Create CitrixReceiverUpdate.sh

Create CitrixReceiverUpdate.sh

#!/bin/bash

#####################################################################################################
#
# ABOUT THIS PROGRAM
#
# NAME
#	CitrixReceiverUpdate.sh -- Installs or updates Citrix Receiver
#
# SYNOPSIS
#	sudo CitrixReceiverUpdate.sh
#
# EXIT CODES
#   0 - Citrix Receiver is current
#   1 - Citrix Receiver installed successfully
#   2 - Citrix Receiver NOT installed
#   3 - Citrix Receiver update unsuccessful
#   4 - Citrix Receiver is running or was attempted to be installed manually and user deferred install
#   5 - Not an Intel-based Mac
#
####################################################################################################
#
# HISTORY
#
#   Version: 1.0
#
#   - v.1.0 Luie Lugo, 09.05.2016 : Updates Citrix Receiver
#
####################################################################################################
# Script to download and install Citrix Receiver.

# Setting variables
receiverProcRunning=0

# Echo function
echoFunc () {
    # Date and Time function for the log file
    fDateTime () { echo $(date +"%a %b %d %T"); }

    # Title for beginning of line in log file
    Title="InstallLatestCitrixReceiver:"

    # Header string function
    fHeader () { echo $(fDateTime) $(hostname) $Title; }

    # Check for the log file
    if [ -e "/Library/Logs/CitrixReceiverUpdateScript.log" ]; then
        echo $(fHeader) "$1" >> "/Library/Logs/CitrixReceiverUpdateScript.log"
    else
        cat > "/Library/Logs/CitrixReceiverUpdateScript.log"
        if [ -e "/Library/Logs/CitrixReceiverUpdateScript.log" ]; then
            echo $(fHeader) "$1" >> "/Library/Logs/CitrixReceiverUpdateScript.log"
        else
            echo "Failed to create log file, writing to JAMF log"
            echo $(fHeader) "$1" >> "/var/log/jamf.log"
        fi
    fi

    # Echo out
    echo $(fDateTime) ": $1"
}

# Exit function
exitFunc () {
    case $1 in
        0) exitCode="0 - Citrix Receiver is current! Version: $2";;
        1) exitCode="1 - SUCCESS: Citrix Receiver has been updated to version $2";;
        2) exitCode="2 - ERROR: Citrix Receiver NOT installed!";;
        3) exitCode="3 - ERROR: Citrix Receiver update unsuccessful, version remains at  $2!";;
        4) exitCode="4 - ERROR: Citrix Receiver is running or was attempted to be installed manually and user deferred install.";;
        5) exitCode="5 - ERROR: Not an Intel-based Mac.";;
        *) exitCode="$1";;
    esac
    echoFunc "Exit code: $exitCode"
    echoFunc "======================== Script Complete ========================"
    exit $1
}

# Check to see if Citrix Receiver is running
receiverRunningCheck () {
    processNum=$(ps aux | grep "Citrix Receiver" | wc -l)
    if [ $processNum -gt 1 ]
    then
        # Receiver is running, prompt the user to close it or defer the upgrade
        receiverRunning
    fi
}

# If Citrix Receiver is running, prompt the user to close it
receiverRunning () {
    echoFunc "Citrix Receiver appears to be running!"
    hudTitle="Citrix Receiver Update"
    hudDescription="Citrix Receiver needs to be updated. Please save your work and close the application to proceed. You can defer if needed.

If you have any questions, please call the help desk."

    #sudo -u $(ls -l /dev/console | awk '{print $3}')    utility
    jamfHelperPrompt=`/Library/Application\ Support/JAMF/bin/jamfHelper.app/Contents/MacOS/jamfHelper -windowType hud -lockHUD -title "$hudTitle" -description "$hudDescription" -button1 "Proceed" -button2 "Defer" -defaultButton 1`

    case $jamfHelperPrompt in
        0)
            echoFunc "Proceed selected"
            receiverProcRunning=1
            receiverRunningCheck
        ;;
        2)
            echoFunc "Deferment Selected"
            exitFunc 4
        ;;
        *)
            echoFunc "Selection: $?"
            #receiverProcRunning=1
            #receiverRunningCheck
            exitFunc 3 "Unknown"
        ;;
    esac
}

# If Citrix Receiver is running, prompt the user to close it
receiverUpdateMan () {
    echoFunc "Citrix Receiver appears to be running!"
    hudTitle="Citrix Receiver Update"
    hudDescription="Citrix Receiver needs to be updated. You will see a program downloading the installer. You can defer if needed.

If you have any questions, please call the help desk."

    jamfHelperPrompt=`/Library/Application\ Support/JAMF/bin/jamfHelper.app/Contents/MacOS/jamfHelper -windowType hud -lockHUD -title "$hudTitle" -description "$hudDescription" -button1 "Defer" -button2 "Proceed" -defaultButton 1 -timeout 60 -countdown`

    case $jamfHelperPrompt in
        0)
            echoFunc "Deferment Selected"
            exitFunc 4
        ;;
        2)
            echoFunc "Proceed selected"
            receiverRunningCheck
        ;;
        *)
            echoFunc "Selection: $?"
            exitFunc 3 "Unknown"
        ;;
    esac
}

echoFunc ""
echoFunc "======================== Starting Script ========================"

# Are we running on Intel?
if [ '`/usr/bin/uname -p`'="i386" -o '`/usr/bin/uname -p`'="x86_64" ]; then
    ## Get OS version and adjust for use with the URL string
    OSvers_URL=$( sw_vers -productVersion | sed 's/[.]/_/g' )

    ## Set the User Agent string for use with curl
    userAgent="Mozilla/5.0 (Macintosh; Intel Mac OS X ${OSvers_URL}) AppleWebKit/535.6.2 (KHTML, like Gecko) Version/5.2 Safari/535.6.2"

    # Get the latest version of Receiver available from Citrix's Receiver page.
    latestver=``
    while [ -z "$latestver" ]
    do
        latestver=`curl -s -L https://www.citrix.com/downloads/citrix-receiver/mac/receiver-for-mac-latest.html | grep "<h1>Receiver " | awk '{print $2}'`
    done

    echoFunc "Latest Citrix Receiver Version is: $latestver"
    latestvernorm=`echo ${latestver}`
    # Get the version number of the currently-installed Citrix Receiver, if any.
    if [ -e "/Applications/Citrix Receiver.app" ]; then
        currentinstalledapp="Citrix Receiver"
        currentinstalledver=`/usr/bin/defaults read /Applications/Citrix\ Receiver.app/Contents/Info CFBundleShortVersionString`
        echoFunc "Current Receiver installed version is: $currentinstalledver"
        if [ ${latestvernorm} = ${currentinstalledver} ]; then
            exitFunc 0 "${currentinstalledapp} ${currentinstalledver}"
        else
            # Not running the latest version, check if Receiver is running
            receiverRunningCheck
        fi
    else
        currentinstalledapp="None"
        currentinstalledver="N/A"
        exitFunc 2
    fi

    # Build URL and dmg file name
    CRCurrVersNormalized=$( echo $latestver | sed -e 's/[.]//g' )
    echoFunc "CRCurrVersNormalized: $CRCurrVersNormalized"
    url1="https:"
    url2=`curl -s -L https://www.citrix.com/downloads/citrix-receiver/mac/receiver-for-mac-latest.html | grep "ctx-dl-link ie10-download-hide promptDw\"" | awk '{print $8}' | cut -c 6-112`
    url=`echo "${url1}${url2}"`
    echoFunc "Latest version of the URL is: $url"
    dmgfile="Citrix_Rec_${CRCurrVersNormalized}.dmg"

    # Compare the two versions, if they are different or Citrix Receiver is not present then download and install the new version.
    if [ "${currentinstalledver}" != "${latestvernorm}" ]; then
        echoFunc "Current Receiver version: ${currentinstalledapp} ${currentinstalledver}"
        echoFunc "Available Receiver version: ${latestver} => ${CRCurrVersNormalized}"
        echoFunc "Downloading newer version."
        curl -s -o /tmp/${dmgfile} ${url}
        case $? in
            0)
                echoFunc "Checking if the file exists after downloading."
                if [ -e "/tmp/${dmgfile}" ]; then
                    receiverFileSize=$(du -k "/tmp/${dmgfile}" | cut -f 1)
                    echoFunc "Downloaded File Size: $receiverFileSize kb"
                else
                    echoFunc "File NOT downloaded!"
                    exitFunc 3 "${currentinstalledapp} ${currentinstalledver}"
                fi
                echoFunc "Checking if Receiver is running one last time before we install"
                receiverRunningCheck
                echoFunc "Mounting installer disk image."
                hdiutil attach /tmp/${dmgfile} -nobrowse -quiet
                echoFunc "Installing..."
                installer -pkg /Volumes/Citrix\ Receiver/Install\ Citrix\ Receiver.pkg -target / > /dev/null

                sleep 10
                echoFunc "Unmounting installer disk image."
                umount "/Volumes/Citrix Receiver"
                sleep 10
                echoFunc "Deleting disk image."
                rm /tmp/${dmgfile}

                #double check to see if the new version got update
                if [ -e "/Applications/Citrix Receiver.app" ]; then
                    newlyinstalledver=`/usr/bin/defaults read /Applications/Citrix\ Receiver.app/Contents/Info CFBundleShortVersionString`
                    if [ "${latestvernorm}" = "${newlyinstalledver}" ]; then
                        echoFunc "SUCCESS: Citrix Receiver has been updated to version ${newlyinstalledver}, issuing JAMF recon command"
                        jamf recon
                        if [ $receiverProcRunning -eq 1 ];
                        then
                            /Library/Application\ Support/JAMF/bin/jamfHelper.app/Contents/MacOS/jamfHelper -windowType hud -lockHUD -title "Citrix Receiver Updated" -description "Citrix Receiver has been updated to version ${newlyinstalledver}." -button1 "OK" -defaultButton 1
                        fi
                        exitFunc 1 "${newlyinstalledver}"
                    else
                        exitFunc 3 "${currentinstalledapp} ${currentinstalledver}"
                    fi
                else
                    exitFunc 3 "${currentinstalledapp} ${currentinstalledver}"
                fi
            ;;
            *)
                echoFunc "Curl function failed on download! Error: $?. Review error codes here: https://curl.haxx.se/libcurl/c/libcurl-errors.html"
            ;;
        esac
    else
        # If Citrix Receiver is up to date already, just log it and exit.
        exitFunc 0 "${currentinstalledapp} ${currentinstalledver}"
    fi
else
    # This script is for Intel Macs only.
    exitFunc 5
fi

error: pathspec 'Other-Products/CitrixReceiverUpdate.sh' did not match any file(s) known to git


script to look for issues in porting to gles

#!/bin/sh

search() {
find . \( -name "*.h" -o -name "*.c" -o -name "*.cpp" -o -name "*.cxx" \) -exec grep -Hn "$*" {} \;
}

listIssues() {
search SDL_WM_GrabInput
search SDL_WarpMouse
search SDL_GL
search GL_CLAMP
search glColor|grep -v glColor4f
search glPushAttrib
search glPolygonMode
search GL_QUADS |grep glDrawArrays
search SDL_Init
search SDL_OPENGL
search SDL_ShowCursor
search glClearDepth
search SDL_Quit
search gluBuild2DMipmaps
search glTexImage2D
search glBegin
search glX
search glRect
search glNewList
}

listIssues|sort -u

error: pathspec 'src/libprojectM/gles.sh' did not match any file(s) known to git


Update tools.sh

Update tools.sh

#!/usr/bin/env bash
### every exit != 0 fails the script
set -e

echo "Install some common tools for further installation"
apt-get update
apt-get install -y --allow-unauthenticated autotools-dev automake cmake curl make grep sed sudo dpkg fuse git wget zip \
openjdk-8-jre build-essential pkg-config python python-dev python-pip bzip2 git libglib2.0-0 libxext6 libsm6 libxrender1 \
mercurial zlib1g-dev openssh-server locales ca-certificates subversion vim
apt-get clean -y


Update tools.sh

Update tools.sh

#!/usr/bin/env bash
### every exit != 0 fails the script
set -e

echo "Install some common tools for further installation"
apt-get update
apt-get install -y --allow-unauthenticated make grep sed sudo dpkg git wget zip build-essential python python-dev python-pip \
bzip2 mercurial openssh-server locales ca-certificates subversion vim gdebi-core bash-completion apt-utils software-properties-common \
python-software-properties libgtk-3-bin libgtk-3-dev
apt-get clean -y

#!/usr/bin/env bash
### every exit != 0 fails the script
set -e

echo "Install some common tools for further installation"
apt-get update
apt-get install -y --allow-unauthenticated make grep sed sudo dpkg git wget zip build-essential python python-dev python-pip \
bzip2 mercurial openssh-server locales ca-certificates subversion vim gdebi-core bash-completion apt-utils software-properties-common \
python-software-properties
apt-get clean -y


基于pandoc批量转换markdown格式纯文本到.docx格式文件

基于pandoc批量转换markdown格式纯文本到.docx格式文件

#!/bin/bash

IFS=$'\n'
pandoc=$(which pandoc)

if [[ ! -x "${pandoc}" ]];then
  echo "pandoc binary is required to run this script"
  exit 1
fi

input_dir="${1:-/tmp/write-ups}"
input_file="README.md"

if [[ ! -d "${input_dir}" ]];then
  echo "batch convert markdown files to .docx"
  echo "usage: $0 <path to README.md>"
  echo "       $0 /tmp/write-ups"
  exit 1
fi

docs=($(find ${input_dir} -iname "${input_file}"))

for doc in ${docs[@]};do
  echo "converting ${doc}"
  pushd $(dirname "$doc") >/dev/null
  "$pandoc" "$input_file" -o "${input_file}.docx"
  popd > /dev/null
done

error: pathspec 'bash/md2docx.sh' did not match any file(s) known to git


Added small script to compile python 3 pyexiv2 support.

#!/bin/bash
ver="5.00"
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
# get cur dir of this script progName=$(basename -- "$0")
cd $DIR
echo "$progName $ver  written by Claude Pageau"
echo "INFO  : Install Dependencies"
sudo apt-get install -y git
sudo apt-get install -y python-all-dev libexiv2-dev libboost-python-dev
sudo apt-get install -y g++
echo "INFO  : Done Dependencies ..."
echo "INFO  : git clone https://github.com/mcmclx/py3exiv2.git"
if [ -d py3exiv2 ]; then
    echo "WARN  : py3exiv2 Folder Already Exists"
else
    echo "INFO  : git clone https://github.com/mcmclx/py3exiv2.git"
    git clone https://github.com/mcmclx/py3exiv2.git
fi
cd py3exiv2
echo "INFO  : Running pytho3 configure"
python3 configure.py
echo "INFO  : Compile from Source per build.sh"
echo "INFO  : This will Take Some Time ..."
./build.sh
echo "INFO  : Install per build.sh -i"
sudo ./build.sh -i
echo "INFO  : python3 pyexiv2 Support Added.

        To Test open python3 console and
        test import of pyexiv2 per commands below

    python3
    import pyexiv2

ctrl-d to exit python 3

Bye ...
"

error: pathspec 'source/install-py3exiv2.sh' did not match any file(s) known to git


Add install.sh template

Add install.sh template

#!/bin/sh

psql -U postgres -c 'create database <NAME>db' postgres
psql -U postgres --no-align --tuples-only -c "SELECT 1 FROM pg_roles WHERE rolname='api'" | grep -q 1 || psql -U postgres -c 'create role api login PASSWORD NULL' postgres
psql -U postgres -c 'GRANT ALL ON DATABASE <NAME>db TO api' postgres
sem-apply --url postgresql://api@localhost/<NAME>db

error: pathspec 'templates/postgresql/install.sh' did not match any file(s) known to git


etc: adb_run_wrapper added

etc: adb_run_wrapper added

remote: Support for password authentication was removed on August 13, 2021.
remote: Please see https://docs.github.com/en/get-started/getting-started-with-git/about-remote-repositories#cloning-with-https-urls for information on currently recommended modes of authentication.
fatal: Authentication failed for 'https://github.com/j16r/rust.git/'


Added a PostGIS template creation script for Ubuntu (tested on 10.04 Lucid Lynx).

#!/usr/bin/env bash
POSTGIS_SQL_PATH=/usr/share/postgresql/8.4/contrib
createdb -E UTF8 template_postgis # Create the template spatial database.
createlang -d template_postgis plpgsql # Adding PLPGSQL language support.
psql -d postgres -c "UPDATE pg_database SET datistemplate='true' WHERE datname='template_postgis';"
psql -d template_postgis -f $POSTGIS_SQL_PATH/postgis.sql # Loading the PostGIS SQL routines
psql -d template_postgis -f $POSTGIS_SQL_PATH/spatial_ref_sys.sql
psql -d template_postgis -c "GRANT ALL ON geometry_columns TO PUBLIC;" # Enabling users to alter spatial tables.
psql -d template_postgis -c "GRANT ALL ON spatial_ref_sys TO PUBLIC;"

error: pathspec 'molly/scripts/create_template_postgis-ubuntu.sh' did not match any file(s) known to git


Update system_report.sh (#83)

Update system_report.sh (#83)

remote: Support for password authentication was removed on August 13, 2021.
remote: Please see https://docs.github.com/en/get-started/getting-started-with-git/about-remote-repositories#cloning-with-https-urls for information on currently recommended modes of authentication.
fatal: Authentication failed for 'https://github.com/bitrise-io/osx-box-bootstrap.git/'


included storage container create script

#!/bin/bash

test -n "$1" || {
  echo "Usage: $0 <docker-compose.yml>" >&2
  exit 1
}
DC="$1"

dcreate() {
  name=$1
  path=$2
  if docker inspect --format='{{.Name}}' "${name}" > /dev/null 2>&1
  then
    echo " ${name} exists";
  else
    docker create -v "${path}" --name "${name}" systemapic/ubuntu
  fi
}

grep -A1 volumes_from "${DC}" |
  grep -v -- -- |
  grep -v volumes_from |
  grep -v "\s*#" |
  sed 's/.*- *\([^ ]*\).*/\1/' |
  sort -u |
while read C; do
  # postgis backup
  if echo "$C" | grep -q '^postgis_backup'; then
    POSTGIS_BACKUP=$C
    echo "Creating PostGIS backup volume $POSTGIS_BACKUP"
    dcreate $POSTGIS_BACKUP /backup/postgis || exit 1
  # data store
  elif echo "$C" | grep -q '^data_store'; then
    DATASTORE=$C
    echo "Creating Data Store volume $DATASTORE"
    dcreate $DATASTORE /data || exit 1
  # mongo
  elif echo "$C" | grep -q '^mongo_store'; then
    MONGO=$C
    echo "Creating MongoDB volume $MONGO"
    dcreate $MONGO /data/db || exit 1
  # redis stats
  elif echo "$C" | grep -q '^redis_stats'; then
    REDISSTATS=$C
    echo "Creating Redis Stats volume $REDISSTATS"
    dcreate $REDISSTATS /data || exit 1
  # postgis
  elif echo "$C" | grep -q '^post'; then
    POSTGIS=$C
    echo "Creating PostGIS volume $POSTGIS"
    dcreate $POSTGIS /var/lib/postgresql || exit 1
  # redis tokens
  elif echo "$C" | grep -q 'redis_tokens'; then
    REDISTOKENS=$C
    echo "Creating Redis Tokens volume $REDISTOKENS"
    dcreate $REDISTOKENS /data || exit 1
  # redis layers
  elif echo "$C" | grep -q '^redis'; then
    REDISLAYERS=$C
    echo "Creating Redis Layers volume $REDISLAYERS"
    dcreate $REDISLAYERS /data || exit 1
  else
    echo "Don't know how to build referenced data container '$C'" >&2
  fi
done

exit 0

error: pathspec 'dev/dev.systemapic.com/create_storage_containers.sh' did not match any file(s) known to git


Adding testing script to GitHub

Adding testing script to GitHub

#!/bin/bash

source /cvmfs/oasis.opensciencegrid.org/osg/modules/lmod/5.6.2/init/bash
module load xrootd/4.1.1
echo "Loaded XRootD"
source ./setStashCache.sh
bash ./stashcp $1 $2

error: pathspec 'testingScript.sh' did not match any file(s) known to git


use the rundir log

use the rundir log

git-svn-id: 6ee47c8665bf46900d9baa0afb711c5b4a0eafac@34 e2bb083e-7f23-0410-b3a8-8253ac9ef6d8

#!/bin/bash

OUTDIR=`pwd`
LOGCOUNT=0
SEQ=1
DATE=`date +"%y%m%d"`
TIME=`date +"%T %Z(%z)"`
RUNDIRBASE="run-$DATE"
RUNDIR="$OUTDIR/$RUNDIRBASE"
mkdir -p $RUNDIR

if [ "$1" == "-fco" ]; then
	FCO=1
	shift
elif [ "$2" == "-fco" ]; then
	FCO=1
fi

if [ "$1" == "" ]; then
	OUT=$RUNDIR/"tests.log"
else
	OUT=$RUNDIR/$1
fi

head() {
	HTMLBASE=tests-$DATE.html
	HTML=$OUTDIR/$HTMLBASE
	rm -f $OUTDIR/current.html

	#This doesn't work well with servers that don't follow symlinks
	#ln -s $HTML $OUTDIR/current.html

	cat <<DOH >$OUTDIR/current.html

		<title>Redirecting...</title>
		<meta http-equiv="cache-control" content="no-cache">
		<meta http-equiv="refresh" content="0,url=$HTMLBASE">

		You should be redirected to $HTMLBASE

DOH
	cat <<DOH >$HTML

		<title>VDL2 nightly integration tests and build ($DATE $TIME)</title>

VDL2 nightly integration tests and build ($DATE $TIME)

Test results

Compiled packages

Older tests

How to add new tests

DOH
	FIRSTTEST=1
}

html() {
	echo $@ >>$HTML
}

tail() {
	html "

"

	if [ "$BINPACKAGE" != "" ]; then
		FBP=$OUTDIR/$BINPACKAGE
		SIZE=`ls -hs $FBP`
		SIZE=${SIZE/$FBP}
		cat <<DOH >>$HTML

Binary packages

	$BINPACKAGE ($SIZE)

DOH
	fi

	html "
Older tests
"
	html ''
	for OLDER in `ls $OUTDIR/tests-*.html`; do
		O=`basename $OLDER`
		html "$O
"
	done
	cat <<DOH >>$HTML
	How to add new tests

DOH
}

outecho() {
	TYPE=$1
	shift
	echo "<$TYPE>$1|$2|$3|$4|$5|$6|$7|$8|$9|"
}

out() {
	echo $@
	TYPE=$1
	if [ "$TYPE" == "test" ]; then

		NAME=$2
		SEQ=$3
		CMD=$4
		RES=$5
		LOG=$6

		if [ "$FIRSTTEST" == "1" ]; then
			html "
Test results
"
			html ""
			html "Output log from tests"
			html "
"
			FIRSTTEST=0
		else
			if [ "$FLUSH" == "1" ]; then
				html "

"
			fi
		fi

		if [ "$TESTPART" != "" ]; then
			html "
$TESTPART
"
			TESTPART=
		fi

		if [ "$FLUSH" == "1" ]; then
			html "
$NAME:

"
		fi
		if [ ${#SEQ} -gt 2 ]; then
			WIDTH=""
		else
			WIDTH="width=\"20\""
		fi
		if [ "$RES" == "Passed" ]; then
			html ""
			html "$SEQ"
		else
			html ""
			html "$SEQ"
		fi
		html "	"

	elif [ "$TYPE" == "package" ]; then
		BINPACKAGE=$2
	else
		html $@
	fi

}

aexec() {
	echo Executing "$@" >>$OUT
	rm -f $OUTDIR/x73010test.log
	LASTCMD="$@"
	"$@" >$OUTDIR/x73010test.log 2>&1
	EXITCODE=$?
	if [ "$EXITCODE" == "127" ]; then
		echo "Command not found: $@" >$OUTDIR/x73010test.log
	fi
	if [ -f $OUTDIR/x73010test.log ]; then
		cat $OUTDIR/x73010test.log >>$OUT
	fi

}

tlog() {
	TLOG="output_$LOGCOUNT.txt"
	rm -f $RUNDIR/$TLOG
	banner "$LASTCMD" $RUNDIR/$TLOG
	if [ -f $OUTDIR/x73010test.log ]; then
		cat $OUTDIR/x73010test.log >>$RUNDIR/$TLOG 2>>$OUT
	fi
	TLOG="$RUNDIRBASE/$TLOG"
	let "LOGCOUNT=$LOGCOUNT+1"
}

fexec() {
	FLUSH=1
	banner "$TEST (faked)"
	echo "Faking $TEST"
	EXITCODE=0
	LASTCMD=""
	vtest
}

banner() {
	if [ "$2" == "" ]; then
		BOUT=$OUT
	else
		BOUT=$2
	fi
	echo "">>$BOUT
	echo "*****************************************************************************************">>$BOUT
	echo "* $1" >>$BOUT
	echo "*****************************************************************************************">>$BOUT
}

pexec() {
	banner "$TEST (part $SEQ)"
	echo "Executing $TEST (part $SEQ)"
	aexec "$@"
	ptest
	let "SEQ=$SEQ+1"
	FLUSH=0
}

ssexec() {
	SEQSAVE=$SEQ
	SEQ=$1
	shift
	banner "$TEST (part $SEQ)"
	echo "Executing $TEST (part $SEQ)"
	aexec "$@"
	ptest
	SEQ=$SEQSAVE
	FLUSH=0
}

vexec() {
	if [ "$SEQ" == "1" ]; then
		banner "$TEST"
		echo "Executing $TEST"
	else
		banner "$TEST (part $SEQ)"
		echo "Executing $TEST (part $SEQ)"
	fi
	aexec "$@"
	vtest
	SEQ=1
	FLUSH=1
}

ptest() {
	if [ "$EXITCODE" == "0" ]; then
		RES="Passed"
	else
		RES="Failed"
	fi
	tlog
	out test "$TEST" $SEQ "$LASTCMD" $RES $TLOG
	if [ "$EXITONFAILURE" == "true" ]; then
		if [ "$EXITCODE" != "0" ]; then
			exit
		fi
	fi
}

vtest() {
	EC=$?
	if [ "$EXITCODE" == "0" ]; then
		RES="Passed"
	else
		RES="Failed"
	fi
	tlog
	out test "$TEST" $SEQ "$LASTCMD" $RES $TLOG
	if [ "$EXITCODE" != "0" ]; then
		if [ "$EXITONFAILURE" == "true" ]; then
			exit
		fi
	fi
}

if [ "$1" == "" ]; then
	OUT=$OUTDIR/"tests.log"
else
	OUT=$OUTDIR/$1
fi
date > $OUT
FLUSH=1

head
TESTPART="Part I: Build"
EXITONFAILURE=true
if [ "$FCO" != "1" ]; then
	TEST="Checkout CoG"
	pexec rm -rf cog
	vexec svn co https://svn.sourceforge.net/svnroot/cogkit/trunk/current/src/cog

	TEST="Checkout VDL2"
	pexec rm -rf src
	#vexec cvs -d :pserver:anonymous@cvs.cogkit.org:/cvs/cogkit co src/vdsk
	svn co https://svn.ci.uchicago.edu/svn/vdl2
fi

TEST="Directory setup"
pexec mkdir cog/modules/vdsk
vexec cp -r vdl2/trunk/* cog/modules/vdsk

TEST="Compile"
pexec cd cog/modules/vdsk
pexec rm -rf dist
vexec ant -quiet dist

TEST="Package"
pexec cd dist
VDSK=`ls -d vdsk*.*`
pexec cd $VDSK/lib
pexec rm -f castor*.jar *gt2ft*.jar ant.jar
pexec cd ../..
pexec rm -rf vdsk-$DATE
pexec mv $VDSK vdsk-$DATE
vexec tar -pczf $OUTDIR/vdsk-$DATE.tar.gz vdsk-$DATE
out package "vdsk-$DATE.tar.gz"

PATH=$PWD/vdsk-$DATE/bin:$PATH
cd ..
TESTDIR=$PWD/tests
echo "Path: $PATH" >>$OUT
cd $RUNDIR

EXITONFAILURE=false
TESTPART="Part II: Language Tests"

for TEST in `ls $TESTDIR/*.dtm`; do
	BN=`basename $TEST`
	echo $BN
	cp $TESTDIR/$BN .

	TESTNAME=${BN:0:((${#BN}-4))}
	TEST="$TESTNAME"

	ssexec "Compile" vdlc $TESTNAME.dtm
	for ((i=0; $i<9; i=$i+1)); do
		pexec vdlrun $TESTNAME.kml
	done
	vexec vdlrun $TESTNAME.kml
done

#Don't remove me:
tail


Allow nightly.sh to test shell scripts in addition to SwiftScripts

git-svn-id: 6ee47c8665bf46900d9baa0afb711c5b4a0eafac@3953 e2bb083e-7f23-0410-b3a8-8253ac9ef6d8

#!/bin/bash

# USAGE NOTES:

# The script will (optionally) checkout Swift, run several tests in a
# subdirectory called run-DATE, and generate useful HTML output and
# tests.log

# Usage: nightly.sh <options>* <GROUPLIST>

# Run nightly.sh -h for quick help
# When something goes wrong, find and check tests.log or use -v
# Code is checked out into TOPDIR (PWD by default) (PATH is not used)
# Swift is compiled and installed in its source tree
# The run is executed in RUNDIR (TOPDIR/RUNDIRBASE)
# The build test is started in TOPDIR
# Everything for a Swift test is written in its RUNDIR
# The temporary output always goes to OUTPUT (TOPDIR/exec.out)

# Each *.swift test may be accompanied by a
# *.setup.sh, *.check.sh, and/or *.clean.sh script
# and a *.timeout specifier
# The scripts may setup and inspect files in RUNDIR including exec.out,
# which must be accessed in stdout.txt, because the currently running
# tested process writes to exec.out, stdout.txt is a copy.
# The GROUP scripts can read the GROUP variable
# The timeout number in the *.timeout file overrides the default
# timeout

# Tests are GROUPed into directories
# Each GROUP directory has:
#      1) a list of *.swift tests (plus *.sh scripts)
#      2) optionally a sites.template.xml
#      3) optionally a tc.template.data
#      4) optionally a fs.template.data
#      5) optionally a swift.properties
#      6) optionally a title.txt
#      7) preferably a README.txt
#      7) optionally a *.timeout
# template files are lightly processed by sed before use
# Missing files will be pulled from swift/etc

# The GROUPLIST is obtained from an external script in the
# groups/ subdirectory by the name of GROUPLISTFILE.
# The GROUPLISTFILE:
#      1) sets the array
#      2) checks any variables needed by make_sites_sed()

# OUTPUT is the stdout of the current test
# stdout.txt retains stdout from the previous test (for *.clean.sh)
# output_*.txt is the HTML-linked permanent output from a test

# All timeouts in this script are in seconds

# PID TREE:
# Background processes are used so that hung Swift jobs can be killed
# These are the background processes (PIDs are tracked)
# Note that PID management has not yet been perfected.  Check ps
# in error cases, especially if you kill nightly.sh .
#
# nightly.sh
# +-monitor()
#   +-sleep
# +-process_exec()
#   +-bin/swift
#     +-java

# FAILURE CASES
# Some cases are designed to cause Swift to crash.  These
# SwiftScripts contain the token THIS-SCRIPT-SHOULD-FAIL somewhere.
# The response of nightly.sh to the exit code of these Swift
# executions is reversed.

# SCHEDULERS
# Environment must contain PROJECT, QUEUE, and WORK
# These variables will be incorporated into the sites.xml
#   via make_sites_sed() -> group_sites_xml()
# Note that some schedulers restrict your choice of RUNDIR

# NAMING
# Site-specific test groups are in providers/ .
# These are named:
# providers/<provider description>/
# or:
# providers/<provider description>/<site>
# E.g., providers/local-pbs/PADS

# WARNINGS
# nightly.sh uses shopt

shopt -s nullglob

printhelp() {
  echo "nightly.sh <options> <output>"
  echo ""
  echo "usage:"
  printf "\t -a         Do not run ant dist             \n"
  printf "\t -c         Do not remove dist (clean)      \n"
  printf "\t -g         Do not run grid tests           \n"
  printf "\t -h         This message                    \n"
  printf "\t -k <N>     Skip first N tests              \n"
  printf "\t -n <N>     Run N tests and quit            \n"
  printf "\t -p         Do not build the package        \n"
  printf "\t -s         Do not do a fresh svn checkout  \n"
  printf "\t -x         Do not continue after a failure \n"
  printf "\t -v         Verbose (set -x, HTML comments) \n"
  printf "\t -o output  Location for output (TOPDIR)    \n"
  printf "\t <FILE>     GROUPLIST definition file       \n"
}

# Defaults:
DEFAULT_TIMEOUT=30 # seconds
RUN_ANT=1
CLEAN=1
SKIP_TESTS=0
NUMBER_OF_TESTS=1000000 # Run all tests by default
BUILD_PACKAGE=1
GRID_TESTS=1
SKIP_CHECKOUT=0
ALWAYS_EXITONFAILURE=0
VERBOSE=0
# The directory in which to start:
TOPDIR=$PWD

while [ $# -gt 0 ]; do
  case $1 in
    -a)
      RUN_ANT=0
      shift;;
    -c)
      CLEAN=0
      shift;;
    -g)
      GRID_TESTS=0
      shift;;
    -h)
      printhelp
      exit 0;;
    -k)
      SKIP_TESTS=$2
      shift 2;;
    -n)
      NUMBER_OF_TESTS=$2
      shift 2;;
    -o)
      TOPDIR=$2
      shift 2;;
    -p)
      BUILD_PACKAGE=0
      shift;;
    -s)
      SKIP_CHECKOUT=1
      shift;;
    -x)
      ALWAYS_EXITONFAILURE=1
      shift;;
    -v)
      VERBOSE=1
      shift;;
    *)
      GROUPLISTFILE=$1
      shift;;
  esac
done

if (( VERBOSE )); then
  set -x
  HTML_COMMENTS=1
fi

# Iterations per test (may want to run each test multiple times?)
ITERS_LOCAL=1

LOGCOUNT=0
SEQ=1
DATE=$( date +"%Y-%m-%d" )
TIME=$( date +"%T" )

RUNDIRBASE="run-$DATE"
RUNDIR=$TOPDIR/$RUNDIRBASE
LOGBASE=$RUNDIRBASE/tests.log
LOG=$TOPDIR/$LOGBASE
OUTPUT=$RUNDIR/exec.out

HTMLPATH=$RUNDIRBASE/tests-$DATE.html
HTML=$TOPDIR/$HTMLPATH

BRANCH=trunk
#BRANCH="branches/tests-01"

SCRIPTDIR=$( cd $( dirname $0 ) ; /bin/pwd )

TESTCOUNT=0

echo "RUNNING_IN:  $RUNDIR"
echo "HTML_OUTPUT: $HTML"

TESTDIR=$TOPDIR/cog/modules/swift/tests

# Ensure all given variables are set
checkvars() {
  while (( ${#*} ))
   do
   VAR=$1
   V=$( eval "echo \${${VAR}+1}" )
   [[ $V == 1 ]] || crash "Not set: $VAR"
   shift
  done
  return 0
}

checkfail() {
  ERR=$?
  shift
  MSG=${*}
  if [[ $ERR != 0 ]]; then
    echo "FAILED($ERR): $MSG"
    exit $ERR
  fi
}

crash() {
  MSG=$1
  echo $MSG
  exit 1
}

verbose() {
  MSG="${*}"
  (( $VERBOSE )) && echo $MSG
  return 0
}

header() {
  CURRENT=$SCRIPTDIR/html/current.html
  sed "s@_HTMLBASE_@$HTMLPATH@" < $CURRENT > $TOPDIR/current.html

  HEADER=$SCRIPTDIR/html/header.html
  HOST=$( hostname )
  SEDCMD="s/_DATE_/$DATE/;s/_TIME_/$TIME/;s/_HOST_/$HOST"/
  sed $SEDCMD < $HEADER > $HTML
  FIRSTTEST=1
}

html() {
  printf "$@\n" >>$HTML
  [ $? != 0 ] && echo "html(): write error!" && exit 1
}

html_h1() {
  TEXT=$1
  html "
$TEXT
"
}

html_a_name() {
  NAME=$1
  html ""
}

html_a_href() {
  HREF=$1
  TEXT=$2
  html "$TEXT"
}

html_table() {
  OPTS=""
  while [[ ${#*} > 0 ]]
  do
    case $1 in
      cellpadding)
        OPTS+="cellpadding=\"$2\" "
        shift 2;;
      border)
        OPTS+="border=\"$2\" "
        shift 2;;
      esac
  done
  html "
"
}

html_~table() {
  html "
"
}

html_tr() {
  CLASS=$1
  if [ -n $CLASS ]; then
    html "
"
  else
    html "
"
  fi
}

html_~tr() {
  html "
"
}

html_th() {
  COLSPAN=$1
  if [ -n $COLSPAN ]; then
    html ""
  else
    html ""
  fi
}

html_~th() {
  html "	"
}

html_td() {
  OPTS=""
  while [[ ${#*} > 0 ]]
  do
    case $1 in
      align)
        OPTS+="align=\"$2\" "
        shift 2;;
      class)
        OPTS+="class=\"$2\" "
        shift 2;;
      title)
        OPTS+="title=\"$2\" "
        shift 2;;
      width)
        OPTS+="width=\"$2\" "
        shift 2;;
    esac
  done
  html ""
}

html_~td() {
  html "	"
}

html_~body() {
  html ""
}

html_~html() {
  html ""
}

html_comment() {
  COMMENT=$1
  (( HTML_COMMENTS == 1 )) && html ""
}

footer() {
  html "

"
  html_comment "End of tests"

  MONTHS=("" "Jan" "Feb" "Mar" "Apr" "May" "Jun" \
    "Jul" "Aug" "Sep" "Oct" "Nov" "Dec")

  if [ "$BINPACKAGE" != "" ]; then
    FBP=$RUNDIR/$BINPACKAGE
    SIZE=`ls -hs $FBP`
    SIZE=${SIZE/$FBP}
    cat <<DOH >>$HTML

Binary packages

	$BINPACKAGE ($SIZE)

DOH
  fi

  LASTYR="00"
  LASTMO="00"
  html "
Older tests
"
  html ''
  html "

"
  for OLDER in `ls $OUTDIR/tests-*.html|sort`; do
    O=`basename $OLDER`
    YR=${O:6:2}
    MO=${O:8:2}
    DY=${O:10:2}
    if echo "$DY$MO$YR"|egrep -v "[0-9]{6}"; then
      YR=${O#tests-}
      YR=${YR%.html}
      MO=0
      DY=$YR
    else
      YR="20$YR"
    fi
    if [ $LASTYR != $YR ]; then
      html "

"
      html "
$YR
"
      LASTYR=$YR
    fi
    if [ $LASTMO != $MO ]; then
      html "

"
      html "
${MONTHS[$MO]}
"
      html "

"
      LASTMO=$MO
    fi
    SUCCESS=`grep 'class="success"' $OLDER|wc -l`
    FAILURE=`grep 'class="failure"' $OLDER|wc -l`
    if [ "$SUCCESS$FAILURE" == "00" ]; then
      COLOR="#e0e0e0"
    else
      COLOR=`perl -e "printf \"#%02x%02x%02x\", $FAILURE/($SUCCESS+$FAILURE)*220+35, $SUCCESS/($SUCCESS+$FAILURE)*220+35, 40;"`
    fi
    html "$DY	"
  done
  html "

"
  html_~body
  html_~html
}

printlist() {
  while [[ $1 != "" ]]; do
    echo $1
    shift
  done
}

outecho() {
  TYPE=$1
  shift
  echo "<$TYPE>$1|$2|$3|$4|$5|$6|$7|$8|$9|"
}

# Create HTML output
out() {
        # echo $@
  TYPE=$1
  if [ "$TYPE" == "test" ]; then

    LABEL="$2"  # Text on link to output
    CMD=$3    # Command issued (td title)
    RESULT=$4 # Passed or Failed

    # WIDTH=$( width "$LABEL" )
    if [ "$RESULT" == "Passed" ]; then
      # html ""
      html_td class "success" width 25 title "$CMD"
      html_a_href $TEST_LOG "$LABEL"
    else
      echo "FAILED"
      cat $RUNDIR/$TEST_LOG < /dev/null
      # html ""
      html_td class "failure" width 25 title "$CMD"
      html_a_href $TEST_LOG $LABEL
    fi
    html_~td

  elif [ "$TYPE" == "package" ]; then
    BINPACKAGE=$2
  else
    html $@
  fi
}

start_test_results() {
  html_h1 "Test results"
  html_a_name "tests"
  html_a_href "tests.log" "Output log from tests"
  html_table border 0 cellpadding 1
}

start_part() {
  PART=$1
  echo
  echo $PART
  echo
  html_tr part
  html_th 3
  html "$PART"
  html_~th
  html_~tr
}

start_row() {
  html_tr testline
  html_td align right width 50
  html "$TESTCOUNT"
  html " "
  html_~td
  html_td align right
  html " "
  if [[ -n $TESTLINK ]]; then
    html_a_href $TESTLINK $TESTNAME
  else
    html $TESTNAME
  fi
  html " "
  html_~td
  html_td
  html_table
  html_tr
  SEQ=1
}

end_row() {
  html_~tr
  html_~table
  html_~td
  html_~tr
}

# HTML width of label (for alignment)
width() {
  LABEL="$1"
  if [ ${#LABEL} -gt 2 ]; then
    WIDTH=""
  else
    WIDTH="width=\"20\""
  fi
  echo $WIDTH
}

# TEST_LOG = test log
test_log() {
  TEST_LOG="output_$LOGCOUNT.txt"
  banner "$LASTCMD" $RUNDIR/$TEST_LOG
  if [ -f $OUTPUT ]; then
    cp $OUTPUT $RUNDIR/$TEST_LOG 2>>$LOG
    cp $OUTPUT $RUNDIR/stdout.txt
  fi
  let "LOGCOUNT=$LOGCOUNT+1"
}

stars() {
  for i in {1..90}
  do
    printf "*"
  done
  echo
}

banner() {
  MSG=$1
  FILE=$2

  if [ "$FILE" == "" ]; then
    BANNER_OUTPUT=$LOG
  else
    BANNER_OUTPUT=$2
  fi
  {
    echo ""
          # stars
    echo "* $MSG"
	  # stars
  } >> $BANNER_OUTPUT
}

# Check for early bailout condition
check_bailout() {
  if [ "$EXITONFAILURE" == "true" ]; then
    if [ "$EXITCODE" != "0" ]; then
      exit $EXITCODE
    fi
  fi
}

# Translate (global) test exit code into result (Passed/Failed) string
result() {
  if [ "$EXITCODE" == "0" ]; then
    echo "Passed"
  else
    echo "Failed"
  fi
}

# Execute process in the background
process_exec() {
  printf "\nExecuting: $@" >>$LOG
  rm -f $OUTPUT

  "$@" > $OUTPUT 2>&1 &
  EXEC_PID=$!

  trap "process_exec_trap $EXEC_PID" SIGTERM

  wait $EXEC_PID
  EXITCODE=$?
  if [ "$EXITCODE" == "127" ]; then
    echo "Command not found: $@" > $OUTPUT
  fi
  if [ -f $OUTPUT ]; then
    cat $OUTPUT >> $LOG
  fi
  (( $TEST_SHOULD_FAIL )) && EXITCODE=$(( ! $EXITCODE ))
  return $EXITCODE
}

# Ensure we kill the tested process and any subordinate java processes
# in case of monitor() timeout
# Rationale: Killing bin/swift does not kill the Swift java process
process_exec_trap() {
  EXEC_PID=$1
  echo "process_exec_trap()"
  kill -KILL $EXEC_PID
  killall_swift
}

# Kill all subordinate swift/java processes
killall_swift() {
  echo "killing all swifts..."
  set -x
  echo $$
  ps -f
  kill_this $( ps -f | grep $$'.*'java  | grep -v grep )
  set +x
}

# Kill a process given its line output from "ps -f"
kill_this() {
  [ -n $2 ] && /bin/kill -KILL $2
}

# Execute as part of test set
# Equivalent to monitored_exec() (but w/o monitoring)
test_exec() {
  banner "$TEST (part $SEQ)"
  echo "Executing $TEST (part $SEQ)"
  pwd
  printf "\nExecuting: $@" >>$LOG

  rm -f $OUTPUT
  "$@" > $OUTPUT 2>&1
  EXITCODE=$?

  if [ "$EXITCODE" == "127" ]; then
    echo "Command not found: $@" > $OUTPUT
  fi

  if [ -f $OUTPUT ]; then
    cat $OUTPUT >> $LOG
  fi

  RESULT=$( result )
  test_log
  out test $SEQ "$LASTCMD" $RESULT $TEST_LOG

  check_bailout

  let "SEQ=$SEQ+1"
  return $EXITCODE
}

# Background process monitoring function
# To be executed in the background as well
# If the monitored process times out, monitor() kills it and writes
# a message to OUTPUT
# If monitor() kills the process, it returns 0
# Otherwise, return non-zero (as would result from killing
# this function)
monitor() {
  (( $VERBOSE )) && set -x

  PID=$1
  TIMEOUT=$2 # seconds
  OUTPUT=$3

  V=$TESTCOUNT

  # Use background so kill/trap is immediate
  sleep $TIMEOUT > /dev/null 2>&1 &
  SLEEP_PID=$!
  trap "monitor_trap $SLEEP_PID $V > /dev/null 2>&1" SIGTERM
  wait $SLEEP_PID
  [ $? != 0 ] && verbose "monitor($V) cancelled" && return 0

  if ps | grep $PID
  then
    echo "monitor($V): killing test process..."
    touch killed_test
    /bin/kill -TERM $PID
    KILLCODE=$?
    if [ $KILLCODE == 0 ]; then
      echo "monitor($V): killed process_exec (TERM)"
    fi
  fi

  sleep 1
  MSG="nightly.sh: monitor($V): killed: exceeded $TIMEOUT seconds"
  echo "$MSG" >> $OUTPUT
  # trap
  return 1
}

monitor_trap() {
  SLEEP_PID=$1
  V=$2
  verbose "monitor_trap(): kill sleep"
  /bin/kill -KILL $SLEEP_PID
}

# Execute given command line in background with monitor
# Otherwise equivalent to test_exec()
# usage: monitored_exec <TIMEOUT> <command> <args>*
monitored_exec()
{
  TIMEOUT=$1
  shift

  banner "$TEST (part $SEQ)"
  echo "Executing $TEST (part $SEQ)"

  START=$( date +%s )

  process_exec "$@" &
  PROCESS_PID=$!

  monitor $PROCESS_PID $TIMEOUT $OUTPUT &
  MONITOR_PID=$!

  wait $PROCESS_PID
  EXITCODE=$?

  STOP=$( date +%s )

  # If the test was killed, monitor() may have work to do
  rm killed_test > /dev/null 2>&1 && sleep 5
  verbose "Killing monitor..."
  /bin/kill -TERM $MONITOR_PID

  echo "TOOK: $(( STOP-START ))"

  RESULT=$( result )
  test_log
  LASTCMD="$@"
  out test $SEQ "$LASTCMD" $RESULT $TEST_LOG

  check_bailout

  let "SEQ=$SEQ+1"
}

# Execute helper script (setup, check, or clean)
script_exec() {
  SCRIPT=$1
  SYMBOL="$2"

  process_exec $SCRIPT
  RESULT=$( result )

  test_log
  out test "$SYMBOL" "$LASTCMD" $RESULT

  check_bailout
}

# Execute Swift test case w/ setup, check, clean
swift_test_case() {
  SWIFTSCRIPT=$1
  SETUPSCRIPT=${SWIFTSCRIPT%.swift}.setup.sh
  CHECKSCRIPT=${SWIFTSCRIPT%.swift}.check.sh
  CLEANSCRIPT=${SWIFTSCRIPT%.swift}.clean.sh
  TIMEOUTFILE=${SWIFTSCRIPT%.swift}.timeout

  TEST_SHOULD_FAIL=0
  if [ -x $GROUP/$SETUPSCRIPT ]; then
    script_exec $GROUP/$SETUPSCRIPT "S"
  fi

  CDM=
  [ -r fs.data ] && CDM="-cdm.file fs.data"

  (( TESTCOUNT++ ))

  TIMEOUT=$( gettimeout $GROUP/$TIMEOUTFILE )

  grep THIS-SCRIPT-SHOULD-FAIL $SWIFTSCRIPT > /dev/null
  TEST_SHOULD_FAIL=$(( ! $?  ))

  monitored_exec $TIMEOUT swift                         \
                       -wrapperlog.always.transfer true \
                       -config swift.properties         \
                       -sites.file sites.xml            \
                       -tc.file tc.data                 \
                       $CDM $SWIFTSCRIPT

  TEST_SHOULD_FAIL=0
  if [ -x $GROUP/$CHECKSCRIPT ]; then
    script_exec $GROUP/$CHECKSCRIPT "√"
  fi
  if [ -x $GROUP/$CLEANSCRIPT ]; then
    script_exec $GROUP/$CLEANSCRIPT "C"
  fi
}

# Execute shell test case w/ setup, check, clean
script_test_case() {
  SHELLSCRIPT=$1
  SETUPSCRIPT=${SHELLSCRIPT%.test.sh}.setup.sh
  CHECKSCRIPT=${SHELLSCRIPT%.test.sh}.check.sh
  CLEANSCRIPT=${SHELLSCRIPT%.test.sh}.clean.sh
  TIMEOUTFILE=${SHELLSCRIPT%.test.sh}.timeout

  TEST_SHOULD_FAIL=0
  if [ -x $GROUP/$SETUPSCRIPT ]; then
    script_exec $GROUP/$SETUPSCRIPT "S"
  fi

  (( TESTCOUNT++ ))

  # Not using background for script tests yet
  # TIMEOUT=$( gettimeout $GROUP/$TIMEOUTFILE )

  if [ -x $GROUP/$SETUPSCRIPT ]; then
    script_exec $GROUP/$SETUPSCRIPT "S"
  fi

  if [ -x $GROUP/$SHELLSCRIPT ]; then
    script_exec $SHELLSCRIPT "X"
  fi

  if [ -x $GROUP/$CHECKSCRIPT ]; then
    script_exec $GROUP/$CHECKSCRIPT "√"
  fi
  if [ -x $GROUP/$CLEANSCRIPT ]; then
    script_exec $GROUP/$CLEANSCRIPT "C"
  fi
}

# All timeouts in this script are in seconds
gettimeout() {
  FILE=$1

  if [ -f $FILE ]; then
    cat $FILE
  else
    echo $DEFAULT_TIMEOUT
  fi
  return 0
}

ssexec() {
  SEQSAVE=$SEQ
  SEQ=$1
  shift
  banner "$TEST (part $SEQ)"
  echo "Executing $TEST (part $SEQ)"
  aexec "$@"
  ptest
  SEQ=$SEQSAVE
}

# Fake exec
fexec() {
  banner "$TEST (faked)"
  echo "Faking $TEST"
  EXITCODE=0
  LASTCMD=""
  # ptest
}

build_package() {
  TEST="Package"
  test_exec cd $SWIFT_HOME/lib
  test_exec rm -f castor*.jar *gt2ft*.jar ant.jar
  test_exec cd $TOPDIR
  test_exec tar -pczf $RUNDIR/swift-$DATE.tar.gz $SWIFT_HOME
  out package "swift-$DATE.tar.gz"
}

# Generate the sites.sed file
make_sites_sed() {
  {
    echo "s@_WORK_@$WORK@"
    echo "s@_HOST_@$GLOBUS_HOSTNAME@"
    echo "s@_PROJECT_@$PROJECT@"
    echo "s@_QUEUE_@$QUEUE@"
  } > $RUNDIR/sites.sed
  return 0
}

# Setup coasters variables
if which ifconfig > /dev/null 2>&1; then
  IFCONFIG=ifconfig
else
  IFCONFIG=/sbin/ifconfig
fi
$IFCONFIG > /dev/null 2>&1 || crash "Cannot run ifconfig!"
GLOBUS_HOSTNAME=$( $IFCONFIG | grep inet | head -1 | cut -d ':' -f 2 | \
                   awk '{print $1}' )
[ $? != 0 ] && crash "Could not obtain GLOBUS_HOSTNAME!"

# Generate sites.xml
group_sites_xml() {
  TEMPLATE=$GROUP/sites.template.xml
  if [ -f $TEMPLATE ]; then
    sed -f $RUNDIR/sites.sed < $TEMPLATE > sites.xml
    [ $? != 0 ] && crash "Could not create sites.xml!"
    echo "Using: $GROUP/sites.template.xml"
  else
    sed "s@_WORK_@$PWD/work@" < $TESTDIR/sites/localhost.xml > sites.xml
    [ $? != 0 ] && crash "Could not create sites.xml!"
    echo "Using: $TESTDIR/sites/localhost.xml"
  fi
}

# Generate tc.data
group_tc_data() {
  if [ -f $GROUP/tc.template.data ]; then
    sed "s@_DIR_@$GROUP@" < $GROUP/tc.template.data > tc.data
    [ $? != 0 ] && crash "Could not create tc.data!"
    echo "Using: $GROUP/tc.template.data"
  else
    cp -v $SWIFT_HOME/etc/tc.data .
    [ $? != 0 ] && crash "Could not copy tc.data!"
  fi
}

# Generate the CDM file, fs.data
group_fs_data() {
  if [ -f $GROUP/fs.template.data ]; then
    sed "s@_PWD_@$PWD@" < $GROUP/fs.template.data > fs.data
    [ $? != 0 ] && crash "Could not create fs.data!"
    echo "Using: $GROUP/fs.template.data"
  else
    rm -f fs.data
  fi
}

# Generate swift.properties
group_swift_properties() {
  if [ -f $GROUP/swift.properties ]; then
    cp -v $GROUP/swift.properties .
    [ $? != 0 ] && crash "Could not copy swift.properties!"
  else
    cp -v $SWIFT_HOME/etc/swift.properties .
    [ $? != 0 ] && crash "Could not copy swift.properties!"
  fi
}

# Obtain the group title
group_title() {
  if [ -r $GROUP/title.txt ]; then
    cat $GROUP/title.txt
  else
    echo "untitled"
  fi
}

# Execute all tests in current GROUP
test_group() {

  group_sites_xml
  group_tc_data
  group_fs_data
  group_swift_properties

  SWIFTS=$( echo $GROUP/*.swift )
  checkfail "Could not list: $GROUP"

  for TEST in $SWIFTS; do

    (( SKIP_COUNTER++ < SKIP_TESTS )) && continue

    TESTNAME=$( basename $TEST )
    cp -v $GROUP/$TESTNAME .
    TESTLINK=$TESTNAME

    start_row
    for (( i=0; $i<$ITERS_LOCAL; i=$i+1 )); do
      swift_test_case $TESTNAME
      (( $TESTCOUNT >= $NUMBER_OF_TESTS )) && return
    done
    end_row
  done

  SCRIPTS=$( echo $GROUP/*.test.sh )
  checkfail "Could not list: $GROUP"
  for TEST in $SCRIPTS; do

    (( SKIP_COUNTER++ < SKIP_TESTS )) && continue

    TESTNAME=$( basename $TEST )
    cp -v $GROUP/$TESTNAME .
    TESTLINK=$TESTNAME

    start_row
    for ((i=0; $i<$ITERS_LOCAL; i=$i+1)); do
      script_test_case $TESTNAME
      (( $TESTCOUNT >= $NUMBER_OF_TESTS )) && return
    done
    end_row
  done
}

if [[ $WORK == "" ]]
then
  WORK=$TOPDIR/work
fi

checkvars GROUPLISTFILE
echo "GROUPLISTFILE: $GROUPLISTFILE"
source $GROUPLISTFILE || exit 1

cd $TOPDIR
mkdir -p $RUNDIR
[ $? != 0 ] && echo "Could not mkdir: $RUNDIR" && exit 1

date > $LOG

make_sites_sed

header
start_test_results
cd $TOPDIR

start_part "Prolog: Build"

TESTLINK=
EXITONFAILURE=true
if [ "$SKIP_CHECKOUT" != "1" ]; then
  TESTNAME="Checkout CoG"
  start_row
  test_exec rm -rf cog
  COG="https://cogkit.svn.sourceforge.net/svnroot/cogkit/trunk/current/src/cog"
  test_exec svn co $COG
  end_row

  TESTNAME="Checkout Swift"
  start_row
  test_exec cd cog/modules
  test_exec rm -rf swift
  test_exec svn co https://svn.ci.uchicago.edu/svn/vdl2/$BRANCH swift
  end_row
fi

TESTNAME="Compile"
start_row

test_exec cd $TOPDIR/cog/modules/swift
if (( $CLEAN )); then
  test_exec rm -rf dist
fi
if (( $RUN_ANT )); then
  test_exec ant -quiet dist
fi
SWIFT_HOME=$TOPDIR/cog/modules/swift/dist/swift-svn

if [ $BUILD_PACKAGE = "1" ]; then
  build_package
fi

PATH=$SWIFT_HOME/bin:$PATH
cd $TOPDIR
which swift
cd $RUNDIR

end_row

if [ $ALWAYS_EXITONFAILURE != "1" ]; then
  EXITONFAILURE=false
fi

SKIP_COUNTER=0

GROUPCOUNT=1
for G in ${GROUPLIST[@]}; do
  export GROUP=$G
  TITLE=$( group_title )
  start_part "Part $GROUPCOUNT: $TITLE"
  test_group
  (( GROUPCOUNT++ ))
  (( $TESTCOUNT >= $NUMBER_OF_TESTS )) && break
done

if [ $GRID_TESTS == "0" ]; then
  exit 0
fi

TESTPART="Appendix G: Grid Tests"

for TEST in `ls $TESTDIR/*.dtm $TESTDIR/*.swift`; do
  BN=`basename $TEST`
  echo $BN
  cp $TESTDIR/$BN .

  TESTNAME=${BN%.dtm}
  TESTNAME=${TESTNAME%.swift}
  TEST="$TESTNAME"

  ssexec "Compile" vdlc $BN
  for ((i=0; $i<9; i=$i+1)); do
    test_exec swift -sites.file ~/.vdl2/sites-grid.xml $TESTNAME.kml
  done
  test_exec swift -sites.file ~/.vdl2/sites-grid.xml $TESTNAME.kml
done

footer

exit 0

# Local Variables:
# sh-basic-offset: 2
# End:

#!/bin/bash

# USAGE NOTES:

# The script will (optionally) checkout Swift, run several tests in a
# subdirectory called run-DATE, and generate useful HTML output and
# tests.log

# Usage: nightly.sh <options>* <GROUPLIST>

# Run nightly.sh -h for quick help
# When something goes wrong, find and check tests.log or use -v
# Code is checked out into TOPDIR (PWD by default) (PATH is not used)
# Swift is compiled and installed in its source tree
# The run is executed in RUNDIR (TOPDIR/RUNDIRBASE)
# The build test is started in TOPDIR
# Everything for a Swift test is written in its RUNDIR
# The temporary output always goes to OUTPUT (TOPDIR/exec.out)

# Each *.swift test may be accompanied by a
# *.setup.sh, *.check.sh, and/or *.clean.sh script
# and a *.timeout specifier
# The scripts may setup and inspect files in RUNDIR including exec.out,
# which must be accessed in stdout.txt, because the currently running
# tested process writes to exec.out, stdout.txt is a copy.
# The GROUP scripts can read the GROUP variable
# The timeout number in the *.timeout file overrides the default
# timeout

# Tests are GROUPed into directories
# Each GROUP directory has:
#      1) a list of *.swift tests (plus *.sh scripts)
#      2) optionally a sites.template.xml
#      3) optionally a tc.template.data
#      4) optionally a fs.template.data
#      5) optionally a swift.properties
#      6) optionally a title.txt
#      7) preferably a README.txt
#      7) optionally a *.timeout
# template files are lightly processed by sed before use
# Missing files will be pulled from swift/etc

# The GROUPLIST is obtained from an external script in the
# groups/ subdirectory by the name of GROUPLISTFILE.
# The GROUPLISTFILE:
#      1) sets the array
#      2) checks any variables needed by make_sites_sed()

# OUTPUT is the stdout of the current test
# stdout.txt retains stdout from the previous test (for *.clean.sh)
# output_*.txt is the HTML-linked permanent output from a test

# All timeouts in this script are in seconds

# PID TREE:
# Background processes are used so that hung Swift jobs can be killed
# These are the background processes (PIDs are tracked)
# Note that PID management has not yet been perfected.  Check ps
# in error cases, especially if you kill nightly.sh .
#
# nightly.sh
# +-monitor()
#   +-sleep
# +-process_exec()
#   +-bin/swift
#     +-java

# FAILURE CASES
# Some cases are designed to cause Swift to crash.  These
# SwiftScripts contain the token THIS-SCRIPT-SHOULD-FAIL somewhere.
# The response of nightly.sh to the exit code of these Swift
# executions is reversed.

# SCHEDULERS
# Environment must contain PROJECT, QUEUE, and WORK
# These variables will be incorporated into the sites.xml
#   via make_sites_sed() -> group_sites_xml()
# Note that some schedulers restrict your choice of RUNDIR

# NAMING
# Site-specific test groups are in providers/ .
# These are named:
# providers/<provider description>/
# or:
# providers/<provider description>/<site>
# E.g., providers/local-pbs/PADS

printhelp() {
  echo "nightly.sh <options> <output>"
  echo ""
  echo "usage:"
  printf "\t -a         Do not run ant dist             \n"
  printf "\t -c         Do not remove dist (clean)      \n"
  printf "\t -g         Do not run grid tests           \n"
  printf "\t -h         This message                    \n"
  printf "\t -k <N>     Skip first N tests              \n"
  printf "\t -n <N>     Run N tests and quit            \n"
  printf "\t -p         Do not build the package        \n"
  printf "\t -s         Do not do a fresh svn checkout  \n"
  printf "\t -x         Do not continue after a failure \n"
  printf "\t -v         Verbose (set -x, HTML comments) \n"
  printf "\t -o output  Location for output (TOPDIR)    \n"
  printf "\t <FILE>     GROUPLIST definition file       \n"
}

# Defaults:
DEFAULT_TIMEOUT=30 # seconds
RUN_ANT=1
CLEAN=1
SKIP_TESTS=0
NUMBER_OF_TESTS=1000000 # Run all tests by default
BUILD_PACKAGE=1
GRID_TESTS=1
SKIP_CHECKOUT=0
ALWAYS_EXITONFAILURE=0
VERBOSE=0
# The directory in which to start:
TOPDIR=$PWD

while [ $# -gt 0 ]; do
  case $1 in
    -a)
      RUN_ANT=0
      shift;;
    -c)
      CLEAN=0
      shift;;
    -g)
      GRID_TESTS=0
      shift;;
    -h)
      printhelp
      exit 0;;
    -k)
      SKIP_TESTS=$2
      shift 2;;
    -n)
      NUMBER_OF_TESTS=$2
      shift 2;;
    -o)
      TOPDIR=$2
      shift 2;;
    -p)
      BUILD_PACKAGE=0
      shift;;
    -s)
      SKIP_CHECKOUT=1
      shift;;
    -x)
      ALWAYS_EXITONFAILURE=1
      shift;;
    -v)
      VERBOSE=1
      shift;;
    *)
      GROUPLISTFILE=$1
      shift;;
  esac
done

if (( VERBOSE )); then
  set -x
  HTML_COMMENTS=1
fi

# Iterations
ITERS_LOCAL=1

LOGCOUNT=0
SEQ=1
DATE=$( date +"%Y-%m-%d" )
TIME=$( date +"%T" )

RUNDIRBASE="run-$DATE"
RUNDIR=$TOPDIR/$RUNDIRBASE
LOGBASE=$RUNDIRBASE/tests.log
LOG=$TOPDIR/$LOGBASE
OUTPUT=$RUNDIR/exec.out

HTMLPATH=$RUNDIRBASE/tests-$DATE.html
HTML=$TOPDIR/$HTMLPATH

BRANCH=trunk
#BRANCH="branches/tests-01"

SCRIPTDIR=$( cd $( dirname $0 ) ; /bin/pwd )

SWIFTCOUNT=0

echo "RUNNING_IN:  $RUNDIR"
echo "HTML_OUTPUT: $HTML"

TESTDIR=$TOPDIR/cog/modules/swift/tests

# Ensure all given variables are set
checkvars() {
  while (( ${#*} ))
   do
   VAR=$1
   V=$( eval "echo \${${VAR}+1}" )
   [[ $V == 1 ]] || crash "Not set: $VAR"
   shift
  done
  return 0
}

checkfail() {
  ERR=$?
  shift
  MSG=${*}
  if [[ $ERR != 0 ]]; then
    echo "FAILED($ERR): $MSG"
    exit $ERR
  fi
}

crash() {
  MSG=$1
  echo $MSG
  exit 1
}

verbose() {
  MSG="${*}"
  (( $VERBOSE )) && echo $MSG
  return 0
}

header() {
  CURRENT=$SCRIPTDIR/html/current.html
  sed "s@_HTMLBASE_@$HTMLPATH@" < $CURRENT > $TOPDIR/current.html

  HEADER=$SCRIPTDIR/html/header.html
  HOST=$( hostname )
  SEDCMD="s/_DATE_/$DATE/;s/_TIME_/$TIME/;s/_HOST_/$HOST"/
  sed $SEDCMD < $HEADER > $HTML
  FIRSTTEST=1
}

html() {
  printf "$@\n" >>$HTML
  [ $? != 0 ] && echo "html(): write error!" && exit 1
}

html_h1() {
  TEXT=$1
  html "
$TEXT
"
}

html_a_name() {
  NAME=$1
  html ""
}

html_a_href() {
  HREF=$1
  TEXT=$2
  html "$TEXT"
}

html_table() {
  OPTS=""
  while [[ ${#*} > 0 ]]
  do
    case $1 in
      cellpadding)
        OPTS+="cellpadding=\"$2\" "
        shift 2;;
      border)
        OPTS+="border=\"$2\" "
        shift 2;;
      esac
  done
  html "
"
}

html_~table() {
  html "
"
}

html_tr() {
  CLASS=$1
  if [ -n $CLASS ]; then
    html "
"
  else
    html "
"
  fi
}

html_~tr() {
  html "
"
}

html_th() {
  COLSPAN=$1
  if [ -n $COLSPAN ]; then
    html ""
  else
    html ""
  fi
}

html_~th() {
  html "	"
}

html_td() {
  OPTS=""
  while [[ ${#*} > 0 ]]
  do
    case $1 in
      align)
        OPTS+="align=\"$2\" "
        shift 2;;
      class)
        OPTS+="class=\"$2\" "
        shift 2;;
      title)
        OPTS+="title=\"$2\" "
        shift 2;;
      width)
        OPTS+="width=\"$2\" "
        shift 2;;
    esac
  done
  html ""
}

html_~td() {
  html "	"
}

html_~body() {
  html ""
}

html_~html() {
  html ""
}

html_comment() {
  COMMENT=$1
  (( HTML_COMMENTS == 1 )) && html ""
}

footer() {
  html "

"
  html_comment "End of tests"

  MONTHS=("" "Jan" "Feb" "Mar" "Apr" "May" "Jun" \
    "Jul" "Aug" "Sep" "Oct" "Nov" "Dec")

  if [ "$BINPACKAGE" != "" ]; then
    FBP=$RUNDIR/$BINPACKAGE
    SIZE=`ls -hs $FBP`
    SIZE=${SIZE/$FBP}
    cat <<DOH >>$HTML

Binary packages

	$BINPACKAGE ($SIZE)

DOH
  fi

  LASTYR="00"
  LASTMO="00"
  html "
Older tests
"
  html ''
  html "

"
  for OLDER in `ls $OUTDIR/tests-*.html|sort`; do
    O=`basename $OLDER`
    YR=${O:6:2}
    MO=${O:8:2}
    DY=${O:10:2}
    if echo "$DY$MO$YR"|egrep -v "[0-9]{6}"; then
      YR=${O#tests-}
      YR=${YR%.html}
      MO=0
      DY=$YR
    else
      YR="20$YR"
    fi
    if [ $LASTYR != $YR ]; then
      html "

"
      html "
$YR
"
      LASTYR=$YR
    fi
    if [ $LASTMO != $MO ]; then
      html "

"
      html "
${MONTHS[$MO]}
"
      html "

"
      LASTMO=$MO
    fi
    SUCCESS=`grep 'class="success"' $OLDER|wc -l`
    FAILURE=`grep 'class="failure"' $OLDER|wc -l`
    if [ "$SUCCESS$FAILURE" == "00" ]; then
      COLOR="#e0e0e0"
    else
      COLOR=`perl -e "printf \"#%02x%02x%02x\", $FAILURE/($SUCCESS+$FAILURE)*220+35, $SUCCESS/($SUCCESS+$FAILURE)*220+35, 40;"`
    fi
    html "$DY	"
  done
  html "

"
  html_~body
  html_~html
}

printlist() {
  while [[ $1 != "" ]]; do
    echo $1
    shift
  done
}

outecho() {
  TYPE=$1
  shift
  echo "<$TYPE>$1|$2|$3|$4|$5|$6|$7|$8|$9|"
}

# Create HTML output
out() {
        # echo $@
  TYPE=$1
  if [ "$TYPE" == "test" ]; then

    LABEL="$2"  # Text on link to output
    CMD=$3    # Command issued (td title)
    RESULT=$4 # Passed or Failed

    # WIDTH=$( width "$LABEL" )
    if [ "$RESULT" == "Passed" ]; then
      # html ""
      html_td class "success" width 25 title "$CMD"
      html_a_href $TEST_LOG "$LABEL"
    else
      echo "FAILED"
      cat $RUNDIR/$TEST_LOG < /dev/null
      # html ""
      html_td class "failure" width 25 title "$CMD"
      html_a_href $TEST_LOG $LABEL
    fi
    html_~td

  elif [ "$TYPE" == "package" ]; then
    BINPACKAGE=$2
  else
    html $@
  fi
}

start_test_results() {
  html_h1 "Test results"
  html_a_name "tests"
  html_a_href "tests.log" "Output log from tests"
  html_table border 0 cellpadding 1
}

start_part() {
  PART=$1
  echo
  echo $PART
  echo
  html_tr part
  html_th 3
  html "$PART"
  html_~th
  html_~tr
}

start_row() {
  html_tr testline
  html_td align right width 50
  html "$SWIFTCOUNT"
  html " "
  html_~td
  html_td align right
  html " "
  if [[ -n $TESTLINK ]]; then
    html_a_href $TESTLINK $TESTNAME
  else
    html $TESTNAME
  fi
  html " "
  html_~td
  html_td
  html_table
  html_tr
  SEQ=1
}

end_row() {
  html_~tr
  html_~table
  html_~td
  html_~tr
}

# HTML width of label (for alignment)
width() {
  LABEL="$1"
  if [ ${#LABEL} -gt 2 ]; then
    WIDTH=""
  else
    WIDTH="width=\"20\""
  fi
  echo $WIDTH
}

# TEST_LOG = test log
test_log() {
  TEST_LOG="output_$LOGCOUNT.txt"
  banner "$LASTCMD" $RUNDIR/$TEST_LOG
  if [ -f $OUTPUT ]; then
    cp $OUTPUT $RUNDIR/$TEST_LOG 2>>$LOG
    cp $OUTPUT $RUNDIR/stdout.txt
  fi
  let "LOGCOUNT=$LOGCOUNT+1"
}

stars() {
  for i in {1..90}
  do
    printf "*"
  done
  echo
}

banner() {
  MSG=$1
  FILE=$2

  if [ "$FILE" == "" ]; then
    BANNER_OUTPUT=$LOG
  else
    BANNER_OUTPUT=$2
  fi
  {
    echo ""
          # stars
    echo "* $MSG"
	  # stars
  } >> $BANNER_OUTPUT
}

# Check for early bailout condition
check_bailout() {
  if [ "$EXITONFAILURE" == "true" ]; then
    if [ "$EXITCODE" != "0" ]; then
      exit $EXITCODE
    fi
  fi
}

# Translate (global) test exit code into result (Passed/Failed) string
result() {
  if [ "$EXITCODE" == "0" ]; then
    echo "Passed"
  else
    echo "Failed"
  fi
}

# Execute process in the background
process_exec() {
  printf "\nExecuting: $@" >>$LOG
  rm -f $OUTPUT

  "$@" > $OUTPUT 2>&1 &
  EXEC_PID=$!

  trap "process_exec_trap $EXEC_PID" SIGTERM

  wait $EXEC_PID
  EXITCODE=$?
  if [ "$EXITCODE" == "127" ]; then
    echo "Command not found: $@" > $OUTPUT
  fi
  if [ -f $OUTPUT ]; then
    cat $OUTPUT >> $LOG
  fi
  (( $TEST_SHOULD_FAIL )) && EXITCODE=$(( ! $EXITCODE ))
  return $EXITCODE
}

# Ensure we kill the tested process and any subordinate java processes
# in case of monitor() timeout
# Rationale: Killing bin/swift does not kill the Swift java process
process_exec_trap() {
  EXEC_PID=$1
  echo "process_exec_trap()"
  kill -KILL $EXEC_PID
  killall_swift
}

# Kill all subordinate swift/java processes
killall_swift() {
  echo "killing all swifts..."
  set -x
  echo $$
  ps -f
  kill_this $( ps -f | grep $$'.*'java  | grep -v grep )
  set +x
}

# Kill a process given its line output from "ps -f"
kill_this() {
  [ -n $2 ] && /bin/kill -KILL $2
}

# Execute as part of test set
# Equivalent to monitored_exec() (but w/o monitoring)
test_exec() {
  banner "$TEST (part $SEQ)"
  echo "Executing $TEST (part $SEQ)"
  pwd
  printf "\nExecuting: $@" >>$LOG

  rm -f $OUTPUT
  "$@" > $OUTPUT 2>&1
  EXITCODE=$?

  if [ "$EXITCODE" == "127" ]; then
    echo "Command not found: $@" > $OUTPUT
  fi

  if [ -f $OUTPUT ]; then
    cat $OUTPUT >> $LOG
  fi

  RESULT=$( result )
  test_log
  out test $SEQ "$LASTCMD" $RESULT $TEST_LOG

  check_bailout

  let "SEQ=$SEQ+1"
  return $EXITCODE
}

# Background process monitoring function
# To be executed in the background as well
# If the monitored process times out, monitor() kills it and writes
# a message to OUTPUT
# If monitor() kills the process, it returns 0
# Otherwise, return non-zero (as would result from killing
# this function)
monitor() {
  (( $VERBOSE )) && set -x

  PID=$1
  TIMEOUT=$2 # seconds
  OUTPUT=$3

  V=$SWIFTCOUNT

  # Use background so kill/trap is immediate
  sleep $TIMEOUT > /dev/null 2>&1 &
  SLEEP_PID=$!
  trap "monitor_trap $SLEEP_PID $V > /dev/null 2>&1" SIGTERM
  wait $SLEEP_PID
  [ $? != 0 ] && verbose "monitor($V) cancelled" && return 0

  if ps | grep $PID
  then
    echo "monitor($V): killing test process..."
    touch killed_test
    /bin/kill -TERM $PID
    KILLCODE=$?
    if [ $KILLCODE == 0 ]; then
      echo "monitor($V): killed process_exec (TERM)"
    fi
  fi

  sleep 1
  MSG="nightly.sh: monitor($V): killed: exceeded $TIMEOUT seconds"
  echo "$MSG" >> $OUTPUT
  # trap
  return 1
}

monitor_trap() {
  SLEEP_PID=$1
  V=$2
  verbose "monitor_trap(): kill sleep"
  /bin/kill -KILL $SLEEP_PID
}

# Execute given command line in background with monitor
# Otherwise equivalent to test_exec()
# usage: monitored_exec <TIMEOUT> <command> <args>*
monitored_exec()
{
  TIMEOUT=$1
  shift

  banner "$TEST (part $SEQ)"
  echo "Executing $TEST (part $SEQ)"

  START=$( date +%s )

  process_exec "$@" &
  PROCESS_PID=$!

  monitor $PROCESS_PID $TIMEOUT $OUTPUT &
  MONITOR_PID=$!

  wait $PROCESS_PID
  EXITCODE=$?

  STOP=$( date +%s )

  # If the test was killed, monitor() may have work to do
  rm killed_test > /dev/null 2>&1 && sleep 5
  verbose "Killing monitor..."
  /bin/kill -TERM $MONITOR_PID

  echo "TOOK: $(( STOP-START ))"

  RESULT=$( result )
  test_log
  LASTCMD="$@"
  out test $SEQ "$LASTCMD" $RESULT $TEST_LOG

  check_bailout

  let "SEQ=$SEQ+1"
}

# Execute helper script (setup, check, or clean)
script_exec() {
  SCRIPT=$1
  SYMBOL="$2"

  process_exec $SCRIPT
  RESULT=$( result )

  test_log
  out test "$SYMBOL" "$LASTCMD" $RESULT

  check_bailout
}

# Execute Swift test case w/ setup, check, clean
swift_test_case() {
  SWIFTSCRIPT=$1
  SETUPSCRIPT=${SWIFTSCRIPT%.swift}.setup.sh
  CHECKSCRIPT=${SWIFTSCRIPT%.swift}.check.sh
  CLEANSCRIPT=${SWIFTSCRIPT%.swift}.clean.sh
  TIMEOUTFILE=${SWIFTSCRIPT%.swift}.timeout

  TEST_SHOULD_FAIL=0
  if [ -x $GROUP/$SETUPSCRIPT ]; then
    script_exec $GROUP/$SETUPSCRIPT "S"
  fi

  CDM=
  [ -r fs.data ] && CDM="-cdm.file fs.data"

  (( SWIFTCOUNT++ ))

  TIMEOUT=$( gettimeout $GROUP/$TIMEOUTFILE )

  grep THIS-SCRIPT-SHOULD-FAIL $SWIFTSCRIPT > /dev/null
  TEST_SHOULD_FAIL=$(( ! $?  ))

  monitored_exec $TIMEOUT swift                         \
                       -wrapperlog.always.transfer true \
                       -config swift.properties         \
                       -sites.file sites.xml            \
                       -tc.file tc.data                 \
                       $CDM $SWIFTSCRIPT

  TEST_SHOULD_FAIL=0
  if [ -x $GROUP/$CHECKSCRIPT ]; then
    script_exec $GROUP/$CHECKSCRIPT "√"
  fi
  if [ -x $GROUP/$CLEANSCRIPT ]; then
    script_exec $GROUP/$CLEANSCRIPT "C"
  fi
}

# All timeouts in this script are in seconds
gettimeout() {
  FILE=$1

  if [ -f $FILE ]; then
    cat $FILE
  else
    echo $DEFAULT_TIMEOUT
  fi
  return 0
}

ssexec() {
  SEQSAVE=$SEQ
  SEQ=$1
  shift
  banner "$TEST (part $SEQ)"
  echo "Executing $TEST (part $SEQ)"
  aexec "$@"
  ptest
  SEQ=$SEQSAVE
}

# Fake exec
fexec() {
  banner "$TEST (faked)"
  echo "Faking $TEST"
  EXITCODE=0
  LASTCMD=""
  # ptest
}

build_package() {
  TEST="Package"
  test_exec cd $SWIFT_HOME/lib
  test_exec rm -f castor*.jar *gt2ft*.jar ant.jar
  test_exec cd $TOPDIR
  test_exec tar -pczf $RUNDIR/swift-$DATE.tar.gz $SWIFT_HOME
  out package "swift-$DATE.tar.gz"
}

# Generate the sites.sed file
make_sites_sed() {
  {
    echo "s@_WORK_@$WORK@"
    echo "s@_HOST_@$GLOBUS_HOSTNAME@"
    echo "s@_PROJECT_@$PROJECT@"
    echo "s@_QUEUE_@$QUEUE@"
  } > $RUNDIR/sites.sed
  return 0
}

# Setup coasters variables
if which ifconfig > /dev/null; then
  IFCONFIG=ifconfig
else
  IFCONFIG=/sbin/ifconfig
fi
$IFCONFIG > /dev/null || crash "Cannot run ifconfig!"
GLOBUS_HOSTNAME=$( $IFCONFIG | grep inet | head -1 | cut -d ':' -f 2 | \
                   awk '{print $1}' )
[ $? != 0 ] && crash "Could not obtain GLOBUS_HOSTNAME!"

# Generate sites.xml
group_sites_xml() {
  TEMPLATE=$GROUP/sites.template.xml
  if [ -f $TEMPLATE ]; then
    sed -f $RUNDIR/sites.sed < $TEMPLATE > sites.xml
    [ $? != 0 ] && crash "Could not create sites.xml!"
    echo "Using: $GROUP/sites.template.xml"
  else
    sed "s@_WORK_@$PWD/work@" < $TESTDIR/sites/localhost.xml > sites.xml
    [ $? != 0 ] && crash "Could not create sites.xml!"
    echo "Using: $TESTDIR/sites/localhost.xml"
  fi
}

# Generate tc.data
group_tc_data() {
  if [ -f $GROUP/tc.template.data ]; then
    sed "s@_DIR_@$GROUP@" < $GROUP/tc.template.data > tc.data
    [ $? != 0 ] && crash "Could not create tc.data!"
    echo "Using: $GROUP/tc.template.data"
  else
    cp -v $SWIFT_HOME/etc/tc.data .
    [ $? != 0 ] && crash "Could not copy tc.data!"
  fi
}

# Generate the CDM file, fs.data
group_fs_data() {
  if [ -f $GROUP/fs.template.data ]; then
    sed "s@_PWD_@$PWD@" < $GROUP/fs.template.data > fs.data
    [ $? != 0 ] && crash "Could not create fs.data!"
    echo "Using: $GROUP/fs.template.data"
  else
    rm -f fs.data
  fi
}

# Generate swift.properties
group_swift_properties() {
  if [ -f $GROUP/swift.properties ]; then
    cp -v $GROUP/swift.properties .
    [ $? != 0 ] && crash "Could not copy swift.properties!"
  else
    cp -v $SWIFT_HOME/etc/swift.properties .
    [ $? != 0 ] && crash "Could not copy swift.properties!"
  fi
}

# Obtain the group title
group_title() {
  if [ -r $GROUP/title.txt ]; then
    cat $GROUP/title.txt
  else
    echo "untitled"
  fi
}

# Execute all tests in current GROUP
test_group() {

  group_sites_xml
  group_tc_data
  group_fs_data
  group_swift_properties

  SWIFTS=$( ls $GROUP/*.swift )
  checkfail "Could not ls: $GROUP"

  for TEST in $SWIFTS; do

    (( SKIP_COUNTER++ < SKIP_TESTS )) && continue

    TESTNAME=$( basename $TEST)
    cp -v $GROUP/$TESTNAME .
    TESTLINK=$TESTNAME

    start_row
    for ((i=0; $i<$ITERS_LOCAL; i=$i+1)); do
      swift_test_case $TESTNAME
      (( $SWIFTCOUNT >= $NUMBER_OF_TESTS )) && return
    done
    end_row
  done
}

if [[ $WORK == "" ]]
then
  WORK=$TOPDIR/work
fi

checkvars GROUPLISTFILE
echo "GROUPLISTFILE: $GROUPLISTFILE"
source $GROUPLISTFILE || exit 1

cd $TOPDIR
mkdir -p $RUNDIR
[ $? != 0 ] && echo "Could not mkdir: $RUNDIR" && exit 1

date > $LOG

make_sites_sed

header
start_test_results
cd $TOPDIR

start_part "Prolog: Build"

TESTLINK=
EXITONFAILURE=true
if [ "$SKIP_CHECKOUT" != "1" ]; then
  TESTNAME="Checkout CoG"
  start_row
  test_exec rm -rf cog
  COG="https://cogkit.svn.sourceforge.net/svnroot/cogkit/trunk/current/src/cog"
  test_exec svn co $COG
  end_row

  TESTNAME="Checkout Swift"
  start_row
  test_exec cd cog/modules
  test_exec rm -rf swift
  test_exec svn co https://svn.ci.uchicago.edu/svn/vdl2/$BRANCH swift
  end_row
fi

TESTNAME="Compile"
start_row

test_exec cd $TOPDIR/cog/modules/swift
if (( $CLEAN )); then
  test_exec rm -rf dist
fi
if (( $RUN_ANT )); then
  test_exec ant -quiet dist
fi
SWIFT_HOME=$TOPDIR/cog/modules/swift/dist/swift-svn

if [ $BUILD_PACKAGE = "1" ]; then
  build_package
fi

PATH=$SWIFT_HOME/bin:$PATH
cd $TOPDIR
which swift
cd $RUNDIR

end_row

if [ $ALWAYS_EXITONFAILURE != "1" ]; then
  EXITONFAILURE=false
fi

SKIP_COUNTER=0

GROUPCOUNT=1
for G in ${GROUPLIST[@]}; do
  export GROUP=$G
  TITLE=$( group_title )
  start_part "Part $GROUPCOUNT: $TITLE"
  test_group
  (( GROUPCOUNT++ ))
  (( $SWIFTCOUNT >= $NUMBER_OF_TESTS )) && break
done

if [ $GRID_TESTS == "0" ]; then
  exit 0
fi

TESTPART="Appendix G: Grid Tests"

for TEST in `ls $TESTDIR/*.dtm $TESTDIR/*.swift`; do
  BN=`basename $TEST`
  echo $BN
  cp $TESTDIR/$BN .

  TESTNAME=${BN%.dtm}
  TESTNAME=${TESTNAME%.swift}
  TEST="$TESTNAME"

  ssexec "Compile" vdlc $BN
  for ((i=0; $i<9; i=$i+1)); do
    test_exec swift -sites.file ~/.vdl2/sites-grid.xml $TESTNAME.kml
  done
  test_exec swift -sites.file ~/.vdl2/sites-grid.xml $TESTNAME.kml
done

footer

exit 0

# Local Variables:
# sh-basic-offset: 2
# End:


[Author: dmaclach]

[Author: dmaclach]

Fix up plistcompiler so that it works with paths with spaces in them.

R=thomasvl
DELTA=1  (0 added, 0 deleted, 1 changed)

git-svn-id: 741c308154f1e22deafec9707230e8b816ec4ca6@323 7dc7ac4e-7543-0410-b95c-c1676fc8e2a3

remote: Support for password authentication was removed on August 13, 2021.
remote: Please see https://docs.github.com/en/get-started/getting-started-with-git/about-remote-repositories#cloning-with-https-urls for information on currently recommended modes of authentication.
fatal: Authentication failed for 'https://github.com/chao787/GTM-mirror.git/'