add reportDoneFile()

add reportDoneFile()

#!/usr/bin/env bash
#library of useful PWGPP related bash functions
#it REQUIRES BASH 4 !!!!
#blame: Mikolaj Krzewicki, mkrzewic@cern.ch

if [ ${BASH_VERSINFO} -lt 4 ]; then
  echo "bash version >= 4 needed, you have ${BASH_VERSION}, exiting..."
  exit 1
fi

PWGPP_runMap="
2010 108350 139517
2011 140441 170593
2012 171590 193766
2013 194308 199146
2014 202369 206695
2015 208505 999999
2016 999999 999999
"

parseConfig()
{
  # parse command line arguments, they have to be in the form
  #  option=value
  # they are then set in the environment
  # additionally another variable named: parseConfig__ORIGINAL__${option}
  # is set to have a fallback.
  # The prefix can be changed by specifying the
  # option originalOptionPrefix="some_prefix". Set to "" to switch off.
  #
  # optionally a config file can be specified in the arguments:
  #  configFile=<someFile>
  # config file sets variables: option=value
  # command line options override config file
  #
  # recommended way of using (at the beginning of your funcion/script):
  #  if ! parseConfig "${@}"; then return; fi

  local args=("$@")
  local opt=""
  local originalOptionPrefix="parseConfig__ORIGINAL__"

  #first check if we will need to decode spaces
  local encodedSpaces=""
  for opt in "${args[@]}"; do
    [[ "${opt}" =~ encodedSpaces=.* ]] \
      && encodedSpaces=1
    [[ "${opt}" =~ originalOptionPrefix=.* ]] \
      && originalOptionPrefix="${opt#*=}"
  done

  #then look for a configFile (if any)
  for opt in "${args[@]}"; do
    if [[ ${opt} =~ configFile=.* ]]; then
      eval "${opt}"
      [[ ! -f ${configFile} ]] \
        && echo "configFile ${configFile} not found, exiting..." \
        && return 1
      echo "using config file: ${configFile}"
      source "${configFile}"
      break
    fi
  done

  #then, parse the options as they override the options from configFile
  for opt in "${args[@]}"; do
    [[ -n ${encodedSpaces} ]] && opt="$(decSpaces ${opt})"
    if [[ ! "${opt}" =~ .*=.* ]]; then
      echo "badly formatted option ${var}, should be: option=value, stopping..."
      return 1
    fi
    local var="${opt%%=*}"
    local value="${opt#*=}"
    #echo "${var}=${value}"
    export ${var}="${value}"
    [[ -n ${originalOptionPrefix} ]] && export ${originalOptionPrefix}${var}="${value}"
  done
  return 0
}

guessRunData()
{
  #guess the period from the path, pick the rightmost one
  #if $ocdbStorage is set it will be reset to the anchorYear (for MC)
  period=""
  runNumber=""
  year=""
  pass=""
  legoTrainRunNumber=""
  dataType=""
  dataAOD=""
  originalPass=""
  originalPeriod=""
  anchorYear=""
  shortRunNumber=""

  local oldIFS=${IFS}
  local IFS="/"
  [[ -z ${1} ]] && return 1
  declare -a path=( $1 )
  IFS="${oldIFS}"
  local dirDepth=$(( ${#path[*]}-1 ))
  for ((x=${dirDepth};x>=0;x--)); do

    [[ $((x-2)) -ge 0 ]] && local fieldPrevPrev=${path[$((x-2))]}
    [[ $((x-1)) -ge 0 ]] && local fieldPrev=${path[$((x-1))]}
    local field=${path[${x}]}
    local fieldNext=${path[$((x+1))]}
    local fieldNextNext=${path[$((x+2))]}

    [[ -z ${legoTrainRunNumber} && ${field} =~ ^[0-9]*$ && ${fieldNext} =~ (.*\.zip$|.*\.root$) ]] && legoTrainRunNumber=${field}
    [[ -z ${runNumber} && ${fieldPrev} =~ ^000[0-9][0-9][0-9][0-9][0-9][0-9]$ ]] && runNumber=${fieldPrev#000}
    [[ -z ${pass} && -n ${runNumber} ]] && pass=${field}
    [[ ${field} =~ ^LHC[0-9][0-9][a-z].*$ ]] && period=${field%_*} && originalPeriod=${field}
    [[ ${field} =~ ^[0-9][0-9][0-9][0-9][0-9][0-9]$ ]] && shortRunNumber=${field}
    [[ ${field} =~ ^20[0-9][0-9]$ ]] && year=${field}
    [[ ${field} =~ ^(^sim$|^data$) ]] && dataType=${field}
    [[ "${field}" =~ "${pass}" && ${fieldNext} =~ AOD ]] && legoTrainRunNumber="" && dataAOD="AOD"

    [[ ${field} =~ ^LHC[0-9][0-9][a-z].*$ && ${fieldPrev} =~ ^20[0-9][0-9]$ && ! ${fieldNext} =~ ^000[0-9][0-9][0-9][0-9][0-9][0-9]$ ]] && pass=${fieldNext}

    [[ ${field} =~ "ESDs" && ${fieldPrev} =~ ^000[0-9][0-9][0-9][0-9][0-9][0-9]$ && ${fieldPrevPrev} =~ ^LHC[0-9][0-9][a-z].*$ ]] && pass=${fieldNext}

  done
  originalPass=${pass}

  if [[ ${dataType} =~ sim ]]; then
    [[ -n ${shortRunNumber} && -z ${runNumber} ]] && runNumber=${shortRunNumber}
    pass="passMC"
    originalPass="" #for MC not from lego, the runNumber is identified as lego train number, thus needs to be nulled
    anchorYear=$(run2year $runNumber)
    if [[ -z "${anchorYear}" ]]; then
      echo "WARNING: anchorYear not available for this production: ${originalPeriod}, runNumber: ${runNumber}. Cannot set the OCDB."
      return 1
    fi
    #modify the OCDB: set the year
    ocdbStorage=$(setYear ${anchorYear} ${ocdbStorage})
  else
    ocdbStorage=$(setYear ${year} ${ocdbStorage})
  fi

  [[ -n ${shortRunNumber} && -z ${runNumber} && -z {dataType} ]] && runNumber=${shortRunNumber}
  [[ -n ${shortRunNumber} && "${legoTrainRunNumber}" =~ ${shortRunNumber} ]] && legoTrainRunNumber=""
  [[ -n ${legoTrainRunNumber} ]] && pass+="_lego${legoTrainRunNumber}"
  [[ -n ${dataAOD} ]] && pass="${pass}_AOD"

  #if [[ -z ${dataType} || -z ${year} || -z ${period} || -z ${runNumber}} || -z ${pass} ]];
  #if [[ -z ${runNumber} ]]
  #then
  #  #error condition
  #  return 1
  #fi

  #ALL OK
  return 0
}

guessRunNumber()
(
  #guess the run number from the path, pick the rightmost one
  if guessRunData "${1}"; then
    echo ${runNumber}
    return 0
  fi
  return 1
)

guessYear()
(
  #guess the year from the path, pick the rightmost one
  if guessRunData "${1}"; then
    echo ${year}
    return 0
  fi
  return 1
)

guessPeriod()
(
  #guess the period from the path, pick the rightmost one
  if guessRunData "${1}"; then
    echo ${period}
    return 0
  fi
  return 1
)

setYear()
{
  #set the year in the string
  #usualy used to modify the year in $ocdbStorage
  #  ${1} - year to be set
  #  ${2} - where to set the year
  #if AUTOYEAR is present in target - it will be replaced by the year
  local yearSource=$(guessYearFast ${1})
  local yearTarget=$(guessYearFast ${2})
  local path=${2}
  [[ ${yearSource} -ne ${yearTarget} && -n ${yearTarget} && -n ${yearSource} ]] \
    && path=${2/"/${yearTarget}"/"/${yearSource}"}
  echo ${path}
  return 0
}

guessYearFast()
{
  #guess the year from the path, pick the rightmost one
  #is string AUTOYEAR present, will be returned
  local IFS="/"
  declare -a pathArray=( ${1} )
  local field=""
  local year=""
  local autoYear=""
  for field in ${pathArray[@]}; do
    [[ ${field} =~ ^20[0-9][0-9]$ ]] && year="${field}"
    [[ ${field} == AUTOYEAR ]] && autoYear="${field}"
  done
  [[ -n ${autoYear} ]] && year="${autoYear}"
  echo ${year}
  return 0
}

run2year()
{
  #for a given run print the year.
  #the run-year table is ${PWGPP_runMap} (a string)
  #one line per year, format: year runMin runMax
  local run=$1
  [[ -z ${run} ]] && return 1
  local year=""
  local runMin=""
  local runMax=""
  while read year runMin runMax; do
    [[ -z ${year} || -z ${runMin} || -z ${runMax} ]] && continue
    [[ ${run} -ge ${runMin} && ${run} -le ${runMax} ]] && echo ${year} && break
  done < <(echo "${PWGPP_runMap}")
  return 0
}

hostInfo(){
#
# Hallo world -  Print AliRoot/Root/Alien system info
#

#
# HOST info
#
    echo --------------------------------------
        echo
        echo HOSTINFO
        echo
        echo HOSTINFO HOSTNAME"      "$HOSTNAME
        echo HOSTINFO DATE"          "`date`
        echo HOSTINFO gccpath"       "`which gcc`
        echo HOSTINFO gcc version"   "`gcc --version | grep gcc`
        echo --------------------------------------

#
# ROOT info
#
        echo --------------------------------------
        echo
        echo ROOTINFO
        echo
        echo ROOTINFO ROOT"           "`which root`
        echo ROOTINFO VERSION"        "`root-config --version`
        echo
        echo --------------------------------------

#
# ALIROOT info
#
        echo --------------------------------------
        echo
        echo ALIROOTINFO
        echo
        echo "ALICE_ROOT=$ALICE_ROOT"
        echo "which aliroot: "$(which aliroot)
        echo "git describe:"
        echo "  "$(git -C $ALICE_ROOT/../src/ describe)
        echo
        echo --------------------------------------

#
# ALIPHYSICS info
#
        echo --------------------------------------
        echo
        echo "ALIPHYSICSINFO"
        echo
        echo "ALICE_PHYSICS=$ALICE_PHYSICS"
        echo "git describe:"
        echo "  "$(git -C $ALICE_PHYSICS/../src/ describe)
        echo
        echo --------------------------------------

#
# Alien info
#
#echo --------------------------------------
#echo
#echo ALIENINFO
#for a in `alien --printenv`; do echo ALIENINFO $a; done
#echo
#echo --------------------------------------

#
# Local Info
#
        echo PWD `pwd`
        echo Dir
        ls -al
        echo
        echo
        echo

  return 0
}

parseListOfFiles()
{
  #generate a list of files, one per line out of arguments.
  #names starting with "@" are assumed to be file lists and will
  #be expanded
  for file in "$@"; do
    if [[ ${file:0:1} == @ ]]; then
      [[ -r "${file:1}" ]] && cat "${file:1}"
    else
      echo "$file"
    fi
  done
}

summarizeLogs()
{
  #validate and summarize the status of logs
  #input is a list of logs, or a glob:
  #example (summarizes logs in current and subdirs):
  #  summarizeLogs * */*
  #if no args given, process all files in PWD
  #exit code 1 if some logs are not validated

  #print a summary of logs
  local -a input
  local file=""
  declare -A files
  while IFS= read x; do
    input+=("$x")
  done < <(parseListOfFiles "$@")
  [[ -z "${input[*]}" ]] && input=( "${PWD}"/* )

  #double inclusion protection+make full paths
  for file in "${input[@]}"; do
    [[ ! "${file}" =~ ^/ ]] && file="${PWD}/${file}"
    files["${file}"]="${file}"
  done

  local logFiles
  logFiles="\.*log$|^stdout$|^stderr$"

  #check logs
  local logStatus=0
  local errorSummary=""
  local validationStatus=""
  declare -A coreFiles
  for file in "${files[@]}"; do
    [[ ! -f "${file}" ]] && continue
    #keep track of core files for later processing
    [[ "${file##*/}" =~ ^core$ ]] && coreFiles[${file}]="${file}" && continue
    [[ ! "${file##*/}" =~ ${logFiles} ]] && continue
    errorSummary=$(validateLog "${file}")
    validationStatus=$?
    [[ validationStatus -ne 0 ]] && logStatus=1
    if [[ ${validationStatus} -eq 0 ]]; then
      #in pretend mode randomly report an error in rec.log some cases
      echo "${file} OK"
    elif [[ ${validationStatus} -eq 1 ]]; then
      echo "${file} BAD ${errorSummary}"
    elif [[ ${validationStatus} -eq 2 ]]; then
      echo "${file} OK MWAH ${errorSummary}"
    fi
  done

  #report core files
  for x in "${coreFiles[@]}"; do
    echo "${x}"
    chmod 644 "${x}"
    stacktraceLog=${x}.stacktrace.log
    #gdb --batch --quiet -ex "bt" -ex "quit" aliroot ${x} > stacktrace_${x//\//_}.log
    gdb --batch --quiet -ex "bt" -ex "quit" aliroot "${x}" > "$stacktraceLog"
    local nLines[2]
    #nLines=($(wc -l stacktrace_${x//\//_}.log))
    nLines=($(wc -l "$stacktraceLog"))
    if [[ ${nLines[0]} -eq 0 ]]; then
      #rm stacktrace_${x//\//_}.log
      rm "$stacktraceLog"
    else
      logStatus=1
      echo "stacktrace $stacktraceLog"
    fi
  done

  return ${logStatus}
}

validateLog()
{
  #validate one log file
  #input is path to log file
  #output an error summary on stdout
  #exit code is 0 if validated, 1 otherwise
  log="${1}"
  [[ ! -f "$log" ]] && return 1
  errorConditions=(
            'There was a crash'
            'floating'
            'error while loading shared libraries'
            'std::bad_alloc'
            's_err_syswatch_'
            'Thread [0-9]* (Thread'
            'AliFatal'
            '\.C.*error:.*\.h: No such file'
            'segmentation'
            'Segmentation fault'
            'Interpreter error recovered'
            ': command not found'
            ': comando non trovato'
            'core dumped'
            'core file'
  )

  warningConditions=(
            'This is serious'
  )

  local logStatus=0
  local errorSummary=""
  local warningSummary=""
  local errorCondition=""
  for errorCondition in "${errorConditions[@]}"; do
    local tmp=$(grep -m1 -e "${errorCondition}" "${log}")
    local error=""
    [[ -n "${tmp}" ]] && error=" : ${errorCondition}"
    errorSummary+=${error}
  done

  local warningCondition=""
  for warningCondition in "${warningConditions[@]}"; do
    local tmp=$(grep -m1 -e "${warningCondition}" "${log}")
    local warning=""
    [[ -n "${tmp}" ]] && warning=" : ${warningCondition}"
    warningSummary+="${warning}"
  done

  if [[ -n ${errorSummary} ]]; then
    echo "${errorSummary}"
    return 1
  fi

  if [[ -n ${warningSummary} ]]; then
    echo "${warningSummary}"
    return 2
  fi

  return 0
}

mergeSysLogs()
{
  if [[ $# -lt 2 ]]; then
    echo 'merge syslogs to an output file'
    echo 'usage:'
    echo 'mergeSysLogs outputFile inputFile1 inputFile2 ...'
    echo 'if file name prepended with "@" it is a file list'
    return 0
  fi
  local outputFile
  local inputFiles
  local i
  local x
  local runNumber
  outputFile=${1}
  shift
  inputFiles="$@"
  i=0
  parseListOfFiles ${inputFiles[@]} | while IFS= read x; do
    runNumber=$(guessRunNumber "${x}")
    [[ -z ${runNumber} ]] && runNumber=0
    gawk -v run=${runNumber} -v i=${i} 'NR > 1 {print run" "$0} NR==1 && i==0 {print "run/I:"$0}' "${x}"
    (( i++ ))
  done > "${outputFile}"
  return 0
}

stackTraceTree()
{
  if [[ $# -lt 1 ]]; then
    echo 'make stacktrace processing  in case of standard root crash log'
    echo 'input is a (list of) text files with the stack trace (either gdb aoutput'
    echo 'produced with e.g. gdb --batch --quiet -ex "bt" -ex "quit" aliroot core,'
    echo 'or the root crash log), output is a TTree formatted table.'
    echo 'example usage:'
    echo 'benchmark.sh stackTraceTree /foo/*/rec.log'
    echo 'benchmark.sh stackTraceTree $(cat file.list)'
    echo 'benchmark.sh stackTraceTree `cat file.list`'
    echo 'benchmark.sh stackTraceTree @file.list somefile.log'
    return 0
  fi
  parseListOfFiles "$@" | while IFS= read x; do echo "filename: $x"; cat "$x" 2>/dev/null; done | gawk '
       BEGIN {
       print "frame/I:method/C:line/C:cpass/I:aliroot/I:file/C";
               RS="#[0-9]*";
               aliroot=0;
               read=1;
             }
      /^filename:/ {filename=$2}
      /There was a crash/ {read=1;}
      /The lines below might hint at the cause of the crash/ {read=0;}
      read==1 {
               if ($3 ~ /Ali*/) aliroot=1; else aliroot=0;
               gsub("#","",RT);
               if ($NF!="" && RT!="" && $3!="") print RT" "$3" "$NF" "0" "aliroot" "filename
             }
      ' 2>/dev/null
}

plotStackTraceTree()
{
  #plot the stacktrace tree,
  #first arg    is the text file in the root tree format
  #second arg   is optional: a plot is written to file instead of screen
  #third arg    is optional: selection for plotting, default skip G_ stuff
  local tree=$1
  local plot=${2:-"crashes.png"}
  local selection=${3:-'!strstr(method,\"G__\")'}
  [[ ! -f ${tree} ]] && echo "plotStackTraceTree: no input file given" && return 1
  aliroot -b <<EOF
TTree* t=AliSysInfo::MakeTree("${tree}");
TCanvas* canvas = new TCanvas("QA crashes","QA crashes",1);
t->Draw("method","${selection}","");
canvas->SaveAs("${plot}");
.q
EOF
  return 0
}

#these functions encode strings to and from a space-less form
#use when spaces are not well handled (e.g. in arguments to
#commands in makeflow files, etc.
encSpaces()(echo "${1// /@@@@}")
decSpaces()(echo "${1//@@@@/ }")

get_realpath()
{
  if [[ $# -lt 1 ]]; then
    echo "print the full path of a file or directory, like \"readlink -f\" on linux"
    echo "Usage:"
    echo "  get_realpath <someFileOrDir>"
    return 0
  fi
  if [[ -f "$1" ]]
  then
    # file *must* exist
    if cd "$(echo "${1%/*}")" &>/dev/null
    then
      # file *may* not be local
      # exception is ./file.ext
      # try 'cd .; cd -;' *works!*
      local tmppwd="$PWD"
      cd - &>/dev/null
    else
      # file *must* be local
      local tmppwd="$PWD"
    fi
  elif [[ -d "$1" ]]; then
    if cd "$1" &>/dev/null; then
      local tmppwd="$PWD"
      cd - &>/dev/null
      echo "$tmppwd"
      return 0
    else
      return 1
    fi
  else
    # file *cannot* exist
    return 1 # failure
  fi
  # reassemble realpath
  echo "$tmppwd"/"${1##*/}"
  return 0 # success
}

printLogStatistics()
{
  #this function processes the summary logs and prints some stats
  #relies on the summary log format produced by summarizeLogs()
  # - how many checked logs in total
  # - number of each type of problem
  # example usage:
  #   printLogStatistics */*.log
  echo "log statistics from: ${1%/*}"
  parseListOfFiles "$@" | while IFS= read x; do cat "$x" 2>/dev/null; done | gawk '
  BEGIN {nOK=0; nCores=0; nStackTraces=0; nLogs=0;}
  /\/core/ {nCores++}
  /\/stacktrace.log/ {nStackTraces++}
  /OK/ {nOK++; nLogs++;}
  /BAD/ {
    nLogs++
    err=""
    write=0
    for (i=3; i<=NF; i++)
    {
      if ($i ~ /^\:$/)
        write=1
      else
        write=0

      if (write==0)
      {
        if (err=="") err=$i
        else err=(err FS $i)
      }

      if (err != "" && (write==1 || i==NF))
      {
        sumBAD[err]++
        err=""
      }
    }
  }
  END {
    print ("number of validated logs: " nOK" out of "nLogs )
    for (key in sumBAD)
    {
      print key": "sumBAD[key]
    }
    if (nCores>0 || nStackTraces>0) print "core files: "nCores", stack traces: "nStackTraces
  }
  '
}

createUniquePID()
{
  #create a unique ID for jobs running in parallel
  #consists of the ip address of the default network interface, PID,
  #if an argument is given, append it (e.g. a production ID)
  #the fields are space separated with a tag for easy parsing
  #spaces in the productionID will be encoded using encSpaces()
  local productionID=""
  [[ -n "${1}" ]] && productionID=$(encSpaces "${1}")
  local defaultIP=$(/sbin/route | awk '$1=="default" {print $8}' | xargs /sbin/ifconfig | awk '/inet / {print $2}' | sed 's/.*\([0-9]?\.[0-9]?\.[0-9]?\.[0-9]?\)/$1/')
  local id="ip:${defaultIP} pid:${BASHPID}"
  [[ -n "${productionID}" ]] && id+=" prod:${productionID}"
  echo "${id}"
}

printExec() {
  echo "[command] [PWD=$PWD]: $*" >&2
  "$@"
}

bigEcho() {
  # Outputs big text to stderr.
  which figlet &> /dev/null \
    && figlet -- "$@" >&2 \
    || ( printf "\n\n$*\n\n" | tr -d '[[:lower:]]' '[[:upper:]]' >&2 )
  true
}

listDir() (
  dir="$(cd "$1"; pwd)"
  echo ; echo "[listDir] Content of ${dir}${2:+" ($2)"}"
  find "$dir" -ls
  echo
)

mkdirLocal() (
  # Creates the given directories, with full path, only if local. If not local,
  # print a message and return success.
  # Return 1 if at least one error happened, 0 on success.
  err=0
  while [[ $# -gt 0 ]]; do
    dir=$1
    shift
    if [[ "${dir%%://*}" != "$dir" ]]; then
      echo "[mkdirLocal] skipping creation of $dir: it is not local"
      continue
    fi
    mkdir -p "$dir"
    [[ -d "$dir" ]]
    rv=$?
    err=$((err + ($rv & 1)))
    echo "[mkdirLocal] creation of dir $dir $([[ $rv == 0 ]] && echo "OK" || echo "FAILED")"
  done
  return $((err & 1))
)

statRemote() (
  # Check if file exists, whether it is local or remote. Returns 0 on success, 1 on failure.
  file=$1
  proto="${file%%://*}"
  [[ "$proto" == "$file" ]] && proto=local
  case "$proto" in
    local) [[ -f $file ]] ;;
    root)  path=${file:$((${#proto}+3))}
           host=${path%%/*}
           path=${path:$((${#host}))}
           while [[ ${path:0:2} == // ]]; do path=${path:1}; done
           xrd "$host" stat "$path" 2>&1 | grep -q Modtime: ;;
    *)     echo "[statRemote] for file $file: protocol not supported: $proto"
           return 2 ;;
  esac
  rv=$?
  echo "[statRemote] file $file (proto=${proto}$([[ $proto == local ]] && echo ", pwd=$PWD"))" \
       "$([[ $rv == 0 ]] && echo "exists" || echo "does NOT exist")"
  return $rv
)

lsRemote() (
  # List remote files on stdout. Returns 0 on success, 1 on failure.
  dir=$1
  proto="${dir%%://*}"
  [[ "$proto" == "$dir" ]] && proto=local
  case "$proto" in
    local) find "$dir" -maxdepth 1 -mindepth 1 ;;
    root)  path=${dir:$((${#proto}+3))}
           host=${path%%/*}
           path=${path:$((${#host}))}
           while [[ ${path:0:2} == // ]]; do path=${path:1}; done
           xrd "$host" ls "$path" 2>&1 | grep -v "No such file or directory" | grep -o "${path}.*" ;;
    *)     echo "[lsRemote] for directory $dir: protocol not supported: $proto" >&2
           return 2 ;;
  esac
  rv=$?
  [[ $rv != 0 ]] && echo "[lsRemote] cannot list directory $dir" >&2
  return $rv
)

copyFileFromRemote() (
  # Copy a list of remote files ($1, $2...${n-1}) to a certain dest dir ($n).
  # The last parameter must then be a local directory. Files can be local too.
  # Dest dir is created if not existing.
  # If only one parameter is specified download this file to the current directory.
  # If a source file is in the form @list.txt, then the list of files (one per
  # line) in list.txt will be expanded and copied.
  # On success 0 is returned - 1 otherwise.
  # Example: copyFileFromRemote root://localhost//file1.txt @list.txt /tmp/localdir/foo

  [[ $# == 1 ]] && dstdir=. || dstdir=${!#}
  maxCopyTries=${maxCopyTries-10}
  err=0
  opname="[copyFileFromRemote]"
  mkdir -p "$dstdir"

  while [[ $# -gt 1 ]]; do
    [[ ${1:0:1} == @ ]] && inputcmd="cat ${1:1}" || inputcmd="echo $1"
    while read -u 3 src; do
      thiserr=1
      proto="${src%%://*}"
      [[ "$proto" == "$src" ]] && proto=local
      dst="$dstdir/$(basename "$src")"
      echo "$opname (proto=$proto) started: $src -> $dst"
      for ((i=1; i<=maxCopyTries; i++)); do
        echo "$opname $src -> $dst attempt $i of $maxCopyTries"
        case "$proto" in
          local) printExec cp "$src" "$dst"
                 if [[ $? != 0 ]]; then
                   rm -f "$dst"
                   false
                 fi ;;
          root)  printExec xrdcp -f "$src" "$dst" ;;
          http*) printExec curl -LsSfo "$dst" "$src" ;;
          *)     echo "protocol not supported: $proto"
                 return 2 ;;
        esac
        if [[ $? == 0 ]]; then
          thiserr=0
          break
        fi
      done  # cp attempts
      [[ $thiserr == 0 ]] && echo "$opname (proto=$proto) OK after $i attempt(s): $src -> $dst" \
                          || echo "$opname (proto=$proto) FAILED after $maxCopyTries attempt(s): $src -> $dst"
      err=$((err+thiserr))
    done 3< <($inputcmd)
    shift
  done

  return $((err & 1))
)

copyFileToRemote() (
  # Copy a list of files ($1, $2...${n-1}) to a certain dest dir ($n). The last
  # parameter must then be a directory. The destination might be local or remote
  # if a protocol is specified (e.g. root://).
  # Dest dir is created if not existing (xrdcp does that already).
  # If a source file is in the form @list.txt, then the list of files (one per
  # line) in list.txt will be expanded and copied.
  # Shell glob is supported.
  # On success 0 is returned - 1 otherwise.
  # Example: copyFileToRemote file1.txt files* @list.txt test1 root://host/dir

  dstdir=${!#}
  maxCopyTries=${maxCopyTries-10}
  proto="${dstdir%%://*}"
  err=0
  [[ "$proto" == "$dstdir" ]] && proto=local
  opname="[copyFileToRemote] (proto=$proto)"

  while [[ $# -gt 1 ]]; do
    [[ ${1:0:1} == @ ]] && inputcmd="cat ${1:1}" || inputcmd="echo $1"
    while read -u 3 src; do
      thiserr=1
      dst="$dstdir/$(basename "$src")"
      for ((i=1; i<=maxCopyTries; i++)); do
        [[ -d "$src" ]] && echo "$opname $src -> $dst skipping, is a directory" && thiserr=0 && break
        # TODO use shopt -s nullglob
        [[ ! -r "$src" ]] && echo "$opname $src -> $dst skipping, cannot access source" && thiserr=0 && break
        echo "$opname $src -> $dst attempt $i of $maxCopyTries"
        case "$proto" in
          local) mkdir -p "$(dirname "$dst")"
                 printExec cp "$src" "$dst"
                 if [[ $? != 0 ]]; then
                   rm -f "$dst"
                   false
                 fi ;;
          root)  printExec xrdcp -f "$src" "$dst" ;;
          *)     echo "protocol not supported: $proto"
                 return 2 ;;
        esac
        if [[ $? == 0 ]]; then
          thiserr=0
          break
        fi
      done  # cp attempts
      [[ $thiserr == 0 ]] && echo "$opname OK after $i attempt(s): $src -> $dst" \
                          || echo "$opname FAILED after $maxCopyTries attempt(s): $src -> $dst"
      err=$((err+thiserr))
    done 3< <($inputcmd)
    shift
  done

  return $((err & 1))
)

paranoidCp()
(
  #recursively copy files and directories
  #if target is a directory - it must exist!
  #to avoid using find and the like as they kill
  #the performance on some cluster file systems
  #does not copy links to avoid problems
  sourceFiles=("${@}")
  destination="${sourceFiles[@]:(-1)}" #last element
   #check if we are not trying to copy to the same structure
  if [ $destination == $sourceFiles ] ; then
      echo paranoidCp INFO skip
      echo paranoidCp INFO destination== $destination
      echo paranoidCp INFO sourceFiles== $sourceFiles
      return 1;
  fi

  unset sourceFiles[${#sourceFiles[@]}-1] #remove last element (dst)
  #[[ ! -f "${destination}" ]]
  for src in "${sourceFiles[@]}"; do
    if [[ -f "${src}" && ! -h  "${src}" ]]; then
      paranoidCopyFile "${src}" "${destination}"
    elif [[ -d "${src}" && ! -h "${src}" ]]; then
      src="${src%/}"
      dst="${destination}/${src##*/}"
      mkdir -p "${dst}"
      paranoidCp "${src}"/* "${dst}"
    fi
  done
)

paranoidCopyFile()
(
  #copy a single file to a target in an existing dir
  #repeat a few times if copy fails
  #returns 1 on failure, 0 on success
  src=$(get_realpath "${1}")
  dst=$(get_realpath "${2}")
  [[ -d "${dst}" ]] && dst="${dst}/${src##*/}"
  #some sanity check
  [[ -z "${src}" ]] && echo "$1 does not exist" && return 1
  [[ -z "${dst}" ]] && echo "$2 does not exist" && return 1
  #check if we are not trying to copy to the same file
  [[ "${src}" == "${dst}" ]] && echo "$dst==$src, not copying" && return 0
  ok=0
  [[ -z "${maxCopyTries}" ]] && maxCopyTries=10

  echo "paranoid copy started: $src -> $dst"
  for (( i=1 ; i<=maxCopyTries ; i++ )) ; do

    echo "...attempt $i of $maxCopyTries"
    #rm -f "$dst"
    cp -a -n "$src" "$dst"

    cmp -s "$src" "$dst"
    if [ $? == 0 ] ; then
      ok=1
      break
    else
      rm -f "${dst}"
    fi

  done

  if [[ "$ok" == 1 ]] ; then
    echo "paranoid copy OK after $i attempt(s): $src -> $dst"
    return 0
  fi

  echo "paranoid copy FAILED after $maxCopyTries attempt(s): $src -> $dst"
  return 1
)

generPWD(){
  #
  # generate semirandom pwd using 2 keys
  # Example usage:
  # generPWD  myserviceaccount10 key11
  key0=$1
  key1=$2
  heslo0=`md5sum <<< "$key0 $key1" | cut -c 1-16`
  heslo=`echo $heslo0 | cut -c 1-8| awk '{print toupper($0)}'`
  heslo=$heslo`echo $heslo0 | cut -c 8-15| awk '{print tolower($0)}'`%
  echo $heslo;
}

reformatXMLCollection()
{
  #parse the xml collection on stdinput
  #output a collection in format: file md5 ctime size
  local nfields=""
  local turl=""
  local md5=""
  local ctime=""
  local size=""
  local x=""
  while read -a fields; do
    nfields=${#fields[*]}
    turl=""
    md5=""
    ctime=""
    size=""
    for ((x=1;x<=${nfields};x++)); do
      field=${fields[${x}]}
      if [[ "${field}" == "md5="* ]]; then
        eval ${field}
      fi
      if [[ "${field}" == "turl="* ]]; then
        eval ${field}
      fi
      if [[ "${field}" == "ctime="* ]]; then
        eval "${field} ${fields[((x+1))]}"
      fi
      if [[ "${field}" == "size="* ]]; then
        eval ${field}" "${fields[((x+1))]}
      fi
    done
    ctime=$( date -d "${ctime}" +%s 2>/dev/null)
    [[ -z $md5 ]] && md5="."
    [[ -n "$turl" ]] && echo "${turl//"alien://"/} ${md5} ${ctime} ${size}"
  done
}

reportDoneFile()
{
  #print a report on a file if exists
  #args: tag filename directory
  local tag=$1
  local file=$2
  local dir=$3
  [[ -r $file ]] && echo "$tag ${dir}/${file}"
}

#this makes debugging easier:
#executes the command given as an argument in this environment
#use case:
#  bashdb utilities.sh summarizeLogs * */*
[[ $# != 0 ]] && eval "$@" || true

# vi:syntax=zsh

#!/usr/bin/env bash
#library of useful PWGPP related bash functions
#it REQUIRES BASH 4 !!!!
#blame: Mikolaj Krzewicki, mkrzewic@cern.ch

if [ ${BASH_VERSINFO} -lt 4 ]; then
  echo "bash version >= 4 needed, you have ${BASH_VERSION}, exiting..."
  exit 1
fi

PWGPP_runMap="
2010 108350 139517
2011 140441 170593
2012 171590 193766
2013 194308 199146
2014 202369 206695
2015 208505 999999
2016 999999 999999
"

parseConfig()
{
  # parse command line arguments, they have to be in the form
  #  option=value
  # they are then set in the environment
  # additionally another variable named: parseConfig__ORIGINAL__${option}
  # is set to have a fallback.
  # The prefix can be changed by specifying the
  # option originalOptionPrefix="some_prefix". Set to "" to switch off.
  #
  # optionally a config file can be specified in the arguments:
  #  configFile=<someFile>
  # config file sets variables: option=value
  # command line options override config file
  #
  # recommended way of using (at the beginning of your funcion/script):
  #  if ! parseConfig "${@}"; then return; fi

  local args=("$@")
  local opt=""
  local originalOptionPrefix="parseConfig__ORIGINAL__"

  #first check if we will need to decode spaces
  local encodedSpaces=""
  for opt in "${args[@]}"; do
    [[ "${opt}" =~ encodedSpaces=.* ]] \
      && encodedSpaces=1
    [[ "${opt}" =~ originalOptionPrefix=.* ]] \
      && originalOptionPrefix="${opt#*=}"
  done

  #then look for a configFile (if any)
  for opt in "${args[@]}"; do
    if [[ ${opt} =~ configFile=.* ]]; then
      eval "${opt}"
      [[ ! -f ${configFile} ]] \
        && echo "configFile ${configFile} not found, exiting..." \
        && return 1
      echo "using config file: ${configFile}"
      source "${configFile}"
      break
    fi
  done

  #then, parse the options as they override the options from configFile
  for opt in "${args[@]}"; do
    [[ -n ${encodedSpaces} ]] && opt="$(decSpaces ${opt})"
    if [[ ! "${opt}" =~ .*=.* ]]; then
      echo "badly formatted option ${var}, should be: option=value, stopping..."
      return 1
    fi
    local var="${opt%%=*}"
    local value="${opt#*=}"
    #echo "${var}=${value}"
    export ${var}="${value}"
    [[ -n ${originalOptionPrefix} ]] && export ${originalOptionPrefix}${var}="${value}"
  done
  return 0
}

guessRunData()
{
  #guess the period from the path, pick the rightmost one
  #if $ocdbStorage is set it will be reset to the anchorYear (for MC)
  period=""
  runNumber=""
  year=""
  pass=""
  legoTrainRunNumber=""
  dataType=""
  dataAOD=""
  originalPass=""
  originalPeriod=""
  anchorYear=""
  shortRunNumber=""

  local oldIFS=${IFS}
  local IFS="/"
  [[ -z ${1} ]] && return 1
  declare -a path=( $1 )
  IFS="${oldIFS}"
  local dirDepth=$(( ${#path[*]}-1 ))
  for ((x=${dirDepth};x>=0;x--)); do

    [[ $((x-2)) -ge 0 ]] && local fieldPrevPrev=${path[$((x-2))]}
    [[ $((x-1)) -ge 0 ]] && local fieldPrev=${path[$((x-1))]}
    local field=${path[${x}]}
    local fieldNext=${path[$((x+1))]}
    local fieldNextNext=${path[$((x+2))]}

    [[ -z ${legoTrainRunNumber} && ${field} =~ ^[0-9]*$ && ${fieldNext} =~ (.*\.zip$|.*\.root$) ]] && legoTrainRunNumber=${field}
    [[ -z ${runNumber} && ${fieldPrev} =~ ^000[0-9][0-9][0-9][0-9][0-9][0-9]$ ]] && runNumber=${fieldPrev#000}
    [[ -z ${pass} && -n ${runNumber} ]] && pass=${field}
    [[ ${field} =~ ^LHC[0-9][0-9][a-z].*$ ]] && period=${field%_*} && originalPeriod=${field}
    [[ ${field} =~ ^[0-9][0-9][0-9][0-9][0-9][0-9]$ ]] && shortRunNumber=${field}
    [[ ${field} =~ ^20[0-9][0-9]$ ]] && year=${field}
    [[ ${field} =~ ^(^sim$|^data$) ]] && dataType=${field}
    [[ "${field}" =~ "${pass}" && ${fieldNext} =~ AOD ]] && legoTrainRunNumber="" && dataAOD="AOD"

    [[ ${field} =~ ^LHC[0-9][0-9][a-z].*$ && ${fieldPrev} =~ ^20[0-9][0-9]$ && ! ${fieldNext} =~ ^000[0-9][0-9][0-9][0-9][0-9][0-9]$ ]] && pass=${fieldNext}

    [[ ${field} =~ "ESDs" && ${fieldPrev} =~ ^000[0-9][0-9][0-9][0-9][0-9][0-9]$ && ${fieldPrevPrev} =~ ^LHC[0-9][0-9][a-z].*$ ]] && pass=${fieldNext}

  done
  originalPass=${pass}

  if [[ ${dataType} =~ sim ]]; then
    [[ -n ${shortRunNumber} && -z ${runNumber} ]] && runNumber=${shortRunNumber}
    pass="passMC"
    originalPass="" #for MC not from lego, the runNumber is identified as lego train number, thus needs to be nulled
    anchorYear=$(run2year $runNumber)
    if [[ -z "${anchorYear}" ]]; then
      echo "WARNING: anchorYear not available for this production: ${originalPeriod}, runNumber: ${runNumber}. Cannot set the OCDB."
      return 1
    fi
    #modify the OCDB: set the year
    ocdbStorage=$(setYear ${anchorYear} ${ocdbStorage})
  else
    ocdbStorage=$(setYear ${year} ${ocdbStorage})
  fi

  [[ -n ${shortRunNumber} && -z ${runNumber} && -z {dataType} ]] && runNumber=${shortRunNumber}
  [[ -n ${shortRunNumber} && "${legoTrainRunNumber}" =~ ${shortRunNumber} ]] && legoTrainRunNumber=""
  [[ -n ${legoTrainRunNumber} ]] && pass+="_lego${legoTrainRunNumber}"
  [[ -n ${dataAOD} ]] && pass="${pass}_AOD"

  #if [[ -z ${dataType} || -z ${year} || -z ${period} || -z ${runNumber}} || -z ${pass} ]];
  #if [[ -z ${runNumber} ]]
  #then
  #  #error condition
  #  return 1
  #fi

  #ALL OK
  return 0
}

guessRunNumber()
(
  #guess the run number from the path, pick the rightmost one
  if guessRunData "${1}"; then
    echo ${runNumber}
    return 0
  fi
  return 1
)

guessYear()
(
  #guess the year from the path, pick the rightmost one
  if guessRunData "${1}"; then
    echo ${year}
    return 0
  fi
  return 1
)

guessPeriod()
(
  #guess the period from the path, pick the rightmost one
  if guessRunData "${1}"; then
    echo ${period}
    return 0
  fi
  return 1
)

setYear()
{
  #set the year in the string
  #usualy used to modify the year in $ocdbStorage
  #  ${1} - year to be set
  #  ${2} - where to set the year
  #if AUTOYEAR is present in target - it will be replaced by the year
  local yearSource=$(guessYearFast ${1})
  local yearTarget=$(guessYearFast ${2})
  local path=${2}
  [[ ${yearSource} -ne ${yearTarget} && -n ${yearTarget} && -n ${yearSource} ]] \
    && path=${2/"/${yearTarget}"/"/${yearSource}"}
  echo ${path}
  return 0
}

guessYearFast()
{
  #guess the year from the path, pick the rightmost one
  #is string AUTOYEAR present, will be returned
  local IFS="/"
  declare -a pathArray=( ${1} )
  local field=""
  local year=""
  local autoYear=""
  for field in ${pathArray[@]}; do
    [[ ${field} =~ ^20[0-9][0-9]$ ]] && year="${field}"
    [[ ${field} == AUTOYEAR ]] && autoYear="${field}"
  done
  [[ -n ${autoYear} ]] && year="${autoYear}"
  echo ${year}
  return 0
}

run2year()
{
  #for a given run print the year.
  #the run-year table is ${PWGPP_runMap} (a string)
  #one line per year, format: year runMin runMax
  local run=$1
  [[ -z ${run} ]] && return 1
  local year=""
  local runMin=""
  local runMax=""
  while read year runMin runMax; do
    [[ -z ${year} || -z ${runMin} || -z ${runMax} ]] && continue
    [[ ${run} -ge ${runMin} && ${run} -le ${runMax} ]] && echo ${year} && break
  done < <(echo "${PWGPP_runMap}")
  return 0
}

hostInfo(){
#
# Hallo world -  Print AliRoot/Root/Alien system info
#

#
# HOST info
#
    echo --------------------------------------
        echo
        echo HOSTINFO
        echo
        echo HOSTINFO HOSTNAME"      "$HOSTNAME
        echo HOSTINFO DATE"          "`date`
        echo HOSTINFO gccpath"       "`which gcc`
        echo HOSTINFO gcc version"   "`gcc --version | grep gcc`
        echo --------------------------------------

#
# ROOT info
#
        echo --------------------------------------
        echo
        echo ROOTINFO
        echo
        echo ROOTINFO ROOT"           "`which root`
        echo ROOTINFO VERSION"        "`root-config --version`
        echo
        echo --------------------------------------

#
# ALIROOT info
#
        echo --------------------------------------
        echo
        echo ALIROOTINFO
        echo
        echo "ALICE_ROOT=$ALICE_ROOT"
        echo "which aliroot: "$(which aliroot)
        echo "git describe:"
        echo "  "$(git -C $ALICE_ROOT/../src/ describe)
        echo
        echo --------------------------------------

#
# ALIPHYSICS info
#
        echo --------------------------------------
        echo
        echo "ALIPHYSICSINFO"
        echo
        echo "ALICE_PHYSICS=$ALICE_PHYSICS"
        echo "git describe:"
        echo "  "$(git -C $ALICE_PHYSICS/../src/ describe)
        echo
        echo --------------------------------------

#
# Alien info
#
#echo --------------------------------------
#echo
#echo ALIENINFO
#for a in `alien --printenv`; do echo ALIENINFO $a; done
#echo
#echo --------------------------------------

#
# Local Info
#
        echo PWD `pwd`
        echo Dir
        ls -al
        echo
        echo
        echo

  return 0
}

parseListOfFiles()
{
  #generate a list of files, one per line out of arguments.
  #names starting with "@" are assumed to be file lists and will
  #be expanded
  for file in "$@"; do
    if [[ ${file:0:1} == @ ]]; then
      [[ -r "${file:1}" ]] && cat "${file:1}"
    else
      echo "$file"
    fi
  done
}

summarizeLogs()
{
  #validate and summarize the status of logs
  #input is a list of logs, or a glob:
  #example (summarizes logs in current and subdirs):
  #  summarizeLogs * */*
  #if no args given, process all files in PWD
  #exit code 1 if some logs are not validated

  #print a summary of logs
  local -a input
  local file=""
  declare -A files
  while IFS= read x; do
    input+=("$x")
  done < <(parseListOfFiles "$@")
  [[ -z "${input[*]}" ]] && input=( "${PWD}"/* )

  #double inclusion protection+make full paths
  for file in "${input[@]}"; do
    [[ ! "${file}" =~ ^/ ]] && file="${PWD}/${file}"
    files["${file}"]="${file}"
  done

  local logFiles
  logFiles="\.*log$|^stdout$|^stderr$"

  #check logs
  local logStatus=0
  local errorSummary=""
  local validationStatus=""
  declare -A coreFiles
  for file in "${files[@]}"; do
    [[ ! -f "${file}" ]] && continue
    #keep track of core files for later processing
    [[ "${file##*/}" =~ ^core$ ]] && coreFiles[${file}]="${file}" && continue
    [[ ! "${file##*/}" =~ ${logFiles} ]] && continue
    errorSummary=$(validateLog "${file}")
    validationStatus=$?
    [[ validationStatus -ne 0 ]] && logStatus=1
    if [[ ${validationStatus} -eq 0 ]]; then
      #in pretend mode randomly report an error in rec.log some cases
      echo "${file} OK"
    elif [[ ${validationStatus} -eq 1 ]]; then
      echo "${file} BAD ${errorSummary}"
    elif [[ ${validationStatus} -eq 2 ]]; then
      echo "${file} OK MWAH ${errorSummary}"
    fi
  done

  #report core files
  for x in "${coreFiles[@]}"; do
    echo "${x}"
    chmod 644 "${x}"
    stacktraceLog=${x}.stacktrace.log
    #gdb --batch --quiet -ex "bt" -ex "quit" aliroot ${x} > stacktrace_${x//\//_}.log
    gdb --batch --quiet -ex "bt" -ex "quit" aliroot "${x}" > "$stacktraceLog"
    local nLines[2]
    #nLines=($(wc -l stacktrace_${x//\//_}.log))
    nLines=($(wc -l "$stacktraceLog"))
    if [[ ${nLines[0]} -eq 0 ]]; then
      #rm stacktrace_${x//\//_}.log
      rm "$stacktraceLog"
    else
      logStatus=1
      echo "stacktrace $stacktraceLog"
    fi
  done

  return ${logStatus}
}

validateLog()
{
  #validate one log file
  #input is path to log file
  #output an error summary on stdout
  #exit code is 0 if validated, 1 otherwise
  log="${1}"
  [[ ! -f "$log" ]] && return 1
  errorConditions=(
            'There was a crash'
            'floating'
            'error while loading shared libraries'
            'std::bad_alloc'
            's_err_syswatch_'
            'Thread [0-9]* (Thread'
            'AliFatal'
            '\.C.*error:.*\.h: No such file'
            'segmentation'
            'Segmentation fault'
            'Interpreter error recovered'
            ': command not found'
            ': comando non trovato'
            'core dumped'
            'core file'
  )

  warningConditions=(
            'This is serious'
  )

  local logStatus=0
  local errorSummary=""
  local warningSummary=""
  local errorCondition=""
  for errorCondition in "${errorConditions[@]}"; do
    local tmp=$(grep -m1 -e "${errorCondition}" "${log}")
    local error=""
    [[ -n "${tmp}" ]] && error=" : ${errorCondition}"
    errorSummary+=${error}
  done

  local warningCondition=""
  for warningCondition in "${warningConditions[@]}"; do
    local tmp=$(grep -m1 -e "${warningCondition}" "${log}")
    local warning=""
    [[ -n "${tmp}" ]] && warning=" : ${warningCondition}"
    warningSummary+="${warning}"
  done

  if [[ -n ${errorSummary} ]]; then
    echo "${errorSummary}"
    return 1
  fi

  if [[ -n ${warningSummary} ]]; then
    echo "${warningSummary}"
    return 2
  fi

  return 0
}

mergeSysLogs()
{
  if [[ $# -lt 2 ]]; then
    echo 'merge syslogs to an output file'
    echo 'usage:'
    echo 'mergeSysLogs outputFile inputFile1 inputFile2 ...'
    echo 'if file name prepended with "@" it is a file list'
    return 0
  fi
  local outputFile
  local inputFiles
  local i
  local x
  local runNumber
  outputFile=${1}
  shift
  inputFiles="$@"
  i=0
  parseListOfFiles ${inputFiles[@]} | while IFS= read x; do
    runNumber=$(guessRunNumber "${x}")
    [[ -z ${runNumber} ]] && runNumber=0
    gawk -v run=${runNumber} -v i=${i} 'NR > 1 {print run" "$0} NR==1 && i==0 {print "run/I:"$0}' "${x}"
    (( i++ ))
  done > "${outputFile}"
  return 0
}

stackTraceTree()
{
  if [[ $# -lt 1 ]]; then
    echo 'make stacktrace processing  in case of standard root crash log'
    echo 'input is a (list of) text files with the stack trace (either gdb aoutput'
    echo 'produced with e.g. gdb --batch --quiet -ex "bt" -ex "quit" aliroot core,'
    echo 'or the root crash log), output is a TTree formatted table.'
    echo 'example usage:'
    echo 'benchmark.sh stackTraceTree /foo/*/rec.log'
    echo 'benchmark.sh stackTraceTree $(cat file.list)'
    echo 'benchmark.sh stackTraceTree `cat file.list`'
    echo 'benchmark.sh stackTraceTree @file.list somefile.log'
    return 0
  fi
  parseListOfFiles "$@" | while IFS= read x; do echo "filename: $x"; cat "$x" 2>/dev/null; done | gawk '
       BEGIN {
       print "frame/I:method/C:line/C:cpass/I:aliroot/I:file/C";
               RS="#[0-9]*";
               aliroot=0;
               read=1;
             }
      /^filename:/ {filename=$2}
      /There was a crash/ {read=1;}
      /The lines below might hint at the cause of the crash/ {read=0;}
      read==1 {
               if ($3 ~ /Ali*/) aliroot=1; else aliroot=0;
               gsub("#","",RT);
               if ($NF!="" && RT!="" && $3!="") print RT" "$3" "$NF" "0" "aliroot" "filename
             }
      ' 2>/dev/null
}

plotStackTraceTree()
{
  #plot the stacktrace tree,
  #first arg    is the text file in the root tree format
  #second arg   is optional: a plot is written to file instead of screen
  #third arg    is optional: selection for plotting, default skip G_ stuff
  local tree=$1
  local plot=${2:-"crashes.png"}
  local selection=${3:-'!strstr(method,\"G__\")'}
  [[ ! -f ${tree} ]] && echo "plotStackTraceTree: no input file given" && return 1
  aliroot -b <<EOF
TTree* t=AliSysInfo::MakeTree("${tree}");
TCanvas* canvas = new TCanvas("QA crashes","QA crashes",1);
t->Draw("method","${selection}","");
canvas->SaveAs("${plot}");
.q
EOF
  return 0
}

#these functions encode strings to and from a space-less form
#use when spaces are not well handled (e.g. in arguments to
#commands in makeflow files, etc.
encSpaces()(echo "${1// /@@@@}")
decSpaces()(echo "${1//@@@@/ }")

get_realpath()
{
  if [[ $# -lt 1 ]]; then
    echo "print the full path of a file or directory, like \"readlink -f\" on linux"
    echo "Usage:"
    echo "  get_realpath <someFileOrDir>"
    return 0
  fi
  if [[ -f "$1" ]]
  then
    # file *must* exist
    if cd "$(echo "${1%/*}")" &>/dev/null
    then
      # file *may* not be local
      # exception is ./file.ext
      # try 'cd .; cd -;' *works!*
      local tmppwd="$PWD"
      cd - &>/dev/null
    else
      # file *must* be local
      local tmppwd="$PWD"
    fi
  elif [[ -d "$1" ]]; then
    if cd "$1" &>/dev/null; then
      local tmppwd="$PWD"
      cd - &>/dev/null
      echo "$tmppwd"
      return 0
    else
      return 1
    fi
  else
    # file *cannot* exist
    return 1 # failure
  fi
  # reassemble realpath
  echo "$tmppwd"/"${1##*/}"
  return 0 # success
}

printLogStatistics()
{
  #this function processes the summary logs and prints some stats
  #relies on the summary log format produced by summarizeLogs()
  # - how many checked logs in total
  # - number of each type of problem
  # example usage:
  #   printLogStatistics */*.log
  echo "log statistics from: ${1%/*}"
  parseListOfFiles "$@" | while IFS= read x; do cat "$x" 2>/dev/null; done | gawk '
  BEGIN {nOK=0; nCores=0; nStackTraces=0; nLogs=0;}
  /\/core/ {nCores++}
  /\/stacktrace.log/ {nStackTraces++}
  /OK/ {nOK++; nLogs++;}
  /BAD/ {
    nLogs++
    err=""
    write=0
    for (i=3; i<=NF; i++)
    {
      if ($i ~ /^\:$/)
        write=1
      else
        write=0

      if (write==0)
      {
        if (err=="") err=$i
        else err=(err FS $i)
      }

      if (err != "" && (write==1 || i==NF))
      {
        sumBAD[err]++
        err=""
      }
    }
  }
  END {
    print ("number of validated logs: " nOK" out of "nLogs )
    for (key in sumBAD)
    {
      print key": "sumBAD[key]
    }
    if (nCores>0 || nStackTraces>0) print "core files: "nCores", stack traces: "nStackTraces
  }
  '
}

createUniquePID()
{
  #create a unique ID for jobs running in parallel
  #consists of the ip address of the default network interface, PID,
  #if an argument is given, append it (e.g. a production ID)
  #the fields are space separated with a tag for easy parsing
  #spaces in the productionID will be encoded using encSpaces()
  local productionID=""
  [[ -n "${1}" ]] && productionID=$(encSpaces "${1}")
  local defaultIP=$(/sbin/route | awk '$1=="default" {print $8}' | xargs /sbin/ifconfig | awk '/inet / {print $2}' | sed 's/.*\([0-9]?\.[0-9]?\.[0-9]?\.[0-9]?\)/$1/')
  local id="ip:${defaultIP} pid:${BASHPID}"
  [[ -n "${productionID}" ]] && id+=" prod:${productionID}"
  echo "${id}"
}

printExec() {
  echo "[command] [PWD=$PWD]: $*" >&2
  "$@"
}

bigEcho() {
  # Outputs big text to stderr.
  which figlet &> /dev/null \
    && figlet -- "$@" >&2 \
    || ( printf "\n\n$*\n\n" | tr -d '[[:lower:]]' '[[:upper:]]' >&2 )
  true
}

listDir() (
  dir="$(cd "$1"; pwd)"
  echo ; echo "[listDir] Content of ${dir}${2:+" ($2)"}"
  find "$dir" -ls
  echo
)

mkdirLocal() (
  # Creates the given directories, with full path, only if local. If not local,
  # print a message and return success.
  # Return 1 if at least one error happened, 0 on success.
  err=0
  while [[ $# -gt 0 ]]; do
    dir=$1
    shift
    if [[ "${dir%%://*}" != "$dir" ]]; then
      echo "[mkdirLocal] skipping creation of $dir: it is not local"
      continue
    fi
    mkdir -p "$dir"
    [[ -d "$dir" ]]
    rv=$?
    err=$((err + ($rv & 1)))
    echo "[mkdirLocal] creation of dir $dir $([[ $rv == 0 ]] && echo "OK" || echo "FAILED")"
  done
  return $((err & 1))
)

statRemote() (
  # Check if file exists, whether it is local or remote. Returns 0 on success, 1 on failure.
  file=$1
  proto="${file%%://*}"
  [[ "$proto" == "$file" ]] && proto=local
  case "$proto" in
    local) [[ -f $file ]] ;;
    root)  path=${file:$((${#proto}+3))}
           host=${path%%/*}
           path=${path:$((${#host}))}
           while [[ ${path:0:2} == // ]]; do path=${path:1}; done
           xrd "$host" stat "$path" 2>&1 | grep -q Modtime: ;;
    *)     echo "[statRemote] for file $file: protocol not supported: $proto"
           return 2 ;;
  esac
  rv=$?
  echo "[statRemote] file $file (proto=${proto}$([[ $proto == local ]] && echo ", pwd=$PWD"))" \
       "$([[ $rv == 0 ]] && echo "exists" || echo "does NOT exist")"
  return $rv
)

lsRemote() (
  # List remote files on stdout. Returns 0 on success, 1 on failure.
  dir=$1
  proto="${dir%%://*}"
  [[ "$proto" == "$dir" ]] && proto=local
  case "$proto" in
    local) find "$dir" -maxdepth 1 -mindepth 1 ;;
    root)  path=${dir:$((${#proto}+3))}
           host=${path%%/*}
           path=${path:$((${#host}))}
           while [[ ${path:0:2} == // ]]; do path=${path:1}; done
           xrd "$host" ls "$path" 2>&1 | grep -v "No such file or directory" | grep -o "${path}.*" ;;
    *)     echo "[lsRemote] for directory $dir: protocol not supported: $proto" >&2
           return 2 ;;
  esac
  rv=$?
  [[ $rv != 0 ]] && echo "[lsRemote] cannot list directory $dir" >&2
  return $rv
)

copyFileFromRemote() (
  # Copy a list of remote files ($1, $2...${n-1}) to a certain dest dir ($n).
  # The last parameter must then be a local directory. Files can be local too.
  # Dest dir is created if not existing.
  # If only one parameter is specified download this file to the current directory.
  # If a source file is in the form @list.txt, then the list of files (one per
  # line) in list.txt will be expanded and copied.
  # On success 0 is returned - 1 otherwise.
  # Example: copyFileFromRemote root://localhost//file1.txt @list.txt /tmp/localdir/foo

  [[ $# == 1 ]] && dstdir=. || dstdir=${!#}
  maxCopyTries=${maxCopyTries-10}
  err=0
  opname="[copyFileFromRemote]"
  mkdir -p "$dstdir"

  while [[ $# -gt 1 ]]; do
    [[ ${1:0:1} == @ ]] && inputcmd="cat ${1:1}" || inputcmd="echo $1"
    while read -u 3 src; do
      thiserr=1
      proto="${src%%://*}"
      [[ "$proto" == "$src" ]] && proto=local
      dst="$dstdir/$(basename "$src")"
      echo "$opname (proto=$proto) started: $src -> $dst"
      for ((i=1; i<=maxCopyTries; i++)); do
        echo "$opname $src -> $dst attempt $i of $maxCopyTries"
        case "$proto" in
          local) printExec cp "$src" "$dst"
                 if [[ $? != 0 ]]; then
                   rm -f "$dst"
                   false
                 fi ;;
          root)  printExec xrdcp -f "$src" "$dst" ;;
          http*) printExec curl -LsSfo "$dst" "$src" ;;
          *)     echo "protocol not supported: $proto"
                 return 2 ;;
        esac
        if [[ $? == 0 ]]; then
          thiserr=0
          break
        fi
      done  # cp attempts
      [[ $thiserr == 0 ]] && echo "$opname (proto=$proto) OK after $i attempt(s): $src -> $dst" \
                          || echo "$opname (proto=$proto) FAILED after $maxCopyTries attempt(s): $src -> $dst"
      err=$((err+thiserr))
    done 3< <($inputcmd)
    shift
  done

  return $((err & 1))
)

copyFileToRemote() (
  # Copy a list of files ($1, $2...${n-1}) to a certain dest dir ($n). The last
  # parameter must then be a directory. The destination might be local or remote
  # if a protocol is specified (e.g. root://).
  # Dest dir is created if not existing (xrdcp does that already).
  # If a source file is in the form @list.txt, then the list of files (one per
  # line) in list.txt will be expanded and copied.
  # Shell glob is supported.
  # On success 0 is returned - 1 otherwise.
  # Example: copyFileToRemote file1.txt files* @list.txt test1 root://host/dir

  dstdir=${!#}
  maxCopyTries=${maxCopyTries-10}
  proto="${dstdir%%://*}"
  err=0
  [[ "$proto" == "$dstdir" ]] && proto=local
  opname="[copyFileToRemote] (proto=$proto)"

  while [[ $# -gt 1 ]]; do
    [[ ${1:0:1} == @ ]] && inputcmd="cat ${1:1}" || inputcmd="echo $1"
    while read -u 3 src; do
      thiserr=1
      dst="$dstdir/$(basename "$src")"
      for ((i=1; i<=maxCopyTries; i++)); do
        [[ -d "$src" ]] && echo "$opname $src -> $dst skipping, is a directory" && thiserr=0 && break
        # TODO use shopt -s nullglob
        [[ ! -r "$src" ]] && echo "$opname $src -> $dst skipping, cannot access source" && thiserr=0 && break
        echo "$opname $src -> $dst attempt $i of $maxCopyTries"
        case "$proto" in
          local) mkdir -p "$(dirname "$dst")"
                 printExec cp "$src" "$dst"
                 if [[ $? != 0 ]]; then
                   rm -f "$dst"
                   false
                 fi ;;
          root)  printExec xrdcp -f "$src" "$dst" ;;
          *)     echo "protocol not supported: $proto"
                 return 2 ;;
        esac
        if [[ $? == 0 ]]; then
          thiserr=0
          break
        fi
      done  # cp attempts
      [[ $thiserr == 0 ]] && echo "$opname OK after $i attempt(s): $src -> $dst" \
                          || echo "$opname FAILED after $maxCopyTries attempt(s): $src -> $dst"
      err=$((err+thiserr))
    done 3< <($inputcmd)
    shift
  done

  return $((err & 1))
)

paranoidCp()
(
  #recursively copy files and directories
  #if target is a directory - it must exist!
  #to avoid using find and the like as they kill
  #the performance on some cluster file systems
  #does not copy links to avoid problems
  sourceFiles=("${@}")
  destination="${sourceFiles[@]:(-1)}" #last element
   #check if we are not trying to copy to the same structure
  if [ $destination == $sourceFiles ] ; then
      echo paranoidCp INFO skip
      echo paranoidCp INFO destination== $destination
      echo paranoidCp INFO sourceFiles== $sourceFiles
      return 1;
  fi

  unset sourceFiles[${#sourceFiles[@]}-1] #remove last element (dst)
  #[[ ! -f "${destination}" ]]
  for src in "${sourceFiles[@]}"; do
    if [[ -f "${src}" && ! -h  "${src}" ]]; then
      paranoidCopyFile "${src}" "${destination}"
    elif [[ -d "${src}" && ! -h "${src}" ]]; then
      src="${src%/}"
      dst="${destination}/${src##*/}"
      mkdir -p "${dst}"
      paranoidCp "${src}"/* "${dst}"
    fi
  done
)

paranoidCopyFile()
(
  #copy a single file to a target in an existing dir
  #repeat a few times if copy fails
  #returns 1 on failure, 0 on success
  src=$(get_realpath "${1}")
  dst=$(get_realpath "${2}")
  [[ -d "${dst}" ]] && dst="${dst}/${src##*/}"
  #some sanity check
  [[ -z "${src}" ]] && echo "$1 does not exist" && return 1
  [[ -z "${dst}" ]] && echo "$2 does not exist" && return 1
  #check if we are not trying to copy to the same file
  [[ "${src}" == "${dst}" ]] && echo "$dst==$src, not copying" && return 0
  ok=0
  [[ -z "${maxCopyTries}" ]] && maxCopyTries=10

  echo "paranoid copy started: $src -> $dst"
  for (( i=1 ; i<=maxCopyTries ; i++ )) ; do

    echo "...attempt $i of $maxCopyTries"
    #rm -f "$dst"
    cp -a -n "$src" "$dst"

    cmp -s "$src" "$dst"
    if [ $? == 0 ] ; then
      ok=1
      break
    else
      rm -f "${dst}"
    fi

  done

  if [[ "$ok" == 1 ]] ; then
    echo "paranoid copy OK after $i attempt(s): $src -> $dst"
    return 0
  fi

  echo "paranoid copy FAILED after $maxCopyTries attempt(s): $src -> $dst"
  return 1
)

generPWD(){
  #
  # generate semirandom pwd using 2 keys
  # Example usage:
  # generPWD  myserviceaccount10 key11
  key0=$1
  key1=$2
  heslo0=`md5sum <<< "$key0 $key1" | cut -c 1-16`
  heslo=`echo $heslo0 | cut -c 1-8| awk '{print toupper($0)}'`
  heslo=$heslo`echo $heslo0 | cut -c 8-15| awk '{print tolower($0)}'`%
  echo $heslo;
}

reformatXMLCollection()
{
  #parse the xml collection on stdinput
  #output a collection in format: file md5 ctime size
  local nfields=""
  local turl=""
  local md5=""
  local ctime=""
  local size=""
  local x=""
  while read -a fields; do
    nfields=${#fields[*]}
    turl=""
    md5=""
    ctime=""
    size=""
    for ((x=1;x<=${nfields};x++)); do
      field=${fields[${x}]}
      if [[ "${field}" == "md5="* ]]; then
        eval ${field}
      fi
      if [[ "${field}" == "turl="* ]]; then
        eval ${field}
      fi
      if [[ "${field}" == "ctime="* ]]; then
        eval "${field} ${fields[((x+1))]}"
      fi
      if [[ "${field}" == "size="* ]]; then
        eval ${field}" "${fields[((x+1))]}
      fi
    done
    ctime=$( date -d "${ctime}" +%s 2>/dev/null)
    [[ -z $md5 ]] && md5="."
    [[ -n "$turl" ]] && echo "${turl//"alien://"/} ${md5} ${ctime} ${size}"
  done
}

#this makes debugging easier:
#executes the command given as an argument in this environment
#use case:
#  bashdb utilities.sh summarizeLogs * */*
[[ $# != 0 ]] && eval "$@" || true

# vi:syntax=zsh


Fixed a bug in obneplus.sh and added ability to create working directory.

remote: Support for password authentication was removed on August 13, 2021.
remote: Please see https://docs.github.com/en/get-started/getting-started-with-git/about-remote-repositories#cloning-with-https-urls for information on currently recommended modes of authentication.
fatal: Authentication failed for 'https://github.com/LA-EPFL/obndocker.git/'


*autoload: update-or-status does better condition to spare a fork

remote: Support for password authentication was removed on August 13, 2021.
remote: Please see https://docs.github.com/en/get-started/getting-started-with-git/about-remote-repositories#cloning-with-https-urls for information on currently recommended modes of authentication.
fatal: Authentication failed for 'https://github.com/psprint/zplugin.git/'


permitrootlogin script

permitrootlogin script

tmp=`sudo grep -e '( \t)PermitRootLogin( \t)no' /etc/ssh/sshd_config`
if [ $? ]
then
	echo 'OK'
else
	echo `sudo grep -e PermitRootLogin /etc/ssh/sshd_config`
fi

error: pathspec 'scripts/security/permitrootlogin.sh' did not match any file(s) known to git


Added app launch command

Added app launch command

cd /Users/vorg/Workspace/vorg-droptasks/droptasks/
open "http://localhost:3001"
node app.js

error: pathspec 'tasks.command' did not match any file(s) known to git


bug 629244 - run startSignatureProductdims

git-svn-id: 0429bb2eb6f71f8cd87fe9410a915008a087abca@2900 efcc4bdc-162a-0410-a8af-5b4b46c5376d

remote: Support for password authentication was removed on August 13, 2021.
remote: Please see https://docs.github.com/en/get-started/getting-started-with-git/about-remote-repositories#cloning-with-https-urls for information on currently recommended modes of authentication.
fatal: Authentication failed for 'https://github.com/twobraids/socorro.git/'


Add yarn path

Add yarn path

remote: Support for password authentication was removed on August 13, 2021.
remote: Please see https://docs.github.com/en/get-started/getting-started-with-git/about-remote-repositories#cloning-with-https-urls for information on currently recommended modes of authentication.
fatal: Authentication failed for 'https://github.com/partounian/dotfiles.git/'


[SYSTEMDS-3193] Promote svn repo `dev/systemds/2.x-rc` to `release/systemds/2.x`

Closes #1427.

#!/usr/bin/env bash
#-------------------------------------------------------------
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.
#
#-------------------------------------------------------------

################################################################################
##  File:  release-to-svn.sh
##  Desc:  Promote release candidate from svn dev/systemds to release/systemds
##  Note:  This file to be run only after the succesful voting
################################################################################

SELF=$(cd $(dirname $0) && pwd)

dry_run_flag=0
while getopts ":n" opt; do
  case $opt in
    n) dry_run_flag=1 ;;
    \?) error "Invalid option: $OPTARG" ;;
  esac
done

RELEASE_STAGING_LOCATION="https://dist.apache.org/repos/dist/dev/systemds"
RELEASE_LOCATION="https://dist.apache.org/repos/dist/release/systemds"

RELEASE_VERSION=
APPROVED_RELEASE_TAG=

ASF_USERNAME=

read -p "RELEASE_VERSION : " RELEASE_VERSION
read -p "APPROVED_RELEASE_TAG : " APPROVED_RELEASE_TAG

read -p "ASF_USERNAME : " ASF_USERNAME

tmp_repo=$(mktemp -d systemds-repo-tmp-XXXXX)

pushd "${tmp_repo}"

# 1. Checkout only the directory associated with approved release tag
svn co --depth=empty $RELEASE_STAGING_LOCATION svn-dev-systemds
cd svn-dev-systemds
svn update --set-depth files ${APPROVED_RELEASE_TAG}
cd ..

# 2.1. Checkout the empty repo, and copy the contents from svn dev
svn co --depth=empty $RELEASE_LOCATION svn-release-systemds
mkdir -p svn-release-systemds/$RELEASE_VERSION

cp svn-dev-systemds/${APPROVED_RELEASE_TAG}/systemds-* svn-release-systemds/$RELEASE_VERSION

# 2.2. Add the files to svn
svn add svn-release-systemds/$RELEASE_VERSION
cd svn-release-systemds

# 2.3. Commit and upload the files to the svn repository

if [[ $dry_run_flag != 1 ]]; then
  # This step prompts for the Apache Credentials
  svn ci --username $ASF_USERNAME -m'Apache SystemDS $RELEASE_VERSION Released' --no-auth-cache \n
  [[ $? == 0 ]] && printf '\n Publishing to $RELEASE_LOCATION is complete!\n'
else
  printf "\n==========\n"
  printf "This step would commit to the SVN release repo\n"
  printf "At $RELEASE_LOCATION \n"
  printf "\n==========\n"
  printf "You might want to manually check the files and run the following:\n"
  printf "svn ci --username $ASF_USERNAME -m'Apache SystemDS $RELEASE_VERSION Released' --no-auth-cache \n"
  printf "\n==========\n"
fi

error: pathspec 'dev/release/svn-staging-to-release.sh' did not match any file(s) known to git


Script to replace incorrect md5 checksums for interchange files

#!/bin/sh

############################################################
This will replace the md5 checksum for each interchange file
specified in the control file.
Usage:
> sh md5-replace.sh {path_to_control_file}
############################################################

if test -z $1
then
  echo "No control file specified."
  exit
fi

ctl_file=$1
path=`dirname $ctl_file`
touch $path/new_file.ctl

for line in `cat $ctl_file`
do
  IFS=","
  set $line
  in_checksum=$4
  file=$path/$3
  checksum=`md5 $file | cut -d = -f 2`
  out_checksum=${checksum:1:${#checksum}}
  if [ "$in_checksum" = "$out_checksum" ]
  then
    echo "     MD5 for $3 is correct."
    echo $1,$2,$3,$in_checksum >> $path/new_file.ctl
  else
    echo ">>>> MD5 for $3 is incorrect. Replacing..."
    echo $1,$2,$3,$out_checksum >> $path/new_file.ctl
  fi
done

echo "Backing up old control file...$ctl_file -> $ctl_file.backup"
mv $ctl_file $ctl_file.backup
echo "Writing new control file..."
mv $path/new_file.ctl $ctl_file
echo "Done."

error: pathspec 'sli/acceptance-tests/md5-replace.sh' did not match any file(s) known to git


add shell script driver

add shell script driver

#!/bin/sh

ARTIFACT=xsdwalker-1.0
JAR=$ARTIFACT.jar

java -cp target/$JAR XSDWalker "$@"

# eof

error: pathspec 'xsdwalker.sh' did not match any file(s) known to git


[+] maven build script

[+] maven build script

remote: Support for password authentication was removed on August 13, 2021.
remote: Please see https://docs.github.com/en/get-started/getting-started-with-git/about-remote-repositories#cloning-with-https-urls for information on currently recommended modes of authentication.
fatal: Authentication failed for 'https://github.com/ownport/docker-env.git/'


Allow ssh.

Allow ssh.

remote: Support for password authentication was removed on August 13, 2021.
remote: Please see https://docs.github.com/en/get-started/getting-started-with-git/about-remote-repositories#cloning-with-https-urls for information on currently recommended modes of authentication.
fatal: Authentication failed for 'https://github.com/demonchild2112/travis-test.git/'


[zsh] add path config file

[zsh] add path config file

#!/bin/zsh

export PATH="$HOME/.bin:$PATH"

error: pathspec 'zsh/path.zsh' did not match any file(s) known to git


xtrace dd if=/dev/null of="$q"

xtrace dd if=/dev/null of="$q"

remote: Support for password authentication was removed on August 13, 2021.
remote: Please see https://docs.github.com/en/get-started/getting-started-with-git/about-remote-repositories#cloning-with-https-urls for information on currently recommended modes of authentication.
fatal: Authentication failed for 'https://github.com/svnpenn/sage.git/'


Add haddock script

Add haddock script

Signed-off-by: Ricky Elrod <3de8762d49a778edd8b1aa9f381ea5a9ccb62944@elrod.me>

#!/usr/bin/env bash

cwd="$( cd "${BASH_SOURCE[0]%/*}" && pwd )"
cd "$cwd/.."
cabal haddock
f=`mktemp -d`
git clone git@github.com:relrod/-.git "$f/-.git"
pushd "$f/-.git"
  git checkout gh-pages && git rm -rf *
popd
mv dist/doc/html/dash/* "$f/-.git/"
pushd "$f/-.git"
  git add -A
  git commit -m "Manual docs deploy."
  git push origin gh-pages
popd
rm -rf "$f"

if [ $? == 0 ]; then
  echo "*** Done: http://relrod.github.io/-/"
  exit 0
else
  echo "*** ERROR!!! Fix the above and try again."
  exit 1
fi

error: pathspec 'scripts/deploy-haddock-manually.sh' did not match any file(s) known to git


Add new git-functions.lib.bash library

Add new git-functions.lib.bash library

function new_alias() {
  local alias_name=$1
  local alias_command=$2

  printf "%s\n" "New command: ${YELLOW}git ${BOLD}${alias_name}${NORMAL}"
  git config $scope alias.${alias_name} "${alias_command}"
}

error: pathspec 'scripts/lib/git-functions.lib.bash' did not match any file(s) known to git


Bug fix

Bug fix

remote: Support for password authentication was removed on August 13, 2021.
remote: Please see https://docs.github.com/en/get-started/getting-started-with-git/about-remote-repositories#cloning-with-https-urls for information on currently recommended modes of authentication.
fatal: Authentication failed for 'https://github.com/Eliot-Roxbergh/dotFIles.git/'


remove duplicates added

remove duplicates added

#!/bin/bash

# Script to prepare input files for SFrame analysis at ULB cluster. During its run it also checks the accessibility and readability of files and computes the total number of events.
# Prerequisities: 1) valid grid proxy (voms-proxy-init --voms cms:/cms/becms)  2) Set-up ROOT and SFrame environments

# Usage: source MakeSFInputFile.sh <directory with input ntuples> <name of output file with SFrame input>
# e.g.
# source MakeSFInputFile.sh /pnfs/iihe/cms/store/user/jez/ZHTo2TauAna_53x_HToTauTau-140 53x_HToTauTau-140
#
# The result would be 53x_HToTauTau-140_input.xml containit config snippet that can be included to SFrame config file

# Important parameters: names of ntuples (without suffix), server name and name of the tree inside ntuples

touch temp_input.1
rm -f temp_input.*

input_data=`echo $1`
output_name=`echo $2`
num_files=${3}
config_name=${4}

sframe_dir=$PWD
# input datasets settings
ntuple_name=output_Ntuples
srm_server=dcap://maite.iihe.ac.be
inTreeName=t

full_path=${srm_server}${input_data}/
touch full_path
rm -f full_path
echo "Full path to data is "${full_path}
echo ${full_path} >> full_path

touch ${output_name}_makeXML.sh
rm -f ${output_name}_*makeXML.sh

ls -1tr ${input_data} | grep root | sort > temp_input.1
total=`more temp_input.1 | wc -l`
echo "Total number of input root files is" $total
counter=0

touch temp_input.1.1
touch temp_input.1.2

touch toKeep.list
rm -f toKeep.list
touch toKeep.list

touch duplicates_${output_name}.list
rm -f duplicates_${output_name}.list
touch duplicates_${output_name}.list

#duplicate removal
for block in  `seq -s ' ' 1 ${total}`; do
    more temp_input.1 |  grep "_${block}_[0-9]\{1,\}_....root" > copies.list
    copies=`more copies.list | wc -l`
    toRemove=`expr $copies - 1`
    if [ $copies -gt 0 ]; then
      more copies.list | head -n ${toRemove} >> duplicates_${output_name}.list
      more copies.list | tail -n 1 >> toKeep.list
    fi
    rm -f copies.list
done

rm -f temp_input.1
mv toKeep.list temp_input.1

more temp_input.1 |while read line
do
  file=`echo $line`
  echo "dccp ${full_path}${file} /scratch/${output_name}_input/" >> temp_input.1.1
  echo "/scratch/${output_name}_input/${file}" >> temp_input.1.2
done

echo "The full paths to all files are:"

rm -f temp_input.1
mv temp_input.1.1 temp_input.1

touch SubmitAll_${output_name}.sh
rm -rf SubmitAll_${output_name}.sh
touch SubmitAll_${output_name}.sh

until [ $total -lt 1 ]
do
  echo "Total is" $total
  total=`expr $total - ${num_files}`
  echo "Subtracted total is" $total
  counter=`expr $counter + 1`
  head -n ${num_files} temp_input.1 > input_${counter}.1
  head -n ${num_files} temp_input.1.2 > input_${counter}.2
  tail -n $total temp_input.1 > temp_input_2_${counter}.1
  tail -n $total temp_input.1.2 > temp_input_2_${counter}.1.2
  rm -f temp_input.1
  mv temp_input_2_${counter}.1 temp_input.1
  rm -f temp_input.1.2
  mv temp_input_2_${counter}.1.2 temp_input.1.2
  mkdir ${output_name}_job${counter}
  cp script.sh ${output_name}_job${counter}/${output_name}_${counter}.sh
  echo "mkdir -p /scratch/${output_name}_input" >> ${output_name}_job${counter}/${output_name}_${counter}.sh
  cat  input_${counter}.1 >> ${output_name}_job${counter}/${output_name}_${counter}.sh
  rm -f input_${counter}.1
  more input_${counter}.2 | tr '\n' ' ' > temp_input.2
  rm -f input_${counter}.2
  echo "mkdir /scratch/${output_name}_${counter}_runDir" >> ${output_name}_job${counter}/${output_name}_${counter}.sh
  echo "cd /scratch/${output_name}_${counter}_runDir"    >> ${output_name}_job${counter}/${output_name}_${counter}.sh
  echo -n "sframe_input.py -r -x 1 -d -o input.xml -t ${inTreeName} " >> ${output_name}_job${counter}/${output_name}_${counter}.sh
  more temp_input.2 >> ${output_name}_job${counter}/${output_name}_${counter}.sh
  echo "" >> ${output_name}_job${counter}/${output_name}_${counter}.sh
  echo "cp ${sframe_dir}/JobConfig.dtd ${sframe_dir}/${config_name} ${sframe_dir}/*.root ." >> ${output_name}_job${counter}/${output_name}_${counter}.sh
  echo "ls -ltrh" >> ${output_name}_job${counter}/${output_name}_${counter}.sh
  echo "sframe_main ${config_name}" >> ${output_name}_job${counter}/${output_name}_${counter}.sh
  echo "cp Analysis.Data1.Reco.root ${sframe_dir}/${output_name}_job${counter}/" >> ${output_name}_job${counter}/${output_name}_${counter}.sh
  echo -n "rm -rf " >> ${output_name}_job${counter}/${output_name}_${counter}.sh
  more temp_input.2 >> ${output_name}_job${counter}/${output_name}_${counter}.sh
  echo "" >> ${output_name}_job${counter}/${output_name}_${counter}.sh
  echo "cd /scratch" >>  ${output_name}_job${counter}/${output_name}_${counter}.sh
  echo "rm -rf ${output_name}_${counter}_runDir" >> ${output_name}_job${counter}/${output_name}_${counter}.sh
  echo "echo \"(0) Wrapper finished successfully. Exit code 0\" " >> ${output_name}_job${counter}/${output_name}_${counter}.sh
  echo "cd ${output_name}_job${counter}" >> SubmitAll_${output_name}.sh
  echo "echo \"Submitting job no. ${counter}... \" " >> SubmitAll_${output_name}.sh
  echo "qsub -q localgrid@cream01 -o script.stdout -e script.stderr ${output_name}_${counter}.sh" >> SubmitAll_${output_name}.sh
  echo "cd .." >> SubmitAll_${output_name}.sh
done
rm -f temp_input*
rm -f full_path

chmod +x SubmitAll_${output_name}.sh
echo "To submit jobs, do ./SubmitAll_${output_name}.sh"
echo "Duplicate input files are in duplicates_${output_name}.list"

error: pathspec 'config/PrepareJobs_removeDuplicates.sh' did not match any file(s) known to git


updating to remove more stale files from doc builds

fatal: remote error: upload-pack: not our ref b253723e9012f92bdaf51da089f6dc604570705d
fatal: the remote end hung up unexpectedly


pacman search alias

pacman search alias

remote: Support for password authentication was removed on August 13, 2021.
remote: Please see https://docs.github.com/en/get-started/getting-started-with-git/about-remote-repositories#cloning-with-https-urls for information on currently recommended modes of authentication.
fatal: Authentication failed for 'https://github.com/gregflynn/dotsanity.git/'


Remove unused tools/rfc.sh

Remove unused tools/rfc.sh

This has been deprecated in favour of git-review for quite a while now
and was removed from Nova 10 months ago.

Change-Id: Ic4bfa1aee3a4b6a7588be3678ac033a1b13e2b18

#!/bin/sh -e
# Copyright (c) 2010-2011 Gluster, Inc. <http://www.gluster.com>
# This initial version of this file was taken from the source tree
# of GlusterFS. It was not directly attributed, but is assumed to be
# Copyright (c) 2010-2011 Gluster, Inc and release GPLv3
# Subsequent modifications are Copyright (c) 2011 OpenStack, LLC.
#
# GlusterFS is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published
# by the Free Software Foundation; either version 3 of the License,
# or (at your option) any later version.
#
# GlusterFS is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program.  If not, see
# <http://www.gnu.org/licenses/>.

branch="master";

set_hooks_commit_msg()
{
    top_dir=`git rev-parse --show-toplevel`
    f="${top_dir}/.git/hooks/commit-msg";
    u="https://review.openstack.org/tools/hooks/commit-msg";

    if [ -x "$f" ]; then
        return;
    fi

    curl -o $f $u || wget -O $f $u;

    chmod +x $f;

    GIT_EDITOR=true git commit --amend
}

add_remote()
{
    username=$1
    project=$2

    echo "No remote set, testing ssh://$username@review.openstack.org:29418"
    if project_list=`ssh -p29418 -o StrictHostKeyChecking=no $username@review.openstack.org gerrit ls-projects 2>/dev/null`
    then
        echo "$username@review.openstack.org:29418 worked."
        if echo $project_list | grep $project >/dev/null
        then
            echo "Creating a git remote called gerrit that maps to:"
            echo "  ssh://$username@review.openstack.org:29418/$project"
            git remote add gerrit ssh://$username@review.openstack.org:29418/$project
        else
            echo "The current project name, $project, is not a known project."
            echo "Please either reclone from github/gerrit or create a"
            echo "remote named gerrit that points to the intended project."
            return 1
        fi

        return 0
    fi
    return 1
}

check_remote()
{
    if ! git remote | grep gerrit >/dev/null 2>&1
    then
        origin_project=`git remote show origin | grep 'Fetch URL' | perl -nle '@fields = split(m|[:/]|); $len = $#fields; print $fields[$len-1], "/", $fields[$len];'`
        if add_remote $USERNAME $origin_project
        then
            return 0
        else
            echo "Your local name doesn't work on Gerrit."
            echo -n "Enter Gerrit username (same as launchpad): "
            read gerrit_user
            if add_remote $gerrit_user $origin_project
            then
                return 0
            else
                echo "Can't infer where gerrit is - please set a remote named"
                echo "gerrit manually and then try again."
                echo
                echo "For more information, please see:"
                echo "\thttp://wiki.openstack.org/GerritWorkflow"
                exit 1
            fi
        fi
    fi
}

rebase_changes()
{
    git fetch;

    GIT_EDITOR=true git rebase -i origin/$branch || exit $?;
}

assert_diverge()
{
    if ! git diff origin/$branch..HEAD | grep -q .
    then
	echo "No changes between the current branch and origin/$branch."
	exit 1
    fi
}

main()
{
    set_hooks_commit_msg;

    check_remote;

    rebase_changes;

    assert_diverge;

    bug=$(git show --format='%s %b' | perl -nle 'if (/\b([Bb]ug|[Ll][Pp])\s*[#:]?\s*(\d+)/) {print "$2"; exit}')

    bp=$(git show --format='%s %b' | perl -nle 'if (/\b([Bb]lue[Pp]rint|[Bb][Pp])\s*[#:]?\s*([0-9a-zA-Z-_]+)/) {print "$2"; exit}')

    if [ "$DRY_RUN" = 1 ]; then
        drier='echo -e Please use the following command to send your commits to review:\n\n'
    else
        drier=
    fi

    local_branch=`git branch | grep -Ei "\* (.*)" | cut -f2 -d' '`
    if [ -z "$bug" ]; then
	if [ -z "$bp" ]; then
            $drier git push gerrit HEAD:refs/for/$branch/$local_branch;
	else
	    $drier git push gerrit HEAD:refs/for/$branch/bp/$bp;
	fi
    else
        $drier git push gerrit HEAD:refs/for/$branch/bug/$bug;
    fi
}

main "$@"


Added the update.sh shell script to uninstall and then install the current version of noseyparker on GCE. Dependent packages are updated if required.

#!/bin/sh

set -e

# Directory paths used for nogotofail.
INSTALL_DIR=/opt/nogotofail
CONFIG_DIR=/etc/nogotofail
LOG_DIR=/var/log/nogotofail

# Stop the nogotofail-mitm and other associated services if they're running.
sudo /etc/init.d/nogotofail-mitm stop
sudo /etc/init.d/dnsmasq stop
sudo /etc/init.d/openvpn stop

# Remove compiled python libraries i.e. *.pyc files.
# TODO: Find a more elegant method for uninstalling a Python program.
#rm -rf $INSTALL_DIR
#rm -rf $CONFIG_DIR
#rm -rf $LOG_DIR
find $INSTALL_DIR -type f -name '*.pyc' -delete

# Install toolchain dependencies
sudo apt-get update
sudo apt-get -y upgrade
#sudo apt-get -y install patch make gcc libssl-dev python-openssl liblzo2-dev libpam-dev

# Install OpenVPN and dnsmasq
#sudo apt-get -y install openvpn dnsmasq

# Build and install a patched version of OpenVPN.
# This is needed because the OpenVPN 2.3.x still does not properly handle
# floating clients (those whose source IP address as seen by the server changes
# from time to time) which is a regular occurrence in the mobile world.
# OpenVPN 2.4 might ship with proper support out of the box. In that case, this
# kludge can be removed.
#./build_openvpn.sh

# Build and install a patched version of dnsmasq.
# This is needed because GCE does not support IPv6. We thus blackhole IPv6
# traffic from clients so that they are forced to use IPv4. However, default
# DNS servers will still resolve hostnames to IPv6 addresses causing clients to
# attempt IPv6. To avoid clients attempting IPv6, we run a patched dnsmasq DNS
# server which empties AAAA records thus causing clients to go for A records
# which provide IPv4 addresses.
#./build_dnsmasq.sh

# Set up OpenVPN server
#sudo ./setup_openvpn.sh

# Set up the MiTM daemon
sudo ./setup_mitm.sh

# Restart all the relevant daemons
sudo /etc/init.d/dnsmasq start
sudo /etc/init.d/openvpn start
#sudo /etc/init.d/nogotofail-mitm stop || true
sudo /etc/init.d/nogotofail-mitm start

error: pathspec 'docs/gce/update.sh' did not match any file(s) known to git


replace {curl,w3m} with {wget,sed,perl}

replace {curl,w3m} with {wget,sed,perl}

remote: Support for password authentication was removed on August 13, 2021.
remote: Please see https://docs.github.com/en/get-started/getting-started-with-git/about-remote-repositories#cloning-with-https-urls for information on currently recommended modes of authentication.
fatal: Authentication failed for 'https://github.com/carl-di-ortus/bin.git/'


Try to ignore weak auth key
Use SHA256 for auth
Use 256 keysize

Signed-off-by: bouroo <84b7a42037b9b1d0c88115cedda61c982b57a725@gmail.com>

remote: Support for password authentication was removed on August 13, 2021.
remote: Please see https://docs.github.com/en/get-started/getting-started-with-git/about-remote-repositories#cloning-with-https-urls for information on currently recommended modes of authentication.
fatal: Authentication failed for 'https://github.com/kryptophonik/openvpn-install.git/'


Added script file

Added script file

#!/bin/sh

echo "SSH_Private_Key_Login 2.0 by Andreas Prang"
echo ""

default_server_user="root"
default_public_key_path=""

function installNewKeys {
    read -p "Please enter target ssh server (FQN or IP w/o port!): " server_name
    read -p "Please enter target ssh server port (22): " server_port
    server_port=${server_port:-22}

    read -p "Please enter user on target ssh server ($default_server_user): " server_user
    server_user=${server_user:-$default_server_user}

    read -p "Please enter a local path where to save keys (~/Documents/myKeys/$server_name""_$server_port/): " local_key_path
    local_key_path=${local_key_path:-~/Documents/myKeys/$server_name"_$server_port"/}

    echo "-----------------------------------------------------------------"

    # check for slash at the end
    if [ "${local_key_path:LEN - 1}" != "/" ]; then
      echo "Adding trailing slash"
      local_key_path=$local_key_path"/"
    fi

    echo "Creating Path "$local_key_path
    mkdir -p $local_key_path

    echo "Creating Certificates"
    ssh-keygen -t dsa -f $local_key_path$server_name

    echo "-----------------------------------------------------------------"
    echo "Copy public key to "$server_user"@"$server_name":"$server_port
    scp -P $server_port $local_key_path""$server_name".pub" $server_user@""$server_name":~/"

    echo "-----------------------------------------------------------------"

    echo "Setup key on "$server_user"@"$server_name":"$server_port
    ssh -p $server_port $server_user@$server_name "mkdir -p ~/.ssh/;touch ~/.ssh/authorized_keys2;cat ~/"$server_name".pub >> ~/.ssh/authorized_keys2;rm ~/"$server_name".pub"

    echo "Add private key to known hosts"
    mkdir -p ~/.ssh
    touch ~/.ssh/known_hosts
    cat $local_key_path""$server_name >> ~/.ssh/known_hosts

    echo "Register private key on local machine"
    ssh-add $local_key_path""$server_name
}

function installExistingKey {
    read -p "Please enter target ssh server (FQN or IP w/o port!): " server_name
    read -p "Please enter target ssh server port (22): " server_port
    read -p "Please enter path to public key file ($public_key_path):" public_key_path
    public_key_path=${public_key_path:-"$default_public_key_path"}
    public_key_path=$(find $public_key_path)
    ! [ -e "$public_key_path" ] && echo "Public key not found! Do not use paths starting with: '~'" && exit 0
    default_public_key_path="$public_key_path"

    server_port=${server_port:-22}
    public_key_file=$(basename $public_key_path)

    read -p "Please enter user on target ssh server ($default_server_user): " server_user
    server_user=${server_user:-$default_server_user}
    default_server_user=$server_user

    echo "-----------------------------------------------------------------"
    echo "Copy public key to "$server_user"@"$server_name":"$server_port
    scp -P $server_port $public_key_path $server_user@""$server_name":~/"

    echo "-----------------------------------------------------------------"

    echo "Setup key on "$server_user"@"$server_name":"$server_port
    ssh -p $server_port $server_user@$server_name "mkdir -p ~/.ssh/;touch ~/.ssh/authorized_keys2;cat ~/"$public_key_file" >> ~/.ssh/authorized_keys2;rm ~/"$public_key_file
    echo "Done."

    echo "-----------------------------------------------------------------"
}

function registerKey {
  read -p "Please enter path to private key file:" private_key_path
  private_key_path="${private_key_path%????}"
  if [ ! -f $private_key_path ]; then
    echo "Private key not found!"
    exit 1
  fi
  mkdir -p ~/.ssh
  touch ~/.ssh/known_hosts
  cat $private_key_path >> ~/.ssh/known_hosts

  echo "Register private key on local machine"
  ssh-add $private_key_path
}

while true; do
  echo "1) Install new public/private ssh key"
  echo "2) Install existing public key on remote server"
  echo "3) Register private key locally"
  echo "x) Exit"
  read -p ": " yn
  case $yn in
    [1]* ) installNewKeys;;
    [2]* ) installExistingKey;;
    [3]* ) registerKey;;
    * ) echo "Bye..."; break;;
  esac
done

error: pathspec 'SSH_Public_Private_Key_Loin_Generator.sh' did not match any file(s) known to git


Fix typo

Fix typo

remote: Support for password authentication was removed on August 13, 2021.
remote: Please see https://docs.github.com/en/get-started/getting-started-with-git/about-remote-repositories#cloning-with-https-urls for information on currently recommended modes of authentication.
fatal: Authentication failed for 'https://github.com/ascoderu/opwen-webapp.git/'


Simple script to run tcpdump and our filter/converter script

#! /bin/sh

if [ $# -ne 1 ]; then
  echo "Syntax: $0 interface" >&2
  exit 1
fi

# make a temporary directory
WORK_DIR=`mktemp -d`

function cleanup {
  rm -rf "$WORK_DIR"
}

trap cleanup EXIT

# make a FIFO (named pipe) to push pcap data down
mkfifo $WORK_DIR/pipe

# start capturing data in a background process
(
  while true; do
    tcpdump -i $1 -s0 -w $WORK_DIR/pipe -U port 53
  done
) &

CAPTURE_PID=$!

function shutdown_capture {
  rm -rf "$WORK_DIR"
  kill $CAPTURE_PID
}

trap shutdown_capture EXIT

# now just hang out
while true; do
  ./pcap2ymmv $WORK_DIR/pipe >> ymmv.dat
done

error: pathspec 'pcap2ymmv/capture.sh' did not match any file(s) known to git


Add timeout.

Add timeout.


Add license header to functions bash script

# Copyright (C) 2016 Google Inc., authors, and contributors <see AUTHORS file>
# Licensed under http://www.apache.org/licenses/LICENSE-2.0 <see LICENSE file>
# Created By: miha@reciprocitylabs.com
# Maintained By: miha@reciprocitylabs.com

set -o nounset
set -o errexit

setup () {
  if [ -z "${1:-}" ]; then
    echo "Missing mandatory project parameter"
    exit 1
  else
    PROJECT="${1}"
  fi

	docker-compose --file docker-compose-testing.yml \
		--project-name ${PROJECT} \
		up --force-recreate -d

	if [[ $UID -eq 0 ]]; then
		# This fixes permissions with bindfs only on jenkins.
		chown 1000:1000 -R .
	elif [[ $UID -eq 1000 ]]; then
		docker exec -i ${PROJECT}_dev_1 sh -c "
		  bindfs /vagrant_bind /vagrant --map=$UID/1000 -o nonempty
		"
	fi

	echo "Provisioning ${PROJECT}_dev_1"
	docker exec -i ${PROJECT}_dev_1 su vagrant -c "
		ansible-playbook -i provision/docker/inventory provision/site.yml
	"
}

teardown () {
  if [ -z "${1:-}" ]; then
    echo "Missing mandatory project parameter"
    exit 1
  else
    PROJECT="${1}"
  fi

	docker exec -i ${PROJECT}_dev_1 sh -c "chown $UID -R /vagrant"
	docker exec -i ${PROJECT}_selenium_1 sh -c "chown $UID -R /selenium"

	docker-compose -p ${PROJECT} stop
}


Use a symlink for the inventory directory Prevents accidental "resets" of the inventory or group_vars

remote: Support for password authentication was removed on August 13, 2021.
remote: Please see https://docs.github.com/en/get-started/getting-started-with-git/about-remote-repositories#cloning-with-https-urls for information on currently recommended modes of authentication.
fatal: Authentication failed for 'https://github.com/galexrt/kubernetes-ansible-deployment.git/'


update

update

remote: Support for password authentication was removed on August 13, 2021.
remote: Please see https://docs.github.com/en/get-started/getting-started-with-git/about-remote-repositories#cloning-with-https-urls for information on currently recommended modes of authentication.
fatal: Authentication failed for 'https://github.com/neurobin/test.git/'


remove old cruft no longer necessary with git plugin

#!/bin/bash

# Helper functions
function abort_on_error () {
	if [[ $? -ne "0" ]]; then
		echo FAILED: $1
		exit 1
	fi
}

# build
cd $(dirname $(git rev-parse --git-dir))
python setup.py install
abort_on_error "Failed to build via setup.py"
echo "Build via setup.py passed"

# make sure shared libraries are linked
# FIXME; figure out why jenkins can't see shared libraries without this
cd $(dirname $(git rev-parse --git-dir))
cd crosscat/cython_code/
ln -s /usr/local/lib/python2.7/dist-packages/crosscat/cython_code/State.so || true
ln -s /usr/local/lib/python2.7/dist-packages/crosscat/cython_code/MultinomialComponentModel.so || true
ln -s /usr/local/lib/python2.7/dist-packages/crosscat/cython_code/ContinuousComponentModel.so || true

# run the tests
cd $(dirname $(git rev-parse --git-dir))
cd crosscat/tests/unit_tests
nosetests --with-xunit
exit

# Build and run tests. WORKSPACE is set by jenkins to /var/
# export PYTHONPATH=$PYTHONPATH:$WORKSPACE
# cd $WORKSPACE/$project_name
# make tests
# make cython
# cd $project_name/tests
# python /usr/bin/nosetests --with-xunit cpp_unit_tests.py cpp_long_tests.py test_sampler_enumeration.py

#!/bin/bash

# default values
branch="master"
user="jenkins"
home_dir="/var/lib/jenkins/"
project_name="crosscat"

# print script usage
usage() {
    cat <<EOF
usage: $0 options

    Pull down repo from github and run tests via jenkins

    OPTIONS:
    -h      Show this message
    -p	    project name (default=$project_name)
    -b      set the branch to run (default=$branch)
    -u      set the user to install virtualenv under (default=$user)
    -h      set the dir to install virtualenv under (default=$home_dir)
EOF
exit
}

#Process command line arguments
while getopts hp:b:u:d: opt
do
    case "$opt" in
        h) usage;;
        p) project_name=$OPTARG;;
        b) branch=$OPTARG;;
	u) user=$OPTARG;;
	d) home_dir=$OPTARG;;
    esac
done

echo "jenkins_script.sh:"
echo "project_name: $project_name"
echo "user: $user"
echo "branch: $branch"
echo "home_dir: $home_dir"

cd $(dirname $(git rev-parse --git-dir))
python setup.py install
if [ $? != 0 ]; then
	exit 1
else
	echo "Build via setup.py passed"
fi
#
cd $(dirname $(git rev-parse --git-dir))
cd crosscat/cython_code/
ln -s /usr/local/lib/python2.7/dist-packages/crosscat/cython_code/State.so || true
ln -s /usr/local/lib/python2.7/dist-packages/crosscat/cython_code/MultinomialComponentModel.so || true
ln -s /usr/local/lib/python2.7/dist-packages/crosscat/cython_code/ContinuousComponentModel.so || true
#
cd $(dirname $(git rev-parse --git-dir))
cd crosscat/tests/unit_tests
nosetests --with-xunit
exit

# Build and run tests. WORKSPACE is set by jenkins to /var/
# export PYTHONPATH=$PYTHONPATH:$WORKSPACE
# cd $WORKSPACE/$project_name
# make tests
# make cython
# cd $project_name/tests
# python /usr/bin/nosetests --with-xunit cpp_unit_tests.py cpp_long_tests.py test_sampler_enumeration.py


fix test for failure

fix test for failure

#!/bin/bash

# default values
branch="master"
user="jenkins"
home_dir="/var/lib/jenkins/"
project_name="crosscat"

# print script usage
usage() {
    cat <<EOF
usage: $0 options

    Pull down repo from github and run tests via jenkins

    OPTIONS:
    -h      Show this message
    -p	    project name (default=$project_name)
    -b      set the branch to run (default=$branch)
    -u      set the user to install virtualenv under (default=$user)
    -h      set the dir to install virtualenv under (default=$home_dir)
EOF
exit
}

#Process command line arguments
while getopts hp:b:u:d: opt
do
    case "$opt" in
        h) usage;;
        p) project_name=$OPTARG;;
        b) branch=$OPTARG;;
	u) user=$OPTARG;;
	d) home_dir=$OPTARG;;
    esac
done

echo "jenkins_script.sh:"
echo "project_name: $project_name"
echo "user: $user"
echo "branch: $branch"
echo "home_dir: $home_dir"

cd $(dirname $(git rev-parse --git-dir))
python setup.py install
if [ $? != 0 ]; then
	exit 1
else
	echo "Build via setup.py passed"
fi
#
cd $(dirname $(git rev-parse --git-dir))
cd crosscat/cython_code/
ln -s /usr/local/lib/python2.7/dist-packages/crosscat/cython_code/State.so || true
ln -s /usr/local/lib/python2.7/dist-packages/crosscat/cython_code/MultinomialComponentModel.so || true
ln -s /usr/local/lib/python2.7/dist-packages/crosscat/cython_code/ContinuousComponentModel.so || true
#
cd $(dirname $(git rev-parse --git-dir))
cd crosscat/tests/unit_tests
nosetests --with-xunit
exit

# Remove old source, and checkout newest source from master.
source /var/lib/jenkins/.bashrc
rm -rf $project_name
git clone --depth=1 https://github.com/mit-probabilistic-computing-project/$project_name.git $project_name --branch $branch
cd $project_name

# If the virtualenv isn't set up, do that.
if [ ! -e /var/lib/jenkins/.virtualenvs ]
then
  bash -i install_scripts/virtualenv_setup.sh $user $home_dir >virtualenv.out 2>virtualenv.err
  source /var/lib/jenkins/.bashrc
fi
workon $project_name

# Build and run tests. WORKSPACE is set by jenkins to /var/
export PYTHONPATH=$PYTHONPATH:$WORKSPACE
cd $WORKSPACE/$project_name
make tests
make cython
cd $project_name/tests
python /usr/bin/nosetests --with-xunit cpp_unit_tests.py cpp_long_tests.py test_sampler_enumeration.py

#!/bin/bash

# default values
branch="master"
user="jenkins"
home_dir="/var/lib/jenkins/"
project_name="crosscat"

# print script usage
usage() {
    cat <<EOF
usage: $0 options

    Pull down repo from github and run tests via jenkins

    OPTIONS:
    -h      Show this message
    -p	    project name (default=$project_name)
    -b      set the branch to run (default=$branch)
    -u      set the user to install virtualenv under (default=$user)
    -h      set the dir to install virtualenv under (default=$home_dir)
EOF
exit
}

#Process command line arguments
while getopts hp:b:u:d: opt
do
    case "$opt" in
        h) usage;;
        p) project_name=$OPTARG;;
        b) branch=$OPTARG;;
	u) user=$OPTARG;;
	d) home_dir=$OPTARG;;
    esac
done

echo "jenkins_script.sh:"
echo "project_name: $project_name"
echo "user: $user"
echo "branch: $branch"
echo "home_dir: $home_dir"

cd $(dirname $(git rev-parse --git-dir))
python setup.py install
if [ $? ]; then
	exit 1
else
	echo "Build via setup.py passed"
fi
#
cd $(dirname $(git rev-parse --git-dir))
cd crosscat/cython_code/
ln -s /usr/local/lib/python2.7/dist-packages/crosscat/cython_code/State.so || true
ln -s /usr/local/lib/python2.7/dist-packages/crosscat/cython_code/MultinomialComponentModel.so || true
ln -s /usr/local/lib/python2.7/dist-packages/crosscat/cython_code/ContinuousComponentModel.so || true
#
cd $(dirname $(git rev-parse --git-dir))
cd crosscat/tests/unit_tests
nosetests --with-xunit
exit

# Remove old source, and checkout newest source from master.
source /var/lib/jenkins/.bashrc
rm -rf $project_name
git clone --depth=1 https://github.com/mit-probabilistic-computing-project/$project_name.git $project_name --branch $branch
cd $project_name

# If the virtualenv isn't set up, do that.
if [ ! -e /var/lib/jenkins/.virtualenvs ]
then
  bash -i install_scripts/virtualenv_setup.sh $user $home_dir >virtualenv.out 2>virtualenv.err
  source /var/lib/jenkins/.bashrc
fi
workon $project_name

# Build and run tests. WORKSPACE is set by jenkins to /var/
export PYTHONPATH=$PYTHONPATH:$WORKSPACE
cd $WORKSPACE/$project_name
make tests
make cython
cd $project_name/tests
python /usr/bin/nosetests --with-xunit cpp_unit_tests.py cpp_long_tests.py test_sampler_enumeration.py