FIX improved nc-based aliveness check in accumulatorStart()

# Copyright 2013 Telefonica Investigacion y Desarrollo, S.A.U
#
# This file is part of Orion Context Broker.
#
# Orion Context Broker is free software: you can redistribute it and/or
# modify it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.
#
# Orion Context Broker is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero
# General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with Orion Context Broker. If not, see http://www.gnu.org/licenses/.
#
# For those usages not covered by this license please contact with
# iot_support at tid dot es

# ------------------------------------------------------------------------------
#
# harnessInit -
#

if [ "$CONTEXTBROKER_TESTENV_SOURCED" != "YES" ]
then
  echo
  echo '---------------------------------------------------'
  echo "Test Environment missing - please source testEnv.sh"
  echo '---------------------------------------------------'
  echo
  exit 1
fi

export CONTEXTBROKER_HARNESS_FUNCTIONS_SOURCED="YES"

if [ "$ORION_FT_DEBUG" == "1" ]
then
  _debug='on'
else
  _debug='off'
fi

# ------------------------------------------------------------------------------
#
# harnessExit -
#
function harnessExit()
{
  rm -f headers.out
}

# ------------------------------------------------------------------------------
#
# vMsg - verbose output
#
# Remember that if the '--verbose' flag is used, the verboser messages pollute
# the total answer and the harness test will fail.
#
# Verbose mode is only meant to troubleshoot harness tests that fail.
#
function vMsg()
{
  if [ "$_verbose" == "on" ]
  then
    echo $*
  fi
}

# ------------------------------------------------------------------------------
#
# dMsg - debug output
#
function dMsg()
{
  if [ "$ORION_FT_DEBUG" == 1 ] || [ "$_debug" == "on"  ]
  then
    echo $* >> /tmp/orionFuncTestDebug.log
  fi
}

# ------------------------------------------------------------------------------
#
# dbInit -
#
# ------------------------------------------------------------------------------
#
# dbInit -
#
function dbInit()
{
  role=$1
  tenant=$2

  host="${CB_DATABASE_HOST}"
  if [ "$host" == "" ]
  then
    host="localhost"
  fi

  port="${CB_DATABASE_PORT}"
  if [ "$port" == "" ]
  then
    port="27017"
  fi

  dMsg initializing database;

  if [ "$role" == "CB" ]
  then
    baseDbName=${CB_DB_NAME}
  elif [ "$role" == "CP1" ]
  then
    baseDbName=${CP1_DB_NAME}
  elif [ "$role" == "CP2" ]
  then
    baseDbName=${CP2_DB_NAME}
  elif [ "$role" == "CP3" ]
  then
    baseDbName=${CP3_DB_NAME}
  elif [ "$role" == "CP4" ]
  then
    baseDbName=${CP4_DB_NAME}
  elif [ "$role" == "CP5" ]
  then
    baseDbName=${CP5_DB_NAME}
  else
    baseDbName=$1
  fi

  # If a tenant was provided, then we build the tenant DB, e.g. orion-ftest1
  if [ $# == 2 ]
  then
    db=$baseDbName-$tenant
  else
    db=$baseDbName
  fi

  dMsg "database to drop: <$db>"
  echo 'db.dropDatabase()' | mongo $host:$port/$db --quiet
}

# ------------------------------------------------------------------------------
#
# dbDrop -
#
function dbDrop()
{
  if [ "$CB_DB_DROP" != "No" ]
  then
    # FIXME: Not sure if $@ should be better...
    dbInit $*
  fi
}

# -----------------------------------------------------------------------------
#
# dbList
#
function dbList
{
  name="$1"

  host="${CB_DATABASE_HOST}"
  if [ "$host" == "" ]
  then
    host="localhost"
  fi

  port="${CB_DATABASE_PORT}"
  if [ "$port" == "" ]
  then
    port="27017"
  fi

  if [ "$name" != "" ]
  then
    echo show dbs | mongo $host:$port --quiet | grep "$name" | awk '{ print $1 }'
  else
    echo show dbs | mongo $host:$port --quiet | awk '{ print $1 }'
  fi
}

# ------------------------------------------------------------------------------
#
# dbResetAll -
#
function dbResetAll()
{
  host="${CB_DATABASE_HOST}"
  if [ "$host" == "" ]
  then
    host="localhost"
  fi

  port="${CB_DATABASE_PORT}"
  if [ "$port" == "" ]
  then
    port="27017"
  fi

  all=$(echo show dbs | mongo $host:$port --quiet | grep ftest | awk '{ print $1 }')
  for db in $all
  do
    dbDrop $db
  done
}

# ------------------------------------------------------------------------------
#
# brokerStopAwait
#
function brokerStopAwait
{
  port=$1

  typeset -i loopNo
  typeset -i loops
  loopNo=0
  loops=50

  while [ $loopNo -lt $loops ]
  do
    nc -w 2 localhost $port >/dev/null </dev/null
    if [ "$?" != "0" ]
    then
      vMsg The orion context broker on port $port has stopped
      sleep 1
      break;
    fi

    vMsg Awaiting orion context broker to fully stop '('$loopNo')' ...
    sleep .2
    loopNo=$loopNo+1
  done

  sleep .5

  # Check CB NOT running fine
  curl -s localhost:${port}/version | grep version > /dev/null
  result=$?
  if [ "$result" == "0" ]
  then
    result=1  # ERROR - the broker is still running
  else
    result=0
  fi
}

# ------------------------------------------------------------------------------
#
# brokerStartAwait
#
# For some really strange reason, all functests fail under valgrind when executed by Jenkins
# without the line:
#
#   echo "Broker started after $loopNo checks" >> /tmp/brokerStartCounter
#
# It also works with echo to /dev/null, but using a file we get some more information.
#
# As we only append (>>) to /tmp/brokerStartCounter in this file, /tmp/brokerStartCounter
# is reset in testHarness.sh, using the command "date > tmp/brokerStartCounter".
#
# The reason we send information to a file is that in case the 100 loop is not enough in the future,
# we will have that information in the file /tmp/brokerStartCounter
#
function brokerStartAwait
{
  if [ "$BROKER_AWAIT_SLEEP_TIME" != "" ]
  then
    sleep $BROKER_AWAIT_SLEEP_TIME
    result=0
    return
  fi

  port=$1

  typeset -i loopNo
  typeset -i loops
  loopNo=0
  loops=100

  while [ $loopNo -lt $loops ]
  do
    nc -w 2 localhost $port >/dev/null </dev/null
    if [ "$?" == "0" ]
    then
      vMsg The orion context broker has started, listening on port $port
      echo "Broker started after $loopNo checks" >> /tmp/brokerStartCounter
      sleep 1
      break;
    fi

    vMsg Awaiting valgrind to fully start the orion context broker '('$loopNo')' ...
    sleep .2
    loopNo=$loopNo+1
  done

  sleep .5

  # Check that CB started
  curl -s localhost:${port}/version | grep version >> /tmp/brokerStartCounter
  result=$?
  echo "result: $result" >> /tmp/brokerStartCounter
}

# ------------------------------------------------------------------------------
#
# localBrokerStart
#
function localBrokerStart()
{
  role=$1
  traceLevels=$2
  ipVersion=$3

  POOL_SIZE=${POOL_SIZE:-10}

  shift
  shift
  shift

  if [ "$traceLevels" != "" ]
  then
    extraParams="-logLevel DEBUG "$*
  else
    extraParams=$*
  fi

  IPvOption=""

  dbHost="${CB_DATABASE_HOST}"
  if [ "$dbHost" == "" ]
  then
    dbHost="localhost"
  fi

  dbPort="${CB_DATABASE_PORT}"
  if [ "$dbPort" == "" ]
  then
    dbPort="27017"
  fi

  dMsg starting broker for role $role

  if [ "$ipVersion" == "IPV4" ]
  then
    IPvOption="-ipv4"
  elif [ "$ipVersion" == "IPV6" ]
  then
    IPvOption="-ipv6"
  fi

  # FIXME P10: eventually the -paranoidV1Indent flag has to be removed
  CB_START_CMD_PREFIX="contextBroker -harakiri -paranoidV1Indent"
  if [ "$role" == "CB" ]
  then
    port=$CB_PORT
    CB_START_CMD="$CB_START_CMD_PREFIX -port $CB_PORT  -pidpath $CB_PID_FILE  -dbhost $dbHost:$dbPort -db $CB_DB_NAME -dbPoolSize $POOL_SIZE -t $traceLevels $IPvOption $extraParams"
  elif [ "$role" == "CP1" ]
  then
    mkdir -p $CP1_LOG_DIR
    port=$CP1_PORT
    CB_START_CMD="$CB_START_CMD_PREFIX -port $CP1_PORT -pidpath $CP1_PID_FILE -dbhost $dbHost:$dbPort -db $CP1_DB_NAME -dbPoolSize $POOL_SIZE -t $traceLevels $IPvOption -logDir $CP1_LOG_DIR $extraParams"
  elif [ "$role" == "CP2" ]
  then
    mkdir -p $CP2_LOG_DIR
    port=$CP2_PORT
    CB_START_CMD="$CB_START_CMD_PREFIX -port $CP2_PORT -pidpath $CP2_PID_FILE -dbhost $dbHost:$dbPort -db $CP2_DB_NAME -dbPoolSize $POOL_SIZE -t $traceLevels $IPvOption -logDir $CP2_LOG_DIR $extraParams"
  elif [ "$role" == "CP3" ]
  then
    mkdir -p $CP3_LOG_DIR
    port=$CP3_PORT
    CB_START_CMD="$CB_START_CMD_PREFIX -port $CP3_PORT -pidpath $CP3_PID_FILE -dbhost $dbHost:$dbPort -db $CP3_DB_NAME -dbPoolSize $POOL_SIZE -t $traceLevels $IPvOption -logDir $CP3_LOG_DIR $extraParams"
  elif [ "$role" == "CP4" ]
  then
    mkdir -p $CP4_LOG_DIR
    port=$CP4_PORT
    CB_START_CMD="$CB_START_CMD_PREFIX -port $CP4_PORT -pidpath $CP4_PID_FILE -dbhost $dbHost:$dbPort -db $CP4_DB_NAME -dbPoolSize $POOL_SIZE -t $traceLevels $IPvOption -logDir $CP4_LOG_DIR $extraParams"
  elif [ "$role" == "CP5" ]
  then
    mkdir -p $CP5_LOG_DIR
    port=$CP5_PORT
    CB_START_CMD="$CB_START_CMD_PREFIX -port $CP5_PORT -pidpath $CP5_PID_FILE -dbhost $dbHost:$dbPort -db $CP5_DB_NAME -dbPoolSize $POOL_SIZE -t $traceLevels $IPvOption -logDir $CP5_LOG_DIR $extraParams"
  fi

  #
  # Start broker under valgrind if VALGRIND set to 1 and if it's the 'main' broker
  #
  # This is IMPORTANT
  # In test cases involving more than **one** broker - valgrind is run only for the CB, not CP1 etc.
  # Having valgrind run for *every broker* destroys the result of the main broker (the 'CB' broker).
  # The 'other' brokers mess up the output file from valgrind, i.e. the result of the main broker
  # is destroyed.
  # So, now ONLY the broker started as 'CB' is started under VALGRIND
  #
  # [ A *number* of old leaks were discovered when this modification was made. ]
  #
  if [ "$VALGRIND" == "" ] || [ "$port" != "$CB_PORT" ]; then
    $CB_START_CMD
    # Wait some time so that the contextBroker is able to do its initial steps (reset database, start HTTP server, etc.)
    sleep 1
    # FIXME: brokerStartAwait $port  instead of sleep 1?
  else
    #
    # Important: the -v flag must be present so that the text "X errors in context Y of Z" is present in the output
    #
    valgrind -v --leak-check=full --track-origins=yes --trace-children=yes $CB_START_CMD > /tmp/valgrind.out 2>&1 &

    # Waiting for valgrind to start (sleep a maximum of 10 secs)
    brokerStartAwait $port
    if [ "$result" != 0 ]
    then
      echo "Unable to start contextBroker"
      exit 1
    fi
  fi

  # Test to see whether we have a broker running on $port. If not raise an error
  running_broker=$(ps -fe | grep contextBroker | grep $port | wc -l)
  if [ $running_broker -ne 1 ]; then
    echo "Unable to start contextBroker"
    exit 1
  fi

  ps=$(ps aux | grep contextBroker)
  dMsg $ps

  # Some times (specially when using remote DB) CB needs some time to connect DB and
  # the test execution needs to a guard time. The righ value for CB_WAIT_AFTER_START
  # needs to be find empirically (it seems that when DB is localhost this time is
  # not needed at all)
  #
  # This guard could be removed once issue #2123 gets completed
  if [ "$CB_WAIT_AFTER_START" != "" ]; then
    sleep $CB_WAIT_AFTER_START
  fi
}

# ------------------------------------------------------------------------------
#
# localBrokerStop
#
function localBrokerStop
{
  role=$1

  if [ "$role" == "CB" ]
  then
    port=$CB_PORT
  elif [ "$role" == "CP1" ]
  then
    port=$CP1_PORT
  elif [ "$role" == "CP2" ]
  then
    port=$CP2_PORT
  elif [ "$role" == "CP3" ]
  then
    port=$CP3_PORT
  elif [ "$role" == "CP4" ]
  then
    port=$CP4_PORT
  elif [ "$role" == "CP5" ]
  then
    port=$CP5_PORT
  fi

  # Test to see if we have a broker running on $port if so kill it!
  running_broker=$(ps -fe | grep contextBroker | grep $port | wc -l)
  if [ $running_broker -ne 0 ]; then
    kill $(ps -fe | grep contextBroker | grep $port | awk '{print $2}') 2> /dev/null
    # Wait some time so the broker can finish properly
    sleep 1
    running_broker=$(ps -fe | grep contextBroker | grep $port | wc -l)
    if [ $running_broker -ne 0 ]; then
      # If the broker refuses to stop politely, kill the process by brute force
      kill -9 $(ps -fe | grep contextBroker | grep $port | awk '{print $2}') 2> /dev/null
      sleep 1
      running_broker=$(ps -fe | grep contextBroker | grep $port | wc -l)
      if [ $running_broker -ne 0 ]; then
        echo "Existing contextBroker is immortal, can not be killed!"
        exit 1
      fi
    fi
  fi
}

# ------------------------------------------------------------------------------
#
# brokerStart
#
function brokerStart()
{
  role=$1
  traceLevels=$2
  ipVersion=$3

  shift
  shift
  shift

  notificationModeGiven=FALSE

  # Check for --noCache and --cache options in 'extraParams'
  xParams=""
  while [ "$#" != 0 ]
  do
    if   [ "$1" == "--noCache" ];            then noCache=ON;
    elif [ "$1" == "--cache" ];              then noCache=OFF;
    elif [ "$1" == "-noCache" ];             then noCache=ON;
    elif [ "$1" == "-cache" ];               then noCache=OFF;
    elif [ "$1" == "-notificationMode" ] || [ "$1" == "--notificationMode" ]
    then
        notificationModeGiven=TRUE
        xParams="$xParams $1 $2"
        shift
    else xParams=$xParams' '$1
    fi
    shift
  done

  if [ "$noCache" != "OFF" ]
  then
    if [ "$CB_NO_CACHE" == "ON" ] || [ "$noCache"  == "ON" ]
    then
      xParams=$xParams' -noCache'
    fi
  fi

 # Not given notificationMode but not forbidden, use default
 if [ "$notificationModeGiven" == "FALSE" ]  &&  [ "$CB_THREADPOOL" == "ON" ]
 then
    xParams=$xParams' -notificationMode threadpool:200:20'
 fi

 if [ "$role" == "" ]
 then
    echo "No role given as first parameter for brokerStart"
    return
 fi

  if [ "$traceLevels" == "" ]
  then
    traceLevels=0-255
  fi

  if [ "$ipVersion" == "" ]
  then
    ipVersion=BOTH
  fi

  localBrokerStop $role
  localBrokerStart $role $traceLevels $ipVersion $xParams

}

# ------------------------------------------------------------------------------
#
# brokerStop -
#
function brokerStop
{
  if [ "$1" == "-v" ]
  then
    _verbose=1
    shift
  fi

  role=$1
  if [ "$role" == "" ]
  then
    role=CB
  fi

  vMsg "Stopping broker $1"

  if [ "$role" == "CB" ]
  then
    pidFile=$CB_PID_FILE
    port=$CB_PORT
    vMsg pidFile: $pidFile
    vMsg port: $port
  elif [ "$role" == "CP1" ]
  then
    pidFile=$CP1_PID_FILE
    port=$CP1_PORT
  elif [ "$role" == "CP2" ]
  then
    pidFile=$CP2_PID_FILE
    port=$CP2_PORT
  elif [ "$role" == "CP3" ]
  then
    pidFile=$CP3_PID_FILE
    port=$CP3_PORT
  elif [ "$role" == "CP4" ]
  then
    pidFile=$CP4_PID_FILE
    port=$CP4_PORT
  elif [ "$role" == "CP5" ]
  then
    pidFile=$CP5_PID_FILE
    port=$CP5_PORT
  fi

  if [ "$VALGRIND" == "" ]; then
    vMsg "killing with PID from pidFile"
    kill $(cat $pidFile 2> /dev/null) 2> /dev/null
    vMsg "should be dead"
    if [ -f /tmp/orion_${port}.pid ]
    then
      rm -f /tmp/orion_${port}.pid 2> /dev/null
    fi
  else
    curl localhost:${port}/exit/harakiri 2> /dev/null >> ${TEST_BASENAME}.valgrind.stop.out
    # Waiting for valgrind to terminate (sleep a max of 10)
    brokerStopAwait $port  # FIXME
    # sleep 4
  fi
}

# ------------------------------------------------------------------------------
#
# accumulatorStop -
#
function accumulatorStop()
{
  port=$1

  # If port is missing, we use the default LISTENER_PORT
  if [ -z "$port" ]
  then
    port=${LISTENER_PORT}
  fi

  pid=$(cat /tmp/accumulator.$port.pid 2> /dev/null)
  if [ "$pid" != "" ]
  then
    kill -15 $pid 2> /dev/null
    sleep .1
    kill -2 $pid 2> /dev/null
    sleep .1
    kill -9 $pid 2> /dev/null
    rm -f /tmp/accumulator.$port.pid
  fi
}

# ------------------------------------------------------------------------------
#
# accumulatorStart -
#
function accumulatorStart()
{
  # FIXME P6: note that due to the way argument processing work, the arguments have to be
  # in a fixed order in the .test, i.e.: --pretty-print, --https, --key, --cert

  if [ "$1" = "--pretty-print" ]
  then
    pretty="$1"
    shift
  fi

  if [ "$1" = "--https" ]
  then
    https="$1"
    shift
  fi

  if [ "$1" = "--key" ]
  then
    key="$1 $2"
    shift
    shift
  fi

  if [ "$1" = "--cert" ]
  then
    cert="$1 $2"
    shift
    shift
  fi

  bindIp=$1
  port=$2

  # If port is missing, we use the default LISTENER_PORT
  if [ -z "$port" ]
  then
    port=$LISTENER_PORT
  fi

  if [ -z "$bindIp" ]
  then
    bindIp='localhost'
  fi

  accumulatorStop $port

  accumulator-server.py --port $port --url /notify --host $bindIp $pretty $https $key $cert > /tmp/accumulator_${port}_stdout 2> /tmp/accumulator_${port}_stderr &
  echo accumulator running as PID $$

  # Wait until accumulator has started or we have waited a given maximum time
  port_not_ok=1
  typeset -i time
  time=0

  until [ $port_not_ok -eq 0 ]
  do
   if [ "$time" -eq "$MAXIMUM_WAIT" ]
   then
      echo "Unable to start listening application after waiting ${MAXIMUM_WAIT}"
      exit 1
   fi
   sleep 1

   time=$time+1
   nc -w 2 $bindIp $port >/dev/null </dev/null
   port_not_ok=$?
  done
}

# ------------------------------------------------------------------------------
#
# accumulatorDump
#
function accumulatorDump()
{
  # FIXME P6: Argument processing in this is ugly... needs a cleanup

  valgrindSleep 2

  if [ "$1" == "IPV6" ]
  then
    url="[::1]:${LISTENER_PORT}/dump"
    g_flag="-g"
  else
    url="localhost:${LISTENER_PORT}/dump"
  fi

  if [ "$2" == "HTTPS" ]
  then
    schema="https://"
    k_flag="-k"
  else
    schema="http://"
  fi

  curl $k_flag $g_flag $schema$url -s -S 2> /dev/null
}

# ------------------------------------------------------------------------------
#
# accumulator2Dump
#
function accumulator2Dump()
{
  valgrindSleep 2

  if [ "$1" == "IPV6" ]
  then
    curl -g [::1]:${LISTENER2_PORT}/dump -s -S 2> /dev/null
  else
    curl localhost:${LISTENER2_PORT}/dump -s -S 2> /dev/null
  fi
}

# ------------------------------------------------------------------------------
#
# accumulator3Dump
#
function accumulator3Dump()
{
  valgrindSleep 2

  if [ "$1" == "IPV6" ]
  then
    curl -g [::1]:${LISTENER3_PORT}/dump -s -S 2> /dev/null
  else
    curl localhost:${LISTENER3_PORT}/dump -s -S 2> /dev/null
  fi
}

# ------------------------------------------------------------------------------
#
# accumulatorCount
#
function accumulatorCount()
{
  valgrindSleep 2

  if [ "$1" == "IPV6" ]
  then
    curl -g [::1]:${LISTENER_PORT}/number -s -S 2> /dev/null
  else
    curl localhost:${LISTENER_PORT}/number -s -S 2> /dev/null
  fi
}

# ------------------------------------------------------------------------------
#
# accumulator2Count
#
function accumulator2Count()
{
  valgrindSleep 2

  if [ "$1" == "IPV6" ]
  then
    curl -g [::1]:${LISTENER2_PORT}/number -s -S 2> /dev/null
  else
    curl localhost:${LISTENER2_PORT}/number -s -S 2> /dev/null
  fi
}

# ------------------------------------------------------------------------------
#
# accumulator3Count
#
function accumulator3Count()
{
  valgrindSleep 2

  if [ "$1" == "IPV6" ]
  then
    curl -g [::1]:${LISTENER3_PORT}/number -s -S 2> /dev/null
  else
    curl localhost:${LISTENER3_PORT}/number -s -S 2> /dev/null
  fi
}

# ------------------------------------------------------------------------------
#
# accumulatorReset -
#
function accumulatorReset()
{
  curl localhost:${LISTENER_PORT}/reset -s -S -X POST
}

# ------------------------------------------------------------------------------
#
# valgrindSleep
#
function valgrindSleep()
{
  if [ "$FUNC_TEST_RUNNING_UNDER_VALGRIND" == "true" ]
  then
    sleep $1
  fi
}

# ------------------------------------------------------------------------------
#
# mongoCmd -
#
# This functions is needed due to some problems with jenkins that seems to avoid
# the usage of 'mongo --quiet ...' directly. Thus, we need to use mongo without
# --quiet, but we need to get rid of some preamble lines about mongo version and
# connection information and a final 'bye' line
#
function mongoCmd()
{

  host="${CB_DATABASE_HOST}"
  if [ "$host" == "" ]
  then
    host="localhost"
  fi

  port="${CB_DATABASE_PORT}"
  if [ "$port" == "" ]
  then
    port="27017"
  fi

  db=$1
  cmd=$2
  echo $cmd | mongo $host:$port/$db | tail -n 2 | head -n 1
}

# ------------------------------------------------------------------------------
#
# dbInsertEntity -
#
# Insert a crafted entity which and "inflated" attribute. The database is pased
# as first argument and the id of the entity as second. The size of the "inflated"
# attribute is the third argument x 10.
#
function dbInsertEntity()
{
  db=$1
  entId=$2
  size=$3

  # This is a JavaScript code that generates a string variable which length is the
  # one in the argument x 10
  jsCode="s = \"\"; for (var i = 0; i < $size ; i++) { s += \"0123456789\"; }"

  ent="entity = \"$entId\""

  doc='doc = {
    "_id": {
        "id": entity,
        "type": "T"
    },
    "attrNames": [ "A" ],
    "attrs": {
        "A": {
            "type": "TA",
            "value": s,
            "creDate" : 1389376081,
            "modDate" : 1389376081
        },
    },
    "creDate": 1389376081,
    "modDate": 1389376081
  }'

  cmd='db.entities.insert(doc)'

  host="${CB_DATABASE_HOST}"
  if [ "$host" == "" ]
  then
    host="localhost"
  fi

  port="${CB_DATABASE_PORT}"
  if [ "$port" == "" ]
  then
    port="27017"
  fi

  echo "$jsCode ; $ent ; $doc ; $cmd" | mongo $host:$port/$db
}

# ------------------------------------------------------------------------------
#
# orionCurl
#
# Options:
#   -X            <HTTP method>    (default: according to curl. GET if no payload, POST if with payload)
#   --host        <host>           (default: localhost)
#   --port        <port>           (default: $CB_PORT)
#   --url         <URL>            (default: empty string)
#   --urlParams   <params>         (URI parameters 'in' URL-string)
#   --payload     <payload>        (default: NO PAYLOAD. Possible values: [filename | "${string}"])
#   --in          (input payload)  (default: xml => application/xml, If 'json': application/json)
#   --out         (output payload  (default: xml => application/xml, If 'json': application/json)
#   --json        (in/out JSON)    (if --in/out is used AFTER --json, it overrides)
#   --tenant      <tenant>         (tenant in HTTP header)
#   --servicePath <path>           (Service Path in HTTP header)
#   --xauthToken  <token>          (X-Auth token value)
#   --origin      <origin>         (Origin in HTTP header)
#   --correlator  <correlatorId>   (default: no correlator ID)
#   --noPayloadCheck               (don't check the payload)
#   --payloadCheck <format>        (force specific treatment of payload)
#   --header      <HTTP header>    (more headers)
#   -H            <HTTP header>    (more headers)
#   --verbose                      (verbose output)
#   -v                             (verbose output)
#   --debug                        (debug mode - output to /tmp/orionFuncTestDebug.log
#
# Any parameters are sent as is to 'curl'
#
function orionCurl()
{
  payloadCheckFormat='json'

  dMsg
  dMsg $(date)
  dMsg orionCurl $*

  _host='localhost'
  _port=$CB_PORT
  _url=''
  _method=''
  _payload=''
  _servicePath=''
  _xtra=''
  _headers=''
  _noPayloadCheck='off'
  _forcedNoPayloadCheck='off'
  _tenant=''
  _origin=''
  _correlator=''
  _inFormat=''
  _outFormat='--header "Accept: application/json"'
  _in='';
  _out='';
  _urlParams=''
  _xauthToken=''
  _payloadCheck=''

  #
  # Parsing parameters
  #
  while [ "$#" != 0 ]
  do
    if   [ "$1" == "--host" ]; then            _host=$2; shift;
    elif [ "$1" == "--port" ]; then            _port=$2; shift;
    elif [ "$1" == "--url" ]; then             _url=$2; shift;
    elif [ "$1" == "--urlParams" ]; then       _urlParams=$2; shift;
    elif [ "$1" == "-X" ]; then                _method="-X $2"; shift;
    elif [ "$1" == "--payload" ]; then         _payload=$2; shift;
    elif [ "$1" == "--noPayloadCheck" ]; then  _noPayloadCheck='on'; _forcedNoPayloadCheck='on'
    elif [ "$1" == "--payloadCheck" ]; then    _payloadCheck=$2; _noPayloadCheck='off'; shift;
    elif [ "$1" == "--servicePath" ]; then     _servicePath='--header "Fiware-ServicePath: '${2}'"'; shift;
    elif [ "$1" == "--tenant" ]; then          _tenant='--header "Fiware-Service: '${2}'"'; shift;
    elif [ "$1" == "--origin" ]; then          _origin='--header "Origin: '${2}'"'; shift;
    elif [ "$1" == "--correlator" ]; then      _correlator='--header "Fiware-Correlator: '${2}'"'; shift;
    elif [ "$1" == "-H" ]; then                _headers=${_headers}" --header \"$2\""; shift;
    elif [ "$1" == "--header" ]; then          _headers=${_headers}" --header \"$2\""; shift;
    elif [ "$1" == "--in" ]; then              _in="$2"; shift;
    elif [ "$1" == "--out" ]; then             _out="$2"; shift;
    elif [ "$1" == "--xauthToken" ]; then      _xauthToken='--header "X-Auth-Token: '${2}'"'; shift;
    elif [ "$1" == "-v" ]; then                _verbose=on;
    elif [ "$1" == "--verbose" ]; then         _verbose=on;
    elif [ "$1" == "--debug" ]; then           _debug=on;
    elif [ "$1" == "-d" ]; then                _debug=on;
    else                                       _xtra="$_xtra $1";
    fi
    shift
  done

  if [ "$_payload" != "" ]
  then
    _inFormat='--header "Content-Type: application/json"'
  fi

  #
  # Check the parameters
  #

  # 1. Do we have a URL present?
  if [ "$_url" == "" ]
  then
    echo "No URL";
    return 1;
  fi

  # 2. Payload can be either data or a path to a file - if data, a file is created with the data as contents
  _PAYLOAD=''
  if [ "$_payload" != "" ]
  then
    if [ -f "$_payload" ]
    then
      _PAYLOAD="-d @$_payload"
    else
      echo $_payload > /tmp/orionFuncTestPayload
      _PAYLOAD="-d @/tmp/orionFuncTestPayload"
    fi
  fi

  # 3. Fix for 'Content-Type' and 'Accept' short names 'xml' and 'json'
  if   [ "$_in"   == "application/xml" ];  then _in='xml';   fi
  if   [ "$_in"   == "application/json" ]; then _in='json';  fi
  if   [ "$_out"  == "application/xml" ];  then _out='xml';  fi
  if   [ "$_out"  == "application/json" ]; then _out='json'; fi
  if   [ "$_out"  == "text/plain" ];       then _out='text'; fi

  if   [ "$_in"  == "xml" ];   then _inFormat='--header "Content-Type: application/xml"'
  elif [ "$_in"  == "json" ];  then _inFormat='--header "Content-Type: application/json"'
  elif [ "$_in"  == "text" ];  then _inFormat='--header "Content-Type: text/plain"'
  elif [ "$_in"  != "" ];      then _inFormat='--header "Content-Type: '${_in}'"'
  fi

  # Note that payloadCheckFormat is also json in the case of --in xml, as the CB also returns error in JSON in this case
  if   [ "$_out" == "xml" ];   then _outFormat='--header "Accept: application/xml"'; payloadCheckFormat='json'
  elif [ "$_out" == "json" ];  then _outFormat='--header "Accept: application/json"'; payloadCheckFormat='json'
  elif [ "$_out" == "text" ];  then _outFormat='--header "Accept: text/plain"'; _noPayloadCheck='on'
  elif [ "$_out" == "any" ];   then _outFormat='--header "Accept: */*"'; _noPayloadCheck='on'
  elif [ "$_out" == "EMPTY" ]; then _outFormat='--header "Accept:"'
  elif [ "$_out" != "" ];      then _outFormat='--header "Accept: '${_out}'"'; _noPayloadCheck='off'
  fi

  if [ "$_payloadCheck" != "" ]
  then
    payloadCheckFormat=$_payloadCheck
    _noPayloadCheck='off'
  fi

  if [ "$_forcedNoPayloadCheck" == 'on' ]
  then
    _noPayloadCheck='on'
  fi

  dMsg $_in: $_in
  dMsg _out: $_out
  dMsg _outFormat: $_outFormat
  dMsg _inFormat: $_inFormat
  dMsg payloadCheckFormat: $payloadCheckFormat
  dMsg _noPayloadCheck: $_noPayloadCheck

  #
  # Assemble the command
  #
  command='curl "'$_host':'${_port}${_url}'"'

  if [ "$_urlParams"   != "" ]; then  command=${command}'?'${_urlParams};   fi
  if [ "$_PAYLOAD"     != "" ]; then  command=${command}' '${_PAYLOAD};     fi
  if [ "$_method"      != "" ]; then  command=${command}' '${_method};      fi
  if [ "$_tenant"      != "" ]; then  command=${command}' '${_tenant};      fi
  if [ "$_servicePath" != "" ]; then  command=${command}' '${_servicePath}; fi
  if [ "$_inFormat"    != "" ]; then  command=${command}' '${_inFormat};    fi
  if [ "$_outFormat"   != "" ]; then  command=${command}' '${_outFormat};   fi
  if [ "$_origin"      != "" ]; then  command=${command}' '${_origin};      fi
  if [ "$_correlator"  != "" ]; then  command=${command}' '${_correlator};  fi
  if [ "$_xauthToken"  != "" ]; then  command=${command}' '${_xauthToken};  fi
  if [ "$_headers"     != "" ]; then  command=${command}' '${_headers};     fi
  if [ "$_xtra"        != "" ]; then  command=${command}' '${_xtra};        fi

  command=${command}' --header "Expect:"'
  command=${command}' -s -S --dump-header /tmp/httpHeaders.out'
  dMsg command: $command

  #
  # Execute the command
  #
  dMsg Executing the curl-command
  eval $command > /tmp/orionCurl.response 2> /dev/null
  _response=$(cat /tmp/orionCurl.response)

  if [ ! -f /tmp/httpHeaders.out ]
  then
    echo "Broker seems to have died ..."
  else
    _responseHeaders=$(cat /tmp/httpHeaders.out)
    #
    # Remove "Connection: Keep-Alive" and "Connection: close" headers
    #
    sed '/Connection: Keep-Alive/d' /tmp/httpHeaders.out  > /tmp/httpHeaders2.out
    sed '/Connection: close/d'      /tmp/httpHeaders2.out > /tmp/httpHeaders.out
    sed '/Connection: Close/d'      /tmp/httpHeaders.out
  fi

  #
  # Unless we remove the HTTP header file, it will remain for the next execution
  #
  \rm -f /tmp/httpHeaders2.out /tmp/httpHeaders.out

  #
  # Print and beautify response body, if any - and if option --noPayloadCheck hasn't been set
  #
  if [ "$_noPayloadCheck" == "on" ]
  then
    cat /tmp/orionCurl.response
    echo
    rm -f /tmp/orionCurl.response
  else
    if [ "$_response" != "" ]
    then
      dMsg buffer to $payloadCheckFormat beautify: $_response

      if [ "$payloadCheckFormat" == json ] || [ "$payloadCheckFormat" == "" ]
      then
        vMsg Running python tool for $_response
        #
        # We need to apply pretty-print on _response. Otherwise positional processing used in .test
        # (e.g. to get SUB_ID typically grep and awk are used) will break
        #
        _response=$(echo $_response | python -mjson.tool)
        echo "$_response"
      else
        dMsg Unknown payloadCheckFormat
      fi
    fi
  fi
}

export -f dbInit
export -f dbList
export -f dbDrop
export -f dbResetAll
export -f brokerStart
export -f localBrokerStop
export -f localBrokerStart
export -f brokerStop
export -f accumulatorStart
export -f accumulatorStop
export -f accumulatorDump
export -f accumulator2Dump
export -f accumulator3Dump
export -f accumulatorCount
export -f accumulator2Count
export -f accumulator3Count
export -f accumulatorReset
export -f orionCurl
export -f dbInsertEntity
export -f mongoCmd
export -f vMsg
export -f dMsg
export -f valgrindSleep
export -f brokerStartAwait
export -f brokerStopAwait

# Copyright 2013 Telefonica Investigacion y Desarrollo, S.A.U
#
# This file is part of Orion Context Broker.
#
# Orion Context Broker is free software: you can redistribute it and/or
# modify it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.
#
# Orion Context Broker is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero
# General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with Orion Context Broker. If not, see http://www.gnu.org/licenses/.
#
# For those usages not covered by this license please contact with
# iot_support at tid dot es

# ------------------------------------------------------------------------------
#
# harnessInit -
#

if [ "$CONTEXTBROKER_TESTENV_SOURCED" != "YES" ]
then
  echo
  echo '---------------------------------------------------'
  echo "Test Environment missing - please source testEnv.sh"
  echo '---------------------------------------------------'
  echo
  exit 1
fi

export CONTEXTBROKER_HARNESS_FUNCTIONS_SOURCED="YES"

if [ "$ORION_FT_DEBUG" == "1" ]
then
  _debug='on'
else
  _debug='off'
fi

# ------------------------------------------------------------------------------
#
# harnessExit -
#
function harnessExit()
{
  rm -f headers.out
}

# ------------------------------------------------------------------------------
#
# vMsg - verbose output
#
# Remember that if the '--verbose' flag is used, the verboser messages pollute
# the total answer and the harness test will fail.
#
# Verbose mode is only meant to troubleshoot harness tests that fail.
#
function vMsg()
{
  if [ "$_verbose" == "on" ]
  then
    echo $*
  fi
}

# ------------------------------------------------------------------------------
#
# dMsg - debug output
#
function dMsg()
{
  if [ "$ORION_FT_DEBUG" == 1 ] || [ "$_debug" == "on"  ]
  then
    echo $* >> /tmp/orionFuncTestDebug.log
  fi
}

# ------------------------------------------------------------------------------
#
# dbInit -
#
# ------------------------------------------------------------------------------
#
# dbInit -
#
function dbInit()
{
  role=$1
  tenant=$2

  host="${CB_DATABASE_HOST}"
  if [ "$host" == "" ]
  then
    host="localhost"
  fi

  port="${CB_DATABASE_PORT}"
  if [ "$port" == "" ]
  then
    port="27017"
  fi

  dMsg initializing database;

  if [ "$role" == "CB" ]
  then
    baseDbName=${CB_DB_NAME}
  elif [ "$role" == "CP1" ]
  then
    baseDbName=${CP1_DB_NAME}
  elif [ "$role" == "CP2" ]
  then
    baseDbName=${CP2_DB_NAME}
  elif [ "$role" == "CP3" ]
  then
    baseDbName=${CP3_DB_NAME}
  elif [ "$role" == "CP4" ]
  then
    baseDbName=${CP4_DB_NAME}
  elif [ "$role" == "CP5" ]
  then
    baseDbName=${CP5_DB_NAME}
  else
    baseDbName=$1
  fi

  # If a tenant was provided, then we build the tenant DB, e.g. orion-ftest1
  if [ $# == 2 ]
  then
    db=$baseDbName-$tenant
  else
    db=$baseDbName
  fi

  dMsg "database to drop: <$db>"
  echo 'db.dropDatabase()' | mongo $host:$port/$db --quiet
}

# ------------------------------------------------------------------------------
#
# dbDrop -
#
function dbDrop()
{
  if [ "$CB_DB_DROP" != "No" ]
  then
    # FIXME: Not sure if $@ should be better...
    dbInit $*
  fi
}

# -----------------------------------------------------------------------------
#
# dbList
#
function dbList
{
  name="$1"

  host="${CB_DATABASE_HOST}"
  if [ "$host" == "" ]
  then
    host="localhost"
  fi

  port="${CB_DATABASE_PORT}"
  if [ "$port" == "" ]
  then
    port="27017"
  fi

  if [ "$name" != "" ]
  then
    echo show dbs | mongo $host:$port --quiet | grep "$name" | awk '{ print $1 }'
  else
    echo show dbs | mongo $host:$port --quiet | awk '{ print $1 }'
  fi
}

# ------------------------------------------------------------------------------
#
# dbResetAll -
#
function dbResetAll()
{
  host="${CB_DATABASE_HOST}"
  if [ "$host" == "" ]
  then
    host="localhost"
  fi

  port="${CB_DATABASE_PORT}"
  if [ "$port" == "" ]
  then
    port="27017"
  fi

  all=$(echo show dbs | mongo $host:$port --quiet | grep ftest | awk '{ print $1 }')
  for db in $all
  do
    dbDrop $db
  done
}

# ------------------------------------------------------------------------------
#
# brokerStopAwait
#
function brokerStopAwait
{
  port=$1

  typeset -i loopNo
  typeset -i loops
  loopNo=0
  loops=50

  while [ $loopNo -lt $loops ]
  do
    nc -w 2 localhost $port >/dev/null </dev/null
    if [ "$?" != "0" ]
    then
      vMsg The orion context broker on port $port has stopped
      sleep 1
      break;
    fi

    vMsg Awaiting orion context broker to fully stop '('$loopNo')' ...
    sleep .2
    loopNo=$loopNo+1
  done

  sleep .5

  # Check CB NOT running fine
  curl -s localhost:${port}/version | grep version > /dev/null
  result=$?
  if [ "$result" == "0" ]
  then
    result=1  # ERROR - the broker is still running
  else
    result=0
  fi
}

# ------------------------------------------------------------------------------
#
# brokerStartAwait
#
# For some really strange reason, all functests fail under valgrind when executed by Jenkins
# without the line:
#
#   echo "Broker started after $loopNo checks" >> /tmp/brokerStartCounter
#
# It also works with echo to /dev/null, but using a file we get some more information.
#
# As we only append (>>) to /tmp/brokerStartCounter in this file, /tmp/brokerStartCounter
# is reset in testHarness.sh, using the command "date > tmp/brokerStartCounter".
#
# The reason we send information to a file is that in case the 100 loop is not enough in the future,
# we will have that information in the file /tmp/brokerStartCounter
#
function brokerStartAwait
{
  if [ "$BROKER_AWAIT_SLEEP_TIME" != "" ]
  then
    sleep $BROKER_AWAIT_SLEEP_TIME
    result=0
    return
  fi

  port=$1

  typeset -i loopNo
  typeset -i loops
  loopNo=0
  loops=100

  while [ $loopNo -lt $loops ]
  do
    nc -w 2 localhost $port >/dev/null </dev/null
    if [ "$?" == "0" ]
    then
      vMsg The orion context broker has started, listening on port $port
      echo "Broker started after $loopNo checks" >> /tmp/brokerStartCounter
      sleep 1
      break;
    fi

    vMsg Awaiting valgrind to fully start the orion context broker '('$loopNo')' ...
    sleep .2
    loopNo=$loopNo+1
  done

  sleep .5

  # Check that CB started
  curl -s localhost:${port}/version | grep version >> /tmp/brokerStartCounter
  result=$?
  echo "result: $result" >> /tmp/brokerStartCounter
}

# ------------------------------------------------------------------------------
#
# localBrokerStart
#
function localBrokerStart()
{
  role=$1
  traceLevels=$2
  ipVersion=$3

  POOL_SIZE=${POOL_SIZE:-10}

  shift
  shift
  shift

  if [ "$traceLevels" != "" ]
  then
    extraParams="-logLevel DEBUG "$*
  else
    extraParams=$*
  fi

  IPvOption=""

  dbHost="${CB_DATABASE_HOST}"
  if [ "$dbHost" == "" ]
  then
    dbHost="localhost"
  fi

  dbPort="${CB_DATABASE_PORT}"
  if [ "$dbPort" == "" ]
  then
    dbPort="27017"
  fi

  dMsg starting broker for role $role

  if [ "$ipVersion" == "IPV4" ]
  then
    IPvOption="-ipv4"
  elif [ "$ipVersion" == "IPV6" ]
  then
    IPvOption="-ipv6"
  fi

  # FIXME P10: eventually the -paranoidV1Indent flag has to be removed
  CB_START_CMD_PREFIX="contextBroker -harakiri -paranoidV1Indent"
  if [ "$role" == "CB" ]
  then
    port=$CB_PORT
    CB_START_CMD="$CB_START_CMD_PREFIX -port $CB_PORT  -pidpath $CB_PID_FILE  -dbhost $dbHost:$dbPort -db $CB_DB_NAME -dbPoolSize $POOL_SIZE -t $traceLevels $IPvOption $extraParams"
  elif [ "$role" == "CP1" ]
  then
    mkdir -p $CP1_LOG_DIR
    port=$CP1_PORT
    CB_START_CMD="$CB_START_CMD_PREFIX -port $CP1_PORT -pidpath $CP1_PID_FILE -dbhost $dbHost:$dbPort -db $CP1_DB_NAME -dbPoolSize $POOL_SIZE -t $traceLevels $IPvOption -logDir $CP1_LOG_DIR $extraParams"
  elif [ "$role" == "CP2" ]
  then
    mkdir -p $CP2_LOG_DIR
    port=$CP2_PORT
    CB_START_CMD="$CB_START_CMD_PREFIX -port $CP2_PORT -pidpath $CP2_PID_FILE -dbhost $dbHost:$dbPort -db $CP2_DB_NAME -dbPoolSize $POOL_SIZE -t $traceLevels $IPvOption -logDir $CP2_LOG_DIR $extraParams"
  elif [ "$role" == "CP3" ]
  then
    mkdir -p $CP3_LOG_DIR
    port=$CP3_PORT
    CB_START_CMD="$CB_START_CMD_PREFIX -port $CP3_PORT -pidpath $CP3_PID_FILE -dbhost $dbHost:$dbPort -db $CP3_DB_NAME -dbPoolSize $POOL_SIZE -t $traceLevels $IPvOption -logDir $CP3_LOG_DIR $extraParams"
  elif [ "$role" == "CP4" ]
  then
    mkdir -p $CP4_LOG_DIR
    port=$CP4_PORT
    CB_START_CMD="$CB_START_CMD_PREFIX -port $CP4_PORT -pidpath $CP4_PID_FILE -dbhost $dbHost:$dbPort -db $CP4_DB_NAME -dbPoolSize $POOL_SIZE -t $traceLevels $IPvOption -logDir $CP4_LOG_DIR $extraParams"
  elif [ "$role" == "CP5" ]
  then
    mkdir -p $CP5_LOG_DIR
    port=$CP5_PORT
    CB_START_CMD="$CB_START_CMD_PREFIX -port $CP5_PORT -pidpath $CP5_PID_FILE -dbhost $dbHost:$dbPort -db $CP5_DB_NAME -dbPoolSize $POOL_SIZE -t $traceLevels $IPvOption -logDir $CP5_LOG_DIR $extraParams"
  fi

  #
  # Start broker under valgrind if VALGRIND set to 1 and if it's the 'main' broker
  #
  # This is IMPORTANT
  # In test cases involving more than **one** broker - valgrind is run only for the CB, not CP1 etc.
  # Having valgrind run for *every broker* destroys the result of the main broker (the 'CB' broker).
  # The 'other' brokers mess up the output file from valgrind, i.e. the result of the main broker
  # is destroyed.
  # So, now ONLY the broker started as 'CB' is started under VALGRIND
  #
  # [ A *number* of old leaks were discovered when this modification was made. ]
  #
  if [ "$VALGRIND" == "" ] || [ "$port" != "$CB_PORT" ]; then
    $CB_START_CMD
    # Wait some time so that the contextBroker is able to do its initial steps (reset database, start HTTP server, etc.)
    sleep 1
    # FIXME: brokerStartAwait $port  instead of sleep 1?
  else
    #
    # Important: the -v flag must be present so that the text "X errors in context Y of Z" is present in the output
    #
    valgrind -v --leak-check=full --track-origins=yes --trace-children=yes $CB_START_CMD > /tmp/valgrind.out 2>&1 &

    # Waiting for valgrind to start (sleep a maximum of 10 secs)
    brokerStartAwait $port
    if [ "$result" != 0 ]
    then
      echo "Unable to start contextBroker"
      exit 1
    fi
  fi

  # Test to see whether we have a broker running on $port. If not raise an error
  running_broker=$(ps -fe | grep contextBroker | grep $port | wc -l)
  if [ $running_broker -ne 1 ]; then
    echo "Unable to start contextBroker"
    exit 1
  fi

  ps=$(ps aux | grep contextBroker)
  dMsg $ps

  # Some times (specially when using remote DB) CB needs some time to connect DB and
  # the test execution needs to a guard time. The righ value for CB_WAIT_AFTER_START
  # needs to be find empirically (it seems that when DB is localhost this time is
  # not needed at all)
  #
  # This guard could be removed once issue #2123 gets completed
  if [ "$CB_WAIT_AFTER_START" != "" ]; then
    sleep $CB_WAIT_AFTER_START
  fi
}

# ------------------------------------------------------------------------------
#
# localBrokerStop
#
function localBrokerStop
{
  role=$1

  if [ "$role" == "CB" ]
  then
    port=$CB_PORT
  elif [ "$role" == "CP1" ]
  then
    port=$CP1_PORT
  elif [ "$role" == "CP2" ]
  then
    port=$CP2_PORT
  elif [ "$role" == "CP3" ]
  then
    port=$CP3_PORT
  elif [ "$role" == "CP4" ]
  then
    port=$CP4_PORT
  elif [ "$role" == "CP5" ]
  then
    port=$CP5_PORT
  fi

  # Test to see if we have a broker running on $port if so kill it!
  running_broker=$(ps -fe | grep contextBroker | grep $port | wc -l)
  if [ $running_broker -ne 0 ]; then
    kill $(ps -fe | grep contextBroker | grep $port | awk '{print $2}') 2> /dev/null
    # Wait some time so the broker can finish properly
    sleep 1
    running_broker=$(ps -fe | grep contextBroker | grep $port | wc -l)
    if [ $running_broker -ne 0 ]; then
      # If the broker refuses to stop politely, kill the process by brute force
      kill -9 $(ps -fe | grep contextBroker | grep $port | awk '{print $2}') 2> /dev/null
      sleep 1
      running_broker=$(ps -fe | grep contextBroker | grep $port | wc -l)
      if [ $running_broker -ne 0 ]; then
        echo "Existing contextBroker is immortal, can not be killed!"
        exit 1
      fi
    fi
  fi
}

# ------------------------------------------------------------------------------
#
# brokerStart
#
function brokerStart()
{
  role=$1
  traceLevels=$2
  ipVersion=$3

  shift
  shift
  shift

  notificationModeGiven=FALSE

  # Check for --noCache and --cache options in 'extraParams'
  xParams=""
  while [ "$#" != 0 ]
  do
    if   [ "$1" == "--noCache" ];            then noCache=ON;
    elif [ "$1" == "--cache" ];              then noCache=OFF;
    elif [ "$1" == "-noCache" ];             then noCache=ON;
    elif [ "$1" == "-cache" ];               then noCache=OFF;
    elif [ "$1" == "-notificationMode" ] || [ "$1" == "--notificationMode" ]
    then
        notificationModeGiven=TRUE
        xParams="$xParams $1 $2"
        shift
    else xParams=$xParams' '$1
    fi
    shift
  done

  if [ "$noCache" != "OFF" ]
  then
    if [ "$CB_NO_CACHE" == "ON" ] || [ "$noCache"  == "ON" ]
    then
      xParams=$xParams' -noCache'
    fi
  fi

 # Not given notificationMode but not forbidden, use default
 if [ "$notificationModeGiven" == "FALSE" ]  &&  [ "$CB_THREADPOOL" == "ON" ]
 then
    xParams=$xParams' -notificationMode threadpool:200:20'
 fi

 if [ "$role" == "" ]
 then
    echo "No role given as first parameter for brokerStart"
    return
 fi

  if [ "$traceLevels" == "" ]
  then
    traceLevels=0-255
  fi

  if [ "$ipVersion" == "" ]
  then
    ipVersion=BOTH
  fi

  localBrokerStop $role
  localBrokerStart $role $traceLevels $ipVersion $xParams

}

# ------------------------------------------------------------------------------
#
# brokerStop -
#
function brokerStop
{
  if [ "$1" == "-v" ]
  then
    _verbose=1
    shift
  fi

  role=$1
  if [ "$role" == "" ]
  then
    role=CB
  fi

  vMsg "Stopping broker $1"

  if [ "$role" == "CB" ]
  then
    pidFile=$CB_PID_FILE
    port=$CB_PORT
    vMsg pidFile: $pidFile
    vMsg port: $port
  elif [ "$role" == "CP1" ]
  then
    pidFile=$CP1_PID_FILE
    port=$CP1_PORT
  elif [ "$role" == "CP2" ]
  then
    pidFile=$CP2_PID_FILE
    port=$CP2_PORT
  elif [ "$role" == "CP3" ]
  then
    pidFile=$CP3_PID_FILE
    port=$CP3_PORT
  elif [ "$role" == "CP4" ]
  then
    pidFile=$CP4_PID_FILE
    port=$CP4_PORT
  elif [ "$role" == "CP5" ]
  then
    pidFile=$CP5_PID_FILE
    port=$CP5_PORT
  fi

  if [ "$VALGRIND" == "" ]; then
    vMsg "killing with PID from pidFile"
    kill $(cat $pidFile 2> /dev/null) 2> /dev/null
    vMsg "should be dead"
    if [ -f /tmp/orion_${port}.pid ]
    then
      rm -f /tmp/orion_${port}.pid 2> /dev/null
    fi
  else
    curl localhost:${port}/exit/harakiri 2> /dev/null >> ${TEST_BASENAME}.valgrind.stop.out
    # Waiting for valgrind to terminate (sleep a max of 10)
    brokerStopAwait $port  # FIXME
    # sleep 4
  fi
}

# ------------------------------------------------------------------------------
#
# accumulatorStop -
#
function accumulatorStop()
{
  port=$1

  # If port is missing, we use the default LISTENER_PORT
  if [ -z "$port" ]
  then
    port=${LISTENER_PORT}
  fi

  pid=$(cat /tmp/accumulator.$port.pid 2> /dev/null)
  if [ "$pid" != "" ]
  then
    kill -15 $pid 2> /dev/null
    sleep .1
    kill -2 $pid 2> /dev/null
    sleep .1
    kill -9 $pid 2> /dev/null
    rm -f /tmp/accumulator.$port.pid
  fi
}

# ------------------------------------------------------------------------------
#
# accumulatorStart -
#
function accumulatorStart()
{
  # FIXME P6: note that due to the way argument processing work, the arguments have to be
  # in a fixed order in the .test, i.e.: --pretty-print, --https, --key, --cert

  if [ "$1" = "--pretty-print" ]
  then
    pretty="$1"
    shift
  fi

  if [ "$1" = "--https" ]
  then
    https="$1"
    shift
  fi

  if [ "$1" = "--key" ]
  then
    key="$1 $2"
    shift
    shift
  fi

  if [ "$1" = "--cert" ]
  then
    cert="$1 $2"
    shift
    shift
  fi

  bindIp=$1
  port=$2

  # If port is missing, we use the default LISTENER_PORT
  if [ -z "$port" ]
  then
    port=$LISTENER_PORT
  fi

  if [ -z "$bindIp" ]
  then
    bindIp='localhost'
  fi

  accumulatorStop $port

  accumulator-server.py --port $port --url /notify --host $bindIp $pretty $https $key $cert > /tmp/accumulator_${port}_stdout 2> /tmp/accumulator_${port}_stderr &
  echo accumulator running as PID $$

  # Wait until accumulator has started or we have waited a given maximum time
  port_not_ok=1
  typeset -i time
  time=0

  until [ $port_not_ok -eq 0 ]
  do
   if [ "$time" -eq "$MAXIMUM_WAIT" ]
   then
      echo "Unable to start listening application after waiting ${MAXIMUM_WAIT}"
      exit 1
   fi
   sleep 1

   time=$time+1
   nc -w 2 localhost $port >/dev/null </dev/null
   port_not_ok=$?
  done
}

# ------------------------------------------------------------------------------
#
# accumulatorDump
#
function accumulatorDump()
{
  # FIXME P6: Argument processing in this is ugly... needs a cleanup

  valgrindSleep 2

  if [ "$1" == "IPV6" ]
  then
    url="[::1]:${LISTENER_PORT}/dump"
    g_flag="-g"
  else
    url="localhost:${LISTENER_PORT}/dump"
  fi

  if [ "$2" == "HTTPS" ]
  then
    schema="https://"
    k_flag="-k"
  else
    schema="http://"
  fi

  curl $k_flag $g_flag $schema$url -s -S 2> /dev/null
}

# ------------------------------------------------------------------------------
#
# accumulator2Dump
#
function accumulator2Dump()
{
  valgrindSleep 2

  if [ "$1" == "IPV6" ]
  then
    curl -g [::1]:${LISTENER2_PORT}/dump -s -S 2> /dev/null
  else
    curl localhost:${LISTENER2_PORT}/dump -s -S 2> /dev/null
  fi
}

# ------------------------------------------------------------------------------
#
# accumulator3Dump
#
function accumulator3Dump()
{
  valgrindSleep 2

  if [ "$1" == "IPV6" ]
  then
    curl -g [::1]:${LISTENER3_PORT}/dump -s -S 2> /dev/null
  else
    curl localhost:${LISTENER3_PORT}/dump -s -S 2> /dev/null
  fi
}

# ------------------------------------------------------------------------------
#
# accumulatorCount
#
function accumulatorCount()
{
  valgrindSleep 2

  if [ "$1" == "IPV6" ]
  then
    curl -g [::1]:${LISTENER_PORT}/number -s -S 2> /dev/null
  else
    curl localhost:${LISTENER_PORT}/number -s -S 2> /dev/null
  fi
}

# ------------------------------------------------------------------------------
#
# accumulator2Count
#
function accumulator2Count()
{
  valgrindSleep 2

  if [ "$1" == "IPV6" ]
  then
    curl -g [::1]:${LISTENER2_PORT}/number -s -S 2> /dev/null
  else
    curl localhost:${LISTENER2_PORT}/number -s -S 2> /dev/null
  fi
}

# ------------------------------------------------------------------------------
#
# accumulator3Count
#
function accumulator3Count()
{
  valgrindSleep 2

  if [ "$1" == "IPV6" ]
  then
    curl -g [::1]:${LISTENER3_PORT}/number -s -S 2> /dev/null
  else
    curl localhost:${LISTENER3_PORT}/number -s -S 2> /dev/null
  fi
}

# ------------------------------------------------------------------------------
#
# accumulatorReset -
#
function accumulatorReset()
{
  curl localhost:${LISTENER_PORT}/reset -s -S -X POST
}

# ------------------------------------------------------------------------------
#
# valgrindSleep
#
function valgrindSleep()
{
  if [ "$FUNC_TEST_RUNNING_UNDER_VALGRIND" == "true" ]
  then
    sleep $1
  fi
}

# ------------------------------------------------------------------------------
#
# mongoCmd -
#
# This functions is needed due to some problems with jenkins that seems to avoid
# the usage of 'mongo --quiet ...' directly. Thus, we need to use mongo without
# --quiet, but we need to get rid of some preamble lines about mongo version and
# connection information and a final 'bye' line
#
function mongoCmd()
{

  host="${CB_DATABASE_HOST}"
  if [ "$host" == "" ]
  then
    host="localhost"
  fi

  port="${CB_DATABASE_PORT}"
  if [ "$port" == "" ]
  then
    port="27017"
  fi

  db=$1
  cmd=$2
  echo $cmd | mongo $host:$port/$db | tail -n 2 | head -n 1
}

# ------------------------------------------------------------------------------
#
# dbInsertEntity -
#
# Insert a crafted entity which and "inflated" attribute. The database is pased
# as first argument and the id of the entity as second. The size of the "inflated"
# attribute is the third argument x 10.
#
function dbInsertEntity()
{
  db=$1
  entId=$2
  size=$3

  # This is a JavaScript code that generates a string variable which length is the
  # one in the argument x 10
  jsCode="s = \"\"; for (var i = 0; i < $size ; i++) { s += \"0123456789\"; }"

  ent="entity = \"$entId\""

  doc='doc = {
    "_id": {
        "id": entity,
        "type": "T"
    },
    "attrNames": [ "A" ],
    "attrs": {
        "A": {
            "type": "TA",
            "value": s,
            "creDate" : 1389376081,
            "modDate" : 1389376081
        },
    },
    "creDate": 1389376081,
    "modDate": 1389376081
  }'

  cmd='db.entities.insert(doc)'

  host="${CB_DATABASE_HOST}"
  if [ "$host" == "" ]
  then
    host="localhost"
  fi

  port="${CB_DATABASE_PORT}"
  if [ "$port" == "" ]
  then
    port="27017"
  fi

  echo "$jsCode ; $ent ; $doc ; $cmd" | mongo $host:$port/$db
}

# ------------------------------------------------------------------------------
#
# orionCurl
#
# Options:
#   -X            <HTTP method>    (default: according to curl. GET if no payload, POST if with payload)
#   --host        <host>           (default: localhost)
#   --port        <port>           (default: $CB_PORT)
#   --url         <URL>            (default: empty string)
#   --urlParams   <params>         (URI parameters 'in' URL-string)
#   --payload     <payload>        (default: NO PAYLOAD. Possible values: [filename | "${string}"])
#   --in          (input payload)  (default: xml => application/xml, If 'json': application/json)
#   --out         (output payload  (default: xml => application/xml, If 'json': application/json)
#   --json        (in/out JSON)    (if --in/out is used AFTER --json, it overrides)
#   --tenant      <tenant>         (tenant in HTTP header)
#   --servicePath <path>           (Service Path in HTTP header)
#   --xauthToken  <token>          (X-Auth token value)
#   --origin      <origin>         (Origin in HTTP header)
#   --correlator  <correlatorId>   (default: no correlator ID)
#   --noPayloadCheck               (don't check the payload)
#   --payloadCheck <format>        (force specific treatment of payload)
#   --header      <HTTP header>    (more headers)
#   -H            <HTTP header>    (more headers)
#   --verbose                      (verbose output)
#   -v                             (verbose output)
#   --debug                        (debug mode - output to /tmp/orionFuncTestDebug.log
#
# Any parameters are sent as is to 'curl'
#
function orionCurl()
{
  payloadCheckFormat='json'

  dMsg
  dMsg $(date)
  dMsg orionCurl $*

  _host='localhost'
  _port=$CB_PORT
  _url=''
  _method=''
  _payload=''
  _servicePath=''
  _xtra=''
  _headers=''
  _noPayloadCheck='off'
  _forcedNoPayloadCheck='off'
  _tenant=''
  _origin=''
  _correlator=''
  _inFormat=''
  _outFormat='--header "Accept: application/json"'
  _in='';
  _out='';
  _urlParams=''
  _xauthToken=''
  _payloadCheck=''

  #
  # Parsing parameters
  #
  while [ "$#" != 0 ]
  do
    if   [ "$1" == "--host" ]; then            _host=$2; shift;
    elif [ "$1" == "--port" ]; then            _port=$2; shift;
    elif [ "$1" == "--url" ]; then             _url=$2; shift;
    elif [ "$1" == "--urlParams" ]; then       _urlParams=$2; shift;
    elif [ "$1" == "-X" ]; then                _method="-X $2"; shift;
    elif [ "$1" == "--payload" ]; then         _payload=$2; shift;
    elif [ "$1" == "--noPayloadCheck" ]; then  _noPayloadCheck='on'; _forcedNoPayloadCheck='on'
    elif [ "$1" == "--payloadCheck" ]; then    _payloadCheck=$2; _noPayloadCheck='off'; shift;
    elif [ "$1" == "--servicePath" ]; then     _servicePath='--header "Fiware-ServicePath: '${2}'"'; shift;
    elif [ "$1" == "--tenant" ]; then          _tenant='--header "Fiware-Service: '${2}'"'; shift;
    elif [ "$1" == "--origin" ]; then          _origin='--header "Origin: '${2}'"'; shift;
    elif [ "$1" == "--correlator" ]; then      _correlator='--header "Fiware-Correlator: '${2}'"'; shift;
    elif [ "$1" == "-H" ]; then                _headers=${_headers}" --header \"$2\""; shift;
    elif [ "$1" == "--header" ]; then          _headers=${_headers}" --header \"$2\""; shift;
    elif [ "$1" == "--in" ]; then              _in="$2"; shift;
    elif [ "$1" == "--out" ]; then             _out="$2"; shift;
    elif [ "$1" == "--xauthToken" ]; then      _xauthToken='--header "X-Auth-Token: '${2}'"'; shift;
    elif [ "$1" == "-v" ]; then                _verbose=on;
    elif [ "$1" == "--verbose" ]; then         _verbose=on;
    elif [ "$1" == "--debug" ]; then           _debug=on;
    elif [ "$1" == "-d" ]; then                _debug=on;
    else                                       _xtra="$_xtra $1";
    fi
    shift
  done

  if [ "$_payload" != "" ]
  then
    _inFormat='--header "Content-Type: application/json"'
  fi

  #
  # Check the parameters
  #

  # 1. Do we have a URL present?
  if [ "$_url" == "" ]
  then
    echo "No URL";
    return 1;
  fi

  # 2. Payload can be either data or a path to a file - if data, a file is created with the data as contents
  _PAYLOAD=''
  if [ "$_payload" != "" ]
  then
    if [ -f "$_payload" ]
    then
      _PAYLOAD="-d @$_payload"
    else
      echo $_payload > /tmp/orionFuncTestPayload
      _PAYLOAD="-d @/tmp/orionFuncTestPayload"
    fi
  fi

  # 3. Fix for 'Content-Type' and 'Accept' short names 'xml' and 'json'
  if   [ "$_in"   == "application/xml" ];  then _in='xml';   fi
  if   [ "$_in"   == "application/json" ]; then _in='json';  fi
  if   [ "$_out"  == "application/xml" ];  then _out='xml';  fi
  if   [ "$_out"  == "application/json" ]; then _out='json'; fi
  if   [ "$_out"  == "text/plain" ];       then _out='text'; fi

  if   [ "$_in"  == "xml" ];   then _inFormat='--header "Content-Type: application/xml"'
  elif [ "$_in"  == "json" ];  then _inFormat='--header "Content-Type: application/json"'
  elif [ "$_in"  == "text" ];  then _inFormat='--header "Content-Type: text/plain"'
  elif [ "$_in"  != "" ];      then _inFormat='--header "Content-Type: '${_in}'"'
  fi

  # Note that payloadCheckFormat is also json in the case of --in xml, as the CB also returns error in JSON in this case
  if   [ "$_out" == "xml" ];   then _outFormat='--header "Accept: application/xml"'; payloadCheckFormat='json'
  elif [ "$_out" == "json" ];  then _outFormat='--header "Accept: application/json"'; payloadCheckFormat='json'
  elif [ "$_out" == "text" ];  then _outFormat='--header "Accept: text/plain"'; _noPayloadCheck='on'
  elif [ "$_out" == "any" ];   then _outFormat='--header "Accept: */*"'; _noPayloadCheck='on'
  elif [ "$_out" == "EMPTY" ]; then _outFormat='--header "Accept:"'
  elif [ "$_out" != "" ];      then _outFormat='--header "Accept: '${_out}'"'; _noPayloadCheck='off'
  fi

  if [ "$_payloadCheck" != "" ]
  then
    payloadCheckFormat=$_payloadCheck
    _noPayloadCheck='off'
  fi

  if [ "$_forcedNoPayloadCheck" == 'on' ]
  then
    _noPayloadCheck='on'
  fi

  dMsg $_in: $_in
  dMsg _out: $_out
  dMsg _outFormat: $_outFormat
  dMsg _inFormat: $_inFormat
  dMsg payloadCheckFormat: $payloadCheckFormat
  dMsg _noPayloadCheck: $_noPayloadCheck

  #
  # Assemble the command
  #
  command='curl "'$_host':'${_port}${_url}'"'

  if [ "$_urlParams"   != "" ]; then  command=${command}'?'${_urlParams};   fi
  if [ "$_PAYLOAD"     != "" ]; then  command=${command}' '${_PAYLOAD};     fi
  if [ "$_method"      != "" ]; then  command=${command}' '${_method};      fi
  if [ "$_tenant"      != "" ]; then  command=${command}' '${_tenant};      fi
  if [ "$_servicePath" != "" ]; then  command=${command}' '${_servicePath}; fi
  if [ "$_inFormat"    != "" ]; then  command=${command}' '${_inFormat};    fi
  if [ "$_outFormat"   != "" ]; then  command=${command}' '${_outFormat};   fi
  if [ "$_origin"      != "" ]; then  command=${command}' '${_origin};      fi
  if [ "$_correlator"  != "" ]; then  command=${command}' '${_correlator};  fi
  if [ "$_xauthToken"  != "" ]; then  command=${command}' '${_xauthToken};  fi
  if [ "$_headers"     != "" ]; then  command=${command}' '${_headers};     fi
  if [ "$_xtra"        != "" ]; then  command=${command}' '${_xtra};        fi

  command=${command}' --header "Expect:"'
  command=${command}' -s -S --dump-header /tmp/httpHeaders.out'
  dMsg command: $command

  #
  # Execute the command
  #
  dMsg Executing the curl-command
  eval $command > /tmp/orionCurl.response 2> /dev/null
  _response=$(cat /tmp/orionCurl.response)

  if [ ! -f /tmp/httpHeaders.out ]
  then
    echo "Broker seems to have died ..."
  else
    _responseHeaders=$(cat /tmp/httpHeaders.out)
    #
    # Remove "Connection: Keep-Alive" and "Connection: close" headers
    #
    sed '/Connection: Keep-Alive/d' /tmp/httpHeaders.out  > /tmp/httpHeaders2.out
    sed '/Connection: close/d'      /tmp/httpHeaders2.out > /tmp/httpHeaders.out
    sed '/Connection: Close/d'      /tmp/httpHeaders.out
  fi

  #
  # Unless we remove the HTTP header file, it will remain for the next execution
  #
  \rm -f /tmp/httpHeaders2.out /tmp/httpHeaders.out

  #
  # Print and beautify response body, if any - and if option --noPayloadCheck hasn't been set
  #
  if [ "$_noPayloadCheck" == "on" ]
  then
    cat /tmp/orionCurl.response
    echo
    rm -f /tmp/orionCurl.response
  else
    if [ "$_response" != "" ]
    then
      dMsg buffer to $payloadCheckFormat beautify: $_response

      if [ "$payloadCheckFormat" == json ] || [ "$payloadCheckFormat" == "" ]
      then
        vMsg Running python tool for $_response
        #
        # We need to apply pretty-print on _response. Otherwise positional processing used in .test
        # (e.g. to get SUB_ID typically grep and awk are used) will break
        #
        _response=$(echo $_response | python -mjson.tool)
        echo "$_response"
      else
        dMsg Unknown payloadCheckFormat
      fi
    fi
  fi
}

export -f dbInit
export -f dbList
export -f dbDrop
export -f dbResetAll
export -f brokerStart
export -f localBrokerStop
export -f localBrokerStart
export -f brokerStop
export -f accumulatorStart
export -f accumulatorStop
export -f accumulatorDump
export -f accumulator2Dump
export -f accumulator3Dump
export -f accumulatorCount
export -f accumulator2Count
export -f accumulator3Count
export -f accumulatorReset
export -f orionCurl
export -f dbInsertEntity
export -f mongoCmd
export -f vMsg
export -f dMsg
export -f valgrindSleep
export -f brokerStartAwait
export -f brokerStopAwait


build changes

build changes

remote: Support for password authentication was removed on August 13, 2021.
remote: Please see https://docs.github.com/en/get-started/getting-started-with-git/about-remote-repositories#cloning-with-https-urls for information on currently recommended modes of authentication.
fatal: Authentication failed for 'https://github.com/altamiracorp/honeycomb.git/'


only reload apache if it is active

only reload apache if it is active

the command `service apache2 reload` will cause a fatal error if apache
is not loaded.

this update first checks to see if apache is active, before triggering
the reload

remote: Support for password authentication was removed on August 13, 2021.
remote: Please see https://docs.github.com/en/get-started/getting-started-with-git/about-remote-repositories#cloning-with-https-urls for information on currently recommended modes of authentication.
fatal: Authentication failed for 'https://github.com/paulgeneres/homestead.git/'


Change start_client.sh script to pass IP

Apparently required for a working RMI-Communication

remote: Support for password authentication was removed on August 13, 2021.
remote: Please see https://docs.github.com/en/get-started/getting-started-with-git/about-remote-repositories#cloning-with-https-urls for information on currently recommended modes of authentication.
fatal: Authentication failed for 'https://github.com/BafDyce/alcatraz_ctf.git/'


updating start.sh

updating start.sh

remote: Support for password authentication was removed on August 13, 2021.
remote: Please see https://docs.github.com/en/get-started/getting-started-with-git/about-remote-repositories#cloning-with-https-urls for information on currently recommended modes of authentication.
fatal: Authentication failed for 'https://github.com/wenchenglai/firehose-ember-shared.git/'


Uses environment variables for adapter and name

remote: Support for password authentication was removed on August 13, 2021.
remote: Please see https://docs.github.com/en/get-started/getting-started-with-git/about-remote-repositories#cloning-with-https-urls for information on currently recommended modes of authentication.
fatal: Authentication failed for 'https://github.com/danstiner/docker-hubot.git/'


Reorder and categorize aliases, add -F flag to more and less alias.

remote: Support for password authentication was removed on August 13, 2021.
remote: Please see https://docs.github.com/en/get-started/getting-started-with-git/about-remote-repositories#cloning-with-https-urls for information on currently recommended modes of authentication.
fatal: Authentication failed for 'https://github.com/meyerlasse/Linux-Startup.git/'


Added gdiff function

Added gdiff function

remote: Support for password authentication was removed on August 13, 2021.
remote: Please see https://docs.github.com/en/get-started/getting-started-with-git/about-remote-repositories#cloning-with-https-urls for information on currently recommended modes of authentication.
fatal: Authentication failed for 'https://github.com/davidmatas/dotfiles.git/'


update git status alias

update git status alias

remote: Support for password authentication was removed on August 13, 2021.
remote: Please see https://docs.github.com/en/get-started/getting-started-with-git/about-remote-repositories#cloning-with-https-urls for information on currently recommended modes of authentication.
fatal: Authentication failed for 'https://github.com/artivilla99/dotfiles.git/'


use own yaio plantuml-servic

use own yaio plantuml-servic

remote: Support for password authentication was removed on August 13, 2021.
remote: Please see https://docs.github.com/en/get-started/getting-started-with-git/about-remote-repositories#cloning-with-https-urls for information on currently recommended modes of authentication.
fatal: Authentication failed for 'https://github.com/das-praktische-schreinerlein/your-all-in-one.git/'


update

update

remote: Support for password authentication was removed on August 13, 2021.
remote: Please see https://docs.github.com/en/get-started/getting-started-with-git/about-remote-repositories#cloning-with-https-urls for information on currently recommended modes of authentication.
fatal: Authentication failed for 'https://github.com/kenzietandun/pokapoka.git/'


Script to check that "too big" metadata is handled correctly by Cromwell in live environments

#!/bin/bash

set -euo pipefail

VAULT_TOKEN=$(cat /etc/vault-token-dsde)
WORKING_DIR=$PWD

DB_PASSWORD=$(docker run --rm \
    -e VAULT_TOKEN=$VAULT_TOKEN \
    broadinstitute/dsde-toolbox vault read \
    -format=json \
    secret/dsde/firecloud/alpha/cromwell/cromwell1/secrets \
    | jq '.data.db_password' \
    | sed -e 's/^"//' -e 's/"$//')

# Update the archive status to be 'Unarchived' (marked as NULL in the DB).
# This will trigger the carboniting process
WORKFLOW_ID="8fcff291-f4aa-401a-bf16-70e7fa76e01d"
docker run --rm \
    -e VAULT_TOKEN=$VAULT_TOKEN \
    broadinstitute/dsde-toolbox mysql-connect.sh -p firecloud -a cromwell1 -e alpha \
    "UPDATE WORKFLOW_METADATA_SUMMARY_ENTRY SET METADATA_ARCHIVE_STATUS = NULL WHERE WORKFLOW_EXECUTION_UUID = '${WORKFLOW_ID}';"

# Define a function to ensure that the carboniting has failed because it was too big. Then call it:
ATTEMPT=1
MAX_ATTEMPTS = 30 # At 10s per attempt, this allows 5 minutes for the carboniter to run.
check_carbonite_failed_too_big () {
  ARCHIVE_STATUS=$(
  docker run --rm \
     -e VAULT_TOKEN=$VAULT_TOKEN \
     broadinstitute/dsde-toolbox mysql-connect.sh -p firecloud -a cromwell1 -e alpha \
     "SELECT METADATA_ARCHIVE_STATUS FROM WORKFLOW_METADATA_SUMMARY_ENTRY WHERE WORKFLOW_EXECUTION_UUID = '${WORKFLOW_ID}';")

  if grep -q "TooLargeToArchive" <<< "${ARCHIVE_STATUS}"; then
    echo "$(date): [ATTEMPT ${ATTEMPT}] Workflow ${WORKFLOW_ID} was considered too big to carbonite (as expected)"
    return 0
  elif [[ "ATTEMPT" -gt "MAX_ATTEMPTS" ]]; then
    echo "$(date): [ATTEMPT ${ATTEMPT}] Workflow ${WORKFLOW_ID} took too long to re-carbonite..."
    exit 1
  elif grep -q "NULL" <<< "${ARCHIVE_STATUS}"; then
    echo "$(date): [ATTEMPT ${ATTEMPT}] Workflow ${WORKFLOW_ID} is still awaiting carbonite..."
    sleep 10
    check_carbonite_failed_too_big
  else
    echo "$(date): [ATTEMPT ${ATTEMPT}] Workflow ${WORKFLOW_ID} finished carboniting but not in a way we expected:"
    echo "${ARCHIVE_STATUS}"
    exit 1
}

check_carbonite_failed_too_big

error: pathspec 'automation/carbonite-too-big-check.sh' did not match any file(s) known to git


containers matrics

containers matrics

remote: Support for password authentication was removed on August 13, 2021.
remote: Please see https://docs.github.com/en/get-started/getting-started-with-git/about-remote-repositories#cloning-with-https-urls for information on currently recommended modes of authentication.
fatal: Authentication failed for 'https://github.com/gitarte/redis-sentinel-nodejs-docker.git/'


Added all api docs to tarball created by 'make dist'

git-svn-id: 938b416e21db2f01dc5c93b2a51c543066b9c9b8@10720 a5903396-d722-0410-b921-86c7d4935375

remote: Support for password authentication was removed on August 13, 2021.
remote: Please see https://docs.github.com/en/get-started/getting-started-with-git/about-remote-repositories#cloning-with-https-urls for information on currently recommended modes of authentication.
fatal: Authentication failed for 'https://github.com/techhead/wsf_c.git/'


initial code commit

initial code commit

#!/bin/bash

#                            hdd-spindown.sh
#
#        Automatic Disk Standby using Kernel diskstats and hdparm
#             2011-2015 by Alexander Koch <lynix47@gmail.com>

# configuration file, (ba)sh-style
CONFIG="/etc/hdd-spindown.rc"

# default setting for watch interval: 300s
INTERV=${CONF_INT:-300}

function check_req() {
	FAIL=0
	for CMD in $@; do
		which $CMD >/dev/null && continue
		echo "error: unable to execute: '$CMD'"
		FAIL=1
	done
	[ $FAIL -ne 0 ] && exit 1
}

function log() {
	logger -t "hdd-spindown.sh" --id=$$ "$1"
}

function selftest_active() {
	smartctl -a "/dev/$1" | grep -q "Self-test routine in progress"
	return $?
}

function dev_stats() {
	awk '{printf "%s|%s\n", $1, $5}' < "/sys/block/$1/stat"
}

function check_dev() {
	NUM=$1

	DEV="${DEVICES[$NUM]}"
	if ! [ -e "/dev/$DEV" ]; then
		if [ -L "/dev/disk/by-id/$DEV" ]; then
			DEV="$(basename "$(readlink "/dev/disk/by-id/$DEV")")"
			log "recognized disk: ${DEVICES[$NUM]} --> $DEV"
			DEVICES[$NUM]="$DEV"
		else
			log "device not found: $DEV, skipping"
			return 1
		fi
	fi

	[ -z "${STAMP[$NUM]}" ] && STAMP[$NUM]=$(date +%s)

	COUNT_NEW="$(dev_stats "$DEV")"
	if [ "${COUNT[$NUM]}" == "$COUNT_NEW" ]; then
		if [ $(($(date +%s) - ${STAMP[$NUM]})) -ge ${TIMEOUT[$NUM]} ]; then
			# omit spindown if SMART Self-Test in progress
			selftest_active "$DEV" && return 0
			if hdparm -C "/dev/$DEV" | grep -q active; then
				log "suspending $DEV"
				hdparm -qy "/dev/$DEV"
				if [ $? -gt 0 ]; then
					log "failed to suspend $DEV"
					return 1
				fi
			fi
		fi
	else
		COUNT[$NUM]="$COUNT_NEW"
		STAMP[$NUM]=$(date +%s)
	fi
}

# check prerequisites
check_req date awk hdparm smartctl logger

# check config file
if ! [ -r "$CONFIG" ]; then
	echo "error: unable to read config file '$CONFIG', aborting."
	exit 1
fi
source "$CONFIG"
if [ -z "$CONF_DEV" ]; then
	echo "error: missing configuration parameter 'CONFIG_DEV', aborting."
	exit 1
fi

# initialize device arrays
I_MAX=$((${#CONF_DEV[@]} - 1))
for I in $(seq 0 $I_MAX); do
	DEVICES[$I]="$(echo "${CONF_DEV[$I]}" | cut -d '|' -f 1)"
	TIMEOUT[$I]="$(echo "${CONF_DEV[$I]}" | cut -d '|' -f 2)"
done

# main loop
while true; do
	for I in $(seq 0 $I_MAX); do
		check_dev $I
	done

	sleep $INTERV
done

error: pathspec 'hdd-spindown.sh' did not match any file(s) known to git


Add cht.sh

Add cht.sh

#!/bin/sh
#
## The only cheat sheet you need
##  / __| '_ \ / _ \/ _` | __| / __| '_ \ \ \     Unified access to the best
## | (__| | | |  __/ (_| | |_ _\__ \ | | |/ /     community driven documentation
## \___|_| |_|\___|\__,_|\__(_)___/_| |_/_/      repositories of the world
##
# [X] open section
# [X] one shot mode
# [X] usage info
# [X] dependencies check
# [X] help
# [X] yank/y/copy/c
# [X] Y/C
# [X] eof problem
# [X] more
# [X] stealth mode
#
# here are several examples for the stealth mode:
#
# zip lists
# list permutation
# random list element
# reverse a list
# read json from file
# append string to a file
# run process in background
# count words in text counter
# group elements list

__CHTSH_VERSION=4
__CHTSH_DATETIME="2018-07-08 22:26:46 +0200"

export LESSSECURE=1
STEALTH_MAX_SELECTION_LENGTH=5

case `uname -s` in
  Darwin) is_macos=yes ;;
  *) is_macos=no ;;
esac

# for KSH93
if echo $KSH_VERSION | grep -q ' 93' && ! local foo 2>/dev/null; then
  alias local=typeset
fi

# any better test not involving either OS matching or actual query?
if [ `uname -s` = OpenBSD ] && [ -x /usr/bin/ftp ]; then
  curl() {
    local opt args="-o -"
    while getopts "b:s" opt; do
      case $opt in
        b) args="$args -c $OPTARG";;
        s) args="$args -M -V";;
        *) echo "internal error: unsupported cURL option '$opt'" >&2; exit 1;;
      esac
    done
    shift $(($OPTIND - 1))
    /usr/bin/ftp $args "$@"
  }
fi

get_query_options()
{
  local query="$*"
  if [ -n "$CHTSH_QUERY_OPTIONS" ]; then
    case $query in
      *\?*)   query="$query&${CHTSH_QUERY_OPTIONS}";;
      *)      query="$query?${CHTSH_QUERY_OPTIONS}";;
    esac
  fi
  printf "%s" "$query"
}

do_query()
{
  local query="$*"
  local b_opts=
  local uri="https://cht.sh/\"\$(get_query_options $query)\""

  if [ -e "$HOME/.cht.sh/id" ]; then
    b_opts="-b \"\$HOME/.cht.sh/id\""
  fi

  eval curl $b_opts -s $uri > "$TMP1"

  if [ -z "$lines" ] || [ "$(wc -l "$TMP1" | awk '{print $1}')" -lt "$lines" ]; then
    cat "$TMP1"
  else
    ${PAGER:-$defpager} "$TMP1"
  fi
}

prepare_query()
{
  local section="$1"; shift
  local input="$1"; shift
  local arguments="$1"

  local query
  if [ -z "$section" ] || [ x"${input}" != x"${input#/}" ]; then
    query=$(printf %s "$input" | sed 's@ @/@; s@ @+@g')
  else
    query=$(printf %s "$section/$input" | sed 's@ @+@g')
  fi

  [ -n "$arguments" ] && arguments="?$arguments"
  printf %s "$query$arguments"
}

get_list_of_sections()
{
  curl -s https://cht.sh/:list | grep -v '/.*/' | grep '/$' | xargs
}

gen_random_str()
(
  len=$1
  if command -v openssl >/dev/null; then
    openssl rand -base64 $(($len*3/4)) | awk -v ORS='' //
  else
    rdev=/dev/urandom
    for d in /dev/{srandom,random,arandom}; do
      test -r $d && rdev=$d
    done
    if command -v hexdump >/dev/null; then
      hexdump -vn $(($len/2)) -e '1/1 "%02X" 1 ""' $rdev
    elif command -v xxd >/dev/null; then
      xxd -l $(($len/2)) -ps $dev | awk -v ORS='' //
    else
      cd /tmp
      s=
      while [ $(echo "$s" | wc -c) -lt $len ]; do
        s="$s$(mktemp -u XXXXXXXXXX)"
      done
      printf %.${len}s "$s"
    fi
  fi
)

if [ -e "$HOME"/.cht.sh/cht.sh.conf ]; then
  # shellcheck disable=SC1090,SC2002
  . "$HOME"/.cht.sh/cht.sh.conf
fi

if [ "$1" = --read ]; then
  read -r a || a=exit
  printf "%s\n" "$a"
  exit 0
elif [ x"$1" = x--help ] || [ -z "$1" ]; then
  cat <<EOF
Usage:

    ${0##*/} --help           show this help
    ${0##*/} #                show extended help
    ${0##*/} --shell [LANG]   shell mode (open LANG if specified)
    ${0##*/} QUERY            process QUERY and exit
EOF
  exit 0
elif [ x"$1" = x--shell ]; then
  shell_mode=yes
  shift
fi

prompt="cht.sh"

prompt="cht.sh"
opts=""
input=""
for o; do
  if [ x"$o" != x"${o#-}" ]; then
    opts="${opts}${o#-}"
  else
    input="$input $o"
  fi
done
query=$(echo "$input" | sed 's@ *$@@; s@^ *@@; s@ @/@; s@ @+@g')

if [ "$shell_mode" != yes ]; then
  curl -s https://cht.sh/"$(get_query_options "$query")"
  exit 0
else
  new_section="$1"
  valid_sections=$(get_list_of_sections)
  valid=no; for q in $valid_sections; do [ "$q" = "$new_section/" ] && { valid=yes; break; }; done

  if [ "$valid" = yes ]; then
    section="$new_section"
    this_query="$(echo "$input" | sed 's@ *[^ ]* *@@')"
    this_prompt="\033[0;32mcht.sh/$section>\033[0m "
  else
    this_query="$input"
    this_prompt="\033[0;32mcht.sh>\033[0m "
  fi
  if [ -n "$this_query" ] && [ -z "$CHEATSH_RESTART" ]; then
    printf "$this_prompt$this_query\n"
    curl -s https://cht.sh/"$(get_query_options "$query")"
  fi
fi

if [ "$is_macos" != yes ]; then
  command -v xsel >/dev/null ||   echo 'DEPENDENCY: please install "xsel" for "copy"' >&2
fi
command -v rlwrap >/dev/null || { echo 'DEPENDENCY: install "rlwrap" to use cht.sh in the shell mode' >&2; exit 1; }
command -v curl   >/dev/null || { echo 'DEPENDENCY: install "curl" to use cht.sh' >&2; exit 1; }

mkdir -p "$HOME/.cht.sh/"
lines=$(tput lines)

if command -v less >/dev/null; then
  defpager="less -R"
elif command -v more >/dev/null; then
  defpager=more
else
  defpager=cat
fi

TMP1=$(mktemp /tmp/cht.sh.XXXXXXXXXXXXX)
trap 'rm -f $TMP1 $TMP2' EXIT
trap 'true' INT

if ! [ -e "$HOME/.cht.sh/.hushlogin" ] && [ -z "$this_query" ]; then
  echo "type 'help' for the cht.sh shell help"
fi

while true; do
  if [ "$section" != "" ]; then
    full_prompt="$prompt/$section> "
  else
    full_prompt="$prompt> "
  fi

  input=$(
    rlwrap -H "$HOME/.cht.sh/history" -pgreen -C cht.sh -S "$full_prompt" sh "$0" --read | sed 's/ *#.*//'
  )

  case "$input" in
    '?'|h|help)
      cat <<EOF
help    - show this help
hush    - do not show the 'help' string at start anymore
cd LANG - change the language context
copy    - copy the last answer in the clipboard (aliases: yank, y, c)
ccopy   - copy the last answer w/o comments (cut comments; aliases: cc, Y, C)
exit    - exit the cheat shell (aliases: quit, ^D)
id [ID] - set/show an unique session id ("reset" to reset, "remove" to remove)
stealth - stealth mode (automatic queries for selected text)
update  - self update (only if the scriptfile is writeable)
version - show current cht.sh version
/:help  - service help
QUERY   - space ceparated query staring (examples are below)
              cht.sh> python zip list
              cht.sh/python> zip list
              cht.sh/go> /python zip list
EOF
      continue
      ;;
    hush)
      mkdir -p $HOME/.cht.sh/ && touch $HOME/.cht.sh/.hushlogin && echo "Initial 'use help' message was disabled"
      continue
      ;;
    cd)
      section=""
      continue
      ;;
    "cd "*)
      new_section=$(echo "$input" | sed 's/cd  *//; s@/*$@@; s@^/*@@')
      if [ -z "$new_section" ] || [ ".." = "$new_section" ]; then
        section=""
      else
        valid_sections=$(get_list_of_sections)
        valid=no; for q in $valid_sections; do [ "$q" = "$new_section/" ] && { valid=yes; break; }; done
        if [ "$valid" = no ]; then
          echo "Invalid section: $new_section"
          echo "Valid sections:"
          echo $valid_sections | xargs printf "%-10s\n" | tr ' ' .  | xargs -n 10 | sed 's/\./ /g; s/^/  /'
          continue
        else
          section="$new_section"
        fi
      fi
      continue
      ;;
    exit|quit)
      break
      ;;
    copy|yank|c|y)
      if [ -z "$DISPLAY" ]; then
        echo copy: supported only in the Desktop version
        continue
      fi
      if [ -z "$input" ]; then
        echo copy: Make at least one query first.
        continue
      else
        curl -s https://cht.sh/"$(get_query_options "$query"?T)" > "$TMP1"
        if [ "$is_macos" != yes ]; then
          xsel -i < "$TMP1"
        else
          cat "$TMP1" | pbcopy
        fi
        echo "copy: $(wc -l "$TMP1" | awk '{print $1}') lines copied to the selection"
        continue
      fi
      ;;
    ccopy|cc|C|Y)
      if [ -z "$DISPLAY" ]; then
        echo copy: supported only in the Desktop version
        continue
      fi
      if [ -z "$input" ]; then
        echo copy: Make at least one query first.
        continue
      else
        curl -s https://cht.sh/"$(get_query_options "$query"?TQ)" > "$TMP1"
        if [ "$is_macos" != yes ]; then
          xsel -i < "$TMP1"
        else
          cat "$TMP1" | pbcopy
        fi
        echo "copy: $(wc -l "$TMP1" | awk '{print $1}') lines copied to the selection"
        continue
      fi
      ;;
    id|"id "*)
      id_file="$HOME/.cht.sh/id"

      if [ id = "$input" ]; then
        new_id=""
      else
        new_id=$(echo "$input" | sed 's/id  *//; s/ *$//; s/ /+/g')
      fi
      if [ "$new_id" = remove ]; then
        if [ -e "$id_file" ]; then
          rm -f -- "$id_file" && echo "id is removed"
        else
          echo "id was not set, so you can't remove it"
        fi
        continue
      fi
      if [ -n "$new_id" ] && [ reset != "$new_id" ] && [ $(/bin/echo -n "$new_id" | wc -c) -lt 16 ]; then
        echo "ERROR: $new_id: Too short id. Minimal id length is 16. Use 'id reset' for a random id"
        continue
      fi
      if [ -z "$new_id" ]; then
        # if new_id is not specified check if we have some id already
        # if yes, just show it
        # if not, generate a new id
        if [ -e "$id_file" ]; then
          echo $(awk '$6 == "id" {print $NF}' <"$id_file" | tail -n 1)
          continue
        else
          new_id=reset
        fi
      fi
      if [ "$new_id" = reset ]; then
        new_id=$(gen_random_str 12)
      else
        echo WARNING: if someone gueses your id, he can read your cht.sh search history
      fi
      if [ -e "$id_file" ] && grep -q '\tid\t[^\t][^\t]*$' "$id_file" 2> /dev/null; then
        sed -i 's/\tid\t[^\t][^\t]*$/ id '"$new_id"'/' "$id_file"
      else
        if ! [ -e "$id_file" ]; then
          printf '#\n\n' > "$id_file"
        fi
        printf ".cht.sh\tTRUE\t/\tTRUE\t0\tid\t$new_id\n" >> "$id_file"
      fi
      echo "$new_id"
      continue
      ;;
    stealth|"stealth "*)
        if [ "$input" != stealth ]; then
          arguments=$(echo "$input" | sed 's/stealth //; s/ /\&/')
        fi
        trap break INT
        if [ "$is_macos" = yes ]; then
          past=$(pbpaste)
        else
          past=$(xsel -o)
        fi
        printf "\033[0;31mstealth:\033[0m you are in the stealth mode; select any text in any window for a query\n"
        printf "\033[0;31mstealth:\033[0m selections longer than $STEALTH_MAX_SELECTION_LENGTH words are ignored\n"
        if [ -n "$arguments" ]; then
          printf "\033[0;31mstealth:\033[0m query arguments: ?$arguments\n"
        fi
        printf "\033[0;31mstealth:\033[0m use ^C to leave this mode\n"
        while true; do
          if [ "$is_macos" = yes ]; then
            current=$(pbpaste)
          else
            current=$(xsel -o)
          fi
          if [ "$past" != "$current" ]; then
            past=$current
            current_text="$(echo $current | tr -c '[a-zA-Z0-9]' ' ')"
            if [ $(echo $current_text | wc -w) -gt "$STEALTH_MAX_SELECTION_LENGTH" ]; then
              echo "\033[0;31mstealth:\033[0m selection length is longer than $STEALTH_MAX_SELECTION_LENGTH words; ignoring"
              continue
            else
              printf "\n\033[0;31mstealth: \033[7m $current_text\033[0m\n"
              query=$(prepare_query "$section" "$current_text" "$arguments")
              do_query "$query"
            fi
          fi
          sleep 1;
        done
        trap - INT
        continue
      ;;
    update)
      [ -w "$0" ] || { echo "The script is readonly; please update manually: curl -s https://cht.sh/:bash | sudo tee $0"; continue; }
      TMP2=$(mktemp /tmp/cht.sh.XXXXXXXXXXXXX)
      curl -s https://cht.sh/:cht.sh > "$TMP2"
      if ! cmp "$0" "$TMP2" > /dev/null 2>&1; then
        if grep -q ^__CHTSH_VERSION= "$TMP2"; then
          # section was vaildated by us already
          args="--shell $section"
          cp "$TMP2" "$0" && echo "Updated. Restarting..." && rm "$TMP2" && CHEATSH_RESTART=1 exec "$0" $args
        else
          echo "Something went wrong. Please update manually"
        fi
      else
        echo "cht.sh is up to date. No update needed"
      fi
      rm -f "$TMP2" > /dev/null 2>&1
      continue
      ;;
    version)
      insttime=$(ls -l -- "$0" | sed 's/  */ /g' | cut -d ' ' -f 6-8)
      echo "cht.sh version $__CHTSH_VERSION of $__CHTSH_DATETIME; installed at: $insttime"
      TMP2=$(mktemp /tmp/cht.sh.XXXXXXXXXXXXX)
      if curl -s https://cht.sh/:cht.sh > "$TMP2"; then
        if ! cmp "$0" "$TMP2" > /dev/null 2>&1; then
          echo "Update needed (type 'update' for that)".
        else
          echo "Up to date. No update needed"
        fi
      fi
      rm -f "$TMP2" > /dev/null 2>&1
      continue
      ;;
    "")
      continue
      ;;
  esac

  query=$(prepare_query "$section" "$input")
  do_query "$query"
done

error: pathspec 'home/bin/cht.sh' did not match any file(s) known to git


Clearer command check

Clearer command check

if [ -n "$(command -v hub)" ]; then
  # hub
  function git(){hub "$@"}
fi

# git
alias g=git
alias gst='git status -sb'
alias gp='git push'
alias gpo='gp origin master'
alias gps='gp stage'
alias gd='git diff'
alias gdc='git diff --cached'
alias ga='git add'
alias gcl='git config --list'
alias gc='git commit -v'
alias gca='git commit -v -a'
alias gco='git checkout'
alias gb='git branch'
alias gba='git branch -a'
alias gx=gitx

if [ "$commands[(I)hub]" ]; then
  # hub
  function git(){hub "$@"}
fi

# git
alias g=git
alias gst='git status -sb'
alias gp='git push'
alias gpo='gp origin master'
alias gps='gp stage'
alias gd='git diff'
alias gdc='git diff --cached'
alias ga='git add'
alias gcl='git config --list'
alias gc='git commit -v'
alias gca='git commit -v -a'
alias gco='git checkout'
alias gb='git branch'
alias gba='git branch -a'
alias gx=gitx


add a build script

add a build script

#!/bin/sh

cabal update
cabal sandbox init
cabal configure
cabal build

error: pathspec 'dsl/build.sh' did not match any file(s) known to git


Added bash script for watchers

Added bash script for watchers

error: pathspec 'watch.sh' did not match any file(s) known to git


add steering script

add steering script

#!/bin/bash

usage() {
        echo "usage: $0 <def_file> <rule_file>"
        exit 1
}

if (( $# < 2)); then
        usage
fi

while (($# > 0)); do
        case $1 in
        "-b" | "--build") BUILDONLY=true; shift;;
        \-*) usage;;
        *) break;;
        esac
done

DEFS="$1"
RULES="$2"

[[ ! -d bin ]] && mkdir bin
make build &&
./bin/parser "$DEFS" &&
./parser/create_rules.sh "$RULES" &&
make runtime

if [[ $? -eq 0 ]]; then
        [[ $BUILDONLY ]] && ./bin/yafam || exit 0
else
        echo "there was an error building yafam"
        exit 2
fi

error: pathspec 'yafam.sh' did not match any file(s) known to git


- Added release script

- Added release script

#!/bin/bash
function advance_version () {
    local v=$1
    # Get the last number. First remove any suffixes (such as '-SNAPSHOT').
    local cleaned=`echo $v | sed -e 's/[^0-9][^0-9]*$//'`
    local last_num=`echo $cleaned | sed -e 's/[0-9]*\.//g'`
    local next_num=$(($last_num+1))
    # Finally replace the last number in version string with the new one.
    echo $v | sed -e "s/[0-9][0-9]*\([^0-9]*\)$/$next_num/"
}

export GIT_MERGE_AUTOEDIT=no
VERSION=`mvn org.apache.maven.plugins:maven-help-plugin:2.1.1:evaluate -Dexpression=project.version | grep -v '\[' | grep -v Download`
RELEASE_VERSION=`echo $VERSION | sed -e 's/[^0-9][^0-9]*$//'`
echo $RELEASE_VERSION
NEW_VERSION=`advance_version $VERSION`
NEW_VERSION=$NEW_VERSION-SNAPSHOT
echo $NEW_VERSION

git flow release start $RELEASE_VERSION
mvn versions:set -DnewVersion=$RELEASE_VERSION versions:commit -q
git add -A
git commit -m "Bump version to $RELEASE_VERSION"
git flow release finish -m "$RELEASE_VERSION" $RELEASE_VERSION
mvn versions:set -DnewVersion=$NEW_VERSION versions:commit -q
git add -A
git commit -m "Bump version to $NEW_VERSION"
git push origin --all
git push origin --tags

error: pathspec 'dorelease.sh' did not match any file(s) known to git


Added versioning for featureflow.js cdn

Added versioning for featureflow.js cdn

#!/usr/bin/env bash
#DEFAULT="default"
#PROFILE=${AWS_PROFILE:-$DEFAULT}
#BUCKET=site-staging
#DIR=./dist
#aws s3 sync $DIR s3://$BUCKET/

PACKAGE_VERSION=$(cat package.json \
  | grep version \
  | head -1 \
  | awk -F: '{ print $2 }' \
  | sed 's/[",]//g' \
  | tr -d '[[:space:]]')

aws s3 cp dist/featureflow.js s3://cdn.featureflow.io/featureflow.js
aws s3 cp dist/featureflow.min.js s3://cdn.featureflow.io/featureflow.min.js
aws s3 cp dist/featureflow.js s3://cdn.featureflow.io/v$PACKAGE_VERSION/featureflow.js
aws s3 cp dist/featureflow.min.js s3://cdn.featureflow.io/v$PACKAGE_VERSION/featureflow.min.js

#!/usr/bin/env bash
#DEFAULT="default"
#PROFILE=${AWS_PROFILE:-$DEFAULT}
#BUCKET=site-staging
#DIR=./dist
#aws s3 sync $DIR s3://$BUCKET/
aws s3 cp dist/featureflow.js s3://cdn.featureflow.io/featureflow.js
aws s3 cp dist/featureflow.min.js s3://cdn.featureflow.io/featureflow.min.js
aws s3 cp dist/featureflow.js s3://controls.featureflow.io/featureflow.js
aws s3 cp dist/featureflow.min.js s3://controls.featureflow.io/featureflow.min.js


Refactor the webui integration test to support external image

This change refactors the integration test so that it can be
run using a local image or an external image. It adds a test/e2e.sh
script for running the test which handles the details of pushing
images to integrated/external registries or allowing the image
to be referenced directly from the host in the case of 'oc cluster up'.

#!/bin/bash

TOP_DIR=$(readlink -f `dirname "${BASH_SOURCE[0]}"` | grep -o '.*/oshinko-webui')
PROJECT=$(oc project -q)

WEBUI_TEST_SECURE=${WEBUI_TEST_SECURE:-false}
WEBUI_TEST_LOCAL_IMAGE=${WEBUI_TEST_LOCAL_IMAGE:-true}
WEBUI_TEST_INTEGRATED_REGISTRY=${WEBUI_TEST_INTEGRATED_REGISTRY:-}
WEBUI_TEST_EXTERNAL_REGISTRY=${WEBUI_TEST_EXTERNAL_REGISTRY:-}
WEBUI_TEST_EXTERNAL_USER=${WEBUI_TEST_EXTERNAL_USER:-}
WEBUI_TEST_EXTERNAL_PASSWORD=${WEBUI_TEST_EXTERNAL_PASSWORD:-}
WEBUI_TEST_RESOURCES=${WEBUI_TEST_RESOURCES:-$TOP_DIR/tools/resources.yaml}
if [ -z "$WEBUI_TEST_IMAGE" ]; then
    if [ "$WEBUI_TEST_LOCAL_IMAGE" == true ]; then
        WEBUI_TEST_IMAGE=oshinko-webui:latest
    else
        WEBUI_TEST_IMAGE=docker.io/radanalyticsio/oshinko-webui
    fi
fi

function print_test_env {
    echo Using image $WEBUI_TEST_IMAGE
    echo Using resources.yaml from $WEBUI_TEST_RESOURCES

    if [ "$WEBUI_TEST_LOCAL_IMAGE" != true ]; then
	echo WEBUI_TEST_LOCAL_IMAGE = $WEBUI_TEST_LOCAL_IMAGE, webui image is external, ignoring registry env vars
    elif [ -n "$WEBUI_TEST_EXTERNAL_REGISTRY" ]; then
        echo Using external registry $WEBUI_TEST_EXTERNAL_REGISTRY
        if [ -z "$WEBUI_TEST_EXTERNAL_USER" ]; then
            echo "Error: WEBUI_TEST_EXTERNAL_USER not set!"
	    exit 1
        else
	    echo Using external registry user $WEBUI_TEST_EXTERNAL_USER
        fi
        if [ -z "$WEBUI_TEST_EXTERNAL_PASSWORD" ]; then
            echo "WEBUI_TEST_EXTERNAL_PASSWORD not set, assuming current docker login"
        else
            echo External registry password set
        fi
    elif [ -n "$WEBUI_TEST_INTEGRATED_REGISTRY" ]; then
        echo Using integrated registry $WEBUI_TEST_INTEGRATED_REGISTRY
    else
        echo Not using external or integrated registry
    fi
}

function push_image {
    # The ip address of an internal/external registry may be set to support running against
    # an openshift that is not "oc cluster up" when using images that have been built locally.
    # In the case of "oc cluster up", the docker on the host is available from openshift so
    # no special pushes of images have to be done.
    # In the case of a "normal" openshift cluster, a local image we'll use for build has to be
    # available from the designated registry.
    # If we're using a pyspark image already in an external registry, openshift can pull it from
    # there and we don't have to do anything.
    local user=
    local password=
    local pushproj=
    local pushimage=
    local registry=
    if [ "$WEBUI_TEST_LOCAL_IMAGE" == true ]; then
	if [ -n  "$WEBUI_TEST_EXTERNAL_REGISTRY" ]; then
	    user=$WEBUI_TEST_EXTERNAL_USER
	    password=$WEBUI_TEST_EXTERNAL_PASSWORD
	    pushproj=$user
	    pushimage=scratch-oshinko-webui
	    registry=$WEBUI_TEST_EXTERNAL_REGISTRY
	elif [ -n "$WEBUI_TEST_INTEGRATED_REGISTRY" ]; then
	    user=$(oc whoami)
	    password=$(oc whoami -t)
	    pushproj=$PROJECT
	    pushimage=oshinko-webui
	    registry=$WEBUI_TEST_INTEGRATED_REGISTRY
	fi
    fi
    if [ -n "$registry" ]; then
	set +e
	docker login --help | grep email &> /dev/null
	res=$?
	set -e
	if [ -n "$password" ]; then
	    if [ "$res" -eq 0 ]; then
		docker login -u ${user} -e jack@jack.com -p ${password} ${registry}
	    else
		docker login -u ${user} -p ${password} ${registry}
	    fi
	fi
	docker tag ${WEBUI_TEST_IMAGE} ${registry}/${pushproj}/${pushimage}
	docker push ${registry}/${pushproj}/${pushimage}
	OSHINKO_WEB_IMAGE=${registry}/${pushproj}/${pushimage}
    else
	OSHINKO_WEB_IMAGE=$WEBUI_TEST_IMAGE
    fi
}

function tweak_template {
    if [ "$WEBUI_TEST_LOCAL_IMAGE" == true ] && [ -z "$WEBUI_TEST_INTEGRATED_REGISTRY" ] && [ -z "$WEBUI_TEST_EXTERNAL_REGISTRY" ]; then
	echo Updating $WEBUI_TEST_RESOURCES to pull local webui image
        if [ -f "$WEBUI_TEST_RESOURCES" ]; then
            cp "$WEBUI_TEST_RESOURCES" `pwd`/resources_mod.yaml
        else
            wget "$WEBUI_TEST_RESOURCES" -O `pwd`/resources_mod.yaml
        fi
        # This sed command finds the line with image: ${OSHINKO_WEB_IMAGE}
        # The next line sets the pull policy in the standard template (this should be maintained)
        # !b;n;s throws away the current processing, reads the next line, and starts a new substitution
        # Ultimately, this sets the pull policy for the web image to IfNotPresent so that an 'oc cluster up'
        # case can just reference the image from the host
        sed -i -r '/image.*OSHINKO_WEB_IMAGE/!b;n;s/(.*imagePullPolicy: *)Always/\1IfNotPresent/' resources_mod.yaml
        WEBUI_TEST_RESOURCES=`pwd`/resources_mod.yaml
    fi
}

# Modify the template if we're using a local image with no registry, i.e. we're in an oc cluster up case
# In this case we don't need a push at all, but we to have a pullpolicy of IfNotPresent
tweak_template
print_test_env
# Push the image if required
push_image

set +e
oc create -f $WEBUI_TEST_RESOURCES
oc policy add-role-to-user admin system:serviceaccount:$PROJECT:oshinko
if [ "$WEBUI_TEST_SECURE" == true ]; then
    oc new-app --template=oshinko-webui-secure -p OSHINKO_WEB_IMAGE=$OSHINKO_WEB_IMAGE
else
    oc new-app --template=oshinko-webui -p OSHINKO_WEB_IMAGE=$OSHINKO_WEB_IMAGE
fi
oc create configmap storedconfig --from-literal=mastercount=1 --from-literal=workercount=4
set -e

error: pathspec 'test/e2e.sh' did not match any file(s) known to git


Create gifrender.sh

Create gifrender.sh

convert input.gif frame.png

for f in *.png; do
./distvolve l $f b 8 s rendered.$f
done

ffmpeg -i %05d.render.png output.mp4

error: pathspec 'gifrender.sh' did not match any file(s) known to git


Início projeto backup mysql

Início projeto backup mysql

Signed-off-by: Rodrigo Cabral <793e1455c1fdf3353d934664fdf5c66470c91d73@gmail.com>

remote: Support for password authentication was removed on August 13, 2021.
remote: Please see https://docs.github.com/en/get-started/getting-started-with-git/about-remote-repositories#cloning-with-https-urls for information on currently recommended modes of authentication.
fatal: Authentication failed for 'https://github.com/rodrixcornell/bin.git/'


POC script of recording terminal and audio

Recording terminal turning it into gif then an mp4.
Audio is recorded during the terminal session, and is added to the mp4 file.

#!/bin/bash
ASCII_FNAME=term_session
GIF_FNAME=term_session.gif
MP4_FNAME=term_session.mp4
WAV_FNAME=term_session.wav
FINAL_FNAME=full_term_session.mp4

# Start audio recording and place in background and then start terminal recording
sox -q -d $WAV_FNAME & asciinema rec $ASCII_FNAME
# Kill the sox program
ps -ef | grep sox | awk '{print $2}' | head -n 1 | xargs kill -15
# Upload the gif to online
url=$(asciinema upload $ASCII_FNAME)
echo "URL: $url"
# Extract the ID from the URL
id=$(echo $url | awk -F '/' '{print $5}')
echo "ID: $id"
# Turn the ASCIINEMA file into a gif
res=`./asciinema2gif --output $GIF_FNAME $id`
echo "$res"
# Turn that gif into an mp4 file
ffmpeg -f gif -i $GIF_FNAME -vcodec libx264 -pix_fmt yuv420p $MP4_FNAME
# Combine audio into the MP4
ffmpeg -i $MP4_FNAME -i $WAV_FNAME -c:v copy -c:a aac -strict experimental $FINAL_FNAME

echo "Your MP4 file with audio is ready: $FINAL_NAME"

error: pathspec 'recterm.sh' did not match any file(s) known to git


Fix test script permissions.

Fix test script permissions.

#!/bin/sh

cd ${CURR_DIR}/issue813-pure-sub-dependency
rm -rf main/.dub
rm -rf sub/.dub
rm -rf sub/sub/.dub
rm -f main/dub.selections.json
${DUB} build --bare --compiler=${DC} main || exit 1


Added fff.sh, aka "Find Files/Folders Fast" shell script to make finding files easier.

#!/opt/local/bin/bash

if [ -z "$1" ]; then
	echo "An argument is required."
fi

SEARCH_DIR="$2"

if [ -z "SEARCH_DIR" ]; then
	SEARCH_DIR="./"
fi

/opt/local/bin/gfind $SEARCH_DIR -regextype posix-extended -regex ".*/?$1[^/]*"

error: pathspec 'Tools/my-tools/fff.sh' did not match any file(s) known to git


Add saveConfig.sh for saving git and gitk config.

#/bin/bash

# Save git global config

cp ~/.gitconfig ./

# Save gitk config

cp -pr ~/.config/git ./

error: pathspec 'Ubuntu_1404_LTS/saveConfig.sh' did not match any file(s) known to git


Create init_container.sh

Create init_container.sh

#!/bin/bash
touch /home/LogFiles/node_$WEBSITE_ROLE_INSTANCE_ID_out.log
echo "$(date) Container started" >> /home/LogFiles/node_$WEBSITE_ROLE_INSTANCE_ID_out.log

error: pathspec '4.4.7-0/init_container.sh' did not match any file(s) known to git


bash script for replacing space

bash script for replacing space

error: pathspec 'rp-space.sh' did not match any file(s) known to git