daemon: try to prevent race on partitions

We wait for the device to appear, this will avoid the following error:

chown: cannot access '/dev/sda1': No such file or directory
failed to change ownership of '/dev/sda1' to ceph:ceph

Signed-off-by: Sébastien Han <62d754cc350e84d3b1c32ae79f976f5348e74a40@redhat.com>

#!/bin/bash
set -e

# log arguments with timestamp
function log {
  if [ -z "$*" ]; then
    return 1
  fi

  local timestamp
  timestamp=$(date '+%F %T')
  echo "$timestamp  $0: $*"
  return 0
}

# ceph config file exists or die
function check_config {
  if [[ ! -e /etc/ceph/${CLUSTER}.conf ]]; then
    log "ERROR- /etc/ceph/${CLUSTER}.conf must exist; get it from your existing mon"
    exit 1
  fi
}

# ceph admin key exists or die
function check_admin_key {
  if [[ ! -e $ADMIN_KEYRING ]]; then
    log "ERROR- $ADMIN_KEYRING must exist; get it from your existing mon"
    exit 1
  fi
}

# Given two strings, return the length of the shared prefix
function prefix_length {
  local maxlen
  maxlen=${#1}
  for ((i=maxlen-1;i>=0;i--)); do
    if [[ "${1:0:i}" == "${2:0:i}" ]]; then
      echo $i
      return
    fi
  done
}

# Test if a command line tool is available
function is_available {
  command -v "$@" &>/dev/null
}

# create the mandatory directories
function create_mandatory_directories {
  # Let's create the bootstrap directories
  for keyring in $OSD_BOOTSTRAP_KEYRING $MDS_BOOTSTRAP_KEYRING $RGW_BOOTSTRAP_KEYRING; do
    mkdir -p "$(dirname "$keyring")"
  done

  # Let's create the ceph directories
  for directory in mon osd mds radosgw tmp mgr; do
    mkdir -p /var/lib/ceph/$directory
  done

  # Make the monitor directory
  mkdir -p "$MON_DATA_DIR"

  # Create socket directory
  mkdir -p /var/run/ceph

  # Creating rados directories
  mkdir -p /var/lib/ceph/radosgw/"${RGW_NAME}"

  # Create the MDS directory
  mkdir -p /var/lib/ceph/mds/"${CLUSTER}-${MDS_NAME}"

  # Create the MGR directory
  mkdir -p /var/lib/ceph/mgr/"${CLUSTER}-$MGR_NAME"

  # Adjust the owner of all those directories
  chown --verbose -R ceph. /var/run/ceph/ /var/lib/ceph/*
}

# Print resolved symbolic links of a device
function resolve_symlink {
  readlink -f "${@}"
}

# Calculate proper device names, given a device and partition number
function dev_part {
  local osd_device=${1}
  local osd_partition=${2}

  if [[ -L ${osd_device} ]]; then
    # This device is a symlink. Work out it's actual device
    local actual_device
    actual_device=$(readlink -f "${osd_device}")
    local bn
    bn=$(basename "${osd_device}")
    if [[ "${actual_device:0-1:1}" == [0-9] ]]; then
      local desired_partition="${actual_device}p${osd_partition}"
    else
      local desired_partition="${actual_device}${osd_partition}"
    fi
    # Now search for a symlink in the directory of $osd_device
    # that has the correct desired partition, and the longest
    # shared prefix with the original symlink
    local symdir
    symdir=$(dirname "${osd_device}")
    local link=""
    local pfxlen=0
    for option in ${symdir}*; do
      [[ -e $option ]] || break
      if [[ $(readlink -f "$symdir"/"$option") == "$desired_partition" ]]; then
        local optprefixlen
        optprefixlen=$(prefix_length "$option" "$bn")
        if [[ $optprefixlen > $pfxlen ]]; then
          link=$symdir/$option
          pfxlen=$optprefixlen
        fi
      fi
    done
    if [[ $pfxlen -eq 0 ]]; then
      >&2 log "Could not locate appropriate symlink for partition ${osd_partition} of ${osd_device}"
      exit 1
    fi
    echo "$link"
  elif [[ "${osd_device:0-1:1}" == [0-9] ]]; then
    echo "${osd_device}p${osd_partition}"
  else
    echo "${osd_device}${osd_partition}"
  fi
}

function osd_trying_to_determine_scenario {
  ${OSD_DEVICE:-none}
  if [[ ${OSD_DEVICE} == "none" ]]; then
    log "Bootstrapped OSD(s) found; using OSD directory"
    source osd_directory.sh
    osd_directory
  elif parted --script "${OSD_DEVICE}" print | grep -sqE '^ 1.*ceph data'; then
    log "Bootstrapped OSD found; activating ${OSD_DEVICE}"
    source osd_disk_activate.sh
    osd_activate
  else
    log "Device detected, assuming ceph-disk scenario is desired"
    log "Preparing and activating ${OSD_DEVICE}"
    osd_disk
  fi
}

function get_osd_dev {
  for i in ${OSD_DISKS}; do
    local osd_id
    osd_id=$(echo "${i}"|sed 's/\(.*\):\(.*\)/\1/')
    local osd_dev
    osd_dev="/dev/$(echo "${i}"|sed 's/\(.*\):\(.*\)/\2/')"
    if [[ "${osd_id}" == "${1}" ]]; then
      echo -n "${osd_dev}"
    fi
  done
}

function unsupported_scenario {
  echo "ERROR: '${CEPH_DAEMON}' scenario or key/value store '${KV_TYPE}' is not supported by this distribution."
  echo "ERROR: for the list of supported scenarios, please refer to your vendor."
  exit 1
}

function is_integer {
  # This function is about saying if the passed argument is an integer
  # Supports also negative integers
  # We use $@ here to consider everything given as parameter and not only the
  # first one : that's mainly for splited strings like "10 10"
  for arg in "$@"; do
    [[ $arg =~ ^-?[0-9]+$ ]]
  done
}

# Transform any set of strings to lowercase
function to_lowercase {
  echo "${@,,}"
}

# Transform any set of strings to uppercase
function to_uppercase {
  echo "${@^^}"
}

# Replace any variable separated with comma with space
# e.g: DEBUG=foo,bar will become:
# echo ${DEBUG//,/ }
# foo bar
function comma_to_space {
  echo "${@//,/ }"
}

# Get based distro by discovering the package manager
function get_package_manager {
  if is_available rpm; then
    OS_VENDOR=redhat
  elif is_available dpkg; then
    OS_VENDOR=ubuntu
  fi
}

# Determine if current distribution is an Ubuntu-based distribution
function is_ubuntu {
  get_package_manager
  [[ "$OS_VENDOR" == "ubuntu" ]]
}

# Determine if current distribution is a RedHat-based distribution
function is_redhat {
  get_package_manager
  [[ "$OS_VENDOR" == "redhat" ]]
}

# Wait for a file to exist, regardless of the type
function wait_for_file {
  timeout 10 bash -c "while [ ! -e ${1} ]; do echo 'Waiting for ${1} to show up' && sleep 1 ; done"
}

function valid_scenarios {
  if [ -n "$EXCLUDED_TAGS" ]; then
    for tag in $EXCLUDED_TAGS; do
      ALL_SCENARIOS=${ALL_SCENARIOS/$tag /}
    done
  fi
  log "Valid values for CEPH_DAEMON are $(to_uppercase "$ALL_SCENARIOS")."
  log "Valid values for the daemon parameter are $ALL_SCENARIOS"
}

function invalid_ceph_daemon {
  if [ -z "$CEPH_DAEMON" ]; then
    log "ERROR- One of CEPH_DAEMON or a daemon parameter must be defined as the name of the daemon you want to deploy."
    valid_scenarios
    exit 1
  else
    log "ERROR- unrecognized scenario."
    valid_scenarios
  fi
}

function get_osd_path {
  echo "$OSD_PATH_BASE-$1/"
}

# List all the partitions on a block device
function list_dev_partitions {
  # We need to remove the /dev/ part of the device name
  # since /proc/partitions has entries like sda only.
  # However we return a complete device name e.g: /dev/sda
  for args in "${@}"; do
    grep -Eo "${args#/dev/}[0-9]" < /proc/partitions | while read -r line; do
      echo "/dev/$line"
    done
  done
}

# Find the typecode of a partition
function get_part_typecode {
  for part in "${@}"; do
    sgdisk --info="${part: -1}" "${part%?}" | awk '/Partition GUID code/ {print tolower($4)}'
  done
}

function apply_ceph_ownership_to_disks {
  if [[ -n "${OSD_JOURNAL}" ]]; then
    wait_for_file "${OSD_JOURNAL}"
    chown --verbose ceph. "${OSD_JOURNAL}"
  elif [[ ${OSD_DMCRYPT} -eq 1 ]]; then
    # apply permission on the lockbox partition
    wait_for_file "$(dev_part "${OSD_DEVICE}" 3)"
    chown --verbose ceph. "$(dev_part "${OSD_DEVICE}" 3)"
  elif [[ ${OSD_BLUESTORE} -eq 1 ]]; then
    dev_real_path=$(resolve_symlink "$OSD_BLUESTORE_BLOCK_WAL" "$OSD_BLUESTORE_BLOCK_DB")
    for partition in $(list_dev_partitions "$OSD_DEVICE" "$dev_real_path"); do
      part_code=$(get_part_typecode "$partition")
      if [[ "$part_code" == "5ce17fce-4087-4169-b7ff-056cc58472be" ||
            "$part_code" == "5ce17fce-4087-4169-b7ff-056cc58473f9" ||
            "$part_code" == "30cd0809-c2b2-499c-8879-2d6b785292be" ||
            "$part_code" == "30cd0809-c2b2-499c-8879-2d6b78529876" ||
            "$part_code" == "89c57f98-2fe5-4dc0-89c1-f3ad0ceff2be" ||
            "$part_code" == "cafecafe-9b03-4f30-b4c6-b4b80ceff106" ]]; then
        chown --verbose ceph. "$partition"
      fi
    done
  else
    wait_for_file "$(dev_part "${OSD_DEVICE}" 2)"
    chown --verbose ceph. "$(dev_part "${OSD_DEVICE}" 2)"
  fi
  wait_for_file "$(dev_part "${OSD_DEVICE}" 1)"
  chown --verbose ceph. "$(dev_part "${OSD_DEVICE}" 1)"
}

# Get partition uuid of a given partition
function get_part_uuid {
  blkid -o value -s PARTUUID "${1}"
}

function ceph_health {
  local bootstrap_user=$1
  local bootstrap_key=$2

  if ! timeout 10 ceph "${CLI_OPTS[@]}" --name "$bootstrap_user" --keyring "$bootstrap_key" health; then
    log "Timed out while trying to reach out to the Ceph Monitor(s)."
    log "Make sure your Ceph monitors are up and running in quorum."
    log "Also verify the validity of $bootstrap_user keyring."
    exit 1
  fi
}

function is_net_ns {
  # if we run a container with --net=host we will see all the connections
  # if we don't, we should see the file header
  [[ $(wc -l < /proc/net/tcp) == 1 ]]
}

function is_pid_ns {
  # if we run a container with --pid=host we will see all the processes
  # if we don't, we should see 3 (pid 1 and ps and the new line)
  [[ $(ps --no-header x | wc -l) -gt 3 ]]
}

# This function is only used when CEPH_DAEMON=demo
# For a 'demo' container, we must ensure there is no Ceph files
function detect_ceph_files {
  if [ -f /etc/ceph/I_AM_A_DEMO ] || [ -f /var/lib/ceph/I_AM_A_DEMO ]; then
    log "Found residual files of a demo container."
    log "This looks like a restart, processing."
    return 0
  fi
  if [ -d /var/lib/ceph ] || [ -d /etc/ceph ]; then
    if [[ "$(find /var/lib/ceph/ -mindepth 3 -maxdepth 3 -type f | wc -l)" != 0 ]] || [[ -z "$(find /etc/ceph -prune -empty)" ]]; then
      log "I can see existing Ceph files, please remove them!"
      log "To run the demo container, remove the content of /var/lib/ceph/ and /etc/ceph/"
      log "Before doing this, make sure you are removing any sensitive data."
      exit 1
    fi
  fi
}

#!/bin/bash
set -e

# log arguments with timestamp
function log {
  if [ -z "$*" ]; then
    return 1
  fi

  local timestamp
  timestamp=$(date '+%F %T')
  echo "$timestamp  $0: $*"
  return 0
}

# ceph config file exists or die
function check_config {
  if [[ ! -e /etc/ceph/${CLUSTER}.conf ]]; then
    log "ERROR- /etc/ceph/${CLUSTER}.conf must exist; get it from your existing mon"
    exit 1
  fi
}

# ceph admin key exists or die
function check_admin_key {
  if [[ ! -e $ADMIN_KEYRING ]]; then
    log "ERROR- $ADMIN_KEYRING must exist; get it from your existing mon"
    exit 1
  fi
}

# Given two strings, return the length of the shared prefix
function prefix_length {
  local maxlen
  maxlen=${#1}
  for ((i=maxlen-1;i>=0;i--)); do
    if [[ "${1:0:i}" == "${2:0:i}" ]]; then
      echo $i
      return
    fi
  done
}

# Test if a command line tool is available
function is_available {
  command -v "$@" &>/dev/null
}

# create the mandatory directories
function create_mandatory_directories {
  # Let's create the bootstrap directories
  for keyring in $OSD_BOOTSTRAP_KEYRING $MDS_BOOTSTRAP_KEYRING $RGW_BOOTSTRAP_KEYRING; do
    mkdir -p "$(dirname "$keyring")"
  done

  # Let's create the ceph directories
  for directory in mon osd mds radosgw tmp mgr; do
    mkdir -p /var/lib/ceph/$directory
  done

  # Make the monitor directory
  mkdir -p "$MON_DATA_DIR"

  # Create socket directory
  mkdir -p /var/run/ceph

  # Creating rados directories
  mkdir -p /var/lib/ceph/radosgw/"${RGW_NAME}"

  # Create the MDS directory
  mkdir -p /var/lib/ceph/mds/"${CLUSTER}-${MDS_NAME}"

  # Create the MGR directory
  mkdir -p /var/lib/ceph/mgr/"${CLUSTER}-$MGR_NAME"

  # Adjust the owner of all those directories
  chown --verbose -R ceph. /var/run/ceph/ /var/lib/ceph/*
}

# Print resolved symbolic links of a device
function resolve_symlink {
  readlink -f "${@}"
}

# Calculate proper device names, given a device and partition number
function dev_part {
  local osd_device=${1}
  local osd_partition=${2}

  if [[ -L ${osd_device} ]]; then
    # This device is a symlink. Work out it's actual device
    local actual_device
    actual_device=$(readlink -f "${osd_device}")
    local bn
    bn=$(basename "${osd_device}")
    if [[ "${actual_device:0-1:1}" == [0-9] ]]; then
      local desired_partition="${actual_device}p${osd_partition}"
    else
      local desired_partition="${actual_device}${osd_partition}"
    fi
    # Now search for a symlink in the directory of $osd_device
    # that has the correct desired partition, and the longest
    # shared prefix with the original symlink
    local symdir
    symdir=$(dirname "${osd_device}")
    local link=""
    local pfxlen=0
    for option in ${symdir}*; do
      [[ -e $option ]] || break
      if [[ $(readlink -f "$symdir"/"$option") == "$desired_partition" ]]; then
        local optprefixlen
        optprefixlen=$(prefix_length "$option" "$bn")
        if [[ $optprefixlen > $pfxlen ]]; then
          link=$symdir/$option
          pfxlen=$optprefixlen
        fi
      fi
    done
    if [[ $pfxlen -eq 0 ]]; then
      >&2 log "Could not locate appropriate symlink for partition ${osd_partition} of ${osd_device}"
      exit 1
    fi
    echo "$link"
  elif [[ "${osd_device:0-1:1}" == [0-9] ]]; then
    echo "${osd_device}p${osd_partition}"
  else
    echo "${osd_device}${osd_partition}"
  fi
}

function osd_trying_to_determine_scenario {
  ${OSD_DEVICE:-none}
  if [[ ${OSD_DEVICE} == "none" ]]; then
    log "Bootstrapped OSD(s) found; using OSD directory"
    source osd_directory.sh
    osd_directory
  elif parted --script "${OSD_DEVICE}" print | grep -sqE '^ 1.*ceph data'; then
    log "Bootstrapped OSD found; activating ${OSD_DEVICE}"
    source osd_disk_activate.sh
    osd_activate
  else
    log "Device detected, assuming ceph-disk scenario is desired"
    log "Preparing and activating ${OSD_DEVICE}"
    osd_disk
  fi
}

function get_osd_dev {
  for i in ${OSD_DISKS}; do
    local osd_id
    osd_id=$(echo "${i}"|sed 's/\(.*\):\(.*\)/\1/')
    local osd_dev
    osd_dev="/dev/$(echo "${i}"|sed 's/\(.*\):\(.*\)/\2/')"
    if [[ "${osd_id}" == "${1}" ]]; then
      echo -n "${osd_dev}"
    fi
  done
}

function unsupported_scenario {
  echo "ERROR: '${CEPH_DAEMON}' scenario or key/value store '${KV_TYPE}' is not supported by this distribution."
  echo "ERROR: for the list of supported scenarios, please refer to your vendor."
  exit 1
}

function is_integer {
  # This function is about saying if the passed argument is an integer
  # Supports also negative integers
  # We use $@ here to consider everything given as parameter and not only the
  # first one : that's mainly for splited strings like "10 10"
  for arg in "$@"; do
    [[ $arg =~ ^-?[0-9]+$ ]]
  done
}

# Transform any set of strings to lowercase
function to_lowercase {
  echo "${@,,}"
}

# Transform any set of strings to uppercase
function to_uppercase {
  echo "${@^^}"
}

# Replace any variable separated with comma with space
# e.g: DEBUG=foo,bar will become:
# echo ${DEBUG//,/ }
# foo bar
function comma_to_space {
  echo "${@//,/ }"
}

# Get based distro by discovering the package manager
function get_package_manager {
  if is_available rpm; then
    OS_VENDOR=redhat
  elif is_available dpkg; then
    OS_VENDOR=ubuntu
  fi
}

# Determine if current distribution is an Ubuntu-based distribution
function is_ubuntu {
  get_package_manager
  [[ "$OS_VENDOR" == "ubuntu" ]]
}

# Determine if current distribution is a RedHat-based distribution
function is_redhat {
  get_package_manager
  [[ "$OS_VENDOR" == "redhat" ]]
}

# Wait for a file to exist, regardless of the type
function wait_for_file {
  timeout 10 bash -c "while [ ! -e ${1} ]; do echo 'Waiting for ${1} to show up' && sleep 1 ; done"
}

function valid_scenarios {
  if [ -n "$EXCLUDED_TAGS" ]; then
    for tag in $EXCLUDED_TAGS; do
      ALL_SCENARIOS=${ALL_SCENARIOS/$tag /}
    done
  fi
  log "Valid values for CEPH_DAEMON are $(to_uppercase "$ALL_SCENARIOS")."
  log "Valid values for the daemon parameter are $ALL_SCENARIOS"
}

function invalid_ceph_daemon {
  if [ -z "$CEPH_DAEMON" ]; then
    log "ERROR- One of CEPH_DAEMON or a daemon parameter must be defined as the name of the daemon you want to deploy."
    valid_scenarios
    exit 1
  else
    log "ERROR- unrecognized scenario."
    valid_scenarios
  fi
}

function get_osd_path {
  echo "$OSD_PATH_BASE-$1/"
}

# List all the partitions on a block device
function list_dev_partitions {
  # We need to remove the /dev/ part of the device name
  # since /proc/partitions has entries like sda only.
  # However we return a complete device name e.g: /dev/sda
  for args in "${@}"; do
    grep -Eo "${args#/dev/}[0-9]" < /proc/partitions | while read -r line; do
      echo "/dev/$line"
    done
  done
}

# Find the typecode of a partition
function get_part_typecode {
  for part in "${@}"; do
    sgdisk --info="${part: -1}" "${part%?}" | awk '/Partition GUID code/ {print tolower($4)}'
  done
}

function apply_ceph_ownership_to_disks {
  if [[ -n "${OSD_JOURNAL}" ]]; then
    wait_for_file "${OSD_JOURNAL}"
    chown --verbose ceph. "${OSD_JOURNAL}"
  elif [[ ${OSD_DMCRYPT} -eq 1 ]]; then
    # apply permission on the lockbox partition
    wait_for_file "$(dev_part "${OSD_DEVICE}" 3)"
    chown --verbose ceph. "$(dev_part "${OSD_DEVICE}" 3)"
  elif [[ ${OSD_BLUESTORE} -eq 1 ]]; then
    dev_real_path=$(resolve_symlink "$OSD_BLUESTORE_BLOCK_WAL" "$OSD_BLUESTORE_BLOCK_DB")
    for partition in $(list_dev_partitions "$OSD_DEVICE" "$dev_real_path"); do
      part_code=$(get_part_typecode "$partition")
      if [[ "$part_code" == "5ce17fce-4087-4169-b7ff-056cc58472be" ||
            "$part_code" == "5ce17fce-4087-4169-b7ff-056cc58473f9" ||
            "$part_code" == "30cd0809-c2b2-499c-8879-2d6b785292be" ||
            "$part_code" == "30cd0809-c2b2-499c-8879-2d6b78529876" ||
            "$part_code" == "89c57f98-2fe5-4dc0-89c1-f3ad0ceff2be" ||
            "$part_code" == "cafecafe-9b03-4f30-b4c6-b4b80ceff106" ]]; then
        chown --verbose ceph. "$partition"
      fi
    done
  else
    wait_for_file "$(dev_part "${OSD_DEVICE}" 2)"
    chown --verbose ceph. "$(dev_part "${OSD_DEVICE}" 2)"
  fi
  chown --verbose ceph. "$(dev_part "${OSD_DEVICE}" 1)"
}

# Get partition uuid of a given partition
function get_part_uuid {
  blkid -o value -s PARTUUID "${1}"
}

function ceph_health {
  local bootstrap_user=$1
  local bootstrap_key=$2

  if ! timeout 10 ceph "${CLI_OPTS[@]}" --name "$bootstrap_user" --keyring "$bootstrap_key" health; then
    log "Timed out while trying to reach out to the Ceph Monitor(s)."
    log "Make sure your Ceph monitors are up and running in quorum."
    log "Also verify the validity of $bootstrap_user keyring."
    exit 1
  fi
}

function is_net_ns {
  # if we run a container with --net=host we will see all the connections
  # if we don't, we should see the file header
  [[ $(wc -l < /proc/net/tcp) == 1 ]]
}

function is_pid_ns {
  # if we run a container with --pid=host we will see all the processes
  # if we don't, we should see 3 (pid 1 and ps and the new line)
  [[ $(ps --no-header x | wc -l) -gt 3 ]]
}

# This function is only used when CEPH_DAEMON=demo
# For a 'demo' container, we must ensure there is no Ceph files
function detect_ceph_files {
  if [ -f /etc/ceph/I_AM_A_DEMO ] || [ -f /var/lib/ceph/I_AM_A_DEMO ]; then
    log "Found residual files of a demo container."
    log "This looks like a restart, processing."
    return 0
  fi
  if [ -d /var/lib/ceph ] || [ -d /etc/ceph ]; then
    if [[ "$(find /var/lib/ceph/ -mindepth 3 -maxdepth 3 -type f | wc -l)" != 0 ]] || [[ -z "$(find /etc/ceph -prune -empty)" ]]; then
      log "I can see existing Ceph files, please remove them!"
      log "To run the demo container, remove the content of /var/lib/ceph/ and /etc/ceph/"
      log "Before doing this, make sure you are removing any sensitive data."
      exit 1
    fi
  fi
}


add kitchen hook

add kitchen hook

#!/bin/bash -x
set +e

# get list of nodes
hosts_list=""
for node in `kitchen list | grep -v '^Instance' | awk -F' ' '{print $1}'`; do
    output=$(kitchen exec $node -c "/sbin/ifconfig eth0 | grep 'inet addr:'")
    ip=$(echo "$output" | grep -v LC_ALL | grep -v '^---' | cut -d: -f2 | awk '{print $1}')
    output=$(kitchen exec $node -c "hostname")
    hostname=$(echo "$output" | grep -v LC_ALL | grep -v '^---')
    # trim whitespace
    hostname=$(echo "${hostname}" | sed -e 's/^[ \t]*//')
    # TODO: verify ip and hostname are valid
    hosts_list="${hosts_list},${ip}:${hostname}"
done
echo "hosts_list: $hosts_list"

hosts_arr=(${hosts_list//,/ })
# update /etc/hosts
for host in ${hosts_arr[@]}
do
    host_split=(${host//:/ })
    ip=${host_split[0]}
    domain=${host_split[1]}

    for node in `kitchen list | grep -v '^Instance' | awk -F' ' '{print $1}'`; do
        kitchen exec $node -c "cp -f /etc/hosts /tmp/hosts"
        if kitchen exec $node -c "grep ${domain} /tmp/hosts"; then
            command="sed -i \"/${domain}/c\\${ip}    ${domain}\" /tmp/hosts"
        else
            command="echo \"${ip}    ${domain}\" >> /tmp/hosts"
        fi
        kitchen exec $node -c "$command"
        kitchen exec $node -c "sudo cp -f /tmp/hosts /etc/hosts"
    done
done

error: pathspec 'chef/kitchen_hook_hack_hosts.sh' did not match any file(s) known to git


Added simple time-basd testrunner

Added simple time-basd testrunner

#!/bin/bash

# Set sleep period as first argument, defaults to 30 seconds
SLEEP_PERIOD=$1
: ${SLEEP_PERIOD:=30}

while true; do
	./manage.py test oscar --settings=test_settings
	sleep $SLEEP_PERIOD
done

error: pathspec 'examples/vanilla/run.sh' did not match any file(s) known to git


Adapt install_deb_dependencies to container use avoiding use of sudo

Change-Id: I45f44616bdfaf9fda66d153898dfd2c80b93d336

#!/bin/bash

if [ ! -s debian/control ] ; then
  echo "Warning, no debian/control file found"
  exit 1
fi

if [ $(dpkg -l | grep postpone | wc -l) -lt 1 ]
then
  DEBIAN_FRONTEND=noninteractive apt-get install --force-yes -y postpone
fi

DEBIAN_FRONTEND=noninteractive postpone -d -f apt-get update || echo "Error updating packages"

pkgs=$(cat debian/control | sed -e "s/$/\!\!/g" | tr -d '\n' | sed "s/\!\![[:space:]]/ /g" | sed "s/\!\!/\n/g" | grep "Build-Depends" | sed "s/Build-Depends://g" | sed "s/Build-Depends-Indep://g" | sed "s/([^)]*)//g" | sed "s/,\([^,^|]*\)[^,]*/\1/g" | sed 's/\[[^]]*\]//g' | sed "s/[[:space:]]+/ /g")
echo "Installing packages: $pkgs"
DEBIAN_FRONTEND=noninteractive postpone -d -f apt-get install --force-yes -y $pkgs || exit 1

error: pathspec 'kurento_container_install_deb_dependencies.sh' did not match any file(s) known to git


Create process_client_csr.sh

Create process_client_csr.sh

#!/bin/bash

docker cp cnt_tkdopvn_client:/etc/openvpn/easy-rsa/keys/client1.csr .

error: pathspec 'process_client_csr.sh' did not match any file(s) known to git


debug: Deleting patch.tar

debug: Deleting patch.tar

Once the patched filesystem is downloaded & installed, let's remove it.
That could take some useless space on the fs.

#!/bin/bash
set -e

# Bash substitution to remove everything before '='
# and only keep what is after
function extract_param {
  echo "${1##*=}"
}

for option in $(comma_to_space ${DEBUG}); do
  case $option in
    verbose)
      echo "VERBOSE: activating bash debugging mode."
      set -x
      ;;
    fstree*)
      echo "FSTREE: uncompressing content of $(extract_param $option)"
      # NOTE (leseb): the entrypoint should already be running from /
      # This is just a safeguard
      pushd / > /dev/null
      wget -q $(extract_param $option) -O patch.tar
      echo "SHA1 of the archive is: $(sha1sum patch.tar)"
      echo ""
      echo "Now, we print the SHA1 of each file."
      for f in $(tar xfpv patch.tar); do
        if [[ ! -d $f ]]; then
          sha1sum $f
        fi
      done
      rm -f patch.tar
      popd > /dev/null
      ;;
    stayalive)
      echo "STAYALIVE: container will not die if a command fails."
      source docker_exec.sh
      ;;
    *)
      echo "$option is not a valid debug option."
      echo "Available options are: verbose,fstree and stayalive."
      echo "They can be used altogether like this: '-e DEBUG=verbose,fstree=http://myfstree,stayalive"
      exit 1
      ;;
  esac
done

#!/bin/bash
set -e

# Bash substitution to remove everything before '='
# and only keep what is after
function extract_param {
  echo "${1##*=}"
}

for option in $(comma_to_space ${DEBUG}); do
  case $option in
    verbose)
      echo "VERBOSE: activating bash debugging mode."
      set -x
      ;;
    fstree*)
      echo "FSTREE: uncompressing content of $(extract_param $option)"
      # NOTE (leseb): the entrypoint should already be running from /
      # This is just a safeguard
      pushd / > /dev/null
      wget -q $(extract_param $option) -O patch.tar
      echo "SHA1 of the archive is: $(sha1sum patch.tar)"
      echo ""
      echo "Now, we print the SHA1 of each file."
      for f in $(tar xfpv patch.tar); do
        if [[ ! -d $f ]]; then
          sha1sum $f
        fi
      done
      popd > /dev/null
      ;;
    stayalive)
      echo "STAYALIVE: container will not die if a command fails."
      source docker_exec.sh
      ;;
    *)
      echo "$option is not a valid debug option."
      echo "Available options are: verbose,fstree and stayalive."
      echo "They can be used altogether like this: '-e DEBUG=verbose,fstree=http://myfstree,stayalive"
      exit 1
      ;;
  esac
done


tor: fix build (#7154)

tor: fix build (#7154)

#!/bin/bash -eu
# Copyright 2016 Google Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
##############################################################################

# Create a directory for instrumented dependencies.
TOR_DEPS=${SRC}/deps
mkdir -p $TOR_DEPS

# Build libevent with proper instrumentation.
cd ${SRC}/libevent
mkdir build && cd build
cmake -DEVENT__DISABLE_MBEDTLS=ON -DEVENT__DISABLE_OPENSSL=ON -DEVENT__LIBRARY_TYPE=STATIC ../
make && make install

# Build OpenSSL with proper instrumentation.
cd ${SRC}/openssl
OPENSSL_CONFIGURE_FLAGS=""
if [[ $CFLAGS = *sanitize=memory* ]]
then
  OPENSSL_CONFIGURE_FLAGS="no-asm"
fi

./config no-shared --prefix=${TOR_DEPS} \
    enable-tls1_3 enable-rc5 enable-md2 enable-ec_nistp_64_gcc_128 enable-ssl3 \
    enable-ssl3-method enable-nextprotoneg enable-weak-ssl-ciphers $CFLAGS \
    -fno-sanitize=alignment $OPENSSL_CONFIGURE_FLAGS

make -j$(nproc) LDCMD="$CXX $CXXFLAGS"
make install

# Build zlib with proper instrumentation,
cd ${SRC}/zlib
./configure --prefix=${TOR_DEPS}
make -j$(nproc) clean
make -j$(nproc) all
make install

# Build tor and the fuzz targets.
cd ${SRC}/tor

sh autogen.sh

# We need to run configure with leak-checking disabled, or many of the
# test functions will fail.
export ASAN_OPTIONS=detect_leaks=0

./configure --disable-asciidoc --enable-oss-fuzz --disable-memory-sentinels \
    --with-libevent-dir=${SRC}/deps \
    --with-openssl-dir=${SRC}/deps \
    --with-zlib-dir=${SRC}/deps \
    --disable-gcc-hardening \
    LDFLAGS="-L${TOR_DEPS}/lib64"

make clean
make micro-revision.i  # Workaround from https://gitlab.torproject.org/tpo/core/tor/-/issues/29520#note_2749427
make -j$(nproc) oss-fuzz-fuzzers

TORLIBS="`make show-testing-libs`"
TORLIBS="$TORLIBS -lm -Wl,-Bstatic -lssl -lcrypto -levent -lz -L${TOR_DEPS}/lib -L${TOR_DEPS}/lib64"
TORLIBS="$TORLIBS -Wl,-Bdynamic"

for fuzzer in src/test/fuzz/*.a; do
    output="${fuzzer%.a}"
    output="${output##*lib}"
    ${CXX} ${CXXFLAGS} -std=c++11 $LIB_FUZZING_ENGINE ${fuzzer} ${TORLIBS} -o ${OUT}/${output}

    corpus_dir="${SRC}/tor-fuzz-corpora/${output#oss-fuzz-}"
    if [ -d "${corpus_dir}" ]; then
      set +x
      zip -q -j ${OUT}/${output}_seed_corpus.zip ${corpus_dir}/*
      set -x
    fi
done

#!/bin/bash -eu
# Copyright 2016 Google Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
##############################################################################

# Create a directory for instrumented dependencies.
TOR_DEPS=${SRC}/deps
mkdir -p $TOR_DEPS

# Build libevent with proper instrumentation.
cd ${SRC}/libevent
sh autogen.sh
./configure --prefix=${TOR_DEPS} --disable-openssl
make -j$(nproc) clean
make -j$(nproc) all
make install

# Build OpenSSL with proper instrumentation.
cd ${SRC}/openssl
OPENSSL_CONFIGURE_FLAGS=""
if [[ $CFLAGS = *sanitize=memory* ]]
then
  OPENSSL_CONFIGURE_FLAGS="no-asm"
fi

./config no-shared --prefix=${TOR_DEPS} \
    enable-tls1_3 enable-rc5 enable-md2 enable-ec_nistp_64_gcc_128 enable-ssl3 \
    enable-ssl3-method enable-nextprotoneg enable-weak-ssl-ciphers $CFLAGS \
    -fno-sanitize=alignment $OPENSSL_CONFIGURE_FLAGS

make -j$(nproc) LDCMD="$CXX $CXXFLAGS"
make install

# Build zlib with proper instrumentation,
cd ${SRC}/zlib
./configure --prefix=${TOR_DEPS}
make -j$(nproc) clean
make -j$(nproc) all
make install

# Build tor and the fuzz targets.
cd ${SRC}/tor

sh autogen.sh

# We need to run configure with leak-checking disabled, or many of the
# test functions will fail.
export ASAN_OPTIONS=detect_leaks=0

./configure --disable-asciidoc --enable-oss-fuzz --disable-memory-sentinels \
    --with-libevent-dir=${SRC}/deps \
    --with-openssl-dir=${SRC}/deps \
    --with-zlib-dir=${SRC}/deps \
    --disable-gcc-hardening \
    LDFLAGS="-L${TOR_DEPS}/lib64"

make clean
make micro-revision.i  # Workaround from https://gitlab.torproject.org/tpo/core/tor/-/issues/29520#note_2749427
make -j$(nproc) oss-fuzz-fuzzers

TORLIBS="`make show-testing-libs`"
TORLIBS="$TORLIBS -lm -Wl,-Bstatic -lssl -lcrypto -levent -lz -L${TOR_DEPS}/lib -L${TOR_DEPS}/lib64"
TORLIBS="$TORLIBS -Wl,-Bdynamic"

for fuzzer in src/test/fuzz/*.a; do
    output="${fuzzer%.a}"
    output="${output##*lib}"
    ${CXX} ${CXXFLAGS} -std=c++11 $LIB_FUZZING_ENGINE ${fuzzer} ${TORLIBS} -o ${OUT}/${output}

    corpus_dir="${SRC}/tor-fuzz-corpora/${output#oss-fuzz-}"
    if [ -d "${corpus_dir}" ]; then
      set +x
      zip -q -j ${OUT}/${output}_seed_corpus.zip ${corpus_dir}/*
      set -x
    fi
done


#126 Reworked the installer and added an uninstaller for Ubuntu

#!/bin/sh
echo Remove MultiBit HD...
rm -Rf /opt/multibit-hd
rm -Rf "~/.JWrapper/JWrapper-MultiBit HD"

echo Remove symlink...
rm -f /usr/bin/multibit-hd

echo Remove HID USB rules...
rm -f /etc/udev/rules.d/99-multibit-hd.rules

echo Remove from Unity...
rm -f /usr/share/applications/multibit-hd.desktop
rm -f ~/.local/share/applications/multibit-hd.desktop
rm -f /usr/local/share/applications/multibit-hd.desktop
update-desktop-database

echo MultiBit HD application is removed.
echo Your wallet files remain intact in ~/.MultiBitHD
echo Your backup files remain inact.

error: pathspec 'mbhd-install/src/main/resources/uninstall-ubuntu.sh' did not match any file(s) known to git


Github Changes

Github Changes

error: pathspec 'Eazy' did not match any file(s) known to git
error: pathspec 'Hax' did not match any file(s) known to git
error: pathspec 'Toolkit/Install' did not match any file(s) known to git
error: pathspec 'Powerbutton.sh' did not match any file(s) known to git


Create docker-stress-test.sh

Create docker-stress-test.sh

remote: Support for password authentication was removed on August 13, 2021.
remote: Please see https://docs.github.com/en/get-started/getting-started-with-git/about-remote-repositories#cloning-with-https-urls for information on currently recommended modes of authentication.
fatal: Authentication failed for 'https://github.com/atomney/scripts.git/'


added script for running all system tests in sequence

#!/bin/sh
#
# Copyright (C) 2000  Internet Software Consortium.
#
# Permission to use, copy, modify, and distribute this software for any
# purpose with or without fee is hereby granted, provided that the above
# copyright notice and this permission notice appear in all copies.
#
# THE SOFTWARE IS PROVIDED "AS IS" AND INTERNET SOFTWARE CONSORTIUM DISCLAIMS
# ALL WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES
# OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL INTERNET SOFTWARE
# CONSORTIUM BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL
# DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR
# PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS
# ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS
# SOFTWARE.

# $Id: runall.sh,v 1.1 2000/07/06 19:32:37 gson Exp $

#
# Run all the system tests.
#

SYSTEMTESTTOP=.
. $SYSTEMTESTTOP/conf.sh

status=0

for d in $SUBDIRS
do
	sh run.sh $d || status=1
done

exit $status

error: pathspec 'bin/tests/system/runall.sh' did not match any file(s) known to git


exerun

exerun

remote: Support for password authentication was removed on August 13, 2021.
remote: Please see https://docs.github.com/en/get-started/getting-started-with-git/about-remote-repositories#cloning-with-https-urls for information on currently recommended modes of authentication.
fatal: Authentication failed for 'https://github.com/cohadar/bash_profile.git/'


improve uninstaller

improve uninstaller

remote: Support for password authentication was removed on August 13, 2021.
remote: Please see https://docs.github.com/en/get-started/getting-started-with-git/about-remote-repositories#cloning-with-https-urls for information on currently recommended modes of authentication.
fatal: Authentication failed for 'https://github.com/sjugge/lush.git/'


Switching to clone only latest revision

Switching to clone only latest revision

remote: Support for password authentication was removed on August 13, 2021.
remote: Please see https://docs.github.com/en/get-started/getting-started-with-git/about-remote-repositories#cloning-with-https-urls for information on currently recommended modes of authentication.
fatal: Authentication failed for 'https://github.com/CanTireInnovations/arduino-studio-builder.git/'


Fixed manual HD run

Fixed manual HD run

remote: Support for password authentication was removed on August 13, 2021.
remote: Please see https://docs.github.com/en/get-started/getting-started-with-git/about-remote-repositories#cloning-with-https-urls for information on currently recommended modes of authentication.
fatal: Authentication failed for 'https://github.com/ENIGMA-git/ENIGMA.git/'


Overlays for Global removal in pretyper

Overlays for Global removal in pretyper

if [ "$CI_PULL_REQUEST" = "9909" ] || [ "$CI_BRANCH" = "pretyping-rm-global" ]; then

    elpi_CI_REF=pretyping-rm-global
    elpi_CI_GITURL=https://github.com/maximedenes/coq-elpi

    coqhammer_CI_REF=pretyping-rm-global
    coqhammer_CI_GITURL=https://github.com/maximedenes/coqhammer

    equations_CI_REF=pretyping-rm-global
    equations_CI_GITURL=https://github.com/maximedenes/Coq-Equations

    ltac2_CI_REF=pretyping-rm-global
    ltac2_CI_GITURL=https://github.com/maximedenes/ltac2

    paramcoq_CI_REF=pretyping-rm-global
    paramcoq_CI_GITURL=https://github.com/maximedenes/paramcoq

    mtac2_CI_REF=pretyping-rm-global
    mtac2_CI_GITURL=https://github.com/maximedenes/Mtac2

fi

error: pathspec 'dev/ci/user-overlays/09909-maximedenes-pretyping-rm-global.sh' did not match any file(s) known to git


FreeBSD manual partitioning on ZFS for FreeBSD10.2.

#!/bin/sh
#
# $FreeBSD10.2: zfs_disk.sh,v 1.1 2015/10/14 08:45:18 Bleakwind (at) www.weaverdream.com Exp $
#
# - First when [Partitioning] select [Shell] and configure network:
#
# # ifconfig "em0" inet "192.168.8.201" netmask "255.255.255.0"
# # route add default "192.168.8.1"
# # echo "nameserver 202.96.134.133" >> /etc/resolv.conf
# # echo "nameserver 8.8.8.8" >> /etc/resolv.conf
#
# - Second fetch the shell file:
#
# # cd /tmp/
# # fetch https://raw.githubusercontent.com/bleakwind/freebsd/master/freebsd10.2/zfs_disk.sh && chmod +x zfs_disk.sh
# #     at: https://github.com/bleakwind/freebsd
#
# - Third allocating disk space:
#
# step 1
# # ./zfs_disk.sh -i zroot
#
# step 2
# # ./zfs_disk.sh -o
# # ./zfs_disk.sh -g da0 0 512k 4g 120g
#       or # ./zfs_disk.sh -g da0 0
# # ./zfs_disk.sh -g da1 1 512k 4g 120g
# # ./zfs_disk.sh -g da2 2 512k 4g 120g
# # ./zfs_disk.sh -g da3 3 512k 4g 120g
#
# step 3
# # ./zfs_disk.sh -p zroot "raidz2 da0p3 da1p3 da2p3 da3p3"
#       or: # ./zfs_disk.sh -p zroot "mirror da0p3 da1p3 mirror da2p3 da3p3"
#
# step 4
# # ./zfs_disk.sh -z zroot auto
# # ./zfs_disk.sh -z zroot "-o mountpoint=/pub zroot/pub"
# # ./zfs_disk.sh -z zroot "-o mountpoint=/db -o recordsize=8K zroot/db"
# #     option: -o copies=2
#
# step 5
# # ./zfs_disk.sh -f zroot
#
# step 6
# # rm zfs_disk.sh zfs_disk.sh.log
# # exit
#
# other
# # ./zfs_disk.sh -o
#       or: # ./zfs_disk.sh -o da0
# # ./zfs_disk.sh -h
#

THIS_FILE=$0
THIS_OPE=$1
FSTAB_FMT="%s\t\t%s\t%s\t%s\t\t%s\t%s\n"
HELP_INFO="usage: ${THIS_FILE} command args ...
where 'command' is one of the following:

    <init|-i> <poolname>
    <gpart|-g> <device> <label> [size_boot] [size_swap] [size_zfs]
    <pool|-p> <poolname> <"poolcommand">
    <zfs|-z> <poolname> <auto|"zfscommand">
    <finish|-f> <poolname> <"noplist">
    <info|-o> [device]
    <help|-h>"

if [ $# -le 0 ]; then
    echo "missing command"
    echo -e "${HELP_INFO}"
    exit 1
fi

if [ ! -f "${THIS_FILE}.log" ]; then
    echo -e "Log for ${THIS_FILE}: `date +%Y%m%d%H%M%S`" > ${THIS_FILE}.log
else
    echo -e "\nLog for ${THIS_FILE}: `date +%Y%m%d%H%M%S`" >> ${THIS_FILE}.log
fi
echo "\${THIS_FILE} > ${THIS_FILE}" >> ${THIS_FILE}.log
echo "\${THIS_OPE} > ${THIS_OPE}" >> ${THIS_FILE}.log

# begin
case "${THIS_OPE}" in
    # init: # ./zfs_disk.sh -i zroot
    init|-i)
        THIS_POOLNAME=$2

        if [ -z "${THIS_POOLNAME}" ]; then
            echo "missing argument"
            echo -e "${HELP_INFO}"
            exit 1
        fi
        echo "Begin init:" >> ${THIS_FILE}.log
        echo "\${THIS_POOLNAME} > ${THIS_POOLNAME}" >> ${THIS_FILE}.log

        zpool destroy "${THIS_POOLNAME}" >> ${THIS_FILE}.log 2>&1
        sysctl vfs.zfs.min_auto_ashift=12 >> ${THIS_FILE}.log 2>&1

        printf "" > "/tmp/bsdinstall_etc/fstab"
        printf "$FSTAB_FMT" "# Device" "Mountpoint" "FStype" "Options" "Dump" "Pass#" >> "/tmp/bsdinstall_etc/fstab"

        echo "End init---" >> ${THIS_FILE}.log
        echo "init successful..."
        ;;

    # gpart: # ./zfs_disk.sh -g da0 0 512k 4g 120g
    #    or: # ./zfs_disk.sh -g da0 0
    gpart|-g)
        THIS_DEVICE=$2
        THIS_LABEL=$3
        THIS_SIZE_BOOT=$4
        THIS_SIZE_SWAP=$5
        THIS_SIZE_ZFS=$6

        if [ -z "${THIS_DEVICE}" ] || [ -z "${THIS_LABEL}" ]; then
            echo "missing argument"
            echo -e "${HELP_INFO}"
            exit 1
        fi
        echo "Begin gpart---" >> ${THIS_FILE}.log

        if [ -z "${THIS_SIZE_BOOT}" ]; then
            THIS_SIZE_BOOT="512k"
        fi
        if [ -z "${THIS_SIZE_SWAP}" ]; then
            THIS_SIZE_SWAP=`sysctl hw.physmem | sed -e 's/.*hw\.physmem: \([0-9]*\).*/\1/g'`
            THIS_SIZE_SWAP=`echo "scale=2; ${THIS_SIZE_SWAP}/1024/1024/1024" | bc`
            THIS_SIZE_SWAP="`printf "%.0f" ${THIS_SIZE_SWAP}`g"
        fi

        echo "\${THIS_DEVICE} > ${THIS_DEVICE}" >> ${THIS_FILE}.log
        echo "\${THIS_LABEL} > ${THIS_LABEL}" >> ${THIS_FILE}.log
        echo "\${THIS_SIZE_BOOT} > ${THIS_SIZE_BOOT}" >> ${THIS_FILE}.log
        echo "\${THIS_SIZE_SWAP} > ${THIS_SIZE_SWAP}" >> ${THIS_FILE}.log
        echo "\${THIS_SIZE_ZFS} > ${THIS_SIZE_ZFS}" >> ${THIS_FILE}.log

        gpart destroy -F "${THIS_DEVICE}" >> ${THIS_FILE}.log 2>&1
        graid delete "${THIS_DEVICE}" >> ${THIS_FILE}.log 2>&1
        zpool labelclear -f "/dev/${THIS_DEVICE}" >> ${THIS_FILE}.log 2>&1
        gpart create -s gpt "${THIS_DEVICE}" >> ${THIS_FILE}.log 2>&1
        gpart destroy -F "${THIS_DEVICE}" >> ${THIS_FILE}.log 2>&1
        gpart create -s gpt "${THIS_DEVICE}" >> ${THIS_FILE}.log 2>&1

        THIS_BOOT=`gpart add -a 4k -l gptboot${THIS_LABEL} -t freebsd-boot -s ${THIS_SIZE_BOOT} "${THIS_DEVICE}"`
        THIS_BOOT_NAME=`echo ${THIS_BOOT} | sed -e 's/\([a-zA-Z0-9]*\) added/\1/g'`
        echo "\${THIS_BOOT} > ${THIS_BOOT}" >> ${THIS_FILE}.log 2>&1
        gpart bootcode -b "/boot/pmbr" -p "/boot/gptzfsboot" -i 1 "${THIS_DEVICE}" >> ${THIS_FILE}.log 2>&1

        THIS_SWAP=`gpart add -a 1m -l swap${THIS_LABEL} -t freebsd-swap -s ${THIS_SIZE_SWAP} "${THIS_DEVICE}"`
        THIS_SWAP_NAME=`echo ${THIS_SWAP} | sed -e 's/\([a-zA-Z0-9]*\) added/\1/g'`
        echo "\${THIS_SWAP} > ${THIS_SWAP}" >> ${THIS_FILE}.log 2>&1
        zpool labelclear -f "/dev/${THIS_SWAP_NAME}" >> ${THIS_FILE}.log 2>&1

        if [ -z "${THIS_SIZE_ZFS}" ]; then
            THIS_ZFS=`gpart add -a 1m -l zfs${THIS_LABEL} -t freebsd-zfs "${THIS_DEVICE}"`
        else
            THIS_ZFS=`gpart add -a 1m -l zfs${THIS_LABEL} -t freebsd-zfs -s ${THIS_SIZE_ZFS} "${THIS_DEVICE}"`
        fi
        THIS_ZFS_NAME=`echo ${THIS_ZFS} | sed -e 's/\([a-zA-Z0-9]*\) added/\1/g'`
        echo "\${THIS_ZFS} > ${THIS_ZFS}" >> ${THIS_FILE}.log 2>&1
        zpool labelclear -f "/dev/${THIS_ZFS_NAME}" >> ${THIS_FILE}.log 2>&1

        printf "${FSTAB_FMT}" "/dev/${THIS_SWAP_NAME}" "none" "swap" "sw" "0" "0" >> "/tmp/bsdinstall_etc/fstab"

        echo "remenber this => freebsd-boot:${THIS_BOOT_NAME}"
        echo "                 freebsd-swap:${THIS_SWAP_NAME}"
        echo "                 freebsd-zfs :${THIS_ZFS_NAME}"

        echo "End gpart---" >> ${THIS_FILE}.log
        echo "create partition successful..."
        ;;

    # pool: # ./zfs_disk.sh -p zroot "raidz2 da0p3 da1p3 da2p3 da3p3"
    #   or: # ./zfs_disk.sh -p zroot "mirror da0p3 da1p3 mirror da2p3 da3p3"
    pool|-p)
        THIS_POOLNAME=$2
        THIS_POOLCOMMAND=$3

        if [ -z "${THIS_POOLNAME}" ] || [ -z "${THIS_POOLCOMMAND}" ]; then
            echo "missing argument"
            echo -e "${HELP_INFO}"
            exit 1
        fi
        echo "Begin pool---" >> ${THIS_FILE}.log
        echo "\${THIS_POOLNAME} > ${THIS_POOLNAME}" >> ${THIS_FILE}.log
        echo "\${THIS_POOLCOMMAND} > ${THIS_POOLCOMMAND}" >> ${THIS_FILE}.log
        echo "zpool create -o altroot=/mnt -O compress=lz4 -O atime=off -m none -f "${THIS_POOLNAME}" ${THIS_POOLCOMMAND}" >> ${THIS_FILE}.log

        zpool create -o altroot=/mnt -O compress=lz4 -O atime=off -m none -f "${THIS_POOLNAME}" ${THIS_POOLCOMMAND} >> ${THIS_FILE}.log 2>&1
        zfs create -o mountpoint=none "${THIS_POOLNAME}/ROOT" >> ${THIS_FILE}.log 2>&1
        zfs create -o mountpoint=/ "${THIS_POOLNAME}/ROOT/default" >> ${THIS_FILE}.log 2>&1

        echo "End pool---" >> ${THIS_FILE}.log
        echo "create pool successful..."
        ;;

    # zfs: # ./zfs_disk.sh -z zroot auto
    #      # ./zfs_disk.sh -z zroot "-o mountpoint=/pub zroot/pub"
    #      # ./zfs_disk.sh -z zroot "-o mountpoint=/db -o recordsize=8K zroot/db"
    zfs|-z)
        THIS_POOLNAME=$2
        THIS_ZFSCOMMAND=$3

        if [ -z "${THIS_POOLNAME}" ] || [ -z "${THIS_ZFSCOMMAND}" ]; then
            echo "missing argument"
            echo -e "${HELP_INFO}"
            exit 1
        fi
        echo "Begin zfs---" >> ${THIS_FILE}.log
        echo "\${THIS_POOLNAME} > ${THIS_POOLNAME}" >> ${THIS_FILE}.log
        echo "\${THIS_ZFSCOMMAND} > ${THIS_ZFSCOMMAND}" >> ${THIS_FILE}.log

        if [ "${THIS_ZFSCOMMAND}" = "auto" ]; then
            zfs create -o mountpoint=/tmp -o exec=on -o setuid=off "${THIS_POOLNAME}/tmp" >> ${THIS_FILE}.log 2>&1

            zfs create -o mountpoint=/usr -o canmount=off "${THIS_POOLNAME}/usr" >> ${THIS_FILE}.log 2>&1
            zfs create -o setuid=off "${THIS_POOLNAME}/usr/ports" >> ${THIS_FILE}.log 2>&1
            zfs create  "${THIS_POOLNAME}/usr/src" >> ${THIS_FILE}.log 2>&1

            zfs create -o mountpoint=/var -o canmount=off "${THIS_POOLNAME}/var" >> ${THIS_FILE}.log 2>&1
            zfs create -o exec=off -o setuid=off "${THIS_POOLNAME}/var/audit" >> ${THIS_FILE}.log 2>&1
            zfs create -o exec=off -o setuid=off "${THIS_POOLNAME}/var/crash" >> ${THIS_FILE}.log 2>&1
            zfs create -o exec=off -o setuid=off "${THIS_POOLNAME}/var/log" >> ${THIS_FILE}.log 2>&1
            zfs create -o atime=on "${THIS_POOLNAME}/var/mail" >> ${THIS_FILE}.log 2>&1
            zfs create -o setuid=off "${THIS_POOLNAME}/var/tmp" >> ${THIS_FILE}.log 2>&1

            zfs create -o mountpoint=/home "${THIS_POOLNAME}/home" >> ${THIS_FILE}.log 2>&1
        else
            zfs create ${THIS_ZFSCOMMAND} >> ${THIS_FILE}.log 2>&1
        fi

        echo "End zfs---" >> ${THIS_FILE}.log
        echo "create dataset successful..."
        ;;

    # finish: # ./zfs_disk.sh -f zroot
    finish|-f)
        THIS_POOLNAME=$2

        if [ -z "${THIS_POOLNAME}" ]; then
            echo "missing argument"
            echo -e "${HELP_INFO}"
            exit 1
        fi
        echo "Begin finish---" >> ${THIS_FILE}.log
        echo "\${THIS_POOLNAME} > ${THIS_POOLNAME}" >> ${THIS_FILE}.log

        zfs set "mountpoint=/${THIS_POOLNAME}" "${THIS_POOLNAME}" >> ${THIS_FILE}.log 2>&1
        chmod 1777 "/mnt/tmp" >> ${THIS_FILE}.log 2>&1
        chmod 1777 "/mnt/var/tmp" >> ${THIS_FILE}.log 2>&1

        zpool set bootfs="${THIS_POOLNAME}/ROOT/default" "${THIS_POOLNAME}" >> ${THIS_FILE}.log 2>&1
        zpool export "${THIS_POOLNAME}" >> ${THIS_FILE}.log 2>&1
        zpool import -o altroot="/mnt" "${THIS_POOLNAME}" >> ${THIS_FILE}.log 2>&1

        mkdir -p "/mnt/boot/zfs" >> ${THIS_FILE}.log 2>&1
        zpool set cachefile="/mnt/boot/zfs/zpool.cache" "${THIS_POOLNAME}" >> ${THIS_FILE}.log 2>&1
        echo "zfs_enable=\"YES\"" >> "/tmp/bsdinstall_etc/rc.conf.zfs"
        echo "kern.geom.label.gptid.enable=\"0\"" >> "/tmp/bsdinstall_boot/loader.conf.zfs"

        echo "End finish---" >> ${THIS_FILE}.log
        echo "finish successful..."
        ;;

    # help: # ./zfs_disk.sh -h
    help|-h)
        echo -e "${HELP_INFO}"
        exit 0
        ;;

    # info: # ./zfs_disk.sh -o
    #   or: # ./zfs_disk.sh -o da0
    info|-o)
        THIS_DEVICE=$2

        echo "Begin info---" >> ${THIS_FILE}.log
        echo "\${THIS_DEVICE} > ${THIS_DEVICE}" >> ${THIS_FILE}.log

        if [ -z "${THIS_DEVICE}" ]; then
            gpart show
        else
            gpart show "${THIS_DEVICE}"
        fi

        echo "End info---" >> ${THIS_FILE}.log
        echo "show disk info successful..."
        ;;

    # error
    *)
        echo "unrecognized command '${THIS_OPE}'"
        echo -e "${HELP_INFO}"
        exit 1
        ;;
esac

error: pathspec 'freebsd10.2/zfs_disk.sh' did not match any file(s) known to git


Better description in purge_styleguide script

remote: Support for password authentication was removed on August 13, 2021.
remote: Please see https://docs.github.com/en/get-started/getting-started-with-git/about-remote-repositories#cloning-with-https-urls for information on currently recommended modes of authentication.
fatal: Authentication failed for 'https://github.com/frontcraft/fronthack-repo.git/'


Add measure cpu temp for Raspberry Pi

Add measure cpu temp for Raspberry Pi

#!/bin/sh
SECONDS=`date '+%s'`

NAME='temp.cpu'
TEMP=`cat /sys/class/thermal/thermal_zone0/temp`
VALUE=`echo "scale=3; ${TEMP} / 1000" | bc`
echo "${NAME}\t${VALUE}\t${SECONDS}"

error: pathspec 'raspberrypi/temp.sh' did not match any file(s) known to git


Initial Commit

Initial Commit

#!/bin/bash

# Variables

#PID
PIDFILE=

#Rclone transfer type
TYPE=

#Rclone source dir
SOURCE=

#Rclone dest dir
DEST=

#Rclone option flags
OPT1=
OPT2=
OPT3=
OPT4=
OPT5=

# Check if PID file exists, if it does, check it's valid, and if not create new
if [ -f $PIDFILE ]
then
  PID=$(cat $PIDFILE)
  ps -p $PID > /dev/null 2>&1
  if [ $? -eq 0 ]
  then
    echo "Rclone sync already running"
    exit 1
  else
    ## If process not found assume not running
    echo $$ > $PIDFILE
    if [ $? -ne 0 ]
    then
      echo "Unable to create PID file"
      exit 1
    fi
  fi
else
  echo $$ > $PIDFILE
  if [ $? -ne 0 ]
  then
    echo "Unable to create PID file"
    exit 1
  fi
fi

# Run the sync job
rclone $TYPE "$SOURCE" "$DEST" $OPT1 $OPT2 $OPT3 $OPT4 -v

# Remove the PID file
rm $PIDFILE


RHSA: add one fix

RHSA: add one fix

#!/bin/sh
# CYBERWATCH SAS - 2016
#
# Security fix for RHSA-2016:1617
#
# Security announcement date: 2016-08-16 11:35:25 UTC
# Script generation date:     2016-08-18 21:45:00 UTC
#
# Operating System: Red Hat 6
# Architecture: x86_64
#
# Vulnerable packages fix on version:
#   - kernel.x86_64:2.6.32-220.67.1.el6
#   - kernel-debug.x86_64:2.6.32-220.67.1.el6
#   - kernel-debug-debuginfo.x86_64:2.6.32-220.67.1.el6
#   - kernel-debug-devel.x86_64:2.6.32-220.67.1.el6
#   - kernel-debuginfo.x86_64:2.6.32-220.67.1.el6
#   - kernel-debuginfo-common-x86_64.x86_64:2.6.32-220.67.1.el6
#   - kernel-devel.x86_64:2.6.32-220.67.1.el6
#   - kernel-headers.x86_64:2.6.32-220.67.1.el6
#   - perf.x86_64:2.6.32-220.67.1.el6
#   - perf-debuginfo.x86_64:2.6.32-220.67.1.el6
#   - python-perf-debuginfo.x86_64:2.6.32-220.67.1.el6
#   - python-perf.x86_64:2.6.32-220.67.1.el6
#
# Last versions recommanded by security team:
#   - kernel.x86_64:2.6.32-220.67.1.el6
#   - kernel-debug.x86_64:2.6.32-220.67.1.el6
#   - kernel-debug-debuginfo.x86_64:2.6.32-220.67.1.el6
#   - kernel-debug-devel.x86_64:2.6.32-220.67.1.el6
#   - kernel-debuginfo.x86_64:2.6.32-220.67.1.el6
#   - kernel-debuginfo-common-x86_64.x86_64:2.6.32-220.67.1.el6
#   - kernel-devel.x86_64:2.6.32-220.67.1.el6
#   - kernel-headers.x86_64:2.6.32-220.67.1.el6
#   - perf.x86_64:2.6.32-220.67.1.el6
#   - perf-debuginfo.x86_64:2.6.32-220.67.1.el6
#   - python-perf-debuginfo.x86_64:2.6.32-220.67.1.el6
#   - python-perf.x86_64:2.6.32-220.67.1.el6
#
# CVE List:
#   - CVE-2016-4565
#
# More details:
#   - https://www.cyberwatch.fr/vulnerabilites
#
# Licence: Released under The MIT License (MIT), See LICENSE FILE
sudo yum install kernel.x86_64-2.6.32 -y
sudo yum install kernel-debug.x86_64-2.6.32 -y
sudo yum install kernel-debug-debuginfo.x86_64-2.6.32 -y
sudo yum install kernel-debug-devel.x86_64-2.6.32 -y
sudo yum install kernel-debuginfo.x86_64-2.6.32 -y
sudo yum install kernel-debuginfo-common-x86_64.x86_64-2.6.32 -y
sudo yum install kernel-devel.x86_64-2.6.32 -y
sudo yum install kernel-headers.x86_64-2.6.32 -y
sudo yum install perf.x86_64-2.6.32 -y
sudo yum install perf-debuginfo.x86_64-2.6.32 -y
sudo yum install python-perf-debuginfo.x86_64-2.6.32 -y
sudo yum install python-perf.x86_64-2.6.32 -y

error: pathspec 'Red_Hat_6/x86_64/2016/RHSA-2016:1617.sh' did not match any file(s) known to git


Add MySQL and Java dev packages to Vagrant

Add MySql JDK7 and Maven to the list of packages that are installed during
the boostrapUbuntuServer.sh file.  These packages are added in response to
some inquiries by the OpenInfoButton developers, but will be useful for
others.

Change-Id: I87b94a896a844a37b97580a66c14d43393c09e2a
OSEHRA-Id: http://issues.osehra.org/browse/OAT-135

#!/usr/bin/env bash
#---------------------------------------------------------------------------
# Copyright 2011-2012 The Open Source Electronic Health Record Agent
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#---------------------------------------------------------------------------

# Make sure we are root
if [[ $EUID -ne 0 ]]; then
    echo "This script must be run as root" 1>&2
    exit 1
fi

# Update the server from repositories
apt-get -y -qq update > /dev/null
apt-get -y -qq upgrade > /dev/null

# Install baseline packages
apt-get install -y -qq git xinetd perl wget curl python ssh mysql-server openjdk-7-jdk maven > /dev/null

#!/usr/bin/env bash
#---------------------------------------------------------------------------
# Copyright 2011-2012 The Open Source Electronic Health Record Agent
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#---------------------------------------------------------------------------

# Make sure we are root
if [[ $EUID -ne 0 ]]; then
    echo "This script must be run as root" 1>&2
    exit 1
fi

# Update the server from repositories
apt-get -y -qq update > /dev/null
apt-get -y -qq upgrade > /dev/null

# Install baseline packages
apt-get install -y -qq git xinetd perl wget curl python ssh > /dev/null


feat(bin): added script to create task on vim.

#!/bin/bash

# Creates a task file inside the current folder

CARDS_DIR=/storage/docs/notes/tasks
TIMESTAMP="$(date "+%Y-%m-%d-%H%M%S-%3N")"
FILENAME=$CARDS_DIR/$TIMESTAMP.md

mkdir -p $CARDS_DIR

vim "$FILENAME" +'normal!ggItask'

error: pathspec 'bin/create-task-card.sh' did not match any file(s) known to git


Added LF

Added LF

remote: Support for password authentication was removed on August 13, 2021.
remote: Please see https://docs.github.com/en/get-started/getting-started-with-git/about-remote-repositories#cloning-with-https-urls for information on currently recommended modes of authentication.
fatal: Authentication failed for 'https://github.com/gurinderu/dcos-gitlab-runner-service.git/'


Update debian-repo.sh

Update debian-repo.sh

remote: Support for password authentication was removed on August 13, 2021.
remote: Please see https://docs.github.com/en/get-started/getting-started-with-git/about-remote-repositories#cloning-with-https-urls for information on currently recommended modes of authentication.
fatal: Authentication failed for 'https://github.com/mjasion/linux-stuff.git/'


Create make_osm_map.sh

Create make_osm_map.sh

error: pathspec 'gpx2osm/make_osm_map.sh' did not match any file(s) known to git


add script to compare gcc and asm outputs

#!/usr/bin/env bash

DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"

if [ $# -ne 0 ]; then
  a=1
else
  echo "usage: compare_instruction_encoding.sh <instr>"
  exit 1
fi

p=$1
echo $p > asm.s
g++ -c asm.s -o asm.o
if [ $? -ne 0 ]; then
  echo "GCC ERROR for "$p
  rm -rf asm.s asm.o
  exit 1
else
  echo "GCC encoding:"
  objdump -D -Msuffix asm.o | grep -A 10 00000000 | grep -v 00000000

  echo ""

  echo "x64asm encoding:"
  cat asm.s | $DIR/../../src/ext/x64asm/bin/asm | tail -n +6 | sed -e "s/^/        /"
  rm -rf asm.s asm.o

  exit 0
fi

error: pathspec 'tools/scripts/compare_instruction_encoding.sh' did not match any file(s) known to git


stupid script to conver hamt32 code to hamt64; works on most files without manual fixing

#!/usr/bin/env bash

for f in "$@"; do
	echo $f
	echo "perl -pi -e 's/32/64/g' $f"
	perl -pi -e 's/32/64/g' $f
	echo "perl -pi -e 's/30/60/g' $f"
	perl -pi -e 's/30/60/g' $f
	#echo "perl -pi -e 's/six/ten/g'"
done

error: pathspec 'convert32to64.sh' did not match any file(s) known to git


simple wd script to monitor batch conversion queue size and lunch more EC2 instances with Chef setting them up a batch nodes.

#!/bin/bash -
#===============================================================================
#          FILE: batch-conversions-ec2.sh
#         USAGE: ./batch-conversions-ec2.sh
#   DESCRIPTION:
#       OPTIONS: ---
#  REQUIREMENTS: ---
#          BUGS: ---
#         NOTES: ---
#        AUTHOR: Jess Portnoy <jess.portnoy@kaltura.com>
#  ORGANIZATION: Kaltura, inc.
#       CREATED: 05/07/14 08:23:48 EDT
#      REVISION:  ---
#===============================================================================

#set -o nounset                              # Treat unset variables as an error
#!/bin/sh
SYSTEM_INI=/etc/kaltura.d/system.ini
if [ -r "$SYSTEM_INI" ];then
        . /etc/kaltura.d/system.ini
else
        echo "Could not source $SYSTEM_INI:("
        exit 1
fi
if [ $# -lt 3 ];then
        echo "Usage: $0 <AMI id> <warning threshold> <critical threshold>"
        exit 2
fi
AMI_IMG=$1
LOWER_THRESHOLD=$2
UPPER_THRESHOLD=$3
OUT=`php /opt/kaltura/app/plugins/monitor/nagios/exec.php --warning-threshold $LOWER_THRESHOLD --error-threshold $UPPER_THRESHOLD --script "/opt/kaltura/app/tests/monitoring/api_v3/getBatchQueueSize.php --service-url $KALTURA_VIRTUAL_HOST_NAME --job-type CONVERT"`
RC=$?
if [ $RC -eq 1 ];then
echo "Reached WARNING threshold"
# here you can do some actions, for example:
# send yourself an email
# launch instances
fi

# we are at CRITICAL, lets launch

if [ $RC -eq 2 ];then
echo "$OUT
Lunching another batch instance ..
"
knife ec2 server create --availability-zone us-east-1d --flavor m3.medium --image $AMI_IMG --identity-file ~/csi.pem --run-list "recipe[nfs],recipe[kaltura::batch]" --ssh-user ec2-user 2>&1 | tee /tmp/log
fi

error: pathspec 'RPM/scripts/watchdogs/batch-conversions-ec2.sh' did not match any file(s) known to git


added script to create and push nightlies (use cron , or something,for scheduling)

#!/usr/bin/env bash
#
# Copyright 2010-2012 Tasos Laskos <tasos.laskos@gmail.com>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

path_to_readlink_function=`dirname $0`"/lib/readlink_f.sh"
if [[ ! -e "$path_to_readlink_function" ]]; then
    echo "Could not find $path_to_readlink_function"
    exit
fi

source $path_to_readlink_function

root="$(dirname "$(readlink_f "${0}")")"
nightlies="$HOME/builds/nightlies"

package_patterns="arachn*.gz arachn*installer.sh"
dest="segfault@downloads.arachni-scanner.com:www/arachni/downloads/nightlies/"

mkdir -p $nightlies
cd $nightlies

# remove the old nightlies
rm -f arachn*.gz arachn*installer.sh

$root/build_all.sh

# wait for the locks to be created
while [ ! -e "32bit_build.lock" ]; do sleep 0.1; done
while [ ! -e "64bit_build.lock" ]; do sleep 0.1; done

# and then wait for the locks to be removed
while [ -e "32bit_build.lock" ]; do sleep 0.1; done
while [ -e "64bit_build.lock" ]; do sleep 0.1; done

rsync --human-readable --progress --executability --compress --stats \
    $package_patterns $dest

error: pathspec 'external/scripts/push_nightlies.sh' did not match any file(s) known to git


Fixed typo for kegs setdiff command

Fixed typo for kegs setdiff command

remote: Support for password authentication was removed on August 13, 2021.
remote: Please see https://docs.github.com/en/get-started/getting-started-with-git/about-remote-repositories#cloning-with-https-urls for information on currently recommended modes of authentication.
fatal: Authentication failed for 'https://github.com/dav1k/dotfiles.git/'


installer script for installing dependencies added

#!/usr/bin/sh

echo 'Downloading & installing python development headers'
sudo apt-get install python-dev

echo 'Downloading & installing distribute'
sudo curl http://python-distribute.org/distribute_setup.py | sudo python

echo 'Downloading libevent'
wget -0 /tmp/libevent-2.0.17-stable.tar.gz https://github.com/downloads/libevent/libevent/libevent-2.0.17-stable.tar.gz
echo 'installing libevent'
tar -C /tmp -xzf /tmp/libevent-2.0.17-stable.tar.gz
cd /tmp/libevent-20.17-stable
./configure
make
sudo make install

echo 'Downloading & installing gevent'
sudo pip install gevent

echo 'Downloading gunicorn'
sudo pip install gunicorn

echo 'Downloading nginx'
sudo pip install nginx

echo 'Downloading zeromq'
wget -O /tmp/zeromq-2.1.11.tar.gz http://download.zeromq.org/zeromq-2.1.11.tar.gz
echo 'Installing zeromq'
tar -C /tmp -xzf /tmp/zeromq-2.1.11.tar.gz
cd /tmp/zeromq-2.1.11
./configure
make
sudo make install

echo 'Downloading pyzmq'
sudo pip install pyzmq

echo 'Downloading libpng'
wget -O /tmp/libpng-1.5.7.tar.xz http://downloads.sourceforge.net/project/libpng/libpng15/older-releases/1.5.7/libpng-1.5.7.tar.xz?r=http%3A%2F%2Fsourceforge.net%2Fprojects%2Flibpng%2Ffiles%2F&ts=1329403713&use_mirror=space
echo 'Installing libpng'
tar -C /tmp -xzf /tmp/libpng-1.5.7.tar.xz
cd /tmp/libpng-1.5.7
./configure
make
sudo make install

echo 'Downloading libpoppler'
wget -O /tmp/poppler-0.18.4.tar.gz http://poppler.freedesktop.org/poppler-0.18.4.tar.gz
echo 'Installing libpoppler'
tar -C /tmp -xzf /tmp/poppler-0.18.4.tar.gz
cd /tmp/poppler-0.18.4
./configure
make
sudo make install

echo 'Downloading mongodb 64-bit'
wget -O /tmp/mongodb-linux-x86_64-2.0.2.tgz http://fastdl.mongodb.org/linux/mongodb-linux-x86_64-2.0.2.tgz
echo 'Installing mongodb'
tar -C /tmp -xzf /tmp/mongodb-linux-x86_64-2.0.2.tgz
cd /tmp/mongodb-linux-x86_64-2.0.2
./configure
make
sudo make install

echo 'Downloading pymongo'
sudo pip install pymongo

echo 'Downloading mongokit'
sudo pip install mongokit

echo 'Downloading flask'
sudo pip install flask

echo 'Downloading werkzeug'
sudo pip install werkzeug

echo 'Downloading jinja2'
sudo pip install jinja2

echo 'Downloading wtform-fork'
sudo pip install hg + https://bitbucket.org/kailashbuki/wtforms-fork

error: pathspec 'install_dependencies.sh' did not match any file(s) known to git


GitHub workflow: Add script to Copybara config

PiperOrigin-RevId: 347830403
Change-Id: Ia6ee679918a5047ca9b258d873a8834872a2a5bf

#!/bin/bash
#
# Decrypts the checked-in IDA SDK. In order to not leak the passphrase in the
# workflow log, it is set as an environment variable before being passed to
# GPG.

set -e

gpg --quiet --batch --yes --decrypt \
  "--passphrase=${IDASDK_SECRET}" \
  --output "${RUNNER_WORKSPACE}/build/idasdk75.zip" \
  "${GITHUB_WORKSPACE}/third_party/idasdk75.zip.gpg"

error: pathspec '.github/scripts/decrypt_secret.sh' did not match any file(s) known to git


checks for gzip/deflate compression by web server with redirect follows

#!/bin/bash
# Scriptacular - curl-compress-check.sh
# Determine whether a web server is serving gzip or deflate compressed files
# Copyright 2013 Christopher Simpkins
# MIT License

if [ $# -eq 0 ]; then
	echo "Please include the URL(s) for the site(s) that you would like to test." 1>&2
	exit 1
fi

for url in "$@";
do
	printf "Testing %s ---> " "$url"
	curl -I -L -s -H 'Accept-Encoding: gzip,deflate' $url | grep "Content-Encoding"
	if (( $? )); then
		echo "Unable to evaluate the URL $url" >&2
		exit 1
	fi
done
exit 0

error: pathspec 'network/curl-compress-check.sh' did not match any file(s) known to git


Add bash completions

Add bash completions

This contains bash completion functionality for the resin CLI, including completion for sub-commands.

Change-type: patch

#!/bin/bash

_resin_complete()
{
  local cur prev

  # Valid top-level completions
  commands="app apps build config deploy device devices env envs help key \
            keys local login logout logs note os preload quickstart settings \
            signup ssh sync util version whoami"
  # Sub-completions
  app_cmds="create restart rm"
  config_cmds="generate inject read reconfigure write"
  device_cmds="identify init move public-url reboot register rename rm \
               shutdown"
  device_public_url_cmds="disable enable status"
  env_cmds="add rename rm"
  key_cmds="add rm"
  local_cmds="configure flash logs push scan ssh stop"
  os_cmds="build-config configure download initialize versions"
  util_cmds="available-drives"

  COMPREPLY=()
  cur=${COMP_WORDS[COMP_CWORD]}
  prev=${COMP_WORDS[COMP_CWORD-1]}

  if [ $COMP_CWORD -eq 1 ]
  then
    COMPREPLY=( $(compgen -W "${commands}" -- $cur) )
  elif [ $COMP_CWORD -eq 2 ]
  then
    case "$prev" in
      "app")
        COMPREPLY=( $(compgen -W "$app_cmds" -- $cur) )
        ;;
      "config")
        COMPREPLY=( $(compgen -W "$config_cmds" -- $cur) )
        ;;
      "device")
        COMPREPLY=( $(compgen -W "$device_cmds" -- $cur) )
        ;;
      "env")
        COMPREPLY=( $(compgen -W "$env_cmds" -- $cur) )
        ;;
      "key")
        COMPREPLY=( $(compgen -W "$key_cmds" -- $cur) )
        ;;
      "local")
        COMPREPLY=( $(compgen -W "$local_cmds" -- $cur) )
        ;;
      "os")
        COMPREPLY=( $(compgen -W "$os_cmds" -- $cur) )
        ;;
      "util")
        COMPREPLY=( $(compgen -W "$util_cmds" -- $cur) )
        ;;
      "*")
        ;;
    esac
  elif [ $COMP_CWORD -eq 3 ]
  then
    case "$prev" in
      "public-url")
        COMPREPLY=( $(compgen -W "$device_public_url_cmds" -- $cur) )
        ;;
      "*")
        ;;
    esac
  fi

}
complete -F _resin_complete resin

error: pathspec 'resin-completion.bash' did not match any file(s) known to git


add 5services.

add 5services.

remote: Support for password authentication was removed on August 13, 2021.
remote: Please see https://docs.github.com/en/get-started/getting-started-with-git/about-remote-repositories#cloning-with-https-urls for information on currently recommended modes of authentication.
fatal: Authentication failed for 'https://github.com/tonkachi/aws-monitors.git/'


Initial commit

Initial commit

#!/bin/bash

################################################################################
#
# run_mothur_mice.sh
#
#
# Here we run the data/raw/*metag/*.files through mothur like we would in a
# normal study
#
# Dependencies...
# * data/raw/*metag/*.files
#
# Produces...
# * *.thetayc.ave.dist
# * *.nmds.axes
# * *.error.summary
#
################################################################################

FILES_FILE=$1
RAW_PATH=$(echo $FILES_FILE | sed -E 's/(.*)\/[^\/]*/\1/')
PROCESS_PATH=$(echo $RAW_PATH | sed -E 's/raw/process/')
mkdir -p $PROCESS_PATH

mothur "#pcr.seqs(fasta=data/references/silva.bacteria.align, start=11894, end=25319, keepdots=F, processors=12)"
mv data/references/silva.bacteria.pcr.align $PROCESS_PATH/silva.v4.align

#change the name of the file from stability.files to whatever suits your study
mothur "#set.dir(output=$PROCESS_PATH);
	make.contigs(file=$FILES_FILE, processors=12);
	screen.seqs(fasta=current, group=current, maxambig=0, maxlength=275);
	unique.seqs();
	count.seqs(name=current, group=current);
	align.seqs(fasta=current, reference=silva.v4.align);
	screen.seqs(fasta=current, count=current, start=1968, end=11550, maxhomop=8);
	filter.seqs(fasta=current, vertical=T, trump=.);
	unique.seqs(fasta=current, count=current);
	pre.cluster(fasta=current, count=current, diffs=2);
	chimera.uchime(fasta=current, count=current, dereplicate=t);
	remove.seqs(fasta=current, accnos=current);
	classify.seqs(fasta=current, count=current, reference=data/references/trainset9_032012.pds.fasta, taxonomy=data/references/trainset9_032012.pds.tax, cutoff=80);
	remove.lineage(fasta=current, count=current, taxonomy=current, taxon=Chloroplast-Mitochondria-unknown-Archaea-Eukaryota);
	remove.groups(count=current, fasta=current, taxonomy=current, groups=Mock-Mock2);
	cluster.split(fasta=current, count=current, taxonomy=current, splitmethod=classify, taxlevel=4, cutoff=0.15);
	make.shared(list=current, count=current, label=0.03);
	classify.otu(list=current, count=current, taxonomy=current, label=0.03);
	phylotype(taxonomy=current);
	make.shared(list=current, count=current, label=1);"

error: pathspec 'code/run_mothur_mice.sh' did not match any file(s) known to git


what it says

what it says

echo "jenkins causes these, since he dispatches h2o as either. So he needs to clean up"
# the 0xdiag ones may be shared with other people doing python tests, so this will whack them

#****************************************************************************
# got the private id from 180
ssh -i ~/.0xcustomer/0xcustomer_id_rsa 0xcustomer@192.168.1.161 rm -f -r /home/0xcustomer/ice*
ssh -i ~/.0xcustomer/0xcustomer_id_rsa 0xcustomer@192.168.1.162 rm -f -r /home/0xcustomer/ice*
ssh -i ~/.0xcustomer/0xcustomer_id_rsa 0xcustomer@192.168.1.163 rm -f -r /home/0xcustomer/ice*
ssh -i ~/.0xcustomer/0xcustomer_id_rsa 0xcustomer@192.168.1.164 rm -f -r /home/0xcustomer/ice*

ssh -i ~/.0xcustomer/0xcustomer_id_rsa 0xcustomer@192.168.1.171 rm -f -r /home/0xcustomer/ice*
ssh -i ~/.0xcustomer/0xcustomer_id_rsa 0xcustomer@192.168.1.172 rm -f -r /home/0xcustomer/ice*
ssh -i ~/.0xcustomer/0xcustomer_id_rsa 0xcustomer@192.168.1.173 rm -f -r /home/0xcustomer/ice*
ssh -i ~/.0xcustomer/0xcustomer_id_rsa 0xcustomer@192.168.1.174 rm -f -r /home/0xcustomer/ice*
ssh -i ~/.0xcustomer/0xcustomer_id_rsa 0xcustomer@192.168.1.175 rm -f -r /home/0xcustomer/ice*
ssh -i ~/.0xcustomer/0xcustomer_id_rsa 0xcustomer@192.168.1.176 rm -f -r /home/0xcustomer/ice*
ssh -i ~/.0xcustomer/0xcustomer_id_rsa 0xcustomer@192.168.1.177 rm -f -r /home/0xcustomer/ice*
ssh -i ~/.0xcustomer/0xcustomer_id_rsa 0xcustomer@192.168.1.178 rm -f -r /home/0xcustomer/ice*
ssh -i ~/.0xcustomer/0xcustomer_id_rsa 0xcustomer@192.168.1.179 rm -f -r /home/0xcustomer/ice*
ssh -i ~/.0xcustomer/0xcustomer_id_rsa 0xcustomer@192.168.1.180 rm -f -r /home/0xcustomer/ice*

#****************************************************************************
# got the private id from 180
ssh -i ~/.0xdiag/0xdiag_id_rsa 0xdiag@192.168.1.161 rm -f -r /home/0xdiag/ice*
ssh -i ~/.0xdiag/0xdiag_id_rsa 0xdiag@192.168.1.162 rm -f -r /home/0xdiag/ice*
ssh -i ~/.0xdiag/0xdiag_id_rsa 0xdiag@192.168.1.163 rm -f -r /home/0xdiag/ice*
ssh -i ~/.0xdiag/0xdiag_id_rsa 0xdiag@192.168.1.164 rm -f -r /home/0xdiag/ice*

ssh -i ~/.0xdiag/0xdiag_id_rsa 0xdiag@192.168.1.171 rm -f -r /home/0xdiag/ice*
ssh -i ~/.0xdiag/0xdiag_id_rsa 0xdiag@192.168.1.172 rm -f -r /home/0xdiag/ice*
ssh -i ~/.0xdiag/0xdiag_id_rsa 0xdiag@192.168.1.173 rm -f -r /home/0xdiag/ice*
ssh -i ~/.0xdiag/0xdiag_id_rsa 0xdiag@192.168.1.174 rm -f -r /home/0xdiag/ice*
ssh -i ~/.0xdiag/0xdiag_id_rsa 0xdiag@192.168.1.175 rm -f -r /home/0xdiag/ice*
ssh -i ~/.0xdiag/0xdiag_id_rsa 0xdiag@192.168.1.176 rm -f -r /home/0xdiag/ice*
ssh -i ~/.0xdiag/0xdiag_id_rsa 0xdiag@192.168.1.177 rm -f -r /home/0xdiag/ice*
ssh -i ~/.0xdiag/0xdiag_id_rsa 0xdiag@192.168.1.178 rm -f -r /home/0xdiag/ice*
ssh -i ~/.0xdiag/0xdiag_id_rsa 0xdiag@192.168.1.179 rm -f -r /home/0xdiag/ice*
ssh -i ~/.0xdiag/0xdiag_id_rsa 0xdiag@192.168.1.180 rm -f -r /home/0xdiag/ice*

error: pathspec 'py/testdir_release/clean_0xdiag_0xcustomer_ice_everywhere.sh' did not match any file(s) known to git


A minor update

A minor update

remote: Support for password authentication was removed on August 13, 2021.
remote: Please see https://docs.github.com/en/get-started/getting-started-with-git/about-remote-repositories#cloning-with-https-urls for information on currently recommended modes of authentication.
fatal: Authentication failed for 'https://github.com/bus710/scripts.git/'


CrClim full build script (#37)

CrClim full build script (#37)

* Script to build STELLA, Dycore and COSMO

* Code cleaningm bugs corrections and add configuration print

* Correct usage of the options formalism

* Clumsy print removed

* Make the K-level flag take a value

* Remove debug exit code and add columns in config print

* Make config more verbose

* Typo in config print

* Add Dycore build options and rename variables

* Indentation fixes

* Add a clean step before clone

* Target management with a default GPU target in a case of a CRCLIM_BUILD (=ON)

* Add project name options and build repro flag

* Correct/add the repro options/flag and help messages

* Conditional removal of installation (depend on build request)

* Add configurable STELLA and COSMO branch and organisation

* Correct missing options order, check and description

* Protect the build against incorrect cleaning

#!/usr/bin/env bash

exitError()
{
	echo "ERROR $1: $3" 1>&2
	echo "ERROR     LOCATION=$0" 1>&2
	echo "ERROR     LINE=$2" 1>&2
	exit "$1"
}

tryExit()
{
	status=$1
	action=$2
	if [ "${status}" -ne 0 ]; then
		echo "ERROR in ${action} with ${status}" >&2
		exit "${status}"
  fi
}

showUsage()
{
	echo "usage: $(basename "$0") -i install_prefix -t target -c compiler -s slave -b stella_branch -o stella_org -a cosmo_branch -q cosmo_org [-4] [-f kflat] [-l klevel] [-z] [-h]"
	echo ""
	echo "mandatory arguments:"
	echo "-t        Target (e.g. cpu or gpu)"
	echo "-s        Slave (the machine)"
	echo "-f        STELLA K-Flat"
	echo "-l        STELLA K-Level"
	echo "-n        The name of the project"
	echo "-b        The STELLA branch to checkout (e.g. crclim)"
	echo "-o        The STELLA github repository organisation (e.g. C2SM-RCM)"
	echo "-a        The COSMO-POMPA branch to checkout (e.g. crclim)"
	echo "-q        The COSMO-POMPA github repository organisation (e.g. C2SM-RCM)"
	echo ""
	echo "optional arguments:"
	echo "-h        Show help"
	echo "-4        Single precision (default: OFF)"
	echo "-c        Compiler (e.g. gnu, cray or pgi)"
	echo "-v        Verbose mode"
	echo "-z        Clean builds"
	echo "-g        Do Stella GNU build"
	echo "-d        Do CPP Dycore GNU build"
	echo "-p        Do Cosmo-Pompa build"
	echo "-i        Install prefix"
	echo "-x        Do bit-reproducible build"
}

# set defaults and process command line options
parseOptions()
{
	local OPTIND
	singleprec=OFF
	projName=""
	compiler=""
	target=""
	slave=""
	kflat=""
	klevel=""
	stellaBranch=""
	stellaOrg=""
	cosmoBranch=""
	cosmoOrg=""
	instPrefix=""
	verbosity=OFF
	cleanup=OFF
	doStella=OFF
	doDycore=OFF
	doPompa=OFF
	doRepro=OFF

	while getopts "h4n:c:b:o:a:q:t:s:f:l:vzgdpi:x" opt; do
		case "${opt}" in
		h)
		    showUsage
		    exit 0
		  	;;
		4)
		    singleprec=ON
		    ;;
		n)
		    projName=$OPTARG
		    ;;
		c)
		    compiler=$OPTARG
		    ;;
		b)
		    stellaBranch=$OPTARG
		    ;;
		o)
		    stellaOrg=$OPTARG
		    ;;
		a)
		    cosmoBranch=$OPTARG
		    ;;
		q)
		    cosmoOrg=$OPTARG
		    ;;
		t)
		    target=$OPTARG
		    ;;
		s)
		    slave=$OPTARG
		    ;;
		f)
		    kflat=$OPTARG
		    ;;
		l)
		    klevel=$OPTARG
		    ;;
		v)
		    verbosity=ON
		    ;;
		z)
		    cleanup=ON
		    ;;
		g)
		    doStella=ON
		    ;;
		d)
		    doDycore=ON
		    ;;
		p)
		    doPompa=ON
		    ;;
		i)
		    instPrefix=$OPTARG
		    ;;
		x)
		    doRepro=ON
		    ;;
		\?)
		    showUsage
		    exitError 601 ${LINENO} "invalid command line option (-${OPTARG})"
		    ;;
		esac
	done
	shift $((OPTIND-1))
}

# make sure the working variable are set
checkOptions()
{
	test -n "${compiler}"     || exitError 603 ${LINENO} "Option <compiler> is not set"
	test -n "${target}"       || exitError 604 ${LINENO} "Option <target> is not set"
	test -n "${slave}"        || exitError 605 ${LINENO} "Option <slave> is not set"
	test -n "${projName}"     || exitError 663 ${LINENO} "Option <projName> is not set"

	if [ ${doStella} == "ON" ] ; then
		test -n "${kflat}"        || exitError 606 ${LINENO} "Option <flat> is not set"
		test -n "${klevel}"       || exitError 607 ${LINENO} "Option <klevel> is not set"
		test -n "${stellaBranch}" || exitError 665 ${LINENO} "Option <stellaBranch> is not set"
		test -n "${stellaOrg}"    || exitError 666 ${LINENO} "Option <stellaOrg> is not set"
	fi

	if [ ${doDycore} == "ON" ] ; then
		test -n "${kflat}"        || exitError 606 ${LINENO} "Option <flat> is not set"
		test -n "${klevel}"       || exitError 607 ${LINENO} "Option <klevel> is not set"
		test -n "${cosmoBranch}"  || exitError 667 ${LINENO} "Option <cosmoBranch> is not set"
		test -n "${cosmoOrg}"     || exitError 668 ${LINENO} "Option <cosmoOrg> is not set"
	fi

	if [ ${doPompa} == "ON" ] ; then
		test -n "${cosmoBranch}"  || exitError 667 ${LINENO} "Option <cosmoBranch> is not set"
		test -n "${cosmoOrg}"     || exitError 668 ${LINENO} "Option <cosmoOrg> is not set"
	fi
}

printConfig()
{
	echo "==============================================================="
	echo "BUILD CONFIGURATION"
	echo "==============================================================="
	echo "PROJECT NAME:             ${projName}"
	echo "SINGLE PRECISION:         ${singleprec}"
	echo "STELLA ORGANISATION:      ${stellaOrg}"
	echo "STELLA BRANCH:            ${stellaBranch}"
	echo "COSMO ORGANISATION:       ${cosmoOrg}"
	echo "COSMO BRANCH:             ${cosmoBranch}"
	echo "COMPILER:                 ${compiler}"
	echo "TARGET:                   ${target}"
	echo "SLAVE:                    ${slave}"
	echo "K-FLAT:                   ${kflat}"
	echo "K-LEVEL:                  ${klevel}"
	echo "BIT-REPRO:                ${doRepro}"
	echo "VERBOSE:                  ${verbosity}"
	echo "CLEAN:                    ${cleanup}"
	echo "DO STELLA COMPILATION:    ${doStella}"
	echo "DO DYCORE COMPILATION:    ${doDycore}"
	echo "DO POMPA COMPILATION:     ${doPompa}"
	echo "INSTALL PREFIX:           ${instPrefix}"
	echo "==============================================================="
}

# clone the repositories
cloneTheRepos()
{
	# note that we clean the previous clone and they're supposed to be installed
	# on another directory (simpler solution)
	if [ ${doStella} == "ON" ] ; then
		echo "Clean previous stella directories"
		\rm -rf stella
		echo "Clone stella"
		git clone git@github.com:"${stellaOrg}"/stella.git --branch "${stellaBranch}"
	fi

	if [ ${doDycore} == "ON" ] || [ ${doPompa} == "ON" ] ; then
		echo "Clean previous cosmo-pompa directories"
		\rm -rf cosmo-pompa
		echo "Clone cosmo-pompa (with dycore)"
		git clone git@github.com:"${cosmoOrg}"/cosmo-pompa.git --branch "${cosmoBranch}"
	fi
}

setupBuilds()
{
	# single precision flag
	moreFlag=""
	if [ ${singleprec} == "ON" ] ; then
		moreFlag="${moreFlag} -4"
	fi

	if [ ${verbosity} == "ON" ] ; then
		moreFlag="${moreFlag} -v"
	fi

	if [ ${cleanup} == "ON" ] ; then
		moreFlag="${moreFlag} -z"
	fi

	# compiler (for Stella and the Dycore)
	gnuCompiler="gnu"

	# path and directory structures
	stellapath="${instPrefix}/${slave}/crclim/stella_kflat${kflat}_klevel${klevel}/${projName}/${target}/${gnuCompiler}"
	dycorepath="${instPrefix}/${slave}/crclim/dycore/${projName}/${target}/${gnuCompiler}"
	cosmopath="${instPrefix}/${slave}/crclim/cosmo/${projName}/${target}/${compiler}"

	# clean previous install path if needed
	if [ ${doStella} == "ON" ] ; then
		\rm -rf "${stellapath:?}/"*
	fi

	if [ ${doDycore} == "ON" ] ; then
		\rm -rf "${dycorepath:?}/"*
	fi

	if [ ${doPompa} == "ON" ] ; then
		\rm -rf "${cosmopath:?}/"*
	fi
}

# compile and install stella
doStellaCompilation()
{
	cd stella || exitError 608 ${LINENO} "Unable to change directory into stella"
	if [ ${doRepro} == "ON" ] ; then
		test/jenkins/build.sh "${moreFlag}" -c "${gnuCompiler}" -i "${stellapath}" -f "${kflat}" -k "${klevel}" -x
		retCode=$?
	else
		test/jenkins/build.sh "${moreFlag}" -c "${gnuCompiler}" -i "${stellapath}" -f "${kflat}" -k "${klevel}"
		retCode=$?
	fi

	tryExit $retCode "STELLA BUILD"
	cd .. || exitError 609 ${LINENO} "Unable to go back"
}

# compile and install the dycore
doDycoreCompilation()
{
	cd cosmo-pompa/dycore || exitError 610 ${LINENO} "Unable to change directory into cosmo-pompa/dycore"
	test/jenkins/build.sh "${moreFlag}" -c "${gnuCompiler}" -t "${target}" -s "${stellapath}" -i "${dycorepath}" -s "${stellapath}"
	retCode=$?
	tryExit $retCode "DYCORE BUILD"
	cd ../.. || exitError 611 ${LINENO} "Unable to go back"
}

# compile and install cosmo-pompa
doCosmoCompilation()
{
	cd cosmo-pompa/cosmo || exitError 612 ${LINENO} "Unable to change directory into cosmo-pompa/cosmo"
	test/jenkins/build.sh "${moreFlag}" -c "${compiler}" -t "${target}" -i "${cosmopath}" -x "${dycorepath}"
	retCode=$?
	tryExit $retCode "COSMO BUILD"
	cd ../.. || exitError 612 ${LINENO} "Unable to go back"
}

# ===================================================
# MAIN LIKE
# ===================================================

# parse command line options (pass all of them to function)
parseOptions "$@"

# check the command line options
checkOptions

printConfig

# clone
cloneTheRepos

# setup
setupBuilds

# compile and install
if [ ${doStella} == "ON" ] ; then
	doStellaCompilation
fi

if [ ${doDycore} == "ON" ] ; then
	doDycoreCompilation
fi

if [ ${doPompa} == "ON" ] ; then
	doCosmoCompilation
fi

# end without errors
echo "####### finished: $0 $* (PID=$$ HOST=$HOSTNAME TIME=$(date '+%D %H:%M:%S'))"
exit 0

error: pathspec 'package_builder/build_cosmo.sh' did not match any file(s) known to git


setup script to help create patches

setup script to help create patches

#!/bin/bash
if [[ $# != 1 ]] ; then
    echo 2>&1 <<EOF
Usage:
  $0 *ruby-tar.gz*

Untar ruby source and set it up for git.
EOF
    exit 1
fi
tar_file="$1"
if [[ ! -r "$tar_file" ]]; then
    echo "tar file: $tar_file does not exist"  2>&1
    exit 2
fi

tar -xzf $tar_file || {
    echo "Error untarring $ruby_name" 2>&1
    exit 3
}
files="
configure
encdb.h
golf_prelude.c
id.h
insns.inc
insns_info.inc
known_errors.inc
lex.c
miniprelude.c
newline.c
node_name.inc
opt_sc.inc
optinsn.inc
optunifs.inc
parse.c
parse.h
revision.h
transdb.h
vm.inc
vmtc.inc
"

ruby_name=$(basename $tar_file .tar.gz)
git_dir="${ruby_name}-git"
mv -v ${ruby_name} ${git_dir}
cd ${git_dir} || {
    error >2 "Error in cd to ${git_dir}"
    exit 4
}
git init .
rm -fv $files
git add *
git commit -m'What we got to work with' .
exit $?

error: pathspec 'script/git-setup.sh' did not match any file(s) known to git


scripts: Add script to create a patch for third-party applications

#!/bin/sh

# This script is used to patch an third-party application.
# For example, if you want to create a patch for third_party/fuse/fuse just do:
#    ./app_diff.sh fuse-2.9.4 build/extbld/third_party/fuse/core third-party/fuse/fuse
# Args:
# $1 - application name
# $2 - path to application within the build folder
# $3 - sources folder

EMBOX_DIR="$(pwd)"
APP_NAME=$1 #fuse-2.9.4
EXTBLD_APP_DIR=$EMBOX_DIR/$2 #build/extbld/third_party/fuse/core
SRC_APP_DIR=$EMBOX_DIR/$3 #third-party/fuse/fuse

cd $EXTBLD_APP_DIR
mkdir -p $APP_NAME-orig
tar xzf $EMBOX_DIR/download/${APP_NAME}.tar.gz -C $APP_NAME-orig --strip-components=1
diff -aur -x configure $APP_NAME-orig $APP_NAME | grep -v "Only in" | tee $SRC_APP_DIR/patch.txt
rm -rf $APP_NAME-orig

error: pathspec 'scripts/app_diff.sh' did not match any file(s) known to git


postgres diagnostics collector

postgres diagnostics collector

#!/bin/sh

#/
 #  Copyright 2016 Amazon.com, Inc. or its affiliates.
 #  All Rights Reserved.
 #
 #  Licensed under the Apache License, Version 2.0 (the "License").
 #  You may not use this file except in compliance with the License.
 #  A copy of the License is located at
 #
 #      http://aws.amazon.com/apache2.0/
 #
 # or in the "license" file accompanying this file.
 # This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
 # either express or implied. See the License for the specific language governing permissions
 # and limitations under the License.
#/

##################
#
# SETTINGS
# 1. PLEASE SET VARIABLES BELOW
# 2. .pgpass is set accordingly and properly as per https://www.postgresql.org/docs/current/static/libpq-pgpass.htmly
# 3. this script is better set as a cron job in a reccuring fashion for collecting stats
# 4. storage consumption will be observed on the executing host
#
#
#
# It's advised to call this script as unprivileged user from cron. Please set proper permissions for STAT_DIR to allow that user to write files in that directory
#################

# This script is intended to be a proof of concept and is provided as is without any support.
#

RDSHOST=''
RDSUSER=''
RDSDB=''
STAT_DIR="${HOME}/postgre_stats/"

##################
# SCRIPT
#################
DATE=`date -u '+%Y-%m-%d-%H%M%S'`
mkdir -p ${STAT_DIR}
if [ $? -eq 0 ]; then

	echo "================ACTIVE SESSION USE================" >> ${STAT_DIR}/${DATE}.output
	psql -h ${RDSHOST} -U ${RDSUSER} ${RDSDB} -c 'SELECT * FROM PG_STAT_ACTIVITY;' >> ${STAT_DIR}/${DATE}.output

#	echo "================LOCKS=============================" >> ${STAT_DIR}/${DATE}.output
#	psql -h ${RDSHOST} -U ${RDSUSER} ${RDSDB} -c 'SELECT blocked_locks.pid AS blocked_pid, blocked_activity.usename  AS blocked_user, blocking_locks.pid     AS blocking_pid, blocking_activity.usename AS blocking_user, blocked_activity.query AS blocked_statement, blocking_activity.query AS blocking_statement FROM  pg_catalog.pg_locks         blocked_locks JOIN pg_catalog.pg_stat_activity blocked_activity  ON blocked_activity.pid = blocked_locks.pid JOIN pg_catalog.pg_locks         blocking_locks ON blocking_locks.locktype = blocked_locks.locktype AND blocking_locks.DATABASE IS NOT DISTINCT FROM blocked_locks.DATABASE AND blocking_locks.relation IS NOT DISTINCT FROM blocked_locks.relation AND blocking_locks.page IS NOT DISTINCT FROM blocked_locks.page AND blocking_locks.tuple IS NOT DISTINCT FROM blocked_locks.tuple AND blocking_locks.virtualxid IS NOT DISTINCT FROM blocked_locks.virtualxid AND blocking_locks.transactionid IS NOT DISTINCT FROM blocked_locks.transactionid AND blocking_locks.classid IS NOT DISTINCT FROM blocked_locks.classid AND blocking_locks.objid IS NOT DISTINCT FROM blocked_locks.objid AND blocking_locks.objsubid IS NOT DISTINCT FROM blocked_locks.objsubid AND blocking_locks.pid != blocked_locks.pid JOIN pg_catalog.pg_stat_activity blocking_activity ON blocking_activity.pid = blocking_locks.pid WHERE NOT blocked_locks.GRANTED;' >> ${STAT_DIR}/${DATE}.output

	echo "================LOCKS -2 ====================" >> ${STAT_DIR}/${DATE}.output
	psql -h ${RDSHOST} -U ${RDSUSER} ${RDSDB} -c 'SELECT * FROM pg_locks;' >> ${STAT_DIR}/${DATE}.output

	echo "================DATABASES INFO====================" >> ${STAT_DIR}/${DATE}.output
	psql -h ${RDSHOST} -U ${RDSUSER} ${RDSDB} -c 'SELECT * FROM pg_stat_database;' >> ${STAT_DIR}/${DATE}.output

	echo "================INDEXES===========================" >> ${STAT_DIR}/${DATE}.output
	psql -h ${RDSHOST} -U ${RDSUSER} ${RDSDB} -c 'SELECT * FROM pg_stat_all_indexes;' >> ${STAT_DIR}/${DATE}.output

#	ATTN: This might perform a vacuum on system databases
#	echo "================ANALYSE VACUUM NEED===============" >> ${STAT_DIR}/${DATE}.output
#	psql -h ${RDSHOST} -U ${RDSUSER} ${RDSDB} -c 'VACUUM VERBOSE ANALYSE;' >> ${STAT_DIR}/${DATE}.output

	echo "================ALL PARAMETERS====================" >> ${STAT_DIR}/${DATE}.output
	psql -h ${RDSHOST} -U ${RDSUSER} ${RDSDB} -x -c 'SHOW ALL;' >> ${STAT_DIR}/${DATE}.output

#	ATTN: might not be needed - generates too much noise
#	echo "================ALL TABLE STATISTICS====================" >> ${STAT_DIR}/${DATE}.output
#	psql -h ${RDSHOST} -U ${RDSUSER} ${RDSDB} -c 'SELECT * FROM pg_stat_all_tables;' >> ${STAT_DIR}/${DATE}.output

	echo "================USER'S TABLE STATISTICS===========" >> ${STAT_DIR}/${DATE}.output
	psql -h ${RDSHOST} -U ${RDSUSER} ${RDSDB} -c 'SELECT * FROM pg_stat_user_tables;' >> ${STAT_DIR}/${DATE}.output

	echo "================USER'S INDEX STATISTICS===========" >> ${STAT_DIR}/${DATE}.output
	psql -h ${RDSHOST} -U ${RDSUSER} ${RDSDB} -c 'SELECT * FROM pg_stat_user_indexes;' >> ${STAT_DIR}/${DATE}.output

else
	echo "DIRECTORY CREATING ERROR"
fi

error: pathspec 'postgres/diag/shell/postgresql-diagnostics.sh' did not match any file(s) known to git


update

update

remote: Support for password authentication was removed on August 13, 2021.
remote: Please see https://docs.github.com/en/get-started/getting-started-with-git/about-remote-repositories#cloning-with-https-urls for information on currently recommended modes of authentication.
fatal: Authentication failed for 'https://github.com/dengxiangyu768/dengxytools.git/'


[cpu_time.sh] give more statistics about polynomial selection

remote: Support for password authentication was removed on August 13, 2021.
remote: Please see https://docs.github.com/en/get-started/getting-started-with-git/about-remote-repositories#cloning-with-https-urls for information on currently recommended modes of authentication.
fatal: Authentication failed for 'https://github.com/mancoast/cado-nfs.git/'


Starting a script to execute the harness in all subfolders and ignore ones where there is nothing to execute

#!/usr/bin/zsh

PROGRAM=dummy
PLATFORM=0
DEVICE=0
SIZE_STRING="1024"
ORIG_DIR=$PWD
INPUTS=/home/s1042579/generated_programs/inputs
CONFIG=/home/s1042579/harness/bla_interface.json

for i in $(seq 1 250)
do
  for low_level in $(find $PROGRAM -mindepth 1 -type d)
  do

    DONE_FILE="$ORIG_DIR/$low_level/done_$SIZE_STRING"
    if [[ ! -a $DONE_FILE ]]
    then
      cd $low_level

      timeout 3m harness_generic --folder $INPUTS --file $CONFIG -p $PLATFORM -d $DEVICE

      cd $ORIG_DIR
    fi
  done
done

error: pathspec 'scripts/execute.zsh' did not match any file(s) known to git


fix(prepare): properly handle ticket-based branch names

remote: Support for password authentication was removed on August 13, 2021.
remote: Please see https://docs.github.com/en/get-started/getting-started-with-git/about-remote-repositories#cloning-with-https-urls for information on currently recommended modes of authentication.
fatal: Authentication failed for 'https://github.com/tomkersten/githooks.git/'


lmp2dump.sh added

lmp2dump.sh added

#!/bin/bash
#
# uses lammps to convert a .lmp file to a LAMMPS dump file
#
# syntax: $0 <dump.xyz> <log.lammps>
#
# TODO: PBS options
# TODO: module load

# list of required software
required="lammps lmpcharges.sh"

# show syntax and exit
fail(){
  echo "syntax: $0 <data.lmp>" >&2
  exit 1
}

# find and print missing programs
missing=$(
for prog in $required; do
  type $prog >/dev/null 2>/dev/null || echo $prog
done
)
if [ -n "$missing" ];then
  echo "This script requires the following libraries:" >&2
  echo -e "$missing" | sed 's/^/* /' >&2
  echo
  fail
fi

# validate input
(( ${#@} == 1 )) || fail
lmpfile="$1"
[ -f "$lmpfile" ] || fail

# prepare directories
mkdir -p rdf

# preparing lmp copy

tmpfile=/tmp/pid$$_$RANDOM.lmp
lmpcharges.sh "$lmpfile" > $tmpfile
tmpfile2=`sed 's/\.lmp$/.dump/' <<< $tmpfile`

[ $tmpfile == $tmpfile2 ] && echo "internal error: tmpfile == tmpfile2" >&2 && exit

echo "running lammps"

numtypes="`sed -n 's/^\s*\([0-9]\+\)\s\+atom\stypes\s*$/\1/p' $tmpfile`"

(( $numtypes > 0 )) || echo "cannot read number of atom types from lmp file" >&2

setmasses="$(for f in `seq 1 $numtypes`; do
	echo "mass $f 1.0"
done)"

echo $setmasses

lammpscmds="
units real
atom_style charge
atom_modify sort 1 2.0
read_data $tmpfile
$setmasses
pair_style none
timestep 1
dump 0 all atom 2 $tmpfile2
fix 1 all nve
run 1
"

#echo -e "$lammpscmds"

# lammps run
lammps -nocite -echo screen -log none -suffix opt << EOF
$lammpscmds
EOF

echo

(( $? != 0 )) && echo "lammps failed!" || echo "dump conversion finished"

if [ "$tmpfile" != "$lmpfile" ]; then
  rm $tmpfile
fi

cat $tmpfile2
rm $tmpfile2

error: pathspec 'scripts/lmp2dump.sh' did not match any file(s) known to git


made make dist(-gz) work with submodules (using git-archive-all.sh)

remote: Support for password authentication was removed on August 13, 2021.
remote: Please see https://docs.github.com/en/get-started/getting-started-with-git/about-remote-repositories#cloning-with-https-urls for information on currently recommended modes of authentication.
fatal: Authentication failed for 'https://github.com/Eurospider/strusPrototypeModuleV1.git/'


dist: added script for merging Arduino sketches

#!/bin/sh

# check if at least the application dir and one sketch is given
if [ $# -lt 2 ]
then
    echo "[Arduino pre-build] Error: not enough arguments given"
    exit 1
fi

# 'consume' the application and arduino directories (first argument)
SRCDIR=$1
shift
APPDIR=$1
shift

# create temporary file and put in the file header
cat ${SRCDIR}/pre.snip > ${APPDIR}/_sketches.cpp
# loop through the given sketches and include them into the temp file
for sketch in $@
do
    cat ${sketch} >> ${APPDIR}/_sketches.cpp
done
# and prepend the file with the arduino bootstrapping code
cat ${SRCDIR}/post.snip >> ${APPDIR}/_sketches.cpp

error: pathspec 'dist/tools/arduino/pre_build.sh' did not match any file(s) known to git


Script to run the mpi example

Script to run the mpi example

git-svn-id: 9146c88ff6d39b48099bf954d15d68f687b3fa69@3069 28e8926c-6b08-0410-baaa-805c5e19b8d6

remote: Support for password authentication was removed on August 13, 2021.
remote: Please see https://docs.github.com/en/get-started/getting-started-with-git/about-remote-repositories#cloning-with-https-urls for information on currently recommended modes of authentication.
fatal: Authentication failed for 'https://github.com/lpellegr/scheduling.git/'


GH #44 - Enable hot deploy in Docker

GH #44 - Enable hot deploy in Docker

remote: Support for password authentication was removed on August 13, 2021.
remote: Please see https://docs.github.com/en/get-started/getting-started-with-git/about-remote-repositories#cloning-with-https-urls for information on currently recommended modes of authentication.
fatal: Authentication failed for 'https://github.com/zatosource/zato-build.git/'


ENH: Add bash template getter script

ENH: Add bash template getter script

#!/bin/bash

MNI_TEMPLATE="https://files.osf.io/v1/resources/fvuh8/providers/osfstorage/57f32a799ad5a101f977eb77"
MNI_SHA256="c7b7ca347c4bb8b7956d1d10515702f0b5e3a8c630e446a1c980e9bf6a549100"
ASYM_09C_TEMPLATE="https://files.osf.io/v1/resources/fvuh8/providers/osfstorage/580705089ad5a101f17944a9"
ASYM_09C_SHA256="a24699ba0d13f72d0f8934cc211cb80bfd9c9a077b481d9b64295cf5275235a9"
OASIS_TEMPLATE="https://files.osf.io/v1/resources/fvuh8/providers/osfstorage/584123a29ad5a1020913609d"
OASIS_SHA256="d87300e91346c16f55baf6f54f5f990bc020b61e8d5df9bcc3abb0cc4b943113"

GET(){
    URL=$1; SHA256=$2;
    mkfifo pipe.tar.gz
    cat pipe.tar.gz | tar zxv -C $CRN_SHARED_DATA &
    SHASUM=$(curl -sSL $URL | tee pipe.tar.gz | sha256sum | cut -d\  -f 1)
    rm pipe.tar.gz

    if [[ "$SHASUM" != "$SHA256" ]]; then
        echo "Failed checksum!"
        return 1
    fi
    return 0
}

set -e
echo "Getting MNI template"
GET "$MNI_TEMPLATE" "$MNI_SHA256"
echo "Getting MNI152NLin2009cAsym template"
GET "$ASYM_09C_TEMPLATE" "$ASYM_09C_SHA256"
echo "Getting OASIS template"
GET "$OASIS_TEMPLATE" "$OASIS_SHA256"
echo "Done!"

error: pathspec 'docker/scripts/get_templates.sh' did not match any file(s) known to git


post_install: removed extraneous  'System Preferences' call

remote: Support for password authentication was removed on August 13, 2021.
remote: Please see https://docs.github.com/en/get-started/getting-started-with-git/about-remote-repositories#cloning-with-https-urls for information on currently recommended modes of authentication.
fatal: Authentication failed for 'https://github.com/vitorgalvao/dotfiles.git/'


added script that facilitates changing debian/* files for building for a different distro

#!/bin/bash
#
# set-target-distro.sh
#
#######################################################################
#
#    Copyright 2013 Michael Chaberski
#
#    Licensed under the Apache License, Version 2.0 (the "License");
#    you may not use this file except in compliance with the License.
#    You may obtain a copy of the License at
#
#        http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS,
#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#    See the License for the specific language governing permissions and
#    limitations under the License.
#
#######################################################################
#~
#~ This script's purpose is to automate the changes to files
#~ in debian/* required in order to build (e.g. using
#~ pbuilder/pdebuild) for Ubuntu or for Debian distributions other
#~ than unstable.
#~
#~ For now, we're only building for Ubuntu/precise, but supporting other
#~ distros should be pretty transparent. (Just add alternatives to the
#~ case statement.
#~
#######################################################################

TARGET_DISTRO=$1

if [ -z "$TARGET_DISTRO" ] ; then
  echo "must specify target distro as argument 1" >&2
  exit 1
fi

DERIVATIVE_NAME=mikeppa

DERIVATIVE_VERSION=$2

if [ -z "$DERIVATIVE_VERSION" ] ; then
  DERIVATIVE_VERSION=1
fi

cp changelog changelog.orig
sed -i 's/nbis (\(.*\)) \(.*\);/nbis (\1'${DERIVATIVE_NAME}${DERIVATIVE_VERSION}') '${TARGET_DISTRO}';/' changelog

if [ $? -ne 0 ] ; then
  echo "modifying changelog failed" >&2
  exit 1
fi

diff changelog.orig changelog

STDS_VERSION="3.9.4"

case "$TARGET_DISTRO" in
  precise )
    STDS_VERSION="3.9.3" ;;
  unstable )
    STDS_VERSION="3.9.4" ;;
esac

cp control control.orig
sed -i 's/^Standards-Version: \(.\+\)$/Standards-Version: '${STDS_VERSION}'/' control
diff control.orig control

error: pathspec 'debian/set-target-distro.sh' did not match any file(s) known to git


add autogen

add autogen

#!/bin/sh

if [ "x$(uname)" = "xOpenBSD" ]; then
        [ -z "$AUTOMAKE_VERSION" ] && export AUTOMAKE_VERSION=1.10
        [ -z "$AUTOCONF_VERSION" ] && export AUTOCONF_VERSION=2.65
fi

die()
{
    echo "$@" >&2
    exit 1
}

mkdir -p etc
aclocal || die "aclocal failed"
autoconf || die "autoconf failed"
autoheader || die "autoheader failed"
automake --add-missing --force-missing --copy --foreign || die "automake failed"
autoreconf || die "autoreconf failed"

error: pathspec 'autogen.sh' did not match any file(s) known to git


Add autogen.sh

Add autogen.sh

#!/bin/sh

PKG_NAME="etherpush"

. gnome-autogen.sh


remove autoheader

remove autoheader

remote: Support for password authentication was removed on August 13, 2021.
remote: Please see https://docs.github.com/en/get-started/getting-started-with-git/about-remote-repositories#cloning-with-https-urls for information on currently recommended modes of authentication.
fatal: Authentication failed for 'https://github.com/tenforward/lxc-doc-ja.git/'