new script for refresh plugin upgrades

new script for refresh plugin upgrades

#!/bin/bash
#Created by Sam Gleske
#Sat 19 Feb 2022 02:28:59 PM EST
#Ubuntu 20.04.3 LTS
#Linux 5.13.0-30-generic x86_64
#GNU bash, version 5.0.17(1)-release (x86_64-pc-linux-gnu)

set -e

function cleanup_on() {
  #clean up jenkins headers file
  [ -n "${JENKINS_HEADERS_FILE}" -a -f "${JENKINS_HEADERS_FILE}" ] && rm -f "${JENKINS_HEADERS_FILE}"
  [ -n "${VAGRANT_SSH_CONFIG}" -a -f "${VAGRANT_SSH_CONFIG}" ] && rm -f "${VAGRANT_SSH_CONFIG}"
  if [ "$1" = '0' ]; then
    echo "Jenkins is ready.  Visit ${JENKINS_WEB}/"
    echo "User: ${JENKINS_USER}"
    echo "Password: ${JENKINS_PASSWORD}"
  fi
}
trap 'cleanup_on $?' EXIT

source env.sh
source "${SCRIPT_LIBRARY_PATH}"/upgrade/setup_environment.sh

#wait for jenkins to become available
"${SCRIPT_LIBRARY_PATH}"/provision_jenkins.sh url-ready "${JENKINS_WEB}/jnlpJars/jenkins-cli.jar"

if [ ! -f pinned-plugins.txt ]; then
  echo 'ERROR: No pinned plugins found so cannot proceed.' >&2
  exit 1
fi

"${SCRIPT_LIBRARY_PATH}"/jenkins-call-url "${SCRIPT_LIBRARY_PATH}"/upgrade/upgradeJenkinsOnly.groovy || (
  echo "Upgrading Jenkins and plugins failed.  Likely, jenkins.war or plugins"
  echo "are not writeable.  This is typical if upgrading jenkins.war via"
  echo "vagrant because jenkins.war is owned by root"
  exit 1
) >&2

# upgrade and restart if necessary
"${SCRIPT_LIBRARY_PATH}"/upgrade/wait_for_upgrade_to_complete.sh
if "${SCRIPT_LIBRARY_PATH}"/upgrade/jenkins_needs_restart.sh; then
  #set up Jenkins env vars post-restart
  source "${SCRIPT_LIBRARY_PATH}"/upgrade/env.sh
fi

"${SCRIPT_LIBRARY_PATH}"/jenkins-call-url "${SCRIPT_LIBRARY_PATH}"/upgrade/installMinimalPlugins.groovy --data-string "plugins='''$(< pinned-plugins.txt)'''.trim();"
"${SCRIPT_LIBRARY_PATH}"/upgrade/wait_for_upgrade_to_complete.sh
export NO_UPGRADE=1
"${SCRIPT_LIBRARY_PATH}"/upgrade/upgrade_build_gradle.sh*

error: pathspec 'scripts/upgrade/refresh_plugins.sh' did not match any file(s) known to git


select-apt-mirror.sh added

select-apt-mirror.sh added

#!/bin/bash
# This script changes the apt mirror to use.

function aptrunner {
    cmd="$1"
    retval=100
    count=0
    if [[ "$retval" -ne "0" && $count -le 5 ]]; then
        count=$count+1
        export DEBIAN_FRONTEND=noninteractive
        if [ "$cmd" == "update" ] ; then
            flags="-uy"
        else
            flags="-fuy"
        fi
        cmdline="apt-get -o Dpkg::Options::="--force-confnew" --force-yes $flags $cmd"
        echo "Running $cmdline"
        $cmdline
        retval=$?
    fi
}

function install {
    packages="$@"

    retval=100
    count=0

    if [[ "$retval" -ne "0" && $count -le 5 ]]; then
        count=$count+1
        DEBIAN_FRONTEND=noninteractive apt-get -o Dpkg::Options::="--force-confnew" --force-yes -fuy install $packages
        retval=$?
    fi
}

rm sources.list
install netselect-apt
sudo netselect-apt -s -o sources.list testing
mv sources.list /etc/apt/sources.list

aptrunner update
aptrunner dist-upgrade
aptrunner autoremove

error: pathspec 'deps/select-apt-mirror.sh' did not match any file(s) known to git


Check in notes on migration so others can view if they want

sudo -u postgres /usr/bin/createdb observatory -E UTF8 -l en_US.UTF8 -T template0

sudo -u postgres /usr/bin/psql -c "ALTER USER postgres PASSWORD 'zaq12wsxcde34rfv'"

sudo -u postgres psql < /vagrant/production/obs.dump.sql

sudo -u www-data /var/www/Observatory/observatory/manage.py migrate dashboard --noinput

sudo -u www-data /var/www/Observatory/observatory/manage.py migrate todo --fake --noinput

sudo -u www-data rsync /vagrant/production/screenshots/screenshots/* /var/www/Observatory/observatory/media/screenshots/ -vzP

error: pathspec 'production/load.sh' did not match any file(s) known to git


Holiday: add utility script to sanitize holiday names

#!/bin/bash

# this is a utility script to sanitize holiday names, it can be run whenever new holidays are introduced

for country in `find ./countries -iname '*.txt' | xargs`; do

    # sanitize holidays and move any alt names onto seperate lines
    cp $country $country.tmp
    cat $country.tmp | \
	tr '(' '\n' | \
	tr -d ')' | \
	tr '/' '\n' | \
	tr -d "'" | \
	tr -d '.' | \
	tr '-' ' ' | \
	tr -d '´' \
	> $country
    rm $country.tmp

    # remove duplicate entries that have a suffix of 2-9, eg: "eid al fitr holiday 2"
    sed -i '/day [2-9]/d' $country

    # sanitize any holiday names that have a suffix of 1
    sed -i 's/\(.*day\) 1$/\1/' $country

    # remove stray entries which result from "(day 1)" appearing in brackets after a holiday name
    sed -i '/^day$/d' $country

    # remove leading/trailing whitespace
    sed -i 's/^[[:space:]]*//' $country
    sed -i 's/[[:space:]]*$//' $country

    # sort and remove duplicate entries
    sort -u -o $country $country

done

error: pathspec 'share/spice/holiday/sanitize-holidays.sh' did not match any file(s) known to git


Create The World of Numbers.sh

Create The World of Numbers.sh

error: pathspec 'Bash/HackerRank/The' did not match any file(s) known to git
error: pathspec 'World' did not match any file(s) known to git
error: pathspec 'of' did not match any file(s) known to git
error: pathspec 'Numbers.sh' did not match any file(s) known to git


Add script for unpacking & transcoding Fineli zip

Download the zip and point the script at it and it does the rest,
unpacking into the backend seed data dir.

#!/bin/bash
# Update seed data from downloaded Fineli zip archive.
# Download the archive from http://www.fineli.fi/showpage.php?page=opendata.
set -e -o pipefail

SCRIPTDIR=$(cd -P $(dirname $0); pwd)
SEEDDIR=$SCRIPTDIR/seed_data
HELP_WANTED=0
RETCODE=0

err () {
  echo ERROR: "$@" >&2
}

while (( "$#" )); do
  arg=$1
  case "$arg" in
    -h|--help)
      HELP_WANTED=1
      break
      ;;
    *)
      if [[ -n $ARCHIVE ]]; then
        RETCODE=1
        HELP_WANTED=1
        break
      fi
      ARCHIVE=$arg
      ;;
  esac
  shift
done

if [[ 1 -eq $HELP_WANTED ]]; then
  cat <<EOF
Usage: $0 [OPTIONS] ARCHIVE

Where ARCHIVE is the path of the archive to import as new seed data, and
OPTIONS can be one or more of:

    -h | --help             Output this help.
EOF
elif [[ ! -f $ARCHIVE ]]; then
  err "$ARCHIVE is not a file."
  RETCODE=1
elif [[ ! -e $ARCHIVE ]]; then
  err "$ARCHIVE is not readable."
  RETCODE=1
elif [[ -z $(which unzip) ]]; then
  err "unzip not found on PATH."
  RETCODE=1
elif [[ -z $(which iconv) ]]; then
  err "iconv not found on PATH."
  RETCODE=1
else
  echo "Unpacking:"
  unzip -j "$ARCHIVE" -x '*.pdf' -d "$SEEDDIR"
  echo "Done unpacking."
  echo "Transcoding to UTF-8:"
  CSVS=($SEEDDIR/*.csv)
  for f in "${CSVS[@]}"; do
    echo "  $f"
    mv "$f" "$f".orig
    iconv -f iso-8859-1 -t utf-8 < "$f".orig > "$f"
    rm -f "$f".orig
  done
  echo "Done transcoding."
fi

exit $RETCODE

error: pathspec 'backend/update_seed_data.sh' did not match any file(s) known to git


Added dockerbuild-runner script that starts the contauner which builds the source

#!/bin/bash

docker run --rm --volume=$PWD:/tmp --workdir=/tmp build-image build.sh

$SHELL

error: pathspec 'dockerbuild.sh' did not match any file(s) known to git


Remove output SQL from tmp directory

Remove output SQL from tmp directory

Rudimentary cleanup system to remove output SQL from tmp directory to stop system from disk flooding.

remote: Support for password authentication was removed on August 13, 2021.
remote: Please see https://docs.github.com/en/get-started/getting-started-with-git/about-remote-repositories#cloning-with-https-urls for information on currently recommended modes of authentication.
fatal: Authentication failed for 'https://github.com/fresh8/script-repo.git/'


HADOOP-7813. Fix test-patch to use proper numerical comparison when checking javadoc and findbugs warning counts. Contributed by Jonathan Eagles.

git-svn-id: 092c8d3bf3a2a4ce66c091ad9796293e212d814c@1201055 13f79535-47bb-0310-9956-ffa450edef68

#!/usr/bin/env bash
#   Licensed under the Apache License, Version 2.0 (the "License");
#   you may not use this file except in compliance with the License.
#   You may obtain a copy of the License at
#
#       http://www.apache.org/licenses/LICENSE-2.0
#
#   Unless required by applicable law or agreed to in writing, software
#   distributed under the License is distributed on an "AS IS" BASIS,
#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#   See the License for the specific language governing permissions and
#   limitations under the License.

#set -x
ulimit -n 1024

### Setup some variables.
### SVN_REVISION and BUILD_URL are set by Hudson if it is run by patch process
### Read variables from properties file
bindir=$(dirname $0)

# Defaults
if [ -z "$MAVEN_HOME" ]; then
  MVN=mvn
else
  MVN=$MAVEN_HOME/bin/mvn
fi

PROJECT_NAME=Hadoop
JENKINS=false
PATCH_DIR=/tmp
SUPPORT_DIR=/tmp
BASEDIR=$(pwd)

PS=${PS:-ps}
AWK=${AWK:-awk}
WGET=${WGET:-wget}
SVN=${SVN:-svn}
GREP=${GREP:-grep}
PATCH=${PATCH:-patch}
JIRACLI=${JIRA:-jira}
FINDBUGS_HOME=${FINDBUGS_HOME}
FORREST_HOME=${FORREST_HOME}
ECLIPSE_HOME=${ECLIPSE_HOME}

###############################################################################
printUsage() {
  echo "Usage: $0 [options] patch-file | defect-number"
  echo
  echo "Where:"
  echo "  patch-file is a local patch file containing the changes to test"
  echo "  defect-number is a JIRA defect number (e.g. 'HADOOP-1234') to test (Jenkins only)"
  echo
  echo "Options:"
  echo "--patch-dir=<dir>      The directory for working and output files (default '/tmp')"
  echo "--basedir=<dir>        The directory to apply the patch to (default current directory)"
  echo "--mvn-cmd=<cmd>        The 'mvn' command to use (default \$MAVEN_HOME/bin/mvn, or 'mvn')"
  echo "--ps-cmd=<cmd>         The 'ps' command to use (default 'ps')"
  echo "--awk-cmd=<cmd>        The 'awk' command to use (default 'awk')"
  echo "--svn-cmd=<cmd>        The 'svn' command to use (default 'svn')"
  echo "--grep-cmd=<cmd>       The 'grep' command to use (default 'grep')"
  echo "--patch-cmd=<cmd>      The 'patch' command to use (default 'patch')"
  echo "--findbugs-home=<path> Findbugs home directory (default FINDBUGS_HOME environment variable)"
  echo "--forrest-home=<path>  Forrest home directory (default FORREST_HOME environment variable)"
  echo "--dirty-workspace      Allow the local SVN workspace to have uncommitted changes"
  echo "--run-tests            Run all tests below the base directory"
  echo
  echo "Jenkins-only options:"
  echo "--jenkins              Run by Jenkins (runs tests and posts results to JIRA)"
  echo "--support-dir=<dir>    The directory to find support files in"
  echo "--wget-cmd=<cmd>       The 'wget' command to use (default 'wget')"
  echo "--jira-cmd=<cmd>       The 'jira' command to use (default 'jira')"
  echo "--jira-password=<pw>   The password for the 'jira' command"
  echo "--eclipse-home=<path>  Eclipse home directory (default ECLIPSE_HOME environment variable)"
}

###############################################################################
parseArgs() {
  for i in $*
  do
    case $i in
    --jenkins)
      JENKINS=true
      ;;
    --patch-dir=*)
      PATCH_DIR=${i#*=}
      ;;
    --support-dir=*)
      SUPPORT_DIR=${i#*=}
      ;;
    --basedir=*)
      BASEDIR=${i#*=}
      ;;
    --mvn-cmd=*)
      MVN=${i#*=}
      ;;
    --ps-cmd=*)
      PS=${i#*=}
      ;;
    --awk-cmd=*)
      AWK=${i#*=}
      ;;
    --wget-cmd=*)
      WGET=${i#*=}
      ;;
    --svn-cmd=*)
      SVN=${i#*=}
      ;;
    --grep-cmd=*)
      GREP=${i#*=}
      ;;
    --patch-cmd=*)
      PATCH=${i#*=}
      ;;
    --jira-cmd=*)
      JIRACLI=${i#*=}
      ;;
    --jira-password=*)
      JIRA_PASSWD=${i#*=}
      ;;
    --findbugs-home=*)
      FINDBUGS_HOME=${i#*=}
      ;;
    --forrest-home=*)
      FORREST_HOME=${i#*=}
      ;;
    --eclipse-home=*)
      ECLIPSE_HOME=${i#*=}
      ;;
    --dirty-workspace)
      DIRTY_WORKSPACE=true
      ;;
    --run-tests)
      RUN_TESTS=true
      ;;
    *)
      PATCH_OR_DEFECT=$i
      ;;
    esac
  done
  if [ -z "$PATCH_OR_DEFECT" ]; then
    printUsage
    exit 1
  fi
  if [[ $JENKINS == "true" ]] ; then
    echo "Running in Jenkins mode"
    defect=$PATCH_OR_DEFECT
    ECLIPSE_PROPERTY="-Declipse.home=$ECLIPSE_HOME"
  else
    echo "Running in developer mode"
    JENKINS=false
    ### PATCH_FILE contains the location of the patchfile
    PATCH_FILE=$PATCH_OR_DEFECT
    if [[ ! -e "$PATCH_FILE" ]] ; then
      echo "Unable to locate the patch file $PATCH_FILE"
      cleanupAndExit 0
    fi
    ### Check if $PATCH_DIR exists. If it does not exist, create a new directory
    if [[ ! -e "$PATCH_DIR" ]] ; then
      mkdir "$PATCH_DIR"
      if [[ $? == 0 ]] ; then
        echo "$PATCH_DIR has been created"
      else
        echo "Unable to create $PATCH_DIR"
        cleanupAndExit 0
      fi
    fi
    ### Obtain the patch filename to append it to the version number
    defect=`basename $PATCH_FILE`
  fi
}

###############################################################################
checkout () {
  echo ""
  echo ""
  echo "======================================================================"
  echo "======================================================================"
  echo "    Testing patch for ${defect}."
  echo "======================================================================"
  echo "======================================================================"
  echo ""
  echo ""
  ### When run by a developer, if the workspace contains modifications, do not continue
  ### unless the --dirty-workspace option was set
  status=`$SVN stat --ignore-externals | sed -e '/^X[ ]*/D'`
  if [[ $JENKINS == "false" ]] ; then
    if [[ "$status" != "" && -z $DIRTY_WORKSPACE ]] ; then
      echo "ERROR: can't run in a workspace that contains the following modifications"
      echo "$status"
      cleanupAndExit 1
    fi
    echo
  else
    cd $BASEDIR
    $SVN revert -R .
    rm -rf `$SVN status --no-ignore`
    $SVN update
  fi
  return $?
}

###############################################################################
setup () {
  ### Download latest patch file (ignoring .htm and .html) when run from patch process
  if [[ $JENKINS == "true" ]] ; then
    $WGET -q -O $PATCH_DIR/jira http://issues.apache.org/jira/browse/$defect
    if [[ `$GREP -c 'Patch Available' $PATCH_DIR/jira` == 0 ]] ; then
      echo "$defect is not \"Patch Available\".  Exiting."
      cleanupAndExit 0
    fi
    relativePatchURL=`$GREP -o '"/jira/secure/attachment/[0-9]*/[^"]*' $PATCH_DIR/jira | $GREP -v -e 'htm[l]*$' | sort | tail -1 | $GREP -o '/jira/secure/attachment/[0-9]*/[^"]*'`
    patchURL="http://issues.apache.org${relativePatchURL}"
    patchNum=`echo $patchURL | $GREP -o '[0-9]*/' | $GREP -o '[0-9]*'`
    echo "$defect patch is being downloaded at `date` from"
    echo "$patchURL"
    $WGET -q -O $PATCH_DIR/patch $patchURL
    VERSION=${SVN_REVISION}_${defect}_PATCH-${patchNum}
    JIRA_COMMENT="Here are the results of testing the latest attachment
  $patchURL
  against trunk revision ${SVN_REVISION}."

    ### Copy in any supporting files needed by this process
    cp -r $SUPPORT_DIR/lib/* ./lib
    #PENDING: cp -f $SUPPORT_DIR/etc/checkstyle* ./src/test
  ### Copy the patch file to $PATCH_DIR
  else
    VERSION=PATCH-${defect}
    cp $PATCH_FILE $PATCH_DIR/patch
    if [[ $? == 0 ]] ; then
      echo "Patch file $PATCH_FILE copied to $PATCH_DIR"
    else
      echo "Could not copy $PATCH_FILE to $PATCH_DIR"
      cleanupAndExit 0
    fi
  fi
  . $BASEDIR/dev-support/test-patch.properties
  ### exit if warnings are NOT defined in the properties file
  if [ -z "$OK_FINDBUGS_WARNINGS" ] || [[ -z "$OK_JAVADOC_WARNINGS" ]] || [[ -z $OK_RELEASEAUDIT_WARNINGS ]]; then
    echo "Please define the following properties in test-patch.properties file"
	 echo  "OK_FINDBUGS_WARNINGS"
	 echo  "OK_RELEASEAUDIT_WARNINGS"
	 echo  "OK_JAVADOC_WARNINGS"
    cleanupAndExit 1
  fi
  echo ""
  echo ""
  echo "======================================================================"
  echo "======================================================================"
  echo " Pre-build trunk to verify trunk stability and javac warnings"
  echo "======================================================================"
  echo "======================================================================"
  echo ""
  echo ""
  if [[ ! -d hadoop-common-project ]]; then
    cd $bindir/..
    echo "Compiling $(pwd)"
    echo "$MVN clean test -DskipTests > $PATCH_DIR/trunkCompile.txt 2>&1"
    $MVN clean test -DskipTests > $PATCH_DIR/trunkCompile.txt 2>&1
    if [[ $? != 0 ]] ; then
      echo "Top-level trunk compilation is broken?"
      cleanupAndExit 1
    fi
    cd -
  fi
  echo "Compiling $(pwd)"
  echo "$MVN clean test -DskipTests -D${PROJECT_NAME}PatchProcess -Ptest-patch > $PATCH_DIR/trunkJavacWarnings.txt 2>&1"
  $MVN clean test -DskipTests -D${PROJECT_NAME}PatchProcess -Ptest-patch > $PATCH_DIR/trunkJavacWarnings.txt 2>&1
  if [[ $? != 0 ]] ; then
    echo "Trunk compilation is broken?"
    cleanupAndExit 1
  fi
}

###############################################################################
### Check for @author tags in the patch
checkAuthor () {
  echo ""
  echo ""
  echo "======================================================================"
  echo "======================================================================"
  echo "    Checking there are no @author tags in the patch."
  echo "======================================================================"
  echo "======================================================================"
  echo ""
  echo ""
  authorTags=`$GREP -c -i '@author' $PATCH_DIR/patch`
  echo "There appear to be $authorTags @author tags in the patch."
  if [[ $authorTags != 0 ]] ; then
    JIRA_COMMENT="$JIRA_COMMENT

    -1 @author.  The patch appears to contain $authorTags @author tags which the Hadoop community has agreed to not allow in code contributions."
    return 1
  fi
  JIRA_COMMENT="$JIRA_COMMENT

    +1 @author.  The patch does not contain any @author tags."
  return 0
}

###############################################################################
### Check for tests in the patch
checkTests () {
  echo ""
  echo ""
  echo "======================================================================"
  echo "======================================================================"
  echo "    Checking there are new or changed tests in the patch."
  echo "======================================================================"
  echo "======================================================================"
  echo ""
  echo ""
  testReferences=`$GREP -c -i '/test' $PATCH_DIR/patch`
  echo "There appear to be $testReferences test files referenced in the patch."
  if [[ $testReferences == 0 ]] ; then
    if [[ $JENKINS == "true" ]] ; then
      patchIsDoc=`$GREP -c -i 'title="documentation' $PATCH_DIR/jira`
      if [[ $patchIsDoc != 0 ]] ; then
        echo "The patch appears to be a documentation patch that doesn't require tests."
        JIRA_COMMENT="$JIRA_COMMENT

    +0 tests included.  The patch appears to be a documentation patch that doesn't require tests."
        return 0
      fi
    fi
    JIRA_COMMENT="$JIRA_COMMENT

    -1 tests included.  The patch doesn't appear to include any new or modified tests.
                        Please justify why no new tests are needed for this patch.
                        Also please list what manual steps were performed to verify this patch."
    return 1
  fi
  JIRA_COMMENT="$JIRA_COMMENT

    +1 tests included.  The patch appears to include $testReferences new or modified tests."
  return 0
}

cleanUpXml () {
  cd $BASEDIR/conf
  for file in `ls *.xml.template`
    do
      rm -f `basename $file .template`
    done
  cd $BASEDIR
}

###############################################################################
### Attempt to apply the patch
applyPatch () {
  echo ""
  echo ""
  echo "======================================================================"
  echo "======================================================================"
  echo "    Applying patch."
  echo "======================================================================"
  echo "======================================================================"
  echo ""
  echo ""
  export PATCH
  $bindir/smart-apply-patch.sh $PATCH_DIR/patch
  if [[ $? != 0 ]] ; then
    echo "PATCH APPLICATION FAILED"
    JIRA_COMMENT="$JIRA_COMMENT

    -1 patch.  The patch command could not apply the patch."
    return 1
  fi
  return 0
}

###############################################################################
### Check there are no javadoc warnings
checkJavadocWarnings () {
  echo ""
  echo ""
  echo "======================================================================"
  echo "======================================================================"
  echo "    Determining number of patched javadoc warnings."
  echo "======================================================================"
  echo "======================================================================"
  echo ""
  echo ""
  echo "$MVN clean test javadoc:javadoc -DskipTests -Pdocs -D${PROJECT_NAME}PatchProcess > $PATCH_DIR/patchJavadocWarnings.txt 2>&1"
  if [ -d hadoop-project ]; then
    (cd hadoop-project; $MVN install)
  fi
  if [ -d hadoop-common-project/hadoop-annotations ]; then
    (cd hadoop-common-project/hadoop-annotations; $MVN install)
  fi
  $MVN clean test javadoc:javadoc -DskipTests -Pdocs -D${PROJECT_NAME}PatchProcess > $PATCH_DIR/patchJavadocWarnings.txt 2>&1
  javadocWarnings=`$GREP '\[WARNING\]' $PATCH_DIR/patchJavadocWarnings.txt | $AWK '/Javadoc Warnings/,EOF' | $GREP warning | $AWK 'BEGIN {total = 0} {total += 1} END {print total}'`
  echo ""
  echo ""
  echo "There appear to be $javadocWarnings javadoc warnings generated by the patched build."

  ### if current warnings greater than OK_JAVADOC_WARNINGS
  if [[ $javadocWarnings -gt $OK_JAVADOC_WARNINGS ]] ; then
    JIRA_COMMENT="$JIRA_COMMENT

    -1 javadoc.  The javadoc tool appears to have generated `expr $(($javadocWarnings-$OK_JAVADOC_WARNINGS))` warning messages."
    return 1
  fi
  JIRA_COMMENT="$JIRA_COMMENT

    +1 javadoc.  The javadoc tool did not generate any warning messages."
  return 0
}

###############################################################################
### Check there are no changes in the number of Javac warnings
checkJavacWarnings () {
  echo ""
  echo ""
  echo "======================================================================"
  echo "======================================================================"
  echo "    Determining number of patched javac warnings."
  echo "======================================================================"
  echo "======================================================================"
  echo ""
  echo ""
  echo "$MVN clean test -DskipTests -D${PROJECT_NAME}PatchProcess -Ptest-patch > $PATCH_DIR/patchJavacWarnings.txt 2>&1"
  $MVN clean test -DskipTests -D${PROJECT_NAME}PatchProcess -Ptest-patch > $PATCH_DIR/patchJavacWarnings.txt 2>&1
  if [[ $? != 0 ]] ; then
    JIRA_COMMENT="$JIRA_COMMENT

    -1 javac.  The patch appears to cause tar ant target to fail."
    return 1
  fi
  ### Compare trunk and patch javac warning numbers
  if [[ -f $PATCH_DIR/patchJavacWarnings.txt ]] ; then
    trunkJavacWarnings=`$GREP '\[WARNING\]' $PATCH_DIR/trunkJavacWarnings.txt | $AWK 'BEGIN {total = 0} {total += 1} END {print total}'`
    patchJavacWarnings=`$GREP '\[WARNING\]' $PATCH_DIR/patchJavacWarnings.txt | $AWK 'BEGIN {total = 0} {total += 1} END {print total}'`
    echo "There appear to be $trunkJavacWarnings javac compiler warnings before the patch and $patchJavacWarnings javac compiler warnings after applying the patch."
    if [[ $patchJavacWarnings != "" && $trunkJavacWarnings != "" ]] ; then
      if [[ $patchJavacWarnings -gt $trunkJavacWarnings ]] ; then
        JIRA_COMMENT="$JIRA_COMMENT

    -1 javac.  The applied patch generated $patchJavacWarnings javac compiler warnings (more than the trunk's current $trunkJavacWarnings warnings)."
        return 1
      fi
    fi
  fi
  JIRA_COMMENT="$JIRA_COMMENT

    +1 javac.  The applied patch does not increase the total number of javac compiler warnings."
  return 0
}

###############################################################################
### Check there are no changes in the number of release audit (RAT) warnings
checkReleaseAuditWarnings () {
  echo ""
  echo ""
  echo "======================================================================"
  echo "======================================================================"
  echo "    Determining number of patched release audit warnings."
  echo "======================================================================"
  echo "======================================================================"
  echo ""
  echo ""
  echo "$MVN apache-rat:check -D${PROJECT_NAME}PatchProcess 2>&1"
  $MVN apache-rat:check -D${PROJECT_NAME}PatchProcess 2>&1
  find $BASEDIR -name rat.txt | xargs cat > $PATCH_DIR/patchReleaseAuditWarnings.txt

  ### Compare trunk and patch release audit warning numbers
  if [[ -f $PATCH_DIR/patchReleaseAuditWarnings.txt ]] ; then
    patchReleaseAuditWarnings=`$GREP -c '\!?????' $PATCH_DIR/patchReleaseAuditWarnings.txt`
    echo ""
    echo ""
    echo "There appear to be $OK_RELEASEAUDIT_WARNINGS release audit warnings before the patch and $patchReleaseAuditWarnings release audit warnings after applying the patch."
    if [[ $patchReleaseAuditWarnings != "" && $OK_RELEASEAUDIT_WARNINGS != "" ]] ; then
      if [[ $patchReleaseAuditWarnings -gt $OK_RELEASEAUDIT_WARNINGS ]] ; then
        JIRA_COMMENT="$JIRA_COMMENT

    -1 release audit.  The applied patch generated $patchReleaseAuditWarnings release audit warnings (more than the trunk's current $OK_RELEASEAUDIT_WARNINGS warnings)."
        $GREP '\!?????' $PATCH_DIR/patchReleaseAuditWarnings.txt > $PATCH_DIR/patchReleaseAuditProblems.txt
        echo "Lines that start with ????? in the release audit report indicate files that do not have an Apache license header." >> $PATCH_DIR/patchReleaseAuditProblems.txt
        JIRA_COMMENT_FOOTER="Release audit warnings: $BUILD_URL/artifact/trunk/patchprocess/patchReleaseAuditProblems.txt
$JIRA_COMMENT_FOOTER"
        return 1
      fi
    fi
  fi
  JIRA_COMMENT="$JIRA_COMMENT

    +1 release audit.  The applied patch does not increase the total number of release audit warnings."
  return 0
}

###############################################################################
### Check there are no changes in the number of Checkstyle warnings
checkStyle () {
  echo ""
  echo ""
  echo "======================================================================"
  echo "======================================================================"
  echo "    Determining number of patched checkstyle warnings."
  echo "======================================================================"
  echo "======================================================================"
  echo ""
  echo ""
  echo "THIS IS NOT IMPLEMENTED YET"
  echo ""
  echo ""
  echo "$MVN test checkstyle:checkstyle -DskipTests -D${PROJECT_NAME}PatchProcess"
  $MVN test checkstyle:checkstyle -DskipTests -D${PROJECT_NAME}PatchProcess

  JIRA_COMMENT_FOOTER="Checkstyle results: $BUILD_URL/artifact/trunk/build/test/checkstyle-errors.html
$JIRA_COMMENT_FOOTER"
  ### TODO: calculate actual patchStyleErrors
#  patchStyleErrors=0
#  if [[ $patchStyleErrors != 0 ]] ; then
#    JIRA_COMMENT="$JIRA_COMMENT
#
#    -1 checkstyle.  The patch generated $patchStyleErrors code style errors."
#    return 1
#  fi
#  JIRA_COMMENT="$JIRA_COMMENT
#
#    +1 checkstyle.  The patch generated 0 code style errors."
  return 0
}

###############################################################################
### Check there are no changes in the number of Findbugs warnings
checkFindbugsWarnings () {
  findbugs_version=`${FINDBUGS_HOME}/bin/findbugs -version`
  echo ""
  echo ""
  echo "======================================================================"
  echo "======================================================================"
  echo "    Determining number of patched Findbugs warnings."
  echo "======================================================================"
  echo "======================================================================"
  echo ""
  echo ""
  echo "$MVN clean test findbugs:findbugs -DskipTests -D${PROJECT_NAME}PatchProcess"
  $MVN clean test findbugs:findbugs -DskipTests -D${PROJECT_NAME}PatchProcess < /dev/null

  if [ $? != 0 ] ; then
    JIRA_COMMENT="$JIRA_COMMENT

    -1 findbugs.  The patch appears to cause Findbugs (version ${findbugs_version}) to fail."
    return 1
  fi

  findbugsWarnings=0
  for file in $(find $BASEDIR -name findbugsXml.xml)
  do
    relative_file=${file#$BASEDIR/} # strip leading $BASEDIR prefix
    if [ ! $relative_file == "target/findbugsXml.xml" ]; then
      module_suffix=${relative_file%/target/findbugsXml.xml} # strip trailing path
      module_suffix=`basename ${module_suffix}`
    fi

    cp $file $PATCH_DIR/patchFindbugsWarnings${module_suffix}.xml
    $FINDBUGS_HOME/bin/setBugDatabaseInfo -timestamp "01/01/2000" \
      $PATCH_DIR/patchFindbugsWarnings${module_suffix}.xml \
      $PATCH_DIR/patchFindbugsWarnings${module_suffix}.xml
    newFindbugsWarnings=`$FINDBUGS_HOME/bin/filterBugs -first "01/01/2000" $PATCH_DIR/patchFindbugsWarnings${module_suffix}.xml \
      $PATCH_DIR/newPatchFindbugsWarnings${module_suffix}.xml | $AWK '{print $1}'`
    echo "Found $newFindbugsWarnings Findbugs warnings ($file)"
    findbugsWarnings=$((findbugsWarnings+newFindbugsWarnings))
    $FINDBUGS_HOME/bin/convertXmlToText -html \
      $PATCH_DIR/newPatchFindbugsWarnings${module_suffix}.xml \
      $PATCH_DIR/newPatchFindbugsWarnings${module_suffix}.html
    JIRA_COMMENT_FOOTER="Findbugs warnings: $BUILD_URL/artifact/trunk/patchprocess/newPatchFindbugsWarnings${module_suffix}.html
$JIRA_COMMENT_FOOTER"
  done

  ### if current warnings greater than OK_FINDBUGS_WARNINGS
  if [[ $findbugsWarnings -gt $OK_FINDBUGS_WARNINGS ]] ; then
    JIRA_COMMENT="$JIRA_COMMENT

    -1 findbugs.  The patch appears to introduce `expr $(($findbugsWarnings-$OK_FINDBUGS_WARNINGS))` new Findbugs (version ${findbugs_version}) warnings."
    return 1
  fi
  JIRA_COMMENT="$JIRA_COMMENT

    +1 findbugs.  The patch does not introduce any new Findbugs (version ${findbugs_version}) warnings."
  return 0
}

###############################################################################
### Run the tests
runTests () {
  echo ""
  echo ""
  echo "======================================================================"
  echo "======================================================================"
  echo "    Running tests."
  echo "======================================================================"
  echo "======================================================================"
  echo ""
  echo ""

  echo "$MVN clean install -Pnative -D${PROJECT_NAME}PatchProcess"
  $MVN clean install -Pnative -D${PROJECT_NAME}PatchProcess
  if [[ $? != 0 ]] ; then
    ### Find and format names of failed tests
    failed_tests=`find . -name 'TEST*.xml' | xargs $GREP  -l -E "<failure|<error" | sed -e "s|.*target/surefire-reports/TEST-|                  |g" | sed -e "s|\.xml||g"`

    if [[ -n "$failed_tests" ]] ; then
      JIRA_COMMENT="$JIRA_COMMENT

    -1 core tests.  The patch failed these unit tests:
$failed_tests"
    else
      JIRA_COMMENT="$JIRA_COMMENT

    -1 core tests.  The patch failed the unit tests build"
    fi
    return 1
  fi
  JIRA_COMMENT="$JIRA_COMMENT

    +1 core tests.  The patch passed unit tests in $modules."
  return 0
}

###############################################################################
### Run the test-contrib target
runContribTests () {
  echo ""
  echo ""
  echo "======================================================================"
  echo "======================================================================"
  echo "    Running contrib tests."
  echo "======================================================================"
  echo "======================================================================"
  echo ""
  echo ""

  if [[ `$GREP -c 'test-contrib' build.xml` == 0 ]] ; then
    echo "No contrib tests in this project."
    return 0
  fi

  ### Kill any rogue build processes from the last attempt
  $PS auxwww | $GREP ${PROJECT_NAME}PatchProcess | $AWK '{print $2}' | /usr/bin/xargs -t -I {} /bin/kill -9 {} > /dev/null

  #echo "$ANT_HOME/bin/ant -Dversion="${VERSION}" $ECLIPSE_PROPERTY -DHadoopPatchProcess= -Dtest.junit.output.format=xml -Dtest.output=no test-contrib"
  #$ANT_HOME/bin/ant -Dversion="${VERSION}" $ECLIPSE_PROPERTY -DHadoopPatchProcess= -Dtest.junit.output.format=xml -Dtest.output=no test-contrib
  echo "NOP"
  if [[ $? != 0 ]] ; then
    JIRA_COMMENT="$JIRA_COMMENT

    -1 contrib tests.  The patch failed contrib unit tests."
    return 1
  fi
  JIRA_COMMENT="$JIRA_COMMENT

    +1 contrib tests.  The patch passed contrib unit tests."
  return 0
}

###############################################################################
### Run the inject-system-faults target
checkInjectSystemFaults () {
  echo ""
  echo ""
  echo "======================================================================"
  echo "======================================================================"
  echo "    Checking the integrity of system test framework code."
  echo "======================================================================"
  echo "======================================================================"
  echo ""
  echo ""

  ### Kill any rogue build processes from the last attempt
  $PS auxwww | $GREP ${PROJECT_NAME}PatchProcess | $AWK '{print $2}' | /usr/bin/xargs -t -I {} /bin/kill -9 {} > /dev/null

  #echo "$ANT_HOME/bin/ant -Dversion="${VERSION}" -DHadoopPatchProcess= -Dtest.junit.output.format=xml -Dtest.output=no -Dcompile.c++=yes -Dforrest.home=$FORREST_HOME inject-system-faults"
  #$ANT_HOME/bin/ant -Dversion="${VERSION}" -DHadoopPatchProcess= -Dtest.junit.output.format=xml -Dtest.output=no -Dcompile.c++=yes -Dforrest.home=$FORREST_HOME inject-system-faults
  echo "NOP"
  return 0
  if [[ $? != 0 ]] ; then
    JIRA_COMMENT="$JIRA_COMMENT

    -1 system test framework.  The patch failed system test framework compile."
    return 1
  fi
  JIRA_COMMENT="$JIRA_COMMENT

    +1 system test framework.  The patch passed system test framework compile."
  return 0
}

###############################################################################
### Submit a comment to the defect's Jira
submitJiraComment () {
  local result=$1
  ### Do not output the value of JIRA_COMMENT_FOOTER when run by a developer
  if [[  $JENKINS == "false" ]] ; then
    JIRA_COMMENT_FOOTER=""
  fi
  if [[ $result == 0 ]] ; then
    comment="+1 overall.  $JIRA_COMMENT

$JIRA_COMMENT_FOOTER"
  else
    comment="-1 overall.  $JIRA_COMMENT

$JIRA_COMMENT_FOOTER"
  fi
  ### Output the test result to the console
  echo "

$comment"

  if [[ $JENKINS == "true" ]] ; then
    echo ""
    echo ""
    echo "======================================================================"
    echo "======================================================================"
    echo "    Adding comment to Jira."
    echo "======================================================================"
    echo "======================================================================"
    echo ""
    echo ""
    ### Update Jira with a comment
    export USER=hudson
    $JIRACLI -s https://issues.apache.org/jira -a addcomment -u hadoopqa -p $JIRA_PASSWD --comment "$comment" --issue $defect
    $JIRACLI -s https://issues.apache.org/jira -a logout -u hadoopqa -p $JIRA_PASSWD
  fi
}

###############################################################################
### Cleanup files
cleanupAndExit () {
  local result=$1
  if [[ $JENKINS == "true" ]] ; then
    if [ -e "$PATCH_DIR" ] ; then
      mv $PATCH_DIR $BASEDIR
    fi
  fi
  echo ""
  echo ""
  echo "======================================================================"
  echo "======================================================================"
  echo "    Finished build."
  echo "======================================================================"
  echo "======================================================================"
  echo ""
  echo ""
  exit $result
}

###############################################################################
###############################################################################
###############################################################################

JIRA_COMMENT=""
JIRA_COMMENT_FOOTER="Console output: $BUILD_URL/console

This message is automatically generated."

### Check if arguments to the script have been specified properly or not
parseArgs $@
cd $BASEDIR

checkout
RESULT=$?
if [[ $JENKINS == "true" ]] ; then
  if [[ $RESULT != 0 ]] ; then
    exit 100
  fi
fi
setup
checkAuthor
RESULT=$?

if [[ $JENKINS == "true" ]] ; then
  cleanUpXml
fi
checkTests
(( RESULT = RESULT + $? ))
applyPatch
if [[ $? != 0 ]] ; then
  submitJiraComment 1
  cleanupAndExit 1
fi
checkJavadocWarnings
(( RESULT = RESULT + $? ))
checkJavacWarnings
(( RESULT = RESULT + $? ))
### Checkstyle not implemented yet
#checkStyle
#(( RESULT = RESULT + $? ))
checkFindbugsWarnings
(( RESULT = RESULT + $? ))
checkReleaseAuditWarnings
(( RESULT = RESULT + $? ))
### Run tests for Jenkins or if explictly asked for by a developer
if [[ $JENKINS == "true" || $RUN_TESTS == "true" ]] ; then
  runTests
  (( RESULT = RESULT + $? ))
  runContribTests
  (( RESULT = RESULT + $? ))
fi
checkInjectSystemFaults
(( RESULT = RESULT + $? ))
JIRA_COMMENT_FOOTER="Test results: $BUILD_URL/testReport/
$JIRA_COMMENT_FOOTER"

submitJiraComment $RESULT
cleanupAndExit $RESULT

#!/usr/bin/env bash
#   Licensed under the Apache License, Version 2.0 (the "License");
#   you may not use this file except in compliance with the License.
#   You may obtain a copy of the License at
#
#       http://www.apache.org/licenses/LICENSE-2.0
#
#   Unless required by applicable law or agreed to in writing, software
#   distributed under the License is distributed on an "AS IS" BASIS,
#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#   See the License for the specific language governing permissions and
#   limitations under the License.

#set -x
ulimit -n 1024

### Setup some variables.
### SVN_REVISION and BUILD_URL are set by Hudson if it is run by patch process
### Read variables from properties file
bindir=$(dirname $0)

# Defaults
if [ -z "$MAVEN_HOME" ]; then
  MVN=mvn
else
  MVN=$MAVEN_HOME/bin/mvn
fi

PROJECT_NAME=Hadoop
JENKINS=false
PATCH_DIR=/tmp
SUPPORT_DIR=/tmp
BASEDIR=$(pwd)

PS=${PS:-ps}
AWK=${AWK:-awk}
WGET=${WGET:-wget}
SVN=${SVN:-svn}
GREP=${GREP:-grep}
PATCH=${PATCH:-patch}
JIRACLI=${JIRA:-jira}
FINDBUGS_HOME=${FINDBUGS_HOME}
FORREST_HOME=${FORREST_HOME}
ECLIPSE_HOME=${ECLIPSE_HOME}

###############################################################################
printUsage() {
  echo "Usage: $0 [options] patch-file | defect-number"
  echo
  echo "Where:"
  echo "  patch-file is a local patch file containing the changes to test"
  echo "  defect-number is a JIRA defect number (e.g. 'HADOOP-1234') to test (Jenkins only)"
  echo
  echo "Options:"
  echo "--patch-dir=<dir>      The directory for working and output files (default '/tmp')"
  echo "--basedir=<dir>        The directory to apply the patch to (default current directory)"
  echo "--mvn-cmd=<cmd>        The 'mvn' command to use (default \$MAVEN_HOME/bin/mvn, or 'mvn')"
  echo "--ps-cmd=<cmd>         The 'ps' command to use (default 'ps')"
  echo "--awk-cmd=<cmd>        The 'awk' command to use (default 'awk')"
  echo "--svn-cmd=<cmd>        The 'svn' command to use (default 'svn')"
  echo "--grep-cmd=<cmd>       The 'grep' command to use (default 'grep')"
  echo "--patch-cmd=<cmd>      The 'patch' command to use (default 'patch')"
  echo "--findbugs-home=<path> Findbugs home directory (default FINDBUGS_HOME environment variable)"
  echo "--forrest-home=<path>  Forrest home directory (default FORREST_HOME environment variable)"
  echo "--dirty-workspace      Allow the local SVN workspace to have uncommitted changes"
  echo "--run-tests            Run all tests below the base directory"
  echo
  echo "Jenkins-only options:"
  echo "--jenkins              Run by Jenkins (runs tests and posts results to JIRA)"
  echo "--support-dir=<dir>    The directory to find support files in"
  echo "--wget-cmd=<cmd>       The 'wget' command to use (default 'wget')"
  echo "--jira-cmd=<cmd>       The 'jira' command to use (default 'jira')"
  echo "--jira-password=<pw>   The password for the 'jira' command"
  echo "--eclipse-home=<path>  Eclipse home directory (default ECLIPSE_HOME environment variable)"
}

###############################################################################
parseArgs() {
  for i in $*
  do
    case $i in
    --jenkins)
      JENKINS=true
      ;;
    --patch-dir=*)
      PATCH_DIR=${i#*=}
      ;;
    --support-dir=*)
      SUPPORT_DIR=${i#*=}
      ;;
    --basedir=*)
      BASEDIR=${i#*=}
      ;;
    --mvn-cmd=*)
      MVN=${i#*=}
      ;;
    --ps-cmd=*)
      PS=${i#*=}
      ;;
    --awk-cmd=*)
      AWK=${i#*=}
      ;;
    --wget-cmd=*)
      WGET=${i#*=}
      ;;
    --svn-cmd=*)
      SVN=${i#*=}
      ;;
    --grep-cmd=*)
      GREP=${i#*=}
      ;;
    --patch-cmd=*)
      PATCH=${i#*=}
      ;;
    --jira-cmd=*)
      JIRACLI=${i#*=}
      ;;
    --jira-password=*)
      JIRA_PASSWD=${i#*=}
      ;;
    --findbugs-home=*)
      FINDBUGS_HOME=${i#*=}
      ;;
    --forrest-home=*)
      FORREST_HOME=${i#*=}
      ;;
    --eclipse-home=*)
      ECLIPSE_HOME=${i#*=}
      ;;
    --dirty-workspace)
      DIRTY_WORKSPACE=true
      ;;
    --run-tests)
      RUN_TESTS=true
      ;;
    *)
      PATCH_OR_DEFECT=$i
      ;;
    esac
  done
  if [ -z "$PATCH_OR_DEFECT" ]; then
    printUsage
    exit 1
  fi
  if [[ $JENKINS == "true" ]] ; then
    echo "Running in Jenkins mode"
    defect=$PATCH_OR_DEFECT
    ECLIPSE_PROPERTY="-Declipse.home=$ECLIPSE_HOME"
  else
    echo "Running in developer mode"
    JENKINS=false
    ### PATCH_FILE contains the location of the patchfile
    PATCH_FILE=$PATCH_OR_DEFECT
    if [[ ! -e "$PATCH_FILE" ]] ; then
      echo "Unable to locate the patch file $PATCH_FILE"
      cleanupAndExit 0
    fi
    ### Check if $PATCH_DIR exists. If it does not exist, create a new directory
    if [[ ! -e "$PATCH_DIR" ]] ; then
      mkdir "$PATCH_DIR"
      if [[ $? == 0 ]] ; then
        echo "$PATCH_DIR has been created"
      else
        echo "Unable to create $PATCH_DIR"
        cleanupAndExit 0
      fi
    fi
    ### Obtain the patch filename to append it to the version number
    defect=`basename $PATCH_FILE`
  fi
}

###############################################################################
checkout () {
  echo ""
  echo ""
  echo "======================================================================"
  echo "======================================================================"
  echo "    Testing patch for ${defect}."
  echo "======================================================================"
  echo "======================================================================"
  echo ""
  echo ""
  ### When run by a developer, if the workspace contains modifications, do not continue
  ### unless the --dirty-workspace option was set
  status=`$SVN stat --ignore-externals | sed -e '/^X[ ]*/D'`
  if [[ $JENKINS == "false" ]] ; then
    if [[ "$status" != "" && -z $DIRTY_WORKSPACE ]] ; then
      echo "ERROR: can't run in a workspace that contains the following modifications"
      echo "$status"
      cleanupAndExit 1
    fi
    echo
  else
    cd $BASEDIR
    $SVN revert -R .
    rm -rf `$SVN status --no-ignore`
    $SVN update
  fi
  return $?
}

###############################################################################
setup () {
  ### Download latest patch file (ignoring .htm and .html) when run from patch process
  if [[ $JENKINS == "true" ]] ; then
    $WGET -q -O $PATCH_DIR/jira http://issues.apache.org/jira/browse/$defect
    if [[ `$GREP -c 'Patch Available' $PATCH_DIR/jira` == 0 ]] ; then
      echo "$defect is not \"Patch Available\".  Exiting."
      cleanupAndExit 0
    fi
    relativePatchURL=`$GREP -o '"/jira/secure/attachment/[0-9]*/[^"]*' $PATCH_DIR/jira | $GREP -v -e 'htm[l]*$' | sort | tail -1 | $GREP -o '/jira/secure/attachment/[0-9]*/[^"]*'`
    patchURL="http://issues.apache.org${relativePatchURL}"
    patchNum=`echo $patchURL | $GREP -o '[0-9]*/' | $GREP -o '[0-9]*'`
    echo "$defect patch is being downloaded at `date` from"
    echo "$patchURL"
    $WGET -q -O $PATCH_DIR/patch $patchURL
    VERSION=${SVN_REVISION}_${defect}_PATCH-${patchNum}
    JIRA_COMMENT="Here are the results of testing the latest attachment
  $patchURL
  against trunk revision ${SVN_REVISION}."

    ### Copy in any supporting files needed by this process
    cp -r $SUPPORT_DIR/lib/* ./lib
    #PENDING: cp -f $SUPPORT_DIR/etc/checkstyle* ./src/test
  ### Copy the patch file to $PATCH_DIR
  else
    VERSION=PATCH-${defect}
    cp $PATCH_FILE $PATCH_DIR/patch
    if [[ $? == 0 ]] ; then
      echo "Patch file $PATCH_FILE copied to $PATCH_DIR"
    else
      echo "Could not copy $PATCH_FILE to $PATCH_DIR"
      cleanupAndExit 0
    fi
  fi
  . $BASEDIR/dev-support/test-patch.properties
  ### exit if warnings are NOT defined in the properties file
  if [ -z "$OK_FINDBUGS_WARNINGS" ] || [[ -z "$OK_JAVADOC_WARNINGS" ]] || [[ -z $OK_RELEASEAUDIT_WARNINGS ]]; then
    echo "Please define the following properties in test-patch.properties file"
	 echo  "OK_FINDBUGS_WARNINGS"
	 echo  "OK_RELEASEAUDIT_WARNINGS"
	 echo  "OK_JAVADOC_WARNINGS"
    cleanupAndExit 1
  fi
  echo ""
  echo ""
  echo "======================================================================"
  echo "======================================================================"
  echo " Pre-build trunk to verify trunk stability and javac warnings"
  echo "======================================================================"
  echo "======================================================================"
  echo ""
  echo ""
  if [[ ! -d hadoop-common-project ]]; then
    cd $bindir/..
    echo "Compiling $(pwd)"
    echo "$MVN clean test -DskipTests > $PATCH_DIR/trunkCompile.txt 2>&1"
    $MVN clean test -DskipTests > $PATCH_DIR/trunkCompile.txt 2>&1
    if [[ $? != 0 ]] ; then
      echo "Top-level trunk compilation is broken?"
      cleanupAndExit 1
    fi
    cd -
  fi
  echo "Compiling $(pwd)"
  echo "$MVN clean test -DskipTests -D${PROJECT_NAME}PatchProcess -Ptest-patch > $PATCH_DIR/trunkJavacWarnings.txt 2>&1"
  $MVN clean test -DskipTests -D${PROJECT_NAME}PatchProcess -Ptest-patch > $PATCH_DIR/trunkJavacWarnings.txt 2>&1
  if [[ $? != 0 ]] ; then
    echo "Trunk compilation is broken?"
    cleanupAndExit 1
  fi
}

###############################################################################
### Check for @author tags in the patch
checkAuthor () {
  echo ""
  echo ""
  echo "======================================================================"
  echo "======================================================================"
  echo "    Checking there are no @author tags in the patch."
  echo "======================================================================"
  echo "======================================================================"
  echo ""
  echo ""
  authorTags=`$GREP -c -i '@author' $PATCH_DIR/patch`
  echo "There appear to be $authorTags @author tags in the patch."
  if [[ $authorTags != 0 ]] ; then
    JIRA_COMMENT="$JIRA_COMMENT

    -1 @author.  The patch appears to contain $authorTags @author tags which the Hadoop community has agreed to not allow in code contributions."
    return 1
  fi
  JIRA_COMMENT="$JIRA_COMMENT

    +1 @author.  The patch does not contain any @author tags."
  return 0
}

###############################################################################
### Check for tests in the patch
checkTests () {
  echo ""
  echo ""
  echo "======================================================================"
  echo "======================================================================"
  echo "    Checking there are new or changed tests in the patch."
  echo "======================================================================"
  echo "======================================================================"
  echo ""
  echo ""
  testReferences=`$GREP -c -i '/test' $PATCH_DIR/patch`
  echo "There appear to be $testReferences test files referenced in the patch."
  if [[ $testReferences == 0 ]] ; then
    if [[ $JENKINS == "true" ]] ; then
      patchIsDoc=`$GREP -c -i 'title="documentation' $PATCH_DIR/jira`
      if [[ $patchIsDoc != 0 ]] ; then
        echo "The patch appears to be a documentation patch that doesn't require tests."
        JIRA_COMMENT="$JIRA_COMMENT

    +0 tests included.  The patch appears to be a documentation patch that doesn't require tests."
        return 0
      fi
    fi
    JIRA_COMMENT="$JIRA_COMMENT

    -1 tests included.  The patch doesn't appear to include any new or modified tests.
                        Please justify why no new tests are needed for this patch.
                        Also please list what manual steps were performed to verify this patch."
    return 1
  fi
  JIRA_COMMENT="$JIRA_COMMENT

    +1 tests included.  The patch appears to include $testReferences new or modified tests."
  return 0
}

cleanUpXml () {
  cd $BASEDIR/conf
  for file in `ls *.xml.template`
    do
      rm -f `basename $file .template`
    done
  cd $BASEDIR
}

###############################################################################
### Attempt to apply the patch
applyPatch () {
  echo ""
  echo ""
  echo "======================================================================"
  echo "======================================================================"
  echo "    Applying patch."
  echo "======================================================================"
  echo "======================================================================"
  echo ""
  echo ""
  export PATCH
  $bindir/smart-apply-patch.sh $PATCH_DIR/patch
  if [[ $? != 0 ]] ; then
    echo "PATCH APPLICATION FAILED"
    JIRA_COMMENT="$JIRA_COMMENT

    -1 patch.  The patch command could not apply the patch."
    return 1
  fi
  return 0
}

###############################################################################
### Check there are no javadoc warnings
checkJavadocWarnings () {
  echo ""
  echo ""
  echo "======================================================================"
  echo "======================================================================"
  echo "    Determining number of patched javadoc warnings."
  echo "======================================================================"
  echo "======================================================================"
  echo ""
  echo ""
  echo "$MVN clean test javadoc:javadoc -DskipTests -Pdocs -D${PROJECT_NAME}PatchProcess > $PATCH_DIR/patchJavadocWarnings.txt 2>&1"
  if [ -d hadoop-project ]; then
    (cd hadoop-project; $MVN install)
  fi
  if [ -d hadoop-common-project/hadoop-annotations ]; then
    (cd hadoop-common-project/hadoop-annotations; $MVN install)
  fi
  $MVN clean test javadoc:javadoc -DskipTests -Pdocs -D${PROJECT_NAME}PatchProcess > $PATCH_DIR/patchJavadocWarnings.txt 2>&1
  javadocWarnings=`$GREP '\[WARNING\]' $PATCH_DIR/patchJavadocWarnings.txt | $AWK '/Javadoc Warnings/,EOF' | $GREP warning | $AWK 'BEGIN {total = 0} {total += 1} END {print total}'`
  echo ""
  echo ""
  echo "There appear to be $javadocWarnings javadoc warnings generated by the patched build."

  ### if current warnings greater than OK_JAVADOC_WARNINGS
  if [[ $javadocWarnings > $OK_JAVADOC_WARNINGS ]] ; then
    JIRA_COMMENT="$JIRA_COMMENT

    -1 javadoc.  The javadoc tool appears to have generated `expr $(($javadocWarnings-$OK_JAVADOC_WARNINGS))` warning messages."
    return 1
  fi
  JIRA_COMMENT="$JIRA_COMMENT

    +1 javadoc.  The javadoc tool did not generate any warning messages."
  return 0
}

###############################################################################
### Check there are no changes in the number of Javac warnings
checkJavacWarnings () {
  echo ""
  echo ""
  echo "======================================================================"
  echo "======================================================================"
  echo "    Determining number of patched javac warnings."
  echo "======================================================================"
  echo "======================================================================"
  echo ""
  echo ""
  echo "$MVN clean test -DskipTests -D${PROJECT_NAME}PatchProcess -Ptest-patch > $PATCH_DIR/patchJavacWarnings.txt 2>&1"
  $MVN clean test -DskipTests -D${PROJECT_NAME}PatchProcess -Ptest-patch > $PATCH_DIR/patchJavacWarnings.txt 2>&1
  if [[ $? != 0 ]] ; then
    JIRA_COMMENT="$JIRA_COMMENT

    -1 javac.  The patch appears to cause tar ant target to fail."
    return 1
  fi
  ### Compare trunk and patch javac warning numbers
  if [[ -f $PATCH_DIR/patchJavacWarnings.txt ]] ; then
    trunkJavacWarnings=`$GREP '\[WARNING\]' $PATCH_DIR/trunkJavacWarnings.txt | $AWK 'BEGIN {total = 0} {total += 1} END {print total}'`
    patchJavacWarnings=`$GREP '\[WARNING\]' $PATCH_DIR/patchJavacWarnings.txt | $AWK 'BEGIN {total = 0} {total += 1} END {print total}'`
    echo "There appear to be $trunkJavacWarnings javac compiler warnings before the patch and $patchJavacWarnings javac compiler warnings after applying the patch."
    if [[ $patchJavacWarnings != "" && $trunkJavacWarnings != "" ]] ; then
      if [[ $patchJavacWarnings -gt $trunkJavacWarnings ]] ; then
        JIRA_COMMENT="$JIRA_COMMENT

    -1 javac.  The applied patch generated $patchJavacWarnings javac compiler warnings (more than the trunk's current $trunkJavacWarnings warnings)."
        return 1
      fi
    fi
  fi
  JIRA_COMMENT="$JIRA_COMMENT

    +1 javac.  The applied patch does not increase the total number of javac compiler warnings."
  return 0
}

###############################################################################
### Check there are no changes in the number of release audit (RAT) warnings
checkReleaseAuditWarnings () {
  echo ""
  echo ""
  echo "======================================================================"
  echo "======================================================================"
  echo "    Determining number of patched release audit warnings."
  echo "======================================================================"
  echo "======================================================================"
  echo ""
  echo ""
  echo "$MVN apache-rat:check -D${PROJECT_NAME}PatchProcess 2>&1"
  $MVN apache-rat:check -D${PROJECT_NAME}PatchProcess 2>&1
  find $BASEDIR -name rat.txt | xargs cat > $PATCH_DIR/patchReleaseAuditWarnings.txt

  ### Compare trunk and patch release audit warning numbers
  if [[ -f $PATCH_DIR/patchReleaseAuditWarnings.txt ]] ; then
    patchReleaseAuditWarnings=`$GREP -c '\!?????' $PATCH_DIR/patchReleaseAuditWarnings.txt`
    echo ""
    echo ""
    echo "There appear to be $OK_RELEASEAUDIT_WARNINGS release audit warnings before the patch and $patchReleaseAuditWarnings release audit warnings after applying the patch."
    if [[ $patchReleaseAuditWarnings != "" && $OK_RELEASEAUDIT_WARNINGS != "" ]] ; then
      if [[ $patchReleaseAuditWarnings -gt $OK_RELEASEAUDIT_WARNINGS ]] ; then
        JIRA_COMMENT="$JIRA_COMMENT

    -1 release audit.  The applied patch generated $patchReleaseAuditWarnings release audit warnings (more than the trunk's current $OK_RELEASEAUDIT_WARNINGS warnings)."
        $GREP '\!?????' $PATCH_DIR/patchReleaseAuditWarnings.txt > $PATCH_DIR/patchReleaseAuditProblems.txt
        echo "Lines that start with ????? in the release audit report indicate files that do not have an Apache license header." >> $PATCH_DIR/patchReleaseAuditProblems.txt
        JIRA_COMMENT_FOOTER="Release audit warnings: $BUILD_URL/artifact/trunk/patchprocess/patchReleaseAuditProblems.txt
$JIRA_COMMENT_FOOTER"
        return 1
      fi
    fi
  fi
  JIRA_COMMENT="$JIRA_COMMENT

    +1 release audit.  The applied patch does not increase the total number of release audit warnings."
  return 0
}

###############################################################################
### Check there are no changes in the number of Checkstyle warnings
checkStyle () {
  echo ""
  echo ""
  echo "======================================================================"
  echo "======================================================================"
  echo "    Determining number of patched checkstyle warnings."
  echo "======================================================================"
  echo "======================================================================"
  echo ""
  echo ""
  echo "THIS IS NOT IMPLEMENTED YET"
  echo ""
  echo ""
  echo "$MVN test checkstyle:checkstyle -DskipTests -D${PROJECT_NAME}PatchProcess"
  $MVN test checkstyle:checkstyle -DskipTests -D${PROJECT_NAME}PatchProcess

  JIRA_COMMENT_FOOTER="Checkstyle results: $BUILD_URL/artifact/trunk/build/test/checkstyle-errors.html
$JIRA_COMMENT_FOOTER"
  ### TODO: calculate actual patchStyleErrors
#  patchStyleErrors=0
#  if [[ $patchStyleErrors != 0 ]] ; then
#    JIRA_COMMENT="$JIRA_COMMENT
#
#    -1 checkstyle.  The patch generated $patchStyleErrors code style errors."
#    return 1
#  fi
#  JIRA_COMMENT="$JIRA_COMMENT
#
#    +1 checkstyle.  The patch generated 0 code style errors."
  return 0
}

###############################################################################
### Check there are no changes in the number of Findbugs warnings
checkFindbugsWarnings () {
  findbugs_version=`${FINDBUGS_HOME}/bin/findbugs -version`
  echo ""
  echo ""
  echo "======================================================================"
  echo "======================================================================"
  echo "    Determining number of patched Findbugs warnings."
  echo "======================================================================"
  echo "======================================================================"
  echo ""
  echo ""
  echo "$MVN clean test findbugs:findbugs -DskipTests -D${PROJECT_NAME}PatchProcess"
  $MVN clean test findbugs:findbugs -DskipTests -D${PROJECT_NAME}PatchProcess < /dev/null

  if [ $? != 0 ] ; then
    JIRA_COMMENT="$JIRA_COMMENT

    -1 findbugs.  The patch appears to cause Findbugs (version ${findbugs_version}) to fail."
    return 1
  fi

  findbugsWarnings=0
  for file in $(find $BASEDIR -name findbugsXml.xml)
  do
    relative_file=${file#$BASEDIR/} # strip leading $BASEDIR prefix
    if [ ! $relative_file == "target/findbugsXml.xml" ]; then
      module_suffix=${relative_file%/target/findbugsXml.xml} # strip trailing path
      module_suffix=`basename ${module_suffix}`
    fi

    cp $file $PATCH_DIR/patchFindbugsWarnings${module_suffix}.xml
    $FINDBUGS_HOME/bin/setBugDatabaseInfo -timestamp "01/01/2000" \
      $PATCH_DIR/patchFindbugsWarnings${module_suffix}.xml \
      $PATCH_DIR/patchFindbugsWarnings${module_suffix}.xml
    newFindbugsWarnings=`$FINDBUGS_HOME/bin/filterBugs -first "01/01/2000" $PATCH_DIR/patchFindbugsWarnings${module_suffix}.xml \
      $PATCH_DIR/newPatchFindbugsWarnings${module_suffix}.xml | $AWK '{print $1}'`
    echo "Found $newFindbugsWarnings Findbugs warnings ($file)"
    findbugsWarnings=$((findbugsWarnings+newFindbugsWarnings))
    $FINDBUGS_HOME/bin/convertXmlToText -html \
      $PATCH_DIR/newPatchFindbugsWarnings${module_suffix}.xml \
      $PATCH_DIR/newPatchFindbugsWarnings${module_suffix}.html
    JIRA_COMMENT_FOOTER="Findbugs warnings: $BUILD_URL/artifact/trunk/patchprocess/newPatchFindbugsWarnings${module_suffix}.html
$JIRA_COMMENT_FOOTER"
  done

  ### if current warnings greater than OK_FINDBUGS_WARNINGS
  if [[ $findbugsWarnings > $OK_FINDBUGS_WARNINGS ]] ; then
    JIRA_COMMENT="$JIRA_COMMENT

    -1 findbugs.  The patch appears to introduce `expr $(($findbugsWarnings-$OK_FINDBUGS_WARNINGS))` new Findbugs (version ${findbugs_version}) warnings."
    return 1
  fi
  JIRA_COMMENT="$JIRA_COMMENT

    +1 findbugs.  The patch does not introduce any new Findbugs (version ${findbugs_version}) warnings."
  return 0
}

###############################################################################
### Run the tests
runTests () {
  echo ""
  echo ""
  echo "======================================================================"
  echo "======================================================================"
  echo "    Running tests."
  echo "======================================================================"
  echo "======================================================================"
  echo ""
  echo ""

  echo "$MVN clean install -Pnative -D${PROJECT_NAME}PatchProcess"
  $MVN clean install -Pnative -D${PROJECT_NAME}PatchProcess
  if [[ $? != 0 ]] ; then
    ### Find and format names of failed tests
    failed_tests=`find . -name 'TEST*.xml' | xargs $GREP  -l -E "<failure|<error" | sed -e "s|.*target/surefire-reports/TEST-|                  |g" | sed -e "s|\.xml||g"`

    if [[ -n "$failed_tests" ]] ; then
      JIRA_COMMENT="$JIRA_COMMENT

    -1 core tests.  The patch failed these unit tests:
$failed_tests"
    else
      JIRA_COMMENT="$JIRA_COMMENT

    -1 core tests.  The patch failed the unit tests build"
    fi
    return 1
  fi
  JIRA_COMMENT="$JIRA_COMMENT

    +1 core tests.  The patch passed unit tests in $modules."
  return 0
}

###############################################################################
### Run the test-contrib target
runContribTests () {
  echo ""
  echo ""
  echo "======================================================================"
  echo "======================================================================"
  echo "    Running contrib tests."
  echo "======================================================================"
  echo "======================================================================"
  echo ""
  echo ""

  if [[ `$GREP -c 'test-contrib' build.xml` == 0 ]] ; then
    echo "No contrib tests in this project."
    return 0
  fi

  ### Kill any rogue build processes from the last attempt
  $PS auxwww | $GREP ${PROJECT_NAME}PatchProcess | $AWK '{print $2}' | /usr/bin/xargs -t -I {} /bin/kill -9 {} > /dev/null

  #echo "$ANT_HOME/bin/ant -Dversion="${VERSION}" $ECLIPSE_PROPERTY -DHadoopPatchProcess= -Dtest.junit.output.format=xml -Dtest.output=no test-contrib"
  #$ANT_HOME/bin/ant -Dversion="${VERSION}" $ECLIPSE_PROPERTY -DHadoopPatchProcess= -Dtest.junit.output.format=xml -Dtest.output=no test-contrib
  echo "NOP"
  if [[ $? != 0 ]] ; then
    JIRA_COMMENT="$JIRA_COMMENT

    -1 contrib tests.  The patch failed contrib unit tests."
    return 1
  fi
  JIRA_COMMENT="$JIRA_COMMENT

    +1 contrib tests.  The patch passed contrib unit tests."
  return 0
}

###############################################################################
### Run the inject-system-faults target
checkInjectSystemFaults () {
  echo ""
  echo ""
  echo "======================================================================"
  echo "======================================================================"
  echo "    Checking the integrity of system test framework code."
  echo "======================================================================"
  echo "======================================================================"
  echo ""
  echo ""

  ### Kill any rogue build processes from the last attempt
  $PS auxwww | $GREP ${PROJECT_NAME}PatchProcess | $AWK '{print $2}' | /usr/bin/xargs -t -I {} /bin/kill -9 {} > /dev/null

  #echo "$ANT_HOME/bin/ant -Dversion="${VERSION}" -DHadoopPatchProcess= -Dtest.junit.output.format=xml -Dtest.output=no -Dcompile.c++=yes -Dforrest.home=$FORREST_HOME inject-system-faults"
  #$ANT_HOME/bin/ant -Dversion="${VERSION}" -DHadoopPatchProcess= -Dtest.junit.output.format=xml -Dtest.output=no -Dcompile.c++=yes -Dforrest.home=$FORREST_HOME inject-system-faults
  echo "NOP"
  return 0
  if [[ $? != 0 ]] ; then
    JIRA_COMMENT="$JIRA_COMMENT

    -1 system test framework.  The patch failed system test framework compile."
    return 1
  fi
  JIRA_COMMENT="$JIRA_COMMENT

    +1 system test framework.  The patch passed system test framework compile."
  return 0
}

###############################################################################
### Submit a comment to the defect's Jira
submitJiraComment () {
  local result=$1
  ### Do not output the value of JIRA_COMMENT_FOOTER when run by a developer
  if [[  $JENKINS == "false" ]] ; then
    JIRA_COMMENT_FOOTER=""
  fi
  if [[ $result == 0 ]] ; then
    comment="+1 overall.  $JIRA_COMMENT

$JIRA_COMMENT_FOOTER"
  else
    comment="-1 overall.  $JIRA_COMMENT

$JIRA_COMMENT_FOOTER"
  fi
  ### Output the test result to the console
  echo "

$comment"

  if [[ $JENKINS == "true" ]] ; then
    echo ""
    echo ""
    echo "======================================================================"
    echo "======================================================================"
    echo "    Adding comment to Jira."
    echo "======================================================================"
    echo "======================================================================"
    echo ""
    echo ""
    ### Update Jira with a comment
    export USER=hudson
    $JIRACLI -s https://issues.apache.org/jira -a addcomment -u hadoopqa -p $JIRA_PASSWD --comment "$comment" --issue $defect
    $JIRACLI -s https://issues.apache.org/jira -a logout -u hadoopqa -p $JIRA_PASSWD
  fi
}

###############################################################################
### Cleanup files
cleanupAndExit () {
  local result=$1
  if [[ $JENKINS == "true" ]] ; then
    if [ -e "$PATCH_DIR" ] ; then
      mv $PATCH_DIR $BASEDIR
    fi
  fi
  echo ""
  echo ""
  echo "======================================================================"
  echo "======================================================================"
  echo "    Finished build."
  echo "======================================================================"
  echo "======================================================================"
  echo ""
  echo ""
  exit $result
}

###############################################################################
###############################################################################
###############################################################################

JIRA_COMMENT=""
JIRA_COMMENT_FOOTER="Console output: $BUILD_URL/console

This message is automatically generated."

### Check if arguments to the script have been specified properly or not
parseArgs $@
cd $BASEDIR

checkout
RESULT=$?
if [[ $JENKINS == "true" ]] ; then
  if [[ $RESULT != 0 ]] ; then
    exit 100
  fi
fi
setup
checkAuthor
RESULT=$?

if [[ $JENKINS == "true" ]] ; then
  cleanUpXml
fi
checkTests
(( RESULT = RESULT + $? ))
applyPatch
if [[ $? != 0 ]] ; then
  submitJiraComment 1
  cleanupAndExit 1
fi
checkJavadocWarnings
(( RESULT = RESULT + $? ))
checkJavacWarnings
(( RESULT = RESULT + $? ))
### Checkstyle not implemented yet
#checkStyle
#(( RESULT = RESULT + $? ))
checkFindbugsWarnings
(( RESULT = RESULT + $? ))
checkReleaseAuditWarnings
(( RESULT = RESULT + $? ))
### Run tests for Jenkins or if explictly asked for by a developer
if [[ $JENKINS == "true" || $RUN_TESTS == "true" ]] ; then
  runTests
  (( RESULT = RESULT + $? ))
  runContribTests
  (( RESULT = RESULT + $? ))
fi
checkInjectSystemFaults
(( RESULT = RESULT + $? ))
JIRA_COMMENT_FOOTER="Test results: $BUILD_URL/testReport/
$JIRA_COMMENT_FOOTER"

submitJiraComment $RESULT
cleanupAndExit $RESULT


Add screenshot.sh script for screenshot keybinding

##################################################################################
#
#    New Start: A modern Arch workflow built with an emphasis on functionality.
#    Copyright (C) 2017 Donovan Glover
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU General Public License as published by
#    the Free Software Foundation, either version 3 of the License, or
#    (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU General Public License for more details.
#
#    You should have received a copy of the GNU General Public License
#    along with this program.  If not, see <https://www.gnu.org/licenses/>.
#
##################################################################################

DATETIME=`date +%F_%H%M%S`
shotgun ~/screenshots/$DATETIME.png
feh ~/screenshots/$DATETIME.png

error: pathspec 'dotfiles/.config/i3/screenshot.sh' did not match any file(s) known to git


feat: add script to create and configure "/var/www" directory

#!/bin/bash

set -e

WWW_PATH="/var/www"

# creating directory
sudo mkdir -p $WWW_PATH

# configuring permissions
sudo chown $USER:www-data $WWW_PATH -Rf

exit 0

error: pathspec 'configure-www-directory.sh' did not match any file(s) known to git


Add cdfzf.zsh shell function

Add cdfzf.zsh shell function

I often make aliases to cd into specific directories that I use often,
but creating and updating many under a certain root directory, like my
~/Projects directory, can be tedious.  By aliasing to cdfzf instead, I
can get fzf's fuzzy matching to change to the target directory instead.
Obviously this requires fzf to be installed to work
(https://github.com/junegunn/fzf).

# cdfzf.zsh
#
# Function for creating aliases to cd into a child directory of a root path
# using fzf fuzzy matching.  https://github.com/junegunn/fzf
#
# Example:
# cdfzf ~/Documents 2

function cdfzf() {
  ROOT_DIR=$1
  MAX_DEPTH=$2
  cd $ROOT_DIR/$((find $ROOT_DIR -type d -maxdepth $MAX_DEPTH ; echo) | fzf --no-multi | sed "s|^$ROOT_DIR/||g") ; pwd
}

error: pathspec 'zsh/bin/cdfzf.zsh' did not match any file(s) known to git


updated aws crap

updated aws crap

remote: Support for password authentication was removed on August 13, 2021.
remote: Please see https://docs.github.com/en/get-started/getting-started-with-git/about-remote-repositories#cloning-with-https-urls for information on currently recommended modes of authentication.
fatal: Authentication failed for 'https://github.com/jkais/dotfiles.git/'


Bug fix.

Bug fix.

Didn't even have to rely on seq(1). I rule.

remote: Support for password authentication was removed on August 13, 2021.
remote: Please see https://docs.github.com/en/get-started/getting-started-with-git/about-remote-repositories#cloning-with-https-urls for information on currently recommended modes of authentication.
fatal: Authentication failed for 'https://github.com/GA114/pomodoro.git/'


upload

upload

remote: Support for password authentication was removed on August 13, 2021.
remote: Please see https://docs.github.com/en/get-started/getting-started-with-git/about-remote-repositories#cloning-with-https-urls for information on currently recommended modes of authentication.
fatal: Authentication failed for 'https://github.com/relic7/prodimages.git/'


Update script

Update script

git-svn-id: 86a4f41f9d0383640198789122b520d4105b319a@12399 dc9f47b5-9b13-0410-9fdd-eb0c1a62fdaf

#!/bin/bash

#
# This script assumes a local clone of the openlayers github repository. This
# clone needs to be configured with an svn remote to the openlayers svn. Here
# are the actual steps to get the clone and configure it properly:
#
# $ git clone git@github.com:openlayers/openlayers.git
# $ git svn init -T trunk/openlayers -t tags/openlayers -b branches/openlayers http://svn.openlayers.org/
#
# To run this script change to the local clone first:
#
# $ cd openlayers
# $ git_to_svn.sh

git pull origin master
git svn dcommit

#!/bin/bash

#
# This script assumes a local clone of the openlayers github repository. This
# clone needs to be configured with an svn remote to the openlayers svn. Here
# are the actual steps to get the clone and configure it properly:
#
# $ git clone git@github.com:openlayers/openlayers.git
# $ git svn init -T trunk/openlayers -t tags/openlayers -b branches/openlayers http://svn.openlayers.org/
#
# To run this script change to the local clone first:
#
# $ cd openlayers
# $ git_to_svn.sh
#

git pull origin master
git svn dcommit


Update e2e.sh (#556)

Update e2e.sh (#556)

* Update e2e.sh

* Update e2e.sh

Former-commit-id: 7a749147f57f61ea010e9c1ed975ed94e17c16c8

remote: Support for password authentication was removed on August 13, 2021.
remote: Please see https://docs.github.com/en/get-started/getting-started-with-git/about-remote-repositories#cloning-with-https-urls for information on currently recommended modes of authentication.
fatal: Authentication failed for 'https://github.com/geeknoid/istio.git/'


add knife script to get kafka addrs based on chef role/env

#!/bin/bash
# uses knife and formats kafka addresses into flag for kcli

if [ "$#" -ne 3 ]; then
  >&2 echo "usage: chef-role chef-environment port"
  exit 1
fi

role=${1}
env=${2}
port=${3}

function ksn() {
	knife search node "roles:${1} AND chef_environment:${2}" -i
}

kafkas_w_commas=$(tr '\n' ',' < <(ksn ${role} ${env} | grep '.sendgrid.net' | grep 'kafka'))
kafkas_wo_last_comma=${kafkas_w_commas%?}

if [ ${#kafkas_wo_last_comma} -eq 0 ]; then
  >&2 echo "no kafkas found with chef-role: ${role}, chef-environment: ${env}"
  exit 1
fi

echo ${kafkas_wo_last_comma} | sed -e "s/.sendgrid.net/.sendgrid.net:${port}/g"

error: pathspec 'get-kafkas.sh' did not match any file(s) known to git


Rename aliases; add aliases.

Rename aliases; add aliases.

# Copyright 2014 Joseph Block <jpb@unixorn.net>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at

#     http://www.apache.org/licenses/LICENSE-2.0

# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

if (which docker > /dev/null)
then
  # Add our plugin's bin diretory to user's path
  PLUGIN_D="$(dirname $0)"
  export PATH=${PATH}:${PLUGIN_D}/bin

  alias d='docker'
  # Most of these aliases are from http://www.kartar.net/2014/03/some-useful-docker-bash-functions-and-aliases/
  alias dkd='docker run -d -P'
  alias dki='docker run -t -i -P'
  alias dkl='docker ps -l -q'
  alias docker-daemonize='docker run -d -P'
  alias docker-interactive='docker run -t -i -P'
  alias docker-last='docker ps -l -q'
  alias dockerbuild='docker build'
  alias dockerimages='docker images'
  alias dockerps='docker ps'

  d-ip() {
    docker inspect --format "{{ .NetworkSettings.IPAddress }}" $1
  }

  dkb() {
    docker build -t="$1" .
  }

  d-pid() {
    docker inspect --format "{{ .State.Pid }}" $1
  }

  d-purge() {
    docker rm $(docker ps -a -q);
  }

  d-stats() {
    docker ps -q | xargs docker stats
  }

  d-stop() {
    # Stop all running containers
    docker stop $(docker ps -a -q);
  }

  d-zsh() {
    local TAG=$1
    docker run -v /tmp:/host_tmp:rw -i -t $TAG /bin/zsh
  }

  dps-monitor() {
     while true
     do
         clear
         docker ps | cut -c -$(tput cols)
         sleep 0.5
     done
  }

  # helpers for starting a container with access to the current directory

  d-busybox() {
    docker run -v $(pwd):/shared --rm -it busybox:latest /bin/sh
  }

  d-centos() {
    docker run -v $(pwd):/shared --rm -it centos:centos7 /bin/bash
  }

  d-centos6() {
    docker run -v $(pwd):/shared --rm -it centos:centos6 /bin/bash
  }

  d-fedora() {
    docker run -v $(pwd):/shared --rm -it fedora:20 /bin/bash
  }

  d-ubuntu() {
    docker run -v $(pwd):/shared --rm -it ubuntu:14.04 /bin/bash
  }
fi

if (which docker-compose > /dev/null)
then
  alias d-cp=docker-compose
fi

# Copyright 2014 Joseph Block <jpb@unixorn.net>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at

#     http://www.apache.org/licenses/LICENSE-2.0

# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

if (which docker > /dev/null)
then
  # Add our plugin's bin diretory to user's path
  PLUGIN_D="$(dirname $0)"
  export PATH=${PATH}:${PLUGIN_D}/bin

  # Most of these aliases are from http://www.kartar.net/2014/03/some-useful-docker-bash-functions-and-aliases/
  alias dkd="docker run -d -P"
  alias dki="docker run -t -i -P"
  alias dkl='docker ps -l -q'
  alias docker-daemonize="docker run -d -P"
  alias docker-interactive="docker run -t -i -P"
  alias docker-last='docker ps -l -q'
  alias dockerbuild='docker build'
  alias dockerimages='docker images'
  alias dockerps='docker ps'

  dkb() {
    docker build -t="$1" .
  }

  d-ip() {
    docker inspect --format "{{ .NetworkSettings.IPAddress }}" $1
  }

  d-pid() {
    docker inspect --format "{{ .State.Pid }}" $1
  }

  d-zsh() {
    local TAG=$1
    docker run -v /tmp:/host_tmp:rw -i -t $TAG /bin/zsh
  }

  dps-monitor() {
     while true
     do
         clear
         docker ps | cut -c -$(tput cols)
         sleep 0.5
     done
  }

  # helpers for starting a container with access to the current directory

  d-busybox() {
    docker run -v $(pwd):/shared --rm -it busybox:latest /bin/sh
  }

  d-centos() {
    docker run -v $(pwd):/shared --rm -it centos:centos7 /bin/bash
  }

  d-centos6() {
    docker run -v $(pwd):/shared --rm -it centos:centos6 /bin/bash
  }

  d-fedora() {
    docker run -v $(pwd):/shared --rm -it fedora:20 /bin/bash
  }

  d-ubuntu() {
    docker run -v $(pwd):/shared --rm -it ubuntu:14.04 /bin/bash
  }
fi


Create install.sh

Create install.sh

#!/bin/bash

sudo add-apt-repository ppa:gophers/archive
sudo apt-get update
sudo apt-get install golang-1.9-go

error: pathspec 'environments/go-pkgmgr/install.sh' did not match any file(s) known to git


Create Main.sh

Create Main.sh

https://paiza.jp/poh/hatsukoi/share/a090b0ad

#!/bin/bash
# Try POH
# author: Leonardone @ NEETSDKASU
# -------------------------------
typeset -i n m #
read n m #
if (( m % n )); then #
    echo 'ng' #
else #
    echo 'ok' #
fi #

error: pathspec 'POH8/CuteIshou/Main.sh' did not match any file(s) known to git


fixes for ScotchBox 2.0 -> 2.5

fixes for ScotchBox 2.0 -> 2.5

- remove Node 4.x repo (ScotchBox 2.5 comes with Node 5)
- change Node bin directory from `usr/bin` to `usr/local/bin`

remote: Support for password authentication was removed on August 13, 2021.
remote: Please see https://docs.github.com/en/get-started/getting-started-with-git/about-remote-repositories#cloning-with-https-urls for information on currently recommended modes of authentication.
fatal: Authentication failed for 'https://github.com/TehSeph/Neko.js.git/'


Create miRanalysis.sh

Create miRanalysis.sh

#!/bin/bash

# Add the modules needed for the analysis
module add bowtie/0.12.7
module add mirdeep2/2.0.0.5
module add SAMtools/1.1
module add fastx_tools/0.0.13

# Variables
FILENAME=$1
myDIR=$2
myREF="/RQexec/johnsonr/Mus_musculus/UCSC/mm10/Sequence/BowtieIndex"
myMIR="/RQusagers/johnsonr/miRNA_reference"
count=0

# Build the bowtie index for hairpin.fa
#cd  "${myMIR}"/
#bowtie-build "${myMIR}"/hairpindna.fa  hairpindna_bowtie
#bowtie-build "${myMIR}"/hairpindna.fa  hairpindna
#bowtie-build "${myMIR}"/mature.fa  mature

cd /RQusagers/johnsonr/PARK_LAB/scripts
# For mouse mm10 and Dr. Duchaines precursor miRNA files
while read mySIZE
        do
                let count++
                echo "$count $mySIZE"

                # Go to the sample directory in the DUCHAINE_LAB folder
                cd  "${myDIR}"/"${mySIZE}"/

                # Align the reads with bowtie
                bowtie -n 1 -l 8 -a --best --strata --phred33-quals -p 4 --un "${mySIZE}"_unmapped.fq "${myMIR}"/hairpindna_bowtie "${mySIZE}".fq "${mySIZE}"_bowtie.sam

                # Run quantifier script from mirDeep2
                # Convert the sam file to a fasta file
                cat "${mySIZE}"_bowtie.sam | grep -v ^@  | awk '{print ">"$1" "$2"\n"$6}' > "${mySIZE}".mapped.fa

                # Run the mapper on the mapped fasta file
                mapper.pl "${mySIZE}".mapped.fa -p /RQusagers/johnsonr/miRNA_reference/hairpindna -c -m -s "${mySIZE}".mapped.fa.col -t "${mySIZE}".mapped.arf -n

                # Run quantification with most recent quantifier.pl script
                perl ~/mirdeep2_core-master/quantifier.pl -p "${myMIR}"/hairpindna.fa -m "${myMIR}"/mature.fa -r "${mySIZE}".mapped.fa.col -t mmu -y "${mySIZE}"v2

done < $FILENAME

error: pathspec 'miRs/miRanalysis.sh' did not match any file(s) known to git


modify .gitignore add add a shell

modify .gitignore add add a shell

#!/bin/bash

# your account file
ACCOUNT_FILE="/Users/jerry/Documents/Blog/account"

USERNAME=`sed -n 1p $ACCOUNT_FILE`
PASSWORD=`sed -n 2p $ACCOUNT_FILE`
TOKEN=`sed -n 3p $ACCOUNT_FILE`

echo "Fetching visit data from Baidu Tongji"
python baidu_tongji.py $USERNAME $PASSWORD $TOKEN

echo "Generate visit map"
phantomjs capture.js

echo "Replace home-bg.png"
mv home-bg.png ../img/

error: pathspec 'home-map/generator.sh' did not match any file(s) known to git


A simple Concourse task script to a deploy BOSH director

Used for my blog post; this should be tested.

#!/bin/bash

# THIS IS A SIMPLIFIED SCRIPT FOR DEPLOYING A BOSH DIRECTOR
# It does not customize the passwords nor the SSL certificates

# We abort the script as soon as we hit an error (as soon as a command exits
# with a non-zero exit status)
set -e

# CUNNIE_DEPLOYMENTS_DIR is the root directory of the checked-out repo of
# `cunnie-deployments`; it contains our BOSH manifests and our directors'
# `-state.json` files; it also contains this script (task script) and task
# definition.
CUNNIE_DEPLOYMENTS_DIR="$( cd "${BASH_SOURCE[0]%/*}" && pwd )/.."
pushd $CUNNIE_DEPLOYMENTS_DIR

# We attempt to deploy our BOSH director. We prepare a git commit message
# regardless whether our attempt succeeds or fails because we need to retain any
# change to the BOSH director's `-state.json` file. This is necessary in cases
# where a deploy proceeds far enough to create a broken director VM, for
# subsequent deploys must be able to destroy the broken director VM in order to
# free up its IP address so that the current deploy will succeed. The crucial
# information needed to destroy the  broken director VM is its VM's ID, which is
# recorded in the `-state.json` file.

# Note that `set -e` does not trigger an abort if the command that returns a
# non-zero exit code is the subject of an `if` block, i.e. `if bosh create-env`;
# this gives us the breathing room to commit our results regardless of whether
# `bosh create-env` succeeded or failed
if bosh create-env $CUNNIE_DEPLOYMENTS_DIR/../bosh-deployment/bosh.yml \
  -o $CUNNIE_DEPLOYMENTS_DIR/../bosh-deployment/misc/powerdns.yml \
  -o $CUNNIE_DEPLOYMENTS_DIR/../bosh-deployment/gcp/cpi.yml \
  -o $CUNNIE_DEPLOYMENTS_DIR/../bosh-deployment/external-ip-not-recommended.yml \
  -o $CUNNIE_DEPLOYMENTS_DIR/../bosh-deployment/jumpbox-user.yml \
  -l <(echo "$DEPLOYMENTS_YML") \
  --vars-store=simple-creds.yml \
  -v dns_recursor_ip="169.254.169.254" \
  -v internal_gw="10.128.0.1" \
  -v internal_cidr="10.128.0.0/20" \
  -v internal_ip="10.128.0.20" \
  -v external_ip="35.202.16.114" \
  -v network="cf" \
  -v subnetwork="cf-e6ecf3fd8a498fbe" \
  -v tags="[ cf-internal, cf-bosh, cf-bosh-cli ]" \
  -v zone="us-central1-b" \
  -v project_id="blabbertabber" \
  -v director_name="gce"
then
  GIT_COMMIT_MESSAGE="CI PASS: $IAAS BOSH deploy :airplane:"
  DEPLOY_EXIT_STATUS=0
else
  GIT_COMMIT_MESSAGE="CI FAIL: $IAAS BOSH deploy :airplane:"
  DEPLOY_EXIT_STATUS=1
fi

# Do we need to commit anything? If a new director hasn't been deployed (most
# often because there's been no change to the manifest, releases, or stemcell),
# then we don't need to commit
if ! git diff --quiet HEAD --; then
  # If we're in this block, then there has been a deployment. Let's set our
  # git author to avoid git's `*** Please tell me who you are.` error.
  git config --global user.name "Concourse CI"
  git config --global user.email brian.cunnie@gmail.com

  # We check out our branch's HEAD because Concourse's git-resource leaves us
  # in `detached HEAD` state. ${DEPLOYMENTS_BRANCH} is typically set to
  # `master`, but may be set to something else (usually while testing).
  git checkout $DEPLOYMENTS_BRANCH
  git add .
  git commit -m"$GIT_COMMIT_MESSAGE"
fi
popd

# We copy our repo with its new commit to a new directory. The Concourse job,
# after it finishes running this task, will push the new commit to GitHub.
# Note that `cp -R` works as well as `rsync`; we use `rsync` by force of
# habit.
rsync -aH cunnie-deployments/ cunnie-deployments-with-state/

# We exit with the return code of `bosh create-env`; if the deploy failed, then
# this Concourse task failed
exit $DEPLOY_EXIT_STATUS

error: pathspec 'ci/tasks/bosh-simple.sh' did not match any file(s) known to git


Added multi-site-tasks

Added multi-site-tasks

Multi site tasks are a series of commands that will run against every
site in the docroot based on a relative path.

#!/usr/bin/env bash
#
# Run mutliple commands within the multisite environments.
#
# Usage:
#   ./multiple-site-tasks.sh
#
# Author: Jacob Bednarz <jacob_bednarz@flightcentre.com>

# Get all Drupal sites
sites=`find ../docroot/sites -maxdepth 1 -type d -print | grep -v '/all$' | grep -v '/default$' | grep -v '\.$'`

echo "Choose the commande to execute : "
echo "[ 1 ] Run DB updates"
echo "[ 2 ] Put sites offline"
echo "[ 3 ] Put sites online"
echo "[ 4 ] Clear all caches"
echo "[ 5 ] Clear CSS and JavaScript caches"
echo "[ 6 ] Clear another specific cache"
echo "[ 7 ] Install specific module"
echo "[ 8 ] Disable specific module"
echo "[ 9 ] Find the status of a module on a site"
read choice

if [ $choice -gt 6 ] ; then
  echo -n "Extension (module/theme) name? "
  read ext
fi

# For each site, execute the command
for site in $sites
do
  echo ----------
  echo $site
  cd $site
  if [ $choice -eq 1 ] ; then
    drush updatedb
  elif [ $choice -eq 2 ] ; then
    drush vset --always-set maintenance_mode 1
  elif [ $choice -eq 3 ] ; then
    drush vset --always-set maintenance_mode 0
  elif [ $choice -eq 4 ] ; then
    drush cc all
  elif [ $choice -eq 5 ] ; then
    drush cc css+js
  elif [ $choice -eq 6 ] ; then
    drush cc
  elif [ $choice -eq 7 ] ; then
    drush pm-enable -y $ext
  elif [ $choice -eq 8 ] ; then
    drush pm-disable -y $ext
  elif [ $choice -eq 9 ] ; then
    drush pm-list | grep "$ext"
  fi
  cd ../
done

error: pathspec 'multi-site-tasks.sh' did not match any file(s) known to git


The quick Homebrew upgrade command

The quick Homebrew upgrade command

function upgrade-brews {
  brew update
  brew upgrade
  brew cleanup -s
}

error: pathspec 'env/brew-quick-upgrade.sh' did not match any file(s) known to git


Adding test driver script.

Adding test driver script.

remote: Support for password authentication was removed on August 13, 2021.
remote: Please see https://docs.github.com/en/get-started/getting-started-with-git/about-remote-repositories#cloning-with-https-urls for information on currently recommended modes of authentication.
fatal: Authentication failed for 'https://github.com/iorbitearth/strcat.git/'


Added shell scripts for running tests.

Added shell scripts for running tests.

#!/bin/sh
echo
echo "Transaction Processor Tests"
echo "==========================="
python -m tests.test_TransactionProcessor
echo
echo "Parse Tests"
echo "==========="
python -m tests.test_Parse

error: pathspec 'runtests.sh' did not match any file(s) known to git


Add a script that runs everything

Add a script that runs everything

#! /bin/bash

for ex in main*; do
    echo -e "\nExecuting $ex\n"
    ./$ex
done

command -v dot >/dev/null 2>&1 || { echo >&2 "I require dot but it's not installed.  Aborting."; exit 1; }

for df in *.dot; do
    echo -e "\nInterpreting $df"
    dot $df -Tsvg -O
    echo -e "Successfully outputted $df.svg\n"
done