[build-and-test] add short-circuiting for RUN_FLAKY_ONLY

Added a short circuit into build-and-test.sh: in case of
RUN_FLAKY_ONLY=1 and no flaky tests are reported, don't build
anything and report success right away.

Change-Id: I69278c08c496749cd5f30e2d042b9291efde202c
Reviewed-on: http://gerrit.cloudera.org:8080/12154
Tested-by: Kudu Jenkins
Reviewed-by: Adar Dembo <866ca730bfc7f544e3ebce604ceeba4879f23df0@cloudera.com>

#!/bin/bash
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.
#
# This script is invoked from the Jenkins builds to build Kudu
# and run all the unit tests.
#
# Environment variables may be used to customize operation:
#   BUILD_TYPE: Default: DEBUG
#     Maybe be one of ASAN|TSAN|DEBUG|RELEASE|COVERAGE|LINT|IWYU
#
#   KUDU_ALLOW_SLOW_TESTS   Default: 1
#     Runs the "slow" version of the unit tests. Set to 0 to
#     run the tests more quickly.
#
#   TEST_TMPDIR   Default: /tmp/kudutest-$UID
#     Specifies the temporary directory where tests should write their
#     data. It is expected that following the completion of all tests, this
#     directory is empty (i.e. every test cleaned up after itself).
#
#   RUN_FLAKY_ONLY    Default: 0
#     Only runs tests which have failed recently, if this is 1.
#     Used by the kudu-flaky-tests jenkins build.
#
#   KUDU_FLAKY_TEST_ATTEMPTS  Default: 1
#     If more than 1, will fetch the list of known flaky tests
#     from the kudu-test jenkins job, and allow those tests to
#     be flaky in this build.
#
#   TEST_RESULT_SERVER  Default: none
#     The host:port pair of a server running test_result_server.py.
#     This must be configured for flaky test resistance or test result
#     reporting to work.
#
#   ENABLE_DIST_TEST  Default: 0
#     If set to 1, will submit C++ and Java tests to be run by the
#     distributed test runner instead of running them locally.
#     This requires that $DIST_TEST_HOME be set to a working dist_test
#     checkout (and that dist_test itself be appropriately configured to
#     point to a cluster)
#
#   BUILD_JAVA        Default: 1
#     Build and test java code if this is set to 1.
#
#   BUILD_PYTHON       Default: 1
#     Build and test the Python wrapper of the client API.
#
#   BUILD_PYTHON3      Default: 1
#     Build and test the Python wrapper of the client API in Python. This
#     option is not mutually exclusive from BUILD_PYTHON. If both options
#     are set (default), then both will be run.
#
#   PIP_INSTALL_FLAGS  Default: ""
#     Extra flags which are passed to 'pip install' when setting up the build
#     environment for the Python wrapper.
#
#   GRADLE_FLAGS       Default: ""
#     Extra flags which are passed to 'gradle' when running Gradle commands.
#
#   ERROR_ON_TEST_FAILURE    Default: 1
#     Whether test failures will cause this script to return an error.

# If a commit messages contains a line that says 'DONT_BUILD', exit
# immediately.
DONT_BUILD=$(git show|egrep '^\s{4}DONT_BUILD$')
if [ "x$DONT_BUILD" != "x" ]; then
    echo "*** Build not requested. Exiting."
    exit 1
fi

set -e
# We pipe our build output to a log file with tee.
# This bash setting ensures that the script exits if the build fails.
set -o pipefail
# gather core dumps
ulimit -c unlimited

BUILD_TYPE=${BUILD_TYPE:-DEBUG}
BUILD_TYPE=$(echo "$BUILD_TYPE" | tr a-z A-Z) # capitalize
BUILD_TYPE_LOWER=$(echo "$BUILD_TYPE" | tr A-Z a-z)

# Set up defaults for environment variables.
DEFAULT_ALLOW_SLOW_TESTS=1

# TSAN builds are pretty slow, so don't do SLOW tests unless explicitly
# requested. Setting KUDU_USE_TSAN influences the thirdparty build.
if [ "$BUILD_TYPE" = "TSAN" ]; then
  DEFAULT_ALLOW_SLOW_TESTS=0
  export KUDU_USE_TSAN=1
fi

export KUDU_FLAKY_TEST_ATTEMPTS=${KUDU_FLAKY_TEST_ATTEMPTS:-1}
export KUDU_ALLOW_SLOW_TESTS=${KUDU_ALLOW_SLOW_TESTS:-$DEFAULT_ALLOW_SLOW_TESTS}
export KUDU_COMPRESS_TEST_OUTPUT=${KUDU_COMPRESS_TEST_OUTPUT:-1}
export TEST_TMPDIR=${TEST_TMPDIR:-/tmp/kudutest-$UID}
BUILD_JAVA=${BUILD_JAVA:-1}
BUILD_GRADLE=${BUILD_GRADLE:-1}
BUILD_PYTHON=${BUILD_PYTHON:-1}
BUILD_PYTHON3=${BUILD_PYTHON3:-1}
ERROR_ON_TEST_FAILURE=${ERROR_ON_TEST_FAILURE:-1}

# Ensure that the test data directory is usable.
mkdir -p "$TEST_TMPDIR"
if [ ! -w "$TEST_TMPDIR" ]; then
  echo "Error: Test output directory ($TEST_TMPDIR) is not writable on $(hostname) by user $(whoami)"
  exit 1
fi

SOURCE_ROOT=$(cd $(dirname "$BASH_SOURCE")/../..; pwd)
BUILD_ROOT=$SOURCE_ROOT/build/$BUILD_TYPE_LOWER

# Remove testing artifacts from the previous run before we do anything
# else. Otherwise, if we fail during the "build" step, Jenkins will
# archive the test logs from the previous run, thinking they came from
# this run, and confuse us when we look at the failed build.
rm -rf $BUILD_ROOT
mkdir -p $BUILD_ROOT

# Same for the Java tests, which aren't inside BUILD_ROOT
rm -rf $SOURCE_ROOT/java/*/build

list_flaky_tests() {
  local url="http://$TEST_RESULT_SERVER/list_failed_tests?num_days=3&build_pattern=%25kudu-test%25"
  >&2 echo Fetching flaky test list from "$url" ...
  curl -s --show-error "$url"
  return $?
}

TEST_LOGDIR="$BUILD_ROOT/test-logs"
TEST_DEBUGDIR="$BUILD_ROOT/test-debug"

cleanup() {
  echo Cleaning up all build artifacts...
  $SOURCE_ROOT/build-support/jenkins/post-build-clean.sh
}
# If we're running inside Jenkins (the BUILD_ID is set), then install
# an exit handler which will clean up all of our build results.
if [ -n "$BUILD_ID" ]; then
  trap cleanup EXIT
fi

# Configure the build
#
# ASAN/TSAN can't build the Python bindings because the exported Kudu client
# library (which the bindings depend on) is missing ASAN/TSAN symbols.
if [ "$BUILD_TYPE" = "ASAN" ]; then
  USE_CLANG=1
  CMAKE_BUILD=fastdebug
  EXTRA_BUILD_FLAGS="-DKUDU_USE_ASAN=1 -DKUDU_USE_UBSAN=1"
  BUILD_PYTHON=0
  BUILD_PYTHON3=0
elif [ "$BUILD_TYPE" = "TSAN" ]; then
  USE_CLANG=1
  CMAKE_BUILD=fastdebug
  EXTRA_BUILD_FLAGS="-DKUDU_USE_TSAN=1"
  EXTRA_TEST_FLAGS="$EXTRA_TEST_FLAGS -LE no_tsan"
  BUILD_PYTHON=0
  BUILD_PYTHON3=0
elif [ "$BUILD_TYPE" = "COVERAGE" ]; then
  USE_CLANG=1
  CMAKE_BUILD=debug
  EXTRA_BUILD_FLAGS="-DKUDU_GENERATE_COVERAGE=1"
  DO_COVERAGE=1

  # We currently dont capture coverage for Java or Python.
  BUILD_JAVA=0
  BUILD_PYTHON=0
  BUILD_PYTHON3=0
elif [ "$BUILD_TYPE" = "LINT" ]; then
  CMAKE_BUILD=debug
elif [ "$BUILD_TYPE" = "IWYU" ]; then
  USE_CLANG=1
  CMAKE_BUILD=debug
else
  # Must be DEBUG or RELEASE
  CMAKE_BUILD=$BUILD_TYPE
fi

# If we are supposed to be resistant to flaky tests or run just flaky tests,
# we need to fetch the list.
if [ "$RUN_FLAKY_ONLY" == "1" -o "$KUDU_FLAKY_TEST_ATTEMPTS" -gt 1 ]; then
  export KUDU_FLAKY_TEST_LIST=$BUILD_ROOT/flaky-tests.txt
  mkdir -p $(dirname $KUDU_FLAKY_TEST_LIST)
  if list_flaky_tests > $KUDU_FLAKY_TEST_LIST; then
    echo The list of flaky tests:
    echo ------------------------------------------------------------
    cat $KUDU_FLAKY_TEST_LIST
    echo
    echo ------------------------------------------------------------
  else
    echo "Could not fetch flaky tests list."
    if [ "$RUN_FLAKY_ONLY" == "1" ]; then
      exit 1
    fi

    echo No list of flaky tests, disabling the flaky test resistance.
    export KUDU_FLAKY_TEST_ATTEMPTS=1
  fi

  if [ "$RUN_FLAKY_ONLY" == "1" ]; then
    test_regex=$(perl -e '
      chomp(my @lines = <>);
      print join("|", map { "^" . quotemeta($_) } @lines);
     ' $KUDU_FLAKY_TEST_LIST)
    if [ -z "$test_regex" ]; then
      echo No tests are flaky.
      exit 0
    fi

    EXTRA_TEST_FLAGS="$EXTRA_TEST_FLAGS -R $test_regex"

    # We don't support detecting java and python flaky tests at the moment.
    echo "RUN_FLAKY_ONLY=1: running flaky tests only,"\
         "disabling Java and python builds."
    BUILD_PYTHON=0
    BUILD_PYTHON3=0
    BUILD_JAVA=0
  elif [ "$KUDU_FLAKY_TEST_ATTEMPTS" -gt 1 ]; then
    echo Will retry the flaky tests up to $KUDU_FLAKY_TEST_ATTEMPTS times.
  fi
fi

THIRDPARTY_TYPE=
if [ "$BUILD_TYPE" = "TSAN" ]; then
  THIRDPARTY_TYPE=tsan
fi
$SOURCE_ROOT/build-support/enable_devtoolset.sh thirdparty/build-if-necessary.sh $THIRDPARTY_TYPE

THIRDPARTY_BIN=$(pwd)/thirdparty/installed/common/bin
export PPROF_PATH=$THIRDPARTY_BIN/pprof

if which ccache >/dev/null ; then
  CLANG=$(pwd)/build-support/ccache-clang/clang
else
  CLANG=$(pwd)/thirdparty/clang-toolchain/bin/clang
fi

# Assemble the cmake command line, starting with environment variables.

# There's absolutely no reason to rebuild the thirdparty tree; we just ran
# build-if-necessary.sh above.
CMAKE="env NO_REBUILD_THIRDPARTY=1"

# If using clang, we have to change the compiler via environment variables.
if [ -n "$USE_CLANG" ]; then
  CMAKE="$CMAKE CC=$CLANG CXX=$CLANG++"
fi

# This will be a passthrough for systems without devtoolset.
CMAKE="$CMAKE $SOURCE_ROOT/build-support/enable_devtoolset.sh"

CMAKE="$CMAKE $THIRDPARTY_BIN/cmake"
CMAKE="$CMAKE -DCMAKE_BUILD_TYPE=$CMAKE_BUILD"

# On distributed tests, force dynamic linking even for release builds. Otherwise,
# the test binaries are too large and we spend way too much time uploading them
# to the test slaves.
if [ "$ENABLE_DIST_TEST" == "1" ]; then
  CMAKE="$CMAKE -DKUDU_LINK=dynamic"
fi
if [ -n "$EXTRA_BUILD_FLAGS" ]; then
  CMAKE="$CMAKE $EXTRA_BUILD_FLAGS"
fi
CMAKE="$CMAKE $SOURCE_ROOT"
cd $BUILD_ROOT
$CMAKE

# Create empty test logs or else Jenkins fails to archive artifacts, which
# results in the build failing.
mkdir -p Testing/Temporary
mkdir -p $TEST_LOGDIR

# Short circuit for LINT builds.
if [ "$BUILD_TYPE" = "LINT" ]; then
  make lint | tee $TEST_LOGDIR/lint.log
  exit $?
fi

# Short circuit for IWYU builds: run the include-what-you-use tool on the files
# modified since the last committed changelist committed upstream.
if [ "$BUILD_TYPE" = "IWYU" ]; then
  make iwyu | tee $TEST_LOGDIR/iwyu.log
  exit $?
fi

# Only enable test core dumps for certain build types.
if [ "$BUILD_TYPE" != "ASAN" ]; then
  export KUDU_TEST_ULIMIT_CORE=unlimited
fi

# our tests leave lots of data lying around, clean up before we run
if [ -d "$TEST_TMPDIR" ]; then
  rm -Rf $TEST_TMPDIR/*
fi

# actually do the build
echo
echo Building C++ code.
echo ------------------------------------------------------------
NUM_PROCS=$(getconf _NPROCESSORS_ONLN)
make -j$NUM_PROCS 2>&1 | tee build.log

# If compilation succeeds, try to run all remaining steps despite any failures.
set +e

TESTS_FAILED=0
EXIT_STATUS=0
FAILURES=""

# If we're running distributed C++ tests, submit them asynchronously while
# we run the Java and Python tests.
if [ "$ENABLE_DIST_TEST" == "1" ]; then
  echo
  echo Submitting C++ distributed-test job.
  echo ------------------------------------------------------------
  # dist-test uses DIST_TEST_JOB_PATH to define where to output it's id file.
  export DIST_TEST_JOB_PATH=$BUILD_ROOT/c-dist-test-job-id
  rm -f $DIST_TEST_JOB_PATH
  if ! $SOURCE_ROOT/build-support/dist_test.py --no-wait run ; then
    EXIT_STATUS=1
    FAILURES="$FAILURES"$'Could not submit C++ distributed test job\n'
  fi
  # Still need to run a few non-dist-test-capable tests locally.
  EXTRA_TEST_FLAGS="$EXTRA_TEST_FLAGS -L no_dist_test"
fi

if ! $THIRDPARTY_BIN/ctest -j$NUM_PROCS $EXTRA_TEST_FLAGS ; then
  TESTS_FAILED=1
  FAILURES="$FAILURES"$'C++ tests failed\n'
fi

if [ "$DO_COVERAGE" == "1" ]; then
  echo
  echo Generating coverage report...
  echo ------------------------------------------------------------
  if ! $SOURCE_ROOT/thirdparty/installed/common/bin/gcovr \
      -r $SOURCE_ROOT \
      --gcov-filter='.*src#kudu.*' \
      --gcov-executable=$SOURCE_ROOT/build-support/llvm-gcov-wrapper \
      --xml \
      > $BUILD_ROOT/coverage.xml ; then
    EXIT_STATUS=1
    FAILURES="$FAILURES"$'Coverage report failed\n'
  fi
fi

if [ "$BUILD_JAVA" == "1" ]; then
  echo
  echo Building and testing java...
  echo ------------------------------------------------------------

  # Make sure we use JDK8
  export JAVA_HOME=$JAVA8_HOME
  export PATH=$JAVA_HOME/bin:$PATH
  pushd $SOURCE_ROOT/java
  set -x

  # Run the full Gradle build.
  export EXTRA_GRADLE_FLAGS="--console=plain"
  EXTRA_GRADLE_FLAGS="$EXTRA_GRADLE_FLAGS --no-daemon"
  EXTRA_GRADLE_FLAGS="$EXTRA_GRADLE_FLAGS --continue"
  EXTRA_GRADLE_FLAGS="$EXTRA_GRADLE_FLAGS -DrerunFailingTestsCount=3"
  # KUDU-2524: temporarily disable scalafmt until we can work out its JDK
  # incompatibility issue.
  EXTRA_GRADLE_FLAGS="$EXTRA_GRADLE_FLAGS -DskipFormat"
  EXTRA_GRADLE_FLAGS="$EXTRA_GRADLE_FLAGS $GRADLE_FLAGS"
  # If we're running distributed Java tests, submit them asynchronously.
  if [ "$ENABLE_DIST_TEST" == "1" ]; then
    echo
    echo Submitting Java distributed-test job.
    echo ------------------------------------------------------------
    # dist-test uses DIST_TEST_JOB_PATH to define where to output it's id file.
    export DIST_TEST_JOB_PATH=$BUILD_ROOT/java-dist-test-job-id
    rm -f $DIST_TEST_JOB_PATH
    if ! $SOURCE_ROOT/build-support/dist_test.py --no-wait java run-all ; then
      EXIT_STATUS=1
      FAILURES="$FAILURES"$'Could not submit Java distributed test job\n'
    fi
  else
    # TODO: Run `gradle check` in BUILD_TYPE DEBUG when static code analysis is fixed
    if ! ./gradlew $EXTRA_GRADLE_FLAGS clean test ; then
      TESTS_FAILED=1
      FAILURES="$FAILURES"$'Java Gradle build/test failed\n'
    fi
  fi

  # Run a script to verify the contents of the JARs to ensure the shading and
  # packaging is correct.
  $SOURCE_ROOT/build-support/verify_jars.pl .

  set +x
  popd
fi

if [ "$BUILD_PYTHON" == "1" ]; then
  echo
  echo Building and testing python.
  echo ------------------------------------------------------------

  # Failing to compile the Python client should result in a build failure.
  set -e
  export KUDU_HOME=$SOURCE_ROOT
  export KUDU_BUILD=$BUILD_ROOT
  pushd $SOURCE_ROOT/python

  # Create a sane test environment.
  rm -Rf $KUDU_BUILD/py_env
  virtualenv $KUDU_BUILD/py_env
  source $KUDU_BUILD/py_env/bin/activate

  # Old versions of pip (such as the one found in el6) default to pypi's http://
  # endpoint which no longer exists. The -i option lets us switch to the
  # https:// endpoint in such cases.
  #
  # Unfortunately, in these old versions of pip, -i doesn't appear to apply
  # recursively to transitive dependencies installed via a direct dependency's
  # "python setup.py" command. Therefore we have no choice but to upgrade to a
  # new version of pip to proceed.
  #
  # Beginning with pip 10, Python 2.6 is no longer supported. Attempting to
  # upgrade to pip 10 on Python 2.6 yields syntax errors. We don't need any new
  # pip features, so let's pin to the last pip version to support Python 2.6.
  pip install -i https://pypi.python.org/simple $PIP_INSTALL_FLAGS --upgrade 'pip < 10.0.0b1'

  # New versions of pip raise an exception when upgrading old versions of
  # setuptools (such as the one found in el6). The workaround is to upgrade
  # setuptools on its own, outside of requirements.txt, and with the pip version
  # check disabled.
  pip install --disable-pip-version-check $PIP_INSTALL_FLAGS --upgrade 'setuptools >= 0.8'

  # One of our dependencies is pandas, installed below. It depends on numpy, and
  # if we don't install numpy directly, the pandas installation will install the
  # latest numpy which is incompatible with Python 2.6.
  #
  # To work around this, we need to install a 2.6-compatible version of numpy
  # before installing pandas. Installing numpy may involve some compiler work,
  # so we must pass in the current values of CC and CXX.
  #
  # See https://github.com/numpy/numpy/releases/tag/v1.12.0 for more details.
  CC=$CLANG CXX=$CLANG++ pip install $PIP_INSTALL_FLAGS 'numpy < 1.12.0'

  # We've got a new pip and new setuptools. We can now install the rest of the
  # Python client's requirements.
  #
  # Installing the Cython dependency may involve some compiler work, so we must
  # pass in the current values of CC and CXX.
  CC=$CLANG CXX=$CLANG++ pip install $PIP_INSTALL_FLAGS -r requirements.txt

  # We need to install Pandas manually because although it's not a required
  # package, it is needed to run all of the tests.
  #
  # Installing pandas may involve some compiler work, so we must pass in the
  # current values of CC and CXX.
  #
  # pandas 0.18 dropped support for python 2.6. See https://pandas.pydata.org/pandas-docs/version/0.23.0/whatsnew.html#v0-18-0-march-13-2016
  # for more details.
  CC=$CLANG CXX=$CLANG++ pip install $PIP_INSTALL_FLAGS 'pandas < 0.18'

  # Delete old Cython extensions to force them to be rebuilt.
  rm -Rf build kudu_python.egg-info kudu/*.so

  # Build the Python bindings. This assumes we run this script from base dir.
  CC=$CLANG CXX=$CLANG++ python setup.py build_ext
  set +e

  # Run the Python tests. This may also involve some compiler work.
  if ! CC=$CLANG CXX=$CLANG++ python setup.py test \
      --addopts="kudu --junit-xml=$TEST_LOGDIR/python_client.xml" \
      2> $TEST_LOGDIR/python_client.log ; then
    TESTS_FAILED=1
    FAILURES="$FAILURES"$'Python tests failed\n'
  fi

  deactivate
  popd
fi

if [ "$BUILD_PYTHON3" == "1" ]; then
  echo
  echo Building and testing python 3.
  echo ------------------------------------------------------------

  # Failing to compile the Python client should result in a build failure.
  set -e
  export KUDU_HOME=$SOURCE_ROOT
  export KUDU_BUILD=$BUILD_ROOT
  pushd $SOURCE_ROOT/python

  # Create a sane test environment.
  rm -Rf $KUDU_BUILD/py_env
  virtualenv -p python3 $KUDU_BUILD/py_env
  source $KUDU_BUILD/py_env/bin/activate

  # Old versions of pip (such as the one found in el6) default to pypi's http://
  # endpoint which no longer exists. The -i option lets us switch to the
  # https:// endpoint in such cases.
  #
  # Unfortunately, in these old versions of pip, -i doesn't appear to apply
  # recursively to transitive dependencies installed via a direct dependency's
  # "python setup.py" command. Therefore we have no choice but to upgrade to a
  # new version of pip to proceed.
  pip install -i https://pypi.python.org/simple $PIP_INSTALL_FLAGS --upgrade pip

  # New versions of pip raise an exception when upgrading old versions of
  # setuptools (such as the one found in el6). The workaround is to upgrade
  # setuptools on its own, outside of requirements.txt, and with the pip version
  # check disabled.
  pip install --disable-pip-version-check $PIP_INSTALL_FLAGS --upgrade 'setuptools >= 0.8'

  # One of our dependencies is pandas, installed below. It depends on numpy, and
  # if we don't install numpy directly, the pandas installation will install the
  # latest numpy which is incompatible with Python 3.4 (the version of Python 3
  # shipped with Ubuntu 14.04).
  #
  # To work around this, we need to install a 3.4-compatible version of numpy
  # before installing pandas. Installing numpy may involve some compiler work,
  # so we must pass in the current values of CC and CXX.
  #
  # See https://github.com/numpy/numpy/releases/tag/v1.16.0rc1 for more details.
  CC=$CLANG CXX=$CLANG++ pip install $PIP_INSTALL_FLAGS 'numpy < 1.16.0'

  # We've got a new pip and new setuptools. We can now install the rest of the
  # Python client's requirements.
  #
  # Installing the Cython dependency may involve some compiler work, so we must
  # pass in the current values of CC and CXX.
  CC=$CLANG CXX=$CLANG++ pip install $PIP_INSTALL_FLAGS -r requirements.txt

  # We need to install Pandas manually because although it's not a required
  # package, it is needed to run all of the tests.
  #
  # Installing pandas may involve some compiler work, so we must pass in the
  # current values of CC and CXX.
  CC=$CLANG CXX=$CLANG++ pip install $PIP_INSTALL_FLAGS pandas

  # Delete old Cython extensions to force them to be rebuilt.
  rm -Rf build kudu_python.egg-info kudu/*.so

  # Build the Python bindings. This assumes we run this script from base dir.
  CC=$CLANG CXX=$CLANG++ python setup.py build_ext
  set +e

  # Run the Python tests. This may also involve some compiler work.
  if ! CC=$CLANG CXX=$CLANG++ python setup.py test \
      --addopts="kudu --junit-xml=$TEST_LOGDIR/python3_client.xml" \
      2> $TEST_LOGDIR/python3_client.log ; then
    TESTS_FAILED=1
    FAILURES="$FAILURES"$'Python 3 tests failed\n'
  fi

  deactivate
  popd
fi

# If we submitted the tasks earlier, go fetch the results now
if [ "$ENABLE_DIST_TEST" == "1" ]; then
  echo
  echo Fetching previously submitted C++ dist-test results...
  echo ------------------------------------------------------------
  C_DIST_TEST_ID=`cat $BUILD_ROOT/c-dist-test-job-id`
  if ! $DIST_TEST_HOME/bin/client watch $C_DIST_TEST_ID ; then
    TESTS_FAILED=1
    FAILURES="$FAILURES"$'Distributed C++ tests failed\n'
  fi
  DT_DIR=$TEST_LOGDIR/dist-test-out
  rm -Rf $DT_DIR
  $DIST_TEST_HOME/bin/client fetch --artifacts -d $DT_DIR $C_DIST_TEST_ID
  # Fetching the artifacts expands each log into its own directory.
  # Move them back into the main log directory
  rm -f $DT_DIR/*zip
  for arch_dir in $DT_DIR/* ; do
    mv $arch_dir/build/$BUILD_TYPE_LOWER/test-logs/* $TEST_LOGDIR
    rm -Rf $arch_dir
  done

  echo
  echo Fetching previously submitted Java dist-test results...
  echo ------------------------------------------------------------
  JAVA_DIST_TEST_ID=`cat $BUILD_ROOT/java-dist-test-job-id`
  if ! $DIST_TEST_HOME/bin/client watch $JAVA_DIST_TEST_ID ; then
    TESTS_FAILED=1
    FAILURES="$FAILURES"$'Distributed Java tests failed\n'
  fi
  DT_DIR=$TEST_LOGDIR/java-dist-test-out
  rm -Rf $DT_DIR
  $DIST_TEST_HOME/bin/client fetch --artifacts -d $DT_DIR $JAVA_DIST_TEST_ID
  # Fetching the artifacts expands each log into its own directory.
  # Move them back into the main log directory
  rm -f $DT_DIR/*zip
  for arch_dir in $DT_DIR/* ; do
    mv $arch_dir/build/java/test-logs/* $TEST_LOGDIR
    rm -Rf $arch_dir
  done
fi

if [ "$TESTS_FAILED" != "0" -o "$EXIT_STATUS" != "0" ]; then
  echo
  echo Tests failed, making sure we have XML files for all tests.
  echo ------------------------------------------------------------

  # Tests that crash do not generate JUnit report XML files.
  # We go through and generate a kind of poor-man's version of them in those cases.
  for GTEST_OUTFILE in $TEST_LOGDIR/*.txt.gz; do
    TEST_EXE=$(basename $GTEST_OUTFILE .txt.gz)
    GTEST_XMLFILE="$TEST_LOGDIR/$TEST_EXE.xml"
    if [ ! -f "$GTEST_XMLFILE" ]; then
      echo "JUnit report missing:" \
           "generating fake JUnit report file from $GTEST_OUTFILE and saving it to $GTEST_XMLFILE"
      zcat $GTEST_OUTFILE | $SOURCE_ROOT/build-support/parse_test_failure.py -x > $GTEST_XMLFILE
    fi
  done
else
  # If all tests passed, ensure that they cleaned up their test output.
  TEST_TMPDIR_CONTENTS=$(ls $TEST_TMPDIR)
  if [ -n "$TEST_TMPDIR_CONTENTS" ]; then
    echo "All tests passed, yet some left behind their test output."
    echo "TEST_TMPDIR: $TEST_TMPDIR"
    find $TEST_TMPDIR -ls
    EXIT_STATUS=1
  fi
fi

set -e

if [ -n "$FAILURES" ]; then
  echo
  echo
  echo ======================================================================
  echo Failure summary
  echo ======================================================================
  echo $FAILURES
  echo
  echo
fi

# If any of the tests failed and we are honoring test failures, set the exit
# status to 1. Note that it may have already been set to 1 above.
if [ "$ERROR_ON_TEST_FAILURE" == "1" -a "$TESTS_FAILED" == "1" ]; then
  EXIT_STATUS=1
fi

exit $EXIT_STATUS

#!/bin/bash
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.
#
# This script is invoked from the Jenkins builds to build Kudu
# and run all the unit tests.
#
# Environment variables may be used to customize operation:
#   BUILD_TYPE: Default: DEBUG
#     Maybe be one of ASAN|TSAN|DEBUG|RELEASE|COVERAGE|LINT|IWYU
#
#   KUDU_ALLOW_SLOW_TESTS   Default: 1
#     Runs the "slow" version of the unit tests. Set to 0 to
#     run the tests more quickly.
#
#   TEST_TMPDIR   Default: /tmp/kudutest-$UID
#     Specifies the temporary directory where tests should write their
#     data. It is expected that following the completion of all tests, this
#     directory is empty (i.e. every test cleaned up after itself).
#
#   RUN_FLAKY_ONLY    Default: 0
#     Only runs tests which have failed recently, if this is 1.
#     Used by the kudu-flaky-tests jenkins build.
#
#   KUDU_FLAKY_TEST_ATTEMPTS  Default: 1
#     If more than 1, will fetch the list of known flaky tests
#     from the kudu-test jenkins job, and allow those tests to
#     be flaky in this build.
#
#   TEST_RESULT_SERVER  Default: none
#     The host:port pair of a server running test_result_server.py.
#     This must be configured for flaky test resistance or test result
#     reporting to work.
#
#   ENABLE_DIST_TEST  Default: 0
#     If set to 1, will submit C++ and Java tests to be run by the
#     distributed test runner instead of running them locally.
#     This requires that $DIST_TEST_HOME be set to a working dist_test
#     checkout (and that dist_test itself be appropriately configured to
#     point to a cluster)
#
#   BUILD_JAVA        Default: 1
#     Build and test java code if this is set to 1.
#
#   BUILD_PYTHON       Default: 1
#     Build and test the Python wrapper of the client API.
#
#   BUILD_PYTHON3      Default: 1
#     Build and test the Python wrapper of the client API in Python. This
#     option is not mutually exclusive from BUILD_PYTHON. If both options
#     are set (default), then both will be run.
#
#   PIP_INSTALL_FLAGS  Default: ""
#     Extra flags which are passed to 'pip install' when setting up the build
#     environment for the Python wrapper.
#
#   GRADLE_FLAGS       Default: ""
#     Extra flags which are passed to 'gradle' when running Gradle commands.
#
#   ERROR_ON_TEST_FAILURE    Default: 1
#     Whether test failures will cause this script to return an error.

# If a commit messages contains a line that says 'DONT_BUILD', exit
# immediately.
DONT_BUILD=$(git show|egrep '^\s{4}DONT_BUILD$')
if [ "x$DONT_BUILD" != "x" ]; then
    echo "*** Build not requested. Exiting."
    exit 1
fi

set -e
# We pipe our build output to a log file with tee.
# This bash setting ensures that the script exits if the build fails.
set -o pipefail
# gather core dumps
ulimit -c unlimited

BUILD_TYPE=${BUILD_TYPE:-DEBUG}
BUILD_TYPE=$(echo "$BUILD_TYPE" | tr a-z A-Z) # capitalize
BUILD_TYPE_LOWER=$(echo "$BUILD_TYPE" | tr A-Z a-z)

# Set up defaults for environment variables.
DEFAULT_ALLOW_SLOW_TESTS=1

# TSAN builds are pretty slow, so don't do SLOW tests unless explicitly
# requested. Setting KUDU_USE_TSAN influences the thirdparty build.
if [ "$BUILD_TYPE" = "TSAN" ]; then
  DEFAULT_ALLOW_SLOW_TESTS=0
  export KUDU_USE_TSAN=1
fi

export KUDU_FLAKY_TEST_ATTEMPTS=${KUDU_FLAKY_TEST_ATTEMPTS:-1}
export KUDU_ALLOW_SLOW_TESTS=${KUDU_ALLOW_SLOW_TESTS:-$DEFAULT_ALLOW_SLOW_TESTS}
export KUDU_COMPRESS_TEST_OUTPUT=${KUDU_COMPRESS_TEST_OUTPUT:-1}
export TEST_TMPDIR=${TEST_TMPDIR:-/tmp/kudutest-$UID}
BUILD_JAVA=${BUILD_JAVA:-1}
BUILD_GRADLE=${BUILD_GRADLE:-1}
BUILD_PYTHON=${BUILD_PYTHON:-1}
BUILD_PYTHON3=${BUILD_PYTHON3:-1}
ERROR_ON_TEST_FAILURE=${ERROR_ON_TEST_FAILURE:-1}

# Ensure that the test data directory is usable.
mkdir -p "$TEST_TMPDIR"
if [ ! -w "$TEST_TMPDIR" ]; then
  echo "Error: Test output directory ($TEST_TMPDIR) is not writable on $(hostname) by user $(whoami)"
  exit 1
fi

SOURCE_ROOT=$(cd $(dirname "$BASH_SOURCE")/../..; pwd)
BUILD_ROOT=$SOURCE_ROOT/build/$BUILD_TYPE_LOWER

# Remove testing artifacts from the previous run before we do anything
# else. Otherwise, if we fail during the "build" step, Jenkins will
# archive the test logs from the previous run, thinking they came from
# this run, and confuse us when we look at the failed build.
rm -rf $BUILD_ROOT
mkdir -p $BUILD_ROOT

# Same for the Java tests, which aren't inside BUILD_ROOT
rm -rf $SOURCE_ROOT/java/*/build

list_flaky_tests() {
  local url="http://$TEST_RESULT_SERVER/list_failed_tests?num_days=3&build_pattern=%25kudu-test%25"
  >&2 echo Fetching flaky test list from "$url" ...
  curl -s --show-error "$url"
  return $?
}

TEST_LOGDIR="$BUILD_ROOT/test-logs"
TEST_DEBUGDIR="$BUILD_ROOT/test-debug"

cleanup() {
  echo Cleaning up all build artifacts...
  $SOURCE_ROOT/build-support/jenkins/post-build-clean.sh
}
# If we're running inside Jenkins (the BUILD_ID is set), then install
# an exit handler which will clean up all of our build results.
if [ -n "$BUILD_ID" ]; then
  trap cleanup EXIT
fi

THIRDPARTY_TYPE=
if [ "$BUILD_TYPE" = "TSAN" ]; then
  THIRDPARTY_TYPE=tsan
fi
$SOURCE_ROOT/build-support/enable_devtoolset.sh thirdparty/build-if-necessary.sh $THIRDPARTY_TYPE

THIRDPARTY_BIN=$(pwd)/thirdparty/installed/common/bin
export PPROF_PATH=$THIRDPARTY_BIN/pprof

if which ccache >/dev/null ; then
  CLANG=$(pwd)/build-support/ccache-clang/clang
else
  CLANG=$(pwd)/thirdparty/clang-toolchain/bin/clang
fi

# Configure the build
#
# ASAN/TSAN can't build the Python bindings because the exported Kudu client
# library (which the bindings depend on) is missing ASAN/TSAN symbols.
cd $BUILD_ROOT
if [ "$BUILD_TYPE" = "ASAN" ]; then
  USE_CLANG=1
  CMAKE_BUILD=fastdebug
  EXTRA_BUILD_FLAGS="-DKUDU_USE_ASAN=1 -DKUDU_USE_UBSAN=1"
  BUILD_PYTHON=0
  BUILD_PYTHON3=0
elif [ "$BUILD_TYPE" = "TSAN" ]; then
  USE_CLANG=1
  CMAKE_BUILD=fastdebug
  EXTRA_BUILD_FLAGS="-DKUDU_USE_TSAN=1"
  EXTRA_TEST_FLAGS="$EXTRA_TEST_FLAGS -LE no_tsan"
  BUILD_PYTHON=0
  BUILD_PYTHON3=0
elif [ "$BUILD_TYPE" = "COVERAGE" ]; then
  USE_CLANG=1
  CMAKE_BUILD=debug
  EXTRA_BUILD_FLAGS="-DKUDU_GENERATE_COVERAGE=1"
  DO_COVERAGE=1

  # We currently dont capture coverage for Java or Python.
  BUILD_JAVA=0
  BUILD_PYTHON=0
  BUILD_PYTHON3=0
elif [ "$BUILD_TYPE" = "LINT" ]; then
  CMAKE_BUILD=debug
elif [ "$BUILD_TYPE" = "IWYU" ]; then
  USE_CLANG=1
  CMAKE_BUILD=debug
else
  # Must be DEBUG or RELEASE
  CMAKE_BUILD=$BUILD_TYPE
fi

# Assemble the cmake command line, starting with environment variables.

# There's absolutely no reason to rebuild the thirdparty tree; we just ran
# build-if-necessary.sh above.
CMAKE="env NO_REBUILD_THIRDPARTY=1"

# If using clang, we have to change the compiler via environment variables.
if [ -n "$USE_CLANG" ]; then
  CMAKE="$CMAKE CC=$CLANG CXX=$CLANG++"
fi

# This will be a passthrough for systems without devtoolset.
CMAKE="$CMAKE $SOURCE_ROOT/build-support/enable_devtoolset.sh"

CMAKE="$CMAKE $THIRDPARTY_BIN/cmake"
CMAKE="$CMAKE -DCMAKE_BUILD_TYPE=$CMAKE_BUILD"

# On distributed tests, force dynamic linking even for release builds. Otherwise,
# the test binaries are too large and we spend way too much time uploading them
# to the test slaves.
if [ "$ENABLE_DIST_TEST" == "1" ]; then
  CMAKE="$CMAKE -DKUDU_LINK=dynamic"
fi
if [ -n "$EXTRA_BUILD_FLAGS" ]; then
  CMAKE="$CMAKE $EXTRA_BUILD_FLAGS"
fi
CMAKE="$CMAKE $SOURCE_ROOT"
$CMAKE

# Create empty test logs or else Jenkins fails to archive artifacts, which
# results in the build failing.
mkdir -p Testing/Temporary
mkdir -p $TEST_LOGDIR

# Short circuit for LINT builds.
if [ "$BUILD_TYPE" = "LINT" ]; then
  make lint | tee $TEST_LOGDIR/lint.log
  exit $?
fi

# Short circuit for IWYU builds: run the include-what-you-use tool on the files
# modified since the last committed changelist committed upstream.
if [ "$BUILD_TYPE" = "IWYU" ]; then
  make iwyu | tee $TEST_LOGDIR/iwyu.log
  exit $?
fi

# Only enable test core dumps for certain build types.
if [ "$BUILD_TYPE" != "ASAN" ]; then
  export KUDU_TEST_ULIMIT_CORE=unlimited
fi

# If we are supposed to be resistant to flaky tests, we need to fetch the
# list of tests to ignore
if [ "$KUDU_FLAKY_TEST_ATTEMPTS" -gt 1 ]; then
  echo Fetching flaky test list...
  export KUDU_FLAKY_TEST_LIST=$BUILD_ROOT/flaky-tests.txt
  mkdir -p $(dirname $KUDU_FLAKY_TEST_LIST)
  echo -n > $KUDU_FLAKY_TEST_LIST
    if [ -n "$TEST_RESULT_SERVER" ] && \
        list_flaky_tests > $KUDU_FLAKY_TEST_LIST ; then
    echo Will retry flaky tests up to $KUDU_FLAKY_TEST_ATTEMPTS times:
    cat $KUDU_FLAKY_TEST_LIST
    echo ----------
  else
    echo Unable to fetch flaky test list. Disabling flaky test resistance.
    export KUDU_FLAKY_TEST_ATTEMPTS=1
  fi
fi

# our tests leave lots of data lying around, clean up before we run
if [ -d "$TEST_TMPDIR" ]; then
  rm -Rf $TEST_TMPDIR/*
fi

# actually do the build
echo
echo Building C++ code.
echo ------------------------------------------------------------
NUM_PROCS=$(getconf _NPROCESSORS_ONLN)
make -j$NUM_PROCS 2>&1 | tee build.log

# If compilation succeeds, try to run all remaining steps despite any failures.
set +e

# Run tests
if [ "$RUN_FLAKY_ONLY" == "1" ] ; then
  if [ -z "$TEST_RESULT_SERVER" ]; then
    echo Must set TEST_RESULT_SERVER to use RUN_FLAKY_ONLY
    exit 1
  fi
  echo
  echo Running flaky tests only:
  echo ------------------------------------------------------------
  if ! ( set -o pipefail ;
         list_flaky_tests | tee $BUILD_ROOT/flaky-tests.txt) ; then
    echo Could not fetch flaky tests list.
    exit 1
  fi
  test_regex=$(perl -e '
    chomp(my @lines = <>);
    print join("|", map { "^" . quotemeta($_) } @lines);
   ' $BUILD_ROOT/flaky-tests.txt)
  if [ -z "$test_regex" ]; then
    echo No tests are flaky.
    exit 0
  fi
  EXTRA_TEST_FLAGS="$EXTRA_TEST_FLAGS -R $test_regex"

  # We don't support detecting java and python flaky tests at the moment.
  echo Disabling Java and python build since RUN_FLAKY_ONLY=1
  BUILD_PYTHON=0
  BUILD_PYTHON3=0
  BUILD_JAVA=0
fi

TESTS_FAILED=0
EXIT_STATUS=0
FAILURES=""

# If we're running distributed C++ tests, submit them asynchronously while
# we run the Java and Python tests.
if [ "$ENABLE_DIST_TEST" == "1" ]; then
  echo
  echo Submitting C++ distributed-test job.
  echo ------------------------------------------------------------
  # dist-test uses DIST_TEST_JOB_PATH to define where to output it's id file.
  export DIST_TEST_JOB_PATH=$BUILD_ROOT/c-dist-test-job-id
  rm -f $DIST_TEST_JOB_PATH
  if ! $SOURCE_ROOT/build-support/dist_test.py --no-wait run ; then
    EXIT_STATUS=1
    FAILURES="$FAILURES"$'Could not submit C++ distributed test job\n'
  fi
  # Still need to run a few non-dist-test-capable tests locally.
  EXTRA_TEST_FLAGS="$EXTRA_TEST_FLAGS -L no_dist_test"
fi

if ! $THIRDPARTY_BIN/ctest -j$NUM_PROCS $EXTRA_TEST_FLAGS ; then
  TESTS_FAILED=1
  FAILURES="$FAILURES"$'C++ tests failed\n'
fi

if [ "$DO_COVERAGE" == "1" ]; then
  echo
  echo Generating coverage report...
  echo ------------------------------------------------------------
  if ! $SOURCE_ROOT/thirdparty/installed/common/bin/gcovr \
      -r $SOURCE_ROOT \
      --gcov-filter='.*src#kudu.*' \
      --gcov-executable=$SOURCE_ROOT/build-support/llvm-gcov-wrapper \
      --xml \
      > $BUILD_ROOT/coverage.xml ; then
    EXIT_STATUS=1
    FAILURES="$FAILURES"$'Coverage report failed\n'
  fi
fi

if [ "$BUILD_JAVA" == "1" ]; then
  echo
  echo Building and testing java...
  echo ------------------------------------------------------------

  # Make sure we use JDK8
  export JAVA_HOME=$JAVA8_HOME
  export PATH=$JAVA_HOME/bin:$PATH
  pushd $SOURCE_ROOT/java
  set -x

  # Run the full Gradle build.
  export EXTRA_GRADLE_FLAGS="--console=plain"
  EXTRA_GRADLE_FLAGS="$EXTRA_GRADLE_FLAGS --no-daemon"
  EXTRA_GRADLE_FLAGS="$EXTRA_GRADLE_FLAGS --continue"
  EXTRA_GRADLE_FLAGS="$EXTRA_GRADLE_FLAGS -DrerunFailingTestsCount=3"
  # KUDU-2524: temporarily disable scalafmt until we can work out its JDK
  # incompatibility issue.
  EXTRA_GRADLE_FLAGS="$EXTRA_GRADLE_FLAGS -DskipFormat"
  EXTRA_GRADLE_FLAGS="$EXTRA_GRADLE_FLAGS $GRADLE_FLAGS"
  # If we're running distributed Java tests, submit them asynchronously.
  if [ "$ENABLE_DIST_TEST" == "1" ]; then
    echo
    echo Submitting Java distributed-test job.
    echo ------------------------------------------------------------
    # dist-test uses DIST_TEST_JOB_PATH to define where to output it's id file.
    export DIST_TEST_JOB_PATH=$BUILD_ROOT/java-dist-test-job-id
    rm -f $DIST_TEST_JOB_PATH
    if ! $SOURCE_ROOT/build-support/dist_test.py --no-wait java run-all ; then
      EXIT_STATUS=1
      FAILURES="$FAILURES"$'Could not submit Java distributed test job\n'
    fi
  else
    # TODO: Run `gradle check` in BUILD_TYPE DEBUG when static code analysis is fixed
    if ! ./gradlew $EXTRA_GRADLE_FLAGS clean test ; then
      TESTS_FAILED=1
      FAILURES="$FAILURES"$'Java Gradle build/test failed\n'
    fi
  fi

  # Run a script to verify the contents of the JARs to ensure the shading and
  # packaging is correct.
  $SOURCE_ROOT/build-support/verify_jars.pl .

  set +x
  popd
fi

if [ "$BUILD_PYTHON" == "1" ]; then
  echo
  echo Building and testing python.
  echo ------------------------------------------------------------

  # Failing to compile the Python client should result in a build failure.
  set -e
  export KUDU_HOME=$SOURCE_ROOT
  export KUDU_BUILD=$BUILD_ROOT
  pushd $SOURCE_ROOT/python

  # Create a sane test environment.
  rm -Rf $KUDU_BUILD/py_env
  virtualenv $KUDU_BUILD/py_env
  source $KUDU_BUILD/py_env/bin/activate

  # Old versions of pip (such as the one found in el6) default to pypi's http://
  # endpoint which no longer exists. The -i option lets us switch to the
  # https:// endpoint in such cases.
  #
  # Unfortunately, in these old versions of pip, -i doesn't appear to apply
  # recursively to transitive dependencies installed via a direct dependency's
  # "python setup.py" command. Therefore we have no choice but to upgrade to a
  # new version of pip to proceed.
  #
  # Beginning with pip 10, Python 2.6 is no longer supported. Attempting to
  # upgrade to pip 10 on Python 2.6 yields syntax errors. We don't need any new
  # pip features, so let's pin to the last pip version to support Python 2.6.
  pip install -i https://pypi.python.org/simple $PIP_INSTALL_FLAGS --upgrade 'pip < 10.0.0b1'

  # New versions of pip raise an exception when upgrading old versions of
  # setuptools (such as the one found in el6). The workaround is to upgrade
  # setuptools on its own, outside of requirements.txt, and with the pip version
  # check disabled.
  pip install --disable-pip-version-check $PIP_INSTALL_FLAGS --upgrade 'setuptools >= 0.8'

  # One of our dependencies is pandas, installed below. It depends on numpy, and
  # if we don't install numpy directly, the pandas installation will install the
  # latest numpy which is incompatible with Python 2.6.
  #
  # To work around this, we need to install a 2.6-compatible version of numpy
  # before installing pandas. Installing numpy may involve some compiler work,
  # so we must pass in the current values of CC and CXX.
  #
  # See https://github.com/numpy/numpy/releases/tag/v1.12.0 for more details.
  CC=$CLANG CXX=$CLANG++ pip install $PIP_INSTALL_FLAGS 'numpy < 1.12.0'

  # We've got a new pip and new setuptools. We can now install the rest of the
  # Python client's requirements.
  #
  # Installing the Cython dependency may involve some compiler work, so we must
  # pass in the current values of CC and CXX.
  CC=$CLANG CXX=$CLANG++ pip install $PIP_INSTALL_FLAGS -r requirements.txt

  # We need to install Pandas manually because although it's not a required
  # package, it is needed to run all of the tests.
  #
  # Installing pandas may involve some compiler work, so we must pass in the
  # current values of CC and CXX.
  #
  # pandas 0.18 dropped support for python 2.6. See https://pandas.pydata.org/pandas-docs/version/0.23.0/whatsnew.html#v0-18-0-march-13-2016
  # for more details.
  CC=$CLANG CXX=$CLANG++ pip install $PIP_INSTALL_FLAGS 'pandas < 0.18'

  # Delete old Cython extensions to force them to be rebuilt.
  rm -Rf build kudu_python.egg-info kudu/*.so

  # Build the Python bindings. This assumes we run this script from base dir.
  CC=$CLANG CXX=$CLANG++ python setup.py build_ext
  set +e

  # Run the Python tests. This may also involve some compiler work.
  if ! CC=$CLANG CXX=$CLANG++ python setup.py test \
      --addopts="kudu --junit-xml=$TEST_LOGDIR/python_client.xml" \
      2> $TEST_LOGDIR/python_client.log ; then
    TESTS_FAILED=1
    FAILURES="$FAILURES"$'Python tests failed\n'
  fi

  deactivate
  popd
fi

if [ "$BUILD_PYTHON3" == "1" ]; then
  echo
  echo Building and testing python 3.
  echo ------------------------------------------------------------

  # Failing to compile the Python client should result in a build failure.
  set -e
  export KUDU_HOME=$SOURCE_ROOT
  export KUDU_BUILD=$BUILD_ROOT
  pushd $SOURCE_ROOT/python

  # Create a sane test environment.
  rm -Rf $KUDU_BUILD/py_env
  virtualenv -p python3 $KUDU_BUILD/py_env
  source $KUDU_BUILD/py_env/bin/activate

  # Old versions of pip (such as the one found in el6) default to pypi's http://
  # endpoint which no longer exists. The -i option lets us switch to the
  # https:// endpoint in such cases.
  #
  # Unfortunately, in these old versions of pip, -i doesn't appear to apply
  # recursively to transitive dependencies installed via a direct dependency's
  # "python setup.py" command. Therefore we have no choice but to upgrade to a
  # new version of pip to proceed.
  pip install -i https://pypi.python.org/simple $PIP_INSTALL_FLAGS --upgrade pip

  # New versions of pip raise an exception when upgrading old versions of
  # setuptools (such as the one found in el6). The workaround is to upgrade
  # setuptools on its own, outside of requirements.txt, and with the pip version
  # check disabled.
  pip install --disable-pip-version-check $PIP_INSTALL_FLAGS --upgrade 'setuptools >= 0.8'

  # One of our dependencies is pandas, installed below. It depends on numpy, and
  # if we don't install numpy directly, the pandas installation will install the
  # latest numpy which is incompatible with Python 3.4 (the version of Python 3
  # shipped with Ubuntu 14.04).
  #
  # To work around this, we need to install a 3.4-compatible version of numpy
  # before installing pandas. Installing numpy may involve some compiler work,
  # so we must pass in the current values of CC and CXX.
  #
  # See https://github.com/numpy/numpy/releases/tag/v1.16.0rc1 for more details.
  CC=$CLANG CXX=$CLANG++ pip install $PIP_INSTALL_FLAGS 'numpy < 1.16.0'

  # We've got a new pip and new setuptools. We can now install the rest of the
  # Python client's requirements.
  #
  # Installing the Cython dependency may involve some compiler work, so we must
  # pass in the current values of CC and CXX.
  CC=$CLANG CXX=$CLANG++ pip install $PIP_INSTALL_FLAGS -r requirements.txt

  # We need to install Pandas manually because although it's not a required
  # package, it is needed to run all of the tests.
  #
  # Installing pandas may involve some compiler work, so we must pass in the
  # current values of CC and CXX.
  CC=$CLANG CXX=$CLANG++ pip install $PIP_INSTALL_FLAGS pandas

  # Delete old Cython extensions to force them to be rebuilt.
  rm -Rf build kudu_python.egg-info kudu/*.so

  # Build the Python bindings. This assumes we run this script from base dir.
  CC=$CLANG CXX=$CLANG++ python setup.py build_ext
  set +e

  # Run the Python tests. This may also involve some compiler work.
  if ! CC=$CLANG CXX=$CLANG++ python setup.py test \
      --addopts="kudu --junit-xml=$TEST_LOGDIR/python3_client.xml" \
      2> $TEST_LOGDIR/python3_client.log ; then
    TESTS_FAILED=1
    FAILURES="$FAILURES"$'Python 3 tests failed\n'
  fi

  deactivate
  popd
fi

# If we submitted the tasks earlier, go fetch the results now
if [ "$ENABLE_DIST_TEST" == "1" ]; then
  echo
  echo Fetching previously submitted C++ dist-test results...
  echo ------------------------------------------------------------
  C_DIST_TEST_ID=`cat $BUILD_ROOT/c-dist-test-job-id`
  if ! $DIST_TEST_HOME/bin/client watch $C_DIST_TEST_ID ; then
    TESTS_FAILED=1
    FAILURES="$FAILURES"$'Distributed C++ tests failed\n'
  fi
  DT_DIR=$TEST_LOGDIR/dist-test-out
  rm -Rf $DT_DIR
  $DIST_TEST_HOME/bin/client fetch --artifacts -d $DT_DIR $C_DIST_TEST_ID
  # Fetching the artifacts expands each log into its own directory.
  # Move them back into the main log directory
  rm -f $DT_DIR/*zip
  for arch_dir in $DT_DIR/* ; do
    mv $arch_dir/build/$BUILD_TYPE_LOWER/test-logs/* $TEST_LOGDIR
    rm -Rf $arch_dir
  done

  echo
  echo Fetching previously submitted Java dist-test results...
  echo ------------------------------------------------------------
  JAVA_DIST_TEST_ID=`cat $BUILD_ROOT/java-dist-test-job-id`
  if ! $DIST_TEST_HOME/bin/client watch $JAVA_DIST_TEST_ID ; then
    TESTS_FAILED=1
    FAILURES="$FAILURES"$'Distributed Java tests failed\n'
  fi
  DT_DIR=$TEST_LOGDIR/java-dist-test-out
  rm -Rf $DT_DIR
  $DIST_TEST_HOME/bin/client fetch --artifacts -d $DT_DIR $JAVA_DIST_TEST_ID
  # Fetching the artifacts expands each log into its own directory.
  # Move them back into the main log directory
  rm -f $DT_DIR/*zip
  for arch_dir in $DT_DIR/* ; do
    mv $arch_dir/build/java/test-logs/* $TEST_LOGDIR
    rm -Rf $arch_dir
  done
fi

if [ "$TESTS_FAILED" != "0" -o "$EXIT_STATUS" != "0" ]; then
  echo
  echo Tests failed, making sure we have XML files for all tests.
  echo ------------------------------------------------------------

  # Tests that crash do not generate JUnit report XML files.
  # We go through and generate a kind of poor-man's version of them in those cases.
  for GTEST_OUTFILE in $TEST_LOGDIR/*.txt.gz; do
    TEST_EXE=$(basename $GTEST_OUTFILE .txt.gz)
    GTEST_XMLFILE="$TEST_LOGDIR/$TEST_EXE.xml"
    if [ ! -f "$GTEST_XMLFILE" ]; then
      echo "JUnit report missing:" \
           "generating fake JUnit report file from $GTEST_OUTFILE and saving it to $GTEST_XMLFILE"
      zcat $GTEST_OUTFILE | $SOURCE_ROOT/build-support/parse_test_failure.py -x > $GTEST_XMLFILE
    fi
  done
else
  # If all tests passed, ensure that they cleaned up their test output.
  TEST_TMPDIR_CONTENTS=$(ls $TEST_TMPDIR)
  if [ -n "$TEST_TMPDIR_CONTENTS" ]; then
    echo "All tests passed, yet some left behind their test output."
    echo "TEST_TMPDIR: $TEST_TMPDIR"
    find $TEST_TMPDIR -ls
    EXIT_STATUS=1
  fi
fi

set -e

if [ -n "$FAILURES" ]; then
  echo
  echo
  echo ======================================================================
  echo Failure summary
  echo ======================================================================
  echo $FAILURES
  echo
  echo
fi

# If any of the tests failed and we are honoring test failures, set the exit
# status to 1. Note that it may have already been set to 1 above.
if [ "$ERROR_ON_TEST_FAILURE" == "1" -a "$TESTS_FAILED" == "1" ]; then
  EXIT_STATUS=1
fi

exit $EXIT_STATUS


setup: update rust

setup: update rust

remote: Support for password authentication was removed on August 13, 2021.
remote: Please see https://docs.github.com/en/get-started/getting-started-with-git/about-remote-repositories#cloning-with-https-urls for information on currently recommended modes of authentication.
fatal: Authentication failed for 'https://github.com/suyash/dotfiles.git/'


made finish message look nicer

made finish message look nicer

remote: Support for password authentication was removed on August 13, 2021.
remote: Please see https://docs.github.com/en/get-started/getting-started-with-git/about-remote-repositories#cloning-with-https-urls for information on currently recommended modes of authentication.
fatal: Authentication failed for 'https://github.com/satellitesharing/bloonix-satellite-raspberrypi.git/'


Create setup.sh

Create setup.sh

#!/bin/bash

echo "Installing all working bash scripts"

echo "Done installing all scripts"

error: pathspec 'setup.sh' did not match any file(s) known to git


Add vim-easymotion

Add vim-easymotion

remote: Support for password authentication was removed on August 13, 2021.
remote: Please see https://docs.github.com/en/get-started/getting-started-with-git/about-remote-repositories#cloning-with-https-urls for information on currently recommended modes of authentication.
fatal: Authentication failed for 'https://github.com/traib/.vim.git/'


setup: fonts: ignore roboto for now

setup: fonts: ignore roboto for now


Create setup.sh

Create setup.sh

remote: Support for password authentication was removed on August 13, 2021.
remote: Please see https://docs.github.com/en/get-started/getting-started-with-git/about-remote-repositories#cloning-with-https-urls for information on currently recommended modes of authentication.
fatal: Authentication failed for 'https://github.com/nsuchy/nginx-naxsi-installer.git/'


setup: go: install gometalinter deps

setup: go: install gometalinter deps


setup shell script

setup shell script


Add setup.sh script

Add setup.sh script

remote: Support for password authentication was removed on August 13, 2021.
remote: Please see https://docs.github.com/en/get-started/getting-started-with-git/about-remote-repositories#cloning-with-https-urls for information on currently recommended modes of authentication.
fatal: Authentication failed for 'https://github.com/crushmaster/rpm-build-tools.git/'


handle config files that aren’t dotfiles

remote: Support for password authentication was removed on August 13, 2021.
remote: Please see https://docs.github.com/en/get-started/getting-started-with-git/about-remote-repositories#cloning-with-https-urls for information on currently recommended modes of authentication.
fatal: Authentication failed for 'https://github.com/daveworth/dotfiles.git/'


Auto setup for teamspeak server

Auto setup for teamspeak server

remote: Support for password authentication was removed on August 13, 2021.
remote: Please see https://docs.github.com/en/get-started/getting-started-with-git/about-remote-repositories#cloning-with-https-urls for information on currently recommended modes of authentication.
fatal: Authentication failed for 'https://github.com/overshard/nebula.bythewood.me.git/'


Create setup.sh

Create setup.sh


added option to setup Node.js development

remote: Support for password authentication was removed on August 13, 2021.
remote: Please see https://docs.github.com/en/get-started/getting-started-with-git/about-remote-repositories#cloning-with-https-urls for information on currently recommended modes of authentication.
fatal: Authentication failed for 'https://github.com/paraschas/dotfiles.git/'


first commit

first commit

#! /bin/bash

ln -s $HOME/dotfiles/.vimrc $HOME/.vimrc
ln -s $HOME/dotfiles/.zshenv $HOME/.zshenv
ln -s $HOME/dotfiles/.zshrc $HOME/.zshrc


adds missing line break

adds missing line break

remote: Support for password authentication was removed on August 13, 2021.
remote: Please see https://docs.github.com/en/get-started/getting-started-with-git/about-remote-repositories#cloning-with-https-urls for information on currently recommended modes of authentication.
fatal: Authentication failed for 'https://github.com/elixytics/monolith.git/'


script to tag releases and build binaries

#!/usr/bin/env sh

TAG_NAME="$1"

if [ -z "$TAG_NAME" ]; then
    echo 1>&2 'no tag name given'
    exit 1
fi

build() {
    os=$1
    arch=$2
    echo $os $arch
    GOOS=$os GOARCH=$arch go build && tar czf "jqsh${TAG_NAME}.${os}-${arch}.tar.gz" jqsh && rm jqsh
    [ $? -eq 0 ] || exit 1
}

if [ -n "`git status --porcelain`" ]; then
    echo 1>&2 'repository is not clean'
    exit 1
fi

if [ -n "`ls *.tar.gz 2>/dev/null`" ]; then
    echo 1>&2 'remove previous distribution files'
    exit 1
fi

git tag -a "$1"

build darwin amd64
build linux amd64

echo "all distributions built successfully"

error: pathspec 'tag.bash' did not match any file(s) known to git


Create szy.sh

Create szy.sh

internal_interface=$(ip route|egrep "dev br0|dev br-lan"|head -1|cut -d" " -f3)
iptables -t filter -D FORWARD -i $internal_interface -j easucks-FORWARD 2>/dev/null
iptables -t filter -F easucks-FORWARD 2>/dev/null
iptables -t filter -X easucks-FORWARD 2>/dev/null
iptables -t filter -N easucks-FORWARD
iptables -t filter -I FORWARD -i $internal_interface -j easucks-FORWARD
iptables -t filter -A easucks-FORWARD -i $internal_interface -p udp -d 13.112.32.135/32 -j DROP
iptables -t filter -A easucks-FORWARD -i $internal_interface -p udp -d 13.230.248.238/32 -j DROP

iptables -t filter -A easucks-FORWARD -i $internal_interface -p udp -d 13.211.52.40/32 -j DROP
iptables -t filter -A easucks-FORWARD -i $internal_interface -p udp -d 13.211.232.151/32 -j DROP
iptables -t filter -A easucks-FORWARD -i $internal_interface -p udp -d 13.233.50.52/32 -j DROP
iptables -t filter -A easucks-FORWARD -i $internal_interface -p udp -d 18.196.167.42/32 -j DROP
iptables -t filter -A easucks-FORWARD -i $internal_interface -p udp -d 18.197.48.114/32 -j DROP
iptables -t filter -A easucks-FORWARD -i $internal_interface -p udp -d 18.206.67.50/32 -j DROP
iptables -t filter -A easucks-FORWARD -i $internal_interface -p udp -d 159.153.28.50/31 -j DROP
iptables -t filter -A easucks-FORWARD -i $internal_interface -p udp -d 169.57.76.162/32 -j DROP
iptables -t filter -A easucks-FORWARD -i $internal_interface -p udp -d 169.57.76.171/32 -j DROP
iptables -t filter -A easucks-FORWARD -i $internal_interface -p udp -d 185.179.200.211/32 -j DROP
iptables -t filter -A easucks-FORWARD -i $internal_interface -p udp -d 185.179.200.226/32 -j DROP
iptables -t filter -A easucks-FORWARD -i $internal_interface -p udp -d 185.179.203.68/32 -j DROP
iptables -t filter -A easucks-FORWARD -i $internal_interface -p udp -d 185.179.203.80/32 -j DROP
iptables -t filter -A easucks-FORWARD -i $internal_interface -p udp -d 34.246.50.222/32 -j DROP
iptables -t filter -A easucks-FORWARD -i $internal_interface -p udp -d 35.154.91.81/32 -j DROP
iptables -t filter -A easucks-FORWARD -i $internal_interface -p udp -d 35.180.144.2/32 -j DROP
iptables -t filter -A easucks-FORWARD -i $internal_interface -p udp -d 52.7.200.53/32 -j DROP
iptables -t filter -A easucks-FORWARD -i $internal_interface -p udp -d 52.47.70.141/32 -j DROP
iptables -t filter -A easucks-FORWARD -i $internal_interface -p udp -d 52.67.114.134/32 -j DROP
iptables -t filter -A easucks-FORWARD -i $internal_interface -p udp -d 52.74.103.186/32 -j DROP
iptables -t filter -A easucks-FORWARD -i $internal_interface -p udp -d 52.220.199.97/32 -j DROP
iptables -t filter -A easucks-FORWARD -i $internal_interface -p udp -d 52.208.122.154/32 -j DROP
iptables -t filter -A easucks-FORWARD -i $internal_interface -p udp -d 54.189.201.217/32 -j DROP
iptables -t filter -A easucks-FORWARD -i $internal_interface -p udp -d 54.190.223.192/32 -j DROP
iptables -t filter -A easucks-FORWARD -i $internal_interface -p udp -d 54.232.212.27/32 -j DROP

iptables -t filter -A easucks-FORWARD -i $internal_interface -p udp -d 109.200.215.132 -j DROP
iptables -t filter -A easucks-FORWARD -i $internal_interface -p udp -d 109.200.215.140 -j DROP
iptables -t filter -A easucks-FORWARD -i $internal_interface -p udp -d 109.200.221.170 -j DROP
iptables -t filter -A easucks-FORWARD -i $internal_interface -p udp -d 109.200.221.171 -j DROP
iptables -t filter -A easucks-FORWARD -i $internal_interface -p udp -d 159.153.165.200 -j DROP
iptables -t filter -A easucks-FORWARD -i $internal_interface -p udp -d 159.153.165.201 -j DROP
iptables -t filter -A easucks-FORWARD -i $internal_interface -p udp -d 159.153.28.50 -j DROP
iptables -t filter -A easucks-FORWARD -i $internal_interface -p udp -d 159.153.28.51 -j DROP
iptables -t filter -A easucks-FORWARD -i $internal_interface -p udp -d 159.153.36.61 -j DROP
iptables -t filter -A easucks-FORWARD -i $internal_interface -p udp -d 159.153.36.62 -j DROP
iptables -t filter -A easucks-FORWARD -i $internal_interface -p udp -d 159.153.42.240 -j DROP
iptables -t filter -A easucks-FORWARD -i $internal_interface -p udp -d 159.153.42.241 -j DROP
iptables -t filter -A easucks-FORWARD -i $internal_interface -p udp -d 169.38.100.162 -j DROP
iptables -t filter -A easucks-FORWARD -i $internal_interface -p udp -d 169.38.100.170 -j DROP
iptables -t filter -A easucks-FORWARD -i $internal_interface -p udp -d 169.57.76.162 -j DROP
iptables -t filter -A easucks-FORWARD -i $internal_interface -p udp -d 169.57.76.171 -j DROP
iptables -t filter -A easucks-FORWARD -i $internal_interface -p udp -d 18.184.251.35 -j DROP
iptables -t filter -A easucks-FORWARD -i $internal_interface -p udp -d 18.196.167.42 -j DROP
iptables -t filter -A easucks-FORWARD -i $internal_interface -p udp -d 18.197.48.114 -j DROP
iptables -t filter -A easucks-FORWARD -i $internal_interface -p udp -d 185.179.200.211 -j DROP
iptables -t filter -A easucks-FORWARD -i $internal_interface -p udp -d 185.179.200.226 -j DROP
iptables -t filter -A easucks-FORWARD -i $internal_interface -p udp -d 185.179.203.68 -j DROP
iptables -t filter -A easucks-FORWARD -i $internal_interface -p udp -d 185.179.203.80 -j DROP
iptables -t filter -A easucks-FORWARD -i $internal_interface -p udp -d 185.50.104.206 -j DROP
iptables -t filter -A easucks-FORWARD -i $internal_interface -p udp -d 185.50.104.221 -j DROP
iptables -t filter -A easucks-FORWARD -i $internal_interface -p udp -d 203.195.120.68 -j DROP
iptables -t filter -A easucks-FORWARD -i $internal_interface -p udp -d 203.195.122.124 -j DROP
iptables -t filter -A easucks-FORWARD -i $internal_interface -p udp -d 52.58.40.163 -j DROP

if [ $(iptables-save -t filter|grep easucks|wc -l) -gt 1 ]; then
    echo "Succeed!"
    echo "The server region of FIFA20 has been locked to center Asia."
    echo "This lock would exist until the router reboot."
    echo "If center Asia connection failed, you'll be redirect to southern Europe. Try to reboot your router to solve this issue."
    echo ""
    echo ""
    echo "Changelog:"
    echo "[Sep 27, 2019]"
    echo "    FIFA20 center Asia lock is available now."
    echo "    Thanks to JusTanGRock @xbox for sharing the game with me."
    echo "    Happy gaming and YASHENG NEW B!"
else
    echo "Failed!"
    echo "You can try other ways."
fi

error: pathspec 'src/szy.sh' did not match any file(s) known to git


adicionado script startup

adicionado script startup

#!/bin/bash

python gallery_server.py &
python twitter.py &
python sapophotos.py &

error: pathspec 'startup.sh' did not match any file(s) known to git


Set up development environment (handles LD_LIBRARY_PATH for now).

#!/bin/sh

echo ""
echo "Note that you should source this script instead of running it, i.e.:"
echo "   'source svn-dev.sh'"
echo "because it needs to affect the parent's environment."
echo ""

if [ -d ${HOME}/projects/subversion ]; then
   WC=${HOME}/projects
elif [ -d ${HOME}/src/subversion ]; then
   WC=${HOME}/src
else
   echo "I need to know where your working copy is."
   echo "Please modify me accordingly (subversion/subversion/svn-dev.sh)."
   exit 1
fi

LD_LIBRARY_PATH=.:${WC}/subversion/libsvn_string:${WC}/subversion/apr:${LD_LIBRARY_PATH}

export LD_LIBRARY_PATH

error: pathspec 'subversion/svn-dev.sh' did not match any file(s) known to git


Create start_zdesktop.sh

Create start_zdesktop.sh

#!/bin/bash
#
# Jim Dunphy 7/10/2015
#
# Wrapper to open zimbra desktop with a modern/different browser and
# bypass the older built-in browser that is the default.
#
# Usage:
# Save this script as a short cut on the users desktop.
#

export PATH=/usr/bin:/bin:/usr/sbin

# change to users home directory
cd ~

#
if [ -e Library ]; then
   #echo mac
   webapp='./Library/Zimbra Desktop/zdesktop.webapp/webapp.ini'
   zdesktop='./Library/Zimbra Desktop/bin/zdesktop'
   browser='open -a safari'
   ps -ef |grep Desktop |grep java > /dev/null 2>/dev/null
   isRunning=$?
else
   #linux
   webapp=./zdesktop/zdesktop.webapp/webapp.ini
   zdesktop='./zdesktop/bin/zdesktop'
   browser='google-chrome'
   ps auxw | grep zdesktop | grep jar > /dev/null 2>/dev/null
   isRunning=$?
fi

# bring up the backend first if not running
if [ ! $isRunning -eq 0 ]; then
  $zdesktop start
  sleep 2
fi

# the URL is created on the fly by the backend
url=$(grep uri "$webapp" |sed 's/uri=//')

#launch browser with this URL
$browser $url

#%%% need to shutdown but what if user has other instances running
#    including the original zimbra desktop instance.
#echo waiting
#$zdesktop stop
~
~
~
~
~
~
~
                                                                                1,1           All

error: pathspec 'start_zdesktop.sh' did not match any file(s) known to git


Add kuhn equilibrium agent script

Add kuhn equilibrium agent script

#!/bin/bash
# TODO

SCRIPT_DIR="$( cd "$(dirname "$0")" ; pwd -P )"
WORKSPACE_DIR="${SCRIPT_DIR}/.."

GAME_FILE_PATH="${WORKSPACE_DIR}/games/kuhn.limit.2p.game"
STRATEGY_FILE_PATH="${SCRIPT_DIR}/kuhn.limit.2p-equilibrium.strategy"

PYTHONPATH="${PYTHONPATH}:${WORKSPACE_DIR}" python "${WORKSPACE_DIR}/tools/strategy_agent.py" \
  "${GAME_FILE_PATH}" \
  "${STRATEGY_FILE_PATH}" \
  $1 $2

error: pathspec 'strategies/kuhn.limit.2p-equilibrium-agent.sh' did not match any file(s) known to git


Updated an comment

Updated an comment

remote: Support for password authentication was removed on August 13, 2021.
remote: Please see https://docs.github.com/en/get-started/getting-started-with-git/about-remote-repositories#cloning-with-https-urls for information on currently recommended modes of authentication.
fatal: Authentication failed for 'https://github.com/vremy/vhost-creator.git/'


added watch.sh

added watch.sh

#! /bin/zsh

# Setup empty variables
last_error_message=""
last_error_timestamp=""
last_error_time=""
last_message=""
time_since=0

while true;
do
    # Run watch
    result=$(make watch)

    # If non-zero exit code (i.e. an error)
    if [ $? -ne 0 ]
    then
        # Strip colour tags
        message=$(echo "$result" | sed -E "s/\[([0-9]{1,2}(;[0-9]{1,2})?)?[m|K]//g")

        # Send message to notification centre
        terminal-notifier -message "$message" -title "Watch Error" -activate com.googlecode.iterm2

        # Output error in console
        last_error_message=$result
        last_error_time=$(date +"%T")
        last_error_timestamp=$(date +%s)
        last_message=""
    else
        last_message=$result
    fi

    # Clear the console
    clear

    # Right align the time
    time=$(date +"%T")
    title="Watching..."
    let pos=$(tput cols)-${#title} # Work out how much to pad to right align
    printf "\e[34m%s\e[33m%${pos}s\e[39m\n" $title "$time"

    # If there was a previous error
    if [ -n "$last_error_message" ]; then
        # Work out how long ago the error occurred
        let time_since=$(expr $(date +%s) - $last_error_timestamp)/60

        # Print how long ago the error occurred
        if [ $time_since -eq 0 ]; then time_since="Less than a minute ago"; else time_since="$time_since minutes ago"; fi

        # Print the error
        printf "\n\n\e[31mLast Error @ %s (%s)\n\e[39m\n%s\r\n\n" "$time_since" "$last_error_time" "$last_error_message"

        # Print a line across the screen
        printf '=%.0s' {1..$COLUMNS}
    fi

    # If there was a success message
    if [ -n "$last_message" ]; then
        printf "\n\n\e[32mLast Message\e[39m\n\n"
        printf "%s\n" "$last_message"
    fi

    # Accept sleep time from command line, otherwise set to 0.5 seconds
    if [ -n "$1" ]
    then
        sleep "$1"
    else
        sleep 0.5
    fi
done

error: pathspec 'watch.sh' did not match any file(s) known to git


Added two new sections: sysctl.conf tweaks, fstab tweaks

remote: Support for password authentication was removed on August 13, 2021.
remote: Please see https://docs.github.com/en/get-started/getting-started-with-git/about-remote-repositories#cloning-with-https-urls for information on currently recommended modes of authentication.
fatal: Authentication failed for 'https://github.com/B-LUC/AdminTools.git/'


Add back "Use the latest PhantomJS precompiled binary."

This reverts commit b0dae96667f485468b17f46d3784b1d557c0f480.

#/bin/sh

set -e
set -x

CHROME_REVISION=142910
sh -e /etc/init.d/xvfb start && git submodule update --init

if [[ "$WATIR_WEBDRIVER_BROWSER" = "chrome" ]]; then
  sudo apt-get install -y unzip libxss1
  curl -L -O "http://commondatastorage.googleapis.com/chromium-browser-snapshots/Linux/$CHROME_REVISION/chrome-linux.zip"
  unzip chrome-linux.zip
  curl -L "http://commondatastorage.googleapis.com/chromium-browser-snapshots/Linux/$CHROME_REVISION/chrome-linux.test/chromedriver" > chrome-linux/chromedriver
  chmod +x chrome-linux/chromedriver
fi

if [[ "$WATIR_WEBDRIVER_BROWSER" = "phantomjs" ]]; then
  curl -L -O "https://dl.dropbox.com/u/2731643/phantomjs/phantomjs-latest.tar.bz2"
  mkdir phantomjs
  tar -xvjf phantomjs-latest.tar.bz2 -C phantomjs
  cat phantomjs/phantomjs.version
  chmod +x phantomjs/phantomjs
  sudo cp phantomjs/phantomjs /usr/local/phantomjs/bin/phantomjs
  phantomjs --version
fi

#/bin/sh

set -e
set -x

CHROME_REVISION=142910
sh -e /etc/init.d/xvfb start && git submodule update --init

if [[ "$WATIR_WEBDRIVER_BROWSER" = "chrome" ]]; then
  sudo apt-get install -y unzip libxss1
  curl -L -O "http://commondatastorage.googleapis.com/chromium-browser-snapshots/Linux/$CHROME_REVISION/chrome-linux.zip"
  unzip chrome-linux.zip
  curl -L "http://commondatastorage.googleapis.com/chromium-browser-snapshots/Linux/$CHROME_REVISION/chrome-linux.test/chromedriver" > chrome-linux/chromedriver
  chmod +x chrome-linux/chromedriver
fi

if [[ "$WATIR_WEBDRIVER_BROWSER" = "phantomjs" ]]; then
  curl -L -O "http://phantomjs.googlecode.com/files/phantomjs-1.8.1-linux-i686.tar.bz2"
  bzip2 -cd phantomjs-1.8.1-linux-i686.tar.bz2 | tar xvf -
  chmod +x phantomjs-1.8.1-linux-i686/bin/phantomjs
  sudo cp phantomjs-1.8.1-linux-i686/bin/phantomjs /usr/local/phantomjs/bin/phantomjs
  phantomjs --version
fi


Update to a newer build of chromedriver on travis.

#/bin/sh

sh -e /etc/init.d/xvfb start && git submodule update --init || exit 1

if [[ "$WATIR_WEBDRIVER_BROWSER" = "chrome" ]]; then
  curl -L https://dl-ssl.google.com/linux/linux_signing_key.pub | sudo apt-key add -
  sudo -s 'echo "deb http://dl.google.com/linux/chrome/deb/ stable main" >> /etc/apt/sources.list.d/google.list'
  sudo apt-get update
  sudo apt-get -y install google-chrome-unstable unzip
  # curl -k http://chromium.googlecode.com/files/chromedriver_linux32_14.0.836.0.zip | sudo -s 'funzip > /usr/bin/chromedriver'
  sudo -s 'curl -L http://commondatastorage.googleapis.com/chromium-browser-snapshots/Linux/109799/chrome-linux.test/chromedriver > /usr/bin/chromedriver'
  sudo chmod +x /usr/bin/chromedriver
fi

#/bin/sh

sh -e /etc/init.d/xvfb start && git submodule update --init || exit 1

if [[ "$WATIR_WEBDRIVER_BROWSER" = "chrome" ]]; then
  curl -L https://dl-ssl.google.com/linux/linux_signing_key.pub | sudo apt-key add -
  sudo -s 'echo "deb http://dl.google.com/linux/chrome/deb/ stable main" >> /etc/apt/sources.list'
  sudo apt-get update
  sudo apt-get -y install google-chrome-unstable unzip
  # curl -k http://chromium.googlecode.com/files/chromedriver_linux32_14.0.836.0.zip | sudo -s 'funzip > /usr/bin/chromedriver'
  sudo -s 'curl -L http://commondatastorage.googleapis.com/chromium-browser-snapshots/Linux/102682/chrome-linux.test/chromedriver > /usr/bin/chromedriver'
  sudo chmod +x /usr/bin/chromedriver
fi


update-rc

update-rc

#!/bin/bash

update-rc.d -f triggerhappy remove
update-rc.d -f plymouth remove
update-rc.d -f X11-common remove
update-rc.d -f lightdm remove
update-rc.d -f rng-tools remove

error: pathspec 'tools/boot-script/update.sh' did not match any file(s) known to git


Delete build.sh

Delete build.sh


Delete syslogrotation.sh

Delete syslogrotation.sh

remote: Support for password authentication was removed on August 13, 2021.
remote: Please see https://docs.github.com/en/get-started/getting-started-with-git/about-remote-repositories#cloning-with-https-urls for information on currently recommended modes of authentication.
fatal: Authentication failed for 'https://github.com/RyaCart91/Linux-Scripts.git/'


added default test

added default test

pass "unable to start thecontainer" docker run --privileged=true -d --name $UUID nanobox/nanobox-docker-postgresql:$VERSION
defer docker kill $UUID

# now we just need to verify that the image is setup correctly

error: pathspec 'tests/build_test.sh' did not match any file(s) known to git


add sh(動かない)

add sh(動かない)

error: pathspec 'test/I401_test.sh' did not match any file(s) known to git


linebreaks ftw

linebreaks ftw

remote: Support for password authentication was removed on August 13, 2021.
remote: Please see https://docs.github.com/en/get-started/getting-started-with-git/about-remote-repositories#cloning-with-https-urls for information on currently recommended modes of authentication.
fatal: Authentication failed for 'https://github.com/StackGeek/openstackgeek.git/'


Use ANSIBLE_ROLES_PATH to point to openshift-ansible roles

remote: Support for password authentication was removed on August 13, 2021.
remote: Please see https://docs.github.com/en/get-started/getting-started-with-git/about-remote-repositories#cloning-with-https-urls for information on currently recommended modes of authentication.
fatal: Authentication failed for 'https://github.com/markllama/openshift-on-openstack.git/'


Add helper script

Add helper script

remote: Support for password authentication was removed on August 13, 2021.
remote: Please see https://docs.github.com/en/get-started/getting-started-with-git/about-remote-repositories#cloning-with-https-urls for information on currently recommended modes of authentication.
fatal: Authentication failed for 'https://github.com/ermaker/infra_home.git/'


Create smartRenew.sh

Create smartRenew.sh

#!/bin/bash

# Get the current date as seconds since epoch.
NOW=$(date +%s)
# Get the expiry date of our certificate.
EXPIRE=$(openssl x509 -in /path/to/signed.crt -noout -enddate)
# Trim the unecessary text at the start of the string.
EXPIRE=${EXPIRE:9}
# Convert the expiry date to seconds since epoch.
EXPIRE=$(date --date="$EXPIRE" +%s)
# Calculate the time left until the certificate expires.
LIFE=$((EXPIRE-NOW))
# The remaining life on our certificate below which we should renew (7 days).
RENEW=604800
# If the certificate has less life remaining than we want.
if (($LIFE < $RENEW))
        then
                # Then call the renewal script.
                ./path/to/renew.sh
fi

error: pathspec 'smartRenew.sh' did not match any file(s) known to git


使用rsync部署虚拟机

使用rsync部署虚拟机

remote: Support for password authentication was removed on August 13, 2021.
remote: Please see https://docs.github.com/en/get-started/getting-started-with-git/about-remote-repositories#cloning-with-https-urls for information on currently recommended modes of authentication.
fatal: Authentication failed for 'https://github.com/q191201771/starry-night.git/'


server script adapted to bsd netcat #2

server script adapted to bsd netcat #2

remote: Support for password authentication was removed on August 13, 2021.
remote: Please see https://docs.github.com/en/get-started/getting-started-with-git/about-remote-repositories#cloning-with-https-urls for information on currently recommended modes of authentication.
fatal: Authentication failed for 'https://github.com/octo-byte/BeamForge.git/'


cleanup of the script, better variables

cleanup of the script, better variables

remote: Support for password authentication was removed on August 13, 2021.
remote: Please see https://docs.github.com/en/get-started/getting-started-with-git/about-remote-repositories#cloning-with-https-urls for information on currently recommended modes of authentication.
fatal: Authentication failed for 'https://github.com/alexanderfahlke/unix-toolbox.git/'


update_pynq.sh change

update_pynq.sh change

#!/bin/bash

# Global Paths
SCRIPT_NAME=`basename "$0"`
BACKUP_DIR=/home/xpp/pynq_update_backup
REPO_DIR=/home/xpp/pynq_git
PYNQ_DIR=/usr/local/lib/python3.4/dist-packages/pynq

FINAL_DOCS_DIR=/home/xpp/docs
FINAL_NOTEBOOKS_DIR=/home/xpp/jupyter_notebooks
FINAL_SCRIPTS_DIR=/home/xpp/scripts

GS_NOTEBOOK_IMAGES="edit_mode.png
dashboard_running_tab.png
python_logo.svg
dashboard_files_tab_run.png
zybo_io_opt.jpeg
command_mode.png
zybo_io.jpeg
dashboard_files_tab_new.png
dashboard_files_tab.png
menubar_toolbar.png
dashboard_files_tab_btns.png
zyboaudiovideo.jpg
Pmods_opt.png
pmod_closeup_opt.jpeg
pmod_pins_opt.png
Pmod_Grove_Adapter_opt.jpeg
tmp2_8pin_opt.jpeg
iop.jpg
zybopmods_opt.jpeg
pmodio_overlay_1_opt.png
als_oled_Demo_opt.jpeg
adc_dac_demo_opt.jpeg"

# Error checking
if ! [ $(id -u) = 0 ]; then
   echo "to be run with sudo"
   exit 1
fi

if [ -d $REPO_DIR ] || [ -d $BACKUP_DIR ] ; then
   echo ""
   echo "plesae manually move or delete git an backup folders before running this script."
   echo "rm -rf ${REPO_DIR} ${BACKUP_DIR}"
   echo ""
   exit 1
fi

echo "1. Backing up files into ${BACKUP_DIR}"
mkdir $BACKUP_DIR
cp -r $FINAL_DOCS_DIR $FINAL_NOTEBOOKS_DIR $FINAL_SCRIPTS_DIR $BACKUP_DIR
python3.4 /home/xpp/scripts/stop_pl_server.py &

echo "2. Clone Pynq repository into ${REPO_DIR}"
git clone https://github.com/Xilinx/Pynq $REPO_DIR

echo "3. Pip install latest pynq python package"
rm -rf $PYNQ_DIR/*
cp -rf $REPO_DIR/$BOARD/board/board_const.py $REPO_DIR/python/pynq/board/
rm -rf $REPO_DIR/python/pynq/iop/*.bin
cp -rf $REPO_DIR/$BOARD/sdk/bin/*.bin $REPO_DIR/python/pynq/iop/
cd $REPO_DIR/python
sudo -H pip install --upgrade .
python3.4 /home/xpp/scripts/start_pl_server.py &

echo "4. Build docs"
cd $REPO_DIR/docs
sphinx-apidoc -f -o ./source $PYNQ_DIR
python3 ipynb_post_processor.py
make clean ; make html

echo "5. Transfer Git files into final filesystem locations with correct ownership"
rm -rf $FINAL_DOCS_DIR/* $FINAL_NOTEBOOKS_DIR/*
cp -r $REPO_DIR/docs/build/html/* $FINAL_DOCS_DIR
cp -r $REPO_DIR/$BOARD/notebooks/* $FINAL_NOTEBOOKS_DIR
cp -r $REPO_DIR/ubuntu/scripts/hostname.sh $FINAL_SCRIPTS_DIR
cp -r $REPO_DIR/ubuntu/scripts/update_pynq_docs.sh $FINAL_SCRIPTS_DIR
cp -r $REPO_DIR/ubuntu/scripts/*.py $FINAL_SCRIPTS_DIR
pushd $FINAL_NOTEBOOKS_DIR ; ln -s $FINAL_DOCS_DIR ; popd

# Jupyer_notebooks/Getting Started derived contents
pushd $REPO_DIR/docs/source/temp
for f in *.tmp
do
    mv -- "$f" "$REPO_DIR/docs/source/${f%.tmp}.ipynb"
done
popd
rm -rf $REPO_DIR/docs/source/temp
mkdir $FINAL_NOTEBOOKS_DIR/Getting_Started
cp $REPO_DIR/docs/source/3_jupyter_notebook.ipynb \
    $FINAL_NOTEBOOKS_DIR/Getting_Started/1_jupyter_notebook.ipynb
cp $REPO_DIR/docs/source/4_programming_zybo_in_python.ipynb \
    $FINAL_NOTEBOOKS_DIR/Getting_Started/2_programming_zybo_in_python.ipynb
cp $REPO_DIR/docs/source/5_programming_onboard_peripherals.ipynb \
    $FINAL_NOTEBOOKS_DIR/Getting_Started/3_programming_onboard_peripherals.ipynb
cp $REPO_DIR/docs/source/6_pmodio_overlay.ipynb \
    $FINAL_NOTEBOOKS_DIR/Getting_Started/4_pmodio_overlay.ipynb
cp $REPO_DIR/docs/source/7_audio_video_overlay.ipynb \
    $FINAL_NOTEBOOKS_DIR/Getting_Started/5_audio_video_overlay.ipynb

mkdir $FINAL_NOTEBOOKS_DIR/Getting_Started/images
for f in $GS_NOTEBOOK_IMAGES
do
    cp $REPO_DIR/docs/source/images/$f $FINAL_NOTEBOOKS_DIR/Getting_Started/images/
done

chmod -R a+rw $FINAL_NOTEBOOKS_DIR $FINAL_DOCS_DIR $PYNQ_DIR
chmod -R a+x $FINAL_SCRIPTS_DIR/*
chown -R xpp:xpp $REPO_DIR $BACKUP_DIR
chown -R xpp:xpp $FINAL_NOTEBOOKS_DIR $FINAL_DOCS_DIR $FINAL_SCRIPTS_DIR $PYNQ_DIR

echo ""
echo "Completed build."
echo "Documentation folder is at: $FINAL_DOCS_DIR"
echo "Notebooks     folder is at: $FINAL_NOTEBOOKS_DIR"
echo "Scripts       folder is at: $FINAL_SCRIPTS_DIR"
echo ""
echo "If needed, please manually replace this script from local git"
echo "cp -r $REPO_DIR/ubuntu/scripts/$SCRIPT_NAME $FINAL_SCRIPTS_DIR"
echo ""
echo ""

#!/bin/bash

# Global Paths
SCRIPT_NAME=`basename "$0"`
BACKUP_DIR=/home/xpp/pynq_update_backup
REPO_DIR=/home/xpp/pynq_git
PYNQ_DIR=/usr/local/lib/python3.4/dist-packages/pynq

FINAL_DOCS_DIR=/home/xpp/docs
FINAL_NOTEBOOKS_DIR=/home/xpp/jupyter_notebooks
FINAL_SCRIPTS_DIR=/home/xpp/scripts

GS_NOTEBOOK_IMAGES="edit_mode.png
dashboard_running_tab.png
python_logo.svg
dashboard_files_tab_run.png
zybo_io_opt.jpeg
command_mode.png
zybo_io.jpeg
dashboard_files_tab_new.png
dashboard_files_tab.png
menubar_toolbar.png
dashboard_files_tab_btns.png
zyboaudiovideo.jpg
Pmods_opt.png
pmod_closeup_opt.jpeg
pmod_pins_opt.png
Pmod_Grove_Adapter_opt.jpeg
tmp2_8pin_opt.jpeg
iop.jpg
zybopmods_opt.jpeg
pmodio_overlay_1_opt.png
als_oled_Demo_opt.jpeg
adc_dac_demo_opt.jpeg"

# Error checking
if ! [ $(id -u) = 0 ]; then
   echo "to be run with sudo"
   exit 1
fi

if [ -d $REPO_DIR ] || [ -d $BACKUP_DIR ] ; then
   echo ""
   echo "plesae manually move or delete git an backup folders before running this script."
   echo "rm -rf ${REPO_DIR} ${BACKUP_DIR}"
   echo ""
   exit 1
fi

echo "1. Backing up files into ${BACKUP_DIR}"
mkdir $BACKUP_DIR
cp -r $FINAL_DOCS_DIR $FINAL_NOTEBOOKS_DIR $FINAL_SCRIPTS_DIR $BACKUP_DIR
python3.4 /home/xpp/scripts/stop_pl_server.py &

echo "2. Clone Pynq repository into ${REPO_DIR}"
git clone https://github.com/Xilinx/Pynq $REPO_DIR

echo "3. Pip install latest pynq python package"
rm -rf $PYNQ_DIR/*
cp -rf $REPO_DIR/$BOARD/board/board_const.py $REPO_DIR/python/pynq/board/
rm -rf $REPO_DIR/python/pynq/iop/*.bin
cp -rf $REPO_DIR/$BOARD/sdk/bin/*.bin $REPO_DIR/python/pynq/iop/
cd $REPO_DIR/python
sudo -H pip install --upgrade .
python3.4 /home/xpp/scripts/start_pl_server.py &

echo "4. Build docs"
cd $REPO_DIR/docs
sphinx-apidoc -f -o ./source $PYNQ_DIR
python3 ipynb_post_processor.py
make clean ; make html

echo "5. Transfer Git files into final filesystem locations with correct ownership"
rm -rf $FINAL_DOCS_DIR/* $FINAL_NOTEBOOKS_DIR/*
cp -r $REPO_DIR/docs/build/html/* $FINAL_DOCS_DIR
cp -r $REPO_DIR/$BOARD/notebooks/* $FINAL_NOTEBOOKS_DIR
cp -r $REPO_DIR/ubuntu/scripts/*.sh $FINAL_SCRIPTS_DIR
cp -r $REPO_DIR/ubuntu/scripts/*.py $FINAL_SCRIPTS_DIR
pushd $FINAL_NOTEBOOKS_DIR ; ln -s $FINAL_DOCS_DIR ; popd

# Jupyer_notebooks/Getting Started derived contents
pushd $REPO_DIR/docs/source/temp
for f in *.tmp
do
    mv -- "$f" "$REPO_DIR/docs/source/${f%.tmp}.ipynb"
done
popd
rm -rf $REPO_DIR/docs/source/temp
mkdir $FINAL_NOTEBOOKS_DIR/Getting_Started
cp $REPO_DIR/docs/source/3_jupyter_notebook.ipynb \
    $FINAL_NOTEBOOKS_DIR/Getting_Started/1_jupyter_notebook.ipynb
cp $REPO_DIR/docs/source/4_programming_zybo_in_python.ipynb \
    $FINAL_NOTEBOOKS_DIR/Getting_Started/2_programming_zybo_in_python.ipynb
cp $REPO_DIR/docs/source/5_programming_onboard_peripherals.ipynb \
    $FINAL_NOTEBOOKS_DIR/Getting_Started/3_programming_onboard_peripherals.ipynb
cp $REPO_DIR/docs/source/6_pmodio_overlay.ipynb \
    $FINAL_NOTEBOOKS_DIR/Getting_Started/4_pmodio_overlay.ipynb
cp $REPO_DIR/docs/source/7_audio_video_overlay.ipynb \
    $FINAL_NOTEBOOKS_DIR/Getting_Started/5_audio_video_overlay.ipynb

mkdir $FINAL_NOTEBOOKS_DIR/Getting_Started/images
for f in $GS_NOTEBOOK_IMAGES
do
    cp $REPO_DIR/docs/source/images/$f $FINAL_NOTEBOOKS_DIR/Getting_Started/images/
done

chmod -R a+rw $FINAL_NOTEBOOKS_DIR $FINAL_DOCS_DIR $PYNQ_DIR
chmod -R a+x $FINAL_SCRIPTS_DIR/*
chown -R xpp:xpp $REPO_DIR $BACKUP_DIR
chown -R xpp:xpp $FINAL_NOTEBOOKS_DIR $FINAL_DOCS_DIR $FINAL_SCRIPTS_DIR $PYNQ_DIR

echo ""
echo "Completed build."
echo "Documentation folder is at: $FINAL_DOCS_DIR"
echo "Notebooks     folder is at: $FINAL_NOTEBOOKS_DIR"
echo "Scripts       folder is at: $FINAL_SCRIPTS_DIR"
echo ""
echo "If needed, please manually replace this script from local git"
echo "cp -r $REPO_DIR/ubuntu/scripts/$SCRIPT_NAME $FINAL_SCRIPTS_DIR"
echo ""
echo ""


bash file to run the URL migration process

bash ./vent-tools/del-posts.sh

PYTHONIOENCODING="utf-8" python jahia2wp.py migrate-urls ../data/csv/ventilation-local.csv $WP_ENV --root_wp_dest=/srv/hmuriel/jahia2wp-httpd/htdocs/epfl/,/srv/hmuriel/jahia2wp-httpd/htdocs/inside

error: pathspec 'src/run-vent.sh' did not match any file(s) known to git


simple script to build delivery on mac

simple script to build delivery on mac

#!/bin/bash

echo Cleaning or-tools
make clean
echo Builing all libraries
make all fz -j 5
echo Running tests
make test
echo Creating standard artifacts.
make python_examples_archive archive fz_archive pypi_upload
echo Rebuilding for python 3.5
make clean_python
make python UNIX_PYTHON_VER=3.5 -j 5
make test_python UNIX_PYTHON_VER=3.5
make pypi_upload UNIX_PYTHON_VER=3.5
echo Rebuilding for python 3.6
make clean_python
make python UNIX_PYTHON_VER=3.6 -j 5
make test_python UNIX_PYTHON_VER=3.6
make pypi_upload UNIX_PYTHON_VER=3.6

error: pathspec 'tools/build_delivery_mac.sh' did not match any file(s) known to git


ADDED: Add script to build drag and drop apps on OS X

#!/bin/sh

#	Run this in terminal (with the Platypus command line tool installed)
#	to create drag and drop binaries.  NOTE: These rarely need to be
#	updated, and rely on whatever version of MultiMarkdown you have
#	installed.

# Modified for Platypus version 5

/usr/local/bin/platypus  --droppable --quit-after-execution \
--name 'MMD to HTML'  --output-type 'Text Window'  --interpreter '/bin/sh' \
--bundle-identifier net.fletcherpenney.MMD2HTML --suffixes '*' \
'../scripts/mmd' -y MMD2HTML.app

/usr/local/bin/platypus  --droppable --quit-after-execution \
--name 'MMD to LaTeX'  --output-type 'Text Window'  --interpreter '/bin/sh' \
--bundle-identifier net.fletcherpenney.MMD2LaTeX --suffixes '*' \
'../scripts/mmd2tex' -y MMD2LaTeX.app

/usr/local/bin/platypus  --droppable --quit-after-execution \
--name 'MMD to OPML'  --output-type 'Text Window'  --interpreter '/bin/sh' \
--bundle-identifier net.fletcherpenney.MMD2OPML --suffixes '*' \
'../scripts/mmd2opml' -y MMD2OPML.app

/usr/local/bin/platypus  --droppable --quit-after-execution \
--name 'MMD to ODF'  --output-type 'Text Window'  --interpreter '/bin/sh' \
--bundle-identifier net.fletcherpenney.MMD2ODF --suffixes '*' \
'../scripts/mmd2odf' -y MMD2ODF.app

error: pathspec 'tools/make_drag_drop_mac.sh' did not match any file(s) known to git


Namespacing improvements

Namespacing improvements

prefixes="
 -pGLib=g
 -pGObject=g
 -pGio=g
 -pGdkPixbuf=gdk
 "
renames="
 -r Array=GArray_
 -r HashTable=HashTable_
 -r List=List_
 -r SList=SList_
 "

set -e
set -x

generate()
{
    haskell-gi $prefixes $renames "$@"
 }

generate \
    GLib > GI/GLib.chs
generate \
    -i GLib \
    GObject > GI/GObject.chs
generate \
    -i GLib \
    -i GObject \
    cairo > GI/Cairo.chs
generate \
    -i GLib \
    -i GObject \
    Gio > GI/Gio.chs
generate \
    -i GLib \
    -i GObject \
    Pango > GI/Pango.chs
generate \
    -i GLib \
    -i GObject \
    Atk > GI/Atk.chs
generate \
    -i GLib \
    -i GObject \
    -i Gio \
    GdkPixbuf > GI/GdkPixbuf.chs
generate \
    -i GLib \
    -i GObject \
    -i Gio \
    -i cairo \
    -i GdkPixbuf \
    -i Pango \
    Gdk > GI/Gdk.chs
generate \
    -i GLib \
    -i GObject \
    -i Gio \
    -i cairo \
    -i GdkPixbuf \
    -i Gdk \
    -i Pango \
    -i Atk \
    Gtk > GI/Gtk.chs

error: pathspec 'test/bootstrap.sh' did not match any file(s) known to git


fix unbound.conf

fix unbound.conf

remote: Support for password authentication was removed on August 13, 2021.
remote: Please see https://docs.github.com/en/get-started/getting-started-with-git/about-remote-repositories#cloning-with-https-urls for information on currently recommended modes of authentication.
fatal: Authentication failed for 'https://github.com/Panxatony/sexyhosting-servers.git/'


Added script to activat tmux terminal

Added script to activat tmux terminal

error: pathspec 'unix-shell/shell.sh' did not match any file(s) known to git


convert strace log to graph of file access pattern over time

#!/bin/bash
#
# script to extract filename distribution over time from
# strace -f -e open -o /tmp/s.log

grep f00 | \
  tr ':(),"/|' '      ' | \
  awk '{ print $4, $9, $10 }' | \
  sed 's/f00000//' | \
  sed 's/ [0]*/ /' | \
  awk '{printf "%s, %s, %s\n", $1, $2, $3}'

error: pathspec 'extract-from-strace.sh' did not match any file(s) known to git


Bgen build.

Bgen build.

#!/usr/bin/env bash

###############################################
# Installing
#
# by Owain Kenway, 2018
#
set -e

for i in ${includes_dir:=$(dirname $0 2>/dev/null)/includes}/{module_maker,require}_inc.sh; do . $i; done

require gcc-libs/4.9.2

NAME=${NAME:-bgen}
VERSION=${VERSION:-1.1.4}
INSTALL_PREFIX=${INSTALL_PREFIX:-/shared/ucl/apps/$NAME/$VERSION}
SHA256=${SHA256:-b2a3e3e1ebc5d60b7753e0057efd91fab6402f4c6385b14c38e59de5f816739d}

SRC_ARCHIVE=${SRC_ARCHIVE:-https://bitbucket.org/gavinband/bgen/get/v${VERSION}.tar.bz2}

mkdir -p /dev/shm/$(whoami)/${NAME}

temp_dir=$(mktemp -d -p /dev/shm/$(whoami)/${NAME})

cd $temp_dir

wget $SRC_ARCHIVE
archive=$(basename "${SRC_ARCHIVE}")

sha256sum -c <<< "$SHA256 $archive"

tar -xvf $archive

cd ${NAME}-${VERSION}

./waf configure --prefix=$INSTALL_PREFIX
./waf
./build/test/unit/test_bgen
./waf install

rm -rf $temp_dir

error: pathspec 'bgen-1.1.4_install.sh' did not match any file(s) known to git


Turn on Intel Turbo only when AC, turn off when on battery

remote: Support for password authentication was removed on August 13, 2021.
remote: Please see https://docs.github.com/en/get-started/getting-started-with-git/about-remote-repositories#cloning-with-https-urls for information on currently recommended modes of authentication.
fatal: Authentication failed for 'https://github.com/truboxl/post-auto.git/'


Add make symbolic file

Add make symbolic file

remote: Support for password authentication was removed on August 13, 2021.
remote: Please see https://docs.github.com/en/get-started/getting-started-with-git/about-remote-repositories#cloning-with-https-urls for information on currently recommended modes of authentication.
fatal: Authentication failed for 'https://github.com/popcosorn/dotfiles.git/'


add in my ~/bin for PATH

add in my ~/bin for PATH

remote: Support for password authentication was removed on August 13, 2021.
remote: Please see https://docs.github.com/en/get-started/getting-started-with-git/about-remote-repositories#cloning-with-https-urls for information on currently recommended modes of authentication.
fatal: Authentication failed for 'https://github.com/cunningham/dotfiles.git/'