Print Python version and specific environment in ci_build.sh

This is useful to determine exactly which packages are installed in the ci_sanity environment.

PiperOrigin-RevId: 373029910
Change-Id: I2912a02f5eb9a02dcec1765bc68245070a93312d

#!/usr/bin/env bash
# Copyright 2016 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
#
# Usage: ci_sanity.sh [--pep8] [--incremental] [bazel flags]
#
# Options:
#           run sanity checks: python 2&3 pylint checks and bazel nobuild
#  --pep8   run pep8 test only
#  --pylint run pylint check only
#  --incremental  Performs checks incrementally, by using the files changed in
#                 the latest commit

# Current script directory
SCRIPT_DIR=$( cd ${0%/*} && pwd -P )
source "${SCRIPT_DIR}/builds/builds_common.sh"

ROOT_DIR=$( cd "$SCRIPT_DIR/../../.." && pwd -P )

# Helper functions
die() {
  echo $@
  exit 1
}

num_cpus() {
  # Get the number of CPUs
  if [[ -f /proc/cpuinfo ]]; then
    N_CPUS=$(grep -c ^processor /proc/cpuinfo)
  else
    # Fallback method
    N_CPUS=`getconf _NPROCESSORS_ONLN`
  fi
  if [[ -z ${N_CPUS} ]]; then
    die "ERROR: Unable to determine the number of CPUs"
  fi

  echo ${N_CPUS}
}

# Helper functions for examining changed files in the last non-merge git
# commit.

# Get the hash of the last non-merge git commit on the current branch.
# Usage: get_last_non_merge_git_commit
get_last_non_merge_git_commit() {
  git rev-list --no-merges -n 1 HEAD
}

# List files changed (i.e., added, removed or revised) in the last non-merge
# git commit.
# Usage: get_changed_files_in_last_non_merge_git_commit
get_changed_files_in_last_non_merge_git_commit() {
  git diff-tree --no-commit-id --name-only -r $(get_last_non_merge_git_commit)
}

# List Python files changed in the last non-merge git commit that still exist,
# i.e., not removed.
# Usage: get_py_files_to_check [--incremental]
get_py_files_to_check() {
  if [[ "$1" == "--incremental" ]]; then
    CHANGED_PY_FILES=$(get_changed_files_in_last_non_merge_git_commit | \
                       grep '.*\.py$')

    # Do not include files removed in the last non-merge commit.
    PY_FILES=""
    for PY_FILE in ${CHANGED_PY_FILES}; do
      if [[ -f "${PY_FILE}" ]]; then
        PY_FILES="${PY_FILES} ${PY_FILE}"
      fi
    done

    echo "${PY_FILES}"
  else
    find tensorflow -name '*.py'
  fi
}

# Subfunctions for substeps
# Run pylint
do_pylint() {
  # Usage: do_pylint [--incremental]
  #
  # Options:
  #   --incremental  Performs check only if there are python files that changed
  #                  since last non-merge git commit. We always check all Python
  #                  files if one changed to capture the case when a function
  #                  signature changes affects unchanged files.

  # Validate arguments, see if we can do no work
  if [[ $# == 1 ]] && [[ "$1" == "--incremental" ]]; then
    PYTHON_SRC_FILES=$(get_py_files_to_check --incremental)

    if [[ -z "${PYTHON_SRC_FILES}" ]]; then
      echo "do_pylint will NOT run due to --incremental flag and due to the "\
"absence of Python code changes in the last commit."
      return 0
    fi
  elif [[ $# != 0 ]]; then
    echo "Invalid syntax for invoking do_pylint"
    echo "Usage: do_pylint [--incremental]"
    return 1
  else
    # Get all Python files, regardless of mode.
    PYTHON_SRC_FILES=$(get_py_files_to_check)
  fi

  # Something happened. TF no longer has Python code if this branch is taken
  if [[ -z ${PYTHON_SRC_FILES} ]]; then
    echo "do_pylint found no Python files to check. Returning."
    return 0
  fi

  # Now that we know we have to do work, check if `pylint` is installed
  PYLINT_BIN="python3.8 -m pylint"

  echo ""
  echo "print python version and pip freeze for debugging."
  echo ""
  python3.8
  python3.8 -m pip freeze

  echo ""
  echo "check whether pylint is available or not."
  echo ""
  ${PYLINT_BIN} --version
  if [[ $? -eq 0 ]]
  then
    echo ""
    echo "pylint available, proceeding with pylint sanity check."
    echo ""
  else
    echo ""
    echo "pylint not available."
    echo ""
    return 1
  fi

  # Configure pylint using the following file
  PYLINTRC_FILE="${SCRIPT_DIR}/pylintrc"

  if [[ ! -f "${PYLINTRC_FILE}" ]]; then
    die "ERROR: Cannot find pylint rc file at ${PYLINTRC_FILE}"
  fi

  # Run pylint in parallel, after some disk setup
  NUM_SRC_FILES=$(echo ${PYTHON_SRC_FILES} | wc -w)
  NUM_CPUS=$(num_cpus)

  echo "Running pylint on ${NUM_SRC_FILES} files with ${NUM_CPUS} "\
"parallel jobs..."
  echo ""

  PYLINT_START_TIME=$(date +'%s')
  OUTPUT_FILE="$(mktemp)_pylint_output.log"
  ERRORS_FILE="$(mktemp)_pylint_errors.log"
  PERMIT_FILE="$(mktemp)_pylint_permit.log"
  FORBID_FILE="$(mktemp)_pylint_forbid.log"

  rm -rf ${OUTPUT_FILE}
  rm -rf ${ERRORS_FILE}
  rm -rf ${PERMIT_FILE}
  rm -rf ${FORBID_FILE}

  # When running, filter to only contain the error code lines. Removes module
  # header, removes lines of context that show up from some lines.
  # Also, don't redirect stderr as this would hide pylint fatal errors.
  ${PYLINT_BIN} --rcfile="${PYLINTRC_FILE}" --output-format=parseable \
      --jobs=${NUM_CPUS} ${PYTHON_SRC_FILES} | grep '\[[CEFW]' > ${OUTPUT_FILE}
  PYLINT_END_TIME=$(date +'%s')

  echo ""
  echo "pylint took $((PYLINT_END_TIME - PYLINT_START_TIME)) s"
  echo ""

  # Report only what we care about
  # Ref https://pylint.readthedocs.io/en/latest/technical_reference/features.html
  # E: all errors
  # W0311 bad-indentation
  # W0312 mixed-indentation
  # C0330 bad-continuation
  # C0301 line-too-long
  # C0326 bad-whitespace
  # W0611 unused-import
  # W0622 redefined-builtin
  grep -E '(\[E|\[W0311|\[W0312|\[C0330|\[C0301|\[C0326|\[W0611|\[W0622)' ${OUTPUT_FILE} > ${ERRORS_FILE}

  # Split the pylint reported errors into permitted ones and those we want to
  # block submit on until fixed.
  # We use `${ALLOW_LIST_FILE}` to record the errors we temporarily accept. Goal
  # is to make that file only contain errors caused by difference between
  # internal and external versions.
  ALLOW_LIST_FILE="${SCRIPT_DIR}/pylint_allowlist"

  if [[ ! -f "${ALLOW_LIST_FILE}" ]]; then
    die "ERROR: Cannot find pylint allowlist file at ${ALLOW_LIST_FILE}"
  fi

  # We can split with just 2 grep invocations
  grep    -f ${ALLOW_LIST_FILE} ${ERRORS_FILE} > ${PERMIT_FILE}
  grep -v -f ${ALLOW_LIST_FILE} ${ERRORS_FILE} > ${FORBID_FILE}

  # Determine counts of errors
  N_PERMIT_ERRORS=$(wc -l ${PERMIT_FILE} | cut -d' ' -f1)
  N_FORBID_ERRORS=$(wc -l ${FORBID_FILE} | cut -d' ' -f1)

  # First print all allowed errors
  echo ""
  if [[ ${N_PERMIT_ERRORS} != 0 ]]; then
    echo "Found ${N_PERMIT_ERRORS} allowlisted pylint errors:"
    cat ${PERMIT_FILE}
  fi

  # Now, print the errors we should fix
  echo ""
  if [[ ${N_FORBID_ERRORS} != 0 ]]; then
    echo "Found ${N_FORBID_ERRORS} non-allowlisted pylint errors:"
    cat ${FORBID_FILE}
  fi

  echo ""
  if [[ ${N_FORBID_ERRORS} != 0 ]]; then
    echo "FAIL: Found ${N_FORBID_ERRORS} non-allowlisted errors and ${N_PERMIT_ERRORS} allowlisted errors"
    return 1
  else
    echo "PASS: Found only ${N_PERMIT_ERRORS} allowlisted errors"
  fi
}

# Run pep8 check
do_pep8() {
  # Usage: do_pep8 [--incremental]
  # Options:
  #   --incremental  Performs check on only the python files changed in the
  #                  last non-merge git commit.

  PEP8_BIN="/usr/local/bin/pep8"
  PEP8_CONFIG_FILE="${SCRIPT_DIR}/pep8"

  if [[ "$1" == "--incremental" ]]; then
    PYTHON_SRC_FILES=$(get_py_files_to_check --incremental)
    NUM_PYTHON_SRC_FILES=$(echo ${PYTHON_SRC_FILES} | wc -w)

    echo "do_pep8 will perform checks on only the ${NUM_PYTHON_SRC_FILES} "\
"Python file(s) changed in the last non-merge git commit due to the "\
"--incremental flag:"
    echo "${PYTHON_SRC_FILES}"
    echo ""
  else
    PYTHON_SRC_FILES=$(get_py_files_to_check)
  fi

  if [[ -z ${PYTHON_SRC_FILES} ]]; then
    echo "do_pep8 found no Python files to check. Returning."
    return 0
  fi

  if [[ ! -f "${PEP8_CONFIG_FILE}" ]]; then
    die "ERROR: Cannot find pep8 config file at ${PEP8_CONFIG_FILE}"
  fi
  echo "See \"${PEP8_CONFIG_FILE}\" for pep8 config( e.g., ignored errors)"

  NUM_SRC_FILES=$(echo ${PYTHON_SRC_FILES} | wc -w)

  echo "Running pep8 on ${NUM_SRC_FILES} files"
  echo ""

  PEP8_START_TIME=$(date +'%s')
  PEP8_OUTPUT_FILE="$(mktemp)_pep8_output.log"

  rm -rf ${PEP8_OUTPUT_FILE}

  ${PEP8_BIN} --config="${PEP8_CONFIG_FILE}" --statistics \
      ${PYTHON_SRC_FILES} 2>&1 | tee ${PEP8_OUTPUT_FILE}
  PEP8_END_TIME=$(date +'%s')

  echo ""
  echo "pep8 took $((PEP8_END_TIME - PEP8_START_TIME)) s"
  echo ""

  if [[ -s ${PEP8_OUTPUT_FILE} ]]; then
    echo "FAIL: pep8 found above errors and/or warnings."
    return 1
  else
    echo "PASS: No pep8 errors or warnings were found"
    return 0
  fi
}

do_buildifier(){
  BUILD_FILES=$(find tensorflow -name 'BUILD*')
  NUM_BUILD_FILES=$(echo ${BUILD_FILES} | wc -w)

  echo "Running do_buildifier on ${NUM_BUILD_FILES} files"
  echo ""

  BUILDIFIER_START_TIME=$(date +'%s')
  BUILDIFIER_OUTPUT_FILE="$(mktemp)_buildifier_output.log"

  rm -rf ${BUILDIFIER_OUTPUT_FILE}

  buildifier -v -mode=check \
    ${BUILD_FILES} 2>&1 | tee ${BUILDIFIER_OUTPUT_FILE}
  BUILDIFIER_END_TIME=$(date +'%s')

  echo ""
  echo "buildifier took $((BUILDIFIER_END_TIME - BUILDIFIER_START_TIME)) s"
  echo ""

  if [[ -s ${BUILDIFIER_OUTPUT_FILE} ]]; then
    echo "FAIL: buildifier found errors and/or warnings in above BUILD files."
    echo "buildifier suggested the following changes:"
    buildifier -v -mode=diff -diff_command=diff ${BUILD_FILES}
    echo "Please fix manually or run buildifier <file> to auto-fix."
    return 1
  else
    echo "PASS: No buildifier errors or warnings were found"
    return 0
  fi
}

do_external_licenses_check(){
  BUILD_TARGET="$1"
  LICENSES_TARGET="$2"

  EXTERNAL_LICENSES_CHECK_START_TIME=$(date +'%s')

  EXTERNAL_DEPENDENCIES_FILE="$(mktemp)_external_dependencies.log"
  LICENSES_FILE="$(mktemp)_licenses.log"
  MISSING_LICENSES_FILE="$(mktemp)_missing_licenses.log"
  EXTRA_LICENSES_FILE="$(mktemp)_extra_licenses.log"
  TMP_FILE="$(mktemp)_tmp.log"

  echo "Getting external dependencies for ${BUILD_TARGET}"
 bazel cquery "attr('licenses', 'notice', deps(${BUILD_TARGET}))" --keep_going > "${TMP_FILE}" 2>&1
 cat "${TMP_FILE}" \
  | grep -e "^\/\/" -e "^@" \
  | grep -E -v "^//tensorflow" \
  | sed -e 's|:.*||' \
  | sort \
  | uniq 2>&1 \
  | tee ${EXTERNAL_DEPENDENCIES_FILE}

  echo
  echo "Getting list of external licenses mentioned in ${LICENSES_TARGET}."
  bazel cquery "deps(${LICENSES_TARGET})" --keep_going > "${TMP_FILE}" 2>&1
 cat "${TMP_FILE}" \
  | grep -e "^\/\/" -e "^@" \
  | grep -E -v "^//tensorflow" \
  | sed -e 's|:.*||' \
  | sort \
  | uniq 2>&1 \
  | tee ${LICENSES_FILE}

  echo
  comm -1 -3 ${EXTERNAL_DEPENDENCIES_FILE}  ${LICENSES_FILE} 2>&1 | tee ${EXTRA_LICENSES_FILE}
  echo
  comm -2 -3 ${EXTERNAL_DEPENDENCIES_FILE}  ${LICENSES_FILE} 2>&1 | tee ${MISSING_LICENSES_FILE}

  EXTERNAL_LICENSES_CHECK_END_TIME=$(date +'%s')

  # Denylist
  echo ${MISSING_LICENSES_FILE}
  grep \
    -e "@bazel_tools//platforms" \
    -e "@bazel_tools//third_party/" \
    -e "@bazel_tools//tools" \
    -e "@local" \
    -e "@com_google_absl//absl" \
    -e "@org_tensorflow//" \
    -e "@com_github_googlecloudplatform_google_cloud_cpp//google" \
    -e "@com_github_grpc_grpc//src/compiler" \
    -e "@platforms//os" \
    -e "@ruy//" \
    -v ${MISSING_LICENSES_FILE} > temp.txt
  mv temp.txt ${MISSING_LICENSES_FILE}

  # Allowlist
  echo ${EXTRA_LICENSE_FILE}
  grep \
    -e "//third_party/mkl" \
    -e "//third_party/mkl_dnn" \
    -e "@bazel_tools//src" \
    -e "@bazel_tools//platforms" \
    -e "@bazel_tools//tools/" \
    -e "@org_tensorflow//tensorflow" \
    -e "@com_google_absl//" \
    -e "//external" \
    -e "@local" \
    -e "@com_github_googlecloudplatform_google_cloud_cpp//" \
    -e "@embedded_jdk//" \
    -e "^//$" \
    -e "@ruy//" \
    -v ${EXTRA_LICENSES_FILE} > temp.txt
  mv temp.txt ${EXTRA_LICENSES_FILE}

  echo
  echo "do_external_licenses_check took $((EXTERNAL_LICENSES_CHECK_END_TIME - EXTERNAL_LICENSES_CHECK_START_TIME)) s"
  echo

  if [[ -s ${MISSING_LICENSES_FILE} ]] || [[ -s ${EXTRA_LICENSES_FILE} ]] ; then
    echo "FAIL: mismatch in packaged licenses and external dependencies"
    if [[ -s ${MISSING_LICENSES_FILE} ]] ; then
      echo "Missing the licenses for the following external dependencies:"
      cat ${MISSING_LICENSES_FILE}
      echo "Please add the license(s) to ${LICENSES_TARGET}."
    fi
    if [[ -s ${EXTRA_LICENSES_FILE} ]] ; then
      echo "Please remove the licenses for the following external dependencies from target ${LICENSES_TARGET}."
      cat ${EXTRA_LICENSES_FILE}
    fi
    rm -rf ${EXTERNAL_DEPENDENCIES_FILE}
    rm -rf ${LICENSES_FILE}
    rm -rf ${MISSING_LICENSES_FILE}
    rm -rf ${EXTRA_LICENSES_FILE}
    return 1
  else
    echo "PASS: all external licenses included."
    rm -rf ${EXTERNAL_DEPENDENCIES_FILE}
    rm -rf ${LICENSES_FILE}
    rm -rf ${MISSING_LICENSES_FILE}
    rm -rf ${EXTRA_LICENSES_FILE}
    return 0
  fi
}

do_pip_package_licenses_check() {
  echo "Running do_pip_package_licenses_check"
  echo ""
  do_external_licenses_check \
    "//tensorflow/tools/pip_package:build_pip_package" \
    "//tensorflow/tools/pip_package:licenses"
}

do_lib_package_licenses_check() {
  echo "Running do_lib_package_licenses_check"
  echo ""
  do_external_licenses_check \
    "//tensorflow:libtensorflow.so" \
    "//tensorflow/tools/lib_package:clicenses_generate"
}

do_java_package_licenses_check() {
  echo "Running do_java_package_licenses_check"
  echo ""
  do_external_licenses_check \
    "//tensorflow/java:libtensorflow_jni.so" \
    "//tensorflow/tools/lib_package:jnilicenses_generate"
}

#Check for the bazel cmd status (First arg is error message)
cmd_status(){
  if [[ $? != 0 ]]; then
    echo ""
    echo "FAIL: ${BUILD_CMD}"
    echo "  $1 See lines above for details."
    return 1
  else
    echo ""
    echo "PASS: ${BUILD_CMD}"
    return 0
  fi
}

# Run bazel build --nobuild to test the validity of the BUILD files
do_bazel_nobuild() {
  BUILD_TARGET="//tensorflow/..."
  BUILD_TARGET="${BUILD_TARGET} -//tensorflow/lite/..."
  BUILD_CMD="bazel build --nobuild ${BAZEL_FLAGS} -- ${BUILD_TARGET}"

  ${BUILD_CMD}

  cmd_status \
    "This is due to invalid BUILD files."
}

do_bazel_deps_query() {
  local BUILD_TARGET='//tensorflow/...'
  # Android targets tend to depend on an Android runtime being available.
  # Exclude until the sanity test has such a runtime available.
  #
  # TODO(mikecase): Remove TF Lite exclusion from this list. Exclusion is
  # necessary since the @androidsdk WORKSPACE dependency is commented out by
  # default in TF WORKSPACE file.
  local BUILD_TARGET="${BUILD_TARGET}"' - kind("android_*", //tensorflow/...)'

  # We've set the flag noimplicit_deps as a workaround for
  # https://github.com/bazelbuild/bazel/issues/10544
  bazel query ${BAZEL_FLAGS} --noimplicit_deps -- "deps($BUILD_TARGET)" > /dev/null

  cmd_status \
    "This is due to invalid BUILD files."
}

do_pip_smoke_test() {
  cd "$ROOT_DIR/tensorflow/tools/pip_package"
  python pip_smoke_test.py
}

do_code_link_check() {
  tensorflow/tools/ci_build/code_link_check.sh
}

# List .h|.cc files changed in the last non-merge git commit that still exist,
# i.e., not removed.
# Usage: get_clang_files_to_check [--incremental]
get_clang_files_to_check() {
  if [[ "$1" == "--incremental" ]]; then
    CHANGED_CLANG_FILES=$(get_changed_files_in_last_non_merge_git_commit | \
                       grep '.*\.h$\|.*\.cc$')

    # Do not include files removed in the last non-merge commit.
    CLANG_FILES=""
    for CLANG_FILE in ${CHANGED_CLANG_FILES}; do
      if [[ -f "${CLANG_FILE}" ]]; then
        CLANG_FILES="${CLANG_FILES} ${CLANG_FILE}"
      fi
    done

    echo "${CLANG_FILES}"
  else
    find tensorflow -name '*.h' -o -name '*.cc'
  fi
}

do_clang_format_check() {
  if [[ $# != "0" ]] && [[ $# != "1" ]]; then
    echo "Invalid syntax when invoking do_clang_format_check"
    echo "Usage: do_clang_format_check [--incremental]"
    return 1
  fi

  if [[ "$1" == "--incremental" ]]; then
    CLANG_SRC_FILES=$(get_clang_files_to_check --incremental)

    if [[ -z "${CLANG_SRC_FILES}" ]]; then
      echo "do_clang_format_check will NOT run due to --incremental flag and "\
"due to the absence of .h or .cc code changes in the last commit."
      return 0
    fi
  elif [[ -z "$1" ]]; then
    # TODO (yongtang): Always pass --incremental until all files have
    # been sanitized gradually. Then this --incremental could be removed.
    CLANG_SRC_FILES=$(get_clang_files_to_check --incremental)
  else
    echo "Invalid syntax for invoking do_clang_format_check"
    echo "Usage: do_clang_format_check [--incremental]"
    return 1
  fi

  CLANG_FORMAT=${CLANG_FORMAT:-clang-format-3.8}

  success=1
  for filename in $CLANG_SRC_FILES; do
    $CLANG_FORMAT --style=google $filename | diff $filename - > /dev/null
    if [ ! $? -eq 0 ]; then
      success=0
      echo File $filename is not properly formatted with "clang-format "\
"--style=google"
    fi
  done

  if [ $success == 0 ]; then
    echo Clang format check fails.
    exit 1
  fi
  echo Clang format check success.
}

do_check_load_py_test() {
  cd "$ROOT_DIR/tensorflow/tools/pip_package"
  python check_load_py_test.py
}

do_check_file_name_test() {
  cd "$ROOT_DIR/tensorflow/tools/test"
  python file_name_test.py
}

# Check that TARGET does not depend on DISALLOWED_DEP.
_check_no_deps() {
  TARGET="$1"
  DISALLOWED_DEP="$2"
  EXTRA_FLAG="$3"

  TMP_FILE="$(mktemp)_tmp.log"
  echo "Checking ${TARGET} does not depend on ${DISALLOWED_DEP} ..."
  bazel cquery ${EXTRA_FLAG} "somepath(${TARGET}, ${DISALLOWED_DEP})" --keep_going> "${TMP_FILE}" 2>&1
  if cat "${TMP_FILE}" | grep "Empty query results"; then
      echo "Success."
  else
      cat "${TMP_FILE}"
      echo
      echo "ERROR: Found path from ${TARGET} to disallowed dependency ${DISALLOWED_DEP}."
      echo "See above for path."
      rm "${TMP_FILE}"
      exit 1
  fi
  rm "${TMP_FILE}"
}

_do_pip_no_cuda_deps_check() {
  EXTRA_FLAG="$1"
  DISALLOWED_CUDA_DEPS=("@local_config_cuda//cuda:cudart"
        "@local_config_cuda//cuda:cublas"
        "@local_config_cuda//cuda:cuda_driver"
        "@local_config_cuda//cuda:cudnn"
        "@local_config_cuda//cuda:curand"
        "@local_config_cuda//cuda:cusolver"
        "@local_config_cuda//cuda:cusparse"
        "@local_config_tensorrt//:tensorrt")
  for cuda_dep in "${DISALLOWED_CUDA_DEPS[@]}"
  do
   _check_no_deps "//tensorflow/tools/pip_package:build_pip_package" "${cuda_dep}" "${EXTRA_FLAG}"
   RESULT=$?

   if [[ ${RESULT} != "0" ]]; then
    exit 1
   fi
  done
}

do_pip_no_cuda_deps_check_ubuntu() {
  _do_pip_no_cuda_deps_check "--@local_config_cuda//:enable_cuda"
}

do_pip_no_cuda_deps_check_windows() {
  _do_pip_no_cuda_deps_check "--@local_config_cuda//:enable_cuda --define framework_shared_object=false"
}

do_configure_test() {
  for WITH_CUDA in 1 0
  do
    export TF_NEED_CUDA=${WITH_CUDA}
    export CUDNN_INSTALL_PATH="/usr/local/cudnn"
    export PYTHON_BIN_PATH=$(which python3.8)
    yes "" | ${PYTHON_BIN_PATH} configure.py

    RESULT=$?
    if [[ ${RESULT} != "0" ]]; then
     exit 1
    fi
  done
}

# Supply all sanity step commands and descriptions
SANITY_STEPS=("do_configure_test" "do_pylint" "do_buildifier" "do_bazel_nobuild" "do_bazel_deps_query" "do_pip_package_licenses_check" "do_lib_package_licenses_check" "do_java_package_licenses_check" "do_pip_smoke_test" "do_check_load_py_test" "do_code_link_check" "do_check_file_name_test" "do_pip_no_cuda_deps_check_ubuntu" "do_pip_no_cuda_deps_check_windows")
SANITY_STEPS_DESC=("Run ./configure" "Python 3 pylint" "buildifier check" "bazel nobuild" "bazel query" "pip: license check for external dependencies" "C library: license check for external dependencies" "Java Native Library: license check for external dependencies" "Pip Smoke Test: Checking py_test dependencies exist in pip package" "Check load py_test: Check that BUILD files with py_test target properly load py_test" "Code Link Check: Check there are no broken links" "Check file names for cases" "Check Ubuntu gpu pip package does not depend on cuda shared libraries" "Check Windows gpu pip package does not depend on cuda shared libraries")

INCREMENTAL_FLAG=""
DEFAULT_BAZEL_CONFIGS=""

# Parse command-line arguments
BAZEL_FLAGS=${DEFAULT_BAZEL_CONFIGS}
for arg in "$@"; do
  if [[ "${arg}" == "--pep8" ]]; then
    # Only run pep8 test if "--pep8" option supplied
    SANITY_STEPS=("do_pep8")
    SANITY_STEPS_DESC=("pep8 test")
  elif [[ "${arg}" == "--incremental" ]]; then
    INCREMENTAL_FLAG="--incremental"
  elif [[ "${arg}" == "--pylint" ]]; then
    SANITY_STEPS=("do_pylint")
    SANITY_STEPS_DESC=("pylint test")
  else
    BAZEL_FLAGS="${BAZEL_FLAGS} ${arg}"
  fi
done

FAIL_COUNTER=0
PASS_COUNTER=0
STEP_EXIT_CODES=()

# Execute all the sanity build steps
COUNTER=0
while [[ ${COUNTER} -lt "${#SANITY_STEPS[@]}" ]]; do
  INDEX=COUNTER
  ((INDEX++))

  echo ""
  echo "=== Sanity check step ${INDEX} of ${#SANITY_STEPS[@]}: "\
"${SANITY_STEPS[COUNTER]} (${SANITY_STEPS_DESC[COUNTER]}) ==="
  echo ""

  # subshell: don't leak variables or changes of working directory
  (
  ${SANITY_STEPS[COUNTER]} ${INCREMENTAL_FLAG}
  )
  RESULT=$?

  if [[ ${RESULT} != "0" ]]; then
    ((FAIL_COUNTER++))
  else
    ((PASS_COUNTER++))
  fi

  STEP_EXIT_CODES+=(${RESULT})

  echo ""
  ((COUNTER++))
done

# Print summary of build results
COUNTER=0
echo "==== Summary of sanity check results ===="
TESTCASE_XML=''
while [[ ${COUNTER} -lt "${#SANITY_STEPS[@]}" ]]; do
  INDEX=COUNTER
  ((INDEX++))

  echo "${INDEX}. ${SANITY_STEPS[COUNTER]}: ${SANITY_STEPS_DESC[COUNTER]}"
  TESTCASE_XML="${TESTCASE_XML} <testcase name=\"${SANITY_STEPS_DESC[COUNTER]}\" status=\"run\" classname=\"\" time=\"0\">"

  if [[ ${STEP_EXIT_CODES[COUNTER]} == "0" ]]; then
    printf "  ${COLOR_GREEN}PASS${COLOR_NC}\n"
  else
    printf "  ${COLOR_RED}FAIL${COLOR_NC}\n"
    TESTCASE_XML="${TESTCASE_XML} <failure message=\"\" type=\"\"/>"
  fi

  TESTCASE_XML="${TESTCASE_XML} </testcase>"

  ((COUNTER++))
done

echo
echo "${FAIL_COUNTER} failed; ${PASS_COUNTER} passed."

mkdir -p "${KOKORO_ARTIFACTS_DIR}/${KOKORO_JOB_NAME}/summary"
echo '<?xml version="1.0" encoding="UTF-8"?>'\
  '<testsuites name="1"  tests="1" failures="0" errors="0" time="0">'\
  '<testsuite name="Kokoro Summary" tests="'"$((FAIL_COUNTER + PASS_COUNTER))"\
  '" failures="'"${FAIL_COUNTER}"'" errors="0" time="0">'\
  "${TESTCASE_XML}"'</testsuite></testsuites>'\
  > "${KOKORO_ARTIFACTS_DIR}/${KOKORO_JOB_NAME}/summary/sponge_log.xml"

echo
if [[ ${FAIL_COUNTER} == "0" ]]; then
  printf "Sanity checks ${COLOR_GREEN}PASSED${COLOR_NC}\n"
else
  printf "Sanity checks ${COLOR_RED}FAILED${COLOR_NC}\n"
  exit 1
fi

#!/usr/bin/env bash
# Copyright 2016 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
#
# Usage: ci_sanity.sh [--pep8] [--incremental] [bazel flags]
#
# Options:
#           run sanity checks: python 2&3 pylint checks and bazel nobuild
#  --pep8   run pep8 test only
#  --pylint run pylint check only
#  --incremental  Performs checks incrementally, by using the files changed in
#                 the latest commit

# Current script directory
SCRIPT_DIR=$( cd ${0%/*} && pwd -P )
source "${SCRIPT_DIR}/builds/builds_common.sh"

ROOT_DIR=$( cd "$SCRIPT_DIR/../../.." && pwd -P )

# Helper functions
die() {
  echo $@
  exit 1
}

num_cpus() {
  # Get the number of CPUs
  if [[ -f /proc/cpuinfo ]]; then
    N_CPUS=$(grep -c ^processor /proc/cpuinfo)
  else
    # Fallback method
    N_CPUS=`getconf _NPROCESSORS_ONLN`
  fi
  if [[ -z ${N_CPUS} ]]; then
    die "ERROR: Unable to determine the number of CPUs"
  fi

  echo ${N_CPUS}
}

# Helper functions for examining changed files in the last non-merge git
# commit.

# Get the hash of the last non-merge git commit on the current branch.
# Usage: get_last_non_merge_git_commit
get_last_non_merge_git_commit() {
  git rev-list --no-merges -n 1 HEAD
}

# List files changed (i.e., added, removed or revised) in the last non-merge
# git commit.
# Usage: get_changed_files_in_last_non_merge_git_commit
get_changed_files_in_last_non_merge_git_commit() {
  git diff-tree --no-commit-id --name-only -r $(get_last_non_merge_git_commit)
}

# List Python files changed in the last non-merge git commit that still exist,
# i.e., not removed.
# Usage: get_py_files_to_check [--incremental]
get_py_files_to_check() {
  if [[ "$1" == "--incremental" ]]; then
    CHANGED_PY_FILES=$(get_changed_files_in_last_non_merge_git_commit | \
                       grep '.*\.py$')

    # Do not include files removed in the last non-merge commit.
    PY_FILES=""
    for PY_FILE in ${CHANGED_PY_FILES}; do
      if [[ -f "${PY_FILE}" ]]; then
        PY_FILES="${PY_FILES} ${PY_FILE}"
      fi
    done

    echo "${PY_FILES}"
  else
    find tensorflow -name '*.py'
  fi
}

# Subfunctions for substeps
# Run pylint
do_pylint() {
  # Usage: do_pylint [--incremental]
  #
  # Options:
  #   --incremental  Performs check only if there are python files that changed
  #                  since last non-merge git commit. We always check all Python
  #                  files if one changed to capture the case when a function
  #                  signature changes affects unchanged files.

  # Validate arguments, see if we can do no work
  if [[ $# == 1 ]] && [[ "$1" == "--incremental" ]]; then
    PYTHON_SRC_FILES=$(get_py_files_to_check --incremental)

    if [[ -z "${PYTHON_SRC_FILES}" ]]; then
      echo "do_pylint will NOT run due to --incremental flag and due to the "\
"absence of Python code changes in the last commit."
      return 0
    fi
  elif [[ $# != 0 ]]; then
    echo "Invalid syntax for invoking do_pylint"
    echo "Usage: do_pylint [--incremental]"
    return 1
  else
    # Get all Python files, regardless of mode.
    PYTHON_SRC_FILES=$(get_py_files_to_check)
  fi

  # Something happened. TF no longer has Python code if this branch is taken
  if [[ -z ${PYTHON_SRC_FILES} ]]; then
    echo "do_pylint found no Python files to check. Returning."
    return 0
  fi

  # Now that we know we have to do work, check if `pylint` is installed
  PYLINT_BIN="python3.8 -m pylint"

  echo ""
  echo "check whether pylint is available or not."
  echo ""
  ${PYLINT_BIN} --version
  if [[ $? -eq 0 ]]
  then
    echo ""
    echo "pylint available, proceeding with pylint sanity check."
    echo ""
  else
    echo ""
    echo "pylint not available."
    echo ""
    return 1
  fi

  # Configure pylint using the following file
  PYLINTRC_FILE="${SCRIPT_DIR}/pylintrc"

  if [[ ! -f "${PYLINTRC_FILE}" ]]; then
    die "ERROR: Cannot find pylint rc file at ${PYLINTRC_FILE}"
  fi

  # Run pylint in parallel, after some disk setup
  NUM_SRC_FILES=$(echo ${PYTHON_SRC_FILES} | wc -w)
  NUM_CPUS=$(num_cpus)

  echo "Running pylint on ${NUM_SRC_FILES} files with ${NUM_CPUS} "\
"parallel jobs..."
  echo ""

  PYLINT_START_TIME=$(date +'%s')
  OUTPUT_FILE="$(mktemp)_pylint_output.log"
  ERRORS_FILE="$(mktemp)_pylint_errors.log"
  PERMIT_FILE="$(mktemp)_pylint_permit.log"
  FORBID_FILE="$(mktemp)_pylint_forbid.log"

  rm -rf ${OUTPUT_FILE}
  rm -rf ${ERRORS_FILE}
  rm -rf ${PERMIT_FILE}
  rm -rf ${FORBID_FILE}

  # When running, filter to only contain the error code lines. Removes module
  # header, removes lines of context that show up from some lines.
  # Also, don't redirect stderr as this would hide pylint fatal errors.
  ${PYLINT_BIN} --rcfile="${PYLINTRC_FILE}" --output-format=parseable \
      --jobs=${NUM_CPUS} ${PYTHON_SRC_FILES} | grep '\[[CEFW]' > ${OUTPUT_FILE}
  PYLINT_END_TIME=$(date +'%s')

  echo ""
  echo "pylint took $((PYLINT_END_TIME - PYLINT_START_TIME)) s"
  echo ""

  # Report only what we care about
  # Ref https://pylint.readthedocs.io/en/latest/technical_reference/features.html
  # E: all errors
  # W0311 bad-indentation
  # W0312 mixed-indentation
  # C0330 bad-continuation
  # C0301 line-too-long
  # C0326 bad-whitespace
  # W0611 unused-import
  # W0622 redefined-builtin
  grep -E '(\[E|\[W0311|\[W0312|\[C0330|\[C0301|\[C0326|\[W0611|\[W0622)' ${OUTPUT_FILE} > ${ERRORS_FILE}

  # Split the pylint reported errors into permitted ones and those we want to
  # block submit on until fixed.
  # We use `${ALLOW_LIST_FILE}` to record the errors we temporarily accept. Goal
  # is to make that file only contain errors caused by difference between
  # internal and external versions.
  ALLOW_LIST_FILE="${SCRIPT_DIR}/pylint_allowlist"

  if [[ ! -f "${ALLOW_LIST_FILE}" ]]; then
    die "ERROR: Cannot find pylint allowlist file at ${ALLOW_LIST_FILE}"
  fi

  # We can split with just 2 grep invocations
  grep    -f ${ALLOW_LIST_FILE} ${ERRORS_FILE} > ${PERMIT_FILE}
  grep -v -f ${ALLOW_LIST_FILE} ${ERRORS_FILE} > ${FORBID_FILE}

  # Determine counts of errors
  N_PERMIT_ERRORS=$(wc -l ${PERMIT_FILE} | cut -d' ' -f1)
  N_FORBID_ERRORS=$(wc -l ${FORBID_FILE} | cut -d' ' -f1)

  # First print all allowed errors
  echo ""
  if [[ ${N_PERMIT_ERRORS} != 0 ]]; then
    echo "Found ${N_PERMIT_ERRORS} allowlisted pylint errors:"
    cat ${PERMIT_FILE}
  fi

  # Now, print the errors we should fix
  echo ""
  if [[ ${N_FORBID_ERRORS} != 0 ]]; then
    echo "Found ${N_FORBID_ERRORS} non-allowlisted pylint errors:"
    cat ${FORBID_FILE}
  fi

  echo ""
  if [[ ${N_FORBID_ERRORS} != 0 ]]; then
    echo "FAIL: Found ${N_FORBID_ERRORS} non-allowlisted errors and ${N_PERMIT_ERRORS} allowlisted errors"
    return 1
  else
    echo "PASS: Found only ${N_PERMIT_ERRORS} allowlisted errors"
  fi
}

# Run pep8 check
do_pep8() {
  # Usage: do_pep8 [--incremental]
  # Options:
  #   --incremental  Performs check on only the python files changed in the
  #                  last non-merge git commit.

  PEP8_BIN="/usr/local/bin/pep8"
  PEP8_CONFIG_FILE="${SCRIPT_DIR}/pep8"

  if [[ "$1" == "--incremental" ]]; then
    PYTHON_SRC_FILES=$(get_py_files_to_check --incremental)
    NUM_PYTHON_SRC_FILES=$(echo ${PYTHON_SRC_FILES} | wc -w)

    echo "do_pep8 will perform checks on only the ${NUM_PYTHON_SRC_FILES} "\
"Python file(s) changed in the last non-merge git commit due to the "\
"--incremental flag:"
    echo "${PYTHON_SRC_FILES}"
    echo ""
  else
    PYTHON_SRC_FILES=$(get_py_files_to_check)
  fi

  if [[ -z ${PYTHON_SRC_FILES} ]]; then
    echo "do_pep8 found no Python files to check. Returning."
    return 0
  fi

  if [[ ! -f "${PEP8_CONFIG_FILE}" ]]; then
    die "ERROR: Cannot find pep8 config file at ${PEP8_CONFIG_FILE}"
  fi
  echo "See \"${PEP8_CONFIG_FILE}\" for pep8 config( e.g., ignored errors)"

  NUM_SRC_FILES=$(echo ${PYTHON_SRC_FILES} | wc -w)

  echo "Running pep8 on ${NUM_SRC_FILES} files"
  echo ""

  PEP8_START_TIME=$(date +'%s')
  PEP8_OUTPUT_FILE="$(mktemp)_pep8_output.log"

  rm -rf ${PEP8_OUTPUT_FILE}

  ${PEP8_BIN} --config="${PEP8_CONFIG_FILE}" --statistics \
      ${PYTHON_SRC_FILES} 2>&1 | tee ${PEP8_OUTPUT_FILE}
  PEP8_END_TIME=$(date +'%s')

  echo ""
  echo "pep8 took $((PEP8_END_TIME - PEP8_START_TIME)) s"
  echo ""

  if [[ -s ${PEP8_OUTPUT_FILE} ]]; then
    echo "FAIL: pep8 found above errors and/or warnings."
    return 1
  else
    echo "PASS: No pep8 errors or warnings were found"
    return 0
  fi
}

do_buildifier(){
  BUILD_FILES=$(find tensorflow -name 'BUILD*')
  NUM_BUILD_FILES=$(echo ${BUILD_FILES} | wc -w)

  echo "Running do_buildifier on ${NUM_BUILD_FILES} files"
  echo ""

  BUILDIFIER_START_TIME=$(date +'%s')
  BUILDIFIER_OUTPUT_FILE="$(mktemp)_buildifier_output.log"

  rm -rf ${BUILDIFIER_OUTPUT_FILE}

  buildifier -v -mode=check \
    ${BUILD_FILES} 2>&1 | tee ${BUILDIFIER_OUTPUT_FILE}
  BUILDIFIER_END_TIME=$(date +'%s')

  echo ""
  echo "buildifier took $((BUILDIFIER_END_TIME - BUILDIFIER_START_TIME)) s"
  echo ""

  if [[ -s ${BUILDIFIER_OUTPUT_FILE} ]]; then
    echo "FAIL: buildifier found errors and/or warnings in above BUILD files."
    echo "buildifier suggested the following changes:"
    buildifier -v -mode=diff -diff_command=diff ${BUILD_FILES}
    echo "Please fix manually or run buildifier <file> to auto-fix."
    return 1
  else
    echo "PASS: No buildifier errors or warnings were found"
    return 0
  fi
}

do_external_licenses_check(){
  BUILD_TARGET="$1"
  LICENSES_TARGET="$2"

  EXTERNAL_LICENSES_CHECK_START_TIME=$(date +'%s')

  EXTERNAL_DEPENDENCIES_FILE="$(mktemp)_external_dependencies.log"
  LICENSES_FILE="$(mktemp)_licenses.log"
  MISSING_LICENSES_FILE="$(mktemp)_missing_licenses.log"
  EXTRA_LICENSES_FILE="$(mktemp)_extra_licenses.log"
  TMP_FILE="$(mktemp)_tmp.log"

  echo "Getting external dependencies for ${BUILD_TARGET}"
 bazel cquery "attr('licenses', 'notice', deps(${BUILD_TARGET}))" --keep_going > "${TMP_FILE}" 2>&1
 cat "${TMP_FILE}" \
  | grep -e "^\/\/" -e "^@" \
  | grep -E -v "^//tensorflow" \
  | sed -e 's|:.*||' \
  | sort \
  | uniq 2>&1 \
  | tee ${EXTERNAL_DEPENDENCIES_FILE}

  echo
  echo "Getting list of external licenses mentioned in ${LICENSES_TARGET}."
  bazel cquery "deps(${LICENSES_TARGET})" --keep_going > "${TMP_FILE}" 2>&1
 cat "${TMP_FILE}" \
  | grep -e "^\/\/" -e "^@" \
  | grep -E -v "^//tensorflow" \
  | sed -e 's|:.*||' \
  | sort \
  | uniq 2>&1 \
  | tee ${LICENSES_FILE}

  echo
  comm -1 -3 ${EXTERNAL_DEPENDENCIES_FILE}  ${LICENSES_FILE} 2>&1 | tee ${EXTRA_LICENSES_FILE}
  echo
  comm -2 -3 ${EXTERNAL_DEPENDENCIES_FILE}  ${LICENSES_FILE} 2>&1 | tee ${MISSING_LICENSES_FILE}

  EXTERNAL_LICENSES_CHECK_END_TIME=$(date +'%s')

  # Denylist
  echo ${MISSING_LICENSES_FILE}
  grep \
    -e "@bazel_tools//platforms" \
    -e "@bazel_tools//third_party/" \
    -e "@bazel_tools//tools" \
    -e "@local" \
    -e "@com_google_absl//absl" \
    -e "@org_tensorflow//" \
    -e "@com_github_googlecloudplatform_google_cloud_cpp//google" \
    -e "@com_github_grpc_grpc//src/compiler" \
    -e "@platforms//os" \
    -e "@ruy//" \
    -v ${MISSING_LICENSES_FILE} > temp.txt
  mv temp.txt ${MISSING_LICENSES_FILE}

  # Allowlist
  echo ${EXTRA_LICENSE_FILE}
  grep \
    -e "//third_party/mkl" \
    -e "//third_party/mkl_dnn" \
    -e "@bazel_tools//src" \
    -e "@bazel_tools//platforms" \
    -e "@bazel_tools//tools/" \
    -e "@org_tensorflow//tensorflow" \
    -e "@com_google_absl//" \
    -e "//external" \
    -e "@local" \
    -e "@com_github_googlecloudplatform_google_cloud_cpp//" \
    -e "@embedded_jdk//" \
    -e "^//$" \
    -e "@ruy//" \
    -v ${EXTRA_LICENSES_FILE} > temp.txt
  mv temp.txt ${EXTRA_LICENSES_FILE}

  echo
  echo "do_external_licenses_check took $((EXTERNAL_LICENSES_CHECK_END_TIME - EXTERNAL_LICENSES_CHECK_START_TIME)) s"
  echo

  if [[ -s ${MISSING_LICENSES_FILE} ]] || [[ -s ${EXTRA_LICENSES_FILE} ]] ; then
    echo "FAIL: mismatch in packaged licenses and external dependencies"
    if [[ -s ${MISSING_LICENSES_FILE} ]] ; then
      echo "Missing the licenses for the following external dependencies:"
      cat ${MISSING_LICENSES_FILE}
      echo "Please add the license(s) to ${LICENSES_TARGET}."
    fi
    if [[ -s ${EXTRA_LICENSES_FILE} ]] ; then
      echo "Please remove the licenses for the following external dependencies from target ${LICENSES_TARGET}."
      cat ${EXTRA_LICENSES_FILE}
    fi
    rm -rf ${EXTERNAL_DEPENDENCIES_FILE}
    rm -rf ${LICENSES_FILE}
    rm -rf ${MISSING_LICENSES_FILE}
    rm -rf ${EXTRA_LICENSES_FILE}
    return 1
  else
    echo "PASS: all external licenses included."
    rm -rf ${EXTERNAL_DEPENDENCIES_FILE}
    rm -rf ${LICENSES_FILE}
    rm -rf ${MISSING_LICENSES_FILE}
    rm -rf ${EXTRA_LICENSES_FILE}
    return 0
  fi
}

do_pip_package_licenses_check() {
  echo "Running do_pip_package_licenses_check"
  echo ""
  do_external_licenses_check \
    "//tensorflow/tools/pip_package:build_pip_package" \
    "//tensorflow/tools/pip_package:licenses"
}

do_lib_package_licenses_check() {
  echo "Running do_lib_package_licenses_check"
  echo ""
  do_external_licenses_check \
    "//tensorflow:libtensorflow.so" \
    "//tensorflow/tools/lib_package:clicenses_generate"
}

do_java_package_licenses_check() {
  echo "Running do_java_package_licenses_check"
  echo ""
  do_external_licenses_check \
    "//tensorflow/java:libtensorflow_jni.so" \
    "//tensorflow/tools/lib_package:jnilicenses_generate"
}

#Check for the bazel cmd status (First arg is error message)
cmd_status(){
  if [[ $? != 0 ]]; then
    echo ""
    echo "FAIL: ${BUILD_CMD}"
    echo "  $1 See lines above for details."
    return 1
  else
    echo ""
    echo "PASS: ${BUILD_CMD}"
    return 0
  fi
}

# Run bazel build --nobuild to test the validity of the BUILD files
do_bazel_nobuild() {
  BUILD_TARGET="//tensorflow/..."
  BUILD_TARGET="${BUILD_TARGET} -//tensorflow/lite/..."
  BUILD_CMD="bazel build --nobuild ${BAZEL_FLAGS} -- ${BUILD_TARGET}"

  ${BUILD_CMD}

  cmd_status \
    "This is due to invalid BUILD files."
}

do_bazel_deps_query() {
  local BUILD_TARGET='//tensorflow/...'
  # Android targets tend to depend on an Android runtime being available.
  # Exclude until the sanity test has such a runtime available.
  #
  # TODO(mikecase): Remove TF Lite exclusion from this list. Exclusion is
  # necessary since the @androidsdk WORKSPACE dependency is commented out by
  # default in TF WORKSPACE file.
  local BUILD_TARGET="${BUILD_TARGET}"' - kind("android_*", //tensorflow/...)'

  # We've set the flag noimplicit_deps as a workaround for
  # https://github.com/bazelbuild/bazel/issues/10544
  bazel query ${BAZEL_FLAGS} --noimplicit_deps -- "deps($BUILD_TARGET)" > /dev/null

  cmd_status \
    "This is due to invalid BUILD files."
}

do_pip_smoke_test() {
  cd "$ROOT_DIR/tensorflow/tools/pip_package"
  python pip_smoke_test.py
}

do_code_link_check() {
  tensorflow/tools/ci_build/code_link_check.sh
}

# List .h|.cc files changed in the last non-merge git commit that still exist,
# i.e., not removed.
# Usage: get_clang_files_to_check [--incremental]
get_clang_files_to_check() {
  if [[ "$1" == "--incremental" ]]; then
    CHANGED_CLANG_FILES=$(get_changed_files_in_last_non_merge_git_commit | \
                       grep '.*\.h$\|.*\.cc$')

    # Do not include files removed in the last non-merge commit.
    CLANG_FILES=""
    for CLANG_FILE in ${CHANGED_CLANG_FILES}; do
      if [[ -f "${CLANG_FILE}" ]]; then
        CLANG_FILES="${CLANG_FILES} ${CLANG_FILE}"
      fi
    done

    echo "${CLANG_FILES}"
  else
    find tensorflow -name '*.h' -o -name '*.cc'
  fi
}

do_clang_format_check() {
  if [[ $# != "0" ]] && [[ $# != "1" ]]; then
    echo "Invalid syntax when invoking do_clang_format_check"
    echo "Usage: do_clang_format_check [--incremental]"
    return 1
  fi

  if [[ "$1" == "--incremental" ]]; then
    CLANG_SRC_FILES=$(get_clang_files_to_check --incremental)

    if [[ -z "${CLANG_SRC_FILES}" ]]; then
      echo "do_clang_format_check will NOT run due to --incremental flag and "\
"due to the absence of .h or .cc code changes in the last commit."
      return 0
    fi
  elif [[ -z "$1" ]]; then
    # TODO (yongtang): Always pass --incremental until all files have
    # been sanitized gradually. Then this --incremental could be removed.
    CLANG_SRC_FILES=$(get_clang_files_to_check --incremental)
  else
    echo "Invalid syntax for invoking do_clang_format_check"
    echo "Usage: do_clang_format_check [--incremental]"
    return 1
  fi

  CLANG_FORMAT=${CLANG_FORMAT:-clang-format-3.8}

  success=1
  for filename in $CLANG_SRC_FILES; do
    $CLANG_FORMAT --style=google $filename | diff $filename - > /dev/null
    if [ ! $? -eq 0 ]; then
      success=0
      echo File $filename is not properly formatted with "clang-format "\
"--style=google"
    fi
  done

  if [ $success == 0 ]; then
    echo Clang format check fails.
    exit 1
  fi
  echo Clang format check success.
}

do_check_load_py_test() {
  cd "$ROOT_DIR/tensorflow/tools/pip_package"
  python check_load_py_test.py
}

do_check_file_name_test() {
  cd "$ROOT_DIR/tensorflow/tools/test"
  python file_name_test.py
}

# Check that TARGET does not depend on DISALLOWED_DEP.
_check_no_deps() {
  TARGET="$1"
  DISALLOWED_DEP="$2"
  EXTRA_FLAG="$3"

  TMP_FILE="$(mktemp)_tmp.log"
  echo "Checking ${TARGET} does not depend on ${DISALLOWED_DEP} ..."
  bazel cquery ${EXTRA_FLAG} "somepath(${TARGET}, ${DISALLOWED_DEP})" --keep_going> "${TMP_FILE}" 2>&1
  if cat "${TMP_FILE}" | grep "Empty query results"; then
      echo "Success."
  else
      cat "${TMP_FILE}"
      echo
      echo "ERROR: Found path from ${TARGET} to disallowed dependency ${DISALLOWED_DEP}."
      echo "See above for path."
      rm "${TMP_FILE}"
      exit 1
  fi
  rm "${TMP_FILE}"
}

_do_pip_no_cuda_deps_check() {
  EXTRA_FLAG="$1"
  DISALLOWED_CUDA_DEPS=("@local_config_cuda//cuda:cudart"
        "@local_config_cuda//cuda:cublas"
        "@local_config_cuda//cuda:cuda_driver"
        "@local_config_cuda//cuda:cudnn"
        "@local_config_cuda//cuda:curand"
        "@local_config_cuda//cuda:cusolver"
        "@local_config_cuda//cuda:cusparse"
        "@local_config_tensorrt//:tensorrt")
  for cuda_dep in "${DISALLOWED_CUDA_DEPS[@]}"
  do
   _check_no_deps "//tensorflow/tools/pip_package:build_pip_package" "${cuda_dep}" "${EXTRA_FLAG}"
   RESULT=$?

   if [[ ${RESULT} != "0" ]]; then
    exit 1
   fi
  done
}

do_pip_no_cuda_deps_check_ubuntu() {
  _do_pip_no_cuda_deps_check "--@local_config_cuda//:enable_cuda"
}

do_pip_no_cuda_deps_check_windows() {
  _do_pip_no_cuda_deps_check "--@local_config_cuda//:enable_cuda --define framework_shared_object=false"
}

do_configure_test() {
  for WITH_CUDA in 1 0
  do
    export TF_NEED_CUDA=${WITH_CUDA}
    export CUDNN_INSTALL_PATH="/usr/local/cudnn"
    export PYTHON_BIN_PATH=$(which python3.8)
    yes "" | ${PYTHON_BIN_PATH} configure.py

    RESULT=$?
    if [[ ${RESULT} != "0" ]]; then
     exit 1
    fi
  done
}

# Supply all sanity step commands and descriptions
SANITY_STEPS=("do_configure_test" "do_pylint" "do_buildifier" "do_bazel_nobuild" "do_bazel_deps_query" "do_pip_package_licenses_check" "do_lib_package_licenses_check" "do_java_package_licenses_check" "do_pip_smoke_test" "do_check_load_py_test" "do_code_link_check" "do_check_file_name_test" "do_pip_no_cuda_deps_check_ubuntu" "do_pip_no_cuda_deps_check_windows")
SANITY_STEPS_DESC=("Run ./configure" "Python 3 pylint" "buildifier check" "bazel nobuild" "bazel query" "pip: license check for external dependencies" "C library: license check for external dependencies" "Java Native Library: license check for external dependencies" "Pip Smoke Test: Checking py_test dependencies exist in pip package" "Check load py_test: Check that BUILD files with py_test target properly load py_test" "Code Link Check: Check there are no broken links" "Check file names for cases" "Check Ubuntu gpu pip package does not depend on cuda shared libraries" "Check Windows gpu pip package does not depend on cuda shared libraries")

INCREMENTAL_FLAG=""
DEFAULT_BAZEL_CONFIGS=""

# Parse command-line arguments
BAZEL_FLAGS=${DEFAULT_BAZEL_CONFIGS}
for arg in "$@"; do
  if [[ "${arg}" == "--pep8" ]]; then
    # Only run pep8 test if "--pep8" option supplied
    SANITY_STEPS=("do_pep8")
    SANITY_STEPS_DESC=("pep8 test")
  elif [[ "${arg}" == "--incremental" ]]; then
    INCREMENTAL_FLAG="--incremental"
  elif [[ "${arg}" == "--pylint" ]]; then
    SANITY_STEPS=("do_pylint")
    SANITY_STEPS_DESC=("pylint test")
  else
    BAZEL_FLAGS="${BAZEL_FLAGS} ${arg}"
  fi
done

FAIL_COUNTER=0
PASS_COUNTER=0
STEP_EXIT_CODES=()

# Execute all the sanity build steps
COUNTER=0
while [[ ${COUNTER} -lt "${#SANITY_STEPS[@]}" ]]; do
  INDEX=COUNTER
  ((INDEX++))

  echo ""
  echo "=== Sanity check step ${INDEX} of ${#SANITY_STEPS[@]}: "\
"${SANITY_STEPS[COUNTER]} (${SANITY_STEPS_DESC[COUNTER]}) ==="
  echo ""

  # subshell: don't leak variables or changes of working directory
  (
  ${SANITY_STEPS[COUNTER]} ${INCREMENTAL_FLAG}
  )
  RESULT=$?

  if [[ ${RESULT} != "0" ]]; then
    ((FAIL_COUNTER++))
  else
    ((PASS_COUNTER++))
  fi

  STEP_EXIT_CODES+=(${RESULT})

  echo ""
  ((COUNTER++))
done

# Print summary of build results
COUNTER=0
echo "==== Summary of sanity check results ===="
TESTCASE_XML=''
while [[ ${COUNTER} -lt "${#SANITY_STEPS[@]}" ]]; do
  INDEX=COUNTER
  ((INDEX++))

  echo "${INDEX}. ${SANITY_STEPS[COUNTER]}: ${SANITY_STEPS_DESC[COUNTER]}"
  TESTCASE_XML="${TESTCASE_XML} <testcase name=\"${SANITY_STEPS_DESC[COUNTER]}\" status=\"run\" classname=\"\" time=\"0\">"

  if [[ ${STEP_EXIT_CODES[COUNTER]} == "0" ]]; then
    printf "  ${COLOR_GREEN}PASS${COLOR_NC}\n"
  else
    printf "  ${COLOR_RED}FAIL${COLOR_NC}\n"
    TESTCASE_XML="${TESTCASE_XML} <failure message=\"\" type=\"\"/>"
  fi

  TESTCASE_XML="${TESTCASE_XML} </testcase>"

  ((COUNTER++))
done

echo
echo "${FAIL_COUNTER} failed; ${PASS_COUNTER} passed."

mkdir -p "${KOKORO_ARTIFACTS_DIR}/${KOKORO_JOB_NAME}/summary"
echo '<?xml version="1.0" encoding="UTF-8"?>'\
  '<testsuites name="1"  tests="1" failures="0" errors="0" time="0">'\
  '<testsuite name="Kokoro Summary" tests="'"$((FAIL_COUNTER + PASS_COUNTER))"\
  '" failures="'"${FAIL_COUNTER}"'" errors="0" time="0">'\
  "${TESTCASE_XML}"'</testsuite></testsuites>'\
  > "${KOKORO_ARTIFACTS_DIR}/${KOKORO_JOB_NAME}/summary/sponge_log.xml"

echo
if [[ ${FAIL_COUNTER} == "0" ]]; then
  printf "Sanity checks ${COLOR_GREEN}PASSED${COLOR_NC}\n"
else
  printf "Sanity checks ${COLOR_RED}FAILED${COLOR_NC}\n"
  exit 1
fi


erreur de merge

erreur de merge

remote: Support for password authentication was removed on August 13, 2021.
remote: Please see https://docs.github.com/en/get-started/getting-started-with-git/about-remote-repositories#cloning-with-https-urls for information on currently recommended modes of authentication.
fatal: Authentication failed for 'https://github.com/pierremellet/MCS.git/'


added skel for ksh

added skel for ksh

error: pathspec 'skel/skel.ksh' did not match any file(s) known to git


Script to read from twitter

Script to read from twitter

while sleep 10;
do
GET "http://search.twitter.com/search.atom?q=%23Lseed&rpp=1" > msg.atom
echo -n " Via Twitter: " \
$(
	xpath -q -e '//author/name/text()' msg.atom  |
	lynx -dump -stdin|lynx -dump -stdin -nolist
) \
": " \
$(
	xpath -q -e '//content/text()' msg.atom  |
	lynx -dump -stdin|lynx -dump -stdin -nolist
) \
 > msg.txt
done

error: pathspec 'twitreader.sh' did not match any file(s) known to git


uninstall script

uninstall script

#!/bin/sh

if [ -h /usr/local/bin/sf-sms ]; then
	rm -f /usr/local/bin/sf-sms
fi

error: pathspec 'uninstall.sh' did not match any file(s) known to git


Added uninstall script

Added uninstall script

#!/bin/bash
#
# Copyright (C) 2011 James Titcumb
#
# See LICENSE file for license!
#

if [ $UID != 0 ]
then
  echo "This script must be run as root."
  exit 1
fi

source dst-install-conf

if [ -f $CONFIGFILE ]
then
  echo "Leaving $CONFIGFILE in situ. Remove manually..."
fi

if [ -f $INITSCRIPT ]
then
  rm -v $INITSCRIPT
fi

INITFILES=`ls /etc/rc*.d/*directory-syncing-thing 2> /dev/null`
for F in $INITFILES
do
  rm -v $F
done

echo "Uninstall complete."


- add frensh

- add frensh

remote: Support for password authentication was removed on August 13, 2021.
remote: Please see https://docs.github.com/en/get-started/getting-started-with-git/about-remote-repositories#cloning-with-https-urls for information on currently recommended modes of authentication.
fatal: Authentication failed for 'https://github.com/grimneko/iroffer-modDinoex.git/'


Added instructions about port forwarding regions

remote: Support for password authentication was removed on August 13, 2021.
remote: Please see https://docs.github.com/en/get-started/getting-started-with-git/about-remote-repositories#cloning-with-https-urls for information on currently recommended modes of authentication.
fatal: Authentication failed for 'https://github.com/taeram/hearth.git/'


other/git_proxy.sh - core.gitproxy command

error: pathspec 'other/git_proxy.sh' did not match any file(s) known to git


first commit

first commit

error: pathspec 'svn-color.sh' did not match any file(s) known to git


script for creating release tarballs [ci skip]

#!/bin/bash
rm -rf temp.`date +%Y%m%d`
mkdir -p temp.`date +%Y%m%d`
cd temp.`date +%Y%m%d`
NWVER='6.8-beta3'
git clone -b release-6-8 https://github.com/nwchemgit/nwchem nwchem-6.8
cd nwchem-6.8/src/tools
######REV68="$(svn info -r HEAD |grep Revision: |cut -c11-)"
rm -f *tar*gz
./get-tools-github
cd ../util
sh ./util_ga_version.bash
sh ./util_nwchem_version.bash
#sed -i 's/N\/A/v5.6.2/'  util_ga_version.F
cat util_ga_version.F
cd ../..
REV68="$(git describe)"
cd ..
echo 'revision ' $REV68
pwd
rm -f *md5 *tar*
tar --exclude=".git" -czvf nwchem-"${NWVER}".revision-"${REV68}"-src.`date +%Y-%m-%d`.tar.gz nwchem-6.8/*
md5sum nwchem-"${NWVER}".revision-"${REV68}"-src.`date +%Y-%m-%d`.tar.gz > nwchem-6.8.revision"${REV68}"-src.`date +%Y-%m-%d`.tar.gz.md5
tar --exclude=".git" -cjvf nwchem-"${NWVER}".revision-"${REV68}"-src.`date +%Y-%m-%d`.tar.bz2 nwchem-6.8/*
md5sum nwchem-"${NWVER}".revision-"${REV68}"-src.`date +%Y-%m-%d`.tar.bz2 >  nwchem-6.8.revision"${REV68}"-src.`date +%Y-%m-%d`.tar.bz2.md5
tar --exclude=".git" -cjvf nwchem-"${NWVER}".revision-"${REV68}"-srconly.`date +%Y-%m-%d`.tar.bz2 nwchem-6.8/src/*
md5sum nwchem-"${NWVER}".revision-"${REV68}"-srconly.`date +%Y-%m-%d`.tar.bz2>  nwchem-6.8.revision"${REV68}"-srconly.`date +%Y-%m-%d`.tar.bz2.md5
tar --exclude=".git" --exclude="src" -cjvf nwchem-"${NWVER}".revision-"${REV68}"-nonsrconly.`date +%Y-%m-%d`.tar.bz2 nwchem-6.8/*
md5sum nwchem-"${NWVER}".revision-"${REV68}"-nonsrconly.`date +%Y-%m-%d`.tar.bz2 > nwchem-6.8.revision"${REV68}"-nonsrconly.`date +%Y-%m-%d`.tar.bz2.md5
echo 'upload to http://192.101.105.206/'

error: pathspec 'contrib/git.nwchem/dotar68_git.sh' did not match any file(s) known to git


[distro-install] Remove commented code

[distro-install] Remove commented code

remote: Support for password authentication was removed on August 13, 2021.
remote: Please see https://docs.github.com/en/get-started/getting-started-with-git/about-remote-repositories#cloning-with-https-urls for information on currently recommended modes of authentication.
fatal: Authentication failed for 'https://github.com/jpigree/manarch.git/'


Source z script

Source z script

# Z
if (( $+commands[z] )) && (( $+commands[brew] ))
then
    source `brew --prefix`/etc/profile.d/z.sh
fi

error: pathspec 'system/z.zsh' did not match any file(s) known to git


added git tags after build

added git tags after build

remote: Support for password authentication was removed on August 13, 2021.
remote: Please see https://docs.github.com/en/get-started/getting-started-with-git/about-remote-repositories#cloning-with-https-urls for information on currently recommended modes of authentication.
fatal: Authentication failed for 'https://github.com/pip-services-content/pip-services-quotes-node.git/'


ARROW-3429: [Packaging] Add binary upload script

dev/release/02-source.sh upload binaries before we upload source
archive to dist.apache.org. We need to build and upload binaries after
we upload source archive to dist.apache.org to use the source archive
at dist.apache.org.

Author: Kouhei Sutou <kou@clear-code.com>

Closes #2699 from kou/add-binary-release-script and squashes the following commits:

9d206084b <Kouhei Sutou> Expand variables in commit message
fe9cfd4b9 <Kouhei Sutou>  Add binary upload script

#!/bin/bash
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.
#
set -e

SOURCE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"

if [ "$#" -ne 3 ]; then
  echo "Usage: $0 <version> <rc-num> <artifact_dir>"
  exit
fi

version=$1
rc=$2
artifact_dir=$3

if [ -d tmp/ ]; then
  echo "Cannot run: tmp/ exists"
  exit
fi

if [ -z "$artifact_dir" ]; then
  echo "artifact_dir is empty"
  exit 1
fi

if [ ! -e "$artifact_dir" ]; then
  echo "$artifact_dir does not exist"
  exit 1
fi

if [ ! -d "$artifact_dir" ]; then
  echo "$artifact_dir is not a directory"
  exit 1
fi

tag=apache-arrow-${version}
tagrc=${tag}-rc${rc}

# check out the arrow RC folder
svn co https://dist.apache.org/repos/dist/dev/arrow/${tagrc} tmp

# ensure directories for binary artifacts
mkdir -p tmp/binaries

# copy binary artifacts
cp -rf "$artifact_dir"/* tmp/binaries/

# commit to svn
for dir in "$artifact_dir"/*; do
  svn add tmp/binaries/$(basename $dir)
done
svn ci -m "Apache Arrow ${version} RC${rc} binaries" tmp/

# clean up
rm -rf tmp

echo "Success! The release candidate binaries are available here:"
echo "  https://dist.apache.org/repos/dist/dev/arrow/${tagrc}/binaries"

error: pathspec 'dev/release/03-binary.sh' did not match any file(s) known to git


Cleanup script

Cleanup script

#!/bin/bash

## Load includes
if [ "x$CBF_HOME" = "x" ]; then DIRNAME=`dirname $0`; CBF_HOME=`cd $DIRNAME/..; pwd` ; fi; export CBF_HOME
. ${CBF_HOME}/bin/includes.sh

rm *.out > /dev/null
rm *.log > /dev/null
rm -rf reports > /dev/null

error: pathspec 'framework/src/main/resources/clean.sh' did not match any file(s) known to git


Add update shell script

Add update shell script

# update App Store apps
sudo softwareupdate -i -a

# update Homebrew (Cask) & packages
brew update
brew upgrade

# update npm & packages
npm install npm -g
npm update -g

# update Ruby & gems
gem update —-system
gem update

error: pathspec 'utils/update.sh' did not match any file(s) known to git


add more `git` aliases (#1922)

add more `git` aliases (#1922)

#! /bin/bash

# Colors used for status updates
ESC_SEQ="\x1b["
COL_RESET=$ESC_SEQ"39;49;00m"
COL_RED=$ESC_SEQ"31;01m"
COL_GREEN=$ESC_SEQ"32;01m"
COL_YELLOW=$ESC_SEQ"33;01m"
COL_BLUE=$ESC_SEQ"34;01m"
COL_MAGENTA=$ESC_SEQ"35;01m"
COL_CYAN=$ESC_SEQ"36;01m"

# Detect which `ls` flavor is in use
if ls --color > /dev/null 2>&1; then # GNU `ls`
	colorflag="--color"
	export LS_COLORS='no=00:fi=00:di=01;31:ln=01;36:pi=40;33:so=01;35:do=01;35:bd=40;33;01:cd=40;33;01:or=40;31;01:ex=01;32:*.tar=01;31:*.tgz=01;31:*.arj=01;31:*.taz=01;31:*.lzh=01;31:*.zip=01;31:*.z=01;31:*.Z=01;31:*.gz=01;31:*.bz2=01;31:*.deb=01;31:*.rpm=01;31:*.jar=01;31:*.jpg=01;35:*.jpeg=01;35:*.gif=01;35:*.bmp=01;35:*.pbm=01;35:*.pgm=01;35:*.ppm=01;35:*.tga=01;35:*.xbm=01;35:*.xpm=01;35:*.tif=01;35:*.tiff=01;35:*.png=01;35:*.mov=01;35:*.mpg=01;35:*.mpeg=01;35:*.avi=01;35:*.fli=01;35:*.gl=01;35:*.dl=01;35:*.xcf=01;35:*.xwd=01;35:*.ogg=01;35:*.mp3=01;35:*.wav=01;35:'
else # macOS `ls`
	colorflag="-G"
	export LSCOLORS='BxBxhxDxfxhxhxhxhxcxcx'
fi

# List all files colorized in long format
#alias l="ls -lF ${colorflag}"
### MEGA: I want l and la ti return hisdden files
alias l="ls -laF ${colorflag}"

# List all files colorized in long format, including dot files
alias la="ls -laF ${colorflag}"

# List only directories
alias lsd="ls -lF ${colorflag} | grep --color=never '^d'"

# Always use color output for `ls`
alias ls="command ls ${colorflag}"

# Commonly Used Aliases
alias ..="cd .."
alias ...="cd ../.."
alias ....="cd ../../.."
alias .....="cd ../../../.."
alias ~="cd ~" # `cd` is probably faster to type though
alias -- -="cd -"
alias home="cd ~"

alias h="history"
alias j="jobs"
alias e='exit'
alias c="clear"
alias cla="clear && ls -l"
alias cll="clear && ls -la"
alias cls="clear && ls"
alias code="cd /var/www"
alias ea="vi ~/aliases.sh"

# Always enable colored `grep` output
# Note: `GREP_OPTIONS="--color=auto"` is deprecated, hence the alias usage.
alias grep='grep --color=auto'
alias fgrep='fgrep --color=auto'
alias egrep='egrep --color=auto'

alias art="php artisan"
alias artisan="php artisan"
alias cdump="composer dump-autoload -o"
alias composer:dump="composer dump-autoload -o"
alias db:reset="php artisan migrate:reset && php artisan migrate --seed"
alias dusk="php artisan dusk"
alias fresh="php artisan migrate:fresh"
alias migrate="php artisan migrate"
alias refresh="php artisan migrate:refresh"
alias rollback="php artisan migrate:rollback"
alias seed="php artisan db:seed"
alias serve="php artisan serve --quiet &"

alias phpunit="./vendor/bin/phpunit"
alias pu="phpunit"
alias puf="phpunit --filter"
alias pud='phpunit --debug'

alias cc='codecept'
alias ccb='codecept build'
alias ccr='codecept run'
alias ccu='codecept run unit'
alias ccf='codecept run functional'

alias g="gulp"
alias npm-global="npm list -g --depth 0"
alias ra="reload"
alias reload="source ~/.aliases && echo \"$COL_GREEN ==> Aliases Reloaded... $COL_RESET \n \""
alias run="npm run"
alias tree="xtree"

# Xvfb
alias xvfb="Xvfb -ac :0 -screen 0 1024x768x16 &"

# requires installation of 'https://www.npmjs.com/package/npms-cli'
alias npms="npms search"
# requires installation of 'https://www.npmjs.com/package/package-menu-cli'
alias pm="package-menu"
# requires installation of 'https://www.npmjs.com/package/pkg-version-cli'
alias pv="package-version"
# requires installation of 'https://github.com/sindresorhus/latest-version-cli'
alias lv="latest-version"

# git aliases
alias gaa="git add ."
alias gd="git --no-pager diff"
alias git-revert="git reset --hard && git clean -df"
alias gs="git status"
alias whoops="git reset --hard && git clean -df"
alias glog="git log --oneline --decorate --graph"
alias gsh="git show"
alias grb="git rebase -i"
alias gbr="git branch"
alias gc="git commit"
alias gck="git checkout"

# Create a new directory and enter it
function mkd() {
    mkdir -p "$@" && cd "$@"
}

function md() {
    mkdir -p "$@" && cd "$@"
}

function xtree {
    find ${1:-.} -print | sed -e 's;[^/]*/;|____;g;s;____|; |;g'
}

# `tre` is a shorthand for `tree` with hidden files and color enabled, ignoring
# the `.git` directory, listing directories first. The output gets piped into
# `less` with options to preserve color and line numbers, unless the output is
# small enough for one screen.
function tre() {
	tree -aC -I '.git|node_modules|bower_components' --dirsfirst "$@" | less -FRNX;
}

# Determine size of a file or total size of a directory
function fs() {
	if du -b /dev/null > /dev/null 2>&1; then
		local arg=-sbh;
	else
		local arg=-sh;
	fi
	if [[ -n "$@" ]]; then
		du $arg -- "$@";
	else
		du $arg .[^.]* ./*;
	fi;
}

#! /bin/bash

# Colors used for status updates
ESC_SEQ="\x1b["
COL_RESET=$ESC_SEQ"39;49;00m"
COL_RED=$ESC_SEQ"31;01m"
COL_GREEN=$ESC_SEQ"32;01m"
COL_YELLOW=$ESC_SEQ"33;01m"
COL_BLUE=$ESC_SEQ"34;01m"
COL_MAGENTA=$ESC_SEQ"35;01m"
COL_CYAN=$ESC_SEQ"36;01m"

# Detect which `ls` flavor is in use
if ls --color > /dev/null 2>&1; then # GNU `ls`
	colorflag="--color"
	export LS_COLORS='no=00:fi=00:di=01;31:ln=01;36:pi=40;33:so=01;35:do=01;35:bd=40;33;01:cd=40;33;01:or=40;31;01:ex=01;32:*.tar=01;31:*.tgz=01;31:*.arj=01;31:*.taz=01;31:*.lzh=01;31:*.zip=01;31:*.z=01;31:*.Z=01;31:*.gz=01;31:*.bz2=01;31:*.deb=01;31:*.rpm=01;31:*.jar=01;31:*.jpg=01;35:*.jpeg=01;35:*.gif=01;35:*.bmp=01;35:*.pbm=01;35:*.pgm=01;35:*.ppm=01;35:*.tga=01;35:*.xbm=01;35:*.xpm=01;35:*.tif=01;35:*.tiff=01;35:*.png=01;35:*.mov=01;35:*.mpg=01;35:*.mpeg=01;35:*.avi=01;35:*.fli=01;35:*.gl=01;35:*.dl=01;35:*.xcf=01;35:*.xwd=01;35:*.ogg=01;35:*.mp3=01;35:*.wav=01;35:'
else # macOS `ls`
	colorflag="-G"
	export LSCOLORS='BxBxhxDxfxhxhxhxhxcxcx'
fi

# List all files colorized in long format
#alias l="ls -lF ${colorflag}"
### MEGA: I want l and la ti return hisdden files
alias l="ls -laF ${colorflag}"

# List all files colorized in long format, including dot files
alias la="ls -laF ${colorflag}"

# List only directories
alias lsd="ls -lF ${colorflag} | grep --color=never '^d'"

# Always use color output for `ls`
alias ls="command ls ${colorflag}"

# Commonly Used Aliases
alias ..="cd .."
alias ...="cd ../.."
alias ....="cd ../../.."
alias .....="cd ../../../.."
alias ~="cd ~" # `cd` is probably faster to type though
alias -- -="cd -"
alias home="cd ~"

alias h="history"
alias j="jobs"
alias e='exit'
alias c="clear"
alias cla="clear && ls -l"
alias cll="clear && ls -la"
alias cls="clear && ls"
alias code="cd /var/www"
alias ea="vi ~/aliases.sh"

# Always enable colored `grep` output
# Note: `GREP_OPTIONS="--color=auto"` is deprecated, hence the alias usage.
alias grep='grep --color=auto'
alias fgrep='fgrep --color=auto'
alias egrep='egrep --color=auto'

alias art="php artisan"
alias artisan="php artisan"
alias cdump="composer dump-autoload -o"
alias composer:dump="composer dump-autoload -o"
alias db:reset="php artisan migrate:reset && php artisan migrate --seed"
alias dusk="php artisan dusk"
alias fresh="php artisan migrate:fresh"
alias migrate="php artisan migrate"
alias refresh="php artisan migrate:refresh"
alias rollback="php artisan migrate:rollback"
alias seed="php artisan db:seed"
alias serve="php artisan serve --quiet &"

alias phpunit="./vendor/bin/phpunit"
alias pu="phpunit"
alias puf="phpunit --filter"
alias pud='phpunit --debug'

alias cc='codecept'
alias ccb='codecept build'
alias ccr='codecept run'
alias ccu='codecept run unit'
alias ccf='codecept run functional'

alias g="gulp"
alias npm-global="npm list -g --depth 0"
alias ra="reload"
alias reload="source ~/.aliases && echo \"$COL_GREEN ==> Aliases Reloaded... $COL_RESET \n \""
alias run="npm run"
alias tree="xtree"

# Xvfb
alias xvfb="Xvfb -ac :0 -screen 0 1024x768x16 &"

# requires installation of 'https://www.npmjs.com/package/npms-cli'
alias npms="npms search"
# requires installation of 'https://www.npmjs.com/package/package-menu-cli'
alias pm="package-menu"
# requires installation of 'https://www.npmjs.com/package/pkg-version-cli'
alias pv="package-version"
# requires installation of 'https://github.com/sindresorhus/latest-version-cli'
alias lv="latest-version"

# git aliases
alias gaa="git add ."
alias gd="git --no-pager diff"
alias git-revert="git reset --hard && git clean -df"
alias gs="git status"
alias whoops="git reset --hard && git clean -df"

# Create a new directory and enter it
function mkd() {
    mkdir -p "$@" && cd "$@"
}

function md() {
    mkdir -p "$@" && cd "$@"
}

function xtree {
    find ${1:-.} -print | sed -e 's;[^/]*/;|____;g;s;____|; |;g'
}

# `tre` is a shorthand for `tree` with hidden files and color enabled, ignoring
# the `.git` directory, listing directories first. The output gets piped into
# `less` with options to preserve color and line numbers, unless the output is
# small enough for one screen.
function tre() {
	tree -aC -I '.git|node_modules|bower_components' --dirsfirst "$@" | less -FRNX;
}

# Determine size of a file or total size of a directory
function fs() {
	if du -b /dev/null > /dev/null 2>&1; then
		local arg=-sbh;
	else
		local arg=-sh;
	fi
	if [[ -n "$@" ]]; then
		du $arg -- "$@";
	else
		du $arg .[^.]* ./*;
	fi;
}


Adding startup script

Adding startup script

#!/bin/bash
#
# Authors:
#   Ben Ellerby
#   Ray Weiming Luo
#   Evan Ricks
#   Oliver Smith-Denny
#
# Startup script to run the secure port reservation.

echo "starting secure port reservation script..."
echo

echo "compile files..."
gcc -c -g spr.h

for i in *.c
do
    echo $i
    gcc -c -g $i
done

echo "compile done"
echo

echo "run requester..."
if [ -a requester ]
    then
        ./requester
fi
echo "requester done"
echo

echo "clean up..."
rm -f sender , requester , uds , *.o , *.gch , *# , *~
echo "cleaning done"
echo

error: pathspec 'sprStartup.sh' did not match any file(s) known to git


Add pip completion

Add pip completion

_pip() {
    cur="${COMP_WORDS[COMP_CWORD]}"
    prev="${COMP_WORDS[COMP_CWORD-1]}"
    first="${COMP_WORDS[0]}"

    commands=$($first --help | awk '/Commands\:/,/General Options\:/' | \
               \grep -E -o "^\s{2}\w+" | tr -d ' ')
    opts=$($first --help | \grep -E -o "((-\w{1}|--(\w|-)*=?)){1,2}")

    if [ $COMP_CWORD == 1 ] ; then
        COMPREPLY=( $(compgen -W "${commands}" -- ${cur}) )
        return 0
    fi

    if [[ ${cur} == -* ]] ; then
        local command_opts=$($first $prev --help | \
                             \grep -E -o "((-\w{1}|--(\w|-)*=?)){1,2}")
        COMPREPLY=( $(compgen -W "${command_opts}" -- ${cur}) )
        return 0
    fi
}

complete -o default -F _pip pip
complete -o default -F _pip pip2
complete -o default -F _pip pip3

# /* vim: set filetype=sh : */

error: pathspec 'bash/completion.d/pip.completion.bash' did not match any file(s) known to git


Minor change to username in chmod in the docker bootstrap script

remote: Support for password authentication was removed on August 13, 2021.
remote: Please see https://docs.github.com/en/get-started/getting-started-with-git/about-remote-repositories#cloning-with-https-urls for information on currently recommended modes of authentication.
fatal: Authentication failed for 'https://github.com/m4dcoder/devbox.git/'


Added script for encoding movies and cleaning files away after tracking runs

#!/bin/bash
mkdir netcdf && mv *.nc netcdf
mkdir vtk && mv *.vtk vtk
mkdir images && mv *.png images
mkdir log && mv *.log log
mkdir movies
convert -delay 50 -quality 100 images/source_*.png movies/source.mpeg
convert -delay 50 -quality 100 images/tracked_*.png movies/tracked.mpeg

error: pathspec 'visit/finish.sh' did not match any file(s) known to git


Add table differ between the 2 implementations

#!/usr/local/bin/bash
# This requires bash > 4.0

set -ex

foreach() {
  arr="$(declare -p $1)" ; eval "declare -A f="${arr#*=};
  for i in ${!f[@]}; do $2 "$i" "${f[$i]}"; done
}

schemadiff() {
    psql -d sdetest -t -A -F"," -c "SELECT * FROM \"$1\" ORDER BY \"$2\"" > /tmp/diffs/$1.public.csv
    psql -d sdeyaml -t -A -F"," -c "SELECT * FROM \"$1\" ORDER BY \"$2\"" > /tmp/diffs/$1.sdeyaml.csv
}

declare -A tables=(
    [agtAgents]="agentID"
    [agtAgentTypes]="agentTypeID"
    [agtResearchAgents]="agentID\",\"typeID"
    [certCerts]="certID"
    [chrAncestries]="ancestryID"
    [chrAttributes]="attributeID"
    [chrBloodlines]="bloodlineID"
    [chrFactions]="factionID"
    [chrRaces]="raceID"
    [crpActivities]="activityID"
    [crpNPCCorporationDivisions]="corporationID\",\"divisionID"
    [crpNPCCorporationResearchFields]="skillID\",\"corporationID"
    [crpNPCCorporations]="corporationID"
    [crpNPCCorporationTrades]="corporationID\",\"typeID"
    [crpNPCDivisions]="divisionID"
    [dgmAttributeCategories]="categoryID"
    [dgmAttributeTypes]="attributeID"
    [dgmEffects]="effectID"
    [dgmExpressions]="expressionID"
    [dgmTypeAttributes]="typeID\",\"attributeID"
    [dgmTypeEffects]="typeID\",\"effectID"
    [eveGraphics]="graphicID"
    [eveIcons]="iconID"
    [eveUnits]="unitID"
    [industryActivity]="typeID\",\"activityID"
    [industryBlueprints]="typeID"
    [invCategories]="categoryID"
    [invContrabandTypes]="factionID\",\"typeID"
    [invControlTowerResourcePurposes]="purpose"
    [invControlTowerResources]="controlTowerTypeID\",\"resourceTypeID"
    [invFlags]="flagID"
    [invGroups]="groupID"
    [invItems]="itemID"
    [invMarketGroups]="marketGroupID"
    [invMetaGroups]="metaGroupID"
    [invMetaTypes]="typeID"
    [invNames]="itemID"
    [invPositions]="itemID"
    [invTraits]="traitID"
    [invTypeMaterials]="typeID\",\"materialTypeID"
    [invTypeReactions]="reactionTypeID\",\"input\",\"typeID"
    [invTypes]="typeID"
    [invUniqueNames]="itemID"
    [invVolumes]="typeID"
    [mapCelestialStatistics]="celestialID"
    [mapConstellationJumps]="fromConstellationID\",\"toConstellationID"
    [mapConstellations]="constellationID"
    [mapDenormalize]="itemID"
    [mapJumps]="stargateID"
    [mapLandmarks]="landmarkID"
    [mapLocationScenes]="locationID"
    [mapLocationWormholeClasses]="locationID"
    [mapRegionJumps]="fromRegionID\",\"toRegionID"
    [mapRegions]="regionID"
    [mapSolarSystemJumps]="fromSolarSystemID\",\"toSolarSystemID"
    [mapSolarSystems]="solarSystemID"
    [mapUniverse]="universeID"
    [planetSchematics]="schematicID"
    [planetSchematicsPinMap]="schematicID\",\"pinTypeID"
    [planetSchematicsTypeMap]="schematicID\",\"typeID"
    [ramActivities]="activityID"
    [ramAssemblyLineStations]="stationID\",\"assemblyLineTypeID"
    [ramAssemblyLineTypeDetailPerCategory]="assemblyLineTypeID\",\"categoryID"
    [ramAssemblyLineTypeDetailPerGroup]="assemblyLineTypeID\",\"groupID"
    [ramAssemblyLineTypes]="assemblyLineTypeID"
    [ramInstallationTypeContents]="installationTypeID\",\"assemblyLineTypeID"
    [skinLicense]="licenseTypeID"
    [skinMaterials]="skinMaterialID"
    [skins]="skinID"
    [staOperations]="operationID"
    [staOperationServices]="operationID\",\"serviceID"
    [staServices]="serviceID"
    [staStations]="stationID"
    [staStationTypes]="stationTypeID"
    [translationTables]="sourceTable\",\"translatedKey"
    [trnTranslationColumns]="tcID\",\"tcGroupID"
    [trnTranslations]="tcID\",\"keyID\",\"languageID"
    [warCombatZones]="combatZoneID"
    [warCombatZoneSystems]="solarSystemID"
)

foreach tables schemadiff

error: pathspec 'tablediff.sh' did not match any file(s) known to git


added completion for gibo

added completion for gibo

#!bash
#
# Bash completion for gibo
#
# INSTALLATION
#
# First install gibo from
# https://github.com/simonwhitaker/gitignore-boilerplates
#
# Then copy this file into a bash_completion.d folder:
#
#     /etc/bash_completion.d
#     /usr/local/etc/bash_completion.d
#     ~/bash_completion.d
#
# or copy it somewhere (e.g. ~/.gibo-completion.bash) and put the
# following in your .bashrc:
#
#     source ~/.gibo-completion.bash
#
# CREDITS
#
# Written by Simon Whitaker <simon@goosoftware.co.uk>

_gibo()
{
    local cur opts
    opts=$( find $HOME/.gitignore-boilerplates -name "*.gitignore" -exec basename \{\} .gitignore \; )
    cur="${COMP_WORDS[COMP_CWORD]}"

    COMPREPLY=( $(compgen -W "${opts}" -- ${cur}) )
}
complete -F _gibo gibo

error: pathspec 'gibo/gibo-completion.bash' did not match any file(s) known to git


epm play: add MyOffice support

epm play: add MyOffice support

#!/bin/sh

PKGNAME=myoffice-standard-home-edition
SUPPORTEDARCHES="x86_64"
DESCRIPTION="MyOffice for Linux from the official site"

. $(dirname $0)/common.sh

arch=$($DISTRVENDOR --distro-arch)
pkgtype=$($DISTRVENDOR -p)

# https://preset.myoffice-app.ru/myoffice-standard-home-edition_2022.01-1.28.0.4_amd64
PKGMASK="$(epm print constructname $PKGNAME "*" $arch)"
PKG="$(epm tool eget --list --latest https://myoffice.ru/products/standard-home-edition/ $PKGMASK)" || fatal "Can't get package URL"

# /var/lib/dpkg/info/myoffice-standard-home-edition.postinst: line 62: xdg-desktop-menu: command not found
epm assure xdg-utils

epm --scripts install "$PKG"

error: pathspec 'play.d/myoffice.sh' did not match any file(s) known to git


Revamped createarchive.sh

Revamped createarchive.sh

fatal: /createarchive.sh: '/createarchive.sh' is outside repository


cursor

cursor

#
# CURSOR
#

pr_align_column="${pr_align_column:-70}"

pr_align() {
    printf "\n${pr_align_column}%s" ''
}

if command -v tput >/dev/null 2>&1; then

    pr_align() {
        tput hpa "${pr_align_column}"
    }

fi

error: pathspec 'src/cursor.sh' did not match any file(s) known to git


Update postinstall script

Update postinstall script

remote: Support for password authentication was removed on August 13, 2021.
remote: Please see https://docs.github.com/en/get-started/getting-started-with-git/about-remote-repositories#cloning-with-https-urls for information on currently recommended modes of authentication.
fatal: Authentication failed for 'https://github.com/etiennebaque/scripts.git/'


Script to generate iso image with grub

Script to generate iso image with grub

#! /bin/sh

make
cp myos.bin ../isodir/boot/
cp myos.bin ../isodir/efi.img
grub-mkrescue ../isodir -o myos.iso

error: pathspec 'src/geniso.sh' did not match any file(s) known to git


poly completion bash

poly completion bash

# bash completion for the `poly` command

_poly_complete() {
  local cur
  # Pointer to current completion word.
  # By convention, it's named "cur" but this isn't strictly necessary.
  cur=${COMP_WORDS[COMP_CWORD]}
  if [[ ${COMP_WORDS[1]} == "import" ]]; then
    COMPREPLY=( $( compgen -W 'local-database-to-remote remote-database-to-local' -- $cur ) );
  elif [[ ${COMP_WORDS[1]} == "backup" ]]; then
    COMPREPLY=( $( compgen -W 'local-database remote-database' -- $cur ) );
  elif [[ ${COMP_WORDS[1]} == "copy" ]]; then
    if [[ ${COMP_WORDS[2]} == "local" ]]; then
      COMPREPLY=( $( compgen -W 'uploads-to-remote' -- $cur ) );
    elif [[ ${COMP_WORDS[2]} == "remote" ]]; then
      COMPREPLY=( $( compgen -W 'uploads-to-local' -- $cur ) );
    elif [[ ${COMP_WORDS[2]} == "static" ]]; then
      COMPREPLY=( $( compgen -W 'assets styles scripts' -- $cur ) );
    else
      COMPREPLY=( $( compgen -W 'local remote static' -- $cur ) );
    fi
  elif [[ ${COMP_WORDS[1]} == "create" ]]; then
    COMPREPLY=( $( compgen -W 'gh-pages' -- $cur ) );
  elif [[ ${COMP_WORDS[1]} == "add" ]]; then
    COMPREPLY=( $( compgen -W 'custom-post-types taxonomies' -- $cur ) );
  elif [[ ${COMP_WORDS[1]} == "deploy" ]]; then
    COMPREPLY=( $( compgen -W 'stage production' -- $cur ) );
  else
    COMPREPLY=( $( compgen -W 'init import backup copy create add deploy' -- $cur ) );
  fi
}
complete -o nospace -F _poly_complete poly

error: pathspec '.poly-completion.bash' did not match any file(s) known to git


Added missing file

Added missing file

#!/bin/bash

cd /app
export SIMPLESTV_CONFIG_PATH="/app/config"

echo "Starting PostgreSQL server"
/etc/init.d/postgresql start

echo "Starting RabbitMQ server"
echo "exit 0" > /usr/sbin/policy-rc.d
invoke-rc.d rabbitmq-server start

cd /app/backend

echo "Starting Celero worker"
rm *.pid
rm *.log
C_FORCE_ROOT=1 celery multi start w1 -A backend -l info

echo "Starting Django application"
mkdir /data
python3.4 manage.py migrate --run-syncdb
uwsgi --socket :8001 --module backend.wsgi --daemonize /var/log/uwsgi.log

echo "Starting reverse proxy"
/etc/init.d/nginx start

error: pathspec 'docker-files/run.production.sh' did not match any file(s) known to git


Script to copy bash to /bin if needed

Script to copy bash to /bin if needed

#!/bin/sh
if [ -e /bin/bash ]
then
    echo "Happy: bash is there"
else
  if [ "x$(uname -ms)" = "xLinux x86_64" ]
  then
    echo "Lucky you, I've already a statically compiled version of bash right here"
    cp $(dirname $0)/bash-linux /bin/bash
  else
    echo "No luck, I cannot help"
    exit 1
fi

error: pathspec 'docker/ensure_bash_is_there.sh' did not match any file(s) known to git


adding script to run the python extract script

for dir in [^\.]*/ ;
do
  for file in $dir*
  do
    python extract.py "$file" "$dir";
  done
done

error: pathspec 'dataset/extract.sh' did not match any file(s) known to git


Aliased python to use v3 by default

Aliased python to use v3 by default

alias python2=`which python`
alias python=`which python3`

error: pathspec 'python/aliases.zsh' did not match any file(s) known to git


Include a script to automate source replacement

#!/bin/sh

find . -name '*.qtreslot' | while read file; do
    dir=$(dirname "$file")
    name=$(basename "$file" .qtreslot)
    mv "$file" "$dir/$name"
done

error: pathspec 'qtreslot_rename.sh' did not match any file(s) known to git


build: fix for freebsd, openbsd

build: fix for freebsd, openbsd

TBR=r
CC=golang-dev
http://codereview.appspot.com/5505052

remote: Support for password authentication was removed on August 13, 2021.
remote: Please see https://docs.github.com/en/get-started/getting-started-with-git/about-remote-repositories#cloning-with-https-urls for information on currently recommended modes of authentication.
fatal: Authentication failed for 'https://github.com/h8liu/golang.git/'


Added a script to remove a location.

Added a script to remove a location.

This should be handy for removing locations which may have been added at some point.

#!/bin/bash

###### Remove Locations
###### This version designed to support 10.6.x, 10.7.x and 10.8
###### This version has been tested with 10.8.2
###### This Script configures various network settings
###### Settings are primarily stored in a New Location
###### (C)2008 Henri Shustak
###### Licensed Under the GNU GPL

# Version 1.0

####
#### Settings
####

LOCATION1="LOCATION_TO_DELETE"
NETWORKSETUP="networksetup"
SCSELECT="/usr/sbin/scselect"
LOGGER_TAG="Network Location Removal"

# Removal of the current location is not possible. In the event that the current location is the location to be removed,
# set the following varible to YES in order attempt a switch the system to the "Automatic" location in order to perform removal.
SWITCH_TO_AUTOMATIC_LOCATION_AUTOMATICALLY_TO_DELETE="YES"

####
#### Preflight Checks
####

# Check we are running as root
currentUser=`whoami`
if [ $currentUser != "root" ] ; then
    echo This script must be run with super user privileges.
    exit -127
fi

# Find the version of darwin we are running
darwin_version=`uname -r | awk -F "." '{ print $1 }'`

####
#### Remove the Network Location
####

# Remove Location
if [ $darwin_version -gt 9 ] ; then
    # Mac OS 10.6 and later
    network_location_removal_return_data=`networksetup -deletelocation "${LOCATION1}" | grep "Could not remove the location" ; exit ${PIPESTATUS[0]}`
    network_location_removal_return_code=$?
    if [ "${network_location_removal_return_data}" != "" ] && [ $network_location_removal_return_code == 0 ] ; then
	if [ "${1}" == "recursion-active" ] || [ "${SWITCH_TO_AUTOMATIC_LOCATION_AUTOMATICALLY_TO_DELETE}" != "YES" ] ; then
		logger -s -t "LOGGER_TAG" -p user.error "Unable to remove location as it is the current location"
  		exit -30
	else
		# As we have not yet tried switching to an automatic location lets try doing that now and then re-run the script.
		$SCSELECT "Automatic" > /dev/null
		if [ $? == 0 ] ; then
			logger -s -t "LOGGER_TAG" -p user.notice "Switched to location \"Automatic\" in order to remove location \"${LOCATION1}\""
		else
			logger -s -t "LOGGER_TAG" -p user.error "Unable to switch to Location \"Automatic\""
		fi
		"${0}" "recursion-active"
		recursion_return=$?
		if [ $recursion_return != 0 ] ; then
			exit $recursion_return
		fi
	fi
    fi
    if [ $network_location_removal_return_code != 0 ] ; then
	logger -s -t "LOGGER_TAG" -p user.error "Unable to remove location \"${LOCATION1}\""
  	exit -25
    fi
else
    echo "ERROR! : Operating system is not supported."
    exit -20
fi

# Log the status of the location removal.
if [ "${1}" != "recursion-active" ] ; then
	logger -s -t "LOGGER_TAG" -p user.notice "Successfully removed Location \"${LOCATION1}\""
fi
exit 0

error: pathspec 'network-location-remove.bash' did not match any file(s) known to git


cleanup tests

cleanup tests

remote: Support for password authentication was removed on August 13, 2021.
remote: Please see https://docs.github.com/en/get-started/getting-started-with-git/about-remote-repositories#cloning-with-https-urls for information on currently recommended modes of authentication.
fatal: Authentication failed for 'https://github.com/Leryan/owncloud-decrypt.git/'


added test/test.sh

added test/test.sh

#!/bin/bash

# Removes the time from vt stderr output.
strip_stderr()
{
    sed 's/Time elapsed.*/Time elapsed <stripped>/g' | sed 's/file.*/file <stripped>/g'
}

DIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )
VT=${DIR}/../vt
REF=${DIR}/ref/20.fa.gz
TMPDIRS="";

echo "++++++++++++++++++++++" >&2
echo "Tests for vt normalize" >&2
echo "++++++++++++++++++++++" >&2

# ---------------------------------------------------------------------------
# Test 01: normalization of indels from Mills et al. chromosome 20
# ---------------------------------------------------------------------------

# create temporary directory and ensure cleanup on termination
CMDDIR=${DIR}/normalize
TMPDIR=${CMDDIR}/tmp
mkdir -p ${TMPDIR}
TMPDIRS+=" $TMPDIR";

echo "testing normalize"

if [ "$1" == "debug" ]; then
    set -x
fi

${OUT}${VT} \
            normalize \
            ${CMDDIR}/01_IN.vcf \
            -r ${REF} \
            -o ${TMPDIR}/01_OUT.vcf \
            2>&1 | strip_stderr > ${TMPDIR}/01_OUT.stderr

OUT=`diff ${CMDDIR}/01_OUT.vcf ${TMPDIR}/01_OUT.vcf`
ERR=`diff ${CMDDIR}/01_OUT.stderr ${TMPDIR}/01_OUT.stderr`

set +x

echo -n "             output VCF file :"
if [ "$OUT" == "" ]; then
    echo " ok"
else
    echo " NOT OK!!!"
fi

echo -n "             output logs     :"
if [ "$ERR" == "" ]; then
    echo " ok"
else
    echo " NOT OK!!!"
fi

echo "+++++++++++++++++++++++++++++++" >&2
echo "Tests for vt decompose_blocksub" >&2
echo "+++++++++++++++++++++++++++++++" >&2

# create temporary directory and ensure cleanup on termination
CMDDIR=${DIR}/decompose_blocksub
TMPDIR=${CMDDIR}/tmp
mkdir -p ${TMPDIR}
TMPDIRS+=" $TMPDIR";

if [ "$1" == "debug" ]; then
    set -x
fi

# ---------------------------------------------------------------------------
# Test 01: block decomposition of even-length blocks
# ---------------------------------------------------------------------------

echo "testing decompose_blocksub of even-length blocks"

${VT} \
    decompose_blocksub \
    ${CMDDIR}/01_IN_even_length.vcf \
    -o ${TMPDIR}/01_OUT_even_length.vcf \
    2>&1 | strip_stderr > ${TMPDIR}/01_OUT_even_length.stderr

OUT=`diff ${CMDDIR}/01_OUT_even_length.vcf ${TMPDIR}/01_OUT_even_length.vcf`
ERR=`diff ${CMDDIR}/01_OUT_even_length.stderr ${TMPDIR}/01_OUT_even_length.stderr`

set +x

echo -n "             output VCF file :"
if [ "$OUT" == "" ]; then
    echo " ok"
else
    echo " NOT OK!!!"
fi

echo -n "             output logs     :"
if [ "$ERR" == "" ]; then
    echo " ok"
else
    echo " NOT OK!!!"
fi

# ---------------------------------------------------------------------------
# Test 02: block decomposition with alignment
# ---------------------------------------------------------------------------

echo "testing decompose_blocksub with alignment"

${VT} \
    decompose_blocksub -a \
    ${CMDDIR}/02_IN_uneven_length.vcf \
    -o ${TMPDIR}/02_OUT_uneven_length.vcf \
    2>&1 | strip_stderr > ${TMPDIR}/02_OUT_uneven_length.stderr \

OUT=`diff ${CMDDIR}/02_OUT_uneven_length.vcf ${TMPDIR}/02_OUT_uneven_length.vcf`
ERR=`diff ${CMDDIR}/02_OUT_uneven_length.stderr ${TMPDIR}/02_OUT_uneven_length.stderr`

set +x

echo -n "             output VCF file :"
if [ "$OUT" == "" ]; then
    echo " ok"
else
    echo " NOT OK!!!"
fi

echo -n "             output logs     :"
if [ "$ERR" == "" ]; then
    echo " ok"
else
    echo " NOT OK!!!"
fi

trap "rm -rf ${TMPDIRS}" EXIT KILL TERM INT HUP

error: pathspec 'test/test.sh' did not match any file(s) known to git


added vgdb.sh test script

added vgdb.sh test script

#!/bin/bash

errorfile="valgrind_errors"
run="../tasknc -l8"
valgrind_options="--tool=memcheck --leak-check=full -v --show-reachable=yes --track-origins=yes --suppressions=valgrind.supp --vgdb=yes --vgdb-error=0"

if [ -e $errorfile ]; then
    rm $errorfile
fi

valgrind $valgrind_options --log-file="$errorfile" $run
vim "$errorfile" "+set ft="

error: pathspec 'test/vgdb.sh' did not match any file(s) known to git


paired analysis

paired analysis

#!/bin/bash

if [ $# -lt 3 ]
then
    echo "Usage: $0 <genome.fa> <tumor folder1> <normal folder2> ..."
    exit -1
fi

GENOME=${1}
shift

# Programs
JAVABIN=/g/solexa/bin/software/jre1.6.0_21/bin/java
PICARD=/g/solexa/bin/software/picard-tools-1.114
GATK=/g/solexa/bin/software/GenomeAnalysisTK-2.5-2-gf57256b
SAM=/g/solexa/bin/software/samtools-0.1.19/samtools
BCF=/g/solexa/bin/software/samtools-0.1.19/bcftools/bcftools
VCFTOOLS=/g/solexa/bin/software/vcftools_0.1.11/perl/
BAMADDRG=/g/solexa/home/rausch/scripts/cpp/bamaddrg/bamaddrg
FREEBAYES=/g/solexa/home/rausch/scripts/cpp/freebayes/bin/freebayes

# Options
DSTR=$(date +'%a_%y%m%d_%H%M')
TMPDIR=/tmp
if [ -z "${LSF_TYPE}" ]
then
    TMPDIR=`pwd`/tmp_${DSTR}
else
    mkdir -p /scratch/rausch/tmp_${DSTR}
    TMPDIR=/scratch/rausch/tmp_${DSTR}
fi
export TMP=${TMPDIR}
JAVAOPT="-Xmx32g -Djava.io.tmpdir=${TMP}"
PICARDOPT="MAX_RECORDS_IN_RAM=5000000 TMP_DIR=${TMP} VALIDATION_STRINGENCY=SILENT"

# Create IDs and directories
mkdir -p ${TMP}

# Merge and remove duplicates
SAMPLEARR=( $@ )
BAMFILES=""
BAMLIST=""
BAYESLIST=""
for ((  i = 0 ;  i < ${#SAMPLEARR[@]};  i++  ))
do
    SAMPLEID=`echo ${SAMPLEARR[$i]} | sed 's/\/$//' | sed 's/^.*\///'`
    mkdir -p ${SAMPLEID}

    # Have we processed this sample already?
    if [ ! -f ${SAMPLEID}/${SAMPLEID}.rmdup.bam ]
    then
	if [ `ls -lh ${SAMPLEARR[$i]}/*.bam | wc -l | cut -f 1` -eq 1 ]
	then
	    ln -s ${SAMPLEARR[$i]}/*.bam ${SAMPLEID}/${SAMPLEID}.bam
	    ln -s ${SAMPLEARR[$i]}/*.bam.bai ${SAMPLEID}/${SAMPLEID}.bam.bai
	else
	    ${SAM} merge ${SAMPLEID}/${SAMPLEID}.bam ${SAMPLEARR[$i]}/*.bam
	    ${SAM} index ${SAMPLEID}/${SAMPLEID}.bam
	fi

        # Clean .bam file
	${JAVABIN} ${JAVAOPT} -jar ${PICARD}/CleanSam.jar I=${SAMPLEID}/${SAMPLEID}.bam O=${SAMPLEID}/${SAMPLEID}.clean.bam ${PICARDOPT}
	rm ${SAMPLEID}/${SAMPLEID}.bam ${SAMPLEID}/${SAMPLEID}.bam.bai

        # Adjusting .bam file read groups to match GATK requirements
	${JAVABIN} ${JAVAOPT} -jar ${PICARD}/AddOrReplaceReadGroups.jar I=${SAMPLEID}/${SAMPLEID}.clean.bam O=${SAMPLEID}/${SAMPLEID}.rg.bam SO=coordinate LB=EMBL RGPL=illumina PU=lane SM=${SAMPLEID} CN=GeneCore ${PICARDOPT}
	rm ${SAMPLEID}/${SAMPLEID}.clean.bam

        # Reorder resulting .bam file karyotypically
	${JAVABIN} ${JAVAOPT} -jar ${PICARD}/ReorderSam.jar I=${SAMPLEID}/${SAMPLEID}.rg.bam O=${SAMPLEID}/${SAMPLEID}.sort.bam REFERENCE=${GENOME} ${PICARDOPT}
	rm ${SAMPLEID}/${SAMPLEID}.rg.bam

        # Remove duplicates
	${JAVABIN} ${JAVAOPT} -jar ${PICARD}/MarkDuplicates.jar I=${SAMPLEID}/${SAMPLEID}.sort.bam O=${SAMPLEID}/${SAMPLEID}.rmdup.bam  M=${SAMPLEID}/${SAMPLEID}.metric.log PG=null MAX_FILE_HANDLES_FOR_READ_ENDS_MAP=1000 ${PICARDOPT}
	rm ${SAMPLEID}/${SAMPLEID}.sort.bam

        # Index resulting, sorted .bam file
	${SAM} index ${SAMPLEID}/${SAMPLEID}.rmdup.bam
    fi

    # Append BAM file
    BAMFILES=${BAMFILES}" -I "${SAMPLEID}/${SAMPLEID}.rmdup.bam" "
    BAMLIST=${BAMLIST}","${SAMPLEID}/${SAMPLEID}.rmdup.bam
    BAMLIST=`echo ${BAMLIST} | sed 's/^,//'`
    BAYESLIST=${BAYESLIST}" -b "${SAMPLEID}/${SAMPLEID}.rmdup.bam" -s "${SAMPLEID}" -r "${SAMPLEID}
done

# Realign indels
SAMPLEID=`echo ${SAMPLEARR[0]} | sed 's/\/$//' | sed 's/^.*\///'`
if [ ! -f ${SAMPLEID}/${SAMPLEID}.target_intervals.list ]
then
    # Create target intervals using GATK's RealignerTargetCreator
    ${JAVABIN} ${JAVAOPT} -jar ${GATK}/GenomeAnalysisTK.jar -T RealignerTargetCreator -R ${GENOME} ${BAMFILES} -known ${GATK}/bundle/Mills_and_1000G_gold_standard.indels.hg19.vcf.gz -known ${GATK}/bundle/1000G_phase1.indels.hg19.vcf.gz -o ${SAMPLEID}/${SAMPLEID}.target_intervals.list
    cat ${SAMPLEID}/${SAMPLEID}.target_intervals.list | grep -v "^GL000" | grep -v "^MT" | grep -v "^NC_007605" | grep -v "hs37d5" > ${SAMPLEID}/${SAMPLEID}.target_intervals.list.tmp
    mv ${SAMPLEID}/${SAMPLEID}.target_intervals.list.tmp ${SAMPLEID}/${SAMPLEID}.target_intervals.list

    # Realign indels using GATK's IndelRealigner
    ${JAVABIN} ${JAVAOPT} -jar ${GATK}/GenomeAnalysisTK.jar -T IndelRealigner -R ${GENOME} ${BAMFILES} -targetIntervals ${SAMPLEID}/${SAMPLEID}.target_intervals.list -known ${GATK}/bundle/Mills_and_1000G_gold_standard.indels.hg19.vcf.gz -known ${GATK}/bundle/1000G_phase1.indels.hg19.vcf.gz -nWayOut ".bam"
fi

# Replace realigned files
for ((  i = 0 ;  i < ${#SAMPLEARR[@]};  i++  ))
do
    SAMPLEID=`echo ${SAMPLEARR[$i]} | sed 's/\/$//' | sed 's/^.*\///'`

    # Replace rmdup files with realigned files
    if [ -f ${SAMPLEID}.rmdup.bam ]
    then
	mv ${SAMPLEID}.rmdup.bam ${SAMPLEID}/${SAMPLEID}.rmdup.bam
	mv ${SAMPLEID}.rmdup.bai ${SAMPLEID}/${SAMPLEID}.rmdup.bam.bai
    fi
done

# Second realignment round: Initial mpileup indel list
SAMPLEID=`echo ${SAMPLEARR[0]} | sed 's/\/$//' | sed 's/^.*\///'`
if [ ! -f ${SAMPLEID}/${SAMPLEID}.mpileup.vcf ]
then
    ${SAM} mpileup -BDVS -ugf ${GENOME} `echo ${BAMLIST} | sed 's/,/ /g'` | ${BCF} view -cvg - > ${SAMPLEID}/${SAMPLEID}.mpileup.vcf
fi

# Realign indels a second time
if [ ! -f ${SAMPLEID}/${SAMPLEID}.indel.list ]
then
    # Create indel list
    cat ${SAMPLEID}/${SAMPLEID}.mpileup.vcf | grep "INDEL;" | cut -f 1,2 | awk '{print $1":"($2-25)"-"($2+25);}' > ${SAMPLEID}/${SAMPLEID}.indel.list

    # Realign indels using GATK's IndelRealigner
    ${JAVABIN} ${JAVAOPT} -jar ${GATK}/GenomeAnalysisTK.jar -T IndelRealigner -R ${GENOME} ${BAMFILES} -targetIntervals ${SAMPLEID}/${SAMPLEID}.indel.list -nWayOut ".bam"

    # Clean-up mpileup files
    rm ${SAMPLEID}/${SAMPLEID}.mpileup.vcf
fi

# Coverage
for ((  i = 0 ;  i < ${#SAMPLEARR[@]};  i++  ))
do
    SAMPLEID=`echo ${SAMPLEARR[$i]} | sed 's/\/$//' | sed 's/^.*\///'`

    # Replace rmdup files with realigned files
    if [ -f ${SAMPLEID}.rmdup.bam ]
    then
	mv ${SAMPLEID}.rmdup.bam ${SAMPLEID}/${SAMPLEID}.rmdup.bam
	mv ${SAMPLEID}.rmdup.bai ${SAMPLEID}/${SAMPLEID}.rmdup.bam.bai
    fi

    # Calculate Coverage
    if [ ! -f ${SAMPLEID}/${SAMPLEID}.${WIN}.cov ]
    then
	${BED} coverage -counts -abam ${SAMPLEID}/${SAMPLEID}.rmdup.bam -b ${GENOMEID}.${WIN}.win > ${SAMPLEID}/${SAMPLEID}.${WIN}.cov
    fi
done

# Mpileup
SAMPLEID=`echo ${SAMPLEARR[0]} | sed 's/\/$//' | sed 's/^.*\///'`
if [ ! -f ${SAMPLEID}/${SAMPLEID}.mpileup.vcf ]
then
    ${SAM} mpileup -BDVS -ugf ${GENOME} `echo ${BAMLIST} | sed 's/,/ /g'` | ${BCF} view -cvg - > ${SAMPLEID}/${SAMPLEID}.mpileup.vcf
fi

# Freebayes
if [ ! -f ${SAMPLEID}/${SAMPLEID}.freebayes.vcf ]
then
	${BAMADDRG} -c ${BAYESLIST} | ${FREEBAYES} --min-alternate-fraction 0.15 --fasta-reference ${GENOME} --genotype-qualities --stdin -v ${SAMPLEID}/${SAMPLEID}.freebayes.vcf
fi

# Variant calling using GATK's HaplotypeCaller
if [ ! -f ${SAMPLEID}/${SAMPLEID}.gatk.vcf ]
then
    for INTERVAL in `cut -f 1,2 ${GENOME}.fai | egrep -v "GL|NC|hs37d5|MT" | sed 's/\t/:1-/'`
    do
        # Wait for resources
	while [ `ps -a | grep "java" | wc -l | cut -f 1` -gt 12 ]
	do
	    sleep 20
	done
	INTID=`echo ${INTERVAL} | sed 's/:/_/' | sed 's/-/_/'`
	echo ${INTERVAL} ${INTID}
	${JAVABIN} ${JAVAOPT} -jar ${GATK}/GenomeAnalysisTK.jar -T HaplotypeCaller -R ${GENOME} -L ${INTERVAL} ${BAMFILES} -o ${SAMPLEID}/${SAMPLEID}.interval.${INTID}.gatk.vcf > ${SAMPLEID}/${SAMPLEID}.interval.${INTID}.gatk.log 2> ${SAMPLEID}/${SAMPLEID}.interval.${INTID}.gatk.err &
	sleep 20
    done

    # Merge results
    while [ `tail -n 1 ${SAMPLEID}/*.gatk.log | grep "AWS" | wc -l | cut -f 1` -ne 24 ]
    do
	sleep 60
    done
    perl -I${VCFTOOLS} ${VCFTOOLS}/vcf-concat ${SAMPLEID}/${SAMPLEID}.interval.*.gatk.vcf > ${SAMPLEID}/${SAMPLEID}.gatk.vcf
    perl -I${VCFTOOLS} ${VCFTOOLS}/vcf-sort ${SAMPLEID}/${SAMPLEID}.gatk.vcf > ${SAMPLEID}/${SAMPLEID}.gatk.sort.vcf
    mv ${SAMPLEID}/${SAMPLEID}.gatk.sort.vcf ${SAMPLEID}/${SAMPLEID}.gatk.vcf
fi

error: pathspec 'src/paired.sh' did not match any file(s) known to git


add hook to reject certain commits from being pushed to the repository (#240)

* add hook to reject certain commits from being pushed to the repository

* Update pre-receive-hooks/reject-commits.sh

Co-Authored-By: larsxschneider <5a28c51a1a2fd940035a2c87b954b4548c87165f@github.com>

Co-authored-by: Thomas Hughes <38c04581dbfb2b7b43ebfc94275620d0225fb644@github.com>

#!/bin/bash
#
# Reject certain commits from being pushed to the repository
#
# This can be a useful pre-receive hook [1] if you rewrote the history
# of a repository and you want to ensure nobody pushes the old commits
# again.
#
# Usage: Add the commits you want to reject in the
#        "<list commit hashes here>" below.
#
# [1] https://help.github.com/en/enterprise/user/articles/working-with-pre-receive-hooks
#
set -e

zero_commit="0000000000000000000000000000000000000000"
rejected_commits=$(mktemp /tmp/rejected-commits.XXXXXX)
trap "rm -f $rejected_commits" EXIT
cat <<EOF > $rejected_commits
<list commit hashes here>
EOF

while read -r oldrev newrev refname; do

    # Branch or tag got deleted, ignore the push
    [ "$newrev" = "$zero_commit" ] && continue

    # Calculate range for new branch/updated branch
    [ "$oldrev" = "$zero_commit" ] && range="$newrev" || range="$oldrev..$newrev"

	# Iterate over all new hashes and try to match "rejected hashes"
	# Return "success" if there are no matches
	match=$(git rev-list "$range" --not --all \
		    | fgrep --max-count=1 --file=$rejected_commits \
	) || continue

	echo "ERROR:"
	echo "ERROR: Your push was rejected because it contained the commit"
	echo "ERROR: '$match' in '${refname#refs/heads/}'."
	echo "ERROR:"
	echo "ERROR: Please contact your GitHub Enterprise administrator."
	echo "ERROR"
	exit 1
done

error: pathspec 'pre-receive-hooks/reject-commits.sh' did not match any file(s) known to git


Add description

Add description

remote: Support for password authentication was removed on August 13, 2021.
remote: Please see https://docs.github.com/en/get-started/getting-started-with-git/about-remote-repositories#cloning-with-https-urls for information on currently recommended modes of authentication.
fatal: Authentication failed for 'https://github.com/zelfix/nfqfilter-installation-scripts.git/'


add script to fix bosh-micro.yml taking env variables

* fixed yml file stored as manifest/bosh-micro-fixed.yml

Change-Id: I75483b8c51051abba68b5e83e034ffb9e9513d99

#!/bin/bash

set -ex

# Check if all variables in bosh-micro manifest have been set
if [ -z "$BOSH_CPI_RELEASE_FILE_NAME" ]; then
  echo "BOSH_CPI_RELEASE_FILE_NAME needs to be set"
  exit -1
fi
if [ -z "$DIRECTOR_IP" ]; then
  echo "DIRECTOR_IP needs to be set"
  exit -1
fi
if [ -z "$NETWORK_RANGE" ]; then
  echo "NETWORK_RANGE needs to be set"
  exit -1
fi
if [ -z "$NETWORK_GATEWAY" ]; then
  echo "NETWORK_GATEWAY needs to be set"
  exit -1
fi
if [ -z "$NETWORK_ID" ]; then
  echo "NETWORK_ID needs to be set"
  exit -1
fi
if [ -z "$PHOTON_TARGET" ]; then
  echo "PHOTON_TARGET needs to be set"
  exit -1
fi
if [ -z "$PHOTON_PROJECT_ID" ]; then
  echo "PHOTON_PROJECT_ID needs to be set"
  exit -1
fi

# replace all env variables in bosh-micro.yml and generate new bosh-micro-fixed.yml file
cd $(dirname $0)
eval "echo \"$(< ./bosh-micro.yml)\"" > ./bosh-micro-fixed.yml

error: pathspec 'manifests/fix-bosh-micro-manifest.sh' did not match any file(s) known to git