Reverse logic

Reverse logic

#!/usr/bin/env bash
# Copyright 2015 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================

set -e

function is_absolute {
  [[ "$1" = /* ]] || [[ "$1" =~ ^[a-zA-Z]:[/\\].* ]]
}

function real_path() {
  is_absolute "$1" && echo "$1" || echo "$PWD/${1#./}"
}

function cp_external() {
  local src_dir=$1
  local dest_dir=$2

  pushd .
  cd "$src_dir"
  for f in `find . ! -type d ! -name '*.py' ! -path '*local_config_cuda*' ! -path '*local_config_tensorrt*' ! -path '*local_config_syslibs*' ! -path '*org_tensorflow*'`; do
    mkdir -p "${dest_dir}/$(dirname ${f})"
    cp "${f}" "${dest_dir}/$(dirname ${f})/"
  done
  popd

  mkdir -p "${dest_dir}/local_config_cuda/cuda/cuda/"
  cp "${src_dir}/local_config_cuda/cuda/cuda/cuda_config.h" "${dest_dir}/local_config_cuda/cuda/cuda/"
}

function move_to_root_if_exists () {
  arg_to_move="$1"
  if [ -e "${arg_to_move}" ]; then
    mv ${arg_to_move} ./
  fi
}

function reorganize_includes() {
  TMPDIR="${1%/}"
  pushd "${TMPDIR}/tensorflow/include/"

  move_to_root_if_exists external/com_google_absl/absl

  move_to_root_if_exists external/eigen_archive/Eigen
  move_to_root_if_exists external/eigen_archive/unsupported

  move_to_root_if_exists external/jsoncpp_git/include
  rm -rf external/jsoncpp_git

  move_to_root_if_exists external/com_google_protobuf/src/google
  rm -rf external/com_google_protobuf/python

  popd
}

PLATFORM="$(uname -s | tr 'A-Z' 'a-z')"
function is_windows() {
  if [[ "${PLATFORM}" =~ (cygwin|mingw32|mingw64|msys)_nt* ]]; then
    true
  else
    false
  fi
}

function prepare_src() {
  if [ $# -lt 1 ] ; then
    echo "No destination dir provided"
    exit 1
  fi

  TMPDIR="${1%/}"
  mkdir -p "$TMPDIR"
  EXTERNAL_INCLUDES="${TMPDIR}/tensorflow/include/external"

  echo $(date) : "=== Preparing sources in dir: ${TMPDIR}"

  if [ ! -d bazel-bin/tensorflow ]; then
    echo "Could not find bazel-bin.  Did you run from the root of the build tree?"
    exit 1
  fi

  if is_windows; then
    rm -rf ./bazel-bin/tensorflow/tools/pip_package/simple_console_for_window_unzip
    mkdir -p ./bazel-bin/tensorflow/tools/pip_package/simple_console_for_window_unzip
    echo "Unzipping simple_console_for_windows.zip to create runfiles tree..."
    unzip -o -q ./bazel-bin/tensorflow/tools/pip_package/simple_console_for_windows.zip -d ./bazel-bin/tensorflow/tools/pip_package/simple_console_for_window_unzip
    echo "Unzip finished."
    # runfiles structure after unzip the python binary
    cp \
      bazel-bin/tensorflow/tools/pip_package/simple_console_for_window_unzip/runfiles/org_tensorflow/LICENSE \
      "${TMPDIR}"
    cp -R \
      bazel-bin/tensorflow/tools/pip_package/simple_console_for_window_unzip/runfiles/org_tensorflow/tensorflow \
      "${TMPDIR}"
    cp_external \
      bazel-bin/tensorflow/tools/pip_package/simple_console_for_window_unzip/runfiles \
      "${EXTERNAL_INCLUDES}/"
    RUNFILES=bazel-bin/tensorflow/tools/pip_package/simple_console_for_window_unzip/runfiles/org_tensorflow
  else
    RUNFILES=bazel-bin/tensorflow/tools/pip_package/build_pip_package.runfiles/org_tensorflow
    if [ -d bazel-bin/tensorflow/tools/pip_package/build_pip_package.runfiles/org_tensorflow/external ]; then
      # Old-style runfiles structure (--legacy_external_runfiles).
      cp \
        bazel-bin/tensorflow/tools/pip_package/build_pip_package.runfiles/org_tensorflow/LICENSE \
        "${TMPDIR}"
      cp -R \
        bazel-bin/tensorflow/tools/pip_package/build_pip_package.runfiles/org_tensorflow/tensorflow \
        "${TMPDIR}"
      cp_external \
        bazel-bin/tensorflow/tools/pip_package/build_pip_package.runfiles/org_tensorflow/external \
        "${EXTERNAL_INCLUDES}"
      # Copy MKL libs over so they can be loaded at runtime
      so_lib_dir=$(ls $RUNFILES | grep solib) || true
      if [ -n "${so_lib_dir}" ]; then
        mkl_so_dir=$(ls ${RUNFILES}/${so_lib_dir} | grep mkl) || true
        if [ -n "${mkl_so_dir}" ]; then
          mkdir "${TMPDIR}/${so_lib_dir}"
          cp -R ${RUNFILES}/${so_lib_dir}/${mkl_so_dir} "${TMPDIR}/${so_lib_dir}"
        fi
      fi
    else
      # New-style runfiles structure (--nolegacy_external_runfiles).
      cp \
        bazel-bin/tensorflow/tools/pip_package/build_pip_package.runfiles/org_tensorflow/LICENSE \
        "${TMPDIR}"
      cp -R \
        bazel-bin/tensorflow/tools/pip_package/build_pip_package.runfiles/org_tensorflow/tensorflow \
        "${TMPDIR}"
      cp_external \
        bazel-bin/tensorflow/tools/pip_package/build_pip_package.runfiles \
        "${EXTERNAL_INCLUDES}"
      # Copy MKL libs over so they can be loaded at runtime
      so_lib_dir=$(ls $RUNFILES | grep solib) || true
      if [ -n "${so_lib_dir}" ]; then
        mkl_so_dir=$(ls ${RUNFILES}/${so_lib_dir} | grep mkl) || true
        if [ -n "${mkl_so_dir}" ]; then
          mkdir "${TMPDIR}/${so_lib_dir}"
          cp -R ${RUNFILES}/${so_lib_dir}/${mkl_so_dir} "${TMPDIR}/${so_lib_dir}"
        fi
      fi
    fi
  fi

  mkdir -p ${TMPDIR}/third_party
  cp -R $RUNFILES/third_party/eigen3 ${TMPDIR}/third_party

  reorganize_includes "${TMPDIR}"

  cp tensorflow/tools/pip_package/MANIFEST.in ${TMPDIR}
  cp tensorflow/tools/pip_package/README ${TMPDIR}/README.md
  cp tensorflow/tools/pip_package/setup.py ${TMPDIR}

  rm -f ${TMPDIR}/tensorflow/libtensorflow_framework.so
  rm -f ${TMPDIR}/tensorflow/libtensorflow_framework.so.[0-9].*

  # In order to break the circular dependency between tensorflow and
  # tensorflow_estimator which forces us to do a multi-step release, we are
  # creating a virtual python package called tensorflow and moving all the tf
  # code into another python package called tensorflow_core:
  #
  #   * move code from tensorflow to tensorflow_core
  #   * create the virtual pip package: create folder and __init__.py file with
  #     needed code for transparent forwarding
  #
  # This is transparent to internal code or to code not using the pip packages.
  mv "${TMPDIR}/tensorflow" "${TMPDIR}/tensorflow_core"
  mkdir "${TMPDIR}/tensorflow"
  mv "${TMPDIR}/tensorflow_core/virtual_root.__init__.py" "${TMPDIR}/tensorflow/__init__.py"

  # In V1 API, we need to remove deprecation warnings from
  # ${TMPDIR}/tensorflow_core/__init__.py as those exists in
  # ${TMPDIR}/tensorflow/__init__.py which does an import* and if these don't
  # get removed users get 6 deprecation warning just on
  #
  #   import tensorflow as tf
  #
  # which is not ok. We disable deprecation by using sed to toggle the flag
  # TODO(mihaimaruseac): When we move the API to root, remove this hack
  # Note: Can't do in place sed that works on all OS, so use a temp file instead
  sed \
    "s/deprecation=True/deprecation=False/g" \
    "${TMPDIR}/tensorflow_core/__init__.py" > "${TMPDIR}/tensorflow_core/__init__.out"
  mv "${TMPDIR}/tensorflow_core/__init__.out" "${TMPDIR}/tensorflow_core/__init__.py"
}

function build_wheel() {
  if [ $# -lt 2 ] ; then
    echo "No src and dest dir provided"
    exit 1
  fi

  TMPDIR="$1"
  DEST="$2"
  PKG_NAME_FLAG="$3"

  # Before we leave the top-level directory, make sure we know how to
  # call python.
  if [[ -e tools/python_bin_path.sh ]]; then
    source tools/python_bin_path.sh
  fi

  pushd ${TMPDIR} > /dev/null

  rm -f MANIFEST
  echo $(date) : "=== Building wheel"
  "${PYTHON_BIN_PATH:-python}" setup.py bdist_wheel ${PKG_NAME_FLAG} >/dev/null
  mkdir -p ${DEST}
  cp dist/* ${DEST}
  popd > /dev/null
  echo $(date) : "=== Output wheel file is in: ${DEST}"
}

function usage() {
  echo "Usage:"
  echo "$0 [--src srcdir] [--dst dstdir] [options]"
  echo "$0 dstdir [options]"
  echo ""
  echo "    --src                 prepare sources in srcdir"
  echo "                              will use temporary dir if not specified"
  echo ""
  echo "    --dst                 build wheel in dstdir"
  echo "                              if dstdir is not set do not build, only prepare sources"
  echo ""
  echo "  Options:"
  echo "    --project_name <name> set project name to name"
  echo "    --cpu                 build tensorflow_cpu"
  echo "    --gpu                 build tensorflow_gpu"
  echo "    --gpudirect           build tensorflow_gpudirect"
  echo "    --rocm                build tensorflow_rocm"
  echo "    --nightly_flag        build tensorflow nightly"
  echo ""
  exit 1
}

function main() {
  PKG_NAME_FLAG=""
  PROJECT_NAME=""
  CPU_BUILD=0
  GPU_BUILD=0
  GPUDIRECT_BUILD=0
  ROCM_BUILD=0
  NIGHTLY_BUILD=0
  SRCDIR=""
  DSTDIR=""
  CLEANSRC=1
  while true; do
    if [[ "$1" == "--help" ]]; then
      usage
      exit 1
    elif [[ "$1" == "--nightly_flag" ]]; then
      NIGHTLY_BUILD=1
    elif [[ "$1" == "--gpu" ]]; then
      GPU_BUILD=1
    elif [[ "$1" == "--cpu" ]]; then
      CPU_BUILD=1
    elif [[ "$1" == "--gpudirect" ]]; then
      GPUDIRECT_BUILD=1
    elif [[ "$1" == "--rocm" ]]; then
      ROCM_BUILD=1
    elif [[ "$1" == "--project_name" ]]; then
      shift
      if [[ -z "$1" ]]; then
        break
      fi
      PROJECT_NAME="$1"
    elif [[ "$1" == "--src" ]]; then
      shift
      SRCDIR="$(real_path $1)"
      CLEANSRC=0
    elif [[ "$1" == "--dst" ]]; then
      shift
      DSTDIR="$(real_path $1)"
    else
      DSTDIR="$(real_path $1)"
    fi
    shift

    if [[ -z "$1" ]]; then
      break
    fi
  done

  if [[ $(( GPU_BUILD + CPU_BUILD + GPUDIRECT_BUILD + ROCM_BUILD )) -gt "1" ]]; then
    echo "Only one of [--gpu, --cpu, --gpudirect, --rocm] may be provided."
    usage
    exit 1
  fi

  if [[ -z "$DSTDIR" ]] && [[ -z "$SRCDIR" ]]; then
    echo "No destination dir provided"
    usage
    exit 1
  fi

  if [[ -z "$SRCDIR" ]]; then
    # make temp srcdir if none set
    SRCDIR="$(mktemp -d -t tmp.XXXXXXXXXX)"
  fi

  prepare_src "$SRCDIR"

  if [[ -z "$DSTDIR" ]]; then
      # only want to prepare sources
      exit
  fi

  if [[ -n ${PROJECT_NAME} ]]; then
    PKG_NAME_FLAG="--project_name ${PROJECT_NAME}"
  elif [[ ${NIGHTLY_BUILD} == "1" && ${GPU_BUILD} == "1" ]]; then
    PKG_NAME_FLAG="--project_name tf_nightly_gpu"
  elif [[ ${NIGHTLY_BUILD} == "1" && ${GPUDIRECT_BUILD} == "1" ]]; then
    PKG_NAME_FLAG="--project_name tf_nightly_gpudirect"
  elif [[ ${NIGHTLY_BUILD} == "1" && ${ROCM_BUILD} == "1" ]]; then
    PKG_NAME_FLAG="--project_name tf_nightly_rocm"
  elif [[ ${NIGHTLY_BUILD} == "1" && ${CPU_BUILD} == "1" ]]; then
    PKG_NAME_FLAG="--project_name tf_nightly_cpu"
  elif [[ ${NIGHTLY_BUILD} == "1" ]]; then
    PKG_NAME_FLAG="--project_name tf_nightly"
  elif [[ ${GPU_BUILD} == "1" ]]; then
    PKG_NAME_FLAG="--project_name tensorflow_gpu"
  elif [[ ${GPUDIRECT_BUILD} == "1" ]]; then
    PKG_NAME_FLAG="--project_name tensorflow_gpudirect"
  elif [[ ${ROCM_BUILD} == "1" ]]; then
    PKG_NAME_FLAG="--project_name tensorflow_rocm"
  elif [[ ${CPU_BUILD} == "1" ]]; then
    PKG_NAME_FLAG="--project_name tensorflow_cpu"
  fi

  build_wheel "$SRCDIR" "$DSTDIR" "$PKG_NAME_FLAG"

  if [[ $CLEANSRC -ne 0 ]]; then
    rm -rf "${TMPDIR}"
  fi
}

main "$@"

#!/usr/bin/env bash
# Copyright 2015 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================

set -e

function is_absolute {
  [[ "$1" = /* ]] || [[ "$1" =~ ^[a-zA-Z]:[/\\].* ]]
}

function real_path() {
  is_absolute "$1" && echo "$1" || echo "$PWD/${1#./}"
}

function cp_external() {
  local src_dir=$1
  local dest_dir=$2

  pushd .
  cd "$src_dir"
  for f in `find . ! -type d ! -name '*.py' ! -path '*local_config_cuda*' ! -path '*local_config_tensorrt*' ! -path '*local_config_syslibs*' ! -path '*org_tensorflow*'`; do
    mkdir -p "${dest_dir}/$(dirname ${f})"
    cp "${f}" "${dest_dir}/$(dirname ${f})/"
  done
  popd

  mkdir -p "${dest_dir}/local_config_cuda/cuda/cuda/"
  cp "${src_dir}/local_config_cuda/cuda/cuda/cuda_config.h" "${dest_dir}/local_config_cuda/cuda/cuda/"
}

function move_to_root_if_exists () {
  arg_to_move="$1"
  if [ -e "${arg_to_move}" ]; then
    mv ${arg_to_move} ./
  fi
}

function reorganize_includes() {
  TMPDIR="${1%/}"
  pushd "${TMPDIR}/tensorflow/include/"

  move_to_root_if_exists external/com_google_absl/absl

  move_to_root_if_exists external/eigen_archive/Eigen
  move_to_root_if_exists external/eigen_archive/unsupported

  move_to_root_if_exists external/jsoncpp_git/include
  rm -rf external/jsoncpp_git

  move_to_root_if_exists external/com_google_protobuf/src/google
  rm -rf external/com_google_protobuf/python

  popd
}

PLATFORM="$(uname -s | tr 'A-Z' 'a-z')"
function is_windows() {
  if [[ "${PLATFORM}" =~ (cygwin|mingw32|mingw64|msys)_nt* ]]; then
    true
  else
    false
  fi
}

function prepare_src() {
  if [ $# -lt 1 ] ; then
    echo "No destination dir provided"
    exit 1
  fi

  TMPDIR="${1%/}"
  mkdir -p "$TMPDIR"
  EXTERNAL_INCLUDES="${TMPDIR}/tensorflow/include/external"

  echo $(date) : "=== Preparing sources in dir: ${TMPDIR}"

  if [ ! -d bazel-bin/tensorflow ]; then
    echo "Could not find bazel-bin.  Did you run from the root of the build tree?"
    exit 1
  fi

  if is_windows; then
    rm -rf ./bazel-bin/tensorflow/tools/pip_package/simple_console_for_window_unzip
    mkdir -p ./bazel-bin/tensorflow/tools/pip_package/simple_console_for_window_unzip
    echo "Unzipping simple_console_for_windows.zip to create runfiles tree..."
    unzip -o -q ./bazel-bin/tensorflow/tools/pip_package/simple_console_for_windows.zip -d ./bazel-bin/tensorflow/tools/pip_package/simple_console_for_window_unzip
    echo "Unzip finished."
    # runfiles structure after unzip the python binary
    cp \
      bazel-bin/tensorflow/tools/pip_package/simple_console_for_window_unzip/runfiles/org_tensorflow/LICENSE \
      "${TMPDIR}"
    cp -R \
      bazel-bin/tensorflow/tools/pip_package/simple_console_for_window_unzip/runfiles/org_tensorflow/tensorflow \
      "${TMPDIR}"
    cp_external \
      bazel-bin/tensorflow/tools/pip_package/simple_console_for_window_unzip/runfiles \
      "${EXTERNAL_INCLUDES}/"
    RUNFILES=bazel-bin/tensorflow/tools/pip_package/simple_console_for_window_unzip/runfiles/org_tensorflow
  else
    RUNFILES=bazel-bin/tensorflow/tools/pip_package/build_pip_package.runfiles/org_tensorflow
    if [ -d bazel-bin/tensorflow/tools/pip_package/build_pip_package.runfiles/org_tensorflow/external ]; then
      # Old-style runfiles structure (--legacy_external_runfiles).
      cp \
        bazel-bin/tensorflow/tools/pip_package/build_pip_package.runfiles/org_tensorflow/LICENSE \
        "${TMPDIR}"
      cp -R \
        bazel-bin/tensorflow/tools/pip_package/build_pip_package.runfiles/org_tensorflow/tensorflow \
        "${TMPDIR}"
      cp_external \
        bazel-bin/tensorflow/tools/pip_package/build_pip_package.runfiles/org_tensorflow/external \
        "${EXTERNAL_INCLUDES}"
      # Copy MKL libs over so they can be loaded at runtime
      so_lib_dir=$(ls $RUNFILES | grep solib) || true
      if [ -n "${so_lib_dir}" ]; then
        mkl_so_dir=$(ls ${RUNFILES}/${so_lib_dir} | grep mkl) || true
        if [ -n "${mkl_so_dir}" ]; then
          mkdir "${TMPDIR}/${so_lib_dir}"
          cp -R ${RUNFILES}/${so_lib_dir}/${mkl_so_dir} "${TMPDIR}/${so_lib_dir}"
        fi
      fi
    else
      # New-style runfiles structure (--nolegacy_external_runfiles).
      cp \
        bazel-bin/tensorflow/tools/pip_package/build_pip_package.runfiles/org_tensorflow/LICENSE \
        "${TMPDIR}"
      cp -R \
        bazel-bin/tensorflow/tools/pip_package/build_pip_package.runfiles/org_tensorflow/tensorflow \
        "${TMPDIR}"
      cp_external \
        bazel-bin/tensorflow/tools/pip_package/build_pip_package.runfiles \
        "${EXTERNAL_INCLUDES}"
      # Copy MKL libs over so they can be loaded at runtime
      so_lib_dir=$(ls $RUNFILES | grep solib) || true
      if [ -n "${so_lib_dir}" ]; then
        mkl_so_dir=$(ls ${RUNFILES}/${so_lib_dir} | grep mkl) || true
        if [ -n "${mkl_so_dir}" ]; then
          mkdir "${TMPDIR}/${so_lib_dir}"
          cp -R ${RUNFILES}/${so_lib_dir}/${mkl_so_dir} "${TMPDIR}/${so_lib_dir}"
        fi
      fi
    fi
  fi

  mkdir -p ${TMPDIR}/third_party
  cp -R $RUNFILES/third_party/eigen3 ${TMPDIR}/third_party

  reorganize_includes "${TMPDIR}"

  cp tensorflow/tools/pip_package/MANIFEST.in ${TMPDIR}
  cp tensorflow/tools/pip_package/README ${TMPDIR}/README.md
  cp tensorflow/tools/pip_package/setup.py ${TMPDIR}

  rm -f ${TMPDIR}/tensorflow/libtensorflow_framework.so
  rm -f ${TMPDIR}/tensorflow/libtensorflow_framework.so.[0-9].*

  # In order to break the circular dependency between tensorflow and
  # tensorflow_estimator which forces us to do a multi-step release, we are
  # creating a virtual python package called tensorflow and moving all the tf
  # code into another python package called tensorflow_core:
  #
  #   * move code from tensorflow to tensorflow_core
  #   * create the virtual pip package: create folder and __init__.py file with
  #     needed code for transparent forwarding
  #
  # This is transparent to internal code or to code not using the pip packages.
  mv "${TMPDIR}/tensorflow" "${TMPDIR}/tensorflow_core"
  mkdir "${TMPDIR}/tensorflow"
  mv "${TMPDIR}/tensorflow_core/virtual_root.__init__.py" "${TMPDIR}/tensorflow/__init__.py"

  # In V1 API, we need to remove deprecation warnings from
  # ${TMPDIR}/tensorflow_core/__init__.py as those exists in
  # ${TMPDIR}/tensorflow/__init__.py which does an import* and if these don't
  # get removed users get 6 deprecation warning just on
  #
  #   import tensorflow as tf
  #
  # which is not ok. We disable deprecation by using sed to toggle the flag
  # TODO(mihaimaruseac): When we move the API to root, remove this hack
  # Note: Can't do in place sed that works on all OS, so use a temp file instead
  sed \
    "s/deprecation=True/deprecation=False/g" \
    "${TMPDIR}/tensorflow_core/__init__.py" > "${TMPDIR}/tensorflow_core/__init__.out"
  mv "${TMPDIR}/tensorflow_core/__init__.out" "${TMPDIR}/tensorflow_core/__init__.py"
}

function build_wheel() {
  if [ $# -lt 2 ] ; then
    echo "No src and dest dir provided"
    exit 1
  fi

  TMPDIR="$1"
  DEST="$2"
  PKG_NAME_FLAG="$3"

  # Before we leave the top-level directory, make sure we know how to
  # call python.
  if [[ -e tools/python_bin_path.sh ]]; then
    source tools/python_bin_path.sh
  fi

  pushd ${TMPDIR} > /dev/null

  rm -f MANIFEST
  echo $(date) : "=== Building wheel"
  "${PYTHON_BIN_PATH:-python}" setup.py bdist_wheel ${PKG_NAME_FLAG} >/dev/null
  mkdir -p ${DEST}
  cp dist/* ${DEST}
  popd > /dev/null
  echo $(date) : "=== Output wheel file is in: ${DEST}"
}

function usage() {
  echo "Usage:"
  echo "$0 [--src srcdir] [--dst dstdir] [options]"
  echo "$0 dstdir [options]"
  echo ""
  echo "    --src                 prepare sources in srcdir"
  echo "                              will use temporary dir if not specified"
  echo ""
  echo "    --dst                 build wheel in dstdir"
  echo "                              if dstdir is not set do not build, only prepare sources"
  echo ""
  echo "  Options:"
  echo "    --project_name <name> set project name to name"
  echo "    --cpu                 build tensorflow_cpu"
  echo "    --gpu                 build tensorflow_gpu"
  echo "    --gpudirect           build tensorflow_gpudirect"
  echo "    --rocm                build tensorflow_rocm"
  echo "    --nightly_flag        build tensorflow nightly"
  echo ""
  exit 1
}

function main() {
  PKG_NAME_FLAG=""
  PROJECT_NAME=""
  CPU_BUILD=0
  GPU_BUILD=0
  GPUDIRECT_BUILD=0
  ROCM_BUILD=0
  NIGHTLY_BUILD=0
  SRCDIR=""
  DSTDIR=""
  CLEANSRC=1
  while true; do
    if [[ "$1" == "--help" ]]; then
      usage
      exit 1
    elif [[ "$1" == "--nightly_flag" ]]; then
      NIGHTLY_BUILD=1
    elif [[ "$1" == "--gpu" ]]; then
      GPU_BUILD=1
    elif [[ "$1" == "--cpu" ]]; then
      CPU_BUILD=1
    elif [[ "$1" == "--gpudirect" ]]; then
      GPUDIRECT_BUILD=1
    elif [[ "$1" == "--rocm" ]]; then
      ROCM_BUILD=1
    elif [[ "$1" == "--project_name" ]]; then
      shift
      if [[ -z "$1" ]]; then
        break
      fi
      PROJECT_NAME="$1"
    elif [[ "$1" == "--src" ]]; then
      shift
      SRCDIR="$(real_path $1)"
      CLEANSRC=0
    elif [[ "$1" == "--dst" ]]; then
      shift
      DSTDIR="$(real_path $1)"
    else
      DSTDIR="$(real_path $1)"
    fi
    shift

    if [[ -z "$1" ]]; then
      break
    fi
  done

  if [[ $(( GPU_BUILD + CPU_BUILD + GPUDIRECT_BUILD + ROCM_BUILD )) -le "1" ]]; then
    echo "Only one of [--gpu, --cpu, --gpudirect, --rocm] may be provided."
    usage
    exit 1
  fi

  if [[ -z "$DSTDIR" ]] && [[ -z "$SRCDIR" ]]; then
    echo "No destination dir provided"
    usage
    exit 1
  fi

  if [[ -z "$SRCDIR" ]]; then
    # make temp srcdir if none set
    SRCDIR="$(mktemp -d -t tmp.XXXXXXXXXX)"
  fi

  prepare_src "$SRCDIR"

  if [[ -z "$DSTDIR" ]]; then
      # only want to prepare sources
      exit
  fi

  if [[ -n ${PROJECT_NAME} ]]; then
    PKG_NAME_FLAG="--project_name ${PROJECT_NAME}"
  elif [[ ${NIGHTLY_BUILD} == "1" && ${GPU_BUILD} == "1" ]]; then
    PKG_NAME_FLAG="--project_name tf_nightly_gpu"
  elif [[ ${NIGHTLY_BUILD} == "1" && ${GPUDIRECT_BUILD} == "1" ]]; then
    PKG_NAME_FLAG="--project_name tf_nightly_gpudirect"
  elif [[ ${NIGHTLY_BUILD} == "1" && ${ROCM_BUILD} == "1" ]]; then
    PKG_NAME_FLAG="--project_name tf_nightly_rocm"
  elif [[ ${NIGHTLY_BUILD} == "1" && ${CPU_BUILD} == "1" ]]; then
    PKG_NAME_FLAG="--project_name tf_nightly_cpu"
  elif [[ ${NIGHTLY_BUILD} == "1" ]]; then
    PKG_NAME_FLAG="--project_name tf_nightly"
  elif [[ ${GPU_BUILD} == "1" ]]; then
    PKG_NAME_FLAG="--project_name tensorflow_gpu"
  elif [[ ${GPUDIRECT_BUILD} == "1" ]]; then
    PKG_NAME_FLAG="--project_name tensorflow_gpudirect"
  elif [[ ${ROCM_BUILD} == "1" ]]; then
    PKG_NAME_FLAG="--project_name tensorflow_rocm"
  elif [[ ${CPU_BUILD} == "1" ]]; then
    PKG_NAME_FLAG="--project_name tensorflow_cpu"
  fi

  build_wheel "$SRCDIR" "$DSTDIR" "$PKG_NAME_FLAG"

  if [[ $CLEANSRC -ne 0 ]]; then
    rm -rf "${TMPDIR}"
  fi
}

main "$@"


Script now commits just before pushing

Script now commits just before pushing

remote: Support for password authentication was removed on August 13, 2021.
remote: Please see https://docs.github.com/en/get-started/getting-started-with-git/about-remote-repositories#cloning-with-https-urls for information on currently recommended modes of authentication.
fatal: Authentication failed for 'https://github.com/StuyCSDojo/Scanner.git/'


Update brew_install.sh

Update brew_install.sh

remote: Support for password authentication was removed on August 13, 2021.
remote: Please see https://docs.github.com/en/get-started/getting-started-with-git/about-remote-repositories#cloning-with-https-urls for information on currently recommended modes of authentication.
fatal: Authentication failed for 'https://github.com/steinbaugh/steinbaugh.github.io.git/'


Add bashunit.sh

Add bashunit.sh

######################################################################
### bashunit.sh
######################################################################

[[ -z "$BASH" || "$__BASHUNIT_SH__" == 'true' ]] && return

declare -r __BASHUNIT_SH__='true'
declare -a __TEST_FUNCTIONS__=()

shopt -s expand_aliases

function regist_test(){
	__TEST_FUNCTIONS__=( ${__TEST_FUNCTIONS__[@]} $1 )
}

function regist_all_tests(){
	__TEST_FUNCTIONS__=( $(typeset -F | awk '/^declare -f '"${1:-test_}"'.+$/ {print $3}') )
}

function run_tests(){
	(
		local         _setup_once_function=$(typeset -F setup_once        )
		local      _teardown_once_function=$(typeset -F teardown_once     )
		local    _setup_everytime_function=$(typeset -F setup_everytime   )
		local _teardown_everytime_function=$(typeset -F teardown_everytime)
		local _test_function _test_cnt=0 _success_cnt=0 _failure_cnt=0 _failure_record_file

		_failure_record_file=$(mktemp) || return $?

		trap 'exit 2' 1 2 3 15
		trap 'rm -f -- "$_failure_record_file"' 0

		echo -e "$(uname -a)\n$BASH $BASH_VERSION (${BASH_VERSINFO[@]})\n\nExecuting $0 ...\n"

		$_setup_once_function

		for _test_function in "${__TEST_FUNCTIONS__[@]}"; do
			$_setup_everytime_function

			printf -- "%-65s" "$_test_function"
			$_test_function

			((_test_cnt++))
			if is_failed; then
				echo '[NG]'; print_failure; clear_failure
				((_failure_cnt++))
			else
				echo '[OK]'
				((_success_cnt++))
			fi

			$_teardown_everytime_function
		done

		$_teardown_once_function

		echo -e "\nRan $_test_cnt test(s) (Passed $_success_cnt / Failed $_failure_cnt)"

		exit $((_failure_cnt == 0 ? 0 : 1))
	)
}

function _assert(){
	eval "$2" && return 0

	record_failure 'Line: %d\nFailed: [%s]\n\n' "$1" "$2"

	return 1
}
alias assert='_assert "$LINENO"'

function _assert_eq(){
	[[ "$2" -eq "$3" ]] && return 0

	record_failure 'Line: %d\nExpected number: [%s]\nActual number:   [%s]\n\n' "$1" "$2" "$3"

	return 1
}
alias assert_eq='_assert_eq "$LINENO"'

function _assert_ne(){
	[[ "$2" -ne "$3" ]] && return 0

	record_failure 'Line: %d\nUnexpected number: [%s]\nActual number:     [%s]\n\n' "$1" "$2" "$3"

	return 1
}
alias assert_ne='_assert_ne "$LINENO"'

function _assert_equal(){
	[[ "$2" == "$3" ]] && return 0

	record_failure 'Line: %d\nExpected string: [%s]\nActual string:   [%s]\n\n' "$1" "$2" "$3"

	return 1
}
alias assert_equal='_assert_equal "$LINENO"'

function _assert_not_equal(){
	[[ "$2" != "$3" ]] && return 0

	record_failure 'Line: %d\nUnexpected string: [%s]\nActual string:     [%s]\n\n' "$1" "$2" "$3"

	return 1
}
alias assert_not_equal='_assert_not_equal "$LINENO"'

function _assert_equal_pattern(){
	[[ "$3" == $2 ]] && return 0

	record_failure 'Line: %d\nExpected string: [%s]\nActual string:   [%s]\n\n' "$1" "$2" "$3"

	return 1
}
alias assert_equal_pattern='_assert_equal_pattern "$LINENO"'

function _assert_not_equal_pattern(){
	[[ "$3" != $2 ]] && return 0

	record_failure 'Line: %d\nUnexpected string: [%s]\nActual string:     [%s]\n\n' "$1" "$2" "$3"

	return 1
}
alias assert_not_equal_pattern='_assert_not_equal_pattern "$LINENO"'

function record_failure(){
	printf -- "$@" >> "$_failure_record_file"
}

function print_failure(){
	cat -- "$_failure_record_file"
}

function clear_failure(){
	: >| "$_failure_record_file"
}

function is_failed(){
	[[ -s "$_failure_record_file" ]] && return 0
	return 1
}

######################################################################
### Local Variables:
### mode: shell-script
### coding: utf-8-unix
### tab-width: 4
### End:
######################################################################

error: pathspec 'bashunit.sh' did not match any file(s) known to git


first commit

first commit

#!/bin/bash

clear

echo "syncing DAM files..."

mv /home/archivesuser/drmc_dam_bulk/Assets/* /home/archivesuser/dam_watched_sync/WatchFolder
cp /home/archivesuser/drmc_dam_bulk/CSV/metadata.csv /home/archivesuser/drmc_dam_bulk/csvbackups/$(date -d "today" +"%Y%m%d%H%M").metadata.csv
mv /home/archivesuser/drmc_dam_bulk/CSV/metadata.csv /home/archivesuser/dam_watched_sync/CSV

echo "done"

error: pathspec 'dam_sync.sh' did not match any file(s) known to git


Add test for max length filter (#46)

Add test for max length filter (#46)

#!/bin/bash -e
#
# Confirms that GitHub issue #41 is fixed
#
# https://github.com/dharple/detox/issues/41
#

if [ -z "$TESTBASE" ] ; then
	echo TESTBASE needs to be passed via the env
	exit 1
fi

. $TESTBASE/test-functions

DETOX=$1
TABLEPATH=$(dirname $TESTBASE)/table

# ---------------------------------------------------------------------------

INPUT="safe.txt"
OUTPUT=$INPUT
METHOD1=max-length-32

test_sequence "$DETOX" "$INPUT" "$OUTPUT" "$TABLEPATH" "$METHOD1"

# ---------------------------------------------------------------------------

INPUT="safe and stuff.txt"
OUTPUT="safe and.txt"
METHOD1=max-length-12

test_sequence "$DETOX" "$INPUT" "$OUTPUT" "$TABLEPATH" "$METHOD1"

# ---------------------------------------------------------------------------

INPUT="safe and stuff.txt"
OUTPUT="safe.txt"
METHOD1=max-length-8

test_sequence "$DETOX" "$INPUT" "$OUTPUT" "$TABLEPATH" "$METHOD1"

# ---------------------------------------------------------------------------

INPUT="safe.tar.gz"
OUTPUT=$INPUT
METHOD1=max-length-32

test_sequence "$DETOX" "$INPUT" "$OUTPUT" "$TABLEPATH" "$METHOD1"

# ---------------------------------------------------------------------------

INPUT="safe and stuff.tar.gz"
OUTPUT="safe .tar.gz"
METHOD1=max-length-12

test_sequence "$DETOX" "$INPUT" "$OUTPUT" "$TABLEPATH" "$METHOD1"

# ---------------------------------------------------------------------------

INPUT="safe and stuff.tar.gz"
OUTPUT="s.tar.gz"
METHOD1=max-length-8

test_sequence "$DETOX" "$INPUT" "$OUTPUT" "$TABLEPATH" "$METHOD1"

error: pathspec 'tests/tests/github-issue-0046/test.sh' did not match any file(s) known to git


Fix the KB repeat rate in ubuntu-gnome

Fix the KB repeat rate in ubuntu-gnome

#!/bin/bash
gsettings set org.gnome.desktop.peripherals.keyboard delay 200
gsettings set org.gnome.desktop.peripherals.keyboard repeat-interval 15

error: pathspec 'fix_kb_repeat.sh' did not match any file(s) known to git


Fix //tools/build_defs/docker:build_test

This is a fallout of https://bazel-review.googlesource.com/#/c/2212/

--
MOS_MIGRATED_REVID=107496707

#!/bin/bash

# Copyright 2015 The Bazel Authors. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# Unit tests for docker_build

DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
source ${DIR}/testenv.sh || { echo "testenv.sh not found!" >&2; exit 1; }

readonly PLATFORM="$(uname -s | tr 'A-Z' 'a-z')"
if [ "${PLATFORM}" = "darwin" ]; then
  readonly MAGIC_TIMESTAMP="$(date -r 0 "+%b %e  %Y")"
else
  readonly MAGIC_TIMESTAMP="$(date --date=@0 "+%F %R")"
fi

function EXPECT_CONTAINS() {
  local complete="${1}"
  local substring="${2}"
  local message="${3:-Expected '${substring}' not found in '${complete}'}"

  echo "${complete}" | grep -Fsq -- "${substring}" \
    || fail "$message"
}

function check_property() {
  local property="${1}"
  local tarball="${2}"
  local layer="${3}"
  local expected="${4}"
  local test_data="${TEST_DATA_DIR}/${tarball}.tar"

  local metadata="$(tar xOf "${test_data}" "./${layer}/json")"

  # This would be much more accurate if we had 'jq' everywhere.
  EXPECT_CONTAINS "${metadata}" "\"${property}\": ${expected}"
}

function check_no_property() {
  local property="${1}"
  local tarball="${2}"
  local layer="${3}"
  local test_data="${TEST_DATA_DIR}/${tarball}.tar"

  tar xOf "${test_data}" "./${layer}/json" >$TEST_log
  expect_not_log "\"${property}\":"

  # notop variant
  test_data="${TEST_DATA_DIR}/notop_${tarball}.tar"
  tar xOf "${test_data}" "./${layer}/json" >$TEST_log
  expect_not_log "\"${property}\":"
}

function check_size() {
  check_property Size "${@}"
}

function check_id() {
  check_property id "${@}"
}

function check_parent() {
  check_property parent "${@}"
}

function check_entrypoint() {
  input="$1"
  shift
  check_property Entrypoint "${input}" "${@}"
  check_property Entrypoint "notop_${input}" "${@}"
}

function check_cmd() {
  input="$1"
  shift
  check_property Cmd "${input}" "${@}"
  check_property Cmd "notop_${input}" "${@}"
}

function check_ports() {
  input="$1"
  shift
  check_property ExposedPorts "${input}" "${@}"
  check_property ExposedPorts "${input}" "${@}"
}

function check_volumes() {
  input="$1"
  shift
  check_property Volumes "${input}" "${@}"
  check_property Volumes "notop_${input}" "${@}"
}

function check_env() {
  input="$1"
  shift
  check_property Env "${input}" "${@}"
  check_property Env "notop_${input}" "${@}"
}

function check_workdir() {
  input="$1"
  shift
  check_property WorkingDir "${input}" "${@}"
  check_property WorkingDir "notop_${input}" "${@}"
}

function check_layers_aux() {
  local input=${1}
  shift 1
  local expected_layers=(${*})

  local expected_layers_sorted=(
    $(for i in ${expected_layers[*]}; do echo $i; done | sort)
  )
  local test_data="${TEST_DATA_DIR}/${input}.tar"

  # Verbose output for testing.
  tar tvf "${test_data}"

  local actual_layers=(
    $(tar tvf ${test_data} | tr -s ' ' | cut -d' ' -f 4- | sort \
      | cut -d'/' -f 2 | grep -E '^[0-9a-f]+$' | sort | uniq))

  # Verbose output for testing.
  echo Expected: ${expected_layers_sorted[@]}
  echo Actual: ${actual_layers[@]}

  check_eq "${#expected_layers[@]}" "${#actual_layers[@]}"

  local index=0
  local parent=
  while [ "${index}" -lt "${#expected_layers[@]}" ]
  do
    # Check that the nth sorted layer matches
    check_eq "${expected_layers_sorted[$index]}" "${actual_layers[$index]}"

    # Grab the ordered layer and check it.
    local layer="${expected_layers[$index]}"

    # Verbose output for testing.
    echo Checking layer: "${layer}"

    local listing="$(tar xOf "${test_data}" "./${layer}/layer.tar" | tar tv)"

    # Check that all files in the layer, if any, have the magic timestamp
    check_eq "$(echo "${listing}" | grep -Fv "${MAGIC_TIMESTAMP}")" ""

    check_id "${input}" "${layer}" "\"${layer}\""

    # Check that the layer contains its predecessor as its parent in the JSON.
    if [[ -n "${parent}" ]]; then
      check_parent "${input}" "${layer}" "\"${parent}\""
    fi

    # Check that the layer's size metadata matches the layer's tarball's size.
    local layer_size=$(tar xOf "${test_data}" "./${layer}/layer.tar" | wc -c | xargs)
    check_size "${input}" "${layer}" "${layer_size}"

    index=$((index + 1))
    parent=$layer
  done
}

function check_layers() {
  local input=$1
  shift
  check_layers_aux "$input" "$@"
  check_layers_aux "notop_$input" "$@"
}

function test_gen_image() {
  grep -Fsq "./gen.out" "$TEST_DATA_DIR/gen_image.tar" \
    || fail "'./gen.out' not found in '$TEST_DATA_DIR/gen_image.tar'"
}

function test_dummy_repository() {
  local layer="0279f3ce8b08d10506abcf452393b3e48439f5eca41b836fae59a0d509fbafea"
  local test_data="${TEST_DATA_DIR}/dummy_repository.tar"
  check_layers_aux "dummy_repository" "$layer"

  local repositories="$(tar xOf "${test_data}" "./repositories")"
  # This would really need to use `jq` instead.
  echo "${repositories}" | \
    grep -Esq -- "\"gcr.io/dummy/[a-zA-Z_]*_docker_testdata\": {" \
    || fail "Cannot find image in repository gcr.io/dummy in '${repositories}'"
  EXPECT_CONTAINS "${repositories}" "\"dummy_repository\": \"$layer\""
}

function test_files_base() {
  check_layers "files_base" \
    "82ca3945f7d07df82f274d7fafe83fd664c2154e5c64c988916ccd5b217bb710"
}

function test_files_with_files_base() {
  check_layers "files_with_files_base" \
    "82ca3945f7d07df82f274d7fafe83fd664c2154e5c64c988916ccd5b217bb710" \
    "84c0d09919ae8b06cb6b064d8cd5eab63341a46f11ccc7ecbe270ad3e1f52744"
}

function test_tar_base() {
  check_layers "tar_base" \
    "8b9e4db9dd4b990ee6d8adc2843ad64702ad9063ae6c22e8ca5f94aa54e71277"

  # Check that this layer doesn't have any entrypoint data by looking
  # for *any* entrypoint.
  check_no_property "Entrypoint" "tar_base" \
    "8b9e4db9dd4b990ee6d8adc2843ad64702ad9063ae6c22e8ca5f94aa54e71277"
}

function test_tar_with_tar_base() {
  check_layers "tar_with_tar_base" \
    "8b9e4db9dd4b990ee6d8adc2843ad64702ad9063ae6c22e8ca5f94aa54e71277" \
    "1cc81a2aaec2e3727d98d48bf9ba09d3ac96ef48adf5edae861d15dd0191dc40"
}

function test_files_with_tar_base() {
  check_layers "files_with_tar_base" \
    "8b9e4db9dd4b990ee6d8adc2843ad64702ad9063ae6c22e8ca5f94aa54e71277" \
    "f099727fa58f9b688e77b511b3cc728b86ae0e84d197b9330bd51082ad5589f2"
}

function test_workdir_with_tar_base() {
  check_layers "workdir_with_tar_base" \
    "8b9e4db9dd4b990ee6d8adc2843ad64702ad9063ae6c22e8ca5f94aa54e71277" \
    "f24cbe53bd1b78909c6dba0bd47016354f3488b35b85aeee68ecc423062b927e"
}

function test_tar_with_files_base() {
  check_layers "tar_with_files_base" \
    "82ca3945f7d07df82f274d7fafe83fd664c2154e5c64c988916ccd5b217bb710" \
    "bee1a325e4b51a1dcfd7e447987b4e130590815865ab22e8744878053d525f20"
}

function test_base_with_entrypoint() {
  check_layers "base_with_entrypoint" \
    "4acbeb0495918726c0107e372b421e1d2a6fd4825d58fc3f0b0b2a719fb3ce1b"

  check_entrypoint "base_with_entrypoint" \
    "4acbeb0495918726c0107e372b421e1d2a6fd4825d58fc3f0b0b2a719fb3ce1b" \
    '["/bar"]'

  # Check that the base layer has a port exposed.
  check_ports "base_with_entrypoint" \
    "4acbeb0495918726c0107e372b421e1d2a6fd4825d58fc3f0b0b2a719fb3ce1b" \
    '{"8080/tcp": {}}'
}

function test_derivative_with_shadowed_cmd() {
  check_layers "derivative_with_shadowed_cmd" \
    "4acbeb0495918726c0107e372b421e1d2a6fd4825d58fc3f0b0b2a719fb3ce1b" \
    "e35f57dc6c1e84ae67dcaaf3479a3a3c0f52ac4d194073bd6214e04c05beab42"
}

function test_derivative_with_cmd() {
  check_layers "derivative_with_cmd" \
    "4acbeb0495918726c0107e372b421e1d2a6fd4825d58fc3f0b0b2a719fb3ce1b" \
    "e35f57dc6c1e84ae67dcaaf3479a3a3c0f52ac4d194073bd6214e04c05beab42" \
    "186289545131e34510006ac79498078dcf41736a5eb9a36920a6b30d3f45bc01"

  check_entrypoint "derivative_with_cmd" \
    "186289545131e34510006ac79498078dcf41736a5eb9a36920a6b30d3f45bc01" \
    '["/bar"]'

  # Check that the middle layer has our shadowed arg.
  check_cmd "derivative_with_cmd" \
    "e35f57dc6c1e84ae67dcaaf3479a3a3c0f52ac4d194073bd6214e04c05beab42" \
    '["shadowed-arg"]'

  # Check that our topmost layer excludes the shadowed arg.
  check_cmd "derivative_with_cmd" \
    "186289545131e34510006ac79498078dcf41736a5eb9a36920a6b30d3f45bc01" \
    '["arg1", "arg2"]'

  # Check that the topmost layer has the ports exposed by the bottom
  # layer, and itself.
  check_ports "derivative_with_cmd" \
    "186289545131e34510006ac79498078dcf41736a5eb9a36920a6b30d3f45bc01" \
    '{"80/tcp": {}, "8080/tcp": {}}'
}

function test_derivative_with_volume() {
  check_layers "derivative_with_volume" \
    "125e7cfb9d4a6d803a57b88bcdb05d9a6a47ac0d6312a8b4cff52a2685c5c858" \
    "08424283ad3a7e020e210bec22b166d7ebba57f7ba2d0713c2fd7bd1e2038f88"

  # Check that the topmost layer has the ports exposed by the bottom
  # layer, and itself.
  check_volumes "derivative_with_volume" \
    "125e7cfb9d4a6d803a57b88bcdb05d9a6a47ac0d6312a8b4cff52a2685c5c858" \
    '{"/logs": {}}'

  check_volumes "derivative_with_volume" \
    "08424283ad3a7e020e210bec22b166d7ebba57f7ba2d0713c2fd7bd1e2038f88" \
    '{"/asdf": {}, "/blah": {}, "/logs": {}}'
}

function test_generated_tarball() {
  check_layers "generated_tarball" \
    "54b8328604115255cc76c12a2a51939be65c40bf182ff5a898a5fb57c38f7772"
}

function test_with_env() {
  check_layers "with_env" \
    "125e7cfb9d4a6d803a57b88bcdb05d9a6a47ac0d6312a8b4cff52a2685c5c858" \
    "42a1bd0f449f61a23b8a7776875ffb6707b34ee99c87d6428a7394f5e55e8624"

  check_env "with_env" \
    "42a1bd0f449f61a23b8a7776875ffb6707b34ee99c87d6428a7394f5e55e8624" \
    '["bar=blah blah blah", "foo=/asdf"]'
}

function test_with_double_env() {
  check_layers "with_double_env" \
    "125e7cfb9d4a6d803a57b88bcdb05d9a6a47ac0d6312a8b4cff52a2685c5c858" \
    "42a1bd0f449f61a23b8a7776875ffb6707b34ee99c87d6428a7394f5e55e8624" \
    "576a9fd9c690be04dc7aacbb9dbd1f14816e32dbbcc510f4d42325bbff7163dd"

  # Check both the aggregation and the expansion of embedded variables.
  check_env "with_double_env" \
    "576a9fd9c690be04dc7aacbb9dbd1f14816e32dbbcc510f4d42325bbff7163dd" \
    '["bar=blah blah blah", "baz=/asdf blah blah blah", "foo=/asdf"]'
}

function get_layer_listing() {
  local input=$1
  local layer=$2
  local test_data="${TEST_DATA_DIR}/${input}.tar"
  tar xOf "${test_data}" \
    "./${layer}/layer.tar" | tar tv | sed -e 's/^.*:00 //'
}

function test_data_path() {
  local no_data_path_sha="451d182e5c71840f00ba9726dc0239db73a21b7e89e79c77f677e3f7c5c23d44"
  local data_path_sha="9a41c9e1709558f7ef06f28f66e9056feafa7e0f83990801e1b27c987278d8e8"

  check_layers_aux "no_data_path_image" "${no_data_path_sha}"
  check_layers_aux "data_path_image" "${data_path_sha}"

  # Without data_path = "." the file will be inserted as `./test`
  # (since it is the path in the package) and with data_path = "."
  # the file will be inserted relatively to the testdata package
  # (so `./test/test`).
  check_eq "$(get_layer_listing "no_data_path_image" "${no_data_path_sha}")" \
    './
./test'
  check_eq "$(get_layer_listing "data_path_image" "${data_path_sha}")" \
    './
./test/
./test/test'
}

function test_extras_with_deb() {
  local test_data="${TEST_DATA_DIR}/extras_with_deb.tar"
  local sha=$(tar xOf ${test_data} ./top)

  # The content of the layer should have no duplicate
  local layer_listing="$(get_layer_listing "extras_with_deb" "${sha}" | sort)"
  check_eq "${layer_listing}" \
"./
./etc/
./etc/nsswitch.conf
./tmp/
./usr/
./usr/bin/
./usr/bin/java -> /path/to/bin/java
./usr/titi"
}

run_suite "build_test"

#!/bin/bash

# Copyright 2015 The Bazel Authors. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# Unit tests for docker_build

DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
source ${DIR}/testenv.sh || { echo "testenv.sh not found!" >&2; exit 1; }

readonly PLATFORM="$(uname -s | tr 'A-Z' 'a-z')"
if [ "${PLATFORM}" = "darwin" ]; then
  readonly MAGIC_TIMESTAMP="$(date -r 0 "+%b %e  %Y")"
else
  readonly MAGIC_TIMESTAMP="$(date --date=@0 "+%F %R")"
fi

function EXPECT_CONTAINS() {
  local complete="${1}"
  local substring="${2}"
  local message="${3:-Expected '${substring}' not found in '${complete}'}"

  echo "${complete}" | grep -Fsq -- "${substring}" \
    || fail "$message"
}

function check_property() {
  local property="${1}"
  local tarball="${2}"
  local layer="${3}"
  local expected="${4}"
  local test_data="${TEST_DATA_DIR}/${tarball}.tar"

  local metadata="$(tar xOf "${test_data}" "./${layer}/json")"

  # This would be much more accurate if we had 'jq' everywhere.
  EXPECT_CONTAINS "${metadata}" "\"${property}\": ${expected}"
}

function check_no_property() {
  local property="${1}"
  local tarball="${2}"
  local layer="${3}"
  local test_data="${TEST_DATA_DIR}/${tarball}.tar"

  tar xOf "${test_data}" "./${layer}/json" >$TEST_log
  expect_not_log "\"${property}\":"

  # notop variant
  test_data="${TEST_DATA_DIR}/notop_${tarball}.tar"
  tar xOf "${test_data}" "./${layer}/json" >$TEST_log
  expect_not_log "\"${property}\":"
}

function check_size() {
  check_property Size "${@}"
}

function check_id() {
  check_property id "${@}"
}

function check_parent() {
  check_property parent "${@}"
}

function check_entrypoint() {
  input="$1"
  shift
  check_property Entrypoint "${input}" "${@}"
  check_property Entrypoint "notop_${input}" "${@}"
}

function check_cmd() {
  input="$1"
  shift
  check_property Cmd "${input}" "${@}"
  check_property Cmd "notop_${input}" "${@}"
}

function check_ports() {
  input="$1"
  shift
  check_property ExposedPorts "${input}" "${@}"
  check_property ExposedPorts "${input}" "${@}"
}

function check_volumes() {
  input="$1"
  shift
  check_property Volumes "${input}" "${@}"
  check_property Volumes "notop_${input}" "${@}"
}

function check_env() {
  input="$1"
  shift
  check_property Env "${input}" "${@}"
  check_property Env "notop_${input}" "${@}"
}

function check_workdir() {
  input="$1"
  shift
  check_property WorkingDir "${input}" "${@}"
  check_property WorkingDir "notop_${input}" "${@}"
}

function check_layers_aux() {
  local input=${1}
  shift 1
  local expected_layers=(${*})

  local expected_layers_sorted=(
    $(for i in ${expected_layers[*]}; do echo $i; done | sort)
  )
  local test_data="${TEST_DATA_DIR}/${input}.tar"

  # Verbose output for testing.
  tar tvf "${test_data}"

  local actual_layers=(
    $(tar tvf ${test_data} | tr -s ' ' | cut -d' ' -f 4- | sort \
      | cut -d'/' -f 2 | grep -E '^[0-9a-f]+$' | sort | uniq))

  # Verbose output for testing.
  echo Expected: ${expected_layers_sorted[@]}
  echo Actual: ${actual_layers[@]}

  check_eq "${#expected_layers[@]}" "${#actual_layers[@]}"

  local index=0
  local parent=
  while [ "${index}" -lt "${#expected_layers[@]}" ]
  do
    # Check that the nth sorted layer matches
    check_eq "${expected_layers_sorted[$index]}" "${actual_layers[$index]}"

    # Grab the ordered layer and check it.
    local layer="${expected_layers[$index]}"

    # Verbose output for testing.
    echo Checking layer: "${layer}"

    local listing="$(tar xOf "${test_data}" "./${layer}/layer.tar" | tar tv)"

    # Check that all files in the layer, if any, have the magic timestamp
    check_eq "$(echo "${listing}" | grep -Fv "${MAGIC_TIMESTAMP}")" ""

    check_id "${input}" "${layer}" "\"${layer}\""

    # Check that the layer contains its predecessor as its parent in the JSON.
    if [[ -n "${parent}" ]]; then
      check_parent "${input}" "${layer}" "\"${parent}\""
    fi

    # Check that the layer's size metadata matches the layer's tarball's size.
    local layer_size=$(tar xOf "${test_data}" "./${layer}/layer.tar" | wc -c | xargs)
    check_size "${input}" "${layer}" "${layer_size}"

    index=$((index + 1))
    parent=$layer
  done
}

function check_layers() {
  local input=$1
  shift
  check_layers_aux "$input" "$@"
  check_layers_aux "notop_$input" "$@"
}

function test_gen_image() {
  grep -Fsq "./gen.out" "$TEST_DATA_DIR/gen_image.tar" \
    || fail "'./gen.out' not found in '$TEST_DATA_DIR/gen_image.tar'"
}

function test_dummy_repository() {
  local layer="eae4fa9baf743667fbe3f8d76fd598cf9ea5052261bbfaa552780dd2744c47a4"
  local test_data="${TEST_DATA_DIR}/dummy_repository.tar"
  check_layers_aux "dummy_repository" "$layer"

  local repositories="$(tar xOf "${test_data}" "./repositories")"
  # This would really need to use `jq` instead.
  echo "${repositories}" | \
    grep -Esq -- "\"gcr.io/dummy/[a-zA-Z_]*_docker_testdata\": {" \
    || fail "Cannot find image in repository gcr.io/dummy in '${repositories}'"
  EXPECT_CONTAINS "${repositories}" "\"dummy_repository\": \"$layer\""
}

function test_files_base() {
  check_layers "files_base" \
    "240dd12c02aee796394ce18eee3108475f7d544294b17fc90ec54e983601fe1b"
}

function test_files_with_files_base() {
  check_layers "files_with_files_base" \
    "240dd12c02aee796394ce18eee3108475f7d544294b17fc90ec54e983601fe1b" \
    "a9fd8cab2b9831ca2a13f371c04667a7698ef3baa90f3e820c4568d774cc69ab"
}

function test_tar_base() {
  check_layers "tar_base" \
    "8b9e4db9dd4b990ee6d8adc2843ad64702ad9063ae6c22e8ca5f94aa54e71277"

  # Check that this layer doesn't have any entrypoint data by looking
  # for *any* entrypoint.
  check_no_property "Entrypoint" "tar_base" \
    "8b9e4db9dd4b990ee6d8adc2843ad64702ad9063ae6c22e8ca5f94aa54e71277"
}

function test_tar_with_tar_base() {
  check_layers "tar_with_tar_base" \
    "8b9e4db9dd4b990ee6d8adc2843ad64702ad9063ae6c22e8ca5f94aa54e71277" \
    "1cc81a2aaec2e3727d98d48bf9ba09d3ac96ef48adf5edae861d15dd0191dc40"
}

function test_files_with_tar_base() {
  check_layers "files_with_tar_base" \
    "8b9e4db9dd4b990ee6d8adc2843ad64702ad9063ae6c22e8ca5f94aa54e71277" \
    "b0fe5685bf89a2c875a93495a4e2b7a3f1fb6f27a9ac5dc2b174e7d74cb6fe27"
}

function test_workdir_with_tar_base() {
  check_layers "workdir_with_tar_base" \
    "8b9e4db9dd4b990ee6d8adc2843ad64702ad9063ae6c22e8ca5f94aa54e71277" \
    "f24cbe53bd1b78909c6dba0bd47016354f3488b35b85aeee68ecc423062b927e"
}

function test_tar_with_files_base() {
  check_layers "tar_with_files_base" \
    "240dd12c02aee796394ce18eee3108475f7d544294b17fc90ec54e983601fe1b" \
    "c1ec3719c075f54f65045f127a6cfd77675c547446cdda7193de81b9b59afb51"
}

function test_base_with_entrypoint() {
  check_layers "base_with_entrypoint" \
    "b5f7dbb1097fa15ac31e86a149b8de2efeaedd5b8f008ba88cf2ae51adf1bd00"

  check_entrypoint "base_with_entrypoint" \
    "b5f7dbb1097fa15ac31e86a149b8de2efeaedd5b8f008ba88cf2ae51adf1bd00" \
    '["/bar"]'

  # Check that the base layer has a port exposed.
  check_ports "base_with_entrypoint" \
    "b5f7dbb1097fa15ac31e86a149b8de2efeaedd5b8f008ba88cf2ae51adf1bd00" \
    '{"8080/tcp": {}}'
}

function test_derivative_with_shadowed_cmd() {
  check_layers "derivative_with_shadowed_cmd" \
    "b5f7dbb1097fa15ac31e86a149b8de2efeaedd5b8f008ba88cf2ae51adf1bd00" \
    "f1303868df182b4bdbf3abeba580e4fe9fe9b976615dcaa67ea167d954f364d3"
}

function test_derivative_with_cmd() {
  check_layers "derivative_with_cmd" \
    "b5f7dbb1097fa15ac31e86a149b8de2efeaedd5b8f008ba88cf2ae51adf1bd00" \
    "f1303868df182b4bdbf3abeba580e4fe9fe9b976615dcaa67ea167d954f364d3" \
    "e050a01061b34d8703a672e0c6e573911ba1925fadfbcaf76999ace937108cee"

  check_entrypoint "derivative_with_cmd" \
    "e050a01061b34d8703a672e0c6e573911ba1925fadfbcaf76999ace937108cee" \
    '["/bar"]'

  # Check that the middle layer has our shadowed arg.
  check_cmd "derivative_with_cmd" \
    "f1303868df182b4bdbf3abeba580e4fe9fe9b976615dcaa67ea167d954f364d3" \
    '["shadowed-arg"]'

  # Check that our topmost layer excludes the shadowed arg.
  check_cmd "derivative_with_cmd" \
    "e050a01061b34d8703a672e0c6e573911ba1925fadfbcaf76999ace937108cee" \
    '["arg1", "arg2"]'

  # Check that the topmost layer has the ports exposed by the bottom
  # layer, and itself.
  check_ports "derivative_with_cmd" \
    "e050a01061b34d8703a672e0c6e573911ba1925fadfbcaf76999ace937108cee" \
    '{"80/tcp": {}, "8080/tcp": {}}'
}

function test_derivative_with_volume() {
  check_layers "derivative_with_volume" \
    "f86da639a9346bec6d3a821ad1f716a177a8ff8f71d66f8b70238ce7e7ba51b8" \
    "839bbd055b732c784847b3ec112d88c94f3bb752147987daef916bc956f9adf0"

  # Check that the topmost layer has the ports exposed by the bottom
  # layer, and itself.
  check_volumes "derivative_with_volume" \
    "f86da639a9346bec6d3a821ad1f716a177a8ff8f71d66f8b70238ce7e7ba51b8" \
    '{"/logs": {}}'

  check_volumes "derivative_with_volume" \
    "839bbd055b732c784847b3ec112d88c94f3bb752147987daef916bc956f9adf0" \
    '{"/asdf": {}, "/blah": {}, "/logs": {}}'
}

function test_generated_tarball() {
  check_layers "generated_tarball" \
    "54b8328604115255cc76c12a2a51939be65c40bf182ff5a898a5fb57c38f7772"
}

function test_with_env() {
  check_layers "with_env" \
    "f86da639a9346bec6d3a821ad1f716a177a8ff8f71d66f8b70238ce7e7ba51b8" \
    "80b94376a90de45256c3e94c82bc3812bc5cbd05b7d01947f29e6805e8cd7018"

  check_env "with_env" \
    "80b94376a90de45256c3e94c82bc3812bc5cbd05b7d01947f29e6805e8cd7018" \
    '["bar=blah blah blah", "foo=/asdf"]'
}

function test_with_double_env() {
  check_layers "with_double_env" \
    "f86da639a9346bec6d3a821ad1f716a177a8ff8f71d66f8b70238ce7e7ba51b8" \
    "80b94376a90de45256c3e94c82bc3812bc5cbd05b7d01947f29e6805e8cd7018" \
    "548e1d847a1d051e3cb3af383b0ebe40d341c01c97e735ae5a78ee3e10353b93"

  # Check both the aggregation and the expansion of embedded variables.
  check_env "with_double_env" \
    "548e1d847a1d051e3cb3af383b0ebe40d341c01c97e735ae5a78ee3e10353b93" \
    '["bar=blah blah blah", "baz=/asdf blah blah blah", "foo=/asdf"]'
}

function test_with_double_env() {
  check_layers "with_double_env" \
    "f86da639a9346bec6d3a821ad1f716a177a8ff8f71d66f8b70238ce7e7ba51b8" \
    "80b94376a90de45256c3e94c82bc3812bc5cbd05b7d01947f29e6805e8cd7018" \
    "548e1d847a1d051e3cb3af383b0ebe40d341c01c97e735ae5a78ee3e10353b93"

  # Check both the aggregation and the expansion of embedded variables.
  check_env "with_double_env" \
    "548e1d847a1d051e3cb3af383b0ebe40d341c01c97e735ae5a78ee3e10353b93" \
    '["bar=blah blah blah", "baz=/asdf blah blah blah", "foo=/asdf"]'
}

function get_layer_listing() {
  local input=$1
  local layer=$2
  local test_data="${TEST_DATA_DIR}/${input}.tar"
  tar xOf "${test_data}" \
    "./${layer}/layer.tar" | tar tv | sed -e 's/^.*:00 //'
}

function test_data_path() {
  local no_data_path_sha="29bab1d66ea34ebbb9604704322458e0f06a3a9a7b0ac08df8b95a6c9c3d9320"
  local data_path_sha="9a5c6b96227af212d1300964aaeb0723e78e78de019de1ba2382603f3050558a"

  check_layers_aux "no_data_path_image" "${no_data_path_sha}"
  check_layers_aux "data_path_image" "${data_path_sha}"

  # Without data_path = "." the file will be inserted as `./test`
  # (since it is the path in the package) and with data_path = "."
  # the file will be inserted relatively to the testdata package
  # (so `./test/test`).
  check_eq $(get_layer_listing "no_data_path_image" "${no_data_path_sha}") \
    ./test
  check_eq $(get_layer_listing "data_path_image" "${data_path_sha}") \
    ./test/test
}

function test_extras_with_deb() {
  local test_data="${TEST_DATA_DIR}/extras_with_deb.tar"
  local sha=$(tar xOf ${test_data} ./top)

  # The content of the layer should have no duplicate
  local layer_listing="$(get_layer_listing "extras_with_deb" "${sha}" | sort)"
  check_eq "${layer_listing}" \
"./
./etc/
./etc/nsswitch.conf
./tmp/
./usr/
./usr/bin/
./usr/bin/java -> /path/to/bin/java
./usr/titi"
}

run_suite "build_test"


Only remove `external/org_tensorflow` if it exists (#5494)

This should fix coverage build broken again today. It seems that a
recent change in TF toolchains removed the creation of `org_tensorflow`
but it is likely that this change will be rolled back in the future.
Hence, to keep OSSFuzz working, we conditionally remove this symlink.

#!/bin/bash -eu
# Copyright 2018 Google Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
################################################################################

# Force Python3, run configure.py to pick the right build config
PYTHON=python3
yes "" | ${PYTHON} configure.py

# Since Bazel passes flags to compilers via `--copt`, `--conlyopt` and
# `--cxxopt`, we need to move all flags from `$CFLAGS` and `$CXXFLAGS` to these.
# We don't use `--copt` as warnings issued by C compilers when encountering a
# C++-only option results in errors during build.
#
# Note: Make sure that by this line `$CFLAGS` and `$CXXFLAGS` are properly set
# up as further changes to them won't be visible to Bazel.
#
# Note: for builds using the undefined behavior sanitizer we need to link
# `clang_rt` ubsan library. Since Bazel uses `clang` for linking instead of
# `clang++`, we need to add the additional `--linkopt` flag.
# See issue: https://github.com/bazelbuild/bazel/issues/8777
declare -r EXTRA_FLAGS="\
$(
for f in ${CFLAGS}; do
  echo "--conlyopt=${f}" "--linkopt=${f}"
done
for f in ${CXXFLAGS}; do
    echo "--cxxopt=${f}" "--linkopt=${f}"
done
if [ "$SANITIZER" = "undefined" ]
then
  echo "--linkopt=$(find $(llvm-config --libdir) -name libclang_rt.ubsan_standalone_cxx-x86_64.a | head -1)"
fi
)"

# Determine all fuzz targets. To control what gets fuzzed with OSSFuzz, all
# supported fuzzers are in `//tensorflow/security/fuzzing`.
# Ignore fuzzers tagged with `no_oss` in opensource.
declare -r FUZZERS=$(bazel query 'kind(cc_.*, tests(//tensorflow/security/fuzzing/...)) - attr(tags, no_oss, kind(cc_.*, tests(//tensorflow/security/fuzzing/...)))')

# Build the fuzzer targets.
# Pass in `--config=libc++` to link against libc++.
# Pass in `--verbose_failures` so it is easy to debug compile crashes.
# Pass in `--strip=never` to ensure coverage support.
# Pass in `$LIB_FUZZING_ENGINE` to `--copt` and `--linkopt` to ensure we have a
# `main` symbol defined (all these fuzzers build without a `main` and by default
# `$CFLAGS` and `CXXFLAGS` compile with `-fsanitize=fuzzer-no-link`).
# Since we have `assert` in fuzzers, make sure `NDEBUG` is not defined
bazel build \
  --config=libc++ \
  ${EXTRA_FLAGS} \
  --verbose_failures \
  --strip=never \
  --copt=${LIB_FUZZING_ENGINE} \
  --linkopt=${LIB_FUZZING_ENGINE} \
  --copt='-UNDEBUG' \
  -- ${FUZZERS}

# The fuzzers built above are in the `bazel-bin/` symlink. But they need to be
# in `$OUT`, so move them accordingly.
for bazel_target in ${FUZZERS}; do
  colon_index=$(expr index "${bazel_target}" ":")
  fuzz_name="${bazel_target:$colon_index}"
  bazel_location="bazel-bin/${bazel_target/:/\/}"
  cp ${bazel_location} ${OUT}/$fuzz_name
done

# For coverage, we need to remap source files to correspond to the Bazel build
# paths. We also need to resolve all symlinks that Bazel creates.
if [ "$SANITIZER" = "coverage" ]
then
  declare -r RSYNC_CMD="rsync -aLkR"
  declare -r REMAP_PATH=${OUT}/proc/self/cwd/
  mkdir -p ${REMAP_PATH}

  # Sync existing code.
  ${RSYNC_CMD} tensorflow/ ${REMAP_PATH}

  # Sync generated proto files.
  ${RSYNC_CMD} ./bazel-out/k8-opt/bin/tensorflow/core/protobuf ${REMAP_PATH}

  # Sync external dependencies. We don't need to include `bazel-tensorflow`.
  # Also, remove `external/org_tensorflow` which is a copy of the entire source
  # code that Bazel creates. Not removing this would cause `rsync` to expand a
  # symlink that ends up pointing to itself!
  pushd bazel-tensorflow
  [[ -e external/org_tensorflow ]] && unlink external/org_tensorflow
  ${RSYNC_CMD} external/ ${REMAP_PATH}
  popd
fi

# Finally, make sure we don't accidentally run with stuff from the bazel cache.
rm -f bazel-*

#!/bin/bash -eu
# Copyright 2018 Google Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
################################################################################

# Force Python3, run configure.py to pick the right build config
PYTHON=python3
yes "" | ${PYTHON} configure.py

# Since Bazel passes flags to compilers via `--copt`, `--conlyopt` and
# `--cxxopt`, we need to move all flags from `$CFLAGS` and `$CXXFLAGS` to these.
# We don't use `--copt` as warnings issued by C compilers when encountering a
# C++-only option results in errors during build.
#
# Note: Make sure that by this line `$CFLAGS` and `$CXXFLAGS` are properly set
# up as further changes to them won't be visible to Bazel.
#
# Note: for builds using the undefined behavior sanitizer we need to link
# `clang_rt` ubsan library. Since Bazel uses `clang` for linking instead of
# `clang++`, we need to add the additional `--linkopt` flag.
# See issue: https://github.com/bazelbuild/bazel/issues/8777
declare -r EXTRA_FLAGS="\
$(
for f in ${CFLAGS}; do
  echo "--conlyopt=${f}" "--linkopt=${f}"
done
for f in ${CXXFLAGS}; do
    echo "--cxxopt=${f}" "--linkopt=${f}"
done
if [ "$SANITIZER" = "undefined" ]
then
  echo "--linkopt=$(find $(llvm-config --libdir) -name libclang_rt.ubsan_standalone_cxx-x86_64.a | head -1)"
fi
)"

# Determine all fuzz targets. To control what gets fuzzed with OSSFuzz, all
# supported fuzzers are in `//tensorflow/security/fuzzing`.
# Ignore fuzzers tagged with `no_oss` in opensource.
declare -r FUZZERS=$(bazel query 'kind(cc_.*, tests(//tensorflow/security/fuzzing/...)) - attr(tags, no_oss, kind(cc_.*, tests(//tensorflow/security/fuzzing/...)))')

# Build the fuzzer targets.
# Pass in `--config=libc++` to link against libc++.
# Pass in `--verbose_failures` so it is easy to debug compile crashes.
# Pass in `--strip=never` to ensure coverage support.
# Pass in `$LIB_FUZZING_ENGINE` to `--copt` and `--linkopt` to ensure we have a
# `main` symbol defined (all these fuzzers build without a `main` and by default
# `$CFLAGS` and `CXXFLAGS` compile with `-fsanitize=fuzzer-no-link`).
# Since we have `assert` in fuzzers, make sure `NDEBUG` is not defined
bazel build \
  --config=libc++ \
  ${EXTRA_FLAGS} \
  --verbose_failures \
  --strip=never \
  --copt=${LIB_FUZZING_ENGINE} \
  --linkopt=${LIB_FUZZING_ENGINE} \
  --copt='-UNDEBUG' \
  -- ${FUZZERS}

# The fuzzers built above are in the `bazel-bin/` symlink. But they need to be
# in `$OUT`, so move them accordingly.
for bazel_target in ${FUZZERS}; do
  colon_index=$(expr index "${bazel_target}" ":")
  fuzz_name="${bazel_target:$colon_index}"
  bazel_location="bazel-bin/${bazel_target/:/\/}"
  cp ${bazel_location} ${OUT}/$fuzz_name
done

# For coverage, we need to remap source files to correspond to the Bazel build
# paths. We also need to resolve all symlinks that Bazel creates.
if [ "$SANITIZER" = "coverage" ]
then
  declare -r RSYNC_CMD="rsync -aLkR"
  declare -r REMAP_PATH=${OUT}/proc/self/cwd/
  mkdir -p ${REMAP_PATH}

  # Sync existing code.
  ${RSYNC_CMD} tensorflow/ ${REMAP_PATH}

  # Sync generated proto files.
  ${RSYNC_CMD} ./bazel-out/k8-opt/bin/tensorflow/core/protobuf ${REMAP_PATH}

  # Sync external dependencies. We don't need to include `bazel-tensorflow`.
  # Also, remove `external/org_tensorflow` which is a copy of the entire source
  # code that Bazel creates. Not removing this would cause `rsync` to expand a
  # symlink that ends up pointing to itself!
  pushd bazel-tensorflow
  unlink external/org_tensorflow
  ${RSYNC_CMD} external/ ${REMAP_PATH}
  popd
fi

# Finally, make sure we don't accidentally run with stuff from the bazel cache.
rm -f bazel-*


Add create sciCloud Drive

Add create sciCloud Drive

#!/bin/bash

# if [ ! -e ~/iCloud\ Drive ]; then
    mkdir ~/iCloud\ Drive
# else
    # echo "\"~/iCloud Drive\" directory already exists."
# fi

iWorks="Keynote Numbers Pages"
other="Automator Notes Preview TextEdit QuickTimePlayerX ScriptEditor2"

for dir in $iWorks $other; do
    ln -ns ~/Library/Mobile\ Documents/com~apple~${dir}/Documents \
        ~/iCloud\ Drive/${dir}
done

ln -ns ~/Library/Mobile\ Documents/com~apple~CloudDocs ~/iCloud\ Drive/CloudDocs

error: pathspec 'create_icloud_drive_simlinks.sh' did not match any file(s) known to git


Comment out evernote

Comment out evernote

remote: Support for password authentication was removed on August 13, 2021.
remote: Please see https://docs.github.com/en/get-started/getting-started-with-git/about-remote-repositories#cloning-with-https-urls for information on currently recommended modes of authentication.
fatal: Authentication failed for 'https://github.com/hectorhuertas/.dotfiles.git/'


Add gcloud brew

Add gcloud brew


Added copy script to copy sensitive files from Dropbox

#!/usr/bin/env bash
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"

if [[ ! "$(type -P rsync)" ]]; then
  echo "RSync must be installed first."
  exit 1
fi

rsync -avz "$HOME"/Dropbox/Apps/dotfiles/ "$HOME"/.dotfiles/

error: pathspec 'bin/copy.sh' did not match any file(s) known to git


add npm update script

add npm update script

error: pathspec 'lang/nodejs/update-npm.sh' did not match any file(s) known to git


Created script to wget the opennlp models.

#!/bin/bash

wget http://opennlp.sourceforge.net/models-1.5/en-ner-location.bin
wget http://opennlp.sourceforge.net/models-1.5/en-token.bin
wget http://opennlp.sourceforge.net/models-1.5/en-sent.bin

error: pathspec 'data/models/getOpenNLPModels.sh' did not match any file(s) known to git


Revised pep257 bash script

Revised pep257 bash script

error: pathspec '{{' did not match any file(s) known to git
error: pathspec 'cookiecutter.repo_name' did not match any file(s) known to git
error: pathspec '}}/pep257.sh' did not match any file(s) known to git


added restart django server bash command

#!/bin/bash

source ${BASH_SOURCE%/*}/functions.sh
run_django_server

error: pathspec 'scripts/restart_django_server.sh' did not match any file(s) known to git


Set nvm paths

Set nvm paths

export INIT_NVM_SCRIPT_PATH=/usr/share/nvm/init-nvm.sh
export NVM_BASH_COMPLETION_PATH=/usr/share/nvm/bash_completion

error: pathspec 'node/node.zsh' did not match any file(s) known to git


db/Re-add build.sh for testing

db/Re-add build.sh for testing

#!/bin/sh

set -e

IMG=registry.ayhan.biz/cms-data
VDA=cms_data
VDB=cms_db

#
# Commit
#
CONT=$(docker run -d -v $VDA:/vol-data -v $VDB:/vol-db akilli/postgres)
sleep 10
docker exec $CONT ash -c 'rm -rf /app && cp -R /vol-data /app && rm -rf /data && cp -R /vol-db /data && chown -R app:app /app /data'
docker commit $CONT $IMG
docker rm -f $CONT

error: pathspec 'db/build.sh' did not match any file(s) known to git


Fix typo in backup script

Fix typo in backup script

remote: Support for password authentication was removed on August 13, 2021.
remote: Please see https://docs.github.com/en/get-started/getting-started-with-git/about-remote-repositories#cloning-with-https-urls for information on currently recommended modes of authentication.
fatal: Authentication failed for 'https://github.com/The-Unknown-Minecraft/The-Unknown-Server.git/'


add configure_git.sh, to set up default git config

#!/usr/bin/env bash

# Stop on any error, print all commands
set -ev

git config --global user.name "Jonathan Hanson"
git config --global user.email "jonathan@jonathan-hanson.org"

git config --global core.editor "subl -n -w"
git config --global diff.compactionHeuristic true

error: pathspec 'scripts/configure_git.sh' did not match any file(s) known to git


Refactor remove SS

Refactor remove SS

remote: Support for password authentication was removed on August 13, 2021.
remote: Please see https://docs.github.com/en/get-started/getting-started-with-git/about-remote-repositories#cloning-with-https-urls for information on currently recommended modes of authentication.
fatal: Authentication failed for 'https://github.com/ankushagarwal/dotfiles.git/'