Create pkg-manager-bash.sh

Create pkg-manager-bash.sh

Test for yum or apt based linux

#!/bin/bash
## Description: Test for yum or apt based linux

export IS_YUM_PKG_MANAGER=`command -v yum &>/dev/null && echo 1 || echo ''`
export IS_APT_GET_PKG_MANAGER=`command -v apt-get &>/dev/null && echo 1 || echo ''`
export PKG_MANAGER=$( command -v yum || command -v apt-get ) || exit 1

error: pathspec 'linux/pkg-manager-bash.sh' did not match any file(s) known to git


Added helper script to make it easy to create headless versions of KillrVideo as Docker images

#!/bin/bash

# This is a simple helper script that creates a Docker image of the
# killrvideo-java application to run in "headless" mode.
version=$1

if [[ -z "$1" ]]; then
  version="latest"
fi

docker build -t killrvideo/killrvideo-java-server:$version .

# I wasn't kidding, it's really simple

error: pathspec 'create-headless-server.sh' did not match any file(s) known to git


Forgot to put back LDA training things here

remote: Support for password authentication was removed on August 13, 2021.
remote: Please see https://docs.github.com/en/get-started/getting-started-with-git/about-remote-repositories#cloning-with-https-urls for information on currently recommended modes of authentication.
fatal: Authentication failed for 'https://github.com/cybeliak/mispronounceableness.git/'


Remove set

Remove set

remote: Support for password authentication was removed on August 13, 2021.
remote: Please see https://docs.github.com/en/get-started/getting-started-with-git/about-remote-repositories#cloning-with-https-urls for information on currently recommended modes of authentication.
fatal: Authentication failed for 'https://github.com/acksin/acksin.git/'


Better local build script [skip build].

Better local build script [skip build].

remote: Support for password authentication was removed on August 13, 2021.
remote: Please see https://docs.github.com/en/get-started/getting-started-with-git/about-remote-repositories#cloning-with-https-urls for information on currently recommended modes of authentication.
fatal: Authentication failed for 'https://github.com/FEniCS/docker.git/'


Added a script that installs Vim Janus and removes the git repo history.

remote: Support for password authentication was removed on August 13, 2021.
remote: Please see https://docs.github.com/en/get-started/getting-started-with-git/about-remote-repositories#cloning-with-https-urls for information on currently recommended modes of authentication.
fatal: Authentication failed for 'https://github.com/nickchappell/packer-templates.git/'


added lesscss plugin

added lesscss plugin

# basics taken from here:
# http://www.ravelrumba.com/blog/watch-compile-less-command-line/comment-page-1/#comment-2464
# Requires watchr: https://github.com/mynyml/watchr

watchless () {
    compressed=0
    compile=""

    while getopts ":xc:h" option
    do
      case $option in
        x ) compressed=1 ;;
        c ) compile=$OPTARG ;;
        h ) _watchless_usage; return ;;
      esac
    done

    if [ $compressed -eq 1 ]; then
        x=' -x'
    else
        x=''
    fi

    if [ -n "$compile" ]; then
        if [ ! -e $compile ]; then
            echo "\033[337;41m\n$compile doesn't exist!\n\033[0m"
            return
        fi

        name=$(echo $compile | cut -d . -f 1)

        watchr -e 'watch(".*\.less$") { |f| system("lessc '$name'.less > '$name'.css'$x' && echo \"'$name'.less > '$name'.css\" ") }'
    else
        watchr -e 'watch(".*\.less$") { |f| system("lessc #{f[0]} > $(echo #{f[0]} | cut -d\. -f1).css'$x' && echo \"#{f[0]} > $(echo #{f[0]} | cut -d\. -f1).css\" ") }'
    fi
}

_watchless_usage () {
    echo "Usage: watchless [options]"
    echo
    echo "Options"
    echo "  -x   Compiles less files into minified css files"
    echo "  -c   Watch all files but compile only the one given here"
    echo "  -h   Get this help message"
    return
}

error: pathspec 'plugins/lesscss/lesscss.plugin.zsh' did not match any file(s) known to git


Add script for formatting C and C++ code with clang-format

#!/bin/bash
clang-format -style='{
    BasedOnStyle: llvm,
    IndentWidth: 4,
    BreakBeforeBraces: Custom,
    BraceWrapping: {
        AfterFunction: true,
        AfterClass: true,
        AfterStruct: true,
        AfterNamespace: true,
        AfterUnion: true,
        AfterEnum: true,
        },
    ColumnLimit: 100,
    AllowShortIfStatementsOnASingleLine: true,
    AllowShortLoopsOnASingleLine: true,
    PointerAlignment: Left,
    AccessModifierOffset: -4,
    AllowShortFunctionsOnASingleLine: Empty,
    }' -i $@

error: pathspec 'format-with-clang.sh' did not match any file(s) known to git


git_subtree_pull.zsh

#!zsh -x

git fetch origin
foreach this (cmake-utils dlfcn-win32 genericLogger optparse winiconv) {
  git fetch $this-remote master
}

git reset --hard origin/master
git clean -ffdx
foreach this (cmake-utils dlfcn-win32 genericLogger optparse winiconv) {
  git subtree pull --prefix 3rdparty/github/$this-remote $this-remote master --squash
}

exit 0

error: pathspec 'git_subtree_pull.zsh' did not match any file(s) known to git


added bash script for initialization

added bash script for initialization

#!/bin/bash
############################
# .make.sh
# makes all necessary symlinks to files/dirs in ~/dotfiles,
# and moves old ones to ~/dotfiles_old
############################

dir=~/dotfiles                  # dotfiles dir
olddir=~/dotfiles_old           # old dotfiles backup dir
files="vimrc vim"               # files/dirs to symlink in ~

echo -n "Creating $olddir for backup of any existing dotfiles in ~ ..."
mkdir -p $olddir
echo "done"

echo -n "Changing to the $dir directory ..."
cd $dir
echo "done"

for file in $files; do
    echo "Moving any existing dotfiles from ~ to $olddir"
    mv ~/.$file ~/dotfiles_old/
    echo "Creating symlink to $file in home directory."
    ln -s $dir/$file ~/.$file
done

error: pathspec 'furnish.sh' did not match any file(s) known to git


add plugin-dir entry-config

add plugin-dir entry-config

#!/bin/bash

sed -i'' -e 's#plugins = .*#plugins = plugins#' /opt/grafana/conf/defaults.ini

error: pathspec 'opt/qnib/entry/10-grafana-plugins-dir.sh' did not match any file(s) known to git


Add script to copy raspbian on the SD card

#!/bin/bash

IMG=$(ls raspbian/*raspbian*.img)
TARGET=/dev/sdd

echo "Copying $IMG to $TARGET..."

# Copy raspbian on the target
sudo dd bs=4M if=$IMG of=$TARGET status=progress

# Ensure the cache is flushed
sudo sync

echo "Done! You can eject the SD card."

error: pathspec 'scripts/copy-raspbian-on-sd.sh' did not match any file(s) known to git


added script for generating release

added script for generating release

#!/usr/bin/env bash

PROJECT=jgum
VERSION=0.0.1
gpg -ab $PROJECT-$VERSION.pom
gpg -ab $PROJECT-$VERSION.jar
gpg -ab $PROJECT-$VERSION-sources.jar
gpg -ab $PROJECT-$VERSION-javadoc.jar

jar -cvf bundle.jar $PROJECT-$VERSION.pom $PROJECT-$VERSION.pom.asc $PROJECT-$VERSION.jar $PROJECT-$VERSION.jar.asc $PROJECT-$VERSION-javadoc.jar $PROJECT-$VERSION-javadoc.jar.asc $PROJECT-$VERSION-sources.jar $PROJECT-$VERSION-sources.jar.asc

error: pathspec 'scripts/maven-central/build.sh' did not match any file(s) known to git


Add an attempt at truth-free variant evaluation

#!/usr/bin/env bash
# Evaluates variant calls without recourse to a truth set
# Run after parallelMappingEvaluation.sh
# Usage: ./scripts/truthFreeEvaluation.sh <alignment input directory> <read source directory>

set -ex

# What plot filetype should we produce?
PLOT_FILETYPE="svg"

# Grab the input directory to look in
INPUT_DIR=${1}

if [[ ! -d "${INPUT_DIR}" ]]
then
    echo "Specify input directory!"
    exit 1
fi

# Grab the read directory to look in
READ_DIR=${2}

if [[ ! -d "${READ_DIR}" ]]
then
    echo "Specify read directory!"
    exit 1
fi

# Set up the plot parameters
# Include both versions of the 1kg SNPs graph name
PLOT_PARAMS=(
    --categories
    snp1kg
    snp1000g
    sbg
    cactus
    camel
    curoverse
    debruijn-k31
    debruijn-k63
    level1
    level2
    level3
    prg
    refonly
    simons
    trivial
    vglr
    haplo1kg30
    haplo1kg50
    shifted1kg
    --category_labels
    1KG
    1KG
    7BG
    Cactus
    Camel
    Curoverse
    "De Bruijn 31"
    "De Bruijn 63"
    Level1
    Level2
    Level3
    PRG
    Primary
    SGDP
    Unmerged
    VGLR
    "1KG Haplo 30"
    "1KG Haplo 50"
    Scrambled
    --colors
    "#fb9a99"
    "#fb9a99"
    "#b15928"
    "#1f78b4"
    "#33a02c"
    "#a6cee3"
    "#e31a1c"
    "#ff7f00"
    "#FF0000"
    "#00FF00"
    "#0000FF"
    "#6a3d9a"
    "#000000"
    "#b2df8a"
    "#b1b300"
    "#cab2d6"
    "#00FF00"
    "#0000FF"
    "#FF0000"
    --dpi 90 --no_n
)

# Color "#fdbf6f" from haplo1kg is free.

# Extract the graphs
EXTRACTED_DIR="${INPUT_DIR}/evals/truthfree/extracted"
if [[ ! -d "${EXTRACTED_DIR}" ]]; then
    # Extract all the graphs
    mkdir -p "${EXTRACTED_DIR}"
    ./scripts/extractGraphs.py "${INPUT_DIR}"/indexes/*/*/*.tar.gz "${EXTRACTED_DIR}"
fi

# Make a place to index reads
READ_INDEX_DIR="${INPUT_DIR}/evals/truthfree/read_indexes"

# Where do we put the plots?
PLOTS_ROOT_DIR="${INPUT_DIR}/evals/truthfree/plots"
mkdir -p "${PLOTS_ROOT_DIR}"

# And the stats files to make the plots?
STATS_ROOT_DIR="${INPUT_DIR}/evals/truthfree/stats"
mkdir -p "${STATS_ROOT_DIR}"

for REGION in brca2; do
    # Go through all the regions (TODO: hardcoded for now)

    # Make a file to put per-graph stats in
    mkdir -p "${STATS_ROOT_DIR}/${REGION}"
    REGION_STATS_FILE="${STATS_ROOT_DIR}/${REGION}/stats.tsv"
    true > ${REGION_STATS_FILE}

    for SAMPLE in NA12878; do
        # Go through all the samples (TODO: hardcoded for now)

        for GRAPH in _truth_ snp1kg refonly shifted1kg cactus prg sbg; do
            # Go through all the graphs (TODO: hardcoded for now)

            # We need a directory for the .vcf to live in
            VCF_DIR="${INPUT_DIR}/evals/truthfree/stats/${REGION}/${GRAPH}/${SAMPLE}"

            if [[ -d "${VCF_DIR}" ]]; then
                # Assume we already ran this one and put it in the stats TSV.
                # TODO: add a rerun flag somehow?
                continue
            fi

            mkdir -p "${VCF_DIR}"

            if [[ "${GRAPH}" == "_truth_" ]]; then
                # Special graph: grab from the truth set

                cat "data/platinum/${SAMPLE}/${REGION^^}.vcf" | grep "#" > "${VCF_DIR}/unsorted.vcf"
                echo "##contig=<ID=ref,length=10000000>" >> "${VCF_DIR}/unsorted.vcf"

                # Subtract out a different base offset to convert to region-relative coordinates for every region.
                if [[ "${REGION}" == "brca2" ]]; then
                    OFFSET=32314860
                elif [[ "${REGION}" == "mhc" ]]; then
                    OFFSET=28510119
                elif [[ "${REGION}" == "lrc_kir" ]]; then
                    OFFSET=54025633
                else
                    echo "Need base offset for region ${REGION}!"
                    exit
                fi

                # TODO: filter out filtered variants somehow?

                cat "data/platinum/${SAMPLE}/${REGION^^}.vcf" | grep -v "^#" | awk -v offset="${OFFSET}" -F $'\t' 'BEGIN {OFS = FS} { $1 = "ref"; $2 -= offset; print $0 }' >> "${VCF_DIR}/unsorted.vcf"

                cat "${VCF_DIR}/unsorted.vcf" | sort -n -k2 > "${VCF_DIR}/sorted.vcf"

                rm -f "${VCF_DIR}/sorted.vcf.gz"
                bgzip "${VCF_DIR}/sorted.vcf" -c > "${VCF_DIR}/sorted.vcf.gz"
                tabix -f -p vcf "${VCF_DIR}/sorted.vcf.gz"

            else
                # Use the genotypes we get from this graph

                # Where do we index these reads?
                INDEX_DIR="${READ_INDEX_DIR}/${REGION}/${GRAPH}/${SAMPLE}/reads.index"

                if [[ ! -d "${INDEX_DIR}" ]]; then
                    # Index the reads
                    mkdir -p "${INDEX_DIR}"
                    FILTERED_FILE=`mktemp`
                    # Get just the primary alignments
                    vg view -aj "${INPUT_DIR}/alignments/${REGION}/${GRAPH}/${SAMPLE}.gam" | jq -c 'select(.is_secondary | not)' | vg view -JGa - > "${FILTERED_FILE}"
                    # Index them into the index
                    vg index -d "${INDEX_DIR}" -N "${FILTERED_FILE}"
                    rm "${FILTERED_FILE}"
                fi

                # Do the genotyping
                # We re-do this every time because the genotype changes pretty fast
                vg genotype "${EXTRACTED_DIR}/${GRAPH}-${REGION}.vg" "${INDEX_DIR}" -v -q -i -C > "${VCF_DIR}/called.vcf"

                # Sort, compress, and index the VCF
                cat "${VCF_DIR}/called.vcf" | sort -n -k2 > "${VCF_DIR}/sorted.vcf"
                rm -f "${VCF_DIR}/sorted.vcf.gz"
                bgzip "${VCF_DIR}/sorted.vcf" -c > "${VCF_DIR}/sorted.vcf.gz"
                tabix -f -p vcf "${VCF_DIR}/sorted.vcf.gz"

            fi

            # Grab the reference FASTA
            ln -f "data/altRegions/${REGION^^}/ref.fa" "${VCF_DIR}/ref.fa"

            # Make a sample graph
            vg construct -r "${VCF_DIR}/ref.fa" -v "${VCF_DIR}/sorted.vcf.gz" -a -f > "${VCF_DIR}/reconstructed.vg"
            vg mod -v "${VCF_DIR}/sorted.vcf.gz" "${VCF_DIR}/reconstructed.vg" > "${VCF_DIR}/sample.vg"

            # Remap reads
            vg map -V "${VCF_DIR}/sample.vg" -f "${READ_DIR}/${REGION^^}/${SAMPLE}/${SAMPLE}.bam.fq" -k 16 -i > "${VCF_DIR}/realigned.gam"

            # Collect scores
            vg view -aj "${VCF_DIR}/realigned.gam" | jq '.score' | sed 's/null/0/' > "${VCF_DIR}/scores.txt"

            # Collect portion of het sites that are significantly biased
            # Sed escaped parens are the regex capture ones for some reason...
            PORTION_BIASED=$(vg stats "${VCF_DIR}/sample.vg" -a "${VCF_DIR}/realigned.gam" | tail -n1 | sed 's/.*(\(.*\)%).*/\1/g')

            # Find the mean score
            MEAN_SCORE=$(cat "${VCF_DIR}/scores.txt" | ./scripts/mean.sh)

            # Put them in as a point for this graph
            printf "${GRAPH}\t${PORTION_BIASED}\t${MEAN_SCORE}\n" >> "${REGION_STATS_FILE}"

        done
    done

    # Now make the plot for the region
    ./scripts/scatter.py "${STATS_ROOT_DIR}/${REGION}/stats.tsv" \
        --save "${PLOTS_ROOT_DIR}/${REGION}.png" \
        --x_label "Portion Biased (%)" \
        --y_label "Mean Score (points)" \
        --title "$(printf "Mean read score vs.\nsite bias in ${REGION}")" \
        "${PLOT_PARAMS[@]}"

done

error: pathspec 'scripts/truthFreeEvaluation.sh' did not match any file(s) known to git


Added rsync to setup script

Added rsync to setup script

remote: Support for password authentication was removed on August 13, 2021.
remote: Please see https://docs.github.com/en/get-started/getting-started-with-git/about-remote-repositories#cloning-with-https-urls for information on currently recommended modes of authentication.
fatal: Authentication failed for 'https://github.com/asyounis/PerfectSwerve.git/'


We need to move the mountpoint

We need to move the mountpoint

#!/bin/sh
# Functions which perform the final cleanup after an install

# Finishes up with ZFS setup before unmounting
zfs_cleanup_unmount()
{
  # Loop through our FS and see if we have any ZFS partitions to cleanup
  for PART in `ls ${PARTDIR}`
  do
    PARTFS="`cat ${PARTDIR}/${PART} | cut -d ':' -f 1`"
    PARTMNT="`cat ${PARTDIR}/${PART} | cut -d ':' -f 2`"
    ZPOOLNAME=$(get_zpool_name "${PART}")

    if [ "$PARTFS" = "ZFS" ]
    then
      # Check if we have multiple zfs mounts specified
      for ZMNT in `echo ${PARTMNT} | sed 's|,| |g'`
      do
        if [ "${ZMNT}" = "/" ]
        then
          # Make sure we haven't already added the zfs boot line when
          # Creating a dedicated "/boot" partition
          cat ${FSMNT}/boot/loader.conf 2>/dev/null | grep "vfs.root.mountfrom=" >/dev/null 2>/dev/null
          if [ "$?" != "0" ] ; then
            echo "vfs.root.mountfrom=\"zfs:${ZPOOLNAME}\"" >> ${FSMNT}/boot/loader.conf
          fi
          FOUNDZFSROOT="${ZPOOLNAME}" ; export FOUNDZFSROOT
        fi
      done
      FOUNDZFS="1"
    fi
  done

  if [ ! -z "${FOUNDZFS}" ]
  then
    # Check if we need to add our ZFS flags to rc.conf, src.conf and loader.conf
    cat ${FSMNT}/boot/loader.conf 2>/dev/null | grep 'zfs_load="YES"' >/dev/null 2>/dev/null
    if [ "$?" != "0" ]
    then
      echo 'zfs_load="YES"' >>${FSMNT}/boot/loader.conf
    fi
    cat ${FSMNT}/etc/rc.conf 2>/dev/null | grep 'zfs_enable="YES"' >/dev/null 2>/dev/null
    if [ "$?" != "0" ]
    then
      #echo 'zfs_enable="YES"' >>${FSMNT}/etc/rc.conf
    fi

    sleep 2
    # Copy over any ZFS cache data
    cp /boot/zfs/* ${FSMNT}/boot/zfs/

    # Copy the hostid so that our zfs cache works
    cp /etc/hostid ${FSMNT}/etc/hostid
  fi

  # Loop through our FS and see if we have any ZFS partitions to cleanup
  for PART in `ls ${PARTDIR}`
  do
    PARTFS="`cat ${PARTDIR}/${PART} | cut -d ':' -f 1`"
    PARTMNT="`cat ${PARTDIR}/${PART} | cut -d ':' -f 2`"
    PARTENC="`cat ${PARTDIR}/${PART} | cut -d ':' -f 3`"
    ZPOOLNAME=$(get_zpool_name "${PART}")

    if [ "$PARTFS" = "ZFS" ]
    then
      # Check if we have multiple zfs mounts specified
      for ZMNT in `echo ${PARTMNT} | sed 's|,| |g'`
      do
        PARTMNTREV="${ZMNT} ${PARTMNTREV}"
      done

      for ZMNT in ${PARTMNTREV}
      do
        if [ "${ZMNT}" != "/" ]
        then
          rc_halt "zfs set mountpoint=${ZMNT} ${ZPOOLNAME}${ZMNT}"
          #rc_halt "zfs unmount ${ZPOOLNAME}${ZMNT}"
          #sleep 2
        fi
      done
    fi
  done

};

# Function which performs the specific setup for using a /boot partition
setup_dedicated_boot_part()
{
  ROOTFS="${1}"
  ROOTFSTYPE="${2}"
  BOOTFS="${3}"
  BOOTMNT="${4}"

  # Set the root mount in loader.conf
  echo "vfs.root.mountfrom=\"${ROOTFSTYPE}:${ROOTFS}\"" >> ${FSMNT}/boot/loader.conf
  rc_halt "mkdir -p ${FSMNT}/${BOOTMNT}/boot"
  rc_halt "mv ${FSMNT}/boot/* ${FSMNT}${BOOTMNT}/boot/"
  rc_halt "mv ${FSMNT}${BOOTMNT}/boot ${FSMNT}/boot/"
  rc_halt "umount /dev/${BOOTFS}"
  rc_halt "mount /dev/${BOOTFS} ${FSMNT}${BOOTMNT}"
  rc_halt "rmdir ${FSMNT}/boot"

  # Strip the '/' from BOOTMNT before making symlink
  BOOTMNTNS="`echo ${BOOTMNT} | sed 's|/||g'`"
  rc_halt "chroot ${FSMNT} ln -s ${BOOTMNTNS}/boot /boot"

};

# Function which creates the /etc/fstab for the installed system
setup_fstab()
{
  FSTAB="${FSMNT}/etc/fstab"
  rm ${FSTAB} >/dev/null 2>/dev/null

  # Create the header
  echo "# Device		Mountpoint		FStype		Options	Dump Pass" >> ${FSTAB}

  # Loop through the partitions, and start creating /etc/fstab
  for PART in `ls ${PARTDIR}`
  do
    PARTFS="`cat ${PARTDIR}/${PART} | cut -d ':' -f 1`"
    PARTMNT="`cat ${PARTDIR}/${PART} | cut -d ':' -f 2`"
    PARTENC="`cat ${PARTDIR}/${PART} | cut -d ':' -f 3`"
    PARTLABEL="`cat ${PARTDIR}/${PART} | cut -d ':' -f 4`"

    DRIVE="`echo ${PART} | rev | cut -b 4- | rev`"
    # Check if this device is being mirrored
    if [ -e "${MIRRORCFGDIR}/${DRIVE}" ]
    then
      # This device is apart of a gmirror, lets reset PART to correct value
      MDRIVE="mirror/`cat ${MIRRORCFGDIR}/${DRIVE} | cut -d ':' -f 3`"
      TMP="`echo ${PART} | rev | cut -b -3 | rev`"
      PART="${MDRIVE}${TMP}"
      PARTLABEL=""
    fi

    # Unset EXT
    EXT=""

    # Figure out if we are using a glabel, or the raw name for this entry
    if [ ! -z "${PARTLABEL}" ]
    then
      DEVICE="label/${PARTLABEL}"
    else
      # Check if using encryption
      if [ "${PARTENC}" = "ON" ] ; then
        EXT=".eli"
      fi

      if [ "${PARTFS}" = "UFS+J" ] ; then
        EXT="${EXT}.journal"
      fi
      DEVICE="${PART}${EXT}"
    fi

    # Set our ROOTFSTYPE for loader.conf if necessary
    check_for_mount "${PARTMNT}" "/"
    if [ "$?" = "0" ] ; then
      if [ "${PARTFS}" = "ZFS" ] ; then
        ROOTFSTYPE="zfs"
        XPOOLNAME=$(get_zpool_name "${PART}")
        ROOTFS="${ZPOOLNAME}"
      else
        ROOTFS="${DEVICE}"
        ROOTFSTYPE="ufs"
      fi
    fi

    # Only create non-zfs partitions
    if [ "${PARTFS}" != "ZFS" ]
    then

      # Make sure geom_journal is loaded
      if [ "${PARTFS}" = "UFS+J" ] ; then
        setup_gjournal
      fi

      # Save the BOOTFS for call at the end
      if [ "${PARTMNT}" = "/boot" ] ; then
        BOOTFS="${PART}${EXT}"
        BOOTMNT="${BOOT_PART_MOUNT}"
        PARTMNT="${BOOTMNT}"
      fi

      # Echo out the fstab entry now
      if [ "${PARTFS}" = "SWAP" ]
      then
        echo "/dev/${DEVICE}	none			swap		sw		0	0" >> ${FSTAB}
      else
        echo "/dev/${DEVICE}  	${PARTMNT}			ufs		rw,noatime	1	1" >> ${FSTAB}
      fi

    fi # End of ZFS Check
  done

  # Setup some specific PC-BSD fstab options
  if [ "$INSTALLTYPE" != "FreeBSD" ]
  then
    echo "procfs			/proc			procfs		rw		0	0" >> ${FSTAB}
    echo "linprocfs		/compat/linux/proc	linprocfs	rw		0	0" >> ${FSTAB}
    echo "tmpfs				/tmp			tmpfs		rw,mode=1777	0	0" >> ${FSTAB}
  fi

  # If we have a dedicated /boot, run the post-install setup of it now
  if [ ! -z "${BOOTMNT}" ] ; then
    setup_dedicated_boot_part "${ROOTFS}" "${ROOTFSTYPE}" "${BOOTFS}" "${BOOTMNT}"
  fi

};

# Setup our disk mirroring with gmirror
setup_gmirror()
{
  NUM="0"

  cd ${MIRRORCFGDIR}
  for DISK in `ls *`
  do
    MIRRORDISK="`cat ${DISK} | cut -d ':' -f 1`"
    MIRRORBAL="`cat ${DISK} | cut -d ':' -f 2`"

    # Create this mirror device
    gmirror label -vb $MIRRORBAL gm${NUM} /dev/${DISK}

    sleep 3

    # Save the gm<num> device in our config
    echo "${MIRRORDISK}:${MIRRORBAL}:gm${NUM}" > ${DISK}

    sleep 3

    NUM="`expr ${NUM} + 1`"
  done

  cat ${FSMNT}/boot/loader.conf 2>/dev/null | grep 'geom_mirror_load="YES"' >/dev/null 2>/dev/null
  if [ "$?" != "0" ]
  then
    echo 'geom_mirror_load="YES"' >>${FSMNT}/boot/loader.conf
  fi

};

# Function which saves geli keys and sets up loading of them at boot
setup_geli_loading()
{

  # Make our keys dir
  mkdir -p ${FSMNT}/boot/keys >/dev/null 2>/dev/null

  cd ${GELIKEYDIR}
  for KEYFILE in `ls *`
  do
     # Figure out the partition name based on keyfile name removing .key
     PART="`echo ${KEYFILE} | cut -d '.' -f 1`"

     # Add the entries to loader.conf to start this geli provider at boot
     echo "geli_${PART}_keyfile0_load=\"YES\"" >> ${FSMNT}/boot/loader.conf
     echo "geli_${PART}_keyfile0_type=\"${PART}:geli_keyfile0\"" >> ${FSMNT}/boot/loader.conf
     echo "geli_${PART}_keyfile0_name=\"/boot/keys/${KEYFILE}\"" >> ${FSMNT}/boot/loader.conf

     # If we have a passphrase, set it up now
     if [ -e "${PARTDIR}-enc/${PART}-encpass" ] ; then
       cat ${PARTDIR}-enc/${PART}-encpass | geli setkey -S -n 0 -p -k ${KEYFILE} -K ${KEYFILE} ${PART}
       geli configure -b ${PART}
     fi

     # Copy the key to the disk
     cp ${KEYFILE} ${FSMNT}/boot/keys/${KEYFILE}
  done

  # Make sure we have geom_eli set to load at boot
  cat ${FSMNT}/boot/loader.conf 2>/dev/null | grep 'geom_eli_load="YES"' >/dev/null 2>/dev/null
  if [ "$?" != "0" ]
  then
    echo 'geom_eli_load="YES"' >>${FSMNT}/boot/loader.conf
  fi

};

# Function to generate a random hostname if none was specified
gen_hostname()
{
  RAND="`jot -r 1 1 9000`"

  if [ "$INSTALLTYPE" = "FreeBSD" ]
  then
    VAL="freebsd-${RAND}"
  else
    VAL="pcbsd-${RAND}"
  fi

  export VAL

};

# Function which sets up the hostname for the system
setup_hostname()
{

  get_value_from_cfg hostname
  HOSTNAME="${VAL}"

  # If we don't have a hostname, make one up
  if [ -z "${HOSTNAME}" ]
  then
    gen_hostname
    HOSTNAME="${VAL}"
  fi

  # Clean up any saved hostname
  cat ${FSMNT}/etc/rc.conf | grep -v "hostname=" >${FSMNT}/etc/rc.conf.new
  mv ${FSMNT}/etc/rc.conf.new ${FSMNT}/etc/rc.conf

  # Set the hostname now
  echo_log "Setting hostname: ${HOSTNAME}"
  echo "hostname=\"${HOSTNAME}\"" >> ${FSMNT}/etc/rc.conf
  sed -i -e "s|my.domain|${HOSTNAME} ${HOSTNAME}|g" ${FSMNT}/etc/hosts

};

# Check and make sure geom_journal is enabled on the system
setup_gjournal()
{

  # Make sure we have geom_journal set to load at boot
  cat ${FSMNT}/boot/loader.conf 2>/dev/null | grep 'geom_journal_load="YES"' >/dev/null 2>/dev/null
  if [ "$?" != "0" ]
  then
    echo 'geom_journal_load="YES"' >>${FSMNT}/boot/loader.conf
  fi

};

# Function which sets the root password from the install config
set_root_pw()
{
  get_value_from_cfg_with_spaces rootPass
  PW="${VAL}"

  # If we don't have a root pass, return
  if [ -z "${PW}" ]
  then
    return 0
  fi

  echo_log "Setting root password"
  echo "${PW}" > ${FSMNT}/.rootpw
  run_chroot_cmd "cat /.rootpw | pw usermod root -h 0"
  rc_halt "rm ${FSMNT}/.rootpw"

};

run_final_cleanup()
{

 # Check if we need to run any gmirror setup
 ls ${MIRRORCFGDIR}/* >/dev/null 2>/dev/null
 if [ "$?" = "0" ]
 then
   # Lets setup gmirror now
   setup_gmirror
 fi

 # Check if we need to save any geli keys
 ls ${GELIKEYDIR}/* >/dev/null 2>/dev/null
 if [ "$?" = "0" ]
 then
   # Lets setup geli loading
   setup_geli_loading
 fi

 # Set the root_pw if it is specified
 set_root_pw

 # Generate the fstab for the installed system
 setup_fstab

};

#!/bin/sh
# Functions which perform the final cleanup after an install

# Finishes up with ZFS setup before unmounting
zfs_cleanup_unmount()
{
  # Loop through our FS and see if we have any ZFS partitions to cleanup
  for PART in `ls ${PARTDIR}`
  do
    PARTFS="`cat ${PARTDIR}/${PART} | cut -d ':' -f 1`"
    PARTMNT="`cat ${PARTDIR}/${PART} | cut -d ':' -f 2`"
    ZPOOLNAME=$(get_zpool_name "${PART}")

    if [ "$PARTFS" = "ZFS" ]
    then
      # Check if we have multiple zfs mounts specified
      for ZMNT in `echo ${PARTMNT} | sed 's|,| |g'`
      do
        if [ "${ZMNT}" = "/" ]
        then
          # Make sure we haven't already added the zfs boot line when
          # Creating a dedicated "/boot" partition
          cat ${FSMNT}/boot/loader.conf 2>/dev/null | grep "vfs.root.mountfrom=" >/dev/null 2>/dev/null
          if [ "$?" != "0" ] ; then
            echo "vfs.root.mountfrom=\"zfs:${ZPOOLNAME}\"" >> ${FSMNT}/boot/loader.conf
          fi
          FOUNDZFSROOT="${ZPOOLNAME}" ; export FOUNDZFSROOT
        fi
      done
      FOUNDZFS="1"
    fi
  done

  if [ ! -z "${FOUNDZFS}" ]
  then
    # Check if we need to add our ZFS flags to rc.conf, src.conf and loader.conf
    cat ${FSMNT}/boot/loader.conf 2>/dev/null | grep 'zfs_load="YES"' >/dev/null 2>/dev/null
    if [ "$?" != "0" ]
    then
      echo 'zfs_load="YES"' >>${FSMNT}/boot/loader.conf
    fi
    cat ${FSMNT}/etc/rc.conf 2>/dev/null | grep 'zfs_enable="YES"' >/dev/null 2>/dev/null
    if [ "$?" != "0" ]
    then
      #echo 'zfs_enable="YES"' >>${FSMNT}/etc/rc.conf
    fi

    sleep 2
    # Copy over any ZFS cache data
    cp /boot/zfs/* ${FSMNT}/boot/zfs/

    # Copy the hostid so that our zfs cache works
    cp /etc/hostid ${FSMNT}/etc/hostid
  fi

  # Loop through our FS and see if we have any ZFS partitions to cleanup
  for PART in `ls ${PARTDIR}`
  do
    PARTFS="`cat ${PARTDIR}/${PART} | cut -d ':' -f 1`"
    PARTMNT="`cat ${PARTDIR}/${PART} | cut -d ':' -f 2`"
    PARTENC="`cat ${PARTDIR}/${PART} | cut -d ':' -f 3`"
    ZPOOLNAME=$(get_zpool_name "${PART}")

    if [ "$PARTFS" = "ZFS" ]
    then
      # Check if we have multiple zfs mounts specified
      for ZMNT in `echo ${PARTMNT} | sed 's|,| |g'`
      do
        PARTMNTREV="${ZMNT} ${PARTMNTREV}"
      done

      for ZMNT in ${PARTMNTREV}
      do
        if [ "${ZMNT}" != "/" ]
        then
          #rc_halt "zfs set mountpoint=${ZMNT} ${ZPOOLNAME}${ZMNT}"
          #rc_halt "zfs unmount ${ZPOOLNAME}${ZMNT}"
          #sleep 2
        fi
      done
    fi
  done

};

# Function which performs the specific setup for using a /boot partition
setup_dedicated_boot_part()
{
  ROOTFS="${1}"
  ROOTFSTYPE="${2}"
  BOOTFS="${3}"
  BOOTMNT="${4}"

  # Set the root mount in loader.conf
  echo "vfs.root.mountfrom=\"${ROOTFSTYPE}:${ROOTFS}\"" >> ${FSMNT}/boot/loader.conf
  rc_halt "mkdir -p ${FSMNT}/${BOOTMNT}/boot"
  rc_halt "mv ${FSMNT}/boot/* ${FSMNT}${BOOTMNT}/boot/"
  rc_halt "mv ${FSMNT}${BOOTMNT}/boot ${FSMNT}/boot/"
  rc_halt "umount /dev/${BOOTFS}"
  rc_halt "mount /dev/${BOOTFS} ${FSMNT}${BOOTMNT}"
  rc_halt "rmdir ${FSMNT}/boot"

  # Strip the '/' from BOOTMNT before making symlink
  BOOTMNTNS="`echo ${BOOTMNT} | sed 's|/||g'`"
  rc_halt "chroot ${FSMNT} ln -s ${BOOTMNTNS}/boot /boot"

};

# Function which creates the /etc/fstab for the installed system
setup_fstab()
{
  FSTAB="${FSMNT}/etc/fstab"
  rm ${FSTAB} >/dev/null 2>/dev/null

  # Create the header
  echo "# Device		Mountpoint		FStype		Options	Dump Pass" >> ${FSTAB}

  # Loop through the partitions, and start creating /etc/fstab
  for PART in `ls ${PARTDIR}`
  do
    PARTFS="`cat ${PARTDIR}/${PART} | cut -d ':' -f 1`"
    PARTMNT="`cat ${PARTDIR}/${PART} | cut -d ':' -f 2`"
    PARTENC="`cat ${PARTDIR}/${PART} | cut -d ':' -f 3`"
    PARTLABEL="`cat ${PARTDIR}/${PART} | cut -d ':' -f 4`"

    DRIVE="`echo ${PART} | rev | cut -b 4- | rev`"
    # Check if this device is being mirrored
    if [ -e "${MIRRORCFGDIR}/${DRIVE}" ]
    then
      # This device is apart of a gmirror, lets reset PART to correct value
      MDRIVE="mirror/`cat ${MIRRORCFGDIR}/${DRIVE} | cut -d ':' -f 3`"
      TMP="`echo ${PART} | rev | cut -b -3 | rev`"
      PART="${MDRIVE}${TMP}"
      PARTLABEL=""
    fi

    # Unset EXT
    EXT=""

    # Figure out if we are using a glabel, or the raw name for this entry
    if [ ! -z "${PARTLABEL}" ]
    then
      DEVICE="label/${PARTLABEL}"
    else
      # Check if using encryption
      if [ "${PARTENC}" = "ON" ] ; then
        EXT=".eli"
      fi

      if [ "${PARTFS}" = "UFS+J" ] ; then
        EXT="${EXT}.journal"
      fi
      DEVICE="${PART}${EXT}"
    fi

    # Set our ROOTFSTYPE for loader.conf if necessary
    check_for_mount "${PARTMNT}" "/"
    if [ "$?" = "0" ] ; then
      if [ "${PARTFS}" = "ZFS" ] ; then
        ROOTFSTYPE="zfs"
        XPOOLNAME=$(get_zpool_name "${PART}")
        ROOTFS="${ZPOOLNAME}"
      else
        ROOTFS="${DEVICE}"
        ROOTFSTYPE="ufs"
      fi
    fi

    # Only create non-zfs partitions
    if [ "${PARTFS}" != "ZFS" ]
    then

      # Make sure geom_journal is loaded
      if [ "${PARTFS}" = "UFS+J" ] ; then
        setup_gjournal
      fi

      # Save the BOOTFS for call at the end
      if [ "${PARTMNT}" = "/boot" ] ; then
        BOOTFS="${PART}${EXT}"
        BOOTMNT="${BOOT_PART_MOUNT}"
        PARTMNT="${BOOTMNT}"
      fi

      # Echo out the fstab entry now
      if [ "${PARTFS}" = "SWAP" ]
      then
        echo "/dev/${DEVICE}	none			swap		sw		0	0" >> ${FSTAB}
      else
        echo "/dev/${DEVICE}  	${PARTMNT}			ufs		rw,noatime	1	1" >> ${FSTAB}
      fi

    fi # End of ZFS Check
  done

  # Setup some specific PC-BSD fstab options
  if [ "$INSTALLTYPE" != "FreeBSD" ]
  then
    echo "procfs			/proc			procfs		rw		0	0" >> ${FSTAB}
    echo "linprocfs		/compat/linux/proc	linprocfs	rw		0	0" >> ${FSTAB}
    echo "tmpfs				/tmp			tmpfs		rw,mode=1777	0	0" >> ${FSTAB}
  fi

  # If we have a dedicated /boot, run the post-install setup of it now
  if [ ! -z "${BOOTMNT}" ] ; then
    setup_dedicated_boot_part "${ROOTFS}" "${ROOTFSTYPE}" "${BOOTFS}" "${BOOTMNT}"
  fi

};

# Setup our disk mirroring with gmirror
setup_gmirror()
{
  NUM="0"

  cd ${MIRRORCFGDIR}
  for DISK in `ls *`
  do
    MIRRORDISK="`cat ${DISK} | cut -d ':' -f 1`"
    MIRRORBAL="`cat ${DISK} | cut -d ':' -f 2`"

    # Create this mirror device
    gmirror label -vb $MIRRORBAL gm${NUM} /dev/${DISK}

    sleep 3

    # Save the gm<num> device in our config
    echo "${MIRRORDISK}:${MIRRORBAL}:gm${NUM}" > ${DISK}

    sleep 3

    NUM="`expr ${NUM} + 1`"
  done

  cat ${FSMNT}/boot/loader.conf 2>/dev/null | grep 'geom_mirror_load="YES"' >/dev/null 2>/dev/null
  if [ "$?" != "0" ]
  then
    echo 'geom_mirror_load="YES"' >>${FSMNT}/boot/loader.conf
  fi

};

# Function which saves geli keys and sets up loading of them at boot
setup_geli_loading()
{

  # Make our keys dir
  mkdir -p ${FSMNT}/boot/keys >/dev/null 2>/dev/null

  cd ${GELIKEYDIR}
  for KEYFILE in `ls *`
  do
     # Figure out the partition name based on keyfile name removing .key
     PART="`echo ${KEYFILE} | cut -d '.' -f 1`"

     # Add the entries to loader.conf to start this geli provider at boot
     echo "geli_${PART}_keyfile0_load=\"YES\"" >> ${FSMNT}/boot/loader.conf
     echo "geli_${PART}_keyfile0_type=\"${PART}:geli_keyfile0\"" >> ${FSMNT}/boot/loader.conf
     echo "geli_${PART}_keyfile0_name=\"/boot/keys/${KEYFILE}\"" >> ${FSMNT}/boot/loader.conf

     # If we have a passphrase, set it up now
     if [ -e "${PARTDIR}-enc/${PART}-encpass" ] ; then
       cat ${PARTDIR}-enc/${PART}-encpass | geli setkey -S -n 0 -p -k ${KEYFILE} -K ${KEYFILE} ${PART}
       geli configure -b ${PART}
     fi

     # Copy the key to the disk
     cp ${KEYFILE} ${FSMNT}/boot/keys/${KEYFILE}
  done

  # Make sure we have geom_eli set to load at boot
  cat ${FSMNT}/boot/loader.conf 2>/dev/null | grep 'geom_eli_load="YES"' >/dev/null 2>/dev/null
  if [ "$?" != "0" ]
  then
    echo 'geom_eli_load="YES"' >>${FSMNT}/boot/loader.conf
  fi

};

# Function to generate a random hostname if none was specified
gen_hostname()
{
  RAND="`jot -r 1 1 9000`"

  if [ "$INSTALLTYPE" = "FreeBSD" ]
  then
    VAL="freebsd-${RAND}"
  else
    VAL="pcbsd-${RAND}"
  fi

  export VAL

};

# Function which sets up the hostname for the system
setup_hostname()
{

  get_value_from_cfg hostname
  HOSTNAME="${VAL}"

  # If we don't have a hostname, make one up
  if [ -z "${HOSTNAME}" ]
  then
    gen_hostname
    HOSTNAME="${VAL}"
  fi

  # Clean up any saved hostname
  cat ${FSMNT}/etc/rc.conf | grep -v "hostname=" >${FSMNT}/etc/rc.conf.new
  mv ${FSMNT}/etc/rc.conf.new ${FSMNT}/etc/rc.conf

  # Set the hostname now
  echo_log "Setting hostname: ${HOSTNAME}"
  echo "hostname=\"${HOSTNAME}\"" >> ${FSMNT}/etc/rc.conf
  sed -i -e "s|my.domain|${HOSTNAME} ${HOSTNAME}|g" ${FSMNT}/etc/hosts

};

# Check and make sure geom_journal is enabled on the system
setup_gjournal()
{

  # Make sure we have geom_journal set to load at boot
  cat ${FSMNT}/boot/loader.conf 2>/dev/null | grep 'geom_journal_load="YES"' >/dev/null 2>/dev/null
  if [ "$?" != "0" ]
  then
    echo 'geom_journal_load="YES"' >>${FSMNT}/boot/loader.conf
  fi

};

# Function which sets the root password from the install config
set_root_pw()
{
  get_value_from_cfg_with_spaces rootPass
  PW="${VAL}"

  # If we don't have a root pass, return
  if [ -z "${PW}" ]
  then
    return 0
  fi

  echo_log "Setting root password"
  echo "${PW}" > ${FSMNT}/.rootpw
  run_chroot_cmd "cat /.rootpw | pw usermod root -h 0"
  rc_halt "rm ${FSMNT}/.rootpw"

};

run_final_cleanup()
{

 # Check if we need to run any gmirror setup
 ls ${MIRRORCFGDIR}/* >/dev/null 2>/dev/null
 if [ "$?" = "0" ]
 then
   # Lets setup gmirror now
   setup_gmirror
 fi

 # Check if we need to save any geli keys
 ls ${GELIKEYDIR}/* >/dev/null 2>/dev/null
 if [ "$?" = "0" ]
 then
   # Lets setup geli loading
   setup_geli_loading
 fi

 # Set the root_pw if it is specified
 set_root_pw

 # Generate the fstab for the installed system
 setup_fstab

};


Customized running script for EPOS 3.1

Customized running script for EPOS 3.1

#!/bin/bash
set -e

if [[ $# -lt 2 ]]; then
    echo "You must specify at least two parameters where to write output."
    exit -1
fi

EPOSFILE1=${1}
EPOSFILE2=${2}

echo "running in `pwd`, writing EPOSroot to ${EPOSFILE1} and ${EPOSFILE2}"

# prepare environment
source /cvmfs/alice.cern.ch/etc/login.sh
eval $(alienv printenv EPOS::v3.111-8)

# prepare EPOS configuration
OptnsEpos3="pp"
ENERGY="13000"
MAXEPOSEVENTS=100
EPOSNFREEZE=10

cp ${ALICE_PHYSICS}/PWG/MCLEGO/EPOS/${OptnsEpos3}.optns .
sed -i "s/\(istmax\)\(\s\+[0-9]\+\)/\1 10/g" ${OptnsEpos3}.optns
sed -i "s/\(ecms\)\(\s\+[0-9]\+\)/\1 $ENERGY/g" ${OptnsEpos3}.optns
sed -i "s/\(nfull\)\(\s\+[0-9]\+\)/\1 $(( $MAXEPOSEVENTS / $EPOSNFREEZE ))/g" ${OptnsEpos3}.optns
sed -i "s/\(nfreeze\)\(\s\+[0-9]\+\)/\1 $EPOSNFREEZE/g" ${OptnsEpos3}.optns

# generate events and alternate between two files
until true; do
    epos $OptnsEpos3 > epos.out 2> epos.err;
    if [ ! -f $EPOSFILE1 ]; then
	mv z-${OptnsEpos3}.root $EPOSFILE1;
    elif [ ! -f $EPOSFILE2 ]; then
	mv z-${OptnsEpos3}.root $EPOSFILE2;
    else
	sleep 1s
    fi
done

error: pathspec 'PWGLF/CommonUtils/MConthefly/macros/gen_epos3.sh' did not match any file(s) known to git


Add sync script

Add sync script

#!/bin/bash

DEST="$1"

if [ -z "$DEST" ]; then exit 1; fi

inotifywait -r -m -e close_write --format '%w%f' . |\
while read file
do
        echo $file
	rsync -azvq $file ${DEST}/$file
	echo -n 'Completed at '
	date
done

error: pathspec 'bin/rsync.sh' did not match any file(s) known to git


better ordering

better ordering

DOKKU_REPO=${DOKKU_REPO:-"https://github.com/progrium/dokku.git"}

apt-get install -y linux-image-extra-`uname -r` software-properties-common
add-apt-repository -y ppa:dotcloud/lxc-docker
apt-get update && apt-get install -y lxc-docker git ruby nginx make

cd ~ && git clone ${DOKKU_REPO}
cd dokku && make install
cd buildstep && make build

echo "include /home/git/*/nginx.conf;" > /etc/nginx/conf.d/dokku.conf
/etc/init.d/nginx start
start nginx-reloader

[[ $(dig +short $HOSTNAME) ]] && HOSTFILE=DOMAIN || HOSTFILE=HOSTNAME
echo $HOSTNAME > /home/git/$HOSTFILE

echo
echo "Be sure to upload a public key for your user:"
echo "  cat ~/.ssh/id_rsa.pub | ssh root@$HOSTNAME \"gitreceive upload-key progrium\""

DOKKU_REPO=${DOKKU_REPO:-"https://github.com/progrium/dokku.git"}

apt-get install -y linux-image-extra-`uname -r` software-properties-common
add-apt-repository -y ppa:dotcloud/lxc-docker
apt-get update && apt-get install -y lxc-docker git ruby nginx make

cd ~ && git clone ${DOKKU_REPO}
cd dokku && make install
cd buildstep && make build

start nginx-reloader
/etc/init.d/nginx start

echo "include /home/git/*/nginx.conf;" > /etc/nginx/conf.d/dokku.conf
[[ $(dig +short $HOSTNAME) ]] && HOSTFILE=DOMAIN || HOSTFILE=HOSTNAME
echo $HOSTNAME > /home/git/$HOSTFILE

echo
echo "Be sure to upload a public key for your user:"
echo "  cat ~/.ssh/id_rsa.pub | ssh root@$HOSTNAME \"gitreceive upload-key progrium\""


[build] Missing bootstrap.sh

[build] Missing bootstrap.sh

#!/usr/bin/env bash
autoreconf -fi --warning=no-portability

error: pathspec 'bootstrap.sh' did not match any file(s) known to git


Create requeriments_dimredux_install.sh

Create requeriments_dimredux_install.sh

Criei esse script básico para instalar os pacotes necessários. Verifiquei que a
linha de commando muito extensa ele acabava não instalando os pacotes.

A ideia é instalar tudo que é básico e essencial para uma máquina recem
instalada.

remote: Support for password authentication was removed on August 13, 2021.
remote: Please see https://docs.github.com/en/get-started/getting-started-with-git/about-remote-repositories#cloning-with-https-urls for information on currently recommended modes of authentication.
fatal: Authentication failed for 'https://github.com/emersonclima/dimredux.git/'


use branch artifact

use branch artifact

remote: Support for password authentication was removed on August 13, 2021.
remote: Please see https://docs.github.com/en/get-started/getting-started-with-git/about-remote-repositories#cloning-with-https-urls for information on currently recommended modes of authentication.
fatal: Authentication failed for 'https://github.com/VirusTotal/content.git/'


VP9VideoRecompressor_V2 updated

VP9VideoRecompressor_V2 updated

remote: Support for password authentication was removed on August 13, 2021.
remote: Please see https://docs.github.com/en/get-started/getting-started-with-git/about-remote-repositories#cloning-with-https-urls for information on currently recommended modes of authentication.
fatal: Authentication failed for 'https://github.com/DarkCaster/Linux-Helper-Tools.git/'


Add Fedora 28

Add Fedora 28

DISTNAME=fedora
RELVER=28
BASEURL=http://ftp.fi.muni.cz/pub/linux/fedora/linux/releases/$RELVER/Everything/x86_64/os
UPDATES=http://ftp.fi.muni.cz/pub/linux/fedora/linux/updates/$RELVER/x86_64
RELEASE="$BASEURL/Packages/f/fedora-release-$RELVER-1.noarch.rpm
	$BASEURL/Packages/f/fedora-repos-$RELVER-1.noarch.rpm"
GROUPNAME="minimal install"
EXTRAPKGS="vim man"
REMOVEPKGS="plymouth"

. $INCLUDE/redhat-family.sh

bootstrap
configure-common

configure-redhat-common

configure-append <<EOF
systemctl disable NetworkManager.service
systemctl disable NetworkManager-wait-online.service
systemctl disable NetworkManager-dispatcher.service
systemctl enable  network.service
systemctl disable firewalld.service
EOF

run-configure

error: pathspec 'templates/fedora-28.sh' did not match any file(s) known to git


upgrade script from 6.2 to 6.3

upgrade script from 6.2 to 6.3

#!/bin/sh -e

# scripted installation of 6.2 to 6.3 upgrade
# https://github.com/gitlabhq/gitlabhq/blob/master/doc/update/6.2-to-6.3.md

# some options
GITUSER=git
GITHOME="/home/$GITUSER"
GITSUDO="sudo -u $GITUSER -H"

# All commands expect root access.
[ "$(whoami)" != "root" ] && echo "error: need to be root" && exit 1

# take backup
cd $GITHOME/gitlab
$GITSUDO bundle exec rake gitlab:backup:create RAILS_ENV=production
service gitlab stop

# checkout 6.3
$GITSUDO git fetch
# config/application.rb is not .example based, thus we need to manually fiddle with the one line difference
#  (6.2 has changed rack attack line and the comment about it is extra)
$GITSUDO grep -v "    # Uncomment to enable rack attack middleware" config/application.rb > config/application.rb.new && $GITSUDO mv config/application.rb.new config/application.rb
$GITSUDO git stash
$GITSUDO git checkout v6.3.0
$GITSUDO git stash pop

# checkout shell 1.7.9
cd $GITHOME/gitlab-shell
$GITSUDO git fetch
$GITSUDO git checkout v1.7.9

# install libs and migrations
cd $GITHOME/gitlab
$GITSUDO bundle install --without development test postgres --deployment
$GITSUDO bundle exec rake db:migrate RAILS_ENV=production
$GITSUDO bundle exec rake assets:clean RAILS_ENV=production
$GITSUDO bundle exec rake assets:precompile RAILS_ENV=production
$GITSUDO bundle exec rake cache:clear RAILS_ENV=production

# setup config files - nothing really has changed
cd $GITHOME/gitlab/config
git diff v6.2.0:config/gitlab.yml.example v6.3.0:config/gitlab.yml.example > gitlab-62-to-63.patch
sed -i 's,/v6.2.0:config/gitlab.yml.example,/v6.2.0:config/gitlab.yml,' gitlab-62-to-63.patch
sed -i 's,/v6.3.0:config/gitlab.yml.example,/v6.3.0:config/gitlab.yml,' gitlab-62-to-63.patch
patch -p2 < gitlab-62-to-63.patch && rm gitlab-62-to-63.patch

# update init script
rm /etc/init.d/gitlab
curl --output /etc/init.d/gitlab https://raw.github.com/gitlabhq/gitlabhq/v6.3.0/lib/support/init.d/gitlab
chmod +x /etc/init.d/gitlab

# restart services
service gitlab start
service nginx restart

# check health
$GITSUDO bundle exec rake gitlab:env:info RAILS_ENV=production
$GITSUDO bundle exec rake gitlab:check RAILS_ENV=production

# victory!
echo "VICTORY! Running 6.3!"

error: pathspec 'upgrades/6.2-to-6.3.sh' did not match any file(s) known to git


add missing actual install script for local tools/scripts

#! /bin/bash

for tool in $FILEDIR/local-bin/*
do
	cp $tool /usr/local/bin/
	chmod a+x /usr/local/bin/$(basename $tool)
done

error: pathspec 'inc/install-tools.sh' did not match any file(s) known to git


Create init/setup-psysh.zsh script

Create init/setup-psysh.zsh script

#!/usr/bin/env zsh

set -eu

echo "Setup psysh"

download_url="https://psysh.org/manual/ja/php_manual.sqlite"
download_to="${HOME}/.local/share/psysh/php_manual.sqlite"
download_to_dir=`dirname ${download_to}`

echo "Create ${download_to_dir} directory"
mkdir -p ${download_to_dir}

if [ -e ${download_to} ]; then
    echo "File ${download_to} is already exist."
    echo "Delete the old file."
    rm ${download_to}
fi

echo "Gets manual file from ${download_url}."
echo "Save to ${download_to}"
if type "wget" > /dev/null 2>&1; then
    echo "Use wget command."
    wget -O ${download_to} ${download_url}
elif type "curl" > /dev/null 2>&1; then
    echo "Use curl command."
    curl -o ${download_to} ${download_url}
else
    echo "\"wget\" or \"curl\" command is missing."
    echo "Skip psysh manual download."
fi

error: pathspec 'init/setup-psysh.zsh' did not match any file(s) known to git


Added file to configure Enviornment

Added file to configure Enviornment

#!/bin/bash

sudo apt-get update -y

sudo apt-get install -y apache2

echo "Hello! Sultan Here!" > /tmp/hello.txt

error: pathspec 'install-webserver.sh' did not match any file(s) known to git


alias u to ..

alias u to ..

remote: Support for password authentication was removed on August 13, 2021.
remote: Please see https://docs.github.com/en/get-started/getting-started-with-git/about-remote-repositories#cloning-with-https-urls for information on currently recommended modes of authentication.
fatal: Authentication failed for 'https://github.com/demhydraz/til.git/'


Added

Added

# THIS FILE IS UNDER VERSION CONTROL.  MAKE CHANGES IN YOUR REPO!!!!! #
#*********************************************************************#
#!/usr/bin/env bash

# AWS CDP-DEV account
cdp-dev() {
  aws-okta exec cdp-dev -- /bin/bash
}

# AWS CDP-DEV account
cdp-dev() {
  aws-okta exec cdp-cicd -- /bin/bash
}

# AWS fp-cloudsec-main account
cdp-dev() {
  aws-okta exec cdp-main -- /bin/bash
}

error: pathspec '.functions/forcepoint.sh' did not match any file(s) known to git


use most recent SAW

use most recent SAW

error: pathspec '.travis/.#install_saw.sh' did not match any file(s) known to git


Add vim-ack search plugin.

Add vim-ack search plugin.

remote: Support for password authentication was removed on August 13, 2021.
remote: Please see https://docs.github.com/en/get-started/getting-started-with-git/about-remote-repositories#cloning-with-https-urls for information on currently recommended modes of authentication.
fatal: Authentication failed for 'https://github.com/julianespinel/configfiles.git/'


Create hook.sh

Create hook.sh

#!/bin/bash

cd .git/hooks
wget https://gist.githubusercontent.com/Desgard/f2c34e4586eb92d50a6a10d0cac7dc0a/raw/61e51756b5fdc8ddae617d01d684ff3f2a4cc331/commit-msg.py -O commit-msg >/dev/null 2>&1

echo -e '\033[44;37;5m The deployment for githook is completed ! \033[0m  😎 '

error: pathspec 'hook.sh' did not match any file(s) known to git


Initialization script

Initialization script

#!/bin/bash

# common initialization of repository update and installation of python

sudo apt-get update
sudo apt-get upgrade

sudo apt-get install python2.7

error: pathspec 'init.sh' did not match any file(s) known to git


ubuntu14.04 c update

ubuntu14.04 c update

error: pathspec 'ubuntu14.04/c_userConfigure.sh' did not match any file(s) known to git


Add the new structWSF upgrade script from version 1.0a93 to 1.0a94.

#!/bin/bash

#
# This version upgrade script is a shell that explains how an upgrade script should be
# developed. It also includes a list of steps to perform depending on the kind of upgrade
# necessary for a specific upgrade of a version A to a version B.
#
# NOTE: this first upgrade script is just in that package for demonstration purposes only
#       the first structWSF version that can be upgraded is 1.0a92
#

STRUCTWSFVERSION="1.0a94"
STRUCTWSFPREVIOUSVERSION="1.0a93"

STRUCTWSFFOLDER=$1

STRUCTWSFDOWNLOADURL="https://github.com/downloads/structureddynamics/structWSF-Open-Semantic-Framework/structWSF-v$STRUCTWSFVERSION.zip"

# From: http://tldp.org/LDP/abs/html/colorizing.html
# Colorizing the installation process.

black='\E[1;30;40m'
red='\E[1;31;40m'
green='\E[1;32;40m'
yellow='\E[1;33;40m'
blue='\E[1;34;40m'
magenta='\E[1;35;40m'
cyan='\E[1;36;40m'
white='\E[1;37;40m'

cecho ()                     # Color-echo.
                             # Argument $1 = message
                             # Argument $2 = color
{
  local default_msg="No message passed."
                             # Doesn't really need to be a local variable.

  message=${1:-$default_msg}   # Defaults to default message.
  color=${2:-$white}           # Defaults to white, if not specified.

  echo -e "$color"
  echo -e "$message"

  tput sgr0                     # Reset to normal.

  return
}

cecho "\n\nBackuping the current version's files...\n"

DATA_INI=$( sed -rn '\|data_ini| s|.*public.*static.*=.*"(.+)".*|\1|p' ${STRUCTWSFFOLDER}/framework/WebService.php )
NETWORK_INI=$( sed -rn '\|network_ini| s|.*public.*static.*=.*"(.+)".*|\1|p' ${STRUCTWSFFOLDER}/framework/WebService.php )
cecho "Debug[2] (NETWORK_INI): $NETWORK_INI" $red
sudo cp -af $STRUCTWSFFOLDER"/" "/tmp/structWSF-backup-"$STRUCTWSFPREVIOUSVERSION"/"

sudo mkdir upgrade

cd upgrade

cecho "\n\nDownload structWSF...\n"

sudo wget $STRUCTWSFDOWNLOADURL

cecho "\n\nDecompressing structWSF...\n"

sudo unzip "structWSF-v"$STRUCTWSFVERSION".zip"

cd `ls -d structureddynamics*/`

sudo mv -f * ../

cd ../

sudo rm -rf `ls -d structureddynamics*/`

cecho "\n\nRemove default settings in the new version...\n"

sudo rm data.ini
sudo rm network.ini
sudo rm index.php
sudo rm auth/wsf_indexer.php
sudo rm scones/config.ini
sudo rm "structWSF-v$STRUCTWSFVERSION.zip"

cecho "\n\nMove new files to the current structWSF installation folder...\n"

sudo cp -af * $STRUCTWSFFOLDER"/"

#
# These are the steps that needs to be performed for each upgrade.
# Some of these steps may not apply to a specific version upgrade
# in which case it will simply be ignored.
#

# 1) Delete files in the previous version of structWSF that are not needed anymore
# 2) If the WebService.php file got modified, do re-create it using the same $data_ini and $network_ini settings

cecho "\n\nUpgrade WebService.php while keeping previous configs in it...\n"

sudo sed -i 's>public static $data_ini = "/data/";>public static $data_ini = "'$DATA_INI'";>' $STRUCTWSFFOLDER"/framework/WebService.php"
sudo sed -i 's>public static $network_ini = "/usr/share/structwsf/";>public static $network_ini = "'$NETWORK_INI'";>' $STRUCTWSFFOLDER"/framework/WebService.php"

# 3) If new data.ini settings got added, add them to the end of the current data.ini file
# 4) If new network.ini settings got added, add them to the end of the current network.ini file
# 5) If changes have been made to the Triple Store, do perform these changes
# 6) If the Solr schema changed, instruct the user what to do (upgrading the schema, and then reloading its data)
# 7) If new software or libraries are needed for this upgrade, then simply install and configure them.
# 8) Delete deprecated files

cecho "\n\nDelete the deprecated Browser web service endpoint...\n"

sudo rm -rf $STRUCTWSFFOLDER"/browse/"

cecho "\n\nDelete the deprecated CompoundWebService class...\n"

sudo rm -f $STRUCTWSFFOLDER"/framework/CompoundWebService.php"

cecho "\n\nDelete the deprecated RdfClass class...\n"

sudo rm -f $STRUCTWSFFOLDER"/framework/RdfClass.php"

cecho "\n\nDelete the deprecated RdfProperty class...\n"

sudo rm -f $STRUCTWSFFOLDER"/framework/RdfProperty.php"

cd ..
rm -rf upgrade

cecho "\n\nThe files of the previous version are still available in that folder: /tmp/structWSF-backup-"$STRUCTWSFPREVIOUSVERSION"/\n" $green

error: pathspec 'versions/v1_0a93_to_v1_0a94.sh' did not match any file(s) known to git


Add more details to how to fix license check failure in sanity build.

PiperOrigin-RevId: 301725066
Change-Id: Ie45918d16efee9298e5e2061fe3b7275b419e417

#!/usr/bin/env bash
# Copyright 2016 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
#
# Usage: ci_sanity.sh [--pep8] [--incremental] [bazel flags]
#
# Options:
#           run sanity checks: python 2&3 pylint checks and bazel nobuild
#  --pep8   run pep8 test only
#  --incremental  Performs checks incrementally, by using the files changed in
#                 the latest commit

# Current script directory
SCRIPT_DIR=$( cd ${0%/*} && pwd -P )
source "${SCRIPT_DIR}/builds/builds_common.sh"

ROOT_DIR=$( cd "$SCRIPT_DIR/../../.." && pwd -P )

# Helper functions
die() {
  echo $@
  exit 1
}

num_cpus() {
  # Get the number of CPUs
  if [[ -f /proc/cpuinfo ]]; then
    N_CPUS=$(grep -c ^processor /proc/cpuinfo)
  else
    # Fallback method
    N_CPUS=`getconf _NPROCESSORS_ONLN`
  fi
  if [[ -z ${N_CPUS} ]]; then
    die "ERROR: Unable to determine the number of CPUs"
  fi

  echo ${N_CPUS}
}

# Helper functions for examining changed files in the last non-merge git
# commit.

# Get the hash of the last non-merge git commit on the current branch.
# Usage: get_last_non_merge_git_commit
get_last_non_merge_git_commit() {
  git rev-list --no-merges -n 1 HEAD
}

# List files changed (i.e., added, removed or revised) in the last non-merge
# git commit.
# Usage: get_changed_files_in_last_non_merge_git_commit
get_changed_files_in_last_non_merge_git_commit() {
  git diff-tree --no-commit-id --name-only -r $(get_last_non_merge_git_commit)
}

# List Python files changed in the last non-merge git commit that still exist,
# i.e., not removed.
# Usage: get_py_files_to_check [--incremental]
get_py_files_to_check() {
  if [[ "$1" == "--incremental" ]]; then
    CHANGED_PY_FILES=$(get_changed_files_in_last_non_merge_git_commit | \
                       grep '.*\.py$')

    # Do not include files removed in the last non-merge commit.
    PY_FILES=""
    for PY_FILE in ${CHANGED_PY_FILES}; do
      if [[ -f "${PY_FILE}" ]]; then
        PY_FILES="${PY_FILES} ${PY_FILE}"
      fi
    done

    echo "${PY_FILES}"
  else
    find tensorflow -name '*.py'
  fi
}

# Subfunctions for substeps
# Run pylint
do_pylint() {
  # Usage: do_pylint [--incremental]
  #
  # Options:
  #   --incremental  Performs check on only the python files changed in the
  #                  last non-merge git commit.

  # Use this list to whitelist pylint errors
  ERROR_WHITELIST="^tensorflow/python/framework/function_test\.py.*\[E1123.*noinline "\
"^tensorflow/python/platform/default/_gfile\.py.*\[E0301.*non-iterator "\
"^tensorflow/python/platform/default/_googletest\.py.*\[E0102.*function\salready\sdefined "\
"^tensorflow/python/feature_column/feature_column_test\.py.*\[E0110.*abstract-class-instantiated "\
"^tensorflow/contrib/layers/python/layers/feature_column\.py.*\[E0110.*abstract-class-instantiated "\
"^tensorflow/contrib/eager/python/evaluator\.py.*\[E0202.*method-hidden "\
"^tensorflow/contrib/eager/python/metrics_impl\.py.*\[E0202.*method-hidden "\
"^tensorflow/contrib/rate/rate\.py.*\[E0202.*method-hidden "\
"^tensorflow/python/training/tracking/tracking\.py.*\[E0202.*method-hidden "\
"^tensorflow/python/platform/gfile\.py.*\[E0301.*non-iterator "\
"^tensorflow/python/keras/callbacks\.py.*\[E1133.*not-an-iterable "\
"^tensorflow/python/keras/engine/base_layer.py.*\[E0203.*access-member-before-definition "\
"^tensorflow/python/keras/layers/recurrent\.py.*\[E0203.*access-member-before-definition "\
"^tensorflow/python/kernel_tests/constant_op_eager_test.py.*\[E0303.*invalid-length-returned "\
"^tensorflow/python/keras/utils/data_utils.py.*\[E1102.*not-callable "\
"^tensorflow/python/autograph/.*_py3_test\.py.*\[E0001.*syntax-error "\
"^tensorflow/python/keras/preprocessing/image\.py.*\[E0240.*Inconsistent method resolution "

  echo "ERROR_WHITELIST=\"${ERROR_WHITELIST}\""

  if [[ $# != "0" ]]  && [[ $# != "1" ]]; then
    echo "Invalid syntax when invoking do_pylint"
    echo "Usage: do_pylint [--incremental]"
    return 1
  fi

  PYLINT_BIN="python3 -m pylint"

  if [[ "$1" == "--incremental" ]]; then
    PYTHON_SRC_FILES=$(get_py_files_to_check --incremental)

    if [[ -z "${PYTHON_SRC_FILES}" ]]; then
      echo "do_pylint will NOT run due to --incremental flag and due to the "\
"absence of Python code changes in the last commit."
      return 0
    else
      # For incremental builds, we still check all Python files in cases there
      # are function signature changes that affect unchanged Python files.
      PYTHON_SRC_FILES=$(get_py_files_to_check)
    fi
  elif [[ -z "$1" ]]; then
    PYTHON_SRC_FILES=$(get_py_files_to_check)
  else
    echo "Invalid syntax for invoking do_pylint"
    echo "Usage: do_pylint [--incremental]"
    return 1
  fi

  if [[ -z ${PYTHON_SRC_FILES} ]]; then
    echo "do_pylint found no Python files to check. Returning."
    return 0
  fi

  PYLINTRC_FILE="${SCRIPT_DIR}/pylintrc"

  if [[ ! -f "${PYLINTRC_FILE}" ]]; then
    die "ERROR: Cannot find pylint rc file at ${PYLINTRC_FILE}"
  fi

  NUM_SRC_FILES=$(echo ${PYTHON_SRC_FILES} | wc -w)
  NUM_CPUS=$(num_cpus)

  echo "Running pylint on ${NUM_SRC_FILES} files with ${NUM_CPUS} "\
"parallel jobs..."
  echo ""

  PYLINT_START_TIME=$(date +'%s')
  OUTPUT_FILE="$(mktemp)_pylint_output.log"
  ERRORS_FILE="$(mktemp)_pylint_errors.log"
  NONWL_ERRORS_FILE="$(mktemp)_pylint_nonwl_errors.log"

  rm -rf ${OUTPUT_FILE}
  rm -rf ${ERRORS_FILE}
  rm -rf ${NONWL_ERRORS_FILE}
  touch ${NONWL_ERRORS_FILE}

  ${PYLINT_BIN} --rcfile="${PYLINTRC_FILE}" --output-format=parseable \
      --jobs=${NUM_CPUS} ${PYTHON_SRC_FILES} > ${OUTPUT_FILE} 2>&1
  PYLINT_END_TIME=$(date +'%s')

  echo ""
  echo "pylint took $((PYLINT_END_TIME - PYLINT_START_TIME)) s"
  echo ""

  # Report only what we care about
  # Ref https://pylint.readthedocs.io/en/latest/technical_reference/features.html
  # E: all errors
  # W0311 bad-indentation
  # W0312 mixed-indentation
  # C0330 bad-continuation
  # C0301 line-too-long
  # C0326 bad-whitespace
  # W0611 unused-import
  # W0622 redefined-builtin
  grep -E '(\[E|\[W0311|\[W0312|\[C0330|\[C0301|\[C0326|\[W0611|\[W0622)' ${OUTPUT_FILE} > ${ERRORS_FILE}

  N_ERRORS=0
  while read -r LINE; do
    IS_WHITELISTED=0
    for WL_REGEX in ${ERROR_WHITELIST}; do
      if echo ${LINE} | grep -q "${WL_REGEX}"; then
        echo "Found a whitelisted error:"
        echo "  ${LINE}"
        IS_WHITELISTED=1
      fi
    done

    if [[ ${IS_WHITELISTED} == "0" ]]; then
      echo "${LINE}" >> ${NONWL_ERRORS_FILE}
      echo "" >> ${NONWL_ERRORS_FILE}
      ((N_ERRORS++))
    fi
  done <${ERRORS_FILE}

  echo ""
  if [[ ${N_ERRORS} != 0 ]]; then
    echo "FAIL: Found ${N_ERRORS} non-whitelisted pylint errors:"
    cat "${NONWL_ERRORS_FILE}"
    return 1
  else
    echo "PASS: No non-whitelisted pylint errors were found."
    return 0
  fi
}

# Run pep8 check
do_pep8() {
  # Usage: do_pep8 [--incremental]
  # Options:
  #   --incremental  Performs check on only the python files changed in the
  #                  last non-merge git commit.

  PEP8_BIN="/usr/local/bin/pep8"
  PEP8_CONFIG_FILE="${SCRIPT_DIR}/pep8"

  if [[ "$1" == "--incremental" ]]; then
    PYTHON_SRC_FILES=$(get_py_files_to_check --incremental)
    NUM_PYTHON_SRC_FILES=$(echo ${PYTHON_SRC_FILES} | wc -w)

    echo "do_pep8 will perform checks on only the ${NUM_PYTHON_SRC_FILES} "\
"Python file(s) changed in the last non-merge git commit due to the "\
"--incremental flag:"
    echo "${PYTHON_SRC_FILES}"
    echo ""
  else
    PYTHON_SRC_FILES=$(get_py_files_to_check)
  fi

  if [[ -z ${PYTHON_SRC_FILES} ]]; then
    echo "do_pep8 found no Python files to check. Returning."
    return 0
  fi

  if [[ ! -f "${PEP8_CONFIG_FILE}" ]]; then
    die "ERROR: Cannot find pep8 config file at ${PEP8_CONFIG_FILE}"
  fi
  echo "See \"${PEP8_CONFIG_FILE}\" for pep8 config( e.g., ignored errors)"

  NUM_SRC_FILES=$(echo ${PYTHON_SRC_FILES} | wc -w)

  echo "Running pep8 on ${NUM_SRC_FILES} files"
  echo ""

  PEP8_START_TIME=$(date +'%s')
  PEP8_OUTPUT_FILE="$(mktemp)_pep8_output.log"

  rm -rf ${PEP8_OUTPUT_FILE}

  ${PEP8_BIN} --config="${PEP8_CONFIG_FILE}" --statistics \
      ${PYTHON_SRC_FILES} 2>&1 | tee ${PEP8_OUTPUT_FILE}
  PEP8_END_TIME=$(date +'%s')

  echo ""
  echo "pep8 took $((PEP8_END_TIME - PEP8_START_TIME)) s"
  echo ""

  if [[ -s ${PEP8_OUTPUT_FILE} ]]; then
    echo "FAIL: pep8 found above errors and/or warnings."
    return 1
  else
    echo "PASS: No pep8 errors or warnings were found"
    return 0
  fi
}

do_buildifier(){
  BUILD_FILES=$(find tensorflow -name 'BUILD*')
  NUM_BUILD_FILES=$(echo ${BUILD_FILES} | wc -w)

  echo "Running do_buildifier on ${NUM_BUILD_FILES} files"
  echo ""

  BUILDIFIER_START_TIME=$(date +'%s')
  BUILDIFIER_OUTPUT_FILE="$(mktemp)_buildifier_output.log"

  rm -rf ${BUILDIFIER_OUTPUT_FILE}

  buildifier -v -mode=check \
    ${BUILD_FILES} 2>&1 | tee ${BUILDIFIER_OUTPUT_FILE}
  BUILDIFIER_END_TIME=$(date +'%s')

  echo ""
  echo "buildifier took $((BUILDIFIER_END_TIME - BUILDIFIER_START_TIME)) s"
  echo ""

  if [[ -s ${BUILDIFIER_OUTPUT_FILE} ]]; then
    echo "FAIL: buildifier found errors and/or warnings in above BUILD files."
    echo "buildifier suggested the following changes:"
    buildifier -v -mode=diff -diff_command=diff ${BUILD_FILES}
    echo "Please fix manually or run buildifier <file> to auto-fix."
    return 1
  else
    echo "PASS: No buildifier errors or warnings were found"
    return 0
  fi
}

do_external_licenses_check(){
  BUILD_TARGET="$1"
  LICENSES_TARGET="$2"

  EXTERNAL_LICENSES_CHECK_START_TIME=$(date +'%s')

  EXTERNAL_DEPENDENCIES_FILE="$(mktemp)_external_dependencies.log"
  LICENSES_FILE="$(mktemp)_licenses.log"
  MISSING_LICENSES_FILE="$(mktemp)_missing_licenses.log"
  EXTRA_LICENSES_FILE="$(mktemp)_extra_licenses.log"
  TMP_FILE="$(mktemp)_tmp.log"

  echo "Getting external dependencies for ${BUILD_TARGET}"
 bazel cquery "attr('licenses', 'notice', deps(${BUILD_TARGET}))" --keep_going > "${TMP_FILE}" 2>&1
 cat "${TMP_FILE}" \
  | grep -e "^\/\/" -e "^@" \
  | grep -E -v "^//tensorflow" \
  | sed -e 's|:.*||' \
  | sort \
  | uniq 2>&1 \
  | tee ${EXTERNAL_DEPENDENCIES_FILE}

  echo
  echo "Getting list of external licenses mentioned in ${LICENSES_TARGET}."
  bazel cquery "deps(${LICENSES_TARGET})" --keep_going > "${TMP_FILE}" 2>&1
 cat "${TMP_FILE}" \
  | grep -e "^\/\/" -e "^@" \
  | grep -E -v "^//tensorflow" \
  | sed -e 's|:.*||' \
  | sort \
  | uniq 2>&1 \
  | tee ${LICENSES_FILE}

  echo
  comm -1 -3 ${EXTERNAL_DEPENDENCIES_FILE}  ${LICENSES_FILE} 2>&1 | tee ${EXTRA_LICENSES_FILE}
  echo
  comm -2 -3 ${EXTERNAL_DEPENDENCIES_FILE}  ${LICENSES_FILE} 2>&1 | tee ${MISSING_LICENSES_FILE}

  EXTERNAL_LICENSES_CHECK_END_TIME=$(date +'%s')

  # Blacklist
  echo ${MISSING_LICENSES_FILE}
  grep \
    -e "@bazel_tools//third_party/" \
    -e "@bazel_tools//tools" \
    -e "@local" \
    -e "@com_google_absl//absl" \
    -e "@org_tensorflow//" \
    -e "@com_github_googlecloudplatform_google_cloud_cpp//google" \
    -e "@com_github_grpc_grpc//src/compiler" \
    -e "@platforms//os" \
    -v ${MISSING_LICENSES_FILE} > temp.txt
  mv temp.txt ${MISSING_LICENSES_FILE}

  # Whitelist
  echo ${EXTRA_LICENSE_FILE}
  grep \
    -e "//third_party/mkl" \
    -e "//third_party/mkl_dnn" \
    -e "@bazel_tools//src" \
    -e "@bazel_tools//tools/" \
    -e "@org_tensorflow//tensorflow" \
    -e "@com_google_absl//" \
    -e "//external" \
    -e "@local" \
    -e "@com_github_googlecloudplatform_google_cloud_cpp//" \
    -e "@embedded_jdk//" \
    -e "^//$" \
    -v ${EXTRA_LICENSES_FILE} > temp.txt
  mv temp.txt ${EXTRA_LICENSES_FILE}

  echo
  echo "do_external_licenses_check took $((EXTERNAL_LICENSES_CHECK_END_TIME - EXTERNAL_LICENSES_CHECK_START_TIME)) s"
  echo

  if [[ -s ${MISSING_LICENSES_FILE} ]] || [[ -s ${EXTRA_LICENSES_FILE} ]] ; then
    echo "FAIL: mismatch in packaged licenses and external dependencies"
    if [[ -s ${MISSING_LICENSES_FILE} ]] ; then
      echo "Missing the licenses for the following external dependencies:"
      cat ${MISSING_LICENSES_FILE}
      echo "Please add the license(s) to ${LICENSES_TARGET}."
    fi
    if [[ -s ${EXTRA_LICENSES_FILE} ]] ; then
      echo "Please remove the licenses for the following external dependencies from target ${LICENSES_TARGET}."
      cat ${EXTRA_LICENSES_FILE}
    fi
    rm -rf ${EXTERNAL_DEPENDENCIES_FILE}
    rm -rf ${LICENSES_FILE}
    rm -rf ${MISSING_LICENSES_FILE}
    rm -rf ${EXTRA_LICENSES_FILE}
    return 1
  else
    echo "PASS: all external licenses included."
    rm -rf ${EXTERNAL_DEPENDENCIES_FILE}
    rm -rf ${LICENSES_FILE}
    rm -rf ${MISSING_LICENSES_FILE}
    rm -rf ${EXTRA_LICENSES_FILE}
    return 0
  fi
}

do_pip_package_licenses_check() {
  echo "Running do_pip_package_licenses_check"
  echo ""
  do_external_licenses_check \
    "//tensorflow/tools/pip_package:build_pip_package" \
    "//tensorflow/tools/pip_package:licenses"
}

do_lib_package_licenses_check() {
  echo "Running do_lib_package_licenses_check"
  echo ""
  do_external_licenses_check \
    "//tensorflow:libtensorflow.so" \
    "//tensorflow/tools/lib_package:clicenses_generate"
}

do_java_package_licenses_check() {
  echo "Running do_java_package_licenses_check"
  echo ""
  do_external_licenses_check \
    "//tensorflow/java:libtensorflow_jni.so" \
    "//tensorflow/tools/lib_package:jnilicenses_generate"
}

#Check for the bazel cmd status (First arg is error message)
cmd_status(){
  if [[ $? != 0 ]]; then
    echo ""
    echo "FAIL: ${BUILD_CMD}"
    echo "  $1 See lines above for details."
    return 1
  else
    echo ""
    echo "PASS: ${BUILD_CMD}"
    return 0
  fi
}

# Run bazel build --nobuild to test the validity of the BUILD files
# TODO(mikecase): Remove TF Lite exclusion from this list. Exclusion is
# necessary since the @androidsdk WORKSPACE dependency is commented
# out by default in TF WORKSPACE file.
do_bazel_nobuild() {
  BUILD_TARGET="//tensorflow/..."
  BUILD_TARGET="${BUILD_TARGET} -//tensorflow/lite/delegates/gpu/..."
  BUILD_TARGET="${BUILD_TARGET} -//tensorflow/lite/java/demo/app/..."
  BUILD_TARGET="${BUILD_TARGET} -//tensorflow/lite/schema/..."
  BUILD_CMD="bazel build --nobuild ${BAZEL_FLAGS} -- ${BUILD_TARGET}"

  ${BUILD_CMD}

  cmd_status \
    "This is due to invalid BUILD files."
}

do_bazel_deps_query() {
  local BUILD_TARGET='//tensorflow/...'
  # Android targets tend to depend on an Android runtime being available.
  # Exclude until the sanity test has such a runtime available.
  #
  # TODO(mikecase): Remove TF Lite exclusion from this list. Exclusion is
  # necessary since the @androidsdk WORKSPACE dependency is commented out by
  # default in TF WORKSPACE file.
  local BUILD_TARGET="${BUILD_TARGET}"' - kind("android_*", //tensorflow/...)'

  # We've set the flag noimplicit_deps as a workaround for
  # https://github.com/bazelbuild/bazel/issues/10544
  bazel query ${BAZEL_FLAGS} --noimplicit_deps -- "deps($BUILD_TARGET)" > /dev/null

  cmd_status \
    "This is due to invalid BUILD files."
}

do_pip_smoke_test() {
  cd "$ROOT_DIR/tensorflow/tools/pip_package"
  python pip_smoke_test.py
}

do_code_link_check() {
  tensorflow/tools/ci_build/code_link_check.sh
}

# List .h|.cc files changed in the last non-merge git commit that still exist,
# i.e., not removed.
# Usage: get_clang_files_to_check [--incremental]
get_clang_files_to_check() {
  if [[ "$1" == "--incremental" ]]; then
    CHANGED_CLANG_FILES=$(get_changed_files_in_last_non_merge_git_commit | \
                       grep '.*\.h$\|.*\.cc$')

    # Do not include files removed in the last non-merge commit.
    CLANG_FILES=""
    for CLANG_FILE in ${CHANGED_CLANG_FILES}; do
      if [[ -f "${CLANG_FILE}" ]]; then
        CLANG_FILES="${CLANG_FILES} ${CLANG_FILE}"
      fi
    done

    echo "${CLANG_FILES}"
  else
    find tensorflow -name '*.h' -o -name '*.cc'
  fi
}

do_clang_format_check() {
  if [[ $# != "0" ]] && [[ $# != "1" ]]; then
    echo "Invalid syntax when invoking do_clang_format_check"
    echo "Usage: do_clang_format_check [--incremental]"
    return 1
  fi

  if [[ "$1" == "--incremental" ]]; then
    CLANG_SRC_FILES=$(get_clang_files_to_check --incremental)

    if [[ -z "${CLANG_SRC_FILES}" ]]; then
      echo "do_clang_format_check will NOT run due to --incremental flag and "\
"due to the absence of .h or .cc code changes in the last commit."
      return 0
    fi
  elif [[ -z "$1" ]]; then
    # TODO (yongtang): Always pass --incremental until all files have
    # been sanitized gradually. Then this --incremental could be removed.
    CLANG_SRC_FILES=$(get_clang_files_to_check --incremental)
  else
    echo "Invalid syntax for invoking do_clang_format_check"
    echo "Usage: do_clang_format_check [--incremental]"
    return 1
  fi

  CLANG_FORMAT=${CLANG_FORMAT:-clang-format-3.8}

  success=1
  for filename in $CLANG_SRC_FILES; do
    $CLANG_FORMAT --style=google $filename | diff $filename - > /dev/null
    if [ ! $? -eq 0 ]; then
      success=0
      echo File $filename is not properly formatted with "clang-format "\
"--style=google"
    fi
  done

  if [ $success == 0 ]; then
    echo Clang format check fails.
    exit 1
  fi
  echo Clang format check success.
}

do_check_load_py_test() {
  cd "$ROOT_DIR/tensorflow/tools/pip_package"
  python check_load_py_test.py
}

do_check_futures_test() {
  cd "$ROOT_DIR/tensorflow/tools/test"
  python check_futures_test.py
}

do_check_file_name_test() {
  cd "$ROOT_DIR/tensorflow/tools/test"
  python file_name_test.py
}

# Check that TARGET does not depend on DISALLOWED_DEP.
_check_no_deps() {
  TARGET="$1"
  DISALLOWED_DEP="$2"
  EXTRA_FLAG="$3"

  TMP_FILE="$(mktemp)_tmp.log"
  echo "Checking ${TARGET} does not depend on ${DISALLOWED_DEP} ..."
  bazel cquery ${EXTRA_FLAG} "somepath(${TARGET}, ${DISALLOWED_DEP})" --keep_going> "${TMP_FILE}" 2>&1
  if cat "${TMP_FILE}" | grep "Empty query results"; then
      echo "Success."
  else
      cat "${TMP_FILE}"
      echo
      echo "ERROR: Found path from ${TARGET} to disallowed dependency ${DISALLOWED_DEP}."
      echo "See above for path."
      rm "${TMP_FILE}"
      exit 1
  fi
  rm "${TMP_FILE}"
}

_do_pip_no_cuda_deps_check() {
  EXTRA_FLAG="$1"
  DISALLOWED_CUDA_DEPS=("@local_config_cuda//cuda:cudart"
        "@local_config_cuda//cuda:cublas"
        "@local_config_cuda//cuda:cuda_driver"
        "@local_config_cuda//cuda:cudnn"
        "@local_config_cuda//cuda:curand"
        "@local_config_cuda//cuda:cusolver"
        "@local_config_cuda//cuda:cusparse"
        "@local_config_tensorrt//:tensorrt")
  for cuda_dep in "${DISALLOWED_CUDA_DEPS[@]}"
  do
   _check_no_deps "//tensorflow/tools/pip_package:build_pip_package" "${cuda_dep}" "${EXTRA_FLAG}"
   RESULT=$?

   if [[ ${RESULT} != "0" ]]; then
    exit 1
   fi
  done
}

do_pip_no_cuda_deps_check_ubuntu() {
  _do_pip_no_cuda_deps_check "--define using_cuda=true --define using_cuda_nvcc=true"
}

do_pip_no_cuda_deps_check_windows() {
  _do_pip_no_cuda_deps_check "--define using_cuda=true --define using_cuda_nvcc=true --define framework_shared_object=false"
}

do_configure_test() {
  for WITH_CUDA in 1 0
  do
    export TF_NEED_CUDA=${WITH_CUDA}
    export CUDNN_INSTALL_PATH="/usr/local/cudnn"
    export PYTHON_BIN_PATH=$(which python)
    yes "" | ./configure

    RESULT=$?
    if [[ ${RESULT} != "0" ]]; then
     exit 1
    fi
  done
}

# Supply all sanity step commands and descriptions
SANITY_STEPS=("do_configure_test" "do_pylint" "do_check_futures_test" "do_buildifier" "do_bazel_nobuild" "do_bazel_deps_query" "do_pip_package_licenses_check" "do_lib_package_licenses_check" "do_java_package_licenses_check" "do_pip_smoke_test" "do_check_load_py_test" "do_code_link_check" "do_check_file_name_test" "do_pip_no_cuda_deps_check_ubuntu" "do_pip_no_cuda_deps_check_windows")
SANITY_STEPS_DESC=("Run ./configure" "Python 3 pylint" "Check that python files have certain __future__ imports" "buildifier check" "bazel nobuild" "bazel query" "pip: license check for external dependencies" "C library: license check for external dependencies" "Java Native Library: license check for external dependencies" "Pip Smoke Test: Checking py_test dependencies exist in pip package" "Check load py_test: Check that BUILD files with py_test target properly load py_test" "Code Link Check: Check there are no broken links" "Check file names for cases" "Check Ubuntu gpu pip package does not depend on cuda shared libraries" "Check Windows gpu pip package does not depend on cuda shared libraries")

INCREMENTAL_FLAG=""
DEFAULT_BAZEL_CONFIGS=""

# Parse command-line arguments
BAZEL_FLAGS=${DEFAULT_BAZEL_CONFIGS}
for arg in "$@"; do
  if [[ "${arg}" == "--pep8" ]]; then
    # Only run pep8 test if "--pep8" option supplied
    SANITY_STEPS=("do_pep8")
    SANITY_STEPS_DESC=("pep8 test")
  elif [[ "${arg}" == "--incremental" ]]; then
    INCREMENTAL_FLAG="--incremental"
  else
    BAZEL_FLAGS="${BAZEL_FLAGS} ${arg}"
  fi
done

FAIL_COUNTER=0
PASS_COUNTER=0
STEP_EXIT_CODES=()

# Execute all the sanity build steps
COUNTER=0
while [[ ${COUNTER} -lt "${#SANITY_STEPS[@]}" ]]; do
  INDEX=COUNTER
  ((INDEX++))

  echo ""
  echo "=== Sanity check step ${INDEX} of ${#SANITY_STEPS[@]}: "\
"${SANITY_STEPS[COUNTER]} (${SANITY_STEPS_DESC[COUNTER]}) ==="
  echo ""

  # subshell: don't leak variables or changes of working directory
  (
  ${SANITY_STEPS[COUNTER]} ${INCREMENTAL_FLAG}
  )
  RESULT=$?

  if [[ ${RESULT} != "0" ]]; then
    ((FAIL_COUNTER++))
  else
    ((PASS_COUNTER++))
  fi

  STEP_EXIT_CODES+=(${RESULT})

  echo ""
  ((COUNTER++))
done

# Print summary of build results
COUNTER=0
echo "==== Summary of sanity check results ===="
while [[ ${COUNTER} -lt "${#SANITY_STEPS[@]}" ]]; do
  INDEX=COUNTER
  ((INDEX++))

  echo "${INDEX}. ${SANITY_STEPS[COUNTER]}: ${SANITY_STEPS_DESC[COUNTER]}"
  if [[ ${STEP_EXIT_CODES[COUNTER]} == "0" ]]; then
    printf "  ${COLOR_GREEN}PASS${COLOR_NC}\n"
  else
    printf "  ${COLOR_RED}FAIL${COLOR_NC}\n"
  fi

  ((COUNTER++))
done

echo
echo "${FAIL_COUNTER} failed; ${PASS_COUNTER} passed."

echo
if [[ ${FAIL_COUNTER} == "0" ]]; then
  printf "Sanity checks ${COLOR_GREEN}PASSED${COLOR_NC}\n"
else
  printf "Sanity checks ${COLOR_RED}FAILED${COLOR_NC}\n"
  exit 1
fi

#!/usr/bin/env bash
# Copyright 2016 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
#
# Usage: ci_sanity.sh [--pep8] [--incremental] [bazel flags]
#
# Options:
#           run sanity checks: python 2&3 pylint checks and bazel nobuild
#  --pep8   run pep8 test only
#  --incremental  Performs checks incrementally, by using the files changed in
#                 the latest commit

# Current script directory
SCRIPT_DIR=$( cd ${0%/*} && pwd -P )
source "${SCRIPT_DIR}/builds/builds_common.sh"

ROOT_DIR=$( cd "$SCRIPT_DIR/../../.." && pwd -P )

# Helper functions
die() {
  echo $@
  exit 1
}

num_cpus() {
  # Get the number of CPUs
  if [[ -f /proc/cpuinfo ]]; then
    N_CPUS=$(grep -c ^processor /proc/cpuinfo)
  else
    # Fallback method
    N_CPUS=`getconf _NPROCESSORS_ONLN`
  fi
  if [[ -z ${N_CPUS} ]]; then
    die "ERROR: Unable to determine the number of CPUs"
  fi

  echo ${N_CPUS}
}

# Helper functions for examining changed files in the last non-merge git
# commit.

# Get the hash of the last non-merge git commit on the current branch.
# Usage: get_last_non_merge_git_commit
get_last_non_merge_git_commit() {
  git rev-list --no-merges -n 1 HEAD
}

# List files changed (i.e., added, removed or revised) in the last non-merge
# git commit.
# Usage: get_changed_files_in_last_non_merge_git_commit
get_changed_files_in_last_non_merge_git_commit() {
  git diff-tree --no-commit-id --name-only -r $(get_last_non_merge_git_commit)
}

# List Python files changed in the last non-merge git commit that still exist,
# i.e., not removed.
# Usage: get_py_files_to_check [--incremental]
get_py_files_to_check() {
  if [[ "$1" == "--incremental" ]]; then
    CHANGED_PY_FILES=$(get_changed_files_in_last_non_merge_git_commit | \
                       grep '.*\.py$')

    # Do not include files removed in the last non-merge commit.
    PY_FILES=""
    for PY_FILE in ${CHANGED_PY_FILES}; do
      if [[ -f "${PY_FILE}" ]]; then
        PY_FILES="${PY_FILES} ${PY_FILE}"
      fi
    done

    echo "${PY_FILES}"
  else
    find tensorflow -name '*.py'
  fi
}

# Subfunctions for substeps
# Run pylint
do_pylint() {
  # Usage: do_pylint [--incremental]
  #
  # Options:
  #   --incremental  Performs check on only the python files changed in the
  #                  last non-merge git commit.

  # Use this list to whitelist pylint errors
  ERROR_WHITELIST="^tensorflow/python/framework/function_test\.py.*\[E1123.*noinline "\
"^tensorflow/python/platform/default/_gfile\.py.*\[E0301.*non-iterator "\
"^tensorflow/python/platform/default/_googletest\.py.*\[E0102.*function\salready\sdefined "\
"^tensorflow/python/feature_column/feature_column_test\.py.*\[E0110.*abstract-class-instantiated "\
"^tensorflow/contrib/layers/python/layers/feature_column\.py.*\[E0110.*abstract-class-instantiated "\
"^tensorflow/contrib/eager/python/evaluator\.py.*\[E0202.*method-hidden "\
"^tensorflow/contrib/eager/python/metrics_impl\.py.*\[E0202.*method-hidden "\
"^tensorflow/contrib/rate/rate\.py.*\[E0202.*method-hidden "\
"^tensorflow/python/training/tracking/tracking\.py.*\[E0202.*method-hidden "\
"^tensorflow/python/platform/gfile\.py.*\[E0301.*non-iterator "\
"^tensorflow/python/keras/callbacks\.py.*\[E1133.*not-an-iterable "\
"^tensorflow/python/keras/engine/base_layer.py.*\[E0203.*access-member-before-definition "\
"^tensorflow/python/keras/layers/recurrent\.py.*\[E0203.*access-member-before-definition "\
"^tensorflow/python/kernel_tests/constant_op_eager_test.py.*\[E0303.*invalid-length-returned "\
"^tensorflow/python/keras/utils/data_utils.py.*\[E1102.*not-callable "\
"^tensorflow/python/autograph/.*_py3_test\.py.*\[E0001.*syntax-error "\
"^tensorflow/python/keras/preprocessing/image\.py.*\[E0240.*Inconsistent method resolution "

  echo "ERROR_WHITELIST=\"${ERROR_WHITELIST}\""

  if [[ $# != "0" ]]  && [[ $# != "1" ]]; then
    echo "Invalid syntax when invoking do_pylint"
    echo "Usage: do_pylint [--incremental]"
    return 1
  fi

  PYLINT_BIN="python3 -m pylint"

  if [[ "$1" == "--incremental" ]]; then
    PYTHON_SRC_FILES=$(get_py_files_to_check --incremental)

    if [[ -z "${PYTHON_SRC_FILES}" ]]; then
      echo "do_pylint will NOT run due to --incremental flag and due to the "\
"absence of Python code changes in the last commit."
      return 0
    else
      # For incremental builds, we still check all Python files in cases there
      # are function signature changes that affect unchanged Python files.
      PYTHON_SRC_FILES=$(get_py_files_to_check)
    fi
  elif [[ -z "$1" ]]; then
    PYTHON_SRC_FILES=$(get_py_files_to_check)
  else
    echo "Invalid syntax for invoking do_pylint"
    echo "Usage: do_pylint [--incremental]"
    return 1
  fi

  if [[ -z ${PYTHON_SRC_FILES} ]]; then
    echo "do_pylint found no Python files to check. Returning."
    return 0
  fi

  PYLINTRC_FILE="${SCRIPT_DIR}/pylintrc"

  if [[ ! -f "${PYLINTRC_FILE}" ]]; then
    die "ERROR: Cannot find pylint rc file at ${PYLINTRC_FILE}"
  fi

  NUM_SRC_FILES=$(echo ${PYTHON_SRC_FILES} | wc -w)
  NUM_CPUS=$(num_cpus)

  echo "Running pylint on ${NUM_SRC_FILES} files with ${NUM_CPUS} "\
"parallel jobs..."
  echo ""

  PYLINT_START_TIME=$(date +'%s')
  OUTPUT_FILE="$(mktemp)_pylint_output.log"
  ERRORS_FILE="$(mktemp)_pylint_errors.log"
  NONWL_ERRORS_FILE="$(mktemp)_pylint_nonwl_errors.log"

  rm -rf ${OUTPUT_FILE}
  rm -rf ${ERRORS_FILE}
  rm -rf ${NONWL_ERRORS_FILE}
  touch ${NONWL_ERRORS_FILE}

  ${PYLINT_BIN} --rcfile="${PYLINTRC_FILE}" --output-format=parseable \
      --jobs=${NUM_CPUS} ${PYTHON_SRC_FILES} > ${OUTPUT_FILE} 2>&1
  PYLINT_END_TIME=$(date +'%s')

  echo ""
  echo "pylint took $((PYLINT_END_TIME - PYLINT_START_TIME)) s"
  echo ""

  # Report only what we care about
  # Ref https://pylint.readthedocs.io/en/latest/technical_reference/features.html
  # E: all errors
  # W0311 bad-indentation
  # W0312 mixed-indentation
  # C0330 bad-continuation
  # C0301 line-too-long
  # C0326 bad-whitespace
  # W0611 unused-import
  # W0622 redefined-builtin
  grep -E '(\[E|\[W0311|\[W0312|\[C0330|\[C0301|\[C0326|\[W0611|\[W0622)' ${OUTPUT_FILE} > ${ERRORS_FILE}

  N_ERRORS=0
  while read -r LINE; do
    IS_WHITELISTED=0
    for WL_REGEX in ${ERROR_WHITELIST}; do
      if echo ${LINE} | grep -q "${WL_REGEX}"; then
        echo "Found a whitelisted error:"
        echo "  ${LINE}"
        IS_WHITELISTED=1
      fi
    done

    if [[ ${IS_WHITELISTED} == "0" ]]; then
      echo "${LINE}" >> ${NONWL_ERRORS_FILE}
      echo "" >> ${NONWL_ERRORS_FILE}
      ((N_ERRORS++))
    fi
  done <${ERRORS_FILE}

  echo ""
  if [[ ${N_ERRORS} != 0 ]]; then
    echo "FAIL: Found ${N_ERRORS} non-whitelisted pylint errors:"
    cat "${NONWL_ERRORS_FILE}"
    return 1
  else
    echo "PASS: No non-whitelisted pylint errors were found."
    return 0
  fi
}

# Run pep8 check
do_pep8() {
  # Usage: do_pep8 [--incremental]
  # Options:
  #   --incremental  Performs check on only the python files changed in the
  #                  last non-merge git commit.

  PEP8_BIN="/usr/local/bin/pep8"
  PEP8_CONFIG_FILE="${SCRIPT_DIR}/pep8"

  if [[ "$1" == "--incremental" ]]; then
    PYTHON_SRC_FILES=$(get_py_files_to_check --incremental)
    NUM_PYTHON_SRC_FILES=$(echo ${PYTHON_SRC_FILES} | wc -w)

    echo "do_pep8 will perform checks on only the ${NUM_PYTHON_SRC_FILES} "\
"Python file(s) changed in the last non-merge git commit due to the "\
"--incremental flag:"
    echo "${PYTHON_SRC_FILES}"
    echo ""
  else
    PYTHON_SRC_FILES=$(get_py_files_to_check)
  fi

  if [[ -z ${PYTHON_SRC_FILES} ]]; then
    echo "do_pep8 found no Python files to check. Returning."
    return 0
  fi

  if [[ ! -f "${PEP8_CONFIG_FILE}" ]]; then
    die "ERROR: Cannot find pep8 config file at ${PEP8_CONFIG_FILE}"
  fi
  echo "See \"${PEP8_CONFIG_FILE}\" for pep8 config( e.g., ignored errors)"

  NUM_SRC_FILES=$(echo ${PYTHON_SRC_FILES} | wc -w)

  echo "Running pep8 on ${NUM_SRC_FILES} files"
  echo ""

  PEP8_START_TIME=$(date +'%s')
  PEP8_OUTPUT_FILE="$(mktemp)_pep8_output.log"

  rm -rf ${PEP8_OUTPUT_FILE}

  ${PEP8_BIN} --config="${PEP8_CONFIG_FILE}" --statistics \
      ${PYTHON_SRC_FILES} 2>&1 | tee ${PEP8_OUTPUT_FILE}
  PEP8_END_TIME=$(date +'%s')

  echo ""
  echo "pep8 took $((PEP8_END_TIME - PEP8_START_TIME)) s"
  echo ""

  if [[ -s ${PEP8_OUTPUT_FILE} ]]; then
    echo "FAIL: pep8 found above errors and/or warnings."
    return 1
  else
    echo "PASS: No pep8 errors or warnings were found"
    return 0
  fi
}

do_buildifier(){
  BUILD_FILES=$(find tensorflow -name 'BUILD*')
  NUM_BUILD_FILES=$(echo ${BUILD_FILES} | wc -w)

  echo "Running do_buildifier on ${NUM_BUILD_FILES} files"
  echo ""

  BUILDIFIER_START_TIME=$(date +'%s')
  BUILDIFIER_OUTPUT_FILE="$(mktemp)_buildifier_output.log"

  rm -rf ${BUILDIFIER_OUTPUT_FILE}

  buildifier -v -mode=check \
    ${BUILD_FILES} 2>&1 | tee ${BUILDIFIER_OUTPUT_FILE}
  BUILDIFIER_END_TIME=$(date +'%s')

  echo ""
  echo "buildifier took $((BUILDIFIER_END_TIME - BUILDIFIER_START_TIME)) s"
  echo ""

  if [[ -s ${BUILDIFIER_OUTPUT_FILE} ]]; then
    echo "FAIL: buildifier found errors and/or warnings in above BUILD files."
    echo "buildifier suggested the following changes:"
    buildifier -v -mode=diff -diff_command=diff ${BUILD_FILES}
    echo "Please fix manually or run buildifier <file> to auto-fix."
    return 1
  else
    echo "PASS: No buildifier errors or warnings were found"
    return 0
  fi
}

do_external_licenses_check(){
  BUILD_TARGET="$1"
  LICENSES_TARGET="$2"

  EXTERNAL_LICENSES_CHECK_START_TIME=$(date +'%s')

  EXTERNAL_DEPENDENCIES_FILE="$(mktemp)_external_dependencies.log"
  LICENSES_FILE="$(mktemp)_licenses.log"
  MISSING_LICENSES_FILE="$(mktemp)_missing_licenses.log"
  EXTRA_LICENSES_FILE="$(mktemp)_extra_licenses.log"
  TMP_FILE="$(mktemp)_tmp.log"

  echo "Getting external dependencies for ${BUILD_TARGET}"
 bazel cquery "attr('licenses', 'notice', deps(${BUILD_TARGET}))" --keep_going > "${TMP_FILE}" 2>&1
 cat "${TMP_FILE}" \
  | grep -e "^\/\/" -e "^@" \
  | grep -E -v "^//tensorflow" \
  | sed -e 's|:.*||' \
  | sort \
  | uniq 2>&1 \
  | tee ${EXTERNAL_DEPENDENCIES_FILE}

  echo
  echo "Getting list of external licenses mentioned in ${LICENSES_TARGET}."
  bazel cquery "deps(${LICENSES_TARGET})" --keep_going > "${TMP_FILE}" 2>&1
 cat "${TMP_FILE}" \
  | grep -e "^\/\/" -e "^@" \
  | grep -E -v "^//tensorflow" \
  | sed -e 's|:.*||' \
  | sort \
  | uniq 2>&1 \
  | tee ${LICENSES_FILE}

  echo
  comm -1 -3 ${EXTERNAL_DEPENDENCIES_FILE}  ${LICENSES_FILE} 2>&1 | tee ${EXTRA_LICENSES_FILE}
  echo
  comm -2 -3 ${EXTERNAL_DEPENDENCIES_FILE}  ${LICENSES_FILE} 2>&1 | tee ${MISSING_LICENSES_FILE}

  EXTERNAL_LICENSES_CHECK_END_TIME=$(date +'%s')

  # Blacklist
  echo ${MISSING_LICENSES_FILE}
  grep \
    -e "@bazel_tools//third_party/" \
    -e "@bazel_tools//tools" \
    -e "@local" \
    -e "@com_google_absl//absl" \
    -e "@org_tensorflow//" \
    -e "@com_github_googlecloudplatform_google_cloud_cpp//google" \
    -e "@com_github_grpc_grpc//src/compiler" \
    -e "@platforms//os" \
    -v ${MISSING_LICENSES_FILE} > temp.txt
  mv temp.txt ${MISSING_LICENSES_FILE}

  # Whitelist
  echo ${EXTRA_LICENSE_FILE}
  grep \
    -e "//third_party/mkl" \
    -e "//third_party/mkl_dnn" \
    -e "@bazel_tools//src" \
    -e "@bazel_tools//tools/" \
    -e "@org_tensorflow//tensorflow" \
    -e "@com_google_absl//" \
    -e "//external" \
    -e "@local" \
    -e "@com_github_googlecloudplatform_google_cloud_cpp//" \
    -e "@embedded_jdk//" \
    -e "^//$" \
    -v ${EXTRA_LICENSES_FILE} > temp.txt
  mv temp.txt ${EXTRA_LICENSES_FILE}

  echo
  echo "do_external_licenses_check took $((EXTERNAL_LICENSES_CHECK_END_TIME - EXTERNAL_LICENSES_CHECK_START_TIME)) s"
  echo

  if [[ -s ${MISSING_LICENSES_FILE} ]] || [[ -s ${EXTRA_LICENSES_FILE} ]] ; then
    echo "FAIL: mismatch in packaged licenses and external dependencies"
    if [[ -s ${MISSING_LICENSES_FILE} ]] ; then
      echo "Missing the licenses for the following external dependencies:"
      cat ${MISSING_LICENSES_FILE}
    fi
    if [[ -s ${EXTRA_LICENSES_FILE} ]] ; then
      echo "Please remove the licenses for the following external dependencies:"
      cat ${EXTRA_LICENSES_FILE}
    fi
    rm -rf ${EXTERNAL_DEPENDENCIES_FILE}
    rm -rf ${LICENSES_FILE}
    rm -rf ${MISSING_LICENSES_FILE}
    rm -rf ${EXTRA_LICENSES_FILE}
    return 1
  else
    echo "PASS: all external licenses included."
    rm -rf ${EXTERNAL_DEPENDENCIES_FILE}
    rm -rf ${LICENSES_FILE}
    rm -rf ${MISSING_LICENSES_FILE}
    rm -rf ${EXTRA_LICENSES_FILE}
    return 0
  fi
}

do_pip_package_licenses_check() {
  echo "Running do_pip_package_licenses_check"
  echo ""
  do_external_licenses_check \
    "//tensorflow/tools/pip_package:build_pip_package" \
    "//tensorflow/tools/pip_package:licenses"
}

do_lib_package_licenses_check() {
  echo "Running do_lib_package_licenses_check"
  echo ""
  do_external_licenses_check \
    "//tensorflow:libtensorflow.so" \
    "//tensorflow/tools/lib_package:clicenses_generate"
}

do_java_package_licenses_check() {
  echo "Running do_java_package_licenses_check"
  echo ""
  do_external_licenses_check \
    "//tensorflow/java:libtensorflow_jni.so" \
    "//tensorflow/tools/lib_package:jnilicenses_generate"
}

#Check for the bazel cmd status (First arg is error message)
cmd_status(){
  if [[ $? != 0 ]]; then
    echo ""
    echo "FAIL: ${BUILD_CMD}"
    echo "  $1 See lines above for details."
    return 1
  else
    echo ""
    echo "PASS: ${BUILD_CMD}"
    return 0
  fi
}

# Run bazel build --nobuild to test the validity of the BUILD files
# TODO(mikecase): Remove TF Lite exclusion from this list. Exclusion is
# necessary since the @androidsdk WORKSPACE dependency is commented
# out by default in TF WORKSPACE file.
do_bazel_nobuild() {
  BUILD_TARGET="//tensorflow/..."
  BUILD_TARGET="${BUILD_TARGET} -//tensorflow/lite/delegates/gpu/..."
  BUILD_TARGET="${BUILD_TARGET} -//tensorflow/lite/java/demo/app/..."
  BUILD_TARGET="${BUILD_TARGET} -//tensorflow/lite/schema/..."
  BUILD_CMD="bazel build --nobuild ${BAZEL_FLAGS} -- ${BUILD_TARGET}"

  ${BUILD_CMD}

  cmd_status \
    "This is due to invalid BUILD files."
}

do_bazel_deps_query() {
  local BUILD_TARGET='//tensorflow/...'
  # Android targets tend to depend on an Android runtime being available.
  # Exclude until the sanity test has such a runtime available.
  #
  # TODO(mikecase): Remove TF Lite exclusion from this list. Exclusion is
  # necessary since the @androidsdk WORKSPACE dependency is commented out by
  # default in TF WORKSPACE file.
  local BUILD_TARGET="${BUILD_TARGET}"' - kind("android_*", //tensorflow/...)'

  # We've set the flag noimplicit_deps as a workaround for
  # https://github.com/bazelbuild/bazel/issues/10544
  bazel query ${BAZEL_FLAGS} --noimplicit_deps -- "deps($BUILD_TARGET)" > /dev/null

  cmd_status \
    "This is due to invalid BUILD files."
}

do_pip_smoke_test() {
  cd "$ROOT_DIR/tensorflow/tools/pip_package"
  python pip_smoke_test.py
}

do_code_link_check() {
  tensorflow/tools/ci_build/code_link_check.sh
}

# List .h|.cc files changed in the last non-merge git commit that still exist,
# i.e., not removed.
# Usage: get_clang_files_to_check [--incremental]
get_clang_files_to_check() {
  if [[ "$1" == "--incremental" ]]; then
    CHANGED_CLANG_FILES=$(get_changed_files_in_last_non_merge_git_commit | \
                       grep '.*\.h$\|.*\.cc$')

    # Do not include files removed in the last non-merge commit.
    CLANG_FILES=""
    for CLANG_FILE in ${CHANGED_CLANG_FILES}; do
      if [[ -f "${CLANG_FILE}" ]]; then
        CLANG_FILES="${CLANG_FILES} ${CLANG_FILE}"
      fi
    done

    echo "${CLANG_FILES}"
  else
    find tensorflow -name '*.h' -o -name '*.cc'
  fi
}

do_clang_format_check() {
  if [[ $# != "0" ]] && [[ $# != "1" ]]; then
    echo "Invalid syntax when invoking do_clang_format_check"
    echo "Usage: do_clang_format_check [--incremental]"
    return 1
  fi

  if [[ "$1" == "--incremental" ]]; then
    CLANG_SRC_FILES=$(get_clang_files_to_check --incremental)

    if [[ -z "${CLANG_SRC_FILES}" ]]; then
      echo "do_clang_format_check will NOT run due to --incremental flag and "\
"due to the absence of .h or .cc code changes in the last commit."
      return 0
    fi
  elif [[ -z "$1" ]]; then
    # TODO (yongtang): Always pass --incremental until all files have
    # been sanitized gradually. Then this --incremental could be removed.
    CLANG_SRC_FILES=$(get_clang_files_to_check --incremental)
  else
    echo "Invalid syntax for invoking do_clang_format_check"
    echo "Usage: do_clang_format_check [--incremental]"
    return 1
  fi

  CLANG_FORMAT=${CLANG_FORMAT:-clang-format-3.8}

  success=1
  for filename in $CLANG_SRC_FILES; do
    $CLANG_FORMAT --style=google $filename | diff $filename - > /dev/null
    if [ ! $? -eq 0 ]; then
      success=0
      echo File $filename is not properly formatted with "clang-format "\
"--style=google"
    fi
  done

  if [ $success == 0 ]; then
    echo Clang format check fails.
    exit 1
  fi
  echo Clang format check success.
}

do_check_load_py_test() {
  cd "$ROOT_DIR/tensorflow/tools/pip_package"
  python check_load_py_test.py
}

do_check_futures_test() {
  cd "$ROOT_DIR/tensorflow/tools/test"
  python check_futures_test.py
}

do_check_file_name_test() {
  cd "$ROOT_DIR/tensorflow/tools/test"
  python file_name_test.py
}

# Check that TARGET does not depend on DISALLOWED_DEP.
_check_no_deps() {
  TARGET="$1"
  DISALLOWED_DEP="$2"
  EXTRA_FLAG="$3"

  TMP_FILE="$(mktemp)_tmp.log"
  echo "Checking ${TARGET} does not depend on ${DISALLOWED_DEP} ..."
  bazel cquery ${EXTRA_FLAG} "somepath(${TARGET}, ${DISALLOWED_DEP})" --keep_going> "${TMP_FILE}" 2>&1
  if cat "${TMP_FILE}" | grep "Empty query results"; then
      echo "Success."
  else
      cat "${TMP_FILE}"
      echo
      echo "ERROR: Found path from ${TARGET} to disallowed dependency ${DISALLOWED_DEP}."
      echo "See above for path."
      rm "${TMP_FILE}"
      exit 1
  fi
  rm "${TMP_FILE}"
}

_do_pip_no_cuda_deps_check() {
  EXTRA_FLAG="$1"
  DISALLOWED_CUDA_DEPS=("@local_config_cuda//cuda:cudart"
        "@local_config_cuda//cuda:cublas"
        "@local_config_cuda//cuda:cuda_driver"
        "@local_config_cuda//cuda:cudnn"
        "@local_config_cuda//cuda:curand"
        "@local_config_cuda//cuda:cusolver"
        "@local_config_cuda//cuda:cusparse"
        "@local_config_tensorrt//:tensorrt")
  for cuda_dep in "${DISALLOWED_CUDA_DEPS[@]}"
  do
   _check_no_deps "//tensorflow/tools/pip_package:build_pip_package" "${cuda_dep}" "${EXTRA_FLAG}"
   RESULT=$?

   if [[ ${RESULT} != "0" ]]; then
    exit 1
   fi
  done
}

do_pip_no_cuda_deps_check_ubuntu() {
  _do_pip_no_cuda_deps_check "--define using_cuda=true --define using_cuda_nvcc=true"
}

do_pip_no_cuda_deps_check_windows() {
  _do_pip_no_cuda_deps_check "--define using_cuda=true --define using_cuda_nvcc=true --define framework_shared_object=false"
}

do_configure_test() {
  for WITH_CUDA in 1 0
  do
    export TF_NEED_CUDA=${WITH_CUDA}
    export CUDNN_INSTALL_PATH="/usr/local/cudnn"
    export PYTHON_BIN_PATH=$(which python)
    yes "" | ./configure

    RESULT=$?
    if [[ ${RESULT} != "0" ]]; then
     exit 1
    fi
  done
}

# Supply all sanity step commands and descriptions
SANITY_STEPS=("do_configure_test" "do_pylint" "do_check_futures_test" "do_buildifier" "do_bazel_nobuild" "do_bazel_deps_query" "do_pip_package_licenses_check" "do_lib_package_licenses_check" "do_java_package_licenses_check" "do_pip_smoke_test" "do_check_load_py_test" "do_code_link_check" "do_check_file_name_test" "do_pip_no_cuda_deps_check_ubuntu" "do_pip_no_cuda_deps_check_windows")
SANITY_STEPS_DESC=("Run ./configure" "Python 3 pylint" "Check that python files have certain __future__ imports" "buildifier check" "bazel nobuild" "bazel query" "pip: license check for external dependencies" "C library: license check for external dependencies" "Java Native Library: license check for external dependencies" "Pip Smoke Test: Checking py_test dependencies exist in pip package" "Check load py_test: Check that BUILD files with py_test target properly load py_test" "Code Link Check: Check there are no broken links" "Check file names for cases" "Check Ubuntu gpu pip package does not depend on cuda shared libraries" "Check Windows gpu pip package does not depend on cuda shared libraries")

INCREMENTAL_FLAG=""
DEFAULT_BAZEL_CONFIGS=""

# Parse command-line arguments
BAZEL_FLAGS=${DEFAULT_BAZEL_CONFIGS}
for arg in "$@"; do
  if [[ "${arg}" == "--pep8" ]]; then
    # Only run pep8 test if "--pep8" option supplied
    SANITY_STEPS=("do_pep8")
    SANITY_STEPS_DESC=("pep8 test")
  elif [[ "${arg}" == "--incremental" ]]; then
    INCREMENTAL_FLAG="--incremental"
  else
    BAZEL_FLAGS="${BAZEL_FLAGS} ${arg}"
  fi
done

FAIL_COUNTER=0
PASS_COUNTER=0
STEP_EXIT_CODES=()

# Execute all the sanity build steps
COUNTER=0
while [[ ${COUNTER} -lt "${#SANITY_STEPS[@]}" ]]; do
  INDEX=COUNTER
  ((INDEX++))

  echo ""
  echo "=== Sanity check step ${INDEX} of ${#SANITY_STEPS[@]}: "\
"${SANITY_STEPS[COUNTER]} (${SANITY_STEPS_DESC[COUNTER]}) ==="
  echo ""

  # subshell: don't leak variables or changes of working directory
  (
  ${SANITY_STEPS[COUNTER]} ${INCREMENTAL_FLAG}
  )
  RESULT=$?

  if [[ ${RESULT} != "0" ]]; then
    ((FAIL_COUNTER++))
  else
    ((PASS_COUNTER++))
  fi

  STEP_EXIT_CODES+=(${RESULT})

  echo ""
  ((COUNTER++))
done

# Print summary of build results
COUNTER=0
echo "==== Summary of sanity check results ===="
while [[ ${COUNTER} -lt "${#SANITY_STEPS[@]}" ]]; do
  INDEX=COUNTER
  ((INDEX++))

  echo "${INDEX}. ${SANITY_STEPS[COUNTER]}: ${SANITY_STEPS_DESC[COUNTER]}"
  if [[ ${STEP_EXIT_CODES[COUNTER]} == "0" ]]; then
    printf "  ${COLOR_GREEN}PASS${COLOR_NC}\n"
  else
    printf "  ${COLOR_RED}FAIL${COLOR_NC}\n"
  fi

  ((COUNTER++))
done

echo
echo "${FAIL_COUNTER} failed; ${PASS_COUNTER} passed."

echo
if [[ ${FAIL_COUNTER} == "0" ]]; then
  printf "Sanity checks ${COLOR_GREEN}PASSED${COLOR_NC}\n"
else
  printf "Sanity checks ${COLOR_RED}FAILED${COLOR_NC}\n"
  exit 1
fi