Oops, need to actually export $CROWBAR_DIR

#!/bin/bash
# Copyright 2011, Dell
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#  http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# Author: VictorLowther
#

# This script expects to be able to run certian commands as root.
# Either run it as a user who can sudo to root, or give the user
# you are running it as the following sudo rights:
# crowbar-tester ALL = NOPASSWD: /bin/mount, /bin/umount, /usr/sbin/debootstrap, /bin/cp, /usr/sbin/chroot

# When running this script for the first time, it will automatically create a
# cache directory and try to populate it with all the build dependencies.
# After that, if you need to pull in new dependencies, you will need to
# call the script with the --update-cache parameter.  If you are going to
# develop on Crowbar, it is a good idea to put the build cache in its own git
# repository, and create a branching structure for the packages that mirrors
# the branching structure in the crowbar repository -- if you do that, then
# this build script can be smarter about what packages it should pull in
# whenever you invoke it to build an iso.

GEM_RE='([^0-9].*)-([0-9].*)'
[[ $DEBUG ]] && {
    set -x
    export PS4='${BASH_SOURCE}@${LINENO}(${FUNCNAME[0]}): '
}

readonly currdir="$PWD"
export PATH="$PATH:/sbin:/usr/sbin:/usr/local/sbin"

# Our general cleanup function.  It is called as a trap whenever the
# build script exits, and it's job is to make sure we leave the local
# system in the same state we cound it, modulo a few calories of wasted heat
# and a shiny new .iso.
cleanup() {
    # Clean up any stray mounts we may have left behind.
    # The paranoia with the grepping is to ensure that we do not
    # inadvertently umount everything.
    GREPOPTS=()
    [[ $CACHE_DIR ]] && GREPOPTS=(-e "$CACHE_DIR")
    [[ $IMAGE_DIR && $CACHE_DIR =~ $IMAGE_DIR ]] && GREPOPTS+=(-e "$IMAGE_DIR")
    [[ $BUILD_DIR && $CACHE_DIR =~ $BUILD_DIR ]] && GREPOPTS+=(-e "$BUILD_DIR")
    [[ $CHROOT && $CACHE_DIR =~ $CHROOT ]] && GREPOPTS+=(-e "$CHROOT")
    if [[ $GREPOPTS ]]; then
	while read dev fs type opts rest; do
	    sudo umount -d -l "$fs"
	done < <(tac /proc/self/mounts |grep "${GREPOPTS[@]}")
    fi
    # If the build process spawned a copy of webrick, make sure it is dead.
    [[ $webrick_pid && -d /proc/$webrick_pid ]] && kill -9 $webrick_pid
    # clean up after outselves from merging branches, if needed.
    cd "$CROWBAR_DIR"
    if [[ $THROWAWAY_BRANCH ]]; then
	# Check out the branch we started the build process, and then
	# nuke whatever throwaway branch we may have created.
	git checkout -f "${CURRENT_BRANCH##*/}" &>/dev/null
	git branch -D "$THROWAWAY_BRANCH" &>/dev/null
    fi
    # If we saved unadded changes, resurrect them.
    [[ $THROWAWAY_STASH ]] && git stash apply "$THROWAWAY_STASH" &>/dev/null
    # Do the same thing as above, but for the build cache instead.
    cd "$CACHE_DIR"
    if [[ $CACHE_THROWAWAY_BRANCH ]]; then
	git checkout -f "$CURRENT_CACHE_BRANCH" &>/dev/null
	git branch -D "$CACHE_THROWAWAY_BRANCH" &>/dev/null
    fi
    [[ $CACHE_THROWAWAY_STASH ]] && git stash apply "$CACHE_THROWAWAY_STASH"
    for d in "$IMAGE_DIR" "$BUILD_DIR" "$CHROOT"; do
	[[ -d $d ]] && sudo rm -rf -- "$d"
    done
}

# Test to see if $1 is in the rest of the args.
is_in() {
    local t="(^| )$1( |\$)"
    shift
    [[ $* =~ $t ]]
}

# Run a command in our chroot environment.
in_chroot() { sudo -H /usr/sbin/chroot "$CHROOT" "$@"; }

# A little helper function for doing bind mounts.
bind_mount() {
    [[ -d $2 ]] || mkdir -p "$2"
    grep -q "$2" /proc/self/mounts || sudo mount --bind "$1" "$2"
}

# Write a string with \n translated to a real line break.
write_lines() { [[ $1 ]] && printf "%b" "$1"; }

# Read base repository information from the *.list files.
read_base_repos() {
    for pkgfile in "$BUILD_DIR/extra/packages/"*.list; do
	[[ -f $pkgfile ]] || continue
	while read line; do
	    line=${line%%#*}
	    [[ $line ]] || continue
	    if [[ $line = repository* ]]; then
		REPOS+=("${line#* }")
	    else
		echo "$line in $pkgfile has been superceded by crowbar.yml metadata." >&2
		die "Please migrate $line into the appropriate barclamp config file."
	    fi
	done <"$pkgfile"
    done
}

# Worker function for all_deps
__all_deps() {
    local dep
    if [[ ${BC_DEPS["$1"]} ]]; then
	for dep in ${BC_DEPS["$1"]}; do
	    is_in "$dep" "${deps[@]}" && continue
	    __all_deps "$dep"
	done
    fi
    is_in "$1" "${deps[@]}" || deps+=("$1")
    return 0
}

# Given a barclamp, echo all of the dependencies of that barclamp
all_deps() {
    local deps=() dep
    __all_deps "$1"
    echo "${deps[*]}"
}

# A couple of utility functions for comparing version numbers.
num_re='^[0-9]+$'
__cmp() {
    [[ $1 || $2 ]] || return 255 # neither 1 nor 2 or set, we are done
    [[ $1 && ! $2 ]] && return 2 # 1 is set and 2 is not, 1 > 2
    [[ ! $1 && $2 ]] && return 0 # 2 is set and 1 is not, 1 < 2
    local a="$1" b="$2"
    if [[ $a =~ $num_re && $b =~ $num_re ]]; then #both numbers, numeric cmp.
	# make sure leading zeros do not confuse us
	a=${a##0} b=${b##0}
	((${a:=0} > ${b:=0})) && return 2
	(($a < $b)) && return 0
	return 1
    else # string compare
	[[ $a > $b ]] && return 2
	[[ $a < $b ]] && return 0
	return 1
    fi
}

# Compare a version string, handling mixed numeric and alpha sequences
# as appropriate.
vercmp(){
    # $1 = version string of first package
    # $2 = version string of second package
    # Returns 0 if $1 > $2, 1 otherwise.
    # urldecode the version strings.
    local a="$(printf "%b" "${1//%/\\x}")"
    local b="$(printf "%b" "${2//%/\\x}")"
    # prepend a version epoch if one is not already there.
    [[ $a =~ ^[0-9]+: ]] || a="0:$a"
    [[ $b =~ ^[0-9]+: ]] || b="0:$b"
    local ver1=()
    local ver2=()
    local i=0
    # split the version strings into arrays using the usual field splitters
    IFS=':.-_ +' read -rs -a ver1 <<< "$a"
    IFS=':.-_ +' read -rs -a ver2 <<< "$b"
    for ((i=0;;i++)); do
	__cmp "${ver1[$i]}" "${ver2[$i]}"
	case $? in
	    2) return 0;;
	    0) return 1;;
	    255) return 1;;
	esac
    done
}

# Index the pool of packages in the CD.
index_cd_pool() {
    # Scan through our pool to find pkgs we can easily omit.

    local pkgname='' pkg=''
    while read pkg; do
	[[ -f $pkg ]] && is_pkg "$pkg" || continue
	pkgname="$(pkg_name "$pkg")"
	CD_POOL["$pkgname"]="${pkg}"
    done < <(find "$(find_cd_pool)" -type f)
}

# Make a chroot environment for package-fetching purposes.
make_chroot() {
    [[ -f $CHROOT/etc/resolv.conf ]] && return 0
    local bc repo
    debug "Making package-fetching chroot"
    mkdir -p "$CHROOT/$CHROOT_PKGDIR"
    mkdir -p "$CHROOT/$CHROOT_GEMDIR"
    __make_chroot

    read_base_repos
    # Add our basic repositories
    add_repos "${REPOS[@]}" || \
	die "Could not add base repositories for $OS_TOKEN chroot!"

    # Add the repos from the barclamps.
    # We do it here because importing the metadata takes forever
    # if we refresh on every barclamp.
    for bc in "${BARCLAMPS[@]}"; do
	while read repo; do
	    add_repos "$repo"
	done < <(write_lines "${BC_REPOS[$bc]}")
    done
    chroot_update
}

stage_pkgs() {
    # $1 = cache to copy from.
    # $2 = location to copy to
    local pkg pkgname pkg_t
    while read pkg; do
	# If it is not a package, skip it.
	is_pkg "$pkg" || continue
	pkgname="$(pkg_name "$pkg")"
	# Check to see if it is in the CD pool.
	pkg_t="${CD_POOL["$pkgname"]}"
	# If it is, and the one in the pool is not older than this one,
	# skip it.
	if [[ $pkg_t && -f $pkg_t ]] && ( ! pkg_cmp "$pkg" "$pkg_t" ); then
	    #debug "Skipping copy of ${pkg##*/}, it is on the install media"
	    # if we are shrinking our ISO, make sure this one is in.
	    [[ $SHRINK_ISO = true ]] && INSTALLED_PKGS["$pkgname"]="true"
	    continue
	fi
	# Now check to see if we have already staged it
	pkg_t="${STAGED_POOL["$pkgname"]}"
	if [[ $pkg_t && -f $pkg_t ]]; then
	    # We have already staged it.  Check to see if ours is newer than
	    # the one already staged.
	    if pkg_cmp "$pkg" "$pkg_t"; then
		# We are newer.  Delete the old one, copy us,
		# and update $STAGED_POOL
		#debug "Replacing ${pkg_t##*/} with ${pkg##*/}"
		rm -f "$pkg_t"
		cp "$pkg" "$2"
		STAGED_POOL["$pkgname"]="$2/${pkg##*/}"
	    fi
	else
	    # We have not seen this package before.  Copy it.
	    cp "$pkg" "$2"
	    STAGED_POOL["$pkgname"]="$2/${pkg##*/}"
	fi
    done < <(find "$1" -type f)
}

# Update the package cache for a barclamp.
update_barclamp_pkg_cache() {
    # $1 = barclamp we are working with
    local bc_cache="$CACHE_DIR/barclamps/$1/$OS_TOKEN/pkgs" pkg dest bc
    local -A pkgs
    # Wipe out the packages already in the chroot package directory.
    while read pkg; do
	is_pkg "$pkg" && sudo rm -f "$pkg"
    done < <(find "$CHROOT/$CHROOT_PKGDIR" -type f)
    for bc in $(all_deps "$1"); do
	[[ -d "$CACHE_DIR/barclamps/$bc/$OS_TOKEN/pkgs/." ]] || continue
	sudo cp -a "$CACHE_DIR/barclamps/$bc/$OS_TOKEN/pkgs/." \
	    "$CHROOT/$CHROOT_PKGDIR"
    done
    # Remember what packages we already have.
    while read pkg; do
	is_pkg "$pkg" || continue
	pkgs["$pkg"]="true"
    done < <(cd "$CHROOT/$CHROOT_PKGDIR"; find -type f)
    chroot_fetch ${BC_PKGS["$1"]} ${BC_BUILD_PKGS["$1"]} || \
	die "Could not fetch packages required by barclamp $1"
    while read pkg; do
	is_pkg "$CHROOT/$CHROOT_PKGDIR/$pkg" || continue
	[[ ${pkgs["$pkg"]} = true ]] && continue
	if [[ ${pkg%/*} != '.' ]]; then
	    [[ -d $bc_cache/${pkg%/*} ]] || mkdir -p "$bc_cache/${pkg%/*}"
	    [[ -f $bc_cache/${pkg##*/} ]] && rm -f "$bc_cache/${pkg##*/}"
	fi
	cp "$CHROOT/$CHROOT_PKGDIR/$pkg" "$bc_cache/$pkg"
    done < <(cd "$CHROOT/$CHROOT_PKGDIR"; find -type f)
}

# Update the gem cache for a barclamp
update_barclamp_gem_cache() {
    local -A gems
    local gemname gemver gemopts bc gem
    local bc_cache="$CACHE_DIR/barclamps/$1/gems"

    # Wipe out the caches.
    ( cd "$CHROOT/$CHROOT_GEMDIR" && sudo rm -rf * )
    # if we have deb or gem caches, copy them back in.
    # Make sure we copy the caches for all our dependent barclamps.
    for bc in $(all_deps "$1"); do
	[[ -d "$CACHE_DIR/barclamps/$bc/$OS_TOKEN/pkgs/." ]] && \
	    sudo cp -a "$CACHE_DIR/barclamps/$bc/$OS_TOKEN/pkgs/." \
	    "$CHROOT/$CHROOT_PKGDIR"
	[[ -d "$CACHE_DIR/barclamps/$bc/gems/." ]] && \
	    sudo cp -a "$CACHE_DIR/barclamps/$bc/gems/." \
	    "$CHROOT/$CHROOT_GEMDIR"
    done

    while read gem; do
	[[ $gem = *.gem ]] || continue
	gems["$gem"]="true"
    done < <(find "$CHROOT/$CHROOT_GEMDIR" -type f)

    # install any build dependencies we need.
    chroot_install ${BC_BUILD_PKGS["$1"]}
    # Grab the gems needed for this barclamp.
    for gem in ${BC_GEMS["$1"]}; do
	if [[ $gem =~ $GEM_RE ]]; then
	    gemname="${BASH_REMATCH[1]}"
	    gemver="${BASH_REMATCH[2]}"
	else
	    gemname="$gem"
	    gemver=''
	fi
	gemopts=(install --no-ri --no-rdoc)
	[[ $gemver ]] && gemopts+=(--version "= ${gemver}")
	[[ $http_proxy ]] && gemopts+=(-p "$http_proxy")
	in_chroot /usr/bin/gem "${gemopts[@]}" "$gemname"
    done

    # Save our updated gems and pkgs in the cache for later.
     while read gem; do
	[[ $gem = *.gem ]] || continue
	[[ ${gems["$gem"]} = "true" ]] && continue
	cp "$gem" "$bc_cache"
    done < <(find "$CHROOT/$CHROOT_GEMDIR" -type f)
}

# Fetch any raw packages we do not already have.
update_barclamp_raw_pkg_cache() {
    local pkg bc_cache="$CACHE_DIR/barclamps/$1/$OS_TOKEN/pkgs"
    # Fetch any raw_pkgs we were asked to.
    for pkg in ${BC_RAW_PKGS["$1"]}; do
	[[ -f $bc_cache/${pkg##*/} ]] && continue
	echo "Caching $pkg:"
	curl -L -o "$bc_cache/${pkg##*/}" "$pkg"
    done
}

# Fetch any bare files that we do not already have.
update_barclamp_file_cache() {
    local dest pkg bc_cache="$CACHE_DIR/barclamps/$1/files"
    # Fetch any extra_pkgs we need.
    while read pkg; do
	dest=${pkg#* }
	[[ $dest = $pkg ]] && dest=''
	pkg=${pkg%% *}
	[[ -f $bc_cache/files/$dest/${pkg##*/} ]] && continue
	mkdir -p "$bc_cache/$dest"
	echo "Caching $pkg:"
	curl -L -o "$bc_cache/$dest/${pkg##*/}" "$pkg"
    done < <(write_lines "${BC_EXTRA_FILES[$1]}")
}

# Check to see if the barclamp package cache needs update.
barclamp_pkg_cache_needs_update() {
    local pkg pkgname arch bc
    local -A pkgs

    # First, check to see if we have all the packages we need.
    for bc in $(all_deps "$1"); do
	local bc_cache="$CACHE_DIR/barclamps/$bc/$OS_TOKEN/pkgs"
	mkdir -p "$bc_cache"
	while read pkg; do
	    is_pkg "$pkg" || continue
	    pkgname="$(pkg_name "$pkg")"
	    #debug "$pkgname is cached"
	    pkgs["$pkgname"]="$pkg"
	done < <(find "$bc_cache" -type f)
    done
    for pkg in ${BC_PKGS["$1"]} ${BC_BUILD_PKGS["$1"]}; do
	[[ $pkg ]] || continue
	for arch in "${PKG_ALLOWED_ARCHES[@]}"; do
	    [[ ${pkgs["$pkg-$arch"]} ]] && continue 2
	    if [[ ${CD_POOL["$pkg-$arch"]} ]]; then
		INSTALLED_PKGS["$pkg-$arch"]="true"
		continue 2
	    fi
	    #debug "Could not find $pkg-$arch"
	done
	return 0
    done
    return 1
}

# Check to see if the barclamp gem cache needs an update.
barclamp_gem_cache_needs_update() {
    local pkg pkgname
    local -A pkgs
    # Second, check to see if we have all the gems we need.
    for pkg in ${BC_GEMS["$1"]}; do
	for bc in $(all_deps "$1"); do
	    local bc_cache="$CACHE_DIR/barclamps/$bc/gems"
	    mkdir -p "$bc_cache"
	    [[ $(find "$bc_cache" \
		-name "$pkg*.gem" -type f) = *.gem ]] && continue 2
	done
	return 0
    done
    return 1
}

# CHeck to see if we are missing any raw packages.
barclamp_raw_pkg_cache_needs_update() {
    local pkg bc_cache="$CACHE_DIR/barclamps/$1/$OS_TOKEN/pkgs"
    mkdir -p "$bc_cache"
    # Third, check to see if we have all the raw_pkgs we need.
    for pkg in ${BC_RAW_PKGS["$1"]}; do
	[[ -f $bc_cache/${pkg##*/} ]] || return 0
    done
    return 1
}

# Check to see if we are missing any raw files.
barclamp_file_cache_needs_update() {
    local pkg dest bc_cache="$CACHE_DIR/barclamps/$1/files"
    mkdir -p "$bc_cache"
    # Fourth, check to make sure we have all the extra_pkgs we need.
    while read pkg; do
	dest=${pkg#* }
	[[ $dest = $pkg ]] && dest=''
	pkg=${pkg%% *}
	[[ -f $bc_cache/$dest/${pkg##*/} ]] || return 0
    done < <(write_lines "${BC_EXTRA_FILES[$1]}")
    return 1
}

# Arrange for cleanup to be called at the most common exit points.
trap cleanup 0 INT QUIT TERM

# Source our config file if we have one
[[ -f $HOME/.build-crowbar.conf ]] && \
    . "$HOME/.build-crowbar.conf"

# Look for a local one.
[[ -f build-crowbar.conf ]] && \
    . "build-crowbar.conf"

# Set up our proxies if we were asked to.
if [[ $USE_PROXY = "1" && $PROXY_HOST ]]; then
    proxy_str="http://"
    if [[ $PROXY_PASSWORD && $PROXY_USER ]]; then
	proxy_str+="$PROXY_USER:$PROXY_PASSWORD@"
    elif [[ $PROXY_USER ]]; then
	proxy_str+="$PROXY_USER@"
    fi
    proxy_str+="$PROXY_HOST"
    [[ $PROXY_PORT ]] && proxy_str+=":$PROXY_PORT"
    [[ $no_proxy ]] || no_proxy="localhost,localhost.localdomain,127.0.0.0/8,$PROXY_HOST"
    [[ $http_proxy ]] || http_proxy="$proxy_str/"
    [[ $https_proxy ]] || https_proxy="$http_proxy"
    export no_proxy http_proxy https_proxy
else
    unset no_proxy http_proxy https_proxy
fi

# Next, some configuration variables that can be used to tune how the
# build process works.

# Barclamps to include.  By default, start with jsut crowbar and let
# the dependency machinery and the command line pull in the rest.
# Note that BARCLAMPS is an array, not a string!
[[ $BARCLAMPS ]] || BARCLAMPS=()

# Location for caches that should not be erased between runs
[[ $CACHE_DIR ]] || CACHE_DIR="$HOME/.crowbar-build-cache"

# Location to store .iso images that we use in the build process.
# These are usually OS install DVDs that we will stage Crowbar on to.
[[ $ISO_LIBRARY ]] || ISO_LIBRARY="$CACHE_DIR/iso"

# This is the location that we will save the generated .iso to.
[[ $ISO_DEST ]] || ISO_DEST="$PWD"

# Directory that holds our Sledgehammer PXE tree.
[[ $SLEDGEHAMMER_PXE_DIR ]] || SLEDGEHAMMER_PXE_DIR="$CACHE_DIR/tftpboot"

# Location of the Crowbar checkout we are building from.
[[ $CROWBAR_DIR ]] ||CROWBAR_DIR="${0%/*}"
export CROWBAR_DIR

# Location of the Sledgehammer source tree.  Only used if we cannot
# find Sledgehammer in $SLEDGEHAMMER_PXE_DIR above.
[[ $SLEDGEHAMMER_DIR ]] || SLEDGEHAMMER_DIR="${CROWBAR_DIR}/../sledgehammer"

# Command to run to clean out the tree before starting the build.
# By default we want to be relatively pristine.
[[ $VCS_CLEAN_CMD ]] || VCS_CLEAN_CMD='git clean -f -x -d'

# Arrays holding the additional pkgs and gems populate Crowbar with.
REPOS=()

declare -A CD_POOL STAGED_POOL INSTALLED_PKGS

# Some helper functions

# Print a message to stderr and exit.  cleanup will be called.
die() { echo "$(date '+%F %T %z'): $*" >&2; exit 1; }

# Print a message to stderr and keep going.
debug() { echo "$(date '+%F %T %z'): $*" >&2; }

# Clean up any cruft that we might have left behind from the last run.
clean_dirs() {
    local d=''
    for d in "$@"; do
	(   mkdir -p "$d"
	    cd "$d"
	    chmod -R u+w .
	    rm -rf * )
    done
}

# Verify that the passed name is really a branch in the git repo.
branch_exists() { git show-ref --quiet --verify --heads -- "refs/heads/$1"; }

# Run a git command in the crowbar repo.
in_repo() ( cd "$CROWBAR_DIR"; "$@")

# Get the head revision of a git repository.
get_rev() (
    cd "$1"
    if [[ -d .git ]]; then
	git rev-parse HEAD
    else
	echo "Not a Git Repository"
    fi
)

# Run a git command in the build cache, assuming it is a git repository.
in_cache() (
    [[ $CURRENT_CACHE_BRANCH ]] || return
    cd "$CACHE_DIR"
    "$@"
)

# Check to see if something is a barclamp.
is_barclamp() { [[ -f $CROWBAR_DIR/barclamps/$1/crowbar.yml ]]; }

# Build our ISO image.
build_iso() (
    cd "$BUILD_DIR"
    rm -f isolinux/boot.cat
    find -name '.svn' -type d -exec rm -rf '{}' ';' 2>/dev/null >/dev/null
    find . -type f -not -name isolinux.bin -not -name sha1sums -not -path '*/.git/*' | \
	xargs sha1sum -b >sha1sums
    mkdir -p "$ISO_DEST"
	# Save the sha1sums and the build-info files along side the iso.
    cp sha1sums build-info "$ISO_DEST"
    mkisofs -r -V "${VERSION:0:30}" -cache-inodes -J -l -quiet \
	-b isolinux/isolinux.bin -c isolinux/boot.cat \
	-no-emul-boot --boot-load-size 4 -boot-info-table \
	-o "$ISO_DEST/$BUILT_ISO" "$IMAGE_DIR" "$BUILD_DIR"
)

# Have the smoketest framework do its thing with the ISO we just made.
test_iso() {
    [[ -L $HOME/test_framework ]] && rm -f "$HOME/test_framework"
    [[ -d $HOME/test_framework ]] && rm -rf "$HOME/test_framework"
    if [[ $CROWBAR_DIR = /* ]]; then
	ln -sf "$CROWBAR_DIR/test_framework" "$HOME/test_framework"
    else
	ln -sf "$currdir/$CROWBAR_DIR/test_framework" "$HOME/test_framework"
    fi
    (   unset no_proxy http_proxy https_proxy
	"$HOME/test_framework/test_crowbar.sh" "$@" \
	    use-iso "$ISO_DEST/$BUILT_ISO"
    ) || \
	die "$(date '+%F %T %z'): Smoketest of $ISO_DEST/$BUILT_ISO failed."
}

# Get the OS we were asked to stage Crowbar on to.  Assume it is Ubuntu 10.10
# unless we specify otherwise.
OS_TO_STAGE="${1-ubuntu-10.10}"
shift

# Make sure that we actually know how to build the ISO we were asked to
# build.  If we do not, print a helpful error message.
if ! [[ $OS_TO_STAGE && -d $CROWBAR_DIR/$OS_TO_STAGE-extra && \
    -f $CROWBAR_DIR/$OS_TO_STAGE-extra/build_lib.sh ]]; then
    cat <<EOF
You must pass the name of the operating system you want to stage Crowbar
on to.  Valid choices are:
EOF
    cd "$CROWBAR_DIR"
    for d in *-extra; do
	[[ -d $d && -f $d/build_lib.sh ]] || continue
	echo "    ${d%-extra}"
    done
    exit 1
fi

# Source OS specific build knowledge.  This includes:
# Parameters that build_crowbar.sh needs to know:
# OS = the distribution we are staging on to, such as redhat or ubuntu.
# OS_VERSION = the version of the distribution we are staging on to.
#              For redhat, it would be somethibng like 5.6
# OS_TOKEN = Defaults to "$OS-$OS_VERSION"
# ISO = the name of the install ISO image we are going to stage Crowbar on to.
# PKG_TYPE = The package type we should look for in crowbar.yml for OS
#            specific packages.
# PKG_ALLOWED_ARCHES = The allowed package arches we will look to stage.
#                      These are usually whatever the OS equivalent of
#                      amd64 and all are.
# CHROOT_GEMDIR = The location that the OS keeps its gem cache in.
# CHROOT_PKGDIR = The location that the OS keeps its package cache in.
# Functions that build_crowbar needs to call:
# find_cd_pool(): This function should echo the current location of the
#   package pool on $ISO when it is mounted on $IMAGE_DIR.
# fetch_os_iso(): This function should try to fetch the OS iso from
#   a well-known location on the Internet, and die if it cannot.
# chroot_update(): This function should ask the chroot to update its
#   package metadata.
# chroot_install(): This function should try to install the packages passed
#   to it as args.
# chroot_fetch(): This function should ask to download (but not install)
#   the packages passed to it as arguments and all their dependencies.
# add_repos(): This function should try to add repositories passed to it
#   as args to the chroot environment.
# is_pkg(): This function should check to see if the string passed to it
#   is a valid package filename.  The file need not actually exist.
# pkg_name(): This function should extract the package name and arch as the
#   package management system will see them from the file passed as an argument,
#   and echo that information in $name-$arch format.
# __make_chroot(): This function should handle all of the OS-specific actions
#   needed to set up the package-fetching chroot environment.
# pkg_cmp(): This function should check to see that both of the files passed to
#   it refer to the same package name.  If it does, this function should return
#   0 if the first package is a higher revision than the second one, and 1 if
#   the first package is the same or lower revision.  It should die if the
#   package names are not the same.
# final_build_fixups(): This function should take wahtever steps are needed
#   to make the default OS install process also ensure that the Crowbar bits
#   are properly staged and to completly automate the admin node install
#   process, either as an install from CD or an install via PXE.  This
#   usually entails modifying initrd files, adding kickstarts/install seeds,
#   modifying boot config files, and so on.
. "$CROWBAR_DIR/$OS_TO_STAGE-extra/build_lib.sh"

{
    # Make sure only one instance of the ISO build runs at a time.
    # Otherwise you can easily end up with a corrupted image.
    debug "Acquiring the build lock."
    flock 65
    # Figure out what our current branch is, in case we need to merge
    # other branches in to the iso to create our build.
    CURRENT_BRANCH="$(in_repo git symbolic-ref HEAD)" || \
	die "Not on a branch we can build from!"
    CURRENT_BRANCH=${CURRENT_BRANCH##*/}
    [[ $CURRENT_BRANCH ]] || die "Not on a branch we can merge from!"

    # Check and see if our local build repository is a git repo. If it is,
    # we may need to do the same sort of merging in it that we might do in the
    # Crowbar repository.
    if [[ -d $CACHE_DIR/.git ]]; then
	for br in "$CURRENT_BRANCH" master ''; do
	    [[ $br ]] || die "Cannot find $CURRENT_BRANCH or master in $CACHE_DIR"
	    (cd "$CACHE_DIR"; branch_exists "$br") || continue
	    CURRENT_CACHE_BRANCH="$br"
	    break
	done
	# If there are packages that have not been comitted, save them
	# in a stash before continuing.  We do this on the assumption that
	# these packages were added manually for testing purposes, or were
	# added in an earlier update-cache operation, but that the user has
	# not gotten around to comitting yet.
	if [[ ! $(in_cache git status) =~ working\ directory\ clean ]]; then
	    CACHE_THROWAWAY_STASH=$(in_cache git stash create)
	    in_cache git checkout -f .
	fi
    fi

    # Parse our options.
    while [[ $1 ]]; do
	case $1 in
	    # Merge a list of branches into a throwaway branch with the
	    # current branch as a baseline before starting the rest of the
	    # build process.  This makes it easier to spin up iso images
	    # with local changes without having to manually merge those
	    # changes in with any other branches of interest first.
	    # This code takes heavy advantage of the lightweight nature of
	    # git branches and takes care to leave uncomitted changes in place.
	    -m|--merge)
		shift
		# Loop through the rest of the arguments, as long as they
		# do not start with a -.
		while [[ $1 && ! ( $1 = -* ) ]]; do
		    # Check to make sure that this argument refers to a branch
		    # in the crowbar git tree.  Die if it does not.
		    in_repo branch_exists "$1" || die "$1 is not a git branch!"
		    # If we have not already created a throwaway branch to
		    # merge these branches into, do so now. If we have
		    # uncomitted changes that need to be stashed, do so here.
		    if [[ ! $THROWAWAY_BRANCH ]]; then
			THROWAWAY_BRANCH="build-throwaway-$$-$RANDOM"
			REPO_PWD="$PWD"
			if [[ ! $(in_repo git status) =~ working\ directory\ clean ]]; then
			    THROWAWAY_STASH=$(in_repo git stash create)
			    in_repo git checkout -f .
			fi
			in_repo git checkout -b "$THROWAWAY_BRANCH"
		    fi
		    # Merge the requested branch into the throwaway branch.
		    # Die if the merge failed -- there must have been a
		    # conflict, and the user needs to fix it up.
		    in_repo git merge "$1" || \
			die "Merge of $1 failed, fix things up and continue"
		    # If there is n identically named branch in the build cache,
		    # merge it into a throwaway branch of the build cache
		    # along with the current branch in the build cache.
		    # This makes it easier to include and manage packages that
		    # are branch-specific, but that do not need to be included
		    # in every build.
		    if in_cache branch_exists "$1"; then
			if [[ ! $CACHE_THROWAWAY_BRANCH ]]; then
			    CACHE_THROWAWAY_BRANCH=${THROWAWAY_BRANCH/build/cache}
			    in_cache git checkout -b "$CACHE_THROWAWAY_BRANCH"
			fi
			in_cache git merge "$1" || \
			    die "Could not merge build cache branch $1"
		    fi
		    shift
		done
		;;
	    # Force an update of the cache
	    update-cache|--update-cache) shift; need_update=true;;
	    # Pull in additional barclamps.
	    --barclamps)
		shift
		while [[ $1 && $1 != -* ]]; do
		    BARCLAMPS+=("$1")
		    shift
		done;;
	    --test)
		NEED_TEST=true
		test_params=(run-test)
		shift
		while [[ $1 && $1 != -* ]]; do
		    test_params+=("$1")
		    shift
		done;;
	    --shrink)
		type shrink_iso >&/dev/null || \
		    die "The build system does not know how to shrink $OS_TO_STAGE"
		SHRINK_ISO=true
		shift;;
	    --generate-minimal-install)
		type generate_minimal_install &>/dev/null || \
		    die "The build system does not know how to generate a minimal install list for $OS_TO_STAGE!"
		GENERATE_MINIMAL_INSTALL=true
		shift;;
	    *) 	die "Unknown command line parameter $1";;
	esac
    done

    # If we stached changes to the crowbar repo, apply them now.
    [[ $THROWAWAY_STASH ]] && in_repo git stash apply "$THROWAWAY_STASH"
    # Ditto for the build cache.
    [[ $CACHE_THROWAWAY_STASH ]] && \
	in_cache git stash apply "$CACHE_THROWAWAY_STASH"

    # Finalize where we expect to find our caches and out chroot.
    # If they were set in one of the conf files, don't touch them.

    # The directory we perform a minimal install into if we need
    # to refresh our gem or pkg caches
    [[ $CHROOT ]] || CHROOT="$CACHE_DIR/$OS_TOKEN/chroot"

    # Make sure that the $OS_TOKEN directory exist.
    mkdir -p "$CACHE_DIR/$OS_TOKEN"

    # The directory we will stage the build into.
    [[ $BUILD_DIR ]] || \
	BUILD_DIR="$CACHE_DIR/$OS_TOKEN/build"
    # The directory that we will mount the OS .ISO on .
    [[ $IMAGE_DIR ]] || \
	IMAGE_DIR="$CACHE_DIR/$OS_TOKEN/image"

    # Directory where we will look for our package lists
    [[ $PACKAGE_LISTS ]] || PACKAGE_LISTS="$BUILD_DIR/extra/packages"

    # Hashes to hold our "interesting" information.
    # Key = barclamp name
    # Value = whatever interesting thing we are looking for.
    declare -A BC_DEPS BC_GROUPS BC_PKGS BC_EXTRA_FILES BC_OS_DEPS BC_GEMS
    declare -A BC_REPOS BC_PPAS BC_RAW_PKGS BC_BUILD_PKGS BC_QUERY_STRINGS

    # Query strings to pull info we are interested out of crowbar.yml
    BC_QUERY_STRINGS["deps"]="barclamp requires"
    BC_QUERY_STRINGS["groups"]="barclamp member"
    BC_QUERY_STRINGS["pkgs"]="$PKG_TYPE pkgs"
    BC_QUERY_STRINGS["extra_files"]="extra_files"
    BC_QUERY_STRINGS["os_support"]="barclamp os_support"
    BC_QUERY_STRINGS["gems"]="gems pkgs"
    BC_QUERY_STRINGS["repos"]="$PKG_TYPE repos"
    BC_QUERY_STRINGS["ppas"]="$PKG_TYPE ppas"
    BC_QUERY_STRINGS["build_pkgs"]="$PKG_TYPE build_pkgs"
    BC_QUERY_STRINGS["raw_pkgs"]="$PKG_TYPE raw_pkgs"

    # Pull in interesting information from all our barclamps
    for bc in $CROWBAR_DIR/barclamps/*; do
	[[ -d $bc ]] || continue
	bc=${bc##*/}
	debug "Reading metadata for $bc barclamp."
	is_barclamp "$bc" || die "$bc is not a barclamp!"
	yml_file="$CROWBAR_DIR/barclamps/$bc/crowbar.yml"
	[[ $bc = crowbar ]] || BC_DEPS["$bc"]+="crowbar "
	for query in "${!BC_QUERY_STRINGS[@]}"; do
	    while read line; do
		[[ $line = nil ]] && continue
		case $query in
		    deps) is_in "$line "${BC_DEPS["$bc"]} || \
			BC_DEPS["$bc"]+="$line ";;
		    groups) is_in "$line" ${BC_GROUPS["$bc"]} ||
			BC_GROUPS["$line"]+="$bc ";;
		    pkgs) is_in "$line" ${BC_PKGS["$bc"]} || \
			BC_PKGS["$bc"]+="$line ";;
		    extra_files) BC_EXTRA_FILES["$bc"]+="$line\n";;
		    os_support) BC_OS_SUPPORT["$bc"]+="$line ";;
		    gems) BC_GEMS["$bc"]+="$line ";;
		    repos) BC_REPOS["$bc"]+="$line\n";;
		    ppas) [[ $PKG_TYPE = debs ]] || \
			die "Cannot declare a PPA for $PKG_TYPE!"
			BC_REPOS["$bc"]+="ppa $line\n";;
		    build_pkgs) BC_BUILD_PKGS["$bc"]+="$line ";;
		    raw_pkgs) BC_RAW_PKGS["$bc"]+="$line ";;
		    *) die "Cannot handle query for $query."
		esac
	    done < <("$CROWBAR_DIR/parse_yml.rb" \
		"$yml_file" \
		${BC_QUERY_STRINGS["$query"]} 2>/dev/null)
	done
    done

    debug "Analyzing barclamp group membership"
    # If any barclamps need group expansion, do it.
    for bc in "${!BC_DEPS[@]}"; do
	newdeps=''
	for dep in ${BC_DEPS["$bc"]}; do
	    if [[ $dep = @* ]]; then
		[[ ${BC_GROUPS["${dep#@}"]} ]] || \
		    die "$bc depends on group ${dep#@}, but that group does not exist!"
		for d in ${BC_GROUPS["${dep#@}"]}; do
		    is_barclamp "$d" || \
			die "$bc depends on barclamp $d from group ${dep#@}, but $d does not exist!"
		    newdeps+="$d "
		done
	    else
		is_barclamp "$dep" || \
		    die "$bc depends on barclamp $dep, but $dep is not a barclamp!"
		newdeps+="$dep "
	    fi
	done
	BC_DEPS["$bc"]="$newdeps"
    done

    # Proxy Variables
    [[ $USE_PROXY ]] || USE_PROXY=0
    [[ $PROXY_HOST ]] || PROXY_HOST=""
    [[ $PROXY_PORT ]] || PROXY_PORT=""
    [[ $PROXY_USER ]] || PROXY_USER=""
    [[ $PROXY_ESC_USER ]] || PROXY_ESC_USER=""
    [[ $PROXY_PASSWORD ]] || PROXY_PASSWORD=""

    # Version for ISO
    [[ $VERSION ]] || VERSION="$(cd "$CROWBAR_DIR"; git describe --long --tags)-dev"

    # Name of the built iso we will build
    [[ $BUILT_ISO ]] || BUILT_ISO="crowbar-${VERSION}.iso"

    # If we were not passed a list of barclamps to include,
    # pull in all of the ones declared as submodules.
    [[ $BARCLAMPS ]] || BARCLAMPS=($(cd "$CROWBAR_DIR"
	    while read sha submod branch; do
		[[ $submod = barclamps/* ]] || continue
		[[ -f $submod/crowbar.yml ]] || \
		    echo "Cannot find crowbar.yml for $submod, exiting."
		echo "${submod##*/}"
	    done < <(git submodule status))
	)

    # Group-expand barclamps if needed, and unset groups after they are expanded.
    for i in "${!BARCLAMPS[@]}"; do
	bc="${BARCLAMPS[$i]}"
	if [[ $bc = @* ]]; then
	    [[ ${BC_GROUPS["${bc#@}"]} ]] || \
		die "No such group ${bc#@}!"
	    BARCLAMPS+=(${BC_GROUPS["${bc#@}"]})
	    unset BARCLAMPS[$i]
	else
	    is_barclamp "$bc" || die "$bc is not a barclamp!"
	fi
    done
    BARCLAMPS=("${BARCLAMPS[@]//@*}")

    # Pull in dependencies for the barclamps.
    # Everything depends on the crowbar barclamp, so include it first.
    new_barclamps=("crowbar")
    while [[ t = t ]]; do
	for bc in "${BARCLAMPS[@]}"; do
	    for dep in ${BC_DEPS["$bc"]}; do
		is_in "$dep" "${new_barclamps[@]}" && continue
		new_barclamps+=("$dep")
	    done
	    is_in "$bc" "${new_barclamps[@]}" || new_barclamps+=("$bc")
	done
	[[ ${BARCLAMPS[*]} = ${new_barclamps[*]} ]] && break
	BARCLAMPS=("${new_barclamps[@]}")
    done

    # Make any directories we don't already have
    for d in "$ISO_LIBRARY" "$ISO_DEST" "$IMAGE_DIR" "$BUILD_DIR" \
	"$SLEDGEHAMMER_PXE_DIR" "$CHROOT"; do
	mkdir -p "$d"
    done

    debug "Checking for Sledgehammer."
    # Make sure Sledgehammer has already been built and pre-staged.
    if ! [[ -f $SLEDGEHAMMER_DIR/bin/sledgehammer-tftpboot.tar.gz || \
	-f $SLEDGEHAMMER_PXE_DIR/initrd0.img ]]; then
	echo "Slegehammer TFTP image missing!"
	echo "Please build Sledgehammer from $SLEDGEHAMMER_DIR before building Crowbar."
	exit 1
    fi

    # Fetch the OS ISO if we need to.
    [[ -f $ISO_LIBRARY/$ISO ]] || fetch_os_iso

    # Start with a clean slate.
    clean_dirs "$IMAGE_DIR" "$BUILD_DIR"

    debug "Cleaning up any VCS cruft."
    # Clean up any cruft that the editor may have left behind.
    (cd "$CROWBAR_DIR"; $VCS_CLEAN_CMD)

    # Make additional directories we will need.
    for d in discovery extra; do
	mkdir -p "$BUILD_DIR/$d"
    done

    # Mount our ISO for the build process.
    debug "Mounting $ISO"
    sudo mount -t iso9660 -o loop "$ISO_LIBRARY/$ISO" "$IMAGE_DIR" || \
	die "Could not mount $ISO"
    debug "Indexing CD package pool."
    index_cd_pool

    # Copy over the Crowbar bits and their prerequisites
    cp -r "$CROWBAR_DIR/extra"/* "$BUILD_DIR/extra"
    cp -r "$CROWBAR_DIR/$OS_TOKEN-extra"/* "$BUILD_DIR/extra"
    cp -r "$CROWBAR_DIR/change-image"/* "$BUILD_DIR"
    mkdir -p "$BUILD_DIR/dell/barclamps"
    echo "build-timestamp: $(date '+%F %T %z')" > "$BUILD_DIR/build-info"
    echo "build-os: $OS_TOKEN" >>"$BUILD_DIR/build-info"
    echo "build-os-iso: $ISO" >>"$BUILD_DIR/build-info"
    echo "crowbar: $(get_rev "$CROWBAR_DIR")" >>"$BUILD_DIR/build-info"
    for bc in "${BARCLAMPS[@]}"; do
	is_barclamp "$bc" || die "Cannot find barclamp $bc!"
	debug "Staging $bc barclamp."
	for cache in pkg gem raw_pkg file; do
	    checker="barclamp_${cache}_cache_needs_update"
	    updater="update_barclamp_${cache}_cache"
	    [[ $(type $checker) = "$checker is a function"* ]] || \
		die "Asked to check $cache cache, but no checker function!"
	    [[ $(type $updater) = "$updater is a function"* ]] || \
		die "Might need to update $cache cache, but no updater!"
	    if $checker "$bc" || [[ $need_update = true ]]; then
		debug "Updating $cache cache for $bc"
		[[ $cache =~ ^(pkg|gem)$ ]] && make_chroot
		$updater "$bc"
	    fi
	done
	cp -r "$CROWBAR_DIR/barclamps/$bc" "$BUILD_DIR/dell/barclamps"
	mkdir -p "$BUILD_DIR/extra/pkgs/"
	stage_pkgs "$CACHE_DIR/barclamps/$bc/$OS_TOKEN/pkgs" \
	    "$BUILD_DIR/extra/pkgs"
	for f in file gems; do
	    [[ -d "$CACHE_DIR/barclamps/$bc/$f" ]] || continue
	    cp -r "$CACHE_DIR/barclamps/$bc/$f" "$BUILD_DIR/extra"
	done

	echo "barclamps/$bc: $(get_rev "$CROWBAR_DIR/barclamps/$bc")" >> "$BUILD_DIR/build-info"
    done

    (cd "$BUILD_DIR"
	find extra/pkgs extra/gems extra/files -type f -print | \
	    sort >> "build-info")
    # Make sure we still provide the legacy ami location
    (cd "$BUILD_DIR"; ln -sf extra/files/ami)
    # Store off the version
    echo "$VERSION" >> "$BUILD_DIR/dell/Version"

    # Custom start-up in place
    if [ -f "$CROWBAR_DIR/crowbar.json" ] ; then
      mkdir -p "$BUILD_DIR/extra/config"
      cp "$CROWBAR_DIR/crowbar.json" "$BUILD_DIR/extra/config"
    fi

    final_build_fixups

    # Copy over the bits that Sledgehammer will look for.
    debug "Copying over Sledgehammer bits"
    # If we need to copy over a new Sledgehammer image, do so.
    if [[ $SLEDGEHAMMER_DIR/bin/sledgehammer-tftpboot.tar.gz -nt \
	$SLEDGEHAMMER_PXE_DIR/initrd0.img ]]; then
	(   cd $SLEDGEHAMMER_PXE_DIR
	    debug "Extracting new Sledgehammer TFTP boot image"
	    rm -rf .
	    cd ..
	    tar xzf "$SLEDGEHAMMER_DIR/bin/sledgehammer-tftpboot.tar.gz"
	    rm -f "$SLEDGEHAMMER_DIR/bin/sledgehammer-tftpboot.tar.gz"
	)
    fi
    cp -a "$SLEDGEHAMMER_PXE_DIR"/* "$BUILD_DIR/discovery"

    # Make our image
    debug "Creating new ISO"
    # Find files and directories that mkisofs will complain about.
    # Do just top-level overlapping directories for now.
    for d in $(cat <(cd "$BUILD_DIR"; find -maxdepth 1 -type d) \
	           <(cd "$IMAGE_DIR"; find -maxdepth 1 -type d) | \
	           sort |uniq -d); do
	[[ $d = . ]] && continue
	d=${d#./}
	# Copy contents of the found directories into $BUILD_DIR, taking care
	# to not clobber existing files.
	mkdir -p "$BUILD_DIR/$d"
	chmod u+wr "$BUILD_DIR/$d"
	# We could also use cp -n, but rhel5 and centos5 do not understand it.
	rsync -rl --ignore-existing --inplace "$IMAGE_DIR/$d/." "$BUILD_DIR/$d/."
	chmod -R u+wr "$BUILD_DIR/$d"
	# Bind mount an empty directory on the $IMAGE_DIR instance.
	sudo mount -t tmpfs -o size=1K tmpfs "$IMAGE_DIR/$d"
    done
    mkdir -p "$BUILD_DIR/isolinux"
    chmod u+wr "$BUILD_DIR/isolinux"
    rsync -rl --ignore-existing --inplace \
	"$IMAGE_DIR/isolinux/." "$BUILD_DIR/isolinux/."
    chmod -R u+wr "$BUILD_DIR/isolinux"
    sudo mount -t tmpfs -o size=1K tmpfs "$IMAGE_DIR/isolinux"

    [[ $SHRINK_ISO && ! $GENERATE_MINIMAL_ISO ]] && shrink_iso
    # Make a file list and a link list.
    ( cd $BUILD_DIR
      find . -type f | \
	  sort > crowbar_files.list
      find . -type l | \
	  xargs ls -ld | \
	  awk '{ print $8 " " $10 }' | \
	  sort > crowbar_links.list
    )
    ( cd $IMAGE_DIR
      find . -type f | \
	  sort >> $BUILD_DIR/crowbar_files.list
      find . -type l | \
	  xargs ls -ld | \
	  awk '{ print $8 " " $10 }' | \
	  sort >> $BUILD_DIR/crowbar_links.list
    )

    # Make an ISO
    build_iso || die "There was a problem building our ISO."
    if [[ $GENERATE_MINIMAL_INSTALL = true ]]; then
	if [[ ! -f "$CROWBAR_DIR/$OS_TOKEN-extra/minimal-install" ]]; then
	    if [[ ! -f "$HOME/admin-installed.list" ]]; then
		test_params=(run-test scratch admin-only)
		    test_iso run-test admin-only
	    fi
	    [[ ! -f "$HOME/crowbar-installed.$OS_TOKEN.list" ]] || \
		die "Asked to shrink build for $OS_TOKEN, but package list missing!"
	    mv "$HOME/admin-installed.list" \
		"$CROWBAR_DIR/$OS_TOKEN-extra/minimal-install"
	fi

	build_iso
    fi
    echo "$(date '+%F %T %z'): Image at $ISO_DEST/$BUILT_ISO"
    if [[ $NEED_TEST = true ]]; then
	echo "$(date '+%F %T %z'): Testing new iso"
	test_iso "${test_params[@]}" || die "Test failed."
    fi
    echo "$(date '+%F %T %z'): Finished."
} 65> /tmp/.build_crowbar.lock

#!/bin/bash
# Copyright 2011, Dell
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#  http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# Author: VictorLowther
#

# This script expects to be able to run certian commands as root.
# Either run it as a user who can sudo to root, or give the user
# you are running it as the following sudo rights:
# crowbar-tester ALL = NOPASSWD: /bin/mount, /bin/umount, /usr/sbin/debootstrap, /bin/cp, /usr/sbin/chroot

# When running this script for the first time, it will automatically create a
# cache directory and try to populate it with all the build dependencies.
# After that, if you need to pull in new dependencies, you will need to
# call the script with the --update-cache parameter.  If you are going to
# develop on Crowbar, it is a good idea to put the build cache in its own git
# repository, and create a branching structure for the packages that mirrors
# the branching structure in the crowbar repository -- if you do that, then
# this build script can be smarter about what packages it should pull in
# whenever you invoke it to build an iso.

GEM_RE='([^0-9].*)-([0-9].*)'
[[ $DEBUG ]] && {
    set -x
    export PS4='${BASH_SOURCE}@${LINENO}(${FUNCNAME[0]}): '
}

readonly currdir="$PWD"
export PATH="$PATH:/sbin:/usr/sbin:/usr/local/sbin"

# Our general cleanup function.  It is called as a trap whenever the
# build script exits, and it's job is to make sure we leave the local
# system in the same state we cound it, modulo a few calories of wasted heat
# and a shiny new .iso.
cleanup() {
    # Clean up any stray mounts we may have left behind.
    # The paranoia with the grepping is to ensure that we do not
    # inadvertently umount everything.
    GREPOPTS=()
    [[ $CACHE_DIR ]] && GREPOPTS=(-e "$CACHE_DIR")
    [[ $IMAGE_DIR && $CACHE_DIR =~ $IMAGE_DIR ]] && GREPOPTS+=(-e "$IMAGE_DIR")
    [[ $BUILD_DIR && $CACHE_DIR =~ $BUILD_DIR ]] && GREPOPTS+=(-e "$BUILD_DIR")
    [[ $CHROOT && $CACHE_DIR =~ $CHROOT ]] && GREPOPTS+=(-e "$CHROOT")
    if [[ $GREPOPTS ]]; then
	while read dev fs type opts rest; do
	    sudo umount -d -l "$fs"
	done < <(tac /proc/self/mounts |grep "${GREPOPTS[@]}")
    fi
    # If the build process spawned a copy of webrick, make sure it is dead.
    [[ $webrick_pid && -d /proc/$webrick_pid ]] && kill -9 $webrick_pid
    # clean up after outselves from merging branches, if needed.
    cd "$currdir/$CROWBAR_DIR"
    if [[ $THROWAWAY_BRANCH ]]; then
	# Check out the branch we started the build process, and then
	# nuke whatever throwaway branch we may have created.
	git checkout -f "${CURRENT_BRANCH##*/}" &>/dev/null
	git branch -D "$THROWAWAY_BRANCH" &>/dev/null
    fi
    # If we saved unadded changes, resurrect them.
    [[ $THROWAWAY_STASH ]] && git stash apply "$THROWAWAY_STASH" &>/dev/null
    # Do the same thing as above, but for the build cache instead.
    cd "$CACHE_DIR"
    if [[ $CACHE_THROWAWAY_BRANCH ]]; then
	git checkout -f "$CURRENT_CACHE_BRANCH" &>/dev/null
	git branch -D "$CACHE_THROWAWAY_BRANCH" &>/dev/null
    fi
    [[ $CACHE_THROWAWAY_STASH ]] && git stash apply "$CACHE_THROWAWAY_STASH"
    for d in "$IMAGE_DIR" "$BUILD_DIR" "$CHROOT"; do
	[[ -d $d ]] && sudo rm -rf -- "$d"
    done
}

# Test to see if $1 is in the rest of the args.
is_in() {
    local t="(^| )$1( |\$)"
    shift
    [[ $* =~ $t ]]
}

# Run a command in our chroot environment.
in_chroot() { sudo -H /usr/sbin/chroot "$CHROOT" "$@"; }

# A little helper function for doing bind mounts.
bind_mount() {
    [[ -d $2 ]] || mkdir -p "$2"
    grep -q "$2" /proc/self/mounts || sudo mount --bind "$1" "$2"
}

# Write a string with \n translated to a real line break.
write_lines() { [[ $1 ]] && printf "%b" "$1"; }

# Read base repository information from the *.list files.
read_base_repos() {
    for pkgfile in "$BUILD_DIR/extra/packages/"*.list; do
	[[ -f $pkgfile ]] || continue
	while read line; do
	    line=${line%%#*}
	    [[ $line ]] || continue
	    if [[ $line = repository* ]]; then
		REPOS+=("${line#* }")
	    else
		echo "$line in $pkgfile has been superceded by crowbar.yml metadata." >&2
		die "Please migrate $line into the appropriate barclamp config file."
	    fi
	done <"$pkgfile"
    done
}

# Worker function for all_deps
__all_deps() {
    local dep
    if [[ ${BC_DEPS["$1"]} ]]; then
	for dep in ${BC_DEPS["$1"]}; do
	    is_in "$dep" "${deps[@]}" && continue
	    __all_deps "$dep"
	done
    fi
    is_in "$1" "${deps[@]}" || deps+=("$1")
    return 0
}

# Given a barclamp, echo all of the dependencies of that barclamp
all_deps() {
    local deps=() dep
    __all_deps "$1"
    echo "${deps[*]}"
}

# A couple of utility functions for comparing version numbers.
num_re='^[0-9]+$'
__cmp() {
    [[ $1 || $2 ]] || return 255 # neither 1 nor 2 or set, we are done
    [[ $1 && ! $2 ]] && return 2 # 1 is set and 2 is not, 1 > 2
    [[ ! $1 && $2 ]] && return 0 # 2 is set and 1 is not, 1 < 2
    local a="$1" b="$2"
    if [[ $a =~ $num_re && $b =~ $num_re ]]; then #both numbers, numeric cmp.
	# make sure leading zeros do not confuse us
	a=${a##0} b=${b##0}
	((${a:=0} > ${b:=0})) && return 2
	(($a < $b)) && return 0
	return 1
    else # string compare
	[[ $a > $b ]] && return 2
	[[ $a < $b ]] && return 0
	return 1
    fi
}

# Compare a version string, handling mixed numeric and alpha sequences
# as appropriate.
vercmp(){
    # $1 = version string of first package
    # $2 = version string of second package
    # Returns 0 if $1 > $2, 1 otherwise.
    # urldecode the version strings.
    local a="$(printf "%b" "${1//%/\\x}")"
    local b="$(printf "%b" "${2//%/\\x}")"
    # prepend a version epoch if one is not already there.
    [[ $a =~ ^[0-9]+: ]] || a="0:$a"
    [[ $b =~ ^[0-9]+: ]] || b="0:$b"
    local ver1=()
    local ver2=()
    local i=0
    # split the version strings into arrays using the usual field splitters
    IFS=':.-_ +' read -rs -a ver1 <<< "$a"
    IFS=':.-_ +' read -rs -a ver2 <<< "$b"
    for ((i=0;;i++)); do
	__cmp "${ver1[$i]}" "${ver2[$i]}"
	case $? in
	    2) return 0;;
	    0) return 1;;
	    255) return 1;;
	esac
    done
}

# Index the pool of packages in the CD.
index_cd_pool() {
    # Scan through our pool to find pkgs we can easily omit.

    local pkgname='' pkg=''
    while read pkg; do
	[[ -f $pkg ]] && is_pkg "$pkg" || continue
	pkgname="$(pkg_name "$pkg")"
	CD_POOL["$pkgname"]="${pkg}"
    done < <(find "$(find_cd_pool)" -type f)
}

# Make a chroot environment for package-fetching purposes.
make_chroot() {
    [[ -f $CHROOT/etc/resolv.conf ]] && return 0
    local bc repo
    debug "Making package-fetching chroot"
    mkdir -p "$CHROOT/$CHROOT_PKGDIR"
    mkdir -p "$CHROOT/$CHROOT_GEMDIR"
    __make_chroot

    read_base_repos
    # Add our basic repositories
    add_repos "${REPOS[@]}" || \
	die "Could not add base repositories for $OS_TOKEN chroot!"

    # Add the repos from the barclamps.
    # We do it here because importing the metadata takes forever
    # if we refresh on every barclamp.
    for bc in "${BARCLAMPS[@]}"; do
	while read repo; do
	    add_repos "$repo"
	done < <(write_lines "${BC_REPOS[$bc]}")
    done
    chroot_update
}

stage_pkgs() {
    # $1 = cache to copy from.
    # $2 = location to copy to
    local pkg pkgname pkg_t
    while read pkg; do
	# If it is not a package, skip it.
	is_pkg "$pkg" || continue
	pkgname="$(pkg_name "$pkg")"
	# Check to see if it is in the CD pool.
	pkg_t="${CD_POOL["$pkgname"]}"
	# If it is, and the one in the pool is not older than this one,
	# skip it.
	if [[ $pkg_t && -f $pkg_t ]] && ( ! pkg_cmp "$pkg" "$pkg_t" ); then
	    #debug "Skipping copy of ${pkg##*/}, it is on the install media"
	    # if we are shrinking our ISO, make sure this one is in.
	    [[ $SHRINK_ISO = true ]] && INSTALLED_PKGS["$pkgname"]="true"
	    continue
	fi
	# Now check to see if we have already staged it
	pkg_t="${STAGED_POOL["$pkgname"]}"
	if [[ $pkg_t && -f $pkg_t ]]; then
	    # We have already staged it.  Check to see if ours is newer than
	    # the one already staged.
	    if pkg_cmp "$pkg" "$pkg_t"; then
		# We are newer.  Delete the old one, copy us,
		# and update $STAGED_POOL
		#debug "Replacing ${pkg_t##*/} with ${pkg##*/}"
		rm -f "$pkg_t"
		cp "$pkg" "$2"
		STAGED_POOL["$pkgname"]="$2/${pkg##*/}"
	    fi
	else
	    # We have not seen this package before.  Copy it.
	    cp "$pkg" "$2"
	    STAGED_POOL["$pkgname"]="$2/${pkg##*/}"
	fi
    done < <(find "$1" -type f)
}

# Update the package cache for a barclamp.
update_barclamp_pkg_cache() {
    # $1 = barclamp we are working with
    local bc_cache="$CACHE_DIR/barclamps/$1/$OS_TOKEN/pkgs" pkg dest bc
    local -A pkgs
    # Wipe out the packages already in the chroot package directory.
    while read pkg; do
	is_pkg "$pkg" && sudo rm -f "$pkg"
    done < <(find "$CHROOT/$CHROOT_PKGDIR" -type f)
    for bc in $(all_deps "$1"); do
	[[ -d "$CACHE_DIR/barclamps/$bc/$OS_TOKEN/pkgs/." ]] || continue
	sudo cp -a "$CACHE_DIR/barclamps/$bc/$OS_TOKEN/pkgs/." \
	    "$CHROOT/$CHROOT_PKGDIR"
    done
    # Remember what packages we already have.
    while read pkg; do
	is_pkg "$pkg" || continue
	pkgs["$pkg"]="true"
    done < <(cd "$CHROOT/$CHROOT_PKGDIR"; find -type f)
    chroot_fetch ${BC_PKGS["$1"]} ${BC_BUILD_PKGS["$1"]} || \
	die "Could not fetch packages required by barclamp $1"
    while read pkg; do
	is_pkg "$CHROOT/$CHROOT_PKGDIR/$pkg" || continue
	[[ ${pkgs["$pkg"]} = true ]] && continue
	if [[ ${pkg%/*} != '.' ]]; then
	    [[ -d $bc_cache/${pkg%/*} ]] || mkdir -p "$bc_cache/${pkg%/*}"
	    [[ -f $bc_cache/${pkg##*/} ]] && rm -f "$bc_cache/${pkg##*/}"
	fi
	cp "$CHROOT/$CHROOT_PKGDIR/$pkg" "$bc_cache/$pkg"
    done < <(cd "$CHROOT/$CHROOT_PKGDIR"; find -type f)
}

# Update the gem cache for a barclamp
update_barclamp_gem_cache() {
    local -A gems
    local gemname gemver gemopts bc gem
    local bc_cache="$CACHE_DIR/barclamps/$1/gems"

    # Wipe out the caches.
    ( cd "$CHROOT/$CHROOT_GEMDIR" && sudo rm -rf * )
    # if we have deb or gem caches, copy them back in.
    # Make sure we copy the caches for all our dependent barclamps.
    for bc in $(all_deps "$1"); do
	[[ -d "$CACHE_DIR/barclamps/$bc/$OS_TOKEN/pkgs/." ]] && \
	    sudo cp -a "$CACHE_DIR/barclamps/$bc/$OS_TOKEN/pkgs/." \
	    "$CHROOT/$CHROOT_PKGDIR"
	[[ -d "$CACHE_DIR/barclamps/$bc/gems/." ]] && \
	    sudo cp -a "$CACHE_DIR/barclamps/$bc/gems/." \
	    "$CHROOT/$CHROOT_GEMDIR"
    done

    while read gem; do
	[[ $gem = *.gem ]] || continue
	gems["$gem"]="true"
    done < <(find "$CHROOT/$CHROOT_GEMDIR" -type f)

    # install any build dependencies we need.
    chroot_install ${BC_BUILD_PKGS["$1"]}
    # Grab the gems needed for this barclamp.
    for gem in ${BC_GEMS["$1"]}; do
	if [[ $gem =~ $GEM_RE ]]; then
	    gemname="${BASH_REMATCH[1]}"
	    gemver="${BASH_REMATCH[2]}"
	else
	    gemname="$gem"
	    gemver=''
	fi
	gemopts=(install --no-ri --no-rdoc)
	[[ $gemver ]] && gemopts+=(--version "= ${gemver}")
	[[ $http_proxy ]] && gemopts+=(-p "$http_proxy")
	in_chroot /usr/bin/gem "${gemopts[@]}" "$gemname"
    done

    # Save our updated gems and pkgs in the cache for later.
     while read gem; do
	[[ $gem = *.gem ]] || continue
	[[ ${gems["$gem"]} = "true" ]] && continue
	cp "$gem" "$bc_cache"
    done < <(find "$CHROOT/$CHROOT_GEMDIR" -type f)
}

# Fetch any raw packages we do not already have.
update_barclamp_raw_pkg_cache() {
    local pkg bc_cache="$CACHE_DIR/barclamps/$1/$OS_TOKEN/pkgs"
    # Fetch any raw_pkgs we were asked to.
    for pkg in ${BC_RAW_PKGS["$1"]}; do
	[[ -f $bc_cache/${pkg##*/} ]] && continue
	echo "Caching $pkg:"
	curl -L -o "$bc_cache/${pkg##*/}" "$pkg"
    done
}

# Fetch any bare files that we do not already have.
update_barclamp_file_cache() {
    local dest pkg bc_cache="$CACHE_DIR/barclamps/$1/files"
    # Fetch any extra_pkgs we need.
    while read pkg; do
	dest=${pkg#* }
	[[ $dest = $pkg ]] && dest=''
	pkg=${pkg%% *}
	[[ -f $bc_cache/files/$dest/${pkg##*/} ]] && continue
	mkdir -p "$bc_cache/$dest"
	echo "Caching $pkg:"
	curl -L -o "$bc_cache/$dest/${pkg##*/}" "$pkg"
    done < <(write_lines "${BC_EXTRA_FILES[$1]}")
}

# Check to see if the barclamp package cache needs update.
barclamp_pkg_cache_needs_update() {
    local pkg pkgname arch bc
    local -A pkgs

    # First, check to see if we have all the packages we need.
    for bc in $(all_deps "$1"); do
	local bc_cache="$CACHE_DIR/barclamps/$bc/$OS_TOKEN/pkgs"
	mkdir -p "$bc_cache"
	while read pkg; do
	    is_pkg "$pkg" || continue
	    pkgname="$(pkg_name "$pkg")"
	    #debug "$pkgname is cached"
	    pkgs["$pkgname"]="$pkg"
	done < <(find "$bc_cache" -type f)
    done
    for pkg in ${BC_PKGS["$1"]} ${BC_BUILD_PKGS["$1"]}; do
	[[ $pkg ]] || continue
	for arch in "${PKG_ALLOWED_ARCHES[@]}"; do
	    [[ ${pkgs["$pkg-$arch"]} ]] && continue 2
	    if [[ ${CD_POOL["$pkg-$arch"]} ]]; then
		INSTALLED_PKGS["$pkg-$arch"]="true"
		continue 2
	    fi
	    #debug "Could not find $pkg-$arch"
	done
	return 0
    done
    return 1
}

# Check to see if the barclamp gem cache needs an update.
barclamp_gem_cache_needs_update() {
    local pkg pkgname
    local -A pkgs
    # Second, check to see if we have all the gems we need.
    for pkg in ${BC_GEMS["$1"]}; do
	for bc in $(all_deps "$1"); do
	    local bc_cache="$CACHE_DIR/barclamps/$bc/gems"
	    mkdir -p "$bc_cache"
	    [[ $(find "$bc_cache" \
		-name "$pkg*.gem" -type f) = *.gem ]] && continue 2
	done
	return 0
    done
    return 1
}

# CHeck to see if we are missing any raw packages.
barclamp_raw_pkg_cache_needs_update() {
    local pkg bc_cache="$CACHE_DIR/barclamps/$1/$OS_TOKEN/pkgs"
    mkdir -p "$bc_cache"
    # Third, check to see if we have all the raw_pkgs we need.
    for pkg in ${BC_RAW_PKGS["$1"]}; do
	[[ -f $bc_cache/${pkg##*/} ]] || return 0
    done
    return 1
}

# Check to see if we are missing any raw files.
barclamp_file_cache_needs_update() {
    local pkg dest bc_cache="$CACHE_DIR/barclamps/$1/files"
    mkdir -p "$bc_cache"
    # Fourth, check to make sure we have all the extra_pkgs we need.
    while read pkg; do
	dest=${pkg#* }
	[[ $dest = $pkg ]] && dest=''
	pkg=${pkg%% *}
	[[ -f $bc_cache/$dest/${pkg##*/} ]] || return 0
    done < <(write_lines "${BC_EXTRA_FILES[$1]}")
    return 1
}

# Arrange for cleanup to be called at the most common exit points.
trap cleanup 0 INT QUIT TERM

# Source our config file if we have one
[[ -f $HOME/.build-crowbar.conf ]] && \
    . "$HOME/.build-crowbar.conf"

# Look for a local one.
[[ -f build-crowbar.conf ]] && \
    . "build-crowbar.conf"

# Set up our proxies if we were asked to.
if [[ $USE_PROXY = "1" && $PROXY_HOST ]]; then
    proxy_str="http://"
    if [[ $PROXY_PASSWORD && $PROXY_USER ]]; then
	proxy_str+="$PROXY_USER:$PROXY_PASSWORD@"
    elif [[ $PROXY_USER ]]; then
	proxy_str+="$PROXY_USER@"
    fi
    proxy_str+="$PROXY_HOST"
    [[ $PROXY_PORT ]] && proxy_str+=":$PROXY_PORT"
    [[ $no_proxy ]] || no_proxy="localhost,localhost.localdomain,127.0.0.0/8,$PROXY_HOST"
    [[ $http_proxy ]] || http_proxy="$proxy_str/"
    [[ $https_proxy ]] || https_proxy="$http_proxy"
    export no_proxy http_proxy https_proxy
else
    unset no_proxy http_proxy https_proxy
fi

# Next, some configuration variables that can be used to tune how the
# build process works.

# Barclamps to include.  By default, start with jsut crowbar and let
# the dependency machinery and the command line pull in the rest.
# Note that BARCLAMPS is an array, not a string!
[[ $BARCLAMPS ]] || BARCLAMPS=()

# Location for caches that should not be erased between runs
[[ $CACHE_DIR ]] || CACHE_DIR="$HOME/.crowbar-build-cache"

# Location to store .iso images that we use in the build process.
# These are usually OS install DVDs that we will stage Crowbar on to.
[[ $ISO_LIBRARY ]] || ISO_LIBRARY="$CACHE_DIR/iso"

# This is the location that we will save the generated .iso to.
[[ $ISO_DEST ]] || ISO_DEST="$PWD"

# Directory that holds our Sledgehammer PXE tree.
[[ $SLEDGEHAMMER_PXE_DIR ]] || SLEDGEHAMMER_PXE_DIR="$CACHE_DIR/tftpboot"

# Location of the Crowbar checkout we are building from.
[[ $CROWBAR_DIR ]] ||CROWBAR_DIR="${0%/*}"

export CROWBAR_DIR
# Location of the Sledgehammer source tree.  Only used if we cannot
# find Sledgehammer in $SLEDGEHAMMER_PXE_DIR above.
[[ $SLEDGEHAMMER_DIR ]] || SLEDGEHAMMER_DIR="${CROWBAR_DIR}/../sledgehammer"

# Command to run to clean out the tree before starting the build.
# By default we want to be relatively pristine.
[[ $VCS_CLEAN_CMD ]] || VCS_CLEAN_CMD='git clean -f -x -d'

# Arrays holding the additional pkgs and gems populate Crowbar with.
REPOS=()

declare -A CD_POOL STAGED_POOL INSTALLED_PKGS
# Some helper functions

# Print a message to stderr and exit.  cleanup will be called.
die() { echo "$(date '+%F %T %z'): $*" >&2; exit 1; }

# Print a message to stderr and keep going.
debug() { echo "$(date '+%F %T %z'): $*" >&2; }

# Clean up any cruft that we might have left behind from the last run.
clean_dirs() {
    local d=''
    for d in "$@"; do
	(   mkdir -p "$d"
	    cd "$d"
	    chmod -R u+w .
	    rm -rf * )
    done
}

# Verify that the passed name is really a branch in the git repo.
branch_exists() { git show-ref --quiet --verify --heads -- "refs/heads/$1"; }

# Run a git command in the crowbar repo.
in_repo() ( cd "$CROWBAR_DIR"; "$@")

# Get the head revision of a git repository.
get_rev() (
    cd "$1"
    if [[ -d .git ]]; then
	git rev-parse HEAD
    else
	echo "Not a Git Repository"
    fi
)

# Run a git command in the build cache, assuming it is a git repository.
in_cache() (
    [[ $CURRENT_CACHE_BRANCH ]] || return
    cd "$CACHE_DIR"
    "$@"
)

# Check to see if something is a barclamp.
is_barclamp() { [[ -f $CROWBAR_DIR/barclamps/$1/crowbar.yml ]]; }

# Build our ISO image.
build_iso() (
    cd "$BUILD_DIR"
    rm -f isolinux/boot.cat
    find -name '.svn' -type d -exec rm -rf '{}' ';' 2>/dev/null >/dev/null
    find . -type f -not -name isolinux.bin -not -name sha1sums -not -path '*/.git/*' | \
	xargs sha1sum -b >sha1sums
    mkdir -p "$ISO_DEST"
	# Save the sha1sums and the build-info files along side the iso.
    cp sha1sums build-info "$ISO_DEST"
    mkisofs -r -V "${VERSION:0:30}" -cache-inodes -J -l -quiet \
	-b isolinux/isolinux.bin -c isolinux/boot.cat \
	-no-emul-boot --boot-load-size 4 -boot-info-table \
	-o "$ISO_DEST/$BUILT_ISO" "$IMAGE_DIR" "$BUILD_DIR"
)

# Have the smoketest framework do its thing with the ISO we just made.
test_iso() {
    [[ -L $HOME/test_framework ]] && rm -f "$HOME/test_framework"
    [[ -d $HOME/test_framework ]] && rm -rf "$HOME/test_framework"
    if [[ $CROWBAR_DIR = /* ]]; then
	ln -sf "$CROWBAR_DIR/test_framework" "$HOME/test_framework"
    else
	ln -sf "$currdir/$CROWBAR_DIR/test_framework" "$HOME/test_framework"
    fi
    (   unset no_proxy http_proxy https_proxy
	"$HOME/test_framework/test_crowbar.sh" "$@" \
	    use-iso "$ISO_DEST/$BUILT_ISO"
    ) || \
	die "$(date '+%F %T %z'): Smoketest of $ISO_DEST/$BUILT_ISO failed."
}

# Get the OS we were asked to stage Crowbar on to.  Assume it is Ubuntu 10.10
# unless we specify otherwise.
OS_TO_STAGE="${1-ubuntu-10.10}"
shift

# Make sure that we actually know how to build the ISO we were asked to
# build.  If we do not, print a helpful error message.
if ! [[ $OS_TO_STAGE && -d $CROWBAR_DIR/$OS_TO_STAGE-extra && \
    -f $CROWBAR_DIR/$OS_TO_STAGE-extra/build_lib.sh ]]; then
    cat <<EOF
You must pass the name of the operating system you want to stage Crowbar
on to.  Valid choices are:
EOF
    cd "$CROWBAR_DIR"
    for d in *-extra; do
	[[ -d $d && -f $d/build_lib.sh ]] || continue
	echo "    ${d%-extra}"
    done
    exit 1
fi

# Source OS specific build knowledge.  This includes:
# Parameters that build_crowbar.sh needs to know:
# OS = the distribution we are staging on to, such as redhat or ubuntu.
# OS_VERSION = the version of the distribution we are staging on to.
#              For redhat, it would be somethibng like 5.6
# OS_TOKEN = Defaults to "$OS-$OS_VERSION"
# ISO = the name of the install ISO image we are going to stage Crowbar on to.
# PKG_TYPE = The package type we should look for in crowbar.yml for OS
#            specific packages.
# PKG_ALLOWED_ARCHES = The allowed package arches we will look to stage.
#                      These are usually whatever the OS equivalent of
#                      amd64 and all are.
# CHROOT_GEMDIR = The location that the OS keeps its gem cache in.
# CHROOT_PKGDIR = The location that the OS keeps its package cache in.
# Functions that build_crowbar needs to call:
# find_cd_pool(): This function should echo the current location of the
#   package pool on $ISO when it is mounted on $IMAGE_DIR.
# fetch_os_iso(): This function should try to fetch the OS iso from
#   a well-known location on the Internet, and die if it cannot.
# chroot_update(): This function should ask the chroot to update its
#   package metadata.
# chroot_install(): This function should try to install the packages passed
#   to it as args.
# chroot_fetch(): This function should ask to download (but not install)
#   the packages passed to it as arguments and all their dependencies.
# add_repos(): This function should try to add repositories passed to it
#   as args to the chroot environment.
# is_pkg(): This function should check to see if the string passed to it
#   is a valid package filename.  The file need not actually exist.
# pkg_name(): This function should extract the package name and arch as the
#   package management system will see them from the file passed as an argument,
#   and echo that information in $name-$arch format.
# __make_chroot(): This function should handle all of the OS-specific actions
#   needed to set up the package-fetching chroot environment.
# pkg_cmp(): This function should check to see that both of the files passed to
#   it refer to the same package name.  If it does, this function should return
#   0 if the first package is a higher revision than the second one, and 1 if
#   the first package is the same or lower revision.  It should die if the
#   package names are not the same.
# final_build_fixups(): This function should take wahtever steps are needed
#   to make the default OS install process also ensure that the Crowbar bits
#   are properly staged and to completly automate the admin node install
#   process, either as an install from CD or an install via PXE.  This
#   usually entails modifying initrd files, adding kickstarts/install seeds,
#   modifying boot config files, and so on.
. "$CROWBAR_DIR/$OS_TO_STAGE-extra/build_lib.sh"

{
    # Make sure only one instance of the ISO build runs at a time.
    # Otherwise you can easily end up with a corrupted image.
    debug "Acquiring the build lock."
    flock 65
    # Figure out what our current branch is, in case we need to merge
    # other branches in to the iso to create our build.
    CURRENT_BRANCH="$(in_repo git symbolic-ref HEAD)" || \
	die "Not on a branch we can build from!"
    CURRENT_BRANCH=${CURRENT_BRANCH##*/}
    [[ $CURRENT_BRANCH ]] || die "Not on a branch we can merge from!"

    # Check and see if our local build repository is a git repo. If it is,
    # we may need to do the same sort of merging in it that we might do in the
    # Crowbar repository.
    if [[ -d $CACHE_DIR/.git ]]; then
	for br in "$CURRENT_BRANCH" master ''; do
	    [[ $br ]] || die "Cannot find $CURRENT_BRANCH or master in $CACHE_DIR"
	    (cd "$CACHE_DIR"; branch_exists "$br") || continue
	    CURRENT_CACHE_BRANCH="$br"
	    break
	done
	# If there are packages that have not been comitted, save them
	# in a stash before continuing.  We do this on the assumption that
	# these packages were added manually for testing purposes, or were
	# added in an earlier update-cache operation, but that the user has
	# not gotten around to comitting yet.
	if [[ ! $(in_cache git status) =~ working\ directory\ clean ]]; then
	    CACHE_THROWAWAY_STASH=$(in_cache git stash create)
	    in_cache git checkout -f .
	fi
    fi

    # Parse our options.
    while [[ $1 ]]; do
	case $1 in
	    # Merge a list of branches into a throwaway branch with the
	    # current branch as a baseline before starting the rest of the
	    # build process.  This makes it easier to spin up iso images
	    # with local changes without having to manually merge those
	    # changes in with any other branches of interest first.
	    # This code takes heavy advantage of the lightweight nature of
	    # git branches and takes care to leave uncomitted changes in place.
	    -m|--merge)
		shift
		# Loop through the rest of the arguments, as long as they
		# do not start with a -.
		while [[ $1 && ! ( $1 = -* ) ]]; do
		    # Check to make sure that this argument refers to a branch
		    # in the crowbar git tree.  Die if it does not.
		    in_repo branch_exists "$1" || die "$1 is not a git branch!"
		    # If we have not already created a throwaway branch to
		    # merge these branches into, do so now. If we have
		    # uncomitted changes that need to be stashed, do so here.
		    if [[ ! $THROWAWAY_BRANCH ]]; then
			THROWAWAY_BRANCH="build-throwaway-$$-$RANDOM"
			REPO_PWD="$PWD"
			if [[ ! $(in_repo git status) =~ working\ directory\ clean ]]; then
			    THROWAWAY_STASH=$(in_repo git stash create)
			    in_repo git checkout -f .
			fi
			in_repo git checkout -b "$THROWAWAY_BRANCH"
		    fi
		    # Merge the requested branch into the throwaway branch.
		    # Die if the merge failed -- there must have been a
		    # conflict, and the user needs to fix it up.
		    in_repo git merge "$1" || \
			die "Merge of $1 failed, fix things up and continue"
		    # If there is n identically named branch in the build cache,
		    # merge it into a throwaway branch of the build cache
		    # along with the current branch in the build cache.
		    # This makes it easier to include and manage packages that
		    # are branch-specific, but that do not need to be included
		    # in every build.
		    if in_cache branch_exists "$1"; then
			if [[ ! $CACHE_THROWAWAY_BRANCH ]]; then
			    CACHE_THROWAWAY_BRANCH=${THROWAWAY_BRANCH/build/cache}
			    in_cache git checkout -b "$CACHE_THROWAWAY_BRANCH"
			fi
			in_cache git merge "$1" || \
			    die "Could not merge build cache branch $1"
		    fi
		    shift
		done
		;;
	    # Force an update of the cache
	    update-cache|--update-cache) shift; need_update=true;;
	    # Pull in additional barclamps.
	    --barclamps)
		shift
		while [[ $1 && $1 != -* ]]; do
		    BARCLAMPS+=("$1")
		    shift
		done;;
	    --test)
		NEED_TEST=true
		test_params=(run-test)
		shift
		while [[ $1 && $1 != -* ]]; do
		    test_params+=("$1")
		    shift
		done;;
	    --shrink)
		type shrink_iso >&/dev/null || \
		    die "The build system does not know how to shrink $OS_TO_STAGE"
		SHRINK_ISO=true
		shift;;
	    --generate-minimal-install)
		type generate_minimal_install &>/dev/null || \
		    die "The build system does not know how to generate a minimal install list for $OS_TO_STAGE!"
		GENERATE_MINIMAL_INSTALL=true
		SHRINK_ISO=true
		shift;;
	    *) 	die "Unknown command line parameter $1";;
	esac
    done

    # If we stached changes to the crowbar repo, apply them now.
    [[ $THROWAWAY_STASH ]] && in_repo git stash apply "$THROWAWAY_STASH"
    # Ditto for the build cache.
    [[ $CACHE_THROWAWAY_STASH ]] && \
	in_cache git stash apply "$CACHE_THROWAWAY_STASH"

    # Finalize where we expect to find our caches and out chroot.
    # If they were set in one of the conf files, don't touch them.

    # The directory we perform a minimal install into if we need
    # to refresh our gem or pkg caches
    [[ $CHROOT ]] || CHROOT="$CACHE_DIR/$OS_TOKEN/chroot"

    # Make sure that the $OS_TOKEN directory exist.
    mkdir -p "$CACHE_DIR/$OS_TOKEN"

    # The directory we will stage the build into.
    [[ $BUILD_DIR ]] || \
	BUILD_DIR="$CACHE_DIR/$OS_TOKEN/build"
    # The directory that we will mount the OS .ISO on .
    [[ $IMAGE_DIR ]] || \
	IMAGE_DIR="$CACHE_DIR/$OS_TOKEN/image"

    # Directory where we will look for our package lists
    [[ $PACKAGE_LISTS ]] || PACKAGE_LISTS="$BUILD_DIR/extra/packages"

    # Hashes to hold our "interesting" information.
    # Key = barclamp name
    # Value = whatever interesting thing we are looking for.
    declare -A BC_DEPS BC_GROUPS BC_PKGS BC_EXTRA_FILES BC_OS_DEPS BC_GEMS
    declare -A BC_REPOS BC_PPAS BC_RAW_PKGS BC_BUILD_PKGS BC_QUERY_STRINGS

    # Query strings to pull info we are interested out of crowbar.yml
    BC_QUERY_STRINGS["deps"]="barclamp requires"
    BC_QUERY_STRINGS["groups"]="barclamp member"
    BC_QUERY_STRINGS["pkgs"]="$PKG_TYPE pkgs"
    BC_QUERY_STRINGS["extra_files"]="extra_files"
    BC_QUERY_STRINGS["os_support"]="barclamp os_support"
    BC_QUERY_STRINGS["gems"]="gems pkgs"
    BC_QUERY_STRINGS["repos"]="$PKG_TYPE repos"
    BC_QUERY_STRINGS["ppas"]="$PKG_TYPE ppas"
    BC_QUERY_STRINGS["build_pkgs"]="$PKG_TYPE build_pkgs"
    BC_QUERY_STRINGS["raw_pkgs"]="$PKG_TYPE raw_pkgs"

    # Pull in interesting information from all our barclamps
    for bc in $CROWBAR_DIR/barclamps/*; do
	[[ -d $bc ]] || continue
	bc=${bc##*/}
	debug "Reading metadata for $bc barclamp."
	is_barclamp "$bc" || die "$bc is not a barclamp!"
	yml_file="$CROWBAR_DIR/barclamps/$bc/crowbar.yml"
	[[ $bc = crowbar ]] || BC_DEPS["$bc"]+="crowbar "
	for query in "${!BC_QUERY_STRINGS[@]}"; do
	    while read line; do
		[[ $line = nil ]] && continue
		case $query in
		    deps) is_in "$line "${BC_DEPS["$bc"]} || \
			BC_DEPS["$bc"]+="$line ";;
		    groups) is_in "$line" ${BC_GROUPS["$bc"]} ||
			BC_GROUPS["$line"]+="$bc ";;
		    pkgs) is_in "$line" ${BC_PKGS["$bc"]} || \
			BC_PKGS["$bc"]+="$line ";;
		    extra_files) BC_EXTRA_FILES["$bc"]+="$line\n";;
		    os_support) BC_OS_SUPPORT["$bc"]+="$line ";;
		    gems) BC_GEMS["$bc"]+="$line ";;
		    repos) BC_REPOS["$bc"]+="$line\n";;
		    ppas) [[ $PKG_TYPE = debs ]] || \
			die "Cannot declare a PPA for $PKG_TYPE!"
			BC_REPOS["$bc"]+="ppa $line\n";;
		    build_pkgs) BC_BUILD_PKGS["$bc"]+="$line ";;
		    raw_pkgs) BC_RAW_PKGS["$bc"]+="$line ";;
		    *) die "Cannot handle query for $query."
		esac
	    done < <("$CROWBAR_DIR/parse_yml.rb" \
		"$yml_file" \
		${BC_QUERY_STRINGS["$query"]} 2>/dev/null)
	done
    done

    debug "Analyzing barclamp group membership"
    # If any barclamps need group expansion, do it.
    for bc in "${!BC_DEPS[@]}"; do
	newdeps=''
	for dep in ${BC_DEPS["$bc"]}; do
	    if [[ $dep = @* ]]; then
		[[ ${BC_GROUPS["${dep#@}"]} ]] || \
		    die "$bc depends on group ${dep#@}, but that group does not exist!"
		for d in ${BC_GROUPS["${dep#@}"]}; do
		    is_barclamp "$d" || \
			die "$bc depends on barclamp $d from group ${dep#@}, but $d does not exist!"
		    newdeps+="$d "
		done
	    else
		is_barclamp "$dep" || \
		    die "$bc depends on barclamp $dep, but $dep is not a barclamp!"
		newdeps+="$dep "
	    fi
	done
	BC_DEPS["$bc"]="$newdeps"
    done

    # Proxy Variables
    [[ $USE_PROXY ]] || USE_PROXY=0
    [[ $PROXY_HOST ]] || PROXY_HOST=""
    [[ $PROXY_PORT ]] || PROXY_PORT=""
    [[ $PROXY_USER ]] || PROXY_USER=""
    [[ $PROXY_ESC_USER ]] || PROXY_ESC_USER=""
    [[ $PROXY_PASSWORD ]] || PROXY_PASSWORD=""

    # Version for ISO
    [[ $VERSION ]] || VERSION="$(cd "$CROWBAR_DIR"; git describe --long --tags)-dev"

    # Name of the built iso we will build
    [[ $BUILT_ISO ]] || BUILT_ISO="crowbar-${VERSION}.iso"

    # If we were not passed a list of barclamps to include,
    # pull in all of the ones declared as submodules.
    [[ $BARCLAMPS ]] || BARCLAMPS=($(cd "$CROWBAR_DIR"
	    while read sha submod branch; do
		[[ $submod = barclamps/* ]] || continue
		[[ -f $submod/crowbar.yml ]] || \
		    echo "Cannot find crowbar.yml for $submod, exiting."
		echo "${submod##*/}"
	    done < <(git submodule status))
	)

    # Group-expand barclamps if needed, and unset groups after they are expanded.
    for i in "${!BARCLAMPS[@]}"; do
	bc="${BARCLAMPS[$i]}"
	if [[ $bc = @* ]]; then
	    [[ ${BC_GROUPS["${bc#@}"]} ]] || \
		die "No such group ${bc#@}!"
	    BARCLAMPS+=(${BC_GROUPS["${bc#@}"]})
	    unset BARCLAMPS[$i]
	else
	    is_barclamp "$bc" || die "$bc is not a barclamp!"
	fi
    done
    BARCLAMPS=("${BARCLAMPS[@]//@*}")

    # Pull in dependencies for the barclamps.
    # Everything depends on the crowbar barclamp, so include it first.
    new_barclamps=("crowbar")
    while [[ t = t ]]; do
	for bc in "${BARCLAMPS[@]}"; do
	    for dep in ${BC_DEPS["$bc"]}; do
		is_in "$dep" "${new_barclamps[@]}" && continue
		new_barclamps+=("$dep")
	    done
	    is_in "$bc" "${new_barclamps[@]}" || new_barclamps+=("$bc")
	done
	[[ ${BARCLAMPS[*]} = ${new_barclamps[*]} ]] && break
	BARCLAMPS=("${new_barclamps[@]}")
    done

    # Make any directories we don't already have
    for d in "$ISO_LIBRARY" "$ISO_DEST" "$IMAGE_DIR" "$BUILD_DIR" \
	"$SLEDGEHAMMER_PXE_DIR" "$CHROOT"; do
	mkdir -p "$d"
    done

    debug "Checking for Sledgehammer."
    # Make sure Sledgehammer has already been built and pre-staged.
    if ! [[ -f $SLEDGEHAMMER_DIR/bin/sledgehammer-tftpboot.tar.gz || \
	-f $SLEDGEHAMMER_PXE_DIR/initrd0.img ]]; then
	echo "Slegehammer TFTP image missing!"
	echo "Please build Sledgehammer from $SLEDGEHAMMER_DIR before building Crowbar."
	exit 1
    fi

    # Fetch the OS ISO if we need to.
    [[ -f $ISO_LIBRARY/$ISO ]] || fetch_os_iso

    # Start with a clean slate.
    clean_dirs "$IMAGE_DIR" "$BUILD_DIR"

    debug "Cleaning up any VCS cruft."
    # Clean up any cruft that the editor may have left behind.
    (cd "$CROWBAR_DIR"; $VCS_CLEAN_CMD)

    # Make additional directories we will need.
    for d in discovery extra; do
	mkdir -p "$BUILD_DIR/$d"
    done

    # Mount our ISO for the build process.
    debug "Mounting $ISO"
    sudo mount -t iso9660 -o loop "$ISO_LIBRARY/$ISO" "$IMAGE_DIR" || \
	die "Could not mount $ISO"
    debug "Indexing CD package pool."
    index_cd_pool

    # Copy over the Crowbar bits and their prerequisites
    cp -r "$CROWBAR_DIR/extra"/* "$BUILD_DIR/extra"
    cp -r "$CROWBAR_DIR/$OS_TOKEN-extra"/* "$BUILD_DIR/extra"
    cp -r "$CROWBAR_DIR/change-image"/* "$BUILD_DIR"
    mkdir -p "$BUILD_DIR/dell/barclamps"
    echo "build-timestamp: $(date '+%F %T %z')" > "$BUILD_DIR/build-info"
    echo "build-os: $OS_TOKEN" >>"$BUILD_DIR/build-info"
    echo "build-os-iso: $ISO" >>"$BUILD_DIR/build-info"
    echo "crowbar: $(get_rev "$CROWBAR_DIR")" >>"$BUILD_DIR/build-info"
    for bc in "${BARCLAMPS[@]}"; do
	is_barclamp "$bc" || die "Cannot find barclamp $bc!"
	debug "Staging $bc barclamp."
	for cache in pkg gem raw_pkg file; do
	    checker="barclamp_${cache}_cache_needs_update"
	    updater="update_barclamp_${cache}_cache"
	    [[ $(type $checker) = "$checker is a function"* ]] || \
		die "Asked to check $cache cache, but no checker function!"
	    [[ $(type $updater) = "$updater is a function"* ]] || \
		die "Might need to update $cache cache, but no updater!"
	    if $checker "$bc" || [[ $need_update = true ]]; then
		debug "Updating $cache cache for $bc"
		[[ $cache =~ ^(pkg|gem)$ ]] && make_chroot
		$updater "$bc"
	    fi
	done
	if [[ $NEED_TEST = true && -d $CROWBAR_DIR/barclamps/$bc/smoketest ]]
	then
	    is_in "$bc" "${test_params[@]}" || test_params+=("$bc")
	fi
	cp -r "$CROWBAR_DIR/barclamps/$bc" "$BUILD_DIR/dell/barclamps"
	mkdir -p "$BUILD_DIR/extra/pkgs/"
	stage_pkgs "$CACHE_DIR/barclamps/$bc/$OS_TOKEN/pkgs" \
	    "$BUILD_DIR/extra/pkgs"
	cp -r "$CACHE_DIR/barclamps/$bc/files" \
	    "$CACHE_DIR/barclamps/$bc/gems" \
	    "$BUILD_DIR/extra"
	echo "barclamps/$bc: $(get_rev "$CROWBAR_DIR/barclamps/$bc")" >> "$BUILD_DIR/build-info"
    done

    (cd "$BUILD_DIR"
	find extra/pkgs extra/gems extra/files -type f -print | \
	    sort >> "build-info")
    # Make sure we still provide the legacy ami location
    (cd "$BUILD_DIR"; ln -sf extra/files/ami)
    # Store off the version
    echo "$VERSION" >> "$BUILD_DIR/dell/Version"

    # Custom start-up in place
    if [ -f "$CROWBAR_DIR/crowbar.json" ] ; then
      mkdir -p "$BUILD_DIR/extra/config"
      cp "$CROWBAR_DIR/crowbar.json" "$BUILD_DIR/extra/config"
    fi

    final_build_fixups

    # Copy over the bits that Sledgehammer will look for.
    debug "Copying over Sledgehammer bits"
    # If we need to copy over a new Sledgehammer image, do so.
    if [[ $SLEDGEHAMMER_DIR/bin/sledgehammer-tftpboot.tar.gz -nt \
	$SLEDGEHAMMER_PXE_DIR/initrd0.img ]]; then
	(   cd $SLEDGEHAMMER_PXE_DIR
	    debug "Extracting new Sledgehammer TFTP boot image"
	    rm -rf .
	    cd ..
	    tar xzf "$SLEDGEHAMMER_DIR/bin/sledgehammer-tftpboot.tar.gz"
	    rm -f "$SLEDGEHAMMER_DIR/bin/sledgehammer-tftpboot.tar.gz"
	)
    fi
    cp -a "$SLEDGEHAMMER_PXE_DIR"/* "$BUILD_DIR/discovery"

    # Make our image
    debug "Creating new ISO"
    # Find files and directories that mkisofs will complain about.
    # Do just top-level overlapping directories for now.
    for d in $(cat <(cd "$BUILD_DIR"; find -maxdepth 1 -type d) \
	           <(cd "$IMAGE_DIR"; find -maxdepth 1 -type d) | \
	           sort |uniq -d); do
	[[ $d = . ]] && continue
	d=${d#./}
	# Copy contents of the found directories into $BUILD_DIR, taking care
	# to not clobber existing files.
	mkdir -p "$BUILD_DIR/$d"
	chmod u+wr "$BUILD_DIR/$d"
	# We could also use cp -n, but rhel5 and centos5 do not understand it.
	rsync -rl --ignore-existing --inplace "$IMAGE_DIR/$d/." "$BUILD_DIR/$d/."
	chmod -R u+wr "$BUILD_DIR/$d"
	# Bind mount an empty directory on the $IMAGE_DIR instance.
	sudo mount -t tmpfs -o size=1K tmpfs "$IMAGE_DIR/$d"
    done
    mkdir -p "$BUILD_DIR/isolinux"
    chmod u+wr "$BUILD_DIR/isolinux"
    rsync -rl --ignore-existing --inplace \
	"$IMAGE_DIR/isolinux/." "$BUILD_DIR/isolinux/."
    chmod -R u+wr "$BUILD_DIR/isolinux"
    sudo mount -t tmpfs -o size=1K tmpfs "$IMAGE_DIR/isolinux"

    [[ $SHRINK_ISO && ! $GENERATE_MINIMAL_ISO ]] && shrink_iso
    # Make a file list and a link list.
    ( cd $BUILD_DIR
      find . -type f | \
	  sort > crowbar_files.list
      find . -type l | \
	  xargs ls -ld | \
	  awk '{ print $8 " " $10 }' | \
	  sort > crowbar_links.list
    )
    ( cd $IMAGE_DIR
      find . -type f | \
	  sort >> $BUILD_DIR/crowbar_files.list
      find . -type l | \
	  xargs ls -ld | \
	  awk '{ print $8 " " $10 }' | \
	  sort >> $BUILD_DIR/crowbar_links.list
    )

    # Make an ISO
    build_iso || die "There was a problem building our ISO."
    if [[ $GENERATE_MINIMAL_INSTALL = true ]]; then
	if [[ ! -f "$CROWBAR_DIR/$OS_TOKEN-extra/minimal-install" ]]; then
	    if [[ ! -f "$HOME/admin-installed.list" ]]; then
		test_params=(run-test scratch admin-only)
		    test_iso run-test scratch admin-only
	    fi
	    [[ ! -f "$HOME/crowbar-installed.$OS_TOKEN.list" ]] || \
		die "Asked to shrink build for $OS_TOKEN, but package list missing!"
	    mv "$HOME/admin-installed.list" \
		"$CROWBAR_DIR/$OS_TOKEN-extra/minimal-install"
	fi

	build_iso
    fi
    if [[ $NEED_TEST = true ]]; then test_iso "${test_params[@]}"; fi

    echo "$(date '+%F %T %z'): Finshed. Image at $ISO_DEST/$BUILT_ISO"
} 65> /tmp/.build_crowbar.lock


Update docker-build.sh

Update docker-build.sh

remote: Support for password authentication was removed on August 13, 2021.
remote: Please see https://docs.github.com/en/get-started/getting-started-with-git/about-remote-repositories#cloning-with-https-urls for information on currently recommended modes of authentication.
fatal: Authentication failed for 'https://github.com/BrentDorsey/pipeline.git/'


Add examples of find command.

Add examples of find command.

# Description: find - Search for files in a directory hierarchy

# Notes
# 1. This command can be used to search for files by name, owner, group, type, permissions, date, and other criteria.
# 2. The general syntax is: find [Where to look] [Criteria] [What to do]
#   - Where to look: This defaults to . ie the current working directory.
#   - Criteria: This defaults to none ie select all files.
#   - What to do: This is also known as the find action. This defaults to ‑print ie display the names of found files to
#     standard output.
# 3. The search is recursive and it will search all subdirectories.

# Common Examples

# Examples with details
find                                    # Display the pathnames of all files in the current directory recursively.
find .                                  # Same as above. Where to look defaults to the current directory.
find -print                             # Same as above. What to do defaults to -print.
find . -print                           # Same as above. Criteria defaults to none.
find / -name foo 2>/dev/null            # Not running this command as root will show error message for each directory.
find /tmp /var/tmp . $HOME -name foo    # Specify multiple 'Where to look' places to search.
find . -name foo\*bar                   # Use shell-style wildcards in the ‑name search argument.
find / -type f -mtime -7                # Find any regular files (i.e., not directories or other special files) with
                                        # the criteria '‑type f', and only those modified seven or fewer days ago with
                                        # the criteria '‑mtime ‑7'.
find / -mmin -10                        # Find files modified less than 10 minutes ago.
find . -perm -o=w                       # Find files that are writable by “others”. This will include symlinks, which
                                        #  should be writable by all.
find . -name Hello.java                 # Finds Hello.java in the current directory resursively. Use the switch -name
                                        # is to perform a name search for the filename Hello.java.

# Cool Tricks

# Find all files with name 'CVS' and delete it recursively using xargs.
find . -name 'CVS' | xargs rm -rf

# Same as above.
# 1. Run the rm command once per file.
# 2. Not as efficient as the above command
# 3. This one is safer if file names contain spaces or newlines.
find . -name core -exec /bin/rm -f '{}' \;

# Same as above.
# 1. Run the rm command once per file.
# 2. Not as efficient as the above command
# 3. This one is safer if file names contain spaces or newlines.
find . -name core -delete

# TODO
# 1. Explore man pages.

error: pathspec 'src/commands/find.sh' did not match any file(s) known to git


Fix git-add -p alias

Fix git-add -p alias

remote: Support for password authentication was removed on August 13, 2021.
remote: Please see https://docs.github.com/en/get-started/getting-started-with-git/about-remote-repositories#cloning-with-https-urls for information on currently recommended modes of authentication.
fatal: Authentication failed for 'https://github.com/looselyrigorous/dotfiles.git/'


add check_code script

add check_code script

#!/bin/sh

# http://mywiki.wooledge.org/Bashism
# https://wiki.ubuntu.com/DashAsBinSh

checkbashisms -f bin/*
checkbashisms -f Makefile eepm.spec

error: pathspec 'check_code.sh' did not match any file(s) known to git


Outage monitor.

Outage monitor.

#!/bin/bash
NOTIFYEMAIL=mike@mzoo.org
SMSEMAIL=<2016794168@pm.sprint.com>
SENDEREMAIL=alert@localhost
SERVER=http://infiniteglitchxxx.net/
PAUSE=5
FAILED=0
DEBUG=1

while true
do
/usr/bin/curl -sSf $SERVER > /dev/null 2>&1
CS=$?
# For debugging purposes
if [ $DEBUG -eq 1 ]
then
    echo "STATUS = $CS"
    echo "FAILED = $FAILED"
    if [ $CS -ne 0 ]
    then
        echo "$SERVER is down"

    elif [ $CS -eq 0 ]
    then
        echo "$SERVER is up"
    fi
fi

# If the server is down and no alert is sent - alert
if [ $CS -ne 0 ] && [ $FAILED -eq 0 ]
then
    FAILED=1
    if [ $DEBUG -eq 1 ]
    then
        echo "$SERVER failed"
    fi
    if [ $DEBUG = 0 ]
    then
        echo "$SERVER went down $(date)" | /usr/bin/mailx -s "$SERVER went down" -r "$SENDEREMAIL" "$SMSEMAIL"
        echo "$SERVER went down $(date)" | /usr/bin/mailx -s "$SERVER went down" -r "$SENDEREMAIL" "$NOTIFYEMAIL"
    fi

# If the server is back up and no alert is sent - alert
elif [ $CS -eq 0 ] && [ $FAILED -eq 1 ]
then
    FAILED=0
    if [ $DEBUG -eq 1 ]
    then
        echo "$SERVER is back up"
    fi
    if [ $DEBUG = 0 ]
    then
        echo "$SERVER is back up $(date)" | /usr/bin/mailx -s "$SERVER is back up again" -r "$SENDEREMAIL" "$SMSEMAIL"
        echo "$SERVER is back up $(date)" | /usr/bin/mailx -s "$SERVER is back up again" -r "$SENDEREMAIL" "$NOTIFYEMAIL"
    fi
fi
sleep $PAUSE
done

error: pathspec 'outage.sh' did not match any file(s) known to git


Added launch-rds.sh

Added launch-rds.sh

#!/bin/bash

mapfile -t dbInstanceARR < <(aws rds describe-db-instances --output json | grep "\"DBInstanceIdentifier" | sed "s/[\"\:\, ]//g" | sed "s/DBInstanceIdentifier//g" )

if [ ${#dbInstanceARR[@]} -gt 0 ]
   then
   echo "Deleting existing RDS database-instances"
   LENGTH=${#dbInstanceARR[@]}

      for (( i=0; i<${LENGTH}; i++));
      do
      if [ ${dbInstanceARR[i]} == "lsl-db" ]
     then
      echo "db exists"
     else
     aws rds create-db-instance --db-instance-identifier lsl-db --db-instance-class db.t1.micro --engine MySQL --master-username controller --master-user-password ilovebunnies --allocated-storage 5
      fi
     done
fi

error: pathspec 'launch-rds.sh' did not match any file(s) known to git


Create make-ovpn.sh

Create make-ovpn.sh

remote: Support for password authentication was removed on August 13, 2021.
remote: Please see https://docs.github.com/en/get-started/getting-started-with-git/about-remote-repositories#cloning-with-https-urls for information on currently recommended modes of authentication.
fatal: Authentication failed for 'https://github.com/Smiril/raspBian-scripts.git/'


add make_proj.sh, close #2

add make_proj.sh, close #2

#!/bin/sh

if [ "$#" -ne 1 ]
then
  echo "Usage: ./make_proj.sh <project name>"
  exit 1
fi

PROJ=$1

rm -rf .git
find . -type f ! -iname "make_proj.sh" -exec sed -i "s/pystub/$PROJ/g" {} \;
mv pystub $PROJ
git init
git add .
git reset make_proj.sh
git commit -am "initial commit"

error: pathspec 'make_proj.sh' did not match any file(s) known to git


luiza

luiza

remote: Support for password authentication was removed on August 13, 2021.
remote: Please see https://docs.github.com/en/get-started/getting-started-with-git/about-remote-repositories#cloning-with-https-urls for information on currently recommended modes of authentication.
fatal: Authentication failed for 'https://github.com/healeyj/task-6-be-presentable.git/'


Added test to verify config issue II

Added test to verify config issue II

#!/usr/bin/env bash

# shellcheck disable=SC2034
test_description="Test init command for success"

# shellcheck disable=SC1091
source ./setup.sh

test_expect_success "Should not dump config file" "
  cd $OUTPUTDIR &&
  zephirc --version -Wunused-variable 2>&1 >/dev/null &&
  test ! -f ./config.json
"

test_expect_success "Should generate a project and dump config file" "
  cd $OUTPUTDIR &&
  zephirc init success &&
  test -d ./success &&
  test -d ./success/success &&
  test -d ./success/ext &&
  test -f ./success/config.json
"

# Cleanup
[ -d "$OUTPUTDIR/config.json" ] && rm -rf "$OUTPUTDIR/config.json"
[ -d "$OUTPUTDIR/success" ] && rm -rf "$OUTPUTDIR/success"

test_done

error: pathspec 'unit-tests/sharness/t0003-init-success.sh' did not match any file(s) known to git


Added dos2unix and openconnect for Mac OS X

remote: Support for password authentication was removed on August 13, 2021.
remote: Please see https://docs.github.com/en/get-started/getting-started-with-git/about-remote-repositories#cloning-with-https-urls for information on currently recommended modes of authentication.
fatal: Authentication failed for 'https://github.com/kirktrue/kpm.git/'


misc updates

misc updates

remote: Support for password authentication was removed on August 13, 2021.
remote: Please see https://docs.github.com/en/get-started/getting-started-with-git/about-remote-repositories#cloning-with-https-urls for information on currently recommended modes of authentication.
fatal: Authentication failed for 'https://github.com/cdsi/theoden.git/'


starts to implement lower-case masking tag

#!/bin/bash
if [[ ($1 == "-h") || ($1 == "--help") ]]
then
    echo "Usage:"
    echo "$0 [-h,-help] [-i,--input-fasta] [-d,--database] <-e,--evalue>
    <-nd,--num_descriptions> <-na,--num_alignments> <-o --output-filename>
    <-cl,--clean> <-nrh,--num-report-hits> <-nthr, --num-threads>
    <-bks, --backward-search>

    where:
        -h, --help  show this help text
	-i, --input-fasta the input FASTA file
	-d, --database database FASTA file
	-e, --evalue the E value
	-nd, --num_descriptions
	-na, --num_alignments
	-o, --output-filename
	-cl, --clean
	-nrh, --num-report-hits
	-nthr, --num-threads
	-bks, --backward-search "
    exit 0
fi

# parsing input arguments
while [[ $# > 1 ]]
do
    key="$1"

    case $key in
	-i|--input-fasta)
	    INPUT_FASTA="$2"
	    shift
	    ;;

	-d|--database)
	    DATABASE="$2"
	    shift
	    ;;

	-e|--evalue)
	    EVALUE_SET="$2"
	    shift
	    ;;

	-nd|--num_descriptions)
	    NUM_DESCRIPTIONS_SET="$2"
	    shift
	    ;;

	-na|--num_alignments)
	    NUM_ALIGNMENTS_SET="$2"
	    shift
	    ;;

	-o|--output-filename)
	    OUTPUTREPORT="$2"
	    shift
	    ;;

	-cl|--clean)
	    DEBUG_CLEAN="$2"
	    shift
	    ;;

	-bks|--backward-search)
	    BACK="$2"
	    shift
	    ;;

	-nrh|--num-report-hits)
	    NUM_REPORT_HITS_SET="$2"
	    shift
	    ;;

	-nthr|--num-threads)
	    NUM_THREDS_SET="$2"
	    shift
	    ;;

    esac
    shift
done

# processing input arguments
DATABASE_NAME=${DATABASE%.*}
EVALUE=${EVALUE_SET:-0.001}
NUM_THREDS=${NUM_THREDS_SET:-4}
NUM_REPORT_HITS=${NUM_REPORT_HITS_SET:-2}
NUM_DESCRIPTIONS=${NUM_DESCRIPTIONS_SET:-9}
NUM_ALIGNMENTS=${NUM_ALIGNMENTS_SET:-9}
OUTPUTREPORT="${OUTPUTREPORT%.*:-"report"}-MaxHits_$NUM_REPORT_HITS-input_${INPUT_FASTA%.*}-db_$DATABASE_NAME-evalue_$EVALUE.txt"

############################################
# generage database if they do not exist
[ ! -e "$DATABASE_NAME.psq" ] && \
[ ! -e "$DATABASE_NAME.psi" ] && \
[ ! -e "$DATABASE_NAME.psd" ] && \
[ ! -e "$DATABASE_NAME.pog" ] && \
[ ! -e "$DATABASE_NAME.pni" ] && \
[ ! -e "$DATABASE_NAME.pnd" ] && \
[ ! -e "$DATABASE_NAME.pin" ] && \
[ ! -e "$DATABASE_NAME.phr" ] && \
makeblastdb -in $DATABASE -out $DATABASE_NAME -dbtype prot -parse_seqids

# final report
echo "\
# blastp report:
#    Input: $INPUT_FASTA
#    Database: $DATABASE_NAME
#    Evalue: $EVALUE
# Fields: subject id, subject length, alignment length, evalue, bit score, % identity, % percid, annotation
" > $OUTPUTREPORT

printf "%30s %10s %10s %10s %8s %10s %10s     %s\n" \
	"HitID" "Length" "Alignment" "Evalue" "BitScore" "%id" \
	"%percid" "Annotation" >> $OUTPUTREPORT
echo \
---------------------------------------------------------------------------------------------------------------------------------------------------------- \
>> $OUTPUTREPORT

NUM_LINES=$(wc -l < $INPUT_FASTA)

ENTRY=0
COUNT=1
NEW_ENTRY=1
while [[ $COUNT -le $NUM_LINES ]]
do
    THIS_LINE=$(sed "${COUNT}q;d" $INPUT_FASTA)

    if [[ $NEW_ENTRY == 1 ]] # '>' line of a new entry
    then
	echo $THIS_LINE > query.$ENTRY.fasta
	COUNT=$((COUNT+1))
	NEW_ENTRY=0
	continue
    else
	if [[ $THIS_LINE =~ ^\> ]] # '>' line of the next new entry
	then
	    NEW_ENTRY=1
	else                       # sequence lines
	    echo $THIS_LINE >> query.$ENTRY.fasta
	    COUNT=$((COUNT+1))
	    if [[ $COUNT -gt $NUM_LINES ]] # the last line
	    then
		echo
	    else                          # not the last line
	        continue
	    fi
	fi
    fi

    # Blast outputs archive
    blastp -outfmt \
	"7 sseqid slen length evalue bitscore pident" \
	-query query.$ENTRY.fasta -out blastp.$ENTRY.report \
	-db $DATABASE_NAME -evalue $EVALUE -num_threads	$NUM_THREDS -lcase_masking
	#-num_descriptions $NUM_DESCRIPTIONS -num_alignments $NUM_ALIGNMENTS

    # build a fasta file for all hits
    grep "ref|\|gi|" blastp.$ENTRY.report | awk 'BEGIN { FS="|" } { print $2 }' > REF.$ENTRY.fasta
    blastdbcmd -db $DATABASE_NAME -entry_batch REF.$ENTRY.fasta -out blastdbcmd.$ENTRY.fasta

    # align query sequence with all hits
    cat query.$ENTRY.fasta blastdbcmd.$ENTRY.fasta > Match.$ENTRY.fasta
    mafft Match.$ENTRY.fasta > Match.$ENTRY.fasta.aligned

    # calculate PID using percid
    percid Match.$ENTRY.fasta.aligned percid.$ENTRY.percid_matrix

    # output report
    ## header for every entry
    echo "Query $(sed "1q;d" query.$ENTRY.fasta):" >> $OUTPUTREPORT
    echo  >> $OUTPUTREPORT

    ## append percid to report
    LINE_NUM=2
    while read line
    do
	if [[ $line == *"0 hits found"* ]] # Append to Fields description with "% percid"
	then

	    echo "0 hits found" >> $OUTPUTREPORT

	elif [[ $line == \#* ]] # other comment line
	then

	    # not include comments
	    echo

	else # entries that need to be processed
	    PERCENT=$(cat percid.$ENTRY.percid_matrix | head -n$LINE_NUM | tail -n1 | awk '{print $1}')
	    PERCENT=$(echo "scale=2; $PERCENT*100" | bc)
	    PERCENT=$(printf '%*.*f' 0 2 "$PERCENT")

	    # extract annotation
	    ID=$(echo $line | awk 'BEGIN { FS="|" } { print $2 }')
	    ID_header=$(grep $ID $DATABASE)
	    SCINAME=$(echo $ID_header | awk 'BEGIN { FS="|" } { print $5 }')

	    IFS="	"
	    printf "%30s %10s %10s %10s %8s %10s %10s %s\n" ${line[0]} ${line[1]} ${line[2]} ${line[3]} ${line[4]} ${line[5]} $PERCENT $SCINAME >> $OUTPUTREPORT

	    if [[ $LINE_NUM -gt $NUM_REPORT_HITS ]]
	    then
		break
	    fi

	    LINE_NUM=$((LINE_NUM+1))

	fi
    done <blastp.$ENTRY.report

    echo  >> $OUTPUTREPORT

    if [[ $BACK == 1 ]]
    then
	if [[ ! -e "input_as_db.phr" ]]
	then
	    makeblastdb -in $INPUT_FASTA -out input_as_db -dbtype prot -parse_seqids
	fi
	BACKWARD_SEARCH_ENTRY_ID=$(grep "ref|\|gi|" blastp.$ENTRY.report | head -n1 | awk 'BEGIN { FS="|" } { print $2 }')
	ENTRY_STRING="'$BACKWARD_SEARCH_ENTRY_ID'"
	eval "blastdbcmd -db $DATABASE_NAME -dbtype prot -entry $ENTRY_STRING > backward_search.$ENTRY.fasta"

	blastp -outfmt \
	    "7 qseqid sseqid slen length evalue bitscore pident" \
	    -query backward_search.$ENTRY.fasta -out backward_search.$ENTRY.report \
	    -db input_as_db -evalue $EVALUE -num_threads $NUM_THREDS -lcase_masking

	echo "  Backward Search hits:" >> $OUTPUTREPORT
	#grep "ref|\|gi|" backward_search.$ENTRY.report | head -n1 | awk 'BEGIN { FS="|" } { print $2 }' >> $OUTPUTREPORT
	REPORT_STRING=$(grep "^gi|" backward_search.$ENTRY.report | head -n1)

        # align query sequence with all hits
        cat backward_search.$ENTRY.fasta query.$ENTRY.fasta > backward_search.Match.$ENTRY.fasta
        mafft backward_search.Match.$ENTRY.fasta > backward_search.Match.$ENTRY.fasta.aligned

        # calculate PID using percid
        percid backward_search.Match.$ENTRY.fasta.aligned backward_search.percid.$ENTRY.percid_matrix
	PERCENT=$(cat backward_search.percid.$ENTRY.percid_matrix | tail -n1 | awk '{print $1}')
	PERCENT=$(echo "scale=2; $PERCENT*100" | bc)
	PERCENT=$(printf '%*.*f' 0 2 "$PERCENT")

	arr=($REPORT_STRING)
	printf "%30s %10s %10s %10s %8s %10s %10s\n" ${arr[1]} \
	${arr[2]} ${arr[3]} ${arr[4]} \
	${arr[5]} ${arr[6]} $PERCENT >> $OUTPUTREPORT

	echo  >> $OUTPUTREPORT
    fi

    ENTRY=$((ENTRY+1))
done

# cleanup
if [[ $DEBUG_CLEAN == 1 ]]
then
    rm Match.*.fasta Match.*.fasta.aligned \
    percid.*.percid_matrix query.*.fasta REF.*.fasta \
    blastp.*.report blastdbcmd.*.fasta backward_search.*
fi

error: pathspec 'blastp_formater_lcasemasking.sh' did not match any file(s) known to git


Updated how check for APT installation sources is performed


Added speedtest-cli for Linux

Added speedtest-cli for Linux


Fixing tmp issue

Fixing tmp issue

remote: Support for password authentication was removed on August 13, 2021.
remote: Please see https://docs.github.com/en/get-started/getting-started-with-git/about-remote-repositories#cloning-with-https-urls for information on currently recommended modes of authentication.
fatal: Authentication failed for 'https://github.com/psd401/psd-osx-bootstrap.git/'


make correction work; this is a nicer way of handling typos in commands

setopt correct_all

alias man='nocorrect man'
alias mv='nocorrect mv'
alias mysql='nocorrect mysql'
alias mv='nocorrect mv'

error: pathspec 'correction.zsh' did not match any file(s) known to git