install-chef-suse: Add check for installed cloud_admin pattern

#! /bin/bash -e

# This script is called after being installed by the Crowbar RPM from the SUSE
# Cloud ISO. In this context, it is expected that all other required
# repositories (eg. SLES, Updates) are already set up, with the required files
# in place.
#
# For development and testing use, call the script with the '--from-git'
# option. Use the appropriate dev VM and follow the corresponding setup
# instructions.

BARCLAMP_INSTALL_OPTS="--rpm"

if [ "$1" = "--from-git" ]; then
    CROWBAR_FROM_GIT=true
    BARCLAMP_INSTALL_OPTS="--force"
    : ${CROWBAR_FILE:=/root/crowbar/crowbar.json}
    : ${BARCLAMP_SRC:=/root/crowbar/barclamps/}
    sed -i -e '/"nagios":/d' -e '/"ganglia":/d' $CROWBAR_FILE
fi

LOGFILE=/var/log/crowbar/install.log
mkdir -p "`dirname "$LOGFILE"`"

: ${BARCLAMP_SRC:="/opt/dell/barclamps/"}

run_succeeded=

# Infrastructure for nice output/logging
# --------------------------------------

# Copy stdout to fd 3
exec 3>&1
# Create fd 4 for logfile
exec 4>> "$LOGFILE"

if [ -z "$CROWBAR_VERBOSE" ]; then
    # Set fd 1 and 2 to logfile
    exec 1>&4 2>&1
else
    # Set fd 1 and 2 to logfile (and keep stdout too)
    exec 1> >( tee -a /dev/fd/4 ) 2>&1
fi
# Send summary fd to original stdout
exec 6>&3

pipe_stdout_and_logfile () {
    tee -a /dev/fd/3 /dev/fd/4 > /dev/null
}

# Draw a spinner so the user knows something is happening
spinner () {
    local delay=0.75
    local spinstr='/-\|'
    printf "... " >&3
    while [ true ]; do
        local temp=${spinstr#?}
        printf "[%c]" "$spinstr" >&3
        local spinstr=$temp${spinstr%"$temp"}
        sleep $delay
        printf "\b\b\b" >&3
    done
}

kill_spinner () {
    if [ ! -z "$LAST_SPINNER_PID" ]; then
        kill >/dev/null 2>&1 $LAST_SPINNER_PID
        if [ $# -eq 0 ]; then
            printf "\b\b\bdone\n" >&3
        else
            printf "\b\b\b$*\n" >&3
        fi
        unset LAST_SPINNER_PID
    fi
}

kill_spinner_with_failed () {
    kill_spinner "failed"
}

echo_log () {
    echo -e === "$(date '+%F %T %z'): $@" >&4
}

echo_summary () {
    # Also send summary to logfile
    echo_log $@

    kill_spinner

    if [ -z "$CROWBAR_VERBOSE" ]; then
        if [ -t 3 ]; then
            echo -n -e $@ >&3
            # Use disown to lose job control messages (especially the
            # "Completed" message when spinner will be killed)
            spinner & disown
            LAST_SPINNER_PID=$!
        else
            echo -e $@ >&3
        fi
    else
        echo -e === $@ >&3
    fi
}

echo_summary_no_spinner () {
    # Also send summary to logfile
    echo_log $@

    kill_spinner

    if [ -z "$CROWBAR_VERBOSE" ]; then
        echo -e $@ >&3
    else
        echo -e === $@ >&3
    fi
}

die() {
    # Send empty line & error to logfile
    echo >&4
    echo_log "Error: $@"

    kill_spinner_with_failed

    echo >&3
    echo -e "Error: $@" >&3

    res=1
    exit 1
}

exit_handler () {
    if [ -z "$run_succeeded" ]; then
        kill_spinner_with_failed
        cat <<EOF | pipe_stdout_and_logfile

Crowbar installation terminated prematurely.  Please examine the above
output or $LOGFILE for clues as to what went wrong.
You should also check the SUSE Cloud Installation Manual, in
particular the Troubleshooting section.  Note that this script can
safely be re-run multiple times if required.
EOF
    else
        kill_spinner
    fi
}

trap exit_handler EXIT

# Real work starts here
# ---------------------

echo "`date` $0 started with args: $*"

ensure_service_running () {
    service="$1"
    regexp="${2:-running}"
    if service $service status | egrep -q "$regexp"; then
        echo "$service is already running - no need to start."
    else
        service $service start
        sleep 4
    fi
}

# Sanity checks
# -------------

echo_summary "Performing sanity checks"

rootpw=$( getent shadow root | cut -d: -f2 )
case "$rootpw" in
    \*|\!*)
        die "root password is unset or locked.  Chef will rewrite /root/.ssh/authorized_keys; therefore to avoid being accidentally locked out of this admin node, you should first ensure you have a working root password."
        ;;
esac

# It is exceedingly important that 'hostname -f' actually returns an FQDN!
# if it doesn't, add an entry to /etc/hosts, e.g.:
#    192.168.124.10 cb-admin.example.com cb-admin
if ! FQDN=$(hostname -f 2>/dev/null); then
    die "Unable to detect fully-qualified hostname. Aborting."
fi

if ! DOMAIN=$(hostname -d 2>/dev/null); then
    die "Unable to detect DNS domain name. Aborting."
fi

if [ -z "$FQDN" -o -z "$DOMAIN" ]; then
    die "Unable to detect fully-qualified hostname. Aborting."
fi

if ! resolved=$(getent ahosts $FQDN 2>/dev/null); then
    die "Unable to resolve hostname $FQDN via host(1). Please check your configuration of DNS, hostname, and /etc/hosts. Aborting."
fi

IPv4_addr=$( echo "$resolved" | awk '{ if ($1 !~ /:/) { print $1; exit } }' )
IPv6_addr=$( echo "$resolved" | awk '{ if ($1  ~ /:/) { print $1; exit } }' )
if [ -z "$IPv4_addr" -a -z "$IPv6_addr" ]; then
    die "Could not resolve $FQDN to an IPv4 or IPv6 address. Aborting."
fi

if [ -n "$CROWBAR_FROM_GIT" ]; then
    REPOS_SKIP_CHECKS+=" SLES11-SP1-Pool SLES11-SP1-Updates SLES11-SP2-Core SLES11-SP2-Updates SLES11-SP3-Pool SLES11-SP3-Updates SUSE-Cloud-2.0-Pool SUSE-Cloud-2.0-Updates"
    zypper in rubygems rubygem-json createrepo
fi

if [ -n "$IPv4_addr" ]; then
    echo "$FQDN resolved to IPv4 address: $IPv4_addr"
    if ! ip addr | grep -q "inet $IPv4_addr"; then
        die "No local interfaces configured with address $IPv4_addr. Aborting."
    fi
    if [[ "$IPv4_addr" =~ ^127 ]]; then
        die "$FQDN resolves to a loopback address. Aborting."
    fi

    if [ -n "$CROWBAR_FROM_GIT" ]; then
        NETWORK_JSON=$BARCLAMP_SRC/network/chef/data_bags/crowbar/bc-template-network.json
    else
        NETWORK_JSON=/opt/dell/chef/data_bags/crowbar/bc-template-network.json
    fi
    if ! /opt/dell/bin/bc-network-admin-helper.rb "$IPv4_addr" < $NETWORK_JSON; then
        die "IPv4 address $IPv4_addr of admin node not in admin range of admin network. Please check and fix with yast2 crowbar. Aborting."
    fi
fi
if [ -n "$IPv6_addr" ]; then
    echo "$FQDN resolved to IPv6 address: $IPv6_addr"
    if ! ip addr | grep -q "inet6 $IPv6_addr"; then
        die "No local interfaces configured with address $IPv6_addr. Aborting."
    fi
fi

# Note that the grep will fail if iptables' output changes; unlikely to happen,
# but...
if LANG=C iptables -n -L | grep -qvE '^$|^Chain [^ ]|^target     prot'; then
    die "Firewall is not completely disabled. Aborting."
fi

if ! ping -c 1 $FQDN >/dev/null 2>&1; then
    die "Failed to ping $FQDN; please check your network configuration. Aborting."
fi

# output details, that will make remote debugging via bugzilla much easier
# for us
/usr/bin/zypper lr -d
/bin/rpm -qV crowbar || :
/usr/bin/lscpu
/bin/df -h
/usr/bin/free -m
/bin/ls -la /srv/tftpboot/repos/ /srv/tftpboot/repos/Cloud/ /srv/tftpboot/suse-11.3/install/

if [ -f /opt/dell/chef/cookbooks/provisioner/templates/default/autoyast.xml.erb ]; then
    # The autoyast profile might not exist yet when CROWBAR_FROM_GIT is enabled
    /usr/bin/grep media_url /opt/dell/chef/cookbooks/provisioner/templates/default/autoyast.xml.erb
fi

CROWBAR=/opt/dell/bin/crowbar

skip_check_for_repo () {
    repo="$1"
    for skipped_repo in $REPOS_SKIP_CHECKS; do
        if [ "$repo" = "$skipped_repo" ]; then
            return 0
        fi
    done
    return 1
}

check_repo_content () {
    repo_name="$1" repo_path="$2" md5="$3"

    if skip_check_for_repo "$repo_name"; then
        echo "Skipping check for $repo_name due to \$REPOS_SKIP_CHECKS"
        return 0
    fi

    if ! [ -e "$repo_path/content.asc" ]; then
        if [ -n "$CROWBAR_FROM_GIT" ]; then
            die "$repo has not been set up yet; please see https://github.com/SUSE/cloud/wiki/Crowbar"
        else
            die "$repo_name has not been set up yet; please check you didn't miss a step in the installation guide."
        fi
    fi

    if [ -n "$CROWBAR_FROM_GIT" ]; then
        echo "Skipping md5 check for $repo_name due to \$CROWBAR_FROM_GIT"
    else
        if [ "`md5sum $repo_path/content | awk '{print $1}'`" != "$md5" ]; then
            die "$repo_name does not contain the expected repository ($repo_path/content failed MD5 checksum)"
        fi
    fi
}

check_repo_product () {
    repo="$1" expected_summary="$2"
    products_xml=/srv/tftpboot/repos/$repo/repodata/products.xml
    if ! grep -q "<summary>$2</summary>" $products_xml; then
        if skip_check_for_repo "$repo"; then
            echo "Ignoring failed repo check for $repo due to \$REPOS_SKIP_CHECKS ($products_xml is missing summary '$expected_summary')"
            if ! [ -d /srv/tftpboot/repos/$repo ]; then
                echo "Creating repo skeleton to make AutoYaST happy."
                mkdir /srv/tftpboot/repos/$repo
                /usr/bin/createrepo /srv/tftpboot/repos/$repo
            fi
            return 0
        fi
        die "$repo does not contain the right repository ($products_xml is missing summary '$expected_summary')"
    fi
}

# FIXME: repos that we cannot check yet:
#   SP3-Updates is lacking products.xml
#   Cloud: we don't have the final md5
#   SUSE-Cloud-2.0-*: not existing yet
REPOS_SKIP_CHECKS+=" Cloud SLES11-SP3-Updates SUSE-Cloud-2.0-Pool SUSE-Cloud-2.0-Updates"

# FIXME: this is for SP3 RC2
check_repo_content \
    SLES11_SP3 \
    /srv/tftpboot/suse-11.3/install \
    f9a7aa4950fbee8079844f5973169db8

check_repo_content \
    Cloud \
    /srv/tftpboot/repos/Cloud \
    1558be86e7354d31e71e7c8c2574031a

if skip_check_for_repo "Cloud-PTF"; then
    echo "Skipping check for Cloud-PTF due to \$REPOS_SKIP_CHECKS"
else
    if ! [ -e "/srv/tftpboot/repos/Cloud-PTF/repodata/repomd.xml" ]; then
        # Only do this for CROWBAR_FROM_GIT , as usually the crowbar package
        # creates the repo metadata for Cloud-PTF
        if [ -n $CROWBAR_FROM_GIT ]; then
            echo "Creating repo skeleton to make AutoYaST happy."
            if ! [ -d /srv/tftpboot/repos/Cloud-PTF ]; then
                mkdir /srv/tftpboot/repos/Cloud-PTF
            fi
            /usr/bin/createrepo /srv/tftpboot/repos/Cloud-PTF
        else
            die "Cloud-PTF has not been set up correctly; did the crowbar rpm fail to install correctly?"
        fi
    fi
fi

check_repo_product SLES11-SP1-Pool        'SUSE Linux Enterprise Server 11 SP1'
check_repo_product SLES11-SP1-Updates     'SUSE Linux Enterprise Server 11 SP1'
check_repo_product SLES11-SP1-Updates     'SUSE_SLES Service Pack 2 Migration Product'
check_repo_product SLES11-SP2-Core        'SUSE Linux Enterprise Server 11 SP2'
check_repo_product SLES11-SP2-Updates     'SUSE Linux Enterprise Server 11 SP2'
check_repo_product SLES11-SP3-Pool        'SUSE Linux Enterprise Server 11 SP3'
check_repo_product SLES11-SP3-Updates     'SUSE Linux Enterprise Server 11 SP3'
check_repo_product SUSE-Cloud-2.0-Pool    'SUSE Cloud 2.0'
check_repo_product SUSE-Cloud-2.0-Updates 'SUSE Cloud 2.0'

if [ -z "$CROWBAR_FROM_GIT" ]; then
    if ! LANG=C zypper if -t pattern cloud_admin 2> /dev/null | grep -q "^Installed: Yes$"; then
        die "cloud_admin pattern is not installed; please install with \"zypper in -t pattern cloud_admin\". Aborting."
    fi
fi

# Setup helper for git
# --------------------

add_ibs_repo () {
    url="$1"
    alias="$2"
    if ! [ -f /etc/zypp/repos.d/$alias.repo ]; then
        zypper ar $url $alias
    else
        echo "Repo: $alias already exists. Skipping."
    fi
}

if [ -n "$CROWBAR_FROM_GIT" ]; then

    echo_summary "Performing additional setup for git"

    # FIXME: This is useful only for testing the crowbar admin node setup.
    #        Additional work (e.g. on the autoyast profile) is required to make
    #        those repos available to any client nodes.
    if [ $CROWBAR_FROM_GIT = "ibs" ]; then
        add_ibs_repo http://dist.suse.de/install/SLP/SLES-11-SP3-LATEST/x86_64/DVD1 sp3
        add_ibs_repo http://dist.suse.de/install/SLP/SLE-11-SP3-SDK-LATEST/x86_64/DVD1/ sdk-sp3
        add_ibs_repo http://dist.suse.de/ibs/SUSE:/SLE-11-SP1:/GA/standard/ sp1-ga
        add_ibs_repo http://dist.suse.de/ibs/SUSE:/SLE-11-SP2:/GA/standard/ sp2-ga
        add_ibs_repo http://dist.suse.de/ibs/SUSE:/SLE-11-SP3:/GA/standard/ sp3-ga
        add_ibs_repo http://dist.suse.de/ibs/SUSE:/SLE-11-SP1:/Update/standard/ sp1-update
        add_ibs_repo http://dist.suse.de/ibs/SUSE:/SLE-11-SP2:/Update/standard/ sp2-update
        add_ibs_repo http://dist.suse.de/ibs/SUSE:/SLE-11-SP3:/Update/standard/ sp3-update
        add_ibs_repo http://dist.suse.de/ibs/Devel:/Cloud:/2.0/SLE_11_SP3/ cloud
    fi

    # install chef and its dependencies
    zypper --gpg-auto-import-keys in rubygem-chef-server rubygem-chef rabbitmq-server \
            couchdb java-1_6_0-ibm rubygem-activesupport

    # also need these (crowbar dependencies):
    zypper in rubygem-cstruct rubygem-kwalify rubygem-ruby-shadow rubygem-sass rubygem-i18n sleshammer tcpdump

    # Need this for provisioner to work:
    mkdir -p /srv/tftpboot/discovery/pxelinux.cfg
    cat > /srv/tftpboot/discovery/pxelinux.cfg/default <<EOF
DEFAULT pxeboot
TIMEOUT 20
PROMPT 0
LABEL pxeboot
        KERNEL vmlinuz0
        APPEND initrd=initrd0.img root=/sledgehammer.iso rootfstype=iso9660 rootflags=loop
ONERROR LOCALBOOT 0
EOF
    # create Compatibility link /tftpboot -> /srv/tftpboot (this is part of
    # the crowbar package when not in $CROWBAR_FROM_GIT)
    if ! [ -e /tftpboot ]; then
        ln -s /srv/tftpboot /tftpboot
    elif [ "$( /usr/bin/readlink /tftpboot )" != "/srv/tftpboot" ]; then
        die "/tftpboot exist but is not a symbolic link to /srv/tftpboot. Please fix!"
    fi

    # log directory needs to exist
    mkdir -p /var/log/crowbar
    chmod 0750 /var/log/crowbar

    # You'll also need:
    #   /srv/tftpboot/discovery/initrd0.img
    #   /srv/tftpboot/discovery/vmlinuz0
    # These can be obtained from a sleshammer image or from an existing
    # ubuntu admin node.
fi

# Starting services
# -----------------

echo_summary "Starting required services"

chkconfig rabbitmq-server on
ensure_service_running rabbitmq-server '^Node .+ with Pid [0-9]+: running'

if rabbitmqctl list_vhosts | grep -q '^/chef$'; then
    : /chef vhost already added
else
    rabbitmqctl add_vhost /chef
fi

if rabbitmqctl list_users 2>&1 | grep -q '^chef	'; then
    : chef user already added
else
    rabbit_chef_password=$( dd if=/dev/urandom count=1 bs=16 2>/dev/null | base64 | tr -d / )
    rabbitmqctl add_user chef "$rabbit_chef_password"
    # Update "amqp_pass" in  /etc/chef/server.rb and solr.rb
    sed -i 's/amqp_pass ".*"/amqp_pass "'"$rabbit_chef_password"'"/' /etc/chef/{server,solr}.rb
fi

rabbitmqctl set_permissions -p /chef chef ".*" ".*" ".*"

chkconfig couchdb on
ensure_service_running couchdb

chmod o-rwx /etc/chef /etc/chef/{server,solr}.rb

# increase chef-solr index field size
perl -i -pe 's{<maxFieldLength>.*</maxFieldLength>}{<maxFieldLength>200000</maxFieldLength>}' /var/lib/chef/solr/conf/solrconfig.xml

services='solr expander server'
for service in $services; do
    chkconfig chef-${service} on
done

for service in $services; do
    ensure_service_running chef-${service}
done

# Initial chef-client run
# -----------------------

echo_summary "Performing initial chef-client run"

if ! [ -e ~/.chef/knife.rb ]; then
    yes '' | knife configure -i
fi

node_info=$(knife node show $FQDN 2>/dev/null || :)
if echo "$node_info" | grep -q 'Run List:.*role'; then
    echo "Chef runlist for $FQDN is already populated; skipping initial chef-client run."
else
    cat <<EOF
This can cause warnings about /etc/chef/client.rb missing and
the run list being empty; they can be safely ignored.
EOF
    chef-client
fi

# Barclamp installation
# ---------------------

echo_summary "Installing barclamps"

# Don't use this one - crowbar barfs due to hyphens in the "id" attribute.
#CROWBAR_FILE="/opt/dell/barclamps/crowbar/chef/data_bags/crowbar/bc-template-crowbar.json"
# See also https://bugzilla.novell.com/show_bug.cgi?id=788161#c9
# for the history behind this location.
: ${CROWBAR_FILE:="/etc/crowbar/crowbar.json"}

mkdir -p /opt/dell/crowbar_framework
CROWBAR_REALM=$($BARCLAMP_SRC/provisioner/updates/parse_node_data $CROWBAR_FILE -a attributes.crowbar.realm)
CROWBAR_REALM=${CROWBAR_REALM##*=}

# Generate the machine install username and password.
if [[ ! -e /etc/crowbar.install.key && $CROWBAR_REALM ]]; then
    dd if=/dev/urandom bs=65536 count=1 2>/dev/null |sha512sum - 2>/dev/null | \
        (read key rest; echo "machine-install:$key" >/etc/crowbar.install.key)
fi

if [[ $CROWBAR_REALM && -f /etc/crowbar.install.key ]]; then
    export CROWBAR_KEY=$(cat /etc/crowbar.install.key)
    sed -i -e "s/machine_password/${CROWBAR_KEY##*:}/g" $CROWBAR_FILE
fi

/opt/dell/bin/barclamp_install.rb $BARCLAMP_INSTALL_OPTS $BARCLAMP_SRC/crowbar

#
# Take care that the barclamps are installed in the right order
# If you've got a full openstack set installed, e.g.: nagios has to be
# installed before keystone, etc.
#
for i in deployer dns ipmi logging nagios network ntp provisioner \
         database rabbitmq ceph \
         keystone glance cinder quantum nova nova_dashboard swift openstack ; do
    if [ -e /opt/dell/crowbar_framework/barclamps/$i.yml ]; then
        echo "$i barclamp is already installed"
    else
        /opt/dell/bin/barclamp_install.rb $BARCLAMP_INSTALL_OPTS $BARCLAMP_SRC/$i
    fi
done

# First step of crowbar bootstrap
# -------------------------------

echo_summary "Bootstrapping Crowbar setup"

# Configure chef to set up bind with correct local domain and DNS forwarders.
dns_template=/opt/dell/chef/data_bags/crowbar/bc-template-dns.json
[ -f $dns_template ] || die "$dns_template doesn't exist"
nameservers=$( awk '/^nameserver/ {print $2}' /etc/resolv.conf )
# This will still work if there are no nameservers.
/opt/dell/bin/bc-dns-json.rb $DOMAIN $nameservers < $dns_template > /tmp/bc-template-dns.json
echo "Instructing chef to configure bind with the following DNS forwarders: $nameservers"
knife data bag from file crowbar /tmp/bc-template-dns.json

echo "Create Admin node role"
NODE_ROLE="crowbar-${FQDN//./_}"
cat > /tmp/role.rb <<EOF
name "$NODE_ROLE"
description "Role for $FQDN"
run_list()
default_attributes( "crowbar" => { "network" => {} } )
override_attributes()
EOF
knife role from file /tmp/role.rb

knife node run_list add "$FQDN" role["crowbar"]
knife node run_list add "$FQDN" role["deployer-client"]
knife node run_list add "$FQDN" role["$NODE_ROLE"]

# at this point you can run chef-client from the command line to start
# the crowbar bootstrapping

chef-client

# OOC, what, if anything, is responsible for starting rainbows/crowbar under bluepill?
ensure_service_running crowbar

# Second step of crowbar bootstrap
# -------------------------------

echo_summary "Applying Crowbar configuration for administration server"

# Make sure looper_chef_client is a NOOP until we are finished deploying
touch /tmp/deploying
# This works because
#
#   crowbar_framework/app/models/provisioner_service.rb
#   crowbar_framework/app/models/service_object.rb
#
# both invoke
#
#   /opt/dell/bin/single_chef_client.sh (from barclamps/crowbar/bin)
#
# which invokes
#
#   /opt/dell/bin/looper_chef_client.sh
#
# which exits immediately if /tmp/deploying exists.

# From here, you should probably read along with the equivalent steps in
# install-chef.sh for comparison

if [ "$($CROWBAR crowbar proposal list)" != "default" ] ; then
    proposal_opts=()
    # If your custom crowbar.json is somewhere else, probably substitute that here
    if [[ -e $CROWBAR_FILE ]]; then
        proposal_opts+=(--file $CROWBAR_FILE)
    fi
    proposal_opts+=(proposal create default)

    # Sometimes proposal creation fails if Chef and Crowbar are not quite
    # fully prepared -- this can happen due to solr not having everything
    # fully indexed yet.  So we don't want to just fail immediatly if
    # we fail to create a proposal -- instead, we will kick Chef, sleep a bit,
    # and try again up to 5 times before bailing out.
    for ((x=1; x<6; x++)); do
        $CROWBAR crowbar "${proposal_opts[@]}" && { proposal_created=true; break; }
        echo "Proposal create failed, pass $x.  Will kick Chef and try again."
        chef-client
        sleep 1
    done
    if [[ ! $proposal_created ]]; then
        die "Could not create default proposal"
    fi
fi

# this has machine key world readable? care?
$CROWBAR crowbar proposal show default >/var/log/crowbar/default-proposal.json

# next will fail if ntp barclamp not present (or did for me...)
$CROWBAR crowbar proposal commit default || \
    die "Could not commit default proposal!"

$CROWBAR crowbar show default >/var/log/crowbar/default.json

crowbar_up=true
chef-client

# here we whould check indexer/expander is finished

# original script has several calls to check_machine_role -- see source for
# this, in my limited testing it wasn't necessary on SUSE, but we should still
# do it anyway (if the role isn't present things break, apparently)

# BMC support?

# Third step of crowbar bootstrap
# -------------------------------

echo_summary "Transitioning administration server to \"ready\""

# transition though all the states to ready.  Make sure that
# Chef has completly finished with transition before proceeding
# to the next.

for state in "discovering" "discovered" "hardware-installing" \
    "hardware-installed" "installing" "installed" "readying" "ready"
do
    while [[ -f "/tmp/chef-client.lock" ]]; do sleep 1; done
    printf "$state: "
    $CROWBAR crowbar transition "$FQDN" "$state" || \
        die "Transition to $state failed!"
    if type -f "transition_check_$state"&>/dev/null; then
        "transition_check_$state" || \
            die "Sanity check for transitioning to $state failed!"
    fi
    # chef_or_die "Chef run for $state transition failed!"
    chef-client
    # check_machine_role
done

# OK, let looper_chef_client run normally now.
rm /tmp/deploying

# Starting more services
# ----------------------

echo_summary "Starting chef-client"

# Need chef-client daemon now
chkconfig chef-client on
ensure_service_running chef-client

# Final sanity checks
# -------------------

echo_summary "Performing post-installation sanity checks"

# Spit out a warning message if we managed to not get an IP address
IPSTR=$($CROWBAR network show default | /opt/dell/barclamps/provisioner/updates/parse_node_data -a attributes.network.networks.admin.ranges.admin.start)
IP=${IPSTR##*=}
ip addr | grep -q $IP || {
    die "eth0 not configured, but should have been."
}

# Run tests -- currently the host will run this.
# 2012-05-24: test is failing, so only run if CROWBAR_RUN_TESTS=true
# (this is distinct from CROWBAR_FROM_GIT, which would add extra repos etc.)
if [ -n "$CROWBAR_RUN_TESTS" ]; then
    /opt/dell/bin/barclamp_test.rb -t || \
        die "Crowbar validation has errors! Please check the logs and correct."
fi

for s in xinetd dhcpd apache2 ; do
    if ! /etc/init.d/$s status >/dev/null ; then
        die "service $s missing"
    fi
done

# We're done!
# -----------

touch /opt/dell/crowbar_framework/.crowbar-installed-ok

kill_spinner

cat <<EOF | pipe_stdout_and_logfile

Admin node deployed.

You can now visit the Crowbar web UI at:

    http://$IP:3000/

You should also now be able to PXE-boot a client.  Please refer
to the documentation for the next steps.

Note that to run the crowbar CLI tool, you will need to log out
and log back in again for the correct environment variables to
be set up.
EOF

run_succeeded=hooray

#! /bin/bash -e

# This script is called after being installed by the Crowbar RPM from the SUSE
# Cloud ISO. In this context, it is expected that all other required
# repositories (eg. SLES, Updates) are already set up, with the required files
# in place.
#
# For development and testing use, call the script with the '--from-git'
# option. Use the appropriate dev VM and follow the corresponding setup
# instructions.

BARCLAMP_INSTALL_OPTS="--rpm"

if [ "$1" = "--from-git" ]; then
    CROWBAR_FROM_GIT=true
    BARCLAMP_INSTALL_OPTS="--force"
    : ${CROWBAR_FILE:=/root/crowbar/crowbar.json}
    : ${BARCLAMP_SRC:=/root/crowbar/barclamps/}
    sed -i -e '/"nagios":/d' -e '/"ganglia":/d' $CROWBAR_FILE
fi

LOGFILE=/var/log/crowbar/install.log
mkdir -p "`dirname "$LOGFILE"`"

: ${BARCLAMP_SRC:="/opt/dell/barclamps/"}

run_succeeded=

# Infrastructure for nice output/logging
# --------------------------------------

# Copy stdout to fd 3
exec 3>&1
# Create fd 4 for logfile
exec 4>> "$LOGFILE"

if [ -z "$CROWBAR_VERBOSE" ]; then
    # Set fd 1 and 2 to logfile
    exec 1>&4 2>&1
else
    # Set fd 1 and 2 to logfile (and keep stdout too)
    exec 1> >( tee -a /dev/fd/4 ) 2>&1
fi
# Send summary fd to original stdout
exec 6>&3

pipe_stdout_and_logfile () {
    tee -a /dev/fd/3 /dev/fd/4 > /dev/null
}

# Draw a spinner so the user knows something is happening
spinner () {
    local delay=0.75
    local spinstr='/-\|'
    printf "... " >&3
    while [ true ]; do
        local temp=${spinstr#?}
        printf "[%c]" "$spinstr" >&3
        local spinstr=$temp${spinstr%"$temp"}
        sleep $delay
        printf "\b\b\b" >&3
    done
}

kill_spinner () {
    if [ ! -z "$LAST_SPINNER_PID" ]; then
        kill >/dev/null 2>&1 $LAST_SPINNER_PID
        if [ $# -eq 0 ]; then
            printf "\b\b\bdone\n" >&3
        else
            printf "\b\b\b$*\n" >&3
        fi
        unset LAST_SPINNER_PID
    fi
}

kill_spinner_with_failed () {
    kill_spinner "failed"
}

echo_log () {
    echo -e === "$(date '+%F %T %z'): $@" >&4
}

echo_summary () {
    # Also send summary to logfile
    echo_log $@

    kill_spinner

    if [ -z "$CROWBAR_VERBOSE" ]; then
        if [ -t 3 ]; then
            echo -n -e $@ >&3
            # Use disown to lose job control messages (especially the
            # "Completed" message when spinner will be killed)
            spinner & disown
            LAST_SPINNER_PID=$!
        else
            echo -e $@ >&3
        fi
    else
        echo -e === $@ >&3
    fi
}

echo_summary_no_spinner () {
    # Also send summary to logfile
    echo_log $@

    kill_spinner

    if [ -z "$CROWBAR_VERBOSE" ]; then
        echo -e $@ >&3
    else
        echo -e === $@ >&3
    fi
}

die() {
    # Send empty line & error to logfile
    echo >&4
    echo_log "Error: $@"

    kill_spinner_with_failed

    echo >&3
    echo -e "Error: $@" >&3

    res=1
    exit 1
}

exit_handler () {
    if [ -z "$run_succeeded" ]; then
        kill_spinner_with_failed
        cat <<EOF | pipe_stdout_and_logfile

Crowbar installation terminated prematurely.  Please examine the above
output or $LOGFILE for clues as to what went wrong.
You should also check the SUSE Cloud Installation Manual, in
particular the Troubleshooting section.  Note that this script can
safely be re-run multiple times if required.
EOF
    else
        kill_spinner
    fi
}

trap exit_handler EXIT

# Real work starts here
# ---------------------

echo "`date` $0 started with args: $*"

ensure_service_running () {
    service="$1"
    regexp="${2:-running}"
    if service $service status | egrep -q "$regexp"; then
        echo "$service is already running - no need to start."
    else
        service $service start
        sleep 4
    fi
}

# Sanity checks
# -------------

echo_summary "Performing sanity checks"

rootpw=$( getent shadow root | cut -d: -f2 )
case "$rootpw" in
    \*|\!*)
        die "root password is unset or locked.  Chef will rewrite /root/.ssh/authorized_keys; therefore to avoid being accidentally locked out of this admin node, you should first ensure you have a working root password."
        ;;
esac

# It is exceedingly important that 'hostname -f' actually returns an FQDN!
# if it doesn't, add an entry to /etc/hosts, e.g.:
#    192.168.124.10 cb-admin.example.com cb-admin
if ! FQDN=$(hostname -f 2>/dev/null); then
    die "Unable to detect fully-qualified hostname. Aborting."
fi

if ! DOMAIN=$(hostname -d 2>/dev/null); then
    die "Unable to detect DNS domain name. Aborting."
fi

if [ -z "$FQDN" -o -z "$DOMAIN" ]; then
    die "Unable to detect fully-qualified hostname. Aborting."
fi

if ! resolved=$(getent ahosts $FQDN 2>/dev/null); then
    die "Unable to resolve hostname $FQDN via host(1). Please check your configuration of DNS, hostname, and /etc/hosts. Aborting."
fi

IPv4_addr=$( echo "$resolved" | awk '{ if ($1 !~ /:/) { print $1; exit } }' )
IPv6_addr=$( echo "$resolved" | awk '{ if ($1  ~ /:/) { print $1; exit } }' )
if [ -z "$IPv4_addr" -a -z "$IPv6_addr" ]; then
    die "Could not resolve $FQDN to an IPv4 or IPv6 address. Aborting."
fi

if [ -n "$CROWBAR_FROM_GIT" ]; then
    REPOS_SKIP_CHECKS+=" SLES11-SP1-Pool SLES11-SP1-Updates SLES11-SP2-Core SLES11-SP2-Updates SLES11-SP3-Pool SLES11-SP3-Updates SUSE-Cloud-2.0-Pool SUSE-Cloud-2.0-Updates"
    zypper in rubygems rubygem-json createrepo
fi

if [ -n "$IPv4_addr" ]; then
    echo "$FQDN resolved to IPv4 address: $IPv4_addr"
    if ! ip addr | grep -q "inet $IPv4_addr"; then
        die "No local interfaces configured with address $IPv4_addr. Aborting."
    fi
    if [[ "$IPv4_addr" =~ ^127 ]]; then
        die "$FQDN resolves to a loopback address. Aborting."
    fi

    if [ -n "$CROWBAR_FROM_GIT" ]; then
        NETWORK_JSON=$BARCLAMP_SRC/network/chef/data_bags/crowbar/bc-template-network.json
    else
        NETWORK_JSON=/opt/dell/chef/data_bags/crowbar/bc-template-network.json
    fi
    if ! /opt/dell/bin/bc-network-admin-helper.rb "$IPv4_addr" < $NETWORK_JSON; then
        die "IPv4 address $IPv4_addr of admin node not in admin range of admin network. Please check and fix with yast2 crowbar. Aborting."
    fi
fi
if [ -n "$IPv6_addr" ]; then
    echo "$FQDN resolved to IPv6 address: $IPv6_addr"
    if ! ip addr | grep -q "inet6 $IPv6_addr"; then
        die "No local interfaces configured with address $IPv6_addr. Aborting."
    fi
fi

# Note that the grep will fail if iptables' output changes; unlikely to happen,
# but...
if LANG=C iptables -n -L | grep -qvE '^$|^Chain [^ ]|^target     prot'; then
    die "Firewall is not completely disabled. Aborting."
fi

if ! ping -c 1 $FQDN >/dev/null 2>&1; then
    die "Failed to ping $FQDN; please check your network configuration. Aborting."
fi

# output details, that will make remote debugging via bugzilla much easier
# for us
/usr/bin/zypper lr -d
/bin/rpm -qV crowbar || :
/usr/bin/lscpu
/bin/df -h
/usr/bin/free -m
/bin/ls -la /srv/tftpboot/repos/ /srv/tftpboot/repos/Cloud/ /srv/tftpboot/suse-11.3/install/

if [ -f /opt/dell/chef/cookbooks/provisioner/templates/default/autoyast.xml.erb ]; then
    # The autoyast profile might not exist yet when CROWBAR_FROM_GIT is enabled
    /usr/bin/grep media_url /opt/dell/chef/cookbooks/provisioner/templates/default/autoyast.xml.erb
fi

CROWBAR=/opt/dell/bin/crowbar

skip_check_for_repo () {
    repo="$1"
    for skipped_repo in $REPOS_SKIP_CHECKS; do
        if [ "$repo" = "$skipped_repo" ]; then
            return 0
        fi
    done
    return 1
}

check_repo_content () {
    repo_name="$1" repo_path="$2" md5="$3"

    if skip_check_for_repo "$repo_name"; then
        echo "Skipping check for $repo_name due to \$REPOS_SKIP_CHECKS"
        return 0
    fi

    if ! [ -e "$repo_path/content.asc" ]; then
        if [ -n "$CROWBAR_FROM_GIT" ]; then
            die "$repo has not been set up yet; please see https://github.com/SUSE/cloud/wiki/Crowbar"
        else
            die "$repo_name has not been set up yet; please check you didn't miss a step in the installation guide."
        fi
    fi

    if [ -n "$CROWBAR_FROM_GIT" ]; then
        echo "Skipping md5 check for $repo_name due to \$CROWBAR_FROM_GIT"
    else
        if [ "`md5sum $repo_path/content | awk '{print $1}'`" != "$md5" ]; then
            die "$repo_name does not contain the expected repository ($repo_path/content failed MD5 checksum)"
        fi
    fi
}

check_repo_product () {
    repo="$1" expected_summary="$2"
    products_xml=/srv/tftpboot/repos/$repo/repodata/products.xml
    if ! grep -q "<summary>$2</summary>" $products_xml; then
        if skip_check_for_repo "$repo"; then
            echo "Ignoring failed repo check for $repo due to \$REPOS_SKIP_CHECKS ($products_xml is missing summary '$expected_summary')"
            if ! [ -d /srv/tftpboot/repos/$repo ]; then
                echo "Creating repo skeleton to make AutoYaST happy."
                mkdir /srv/tftpboot/repos/$repo
                /usr/bin/createrepo /srv/tftpboot/repos/$repo
            fi
            return 0
        fi
        die "$repo does not contain the right repository ($products_xml is missing summary '$expected_summary')"
    fi
}

# FIXME: repos that we cannot check yet:
#   SP3-Updates is lacking products.xml
#   Cloud: we don't have the final md5
#   SUSE-Cloud-2.0-*: not existing yet
REPOS_SKIP_CHECKS+=" Cloud SLES11-SP3-Updates SUSE-Cloud-2.0-Pool SUSE-Cloud-2.0-Updates"

# FIXME: this is for SP3 RC2
check_repo_content \
    SLES11_SP3 \
    /srv/tftpboot/suse-11.3/install \
    f9a7aa4950fbee8079844f5973169db8

check_repo_content \
    Cloud \
    /srv/tftpboot/repos/Cloud \
    1558be86e7354d31e71e7c8c2574031a

if skip_check_for_repo "Cloud-PTF"; then
    echo "Skipping check for Cloud-PTF due to \$REPOS_SKIP_CHECKS"
else
    if ! [ -e "/srv/tftpboot/repos/Cloud-PTF/repodata/repomd.xml" ]; then
        # Only do this for CROWBAR_FROM_GIT , as usually the crowbar package
        # creates the repo metadata for Cloud-PTF
        if [ -n $CROWBAR_FROM_GIT ]; then
            echo "Creating repo skeleton to make AutoYaST happy."
            if ! [ -d /srv/tftpboot/repos/Cloud-PTF ]; then
                mkdir /srv/tftpboot/repos/Cloud-PTF
            fi
            /usr/bin/createrepo /srv/tftpboot/repos/Cloud-PTF
        else
            die "Cloud-PTF has not been set up correctly; did the crowbar rpm fail to install correctly?"
        fi
    fi
fi

check_repo_product SLES11-SP1-Pool        'SUSE Linux Enterprise Server 11 SP1'
check_repo_product SLES11-SP1-Updates     'SUSE Linux Enterprise Server 11 SP1'
check_repo_product SLES11-SP1-Updates     'SUSE_SLES Service Pack 2 Migration Product'
check_repo_product SLES11-SP2-Core        'SUSE Linux Enterprise Server 11 SP2'
check_repo_product SLES11-SP2-Updates     'SUSE Linux Enterprise Server 11 SP2'
check_repo_product SLES11-SP3-Pool        'SUSE Linux Enterprise Server 11 SP3'
check_repo_product SLES11-SP3-Updates     'SUSE Linux Enterprise Server 11 SP3'
check_repo_product SUSE-Cloud-2.0-Pool    'SUSE Cloud 2.0'
check_repo_product SUSE-Cloud-2.0-Updates 'SUSE Cloud 2.0'

# Setup helper for git
# --------------------

add_ibs_repo () {
    url="$1"
    alias="$2"
    if ! [ -f /etc/zypp/repos.d/$alias.repo ]; then
        zypper ar $url $alias
    else
        echo "Repo: $alias already exists. Skipping."
    fi
}

if [ -n "$CROWBAR_FROM_GIT" ]; then

    echo_summary "Performing additional setup for git"

    # FIXME: This is useful only for testing the crowbar admin node setup.
    #        Additional work (e.g. on the autoyast profile) is required to make
    #        those repos available to any client nodes.
    if [ $CROWBAR_FROM_GIT = "ibs" ]; then
        add_ibs_repo http://dist.suse.de/install/SLP/SLES-11-SP3-LATEST/x86_64/DVD1 sp3
        add_ibs_repo http://dist.suse.de/install/SLP/SLE-11-SP3-SDK-LATEST/x86_64/DVD1/ sdk-sp3
        add_ibs_repo http://dist.suse.de/ibs/SUSE:/SLE-11-SP1:/GA/standard/ sp1-ga
        add_ibs_repo http://dist.suse.de/ibs/SUSE:/SLE-11-SP2:/GA/standard/ sp2-ga
        add_ibs_repo http://dist.suse.de/ibs/SUSE:/SLE-11-SP3:/GA/standard/ sp3-ga
        add_ibs_repo http://dist.suse.de/ibs/SUSE:/SLE-11-SP1:/Update/standard/ sp1-update
        add_ibs_repo http://dist.suse.de/ibs/SUSE:/SLE-11-SP2:/Update/standard/ sp2-update
        add_ibs_repo http://dist.suse.de/ibs/SUSE:/SLE-11-SP3:/Update/standard/ sp3-update
        add_ibs_repo http://dist.suse.de/ibs/Devel:/Cloud:/2.0/SLE_11_SP3/ cloud
    fi

    # install chef and its dependencies
    zypper --gpg-auto-import-keys in rubygem-chef-server rubygem-chef rabbitmq-server \
            couchdb java-1_6_0-ibm rubygem-activesupport

    # also need these (crowbar dependencies):
    zypper in rubygem-cstruct rubygem-kwalify rubygem-ruby-shadow rubygem-sass rubygem-i18n sleshammer tcpdump

    # Need this for provisioner to work:
    mkdir -p /srv/tftpboot/discovery/pxelinux.cfg
    cat > /srv/tftpboot/discovery/pxelinux.cfg/default <<EOF
DEFAULT pxeboot
TIMEOUT 20
PROMPT 0
LABEL pxeboot
        KERNEL vmlinuz0
        APPEND initrd=initrd0.img root=/sledgehammer.iso rootfstype=iso9660 rootflags=loop
ONERROR LOCALBOOT 0
EOF
    # create Compatibility link /tftpboot -> /srv/tftpboot (this is part of
    # the crowbar package when not in $CROWBAR_FROM_GIT)
    if ! [ -e /tftpboot ]; then
        ln -s /srv/tftpboot /tftpboot
    elif [ "$( /usr/bin/readlink /tftpboot )" != "/srv/tftpboot" ]; then
        die "/tftpboot exist but is not a symbolic link to /srv/tftpboot. Please fix!"
    fi

    # log directory needs to exist
    mkdir -p /var/log/crowbar
    chmod 0750 /var/log/crowbar

    # You'll also need:
    #   /srv/tftpboot/discovery/initrd0.img
    #   /srv/tftpboot/discovery/vmlinuz0
    # These can be obtained from a sleshammer image or from an existing
    # ubuntu admin node.
fi

# Starting services
# -----------------

echo_summary "Starting required services"

chkconfig rabbitmq-server on
ensure_service_running rabbitmq-server '^Node .+ with Pid [0-9]+: running'

if rabbitmqctl list_vhosts | grep -q '^/chef$'; then
    : /chef vhost already added
else
    rabbitmqctl add_vhost /chef
fi

if rabbitmqctl list_users 2>&1 | grep -q '^chef	'; then
    : chef user already added
else
    rabbit_chef_password=$( dd if=/dev/urandom count=1 bs=16 2>/dev/null | base64 | tr -d / )
    rabbitmqctl add_user chef "$rabbit_chef_password"
    # Update "amqp_pass" in  /etc/chef/server.rb and solr.rb
    sed -i 's/amqp_pass ".*"/amqp_pass "'"$rabbit_chef_password"'"/' /etc/chef/{server,solr}.rb
fi

rabbitmqctl set_permissions -p /chef chef ".*" ".*" ".*"

chkconfig couchdb on
ensure_service_running couchdb

chmod o-rwx /etc/chef /etc/chef/{server,solr}.rb

# increase chef-solr index field size
perl -i -pe 's{<maxFieldLength>.*</maxFieldLength>}{<maxFieldLength>200000</maxFieldLength>}' /var/lib/chef/solr/conf/solrconfig.xml

services='solr expander server'
for service in $services; do
    chkconfig chef-${service} on
done

for service in $services; do
    ensure_service_running chef-${service}
done

# Initial chef-client run
# -----------------------

echo_summary "Performing initial chef-client run"

if ! [ -e ~/.chef/knife.rb ]; then
    yes '' | knife configure -i
fi

node_info=$(knife node show $FQDN 2>/dev/null || :)
if echo "$node_info" | grep -q 'Run List:.*role'; then
    echo "Chef runlist for $FQDN is already populated; skipping initial chef-client run."
else
    cat <<EOF
This can cause warnings about /etc/chef/client.rb missing and
the run list being empty; they can be safely ignored.
EOF
    chef-client
fi

# Barclamp installation
# ---------------------

echo_summary "Installing barclamps"

# Don't use this one - crowbar barfs due to hyphens in the "id" attribute.
#CROWBAR_FILE="/opt/dell/barclamps/crowbar/chef/data_bags/crowbar/bc-template-crowbar.json"
# See also https://bugzilla.novell.com/show_bug.cgi?id=788161#c9
# for the history behind this location.
: ${CROWBAR_FILE:="/etc/crowbar/crowbar.json"}

mkdir -p /opt/dell/crowbar_framework
CROWBAR_REALM=$($BARCLAMP_SRC/provisioner/updates/parse_node_data $CROWBAR_FILE -a attributes.crowbar.realm)
CROWBAR_REALM=${CROWBAR_REALM##*=}

# Generate the machine install username and password.
if [[ ! -e /etc/crowbar.install.key && $CROWBAR_REALM ]]; then
    dd if=/dev/urandom bs=65536 count=1 2>/dev/null |sha512sum - 2>/dev/null | \
        (read key rest; echo "machine-install:$key" >/etc/crowbar.install.key)
fi

if [[ $CROWBAR_REALM && -f /etc/crowbar.install.key ]]; then
    export CROWBAR_KEY=$(cat /etc/crowbar.install.key)
    sed -i -e "s/machine_password/${CROWBAR_KEY##*:}/g" $CROWBAR_FILE
fi

/opt/dell/bin/barclamp_install.rb $BARCLAMP_INSTALL_OPTS $BARCLAMP_SRC/crowbar

#
# Take care that the barclamps are installed in the right order
# If you've got a full openstack set installed, e.g.: nagios has to be
# installed before keystone, etc.
#
for i in deployer dns ipmi logging nagios network ntp provisioner \
         database rabbitmq ceph \
         keystone glance cinder quantum nova nova_dashboard swift openstack ; do
    if [ -e /opt/dell/crowbar_framework/barclamps/$i.yml ]; then
        echo "$i barclamp is already installed"
    else
        /opt/dell/bin/barclamp_install.rb $BARCLAMP_INSTALL_OPTS $BARCLAMP_SRC/$i
    fi
done

# First step of crowbar bootstrap
# -------------------------------

echo_summary "Bootstrapping Crowbar setup"

# Configure chef to set up bind with correct local domain and DNS forwarders.
dns_template=/opt/dell/chef/data_bags/crowbar/bc-template-dns.json
[ -f $dns_template ] || die "$dns_template doesn't exist"
nameservers=$( awk '/^nameserver/ {print $2}' /etc/resolv.conf )
# This will still work if there are no nameservers.
/opt/dell/bin/bc-dns-json.rb $DOMAIN $nameservers < $dns_template > /tmp/bc-template-dns.json
echo "Instructing chef to configure bind with the following DNS forwarders: $nameservers"
knife data bag from file crowbar /tmp/bc-template-dns.json

echo "Create Admin node role"
NODE_ROLE="crowbar-${FQDN//./_}"
cat > /tmp/role.rb <<EOF
name "$NODE_ROLE"
description "Role for $FQDN"
run_list()
default_attributes( "crowbar" => { "network" => {} } )
override_attributes()
EOF
knife role from file /tmp/role.rb

knife node run_list add "$FQDN" role["crowbar"]
knife node run_list add "$FQDN" role["deployer-client"]
knife node run_list add "$FQDN" role["$NODE_ROLE"]

# at this point you can run chef-client from the command line to start
# the crowbar bootstrapping

chef-client

# OOC, what, if anything, is responsible for starting rainbows/crowbar under bluepill?
ensure_service_running crowbar

# Second step of crowbar bootstrap
# -------------------------------

echo_summary "Applying Crowbar configuration for administration server"

# Make sure looper_chef_client is a NOOP until we are finished deploying
touch /tmp/deploying
# This works because
#
#   crowbar_framework/app/models/provisioner_service.rb
#   crowbar_framework/app/models/service_object.rb
#
# both invoke
#
#   /opt/dell/bin/single_chef_client.sh (from barclamps/crowbar/bin)
#
# which invokes
#
#   /opt/dell/bin/looper_chef_client.sh
#
# which exits immediately if /tmp/deploying exists.

# From here, you should probably read along with the equivalent steps in
# install-chef.sh for comparison

if [ "$($CROWBAR crowbar proposal list)" != "default" ] ; then
    proposal_opts=()
    # If your custom crowbar.json is somewhere else, probably substitute that here
    if [[ -e $CROWBAR_FILE ]]; then
        proposal_opts+=(--file $CROWBAR_FILE)
    fi
    proposal_opts+=(proposal create default)

    # Sometimes proposal creation fails if Chef and Crowbar are not quite
    # fully prepared -- this can happen due to solr not having everything
    # fully indexed yet.  So we don't want to just fail immediatly if
    # we fail to create a proposal -- instead, we will kick Chef, sleep a bit,
    # and try again up to 5 times before bailing out.
    for ((x=1; x<6; x++)); do
        $CROWBAR crowbar "${proposal_opts[@]}" && { proposal_created=true; break; }
        echo "Proposal create failed, pass $x.  Will kick Chef and try again."
        chef-client
        sleep 1
    done
    if [[ ! $proposal_created ]]; then
        die "Could not create default proposal"
    fi
fi

# this has machine key world readable? care?
$CROWBAR crowbar proposal show default >/var/log/crowbar/default-proposal.json

# next will fail if ntp barclamp not present (or did for me...)
$CROWBAR crowbar proposal commit default || \
    die "Could not commit default proposal!"

$CROWBAR crowbar show default >/var/log/crowbar/default.json

crowbar_up=true
chef-client

# here we whould check indexer/expander is finished

# original script has several calls to check_machine_role -- see source for
# this, in my limited testing it wasn't necessary on SUSE, but we should still
# do it anyway (if the role isn't present things break, apparently)

# BMC support?

# Third step of crowbar bootstrap
# -------------------------------

echo_summary "Transitioning administration server to \"ready\""

# transition though all the states to ready.  Make sure that
# Chef has completly finished with transition before proceeding
# to the next.

for state in "discovering" "discovered" "hardware-installing" \
    "hardware-installed" "installing" "installed" "readying" "ready"
do
    while [[ -f "/tmp/chef-client.lock" ]]; do sleep 1; done
    printf "$state: "
    $CROWBAR crowbar transition "$FQDN" "$state" || \
        die "Transition to $state failed!"
    if type -f "transition_check_$state"&>/dev/null; then
        "transition_check_$state" || \
            die "Sanity check for transitioning to $state failed!"
    fi
    # chef_or_die "Chef run for $state transition failed!"
    chef-client
    # check_machine_role
done

# OK, let looper_chef_client run normally now.
rm /tmp/deploying

# Starting more services
# ----------------------

echo_summary "Starting chef-client"

# Need chef-client daemon now
chkconfig chef-client on
ensure_service_running chef-client

# Final sanity checks
# -------------------

echo_summary "Performing post-installation sanity checks"

# Spit out a warning message if we managed to not get an IP address
IPSTR=$($CROWBAR network show default | /opt/dell/barclamps/provisioner/updates/parse_node_data -a attributes.network.networks.admin.ranges.admin.start)
IP=${IPSTR##*=}
ip addr | grep -q $IP || {
    die "eth0 not configured, but should have been."
}

# Run tests -- currently the host will run this.
# 2012-05-24: test is failing, so only run if CROWBAR_RUN_TESTS=true
# (this is distinct from CROWBAR_FROM_GIT, which would add extra repos etc.)
if [ -n "$CROWBAR_RUN_TESTS" ]; then
    /opt/dell/bin/barclamp_test.rb -t || \
        die "Crowbar validation has errors! Please check the logs and correct."
fi

for s in xinetd dhcpd apache2 ; do
    if ! /etc/init.d/$s status >/dev/null ; then
        die "service $s missing"
    fi
done

# We're done!
# -----------

touch /opt/dell/crowbar_framework/.crowbar-installed-ok

kill_spinner

cat <<EOF | pipe_stdout_and_logfile

Admin node deployed.

You can now visit the Crowbar web UI at:

    http://$IP:3000/

You should also now be able to PXE-boot a client.  Please refer
to the documentation for the next steps.

Note that to run the crowbar CLI tool, you will need to log out
and log back in again for the correct environment variables to
be set up.
EOF

run_succeeded=hooray


Add rpm build script

Add rpm build script

#!/bin/bash

## From https://github.com/bisq-network/bisq-desktop/issues/401#issuecomment-372091261

version=0.7.0

alien -r -g /home/$USER/Desktop/Bisq-64bit-$version.deb
find bisq-$version -type f | while read LIB; do LDDOUT=$(ldd $LIB 2>&1); LDDRETVAL=$?;if [ \( -z "${LDDOUT%%*you do not have execution permission for*}" \) -a \( $LDDRETVAL -eq 0 \) ]; then chmod -v +x $LIB;fi;done
cat bisq-$version/bisq-$version-2.spec | while read LINE; do if echo "$LINE" | grep -q "_unpackaged_files_terminate_build" ;then break; else echo "$LINE";fi;done > bisq.spec
rm bisq-$version/bisq-$version-2.spec
echo "AutoReqProv: no" >> bisq.spec
find bisq-$version | /usr/lib/rpm/rpmdeps --requires 2>&1| while read LIB; do if [ -z "$(find bisq-$version -name  ${LIB%%(*}\*)" ]; then echo "Requires: $LIB"; fi; done | egrep -v '(libavcodec|libavformat|libavutil)' >> bisq.spec
cat >> bisq.spec << "EOF"
%description
Bisq is ....

%preun
if [ "$1" = 0 ]; then
   xdg-desktop-menu uninstall --novendor /opt/Bisq/Bisq.desktop
fi

%post
xdg-desktop-menu install --novendor /opt/Bisq/Bisq.desktop

%files
/opt/Bisq
EOF

pushd bisq-$version
rpmbuild --buildroot=$(pwd) -bb --target x86_64 ../bisq.spec
popd

error: pathspec 'package/linux/rpm.sh' did not match any file(s) known to git


small script to add and remove lines items from a todo list

#!/bin/zsh

usage="todo {add,remove} 'task'"
todo_file=/home/fin/.zsh/todo.txt

if [ $1 = "add" ]; then
    echo $2 >> $todo_file
elif [ $1 = "remove" ]; then
    sed -i "/${2}/d" $todo_file
fi

cat $todo_file

error: pathspec 'general_utilities/todo.sh' did not match any file(s) known to git


Quickly stash & pull & pop

Quickly stash & pull & pop

function git_stash_pull_pop() {
  git stash && git pull && git stash pop
  return $?
}

error: pathspec 'bash_profile.d/git_stash_pull_pop.sh' did not match any file(s) known to git


Import gic/CVS - 2014/01/14 17:05:04

Import gic/CVS - 2014/01/14 17:05:04

remote: Support for password authentication was removed on August 13, 2021.
remote: Please see https://docs.github.com/en/get-started/getting-started-with-git/about-remote-repositories#cloning-with-https-urls for information on currently recommended modes of authentication.
fatal: Authentication failed for 'https://github.com/reaper/rubtools.git/'


Add script for ignored files in the working tree

This adds a script listing all files that are ignored by .gitignore but
present in the working tree nevertheless. The option “--sort-by-size”
sorts the output by file size, whereas the default is to sort the files
lexicographically with numeric sorting.

#!/usr/bin/env bash
#
# git-find-ignored-files.sh
#
# Find all files present in the index and working tree ignored by .gitignore
#
# Usage: git-find-ignored-files [-s | --sort-by-size] [--help]
#
# Author: Patrick Lühne, https://www.luehne.de/
#

function print_help
{
	grep "^# Usage" < "$0" | cut -c 3-
}

if [[ $# -gt 1 ]]
then
	print_help
	exit 1
fi

command="git ls-files --ignored --exclude-standard -z | xargs -0r du -sh"

case "$1" in
	-h|--help)
		print_help
		exit 0
		;;
	-s|--sort-by-size)
		command="$command | sort -h"
		;;
	*)
		if [[ $# -gt 0 ]]
		then
			(>&2 echo "error: unknown option “$1”")
			print_help
			exit 1
		fi
		;;
esac

eval "$command"

error: pathspec 'git-find-ignored-files.sh' did not match any file(s) known to git


Replacing librabbitmq with amqp

Replacing librabbitmq with amqp

remote: Support for password authentication was removed on August 13, 2021.
remote: Please see https://docs.github.com/en/get-started/getting-started-with-git/about-remote-repositories#cloning-with-https-urls for information on currently recommended modes of authentication.
fatal: Authentication failed for 'https://github.com/zatosource/zato-build.git/'


Add script/npm.sh

Add script/npm.sh

npm.sh provides a convenient way to install various global node.js modules

# npm.sh
#
# Install various global node modules

# http://bower.io/
npm install -g bower

# http://coffeescript.org/
npm install -g coffee-script

# http://jashkenas.github.io/docco/
npm install -g docco

# http://gruntjs.com/
npm install -g grunt-cli

# http://yeoman.io/
npm install -g yo

error: pathspec 'script/npm.sh' did not match any file(s) known to git


use sudo only when required allow install in Centos

remote: Support for password authentication was removed on August 13, 2021.
remote: Please see https://docs.github.com/en/get-started/getting-started-with-git/about-remote-repositories#cloning-with-https-urls for information on currently recommended modes of authentication.
fatal: Authentication failed for 'https://github.com/jig/myubuntu.git/'


Added a dev install script

Added a dev install script

#!/bin/bash -e
{ # this ensures the entire script is downloaded #

# Stops the execution of a script if a command or pipeline has an error
set -e

echo "Downloading dev version from github"

#download latest
wget https://github.com/MKHenson/modepress/archive/dev.zip
unzip -o "modepress-dev.zip" "modepress-dev/*"

# Moves the server folder to the current directory
cp -r modepress-dev/server/* .

# Remove modepress-master
if [ -d "modepress-dev" ]; then
	rm modepress-dev -R
fi

# Remove the zip file
rm "modepress-dev.zip"

if [ ! -f "config.json" ]; then
	# Copy the example config to a config.json
	cp "example-config.json" "config.json"
fi

# All done
echo "Modepress successfully installed"
echo "Please run an NPM update and edit the config.json"
exit
} # this ensures the entire script is downloaded #

error: pathspec 'install-script-dev.sh' did not match any file(s) known to git


adding bootstrap script

adding bootstrap script

#!/bin/bash

set -e

echo 'updating apt-get tree and installing python-pip'
sudo apt-get update && sudo apt-get install -y python-pip python-dev

echo 'installing ansible...'
sudo pip install ansible==1.9.4

echo 'running ansible...'
ansible-playbook -i "localhost," -c local deployment/ubuntu14/ansible/localhost.yml

echo 'testing connectivity'
cif -d -p

echo 'testing query'
cif --search example.com -d

error: pathspec 'deployment/ubuntu14/ansible/bootstrap.sh' did not match any file(s) known to git


Added eclipse director script, for easy command-line  provisioning operations. Modify it to point to your eclipse directory (or pass it using --eclipse-dir), and easily install or uninstall scala plugin versions.

#!/bin/bash

eclipse_dir="/Applications/Programming/eclipse-helios/"
eclipse_opt="-nosplash -consoleLog"
app="org.eclipse.equinox.p2.director"

eclipse_repo="http://download.eclipse.org/releases/helios/"
branch="master"
scala_repo="http://scala-ide.dreamhosters.com/nightly-update"

function usage()
{
    cat <<EOF
`basename $0` [opt] command [pluginid]

Options:
    --eclipse-dir <path>      Path to the Eclipse installation that you want to modify

Commands:
    list                      List available plugins. (useless right now, as it uses the Eclipse repo).

    install <id>              Install plugin. It is the version number of the Scala installation
                              For instance: 2.9.1.final or trunk

    uninstall                 Uninstall the currently installed Scala plugin

    uninstall-bundle <id>     Uninstall the given bundle

    help                      Print this help screen

EOF
    exit 1
}

while [ $# -gt 0 ]; do

    case $1 in
        "" | "help")
            usage
            ;;

        "--eclipse-dir")
            eclipse_dir=$2
            echo "Eclipse installation dir is $eclipse_dir"
            shift 2
            ;;

        "--branch")
            branch=$2
            echo "Eclipse repository is $scala_repo-$branch-?"
            shift 2
            ;;

        "list")
            $eclipse_dir/eclipse $eclipse_opt \
                -application $app \
                -repository $eclipse_repo \
                -list
            shift
            ;;

        "install")
            echo "Installing $2.."
            $eclipse_dir/eclipse $eclipse_opt \
                -application $app \
                -repository "$scala_repo-$branch-$2" \
                -installIU org.scala-ide.sdt.feature.feature.group
            shift 2
            ;;

        "uninstall")
            echo "Unnstalling.."
            shift 1
            $eclipse_dir/eclipse $eclipse_opt \
                -application $app \
                -repository "$eclipse_repo" \
                -uninstallIU org.scala-ide.sdt.feature.feature.group
            ;;

        "uninstall-bundle")
            echo "Unnstalling.. $2"
            $eclipse_dir/eclipse $eclipse_opt \
                -application $app \
                -repository "$eclipse_repo" \
                -uninstallIU $2
            shift 2
            ;;

        *)
            usage
            ;;
    esac
done

error: pathspec 'eclipse-director.sh' did not match any file(s) known to git


add kompose installation script

add kompose installation script

#!/usr/bin/env bash
set -eu

KOMPOSE_VERSION="v0.6.0"
curl -L https://github.com/kubernetes-incubator/kompose/releases/download/${KOMPOSE_VERSION}/kompose-linux-amd64 -o /usr/local/bin/kompose
sudo chmod +x /usr/local/bin/whalebrew

error: pathspec 'script/install_kompose.sh' did not match any file(s) known to git


実機上でAPI Keyを登録するためのスクリプトを作成

実機上でAPI Keyを登録するためのスクリプトを作成

#!/bin/bash

# get current registration

clientId=`grep GOOGLE_DEFAULT_CLIENT_ID /etc/chrome_dev.conf | cut -d "=" -f 2`
clientSecret=`grep GOOGLE_DEFAULT_CLIENT_SECRET /etc/chrome_dev.conf | cut -d "=" -f 2`
apiKey=`grep GOOGLE_API_KEY /etc/chrome_dev.conf | cut -d "=" -f 2`

if [ -n "${apiKey}" ]; then
  status="N"
  echo API Key has been already registered.
  echo
  echo "CLIENT_ID:    ${clientId}"
  echo "CLIENT_SECRET:${clientSecret}"
  echo "API_KEY:      ${apiKey}"
  echo
  read -p "Do you want to change API Key?(y/N)" status
  if [ ! "y" = ${status} ]; then
    exit
  fi
fi

status="N"
while [ ! "y" = ${status} ]
do
  read -p "Input CLIENT ID:" clientId
  read -p "Input CLIENT SECRET:" clientSecret
  read -p "Input API KEY:" apiKey
  echo
  echo "CLIENT ID is     " ${clientId}
  echo "CLIENT SECRET is " ${clientSecret}
  echo "API KEY is       " ${apiKey}
  echo
  read -p "Is this correct?(y/N)" status
done

mount -o remount,rw /
if [ 0 -ne $? ]; then
  echo remount root partition failed . Abort.
  exit
fi
sed -e '/^GOOGLE_DEFAULT_CLIENT_ID/d' -i /etc/chrome_dev.conf
sed -e '/^GOOGLE_DEFAULT_CLIENT_SECRET/d' -i /etc/chrome_dev.conf
sed -e '/^GOOGLE_API_KEY/d' -i /etc/chrome_dev.conf

echo GOOGLE_DEFAULT_CLIENT_ID=${clientId} >> /etc/chrome_dev.conf
echo GOOGLE_DEFAULT_CLIENT_SECRET=${clientSecret} >> /etc/chrome_dev.conf
echo GOOGLE_API_KEY=${apiKey} >> /etc/chrome_dev.conf

echo
echo API Key registered.
echo Please restart your machine.

error: pathspec 'script_dev/setupApiKey.sh' did not match any file(s) known to git


Updating code formatting rules.

Updating code formatting rules.

remote: Support for password authentication was removed on August 13, 2021.
remote: Please see https://docs.github.com/en/get-started/getting-started-with-git/about-remote-repositories#cloning-with-https-urls for information on currently recommended modes of authentication.
fatal: Authentication failed for 'https://github.com/d-tk/dtk.git/'


Add simple demo script for desktop Precog

##
##  ____    ____    _____    ____    ___     ____
## |  _ \  |  _ \  | ____|  / ___|  / _/    / ___|        Precog (R)
## | |_) | | |_) | |  _|   | |     | |  /| | |  _         Advanced Analytics Engine for NoSQL Data
## |  __/  |  _ <  | |___  | |___  |/ _| | | |_| |        Copyright (C) 2010 - 2013 SlamData, Inc.
## |_|     |_| \_\ |_____|  \____|   /__/   \____|        All Rights Reserved.
##
## This program is free software: you can redistribute it and/or modify it under the terms of the
## GNU Affero General Public License as published by the Free Software Foundation, either version
## 3 of the License, or (at your option) any later version.
##
## This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY;
## without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See
## the GNU Affero General Public License for more details.
##
## You should have received a copy of the GNU Affero General Public License along with this
## program. If not, see <http://www.gnu.org/licenses/>.
##
##
#!/bin/bash

# Static configuration here!
WORKDIR=/tmp/precogdesktop
ZOOKEEPER_PORT=2181
KAFKA_GLOBAL_PORT=9001
KAFKA_LOCAL_PORT=9011
INGEST_PORT=30060
SHARD_PORT=30070

# Taken from http://blog.publicobject.com/2006/06/canonical-path-of-file-in-bash.html
function path-canonical-simple() {
    local dst="${1}"
    cd -P -- "$(dirname -- "${dst}")" &> /dev/null && echo "$(pwd -P)/$(basename -- "${dst}")" | sed 's#/\.##'
}

function port_is_open() {
   netstat -an | egrep "[\.:]$1[[:space:]]+.*LISTEN" > /dev/null
}

function wait_until_port_open () {
    while ! port_is_open $1; do
        sleep 1
    done
}

BASEDIR=$(path-canonical-simple `dirname $0`)/..

echo "Using base: $BASEDIR"

VERSION=`git describe`
DESKTOP_ASSEMBLY="$BASEDIR"/desktop/target/desktop-assembly-$VERSION.jar
YGGDRASIL_ASSEMBLY="$BASEDIR"/yggdrasil/target/yggdrasil-assembly-$VERSION.jar

GC_OPTS="-XX:+UseConcMarkSweepGC -XX:+CMSIncrementalMode -XX:-CMSIncrementalPacing -XX:CMSIncrementalDutyCycle=100"

JAVA="java $GC_OPTS"

# Make sure we have the tools we need on OSX
if [ `uname` == "Darwin" ]; then
    echo "Running on OSX"
    alias tar="/usr/bin/gnutar"
fi

function exists {
    for fl in "$@"; do
        if [ -f "$fl" ]; then return 0; fi
    done
    return 1
}

# Check for prereqs first
ARTIFACTDIR="$BASEDIR/standaloneArtifacts"

echo "Using artifacts in $ARTIFACTDIR"

[ -d "$ARTIFACTDIR" ] || mkdir -p "$ARTIFACTDIR"

(exists "$ARTIFACTDIR"/zookeeper* && echo "  ZooKeeper exists") || {
    echo "Downloading current ZooKeeper artifact"
    pushd "$ARTIFACTDIR" > /dev/null
    wget -nd -q http://ops.reportgrid.com.s3.amazonaws.com/zookeeper/zookeeper-3.4.3.tar.gz || {
        echo "Failed to download zookeeper" >&2
        exit 3
    }
    popd > /dev/null
}

(exists "$ARTIFACTDIR"/kafka* && echo "  Kafka exists") || {
    echo "Downloading current Kafka artifact"
    pushd "$ARTIFACTDIR" > /dev/null
    wget -nd -q http://s3.amazonaws.com/ops.reportgrid.com/kafka/kafka-0.7.5.zip || {
        echo "Failed to download kafka" >&2
        exit 3
    }
    popd > /dev/null
}

unset REBEL_OPTS
if [ -e "$REBEL_HOME" ]; then
    REBEL_OPTS="-noverify -javaagent:$REBEL_HOME/jrebel.jar -Dplatform.root=`dirname $0`"
else
    REBEL_OPTS=''
fi

# Set up dirs for all components
ZKBASE="$WORKDIR"/zookeeper
ZKDATA="$WORKDIR"/zookeeper-data

KFBASE="$WORKDIR"/kafka
KFGLOBALDATA="$WORKDIR"/kafka-global
KFLOCALDATA="$WORKDIR"/kafka-local

mkdir -p $ZKBASE $KFBASE $ZKDATA "$WORKDIR"/{configs,logs,shard-data/data,shard-data/archive,shard-data/scratch,shard-data/ingest_failures}

echo "Running standalone shard under $WORKDIR"

# Set shutdown hook
function on_exit() {
    echo "========== Shutting down system =========="

    function is_running() {
        [ ! -z "$1" ] && kill -0 "$1" &> /dev/null
    }

    if is_running $INGESTPID; then
        echo "Stopping ingest..."
        kill $INGESTPID
        wait $INGESTPID
    fi

    if is_running $SHARDPID; then
        echo "Stopping shard..."
        kill $SHARDPID
        wait $SHARDPID
    fi

    if is_running $KFGLOBALPID; then
        echo "Stopping kafka..."
        # Kafka is somewhat of a pain, since the Java process daemonizes from within the startup script. That means that killing the script detaches
        # the Java process, leaving it running. Instead, we kill all child processes
        for pid in `ps -o pid,ppid | awk -v PID=$KFGLOBALPID '{ if($2 == PID) print $1}'`; do kill $pid; done
        wait $KFGLOBALPID
    fi

    if is_running $KFLOCALPID; then
        for pid in `ps -o pid,ppid | awk -v PID=$KFLOCALPID '{ if($2 == PID) print $1}'`; do kill $pid; done
        wait $KFLOCALPID
    fi

    echo "Stopping zookeeper..."
    cd $ZKBASE/bin
    ./zkServer.sh stop

    echo "Shutdown complete"
}

trap on_exit EXIT

# Get zookeeper up and running first
pushd $ZKBASE > /dev/null
tar --strip-components=1 --exclude='docs*' --exclude='src*' --exclude='dist-maven*' --exclude='contrib*' --exclude='recipes*' -xvzf "$ARTIFACTDIR"/zookeeper* > /dev/null 2>&1 || {
    echo "Failed to unpack zookeeper" >&2
    exit 3
}
popd > /dev/null

# Copy in a simple config
echo "# the directory where the snapshot is stored." >> $ZKBASE/conf/zoo.cfg
echo "dataDir=$ZKDATA" >> $ZKBASE/conf/zoo.cfg
echo "# the port at which the clients will connect" >> $ZKBASE/conf/zoo.cfg
echo "clientPort=$ZOOKEEPER_PORT" >> $ZKBASE/conf/zoo.cfg

# Set up logging for zookeeper
cat > $ZKBASE/bin/log4j.properties <<EOF
log4j.rootLogger=INFO, file
log4j.appender.file=org.apache.log4j.RollingFileAppender
log4j.appender.file.File=$WORKDIR/logs/zookeeper.log
log4j.appender.file.MaxFileSize=1MB
log4j.appender.file.MaxBackupIndex=1
log4j.appender.file.layout=org.apache.log4j.PatternLayout
log4j.appender.file.layout.ConversionPattern=%d{ABSOLUTE} %5p %c{1}:%L - %m%n
EOF

# Start it up!
echo "Starting zookeeper on port $ZOOKEEPER_PORT"
cd $ZKBASE/bin
./zkServer.sh start &> $WORKDIR/logs/zookeeper.stdout

wait_until_port_open $ZOOKEEPER_PORT

# Now, start global and local kafkas
cd "$WORKDIR"
unzip "$ARTIFACTDIR"/kafka* > /dev/null || {
    echo "Failed to unpack kafka" >&2
    exit 3
}

# Transform the provided config into global and local configs, and start services
cd "$WORKDIR"/kafka/config
chmod +x $KFBASE/bin/kafka-server-start.sh

sed -e "s#log.dir=.*#log.dir=$KFGLOBALDATA#; s/port=.*/port=$KAFKA_GLOBAL_PORT/; s/zk.connect=localhost:2181/zk.connect=localhost:$ZOOKEEPER_PORT/" < server.properties > server-global.properties
$KFBASE/bin/kafka-server-start.sh $KFBASE/config/server-global.properties &> $WORKDIR/logs/kafka-global.stdout &
KFGLOBALPID=$!

wait_until_port_open $KAFKA_GLOBAL_PORT

sed -e "s#log.dir=.*#log.dir=$KFLOCALDATA#; s/port=.*/port=$KAFKA_LOCAL_PORT/; s/enable.zookeeper=.*/enable.zookeeper=false/; s/zk.connect=localhost:2181/zk.connect=localhost:$ZOOKEEPER_PORT/" < server.properties > server-local.properties
$KFBASE/bin/kafka-server-start.sh $KFBASE/config/server-local.properties &> $WORKDIR/logs/kafka-local.stdout &
KFLOCALPID=$!

wait_until_port_open $KAFKA_LOCAL_PORT

echo "Kafka Global = $KFGLOBALPID"
echo "Kafka Local = $KFLOCALPID"

# Set up ingest and shard service configs
sed -e "s#/var/log#$WORKDIR/logs#; s#\[\"localhost\"\]#\[\"localhost:$MONGO_PORT\"\]#; s#/accounts/v1/#/#; s/port = 9082/port = $KAFKA_LOCAL_PORT/; s/port = 9092/port = $KAFKA_GLOBAL_PORT/; s/connect = localhost:2181/connect = localhost:$ZOOKEEPER_PORT/; s/port = 30060/port = $INGEST_PORT/" < "$BASEDIR"/desktop/configs/test/ingest-v1.conf > "$WORKDIR"/configs/ingest-v1.conf || echo "Failed to update ingest config"
sed -e "s#/var/log/precog#$WORKDIR/logs#" < "$BASEDIR"/desktop/configs/test/ingest-v1.logging.xml > "$WORKDIR"/configs/ingest-v1.logging.xml

sed -e "s#/var/log#$WORKDIR/logs#; s#\[\"localhost\"\]#\[\"localhost:$MONGO_PORT\"\]#; s#/opt/precog/shard#$WORKDIR/shard-data#; s/port = 9092/port = $KAFKA_GLOBAL_PORT/; s/hosts = localhost:2181/hosts = localhost:$ZOOKEEPER_PORT/; s/port = 30070/port = $SHARD_PORT/; s/port = 30064/port = $ACCOUNTS_PORT/" < "$BASEDIR"/desktop/configs/test/shard-v1.conf > "$WORKDIR"/configs/shard-v1.conf || echo "Failed to update shard config"
sed -e "s#/var/log/precog#$WORKDIR/logs#" < "$BASEDIR"/desktop/configs/test/shard-v1.logging.xml > "$WORKDIR"/configs/shard-v1.logging.xml

cd "$BASEDIR"

# Prior to ingest startup, we need to set an initial checkpoint if it's not already there
if [ ! -e "$WORKDIR"/initial_checkpoint.json ]; then
    $JAVA $REBEL_OPTS -jar "$YGGDRASIL_ASSEMBLY" zk -z "localhost:$ZOOKEEPER_PORT" -uc "/precog-desktop/shard/checkpoint/`hostname`:{\"offset\":0, \"messageClock\":[]}" &> $WORKDIR/logs/checkpoint_init.stdout || {
        echo "Couldn't set initial checkpoint!" >&2
        exit 3
    }
    touch "$WORKDIR"/initial_checkpoint.json
fi

echo "Starting ingest service"
$JAVA $REBEL_OPTS -Dlogback.configurationFile="$WORKDIR"/configs/ingest-v1.logging.xml -classpath "$DESKTOP_ASSEMBLY" com.precog.standalone.StandaloneIngestServer --configFile "$WORKDIR"/configs/ingest-v1.conf &> $WORKDIR/logs/ingest-v1.stdout &
INGESTPID=$!

echo "Starting shard service"
$JAVA $REBEL_OPTS -Dlogback.configurationFile="$WORKDIR"/configs/shard-v1.logging.xml -classpath "$DESKTOP_ASSEMBLY" com.precog.shard.desktop.DesktopShardServer --configFile "$WORKDIR"/configs/shard-v1.conf &> $WORKDIR/logs/shard-v1.stdout &
SHARDPID=$!

# Let the ingest/shard services startup in parallel
wait_until_port_open $INGEST_PORT
wait_until_port_open $SHARD_PORT

cat > $WORKDIR/ports.txt <<EOF
KAFKA_LOCAL_PORT=$KAFKA_LOCAL_PORT
KAFKA_GLOBAL_PORT=$KAFKA_GLOBAL_PORT
ZOOKEEPER_PORT=$ZOOKEEPER_PORT
INGEST_PORT=$INGEST_PORT
SHARD_PORT=$SHARD_PORT
EOF

echo "Startup complete, running in $WORKDIR"

echo "============================================================"
echo "Base path: $WORKDIR"
cat <<EOF
KAFKA_LOCAL_PORT:  $KAFKA_LOCAL_PORT
KAFKA_GLOBAL_PORT: $KAFKA_GLOBAL_PORT
ZOOKEEPER_PORT:    $ZOOKEEPER_PORT
INGEST_PORT:       $INGEST_PORT
SHARD_PORT:        $SHARD_PORT
EOF
echo "============================================================"

function query() {
    curl -s -G \
      --data-urlencode "q=$1" \
      --data-urlencode "apiKey=$ACCOUNTTOKEN" \
      "http://localhost:$SHARD_PORT/analytics/fs/$ACCOUNTID"
}

function count() {
    query "count(//xyz)" | tr -d "[]"
}

function wait_til_nonzero() {
    wait_til_n_rows 1 $1
    return $?
}

function now() {
    date "+%s"
}

function check_time() {
    expr `now` '>' $1
}

function wait_til_n_rows() {
    N=$1
    LIMIT=$( expr `now` '+' $2 )
    RESULT=$( count )
    echo "!!! count returned $RESULT"
    while [ -z "$RESULT" ] || [ "$RESULT" -lt "$N" ]; do
        sleep 0.05
        [ `check_time $LIMIT` -eq 1 ] && return 1
        RESULT=$( count )
        echo "!!! count returned $RESULT"
    done
    return 0
}

function count_lines() {
    wc -l $1 | awk '{print $1}'
}

TESTJSON="n100k.json"
TESTURL="http://ops.reportgrid.com.s3.amazonaws.com/datasets/$TESTJSON"

function download_testjson() {
    echo "downloading json"
    wget $TESTURL
    if [ $? -ne 0 ]; then
        echo "Failed to download $TESTURL" >&2
        exit 3
    fi
    echo "done"
}

if [ -n "$TESTQUIT" ]; then

    echo "trying??"
    ( [ -r $TESTJSON ] && [ `count_lines $TESTJSON` -eq 100000 ] ) || download_testjson
    echo "what????"

    echo ";;; ingesting $TESTJSON"
    curl -o /dev/null -v \
      -H 'Content-Type: application/json' \
      --data-bin "@$TESTJSON" \
      "http://localhost:$INGEST_PORT/sync/fs/$ACCOUNTID/xyz?apiKey=$ACCOUNTTOKEN"

    echo ";;; polling for rows via count()"
    wait_til_nonzero 60

    trap EXIT

    if [ $? -eq 0 ]; then
        echo ";;; ingest rows detected--killing service now!"
        kill -9 $INGESTPID
        kill -9 $SHARDPID
        kill -9 $ACCOUNTSPID
        kill -9 $JOBSPID
        kill -9 $AUTHPID
        on_exit
        exit 0
    else
        echo ";;; no rows ingested after 60s--failed!"
        on_exit
        exit 1
    fi
fi

if [ -n "$TESTRESUME" ]; then
    NROWS=`wc -l $TESTJSON | awk '{print $1}'`
    wait_til_n_rows $NROWS 60
    trap EXIT
    echo ";;; verifying that shard resumes ingest"
    if [ $? -eq 0 ]; then
        echo ";;; resume succeeded (found $NROWS rows)"
        echo ";;; ok"
        on_exit
        exit 0
    else
        echo ";;; resume failed (timed out before seeing $NROWS rows)"
        echo ";;; ERROR!"
        on_exit 1
        exit 1
    fi
fi

# Wait forever until the user Ctrl-C's the system
while true; do sleep 30; done

error: pathspec 'desktop/simple.sh' did not match any file(s) known to git


adding missing file

adding missing file

#!/usr/bin/env bash
##
# This section should match your Makefile
##
PY=python
PELICAN=pelican
PELICANOPTS=

BASEDIR=$(pwd)
INPUTDIR=$BASEDIR/content
OUTPUTDIR=$BASEDIR/output
CONFFILE=$BASEDIR/pelicanconf.py

###
# Don't change stuff below here unless you are sure
###

SRV_PID=$BASEDIR/srv.pid
PELICAN_PID=$BASEDIR/pelican.pid

function usage(){
  echo "usage: $0 (stop) (start) (restart) [port]"
  echo "This starts pelican in debug and reload mode and then launches"
  echo "A pelican.server to help site development. It doesn't read"
  echo "your pelican options so you edit any paths in your Makefile"
  echo "you will need to edit it as well"
  exit 3
}

function alive() {
  kill -0 $1 >/dev/null 2>&1
}

function shut_down(){
  PID=$(cat $SRV_PID)
  if [[ $? -eq 0 ]]; then
    if alive $PID; then
      echo "Killing pelican.server"
      kill $PID
    else
      echo "Stale PID, deleting"
    fi
    rm $SRV_PID
  else
    echo "pelican.server PIDFile not found"
  fi

  PID=$(cat $PELICAN_PID)
  if [[ $? -eq 0 ]]; then
    if alive $PID; then
      echo "Killing Pelican"
      kill $PID
    else
      echo "Stale PID, deleting"
    fi
    rm $PELICAN_PID
  else
    echo "Pelican PIDFile not found"
  fi
}

function start_up(){
  local port=$1
  echo "Starting up Pelican and pelican.server"
  shift
  $PELICAN --debug --autoreload -r $INPUTDIR -o $OUTPUTDIR -s $CONFFILE $PELICANOPTS &
  pelican_pid=$!
  echo $pelican_pid > $PELICAN_PID
  cd $OUTPUTDIR
  $PY -m pelican.server $port &
  srv_pid=$!
  echo $srv_pid > $SRV_PID
  cd $BASEDIR
  sleep 1
  if ! alive $pelican_pid ; then
    echo "Pelican didn't start. Is the pelican package installed?"
    return 1
  elif ! alive $srv_pid ; then
    echo "pelican.server didn't start. Is there something else which uses port 8000?"
    return 1
  fi
  echo 'Pelican and pelican.server processes now running in background.'
}

###
#  MAIN
###
[[ ($# -eq 0) || ($# -gt 2) ]] && usage
port=''
[[ $# -eq 2 ]] && port=$2

if [[ $1 == "stop" ]]; then
  shut_down
elif [[ $1 == "restart" ]]; then
  shut_down
  start_up $port
elif [[ $1 == "start" ]]; then
  if ! start_up $port; then
    shut_down
  fi
else
  usage
fi

error: pathspec 'develop_server.sh' did not match any file(s) known to git


Set initial user login group

Set initial user login group

remote: Support for password authentication was removed on August 13, 2021.
remote: Please see https://docs.github.com/en/get-started/getting-started-with-git/about-remote-repositories#cloning-with-https-urls for information on currently recommended modes of authentication.
fatal: Authentication failed for 'https://github.com/hwong/provisioning.git/'


No bug - Update INCOMPLETE_LOCALES to remove lo

remote: Support for password authentication was removed on August 13, 2021.
remote: Please see https://docs.github.com/en/get-started/getting-started-with-git/about-remote-repositories#cloning-with-https-urls for information on currently recommended modes of authentication.
fatal: Authentication failed for 'https://github.com/kesun421/firefox-ios.git/'


fix possible depend issue

fix possible depend issue

remote: Support for password authentication was removed on August 13, 2021.
remote: Please see https://docs.github.com/en/get-started/getting-started-with-git/about-remote-repositories#cloning-with-https-urls for information on currently recommended modes of authentication.
fatal: Authentication failed for 'https://github.com/netdrg/hstacklet.git/'


Added update to subversion install

Added update to subversion install

Fresh images are getting old so update is needed to pull down packages.

cfg/Install Eazy Hax RetroPie Toolkit.sh

error: pathspec 'cfg/Install' did not match any file(s) known to git
error: pathspec 'Eazy' did not match any file(s) known to git
error: pathspec 'Hax' did not match any file(s) known to git
error: pathspec 'RetroPie' did not match any file(s) known to git
error: pathspec 'Toolkit.sh' did not match any file(s) known to git


use java 7

use java 7

remote: Support for password authentication was removed on August 13, 2021.
remote: Please see https://docs.github.com/en/get-started/getting-started-with-git/about-remote-repositories#cloning-with-https-urls for information on currently recommended modes of authentication.
fatal: Authentication failed for 'https://github.com/Apptimize/ve.git/'


[skip ci] change/add-to env options to support travis run of mcapi/python, the python API

#!/usr/bin/env bash

# travisrun.sh - to support travis tests of mcapi/python, the Python API
# initially a copy of dev.sh (and should probaly be maintained as that)

# Note: often this will be redundent, but it simplifies use
# source to set all environment variables for the type of server
export SERVERTYPE=dev

export MC_SERVICE_PORT=5002
export MC_API_SERVICE_PORT=5004
export MC_PUB_SERVICE_PORT=5026
export MCDB_PORT=30815
export MCDB_CONNECTION="localhost:$MCDB_PORT"
export MCDB_DIR=~/testdb
export RETHINKDB_HTTP_PORT=8090
export RETHINKDB_CLUSTER_PORT=31815
export MCSTOREDBIN=testbin/mcstored
export MC_ES_URL="http://localhost:9500"
export MC_ES_NAME="mc-es-test"
export MC_LOG_DIR=/tmp
if [ "$MCDB_FILE" = "" ]; then
    export MCDB_FILE=../test_data/rethinkdb_dump_test_data.tar.gz
fi
export MCDIR=~/mcdir/mcfs/data/test:/mcfs/data/materialscommons
export MCFS_HTTP_PORT=5012

if [ -f /etc/materialscommons/config.dev ]; then
    . /etc/materialscommons/config.dev
fi

if [ ! -d ${MCDB_DIR} ]; then
    mkdir -p ${MCDB_DIR}
fi

if [ "$REINIT" = "t" ]; then
    (cd ${MCDB_DIR}; rm -rf rethinkdb_data)
fi

error: pathspec 'backend/env/travisrun.sh' did not match any file(s) known to git


Linux Script

Linux Script

Install and Run /LINUX/runGUIdockLinux.sh

fatal: Invalid path '/LINUX': No such file or directory


Remove outer TEST_TIME dependency

Remove outer TEST_TIME dependency

#!/bin/bash -x

# receive list of apps to run coldlaunch test against
# i.e. 'calendar clock communications@dialer email'
APP_LIST=$1
JOB_TIME=$(node -e "console.log(Date.now());")

post_to_treeherder() {
  local RAPTOR_BUILD_STATE=$1
  local RAPTOR_APP_NAME=$2
  local TEST_START_TIME=$3
  local TEST_FINISH_TIME=$4
  local TEST_FAILURE=$5

  python --version
  source raptor-env/bin/activate
  python --version

  TEST_TIME="${JOB_TIME}" post-to-treeherder/submit-to-treeherder.py \
    --repository b2g-inbound \
    --build-state ${RAPTOR_BUILD_STATE} \
    --treeherder-url https://treeherder.allizom.org/ \
    --treeherder-client-id raptor \
    --treeherder-secret ${RAPTOR_TREEHERDER_SECRET} \
    --test-type cold-launch \
    --app-name ${RAPTOR_APP_NAME} \
    --start-time ${TEST_START_TIME} \
    --finish-time ${TEST_FINISH_TIME} \
    --test-failure ${TEST_FAILURE}
}

run_coldlaunch_test() {
  local APP_ORIGIN=$1
  local ENTRY_POINT=$2

  APP_FAILURE=0

  local FILE_FORMAT="${APP_ORIGIN}"
  local ENTRY_FLAG=""

  # run coldlaunch
  if [ -n "$ENTRY_POINT" ] ; then
    local FILE_FORMAT="${APP_ORIGIN}-${ENTRY_POINT}"
    local ENTRY_FLAG="--entry-point ${ENTRY_POINT}"
  fi

  if [ -n "$RAPTOR_DATABASE_NAME" ] ; then
    local DATABASE_FLAG="--database ${RAPTOR_DATABASE_NAME}"
  fi

  DEBUG=* ${PWD}/node_modules/.bin/raptor test coldlaunch \
    --serial ${ANDROID_SERIAL} \
    --app ${APP_ORIGIN} \
    --runs ${RUNS} \
    --timeout ${TIMEOUT} \
    --time ${JOB_TIME} \
    --metrics ${FILE_FORMAT}.ldjson \
    --logcat ${FILE_FORMAT}.logcat \
    ${ENTRY_FLAG} 2>&1 | tee ${FILE_FORMAT}.log

  APP_FAILURE=${PIPESTATUS[0]}

  local LINES="$(wc -l < ${FILE_FORMAT}.ldjson)"
  local EXPECTED="$((RUNS + 1))"

  # submit results to db
  if [[ ${LINES} -eq ${EXPECTED} ]] && [[ ${APP_FAILURE} == 0 ]] ; then
    DEBUG=* ${PWD}/node_modules/.bin/raptor submit ${FILE_FORMAT}.ldjson \
      --host ${RAPTOR_HOST} \
      --port ${RAPTOR_PORT} \
      --username ${RAPTOR_USERNAME} \
      --password ${RAPTOR_PASSWORD} \
      --protocol ${RAPTOR_PROTOCOL} \
      ${DATABASE_FLAG}
  else
    APP_FAILURE=1
    SUITE_FAILURE=1
  fi
}

# main coldlaunch loop
SUITE_FAILURE=0
APP_FAILURE=0

for i in ${APP_LIST[@]}
do
  APP=${i}

  # origin@entrypoint
  if [[ "${APP}" =~ "@" ]] ; then
    ORIGIN=${APP%@*}
    ENTRY=${APP#*@}
  else
    ORIGIN=${APP}
    ENTRY=''
  fi

  APP_START_TIME=$(node -e "console.log(Date.now());")
  post_to_treeherder running ${APP} ${APP_START_TIME}

  run_coldlaunch_test ${ORIGIN} ${ENTRY}

  APP_FINISH_TIME=$(node -e "console.log(Date.now());")
  post_to_treeherder completed ${APP} ${APP_START_TIME} ${APP_FINISH_TIME} ${APP_FAILURE}
done

exit ${SUITE_FAILURE}

#!/bin/bash -x

# receive list of apps to run coldlaunch test against
# i.e. 'calendar clock communications@dialer email'
APP_LIST=$1

post_to_treeherder() {
  local RAPTOR_BUILD_STATE=$1
  local RAPTOR_APP_NAME=$2
  local TEST_START_TIME=$3
  local TEST_FINISH_TIME=$4
  local TEST_FAILURE=$5

  python --version
  source raptor-env/bin/activate
  python --version

  post-to-treeherder/submit-to-treeherder.py \
    --repository b2g-inbound \
    --build-state ${RAPTOR_BUILD_STATE} \
    --treeherder-url https://treeherder.allizom.org/ \
    --treeherder-client-id raptor \
    --treeherder-secret ${RAPTOR_TREEHERDER_SECRET} \
    --test-type cold-launch \
    --app-name ${RAPTOR_APP_NAME} \
    --start-time ${TEST_START_TIME} \
    --finish-time ${TEST_FINISH_TIME} \
    --test-failure ${TEST_FAILURE}
}

run_coldlaunch_test() {
  local APP_ORIGIN=$1
  local ENTRY_POINT=$2

  APP_FAILURE=0

  local FILE_FORMAT="${APP_ORIGIN}"
  local ENTRY_FLAG=""

  # run coldlaunch
  if [ -n "$ENTRY_POINT" ] ; then
    local FILE_FORMAT="${APP_ORIGIN}-${ENTRY_POINT}"
    local ENTRY_FLAG="--entry-point ${ENTRY_POINT}"
  fi

  if [ -n "$RAPTOR_DATABASE_NAME" ] ; then
    local DATABASE_FLAG="--database ${RAPTOR_DATABASE_NAME}"
  fi

  DEBUG=* ${PWD}/node_modules/.bin/raptor test coldlaunch \
    --serial ${ANDROID_SERIAL} \
    --app ${APP_ORIGIN} \
    --runs ${RUNS} \
    --timeout ${TIMEOUT} \
    --time ${TEST_TIME} \
    --metrics ${FILE_FORMAT}.ldjson \
    --logcat ${FILE_FORMAT}.logcat \
    ${ENTRY_FLAG} 2>&1 | tee ${FILE_FORMAT}.log

  APP_FAILURE=${PIPESTATUS[0]}

  local LINES="$(wc -l < ${FILE_FORMAT}.ldjson)"
  local EXPECTED="$((RUNS + 1))"

  # submit results to db
  if [[ ${LINES} -eq ${EXPECTED} ]] && [[ ${APP_FAILURE} == 0 ]] ; then
    DEBUG=* ${PWD}/node_modules/.bin/raptor submit ${FILE_FORMAT}.ldjson \
      --host ${RAPTOR_HOST} \
      --port ${RAPTOR_PORT} \
      --username ${RAPTOR_USERNAME} \
      --password ${RAPTOR_PASSWORD} \
      --protocol ${RAPTOR_PROTOCOL} \
      ${DATABASE_FLAG}
  else
    APP_FAILURE=1
    SUITE_FAILURE=1
  fi
}

# main coldlaunch loop
SUITE_FAILURE=0
APP_FAILURE=0

for i in ${APP_LIST[@]}
do
  APP=${i}

  # origin@entrypoint
  if [[ "${APP}" =~ "@" ]] ; then
    ORIGIN=${APP%@*}
    ENTRY=${APP#*@}
  else
    ORIGIN=${APP}
    ENTRY=''
  fi

  APP_START_TIME=$(node -e "console.log(Date.now());")
  post_to_treeherder running ${APP} ${APP_START_TIME}

  run_coldlaunch_test ${ORIGIN} ${ENTRY}

  APP_FINISH_TIME=$(node -e "console.log(Date.now());")
  post_to_treeherder completed ${APP} ${APP_START_TIME} ${APP_FINISH_TIME} ${APP_FAILURE}
done

exit ${SUITE_FAILURE}