Create for Oracle Linux Script

Create for Oracle Linux Script

#!/bin/bash -v

# Logger
exec > >(tee /var/log/user-data_3rd-bootstrap.log || logger -t user-data -s 2> /dev/console) 2>&1

#-------------------------------------------------------------------------------
# Set UserData Parameter
#-------------------------------------------------------------------------------

if [ -f /tmp/userdata-parameter ]; then
    source /tmp/userdata-parameter
fi

if [[ -z "${Language}" || -z "${Timezone}" || -z "${VpcNetwork}" ]]; then
    # Default Language
	Language="ja_JP.UTF-8"
    # Default Timezone
	Timezone="Asia/Tokyo"
	# Default VPC Network
	VpcNetwork="IPv4"
fi

# echo
echo $Language
echo $Timezone
echo $VpcNetwork

#-------------------------------------------------------------------------------
# Yum Configuration
#-------------------------------------------------------------------------------
yum clean all
yum install -y yum-utils
yum clean all

# Delete AMI Defalut YUM Repositories File
rm -rf /etc/yum.repos.d/public-yum-ol7.repo

# Add Oralce Linux v7 Public YUM Repositories File
yum-config-manager --add-repo http://public-yum.oracle.com/public-yum-ol7.repo

#-------------------------------------------------------------------------------
# Enable Repositories (Oracle Linux v7) - [Default Enable]
#  http://yum.oracle.com/oracle-linux-7.html
#-------------------------------------------------------------------------------

# Latest packages released for Oracle Linux 7.
#  http://yum.oracle.com/repo/OracleLinux/OL7/latest/x86_64/index.html
yum-config-manager --enable ol7_latest

# Latest Unbreakable Enterprise Kernel Release 4 packages for Oracle Linux 7.
#  http://yum.oracle.com/repo/OracleLinux/OL7/UEKR4/x86_64/index.html
yum-config-manager --enable ol7_UEKR4

#-------------------------------------------------------------------------------
# Enable Repositories (Oracle Linux v7) - [Default Disable]
#  http://yum.oracle.com/oracle-linux-7.html
#-------------------------------------------------------------------------------

# Latest packages released for Oracle Linux 7.
#  http://yum.oracle.com/repo/OracleLinux/OL7/optional/latest/x86_64/index.html
yum-config-manager --enable ol7_optional_latest

# Latest add-on packages for Oracle Linux 7.
#  http://yum.oracle.com/repo/OracleLinux/OL7/addons/x86_64/index.html
yum-config-manager --enable ol7_addons

# Latest packages for test and development for Oracle Linux 7.
#  http://yum.oracle.com/repo/OracleLinux/OL7/developer/x86_64/index.html
yum-config-manager --enable ol7_developer

# Latest EPEL packages for test and development for Oracle Linux 7.
#  http://yum.oracle.com/repo/OracleLinux/OL7/developer_EPEL/x86_64/index.html
yum-config-manager --enable ol7_developer_EPEL

# Latest Software Collection Library packages for Oracle Linux 7.
#  http://yum.oracle.com/repo/OracleLinux/OL7/SoftwareCollections/x86_64/index.html
# yum-config-manager --enable ol7_software_collections

#-------------------------------------------------------------------------------
# Default Package Update
#-------------------------------------------------------------------------------

# yum repository metadata Clean up
yum clean all

# Default Package Update
yum update -y

#-------------------------------------------------------------------------------
# Custom Package Installation
#-------------------------------------------------------------------------------

# Package Install Oracle Linux System Administration Tools (from Oracle Linux Official Repository)
yum install -y arptables bash-completion bind-utils dstat ebtables gdisk git hdparm lsof lzop iotop mtr nc nmap sos sysstat tcpdump traceroute vim-enhanced yum-priorities yum-plugin-versionlock wget
yum install -y setroubleshoot-server

# Package Install Oracle Linux System Administration Tools (from Oracle Linux EPEL Repository)
yum --enablerepo=ol7_developer_EPEL install -y jq cloud-utils-growpart

# Package Install EPEL(Extra Packages for Enterprise Linux) Repository Package
# yum localinstall -y https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm

cat > /etc/yum.repos.d/epel-bootstrap.repo << __EOF__
[epel]
name=Bootstrap EPEL
mirrorlist=https://mirrors.fedoraproject.org/metalink?repo=epel-7&arch=\$basearch
failovermethod=priority
enabled=0
gpgcheck=0
__EOF__

yum --enablerepo=epel -y install epel-release
rm -f /etc/yum.repos.d/epel-bootstrap.repo

sed -i 's/enabled=1/enabled=0/g' /etc/yum.repos.d/epel.repo
# yum-config-manager --disable epel epel-debuginfo epel-source

yum clean all

# Package Install Oracle Linux System Administration Tools (from EPEL Repository)
yum --enablerepo=epel install -y atop collectl fio

#-------------------------------------------------------------------------------
# Set AWS Instance MetaData
#-------------------------------------------------------------------------------

# Instance MetaData
AZ=$(curl -s "http://169.254.169.254/latest/meta-data/placement/availability-zone")
Region=$(echo $AZ | sed -e 's/.$//g')
InstanceId=$(curl -s "http://169.254.169.254/latest/meta-data/instance-id")
InstanceType=$(curl -s "http://169.254.169.254/latest/meta-data/instance-type")
PrivateIp=$(curl -s "http://169.254.169.254/latest/meta-data/local-ipv4")
AmiId=$(curl -s "http://169.254.169.254/latest/meta-data/ami-id")

# IAM Role & STS Information
RoleArn=$(curl -s "http://169.254.169.254/latest/meta-data/iam/info" | jq -r '.InstanceProfileArn')
RoleName=$(echo $RoleArn | cut -d '/' -f 2)

StsCredential=$(curl -s "http://169.254.169.254/latest/meta-data/iam/security-credentials/$RoleName")
StsAccessKeyId=$(echo $StsCredential | jq -r '.AccessKeyId')
StsSecretAccessKey=$(echo $StsCredential | jq -r '.SecretAccessKey')
StsToken=$(echo $StsCredential | jq -r '.Token')

# AWS Account ID
AwsAccountId=$(curl -s "http://169.254.169.254/latest/dynamic/instance-identity/document" | jq -r '.accountId')

#-------------------------------------------------------------------------------
# Custom Package Installation [AWS-CLI]
#-------------------------------------------------------------------------------
yum --enablerepo=epel install -y python2-pip
pip install --upgrade pip

pip install awscli

cat > /etc/profile.d/aws-cli.sh << __EOF__
if [ -n "\$BASH_VERSION" ]; then
   complete -C /usr/bin/aws_completer aws
fi
__EOF__

aws --version

# Setting AWS-CLI default Region & Output format
aws configure << __EOF__

${Region}
json

__EOF__

sleep 3

# Getting AWS-CLI default Region & Output format
aws configure list
cat ~/.aws/config

# Get AWS Region Information
if [ -n "$RoleName" ]; then
	echo "# Get AWS Region Infomation"
	aws ec2 describe-regions --region ${Region}
fi

# Get AMI Information
if [ -n "$RoleName" ]; then
	echo "# Get AMI Information"
	aws ec2 describe-images --image-ids ${AmiId} --output json --region ${Region}
fi

# Get EC2 Instance Information
if [ -n "$RoleName" ]; then
	echo "# Get EC2 Instance Information"
	aws ec2 describe-instances --instance-ids ${InstanceId} --output json --region ${Region}
fi

# Get EC2 Instance attached EBS Volume Information
if [ -n "$RoleName" ]; then
	echo "# Get EC2 Instance attached EBS Volume Information"
	aws ec2 describe-volumes --filters Name=attachment.instance-id,Values=${InstanceId} --output json --region ${Region}
fi

# Get EC2 Instance Attribute[Network Interface Performance Attribute]
#
# - ENA (Elastic Network Adapter)
#   http://docs.aws.amazon.com/ja_jp/AWSEC2/latest/UserGuide/enhanced-networking-ena.html
# - SR-IOV
#   http://docs.aws.amazon.com/ja_jp/AWSEC2/latest/UserGuide/sriov-networking.html
#
if [ -n "$RoleName" ]; then
	if [[ "$InstanceType" =~ ^(e3.*|f1.*|g3.*|i3.*|p2.*|r4.*|x1.*|x1e.*|m4.16xlarge)$ ]]; then
		# Get EC2 Instance Attribute(Elastic Network Adapter Status)
		echo "# Get EC2 Instance Attribute(Elastic Network Adapter Status)"
		aws ec2 describe-instances --instance-id ${InstanceId} --query Reservations[].Instances[].EnaSupport --output json --region ${Region}
		echo "# Get Linux Kernel Module(modinfo ena)"
		modinfo ena
	elif [[ "$InstanceType" =~ ^(c3.*|c4.*|d2.*|i2.*|r3.*|m4.*)$ ]]; then
		# Get EC2 Instance Attribute(Single Root I/O Virtualization Status)
		echo "# Get EC2 Instance Attribute(Single Root I/O Virtualization Status)"
		aws ec2 describe-instance-attribute --instance-id ${InstanceId} --attribute sriovNetSupport --output json --region ${Region}
		echo "# Get Linux Kernel Module(modinfo ixgbevf)"
		modinfo ixgbevf
	else
		echo "# Not Target Instance Type :" $InstanceType
	fi
fi

# Get EC2 Instance Attribute[Storage Interface Performance Attribute]
#
# - EBS Optimized Instance
#   http://docs.aws.amazon.com/ja_jp/AWSEC2/latest/UserGuide/EBSOptimized.html
#
if [ -n "$RoleName" ]; then
	if [[ "$InstanceType" =~ ^(c1.*|c3.*|c4.*|d2.*|e3.*|f1.*|g2.*|g3.*|i2.*|i3.*|m1.*|m2.*|m3.*|m4.*|p2.*|r3.*|r4.*|x1.*|x1e.*)$ ]]; then
		# Get EC2 Instance Attribute(EBS-optimized instance Status)
		echo "# Get EC2 Instance Attribute(EBS-optimized instance Status)"
		aws ec2 describe-instance-attribute --instance-id ${InstanceId} --attribute ebsOptimized --output json --region ${Region}
		echo "# Get Linux Block Device Read-Ahead Value(blockdev --report)"
		blockdev --report
	else
		echo "# Get Linux Block Device Read-Ahead Value(blockdev --report)"
		blockdev --report
	fi
fi

#-------------------------------------------------------------------------------
# Custom Package Installation [AWS CloudFormation Helper Scripts]
#-------------------------------------------------------------------------------
# yum --enablerepo=epel localinstall -y https://s3.amazonaws.com/cloudformation-examples/aws-cfn-bootstrap-latest.amzn1.noarch.rpm
# yum --enablerepo=epel install -y python2-pip
# pip install --upgrade pip

pip install pystache
pip install argparse
pip install python-daemon
pip install requests

curl https://s3.amazonaws.com/cloudformation-examples/aws-cfn-bootstrap-latest.tar.gz -o /tmp/aws-cfn-bootstrap-latest.tar.gz
tar -pxvzf /tmp/aws-cfn-bootstrap-latest.tar.gz -C /tmp

cd /tmp/aws-cfn-bootstrap-1.4/
python setup.py build
python setup.py install

chmod 775 /usr/init/redhat/cfn-hup
ln -s /usr/init/redhat/cfn-hup /etc/init.d/cfn-hup

cd /tmp

#-------------------------------------------------------------------------------
# Custom Package Installation [AWS Systems Service Manager (aka SSM) agent]
#-------------------------------------------------------------------------------
# yum localinstall -y "https://s3.amazonaws.com/ec2-downloads-windows/SSMAgent/latest/linux_amd64/amazon-ssm-agent.rpm"

yum localinstall -y "https://amazon-ssm-${Region}.s3.amazonaws.com/latest/linux_amd64/amazon-ssm-agent.rpm"

systemctl daemon-reload

systemctl status -l amazon-ssm-agent
systemctl enable amazon-ssm-agent
systemctl is-enabled amazon-ssm-agent

systemctl restart amazon-ssm-agent
systemctl status -l amazon-ssm-agent

ssm-cli get-instance-information

#-------------------------------------------------------------------------------
# Custom Package Installation [Amazon EC2 Rescue for Linux (ec2rl)]
# http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/Linux-Server-EC2Rescue.html
#-------------------------------------------------------------------------------

# Package Download Amazon Linux System Administration Tools (from S3 Bucket)
curl -sS "https://s3.amazonaws.com/ec2rescuelinux/ec2rl.tgz" -o "/tmp/ec2rl.tgz"

mkdir -p "/opt/aws"

tar -xzvf "/tmp/ec2rl.tgz" -C "/opt/aws"

cat > /etc/profile.d/ec2rl.sh << __EOF__
export PATH=\$PATH:/opt/aws/ec2rl
__EOF__

# Check Version
/opt/aws/ec2rl/ec2rl version

/opt/aws/ec2rl/ec2rl version-check

# Required Software Package
/opt/aws/ec2rl/ec2rl software-check

# Diagnosis [dig modules]
# /opt/aws/ec2rl/ec2rl run --only-modules=dig --domain=amazon.com

#-------------------------------------------------------------------------------
# Custom Package Installation [Ansible]
#-------------------------------------------------------------------------------

# Package Install Oracle Linux System Administration Tools (from Oracle Linux EPEL Repository)
yum --enablerepo=ol7_developer_EPEL install -y ansible ansible-doc

ansible --version

ansible localhost -m setup

#-------------------------------------------------------------------------------
# Custom Package Installation [Oracle Developer Package:PowerShell]
#-------------------------------------------------------------------------------

# Package Install Oracle Linux System Administration Tools (from Oracle Linux Development Repository)
# yum --enablerepo=ol7_developer install -y powershell

# powershell --version

#-------------------------------------------------------------------------------
# Custom Package Installation [Oracle Developer Package:td-agent]
#-------------------------------------------------------------------------------

# Package Install Oracle Linux System Administration Tools (from Oracle Linux Development Repository)
# yum --enablerepo=ol7_developer install -y td-agent

# /opt/td-agent/usr/sbin/td-agent --version
# /opt/td-agent/usr/bin/td --version

# cat /etc/td-agent/td-agent.conf

# systemctl daemon-reload

# systemctl status -l td-agent
# systemctl enable td-agent
# systemctl is-enabled td-agent

# systemctl restart td-agent
# systemctl status -l td-agent

# Package Install Fluentd (td-agent) Gem Packages (from Ruby Gem Package)

# /opt/td-agent/usr/sbin/td-agent-gem list

# /opt/td-agent/usr/sbin/td-agent-gem search -r fluent-plugin

# /opt/td-agent/usr/sbin/td-agent-gem install fluent-plugin-aws-elasticsearch-service
# /opt/td-agent/usr/sbin/td-agent-gem install fluent-plugin-cloudwatch-logs
# /opt/td-agent/usr/sbin/td-agent-gem install fluent-plugin-kinesis
# /opt/td-agent/usr/sbin/td-agent-gem install fluent-plugin-kinesis-firehose

# /opt/td-agent/usr/sbin/td-agent-gem update fluent-plugin-s3

# /opt/td-agent/usr/sbin/td-agent-gem list

#-------------------------------------------------------------------------------
# Custom Package Installation [Oracle Developer Package:Terraform]
#-------------------------------------------------------------------------------

# Package Install Oracle Linux System Administration Tools (from Oracle Linux Development Repository)
# yum --enablerepo=ol7_developer install -y terraform

# terraform --version

#-------------------------------------------------------------------------------
# Custom Package Clean up
#-------------------------------------------------------------------------------
yum clean all

#-------------------------------------------------------------------------------
# System Setting
#-------------------------------------------------------------------------------

# CPU Information [cat /proc/cpuinfo]
cat /proc/cpuinfo

# CPU Information [lscpu]
lscpu

lscpu --extended

# Memory Information [cat /proc/meminfo]
cat /proc/meminfo

# Memory Information [free]
free

# Disk Information(Partition) [parted -l]
parted -l

# Disk Information(MountPoint) [lsblk]
lsblk

# Disk Information(File System) [df -h]
df -h

# Network Information(Network Interface) [ip addr show]
ip addr show

# Network Information(Routing Table) [ip route show]
ip route show

# Network Information(Firewall Service) [firewalld]
if [ $(command -v firewall-cmd) ]; then
    # Network Information(Firewall Service) [systemctl status -l firewalld]
    systemctl status -l firewalld
    # Network Information(Firewall Service) [firewall-cmd --list-all]
    firewall-cmd --list-all
fi

# Linux Security Information(SELinux) [getenforce] [sestatus]
getenforce

sestatus

#-------------------------------------------------------------------------------
# System Setting
#-------------------------------------------------------------------------------

# NTP Service Enabled(chronyd)
systemctl restart chronyd
systemctl enable chronyd
systemctl is-enabled chronyd
sleep 3
chronyc tracking
chronyc sources -v
chronyc sourcestats -v

# Setting SystemClock and Timezone
if [ "${Timezone}" = "Asia/Tokyo" ]; then
	echo "# Setting SystemClock and Timezone -> $Timezone"
	date
	# timedatectl status
	timedatectl set-timezone Asia/Tokyo
	date
	# timedatectl status
elif [ "${Timezone}" = "UTC" ]; then
	echo "# Setting SystemClock and Timezone -> $Timezone"
	date
	# timedatectl status
	timedatectl set-timezone UTC
	date
	# timedatectl status
else
	echo "# Default SystemClock and Timezone"
	# timedatectl status
	date
fi

# Time synchronization with NTP server
date
chronyc tracking
chronyc sources -v
chronyc sourcestats -v
date

# Setting System Language
if [ "${Language}" = "ja_JP.UTF-8" ]; then
	echo "# Setting System Language -> $Language"
	locale
	# localectl status
	localectl set-locale LANG=ja_JP.utf8
	locale
	# localectl status
	cat /etc/locale.conf
elif [ "${Language}" = "en_US.UTF-8" ]; then
	echo "# Setting System Language -> $Language"
	locale
	# localectl status
	localectl set-locale LANG=en_US.utf8
	locale
	# localectl status
	cat /etc/locale.conf
else
	echo "# Default Language"
	locale
	cat /etc/locale.conf
fi

# Setting IP Protocol Stack (IPv4 Only) or (IPv4/IPv6 Dual stack)
if [ "${VpcNetwork}" = "IPv4" ]; then
	echo "# Setting IP Protocol Stack -> $VpcNetwork"
	# Setting NTP Deamon
	sed -i 's/bindcmdaddress ::1/#bindcmdaddress ::1/g' /etc/chrony.conf
	systemctl restart chronyd
	# Disable IPv6 Kernel Module
	echo "options ipv6 disable=1" >> /etc/modprobe.d/ipv6.conf
	# Disable IPv6 Kernel Parameter
	sysctl -a

	DisableIPv6Conf="/etc/sysctl.d/99-ipv6-disable.conf"

	cat /dev/null > $DisableIPv6Conf
	echo '# Custom sysctl Parameter for ipv6 disable' >> $DisableIPv6Conf
	echo 'net.ipv6.conf.all.disable_ipv6 = 1' >> $DisableIPv6Conf
	echo 'net.ipv6.conf.default.disable_ipv6 = 1' >> $DisableIPv6Conf

	sysctl --system
	sysctl -p

	sysctl -a | grep -ie "local_port" -ie "ipv6" | sort
elif [ "${VpcNetwork}" = "IPv6" ]; then
	echo "# Show IP Protocol Stack -> $VpcNetwork"
	echo "# Show IPv6 Network Interface Address"
	ifconfig
	echo "# Show IPv6 Kernel Module"
	lsmod | grep ipv6
	echo "# Show Network Listen Address and report"
	netstat -an -A inet6
	echo "# Show Network Routing Table"
	netstat -r -A inet6
else
	echo "# Default IP Protocol Stack"
	echo "# Show IPv6 Network Interface Address"
	ifconfig
	echo "# Show IPv6 Kernel Module"
	lsmod | grep ipv6
	echo "# Show Network Listen Address and report"
	netstat -an -A inet6
	echo "# Show Network Routing Table"
	netstat -r -A inet6
fi

#-------------------------------------------------------------------------------
# System Setting (Root Disk Extension)
#-------------------------------------------------------------------------------
# Disk Information(Partition) [parted -l]
parted -l

# Disk Information(Partition) [file -s]
file -s /dev/xvd*

# Disk Information(MountPoint) [lsblk]
lsblk

# Disk Information(File System) [df -h]
df -h

# Expansion of disk partition
parted -l

LANG=C growpart --dry-run /dev/xvda 1
LANG=C growpart /dev/xvda 1

parted -l

# Expansion of disk partition
df -h

resize2fs /dev/xvda1

df -h

#-------------------------------------------------------------------------------
# Reboot
#-------------------------------------------------------------------------------

# Instance Reboot
reboot

error: pathspec '3rd-Bootstrap_OracleLinux-v7-HVM.sh' did not match any file(s) known to git


rust lang server

rust lang server

#!/usr/bin/env bash

rustup component add rls rust-analysis rust-src

error: pathspec 'LanguageServers/install_rust_lang_components.sh' did not match any file(s) known to git


added build script that can be used if only frontend server is changed

#!/bin/bash

docker image build -t hello-compsci .

error: pathspec 'frontend-server/build.sh' did not match any file(s) known to git


overlay for #15645

overlay for #15645

overlay vscoq https://github.com/alizter/vscoq alizter+improve-coqide-help 15645

error: pathspec 'dev/ci/user-overlays/15645-alizter-alizter+improve-coqide-help.sh' did not match any file(s) known to git


Core: add update-truststore admin commands (refs #5789)

remote: Support for password authentication was removed on August 13, 2021.
remote: Please see https://docs.github.com/en/get-started/getting-started-with-git/about-remote-repositories#cloning-with-https-urls for information on currently recommended modes of authentication.
fatal: Authentication failed for 'https://github.com/it-toolbox/project-factory-core.git/'


Implementation of run-benchmark bash script

#!/bin/bash

declare -a frameworks=("angular" "angular2")
declare -a virtualMachines=("64" "128")

for framework in "${frameworks[@]}"
do
    echo "$framework"

    for virtualMachine in "${virtualMachines[@]}"
    do
        echo "$virtualMachine"

        node ./chrome-performance-logs-recorder/build/main.js --verbose --url="http://localhost:8080/$framework" --filter-event-names="DrawFrame,Layout,Paint" --duration=1000 --local-storage="virtualMachines:$virtualMachine" --output-filename="$framework.$virtualMachine.json"
        node ./trace-event-format-analyzer/build/main.js --verbose --paths="$framework.$virtualMachine.json" --complete-events="Paint" --duration-events="Layout" --immediate-events="DrawFrame" --output
    done
done

error: pathspec 'run-benchmark.sh' did not match any file(s) known to git


Add filename and template

Add filename and template

remote: Support for password authentication was removed on August 13, 2021.
remote: Please see https://docs.github.com/en/get-started/getting-started-with-git/about-remote-repositories#cloning-with-https-urls for information on currently recommended modes of authentication.
fatal: Authentication failed for 'https://github.com/OddState/resume-product.git/'


* build.sh + modif dans le README

* build.sh + modif dans le README

if [ -z "$releaseVersion" ]; then
    # incremental release
    make zip
    scp dist/emacs-runtime-`git describe --tags`.zip buildfix@porthos:/u01/web/versions/emacs_runtime/emacs-runtime-dev.zip
else
    VERSION=$releaseVersion make zip
    git tag $releaseVersion -m "Emacs runtime $releaseVersion released"
    git push origin $releaseVersion
    scp dist/emacs-runtime-$releaseVersion.zip buildfix@porthos:/u01/web/versions/emacs_runtime/
    ssh buildfix@porthos "cd /u01/web/versions/emacs_runtime/ && ln -sf emacs-runtime-$releaseVersion.zip emacs-runtime-latest.zip"
fi

error: pathspec 'runtime/build.sh' did not match any file(s) known to git


add pytorch horovod example test (#3192)

#!/bin/bash

export SPARK_HOME=$SPARK_HOME
export MASTER=local[4]
export FTP_URI=$FTP_URI
export ANALYTICS_ZOO_ROOT=$ANALYTICS_ZOO_ROOT
export ANALYTICS_ZOO_HOME=$ANALYTICS_ZOO_ROOT/dist
export ANALYTICS_ZOO_JAR=`find ${ANALYTICS_ZOO_HOME}/lib -type f -name "analytics-zoo*jar-with-dependencies.jar"`
export ANALYTICS_ZOO_PYZIP=`find ${ANALYTICS_ZOO_HOME}/lib -type f -name "analytics-zoo*python-api.zip"`
export ANALYTICS_ZOO_CONF=${ANALYTICS_ZOO_HOME}/conf/spark-analytics-zoo.conf
export PYTHONPATH=${ANALYTICS_ZOO_PYZIP}:$PYTHONPATH
export BIGDL_JARS=`find ${ANALYTICS_ZOO_HOME}/lib -type f -name "analytics-zoo*jar-with-dependencies.jar"`

set -e

echo "Start ray horovod pytorch example tests"
#start execute
echo "#1 pytorch estimator example"
start=$(date "+%s")
python ${ANALYTICS_ZOO_ROOT}/pyzoo/zoo/examples/orca/learn/horovod/pytorch_estimator.py
now=$(date "+%s")
time1=$((now-start))

echo "horovod pytorch example tests finished"
echo "#1 pytorch estimator example time used:$time1 seconds"

error: pathspec 'python/orca/dev/example/run-example-test-ray-horovod-pytorch.sh' did not match any file(s) known to git


Turn on error checking in ensure_android_bundle.sh

#!/bin/bash

set -e

BUNDLE_VERSION=0.1

# OS Check. Put here because here is where we download the precompiled
# bundles that are arch specific.
UNAME=$(uname)
if [ "$UNAME" != "Linux" -a "$UNAME" != "Darwin" ] ; then
  echo "Sorry, this OS is not supported."
  exit 1
fi

# import all the environment
source $(dirname $0)/common_env.sh

command -v java >/dev/null 2>&1 || {
  if [ ${UNAME} == "Linux" ] ; then
    echo "Please install Java before running this command.";
    echo "Directions can be found at: http://openjdk.java.net/install/"
  else
    echo "The android platform needs Java to be installed on your system."
    java -version
  fi

  exit 1;
}

cd "$ORIG_DIR"

if [ -z "$USE_GLOBAL_ADK" ] ; then
  # not using global ADK
  true
else
  # using global ADK, check all utilities
  set -e
  trap "echo One of the required utilities wasn't found in global PATH: java javac ant android" EXIT

  which java
  which javac
  which ant
  which android

  trap - EXIT
  set +e
  exit 0
fi

install_android_bundle () {
  echo "Going to install Android Bundle (300M-400M)."
  echo "This might take a while, please hold on."

  set -e
  trap "echo Failed to install dependency kit." EXIT

  TARBALL="android_bundle_${UNAME}_${BUNDLE_VERSION}.tar.gz"
  DEST_DIR=$(dirname "$ANDROID_BUNDLE")
  BUNDLE_TMPDIR="$DEST_DIR/android_bundle.xxx"

  rm -rf "$BUNDLE_TMPDIR"
  mkdir "$BUNDLE_TMPDIR"

  ANDROID_BUNDLE_URL_ROOT="http://s3.amazonaws.com/android-bundle/"

  if [ -f "$DEST_DIR/$TARBALL" ] ; then
      echo "Skipping download and installing kit from $DEST_DIR/$TARBALL" >&2
      tar -xzf "$DEST_DIR/$TARBALL" -C "$BUNDLE_TMPDIR"
  else
      curl "$ANDROID_BUNDLE_URL_ROOT$TARBALL" | tar -xzf - -C "$BUNDLE_TMPDIR"
  fi

  # Delete old dev bundle and rename the new one on top of it.
  rm -rf "$DEST_DIR/android_bundle"
  mv "$BUNDLE_TMPDIR" "$DEST_DIR/android_bundle"

  echo "Installed dependency kit v${BUNDLE_VERSION} in android_bundle." >&2
  echo >&2

  trap - EXIT
  set +e;
}

if [ ! -d "$ANDROID_BUNDLE" ] ; then
  install_android_bundle
elif [ ! -f "$ANDROID_BUNDLE/.bundle_version.txt" ] ||
  # we might need an android bundle version?
  grep -qvx "$BUNDLE_VERSION" "$ANDROID_BUNDLE/.bundle_version.txt" ; then
  install_android_bundle
fi

command -v javac >/dev/null 2>&1 || {
  echo >&2 "To add the android platform, please install a JDK. Here are some directions: http://openjdk.java.net/install/"; exit 1;
}

set_config () {
  KEY=$1
  VALUE=$2

  CONFIG_FILE=${ANDROID_BUNDLE}/meteor_avd/config.ini

  TEMP_FILE=`mktemp`
  grep -v "^${KEY}=" ${CONFIG_FILE} > ${TEMP_FILE}
  echo "${KEY}=${VALUE}" >> ${TEMP_FILE}
  mv -f ${TEMP_FILE} ${CONFIG_FILE}
}

# create avd if necessary
if [[ ! $("${ANDROID_BUNDLE}/android-sdk/tools/android" list avd | grep Name) ]] ; then
  #ABI="default/armeabi-v7a"
  ABI="default/x86"

  echo "
" | "${ANDROID_BUNDLE}/android-sdk/tools/android" create avd --target 1 --name meteor --abi ${ABI} --path ${ANDROID_BUNDLE}/meteor_avd/ 1>&2

  # Nice keyboard support
  set_config "hw.keyboard" "yes"
  set_config "hw.mainKeys" "no"

  # More RAM than the default
  set_config "hw.ramSize" "1024"
  set_config "vm.heapSize" "64"

  # These are the settings for a Nexus 4, but it's a bit big for some screens
  #  (and likely a bit slow without GPU & KVM/HAXM acceleration)
  #set_config "skin.dynamic" "yes"
  #set_config "hw.lcd.density" "320"
  #set_config "hw.device.name" "Nexus 4"
  #set_config "hw.device.manufacturer" "Google"

  # XXX: hw.gpu.enabled=yes ?

fi

#!/bin/bash
BUNDLE_VERSION=0.1

# OS Check. Put here because here is where we download the precompiled
# bundles that are arch specific.
UNAME=$(uname)
if [ "$UNAME" != "Linux" -a "$UNAME" != "Darwin" ] ; then
  echo "Sorry, this OS is not supported."
  exit 1
fi

# import all the environment
source $(dirname $0)/common_env.sh

command -v java >/dev/null 2>&1 || {
  if [ ${UNAME} == "Linux" ] ; then
    echo "Please install Java before running this command.";
    echo "Directions can be found at: http://openjdk.java.net/install/"
  else
    echo "The android platform needs Java to be installed on your system."
    java -version
  fi

  exit 1;
}

cd "$ORIG_DIR"

if [ -z "$USE_GLOBAL_ADK" ] ; then
  # not using global ADK
  true
else
  # using global ADK, check all utilities
  set -e
  trap "echo One of the required utilities wasn't found in global PATH: java javac ant android" EXIT

  which java
  which javac
  which ant
  which android

  trap - EXIT
  set +e
  exit 0
fi

install_android_bundle () {
  echo "Going to install Android Bundle (300M-400M)."
  echo "This might take a while, please hold on."

  set -e
  trap "echo Failed to install dependency kit." EXIT

  TARBALL="android_bundle_${UNAME}_${BUNDLE_VERSION}.tar.gz"
  DEST_DIR=$(dirname "$ANDROID_BUNDLE")
  BUNDLE_TMPDIR="$DEST_DIR/android_bundle.xxx"

  rm -rf "$BUNDLE_TMPDIR"
  mkdir "$BUNDLE_TMPDIR"

  ANDROID_BUNDLE_URL_ROOT="http://s3.amazonaws.com/android-bundle/"

  if [ -f "$DEST_DIR/$TARBALL" ] ; then
      echo "Skipping download and installing kit from $DEST_DIR/$TARBALL" >&2
      tar -xzf "$DEST_DIR/$TARBALL" -C "$BUNDLE_TMPDIR"
  else
      curl "$ANDROID_BUNDLE_URL_ROOT$TARBALL" | tar -xzf - -C "$BUNDLE_TMPDIR"
  fi

  # Delete old dev bundle and rename the new one on top of it.
  rm -rf "$DEST_DIR/android_bundle"
  mv "$BUNDLE_TMPDIR" "$DEST_DIR/android_bundle"

  echo "Installed dependency kit v${BUNDLE_VERSION} in android_bundle." >&2
  echo >&2

  trap - EXIT
  set +e;
}

if [ ! -d "$ANDROID_BUNDLE" ] ; then
  install_android_bundle
elif [ ! -f "$ANDROID_BUNDLE/.bundle_version.txt" ] ||
  # we might need an android bundle version?
  grep -qvx "$BUNDLE_VERSION" "$ANDROID_BUNDLE/.bundle_version.txt" ; then
  install_android_bundle
fi

command -v javac >/dev/null 2>&1 || {
  echo >&2 "To add the android platform, please install a JDK. Here are some directions: http://openjdk.java.net/install/"; exit 1;
}

set_config () {
  KEY=$1
  VALUE=$2

  CONFIG_FILE=${ANDROID_BUNDLE}/meteor_avd/config.ini

  TEMP_FILE=`mktemp`
  grep -v "^${KEY}=" ${CONFIG_FILE} > ${TEMP_FILE}
  echo "${KEY}=${VALUE}" >> ${TEMP_FILE}
  mv -f ${TEMP_FILE} ${CONFIG_FILE}
}

# create avd if necessary
if [[ ! $("${ANDROID_BUNDLE}/android-sdk/tools/android" list avd | grep Name) ]] ; then
  #ABI="default/armeabi-v7a"
  ABI="default/x86"

  echo "
" | "${ANDROID_BUNDLE}/android-sdk/tools/android" create avd --target 1 --name meteor --abi ${ABI} --path ${ANDROID_BUNDLE}/meteor_avd/ 1>&2

  # Nice keyboard support
  set_config "hw.keyboard" "yes"
  set_config "hw.mainKeys" "no"

  # More RAM than the default
  set_config "hw.ramSize" "1024"
  set_config "vm.heapSize" "64"

  # These are the settings for a Nexus 4, but it's a bit big for some screens
  #  (and likely a bit slow without GPU & KVM/HAXM acceleration)
  #set_config "skin.dynamic" "yes"
  #set_config "hw.lcd.density" "320"
  #set_config "hw.device.name" "Nexus 4"
  #set_config "hw.device.manufacturer" "Google"

  # XXX: hw.gpu.enabled=yes ?

fi


Fixed setup.sh file so the e2e tests can be run locally. (#287)

* Fixed setup.sh file so the e2e tests can be run locally.

* Forgot the space!

#!/usr/bin/env bash

# Copyright 2017 The TIE Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS-IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# Flags:
#   --disable-presubmit-checks: set to disable hooks that run presubmit checks.

set -e

export OS=`uname`
export MACHINE_TYPE=`uname -m`
export TOOLS_DIR=./tools
export NODE_DIR=$TOOLS_DIR/node-6.9.1
if [ ${OS} == "MINGW_NT-10.0" ]; then
  export NPM_CMD=$NODE_DIR/npm
else
  export NPM_CMD=$NODE_DIR/bin/npm
fi
export NODE_MODULES_DIR=./node_modules
# Adjust PATH to include a reference to node.
export PATH=$NODE_DIR/bin:$PATH

#############################################################
# Installs a node module at a particular version.
# Usage: install_node_module [module_name] [module_version]
# Globals:
#   NPM_INSTALLED_MODULES
# Arguments:
#   module_name: the name of the node module
#   module_version: the expected version of the module
# Returns:
#   None
#############################################################
install_node_module() {
  if [[ $NPM_INSTALLED_MODULES != *"$1@$2"* ]]; then
    echo Installing $1@$2
    $NPM_CMD install $1@$2
    NPM_INSTALLED_MODULES="$(npm list)"
  fi
}

# Set up hooks if not disabled.
if ! [[ $* == *--disable-presubmit-checks* ]]
then
  git config core.hooksPath "./hooks"
fi

export -f install_node_module

# Ensure that there is a node_modules folder in the root, otherwise node may
# put libraries in the wrong place. See
#
#     https://docs.npmjs.com/files/folders#more-information
if [ ! -d "$NODE_MODULES_DIR" ]; then
  mkdir $NODE_MODULES_DIR
fi

# Download and install node.js, if necessary.
echo Checking if node.js is installed in $NODE_DIR
if [ ! -d "$NODE_DIR" ]; then
  echo Installing Node.js for ${OS}
  ON_WIN=false
  if [ ${OS} == "Darwin" ]; then
    if [ ${MACHINE_TYPE} == 'x86_64' ]; then
      NODE_FILE_NAME=node-v6.9.1-darwin-x64
    else
      NODE_FILE_NAME=node-v6.9.1-darwin-x86
    fi
  elif [ ${OS} == "Linux" ]; then
    if [ ${MACHINE_TYPE} == 'x86_64' ]; then
      NODE_FILE_NAME=node-v6.9.1-linux-x64
    else
      NODE_FILE_NAME=node-v6.9.1-linux-x86
    fi
  elif [ ${OS} == MINGW64_NT-10.0 ]; then
    ON_WIN=true
    NODE_FILE_NAME=node-v6.9.1-win-x64
  fi

  if [ ! -d "$TOOLS_DIR" ]; then
    mkdir tools
  fi

  if $ON_WIN; then
    curl -o node-download.zip https://nodejs.org/dist/v6.9.1/$NODE_FILE_NAME.zip
    unzip node-download.zip -d $TOOLS_DIR
    mv $TOOLS_DIR/$NODE_FILE_NAME $NODE_DIR
    rm node-download.zip
  else
    curl -o node-download.tgz https://nodejs.org/dist/v6.9.1/$NODE_FILE_NAME.tar.gz
    tar xzf node-download.tgz --directory $TOOLS_DIR
    mv $TOOLS_DIR/$NODE_FILE_NAME $NODE_DIR
    rm node-download.tgz
  fi
fi

# Generate a list of already-installed modules.
NPM_INSTALLED_MODULES="$(npm list)"

#!/usr/bin/env bash

# Copyright 2017 The TIE Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS-IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# Flags:
#   --disable-presubmit-checks: set to disable hooks that run presubmit checks.

set -e

export OS=`uname`
export MACHINE_TYPE=`uname -m`
export TOOLS_DIR=./tools
export NODE_DIR=$TOOLS_DIR/node-6.9.1
if [ ${OS}==MINGW_NT-10.0 ]; then
  export NPM_CMD=$NODE_DIR/npm
else
  export NPM_CMD=$NODE_DIR/bin/npm
fi
export NODE_MODULES_DIR=./node_modules
# Adjust PATH to include a reference to node.
export PATH=$NODE_DIR/bin:$PATH

#############################################################
# Installs a node module at a particular version.
# Usage: install_node_module [module_name] [module_version]
# Globals:
#   NPM_INSTALLED_MODULES
# Arguments:
#   module_name: the name of the node module
#   module_version: the expected version of the module
# Returns:
#   None
#############################################################
install_node_module() {
  if [[ $NPM_INSTALLED_MODULES != *"$1@$2"* ]]; then
    echo Installing $1@$2
    $NPM_CMD install $1@$2
    NPM_INSTALLED_MODULES="$(npm list)"
  fi
}

# Set up hooks if not disabled.
if ! [[ $* == *--disable-presubmit-checks* ]]
then
  git config core.hooksPath "./hooks"
fi

export -f install_node_module

# Ensure that there is a node_modules folder in the root, otherwise node may
# put libraries in the wrong place. See
#
#     https://docs.npmjs.com/files/folders#more-information
if [ ! -d "$NODE_MODULES_DIR" ]; then
  mkdir $NODE_MODULES_DIR
fi

# Download and install node.js, if necessary.
echo Checking if node.js is installed in $NODE_DIR
if [ ! -d "$NODE_DIR" ]; then
  echo Installing Node.js for ${OS}
  ON_WIN=false
  if [ ${OS} == "Darwin" ]; then
    if [ ${MACHINE_TYPE} == 'x86_64' ]; then
      NODE_FILE_NAME=node-v6.9.1-darwin-x64
    else
      NODE_FILE_NAME=node-v6.9.1-darwin-x86
    fi
  elif [ ${OS} == "Linux" ]; then
    if [ ${MACHINE_TYPE} == 'x86_64' ]; then
      NODE_FILE_NAME=node-v6.9.1-linux-x64
    else
      NODE_FILE_NAME=node-v6.9.1-linux-x86
    fi
  elif [ ${OS} == MINGW64_NT-10.0 ]; then
    ON_WIN=true
    NODE_FILE_NAME=node-v6.9.1-win-x64
  fi

  if [ ! -d "$TOOLS_DIR" ]; then
    mkdir tools
  fi

  if $ON_WIN; then
    curl -o node-download.zip https://nodejs.org/dist/v6.9.1/$NODE_FILE_NAME.zip
    unzip node-download.zip -d $TOOLS_DIR
    mv $TOOLS_DIR/$NODE_FILE_NAME $NODE_DIR
    rm node-download.zip
  else
    curl -o node-download.tgz https://nodejs.org/dist/v6.9.1/$NODE_FILE_NAME.tar.gz
    tar xzf node-download.tgz --directory $TOOLS_DIR
    mv $TOOLS_DIR/$NODE_FILE_NAME $NODE_DIR
    rm node-download.tgz
  fi
fi

# Generate a list of already-installed modules.
NPM_INSTALLED_MODULES="$(npm list)"


Fixed setup and launch of PostgreSQL, Nginx, Memcached, and Redis processes.

#!/bin/bash

# DESCRIPTION
# Sets up and launches (if necessary) installed software.

# Bash
sudo bash -c "printf '/usr/local/bin/bash\n' >> /etc/shells"

# Bash Completion
chsh -s /usr/local/bin/bash

# rbenv
export PATH="/usr/local/bin:$PATH"
eval "$(rbenv init -)"
mkdir -p $HOME/.rbenv && cp settings/rbenv-vars.txt $HOME/.rbenv/vars
for ruby in $MRI $RUBINIUS $JRUBY; do
  rbenv install $ruby
done
rbenv global $MRI

# Ruby
git clone git://github.com/bkuhlmann/ruby_setup.git
(
  cd ruby_setup
  for ruby in $MRI $RUBINIUS $JRUBY; do
    rbenv shell $ruby
    ./run.sh i
  done
)
rm -rf ruby_setup

# Go
install_git_project "git://github.com/bkuhlmann/go_setup.git" "go_setup" "./run.sh i"

# NPM
install_git_project "git://github.com/bkuhlmann/npm_setup.git" "npm_setup" "./run.sh i"

# PostgreSQL
plist="homebrew.mxcl.postgresql.plist"
ln -sfv /usr/local/opt/postgresql/$plist $HOME/Library/LaunchAgents/$plist
initdb /usr/local/var/postgres -E utf8
launchctl load $HOME/Library/LaunchAgents/$plist

# Nginx
plist="homebrew.mxcl.nginx.plist"
ln -sfv /usr/local/opt/nginx/$plist $HOME/Library/LaunchAgents/$plist
launchctl load $HOME/Library/LaunchAgents/$plist

# Memcached
plist="homebrew.mxcl.memcached.plist"
ln -sfv /usr/local/opt/memcached/$plist $HOME/Library/LaunchAgents/$plist
launchctl load $HOME/Library/LaunchAgents/$plist

# Redis
plist="homebrew.mxcl.redis.plist"
ln -sfv /usr/local/opt/redis/$plist $HOME/Library/LaunchAgents/$plist
launchctl load $HOME/Library/LaunchAgents/$plist

# Pow
curl get.pow.cx | sh

# Google Chrome Ember Inspector
(
  cd "$EMBER_INSPECTOR_EXTENSION_PATH"
  grunt
)

# Sublime Text
if [ ! -e "/usr/bin/sublime" ]; then
  sudo ln -sv "/Applications/Sublime Text 2.app/Contents/SharedSupport/bin/subl" /usr/local/bin/sublime
fi
install_git_project "git://github.com/bkuhlmann/sublime_text.git" "sublime_text" "./run.sh i"

# Dotfiles
rm -f $HOME/.bash_profile
install_git_project "git://github.com/bkuhlmann/dotfiles.git" "dotfiles" "./run.sh i"
source $HOME/.bashrc

#!/bin/bash

# DESCRIPTION
# Sets up and launches (if necessary) installed software.

# Bash
sudo bash -c "printf '/usr/local/bin/bash\n' >> /etc/shells"

# Bash Completion
chsh -s /usr/local/bin/bash

# rbenv
export PATH="/usr/local/bin:$PATH"
eval "$(rbenv init -)"
mkdir -p $HOME/.rbenv && cp settings/rbenv-vars.txt $HOME/.rbenv/vars
for ruby in $MRI $RUBINIUS $JRUBY; do
  rbenv install $ruby
done
rbenv global $MRI

# Ruby
git clone git://github.com/bkuhlmann/ruby_setup.git
(
  cd ruby_setup
  for ruby in $MRI $RUBINIUS $JRUBY; do
    rbenv shell $ruby
    ./run.sh i
  done
)
rm -rf ruby_setup

# Go
install_git_project "git://github.com/bkuhlmann/go_setup.git" "go_setup" "./run.sh i"

# NPM
install_git_project "git://github.com/bkuhlmann/npm_setup.git" "npm_setup" "./run.sh i"

# Nginx
ln -sfv /usr/local/opt/nginx/*.plist $HOME/Library/LaunchAgents
launchctl load $HOME/Library/LaunchAgents/homebrew.mxcl.nginx.plist

# Memcached
ln -sfv /usr/local/opt/memcached/*.plist $HOME/Library/LaunchAgents
launchctl load $HOME/Library/LaunchAgents/homebrew.mxcl.memcached.plist

# Redis
ln -sfv /usr/local/opt/redis/*.plist $HOME/Library/LaunchAgents
launchctl load $HOME/Library/LaunchAgents/homebrew.mxcl.redis.plist

# PostgreSQL
ln -sfv /usr/local/opt/postgresql/*.plist $HOME/Library/LaunchAgents
initdb /usr/local/var/postgres -E utf8
launchctl load $HOME/Library/LaunchAgents/homebrew.mxcl.postgresql.plist

# Pow
curl get.pow.cx | sh

# Google Chrome Ember Inspector
(
  cd "$EMBER_INSPECTOR_EXTENSION_PATH"
  grunt
)

# Sublime Text
if [ ! -e "/usr/bin/sublime" ]; then
  sudo ln -sv "/Applications/Sublime Text 2.app/Contents/SharedSupport/bin/subl" /usr/local/bin/sublime
fi
install_git_project "git://github.com/bkuhlmann/sublime_text.git" "sublime_text" "./run.sh i"

# Dotfiles
rm -f $HOME/.bash_profile
install_git_project "git://github.com/bkuhlmann/dotfiles.git" "dotfiles" "./run.sh i"
source $HOME/.bashrc


Update.

Update.

#!/bin/bash
script_path="$( cd "$( dirname "$0" )" && pwd )"
project_path=$( readlink -e "$script_path/.." )
venv_path="$project_path/.venv"

# Install required packages
sudo apt --assume-yes install build-essential
sudo apt --assume-yes install python-dev
sudo apt --assume-yes install python-pip
sudo apt --assume-yes install virtualenv
sudo apt --assume-yes install libffi-dev
sudo apt --assume-yes install libssl-dev

# Create a virtualenv
virtualenv $venv_path
source "$venv_path/bin/activate"
pip install -r "$script_path/requirements.txt"

# Create a self-signed certicate
bash "$script_path/protocol-setup.sh"

#!/bin/bash
script_path="$( cd "$( dirname "$0" )" && pwd )"
project_path=$( readlink -e "$script_path/.." )
venv_path="$project_path/.venv"

# Install required packages
sudo apt --assume-yes install build-essential
sudo apt --assume-yes install python-dev
sudo apt --assume-yes install python-pip
sudo apt --assume-yes install virtualenv
sudo apt --assume-yes install libffi-dev

# Create a virtualenv
virtualenv $venv_path
source "$venv_path/bin/activate"
pip install -r "$script_path/requirements.txt"

# Create a self-signed certicate
bash "$script_path/protocol-setup.sh"


Added python setup script.

Added python setup script.

#!/usr/bin/env bash

pip install watchdog
pip install pyobjc-framework-Quartz

error: pathspec 'scripts/setup.sh' did not match any file(s) known to git


Update arrayfire binary (#141)

Update arrayfire binary (#141)

* Update arrayfire binary

* Use version variable

* Remove `export`

* Minor fix

#!/usr/bin/env bash
#######################################################################
# PackageEvaluator
# https://github.com/JuliaCI/PackageEvaluator.jl
# (c) Iain Dunning and contributors 2015
# Licensed under the MIT License
#######################################################################
# This script file is the provisioning script run by Vagrant after
# the VM is created. It sets up the environment for running PkgEval,
# then runs it to produce the JSON result files.
#
# Commandline arguments for this script, passed through by Vagrant.
# 1st: Julia version:   0.3 | 0.4 | 0.5
# 2nd: Test set to run: setup | all | AL | MZ
#######################################################################

#######################################################################
# Accept all apt-gets
# This is an attempt to deal with BinDeps packages trying to install
# dependencies with apt-get and asking for permissions. Normally we
# could pass a --yes argument to avoid this, but there isn't a way to
# do that with BinDeps. We can actually do a global override, which
# is what we do here. Since we are piping stuff into a file, the
# easiest way to sudo the whole thing is to do it in this style.
sudo su -c 'echo "APT::Get::Assume-Yes \"true\";" > /etc/apt/apt.conf.d/pkgevalforceyes'
sudo su -c 'echo "APT::Get::force-yes \"true\";" >> /etc/apt/apt.conf.d/pkgevalforceyes'
# Uncomment following line to check it did indeed work
#cat /etc/apt/apt.conf.d/pkgevalforceyes

#######################################################################
# Use newest R release
sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys E084DAB9
sudo add-apt-repository -y "deb http://cran.rstudio.com/bin/linux/ubuntu trusty/"

# Upgrade the installation and install Julia
sudo apt-get update    # Pull in latest versions
sudo apt-get upgrade   # Upgrade system packages
# Use first argument to script to distinguish between the versions
if [ "$1" == "0.3" ]
then
    wget -q -O julia.tar.gz https://julialang.s3.amazonaws.com/bin/linux/x64/0.3/julia-0.3-latest-linux-x86_64.tar.gz
elif [ "$1" == "0.4" ]
then
    wget -q -O julia.tar.gz https://julialang.s3.amazonaws.com/bin/linux/x64/0.4/julia-0.4-latest-linux-x86_64.tar.gz
elif [ "$1" == "0.5" ]
then
    wget -q -O julia.tar.gz https://julialang.s3.amazonaws.com/bin/linux/x64/0.5/julia-0.5-latest-linux-x86_64.tar.gz
else
    # Nightly
    wget -q -O julia.tar.gz https://julianightlies.s3.amazonaws.com/bin/linux/x64/julia-latest-linux64.tar.gz
fi
mkdir julia
tar -zxf julia.tar.gz -C ./julia --strip-components=1
rm julia.tar.gz
export PATH="${PATH}:/home/vagrant/julia/bin/"
# Retain PATH to make it easier to use VM for debugging
echo "export PATH=\"\${PATH}:/home/vagrant/julia/bin/\"" >> /home/vagrant/.profile

# Set a 10 minute absolute timeout and 2 minute low speed timeout
# for curl because of unreliable downloads like wcslib, graphviz
echo "max-time = 600" >> ~/.curlrc
echo "speed-time = 120" >> ~/.curlrc

#######################################################################
# Install any dependencies that aren't handled by BinDeps
# Somethings can't be or don't make sense to be installed by BinDeps,
# so we will do them manually. Package maintainers can submit PRs to
# extend this list as needed.
# Need git of course, doesn't come by default in the image
sudo apt-get install git
# Need X virtual frame buffer for many packages
sudo apt-get install xvfb
# Need GMP for e.g. GLPK, why not get some PCRE too
sudo apt-get install libpcre3-dev libgmp-dev
# Need gfortran for e.g. GLMNet.jl, Ipopt.jl, KernSmooth.jl...
sudo apt-get install gfortran pkg-config
# Need unzip for e.g. Blink.jl, FLANN.jl
sudo apt-get install unzip
# Need cmake for e.g. GLFW.jl, Metis.jl
sudo apt-get install cmake
# Install R for e.g. Rif.jl, RCall.jl
sudo apt-get install r-base r-base-dev
# Install Java for e.g. JavaCall.jl, Taro.jl
# From: http://stackoverflow.com/q/19275856/3822752
sudo add-apt-repository -y ppa:webupd8team/java
sudo apt-get update
echo debconf shared/accepted-oracle-license-v1-1 select true | sudo debconf-set-selections
echo debconf shared/accepted-oracle-license-v1-1 seen true | sudo debconf-set-selections
sudo apt-get -y install oracle-java7-installer
export JAVA_HOME=/usr/lib/jvm/java-7-oracle
echo "export JAVA_HOME=/usr/lib/jvm/java-7-oracle" >> /home/vagrant/.profile
# Use the Conda Package as Python environment
export PYTHON=""
# need /usr/share/dict/words for TextAnalysis.jl
sudo apt-get install wamerican
# Need xmllint (and others?) for XMLDict.jl
sudo apt-get install libxml2-utils
# ArrayFire
AFVERSION=v3.4.0
wget -q http://ci.arrayfire.org/userContent/Linux/ArrayFire-no-gl-${AFVERSION}_Linux_x86_64.sh
sudo chmod +x ArrayFire-no-gl-${AFVERSION}_Linux_x86_64.sh
sudo ./ArrayFire-no-gl-${AFVERSION}_Linux_x86_64.sh --exclude-subdir --prefix=/usr/local
export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/lib/
rm ArrayFire-no-gl-${AFVERSION}_Linux_x86_64.sh

#######################################################################
# Get PackageEvaluator scripts
PKGEVALDIR="/home/vagrant/pkgeval"
git clone https://github.com/JuliaCI/PackageEvaluator.jl.git $PKGEVALDIR
# Make results folders. Folder name is second argument to this script.
# These folders are shared - i.e. we are writing to outside the VM,
# most likely the PackageEvaluator.jl/scripts folder.
rm -rf /vagrant/${1}${2}
mkdir /vagrant/${1}${2}
cd /vagrant/${1}${2}
# Initialize METADATA for testing
# Note that it is important for it to not be in the /vagrant/ folder as
# that seems to mess with symlinks quite badly.
# See https://github.com/JuliaOpt/Ipopt.jl/issues/31 for discussion
# about this breaking a build.
julia -e "Pkg.init(); println(Pkg.dir())"

#######################################################################
# Run PackageEvaluator
if [ "$2" == "all" ]
then
    LOOPOVER=/home/vagrant/.julia/v${1}/METADATA/*
elif [ "$2" == "AL" ]
then
    LOOPOVER=/home/vagrant/.julia/v${1}/METADATA/[A-L]*;
elif [ "$2" == "MZ" ]
then
    LOOPOVER=/home/vagrant/.julia/v${1}/METADATA/[M-Z]*;
fi
# For every package name...
for f in $LOOPOVER;
do
    # Extract just the package name from the path
    PKGNAME=$(basename "$f")
    # Attempt to add the package. We give it half an hour - most
    # use far less, some do push it. CoinOptServices.jl set the
    # standard for build time and memory consumption, see
    # https://github.com/JuliaCI/PackageEvaluator.jl/issues/83
    # The log for adding the package will go in the results folder.
    timeout 2000s julia -e "Pkg.add(\"${PKGNAME}\")" 2>&1 | tee PKGEVAL_${PKGNAME}_add.log
    # A package can have four states:
    # - Not testable: for some reason, we can't even analyze how
    #   broken or not the package is, usually due to a limitation
    #   of PackageEvaluaor itself.
    # - No tests: the package doesn't even have tests. We used to
    #   further distinguish this by seeing if the package loads,
    #   but loading doesn't mean it actually works so this is bit
    #   misleading.
    # - Tests fail: the package has tests, and they don't pass.
    # - Tests pass: the tests pass!
    # We first run a script that:
    # - Identifies if we are testable or not. If we are not,
    #   exit with status 1 (not testable), or 2 (no tests).
    # - If we are, create a shell script to run the tests that
    #   includes xvfb, timeout, etc.
    julia $PKGEVALDIR/src/preptest.jl $PKGNAME
    TESTSTATUS=$?
    if [ $TESTSTATUS -eq 255 ]
    then
        # Not testable
        echo "NOT TESTABLE"
    elif [ $TESTSTATUS -eq 254 ]
    then
        # No tests
        echo "NO TESTS"
    else
        # Has tests, we need to run them
        # preptest.jl should have created a shell script to
        # run them. We just need to run that shell script
        # and store the error code. A code of 0 means the
        # tests passed, a code of 1 means Julia threw an
        # error, and a code of 124 means timeout triggered
        chmod +x $PKGNAME.sh
        ./$PKGNAME.sh
        TESTSTATUS=$?
    fi
    # We now want to bundle up the adding and testing results,
    # as well as other useful information about the package,
    # into a JSON that we can concatenate with the rest of the
    # results later on.
    # TODO: Avoid the need to add JSON each time,
    # even if it is just a simple move effectively. Can we
    # just pull the encoding part out of JSON, which is all
    # we really need because most of it is parsing?
    julia -e 'Pkg.add("JSON")'
    julia $PKGEVALDIR/src/prepjson.jl $PKGNAME $TESTSTATUS /vagrant/${1}${2}
    # Finish up by removing the package. Doesn't actually remove
    # it in the sense of deleting the files - this helps the
    # overall process run faster, if my understanding of how
    # the .trash folder works is correct.
    # Called twice in case the package makes itself dirty
    julia -e "Pkg.rm(\"${PKGNAME}\"); Pkg.rm(\"${PKGNAME}\")"
    # In case something confusing happens?
    julia -e "Pkg.status()"
done

#######################################################################
# Bundle results together
echo "Bundling results"
cd /vagrant/
julia $PKGEVALDIR/src/joinjson.jl /vagrant/${1}${2} ${1}${2}

#######################################################################
echo "Finished normally! $1 $2"

#!/usr/bin/env bash
#######################################################################
# PackageEvaluator
# https://github.com/JuliaCI/PackageEvaluator.jl
# (c) Iain Dunning and contributors 2015
# Licensed under the MIT License
#######################################################################
# This script file is the provisioning script run by Vagrant after
# the VM is created. It sets up the environment for running PkgEval,
# then runs it to produce the JSON result files.
#
# Commandline arguments for this script, passed through by Vagrant.
# 1st: Julia version:   0.3 | 0.4 | 0.5
# 2nd: Test set to run: setup | all | AL | MZ
#######################################################################

#######################################################################
# Accept all apt-gets
# This is an attempt to deal with BinDeps packages trying to install
# dependencies with apt-get and asking for permissions. Normally we
# could pass a --yes argument to avoid this, but there isn't a way to
# do that with BinDeps. We can actually do a global override, which
# is what we do here. Since we are piping stuff into a file, the
# easiest way to sudo the whole thing is to do it in this style.
sudo su -c 'echo "APT::Get::Assume-Yes \"true\";" > /etc/apt/apt.conf.d/pkgevalforceyes'
sudo su -c 'echo "APT::Get::force-yes \"true\";" >> /etc/apt/apt.conf.d/pkgevalforceyes'
# Uncomment following line to check it did indeed work
#cat /etc/apt/apt.conf.d/pkgevalforceyes

#######################################################################
# Use newest R release
sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys E084DAB9
sudo add-apt-repository -y "deb http://cran.rstudio.com/bin/linux/ubuntu trusty/"

# Upgrade the installation and install Julia
sudo apt-get update    # Pull in latest versions
sudo apt-get upgrade   # Upgrade system packages
# Use first argument to script to distinguish between the versions
if [ "$1" == "0.3" ]
then
    wget -q -O julia.tar.gz https://julialang.s3.amazonaws.com/bin/linux/x64/0.3/julia-0.3-latest-linux-x86_64.tar.gz
elif [ "$1" == "0.4" ]
then
    wget -q -O julia.tar.gz https://julialang.s3.amazonaws.com/bin/linux/x64/0.4/julia-0.4-latest-linux-x86_64.tar.gz
elif [ "$1" == "0.5" ]
then
    wget -q -O julia.tar.gz https://julialang.s3.amazonaws.com/bin/linux/x64/0.5/julia-0.5-latest-linux-x86_64.tar.gz
else
    # Nightly
    wget -q -O julia.tar.gz https://julianightlies.s3.amazonaws.com/bin/linux/x64/julia-latest-linux64.tar.gz
fi
mkdir julia
tar -zxf julia.tar.gz -C ./julia --strip-components=1
rm julia.tar.gz
export PATH="${PATH}:/home/vagrant/julia/bin/"
# Retain PATH to make it easier to use VM for debugging
echo "export PATH=\"\${PATH}:/home/vagrant/julia/bin/\"" >> /home/vagrant/.profile

# Set a 10 minute absolute timeout and 2 minute low speed timeout
# for curl because of unreliable downloads like wcslib, graphviz
echo "max-time = 600" >> ~/.curlrc
echo "speed-time = 120" >> ~/.curlrc

#######################################################################
# Install any dependencies that aren't handled by BinDeps
# Somethings can't be or don't make sense to be installed by BinDeps,
# so we will do them manually. Package maintainers can submit PRs to
# extend this list as needed.
# Need git of course, doesn't come by default in the image
sudo apt-get install git
# Need X virtual frame buffer for many packages
sudo apt-get install xvfb
# Need GMP for e.g. GLPK, why not get some PCRE too
sudo apt-get install libpcre3-dev libgmp-dev
# Need gfortran for e.g. GLMNet.jl, Ipopt.jl, KernSmooth.jl...
sudo apt-get install gfortran pkg-config
# Need unzip for e.g. Blink.jl, FLANN.jl
sudo apt-get install unzip
# Need cmake for e.g. GLFW.jl, Metis.jl
sudo apt-get install cmake
# Install R for e.g. Rif.jl, RCall.jl
sudo apt-get install r-base r-base-dev
# Install Java for e.g. JavaCall.jl, Taro.jl
# From: http://stackoverflow.com/q/19275856/3822752
sudo add-apt-repository -y ppa:webupd8team/java
sudo apt-get update
echo debconf shared/accepted-oracle-license-v1-1 select true | sudo debconf-set-selections
echo debconf shared/accepted-oracle-license-v1-1 seen true | sudo debconf-set-selections
sudo apt-get -y install oracle-java7-installer
export JAVA_HOME=/usr/lib/jvm/java-7-oracle
echo "export JAVA_HOME=/usr/lib/jvm/java-7-oracle" >> /home/vagrant/.profile
# Use the Conda Package as Python environment
export PYTHON=""
# need /usr/share/dict/words for TextAnalysis.jl
sudo apt-get install wamerican
# Need xmllint (and others?) for XMLDict.jl
sudo apt-get install libxml2-utils
# ArrayFire
wget -q http://ci.arrayfire.org/userContent/Linux/ArrayFire-no-gl-v3.3.2_Linux_x86_64.sh
sudo chmod +x ArrayFire-no-gl-v3.3.2_Linux_x86_64.sh
sudo ./ArrayFire-no-gl-v3.3.2_Linux_x86_64.sh --exclude-subdir --prefix=/usr/local
export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/lib/
rm ArrayFire-no-gl-v3.3.2_Linux_x86_64.sh

#######################################################################
# Get PackageEvaluator scripts
PKGEVALDIR="/home/vagrant/pkgeval"
git clone https://github.com/JuliaCI/PackageEvaluator.jl.git $PKGEVALDIR
# Make results folders. Folder name is second argument to this script.
# These folders are shared - i.e. we are writing to outside the VM,
# most likely the PackageEvaluator.jl/scripts folder.
rm -rf /vagrant/${1}${2}
mkdir /vagrant/${1}${2}
cd /vagrant/${1}${2}
# Initialize METADATA for testing
# Note that it is important for it to not be in the /vagrant/ folder as
# that seems to mess with symlinks quite badly.
# See https://github.com/JuliaOpt/Ipopt.jl/issues/31 for discussion
# about this breaking a build.
julia -e "Pkg.init(); println(Pkg.dir())"

#######################################################################
# Run PackageEvaluator
if [ "$2" == "all" ]
then
    LOOPOVER=/home/vagrant/.julia/v${1}/METADATA/*
elif [ "$2" == "AL" ]
then
    LOOPOVER=/home/vagrant/.julia/v${1}/METADATA/[A-L]*;
elif [ "$2" == "MZ" ]
then
    LOOPOVER=/home/vagrant/.julia/v${1}/METADATA/[M-Z]*;
fi
# For every package name...
for f in $LOOPOVER;
do
    # Extract just the package name from the path
    PKGNAME=$(basename "$f")
    # Attempt to add the package. We give it half an hour - most
    # use far less, some do push it. CoinOptServices.jl set the
    # standard for build time and memory consumption, see
    # https://github.com/JuliaCI/PackageEvaluator.jl/issues/83
    # The log for adding the package will go in the results folder.
    timeout 2000s julia -e "Pkg.add(\"${PKGNAME}\")" 2>&1 | tee PKGEVAL_${PKGNAME}_add.log
    # A package can have four states:
    # - Not testable: for some reason, we can't even analyze how
    #   broken or not the package is, usually due to a limitation
    #   of PackageEvaluaor itself.
    # - No tests: the package doesn't even have tests. We used to
    #   further distinguish this by seeing if the package loads,
    #   but loading doesn't mean it actually works so this is bit
    #   misleading.
    # - Tests fail: the package has tests, and they don't pass.
    # - Tests pass: the tests pass!
    # We first run a script that:
    # - Identifies if we are testable or not. If we are not,
    #   exit with status 1 (not testable), or 2 (no tests).
    # - If we are, create a shell script to run the tests that
    #   includes xvfb, timeout, etc.
    julia $PKGEVALDIR/src/preptest.jl $PKGNAME
    TESTSTATUS=$?
    if [ $TESTSTATUS -eq 255 ]
    then
        # Not testable
        echo "NOT TESTABLE"
    elif [ $TESTSTATUS -eq 254 ]
    then
        # No tests
        echo "NO TESTS"
    else
        # Has tests, we need to run them
        # preptest.jl should have created a shell script to
        # run them. We just need to run that shell script
        # and store the error code. A code of 0 means the
        # tests passed, a code of 1 means Julia threw an
        # error, and a code of 124 means timeout triggered
        chmod +x $PKGNAME.sh
        ./$PKGNAME.sh
        TESTSTATUS=$?
    fi
    # We now want to bundle up the adding and testing results,
    # as well as other useful information about the package,
    # into a JSON that we can concatenate with the rest of the
    # results later on.
    # TODO: Avoid the need to add JSON each time,
    # even if it is just a simple move effectively. Can we
    # just pull the encoding part out of JSON, which is all
    # we really need because most of it is parsing?
    julia -e 'Pkg.add("JSON")'
    julia $PKGEVALDIR/src/prepjson.jl $PKGNAME $TESTSTATUS /vagrant/${1}${2}
    # Finish up by removing the package. Doesn't actually remove
    # it in the sense of deleting the files - this helps the
    # overall process run faster, if my understanding of how
    # the .trash folder works is correct.
    # Called twice in case the package makes itself dirty
    julia -e "Pkg.rm(\"${PKGNAME}\"); Pkg.rm(\"${PKGNAME}\")"
    # In case something confusing happens?
    julia -e "Pkg.status()"
done

#######################################################################
# Bundle results together
echo "Bundling results"
cd /vagrant/
julia $PKGEVALDIR/src/joinjson.jl /vagrant/${1}${2} ${1}${2}

#######################################################################
echo "Finished normally! $1 $2"


fixed endpoint

fixed endpoint

#!/bin/bash
set -e

# help
if test "$help" = "1";  then
  cat /HELP.md

  exit 0
else
  echo "show all options:"
  echo "  docker run -e help=1 arangodb"
fi

echo
echo "starting ArangoDB in stand-alone mode"

# start in development mode
if test "$development" = "1";  then
  D1="--javascript.dev-app-path"
  D2="/apps-dev"
fi

# fix permissions
touch /logs/arangodb.log
rm -rf /tmp/arangodb
mkdir /tmp/arangodb

chown arangodb:arangodb /data /apps /apps-dev /logs /logs/arangodb.log /tmp/arangodb

# pipe logfile to standard out
if test "$verbose" = "1";  then
  tail -f /logs/arangodb.log &
fi

# start server
if test "$console" = "1";  then
  /usr/sbin/arangod \
        --uid arangodb \
        --gid arangodb \
        --database.directory /data \
        --javascript.app-path /apps \
        --log.file /logs/arangodb.log \
        --temp-path /tmp/arangodb \
	--server.endpoint tcp://0.0.0.0:8529/ \
        $D1 $D2 \
        "$@" &
  /bin/bash
else
  exec /usr/sbin/arangod \
	--uid arangodb \
	--gid arangodb \
        --database.directory /data \
        --javascript.app-path /apps \
	--log.file /logs/arangodb.log \
        --temp-path /tmp/arangodb \
	--server.endpoint tcp://0.0.0.0:8529/ \
	$D1 $D2 \
	"$@"
fi

#!/bin/bash
set -e

# help
if test "$help" = "1";  then
  cat /HELP.md

  exit 0
else
  echo "show all options:"
  echo "  docker run -e help=1 arangodb"
fi

echo
echo "starting ArangoDB in stand-alone mode"

# start in development mode
if test "$development" = "1";  then
  D1="--javascript.dev-app-path"
  D2="/apps-dev"
fi

# fix permissions
touch /logs/arangodb.log
rm -rf /tmp/arangodb
mkdir /tmp/arangodb

chown arangodb:arangodb /data /apps /apps-dev /logs /logs/arangodb.log /tmp/arangodb

# pipe logfile to standard out
if test "$verbose" = "1";  then
  tail -f /logs/arangodb.log &
fi

# start server
if test "$console" = "1";  then
  /usr/sbin/arangod \
        --uid arangodb \
        --gid arangodb \
        --database.directory /data \
        --javascript.app-path /apps \
        --log.file /logs/arangodb.log \
        --temp-path /tmp/arangodb \
        $D1 $D2 \
        "$@" &
  /bin/bash
else
  exec /usr/sbin/arangod \
	--uid arangodb \
	--gid arangodb \
        --database.directory /data \
        --javascript.app-path /apps \
	--log.file /logs/arangodb.log \
        --temp-path /tmp/arangodb \
	$D1 $D2 \
	"$@"
fi


test gh gap pages

test gh gap pages

#!/usr/bin/env bash
set -ex

CURRENT=`pwd`

# Test GAP
GAP="$GAPROOT/bin/gap.sh --quitonbreak -q"

mkdir $COVDIR
$GAP --cover $COVDIR/test.coverage $GAPROOT/pkg/francy/tst/testall.g

# Test Javascript
cd js

npm run lint
npm run coverage
npm run docs

# Test Extensions - won't produce any coverage obviously
cd packages/francy-extension-jupyter

pyenv local 3.6

# install jupyter
pip3.6 install jupyter
#pip3.6 install jupyterlab

# install extension on pip
pip3.6 install -e .

# install extension link on jupyter notebook
`pyenv which jupyter` nbextension install --symlink --py --sys-prefix jupyter_francy
`pyenv which jupyter` nbextension enable --py --sys-prefix jupyter_francy
`pyenv which jupyter` nbextension list 2>&1 | grep -q jupyter_francy

# TODO FIXME - labextension does not work!
# install extension link on jupyter lab
#`pyenv which jupyter` labextension link
#`pyenv which jupyter` labextension list 2>&1 | grep -q jupyter_francy

cd $CURRENT

cd gap

cat makedoc.g | $GAP

cd $CURRENT/gap
# Add a new remote pointing to the GitHubPagesForGAP repository
git remote add gh-gap https://github.com/gap-system/GitHubPagesForGAP
git fetch gh-gap

# Create a fresh gh-pages branch from the new remote
git branch gh-pages gh-gap/gh-pages --no-track

# Create a new worktree and change into it
git worktree add gh-pages gh-pages
cd gh-pages

cp -f ../PackageInfo.g ../README* .
cp -f ../doc/*.{css,html,js,txt} doc/

$GAPROOT/bin/gap.sh update.g

git add PackageInfo.g README* doc/ _data/package.yml
git commit -m "Setup gh-pages based on GitHubPagesForGAP"
git push -f --set-upstream origin gh-pages


Check for root password to avoid accidental lock-out from admin node (bnc#782525)

#! /bin/bash

# This script is supposed to be run after being installed via the
# crowbar rpm from the SUSE Cloud ISO.  In that context, it is
# expected that all other required repositories (SP2, Updates, etc.)
# are already set up, and a lot of the required files will already be
# in the right place.  However if you want to test setup on a vanilla
# SLES system, you can follow the manual steps below:
#
# 0. export CROWBAR_TESTING=true
# 1. Copy all barclamps to /opt/dell/barclamps
#    You'll want:
#      crowbar database deployer dns glance ipmi keystone logging
#      nagios network nova nova_dashboard ntp openstack
#      provisioner swift
# 2. Copy extra/barclamp* to /opt/dell/bin/
# 4. You should probably set eth0 to be static IP 192.168.124.10/24.
# 5. rsync the Devel:Cloud, SLES-11-SP2-LATEST, SLE-11-SP2-SDK-LATEST,
#    and possibly other repos into locations under /srv/tftpboot -
#    see https://github.com/SUSE/cloud/wiki/Crowbar for details.

LOGFILE=/var/log/chef/install.log

set -e

run_succeeded=

exit_handler () {
    if [ -z "$run_succeeded" ]; then
        cat <<EOF

Crowbar installation terminated prematurely.  Please examine the above
output or $LOGFILE for clues as to what went wrong.
You should also check the SUSE Cloud Installation Manual, in
particular the Troubleshooting section.  Note that this script can
safely be re-run multiple times if required.
EOF
    fi
}

trap exit_handler EXIT

exec >  >(tee -a $LOGFILE    )
exec 2> >(tee -a $LOGFILE >&2)

echo "`date` $0 started with args: $*"

die() { echo "$(date '+%F %T %z'): $*" >&2; res=1; exit 1; }

ensure_service_running () {
    service="$1"
    regexp="${2:-running}"
    if service $service status | egrep -q "$regexp"; then
        echo "$service is already running - no need to start."
    else
        service $service start
        sleep 4
    fi
}

rootpw=$( getent shadow root | cut -d: -f2 )
case "$rootpw" in
    \*|\!*)
        die "root password is unset or locked.  Chef will rewrite /root/.ssh/authorized_keys; therefore to avoid being accidentally locked out of this admin node, you should first ensure you have a working root password."
        ;;
esac

# It is exceedingly important that 'hostname -f' actually returns an FQDN!
# if it doesn't, add an entry to /etc/hosts, e.g.:
#    192.168.124.10 cb-admin.example.com cb-admin
if ! FQDN=$(hostname -f 2>/dev/null); then
    die "Unable to detect fully-qualified hostname. Aborting."
fi

if ! DOMAIN=$(hostname -d 2>/dev/null); then
    die "Unable to detect DNS domain name. Aborting."
fi

if [ -z "$FQDN" -o -z "$DOMAIN" ]; then
    die "Unable to detect fully-qualified hostname. Aborting."
fi

if ! resolved=$(getent ahosts $FQDN 2>/dev/null); then
    die "Unable to resolve hostname $FQDN via host(1). Please check your configuration of DNS, hostname, and /etc/hosts. Aborting."
fi

IPv4_addr=$( echo "$resolved" | awk '{ if ($1 !~ /:/) { print $1; exit } }' )
IPv6_addr=$( echo "$resolved" | awk '{ if ($1  ~ /:/) { print $1; exit } }' )
if [ -z "$IPv4_addr" -a -z "$IPv6_addr" ]; then
    die "Could not resolve $FQDN to an IPv4 or IPv6 address. Aborting."
fi

if [ -n "$IPv4_addr" ]; then
    echo "$FQDN resolved to IPv4 address: $IPv4_addr"
    if ! ip addr | grep -q "inet $IPv4_addr"; then
        die "No local interfaces configured with address $IPv4_addr. Aborting."
    fi
    if [[ "$IPv4_addr" =~ ^127 ]]; then
        die "$FQDN resolves to a loopback address. Aborting."
    fi
    if ! /opt/dell/bin/bc-network-admin-helper.rb "$IPv4_addr" < /opt/dell/chef/data_bags/crowbar/bc-template-network.json; then
        die "IPv4 address of admin node not in admin range of admin network. Please check and fix with yast2 crowbar. Aborting."
    fi
fi
if [ -n "$IPv6_addr" ]; then
    echo "$FQDN resolved to IPv6 address: $IPv6_addr"
    if ! ip addr | grep -q "inet6 $IPv6_addr"; then
        die "No local interfaces configured with address $IPv6_addr. Aborting."
    fi
fi

# Note that the grep will fail if iptables' output changes; unlikely to happen,
# but...
if LANG=C iptables -n -L | grep -qvE '^$|^Chain [^ ]|^target     prot'; then
    die "Firewall is not completely disabled. Aborting."
fi

if ! ping -c 1 $FQDN >/dev/null 2>&1; then
    die "Failed to ping $FQDN; please check your network configuration. Aborting."
fi

# output details, that will make remote debugging via bugzilla much easier
# for us
/usr/bin/zypper lr -d
/bin/rpm -qV crowbar || :
/usr/bin/lscpu
/bin/df -h
/usr/bin/free -m
/bin/ls -la /srv/tftpboot/repos/ /srv/tftpboot/repos/Cloud/ /srv/tftpboot/suse-11.2/install/
/usr/bin/grep media_url /opt/dell/chef/cookbooks/provisioner/templates/default/autoyast.xml.erb

CROWBAR=/opt/dell/bin/crowbar

for repo in suse-11.2/install repos/Cloud; do
    repo=/srv/tftpboot/$repo
    if ! [ -e $repo/content.asc ] && ! [ -e $repo/repodata/repomd.xml.asc ]; then
        if [ -n "$CROWBAR_TESTING" ]; then
            die "$repo has not been set up yet; please see https://github.com/SUSE/cloud/wiki/Crowbar"
        else
            die "$repo has not been set up yet; please check you didn't miss a step in the installation guide."
        fi
    fi
done

if [ -n "$CROWBAR_TESTING" ]; then
    # This is supposed to go away once the Chef dependencies are included in the
    # add-on image.  Note that SP1 is required for rubygem-haml at least, but
    # the new maintenance model requires SP1 repos alongside SP2 anyway.
    zypper ar    http://dist.suse.de/install/SLP/SLES-11-SP2-GM/x86_64/DVD1 sp2
    zypper ar    http://dist.suse.de/install/SLP/SLE-11-SP2-SDK-GM/x86_64/DVD1/ sdk-sp2
    zypper ar    http://dist.suse.de/ibs/SUSE:/SLE-11-SP1:/GA/standard/ sp1-ga
    zypper ar    http://dist.suse.de/ibs/SUSE:/SLE-11-SP2:/GA/standard/ sp2-ga
    zypper ar -f http://dist.suse.de/ibs/SUSE:/SLE-11-SP1:/Update/standard/ sp1-update
    zypper ar -f http://dist.suse.de/ibs/SUSE:/SLE-11-SP2:/Update/standard/ sp2-update
    zypper ar -f http://dist.suse.de/ibs/Devel:/Cloud/SLE_11_SP2/ cloud

    # install chef and its dependencies
    zypper --gpg-auto-import-keys in rubygem-chef-server rubygem-chef rabbitmq-server couchdb java-1_6_0-ibm rubygem-activesupport

    # also need these (crowbar dependencies):
    zypper in rubygem-kwalify rubygem-ruby-shadow tcpdump

    # Need this for provisioner to work:
    mkdir -p /srv/tftpboot/discovery/pxelinux.cfg
    cat > /srv/tftpboot/discovery/pxelinux.cfg/default <<EOF
DEFAULT pxeboot
TIMEOUT 20
PROMPT 0
LABEL pxeboot
        KERNEL vmlinuz0
        APPEND initrd=initrd0.img root=/sledgehammer.iso rootfstype=iso9660 rootflags=loop
ONERROR LOCALBOOT 0
EOF

    # You'll also need:
    #   /srv/tftpboot/discovery/initrd0.img
    #   /srv/tftpboot/discovery/vmlinuz0
    # These can be obtained from a sleshammer image or from an existing
    # ubuntu admin node.
fi

chkconfig rabbitmq-server on
ensure_service_running rabbitmq-server '^Node .+ with Pid [0-9]+: running'

if rabbitmqctl list_vhosts | grep -q '^/chef$'; then
    : /chef vhost already added
else
    rabbitmqctl add_vhost /chef
fi

if rabbitmqctl list_users 2>&1 | grep -q '^chef	'; then
    : chef user already added
else
    rabbit_chef_password=$( dd if=/dev/urandom count=1 bs=16 2>/dev/null | base64 | tr -d / )
    rabbitmqctl add_user chef "$rabbit_chef_password"
    # Update "amqp_pass" in  /etc/chef/server.rb and solr.rb
    sed -i 's/amqp_pass ".*"/amqp_pass "'"$rabbit_chef_password"'"/' /etc/chef/{server,solr}.rb
fi

chef_webui_password=$( dd if=/dev/urandom count=1 bs=16 2>/dev/null | base64 | tr -d / )
sed -i "s/^\s*web_ui_admin_default_password\s*\".*\"/web_ui_admin_default_password \"$chef_webui_password\"/" /etc/chef/webui.rb
chmod o-rwx /etc/chef /etc/chef/{server,solr,webui}.rb

rabbitmqctl set_permissions -p /chef chef ".*" ".*" ".*"

chkconfig couchdb on
ensure_service_running couchdb

# increase chef-solr index field size
perl -i -pe 's{<maxFieldLength>.*</maxFieldLength>}{<maxFieldLength>200000</maxFieldLength>}' /var/lib/chef/solr/conf/solrconfig.xml

services='solr expander server server-webui'
for service in $services; do
    chkconfig chef-${service} on
done

for service in $services; do
    ensure_service_running chef-${service}
done

# chef-server-webui won't start if /etc/chef/webui.pem doesn't exist, which
# may be the case (chef-server generates it if not present, and if the webui
# starts too soon, it won't be there yet).
for ((x=1; x<6; x++)); do
    sleep 10
    service chef-server-webui status >/dev/null && { chef_webui_running=true; break; }
    service chef-server-webui start
done
if [[ ! $chef_webui_running ]]; then
    echo "WARNING: unable to start chef-server-webui"
fi

if ! [ -e ~/.chef/knife.rb ]; then
    yes '' | knife configure -i
fi

node_info=$(knife node show $FQDN 2>/dev/null || :)
if echo "$node_info" | grep -q 'Run List:.*role'; then
    echo "Chef runlist for $FQDN is already populated; skipping initial chef-client run."
else
    cat <<EOF
Performing initial chef-client run ...
This can cause warnings about /etc/chef/client.rb missing and
the run list being empty; they can be safely ignored.
EOF
    chef-client
fi

# Don't use this one - crowbar barfs due to hyphens in the "id" attribute.
#CROWBAR_FILE="/opt/dell/barclamps/crowbar/chef/data_bags/crowbar/bc-template-crowbar.json"
: ${CROWBAR_FILE:="/etc/crowbar/crowbar.json"}

mkdir -p /opt/dell/crowbar_framework
CROWBAR_REALM=$(/opt/dell/barclamps/provisioner/updates/parse_node_data $CROWBAR_FILE -a attributes.crowbar.realm)
CROWBAR_REALM=${CROWBAR_REALM##*=}

# Generate the machine install username and password.
if [[ ! -e /etc/crowbar.install.key && $CROWBAR_REALM ]]; then
    dd if=/dev/urandom bs=65536 count=1 2>/dev/null |sha512sum - 2>/dev/null | \
        (read key rest; echo "machine-install:$key" >/etc/crowbar.install.key)
fi

if [[ $CROWBAR_REALM && -f /etc/crowbar.install.key ]]; then
    export CROWBAR_KEY=$(cat /etc/crowbar.install.key)
    sed -i -e "s/machine_password/${CROWBAR_KEY##*:}/g" $CROWBAR_FILE
fi

/opt/dell/bin/barclamp_install.rb /opt/dell/barclamps/crowbar

#
# Take care that the barclamps are installed in the right order
# If you've got a full openstack set installed, e.g.: nagios has to be
# installed before keystone, etc.
#
for i in deployer dns database ipmi nagios keystone \
         glance logging network ceph nova nova_dashboard ntp openstack \
         provisioner swift ; do
    if [ -e /opt/dell/crowbar_framework/barclamps/$i.yml ]; then
        echo "$i barclamp is already installed"
    else
        /opt/dell/bin/barclamp_install.rb /opt/dell/barclamps/$i
    fi
done

# Configure chef to set up bind with correct local domain and DNS forwarders.
dns_template=/opt/dell/chef/data_bags/crowbar/bc-template-dns.json
[ -f $dns_template ] || die "$dns_template doesn't exist"
nameservers=$( awk '/^nameserver/ {print $2}' /etc/resolv.conf )
# This will still work if there are no nameservers.
/opt/dell/bin/bc-dns-json.rb $DOMAIN $nameservers < $dns_template > /tmp/bc-template-dns.json
echo "Instructing chef to configure bind with the following DNS forwarders: $nameservers"
knife data bag from file crowbar /tmp/bc-template-dns.json

echo "Create Admin node role"
NODE_ROLE="crowbar-${FQDN//./_}"
cat > /tmp/role.rb <<EOF
name "$NODE_ROLE"
description "Role for $FQDN"
run_list()
default_attributes( "crowbar" => { "network" => {} } )
override_attributes()
EOF
knife role from file /tmp/role.rb

knife node run_list add "$FQDN" role["crowbar"]
knife node run_list add "$FQDN" role["deployer-client"]
knife node run_list add "$FQDN" role["$NODE_ROLE"]

# at this point you can run chef-client from the command line to start
# the crowbar bootstrapping

chef-client

# OOC, what, if anything, is responsible for starting rainbows/crowbar under bluepill?
ensure_service_running crowbar

# Make sure looper_chef_client is a NOOP until we are finished deploying
touch /tmp/deploying
# This works because
#
#   crowbar_framework/app/models/provisioner_service.rb
#   crowbar_framework/app/models/service_object.rb
#
# both invoke
#
#   /opt/dell/bin/single_chef_client.sh (from barclamps/crowbar/bin)
#
# which invokes
#
#   /opt/dell/bin/looper_chef_client.sh
#
# which exits immediately if /tmp/deploying exists.

# From here, you should probably read along with the equivalent steps in
# install-chef.sh for comparison

if [ "$($CROWBAR crowbar proposal list)" != "default" ] ; then
    proposal_opts=()
    # If your custom crowbar.json is somewhere else, probably substitute that here
    if [[ -e $CROWBAR_FILE ]]; then
        proposal_opts+=(--file $CROWBAR_FILE)
    fi
    proposal_opts+=(proposal create default)

    # Sometimes proposal creation fails if Chef and Crowbar are not quite
    # fully prepared -- this can happen due to solr not having everything
    # fully indexed yet.  So we don't want to just fail immediatly if
    # we fail to create a proposal -- instead, we will kick Chef, sleep a bit,
    # and try again up to 5 times before bailing out.
    for ((x=1; x<6; x++)); do
        $CROWBAR crowbar "${proposal_opts[@]}" && { proposal_created=true; break; }
        echo "Proposal create failed, pass $x.  Will kick Chef and try again."
        chef-client
        sleep 1
    done
    if [[ ! $proposal_created ]]; then
        die "Could not create default proposal"
    fi
fi

# this has machine key world readable? care?
$CROWBAR crowbar proposal show default >/var/log/default-proposal.json

# next will fail if ntp barclamp not present (or did for me...)
$CROWBAR crowbar proposal commit default || \
    die "Could not commit default proposal!"

$CROWBAR crowbar show default >/var/log/default.json

crowbar_up=true
chef-client

# here we whould check indexer/expander is finished

# original script has several calls to check_machine_role -- see source for
# this, in my limited testing it wasn't necessary on SUSE, but we should still
# do it anyway (if the role isn't present things break, apparently)

# BMC support?

# transition though all the states to ready.  Make sure that
# Chef has completly finished with transition before proceeding
# to the next.

for state in "discovering" "discovered" "hardware-installing" \
    "hardware-installed" "installing" "installed" "readying" "ready"
do
    while [[ -f "/tmp/chef-client.lock" ]]; do sleep 1; done
    printf "$state: "
    $CROWBAR crowbar transition "$FQDN" "$state" || \
        die "Transition to $state failed!"
    if type -f "transition_check_$state"&>/dev/null; then
        "transition_check_$state" || \
            die "Sanity check for transitioning to $state failed!"
    fi
    # chef_or_die "Chef run for $state transition failed!"
    chef-client
    # check_machine_role
done

# OK, let looper_chef_client run normally now.
rm /tmp/deploying

# Need chef-client daemon now
chkconfig chef-client on
ensure_service_running chef-client

# Spit out a warning message if we managed to not get an IP address
IPSTR=$($CROWBAR network show default | /opt/dell/barclamps/provisioner/updates/parse_node_data -a attributes.network.networks.admin.ranges.admin.start)
IP=${IPSTR##*=}
ip addr | grep -q $IP || {
    die "eth0 not configured, but should have been."
}

# Run tests -- currently the host will run this.
# 2012-05-24: test is failing, so only run if CROWBAR_RUN_TESTS=true
# (this is distinct from CROWBAR_TESTING, which would add extra repos etc.)
if [ -n "$CROWBAR_RUN_TESTS" ]; then
    /opt/dell/bin/barclamp_test.rb -t || \
        die "Crowbar validation has errors! Please check the logs and correct."
fi

for s in xinetd dhcpd apache2 ; do
    if ! /etc/init.d/$s status >/dev/null ; then
        die "service $s missing"
    fi
done

touch /opt/dell/crowbar_framework/.crowbar-installed-ok

cat <<EOF
Admin node deployed.

You can now visit the Crowbar web UI at:

    http://$IP:3000/

You should also now be able to PXE-boot a client.  Please refer
to the documentation for the next steps.

Note that to run the crowbar CLI tool, you will need to log out
and log back in again for the correct environment variables to
be set up.
EOF

run_succeeded=hooray

#! /bin/bash

# This script is supposed to be run after being installed via the
# crowbar rpm from the SUSE Cloud ISO.  In that context, it is
# expected that all other required repositories (SP2, Updates, etc.)
# are already set up, and a lot of the required files will already be
# in the right place.  However if you want to test setup on a vanilla
# SLES system, you can follow the manual steps below:
#
# 0. export CROWBAR_TESTING=true
# 1. Copy all barclamps to /opt/dell/barclamps
#    You'll want:
#      crowbar database deployer dns glance ipmi keystone logging
#      nagios network nova nova_dashboard ntp openstack
#      provisioner swift
# 2. Copy extra/barclamp* to /opt/dell/bin/
# 4. You should probably set eth0 to be static IP 192.168.124.10/24.
# 5. rsync the Devel:Cloud, SLES-11-SP2-LATEST, SLE-11-SP2-SDK-LATEST,
#    and possibly other repos into locations under /srv/tftpboot -
#    see https://github.com/SUSE/cloud/wiki/Crowbar for details.

LOGFILE=/var/log/chef/install.log

set -e

run_succeeded=

exit_handler () {
    if [ -z "$run_succeeded" ]; then
        cat <<EOF

Crowbar installation terminated prematurely.  Please examine the above
output or $LOGFILE for clues as to what went wrong.
You should also check the SUSE Cloud Installation Manual, in
particular the Troubleshooting section.  Note that this script can
safely be re-run multiple times if required.
EOF
    fi
}

trap exit_handler EXIT

exec >  >(tee -a $LOGFILE    )
exec 2> >(tee -a $LOGFILE >&2)

echo "`date` $0 started with args: $*"

die() { echo "$(date '+%F %T %z'): $*" >&2; res=1; exit 1; }

ensure_service_running () {
    service="$1"
    regexp="${2:-running}"
    if service $service status | egrep -q "$regexp"; then
        echo "$service is already running - no need to start."
    else
        service $service start
        sleep 4
    fi
}

# It is exceedingly important that 'hostname -f' actually returns an FQDN!
# if it doesn't, add an entry to /etc/hosts, e.g.:
#    192.168.124.10 cb-admin.example.com cb-admin
if ! FQDN=$(hostname -f 2>/dev/null); then
    die "Unable to detect fully-qualified hostname. Aborting."
fi

if ! DOMAIN=$(hostname -d 2>/dev/null); then
    die "Unable to detect DNS domain name. Aborting."
fi

if [ -z "$FQDN" -o -z "$DOMAIN" ]; then
    die "Unable to detect fully-qualified hostname. Aborting."
fi

if ! resolved=$(getent ahosts $FQDN 2>/dev/null); then
    die "Unable to resolve hostname $FQDN via host(1). Please check your configuration of DNS, hostname, and /etc/hosts. Aborting."
fi

IPv4_addr=$( echo "$resolved" | awk '{ if ($1 !~ /:/) { print $1; exit } }' )
IPv6_addr=$( echo "$resolved" | awk '{ if ($1  ~ /:/) { print $1; exit } }' )
if [ -z "$IPv4_addr" -a -z "$IPv6_addr" ]; then
    die "Could not resolve $FQDN to an IPv4 or IPv6 address. Aborting."
fi

if [ -n "$IPv4_addr" ]; then
    echo "$FQDN resolved to IPv4 address: $IPv4_addr"
    if ! ip addr | grep -q "inet $IPv4_addr"; then
        die "No local interfaces configured with address $IPv4_addr. Aborting."
    fi
    if [[ "$IPv4_addr" =~ ^127 ]]; then
        die "$FQDN resolves to a loopback address. Aborting."
    fi
    if ! /opt/dell/bin/bc-network-admin-helper.rb "$IPv4_addr" < /opt/dell/chef/data_bags/crowbar/bc-template-network.json; then
        die "IPv4 address of admin node not in admin range of admin network. Please check and fix with yast2 crowbar. Aborting."
    fi
fi
if [ -n "$IPv6_addr" ]; then
    echo "$FQDN resolved to IPv6 address: $IPv6_addr"
    if ! ip addr | grep -q "inet6 $IPv6_addr"; then
        die "No local interfaces configured with address $IPv6_addr. Aborting."
    fi
fi

# Note that the grep will fail if iptables' output changes; unlikely to happen,
# but...
if LANG=C iptables -n -L | grep -qvE '^$|^Chain [^ ]|^target     prot'; then
    die "Firewall is not completely disabled. Aborting."
fi

if ! ping -c 1 $FQDN >/dev/null 2>&1; then
    die "Failed to ping $FQDN; please check your network configuration. Aborting."
fi

# output details, that will make remote debugging via bugzilla much easier
# for us
/usr/bin/zypper lr -d
/bin/rpm -qV crowbar || :
/usr/bin/lscpu
/bin/df -h
/usr/bin/free -m
/bin/ls -la /srv/tftpboot/repos/ /srv/tftpboot/repos/Cloud/ /srv/tftpboot/suse-11.2/install/
/usr/bin/grep media_url /opt/dell/chef/cookbooks/provisioner/templates/default/autoyast.xml.erb

CROWBAR=/opt/dell/bin/crowbar

for repo in suse-11.2/install repos/Cloud; do
    repo=/srv/tftpboot/$repo
    if ! [ -e $repo/content.asc ] && ! [ -e $repo/repodata/repomd.xml.asc ]; then
        if [ -n "$CROWBAR_TESTING" ]; then
            die "$repo has not been set up yet; please see https://github.com/SUSE/cloud/wiki/Crowbar"
        else
            die "$repo has not been set up yet; please check you didn't miss a step in the installation guide."
        fi
    fi
done

if [ -n "$CROWBAR_TESTING" ]; then
    # This is supposed to go away once the Chef dependencies are included in the
    # add-on image.  Note that SP1 is required for rubygem-haml at least, but
    # the new maintenance model requires SP1 repos alongside SP2 anyway.
    zypper ar    http://dist.suse.de/install/SLP/SLES-11-SP2-GM/x86_64/DVD1 sp2
    zypper ar    http://dist.suse.de/install/SLP/SLE-11-SP2-SDK-GM/x86_64/DVD1/ sdk-sp2
    zypper ar    http://dist.suse.de/ibs/SUSE:/SLE-11-SP1:/GA/standard/ sp1-ga
    zypper ar    http://dist.suse.de/ibs/SUSE:/SLE-11-SP2:/GA/standard/ sp2-ga
    zypper ar -f http://dist.suse.de/ibs/SUSE:/SLE-11-SP1:/Update/standard/ sp1-update
    zypper ar -f http://dist.suse.de/ibs/SUSE:/SLE-11-SP2:/Update/standard/ sp2-update
    zypper ar -f http://dist.suse.de/ibs/Devel:/Cloud/SLE_11_SP2/ cloud

    # install chef and its dependencies
    zypper --gpg-auto-import-keys in rubygem-chef-server rubygem-chef rabbitmq-server couchdb java-1_6_0-ibm rubygem-activesupport

    # also need these (crowbar dependencies):
    zypper in rubygem-kwalify rubygem-ruby-shadow tcpdump

    # Need this for provisioner to work:
    mkdir -p /srv/tftpboot/discovery/pxelinux.cfg
    cat > /srv/tftpboot/discovery/pxelinux.cfg/default <<EOF
DEFAULT pxeboot
TIMEOUT 20
PROMPT 0
LABEL pxeboot
        KERNEL vmlinuz0
        APPEND initrd=initrd0.img root=/sledgehammer.iso rootfstype=iso9660 rootflags=loop
ONERROR LOCALBOOT 0
EOF

    # You'll also need:
    #   /srv/tftpboot/discovery/initrd0.img
    #   /srv/tftpboot/discovery/vmlinuz0
    # These can be obtained from a sleshammer image or from an existing
    # ubuntu admin node.
fi

chkconfig rabbitmq-server on
ensure_service_running rabbitmq-server '^Node .+ with Pid [0-9]+: running'

if rabbitmqctl list_vhosts | grep -q '^/chef$'; then
    : /chef vhost already added
else
    rabbitmqctl add_vhost /chef
fi

if rabbitmqctl list_users 2>&1 | grep -q '^chef	'; then
    : chef user already added
else
    rabbit_chef_password=$( dd if=/dev/urandom count=1 bs=16 2>/dev/null | base64 | tr -d / )
    rabbitmqctl add_user chef "$rabbit_chef_password"
    # Update "amqp_pass" in  /etc/chef/server.rb and solr.rb
    sed -i 's/amqp_pass ".*"/amqp_pass "'"$rabbit_chef_password"'"/' /etc/chef/{server,solr}.rb
fi

chef_webui_password=$( dd if=/dev/urandom count=1 bs=16 2>/dev/null | base64 | tr -d / )
sed -i "s/^\s*web_ui_admin_default_password\s*\".*\"/web_ui_admin_default_password \"$chef_webui_password\"/" /etc/chef/webui.rb
chmod o-rwx /etc/chef /etc/chef/{server,solr,webui}.rb

rabbitmqctl set_permissions -p /chef chef ".*" ".*" ".*"

chkconfig couchdb on
ensure_service_running couchdb

# increase chef-solr index field size
perl -i -pe 's{<maxFieldLength>.*</maxFieldLength>}{<maxFieldLength>200000</maxFieldLength>}' /var/lib/chef/solr/conf/solrconfig.xml

services='solr expander server server-webui'
for service in $services; do
    chkconfig chef-${service} on
done

for service in $services; do
    ensure_service_running chef-${service}
done

# chef-server-webui won't start if /etc/chef/webui.pem doesn't exist, which
# may be the case (chef-server generates it if not present, and if the webui
# starts too soon, it won't be there yet).
for ((x=1; x<6; x++)); do
    sleep 10
    service chef-server-webui status >/dev/null && { chef_webui_running=true; break; }
    service chef-server-webui start
done
if [[ ! $chef_webui_running ]]; then
    echo "WARNING: unable to start chef-server-webui"
fi

if ! [ -e ~/.chef/knife.rb ]; then
    yes '' | knife configure -i
fi

node_info=$(knife node show $FQDN 2>/dev/null || :)
if echo "$node_info" | grep -q 'Run List:.*role'; then
    echo "Chef runlist for $FQDN is already populated; skipping initial chef-client run."
else
    cat <<EOF
Performing initial chef-client run ...
This can cause warnings about /etc/chef/client.rb missing and
the run list being empty; they can be safely ignored.
EOF
    chef-client
fi

# Don't use this one - crowbar barfs due to hyphens in the "id" attribute.
#CROWBAR_FILE="/opt/dell/barclamps/crowbar/chef/data_bags/crowbar/bc-template-crowbar.json"
: ${CROWBAR_FILE:="/etc/crowbar/crowbar.json"}

mkdir -p /opt/dell/crowbar_framework
CROWBAR_REALM=$(/opt/dell/barclamps/provisioner/updates/parse_node_data $CROWBAR_FILE -a attributes.crowbar.realm)
CROWBAR_REALM=${CROWBAR_REALM##*=}

# Generate the machine install username and password.
if [[ ! -e /etc/crowbar.install.key && $CROWBAR_REALM ]]; then
    dd if=/dev/urandom bs=65536 count=1 2>/dev/null |sha512sum - 2>/dev/null | \
        (read key rest; echo "machine-install:$key" >/etc/crowbar.install.key)
fi

if [[ $CROWBAR_REALM && -f /etc/crowbar.install.key ]]; then
    export CROWBAR_KEY=$(cat /etc/crowbar.install.key)
    sed -i -e "s/machine_password/${CROWBAR_KEY##*:}/g" $CROWBAR_FILE
fi

/opt/dell/bin/barclamp_install.rb /opt/dell/barclamps/crowbar

#
# Take care that the barclamps are installed in the right order
# If you've got a full openstack set installed, e.g.: nagios has to be
# installed before keystone, etc.
#
for i in deployer dns database ipmi nagios keystone \
         glance logging network ceph nova nova_dashboard ntp openstack \
         provisioner swift ; do
    if [ -e /opt/dell/crowbar_framework/barclamps/$i.yml ]; then
        echo "$i barclamp is already installed"
    else
        /opt/dell/bin/barclamp_install.rb /opt/dell/barclamps/$i
    fi
done

# Configure chef to set up bind with correct local domain and DNS forwarders.
dns_template=/opt/dell/chef/data_bags/crowbar/bc-template-dns.json
[ -f $dns_template ] || die "$dns_template doesn't exist"
nameservers=$( awk '/^nameserver/ {print $2}' /etc/resolv.conf )
# This will still work if there are no nameservers.
/opt/dell/bin/bc-dns-json.rb $DOMAIN $nameservers < $dns_template > /tmp/bc-template-dns.json
echo "Instructing chef to configure bind with the following DNS forwarders: $nameservers"
knife data bag from file crowbar /tmp/bc-template-dns.json

echo "Create Admin node role"
NODE_ROLE="crowbar-${FQDN//./_}"
cat > /tmp/role.rb <<EOF
name "$NODE_ROLE"
description "Role for $FQDN"
run_list()
default_attributes( "crowbar" => { "network" => {} } )
override_attributes()
EOF
knife role from file /tmp/role.rb

knife node run_list add "$FQDN" role["crowbar"]
knife node run_list add "$FQDN" role["deployer-client"]
knife node run_list add "$FQDN" role["$NODE_ROLE"]

# at this point you can run chef-client from the command line to start
# the crowbar bootstrapping

chef-client

# OOC, what, if anything, is responsible for starting rainbows/crowbar under bluepill?
ensure_service_running crowbar

# Make sure looper_chef_client is a NOOP until we are finished deploying
touch /tmp/deploying
# This works because
#
#   crowbar_framework/app/models/provisioner_service.rb
#   crowbar_framework/app/models/service_object.rb
#
# both invoke
#
#   /opt/dell/bin/single_chef_client.sh (from barclamps/crowbar/bin)
#
# which invokes
#
#   /opt/dell/bin/looper_chef_client.sh
#
# which exits immediately if /tmp/deploying exists.

# From here, you should probably read along with the equivalent steps in
# install-chef.sh for comparison

if [ "$($CROWBAR crowbar proposal list)" != "default" ] ; then
    proposal_opts=()
    # If your custom crowbar.json is somewhere else, probably substitute that here
    if [[ -e $CROWBAR_FILE ]]; then
        proposal_opts+=(--file $CROWBAR_FILE)
    fi
    proposal_opts+=(proposal create default)

    # Sometimes proposal creation fails if Chef and Crowbar are not quite
    # fully prepared -- this can happen due to solr not having everything
    # fully indexed yet.  So we don't want to just fail immediatly if
    # we fail to create a proposal -- instead, we will kick Chef, sleep a bit,
    # and try again up to 5 times before bailing out.
    for ((x=1; x<6; x++)); do
        $CROWBAR crowbar "${proposal_opts[@]}" && { proposal_created=true; break; }
        echo "Proposal create failed, pass $x.  Will kick Chef and try again."
        chef-client
        sleep 1
    done
    if [[ ! $proposal_created ]]; then
        die "Could not create default proposal"
    fi
fi

# this has machine key world readable? care?
$CROWBAR crowbar proposal show default >/var/log/default-proposal.json

# next will fail if ntp barclamp not present (or did for me...)
$CROWBAR crowbar proposal commit default || \
    die "Could not commit default proposal!"

$CROWBAR crowbar show default >/var/log/default.json

crowbar_up=true
chef-client

# here we whould check indexer/expander is finished

# original script has several calls to check_machine_role -- see source for
# this, in my limited testing it wasn't necessary on SUSE, but we should still
# do it anyway (if the role isn't present things break, apparently)

# BMC support?

# transition though all the states to ready.  Make sure that
# Chef has completly finished with transition before proceeding
# to the next.

for state in "discovering" "discovered" "hardware-installing" \
    "hardware-installed" "installing" "installed" "readying" "ready"
do
    while [[ -f "/tmp/chef-client.lock" ]]; do sleep 1; done
    printf "$state: "
    $CROWBAR crowbar transition "$FQDN" "$state" || \
        die "Transition to $state failed!"
    if type -f "transition_check_$state"&>/dev/null; then
        "transition_check_$state" || \
            die "Sanity check for transitioning to $state failed!"
    fi
    # chef_or_die "Chef run for $state transition failed!"
    chef-client
    # check_machine_role
done

# OK, let looper_chef_client run normally now.
rm /tmp/deploying

# Need chef-client daemon now
chkconfig chef-client on
ensure_service_running chef-client

# Spit out a warning message if we managed to not get an IP address
IPSTR=$($CROWBAR network show default | /opt/dell/barclamps/provisioner/updates/parse_node_data -a attributes.network.networks.admin.ranges.admin.start)
IP=${IPSTR##*=}
ip addr | grep -q $IP || {
    die "eth0 not configured, but should have been."
}

# Run tests -- currently the host will run this.
# 2012-05-24: test is failing, so only run if CROWBAR_RUN_TESTS=true
# (this is distinct from CROWBAR_TESTING, which would add extra repos etc.)
if [ -n "$CROWBAR_RUN_TESTS" ]; then
    /opt/dell/bin/barclamp_test.rb -t || \
        die "Crowbar validation has errors! Please check the logs and correct."
fi

for s in xinetd dhcpd apache2 ; do
    if ! /etc/init.d/$s status >/dev/null ; then
        die "service $s missing"
    fi
done

touch /opt/dell/crowbar_framework/.crowbar-installed-ok

cat <<EOF
Admin node deployed.

You can now visit the Crowbar web UI at:

    http://$IP:3000/

You should also now be able to PXE-boot a client.  Please refer
to the documentation for the next steps.

Note that to run the crowbar CLI tool, you will need to log out
and log back in again for the correct environment variables to
be set up.
EOF

run_succeeded=hooray