Oops, operate on right argument

Oops, operate on right argument

#!/bin/sh

# Generate environment for using Farsight 2 from Git checkout.

realpath_ ()
{
    ( cd $1 2>/dev/null; pwd )
}

pjoin ()
{
    a=$(realpath_ $1)
    b="$2"

    [ -z "$a" ] && { echo $b ; return ; }

    if [ -z "$b" ]; then
        # Existing path list is empty.
        echo "$a"
    elif ! echo "$b" | sed -e 's/:/\\n/g' | grep -q "^$a$"; then
        # New path is not in path list.
        echo "$a:$b"
    else
        # New path is already in path list.
        echo "$b"
    fi
}

p=$(realpath_ $(dirname $0))
ppath="$FS_PLUGIN_PATH"

echo export PYTHONPATH=`pjoin "$p/python/.libs" "$PYTHONPATH"`
echo export GST_PLUGIN_PATH=`pjoin "$p/gst" "$GST_PLUGIN_PATH"`

for i in `find $p/transmitters -maxdepth 1 -type d -exec basename '{}' ';'`; do
    ppath=`pjoin "$p/transmitters/$i/.libs" "$ppath"`
done

echo "export FS_PLUGIN_PATH=$ppath"

#!/bin/sh

# Generate environment for using Farsight 2 from Git checkout.

realpath_ ()
{
    ( cd $0 2>/dev/null; pwd )
}

pjoin ()
{
    a=$(realpath_ $1)
    b="$2"

    [ -z "$a" ] && { echo $b ; return ; }

    if [ -z "$b" ]; then
        # Existing path list is empty.
        echo "$a"
    elif ! echo "$b" | sed -e 's/:/\\n/g' | grep -q "^$a$"; then
        # New path is not in path list.
        echo "$a:$b"
    else
        # New path is already in path list.
        echo "$b"
    fi
}

p=$(realpath_ $(dirname $0))
ppath="$FS_PLUGIN_PATH"

echo export PYTHONPATH=`pjoin "$p/python/.libs" "$PYTHONPATH"`
echo export GST_PLUGIN_PATH=`pjoin "$p/gst" "$GST_PLUGIN_PATH"`

for i in `find $p/transmitters -maxdepth 1 -type d -exec basename '{}' ';'`; do
    ppath=`pjoin "$p/transmitters/$i/.libs" "$ppath"`
done

echo "export FS_PLUGIN_PATH=$ppath"


Stub wrapper for the Payne

Stub wrapper for the Payne

#!/bin/bash

# This script updates the local installation of 4GP in a python virtual environment

# Make absolutely sure we're in the right directory to start with
cd "$(dirname "$0")"
cwd=`pwd`

# Sometimes this line is necessary, if your locale settings are broken
export LC_ALL=C

# Below we do a bit of brute-force uninstallation and cleaning of the python environment, as otherwise we can
# end up reinstalling a cached old version of packages, rather than forcing a download of a new copy of a package

rm -Rf ../virtualenv
virtualenv -p python3 ../virtualenv

# Activate python virtual environment
source ../virtualenv/bin/activate

# Install some of the python packages we required
pip install numpy==1.15.3  # Currently, 1.16.0 doesn't actually install
pip install scipy astropy mysqlclient flask matplotlib tables

# These additional packages are required by Sergey's RV code
pip install pyyaml numdifftools pandas

# Make sure we've got the latest version of the 4GP libraries installed in virtual environment
cd ${cwd}

cd ../4most-4gp/src/pythonModules/fourgp_speclib
rm -Rf build dist *.egg-info  # Clear out the cache to make sure we install latest version of code
python3 setup.py install

cd ../fourgp_cannon
rm -Rf build dist *.egg-info  # Clear out the cache to make sure we install latest version of code
python3 setup.py install

cd ../fourgp_payne
rm -Rf build dist *.egg-info  # Clear out the cache to make sure we install latest version of code
python3 setup.py install

cd ../fourgp_degrade
rm -Rf build dist *.egg-info  # Clear out the cache to make sure we install latest version of code
python3 setup.py install

cd ../fourgp_rv
rm -Rf build dist *.egg-info  # Clear out the cache to make sure we install latest version of code
python3 setup.py install

cd ../fourgp_specsynth
rm -Rf build dist *.egg-info  # Clear out the cache to make sure we install latest version of code
python3 setup.py install

cd ../fourgp_telescope_data
rm -Rf build dist *.egg-info  # Clear out the cache to make sure we install latest version of code
python3 setup.py install

cd ../fourgp_fourfs
rm -Rf build dist *.egg-info  # Clear out the cache to make sure we install latest version of code
python3 setup.py install

# Make sure we've got the latest version of Andy Casey's Cannon installed in virtual environment
cd ${cwd}
cd ../AnniesLasso
rm -Rf build dist *.egg-info  # Clear out the cache to make sure we install latest version of code
python3 setup.py install

# Make sure we've got the latest version of Anno Ho's Cannon installed in virtual environment
cd ${cwd}
cd ../TheCannon
rm -Rf build dist *.egg-info  # Clear out the cache to make sure we install latest version of code
python3 setup.py install

# Make sure we've got the latest version of pyphot installed in virtual environment
cd ${cwd}
cd ../pyphot
rm -Rf build dist *.egg-info  # Clear out the cache to make sure we install latest version of code
python3 setup.py install

# Make sure we've got the latest version of Sergey's RV code installed in virtual environment
cd ${cwd}
cd ../rvspecfit
rm -Rf build dist *.egg-info  # Clear out the cache to make sure we install latest version of code
python3 setup.py install

#!/bin/bash

# This script updates the local installation of 4GP in a python virtual environment

# Make absolutely sure we're in the right directory to start with
cd "$(dirname "$0")"
cwd=`pwd`

# Sometimes this line is necessary, if your locale settings are broken
export LC_ALL=C

# Below we do a bit of brute-force uninstallation and cleaning of the python environment, as otherwise we can
# end up reinstalling a cached old version of packages, rather than forcing a download of a new copy of a package

rm -Rf ../virtualenv
virtualenv -p python3 ../virtualenv

# Activate python virtual environment
source ../virtualenv/bin/activate

# Install some of the python packages we required
pip install numpy==1.15.3  # Currently, 1.16.0 doesn't actually install
pip install scipy astropy mysqlclient flask matplotlib tables

# These additional packages are required by Sergey's RV code
pip install pyyaml numdifftools pandas

# Make sure we've got the latest version of the 4GP libraries installed in virtual environment
cd ${cwd}

cd ../4most-4gp/src/pythonModules/fourgp_speclib
rm -Rf build dist *.egg-info  # Clear out the cache to make sure we install latest version of code
python3 setup.py install

cd ../fourgp_cannon
rm -Rf build dist *.egg-info  # Clear out the cache to make sure we install latest version of code
python3 setup.py install

cd ../fourgp_degrade
rm -Rf build dist *.egg-info  # Clear out the cache to make sure we install latest version of code
python3 setup.py install

cd ../fourgp_rv
rm -Rf build dist *.egg-info  # Clear out the cache to make sure we install latest version of code
python3 setup.py install

cd ../fourgp_specsynth
rm -Rf build dist *.egg-info  # Clear out the cache to make sure we install latest version of code
python3 setup.py install

cd ../fourgp_telescope_data
rm -Rf build dist *.egg-info  # Clear out the cache to make sure we install latest version of code
python3 setup.py install

cd ../fourgp_fourfs
rm -Rf build dist *.egg-info  # Clear out the cache to make sure we install latest version of code
python3 setup.py install

# Make sure we've got the latest version of Andy Casey's Cannon installed in virtual environment
cd ${cwd}
cd ../AnniesLasso
rm -Rf build dist *.egg-info  # Clear out the cache to make sure we install latest version of code
python3 setup.py install

# Make sure we've got the latest version of Anno Ho's Cannon installed in virtual environment
cd ${cwd}
cd ../TheCannon
rm -Rf build dist *.egg-info  # Clear out the cache to make sure we install latest version of code
python3 setup.py install

# Make sure we've got the latest version of pyphot installed in virtual environment
cd ${cwd}
cd ../pyphot
rm -Rf build dist *.egg-info  # Clear out the cache to make sure we install latest version of code
python3 setup.py install

# Make sure we've got the latest version of Sergey's RV code installed in virtual environment
cd ${cwd}
cd ../rvspecfit
rm -Rf build dist *.egg-info  # Clear out the cache to make sure we install latest version of code
python3 setup.py install


Create gen_letters.sh

Create gen_letters.sh

for i in A B C D E F G H I J K L M N O P Q R S T U V W X Y Z; do banner $i; done

error: pathspec 'gen_letters.sh' did not match any file(s) known to git


Add a install script (initial idea).

Add a install script (initial idea).

#!/bin/bh -e

ARCHIVE=seacatio-arm-linux-gnueabihf-vv1703-alpha.1.tar.gz
URL=https://getseacatiostoracc.blob.core.windows.net/getseacatio/releases/

curl ${URL}${ARCHIVE} | tar xz -C /opt/
chown -Rv root:root /opt/seacatio/

error: pathspec 'get/install.sh' did not match any file(s) known to git


Add test-lib.sh for our shell test framework

Our test framework is based on Sharness.
So the first thing to do is to source it.

# Test framework for go-ipfs
#
# Copyright (c) 2014 Christian Couder
# MIT Licensed; see the LICENSE file in this repository.
#
# We are using sharness (https://github.com/mlafeldt/sharness)
# which was extracted from the Git test framework.

# You need either sharness to be installed system-wide
# or to set SHARNESS_DIRECTORY properly

if test -z "$SHARNESS_DIRECTORY"
then
	SHARNESS_DIRECTORY=/usr/local/share/sharness
fi

SHARNESS_LIB="$SHARNESS_DIRECTORY/sharness.sh"

test -f "$SHARNESS_LIB" || {
	echo >&2 "Cannot find sharness.sh in: $SHARNESS_DIRECTORY"
	echo >&2 "Please install Sharness system-wide or set the"
	echo >&2 "SHARNESS_DIRECTORY environment variable."
	echo >&2 "See: https://github.com/mlafeldt/sharness"
	exit 1
}

. "$SHARNESS_LIB" || {
	echo >&2 "Cannot source: $SHARNESS_LIB"
	echo >&2 "Please check Sharness installation."
	exit 1
}

# Please put go-ipfs specific shell functions below

error: pathspec 'test/test-lib.sh' did not match any file(s) known to git


run on linux

run on linux

sudo -i

export HOME=/home/ec2-user
export PYTHONPATH=.
cd ~/temp
python active_data/app.py  --settings=tests/config/app_staging_settings.json
export HOME=/home/ec2-user
export PYTHONPATH=.
cd ~/temp
python active_data/app.py  --settings=tests/config/app_staging_settings.json

error: pathspec 'resources/scripts/app.sh' did not match any file(s) known to git


Add OpenSSL script to remove passphrase from private keys

#!/bin/bash

## Remove a pass phrase from a key
## Optionally create a copy of the same removing the pass phrase.
#
# http://www.akadia.com/services/ssh_test_certificate.html
# https://serversforhackers.com/self-signed-ssl-certificates
# https://www.sslshopper.com/article-most-common-openssl-commands.html

# A blank passphrase: change with your own
PASSPHRASE=''

if [[ "$#" -ne 2 ]]
then
    echo "Syntax: $(basename "$0") KEYFILE_PASSPHRASE_PROTECTED KEYFILE_PASSPHRASE_FREE"
    echo "Iillegal number of arguments!"
    exit 1
fi

KEYFILE_PASSPHRASE_PROTECTED="$1"
KEYFILE_PASSPHRASE_FREE="$2"
if [ "$KEYFILE_PASSPHRASE_FREE" == "$KEYFILE_PASSPHRASE_PROTECTED" ]
then
    echo "Syntax: $(basename "$0") KEYFILE_PASSPHRASE_PROTECTED KEYFILE_PASSPHRASE_FREE"
    echo "Protected and free key files must be different!"
    exit 1
fi

echo "#### PEM pass phrase free Private Key file in $KEYFILE_PASSPHRASE_FREE"
openssl rsa -in "$KEYFILE_PASSPHRASE_PROTECTED" -out "$KEYFILE_PASSPHRASE_FREE" -passin pass:$PASSPHRASE

error: pathspec 'web/ssl_util/ssl_remove_passphrase.sh' did not match any file(s) known to git


adding vcf_unphase.sh

adding vcf_unphase.sh

#!/usr/bin/env bash

awk -F $'\t' '\
BEGIN {OFS = FS}
/^[#]/ {print; next}
{
  for (i = 10; i<=NF; i++) {
    gsub("\\|","/",$i)
  }
print
}
'

# -F $'\t' BEGIN {OFS = FS} # split on tabs
# /^[#]/ {print; next} # just output lines starting with '#' (header)
# the rest replaces '|' with '/' in fields 10+ and prints the line

error: pathspec 'vcf_unphase.sh' did not match any file(s) known to git


Added autogen.sh

Added autogen.sh

error: pathspec 'sxc/autogen.sh' did not match any file(s) known to git


Set vscode as default editor

Set vscode as default editor

remote: Support for password authentication was removed on August 13, 2021.
remote: Please see https://docs.github.com/en/get-started/getting-started-with-git/about-remote-repositories#cloning-with-https-urls for information on currently recommended modes of authentication.
fatal: Authentication failed for 'https://github.com/danielgavrilov/dotfiles.git/'


system: make atom my default editor

system: make atom my default editor


Remove journals from sandbox builds

Remove journals from sandbox builds

#!/usr/bin/env bash

# Ansible provisioning wrapper script that
# assumes the following parameters set
# as environment variables
#
# - github_username
# - server_type
# - instance_type
# - region
# - aws_account
# - keypair
# - ami
# - root_ebs_size
# - security_group
# - dns_zone
# - dns_name
# - environment
# - name_tag
set -x

# Seeing the environment is fine, spewing secrets to the log isn't ok
env | grep -v AWS | grep -v ARN

export PYTHONUNBUFFERED=1
export BOTO_CONFIG=/var/lib/jenkins/${aws_account}.boto

# docker on OS-X includes your Mac's home directory in the socket path
# that SSH/Ansible uses for the control socket, pushing you over
# the 108 character limit.
if [ -f /.dockerenv ]; then
    export ANSIBLE_SSH_CONTROL_PATH=/tmp/%%C
fi

run_ansible() {
  if [[ "$VERBOSE" == "true" ]]; then
    verbose_arg='-vvv'
  else
    verbose_arg=''
  fi

  ansible-playbook $verbose_arg $@
  ret=$?
  if [[ $ret -ne 0 ]]; then
    exit $ret
  fi
}

# This DATE_TIME will be used as instance launch time tag
if [[ ! -n ${sandbox_life//[0-9]/} ]]  && [[ ${sandbox_life} -le 30 ]]; then
    TERMINATION_DATE_TIME=`date +"%m-%d-%Y %T" --date "${sandbox_life=7} days"`
else
   echo "Please enter the valid value for the sandbox_life(between 1 to 30)"
   exit 1
fi

if [[ -z $BUILD_USER ]]; then
    BUILD_USER=jenkins
fi

if [[ -z $BUILD_USER_ID ]]; then
    BUILD_USER_ID=edx-sandbox
fi

if [[ -z $WORKSPACE ]]; then
    dir=$(dirname $0)
    source "$dir/ascii-convert.sh"
else
    source "$WORKSPACE/configuration/util/jenkins/ascii-convert.sh"
fi

if [[ -z $static_url_base ]]; then
  static_url_base="/static"
fi

if [[ -z $github_username  ]]; then
  github_username=$BUILD_USER_ID
fi

# Having access keys OR a boto config allows sandboxes to be built.
if [[ ( -z $AWS_ACCESS_KEY_ID || -z $AWS_SECRET_ACCESS_KEY ) && (! -f $BOTO_CONFIG) ]]; then
  echo "AWS credentials not found for $aws_account"
  exit 1
fi

extra_vars_file="/var/tmp/extra-vars-$$.yml"
sandbox_secure_vars_file="${WORKSPACE}/configuration-secure/ansible/vars/developer-sandbox.yml"
sandbox_internal_vars_file="${WORKSPACE}/configuration-internal/ansible/vars/developer-sandbox.yml"
extra_var_arg="-e@${extra_vars_file}"
program_manager="false"

if [[ $edx_internal == "true" ]]; then
    # if this is a an edx server include
    # the secret var file
    extra_var_arg="-e@${sandbox_internal_vars_file} -e@${sandbox_secure_vars_file} -e@${extra_vars_file}"
fi

if [[ -z $region ]]; then
  region="us-east-1"
fi

# edX has reservations for sandboxes in this zone, don't change without updating reservations.
if [[ -z $zone ]]; then
  zone="us-east-1c"
fi

if [[ -z $vpc_subnet_id ]]; then
  vpc_subnet_id="subnet-cd867aba"
fi

if [[ -z $elb ]]; then
  elb="false"
fi

if [[ -z $dns_name ]]; then
  dns_name=${github_username}
fi

if [[ -z $name_tag ]]; then
  name_tag=${github_username}-${environment}
fi

if [[ -z $sandbox_platform_name ]]; then
    sandbox_platform_name=$dns_name
fi

if [[ -z $ami ]]; then
  if [[ $server_type == "full_edx_installation" ]]; then
    ami="ami-0010ced4e81b9297f"
  elif [[ $server_type == "ubuntu_16.04" || $server_type == "full_edx_installation_from_scratch" ]]; then
    ami="ami-04169656fea786776"
  fi
fi

if [[ -z $instance_type ]]; then
  instance_type="r5.large"
fi

if [[ -z $instance_initiated_shutdown_behavior ]]; then
  instance_initiated_shutdown_behavior="terminate"
fi

if [[ -z $enable_newrelic ]]; then
  enable_newrelic="false"
fi

if [[ -z $enable_datadog ]]; then
  enable_datadog="false"
fi

if [[ -z $performance_course ]]; then
  performance_course="false"
fi

if [[ -z $demo_test_course ]]; then
  demo_test_course="false"
fi

if [[ -z $edx_demo_course ]]; then
  edx_demo_course="false"
fi

if [[ -z $enable_automatic_auth_for_testing ]]; then
  enable_automatic_auth_for_testing="false"
fi

if [[ -z $enable_client_profiling ]]; then
  enable_client_profiling="false"
fi

if [[ -z $set_whitelabel ]]; then
  set_whitelabel="true"
fi

if [[ -z $registrar ]]; then
  registrar="false"
fi

if [[ -z $registrar_version ]]; then
  registrar_version="master"
fi

if [[ -z $learner_portal ]]; then
  learner_portal="false"
fi

if [[ -z $learner_portal_version ]]; then
  learner_portal_version="master"
fi

if [[ $registrar == 'true' ]]; then
  program_manager="true"
fi

# Lowercase the dns name to deal with an ansible bug
dns_name="${dns_name,,}"

deploy_host="${dns_name}.${dns_zone}"
ssh-keygen -f "/var/lib/jenkins/.ssh/known_hosts" -R "$deploy_host"

cd playbooks

cat << EOF > $extra_vars_file
edx_platform_version: $edxapp_version
forum_version: $forum_version
notifier_version: $notifier_version
XQUEUE_VERSION: $xqueue_version
certs_version: $certs_version
configuration_version: $configuration_version
demo_version: $demo_version
THEMES_VERSION: $themes_version
registrar_version: $registrar_version
learner_portal_version: $learner_portal_version
program_manager_version: $program_manager_version

edx_ansible_source_repo: ${configuration_source_repo}
edx_platform_repo: ${edx_platform_repo}

EDXAPP_PLATFORM_NAME: $sandbox_platform_name
COMMON_SANDBOX_BUILD: True

EDXAPP_STATIC_URL_BASE: $static_url_base
EDXAPP_LMS_NGINX_PORT: 80
EDXAPP_CMS_NGINX_PORT: 80

ECOMMERCE_NGINX_PORT: 80
ECOMMERCE_SSL_NGINX_PORT: 443
ECOMMERCE_VERSION: $ecommerce_version

CREDENTIALS_NGINX_PORT: 80
CREDENTIALS_SSL_NGINX_PORT: 443
CREDENTIALS_VERSION: $credentials_version

ANALYTICS_API_NGINX_PORT: 80
ANALYTICS_API_SSL_NGINX_PORT: 443
ANALYTICS_API_VERSION: $analytics_api_version

REGISTRAR_NGINX_PORT: 80
REGISTRAR_SSL_NGINX_PORT: 443
REGISTRAR_VERSION: $registrar_version
REGISTRAR_ENABLED: $registrar
REGISTRAR_SANDBOX_BUILD: True

LEARNER_PORTAL_NGINX_PORT: 80
LEARNER_PORTAL_SSL_NGINX_PORT: 443
LEARNER_PORTAL_VERSION: $learner_portal_version
LEARNER_PORTAL_ENABLED: $learner_portal
LEARNER_PORTAL_SANDBOX_BUILD: True

PROGRAM_MANAGER_NGINX_PORT: 80
PROGRAM_MANAGER_SSL_NGINX_PORT: 443
PROGRAM_MANAGER_VERSION: $program_manager_version
PROGRAM_MANAGER_ENABLED: $program_manager
PROGRAM_MANAGER_SANDBOX_BUILD: True

VIDEO_PIPELINE_BASE_NGINX_PORT: 80
VIDEO_PIPELINE_BASE_SSL_NGINX_PORT: 443

DISCOVERY_NGINX_PORT: 80
DISCOVERY_SSL_NGINX_PORT: 443
DISCOVERY_VERSION: $discovery_version
NGINX_SET_X_FORWARDED_HEADERS: True
NGINX_REDIRECT_TO_HTTPS: True
EDX_ANSIBLE_DUMP_VARS: true
migrate_db: "yes"
dns_name: $dns_name
COMMON_HOSTNAME: $dns_name
COMMON_DEPLOYMENT: edx
COMMON_ENVIRONMENT: sandbox
COMMON_LMS_BASE_URL: https://${deploy_host}

nginx_default_sites:
  - lms

# User provided extra vars
$extra_vars
EOF

if [[ $basic_auth == "true" ]]; then
    # vars specific to provisioning added to $extra-vars
    cat << EOF_AUTH >> $extra_vars_file
COMMON_ENABLE_BASIC_AUTH: True
COMMON_HTPASSWD_USER: $auth_user
COMMON_HTPASSWD_PASS: $auth_pass
XQUEUE_BASIC_AUTH_USER: $auth_user
XQUEUE_BASIC_AUTH_PASSWORD: $auth_pass
EOF_AUTH

else
    cat << EOF_AUTH >> $extra_vars_file
COMMON_ENABLE_BASIC_AUTH: False
EOF_AUTH

fi

if [[ -n $nginx_users ]]; then
   cat << EOF_AUTH >> $extra_vars_file
NGINX_USERS: $nginx_users
EOF_AUTH
fi

if [[ $enable_client_profiling == "true" ]]; then
    cat << EOF_PROFILING >> $extra_vars_file
EDXAPP_SESSION_SAVE_EVERY_REQUEST: True
EOF_PROFILING
fi

if [[ $edx_internal == "true" ]]; then
    cat << EOF >> $extra_vars_file
EDXAPP_PREVIEW_LMS_BASE: preview-${deploy_host}
EDXAPP_LMS_BASE: ${deploy_host}
EDXAPP_CMS_BASE: studio-${deploy_host}
EDXAPP_SITE_NAME: ${deploy_host}
CERTS_DOWNLOAD_URL: "http://${deploy_host}:18090"
CERTS_VERIFY_URL: "http://${deploy_host}:18090"
edx_internal: True
COMMON_USER_INFO:
  - name: ${github_username}
    github: true
    type: admin
USER_CMD_PROMPT: '[$name_tag] '
COMMON_ENABLE_NEWRELIC_APP: $enable_newrelic
COMMON_ENABLE_DATADOG: $enable_datadog
COMMON_OAUTH_BASE_URL: "https://${deploy_host}"
FORUM_NEW_RELIC_ENABLE: $enable_newrelic
ENABLE_PERFORMANCE_COURSE: $performance_course
ENABLE_DEMO_TEST_COURSE: $demo_test_course
ENABLE_EDX_DEMO_COURSE: $edx_demo_course
EDXAPP_ENABLE_AUTO_AUTH: $enable_automatic_auth_for_testing
EDXAPP_NEWRELIC_LMS_APPNAME: sandbox-${dns_name}-edxapp-lms
EDXAPP_NEWRELIC_CMS_APPNAME: sandbox-${dns_name}-edxapp-cms
EDXAPP_NEWRELIC_WORKERS_APPNAME: sandbox-${dns_name}-edxapp-workers
XQUEUE_NEWRELIC_APPNAME: sandbox-${dns_name}-xqueue
XQUEUE_CONSUMER_NEWRELIC_APPNAME: sandbox-${dns_name}-xqueue_consumer
FORUM_NEW_RELIC_APP_NAME: sandbox-${dns_name}-forums
SANDBOX_USERNAME: $github_username
EDXAPP_ECOMMERCE_PUBLIC_URL_ROOT: "https://ecommerce-${deploy_host}"
EDXAPP_ECOMMERCE_API_URL: "https://ecommerce-${deploy_host}/api/v2"
EDXAPP_DISCOVERY_API_URL: "https://discovery-${deploy_host}/api/v1"
EDXAPP_COURSE_CATALOG_API_URL: "{{ EDXAPP_DISCOVERY_API_URL }}"

ANALYTICS_API_LMS_BASE_URL: "https://{{ EDXAPP_LMS_BASE }}/"

# NOTE: This is the same as DISCOVERY_URL_ROOT below
ECOMMERCE_DISCOVERY_SERVICE_URL: "https://discovery-${deploy_host}"
ECOMMERCE_ECOMMERCE_URL_ROOT: "https://ecommerce-${deploy_host}"
ECOMMERCE_LMS_URL_ROOT: "https://${deploy_host}"
ECOMMERCE_SOCIAL_AUTH_REDIRECT_IS_HTTPS: true
ecommerce_create_demo_data: true

DISCOVERY_URL_ROOT: "https://discovery-${deploy_host}"
DISCOVERY_SOCIAL_AUTH_REDIRECT_IS_HTTPS: true

REGISTRAR_URL_ROOT: "https://registrar-${deploy_host}"
REGISTRAR_API_ROOT: "https://registrar-${deploy_host}/api"
REGISTRAR_DISCOVERY_BASE_URL: "https://discovery-${deploy_host}"
REGISTRAR_LMS_BASE_URL: "https://${deploy_host}"
REGISTRAR_SOCIAL_AUTH_REDIRECT_IS_HTTPS: true

LEARNER_PORTAL_URL_ROOT: "https://learner-portal-${deploy_host}"
LEARNER_PORTAL_DISCOVERY_BASE_URL: "https://discovery-${deploy_host}"
LEARNER_PORTAL_LMS_BASE_URL: "https://${deploy_host}"

PROGRAM_MANAGER_URL_ROOT: "https://program-manager-${deploy_host}"
PROGRAM_MANAGER_DISCOVERY_BASE_URL: "https://discovery-${deploy_host}"
PROGRAM_MANAGER_LMS_BASE_URL: "https://${deploy_host}"
PROGRAM_MANAGER_REGISTRAR_API_BASE_URL: "https://registrar-${deploy_host}/api"

credentials_create_demo_data: true
CREDENTIALS_LMS_URL_ROOT: "https://${deploy_host}"
CREDENTIALS_DOMAIN: "credentials-${deploy_host}"
CREDENTIALS_URL_ROOT: "https://{{ CREDENTIALS_DOMAIN }}"
CREDENTIALS_SOCIAL_AUTH_REDIRECT_IS_HTTPS: true
CREDENTIALS_DISCOVERY_API_URL: "{{ DISCOVERY_URL_ROOT }}/api/v1/"

VIDEO_PIPELINE_DOMAIN: "veda-${deploy_host}"
VIDEO_PIPELINE_BASE_URL_ROOT: "https://{{ VIDEO_PIPELINE_DOMAIN }}"
VIDEO_PIPELINE_BASE_LMS_BASE_URL: "https://{{ EDXAPP_LMS_BASE }}"

VEDA_WEB_FRONTEND_VERSION: ${video_pipeline_version:-master}
VEDA_PIPELINE_WORKER_VERSION: ${video_pipeline_version:-master}
VEDA_ENCODE_WORKER_VERSION: ${video_encode_worker_version:-master}

EOF
fi

if [[ $recreate == "true" ]]; then
    # vars specific to provisioning added to $extra-vars
    cat << EOF >> $extra_vars_file
dns_name: $dns_name
keypair: $keypair
instance_type: $instance_type
security_group: $security_group
ami: $ami
region: $region
zone: $zone
instance_initiated_shutdown_behavior: $instance_initiated_shutdown_behavior
instance_tags:
    environment: $environment
    github_username: $github_username
    Name: $name_tag
    source: jenkins
    owner: $BUILD_USER
    instance_termination_time: $TERMINATION_DATE_TIME
    datadog: monitored
root_ebs_size: $root_ebs_size
name_tag: $name_tag
dns_zone: $dns_zone
elb: $elb
EOF

    if [[ $server_type == "full_edx_installation" ]]; then
        extra_var_arg+=' -e instance_userdata="" -e launch_wait_time=0 -e elb_pre_post=false'
    fi
    # run the tasks to launch an ec2 instance from AMI
    cat $extra_vars_file
    run_ansible edx_provision.yml -i inventory.ini $extra_var_arg --user ubuntu

    if [[ $server_type == "full_edx_installation" ]]; then
        # additional tasks that need to be run if the
        # entire edx stack is brought up from an AMI
        run_ansible redis.yml -i "${deploy_host}," $extra_var_arg --user ubuntu
        run_ansible restart_supervisor.yml -i "${deploy_host}," $extra_var_arg --user ubuntu
    fi
fi

veda_web_frontend=${video_pipeline:-false}
veda_pipeline_worker=${video_pipeline:-false}
veda_encode_worker=${video_encode_worker:-false}
video_pipeline_integration=${video_pipeline:-false}

# ansible overrides for master's integration environment setup
if [[ $registrar == "true" ]]; then
    cat << EOF >> $extra_vars_file
COMMON_ENABLE_SPLUNKFORWARDER: true,
EDXAPP_ENABLE_ENROLLMENT_RESET: true,
EOF
fi

declare -A deploy
plays="edxapp forum ecommerce credentials discovery analyticsapi veda_web_frontend veda_pipeline_worker veda_encode_worker video_pipeline_integration notifier xqueue certs demo testcourses registrar program_manager learner_portal"

for play in $plays; do
    deploy[$play]=${!play}
done

# If reconfigure was selected or if starting from an ubuntu 16.04 AMI
# run non-deploy tasks for all plays
if [[ $reconfigure == "true" || $server_type == "full_edx_installation_from_scratch" ]]; then
    cat $extra_vars_file
    run_ansible edx_continuous_integration.yml -i "${deploy_host}," $extra_var_arg --user ubuntu
fi

if [[ $reconfigure != "true" && $server_type == "full_edx_installation" ]]; then
    # Run deploy tasks for the plays selected
    for i in $plays; do
        if [[ ${deploy[$i]} == "true" ]]; then
            cat $extra_vars_file
            run_ansible ${i}.yml -i "${deploy_host}," $extra_var_arg --user ubuntu
            if [[ ${i} == "edxapp" ]]; then
                run_ansible worker.yml -i "${deploy_host}," $extra_var_arg --user ubuntu
            fi
        fi
    done
fi

# deploy the edx_ansible play
run_ansible edx_ansible.yml -i "${deploy_host}," $extra_var_arg --user ubuntu
cat $sandbox_secure_vars_file $sandbox_internal_vars_file $extra_vars_file | grep -v -E "_version|migrate_db" > ${extra_vars_file}_clean
ansible -c ssh -i "${deploy_host}," $deploy_host -m copy -a "src=${extra_vars_file}_clean dest=/edx/app/edx_ansible/server-vars.yml" -u ubuntu -b
ret=$?
if [[ $ret -ne 0 ]]; then
  exit $ret
fi

if [[ $run_oauth == "true" ]]; then
    # Setup the OAuth2 clients
    run_ansible oauth_client_setup.yml -i "${deploy_host}," $extra_var_arg --user ubuntu
fi

# set the hostname
run_ansible set_hostname.yml -i "${deploy_host}," -e hostname_fqdn=${deploy_host} --user ubuntu

# master's integration environment setup
if [[ $registrar == "true" ]]; then
  # vars specific to master's integration environment
  cat << EOF >> $extra_vars_file
username: $registrar_user_email
email: $registrar_user_email
organization_key: $registrar_org_key
registrar_role: "organization_read_write_enrollments"
EOF
  run_ansible masters_sandbox.yml -i "${deploy_host}," $extra_var_arg --user ubuntu
fi

if [[ $set_whitelabel == "true" ]]; then
    # Setup Whitelabel themes
    run_ansible whitelabel.yml -i "${deploy_host}," $extra_var_arg --user ubuntu
fi

if [[ $enable_newrelic == "true" ]]; then
    run_ansible ../run_role.yml -i "${deploy_host}," -e role=newrelic_infrastructure $extra_var_arg  --user ubuntu
fi

rm -f "$extra_vars_file"
rm -f ${extra_vars_file}_clean

#!/usr/bin/env bash

# Ansible provisioning wrapper script that
# assumes the following parameters set
# as environment variables
#
# - github_username
# - server_type
# - instance_type
# - region
# - aws_account
# - keypair
# - ami
# - root_ebs_size
# - security_group
# - dns_zone
# - dns_name
# - environment
# - name_tag
set -x

# Seeing the environment is fine, spewing secrets to the log isn't ok
env | grep -v AWS | grep -v ARN

export PYTHONUNBUFFERED=1
export BOTO_CONFIG=/var/lib/jenkins/${aws_account}.boto

# docker on OS-X includes your Mac's home directory in the socket path
# that SSH/Ansible uses for the control socket, pushing you over
# the 108 character limit.
if [ -f /.dockerenv ]; then
    export ANSIBLE_SSH_CONTROL_PATH=/tmp/%%C
fi

run_ansible() {
  if [[ "$VERBOSE" == "true" ]]; then
    verbose_arg='-vvv'
  else
    verbose_arg=''
  fi

  ansible-playbook $verbose_arg $@
  ret=$?
  if [[ $ret -ne 0 ]]; then
    exit $ret
  fi
}

# This DATE_TIME will be used as instance launch time tag
if [[ ! -n ${sandbox_life//[0-9]/} ]]  && [[ ${sandbox_life} -le 30 ]]; then
    TERMINATION_DATE_TIME=`date +"%m-%d-%Y %T" --date "${sandbox_life=7} days"`
else
   echo "Please enter the valid value for the sandbox_life(between 1 to 30)"
   exit 1
fi

if [[ -z $BUILD_USER ]]; then
    BUILD_USER=jenkins
fi

if [[ -z $BUILD_USER_ID ]]; then
    BUILD_USER_ID=edx-sandbox
fi

if [[ -z $WORKSPACE ]]; then
    dir=$(dirname $0)
    source "$dir/ascii-convert.sh"
else
    source "$WORKSPACE/configuration/util/jenkins/ascii-convert.sh"
fi

if [[ -z $static_url_base ]]; then
  static_url_base="/static"
fi

if [[ -z $github_username  ]]; then
  github_username=$BUILD_USER_ID
fi

# Having access keys OR a boto config allows sandboxes to be built.
if [[ ( -z $AWS_ACCESS_KEY_ID || -z $AWS_SECRET_ACCESS_KEY ) && (! -f $BOTO_CONFIG) ]]; then
  echo "AWS credentials not found for $aws_account"
  exit 1
fi

extra_vars_file="/var/tmp/extra-vars-$$.yml"
sandbox_secure_vars_file="${WORKSPACE}/configuration-secure/ansible/vars/developer-sandbox.yml"
sandbox_internal_vars_file="${WORKSPACE}/configuration-internal/ansible/vars/developer-sandbox.yml"
extra_var_arg="-e@${extra_vars_file}"
program_manager="false"

if [[ $edx_internal == "true" ]]; then
    # if this is a an edx server include
    # the secret var file
    extra_var_arg="-e@${sandbox_internal_vars_file} -e@${sandbox_secure_vars_file} -e@${extra_vars_file}"
fi

if [[ -z $region ]]; then
  region="us-east-1"
fi

# edX has reservations for sandboxes in this zone, don't change without updating reservations.
if [[ -z $zone ]]; then
  zone="us-east-1c"
fi

if [[ -z $vpc_subnet_id ]]; then
  vpc_subnet_id="subnet-cd867aba"
fi

if [[ -z $elb ]]; then
  elb="false"
fi

if [[ -z $dns_name ]]; then
  dns_name=${github_username}
fi

if [[ -z $name_tag ]]; then
  name_tag=${github_username}-${environment}
fi

if [[ -z $sandbox_platform_name ]]; then
    sandbox_platform_name=$dns_name
fi

if [[ -z $ami ]]; then
  if [[ $server_type == "full_edx_installation" ]]; then
    ami="ami-0010ced4e81b9297f"
  elif [[ $server_type == "ubuntu_16.04" || $server_type == "full_edx_installation_from_scratch" ]]; then
    ami="ami-04169656fea786776"
  fi
fi

if [[ -z $instance_type ]]; then
  instance_type="r5.large"
fi

if [[ -z $instance_initiated_shutdown_behavior ]]; then
  instance_initiated_shutdown_behavior="terminate"
fi

if [[ -z $enable_newrelic ]]; then
  enable_newrelic="false"
fi

if [[ -z $enable_datadog ]]; then
  enable_datadog="false"
fi

if [[ -z $performance_course ]]; then
  performance_course="false"
fi

if [[ -z $demo_test_course ]]; then
  demo_test_course="false"
fi

if [[ -z $edx_demo_course ]]; then
  edx_demo_course="false"
fi

if [[ -z $enable_automatic_auth_for_testing ]]; then
  enable_automatic_auth_for_testing="false"
fi

if [[ -z $enable_client_profiling ]]; then
  enable_client_profiling="false"
fi

if [[ -z $set_whitelabel ]]; then
  set_whitelabel="true"
fi

if [[ -z $journals ]]; then
  journals="false"
fi

if [[ -z $journals_version ]]; then
  journals_version="master"
fi

if [[ -z $registrar ]]; then
  registrar="false"
fi

if [[ -z $registrar_version ]]; then
  registrar_version="master"
fi

if [[ -z $learner_portal ]]; then
  learner_portal="false"
fi

if [[ -z $learner_portal_version ]]; then
  learner_portal_version="master"
fi

if [[ $registrar == 'true' ]]; then
  program_manager="true"
fi

# Lowercase the dns name to deal with an ansible bug
dns_name="${dns_name,,}"

deploy_host="${dns_name}.${dns_zone}"
ssh-keygen -f "/var/lib/jenkins/.ssh/known_hosts" -R "$deploy_host"

cd playbooks

cat << EOF > $extra_vars_file
edx_platform_version: $edxapp_version
forum_version: $forum_version
notifier_version: $notifier_version
XQUEUE_VERSION: $xqueue_version
certs_version: $certs_version
configuration_version: $configuration_version
demo_version: $demo_version
THEMES_VERSION: $themes_version
journals_version: $journals_version
registrar_version: $registrar_version
learner_portal_version: $learner_portal_version
program_manager_version: $program_manager_version

edx_ansible_source_repo: ${configuration_source_repo}
edx_platform_repo: ${edx_platform_repo}

EDXAPP_PLATFORM_NAME: $sandbox_platform_name
COMMON_SANDBOX_BUILD: True

EDXAPP_STATIC_URL_BASE: $static_url_base
EDXAPP_LMS_NGINX_PORT: 80
EDXAPP_CMS_NGINX_PORT: 80

ECOMMERCE_NGINX_PORT: 80
ECOMMERCE_SSL_NGINX_PORT: 443
ECOMMERCE_VERSION: $ecommerce_version

CREDENTIALS_NGINX_PORT: 80
CREDENTIALS_SSL_NGINX_PORT: 443
CREDENTIALS_VERSION: $credentials_version

ANALYTICS_API_NGINX_PORT: 80
ANALYTICS_API_SSL_NGINX_PORT: 443
ANALYTICS_API_VERSION: $analytics_api_version

JOURNALS_NGINX_PORT: 80
JOURNALS_SSL_NGINX_PORT: 443
JOURNALS_VERSION: $journals_version
JOURNALS_ENABLED: $journals
JOURNALS_SANDBOX_BUILD: True

REGISTRAR_NGINX_PORT: 80
REGISTRAR_SSL_NGINX_PORT: 443
REGISTRAR_VERSION: $registrar_version
REGISTRAR_ENABLED: $registrar
REGISTRAR_SANDBOX_BUILD: True

LEARNER_PORTAL_NGINX_PORT: 80
LEARNER_PORTAL_SSL_NGINX_PORT: 443
LEARNER_PORTAL_VERSION: $learner_portal_version
LEARNER_PORTAL_ENABLED: $learner_portal
LEARNER_PORTAL_SANDBOX_BUILD: True

PROGRAM_MANAGER_NGINX_PORT: 80
PROGRAM_MANAGER_SSL_NGINX_PORT: 443
PROGRAM_MANAGER_VERSION: $program_manager_version
PROGRAM_MANAGER_ENABLED: $program_manager
PROGRAM_MANAGER_SANDBOX_BUILD: True

VIDEO_PIPELINE_BASE_NGINX_PORT: 80
VIDEO_PIPELINE_BASE_SSL_NGINX_PORT: 443

DISCOVERY_NGINX_PORT: 80
DISCOVERY_SSL_NGINX_PORT: 443
DISCOVERY_VERSION: $discovery_version
NGINX_SET_X_FORWARDED_HEADERS: True
NGINX_REDIRECT_TO_HTTPS: True
EDX_ANSIBLE_DUMP_VARS: true
migrate_db: "yes"
dns_name: $dns_name
COMMON_HOSTNAME: $dns_name
COMMON_DEPLOYMENT: edx
COMMON_ENVIRONMENT: sandbox
COMMON_LMS_BASE_URL: https://${deploy_host}

nginx_default_sites:
  - lms

# User provided extra vars
$extra_vars
EOF

if [[ $basic_auth == "true" ]]; then
    # vars specific to provisioning added to $extra-vars
    cat << EOF_AUTH >> $extra_vars_file
COMMON_ENABLE_BASIC_AUTH: True
COMMON_HTPASSWD_USER: $auth_user
COMMON_HTPASSWD_PASS: $auth_pass
XQUEUE_BASIC_AUTH_USER: $auth_user
XQUEUE_BASIC_AUTH_PASSWORD: $auth_pass
EOF_AUTH

else
    cat << EOF_AUTH >> $extra_vars_file
COMMON_ENABLE_BASIC_AUTH: False
EOF_AUTH

fi

if [[ -n $nginx_users ]]; then
   cat << EOF_AUTH >> $extra_vars_file
NGINX_USERS: $nginx_users
EOF_AUTH
fi

if [[ $enable_client_profiling == "true" ]]; then
    cat << EOF_PROFILING >> $extra_vars_file
EDXAPP_SESSION_SAVE_EVERY_REQUEST: True
EOF_PROFILING
fi

if [[ $edx_internal == "true" ]]; then
    cat << EOF >> $extra_vars_file
EDXAPP_PREVIEW_LMS_BASE: preview-${deploy_host}
EDXAPP_LMS_BASE: ${deploy_host}
EDXAPP_CMS_BASE: studio-${deploy_host}
EDXAPP_SITE_NAME: ${deploy_host}
CERTS_DOWNLOAD_URL: "http://${deploy_host}:18090"
CERTS_VERIFY_URL: "http://${deploy_host}:18090"
edx_internal: True
COMMON_USER_INFO:
  - name: ${github_username}
    github: true
    type: admin
USER_CMD_PROMPT: '[$name_tag] '
COMMON_ENABLE_NEWRELIC_APP: $enable_newrelic
COMMON_ENABLE_DATADOG: $enable_datadog
COMMON_OAUTH_BASE_URL: "https://${deploy_host}"
FORUM_NEW_RELIC_ENABLE: $enable_newrelic
ENABLE_PERFORMANCE_COURSE: $performance_course
ENABLE_DEMO_TEST_COURSE: $demo_test_course
ENABLE_EDX_DEMO_COURSE: $edx_demo_course
EDXAPP_ENABLE_AUTO_AUTH: $enable_automatic_auth_for_testing
EDXAPP_NEWRELIC_LMS_APPNAME: sandbox-${dns_name}-edxapp-lms
EDXAPP_NEWRELIC_CMS_APPNAME: sandbox-${dns_name}-edxapp-cms
EDXAPP_NEWRELIC_WORKERS_APPNAME: sandbox-${dns_name}-edxapp-workers
XQUEUE_NEWRELIC_APPNAME: sandbox-${dns_name}-xqueue
XQUEUE_CONSUMER_NEWRELIC_APPNAME: sandbox-${dns_name}-xqueue_consumer
FORUM_NEW_RELIC_APP_NAME: sandbox-${dns_name}-forums
SANDBOX_USERNAME: $github_username
EDXAPP_ECOMMERCE_PUBLIC_URL_ROOT: "https://ecommerce-${deploy_host}"
EDXAPP_ECOMMERCE_API_URL: "https://ecommerce-${deploy_host}/api/v2"
EDXAPP_DISCOVERY_API_URL: "https://discovery-${deploy_host}/api/v1"
EDXAPP_COURSE_CATALOG_API_URL: "{{ EDXAPP_DISCOVERY_API_URL }}"

ANALYTICS_API_LMS_BASE_URL: "https://{{ EDXAPP_LMS_BASE }}/"

# NOTE: This is the same as DISCOVERY_URL_ROOT below
ECOMMERCE_DISCOVERY_SERVICE_URL: "https://discovery-${deploy_host}"
ECOMMERCE_ECOMMERCE_URL_ROOT: "https://ecommerce-${deploy_host}"
ECOMMERCE_LMS_URL_ROOT: "https://${deploy_host}"
ECOMMERCE_SOCIAL_AUTH_REDIRECT_IS_HTTPS: true
ecommerce_create_demo_data: true

JOURNALS_URL_ROOT: "https://journals-{{ EDXAPP_LMS_BASE }}"
JOURNALS_FRONTEND_URL: "https://journalsapp-{{ EDXAPP_LMS_BASE }}"
JOURNALS_API_URL: "https://journals-{{ EDXAPP_LMS_BASE }}/api/v1/"
JOURNALS_DISCOVERY_SERVICE_URL: "https://discovery-{{ EDXAPP_LMS_BASE }}"
JOURNALS_LMS_URL_ROOT: "https://{{ EDXAPP_LMS_BASE }}"
JOURNALS_SOCIAL_AUTH_REDIRECT_IS_HTTPS: true
JOURNALS_DISCOVERY_API_URL: "{{ JOURNALS_DISCOVERY_SERVICE_URL }}/api/v1/"
JOURNALS_DISCOVERY_JOURNALS_API_URL: "{{ JOURNALS_DISCOVERY_SERVICE_URL }}/journal/api/v1/"
JOURNALS_ECOMMERCE_BASE_URL: "{{ ECOMMERCE_ECOMMERCE_URL_ROOT }}"
JOURNALS_ECOMMERCE_API_URL: "{{ JOURNALS_ECOMMERCE_BASE_URL }}/api/v2/"
JOURNALS_ECOMMERCE_JOURNALS_API_URL: "{{ JOURNALS_ECOMMERCE_BASE_URL }}/journal/api/v1"
journals_create_demo_data: true

DISCOVERY_URL_ROOT: "https://discovery-${deploy_host}"
DISCOVERY_SOCIAL_AUTH_REDIRECT_IS_HTTPS: true

REGISTRAR_URL_ROOT: "https://registrar-${deploy_host}"
REGISTRAR_API_ROOT: "https://registrar-${deploy_host}/api"
REGISTRAR_DISCOVERY_BASE_URL: "https://discovery-${deploy_host}"
REGISTRAR_LMS_BASE_URL: "https://${deploy_host}"
REGISTRAR_SOCIAL_AUTH_REDIRECT_IS_HTTPS: true

LEARNER_PORTAL_URL_ROOT: "https://learner-portal-${deploy_host}"
LEARNER_PORTAL_DISCOVERY_BASE_URL: "https://discovery-${deploy_host}"
LEARNER_PORTAL_LMS_BASE_URL: "https://${deploy_host}"

PROGRAM_MANAGER_URL_ROOT: "https://program-manager-${deploy_host}"
PROGRAM_MANAGER_DISCOVERY_BASE_URL: "https://discovery-${deploy_host}"
PROGRAM_MANAGER_LMS_BASE_URL: "https://${deploy_host}"
PROGRAM_MANAGER_REGISTRAR_API_BASE_URL: "https://registrar-${deploy_host}/api"

credentials_create_demo_data: true
CREDENTIALS_LMS_URL_ROOT: "https://${deploy_host}"
CREDENTIALS_DOMAIN: "credentials-${deploy_host}"
CREDENTIALS_URL_ROOT: "https://{{ CREDENTIALS_DOMAIN }}"
CREDENTIALS_SOCIAL_AUTH_REDIRECT_IS_HTTPS: true
CREDENTIALS_DISCOVERY_API_URL: "{{ DISCOVERY_URL_ROOT }}/api/v1/"

VIDEO_PIPELINE_DOMAIN: "veda-${deploy_host}"
VIDEO_PIPELINE_BASE_URL_ROOT: "https://{{ VIDEO_PIPELINE_DOMAIN }}"
VIDEO_PIPELINE_BASE_LMS_BASE_URL: "https://{{ EDXAPP_LMS_BASE }}"

VEDA_WEB_FRONTEND_VERSION: ${video_pipeline_version:-master}
VEDA_PIPELINE_WORKER_VERSION: ${video_pipeline_version:-master}
VEDA_ENCODE_WORKER_VERSION: ${video_encode_worker_version:-master}

EOF
fi

if [[ $recreate == "true" ]]; then
    # vars specific to provisioning added to $extra-vars
    cat << EOF >> $extra_vars_file
dns_name: $dns_name
keypair: $keypair
instance_type: $instance_type
security_group: $security_group
ami: $ami
region: $region
zone: $zone
instance_initiated_shutdown_behavior: $instance_initiated_shutdown_behavior
instance_tags:
    environment: $environment
    github_username: $github_username
    Name: $name_tag
    source: jenkins
    owner: $BUILD_USER
    instance_termination_time: $TERMINATION_DATE_TIME
    datadog: monitored
root_ebs_size: $root_ebs_size
name_tag: $name_tag
dns_zone: $dns_zone
elb: $elb
EOF

    if [[ $server_type == "full_edx_installation" ]]; then
        extra_var_arg+=' -e instance_userdata="" -e launch_wait_time=0 -e elb_pre_post=false'
    fi
    # run the tasks to launch an ec2 instance from AMI
    cat $extra_vars_file
    run_ansible edx_provision.yml -i inventory.ini $extra_var_arg --user ubuntu

    if [[ $server_type == "full_edx_installation" ]]; then
        # additional tasks that need to be run if the
        # entire edx stack is brought up from an AMI
        run_ansible redis.yml -i "${deploy_host}," $extra_var_arg --user ubuntu
        run_ansible restart_supervisor.yml -i "${deploy_host}," $extra_var_arg --user ubuntu
    fi
fi

veda_web_frontend=${video_pipeline:-false}
veda_pipeline_worker=${video_pipeline:-false}
veda_encode_worker=${video_encode_worker:-false}
video_pipeline_integration=${video_pipeline:-false}

# ansible overrides for master's integration environment setup
if [[ $registrar == "true" ]]; then
    cat << EOF >> $extra_vars_file
COMMON_ENABLE_SPLUNKFORWARDER: true,
EDXAPP_ENABLE_ENROLLMENT_RESET: true,
EOF
fi

declare -A deploy
plays="edxapp forum ecommerce credentials discovery journals analyticsapi veda_web_frontend veda_pipeline_worker veda_encode_worker video_pipeline_integration notifier xqueue certs demo testcourses registrar program_manager learner_portal"

for play in $plays; do
    deploy[$play]=${!play}
done

# If reconfigure was selected or if starting from an ubuntu 16.04 AMI
# run non-deploy tasks for all plays
if [[ $reconfigure == "true" || $server_type == "full_edx_installation_from_scratch" ]]; then
    cat $extra_vars_file
    run_ansible edx_continuous_integration.yml -i "${deploy_host}," $extra_var_arg --user ubuntu
fi

if [[ $reconfigure != "true" && $server_type == "full_edx_installation" ]]; then
    # Run deploy tasks for the plays selected
    for i in $plays; do
        if [[ ${deploy[$i]} == "true" ]]; then
            cat $extra_vars_file
            run_ansible ${i}.yml -i "${deploy_host}," $extra_var_arg --user ubuntu
            if [[ ${i} == "edxapp" ]]; then
                run_ansible worker.yml -i "${deploy_host}," $extra_var_arg --user ubuntu
            fi
        fi
    done
fi

# deploy the edx_ansible play
run_ansible edx_ansible.yml -i "${deploy_host}," $extra_var_arg --user ubuntu
cat $sandbox_secure_vars_file $sandbox_internal_vars_file $extra_vars_file | grep -v -E "_version|migrate_db" > ${extra_vars_file}_clean
ansible -c ssh -i "${deploy_host}," $deploy_host -m copy -a "src=${extra_vars_file}_clean dest=/edx/app/edx_ansible/server-vars.yml" -u ubuntu -b
ret=$?
if [[ $ret -ne 0 ]]; then
  exit $ret
fi

if [[ $run_oauth == "true" ]]; then
    # Setup the OAuth2 clients
    run_ansible oauth_client_setup.yml -i "${deploy_host}," $extra_var_arg --user ubuntu
fi

# set the hostname
run_ansible set_hostname.yml -i "${deploy_host}," -e hostname_fqdn=${deploy_host} --user ubuntu

# master's integration environment setup
if [[ $registrar == "true" ]]; then
  # vars specific to master's integration environment
  cat << EOF >> $extra_vars_file
username: $registrar_user_email
email: $registrar_user_email
organization_key: $registrar_org_key
registrar_role: "organization_read_write_enrollments"
EOF
  run_ansible masters_sandbox.yml -i "${deploy_host}," $extra_var_arg --user ubuntu
fi

if [[ $set_whitelabel == "true" ]]; then
    # Setup Whitelabel themes
    run_ansible whitelabel.yml -i "${deploy_host}," $extra_var_arg --user ubuntu
fi

if [[ $enable_newrelic == "true" ]]; then
    run_ansible ../run_role.yml -i "${deploy_host}," -e role=newrelic_infrastructure $extra_var_arg  --user ubuntu
fi

rm -f "$extra_vars_file"
rm -f ${extra_vars_file}_clean


OPS-727 Sandbox platform name and logo

OPS-727 Sandbox platform name and logo

#!/usr/bin/env bash

# Ansible provisioning wrapper script that
# assumes the following parameters set
# as environment variables
#
# - github_username
# - server_type
# - instance_type
# - region
# - aws_account
# - keypair
# - ami
# - root_ebs_size
# - security_group
# - dns_zone
# - dns_name
# - environment
# - name_tag
set -x
env
export PYTHONUNBUFFERED=1
export BOTO_CONFIG=/var/lib/jenkins/${aws_account}.boto

run_ansible() {
  if [[ "$VERBOSE" == "true" ]]; then
    verbose_arg='-vvv'
  else
    verbose_arg=''
  fi

  ansible-playbook $verbose_arg $@
  ret=$?
  if [[ $ret -ne 0 ]]; then
    exit $ret
  fi
}

# This DATE_TIME will be used as instance launch time tag
if [[ ! -n ${sandbox_life//[0-9]/} ]]  && [[ ${sandbox_life} -le 30 ]]; then
    TERMINATION_DATE_TIME=`date +"%m-%d-%Y %T" --date "${sandbox_life=7} days"`
else
   echo "Please enter the valid value for the sandbox_life(between 1 to 30)"
   exit 1
fi

if [[ -z $BUILD_USER ]]; then
    BUILD_USER=jenkins
fi

if [[ -z $BUILD_USER_ID ]]; then
    BUILD_USER_ID=edx-sandbox
fi

if [[ -z $WORKSPACE ]]; then
    dir=$(dirname $0)
    source "$dir/ascii-convert.sh"
else
    source "$WORKSPACE/configuration/util/jenkins/ascii-convert.sh"
fi

if [[ -z $static_url_base ]]; then
  static_url_base="/static"
fi

if [[ -z $github_username  ]]; then
  github_username=$BUILD_USER_ID
fi

if [[ ! -f $BOTO_CONFIG ]]; then
  echo "AWS credentials not found for $aws_account"
  exit 1
fi

extra_vars_file="/var/tmp/extra-vars-$$.yml"
sandbox_vars_file="${WORKSPACE}/configuration-secure/ansible/vars/developer-sandbox.yml"
extra_var_arg="-e@${extra_vars_file}"

if [[ $edx_internal == "true" ]]; then
    # if this is a an edx server include
    # the secret var file
    extra_var_arg="-e@${sandbox_vars_file} -e@${extra_vars_file}"
fi

if [[ -z $region ]]; then
  region="us-east-1"
fi

# edX has reservations for sandboxes in this zone, don't change without updating reservations.
if [[ -z $zone ]]; then
  zone="us-east-1c"
fi

if [[ -z $vpc_subnet_id ]]; then
  vpc_subnet_id="subnet-cd867aba"
fi

if [[ -z $elb ]]; then
  elb="false"
fi

if [[ -z $dns_name ]]; then
  dns_name=${github_username}
fi

if [[ -z $name_tag ]]; then
  name_tag=${github_username}-${environment}
fi

if [[ -z $sandbox_platform_name ]]; then
    sandbox_platform_name=$dns_name
fi

if [[ -z $ami ]]; then
  if [[ $server_type == "full_edx_installation" ]]; then
    ami="ami-c8093ea2"
  elif [[ $server_type == "ubuntu_12.04" || $server_type == "full_edx_installation_from_scratch" ]]; then
    ami="ami-94be91fe"
  elif [[ $server_type == "ubuntu_14.04(experimental)" ]]; then
    ami="ami-35d6f95f"
  fi
fi

if [[ -z $instance_type ]]; then
  instance_type="t2.medium"
fi

if [[ -z $enable_newrelic ]]; then
  enable_newrelic="false"
fi

if [[ -z $enable_datadog ]]; then
  enable_datadog="false"
fi

if [[ -z $performance_course ]]; then
  performance_course="false"
fi

if [[ -z $demo_test_course ]]; then
  demo_test_course="false"
fi

if [[ -z $edx_demo_course ]]; then
  edx_demo_course="false"
fi

if [[ -z $enable_client_profiling ]]; then
  enable_client_profiling="false"
fi

# Lowercase the dns name to deal with an ansible bug
dns_name="${dns_name,,}"

deploy_host="${dns_name}.${dns_zone}"
ssh-keygen -f "/var/lib/jenkins/.ssh/known_hosts" -R "$deploy_host"

cd playbooks/edx-east

cat << EOF > $extra_vars_file
ansible_ssh_private_key_file: /var/lib/jenkins/${keypair}.pem
edx_platform_version: $edxapp_version
forum_version: $forum_version
notifier_version: $notifier_version
xqueue_version: $xqueue_version
xserver_version: $xserver_version
certs_version: $certs_version
configuration_version: $configuration_version

edx_ansible_source_repo: ${configuration_source_repo}
edx_platform_repo: ${edx_platform_repo}

EDXAPP_PLATFORM_NAME: $sandbox_platform_name
EDXAPP_COMPREHENSIVE_THEME_DIR: $edxapp_comprehensive_theme_dir

EDXAPP_STATIC_URL_BASE: $static_url_base
EDXAPP_LMS_NGINX_PORT: 80
EDXAPP_LMS_PREVIEW_NGINX_PORT: 80
EDXAPP_CMS_NGINX_PORT: 80

ECOMMERCE_NGINX_PORT: 80
ECOMMERCE_SSL_NGINX_PORT: 443
ECOMMERCE_VERSION: $ecommerce_version

PROGRAMS_NGINX_PORT: 80
PROGRAMS_SSL_NGINX_PORT: 443
PROGRAMS_VERSION: $programs_version

CREDENTIALS_NGINX_PORT: 80
CREDENTIALS_SSL_NGINX_PORT: 443
CREDENTIALS_VERSION: $credentials_version

DISCOVERY_NGINX_PORT: 80
DISCOVERY_SSL_NGINX_PORT: 443
DISCOVERY_VERSION: $discovery_version
NGINX_SET_X_FORWARDED_HEADERS: True
NGINX_REDIRECT_TO_HTTPS: True
EDX_ANSIBLE_DUMP_VARS: true
migrate_db: "yes"
openid_workaround: True
rabbitmq_ip: "127.0.0.1"
rabbitmq_refresh: True
COMMON_HOSTNAME: $dns_name
COMMON_DEPLOYMENT: edx
COMMON_ENVIRONMENT: sandbox

nginx_default_sites:
  - lms

# User provided extra vars
$extra_vars
EOF

if [[ $basic_auth == "true" ]]; then
    # vars specific to provisioning added to $extra-vars
    cat << EOF_AUTH >> $extra_vars_file
COMMON_ENABLE_BASIC_AUTH: True
COMMON_HTPASSWD_USER: $auth_user
COMMON_HTPASSWD_PASS: $auth_pass
XQUEUE_BASIC_AUTH_USER: $auth_user
XQUEUE_BASIC_AUTH_PASSWORD: $auth_pass
EOF_AUTH

else
    cat << EOF_AUTH >> $extra_vars_file
COMMON_ENABLE_BASIC_AUTH: False
EOF_AUTH

fi

if [[ $enable_client_profiling == "true" ]]; then
    cat << EOF_PROFILING >> $extra_vars_file
EDXAPP_SESSION_SAVE_EVERY_REQUEST: True
EOF_PROFILING
fi

if [[ $edx_internal == "true" ]]; then
    # if this isn't a public server add the github
    # user and set edx_internal to True so that
    # xserver is installed
    cat << EOF >> $extra_vars_file
EDXAPP_PREVIEW_LMS_BASE: preview-${deploy_host}
EDXAPP_LMS_BASE: ${deploy_host}
EDXAPP_CMS_BASE: studio-${deploy_host}
EDXAPP_SITE_NAME: ${deploy_host}
CERTS_DOWNLOAD_URL: "http://${deploy_host}:18090"
CERTS_VERIFY_URL: "http://${deploy_host}:18090"
edx_internal: True
COMMON_USER_INFO:
  - name: ${github_username}
    github: true
    type: admin
USER_CMD_PROMPT: '[$name_tag] '
COMMON_ENABLE_NEWRELIC_APP: $enable_newrelic
COMMON_ENABLE_DATADOG: $enable_datadog
FORUM_NEW_RELIC_ENABLE: $enable_newrelic
ENABLE_PERFORMANCE_COURSE: $performance_course
ENABLE_DEMO_TEST_COURSE: $demo_test_course
ENABLE_EDX_DEMO_COURSE: $edx_demo_course
EDXAPP_NEWRELIC_LMS_APPNAME: sandbox-${dns_name}-edxapp-lms
EDXAPP_NEWRELIC_CMS_APPNAME: sandbox-${dns_name}-edxapp-cms
EDXAPP_NEWRELIC_WORKERS_APPNAME: sandbox-${dns_name}-edxapp-workers
XQUEUE_NEWRELIC_APPNAME: sandbox-${dns_name}-xqueue
FORUM_NEW_RELIC_APP_NAME: sandbox-${dns_name}-forums
SANDBOX_USERNAME: $github_username
EDXAPP_ECOMMERCE_PUBLIC_URL_ROOT: "https://ecommerce-${deploy_host}"
EDXAPP_ECOMMERCE_API_URL: "https://ecommerce-${deploy_host}/api/v2"

ECOMMERCE_ECOMMERCE_URL_ROOT: "https://ecommerce-${deploy_host}"
ECOMMERCE_LMS_URL_ROOT: "https://${deploy_host}"
ECOMMERCE_SOCIAL_AUTH_REDIRECT_IS_HTTPS: true

PROGRAMS_LMS_URL_ROOT: "https://${deploy_host}"
PROGRAMS_URL_ROOT: "https://programs-${deploy_host}"
PROGRAMS_SOCIAL_AUTH_REDIRECT_IS_HTTPS: true

CREDENTIALS_LMS_URL_ROOT: "https://${deploy_host}"
CREDENTIALS_URL_ROOT: "https://credentials-${deploy_host}"
CREDENTIALS_SOCIAL_AUTH_REDIRECT_IS_HTTPS: true
COURSE_DISCOVERY_ECOMMERCE_API_URL: "https://ecommerce-${deploy_host}/api/v2"

DISCOVERY_OAUTH_URL_ROOT: "https://${deploy_host}"
DISCOVERY_URL_ROOT: "https://discovery-${deploy_host}"
DISCOVERY_SOCIAL_AUTH_REDIRECT_IS_HTTPS: true

EOF
fi

if [[ $recreate == "true" ]]; then
    # vars specific to provisioning added to $extra-vars
    cat << EOF >> $extra_vars_file
dns_name: $dns_name
keypair: $keypair
instance_type: $instance_type
security_group: $security_group
ami: $ami
region: $region
zone: $zone
instance_tags:
    environment: $environment
    github_username: $github_username
    Name: $name_tag
    source: jenkins
    owner: $BUILD_USER
    instance_termination_time: $TERMINATION_DATE_TIME
    datadog: monitored
root_ebs_size: $root_ebs_size
name_tag: $name_tag
dns_zone: $dns_zone
rabbitmq_refresh: True
elb: $elb
EOF

    # run the tasks to launch an ec2 instance from AMI
    cat $extra_vars_file
    run_ansible edx_provision.yml -i inventory.ini $extra_var_arg --user ubuntu

    if [[ $server_type == "full_edx_installation" ]]; then
        # additional tasks that need to be run if the
        # entire edx stack is brought up from an AMI
        run_ansible rabbitmq.yml -i "${deploy_host}," $extra_var_arg --user ubuntu
        run_ansible restart_supervisor.yml -i "${deploy_host}," $extra_var_arg --user ubuntu
    fi
fi

declare -A deploy
roles="edxapp forum ecommerce programs credentials discovery notifier xqueue xserver certs demo testcourses"

for role in $roles; do
    deploy[$role]=${!role}
done

# If reconfigure was selected or if starting from an ubuntu 12.04 AMI
# run non-deploy tasks for all roles
if [[ $reconfigure == "true" || $server_type == "full_edx_installation_from_scratch" ]]; then
    cat $extra_vars_file
    run_ansible edx_continuous_integration.yml -i "${deploy_host}," $extra_var_arg --user ubuntu
fi

if [[ $reconfigure != "true" && $server_type == "full_edx_installation" ]]; then
    # Run deploy tasks for the roles selected
    for i in $roles; do
        if [[ ${deploy[$i]} == "true" ]]; then
            cat $extra_vars_file
            run_ansible ${i}.yml -i "${deploy_host}," $extra_var_arg --user ubuntu
        fi
    done
fi

# deploy the edx_ansible role
run_ansible edx_ansible.yml -i "${deploy_host}," $extra_var_arg --user ubuntu
cat $sandbox_vars_file $extra_vars_file | grep -v -E "_version|migrate_db" > ${extra_vars_file}_clean
ansible -c ssh -i "${deploy_host}," $deploy_host -m copy -a "src=${extra_vars_file}_clean dest=/edx/app/edx_ansible/server-vars.yml" -u ubuntu -b
ret=$?
if [[ $ret -ne 0 ]]; then
  exit $ret
fi

if [[ $run_oauth == "true" ]]; then
    # Setup the OAuth2 clients
    run_ansible oauth_client_setup.yml -i "${deploy_host}," $extra_var_arg --user ubuntu
fi

# set the hostname
run_ansible set_hostname.yml -i "${deploy_host}," -e hostname_fqdn=${deploy_host} --user ubuntu

rm -f "$extra_vars_file"
rm -f ${extra_vars_file}_clean

#!/usr/bin/env bash

# Ansible provisioning wrapper script that
# assumes the following parameters set
# as environment variables
#
# - github_username
# - server_type
# - instance_type
# - region
# - aws_account
# - keypair
# - ami
# - root_ebs_size
# - security_group
# - dns_zone
# - dns_name
# - environment
# - name_tag
set -x
env
export PYTHONUNBUFFERED=1
export BOTO_CONFIG=/var/lib/jenkins/${aws_account}.boto

run_ansible() {
  if [[ "$VERBOSE" == "true" ]]; then
    verbose_arg='-vvv'
  else
    verbose_arg=''
  fi

  ansible-playbook $verbose_arg $@
  ret=$?
  if [[ $ret -ne 0 ]]; then
    exit $ret
  fi
}

# This DATE_TIME will be used as instance launch time tag
if [[ ! -n ${sandbox_life//[0-9]/} ]]  && [[ ${sandbox_life} -le 30 ]]; then
    TERMINATION_DATE_TIME=`date +"%m-%d-%Y %T" --date "${sandbox_life=7} days"`
else
   echo "Please enter the valid value for the sandbox_life(between 1 to 30)"
   exit 1
fi

if [[ -z $BUILD_USER ]]; then
    BUILD_USER=jenkins
fi

if [[ -z $BUILD_USER_ID ]]; then
    BUILD_USER_ID=edx-sandbox
fi

if [[ -z $WORKSPACE ]]; then
    dir=$(dirname $0)
    source "$dir/ascii-convert.sh"
else
    source "$WORKSPACE/configuration/util/jenkins/ascii-convert.sh"
fi

if [[ -z $static_url_base ]]; then
  static_url_base="/static"
fi

if [[ -z $github_username  ]]; then
  github_username=$BUILD_USER_ID
fi

if [[ ! -f $BOTO_CONFIG ]]; then
  echo "AWS credentials not found for $aws_account"
  exit 1
fi

extra_vars_file="/var/tmp/extra-vars-$$.yml"
sandbox_vars_file="${WORKSPACE}/configuration-secure/ansible/vars/developer-sandbox.yml"
extra_var_arg="-e@${extra_vars_file}"

if [[ $edx_internal == "true" ]]; then
    # if this is a an edx server include
    # the secret var file
    extra_var_arg="-e@${sandbox_vars_file} -e@${extra_vars_file}"
fi

if [[ -z $region ]]; then
  region="us-east-1"
fi

# edX has reservations for sandboxes in this zone, don't change without updating reservations.
if [[ -z $zone ]]; then
  zone="us-east-1c"
fi

if [[ -z $vpc_subnet_id ]]; then
  vpc_subnet_id="subnet-cd867aba"
fi

if [[ -z $elb ]]; then
  elb="false"
fi

if [[ -z $dns_name ]]; then
  dns_name=${github_username}
fi

if [[ -z $name_tag ]]; then
  name_tag=${github_username}-${environment}
fi

if [[ -z $ami ]]; then
  if [[ $server_type == "full_edx_installation" ]]; then
    ami="ami-c8093ea2"
  elif [[ $server_type == "ubuntu_12.04" || $server_type == "full_edx_installation_from_scratch" ]]; then
    ami="ami-94be91fe"
  elif [[ $server_type == "ubuntu_14.04(experimental)" ]]; then
    ami="ami-35d6f95f"
  fi
fi

if [[ -z $instance_type ]]; then
  instance_type="t2.medium"
fi

if [[ -z $enable_newrelic ]]; then
  enable_newrelic="false"
fi

if [[ -z $enable_datadog ]]; then
  enable_datadog="false"
fi

if [[ -z $performance_course ]]; then
  performance_course="false"
fi

if [[ -z $demo_test_course ]]; then
  demo_test_course="false"
fi

if [[ -z $edx_demo_course ]]; then
  edx_demo_course="false"
fi

if [[ -z $enable_client_profiling ]]; then
  enable_client_profiling="false"
fi

# Lowercase the dns name to deal with an ansible bug
dns_name="${dns_name,,}"

deploy_host="${dns_name}.${dns_zone}"
ssh-keygen -f "/var/lib/jenkins/.ssh/known_hosts" -R "$deploy_host"

cd playbooks/edx-east

cat << EOF > $extra_vars_file
ansible_ssh_private_key_file: /var/lib/jenkins/${keypair}.pem
edx_platform_version: $edxapp_version
forum_version: $forum_version
notifier_version: $notifier_version
xqueue_version: $xqueue_version
xserver_version: $xserver_version
certs_version: $certs_version
configuration_version: $configuration_version

edx_ansible_source_repo: ${configuration_source_repo}
edx_platform_repo: ${edx_platform_repo}

EDXAPP_COMPREHENSIVE_THEME_DIR: $edxapp_comprehensive_theme_dir

EDXAPP_STATIC_URL_BASE: $static_url_base
EDXAPP_LMS_NGINX_PORT: 80
EDXAPP_LMS_PREVIEW_NGINX_PORT: 80
EDXAPP_CMS_NGINX_PORT: 80

ECOMMERCE_NGINX_PORT: 80
ECOMMERCE_SSL_NGINX_PORT: 443
ECOMMERCE_VERSION: $ecommerce_version

PROGRAMS_NGINX_PORT: 80
PROGRAMS_SSL_NGINX_PORT: 443
PROGRAMS_VERSION: $programs_version

CREDENTIALS_NGINX_PORT: 80
CREDENTIALS_SSL_NGINX_PORT: 443
CREDENTIALS_VERSION: $credentials_version

DISCOVERY_NGINX_PORT: 80
DISCOVERY_SSL_NGINX_PORT: 443
DISCOVERY_VERSION: $discovery_version
NGINX_SET_X_FORWARDED_HEADERS: True
NGINX_REDIRECT_TO_HTTPS: True
EDX_ANSIBLE_DUMP_VARS: true
migrate_db: "yes"
openid_workaround: True
rabbitmq_ip: "127.0.0.1"
rabbitmq_refresh: True
COMMON_HOSTNAME: $dns_name
COMMON_DEPLOYMENT: edx
COMMON_ENVIRONMENT: sandbox

nginx_default_sites:
  - lms

# User provided extra vars
$extra_vars
EOF

if [[ $basic_auth == "true" ]]; then
    # vars specific to provisioning added to $extra-vars
    cat << EOF_AUTH >> $extra_vars_file
COMMON_ENABLE_BASIC_AUTH: True
COMMON_HTPASSWD_USER: $auth_user
COMMON_HTPASSWD_PASS: $auth_pass
XQUEUE_BASIC_AUTH_USER: $auth_user
XQUEUE_BASIC_AUTH_PASSWORD: $auth_pass
EOF_AUTH

else
    cat << EOF_AUTH >> $extra_vars_file
COMMON_ENABLE_BASIC_AUTH: False
EOF_AUTH

fi

if [[ $enable_client_profiling == "true" ]]; then
    cat << EOF_PROFILING >> $extra_vars_file
EDXAPP_SESSION_SAVE_EVERY_REQUEST: True
EOF_PROFILING
fi

if [[ $edx_internal == "true" ]]; then
    # if this isn't a public server add the github
    # user and set edx_internal to True so that
    # xserver is installed
    cat << EOF >> $extra_vars_file
EDXAPP_PREVIEW_LMS_BASE: preview-${deploy_host}
EDXAPP_LMS_BASE: ${deploy_host}
EDXAPP_CMS_BASE: studio-${deploy_host}
EDXAPP_SITE_NAME: ${deploy_host}
CERTS_DOWNLOAD_URL: "http://${deploy_host}:18090"
CERTS_VERIFY_URL: "http://${deploy_host}:18090"
edx_internal: True
COMMON_USER_INFO:
  - name: ${github_username}
    github: true
    type: admin
USER_CMD_PROMPT: '[$name_tag] '
COMMON_ENABLE_NEWRELIC_APP: $enable_newrelic
COMMON_ENABLE_DATADOG: $enable_datadog
FORUM_NEW_RELIC_ENABLE: $enable_newrelic
ENABLE_PERFORMANCE_COURSE: $performance_course
ENABLE_DEMO_TEST_COURSE: $demo_test_course
ENABLE_EDX_DEMO_COURSE: $edx_demo_course
EDXAPP_NEWRELIC_LMS_APPNAME: sandbox-${dns_name}-edxapp-lms
EDXAPP_NEWRELIC_CMS_APPNAME: sandbox-${dns_name}-edxapp-cms
EDXAPP_NEWRELIC_WORKERS_APPNAME: sandbox-${dns_name}-edxapp-workers
XQUEUE_NEWRELIC_APPNAME: sandbox-${dns_name}-xqueue
FORUM_NEW_RELIC_APP_NAME: sandbox-${dns_name}-forums
SANDBOX_USERNAME: $github_username
EDXAPP_ECOMMERCE_PUBLIC_URL_ROOT: "https://ecommerce-${deploy_host}"
EDXAPP_ECOMMERCE_API_URL: "https://ecommerce-${deploy_host}/api/v2"

ECOMMERCE_ECOMMERCE_URL_ROOT: "https://ecommerce-${deploy_host}"
ECOMMERCE_LMS_URL_ROOT: "https://${deploy_host}"
ECOMMERCE_SOCIAL_AUTH_REDIRECT_IS_HTTPS: true

PROGRAMS_LMS_URL_ROOT: "https://${deploy_host}"
PROGRAMS_URL_ROOT: "https://programs-${deploy_host}"
PROGRAMS_SOCIAL_AUTH_REDIRECT_IS_HTTPS: true

CREDENTIALS_LMS_URL_ROOT: "https://${deploy_host}"
CREDENTIALS_URL_ROOT: "https://credentials-${deploy_host}"
CREDENTIALS_SOCIAL_AUTH_REDIRECT_IS_HTTPS: true
COURSE_DISCOVERY_ECOMMERCE_API_URL: "https://ecommerce-${deploy_host}/api/v2"

DISCOVERY_OAUTH_URL_ROOT: "https://${deploy_host}"
DISCOVERY_URL_ROOT: "https://discovery-${deploy_host}"
DISCOVERY_SOCIAL_AUTH_REDIRECT_IS_HTTPS: true

EOF
fi

if [[ $recreate == "true" ]]; then
    # vars specific to provisioning added to $extra-vars
    cat << EOF >> $extra_vars_file
dns_name: $dns_name
keypair: $keypair
instance_type: $instance_type
security_group: $security_group
ami: $ami
region: $region
zone: $zone
instance_tags:
    environment: $environment
    github_username: $github_username
    Name: $name_tag
    source: jenkins
    owner: $BUILD_USER
    instance_termination_time: $TERMINATION_DATE_TIME
    datadog: monitored
root_ebs_size: $root_ebs_size
name_tag: $name_tag
dns_zone: $dns_zone
rabbitmq_refresh: True
elb: $elb
EOF

    # run the tasks to launch an ec2 instance from AMI
    cat $extra_vars_file
    run_ansible edx_provision.yml -i inventory.ini $extra_var_arg --user ubuntu

    if [[ $server_type == "full_edx_installation" ]]; then
        # additional tasks that need to be run if the
        # entire edx stack is brought up from an AMI
        run_ansible rabbitmq.yml -i "${deploy_host}," $extra_var_arg --user ubuntu
        run_ansible restart_supervisor.yml -i "${deploy_host}," $extra_var_arg --user ubuntu
    fi
fi

declare -A deploy
roles="edxapp forum ecommerce programs credentials discovery notifier xqueue xserver certs demo testcourses"

for role in $roles; do
    deploy[$role]=${!role}
done

# If reconfigure was selected or if starting from an ubuntu 12.04 AMI
# run non-deploy tasks for all roles
if [[ $reconfigure == "true" || $server_type == "full_edx_installation_from_scratch" ]]; then
    cat $extra_vars_file
    run_ansible edx_continuous_integration.yml -i "${deploy_host}," $extra_var_arg --user ubuntu
fi

if [[ $reconfigure != "true" && $server_type == "full_edx_installation" ]]; then
    # Run deploy tasks for the roles selected
    for i in $roles; do
        if [[ ${deploy[$i]} == "true" ]]; then
            cat $extra_vars_file
            run_ansible ${i}.yml -i "${deploy_host}," $extra_var_arg --user ubuntu
        fi
    done
fi

# deploy the edx_ansible role
run_ansible edx_ansible.yml -i "${deploy_host}," $extra_var_arg --user ubuntu
cat $sandbox_vars_file $extra_vars_file | grep -v -E "_version|migrate_db" > ${extra_vars_file}_clean
ansible -c ssh -i "${deploy_host}," $deploy_host -m copy -a "src=${extra_vars_file}_clean dest=/edx/app/edx_ansible/server-vars.yml" -u ubuntu -b
ret=$?
if [[ $ret -ne 0 ]]; then
  exit $ret
fi

if [[ $run_oauth == "true" ]]; then
    # Setup the OAuth2 clients
    run_ansible oauth_client_setup.yml -i "${deploy_host}," $extra_var_arg --user ubuntu
fi

# set the hostname
run_ansible set_hostname.yml -i "${deploy_host}," -e hostname_fqdn=${deploy_host} --user ubuntu

rm -f "$extra_vars_file"
rm -f ${extra_vars_file}_clean


Add apt upgrade in debian provisioning

Add apt upgrade in debian provisioning

#!/bin/bash
#wget http://packages.openxpki.org/debian/openxpki.list -O /etc/apt/sources.list.d/openxpki.list
set -e
set -x

DEBIAN_CODENAME=`lsb_release -sc`

apt-get update && apt-get upgrade --assume-yes

# Install the deps
export DEBIAN_FRONTEND=noninteractive
apt-get install --assume-yes  dh-make-perl libapache2-mod-fcgid

# Debian Buster has renamed the mysql server package
case $DEBIAN_CODENAME in
    jessie)
    PACKAGES="mysql-server"
    ;;
    stretch|buster)
    PACKAGES="default-mysql-server"
    ;;
    *)
    echo "Unsupported Debian release $DEBIAN_CODENAME"
    exit 1
    ;;
esac

apt-get install --assume-yes $PACKAGES libdbd-mysql-perl

if [ -e /code-repo/package/debian/build-deps-$DEBIAN_CODENAME.lst ]; then
    cat /code-repo/package/debian/build-deps-$DEBIAN_CODENAME.lst \
        | xargs apt-get install --assume-yes
else
    cat /code-repo/package/debian/build-deps.lst \
        | xargs apt-get install --assume-yes
fi

# packages required for testing only
apt-get install --assume-yes libtest-deep-perl

# make sure libcryptx-perl is found by dh-make-perl and package versions are found
apt-file update
apt-cache dumpavail | dpkg --merge-avail

# This inits the cpan module for dh-make-perl
(echo y;echo o conf prerequisites_policy follow;echo o conf commit)|cpan

git -C /code-repo submodule init
git -C /code-repo submodule update --checkout
HEAD=`git -C /code-repo/config rev-parse HEAD | cut -c1-6`
sed -i -r "s/^commit:.*/commit: $HEAD/" /code-repo/config/config.d/system/version.yaml

# Now chdir to the debian package dir
cd /code-repo/package/debian

# Now build the deps
make clean

make cpan_dependency cpan_dependency2

# Install remaining deps
dpkg -i deb/cpan/*deb

make openxpki

# Install the stuff - this exits with false due to unresolved deps
dpkg -i deb/core/*deb || /bin/true

# This pulls in the deps from the openxpki packages
apt-get  install --fix-broken --yes

# run the setup stuff
/vagrant/setup-dummy.sh

# Need to wait until server and watchdog are up
sleep 30;

# Kick off prove
#cd /code-repo/qatest/backend/nice
#prove .

cd /code-repo/qatest/backend/webui
prove .

#!/bin/bash
#wget http://packages.openxpki.org/debian/openxpki.list -O /etc/apt/sources.list.d/openxpki.list
set -e
set -x

DEBIAN_CODENAME=`lsb_release -sc`

apt-get update

# Install the deps
export DEBIAN_FRONTEND=noninteractive
apt-get install --assume-yes  dh-make-perl libapache2-mod-fcgid

# Debian Buster has renamed the mysql server package
case $DEBIAN_CODENAME in
    jessie)
	PACKAGES="mysql-server"
	;;
    stretch|buster)
	PACKAGES="default-mysql-server"
	;;
    *)
	echo "Unsupported Debian release $DEBIAN_CODENAME"
	exit 1
	;;
esac

apt-get install --assume-yes $PACKAGES libdbd-mysql-perl

if [ -e /code-repo/package/debian/build-deps-$DEBIAN_CODENAME.lst ]; then
    cat /code-repo/package/debian/build-deps-$DEBIAN_CODENAME.lst \
        | xargs apt-get install --assume-yes
else
    cat /code-repo/package/debian/build-deps.lst \
        | xargs apt-get install --assume-yes
fi

# packages required for testing only
apt-get install --assume-yes libtest-deep-perl

# make sure libcryptx-perl is found by dh-make-perl and package versions are found
apt-file update
apt-cache dumpavail | dpkg --merge-avail

# This inits the cpan module for dh-make-perl
(echo y;echo o conf prerequisites_policy follow;echo o conf commit)|cpan

git -C /code-repo submodule init
git -C /code-repo submodule update --checkout
HEAD=`git -C /code-repo/config rev-parse HEAD | cut -c1-6`
sed -i -r "s/^commit:.*/commit: $HEAD/" /code-repo/config/config.d/system/version.yaml

# Now chdir to the debian package dir
cd /code-repo/package/debian

# Now build the deps
make clean

make cpan_dependency cpan_dependency2

# Install remaining deps
dpkg -i deb/cpan/*deb

make openxpki

# Install the stuff - this exits with false due to unresolved deps
dpkg -i deb/core/*deb || /bin/true

# This pulls in the deps from the openxpki packages
apt-get  install --fix-broken --yes

# run the setup stuff
/vagrant/setup-dummy.sh

# Need to wait until server and watchdog are up
sleep 30;

# Kick off prove
#cd /code-repo/qatest/backend/nice
#prove .

cd /code-repo/qatest/backend/webui
prove .


argument

argument

function Usage {
  echo "Usage: $0 [ <testName> ]"
  echo "       where <testName> is for instance  shallow_openad"
  echo "       executed in build_<testName>"
}
if [ $# -eq 1 ]
then
  testName=$1
else
  testName=${PWD##*/build_}
fi
inputDir=../../../../MITgcm_contrib/heimbach/OpenAD/input_${testName}
if [ ! -d ${inputDir} ]
then
  echo "Error: input directory ${inputDir} doesn't exist"
  Usage $*
  exit -1
fi
ln -sf ${inputDir}/* .
tDir=$PWD
cd ../../global_ocean.90x40x15/input
./prepare_run > /dev/null
cd $tDir
ln -sf ../../global_ocean.90x40x15/input/*.bin .

function Usage {
  echo "Usage: $0 <testName>"
  echo "       where <testName> is for instance  shallow_openad"
  echo "       executed in build_<testName>"
}
testName=${PWD##*/build_}
inputDir=../../../../MITgcm_contrib/heimbach/OpenAD/input_${testName}
if [ ! -d ${inputDir} ]
then
  echo "Error: input directory ${inputDir} doesn't exist"
  Usage $*
  exit -1
fi
ln -sf ${inputDir}/* .
cd ../../global_ocean.90x40x15/input
./prepare_run > /dev/null
cd ../../OpenAD/build_${testName}
ln -sf ../../global_ocean.90x40x15/input/*.bin .


use java devel 1.8.0

use java devel 1.8.0

#!/bin/bash
# These are all of the packages that need to be installed before bootstrap
# is run
#
# Note that all apache modules should have already been installed
# in the bootstrap image.  Otherwise you will have the system attempt
# to reload httpd which causes the httpd connection to go down
#
# dokuwiki also needs to be in bootstrap for the same reasons

if grep -q Cauldron /etc/release  ; then
JAVA=java-1.8.0-openjdk-devel
fi

set -e
sudo urpmi --no-suggests \
--auto \
--downloader "curl" \
--curl-options "--retry 5 --speed-time 30 --connect-timeout 30" \
maven \
maven-clean-plugin \
maven-assembly-plugin \
maven-compiler-plugin \
maven-dependency-plugin \
maven-install-plugin \
maven-release-plugin \
aether \
aether-connector-basic \
aether-transport-file \
aether-transport-http \
aether-transport-wagon \
fop \
nodejs \
gcc-c++ \
ipython make postgresql9.3-devel \
strace \
screen \
krb5-appl-clients \
python-flask \
python-pexpect \
R-base \
vim-minimal \
rstudio-server \
fudge-devel \
log4cxx-devel \
postgresql9.3-server \
python-pyzmq \
dokuwiki \
cmake \
ipython \
python-tornado \
python-mglob \
dokuwiki-plugin-s5  \
python-matplotlib \
python-py4j \
python-pyro4 \
python-pytz \
python-pip \
python-devel \
readline-devel \
lapack-devel \
python-pandas \
python-zipline \
$JAVA

#cmake is for building shiny-server
#tornado and mglob is for ipython
#readline-devel, python-devel, lapack-devel are for Rpy

# don't start up server
sudo systemctl disable hsqldb

#gcc-c++ is needed for ethercalc

#!/bin/bash
# These are all of the packages that need to be installed before bootstrap
# is run
#
# Note that all apache modules should have already been installed
# in the bootstrap image.  Otherwise you will have the system attempt
# to reload httpd which causes the httpd connection to go down
#
# dokuwiki also needs to be in bootstrap for the same reasons

if grep -q Cauldron /etc/release  ; then
JAVA=java-1.8.0-openjdk
fi

set -e
sudo urpmi --no-suggests \
--auto \
--downloader "curl" \
--curl-options "--retry 5 --speed-time 30 --connect-timeout 30" \
maven \
maven-clean-plugin \
maven-assembly-plugin \
maven-compiler-plugin \
maven-dependency-plugin \
maven-install-plugin \
maven-release-plugin \
aether \
aether-connector-basic \
aether-transport-file \
aether-transport-http \
aether-transport-wagon \
fop \
nodejs \
gcc-c++ \
ipython make postgresql9.3-devel \
strace \
screen \
krb5-appl-clients \
python-flask \
python-pexpect \
R-base \
vim-minimal \
rstudio-server \
fudge-devel \
log4cxx-devel \
postgresql9.3-server \
python-pyzmq \
dokuwiki \
cmake \
ipython \
python-tornado \
python-mglob \
dokuwiki-plugin-s5  \
python-matplotlib \
python-py4j \
python-pyro4 \
python-pytz \
python-pip \
python-devel \
readline-devel \
lapack-devel \
python-pandas \
python-zipline \
$JAVA

#cmake is for building shiny-server
#tornado and mglob is for ipython
#readline-devel, python-devel, lapack-devel are for Rpy

# don't start up server
sudo systemctl disable hsqldb

#gcc-c++ is needed for ethercalc


initial import of inittab respawn script

#!/bin/bash
#
# Runs TWKE. If it fails/exits, restart with the latest version
#
# This script assumes that you've created a user account e.g. `twke`
# whose home directory can be used for log files. Either this user
# or the system should have an rvm installation with Ruby 1.9.2. Copy
# this script into that user's ~/bin
#
# You can then put a command such as the following in your
# /etc/inittab to ensure twke always restarts with the latest
# code, including through the 'twke respawn' command:
#
# # Run twke
# twke:345:respawn:/bin/su -c '/home/twke/bin/respawn_twke.sh' twke

# The repo you want to clone your twke from, override to use
# your own fork (suggested)
REPO="git@github.com:josephruscio/twke.git"

# This is where we'll run twke from
BASEDIR="/tmp/twke_run"

# monit uses this
PIDFILE="$BASEDIR/twkerun.pid"

# The API key of your bot's campfire user
APIKEY=""

# The domain of your campfire account
DOMAIN="ohai"

# The initial room you'd like to connect to
ROOM="470001"

RUNLOG="$HOME/runtwke.log"
LOG="$HOME/twke.log"

BUNDLE_DIR="/tmp/twke_bundle_cache"

export PATH="$PATH:/usr/local/bin:$HOME/git/awsam/bin"

# Load RVM into a shell session *as a function*
if [[ -s "$HOME/.rvm/scripts/rvm" ]] ; then
	# First try to load from a user install
	source "$HOME/.rvm/scripts/rvm"
elif [[ -s "/usr/local/rvm/scripts/rvm" ]] ; then
	# Then try to load from a root install
	source "/usr/local/rvm/scripts/rvm"
else
	printf "ERROR: An RVM installation was not found.\n"
	exit 1
fi

rvm use ruby-1.9.2 >> $RUNLOG 2>&1 || \
	{ echo "Failed to switch to Ruby 1.9.2" >> $RUNLOG ; exit 1; }

mkdir -p $BASEDIR

rm -rf $BASEDIR/*

REPODIR=`mktemp -d $BASEDIR/repo.XXXXXX`

git clone -q $REPO $REPODIR >> $RUNLOG 2>&1 || \
	{ echo "Failed to clone twke repo!" >> $RUNLOG; exit 1; }

pushd $REPODIR >> $RUNLOG

mkdir -p $BUNDLE_DIR || \
	{ echo "Failed to create bundler cache: $BUNDLE_DIR" >> $RUNLOG; exit 1; }

bundle install --path $BUNDLE_DIR/gems >> $RUNLOG 2>&1 || \
	{ echo "Failed to install twke gems"; exit 1; }

# Use this directory as Twke's base temporary directory
TMPDIR=`mktemp -d $BASEDIR/runtmp.XXXXXX`

export TMPDIR
bundle exec ./bin/twke -k "$APIKEY" -s "$DOMAIN" -r "$ROOM" >> $LOG 2>&1

#echo $! > $PIDFILE

popd > /dev/null

error: pathspec 'contrib/respawn_twke.sh' did not match any file(s) known to git


moved index-milljek uploader into util here

#!/bin/sh

jekyls=$(find /Users/ccmiller/git/milleriajek/_site -d -type f -iname "*.html" -not -iname "index*")

for j in $jekyls; do
	id=$(basename "${j}" .html)

# echo "checking $j..."

# manualstr=""

# locsarr=$(cat $j | grep cartodb\:)

# for l in $locsarr; do

	# echo $l|awk -F '[:]' '{print $(NF-0)}'
	# echo $l| cut -d':' -f 2

# done

# echo $locsarr

# body=$(xml sel -Q -t --value-of //body $j)
# bodypeach=$(echo $body | php -r 'echo htmlentities($argn);')
#
# echo $body;

	# echo "submitting"$id"..."
	# echo "bodypeach:"$bodypeach
	# curl "http://localhost:8983/solr/miljek/update/extract?literal.id="$id"&commit=true" -F "myfile=@"$j
	curl "http://localhost:8983/solr/miljek/update/extract?literal.id="$id"&commit=true&fmap.body=body" -F "myfile=@"$j
	# curl "http://localhost:8983/solr/miljek/update/extract?literal.id="$id"&commit=true&literal.body="$body -F "myfile=@"$j
	# curl "http://localhost:8983/solr/miljek/update/extract?literal.id="$id"&commit=true&literal.body=${bodypeach}" -F "myfile=@"$j
	# curl "http://localhost:8983/solr/miljek/update/extract?literal.id="$id"&commit=true&literal.body=${bodypeach}" -F "myfile=@"$j
done

error: pathspec 'util/index-miljek.sh' did not match any file(s) known to git


Added block_branch_names.sh

Added block_branch_names.sh

#!/usr/bin/env bash

#
# Pre-receive hook that will block any new commits that their names contain
# other than lower-case alphabet characters (a-z).
#
# More details on pre-receive hooks and how to apply them can be found on
# https://help.github.com/enterprise/admin/guides/developer-workflow/managing-pre-receive-hooks-on-the-github-enterprise-appliance/
#

zero_commit="0000000000000000000000000000000000000000"

while read oldrev newrev refname; do
  # Prevent creation of new branches that don't match `^refs/heads/[a-z]+$`
  if [[ $oldrev == $zero_commit && ! $refname =~ ^refs/heads/[a-z]+$ ]]; then
    echo "Blocking creation of new branch $refname because it must only contain lower-case alphabet characters."
    exit 1
  fi
done
exit 0

error: pathspec 'pre-receive-hooks/block_branch_names.sh' did not match any file(s) known to git


remove heroku remote from shell file

remove heroku remote from shell file

remote: Support for password authentication was removed on August 13, 2021.
remote: Please see https://docs.github.com/en/get-started/getting-started-with-git/about-remote-repositories#cloning-with-https-urls for information on currently recommended modes of authentication.
fatal: Authentication failed for 'https://github.com/Fuzlin/Singularity.git/'


Removing locale bolocks

Removing locale bolocks

remote: Support for password authentication was removed on August 13, 2021.
remote: Please see https://docs.github.com/en/get-started/getting-started-with-git/about-remote-repositories#cloning-with-https-urls for information on currently recommended modes of authentication.
fatal: Authentication failed for 'https://github.com/mvdan/tyk.git/'


Create restart.sh

Create restart.sh

error: pathspec 'deployer/src/main/bin/restart.sh' did not match any file(s) known to git


provide env for nodejs memory usage

provide env for nodejs memory usage

# Allows nodejs to use more memory
export NODE_OPTIONS='--max-old-space-size=8096'

error: pathspec 'zsh/config.d/available/nodejs__env.zsh' did not match any file(s) known to git


Search for running_vm_list only in `machines` folder

otherwise it might catch a non-machine id, in my case, for hostmanager plugin

#!/bin/bash

# (The MIT License)
#
# Copyright (c) 2014 Kura
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the 'Software'), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.

__pwdln() {
   pwdmod="${PWD}/"
   itr=0
   until [[ -z "$pwdmod" ]];do
      itr=$(($itr+1))
      pwdmod="${pwdmod#*/}"
   done
   echo -n $(($itr-1))
}

__vagrantinvestigate() {
    if [ -f "${PWD}/.vagrant" -o -d "${PWD}/.vagrant" ];then
      echo "${PWD}/.vagrant"
      return 0
   else
      pwdmod2="${PWD}"
      for (( i=2; i<=$(__pwdln); i++ ));do
         pwdmod2="${pwdmod2%/*}"
         if [ -f "${pwdmod2}/.vagrant" -o -d "${pwdmod2}/.vagrant" ];then
            echo "${pwdmod2}/.vagrant"
            return 0
         fi
      done
   fi
   return 1
}

_vagrant() {
    cur="${COMP_WORDS[COMP_CWORD]}"
    prev="${COMP_WORDS[COMP_CWORD-1]}"
    commands="box cloud connect destroy docker-exec docker-logs docker-run global-status halt help init list-commands login package plugin provision push rdp reload resume rsync rsync-auto share snapshot ssh ssh-config status suspend up version"

    if [ $COMP_CWORD == 1 ]
    then
      COMPREPLY=($(compgen -W "${commands}" -- ${cur}))
      return 0
    fi

    if [ $COMP_CWORD == 2 ]
    then
        case "$prev" in
            "init")
              local box_list=$(find "${VAGRANT_HOME:-${HOME}/.vagrant.d}/boxes" -mindepth 1 -maxdepth 1 -type d -exec basename {} \; | sed -e 's/-VAGRANTSLASH-/\//')
              COMPREPLY=($(compgen -W "${box_list}" -- ${cur}))
              return 0
            ;;
            "up")
              vagrant_state_file=$(__vagrantinvestigate) || return 1
              if [[ -d "${vagrant_state_file}" ]]
              then
                local vm_list=$(find "${vagrant_state_file}/machines" -mindepth 1 -maxdepth 1 -type d -exec basename {} \;)
              fi
              local up_commands="--no-provision"
              COMPREPLY=($(compgen -W "${up_commands} ${vm_list}" -- ${cur}))
              return 0
            ;;
            "destroy"|"ssh"|"provision"|"reload"|"halt"|"suspend"|"resume"|"ssh-config")
              vagrant_state_file=$(__vagrantinvestigate) || return 1
              if [[ -f "${vagrant_state_file}" ]]
              then
                running_vm_list=$(grep 'active' "${vagrant_state_file}" | sed -e 's/"active"://' | tr ',' '\n' | cut -d '"' -f 2 | tr '\n' ' ')
              else
                running_vm_list=$(find "${vagrant_state_file}/machines" -type f -name "id" | awk -F"/" '{print $(NF-2)}')
              fi
              COMPREPLY=($(compgen -W "${running_vm_list}" -- ${cur}))
              return 0
            ;;
            "box")
              box_commands="add help list outdated prune remove repackage update"
              COMPREPLY=($(compgen -W "${box_commands}" -- ${cur}))
              return 0
            ;;
            "cloud")
              cloud_commands="auth box search provider publish version"
              COMPREPLY=($(compgen -W "${cloud_commands}" -- ${cur}))
              return 0
            ;;
            "plugin")
              plugin_commands="install license list uninstall update"
              COMPREPLY=($(compgen -W "${plugin_commands}" -- ${cur}))
              return 0
            ;;
            "help")
              COMPREPLY=($(compgen -W "${commands}" -- ${cur}))
              return 0
            ;;
            "snapshot")
              snapshot_commands="delete list pop push restore save"
              COMPREPLY=($(compgen -W "${snapshot_commands}" -- ${cur}))
              return 0
            ;;
            *)
            ;;
        esac
    fi

    if [ $COMP_CWORD == 3 ]
    then
      action="${COMP_WORDS[COMP_CWORD-2]}"
      case "$action" in
        "up")
          if [ "$prev" == "--no-provision" ]
          then
            if [[ -d "${vagrant_state_file}" ]]
            then
              local vm_list=$(find "${vagrant_state_file}/machines" -mindepth 1 -maxdepth 1 -type d -exec basename {} \;)
            fi
            COMPREPLY=($(compgen -W "${vm_list}" -- ${cur}))
            return 0
          fi
          ;;
        "box")
          case "$prev" in
            "remove"|"repackage")
              local box_list=$(find "${VAGRANT_HOME:-${HOME}/.vagrant.d}/boxes" -mindepth 1 -maxdepth 1 -type d -exec basename {} \; | sed -e 's/-VAGRANTSLASH-/\//')
              COMPREPLY=($(compgen -W "${box_list}" -- ${cur}))
              return 0
              ;;
            *)
              ;;
          esac
          ;;
        "snapshot")
          case "$prev" in
            "restore"|"delete")
              local snapshot_list=$(vagrant snapshot list)
              COMPREPLY=($(compgen -W "${snapshot_list}" -- ${cur}))
              return 0
            ;;
          esac
          ;;
        *)
          ;;
      esac
    fi

}
complete -F _vagrant vagrant

#!/bin/bash

# (The MIT License)
#
# Copyright (c) 2014 Kura
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the 'Software'), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.

__pwdln() {
   pwdmod="${PWD}/"
   itr=0
   until [[ -z "$pwdmod" ]];do
      itr=$(($itr+1))
      pwdmod="${pwdmod#*/}"
   done
   echo -n $(($itr-1))
}

__vagrantinvestigate() {
    if [ -f "${PWD}/.vagrant" -o -d "${PWD}/.vagrant" ];then
      echo "${PWD}/.vagrant"
      return 0
   else
      pwdmod2="${PWD}"
      for (( i=2; i<=$(__pwdln); i++ ));do
         pwdmod2="${pwdmod2%/*}"
         if [ -f "${pwdmod2}/.vagrant" -o -d "${pwdmod2}/.vagrant" ];then
            echo "${pwdmod2}/.vagrant"
            return 0
         fi
      done
   fi
   return 1
}

_vagrant() {
    cur="${COMP_WORDS[COMP_CWORD]}"
    prev="${COMP_WORDS[COMP_CWORD-1]}"
    commands="box cloud connect destroy docker-exec docker-logs docker-run global-status halt help init list-commands login package plugin provision push rdp reload resume rsync rsync-auto share snapshot ssh ssh-config status suspend up version"

    if [ $COMP_CWORD == 1 ]
    then
      COMPREPLY=($(compgen -W "${commands}" -- ${cur}))
      return 0
    fi

    if [ $COMP_CWORD == 2 ]
    then
        case "$prev" in
            "init")
              local box_list=$(find "${VAGRANT_HOME:-${HOME}/.vagrant.d}/boxes" -mindepth 1 -maxdepth 1 -type d -exec basename {} \; | sed -e 's/-VAGRANTSLASH-/\//')
              COMPREPLY=($(compgen -W "${box_list}" -- ${cur}))
              return 0
            ;;
            "up")
              vagrant_state_file=$(__vagrantinvestigate) || return 1
              if [[ -d "${vagrant_state_file}" ]]
              then
                local vm_list=$(find "${vagrant_state_file}/machines" -mindepth 1 -maxdepth 1 -type d -exec basename {} \;)
              fi
              local up_commands="--no-provision"
              COMPREPLY=($(compgen -W "${up_commands} ${vm_list}" -- ${cur}))
              return 0
            ;;
            "destroy"|"ssh"|"provision"|"reload"|"halt"|"suspend"|"resume"|"ssh-config")
              vagrant_state_file=$(__vagrantinvestigate) || return 1
              if [[ -f "${vagrant_state_file}" ]]
              then
                running_vm_list=$(grep 'active' "${vagrant_state_file}" | sed -e 's/"active"://' | tr ',' '\n' | cut -d '"' -f 2 | tr '\n' ' ')
              else
                running_vm_list=$(find "${vagrant_state_file}" -type f -name "id" | awk -F"/" '{print $(NF-2)}')
              fi
              COMPREPLY=($(compgen -W "${running_vm_list}" -- ${cur}))
              return 0
            ;;
            "box")
              box_commands="add help list outdated prune remove repackage update"
              COMPREPLY=($(compgen -W "${box_commands}" -- ${cur}))
              return 0
            ;;
            "cloud")
              cloud_commands="auth box search provider publish version"
              COMPREPLY=($(compgen -W "${cloud_commands}" -- ${cur}))
              return 0
            ;;
            "plugin")
              plugin_commands="install license list uninstall update"
              COMPREPLY=($(compgen -W "${plugin_commands}" -- ${cur}))
              return 0
            ;;
            "help")
              COMPREPLY=($(compgen -W "${commands}" -- ${cur}))
              return 0
            ;;
            "snapshot")
              snapshot_commands="delete list pop push restore save"
              COMPREPLY=($(compgen -W "${snapshot_commands}" -- ${cur}))
              return 0
            ;;
            *)
            ;;
        esac
    fi

    if [ $COMP_CWORD == 3 ]
    then
      action="${COMP_WORDS[COMP_CWORD-2]}"
      case "$action" in
        "up")
          if [ "$prev" == "--no-provision" ]
          then
            if [[ -d "${vagrant_state_file}" ]]
            then
              local vm_list=$(find "${vagrant_state_file}/machines" -mindepth 1 -maxdepth 1 -type d -exec basename {} \;)
            fi
            COMPREPLY=($(compgen -W "${vm_list}" -- ${cur}))
            return 0
          fi
          ;;
        "box")
          case "$prev" in
            "remove"|"repackage")
              local box_list=$(find "${VAGRANT_HOME:-${HOME}/.vagrant.d}/boxes" -mindepth 1 -maxdepth 1 -type d -exec basename {} \; | sed -e 's/-VAGRANTSLASH-/\//')
              COMPREPLY=($(compgen -W "${box_list}" -- ${cur}))
              return 0
              ;;
            *)
              ;;
          esac
          ;;
        "snapshot")
          case "$prev" in
            "restore"|"delete")
              local snapshot_list=$(vagrant snapshot list)
              COMPREPLY=($(compgen -W "${snapshot_list}" -- ${cur}))
              return 0
            ;;
          esac
          ;;
        *)
          ;;
      esac
    fi

}
complete -F _vagrant vagrant


Script to backup current ansible-tree

Script to backup current ansible-tree

#!/bin/bash
pushd ~
tar cJf fgci-ansible-`date +%Y%m%d`.tar.xz -X .fgci-ansible-excludes ./fgci-ansible
popd

error: pathspec 'tools/archive.sh' did not match any file(s) known to git


Include binary packages on cd.

Include binary packages on cd.

remote: Support for password authentication was removed on August 13, 2021.
remote: Please see https://docs.github.com/en/get-started/getting-started-with-git/about-remote-repositories#cloning-with-https-urls for information on currently recommended modes of authentication.
fatal: Authentication failed for 'https://github.com/veritas-shine/minix3-rpi.git/'


Ssycall tool (thanks querat_g !)

Ssycall tool (thanks querat_g !)

#!/bin/bash
## syscall.sh for  in /home/querat_g/tmp/syscall
##
## Made by querat_g
## Login   <querat_g@epitech.net>
##
## Started on  Thu Apr  7 15:05:46 2016 querat_g
## Last update Fri Apr  8 11:07:48 2016 querat_g
##

#set -x

if [ ! -d man ]
then
    mkdir man
fi

function echolor { echo -e "$@"; }
function color() { echo -ne "$@";}

LRED="\e[1;31m";
RED="\e[0;31m";
LGREEN="\e[1;32m";
GREEN="\e[0;32m";
YELLOW="\e[1;33m";
BROWN="\e[0;33m";
BLUE="\e[1;34m";
PINK="\e[1;35m";
CYAN="\e[1;36m";
WHITE="\e[0m";

failFile=failedSyscalls.txt
echo "" > $failFile

listFile=$(find /usr/include -name unistd_64.h | head -n1)

listSysCalls="$(cat $listFile | grep __NR_)"

IFS="
"

for sysCall in $listSysCalls
do
    # on enlève le _NR__, les #define et autres crasses
    line=$(echo $sysCall | sed -E "s/__NR_//g" | sed -E "s/#define //g" | sed -E 's/64//g')

    sysCallId=$(echo $line | cut -d\  -f2)
    sysCallName=$(echo $line | cut -d\  -f1)

    # (rt_)sigaction et autres saletés en début de nom de fonction
    sysCallName=$(echo $sysCallName | sed -E 's/^rt_//g')

    # on ouvre le man
    sysCallManName=$(echo $sysCallName | sed -E 's/[0-9]+$//g' | sed -E 's/^rt_//g')
    echo $sysCallManName
    manFile="./man/$sysCallManName.man"
    echolor $CYAN"reading man $sysCallName"
    man 2 $sysCallName 1> $manFile 2> /dev/null

    # si il est pas trouvé, on gueule
    if [ ! -s $manFile ]
    then
	echolor $RED"man $sysCallName not found"$WHITE;
	echo $sysCallName >> $failFile
	continue
    fi;

    # récupération du man de son Synopsis à sa Description
    synopsisStart=$(cat $manFile | grep -n SYNOPSIS | cut -d ':' -f1)
    synopsisEnd=$(cat $manFile | grep -n DESCRIPTION | cut -d ':' -f1)
    protos=$(cat $manFile | sed -n "$synopsisStart, $synopsisEnd p ")

    # espaces multiples
    protos=$(echo $protos | sed -E 's/\s+/ /g')

    proto1=$(echo $protos | cut -d\; -f1 | egrep -o '((static|internal|extern)\s+)?([a-zA-Z0-9_]+)[\t ]+([a-zA-Z0-9_]+)\((.)+\)')
    proto2=$(echo $protos | cut -d\; -f2 | egrep -o '((static|internal|extern)\s+)?([a-zA-Z0-9_]+)[\t ]+([a-zA-Z0-9_]+)\((.)+\)')
    proto3=$(echo $protos | cut -d\; -f3 | egrep -o '((static|internal|extern)\s+)?([a-zA-Z0-9_]+)[\t ]+([a-zA-Z0-9_]+)\((.)+\)')
    proto4=$(echo $protos | cut -d\; -f4 | egrep -o '((static|internal|extern)\s+)?([a-zA-Z0-9_]+)[\t ]+([a-zA-Z0-9_]+)\((.)+\)')
    proto1=$(echo $protos | cut -d\; -f1 | egrep -o "([a-zA-Z0-9_]+)[\t ]+(\*)?([a-zA-Z0-9_]+)\((.)+\)")
    proto2=$(echo $protos | cut -d\; -f2 | egrep -o "([a-zA-Z0-9_]+)[\t ]+(\*)?([a-zA-Z0-9_]+)\((.)+\)")
    proto3=$(echo $protos | cut -d\; -f3 | egrep -o "([a-zA-Z0-9_]+)[\t ]+(\*)?([a-zA-Z0-9_]+)\((.)+\)")
    proto4=$(echo $protos | cut -d\; -f4 | egrep -o "([a-zA-Z0-9_]+)[\t ]+(\*)?([a-zA-Z0-9_]+)\((.)+\)")

    # on cherche ou se trouve le bon proto
    echo $proto1 | egrep "$sysCallName\(" > /dev/null
    proto1Match=$?
    echo $proto2 | egrep "$sysCallName\(" > /dev/null
    proto2Match=$?
    echo $proto3 | egrep "$sysCallName\(" > /dev/null
    proto3Match=$?
    echo $proto4 | egrep "$sysCallName\(" > /dev/null
    proto4Match=$?
    proto=""
    if [ $proto1Match -eq 0 ]
    then
     	proto=$proto1;
    elif [ $proto2Match -eq 0 ]
    then
     	proto=$proto2;

    elif [ $proto3Match -eq 0 ]
    then
     	proto=$proto3;
    elif [ $proto4Match -eq 0 ]
    then
     	proto=$proto4;
    else
	echolor $LRED "Proto not found for $sysCallName !"
	echo $sysCallName >> $failFile
	# on sort de la boucle générale sans parser de proto pour cette fonction
	continue;
    fi

    # des fois il reste un bout du synopsis
    proto=$(echo $proto | egrep "([a-zA-Z0-9_]+)(\s+)(\*?)$sysCallName\(.*\)$")
    proto=$(echo $proto | sed -E "s/(.*>)//g")

    # Du coup, je décline toute responsabilité en cas de proto disparu
    if [ -z $proto ]
    then
	echolor $LRED "Empty proto $sysCallName" $WHITE
	echo $sysCallName >> $failFile
	continue
    fi

    echolor $LGREEN "proto ! >> $proto" $WHITE;

    echo
done

error: pathspec 'tools/syscall.sh' did not match any file(s) known to git


Add zsh_reload which provides src function, this function will source .zshrc and rebuilds cache

zsh_cache=$HOME/.zsh_cache
mkdir -p $zsh_cache

# reload zshrc
function src()
{
  autoload -U compinit zrecompile
  compinit -d $zsh_cache/zcomp-$HOST
  for f in $HOME/.zshrc $zsh_cache/zcomp-$HOST; do
    zrecompile -p $f && rm -f $f.zwc.old
  done
  source ~/.zshrc
}

error: pathspec 'plugins/zsh_reload/zsh_reload.plugin.zsh' did not match any file(s) known to git


Renamed preset again.

Renamed preset again.

#!/bin/bash
cmake -DWITH="ALL" -DCHECK=off -DARITH=gmp -DFP_PRIME=254 -DFP_QNRES=on -DFP_METHD="BASIC;COMBA;COMBA;MONTY;LOWER;SLIDE" DFP_QNRES=on -DPC_METHD="PRIME" -DPP_METHD="INTEG;INTEG;LAZYR;OATEP" -DCOMP="-O2 -funroll-loops -fomit-frame-pointer" $1

error: pathspec 'preset/gmp-pbc-128.sh' did not match any file(s) known to git


I need yarn back

I need yarn back

remote: Support for password authentication was removed on August 13, 2021.
remote: Please see https://docs.github.com/en/get-started/getting-started-with-git/about-remote-repositories#cloning-with-https-urls for information on currently recommended modes of authentication.
fatal: Authentication failed for 'https://github.com/jrock2004/install-scripts.git/'


added ip alias for networking

added ip alias for networking

remote: Support for password authentication was removed on August 13, 2021.
remote: Please see https://docs.github.com/en/get-started/getting-started-with-git/about-remote-repositories#cloning-with-https-urls for information on currently recommended modes of authentication.
fatal: Authentication failed for 'https://github.com/buonzz/scaleout-php.git/'


Add JSLint-to-xml script.

Add JSLint-to-xml script.

#!/bin/bash

###########################################################################
#
#   CONFIG
#   ------

# The path (relative to this script file) to the Rhino JAR:
RHINO=../lib/js.jar

# The path (relative to this script file) to the JSLint JS:
JSLINT=../lib/rhinoed_jslint.js

# The base of the test name that will be reported in Hudson:
TEST_BASENAME="org.mikewest.static"

###########################################################################
#
#   SCRIPT (no touchies)
#   --------------------
#
SCRIPTPATH=`dirname ${0}`
RHINO="${SCRIPTPATH}/${RHINO}"
JSLINT="${SCRIPTPATH}/${JSLINT}"

if [ -a "${1}" ] && [ -d "${2}" ]; then

    TEMP=`mktemp -t jslintreport.XXXXXX` || {
        echo "FATAL: Couldn't create temp file for reports"
        exit 1
    }

    LINTEE=$1
    REPORT_ROOT=$2
    LINTEE_TYPE=`echo ${LINTEE} | sed 's#\(.*\)\.\([^\.]*\)$#\2#'`
    LINTEE_BASENAME=`basename ${LINTEE} | sed 's#\(.*\)\.\([^\.]*\)$#\1#'`
    TEST_BASENAME="${TEST_BASENAME}.jslint.${LINTEE_TYPE}.${LINTEE_BASENAME}"
    REPORT_FILENAME="${REPORT_ROOT}/${TEST_BASENAME}.xml"
    mkdir -p $REPORT_ROOT || {
        echo "FATAL: Couldn't create directory for reports"
        exit 1
    }

    if [ "${LINTEE_TYPE}" == "css" ]; then
        #
        #   For CSS files, prepend `@charset "UTF-8";`, write to a temp file
        #   and parse _that_ file rather than the original.  This is much simpler
        #   than stripping out the charset from each file when minimizing.
        #
        TOPARSE=`mktemp -t csslintreport.XXXXXX` || {
            echo "FATAL: Couldn't create temp file for reports"
            exit 1
        }
        awk -v PREPEND='@charset "UTF-8";' 'BEGIN {print PREPEND}{print}' ${LINTEE} > ${TOPARSE}
        TODISPLAY=$LINTEE
    else
        TOPARSE=$LINTEE
        TODISPLAY=$LINTEE
    fi

    LINT_TIME=`{ time java -jar ${RHINO} ${JSLINT} "${TOPARSE}" "${TODISPLAY}" 2>&1 1> "${TEMP}"; } 2>&1 | grep real | sed 's#.*m\(.*\)s#\1#'`
    LINT_RESULTS=$(<"${TEMP}")

    WAS_FAILURE=`cat "${TEMP}" | grep -v 'No problems found in'`
    rm ${TEMP}

    if [ "---${WAS_FAILURE}---" != "------" ]; then
        echo "<testsuite failures='1' time='${LINT_TIME}' errors='1' tests='1' skipped='0' name='${TEST_BASENAME}'>" > $REPORT_FILENAME
        echo "  <testcase time='${LINT_TIME}' name='testJSLint' classname='${TEST_BASENAME}'>" >> $REPORT_FILENAME
        echo "    <error message='${LINTEE_BASENAME} failed JSLint.'><![CDATA[${LINT_RESULTS}]]></error>" >> $REPORT_FILENAME
        echo "  </testcase>" >> $REPORT_FILENAME
        echo "</testsuite>" >> $REPORT_FILENAME
    else
        echo "<testsuite failures='0' time='${LINT_TIME}' errors='0' tests='1' skipped='0' name='${TEST_BASENAME}'>" > $REPORT_FILENAME
        echo "  <testcase time='${LINT_TIME}' name='testJSLint' classname='${TEST_BASENAME}'>" >> $REPORT_FILENAME
        echo "  </testcase>" >> $REPORT_FILENAME
        echo "</testsuite>" >> $REPORT_FILENAME
    fi
else
    echo "Usage: ${0} <file to lint> <path to reports>"
    exit 1
fi

error: pathspec 'scripts/jslint-to-xml.sh' did not match any file(s) known to git


Fix theme not found when deployed to GitHub Pages

Signed-off-by: auto deployer <93448d082884f60b4ee171ca9201c3f24fe16f94@gmail.com>

remote: Support for password authentication was removed on August 13, 2021.
remote: Please see https://docs.github.com/en/get-started/getting-started-with-git/about-remote-repositories#cloning-with-https-urls for information on currently recommended modes of authentication.
fatal: Authentication failed for 'https://github.com/trendystore/store.git/'