Create bwa_bedops_paired_align_non-interleaved_general.sh

#!/bin/bash

# Author: Leah Roberts
# Affiliations: Scott Beatson Lab Group - University of Queensland St Lucia
# Date: October 2014

############# Script Description ################

# This script is designed to align reads to a reference using BWA, and then count the
# number of reads overlapping predefined regions. This script generates two outputs: one
# output describes the number of reads physically overlapping the predefined regions, and
# another output describing read-pairs that span across the desired regions.

# The script should be run from inside the directory with the fastq reads:

# $ bash <script.sh> $REFERENCE $BEDMAP_REFERENCE

# It is very important that the fastq files are formatted correctly for the script to work:
# 	name_1.fastq
# 	name_2.fastq

# Reads will be mapped to the REFERENCE. It should be in the format ../reference.fa
# the BEDMAP_REFERENCE defines the desired regions. It should be in the format ../bedmap_format.fa
# To find out more, please read the README file available on Github (https://github.com/LeahRoberts/FiSC).

REFERENCE=$1
BEDMAP_REFERENCE=$2

# First need to index the reference fasta file to be used in the mapping:
echo "indexing " $REFERENCE
bwa index $REFERENCE

# Write a loop that will generate .sai files by reading each of the paired strains to the
# reference strain using the tool BWA. These are outputted as $name_1.sai or $name_2.sai.

for f in *

do

# Used cut to parse out the strain name - THIS WILL ONLY WORK IF THE FASTQ FILE IS
# FORMATTED CORRECTLY (i.e. $strainname_1.fastq).
# Prints the name of the strain currently being processed:

	echo "processing $(echo $f | cut -f1 -d.)"

# Formatted script so that it will take in both zipped and unzipped fastq files

		if [[ $f == *_1.fastq.gz ]]
		then
			gunzip $f
			read1=$(echo $f | cut -f1-2 -d.)
			name1=$(echo $f | cut -f1 -d.)
			bwa aln $REFERENCE $read1 > $name1.sai

		elif [[ $f == *_2.fastq.gz ]]
		then
			gunzip $f
			read2=$(echo $f | cut -f1-2 -d.)
			name2=$(echo $f | cut -f1 -d.)
			bwa aln $REFERENCE $read2 > $name2.sai

		elif [[ $f == *_1.fastq ]]
		then
			name1=$(echo $f | cut -f1 -d.)
                        bwa aln $REFERENCE $f > $name1.sai

		elif [[ $f == *_2.fastq ]]
		then
			name2=$(echo $f | cut -f1 -d.)
                        bwa aln $REFERENCE $f > $name2.sai
	fi
done

# Write another loop that will take the .sai files from the previous section, and map them
# to the reference strain generating .sam and .bam files.

for f in *
do
	if [[ $f == *fastq ]]
	then

# Parse out just the name of the strain (again, in the format $name_1.fastq).
		name=$(echo $f | cut -f1 -d_)

# Want to only perform the alignment once - as there are two .sai files, this has the
# potential to interate through twice. This if statement prevents the script from
# iterating through more than once on the same strain:
		if [[ ! -e $name.bam ]]
		then
			echo "checking for pairs - " $name

            		if [[ $(echo $f | cut -f1 -d. | cut -f2 -d_) == "1" ]]
            		then
            			name1=$f
            			echo "first paired read = " $name1

# Make sure that the names of paired files (.sai and .fastq) are the SAME and that the SAME strain files are being aligned
# to the reference (again using BWA):
                		for g in *
                		do
                			if [[ $(echo $g | cut -f2 -d_) == "2.fastq" ]] && [[ $(echo $g | cut -f1 -d_) == $name ]]
                    			then
                    				echo $f "and" $g "are a pair - performing alignment"
                        			bwa sampe $REFERENCE $name\_1.sai $name\_2.sai $f $g > $name.sam

# '-f 0x0002' tells samtools to only take correctly paired reads. '-F 4' tells samtools to only take reads that have mapped
# to the reference.
                        			samtools view -bS -f 0x0002 -F 4 $name.sam > $name.bam
                        			samtools sort $name.bam $name.sorted
                        			samtools index $name.sorted.bam

# Removes the original fastq files and the SAM file. These should be commented out if using the script for the first time or
# if the user prefers to keep all the data:
						rm $f
						rm $g
						rm $name.sam

                    			fi
                		done

# A second loop exactly the same as the first, except it takes in "read2" files:

			elif [[ $(echo $f | cut -f1 -d. | cut -f2 -d_) == "2" ]]
         		then
                    		name2=$f
                    		echo "second paired read = " $name2

                    		for g in *
                    		do
                    			if [[ $(echo $g | cut -f2 -d_) == "1.fastq" ]] && [[ $(echo $g | cut -f1 -d_) == $name ]]
                        		then
                        			echo $f "and" $g "are a pair"
                            			bwa sampe $REFERENCE $name\_1.sai $name\_2.sai $g $f > $name.sam
                            			samtools view -bS -f 0x0002 -F 4 $name.sam > $name.bam
                            			samtools sort $name.bam $name.bam.sorted
                            			samtools index $name.bam.sorted
						rm $f
						rm $g
						rm $name.sam
                        		fi
                		done
            		fi
		fi
	fi

done

# The above loop should have generated the BAM alignment files for all the reads against the reference.

# Getting rid of the .sai files to clean up the directory.

for f in *
do
        if [[ $f == *.sai ]]
        then
                echo "deleting " $f
                rm $f

# The next command then counts all the reads aligning the "exons" defined by the BEDMAP_REFERENCE file.
# This BEDMAP_REFERENCE file was generated using gff2bed (see README and bedops documentation).

# Taking the $name.sorted.bam files and converting them to .bed files, and then using bedmaps to count the
# reads overlapping the 'exon' regions (in this case, the borders of the invertible DNA switch for fimS).
# The results for each strain are saved as $name.result.bed.

	elif [[ $f == *.sorted.bam ]]
	then
		echo "converting " $f " to bed file"
		bam2bed < $f > $f.bed
		bedmap --echo --count $BEDMAP_REFERENCE $f.bed > $f.result.bed
		echo "results for "$f" have finished compiling"
	fi
done

# Loop to parse all of the $name.result.bed file contents into a single result.csv file with the strain name identifier.

echo "STRAIN,OFF_1,OFF_2,ON_1,ON_2" > fimS_OFF_ON_bed_results.csv

for f in *
do
	if [[ $f == *.result.bed ]]
    	then
		NAME=$(echo $f | cut -f1 -d.)
		OFF_1=$(head -1 $f | cut -f2 -d\|)
        	OFF_2=$(head -2 $f | tail -1 | cut -f2 -d\|)
        	ON_1=$(tail -2 $f | head -1 | cut -f2 -d\|)
        	ON_2=$(tail -1 $f | cut -f2 -d\|)

    		echo $NAME','$OFF_1','$OFF_2','$ON_1','$ON_2 >> fimS_OFF_ON_bed_results.csv

    	fi
done

echo "finished creating csv file containing bedmaps results"

# The next section counts the number of read-pairs that overlap the 3 regions for each orientation, namely Left Flank,
# Switch Region and Right Flank.

for f in *
do
	if [[ $f == *.sorted.bam ]]
	then

# Reads the sorted bam file and obtains all of the read IDs.
# These are then sorted, where duplicate read names are discarded.This ultimately results in a unique list of
# read names that have mapped to the reference.

		samtools view $f | cut -f1 -d$'\t' > readnames
		sort readnames | uniq >> readnames.sorted
		echo "assigning reads to..."$f

# The below variables are used to count the reads overlapping the regions of interest (i.e. OFF_1, OFF_2, ON_1, ON_2).
# The other remaining variables can be used to validate the script and check that it is running accordingly.

		readcount=$(wc -l readnames.sorted)

		OFF_1=0
		OFF_2=0
		ON_1=0
		ON_2=0
		DIFF_REGION=0
		SAME_REGION=0

# The next loop reads in the 'readnames.sorted' file containing all of the read IDs.

		while read name
		do

# For each read that is fed through the while loop, its coordinate (as well as the coordinate of its pair) are
# obtained from the .sorted.bam file using 'grep' and cutting the fourth field (which is the coordinate field).
# This is temporarily saved in a 'positions.txt' file. Variable 'a' is then assigned one of these numbers,
# while variable 'b' is assigned the other. The script then determines what region the read lies in based on its
# starting coordinate, either 'left_flank', 'switch_region' or 'right_flank' for either orientation.

			samtools view $f | grep $name | cut -f4 -d$'\t' > position.txt

			a=$(head -1 position.txt)

			if [ $a -le 1000 ]
			then
				read1='OFF_left_flank'

			elif [ $a -ge 1001 -a $a -le 1313 ]
			then
				read1='OFF_switch_region'

			elif [ $a -ge 1314 -a $a -le 2313 ]
			then
				read1='OFF_right_flank'

			elif [ $a -ge 2314 -a $a -le 3313 ]
			then
				read1='ON_left_flank'

			elif [ $a -ge 3314 -a $a -le 3626 ]
			then
				read1='ON_switch_region'

			elif [ $a -ge 3627 ]
			then
				read1='ON_right_flank'
			fi

			b=$(tail -1 position.txt)

			if [ $b -le 999 ]
			then
				read2='OFF_left_flank'

			elif [ $b -ge 1000 -a $b -le 1313 ]
			then
				read2='OFF_switch_region'

			elif [ $b -ge 1314 -a $b -le 2313 ]
			then
				read2='OFF_right_flank'

			elif [ $b -ge 2314 -a $b -le 3313 ]
                        then
                                read2='ON_left_flank'

			elif [ $b -ge 3314 -a $b -le 3626 ]
                        then
                                read2='ON_switch_region'

			elif [ $b -ge 3627 ]
                        then
                                read2='ON_right_flank'
			fi

# At this point the script counts the number of reads overlapping different region.
# Since some of the reads can be in the same region, the first if statement filters only for
# reads that are in different regions.
# The next if statements encompass all of the possibilites for the read positions. Depending on
# where the reads lie, a '+1' is added to the tally for either OFF_1 (i.e. read pairs in the left flank and
# in the switch region), or OFF_2 (i.e. read pairs in the right flank and in the switch region), and again
# similarly for the ON region.

			if [[ $read1 != $read2  ]]
			then
				DIFF_REGION=$(($DIFF_REGION + 1))

				if [[ $read1 == 'OFF_switch_region' ]] && [[ $read2 == 'OFF_right_flank' ]]
				then
					OFF_2=$(($OFF_2 + 1))
					echo $name >> mapped_reads_OFF.txt

				elif [[ $read1 == 'OFF_switch_region' ]] && [[ $read2 == 'OFF_left_flank' ]]
				then
					OFF_1=$(($OFF_1 + 1))
					echo $name >> mapped_reads_OFF.txt

				elif [[ $read2 == 'OFF_switch_region' ]] && [[ $read1 == 'OFF_right_flank' ]]
				then
					OFF_2=$(($OFF_2 + 1))
					echo $name >> mapped_reads_OFF.txt

				elif [[ $read2 == 'OFF_switch_region' ]] && [[ $read1 == 'OFF_left_flank' ]]
				then
					OFF_1=$(($OFF_1 + 1))
					echo $name >> mapped_reads_OFF.txt

				elif [[ $read1 == 'ON_switch_region' ]] && [[ $read2 == 'ON_right_flank' ]]
                                then
                                        ON_2=$(($ON_2 + 1))
                                        echo $name >> mapped_reads_ON.txt

				elif [[ $read1 == 'ON_switch_region' ]] && [[ $read2 == 'ON_left_flank' ]]
                                then
                                        ON_1=$(($ON_1 + 1))
                                        echo $name >> mapped_reads_ON.txt

				elif [[ $read2 == 'ON_switch_region' ]] && [[ $read1 == 'ON_right_flank' ]]
                                then
                                        ON_2=$(($ON_2 + 1))
                                        echo $name >> mapped_reads_ON.txt

				elif [[ $read2 == 'ON_switch_region' ]] && [[ $read1 == 'ON_left_flank' ]]
                                then
                                        ON_1=$(($ON_1 + 1))
                                        echo $name >> mapped_reads_ON.txt

				fi
			else
				SAME_REGION=$(($SAME_REGION + 1))
			fi

		done <readnames.sorted

### Printing out results (optional)

#	echo 'OFF_1 = ' $OFF_1
#       echo 'OFF_2 = ' $OFF_2
#	echo 'ON_1 = ' $ON_1
#	echo 'ON_2 = ' $ON_2
#       echo 'read_count = ' $readcount
#       echo "reads in the same region = " $SAME_REGION
#       echo "reads in different regions = " $DIFF_REGION

# Creating a csv file for the results output:

	rm readnames.sorted

	NAME=$(echo $f | cut -f1 -d.)
	echo $NAME','$OFF_1','$OFF_2','$ON_1','$ON_2 >> fimS_OFF_ON_positions.csv

	fi
done

# Cleaning up unnecessary files:

rm readnames
rm position.txt

# Move all of the files into directories of their own

for f in *
do
	if [[ $f == *.result.bed ]]
	then
		NAME=$(echo $f | cut -f1 -d.)
		mkdir ana_$NAME
		mv $NAME* ana_$NAME
	fi
done

echo "Finished!"

error: pathspec 'Scripts/bwa_bedops_paired_align_non-interleaved_general.sh' did not match any file(s) known to git


add tmux.centos.all.sh

add tmux.centos.all.sh

yum update -y
yum install -y automake gcc ncurses-devel git
mkdir bzizi.com
cd bzizi.com
wget http://iweb.dl.sourceforge.net/project/levent/release-2.0.22-stable/libevent-2.0.22-stable.tar.gz

tar -zxvf libevent-2.0.22-stable.tar.gz
cd libevent-2.0.22-stable
./configure && make
make install

cd ..

git clone https://github.com/tmux/tmux.git
cd tmux
sh autogen.sh
./configure && make
make install

ln -s /usr/local/lib/libevent-2.0.so.5 /usr/lib/libevent-2.0.so.5     # 32位系统
ln -s /usr/local/lib/libevent-2.0.so.5 /usr/lib64/libevent-2.0.so.5   # 64位系统
clear

echo tmux successful installation,enjoy it
echo Welcome to www.bzizi.com -- haozibi

error: pathspec 'tmux.centos.all.sh' did not match any file(s) known to git


fixed upstream

fixed upstream

remote: Support for password authentication was removed on August 13, 2021.
remote: Please see https://docs.github.com/en/get-started/getting-started-with-git/about-remote-repositories#cloning-with-https-urls for information on currently recommended modes of authentication.
fatal: Authentication failed for 'https://github.com/ReadingCorner/ReadingCorner.git/'


build-nssov: do not use `uname -m`

build-nssov: do not use `uname -m`

remote: Support for password authentication was removed on August 13, 2021.
remote: Please see https://docs.github.com/en/get-started/getting-started-with-git/about-remote-repositories#cloning-with-https-urls for information on currently recommended modes of authentication.
fatal: Authentication failed for 'https://github.com/grawity/cluenet.git/'


Install ripgrep; fixes #37

Install ripgrep; fixes #37

#!/usr/bin/env bash

source lib.sh

DEB_FILE_URL="https://github.com/BurntSushi/ripgrep/releases/download/11.0.1/ripgrep_11.0.1_amd64.deb"
DEST="/tmp/$(basename ${DEB_FILE_URL})"

dpkg -l ripgrep > /dev/null
if [ $? -eq 0 ]; then
    info "ripgrep already installed"
    exit 0
fi

curl --silent --show-error -o "${DEST}" -L "${DEB_FILE_URL}"
sudo dpkg -i "${DEST}"

error: pathspec 'ripgrep/install.sh' did not match any file(s) known to git


Add developer

Add developer

#!/usr/bin/env bash
set -euo pipefail

cd "$HOME"

echo "Bootstrap"
curl -L https://raw.githubusercontent.com/andrewtchin/ansible-common/master/ubuntu-bootstrap.sh | sh

echo "Install devel Ansible"
git clone https://github.com/ansible/ansible.git --recursive
source ./ansible/hacking/env-setup

echo "Clone ansible-ubuntu"
ANSIBLE_UBUNTU_DIR="$HOME/ansible-ubuntu"
if [ -d "$ANSIBLE_UBUNTU_DIR" ]; then
  rm -rf "$ANSIBLE_UBUNTU_DIR"
fi
git clone https://github.com/andrewtchin/ansible-ubuntu.git "$ANSIBLE_UBUNTU_DIR"

cd "$ANSIBLE_UBUNTU_DIR"
echo "Run ansible-ubuntu"
ansible-playbook -vvv playbooks/ubuntu-desktop.yml --ask-become-pass -c local --extra-vars=@vars/ubuntu.json -i inventory

echo "Running developer playbook"
ansible-playbook -vvv playbooks/developer.yml --ask-become-pass -c local --extra-vars=@vars/ubuntu.json -i inventory

echo "Install script exiting"

error: pathspec 'ubuntu-developer.sh' did not match any file(s) known to git


add facebook streaming example

add facebook streaming example

#!/bin/sh

# Go to Facebook.com, and start a new "Live Video" post
# At the top of the video window, select "Connect" instead of "Camera"
# Enter the API key below to start the stream
FACEBOOKURL="rtmp://live-api-a.facebook.com:80/rtmp/"
STREAMKEY="<STREAMKEY>"

ffmpeg -y -nostdin \
	-thread_queue_size 512 \
	-timeout 3000000 \
	-i tcp://localhost:11000 \
	-t 5400 \
	-strict -2 \
	-c:a aac -ac 1 -ar 44100 -b:a 128k \
	-c:v libx264 \
	-preset superfast \
	-pix_fmt yuv420p \
	-r 30 \
	-g 60 \
	-vb 2048k -minrate 2000k -maxrate 4000k \
	-bufsize 4096k -threads 2  \
	-f flv 	"$FACEBOOKURL$STREAMKEY"

error: pathspec 'example-scripts/ffmpeg/stream-facebook.sh' did not match any file(s) known to git


handbrakecli to brew

handbrakecli to brew

remote: Support for password authentication was removed on August 13, 2021.
remote: Please see https://docs.github.com/en/get-started/getting-started-with-git/about-remote-repositories#cloning-with-https-urls for information on currently recommended modes of authentication.
fatal: Authentication failed for 'https://github.com/vitorgalvao/dotfiles.git/'


apps: remove unused casks

apps: remove unused casks


Setup and install script for fedora.

Setup and install script for fedora.

#!/bin/bash

if [ ! $USER = 'root' ]; then
    echo "script must be run as root"
    exit
fi

#configuration options for your environment

DEV_USER=notroot
DOMAIN=mywebiva.com
PASSENGER_TMP_DIR=/var/www/tmp
WEBIVA_BASE_DIR=/var/opt/$DEV_USER
WEBIVA_DIR=/var/opt/$DEV_USER/Webiva
RAILS_ENV=development

if ( ! grep $DEV_USER /etc/passwd >& /dev/null ); then
    echo "$DEV_USER was not found in /etc/passwd"
    echo "update the DEV_USER variable above with the correct user"
    exit
fi

yum -y install mysql-server mysql rubygem-rails ruby-mysql ruby-rdoc ruby-devel ruby-imagesize rubygem-tlsmail libxslt libxslt-devel gcc ImageMagick-devel gcc-c++ apr-devel httpd-devel git mysql-devel memcached

gem install passenger starling fastthread daemons

passenger-install-apache2-module

cd /usr/local/lib
ln -s /usr/lib64/mysql

mkdir $PASSENGER_TMP_DIR
chmod 777 $PASSENGER_TMP_DIR

mkdir $WEBIVA_BASE_DIR
chown $DEV_USER:$DEV_USER $WEBIVA_BASE_DIR
chmod 711 $WEBIVA_BASE_DIR

PASSENGER_CONFIG_DIR=`passenger-config --root`

# Apache Configs

# setup passenger
echo "LoadModule passenger_module $PASSENGER_CONFIG_DIR/ext/apache2/mod_passenger.so
PassengerRoot $PASSENGER_CONFIG_DIR
PassengerRuby /usr/bin/ruby
PassengerTempDir $PASSENGER_TMP_DIR" > /etc/httpd/conf.d/mod_passenger.conf

# setup Webiva

echo "
<VirtualHost *:80>
   ServerName $DOMAIN
   ServerAlias www.$DOMAIN
   DocumentRoot $WEBIVA_DIR/public

   # Optional - set site to run as a specific user
   PassengerDefaultUser $DEV_USER

   # Optional - set to production or development as necessary
   RailsEnv $RAILS_ENV

   # We don't want any user uploaded scripts to be executed in the public directory
   # This should be the public/system directory of your webiva install
   <Directory "$WEBIVA_DIR/public/system">
         Options FollowSymLinks
         AllowOverride Limit
         Order allow,deny
         Allow from all
         <IfModule mod_php5.c>
          php_admin_flag engine off
        </IfModule>
        AddType text/plain .html .htm .shtml .php .php3 .phtml .phtm .pl
   </Directory>
</VirtualHost>
" > /etc/httpd/conf.d/webiva.conf

mv /etc/httpd/conf.d/welcome.conf /etc/httpd/conf.d/welcome.conf.off

# for SeLinux

if ( selinuxenabled ); then
    ANSWER=n
    echo ""
    echo ""
    echo "Setup SeLinux for Webiva (y/n):"
    read -e ANSWER
    if [ $ANSWER = 'y' ]; then
	chcon -R -h -t httpd_sys_content_t $PASSENGER_CONFIG_DIR
	chcon -R -h -t httpd_sys_content_rw_t $PASSENGER_TMP_DIR

	cd /etc/selinux

	echo "
module local 1.0;

require {
        type user_devpts_t;
        type user_home_dir_t;
        type user_home_t;
        type unconfined_t;
        type httpd_tmp_t;
        type httpd_sys_script_t;
        type memcache_port_t;
        type root_t;
        type port_t;
        type httpd_sys_content_rw_t;
        type public_content_rw_t;
        type var_t;
        type http_port_t;
        type httpd_t;
        class process getsched;
        class unix_stream_socket { read write shutdown };
        class chr_file { read write ioctl };
        class capability { setuid dac_read_search chown fsetid setgid fowner dac_override };
        class tcp_socket name_connect;
        class fifo_file { getattr setattr read create unlink };
        class sock_file { write create unlink getattr setattr };
        class dir { write search read create open getattr add_name };
        class file { write read create open getattr setattr };
}

#============= httpd_sys_script_t ==============
allow httpd_sys_script_t http_port_t:tcp_socket name_connect;
allow httpd_sys_script_t memcache_port_t:tcp_socket name_connect;
allow httpd_sys_script_t port_t:tcp_socket name_connect;
allow httpd_sys_script_t root_t:dir { write create add_name };
allow httpd_sys_script_t self:capability { setuid dac_read_search chown fsetid setgid fowner dac_override };
allow httpd_sys_script_t self:process getsched;
allow httpd_sys_script_t var_t:file { write read create open getattr setattr };
allow httpd_sys_script_t user_devpts_t:chr_file { read write ioctl };
allow httpd_sys_script_t user_home_dir_t:dir { write search read create open getattr add_name };
allow httpd_sys_script_t user_home_t:dir { write search read create open getattr add_name };

#============= httpd_t ==============
allow httpd_t httpd_sys_script_t:unix_stream_socket { read write shutdown };
allow httpd_t httpd_sys_content_rw_t:fifo_file { getattr setattr read create unlink };
allow httpd_t httpd_sys_content_rw_t:sock_file { write create unlink getattr setattr };
allow httpd_t httpd_tmp_t:fifo_file { getattr setattr read create unlink };
allow httpd_t self:capability { setuid dac_read_search chown fsetid setgid fowner dac_override };
" > local.te

	checkmodule -M -m -o local.mod local.te
	semodule_package -o local.pp -m local.mod
	semodule -i local.pp
    fi
fi

/sbin/service mysqld start
/sbin/service memcached start

cd $WEBIVA_BASE_DIR

sudo -u $DEV_USER git clone git://github.com/cykod/Webiva.git

cd $WEBIVA_DIR

sudo -u $DEV_USER -s script/quick_install.rb

if ( selinuxenabled ); then
    if [ $ANSWER = 'y' ]; then
	chcon -R -h -t httpd_sys_content_t $WEBIVA_BASE_DIR
    fi
fi

/sbin/service httpd start

ANSWER=n
echo ""
echo ""
echo "Automatically start mysqld, memcached and httpd on boot (y/n):"
read -e ANSWER
if [ $ANSWER = 'y' ]; then
    chkconfig mysqld on
    chkconfig memcached on
    chkconfig httpd on
fi

if [ $RAILS_ENV = 'development' ]; then
    cd /home/$DEV_USER
    echo "127.0.0.1   $DOMAIN www.$DOMAIN" >> /etc/hosts
    sudo -u $DEV_USER ln -s $WEBIVA_DIR
fi

error: pathspec 'doc/install_fedora_packages.sh' did not match any file(s) known to git


Added install script for Windows

Added install script for Windows

#!/bin/bash

pip install --install-option="--without-c-extensions" -r requirements-dev.txt

error: pathspec 'util/install_win.sh' did not match any file(s) known to git


Add docker-compose backup script

Add docker-compose backup script

#!/bin/bash
function backup_compose_instances {(
    if [[ "$EUID" -ne 0 ]]; then
        echo "Please run as root";
        exit;
    fi

    compose_folder="/srv/compose/"
    #backup_destination="backup_server:httpdocs/compose-backups"
    backup_destination=""
    backup_folder="/srv/backups"
    if [[ ! -d "$backup_folder" ]]; then
        mkdir "$backup_folder"
    fi

    function make_instance_backup {
        instance="${1%%/}"
        instance="${instance##$compose_folder}"
        instance_backup_folder="$backup_folder/$instance"
        timestamp="$(date +%Y-%m-%d_%H-%M-%S)"
        filename="${instance##*/}"'_'"${timestamp}"'.tar.gz'

        echo -n "$2Processing ${instance}..."

        if [[ ! -d "$instance_backup_folder" ]]; then
            mkdir -p "$instance_backup_folder"
        fi

        tar -zcpf "$instance_backup_folder/$filename" --transform "s,^${compose_folder#/},," "$1" &>/dev/null

        echo " done. File size: $(du -csh "$instance_backup_folder/$filename" | tail -n 1 | sed 's/[ ]*total//g')"
    }

    function recurse_instances {
        for folder in "${1%/}"/*; do
            if [[ -d "$folder" ]]; then
                if [[ "${folder##*\.}" == "d" ]]; then
                    echo "${2}Processing nested folder: $folder"
                    recurse_instances "$folder" "$2    "
                    echo "$2    Nested folder “$folder” size: $(du -csh "$folder" | tail -n 1 | sed 's/[ ]*total//g')"
                else
                    if [[ ! -f "$folder/.nobackup" ]]; then
                        make_instance_backup "$folder" "$2"
                    fi
                fi
            fi
        done
    }

    echo "Begining backup process."
    echo "Folder to backup: $compose_folder"
    echo "Saving destination: $backup_folder"

    recurse_instances "$compose_folder"
    echo "Total folder size: $(du -csh "$backup_folder" | tail -n 1 | sed 's/[ ]*total//g')"

    if [[ "$backup_folder" != "" ]]; then
        echo "Cleaning files older than 3 days..."
        find "${backup_folder}"/* -type f -mtime +2 -print | xargs rm -rf
        echo "Total folder size after cleaning: $(du -csh "$backup_folder" | tail -n 1 | sed 's/[ ]*total//g')"
    fi

    if [[ "$backup_destination" != "" ]]; then
        echo "Syncing with backup server..."
        rsync -e "ssh -i /srv/.ssh/backup_server -l backups" --progress --delete -azh "$backup_folder" $backup_destination
    fi

    echo "Done."
)}

error: pathspec 'functions/backup_compose_instances.sh' did not match any file(s) known to git


force push

force push

remote: Support for password authentication was removed on August 13, 2021.
remote: Please see https://docs.github.com/en/get-started/getting-started-with-git/about-remote-repositories#cloning-with-https-urls for information on currently recommended modes of authentication.
fatal: Authentication failed for 'https://github.com/AppGyver/supersonic.git/'


document release steps in a bash file.

document release steps in a bash file.

# This file is intended to document release steps.
# Verify each step's result before calling the next command.
# It is documented here with the intention of being automated
# as a shell script later.

echo 'git clean -xdf'
echo 'python setup.py bdist_wheel --universal'
echo 'twine upload dist/* -r https://upload.pypi.org/legacy/ -u kubernetes'

error: pathspec 'scripts/release.sh' did not match any file(s) known to git


add scripts/ directory and release script

#! /bin/sh
#
# release.sh --- Create a signed tarball release for ASDF-INSTALL.
#
# Copyright (C) 2005, James Bielman  <jamesjb@jamesjb.com>
#
# Permission is hereby granted, free of charge, to any person
# obtaining a copy of this software and associated documentation
# files (the "Software"), to deal in the Software without
# restriction, including without limitation the rights to use, copy,
# modify, merge, publish, distribute, sublicense, and/or sell copies
# of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be
# included in all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
# NONINFRINGEMENT.  IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
# DEALINGS IN THE SOFTWARE.
#

SNAPSHOT_DATE=`date +"%Y%m%d"`
TARBALL_NAME="cffi_$SNAPSHOT_DATE"
TARBALL="$TARBALL_NAME.tar.gz"
SIGNATURE="$TARBALL.asc"
RELEASE_DIR="/project/cffi/public_html/snapshots"

echo "Creating distribution..."
darcs dist -d "$TARBALL_NAME"

echo "Signing tarball..."
gpg -b -a "$TARBALL_NAME.tar.gz"

echo "Copying tarball to web server..."
scp "$TARBALL" "$SIGNATURE" common-lisp.net:"$RELEASE_DIR"

echo "Uploaded $TARBALL and $SIGNATURE."
echo "Don't forget to update the link on the CLiki page!"


fix the NULL import problem for the lineage table

remote: Support for password authentication was removed on August 13, 2021.
remote: Please see https://docs.github.com/en/get-started/getting-started-with-git/about-remote-repositories#cloning-with-https-urls for information on currently recommended modes of authentication.
fatal: Authentication failed for 'https://github.com/bmesuere/uni-test.git/'


AMBARI-11159. Kerberos: Add kerberos-setup.sh script back to the resources/scripts directory with appropriate changes (alexantonenko)

#!/bin/bash
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.
#

#
# This script is provided as an example of how to parse the Kerberos CSV file.
# It is for illustrative purposes only and should not be used for any other purpose.
#

############################
## NOTE:
##      1) This script should be executed on NameNode host as that host is guaranteed to have all the users needed while creating keytab file
##      2) The script has been verified to work in gce environment and
##         vagrant environment documented at ambari wiki: https://cwiki.apache.org/confluence/display/AMBARI/Quick+Start+Guide
###########################

usage () {
echo "Usage: keytabs.sh <HOST_PRINCIPAL_KEYTABLE.csv> <SSH_LOGIN_KEY_PATH>";
echo "  <HOST_PRINCIPAL_KEYTABLE.csv>: CSV file generated by 'Enable Security Wizard' of Ambari";
echo "  <SSH_LOGIN_KEY_PATH>: File path to the ssh login key for root user";
exit 1;
}

###################
## processCSVFile()
###################
processCSVFile () {
    csvFile=$1;
    csvFile=$(printf '%q' "$csvFile")
    # Remove blank lines
    sed -i '/^\s*$/d' $csvFile
    touch generate_keytabs.sh;
    chmod 755 generate_keytabs.sh;

    echo "#!/bin/bash" > generate_keytabs.sh;
    echo "###########################################################################" >> generate_keytabs.sh;
    echo "###########################################################################" >> generate_keytabs.sh;
    echo "## " >> generate_keytabs.sh;
    echo "## Ambari Security Script Generator" >> generate_keytabs.sh;
    echo "## "  >> generate_keytabs.sh;
    echo "## Ambari security script is generated which should be run on the" >> generate_keytabs.sh;
    echo "## Kerberos server machine." >> generate_keytabs.sh;
    echo "## " >> generate_keytabs.sh;
    echo "## Running the generated script will create host specific keytabs folders." >> generate_keytabs.sh;
    echo "## Each of those folders will contain service specific keytab files with " >> generate_keytabs.sh;
    echo "## appropriate permissions. There folders should be copied as the appropriate" >> generate_keytabs.sh;
    echo "## host's '/etc/security/keytabs' folder" >> generate_keytabs.sh;
    echo "###########################################################################" >> generate_keytabs.sh;
    echo "###########################################################################" >> generate_keytabs.sh;

    rm -f commands.mkdir;
    rm -f commands.chmod;
    rm -f commands.addprinc;
    rm -f commands.xst
    rm -f commands.xst.cp
    rm -f commands.chown.1
    rm -f commands.chmod.1
    rm -f commands.chmod.2
    rm -f commands.tar

    seenHosts="";
    seenPrincipals="";

    echo "mkdir -p ./tmp_keytabs" >> commands.mkdir;
    sed 1d $csvFile | while read line; do
        hostName=`echo $line|cut -d , -f 1`;
        service=`echo $line|cut -d , -f 2`;
        principal=`echo $line|cut -d , -f 3`;
        localUserName=`echo $line|cut -d , -f 5`;
        keytabFile=`echo $line|cut -d , -f 6 | cut -d , -f 6 | rev | cut -d '/' -f 1 | rev`;
        fullKeytabFilePath=`echo $line|cut -d , -f 6`;
        keytabFilePath=${fullKeytabFilePath%/*};
        owner=`echo $line|cut -d , -f 7`;
        group=`echo $line|cut -d , -f 9`;
        acl=`echo $line|cut -d , -f 11`;

        if [[ $seenHosts != *$hostName* ]]; then
              echo "mkdir -p ./keytabs_$hostName" >> commands.mkdir;
              echo "chmod 755 ./keytabs_$hostName" >> commands.chmod;
              echo "chown -R root:$group `pwd`/keytabs_$hostName" >> commands.chown.1
              echo "tar -cvf keytabs_$hostName.tar -C keytabs_$hostName ." >> commands.tar
              seenHosts="$seenHosts$hostName";
        fi

        if [[ $seenPrincipals != *" $principal"* ]]; then
          echo -e "kadmin.local -q \"addprinc -randkey $principal\"" >> commands.addprinc;
          seenPrincipals="$seenPrincipals $principal"
        fi
        tmpKeytabFile="`pwd`/tmp_keytabs/$keytabFile";
        newKeytabPath="`pwd`/keytabs_$hostName$keytabFilePath";
        newKeytabFile="$newKeytabPath/$keytabFile";
        if [ ! -f $tmpKeytabFile ]; then
          echo "kadmin.local -q \"xst -k $tmpKeytabFile $principal\"" >> commands.xst;
        fi
        if [ ! -d $newKeytabPath ]; then
            echo "mkdir -p $newKeytabPath" >> commands.mkdir;
        fi
        echo "cp $tmpKeytabFile $newKeytabFile" >> commands.xst.cp
        echo "chmod $acl $newKeytabFile" >> commands.chmod.2
        echo "chown $owner:$group $newKeytabFile" >> commands.chown.1
    done;

    echo "" >> generate_keytabs.sh;
    echo "" >> generate_keytabs.sh;
    echo "###########################################################################" >> generate_keytabs.sh;
    echo "# Making host specific keytab folders" >> generate_keytabs.sh;
    echo "###########################################################################" >> generate_keytabs.sh;
    cat commands.mkdir >> generate_keytabs.sh;
    echo "" >> generate_keytabs.sh;
    echo "###########################################################################" >> generate_keytabs.sh;
    echo "# Changing permissions for host specific keytab folders" >> generate_keytabs.sh;
    echo "###########################################################################" >> generate_keytabs.sh;
    cat commands.chmod >> generate_keytabs.sh;
    echo "" >> generate_keytabs.sh;
    echo "###########################################################################" >> generate_keytabs.sh;
    echo "# Creating Kerberos Principals" >> generate_keytabs.sh;
    echo "###########################################################################" >> generate_keytabs.sh;
    cat commands.addprinc >> generate_keytabs.sh;
    echo "" >> generate_keytabs.sh;
    echo "###########################################################################" >> generate_keytabs.sh;
    echo "# Creating Kerberos Principal keytabs in host specific keytab folders" >> generate_keytabs.sh;
    echo "###########################################################################" >> generate_keytabs.sh;
    cat commands.xst >> generate_keytabs.sh;
    cat commands.xst.cp >> generate_keytabs.sh;
    echo "" >> generate_keytabs.sh;
    echo "###########################################################################" >> generate_keytabs.sh;
    echo "# Changing ownerships of host specific keytab files" >> generate_keytabs.sh;
    echo "###########################################################################" >> generate_keytabs.sh;
    cat commands.chown.1 >> generate_keytabs.sh;
    echo "" >> generate_keytabs.sh;
    echo "###########################################################################" >> generate_keytabs.sh;
    echo "# Changing access permissions of host specific keytab files" >> generate_keytabs.sh;
    echo "###########################################################################" >> generate_keytabs.sh;
    #cat commands.chmod.1
    cat commands.chmod.2 >> generate_keytabs.sh;
    echo "" >> generate_keytabs.sh;
    echo "###########################################################################" >> generate_keytabs.sh;
    echo "# Packaging keytab folders" >> generate_keytabs.sh;
    echo "###########################################################################" >> generate_keytabs.sh;
    cat commands.tar >> generate_keytabs.sh;
    echo "" >> generate_keytabs.sh;
    echo "###########################################################################" >> generate_keytabs.sh;
    echo "# Cleanup" >> generate_keytabs.sh;
    echo "###########################################################################" >> generate_keytabs.sh;
    echo "rm -rf ./tmp_keytabs" >> generate_keytabs.sh;
    echo "" >> generate_keytabs.sh;
    echo "echo \"****************************************************************************\"" >> generate_keytabs.sh;
    echo "echo \"****************************************************************************\"" >> generate_keytabs.sh;
    echo "echo \"** Copy and extract 'keytabs_[hostname].tar' files onto respective hosts. **\"" >> generate_keytabs.sh;
    echo "echo \"**                                                                        **\"" >> generate_keytabs.sh;
    echo "echo \"** Generated keytab files are preserved in the 'tmp_keytabs' folder.      **\"" >> generate_keytabs.sh;
    echo "echo \"****************************************************************************\"" >> generate_keytabs.sh;
    echo "echo \"****************************************************************************\"" >> generate_keytabs.sh;

    rm -f commands.mkdir >> generate_keytabs.sh;
    rm -f commands.chmod >> generate_keytabs.sh;
    rm -f commands.addprinc >> generate_keytabs.sh;
    rm -f commands.xst >> generate_keytabs.sh;
    rm -f commands.xst.cp >> generate_keytabs.sh;
    rm -f commands.chown.1 >> generate_keytabs.sh;
    rm -f commands.chmod.1 >> generate_keytabs.sh;
    rm -f commands.chmod.2 >> generate_keytabs.sh;
    rm -f commands.tar >> generate_keytabs.sh;
    # generate keytabs
    sh ./generate_keytabs.sh
}

########################
## installKDC () : Install rng tools,pdsh on KDC host and KDC packages on all host. Modify krb5 file
########################
installKDC () {
  csvFile=$1;
  sshLoginKey=$2;
  HOSTNAME=`hostname --fqdn`
  scriptDir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
  krb5_new_conf=$scriptDir"/krb5.conf"
  krb5_conf="/etc/krb5.conf"
  #export additional path for suse and centos5
  PATH=$PATH:/usr/lib/mit/sbin/:/usr/kerberos/sbin/:/usr/sbin/:/sbin/
  # Install rng tools
  installRngtools
  # Install kdc server on this host
  $inst_cmd $server_packages
  # Configure /etc/krb5.conf
  cp $krb5_conf $krb5_conf".bak"
  cp $krb5_new_conf $krb5_conf
  sed -i "s/\(kdc *= *\).*kerberos.example.com.*/\1$HOSTNAME/" $krb5_conf
  sed -i "s/\(admin_server *= *\).*kerberos.example.com.*/\1$HOSTNAME/" $krb5_conf
  # Create principal key and start services
  if [[ ! -f $principal_file ]]; then
    echo -ne '\n\n' | kdb5_util create -s
  fi
  eval $kdc_service_start
  eval $kadmin_service_start
  # Install pdsh on this host
  $inst_cmd pdsh;
  chown root:root -R /usr;
  eval `ssh-agent`
  ssh-add $sshLoginKey
  hostNames='';

  # remove empty lines
  sed -i "/^\s*$/d" $csvFile;

  sed 1d $csvFile > $csvFile.tmp
  while read line; do
    hostName=`echo $line|cut -d , -f 1`;
    if [ -z "$hostNames" ]; then
      hostNames=$hostName;
      continue;
    fi
    if [[ $hostNames != *$hostName* ]]; then
      hostNames=$hostNames,$hostName;
    fi
    echo "hostNames $hostNames";
  done < $csvFile.tmp;
  rm -f $csvFile.tmp
  # Check all hosts for passwordless ssh
  OLD_IFS=$IFS
  IFS=,
  for host in $hostNames; do

    checkSSH $host
  done
  IFS=$OLD_IFS
  export PDSH_SSH_ARGS_APPEND="-q -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o PreferredAuthentications=publickey"
  pdsh -R ssh -w $hostNames "$inst_cmd $client_packages"
  pdsh -R ssh -w $hostNames "$inst_cmd pdsh"
  pdsh -R ssh -w $hostNames chown root:root -R /usr
  pdcp -R ssh -w $hostNames $krb5_conf $krb5_conf
}

########################
## distributeKeytabs () : Distribute the tar on all respective hosts root directory and untar it
########################
distributeKeytabs () {
  shopt -s nullglob
  filearray=(keytabs_*tar)
  for i in ${filearray[@]}; do
    derivedname=${i%.*}
    derivedname=${derivedname##keytabs_}
    echo $derivedname
    scp -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no $i root@$derivedname:/
    ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no root@$derivedname "cd /;tar xvf $i --no-overwrite-dir"
  done
}

########################
## getEnvironmentCMD () : get linux distribution type and package manager
########################
getEnvironmentCMD () {
  os=`python -c 'import sys; sys.path.append("/usr/lib/python2.6/site-packages/"); from ambari_commons import OSCheck; print OSCheck.get_os_family()'`
  version=`python -c 'import sys; sys.path.append("/usr/lib/python2.6/site-packages/"); from ambari_commons import OSCheck; print OSCheck.get_os_major_version()'`
  os=$os$version;
  case $os in
  'ubuntu12' )
    pkgmgr='apt-get'
    inst_cmd="env DEBIAN_FRONTEND=noninteractive /usr/bin/$pkgmgr --allow-unauthenticated --assume-yes install -f "
    client_packages="krb5-user libpam-krb5 libpam-ccreds auth-client-config"
    server_packages="krb5-kdc krb5-admin-server $client_packages"
    rng_tools="rng-tools"
    principal_file="/etc/krb5kdc/principal"
    kdc_service_start="service krb5-kdc start || service krb5-kdc status"
    kadmin_service_start="service krb5-admin-server start || service krb5-admin-server status"
    ;;
  'redhat5' )
    pkgmgr='yum'
    inst_cmd="/usr/bin/$pkgmgr -y install "
    client_packages="krb5-workstation"
    server_packages="krb5-server krb5-libs krb5-auth-dialog $client_packages"
    rng_tools="rng-utils"
    principal_file="/var/kerberos/krb5kdc/principal"
    kdc_service_start="service kadmin start; service kadmin status"
    kadmin_service_start="service krb5kdc start || service krb5kdc status"
    ;;
  'redhat6' )
    pkgmgr='yum'
    inst_cmd="/usr/bin/$pkgmgr -y install "
    client_packages="krb5-workstation"
    server_packages="krb5-server krb5-libs krb5-auth-dialog $client_packages"
    rng_tools="rng-tools"
    principal_file="/var/kerberos/krb5kdc/principal"
    kdc_service_start="service kadmin start; service kadmin status"
    kadmin_service_start="service krb5kdc start || service krb5kdc status"
    ;;
  'suse11' )
    pkgmgr='zypper'
    inst_cmd="/usr/bin/$pkgmgr install --auto-agree-with-licenses --no-confirm "
    client_packages="krb5-client"
    server_packages="krb5 krb5-server $client_packages"
    rng_tools="rng-tools"
    principal_file="/var/lib/kerberos/krb5kdc/principal"
    kdc_service_start="service kadmind start || service kadmind status"
    kadmin_service_start="service krb5kdc start || service krb5kdc status"
    ;;
  esac
}

########################
## checkUser () : If the user executing the script is not "root" then exit
########################
checkUser () {
  userid=`id -u`;
  if (($userid != 0)); then
    echo "ERROR: The script needs to be executed by root user"
    exit 1;
  fi
}

########################
## checkSSH () : If passwordless ssh for root is not configured then exit
########################
checkSSH () {
  host=$1
  ssh -oPasswordAuthentication=no -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no $host "exit 0" && return_value=0 || return_value=$? && true
  if [[ $return_value != 0 ]]; then
    echo "ERROR: Passwordless ssh for user root is not configured for host $host"
    exit 1;
  fi
}

########################
## installRngtools () : Install and start rng-tools
########################
installRngtools () {
  $inst_cmd $rng_tools
  echo $inst_cmd $rng_utils
  if [ $os == 'ubuntu12' ] || [ $os == 'suse11' ]; then
    echo "HRNGDEVICE=/dev/urandom" >> /etc/default/rng-tools
    /etc/init.d/rng-tools start || true
  elif [ $os == 'redhat5' ]; then
    /sbin/rngd -r /dev/urandom -o /dev/random -f -t .001 --background
  else
    sed -i "s/\(EXTRAOPTIONS *= *\).*/\1\"-r \/dev\/urandom\"/" "/etc/sysconfig/rngd"
    # start rngd
    /etc/init.d/rngd start
  fi
}

if (($# != 2)); then
    usage
fi

set -e
checkUser
getEnvironmentCMD
installKDC $@
processCSVFile $@
distributeKeytabs $@

error: pathspec 'ambari-server/src/main/resources/scripts/kerberos-setup.sh' did not match any file(s) known to git


Put _powerline_prompt at the start

Put _powerline_prompt at the start

Reason: $? may be overridden by the previous commands in PROMPT_COMMAND that do
not keep it like _powerline_prompt does (see `return $last_exit_code`).
(@ascrane)

Closes #836

if test -z "${POWERLINE_COMMAND}" ; then
	if which powerline-client &>/dev/null ; then
		export POWERLINE_COMMAND=powerline-client
	elif which powerline &>/dev/null ; then
		export POWERLINE_COMMAND=powerline
	else
		# `$0` is set to `-bash` when using SSH so that won't work
		export POWERLINE_COMMAND="$(dirname "$BASH_SOURCE")/../../../scripts/powerline"
	fi
fi

_powerline_init_tmux_support() {
	# Note: `test -w ""` returns false, so first condition may be removed
	if test -n "$TMUX" && test -w "$TMUX" ; then
		# TMUX variable may be unset to create new tmux session inside this one
		_POWERLINE_TMUX="$TMUX"

		_powerline_tmux_setenv() {
			TMUX="$_POWERLINE_TMUX" tmux setenv -g TMUX_"$1"_`tmux display -p "#D" | tr -d %` "$2"
			TMUX="$_POWERLINE_TMUX" tmux refresh -S
		}

		_powerline_tmux_set_pwd() {
			if test "x$_POWERLINE_SAVED_PWD" != "x$PWD" ; then
				_POWERLINE_SAVED_PWD="$PWD"
				_powerline_tmux_setenv PWD "$PWD"
			fi
		}

		_powerline_tmux_set_columns() {
			_powerline_tmux_setenv COLUMNS "$COLUMNS"
		}

		trap "_powerline_tmux_set_columns" SIGWINCH
		_powerline_tmux_set_columns
	else
		_powerline_tmux_set_pwd() {
			return 0
		}
	fi
}

_powerline_prompt() {
	local last_exit_code=$?
	PS1="$($POWERLINE_COMMAND shell left -r bash_prompt --last_exit_code=$last_exit_code --jobnum="$(jobs -p|wc -l)")"
	_powerline_tmux_set_pwd
	return $last_exit_code
}

test "x$PROMPT_COMMAND" != "x${PROMPT_COMMAND%_powerline_prompt*}" ||
	export PROMPT_COMMAND=$'_powerline_prompt\n'"${PROMPT_COMMAND}"

_powerline_init_tmux_support

if test -z "${POWERLINE_COMMAND}" ; then
	if which powerline-client &>/dev/null ; then
		export POWERLINE_COMMAND=powerline-client
	elif which powerline &>/dev/null ; then
		export POWERLINE_COMMAND=powerline
	else
		# `$0` is set to `-bash` when using SSH so that won't work
		export POWERLINE_COMMAND="$(dirname "$BASH_SOURCE")/../../../scripts/powerline"
	fi
fi

_powerline_init_tmux_support() {
	# Note: `test -w ""` returns false, so first condition may be removed
	if test -n "$TMUX" && test -w "$TMUX" ; then
		# TMUX variable may be unset to create new tmux session inside this one
		_POWERLINE_TMUX="$TMUX"

		_powerline_tmux_setenv() {
			TMUX="$_POWERLINE_TMUX" tmux setenv -g TMUX_"$1"_`tmux display -p "#D" | tr -d %` "$2"
			TMUX="$_POWERLINE_TMUX" tmux refresh -S
		}

		_powerline_tmux_set_pwd() {
			if test "x$_POWERLINE_SAVED_PWD" != "x$PWD" ; then
				_POWERLINE_SAVED_PWD="$PWD"
				_powerline_tmux_setenv PWD "$PWD"
			fi
		}

		_powerline_tmux_set_columns() {
			_powerline_tmux_setenv COLUMNS "$COLUMNS"
		}

		trap "_powerline_tmux_set_columns" SIGWINCH
		_powerline_tmux_set_columns
	else
		_powerline_tmux_set_pwd() {
			return 0
		}
	fi
}

_powerline_prompt() {
	local last_exit_code=$?
	PS1="$($POWERLINE_COMMAND shell left -r bash_prompt --last_exit_code=$last_exit_code --jobnum="$(jobs -p|wc -l)")"
	_powerline_tmux_set_pwd
	return $last_exit_code
}

[[ "$PROMPT_COMMAND" != "${PROMPT_COMMAND/_powerline_prompt/}" ]] ||
	export PROMPT_COMMAND="${PROMPT_COMMAND}"$'\n'"_powerline_prompt"

_powerline_init_tmux_support


Initial version.

Initial version.

#!/bin/sh
#
# Copyright (c) 2014, Eugene M. Kim.  All rights reserved.
#

set -eu

unset progname progdir
progname="${0##*/}"
progdir="${0%/*}"

msg() {
	case $# in
	[1-9]*)
		echo "$progname: $*" >&2
		;;
	esac
}

: ${LOG_LEVEL=warning}
debug() { _debug DEBUG: "$@"; }
info() { _info INFO: "$@"; }
notice() { _notice NOTICE: "$@"; }
warning() { _warning WARNING: "$@"; }
err() { _err ERROR: "$@"; }
crit() { _crit CRITICAL: "$@"; }
alert() { _alert ALERT: "$@"; }
emerg() { _emerg EMERGENCY: "$@"; }
_debug() { case "$LOG_LEVEL" in debug) msg "$@";; *) : "$@";; esac; }
_info() { case "$LOG_LEVEL" in info) msg "$@";; *) _debug "$@";; esac; }
_notice() { case "$LOG_LEVEL" in notice) msg "$@";; *) _info "$@";; esac; }
_warning() { case "$LOG_LEVEL" in warning) msg "$@";; *) _notice "$@";; esac; }
_err() { case "$LOG_LEVEL" in err) msg "$@";; *) _warning "$@";; esac; }
_crit() { case "$LOG_LEVEL" in crit) msg "$@";; *) _err "$@";; esac; }
_alert() { case "$LOG_LEVEL" in alert) msg "$@";; *) _crit "$@";; esac; }
_emerg() { case "$LOG_LEVEL" in emerg) msg "$@";; *) _alert "$@";; esac; }
fatal() {
	"$@"
	echo "(exit 1);"
	exit 1
}

print_usage() {
	echo "usage: eval \`$progname\`"
}

usage() {
	err "$@"
	print_usage >&2
	exit 64 # EX_USAGE
}

main() {
	set_default_options
	process_options "$@"
	shift $(($OPTIND - 1))
	auto_shell_type
	case $# in
	[1-9]*)
		usage "extra arguments given"
		;;
	esac
	if $kill_agent
	then
		kill_agent
	else
		start_agent
	fi
}

start_agent() {
	while ! agent_env_valid
	do
		if read_agent_info
		then
			if agent_env_valid
			then
				break
			fi
			notice "agent info stale; deleting and restarting"
			delete_agent_info
		fi
		if ! make_agent_info
		then
			err "cannot create agent"
			return 69
		fi
	done
	emit_agent_env
	display_agent_info
}

kill_agent() {
	if ! agent_env_valid
	then
		err "there is no agent to kill"
		return 69
	fi
	if ! has_agent_pid
	then
		err "agent PID unknown, cannot kill agent ($SSH_AUTH_SOCK)"
		return 69
	fi
	notice "killing agent ($SSH_AUTH_SOCK) PID $SSH_AGENT_PID"
	kill "$SSH_AGENT_PID"
	case "$shell_type" in
	sh)
		echo "unset SSH_AUTH_SOCK; unset SSH_AGENT_PID;"
		;;
	csh)
		echo "unsetenv SSH_AUTH_SOCK; unsetenv SSH_AGENT_PID;"
		;;
	esac
}

set_default_options() {
	unset shell_type bind_address default_lifetime kill_agent
	shell_type=
	bind_address=
	default_lifetime=
	kill_agent=false
}

process_options() {
	local _opt
	while getopts :cska:t: _opt
	do
		case "$_opt" in
		':')
			usage "missing argument for -$OPTARG"
			;;
		'?')
			usage "unknown option -$OPTARG"
			;;
		*)
			process_one_option "-$_opt" "${OPTARG-}"
			;;
		esac
	done
}

process_one_option() {
	case "$1" in
	-c)
		shell_type=csh
		;;
	-s)
		shell_type=sh
		;;
	-k)
		kill_agent=true
		;;
	-a)
		bind_address="$2"
		;;
	-t)
		default_lifetime="$2"
		;;
	esac
}

auto_shell_type() {
	case "$shell_type" in
	?*)
		return
		;;
	esac
	case "$SHELL" in
	*csh)
		shell_type=csh
		;;
	*)
		shell_type=sh
		;;
	esac
}

agent_env_set() {
	case "${SSH_AUTH_SOCK+set}" in
	set)
		info "SSH_AUTH_SOCK set: $SSH_AUTH_SOCK"
		return 0
		;;
	*)
		info "SSH_AUTH_SOCK not set"
		return 1
		;;
	esac
}

has_agent_pid() {
	case "${SSH_AGENT_PID+set}" in
	set)
		info "SSH_AGENT_PID set: $SSH_AGENT_PID"
		return 0
		;;
	*)
		info "SSH_AGENT_PID not set"
		return 1
		;;
	esac
}

agent_env_valid() {
	local _code=
	ssh-add -l > /dev/null 2>&1 || _code=$?
	case "$_code" in
	'')
		info "SSH_AUTH_SOCK valid: agent is running with key(s)"
		return 0
		;;
	1)
		info "SSH_AUTH_SOCK valid: agent is running with no keys"
		return 0
		;;
	*)
		info "SSH_AUTH_SOCK invalid (ssh-add returned $_code)"
		return 1
		;;
	esac
}

agent_info_file="/tmp/$progname.`id -un`"

read_agent_info() {
	local _sock _pid
	if [ -e "$agent_info_file" -a ! -f "$agent_info_file" ]
	then
		fatal err "$agent_info_file exists but is not a file"
	fi
	if ! read -r _pid _sock < "$agent_info_file"
	then
		info "cannot read agent info from $agent_info_file"
		return 1
	fi
	SSH_AUTH_SOCK="$_sock"
	SSH_AGENT_PID="$_pid"
	export SSH_AUTH_SOCK SSH_AGENT_PID
}

delete_agent_info() {
	notice "deleting agent info file $agent_info_file"
	rm -f "$agent_info_file"
}

make_agent_info() {
	local _tmpfile
	if ! _tmpfile=$(mktemp "$agent_info_file.XXXXXX")
	then
		fatal err "cannot create a temporary agent info file"
	fi
	if ! build_agent_info_file "$_tmpfile"
	then
		err "cannot build a temporary agent info file"
		rm -f "$_tmpfile"
		exit 1
	fi
	if ! mv -n "$_tmpfile" "$agent_info_file"
	then
		err "cannot move temporary agent info file into place"
		rm -f "$_tmpfile"
		exit 1
	fi
	notice "new agent info file has created"
}

build_agent_info_file() {
	local _code
	_code=
	eval `run_ssh_agent || echo _code=$?` > /dev/null
	case "$_code" in
	?*)
		err "ssh-agent returned $_code"
		return $_code
		;;
	esac
	echo "$SSH_AGENT_PID $SSH_AUTH_SOCK" > "$1"
}

run_ssh_agent() {
	set -- -s
	case "$bind_address" in
	?*)
		set -- -a "$bind_address"
		;;
	esac
	case "$default_lifetime" in
	?*)
		set -- -t "$default_lifetime"
		;;
	esac
	ssh-agent "$@"
}

emit_agent_env() {
	case "$shell_type" in
	sh)
		echo "SSH_AUTH_SOCK='$SSH_AUTH_SOCK'; export SSH_AUTH_SOCK;"
		case "${SSH_AGENT_PID+set}" in
		set)
			echo "SSH_AGENT_PID='$SSH_AGENT_PID'; export SSH_AGENT_PID;"
			;;
		esac
		;;
	csh)
		echo "setenv SSH_AUTH_SOCK '$SSH_AUTH_SOCK';"
		case "${SSH_AGENT_PID+set}" in
		set)
			echo "setenv SSH_AGENT_PID '$SSH_AGENT_PID';"
			;;
		esac
		;;
	esac
}

display_agent_info() {
	echo 'echo "Agent running at $SSH_AUTH_SOCK (PID $SSH_AGENT_PID)";'
}

main "$@"

error: pathspec 'setup-ssh-agent.sh' did not match any file(s) known to git


add portforward for prometheus

add portforward for prometheus

remote: Support for password authentication was removed on August 13, 2021.
remote: Please see https://docs.github.com/en/get-started/getting-started-with-git/about-remote-repositories#cloning-with-https-urls for information on currently recommended modes of authentication.
fatal: Authentication failed for 'https://github.com/rogierlommers/dotfiles.git/'


overlay for removal of optname

overlay for removal of optname

if [ "$CI_PULL_REQUEST" = "11521" ] || [ "$CI_BRANCH" = "no-optname" ]; then

    coqhammer_CI_REF=no-optname
    coqhammer_CI_GITURL=https://github.com/SkySkimmer/coqhammer

    equations_CI_REF=no-optname
    equations_CI_GITURL=https://github.com/SkySkimmer/Coq-Equations

    unicoq_CI_REF=no-optname
    unicoq_CI_GITURL=https://github.com/SkySkimmer/unicoq

    paramcoq_CI_REF=no-optname
    paramcoq_CI_GITURL=https://github.com/SkySkimmer/paramcoq

fi

error: pathspec 'dev/ci/user-overlays/11521-SkySkimmer-no-optname.sh' did not match any file(s) known to git


Add tmux startup file

Add tmux startup file

#!/bin/bash

SESSION_NAME="CDS"

if tmux has-session -t $SESSION_NAME 2> /dev/null; then
	tmux attach -t $SESSION_NAME
	exit
fi

# Open Vim
tmux new-session -d -s $SESSION_NAME -n dev
tmux send-keys -t $SESSION_NAME:dev "vim" C-m

# Open up a tab for git
tmux new-window -n git -t $SESSION_NAME
tmux send-keys -t $SESSION_NAME:git "git hista" C-m

tmux select-window -t $SESSION_NAME:0

tmux attach -t $SESSION_NAME

error: pathspec '.tmux' did not match any file(s) known to git


Build scripts for ppc64le/Linux.

Build scripts for ppc64le/Linux.

# ----------------------------------------------------------------------------
#
# Package	: LightGBM
# Version	: 2.1.0
# Source repo	: https://github.com/Microsoft/LightGBM
# Tested on	: ubuntu_16.04
# Script License: Apache License, Version 2 or later
# Maintainer	: Priya Seth <sethp@us.ibm.com>
#
# Disclaimer: This script has been tested in non-root mode on given
# ==========  platform using the mentioned version of the package.
#             It may not work as expected with newer versions of the
#             package and/or distribution. In such case, please
#             contact "Maintainer" of this script.
#
# ----------------------------------------------------------------------------

#!/bin/bash

sudo apt-get update -y

#Install OpenCL development environment
sudo apt-get install -y nvidia-opencl-icd-375 nvidia-opencl-dev opencl-headers

#Install necessary building tools and dependencies
sudo apt-get install -y git vim python-dev cmake build-essential libboost-dev libboost-system-dev libboost-filesystem-dev liblapack-dev

#Compile light GBM with GPU support
git clone --recursive https://github.com/Microsoft/LightGBM
cd LightGBM
mkdir build ; cd build

#Disable altivec support
export CFLAGS="-mno-altivec -mno-vsx"
export CXXFLAGS="-mno-altivec -mno-vsx"

cmake -DUSE_GPU=1 ..
make -j$(nproc)
cd ..

#Install Python Interface (optional)
sudo apt-get install -y python-pip python-numpy
sudo pip install --upgrade pip

sudo pip install setuptools numpy scipy -U
sudo pip install scikit-learn -U
cd python-package/
sudo python setup.py install --precompile
cd ..

#Execute automated tests
sudo pip install pytest nose pandas
pytest

error: pathspec 'lightGBM/lightGBM_ubuntu_16.04_gpu.sh' did not match any file(s) known to git


formatting is hard

formatting is hard

remote: Support for password authentication was removed on August 13, 2021.
remote: Please see https://docs.github.com/en/get-started/getting-started-with-git/about-remote-repositories#cloning-with-https-urls for information on currently recommended modes of authentication.
fatal: Authentication failed for 'https://github.com/warmchang/minikube.git/'


fix typo

fix typo

remote: Support for password authentication was removed on August 13, 2021.
remote: Please see https://docs.github.com/en/get-started/getting-started-with-git/about-remote-repositories#cloning-with-https-urls for information on currently recommended modes of authentication.
fatal: Authentication failed for 'https://github.com/Zonama/ZonamaDev.git/'


added /etc/profile.d/node-path.sh to have NODE_PATH setup correctly upon login.

#!/bin/sh
# shellcheck shell=dash
export NODE_PATH=/usr/local/lib/node_modules:/usr/lib/node_modules:${NODE_PATH}

error: pathspec 'buildroot-external/overlay/base/etc/profile.d/node-path.sh' did not match any file(s) known to git


update login process of shell

update login process of shell

remote: Support for password authentication was removed on August 13, 2021.
remote: Please see https://docs.github.com/en/get-started/getting-started-with-git/about-remote-repositories#cloning-with-https-urls for information on currently recommended modes of authentication.
fatal: Authentication failed for 'https://github.com/j4ve1in/dotfiles.git/'


Added more explanation

Added more explanation

remote: Support for password authentication was removed on August 13, 2021.
remote: Please see https://docs.github.com/en/get-started/getting-started-with-git/about-remote-repositories#cloning-with-https-urls for information on currently recommended modes of authentication.
fatal: Authentication failed for 'https://github.com/neocryses/dotfiles.git/'


Create c9.sh

Create c9.sh

#! /bin/sh

# https://c9.io/
# wonderful for testing.

error: pathspec 'c9.sh' did not match any file(s) known to git


A simple script for continuous integration

#!/usr/bin/env sh

# A simple continuous integration script. Will do a git clone of the specified
# directory, do a full build and run the unit test suite

VC_DIR=$1
EXTERNAL_TOOLS_DIR=$2
FLAGS=$3

# abort_on_error(action, command...)
function run_action {
    ACTION=$1
    shift
    echo "running ${ACTION}: $@"
    $@ | tee ${TARGET_DIR}/log_${ACTION}.txt

    if [ "$?" -ne "0" ]; then
        echo "${ACTION} failed with $?"
        exit 1
    fi
}

if [ ! -d "${VC_DIR}" ]; then
    echo "Cannot find vc dir '${VC_DIR}' in `pwd`"
    exit 2
fi

if [ ! -d "${EXTERNAL_TOOLS_DIR}" ]; then
    echo "Cannot find external tools dir '${EXTERNAL_TOOLS_DIR}' in `pwd`"
    exit 3
fi

echo "Starting build and test run"

TARGET_DIR_REL=ci_run_`date '+%Y-%m-%d_%H_%M_%S'`
mkdir -p ${TARGET_DIR_REL}
TARGET_DIR=`cd ${TARGET_DIR_REL}; pwd`
echo "Target dir is ${TARGET_DIR}"

run_action "git_clone" git clone ${VC_DIR} ${TARGET_DIR}/zomp_vc
cd ${TARGET_DIR}/zomp_vc

run_action "make_all" make ${FLAGS} all
run_action "make_test" make ${FLAGS} test

error: pathspec 'ci.sh' did not match any file(s) known to git


sap hana preparation taken from pub source document

#!/bin/bash
# Prep for SAP HANA ENV
# Base on http://help.sap.com/hana/red_hat_enterprise_linux_rhel_6_5_configuration_guide_for_sap_hana_en.pdf
# Log all the work

{
# Assuming we're on 6.6
echo "Downgrading redhat release to 6.5..."
sudo yum downgrade redhat-release -y

echo "Synchronizing clock..."
service ntpd stop
ntpdate servertime.service.softlayer.com
service ntpd start

echo "Checking if ntp is on... And turning it on"
chkconfig | grep ntpd
chkconfig ntpd on

echo "Setup ntpdate to adjust clock on reboot for larger drifts"
echo servertime.service.softlayer.com >> /etc/ntp/step-tickers
chkconfig ntpdate on

echo "Installing yum-version lock and security"
yum -y install yum-versionlock yum-security

echo "Checking versions..."
yum --showduplicates nss-softokn # <= 3.14.3-12.el6_5
yum --showduplicates nss-softokn-freebl # <= 3.14.3-12.el6_5
yum --showduplicates kernel # <= 2.6.32-431.29.2.el6
yum --showduplicates kernel-haders # <= 2.6.32-431.29.2.el6
yum --showduplicates kernel-firmware # <= 2.6.32-431.29.2.el6

echo "Creating versionlock file list..."
cat /etc/yum/pluginconf.d/versionlock.list <<EOT
kernel-2.6.32-431.29.2.el6.*
kernel-firmware-2.6.32-431.29.2.el6.*
kernel-headers-2.6.32-431.29.2.el6.*
nss-softokn-freebl-3.14.3-12.el6_5.*
nss-softokn-3.14.3-12.el6_5.*
EOT

echo "Performing security updates..."
yum -y --security update

echo "Installing base + xfsprogs..."
yum -y groupinstall base
yum -y install xfsprogs

echo "Updating kernel..."
yum -y update kernel kernel-firmware

echo "Cleaning up old kernels..."
package-cleanup --oldkernels --count=1 -y

echo "Done. Rebooting..."
reboot

} | tee $0.log

error: pathspec 'sap_hana_rhel_prepare.sh' did not match any file(s) known to git


Script for Linux V2

Script for Linux V2

Install and Run /LINUX/runGUIdockLinux-VER2.sh

fatal: Invalid path '/LINUX': No such file or directory


Use git_timed function from devstack in ggp

Gerrit git prep makes a bunch of git calls, sometimes they fail. We
don't like that - so lift the retry-git function from devstack and
re-use it.

Change-Id: I866374e14691dcdffc77e81be049b62e649750a4

#!/bin/bash -e

GERRIT_SITE=$1
GIT_ORIGIN=$2

# git can sometimes get itself infinitely stuck with transient network
# errors or other issues with the remote end.  This wraps git in a
# timeout/retry loop and is intended to watch over non-local git
# processes that might hang.  GIT_TIMEOUT, if set, is passed directly
# to timeout(1); otherwise the default value of 0 maintains the status
# quo of waiting forever.
# usage: git_timed <git-command>
function git_timed {
    local count=0
    local timeout=0

    if [[ -n "${GIT_TIMEOUT}" ]]; then
        timeout=${GIT_TIMEOUT}
    fi

    until timeout -s SIGINT ${timeout} git "$@"; do
        # 124 is timeout(1)'s special return code when it reached the
        # timeout; otherwise assume fatal failure
        if [[ $? -ne 124 ]]; then
            exitcode=$?
            echo $LINENO "git call failed: [git $@]"
            exit $exitcode
        fi

        count=$(($count + 1))
        echo "timeout ${count} for git call: [git $@]"
        if [ $count -eq 3 ]; then
            echo $LINENO "Maximum of 3 git retries reached"
            exit 1
        fi
        sleep 5
    done
}

if [ -z "$GERRIT_SITE" ]; then
    echo "The gerrit site name (eg 'https://review.openstack.org') must be the first argument."
    exit 1
fi

if [ -z "$ZUUL_URL" ]; then
    echo "The ZUUL_URL must be provided."
    exit 1
fi

if [ -z "$GIT_ORIGIN" ] || [ -n "$ZUUL_NEWREV" ]; then
    GIT_ORIGIN="$GERRIT_SITE/p"
    # git://git.openstack.org/
    # https://review.openstack.org/p
fi

if [ -z "$ZUUL_REF" ]; then
    if [ -n "$BRANCH" ]; then
        echo "No ZUUL_REF so using requested branch $BRANCH from origin."
        ZUUL_REF=$BRANCH
        # use the origin since zuul mergers have outdated branches
        ZUUL_URL=$GIT_ORIGIN
    else
        echo "Provide either ZUUL_REF or BRANCH in the calling enviromnent."
        exit 1
    fi
fi

if [ ! -z "$ZUUL_CHANGE" ]; then
    echo "Triggered by: $GERRIT_SITE/$ZUUL_CHANGE"
fi

set -x
if [[ ! -e .git ]]; then
    ls -a
    rm -fr .[^.]* *
    if [ -d /opt/git/$ZUUL_PROJECT/.git ]; then
        git_timed clone file:///opt/git/$ZUUL_PROJECT .
    else
        git_timed clone $GIT_ORIGIN/$ZUUL_PROJECT .
    fi
fi
git remote set-url origin $GIT_ORIGIN/$ZUUL_PROJECT

# attempt to work around bugs 925790 and 1229352
if ! git_timed remote update; then
    echo "The remote update failed, so garbage collecting before trying again."
    git gc
    git_timed remote update
fi

git reset --hard
if ! git clean -x -f -d -q ; then
    sleep 1
    git clean -x -f -d -q
fi

if echo "$ZUUL_REF" | grep -q ^refs/tags/; then
    git_timed fetch --tags $ZUUL_URL/$ZUUL_PROJECT
    git checkout $ZUUL_REF
    git reset --hard $ZUUL_REF
elif [ -z "$ZUUL_NEWREV" ]; then
    git_timed fetch $ZUUL_URL/$ZUUL_PROJECT $ZUUL_REF
    git checkout FETCH_HEAD
    git reset --hard FETCH_HEAD
else
    git checkout $ZUUL_NEWREV
    git reset --hard $ZUUL_NEWREV
fi

if ! git clean -x -f -d -q ; then
    sleep 1
    git clean -x -f -d -q
fi

if [ -f .gitmodules ]; then
    git_timed submodule init
    git_timed submodule sync
    git_timed submodule update --init
fi

#!/bin/bash -e

GERRIT_SITE=$1
GIT_ORIGIN=$2

if [ -z "$GERRIT_SITE" ]; then
    echo "The gerrit site name (eg 'https://review.openstack.org') must be the first argument."
    exit 1
fi

if [ -z "$ZUUL_URL" ]; then
    echo "The ZUUL_URL must be provided."
    exit 1
fi

if [ -z "$GIT_ORIGIN" ] || [ -n "$ZUUL_NEWREV" ]; then
    GIT_ORIGIN="$GERRIT_SITE/p"
    # git://git.openstack.org/
    # https://review.openstack.org/p
fi

if [ -z "$ZUUL_REF" ]; then
    if [ -n "$BRANCH" ]; then
        echo "No ZUUL_REF so using requested branch $BRANCH from origin."
        ZUUL_REF=$BRANCH
        # use the origin since zuul mergers have outdated branches
        ZUUL_URL=$GIT_ORIGIN
    else
        echo "Provide either ZUUL_REF or BRANCH in the calling enviromnent."
        exit 1
    fi
fi

if [ ! -z "$ZUUL_CHANGE" ]; then
    echo "Triggered by: $GERRIT_SITE/$ZUUL_CHANGE"
fi

set -x
if [[ ! -e .git ]]; then
    ls -a
    rm -fr .[^.]* *
    if [ -d /opt/git/$ZUUL_PROJECT/.git ]; then
        git clone file:///opt/git/$ZUUL_PROJECT .
    else
        git clone $GIT_ORIGIN/$ZUUL_PROJECT .
    fi
fi
git remote set-url origin $GIT_ORIGIN/$ZUUL_PROJECT

# attempt to work around bugs 925790 and 1229352
if ! git remote update; then
    echo "The remote update failed, so garbage collecting before trying again."
    git gc
    git remote update
fi

git reset --hard
if ! git clean -x -f -d -q ; then
    sleep 1
    git clean -x -f -d -q
fi

if echo "$ZUUL_REF" | grep -q ^refs/tags/; then
    git fetch --tags $ZUUL_URL/$ZUUL_PROJECT
    git checkout $ZUUL_REF
    git reset --hard $ZUUL_REF
elif [ -z "$ZUUL_NEWREV" ]; then
    git fetch $ZUUL_URL/$ZUUL_PROJECT $ZUUL_REF
    git checkout FETCH_HEAD
    git reset --hard FETCH_HEAD
else
    git checkout $ZUUL_NEWREV
    git reset --hard $ZUUL_NEWREV
fi

if ! git clean -x -f -d -q ; then
    sleep 1
    git clean -x -f -d -q
fi

if [ -f .gitmodules ]; then
    git submodule init
    git submodule sync
    git submodule update --init
fi


added v2

added v2

#!/bin/bash
#
# Create a rsconf dev box
#
set -euo pipefail

vagrant_rsconf_dev_main() {
    case $(basename "$PWD") in
        v3)
            vagrant_rsconf_dev_master
            ;;
        v2|v4|v5)
            vagrant_dev_barebones=1 \
                install_server=http://v3.radia.run:2916 \
                vagrant_rsconf_dev_worker
            ;;
        *)
            install_err 'Must be run from v2, v3, v4, or v5 dirs'
            ;;
   esac
}

vagrant_rsconf_dev_master() {
    install_repo_eval vagrant-centos7
    bivio_vagrant_ssh <<'EOF'
        bivio_pyenv_2
        set -euo pipefail
        sudo yum install -y nginx
        mkdir -p ~/src/radiasoft
        cd ~/src/radiasoft
        gcl download
        gcl containers
        gcl pykern
        cd pykern
        pip install -e .
        cd ..
        gcl rsconf
        cd rsconf
        pip install -e .
        mkdir -p rpm
        cd rpm
        curl -S -s -L -O https://depot.radiasoft.org/foss/bivio-perl-dev.rpm
        curl -S -s -L -O https://depot.radiasoft.org/foss/perl-Bivio-dev.rpm
        cd ..
        rsconf build
EOF
    local s=file:///home/vagrant/src/radiasoft/rsconf/run/srv
    install_server=$s vagrant_rsconf_dev_run || true
    vagrant reload
    install_server=$s vagrant_rsconf_dev_run
    # For building perl rpms (see build-perl-rpm.sh)
    bivio_vagrant_ssh sudo usermod -aG docker vagrant
}

vagrant_rsconf_dev_run() {
    install_server=$install_server bivio_vagrant_ssh sudo su - <<EOF
        set -euo pipefail
        export install_channel=dev install_server=$install_server
        curl "$install_server/index.html" | bash -s rsconf.sh "\$(hostname -f)" setup_dev
EOF
}

vagrant_rsconf_dev_worker() {
    install_repo_eval vagrant-centos7
    vagrant_rsconf_dev_run || true
    vagrant reload
    vagrant_rsconf_dev_run
}

vagrant_rsconf_dev_main ${install_extra_args[@]+"${install_extra_args[@]}"}

#!/bin/bash
#
# Create a rsconf dev box
#
set -euo pipefail

vagrant_rsconf_dev_main() {
    case $(basename "$PWD") in
        v3)
            vagrant_rsconf_dev_master
            ;;
        v4|v5)
            vagrant_dev_barebones=1 \
                install_server=http://v3.radia.run:2916 \
                vagrant_rsconf_dev_worker
            ;;
        *)
            install_err 'Must be run from v3, v4, or v5 dirs'
            ;;
   esac
}

vagrant_rsconf_dev_master() {
    install_repo_eval vagrant-centos7
    bivio_vagrant_ssh <<'EOF'
        bivio_pyenv_2
        set -euo pipefail
        sudo yum install -y nginx
        mkdir -p ~/src/radiasoft
        cd ~/src/radiasoft
        gcl download
        gcl containers
        gcl pykern
        cd pykern
        pip install -e .
        cd ..
        gcl rsconf
        cd rsconf
        pip install -e .
        mkdir -p rpm
        cd rpm
        curl -S -s -L -O https://depot.radiasoft.org/foss/bivio-perl-dev.rpm
        curl -S -s -L -O https://depot.radiasoft.org/foss/perl-Bivio-dev.rpm
        cd ..
        rsconf build
EOF
    local s=file:///home/vagrant/src/radiasoft/rsconf/run/srv
    install_server=$s vagrant_rsconf_dev_run || true
    vagrant reload
    install_server=$s vagrant_rsconf_dev_run
    # For building perl rpms (see build-perl-rpm.sh)
    bivio_vagrant_ssh sudo usermod -aG docker vagrant
}

vagrant_rsconf_dev_run() {
    install_server=$install_server bivio_vagrant_ssh sudo su - <<EOF
        set -euo pipefail
        export install_channel=dev install_server=$install_server
        curl "$install_server/index.html" | bash -s rsconf.sh "\$(hostname -f)" setup_dev
EOF
}

vagrant_rsconf_dev_worker() {
    install_repo_eval vagrant-centos7
    vagrant_rsconf_dev_run || true
    vagrant reload
    vagrant_rsconf_dev_run
}

vagrant_rsconf_dev_main ${install_extra_args[@]+"${install_extra_args[@]}"}


remove unused setup env vars

remove unused setup env vars

#!/usr/bin/env bash

# Setup env vars and folders for the ctl script
# This helps keep the ctl script as readable
# as possible

# Usage options:
# source /var/vcap/jobs/foobar/helpers/ctl_setup.sh JOB_NAME OUTPUT_LABEL
# source /var/vcap/jobs/foobar/helpers/ctl_setup.sh foobar
# source /var/vcap/jobs/foobar/helpers/ctl_setup.sh foobar foobar
# source /var/vcap/jobs/foobar/helpers/ctl_setup.sh foobar nginx

set -e # exit immediately if a simple command exits with a non-zero status
set -u # report the usage of uninitialized variables

JOB_NAME=$1
output_label=${2:-${JOB_NAME}}

export JOB_DIR=/var/vcap/jobs/$JOB_NAME
chmod 755 $JOB_DIR # to access file via symlink

# Load some bosh deployment properties into env vars
# Try to put all ERb into data/properties.sh.erb
# incl $NAME, $JOB_INDEX, $WEBAPP_DIR
source $JOB_DIR/data/properties.sh

source $JOB_DIR/helpers/ctl_utils.sh
redirect_output ${output_label}

export HOME=${HOME:-/home/vcap}

# Add all packages' /bin & /sbin into $PATH
for package_bin_dir in $(ls -d /var/vcap/packages/*/*bin)
do
  export PATH=${package_bin_dir}:$PATH
done

export LD_LIBRARY_PATH=${LD_LIBRARY_PATH:-''} # default to empty
for package_bin_dir in $(ls -d /var/vcap/packages/*/lib)
do
  export LD_LIBRARY_PATH=${package_bin_dir}:$LD_LIBRARY_PATH
done

# Setup log, run and tmp folders

export RUN_DIR=/var/vcap/sys/run/$JOB_NAME
export LOG_DIR=/var/vcap/sys/log/$JOB_NAME
export TMP_DIR=/var/vcap/sys/tmp/$JOB_NAME
export STORE_DIR=/var/vcap/store/$JOB_NAME
for dir in $RUN_DIR $LOG_DIR $TMP_DIR $STORE_DIR
do
  mkdir -p ${dir}
  chown vcap:vcap ${dir}
  chmod 775 ${dir}
done
export TMPDIR=$TMP_DIR

PIDFILE=$RUN_DIR/$output_label.pid

echo '$PATH' $PATH

#!/usr/bin/env bash

# Setup env vars and folders for the ctl script
# This helps keep the ctl script as readable
# as possible

# Usage options:
# source /var/vcap/jobs/foobar/helpers/ctl_setup.sh JOB_NAME OUTPUT_LABEL
# source /var/vcap/jobs/foobar/helpers/ctl_setup.sh foobar
# source /var/vcap/jobs/foobar/helpers/ctl_setup.sh foobar foobar
# source /var/vcap/jobs/foobar/helpers/ctl_setup.sh foobar nginx

set -e # exit immediately if a simple command exits with a non-zero status
set -u # report the usage of uninitialized variables

JOB_NAME=$1
output_label=${2:-${JOB_NAME}}

export JOB_DIR=/var/vcap/jobs/$JOB_NAME
chmod 755 $JOB_DIR # to access file via symlink

# Load some bosh deployment properties into env vars
# Try to put all ERb into data/properties.sh.erb
# incl $NAME, $JOB_INDEX, $WEBAPP_DIR
source $JOB_DIR/data/properties.sh

source $JOB_DIR/helpers/ctl_utils.sh
redirect_output ${output_label}

export HOME=${HOME:-/home/vcap}

# Add all packages' /bin & /sbin into $PATH
for package_bin_dir in $(ls -d /var/vcap/packages/*/*bin)
do
  export PATH=${package_bin_dir}:$PATH
done

export LD_LIBRARY_PATH=${LD_LIBRARY_PATH:-''} # default to empty
for package_bin_dir in $(ls -d /var/vcap/packages/*/lib)
do
  export LD_LIBRARY_PATH=${package_bin_dir}:$LD_LIBRARY_PATH
done

# Setup log, run and tmp folders

export RUN_DIR=/var/vcap/sys/run/$JOB_NAME
export LOG_DIR=/var/vcap/sys/log/$JOB_NAME
export TMP_DIR=/var/vcap/sys/tmp/$JOB_NAME
export STORE_DIR=/var/vcap/store/$JOB_NAME
for dir in $RUN_DIR $LOG_DIR $TMP_DIR $STORE_DIR
do
  mkdir -p ${dir}
  chown vcap:vcap ${dir}
  chmod 775 ${dir}
done
export TMPDIR=$TMP_DIR

export C_INCLUDE_PATH=/var/vcap/packages/mysqlclient/include/mysql:/var/vcap/packages/sqlite/include:/var/vcap/packages/libpq/include
export LIBRARY_PATH=/var/vcap/packages/mysqlclient/lib/mysql:/var/vcap/packages/sqlite/lib:/var/vcap/packages/libpq/lib

# consistent place for vendoring python libraries within package
if [[ -d ${WEBAPP_DIR:-/xxxx} ]]
then
  export PYTHONPATH=$WEBAPP_DIR/vendor/lib/python
fi

if [[ -d /var/vcap/packages/java7 ]]
then
  export JAVA_HOME="/var/vcap/packages/java7"
fi

# setup CLASSPATH for all jars/ folders within packages
export CLASSPATH=${CLASSPATH:-''} # default to empty
for java_jar in $(ls -d /var/vcap/packages/*/*/*.jar)
do
  export CLASSPATH=${java_jar}:$CLASSPATH
done

PIDFILE=$RUN_DIR/$output_label.pid

echo '$PATH' $PATH


jira #DTRA-437: adjusted finalize script as follows:
 * to update nvp's with spaces for equal signs
 * adjusted all cmdTemplate's to use bracketed commands
 * set the certs nvp values in the apache site definition for the DEP.

git-svn-id: 85c88240758832ecdac9e9f2f6d4112daa3a57f7@26779 cadd33a7-e05c-44d0-9ff7-823b10d47c77

remote: Support for password authentication was removed on August 13, 2021.
remote: Please see https://docs.github.com/en/get-started/getting-started-with-git/about-remote-repositories#cloning-with-https-urls for information on currently recommended modes of authentication.
fatal: Authentication failed for 'https://github.com/OrbisTechnologies/BioIDWidgets.git/'


[CI] Add overlay

[CI] Add overlay

overlay serapi https://github.com/proux01/coq-serapi coq_12324 12324

error: pathspec 'dev/ci/user-overlays/12324-proux01-enable-notation.sh' did not match any file(s) known to git


Add overlay

Add overlay

overlay coqhammer https://github.com/proux01/coqhammer coq_15884 15884

error: pathspec 'dev/ci/user-overlays/15884-proux01-num_not_no_elab.sh' did not match any file(s) known to git


Initial commit

Initial commit

# https://bioconda.github.io/
conda config --add channels conda-forge
conda config --add channels defaults
conda config --add channels r
conda config --add channels bioconda

error: pathspec 'bash/conda_config.sh' did not match any file(s) known to git


format fix

format fix

Signed-off-by: Xian Chaobo <dcd6fd1596156638bd4b273477ce808661e40b45@huawei.com>

#!/usr/bin/env bats

load helpers

function teardown() {
	swarm_manage_cleanup
	stop_docker
}

# FIXME
@test "docker attach" {
	skip
}

@test "docker build" {
      start_docker 3
      swarm_manage

      run docker_swarm images
      [ "$status" -eq 0 ]
      [ "${#lines[@]}" -eq 1 ]

      run docker_swarm build -t test $BATS_TEST_DIRNAME/testdata/build
      [ "$status" -eq 0 ]

      run docker_swarm images
      [ "$status" -eq 0 ]
      [ "${#lines[@]}" -eq 3 ]
}

# FIXME
@test "docker commit" {
	skip
}

# FIXME
@test "docker cp" {
	skip
}

@test "docker create" {
	start_docker 3
	swarm_manage

	# make sure no contaienr exist
	run docker_swarm ps -qa
	[ "${#lines[@]}" -eq 0 ]

	# create
	run docker_swarm create --name test_container busybox sleep 1000
	[ "$status" -eq 0 ]

	# verify, created container exists
	run docker_swarm ps -l
	[ "${#lines[@]}" -eq 2 ]
	[[ "${lines[1]}" ==  *"test_container"* ]]
}

# FIXME
@test "docker diff" {
	skip
}

# FIXME
@test "docker events" {
	skip
}

# FIXME
@test "docker exec" {
	skip
}

# FIXME
@test "docker export" {
	skip
}

# FIXME
@test "docker history" {
	skip
}

# FIXME
@test "docker images" {
	skip
}

# FIXME
@test "docker import" {
	skip
}

@test "docker info" {
	start_docker 3
	swarm_manage
	run docker_swarm info
	[ "$status" -eq 0 ]
	[[ "${lines[3]}" == *"Nodes: 3" ]]
}

# FIXME
@test "docker inspect" {
	skip
}

@test "docker kill" {
	start_docker 3
	swarm_manage
	# run
	run docker_swarm run -d --name test_container busybox sleep 1000
	[ "$status" -eq 0 ]

	# make sure container is up before killing
	run docker_swarm ps -l
	[ "${#lines[@]}" -eq 2 ]
	[[ "${lines[1]}" == *"test_container"* ]]
	[[ "${lines[1]}" == *"Up"* ]]

	# kill
	run docker_swarm kill test_container
	[ "$status" -eq 0 ]
	# verify
	run docker_swarm ps -l
	[ "${#lines[@]}" -eq 2 ]
	[[ "${lines[1]}" == *"test_container"* ]]
	[[ "${lines[1]}" == *"Exited"* ]]
}

# FIXME
@test "docker load" {
	skip
}

# FIXME
@test "docker login" {
	skip
}

# FIXME
@test "docker logout" {
	skip
}

# FIXME
@test "docker logs" {
	skip
}

# FIXME
@test "docker port" {
	skip
}

# FIXME
@test "docker pause" {
	skip
}

@test "docker ps -n" {
	start_docker 1
	swarm_manage
	run docker_swarm run -d busybox sleep 42
	run docker_swarm run -d busybox false
	run docker_swarm ps -n 3
	# Non-running containers should be included in ps -n
	[ "${#lines[@]}" -eq  3 ]

	run docker_swarm run -d busybox true
	run docker_swarm ps -n 3
	[ "${#lines[@]}" -eq  4 ]

	run docker_swarm run -d busybox true
	run docker_swarm ps -n 3
	[ "${#lines[@]}" -eq  4 ]
}

@test "docker ps -l" {
	start_docker 1
	swarm_manage
	run docker_swarm run -d busybox sleep 42
	sleep 1 #sleep so the 2 containers don't start at the same second
	run docker_swarm run -d busybox true
	run docker_swarm ps -l
	[ "${#lines[@]}" -eq  2 ]
	# Last container should be "true", even though it's stopped.
	[[ "${lines[1]}" == *"true"* ]]

	sleep 1 #sleep so the container doesn't start at the same second as 'busybox true'
	run docker_swarm run -d busybox false
	run docker_swarm ps -l
	[ "${#lines[@]}" -eq  2 ]
	[[ "${lines[1]}" == *"false"* ]]
}

# FIXME
@test "docker pull" {
	skip
}

# FIXME
@test "docker push" {
	skip
}

# FIXME
@test "docker rename" {
	skip
}

@test "docker restart" {
	start_docker 3
	swarm_manage
	# run
	run docker_swarm run -d --name test_container busybox sleep 1000
	[ "$status" -eq 0 ]

	# make sure container is up
	run docker_swarm ps -l
	[ "${#lines[@]}" -eq 2 ]
	[[ "${lines[1]}" == *"test_container"* ]]
	[[ "${lines[1]}" == *"Up"* ]]

	# restart
	run docker_swarm restart test_container
	[ "$status" -eq 0 ]
	# verify
	run docker_swarm ps -l
	[ "${#lines[@]}" -eq 2 ]
	[[ "${lines[1]}" == *"test_container"* ]]
	[[ "${lines[1]}" == *"Up"* ]]
}

# FIXME
@test "docker rm" {
	skip
}

# FIXME
@test "docker rmi" {
	skip
}

# FIXME
@test "docker run" {
	skip
}

# FIXME
@test "docker save" {
	skip
}

# FIXME
@test "docker search" {
	skip
}

@test "docker start" {
	start_docker 3
	swarm_manage
	# create
	run docker_swarm create --name test_container busybox sleep 1000
	[ "$status" -eq 0 ]

	# make sure created container exists
	# new created container has no status
	run docker_swarm ps -l
	[ "${#lines[@]}" -eq 2 ]
	[[ "${lines[1]}" ==  *"test_container"* ]]

	# start
	run docker_swarm start test_container
	[ "$status" -eq 0 ]
	run docker_swarm ps -l
	[ "${#lines[@]}" -eq 2 ]
	[[ "${lines[1]}" == *"test_container"* ]]
	[[ "${lines[1]}" ==  *"Up"* ]]
}

# FIXME
@test "docker stats" {
	skip
}

@test "docker stop" {
	start_docker 3
	swarm_manage
	# run
	run docker_swarm run -d --name test_container busybox sleep 500
	[ "$status" -eq 0 ]

	# make sure container is up before stop
	run docker_swarm ps -l
	[ "${#lines[@]}" -eq 2 ]
	[[ "${lines[1]}" == *"test_container"* ]]
	[[ "${lines[1]}" == *"Up"* ]]

	# stop
	run docker_swarm stop test_container
	[ "$status" -eq 0 ]

	# verify
	run docker_swarm ps -l
	[ "${#lines[@]}" -eq 2 ]
	[[ "${lines[1]}" == *"test_container"* ]]
	[[ "${lines[1]}" == *"Exited"* ]]
}

# FIXME
@test "docker tag" {
	skip
}

# FIXME
@test "docker top" {
	skip
}

# FIXME
@test "docker unpause" {
	skip
}

# FIXME
@test "docker version" {
	skip
}

# FIXME
@test "docker wait" {
	skip
}


Fix filter

Fix filter

remote: Support for password authentication was removed on August 13, 2021.
remote: Please see https://docs.github.com/en/get-started/getting-started-with-git/about-remote-repositories#cloning-with-https-urls for information on currently recommended modes of authentication.
fatal: Authentication failed for 'https://github.com/Jumpscale/jumpscale6_core.git/'


bug fix for rpm stage

bug fix for rpm stage

remote: Support for password authentication was removed on August 13, 2021.
remote: Please see https://docs.github.com/en/get-started/getting-started-with-git/about-remote-repositories#cloning-with-https-urls for information on currently recommended modes of authentication.
fatal: Authentication failed for 'https://github.com/jmcanterafonseca/fiware-cygnus.git/'


Added git-dispatcher-completion.bash

Added git-dispatcher-completion.bash

#!/bin/bash

_git_dispatcher() {
    local subcommands="install update uninstall activate deactivate"
	local subcommand="$(__git_find_on_cmdline "$subcommands")"

	if [ -z "$subcommand" ]; then
		__gitcomp "$subcommands"
		return
	fi

	case "$subcommand" in
        install|update)
            __git_dispatcher_version
            return
            ;;
        *)
            COMPREPLY=()
            ;;
	esac
}

__git_dispatcher_version() {
    versions=$(git ls-remote --heads --tags https://github.com/whiskeysierra/git-dispatcher.git | grep -oE "refs/(heads|tags)/.*[^}]$" | sed -E "s/refs\/(heads|tags)\///g")
    __gitcomp "$versions"
}

# alias __git_find_on_cmdline for backwards compatibility
if [ -z "`type -t __git_find_on_cmdline`" ]; then
	alias __git_find_on_cmdline=__git_find_subcommand
fi

error: pathspec 'git-dispatcher-completion.bash' did not match any file(s) known to git


Script upload

Script upload

remote: Support for password authentication was removed on August 13, 2021.
remote: Please see https://docs.github.com/en/get-started/getting-started-with-git/about-remote-repositories#cloning-with-https-urls for information on currently recommended modes of authentication.
fatal: Authentication failed for 'https://github.com/LooneySJ/macOS-Login-Screen-Changer.git/'


Update build.sh

Update build.sh

remote: Support for password authentication was removed on August 13, 2021.
remote: Please see https://docs.github.com/en/get-started/getting-started-with-git/about-remote-repositories#cloning-with-https-urls for information on currently recommended modes of authentication.
fatal: Authentication failed for 'https://github.com/baboune/dockerimages.git/'


Script to retrieve the current schema version

#!/bin/bash
DB=$1
if [ -z "$1" ]
 then
  DB="discord.db"
fi
echo $DB
sqlite3 -line $DB "SELECT * FROM schema LIMIT 1;"

error: pathspec 'tools/getSchemaVersion.sh' did not match any file(s) known to git


Create run-with-redis.sh

Create run-with-redis.sh

#!/bin/bash
_CTXT=/usr/local/tomcat/conf/context.xml
sed -i  's#</Context>##' $_CTXT
echo '<Valve className="com.gopivotal.manager.SessionFlushValve"/>' >> $_CTXT
echo '<Manager className="org.apache.catalina.session.PersistentManager">' >> $_CTXT
echo '<Store className="com.gopivotal.manager.redis.RedisStore"' >> $_CTXT
echo " host=\"$REDIS_HOST\"" >> $_CTXT
echo " port=\"$REDIS_PORT\"" >> $_CTXT
echo '/>' >> $_CTXT
echo '</Manager>' >> $_CTXT
echo '</Context>' >> $_CTXT

exec /run.sh

error: pathspec 'jre8-tomcat8/run-with-redis.sh' did not match any file(s) known to git


Do cleaning before re-generating lang

Do cleaning before re-generating lang

remote: Support for password authentication was removed on August 13, 2021.
remote: Please see https://docs.github.com/en/get-started/getting-started-with-git/about-remote-repositories#cloning-with-https-urls for information on currently recommended modes of authentication.
fatal: Authentication failed for 'https://github.com/cybeliak/mispronounceableness.git/'


feat: add script to configure bash aliases

#!/bin/bash

set -e

# sets "vim" as default editor
sudo sh -c 'echo "\nexport EDITOR='vim'\nexport VISUAL='vim'" >> /etc/bash.bashrc'

# creating alias commands
touch ~/.bash_aliases
echo "alias h='vim /etc/hosts'" >> ~/.bash_aliases

# display hidden files by default
sudo sh -c 'echo "\nalias ls=\"ls -a\"" >> ~/.bashrc'

# reload bash
source ~/.bashrc

exit 0

error: pathspec 'configure-basic-bash-aliases.sh' did not match any file(s) known to git


Don't use ROOT config

Don't use ROOT config

remote: Support for password authentication was removed on August 13, 2021.
remote: Please see https://docs.github.com/en/get-started/getting-started-with-git/about-remote-repositories#cloning-with-https-urls for information on currently recommended modes of authentication.
fatal: Authentication failed for 'https://github.com/just-paja/improvanywhere-platform-scripts.git/'


Add provision resources for tutorial http://www.liviubalan.com/install-ubuntu-14-04-lts-server-on-virtualbox

# (c) Liviu Balan <liv_romania@yahoo.com>
# http://www.liviubalan.com/
#
# For the full copyright and license information, please view the LICENSE
# file that was distributed with this source code.

LIV_TUT_META_URL='http://www.liviubalan.com/install-ubuntu-14-04-lts-server-on-virtualbox'
LIV_TUT_META_DESCR='This is achieved by using a Vagrant box. For more info see http://www.liviubalan.com/vagrant-boxes-overview'

error: pathspec 'provision-shell/tutorials/100/011/meta.sh' did not match any file(s) known to git


added

added

#!/bin/bash
# Author: Hee Won Lee <knowpd@research.att.com>
# Created on 10/20/2017

if [[ "$#" -ne 2 ]]; then
    echo "Usage: $0 <public_network> <cluster_network>"
    echo "  public_network:  e.g. 172.31.0.0/20"
    echo "  cluster_network: e.g. 172.31.0.0/20"
    exit 1
fi

set -x

PUBLIC_NETWORK=$1
CLUSTER_NETWORK=$2

sudo apt install ceph ceph-common	# for every K8s nodes
sudo apt install jq			# used in activate-namespace.sh

# Note: we do not require a specific helm version.
curl -O https://storage.googleapis.com/kubernetes-helm/helm-v2.6.1-linux-amd64.tar.gz
tar xzvf helm-v2.6.1-linux-amd64.tar.gz
sudo cp linux-amd64/helm /usr/local/bin

helm init       # or helm init --upgrade
helm serve &

kubectl create namespace ceph
./create-secret-kube-config.sh ceph

./helm-install-ceph.sh ceph $PUBLIC_NETWORK $CLUSTER_NETWORK

error: pathspec 'sds/ceph-docker/examples/helm/install-allinone-ceph-mon.sh' did not match any file(s) known to git


Add OSX install script (#1947)

Add OSX install script (#1947)

* Add OSX install script

* Address PR comments

#!/usr/bin/env bash

# XXX: this script is intended to be run from
# an MacOS machine

# as written, this script will install
# tendermint core from master branch
REPO=github.com/tendermint/tendermint

# change this to a specific release or branch
BRANCH=master

if ! [ -x "$(command -v brew)" ]; then
  echo 'Error: brew is not installed, to install brew' >&2
  echo 'follow the instructions here: https://docs.brew.sh/Installation' >&2
  exit 1
fi

if ! [ -x "$(command -v go)" ]; then
  echo 'Error: go is not installed, to install go follow' >&2
  echo 'the instructions here: https://golang.org/doc/install#tarball' >&2
  echo 'ALSO MAKE SURE TO SETUP YOUR $GOPATH and $GOBIN in your ~/.profile: https://github.com/golang/go/wiki/SettingGOPATH' >&2
  exit 1
fi

if ! [ -x "$(command -v make)" ]; then
  echo 'Make not installed, installing using brew...'
  brew install make
fi

# get the code and move into repo
go get $REPO
cd $GOPATH/src/$REPO

# build & install
git checkout $BRANCH
# XXX: uncomment if branch isn't master
# git fetch origin $BRANCH
make get_tools
make get_vendor_deps
make install

error: pathspec 'scripts/install/install_tendermint_osx.sh' did not match any file(s) known to git


Add simple Terrafrom shemas exporter

Add simple Terrafrom shemas exporter

#!/usr/bin/env bash

rm -rf 'out/'
mkdir -p 'out/providers'
mkdir -p 'out/provisioners'

ls bin | grep -- '-provider-' | awk -F '-' '{$1=$2=""; print substr($0, 3)}' | sed 's/ /-/'> 'out/providers.list'
ls bin | grep -- '-provisioner-' | awk -F '-' '{$1=$2=""; print substr($0, 3)}' | sed 's/ /-/'> 'out/provisioners.list'

echo "Exporting providers"
for i in $(cat 'out/providers.list'); do
    ./bin/terraform schemas -json -indent -type provider "$i" > "out/providers/$i.json";
done

echo "Exporting provisioners"
for i in $(cat 'out/provisioners.list'); do
    ./bin/terraform schemas -json -indent -type provisioner "$i" > "out/provisioners/$i.json";
done

echo "Exporting functions"
./bin/terraform schemas -json -indent functions > "out/functions.json";

echo "All done"

error: pathspec 'shell-scripts/export-terraform-schemas.sh' did not match any file(s) known to git


fix cluster-sanity-check script to work with new docker for mac

#!/bin/bash -e
#
# It sends a event via the ingestion api and it checks if the event comes
# out on the other side (ELS)
#
# Usage:
#
#    cluster-sanity-check.sh
#
#

BASE=$(cd $(dirname $0)/../docker-images && pwd)
SAM=$BASE/..
VER=${1:-latest}
BVER=${2:-master}
[ `uname` == "Darwin" ] && export SED=gsed || export SED=sed

echo ''
echo ''
echo '------------------------------------------------------------------'
echo "           Samsara's cluster sanity check"
echo '------------------------------------------------------------------'

function container-ip(){
    # if [ "$DOCKER_HOST" != "" ] ; then
    #     echo $DOCKER_HOST | grep -oE '[0-9]+(\.[0-9]+){3}'
    # else
    #     CNTID=`docker ps | grep $1 | cut -d' ' -f1`
    #     docker inspect $CNTID | grep '"IPAddress"' | head -1 | grep -oE '[0-9]+(\.[0-9]+){3}'
    # fi
    echo ${DOCKER_HOST:-127.0.0.1} | grep -oE '[0-9]+(\.[0-9]+){3}'
}

function wait_for {
    echo "Checking if $1 is started."

    while [ "$(nc -z -w 5 $2 $3 || echo 1)" == "1" ] ; do
        echo "Waiting for $1 to start up..."
        sleep 3
    done
}

INGEST=$(container-ip ingestion_1)
ELS=$(container-ip elasticsearch_1)

wait_for ingestion-api $INGEST 9000
wait_for elasticsearch $ELS    9200

echo ''
echo ''
echo "sending event..."
export NONCE=$RANDOM
cat <<EOF | curl -i -H "Content-Type: application/json" \
                 -H "X-Samsara-publishedTimestamp: $(date +%s999)" \
                 -XPOST "http://$INGEST:9000/v1/events" -d @-
[
 {
 "timestamp": $(date +%s000),
 "sourceId": "samsara",
 "eventName": "sanity.check.started",
 "nonce": "$NONCE"
 }
]
EOF

export ATTEMPT=0
export TEST=''
while [ "$TEST" == "" -a $ATTEMPT -le 10 ] ; do
    export ATTEMPT=$(($ATTEMPT + 1))
    echo "Waiting for event to appear: attempt $ATTEMPT"
    curl -sS -XGET "http://$ELS:9200/_all/events/_search?q=nonce:+$NONCE" \
        | grep -qoE '"hits":{"total":1,' && export TEST="OK"
    sleep 3
done

echo ''
echo ''
echo '------------------------------------------------------------------'
echo -n "           Sanity check "
[ "$TEST" == "OK" ] && echo "PASSED" || echo "FAILED"
echo '------------------------------------------------------------------'
echo ''
echo ''

[ "$TEST" == "OK" ] && exit 0 || exit 1

#!/bin/bash -e
#
# It sends a event via the ingestion api and it checks if the event comes
# out on the other side (ELS)
#
# Usage:
#
#    cluster-sanity-check.sh
#
#

BASE=$(cd $(dirname $0)/../docker-images && pwd)
SAM=$BASE/..
VER=${1:-latest}
BVER=${2:-master}
[ `uname` == "Darwin" ] && export SED=gsed || export SED=sed

echo ''
echo ''
echo '------------------------------------------------------------------'
echo "           Samsara's cluster sanity check"
echo '------------------------------------------------------------------'

function container-ip(){
    if [ `uname` == "Darwin" ] ; then
        echo $DOCKER_HOST | grep -oE '[0-9]+(\.[0-9]+){3}'
    else
        CNTID=`docker ps | grep $1 | cut -d' ' -f1`
        docker inspect $CNTID | grep '"IPAddress"' | grep -oE '[0-9]+(\.[0-9]+){3}'
    fi
}

function wait_for {
    echo "Checking if $1 is started."

    while [ "$(nc -z -w 5 $2 $3 || echo 1)" == "1" ] ; do
        echo "Waiting for $1 to start up..."
        sleep 3
    done
}

INGEST=$(container-ip ingestion_1)
ELS=$(container-ip elasticsearch_1)

wait_for ingestion-api $INGEST 9000
wait_for elasticsearch $ELS    9200

echo ''
echo ''
echo "sending event..."
export NONCE=$RANDOM
cat <<EOF | curl -i -H "Content-Type: application/json" \
                 -H "X-Samsara-publishedTimestamp: $(date +%s999)" \
                 -XPOST "http://$INGEST:9000/v1/events" -d @-
[
 {
 "timestamp": $(date +%s000),
 "sourceId": "samsara",
 "eventName": "sanity.check.started",
 "nonce": "$NONCE"
 }
]
EOF

export ATTEMPT=0
export TEST=''
while [ "$TEST" == "" -a $ATTEMPT -le 10 ] ; do
    export ATTEMPT=$(($ATTEMPT + 1))
    echo "Waiting for event to appear: attempt $ATTEMPT"
    curl -sS -XGET "http://$ELS:9200/_all/events/_search?q=nonce:+$NONCE" \
        | grep -qoE '"hits":{"total":1,' && export TEST="OK"
    sleep 3
done

echo ''
echo ''
echo '------------------------------------------------------------------'
echo -n "           Sanity check "
[ "$TEST" == "OK" ] && echo "PASSED" || echo "FAILED"
echo '------------------------------------------------------------------'
echo ''
echo ''

[ "$TEST" == "OK" ] && exit 0 || exit 1