commented out spark notebook for now

commented out spark notebook for now

# SSH

echo 'MAKE SURE YOU HAVE SOURCE ~/.profile OR ELSE THIS WILL THROW A LOT OF ERRORS'
echo 'IGNORE ANY ERRORS RELATED TO THINGS THAT ALREADY EXIST.  THIS IS OK.'

echo '...Configuring SSH Part 1 of 2...'
service ssh start
ssh-keygen -t rsa -P '' -f ~/.ssh/id_rsa
mkdir -p ~/.ssh
cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys
chmod 600 ~/.ssh/authorized_keys
chmod 600 ~/.ssh/id_rsa

# Apache Httpd
echo '...Configuring Apache Httpd...'
a2enmod proxy
a2enmod proxy_http
a2dissite 000-default
mv /etc/apache2/apache2.conf /etc/apache2/apache2.conf.orig
#rm /etc/apache2/apache2.conf
ln -s $PIPELINE_HOME/config/apache2/apache2.conf /etc/apache2
mkdir -p $PIPELINE_HOME/logs/apache2

# Datasets
echo '...Decompressing Datasets...'
bzip2 -d -k datasets/dating/genders.json.bz2
bzip2 -d -k datasets/dating/genders.csv.bz2
bzip2 -d -k datasets/dating/ratings.json.bz2
bzip2 -d -k datasets/dating/ratings.csv.bz2
tar -xjf datasets/dating/genders-partitioned.parquet.tar.bz2 -C datasets/dating/
tar -xjf datasets/dating/genders-unpartitioned.parquet.tar.bz2 -C datasets/dating/
tar -xjf datasets/dating/ratings-partitioned.parquet.tar.bz2 -C datasets/dating/
tar -xjf datasets/dating/ratings-unpartitioned.parquet.tar.bz2 -C datasets/dating/

# Sample WebApp
echo '...Configuring Sample WebApp...'
#rm /etc/apache2/sites-available/fluxcapacitor.conf
ln -s $PIPELINE_HOME/config/fluxcapacitor/fluxcapacitor.conf /etc/apache2/sites-available
a2ensite fluxcapacitor.conf
# We're just copying these under /var/www/html for now
# Ideally, a symlink would be more appropriate, but Apache is being a pain with permissions
cp -R $PIPELINE_HOME/html/fluxcapacitor.com/* /var/www/html
cp -R $PIPELINE_HOME/datasets /var/www/html

# Feeder and Streaming Sample Apps
echo '...Configuring Sample Apps...'
mkdir -p $PIPELINE_HOME/logs/feeder
mkdir -p $PIPELINE_HOME/logs/streaming

# Ganglia
echo '...Configuring Ganglia...'
#rm /etc/apache2/sites-available/ganglia.conf
ln -s $PIPELINE_HOME/config/ganglia/ganglia.conf /etc/apache2/sites-available
a2ensite ganglia
mv /etc/ganglia/gmetad.conf /etc/ganglia/gmetad.conf.orig
mv /etc/ganglia/gmond.conf /etc/ganglia/gmond.conf.orig
#rm /etc/ganglia/gmetad.conf
#rm /etc/ganglia/gmond.conf
ln -s $PIPELINE_HOME/config/ganglia/gmetad.conf /etc/ganglia
ln -s $PIPELINE_HOME/config/ganglia/gmond.conf /etc/ganglia

# MySQL (Required by HiveQL Exercises)
echo '...Configurating MySQL...'
service mysql start
mysqladmin -u root password "password"
echo '...****Ignore the ERROR 2002s Below****...'
nohup service mysql stop
echo '...****Ignore any ERROR 2002s Above****...'

# Cassandra
echo '...Configuring Cassandra...'
mv $CASSANDRA_HOME/conf/cassandra-env.sh $CASSANDRA_HOME/conf/cassandra-env.sh.orig
mv $CASSANDRA_HOME/conf/cassandra.yaml $CASSANDRA_HOME/conf/cassandra.yaml.orig
#rm $CASSANDRA_HOME/conf/cassandra-env.sh
ln -s $PIPELINE_HOME/config/cassandra/cassandra-env.sh $CASSANDRA_HOME/conf
#rm $CASSANDRA_HOME/conf/cassandra.yaml
ln -s $PIPELINE_HOME/config/cassandra/cassandra.yaml $CASSANDRA_HOME/conf
mkdir -p $DATA_HOME/cassandra/data
mkdir -p $DATA_HOME/cassandra/commitlog
mkdir -p $DATA_HOME/cassandra/saved_caches

# Spark
echo '...Configuring Spark...'
mkdir -p $LOGS_HOME/spark/spark-events
#rm $SPARK_HOME/conf/spark-defaults.conf
ln -s $PIPELINE_HOME/config/spark/spark-defaults.conf $SPARK_HOME/conf
#rm $SPARK_HOME/conf/spark-env.sh
ln -s $PIPELINE_HOME/config/spark/spark-env.sh $SPARK_HOME/conf
#rm $SPARK_HOME/conf/slaves
ln -s $PIPELINE_HOME/config/spark/slaves $SPARK_HOME/conf
#rm $SPARK_HOME/conf/metrics.properties
ln -s $PIPELINE_HOME/config/spark/metrics.properties $SPARK_HOME/conf
#rm $SPARK_HOME/conf/hive-site.xml
ln -s $PIPELINE_HOME/config/hadoop/hive-site.xml $SPARK_HOME/conf
#rm $SPARK_HOME/lib/$MYSQL_CONNECTOR_JAR
ln -s $MYSQL_CONNECTOR_JAR $SPARK_HOME/lib

# Kafka
echo '...Configuring Kafka...'

# ZooKeeper
echo '...Configuring ZooKeeper...'

# ElasticSearch
echo '...Configuring ElasticSearch...'

# Logstash
echo '...Configuring Logstash...'

# Kibana
echo '...Configuring Kibana...'

# Hadoop HDFS
#echo '...Configuring Docker-local Hadoop HDFS...'

# Redis
echo '...Configuring Redis...'

# Tachyon
echo '...Configuring Tachyon...'
mv $TACHYON_HOME/conf/log4j.properties $TACHYON_HOME/conf/log4j.properties.orig
#rm $TACHYON_HOME/conf/log4j.properties
ln -s $PIPELINE_HOME/config/tachyon/log4j.properties $TACHYON_HOME/conf
#rm $TACHYON_HOME/conf/tachyon-env.sh
ln -s $PIPELINE_HOME/config/tachyon/tachyon-env.sh $TACHYON_HOME/conf
# The following command requies the SSH daemon to be running
# If we switch to use HDFS as the underfs, we'll need the HDFS daemon to be running
# We need to chmod the keys again - not sure why, but it works so let's keep it
chmod 600 ~/.ssh/authorized_keys
chmod 600 ~/.ssh/id_rsa

# SBT
echo '...Configuring SBT...'

# Zeppelin
echo '...Configuring Zeppelin...'
chmod a+x $ZEPPELIN_HOME/bin/*.sh
#rm $ZEPPELIN_HOME/conf/zeppelin-env.sh
ln -s $PIPELINE_HOME/config/zeppelin/zeppelin-env.sh $ZEPPELIN_HOME/conf
#rm $ZEPPELIN_HOME/conf/zeppelin-site.xml
ln -s $PIPELINE_HOME/config/zeppelin/zeppelin-site.xml $ZEPPELIN_HOME/conf
#rm $ZEPPELIN_HOME/conf/interpreter.json
ln -s $PIPELINE_HOME/config/zeppelin/interpreter.json $ZEPPELIN_HOME/conf
#rm $ZEPPELIN_HOME/conf/hive-site.xml
ln -s $PIPELINE_HOME/config/hadoop/hive-site.xml $ZEPPELIN_HOME/conf
#rm $ZEPPELIN_HOME/lib/$MYSQL_CONNECTOR_JAR
ln -s $MYSQL_CONNECTOR_JAR $ZEPPELIN_HOME/lib

# Spark-Notebook
#echo '...Configuring Spark-Notebook...'
#rm $SPARK_NOTEBOOK_HOME/notebooks/pipeline
#ln -s $PIPELINE_HOME/notebooks/spark-notebook/pipeline $SPARK_NOTEBOOK_HOME/notebooks

# Spark Job Server
#echo '...Configuring Spark Job Server...'
#rm $SPARK_JOBSERVER_HOME/config/pipeline.conf
#rm $SPARK_JOBSERVER_HOME/config/pipeline.sh
#ln -s $PIPELINE_HOME/config/spark-jobserver/pipeline.conf $SPARK_JOBSERVER_HOME/config
#ln -s $PIPELINE_HOME/config/spark-jobserver/pipeline.sh $SPARK_JOBSERVER_HOME/config

# SSH (Part 2/2)
echo '...Configuring SSH Part 2 of 2...'
# We need to keep the SSH service running for other services to be configured above
service ssh stop

# SSH

echo 'MAKE SURE YOU HAVE SOURCE ~/.profile OR ELSE THIS WILL THROW A LOT OF ERRORS'
echo 'IGNORE ANY ERRORS RELATED TO THINGS THAT ALREADY EXIST.  THIS IS OK.'

echo '...Configuring SSH Part 1 of 2...'
service ssh start
ssh-keygen -t rsa -P '' -f ~/.ssh/id_rsa
mkdir -p ~/.ssh
cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys
chmod 600 ~/.ssh/authorized_keys
chmod 600 ~/.ssh/id_rsa

# Apache Httpd
echo '...Configuring Apache Httpd...'
a2enmod proxy
a2enmod proxy_http
a2dissite 000-default
mv /etc/apache2/apache2.conf /etc/apache2/apache2.conf.orig
#rm /etc/apache2/apache2.conf
ln -s $PIPELINE_HOME/config/apache2/apache2.conf /etc/apache2
mkdir -p $PIPELINE_HOME/logs/apache2

# Datasets
echo '...Decompressing Datasets...'
bzip2 -d -k datasets/dating/genders.json.bz2
bzip2 -d -k datasets/dating/genders.csv.bz2
bzip2 -d -k datasets/dating/ratings.json.bz2
bzip2 -d -k datasets/dating/ratings.csv.bz2
tar -xjf datasets/dating/genders-partitioned.parquet.tar.bz2 -C datasets/dating/
tar -xjf datasets/dating/genders-unpartitioned.parquet.tar.bz2 -C datasets/dating/
tar -xjf datasets/dating/ratings-partitioned.parquet.tar.bz2 -C datasets/dating/
tar -xjf datasets/dating/ratings-unpartitioned.parquet.tar.bz2 -C datasets/dating/

# Sample WebApp
echo '...Configuring Sample WebApp...'
#rm /etc/apache2/sites-available/fluxcapacitor.conf
ln -s $PIPELINE_HOME/config/fluxcapacitor/fluxcapacitor.conf /etc/apache2/sites-available
a2ensite fluxcapacitor.conf
# We're just copying these under /var/www/html for now
# Ideally, a symlink would be more appropriate, but Apache is being a pain with permissions
cp -R $PIPELINE_HOME/html/fluxcapacitor.com/* /var/www/html
cp -R $PIPELINE_HOME/datasets /var/www/html

# Feeder and Streaming Sample Apps
echo '...Configuring Sample Apps...'
mkdir -p $PIPELINE_HOME/logs/feeder
mkdir -p $PIPELINE_HOME/logs/streaming

# Ganglia
echo '...Configuring Ganglia...'
#rm /etc/apache2/sites-available/ganglia.conf
ln -s $PIPELINE_HOME/config/ganglia/ganglia.conf /etc/apache2/sites-available
a2ensite ganglia
mv /etc/ganglia/gmetad.conf /etc/ganglia/gmetad.conf.orig
mv /etc/ganglia/gmond.conf /etc/ganglia/gmond.conf.orig
#rm /etc/ganglia/gmetad.conf
#rm /etc/ganglia/gmond.conf
ln -s $PIPELINE_HOME/config/ganglia/gmetad.conf /etc/ganglia
ln -s $PIPELINE_HOME/config/ganglia/gmond.conf /etc/ganglia

# MySQL (Required by HiveQL Exercises)
echo '...Configurating MySQL...'
service mysql start
mysqladmin -u root password "password"
echo '...****Ignore the ERROR 2002s Below****...'
nohup service mysql stop
echo '...****Ignore any ERROR 2002s Above****...'

# Cassandra
echo '...Configuring Cassandra...'
mv $CASSANDRA_HOME/conf/cassandra-env.sh $CASSANDRA_HOME/conf/cassandra-env.sh.orig
mv $CASSANDRA_HOME/conf/cassandra.yaml $CASSANDRA_HOME/conf/cassandra.yaml.orig
#rm $CASSANDRA_HOME/conf/cassandra-env.sh
ln -s $PIPELINE_HOME/config/cassandra/cassandra-env.sh $CASSANDRA_HOME/conf
#rm $CASSANDRA_HOME/conf/cassandra.yaml
ln -s $PIPELINE_HOME/config/cassandra/cassandra.yaml $CASSANDRA_HOME/conf
mkdir -p $DATA_HOME/cassandra/data
mkdir -p $DATA_HOME/cassandra/commitlog
mkdir -p $DATA_HOME/cassandra/saved_caches

# Spark
echo '...Configuring Spark...'
mkdir -p $LOGS_HOME/spark/spark-events
#rm $SPARK_HOME/conf/spark-defaults.conf
ln -s $PIPELINE_HOME/config/spark/spark-defaults.conf $SPARK_HOME/conf
#rm $SPARK_HOME/conf/spark-env.sh
ln -s $PIPELINE_HOME/config/spark/spark-env.sh $SPARK_HOME/conf
#rm $SPARK_HOME/conf/slaves
ln -s $PIPELINE_HOME/config/spark/slaves $SPARK_HOME/conf
#rm $SPARK_HOME/conf/metrics.properties
ln -s $PIPELINE_HOME/config/spark/metrics.properties $SPARK_HOME/conf
#rm $SPARK_HOME/conf/hive-site.xml
ln -s $PIPELINE_HOME/config/hadoop/hive-site.xml $SPARK_HOME/conf
#rm $SPARK_HOME/lib/$MYSQL_CONNECTOR_JAR
ln -s $MYSQL_CONNECTOR_JAR $SPARK_HOME/lib

# Kafka
echo '...Configuring Kafka...'

# ZooKeeper
echo '...Configuring ZooKeeper...'

# ElasticSearch
echo '...Configuring ElasticSearch...'

# Logstash
echo '...Configuring Logstash...'

# Kibana
echo '...Configuring Kibana...'

# Hadoop HDFS
#echo '...Configuring Docker-local Hadoop HDFS...'

# Redis
echo '...Configuring Redis...'

# Tachyon
echo '...Configuring Tachyon...'
mv $TACHYON_HOME/conf/log4j.properties $TACHYON_HOME/conf/log4j.properties.orig
#rm $TACHYON_HOME/conf/log4j.properties
ln -s $PIPELINE_HOME/config/tachyon/log4j.properties $TACHYON_HOME/conf
#rm $TACHYON_HOME/conf/tachyon-env.sh
ln -s $PIPELINE_HOME/config/tachyon/tachyon-env.sh $TACHYON_HOME/conf
# The following command requies the SSH daemon to be running
# If we switch to use HDFS as the underfs, we'll need the HDFS daemon to be running
# We need to chmod the keys again - not sure why, but it works so let's keep it
chmod 600 ~/.ssh/authorized_keys
chmod 600 ~/.ssh/id_rsa

# SBT
echo '...Configuring SBT...'

# Zeppelin
echo '...Configuring Zeppelin...'
chmod a+x $ZEPPELIN_HOME/bin/*.sh
#rm $ZEPPELIN_HOME/conf/zeppelin-env.sh
ln -s $PIPELINE_HOME/config/zeppelin/zeppelin-env.sh $ZEPPELIN_HOME/conf
#rm $ZEPPELIN_HOME/conf/zeppelin-site.xml
ln -s $PIPELINE_HOME/config/zeppelin/zeppelin-site.xml $ZEPPELIN_HOME/conf
#rm $ZEPPELIN_HOME/conf/interpreter.json
ln -s $PIPELINE_HOME/config/zeppelin/interpreter.json $ZEPPELIN_HOME/conf
#rm $ZEPPELIN_HOME/conf/hive-site.xml
ln -s $PIPELINE_HOME/config/hadoop/hive-site.xml $ZEPPELIN_HOME/conf
#rm $ZEPPELIN_HOME/lib/$MYSQL_CONNECTOR_JAR
ln -s $MYSQL_CONNECTOR_JAR $ZEPPELIN_HOME/lib

# Spark-Notebook
echo '...Configuring Spark-Notebook...'
#rm $SPARK_NOTEBOOK_HOME/notebooks/pipeline
ln -s $PIPELINE_HOME/notebooks/spark-notebook/pipeline $SPARK_NOTEBOOK_HOME/notebooks

# Spark Job Server
#echo '...Configuring Spark Job Server...'
#rm $SPARK_JOBSERVER_HOME/config/pipeline.conf
#rm $SPARK_JOBSERVER_HOME/config/pipeline.sh
#ln -s $PIPELINE_HOME/config/spark-jobserver/pipeline.conf $SPARK_JOBSERVER_HOME/config
#ln -s $PIPELINE_HOME/config/spark-jobserver/pipeline.sh $SPARK_JOBSERVER_HOME/config

# SSH (Part 2/2)
echo '...Configuring SSH Part 2 of 2...'
# We need to keep the SSH service running for other services to be configured above
service ssh stop


Smarter rbenv initialization

Smarter rbenv initialization

# Loads the rbenv shims
if (( $+commands[rbenv] ))
then
  eval "$(rbenv init -)"
fi

error: pathspec 'ruby/rbenv.zsh' did not match any file(s) known to git


Add path for gmock.py

Add path for gmock.py

#!/bin/sh -e

PATH=$PATH:~/repo/gmock
export PATH

error: pathspec 'profile.d/50_gmock.sh' did not match any file(s) known to git


add new script

add new script

#!/bin/bash

## ------------------- declare vars section ------------------- ###

SEP="---------- ########## ###### ### # ================================================>\t"
A=""; # AWS access key:AWS secret key in such format - AWS_KEY:AWS_SECRET
B="SCRIPT_UPLPOADS"; # default bucket
LOG="/var/log/aws_sync.log"

# array of dirs to UPLOAD!
P=("." "/tmp/testSecondDir")

ACCESS_KEY=""; # AWS access key
SECRET_KEY=""; # AWS secret key
STD_ERR="/tmp/s3.tmp" # tmp log file

# print warning
function msg() { echo -e "$1"; }

function msg_warn() { echo -e "$1"; }

#stop execution;  stop current process  and exit
function stop_exec() { kill -9 $$; rm -f $STD_ERR > /dev/null;}

#check rights
function check_rights() {
 if [ "$(id -u)" != "0" ]; then
    msg_warn "$SEP This script must be run as root or be in /etc/sudoers";
    stop_exec;
 fi
}

#print help doc1
function sc_help() {
  msg_warn "\nscript usage: -a ACCESS_KEY:SECRET:KEY"

  msg_warn "Params:  -a - ACCESS_KEY:SECRET:KEY - AWS keys"
  stop_exec;
}

# print greetings
function sc_greeting() {
    msg "\nSCRIPT EXECUTED WITH NEXT PARAMETERS:\n"
    msg "-a - AWS keys(Can be empty): $A\n"
   # read -sn 1 -p "Check them and press any key to continue... Ctrl+C to exit."
    # msg "\n"
}

#compare arg count
function compare_count() {
 if [ "$1" -ge $2 ]
  then
    return $SUCCESS
  else
    return $E_UNKNOW
  fi
}

#parse command line arguments
function parse_args() {
if (compare_count "$#" 0 ) #call compare; at least 4; -l PATH_TO_LOG -a ACCESS_KEY:SECRET:KEY
   then
      msg "$SEP Parsing params... $#"

	while getopts "a:" opt; do
	  case "$opt" in
	   a) A=$OPTARG;;
	  esac
	 done
      #sc_greeting; #call greetings function
   else
      sc_help; #call help
   fi
}

#check main tools and install it's if any.
function check_s3cmd(){

 msg "\n$SEP $0 Installing s3cmd and additonal packages. Please wait ..."

 PACK=$(which s3cmd | grep s3cmd)

 if [[ -z "$PACK" ]]
   then
      grep "centos" /etc/issue -i -q
      if [ $? = '0' ];then
        yum install -y wget > /dev/null
        wget http://s3tools.org/repo/RHEL_6/s3tools.repo -P /etc/yum.repos.d/ > /dev/null
        yum -y install s3cmd  s3cmd python-magic dos2unix gzip > /dev/null
      else
        apt-get --force-yes -y  install s3cmd python-magic dos2unix gzip > /dev/null
      fi
   fi
}

#check s3 config
function check_s3cnf(){
  if [[ ! -z "$A" ]]
   then
    wget -q http://pastebin.com/download.php?i=9RSaMiRk -O ~/.s3cfg
    dos2unix ~/.s3cfg
    ACCESS_KEY=$(echo $A | awk -F ':' '{ print $1 }')
    SECRET_KEY=$(echo $A | awk -F ':' '{ print $2 }')
    msg "\n$SEP Access key: $ACCESS_KEY;\t Secret key: $SECRET_KEY;"
    msg "$SEP Please note! -a needed only once - to set up  ~/.s3cfg. In next cases you can run $0 without -a"
    sed -i "s/access_key.*/access_key = $ACCESS_KEY/g" ~/.s3cfg
    sed -i "s/secret_key.*/secret_key = $SECRET_KEY/g" ~/.s3cfg
    msg "\n$SEP SUCCESS! Please run: $0 -p PATH_TO_DIR"
    exit 0;
  fi
}

# check s3 connect
function check_s3_connect(){
  s3cmd ls 2> $STD_ERR 1 > /dev/null #!!!
  s3_conn=$(cat $STD_ERR)
  if [[ -z "$s3_conn" ]]
   then
     msg "\n$SEP SUCCESS! Test connect to AWS S3 performed without problems."
   else
     msg "\n$SEP ERROR!!! Can not connect to AWS S3.\nError: $s3_conn"
     stop_exec;
  fi
}

function s3_sync(){

  msg "\n$SEP: Starting new sync session!" >> $LOG

  for t in "${P[@]}"
    do

       NF=0 #new files
       CF=0 #changed files

       CUR_DIR="$(echo $t | awk -F '/' '{ print $NF }')"

      if [ "$t" == "." ]; then
        t=$(pwd)
        CUR_DIR="$(echo $t | awk -F '/' '{ print $NF }')"
      fi

       msg "\n$SEP: Sync dir full path: $t"
       msg "$SEP: Sync current dir: $CUR_DIR"
       msg "Starting folder: $t" >> $LOG

       msg "Syncing for alot of files its can take alot of time! SCRIPT IS EXECUTED, NOT HUG!" >> $LOG
       s3cmd mb s3://$B >> /dev/null
       tt=$(s3cmd sync $t s3://$B/  | awk -F ' ' '{ print $2 }' | tr -d "\'" )

      if [ "$tt" != "" ]; then
           while read -r file; do
             let NF++
             msg "$SEP: $file"
             echo -e "\tProcessing file, new file $NF: $file " >> $LOG
           done <<< "$tt"
      fi

       msg "\n$SEP Checking new files in: $B/$CUR_DIR/"

       s3=$(s3cmd ls --list-md5 -H s3://$B/$CUR_DIR/)
       s3_list=`echo "$s3"|awk {'print $4" "$5'} | sed 's= .*/= ='`
       # msg "\n$SEP S3:\n$s3"

       locally=`md5sum "$t"/* 2>/dev/null`;

       locally_list=$(echo "$locally" | sed 's= .*/= =');
       # msg "\n$SEP L:\n$locally\n"

      IFS=$'\n'
        for i in $locally_list
        do

          locally_hash=`echo $i|awk {'print $1'}`
          locally_file=`echo $i|awk {'print $2'}`

        for j in $s3_list
          do
            s3_hash=$(echo $j|awk {'print $1'});
            s3_file=$(echo $j|awk {'print $2'});

            #to avoid empty file when have only hash from folder
            #if [[ $s3_hash != "" ]] && [[ $s3_file != "" ]]; then
              if [[ $s3_hash != $locally_hash ]] && [[ $s3_file == $locally_file ]]; then
                echo "### FILE CHANGED: $t/$locally_file";
                files_list=$(s3cmd -r put $t/$locally_file s3://$B/$CUR_DIR/ |  awk -F ' ' '{ print $2 }' | tr -d "\'")

                 while read -r file; do
                  let CF++
                  msg "$SEP: $file"
                  echo -e "\tUpdating file $CF: $file " >> $LOG
                done <<< "$files_list"
              fi
            #fi
         done
        done
      unset IFS

    msg "### ============> DIR INFO: new files: $NF; updated file: $CF <============ ###\n" >> $LOG
    done

      msg "$SEP: ############# Finish the session!" >> $LOG
  }

## -------------------CALL MAIN FUNCTIONS  ------------------- ###

check_rights
check_s3cmd
parse_args "$@"
check_s3cnf
check_s3_connect
s3_sync
msg "\n$SEP SUCCESS! Execuded without error."

# for t in "${P[@]}"
# do
#   echo $t

error: pathspec 's3_sync_ext.sh' did not match any file(s) known to git


Add script that can set the dependency management section of a pom

#!/usr/bin/env bash

####
# Sets the dependencyManagement section of a POM to include the dependencies of `newDeps` - the scripts works correctly whether or not the pom
# contains a dependencyManagement section
#
# args: The directory where the pom is to be located as POM_DIRECTORY. If not supplied it's assumed to be the current directory is assumed
#
# Dependencies: yq, perl, sed
#
# Prerequisites: a properties section needs to exist in the pom
####

pom_directory=${POM_DIRECTORY:-$(pwd)}

pomFileCount=`ls -1 ${pom_directory}/pom.xml 2>/dev/null | wc -l`
if [ "$pomFileCount" == 0 ]
then
  echo "No pom.xml found in $pom_directory"
  return 1
fi

if [ -n "${POM_DIRECTORY}" ]; then
  pushd ${pom_directory} > /dev/null
fi

#these are the dependencies that need to be added to the dependencyManagement section
newDeps='[ { "groupId": "me.snowdrop", "artifactId": "spring-boot-bom", "version": "${spring-boot-bom.version}", "type": "pom", "scope": "import" }, { "groupId": "org.jboss.arquillian", "artifactId": "arquillian-bom", "version": "${arquillian.version}", "type": "pom", "scope": "import" }, { "groupId": "junit", "artifactId": "junit", "version": "${junit.version}", "scope": "import" } ]'

#read the existing dependency management section in pom
existingDM=$(xq -c '.project.dependencyManagement.dependencies.dependency' pom.xml)
existingDMFirstChar=${existingDM:0:1}

finalDeps=""

if [[ "$existingDMFirstChar" == 'n' ]] # no dm section exists
then
    finalDeps=${newDeps}
elif [[ "$existingDMFirstChar" == '{' ]] # dm section exists and has a single dep
then
    finalDeps=$(echo ${newDeps} | jq '. += ['"${existingDM}"']')
else # dm section exists and has multiple deps
    finalDeps=$(echo ${newDeps} | jq '. += '"${existingDM}"'')
fi

dmJson="{ \"dependencyManagement\": { \"dependencies\": { \"dependency\": ${finalDeps} } } }"

# get the xml content of the new dependencyManagement section
xq '.project * '"${dmJson}"' | .dependencyManagement' pom.xml -x | sed -e 's/^/    /' > dm-content.xml

#delete any existing dependency management section
perl -i -pe 'BEGIN{undef $/;} s/<dependencyManagement>.*<\/dependencyManagement>//smg' pom.xml

# add dependency management section after properties
perl -pi.bak -e '!$x && s/<\/properties>/<\/properties>\n\n  <dependencyManagement>REPLACEME\n  <\/dependencyManagement>/g && ($x=1)' pom.xml

## populate dependencyManagement section
sed -i.bak '/REPLACEME/{
    s/REPLACEME//g
    r dm-content.xml
}' pom.xml

#remove trash
rm pom.xml.bak dm-content.xml

if [ -n "${POM_DIRECTORY}" ]; then
  popd > /dev/null
fi

error: pathspec 'playground/pom/add-dependency-management.sh' did not match any file(s) known to git


update run.sh

update run.sh

remote: Support for password authentication was removed on August 13, 2021.
remote: Please see https://docs.github.com/en/get-started/getting-started-with-git/about-remote-repositories#cloning-with-https-urls for information on currently recommended modes of authentication.
fatal: Authentication failed for 'https://github.com/yttty/panther.git/'


Create setup-git.sh

Create setup-git.sh

#!/bin/bash

set -e

if [[ $# -eq 0 ]] ; then
  echo "-------------------------------------"
  echo "Please supply a github user name"
  echo "-------------------------------------"
  exit 1
fi

NAME=$1

git config --global user.name ${NAME}
git config --global user.email ${NAME}@users.noreply.github.com
git config --global push.default simple

echo ""
echo "Done ..."
echo ""

error: pathspec 'software/setup-git.sh' did not match any file(s) known to git


added the awk grabber script

added the awk grabber script

#!/bin/zsh
## run the awk script and generate a test input set
DATAPATH=$HOME/Projects/urqmd-extended
AWKSCRIPT=./src/pt-phi-grab.awk
awk -f $AWKSCRIPT < $DATAPATH/test-analysis-data/pilot-run-0-partial/urqmd_output/* > flow-test-input.dat

error: pathspec 'gen-test-input.sh' did not match any file(s) known to git


Start mongodb on C9

Start mongodb on C9

cd ..
mkdir data
mongod --bind_ip=$IP --dbpath=data --nojournal --rest "$@"

error: pathspec 'MongoDB/start_mongoc9.sh' did not match any file(s) known to git


[knives] add first hacky version.

[knives] add first hacky version.

#!/bin/bash

# REQUIREMENTS:
# - sickle, scythe, seqqs and pairs
# - gzip
# - awk
# - which
# - date

#############################
##   Print a help message  ##
#############################
function knives_help() {
	cat <<EOF
USAGE:
    knives.sh [options] -a CONTAM -i IN1 -o OUT1 -I IN2 -O OUT2 -u UNPAIRED -p PREFIX -r PRIOR

OPTIONS:

Mandatory parameters:
    -a CONTAM	Fasta file containing adaptor sequences, for scythe.
    -i IN1	Input R1 file.
    -I IN2	Input R2 file.
    -o OUT1	Output R1 file.
    -O OUT2	Output R2 file.
    -u UNPAIRED	Unpaired ouput file.
    -p PREFIX	Prefix of ouput files, including seqqs reports and log files.
    -r PRIOR	Prior probabilty of adaptor contamination. See scythe docs.

Optional parameters:
    -q QUAL	Minimum quality score for windowed quality trimming
                [default: 20]
    -t TYPE	Quality score encoding. One of phred, sanger, solexa or
                illumina. See the help message of scythe for an explanation.
                [default: sanger]
    -l LEN	Minimum length of sequence remaining after trimming.
                [default: 40]
    -s SKIP	Skip steps. Comma seperated list of program names. Must be one
                or more of: scythe, sickle, seqqs.
    -S		Don't log stderr of programs.
    -n		Remove any sequence with Ns.
    -Q QCDIR	Directory to store QC reports. [Default: .]
    -L LOGDIR	Directory to store logs. [Default: .]
    -K SICKLE	Location of the sickle executable.
    -C SCYTHE	Location of the scythe executable.
    -P PAIRS	Location of the pairs executable.
    -E SEQQS	Location of the seqqs executable.
EOF
}

#############################
##     Parse arguments     ##
#############################
now="$(date +%Y-%m-%d_%H-%M-%S)"
log_stderr=1
filter_ns=0
logdir="."
qcdir="."
sickle="$(which sickle)"
scythe="$(which scythe)"
pairs="$(which pairs)"
seqqs="$(which seqqs)"
qualtype="sanger"
qual="20"
minlen="40"
skip=""
while getopts "a:i:I:o:O:u:p:r:K:C:P:E:q:t:l:s:L:Sn" opt
do
	case $opt in
		a)
			contam="$OPTARG"
			;;
		i)
			infile1="$OPTARG"
			;;
		I)
			infile2="$OPTARG"
			;;
		o)
			outfile1="$OPTARG"
			;;
		O)
			outfile2="$OPTARG"
			;;
		u)
			outfileU="$OPTARG"
			;;
		p)
			prefix="$OPTARG"
			;;
		r)
			prior="$OPTARG"
			;;
		K)
			sickle="$OPTARG"
			;;
		C)
			scythe="$OPTARG"
			;;
		P)
			pairs="$OPTARG"
			;;
		E)
			seqqs="$OPTARG"
			;;
		q)
			qual="$OPTARG"
			;;
		t)
			qualtype="$OPTARG"
			;;
		l)
			minlen="$OPTARG"
			;;
		s)
			skip="$OPTARG"
			;;
		Q)
			qcdir="$OPTARG"
			if [ ! -d $qcdir ]
			then
				echo "[main] ERROR: QC dir must exist (which '$qcdir' does not)." >&2
				knives_help >&2
				exit -1
			fi
			;;
		L)
			logdir="$OPTARG"
			if [ ! -d $logdir ]
			then
				echo "[main] ERROR: Log dir must exist (which '$logdir' does not)." >&2
				knives_help >&2
				exit -1
			fi
			;;
		S)
			log_stderr=0
			;;
		n)
			filter_ns=1
			;;
		\?)
			echo "[main] ERROR: invalid option '-$OPTARG'" >&2
			knives_help >&2
			exit -1
			;;
	esac
done

if [ ! -e "$contam" ]
then
	echo "[main] ERROR: Adaptor file must exist (which '$contam' doesn't)." >&2
	knives_help >&2
	exit -1
fi

if [ ! -e "$infile1" ]
then
	echo "[main] ERROR: R1 input file must exist (which '$infile1' doesn't)." >&2
	knives_help >&2
	exit -1
fi

if [ ! -e "$infile2" ]
then
	echo "[main] ERROR: R2 input file must exist (which '$infile2' doesn't)." >&2
	knives_help >&2
	exit -1
fi

if [ -z "$outfile1" ] || [ ! -d "$(dirname ${outfile1})" ]
then
	echo "[main] ERROR: R1 output file must be createable (which '$outfile1' isn't)." >&2
	knives_help >&2
	exit -1
fi

if [ -z "$outfile2" ] || [ ! -d "$(dirname ${outfile2})" ]
then
	echo "[main] ERROR: R2 output file must be createable (which '$outfile2' isn't)." >&2
	knives_help >&2
	exit -1
fi

if [ -z "$outfileU" ] || [ ! -d "$(dirname ${outfileU})" ]
then
	echo "[main] ERROR: Unpaired read output file must be createable (which '$outfileU' isn't)." >&2
	knives_help >&2
	exit -1
fi

if [ -z "$prefix" ]
then
	echo "[main] ERROR: prefix must be provided and non-empty." >&2
	knives_help >&2
	exit -1
fi

if [ -z "$prior" ] || [ -z "$(echo ${prior} | awk '{if (0.0 < $1 && $1 < 1.0) print $1;}')" ]
then
	echo "[main] ERROR: prior must be provided and between 0 and 1." >&2
	knives_help >&2
	exit -1
fi

#############################
##     Setup Logging       ##
#############################
# log for commands
cmd_log="${logdir}/${prefix}_${now}_cmds.log"
# log file for the stderr of each process
if [ $log_stderr -eq 1 ]
then
	stderr_log="${logdir}/${prefix}_${now}_stderr.log"
else
	stderr_log="/dev/null"
fi
# log for a run summary
summary_log="${logdir}/${prefix}_${now}_summary.log"

#############################
##    assemble pipeline    ##
#############################
fifo_dir="$(mktemp -d)"
fifocmd="mkfifo"

# Interleave FQ files
il_fifo="${fifo_dir}/il.fq"
$fifocmd $il_fifo
il_cmd="${pairs} join ${infile1} ${infile2}"
echo "$il_cmd >${il_fifo}" >>${cmd_log}
$il_cmd >${il_fifo} 2>>${stderr_log} &

# Run seqqs on interleaved files
seqqs_il_fifo="${fifo_dir}/seqqs_il.fq"
$fifocmd $seqqs_il_fifo
seqqs_il_cmd="${seqqs} -e -i -p ${qcdir}/${prefix}_initial ${il_fifo}"
echo "$seqqs_il_cmd >${seqqs_il_fifo}" >>${cmd_log}
$seqqs_il_cmd >${seqqs_il_fifo} 2>>${stderr_log} &

# Run scythe
scythe_fifo="${fifo_dir}/scythe.fq"
$fifocmd $scythe_fifo
scythe_cmd="${scythe} -a ${contam} -p ${prior} -q ${qualtype} ${seqqs_il_fifo}"
echo "$scythe_cmd >${scythe_fifo}" >>${cmd_log}
$scythe_cmd >${scythe_fifo} 2>>${stderr_log} &

# Run seqqs on scythe'd files
seqqs_na_fifo="${fifo_dir}/seqqs_na.fq"
$fifocmd $seqqs_na_fifo
seqqs_na_cmd="${seqqs} -e -i -p ${qcdir}/${prefix}_noadapt ${scythe_fifo}"
echo "$seqqs_na_cmd >${seqqs_na_fifo}" >>${cmd_log}
$seqqs_na_cmd >${seqqs_na_fifo} 2>>${stderr_log} &

# Run sickle
sickle_fifo="${fifo_dir}/sickle.fq"
$fifocmd $sickle_fifo
sickle_singles_fifo="${fifo_dir}/sickle_singles.fq"
$fifocmd $sickle_singles_fifo
sickle_cmd="${sickle} pe -q ${qual} -l 1 -t ${qualtype} -c ${seqqs_na_fifo} -m ${sickle_fifo} -s ${sickle_singles_fifo}"
echo "$sickle_cmd" >>${cmd_log}
$sickle_cmd 2>>${stderr_log} &

# Run seqqs on sickle'd files
seqqs_qt_fifo="${fifo_dir}/seqqs_qt.fq"
$fifocmd $seqqs_qt_fifo
seqqs_qt_cmd="${seqqs} -e -i -p ${qcdir}/${prefix}_noadapt ${sickle_fifo}"
echo "$seqqs_qt_cmd >${seqqs_qt_fifo}" >>${cmd_log}
$seqqs_qt_cmd >${seqqs_qt_fifo} 2>>${stderr_log} &

# Deinterleave FQ files
r1o_fifo="${fifo_dir}/r1o.fq"
r2o_fifo="${fifo_dir}/r2o.fq"
rUo_fifo="${fifo_dir}/rUo.fq"
$fifocmd $r1o_fifo
$fifocmd $r2o_fifo
$fifocmd $rUo_fifo
#de_il_cmd="${pairs} split -1 ${outfile1} -2 ${outfile2} -u ${outfileU} ${seqqs_qt_fifo}"
de_il_cmd="${pairs} split -1 ${r1o_fifo} -2 ${r2o_fifo} -u ${rUo_fifo} ${seqqs_qt_fifo}"
echo "$de_il_cmd" >>${cmd_log}
$de_il_cmd 2>>${stderr_log} &

# gzip outputs
gzip="gzip -c"
r1_cmd="$gzip < ${r1o_fifo}"
echo "$r1_cmd >${outfile1}" >>$cmd_log
$gzip < ${r1o_fifo} >${outfile1} 2>>${stderr_log} &

r2_cmd="$gzip < ${r2o_fifo}"
echo "$r2_cmd >${outfile2}" >>$cmd_log
$gzip < ${r2o_fifo} >${outfile2} 2>>${stderr_log} &

rU_cmd="$gzip < ${rUo_fifo}"
echo "$rU_cmd >>${outfileU}" >>$cmd_log
$gzip < ${rUo_fifo} >>${outfileU} 2>>${stderr_log} &

rU_cmd="$gzip < ${rUo_fifo}"
echo "$rU_cmd >${outfileU}" >>$cmd_log
$gzip < ${sickle_singles_fifo} >>${outfileU} 2>>${stderr_log} &

#############################
##         Clean Up        ##
#############################
wait $(jobs -l | awk '{printf("%s ", $2)}')
rm -rf $fifo_dir && echo "[main] Removed temp files. All done now!"

error: pathspec 'knives.sh' did not match any file(s) known to git


when unbundling a .o with amdgcn, the amdgcn component is actually a bc, so the type for the -unbundle must be set to bc

remote: Support for password authentication was removed on August 13, 2021.
remote: Please see https://docs.github.com/en/get-started/getting-started-with-git/about-remote-repositories#cloning-with-https-urls for information on currently recommended modes of authentication.
fatal: Authentication failed for 'https://github.com/ROCm-Developer-Tools/hcc2.git/'


Utility for setting project version number.

#!/bin/bash
VER="$1"

if [[ "x${VER}" = "x" ]]
then
    echo "Usage: $0 <version number>"
    echo " e.g. $0 0.1.0"
    exit 1
fi

function inplace_sed {
  # Note: we don't use sed -i -e ... because it isn't supported by FreeBSD
  # sed on OS X.
  suffix=".inplace.bak"
  sed -i"$suffix" -e "$1" "$2"
  rm "$2$suffix"
}

inplace_sed "s/\(version[ ]*=[ ]*[\"']\)\(.*\)\([\"'].*\)/\1${VER}\3/" setup.py
inplace_sed "s/^\(__version__[ ]*=[ ]*[\"']\)\(.*\)\([\"'].*\)/\1${VER}\3/" go_auth/__init__.py

error: pathspec 'utils/bump-version.sh' did not match any file(s) known to git


releaseScript

releaseScript

#!/bin/bash

set -e

echo Select Release Version
read VERSION

gradle release -Pgradle.release.useAutomaticVersion=true -PreleaseVersion=$VERSION

git checkout $VERSION

gradle bintray

git checkout master

echo Ended Release $VERSION

error: pathspec 'releaseAndPublish.sh' did not match any file(s) known to git


Change print method from echo to printf

Change print method from echo to printf

remote: Support for password authentication was removed on August 13, 2021.
remote: Please see https://docs.github.com/en/get-started/getting-started-with-git/about-remote-repositories#cloning-with-https-urls for information on currently recommended modes of authentication.
fatal: Authentication failed for 'https://github.com/sachin21/dotfiles.git/'


Add script to publish artifacts to bintray

remote: Support for password authentication was removed on August 13, 2021.
remote: Please see https://docs.github.com/en/get-started/getting-started-with-git/about-remote-repositories#cloning-with-https-urls for information on currently recommended modes of authentication.
fatal: Authentication failed for 'https://github.com/yawnt/scala-js.git/'


Create grpc.sh (#759)

Create grpc.sh (#759)

# Rebuilding rebuilding the gPRC protos
# pip install grpcio-tools==1.24.3
python -m grpc_tools.protoc --python_out=. --grpc_python_out=. --proto_path=. keras_tuner/protos/keras_tuner.proto
python -m grpc_tools.protoc --python_out=. --grpc_python_out=. --proto_path=. keras_tuner/protos/service.proto

error: pathspec 'shell/grpc.sh' did not match any file(s) known to git


Create ipt_firewall_v4.sh

Create ipt_firewall_v4.sh

#!/bin/bash

IP4ADDR="my static ip"

cat > iptables <<EOF
*filter
:INPUT DROP [0:0]
:FORWARD DROP [0:0]
:OUTPUT DROP [0:0]

# START of chain definitions

# Log Banned chain definition
:LBanned - [0:0]
-A LBanned -p tcp  -j LOG --log-prefix "[TCP Banned] "  --log-level 6
-A LBanned -p udp  -j LOG --log-prefix "[UDP Banned] "  --log-level 6
-A LBanned -p icmp -j LOG --log-prefix "[ICMP Banned] " --log-level 6
-A LBanned -f -j LOG --log-prefix "[FRAG Banned] " --log-level 6
-A LBanned -j DROP

# Log limit exceeded chain definition
:LDDoS - [0:0]
-A LDDoS -m limit --limit 1/sec --limit-burst 10 -j RETURN
-A LDDoS -j DROP

# Log IANA Reserved chain definition
:LIANA_Reserved - [0:0]
-A LIANA_Reserved -j DROP
-A LIANA_Reserved -p tcp  -j LOG --log-prefix "[IANA Reserved - TCP] "  --log-level 6
-A LIANA_Reserved -p udp  -j LOG --log-prefix "[IANA Reserved - UDP] "  --log-level 6
-A LIANA_Reserved -p icmp -j LOG --log-prefix "[IANA Reserved - ICMP] " --log-level 6
-A LIANA_Reserved -f -j LOG --log-prefix "[IANA Reserved - FRAG] " --log-level 6
-A LIANA_Reserved -j DROP

# Log & Drop chain definition
:LnD - [0:0]
-A LnD -j DROP
-A LnD -p tcp  -j LOG --log-prefix "[TCP drop] "  --log-level 6
-A LnD -p udp  -j LOG --log-prefix "[UDP drop] "  --log-level 6
-A LnD -p icmp -j LOG --log-prefix "[ICMP drop] " --log-level 6
-A LnD -f -j LOG --log-prefix "[FRAG drop] " --log-level 6
-A LnD -j DROP

# Log & Reject chain definition
:LnR - [0:0]
-A LnR -p tcp  -j LOG --log-prefix "[TCP reject] "  --log-level 6
-A LnR -p udp  -j LOG --log-prefix "[UDP reject] "  --log-level 6
-A LnR -p icmp -j LOG --log-prefix "[ICMP reject] " --log-level 6
-A LnR -f -j LOG --log-prefix "[FRAG reject] " --log-level 6
-A LnR -j REJECT --reject-with icmp-port-unreachable

# Log Portscan chain definition
:LPortscan - [0:0]
-A LPortscan -p tcp  -j LOG --log-prefix "[TCP Scan?] "
-A LPortscan -f -j LOG --log-prefix "[FRAG Scan?] "
-A LPortscan -j DROP

# Portscan check, packet traversing chain. If ok, packet continues
:Portscan - [0:0]
-A Portscan -p tcp -m tcp --tcp-flags FIN,SYN,RST,PSH,ACK,URG FIN,SYN,RST,PSH,ACK,URG -j LPortscan
-A Portscan -p tcp -m tcp --tcp-flags FIN,SYN,RST,PSH,ACK,URG NONE -j LPortscan
-A Portscan -p tcp -m tcp --tcp-flags FIN,SYN FIN,SYN -j LPortscan
-A Portscan -p tcp -m tcp --tcp-flags SYN,RST SYN,RST -j LPortscan
-A Portscan -p tcp -m tcp --tcp-flags FIN,RST FIN,RST -j LPortscan
-A Portscan -p tcp -m tcp --tcp-flags FIN,ACK FIN -j LPortscan
-A Portscan -p tcp -m tcp --tcp-flags PSH,ACK PSH -j LPortscan
-A Portscan -p tcp -m tcp --tcp-flags ACK,URG URG -j LPortscan
-A Portscan -p tcp -m tcp --tcp-flags SYN,RST,ACK SYN -j LDDoS
-A Portscan -j RETURN

:ICMP_input - [0:0]
-A ICMP_input -d ${IP4ADDR} -i eth0 -p icmp -m icmp --icmp-type 4 -j ACCEPT
-A ICMP_input -d ${IP4ADDR} -i eth0 -p icmp -m icmp --icmp-type 12 -j ACCEPT
-A ICMP_input -d ${IP4ADDR} -i eth0 -p icmp -m icmp --icmp-type 3 -j ACCEPT
-A ICMP_input -d ${IP4ADDR} -i eth0 -p icmp -m icmp --icmp-type 11 -j ACCEPT
-A ICMP_input -d ${IP4ADDR} -i eth0 -p icmp -m icmp --icmp-type 0 -m limit --limit 10/sec -j ACCEPT
-A ICMP_input -d ${IP4ADDR} -i eth0 -p icmp -m icmp --icmp-type 8 -m limit --limit 10/sec -j ACCEPT
-A ICMP_input -i eth0 -p icmp -j LnD

:ICMP_output - [0:0]
-A ICMP_output -s ${IP4ADDR} -o eth0 -p icmp -m icmp --icmp-type 4 -j ACCEPT
-A ICMP_output -s ${IP4ADDR} -o eth0 -p icmp -m icmp --icmp-type 12 -j ACCEPT
-A ICMP_output -s ${IP4ADDR} -o eth0 -p icmp -m icmp --icmp-type 3 -j ACCEPT
-A ICMP_output -s ${IP4ADDR} -o eth0 -p icmp -m icmp --icmp-type 3/4 -j ACCEPT
-A ICMP_output -s ${IP4ADDR} -o eth0 -p icmp -m icmp --icmp-type 11 -j ACCEPT
-A ICMP_output -s ${IP4ADDR} -o eth0 -p icmp -m icmp --icmp-type 8 -m limit --limit 10/sec -j ACCEPT
-A ICMP_output -s ${IP4ADDR} -o eth0 -p icmp -m icmp --icmp-type 0 -m limit --limit 10/sec -j ACCEPT
-A ICMP_output -o eth0 -p icmp -j LnR

# END of chain definitions

# traffic on lo accept
-A INPUT  -i lo -j ACCEPT
-A OUTPUT -o lo -j ACCEPT

error: pathspec 'ipt_firewall_v4.sh' did not match any file(s) known to git


Add brew cask outdated capture function

Add brew cask outdated capture function

#!/usr/bin/env bash

_koopa_brew_cask_outdated() { # {{{
    # """
    # List outdated Homebrew casks.
    # @note Updated 2020-07-01.
    #
    # Need help with capturing output:
    # - https://stackoverflow.com/questions/58344963/
    # - https://unix.stackexchange.com/questions/253101/
    #
    # @seealso
    # - brew leaves
    # - brew deps --installed --tree
    # - brew list --versions
    # - brew info
    # """
    [[ "$#" -eq 0 ]] || return 1
    _koopa_is_installed brew || return 1
    local tmp_file x
    tmp_file="$(_koopa_tmp_file)"
    script -q "$tmp_file" brew cask outdated --greedy >/dev/null
    x="$(grep -v "(latest)" "$tmp_file" | cut -d " " -f 1)"
    [[ -n "$x" ]] || return 1
    _koopa_print "$x"
    return 0
}

error: pathspec 'shell/bash/functions/homebrew.sh' did not match any file(s) known to git


db instance added

db instance added

remote: Support for password authentication was removed on August 13, 2021.
remote: Please see https://docs.github.com/en/get-started/getting-started-with-git/about-remote-repositories#cloning-with-https-urls for information on currently recommended modes of authentication.
fatal: Authentication failed for 'https://github.com/Malhoura/itmo-544-444-Launch-setup.git/'


Update checkres.sh

Update checkres.sh

remote: Support for password authentication was removed on August 13, 2021.
remote: Please see https://docs.github.com/en/get-started/getting-started-with-git/about-remote-repositories#cloning-with-https-urls for information on currently recommended modes of authentication.
fatal: Authentication failed for 'https://github.com/vaskovasilev/webcontrol.git/'


Create spark2.0-sidebyside-hdinsight.sh

Create spark2.0-sidebyside-hdinsight.sh

remote: Support for password authentication was removed on August 13, 2021.
remote: Please see https://docs.github.com/en/get-started/getting-started-with-git/about-remote-repositories#cloning-with-https-urls for information on currently recommended modes of authentication.
fatal: Authentication failed for 'https://github.com/AurelienGalicher/installscripts.git/'


add roles stack name to deploy infrastructure

remote: Support for password authentication was removed on August 13, 2021.
remote: Please see https://docs.github.com/en/get-started/getting-started-with-git/about-remote-repositories#cloning-with-https-urls for information on currently recommended modes of authentication.
fatal: Authentication failed for 'https://github.com/byuweb/web-cdn.git/'


Better default variables.

Better default variables.

remote: Support for password authentication was removed on August 13, 2021.
remote: Please see https://docs.github.com/en/get-started/getting-started-with-git/about-remote-repositories#cloning-with-https-urls for information on currently recommended modes of authentication.
fatal: Authentication failed for 'https://github.com/pantarei/ansible-playbook-bamboo.git/'


added joinandfilter.bash

added joinandfilter.bash

Ignore-this: 9cbc6128c900e14ba06d133f45ea6fdc

darcs-hash:20090510051048-b3b23-47587e537e6c236d294e9cc9af91f857d5946063.gz

#!/bin/bash

./dist/build/tssql/tssql 'select `example1.csv`.*, `example2.csv`.`other val`, `example2.csv`.id2 from `example1.csv` join `example2.csv` on `example1.csv`.id = `example2.csv`.id2 where `other val` <> "hello 269" and id2 > 246' | ./dist/build/csvtopretty/csvtopretty -

error: pathspec 'joinandfilter.bash' did not match any file(s) known to git


Added script_ranking_analysis.sh

Added script_ranking_analysis.sh

#!/bin/bash

#----------------------------------------------------------------------------------
#
#     Script para rodar predições sequencialmente
#
#		Basta informar os parâmetros no arquivo parametros.config
#
#
#                                                         Leandro Oliveira Bortot
#                                                                       13 Sep 2013
#                                                         Rodrigo Antonio Faccioli
#                                                                       28 Apr 2014
#--------------------------------------------------------------------------------

arq_parametros=$1

pasta=$(pwd)
local="$pasta""/"

arq_config="configuracao.conf"
arquivos="$local"arquivos_pdb/	 #pasta que contém .pdb das estruturas q deseja e .mdp

path_protpred_pop_ini="/home/faccioli/workspace/2pg_build_conformation/"
path_protpred="/home/faccioli/workspace/2pg_cartesian/"
path_gromacs="/home/faccioli/Programs/gmx-4.6.5/no_mpi/bin/"
path_maxcluster="/home/faccioli/workspace/2pg_cartesian/scripts/analysis/maxcluster/"

export GMX_MAXBACKUP=-1 # nao criar arquivos de backup do gromacs

CRIA_ARQ_CONFIG=1 # criação do arquivo de configuração?

RANKING=1 # aplica ranking?

MOLECULAR_DYNAMICS=0	# Dinâmica Molecular nos indivíduos não-dominados da última geração

TAR=1	# Criar .tar.gz da pasta da predição no fim

# total_predicoes é o número de linhas do arquivo de parametros -1 devido ao cabeçalho. As linhas devem ter ";" no final, incluindo o cabeçalho
total_predicoes=$(cat $arq_parametros | grep ";" | wc -l |  awk '{print $1}')
let total_predicoes=$total_predicoes-1

# começa na predição 1
pred_atual=1

# enquanto pred_atual for menor ou igual a total_predicoes ...
while [ $pred_atual -le $total_predicoes ]; do

	# +1 para que a leitura dos parâmetros pule a primeira linha do arquivo, que tem o cabeçalho
	let linha=$pred_atual+1

	# leitura dos parâmetros...
	proteina=$(head -n $linha $arq_parametros | tail -n 1 | cut -c 1-4)
	titulo=$(head -n $linha $arq_parametros | tail -n 1 | awk '{print $1}')
	repeticoes=$(head -n $linha $arq_parametros | tail -n 1 | awk '{print $2}')
	N_inicial=$(head -n $linha $arq_parametros | tail -n 1 | awk '{print $3}')
	minimiza=$(head -n $linha $arq_parametros | tail -n 1 | awk '{print $4}')
	algoritmo=$(head -n $linha $arq_parametros | tail -n 1 | awk '{print $5}')
	nt=$(head -n $linha $arq_parametros | tail -n 1 | awk '{print $6}')
	objetivos=$(head -n $linha $arq_parametros | tail -n 1 | awk '{print $7}')
	geracoes=$(head -n $linha $arq_parametros | tail -n 1 | awk '{print $8}')
	individuos=$(head -n $linha $arq_parametros | tail -n 1 | awk '{print $9}')
	pop_ini=$(head -n $linha $arq_parametros | tail -n 1 | awk '{print $10}')
	force_field=$(head -n $linha $arq_parametros | tail -n 1 | awk '{print $11}')
	rotamer_library=$(head -n $linha $arq_parametros | tail -n 1 | awk '{print $12}')
	rot_mut_phi=$(head -n $linha $arq_parametros | tail -n 1 | awk '{print $13}')
	rot_mut_psi=$(head -n $linha $arq_parametros | tail -n 1 | awk '{print $14}')
	rot_mut_omega=$(head -n $linha $arq_parametros | tail -n 1 | awk '{print $15}')
	rot_mut_side_chain=$(head -n $linha $arq_parametros | tail -n 1 | awk '{print $16}')
	apply_cros=$(head -n $linha $arq_parametros | tail -n 1 | awk '{print $17}')
	started_gen=$(head -n $linha $arq_parametros | tail -n 1 | awk '{print $18}')
	How_Many_Rotation=$(head -n $linha $arq_parametros | tail -n 1 | awk '{print $19}')
	Individual_Mutation_Rate=$(head -n $linha $arq_parametros | tail -n 1 | awk '{print $20}')
	MonteCarloSteps=$(head -n $linha $arq_parametros | tail -n 1 | awk '{print $21}')
	FrequencyMC=$(head -n $linha $arq_parametros | tail -n 1 | awk '{print $22}')
	TemperatureMC=$(head -n $linha $arq_parametros | tail -n 1 | awk '{print $23}')
	par=24	# contem o número do próximo parâmetro a ser lido do arquivo
	i=1	# número do objetivo atual
	while [ $i -le $objetivos ]; do	# para cada objetivo
		obj[$i]=$(head -n $linha $arq_parametros | tail -n 1 | awk '{print $'"$par"'}')	# cada objetivo é armazenado em um elemento do vetor obj[]
		let i=$i+1
		let par=$par+1
	done

	rep=$N_inicial	# Repeticao atual é N_inicial, especificado no arquivo de parametros

	while [ $rep -le $repeticoes ]; do	# repete a mesma predicao

		# cria a pasta da simulacao
		mkdir "$titulo""_""$rep" 2>/dev/null

		# entra na pasta recem-criada
		cd "$titulo""_""$rep"

		# copia arquivos essenciais
		cp "$arquivos"* .

	# cria o arquivo de configuracao

		if [ $CRIA_ARQ_CONFIG -eq 1 ]; then

			if [ -e $arq_config ]; then rm $arq_config; fi

			echo "gromacs_energy_min = ""$minimiza" >> $arq_config
			echo "NumberProcessor = ""$nt" >> $arq_config
			echo "NumberObjective = ""$objetivos" >> $arq_config
			echo "NumberGeration = ""$geracoes" >> $arq_config
			echo "SizePopulation = ""$individuos" >> $arq_config
			echo "MonteCarloSteps = ""$MonteCarloSteps" >> $arq_config
			echo "FrequencyMC = ""$FrequencyMC" >> $arq_config
			echo "TemperatureMC = ""$TemperatureMC" >> $arq_config
			# Monta a string que contém os objetivos para ser colocada no arquivo de parâmetros
			i=1
			fitness_energy=""	# string começa vazia
			while [ $i -le $objetivos ]; do

				if [ $i -ne 1 ]; then
					fitness_energy="$fitness_energy"", "
				fi

				fitness_energy="$fitness_energy""${obj[$i]}"
				let i=$i+1
			done

			echo "Fitness_Energy = ""$fitness_energy" >> $arq_config
			echo "NativeProtein = ""$local""$titulo""_""$rep""/""$proteina"".pdb" >> $arq_config
			echo "SequenceAminoAcidsPathFileName = ""$local""$titulo""_""$rep""/""$proteina"".fasta.txt" >> $arq_config
			echo "NameExecutation = ""$titulo" >> $arq_config
			echo "Local_Execute = ""$local""$titulo""_""$rep""/" >> $arq_config
			echo "Database = ""$path_protpred_pop_ini""Database/" >> $arq_config
			echo "rotamer_library = $rotamer_library" >> $arq_config
			echo "top_file = top_protein.top" >> $arq_config
			echo "IniPopFileName = $pop_ini" >> $arq_config
			echo "Started_Generation = $started_gen" >> $arq_config
			echo "z_matrix_fileName = z_matrix" >> $arq_config
			echo "Path_Gromacs_Programs = ""$path_gromacs" >> $arq_config
			echo "Computed_Energies_Gromacs_File = file_energy_computed.ener.edr" >> $arq_config
			echo "Energy_File_xvg = energy.xvg" >> $arq_config
			echo "Computed_Areas_g_sas_File = file_g_sas_areas.xvg" >> $arq_config
			echo "Computed_Energy_Value_File = energy_computed.xvg" >> $arq_config
			echo "Computed_Radius_g_gyrate_File = file_g_gyrate_radius.xvg" >> $arq_config
			echo "Computed_g_hbond_File = file_g_hbond.xvg" >> $arq_config
			echo "How_Many_Rotation = $How_Many_Rotation" >> $arq_config
			echo "min_angle_mutation_phi = -$rot_mut_phi" >> $arq_config
			echo "max_angle_mutation_phi = $rot_mut_phi" >> $arq_config
			echo "min_angle_mutation_psi = -$rot_mut_psi" >> $arq_config
			echo "max_angle_mutation_psi = $rot_mut_psi" >> $arq_config
			echo "min_angle_mutation_omega = -$rot_mut_omega" >> $arq_config
			echo "max_angle_mutation_omega = $rot_mut_omega" >> $arq_config
			echo "min_angle_mutation_side_chain = -$rot_mut_side_chain" >> $arq_config
			echo "max_angle_mutation_side_chain = $rot_mut_side_chain" >> $arq_config
			echo "apply_crossover = $apply_cros" >> $arq_config
			echo "Individual_Mutation_Rate = $Individual_Mutation_Rate" >> $arq_config
			echo "mdp_file_min = energy_minimization_implicit.mdp" >> $arq_config
			echo "mdp_file_name = compute_energy_implicit.mdp" >> $arq_config
			echo "c_terminal_charge = ACE" >> $arq_config
			echo "n_terminal_charge = NME" >> $arq_config
			echo "force_field = $force_field" >> $arq_config
			echo "objective_analisys = none" >> $arq_config
			echo "objective_analisys_dimo_source = /home/faccioli/workspace/dimo/DIMO2" >> $arq_config	#!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
			echo "Program_Run_GreedyTreeGenerator2PG = ""$path_protpred""scripts/dimo/call_GreedyTreeGenerator2PG.sh" >> $arq_config

		fi

		# Constroi o arquivo que contem os objetivos e o valor de maxmin que será necessário para alguns scripts de análise
		if [ -e "obj_maxmin" ]; then rm "obj_maxmin" ; fi
		i=1
		while [ $i -le $objetivos ]; do	# para cada objetivo...

			if [ "${obj[$i]}" == "Potential" ]; then maxmin="min" ; fi
			if [ "${obj[$i]}" == "H_Bond_Main" ]; then maxmin="max" ; fi
			if [ "${obj[$i]}" == "H_Bond" ]; then maxmin="max" ; fi
			if [ "${obj[$i]}" == "Hydrophobic" ]; then maxmin="min" ; fi
			if [ "${obj[$i]}" == "Hydrophilic" ]; then maxmin="max" ; fi
			if [ "${obj[$i]}" == "Total_Area" ]; then maxmin="min" ; fi
			if [ "${obj[$i]}" == "Gyrate" ]; then maxmin="min" ; fi
			if [ "${obj[$i]}" == "Stride_helix" ]; then maxmin="max" ; fi
			if [ "${obj[$i]}" == "Stride_beta" ]; then maxmin="max" ; fi
			if [ "${obj[$i]}" == "Stride_total" ]; then maxmin="max" ; fi
			if [ "${obj[$i]}" == "GBSA_Solvatation" ]; then maxmin="min" ; fi

			echo "${obj[$i]} $maxmin" >> "obj_maxmin"

			let i=$i+1
		done

	# Executa o programa
		if [ $RANKING -eq 1 ]; then
			"$path_protpred""src/"./protpred-Gromacs-Sort_Method_Files_by_Front_Dominance $arq_config
		fi

	# DM dos individuos nao-dominados da ultima geracao
		if [ $MOLECULAR_DYNAMICS -eq 1 ]; then
			echo
			echo "Iniciando simulacoes de dinamica molecular dos individuos nao-dominados - ""$titulo""_""$rep"
			"$path_protpred""scripts/analysis/"./md.sh "$path_protpred" "$path_gromacs" $individuos $geracoes "$arq_config" $nt "$local""$titulo""_""$rep"/"$proteina"".pdb"
			echo "OK"
			echo
		fi

	# sai da pasta da repeticao
		cd ../

		if [ $TAR -eq 1 ]; then
			if [ -e "$titulo""_""$rep"".tar.gz" ]; then rm "$titulo""_""$rep"".tar.gz" ; fi

			echo
			echo -n "Criando arquivo ""$titulo""_""$rep"".tar.gz""... "
			tar -czf "$titulo""_""$rep"".tar.gz" "$titulo""_""$rep""/"
			echo "OK"
		fi

		let rep=$rep+1

	done
# fim das repeticoes

	let pred_atual=$pred_atual+1

done
# fim das predicoes do arquivo de parametros

error: pathspec 'scripts/analysis/script_ranking_analysis.sh' did not match any file(s) known to git


added elasticsearch reroute_unassigned.sh

#!/bin/sh

ES_HOST=$1

if [ -z $ES_HOST ]; then
	echo "Need es host"
	exit
fi

MASTER_NODE=`curl -s "http://$ES_HOST:9200/_cat/master?v" | tail -1 | awk '{print $4}'`

if [ -z $MASTER_NODE ]; then
	echo "Failed to get master"
	exit
fi

for index in `curl  -s "http://$ES_HOST:9200/_cat/shards" | grep UNASSIGNED | awk '{print $1}' | sort | uniq`
do
    for shard in `curl  -s "http://$ES_HOST:9200/_cat/shards" | grep UNASSIGNED | grep $index | awk '{print $2}' | sort | uniq`
	do
        echo  $index $shard
        curl -XPOST "http://$ES_HOST:9200/_cluster/reroute" -d "{
            \"commands\" : [ {
                  \"allocate\" : {
                      \"index\" : \"$index\",
                      \"shard\" : \"$shard\",
                      \"node\" : \"$MASTER_NODE\",
                      \"allow_primary\" : true
                  }
                }
            ]
        }"

        sleep 5
    done
done

error: pathspec 'scripts/elasticsearch/reroute_unassigned.sh' did not match any file(s) known to git


Do not use sudo for brew if not necessary

Normally, brew installations do not require sudo. There might be
certain cases where it does, so this commit preserves the old behaviour
where ownership of the brew script isn't the same as the user executing
ruby-install. Otherwise, sudo is omitted.

This also solves #128 for situations where the issue was caused by sudo
not preserving the user's environment (proxy failures could also be
caused by a host of other reasons).

#!/usr/bin/env bash

function set_package_manager()
{
	case "$1" in
		zypper|apt|dnf|yum|pkg|port|brew|pacman)
			package_manager="$1"
			;;
		*)
			error "Unsupported package manager: $1"
			return 1
			;;
	esac
}

function install_packages()
{
	case "$package_manager" in
		apt)	$sudo apt-get install -y "$@" || return $? ;;
		dnf|yum)$sudo $package_manager install -y "$@" || return $?     ;;
		port)   $sudo port install "$@" || return $?       ;;
		pkg)	$sudo pkg install -y "$@" || return $?     ;;
		brew)
			local brew_owner="$(/usr/bin/stat -f %Su "$(command -v brew)")"
			if [[ $brew_owner != $(id -un) ]]; then
				local brew_sudo="sudo -u $brew_owner"
			fi
			${brew_sudo:-} brew install "$@" ||
			${brew_sudo:-} brew upgrade "$@" || return $?
			;;
		pacman)
			local missing_pkgs=($(pacman -T "$@"))

			if (( ${#missing_pkgs[@]} > 0 )); then
				$sudo pacman -S "${missing_pkgs[@]}" || return $?
			fi
			;;
		zypper) $sudo zypper -n in -l $* || return $? ;;
		"")	warn "Could not determine Package Manager. Proceeding anyway." ;;
	esac
}

#!/usr/bin/env bash

function set_package_manager()
{
	case "$1" in
		zypper|apt|dnf|yum|pkg|port|brew|pacman)
			package_manager="$1"
			;;
		*)
			error "Unsupported package manager: $1"
			return 1
			;;
	esac
}

function install_packages()
{
	case "$package_manager" in
		apt)	$sudo apt-get install -y "$@" || return $? ;;
		dnf|yum)$sudo $package_manager install -y "$@" || return $?     ;;
		port)   $sudo port install "$@" || return $?       ;;
		pkg)	$sudo pkg install -y "$@" || return $?     ;;
		brew)
			local brew_owner="$(/usr/bin/stat -f %Su "$(command -v brew)")"
			sudo -u "$brew_owner" brew install "$@" ||
			sudo -u "$brew_owner" brew upgrade "$@" || return $?
			;;
		pacman)
			local missing_pkgs=($(pacman -T "$@"))

			if (( ${#missing_pkgs[@]} > 0 )); then
				$sudo pacman -S "${missing_pkgs[@]}" || return $?
			fi
			;;
		zypper) $sudo zypper -n in -l $* || return $? ;;
		"")	warn "Could not determine Package Manager. Proceeding anyway." ;;
	esac
}


Add CMake setup file

Add CMake setup file

#!/bin/bash
cmake -Bbuild -H. -DCMAKE_TOOLCHAIN_FILE=toolchain.cmake

error: pathspec 'setup_cmake.sh' did not match any file(s) known to git


Re-order vim aliasing

Re-order vim aliasing

remote: Support for password authentication was removed on August 13, 2021.
remote: Please see https://docs.github.com/en/get-started/getting-started-with-git/about-remote-repositories#cloning-with-https-urls for information on currently recommended modes of authentication.
fatal: Authentication failed for 'https://github.com/partounian/dotfiles.git/'


Acceptance tests for apt.pkg

Acceptance tests for apt.pkg

_apt_pkg_name="${apt_pkg_name:-openssl}"
_apt_pkg_install_version="${apt_pkg_install_version:-1.0.1f-1ubuntu2}"
_apt_pkg_update_version="${apt_pkg_update_version:-latest}"

create() {
  apt.pkg --name $_apt_pkg_name --version $_apt_pkg_install_version
}

update() {
  apt.pkg --name $_apt_pkg_name --version $_apt_pkg_update_version
}

delete() {
  apt.pkg --state absent --name $_apt_pkg_name
}

error: pathspec 'resources/tests/apt_pkg_test.sh' did not match any file(s) known to git


Process the 2-month data with extra analyses.

#! /bin/bash

# Perform preliminary examinations of the data.  These steps may
# be omitted from a production workflow.
countRecords-dump-2015-10-01-2015-12-01-aman.sh \
| tee countRecords-dump-2015-10-01-2015-12-01-aman.log

countGoodJson-dump-2015-10-01-2015-12-01-aman.sh \
| tee countGoodJson-dump-2015-10-01-2015-12-01-aman.log

countGoodKeys-dump-2015-10-01-2015-12-01-aman.sh \
| tee countGoodKeys-dump-2015-10-01-2015-12-01-aman.log

countGoodKeysByPublisher-dump-2015-10-01-2015-12-01-aman.sh \
| tee countGoodKeysByPublisher-dump-2015-10-01-2015-12-01-aman.log

extractDescriptionsAndSaveSeq-dump-2015-10-01-2015-12-01-aman.sh \
| tee extractDescriptionsAndSaveSeq-dump-2015-10-01-2015-12-01-aman.log

# Perform the critical analysis steps that should be part of the
# production workflow:
tokenizeDescriptionsAndSaveSeq-dump-2015-10-01-2015-12-01-aman.sh \
| tee tokenizeDescriptionsAndSaveSeq-dump-2015-10-01-2015-12-01-aman.log

applyCrfSjSparkDescMemexHairEyes-dump-2015-10-01-2015-12-01-aman.sh \
| tee applyCrfSjSparkDescMemexHairEyes-dump-2015-10-01-2015-12-01-aman.log

applyCrfSjSparkDescMemexNameEthnic-dump-2015-10-01-2015-12-01-aman.sh \
| tee applyCrfSjSparkDescMemexNameEthnic-dump-2015-10-01-2015-12-01-aman.log

# Examine the output of the CRF analyses.  These steps may be omitted
# from a production workflow.
countGoodKeys-dump-2015-10-01-2015-12-01-aman-results-hair-eyes.sh \
| tee countGoodKeys-dump-2015-10-01-2015-12-01-aman-results-hair-eyes.log

countCrfResultTokens-dump-2015-10-01-2015-12-01-aman-results-hair-eyes.sh \
| tee countCrfResultTokens-dump-2015-10-01-2015-12-01-aman-results-hair-eyes.log

countCrfResultTokens-dump-2015-10-01-2015-12-01-aman-results-name-ethnic.sh \
| tee countCrfResultTokens-dump-2015-10-01-2015-12-01-aman-results-name-ethnic.log

countGoodKeys-dump-2015-10-01-2015-12-01-aman-results-name-ethnic.sh \
| tee countGoodKeys-dump-2015-10-01-2015-12-01-aman-results-name-ethnic.log

error: pathspec 'process-dump-2015-10-01-2015-12-01-aman.sh' did not match any file(s) known to git


script to run redis cluster locally

script to run redis cluster locally

error: pathspec 'run-redis-cluster.sh' did not match any file(s) known to git


Git creation + getopts + check assembly fasta + final batch after the loop

#!/bin/bash

# TO ADD :
#	- merge results in one fasta file !!
#	- add OutputDir options (currently create batches in current folder)
#	- create fata fai if not already there

# Runs pilon in batches, uses the fasta fai from the assembly to get the scaffolds IDs (then uses pilon --targets option to launch pilon on these Ids)
# Output is one folder for each batch

function usage
{
        echo "USAGE run_pilon_batches.sh -t [Threads] -a [Assembly fasta] -b [Batch size] -f [--frags align.bam] -p [pilon.jar]"
        echo "  -h Print this help message"
        echo "  -t Number of threads to use"
        echo "  -a Assembly in fasta format"
        echo "  -b batch size (how many contigs to process per pilon run)"
        echo "  -f indicate the bam files location, format : '--frags /path/to/file1.bam --frags /path/to/file2.bam'"
	echo "  -p path to pilon jar file"
}

# Default values
batchSize=100
threads=20

while getopts ht:a:b:f:p: opt; do
        case ${opt} in
                h)
                        usage
			exit 1
                ;;
                t)
                        threads=${OPTARG}
                ;;
                a)
                        assemblyFasta=${OPTARG}
                ;;
                b)
                        batchSize=${OPTARG}
                ;;
                f)
                        FRAGS=${OPTARG}
		;;
		p)
			pilonJar=${OPTARG}
                ;;
                \?)
                        echo "Invalid option: -${OPTARG}"
                        exit 1
                ;;
                :)
                        echo "Option -${OPTARG} requires an argument"
                        exit 1
                ;;
        esac
done

# Check user input
if [[ ! -r ${assemblyFasta} ]]; then
	echo ""
	echo "Cannot read the Assembly fasta file: ${assemblyFasta}"
	echo ""
	usage
	exit 1
fi

fastaFaiFile=${assemblyFasta}".fai"
if [[ ! -r ${fastaFaiFile} ]]; then
	echo ""
	echo "Cannot read the fasta.fai file (must be next to the fasta file and have the same name): ${fastaFaiFile}"
	echo ""
	usage
	exit 1
fi

##### Add tests for other inputs (threads --frags etc...)

# Variable for each Batch
count=0
batch=""
batchNumber=0

# Get the list of scaffolds names (fasta fai first column)
scaffoldIDs=`cut -f 1 ${fastaFaiFile}`

for ID in ${scaffoldIDs}; do

	count=$((${count}+1))

	# To avoid having a comma at the beginning (and avoid pilon throwing an error)
	batch="${batch},${ID}"

	# Launch Pilon every "batchSize" IDs
	if [ ${count} -ge ${batchSize} ]; then

		batchNumber=$((${batchNumber}+1))

		cmd="java -jar ${pilonJar} --genome ${assemblyFasta} ${FRAGS} --output pilon_on_batch${batchNumber} --outdir pilon_on_batch${batchNumber}/ --changes --fix all --threads ${threads} --targets ${batch} > pilon_on_batch${batchNumber}.log"

		echo ${cmd}
		eval ${cmd}

		batch=""
		count=0

		echo "BATCH ${batchNumber} Done !"
		echo ""
	fi

done

# The last batch for the Ids left (not useful if number of scaffolds is modulo the batch size)
if [ "$batch" != "" ]; then
	batchNumber=$((${batchNumber}+1))

	cmd="java -jar ${pilonJar} --genome ${assemblyFasta} ${FRAGS} --output pilon_on_batch${batchNumber} --outdir pilon_on_batch${batchNumber}/ --changes --fix all --threads ${threads} --targets ${batch} > pilon_on_batch${batchNumber}.log"

	echo ${cmd}
	eval ${cmd}

	echo "BATCH ${batchNumber} Done !\n\n"
fi

echo "All the batches have been processed !\n"

# Merge fasta here

error: pathspec 'run_pilon_batches.sh' did not match any file(s) known to git


Add overlays

Add overlays

overlay fiat_crypto https://github.com/proux01/fiat-crypto rm_deprecated_813 14819
overlay metacoq https://github.com/proux01/metacoq rm_deprecated_813 14819

error: pathspec 'dev/ci/user-overlays/14819-proux01-rm_deprecated_813.sh' did not match any file(s) known to git


Add script to update examples archetype automatically

remote: Support for password authentication was removed on August 13, 2021.
remote: Please see https://docs.github.com/en/get-started/getting-started-with-git/about-remote-repositories#cloning-with-https-urls for information on currently recommended modes of authentication.
fatal: Authentication failed for 'https://github.com/wangyum/beam.git/'