new script to get mean functions with motion correction

#!/usr/bin/env bash

#  This script gets the mean across a set of functional images by
#  first getting a mean for each motion corrected run
#  and then getting the mean of the motion corrected means
#
# For this script, a user should only need to run the following:
#
#     $0 -o outprefix -w working-directory -i func-file1 [... func-fileN]
#
#
# All the commands with some useful logging information should all be saved.

#### Usage ####

source ${GUNTHERDIR}/include/cmdarg.sh

cmdarg_info "header" "Script for functional motion correction"
cmdarg_info "author" "McCarthy Lab <some address>"
## required inputs
cmdarg "i:" "inputs" "Path to functional runs to motion correct"
cmdarg "o:" "output" "Output mean functional file"
cmdarg "w:" "workdir" "Path to working directory"
## optional inputs
cmdarg "n" "njobs" "Number of parallel jobs" 1
cmdarg "k" "keepwdir" "Keep working directory" false
cmdarg "f" "force" "Will overwrite any existing output" false
cmdarg "l?" "log" "Log file"
## parse
cmdarg_parse "$@"
[ $# == 0 ] && exit

#### Set Variables ####

inputs=( ${cmdarg_cfg['inputs']} )
output=${cmdarg_cfg['output']}
workdir=${cmdarg_cfg['workdir']}
keepwdir=${cmdarg_cfg['keepwdir']}
overwrite=${cmdarg_cfg['force']}
njobs==${cmdarg_cfg['njobs']}
_LOG_FILE=${cmdarg_cfg['log']}
[ ! -z $_LOG_FILE ] && _LOG_FILE=$( readlink -f ${_LOG_FILE} ) # absolute path (if exists)

ext=".nii.gz"

nruns=${#inputs[@]}
outdir=$(dirname $outprefix)
echo "$nruns runs"

old_afni_deconflict=$AFNI_DECONFLICT
if [ $overwrite == true ]; then
  export AFNI_DECONFLICT="OVERWRITE"
fi

#### Log ####

source ${GUNTHERDIR}/include/log.sh

check_logfile

log_echo ""
log_echo "RUNNING: $0 $@"

#### Checks/Setup ####

source ${GUNTHERDIR}/include/io.sh

check_inputs ${inputs[@]}
check_outputs $overwrite "$workdir" "$output"
[ ! -e $workdir ] && mkdir -p $workdir

# get full paths since changing paths
workdir=$( readlink -f ${workdir} )
output=$( readlink -f ${output} )
for (( i = 0; i < ${#inputs[@]}; i++ )); do
  inputs[$i]=$( readlink -f ${inputs[$i]} )
done

# recheck inputs
check_inputs ${inputs[@]}

# Work in the working directory
log_cmd2 "cd $workdir"

###
# First Pass
###

log_echo "=== First Pass"

# Get the range of scan numbers with padding of 2
ris=()
for i in `seq 1 $nruns`; do
  ri=`echo $i | awk '{printf "%02d", $1}'`
  ris+=($ri)
done

log_echo "Generate motion-corrected mean EPIs"

log_echo "...mean EPI of non-motion corrected data"
log_tcmd "parallel --xapply --no-notice -j $njobs --eta 3dTstat -mean -prefix iter0_mean_epi_r{1}${ext} {2} ::: ${ris[*]} ::: ${inputs[*]}"

log_echo "...apply motion correction to mean EPI"
log_tcmd "parallel --xapply --no-notice -j $njobs --eta 3dvolreg -verbose -zpad 4 -base iter0_mean_epi_r{1}${ext} -prefix iter0_epi_volreg_r{1}${ext} -cubic {2} ::: ${ris[*]} ::: ${inputs[*]}"

log_echo "...new mean EPI of motion-corrected data"
log_tcmd "parallel --xapply --no-notice -j $njobs --eta 3dTstat -mean -prefix iter1_mean_epi_r{1}${ext} {2} ::: ${ris[*]} ::: ${inputs[*]}"

#for i in `seq 1 $nruns`; do
#  ri=`echo $i | awk '{printf "%02d", $1}'`
#  input=${inputs[i-1]}
#  log_echo "Generate motion-corrected mean EPI for run ${ri}"
#  # Mean EPI of non-motion-corrected data
#  log_tcmd "3dTstat -mean -prefix iter0_mean_epi_r${ri}${ext} ${input}"
#  # Apply motion correction to mean EPI
#  log_tcmd "3dvolreg -verbose -zpad 4 -base iter0_mean_epi_r${ri}${ext} -prefix iter0_epi_volreg_r${ri}${ext} -cubic ${input}"
#  # New mean EPI of motion-corrected data
#  log_tcmd "3dTstat -mean -prefix iter1_mean_epi_r${ri}${ext} iter0_epi_volreg_r${ri}${ext}"
#done

# Combine mean EPIs from each run
log_echo "Combine mean EPIs from each run"
log_tcmd "3dTcat -prefix iter1_mean_epis${ext} iter1_mean_epi_r*${ext}"

# Get mean EPI across runs
log_echo "Get mean EPI across runs"
log_tcmd "3dTstat -mean -prefix iter1_mean_mean_epi${ext} iter1_mean_epis${ext}"

###
# Second Pass
##

log_echo "=== Second Pass"
# Register mean EPIs from each run to each other
log_echo "Motion correct the mean EPIs"
log_tcmd "3dvolreg -verbose -zpad 4 -base iter1_mean_mean_epi${ext} -prefix iter2_mean_epis_volreg${ext} -cubic iter1_mean_epis${ext}"

# Take the mean of motion-corrected mean EPIs
log_echo "Get mean EPI across the mean EPIs"
log_tcmd "3dTstat -mean -prefix iter2_mean_mean_epi${ext} iter2_mean_epis_volreg${ext}"

###
# Wrap Up
##

log_echo "=== Clean up"

log_tcmd "3dcopy iter2_mean_mean_epi${ext} ${output}"

if [ ${keepwdir} == false ]; then
  log_echo "Removing working directory"
  log_cmd "rm ${workdir}/*"
  log_cmd "rmdir ${workdir}"
fi

# Unset AFNI_DECONFLICT
if [ $overwrite == true ]; then
  export AFNI_DECONFLICT=$old_afni_deconflict
fi

error: pathspec 'func_mean_functionals.sh' did not match any file(s) known to git


adds: basic test for k.

adds: basic test for k.

#!/bin/bash

PASS="PASS"
FAIL="FAILED"

function test_start() {
  k.
  if [ $? -eq 0 ]; then
    echo "-> k. is installed"
  else
    echo "-> k. is not installed. Exiting..."
    exit 1
  fi

  echo "-> Running tests..."
}

function test_done() {
  echo "-> All tests done."
}

function assert_text(){
  if [ "$1" != "$2" ]; then
    echo $FAIL
  else
    echo $PASS
  fi
}

function test() {
  echo "-> $1"
  RESULT=$2
  echo $RESULT
}

test_start

# Assert that calling upon k. will return bye
COMMAND=`k.`
ANSWER="bye"
test "Test 1: Expecting 'k.' to answer 'bye'" `assert_text $COMMAND $ANSWER`

test_done

error: pathspec 'test/basic_test.sh' did not match any file(s) known to git


add scripts

add scripts

error: pathspec 'create_user.sh' did not match any file(s) known to git


First commit, takes the data from file with mock raw dab exported data and creates csv

#!/usr/bin/env bash

# start from
#aa=(adb shell content query --uri content://com.android.calendar/events --where "dtstart > '1424131251000' and dtend < '1424217651000'" ))

#aa=(adb shell content query --uri content://com.android.calendar/events --where "allDay = 1 and dtstart > '1424044851000' and dtend < '1424304051000'" ))

# adb shell "
# content query --uri content://com.android.calendar/events
# " > res.txt

#echo `adb shell`
#echo `content query --uri content://com.android.calendar/events --where "dtstart > '1424131251000' and dtend < '1424217651000'"`

DEBUG=true
outputFileName="result.csv"

# clean the file
> $outputFileName

strindex() {
  x="${1%%$2*}"
  [[ $x = $1 ]] && echo -1 || echo ${#x}
}

# function to convert android row to csv row
# row example:
#	Row: 0 sync_data10=NULL, cal_sync6=NULL, rrule=NULL, sync_data6=NULL, cal_sync8=1423664730492, eventTimezone=Asia/Jerusalem, _sync_id=pcr4ma0g31n6m0e76c6cno4i10_20150217T080000Z, hasAttendeeData=1, customAppPackage=NULL, originalInstanceTime=1424160000000, sync_data2=NULL, allowedReminders=0,1,2, calendar_timezone=Asia/Jerusalem, uid2445=NULL, dirty=0, originalAllDay=0, cal_sync10=NULL, calendar_color=-6306073, exrule=NULL, cal_sync3=NULL, lastDate=1424161800000, canOrganizerRespond=1, guestsCanSeeGuests=1, rdate=NULL, account_type=com.google, eventEndTimezone=NULL, selfAttendeeStatus=0, cal_sync2=NULL, mutators=NULL, exdate=NULL, hasExtendedProperties=0, calendar_color_index=15, organizer=nir@everything.me, eventColor_index=NULL, sync_data9=0, cal_sync5=0, eventColor=NULL, cal_sync4=1, availability=0, ownerAccount=nir@everything.me, dtstart=1424160000000, lastSynced=0, duration=NULL, accessLevel=0, maxReminders=5, displayColor=-6306073, allDay=0, dtend=1424161800000, eventStatus=1, sync_data3=NULL, sync_data4="2835027570380000", original_id=NULL, _id=1215, guestsCanModify=0, calendar_access_level=100, customAppUri=NULL, calendar_displayName=nir@everything.me, sync_data5=2014-12-02T09:49:45.000Z, guestsCanInviteOthers=1, sync_data8=NULL, cal_sync9=NULL, original_sync_id=pcr4ma0g31n6m0e76c6cno4i10, cal_sync1=nir@everything.me, cal_sync7=NULL, canModifyTimeZone=1, visible=1, allowedAttendeeTypes=0,1,2, sync_data1=pcr4ma0g31n6m0e76c6cno4i10@google.com, allowedAvailability=0,1, description=, title=, calendar_id=11, sync_data7=NULL, deleted=0, eventLocation=, account_name=roman@everything.me, hasAlarm=0, isOrganizer=1
toCsv() {
	# $1 - the row (string)
	# $2 - print columns (boolean)

	columns=()
	values=()

	showColumns=$2
	i=0

	# the key value
	keyValue=""
	next=false

	for item in $1
	do
		# skip the first two (they will be Row\n<number>)
		if [ $i -eq 0 -o $i -eq 1 ]; then
			let "i+=1"
			continue
		fi

		# if [ $2 = true ]; then
		# 	echo $item
		# fi

		next=false

		# last char
		concat="${keyValue}${item}"
		# concat=$(printf "%s %s" "$keyValue $item")
		keyValue=$(printf "%s" "$concat")
		#$(printf "%s %s" "$keyValue" "$item")
		# if [ $2 = true ]; then
		# 	echo $keyValue
		# fi
		lastChar="${item: -1}"
		if [ $lastChar = "," ]; then
			let "i+=1"
			next=true
		fi

		if [ $next = true ]; then
			# if [ $2 = true ]; then
			# 	echo $keyValue
			# 	echo "---"
			# fi

			position=$(strindex "$keyValue" "=")
			column=${keyValue:0:position}
			value=${keyValue:position+1}

			# DO IT ONLY FOR CSV
			# except last char and replace all commas in value to space
			value="${value%?}"
			value=$(tr ',' ' ' <<< "$value")
			value="${value},"

			columns+=($column)
			values+=($value)

			next=false
			keyValue=""
		fi

	done

	# add last one
	position=$(strindex "$keyValue" "=")
	column=${keyValue:0:position}
	value=${keyValue:(position+1)}
	columns+=($column)
	values+=($value)
	let "i+=1"

	# check if we need to print columns
	if [ $2 == true ]; then

		# print columns with comma separated
		cols=${columns[*]}

		# print to file
		echo ${cols// /,} >> $outputFileName
	fi

	# print values with comma separated
	vals=${values[*]}

	# print to file end escape new lines
	echo ${vals} | tr "\n\r" " " >> $outputFileName
	echo -n $'\n' >> $outputFileName

}

# some dificulties:
# - it could be new lines in the middle and '+=' doesn't work with just $line. we should use printf string format
# - we can't match by '=' or by ',' since these chars could be part of the field value itself
# - i don't want to match with predefined field names, but keep it dynamic
# - new lines in values

file="raw_query.txt"
seperator="Row:"

# prepare all rows in this array
declare -a rows

# rows counter
count=-1

# let's read line be line and build rows
while read -r line
do
	# it works as follow:
	# - we check if row starts with 'Row:', then we can assume that this is a new row (not pretty bad assumtion after many tests)
	# - the line that doesn't start with 'Row:' is something that contains fields and data whuch are belong to previous row

	# the position of 'Row:' in the line
    index=$(strindex "$line" "$seperator")
    if [ $index -eq 0 ]; then
    	# this is a new row
    	let "count+=1"
    	rows[$count]=$(printf "%s" "$line")
    else
    	# this is still the previous row
    	rows[$count]+=$(printf "%s" "$line")
    fi
done < "$file"

# print all rows
if [ $DEBUG = true ]; then
	for i in "${!rows[@]}"
	do
		echo ${rows[i]}
	done
	echo "===================="
fi

# time to fetch all playing field names :)
# the way to do it:
# - we take first row
# - iterate over items and check if item ends with ','
for i in "${!rows[@]}"
do
	toCsv "${rows[i]}" $(if [ $i -eq 0 ]; then echo "true"; else echo "false"; fi)
done

# declare -a columns
# firstRow=${rows[5]}
# for item in $firstRow; do
# 	echo $item
# done

# export results
echo "Result:"
echo $(printf "Num of rows: %d" "${#rows[@]}")

error: pathspec 'adb-export.sh' did not match any file(s) known to git


add build-nginx.sh

add build-nginx.sh

apt-get install libpcre3-dev libluajit-5.1-dev libcurl4-openssl-dev libssl-dev libboost-regex1.55-dev libboost-system1.55-dev libboost-thread1.55-dev libcrypto++-dev zlib1g-dev

curl -L https://github.com/RedPitaya/RedPitaya/archive/v0.95-1.tar.gz -o RedPitaya-0.95-1.tar.gz
tar -zxf RedPitaya-0.95-1.tar.gz
cd RedPitaya-0.95-1

cat <<- EOF_CAT > patches/lua-nginx-module.patch
diff -rupN lua-nginx-module.old/config lua-nginx-module.new/config
--- lua-nginx-module.old/config
+++ lua-nginx-module.new/config
@@ -295,8 +311,8 @@ fi
 NGX_DTRACE_PROVIDERS="$NGX_DTRACE_PROVIDERS $ngx_addon_dir/dtrace/ngx_lua_provider.d"
 NGX_TAPSET_SRCS="$NGX_TAPSET_SRCS $ngx_addon_dir/tapset/ngx_lua.stp"

-USE_MD5=YES
-USE_SHA1=YES
+USE_MD5=NO
+USE_SHA1=NO

 CORE_INCS="$CORE_INCS $ngx_addon_dir/src/api"

diff -rupN lua-nginx-module.old/src/ngx_http_lua_socket_tcp.c lua-nginx-module.new/src/ngx_http_lua_socket_tcp.c
--- lua-nginx-module.old/src/ngx_http_lua_socket_tcp.c
+++ lua-nginx-module.new/src/ngx_http_lua_socket_tcp.c
@@ -3146,8 +3146,7 @@ ngx_http_lua_req_socket(lua_State *L)
     }

     lua_settop(L, 1);
-    lua_pushnil(L);
-    return 2;
+    return 1;
 }

EOF_CAT

make Bazaar/tools/libjson
make Bazaar/nginx/ngx_ext_modules/ws_server/websocketpp
make Bazaar/nginx/ngx_ext_modules/lua-nginx-module
make Bazaar/nginx/nginx-1.5.3

make -C shared

patch -d Bazaar/nginx/ngx_ext_modules/ws_server/rp_sdk -p1 <<- EOF_PATCH
diff -rupN rp_sdk.old/Makefile rp_sdk.new/Makefile
--- rp_sdk.old/Makefile
+++ rp_sdk.new/Makefile
@@ -22,7 +22,7 @@ CRYPTO_DIR=../../../../tools/cryptopp
 CRYPTO_INSTALL_DIR=../../../../tools/build

 CXX=\$(CROSS_COMPILE)g++
-CXXFLAGS=-c -Wall -O0 -static -std=c++11 -fPIC -I\$(LIBJSON_DIR) -I\$(CRYPTO_INSTALL_DIR)/include/cryptopp -DNDEBUG
+CXXFLAGS=-c -Wall -O0 -static -std=c++11 -fPIC -I\$(LIBJSON_DIR) -I/usr/include/crypto++ -DNDEBUG
 ifeq (\$(ALWAYS_PURCHASED),true)
 CXXFLAGS+=-DALWAYS_PURCHASED
 endif
@@ -51,9 +51,7 @@ \$(CRYPTO_LIB):
 	make -C \$(CRYPTO_DIR) CXX=\${CROSS_COMPILE}g++ PREFIX=../build install

 \$(LIB): \$(OBJECTS)
-	mkdir -p \$(OBJDIR)/cryptopp
-	cd \$(OBJDIR)/cryptopp; ar -x \$(CURRENT_DIR)/\$(CRYPTO_LIB)
-	ar rc \$(LIB) objs/cryptopp/*.o \$(OBJECTS)
+	ar rc \$(LIB) \$(OBJECTS)

 \$(SDKOBJDIR)/%.o: %.cpp
 	mkdir -p \$(dir \$@)
EOF_PATCH

make -C Bazaar/nginx/ngx_ext_modules/ws_server/rp_sdk librp_sdk.a

patch -d Bazaar/nginx/ngx_ext_modules/ws_server/websocketpp/websocketpp -p1 <<- EOF_PATCH
diff -rupN websocketpp.old/transport/asio/endpoint.hpp websocketpp.new/transport/asio/endpoint.hpp
--- websocketpp.old/transport/asio/endpoint.hpp
+++ websocketpp.new/transport/asio/endpoint.hpp
@@ -95,7 +95,7 @@ public:
     explicit endpoint()
       : m_io_service(NULL)
       , m_external_io_service(false)
-      , m_listen_backlog(0)
+      , m_listen_backlog(boost::asio::socket_base::max_connections)
       , m_reuse_addr(false)
       , m_state(UNINITIALIZED)
     {
EOF_PATCH

make -C Bazaar/nginx/ngx_ext_modules/ws_server

cd Bazaar/nginx/nginx-1.5.3
./configure `cat ../configure_withouts.txt` --add-module=../ngx_ext_modules/lua-nginx-module --add-module=../ngx_ext_modules/ngx_http_rp_module
make
cp objs/nginx /opt/redpitaya/sbin/nginx
strip /opt/redpitaya/sbin/nginx
cd -

patch -d Applications/scopegenpro/src -p1 <<- EOF_PATCH
diff -rupN src.old/Makefile src.new/Makefile
--- src.old/Makefile
+++ src.new/Makefile
@@ -3,7 +3,7 @@ RM=rm

 CXXSOURCES=main.cpp

-RP_API=../../../api/lib
+RP_API=/opt/redpitaya/lib
 RP_SDK=../../../Bazaar/nginx/ngx_ext_modules/ws_server/rp_sdk

 INCLUDE = -I\$(RP_SDK)
@@ -14,7 +14,7 @@ COMMON_FLAGS+=-Wall -fPIC -lstdc++ -Os -s
 CXXFLAGS+=\$(COMMON_FLAGS) -std=c++11 \$(INCLUDE)
 LDFLAGS =-shared \$(COMMON_FLAGS) -L\$(RP_SDK)/lib
 LDFLAGS+= -Wl,--whole-archive
-LDFLAGS+=-L\$(RP_SDK) -lrp_sdk
+LDFLAGS+=-L\$(RP_SDK) -lrp_sdk -lcryptopp
 LDFLAGS+=-L\$(RP_API) -lrpapp -lrp
 LDFLAGS+= -Wl,--no-whole-archive

EOF_PATCH

make -C Applications/scopegenpro/src
cp Applications/scopegenpro/controllerhf.so /opt/redpitaya/www/apps/scopegenpro/controllerhf.so

error: pathspec 'scripts/build-nginx.sh' did not match any file(s) known to git


Add script for converting the MS Access database to Postgres

#!/bin/bash

ACC_FILE=kore_tietokanta_ta.accdb
DB=test

mdb-schema $ACC_FILE postgres > schema.sql
cat schema.sql | grep -v CONSTRAINT | grep -v INDEX | psql -q $DB

rm -f bool-columns.txt

echo Sniffing out boolean columns
for t in $(mdb-tables $ACC_FILE) ; do
	for c in $(echo \\d \"$t\" | psql $DB | grep boolean| cut -d '|' -f 1) ; do
		echo $t $c >> bool-columns.txt
	done
done

echo Changing boolean columns to integers
while read -r t c; do
	echo "ALTER TABLE \"$t\" ALTER COLUMN $c TYPE int USING 0;" | psql -q $DB
done < bool-columns.txt

for t in $(mdb-tables $ACC_FILE) ; do
	echo Exporting $t
	mdb-export -I postgres -b strip -q \' $ACC_FILE $t | psql -q $DB
done

echo Creating constraints and indices
cat schema.sql | grep -e CONSTRAINT -e INDEX | psql -q $DB
rm schema.sql

echo Changing columns back to booleans
while read -r t c; do
	echo "ALTER TABLE \"$t\" ALTER COLUMN $c TYPE boolean USING CASE WHEN $c=0 THEN FALSE ELSE TRUE END;" | psql -q $DB
done < bool-columns.txt

rm bool-columns.txt

echo Creating fake primary keys
SQL="
ALTER TABLE \"Rakennuksen_status\" ADD COLUMN id VARCHAR;
UPDATE \"Rakennuksen_status\" SET id = koulun_id || '-' || rakennuksen_id;
ALTER TABLE \"Rakennuksen_status\" DROP CONSTRAINT IF EXISTS \"Rakennuksen_status_pkey\";
ALTER TABLE \"Rakennuksen_status\" ADD PRIMARY KEY (id);
"

psql -q $DB -c "$SQL"

for t in $(psql -c "\di" $DB | grep _pkey | cut -d '|' -f 2 | sed -e 's/_pkey//') ; do
	echo "ALTER TABLE \"$t\" ADD COLUMN id SERIAL;" | psql -q $DB
done

error: pathspec 'scripts/export-psql.sh' did not match any file(s) known to git


Create fix_eth0_ip.sh

Create fix_eth0_ip.sh

## Requirement
if there is ip address on eth0 interface, save it some where
if there is no ip address on eth0 for n seconds
  configure eth0 with last successful ip address manually
this is the dhcp over lay mode

#!/bin/bash

ETH0_IP=$(ifconfig eth0 |grep 'inet addr'|awk '{print $2}' | awk -F: '{print $2}')

SAVED_IP_FILE=/var/run/saved_ip

if [[ -n "${ETH0_IP}" ]]
then
    /bin/rm -f ${SAVED_IP_FILE}
    echo $ETH0_IP > ${SAVED_IP_FILE}
    logger "IP for eth0 detected(${ETH0_IP}), saved in ${SAVED_IP_FILE}"
elif [[ -f "${SAVED_IP_FILE}" ]]
then
    /sbin/ifconfig eth0 $(cat ${SAVED_IP_FILE})
    logger "WARN:NO IP for eth0, set to saved IP - ${SAVED_IP}"
else
    logger "ERROR:NO IP for eth0, and NO saved IP."
fi

error: pathspec 'scripts/fix_eth0_ip.sh' did not match any file(s) known to git


Add mesa recipe

Add mesa recipe

GIT=https://gitlab.redox-os.org/redox-os/mesa.git
GIT_UPSTREAM=git://anongit.freedesktop.org/mesa/mesa
BUILD_DEPENDS=(expat zlib)

function recipe_update {
    echo "skipping update"
    skip=1
}

function recipe_build {
    #NOCONFIGURE=1 ./autogen.sh
    sysroot="${PWD}/../sysroot"
    export CFLAGS="-I$sysroot/include -DHAVE_PTHREAD=1"
    export CPPFLAGS="-I$sysroot/include -DHAVE_PTHREAD=1"
    export LDFLAGS="-L$sysroot/lib"
    EXPAT_LIBS="-lexpat" EXPAT_CFLAGS="." \
    ./configure --host=${HOST} --prefix=/ \
        --disable-gles1 \
        --disable-gles2 \
        --disable-dri \
        --disable-dri3 \
        --disable-glx \
        --disable-egl \
        --disable-driglx-direct \
        --disable-gbm \
        --disable-llvm \
        --enable-gallium-osmesa \
        --with-gallium-drivers=swrast \
        --with-platforms=surfaceless
    make -j"$(nproc)"
    skip=1
}

function recipe_test {
    echo "skipping test"
    skip=1
}

function recipe_clean {
    make clean
    skip=1
}

function recipe_stage {
    dest="$(realpath $1)"
    make DESTDIR="$dest" install
    skip=1
}

error: pathspec 'recipes/mesa/recipe.sh' did not match any file(s) known to git


New script to open errata text window

New script to open errata text window

remote: Support for password authentication was removed on August 13, 2021.
remote: Please see https://docs.github.com/en/get-started/getting-started-with-git/about-remote-repositories#cloning-with-https-urls for information on currently recommended modes of authentication.
fatal: Authentication failed for 'https://github.com/jfriesse/hsu.git/'


Create prereq.sh

Create prereq.sh

#!/bin/bash

function otherpkgs()
{
  step "Installating additional packages: "
  try apt-get update 1>/dev/null
  try apt-get install whois iputils-traceroute iputils-ping dnsutils arpwatch netcat socat nmap ngrep tcpdump --force-yes 1>/dev/null
  try apt-get install git htop sudo nano --force-yes 1>/dev/null
  next
}

function sources()
{
  step "Setting up apt sources"
  try mkdir ./backup
  try mv /etc/apt/sources.list ./backup/sources.list

  try cat >/etc/apt/sources.list <<EOL
# Main
deb  https://deb.debian.org/debian jessie main
deb-src  https://deb.debian.org/debian jessie main
# Updates
deb  https://deb.debian.org/debian jessie-updates main
deb-src  https://deb.debian.org/debian jessie-updates main
# Security
deb https://security.debian.org/ jessie/updates main
deb-src https://security.debian.org/ jessie/updates main
EOL

  try netselect-apt 1>/dev/null
  try apt-get update 1>/dev/null
  next
}

# Source Check
_LOADED_PREREQ=1

error: pathspec 'prereq.sh' did not match any file(s) known to git


fixup! generate-signkeys.sh: encrypt private key

remote: Support for password authentication was removed on August 13, 2021.
remote: Please see https://docs.github.com/en/get-started/getting-started-with-git/about-remote-repositories#cloning-with-https-urls for information on currently recommended modes of authentication.
fatal: Authentication failed for 'https://github.com/DarkCaster/Linux-Helper-Tools.git/'


Add doc publishing script

Add doc publishing script

#!/bin/bash
# From: http://happygiraffe.net/blog/2009/07/04/publishing-a-subdirectory-to-github-pages/

doc_sha=$(git ls-tree -d HEAD doc | awk '{print $3}')
new_commit=$(echo "Auto-update docs." | git commit-tree $doc_sha -p refs/heads/gh-pages)
git update-ref refs/heads/gh-pages $new_commit

error: pathspec 'pubdoc.sh' did not match any file(s) known to git


add reload pelias scripts

add reload pelias scripts

# step 1: download transit, OSM and OR-WA data
cd /mnt/pelias/
mkdir ./old
mv openaddresses  openstreetmap  transit ./old/

mkdir /mnt/pelias/transit
cd /mnt/pelias/transit
scp otp@maps7:~/loader/ott/loader/gtfs/cache/*.zip .

mkdir /mnt/pelias/openstreetmap
cd /mnt/pelias/openstreetmap
scp otp@maps7:~/loader/ott/loader/osm/cache/or-wa.pbf .

mkdir /mnt/pelias/openaddresses
cd /mnt/pelias/openaddresses
wget https://s3.amazonaws.com/data.openaddresses.io/openaddr-collected-us_west.zip .
# FOR TESTING THIS SCRIPT >> cp ~/openaddr-collected-us_west.zip .
unzip openaddr-collected-us_west.zip

# step 2: clone the pelias.transit.loader and set up the config from that project as out pelias.json
cd ~/
mkdir ./old
mv pelias.json old/
rm pelias.json

cd ~/projects/
rm -rf pelias.transit.loader
git clone https://github.com/OpenTransitTools/pelias.transit.loader.git
ln -s ~/projects/pelias.transit.loader/pelias.json ~/pelias.json
cd ~/projects/pelias.transit.loader/
npm install
npm stop

# step 3: create new / empty index
cd ~/projects/schema
curl -XDELETE 'localhost:9200/pelias?pretty'
node scripts/create_index.js

# step 4: load the system...
#loaders=(pelias.transit.loader whosonfirst polylines geonames openstreetmap openaddresses)
loaders=(pelias.transit.loader openaddresses  openstreetmap polylines)
for l in "${loaders[@]}"
do
    echo $l
    cd ~/projects/$l/
    npm install
    npm start
done

error: pathspec 'reload.sh' did not match any file(s) known to git


Update bash script for building stuff

Update bash script for building stuff

repos=(
  "lib/background-size.js"
  "lib/border-radius.js"
  "lib/borders.js"
  "lib/box-sizing.js"
  "lib/clearfix.js"
  "lib/display.js"
  "lib/docs.js"
  "lib/floats.js"
  "lib/font-family.js"
  "lib/font-weight.js"
  "lib/heights.js"
  "lib/hovers.js"
  "lib/images.js"
  "lib/letter-spacing.js"
  "lib/line-height.js"
  "lib/max-widths.js"
  "lib/measure-avenir.js"
  "lib/measure-time.js"
  "lib/measure.js"
  "lib/position.js"
  "lib/spacing.js"
  "lib/text-align.js"
  "lib/text-decoration.js"
  "lib/text-transform.js"
  "lib/type-scale.js"
  "lib/typography.js"
  "lib/vertical-align.js"
  "lib/white-space.js"
  "lib/widths.js"
)

for repo in "${repos[@]}"
do
  node "${repo}"
done

error: pathspec 'script.sh' did not match any file(s) known to git


purego.sh: Omitted in last commit.

purego.sh: Omitted in last commit.

set -e
set -v

CGO_ENABLED=0 go test -purego true
CGO_ENABLED=0 go test -purego true -tags purego
CGO_ENABLED=1 go test -purego false
CGO_ENABLED=1 go test -purego true -tags purego

error: pathspec 'purego.sh' did not match any file(s) known to git


Add script to update and re-configure OS for release

#!/usr/bin/env bash

cd $(dirname $0)

echo Update OS

sudo apt-get update
sudo apt-get upgrade
sudo apt-get dist-upgrade

if [ ! -f "$JAVA_HOME/jre/lib/security/local_policy.jar" ]
then
echo "Enable strong crypto support for Java"

wget --no-check-certificate --no-cookies --header "Cookie: oraclelicense=accept-securebackup-cookie" http://download.oracle.com/otn-pub/java/jce/8/jce_policy-8.zip

checksum=f3020a3922efd6626c2fff45695d527f34a8020e938a49292561f18ad1320b59 # see https://github.com/jonathancross/jc-docs/blob/master/java-strong-crypto-test/README.md

if ! echo "$checksum jce_policy-8.zip" | sha256sum -c -;
then
    echo "Checksum failed" >&2
    exit 1
fi

unzip jce_policy-8.zip
sudo cp UnlimitedJCEPolicyJDK8/{US_export_policy.jar,local_policy.jar} $JAVA_HOME/jre/lib/security/
sudo chmod 664 $JAVA_HOME/jre/lib/security/{US_export_policy.jar,local_policy.jar}
sudo rm -rf UnlimitedJCEPolicyJDK8 jce_policy-8.zip
else
echo "Strong Crypto support for Java already available"
fi

bouncyCastleJar=bcprov-jdk15on-1.56.jar

if [ ! -f "$JAVA_HOME/jre/lib/ext/$bouncyCastleJar" ]
then
echo Configure Bouncy Castle

wget "http://central.maven.org/maven2/org/bouncycastle/bcprov-jdk15on/1.56/$bouncyCastleJar"
sudo mv $bouncyCastleJar $JAVA_HOME/jre/lib/ext/
sudo chmod 777 "$JAVA_HOME/jre/lib/ext/$bouncyCastleJar"
else
echo Bouncy Castle already configured
fi

error: pathspec 'package/linux/prepare-system.sh' did not match any file(s) known to git


lug: worker-script: add git.sh

lug: worker-script: add git.sh

#!/bin/bash

if [ ! -d "${LUG_path}/.git" ]; then
	git clone --mirror "$LUG_origin" "$LUG_path"
fi

cd "$LUG_path"
git pull origin --all
git update-server-info
git gc --auto
git repack -a -b -d

error: pathspec 'lug/worker-script/git.sh' did not match any file(s) known to git


Create upgrade-os.sh

Create upgrade-os.sh

#!/bin/bash
set -e

# check if we are root
if [ "$EUID" -ne 0 ] ; then
  echo
  echo "*** Please run as root ***"
  echo ""
	exit 1
fi

if [[ $# -ne 2 ]] ; then
  echo "-------------------------------------------"
  echo "Please supply the old and new distro name"
  echo "  upgrade-os.sh jessie stretch
  echo "-------------------------------------------"
  exit 1
fi

OLD_DISTRO=$1
DISTRO=$2

echo ""
echo "============================="
echo "| Setting Up ${DISTRO}"
echo "============================="
echo ""

# make sure everything is up-to-date
apt-get update
apt-get upgrade -y
apt-get dist-upgrade -y
apt-get autoremove -y

# change distro name
sed -i 's/${OLD_DISTRO}/${DISTRO}/g' /etc/apt/sources.list

# now do the actual upgrade
apt-get update
apt-get upgrade -y
apt-get dist-upgrade -y

echo ""
echo "============================================="
echo "| ${DISTRO} installed ... reboot in 3 sec"
echo "============================================="
echo ""

sleep 1
echo "."
sleep 1
echo "."
sleep 1
echo "."

reboot now

error: pathspec 'software/upgrade-os.sh' did not match any file(s) known to git


Make group FS subject

Make group FS subject

error: pathspec 'make_group_average_fs.sh' did not match any file(s) known to git


fix if statement

fix if statement

#!/bin/bash
set -e

NUM_CPUS=`nproc`
echo "###############"
echo "### Using ${NUM_CPUS} cores"

# setup some build variables
BUILD_USER=vagrant
BUILD_GROUP=vagrant
BUILD_ROOT=/var/kernel_build
BUILD_CACHE=$BUILD_ROOT/cache
ARM_TOOLS=$BUILD_CACHE/tools
LINUX_KERNEL=$BUILD_CACHE/linux-kernel
# LINUX_KERNEL_COMMIT=""
LINUX_KERNEL_COMMIT=1f58c41a5aba262958c2869263e6fdcaa0aa3c00
RASPBERRY_FIRMWARE=$BUILD_CACHE/rpi_firmware

if [ -d /vagrant ]; then
  # running in vagrant VM
  SRC_DIR=/vagrant
else
  # running in drone build
  SRC_DIR=`pwd`
  BUILD_USER=`whoami`
  BUILD_GROUP=`whoami`
fi

LINUX_KERNEL_CONFIGS=$SRC_DIR/kernel_configs

NEW_VERSION=`date +%Y%m%d-%H%M%S`
BUILD_RESULTS=$BUILD_ROOT/results/kernel-$NEW_VERSION

X64_CROSS_COMPILE_CHAIN=arm-bcm2708/gcc-linaro-arm-linux-gnueabihf-raspbian-x64

declare -A CCPREFIX
CCPREFIX["rpi1"]=$ARM_TOOLS/$X64_CROSS_COMPILE_CHAIN/bin/arm-linux-gnueabihf-
CCPREFIX["rpi2"]=$ARM_TOOLS/$X64_CROSS_COMPILE_CHAIN/bin/arm-linux-gnueabihf-

declare -A IMAGE_NAME
IMAGE_NAME["rpi1"]=kernel.img
IMAGE_NAME["rpi2"]=kernel7.img

function create_dir_for_build_user () {
    local target_dir=$1

    sudo mkdir -p $target_dir
    sudo chown $BUILD_USER:$BUILD_GROUP $target_dir
}

function setup_build_dirs () {
  for dir in $BUILD_ROOT $BUILD_CACHE $BUILD_RESULTS $ARM_TOOLS $LINUX_KERNEL $RASPBERRY_FIRMWARE; do
    create_dir_for_build_user $dir
  done
}

function clone_or_update_repo_for () {
  local repo_url=$1
  local repo_path=$2
  local repo_commit=$3

  if [ ! -z "${repo_commit}" ]; then
    rm -rf $repo_path
  fi
  if [ -d ${repo_path}/.git ]; then
    cd $repo_path && git pull
  else
    echo "Cloning $repo_path with commit $repo_commit"
    git clone --depth 1 $repo_url $repo_path
    if [ ! -z "${repo_commit}" ]; then
      git checkout -qf ${repo_commit}
    fi
  fi
}

function setup_arm_cross_compiler_toolchain () {
  echo "### Check if Raspberry Pi Crosscompiler repository at ${ARM_TOOLS} is still up to date"
  clone_or_update_repo_for 'https://github.com/raspberrypi/tools.git' $ARM_TOOLS ""
}

function setup_linux_kernel_sources () {
  echo "### Check if Raspberry Pi Linux Kernel repository at ${LINUX_KERNEL} is still up to date"
  clone_or_update_repo_for 'https://github.com/raspberrypi/linux.git' $LINUX_KERNEL $LINUX_KERNEL_COMMIT
}

function setup_rpi_firmware () {
  echo "### Check if Raspberry Pi Firmware repository at ${LINUX_KERNEL} is still up to date"
  clone_or_update_repo_for 'https://github.com/asb/firmware' $RASPBERRY_FIRMWARE ""
}

function prepare_kernel_building () {
  setup_build_dirs
  setup_arm_cross_compiler_toolchain
  setup_linux_kernel_sources
  setup_rpi_firmware
}

create_kernel_for () {
  echo "###############"
  echo "### START building kernel for ${PI_VERSION}"

  local PI_VERSION=$1

  cd $LINUX_KERNEL

  # add kernel branding for hyprOS
  sed -i 's/^EXTRAVERSION =.*/EXTRAVERSION = -hypriotos/g' Makefile

  # save git commit id of this build
  local KERNEL_COMMIT=`git rev-parse HEAD`
  echo "### git commit id of this kernel build is ${KERNEL_COMMIT}"

  # clean build artifacts
  make ARCH=arm clean

  # copy kernel configuration file over
  cp $LINUX_KERNEL_CONFIGS/${PI_VERSION}_docker_kernel_config $LINUX_KERNEL/.config

  echo "### building kernel"
  mkdir -p $BUILD_RESULTS/$PI_VERSION
  echo $KERNEL_COMMIT > $BUILD_RESULTS/kernel-commit.txt
  ARCH=arm CROSS_COMPILE=${CCPREFIX[$PI_VERSION]} make -j$NUM_CPUS -k
  cp $LINUX_KERNEL/arch/arm/boot/Image $BUILD_RESULTS/$PI_VERSION/${IMAGE_NAME[${PI_VERSION}]}

  echo "### building kernel modules"
  mkdir -p $BUILD_RESULTS/$PI_VERSION/modules
  ARCH=arm CROSS_COMPILE=${CCPREFIX[${PI_VERSION}]} INSTALL_MOD_PATH=$BUILD_RESULTS/$PI_VERSION/modules make modules_install -j$NUM_CPUS

  # remove symlinks, mustn't be part of raspberrypi-bootloader*.deb
  echo "### removing symlinks"
  rm -f $BUILD_RESULTS/$PI_VERSION/modules/lib/modules/*/build
  rm -f $BUILD_RESULTS/$PI_VERSION/modules/lib/modules/*/source

  echo "### building deb packages"
  KBUILD_DEBARCH=armhf ARCH=arm CROSS_COMPILE=${CCPREFIX[${PI_VERSION}]} make deb-pkg
  mv ../*.deb $BUILD_RESULTS
  echo "###############"
  echo "### END building kernel for ${PI_VERSION}"
  echo "### Check the $BUILD_RESULTS/$PI_VERSION/kernel.img and $BUILD_RESULTS/$PI_VERSION/modules directory on your host machine."
}

function create_kernel_deb_packages () {
  echo "###############"
  echo "### START building kernel DEBIAN PACKAGES"

  PKG_TMP=`mktemp -d`

  NEW_KERNEL=$PKG_TMP/raspberrypi-kernel-${NEW_VERSION}

  create_dir_for_build_user $NEW_KERNEL

  # copy over source files for building the packages
  cp -r $RASPBERRY_FIRMWARE/* $NEW_KERNEL
  cp -r $SRC_DIR/debian $NEW_KERNEL/debian
  touch $NEW_KERNEL/debian/files

  for pi_version in ${!CCPREFIX[@]}; do
    cp $BUILD_RESULTS/$pi_version/${IMAGE_NAME[${pi_version}]} $NEW_KERNEL/boot
    cp -R $BUILD_RESULTS/$pi_version/modules/lib/modules/*hypriotos* $NEW_KERNEL/modules
  done

  # build debian packages
  cd $NEW_KERNEL

  dch -v ${NEW_VERSION} --package raspberrypi-firmware 'add Hypriot custom kernel'
  debuild --no-lintian -ePATH=${PATH}:$ARM_TOOLS/$X64_CROSS_COMPILE_CHAIN/bin -b -aarmhf -us -uc
  cp ../*.deb $BUILD_RESULTS

  echo "###############"
  echo "### FINISH building kernel DEBIAN PACKAGES"
}

##############
###  main  ###
##############

# setup necessary build environment: dir, repos, etc.
prepare_kernel_building

# create kernel, associated modules
for pi_version in ${!CCPREFIX[@]}; do
  create_kernel_for $pi_version
done

# create kernel packages
create_kernel_deb_packages

# running in vagrant VM
if [ -d /vagrant ]; then
  # copy build results to synced vagrant host folder
  FINAL_BUILD_RESULTS=/vagrant/build_results/$NEW_VERSION
else
  # running in drone build
  FINAL_BUILD_RESULTS=$SRC_DIR/output/$NEW_VERSION
fi

echo "###############"
echo "### Copy deb packages to $FINAL_BUILD_RESULTS"
mkdir -p $FINAL_BUILD_RESULTS
cp $BUILD_RESULTS/*.deb $FINAL_BUILD_RESULTS
cp $BUILD_RESULTS/*.txt $FINAL_BUILD_RESULTS

#!/bin/bash
set -e

NUM_CPUS=`nproc`
echo "###############"
echo "### Using ${NUM_CPUS} cores"

# setup some build variables
BUILD_USER=vagrant
BUILD_GROUP=vagrant
BUILD_ROOT=/var/kernel_build
BUILD_CACHE=$BUILD_ROOT/cache
ARM_TOOLS=$BUILD_CACHE/tools
LINUX_KERNEL=$BUILD_CACHE/linux-kernel
# LINUX_KERNEL_COMMIT=""
LINUX_KERNEL_COMMIT=1f58c41a5aba262958c2869263e6fdcaa0aa3c00
RASPBERRY_FIRMWARE=$BUILD_CACHE/rpi_firmware

if [ -d /vagrant ]; then
  # running in vagrant VM
  SRC_DIR=/vagrant
else
  # running in drone build
  SRC_DIR=`pwd`
  BUILD_USER=`whoami`
  BUILD_GROUP=`whoami`
fi

LINUX_KERNEL_CONFIGS=$SRC_DIR/kernel_configs

NEW_VERSION=`date +%Y%m%d-%H%M%S`
BUILD_RESULTS=$BUILD_ROOT/results/kernel-$NEW_VERSION

X64_CROSS_COMPILE_CHAIN=arm-bcm2708/gcc-linaro-arm-linux-gnueabihf-raspbian-x64

declare -A CCPREFIX
CCPREFIX["rpi1"]=$ARM_TOOLS/$X64_CROSS_COMPILE_CHAIN/bin/arm-linux-gnueabihf-
CCPREFIX["rpi2"]=$ARM_TOOLS/$X64_CROSS_COMPILE_CHAIN/bin/arm-linux-gnueabihf-

declare -A IMAGE_NAME
IMAGE_NAME["rpi1"]=kernel.img
IMAGE_NAME["rpi2"]=kernel7.img

function create_dir_for_build_user () {
    local target_dir=$1

    sudo mkdir -p $target_dir
    sudo chown $BUILD_USER:$BUILD_GROUP $target_dir
}

function setup_build_dirs () {
  for dir in $BUILD_ROOT $BUILD_CACHE $BUILD_RESULTS $ARM_TOOLS $LINUX_KERNEL $RASPBERRY_FIRMWARE; do
    create_dir_for_build_user $dir
  done
}

function clone_or_update_repo_for () {
  local repo_url=$1
  local repo_path=$2
  local repo_commit=$3

  if [ ! [ -z "${repo_commit}" ] ]; then
    rm -rf $repo_path
  fi
  if [ -d ${repo_path}/.git ]; then
    cd $repo_path && git pull
  else
    echo "Cloning $repo_path with commit $repo_commit"
    git clone --depth 1 $repo_url $repo_path
    if [ ! [ -z "${repo_commit}" ] ]; then
      git checkout -qf ${repo_commit}
    fi
  fi
}

function setup_arm_cross_compiler_toolchain () {
  echo "### Check if Raspberry Pi Crosscompiler repository at ${ARM_TOOLS} is still up to date"
  clone_or_update_repo_for 'https://github.com/raspberrypi/tools.git' $ARM_TOOLS ""
}

function setup_linux_kernel_sources () {
  echo "### Check if Raspberry Pi Linux Kernel repository at ${LINUX_KERNEL} is still up to date"
  clone_or_update_repo_for 'https://github.com/raspberrypi/linux.git' $LINUX_KERNEL $LINUX_KERNEL_COMMIT
}

function setup_rpi_firmware () {
  echo "### Check if Raspberry Pi Firmware repository at ${LINUX_KERNEL} is still up to date"
  clone_or_update_repo_for 'https://github.com/asb/firmware' $RASPBERRY_FIRMWARE ""
}

function prepare_kernel_building () {
  setup_build_dirs
  setup_arm_cross_compiler_toolchain
  setup_linux_kernel_sources
  setup_rpi_firmware
}

create_kernel_for () {
  echo "###############"
  echo "### START building kernel for ${PI_VERSION}"

  local PI_VERSION=$1

  cd $LINUX_KERNEL

  # add kernel branding for hyprOS
  sed -i 's/^EXTRAVERSION =.*/EXTRAVERSION = -hypriotos/g' Makefile

  # save git commit id of this build
  local KERNEL_COMMIT=`git rev-parse HEAD`
  echo "### git commit id of this kernel build is ${KERNEL_COMMIT}"

  # clean build artifacts
  make ARCH=arm clean

  # copy kernel configuration file over
  cp $LINUX_KERNEL_CONFIGS/${PI_VERSION}_docker_kernel_config $LINUX_KERNEL/.config

  echo "### building kernel"
  mkdir -p $BUILD_RESULTS/$PI_VERSION
  echo $KERNEL_COMMIT > $BUILD_RESULTS/kernel-commit.txt
  ARCH=arm CROSS_COMPILE=${CCPREFIX[$PI_VERSION]} make -j$NUM_CPUS -k
  cp $LINUX_KERNEL/arch/arm/boot/Image $BUILD_RESULTS/$PI_VERSION/${IMAGE_NAME[${PI_VERSION}]}

  echo "### building kernel modules"
  mkdir -p $BUILD_RESULTS/$PI_VERSION/modules
  ARCH=arm CROSS_COMPILE=${CCPREFIX[${PI_VERSION}]} INSTALL_MOD_PATH=$BUILD_RESULTS/$PI_VERSION/modules make modules_install -j$NUM_CPUS

  # remove symlinks, mustn't be part of raspberrypi-bootloader*.deb
  echo "### removing symlinks"
  rm -f $BUILD_RESULTS/$PI_VERSION/modules/lib/modules/*/build
  rm -f $BUILD_RESULTS/$PI_VERSION/modules/lib/modules/*/source

  echo "### building deb packages"
  KBUILD_DEBARCH=armhf ARCH=arm CROSS_COMPILE=${CCPREFIX[${PI_VERSION}]} make deb-pkg
  mv ../*.deb $BUILD_RESULTS
  echo "###############"
  echo "### END building kernel for ${PI_VERSION}"
  echo "### Check the $BUILD_RESULTS/$PI_VERSION/kernel.img and $BUILD_RESULTS/$PI_VERSION/modules directory on your host machine."
}

function create_kernel_deb_packages () {
  echo "###############"
  echo "### START building kernel DEBIAN PACKAGES"

  PKG_TMP=`mktemp -d`

  NEW_KERNEL=$PKG_TMP/raspberrypi-kernel-${NEW_VERSION}

  create_dir_for_build_user $NEW_KERNEL

  # copy over source files for building the packages
  cp -r $RASPBERRY_FIRMWARE/* $NEW_KERNEL
  cp -r $SRC_DIR/debian $NEW_KERNEL/debian
  touch $NEW_KERNEL/debian/files

  for pi_version in ${!CCPREFIX[@]}; do
    cp $BUILD_RESULTS/$pi_version/${IMAGE_NAME[${pi_version}]} $NEW_KERNEL/boot
    cp -R $BUILD_RESULTS/$pi_version/modules/lib/modules/*hypriotos* $NEW_KERNEL/modules
  done

  # build debian packages
  cd $NEW_KERNEL

  dch -v ${NEW_VERSION} --package raspberrypi-firmware 'add Hypriot custom kernel'
  debuild --no-lintian -ePATH=${PATH}:$ARM_TOOLS/$X64_CROSS_COMPILE_CHAIN/bin -b -aarmhf -us -uc
  cp ../*.deb $BUILD_RESULTS

  echo "###############"
  echo "### FINISH building kernel DEBIAN PACKAGES"
}

##############
###  main  ###
##############

# setup necessary build environment: dir, repos, etc.
prepare_kernel_building

# create kernel, associated modules
for pi_version in ${!CCPREFIX[@]}; do
  create_kernel_for $pi_version
done

# create kernel packages
create_kernel_deb_packages

# running in vagrant VM
if [ -d /vagrant ]; then
  # copy build results to synced vagrant host folder
  FINAL_BUILD_RESULTS=/vagrant/build_results/$NEW_VERSION
else
  # running in drone build
  FINAL_BUILD_RESULTS=$SRC_DIR/output/$NEW_VERSION
fi

echo "###############"
echo "### Copy deb packages to $FINAL_BUILD_RESULTS"
mkdir -p $FINAL_BUILD_RESULTS
cp $BUILD_RESULTS/*.deb $FINAL_BUILD_RESULTS
cp $BUILD_RESULTS/*.txt $FINAL_BUILD_RESULTS


Initial

Initial

#!/usr/bin/perl

# If a box doesn't have a statically assigned MAC address, such as NEO, we need to craft a boxid (in MAC format)
# that can be used for phone home
# Derek Maxson, 20211115

# Get eth0 MAC Address
my $mac = `cat /sys/class/net/eth0/address`;

# Get the second character in the string
my $magicBit = substr($mac,1,1);

# Evaluate if the string is indicating a "locally-administered MAC address"
if ($magicBit eq "2" || $magicBit eq "6" ||  lc($magicBit) eq "a" || lc($magicBit) eq "e") {
	#print "Locally Administered\n";
	# Now craft new MAC from machine-id so that it won't change all the time!
	my $machineId = `cat /etc/machine-id`;
	$mac = "0a" . substr($machineId,0,10);
	$mac = join(':', unpack '(A2)*', $mac);
}

print $mac;

error: pathspec 'scripts/determineBoxID.sh' did not match any file(s) known to git


Add "slow" version of benchmark script, that runs a lot more combinations.

#!/bin/bash

set -ex

pwd="`pwd`"
rev=`git rev-parse HEAD | cut -c1-8`
date=`date "+%Y%m%dT%H%M"`
result_path="$pwd/results/$date-$rev"

: ${PARAMETERS:="-t 30 -n 300"}
echo "Parameters: $PARAMETERS"

echo "Installing/checking lein..."
./lein version
echo "lein installed"

echo "Making uberjar"
./lein uberjar
echo "Uberjar made"

echo "Benchmarking..."

mkdir -p "$result_path"

for i in 1 2 3 4 5
do
    # ORIGINAL VERSION
    version="orig"
    for w in 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 18 20 22 24 26 28 30 32 36 40 44 48 52 56 60 64
    do
        ./lein run -- -v $version -w $w $PARAMETERS > "$result_path/$version-w$w-i$i.txt"
    done

    # TXACT VERSION
    version="txact"
    for w in 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 18 20 22 24 26 28 30 32 36 40 44 48 52 56 60 64
    do
        for s in 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 18 20 22 24 26 28 30 32 36 40 44 48 52 56 60 64
        do
            ./lein run -- -v $version -w $w -s $s $PARAMETERS > "$result_path/$version-w$w-s$s-i$i.txt"
        done
    done
done

echo "Benchmark done"

error: pathspec 'run-benchmarks-slow.sh' did not match any file(s) known to git


Added alias for cosmos CLI

Added alias for cosmos CLI

remote: Support for password authentication was removed on August 13, 2021.
remote: Please see https://docs.github.com/en/get-started/getting-started-with-git/about-remote-repositories#cloning-with-https-urls for information on currently recommended modes of authentication.
fatal: Authentication failed for 'https://github.com/leo/dotfiles.git/'


bind to 0.0.0.0

bind to 0.0.0.0

remote: Support for password authentication was removed on August 13, 2021.
remote: Please see https://docs.github.com/en/get-started/getting-started-with-git/about-remote-repositories#cloning-with-https-urls for information on currently recommended modes of authentication.
fatal: Authentication failed for 'https://github.com/CloudCoreo/stackengine-cluster.git/'


Create git.sh

Create git.sh

#!/bin/bash
#Configure github for you.

USERHOME=/home/ubuntu

apt-get update -y && \
apt-get install -y wget && \
wget https://raw.githubusercontent.com/otnt/dotfiles/master/confige/.gitconfig -O $USERHOME/.gitconfig

error: pathspec 'confige/git.sh' did not match any file(s) known to git


kernels: Add support for creating ubuntu kernel packages

This only covers the 4.x Ubuntu kernels and for now does
not include the headers to build modules against.

Signed-off-by: Rolf Neugebauer <6fdd1d8677ab4fbcb4df3eff6e190c3298f7f742@docker.com>

#! /bin/sh

REPO="linuxkit/kernel-ubuntu"
BASE_URL=http://mirrors.kernel.org/ubuntu/pool/main/l/linux/

ARCH=amd64
LINKS=$(curl -s ${BASE_URL}/ | sed -n 's/.*href="\([^"]*\).*/\1/p')
# Just get names for 4.x kernels
KERNELS=$(echo $LINKS | \
    grep -o "linux-image-4\.[0-9]\+\.[0-9]\+-[0-9]\+-generic_[^ ]\+${ARCH}\.deb")

for KERN_DEB in $KERNELS; do
    VERSION=$(echo $KERN_DEB | \
        grep -o "[0-9]\+\.[0-9]\+\.[0-9]\+-[0-9]\+" | head -1)

    echo "$VERSION -> $KERN_DEB"
    DOCKER_CONTENT_TRUST=1 docker pull ${REPO}:${VERSION} && continue

    EXTRA_DEB=$(echo $LINKS | \
        grep -o "linux-image-extra-${VERSION}-generic_[^ ]\+${ARCH}\.deb")

    # Don't pull in the headers. This is mostly for testing
    # HDR_DEB=$(echo $LINKS | \
    #     grep -o "linux-headers-${VERSION}_[^ ]\+_all\.deb")
    # HDR_ARCH_DEB=$(echo $LINKS | \
    #     grep -o "linux-headers-${VERSION}-generic_[^ ]\+_${ARCH}\.deb")

    URLS="${BASE_URL}/${KERN_DEB} ${BASE_URL}/${EXTRA_DEB} ${BASE_URL}/${HDR_DEB} ${BASE_URL}/${HDR_ARCH_DEB}"

    # Doesn't exist build and push
    docker build -t ${REPO}:${VERSION} -f Dockerfile.deb --no-cache \
           --build-arg DEB_URLS="${URLS}" . &&
        DOCKER_CONTENT_TRUST=1 docker push ${REPO}:${VERSION})
done

error: pathspec 'scripts/kernels/ubuntu.sh' did not match any file(s) known to git


Add post build javadoc publisher script

Add post build javadoc publisher script

#! /bin/sh

BRANCH=gh-pages

cd $WORKSPACE

version_line=$(cat pom.xml | egrep -o "<version>(.+)</version>" | head -n 1)
version=$(expr match $version_line "<version>\(.\+\)</version>")

if [[ $version == *-SNAPSHOT ]]; then
    dirname="apidocs-dev"
    commit_version="$version-b$BUILD_NUMBER"
else
    dirname="apidocs"
    commit_version="$version"
fi

rm -rf /tmp/${JOB_NAME}_apidocs
cp -R target/site/apidocs /tmp/${JOB_NAME}_apidocs

git checkout $BRANCH
git pull origin $BRANCH

rm -rf ./$dirname
mv /tmp/${JOB_NAME}_apidocs ./$dirname
git add . && git add -u
git commit -m "Update javadoc for $commit_version"

git push origin $BRANCH
git checkout master

error: pathspec 'publish_javadoc.sh' did not match any file(s) known to git


Coleman - created convenience script for adding 'minion' users on the Mac Pros

#!/bin/bash

NUMBER=$1
HOST=$2

dscl . create /Users/minion${NUMBER}
dscl . create /Users/minion${NUMBER} UserShell /bin/bash
dscl . create /Users/minion${NUMBER} RealName "${HOST} Minion ${NUMBER}
dscl . create /Users/minion${NUMBER} UniqueID "100${NUMBER}"
dscl . create /Users/minion${NUMBER} PrimaryGroupID 20
dscl . create /Users/minion${NUMBER} NFSHomeDirectory /Users/minion${NUMBER}
dscl . passwd /Users/minion${NUMBER} M0bileL0yalty

error: pathspec 'create-user.sh' did not match any file(s) known to git


mode fix on ci.sh

mode fix on ci.sh

#!/bin/bash
test/support/chef_config.sh
go get -v github.com/ctdk/goiardi
test/support/start_server.sh
go get code.google.com/p/go.tools/cmd/vet
go install code.google.com/p/go.tools/cmd/vet
go vet
go test -v -cover
test/support/stop_server.sh


Fix #63 (without using `--attribute-walk`)

Use a more precise query for udev info that expands the number
supported platforms.
Possibly closes #63 by specifying a more clearly defined query that
obviates the need to parse the output of `--attribute-walk`.
Recent udevadm versions use `--query=all --export` as a default action
while older versions do not.

remote: Support for password authentication was removed on August 13, 2021.
remote: Please see https://docs.github.com/en/get-started/getting-started-with-git/about-remote-repositories#cloning-with-https-urls for information on currently recommended modes of authentication.
fatal: Authentication failed for 'https://github.com/rhwood/paducah.git/'


added script for launching ncmpcpp and cava in a tmux session with correct proportions

#!/bin/sh
tmux new-session -d 'ncmpcpp'
tmux split-window -p 20 -v 'cava'\; last-pane
tmux -2 attach-session -d
tmux swap-pane

error: pathspec 'Scripts/music_mux.sh' did not match any file(s) known to git


Added new initialize file

Added new initialize file

rosservice call /InitializeCameraService$1 "{header: auto, hsvColorRanges: [2652260, 5177343, 1407076, 2455652], quadCopterIds: [1, 2]}"

error: pathspec 'test/test_messages/initialize_camera_id_n_2_colors_yellow_green.sh' did not match any file(s) known to git


Auto-mkdir output

Auto-mkdir output

remote: Support for password authentication was removed on August 13, 2021.
remote: Please see https://docs.github.com/en/get-started/getting-started-with-git/about-remote-repositories#cloning-with-https-urls for information on currently recommended modes of authentication.
fatal: Authentication failed for 'https://github.com/mettatw/probable-octo-guacamole.git/'


first draft of a script for automating a lot of the release process

#!/bin/bash

function help_and_exit() {
    echo "Usage: $0 [-go] [-verbose]"
    echo
    echo "Moves minified CSS and JS to distribution directories and"
    echo "creates a branch in SVN."
    echo
    echo "  -go:      Run commands instead of just echoing them."
    echo "  -verbose: More verbose logging."
    exit "$1"
}

# 1 for verbose logging
export VERBOSE="0"
# 1 if commands that have side-effects should actually be run instead of logged
export EFFECT="0"

function panic() {
    echo "$*"
    exit -1
}

function command() {
    if (( $VERBOSE )) || ! (( $EFFECT )); then
	echo '$' "$*"
    fi
    if (( $EFFECT )); then
	"$@" || panic "command failed: $@"
    fi
}

function mime_for_file() {
    local path="$1"
    case "${path##*.}" in
	js)   echo -n "text/javascript;charset=UTF-8";;
	css)  echo -n "text/css;charset=UTF-8";;
	html) echo -n "text/html;charset=UTF-8";;
	*)    panic "unrecognized extension for $path";;
    esac
}

for var in "$@"; do
  case "$var" in
      -verbose)
	  VERBOSE="1"
	  ;;
      -go)
	  EFFECT="1"
	  ;;
      -h)
	  help_and_exit 0
	  ;;
      *)
	  echo "Unrecognized variable $var"
	  help_and_exit -1
	  ;;
  esac
done

# Find svn root
export VERSION_BASE="$(
  pushd "$(dirname "$0")/.." > /dev/null; pwd; popd > /dev/null)"

if [ -z "$VERSION_BASE" ] || ! [ -d "$VERSION_BASE" ]; then
    panic "unknown VERSION_BASE"
fi
if ! [ -d "$VERSION_BASE/trunk" ]; then
    panic "missing trunk in $VERSION_BASE"
fi
if ! [ -d "$VERSION_BASE/loader" ]; then
    panic "missing loader in $VERSION_BASE"
fi
if ! [ -d "$VERSION_BASE/branches" ]; then
    panic "missing branches in $VERSION_BASE"
fi

if (( $VERBOSE )); then
    echo "VERSION_BASE=$VERSION_BASE"
fi

# Choose a release label
export RELEASE_LABEL="$(date -u +release-%e-%b-%Y)"

if (( $VERBOSE )); then
    echo "RELEASE_LABEL=$RELEASE_LABEL"
fi

if [ -e "$VERSION_BASE/branches/$RELEASE_LABEL" ]; then
    panic "duplicate release $VERSION_BASE/branches/$RELEASE_LABEL"
fi

# Make the distribution
function build() {
  pushd "$VERSION_BASE/trunk" > /dev/null
  make distrib
  local status=$?
  popd > /dev/null
  (($status))
}
if build; then
  panic "Make failed"
fi

if [-n "$(svn stat "$VERSION_BASE/trunk")"; then
  svn stat "$VERSION_BASE/trunk"
  panic "Uncommitted changes"
fi

function sync() {
    local action="$1"
    local src_dir="$2"
    local dest_dir="$3"
    shift 3
    local exts=$@
    local ext
    local src_file
    local dest_file
    for ext in $exts; do
	for src_file in "$src_dir"/*."$ext"; do
	    dest_file="$dest_dir"/"$(basename "$src_file")"
	    if ! [ -e "$dest_file" ] || diff -q "$src_file" "$dest_file"; then
		"$action" "$src_file" "$dest_file"
	    fi
	done
	for dest_file in "$dest_dir"/*."$ext"; do
	    src_file="$src_dir"/"$(basename "$dest_file")"
	    if ! [ -e "$src_file" ]; then
		"$action" "$src_file" "$dest_file"
	    fi
	done
    done
}

function svn_sync() {
    local src_file="$1"
    local dest_file="$2"
    if ! [ -e "$src_file" ]; then
	command svn delete "$dest_file"
    else
	command cp "$src_file" "$dest_file"
	if ! [ -e "$dest_file" ]; then
	    command svn add "$dest_file"
	    command svn propset svn:mime-type "$(mime_for_file "$src_file")" \
		"$dest_file"
	fi
    fi
}

# Deploy the current compiled source to /loader
sync svn_sync "$VERSION_BASE/trunk/src" "$VERSION_BASE/loader" js css
sync svn_sync "$VERSION_BASE/trunk/styles" "$VERSION_BASE/loader/skins" css

# Cut branch
command svn copy "$VERSION_BASE/trunk" "$VERSION_BASE/branches/$RELEASE_LABEL"

# Dump final instructions for caller.
echo
if (( $EFFECT )); then
    echo "Finally run"
    echo "    $ svn commit -m 'Release $VERSION_BASE'"
    echo "to commit the new release then upload"
    echo "    $VERSION_BASE/trunk/distrib/prettify-small.*"
    echo "to"
    echo "    http://code.google.com/p/google-code-prettify/downloads/list"
else
   echo "Rerun with -go flag to actually execute these commands."
fi

exit 0

error: pathspec 'cut-release.sh' did not match any file(s) known to git


also manage the queue scripts for farm in git

## Use the current working directory
#$ -cwd
## use bash commands
#$ -S /bin/bash
## combine error and output files
#$ -j y
## Parallel for MPI
##$ -pe mpi 20
## Parallel for openmp:
#$ -pe threaded 16
## Output file name
#$ -o det_figure3.log

module load gcc openmpi R
Rscript -e "source('~/.Rprofile'); library('knitr'); knit('figure3.Rmd')"

error: pathspec 'inst/examples/mpi.sh' did not match any file(s) known to git


created script to collect results of significant enrichments

## Erin Fry
## August 1 2017

## this script finds GO enrichments in all lineages after running IdentifyShiftsInAllLineages.Rmd
## Non-indented lines should be evaluated for modification specific to the user's purpose
## User MUST modify the base path
## The other basic options to change might be the models to evaluate if different than the recommended models

###########################################################

	## read in tissue code and start and end gene numbers

	tissue=$1

###########################################################

	## set directory paths

export path=/Users/lynchlab/Desktop/ErinFry/workflowr/AGER ##full absolute path to main directory

	export pathData=${path}/data/forBAGER
	export pathScripts=${path}/code
	export pathResults=${path}/data/BAGERresults/${tissue}
	export pathCommands=${pathScripts}/commands
	export pathEnrichments=${pathResults}/Enrichments

###########################################################

## create results file

echo These are the enrichments for $tissue in all lineages > $pathEnrichments/enrichmentresults.txt
echo ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ >> $pathEnrichments/enrichmentresults.txt

cd $pathEnrichments

for enrich in Project*/enrichment_results*
do
echo $enrich has the following enrichments >> $pathEnrichments/enrichmentresults.txt
cat $enrich >> $pathEnrichments/enrichmentresults.txt

echo ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ >> $pathEnrichments/enrichmentresults.txt

done

error: pathspec 'code/consolidate_enrichments.sh' did not match any file(s) known to git


Update brew-cask.sh

Update brew-cask.sh

remote: Support for password authentication was removed on August 13, 2021.
remote: Please see https://docs.github.com/en/get-started/getting-started-with-git/about-remote-repositories#cloning-with-https-urls for information on currently recommended modes of authentication.
fatal: Authentication failed for 'https://github.com/dbonner1987/dotfiles.git/'


Create install_synoghost.sh

Create install_synoghost.sh

remote: Support for password authentication was removed on August 13, 2021.
remote: Please see https://docs.github.com/en/get-started/getting-started-with-git/about-remote-repositories#cloning-with-https-urls for information on currently recommended modes of authentication.
fatal: Authentication failed for 'https://github.com/t11z/install-synoghost.git/'


fix install script

fix install script

#!/bin/bash

# Executing the script from the url
# curl -s https://raw.githubusercontent.com/morgangiraud/ecvd-js/master/finaltest/installscript.sh | bash /dev/stdin

if [ -z "$1" ]; then
  echo "No argument supplied"
  exit 1
fi

if [ ! -d ~/tmp ]; then
  mkdir ~/tmp
fi
cd ~/tmp

# Xcode
xcode-select --install

which -s brew
if [[ $? != 0 ]] ; then
    /usr/bin/ruby -e "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install)"
else
    brew update
fi
echo "Installing missing utils"
brew install automake curl git wget cask pidof
echo "Upgrading utils"
brew upgrade automake curl git wget cask pidof

echo "Checking for Sublime text"
which -s subl
if [[ $? != 0 ]] ; then
  brew cask install sublime-text
fi

echo "Checking for Node"
which -s node
if [[ $? != 0 ]] ; then
    brew install node
else
    brew upgrade node
fi

if [ -d ~/tmp/ecvd-js ]; then
  echo "Removing ~/tmp/ecvd-js folder"
  rm -rf ~/tmp/ecvd-js
fi
echo "cloning distant repo"
cd ~/tmp
git clone https://github.com/morgangiraud/ecvd-js.git
cd ecvd-js
git checkout -b $1 && git push -u origin $1
mkdir -p finaltest/$1
cd finaltest/$1
cp -R ../exercices .
subl .
open -a Terminal .
open -a "/Applications/Google Chrome.app" 'https://github.com/morgangiraud/ecvd-js/blob/master/finaltest/finaltest.md'

#!/bin/bash

# Executing the script from the url
# curl -s https://raw.githubusercontent.com/morgangiraud/ecvd-js/master/finaltest/installscript.sh | bash /dev/stdin name

if [ -z "$1" ]; then
  echo "No argument supplied"
  exit 1
fi

if [ ! -d ~/tmp ]; then
  mkdir ~/tmp
fi
cd ~/tmp

# Xcode
xcode-select --install

which -s brew
if [[ $? != 0 ]] ; then
    /usr/bin/ruby -e "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install)"
else
    brew update
fi
echo "Installing missing utils"
brew install automake curl git wget cask pidof
echo "Upgrading utils"
brew upgrade automake curl git wget cask pidof

echo "Checking for Sublime text"
which -s subl
if [[ $? != 0 ]] ; then
  brew cask install sublime-text
fi

echo "Checking for Node"
which -s node
if [[ $? != 0 ]] ; then
    brew install node
else
    brew upgrade node
fi

if [ -d ~/tmp/ecvd-js ]; then
  echo "Removing ~/tmp/ecvd-js folder"
  rm -rf ~/tmp/ecvd-js
fi
echo "cloning distant repo"
cd ~/tmp
git clone https://github.com/morgangiraud/ecvd-js.git
cd ecvd-js
git checkout -b $1 && git push -u origin $1
mkdir -p finaltest/$1
cd finaltest/$1
cp -R ../exercices .
subl .
open -a Terminal .
open -a "/Applications/Google Chrome.app" 'https://github.com/morgangiraud/ecvd-js/blob/master/finaltest/finaltest.md'


update etcd prefix

update etcd prefix

remote: Support for password authentication was removed on August 13, 2021.
remote: Please see https://docs.github.com/en/get-started/getting-started-with-git/about-remote-repositories#cloning-with-https-urls for information on currently recommended modes of authentication.
fatal: Authentication failed for 'https://github.com/khipu/dockerfiles.git/'


setup1 scrip created

setup1 scrip created

#!/bin/sh
echo "SETUP 1"
echo "Script executed from: ${PWD}"
cd ~
echo "Now at: ${PWD}"
export LC_ALL=C

sudo apt-get --assume-yes install virtualenv
sudo apt-get --assume-yes install python-pip

#virtualenv
virtualenv --system-site-packages ~/tensorflow
source ~/tensorflow/bin/activate

#python data sci. libs
sudo apt-get --assume-yes install python-numpy python-scipy python-matplotlib ipython ipython-notebook python-pandas python-sympy python-nose
pip install -U scikit-learn

#install tensorflow
#tensorflow v0.11
#GPU - CUDA:
#export TF_BINARY_URL=https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow-0.11.0rc0-cp27-none-linux_x86_64.whl
#CPU:
export TF_BINARY_URL=https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-0.11.0rc0-cp27-none-linux_x86_64.whl
pip install --upgrade $TF_BINARY_URL

error: pathspec 'setup1.sh' did not match any file(s) known to git


Add Arbeitsplatzliste to resistance_test

Closes #9

#!/bin/bash

#set -x  # uncomment for debugging
incomingdir=/cygdrive/D/Illumina/MiSeqOutput
timavoDST=/data/MiSeq
datamoverDST=data/outgoing

# define these globally so no need to pass it as a function parameter
headers='undefined'
rundir='undefined'
# IEMFileVersion='undefined'
Account=''
Investigator_Name='undefined'
Experiment_Name='undefined'
#Date='undefined'
Workflow='undefined'
Application='undefined'
Assay='undefined'
#Description='undefined'
Chemistry='undefined'
Read1=''
Read2=''
PhiX='undefined'
RGT_box1='undefined'
RGT_box2='undefined'
openbis='n'
timavo='n'

# global variable to allow a function return it
ismolis='undefined'

test_molis(){
    if [[ $1 =~ ^100[0-9]{7} ]]; then
        ismolis=true
        return
    else
        ismolis=false
        return
    fi
}

write_miseq_run(){
    # input: run_name space
    # space is Diagnostics or Research
    run_name_here=$1

    datenum=${run_name_here:0:6}
    YYYY=20${datenum:0:2}
    MM=${datenum:2:2}
    DD=${datenum:4:2}

    echo "Writing properties files for MISEQ_RUN:$1 PROJECT:$2"
    prop_file=sample.properties
    {
        printf "INVESTIGATOR_NAME = %s\n" "$Investigator_Name"
        printf "EXPERIMENT_NAME = %s\n" "$Experiment_Name"
        printf "ACCOUNT = %s\n" "$Account"
        printf "DATE = %s-%s-%s\n" "${YYYY}" "${MM}" "${DD}"
        printf "MISEQ_WORKFLOW = %s\n" "$Workflow"
        printf "APPLICATION = %s\n" "$Application"
        printf "ASSAY = %s\n" "$Assay"
        printf "CHEMISTRY = %s\n" "$Chemistry"
        printf "READ_1 = %s\n" "$Read1"
        printf "READ_2 = %s\n" "$Read2"
        printf "PHIX_CONCENTRATION = %s\n" "$PhiX"
        printf "RGT_Box_1 = %s\n" "$RGT_box1"
        printf "RGT_Box_2 = %s\n" "$RGT_box2"
    } > "$prop_file"

    dst="${datamoverDST}/${run_name_here}"
    rsync -a --rsync-path="mkdir -p $dst && rsync" "$prop_file" "ozagor@datamover:$dst"
    # save rsync exit status, if 0 then success
    rsync1=$?

    ### sample_type MISEQ_RUN
    project_to_write=$2

    prop_file=dataset.properties
    {
        printf "SPACE = IMV\n"
        printf "PROJECT = %s\n" "$project_to_write"
        printf "EXPERIMENT = MISEQ_RUNS\n"
        printf "SAMPLE = %s\n" "$run_name_here"
        printf "SAMPLE_TYPE = MISEQ_RUN\n"
        printf "DATASET_TYPE = DATAMOVER_SAMPLE_CREATOR\n"
    } > "$prop_file"

    rsync -a "$prop_file" "ozagor@datamover:$dst"
    rsync2=$?
    if [ "$rsync1" -eq "0" ] && [ "$rsync2" -eq "0" ]; then
        ssh ozagor@datamover touch "$datamoverDST/.MARKER_is_finished_${run_name_here}" </dev/null
    else
        echo -e "WATCH OUT: touching the void! $rsync1 $rsync2"
    fi

}

write_miseq_sample_zero(){

    sample_number=0
    sample_name='Undetermined'
    run_name="$(basename $rundir)"

    fastq_file=$incomingdir/$run_name/Data/Intensities/BaseCalls/${sample_name}_S${sample_number}_L001_R1_001.fastq.gz
    fastq_file_2=$incomingdir/$run_name/Data/Intensities/BaseCalls/${sample_name}_S${sample_number}_L001_R2_001.fastq.gz

    echo "Syncing to TIMAVO"
    DST2="${timavoDST}/MiSeqOutput/${run_name}/Data/Intensities/BaseCalls/"
    rsync -a --rsync-path="mkdir -p $DST2 && rsync" "$fastq_file" "timavo:$DST2"
    if [ -e "$fastq_file_2" ]; then
        rsync "$fastq_file_2" "timavo:$DST2"
    fi

    ### write properties file

    ### sample_type MISEQ_SAMPLE
    echo "Writing properties files for MISEQ_SAMPLE ID:${sample_number} NAME:${sample_name}"
    prop_file=sample.properties
    {
        printf "SAMPLE_ID=0\n"
        printf "SAMPLE_NAME=Undetermined\n"
        printf "SAMPLE_PLATE=\n"
        printf "SAMPLE_WELL=\n"
        printf "I7_INDEX_ID=\n"
        printf "INDEX_1=\n"
        printf "I5_INDEX_ID=\n"
        printf "INDEX_2=\n"
        printf "DESCRIPTION=\n"
    } > "$prop_file"

    dst="${datamoverDST}/${run_name}-${sample_number}"
    rsync -a --rsync-path="mkdir -p $dst && rsync" "$prop_file" "ozagor@datamover:$dst"
    # save rsync exit status, if 0 then success
    rsync1=$?

    prop_file=dataset.properties
    {
        printf "SPACE = IMV\n"
        printf "PROJECT = METAGENOMICS\n"
        printf "EXPERIMENT = MISEQ_SAMPLES\n"
        printf "SAMPLE = %s-%s\n" "${run_name}" "${sample_number}"
        printf "SAMPLE_TYPE = MISEQ_SAMPLE\n"
        printf "DATASET_TYPE = FASTQ\n"
    } > "$prop_file"

    rsync -a "$prop_file" "ozagor@datamover:$dst"
    rsync2=$?

    rsync3=0
    if [ -e "$fastq_file" ]; then
        rsync -a "$fastq_file" "ozagor@datamover:$dst"
        rsync3=$?
        # echo "R1 file exists, rsync returns $rsync3"
    else
        echo "R1 file does not exist"
    fi

    rsync4=0
    if [ -e "$fastq_file_2" ]; then
        rsync -a "$fastq_file_2" "ozagor@datamover:$dst"
        rsync4=$?
        echo "R2 file exists, rsync returns $rsync4"
    # else
    #     echo "R2 file does not exist"
    fi

    if [ "$rsync1" -eq "0" ] && [ "$rsync2" -eq "0" ] && [ "$rsync3" -eq "0" ] && [ "$rsync4" -eq "0" ]; then
        ssh ozagor@datamover touch "$datamoverDST/.MARKER_is_finished_${run_name}-${sample_number}" </dev/null
    else
        echo -e "WATCH OUT: touching the void! $rsync1 $rsync2 $rsync3 $rsync4"
    fi

}

write_miseq_sample(){
    # input is a line in section [Data] of the samples sheet
    # copy fastq file into $bufferdir/$rundir/S.../ and write dataset.properties
    # if timavo == y, sync fastq file on timavo too

    declare -a sample_line=("${!1}")

    ### move fastq file into folder

    sample_number=${sample_line[0]}
    # this should mimic the behaviour of MiSeq Reporter on sample names with
    # spaces and punctuations:
    # - remove leadin and trailing whitespaces (first sed)
    # - replace punctuation characters and internal spaces with dashes
    sample_name=$(echo ${sample_line[1]} | sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//' | tr -s '[:punct:] ' '-')
    sample_plate=${sample_line[2]}
    sample_well=${sample_line[3]}
    I7_index_id=${sample_line[4]}
    index_1=${sample_line[5]}
    I5_index_id=${sample_line[6]}
    index_2=${sample_line[7]}
    description=${sample_line[9]}
    timavo=${sample_line[14]}

    run_name="$(basename $rundir)"

    fastq_file=$incomingdir/$run_name/Data/Intensities/BaseCalls/${sample_name}_S${sample_number}_L001_R1_001.fastq.gz
    fastq_file_2=$incomingdir/$run_name/Data/Intensities/BaseCalls/${sample_name}_S${sample_number}_L001_R2_001.fastq.gz

    if [[ $timavo == *"y"* ]]; then
        echo "Syncing to TIMAVO"
        DST2="${timavoDST}/MiSeqOutput/${run_name}/Data/Intensities/BaseCalls/"
        rsync -a --rsync-path="mkdir -p $DST2 && rsync" "$fastq_file" "timavo:$DST2"
        if [ -e "$fastq_file_2" ]; then
            rsync "$fastq_file_2" "timavo:$DST2"
        fi
      else
        echo "Not syncing to TIMAVO"
    fi

    ### write properties file

    ### sample_type MISEQ_SAMPLE
    echo "Writing properties files for MISEQ_SAMPLE ID:${sample_number} NAME:${sample_name}"
    prop_file=sample.properties
    {
        printf "SAMPLE_ID=%s\n" "${sample_number}"
        printf "SAMPLE_NAME=%s\n" "${sample_name}"
        printf "SAMPLE_PLATE=%s\n" "${sample_plate}"
        printf "SAMPLE_WELL=%s\n" "$sample_well"
        printf "I7_INDEX_ID=%s\n" "$I7_index_id"
        printf "INDEX_1=%s\n" "$index_1"
        printf "I5_INDEX_ID=%s\n" "$I5_index_id"
        printf "INDEX_2=%s\n" "$index_2"
        printf "DESCRIPTION=%s\n" "$description"
    } > "$prop_file"

    dst="${datamoverDST}/${run_name}-${sample_number}"
    rsync -a --rsync-path="mkdir -p $dst && rsync" "$prop_file" "ozagor@datamover:$dst"
    # save rsync exit status, if 0 then success
    rsync1=$?

    Project=${sample_line[8]}

    prop_file=dataset.properties
    {
        printf "SPACE = IMV\n"
        printf "PROJECT = %s\n" "$Project"
        printf "EXPERIMENT = MISEQ_SAMPLES\n"
        printf "SAMPLE = %s-%s\n" "${run_name}" "${sample_number}"
        printf "SAMPLE_TYPE = MISEQ_SAMPLE\n"
        printf "DATASET_TYPE = FASTQ\n"
    } > "$prop_file"

    rsync -a "$prop_file" "ozagor@datamover:$dst"
    rsync2=$?

    rsync3=0
    if [ -e "$fastq_file" ]; then
        rsync -a "$fastq_file" "ozagor@datamover:$dst"
        rsync3=$?
        # echo "R1 file exists, rsync returns $rsync3"
    else
        echo "R1 file does not exist"
    fi

    rsync4=0
    if [ -e "$fastq_file_2" ]; then
        rsync -a "$fastq_file_2" "ozagor@datamover:$dst"
        rsync4=$?
        echo "R2 file exists, rsync returns $rsync4"
    # else
    #     echo "R2 file does not exist"
    fi

    if [ "$rsync1" -eq "0" ] && [ "$rsync2" -eq "0" ] && [ "$rsync3" -eq "0" ] && [ "$rsync4" -eq "0" ]; then
        ssh ozagor@datamover touch "$datamoverDST/.MARKER_is_finished_${run_name}-${sample_number}" </dev/null
    else
        echo -e "WATCH OUT: touching the void! $rsync1 $rsync2 $rsync3 $rsync4"
    fi

}

write_resistance_test(){

    declare -a sample_line=("${!1}")

    sample_number=${sample_line[0]}
    sample_name=${sample_line[1]}
    #sample_project=${sample_line[8]}
    #Description=${sample_line[9]}
    virus=${sample_line[10]}
    genotype=${sample_line[11]}
    target=${sample_line[12]}
    viral_load=${sample_line[13]}
    apl=${sample_line[15]}

    run_name=$(basename "$rundir")

    echo "Writing properties files for RESISTANCE_TEST RUN:${run_name} SAMPLE:${sample_name}"

    ### write properties file
    prop_file=sample.properties
    {
        printf "SAMPLE_NAME=%s\n" "$sample_name"
        printf "VIRUS=%s\n" "$virus"
        printf "TARGET_REGION=%s\n" "$target"
        printf "GENOTYPE=%s\n" "$genotype"
        printf "VIRAL_LOAD=%s\n" "$viral_load"
        printf "APL=%s\n" "$apl"
    } > "$prop_file"

    dst="${datamoverDST}/${run_name}-${sample_number}_RESISTANCE"
    rsync -a --rsync-path="mkdir -p $dst && rsync" "$prop_file" "ozagor@datamover:$dst"
    # save rsync exit status, if 0 then success
    rsync1=$?

    prop_file=dataset.properties
    {
        printf "SPACE = IMV\n"
        printf "PROJECT = RESISTANCE\n"
        printf "EXPERIMENT = RESISTANCE_TESTS\n"
        printf "SAMPLE = %s-%s_RESISTANCE\n" "${run_name}" "${sample_number}"
        printf "SAMPLE_TYPE = RESISTANCE_TEST\n"
        printf "DATASET_TYPE = DATAMOVER_SAMPLE_CREATOR\n"
    } > "$prop_file"

    rsync -a "$prop_file" "ozagor@datamover:$dst"
    rsync2=$?

    if [ "$rsync1" -eq "0" ] && [ "$rsync2" -eq "0" ]; then
        ssh ozagor@datamover touch "$datamoverDST/.MARKER_is_finished_${run_name}-${sample_number}_RESISTANCE" </dev/null
    else
        echo -e "WATCH OUT: touching the void! $rsync1 $rsync2"
    fi

}

process_runs(){
    # read the samples sheet,
    # save info found in [Header] and [Reads] in global variables
    # call process_sample on all lines in [Data] section

    rundir=$1
    if [[ -e $rundir/.UPLOADED_RUN ]]; then
        echo "Run $rundir already uploaded"
        return
    fi

    # reset headers
    headers='undefined'

    ### remove spaces in sample sheet
    #sed -e "s/ /_/g" < "$rundir/Data/Intensities/BaseCalls/SampleSheet.csv" > sample_sheet.tmp
    # sample sheets are now created with Stefan's template: no need to remove them anymore
    cp "$rundir/SampleSheet.csv" sample_sheet.tmp
    count_openbis=$(grep -c openbis sample_sheet.tmp)
    ### counters for read 1/2 and samples
    r=0
    s=0
    res_sample=false
    meta_sample=false
    plasm_sample=false
    anti_sample=false
    other_sample=false

    ### read sample sheet line by line splitting on commas
    ### || [[ -n "$line"]] allow reading last line also if file does not end
    ### with a newline
    while IFS=',' read -r -a line || [[ -n "$line" ]]
    do
        # if [[ "$count_openbis" == 0 ]]
        # then
        #     break
        # fi
        if [[ ${line[0]} =~ ^\[[[:alpha:]]*\] ]]
        then
            section=${line[0]}

        ### [Header] section
        elif [[ $section == "[Header]" && ${line[1]} ]]
        then
            declare "${line[0]}=${line[1]}"

        ### [Reads] section for read 1
        elif [[ $section == "[Reads]" && ${line[0]} =~ ^[0-9]+$ && $r -eq 0 ]]
        then
            # [Header] has now been parsed, if openbis != y then stop parsing
            # this run and go to next
            if [[ "$openbis" == "n" ]]; then
                break
            fi
            r=1
            # echo $section ${line[0]}
            Read1=${line[0]}

        ### [Reads] section for read 2
        elif [[ $section == "[Reads]" && ${line[0]} =~ ^[0-9]+$ && $r -eq 1 ]]
        then
            r=2
            # echo $section ${line[0]}
            Read2=${line[0]}

        ### [Data] section headers
        elif [[ $section == "[Data]" && ${line[1]} && $s -eq 0 ]]
        then
            headers=( "${line[@]}" )
            echo -e "  Headers found:" "${headers[@]}"
            ((s+=1))

        ### [Data] section values
        elif [[ $section == "[Data]" && ${line[1]} && $s -gt 0 ]]
        then

            write_miseq_sample line[@]

            case ${line[8]} in
              Antibodies)
                anti_sample=true
                ;;
              Metagenomics)
                meta_sample=true
                ;;
              Other)
                other_sample=true
                ;;
              Plasmids)
                plasm_sample=true
                ;;
              Resistance)
                res_sample=true
                write_resistance_test line[@]
                ;;
              esac
            ((s+=1))
        fi

    done < sample_sheet.tmp

    echo "Syncing SampleSheet to timavo"
    run_name=$(basename "$rundir")
    smpshdst="${timavoDST}/MiSeqOutput/${run_name}/Data/Intensities/BaseCalls/"
    rsync -av --rsync-path="mkdir -p $smpshdst && rsync" sample_sheet.tmp "timavo:$smpshdst/SampleSheet.csv"
    rm sample_sheet.tmp

    # if any sample was X then write Miseq run sample with PROJECT = X
    if [ "$anti_sample" = true ]; then
        echo "WRITING ANTIBODIES RUN"
        write_miseq_run "${run_name}_ANTIBODIES" Antibodies
    fi
    if [ "$meta_sample" = true ]; then
        echo "WRITING UNDETERMINED READS"
        write_miseq_sample_zero
        echo "WRITING METAGENOMICS RUN"
        write_miseq_run "${run_name}_METAGENOMICS" Metagenomics
    fi
    if [ "$other_sample" = true ]; then
        echo "WRITING OTHER RUN"
        write_miseq_run "${run_name}_OTHER" Other
    fi
    if [ "$plasm_sample" = true ]; then
        echo "WRITING PLASMIDS RUN"
        write_miseq_run "${run_name}_PLASMIDS" Plasmids
    fi
    if [ "$res_sample" = true ]; then
        echo "WRITING RESISTANCE RUN"
        write_miseq_run "${run_name}_RESISTANCE" Resistance
    fi

    # get sample sheet name from runParameter.xml file and save runParameter.xml file with the sample sheet name
    SAMPLESHEET=$(grep -A 1 ReagentKitRFIDTag "$rundir/runParameters.xml" | grep SerialNumber | sed 's/^.*<SerialNumber>//' | sed 's/<\/SerialNumber>//')
    rsync -av --stats --chmod=ug+rwx -p "$rundir/runParameters.xml" "timavo:$timavoDST/MiSeqRunParameters/${SAMPLESHEET}.xml"

    # reset [Header] and [Reads] information
    Investigator_Name='undefined'
    Experiment_Name='undefined'
#    Date='undefined'
    Workflow='undefined'
    Application='undefined'
    Assay='undefined'
#    Description='undefined'
    Chemistry='undefined'
    Read1=''
    Read2=''
    PhiX='undefined'
    RGT_box1='undefined'
    RGT_box2='undefined'
    openbis='n'

    touch "$rundir/.UPLOADED_RUN"

}

echo 'GO!'
### main loop over all dirs in $incomingdir starting with "1"
#for rundir in $(find $incomingdir -type d -name "1*" -depth 1)
#for rundir in $(ls $incomingdir)
for rundir in "$incomingdir"/1*; do
    [[ -d "$rundir" ]] || continue
    echo -e "\033[1;31m================================================================================\033[0m"
    echo "Now syncing:" "$rundir"
    process_runs "$rundir"
done

# copy SampleSheets
rsync -av --stats --chmod=ug+rwx -p /cygdrive/i/MiSeqSampleSheets "timavo:$timavoDST"

#!/bin/bash

#set -x  # uncomment for debugging
incomingdir=/cygdrive/D/Illumina/MiSeqOutput
timavoDST=/data/MiSeq
datamoverDST=data/outgoing

# define these globally so no need to pass it as a function parameter
headers='undefined'
rundir='undefined'
# IEMFileVersion='undefined'
Account=''
Investigator_Name='undefined'
Experiment_Name='undefined'
#Date='undefined'
Workflow='undefined'
Application='undefined'
Assay='undefined'
#Description='undefined'
Chemistry='undefined'
Read1=''
Read2=''
PhiX='undefined'
RGT_box1='undefined'
RGT_box2='undefined'
openbis='n'
timavo='n'

# global variable to allow a function return it
ismolis='undefined'

test_molis(){
    if [[ $1 =~ ^100[0-9]{7} ]]; then
        ismolis=true
        return
    else
        ismolis=false
        return
    fi
}

write_miseq_run(){
    # input: run_name space
    # space is Diagnostics or Research
    run_name_here=$1

    datenum=${run_name_here:0:6}
    YYYY=20${datenum:0:2}
    MM=${datenum:2:2}
    DD=${datenum:4:2}

    echo "Writing properties files for MISEQ_RUN:$1 PROJECT:$2"
    prop_file=sample.properties
    {
        printf "INVESTIGATOR_NAME = %s\n" "$Investigator_Name"
        printf "EXPERIMENT_NAME = %s\n" "$Experiment_Name"
        printf "ACCOUNT = %s\n" "$Account"
        printf "DATE = %s-%s-%s\n" "${YYYY}" "${MM}" "${DD}"
        printf "MISEQ_WORKFLOW = %s\n" "$Workflow"
        printf "APPLICATION = %s\n" "$Application"
        printf "ASSAY = %s\n" "$Assay"
        printf "CHEMISTRY = %s\n" "$Chemistry"
        printf "READ_1 = %s\n" "$Read1"
        printf "READ_2 = %s\n" "$Read2"
        printf "PHIX_CONCENTRATION = %s\n" "$PhiX"
        printf "RGT_Box_1 = %s\n" "$RGT_box1"
        printf "RGT_Box_2 = %s\n" "$RGT_box2"
    } > "$prop_file"

    dst="${datamoverDST}/${run_name_here}"
    rsync -a --rsync-path="mkdir -p $dst && rsync" "$prop_file" "ozagor@datamover:$dst"
    # save rsync exit status, if 0 then success
    rsync1=$?

    ### sample_type MISEQ_RUN
    project_to_write=$2

    prop_file=dataset.properties
    {
        printf "SPACE = IMV\n"
        printf "PROJECT = %s\n" "$project_to_write"
        printf "EXPERIMENT = MISEQ_RUNS\n"
        printf "SAMPLE = %s\n" "$run_name_here"
        printf "SAMPLE_TYPE = MISEQ_RUN\n"
        printf "DATASET_TYPE = DATAMOVER_SAMPLE_CREATOR\n"
    } > "$prop_file"

    rsync -a "$prop_file" "ozagor@datamover:$dst"
    rsync2=$?
    if [ "$rsync1" -eq "0" ] && [ "$rsync2" -eq "0" ]; then
        ssh ozagor@datamover touch "$datamoverDST/.MARKER_is_finished_${run_name_here}" </dev/null
    else
        echo -e "WATCH OUT: touching the void! $rsync1 $rsync2"
    fi

}

write_miseq_sample_zero(){

    sample_number=0
    sample_name='Undetermined'
    run_name="$(basename $rundir)"

    fastq_file=$incomingdir/$run_name/Data/Intensities/BaseCalls/${sample_name}_S${sample_number}_L001_R1_001.fastq.gz
    fastq_file_2=$incomingdir/$run_name/Data/Intensities/BaseCalls/${sample_name}_S${sample_number}_L001_R2_001.fastq.gz

    echo "Syncing to TIMAVO"
    DST2="${timavoDST}/MiSeqOutput/${run_name}/Data/Intensities/BaseCalls/"
    rsync -a --rsync-path="mkdir -p $DST2 && rsync" "$fastq_file" "timavo:$DST2"
    if [ -e "$fastq_file_2" ]; then
        rsync "$fastq_file_2" "timavo:$DST2"
    fi

    ### write properties file

    ### sample_type MISEQ_SAMPLE
    echo "Writing properties files for MISEQ_SAMPLE ID:${sample_number} NAME:${sample_name}"
    prop_file=sample.properties
    {
        printf "SAMPLE_ID=0\n"
        printf "SAMPLE_NAME=Undetermined\n"
        printf "SAMPLE_PLATE=\n"
        printf "SAMPLE_WELL=\n"
        printf "I7_INDEX_ID=\n"
        printf "INDEX_1=\n"
        printf "I5_INDEX_ID=\n"
        printf "INDEX_2=\n"
        printf "DESCRIPTION=\n"
    } > "$prop_file"

    dst="${datamoverDST}/${run_name}-${sample_number}"
    rsync -a --rsync-path="mkdir -p $dst && rsync" "$prop_file" "ozagor@datamover:$dst"
    # save rsync exit status, if 0 then success
    rsync1=$?

    prop_file=dataset.properties
    {
        printf "SPACE = IMV\n"
        printf "PROJECT = METAGENOMICS\n"
        printf "EXPERIMENT = MISEQ_SAMPLES\n"
        printf "SAMPLE = %s-%s\n" "${run_name}" "${sample_number}"
        printf "SAMPLE_TYPE = MISEQ_SAMPLE\n"
        printf "DATASET_TYPE = FASTQ\n"
    } > "$prop_file"

    rsync -a "$prop_file" "ozagor@datamover:$dst"
    rsync2=$?

    rsync3=0
    if [ -e "$fastq_file" ]; then
        rsync -a "$fastq_file" "ozagor@datamover:$dst"
        rsync3=$?
        # echo "R1 file exists, rsync returns $rsync3"
    else
        echo "R1 file does not exist"
    fi

    rsync4=0
    if [ -e "$fastq_file_2" ]; then
        rsync -a "$fastq_file_2" "ozagor@datamover:$dst"
        rsync4=$?
        echo "R2 file exists, rsync returns $rsync4"
    # else
    #     echo "R2 file does not exist"
    fi

    if [ "$rsync1" -eq "0" ] && [ "$rsync2" -eq "0" ] && [ "$rsync3" -eq "0" ] && [ "$rsync4" -eq "0" ]; then
        ssh ozagor@datamover touch "$datamoverDST/.MARKER_is_finished_${run_name}-${sample_number}" </dev/null
    else
        echo -e "WATCH OUT: touching the void! $rsync1 $rsync2 $rsync3 $rsync4"
    fi

}

write_miseq_sample(){
    # input is a line in section [Data] of the samples sheet
    # copy fastq file into $bufferdir/$rundir/S.../ and write dataset.properties
    # if timavo == y, sync fastq file on timavo too

    declare -a sample_line=("${!1}")

    ### move fastq file into folder

    sample_number=${sample_line[0]}
    # this should mimic the behaviour of MiSeq Reporter on sample names with
    # spaces and punctuations:
    # - remove leadin and trailing whitespaces (first sed)
    # - replace punctuation characters and internal spaces with dashes
    sample_name=$(echo ${sample_line[1]} | sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//' | tr -s '[:punct:] ' '-')
    sample_plate=${sample_line[2]}
    sample_well=${sample_line[3]}
    I7_index_id=${sample_line[4]}
    index_1=${sample_line[5]}
    I5_index_id=${sample_line[6]}
    index_2=${sample_line[7]}
    description=${sample_line[9]}
    timavo=${sample_line[14]}

    run_name="$(basename $rundir)"

    fastq_file=$incomingdir/$run_name/Data/Intensities/BaseCalls/${sample_name}_S${sample_number}_L001_R1_001.fastq.gz
    fastq_file_2=$incomingdir/$run_name/Data/Intensities/BaseCalls/${sample_name}_S${sample_number}_L001_R2_001.fastq.gz

    if [[ $timavo == *"y"* ]]; then
        echo "Syncing to TIMAVO"
        DST2="${timavoDST}/MiSeqOutput/${run_name}/Data/Intensities/BaseCalls/"
        rsync -a --rsync-path="mkdir -p $DST2 && rsync" "$fastq_file" "timavo:$DST2"
        if [ -e "$fastq_file_2" ]; then
            rsync "$fastq_file_2" "timavo:$DST2"
        fi
      else
        echo "Not syncing to TIMAVO"
    fi

    ### write properties file

    ### sample_type MISEQ_SAMPLE
    echo "Writing properties files for MISEQ_SAMPLE ID:${sample_number} NAME:${sample_name}"
    prop_file=sample.properties
    {
        printf "SAMPLE_ID=%s\n" "${sample_number}"
        printf "SAMPLE_NAME=%s\n" "${sample_name}"
        printf "SAMPLE_PLATE=%s\n" "${sample_plate}"
        printf "SAMPLE_WELL=%s\n" "$sample_well"
        printf "I7_INDEX_ID=%s\n" "$I7_index_id"
        printf "INDEX_1=%s\n" "$index_1"
        printf "I5_INDEX_ID=%s\n" "$I5_index_id"
        printf "INDEX_2=%s\n" "$index_2"
        printf "DESCRIPTION=%s\n" "$description"
    } > "$prop_file"

    dst="${datamoverDST}/${run_name}-${sample_number}"
    rsync -a --rsync-path="mkdir -p $dst && rsync" "$prop_file" "ozagor@datamover:$dst"
    # save rsync exit status, if 0 then success
    rsync1=$?

    Project=${sample_line[8]}

    prop_file=dataset.properties
    {
        printf "SPACE = IMV\n"
        printf "PROJECT = %s\n" "$Project"
        printf "EXPERIMENT = MISEQ_SAMPLES\n"
        printf "SAMPLE = %s-%s\n" "${run_name}" "${sample_number}"
        printf "SAMPLE_TYPE = MISEQ_SAMPLE\n"
        printf "DATASET_TYPE = FASTQ\n"
    } > "$prop_file"

    rsync -a "$prop_file" "ozagor@datamover:$dst"
    rsync2=$?

    rsync3=0
    if [ -e "$fastq_file" ]; then
        rsync -a "$fastq_file" "ozagor@datamover:$dst"
        rsync3=$?
        # echo "R1 file exists, rsync returns $rsync3"
    else
        echo "R1 file does not exist"
    fi

    rsync4=0
    if [ -e "$fastq_file_2" ]; then
        rsync -a "$fastq_file_2" "ozagor@datamover:$dst"
        rsync4=$?
        echo "R2 file exists, rsync returns $rsync4"
    # else
    #     echo "R2 file does not exist"
    fi

    if [ "$rsync1" -eq "0" ] && [ "$rsync2" -eq "0" ] && [ "$rsync3" -eq "0" ] && [ "$rsync4" -eq "0" ]; then
        ssh ozagor@datamover touch "$datamoverDST/.MARKER_is_finished_${run_name}-${sample_number}" </dev/null
    else
        echo -e "WATCH OUT: touching the void! $rsync1 $rsync2 $rsync3 $rsync4"
    fi

}

write_resistance_test(){

    declare -a sample_line=("${!1}")

    sample_number=${sample_line[0]}
    sample_name=${sample_line[1]}
    #sample_project=${sample_line[8]}
    #Description=${sample_line[9]}
    virus=${sample_line[10]}
    genotype=${sample_line[11]}
    target=${sample_line[12]}
    viral_load=${sample_line[13]}

    run_name=$(basename "$rundir")

    echo "Writing properties files for RESISTANCE_TEST RUN:${run_name} SAMPLE:${sample_name}"

    ### write properties file
    prop_file=sample.properties
    {
        printf "SAMPLE_NAME=%s\n" "$sample_name"
        printf "VIRUS=%s\n" "$virus"
        printf "TARGET_REGION=%s\n" "$target"
        printf "GENOTYPE=%s\n" "$genotype"
        printf "VIRAL_LOAD=%s\n" "$viral_load"
    } > "$prop_file"

    dst="${datamoverDST}/${run_name}-${sample_number}_RESISTANCE"
    rsync -a --rsync-path="mkdir -p $dst && rsync" "$prop_file" "ozagor@datamover:$dst"
    # save rsync exit status, if 0 then success
    rsync1=$?

    prop_file=dataset.properties
    {
        printf "SPACE = IMV\n"
        printf "PROJECT = RESISTANCE\n"
        printf "EXPERIMENT = RESISTANCE_TESTS\n"
        printf "SAMPLE = %s-%s_RESISTANCE\n" "${run_name}" "${sample_number}"
        printf "SAMPLE_TYPE = RESISTANCE_TEST\n"
        printf "DATASET_TYPE = DATAMOVER_SAMPLE_CREATOR\n"
    } > "$prop_file"

    rsync -a "$prop_file" "ozagor@datamover:$dst"
    rsync2=$?

    if [ "$rsync1" -eq "0" ] && [ "$rsync2" -eq "0" ]; then
        ssh ozagor@datamover touch "$datamoverDST/.MARKER_is_finished_${run_name}-${sample_number}_RESISTANCE" </dev/null
    else
        echo -e "WATCH OUT: touching the void! $rsync1 $rsync2"
    fi

}

process_runs(){
    # read the samples sheet,
    # save info found in [Header] and [Reads] in global variables
    # call process_sample on all lines in [Data] section

    rundir=$1
    if [[ -e $rundir/.UPLOADED_RUN ]]; then
        echo "Run $rundir already uploaded"
        return
    fi

    # reset headers
    headers='undefined'

    ### remove spaces in sample sheet
    #sed -e "s/ /_/g" < "$rundir/Data/Intensities/BaseCalls/SampleSheet.csv" > sample_sheet.tmp
    # sample sheets are now created with Stefan's template: no need to remove them anymore
    cp "$rundir/SampleSheet.csv" sample_sheet.tmp
    count_openbis=$(grep -c openbis sample_sheet.tmp)
    ### counters for read 1/2 and samples
    r=0
    s=0
    res_sample=false
    meta_sample=false
    plasm_sample=false
    anti_sample=false
    other_sample=false

    ### read sample sheet line by line splitting on commas
    ### || [[ -n "$line"]] allow reading last line also if file does not end
    ### with a newline
    while IFS=',' read -r -a line || [[ -n "$line" ]]
    do
        # if [[ "$count_openbis" == 0 ]]
        # then
        #     break
        # fi
        if [[ ${line[0]} =~ ^\[[[:alpha:]]*\] ]]
        then
            section=${line[0]}

        ### [Header] section
        elif [[ $section == "[Header]" && ${line[1]} ]]
        then
            declare "${line[0]}=${line[1]}"

        ### [Reads] section for read 1
        elif [[ $section == "[Reads]" && ${line[0]} =~ ^[0-9]+$ && $r -eq 0 ]]
        then
            # [Header] has now been parsed, if openbis != y then stop parsing
            # this run and go to next
            if [[ "$openbis" == "n" ]]; then
                break
            fi
            r=1
            # echo $section ${line[0]}
            Read1=${line[0]}

        ### [Reads] section for read 2
        elif [[ $section == "[Reads]" && ${line[0]} =~ ^[0-9]+$ && $r -eq 1 ]]
        then
            r=2
            # echo $section ${line[0]}
            Read2=${line[0]}

        ### [Data] section headers
        elif [[ $section == "[Data]" && ${line[1]} && $s -eq 0 ]]
        then
            headers=( "${line[@]}" )
            echo -e "  Headers found:" "${headers[@]}"
            ((s+=1))

        ### [Data] section values
        elif [[ $section == "[Data]" && ${line[1]} && $s -gt 0 ]]
        then

            write_miseq_sample line[@]

            case ${line[8]} in
              Antibodies)
                anti_sample=true
                ;;
              Metagenomics)
                meta_sample=true
                ;;
              Other)
                other_sample=true
                ;;
              Plasmids)
                plasm_sample=true
                ;;
              Resistance)
                res_sample=true
                write_resistance_test line[@]
                ;;
              esac
            ((s+=1))
        fi

    done < sample_sheet.tmp

    echo "Syncing SampleSheet to timavo"
    run_name=$(basename "$rundir")
    smpshdst="${timavoDST}/MiSeqOutput/${run_name}/Data/Intensities/BaseCalls/"
    rsync -av --rsync-path="mkdir -p $smpshdst && rsync" sample_sheet.tmp "timavo:$smpshdst/SampleSheet.csv"
    rm sample_sheet.tmp

    # if any sample was X then write Miseq run sample with PROJECT = X
    if [ "$anti_sample" = true ]; then
        echo "WRITING ANTIBODIES RUN"
        write_miseq_run "${run_name}_ANTIBODIES" Antibodies
    fi
    if [ "$meta_sample" = true ]; then
        echo "WRITING UNDETERMINED READS"
        write_miseq_sample_zero
        echo "WRITING METAGENOMICS RUN"
        write_miseq_run "${run_name}_METAGENOMICS" Metagenomics
    fi
    if [ "$other_sample" = true ]; then
        echo "WRITING OTHER RUN"
        write_miseq_run "${run_name}_OTHER" Other
    fi
    if [ "$plasm_sample" = true ]; then
        echo "WRITING PLASMIDS RUN"
        write_miseq_run "${run_name}_PLASMIDS" Plasmids
    fi
    if [ "$res_sample" = true ]; then
        echo "WRITING RESISTANCE RUN"
        write_miseq_run "${run_name}_RESISTANCE" Resistance
    fi

    # get sample sheet name from runParameter.xml file and save runParameter.xml file with the sample sheet name
    SAMPLESHEET=$(grep -A 1 ReagentKitRFIDTag "$rundir/runParameters.xml" | grep SerialNumber | sed 's/^.*<SerialNumber>//' | sed 's/<\/SerialNumber>//')
    rsync -av --stats --chmod=ug+rwx -p "$rundir/runParameters.xml" "timavo:$timavoDST/MiSeqRunParameters/${SAMPLESHEET}.xml"

    # reset [Header] and [Reads] information
    Investigator_Name='undefined'
    Experiment_Name='undefined'
#    Date='undefined'
    Workflow='undefined'
    Application='undefined'
    Assay='undefined'
#    Description='undefined'
    Chemistry='undefined'
    Read1=''
    Read2=''
    PhiX='undefined'
    RGT_box1='undefined'
    RGT_box2='undefined'
    openbis='n'

    touch "$rundir/.UPLOADED_RUN"

}

echo 'GO!'
### main loop over all dirs in $incomingdir starting with "1"
#for rundir in $(find $incomingdir -type d -name "1*" -depth 1)
#for rundir in $(ls $incomingdir)
for rundir in "$incomingdir"/1*; do
    [[ -d "$rundir" ]] || continue
    echo -e "\033[1;31m================================================================================\033[0m"
    echo "Now syncing:" "$rundir"
    process_runs "$rundir"
done

# copy SampleSheets
rsync -av --stats --chmod=ug+rwx -p /cygdrive/i/MiSeqSampleSheets "timavo:$timavoDST"


updates

updates

remote: Support for password authentication was removed on August 13, 2021.
remote: Please see https://docs.github.com/en/get-started/getting-started-with-git/about-remote-repositories#cloning-with-https-urls for information on currently recommended modes of authentication.
fatal: Authentication failed for 'https://github.com/flurischt/dockerfiles.git/'


ngolo-fuzzing: fix coverage build (#7682)

#!/bin/bash -eu
# Copyright 2021 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
################################################################################

# compile latest go from git
(
cd $SRC/goroot/src
./make.bash
)
rm -Rf /root/.go/
mv $SRC/goroot /root/.go

compile_package () {
    pkg=$1
    pkg_flat=`echo $pkg | sed 's/\//_/g'`
    args=`cat $SRC/ngolo-fuzzing/std/args.txt | grep "^$pkg_flat " | cut -d" " -f2-`
    ./ngolo-fuzzing $args $pkg fuzz_ng_$pkg_flat
    (
        cd fuzz_ng_$pkg_flat
        $SRC/LPM/external.protobuf/bin/protoc --go_out=./ ngolofuzz.proto
        mkdir cpp
        $SRC/LPM/external.protobuf/bin/protoc --cpp_out=./cpp ngolofuzz.proto
        $CXX -stdlib=libc++ -c -I . -I $SRC/LPM/external.protobuf/include cpp/ngolofuzz.pb.cc
        $CXX $CXXFLAGS -c -Icpp -I $SRC/libprotobuf-mutator/ -I $SRC/LPM/external.protobuf/include $SRC/ngolo-fuzzing/lpm/ngolofuzz.cc
    )
    if [ "$SANITIZER" = "coverage" ]
    then
        (
        cd fuzz_ng_$pkg_flat
        compile_go_fuzzer . FuzzNG_unsure fuzz_ngo_$pkg_flat
        )
    else
        compile_go_fuzzer ./fuzz_ng_$pkg_flat FuzzNG_unsure fuzz_ngo_$pkg_flat
        ./go114-fuzz-build/go114-fuzz-build -func FuzzNG_valid -o fuzz_ng_$pkg_flat.a ./fuzz_ng_$pkg_flat

        $CXX $CXXFLAGS $LIB_FUZZING_ENGINE fuzz_ng_$pkg_flat/ngolofuzz.pb.o fuzz_ng_$pkg_flat//ngolofuzz.o fuzz_ng_$pkg_flat.a  $SRC/LPM/src/libfuzzer/libprotobuf-mutator-libfuzzer.a $SRC/LPM/src/libprotobuf-mutator.a $SRC/LPM/external.protobuf/lib/libprotobuf.a -o $OUT/fuzz_ng_$pkg_flat
    fi
}

# in $SRC/ngolo-fuzzing
go build

(
cd go114-fuzz-build
go build
)

# maybe we should git clone --depth 1 https://github.com/golang/go.git
find /root/.go/src/ -type d | cut -d/ -f5- | while read pkg; do
    if [[ `ls /root/.go/src/$pkg/*.go | wc -l` == '0' ]]; then
        continue
    fi
    if [[ `echo $pkg | grep internal | wc -l` == '1' ]]; then
        continue
    fi
    if [[ `echo $pkg | grep vendor | wc -l` == '1' ]]; then
        continue
    fi
    if [[ `echo $pkg | grep testdata | wc -l` == '1' ]]; then
        continue
    fi
    if compile_package $pkg; then
        echo $pkg >> ok.txt
    else
        echo "Failed for $pkg"
        # hard fail if the package is meant to be supported
        grep ^$pkg$ $SRC/ngolo-fuzzing/std/supported.txt && exit 1
        echo $pkg >> ko.txt
    fi

done

echo "Failed packages:"
cat ko.txt

echo "Succesful packages:"
cat ok.txt

#!/bin/bash -eu
# Copyright 2021 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
################################################################################

# compile latest go from git
(
cd $SRC/goroot/src
./make.bash
)
rm -Rf /root/.go/
mv $SRC/goroot /root/.go

compile_package () {
    pkg=$1
    pkg_flat=`echo $pkg | sed 's/\//_/g'`
    args=`cat $SRC/ngolo-fuzzing/std/args.txt | grep "^$pkg_flat " | cut -d" " -f2-`
    ./ngolo-fuzzing $args $pkg fuzz_ng_$pkg_flat
    (
        cd fuzz_ng_$pkg_flat
        $SRC/LPM/external.protobuf/bin/protoc --go_out=./ ngolofuzz.proto
        mkdir cpp
        $SRC/LPM/external.protobuf/bin/protoc --cpp_out=./cpp ngolofuzz.proto
        $CXX -stdlib=libc++ -c -I . -I $SRC/LPM/external.protobuf/include cpp/ngolofuzz.pb.cc
        $CXX $CXXFLAGS -c -Icpp -I $SRC/libprotobuf-mutator/ -I $SRC/LPM/external.protobuf/include $SRC/ngolo-fuzzing/lpm/ngolofuzz.cc
    )
    if [ "$SANITIZER" != "coverage" ]
    then
        (
        cd fuzz_ng_$pkg_flat
        compile_go_fuzzer . FuzzNG_unsure fuzz_ngo_$pkg_flat
        )
    else
        compile_go_fuzzer ./fuzz_ng_$pkg_flat FuzzNG_unsure fuzz_ngo_$pkg_flat
        ./go114-fuzz-build/go114-fuzz-build -func FuzzNG_valid -o fuzz_ng_$pkg_flat.a ./fuzz_ng_$pkg_flat

        $CXX $CXXFLAGS $LIB_FUZZING_ENGINE fuzz_ng_$pkg_flat/ngolofuzz.pb.o fuzz_ng_$pkg_flat//ngolofuzz.o fuzz_ng_$pkg_flat.a  $SRC/LPM/src/libfuzzer/libprotobuf-mutator-libfuzzer.a $SRC/LPM/src/libprotobuf-mutator.a $SRC/LPM/external.protobuf/lib/libprotobuf.a -o $OUT/fuzz_ng_$pkg_flat
    fi
}

# in $SRC/ngolo-fuzzing
go build

(
cd go114-fuzz-build
go build
)

# maybe we should git clone --depth 1 https://github.com/golang/go.git
find /root/.go/src/ -type d | cut -d/ -f5- | while read pkg; do
    if [[ `ls /root/.go/src/$pkg/*.go | wc -l` == '0' ]]; then
        continue
    fi
    if [[ `echo $pkg | grep internal | wc -l` == '1' ]]; then
        continue
    fi
    if [[ `echo $pkg | grep vendor | wc -l` == '1' ]]; then
        continue
    fi
    if [[ `echo $pkg | grep testdata | wc -l` == '1' ]]; then
        continue
    fi
    if compile_package $pkg; then
        echo $pkg >> ok.txt
    else
        echo "Failed for $pkg"
        # hard fail if the package is meant to be supported
        grep ^$pkg$ $SRC/ngolo-fuzzing/std/supported.txt && exit 1
        echo $pkg >> ko.txt
    fi

done

echo "Failed packages:"
cat ko.txt

echo "Succesful packages:"
cat ok.txt


Add Jenkins test script

Add Jenkins test script

#!/bin/bash
# Usage: test.sh
# This script is from the "Passenger binaries test" Jenkins job. It builds
# portable binaries for Passenger and Nginx and runs tests against them.
#
# Required environment variables:
#
#   WORKSPACE
#
# Optional environment variables:
#
#   PASSENGER_ROOT (defaults to $WORKSPACE)
#   CONCURRENCY
#
# Sample invocation in a development environment:
#
#   env WORKSPACE=$HOME PASSENGER_ROOT=/passenger ./jenkins/test/test.sh

set -e
SELFDIR=`dirname "$0"`
cd "$SELFDIR/../.."
source "./shared/lib/library.sh"

require_envvar WORKSPACE "$WORKSPACE"

PASSENGER_ROOT="${PASSENGER_ROOT:-$WORKSPACE}"
CONCURRENCY=${CONCURRENCY:-4}

# Sleep for a random amount of time in order to work around Docker/AUFS bugs
# that may be triggered if multiple containers are shut down at the same time.
echo 'import random, time; time.sleep(random.random() * 4)' | python

run mkdir -p cache output
run ./linux/build \
	-p "$PASSENGER_ROOT" \
	-c "$WORKSPACE/cache/x86" \
	-o "$WORKSPACE/output/x86" \
	-a x86 \
	passenger nginx
run ./linux/build \
	-p "$PASSENGER_ROOT" \
	-c "$WORKSPACE/cache/x86_64" \
	-o "$WORKSPACE/output/x86_64" \
	-a x86_64 \
	passenger nginx

error: pathspec 'jenkins/test/test.sh' did not match any file(s) known to git


New script to increase Inotify limit.

New script to increase Inotify limit.

#!/bin/sh

# Increase the INotify limit in case of "INotify: Too many open files"
LIMIT=512
FILE="/proc/sys/fs/inotify/max_user_instances"

echo $LIMIT > $FILE

error: pathspec 'openvz/increaseInotifyLimit.sh' did not match any file(s) known to git