Update online_gunicorn.sh

Update online_gunicorn.sh

remote: Support for password authentication was removed on August 13, 2021.
remote: Please see https://docs.github.com/en/get-started/getting-started-with-git/about-remote-repositories#cloning-with-https-urls for information on currently recommended modes of authentication.
fatal: Authentication failed for 'https://github.com/saintic/Template.git/'


Add in a script that checks the write endurance of a dell nvme card

#!/bin/bash

# Get current status of cache
CURRENT_CACHE_SIZE=`/opt/dell/srvadmin/bin/omreport storage nvmeadapter controller=3 |awk -F'[ %]' '/^Remaining Rated Write Endurance/ {print $6}'`
CRITICAL=25

if [[ $CURRENT_CACHE_SIZE -le $CRITICAL ]]; then
  STATUS="CRITICAL - Cache is under $CRITICAL%"
  EXIT=2
else
  STATUS="Ok - Move along, nothing to see here"
  EXIT=0
fi

# Print status and exit
echo $STATUS
exit $EXIT

error: pathspec 'check_dell_nvme_write_endurance.sh' did not match any file(s) known to git


   Increase the threshold for LINUX_CPU_MAX_WHL_SIZE to 152M

PiperOrigin-RevId: 345526552
Change-Id: I9c1375463664a33bbec62bc689d59cb046c76270

#!/bin/bash
# Copyright 2016 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
# Pip install TensorFlow and run basic test on the pip package.

set -e
set -x

# CPU size
MAC_CPU_MAX_WHL_SIZE=175M
LINUX_CPU_MAX_WHL_SIZE=152M
WIN_CPU_MAX_WHL_SIZE=113M
# GPU size
LINUX_GPU_MAX_WHL_SIZE=390M
WIN_GPU_MAX_WHL_SIZE=252M

function run_smoke_test() {
  VENV_TMP_DIR=$(mktemp -d)

  ${PYTHON_BIN_PATH} -m virtualenv -p ${PYTHON_BIN_PATH} "${VENV_TMP_DIR}" || \
      die "FAILED: Unable to create virtualenv"

  source "${VENV_TMP_DIR}/bin/activate" || \
      die "FAILED: Unable to activate virtualenv "

  # install tensorflow
  python -m pip install ${WHL_NAME} || \
      die "pip install (forcing to reinstall tensorflow) FAILED"
      echo "Successfully installed pip package ${WHL_NAME}"

  # Test TensorflowFlow imports
  test_tf_imports

  # Test TensorFlow whl file size
  test_tf_whl_size

  RESULT=$?
  # Deactivate from virtualenv.
  deactivate || source deactivate || die "FAILED: Unable to deactivate from existing virtualenv."
  sudo rm -rf "${KOKORO_GFILE_DIR}/venv"
  return $RESULT
}

function test_tf_imports() {
  TMP_DIR=$(mktemp -d)
  pushd "${TMP_DIR}"

  # test for basic import and perform tf.add operation.
  RET_VAL=$(python -c "import tensorflow as tf; t1=tf.constant([1,2,3,4]); t2=tf.constant([5,6,7,8]); print(tf.add(t1,t2).shape)")
  if ! [[ ${RET_VAL} == *'(4,)'* ]]; then
    echo "Unexpected return value: ${RET_VALUE}"
    echo "PIP test on virtualenv FAILED, will not upload ${WHL_NAME} package."
     return 1
  fi

  # test basic keras is available
  RET_VAL=$(python -c "import tensorflow as tf; print(tf.keras.__name__)")
  if ! [[ ${RET_VAL} == *'tensorflow.keras'* ]]; then
    echo "Unexpected return value: ${RET_VALUE}"
    echo "PIP test on virtualenv FAILED, will not upload ${WHL_NAME} package."
    return 1
  fi

  # similar test for estimator
  RET_VAL=$(python -c "import tensorflow as tf; print(tf.estimator.__name__)")
  if ! [[ ${RET_VAL} == *'tensorflow_estimator.python.estimator.api._v2.estimator'* ]]; then
    echo "Unexpected return value: ${RET_VALUE}"
    echo "PIP test on virtualenv FAILED, will not upload ${WHL_NAME} package."
    return 1
  fi

  RESULT=$?

  popd
  return $RESULT
}

function test_tf_whl_size() {
  # Check CPU whl size.
  if [[ "$WHL_NAME" == *"_cpu"* ]]; then
    # Check MAC CPU whl size.
    if [[ "$WHL_NAME" == *"-macos"* ]] && [[ $(find $WHL_NAME -type f -size +${MAC_CPU_MAX_WHL_SIZE}) ]]; then
        echo "Mac CPU whl size has exceeded ${MAC_CPU_MAX_WHL_SIZE}. To keep
within pypi's CDN distribution limit, we must not exceed that threshold."
      return 1
    fi
    # Check Linux CPU whl size.
    if [[ "$WHL_NAME" == *"-manylinux"* ]] && [[ $(find $WHL_NAME -type f -size +${LINUX_CPU_MAX_WHL_SIZE}) ]]; then
        echo "Linux CPU whl size has exceeded ${LINUX_CPU_MAX_WHL_SIZE}. To keep
within pypi's CDN distribution limit, we must not exceed that threshold."
      return 1
    fi
    # Check Windows CPU whl size.
    if [[ "$WHL_NAME" == *"-win"* ]] && [[ $(find $WHL_NAME -type f -size +${WIN_CPU_MAX_WHL_SIZE}) ]]; then
        echo "Windows CPU whl size has exceeded ${WIN_CPU_MAX_WHL_SIZE}. To keep
within pypi's CDN distribution limit, we must not exceed that threshold."
      return 1
    fi
  # Check GPU whl size
  elif [[ "$WHL_NAME" == *"_gpu"* ]]; then
    # Check Linux GPU whl size.
    if [[ "$WHL_NAME" == *"-manylinux"* ]] && [[ $(find $WHL_NAME -type f -size +${LINUX_GPU_MAX_WHL_SIZE}) ]]; then
        echo "Linux GPU whl size has exceeded ${LINUX_GPU_MAX_WHL_SIZE}. To keep
within pypi's CDN distribution limit, we must not exceed that threshold."
      return 1
    fi
    # Check Windows GPU whl size.
    if [[ "$WHL_NAME" == *"-win"* ]] && [[ $(find $WHL_NAME -type f -size +${WIN_GPU_MAX_WHL_SIZE}) ]]; then
        echo "Windows GPU whl size has exceeded ${WIN_GPU_MAX_WHL_SIZE}. To keep
within pypi's CDN distribution limit, we must not exceed that threshold."
      return 1
    fi
  fi
}

###########################################################################
# Main
###########################################################################
if [[ -z "${1}" ]]; then
  echo "TF WHL path not given, unable to install and test."
  return 1
fi

WHL_NAME=${1}
run_smoke_test

#!/bin/bash
# Copyright 2016 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
# Pip install TensorFlow and run basic test on the pip package.

set -e
set -x

# CPU size
MAC_CPU_MAX_WHL_SIZE=175M
LINUX_CPU_MAX_WHL_SIZE=142M
WIN_CPU_MAX_WHL_SIZE=113M
# GPU size
LINUX_GPU_MAX_WHL_SIZE=390M
WIN_GPU_MAX_WHL_SIZE=252M

function run_smoke_test() {
  VENV_TMP_DIR=$(mktemp -d)

  ${PYTHON_BIN_PATH} -m virtualenv -p ${PYTHON_BIN_PATH} "${VENV_TMP_DIR}" || \
      die "FAILED: Unable to create virtualenv"

  source "${VENV_TMP_DIR}/bin/activate" || \
      die "FAILED: Unable to activate virtualenv "

  # install tensorflow
  python -m pip install ${WHL_NAME} || \
      die "pip install (forcing to reinstall tensorflow) FAILED"
      echo "Successfully installed pip package ${WHL_NAME}"

  # Test TensorflowFlow imports
  test_tf_imports

  # Test TensorFlow whl file size
  test_tf_whl_size

  RESULT=$?
  # Deactivate from virtualenv.
  deactivate || source deactivate || die "FAILED: Unable to deactivate from existing virtualenv."
  sudo rm -rf "${KOKORO_GFILE_DIR}/venv"
  return $RESULT
}

function test_tf_imports() {
  TMP_DIR=$(mktemp -d)
  pushd "${TMP_DIR}"

  # test for basic import and perform tf.add operation.
  RET_VAL=$(python -c "import tensorflow as tf; t1=tf.constant([1,2,3,4]); t2=tf.constant([5,6,7,8]); print(tf.add(t1,t2).shape)")
  if ! [[ ${RET_VAL} == *'(4,)'* ]]; then
    echo "Unexpected return value: ${RET_VALUE}"
    echo "PIP test on virtualenv FAILED, will not upload ${WHL_NAME} package."
     return 1
  fi

  # test basic keras is available
  RET_VAL=$(python -c "import tensorflow as tf; print(tf.keras.__name__)")
  if ! [[ ${RET_VAL} == *'tensorflow.keras'* ]]; then
    echo "Unexpected return value: ${RET_VALUE}"
    echo "PIP test on virtualenv FAILED, will not upload ${WHL_NAME} package."
    return 1
  fi

  # similar test for estimator
  RET_VAL=$(python -c "import tensorflow as tf; print(tf.estimator.__name__)")
  if ! [[ ${RET_VAL} == *'tensorflow_estimator.python.estimator.api._v2.estimator'* ]]; then
    echo "Unexpected return value: ${RET_VALUE}"
    echo "PIP test on virtualenv FAILED, will not upload ${WHL_NAME} package."
    return 1
  fi

  RESULT=$?

  popd
  return $RESULT
}

function test_tf_whl_size() {
  # Check CPU whl size.
  if [[ "$WHL_NAME" == *"_cpu"* ]]; then
    # Check MAC CPU whl size.
    if [[ "$WHL_NAME" == *"-macos"* ]] && [[ $(find $WHL_NAME -type f -size +${MAC_CPU_MAX_WHL_SIZE}) ]]; then
        echo "Mac CPU whl size has exceeded ${MAC_CPU_MAX_WHL_SIZE}. To keep
within pypi's CDN distribution limit, we must not exceed that threshold."
      return 1
    fi
    # Check Linux CPU whl size.
    if [[ "$WHL_NAME" == *"-manylinux"* ]] && [[ $(find $WHL_NAME -type f -size +${LINUX_CPU_MAX_WHL_SIZE}) ]]; then
        echo "Linux CPU whl size has exceeded ${LINUX_CPU_MAX_WHL_SIZE}. To keep
within pypi's CDN distribution limit, we must not exceed that threshold."
      return 1
    fi
    # Check Windows CPU whl size.
    if [[ "$WHL_NAME" == *"-win"* ]] && [[ $(find $WHL_NAME -type f -size +${WIN_CPU_MAX_WHL_SIZE}) ]]; then
        echo "Windows CPU whl size has exceeded ${WIN_CPU_MAX_WHL_SIZE}. To keep
within pypi's CDN distribution limit, we must not exceed that threshold."
      return 1
    fi
  # Check GPU whl size
  elif [[ "$WHL_NAME" == *"_gpu"* ]]; then
    # Check Linux GPU whl size.
    if [[ "$WHL_NAME" == *"-manylinux"* ]] && [[ $(find $WHL_NAME -type f -size +${LINUX_GPU_MAX_WHL_SIZE}) ]]; then
        echo "Linux GPU whl size has exceeded ${LINUX_GPU_MAX_WHL_SIZE}. To keep
within pypi's CDN distribution limit, we must not exceed that threshold."
      return 1
    fi
    # Check Windows GPU whl size.
    if [[ "$WHL_NAME" == *"-win"* ]] && [[ $(find $WHL_NAME -type f -size +${WIN_GPU_MAX_WHL_SIZE}) ]]; then
        echo "Windows GPU whl size has exceeded ${WIN_GPU_MAX_WHL_SIZE}. To keep
within pypi's CDN distribution limit, we must not exceed that threshold."
      return 1
    fi
  fi
}

###########################################################################
# Main
###########################################################################
if [[ -z "${1}" ]]; then
  echo "TF WHL path not given, unable to install and test."
  return 1
fi

WHL_NAME=${1}
run_smoke_test


Update wbr-zsh.sh     add a point to back up old .zshrc

remote: Support for password authentication was removed on August 13, 2021.
remote: Please see https://docs.github.com/en/get-started/getting-started-with-git/about-remote-repositories#cloning-with-https-urls for information on currently recommended modes of authentication.
fatal: Authentication failed for 'https://github.com/stuha/wbr-zsh.git/'


Write sample zero: undetermined reads

Write sample zero: undetermined reads

Sample S0 is not listed in the sample sheet, but should be stored for
metagenomics runs, where it is used to assess overall contamination
(issue #1).

Further, fixed fastq_file_2 wrong path.

#!/bin/bash

#set -x  # uncomment for debugging
incomingdir=/cygdrive/D/Illumina/MiSeqOutput
timavoDST=/data/MiSeq
datamoverDST=data/outgoing

# define these globally so no need to pass it as a function parameter
headers='undefined'
rundir='undefined'
# IEMFileVersion='undefined'
Account=''
Investigator_Name='undefined'
Experiment_Name='undefined'
#Date='undefined'
Workflow='undefined'
Application='undefined'
Assay='undefined'
#Description='undefined'
Chemistry='undefined'
Read1=''
Read2=''
PhiX='undefined'
RGT_box1='undefined'
RGT_box2='undefined'
openbis='n'
timavo='n'

# global variable to allow a function return it
ismolis='undefined'

test_molis(){
    if [[ $1 =~ ^100[0-9]{7} ]]; then
        ismolis=true
        return
    else
        ismolis=false
        return
    fi
}

write_miseq_run(){
    # input: run_name space
    # space is Diagnostics or Research
    run_name=$1

    datenum=${run_name:0:6}
    YYYY=20${datenum:0:2}
    MM=${datenum:2:2}
    DD=${datenum:4:2}

    echo "Writing properties files for MISEQ_RUN:$1 PROJECT:$2"
    prop_file=sample.properties
    {
        printf "INVESTIGATOR_NAME = %s\n" "$Investigator_Name"
        printf "EXPERIMENT_NAME = %s\n" "$Experiment_Name"
        printf "ACCOUNT = %s\n" "$Account"
        printf "DATE = %s-%s-%s\n" "${YYYY}" "${MM}" "${DD}"
        printf "MISEQ_WORKFLOW = %s\n" "$Workflow"
        printf "APPLICATION = %s\n" "$Application"
        printf "ASSAY = %s\n" "$Assay"
        printf "CHEMISTRY = %s\n" "$Chemistry"
        printf "READ_1 = %s\n" "$Read1"
        printf "READ_2 = %s\n" "$Read2"
        printf "PHIX_CONCENTRATION = %s\n" "$PhiX"
        printf "RGT_Box_1 = %s\n" "$RGT_box1"
        printf "RGT_Box_2 = %s\n" "$RGT_box2"
    } > "$prop_file"

    dst="${datamoverDST}/${run_name}"
    rsync -a --rsync-path="mkdir -p $dst && rsync" "$prop_file" "ozagor@datamover:$dst"
    # save rsync exit status, if 0 then success
    rsync1=$?

    ### sample_type MISEQ_RUN
    project_to_write=$2

    prop_file=dataset.properties
    {
        printf "SPACE = IMV\n"
        printf "PROJECT = %s\n" "$project_to_write"
        printf "EXPERIMENT = MISEQ_RUNS\n"
        printf "SAMPLE = %s\n" "$run_name"
        printf "SAMPLE_TYPE = MISEQ_RUN\n"
        printf "DATASET_TYPE = DATAMOVER_SAMPLE_CREATOR\n"
    } > "$prop_file"

    rsync -a "$prop_file" "ozagor@datamover:$dst"
    rsync2=$?
    if [ "$rsync1" -eq "0" ] && [ "$rsync2" -eq "0" ]; then
        ssh ozagor@datamover touch "$datamoverDST/.MARKER_is_finished_${run_name}" </dev/null
    else
        echo -e "WATCH OUT: touching the void! $rsync1 $rsync2"
    fi

}

write_miseq_sample_zero(){

    sample_number=0
    sample_name='Undetermined'
    run_name="$(basename $rundir)"

    fastq_file=$incomingdir/$run_name/Data/Intensities/BaseCalls/${sample_name}_S${sample_number}_L001_R1_001.fastq.gz
    fastq_file_2=$incomingdir/$run_name/Data/Intensities/BaseCalls/${sample_name}_S${sample_number}_L001_R2_001.fastq.gz

    echo "Syncing to TIMAVO"
    DST2="${timavoDST}/MiSeqOutput/${run_name}/Data/Intensities/BaseCalls/"
    rsync -a --rsync-path="mkdir -p $DST2 && rsync" "$fastq_file" "timavo:$DST2"
    if [ -e "$fastq_file_2" ]; then
        rsync "$fastq_file_2" "$DST2"
    fi

    ### write properties file

    ### sample_type MISEQ_SAMPLE
    echo "Writing properties files for MISEQ_SAMPLE ID:${sample_number} NAME:${sample_name}"
    prop_file=sample.properties
    {
        printf "SAMPLE_ID=0\n"
        printf "SAMPLE_NAME=Undetermined\n"
        printf "SAMPLE_PLATE=\n"
        printf "SAMPLE_WELL=\n"
        printf "I7_INDEX_ID=\n"
        printf "INDEX_1=\n"
        printf "I5_INDEX_ID=\n"
        printf "INDEX_2=\n"
        printf "DESCRIPTION=\n"
    } > "$prop_file"

    dst="${datamoverDST}/${run_name}-${sample_number}"
    rsync -a --rsync-path="mkdir -p $dst && rsync" "$prop_file" "ozagor@datamover:$dst"
    # save rsync exit status, if 0 then success
    rsync1=$?

    prop_file=dataset.properties
    {
        printf "SPACE = IMV\n"
        printf "PROJECT = METAGENOMICS\n"
        printf "EXPERIMENT = MISEQ_SAMPLES\n"
        printf "SAMPLE = %s-%s\n" "${run_name}" "${sample_number}"
        printf "SAMPLE_TYPE = MISEQ_SAMPLE\n"
        printf "DATASET_TYPE = FASTQ\n"
    } > "$prop_file"

    rsync -a "$prop_file" "ozagor@datamover:$dst"
    rsync2=$?

    rsync3=0
    if [ -e "$fastq_file" ]; then
        rsync -a "$fastq_file" "ozagor@datamover:$dst"
        rsync3=$?
        # echo "R1 file exists, rsync returns $rsync3"
    else
        echo "R1 file does not exist"
    fi

    rsync4=0
    if [ -e "$fastq_file_2" ]; then
        rsync -a "$fastq_file_2" "ozagor@datamover:$dst"
        rsync4=$?
        echo "R2 file exists, rsync returns $rsync4"
    # else
    #     echo "R2 file does not exist"
    fi

    if [ "$rsync1" -eq "0" ] && [ "$rsync2" -eq "0" ] && [ "$rsync3" -eq "0" ] && [ "$rsync4" -eq "0" ]; then
        ssh ozagor@datamover touch "$datamoverDST/.MARKER_is_finished_${run_name}-${sample_number}" </dev/null
    else
        echo -e "WATCH OUT: touching the void! $rsync1 $rsync2 $rsync3 $rsync4"
    fi

}

write_miseq_sample(){
    # input is a line in section [Data] of the samples sheet
    # copy fastq file into $bufferdir/$rundir/S.../ and write dataset.properties
    # if timavo == y, sync fastq file on timavo too

    declare -a sample_line=("${!1}")

    ### move fastq file into folder

    sample_number=${sample_line[0]}
    # this should mimic the behaviour of MiSeq Reporter on sample names with
    # spaces and punctuations:
    # - remove leadin and trailing whitespaces (first sed)
    # - replace punctuation characters and internal spaces with dashes
    sample_name=$(echo ${sample_line[1]} | sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//' | tr -s '[:punct:] ' '-')
    sample_plate=${sample_line[2]}
    sample_well=${sample_line[3]}
    I7_index_id=${sample_line[4]}
    index_1=${sample_line[5]}
    I5_index_id=${sample_line[6]}
    index_2=${sample_line[7]}
    description=${sample_line[9]}
    timavo=${sample_line[14]}

    run_name="$(basename $rundir)"

    fastq_file=$incomingdir/$run_name/Data/Intensities/BaseCalls/${sample_name}_S${sample_number}_L001_R1_001.fastq.gz
    fastq_file_2=$incomingdir/$run_name/Data/Intensities/BaseCalls/${sample_name}_S${sample_number}_L001_R2_001.fastq.gz

    if [[ $timavo == *"y"* ]]; then
        echo "Syncing to TIMAVO"
        DST2="${timavoDST}/MiSeqOutput/${run_name}/Data/Intensities/BaseCalls/"
        rsync -a --rsync-path="mkdir -p $DST2 && rsync" "$fastq_file" "timavo:$DST2"
        if [ -e "$fastq_file_2" ]; then
            rsync "$fastq_file_2" "$DST2"
        fi
      else
        echo "Not syncing to TIMAVO"
    fi

    ### write properties file

    ### sample_type MISEQ_SAMPLE
    echo "Writing properties files for MISEQ_SAMPLE ID:${sample_number} NAME:${sample_name}"
    prop_file=sample.properties
    {
        printf "SAMPLE_ID=%s\n" "${sample_number}"
        printf "SAMPLE_NAME=%s\n" "${sample_name}"
        printf "SAMPLE_PLATE=%s\n" "${sample_plate}"
        printf "SAMPLE_WELL=%s\n" "$sample_well"
        printf "I7_INDEX_ID=%s\n" "$I7_index_id"
        printf "INDEX_1=%s\n" "$index_1"
        printf "I5_INDEX_ID=%s\n" "$I5_index_id"
        printf "INDEX_2=%s\n" "$index_2"
        printf "DESCRIPTION=%s\n" "$description"
    } > "$prop_file"

    dst="${datamoverDST}/${run_name}-${sample_number}"
    rsync -a --rsync-path="mkdir -p $dst && rsync" "$prop_file" "ozagor@datamover:$dst"
    # save rsync exit status, if 0 then success
    rsync1=$?

    Project=${sample_line[8]}

    prop_file=dataset.properties
    {
        printf "SPACE = IMV\n"
        printf "PROJECT = %s\n" "$Project"
        printf "EXPERIMENT = MISEQ_SAMPLES\n"
        printf "SAMPLE = %s-%s\n" "${run_name}" "${sample_number}"
        printf "SAMPLE_TYPE = MISEQ_SAMPLE\n"
        printf "DATASET_TYPE = FASTQ\n"
    } > "$prop_file"

    rsync -a "$prop_file" "ozagor@datamover:$dst"
    rsync2=$?

    rsync3=0
    if [ -e "$fastq_file" ]; then
        rsync -a "$fastq_file" "ozagor@datamover:$dst"
        rsync3=$?
        # echo "R1 file exists, rsync returns $rsync3"
    else
        echo "R1 file does not exist"
    fi

    rsync4=0
    if [ -e "$fastq_file_2" ]; then
        rsync -a "$fastq_file_2" "ozagor@datamover:$dst"
        rsync4=$?
        echo "R2 file exists, rsync returns $rsync4"
    # else
    #     echo "R2 file does not exist"
    fi

    if [ "$rsync1" -eq "0" ] && [ "$rsync2" -eq "0" ] && [ "$rsync3" -eq "0" ] && [ "$rsync4" -eq "0" ]; then
        ssh ozagor@datamover touch "$datamoverDST/.MARKER_is_finished_${run_name}-${sample_number}" </dev/null
    else
        echo -e "WATCH OUT: touching the void! $rsync1 $rsync2 $rsync3 $rsync4"
    fi

}

write_resistance_test(){

    declare -a sample_line=("${!1}")

    sample_number=${sample_line[0]}
    sample_name=${sample_line[1]}
    #sample_project=${sample_line[8]}
    #Description=${sample_line[9]}
    virus=${sample_line[10]}
    genotype=${sample_line[11]}
    target=${sample_line[12]}
    viral_load=${sample_line[13]}

    run_name=$(basename "$rundir")

    echo "Writing properties files for RESISTANCE_TEST RUN:${run_name} SAMPLE:${sample_name}"

    ### write properties file
    prop_file=sample.properties
    {
        printf "SAMPLE_NAME=%s\n" "$sample_name"
        printf "VIRUS=%s\n" "$virus"
        printf "TARGET_REGION=%s\n" "$target"
        printf "GENOTYPE=%s\n" "$genotype"
        printf "VIRAL_LOAD=%s\n" "$viral_load"
    } > "$prop_file"

    dst="${datamoverDST}/${run_name}-${sample_number}_RESISTANCE"
    rsync -a --rsync-path="mkdir -p $dst && rsync" "$prop_file" "ozagor@datamover:$dst"
    # save rsync exit status, if 0 then success
    rsync1=$?

    prop_file=dataset.properties
    {
        printf "SPACE = IMV\n"
        printf "PROJECT = RESISTANCE\n"
        printf "EXPERIMENT = RESISTANCE_TESTS\n"
        printf "SAMPLE = %s-%s_RESISTANCE\n" "${run_name}" "${sample_number}"
        printf "SAMPLE_TYPE = RESISTANCE_TEST\n"
        printf "DATASET_TYPE = DATAMOVER_SAMPLE_CREATOR\n"
    } > "$prop_file"

    rsync -a "$prop_file" "ozagor@datamover:$dst"
    rsync2=$?

    if [ "$rsync1" -eq "0" ] && [ "$rsync2" -eq "0" ]; then
        ssh ozagor@datamover touch "$datamoverDST/.MARKER_is_finished_${run_name}-${sample_number}_RESISTANCE" </dev/null
    else
        echo -e "WATCH OUT: touching the void! $rsync1 $rsync2"
    fi

}

process_runs(){
    # read the samples sheet,
    # save info found in [Header] and [Reads] in global variables
    # call process_sample on all lines in [Data] section

    rundir=$1
    if [[ -e $rundir/.UPLOADED_RUN ]]; then
        echo "Run $rundir already uploaded"
        return
    fi

    # reset headers
    headers='undefined'

    ### remove spaces in sample sheet
    #sed -e "s/ /_/g" < "$rundir/Data/Intensities/BaseCalls/SampleSheet.csv" > sample_sheet.tmp
    # sample sheets are now created with Stefan's template: no need to remove them anymore
    cp "$rundir/Data/Intensities/BaseCalls/SampleSheet.csv" sample_sheet.tmp
    count_openbis=$(grep -c openbis sample_sheet.tmp)
    ### counters for read 1/2 and samples
    r=0
    s=0
    res_sample=false
    meta_sample=false
    plasm_sample=false
    anti_sample=false
    other_sample=false

    ### read sample sheet line by line splitting on commas
    ### || [[ -n "$line"]] allow reading last line also if file does not end
    ### with a newline
    while IFS=',' read -r -a line || [[ -n "$line" ]]
    do
        # if [[ "$count_openbis" == 0 ]]
        # then
        #     break
        # fi
        if [[ ${line[0]} =~ ^\[[[:alpha:]]*\] ]]
        then
            section=${line[0]}

        ### [Header] section
        elif [[ $section == "[Header]" && ${line[1]} ]]
        then
            declare "${line[0]}=${line[1]}"

        ### [Reads] section for read 1
        elif [[ $section == "[Reads]" && ${line[0]} =~ ^[0-9]+$ && $r -eq 0 ]]
        then
            # [Header] has now been parsed, if openbis != y then stop parsing
            # this run and go to next
            if [[ "$openbis" == "n" ]]; then
                break
            fi
            r=1
            # echo $section ${line[0]}
            Read1=${line[0]}

        ### [Reads] section for read 2
        elif [[ $section == "[Reads]" && ${line[0]} =~ ^[0-9]+$ && $r -eq 1 ]]
        then
            r=2
            # echo $section ${line[0]}
            Read2=${line[0]}

        ### [Data] section headers
        elif [[ $section == "[Data]" && ${line[1]} && $s -eq 0 ]]
        then
            headers=( "${line[@]}" )
            echo -e "  Headers found:" "${headers[@]}"
            ((s+=1))

        ### [Data] section values
        elif [[ $section == "[Data]" && ${line[1]} && $s -gt 0 ]]
        then

            write_miseq_sample line[@]

            case ${line[8]} in
              Antibodies)
                anti_sample=true
                ;;
              Metagenomics)
                meta_sample=true
                ;;
              Other)
                other_sample=true
                ;;
              Plasmids)
                plasm_sample=true
                ;;
              Resistance)
                res_sample=true
                write_resistance_test line[@]
                ;;
              esac
            ((s+=1))
        fi

    done < sample_sheet.tmp

    echo "Syncing SampleSheet to timavo"
    run_name=$(basename "$rundir")
    smpshdst="${timavoDST}/MiSeqOutput/${run_name}/Data/Intensities/BaseCalls/"
    rsync -av --rsync-path="mkdir -p $smpshdst && rsync" sample_sheet.tmp "timavo:$smpshdst/SampleSheet.csv"
    rm sample_sheet.tmp

    # if any sample was X then write Miseq run sample with PROJECT = X
    if [ "$anti_sample" = true ]; then
        echo "WRITING ANTIBODIES RUN"
        write_miseq_run "${run_name}_ANTIBODIES" Antibodies
    fi
    if [ "$meta_sample" = true ]; then
        echo "WRITING UNDETERMINED READS"
        write_miseq_sample_zero
        echo "WRITING METAGENOMICS RUN"
        write_miseq_run "${run_name}_METAGENOMICS" Metagenomics
    fi
    if [ "$other_sample" = true ]; then
        echo "WRITING OTHER RUN"
        write_miseq_run "${run_name}_OTHER" Other
    fi
    if [ "$plasm_sample" = true ]; then
        echo "WRITING PLASMIDS RUN"
        write_miseq_run "${run_name}_PLASMIDS" Plasmids
    fi
    if [ "$res_sample" = true ]; then
        echo "WRITING RESISTANCE RUN"
        write_miseq_run "${run_name}_RESISTANCE" Resistance
    fi

    # get sample sheet name from runParameter.xml file and save runParameter.xml file with the sample sheet name
    SAMPLESHEET=$(grep -A 1 ReagentKitRFIDTag "$rundir/runParameters.xml" | grep SerialNumber | sed 's/^.*<SerialNumber>//' | sed 's/<\/SerialNumber>//')
    rsync -av --stats --chmod=ug+rwx -p "$rundir/runParameters.xml" "timavo:$timavoDST/MiSeqRunParameters/${SAMPLESHEET}.xml"

    # reset [Header] and [Reads] information
    Investigator_Name='undefined'
    Experiment_Name='undefined'
#    Date='undefined'
    Workflow='undefined'
    Application='undefined'
    Assay='undefined'
#    Description='undefined'
    Chemistry='undefined'
    Read1=''
    Read2=''
    PhiX='undefined'
    RGT_box1='undefined'
    RGT_box2='undefined'
    openbis='n'

    touch "$rundir/.UPLOADED_RUN"

}

echo 'GO!'
### main loop over all dirs in $incomingdir starting with "1"
#for rundir in $(find $incomingdir -type d -name "1*" -depth 1)
#for rundir in $(ls $incomingdir)
for rundir in "$incomingdir"/1*; do
    [[ -d "$rundir" ]] || continue
    echo -e "\033[1;31m================================================================================\033[0m"
    echo "Now syncing:" "$rundir"
    process_runs "$rundir"
done

# copy SampleSheets
rsync -av --stats --chmod=ug+rwx -p /cygdrive/d/Illumina/MiSeqSampleSheets "timavo:$timavoDST"

#!/bin/bash

#set -x  # uncomment for debugging
incomingdir=/cygdrive/D/Illumina/MiSeqOutput
timavoDST=/data/MiSeq
datamoverDST=data/outgoing

# define these globally so no need to pass it as a function parameter
headers='undefined'
rundir='undefined'
# IEMFileVersion='undefined'
Account=''
Investigator_Name='undefined'
Experiment_Name='undefined'
#Date='undefined'
Workflow='undefined'
Application='undefined'
Assay='undefined'
#Description='undefined'
Chemistry='undefined'
Read1=''
Read2=''
PhiX='undefined'
RGT_box1='undefined'
RGT_box2='undefined'
openbis='n'
timavo='n'

# global variable to allow a function return it
ismolis='undefined'

test_molis(){
    if [[ $1 =~ ^100[0-9]{7} ]]; then
        ismolis=true
        return
    else
        ismolis=false
        return
    fi
}

write_miseq_run(){
    # input: run_name space
    # space is Diagnostics or Research
    run_name=$1

    datenum=${run_name:0:6}
    YYYY=20${datenum:0:2}
    MM=${datenum:2:2}
    DD=${datenum:4:2}

    echo "Writing properties files for MISEQ_RUN:$1 PROJECT:$2"
    prop_file=sample.properties
    {
        printf "INVESTIGATOR_NAME = %s\n" "$Investigator_Name"
        printf "EXPERIMENT_NAME = %s\n" "$Experiment_Name"
        printf "ACCOUNT = %s\n" "$Account"
        printf "DATE = %s-%s-%s\n" "${YYYY}" "${MM}" "${DD}"
        printf "MISEQ_WORKFLOW = %s\n" "$Workflow"
        printf "APPLICATION = %s\n" "$Application"
        printf "ASSAY = %s\n" "$Assay"
        printf "CHEMISTRY = %s\n" "$Chemistry"
        printf "READ_1 = %s\n" "$Read1"
        printf "READ_2 = %s\n" "$Read2"
        printf "PHIX_CONCENTRATION = %s\n" "$PhiX"
        printf "RGT_Box_1 = %s\n" "$RGT_box1"
        printf "RGT_Box_2 = %s\n" "$RGT_box2"
    } > "$prop_file"

    dst="${datamoverDST}/${run_name}"
    rsync -a --rsync-path="mkdir -p $dst && rsync" "$prop_file" "ozagor@datamover:$dst"
    # save rsync exit status, if 0 then success
    rsync1=$?

    ### sample_type MISEQ_RUN
    project_to_write=$2

    prop_file=dataset.properties
    {
        printf "SPACE = IMV\n"
        printf "PROJECT = %s\n" "$project_to_write"
        printf "EXPERIMENT = MISEQ_RUNS\n"
        printf "SAMPLE = %s\n" "$run_name"
        printf "SAMPLE_TYPE = MISEQ_RUN\n"
        printf "DATASET_TYPE = DATAMOVER_SAMPLE_CREATOR\n"
    } > "$prop_file"

    rsync -a "$prop_file" "ozagor@datamover:$dst"
    rsync2=$?
    if [ "$rsync1" -eq "0" ] && [ "$rsync2" -eq "0" ]; then
        ssh ozagor@datamover touch "$datamoverDST/.MARKER_is_finished_${run_name}" </dev/null
    else
        echo -e "WATCH OUT: touching the void! $rsync1 $rsync2 $rsync3 $rsync4"
    fi

}

write_miseq_sample(){
    # input is a line in section [Data] of the samples sheet
    # copy fastq file into $bufferdir/$rundir/S.../ and write dataset.properties
    # if timavo == y, sync fastq file on timavo too

    declare -a sample_line=("${!1}")

    ### move fastq file into folder

    sample_number=${sample_line[0]}
    # this should mimic the behaviour of MiSeq Reporter on sample names with
    # spaces and punctuations:
    # - remove leadin and trailing whitespaces (first sed)
    # - replace punctuation characters and internal spaces with dashes
    sample_name=$(echo ${sample_line[1]} | sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//' | tr -s '[:punct:] ' '-')
    sample_plate=${sample_line[2]}
    sample_well=${sample_line[3]}
    I7_index_id=${sample_line[4]}
    index_1=${sample_line[5]}
    I5_index_id=${sample_line[6]}
    index_2=${sample_line[7]}
    description=${sample_line[9]}
    timavo=${sample_line[14]}

    run_name="$(basename $rundir)"

    fastq_file=$incomingdir/$run_name/Data/Intensities/BaseCalls/${sample_name}_S${sample_number}_L001_R1_001.fastq.gz
    fastq_file_2=$run_name/Data/Intensities/BaseCalls/${sample_name}_S${sample_number}_L001_R2_001.fastq.gz

    if [[ $timavo == *"y"* ]]; then
        echo "Syncing to TIMAVO"
        DST2="${timavoDST}/MiSeqOutput/${run_name}/Data/Intensities/BaseCalls/"
        rsync -a --rsync-path="mkdir -p $DST2 && rsync" "$fastq_file" "timavo:$DST2"
        if [ -e "$fastq_file_2" ]; then
            rsync "$fastq_file_2" "$DST2"
        fi
      else
        echo "Not syncing to TIMAVO"
    fi

    ### write properties file

    ### sample_type MISEQ_SAMPLE
    echo "Writing properties files for MISEQ_SAMPLE ID:${sample_number} NAME:${sample_name}"
    prop_file=sample.properties
    {
        printf "SAMPLE_ID=%s\n" "${sample_number}"
        printf "SAMPLE_NAME=%s\n" "${sample_name}"
        printf "SAMPLE_PLATE=%s\n" "${sample_plate}"
        printf "SAMPLE_WELL=%s\n" "$sample_well"
        printf "I7_INDEX_ID=%s\n" "$I7_index_id"
        printf "INDEX_1=%s\n" "$index_1"
        printf "I5_INDEX_ID=%s\n" "$I5_index_id"
        printf "INDEX_2=%s\n" "$index_2"
        printf "DESCRIPTION=%s\n" "$description"
    } > "$prop_file"

    dst="${datamoverDST}/${run_name}-${sample_number}"
    rsync -a --rsync-path="mkdir -p $dst && rsync" "$prop_file" "ozagor@datamover:$dst"
    # save rsync exit status, if 0 then success
    rsync1=$?

    Project=${sample_line[8]}

    prop_file=dataset.properties
    {
        printf "SPACE = IMV\n"
        printf "PROJECT = %s\n" "$Project"
        printf "EXPERIMENT = MISEQ_SAMPLES\n"
        printf "SAMPLE = %s-%s\n" "${run_name}" "${sample_number}"
        printf "SAMPLE_TYPE = MISEQ_SAMPLE\n"
        printf "DATASET_TYPE = FASTQ\n"
    } > "$prop_file"

    rsync -a "$prop_file" "ozagor@datamover:$dst"
    rsync2=$?

    rsync3=0
    if [ -e "$fastq_file" ]; then
        rsync -a "$fastq_file" "ozagor@datamover:$dst"
        rsync3=$?
        # echo "R1 file exists, rsync returns $rsync3"
    else
        echo "R1 file does not exist"
    fi

    rsync4=0
    if [ -e "$fastq_file_2" ]; then
        rsync -a "$fastq_file_2" "ozagor@datamover:$dst"
        rsync4=$?
        echo "R2 file exists, rsync returns $rsync4"
    # else
    #     echo "R2 file does not exist"
    fi

    if [ "$rsync1" -eq "0" ] && [ "$rsync2" -eq "0" ] && [ "$rsync3" -eq "0" ] && [ "$rsync4" -eq "0" ]; then
        ssh ozagor@datamover touch "$datamoverDST/.MARKER_is_finished_${run_name}-${sample_number}" </dev/null
    else
        echo -e "WATCH OUT: touching the void! $rsync1 $rsync2 $rsync3 $rsync4"
    fi

}

write_resistance_test(){

    declare -a sample_line=("${!1}")

    sample_number=${sample_line[0]}
    sample_name=${sample_line[1]}
    #sample_project=${sample_line[8]}
    #Description=${sample_line[9]}
    virus=${sample_line[10]}
    genotype=${sample_line[11]}
    target=${sample_line[12]}
    viral_load=${sample_line[13]}

    run_name=$(basename "$rundir")

    echo "Writing properties files for RESISTANCE_TEST RUN:${run_name} SAMPLE:${sample_name}"

    ### write properties file
    prop_file=sample.properties
    {
        printf "SAMPLE_NAME=%s\n" "$sample_name"
        printf "VIRUS=%s\n" "$virus"
        printf "TARGET_REGION=%s\n" "$target"
        printf "GENOTYPE=%s\n" "$genotype"
        printf "VIRAL_LOAD=%s\n" "$viral_load"
    } > "$prop_file"

    dst="${datamoverDST}/${run_name}-${sample_number}_RESISTANCE"
    rsync -a --rsync-path="mkdir -p $dst && rsync" "$prop_file" "ozagor@datamover:$dst"
    # save rsync exit status, if 0 then success
    rsync1=$?

    prop_file=dataset.properties
    {
        printf "SPACE = IMV\n"
        printf "PROJECT = RESISTANCE\n"
        printf "EXPERIMENT = RESISTANCE_TESTS\n"
        printf "SAMPLE = %s-%s_RESISTANCE\n" "${run_name}" "${sample_number}"
        printf "SAMPLE_TYPE = RESISTANCE_TEST\n"
        printf "DATASET_TYPE = DATAMOVER_SAMPLE_CREATOR\n"
    } > "$prop_file"

    rsync -a "$prop_file" "ozagor@datamover:$dst"
    rsync2=$?

    if [ "$rsync1" -eq "0" ] && [ "$rsync2" -eq "0" ]; then
        ssh ozagor@datamover touch "$datamoverDST/.MARKER_is_finished_${run_name}-${sample_number}_RESISTANCE" </dev/null
    else
        echo -e "WATCH OUT: touching the void! $rsync1 $rsync2 $rsync3 $rsync4"
    fi

}

process_runs(){
    # read the samples sheet,
    # save info found in [Header] and [Reads] in global variables
    # call process_sample on all lines in [Data] section

    rundir=$1
    if [[ -e $rundir/.UPLOADED_RUN ]]; then
        echo "Run $rundir already uploaded"
        return
    fi

    # reset headers
    headers='undefined'

    ### remove spaces in sample sheet
    #sed -e "s/ /_/g" < "$rundir/Data/Intensities/BaseCalls/SampleSheet.csv" > sample_sheet.tmp
    # sample sheets are now created with Stefan's template: no need to remove them anymore
    cp "$rundir/Data/Intensities/BaseCalls/SampleSheet.csv" sample_sheet.tmp
    count_openbis=$(grep -c openbis sample_sheet.tmp)
    ### counters for read 1/2 and samples
    r=0
    s=0
    res_sample=false
    meta_sample=false
    plasm_sample=false
    anti_sample=false
    other_sample=false

    ### read sample sheet line by line splitting on commas
    ### || [[ -n "$line"]] allow reading last line also if file does not end
    ### with a newline
    while IFS=',' read -r -a line || [[ -n "$line" ]]
    do
        # if [[ "$count_openbis" == 0 ]]
        # then
        #     break
        # fi
        if [[ ${line[0]} =~ ^\[[[:alpha:]]*\] ]]
        then
            section=${line[0]}

        ### [Header] section
        elif [[ $section == "[Header]" && ${line[1]} ]]
        then
            declare "${line[0]}=${line[1]}"

        ### [Reads] section for read 1
        elif [[ $section == "[Reads]" && ${line[0]} =~ ^[0-9]+$ && $r -eq 0 ]]
        then
            # [Header] has now been parsed, if openbis != y then stop parsing
            # this run and go to next
            if [[ "$openbis" == "n" ]]; then
                break
            fi
            r=1
            # echo $section ${line[0]}
            Read1=${line[0]}

        ### [Reads] section for read 2
        elif [[ $section == "[Reads]" && ${line[0]} =~ ^[0-9]+$ && $r -eq 1 ]]
        then
            r=2
            # echo $section ${line[0]}
            Read2=${line[0]}

        ### [Data] section headers
        elif [[ $section == "[Data]" && ${line[1]} && $s -eq 0 ]]
        then
            headers=( "${line[@]}" )
            echo -e "  Headers found:" "${headers[@]}"
            ((s+=1))

        ### [Data] section values
        elif [[ $section == "[Data]" && ${line[1]} && $s -gt 0 ]]
        then

            write_miseq_sample line[@]

            case ${line[8]} in
              Antibodies)
                anti_sample=true
                ;;
              Metagenomics)
                meta_sample=true
                ;;
              Other)
                other_sample=true
                ;;
              Plasmids)
                plasm_sample=true
                ;;
              Resistance)
                res_sample=true
                write_resistance_test line[@]
                ;;
              esac
            ((s+=1))
        fi

    done < sample_sheet.tmp

    echo "Syncing SampleSheet to timavo"
    run_name=$(basename "$rundir")
    smpshdst="${timavoDST}/MiSeqOutput/${run_name}/Data/Intensities/BaseCalls/"
    rsync -av --rsync-path="mkdir -p $smpshdst && rsync" sample_sheet.tmp "timavo:$smpshdst/SampleSheet.csv"
    rm sample_sheet.tmp

    # if any sample was X then write Miseq run sample with PROJECT = X
    if [ "$anti_sample" = true ]; then
        echo "WRITING ANTIBODIES RUN"
        write_miseq_run "${run_name}_ANTIBODIES" Antibodies
    fi
    if [ "$meta_sample" = true ]; then
        echo "WRITING METAGENOMICS RUN"
        write_miseq_run "${run_name}_METAGENOMICS" Metagenomics
    fi
    if [ "$other_sample" = true ]; then
        echo "WRITING OTHER RUN"
        write_miseq_run "${run_name}_OTHER" Other
    fi
    if [ "$plasm_sample" = true ]; then
        echo "WRITING PLASMIDS RUN"
        write_miseq_run "${run_name}_PLASMIDS" Plasmids
    fi
    if [ "$res_sample" = true ]; then
        echo "WRITING RESISTANCE RUN"
        write_miseq_run "${run_name}_RESISTANCE" Resistance
    fi

    # get sample sheet name from runParameter.xml file and save runParameter.xml file with the sample sheet name
    SAMPLESHEET=$(grep -A 1 ReagentKitRFIDTag "$rundir/runParameters.xml" | grep SerialNumber | sed 's/^.*<SerialNumber>//' | sed 's/<\/SerialNumber>//')
    rsync -av --stats --chmod=ug+rwx -p "$rundir/runParameters.xml" "timavo:$timavoDST/MiSeqRunParameters/${SAMPLESHEET}.xml"

    # reset [Header] and [Reads] information
    Investigator_Name='undefined'
    Experiment_Name='undefined'
#    Date='undefined'
    Workflow='undefined'
    Application='undefined'
    Assay='undefined'
#    Description='undefined'
    Chemistry='undefined'
    Read1=''
    Read2=''
    PhiX='undefined'
    RGT_box1='undefined'
    RGT_box2='undefined'
    openbis='n'

    touch "$rundir/.UPLOADED_RUN"

}

echo 'GO!'
### main loop over all dirs in $incomingdir starting with "1"
#for rundir in $(find $incomingdir -type d -name "1*" -depth 1)
#for rundir in $(ls $incomingdir)
for rundir in "$incomingdir"/1*; do
    [[ -d "$rundir" ]] || continue
    echo -e "\033[1;31m================================================================================\033[0m"
    echo "Now syncing:" "$rundir"
    process_runs "$rundir"
done

# copy SampleSheets
rsync -av --stats --chmod=ug+rwx -p /cygdrive/d/Illumina/MiSeqSampleSheets "timavo:$timavoDST"


Create firewall.sh

Create firewall.sh

#!/bin/sh
#------------------------------------------------------------------------------
#
# Firewall script
# Compiler: Cal Harding <cal@calharding.net>
#                       <cal@dcdata.co.za>
# License ISC License
# URL: http://www.calharding.net
# Sources: IRC, man pages, HOWTOs, Linux iptables Reference by Greg Purdy
#
# Copyright (c) 2011, Cal Harding <cal@calharding.net>
#
# Permission to use, copy, modify, and/or distribute this software for any
# purpose with or without fee is hereby granted, provided that the above
# copyright notice and this permission notice appear in all copies.
#
# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
#
#------------------------------------------------------------------------------

IPT=/sbin/iptables
EIF="eth0"
EIP="203.0.113.77"
VIF="tun0"

# Logging options
LOG="LOG --log-level info --log-tcp-sequence"
LOG="$LOG --log-tcp-options --log-ip-options"

# Rate limiting
RLIMIT="-m limit --limit 3/s --limit-burst 8"

#/sbin/modprobe ip_nat_ftp
#/sbin/modprobe ip_conntrack_ftp

# Cleanup
#-----------------------------------------------------------------------------

# Flush firewall
$IPT -F
$IPT -t nat -F
$IPT -t mangle -F

# Delete all non-default chains
$IPT -X
$IPT -t nat -X
$IPT -t mangle -X

# zero out packet counters
$IPT -Z
$IPT -t nat -Z
$IPT -t mangle -Z

# Set up default policies
#----------------------------------------------------------------------------

$IPT -P INPUT ACCEPT
$IPT -P FORWARD ACCEPT
$IPT -P OUTPUT ACCEPT

# Log and accept
$IPT -N AC_LOG
$IPT -A AC_LOG -j $LOG $RLIMIT --log-prefix "ACCEPT: "
$IPT -A AC_LOG -j ACCEPT

# Log and drop
$IPT -N DR_LOG
$IPT -A DR_LOG -j $LOG $RLIMIT --log-prefix "DROP: "
$IPT -A DR_LOG -j DROP

# Log and reject (TCP packets are rejected with a TCP reset)
$IPT -N RE_LOG
$IPT -A RE_LOG -j $LOG $RLIMIT --log-prefix "REJECT: "
$IPT -A RE_LOG -p tcp -j REJECT --reject-with tcp-reset
$IPT -A RE_LOG -j REJECT

# Kernel configuration.
#------------------------------------------------------------------------------

# Disable IP forwarding.
# On => Off = (reset)
#echo 1 > /proc/sys/net/ipv4/ip_forward
#echo 0 > /proc/sys/net/ipv4/ip_forward

# Enable IP spoofing protection
for i in /proc/sys/net/ipv4/conf/*/rp_filter; do echo 1 > $i; done

# Protect against SYN flood attacks
echo 1 > /proc/sys/net/ipv4/tcp_syncookies

# Ignore all incoming ICMP echo requests
echo 0 > /proc/sys/net/ipv4/icmp_echo_ignore_all

# Ignore ICMP echo requests to broadcast
echo 1 > /proc/sys/net/ipv4/icmp_echo_ignore_broadcasts

# Log packets with impossible addresses.
for i in /proc/sys/net/ipv4/conf/*/log_martians; do echo 1 > $i; done

# Don't log invalid responses to broadcast
echo 1 > /proc/sys/net/ipv4/icmp_ignore_bogus_error_responses

# Don't accept or send ICMP redirects.
for i in /proc/sys/net/ipv4/conf/*/accept_redirects; do echo 0 > $i; done
for i in /proc/sys/net/ipv4/conf/*/send_redirects; do echo 0 > $i; done

# Don't accept source routed packets.
for i in /proc/sys/net/ipv4/conf/*/accept_source_route; do echo 0 > $i; done

# Disable multicast routing
#for i in /proc/sys/net/ipv4/conf/*/mc_forwarding; do echo 0 > $i; done

# Disable proxy_arp.
for i in /proc/sys/net/ipv4/conf/*/proxy_arp; do echo 0 > $i; done

# Enable secure redirects, i.e. only accept ICMP redirects for gateways
# Helps against MITM attacks.
for i in /proc/sys/net/ipv4/conf/*/secure_redirects; do echo 1 > $i; done

# Disable bootp_relay
for i in /proc/sys/net/ipv4/conf/*/bootp_relay; do echo 0 > $i; done

# Allow loopback to do anything.
#------------------------------------------------------------------------------

$IPT -A INPUT -i lo -j ACCEPT
$IPT -A OUTPUT -o lo -j ACCEPT

# Syn flood protection
#------------------------------------------------------------------------------

$IPT -N SYN_FLOOD
$IPT -A INPUT -p tcp --syn -j SYN_FLOOD
$IPT -A SYN_FLOOD $RLIMIT -j RETURN
$IPT -A SYN_FLOOD -j DROP

# Test for invalid TCP flags (portscanning)
# -----------------------------------------------------------------------------

$IPT -N BAD_FLAGS
$IPT -A BAD_FLAGS -j LOG --log-level debug --log-prefix "IPT BAD_FLAGS: "
$IPT -A BAD_FLAGS -j DROP

$IPT -N TCP_FLAGS
$IPT -A TCP_FLAGS -p tcp --tcp-flags ACK,FIN FIN -j BAD_FLAGS
$IPT -A TCP_FLAGS -p tcp --tcp-flags ACK,PSH PSH -j BAD_FLAGS
$IPT -A TCP_FLAGS -p tcp --tcp-flags ACK,URG URG -j BAD_FLAGS
$IPT -A TCP_FLAGS -p tcp --tcp-flags FIN,RST FIN,RST -j BAD_FLAGS
$IPT -A TCP_FLAGS -p tcp --tcp-flags SYN,FIN SYN,FIN -j BAD_FLAGS
$IPT -A TCP_FLAGS -p tcp --tcp-flags SYN,RST SYN,RST -j BAD_FLAGS
$IPT -A TCP_FLAGS -p tcp --tcp-flags ALL ALL -j BAD_FLAGS
$IPT -A TCP_FLAGS -p tcp --tcp-flags ALL NONE -j BAD_FLAGS
$IPT -A TCP_FLAGS -p tcp --tcp-flags ALL FIN,PSH,URG -j BAD_FLAGS
$IPT -A TCP_FLAGS -p tcp --tcp-flags ALL SYN,FIN,PSH,URG -j BAD_FLAGS
$IPT -A TCP_FLAGS -p tcp --tcp-flags ALL SYN,RST,ACK,FIN,URG -j BAD_FLAGS

#########################
# FORWARDING
#########################

# for openvpn
$IPT -A FORWARD -p tcp -i $VIF -s 10.17.165.0/24 -j ACCEPT

$IPT -A FORWARD -o $VIF -d 10.17.165.0/24 -m state --state ESTABLISHED,RELATED -j ACCEPT
$IPT -A FORWARD -j RE_LOG

##########################
# INPUT
##########################

# Drop any traffic from IANA-reserved IPs.
#------------------------------------------------------------------------------

$IPT -A INPUT -s 0.0.0.0/7 -j DR_LOG
#$IPT -A INPUT -s 2.0.0.0/8 -j DR_LOG
#$IPT -A INPUT -s 5.0.0.0/8 -j DR_LOG
#$IPT -A INPUT -s 7.0.0.0/8 -j DR_LOG
$IPT -A INPUT -s 10.0.0.0/8 -j DR_LOG
#$IPT -A INPUT -s 23.0.0.0/8 -j DR_LOG
#$IPT -A INPUT -s 27.0.0.0/8 -j DR_LOG
#$IPT -A INPUT -s 31.0.0.0/8 -j DR_LOG
#$IPT -A INPUT -s 36.0.0.0/7 -j DR_LOG
#$IPT -A INPUT -s 39.0.0.0/8 -j DR_LOG
#$IPT -A INPUT -s 42.0.0.0/8 -j DR_LOG
#$IPT -A INPUT -s 49.0.0.0/8 -j DR_LOG
#$IPT -A INPUT -s 50.0.0.0/8 -j DR_LOG
#$IPT -A INPUT -s 77.0.0.0/8 -j DR_LOG
#$IPT -A INPUT -s 78.0.0.0/7 -j DR_LOG
#$IPT -A INPUT -s 92.0.0.0/6 -j DR_LOG
#$IPT -A INPUT -s 96.0.0.0/4 -j DR_LOG # blocks ssh for some reason
#$IPT -A INPUT -s 112.0.0.0/5 -j DR_LOG
#$IPT -A INPUT -s 120.0.0.0/8 -j DR_LOG
$IPT -A INPUT -s 169.254.0.0/16 -j DR_LOG
$IPT -A INPUT -s 172.16.0.0/12 -j DR_LOG
#$IPT -A INPUT -s 173.0.0.0/8 -j DR_LOG
#$IPT -A INPUT -s 174.0.0.0/7 -j DR_LOG
#$IPT -A INPUT -s 176.0.0.0/5 -j DR_LOG
#$IPT -A INPUT -s 184.0.0.0/6 -j DR_LOG
$IPT -A INPUT -s 192.0.2.0/24 -j DR_LOG
$IPT -A INPUT -s 192.88.99.0/24 -j DR_LOG
#$IPT -A INPUT -s 197.0.0.0/8 -j DR_LOG
$IPT -A INPUT -s 198.18.0.0/15 -j DR_LOG
$IPT -A INPUT -s 198.51.100.0/24 -j DR_LOG
$IPT -A INPUT -s 203.0.113.0/24 -j DR_LOG
#$IPT -A INPUT -s 223.0.0.0/8 -j DR_LOG
$IPT -A INPUT -s 224.0.0.0/4 -j DR_LOG
$IPT -A INPUT -s 240.0.0.0/4 -j DR_LOG

# Drop all fragmented ICMP packets (usually malicious)
$IPT -A INPUT -p icmp --fragment -j DR_LOG
$IPT -A OUTPUT -p icmp --fragment -j DR_LOG
$IPT -A FORWARD -p icmp --fragment -j DR_LOG

# Make it hard to multi-ping
$IPT -A INPUT -p icmp -m limit --limit 1/s --limit-burst 2 -j ACCEPT
$IPT -A INPUT -p icmp -m limit --limit 1/s --limit-burst 2 -j LOG --log-prefix "PING-DROP: "
$IPT -A INPUT -p icmp -j DROP

#OpenVPN
$IPT -A INPUT -i $EIF -p tcp --dport 11194 -j ACCEPT

# SSH
# if > 4 packets in 60 seconds from the same ip addr, drop subsequent
$IPT -A INPUT -i $EIF -p tcp --dport 22 -m state --state NEW -m recent --set
$IPT -A INPUT -i $EIF -p tcp --dport 22 -m state --state NEW -m recent --update --seconds 60 --hitcount 4 -j DR_LOG

$IPT -A INPUT -i $EIF -p tcp --dport 22 -j ACCEPT

# Drop invalid packets
$IPT -A INPUT -m state --state INVALID -j DROP
$IPT -A OUTPUT -m state --state INVALID -j DROP

$IPT -A INPUT -i $EIF -m state --state ESTABLISHED,RELATED -j ACCEPT
$IPT -A INPUT -j RE_LOG

error: pathspec 'firewall.sh' did not match any file(s) known to git


Helper scripts for generating a list of panoramas

#!/bin/bash

#
# Sample script for creating a link-list of panoramas generated by wp.sh
#

ROOT=wp_pages
HEADLINE=""
if [ -f wp_conf.sh ]; then
    source wp_conf.sh
fi

echo "
"
echo "$HEADLINE"
echo "
"
pushd "$ROOT" > /dev/null
for META in `ls */meta.dat | sort`; do
    DEST=`cat "$META" | grep dest | cut -d= -f2`
    DESCRIPTION=`cat "$META" | grep description | cut -d= -f2`
    if [ "." == ".$DEST" ]; then
        continue
    fi
    if [ "." == ".$DESCRIPTION" ]; then
        DESCRIPTION="$DEST"
    fi

    echo "
$DESCRIPTION
"
done
popd > /dev/null
echo "
"
echo "
"

error: pathspec 'wp_list.sh' did not match any file(s) known to git


Add wrapper script

Add wrapper script

#!/bin/bash

if [ ! -d ytdl ]; then
	virtualenv ytdl
	source ytdl/bin/activate
	pip install youtube-dl
	deactivate
fi

source ytdl/bin/activate

while true; do
	./offline
	sleep 2h
done

error: pathspec 'wrapper.sh' did not match any file(s) known to git


Deploy crashlytics beta script.

Deploy crashlytics beta script.

[skip ci]

#!/bin/bash

RELEASE_NOTES=$(git log --format="%cn @ "$TRAVIS_BRANCH"
------
%B" -n 1 $TRAVIS_COMMIT)

echo "$RELEASE_NOTES" > crashlytics_release_notes.txt

./gradlew crashlyticsUploadDistributionDebug

error: pathspec 'deploy_crashlytics_beta.sh' did not match any file(s) known to git


Conda build script.

Conda build script.

#!/bin/bash

mkdir tmpconda
cd tmpconda
for pkg in latexcodec tabulate monty pybtex palettable spglib pydispatcher pymatgen
do
    conda skeleton pypi $pkg
    conda build $pkg
    anaconda upload /Users/shyuep/miniconda3/conda-bld/osx-64/$pkg-*py35*.tar.bz2
done
cd ..
rm -r tmpconda

mkdir tmpconda
cd tmpconda
for pkg in latexcodec tabulate monty pybtex palettable spglib pydispatcher pymatgen
do
    conda skeleton pypi --python-version 2.7 $pkg
    conda build --python 2.7 $pkg
    anaconda upload --force /Users/shyuep/miniconda3/conda-bld/osx-64/$pkg-*py27*.tar.bz2
done
cd ..
rm -r tmpconda

error: pathspec 'dev_scripts/build_conda.sh' did not match any file(s) known to git


Added common Drupal modules

Added common Drupal modules

remote: Support for password authentication was removed on August 13, 2021.
remote: Please see https://docs.github.com/en/get-started/getting-started-with-git/about-remote-repositories#cloning-with-https-urls for information on currently recommended modes of authentication.
fatal: Authentication failed for 'https://github.com/PotatoSkin15/linux-scripts.git/'


'/etc/ceph/.mon.keyring' is repeated.

'/etc/ceph/.mon.keyring' is repeated.

#!/bin/bash
set -e

function get_admin_key {
   # No-op for static
   echo "static: does not generate admin key"
}

function get_mon_config {
  if [ ! -e /etc/ceph/${CLUSTER}.conf ]; then
    fsid=$(uuidgen)
    cat <<ENDHERE >/etc/ceph/${CLUSTER}.conf
[global]
fsid = $fsid
mon initial members = ${MON_NAME}
mon host = ${MON_IP}
auth cluster required = cephx
auth service required = cephx
auth client required = cephx
public network = ${CEPH_PUBLIC_NETWORK}
cluster network = ${CEPH_CLUSTER_NETWORK}
osd journal size = ${OSD_JOURNAL_SIZE}
ENDHERE

    if [[ ! -z "$(ip -6 -o a | grep scope.global | awk '/eth/ { sub ("/..", "", $4); print $4 }' | head -n1)" ]]; then
      echo "ms_bind_ipv6 = true" >> /etc/ceph/${CLUSTER}.conf
      sed -i '/mon host/d' /etc/ceph/${CLUSTER}.conf
      echo "mon host = ${MON_IP}" >> /etc/ceph/${CLUSTER}.conf
    fi

    # Generate administrator key
    ceph-authtool /etc/ceph/${CLUSTER}.client.admin.keyring --create-keyring --gen-key -n client.admin --set-uid=0 --cap mon 'allow *' --cap osd 'allow *' --cap mds 'allow'

    # Generate the mon. key
    ceph-authtool /etc/ceph/${CLUSTER}.mon.keyring --create-keyring --gen-key -n mon. --cap mon 'allow *'

    # Create bootstrap key directories
    mkdir -p /var/lib/ceph/bootstrap-{osd,mds,rgw}
    chown ceph. /var/lib/ceph/bootstrap-{osd,mds,rgw}

    # Generate the OSD bootstrap key
    ceph-authtool /var/lib/ceph/bootstrap-osd/${CLUSTER}.keyring --create-keyring --gen-key -n client.bootstrap-osd --cap mon 'allow profile bootstrap-osd'

    # Generate the MDS bootstrap key
    ceph-authtool /var/lib/ceph/bootstrap-mds/${CLUSTER}.keyring --create-keyring --gen-key -n client.bootstrap-mds --cap mon 'allow profile bootstrap-mds'

    # Generate the RGW bootstrap key
    ceph-authtool /var/lib/ceph/bootstrap-rgw/${CLUSTER}.keyring --create-keyring --gen-key -n client.bootstrap-rgw --cap mon 'allow profile bootstrap-rgw'

    # Apply proper permissions to the keys
    chown ceph. /etc/ceph/${CLUSTER}.mon.keyring /var/lib/ceph/bootstrap-osd/${CLUSTER}.keyring /var/lib/ceph/bootstrap-mds/${CLUSTER}.keyring /var/lib/ceph/bootstrap-rgw/${CLUSTER}.keyring

    # Generate initial monitor map
    monmaptool --create --add ${MON_NAME} "${MON_IP}:6789" --fsid ${fsid} /etc/ceph/monmap
    chown ceph. /etc/ceph/monmap
  fi
}

function get_config {
   # No-op for static
   echo "static: does not generate config"
}

#!/bin/bash
set -e

function get_admin_key {
   # No-op for static
   echo "static: does not generate admin key"
}

function get_mon_config {
  if [ ! -e /etc/ceph/${CLUSTER}.conf ]; then
    fsid=$(uuidgen)
    cat <<ENDHERE >/etc/ceph/${CLUSTER}.conf
[global]
fsid = $fsid
mon initial members = ${MON_NAME}
mon host = ${MON_IP}
auth cluster required = cephx
auth service required = cephx
auth client required = cephx
public network = ${CEPH_PUBLIC_NETWORK}
cluster network = ${CEPH_CLUSTER_NETWORK}
osd journal size = ${OSD_JOURNAL_SIZE}
ENDHERE

    if [[ ! -z "$(ip -6 -o a | grep scope.global | awk '/eth/ { sub ("/..", "", $4); print $4 }' | head -n1)" ]]; then
      echo "ms_bind_ipv6 = true" >> /etc/ceph/${CLUSTER}.conf
      sed -i '/mon host/d' /etc/ceph/${CLUSTER}.conf
      echo "mon host = ${MON_IP}" >> /etc/ceph/${CLUSTER}.conf
    fi

    # Generate administrator key
    ceph-authtool /etc/ceph/${CLUSTER}.client.admin.keyring --create-keyring --gen-key -n client.admin --set-uid=0 --cap mon 'allow *' --cap osd 'allow *' --cap mds 'allow'

    # Generate the mon. key
    ceph-authtool /etc/ceph/${CLUSTER}.mon.keyring --create-keyring --gen-key -n mon. --cap mon 'allow *'

    # Create bootstrap key directories
    mkdir -p /var/lib/ceph/bootstrap-{osd,mds,rgw}
    chown ceph. /var/lib/ceph/bootstrap-{osd,mds,rgw}

    # Generate the OSD bootstrap key
    ceph-authtool /var/lib/ceph/bootstrap-osd/${CLUSTER}.keyring --create-keyring --gen-key -n client.bootstrap-osd --cap mon 'allow profile bootstrap-osd'

    # Generate the MDS bootstrap key
    ceph-authtool /var/lib/ceph/bootstrap-mds/${CLUSTER}.keyring --create-keyring --gen-key -n client.bootstrap-mds --cap mon 'allow profile bootstrap-mds'

    # Generate the RGW bootstrap key
    ceph-authtool /var/lib/ceph/bootstrap-rgw/${CLUSTER}.keyring --create-keyring --gen-key -n client.bootstrap-rgw --cap mon 'allow profile bootstrap-rgw'

    # Apply proper permissions to the keys
    chown ceph. /etc/ceph/${CLUSTER}.mon.keyring /etc/ceph/${CLUSTER}.mon.keyring /var/lib/ceph/bootstrap-mds/${CLUSTER}.keyring /var/lib/ceph/bootstrap-rgw/${CLUSTER}.keyring

    # Generate initial monitor map
    monmaptool --create --add ${MON_NAME} "${MON_IP}:6789" --fsid ${fsid} /etc/ceph/monmap
    chown ceph. /etc/ceph/monmap
  fi
}

function get_config {
   # No-op for static
   echo "static: does not generate config"
}


Add script for running ios Tests on simulator (#10999)

#!/bin/sh

set -e

FLUTTER_ENGINE=ios_debug_sim_unopt

if [ $# -eq 1 ]; then
  FLUTTER_ENGINE=$1
fi

PRETTY="cat"
if which xcpretty; then
  PRETTY="xcpretty"
fi

cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd

./compile_ios_jit.sh ../../../out/host_debug_unopt ../../../out/$FLUTTER_ENGINE/clang_x64

pushd ios/Scenarios

set -o pipefail && xcodebuild -sdk iphonesimulator \
  -scheme ScenariosTests \
  -destination 'platform=iOS Simulator,name=iPhone SE,OS=12.2' \
  test \
  FLUTTER_ENGINE=$FLUTTER_ENGINE | $PRETTY

popd

error: pathspec 'testing/scenario_app/run_ios_tests.sh' did not match any file(s) known to git


going to SEBTS_25'

going to SEBTS_25'

remote: Support for password authentication was removed on August 13, 2021.
remote: Please see https://docs.github.com/en/get-started/getting-started-with-git/about-remote-repositories#cloning-with-https-urls for information on currently recommended modes of authentication.
fatal: Authentication failed for 'https://github.com/sebts/util.git/'


Added script to update kissmetrics data.

#!/bin/bash
s3cmd get --recursive --skip-existing s3://appearin-kissmetrics kissmetrics
s3cmd sync s3://appearin-kissmetrics/* kissmetrics

error: pathspec 'data/update.sh' did not match any file(s) known to git


use latest version of sonobuoy for conformance tests

remote: Support for password authentication was removed on August 13, 2021.
remote: Please see https://docs.github.com/en/get-started/getting-started-with-git/about-remote-repositories#cloning-with-https-urls for information on currently recommended modes of authentication.
fatal: Authentication failed for 'https://github.com/warmchang/minikube.git/'


Add script for setting tested version of plugin in scripted tests.

#!/bin/bash

[ "$#" -gt "0" ] || {
  echo "Usage: $0 <version>"; exit 1
}

find -name Plugins.scala | xargs sed -i "s/\(^.*val testedVersion.*\"\)\(.*\)\(\".*$\)/\1$1\3/g"

error: pathspec 'sbt-idea-tests/set_tested_version.sh' did not match any file(s) known to git


Adding project name and group that works

PROJECT_GROUP=org/adligo
PROJECT_NAME=tests4j

export PROJECT_GROUP PROJECT_NAME

error: pathspec 'artProject.sh' did not match any file(s) known to git


Add model test shell script

Add model test shell script

#!/bin/bash
INPUTJPG=$1"*.jpg"
INPUTPNG=$1"*.png"
OUTPUT=$2
MODEL=$3
mkdir -p $2

for f in $INPUTJPG
do
  echo "Processing $f file..."
	python single_image_test.py -g 0 --load_gen_model $3 -i $f -o $2"/"$(basename $f) -r 2
done

for f in $INPUTPNG
do
  echo "Processing $f file..."
	python single_image_test.py -g 0 --load_gen_model $3 -i $f -o $2"/"$(basename $f) -r 2
done

error: pathspec 'scripts/folder_single_images_test.sh' did not match any file(s) known to git


-- resolved WordnetTools dependency, set to master-SNAPSHOT which is the latest version always

#!/usr/bin/env bash
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
ROOT="$( cd $DIR && cd .. && pwd)"
LIB="$ROOT"/lib

RESOURCES="$( cd $ROOT && cd .. && pwd)"/vua-resources
# assumes vua-resources is installed next to this installation
# git clone https://github.com/cltl/vua-resources.git
#
# the software assume that the input files are in NAF format minially with the following layers:
# - tokens, terms, entities, coreference for events, srl, timeExpressions
# to create an event coreference layer, use the event-coreference scripts

#DUTCH
ontology ="$RESOURCES"/dbpedia_nl_types.tsv.gz
#ENGLISH
ontology ="$RESOURCES"/instance_types_en.ttl.gz
java -Xmx2000m -cp "$LIB/EventCoreference-v3.1.2-jar-with-dependencies.jar" eu.newsreader.eventcoreference.naf.GetSimpleSemFromNafFolder --naf-folder $1 --extension ".naf" --ontology $ontology

error: pathspec 'scripts/naf-folder-to-simple-trig.sh' did not match any file(s) known to git


RPPA breast data import

RPPA breast data import

# Import RPPA data
./importProteinArrayData.pl $CGDS_DATA_HOME/breast-rppa/BR410_212Ab_RPPA_Median_Centered.txt BRCA_portal

error: pathspec 'portal/scripts/breast-rppa.sh' did not match any file(s) known to git


rm negative check

rm negative check

remote: Support for password authentication was removed on August 13, 2021.
remote: Please see https://docs.github.com/en/get-started/getting-started-with-git/about-remote-repositories#cloning-with-https-urls for information on currently recommended modes of authentication.
fatal: Authentication failed for 'https://github.com/mko-x/docker-owncloud.git/'


Fix libtensorflow builds due to typo.

Fix libtensorflow builds due to typo.

PiperOrigin-RevId: 363584508
Change-Id: I577541a2d246accaa833720f2664591f38b11de6

#!/usr/bin/env bash
# Copyright 2016 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
#
# Script to generate tarballs:
# (1) The TensorFlow C-library: Containing C API header files and libtensorflow.so
# (2) Native library for the TensorFlow Java API: Containing libtensorflow_jni.so
# And jars:
# (3) Java API .jar
# (4) Java API sources .jar
#
# These binary distributions will allow use of TensorFlow in various languages
# without having to compile the TensorFlow framework from sources, which takes
# a while and also introduces many other dependencies.
#
# Usage:
# - Source this file in another bash script
# - Execute build_libtensorflow_tarball SUFFIX
#
# Produces:
# - lib_package/libtensorflow${SUFFIX}.tar.gz
# - lib_package/libtensorflow_jni${SUFFIX}.tar.gz
# - lib_package/libtensorflow.jar
# - lib_package/libtensorflow-src.jar
# - lib_package/libtensorflow_proto.zip
#
# ASSUMPTIONS:
# - build_libtensorflow_tarball is invoked from the root of the git tree.
# - Any environment variables needed by the "configure" script have been set.

function build_libtensorflow_tarball() {
  # Sanity check that this is being run from the root of the git repository.
  if [ ! -e "WORKSPACE" ]; then
    echo "Must run this from the root of the bazel workspace"
    exit 1
  fi
  # Delete any leftovers from previous builds in this workspace.
  DIR=lib_package
  rm -rf ${DIR}

  TARBALL_SUFFIX="${1}"
  if [ "$(uname)" == "Darwin" ]; then
    BAZEL_OPTS="${BAZEL_OPTS} --config=release_cpu_macos"
  elif [ "${TF_NEED_CUDA}" == "1" ]; then
    BAZEL_OPTS="${BAZEL_OPTS} --config=release_gpu_linux"
  else
    BAZEL_OPTS="${BAZEL_OPTS} --config=release_cpu_linux"
  fi
  bazel clean --expunge
  export PYTHON_BIN_PATH=$(which python3.8)
  yes "" | "$PYTHON_BIN_PATH" configure.py

  # Remove this test call when
  # https://github.com/bazelbuild/bazel/issues/2352
  # and https://github.com/bazelbuild/bazel/issues/1580
  # have been resolved and the "manual" tags on the BUILD targets
  # in tensorflow/tools/lib_package/BUILD are removed.
  # Till then, must manually run the test since these tests are
  # not covered by the continuous integration.
  bazel test ${BAZEL_OPTS} --test_output=errors \
    //tensorflow/tools/lib_package:libtensorflow_test \
    //tensorflow/tools/lib_package:libtensorflow_java_test

  bazel build ${BAZEL_OPTS} \
    //tensorflow/tools/lib_package:libtensorflow.tar.gz \
    //tensorflow/tools/lib_package:libtensorflow_jni.tar.gz \
    //tensorflow/java:libtensorflow.jar \
    //tensorflow/java:libtensorflow-src.jar \
    //tensorflow/tools/lib_package:libtensorflow_proto.zip

  mkdir -p ${DIR}

  cp bazel-bin/tensorflow/tools/lib_package/libtensorflow.tar.gz ${DIR}/libtensorflow${TARBALL_SUFFIX}.tar.gz
  cp bazel-bin/tensorflow/tools/lib_package/libtensorflow_jni.tar.gz ${DIR}/libtensorflow_jni${TARBALL_SUFFIX}.tar.gz
  cp bazel-bin/tensorflow/java/libtensorflow.jar ${DIR}
  cp_normalized_srcjar bazel-bin/tensorflow/java/libtensorflow-src.jar ${DIR}/libtensorflow-src.jar
  cp bazel-bin/tensorflow/tools/lib_package/libtensorflow_proto.zip ${DIR}
  chmod -x ${DIR}/*
}

# Helper function to copy a srcjar after moving any source files
# directly under the root to the "maven-style" src/main/java layout
#
# Source files generated by annotation processors appear directly
# under the root of srcjars jars created by bazel, rather than under
# the maven-style src/main/java subdirectory.
#
# Bazel manages annotation generated source as follows: First, it
# calls javac with options that create generated files under a
# bazel-out directory. Next, it archives the generated source files
# into a srcjar directly under the root. There doesn't appear to be a
# simple way to parameterize this from bazel, hence this helper to
# "normalize" the srcjar layout.
#
# Arguments:
#   src_jar - path to the original srcjar
#   dest_jar - path to the destination
# Returns:
#   None
function cp_normalized_srcjar() {
  local src_jar="$1"
  local dest_jar="$2"
  if [[ -z "${src_jar}" || -z "${dest_jar}" ]]; then
    echo "Unexpected: missing arguments" >&2
    exit 2
  fi
  local tmp_dir
  tmp_dir=$(mktemp -d)
  cp "${src_jar}" "${tmp_dir}/orig.jar"
  pushd "${tmp_dir}"
  # Extract any src/ files
  jar -xf "${tmp_dir}/orig.jar" src/
  # Extract any org/ files under src/main/java
  (mkdir -p src/main/java && cd src/main/java && jar -xf "${tmp_dir}/orig.jar" org/)
  # Repackage src/
  jar -cMf "${tmp_dir}/new.jar" src
  popd
  cp "${tmp_dir}/new.jar" "${dest_jar}"
  rm -rf "${tmp_dir}"
}

#!/usr/bin/env bash
# Copyright 2016 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
#
# Script to generate tarballs:
# (1) The TensorFlow C-library: Containing C API header files and libtensorflow.so
# (2) Native library for the TensorFlow Java API: Containing libtensorflow_jni.so
# And jars:
# (3) Java API .jar
# (4) Java API sources .jar
#
# These binary distributions will allow use of TensorFlow in various languages
# without having to compile the TensorFlow framework from sources, which takes
# a while and also introduces many other dependencies.
#
# Usage:
# - Source this file in another bash script
# - Execute build_libtensorflow_tarball SUFFIX
#
# Produces:
# - lib_package/libtensorflow${SUFFIX}.tar.gz
# - lib_package/libtensorflow_jni${SUFFIX}.tar.gz
# - lib_package/libtensorflow.jar
# - lib_package/libtensorflow-src.jar
# - lib_package/libtensorflow_proto.zip
#
# ASSUMPTIONS:
# - build_libtensorflow_tarball is invoked from the root of the git tree.
# - Any environment variables needed by the "configure" script have been set.

function build_libtensorflow_tarball() {
  # Sanity check that this is being run from the root of the git repository.
  if [ ! -e "WORKSPACE" ]; then
    echo "Must run this from the root of the bazel workspace"
    exit 1
  fi
  # Delete any leftovers from previous builds in this workspace.
  DIR=lib_package
  rm -rf ${DIR}

  TARBALL_SUFFIX="${1}"
  if [ "$(uname)" == "Darwin" ]; then
    BAZEL_OPTS="${BAZEL_OPTS} --config=release_macos_linux"
  elif [ "${TF_NEED_CUDA}" == "1" ]; then
    BAZEL_OPTS="${BAZEL_OPTS} --config=release_gpu_linux"
  else
    BAZEL_OPTS="${BAZEL_OPTS} --config=release_cpu_linux"
  fi
  bazel clean --expunge
  export PYTHON_BIN_PATH=$(which python3.8)
  yes "" | "$PYTHON_BIN_PATH" configure.py

  # Remove this test call when
  # https://github.com/bazelbuild/bazel/issues/2352
  # and https://github.com/bazelbuild/bazel/issues/1580
  # have been resolved and the "manual" tags on the BUILD targets
  # in tensorflow/tools/lib_package/BUILD are removed.
  # Till then, must manually run the test since these tests are
  # not covered by the continuous integration.
  bazel test ${BAZEL_OPTS} --test_output=errors \
    //tensorflow/tools/lib_package:libtensorflow_test \
    //tensorflow/tools/lib_package:libtensorflow_java_test

  bazel build ${BAZEL_OPTS} \
    //tensorflow/tools/lib_package:libtensorflow.tar.gz \
    //tensorflow/tools/lib_package:libtensorflow_jni.tar.gz \
    //tensorflow/java:libtensorflow.jar \
    //tensorflow/java:libtensorflow-src.jar \
    //tensorflow/tools/lib_package:libtensorflow_proto.zip

  mkdir -p ${DIR}

  cp bazel-bin/tensorflow/tools/lib_package/libtensorflow.tar.gz ${DIR}/libtensorflow${TARBALL_SUFFIX}.tar.gz
  cp bazel-bin/tensorflow/tools/lib_package/libtensorflow_jni.tar.gz ${DIR}/libtensorflow_jni${TARBALL_SUFFIX}.tar.gz
  cp bazel-bin/tensorflow/java/libtensorflow.jar ${DIR}
  cp_normalized_srcjar bazel-bin/tensorflow/java/libtensorflow-src.jar ${DIR}/libtensorflow-src.jar
  cp bazel-bin/tensorflow/tools/lib_package/libtensorflow_proto.zip ${DIR}
  chmod -x ${DIR}/*
}

# Helper function to copy a srcjar after moving any source files
# directly under the root to the "maven-style" src/main/java layout
#
# Source files generated by annotation processors appear directly
# under the root of srcjars jars created by bazel, rather than under
# the maven-style src/main/java subdirectory.
#
# Bazel manages annotation generated source as follows: First, it
# calls javac with options that create generated files under a
# bazel-out directory. Next, it archives the generated source files
# into a srcjar directly under the root. There doesn't appear to be a
# simple way to parameterize this from bazel, hence this helper to
# "normalize" the srcjar layout.
#
# Arguments:
#   src_jar - path to the original srcjar
#   dest_jar - path to the destination
# Returns:
#   None
function cp_normalized_srcjar() {
  local src_jar="$1"
  local dest_jar="$2"
  if [[ -z "${src_jar}" || -z "${dest_jar}" ]]; then
    echo "Unexpected: missing arguments" >&2
    exit 2
  fi
  local tmp_dir
  tmp_dir=$(mktemp -d)
  cp "${src_jar}" "${tmp_dir}/orig.jar"
  pushd "${tmp_dir}"
  # Extract any src/ files
  jar -xf "${tmp_dir}/orig.jar" src/
  # Extract any org/ files under src/main/java
  (mkdir -p src/main/java && cd src/main/java && jar -xf "${tmp_dir}/orig.jar" org/)
  # Repackage src/
  jar -cMf "${tmp_dir}/new.jar" src
  popd
  cp "${tmp_dir}/new.jar" "${dest_jar}"
  rm -rf "${tmp_dir}"
}


Add ci script

Add ci script

#!/bin/bash
set -e
set -u

# checksums
MD5SUM="19e50b22e5ee88314ad6a6630d202277"
MD5SUM_ANNO="da002cc4c9c4f2c77e4401c97564be94"

# run ggsashimi without annotation
docker run --rm -w $PWD -v $PWD:$PWD guigolab/ggsashimi -b examples/input_bams.tsv -c chr10:27040584-27048100 -o ci/sashimi
[[ $(grep -avE 'CreationDate|ModDate' ci/sashimi.pdf | md5sum | awk '{$0=$1}1') == $MD5SUM ]]

# run ggsashimi with annotation
docker run --rm -w $PWD -v $PWD:$PWD guigolab/ggsashimi -g examples/annotation.gtf -b examples/input_bams.tsv -c chr10:27040584-27048100 -o ci/sashimi-anno
[[ $(grep -avE 'CreationDate|ModDate' ci/sashimi-anno.pdf | md5sum | awk '{$0=$1}1') == $MD5SUM_ANNO ]]

error: pathspec 'ci/run.sh' did not match any file(s) known to git


gopath

gopath

error: pathspec 'go/path.zsh' did not match any file(s) known to git


Update tmux to 2.9

Update tmux to 2.9

# This tmux statusbar config was created by tmuxline.vim
# on Sun, 28 May 2017

set -g status-justify "left"
set -g status "on"
set -g status-style "none"
set -g message-command-style bg="#504945"
set -g status-left-length "100"
set -g pane-active-border-style fg="#a89984"
set -g status-style bg="#3c3836"
set -g message-command-style fg="#a89984"
set -g pane-border-style fg="#504945"
set -g message-style bg="#504945"
set -g status-left-style "none"
set -g status-right-style "none"
set -g status-right-length "100"
set -g message-style fg="#a89984"
setw -g window-status-style fg="#a89984"
setw -g window-status-style bg="#3c3836","none"
setw -g window-status-activity-style bg="#3c3836",fg="#a89984","none"
setw -g window-status-separator ""
set -g status-left "#[fg=#1d2021,bg=#a89984] #S #[fg=#a89984,bg=#3c3836,nobold,nounderscore,noitalics]"
set -g status-right "#[fg=#504945,bg=#3c3836,nobold,nounderscore,noitalics]#[fg=#a89984,bg=#504945] %Y-%m-%d  %H:%M #[fg=#a89984,bg=#504945,nobold,nounderscore,noitalics]#[fg=#1d2021,bg=#a89984] #h "
setw -g window-status-format "#[fg=#a89984,bg=#3c3836] #I #[fg=#a89984,bg=#3c3836] #W "
setw -g window-status-current-format "#[fg=#3c3836,bg=#504945,nobold,nounderscore,noitalics]#[fg=#a89984,bg=#504945] #I #[fg=#a89984,bg=#504945] #W #[fg=#504945,bg=#3c3836,nobold,nounderscore,noitalics]"

# This tmux statusbar config was created by tmuxline.vim
# on Sun, 28 May 2017

set -g status-justify "left"
set -g status "on"
set -g status-attr "none"
set -g message-command-bg "#504945"
set -g status-left-length "100"
set -g pane-active-border-fg "#a89984"
set -g status-bg "#3c3836"
set -g message-command-fg "#a89984"
set -g pane-border-fg "#504945"
set -g message-bg "#504945"
set -g status-left-attr "none"
set -g status-right-attr "none"
set -g status-right-length "100"
set -g message-fg "#a89984"
setw -g window-status-fg "#a89984"
setw -g window-status-attr "none"
setw -g window-status-activity-bg "#3c3836"
setw -g window-status-activity-attr "none"
setw -g window-status-activity-fg "#a89984"
setw -g window-status-separator ""
setw -g window-status-bg "#3c3836"
set -g status-left "#[fg=#1d2021,bg=#a89984] #S #[fg=#a89984,bg=#3c3836,nobold,nounderscore,noitalics]"
set -g status-right "#[fg=#504945,bg=#3c3836,nobold,nounderscore,noitalics]#[fg=#a89984,bg=#504945] %Y-%m-%d  %H:%M #[fg=#a89984,bg=#504945,nobold,nounderscore,noitalics]#[fg=#1d2021,bg=#a89984] #h "
setw -g window-status-format "#[fg=#a89984,bg=#3c3836] #I #[fg=#a89984,bg=#3c3836] #W "
setw -g window-status-current-format "#[fg=#3c3836,bg=#504945,nobold,nounderscore,noitalics]#[fg=#a89984,bg=#504945] #I #[fg=#a89984,bg=#504945] #W #[fg=#504945,bg=#3c3836,nobold,nounderscore,noitalics]"


added dev context

added dev context

source lib/brew.sh

_installFormula sbt
_installFormula scala
_installFormula gradle
_installFormula maven
_installFormula git
_installFormula mysql

error: pathspec 'dev/install.sh' did not match any file(s) known to git


bash: add color.sh

bash: add color.sh

#!/bin/bash

cat << EOF

Black        0;30     Dark Gray     1;30
Red          0;31     Light Red     1;31
Green        0;32     Light Green   1;32
Brown/Orange 0;33     Yellow        1;33
Blue         0;34     Light Blue    1;34
Purple       0;35     Light Purple  1;35
Cyan         0;36     Light Cyan    1;36
Light Gray   0;37     White         1;37

EOF

RED='\033[0;31m'
NC='\033[0m' # No Color
printf "I ${RED}love${NC} Stack Overflow\n"

# Continued from above example
echo -e "I ${RED}love${NC} Stack Overflow"

error: pathspec 'bash/color.sh' did not match any file(s) known to git


Added Xcode alias for deleting Derived Data.

remote: Support for password authentication was removed on August 13, 2021.
remote: Please see https://docs.github.com/en/get-started/getting-started-with-git/about-remote-repositories#cloning-with-https-urls for information on currently recommended modes of authentication.
fatal: Authentication failed for 'https://github.com/cpcloud/oh-my-zsh.git/'


WIP on script to show logs of failing jobs

#!/bin/bash

set -e

function fetch_job_logs{
  job_name=$1
  job_id_or_uuid=${2:-'0'}
  job=${job_name}-${job_id_or_uuid}
  rm -rf /tmp/${job}
  mkdir -p /tmp/${job}
  bosh logs ${job_name} ${job_id_or_uuid} --dir /tmp/${job}
  tar xfz /tmp/${job}/*.tgz -O
}

task_id=$(boshcurl /deployments/${deployment}/vms\?format\=full | jq -r .id)
boshcurl /tasks/${task_id}/output\?type\=result | jq -r ". | select (.job_state != \"running\") | (.job_name, .id)"

error: pathspec 'ci/tasks/dump_bosh_logs_failing_jobs.sh' did not match any file(s) known to git


[master] colorize

[master] colorize

remote: Support for password authentication was removed on August 13, 2021.
remote: Please see https://docs.github.com/en/get-started/getting-started-with-git/about-remote-repositories#cloning-with-https-urls for information on currently recommended modes of authentication.
fatal: Authentication failed for 'https://github.com/hotpxl/colonise.git/'


Create Task17.sh

Create Task17.sh

#!/bin/sh
for i in {10..29}
do
   head -n 1 fileWithHeader.csv > tmp_file
   cat fileWithoutHeader_00$i >> tmp_file
   mv tmp_file fileWithoutHeader_00$i
done

error: pathspec 'bin/Task17.sh' did not match any file(s) known to git


solve probleem instalation

solve probleem instalation

remote: Support for password authentication was removed on August 13, 2021.
remote: Please see https://docs.github.com/en/get-started/getting-started-with-git/about-remote-repositories#cloning-with-https-urls for information on currently recommended modes of authentication.
fatal: Authentication failed for 'https://github.com/Stratio/Explorer.git/'


add install-rover-packages script

add install-rover-packages script

remote: Support for password authentication was removed on August 13, 2021.
remote: Please see https://docs.github.com/en/get-started/getting-started-with-git/about-remote-repositories#cloning-with-https-urls for information on currently recommended modes of authentication.
fatal: Authentication failed for 'https://github.com/team-diana/scripts.git/'


Berechne Mittelwert einer Spalte Ubuntu only!

#!/usr/bin/ksh

awk '
	BEGIN {
		col="'$2'"+0;
		min=max=$col+0;
	}

	{j++; val=$col+0;
		av+=val;

		if (min > val)
			min = val;
		else if (max < val)
			max = val;
	}

	END {
		printf "%8g %8g %8g %8g\n",j,av/j,min,max;
	} ' $1

error: pathspec 'Thermostaten/src/average.sh' did not match any file(s) known to git


updated version with changes for use on Mac and with new work credentials

remote: Support for password authentication was removed on August 13, 2021.
remote: Please see https://docs.github.com/en/get-started/getting-started-with-git/about-remote-repositories#cloning-with-https-urls for information on currently recommended modes of authentication.
fatal: Authentication failed for 'https://github.com/chriskauffman/eMailTest-Perl.git/'