Raise a timeout in the end-to-end-testscript

#!/bin/bash
# Copyright (C) 2009-2010:
#     Gabes Jean, naparuba@gmail.com
#     Gerhard Lausser, Gerhard.Lausser@consol.de
#
# This file is part of Shinken.
#
# Shinken is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Shinken is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with Shinken.  If not, see <http://www.gnu.org/licenses/>.

#set -x

echo "Begining test END TO END"

DIR=$(cd $(dirname "$0"); pwd)
echo "Going to dir $DIR/.."
cd $DIR/..

NB_CPUS=`cat /proc/cpuinfo  | grep 'processor' | wc -l` || 4
echo "NB CPUS: " $NB_CPUS

# check for a process existance with good number
function check_process_nb {
    NB=`ps -ef | grep -v grep | grep "shinken-"$1 | wc -l`
    if [ $NB != "$2" ]
    then
	echo "Error: There is not enough $1 launched (only $NB)."
	exit 2
    else
	echo "Ok, got $NB $1"
    fi
}

function is_file_present {
    if [ -e $1 ]
    then
	echo "File $1 is present."
    else
	echo "Error: File $1 is missing!"
	exit 2
    fi
}

function string_in_file {
    grep "$1" $2
    if [ $? != 0 ]
    then
	echo "Error: the file $2 is missing string $1!"
	exit 2
    else
	echo "The string $1 is in $2"
    fi
}

function print_date {
    date +%s
}

function check_good_run {
    VAR="$1"
    RUN="$2"
    LOG="$3"

    echo "Check for $NB_SCHEDULERS Scheduler"
    check_process_nb scheduler $NB_SCHEDULERS
    is_file_present $RUN/schedulerd.pid

    echo "Check for $NB_POLLERS pollers (1 master, and 4 workers)"
    check_process_nb poller $NB_POLLERS
    is_file_present $RUN/pollerd.pid

    echo "Check for $NB_REACTIONNERS reactionners (1 master, 1 for multiporcess module (queue manager) 1 worker)"
    check_process_nb reactionner $NB_REACTIONNERS
    is_file_present $RUN/reactionnerd.pid

    echo "Check for $NB_BROKERS brokers (one master, one for livestatus.dati, one for WebUI)"
    check_process_nb broker $NB_BROKERS
    is_file_present $RUN/brokerd.pid

    echo "Check for $NB_RECEIVERS receivers (one master)"
    check_process_nb receiver $NB_RECEIVERS
    is_file_present $RUN/receiverd.pid

    echo "Check for $NB_ARBITERS arbiter"
    check_process_nb arbiter $NB_ARBITERS
    is_file_present $RUN/arbiterd.pid

    echo "Now checking for good file prensence"
    ls var
    is_file_present $LOG/nagios.log
    string_in_file "Waiting for initial configuration" $LOG/nagios.log
    string_in_file "First scheduling" $LOG/nagios.log
    string_in_file "OK, all schedulers configurations are dispatched :)" $LOG/nagios.log
    string_in_file "OK, no more reactionner sent need" $LOG/nagios.log
    string_in_file "OK, no more poller sent need" $LOG/nagios.log
    string_in_file "OK, no more broker sent need" $LOG/nagios.log
}

function localize_config {
    # change paths in config files (/usr/local/shinken/*) to
    # relative paths, so this test runs only in the current directory.
    # takes nagios.cfg and shinken-specific.cfg
    cp $1 /tmp/nagios.cfg.save
    cp $2 /tmp/shinken-specific.cfg.save
    sed -e 's/\/usr\/local\/shinken\///g' < /tmp/nagios.cfg.save > $1
    sed -e 's/\/usr\/local\/shinken\/var\///g' < /tmp/shinken-specific.cfg.save > $2
}

function globalize_config {
    mv /tmp/nagios.cfg.save $1
    mv /tmp/shinken-specific.cfg.save $2
}

# Standard launch process packets
NB_SCHEDULERS=2
# NB Poller is 1 for core + nb cpus
NB_POLLERS=$((2 + $NB_CPUS))
NB_REACTIONNERS=3
NB_BROKERS=4
NB_RECEIVERS=2
NB_ARBITERS=3  # master itself & namedpipe-autogenerated!

echo "Clean old tests and kill remaining processes"
./clean.sh

echo "####################################################################################"
echo "#                                                                                #"
echo "#                           SIMPLE START                                           #"
echo "#                                                                                #"
echo "####################################################################################"

echo "Now we can start some launch tests"
localize_config etc/nagios.cfg etc/shinken-specific.cfg
bin/launch_all_debug.sh
globalize_config etc/nagios.cfg etc/shinken-specific.cfg

echo "Now checking for existing apps"

echo "we can sleep 5sec for conf dispatching and so good number of process"
sleep 10

#Now check if the run looks good with var in the direct directory
check_good_run var var var

echo "First launch check OK"

echo "####################################################################################"

echo "Now we clean it and test an install"
./clean.sh

echo "####################################################################################"
echo "#                                                                                #"
echo "#                           DUMMY INSTALL                                          #"
echo "#                                                                                #"
echo "####################################################################################"

echo "Now installing the application in DUMMY mode"
python setup.py install --root=/tmp/moncul --record=INSTALLED_FILES --install-scripts=/usr/bin

if [ $? != '0' ]
then
    echo "Error: the dummy install failed."
    exit 2
fi
echo "Dummy install OK"

echo "I reclean all for a real install"
./clean.sh

echo "####################################################################################"
echo "#                                                                                #"
echo "#                           REAL INSTALL                                           #"
echo "#                                                                                #"
echo "####################################################################################"

echo "Now a REAL install"
sudo python setup.py install --install-scripts=/usr/bin
if [ $? != '0' ]
then
    echo "Error: the real install failed."
    exit 2
fi
echo "Real install OK"

# Useful to take it from setup_parameter? It's just for coding here
ETC=/etc/shinken
is_file_present $ETC/nagios.cfg
is_file_present $ETC/shinken-specific.cfg
string_in_file "servicegroups.cfg" $ETC/nagios.cfg
is_file_present /usr/bin/shinken-arbiter

ps -fu shinken

echo "Now we can test a real run guy"
sudo /etc/init.d/shinken-scheduler -d start
sudo /etc/init.d/shinken-poller -d start
sudo /etc/init.d/shinken-reactionner -d start
sudo /etc/init.d/shinken-broker -d start
sudo /etc/init.d/shinken-receiver -d start
sudo /etc/init.d/shinken-arbiter -d start

echo "We will sleep again 5sec so every one is quite stable...."
sleep 10
check_good_run /var/lib/shinken /var/run/shinken /var/log/shinken

sudo /etc/init.d/shinken-arbiter status
sudo /etc/init.d/shinken-scheduler status
sudo /etc/init.d/shinken-poller status
sudo /etc/init.d/shinken-reactionner status
sudo /etc/init.d/shinken-broker status
sudo /etc/init.d/shinken-receiver status

sudo /etc/init.d/shinken-arbiter stop
sudo /etc/init.d/shinken-scheduler stop
sudo /etc/init.d/shinken-poller stop
sudo /etc/init.d/shinken-reactionner stop
sudo /etc/init.d/shinken-broker stop
sudo /etc/init.d/shinken-receiver stop

sleep 5
ps -fu shinken

check_process_nb arbiter 0
check_process_nb scheduler 0
check_process_nb broker 0
check_process_nb receiver 0
check_process_nb poller 0
check_process_nb reactionner 0

echo "OK Great. Even the real launch test pass. Great. I can clean after me."
./clean.sh

echo "####################################################################################"
echo "#                                                                                #"
echo "#                                    HA launch                                     #"
echo "#                                                                                #"
echo "####################################################################################"

echo "Now we can start some launch tests"
localize_config etc/nagios.cfg test/etc/test_stack2/shinken-specific-ha-only.cfg
test/bin/launch_all_debug2.sh
globalize_config etc/nagios.cfg test/etc/test_stack2/shinken-specific-ha-only.cfg

echo "Now checking for existing apps"

echo "we can sleep 5sec for conf dispatching and so good number of process"
sleep 10

# The number of process changed, we mush look for it

# Standard launch process packets
NB_SCHEDULERS=4
# 1+NB_CPUS for stack 1, and 1 for 2 (not active, so no worker)
NB_POLLERS=$((2 + $NB_CPUS + 2))
# 2 for stack1, 1 for stack2 (no worker from now)
NB_REACTIONNERS=5
# 2 for stack 1, 1 for stack2 (no livesatus.dat nor log worker launch)
NB_BROKERS=5
# Still 1 receiver
NB_RECEIVERS=2
# still 1
NB_ARBITERS=3

# Now check if the run looks good with var in the direct directory
check_good_run var var var

echo "All launch of HA daemons is OK"

# Now we kill and see if all is OK :)
# We clean the log file
#> $VAR/nagios.log

# We kill the most important thing first: the scheduler-Master
bin/stop_scheduler.sh

# We sleep to be sruethe scheduler see us
sleep 15
NB_SCHEDULERS=2
print_date

# First we look is the arbiter saw the scheduler as dead
string_in_file "Warning : Scheduler scheduler-Master had the configuration 0 but is dead, I am not happy." $VAR/nagios.log
# Then we look if the scheduler-spare got a conf from arbiter (here, view from the arbiter)
string_in_file "Dispatch OK of conf in scheduler scheduler-Spare" $VAR/nagios.log

# then is the broker know it and try to connect to the new scheduler-spare
string_in_file "\[broker-Master\] Connection OK to the scheduler scheduler-Spare" $VAR/nagios.log

echo "Now stop the poller-Master"
# Now we stop the poller. We will see the sapre take the job (we hope in fact :) )
bin/stop_poller.sh
# check_good_run var
sleep 10
print_date

# The master should be look dead
string_in_file "Warning : \[All\] The poller poller-Master seems to be down, I must re-dispatch its role to someone else." $VAR/nagios.log
# The spare should got the conf
string_in_file "\[All\] Dispatch OK of configuration 0 to poller poller-Slave" $VAR/nagios.log
# And he should got the scheduler link (the sapre one)
string_in_file "\[poller-Slave\] Connection OK with scheduler scheduler-Spare" $VAR/nagios.log
#string_in_file "\[poller-Slave\] Connection OK with scheduler scheduler-Spare" $VAR/pollerd-2.log

echo "Now stop the reactionner"
bin/stop_reactionner.sh
# check_good_run var
sleep 10
print_date

# The master should be look dead
string_in_file "Warning : \[All\] The reactionner reactionner-Master seems to be down, I must re-dispatch its role to someone else." $VAR/nagios.log
# The spare should got the conf
string_in_file "\[All\] Dispatch OK of configuration 0 to reactionner reactionner-Spare" $VAR/nagios.log
# And he should got the scheduler link (the sapre one)
string_in_file "\[reactionner-Spare\] Connection OK with scheduler scheduler-Spare" $VAR/nagios.log
# string_in_file "\[reactionner-Spare\] Connection OK with scheduler scheduler-Spare" $VAR/reactionnerd-2.log

echo "Now we stop... the Broker!"
bin/stop_broker.sh
# check_good_run var
sleep 10
print_date

# The master should be look dead
string_in_file "Warning : \[All\] The broker broker-Master seems to be down, I must re-dispatch its role to someone else." $VAR/nagios.log
# The spare should got the conf
string_in_file "\[All\] Dispatch OK of configuration 0 to broker broker-Slave" $VAR/nagios.log
# And he should got the scheduler link (the spare one)
string_in_file "\[broker-Slave\] Connection OK to the scheduler scheduler-Spare" $VAR/nagios.log
# And to other satellites
string_in_file "\[broker-Slave\] Connection OK to the reactionner reactionner-Spare" $VAR/nagios.log
string_in_file "\[broker-Slave\] Connection problem to the poller poller-Master" $VAR/nagios.log
# And should have load the modules
string_in_file "\[broker-Slave\] I correctly loaded the modules: \[Simple-log,Livestatus\]" $VAR/nagios.log

echo "Now we clean it"
./clean.sh

echo "####################################################################################"
echo "#                                                                                #"
echo "#                          Load balancing launch                                   #"
echo "#                                                                                #"
echo "####################################################################################"

echo "Now we can start some launch tests"
localize_config etc/nagios.cfg test/etc/test_stack2/shinken-specific-lb-only.cfg
test/bin/launch_all_debug3.sh
globalize_config etc/nagios.cfg test/etc/test_stack2/shinken-specific-lb-only.cfg

echo "Now checking for existing apps"

echo "we can sleep 5sec for conf dispatching and so good number of process"
sleep 10

# The number of process changed, we mush look for it

# Standard launch process packets
NB_SCHEDULERS=4
# 1 + nb cpus for stack 1, and same for stack 2
NB_POLLERS=$((2 + $NB_CPUS + 2 + $NB_CPUS))
# 2 for stack1, same for stack 2
NB_REACTIONNERS=6
# 2 for stack 1, 1 for stack2 (no livestatus nor log worker launch)
NB_BROKERS=5
# STill one receivers
NB_RECEIVERS=2
# still 1
NB_ARBITERS=3

# Now check if the run looks good with var in the direct directory
check_good_run var var var

echo "All launch of LB daemons is OK"

# Now look if it's also good in the log file too
string_in_file "Dispatch OK of conf in scheduler scheduler-Master-2" $VAR/nagios.log
string_in_file "Dispatch OK of conf in scheduler scheduler-Master-1" $VAR/nagios.log
string_in_file "OK, no more reactionner sent need" $VAR/nagios.log
string_in_file "OK, no more poller sent need" $VAR/nagios.log
string_in_file "OK, no more broker sent need" $VAR/nagios.log

# Now we will check what happened when we will an alive satellite, and if another active
# one got configuration again and again (and so don't work...) or if its managed
echo "Killing Poller 1"

POLLER1_PID=`ps -fu shinken | grep poller | grep -v test_stack2 | grep -v grep |awk '{print $2, $3}' |grep -E " 1$" | awk '{print $1}'`
kill $POLLER1_PID

echo "sleep some few seconds to see the arbiter react"
sleep 5
# And we look if the arbiter find that the other poller do not need another configuration send
string_in_file "Skipping configuration 0 send to the poller poller-Master-2: it already got it" $VAR/nagios.log

echo "Now we clean it"
./clean.sh

echo "####################################################################################"
echo "#                                                                                #"
echo "#                              Passive Poller                                      #"
echo "#                                                                                #"
echo "####################################################################################"

echo "Now we can start some launch tests"
localize_config etc/nagios.cfg test/etc/test_stack2/shinken-specific-passive-poller.cfg
test/bin/launch_all_debug4.sh
globalize_config etc/nagios.cfg test/etc/test_stack2/shinken-specific-passive-poller.cfg

echo "Now checking for existing apps"

echo "we can sleep 5sec for conf dispatching and so good number of process"
sleep 10

# The number of process changed, we mush look for it

# Standard launch process packets
NB_SCHEDULERS=4
# 5 for stack 1, and 5 for stack 2
NB_POLLERS=$((2 + $NB_CPUS + 2 + $NB_CPUS))
# 2 for stack1, Only 1 for stack 2 because it is not active
NB_REACTIONNERS=5
# 2 for stack 1, 1 for stack2 (no livestatus nor log worker launch)
NB_BROKERS=5
# Still oen receiver
NB_RECEIVERS=2
# still 1
NB_ARBITERS=3

# Now check if the run looks good with var in the direct directory
check_good_run var var var

echo "All launch of LB daemons is OK"

# Now look if it's also good in the log file too
string_in_file "Dispatch OK of conf in scheduler scheduler-Master-2" $VAR/nagios.log
string_in_file "Dispatch OK of conf in scheduler scheduler-Master-1" $VAR/nagios.log
string_in_file "OK, no more reactionner sent need" $VAR/nagios.log
string_in_file "OK, no more poller sent need" $VAR/nagios.log
string_in_file "OK, no more broker sent need" $VAR/nagios.log
# We should see the poller 2 say it is passive
string_in_file "\[poller-Master-2\] Passive mode enabled." $VAR/nagios.log
# and the schedulers should connect to it too
string_in_file "Connection OK to the poller poller-Master-2" $VAR/nagios.log

echo "Now we clean it"
./clean.sh

echo "####################################################################################"
echo "#                                                                                #"
echo "#                              Scheduler restart                                   #"
echo "#                                                                                #"
echo "####################################################################################"

echo "Now we can start some launch tests"
localize_config etc/nagios.cfg etc/shinken-specific.cfg
bin/launch_all_debug.sh
globalize_config etc/nagios.cfg etc/shinken-specific.cfg

echo "Now checking for existing apps"

echo "we can sleep 5sec for conf dispatching and so good number of process"
sleep 10

# The number of process changed, we mush look for it

# Standard launch process packets
NB_SCHEDULERS=2
# NB Poller is 1 for core + nb cpus
NB_POLLERS=$((2 + $NB_CPUS))
NB_REACTIONNERS=3
NB_BROKERS=4
NB_RECEIVERS=2
NB_ARBITERS=3  # master itself & namedpipe-autogenerated!

# Now check if the run looks good with var in the direct directory
check_good_run var var var

echo "All launch of LB daemons is OK"

# Now look if it's also good in the log file too
string_in_file "Dispatch OK of conf in scheduler scheduler-1" $VAR/nagios.log
string_in_file "OK, no more reactionner sent need" $VAR/nagios.log
string_in_file "OK, no more poller sent need" $VAR/nagios.log
string_in_file "OK, no more broker sent need" $VAR/nagios.log

# Now we stop the scheduler and restart it
# We clean the log and restart teh scheduler
bin/stop_scheduler.sh
> $VAR/nagios.log
sleep 3
bin/launch_scheduler_debug.sh
sleep 120

string_in_file "Warning : Scheduler scheduler-1 did not managed its configuration 0,I am not happy." $VAR/nagios.log
string_in_file "Dispatch OK of conf in scheduler scheduler-1" $VAR/nagios.log
string_in_file "We already got the conf 0 (scheduler-1)" $VAR/nagios.log

echo "Now we clean it"
./clean.sh

echo "####################################################################################"
echo "#                                                                                #"
echo "#                              Passive Arbiter                                     #"
echo "#                                                                                #"
echo "####################################################################################"

echo "Now we can start some launch tests"
localize_config etc/nagios.cfg test/etc/test_stack2/shinken-specific-passive-arbiter.cfg
test/bin/launch_all_debug5.sh
globalize_config etc/nagios.cfg test/etc/test_stack2/shinken-specific-passive-arbiter.cfg

echo "Now checking for existing apps"

echo "we can sleep 5sec for conf dispatching and so good number of process"
sleep 10

# The number of process changed, we mush look for it

# Standard launch process packets
NB_SCHEDULERS=4
# 5 for stack 1, and 5 for stack 2
NB_POLLERS=$((2 + $NB_CPUS + 2 + $NB_CPUS))
# 2 for stack1, Only 1 for stack 2 because it is not active
NB_REACTIONNERS=5
# 2 for stack 1, 1 for stack2 (no livestatus nor log worker launch)
NB_BROKERS=5
# Still oen receiver
NB_RECEIVERS=2
# still 1
NB_ARBITERS=3

# Now check if the run looks good with var in the direct directory
check_good_run var var var

echo "All launch of LB daemons is OK"

# Now look if it's also good in the log file too
string_in_file "Dispatch OK of conf in scheduler scheduler-Master-2" $VAR/nagios.log
string_in_file "Dispatch OK of conf in scheduler scheduler-Master-1" $VAR/nagios.log
string_in_file "OK, no more reactionner sent need" $VAR/nagios.log
string_in_file "OK, no more poller sent need" $VAR/nagios.log
string_in_file "OK, no more broker sent need" $VAR/nagios.log

# And the string so the spare is taking the control
string_in_file "I (Arbiter-spare) take the lead" $VAR/nagios.log

echo "Now we clean it"
./clean.sh

echo "####################################################################################"
echo "#                                                                                #"
echo "#                          Direct routing for receiver                             #"
echo "#                                                                                #"
echo "####################################################################################"

# Special clean, the previous(?) external command file
CMD_FILE=/tmp/tmp-for-receiver-direct-routing.cmd
rm -f $CMD_FILE

echo "Now we can start some launch tests"
localize_config etc/nagios.cfg test/etc/test_stack2/shinken-specific-receiver-direct-routing.cfg
test/bin/launch_all_debug6.sh
globalize_config etc/nagios.cfg test/etc/test_stack2/shinken-specific-receiver-direct-routing.cfg

echo "Now checking for existing apps"

echo "we can sleep 5sec for conf dispatching and so good number of process"
sleep 10

# The number of process changed, we mush look for it

# Standard launch process packets
NB_SCHEDULERS=4
# 5 for stack 1, and 5 for stack 2
NB_POLLERS=$((2 + $NB_CPUS + 2 + $NB_CPUS))
# 2 for stack1, Only 1 for stack 2 because it is not active
NB_REACTIONNERS=5
# 2 for stack 1, 1 for stack2 (no livestatus nor log worker launch)
NB_BROKERS=5
# Still oen receiver
NB_RECEIVERS=3
# still 1
NB_ARBITERS=3

# Now check if the run looks good with var in the direct directory
check_good_run var var var

echo "All launch of LB daemons is OK"

#Look if the command file is present
is_file_present $CMD_FILE

# Now look if it's also good in the log file too
#string_in_file "Dispatch OK of conf in scheduler scheduler-Master-2" $VAR/nagios.log
string_in_file "Dispatch OK of conf in scheduler scheduler-Master-1" $VAR/nagios.log
string_in_file "OK, no more reactionner sent need" $VAR/nagios.log
string_in_file "OK, no more poller sent need" $VAR/nagios.log
string_in_file "OK, no more broker sent need" $VAR/nagios.log
string_in_file "OK, no more receiver sent need" $VAR/nagios.log

now=$(date +%s)

printf "[111] ADD_SIMPLE_POLLER;All;newpoller;localhost;8771\n" > $CMD_FILE
printf "[111] PROCESS_SERVICE_CHECK_RESULT;localhost;LocalDisks;2;Oh yes\n" > $CMD_FILE
printf "[111] PROCESS_HOST_CHECK_RESULT;localhost;2;Oh yes\n" > $CMD_FILE

sleep 30

string_in_file "Dispatch OK of configuration 0 to poller newpoller"   $VAR/nagios.log
string_in_file "PASSIVE HOST CHECK: localhost;2;Oh yes"   $VAR/nagios.log

# Now we will try to stop the scheduler, and switch to a new one
echo "STOPPING MASTER SCHEDULER"
bin/stop_scheduler.sh

sleep 10

date +%s
#Check if slave scheduler is ok
string_in_file "Dispatch OK of conf in scheduler scheduler-Master-2"   $VAR/nagios.log

# Clean the log
> $VAR/nagios.log

printf "[111] ADD_SIMPLE_POLLER;All;newpoller;localhost;8771\n" > $CMD_FILE
printf "[111] PROCESS_HOST_CHECK_RESULT;localhost;2;Oh yes again\n" > $CMD_FILE

sleep 5

date +%s
string_in_file "PASSIVE HOST CHECK: localhost;2;Oh yes again"   $VAR/nagios.log

echo "Now we clean it"
./clean.sh

echo ""
echo ""
echo "All check are OK. Congrats! You can go take a Beer ;)"

#!/bin/bash
# Copyright (C) 2009-2010:
#     Gabes Jean, naparuba@gmail.com
#     Gerhard Lausser, Gerhard.Lausser@consol.de
#
# This file is part of Shinken.
#
# Shinken is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Shinken is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with Shinken.  If not, see <http://www.gnu.org/licenses/>.

#set -x

echo "Begining test END TO END"

DIR=$(cd $(dirname "$0"); pwd)
echo "Going to dir $DIR/.."
cd $DIR/..

NB_CPUS=`cat /proc/cpuinfo  | grep 'processor' | wc -l` || 4
echo "NB CPUS: " $NB_CPUS

# check for a process existance with good number
function check_process_nb {
    NB=`ps -ef | grep -v grep | grep "shinken-"$1 | wc -l`
    if [ $NB != "$2" ]
    then
	echo "Error: There is not enough $1 launched (only $NB)."
	exit 2
    else
	echo "Ok, got $NB $1"
    fi
}

function is_file_present {
    if [ -e $1 ]
    then
	echo "File $1 is present."
    else
	echo "Error: File $1 is missing!"
	exit 2
    fi
}

function string_in_file {
    grep "$1" $2
    if [ $? != 0 ]
    then
	echo "Error: the file $2 is missing string $1!"
	exit 2
    else
	echo "The string $1 is in $2"
    fi
}

function print_date {
    date +%s
}

function check_good_run {
    VAR="$1"
    RUN="$2"
    LOG="$3"

    echo "Check for $NB_SCHEDULERS Scheduler"
    check_process_nb scheduler $NB_SCHEDULERS
    is_file_present $RUN/schedulerd.pid

    echo "Check for $NB_POLLERS pollers (1 master, and 4 workers)"
    check_process_nb poller $NB_POLLERS
    is_file_present $RUN/pollerd.pid

    echo "Check for $NB_REACTIONNERS reactionners (1 master, 1 for multiporcess module (queue manager) 1 worker)"
    check_process_nb reactionner $NB_REACTIONNERS
    is_file_present $RUN/reactionnerd.pid

    echo "Check for $NB_BROKERS brokers (one master, one for livestatus.dati, one for WebUI)"
    check_process_nb broker $NB_BROKERS
    is_file_present $RUN/brokerd.pid

    echo "Check for $NB_RECEIVERS receivers (one master)"
    check_process_nb receiver $NB_RECEIVERS
    is_file_present $RUN/receiverd.pid

    echo "Check for $NB_ARBITERS arbiter"
    check_process_nb arbiter $NB_ARBITERS
    is_file_present $RUN/arbiterd.pid

    echo "Now checking for good file prensence"
    ls var
    is_file_present $LOG/nagios.log
    string_in_file "Waiting for initial configuration" $LOG/nagios.log
    string_in_file "First scheduling" $LOG/nagios.log
    string_in_file "OK, all schedulers configurations are dispatched :)" $LOG/nagios.log
    string_in_file "OK, no more reactionner sent need" $LOG/nagios.log
    string_in_file "OK, no more poller sent need" $LOG/nagios.log
    string_in_file "OK, no more broker sent need" $LOG/nagios.log
}

function localize_config {
    # change paths in config files (/usr/local/shinken/*) to
    # relative paths, so this test runs only in the current directory.
    # takes nagios.cfg and shinken-specific.cfg
    cp $1 /tmp/nagios.cfg.save
    cp $2 /tmp/shinken-specific.cfg.save
    sed -e 's/\/usr\/local\/shinken\///g' < /tmp/nagios.cfg.save > $1
    sed -e 's/\/usr\/local\/shinken\/var\///g' < /tmp/shinken-specific.cfg.save > $2
}

function globalize_config {
    mv /tmp/nagios.cfg.save $1
    mv /tmp/shinken-specific.cfg.save $2
}

# Standard launch process packets
NB_SCHEDULERS=2
# NB Poller is 1 for core + nb cpus
NB_POLLERS=$((2 + $NB_CPUS))
NB_REACTIONNERS=3
NB_BROKERS=4
NB_RECEIVERS=2
NB_ARBITERS=3  # master itself & namedpipe-autogenerated!

echo "Clean old tests and kill remaining processes"
./clean.sh

echo "####################################################################################"
echo "#                                                                                #"
echo "#                           SIMPLE START                                           #"
echo "#                                                                                #"
echo "####################################################################################"

echo "Now we can start some launch tests"
localize_config etc/nagios.cfg etc/shinken-specific.cfg
bin/launch_all_debug.sh
globalize_config etc/nagios.cfg etc/shinken-specific.cfg

echo "Now checking for existing apps"

echo "we can sleep 5sec for conf dispatching and so good number of process"
sleep 10

#Now check if the run looks good with var in the direct directory
check_good_run var var var

echo "First launch check OK"

echo "####################################################################################"

echo "Now we clean it and test an install"
./clean.sh

echo "####################################################################################"
echo "#                                                                                #"
echo "#                           DUMMY INSTALL                                          #"
echo "#                                                                                #"
echo "####################################################################################"

echo "Now installing the application in DUMMY mode"
python setup.py install --root=/tmp/moncul --record=INSTALLED_FILES --install-scripts=/usr/bin

if [ $? != '0' ]
then
    echo "Error: the dummy install failed."
    exit 2
fi
echo "Dummy install OK"

echo "I reclean all for a real install"
./clean.sh

echo "####################################################################################"
echo "#                                                                                #"
echo "#                           REAL INSTALL                                           #"
echo "#                                                                                #"
echo "####################################################################################"

echo "Now a REAL install"
sudo python setup.py install --install-scripts=/usr/bin
if [ $? != '0' ]
then
    echo "Error: the real install failed."
    exit 2
fi
echo "Real install OK"

# Useful to take it from setup_parameter? It's just for coding here
ETC=/etc/shinken
is_file_present $ETC/nagios.cfg
is_file_present $ETC/shinken-specific.cfg
string_in_file "servicegroups.cfg" $ETC/nagios.cfg
is_file_present /usr/bin/shinken-arbiter

ps -fu shinken

echo "Now we can test a real run guy"
sudo /etc/init.d/shinken-scheduler -d start
sudo /etc/init.d/shinken-poller -d start
sudo /etc/init.d/shinken-reactionner -d start
sudo /etc/init.d/shinken-broker -d start
sudo /etc/init.d/shinken-receiver -d start
sudo /etc/init.d/shinken-arbiter -d start

echo "We will sleep again 5sec so every one is quite stable...."
sleep 10
check_good_run /var/lib/shinken /var/run/shinken /var/log/shinken

sudo /etc/init.d/shinken-arbiter status
sudo /etc/init.d/shinken-scheduler status
sudo /etc/init.d/shinken-poller status
sudo /etc/init.d/shinken-reactionner status
sudo /etc/init.d/shinken-broker status
sudo /etc/init.d/shinken-receiver status

sudo /etc/init.d/shinken-arbiter stop
sudo /etc/init.d/shinken-scheduler stop
sudo /etc/init.d/shinken-poller stop
sudo /etc/init.d/shinken-reactionner stop
sudo /etc/init.d/shinken-broker stop
sudo /etc/init.d/shinken-receiver stop

sleep 5
ps -fu shinken

check_process_nb arbiter 0
check_process_nb scheduler 0
check_process_nb broker 0
check_process_nb receiver 0
check_process_nb poller 0
check_process_nb reactionner 0

echo "OK Great. Even the real launch test pass. Great. I can clean after me."
./clean.sh

echo "####################################################################################"
echo "#                                                                                #"
echo "#                                    HA launch                                     #"
echo "#                                                                                #"
echo "####################################################################################"

echo "Now we can start some launch tests"
localize_config etc/nagios.cfg test/etc/test_stack2/shinken-specific-ha-only.cfg
test/bin/launch_all_debug2.sh
globalize_config etc/nagios.cfg test/etc/test_stack2/shinken-specific-ha-only.cfg

echo "Now checking for existing apps"

echo "we can sleep 5sec for conf dispatching and so good number of process"
sleep 10

# The number of process changed, we mush look for it

# Standard launch process packets
NB_SCHEDULERS=4
# 1+NB_CPUS for stack 1, and 1 for 2 (not active, so no worker)
NB_POLLERS=$((2 + $NB_CPUS + 2))
# 2 for stack1, 1 for stack2 (no worker from now)
NB_REACTIONNERS=5
# 2 for stack 1, 1 for stack2 (no livesatus.dat nor log worker launch)
NB_BROKERS=5
# Still 1 receiver
NB_RECEIVERS=2
# still 1
NB_ARBITERS=3

# Now check if the run looks good with var in the direct directory
check_good_run var var var

echo "All launch of HA daemons is OK"

# Now we kill and see if all is OK :)
# We clean the log file
#> $VAR/nagios.log

# We kill the most important thing first: the scheduler-Master
bin/stop_scheduler.sh

# We sleep to be sruethe scheduler see us
sleep 15
NB_SCHEDULERS=2
print_date

# First we look is the arbiter saw the scheduler as dead
string_in_file "Warning : Scheduler scheduler-Master had the configuration 0 but is dead, I am not happy." $VAR/nagios.log
# Then we look if the scheduler-spare got a conf from arbiter (here, view from the arbiter)
string_in_file "Dispatch OK of conf in scheduler scheduler-Spare" $VAR/nagios.log

# then is the broker know it and try to connect to the new scheduler-spare
string_in_file "\[broker-Master\] Connection OK to the scheduler scheduler-Spare" $VAR/nagios.log

echo "Now stop the poller-Master"
# Now we stop the poller. We will see the sapre take the job (we hope in fact :) )
bin/stop_poller.sh
# check_good_run var
sleep 10
print_date

# The master should be look dead
string_in_file "Warning : \[All\] The poller poller-Master seems to be down, I must re-dispatch its role to someone else." $VAR/nagios.log
# The spare should got the conf
string_in_file "\[All\] Dispatch OK of configuration 0 to poller poller-Slave" $VAR/nagios.log
# And he should got the scheduler link (the sapre one)
string_in_file "\[poller-Slave\] Connection OK with scheduler scheduler-Spare" $VAR/nagios.log
#string_in_file "\[poller-Slave\] Connection OK with scheduler scheduler-Spare" $VAR/pollerd-2.log

echo "Now stop the reactionner"
bin/stop_reactionner.sh
# check_good_run var
sleep 10
print_date

# The master should be look dead
string_in_file "Warning : \[All\] The reactionner reactionner-Master seems to be down, I must re-dispatch its role to someone else." $VAR/nagios.log
# The spare should got the conf
string_in_file "\[All\] Dispatch OK of configuration 0 to reactionner reactionner-Spare" $VAR/nagios.log
# And he should got the scheduler link (the sapre one)
string_in_file "\[reactionner-Spare\] Connection OK with scheduler scheduler-Spare" $VAR/nagios.log
# string_in_file "\[reactionner-Spare\] Connection OK with scheduler scheduler-Spare" $VAR/reactionnerd-2.log

echo "Now we stop... the Broker!"
bin/stop_broker.sh
# check_good_run var
sleep 10
print_date

# The master should be look dead
string_in_file "Warning : \[All\] The broker broker-Master seems to be down, I must re-dispatch its role to someone else." $VAR/nagios.log
# The spare should got the conf
string_in_file "\[All\] Dispatch OK of configuration 0 to broker broker-Slave" $VAR/nagios.log
# And he should got the scheduler link (the spare one)
string_in_file "\[broker-Slave\] Connection OK to the scheduler scheduler-Spare" $VAR/nagios.log
# And to other satellites
string_in_file "\[broker-Slave\] Connection OK to the reactionner reactionner-Spare" $VAR/nagios.log
string_in_file "\[broker-Slave\] Connection problem to the poller poller-Master" $VAR/nagios.log
# And should have load the modules
string_in_file "\[broker-Slave\] I correctly loaded the modules: \[Simple-log,Livestatus\]" $VAR/nagios.log

echo "Now we clean it"
./clean.sh

echo "####################################################################################"
echo "#                                                                                #"
echo "#                          Load balancing launch                                   #"
echo "#                                                                                #"
echo "####################################################################################"

echo "Now we can start some launch tests"
localize_config etc/nagios.cfg test/etc/test_stack2/shinken-specific-lb-only.cfg
test/bin/launch_all_debug3.sh
globalize_config etc/nagios.cfg test/etc/test_stack2/shinken-specific-lb-only.cfg

echo "Now checking for existing apps"

echo "we can sleep 5sec for conf dispatching and so good number of process"
sleep 10

# The number of process changed, we mush look for it

# Standard launch process packets
NB_SCHEDULERS=4
# 1 + nb cpus for stack 1, and same for stack 2
NB_POLLERS=$((2 + $NB_CPUS + 2 + $NB_CPUS))
# 2 for stack1, same for stack 2
NB_REACTIONNERS=6
# 2 for stack 1, 1 for stack2 (no livestatus nor log worker launch)
NB_BROKERS=5
# STill one receivers
NB_RECEIVERS=2
# still 1
NB_ARBITERS=3

# Now check if the run looks good with var in the direct directory
check_good_run var var var

echo "All launch of LB daemons is OK"

# Now look if it's also good in the log file too
string_in_file "Dispatch OK of conf in scheduler scheduler-Master-2" $VAR/nagios.log
string_in_file "Dispatch OK of conf in scheduler scheduler-Master-1" $VAR/nagios.log
string_in_file "OK, no more reactionner sent need" $VAR/nagios.log
string_in_file "OK, no more poller sent need" $VAR/nagios.log
string_in_file "OK, no more broker sent need" $VAR/nagios.log

# Now we will check what happened when we will an alive satellite, and if another active
# one got configuration again and again (and so don't work...) or if its managed
echo "Killing Poller 1"

POLLER1_PID=`ps -fu shinken | grep poller | grep -v test_stack2 | grep -v grep |awk '{print $2, $3}' |grep -E " 1$" | awk '{print $1}'`
kill $POLLER1_PID

echo "sleep some few seconds to see the arbiter react"
sleep 5
# And we look if the arbiter find that the other poller do not need another configuration send
string_in_file "Skipping configuration 0 send to the poller poller-Master-2: it already got it" $VAR/nagios.log

echo "Now we clean it"
./clean.sh

echo "####################################################################################"
echo "#                                                                                #"
echo "#                              Passive Poller                                      #"
echo "#                                                                                #"
echo "####################################################################################"

echo "Now we can start some launch tests"
localize_config etc/nagios.cfg test/etc/test_stack2/shinken-specific-passive-poller.cfg
test/bin/launch_all_debug4.sh
globalize_config etc/nagios.cfg test/etc/test_stack2/shinken-specific-passive-poller.cfg

echo "Now checking for existing apps"

echo "we can sleep 5sec for conf dispatching and so good number of process"
sleep 10

# The number of process changed, we mush look for it

# Standard launch process packets
NB_SCHEDULERS=4
# 5 for stack 1, and 5 for stack 2
NB_POLLERS=$((2 + $NB_CPUS + 2 + $NB_CPUS))
# 2 for stack1, Only 1 for stack 2 because it is not active
NB_REACTIONNERS=5
# 2 for stack 1, 1 for stack2 (no livestatus nor log worker launch)
NB_BROKERS=5
# Still oen receiver
NB_RECEIVERS=2
# still 1
NB_ARBITERS=3

# Now check if the run looks good with var in the direct directory
check_good_run var var var

echo "All launch of LB daemons is OK"

# Now look if it's also good in the log file too
string_in_file "Dispatch OK of conf in scheduler scheduler-Master-2" $VAR/nagios.log
string_in_file "Dispatch OK of conf in scheduler scheduler-Master-1" $VAR/nagios.log
string_in_file "OK, no more reactionner sent need" $VAR/nagios.log
string_in_file "OK, no more poller sent need" $VAR/nagios.log
string_in_file "OK, no more broker sent need" $VAR/nagios.log
# We should see the poller 2 say it is passive
string_in_file "\[poller-Master-2\] Passive mode enabled." $VAR/nagios.log
# and the schedulers should connect to it too
string_in_file "Connection OK to the poller poller-Master-2" $VAR/nagios.log

echo "Now we clean it"
./clean.sh

echo "####################################################################################"
echo "#                                                                                #"
echo "#                              Scheduler restart                                   #"
echo "#                                                                                #"
echo "####################################################################################"

echo "Now we can start some launch tests"
localize_config etc/nagios.cfg etc/shinken-specific.cfg
bin/launch_all_debug.sh
globalize_config etc/nagios.cfg etc/shinken-specific.cfg

echo "Now checking for existing apps"

echo "we can sleep 5sec for conf dispatching and so good number of process"
sleep 10

# The number of process changed, we mush look for it

# Standard launch process packets
NB_SCHEDULERS=2
# NB Poller is 1 for core + nb cpus
NB_POLLERS=$((2 + $NB_CPUS))
NB_REACTIONNERS=3
NB_BROKERS=4
NB_RECEIVERS=2
NB_ARBITERS=3  # master itself & namedpipe-autogenerated!

# Now check if the run looks good with var in the direct directory
check_good_run var var var

echo "All launch of LB daemons is OK"

# Now look if it's also good in the log file too
string_in_file "Dispatch OK of conf in scheduler scheduler-1" $VAR/nagios.log
string_in_file "OK, no more reactionner sent need" $VAR/nagios.log
string_in_file "OK, no more poller sent need" $VAR/nagios.log
string_in_file "OK, no more broker sent need" $VAR/nagios.log

# Now we stop the scheduler and restart it
# We clean the log and restart teh scheduler
bin/stop_scheduler.sh
> $VAR/nagios.log
sleep 3
bin/launch_scheduler_debug.sh
sleep 120

string_in_file "Warning : Scheduler scheduler-1 did not managed its configuration 0,I am not happy." $VAR/nagios.log
string_in_file "Dispatch OK of conf in scheduler scheduler-1" $VAR/nagios.log
string_in_file "We already got the conf 0 (scheduler-1)" $VAR/nagios.log

echo "Now we clean it"
./clean.sh

echo "####################################################################################"
echo "#                                                                                #"
echo "#                              Passive Arbiter                                     #"
echo "#                                                                                #"
echo "####################################################################################"

echo "Now we can start some launch tests"
localize_config etc/nagios.cfg test/etc/test_stack2/shinken-specific-passive-arbiter.cfg
test/bin/launch_all_debug5.sh
globalize_config etc/nagios.cfg test/etc/test_stack2/shinken-specific-passive-arbiter.cfg

echo "Now checking for existing apps"

echo "we can sleep 5sec for conf dispatching and so good number of process"
sleep 10

# The number of process changed, we mush look for it

# Standard launch process packets
NB_SCHEDULERS=4
# 5 for stack 1, and 5 for stack 2
NB_POLLERS=$((2 + $NB_CPUS + 2 + $NB_CPUS))
# 2 for stack1, Only 1 for stack 2 because it is not active
NB_REACTIONNERS=5
# 2 for stack 1, 1 for stack2 (no livestatus nor log worker launch)
NB_BROKERS=5
# Still oen receiver
NB_RECEIVERS=2
# still 1
NB_ARBITERS=3

# Now check if the run looks good with var in the direct directory
check_good_run var var var

echo "All launch of LB daemons is OK"

# Now look if it's also good in the log file too
string_in_file "Dispatch OK of conf in scheduler scheduler-Master-2" $VAR/nagios.log
string_in_file "Dispatch OK of conf in scheduler scheduler-Master-1" $VAR/nagios.log
string_in_file "OK, no more reactionner sent need" $VAR/nagios.log
string_in_file "OK, no more poller sent need" $VAR/nagios.log
string_in_file "OK, no more broker sent need" $VAR/nagios.log

# And the string so the spare is taking the control
string_in_file "I (Arbiter-spare) take the lead" $VAR/nagios.log

echo "Now we clean it"
./clean.sh

echo "####################################################################################"
echo "#                                                                                #"
echo "#                          Direct routing for receiver                             #"
echo "#                                                                                #"
echo "####################################################################################"

# Special clean, the previous(?) external command file
CMD_FILE=/tmp/tmp-for-receiver-direct-routing.cmd
rm -f $CMD_FILE

echo "Now we can start some launch tests"
localize_config etc/nagios.cfg test/etc/test_stack2/shinken-specific-receiver-direct-routing.cfg
test/bin/launch_all_debug6.sh
globalize_config etc/nagios.cfg test/etc/test_stack2/shinken-specific-receiver-direct-routing.cfg

echo "Now checking for existing apps"

echo "we can sleep 5sec for conf dispatching and so good number of process"
sleep 10

# The number of process changed, we mush look for it

# Standard launch process packets
NB_SCHEDULERS=4
# 5 for stack 1, and 5 for stack 2
NB_POLLERS=$((2 + $NB_CPUS + 2 + $NB_CPUS))
# 2 for stack1, Only 1 for stack 2 because it is not active
NB_REACTIONNERS=5
# 2 for stack 1, 1 for stack2 (no livestatus nor log worker launch)
NB_BROKERS=5
# Still oen receiver
NB_RECEIVERS=3
# still 1
NB_ARBITERS=3

# Now check if the run looks good with var in the direct directory
check_good_run var var var

echo "All launch of LB daemons is OK"

#Look if the command file is present
is_file_present $CMD_FILE

# Now look if it's also good in the log file too
#string_in_file "Dispatch OK of conf in scheduler scheduler-Master-2" $VAR/nagios.log
string_in_file "Dispatch OK of conf in scheduler scheduler-Master-1" $VAR/nagios.log
string_in_file "OK, no more reactionner sent need" $VAR/nagios.log
string_in_file "OK, no more poller sent need" $VAR/nagios.log
string_in_file "OK, no more broker sent need" $VAR/nagios.log
string_in_file "OK, no more receiver sent need" $VAR/nagios.log

now=$(date +%s)

printf "[111] ADD_SIMPLE_POLLER;All;newpoller;localhost;8771\n" > $CMD_FILE
printf "[111] PROCESS_SERVICE_CHECK_RESULT;localhost;LocalDisks;2;Oh yes\n" > $CMD_FILE
printf "[111] PROCESS_HOST_CHECK_RESULT;localhost;2;Oh yes\n" > $CMD_FILE

sleep 3

string_in_file "Dispatch OK of configuration 0 to poller newpoller"   $VAR/nagios.log
string_in_file "PASSIVE HOST CHECK: localhost;2;Oh yes"   $VAR/nagios.log

# Now we will try to stop the scheduler, and switch to a new one
echo "STOPPING MASTER SCHEDULER"
bin/stop_scheduler.sh

sleep 10

date +%s
#Check if slave scheduler is ok
string_in_file "Dispatch OK of conf in scheduler scheduler-Master-2"   $VAR/nagios.log

# Clean the log
> $VAR/nagios.log

printf "[111] ADD_SIMPLE_POLLER;All;newpoller;localhost;8771\n" > $CMD_FILE
printf "[111] PROCESS_HOST_CHECK_RESULT;localhost;2;Oh yes again\n" > $CMD_FILE

sleep 5

date +%s
string_in_file "PASSIVE HOST CHECK: localhost;2;Oh yes again"   $VAR/nagios.log

echo "Now we clean it"
./clean.sh

echo ""
echo ""
echo "All check are OK. Congrats! You can go take a Beer ;)"


videostreamer.sh chmodded to be executable

remote: Support for password authentication was removed on August 13, 2021.
remote: Please see https://docs.github.com/en/get-started/getting-started-with-git/about-remote-repositories#cloning-with-https-urls for information on currently recommended modes of authentication.
fatal: Authentication failed for 'https://github.com/themasterchef/pcl-streamer.git/'


Added init file for bootstrapping

Added init file for bootstrapping

remote: Support for password authentication was removed on August 13, 2021.
remote: Please see https://docs.github.com/en/get-started/getting-started-with-git/about-remote-repositories#cloning-with-https-urls for information on currently recommended modes of authentication.
fatal: Authentication failed for 'https://github.com/grerrg/dotfiles.git/'


osd: test: Add script to test LOST state

Signed-off-by: Colin McCabe <c0eabe2887ac1e91c280795b9fca010e21066283@hq.newdream.net>

#!/bin/bash -x

#
# Simple test of recovery logic
#

# Constants
MAX_OBJS=100
OBJ_SIZE=1000000
TEMPDIR=`mktemp -d`
SDIR=`dirname $0`/..

# Initialization
cd $SDIR
[ -e "/dev/urandom" ] || die "need /dev/urandom"
trap cleanup INT TERM EXIT
rm -rf $TEMPDIR
mkdir -p $TEMPDIR || die "failed to make tempdir"

# Functions
cleanup() {
        rm -rf "${TEMPDIR}"
}

die() {
        echo $@
        exit 1
}

stop_osd() {
        osd_index=$1
        pidfile="out/osd.$osd_index.pid"
        if [ -e $pidfile ]; then
                kill `cat $pidfile` && return 0
        else
                echo "cosd process $osd_index is not running"
        fi
        return 1
}

restart_osd() {
        osd_index=$1
        ./cosd -i $osd_index -c ceph.conf &
}

write_objects() {
        start_ver=$1
        stop_ver=$2
        for v in `seq $start_ver $stop_ver`; do
                head -c $OBJ_SIZE /dev/zero  | tr '\0' '$v' > $TEMPDIR/ver$v
                for i in `seq -w 1 $MAX_OBJS`; do
                        ./rados -p data put obj$i $TEMPDIR/ver$v || die "radostool failed"
                done
        done
}

setup() {
        # Start ceph
        ./stop.sh

        # TODO: somehow constrain the number of PGs created by vstart.sh so that
        # situations are more reproducible
        CEPH_NUM_OSD=2 ./vstart.sh -d -n || die "vstart failed"
}

do_test() {
        # Write lots and lots of objects
        write_objects 1 3

        # Take down osd1
        stop_osd 1

        # Continue writing a lot of objects
        write_objects 4 6

        # Bring up osd1
        restart_osd 1

        # Give recovery some time to start
        sleep 30

        # Stop osd0 before recovery can complete
        stop_osd 0

        # Now we should be in LOST_REVERT
}

run() {
        setup

        do_test
}

$@

error: pathspec 'src/test/test_lost.sh' did not match any file(s) known to git


Added build script for Solaris 11 11/11.

remote: Support for password authentication was removed on August 13, 2021.
remote: Please see https://docs.github.com/en/get-started/getting-started-with-git/about-remote-repositories#cloning-with-https-urls for information on currently recommended modes of authentication.
fatal: Authentication failed for 'https://github.com/HongjianLi/igrow.git/'


add script to install pre-commit hook locally

#!/bin/sh

FORMATTER_DIR="$(dirname $0)"
GIT_HOOK_DIR="$FORMATTER_DIR/../.git/hooks"
cp $FORMATTER_DIR/pre-commit $GIT_HOOK_DIR/pre-commit
chmod +x $GIT_HOOK_DIR/pre-commit
echo "pre-commit hook installed in $GIT_HOOK_DIR/pre-commit."

error: pathspec 'formatter/install-pre-commit-hook.sh' did not match any file(s) known to git


Added runner for tests under mpi: unittest/mpitest.sh

#!/bin/bash
for SAMPLER in abc demcz dream fast fscabc lhs mc mcmc mle rope sa # sceua
do
    mpirun -c 6 python spotpy/examples/cli_cmf_lumped.py run -s $SAMPLER -n 10000 -p mpi
done

error: pathspec 'spotpy/unittests/mpitest.sh' did not match any file(s) known to git


copy PNGs from parent directory for the chunked user guide

git-svn-id: 6ee47c8665bf46900d9baa0afb711c5b4a0eafac@2452 e2bb083e-7f23-0410-b3a8-8253ac9ef6d8

#!/bin/sh

mkdir -p userguide/ || exit 1
cd userguide/ || exit 2
rm -f *.html *.php
cp ../*.png .

xsltproc --nonet ../formatting/swiftsh_html_chunked.xsl ../userguide.xml

#!/bin/sh

mkdir -p userguide/ || exit 1
cd userguide/ || exit 2
rm -f *.html *.php

xsltproc --nonet ../formatting/swiftsh_html_chunked.xsl ../userguide.xml


Fixed installation path issue

Fixed installation path issue

remote: Support for password authentication was removed on August 13, 2021.
remote: Please see https://docs.github.com/en/get-started/getting-started-with-git/about-remote-repositories#cloning-with-https-urls for information on currently recommended modes of authentication.
fatal: Authentication failed for 'https://github.com/CanTireInnovations/arduino-studio-builder.git/'


Added code to select the subset of services that the service image is supposed to support

git-svn-id: 2deb3340141ac69a26c0f89cf6cbd56f19c97dea@4411 ef35cc4a-2640-4bbf-98bc-65920db7d8ae

#!/bin/bash -e
# Copyright (c) 2010-2012, Contrail consortium.
# All rights reserved.
#
# Redistribution and use in source and binary forms,
# with or without modification, are permitted provided
# that the following conditions are met:
#
#  1. Redistributions of source code must retain the
#     above copyright notice, this list of conditions
#     and the following disclaimer.
#  2. Redistributions in binary form must reproduce
#     the above copyright notice, this list of
#     conditions and the following disclaimer in the
#     documentation and/or other materials provided
#     with the distribution.
#  3. Neither the name of the Contrail consortium nor the
#     names of its contributors may be used to endorse
#     or promote products derived from this software
#     without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND
# CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES,
# INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
# MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR
# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
# BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
# WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
# OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.

##
# This script generates a VM image for ConPaaS, to be used for OpenNebula with KVM.
# The script should be run on a Debian or Ubuntu system.
# Before running this script, please make sure that you have the following
# executables in your $PATH:
#
# dd parted losetup kpartx mkfs.ext3 tune2fs mount debootstrap chroot umount grub-install (grub2)
#
# NOTE: This script requires the installation of Grub 2 (we recommend Grub 1.98 or newer,
# as we experienced problems with 1.96).
##

##### TO CUSTOMIZE: #####

# The name and size of the image file that will be generated.
FILENAME=conpaas.img
FILESIZE=2048 #MB

# The Debian distribution that you would like to have installed (we recommend squeeze).
DEBIAN_DIST=squeeze
DEBIAN_MIRROR=http://ftp.nl.debian.org/debian

# The architecture and kernel version for the OS that will be installed (please make
# sure to modify the kernel version name accordingly if you modify the architecture).
ARCH=amd64
KERNEL_VERSION=2.6.32-5-amd64

# Services that will be installed:
PHP_SERVICE=true
MYSQL_SERVICE=true
CONDOR_SERVICE=true
IPOP_SERVICE=true
GIT_SERVICE=true
SELENIUM_SERVICE=true
HADOOP_SERVICE=true
SCALARIS_SERVICE=true
XTREEMFS_SERVICE=true
CDS_SERVICE=true

# override above values with those found in config file, if present
[ -f services_config.cfg ] && . services_config.cfg

#########################
export LC_ALL=C

# Function for displaying highlighted messages.
function cecho() {
  echo -en "\033[1m"
  echo -n "#" $@
  echo -e "\033[0m"
}

# Set up message on purpose before root permission check
cecho "Setting up for these services:"
for i in PHP MYSQL CONDOR IPOP GIT SELENIUM HADOOP SCALARIS XTREEMFS CDS
do
	name=`echo $i`_SERVICE
	eval serv=\${$name}
	$serv && cecho "-" "$i"
done

if [ `id -u` -ne 0 ]; then
  cecho 'need root permissions for this script';
  exit 1;
fi

# System rollback function
function cleanup() {
    # Set errormsg if something went wrong
    [ $? -ne 0 ] && errormsg="Script terminated with errors"

    for mpoint in /dev/pts /dev /proc /
    do
      mpoint="${ROOT_DIR?:not set}${mpoint}"

      # Only attempt to umount $ROOT_DIR{/dev/pts,/dev,/proc,/} if necessary
      if [ -d $mpoint ]
      then
        cecho "Umounting $mpoint"
        umount $mpoint || true
      fi
    done

    sleep 1s
    losetup -d $LOOP_DEV_P
    sleep 1s
    kpartx -d $LOOP_DEV
    sleep 1s
    losetup -d $LOOP_DEV
    sleep 1s
    rm -r $ROOT_DIR
    # Print "Done" on success, $errormsg otherwise
    cecho "${errormsg:-Done}"
}

# Check if required binaries are in $PATH
for bin in dd parted losetup kpartx mkfs.ext3 tune2fs mount debootstrap chroot umount grub-install
do
  if [ -z `which $bin` ]
  then
    if [ -x /usr/lib/command-not-found ]
    then
      /usr/lib/command-not-found $bin
    else
      echo "$bin: command not found"
    fi
    exit 1
  fi
done

cecho "Creating empty disk image at" $FILENAME
dd if=/dev/zero of=$FILENAME bs=1M count=$FILESIZE

cecho "Writing partition table"
parted -s $FILENAME mklabel msdos

cecho "Creating primary partition"
cyl_total=`parted -s $FILENAME unit s print | awk '{if (NF > 2 && $1 == "Disk") print $0}' | sed 's/Disk .* \([0-9]\+\)s/\1/'`
cyl_partition=`expr $cyl_total - 2048`
parted -s $FILENAME unit s mkpart primary 2048 $cyl_partition

cecho "Setting boot flag"
parted -s $FILENAME set 1 boot on

LOOP_DEV=`losetup -f`
cecho "Going to use" $LOOP_DEV
losetup $LOOP_DEV $FILENAME

dname=`kpartx -l $LOOP_DEV | awk '{print $1}'`
PART_DEV=/dev/mapper/$dname
cecho "Mapping partition to device"
kpartx -a $LOOP_DEV

cecho "Creating ext3 filesystem"
echo 'y' | mkfs.ext3 $PART_DEV
cecho "Setting label 'root'"
tune2fs $PART_DEV -L root

ROOT_DIR=`mktemp -d`
cecho "Using $ROOT_DIR as mount point"

cecho "Mounting disk image"

LOOP_DEV_P=`losetup -f`
losetup $LOOP_DEV_P $PART_DEV
mount $LOOP_DEV_P $ROOT_DIR
#mount $PART_DEV $ROOT_DIR

# Always clean up on exit
trap "cleanup" EXIT

cecho "Starting debootstrap"
debootstrap --arch $ARCH --include locales $DEBIAN_DIST $ROOT_DIR $DEBIAN_MIRROR

cecho "Writing fstab"
echo "/dev/sda1 / ext3 defaults 0 1" > $ROOT_DIR/etc/fstab
cecho "Writing /etc/network/interfaces"
cat <<EOF > $ROOT_DIR/etc/network/interfaces
auto lo
iface lo inet loopback
auto eth0
iface eth0 inet static
EOF
cecho "Removing udev persistent rules"
rm $ROOT_DIR/etc/udev/rules.d/70-persistent* || true

cecho "Changing hostname"
cat <<EOF > $ROOT_DIR/etc/hostname
conpaas
EOF

sed -i '1i 127.0.0.1  conpaas' $ROOT_DIR/etc/hosts

# mount /dev/pts to avoid error message: Can not write log, openpty() failed (/dev/pts not mounted?)
cecho "Mounting /dev, /dev/pts and /proc in chroot"
mount -obind /dev $ROOT_DIR/dev
mount -obind /dev/pts $ROOT_DIR/dev/pts
mount -t proc proc $ROOT_DIR/proc

cecho "Setting keyboard layout"
chroot $ROOT_DIR /bin/bash -c "echo 'debconf keyboard-configuration/variant  select  USA' | debconf-set-selections"

cecho "Generating and setting locale"
chroot $ROOT_DIR /bin/bash -c "sed --in-place 's/^# en_US.UTF-8/en_US.UTF-8/' /etc/locale.gen"
chroot $ROOT_DIR /bin/bash -c 'locale-gen'
chroot $ROOT_DIR /bin/bash -c 'update-locale LANG=en_US.UTF-8'

cecho "Running apt-get update"
chroot $ROOT_DIR /bin/bash -c 'apt-get -y update'
cecho "Installing linux-image-$KERNEL_VERSION"
chroot $ROOT_DIR /bin/bash -c "apt-get -y install linux-image-$KERNEL_VERSION"
cecho "Installing grub package"
chroot $ROOT_DIR /bin/bash -c 'DEBIAN_FRONTEND=noninteractive apt-get -y --force-yes install grub'

mkdir -p $ROOT_DIR/boot/grub
cat <<EOF > $ROOT_DIR/boot/grub/device.map
(hd0)   $LOOP_DEV
(hd0,1) $LOOP_DEV_P
EOF

chroot $ROOT_DIR grub-mkconfig -o /boot/grub/grub.cfg

cecho "Writing /boot/grub/grub.cfg"
cat <<EOF > $ROOT_DIR/boot/grub/grub.cfg
set default=0
set timeout=0
menuentry 'linux-image-$KERNEL_VERSION' {
  insmod ext2
  set root='(hd0,1)'
  linux  /boot/vmlinuz-$KERNEL_VERSION root=/dev/sda1
  initrd /boot/initrd.img-$KERNEL_VERSION
}
EOF

cecho "Running grub-install"
grub-install --no-floppy --grub-mkdevicemap=$ROOT_DIR/boot/grub/device.map --root-directory=$ROOT_DIR $LOOP_DEV

# disable auto start after package install
cat <<EOF > $ROOT_DIR/usr/sbin/policy-rc.d
#!/bin/sh
exit 101
EOF
chmod 755 $ROOT_DIR/usr/sbin/policy-rc.d

# Generate a script that will install the dependencies in the system.
cat <<EOF > $ROOT_DIR/conpaas_install
#!/bin/bash
# Function for displaying highlighted messages.
function cecho() {
  echo -en "\033[1m"
  echo -n "#" \$@
  echo -e "\033[0m"
}

# set root passwd
echo "root:contrail" | chpasswd

# fix apt sources
sed --in-place 's/main/main contrib non-free/' /etc/apt/sources.list

# install dependencies
apt-get -f -y update
# pre-accept sun-java6 licence
echo "debconf shared/accepted-sun-dlj-v1-1 boolean true" | debconf-set-selections
DEBIAN_FRONTEND=noninteractive apt-get -y --force-yes --no-install-recommends --no-upgrade \
        install openssh-server \
        python python-pycurl python-openssl python-m2crypto \
        python-mysqldb python-cheetah nginx \
        tomcat6-user memcached mysql-server \
        make gcc g++ sun-java6-jdk erlang ant libxslt1-dev yaws subversion git \
        xvfb xinit unzip
update-rc.d memcached disable
update-rc.d nginx disable
update-rc.d yaws disable
update-rc.d mysql disable

# create directory structure
echo > /var/log/cpsagent.log
mkdir /etc/cpsagent/
mkdir /var/tmp/cpsagent/
mkdir /var/run/cpsagent/
mkdir /var/cache/cpsagent/
echo > /var/log/cpsmanager.log
mkdir /etc/cpsmanager/
mkdir /var/tmp/cpsmanager/
mkdir /var/run/cpsmanager/
mkdir /var/cache/cpsmanager/

EOF

!($PHP_SERVICE || $MYSQL_SERVICE) && echo 'cecho "===== Skipped PHP & MYSQL ====="' >> $ROOT_DIR/conpaas_install
($PHP_SERVICE || $MYSQL_SERVICE) && cat <<EOF >> $ROOT_DIR/conpaas_install
cecho "===== add dotdeb repo for php fpm ====="
# add dotdeb repo for php fpm
echo "deb http://packages.dotdeb.org stable all" >> /etc/apt/sources.list
wget -O - http://www.dotdeb.org/dotdeb.gpg 2>/dev/null | apt-key add -
apt-get -f -y update
apt-get -f -y --no-install-recommends --no-upgrade install php5-fpm php5-curl \
              php5-mcrypt php5-mysql php5-odbc \
              php5-pgsql php5-sqlite php5-sybase php5-xmlrpc php5-xsl \
              php5-adodb php5-memcache php5-gd
update-rc.d php5-fpm disable

# remove dotdeb repo
sed --in-place 's%deb http://packages.dotdeb.org stable all%%' /etc/apt/sources.list
apt-get -f -y update

# remove cached .debs from /var/cache/apt/archives to save disk space
apt-get clean
EOF

$CONDOR_SERVICE || echo 'cecho "===== Skipped CONDOR ====="' >> $ROOT_DIR/conpaas_install
$CONDOR_SERVICE && cat <<EOF >> $ROOT_DIR/conpaas_install
cecho "===== install HTCondor package ====="
mkdir -p /var/lib/condor
chown condor /var/lib/condor
chgrp condor /var/lib/condor
chmod 766 /var/lib/condor
# avoid warning: W: GPG error: http://mozilla.debian.net squeeze-backports Release: The following signatures couldn't be verified because the public key is not available: NO_PUBKEY 85A3D26506C4AE2A
#apt-get install debian-keyring
wget -O - -q http://mozilla.debian.net/archive.asc | apt-key add -
# avoid warning: W: GPG error: http://dl.google.com stable Release: The following signatures couldn't be verified because the public key is not available: NO_PUBKEY A040830F7FAC5991
wget -q -O - https://dl-ssl.google.com/linux/linux_signing_key.pub | apt-key add -

# If things go wrong, you may want to read  http://research.cs.wisc.edu/htcondor/debian/
#
echo "deb http://research.cs.wisc.edu/htcondor/debian/stable/ $DEBIAN_DIST contrib" >> /etc/apt/sources.list
apt-get update
apt-get -f -y --force-yes install condor
echo ===== check if HTCondor is running =====
ps -ef | grep condor
echo ===== stop condor =====
/etc/init.d/condor stop
echo =====

# remove cached .debs from /var/cache/apt/archives to save disk space
apt-get clean
EOF

$IPOP_SERVICE || echo 'cecho "===== Skipped IPOP ====="' >> $ROOT_DIR/conpaas_install
$IPOP_SERVICE && cat <<EOF >> $ROOT_DIR/conpaas_install
cecho "===== install IPOP package ====="
echo "deb http://www.grid-appliance.org/files/packages/deb/ stable contrib" >> /etc/apt/sources.list
wget -O - http://www.grid-appliance.org/files/packages/deb/repo.key | apt-key add -
apt-get update
apt-get -f -y install ipop

# remove cached .debs from /var/cache/apt/archives to save disk space
apt-get clean

EOF

$GIT_SERVICE || echo 'cecho "===== Skipped GIT ====="' >> $ROOT_DIR/conpaas_install
$GIT_SERVICE && cat <<EOF >> $ROOT_DIR/conpaas_install
cecho "===== install GIT ====="
# add git user
useradd git --shell /usr/bin/git-shell --create-home -k /dev/null
# create ~git/.ssh and authorized_keys
install -d -m 700 --owner=git --group=git /home/git/.ssh
install -m 600 --owner=git --group=git /dev/null ~git/.ssh/authorized_keys
# create default repository
git init --bare ~git/code
# create SSH key for manager -> agent access
ssh-keygen -N "" -f ~root/.ssh/id_rsa
echo StrictHostKeyChecking no > ~root/.ssh/config
# allow manager -> agent passwordless pushes
cat ~root/.ssh/id_rsa.pub > ~git/.ssh/authorized_keys
# fix repository permissions
chown -R git:git ~git/code

EOF

$SELENIUM_SERVICE || echo 'cecho "===== Skipped SELENIUM ====="' >> $ROOT_DIR/conpaas_install
$SELENIUM_SERVICE && cat <<EOF >> $ROOT_DIR/conpaas_install
cecho "===== install SELENIUM ====="
# recent versions of iceweasel and chrome
echo "deb http://backports.debian.org/debian-backports squeeze-backports main" >> /etc/apt/sources.list
echo "deb http://mozilla.debian.net/ squeeze-backports iceweasel-esr" >> /etc/apt/sources.list
echo "deb http://dl.google.com/linux/deb/ stable main" >> /etc/apt/sources.list

apt-get -f -y update
apt-get -f -y --force-yes install -t squeeze-backports iceweasel
apt-get -f -y --force-yes install google-chrome-beta

EOF

$HADOOP_SERVICE || echo 'cecho "===== Skipped HADOOP ====="' >> $ROOT_DIR/conpaas_install
$HADOOP_SERVICE && cat <<EOF >> $ROOT_DIR/conpaas_install
cecho "===== install cloudera repo for hadoop ====="
# add cloudera repo for hadoop
echo "deb http://archive.cloudera.com/debian $DEBIAN_DIST-cdh3 contrib" >> /etc/apt/sources.list
wget -O - http://archive.cloudera.com/debian/archive.key 2>/dev/null | apt-key add -
apt-get -f -y update
apt-get -f -y --no-install-recommends --no-upgrade install \
  hadoop-0.20 hadoop-0.20-namenode hadoop-0.20-datanode \
  hadoop-0.20-secondarynamenode hadoop-0.20-jobtracker  \
  hadoop-0.20-tasktracker hadoop-pig hue-common  hue-filebrowser \
  hue-jobbrowser hue-jobsub hue-plugins hue-server dnsutils
update-rc.d hadoop-0.20-namenode disable
update-rc.d hadoop-0.20-datanode disable
update-rc.d hadoop-0.20-secondarynamenode disable
update-rc.d hadoop-0.20-jobtracker disable
update-rc.d hadoop-0.20-tasktracker disable
update-rc.d hue disable
# create a default config dir
mkdir -p /etc/hadoop-0.20/conf.contrail
update-alternatives --install /etc/hadoop-0.20/conf hadoop-0.20-conf /etc/hadoop-0.20/conf.contrail 99
# remove cloudera repo
sed --in-place "s%deb http://archive.cloudera.com/debian $DEBIAN_DIST-cdh3 contrib%%" /etc/apt/sources.list
apt-get -f -y update

EOF

$SCALARIS_SERVICE || echo 'cecho "===== Skipped SCALARIS ====="' >> $ROOT_DIR/conpaas_install
$SCALARIS_SERVICE && cat <<EOF >> $ROOT_DIR/conpaas_install
cecho "===== install scalaris repo ====="
# add scalaris repo
echo "deb http://download.opensuse.org/repositories/home:/scalaris/Debian_6.0 /" >> /etc/apt/sources.list
wget -O - http://download.opensuse.org/repositories/home:/scalaris/Debian_6.0/Release.key 2>/dev/null | apt-key add -
apt-get -f -y update
apt-get -f -y --no-install-recommends --no-upgrade install scalaris screen
update-rc.d scalaris disable
# remove scalaris repo
sed --in-place 's%deb http://download.opensuse.org/repositories/home:/scalaris/Debian_6.0 /%%' /etc/apt/sources.list
apt-get -f -y update

EOF

$XTREEMFS_SERVICE || echo 'cecho "===== Skipped XTREEMFS ====="' >> $ROOT_DIR/conpaas_install
$XTREEMFS_SERVICE && cat <<EOF >> $ROOT_DIR/conpaas_install
cecho "===== install xtreemfs repo ====="
# add xtreemfs repo
echo "deb http://download.opensuse.org/repositories/home:/xtreemfs:/unstable/Debian_6.0 /" >> /etc/apt/sources.list
wget -O - http://download.opensuse.org/repositories/home:/xtreemfs:/unstable/Debian_6.0/Release.key 2>/dev/null | apt-key add -
apt-get -f -y update
apt-get -f -y --no-install-recommends --no-upgrade install xtreemfs-server xtreemfs-client
update-rc.d xtreemfs-osd disable
update-rc.d xtreemfs-mrc disable
update-rc.d xtreemfs-dir disable
# remove xtreemfs repo
sed --in-place 's%deb http://download.opensuse.org/repositories/home:/xtreemfs:/unstable/Debian_6.0 /%%' /etc/apt/sources.list
apt-get -f -y update

EOF

$CDS_SERVICE || echo 'cecho "===== Skipped CDS ====="' >> $ROOT_DIR/conpaas_install
$CDS_SERVICE && cat <<EOF >> $ROOT_DIR/conpaas_install
cecho "===== install latest nginx (1.2.2) and other packages required by CDS ====="
# install latest nginx (1.2.2) and other packages required by CDS
DEBIAN_FRONTEND=noninteractive apt-get -y --force-yes --no-install-recommends --no-upgrade \
    install libpcre3-dev libssl-dev libgeoip-dev libperl-dev
wget http://nginx.org/download/nginx-1.2.2.tar.gz
tar xzf nginx-1.2.2.tar.gz
cd nginx-1.2.2
./configure --sbin-path=/usr/sbin/nginx --conf-path=/etc/nginx/nginx.conf --error-log-path=/var/log/nginx/error.log --http-client-body-temp-path=/var/lib/nginx/body --http-fastcgi-temp-path=/var/lib/nginx/fastcgi --http-log-path=/var/log/nginx/access.log --http-proxy-temp-path=/var/lib/nginx/proxy --lock-path=/var/lock/nginx.lock --pid-path=/var/run/nginx.pid --with-debug --with-http_dav_module --with-http_flv_module --with-http_geoip_module --with-http_gzip_static_module --with-http_realip_module --with-http_stub_status_module --with-http_ssl_module --with-http_sub_module --with-ipv6 --with-mail --with-mail_ssl_module --with-http_perl_module
make
make install
cd ..
rm -rf nginx-1.2.2*

EOF

cat <<EOF >> $ROOT_DIR/conpaas_install
apt-get -f -y clean
exit 0
EOF

# Execute the script for installing the dependencies.
chmod a+x $ROOT_DIR/conpaas_install
chroot $ROOT_DIR /bin/bash /conpaas_install
rm -f $ROOT_DIR/conpaas_install

rm -f $ROOT_DIR/usr/sbin/policy-rc.d

##### TO CUSTOMIZE: #####
# This part is for OpenNebula contextualization. The contextualization scripts (and possibly
# other necessary files) will be provided through OpenNebula to the VM as an ISO image.
# We need to mount this image and execute the contextualization scripts. You might need
# to change the dev file associated with the CD-ROM inside your virtual machine from
# "/dev/sr0" to something else (depending on your operating system and on the virtualization
# software, it can be /dev/hdb, /dev/sdb etc.). You can check this in a VM that is already running
# in your OpenNebula system and that has been configured with contextualization.

cat <<"EOF" > $ROOT_DIR/etc/rc.local
#!/bin/sh
mount -t iso9660 /dev/sr0 /mnt

if [ -f /mnt/context.sh ]; then
  . /mnt/context.sh
  if [ -n "$USERDATA" ]; then
    echo "$USERDATA" | /usr/bin/xxd -p -r | /bin/sh
  elif [ -e /mnt/init.sh ]; then
    . /mnt/init.sh
  fi
fi

umount /mnt

exit 0
EOF

#!/bin/bash -e
# Copyright (c) 2010-2012, Contrail consortium.
# All rights reserved.
#
# Redistribution and use in source and binary forms,
# with or without modification, are permitted provided
# that the following conditions are met:
#
#  1. Redistributions of source code must retain the
#     above copyright notice, this list of conditions
#     and the following disclaimer.
#  2. Redistributions in binary form must reproduce
#     the above copyright notice, this list of
#     conditions and the following disclaimer in the
#     documentation and/or other materials provided
#     with the distribution.
#  3. Neither the name of the Contrail consortium nor the
#     names of its contributors may be used to endorse
#     or promote products derived from this software
#     without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND
# CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES,
# INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
# MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR
# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
# BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
# WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
# OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.

##
# This script generates a VM image for ConPaaS, to be used for OpenNebula with KVM.
# The script should be run on a Debian or Ubuntu system.
# Before running this script, please make sure that you have the following
# executables in your $PATH:
#
# dd parted losetup kpartx mkfs.ext3 tune2fs mount debootstrap chroot umount grub-install (grub2)
#
# NOTE: This script requires the installation of Grub 2 (we recommend Grub 1.98 or newer,
# as we experienced problems with 1.96).
##

##### TO CUSTOMIZE: #####

# The name and size of the image file that will be generated.
FILENAME=conpaas.img
FILESIZE=2048 #MB

# The Debian distribution that you would like to have installed (we recommend squeeze).
DEBIAN_DIST=squeeze
DEBIAN_MIRROR=http://ftp.nl.debian.org/debian

# The architecture and kernel version for the OS that will be installed (please make
# sure to modify the kernel version name accordingly if you modify the architecture).
ARCH=i386
KERNEL_VERSION=2.6.32-5-686

#########################
export LC_ALL=C

# Function for displaying highlighted messages.
function cecho() {
  echo -en "\033[1m"
  echo -n "#" $@
  echo -e "\033[0m"
}

if [ `id -u` -ne 0 ]; then
  cecho 'need root permissions for this script';
  exit 1;
fi

# System rollback function
function cleanup() {
    # Set errormsg if something went wrong
    [ $? -ne 0 ] && errormsg="Script terminated with errors"

    for mpoint in /dev /proc /
    do
      mpoint="${ROOT_DIR?:not set}${mpoint}"

      # Only attempt to umount $ROOT_DIR{/dev,/proc,/} if necessary
      if [ -d $mpoint ]
      then
        cecho "Umounting $mpoint"
        umount $mpoint || true
      fi
    done

    sleep 1s
    losetup -d $LOOP_DEV_P
    sleep 1s
    kpartx -d $LOOP_DEV
    sleep 1s
    losetup -d $LOOP_DEV
    sleep 1s
    rm -r $ROOT_DIR
    # Print "Done" on success, $errormsg otherwise
    cecho "${errormsg:-Done}"
}

# Check if required binaries are in $PATH
for bin in dd parted losetup kpartx mkfs.ext3 tune2fs mount debootstrap chroot umount grub-install
do
  if [ -z `which $bin` ]
  then
    if [ -x /usr/lib/command-not-found ]
    then
      /usr/lib/command-not-found $bin
    else
      echo "$bin: command not found"
    fi
    exit 1
  fi
done

cecho "Creating empty disk image at" $FILENAME
dd if=/dev/zero of=$FILENAME bs=1M count=$FILESIZE

cecho "Writing partition table"
parted -s $FILENAME mklabel msdos

cecho "Creating primary partition"
cyl_total=`parted -s $FILENAME unit s print | awk '{if (NF > 2 && $1 == "Disk") print $0}' | sed 's/Disk .* \([0-9]\+\)s/\1/'`
cyl_partition=`expr $cyl_total - 2048`
parted -s $FILENAME unit s mkpart primary 2048 $cyl_partition

cecho "Setting boot flag"
parted -s $FILENAME set 1 boot on

LOOP_DEV=`losetup -f`
cecho "Going to use" $LOOP_DEV
losetup $LOOP_DEV $FILENAME

dname=`kpartx -l $LOOP_DEV | awk '{print $1}'`
PART_DEV=/dev/mapper/$dname
cecho "Mapping partition to device"
kpartx -a $LOOP_DEV

cecho "Creating ext3 filesystem"
echo 'y' | mkfs.ext3 $PART_DEV
cecho "Setting label 'root'"
tune2fs $PART_DEV -L root

ROOT_DIR=`mktemp -d`
cecho "Using $ROOT_DIR as mount point"

cecho "Mounting disk image"

LOOP_DEV_P=`losetup -f`
losetup $LOOP_DEV_P $PART_DEV
mount $LOOP_DEV_P $ROOT_DIR
#mount $PART_DEV $ROOT_DIR

# Always clean up on exit
trap "cleanup" EXIT

cecho "Starting debootstrap"
debootstrap --arch $ARCH --include locales $DEBIAN_DIST $ROOT_DIR $DEBIAN_MIRROR

cecho "Writing fstab"
echo "/dev/sda1 / ext3 defaults 0 1" > $ROOT_DIR/etc/fstab
cecho "Writing /etc/network/interfaces"
cat <<EOF > $ROOT_DIR/etc/network/interfaces
auto lo
iface lo inet loopback
auto eth0
iface eth0 inet static
EOF
cecho "Removing udev persistent rules"
rm $ROOT_DIR/etc/udev/rules.d/70-persistent* || true

cecho "Changing hostname"
cat <<EOF > $ROOT_DIR/etc/hostname
conpaas
EOF

sed -i '1i 127.0.0.1  conpaas' $ROOT_DIR/etc/hosts

cecho "Mounting /dev and /proc in chroot"
mount -obind /dev $ROOT_DIR/dev
mount -t proc proc $ROOT_DIR/proc

cecho "Setting keyboard layout"
chroot $ROOT_DIR /bin/bash -c "echo 'debconf keyboard-configuration/variant  select  USA' | debconf-set-selections"

cecho "Generating and setting locale"
chroot $ROOT_DIR /bin/bash -c "sed --in-place 's/^# en_US.UTF-8/en_US.UTF-8/' /etc/locale.gen"
chroot $ROOT_DIR /bin/bash -c 'locale-gen'
chroot $ROOT_DIR /bin/bash -c 'update-locale LANG=en_US.UTF-8'

cecho "Running apt-get update"
chroot $ROOT_DIR /bin/bash -c 'apt-get -y update'
cecho "Installing linux-image-$KERNEL_VERSION"
chroot $ROOT_DIR /bin/bash -c "apt-get -y install linux-image-$KERNEL_VERSION"
cecho "Installing grub package"
chroot $ROOT_DIR /bin/bash -c 'DEBIAN_FRONTEND=noninteractive apt-get -y --force-yes install grub'

mkdir -p $ROOT_DIR/boot/grub
cat <<EOF > $ROOT_DIR/boot/grub/device.map
(hd0)   $LOOP_DEV
(hd0,1) $LOOP_DEV_P
EOF

chroot $ROOT_DIR grub-mkconfig -o /boot/grub/grub.cfg

cecho "Writing /boot/grub/grub.cfg"
cat <<EOF > $ROOT_DIR/boot/grub/grub.cfg
set default=0
set timeout=0
menuentry 'linux-image-$KERNEL_VERSION' {
  insmod ext2
  set root='(hd0,1)'
  linux  /boot/vmlinuz-$KERNEL_VERSION root=/dev/sda1
  initrd /boot/initrd.img-$KERNEL_VERSION
}
EOF

cecho "Running grub-install"
grub-install --no-floppy --grub-mkdevicemap=$ROOT_DIR/boot/grub/device.map --root-directory=$ROOT_DIR $LOOP_DEV

# disable auto start after package install
cat <<EOF > $ROOT_DIR/usr/sbin/policy-rc.d
#!/bin/sh
exit 101
EOF
chmod 755 $ROOT_DIR/usr/sbin/policy-rc.d

# Generate a script that will install the dependencies in the system.
cat <<EOF > $ROOT_DIR/conpaas_install
#!/bin/bash
# set root passwd
echo "root:contrail" | chpasswd

# fix apt sources
sed --in-place 's/main/main contrib non-free/' /etc/apt/sources.list

# install dependencies
apt-get -f -y update
# pre-accept sun-java6 licence
echo "debconf shared/accepted-sun-dlj-v1-1 boolean true" | debconf-set-selections
DEBIAN_FRONTEND=noninteractive apt-get -y --force-yes --no-install-recommends --no-upgrade \
        install openssh-server \
        python python-pycurl python-openssl python-m2crypto \
        python-mysqldb python-cheetah nginx \
        tomcat6-user memcached mysql-server \
        make gcc g++ sun-java6-jdk erlang ant libxslt1-dev yaws subversion git \
        xvfb xinit unzip
update-rc.d memcached disable
update-rc.d nginx disable
update-rc.d yaws disable
update-rc.d mysql disable

# add dotdeb repo for php fpm
echo "deb http://packages.dotdeb.org stable all" >> /etc/apt/sources.list
wget -O - http://www.dotdeb.org/dotdeb.gpg 2>/dev/null | apt-key add -
apt-get -f -y update
apt-get -f -y --no-install-recommends --no-upgrade install php5-fpm php5-curl \
              php5-mcrypt php5-mysql php5-odbc \
              php5-pgsql php5-sqlite php5-sybase php5-xmlrpc php5-xsl \
              php5-adodb php5-memcache php5-gd
update-rc.d php5-fpm disable

# remove dotdeb repo
sed --in-place 's%deb http://packages.dotdeb.org stable all%%' /etc/apt/sources.list
apt-get -f -y update

echo "deb http://www.grid-appliance.org/files/packages/deb/ stable contrib" >> /etc/apt/sources.list
wget -O - http://www.grid-appliance.org/files/packages/deb/repo.key | apt-key add -
apt-get update
apt-get -f -y install ipop

# remove cached .debs from /var/cache/apt/archives to save disk space
apt-get clean

# create directory structure
echo > /var/log/cpsagent.log
mkdir /etc/cpsagent/
mkdir /var/tmp/cpsagent/
mkdir /var/run/cpsagent/
mkdir /var/cache/cpsagent/
echo > /var/log/cpsmanager.log
mkdir /etc/cpsmanager/
mkdir /var/tmp/cpsmanager/
mkdir /var/run/cpsmanager/
mkdir /var/cache/cpsmanager/

# add git user
useradd git --shell /usr/bin/git-shell --create-home -k /dev/null
# create ~git/.ssh and authorized_keys
install -d -m 700 --owner=git --group=git /home/git/.ssh
install -m 600 --owner=git --group=git /dev/null ~git/.ssh/authorized_keys
# create default repository
git init --bare ~git/code
# create SSH key for manager -> agent access
ssh-keygen -N "" -f ~root/.ssh/id_rsa
echo StrictHostKeyChecking no > ~root/.ssh/config
# allow manager -> agent passwordless pushes
cat ~root/.ssh/id_rsa.pub > ~git/.ssh/authorized_keys
# fix repository permissions
chown -R git:git ~git/code

# recent versions of iceweasel and chrome
echo "deb http://backports.debian.org/debian-backports squeeze-backports main" >> /etc/apt/sources.list
echo "deb http://mozilla.debian.net/ squeeze-backports iceweasel-esr" >> /etc/apt/sources.list
echo "deb http://dl.google.com/linux/deb/ stable main" >> /etc/apt/sources.list

apt-get -f -y update
apt-get -f -y --force-yes install -t squeeze-backports iceweasel
apt-get -f -y --force-yes install google-chrome-beta

# add cloudera repo for hadoop
echo "deb http://archive.cloudera.com/debian $DEBIAN_DIST-cdh3 contrib" >> /etc/apt/sources.list
wget -O - http://archive.cloudera.com/debian/archive.key 2>/dev/null | apt-key add -
apt-get -f -y update
apt-get -f -y --no-install-recommends --no-upgrade install \
  hadoop-0.20 hadoop-0.20-namenode hadoop-0.20-datanode \
  hadoop-0.20-secondarynamenode hadoop-0.20-jobtracker  \
  hadoop-0.20-tasktracker hadoop-pig hue-common  hue-filebrowser \
  hue-jobbrowser hue-jobsub hue-plugins hue-server dnsutils
update-rc.d hadoop-0.20-namenode disable
update-rc.d hadoop-0.20-datanode disable
update-rc.d hadoop-0.20-secondarynamenode disable
update-rc.d hadoop-0.20-jobtracker disable
update-rc.d hadoop-0.20-tasktracker disable
update-rc.d hue disable
# create a default config dir
mkdir -p /etc/hadoop-0.20/conf.contrail
update-alternatives --install /etc/hadoop-0.20/conf hadoop-0.20-conf /etc/hadoop-0.20/conf.contrail 99
# remove cloudera repo
sed --in-place "s%deb http://archive.cloudera.com/debian $DEBIAN_DIST-cdh3 contrib%%" /etc/apt/sources.list
apt-get -f -y update

# add scalaris repo
echo "deb http://download.opensuse.org/repositories/home:/scalaris/Debian_6.0 /" >> /etc/apt/sources.list
wget -O - http://download.opensuse.org/repositories/home:/scalaris/Debian_6.0/Release.key 2>/dev/null | apt-key add -
apt-get -f -y update
apt-get -f -y --no-install-recommends --no-upgrade install scalaris screen
update-rc.d scalaris disable
# remove scalaris repo
sed --in-place 's%deb http://download.opensuse.org/repositories/home:/scalaris/Debian_6.0 /%%' /etc/apt/sources.list
apt-get -f -y update

# add xtreemfs repo
echo "deb http://download.opensuse.org/repositories/home:/xtreemfs:/unstable/Debian_6.0 /" >> /etc/apt/sources.list
wget -O - http://download.opensuse.org/repositories/home:/xtreemfs:/unstable/Debian_6.0/Release.key 2>/dev/null | apt-key add -
apt-get -f -y update
apt-get -f -y --no-install-recommends --no-upgrade install xtreemfs-server xtreemfs-client
update-rc.d xtreemfs-osd disable
update-rc.d xtreemfs-mrc disable
update-rc.d xtreemfs-dir disable
# remove xtreemfs repo
sed --in-place 's%deb http://download.opensuse.org/repositories/home:/xtreemfs:/unstable/Debian_6.0 /%%' /etc/apt/sources.list
apt-get -f -y update

# install latest nginx (1.2.2) and other packages required by CDS
DEBIAN_FRONTEND=noninteractive apt-get -y --force-yes --no-install-recommends --no-upgrade \
    install libpcre3-dev libssl-dev libgeoip-dev libperl-dev
wget http://nginx.org/download/nginx-1.2.2.tar.gz
tar xzf nginx-1.2.2.tar.gz
cd nginx-1.2.2
./configure --sbin-path=/usr/sbin/nginx --conf-path=/etc/nginx/nginx.conf --error-log-path=/var/log/nginx/error.log --http-client-body-temp-path=/var/lib/nginx/body --http-fastcgi-temp-path=/var/lib/nginx/fastcgi --http-log-path=/var/log/nginx/access.log --http-proxy-temp-path=/var/lib/nginx/proxy --lock-path=/var/lock/nginx.lock --pid-path=/var/run/nginx.pid --with-debug --with-http_dav_module --with-http_flv_module --with-http_geoip_module --with-http_gzip_static_module --with-http_realip_module --with-http_stub_status_module --with-http_ssl_module --with-http_sub_module --with-ipv6 --with-mail --with-mail_ssl_module --with-http_perl_module
make
make install
cd ..
rm -rf nginx-1.2.2*

apt-get -f -y clean
exit 0
EOF

# Execute the script for installing the dependencies.
chmod a+x $ROOT_DIR/conpaas_install
chroot $ROOT_DIR /bin/bash /conpaas_install
rm -f $ROOT_DIR/conpaas_install

rm -f $ROOT_DIR/usr/sbin/policy-rc.d

##### TO CUSTOMIZE: #####
# This part is for OpenNebula contextualization. The contextualization scripts (and possibly
# other necessary files) will be provided through OpenNebula to the VM as an ISO image.
# We need to mount this image and execute the contextualization scripts. You might need
# to change the dev file associated with the CD-ROM inside your virtual machine from
# "/dev/sr0" to something else (depending on your operating system and on the virtualization
# software, it can be /dev/hdb, /dev/sdb etc.). You can check this in a VM that is already running
# in your OpenNebula system and that has been configured with contextualization.

cat <<"EOF" > $ROOT_DIR/etc/rc.local
#!/bin/sh
mount -t iso9660 /dev/sr0 /mnt

if [ -f /mnt/context.sh ]; then
  . /mnt/context.sh
  if [ -n "$USERDATA" ]; then
    echo "$USERDATA" | /usr/bin/xxd -p -r | /bin/sh
  elif [ -e /mnt/init.sh ]; then
    . /mnt/init.sh
  fi
fi

umount /mnt

exit 0
EOF


add a release script

add a release script

#!/bin/bash

set -o errexit
set -o nounset

verify_args() {
  echo "Cutting a release for version $version with commit $commit"
  while true; do
    read -rp "Is this correct (y/n)?" yn
    case $yn in
    [Yy]*) break;;
    [Nn]*) exit;;
    *) echo "Please answer y or n.";;
    esac
  done
}

update_version() {
  echo "$version" > VERSION
  echo "VERSION set to $version"
}

if [ $# -le 2 ]; then
  echo "version and commit arguments required"
fi

set -eu
version=$1
commit=$2

verify_args

# create version release branch
git checkout master
git pull
if ! git diff --exit-code master origin/master
then
  echo "ERROR! There are local-only changes on branch 'master'!"
  exit 1
fi
git checkout -b "release-$version" "$commit"

# update VERSION
update_version

# commit VERSION and CHANGELOG updates
git add VERSION
git commit -m "Prep for $version release

[skip ci]"

# merge into release
git checkout release
if ! git diff --exit-code release origin/release
then
  echo "ERROR! There are local-only changes on branch 'release'!"
  exit 1
fi
git merge "release-$version" -m "Release $version"

# tag release
git tag "$version"

# push to release branch
git push origin release
git push origin "$version"

git checkout master
git merge "release-$version"

git push origin master

error: pathspec 'release.sh' did not match any file(s) known to git


Added script to release

Added script to release

remote: Support for password authentication was removed on August 13, 2021.
remote: Please see https://docs.github.com/en/get-started/getting-started-with-git/about-remote-repositories#cloning-with-https-urls for information on currently recommended modes of authentication.
fatal: Authentication failed for 'https://github.com/sixro/ipsteric.git/'


release.sh

release.sh

#!/bin/bash

VERSION=$1

MESSAGE="$2"

function CL() {
    STR="$1"
    STR=${STR//\:/\\:}
    STR=${STR//\//\\/}
    STR=${STR//\_/\\_}
    echo "$STR"
}

UPDATE_LINK=$(CL https://github.com/alexandre-mbm/donatal-firefox/releases/download/$VERSION/verificador_do_diario_oficial_de_natal-$VERSION-fx.xpi)

UPDATE_URL=$(CL https://raw.githubusercontent.com/alexandre-mbm/donatal-firefox/master/update.rdf)

RDF=@donatal-$VERSION.update.rdf

XPI=donatal.xpi

FINAL=$(CL verificador_do_diario_oficial_de_natal-$VERSION-fx.xpi)

JSON=package.json

MANIFEST=update.rdf

# muda versão
# gera pacote
# git push --tags
# editar o release (abrir página)

sed -i '/"version"\:/s/\: ".*"/\: "'$VERSION'"/' $JSON
sed -i '/"updateURL"\:/s/\: ".*"/\: "'$UPDATE_URL'"/' $JSON
sed -i '/"updateLink"\:/s/\: ".*"/\: "'$UPDATE_LINK'"/' $JSON

jpm xpi

jpm sign --api-key "$JPM_API_KEY" --api-secret "$JPM_API_SECRET" --xpi $XPI

mv $RDF $MANIFEST

git add $JSON
git add $MANIFEST
git commit -m "$MESSAGE"
git tag -f $VERSION
git push
git push --tags
git show

echo
echo https://github.com/alexandre-mbm/donatal-firefox/releases/tag/$VERSION
echo


Create git_commit_rewrite_one_at_a_time.sh

# check log and figure out how many commits you going to consider as bad commits
git log

# do rebase with number of commits you going to edit
git rebase -i HEAD~5

# you will get list of commits
# file contents begins
pick e530edb2 bad commit 5
pick 570c043e bad commit 4
pick 71b605be bad commit 3
pick 8b473faa good commit
pick 2759a4ae bad commit 2

# Rebase (5 command(s))
#
# Commands:
# p, pick = use commit
# r, reword = use commit, but edit the commit message
# e, edit = use commit, but stop for amending
# s, squash = use commit, but meld into previous commit
# f, fixup = like "squash", but discard this commit's log message
# x, exec = run command (the rest of the line) using shell
#
# These lines can be re-ordered; they are executed from top to bottom.
#
# If you remove a line here THAT COMMIT WILL BE LOST.
#
# However, if you remove everything, the rebase will be aborted.
#
# Note that empty commits are commented out

# file contents ends

# Next thing you will do is fix this bad commits by placing below line on each bad commit
exec git commit --amend --author="Firstname Lastname <myemail@gmail.com>" -C HEAD

# Go back to file and add this
pick e530edb2 bad commit 5
exec git commit --amend --author="Firstname Lastname <myemail@gmail.com>" -C HEAD
pick 570c043e bad commit 4
exec git commit --amend --author="Firstname Lastname <myemail@gmail.com>" -C HEAD
pick 71b605be bad commit 3
exec git commit --amend --author="Firstname Lastname <myemail@gmail.com>" -C HEAD
pick 8b473faa good commit
pick 2759a4ae bad commit 2
exec git commit --amend --author="Firstname Lastname <myemail@gmail.com>" -C HEAD

# save changes git will exec this file and update all necessary info based on above changes

# Now lets push this changes to remote
git push -f origin master(your branch name)

error: pathspec 'git_commit_rewrite_one_at_a_time.sh' did not match any file(s) known to git


IpTables Default

IpTables Default

# Modify this file accordingly for your specific requirement.
# http://www.thegeekstuff.com
# 1. Delete all existing rules
iptables -F

# 2. Set default chain policies
iptables -P INPUT DROP
iptables -P FORWARD DROP
iptables -P OUTPUT DROP

# 3. Block a specific ip-address
#BLOCK_THIS_IP="x.x.x.x"
#iptables -A INPUT -s "$BLOCK_THIS_IP" -j DROP

# 4. Allow ALL incoming SSH
iptables -A INPUT -i eth0 -p tcp -s 10.0.0.0/7 --dport 22 -m state --state NEW,ESTABLISHED -j ACCEPT
iptables -A OUTPUT -o eth0 -p tcp --sport 22 -m state --state ESTABLISHED -j ACCEPT

# 5. Allow incoming SSH only from a specific network
#iptables -A INPUT -i eth0 -p tcp -s 192.168.200.0/24 --dport 22 -m state --state NEW,ESTABLISHED -j ACCEPT
#iptables -A OUTPUT -o eth0 -p tcp --sport 22 -m state --state ESTABLISHED -j ACCEPT

# 6. Allow incoming HTTP
iptables -A INPUT -i eth0 -p tcp --dport 80 -m state --state NEW,ESTABLISHED -j ACCEPT
iptables -A OUTPUT -o eth0 -p tcp --sport 80 -m state --state ESTABLISHED -j ACCEPT

# Allow incoming HTTPS
iptables -A INPUT -i eth0 -p tcp --dport 443 -m state --state NEW,ESTABLISHED -j ACCEPT
iptables -A OUTPUT -o eth0 -p tcp --sport 443 -m state --state ESTABLISHED -j ACCEPT

# 7. MultiPorts (Allow incoming SSH, HTTP, and HTTPS)
iptables -A INPUT -i eth0 -p tcp -m multiport --dports 22,80,443 -m state --state NEW,ESTABLISHED -j ACCEPT
iptables -A OUTPUT -o eth0 -p tcp -m multiport --sports 22,80,443 -m state --state ESTABLISHED -j ACCEPT

# 8. Allow outgoing SSH
iptables -A OUTPUT -o eth0 -p tcp --dport 22 -m state --state NEW,ESTABLISHED -j ACCEPT
iptables -A INPUT -i eth0 -p tcp --sport 22 -m state --state ESTABLISHED -j ACCEPT

# 9. Allow outgoing SSH only to a specific network
#iptables -A OUTPUT -o eth0 -p tcp -d 192.168.101.0/24 --dport 22 -m state --state NEW,ESTABLISHED -j ACCEPT
#iptables -A INPUT -i eth0 -p tcp --sport 22 -m state --state ESTABLISHED -j ACCEPT

# 10. Allow outgoing HTTPS
iptables -A OUTPUT -o eth0 -p tcp --dport 443 -m state --state NEW,ESTABLISHED -j ACCEPT
iptables -A INPUT -i eth0 -p tcp --sport 443 -m state --state ESTABLISHED -j ACCEPT

# 11. Load balance incoming HTTPS traffic
#iptables -A PREROUTING -i eth0 -p tcp --dport 443 -m state --state NEW -m nth --counter 0 --every 3 --packet 0 -j DNAT --to-destination 192.168.1.101:443
#iptables -A PREROUTING -i eth0 -p tcp --dport 443 -m state --state NEW -m nth --counter 0 --every 3 --packet 1 -j DNAT --to-destination 192.168.1.102:443
#iptables -A PREROUTING -i eth0 -p tcp --dport 443 -m state --state NEW -m nth --counter 0 --every 3 --packet 2 -j DNAT --to-destination 192.168.1.103:443

# 12. Ping from inside to outside
iptables -A OUTPUT -p icmp --icmp-type echo-request -j ACCEPT
iptables -A INPUT -p icmp --icmp-type echo-reply -j ACCEPT

# 13. Ping from outside to inside
iptables -A INPUT -p icmp --icmp-type echo-request -j ACCEPT
iptables -A OUTPUT -p icmp --icmp-type echo-reply -j ACCEPT

# 14. Allow loopback access
iptables -A INPUT -i lo -j ACCEPT
iptables -A OUTPUT -o lo -j ACCEPT

# 15. Allow packets from internal network to reach external network.
# if eth1 is connected to external network (internet)
# if eth0 is connected to internal network (192.168.1.x)
iptables -A FORWARD -i eth0 -o eth1 -j ACCEPT

# 16. Allow outbound DNS
iptables -A OUTPUT -p udp -o eth0 --dport 53 -j ACCEPT
iptables -A INPUT -p udp -i eth0 --sport 53 -j ACCEPT

# 17. Allow NIS Connections
# rpcinfo -p | grep ypbind ; This port is 853 and 850
#iptables -A INPUT -p tcp --dport 111 -j ACCEPT
#iptables -A INPUT -p udp --dport 111 -j ACCEPT
#iptables -A INPUT -p tcp --dport 853 -j ACCEPT
#iptables -A INPUT -p udp --dport 853 -j ACCEPT
#iptables -A INPUT -p tcp --dport 850 -j ACCEPT
#iptables -A INPUT -p udp --dport 850 -j ACCEPT

# 18. Allow rsync from a specific network
#iptables -A INPUT -i eth0 -p tcp -s 192.168.101.0/24 --dport 873 -m state --state NEW,ESTABLISHED -j ACCEPT
#iptables -A OUTPUT -o eth0 -p tcp --sport 873 -m state --state ESTABLISHED -j ACCEPT

# 19. Allow MySQL connection only from a specific network
#iptables -A INPUT -i eth0 -p tcp -s 192.168.200.0/24 --dport 3306 -m state --state NEW,ESTABLISHED -j ACCEPT
#iptables -A OUTPUT -o eth0 -p tcp --sport 3306 -m state --state ESTABLISHED -j ACCEPT

# 20. Allow Sendmail or Postfix
iptables -A INPUT -i eth0 -p tcp --dport 25 -m state --state NEW,ESTABLISHED -j ACCEPT
iptables -A OUTPUT -o eth0 -p tcp --sport 25 -m state --state ESTABLISHED -j ACCEPT

# 21. Allow IMAP and IMAPS
#iptables -A INPUT -i eth0 -p tcp --dport 143 -m state --state NEW,ESTABLISHED -j ACCEPT
#iptables -A OUTPUT -o eth0 -p tcp --sport 143 -m state --state ESTABLISHED -j ACCEPT
#iptables -A INPUT -i eth0 -p tcp --dport 993 -m state --state NEW,ESTABLISHED -j ACCEPT
#iptables -A OUTPUT -o eth0 -p tcp --sport 993 -m state --state ESTABLISHED -j ACCEPT

# 22. Allow POP3 and POP3S
#iptables -A INPUT -i eth0 -p tcp --dport 110 -m state --state NEW,ESTABLISHED -j ACCEPT
#iptables -A OUTPUT -o eth0 -p tcp --sport 110 -m state --state ESTABLISHED -j ACCEPT
#iptables -A INPUT -i eth0 -p tcp --dport 995 -m state --state NEW,ESTABLISHED -j ACCEPT
#iptables -A OUTPUT -o eth0 -p tcp --sport 995 -m state --state ESTABLISHED -j ACCEPT

# 23. Prevent DoS attack
iptables -A INPUT -p tcp --dport 80 -m limit --limit 25/minute --limit-burst 100 -j ACCEPT

# 24. Port forwarding 422 to 22
#iptables -t nat -A PREROUTING -p tcp -d 192.168.102.37 --dport 422 -j DNAT --to 192.168.102.37:22
#iptables -A INPUT -i eth0 -p tcp --dport 422 -m state --state NEW,ESTABLISHED -j ACCEPT
#iptables -A OUTPUT -o eth0 -p tcp --sport 422 -m state --state ESTABLISHED -j ACCEPT

# 25. Log dropped packets
iptables -N LOGGING
iptables -A INPUT -j LOGGING
iptables -A LOGGING -m limit --limit 2/min -j LOG --log-prefix "IPTables Packet Dropped: " --log-level 7
iptables -A LOGGING -j DROP

error: pathspec 'general/networking/iptables-rules.sh' did not match any file(s) known to git


Fixed pip

Fixed pip

remote: Support for password authentication was removed on August 13, 2021.
remote: Please see https://docs.github.com/en/get-started/getting-started-with-git/about-remote-repositories#cloning-with-https-urls for information on currently recommended modes of authentication.
fatal: Authentication failed for 'https://github.com/byuoitav/raspi-deployment-microservice.git/'


Add tex to path

Add tex to path

error: pathspec 'tex/path.zsh' did not match any file(s) known to git


Add a helper to bump the pom-scijava version after editing its properties

This will be used soon in a Jenkins job.

Signed-off-by: Johannes Schindelin <53fb8db7833fca1e1746dd8592f61048d350f64c@gmx.de>

#!/bin/sh

die () {
	echo "$*" >&2
	exit 1
}

skip_commit=
test a--skip-commit != "a$1" || {
	skip_commit=t
	shift
}

test $# = 2 ||
die "Usage: $0 [--skip-commit] <key> <value>"

pom=pom-scijava/pom.xml
cd "$(dirname "$0")/.." &&
test -f $pom ||
die "Could not switch to scijava-common's root directory"

sed_quote () {
        echo "$1" | sed "s/[]\/\"\'\\\\(){}[\!\$  ;]/\\\\&/g"
}

gav="$(sh bin/maven-helper.sh gav-from-pom $pom)"
old_version=${gav##*:}
new_version=${old_version%.*}.$((1 + ${old_version##*.}))

property="$(sed_quote "$1")"
value="$(sed_quote "$2")"
sed \
  -e "/<properties>/,/<\/properties>/s/\(<$property>\)[^<]*\(<\/$property>\)/\1$value\2/" \
  $pom > $pom.new ||
die "Failed to set property $1 = $2"

git diff --quiet --no-index $pom $pom.new &&
die "Property $1 not found in $pom"

sed \
  -e "s/^\(\\t<version>\)$old_version\(<\/version>\)/\1$new_version\2/" \
  $pom.new > $pom ||
die "Failed to increase version of $pom"

rm $pom.new ||
die "Failed to remove intermediate $pom.new"

test f = "$skip_commit" || {
	git commit -s -m "Increase pom-scijava version to $new_version" \
		-m "This changed the property '$1' to '$2'." $pom &&
	git push origin HEAD
}

error: pathspec 'bin/bump-pom-scijava.sh' did not match any file(s) known to git


Create update_program_langs.sh

Create update_program_langs.sh

error: pathspec 'update_program_langs.sh' did not match any file(s) known to git