<NME> test_functions <BEF> # By default, this script runs in testing mode # Although it allows for a server debugging mode, which attaches gdb to # the `cvmfs_server publish` process and allows for both online and # failure debugging. # # To allow for online debugging, export CVMFS_TEST_SRVDEBUG like so: # # `export CVMFS_TEST_SRVDEBUG=fail` - for crash debugging with gdb # `export CVMFS_TEST_SRVDEBUG=startup` - for interactive debugging with gdb CVMFS_SERVER_APACHE_RELOAD_IS_RESTART=true CVMFS_TEST_DEBUGLOG=${CVMFS_TEST_DEBUGLOG:=} CVMFS_TEST_PROXY=${CVMFS_TEST_PROXY:=http://ca-proxy.cern.ch:3128} CVMFS_TEST_SCRATCH=${CVMFS_TEST_SCRATCH:=/tmp/cvmfs-test} CVMFS_TEST_SUITE_NAME=${CVMFS_TEST_SUITE_NAME:=Integration Tests} CVMFS_TEST_CLASS_NAME=${CVMFS_TEST_CLASS_NAME:=IntegrationTests} CVMFS_TEST_EXCLUDE=${CVMFS_TEST_EXCLUDE:=} CVMFS_TEST_SYSLOG_FACILITY=${CVMFS_TEST_SYSLOG_FACILITY:=5} CVMFS_TEST_SYSLOG_TARGET=${CVMFS_TEST_SYSLOG_TARGET:=/var/log/cvmfs-testing.log} CVMFS_TEST_REPO=${CVMFS_TEST_REPO:=test.cern.ch} CVMFS_TEST_REPO_MORE=${CVMFS_TEST_REPO_MORE:=test-more.cern.ch} CVMFS_TEST_USER=${CVMFS_TEST_USER:=sftnight} # user and group are used to over- CVMFS_TEST_GROUP=${CVMFS_TEST_GROUP:=sftnight} # write the owner of files for testing CVMFS_TEST_GEO_LICENSE_KEY=${CVMFS_TEST_GEO_LICENSE_KEY:=} CVMFS_TEST_UNIONFS=${CVMFS_TEST_UNIONFS:=} # union filesystem type to test CVMFS_TEST_SRVDEBUG=${CVMFS_TEST_SRVDEBUG:=} CVMFS_TEST_HASHALGO=${CVMFS_TEST_HASHALGO:=sha1} CVMFS_TEST_S3_CONFIG=${CVMFS_TEST_S3_CONFIG:=} CVMFS_TEST_S3_STORAGE=${CVMFS_TEST_S3_STORAGE:=} CVMFS_TEST_HTTP_BASE=${CVMFS_TEST_HTTP_BASE:=} CVMFS_TEST_STRATUM0=${CVMFS_TEST_STRATUM0:=} # backward compatibility! use CVMFS_TEST_HTTP_BASE instead # These OPT variables are for the benchmark options under the benchmarks folder CVMFS_OPT_WARM_CACHE=${CVMFS_OPT_WARM_CACHE:=yes} CVMFS_OPT_HOT_CACHE=${CVMFS_OPT_HOT_CACHE:=no} CVMFS_OPT_TALK_STATISTICS=${CVMFS_OPT_TALK_STATISTICS:=yes} CVMFS_OPT_VALGRIND=${CVMFS_OPT_VALGRIND:=yes} CVMFS_OPT_ITERATIONS=${CVMFS_OPT_ITERATIONS:=1} CVMFS_OPT_TEST_TYPE=${CVMFS_OPT_TEST_TYPE:=callgrind} # callgrind / memcheck CVMFS_OPT_CONFIG_FILE=${CVMFS_OPT_CONFIG_FILE:=} CVMFS_OPT_OUTPUT_DIR=${CVMFS_OPT_OUTPUT_DIR:=/tmp/cvmfs_benchmarks} CVMFS_OPT_CACHEDIR=${CVMFS_OPT_CACHEDIR:=~/.benchmark} CVMFS_OPT_PARALLEL_RUNS=${CVMFS_OPT_PARALLEL_RUNS:=1} CVMFS_OPT_MTAB_MODIFIED="false" # backward compatibility. # CVMFS_TEST_STRATUM0 was replaced by CVMFS_TEST_HTTP_BASE to better reflect # the purpose of this variable. Please don't use CVMFS_TEST_STRATUM0 anymore if [ -z "$CVMFS_TEST_HTTP_BASE" ] && [ ! -z "$CVMFS_TEST_STRATUM0" ]; then echo "Warning: You are using the deprecated CVMFS_TEST_STRATUM0 variable." echo " Please consider switching to CVMFS_TEST_HTTP_BASE" CVMFS_TEST_HTTP_BASE="$CVMFS_TEST_STRATUM0" fi if [ ! -z "$CVMFS_TEST_GEO_LICENSE_KEY" ]; then echo "CVMFS_GEO_LICENSE_KEY=$CVMFS_TEST_GEO_LICENSE_KEY" | sudo tee /etc/cvmfs/server.local > /dev/null fi if [ -f /.dockerinit ]; then CVMFS_TEST_DOCKER=yes else CVMFS_TEST_DOCKER=no fi die() { echo -e $1 >&2 exit 1 } running_on_osx() { [ "x$(uname)" == "xDarwin" ] } if running_on_osx; then CVMFS_PYTHON2="python" else CVMFS_PYTHON2="python2" fi CVMFS_SYS_PYTHON="python" if ! $CVMFS_SYS_PYTHON -V >/dev/null 2>&1; then CVMFS_SYS_PYTHON="python3" if ! $CVMFS_SYS_PYTHON -V >/dev/null 2>&1; then CVMFS_SYS_PYTHON="python2" fi fi running_on_s3() { [ "x$CVMFS_TEST_S3_CONFIG" != "x" ] } running_with_external_cache() { local repo="$1" sudo cvmfs_talk -i $repo cache instance | head -1 | grep -q External } get_xattr() { local extended_attr="$1" local path="$2" if running_on_osx; then xattr -p user.$extended_attr "$path" else attr -qg $extended_attr "$path" fi } list_xattrs() { local repo="$1" if running_on_osx; then xattr "$repo" else attr -ql "$repo" fi } show_mounted() { if running_on_osx; then mount else cat /proc/mounts fi } get_user_data() { local user=$1 if running_on_osx; then id -P "$user" else getent passwd "$user" fi } stat_wrapper() { local format=$1 local repo=$2 if running_on_osx; then stat -f $format $repo else stat -c $format $repo fi } # Find the service binary (or detect systemd) minpidof() { pidof $1 | tr " " "\n" | sort --numeric-sort | head -n1 } SERVICE_BIN="false" if ! pidof systemd > /dev/null 2>&1 || [ $(minpidof systemd) -ne 1 ]; then if [ -x /sbin/service ]; then SERVICE_BIN="/sbin/service" elif [ -x /usr/sbin/service ]; then SERVICE_BIN="/usr/sbin/service" # Ubuntu elif [ -x /sbin/rc-service ]; then SERVICE_BIN="/sbin/rc-service" # OpenRC elif running_on_osx; then SERVICE_BIN="false" # we don't need to run any service on mac else die "Neither systemd nor service binary detected" fi fi is_systemd() { [ x"$SERVICE_BIN" = x"false" ] } is_redhat() { [ -e /etc/redhat-release ] } is_el7() { is_redhat && which lsb_release > /dev/null 2>&1 && \ [ x"$(lsb_release -rs | cut -f1 -d'.')" = x"7" ] } is_el6() { is_redhat && which lsb_release > /dev/null 2>&1 && \ [ x"$(lsb_release -rs | cut -f1 -d'.')" = x"6" ] } is_el5() { is_redhat && which lsb_release > /dev/null 2>&1 && \ [ x"$(lsb_release -rs | cut -f1 -d'.')" = x"5" ] } # find the name of the httpd service # figure out the installed apache platform if which httpd2 >/dev/null 2>&1; then #SLES/OpenSuSE APACHE_SERVICE="apache2" APACHE_CONF=${APACHE_SERVICE} APACHE_BIN=$(which httpd2) elif which apache2 >/dev/null 2>&1; then # Debian based APACHE_SERVICE="apache2" APACHE_CONF=${APACHE_SERVICE} APACHE_BIN=$(which apache2) elif which /usr/sbin/apache2 >/dev/null 2>&1; then # newer Ubuntu APACHE_SERVICE="apache2" APACHE_CONF=${APACHE_SERVICE} APACHE_BIN=/usr/sbin/apache2 else APACHE_SERVICE="httpd" # EL based APACHE_CONF=${APACHE_SERVICE} APACHE_BIN="/usr/sbin/httpd" fi normalize_version() { local version_string="$1" while [ $(echo "$version_string" | grep -o '\.' | wc -l) -lt 2 ]; do version_string="${version_string}.0" done echo "$version_string" } to_syslog() { logger -t cvmfs_test "$1" } version_major() { echo $1 | cut --delimiter=. --fields=1; } version_minor() { echo $1 | cut --delimiter=. --fields=2; } version_patch() { echo $1 | cut --delimiter=. --fields=3; } prepend_zeros() { printf %05d "$1"; } compare_versions() { local lhs="$(normalize_version $1)" local comparison_operator=$2 local rhs="$(normalize_version $3)" local lhs1=$(prepend_zeros $(version_major $lhs)) local lhs2=$(prepend_zeros $(version_minor $lhs)) local lhs3=$(prepend_zeros $(version_patch $lhs)) local rhs1=$(prepend_zeros $(version_major $rhs)) local rhs2=$(prepend_zeros $(version_minor $rhs)) local rhs3=$(prepend_zeros $(version_patch $rhs)) [ $lhs1$lhs2$lhs3 $comparison_operator $rhs1$rhs2$rhs3 ] } service_request() { local service_name=$1 local request_verb=$2 if is_systemd; then echo "using systemd" # Prevent "service restarted too often" failures sudo systemctl reset-failed $service_name if [ "x$request_verb" = "xrestart" ]; then sudo systemctl stop $service_name if ! sudo systemctl start $service_name; then sleep 20 sudo systemctl start $service_name fi else if ! sudo systemctl $request_verb $service_name; then sleep 20 sudo systemctl $request_verb $service_name fi fi else echo "using traditional service binary" sudo $SERVICE_BIN $service_name $request_verb fi } # ensures that a generic service is running # @param service the name of the service # @param state the desired state of the service (on|off) # @return 0 on success service_switch() { local service_name=$1 local state=$2 running_on_osx && return 0 case $state in on) echo "starting $service_name..." service_request $service_name start || return 100 ;; off) echo "stopping $service_name..." service_request $service_name stop || return 101 ;; restart) echo "restarting $service_name..." service_request $service_name restart || return 102 ;; *) echo "unrecognized state switching for $service_name" return 102 esac return 0 } # checks if a service needs to be switched on or off # @param desired_state the state the service should be switched to # @param state the state the service is currently in # @return 0 if a switch is needed service_should_switch() { local desired_state=$1 local state=$2 if [ $state -eq 0 ]; then # is running and... if [ "$desired_state" = "off" ]; then # should be switched off return 0 fi else # is NOT running and... if [ "$desired_state" = "on" ]; then # should be switched on return 0 fi fi return 1 } # checks if autofs is running on /cvmfs # @return 0 when autofs is mounted on /cvmfs autofs_check() { if running_on_osx; then return 0 else cat /proc/mounts | grep -q "^/etc/auto.cvmfs /cvmfs " fi } # ensures that autofs is on or off # @param state the desired state of autofs (on|off) # @return 0 on success autofs_switch() { autofs_check if service_should_switch $1 $?; then service_switch autofs $1 fi } # switches apache on or off # @param state the desired state of autofs (on|off) # @return 0 on success apache_switch() { service_request $APACHE_SERVICE status > /dev/null 2>&1 if service_should_switch $1 $?; then service_switch $APACHE_SERVICE $1 fi } # Checks whether the given test case is marked with any of the given # test suite tags. If there are no labels, execute the test # @return 0 on success is_in_suite() { local testdir=$1 shift local labels=$@ if [ "x$labels" = "x" ]; then return 0; fi ( unset cvmfs_test_suites . $t/main for wanted_label in $labels; do for test_label in $cvmfs_test_suites; do if [ "$wanted_label" = "$test_label" ]; then exit 0; fi done done exit 1 ) return $? } contains() { local haystack="$1" local needle=$2 for elem in $haystack do if [ $(basename $elem) = $(basename $needle) ]; then return 0 fi done return 1 } uses_aufs() { local repo_name="$1" load_repo_config $repo_name [ x"$CVMFS_UNION_FS_TYPE" = x"aufs" ] } uses_overlayfs() { local repo_name="$1" load_repo_config $repo_name [ x"$CVMFS_UNION_FS_TYPE" = x"overlayfs" ] } is_linux() { [ x"$(uname)" = x"Linux" ] } is_macos() { [ x"$(uname)" = x"Darwin" ] } get_number_of_cpu_cores() { if is_linux; then cat /proc/cpuinfo | grep -e '^processor' | wc -l elif is_macos; then sysctl -n hw.ncpu else echo "1" fi } break_hardlink() { local path="$1" local tmp_file="$(mktemp)" local linkcount="$(stat --format=%h $path)" echo "Note: breaking up hardlink to '$path' linkcount: $linkcount" cp -f $path $tmp_file rm -f $path mv $tmp_file $path } wait_for_umount() { local timeout=60 while $(pgrep -u cvmfs cvmfs2 > /dev/null); do if [ $timeout -eq 0 ]; then return 1 fi timeout=$(($timeout-1)) sleep 1 done return 0 } cvmfs_clean() { sudo cvmfs_config umount > /dev/null || return 100 sudo sh -c "rm -rf /var/lib/cvmfs/*" || return 101 sudo rm -f /etc/cvmfs/default.local || return 102 sudo sh -c "rm -f /etc/cvmfs/config.d/*.local" || return 104 sudo sh -c "cat /dev/null > $CVMFS_TEST_SYSLOG_TARGET" || return 105 wait_for_umount || return 99 return 0 } cvmfs_disable_config_repository() { sudo sh -c "echo 'CVMFS_CONFIG_REPOSITORY=' > /etc/cvmfs/default.d/99_no_config_repo.conf" \ || return $? } cvmfs_enable_config_repository() { sudo rm -f /etc/cvmfs/default.d/99_no_config_repo.conf || return $? } cvmfs_mount_direct() { repositories=$1 eval $(cvmfs_config showconfig | grep -e ^CVMFS_CONFIG_REPOSITORY=) if [ "x$CVMFS_CONFIG_REPOSITORY" != "x" ]; then _cvmfs_mount_manually "$CVMFS_CONFIG_REPOSITORY" fi for i in $(echo "$repositories" | tr "," " "); do _cvmfs_mount_manually "$i" || return 102 done } cvmfs_mount() { if [ x$PARROT_ENABLED = "xTRUE" ]; then return 0 fi repositories=$1 shift 1 sudo sh -c "echo \"CVMFS_REPOSITORIES=$repositories\" > /etc/cvmfs/default.local" || return 100 sudo sh -c "echo \"CVMFS_HTTP_PROXY=\\\"${CVMFS_TEST_PROXY}\\\"\" >> /etc/cvmfs/default.local" || return 100 sudo sh -c 'echo "CVMFS_TIMEOUT=20" >> /etc/cvmfs/default.local' || return 100 sudo sh -c 'echo "CVMFS_TIMEOUT_DIRECT=20" >> /etc/cvmfs/default.local' || return 100 sudo sh -c "echo CVMFS_SYSLOG_FACILITY=$CVMFS_TEST_SYSLOG_FACILITY >> /etc/cvmfs/default.local" || return 100 sudo sh -c "echo CVMFS_REPOSITORY_TAG=TESTING > /etc/cvmfs/config.d/cvmfs-config.cern.ch.local" || return 100 # add additional parameters local proberunner while [ $# -gt 0 ]; do local param="$1" if [ "$param" = "CVMFS_SUID=yes" ]; then proberunner="sudo" fi sudo sh -c "echo \"$param\" >> /etc/cvmfs/default.local" || return 100 shift 1 done if [ "x$CVMFS_TEST_DEBUGLOG" != "x" ]; then sudo sh -c "echo \"CVMFS_DEBUGLOG=$CVMFS_TEST_DEBUGLOG\" >> /etc/cvmfs/default.local" || return 100 fi if running_on_osx; then cvmfs_mount_direct $repositories fi if [ "x$cvmfs_test_autofs_on_startup" = "xfalse" ]; then cvmfs_mount_direct $repositories fi $proberunner cvmfs_config probe > /dev/null 2>&1 || return 101 return 0 } _cvmfs_mount_manually() { local repository=$1 cvmfs_config status | grep -q "/cvmfs/$repository" && return 0 mkdir -p "/cvmfs/$repository" 2>/dev/null || sudo mkdir -p "/cvmfs/$repository" sudo mount -t cvmfs "$repository" "/cvmfs/$repository" >/dev/null 2>&1 } cvmfs_umount() { if [ x$PARROT_ENABLED = "xTRUE" ]; then return 0 fi local repositories=$1 local times=5 local result=-1 for r in $(echo $repositories | tr , " "); do while (( result != 0 && times > 0 )); do sudo umount /cvmfs/$r > /dev/null 2>&1 result=$? times=$(( $times - 1 )) if [ $result -ne 0 ] && [ $times -gt 0 ]; then echo "Failed to umount. Trying again..." sleep 1 fi done if [ $result -ne 0 ]; then lsof /cvmfs/$r return 200 fi local timeout=5 while show_mounted | grep -q " /cvmfs/$r "; do if [ $timeout -eq 0 ]; then return 101 fi timeout=$(($timeout-1)) sleep 1 done times=5 result=-1 done return 0 } cvmfs_remove_breadcrumb() { local repo="$1" if running_with_external_cache $repo; then local pid_cache=$(sudo cvmfs_talk -i $repo pid cachemgr) echo "--- sending SIGUSR2 to cache manager at PID $pid_cache" sudo kill -12 $pid_cache sleep 2 else echo "--- removing $(get_cvmfs_cachedir ${repo})/cvmfschecksum.${repo}" sudo rm -f "$(get_cvmfs_cachedir ${repo})/cvmfschecksum.${repo}" || return 1 fi } has_selinux() { which sestatus > /dev/null 2>&1 && \ which getenforce > /dev/null 2>&1 && \ getenforce | grep -qi "enforc" || return 1 } get_cvmfs_cachedir() { local repository=$1 local cache_dir=$(cvmfs_config showconfig $repository | grep CVMFS_WORKSPACE | awk '{print $1}' | cut -d= -f2) if [ "x$cache_dir" = "x" ]; then cache_dir=$(cvmfs_config showconfig $repository | grep CVMFS_CACHE_DIR | awk '{print $1}' | cut -d= -f2) fi if [ "x$cache_dir" = "x" ]; then echo "Failed to figure out cache directory" exit 1 fi echo $cache_dir } CVMFS_MEMORY_WARNING_FLAG=0 CVMFS_TIME_WARNING_FLAG=0 CVMFS_GENERAL_WARNING_FLAG=0 CVMFS_MEMORY_WARNING=254 CVMFS_TIME_WARNING=253 CVMFS_GENERAL_WARNING=252 CVMFS_TEST_RETVAL_TIMEOUT=251 mangle_test_retval() { local prior_retval=$1 # if the test case failed... report the retval right away if [ $prior_retval -ne 0 ]; then return $prior_retval fi # check if the test case produced memory warnings and report them if [ $CVMFS_MEMORY_WARNING_FLAG -ne 0 ]; then return $CVMFS_MEMORY_WARNING fi # check if the test case produced timeout warnings if [ $CVMFS_TIME_WARNING_FLAG -ne 0 ]; then return $CVMFS_TIME_WARNING fi # check if the test case produced any other warnings if [ $CVMFS_GENERAL_WARNING_FLAG -ne 0 ]; then return $CVMFS_GENERAL_WARNING fi # return 0 if all is good return 0 } reset_test_warning_flags() { CVMFS_MEMORY_WARNING_FLAG=0 CVMFS_TIME_WARNING_FLAG=0 CVMFS_GENERAL_WARNING_FLAG=0 } check_time() { local start_time=$1 local end_time=$2 local limit=$3 local diff_time=$(($end_time-$start_time)) if [ $diff_time -gt $limit ]; then echo "Time limit exceeded" >&2 echo "Limit was $limit but it took $diff_time seconds" CVMFS_TIME_WARNING_FLAG=1 return 100 fi return 0 } check_memory() { local instance=$1 local limit=$2 local pid=$(get_xattr pid /cvmfs/$instance) || return 100 local rss="$(sudo cat /proc/$pid/status | grep VmRSS)" || return 101 local rss_kb=$(echo $rss | awk '{print $2}') if [ $rss_kb -gt $limit ]; then local elements=$(sudo cvmfs_talk -i $instance internal affairs | grep '^inode_tracker.no_reference' | cut -d\| -f2) echo "Memory limit of $limit kB was exceeded by $instance which used $rss_kb kB" echo "We've had $elements items in the inode tracker" echo "Output of cat /proc/$pid/status:" sudo cat /proc/$pid/status 2>&1 CVMFS_MEMORY_WARNING_FLAG=1 return 102 fi return 0 } min() { [ $1 -lt $2 ] && echo $1 || echo $2 } destroy_repo() { local repo=$1 shift 1 local additional_flags="$@" if [ x"$CVMFS_TEST_S3_CONFIG" != x"" ] && [ x"$CVMFS_TEST_S3_STORAGE" != x"" ]; then echo "Wiping repository S3 storage ${CVMFS_TEST_S3_STORAGE}/${repo}" sudo rm -rf "${CVMFS_TEST_S3_STORAGE}/${repo}" fi load_repo_config $repo if [ x"$TEST_CVMFS_RECEIVER_UPSTREAM_STORAGE" != x"" ]; then # It is possible to have publisher and gateway on the same machine for testing purposes. # However, `cvmfs_server rmfs` then incorrectly does not delete the repository backend storage. # Let's do it here instead. echo "Wiping local repository storage in gateway setup /srv/cvmfs/${repo}" sudo rm -rf /srv/cvmfs/${repo} fi sudo cvmfs_server rmfs -f $additional_flags $repo || return 100 } has_repo() { local repo=$1 cvmfs_server list 2>/dev/null | grep "^$repo " | grep -q -F "$repo" } create_repo() { local repo=$1 local uid=$2 local debug_log=$3 shift $(min $# 3) echo "Shutting down autofs for the cvmfs mounts" autofs_switch off || return 100 if has_repo $repo; then echo "Repository $repo is already present... removing it" destroy_repo $repo || return 101 fi # Note: We need to do a non-graceful (brutal?) restart of Apache on Fedora 26+ to avoid # a timeout issue if [ x"$(which apachectl)" != x"" ]; then sudo apachectl restart fi echo -n "Creating new repository $repo (with extra options "$@") ..." local s3_config="" local stratum0="" local unionfs="" if [ x"$CVMFS_TEST_S3_CONFIG" != x"" ]; then echo " S3 Config: $CVMFS_TEST_S3_CONFIG" s3_config=" -s $CVMFS_TEST_S3_CONFIG" fi if [ x"$CVMFS_TEST_HTTP_BASE" != x"" ]; then echo " Stratum0: $CVMFS_TEST_HTTP_BASE" stratum0=" -w $CVMFS_TEST_HTTP_BASE" fi if [ x"$CVMFS_TEST_UNIONFS" != x"" ]; then echo " UnionFS: $CVMFS_TEST_UNIONFS" unionfs=" -f $CVMFS_TEST_UNIONFS" fi sudo -E cvmfs_server mkfs -o $uid -m \ -a ${CVMFS_TEST_HASHALGO} \ $s3_config \ $stratum0 \ $unionfs \ "$@" $repo || return 102 local client_conf="/etc/cvmfs/repositories.d/${repo}/client.conf" if [ x$debug_log != x -a x$debug_log != xNO ]; then echo "CVMFS_DEBUGLOG=$debug_log" | sudo tee -a $client_conf sudo sed -i.bak -e "s/^\(cvmfs2#$repo.*\)\(allow_other.*\)/\1debug,\2/" /etc/fstab fi } create_empty_repo() { local repo=$1 local uid=$2 local debug_log=$3 shift $(min $# 3) create_repo $repo $uid $debug_log "$@" || return 101 if [ -f /cvmfs/$repo/new_repository ]; then sudo cvmfs_server transaction $repo || return 102 rm -f /cvmfs/$repo/new_repository sudo cvmfs_server publish $repo || return 103 fi } # creates a cvmfs repository and fills it with some dummy data # # @param repo_name the name of the repository to create # @param uid the user id of the new repository's owner create_filled_repo() { local repo=$1 local uid=$2 local debug_log=$3 create_empty_repo $repo $uid $debug_log || return 101 sudo cvmfs_server transaction $repo || return 102 pushdir /cvmfs/$repo echo "meaningless file content" > file echo "more clever file content" > clever ln file hardlinkToFile ln -s clever symlinkToClever mkdir -p foo/bar/baz mkdir -p bar/foo/baz touch foo/.cvmfscatalog # Put some meaningful stuff into files echo "Vom Eise befreit sind Strom und Bäche" > foo/bar/verse1 echo "Durch des Frühlings holden, belebenden Blick," > foo/bar/verse2 echo "Im Tale grünet Hoffnungsglück;" > foo/bar/verse3 echo "Der alte Winter, in seiner Schwäche," > foo/bar/verse4 echo "Zog sich in rauhe Berge zurück." > foo/bar/verse5 popdir sudo cvmfs_server publish $repo || return 103 } # generates a huge dummy repository content with 500.000 directory entries # in a somewhat representative directory structure # # @param repo_dir where to put the dummy dirents make_huge_repo() { local repo_dir=$1 $CVMFS_PYTHON2 ${TEST_ROOT}/common/mock_services/make_repo.py \ --max-dir-depth 7 \ --num-subdirs 5 \ --num-files-per-dir 5 \ --min-file-size 0 \ --max-file-size 5120 \ $repo_dir } import_repo() { local repo=$1 local uid=$2 shift 2 local extra_options="$*" echo "Importing repository $repo..." local unionfs="" if [ x"$CVMFS_TEST_UNIONFS" != x"" ]; then echo " UnionFS: $CVMFS_TEST_UNIONFS" unionfs=" -f $CVMFS_TEST_UNIONFS" fi sudo -E cvmfs_server import -o $uid $unionfs $extra_options $repo || return 102 } get_stratum1_name() { local repo_name="$1" echo "$repo_name.$(date +%s%N | md5sum | head -c6).replica" } # creates a repository replica, as is done on stratum 1s # @param replica_name the name of the new replica # @param replica_owner the owner of the new replica # @param stratum0_url the url for the source repository # @param stratum0_pub the public key the repository create_stratum1() { local replica_name=$1 local replica_owner=$2 local stratum0_url=$3 local stratum0_pub=$4 shift 4 local additional_parameters="$*" local s3_config="" if [ x"$CVMFS_TEST_S3_CONFIG" != x"" ]; then local stratum1_url="$CVMFS_TEST_HTTP_BASE" [ ! -z "$stratum1_url" ] || die "\$CVMFS_TEST_HTTP_BASE needs to be set for S3 tests to work!" echo " S3 Config: $CVMFS_TEST_S3_CONFIG" echo " Stratum1: $stratum1_url" s3_config=" -s $CVMFS_TEST_S3_CONFIG -w $stratum1_url -a" fi sudo -E cvmfs_server add-replica -o $replica_owner \ -n $replica_name \ $s3_config \ $additional_parameters \ $stratum0_url \ $stratum0_pub } # creates a repository replica as a pass-through to a stratum 0 # @param replica_name the name of the new replica # @param replica_owner the owner of the new replica # @param stratum0_url the url for the source repository create_passthrough_stratum1() { local replica_name=$1 local replica_owner=$2 local stratum0_url=$3 shift 3 local additional_parameters="$*" local s3_config="" if [ x"$CVMFS_TEST_S3_CONFIG" != x"" ]; then local stratum1_url="$CVMFS_TEST_HTTP_BASE" [ ! -z "$stratum1_url" ] || die "\$CVMFS_TEST_HTTP_BASE needs to be set for S3 tests to work!" echo " S3 Config: $CVMFS_TEST_S3_CONFIG" echo " Stratum1: $stratum1_url" s3_config=" -s $CVMFS_TEST_S3_CONFIG -w $stratum1_url -a" fi sudo -E cvmfs_server add-replica -P \ -o $replica_owner \ -n $replica_name \ $s3_config \ $additional_parameters \ $stratum0_url } # creates a big file at a given location with a given size # @param path the location of the new file # @param size the desired size in megabytes create_big_file() { local path=$1 local size=$2 dd if=/dev/zero of=$path bs=1024k count=$size } # generate a custom style recursive listing with only the following infos: # - file name # - linkcount # - file mode # - file size # - parent directory # - symlink destination (when applicable) # # @param directory the directory to be listed # @param show_linkcount yes be default, otherwise linkcount is left out # @return a custom directory listing create_listing() { local directory=$1 local show_linkcount=$2 local lst lst=$(ls --almost-all --recursive -l --file-type --time-style=+ $directory | \ awk ' { # skip the total file count, ls prints at the end of each directory listing if(substr($1, 0, 5) == "total") next; # length of base directory path base_dir_length = length(base_dir) # truncate base path from the path printed before each new listing if(substr($0, 0, base_dir_length) == base_dir) { print substr($0, base_dir_length + 1, length($0)-1) next; } # first character first = substr($0, 0, 1); # print file meta information # $1 => file mode (rwx bits, ...) # $2 => linkcount # $3,$4 => owner, group (might also be provided from outside) # $5 => file size (skipped for directories) # $6 => file name # $7,$8 => symlink destination (only for symlinks :o) ) if (first != "-" && first != "d" && first != "l") { printf "\n" next; } # print out file information printf $1 " "; if (show_linkcount == "yes") { printf $2 " "; } printf $3 " "; printf $4 " "; printf $6; if(first == "l") printf " " $7 " " $8; if(first != "d") printf " " $5; printf "\n" }' base_dir=$directory show_linkcount=$show_linkcount) echo -e "$lst" } # compares the file contents and file meta data of two directories # Note: function creates the files 'listingFile1' and 'listingFile2' in `pwd` # `pwd` should NOT be part of the comparison!! # # @param dir1 the directory to probe # @param dir2 the ground truth directory # @param repo the underlying repository name (optional - to detect overlayfs) # @return != 0 to indicate inequality, 0 means success compare_directories() { local dir1=$1 local dir2=$2 local repo_name=$3 local show_linkcount="yes" if [ ! -z $repo_name ] && uses_overlayfs $repo_name; then echo "NOTE: $repo_name is based on OverlayFS - ignoring link counts" show_linkcount="no" fi listing1=$(create_listing $dir1 $show_linkcount) listing2=$(create_listing $dir2 $show_linkcount) echo "check if directory structure and file meta data fits" local listingFile1="listing_$(basename $dir1)" local listingFile2="listing_$(basename $dir2)" echo -e "$listing1" > $listingFile1 echo -e "$listing2" > $listingFile2 diff -uN $listingFile1 $listingFile2 2>&1 || return 101 echo "check if the file contents in both directories are the same" diff -ruN $dir1 $dir2 2>&1 || return 102 return 0 } # run a binary in the background and return it's PID through stdout # # @param logfile a path to a logfile for outputs of the started program # @return the process ID of the created background service run_background_service() { local logfile=$1 shift 1 local srv_pid # shell for the win! # This spawns a shell executing the passed command string (sudo sh -c) which # will print it's PID (echo $$) into a fifo and execute the user defined # binary redirecting its outputs into the logfile (exec $@ >> $logfile 2>&1). # Furthermore it disconnects the new process from the executing shell (nohup), # satisfies the I/O requirements of nohup (2>&1 < /dev/null) and sends it to # the background (&) # Credit: Dario Berzano helped with that monster! local fifo="pid_fifo" mkfifo $fifo || return 1 nohup sh -c "echo \$\$ > $fifo; exec $@ >> $logfile 2>&1" 2>&1 < /dev/null & srv_pid=$(cat $fifo) rm -f $fifo # check if the background process is running # TODO(rmeusel): Fix this race. If the background service is a short living # process it might have successfully terminated at this point. if ! sudo kill -0 $srv_pid > /dev/null 2>&1; then return 4 fi # print the PID of the background process and return successfully echo $srv_pid return 0 } # open a port for incoming connections. It will accept the connection but stay # silent on it # # Note: The user is responsible for killing the created server process after # usage # # @param protocol either UDP or TCP # @param port the desired port number to be opened # @param logfile a path to the logfile where connection logs should be written # @return the process ID of the created server script open_silent_port() { local protocol=$1 local port=$2 local logfile=$3 local cmd local mock mock="${CVMFS_PYTHON2} ${TEST_ROOT}/common/mock_services/silent_socket.py $protocol $port" echo "Calling $mock" >> $logfile # Do similar fifo-trick as in run_background_service() to get the PID of the # actual silent_socket.py process rather than just the `sudo` around it. local fifo="pid_silent_port_fifo" mkfifo $fifo || return 1 cmd="sudo sh -c \"echo \\\$\\\$ > $fifo; exec $mock\"" run_background_service $logfile "$cmd" > /dev/null 2>&1 || return $? cat $fifo # return the PID of the `python silent_socket.py` } # spawns a mocked HTTP server on a given port and document root location # Note: The user is responsible to kill the server process once after usage # # @param document_root absolute path to be served via HTTP # @param port http port to be used # @param logfile (optional) where to log the HTTP server's status info # @return PID of HTTP server or >0 on error open_http_server() { local document_root="$1" local port="$2" local logfile="${3:=/dev/null}" local http_server="${CVMFS_PYTHON2} ${TEST_ROOT}/common/mock_services/http_server.py" local http_pid= local http_sentinel="http://localhost:${port}/http_sentinel" # spawning the server http_pid=$(run_background_service $logfile "$http_server -p $port -r $document_root") touch ${document_root}/$(basename $http_sentinel) # waiting for the server to come up local timeout=15 while ! curl -Is $url ${http_sentinel} | head -n1 | grep "200 OK" >> $logfile 2>&1 && \ [ $timeout -gt 0 ]; do sleep 1 if ! kill -0 $http_pid > /dev/null 2>&1;then return 1 fi timeout=$(( $timeout - 1 )) done [ $timeout -gt 0 ] || return 2 echo "$http_pid" } # checks if a nested catalog is part of the current catalog configuration # of the repository # @param catalog_path the catalog root path to be checked # @param repo_name the repository to be checked check_catalog_presence() { local catalog_path=$1 local repo_name=$2 cvmfs_server list-catalogs -x $repo_name | grep -x $catalog_path > /dev/null 2>&1 return $? } # counts the number of present catalogs in the repository # @param repo_name the name of the repository to investigate # @return the number of found catalogs get_catalog_count() { local repo_name=$1 echo $(cvmfs_server list-catalogs -x $repo_name | wc -l) } # read one specific manifest field of the specified repository # @param field_tag the manifest field to be retrieved (one character) # @param (stdin) the manifest to be investigated # return the content of the specified manifest field get_manifest_field_from_manifest() { local field_tag="$1" grep --text -e "^$field_tag" | head -n1 | sed "s/^$field_tag\(.*\)$/\1/" } # read one specific manifest field of the specified repository # @param repo_name the repository to be investigated # @param field_tag the manifest field to be retrieved (one character) # return the content of the specified manifest field get_manifest_field() { local repo_name=$1 local field_tag=$2 load_repo_config $repo_name local manifest_url="$(get_repo_url $repo_name)/.cvmfspublished" curl -s "$manifest_url" | get_manifest_field_from_manifest "$field_tag" } # retrieves the hash of the current root catalog of the given # repository by investigating the manifest # @param repo_name the name of the repository # @return the current root catalog hash get_current_root_catalog() { local repo_name=$1 local manifest_url="$(get_repo_url $repo_name)/.cvmfspublished" get_manifest_field $repo_name "C" } # downloads and decompresses the root catalog of a given repository for deeper # inspection # @param repo_name the repository to download the root catalog from # @param object_hash the object to be downloaded and decompressed # @return prints the file name of the extracted catalog to stdout get_and_decompress_root_catalog() { local repo_name=$1 local catalog_hash="$(get_current_root_catalog $repo_name)C" download_and_decompress_object $repo_name $catalog_hash } # uses `cvmfs_server check` to check the integrity of the catalog structure # additionally it might check the backend storage integrity as well when # provided with -i # @param repo the repository to be checked check_repository() { local repo=$1 shift 1 cvmfs_server check $@ $repo || return 100 return 0 } check_repo_integrity() { local repo=$1 for f in `find /cvmfs/${repo} -type f`; do cat $f >/dev/null if [ $? -ne 0 ]; then echo "Integrity check: corrupted file ${f}" return 1 fi done return 0 } # wrapper function to start a new repository update transaction # @param repo the repository you want to start the transaction for start_transaction() { local repo=$1 shift cvmfs_server transaction $@ $repo || return 100 return 0 } # wrapper function to abort a transaction # @param repo the repository you want to abort a transaction on abort_transaction() { local repo=$1 cvmfs_server abort -f $repo || return 100 return 0 } # wrapper function to rollback a repository to a specified tag # # @param repo the repository you want to rollback # @param tag the name of the tag to be rollbacked to rollback_repo() { local repo=$1 local tag=$2 cvmfs_server rollback -t $tag -f $repo || return 100 return 0 } # wrapper function to publish an repository after its contents were updated. # @param repo the repository name to start the transaction in publish_repo() { local repo=$1 shift 1 # enable the debug mode? # in debug mode we redirect output directly to the interactive shell, # overriding any redirections to logfiles or whatever... We want to hack! case $CVMFS_TEST_SRVDEBUG in fail) cvmfs_server publish -dv "$@" $repo > /dev/tty 2>&1 || return 100 ;; startup) cvmfs_server publish -Dv "$@" $repo > /dev/tty 2>&1 || return 100 ;; *) cvmfs_server publish -v "$@" $repo || return 100 ;; esac return 0 } # sources the repository server configuration file for the given repo name # @param name the name of the repository to load the configs for load_repo_config() { local name=$1 local conf_dir="/etc/cvmfs/repositories.d/${name}" [ ! -f "${conf_dir}/server.conf" ] || . ${conf_dir}/server.conf [ ! -f "${conf_dir}/client.conf" ] || . ${conf_dir}/client.conf } get_millisecond_epoch() { if running_on_osx; then echo $(( $(date +%s) * 1000 )) else echo $(( $(date +%s%N) / 1000000 )) fi } get_iso8601_timestamp() { date -u +'%Y-%m-%dT%H:%M:%SZ' } milliseconds_to_seconds() { local milliseconds="$1" # add a zero padding if necessary while [ $(echo -n "$milliseconds" | wc -c) -lt 4 ]; do milliseconds="0$milliseconds" done # insert a decimal point to make seconds digits=$(echo -n "$milliseconds" | wc -c) echo -n "$milliseconds" | head -c $(( $digits - 3 )) echo -n "." echo -n "$milliseconds" | tail -c 3 } milliseconds_to_human_readable() { local milliseconds="$1" local scratch=$milliseconds local seconds=1000 local minutes=$(( $seconds * 60 )) local hours=$(( $minutes * 60)) local days=$(( $hours * 24 )) local o_days=$(( $scratch / $days )) if [ $o_days -gt 0 ]; then echo -n "$o_days days " scratch=$(( $scratch % $days )) fi local o_hours=$(( $scratch / $hours )) if [ $o_days -gt 0 ] || [ $o_hours -gt 0 ]; then echo -n "$o_hours hours " scratch=$(( $scratch % $hours )) fi local o_minutes=$(( $scratch / $minutes )) if [ $o_days -gt 0 ] || [ $o_hours -gt 0 ] || [ $o_minutes -gt 0 ]; then echo -n "$o_minutes minutes " scratch=$(( $scratch % $minutes )) fi echo "$(milliseconds_to_seconds $scratch) seconds" } # runs the given command and measures the execution time in milliseconds stop_watch() { begin=$(get_millisecond_epoch) cmd="$@" $cmd res=$? end=$(get_millisecond_epoch) echo $((end - begin)) return $res } # wrapper around pushd (that is not available in dash) # if pushd is not available, a normal `cd` is used # Note: We do not reimplement the whole functionality of pushd, with our work- # around it is only possible to have ONE directory-level stored pushdir() { type pushd > /dev/null 2>&1 if [ $? -eq 0 ]; then pushd $1 return $? else cd $1 return $? fi } # wrapper around popd (that is not available in dash) # if popd is not available, a normal `cd -` is used to go back to the last dir # Note: We do not reimplement the whole functionality of popd, with our work- # around it is only possible to have ONE directory-level stored popdir() { type popd > /dev/null 2>&1 if [ $? -eq 0 ]; then popd $1 return $? else cd - return $? fi } # retrieves the apache version string "2.x.xx" get_apache_version() { ${APACHE_BIN} -v | head -n1 | \ sed 's/^Server version: Apache\/\([0-9]\+\.[0-9]\+\.[0-9]\+\).*$/\1/' } # figure out apache config file mode # # @return apache config mode (stdout) (see globals below) APACHE_CONF_MODE_CONFD=1 # *.conf goes to ${APACHE_CONF}/conf.d APACHE_CONF_MODE_CONFAVAIL=2 # *.conf goes to ${APACHE_CONF}/conf-available get_apache_conf_mode() { [ -d /etc/${APACHE_CONF}/conf-available ] && echo $APACHE_CONF_MODE_CONFAVAIL \ || echo $APACHE_CONF_MODE_CONFD } # find location of apache configuration files # # @return the location of apache configuration files (stdout) get_apache_conf_path() { local res_path="/etc/${APACHE_CONF}" if [ x"$(get_apache_conf_mode)" = x"$APACHE_CONF_MODE_CONFAVAIL" ]; then echo "${res_path}/conf-available" elif [ -d "${res_path}/modules.d" ]; then echo "${res_path}/modules.d" else echo "${res_path}/conf.d" fi } # returns the apache configuration string for 'allow from all' # Note: this is necessary, since apache 2.4.x formulates that different # # @return a configuration snippet to allow s'th from all hosts (stdout) get_compatible_apache_allow_from_all_config() { local minor_apache_version=$(version_minor "$(get_apache_version)") if [ $minor_apache_version -ge 4 ]; then echo "Require all granted" else local nl=' ' echo "Order allow,deny${nl} Allow from all" fi } # writes apache configuration file # This figures out where to put the apache configuration file depending # on the running apache version # Note: Configuration file content is expected to come through stdin # # @param file_name the name of the apache config file (no path!) # @return 0 on success create_apache_config_file() { local file_name=$1 local conf_path conf_path="$(get_apache_conf_path)" # create (or append) the conf file cat - | sudo tee ${conf_path}/${file_name} > /dev/null || return 1 # the new apache requires the enable the config afterwards if [ x"$(get_apache_conf_mode)" = x"$APACHE_CONF_MODE_CONFAVAIL" ]; then sudo a2enconf $file_name > /dev/null || return 2 fi return 0 } # removes apache config files dependent on the apache version in place # Note: As of apache 2.4.x `a2disconf` needs to be called before removal # # @param file_name the name of the conf file to be removed (no path!) # @return 0 on successful removal remove_apache_config_file() { local file_name=$1 local conf_path conf_path="$(get_apache_conf_path)/${file_name}" # disable configuration on newer apache versions if [ x"$(get_apache_conf_mode)" = x"$APACHE_CONF_MODE_CONFAVAIL" ]; then sudo a2disconf $file_name > /dev/null 2>&1 || return 1 fi # remove configuration file sudo rm -f $conf_path } get_hash_path() { local object_hash="$1" local object_path="$(echo -n $object_hash | sed -e 's/^\(..\)\(.*\)$/\1\/\2/')" echo "${object_path}" } get_local_repo_storage() { local repo_name="$1" echo "/srv/cvmfs/${repo_name}" } get_local_repo_object() { local repo_name="$1" local object_hash="$2" local repo_storage="$(get_local_repo_storage $repo_name)" local object_path="$(get_hash_path $object_hash)" echo "${repo_storage}/data/${object_path}" } get_local_repo_url() { local repo_name="$1" echo "http://localhost/cvmfs/${repo_name}" } get_s3_repo_url() { local repo_name="$1" local s3_stratum0_url="$CVMFS_TEST_HTTP_BASE" [ x"$s3_stratum0_url" != x"" ] || die "\$CVMFS_TEST_HTTP_BASE needs to be set for S3 tests to work!" [ $(echo -n "$s3_stratum0_url" | tail -c1) = "/" ] || s3_stratum0_url="${s3_stratum0_url}/" echo "${s3_stratum0_url}${repo_name}" } get_repo_url() { local repo_name="$1" if [ x"$CVMFS_TEST_S3_CONFIG" != x"" ]; then get_s3_repo_url "$repo_name" else get_local_repo_url "$repo_name" fi } get_object_url() { local repo_name="$1" local object_hash="$2" local repo_url="$(get_repo_url $repo_name)" local object_path="$(get_hash_path $object_hash)" echo "${repo_url}/data/${object_path}" } get_apache_config_filename() { local repo_name="$1" echo "${repo_name}.conf" } get_backend_spooler() { if [ -n "${TEST_CVMFS_RECEIVER_UPSTREAM_STORAGE}" ] ; then # A gateway test is running: access the backend storage directly echo "${TEST_CVMFS_RECEIVER_UPSTREAM_STORAGE}" else echo "${CVMFS_UPSTREAM_STORAGE}" fi } cleanup_legacy_repo_leftovers() { local legacy_repo_name="$1" local legacy_repo_storage="$(get_local_repo_storage $legacy_repo_name)" local apache_config_file="$(get_apache_config_filename $legacy_repo_name)" echo -n "cleanup (if necessary)... " has_repo $legacy_repo_name && die "fail" if [ -d $legacy_repo_storage ]; then sudo rm -fR $legacy_repo_storage fi if [ -f /etc/cvmfs/keys/${legacy_repo_name}.crt ] || \ [ -f /etc/cvmfs/keys/${legacy_repo_name}.pub ] || \ [ -f /etc/cvmfs/keys/${legacy_repo_name}.key ] || \ [ -f /etc/cvmfs/keys/${legacy_repo_name}.masterkey ]; then sudo rm -f /etc/cvmfs/keys/${legacy_repo_name}.* fi remove_apache_config_file $apache_config_file echo "done" } # resigns a freshly planted legacy repository (see plant_legacy_repo()) # Note: This expects a matching keychain in /etc/keys/{$repo_name} # @param repo_name the FQRN of the legacy repository to be resigned _resign_legacy_repository() { local repo_name="$1" local repo_storage="$(get_local_repo_storage $repo_name)" local legacy_repo_storage="${repo_storage}" [ -d ${repo_storage}/pub ] && legacy_repo_storage="${legacy_repo_storage}/pub/catalogs" # for 2.0.x style repositories local tmp_dir="$(pwd)/tmp" local upstream="local,${legacy_repo_storage}/data/txn,${legacy_repo_storage}" mkdir -p $tmp_dir || return 1 # recreate whitelist local whitelist=${tmp_dir}/whitelist.${repo_name} echo `date -u "+%Y%m%d%H%M%S"` > ${whitelist}.unsigned echo "E`date -u --date='next month' "+%Y%m%d%H%M%S"`" >> ${whitelist}.unsigned echo "N$repo_name" >> ${whitelist}.unsigned openssl x509 -in /etc/cvmfs/keys/${repo_name}.crt -outform der | \ cvmfs_publish hash -a sha1 -f >> ${whitelist}.unsigned || return 2 local hash; hash=`cat ${whitelist}.unsigned | cvmfs_publish hash -a sha1` echo "--" >> ${whitelist}.unsigned echo $hash >> ${whitelist}.unsigned echo -n $hash > ${whitelist}.hash openssl rsautl -inkey /etc/cvmfs/keys/${repo_name}.masterkey -sign -in ${whitelist}.hash -out ${whitelist}.signature || return 3 cat ${whitelist}.unsigned ${whitelist}.signature | sudo tee ${legacy_repo_storage}/.cvmfswhitelist > /dev/null || return 4 rm -f ${whitelist}.unsigned ${whitelist}.signature ${whitelist}.hash || return 5 } # extracts a tarball into the root directory! # Note: it expects the tarball to define the full destination path. # @param tarball the path to the tarball to be planted # @param verbose print the verbose output of tar plant_tarball() { local tarball="$1" local verbose="$2" local olddir="$(pwd)" local tar_params="xzf" if [ x"$verbose" != x"" ]; then tar_params="${tar_params}v" fi cd / || return 101 sudo tar $tar_params $tarball || return 102 cd $olddir || return 103 } # extracts a legacy repository contained in a tarball. # Note: This expects the tarball to define the full destination path. Furthermore # it expects a repo keychain to be planted in /etc/keys/${legacy_repo_name} # @param tarball path to the tarball containing the legacy repo # @param legacy_repo_name FQRN of the repository to be planted # @param repo_owner user ID that is supposed to own the repository plant_legacy_repository_revision() { local tarball="$1" local legacy_repo_name="$2" local repo_owner="$3" local verbose="$4" local legacy_repo_storage="$(get_local_repo_storage $legacy_repo_name)" plant_tarball "$tarball" "$verbose" || return 104 # figure out if we just planted a CVMFS 2.0.x repository or something newer local is_cvmfs_20=0 [ -d ${legacy_repo_storage}/pub ] && is_cvmfs_20=1 # do a couple of sanity checks and adjustments if [ $is_cvmfs_20 -eq 1 ]; then [ -d ${legacy_repo_storage}/pub ] || return 105 [ -f ${legacy_repo_storage}/pub/catalogs/.cvmfscatalog ] || return 106 [ -f ${legacy_repo_storage}/pub/catalogs/.cvmfswhitelist ] || return 107 sudo touch ${legacy_repo_storage}/pub/catalogs/.cvmfs_master_replica || return 108 else [ -d ${legacy_repo_storage}/data ] || return 105 [ -f ${legacy_repo_storage}/.cvmfspublished ] || return 106 [ -f ${legacy_repo_storage}/.cvmfswhitelist ] || return 107 sudo touch ${legacy_repo_storage}/.cvmfs_master_replica || return 108 fi _resign_legacy_repository "$legacy_repo_name" || return 109 } # creates a path into the CAS of CVMFS server # # @param content_hash the content hash to be checked make_path() { local content_hash="$1" local characters=$(echo -n $content_hash | wc -c) local head_cnt=2 local tail_cnt=$(( $characters - $head_cnt )) echo "data/$(echo -n $content_hash | head -c $head_cnt)/$(echo -n $content_hash | tail -c $tail_cnt)" } peek_backend_raw() { local repo_name="$1" local remote_path="$2" load_repo_config $repo_name cvmfs_swissknife peek -d $remote_path -r $(get_backend_spooler) } # peeks in the upstream storage to find a given content hash # # @param repository the repository name to be queried # @param content_hash the content has to be checked for availability # @return 0 on successfully found object peek_backend() { local repo_name="$1" local content_hash="$2" peek_backend_raw $repo_name $(make_path $content_hash) } # downloads a random file from the backend storage of a CVMFS repository # # @param repository the repository name to uploaded into # @param remote_path the relative remote path to be uploaded into # @param local_path the destination path to be downloaded into download_from_backend() { local repo_name="$1" local remote_path="$2" local local_path="$3" load_repo_config $repo_name curl -o $local_path -s "$(get_repo_url $repo_name)/${remote_path}" } # removes a random file from the backend storage of a CVMFS repository # # @param repository the repository name to uploaded into # @param remote_path the relative remote path to be uploaded into delete_from_backend() { local repo_name="$1" local remote_path="$2" load_repo_config $repo_name cvmfs_swissknife remove -o $remote_path -r $(get_backend_spooler) } # uploads a random file into the backend storage of a CVMFS repository # # @param repository the repository name to uploaded into # @param file_path the file to be uploaded # @param remote_path the relative remote path to be uploaded into upload_into_backend() { local repo_name="$1" local file_path="$2" local remote_path="$3" load_repo_config $repo_name local spooler=$(get_backend_spooler) if [ x"$(echo "$spooler" | cut -f1 -d',')" = x"S3" ]; then delete_from_backend $repo_name $remote_path # S3 does a lazy upload fi cvmfs_swissknife upload -i $file_path \ -o $remote_path \ -r $spooler \ -a $CVMFS_HASH_ALGORITHM } # downloads and decompresses a backend object from a given repository # @param repo_name the repository to download the root catalog from # @param object_hash the object to be downloaded and decompressed # @return prints the file name of the extracted catalog to stdout download_and_decompress_object() { local repo_name="$1" local object_hash="$2" local object_url="$(get_object_url $repo_name $object_hash)" local destination="${object_hash}.uncompressed" curl --silent \ --show-error $object_url \ | cvmfs_swissknife zpipe -d > $destination || return 1 echo $destination } # reads the upstream config part of a CernVM-FS server configuration file # This is the third comma-separated field in the upstream string read_upstream_config() { local upstream_string="$1" echo "$upstream_string" | cut -f3 -d',' } # switches on/off the garbage collection for a given repository # # @param name the name of the repository to be changed # @return 0 return code on success toggle_gc() { local name=$1 local cfg_file="/etc/cvmfs/repositories.d/${name}/server.conf" local tmp_file="$(mktemp)" local value="$(cat $cfg_file | grep 'CVMFS_GARBAGE_COLLECTION' | sed -e 's/.*=\([a-z]\+\)/\1/')" local new_value="true" [ x"$value" = x"true" ] && new_value="false" cat $cfg_file | sed -e "s/\(CVMFS_GARBAGE_COLLECTION\)=[a-z]\+/\1=$new_value/" > $tmp_file sudo cp -f $tmp_file $cfg_file || return 1 rm -f $tmp_file || return 2 return 0 } get_object_from_manifest() { local name=$1 local type=$2 local url="$(get_repo_url $name)" hash=$(cvmfs_swissknife info -r $url -R | grep "^$type" | tr -d $type) echo "${hash}${type}" } set_auto_tag_timespan() { local name=$1 local timespan="$2" local cfg_file="/etc/cvmfs/repositories.d/${name}/server.conf" local tmp_file="$(mktemp)" cat "$cfg_file" > "$tmp_file" || return 1 cat >> $tmp_file << EOF CVMFS_AUTO_TAG_TIMESPAN="$timespan" EOF sudo cp $tmp_file $cfg_file || return 2 rm -f $tmp_file return 0 } # disables the automatic garbage collection of a given repository configuration # # @param name the name of the repository to reconfigure disable_auto_garbage_collection() { local name="$1" local cfg_file="/etc/cvmfs/repositories.d/${name}/server.conf" if cat $cfg_file | grep -q 'CVMFS_AUTO_GC'; then local tmp_cfg="$(mktemp)" cat $cfg_file | sed -e 's/^\(CVMFS_AUTO_GC\)=.*$/\1=false/' > $tmp_cfg || return 1 sudo mv -f $tmp_cfg $cfg_file || return 2 sudo chmod 0644 $cfg_file || return 3 fi } # get the current unix timestamp get_timestamp() { date --date='2 seconds ago' +@%s } # display a timestamp in human readable form display_timestamp() { local timestamp=$1 date --date="$timestamp" } # this copies all top-level files from source to destination ignoring symlinks # Note: Using the content of /bin or /usr/bin as integration test guinea pig was # never a good idea. This is pretty much technical debt for this poor # decision. _cp_bindir() { local source_directory="$1" local destination_directory="$2" for f in $(find ${source_directory}/ -maxdepth 1 -type f); do sudo cp $f $destination_directory || return 1 done sudo chmod -R a+rw $destination_directory || return 2 for f in $(find $destination_directory -type f); do sudo chmod 0444 $f || return 3 done } # copy all content of /bin/* into a given directory (for server testing purposes) # # @param destination_directory cp_bin() { local destination_directory="$1" _cp_bindir "/bin" "$destination_directory" } # copy all content of /usr/bin/* into a given directory (for server testing purposes) # # @param destination_directory cp_usrbin() { local destination_directory="$1" _cp_bindir "/usr/bin" "$destination_directory" } # prints a benchmark log message benchmark_log() { local log_message=$1 echo "=========== ${log_message}" } # waits until cvmfs is mounted wait_fqrn_mount() { if [ x$PARROT_ENABLED = "xTRUE" ]; then return 0 fi local waits=0 while ! mount | grep -q "/cvmfs/${FQRN}"; do waits=$(( waits + 1 )) if [ "$waits" -ge 120 ]; then die "${FQRN} could not be mounted" fi sleep 1 done } # runs the command once, mounting and unmounting the repository single_execution() { cvmfs2 -f -o config=cvmfs.conf,disable_watchdog,simple_options_parsing $FQRN /cvmfs/$FQRN 2>&1 & wait_fqrn_mount ( cvmfs_run_benchmark ) local code=$? cvmfs_umount $FQRN return $code } # creates the default cvmfs.conf file for normal testing # # @param destination_file absolute path of the directory create_default_cvmfs_config() { local destination_directory="$1" local config_file=$destination_directory/cvmfs.conf local test_folder=$( cd $(dirname $0) ; pwd -P ) cat > $config_file << EOF CVMFS_CACHE_BASE=${CVMFS_OPT_CACHEDIR} CVMFS_RELOAD_SOCKETS=${CVMFS_OPT_CACHEDIR} CVMFS_SERVER_URL=http://cvmfs-stratum-one.cern.ch/cvmfs/@fqrn@ CVMFS_SHARED_CACHE=no CVMFS_HTTP_PROXY=${CVMFS_TEST_PROXY} CVMFS_AUTO_UPDATE=false CMS_LOCAL_SITE=${test_folder}/benchmarks/004-cms CVMFS_QUOTA_LIMIT=-1 CVMFS_KEYS_DIR=/etc/cvmfs/keys/cern.ch EOF } clean_benchmark_cache() { local cachedir="${CVMFS_OPT_CACHEDIR}/${FQRN}" if [ -d "$cachedir" ] && [ "x$cachedir" != x ]; then list=$(ls $cachedir | grep -P "^[0-9a-f]{2}$") for dir in $list; do rm -f "$cachedir/$dir"/* done fi } # sets the valgrind options depending on the test type # # @param test_type concrete test to execute set_valgrind_options() { local option="$1" local CVMFS_OPT_CALLGRIND="--tool=callgrind --dump-before=fuse_session_loop_mt --dump-after=fuse_session_loop_mt --dsymutil=yes --log-file=${FQRN}.callgrind.log -v" local CVMFS_OPT_MEMCHECK="--tool=memcheck --xml=yes --xml-file=${FQRN}.memcheck.xml --leak-check=yes --log-file=${FQRN}.memcheck.log -v" OPTIONS="" case $option in callgrind) OPTIONS=$CVMFS_OPT_CALLGRIND ;; memcheck) OPTIONS=$CVMFS_OPT_MEMCHECK ;; esac } # creates the environment to run the benchmark # # @param workspace absolute path to the directory where the benchmark will put the files in setup_benchmark_environment() { local workspace="$1" local me=$(whoami) mkdir -p "$CVMFS_OPT_OUTPUT_DIR/$FQRN" mkdir -p $workspace/cache # make sure the user can execute /bin/fusermount, write into /dev/fuse and write into /cvmfs/<repo> sudo chmod 666 /dev/fuse sudo chmod 4755 /bin/fusermount sudo mkdir -p /cvmfs/$FQRN sudo chown $me /cvmfs/$FQRN # mtab trick if [ ! -h /etc/mtab ]; then sudo sh -c "mv /etc/mtab /etc/mtab.save" sudo sh -c "ln -s /proc/mounts /etc/mtab" CVMFS_OPT_MTAB_MODIFIED="true" fi # check that the cache directory exists, is empty and is correctly labeled for SElinux if [ ! -d "$CVMFS_OPT_CACHEDIR" ]; then mkdir -p "$CVMFS_OPT_CACHEDIR" has_selinux local selinux=$? if [ "$selinux" -eq 0 ]; then chcon -Rv --type=cvmfs_cache_t $CVMFS_OPT_CACHEDIR fi else clean_benchmark_cache fi # create a cvmfs.conf file or use a custom one if [ -z "$CVMFS_OPT_CONFIG_FILE" ]; then create_default_cvmfs_config $workspace else cp -f "$CVMFS_OPT_CONFIG_FILE" $workspace/cvmfs.conf fi return 0 } # creates a system.info file file with information about the environment create_info_file() { cat > system.info << EOF - Operating system: $(uname -a) - CernVM-FS version: $(cvmfs2 --version) - cvmfs2 executable md5: $(md5sum $(which cvmfs2)) - CernVM-FS talk: $(cvmfs_talk -p ${CVMFS_OPT_CACHEDIR}/${FQRN}/cvmfs_io.${FQRN} -i ${FQRN} internal affairs) EOF } # creates a statistics file create_statistics_file() { local iteration=$1 local total_time=$2 local memory_median=$3 local memory_max=$4 local statistics_file="${FQRN}_${iteration}.data" benchmark_log "creating the statistics file ${statistics_file} calling cvmfs_talk" cat >$statistics_file << EOF # repo=$FQRN # iterations=$CVMFS_OPT_ITERATIONS # warm_cache=$CVMFS_OPT_WARM_CACHE EOF benchmark_log "Executing cvmfs_talk" cvmfs_talk -p "$CVMFS_OPT_CACHEDIR"/$FQRN/cvmfs_io.$FQRN internal affairs | grep \| | tail -n +2 >> $statistics_file echo "global.sz_total_time|$total_time|Total execution time (seconds)" >> $statistics_file # of only one iteration without valgrind! echo "global.sz_memory_median|$memory_median|Median memory consumption (KB)" >> $statistics_file echo "global.sz_memory_max|$memory_max|Maximum memory consumption (KB)" >> $statistics_file } # collects data during the run statistics_daemon() { local dump_file=$1 local PID=$2 touch $dump_file while [ -f /proc/${PID}/smaps ]; do # memory in KB echo 0 $(sudo awk '/Rss/ {print "+", $2}' /proc/${PID}/smaps) | bc >> $dump_file sleep 2 done } # collects the median of a file collect_median() { local file=$1 local num_lines=$(cat $file | wc -l) local median_pos=$(( num_lines / 2 )) echo $(sort $file | sed "${median_pos}q;d") } # collects the maximum of a file collect_maximum() { echo $(sort $1 | tail -n 1) } # executes the statistics loop execute_statistics_loop() { local code=0 local iteration=1 local PID=0 local processes=() local memory_file="memory.stat" while (( iteration <= $CVMFS_OPT_ITERATIONS && code == 0 )); do local start_time=$(date +%s) # cleaning the cache if necessary before executing the actual benchmark if [ x"$CVMFS_OPT_WARM_CACHE" != x"yes" ]; then clean_benchmark_cache fi cvmfs2 -f -o config=cvmfs.conf,disable_watchdog,simple_options_parsing $FQRN /cvmfs/$FQRN 2>&1 & PID=$! wait_fqrn_mount statistics_daemon $memory_file $PID >/dev/null 2>&1 & # it kills himself when there is no cvmfs process if [ x"$CVMFS_OPT_HOT_CACHE" = x"yes" ]; then benchmark_log "Starting early execution to fill the cache" ( cvmfs_run_benchmark ) > /dev/null 2>&1 local code=$? if [ $code -ne 0 ]; then die "Failed in the hot cache execution" fi fi for i in $(seq 1 1 $CVMFS_OPT_PARALLEL_RUNS); do ( cvmfs_run_benchmark ) > /tmp/$FQRN.log.$i 2>&1 & processes[$i]=$! done for pid in ${processes[@]}; do wait $pid local retval=$? if [ $retval -ne 0 ]; then code=$retval echo "One thread failed in $FQRN. Check /tmp/$FQRN.log" fi done local end_time=$(date +%s) local total_time=$(( end_time - start_time )) # in seconds local memory_median=$(collect_median $memory_file) local memory_max=$(collect_maximum $memory_file) create_statistics_file $iteration $total_time $memory_median $memory_max cvmfs_umount $FQRN wait $PID iteration=$(( iteration + 1 )) done if [ "$code" -ne 0 ]; then benchmark_log "Problem executing the tests. Aborting" benchmark_log $(lsof /cvmfs/${FQRN}) fi return $code } # executes the main valgrind's loop execute_benchmark_loop() { local iteration=1 local code=0 local count=$CVMFS_OPT_ITERATIONS if [ "x$PARROT_ENABLED" = "xTRUE" ]; then benchmark_log "Valgrind is not available with Parrot" exit 100 fi valgrind $OPTIONS cvmfs2 -f -o config=cvmfs.conf,disable_watchdog,simple_options_parsing $FQRN /cvmfs/$FQRN & PID=$! wait_fqrn_mount while (( count > 0 && code == 0 )); do # cleaning the cache if necessary before executing the actual benchmark if [ x"$CVMFS_OPT_WARM_CACHE" != x"yes" ]; then clean_benchmark_cache fi benchmark_log "Starting iteration ${iteration}" ( cvmfs_run_benchmark ) code=$? count=$(( count - 1 )) iteration=$(( iteration + 1 )) done create_info_file if [ "$code" -ne 0 ]; then benchmark_log "Problem executing the tests. Aborting" benchmark_log $(lsof /cvmfs/${FQRN}) exit 100 fi benchmark_log "Iterations finished. Unmounting cvmfs" cvmfs_umount $FQRN wait $PID } # runs a benchmark $CVMFS_OPT_ITERATIONS times run_benchmark() { local statistics_file="$FQRN.data" # use or not warm cache if [ x"$CVMFS_OPT_WARM_CACHE" = x"yes" ]; then benchmark_log "Starting initial execution" single_execution local code=$? if [ $code -ne 0 ]; then die "Failed in the warm cache execution" fi else benchmark_log "Using cold cache" fi # run the statistics collection mode if [ x"$CVMFS_OPT_TALK_STATISTICS" = x"yes" ]; then benchmark_log "Executing the statistics collection" execute_statistics_loop local code=$? if [ $code -ne 0 ]; then die "Failed in the statistics collection" fi fi if [ x"$CVMFS_OPT_VALGRIND" = x"yes" ]; then for tool in $CVMFS_OPT_TEST_TYPE; do set_valgrind_options $tool benchmark_log "Starting loading the profiler and mounting in /cvmfs/${FQRN}" benchmark_log "Running valgrind with the following parameters: $OPTIONS" benchmark_log "Launching the job $CVMFS_OPT_ITERATIONS times" execute_benchmark_loop local code=$? if [ $code -ne 0 ]; then die "Failed in Valgrind" fi done fi } # collects the files and pastes them into $CVMFS_OPT_OUTPUT_DIR/$FQRN collect_benchmark_results() { local benchmark_output_dir="$CVMFS_OPT_OUTPUT_DIR/$FQRN" # undo the mtab trick if [ x"$CVMFS_OPT_MTAB_MODIFIED" = x"true" ]; then cat > $xml_file << EOF <?xml version="1.0" encoding="UTF-8"?> <testsuites tests="$num_tests" failures="$num_fails" disabled="$num_skips" errors="0" timestamp="$t_start" time="$(milliseconds_to_seconds $t_elapsed)" name="CVMFS Test Runner"> <testsuite hostname="$CVMFS_PLATFORM_NAME" name="$CVMFS_TEST_SUITE_NAME" timestamp="$t_start" tests="$num_tests" failures="$num_fails" disabled="$num_skips" errors="0" time="$(milliseconds_to_seconds $t_elapsed)"> EOF } cp -rf $workdir/system.info $workdir/$FQRN.*.log $workdir/callgrind.out.* $workdir/$FQRN.memcheck.xml $workdir/cvmfs.conf $workdir/*.data $benchmark_output_dir > /dev/null 2>&1 benchmark_log "Test results copied into $benchmark_output_dir" >> $logfile } # # XUnit XML generation helper functions # xunit_preamble() { local xml_file="$1" local scratchdir="$2" local num_tests="$(cat ${scratchdir}/num_tests)" local num_fails="$(cat ${scratchdir}/num_failures)" local num_skips="$(cat ${scratchdir}/num_skipped)" local t_start="$(cat ${scratchdir}/starttime)" local t_elapsed="$(cat ${scratchdir}/elapsed)" cat > $xml_file << EOF <?xml version="1.0" encoding="UTF-8"?> <testsuites tests="$num_tests" failures="$num_fails" disabled="$num_skips" errors="0" timestamp="$t_start" time="$(milliseconds_to_seconds $t_elapsed)" name="CVMFS Test Runner"> <testsuite hostname="$CVMFS_PLATFORM_NAME" name="$CVMFS_TEST_SUITE_NAME" timestamp="$CVMFS_TIMESTAMP" tests="$num_tests" failures="$num_fails" disabled="$num_skips" errors="0" time="$(milliseconds_to_seconds $t_elapsed)"> EOF } xunit_testcase() { local xml_file="$1" local test_scratchdir="$2" local logfile="$3" local test_name="$(cat ${test_scratchdir}/name)" local test_number="$(cat ${test_scratchdir}/number)" # check if the test was skipped if [ -f ${test_scratchdir}/skipped ]; then cat >> $xml_file << EOF <testcase name="$test_name" status="notrun" time="0.000" classname="$CVMFS_TEST_CLASS_NAME" /> EOF return 0 fi local t_elapsed=$(cat ${test_scratchdir}/elapsed) # check if the test was a success if [ -f ${test_scratchdir}/success ]; then cat >> $xml_file << EOF <testcase name="$test_name" status="run" time="$(milliseconds_to_seconds $t_elapsed)" classname="$CVMFS_TEST_CLASS_NAME" /> EOF return 0 fi local log_begin=$(cat ${test_scratchdir}/log_begin) local log_end=$(cat ${test_scratchdir}/log_end) local log_length=$(( $log_end - $log_begin )) # check if the test has failed if [ -f ${test_scratchdir}/failure ]; then local retval="$(cat ${test_scratchdir}/retval)" cat >> $xml_file << EOF <testcase name="$test_name" status="run" time="$(milliseconds_to_seconds $t_elapsed)" classname="$CVMFS_TEST_CLASS_NAME"> <failure message="Failed with Retval $retval" type=""> <![CDATA[$(head -n $log_end $logfile | tail -n $log_length | sed 's/[[:cntrl:]]//g')]]> </failure> </testcase> EOF return 0 fi } xunit_epilogue() { local xml_file="$1" cat >> $xml_file
{'addition_count': 1, 'commit_subject': 'Add timestamp and modify default reports', 'deletion_count': 1, 'file_extension': '', 'lang': 'test_functions', 'license': 'bsd-3-clause', 'repo_name': 'cvmfs/cvmfs'}