<NME> main
<BEF> ADDFILE
<MSG> Merge pull request #1542 from reneme/feature-early_nested

Optimisation: Create/Remove Nested Catalogs

Up to now, a nested catalog was created as soon as `cvmfs_server publish` hit a `.cvmfscatalog` file, and removed when it found a deleted `.cvmfscatalog` file. Conceptually that is fine, but it might lead to many unnecessary catalog updates.

For example: Imagine a user adds a new directory release and marks it a nested catalog (i.e. `release/.cvmfscatalog`), furthermore she adds 150,000 files to this new directory. In the worst case, `cvmfs_server publish` would traverse all 150,000 files, adding them to the father catalog of release. Only in the end it would find `.cvmfscatalog` which forced `cvmfs_server publish` to split the father at release and copying all 150,000 entries to the new nested catalog.

Similarly this could happen when `cvmfs_server publish` first finds a deleted `.cvmfscatalog` file and copys-up all entries into the father just before removing all of them, because actually the entire directory was deleted.

That patch disentangles the creation of nested catalogs from the processing of `.cvmfscatalog` in `cvmfs_server publish`. That simplifies the code and also allows to do the catalog creation smarter. I.e. a newly created directory containing a `.cvmfspublished` file will immediate result in a new (and empty) catalog that gets filled right afterwards. Similarly a removed directory containing `.cvmfspublished` is deleting an empty catalog and avoids an expensive and useless copy-up.
<DFF> @@ -0,0 +1,177 @@
+cvmfs_test_name="Early Creation of Nested Catalogs"
+cvmfs_test_autofs_on_startup=false
+
+cvmfs_run_test() {
+  logfile=$1
+  local repo_dir=/cvmfs/$CVMFS_TEST_REPO
+
+  local scratch_dir=$(pwd)
+
+  echo "create a fresh repository named $CVMFS_TEST_REPO with user $CVMFS_TEST_USER"
+  create_empty_repo $CVMFS_TEST_REPO $CVMFS_TEST_USER || return $?
+
+  echo "starting transaction to create a single nested catalog"
+  start_transaction $CVMFS_TEST_REPO || return $?
+
+  local dir_1="nested_1"
+  echo "setting up a new nested catalog ($dir_1)"
+  mkdir ${repo_dir}/${dir_1}               || return 1
+  touch ${repo_dir}/${dir_1}/.cvmfscatalog || return 2
+  local i=1024
+  while [ $i -gt 0 ]; do
+    local f=$(head -c10 /dev/urandom | md5sum | cut -d' ' -f1)
+    head -c1024 /dev/urandom > ${repo_dir}/${dir_1}/${f} || return 3
+    i=$(( $i - 1 ))
+  done
+
+  local log_1="publish_1.log"
+  echo "creating CVMFS snapshot (logging into $log_1)"
+  publish_repo $CVMFS_TEST_REPO -v > $log_1 2>&1 || return $?
+
+  echo "check if the nested catalog was created first"
+  # 1st: the containing directory is added
+  # 2nd: the nested catalog is created in this directory
+  cat $log_1 | grep -e '^\[add\] ' | head -n1 | grep -e "$dir_1\$"                           || return 4
+  cat $log_1 | grep -e '^\[add\] ' | head -n2 | tail -n1 | grep -e "^\[add\] Nested catalog" || return 5
+
+  echo "check if all catalogs are there"
+  check_catalog_presence "/${dir_1}" $CVMFS_TEST_REPO || return 6
+
+  echo "check catalog and data integrity"
+  check_repository $CVMFS_TEST_REPO -i  || return $?
+
+  # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
+
+  echo "starting transaction to create several nested catalogs recursively"
+  start_transaction $CVMFS_TEST_REPO || return $?
+
+  local dir_2="nested_2"
+  echo "setting up a new nested catalog ($dir_2)"
+  mkdir ${repo_dir}/${dir_2}               || return 10
+  for d in $(seq 1 5); do
+    mkdir ${repo_dir}/${dir_2}/${d}
+    touch ${repo_dir}/${dir_2}/${d}/.cvmfscatalog
+    local j=1024
+    while [ $j -gt 0 ]; do
+      local f=$(head -c10 /dev/urandom | md5sum | cut -d' ' -f1)
+      head -c1024 /dev/urandom > ${repo_dir}/${dir_2}/${d}/${f} || return 11
+      j=$(( $j - 1 ))
+    done
+  done
+
+  local log_2="publish_2.log"
+  echo "creating CVMFS snapshot (logging into $log_2)"
+  publish_repo $CVMFS_TEST_REPO -v > $log_2 2>&1 || return $?
+
+  echo "check if the catalogs have been created properly"
+  for d in $(seq 1 5); do
+    cat $log_2 | grep -A1 -e "^\[add\] /.*${dir_2}/${d}\$" | tail -n1 | grep -e "^\[add\] Nested catalog" || return 12
+    check_catalog_presence "/${dir_2}/${d}" $CVMFS_TEST_REPO                                              || return 13
+  done
+
+  echo "check catalog and data integrity"
+  check_repository $CVMFS_TEST_REPO -i  || return $?
+
+  # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
+
+  echo "starting transaction to remove several nested catalogs recursively"
+  start_transaction $CVMFS_TEST_REPO || return $?
+
+  echo "remove directory $dir_2 and the contained nested catalogs"
+  rm -fR ${repo_dir}/$dir_2 || return 20
+
+  local log_3="publish_3.log"
+  echo "creating CVMFS snapshot (logging into $log_3)"
+  publish_repo $CVMFS_TEST_REPO -v > $log_3 2>&1 || return $?
+
+  echo "check if the catalogs have been removed properly"
+  for d in $(seq 1 5); do
+    cat $log_3 | grep -A1 -e "^\[rem\] Nested catalog.*/${dir_2}/${d}\$" \
+               | tail -n1                                                \
+               | grep -e "^\[rem\] /.*${dir_2}/${d}\$"       || return 21
+    check_catalog_presence "/${dir_2}/${d}" $CVMFS_TEST_REPO && return 22
+  done
+
+  echo "check catalog and data integrity"
+  check_repository $CVMFS_TEST_REPO -i  || return $?
+
+  # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
+
+  echo "starting transaction to create several directories"
+  start_transaction $CVMFS_TEST_REPO || return $?
+
+  local dir_3="nested_3"
+  echo "setting up a new testbed ($dir_3)"
+  mkdir ${repo_dir}/${dir_3} || return 30
+  for d in $(seq 1 5); do
+    mkdir ${repo_dir}/${dir_3}/${d}
+    local j=1024
+    while [ $j -gt 0 ]; do
+      local f=$(head -c10 /dev/urandom | md5sum | cut -d' ' -f1)
+      head -c1024 /dev/urandom > ${repo_dir}/${dir_3}/${d}/${f} || return 31
+      j=$(( $j - 1 ))
+    done
+  done
+
+  local log_4="publish_4.log"
+  echo "creating CVMFS snapshot (logging into $log_4)"
+  publish_repo $CVMFS_TEST_REPO -v > $log_4 2>&1 || return $?
+
+  echo "check that no catalogs are created yet"
+  for d in $(seq 1 5); do
+    check_catalog_presence "/${dir_3}/${d}" && return 32
+  done
+
+  echo "check catalog and data integrity"
+  check_repository $CVMFS_TEST_REPO -i  || return $?
+
+  echo "starting transaction to retrofit nested catalogs inside ($dir_3)"
+  start_transaction $CVMFS_TEST_REPO || return $?
+
+  echo "touch .cvmfscatalog files inside ${dir_3}"
+  for d in $(seq 1 5); do
+    touch ${repo_dir}/${dir_3}/${d}/.cvmfscatalog || return 33
+    local j=1024
+    while [ $j -gt 0 ]; do
+      local f=$(head -c10 /dev/urandom | md5sum | cut -d' ' -f1)
+      head -c1024 /dev/urandom > ${repo_dir}/${dir_3}/${d}/${f} || return 34
+      j=$(( $j - 1 ))
+    done
+  done
+
+  local log_5="publish_5.log"
+  echo "publish to create catalogs (logging into $log_5)"
+  publish_repo $CVMFS_TEST_REPO -v > $log_5 2>&1 || return 34
+
+  echo "check if the catalogs have been created properly"
+  for d in $(seq 1 5); do
+    cat $log_5 | grep -A1 -e "^\[tou\] /.*${dir_3}/${d}\$" | tail -n1 | grep -e "^\[add\] Nested catalog" || return 35
+    check_catalog_presence "/${dir_3}/${d}" $CVMFS_TEST_REPO                                              || return 36
+  done
+
+  # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
+
+  echo "starting transaction to update some nested catalogs in ${dir_3}"
+  start_transaction $CVMFS_TEST_REPO || return $?
+
+  rm -f ${repo_dir}/${dir_3}/1/.cvmfscatalog || return 40
+  rm -f ${repo_dir}/${dir_3}/3/.cvmfscatalog || return 41
+  touch ${repo_dir}/${dir_3}/.cvmfscatalog   || return 42
+
+  local log_6="publish_6.log"
+  echo "creating CVMFS snapshot (logging into $log_6)"
+  publish_repo $CVMFS_TEST_REPO -v > $log_6 2>&1 || return $?
+
+  echo "check the catalog modification sequence ($log_6)"
+  cat $log_6 | grep -A1 -e "^\[tou\] /.*${dir_3}/1\$" | tail -n1 | grep -e "^\[rem\] Nested catalog.*${dir_3}/1\$" || return 43
+  cat $log_6 | grep -A1 -e "^\[tou\] /.*${dir_3}/3\$" | tail -n1 | grep -e "^\[rem\] Nested catalog.*${dir_3}/3\$" || return 44
+  cat $log_6 | grep -A1 -e "^\[tou\] /.*${dir_3}\$"   | tail -n1 | grep -e "^\[add\] Nested catalog.*${dir_3}\$"   || return 45
+
+  echo "check that the right catalogs are present"
+  check_catalog_presence "/${dir_3}/2" $CVMFS_TEST_REPO || return 46
+  check_catalog_presence "/${dir_3}/4" $CVMFS_TEST_REPO || return 47
+  check_catalog_presence "/${dir_3}/5" $CVMFS_TEST_REPO || return 48
+  check_catalog_presence "/${dir_3}"   $CVMFS_TEST_REPO || return 49
+
+  return 0
+}

{'addition_count': 177, 'commit_subject': 'Merge pull request #1542 from reneme/feature-early_nested', 'deletion_count': 0, 'file_extension': '', 'lang': 'main', 'license': 'bsd-3-clause', 'repo_name': 'cvmfs/cvmfs'}