#!/usr/bin/env bash

# meant to be run on mercury.dlab.berkeley.edu

DATA_PATH=/var/local/destress/lj-annex/data/events
OUTPUT_PATH=/var/local/destress/combined2/events

MAX_FILE_SIZE=104857600 # 100Mb

mkdir -p $OUTPUT_PATH

# Loop over directories in DATA_PATH
for d in `ls $DATA_PATH`; do
    echo $d
    OUT_FILE_COUNTER=1 # Counter to track when files are split
    for f in `ls $DATA_PATH/$d`; do
        if [ "$OUT_FILE_COUNTER" -eq 1 ]
          then
            OUT_FILE=$OUTPUT_PATH/$d.xml
          else
            OUT_FILE=$OUTPUT_PATH/$d$OUT_FILE_COUNTER.xml
        fi
        cat $DATA_PATH/$d/$f >> $OUT_FILE # Add to combined file

	# Split files if too large
        FILE_SIZE=$(stat -c%s $OUT_FILE)
        if (( "$FILE_SIZE" > "$MAX_FILE_SIZE" ))
          then
           OUT_FILE_COUNTER=$((OUT_FILE_COUNTER+1))
        fi
    done
done

#!/usr/bin/env bash

# meant to be run on mercury.dlab.berkeley.edu

DATA_PATH=/var/local/destress/lj-annex/data/events
OUTPUT_PATH=/home/pierre/combined/events

mkdir -p $OUTPUT_PATH

for f in `ls $DATA_PATH`; do
    echo $f
    # cat $DATA_PATH/$f/* > $OUTPUT_PATH/$f.xml
    find -L $DATA_PATH/$f -maxdepth 1 -type f -print0 | sort -z | xargs -0 cat -- > $OUTPUT_PATH/$f.xml
done


--- 
+++ 
@@ -3,13 +3,30 @@
 # meant to be run on mercury.dlab.berkeley.edu
 
 DATA_PATH=/var/local/destress/lj-annex/data/events
-OUTPUT_PATH=/home/pierre/combined/events
+OUTPUT_PATH=/var/local/destress/combined2/events
+
+MAX_FILE_SIZE=104857600 # 100Mb
 
 mkdir -p $OUTPUT_PATH
 
-for f in `ls $DATA_PATH`; do
-    echo $f
-    # cat $DATA_PATH/$f/* > $OUTPUT_PATH/$f.xml
-    find -L $DATA_PATH/$f -maxdepth 1 -type f -print0 | sort -z | xargs -0 cat -- > $OUTPUT_PATH/$f.xml
+# Loop over directories in DATA_PATH
+for d in `ls $DATA_PATH`; do
+    echo $d
+    OUT_FILE_COUNTER=1 # Counter to track when files are split
+    for f in `ls $DATA_PATH/$d`; do
+        if [ "$OUT_FILE_COUNTER" -eq 1 ]
+          then
+            OUT_FILE=$OUTPUT_PATH/$d.xml
+          else
+            OUT_FILE=$OUTPUT_PATH/$d$OUT_FILE_COUNTER.xml
+        fi
+        cat $DATA_PATH/$d/$f >> $OUT_FILE # Add to combined file
+
+	# Split files if too large
+        FILE_SIZE=$(stat -c%s $OUT_FILE)
+        if (( "$FILE_SIZE" > "$MAX_FILE_SIZE" ))
+          then
+           OUT_FILE_COUNTER=$((OUT_FILE_COUNTER+1))
+        fi
+    done
 done
-
Concatenate files into chunks no larger than 100Mb.
