linkchecker: limit retries and depulicate links
@@ -10,31 +10,60 @@ if [ -z "$1" ]; then
exit
fi
+LINKFILE=$(mktemp)
+awk -F '|' '
+# Ignore links that contain version placeholder
+!/doc\/news\/_preparation_next_release\.md.*<<VERSION>>/ {
+ if($2 ~ /.*\//) {
+ sub("/$", "", $2)
+ }
+ sub("\\+", "%2B", $2)
+ print $2 "|" $1
+}' "$1" | sort -t '|' -k1 | awk -F "|" '
+key != $1 || NR == 1 {
+ key = $1
+ if(data) {
+ print data
+ }
+ data = $0
+ next
+}
+
+{
+ sub($1,"",$0)
+ data = data""$0
+}
+
+END {
+ print data
+}' > "$LINKFILE"
+
NUMTHREADS=10
check() {
- # Ignore links that contain version placeholder
- echo "$1" | grep -Eq 'doc/news/_preparation_next_release\.md.*<<VERSION>>' && return
+ link=$(echo "$1" | grep -oE "(https|http|ftp):[^|]*")
+ files=$(echo "$1" | grep -oE "\|.*" | sed 's/|/ /g')
# The following link only works if you are logged into the build server
# See also: https://github.com/ElektraInitiative/libelektra/issues/2674
- echo "$1" | grep -Eq 'https://build.libelektra.org/jenkins/job/libelektra/pipeline-syntax' && return
- link=$(echo "$1" | grep -oE "(https|http|ftp):.*")
+ echo "$link" | grep -Eq 'https://build.libelektra.org/jenkins/job/libelektra/pipeline-syntax' && return
if [ -z "$link" ]; then
echo "check the link format of $1"
return
fi
- wget --spider --quiet --timeout=10 "$link"
+ wget --spider --quiet --timeout=5 --tries=3 "$link"
if [ "$?" -ne "0" ]; then
- wget -O - --quiet "$link" > /dev/null
+ wget -O - --quiet --timeout=5 --tries=3 "$link" > /dev/null
if [ "$?" -ne "0" ]; then
- echo >&2 "$1"
+ for file in $files; do
+ echo >&2 "$file $link"
+ done
fi
fi
}
COUNTLINKS=0
THREADCOUNT=0
-NUMLINKS=$(wc -l < "$1")
+NUMLINKS=$(wc -l < "$LINKFILE")
PIDS=""
while read -r line; do
@@ -50,7 +79,7 @@ while read -r line; do
PIDS=""
THREADCOUNT=1
fi
-done < "$1"
+done < "$LINKFILE"
for pid in $PIDS; do
wait "$pid"
