From 4643aeec7028902e2146ac6d387decfdb1c32d72 Mon Sep 17 00:00:00 2001 From: xfnw Date: Wed, 20 Oct 2021 15:19:56 -0400 Subject: [PATCH] increase crawling timeout --- urls.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/urls.sh b/urls.sh index e9bf4e4..842c450 100755 --- a/urls.sh +++ b/urls.sh @@ -1,4 +1,4 @@ -wget -w 2 --random-wait --spider --force-html --tries 1 --timeout 1 -r -l1 -H -U 'searplbot/1.0' $@ 2>&1 | tee -a wg +wget -w 2 --random-wait --spider --force-html --tries 1 --timeout 2 -r -l1 -H -U 'searplbot/1.0' $@ 2>&1 | tee -a wg grep '^--' wg | awk '{ print $3 }' \ | grep -v '\.\(css\|js\|png\|gif\|jpg\|txt\|ico\|ttf\|svg\)$' \