xfnw/searpl
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

increase crawling timeout

Browse source
This commit is contained in:
xfnw 2021-10-20 15:19:56 -04:00
parent 34e9d6b8ca
commit 4643aeec70
1 changed files with 1 additions and 1 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
urls.sh
View file

@ -1,4 +1,4 @@
wget -w 2 --random-wait --spider --force-html --tries 1 --timeout 1 -r -l1 -H -U 'searplbot/1.0' $@ 2>&1 | tee -a wg
wget -w 2 --random-wait --spider --force-html --tries 1 --timeout 2 -r -l1 -H -U 'searplbot/1.0' $@ 2>&1 | tee -a wg
grep '^--' wg | awk '{ print $3 }' \
| grep -v '\.\(css\|js\|png\|gif\|jpg\|txt\|ico\|ttf\|svg\)$' \