xfnw/searpl
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

crawl slower so sites wont ban it lol

Browse source
This commit is contained in:
xfnw 2021-01-16 13:37:01 -05:00
parent 3e051c0feb
commit c6b476c4a3
1 changed files with 1 additions and 1 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
urls.sh
View file

@ -1,4 +1,4 @@
wget --spider --force-html --tries 1 --timeout 1 -r -l1 -H -U 'searplbot/1.0' $@ 2>&1 | tee -a wg
wget -w 2 --random-wait --spider --force-html --tries 1 --timeout 1 -r -l1 -H -U 'searplbot/1.0' $@ 2>&1 | tee -a wg
grep '^--' wg | awk '{ print $3 }' \
| grep -v '\.\(css\|js\|png\|gif\|jpg\|txt\|ico\|ttf\|svg\)$' \