xfnw/searpl
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

set a useragent

Browse source
This commit is contained in:
xfnw 2020-12-15 10:17:28 -05:00
parent 8c4421108b
commit 2b46e8899f
1 changed files with 9 additions and 3 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

12
urls.sh
View file

@ -1,3 +1,9 @@
wget --spider --force-html -r -l1 -H $@ 2>&1 \
| grep '^--' | awk '{ print $3 }' \
| grep -v '\.\(css\|js\|png\|gif\|jpg\|txt\|ico\|ttf\|svg\)$'
wget --spider --force-html -r -l1 -H -U 'searplbot/1.0' $@ 2>&1 | tee wg
grep '^--' wg | awk '{ print $3 }' \
| grep -v '\.\(css\|js\|png\|gif\|jpg\|txt\|ico\|ttf\|svg\)$' \
| tee ur
sleep 10
php crawl.php $(cat ur)