xfnw/searpl
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

dont track svgs and drop the / from the end of urls so they wont be duplicated

Browse source
This commit is contained in:
xfnw 2020-12-15 09:59:19 -05:00
parent 75be84f895
commit 8c4421108b
2 changed files with 2 additions and 1 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
crawl.php
View file

@ -28,6 +28,7 @@ $arg = $argv;
array_shift($arg); array_shift($arg);
foreach ($arg as $url) { foreach ($arg as $url) {
$url = preg_replace('/\/$/','',$url);
$file = file_get_contents($url); $file = file_get_contents($url);
if (!$file) if (!$file)
continue; continue;

2
urls.sh
View file

@ -1,3 +1,3 @@
wget --spider --force-html -r -l1 -H $@ 2>&1 \ wget --spider --force-html -r -l1 -H $@ 2>&1 \
| grep '^--' | awk '{ print $3 }' \ | grep '^--' | awk '{ print $3 }' \
| grep -v '\.\(css\|js\|png\|gif\|jpg\|txt\|ico\|ttf\)$' | grep -v '\.\(css\|js\|png\|gif\|jpg\|txt\|ico\|ttf\|svg\)$'