searpl/urls.sh

wget --spider --force-html -r -l1 -H $@ 2>&1 \
  | grep '^--' | awk '{ print $3 }' \
  | grep -v '\.\(css\|js\|png\|gif\|jpg\|txt\|ico\|ttf\|svg\)$'
unique urls 2020-12-14 22:22:28 +00:00			`wget --spider --force-html -r -l1 -H $@ 2>&1 \`
working 2020-12-14 21:59:16 +00:00			`\| grep '^--' \| awk '{ print $3 }' \`
dont track svgs and drop the / from the end of urls so they wont be duplicated 2020-12-15 14:59:19 +00:00			`\| grep -v '\.\(css\\|js\\|png\\|gif\\|jpg\\|txt\\|ico\\|ttf\\|svg\)$'`