uniquely sort urls and put the db the gitignore

2020-12-31 21:05:50 -05:00 · 2020-12-31 21:05:50 -05:00 · d4d5e67da8
commit d4d5e67da8
parent 036e3addb2
2 changed files with 3 additions and 0 deletions
--- a/.gitignore
+++ b/.gitignore
@ -0,0 +1,2 @@
+db.sqlite
+
--- a/urls.sh
+++ b/urls.sh
@ -2,6 +2,7 @@ wget --spider --force-html --tries 1 --timeout 1 -r -l1 -H -U 'searplbot/1.0' $@

 grep '^--' wg | awk '{ print $3 }' \
  | grep -v '\.\(css\|js\|png\|gif\|jpg\|txt\|ico\|ttf\|svg\)$' \
+  | sort | uniq \
  | tee -a ur

 rm wg