From d4d5e67da8212c7ff88b5512269247edd40bf3a2 Mon Sep 17 00:00:00 2001
From: xfnw <xfnw@thunix.net>
Date: Thu, 31 Dec 2020 21:05:50 -0500
Subject: [PATCH] uniquely sort urls and put the db the gitignore

---
 .gitignore | 2 ++
 urls.sh    | 1 +
 2 files changed, 3 insertions(+)
 create mode 100644 .gitignore

diff --git a/.gitignore b/.gitignore
new file mode 100644
index 0000000..022fd15
--- /dev/null
+++ b/.gitignore
@@ -0,0 +1,2 @@
+db.sqlite
+
diff --git a/urls.sh b/urls.sh
index 0494843..2a69adc 100755
--- a/urls.sh
+++ b/urls.sh
@@ -2,6 +2,7 @@ wget --spider --force-html --tries 1 --timeout 1 -r -l1 -H -U 'searplbot/1.0' $@
 
 grep '^--' wg | awk '{ print $3 }' \
   | grep -v '\.\(css\|js\|png\|gif\|jpg\|txt\|ico\|ttf\|svg\)$' \
+  | sort | uniq \
   | tee -a ur
 
 rm wg