From c62b48ae1f52c3f1529ed2a41d3f35fe705f3b21 Mon Sep 17 00:00:00 2001
From: xfnw <xfnw@thunix.net>
Date: Mon, 14 Dec 2020 17:22:28 -0500
Subject: [PATCH] unique urls

---
 create.php | 2 +-
 urls.sh    | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/create.php b/create.php
index 2bba20d..f1d23b7 100644
--- a/create.php
+++ b/create.php
@@ -1,2 +1,2 @@
-CREATE TABLE indexed (id INTEGER PRIMARY KEY, title VARCHAR(255), url VARCHAR(512), content TEXT)
+CREATE TABLE indexed (id INTEGER PRIMARY KEY, title VARCHAR(255), url VARCHAR(512) UNIQUE, content TEXT)
 
diff --git a/urls.sh b/urls.sh
index 1c7543d..fe0f851 100755
--- a/urls.sh
+++ b/urls.sh
@@ -1,3 +1,3 @@
-wget --spider --force-html -r -l2 -H $@ 2>&1 \
+wget --spider --force-html -r -l1 -H $@ 2>&1 \
   | grep '^--' | awk '{ print $3 }' \
-  | grep -v '\.\(css\|js\|png\|gif\|jpg\)$'
+  | grep -v '\.\(css\|js\|png\|gif\|jpg\|txt\|ico\|ttf\)$'