From c6b476c4a35ec4e9b79ee2c9839f9b85cc93ae27 Mon Sep 17 00:00:00 2001
From: xfnw <xfnw@thunix.net>
Date: Sat, 16 Jan 2021 13:37:01 -0500
Subject: [PATCH] crawl slower so sites wont ban it lol

---
 urls.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/urls.sh b/urls.sh
index 2a69adc..e9bf4e4 100755
--- a/urls.sh
+++ b/urls.sh
@@ -1,4 +1,4 @@
-wget --spider --force-html --tries 1 --timeout 1 -r -l1 -H -U 'searplbot/1.0' $@ 2>&1 | tee -a wg
+wget -w 2 --random-wait --spider --force-html --tries 1 --timeout 1 -r -l1 -H -U 'searplbot/1.0' $@ 2>&1 | tee -a wg
 
 grep '^--' wg | awk '{ print $3 }' \
   | grep -v '\.\(css\|js\|png\|gif\|jpg\|txt\|ico\|ttf\|svg\)$' \