From 2b46e8899fdf08b6c67275ef152c1f15b5ea020d Mon Sep 17 00:00:00 2001
From: xfnw <xfnw@thunix.net>
Date: Tue, 15 Dec 2020 10:17:28 -0500
Subject: [PATCH] set a useragent

---
 urls.sh | 12 +++++++++---
 1 file changed, 9 insertions(+), 3 deletions(-)

diff --git a/urls.sh b/urls.sh
index 1e26497..356713a 100755
--- a/urls.sh
+++ b/urls.sh
@@ -1,3 +1,9 @@
-wget --spider --force-html -r -l1 -H $@ 2>&1 \
-  | grep '^--' | awk '{ print $3 }' \
-  | grep -v '\.\(css\|js\|png\|gif\|jpg\|txt\|ico\|ttf\|svg\)$'
+wget --spider --force-html -r -l1 -H -U 'searplbot/1.0' $@ 2>&1 | tee wg
+
+grep '^--' wg | awk '{ print $3 }' \
+  | grep -v '\.\(css\|js\|png\|gif\|jpg\|txt\|ico\|ttf\|svg\)$' \
+  | tee ur
+
+sleep 10
+
+php crawl.php $(cat ur)