xfnw/searpl
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

delete page before downloading new one, so dead pages do not sit in the database

Browse source
This commit is contained in:
xfnw 2021-01-27 20:33:25 -05:00
parent c6b476c4a3
commit 826e3c2b7c
1 changed files with 4 additions and 3 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

7
crawl.php
View file

@ -31,6 +31,10 @@ foreach ($arg as $url) {
echo "\n"; echo "\n";
$url = preg_replace('/\/$/','',$url); $url = preg_replace('/\/$/','',$url);
echo $url."\n"; echo $url."\n";
$stmt = $db->prepare('DELETE FROM indexed WHERE url = ?');
$stmt->execute([$url]);
$file = file_get_contents($url); $file = file_get_contents($url);
if (!$file) if (!$file)
continue; continue;
@ -43,9 +47,6 @@ foreach ($arg as $url) {
echo "title: ".$title."\n"; echo "title: ".$title."\n";
$stmt = $db->prepare('DELETE FROM indexed WHERE url = ?');
$stmt->execute([$url]);
$stmt = $db->prepare('INSERT INTO indexed (title, url, content) VALUES (?, ?, ?)'); $stmt = $db->prepare('INSERT INTO indexed (title, url, content) VALUES (?, ?, ?)');
$stmt->execute([$title, $url, $document]); $stmt->execute([$title, $url, $document]);
} }