xfnw/searpl
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

limit filesize of crawls

Browse source
This commit is contained in:
xfnw 2021-10-22 11:43:33 -04:00
parent f23fdaad28
commit e9b6f113d9
1 changed files with 1 additions and 1 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
crawl.php
View file

@ -42,7 +42,7 @@ foreach ($arg as $url) {
$stmt = $db->prepare('DELETE FROM indexed WHERE url = ?'); $stmt = $db->prepare('DELETE FROM indexed WHERE url = ?');
$stmt->execute([htmlspecialchars(htmlspecialchars_decode($url))]); $stmt->execute([htmlspecialchars(htmlspecialchars_decode($url))]);
$file = file_get_contents($url, false, $context); $file = file_get_contents($url, false, $context, 0, 1000000);
if (!$file || strpos($http_response_header[0],'200 OK') === false) if (!$file || strpos($http_response_header[0],'200 OK') === false)
continue; continue;
$title = page_title($file); $title = page_title($file);