From d65342e304f92643ba922200953cfebc51e1e482 Mon Sep 17 00:00:00 2001 From: ArthurHoaro Date: Sat, 30 Sep 2017 11:04:13 +0200 Subject: Extract the title/charset during page download, and check content type Use CURLOPT_WRITEFUNCTION to check the response code and content type (only allow HTML). Also extract the title and charset during downloading chunk of data, and stop it when everything has been extracted. Closes #579 --- application/HttpUtils.php | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) (limited to 'application/HttpUtils.php') diff --git a/application/HttpUtils.php b/application/HttpUtils.php index 00835966..2edf5ce2 100644 --- a/application/HttpUtils.php +++ b/application/HttpUtils.php @@ -3,9 +3,11 @@ * GET an HTTP URL to retrieve its content * Uses the cURL library or a fallback method * - * @param string $url URL to get (http://...) - * @param int $timeout network timeout (in seconds) - * @param int $maxBytes maximum downloaded bytes (default: 4 MiB) + * @param string $url URL to get (http://...) + * @param int $timeout network timeout (in seconds) + * @param int $maxBytes maximum downloaded bytes (default: 4 MiB) + * @param callable|string $curlWriteFunction Optional callback called during the download (cURL CURLOPT_WRITEFUNCTION). + * Can be used to add download conditions on the headers (response code, content type, etc.). * * @return array HTTP response headers, downloaded content * @@ -29,7 +31,7 @@ * @see http://stackoverflow.com/q/9183178 * @see http://stackoverflow.com/q/1462720 */ -function get_http_response($url, $timeout = 30, $maxBytes = 4194304) +function get_http_response($url, $timeout = 30, $maxBytes = 4194304, $curlWriteFunction = null) { $urlObj = new Url($url); $cleanUrl = $urlObj->idnToAscii(); @@ -75,6 +77,10 @@ function get_http_response($url, $timeout = 30, $maxBytes = 4194304) curl_setopt($ch, CURLOPT_TIMEOUT, $timeout); curl_setopt($ch, CURLOPT_USERAGENT, $userAgent); + if (is_callable($curlWriteFunction)) { + curl_setopt($ch, CURLOPT_WRITEFUNCTION, $curlWriteFunction); + } + // Max download size management curl_setopt($ch, CURLOPT_BUFFERSIZE, 1024); curl_setopt($ch, CURLOPT_NOPROGRESS, false); -- cgit v1.2.3