From 650911ae4b2ce64d7019105b87d8d12ff4fa8444 Mon Sep 17 00:00:00 2001 From: rldhont Date: Fri, 10 Jan 2025 13:52:20 +0100 Subject: [PATCH 1/6] Add X-Request-Id in request to QGIS Server headers --- lizmap/modules/lizmap/lib/Request/Proxy.php | 1 + 1 file changed, 1 insertion(+) diff --git a/lizmap/modules/lizmap/lib/Request/Proxy.php b/lizmap/modules/lizmap/lib/Request/Proxy.php index 0d4841bda8..9e2d203323 100644 --- a/lizmap/modules/lizmap/lib/Request/Proxy.php +++ b/lizmap/modules/lizmap/lib/Request/Proxy.php @@ -307,6 +307,7 @@ protected static function buildHeaders($url, $options) $options['headers'] = array_merge( self::userHttpHeader(), self::$services->wmsServerHeaders, + array('X-Request-Id' => uniqid().'-'.bin2hex(random_bytes(10))), $options['headers'] ); } From 7bf80e06715d822a0e24ef078988d5fba4f3e5a1 Mon Sep 17 00:00:00 2001 From: rldhont Date: Mon, 13 Jan 2025 10:42:59 +0100 Subject: [PATCH 2/6] logRequestIfError: use X-Request-Id if available --- lizmap/modules/lizmap/lib/Request/Proxy.php | 23 ++++++++++++++++----- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/lizmap/modules/lizmap/lib/Request/Proxy.php b/lizmap/modules/lizmap/lib/Request/Proxy.php index 9e2d203323..20acdfc157 100644 --- a/lizmap/modules/lizmap/lib/Request/Proxy.php +++ b/lizmap/modules/lizmap/lib/Request/Proxy.php @@ -487,17 +487,30 @@ protected static function fileProxy($url, $options) /** * Log if the HTTP code is a 4XX or 5XX error code. * - * @param int $httpCode The HTTP code of the request - * @param string $url The URL of the request, for logging + * @param int $httpCode The HTTP code of the request + * @param string $url The URL of the request, for logging + * @param array $headers The headers of the response */ - protected static function logRequestIfError($httpCode, $url) + protected static function logRequestIfError($httpCode, $url, $headers = array()) { if ($httpCode < 400) { return; } - \jLog::log('An HTTP request ended with an error, please check the main error log. HTTP code '.$httpCode, 'lizmapadmin'); - \jLog::log('The HTTP request ended with an error. HTTP code '.$httpCode.' → '.$url, 'error'); + $xRequestId = $headers['X-Request-Id'] ?? ''; + + $lizmapAdmin = 'An HTTP request ended with an error, please check the main error log.'; + $lizmapAdmin .= ' HTTP code '.$httpCode.'.'; + $error = 'The HTTP request ended with an error.'; + $error .= ' HTTP code '.$httpCode.'.'; + if ($xRequestId !== '') { + $lizmapAdmin .= ' The X-Request-Id `'.$xRequestId.'`.'; + $error .= ' X-Request-Id `'.$xRequestId.'` → '.$url; + } else { + $error .= ' → '.$url; + } + \jLog::log($lizmapAdmin, 'lizmapadmin'); + \jLog::log($error, 'error'); } /** From 61871ce01832ff42a824a0fccaf61bbf97588037 Mon Sep 17 00:00:00 2001 From: rldhont Date: Mon, 13 Jan 2025 10:18:42 +0100 Subject: [PATCH 3/6] Response headers : curl and file --- lizmap/modules/lizmap/lib/Request/Proxy.php | 62 ++++++++++++++++----- 1 file changed, 47 insertions(+), 15 deletions(-) diff --git a/lizmap/modules/lizmap/lib/Request/Proxy.php b/lizmap/modules/lizmap/lib/Request/Proxy.php index 20acdfc157..76d65c5f4c 100644 --- a/lizmap/modules/lizmap/lib/Request/Proxy.php +++ b/lizmap/modules/lizmap/lib/Request/Proxy.php @@ -323,7 +323,7 @@ protected static function buildHeaders($url, $options) * @param string $url * @param array $options * - * @return array{0: string, 1: string, 2: int} Array containing data (0: string), mime type (1: string) and HTTP code (2: int) + * @return array{0: string, 1: string, 2: int, 3: array} Array containing data (0: string), mime type (1: string), HTTP code (2: int) and headers */ protected static function curlProxy($url, $options) { @@ -331,7 +331,7 @@ protected static function curlProxy($url, $options) $http_code = null; $ch = curl_init(); - curl_setopt($ch, CURLOPT_HEADER, 0); + curl_setopt($ch, CURLOPT_HEADER, 1); curl_setopt($ch, CURLOPT_SSL_VERIFYPEER, false); curl_setopt($ch, CURLOPT_RETURNTRANSFER, 1); curl_setopt($ch, CURLOPT_FOLLOWLOCATION, true); @@ -382,10 +382,29 @@ protected static function curlProxy($url, $options) curl_setopt($ch, CURLOPT_POSTFIELDS, $options['body']); } } + $data = curl_exec($ch); if (!$data) { $data = ''; } + $header_size = curl_getinfo($ch, CURLINFO_HEADER_SIZE); + $headers_str = substr($data, 0, $header_size); + $data = substr($data, $header_size); + + $headers_arr = array_filter(explode("\r\n", $headers_str)); + $status_message = array_shift($headers_arr); + $headers = array(); + foreach ($headers_arr as $value) { + if (false !== ($matches = explode(':', $value, 2))) { + $key = str_replace( + ' ', + '-', + ucwords(strtolower(str_replace('-', ' ', $matches[0]))) + ); + $headers["{$key}"] = trim($matches[1]); + } + } + $info = curl_getinfo($ch); $mime = $info['content_type']; $http_code = (int) $info['http_code']; @@ -396,14 +415,14 @@ protected static function curlProxy($url, $options) curl_close($ch); - return array($data, $mime, $http_code); + return array($data, $mime, $http_code, $headers); } /** * @param string $url * @param array $options * - * @return array{0: string, 1: string, 2: int} Array containing data (0: string), mime type (1: string) and HTTP code (2: int) + * @return array{0: string, 1: string, 2: int, 3: array} Array containing data (0: string), mime type (1: string), HTTP code (2: int) and headers */ protected static function fileProxy($url, $options) { @@ -460,18 +479,30 @@ protected static function fileProxy($url, $options) $data = ''; } $mime = 'image/png'; - $matches = array(); $http_code = 0; + $headers = array(); // $http_response_header is created by file_get_contents foreach ($http_response_header as $header) { - if (preg_match('#^Content-Type:\s+([\w/\.+]+)(;\s+charset=(\S+))?#i', $header, $matches)) { - $mime = $matches[1]; - if (count($matches) > 3) { - $mime .= '; charset='.$matches[3]; - } - } elseif (substr($header, 0, 5) === 'HTTP/') { + if (substr($header, 0, 5) === 'HTTP/') { list($version, $code, $phrase) = explode(' ', $header, 3) + array('', false, ''); $http_code = (int) $code; + + continue; + } + if (false !== ($matches = explode(':', $header, 2))) { + $key = str_replace( + ' ', + '-', + ucwords(strtolower(str_replace('-', ' ', $matches[0]))) + ); + $headers["{$key}"] = trim($matches[1]); + if ($key === 'Content-Type' + && preg_match('#^Content-Type:\s+([\w/\.+]+)(;\s+charset=(\S+))?#i', $header, $matches)) { + $mime = $matches[1]; + if (count($matches) > 3) { + $mime .= '; charset='.$matches[3]; + } + } } } // optional debug @@ -481,7 +512,7 @@ protected static function fileProxy($url, $options) \jLog::dump($http_response_header, 'getRemoteData, bad response, response headers', 'error'); } - return array($data, $mime, $http_code); + return array($data, $mime, $http_code, $headers); } /** @@ -530,7 +561,7 @@ protected static function logRequestIfError($httpCode, $url, $headers = array()) * @param string|string[] $method deprecated. the http method. * it is ignored if $options is an array. * - * @return array{0: string, 1: string, 2: int} Array containing data (0: string), mime type (1: string) and HTTP code (2: int) + * @return array{0: string, 1: string, 2: int, 3: array} Array containing data (0: string), mime type (1: string), HTTP code (2: int) and headers */ public static function getRemoteData($url, $options = null, $debug = null, $method = 'get') { @@ -548,6 +579,7 @@ public static function getRemoteData($url, $options = null, $debug = null, $meth $content, 'text/json', 200, + array(), ); } // All requests are logged @@ -558,13 +590,13 @@ public static function getRemoteData($url, $options = null, $debug = null, $meth if (extension_loaded('curl') && $options['proxyHttpBackend'] != 'php') { // With curl $curlRequest = self::curlProxy($url, $options); - self::logRequestIfError($curlRequest[2], $url); + self::logRequestIfError($curlRequest[2], $url, $curlRequest[3]); return $curlRequest; } // With file_get_contents $request = self::fileProxy($url, $options); - self::logRequestIfError($request[2], $url); + self::logRequestIfError($request[2], $url, $request[3]); return $request; } From eff786c0aba343ad5b175faa3461bec35e001111 Mon Sep 17 00:00:00 2001 From: rldhont Date: Mon, 13 Jan 2025 12:23:37 +0100 Subject: [PATCH 4/6] Response headers from stream to log X-Request-Id --- lizmap/modules/lizmap/lib/Request/Proxy.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lizmap/modules/lizmap/lib/Request/Proxy.php b/lizmap/modules/lizmap/lib/Request/Proxy.php index 76d65c5f4c..785d8d3ff8 100644 --- a/lizmap/modules/lizmap/lib/Request/Proxy.php +++ b/lizmap/modules/lizmap/lib/Request/Proxy.php @@ -680,7 +680,7 @@ public static function getRemoteDataAsStream($url, $options = null) } $response = $client->send($request, $reqOptions); - self::logRequestIfError($response->getStatusCode(), $url); + self::logRequestIfError($response->getStatusCode(), $url, $response->getHeaders()); return new ProxyResponse( $response->getStatusCode(), From 148b8dc801b22e5fd042ba1c67dd5b7124a47817 Mon Sep 17 00:00:00 2001 From: rldhont Date: Mon, 13 Jan 2025 15:32:42 +0100 Subject: [PATCH 5/6] Tests: Update Proxy test about buildHeaders --- tests/units/classes/Request/ProxyTest.php | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/units/classes/Request/ProxyTest.php b/tests/units/classes/Request/ProxyTest.php index 2770e96c32..3ed0fe98ad 100644 --- a/tests/units/classes/Request/ProxyTest.php +++ b/tests/units/classes/Request/ProxyTest.php @@ -235,7 +235,11 @@ public function testBuildHeaders($options, $expectedHeaders, $expectedBody, $exp $url = 'http://localhost?test=test'; ProxyForTests::setServices((object)array('wmsServerURL' => 'http://localhost', 'wmsServerHeaders' => array())); list($url, $result) = ProxyForTests::buildHeadersForTests($url, $options); - $this->assertEquals($expectedHeaders, $result['headers']); + foreach ($expectedHeaders as $header => $value) { + $this->assertArrayHasKey($header, $result['headers']); + $this->assertEquals($value, $result['headers'][$header]); + } + $this->assertArrayHasKey('X-Request-Id', $result['headers']); $this->assertEquals($expectedBody, $result['body']); if ($expectedUrl) { $this->assertEquals($expectedUrl, $url); From 2ada8d1185bc75348d2e8c12088df3b4a77c4d34 Mon Sep 17 00:00:00 2001 From: rldhont Date: Thu, 16 Jan 2025 13:55:01 +0100 Subject: [PATCH 6/6] Response headers: use it in OGCRequest --- .../modules/lizmap/lib/Request/OGCRequest.php | 16 ++++++++---- lizmap/modules/lizmap/lib/Request/Proxy.php | 5 +++- .../lizmap/lib/Request/ProxyResponse.php | 25 ++++++++++++++++--- .../integration/requests-wfs-ghaction.js | 6 ++++- .../classes/Request/ProxyResponseTest.php | 20 +++++++++++++++ 5 files changed, 61 insertions(+), 11 deletions(-) create mode 100644 tests/units/classes/Request/ProxyResponseTest.php diff --git a/lizmap/modules/lizmap/lib/Request/OGCRequest.php b/lizmap/modules/lizmap/lib/Request/OGCRequest.php index ee32f39dae..ca72fbb813 100644 --- a/lizmap/modules/lizmap/lib/Request/OGCRequest.php +++ b/lizmap/modules/lizmap/lib/Request/OGCRequest.php @@ -233,9 +233,10 @@ private function formatHttpErrorString($parameters, $code) /** * Log if the HTTP code is a 4XX or 5XX error code. * - * @param int $code The HTTP code of the request + * @param int $code The HTTP code of the request + * @param array $headers The headers of the response */ - protected function logRequestIfError($code) + protected function logRequestIfError($code, $headers) { if ($code < 400) { return; @@ -243,6 +244,11 @@ protected function logRequestIfError($code) $message = 'The HTTP OGC request to QGIS Server ended with an error.'; + $xRequestId = $headers['X-Request-Id'] ?? ''; + if ($xRequestId !== '') { + $message .= ' The X-Request-Id `'.$xRequestId.'`.'; + } + // The master error with MAP parameter // This user must have an access to QGIS Server logs $params = $this->parameters(); @@ -308,14 +314,14 @@ protected function request($post = false, $stream = false) if ($stream) { $response = \Lizmap\Request\Proxy::getRemoteDataAsStream($querystring, $options); - $this->logRequestIfError($response->getCode()); + $this->logRequestIfError($response->getCode(), $response->getHeaders()); return new OGCResponse($response->getCode(), $response->getMime(), $response->getBodyAsStream()); } - list($data, $mime, $code) = \Lizmap\Request\Proxy::getRemoteData($querystring, $options); + list($data, $mime, $code, $headers) = \Lizmap\Request\Proxy::getRemoteData($querystring, $options); - $this->logRequestIfError($code); + $this->logRequestIfError($code, $headers); return new OGCResponse($code, $mime, $data); } diff --git a/lizmap/modules/lizmap/lib/Request/Proxy.php b/lizmap/modules/lizmap/lib/Request/Proxy.php index 785d8d3ff8..24010e6972 100644 --- a/lizmap/modules/lizmap/lib/Request/Proxy.php +++ b/lizmap/modules/lizmap/lib/Request/Proxy.php @@ -626,6 +626,7 @@ public static function getRemoteDataAsStream($url, $options = null) return new ProxyResponse( 200, 'text/json', + array('Content-Type' => 'text/json'), $stream ); } @@ -680,11 +681,13 @@ public static function getRemoteDataAsStream($url, $options = null) } $response = $client->send($request, $reqOptions); - self::logRequestIfError($response->getStatusCode(), $url, $response->getHeaders()); + $headers = $response->getHeaders(); + self::logRequestIfError($response->getStatusCode(), $url, $headers); return new ProxyResponse( $response->getStatusCode(), $response->getHeader('Content-Type')[0], + $headers, $response->getBody() ); } diff --git a/lizmap/modules/lizmap/lib/Request/ProxyResponse.php b/lizmap/modules/lizmap/lib/Request/ProxyResponse.php index 38aa1ff8d5..708dc149ab 100644 --- a/lizmap/modules/lizmap/lib/Request/ProxyResponse.php +++ b/lizmap/modules/lizmap/lib/Request/ProxyResponse.php @@ -26,6 +26,11 @@ class ProxyResponse */ protected $mime; + /** + * @var array the headers of the response + */ + protected $headers; + /** * @var StreamInterface the response body as a stream */ @@ -34,14 +39,16 @@ class ProxyResponse /** * constructor. * - * @param int $code the HTTP status code of the response - * @param string $mime the MIME type of the response - * @param StreamInterface $body the response body as a string + * @param int $code the HTTP status code of the response + * @param string $mime the MIME type of the response + * @param array $headers the headers of the response + * @param StreamInterface $body the response body as a string */ - public function __construct($code, $mime, $body) + public function __construct($code, $mime, $headers, $body) { $this->code = $code; $this->mime = $mime; + $this->headers = $headers; $this->body = $body; } @@ -65,6 +72,16 @@ public function getMime() return $this->mime; } + /** + * Get the headers of the response. + * + * @return array + */ + public function getHeaders() + { + return $this->headers; + } + /** * Get the response's body as string. * diff --git a/tests/end2end/cypress/integration/requests-wfs-ghaction.js b/tests/end2end/cypress/integration/requests-wfs-ghaction.js index d8bf8e1b58..24f6ce9bcf 100644 --- a/tests/end2end/cypress/integration/requests-wfs-ghaction.js +++ b/tests/end2end/cypress/integration/requests-wfs-ghaction.js @@ -1247,7 +1247,11 @@ describe('Request service', function () { .then((result) => { expect(result.code).to.eq(0) expect(result.stdout).to.contain('An HTTP request ended with an error, please check the main error log.') - expect(result.stdout).to.contain('HTTP code 400') + expect(result.stdout).to.contain('HTTP code 400.') + expect(result.stdout).to.contain('The HTTP OGC request to QGIS Server ended with an error.') + expect(result.stdout).to.contain( + 'HTTP code 400 on "REPOSITORY" = \'testsrepository\' & "PROJECT" = \'selection\' & "SERVICE" = \'WFS\' & "REQUEST" = \'getfeature\'' + ) clearLizmapAdminLog() }) clearErrorsLog() diff --git a/tests/units/classes/Request/ProxyResponseTest.php b/tests/units/classes/Request/ProxyResponseTest.php new file mode 100644 index 0000000000..fffb0f51e1 --- /dev/null +++ b/tests/units/classes/Request/ProxyResponseTest.php @@ -0,0 +1,20 @@ + 'text/json'), + \GuzzleHttp\Psr7\Utils::streamFor('{}') + ); + $this->assertEquals(200, $response->getCode()); + $this->assertEquals('text/json', $response->getMime()); + $this->assertEquals(array('Content-Type' => 'text/json'), $response->getHeaders()); + } +}