Add support for 10x responses in curlcmd output

Author: saintamh

hublot/engines/curlcmd.py

@@ -50,10 +50,19 @@ def request(self, creq: CompiledRequest, config: Config) -> Response:
             else:
                 raise HublotException(output)
 
-        headers_match = re.search(rb"\r?\n\r?\n", curl.stdout)
+        curl_output = curl.stdout
+
+        # completely ignore 10x headers
+        curl_output = re.sub(
+            rb"^HTTP/\d+(?:\.\d+)? 10\d\b(?:.*\r?\n)+\r?\n(?=HTTP/2 )",
+            b"",
+            curl_output,
+        )
+
+        headers_match = re.search(rb"\r?\n\r?\n", curl_output)
         if not headers_match:  # pragma: no cover
             raise Exception("Failed to find headers in curl output")
-        headers_str = curl.stdout[: headers_match.start()].decode("ISO-8859-1")
+        headers_str = curl_output[: headers_match.start()].decode("ISO-8859-1")
 
         status_code, reason, headers_str = self._parse_status_line(headers_str)
         return Response(
@@ -63,7 +72,7 @@ def request(self, creq: CompiledRequest, config: Config) -> Response:
             status_code=status_code,
             reason=reason,
             headers=self._parse_headers(headers_str),
-            content=curl.stdout[headers_match.end() :],
+            content=curl_output[headers_match.end() :],
         )
 
     def _compose_curl_command(self, creq: CompiledRequest, config: Config) -> Iterable[str]:

hublot/engines/pycurl.py

@@ -26,12 +26,12 @@
 
 
 RE_STATUS = re.compile(
-    r"^\s* HTTP/\d+(?:\.\d+)? \s+ (\d\d\d) \s* (?: (\S.+?) \s* )? $",
+    r"^\s* HTTP/\d+(?:\.\d+)? \ + (\d\d\d) \ * (?: (\S.+?) \ * )? \r?\n",
     flags=re.M | re.X,
 )
 
 RE_HEADER = re.compile(
-    r"^\s* ([^:]+?) \s*:\s* (.+?) \s*$",
+    r"^\ * ([^:]+?) \ *:\ * (.+?) \ *\r?$",
     flags=re.M | re.X,
 )
 

test/test_engine_curlcmd.py

@@ -0,0 +1,85 @@
+#!/usr/bin/env python3
+
+# srandards
+from inspect import cleandoc
+import subprocess
+
+# 3rd parties
+import pytest
+
+# hublot
+from hublot import Headers, HttpClient, Request, Response
+from hublot.compile import compile_request
+
+
+@pytest.mark.parametrize(
+    "curl_output, get_expected_response",
+    [
+        pytest.param(
+            """
+            HTTP/2 200 
+            Content-Type: text/plain; charset=UTF-8
+            Content-Length: 2
+            
+            OK
+            """,
+            lambda creq: Response(
+                creq,
+                from_cache=False,
+                history=[],
+                status_code=200,
+                reason=None,
+                headers=Headers(
+                    {
+                        "Content-Type": "text/plain; charset=UTF-8",
+                        "Content-Length": "2",
+                    }
+                ),
+                content=b"OK",
+            ),
+            id="simple 200 response",
+        ),
+        pytest.param(
+            """
+            HTTP/1.1 101 Switching Protocols
+            Connection: Upgrade
+            Upgrade: h2c
+
+            HTTP/2 200 
+            Content-Type: text/plain; charset=UTF-8
+            Content-Length: 2
+
+            OK
+            """,
+            lambda creq: Response(
+                creq,
+                from_cache=False,
+                history=[],
+                status_code=200,
+                reason=None,
+                headers=Headers(
+                    {
+                        "Content-Type": "text/plain; charset=UTF-8",
+                        "Content-Length": "2",
+                    }
+                ),
+                content=b"OK",
+            ),
+            id="101+200 double-response",
+        ),
+    ],
+)
+def test_engine_curlcmd(mocker, curl_output, get_expected_response):
+    client = HttpClient(engines=["curlcmd"])
+    mocker.patch(
+        "hublot.engines.curlcmd.subprocess.run",
+        return_value=subprocess.CompletedProcess(
+            args=[],
+            returncode=0,
+            stdout=cleandoc(curl_output).replace("\n", "\r\n").encode("UTF-8"),
+        ),
+    )
+    req = Request("http://test.test/", method="GET")
+    creq = compile_request(req, client.config, client.cookies, num_retries=0)
+    res = client.fetch(req)
+    assert res == get_expected_response(creq)