Replace urllib3 with httpx

Author: scottwallacesh

gabbi/case.py

@@ -14,7 +14,7 @@
 
 The test case encapsulates the request headers and body and expected
 response headers and body. When the test is run an HTTP request is
-made using urllib3. Assertions are made against the response.
+made using httpx. Assertions are made against the response.
 """
 
 from collections import OrderedDict
@@ -500,7 +500,7 @@ def _run_request(
         """
 
         if 'user-agent' not in (key.lower() for key in headers):
-            headers['user-agent'] = "gabbi/%s (Python urllib3)" % __version__
+            headers['user-agent'] = "gabbi/%s (Python httpx)" % __version__
 
         try:
             response, content = self.http.request(

gabbi/httpclient.py

@@ -11,53 +11,56 @@
 # License for the specific language governing permissions and limitations
 # under the License.
 
+from gabbi import utils
+from gabbi.handlers import jsonhandler
+import httpx
+import logging
 import os
 import sys
 
-import certifi
-import urllib3
-
-from gabbi.handlers import jsonhandler
-from gabbi import utils
-
-# Disable SSL warnings otherwise tests which process stderr will get
-# extra information.
-urllib3.disable_warnings()
+logging.getLogger("httpx").setLevel(logging.WARNING)
 
 
-class Http(urllib3.PoolManager):
-    """A subclass of the ``urllib3.PoolManager`` to munge the data.
+class Http:
+    """A class to munge the HTTP response.
 
     This transforms the response to look more like what httplib2
     provided when it was used as the HTTP client.
     """
 
+    def __init__(self, **kwargs):
+        self.extensions = {}
+        if "server_hostname" in kwargs:
+            self.extensions["sni_hostname"] = kwargs["server_hostname"]
+        self.client = httpx.Client(verify=kwargs.get("cert_validate", True))
+
     def request(self, absolute_uri, method, body, headers, redirect, timeout):
-        if redirect:
-            retry = urllib3.util.Retry(raise_on_redirect=False, redirect=5)
-        else:
-            retry = urllib3.util.Retry(total=False, redirect=False)
-        response = super(Http, self).request(
-            method,
-            absolute_uri,
-            body=body,
-            headers=headers,
-            retries=retry,
-            timeout=timeout,
-        )
+        try:
+            response = self.client.request(
+                method=method,
+                url=absolute_uri,
+                headers=headers,
+                content=body,
+                timeout=timeout,
+                follow_redirects=redirect,
+                extensions=self.extensions,
+            )
+        except httpx.ConnectError as error:
+            raise RuntimeError(
+                f"{error}.\n\nMethod: {method}\nURL: {absolute_uri}"
+            ) from error
 
         # Transform response into something akin to httplib2
         # response object.
-        content = response.data
-        status = response.status
-        reason = response.reason
+        content = response.content
+        status = response.status_code
+        reason = response.reason_phrase
+        http_version = response.http_version
         headers = response.headers
-        headers['status'] = str(status)
-        headers['reason'] = reason
+        headers["status"] = str(status)
+        headers["reason"] = str(reason)
+        headers["http_protocol_version"] = str(http_version)
 
-        # Shut down open PoolManagers whose connections have completed to
-        # save on socket file descriptors.
-        self.clear()
         return headers, content
 
 
@@ -85,59 +88,71 @@ class VerboseHttp(Http):
     # Can include response object attributes that are not
     # technically headers.
     HEADER_BLACKLIST = [
-        'status',
-        'reason',
+        "status",
+        "reason",
+        "http_protocol_version",
     ]
 
-    REQUEST_PREFIX = '>'
-    RESPONSE_PREFIX = '<'
+    REQUEST_PREFIX = ">"
+    RESPONSE_PREFIX = "<"
     COLORMAP = {
-        'caption': os.environ.get('GABBI_CAPTION_COLOR', 'BLUE').upper(),
-        'header': os.environ.get('GABBI_HEADER_COLOR', 'YELLOW').upper(),
-        'request': os.environ.get('GABBI_REQUEST_COLOR', 'CYAN').upper(),
-        'status': os.environ.get('GABBI_STATUS_COLOR', 'CYAN').upper(),
+        "caption": os.environ.get("GABBI_CAPTION_COLOR", "BLUE").upper(),
+        "header": os.environ.get("GABBI_HEADER_COLOR", "YELLOW").upper(),
+        "request": os.environ.get("GABBI_REQUEST_COLOR", "CYAN").upper(),
+        "status": os.environ.get("GABBI_STATUS_COLOR", "CYAN").upper(),
     }
 
     def __init__(self, **kwargs):
-        self.caption = kwargs.pop('caption')
-        self._show_body = kwargs.pop('body')
-        self._show_headers = kwargs.pop('headers')
-        self._use_color = kwargs.pop('colorize')
-        self._stream = kwargs.pop('stream')
+        self.caption = kwargs.pop("caption")
+        self._show_body = kwargs.pop("body")
+        self._show_headers = kwargs.pop("headers")
+        self._use_color = kwargs.pop("colorize")
+        self._stream = kwargs.pop("stream")
         if self._use_color:
             self.colorize = utils.get_colorizer(self._stream)
-        super(VerboseHttp, self).__init__(**kwargs)
+        super().__init__(**kwargs)
 
     def request(self, absolute_uri, method, body, headers, redirect, timeout):
         """Display request parameters before requesting."""
 
-        self._verbose_output('#### %s ####' % self.caption,
-                             color=self.COLORMAP['caption'])
-        self._verbose_output('%s %s' % (method, absolute_uri),
-                             prefix=self.REQUEST_PREFIX,
-                             color=self.COLORMAP['request'])
+        self._verbose_output(
+            f"#### {self.caption} ####",
+            color=self.COLORMAP["caption"],
+        )
+        self._verbose_output(
+            f"{method} {absolute_uri}",
+            prefix=self.REQUEST_PREFIX,
+            color=self.COLORMAP["request"],
+        )
 
         self._print_headers(headers, prefix=self.REQUEST_PREFIX)
         self._print_body(headers, body)
 
-        response, content = super(VerboseHttp, self).request(
-            absolute_uri, method, body, headers, redirect, timeout)
+        response, content = super().request(
+            absolute_uri,
+            method,
+            body,
+            headers,
+            redirect,
+            timeout,
+        )
 
         # Blank line for division
-        self._verbose_output('')
-        self._verbose_output('%s %s' % (response['status'],
-                                        response['reason']),
-                             prefix=self.RESPONSE_PREFIX,
-                             color=self.COLORMAP['status'])
+        self._verbose_output("")
+        self._verbose_output(
+            f'{response["status"]} {response["reason"]}',
+            prefix=self.RESPONSE_PREFIX,
+            color=self.COLORMAP["status"],
+        )
         self._print_headers(response, prefix=self.RESPONSE_PREFIX)
 
         # response body
         self._print_body(response, content)
-        self._verbose_output('')
+        self._verbose_output("")
 
         return (response, content)
 
-    def _print_headers(self, headers, prefix=''):
+    def _print_headers(self, headers, prefix=""):
         """Output request or response headers."""
         if self._show_headers:
             for key in headers:
@@ -148,11 +163,11 @@ def _print_body(self, headers, content):
         """Output body if not binary."""
         # Use text/plain as the default so that when there is not content-type
         # we can still see the output.
-        content_type = utils.extract_content_type(headers, 'text/plain')[0]
+        content_type = utils.extract_content_type(headers, "text/plain")[0]
         if self._show_body and utils.not_binary(content_type):
             content = utils.decode_response_content(headers, content)
             if isinstance(content, bytes):
-                content = content.decode('utf-8')
+                content = content.decode("utf-8")
             # TODO(cdent): Using the JSONHandler here instead of
             # just the json module to make it clear that eventually
             # we could pretty print any printable output by using a
@@ -166,57 +181,50 @@ def _print_body(self, headers, content):
                 except ValueError:
                     # It it didn't decode for some reason treat it as a string.
                     pass
-            self._verbose_output('')
+            self._verbose_output("")
             if content:
                 self._verbose_output(content)
 
-    def _print_header(self, name, value, prefix='', stream=None):
+    def _print_header(self, name, value, prefix="", stream=None):
         """Output one single header."""
-        header = self.colorize(self.COLORMAP['header'], "%s:" % name)
-        self._verbose_output("%s %s" % (header, value), prefix=prefix,
-                             stream=stream)
+        header = self.colorize(self.COLORMAP["header"], f"{name}:")
+        self._verbose_output(f"{header} {value}", prefix=prefix, stream=stream)
 
-    def _verbose_output(self, message, prefix='', color=None, stream=None):
+    def _verbose_output(self, message, prefix="", color=None, stream=None):
         """Output a message."""
         stream = stream or self._stream
         if prefix and message:
-            print(prefix, end=' ', file=stream)
+            print(prefix, end=" ", file=stream)
         if color:
             message = self.colorize(color, message)
         print(message, file=stream)
 
 
 def get_http(
     verbose=False,
-    caption='',
+    caption="",
     cert_validate=True,
     hostname=None,
     timeout=30,
 ):
     """Return an ``Http`` class for making requests."""
-    cert_validation = {'cert_reqs': 'CERT_NONE'} if not cert_validate else {}
-
     if not verbose:
         return Http(
-            strict=True,
-            ca_certs=certifi.where(),
             server_hostname=hostname,
             timeout=timeout,
-            **cert_validation
+            cert_validate=cert_validate,
         )
 
-    headers = False if verbose == 'body' else True
-    body = False if verbose == 'headers' else True
+    headers = verbose != "body"
+    body = verbose != "headers"
 
     return VerboseHttp(
         headers=headers,
         body=body,
         stream=sys.stdout,
         caption=caption,
         colorize=True,
-        strict=True,
-        ca_certs=certifi.where(),
         server_hostname=hostname,
         timeout=timeout,
-        **cert_validation
+        cert_validate=cert_validate,
     )

requirements.txt

@@ -1,7 +1,7 @@
 pbr
 pytest
 PyYAML
-urllib3>=1.26.9,<2.0.0
+httpx
 certifi
 jsonpath-rw-ext>=1.0.0
 wsgi-intercept>=1.13.0