Add 'disable_response_handler' test data (#301)

Defaults to False. If true, no response handler is
run to transform the response body from its raw content-type
to structured data that can be used by tests managed by
that content-type's handler.

This is useful when a response (over which we probably
have no control) claims a content-type of application/json
but the response is not valid JSON. In this circumstance
there will be an error, the test will fail.

Sometimes we may need the test to not fail so we can
make other checks (such as checking headers) without
worrying about the body. In that case
disable_response_handler may be set to True.

In the failure case, when we do want the error, the
error is now more informative.

Docs are updated to reflect the new field.

Also included in this change:

* Fix (in .stestr.conf) to gabbi's own test runs not
  being properly parallelized due to a bad regex

* Run pep8 tests by default

Fixes #299

Author: cdent

.stestr.conf

@@ -3,4 +3,4 @@ test_path=gabbi/tests
 test_command=${PYTHON:-python} -m subunit.run discover gabbi $LISTOPT $IDOPTION
 test_id_option=--load-list $IDFILE
 test_list_option=--list
-group_regex=(?:gabbi\.suitemaker\.(test_[^_]+_[^_]+)|tests\.test_(?:intercept|inner_fixture)\.([^_]+))
+group_regex=(?:gabbi\.tests\.test_(?:\w+)\.([^_]+))

docs/source/format.rst

@@ -72,6 +72,14 @@ Metadata
        so changing this may be requried. It can also be changed when
        :doc:`loader` or using :doc:`gabbi-run <runner>`.
      - defaults to ``True``
+   * - ``disable_response_handler``
+     - If ``True``, means that the response body will not be processed to
+       Python data. This can be necessary if a response claims a
+       ``content-type`` but the body is not actually that type but it is still
+       necessary to run tests against the response. In that situation, if
+       ``disable_response_handler`` is ``False`` the test will be treated as
+       a failure.
+     - defaults to ``False``
 
 
 .. note:: When tests are generated dynamically, the ``TestCase`` name will

gabbi/case.py

@@ -70,6 +70,7 @@
     'skip': '',
     'poll': {},
     'use_prior_test': True,
+    'disable_response_handler': False
 }
 
 
@@ -482,9 +483,15 @@ def _run_request(self, url, method, headers, body, redirect=False):
         decoded_output = utils.decode_response_content(response, content)
         self.content_type = response.get('content-type', '').lower()
         loader_class = self.get_content_handler(self.content_type)
-        if decoded_output and loader_class:
+        if (decoded_output and loader_class
+                and not self.test_data['disable_response_handler']):
             # save structured response data
-            self.response_data = loader_class.loads(decoded_output)
+            try:
+                self.response_data = loader_class.loads(decoded_output)
+            except exception.GabbiDataLoadError as exc:
+                raise AssertionError(
+                    'unable to load data as %s' % self.content_type) from exc
+
         else:
             self.response_data = None
         self.output = decoded_output

gabbi/exception.py

@@ -13,6 +13,11 @@
 """Gabbi specific exceptions."""
 
 
+class GabbiDataLoadError(ValueError):
+    """An exception to alert when data streams cannot be loaded."""
+    pass
+
+
 class GabbiFormatError(ValueError):
     """An exception to encapsulate poorly formed test data."""
     pass

gabbi/handlers/base.py

@@ -109,7 +109,11 @@ def dumps(data, pretty=False, test=None):
 
     @staticmethod
     def loads(data):
-        """Create structured (Python) data from a stream."""
+        """Create structured (Python) data from a stream.
+
+        If there is a failure decoding then the handler should
+        repackage the error as a gabbi.exception.GabbiDataLoadError.
+        """
         return data
 
     @staticmethod

gabbi/handlers/jsonhandler.py

@@ -16,6 +16,7 @@
 
 import six
 
+from gabbi.exception import GabbiDataLoadError
 from gabbi.handlers import base
 from gabbi import json_parser
 
@@ -54,7 +55,10 @@ def dumps(data, pretty=False, test=None):
 
     @staticmethod
     def loads(data):
-        return json.loads(data)
+        try:
+            return json.loads(data)
+        except ValueError as exc:
+            raise GabbiDataLoadError('unable to parse data') from exc
 
     @staticmethod
     def load_data_file(test, file_path):

gabbi/tests/gabbits_intercept/disable-response-handler.yaml

@@ -0,0 +1,22 @@
+# Test that disabling the response handler despite an accept match
+# works as required.
+
+tests:
+
+- name: get some not json fail
+  desc: This will cause an error, presented as a test failure
+  xfail: True
+  GET: /notjson
+  response_headers:
+    content-type: application/json
+  response_strings:
+    - not valid json
+
+- name: get some not json gloss
+  desc: this will not error because we do not parse
+  GET: /notjson
+  response_headers:
+    content-type: application/json
+  disable_response_handler: True
+  response_strings:
+    - not valid json

gabbi/tests/simple_wsgi.py

@@ -92,6 +92,10 @@ def __call__(self, environ, start_response):
                         </body>
                     </html>
                     """]
+        # Provide response that claims to be json but is not.
+        elif path_info.startswith('/notjson'):
+            start_response('200 OK', [('Content-Type', 'application/json')])
+            return [b'not valid json']
         elif path_info.startswith('/poller'):
             if CURRENT_POLL == 0:
                 CURRENT_POLL = int(query_data.get('count', [5])[0])

gabbi/tests/test_live.py

@@ -47,7 +47,8 @@ def start_fixture(self):
 def load_tests(loader, tests, pattern):
     """Provide a TestSuite to the discovery process."""
     test_dir = os.path.join(os.path.dirname(__file__), TESTS_DIR)
-    return driver.build_tests(test_dir, loader, **BUILD_TEST_ARGS)
+    return driver.build_tests(
+        test_dir, loader, test_loader_name=__name__, **BUILD_TEST_ARGS)
 
 
 def pytest_generate_tests(metafunc):

test-failskip.sh

@@ -7,7 +7,7 @@ shopt -s nocasematch
 [[ "${GABBI_SKIP_NETWORK:-false}" == "true" ]] && SKIP=7 || SKIP=2
 shopt -u nocasematch
 
-FAILS=12
+FAILS=13
 
 GREP_FAIL_MATCH="expected failures=$FAILS"
 GREP_SKIP_MATCH="skipped=$SKIP,"

tox.ini

@@ -1,7 +1,7 @@
 [tox]
 minversion = 3.1.1
 skipsdist = True
-envlist = py36,py37,py38,py39,pypy3,pep8,limit,failskip,docs,py39-prefix,py39-limit,py39-verbosity,py39-failskip,py36-pytest,py37-pytest,py38-pytest,py39-pytest
+envlist = pep8,py36,py37,py38,py39,pypy3,pep8,limit,failskip,docs,py39-prefix,py39-limit,py39-verbosity,py39-failskip,py36-pytest,py37-pytest,py38-pytest,py39-pytest
 
 [testenv]
 deps = -r{toxinidir}/requirements.txt