Merge pull request #603 from crim-ca/wps-header-filter

Author: fmigneault

CHANGES.rst

@@ -12,7 +12,9 @@ Changes
 
 Changes:
 --------
-- No change.
+- Add ``weaver.wps_client_headers_filter`` setting that allows filtering of specific `WPS` request headers from the
+  incoming request to be passed down to the `WPS` client employed to interact with the `WPS` provider
+  (fixes `#600 <https://github.com/crim-ca/weaver/issues/600>`_).
 
 Fixes:
 ------

config/weaver.ini.example

@@ -113,6 +113,10 @@ weaver.wps_output_s3_bucket =
 weaver.wps_output_s3_region =
 weaver.wps_workdir =
 
+# List of comma-separated case-insensitive headers that will be removed from incoming requests before
+# passing them down to invoke an operation with the corresponding WPS provider through the WPS client.
+weaver.wps_client_headers_filter = Host,
+
 # --- Weaver WPS metadata ---
 # all attributes under "metadata:main" can be specified as 'weaver.wps_metadata_<field>'
 # (reference: https://pywps.readthedocs.io/en/master/configuration.html#metadata-main)

docs/source/configuration.rst

@@ -199,6 +199,16 @@ they are optional and which default value or operation is applied in each situat
   |
   | Prefix where process :term:`Job` worker should execute the :term:`Process` from.
 
+- | ``weaver.wps_client_headers_filter = <headers>``
+  | (default: ``Host,``)
+  |
+  | List of comma-separated case-insensitive headers that will be removed from incoming requests before
+  | passing them down to invoke an operation with the corresponding :term:`WPS` provider through the :term:`WPS` client.
+
+.. seealso::
+    - :func:`weaver.wps.utils.get_wps_client_filtered_headers`
+    - :func:`weaver.wps.utils.get_wps_client`
+
 - | ``weaver.wps_restapi = true|false`` [:class:`bool`-like]
   | (default: ``true``)
   |

tests/wps/test_utils.py

@@ -14,9 +14,11 @@
 from weaver import xml_util
 from weaver.formats import AcceptLanguage, ContentType
 from weaver.owsexceptions import OWSAccessForbidden, OWSException, OWSMissingParameterValue, OWSNoApplicableCode
+from weaver.utils import null
 from weaver.wps.utils import (
     get_exception_from_xml_status,
     get_wps_client,
+    get_wps_client_filtered_headers,
     get_wps_output_context,
     map_wps_output_location,
     set_wps_language
@@ -91,14 +93,17 @@ def test_get_wps_client_headers_preserved():
     """
     Validate that original request headers are not modified following WPS client sub-requests.
     """
-    test_wps_url = "http://dont-care.com/wps"
+    test_wps_host = "dont-care.com"
+    test_wps_url = f"http://{test_wps_host}/wps"
     test_headers = {
+        "Host": test_wps_host,  # this should be filtered out
         "Content-Type": ContentType.APP_XML,
         "Content-Length": "0",
         "Accept-Language": AcceptLanguage.FR_CA,
         "Accept": ContentType.APP_JSON,
         "Authorization": "Bearer: FAKE",  # nosec
     }
+    test_settings = {"weaver.wps_client_headers_filter": "Host,"}
     test_copy_headers = copy.deepcopy(test_headers)
     # following are removed for sub-request
     test_wps_headers = {
@@ -112,7 +117,7 @@ def test_get_wps_client_headers_preserved():
         for patch in patches:
             mocks.append(stack.enter_context(patch))
 
-        wps = get_wps_client(test_wps_url, headers=test_headers)
+        wps = get_wps_client(test_wps_url, headers=test_headers, container=test_settings)
 
     for mocked in mocks:
         assert mocked.called
@@ -122,6 +127,31 @@ def test_get_wps_client_headers_preserved():
     assert wps.url == test_wps_url
 
 
+@pytest.mark.parametrize(
+    ["test_headers", "test_filter", "expect_result"],
+    [
+        # 'null' not supported by function itself, only for us to consider no settings container at all
+        ({}, null, {}),
+        ({"Host": "test.com"}, null, {}),  # Host is default if no settings
+        ({}, None, {}),
+        ({"Host": "test.com"}, None, {}),
+        ({}, "", {}),
+        ({}, [], {}),
+        ({"Authorization": "random"}, [], {"Authorization": "random"}),
+        ({"Authorization": "random"}, "Host", {"Authorization": "random"}),
+        ({"X-Test": "OK", "x-custom": "bye", "Host": "test.com"}, ["Host", "X-custom"], {"X-Test": "OK"}),
+        ({"Host": "example.com", "Authorization": "random"}, "host", {"Authorization": "random"}),
+        ({"Host": "example.com", "Authorization": "random"}, "Host,", {"Authorization": "random"}),
+        ({"Host": "example.com", "Authorization": "random"}, "HOST,", {"Authorization": "random"}),
+        ({"Host": "example.com", "Authorization": "random"}, ["Host"], {"Authorization": "random"}),
+    ]
+)
+def test_get_wps_client_filtered_headers(test_headers, test_filter, expect_result):
+    settings = {"weaver.wps_client_headers_filter": test_filter} if test_filter is not null else None
+    result = get_wps_client_filtered_headers(test_headers, settings)
+    assert result == expect_result
+
+
 def test_get_wps_output_context_validation():
     bad_cases = [
         "test/////test",

weaver/utils.py

@@ -149,6 +149,8 @@
         "headers": NotRequired[AnyHeadersContainer],
         "cookies": NotRequired[AnyCookiesContainer],
         "stream": NotRequired[bool],
+        "cache": NotRequired[bool],
+        "cache_enabled": NotRequired[bool],
     }, total=False)
     RequestCachingKeywords = Dict[str, AnyValueType]
     RequestCachingFunction = Callable[[AnyRequestMethod, str, RequestCachingKeywords], Response]
@@ -1670,7 +1672,7 @@ def invalidate_region(caching_args):
 
 
 def get_ssl_verify_option(method, url, settings, request_options=None):
-    # type: (str, str, AnySettingsContainer, Optional[SettingsType]) -> bool
+    # type: (str, str, AnySettingsContainer, Optional[RequestOptions]) -> bool
     """
     Obtains the SSL verification option considering multiple setting definitions and the provided request context.
 
@@ -1695,9 +1697,9 @@ def get_ssl_verify_option(method, url, settings, request_options=None):
 
 
 def get_no_cache_option(request_headers, **cache_options):
-    # type: (HeadersType, **bool) -> bool
+    # type: (HeadersType, **bool | RequestOptions) -> bool
     """
-    Obtains the No-Cache result from request headers and configured request options.
+    Obtains the ``No-Cache`` result from request headers and configured :term:`Request Options`.
 
     .. seealso::
         - :meth:`Request.headers`
@@ -1717,7 +1719,7 @@ def get_no_cache_option(request_headers, **cache_options):
 def get_request_options(method, url, settings):
     # type: (str, str, AnySettingsContainer) -> RequestOptions
     """
-    Obtains the *request options* corresponding to the request from the configuration file.
+    Obtains the :term:`Request Options` corresponding to the request from the configuration file.
 
     The configuration file specified is expected to be pre-loaded within setting ``weaver.request_options``.
     If no file was pre-loaded or no match is found for the request, an empty options dictionary is returned.

weaver/wps/utils.py

@@ -12,6 +12,7 @@
 from owslib.wps import WebProcessingService, WPSExecution
 from pyramid.httpexceptions import HTTPNotFound, HTTPOk, HTTPUnprocessableEntity
 from pywps import configuration as pywps_config
+from requests.structures import CaseInsensitiveDict
 from webob.acceptparse import create_accept_language_header
 
 from weaver import owsexceptions, xml_util
@@ -281,6 +282,7 @@ def get_wps_client(url, container=None, verify=None, headers=None, language=None
         language = str(language)
     if headers is not None and not isinstance(headers, dict):
         headers = dict(headers)
+    headers = get_wps_client_filtered_headers(headers, container)
     request_args = (url, headers, verify, language)
     if get_no_cache_option(headers, request_options=opts):
         for func in (_get_wps_client_cached, _describe_process_cached):
@@ -290,6 +292,30 @@ def get_wps_client(url, container=None, verify=None, headers=None, language=None
     return wps
 
 
+def get_wps_client_filtered_headers(headers, container):
+    # type: (Optional[HeadersType], AnySettingsContainer) -> HeadersType
+    """
+    Filters out any headers configured for the :term:`WPS` client by the ``weaver.wps_client_headers_filter`` setting.
+
+    :param headers: Headers to filter as applicable.
+    :param container: Any settings container to retrieve application settings.
+    :return: Filtered :term:`WPS` headers.
+    """
+    if not headers:
+        return {}
+    settings = get_settings(container) or {}
+    hdr_spec = settings.get("weaver.wps_client_headers_filter") or "Host,"  # default if missing (match docs/example)
+    if isinstance(hdr_spec, str):
+        hdr_spec = [hdr.strip() for hdr in hdr_spec.split(",")]
+    hdr_spec = [hdr for hdr in hdr_spec if hdr]
+    if not hdr_spec:
+        return headers
+    headers = CaseInsensitiveDict(headers.copy())
+    for hdr in hdr_spec:
+        headers.pop(hdr, None)
+    return dict(headers)
+
+
 def check_wps_status(location=None,     # type: Optional[str]
                      response=None,     # type: Optional[xml_util.XML]
                      sleep_secs=2,      # type: int