exclude a specific API endpoints from logging requests fix #44

Author: Bob Bui

CHANGELOG.md

@@ -4,6 +4,9 @@ All notable changes to this project will be documented in this file.
 This project adheres to [Semantic Versioning](http://semver.org/).  
 The format is based on [Keep a Changelog](http://keepachangelog.com/).
 
+## 1.2.5 - 2020-08-04
+ - fix #44
+ 
 ## 1.2.4 - 2020-08-03
  - fix #51
 

README.md

@@ -11,6 +11,7 @@ If you're using Cloud Foundry, it might worth to check out the library [SAP/cf-p
    2.4 [Log extra properties](#24-log-extra-properties)  
    2.5 [Root logger](#25-root-logger)  
    2.6 [Custom log formatter](#26-custom-log-formatter)
+   2.7 [Exclude certain URl from request instrumentation](#27-exclude-certain-url-from-request-instrumentation)
 3. [Configuration](#3-configuration)  
 4. [Python References](#4-python-references)
 5. [Framework support plugin development](#5-framework-support-plugin-development)
@@ -190,6 +191,13 @@ json_logging.config_root_logger()
 Customer JSON log formatter can be passed to init method. see example for more detail: 
 https://github.com/thangbn/json-logging-python/blob/master/example/custom_log_format.py
 
+## 2.7 Exclude certain URl from request instrumentation
+Certain URL can be excluded from request instrumentation by specifying a list of regex into **init_request_instrument** method like below:
+
+```python
+json_logging.init_request_instrument(app, exclude_url_patterns=[r'/exclude_from_request_instrumentation'])
+```
+
 # 3. Configuration
 logging library can be configured by setting the value in json_logging, all configuration must be placed before json_logging.init method call
 

example/connexion-example/hello.py

@@ -1,17 +1,30 @@
 #!/usr/bin/env python3
+import logging
+import sys
 
 import connexion
 import json_logging
 
+# init the logger as usual
+logger = logging.getLogger("test logger")
+logger.setLevel(logging.DEBUG)
+logger.addHandler(logging.StreamHandler(sys.stdout))
+
 
 def post_greeting(name):
+    logger.info("test log statement")
+    logger.info("test log statement with extra props", extra={'props': {"extra_property": 'extra_value'}})
     return 'Hello {name}'.format(name=name)
 
 
+def exclude_from_request_instrumentation(name):
+    return 'Hello {name}. this request wont log request instrumentation information'.format(name=name)
+
+
 def create():
     app = connexion.FlaskApp(__name__, port=9090, specification_dir='openapi/')
     json_logging.init_connexion(enable_json=True)
-    json_logging.init_request_instrument(app)
+    json_logging.init_request_instrument(app, exclude_url_patterns=[r'/exclude_from_request_instrumentation'])
 
     app.add_api('helloworld-api.yaml', arguments={'title': 'Hello World Example'})
     return app

example/connexion-example/openapi/helloworld-api.yaml

@@ -28,3 +28,24 @@ paths:
           schema:
             type: string
             example: "dave"
+  /exclude_from_request_instrumentation:
+    get:
+      summary: Generate greeting
+      description: Generates a greeting message.
+      operationId: hello.exclude_from_request_instrumentation
+      responses:
+        200:
+          description: greeting response
+          content:
+            text/plain:
+              schema:
+                type: string
+                example: "hello dave!"
+      parameters:
+        - name: name
+          in: path
+          description: Name of the person to greet.
+          required: true
+          schema:
+            type: string
+            example: "dave"

example/flask_sample_app.py

@@ -7,7 +7,7 @@
 
 app = flask.Flask(__name__)
 json_logging.init_flask(enable_json=True)
-json_logging.init_request_instrument(app)
+json_logging.init_request_instrument(app, exclude_url_patterns=[r'/exclude_from_request_instrumentation'])
 
 # init the logger as usual
 logger = logging.getLogger("test logger")
@@ -34,5 +34,10 @@ def exception():
     return "Error occurred, check log for detail"
 
 
+@app.route('/exclude_from_request_instrumentation')
+def exclude_from_request_instrumentation():
+    return "this request wont log request instrumentation information"
+
+
 if __name__ == "__main__":
     app.run(host='0.0.0.0', port=int(5000), use_reloader=False)

example/quart_sample_app.py

@@ -8,7 +8,7 @@
 
 app = quart.Quart(__name__)
 json_logging.init_quart(enable_json=True)
-json_logging.init_request_instrument(app)
+json_logging.init_request_instrument(app, exclude_url_patterns=[r'/exclude_from_request_instrumentation'])
 
 # init the logger as usual
 logger = logging.getLogger("test logger")
@@ -25,6 +25,11 @@ async def home():
            "\ncorrelation_id                    : " + correlation_id
 
 
+@app.route('/exclude_from_request_instrumentation')
+def exclude_from_request_instrumentation():
+    return "this request wont log request instrumentation information"
+
+
 if __name__ == "__main__":
     loop = asyncio.get_event_loop()
     app.run(host='0.0.0.0', port=int(5001), use_reloader=False, loop=loop)

example/sanic_sample_app.py

@@ -8,7 +8,7 @@
 
 app = sanic.Sanic(name="sanic-web-app")
 json_logging.init_sanic(enable_json=True)
-json_logging.init_request_instrument(app)
+json_logging.init_request_instrument(app, exclude_url_patterns=[r'/exclude_from_request_instrumentation'])
 
 # init the logger as usual
 logger = logging.getLogger("sanic-integration-test-app")
@@ -31,5 +31,10 @@ def test(request):
         "\ncorrelation_id_without_request_obj: " + correlation_id_without_request_obj)
 
 
+@app.route('/exclude_from_request_instrumentation')
+def exclude_from_request_instrumentation(request):
+    return sanic.response.text("this request wont log request instrumentation information")
+
+
 if __name__ == "__main__":
     app.run(host="0.0.0.0", port=8000)

json_logging/__init__.py

@@ -31,7 +31,6 @@
 _request_util = None
 _default_formatter = None
 
-
 def get_correlation_id(request=None):
     """
     Get current request correlation-id. If one is not present, a new one might be generated
@@ -152,7 +151,7 @@ def __init(framework_name=None, custom_formatter=None, enable_json=False):
     util.update_formatter_for_loggers(existing_loggers, _default_formatter)
 
 
-def init_request_instrument(app=None, custom_formatter=None):
+def init_request_instrument(app=None, custom_formatter=None, exclude_url_patterns=[]):
     """
     Configure the request instrumentation logging configuration for given web app. Must be called after init method
 
@@ -170,7 +169,7 @@ def init_request_instrument(app=None, custom_formatter=None):
             raise ValueError('custom_formatter is not subclass of logging.Formatter', custom_formatter)
 
     configurator = _current_framework['app_request_instrumentation_configurator']()
-    configurator.config(app)
+    configurator.config(app, exclude_url_patterns=exclude_url_patterns)
 
     formatter = custom_formatter if custom_formatter else JSONRequestLogFormatter
     request_logger = configurator.request_logger

json_logging/framework/connexion/__init__.py

@@ -6,6 +6,7 @@
 import json_logging.framework
 from json_logging import JSONLogWebFormatter
 from json_logging.framework_base import AppRequestInstrumentationConfigurator, RequestAdapter, ResponseAdapter
+from json_logging.util import is_not_match_any_pattern
 
 
 def is_connexion_present():
@@ -30,7 +31,7 @@ def is_connexion_present():
 
 
 class ConnexionAppRequestInstrumentationConfigurator(AppRequestInstrumentationConfigurator):
-    def config(self, app):
+    def config(self, app, exclude_url_patterns=[]):
         if not is_connexion_present():
             raise RuntimeError("connexion is not available in system runtime")
         from flask.app import Flask
@@ -45,18 +46,19 @@ def config(self, app):
         # noinspection PyAttributeOutsideInit
         self.request_logger = logging.getLogger('connexion-request-logger')
 
-
         from flask import g
 
         @app.app.before_request
         def before_request():
-            g.request_info = json_logging.RequestInfo(_current_request)
+            if is_not_match_any_pattern(_current_request.path, exclude_url_patterns):
+                g.request_info = json_logging.RequestInfo(_current_request)
 
         @app.app.after_request
         def after_request(response):
-            request_info = g.request_info
-            request_info.update_response_status(response)
-            self.request_logger.info("", extra={'request_info': request_info})
+            if hasattr(g, 'request_info'):
+                request_info = g.request_info
+                request_info.update_response_status(response)
+                self.request_logger.info("", extra={'request_info': request_info})
             return response
 
 

json_logging/framework/flask/__init__.py

@@ -1,11 +1,12 @@
 # coding=utf-8
 import logging
-import sys
 
 import json_logging
 import json_logging.framework
 from json_logging.framework_base import AppRequestInstrumentationConfigurator, RequestAdapter, ResponseAdapter
 
+from json_logging.util import is_not_match_any_pattern
+
 
 def is_flask_present():
     # noinspection PyPep8,PyBroadException
@@ -25,7 +26,7 @@ def is_flask_present():
 
 
 class FlaskAppRequestInstrumentationConfigurator(AppRequestInstrumentationConfigurator):
-    def config(self, app):
+    def config(self, app, exclude_url_patterns=[]):
         if not is_flask_present():
             raise RuntimeError("flask is not available in system runtime")
         from flask.app import Flask
@@ -45,13 +46,15 @@ def config(self, app):
 
         @app.before_request
         def before_request():
-            g.request_info = json_logging.RequestInfo(_current_request)
+            if is_not_match_any_pattern(_current_request.path, exclude_url_patterns):
+                g.request_info = json_logging.RequestInfo(_current_request)
 
         @app.after_request
         def after_request(response):
-            request_info = g.request_info
-            request_info.update_response_status(response)
-            self.request_logger.info("", extra={'request_info': request_info})
+            if hasattr(g, 'request_info'):
+                request_info = g.request_info
+                request_info.update_response_status(response)
+                self.request_logger.info("", extra={'request_info': request_info})
             return response
 
 

json_logging/framework/quart/__init__.py

@@ -6,6 +6,7 @@
 import json_logging.framework
 from json_logging import JSONLogWebFormatter
 from json_logging.framework_base import AppRequestInstrumentationConfigurator, RequestAdapter, ResponseAdapter
+from json_logging.util import is_not_match_any_pattern
 
 
 def is_quart_present():
@@ -26,38 +27,39 @@ def is_quart_present():
 
 
 class QuartAppRequestInstrumentationConfigurator(AppRequestInstrumentationConfigurator):
-    def config(self, app):
+    def config(self, app, exclude_url_patterns=[]):
         if not is_quart_present():
             raise RuntimeError("quart is not available in system runtime")
         from quart.app import Quart
         if not isinstance(app, Quart):
             raise RuntimeError("app is not a valid quart.app.Quart app instance")
 
         # Remove quart logging handlers
-        from quart.logging import default_handler, serving_handler
+        from quart.logging import default_handler
         logging.getLogger('quart.app').removeHandler(default_handler)
-        logging.getLogger('quart.serving').removeHandler(serving_handler)
-
         json_logging.util.update_formatter_for_loggers([
-            # logging.getLogger('quart.app'),
-            # logging.getLogger('quart.serving'),
+            logging.getLogger('quart.app'),
         ], JSONLogWebFormatter)
 
+        logging.getLogger('quart.serving').disabled = True
+
         # noinspection PyAttributeOutsideInit
-        self.request_logger = logging.getLogger('quart.app')
+        self.request_logger = logging.getLogger('quart-request-logger')
 
         from quart import g
 
         @app.before_request
         def before_request():
-            g.request_info = json_logging.RequestInfo(_current_request)
+            if is_not_match_any_pattern(_current_request.path, exclude_url_patterns):
+                g.request_info = json_logging.RequestInfo(_current_request)
 
         @app.after_request
         def after_request(response):
-            request_info = g.request_info
-            request_info.update_response_status(response)
-            # TODO:handle to print out request instrumentation in non-JSON mode
-            self.request_logger.info("", extra={'request_info': request_info})
+            if hasattr(g, 'request_info'):
+                request_info = g.request_info
+                request_info.update_response_status(response)
+                # TODO:handle to print out request instrumentation in non-JSON mode
+                self.request_logger.info("", extra={'request_info': request_info})
             return response
 
 

json_logging/framework/sanic/__init__.py

@@ -8,6 +8,7 @@
 import json_logging.framework
 from json_logging.framework_base import FrameworkConfigurator, AppRequestInstrumentationConfigurator, RequestAdapter, \
     ResponseAdapter
+from json_logging.util import is_not_match_any_pattern
 
 
 def is_sanic_present():
@@ -41,7 +42,7 @@ def config(self):
 
 
 class SanicAppRequestInstrumentationConfigurator(AppRequestInstrumentationConfigurator):
-    def config(self, app):
+    def config(self, app, exclude_url_patterns=[]):
         if not is_sanic_present():
             raise RuntimeError("Sanic is not available in system runtime")
         # noinspection PyPackageRequirements
@@ -56,13 +57,15 @@ def config(self, app):
 
         @app.middleware('request')
         def before_request(request):
-            request.ctx.request_info = json_logging.RequestInfo(request)
+            if is_not_match_any_pattern(request.path, exclude_url_patterns):
+                request.ctx.request_info = json_logging.RequestInfo(request)
 
         @app.middleware('response')
         def after_request(request, response):
-            request_info = request.ctx.request_info
-            request_info.update_response_status(response)
-            self.request_logger.info("", extra={'request_info': request_info, 'type': 'request'})
+            if hasattr(request.ctx, 'request_info'):
+                request_info = request.ctx.request_info
+                request_info.update_response_status(response)
+                self.request_logger.info("", extra={'request_info': request_info, 'type': 'request'})
 
     def get_request_logger(self):
         return self.request_logger

json_logging/framework_base.py

@@ -189,7 +189,7 @@ def __new__(cls, *args, **kw):
             cls._instance.request_logger = None
         return cls._instance
 
-    def config(self, app):
+    def config(self, app, exclude_url_patterns=None):
         """
         configuration logic
 

json_logging/util.py

@@ -1,6 +1,7 @@
 # coding=utf-8
 import logging
 import os
+import re
 import sys
 from datetime import datetime
 from logging import Logger, StreamHandler
@@ -203,3 +204,7 @@ def _get_correlation_id_in_request_header(request_adapter, request):
             if value is not None:
                 return value
         return None
+
+
+def is_not_match_any_pattern(path, patterns):
+    return all(map(lambda pattern: re.search(pattern, path) is None, patterns))

setup.py

@@ -12,7 +12,7 @@
 
 setup(
     name="json-logging",
-    version='1.2.4',
+    version='1.2.5',
     packages=find_packages(exclude=['contrib', 'docs', 'tests*', 'example', 'dist', 'build']),
     license='Apache License 2.0',
     description="JSON Python Logging",