feat(app-b): add otlp sdk configuration

Author: willianantunes

b/.dockerignore

@@ -3,6 +3,7 @@
 !letter_b
 !scripts
 !manage.py
+!otlp.py
 !poetry*
 !pyproject.toml
 !gunicorn_config.py

b/.env

@@ -24,10 +24,11 @@ DB_USE_REPLICA=False
 ####################
 #### Logging config
 ROOT_LOG_LEVEL=INFO
-DEFAULT_LOG_FORMATTER=development
+DEFAULT_LOG_FORMATTER=otlp
 STOMP_LOG_LEVEL=WARNING
 DJANGO_DB_BACKENDS_LOG_LEVEL=INFO
 DJANGO_REQUEST_LOG_LEVEL=INFO
+PROJECT_LOG_LEVEL=INFO
 
 ##################
 #### DEBUG LIB CONFIGURATION
@@ -44,3 +45,15 @@ STOMP_SERVER_USER=guest
 STOMP_SERVER_PASSWORD=guest
 STOMP_USE_SSL=False
 CREATE_AUDIT_ACTION_DESTINATION=/queue/create-audit-action
+
+##################
+#### OTL
+# https://opentelemetry.io/docs/specs/otel/configuration/sdk-environment-variables/
+# https://opentelemetry.io/docs/languages/sdk-configuration/otlp-exporter/
+OTEL_EXPORTER_OTLP_ENDPOINT=http://otel-collector:4317
+OTEL_EXPORTER_OTLP_LOGS_ENDPOINT=http://otel-collector:4317
+OTEL_EXPORTER_OTLP_LOGS_INSECURE=true
+OTEL_RESOURCE_ATTRIBUTES=service.name=letter-b,service.version=unknown
+OTEL_LOG_LEVEL=debug
+OTEL_PYTHON_LOG_LEVEL=DEBUG
+OTEL_PYTHON_LOG_CORRELATION=true

b/README.md

@@ -8,6 +8,8 @@ Execute `docker compose up app-production` to start API server in production mod
 - http://localhost:8080/api/v1/
 - http://localhost:8080/api/v1/users/attributes
 
+Look at traces, logs, and metrics at `http://localhost:8090`.
+
 ## Recurring procedures
 
 ### Updating the lock file

b/docker-compose.yml

@@ -41,6 +41,8 @@ services:
         condition: service_healthy
       rabbitmq:
         condition: service_healthy
+      otel-collector:
+        condition: service_started
     command:
       [
         "./scripts/start-development.sh",
@@ -85,6 +87,8 @@ services:
         condition: service_healthy
       rabbitmq:
         condition: service_healthy
+      otel-collector:
+        condition: service_started
     command:
       [
         "./scripts/start-production.sh",
@@ -95,6 +99,29 @@ services:
       timeout: 5s
       retries: 5
 
+  otel-collector:
+    image: otel/opentelemetry-collector-contrib:0.113.0
+    command: [ "--config=/etc/otel-collector-config.yml" ]
+    volumes:
+      - ./otel-collector-config.yml:/etc/otel-collector-config.yml
+    depends_on:
+      hyperdx:
+        condition: service_healthy
+
+  hyperdx:
+    image: hyperdx/hyperdx-local:1.10.0
+    environment:
+      HYPERDX_APP_PORT: 8090 # HyperDX UI (Next.js)
+      HYPERDX_API_PORT: 8091 # Private HyperDX API for the UI
+    ports:
+      - "8090:8080"
+      - "8091:8000"
+      - "4317:4317"
+    healthcheck:
+      test: curl --fail http://localhost:8000/health || exit 1
+      interval: 10s
+      timeout: 3s
+
   integration-tests:
     build: *dockerfile-dev-build
     env_file: .env

b/gunicorn_config.py

@@ -1,13 +1,25 @@
-import psycogreen.gevent
+from gevent import monkey
 
-# https://github.com/psycopg/psycogreen/blob/39a258cb4040b88b60a7600f6942e651a28db9a7/README.rst#module-psycogreengevent
-psycogreen.gevent.patch_psycopg()
+# If you don't do this, you'll get the following error for example:
+# DatabaseWrapper objects created in a thread can only be used in that same thread. The object with alias 'default' was created in thread id 140208374647872 and this is thread id 140208369922144
+monkey.patch_all()
 
 import json
 import os
 
 from pythonjsonlogger.jsonlogger import JsonFormatter
 
+
+def post_fork(server, worker):
+    """
+    https://opentelemetry-python.readthedocs.io/en/stable/examples/fork-process-model/README.html#gunicorn-post-fork-hook
+    """
+    server.log.info("Worker spawned (pid: %s)", worker.pid)
+    from otlp import configure_opentelemetry
+
+    configure_opentelemetry()
+
+
 bind = os.environ.get("DJANGO_BIND_ADDRESS", "0.0.0.0") + ":" + os.environ.get("DJANGO_BIND_PORT", "8000")
 backlog = int(os.getenv("GUNICORN_BACKLOG", "2048"))
 workers = int(os.getenv("GUNICORN_WORKERS", "3"))

b/letter_b/settings.py

@@ -158,7 +158,7 @@
 
 
 # Logging
-# https://docs.djangoproject.com/en/4.0/topics/logging/
+# https://docs.djangoproject.com/en/5.1/topics/logging/
 
 LOGGING = {
     "version": 1,
@@ -175,6 +175,10 @@
             "()": JsonFormatter,
             "format": "%(levelname)-8s [%(asctime)s] [%(request_id)s] %(name)s: %(message)s",
         },
+        "otlp": {
+            "()": Formatter,
+            "format": "%(asctime)s %(levelname)s [%(name)s] [%(filename)s:%(lineno)d] [trace_id=%(otelTraceID)s span_id=%(otelSpanID)s resource.service.name=%(otelServiceName)s trace_sampled=%(otelTraceSampled)s] - %(message)s",
+        },
         "development": {
             "()": Formatter,
             "format": "%(levelname)-8s [%(asctime)s] [%(request_id)s] %(name)s: %(message)s",
@@ -185,10 +189,11 @@
             "class": "logging.StreamHandler",
             "filters": ["request_id", "redact_filter"],
             "formatter": os.getenv("DEFAULT_LOG_FORMATTER", "standard"),
-        }
+        },
+        "otlp": {"class": "otlp.CustomLoggingHandler"},
     },
     "loggers": {
-        "": {"level": os.getenv("ROOT_LOG_LEVEL", "INFO"), "handlers": ["console"]},
+        "": {"level": os.getenv("ROOT_LOG_LEVEL", "INFO"), "handlers": ["console", "otlp"]},
         "letter_b": {
             "level": os.getenv("PROJECT_LOG_LEVEL", "INFO"),
             "handlers": ["console"],

b/letter_b/wsgi.py

@@ -3,10 +3,13 @@
 from django.conf import settings
 from django.contrib.staticfiles.handlers import StaticFilesHandler
 from django.core.wsgi import get_wsgi_application
+from opentelemetry.instrumentation.wsgi import OpenTelemetryMiddleware
 
 os.environ.setdefault("DJANGO_SETTINGS_MODULE", "letter_b.settings")
 
+wsgi_application = OpenTelemetryMiddleware(get_wsgi_application())
+
 if settings.USE_STATIC_FILE_HANDLER_FROM_WSGI:
-    application = StaticFilesHandler(get_wsgi_application())
+    application = StaticFilesHandler(wsgi_application)
 else:
-    application = get_wsgi_application()
+    application = wsgi_application

b/manage.py

@@ -3,11 +3,15 @@
 import os
 import sys
 
+from otlp import configure_opentelemetry
+
 
 def main():
     os.environ.setdefault("DJANGO_SETTINGS_MODULE", "letter_b.settings")
     try:
         from django.core.management import execute_from_command_line
+
+        configure_opentelemetry()
     except ImportError as exc:
         raise ImportError(
             "Couldn't import Django. Are you sure it's installed and "

b/otel-collector-config.yml

@@ -0,0 +1,51 @@
+# https://opentelemetry.io/docs/collector/configuration/
+
+# Receivers collect telemetry from one or more sources. 
+# They can be pull or push based, and may support one or more data sources.
+receivers:
+  otlp:
+    protocols:
+      grpc:
+        endpoint: 0.0.0.0:4317
+      http:
+        endpoint: 0.0.0.0:4318
+
+# Processors take the data collected by receivers and modify or transform it before sending it to the exporters.
+processors:
+  batch: # https://github.com/open-telemetry/opentelemetry-collector/blob/e1a688fc4ef45306b2782b68423b21334b5ebd18/processor/batchprocessor/README.md
+    timeout: 5s
+    send_batch_size: 10
+    send_batch_max_size: 100
+
+# Exporters send data to one or more backends or destinations.
+exporters:
+  debug:
+    verbosity: detailed
+  otlp/hyperdx:
+    endpoint: "hyperdx:4317"
+    tls:
+      insecure: true
+
+extensions:
+  health_check:
+  pprof:
+  zpages:
+
+service:
+  extensions: [ health_check, pprof, zpages ]
+  telemetry:
+    logs:
+      level: "debug"
+  pipelines:
+    traces:
+      receivers: [ otlp ]
+      processors: [ batch ]
+      exporters: [ otlp/hyperdx ]
+    metrics:
+      receivers: [ otlp ]
+      processors: [ batch ]
+      exporters: [ otlp/hyperdx ]
+    logs:
+      receivers: [ otlp ]
+      processors: [ batch ]
+      exporters: [ otlp/hyperdx ]

b/otlp.py

@@ -0,0 +1,38 @@
+import logging
+
+from opentelemetry import trace
+from opentelemetry._logs import set_logger_provider
+from opentelemetry.exporter.otlp.proto.grpc._log_exporter import OTLPLogExporter
+from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import OTLPSpanExporter
+from opentelemetry.instrumentation.django import DjangoInstrumentor
+from opentelemetry.instrumentation.logging import LoggingInstrumentor
+from opentelemetry.instrumentation.psycopg import PsycopgInstrumentor
+from opentelemetry.sdk._logs import LoggerProvider
+from opentelemetry.sdk._logs import LoggingHandler
+from opentelemetry.sdk._logs.export import BatchLogRecordProcessor
+from opentelemetry.sdk.trace import TracerProvider
+from opentelemetry.sdk.trace.export import BatchSpanProcessor
+
+
+# This is experimental and not recommended for production use.
+# https://github.com/open-telemetry/opentelemetry-python/discussions/4294
+class CustomLoggingHandler(LoggingHandler):
+    def __init__(self, level=logging.NOTSET, logger_provider=None) -> None:
+        logger_provider = LoggerProvider()
+        set_logger_provider(logger_provider)
+        logger_provider.add_log_record_processor(BatchLogRecordProcessor(OTLPLogExporter()))
+        super().__init__(level, logger_provider)
+
+
+def configure_opentelemetry():
+    # https://opentelemetry-python.readthedocs.io/en/stable/exporter/otlp/otlp.html
+    # Don't customize the provider, exporter, and so on programmatically. Use environment variables instead.
+    tracer_provider = TracerProvider()
+    trace.set_tracer_provider(tracer_provider)
+    tracer_provider.add_span_processor(BatchSpanProcessor(OTLPSpanExporter()))
+    # TSHOOT PURPOSES ONLY: tracer_provider.add_span_processor(BatchSpanProcessor(ConsoleSpanExporter()))
+    # https://opentelemetry-python-contrib.readthedocs.io/en/latest/instrumentation/django/django.html#usage
+    LoggingInstrumentor().instrument(tracer_provider=tracer_provider)
+    DjangoInstrumentor().instrument(tracer_provider=tracer_provider)
+    # https://opentelemetry-python-contrib.readthedocs.io/en/latest/instrumentation/psycopg/psycopg.html#module-opentelemetry.instrumentation.psycopg
+    PsycopgInstrumentor().instrument(tracer_provider=tracer_provider, enable_commenter=True)