Split redash/__init__.py to prevent import time side-effects.

Refs #3569 and #3466.

Author: jezdez

redash/__init__.py

@@ -1,27 +1,29 @@
-import sys
 import logging
-import urlparse
+import os
+import sys
 import urllib
+import urlparse
 
 import redis
+<<<<<<< HEAD
 from flask import Flask, current_app
 from werkzeug.contrib.fixers import ProxyFix
 from werkzeug.routing import BaseConverter
 from statsd import StatsClient
+=======
+>>>>>>> Split redash/__init__.py to prevent import time side-effects.
 from flask_mail import Mail
 from flask_limiter import Limiter
 from flask_limiter.util import get_ipaddr
 from flask_migrate import Migrate
+from statsd import StatsClient
+from werkzeug.local import LocalProxy
 
-from redash import settings
-from redash.query_runner import import_query_runners
-from redash.destinations import import_destinations
-
+from . import settings
 
 __version__ = '7.0.0'
 
 
-import os
 if os.environ.get("REMOTE_DEBUG"):
     import ptvsd
     ptvsd.enable_attach(address=('0.0.0.0', 5678))
@@ -36,10 +38,8 @@ def setup_logging():
 
     # Make noisy libraries less noisy
     if settings.LOG_LEVEL != "DEBUG":
-        logging.getLogger("passlib").setLevel("ERROR")
-        logging.getLogger("requests.packages.urllib3").setLevel("ERROR")
-        logging.getLogger("snowflake.connector").setLevel("ERROR")
-        logging.getLogger('apiclient').setLevel("ERROR")
+        for name in ["passlib", "requests.packages.urllib3", "snowflake.connector", "apiclient"]:
+            logging.getLogger(name).setLevel("ERROR")
 
 
 def create_redis_connection():
@@ -67,69 +67,13 @@ def create_redis_connection():
 
 
 setup_logging()
-redis_connection = create_redis_connection()
-
-mail = Mail()
-migrate = Migrate()
-mail.init_mail(settings.all_settings())
-statsd_client = StatsClient(host=settings.STATSD_HOST, port=settings.STATSD_PORT, prefix=settings.STATSD_PREFIX)
-limiter = Limiter(key_func=get_ipaddr, storage_uri=settings.LIMITER_STORAGE)
 
-import_query_runners(settings.QUERY_RUNNERS)
-import_destinations(settings.DESTINATIONS)
+redis_connection = LocalProxy(create_redis_connection)
 
-from redash.version_check import reset_new_version_status
-reset_new_version_status()
+mail = LocalProxy(Mail)
 
+migrate = LocalProxy(Migrate)
 
-class SlugConverter(BaseConverter):
-    def to_python(self, value):
-        # This is ay workaround for when we enable multi-org and some files are being called by the index rule:
-        # for path in settings.STATIC_ASSETS_PATHS:
-        #     full_path = safe_join(path, value)
-        #     if os.path.isfile(full_path):
-        #         raise ValidationError()
-
-        return value
-
-    def to_url(self, value):
-        return value
-
-
-def create_app():
-    from redash import authentication, extensions, handlers, security
-    from redash.handlers.webpack import configure_webpack
-    from redash.handlers import chrome_logger
-    from redash.models import db, users
-    from redash.metrics import request as request_metrics
-    from redash.utils import sentry
-
-    sentry.init()
-
-    app = Flask(__name__,
-                template_folder=settings.STATIC_ASSETS_PATH,
-                static_folder=settings.STATIC_ASSETS_PATH,
-                static_url_path='/static')
-
-    # Make sure we get the right referral address even behind proxies like nginx.
-    app.wsgi_app = ProxyFix(app.wsgi_app, settings.PROXIES_COUNT)
-    app.url_map.converters['org_slug'] = SlugConverter
-
-    # configure our database
-    app.config['SQLALCHEMY_DATABASE_URI'] = settings.SQLALCHEMY_DATABASE_URI
-    app.config.update(settings.all_settings())
-
-    security.init_app(app)
-    request_metrics.init_app(app)
-    db.init_app(app)
-    migrate.init_app(app, db)
-    mail.init_app(app)
-    authentication.init_app(app)
-    limiter.init_app(app)
-    handlers.init_app(app)
-    configure_webpack(app)
-    extensions.init_app(app)
-    chrome_logger.init_app(app)
-    users.init_app(app)
+statsd_client = StatsClient(host=settings.STATSD_HOST, port=settings.STATSD_PORT, prefix=settings.STATSD_PREFIX)
 
-    return app
+limiter = Limiter(key_func=get_ipaddr, storage_uri=settings.LIMITER_STORAGE)

redash/app.py

@@ -0,0 +1,59 @@
+from flask import Flask
+from werkzeug.contrib.fixers import ProxyFix
+
+from . import mail, settings
+from .utils import routes
+
+
+class Redash(Flask):
+    """A custom Flask app for Redash"""
+    def __init__(self, *args, **kwargs):
+        kwargs.update({
+            'template_folder': settings.STATIC_ASSETS_PATH,
+            'static_folder': settings.STATIC_ASSETS_PATH,
+            'static_path': '/static',
+        })
+        super(Redash, self).__init__(__name__, *args, **kwargs)
+        # Make sure we get the right referral address even behind proxies like nginx.
+        self.wsgi_app = ProxyFix(self.wsgi_app, settings.PROXIES_COUNT)
+        # Handle the special case of the org slug
+        self.url_map.converters['org_slug'] = routes.SlugConverter
+        # Configure Redash using our settings
+        self.config.from_object('redash.settings')
+
+
+def create_app():
+    from . import authentication, extensions, handlers, limiter, migrate, security
+    from .destinations import import_destinations
+    from .handlers import chrome_logger
+    from .handlers.webpack import configure_webpack
+    from .metrics import request as request_metrics
+    from .models import db, users
+    from .query_runner import import_query_runners
+    from .utils import sentry
+    from .version_check import reset_new_version_status
+
+    sentry.init()
+    app = Redash()
+
+    # Check and update the cached version for use by the client
+    app.before_first_request(reset_new_version_status)
+
+    # Load query runners and destinations
+    import_query_runners(settings.QUERY_RUNNERS)
+    import_destinations(settings.DESTINATIONS)
+
+    security.init_app(app)
+    request_metrics.init_app(app)
+    db.init_app(app)
+    migrate.init_app(app, db)
+    mail.init_app(app)
+    authentication.init_app(app)
+    limiter.init_app(app)
+    handlers.init_app(app)
+    configure_webpack(app)
+    extensions.init_app(app)
+    chrome_logger.init_app(app)
+    users.init_app(app)
+
+    return app

redash/cli/__init__.py

@@ -4,7 +4,8 @@
 from flask.cli import FlaskGroup, run_command
 from flask import current_app
 
-from redash import create_app, settings, __version__
+from redash import settings, __version__
+from redash.app import create_app
 from redash.cli import users, groups, database, data_sources, organization
 from redash.monitor import get_status
 
@@ -15,9 +16,11 @@ def create(group):
 
     @app.shell_context_processor
     def shell_context():
-        from redash import models
-        return dict(models=models)
-
+        from redash import models, settings
+        return {
+            'models': models,
+            'settings': settings,
+        }
     return app
 
 
@@ -48,7 +51,7 @@ def status():
 @manager.command()
 def check_settings():
     """Show the settings as Redash sees them (useful for debugging)."""
-    for name, item in settings.all_settings().iteritems():
+    for name, item in current_app.config.iteritems():
         print("{} = {}".format(name, item))
 
 

redash/metrics/celery.py

@@ -1,17 +1,13 @@
-from __future__ import absolute_import
-
 import logging
 import socket
 import time
 
-from celery.signals import task_postrun, task_prerun
 from redash import settings, statsd_client
 from redash.utils import json_dumps
 
 tasks_start_time = {}
 
 
-@task_prerun.connect
 def task_prerun_handler(signal, sender, task_id, task, args, kwargs, **kw):
     try:
         tasks_start_time[task_id] = time.time()
@@ -29,7 +25,6 @@ def metric_name(name, tags):
     return "{},{}".format(name, tags_string)
 
 
-@task_postrun.connect
 def task_postrun_handler(signal, sender, task_id, task, args, kwargs, retval, state, **kw):
     try:
         run_time = 1000 * (time.time() - tasks_start_time.pop(task_id))

redash/settings/__init__.py

@@ -4,19 +4,7 @@
 from flask_talisman import talisman
 
 from .helpers import fix_assets_path, array_from_string, parse_boolean, int_or_none, set_from_string
-from .organization import DATE_FORMAT
-
-
-def all_settings():
-    from types import ModuleType
-
-    settings = {}
-    for name, item in globals().iteritems():
-        if not callable(item) and not name.startswith("__") and not isinstance(item, ModuleType):
-            settings[name] = item
-
-    return settings
-
+from .organization import DATE_FORMAT  # noqa
 
 REDIS_URL = os.environ.get('REDASH_REDIS_URL', os.environ.get('REDIS_URL', "redis://localhost:6379/0"))
 PROXIES_COUNT = int(os.environ.get('REDASH_PROXIES_COUNT', "1"))

redash/tasks/queries.py

@@ -11,7 +11,7 @@
 from redash import models, redis_connection, settings, statsd_client
 from redash.query_runner import InterruptException
 from redash.tasks.alerts import check_alerts_for_query
-from redash.utils import gen_query_hash, json_dumps, json_loads, utcnow, mustache_render
+from redash.utils import gen_query_hash, json_dumps, utcnow, mustache_render
 from redash.worker import celery
 
 logger = get_task_logger(__name__)

redash/utils/routes.py

@@ -0,0 +1,15 @@
+from werkzeug.routing import BaseConverter
+
+
+class SlugConverter(BaseConverter):
+    def to_python(self, value):
+        # This is ay workaround for when we enable multi-org and some files are being called by the index rule:
+        # for path in settings.STATIC_ASSETS_PATHS:
+        #     full_path = safe_join(path, value)
+        #     if os.path.isfile(full_path):
+        #         raise ValidationError()
+
+        return value
+
+    def to_url(self, value):
+        return value

redash/worker.py

@@ -7,9 +7,12 @@
 
 from celery import Celery
 from celery.schedules import crontab
-from celery.signals import worker_process_init
-from redash import create_app, settings
-from redash.metrics import celery as celery_metrics
+from celery.signals import task_postrun, task_prerun, worker_process_init
+
+from redash import settings
+from redash.app import create_app
+from redash.metrics.celery import task_postrun_handler, task_prerun_handler
+
 
 celery = Celery('redash',
                 broker=settings.CELERY_BROKER,
@@ -66,6 +69,10 @@ def __call__(self, *args, **kwargs):
 
 celery.Task = ContextTask
 
+# Connect the task signal handles for Redash metrics
+task_prerun.connect(task_prerun_handler)
+task_postrun.connect(task_postrun_handler)
+
 
 # Create Flask app after forking a new worker, to make sure no resources are shared between processes.
 @worker_process_init.connect

redash/wsgi.py

@@ -1,3 +1,3 @@
-from redash import create_app
+from redash.app import create_app
 
 app = create_app()

tests/__init__.py

@@ -16,8 +16,8 @@
 # Make sure rate limit is enabled
 os.environ['REDASH_RATELIMIT_ENABLED'] = "true"
 
-from redash import create_app, limiter
-from redash import redis_connection
+from redash import limiter, redis_connection
+from redash.app import create_app
 from redash.models import db
 from redash.utils import json_dumps, json_loads
 from tests.factories import Factory, user_factory