Add variable expansion (fix joke2k#421)

- Expand variables referenced as `$VAR` or `${VAR}`.
- Detect infinite recursion in expansion (self-reference).

Author: invasy

CHANGELOG.rst

@@ -11,6 +11,8 @@ Added
 +++++
 - Added support for secure Elasticsearch connections
   `#463 <https://github.com/joke2k/django-environ/pull/463>`_.
+- Added variable expansion
+  `#468 <https://github.com/joke2k/django-environ/pull/468>`_.
 
 Changed
 +++++++

docs/quickstart.rst

@@ -23,6 +23,28 @@ And use it with ``settings.py`` as follows:
    :start-after: -code-begin-
    :end-before: -overview-
 
+Variables can contain references to another variables: ``$VAR`` or ``${VAR}``.
+Referenced variables are searched in the environment and within all definitions
+in the ``.env`` file. References are checked for recursion (self-reference).
+Exception is thrown if any reference results in infinite loop on any level
+of recursion. Variable values are substituted similar to shell parameter
+expansion. Example:
+
+.. code-block:: shell
+
+   # shell
+   export POSTGRES_USERNAME='user' POSTGRES_PASSWORD='SECRET'
+
+.. code-block:: shell
+
+   # .env
+   POSTGRES_HOSTNAME='example.com'
+   POSTGRES_DB='database'
+   DATABASE_URL="postgres://${POSTGRES_USERNAME}:${POSTGRES_PASSWORD}@${POSTGRES_HOSTNAME}:5432/${POSTGRES_DB}"
+
+The value of ``DATABASE_URL`` variable will become
+``postgres://user:SECRET@example.com:5432/database``.
+
 The ``.env`` file should be specific to the environment and not checked into
 version control, it is best practice documenting the ``.env`` file with an example.
 For example, you can also add ``.env.dist`` with a template of your variables to

environ/environ.py

@@ -17,7 +17,9 @@
 import os
 import re
 import sys
+import threading
 import warnings
+from os.path import expandvars
 from urllib.parse import (
     parse_qs,
     ParseResult,
@@ -40,6 +42,9 @@
 Openable = (str, os.PathLike)
 logger = logging.getLogger(__name__)
 
+# Variables which values should not be expanded
+NOT_EXPANDED = 'DJANGO_SECRET_KEY', 'CACHE_URL'
+
 
 def _cast(value):
     # Safely evaluate an expression node or a string containing a Python
@@ -189,7 +194,11 @@ class Env:
                             for s in ('', 's')]
     CLOUDSQL = 'cloudsql'
 
+    VAR = re.compile(r'(?<!\\)\$\{?(?P<name>[A-Z_][0-9A-Z_]*)}?',
+                     re.IGNORECASE)
+
     def __init__(self, **scheme):
+        self._local = threading.local()
         self.smart_cast = True
         self.escape_proxy = False
         self.prefix = ""
@@ -343,9 +352,13 @@ def path(self, var, default=NOTSET, **kwargs):
         """
         return Path(self.get_value(var, default=default), **kwargs)
 
-    def get_value(self, var, cast=None, default=NOTSET, parse_default=False):
+    def get_value(self, var, cast=None,  # pylint: disable=R0913
+                  default=NOTSET, parse_default=False, add_prefix=True):
         """Return value for given environment variable.
 
+        - Expand variables referenced as ``$VAR`` or ``${VAR}``.
+        - Detect infinite recursion in expansion (self-reference).
+
         :param str var:
             Name of variable.
         :param collections.abc.Callable or None cast:
@@ -354,15 +367,33 @@ def get_value(self, var, cast=None, default=NOTSET, parse_default=False):
              If var not present in environ, return this instead.
         :param bool parse_default:
             Force to parse default.
+        :param bool add_prefix:
+            Whether to add prefix to variable name.
         :returns: Value from environment or default (if set).
         :rtype: typing.IO[typing.Any]
         """
-
+        var_name = f'{self.prefix}{var}' if add_prefix else var
+        if not hasattr(self._local, 'vars'):
+            self._local.vars = set()
+        if var_name in self._local.vars:
+            error_msg = f"Environment variable '{var_name}' recursively "\
+                        "references itself (eventually)"
+            raise ImproperlyConfigured(error_msg)
+
+        self._local.vars.add(var_name)
+        try:
+            return self._get_value(
+                var_name, cast=cast, default=default,
+                parse_default=parse_default)
+        finally:
+            self._local.vars.remove(var_name)
+
+    def _get_value(self, var_name, cast=None, default=NOTSET,
+                   parse_default=False):
         logger.debug(
             "get '%s' casted as '%s' with default '%s'",
-            var, cast, default)
+            var_name, cast, default)
 
-        var_name = f'{self.prefix}{var}'
         if var_name in self.scheme:
             var_info = self.scheme[var_name]
 
@@ -388,26 +419,37 @@ def get_value(self, var, cast=None, default=NOTSET, parse_default=False):
             value = self.ENVIRON[var_name]
         except KeyError as exc:
             if default is self.NOTSET:
-                error_msg = f'Set the {var} environment variable'
+                error_msg = f'Set the {var_name} environment variable'
                 raise ImproperlyConfigured(error_msg) from exc
 
             value = default
 
+        # Expand variables
+        if isinstance(value, (bytes, str)) and var_name not in NOT_EXPANDED:
+            def repl(match_):
+                return self.get_value(
+                    match_.group('name'), cast=cast, default=default,
+                    parse_default=parse_default, add_prefix=False)
+
+            is_bytes = isinstance(value, bytes)
+            if is_bytes:
+                value = value.decode('utf-8')
+            value = self.VAR.sub(repl, value)
+            value = expandvars(value)
+            if is_bytes:
+                value = value.encode('utf-8')
+
         # Resolve any proxied values
         prefix = b'$' if isinstance(value, bytes) else '$'
         escape = rb'\$' if isinstance(value, bytes) else r'\$'
-        if hasattr(value, 'startswith') and value.startswith(prefix):
-            value = value.lstrip(prefix)
-            value = self.get_value(value, cast=cast, default=default)
 
         if self.escape_proxy and hasattr(value, 'replace'):
             value = value.replace(escape, prefix)
 
         # Smart casting
-        if self.smart_cast:
-            if cast is None and default is not None and \
-                    not isinstance(default, NoValue):
-                cast = type(default)
+        if self.smart_cast and cast is None and default is not None \
+                and not isinstance(default, NoValue):
+            cast = type(default)
 
         value = None if default is None and value == '' else value
 

tests/test_expansion.py

@@ -0,0 +1,27 @@
+import pytest
+
+from environ import Env, Path
+from environ.compat import ImproperlyConfigured
+
+
+class TestExpansion:
+    def setup_method(self, method):
+        Env.ENVIRON = {}
+        self.env = Env()
+        self.env.read_env(Path(__file__, is_file=True)('test_expansion.txt'))
+
+    def test_expansion(self):
+        assert self.env('HELLO') == 'Hello, world!'
+
+    def test_braces(self):
+        assert self.env('BRACES') == 'Hello, world!'
+
+    def test_recursion(self):
+        with pytest.raises(ImproperlyConfigured) as excinfo:
+            self.env('RECURSIVE')
+        assert str(excinfo.value) == "Environment variable 'RECURSIVE' recursively references itself (eventually)"
+
+    def test_transitive(self):
+        with pytest.raises(ImproperlyConfigured) as excinfo:
+            self.env('R4')
+        assert str(excinfo.value) == "Environment variable 'R4' recursively references itself (eventually)"

tests/test_expansion.txt

@@ -0,0 +1,9 @@
+VAR1='Hello'
+VAR2='world'
+HELLO="$VAR1, $VAR2!"
+BRACES="${VAR1}, ${VAR2}!"
+RECURSIVE="This variable is $RECURSIVE"
+R1="$R2"
+R2="$R3"
+R3="$R4"
+R4="$R1"