Workaround docker-compose-v2 env passing (#18887)

* Workaround docker-compose-v2 env passing

Docker Compose v2 has environment parsing broken in many ways.
Until this is fixed, we cannot use env files, instead we must
set all the variables directly, because parsing variables without
values or parsing variables which have empty values is broken
in several ways. Some of the issues are closed but not released,
and until this is fixed, some extra code duplication and explicitly
setting all default variables to "" when needed should solve the
problem for both Docker-Compose v1 and Docker-Compose v2

(cherry picked from commit ab5b2bf)

Author: potiuk

breeze

@@ -567,24 +567,6 @@ EOF
 # Those are a convenience scripts that you might use to debug command execution although
 # In most cases they are used internally by Breeze.
 #
-# Used Globals:
-#   BRANCH_NAME
-#   PYTHON_MAJOR_MINOR_VERSION
-#   BACKEND
-#   AIRFLOW_VERSION
-#   INSTALL_AIRFLOW_VERSION
-#   SSH_PORT
-#   WEBSERVER_HOST_PORT
-#   POSTGRES_HOST_PORT
-#   POSTGRES_VERSION
-#   MYSQL_HOST_PORT
-#   MYSQL_VERSION
-#   AIRFLOW_SOURCES
-#   AIRFLOW_CI_IMAGE
-#   AIRFLOW_PROD_IMAGE
-#   AIRFLOW_IMAGE_KUBERNETES
-#   SQLITE_URL
-#
 # Arguments:
 #
 #   file to prepare
@@ -657,6 +639,17 @@ export SQLITE_URL="${SQLITE_URL}"
 export USE_AIRFLOW_VERSION="${USE_AIRFLOW_VERSION}"
 export USE_PACKAGES_FROM_DIST="${USE_PACKAGES_FROM_DIST}"
 export EXECUTOR="${EXECUTOR}"
+export START_AIRFLOW="${START_AIRFLOW}"
+export ENABLED_INTEGRATIONS="${ENABLED_INTEGRATIONS}"
+export ENABLED_SYSTEMS="${ENABLED_SYSTEMS}"
+export GITHUB_ACTIONS="${GITHUB_ACTIONS}"
+export ISSUE_ID="${ISSUE_ID}"
+export NUM_RUNS="${NUM_RUNS}"
+export RUN_TESTS="${RUN_TESTS}"
+export RUN_INTEGRATION_TESTS="${RUN_INTEGRATION_TESTS}"
+export RUN_SYSTEM_TESTS="${RUN_SYSTEM_TESTS}"
+export VERSION_SUFFIX_FOR_SVN="${VERSION_SUFFIX_FOR_SVN}"
+export VERSION_SUFFIX_FOR_PYPI="${VERSION_SUFFIX_FOR_PYPI}"
 docker-compose ${command}
 EOF
     chmod u+x "${file}"

scripts/ci/docker-compose/_docker_compose.env

@@ -26,8 +26,56 @@ services:
       - CELERY_BROKER_URLS=amqp://guest:guest@rabbitmq:5672,redis://redis:6379/0
       - KUBECONFIG=/files/.kube/config
       - HOST_HOME=${HOME}
-    env_file:
-      - _docker_compose.env
+      # We need all those env variables here because docker-compose-v2 does not really work well
+      # With env files and there are many problems with it:
+      - AIRFLOW_CI_IMAGE=${AIRFLOW_CI_IMAGE}
+      - AIRFLOW_EXTRAS=${AIRFLOW_EXTRAS}
+      - BACKEND=${BACKEND}
+      - BREEZE=${BREEZE}
+      - CI=${CI}
+      - CI_BUILD_ID=${CI_BUILD_ID}
+      - CI_JOB_ID=${CI_JOB_ID}
+      - CI_EVENT_TYPE=${CI_EVENT_TYPE}
+      - CI_TARGET_REPO=${CI_TARGET_REPO}
+      - CI_TARGET_BRANCH=${CI_TARGET_BRANCH}
+      - COMMIT_SHA=${COMMIT_SHA}
+      - DB_RESET=${DB_RESET}
+      - DEFAULT_BRANCH=${DEFAULT_BRANCH}
+      - DEFAULT_CONSTRAINTS_BRANCH=${DEFAULT_CONSTRAINTS_BRANCH}
+      - ENABLED_INTEGRATIONS=${ENABLED_INTEGRATIONS}
+      - ENABLED_SYSTEMS=${ENABLED_SYSTEMS}
+      - GITHUB_ACTIONS=${GITHUB_ACTIONS}
+      - GITHUB_REGISTRY_PULL_IMAGE_TAG=${GITHUB_REGISTRY_PULL_IMAGE_TAG}
+      - HOST_USER_ID=${HOST_USER_ID}
+      - HOST_GROUP_ID=${HOST_GROUP_ID}
+      - HOST_OS=${HOST_OS}
+      - INIT_SCRIPT_FILE=${INIT_SCRIPT_FILE}
+      - INSTALL_AIRFLOW_VERSION=${INSTALL_AIRFLOW_VERSION}
+      - GENERATE_CONSTRAINTS_MODE=${GENERATE_CONSTRAINTS_MODE}
+      - INSTALL_PROVIDERS_FROM_SOURCES=${INSTALL_PROVIDERS_FROM_SOURCES}
+      - USE_AIRFLOW_VERSION=${USE_AIRFLOW_VERSION}
+      - USE_PACKAGES_FROM_DIST=${USE_PACKAGES_FROM_DIST}
+      - ISSUE_ID=${ISSUE_ID}
+      - LOAD_DEFAULT_CONNECTIONS=${LOAD_DEFAULT_CONNECTIONS}
+      - LOAD_EXAMPLES=${LOAD_EXAMPLES}
+      - MYSQL_VERSION=${MYSQL_VERSION}
+      - NUM_RUNS=${NUM_RUNS}
+      - PACKAGE_FORMAT=${PACKAGE_FORMAT}
+      - POSTGRES_VERSION=${POSTGRES_VERSION}
+      - PRINT_INFO_FROM_SCRIPTS=${PRINT_INFO_FROM_SCRIPTS}
+      - PYTHONDONTWRITEBYTECODE=${PYTHONDONTWRITEBYTECODE}
+      - PYTHON_MAJOR_MINOR_VERSION=${PYTHON_MAJOR_MINOR_VERSION}
+      - RUN_TESTS=${RUN_TESTS}
+      - RUN_INTEGRATION_TESTS=${RUN_INTEGRATION_TESTS}
+      - RUN_SYSTEM_TESTS=${RUN_SYSTEM_TESTS}
+      - START_AIRFLOW=${START_AIRFLOW}
+      - TEST_TYPE=${TEST_TYPE}
+      - UPGRADE_TO_NEWER_DEPENDENCIES=${UPGRADE_TO_NEWER_DEPENDENCIES}
+      - VERBOSE=${VERBOSE}
+      - VERBOSE_COMMANDS=${VERBOSE_COMMANDS}
+      - VERSION_SUFFIX_FOR_PYPI=${VERSION_SUFFIX_FOR_PYPI}
+      - VERSION_SUFFIX_FOR_SVN=${VERSION_SUFFIX_FOR_SVN}
+      - WHEEL_VERSION=${WHEEL_VERSION}
     volumes:
       # Pass docker to inside of the container so that Kind and Moto tests can use it.
       - /var/run/docker.sock:/var/run/docker.sock

scripts/ci/docker-compose/base.yml

@@ -311,6 +311,19 @@ function initialization::initialize_force_variables() {
 
     # Can be set to true to skip if the image is newer in registry
     export SKIP_CHECK_REMOTE_IMAGE=${SKIP_CHECK_REMOTE_IMAGE:="false"}
+
+    # integrations are disabled by default
+    export ENABLED_INTEGRATIONS=${ENABLED_INTEGRATIONS:=""}
+
+    # systems are disabled by default
+    export ENABLED_SYSTEMS=${ENABLED_SYSTEMS:=""}
+
+    # no issue id by default (quarantined builds only)
+    export ISSUE_ID=${ISSUE_ID:=""}
+
+    # no NUM_RUNS by default (quarantined builds only)
+    export NUM_RUNS=${NUM_RUNS:=""}
+
 }
 
 # Determine information about the host
@@ -576,10 +589,24 @@ function initialization::initialize_test_variables() {
     # In case we want to force certain test type to run, this variable should be set to this type
     # Otherwise TEST_TYPEs to run will be derived from TEST_TYPES space-separated string
     export FORCE_TEST_TYPE=${FORCE_TEST_TYPE:=""}
+
+    # Do not run tests by default
+    export RUN_TESTS=${RUN_TESTS:="false"}
+
+    # Do not run integration tests by default
+    export RUN_INTEGRATION_TESTS=${RUN_INTEGRATION_TESTS:="false"}
+
+    # Do not run system tests by default
+    export RUN_SYSTEM_TESTS=${RUN_SYSTEM_TESTS:="false"}
+
 }
 
 function initialization::initialize_package_variables() {
+    # default package format
     export PACKAGE_FORMAT=${PACKAGE_FORMAT:="wheel"}
+    # default version suffixes
+    export VERSION_SUFFIX_FOR_PYPI=${VERSION_SUFFIX_FOR_PYPI:=""}
+    export VERSION_SUFFIX_FOR_SVN=${VERSION_SUFFIX_FOR_SVN:=""}
 }
 
 

scripts/ci/libraries/_initialization.sh

@@ -52,7 +52,7 @@ else
     export RUN_AIRFLOW_1_10="false"
 fi
 
-if [[ -z ${USE_AIRFLOW_VERSION=} ]]; then
+if [[ ${USE_AIRFLOW_VERSION} == "" ]]; then
     export PYTHONPATH=${AIRFLOW_SOURCES}
     echo
     echo "Using already installed airflow version"
@@ -184,6 +184,7 @@ ssh-keyscan -H localhost >> ~/.ssh/known_hosts 2>/dev/null
 
 cd "${AIRFLOW_SOURCES}"
 
+echo "START_AIRFLOW:=${START_AIRFLOW}"
 if [[ ${START_AIRFLOW:="false"} == "true" ]]; then
     export AIRFLOW__CORE__LOAD_DEFAULT_CONNECTIONS=${LOAD_DEFAULT_CONNECTIONS}
     export AIRFLOW__CORE__LOAD_EXAMPLES=${LOAD_EXAMPLES}