From ab5b2bfca6b5be57a19bdbfe2206319bd2e71ac9 Mon Sep 17 00:00:00 2001 From: Jarek Potiuk Date: Tue, 12 Oct 2021 00:18:49 +0200 Subject: [PATCH] Workaround docker-compose-v2 env passing (#18887) * Workaround docker-compose-v2 env passing Docker Compose v2 has environment parsing broken in many ways. Until this is fixed, we cannot use env files, instead we must set all the variables directly, because parsing variables without values or parsing variables which have empty values is broken in several ways. Some of the issues are closed but not released, and until this is fixed, some extra code duplication and explicitly setting all default variables to "" when needed should solve the problem for both Docker-Compose v1 and Docker-Compose v2 --- breeze | 29 ++++------- scripts/ci/docker-compose/_docker_compose.env | 49 ----------------- scripts/ci/docker-compose/base.yml | 52 ++++++++++++++++++- scripts/ci/libraries/_initialization.sh | 27 ++++++++++ scripts/in_container/entrypoint_ci.sh | 3 +- 5 files changed, 90 insertions(+), 70 deletions(-) delete mode 100644 scripts/ci/docker-compose/_docker_compose.env diff --git a/breeze b/breeze index d6ef2cbb7d048..72bb8da190f77 100755 --- a/breeze +++ b/breeze @@ -567,24 +567,6 @@ EOF # Those are a convenience scripts that you might use to debug command execution although # In most cases they are used internally by Breeze. # -# Used Globals: -# BRANCH_NAME -# PYTHON_MAJOR_MINOR_VERSION -# BACKEND -# AIRFLOW_VERSION -# INSTALL_AIRFLOW_VERSION -# SSH_PORT -# WEBSERVER_HOST_PORT -# POSTGRES_HOST_PORT -# POSTGRES_VERSION -# MYSQL_HOST_PORT -# MYSQL_VERSION -# AIRFLOW_SOURCES -# AIRFLOW_CI_IMAGE -# AIRFLOW_PROD_IMAGE -# AIRFLOW_IMAGE_KUBERNETES -# SQLITE_URL -# # Arguments: # # file to prepare @@ -659,6 +641,17 @@ export SQLITE_URL="${SQLITE_URL}" export USE_AIRFLOW_VERSION="${USE_AIRFLOW_VERSION}" export USE_PACKAGES_FROM_DIST="${USE_PACKAGES_FROM_DIST}" export EXECUTOR="${EXECUTOR}" +export START_AIRFLOW="${START_AIRFLOW}" +export ENABLED_INTEGRATIONS="${ENABLED_INTEGRATIONS}" +export ENABLED_SYSTEMS="${ENABLED_SYSTEMS}" +export GITHUB_ACTIONS="${GITHUB_ACTIONS}" +export ISSUE_ID="${ISSUE_ID}" +export NUM_RUNS="${NUM_RUNS}" +export RUN_TESTS="${RUN_TESTS}" +export RUN_INTEGRATION_TESTS="${RUN_INTEGRATION_TESTS}" +export RUN_SYSTEM_TESTS="${RUN_SYSTEM_TESTS}" +export VERSION_SUFFIX_FOR_SVN="${VERSION_SUFFIX_FOR_SVN}" +export VERSION_SUFFIX_FOR_PYPI="${VERSION_SUFFIX_FOR_PYPI}" docker-compose ${command} EOF chmod u+x "${file}" diff --git a/scripts/ci/docker-compose/_docker_compose.env b/scripts/ci/docker-compose/_docker_compose.env deleted file mode 100644 index d206af47d0f1d..0000000000000 --- a/scripts/ci/docker-compose/_docker_compose.env +++ /dev/null @@ -1,49 +0,0 @@ -AIRFLOW_CI_IMAGE="${AIRFLOW_CI_IMAGE}" -AIRFLOW_EXTRAS="${AIRFLOW_EXTRAS}" -BACKEND="${BACKEND}" -BREEZE="${BREEZE}" -CI="${CI}" -CI_BUILD_ID="${CI_BUILD_ID}" -CI_JOB_ID="${CI_JOB_ID}" -CI_EVENT_TYPE="${CI_EVENT_TYPE}" -CI_TARGET_REPO="${CI_TARGET_REPO}" -CI_TARGET_BRANCH="${CI_TARGET_BRANCH}" -COMMIT_SHA="${COMMIT_SHA}" -DB_RESET="${DB_RESET}" -DEFAULT_BRANCH="${DEFAULT_BRANCH}" -DEFAULT_CONSTRAINTS_BRANCH="${DEFAULT_CONSTRAINTS_BRANCH}" -ENABLED_INTEGRATIONS="${ENABLED_INTEGRATIONS}" -ENABLED_SYSTEMS="${ENABLED_SYSTEMS}" -GITHUB_ACTIONS="${GITHUB_ACTIONS}" -GITHUB_REGISTRY_PULL_IMAGE_TAG="${GITHUB_REGISTRY_PULL_IMAGE_TAG}" -HOST_USER_ID="${HOST_USER_ID}" -HOST_GROUP_ID="${HOST_GROUP_ID}" -HOST_OS="${HOST_OS}" -HOST_HOME="${HOST_HOME}" -INIT_SCRIPT_FILE="${INIT_SCRIPT_FILE}" -INSTALL_AIRFLOW_VERSION="${INSTALL_AIRFLOW_VERSION}" -GENERATE_CONSTRAINTS_MODE="${GENERATE_CONSTRAINTS_MODE}" -INSTALL_PROVIDERS_FROM_SOURCES="${INSTALL_PROVIDERS_FROM_SOURCES}" -USE_AIRFLOW_VERSION="${USE_AIRFLOW_VERSION}" -USE_PACKAGES_FROM_DIST="${USE_PACKAGES_FROM_DIST}" -ISSUE_ID="${ISSUE_ID}" -LOAD_DEFAULT_CONNECTIONS="${LOAD_DEFAULT_CONNECTIONS}" -LOAD_EXAMPLES="${LOAD_EXAMPLES}" -MYSQL_VERSION="${MYSQL_VERSION}" -NUM_RUNS="${NUM_RUNS}" -PACKAGE_FORMAT="${PACKAGE_FORMAT}" -POSTGRES_VERSION="${POSTGRES_VERSION}" -PRINT_INFO_FROM_SCRIPTS="${PRINT_INFO_FROM_SCRIPTS}" -PYTHONDONTWRITEBYTECODE="${PYTHONDONTWRITEBYTECODE}" -PYTHON_MAJOR_MINOR_VERSION="${PYTHON_MAJOR_MINOR_VERSION}" -RUN_TESTS="${RUN_TESTS}" -RUN_INTEGRATION_TESTS="${RUN_INTEGRATION_TESTS}" -RUN_SYSTEM_TESTS="${RUN_SYSTEM_TESTS}" -START_AIRFLOW="${START_AIRFLOW}" -TEST_TYPE="${TEST_TYPE}" -UPGRADE_TO_NEWER_DEPENDENCIES="${UPGRADE_TO_NEWER_DEPENDENCIES}" -VERBOSE="${VERBOSE}" -VERBOSE_COMMANDS="${VERBOSE_COMMANDS}" -VERSION_SUFFIX_FOR_PYPI="${VERSION_SUFFIX_FOR_PYPI}" -VERSION_SUFFIX_FOR_SVN="${VERSION_SUFFIX_FOR_SVN}" -WHEEL_VERSION="${WHEEL_VERSION}" diff --git a/scripts/ci/docker-compose/base.yml b/scripts/ci/docker-compose/base.yml index c9fed6afca1e7..4ecf6ee3bcb64 100644 --- a/scripts/ci/docker-compose/base.yml +++ b/scripts/ci/docker-compose/base.yml @@ -26,8 +26,56 @@ services: - CELERY_BROKER_URLS=amqp://guest:guest@rabbitmq:5672,redis://redis:6379/0 - KUBECONFIG=/files/.kube/config - HOST_HOME=${HOME} - env_file: - - _docker_compose.env + # We need all those env variables here because docker-compose-v2 does not really work well + # With env files and there are many problems with it: + - AIRFLOW_CI_IMAGE=${AIRFLOW_CI_IMAGE} + - AIRFLOW_EXTRAS=${AIRFLOW_EXTRAS} + - BACKEND=${BACKEND} + - BREEZE=${BREEZE} + - CI=${CI} + - CI_BUILD_ID=${CI_BUILD_ID} + - CI_JOB_ID=${CI_JOB_ID} + - CI_EVENT_TYPE=${CI_EVENT_TYPE} + - CI_TARGET_REPO=${CI_TARGET_REPO} + - CI_TARGET_BRANCH=${CI_TARGET_BRANCH} + - COMMIT_SHA=${COMMIT_SHA} + - DB_RESET=${DB_RESET} + - DEFAULT_BRANCH=${DEFAULT_BRANCH} + - DEFAULT_CONSTRAINTS_BRANCH=${DEFAULT_CONSTRAINTS_BRANCH} + - ENABLED_INTEGRATIONS=${ENABLED_INTEGRATIONS} + - ENABLED_SYSTEMS=${ENABLED_SYSTEMS} + - GITHUB_ACTIONS=${GITHUB_ACTIONS} + - GITHUB_REGISTRY_PULL_IMAGE_TAG=${GITHUB_REGISTRY_PULL_IMAGE_TAG} + - HOST_USER_ID=${HOST_USER_ID} + - HOST_GROUP_ID=${HOST_GROUP_ID} + - HOST_OS=${HOST_OS} + - INIT_SCRIPT_FILE=${INIT_SCRIPT_FILE} + - INSTALL_AIRFLOW_VERSION=${INSTALL_AIRFLOW_VERSION} + - GENERATE_CONSTRAINTS_MODE=${GENERATE_CONSTRAINTS_MODE} + - INSTALL_PROVIDERS_FROM_SOURCES=${INSTALL_PROVIDERS_FROM_SOURCES} + - USE_AIRFLOW_VERSION=${USE_AIRFLOW_VERSION} + - USE_PACKAGES_FROM_DIST=${USE_PACKAGES_FROM_DIST} + - ISSUE_ID=${ISSUE_ID} + - LOAD_DEFAULT_CONNECTIONS=${LOAD_DEFAULT_CONNECTIONS} + - LOAD_EXAMPLES=${LOAD_EXAMPLES} + - MYSQL_VERSION=${MYSQL_VERSION} + - NUM_RUNS=${NUM_RUNS} + - PACKAGE_FORMAT=${PACKAGE_FORMAT} + - POSTGRES_VERSION=${POSTGRES_VERSION} + - PRINT_INFO_FROM_SCRIPTS=${PRINT_INFO_FROM_SCRIPTS} + - PYTHONDONTWRITEBYTECODE=${PYTHONDONTWRITEBYTECODE} + - PYTHON_MAJOR_MINOR_VERSION=${PYTHON_MAJOR_MINOR_VERSION} + - RUN_TESTS=${RUN_TESTS} + - RUN_INTEGRATION_TESTS=${RUN_INTEGRATION_TESTS} + - RUN_SYSTEM_TESTS=${RUN_SYSTEM_TESTS} + - START_AIRFLOW=${START_AIRFLOW} + - TEST_TYPE=${TEST_TYPE} + - UPGRADE_TO_NEWER_DEPENDENCIES=${UPGRADE_TO_NEWER_DEPENDENCIES} + - VERBOSE=${VERBOSE} + - VERBOSE_COMMANDS=${VERBOSE_COMMANDS} + - VERSION_SUFFIX_FOR_PYPI=${VERSION_SUFFIX_FOR_PYPI} + - VERSION_SUFFIX_FOR_SVN=${VERSION_SUFFIX_FOR_SVN} + - WHEEL_VERSION=${WHEEL_VERSION} volumes: # Pass docker to inside of the container so that Kind and Moto tests can use it. - /var/run/docker.sock:/var/run/docker.sock diff --git a/scripts/ci/libraries/_initialization.sh b/scripts/ci/libraries/_initialization.sh index ff9fb293a2864..31e17e45131fd 100644 --- a/scripts/ci/libraries/_initialization.sh +++ b/scripts/ci/libraries/_initialization.sh @@ -311,6 +311,19 @@ function initialization::initialize_force_variables() { # Can be set to true to skip if the image is newer in registry export SKIP_CHECK_REMOTE_IMAGE=${SKIP_CHECK_REMOTE_IMAGE:="false"} + + # integrations are disabled by default + export ENABLED_INTEGRATIONS=${ENABLED_INTEGRATIONS:=""} + + # systems are disabled by default + export ENABLED_SYSTEMS=${ENABLED_SYSTEMS:=""} + + # no issue id by default (quarantined builds only) + export ISSUE_ID=${ISSUE_ID:=""} + + # no NUM_RUNS by default (quarantined builds only) + export NUM_RUNS=${NUM_RUNS:=""} + } # Determine information about the host @@ -576,10 +589,24 @@ function initialization::initialize_test_variables() { # In case we want to force certain test type to run, this variable should be set to this type # Otherwise TEST_TYPEs to run will be derived from TEST_TYPES space-separated string export FORCE_TEST_TYPE=${FORCE_TEST_TYPE:=""} + + # Do not run tests by default + export RUN_TESTS=${RUN_TESTS:="false"} + + # Do not run integration tests by default + export RUN_INTEGRATION_TESTS=${RUN_INTEGRATION_TESTS:="false"} + + # Do not run system tests by default + export RUN_SYSTEM_TESTS=${RUN_SYSTEM_TESTS:="false"} + } function initialization::initialize_package_variables() { + # default package format export PACKAGE_FORMAT=${PACKAGE_FORMAT:="wheel"} + # default version suffixes + export VERSION_SUFFIX_FOR_PYPI=${VERSION_SUFFIX_FOR_PYPI:=""} + export VERSION_SUFFIX_FOR_SVN=${VERSION_SUFFIX_FOR_SVN:=""} } diff --git a/scripts/in_container/entrypoint_ci.sh b/scripts/in_container/entrypoint_ci.sh index 29f5210814248..0e09c8aa4e61d 100755 --- a/scripts/in_container/entrypoint_ci.sh +++ b/scripts/in_container/entrypoint_ci.sh @@ -52,7 +52,7 @@ else export RUN_AIRFLOW_1_10="false" fi -if [[ -z ${USE_AIRFLOW_VERSION=} ]]; then +if [[ ${USE_AIRFLOW_VERSION} == "" ]]; then export PYTHONPATH=${AIRFLOW_SOURCES} echo echo "Using already installed airflow version" @@ -184,6 +184,7 @@ ssh-keyscan -H localhost >> ~/.ssh/known_hosts 2>/dev/null cd "${AIRFLOW_SOURCES}" +echo "START_AIRFLOW:=${START_AIRFLOW}" if [[ ${START_AIRFLOW:="false"} == "true" ]]; then export AIRFLOW__CORE__LOAD_DEFAULT_CONNECTIONS=${LOAD_DEFAULT_CONNECTIONS} export AIRFLOW__CORE__LOAD_EXAMPLES=${LOAD_EXAMPLES}