feat(ingest): use uv for python package installs

.github/scripts/docker_helpers.sh

@@ -24,7 +24,7 @@ function get_tag_full {
 }
 
 function get_python_docker_release_v {
-    echo $(echo ${GITHUB_REF} | sed -e "s,refs/heads/${MAIN_BRANCH},0.0.0+docker.${SHORT_SHA},g" -e 's,refs/tags/v\(.*\),\1+docker,g' -e 's,refs/pull/\([0-9]*\).*,0.0.0+docker.pr\1,g')
+    echo $(echo ${GITHUB_REF} | sed -e "s,refs/heads/${MAIN_BRANCH},1!0.0.0+docker.${SHORT_SHA},g" -e 's,refs/tags/v\(.*\),\1+docker,g' -e 's,refs/pull/\([0-9]*\).*,1!0.0.0+docker.pr\1,g')
 }
 
 function get_unique_tag {

.github/workflows/metadata-ingestion.yml

@@ -51,6 +51,15 @@ jobs:
           java-version: 17
       - uses: gradle/gradle-build-action@v2
       - uses: actions/checkout@v3
+      - uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.python-version }}
+          cache: "pip"
+      - uses: actions/cache@v4
+        with:
+          path: |
+            ~/.cache/uv
+          key: ${{ runner.os }}-uv-${{ hashFiles('**/requirements.txt') }}
       - uses: actions/setup-python@v4
         with:
           python-version: ${{ matrix.python-version }}

docker/datahub-ingestion-base/Dockerfile

@@ -50,7 +50,7 @@ RUN apt-get update && apt-get install -y -qq \
     ldap-utils \
     unixodbc \
     libodbc2 \
-    && python -m pip install --no-cache --upgrade pip wheel setuptools \
+    && python -m pip install --no-cache --upgrade pip uv wheel setuptools \
     && rm -rf /var/lib/apt/lists/* /var/cache/apk/*
 
 # compiled against newer golang for security fixes
@@ -59,16 +59,22 @@ COPY --from=dockerize-binary /go/bin/dockerize /usr/local/bin
 COPY ./docker/datahub-ingestion-base/base-requirements.txt requirements.txt
 COPY ./docker/datahub-ingestion-base/entrypoint.sh /entrypoint.sh
 
-RUN pip install --no-cache -r requirements.txt && \
-    pip uninstall -y acryl-datahub && \
-    chmod +x /entrypoint.sh && \
-    addgroup --gid 1000 datahub && \
-    adduser --disabled-password --uid 1000 --gid 1000 --home /datahub-ingestion datahub
+RUN addgroup --gid 1000 datahub && \
+    adduser --disabled-password --uid 1000 --gid 1000 --home /datahub-ingestion datahub && \
+    chmod +x /entrypoint.sh
+
+USER datahub
+ENV VIRTUAL_ENV=/datahub-ingestion/.venv
+ENV PATH="${VIRTUAL_ENV}/bin:$PATH"
+RUN python3 -m venv $VIRTUAL_ENV && \
+    uv pip install --no-cache -r requirements.txt && \
+    pip uninstall -y acryl-datahub
 
 ENTRYPOINT [ "/entrypoint.sh" ]
 
 FROM ${BASE_IMAGE} as full-install
 
+USER 0
 RUN apt-get update && apt-get install -y -qq \
     default-jre-headless \
     && rm -rf /var/lib/apt/lists/* /var/cache/apk/*
@@ -91,10 +97,11 @@ RUN if [ $(arch) = "x86_64" ]; then \
     ldconfig; \
     fi;
 
+USER datahub
+
 FROM ${BASE_IMAGE} as slim-install
 # Do nothing else on top of base
 
 FROM ${APP_ENV}-install
 
-USER datahub
 ENV PATH="/datahub-ingestion/.local/bin:$PATH"