doc: Rootless lazy pulling with Podman, nerdctl and BuildKit

.github/workflows/tests.yml

@@ -166,6 +166,14 @@ jobs:
         METADATA_STORE: ${{ matrix.metadata-store }}
       run: make test-cri-o
 
+  test-podman:
+    runs-on: ubuntu-22.04
+    name: PodmanRootless
+    steps:
+    - uses: actions/checkout@v3
+    - name: Test Podman (rootless)
+      run: make test-podman
+
   test-k3s:
     runs-on: ubuntu-22.04
     name: K3S

Dockerfile

@@ -23,6 +23,9 @@ ARG CONMON_VERSION=v2.1.5
 ARG COMMON_VERSION=v0.49.2
 ARG CRIO_TEST_PAUSE_IMAGE_NAME=registry.k8s.io/pause:3.6
 
+ARG CONTAINERIZED_SYSTEMD_VERSION=v0.1.1
+ARG SLIRP4NETNS_VERSION=v1.1.12
+
 # Used in CI
 ARG CRI_TOOLS_VERSION=v1.25.0
 
@@ -183,6 +186,36 @@ COPY --from=conmon-dev /out/bin/* /usr/local/bin/
 COPY --from=containers-common-dev /out/seccomp.json /usr/share/containers/
 COPY --from=stargz-store-dev /out/* /usr/local/bin/
 
+# Image for testing rootless Podman with Stargz Store.
+# This takes the same approach as nerdctl CI: https://github.com/containerd/nerdctl/blob/6341c8320984f7148b92dd33472d8eaca6dba756/Dockerfile#L302-L326
+FROM podman-base AS podman-rootless
+ARG CONTAINERIZED_SYSTEMD_VERSION
+ARG SLIRP4NETNS_VERSION
+RUN apt-get update -y && apt-get install -y \
+                         systemd systemd-sysv dbus dbus-user-session \
+                         openssh-server openssh-client uidmap
+RUN curl -o /usr/local/bin/slirp4netns --fail -L https://github.com/rootless-containers/slirp4netns/releases/download/${SLIRP4NETNS_VERSION}/slirp4netns-$(uname -m) && \
+    chmod +x /usr/local/bin/slirp4netns && \
+    curl -L -o /docker-entrypoint.sh https://raw.githubusercontent.com/AkihiroSuda/containerized-systemd/${CONTAINERIZED_SYSTEMD_VERSION}/docker-entrypoint.sh && \
+    chmod +x /docker-entrypoint.sh && \
+    curl -L -o /etc/containers/policy.json https://raw.githubusercontent.com/containers/skopeo/master/default-policy.json
+# storage.conf plugs Stargz Store into Podman as an Additional Layer Store
+COPY ./script/podman/config/storage.conf /home/rootless/.config/containers/storage.conf
+# Stargz Store systemd service for rootless Podman
+COPY ./script/podman/config/podman-rootless-stargz-store.service /home/rootless/.config/systemd/user/
+# test-podman-rootless.sh logins to the user via SSH
+COPY ./script/podman/config/test-podman-rootless.sh /test-podman-rootless.sh
+RUN ssh-keygen -q -t rsa -f /root/.ssh/id_rsa -N '' && \
+    useradd -m -s /bin/bash rootless && \
+    mkdir -p -m 0700 /home/rootless/.ssh && \
+    cp -a /root/.ssh/id_rsa.pub /home/rootless/.ssh/authorized_keys && \
+    mkdir -p /home/rootless/.local/share /home/rootless/.local/share/stargz-store/store && \
+    chown -R rootless:rootless /home/rootless
+VOLUME /home/rootless/.local/share
+
+ENTRYPOINT ["/docker-entrypoint.sh", "/test-podman-rootless.sh"]
+CMD ["/bin/bash", "--login", "-i"]
+
 # Image which can be used for interactive demo environment
 FROM containerd-base AS demo
 ARG CNI_PLUGINS_VERSION

Makefile

@@ -108,6 +108,9 @@ test-cri-containerd:
 test-cri-o:
 	@./script/cri-o/test.sh
 
+test-podman:
+	@./script/podman/test.sh
+
 test-criauth:
 	@./script/criauth/test.sh
 

docs/rootless.md

@@ -0,0 +1,56 @@
+# Rootless execution of stargz snapshotter
+
+This document lists links and information about how to run Stargz Snapshotter and Stargz Store from the non-root user.
+
+## nerdctl (Stargz Snapshotter)
+
+Rootless Stargz Snapshotter for nerdctl can be installed via `containerd-rootless-setuptool.sh install-stargz` command.
+Please see [the doc in nerdctl repo](https://github.com/containerd/nerdctl/blob/v1.1.0/docs/rootless.md#stargz-snapshotter) for details.
+
+## Podman (Stargz Store)
+
+> NOTE: This is an experimental configuration leveraging [`podman unshare`](https://docs.podman.io/en/latest/markdown/podman-unshare.1.html). Limitation: `--uidmap` of `podman run` doesn't work.
+
+First, allow podman using Stargz Store by adding the following store configuration.
+Put the configuration file to [`/etc/containers/storage.conf` or `$HOME/.config/containers/storage.conf`](https://github.com/containers/podman/blob/v4.3.1/docs/tutorials/rootless_tutorial.md#storageconf).
+
+> NOTE: Replace `/path/to/home` to the actual home directory.
+
+```
+[storage]
+driver = "overlay"
+
+[storage.options]
+additionallayerstores = ["/path/to/homedir/.local/share/stargz-store/store:ref"]
+```
+
+Start Stargz Store in the namespace managed by podman via [`podman unshare`](https://docs.podman.io/en/latest/markdown/podman-unshare.1.html) command.
+
+```
+$ podman unshare stargz-store --root $HOME/.local/share/stargz-store/data $HOME/.local/share/stargz-store/store &
+```
+
+Podman performs lazy pulling when it pulls eStargz images.
+
+```
+$ podman pull ghcr.io/stargz-containers/python:3.9-esgz
+```
+
+<details>
+<summary>Creating systemd unit file for Stargz Store</summary>
+
+It's possible to create systemd unit file of Stargz Store for easily managing it.
+An example systemd unit file can be found [here](../script/podman/config/podman-rootless-stargz-store.service)
+
+After installing that file (e.g. to `$HOME/.config/systemd/user/`), start the service using `systemctl`.
+
+```
+$ systemctl --user start podman-rootless-stargz-store
+```
+
+</details>
+
+## BuildKit (Stargz Snapshotter)
+
+BuildKit supports running Stargz Snapshotter from the non-root user.
+Please see [the doc in BuildKit repo](https://github.com/moby/buildkit/blob/8b132188aa7af944c813d02da63c93308d83cf75/docs/stargz-estargz.md) (unmerged 2023/1/18) for details.

script/podman/config/podman-rootless-stargz-store.service

@@ -0,0 +1,11 @@
+[Unit]
+Description=Stargz Store service for Podman
+
+[Service]
+# This pipes stargz-store's IO to the shell otherwise they are lost and can't be managed by systemd.
+# TODO: fix this and do not use pipe
+ExecStart=/bin/bash -c "podman unshare stargz-store --log-level=debug --root %h/.local/share/stargz-store/data %h/.local/share/stargz-store/store 2>&1 | cat"
+ExecStopPost=podman unshare umount %h/.local/share/stargz-store/store
+
+[Install]
+WantedBy=default.target

script/podman/config/storage.conf

@@ -0,0 +1,4 @@
+[storage]
+  driver = "overlay"
+[storage.options]
+  additionallayerstores = ["/home/rootless/.local/share/stargz-store/store:ref"]

script/podman/config/test-podman-rootless.sh

@@ -0,0 +1,25 @@
+#!/bin/bash
+
+#   Copyright The containerd Authors.
+
+#   Licensed under the Apache License, Version 2.0 (the "License");
+#   you may not use this file except in compliance with the License.
+#   You may obtain a copy of the License at
+
+#       http://www.apache.org/licenses/LICENSE-2.0
+
+#   Unless required by applicable law or agreed to in writing, software
+#   distributed under the License is distributed on an "AS IS" BASIS,
+#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#   See the License for the specific language governing permissions and
+#   limitations under the License.
+
+set -eux -o pipefail
+if [[ "$(id -u)" = "0" ]]; then
+	# Switch to the rootless user via SSH; This is the same approach as done in nerdctl CI
+	systemctl start sshd
+	exec ssh -o StrictHostKeyChecking=no rootless@localhost "$0" "$@"
+else
+	systemctl --user start podman-rootless-stargz-store
+	exec "$@"
+fi

script/podman/run_test.sh

@@ -0,0 +1,35 @@
+#!/bin/bash
+
+#   Copyright The containerd Authors.
+
+#   Licensed under the Apache License, Version 2.0 (the "License");
+#   you may not use this file except in compliance with the License.
+#   You may obtain a copy of the License at
+
+#       http://www.apache.org/licenses/LICENSE-2.0
+
+#   Unless required by applicable law or agreed to in writing, software
+#   distributed under the License is distributed on an "AS IS" BASIS,
+#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#   See the License for the specific language governing permissions and
+#   limitations under the License.
+
+set -euo pipefail
+
+STARGZ_STORE_SERVICE=podman-rootless-stargz-store
+
+# Lazy pulling and run
+podman pull ghcr.io/stargz-containers/alpine:3.10.2-esgz
+podman run --rm ghcr.io/stargz-containers/alpine:3.10.2-esgz echo hello
+
+# Run (includes lazy pulling)
+podman run --rm ghcr.io/stargz-containers/python:3.9-esgz echo hello
+
+# Print store log to be checked by the host
+LOG_REMOTE_SNAPSHOT="remote-snapshot-prepared"
+journalctl --user -u "${STARGZ_STORE_SERVICE}" | grep "${LOG_REMOTE_SNAPSHOT}"
+
+# Non-lazy pulling
+podman run --rm ghcr.io/stargz-containers/ubuntu:22.04-org echo hello
+
+systemctl --user stop "${STARGZ_STORE_SERVICE}"

script/podman/test.sh

@@ -0,0 +1,55 @@
+#!/bin/bash
+
+#   Copyright The containerd Authors.
+
+#   Licensed under the Apache License, Version 2.0 (the "License");
+#   you may not use this file except in compliance with the License.
+#   You may obtain a copy of the License at
+
+#       http://www.apache.org/licenses/LICENSE-2.0
+
+#   Unless required by applicable law or agreed to in writing, software
+#   distributed under the License is distributed on an "AS IS" BASIS,
+#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#   See the License for the specific language governing permissions and
+#   limitations under the License.
+
+set -euo pipefail
+
+CONTEXT="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )/"
+REPO="${CONTEXT}../../"
+
+source "${REPO}/script/util/utils.sh"
+
+NODE_IMAGE_NAME_BASE=test-podman-rootless-node-base
+NODE_IMAGE_NAME=test-podman-rootless-node
+
+if [ "${PODMAN_NO_RECREATE:-}" != "true" ] ; then
+    echo "Preparing node image..."
+    docker build ${DOCKER_BUILD_ARGS:-} --progress=plain -t "${NODE_IMAGE_NAME_BASE}" --target podman-rootless "${REPO}"
+fi
+
+TEST_LOG=$(mktemp)
+STORE_LOG=$(mktemp)
+TMP_CONTEXT=$(mktemp -d)
+function cleanup {
+    local ORG_EXIT_CODE="${1}"
+    rm -rf "${TMP_CONTEXT}" || true
+    rm "${TEST_LOG}" || true
+    rm "${STORE_LOG}" || true
+    exit "${ORG_EXIT_CODE}"
+}
+trap 'cleanup "$?"' EXIT SIGHUP SIGINT SIGQUIT SIGTERM
+
+cp "${CONTEXT}/run_test.sh" "${TMP_CONTEXT}/run_test.sh"
+cat <<EOF > "${TMP_CONTEXT}/Dockerfile"
+FROM ${NODE_IMAGE_NAME_BASE}
+
+COPY run_test.sh /run_test.sh
+CMD ["/bin/bash", "--login", "/run_test.sh"]
+EOF
+
+docker build ${DOCKER_BUILD_ARGS:-} --progress=plain -t "${NODE_IMAGE_NAME}" "${TMP_CONTEXT}"
+docker run -t --rm --privileged "${NODE_IMAGE_NAME}" | tee "${TEST_LOG}"
+cat "${TEST_LOG}" | grep "${LOG_REMOTE_SNAPSHOT}" | sed -E 's/^[^\{]*(\{.*)$/\1/g' > "${STORE_LOG}"
+check_remote_snapshots "${STORE_LOG}"