modify optimizer test

Author: billie60

.github/workflows/optimizer.yml

@@ -16,6 +16,7 @@ on:
 
 env:
   CARGO_TERM_COLOR: always
+  REGISTRY: ghcr.io
 
 jobs:
   run_optimizer:
@@ -54,7 +55,8 @@ jobs:
           sudo wget https://github.com/containerd/containerd/releases/download/v1.7.0/containerd-static-1.7.0-linux-amd64.tar.gz
           sudo systemctl stop containerd
           sudo tar -zxf ./containerd-static-1.7.0-linux-amd64.tar.gz -C /usr/
-          sudo install -D -m 755 misc/optimizer/containerd-config.toml /etc/containerd/config.toml
+          # sudo install -D -m 755 misc/optimizer/containerd-config.toml /etc/containerd/config.toml
+          sudo install -D -m 755 misc/example/containerd-config.toml /etc/containerd/config.toml
           sudo systemctl restart containerd
           sudo wget https://github.com/opencontainers/runc/releases/download/v1.1.5/runc.amd64 -O /usr/bin/runc
           sudo chmod +x /usr/bin/runc
@@ -64,6 +66,18 @@ jobs:
           sudo mkdir -p /opt/cni/bin
           sudo tar xzf cni-plugins-linux-amd64-v1.2.0.tgz -C /opt/cni/bin/
           sudo install -D -m 755 misc/example/10-containerd-net.conflist /etc/cni/net.d/10-containerd-net.conflist
+      - name: Setup nydus-snapshotter and nydus service
+        run: |
+          make build && make install
+          sleep 5
+          NYDUS_VER=v$(curl --header 'authorization: Bearer ${{ secrets.GITHUB_TOKEN }}' -s "https://api.github.com/repos/dragonflyoss/nydus/releases/latest" | jq -r .tag_name | sed 's/^v//')
+          wget https://github.com/dragonflyoss/nydus/releases/download/$NYDUS_VER/nydus-static-$NYDUS_VER-linux-amd64.tgz
+          tar xzvf nydus-static-$NYDUS_VER-linux-amd64.tgz
+          sudo systemctl restart nydus-snapshotter.service
+          sudo install -D -m 755 nydus-static/nydusd /usr/local/bin/nydusd 
+          sudo install -D -m 755 nydus-static/nydus-image /usr/local/bin/nydus-image 
+          sudo install -D -m 755 nydus-static/nydusctl /usr/local/bin/nydusctl
+
       - name: Build and install optimizer
         run: |
           rustup component add rustfmt clippy
@@ -90,22 +104,41 @@ jobs:
             echo "containerd is not ready"
             exit 1
           fi
+      - name: Setup prefetch-distribution http server
+        run: |
+          go build -o prefetch-distribution tools/prefetch-distribution/main.go
+          nohup ./prefetch-distribution &  
+          sleep 5
+      - name: Log in to container registry
+        uses: docker/login-action@v2
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
       - name: Generate accessed files list
         run: |
           sed -i "s|host_path: script|host_path: $(pwd)/misc/optimizer/script|g" misc/optimizer/nginx.yaml
           sudo crictl run misc/optimizer/nginx.yaml misc/optimizer/sandbox.yaml
           sleep 20
           sudo crictl rmp  -f --all
+          sudo crictl rmi --all
           tree /opt/nri/optimizer/results/
-          count=$(cat /opt/nri/optimizer/results/library/nginx:1.23.3 | wc -l)
+          # count=$(cat /opt/nri/optimizer/results/library/nginx:1.23.3 | wc -l)
+          count=$(cat /opt/nri/optimizer/results/dragonflyoss/image-service/nginx:nydus-latest | wc -l)
           expected=$(cat misc/optimizer/script/file_list.txt | wc -l)
           echo "count: $count expected minimum value: $expected"
           if [ $count -lt $expected ]; then
-            echo "failed to generate accessed files list for nginx:1.23.3"
+            echo "failed to generate accessed files list for nginx:nydus-latest"
             cat misc/optimizer/script/file_list.txt
             exit 1
           fi
-          cat /opt/nri/optimizer/results/library/nginx:1.23.3.csv
+          cat /opt/nri/optimizer/results/dragonflyoss/image-service/nginx:nydus-latest.csv
+      - name: Transmit the prefetch list to nydusd
+        run: |
+          sed -i "s|host_path: script|host_path: $(pwd)/misc/optimizer/script|g" misc/optimizer/nginx.yaml
+          sudo crictl run misc/optimizer/nginx.yaml misc/optimizer/sandbox.yaml
+          cat ~/logs/nydus-snapshotter.log | grep "nydusd command"
       - name: Dump logs
         if: failure()
         continue-on-error: true

misc/example/containerd-config.toml

@@ -18,6 +18,12 @@ oom_score = 0
   [plugins."io.containerd.grpc.v1.cri".cni]
       bin_dir = "/opt/cni/bin"
       conf_dir = "/etc/cni/net.d"
+  
+[plugins."io.containerd.nri.v1.nri"]
+  config_file = "/etc/nri/nri.conf"
+  disable = false
+  plugin_path = "/opt/nri/plugins"
+  socket_path = "/var/run/nri.sock"
 
 [proxy_plugins]
   [proxy_plugins.nydus]

misc/example/optimizer-nri-plugin.conf

@@ -12,4 +12,4 @@ overwrite = false
 # Do not change this element.
 events = [ "StartContainer", "StopContainer" ]
 # The service url of prefetch distribution.
-prefetch_distribution_url = "http://localhost:1323"
+prefetch_distribution_url = "http://localhost:1323"

misc/optimizer/nginx.yaml

@@ -2,7 +2,7 @@ metadata:
   name: nginx
 
 image:
-  image: nginx:1.23.3
+  image: ghcr.io/dragonflyoss/image-service/nginx:nydus-latest
 
 mounts:
   - host_path: script

misc/snapshotter/config.toml

@@ -48,6 +48,7 @@ memory_limit = ""
 [log]
 # Print logs to stdout rather than logging files
 log_to_stdout = false
+dir = "~/logs"
 # Snapshotter's log level
 level = "info"
 log_rotation_compress = true