[wip]Add eBPF support for optimizer

.github/workflows/ci.yml

@@ -12,7 +12,7 @@ env:
 jobs:
   build:
     name: Build and Lint
-    timeout-minutes: 10
+    timeout-minutes: 20
     runs-on: ubuntu-latest
     steps:
       - name: Set up Go
@@ -33,17 +33,13 @@ jobs:
           go install github.com/golangci/golangci-lint/cmd/golangci-lint@v1.51.2
           export PATH=$PATH:$(go env GOPATH)/bin
           make
-          make test
-          make check
+          make container-test
+          make container-check
   build-optimizer:
     name: Build optimizer
-    timeout-minutes: 10
+    timeout-minutes: 20
     runs-on: ubuntu-latest
     steps:
-      - name: Set up Go
-        uses: actions/setup-go@v3
-        with:
-          go-version: "1.20.1"
       - name: Check out code
         uses: actions/checkout@v3
       - name: cache go mod
@@ -55,19 +51,12 @@ jobs:
             ${{ runner.os }}-go
       - name: Build
         run: |
-          go install github.com/golangci/golangci-lint/cmd/golangci-lint@v1.51.2
-          export PATH=$PATH:$(go env GOPATH)/bin
-          rustup component add rustfmt clippy
-          make build-optimizer
+          make container-build-optimizer
   smoke:
     name: Smoke
-    timeout-minutes: 10
+    timeout-minutes: 20
     runs-on: ubuntu-latest
     steps:
-      - name: Set up Go
-        uses: actions/setup-go@v3
-        with:
-          go-version: "1.19.6"
       - name: Check out code
         uses: actions/checkout@v3
       - name: cache go mod
@@ -118,14 +107,10 @@ jobs:
 
   coverage:
     name: Code coverage
-    timeout-minutes: 10
+    timeout-minutes: 20
     runs-on: ubuntu-latest
     needs: [build]
     steps:
-      - name: Install Go
-        uses: actions/setup-go@v3
-        with:
-          go-version: "1.19.6"
       - name: Checkout code
         uses: actions/checkout@v3
       - name: cache go mod
@@ -136,7 +121,7 @@ jobs:
           restore-keys: |
             ${{ runner.os }}-go
       - name: Run unit tests.
-        run: make cover
+        run: make container-cover
       - name: Upload coverage to Codecov
         uses: codecov/codecov-action@v3
         with:

.github/workflows/optimizer.yml

@@ -19,12 +19,11 @@ env:
 
 jobs:
   run_optimizer:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-20.04
+    env:
+      GOBCC_BUILDER_IMAGE: 'ghcr.io/sctb512/gobcc-builder:20.04'
+      GOBCC_BUILDER_IMAGE_FILE: '~/.image/gobcc-builder:20.04.tar'
     steps:
-      - name: Set up Go
-        uses: actions/setup-go@v3
-        with:
-          go-version: "1.19.6"
       - name: Checkout repository
         uses: actions/checkout@v3
       - name: cache go mod
@@ -46,6 +45,44 @@ jobs:
           key: ${{ runner.os }}-cargo-${{ hashFiles('tools/optimizer-server/Cargo.lock') }}
           restore-keys: |
             ${{ runner.os }}-cargo
+      - name: cache go bcc image
+        uses: actions/cache@v3
+        with:
+          path: |
+            ${GOBCC_BUILDER_IMAGE_FILE}
+          key: ${{ runner.os }}-bogcc-builder-${{ hashFiles('${GOBCC_BUILDER_IMAGE_FILE}') }}
+          restore-keys: |
+            ${{ runner.os }}-bogcc-builder
+      - name: load go bcc image
+        run: |
+          if [ -f ${GOBCC_BUILDER_IMAGE_FILE} ];then
+            sudo docker load < ${GOBCC_BUILDER_IMAGE_FILE}
+          else
+            sudo docker pull ${GOBCC_BUILDER_IMAGE}
+          fi
+      - name: Build optimizer
+        run: |
+          make container-optimizer
+          pwd
+          ldd bin/*optimizer*
+          ls -lh bin/*optimizer*
+      - name: Setup bcc for eBPF
+        run: |
+          sudo apt-get update
+          sudo apt install -y zip bison build-essential cmake flex git libedit-dev \
+          libllvm12 llvm-12-dev libclang-12-dev python zlib1g-dev libelf-dev libfl-dev python3-setuptools \
+          liblzma-dev arping netperf iperf
+          git clone https://github.com/iovisor/bcc.git; cd bcc
+          git checkout v0.24.0
+          mkdir build; cd build
+          cmake ..
+          make
+          sudo make install
+          cmake -DPYTHON_CMD=python3 ..
+          pushd src/python/
+          make
+          sudo make install
+          popd
       - name: containerd runc and crictl
         run: |
           sudo wget https://github.com/kubernetes-sigs/cri-tools/releases/download/v1.26.0/crictl-v1.26.0-linux-amd64.tar.gz
@@ -56,6 +93,7 @@ jobs:
           sudo tar -zxf ./containerd-static-1.7.0-linux-amd64.tar.gz -C /usr/
           sudo install -D -m 755 misc/optimizer/containerd-config.toml /etc/containerd/config.toml
           sudo systemctl restart containerd
+          systemctl status containerd --no-pager -l
           sudo wget https://github.com/opencontainers/runc/releases/download/v1.1.5/runc.amd64 -O /usr/bin/runc
           sudo chmod +x /usr/bin/runc
       - name: Setup CNI
@@ -66,11 +104,6 @@ jobs:
           sudo install -D -m 755 misc/example/10-containerd-net.conflist /etc/cni/net.d/10-containerd-net.conflist
       - name: Build and install optimizer
         run: |
-          rustup component add rustfmt clippy
-          make optimizer
-          sudo chown -R $(id -un):$(id -gn) . ~/.cargo/
-          pwd
-          ls -lh bin/*optimizer*
           sudo make install-optimizer
           sudo install -D -m 755 misc/example/optimizer-nri-plugin.conf /etc/nri/conf.d/02-optimizer-nri-plugin.conf
           sudo systemctl restart containerd
@@ -87,14 +120,15 @@ jobs:
           echo "count: $count expected minimum value: $expected"
           if [ $count -lt $expected ]; then
             echo "failed to generate accessed files list for nginx:1.23.3"
-            cat misc/optimizer/script/file_list.txt
+            cat /opt/nri/optimizer/results/library/nginx:1.23.3
             exit 1
           fi
           cat /opt/nri/optimizer/results/library/nginx:1.23.3.csv
       - name: Dump logs
         if: failure()
         continue-on-error: true
         run: |
-          systemctl status containerd --no-pager -l
           journalctl -xeu containerd --no-pager
+          journalctl -xeu docker --no-pager
+          journalctl -t "optimizer-nri-plugin" --no-pager
 

.github/workflows/release.yml

@@ -15,9 +15,6 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
-      - uses: actions/setup-go@v3
-        with:
-          go-version: "1.19.6"
       - name: cache go mod
         uses: actions/cache@v3
         with:
@@ -39,9 +36,7 @@ jobs:
             ${{ runner.os }}-cargo
       - name: build nydus-snapshotter and optimizer
         run: |
-          go install github.com/golangci/golangci-lint/cmd/golangci-lint@v1.51.2
-          export PATH=$PATH:$(go env GOPATH)/bin
-          make static-release
+          make container-static-release
       - name: upload artifacts
         uses: actions/upload-artifact@v3
         with:

Makefile

@@ -1,10 +1,12 @@
 all: clean build
 optimizer: clean-optimizer build-optimizer
+container-optimizer: clean-optimizer container-build-optimizer
 
 PKG = github.com/containerd/nydus-snapshotter
 PACKAGES ?= $(shell go list ./... | grep -v /tests)
 SUDO = $(shell which sudo)
 GO_EXECUTABLE_PATH ?= $(shell which go)
+CONTAINER_ENGINE ?= $(if $(shell which docker),docker,nerdctl)
 NYDUS_BUILDER ?= /usr/bin/nydus-image
 NYDUS_NYDUSD ?= /usr/bin/nydusd
 GOOS ?= linux
@@ -51,6 +53,10 @@ OPTIMIZER_SERVER_TOML = ${OPTIMIZER_SERVER}/Cargo.toml
 OPTIMIZER_SERVER_BIN = ${OPTIMIZER_SERVER}/target/release/optimizer-server
 STATIC_OPTIMIZER_SERVER_BIN = ${OPTIMIZER_SERVER}/target/x86_64-unknown-linux-gnu/release/optimizer-server
 
+CONTAINER_FLAGS += -v .:/work
+CONTAINER_RUN = ${SUDO} ${CONTAINER_ENGINE} run --rm -t ${CONTAINER_FLAGS} ghcr.io/sctb512/gobcc-builder:20.04 --
+
+
 .PHONY: build
 build:
 	GOOS=${GOOS} GOARCH=${GOARCH} ${PROXY} go build -ldflags "$(LDFLAGS)" -v -o bin/containerd-nydus-grpc ./cmd/containerd-nydus-grpc
@@ -63,11 +69,17 @@ build-optimizer:
 	GOOS=${GOOS} GOARCH=${GOARCH} ${PROXY} go build -ldflags "$(LDFLAGS)" -v -o bin/optimizer-nri-plugin ./cmd/optimizer-nri-plugin
 	make -C tools/optimizer-server release && cp ${OPTIMIZER_SERVER_BIN} ./bin
 
+container-build-optimizer:
+	${CONTAINER_RUN} "make build-optimizer"
+
 static-release:
 	CGO_ENABLED=0 ${PROXY} GOOS=${GOOS} GOARCH=${GOARCH} go build -ldflags "$(LDFLAGS) -extldflags -static" -v -o bin/containerd-nydus-grpc ./cmd/containerd-nydus-grpc
 	CGO_ENABLED=0 ${PROXY} GOOS=${GOOS} GOARCH=${GOARCH} go build -ldflags "$(LDFLAGS) -extldflags -static" -v -o bin/optimizer-nri-plugin ./cmd/optimizer-nri-plugin
 	make -C tools/optimizer-server static-release && cp ${STATIC_OPTIMIZER_SERVER_BIN} ./bin
 
+container-static-release:
+	${CONTAINER_RUN} "make static-release"
+
 # Majorly for cross build for converter package since it is imported by other projects
 converter:
 	GOOS=${GOOS} GOARCH=${GOARCH} ${PROXY} go build -ldflags "$(LDFLAGS)" -v -o bin/converter ./cmd/converter
@@ -106,31 +118,44 @@ install-optimizer:
 
 .PHONY: vet
 vet:
-	go vet $(PACKAGES) ./tests
+	go vet ${PACKAGES} ./tests
+
+container-vet:
+	${CONTAINER_RUN} "go vet ${PACKAGES} ./tests"
 
 .PHONY: check
 check: vet
 	golangci-lint run
 
+container-check: container-vet
+	${CONTAINER_RUN} "golangci-lint run"
+
 .PHONY: test
 test:
 	go test -race -v -mod=mod -cover ${PACKAGES}
 
+container-test:
+	${CONTAINER_RUN} "go test -race -v -mod=mod -cover ${PACKAGES}"
+
 .PHONY: cover
 cover:
-	go test -v -covermode=atomic -coverprofile=coverage.txt $(PACKAGES)
+	go test -v -covermode=atomic -coverprofile=coverage.txt ${PACKAGES}
 	go tool cover -func=coverage.txt
 
+container-cover:
+	${CONTAINER_RUN} "go test -v -covermode=atomic -coverprofile=coverage.txt ${PACKAGES}"
+	${CONTAINER_RUN} "go tool cover -func=coverage.txt"
+
 # make smoke TESTS=TestPack
 smoke:
 	${GO_EXECUTABLE_PATH} test -o smoke.tests -c -race -v -cover ./tests
-	$(SUDO) -E NYDUS_BUILDER=${NYDUS_BUILDER} NYDUS_NYDUSD=${NYDUS_NYDUSD} ./smoke.tests -test.v -test.timeout 10m -test.parallel=8 -test.run=$(TESTS)
+	${SUDO} -E NYDUS_BUILDER=${NYDUS_BUILDER} NYDUS_NYDUSD=${NYDUS_NYDUSD} ./smoke.tests -test.v -test.timeout 10m -test.parallel=8 -test.run=$(TESTS)
 
 .PHONY: integration
 integration:
 	CGO_ENABLED=1 ${PROXY} GOOS=${GOOS} GOARCH=${GOARCH} go build -ldflags '-X "${PKG}/version.Version=${VERSION}" -extldflags "-static"' -race -v -o bin/containerd-nydus-grpc ./cmd/containerd-nydus-grpc
-	$(SUDO) DOCKER_BUILDKIT=1 docker build ${BUILD_ARG_E2E_DOWNLOADS_MIRROR} -t nydus-snapshotter-e2e:0.1 -f integration/Dockerfile .
-	$(SUDO) docker run --cap-add SYS_ADMIN --security-opt seccomp=unconfined --cgroup-parent=system.slice --cgroupns private --name nydus-snapshotter_e2e --rm --privileged -v /root/.docker:/root/.docker -v `go env GOMODCACHE`:/go/pkg/mod \
+	${SUDO} DOCKER_BUILDKIT=1 docker build ${BUILD_ARG_E2E_DOWNLOADS_MIRROR} -t nydus-snapshotter-e2e:0.1 -f integration/Dockerfile .
+	${SUDO} docker run --cap-add SYS_ADMIN --security-opt seccomp=unconfined --cgroup-parent=system.slice --cgroupns private --name nydus-snapshotter_e2e --rm --privileged -v /root/.docker:/root/.docker -v `go env GOMODCACHE`:/go/pkg/mod \
 	-v `go env GOCACHE`:/root/.cache/go-build -v `pwd`:/nydus-snapshotter \
 	-v /usr/src/linux-headers-${KERNEL_VER}:/usr/src/linux-headers-${KERNEL_VER} \
 	${ENV_TARGET_IMAGES_FILE}  \