From ed59ba072f96796db67bbfa5b616749b963ed8ef Mon Sep 17 00:00:00 2001 From: Bin Tang Date: Mon, 10 Jul 2023 15:52:36 +0800 Subject: [PATCH 1/9] pkg: introduce pkg/optimizer package This package is used to abstract the optimizer. Signed-off-by: Bin Tang --- pkg/optimizer/optimizer.go | 113 +++++++++++++++++++++++++++++++++++++ 1 file changed, 113 insertions(+) create mode 100644 pkg/optimizer/optimizer.go diff --git a/pkg/optimizer/optimizer.go b/pkg/optimizer/optimizer.go new file mode 100644 index 0000000000..dfa92fde33 --- /dev/null +++ b/pkg/optimizer/optimizer.go @@ -0,0 +1,113 @@ +package optimizer + +import ( + "fmt" + "log/syslog" + "os" + "path/filepath" + "time" + + "github.com/containerd/containerd/reference/docker" + "github.com/containerd/nri/pkg/api" + "github.com/containerd/nydus-snapshotter/pkg/optimizer/ebpf" + "github.com/containerd/nydus-snapshotter/pkg/optimizer/fanotify" + "github.com/pkg/errors" + "github.com/sirupsen/logrus" +) + +type Server interface { + Start() error + Stop() +} + +const ( + imageNameLabel = "io.kubernetes.cri.image-name" +) + +const ( + EBPF = "ebpf" + FANOTIFY = "fanotify" +) + +type Config struct { + ServerType string `toml:"server_type"` + ServerPath string `toml:"server_path"` + PersistDir string `toml:"persist_dir"` + Readable bool `toml:"readable"` + Timeout int `toml:"timeout"` + Overwrite bool `toml:"overwrite"` +} + +func GetImageName(annotations map[string]string) (string, string, error) { + named, err := docker.ParseDockerRef(annotations[imageNameLabel]) + if err != nil { + return "", "", err + } + nameTagged := named.(docker.NamedTagged) + repo := docker.Path(nameTagged) + + dir := filepath.Dir(repo) + image := filepath.Base(repo) + + imageName := image + ":" + nameTagged.Tag() + + return dir, imageName, nil +} + +func getPersistPath(cfg Config, dir, imageName string) (string, error) { + persistDir := filepath.Join(cfg.PersistDir, dir) + if err := os.MkdirAll(persistDir, os.ModePerm); err != nil { + return "", err + } + + persistFile := filepath.Join(persistDir, imageName) + if cfg.Timeout > 0 { + persistFile = fmt.Sprintf("%s.timeout%ds", persistFile, cfg.Timeout) + } + + return persistFile, nil +} + +func getPersistFile(persistFile string) (*os.File, *os.File, error) { + f, err := os.OpenFile(persistFile, os.O_CREATE|os.O_WRONLY, 0644) + if err != nil { + return nil, nil, errors.Wrapf(err, "failed to open file %q", persistFile) + } + + persistCsvFile := fmt.Sprintf("%s.csv", persistFile) + fCsv, err := os.Create(persistCsvFile) + if err != nil { + return nil, nil, errors.Wrapf(err, "failed to create file %q", persistCsvFile) + } + + return f, fCsv, nil +} + +func NewServer(cfg Config, container *api.Container, logWriter *syslog.Writer) (string, Server, error) { + dir, imageName, err := GetImageName(container.Annotations) + if err != nil { + return "", nil, err + } + + persistPath, err := getPersistPath(cfg, dir, imageName) + if err != nil { + return "", nil, err + } + + if !cfg.Overwrite { + if file, err := os.Stat(persistPath); err == nil && !file.IsDir() { + return imageName, nil, nil + } + } + + file, csvFile, err := getPersistFile(persistPath) + if err != nil { + return "", nil, err + } + + logrus.Infof("start optimizer server for %s, image: %s, persist file: %s", container.Id, imageName, persistPath) + if cfg.ServerType == EBPF { + return imageName, ebpf.NewServer(container.Id, imageName, file, csvFile, cfg.Readable, cfg.Overwrite, time.Duration(cfg.Timeout)*time.Second, logWriter), nil + } + return imageName, fanotify.NewServer(cfg.ServerPath, container.Pid, imageName, file, csvFile, cfg.Readable, cfg.Overwrite, time.Duration(cfg.Timeout)*time.Second, logWriter), nil +} From efdc12975c43861793017bcf20a0065d64c4f0d2 Mon Sep 17 00:00:00 2001 From: Bin Tang Date: Mon, 10 Jul 2023 15:55:18 +0800 Subject: [PATCH 2/9] optimizer: refine fanotify optimizer to optimizer package Signed-off-by: Bin Tang --- pkg/{ => optimizer}/fanotify/conn/conn.go | 0 pkg/{ => optimizer}/fanotify/fanotify.go | 78 ++++++++++------------- 2 files changed, 32 insertions(+), 46 deletions(-) rename pkg/{ => optimizer}/fanotify/conn/conn.go (100%) rename pkg/{ => optimizer}/fanotify/fanotify.go (62%) diff --git a/pkg/fanotify/conn/conn.go b/pkg/optimizer/fanotify/conn/conn.go similarity index 100% rename from pkg/fanotify/conn/conn.go rename to pkg/optimizer/fanotify/conn/conn.go diff --git a/pkg/fanotify/fanotify.go b/pkg/optimizer/fanotify/fanotify.go similarity index 62% rename from pkg/fanotify/fanotify.go rename to pkg/optimizer/fanotify/fanotify.go index 7d272cea42..61608827a8 100644 --- a/pkg/fanotify/fanotify.go +++ b/pkg/optimizer/fanotify/fanotify.go @@ -17,45 +17,41 @@ import ( "syscall" "time" - "github.com/containerd/nydus-snapshotter/pkg/fanotify/conn" + "github.com/containerd/nydus-snapshotter/pkg/optimizer/fanotify/conn" "github.com/containerd/nydus-snapshotter/pkg/utils/display" "github.com/pkg/errors" "github.com/sirupsen/logrus" ) type Server struct { - BinaryPath string - ContainerPid uint32 - ImageName string - PersistFile string - Readable bool - Overwrite bool - Timeout time.Duration - Client *conn.Client - Cmd *exec.Cmd - LogWriter *syslog.Writer + BinaryPath string + ContainerPid uint32 + ImageName string + PersistFile *os.File + PersistCSVFile *os.File + Readable bool + Overwrite bool + Timeout time.Duration + Client *conn.Client + Cmd *exec.Cmd + LogWriter *syslog.Writer } -func NewServer(binaryPath string, containerPid uint32, imageName string, persistFile string, readable bool, overwrite bool, timeout time.Duration, logWriter *syslog.Writer) *Server { - return &Server{ - BinaryPath: binaryPath, - ContainerPid: containerPid, - ImageName: imageName, - PersistFile: persistFile, - Readable: readable, - Overwrite: overwrite, - Timeout: timeout, - LogWriter: logWriter, +func NewServer(binaryPath string, containerPid uint32, imageName string, file *os.File, csvFile *os.File, readable bool, overwrite bool, timeout time.Duration, logWriter *syslog.Writer) Server { + return Server{ + BinaryPath: binaryPath, + ContainerPid: containerPid, + ImageName: imageName, + PersistFile: file, + PersistCSVFile: csvFile, + Readable: readable, + Overwrite: overwrite, + Timeout: timeout, + LogWriter: logWriter, } } -func (fserver *Server) RunServer() error { - if !fserver.Overwrite { - if file, err := os.Stat(fserver.PersistFile); err == nil && !file.IsDir() { - return nil - } - } - +func (fserver Server) Start() error { cmd := exec.Command(fserver.BinaryPath) cmd.SysProcAttr = &syscall.SysProcAttr{ Cloneflags: syscall.CLONE_NEWNS, @@ -79,7 +75,7 @@ func (fserver *Server) RunServer() error { fserver.Cmd = cmd go func() { - if err := fserver.RunReceiver(); err != nil { + if err := fserver.Receive(); err != nil { logrus.WithError(err).Errorf("Failed to receive event information from server") } }() @@ -87,28 +83,18 @@ func (fserver *Server) RunServer() error { if fserver.Timeout > 0 { go func() { time.Sleep(fserver.Timeout) - fserver.StopServer() + fserver.Stop() }() } return nil } -func (fserver *Server) RunReceiver() error { - f, err := os.OpenFile(fserver.PersistFile, os.O_CREATE|os.O_WRONLY, 0644) - if err != nil { - return errors.Wrapf(err, "failed to open file %q", fserver.PersistFile) - } - defer f.Close() - - persistCsvFile := fmt.Sprintf("%s.csv", fserver.PersistFile) - fCsv, err := os.Create(persistCsvFile) - if err != nil { - return errors.Wrapf(err, "failed to create file %q", persistCsvFile) - } - defer fCsv.Close() +func (fserver Server) Receive() error { + defer fserver.PersistFile.Close() + defer fserver.PersistCSVFile.Close() - csvWriter := csv.NewWriter(fCsv) + csvWriter := csv.NewWriter(fserver.PersistCSVFile) if err := csvWriter.Write([]string{"path", "size", "elapsed"}); err != nil { return errors.Wrapf(err, "failed to write csv header") } @@ -125,7 +111,7 @@ func (fserver *Server) RunReceiver() error { } if eventInfo != nil { - fmt.Fprintln(f, eventInfo.Path) + fmt.Fprintln(fserver.PersistFile, eventInfo.Path) var line []string if fserver.Readable { @@ -143,7 +129,7 @@ func (fserver *Server) RunReceiver() error { return nil } -func (fserver *Server) StopServer() { +func (fserver Server) Stop() { if fserver.Cmd != nil { logrus.Infof("Send SIGTERM signal to process group %d", fserver.Cmd.Process.Pid) if err := syscall.Kill(-fserver.Cmd.Process.Pid, syscall.SIGTERM); err != nil { From a8dfa8de3e9df4ab461f79e27608a25e311c298b Mon Sep 17 00:00:00 2001 From: Bin Tang Date: Mon, 10 Jul 2023 15:58:27 +0800 Subject: [PATCH 3/9] optimizer: add support for ebpf optimizer Signed-off-by: Bin Tang --- pkg/optimizer/ebpf/conn/conn.go | 363 ++++++++++++++++++++++++++++++++ pkg/optimizer/ebpf/ebpf.go | 145 +++++++++++++ 2 files changed, 508 insertions(+) create mode 100644 pkg/optimizer/ebpf/conn/conn.go create mode 100644 pkg/optimizer/ebpf/ebpf.go diff --git a/pkg/optimizer/ebpf/conn/conn.go b/pkg/optimizer/ebpf/conn/conn.go new file mode 100644 index 0000000000..379e9862d2 --- /dev/null +++ b/pkg/optimizer/ebpf/conn/conn.go @@ -0,0 +1,363 @@ +/* + * Copyright (c) 2023. Nydus Developers. All rights reserved. + * + * SPDX-License-Identifier: Apache-2.0 + */ + +package conn + +import ( + "bytes" + "encoding/binary" + "strings" + "time" + + bpf "github.com/iovisor/gobpf/bcc" + "github.com/pkg/errors" +) + +import "C" + +const templateSource string = ` +#include +#include +#include +#include +#include +#include + +#define CONTAINER_ID_LEN 128 +#define FILE_PATH_LEN 256 +#define CMPMAX 16 +#define DENTRY_DEPTH_MAX 16 + +struct request_info { + char comm[16]; + char path[FILE_PATH_LEN]; + u64 position; + u32 length; +}; + +struct buf_s { + char buf[FILE_PATH_LEN*2]; +}; + +BPF_PERCPU_ARRAY(path_buf, struct buf_s, 1); +BPF_PERCPU_ARRAY(event_buf, struct request_info, 1); +BPF_HASH(vfs_read_start_trace, u64, struct request_info); +BPF_PERF_OUTPUT(fille_access_events); + +static int container_id_filter(); +static void fill_file_path(struct file *file, char *file_path); +static int local_strcmp(const char *cs, const char *ct); +static int file_filter(struct file *file); +static int trace_read_entry(struct pt_regs *ctx, struct file *file, loff_t *pos); + +static int container_id_filter() { + struct task_struct *curr_task; + struct kernfs_node *knode, *pknode; + char container_id[CONTAINER_ID_LEN]; + char expected_container_id[CONTAINER_ID_LEN] = "EXCEPTED_CONTAINERD_ID"; + char end = 0; + curr_task = (struct task_struct *) bpf_get_current_task(); + + knode = curr_task->cgroups->subsys[0]->cgroup->kn; + pknode = knode->parent; + if(pknode != NULL) + bpf_probe_read_str(container_id, CONTAINER_ID_LEN, knode->name); + else + bpf_probe_read(container_id, 1, &end); + + return local_strcmp(container_id, expected_container_id); +} + +static void fill_file_path(struct file *file, char *file_path) { + struct dentry *de , *de_last, *mnt_root; + unsigned int de_depth, len, buf_pos; + int first_de = 1; + char slash = '/'; + int zero = 0; + struct buf_s *buf = path_buf.lookup(&zero); + + if (!buf) + return; + + mnt_root = file->f_path.mnt->mnt_root; + de = file->f_path.dentry; + de_last = NULL; + buf_pos = FILE_PATH_LEN - 1; + + for (de_depth = 0; de_depth < DENTRY_DEPTH_MAX; de_depth++) { + //found root dentry + if (de == de_last || de == mnt_root) { + //fill slash + if (buf_pos == 0) + break; + buf_pos -= 1; + bpf_probe_read(&buf->buf[buf_pos & (FILE_PATH_LEN -1)], 1, &slash); + break; + } + + //fill dentry name + len = (de->d_name.len + 1) & (FILE_PATH_LEN - 1); + if (buf_pos <= len) + break; + + buf_pos -= len; + if (len != bpf_probe_read_str(&buf->buf[buf_pos & (FILE_PATH_LEN -1)], len, de->d_name.name)) + break; + + //remove null with slash + if (first_de) + first_de = 0; + else + bpf_probe_read(&buf->buf[(buf_pos + len -1) & (FILE_PATH_LEN -1)], 1, &slash); + + de_last = de; + de = de->d_parent; + } + + bpf_probe_read_str(file_path, FILE_PATH_LEN, &buf->buf[buf_pos]); +} + +/* local strcmp function, max length 16 to protect instruction loops */ +static int local_strcmp(const char *cs, const char *ct) +{ + int len = 0; + unsigned char c1, c2; + + while (len++ < CMPMAX) { + c1 = *cs++; + c2 = *ct++; + if (c1 != c2) + return c1 < c2 ? -1 : 1; + if (!c1) + break; + } + return 0; +} + +// only trace common file +static int file_filter(struct file *file) +{ + int mode = file->f_inode->i_mode; + struct super_block *mnt_sb = file->f_path.mnt->mnt_sb; + char ovl_name[CMPMAX] = "overlay"; + char fs_type[CMPMAX] = ""; + + if (!mnt_sb) + return -1; + + if (!S_ISREG(mode)) + return -1; + + bpf_probe_read_str(fs_type, sizeof(fs_type), mnt_sb->s_type->name); + if (local_strcmp(fs_type, ovl_name)) + return -1; + + return 0; +} + +static int trace_read_entry(struct pt_regs *ctx, struct file *file, loff_t *pos) +{ + int zero = 0; + u64 pid = bpf_get_current_pid_tgid(); + struct request_info *event = event_buf.lookup(&zero); + + if (!event) + return 0; + + if (file_filter(file)) { + return 0; + } + + bpf_get_current_comm(event->comm, sizeof(event->comm)); + + if (container_id_filter()) { + return 0; + }; + + bpf_probe_read(&event->position, sizeof(event->position), pos); + fill_file_path(file, event->path); + vfs_read_start_trace.update(&pid, event); + + return 0; +} + +// vfs_read will not nested , this is safe +int trace_read_return(struct pt_regs *ctx) +{ + int ret = PT_REGS_RC(ctx); + u64 pid = bpf_get_current_pid_tgid(); + struct request_info *event = vfs_read_start_trace.lookup(&pid); + if (!event) + return 0; + if (ret <= 0) + return 0; + + event->length = ret; + fille_access_events.perf_submit(ctx, event, sizeof(struct request_info)); + + return 0; +} + +int trace_vfs_read_entry(struct pt_regs *ctx, struct file *file, char __user *buf, size_t count, loff_t *pos) +{ + return trace_read_entry(ctx, file, pos); +} + +int trace_splice_read_entry(struct pt_regs *ctx, struct file *in, loff_t *ppos, + struct pipe_inode_info *pipe, size_t len, + unsigned int flags) +{ + return trace_read_entry(ctx, in, ppos); +} + +int trace_vfs_readv_entry(struct pt_regs *ctx, struct file *file, const struct iovec __user *vec, + unsigned long vlen, loff_t *pos, rwf_t flags) +{ + return trace_read_entry(ctx, file, pos); +} + +//readahead may influence the page fault collection data, disable readahead for tracing mmap read. +int trace_page_fault(struct pt_regs *ctx, struct vm_fault *vmf) +{ + int zero = 0; + u64 page_size = 4096; + u64 pid = bpf_get_current_pid_tgid(); + struct request_info *event = event_buf.lookup(&zero); + struct vm_area_struct *vma = vmf->vma; + struct file *file = vma->vm_file; + int container_id_len; + + if (!vma) + return 0; + + // only trace file backed page fault + if (!file) + return 0; + + if (!event) + return 0; + + if (file_filter(file)) { + return 0; + } + + //skip page fault for write + if ((vmf->flags & FAULT_FLAG_WRITE) && (vma->vm_flags & VM_SHARED)) + return 0; + + bpf_get_current_comm(event->comm, sizeof(event->comm)); + + if (container_id_filter()) { + return 0; + }; + + fill_file_path(file, event->path); + event->position = (vma->vm_pgoff + vmf->pgoff) * page_size; + event->length = page_size; + fille_access_events.perf_submit(ctx, event, sizeof(struct request_info)); + + return 0; +} +` + +func kprobeSyscall(m *bpf.Module, syscall string, kprobeEntry string) error { + vfsReadKprobe, err := m.LoadKprobe(kprobeEntry) + if err != nil { + return errors.Wrapf(err, "load entry %s", kprobeEntry) + } + + err = m.AttachKprobe(syscall, vfsReadKprobe, -1) + if err != nil { + return errors.Wrapf(err, "attach entry %s", kprobeEntry) + } + return nil +} + +func kretprobeSyscall(m *bpf.Module, syscall string, kprobeEntry string) error { + vfsReadKprobe, err := m.LoadKprobe(kprobeEntry) + if err != nil { + return errors.Wrapf(err, "load entry %s", kprobeEntry) + } + + err = m.AttachKretprobe(syscall, vfsReadKprobe, -1) + if err != nil { + return errors.Wrapf(err, "attach entry %s", kprobeEntry) + } + return nil +} + +func InitKprobeTable(id string) (*bpf.Module, *bpf.Table, error) { + source := strings.ReplaceAll(templateSource, "EXCEPTED_CONTAINERD_ID", id) + + m := bpf.NewModule(source, []string{}) + + if err := kprobeSyscall(m, "vfs_read", "trace_vfs_read_entry"); err != nil { + return nil, nil, err + } + if err := kprobeSyscall(m, "vfs_readv", "trace_vfs_readv_entry"); err != nil { + return nil, nil, err + } + if err := kprobeSyscall(m, "generic_file_splice_read", "trace_splice_read_entry"); err != nil { + return nil, nil, err + } + if err := kretprobeSyscall(m, "vfs_read", "trace_read_return"); err != nil { + return nil, nil, err + } + if err := kretprobeSyscall(m, "vfs_readv", "trace_read_return"); err != nil { + return nil, nil, err + } + if err := kretprobeSyscall(m, "generic_file_splice_read", "trace_read_return"); err != nil { + return nil, nil, err + } + if err := kprobeSyscall(m, "__do_fault", "trace_page_fault"); err != nil { + return nil, nil, err + } + + table := bpf.NewTable(m.TableId("fille_access_events"), m) + + return m, table, nil +} + +const ( + filePathLength = 256 +) + +type RawEventInfo struct { + Command [16]byte + Path [filePathLength]byte + Position uint64 + Length uint32 +} + +type EventInfo struct { + Timestamp int64 + Command string + Path string + Position uint64 + Size uint32 +} + +type Client struct { + Channel chan []byte +} + +func (c *Client) GetEventInfo() (*EventInfo, error) { + event := RawEventInfo{} + + data := <-c.Channel + err := binary.Read(bytes.NewBuffer(data), binary.LittleEndian, &event) + if err != nil { + return nil, errors.Wrap(err, "decode received data") + } + + return &EventInfo{ + Timestamp: time.Now().UnixMilli(), + Command: string(event.Command[:bytes.IndexByte(event.Command[:], 0)]), + Path: string(event.Path[:bytes.IndexByte(event.Path[:], 0)]), + Position: event.Position, + Size: event.Length, + }, nil +} diff --git a/pkg/optimizer/ebpf/ebpf.go b/pkg/optimizer/ebpf/ebpf.go new file mode 100644 index 0000000000..94f83007e2 --- /dev/null +++ b/pkg/optimizer/ebpf/ebpf.go @@ -0,0 +1,145 @@ +/* + * Copyright (c) 2023. Nydus Developers. All rights reserved. + * + * SPDX-License-Identifier: Apache-2.0 + */ + +package ebpf + +import ( + "encoding/csv" + "fmt" + "log/syslog" + "os" + "time" + + "github.com/containerd/nydus-snapshotter/pkg/optimizer/ebpf/conn" + "github.com/containerd/nydus-snapshotter/pkg/utils/display" + bpf "github.com/iovisor/gobpf/bcc" + "github.com/pkg/errors" + "github.com/sirupsen/logrus" +) + +type Server struct { + ContainerID string + BPFTable *bpf.Table + Module *bpf.Module + PerfMap *bpf.PerfMap + Client *conn.Client + Close chan struct{} + ImageName string + PersistFile *os.File + PersistCSVFile *os.File + Readable bool + Overwrite bool + Timeout time.Duration + LogWriter *syslog.Writer +} + +func NewServer(containerID string, imageName string, file *os.File, csvFile *os.File, readable bool, overwrite bool, timeout time.Duration, logWriter *syslog.Writer) Server { + return Server{ + ContainerID: containerID, + ImageName: imageName, + PersistFile: file, + PersistCSVFile: csvFile, + Readable: readable, + Overwrite: overwrite, + Timeout: timeout, + LogWriter: logWriter, + } +} + +func (eserver Server) Start() error { + go func() { + m, table, err := conn.InitKprobeTable(eserver.ContainerID) + if err != nil { + logrus.Infof("InitKprobeTable err: %v", err) + return + } + + channel := make(chan []byte) + eserver.PerfMap, err = bpf.InitPerfMapWithPageCnt(table, channel, nil, 1024) + if err != nil { + logrus.Infof("init perf map err: %v", err) + return + } + eserver.Module = m + eserver.Client = &conn.Client{ + Channel: channel, + } + eserver.Close = make(chan struct{}) + + go func() { + if err := eserver.Receive(); err != nil { + logrus.WithError(err).Errorf("Failed to receive event information from server") + } + }() + + eserver.PerfMap.Start() + + if eserver.Timeout > 0 { + go func() { + time.Sleep(eserver.Timeout) + eserver.Stop() + }() + } + + }() + + return nil +} + +func (eserver Server) Stop() { + if eserver.PerfMap != nil { + eserver.PerfMap.Stop() + eserver.Close <- struct{}{} + eserver.Module.Close() + } +} + +func (eserver Server) Receive() error { + defer eserver.PersistFile.Close() + defer eserver.PersistCSVFile.Close() + + csvWriter := csv.NewWriter(eserver.PersistCSVFile) + if err := csvWriter.Write([]string{"timestamp", "command", "path", "position", "size"}); err != nil { + return errors.Wrapf(err, "failed to write csv header") + } + csvWriter.Flush() + + fileList := make(map[string]struct{}) + for { + select { + case <-eserver.Close: + close(eserver.Close) + for key := range fileList { + delete(fileList, key) + } + return nil + default: + eventInfo, err := eserver.Client.GetEventInfo() + if err != nil { + return fmt.Errorf("failed to get event information: %v", err) + } + + if eventInfo != nil { + if _, ok := fileList[eventInfo.Path]; !ok { + fmt.Fprintln(eserver.PersistFile, eventInfo.Path) + fileList[eventInfo.Path] = struct{}{} + } + + var line []string + if eserver.Readable { + eventTime := time.Unix(0, eventInfo.Timestamp*int64(time.Millisecond)).Format("2006-01-02 15:04:05.000") + line = []string{eventTime, eventInfo.Command, eventInfo.Path, fmt.Sprint(eventInfo.Position), display.ByteToReadableIEC(eventInfo.Size)} + } else { + line = []string{fmt.Sprint(eventInfo.Timestamp), eventInfo.Command, eventInfo.Path, fmt.Sprint(eventInfo.Position), fmt.Sprint(eventInfo.Size)} + } + if err := csvWriter.Write(line); err != nil { + return errors.Wrapf(err, "failed to write csv") + } + csvWriter.Flush() + } + } + } +} From f269ad855d0e839c7bda48aa3d66c5595de28abd Mon Sep 17 00:00:00 2001 From: Bin Tang Date: Mon, 10 Jul 2023 15:59:25 +0800 Subject: [PATCH 4/9] optimizer: adjust nri plugin to use pkg/optimizer Remove the 'events' configuration, which does not need to be configured, and add the 'server_type' for optimizer. Signed-off-by: Bin Tang --- cmd/optimizer-nri-plugin/main.go | 108 ++++++++++--------------------- 1 file changed, 34 insertions(+), 74 deletions(-) diff --git a/cmd/optimizer-nri-plugin/main.go b/cmd/optimizer-nri-plugin/main.go index 13c51dffb9..41a9e176d6 100644 --- a/cmd/optimizer-nri-plugin/main.go +++ b/cmd/optimizer-nri-plugin/main.go @@ -8,23 +8,19 @@ package main import ( "context" - "fmt" "io" "log/syslog" "os" - "path/filepath" "strings" - "time" "github.com/pkg/errors" "github.com/sirupsen/logrus" "github.com/urfave/cli/v2" - "github.com/containerd/containerd/reference/docker" "github.com/containerd/nri/pkg/api" "github.com/containerd/nri/pkg/stub" "github.com/containerd/nydus-snapshotter/pkg/errdefs" - "github.com/containerd/nydus-snapshotter/pkg/fanotify" + "github.com/containerd/nydus-snapshotter/pkg/optimizer" "github.com/containerd/nydus-snapshotter/version" "github.com/pelletier/go-toml" ) @@ -33,23 +29,18 @@ const ( defaultEvents = "StartContainer,StopContainer" defaultServerPath = "/usr/local/bin/optimizer-server" defaultPersistDir = "/opt/nri/optimizer/results" + defaultServerType = optimizer.FANOTIFY ) type PluginConfig struct { - Events []string `toml:"events"` - - ServerPath string `toml:"server_path"` - PersistDir string `toml:"persist_dir"` - Readable bool `toml:"readable"` - Timeout int `toml:"timeout"` - Overwrite bool `toml:"overwrite"` + Events []string + optimizerCfg optimizer.Config } type PluginArgs struct { - PluginName string - PluginIdx string - PluginEvents string - Config PluginConfig + PluginName string + PluginIdx string + Config PluginConfig } type Flags struct { @@ -70,39 +61,39 @@ func buildFlags(args *PluginArgs) []cli.Flag { Destination: &args.PluginIdx, }, &cli.StringFlag{ - Name: "events", - Value: defaultEvents, - Usage: "the events that containerd subscribes to. DO NOT CHANGE THIS.", - Destination: &args.PluginEvents, + Name: "server-type", + Value: defaultServerType, + Usage: "the type of optimizer, available value includes [\"ebpf\", \"fanotify\"]", + Destination: &args.Config.optimizerCfg.ServerType, }, &cli.StringFlag{ Name: "server-path", Value: defaultServerPath, Usage: "the path of optimizer server binary", - Destination: &args.Config.ServerPath, + Destination: &args.Config.optimizerCfg.ServerPath, }, &cli.StringFlag{ Name: "persist-dir", Value: defaultPersistDir, Usage: "the directory to persist accessed files list for container", - Destination: &args.Config.PersistDir, + Destination: &args.Config.optimizerCfg.PersistDir, }, &cli.BoolFlag{ Name: "readable", Value: false, Usage: "whether to make the csv file human readable", - Destination: &args.Config.Readable, + Destination: &args.Config.optimizerCfg.Readable, }, &cli.IntFlag{ Name: "timeout", Value: 0, Usage: "the timeout to kill optimizer server, 0 to disable it", - Destination: &args.Config.Timeout, + Destination: &args.Config.optimizerCfg.Timeout, }, &cli.BoolFlag{ Name: "overwrite", Usage: "whether to overwrite the existed persistent files", - Destination: &args.Config.Overwrite, + Destination: &args.Config.optimizerCfg.Overwrite, }, } } @@ -121,15 +112,11 @@ type plugin struct { } var ( - cfg PluginConfig - log *logrus.Logger - logWriter *syslog.Writer - _ = stub.ConfigureInterface(&plugin{}) - globalFanotifyServer = make(map[string]*fanotify.Server) -) - -const ( - imageNameLabel = "io.kubernetes.cri.image-name" + cfg PluginConfig + log *logrus.Logger + logWriter *syslog.Writer + _ = stub.ConfigureInterface(&plugin{}) + globalServer = make(map[string]optimizer.Server) ) func (p *plugin) Configure(config, runtime, version string) (stub.EventMask, error) { @@ -142,7 +129,7 @@ func (p *plugin) Configure(config, runtime, version string) (stub.EventMask, err if err != nil { return 0, errors.Wrap(err, "parse TOML") } - if err := tree.Unmarshal(&cfg); err != nil { + if err := tree.Unmarshal(&cfg.optimizerCfg); err != nil { return 0, err } @@ -157,68 +144,41 @@ func (p *plugin) Configure(config, runtime, version string) (stub.EventMask, err } func (p *plugin) StartContainer(_ *api.PodSandbox, container *api.Container) error { - dir, imageName, err := GetImageName(container.Annotations) + imageName, server, err := optimizer.NewServer(cfg.optimizerCfg, container, logWriter) if err != nil { return err } - persistDir := filepath.Join(cfg.PersistDir, dir) - if err := os.MkdirAll(persistDir, os.ModePerm); err != nil { - return err - } - - persistFile := filepath.Join(persistDir, imageName) - if cfg.Timeout > 0 { - persistFile = fmt.Sprintf("%s.timeout%ds", persistFile, cfg.Timeout) + if server == nil { + return nil } - fanotifyServer := fanotify.NewServer(cfg.ServerPath, container.Pid, imageName, persistFile, cfg.Readable, cfg.Overwrite, time.Duration(cfg.Timeout)*time.Second, logWriter) - - if err := fanotifyServer.RunServer(); err != nil { + if err := server.Start(); err != nil { return err } - globalFanotifyServer[imageName] = fanotifyServer + globalServer[imageName] = server return nil } func (p *plugin) StopContainer(_ *api.PodSandbox, container *api.Container) ([]*api.ContainerUpdate, error) { var update = []*api.ContainerUpdate{} - _, imageName, err := GetImageName(container.Annotations) + _, imageName, err := optimizer.GetImageName(container.Annotations) if err != nil { return update, err } - if fanotifyServer, ok := globalFanotifyServer[imageName]; ok { - fanotifyServer.StopServer() - } else { - return nil, errors.New("can not find fanotify server for container image " + imageName) + if server, ok := globalServer[imageName]; ok { + server.Stop() } return update, nil } -func GetImageName(annotations map[string]string) (string, string, error) { - named, err := docker.ParseDockerRef(annotations[imageNameLabel]) - if err != nil { - return "", "", err - } - nameTagged := named.(docker.NamedTagged) - repo := docker.Path(nameTagged) - - dir := filepath.Dir(repo) - image := filepath.Base(repo) - - imageName := image + ":" + nameTagged.Tag() - - return dir, imageName, nil -} - func (p *plugin) onClose() { - for _, fanotifyServer := range globalFanotifyServer { - fanotifyServer.StopServer() + for _, server := range globalServer { + server.Stop() } - os.Exit(0) } func main() { @@ -255,10 +215,10 @@ func main() { p := &plugin{} - if p.mask, err = api.ParseEventMask(flags.Args.PluginEvents); err != nil { + if p.mask, err = api.ParseEventMask(defaultEvents); err != nil { log.Fatalf("failed to parse events: %v", err) } - cfg.Events = strings.Split(flags.Args.PluginEvents, ",") + cfg.Events = strings.Split(defaultEvents, ",") if p.stub, err = stub.New(p, append(opts, stub.WithOnClose(p.onClose))...); err != nil { log.Fatalf("failed to create plugin stub: %v", err) From a05e69f9ba84b1962283ebabd5e2278109fff506 Mon Sep 17 00:00:00 2001 From: Bin Tang Date: Mon, 10 Jul 2023 16:07:15 +0800 Subject: [PATCH 5/9] misc: remove events and add server_type entry Signed-off-by: Bin Tang --- misc/example/optimizer-nri-plugin.conf | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/misc/example/optimizer-nri-plugin.conf b/misc/example/optimizer-nri-plugin.conf index 7b6a561eb1..2394648abc 100644 --- a/misc/example/optimizer-nri-plugin.conf +++ b/misc/example/optimizer-nri-plugin.conf @@ -1,13 +1,13 @@ +# The type of optimizer, available value includes ["ebpf", "fanotify"] +server_type = "fanotify" # The directory to persist accessed files list for container. persist_dir = "/opt/nri/optimizer/results" # Whether to make the csv file human readable. readable = false # The path of optimizer server binary. +# Only used for fanotify optimizer. server_path = "/usr/local/bin/optimizer-server" # The timeout to kill optimizer server, 0 to disable it. timeout = 0 # Whether to overwrite the existed persistent files. -overwrite = false -# The events that containerd subscribes to. -# Do not change this element. -events = [ "StartContainer", "StopContainer" ] \ No newline at end of file +overwrite = false \ No newline at end of file From 046b59c03f9e37ecd14c1e39a7cea487f3af7b0f Mon Sep 17 00:00:00 2001 From: Bin Tang Date: Mon, 10 Jul 2023 16:08:56 +0800 Subject: [PATCH 6/9] go.mod: update go.mod for ebpf package Signed-off-by: Bin Tang --- go.mod | 1 + go.sum | 2 ++ 2 files changed, 3 insertions(+) diff --git a/go.mod b/go.mod index 799362bcf9..11efb21a2b 100644 --- a/go.mod +++ b/go.mod @@ -108,6 +108,7 @@ require ( github.com/hashicorp/go-hclog v1.2.0 // indirect github.com/hashicorp/go-multierror v1.1.1 // indirect github.com/imdario/mergo v0.3.13 + github.com/iovisor/gobpf v0.2.0 github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect diff --git a/go.sum b/go.sum index 37f0be0f7d..47ac677a98 100644 --- a/go.sum +++ b/go.sum @@ -216,6 +216,8 @@ github.com/hashicorp/go-retryablehttp v0.7.2 h1:AcYqCvkpalPnPF2pn0KamgwamS42TqUD github.com/hashicorp/go-retryablehttp v0.7.2/go.mod h1:Jy/gPYAdjqffZ/yFGCFV2doI5wjtH1ewM9u8iYVjtX8= github.com/imdario/mergo v0.3.13 h1:lFzP57bqS/wsqKssCGmtLAb8A0wKjLGrve2q3PPVcBk= github.com/imdario/mergo v0.3.13/go.mod h1:4lJ1jqUDcsbIECGy0RUJAXNIhg+6ocWgb1ALK2O4oXg= +github.com/iovisor/gobpf v0.2.0 h1:34xkQxft+35GagXBk3n23eqhm0v7q0ejeVirb8sqEOQ= +github.com/iovisor/gobpf v0.2.0/go.mod h1:WSY9Jj5RhdgC3ci1QaacvbFdQ8cbrEjrpiZbLHLt2s4= github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= From bf8615813ce72a4ae88287ec5c0802b0474121a5 Mon Sep 17 00:00:00 2001 From: Bin Tang Date: Thu, 24 Aug 2023 14:45:28 +0800 Subject: [PATCH 7/9] optimizer: transmit container id to optimizer via map Signed-off-by: Bin Tang --- pkg/optimizer/ebpf/conn/conn.go | 24 +++++++++++++++++------- 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/pkg/optimizer/ebpf/conn/conn.go b/pkg/optimizer/ebpf/conn/conn.go index 379e9862d2..e413d063ef 100644 --- a/pkg/optimizer/ebpf/conn/conn.go +++ b/pkg/optimizer/ebpf/conn/conn.go @@ -9,7 +9,6 @@ package conn import ( "bytes" "encoding/binary" - "strings" "time" bpf "github.com/iovisor/gobpf/bcc" @@ -18,7 +17,7 @@ import ( import "C" -const templateSource string = ` +const sourceCode string = ` #include #include #include @@ -42,6 +41,9 @@ struct buf_s { char buf[FILE_PATH_LEN*2]; }; +char container_id[CONTAINER_ID_LEN]; + +BPF_ARRAY(id_buf, container_id, 1); BPF_PERCPU_ARRAY(path_buf, struct buf_s, 1); BPF_PERCPU_ARRAY(event_buf, struct request_info, 1); BPF_HASH(vfs_read_start_trace, u64, struct request_info); @@ -57,8 +59,13 @@ static int container_id_filter() { struct task_struct *curr_task; struct kernfs_node *knode, *pknode; char container_id[CONTAINER_ID_LEN]; - char expected_container_id[CONTAINER_ID_LEN] = "EXCEPTED_CONTAINERD_ID"; char end = 0; + int zero = 0; + char *buf_id = container_id.lookup(&zero); + if (!buf_id) { + return -1; + } + curr_task = (struct task_struct *) bpf_get_current_task(); knode = curr_task->cgroups->subsys[0]->cgroup->kn; @@ -68,7 +75,7 @@ static int container_id_filter() { else bpf_probe_read(container_id, 1, &end); - return local_strcmp(container_id, expected_container_id); + return local_strcmp(container_id, buf_id); } static void fill_file_path(struct file *file, char *file_path) { @@ -290,9 +297,7 @@ func kretprobeSyscall(m *bpf.Module, syscall string, kprobeEntry string) error { } func InitKprobeTable(id string) (*bpf.Module, *bpf.Table, error) { - source := strings.ReplaceAll(templateSource, "EXCEPTED_CONTAINERD_ID", id) - - m := bpf.NewModule(source, []string{}) + m := bpf.NewModule(sourceCode, []string{}) if err := kprobeSyscall(m, "vfs_read", "trace_vfs_read_entry"); err != nil { return nil, nil, err @@ -316,6 +321,11 @@ func InitKprobeTable(id string) (*bpf.Module, *bpf.Table, error) { return nil, nil, err } + containerIDTable := bpf.NewTable(m.TableId("id_buf"), m) + if err := containerIDTable.Set([]byte{0}, []byte(id)); err != nil { + return nil, nil, err + } + table := bpf.NewTable(m.TableId("fille_access_events"), m) return m, table, nil From 37b1ab09a16bc2a7545f1ddb847d1efd986956a9 Mon Sep 17 00:00:00 2001 From: Bin Tang Date: Thu, 24 Aug 2023 14:46:17 +0800 Subject: [PATCH 8/9] Makefile: build optimizer in container Signed-off-by: Bin Tang --- Makefile | 35 ++++++++++++++++++++++++++++++----- 1 file changed, 30 insertions(+), 5 deletions(-) diff --git a/Makefile b/Makefile index 32313a4355..92cd1c73a8 100644 --- a/Makefile +++ b/Makefile @@ -1,10 +1,12 @@ all: clean build optimizer: clean-optimizer build-optimizer +container-optimizer: clean-optimizer container-build-optimizer PKG = github.com/containerd/nydus-snapshotter PACKAGES ?= $(shell go list ./... | grep -v /tests) SUDO = $(shell which sudo) GO_EXECUTABLE_PATH ?= $(shell which go) +CONTAINER_ENGINE ?= $(if $(shell which docker),docker,nerdctl) NYDUS_BUILDER ?= /usr/bin/nydus-image NYDUS_NYDUSD ?= /usr/bin/nydusd GOOS ?= linux @@ -51,6 +53,10 @@ OPTIMIZER_SERVER_TOML = ${OPTIMIZER_SERVER}/Cargo.toml OPTIMIZER_SERVER_BIN = ${OPTIMIZER_SERVER}/target/release/optimizer-server STATIC_OPTIMIZER_SERVER_BIN = ${OPTIMIZER_SERVER}/target/x86_64-unknown-linux-gnu/release/optimizer-server +CONTAINER_FLAGS += -v .:/work +CONTAINER_RUN = ${SUDO} ${CONTAINER_ENGINE} run --rm -t ${CONTAINER_FLAGS} ghcr.io/sctb512/gobcc-builder:20.04 -- + + .PHONY: build build: GOOS=${GOOS} GOARCH=${GOARCH} ${PROXY} go build -ldflags "$(LDFLAGS)" -v -o bin/containerd-nydus-grpc ./cmd/containerd-nydus-grpc @@ -63,11 +69,17 @@ build-optimizer: GOOS=${GOOS} GOARCH=${GOARCH} ${PROXY} go build -ldflags "$(LDFLAGS)" -v -o bin/optimizer-nri-plugin ./cmd/optimizer-nri-plugin make -C tools/optimizer-server release && cp ${OPTIMIZER_SERVER_BIN} ./bin +container-build-optimizer: + ${CONTAINER_RUN} "make build-optimizer" + static-release: CGO_ENABLED=0 ${PROXY} GOOS=${GOOS} GOARCH=${GOARCH} go build -ldflags "$(LDFLAGS) -extldflags -static" -v -o bin/containerd-nydus-grpc ./cmd/containerd-nydus-grpc CGO_ENABLED=0 ${PROXY} GOOS=${GOOS} GOARCH=${GOARCH} go build -ldflags "$(LDFLAGS) -extldflags -static" -v -o bin/optimizer-nri-plugin ./cmd/optimizer-nri-plugin make -C tools/optimizer-server static-release && cp ${STATIC_OPTIMIZER_SERVER_BIN} ./bin +container-static-release: + ${CONTAINER_RUN} "make static-release" + # Majorly for cross build for converter package since it is imported by other projects converter: GOOS=${GOOS} GOARCH=${GOARCH} ${PROXY} go build -ldflags "$(LDFLAGS)" -v -o bin/converter ./cmd/converter @@ -106,31 +118,44 @@ install-optimizer: .PHONY: vet vet: - go vet $(PACKAGES) ./tests + go vet ${PACKAGES} ./tests + +container-vet: + ${CONTAINER_RUN} "go vet ${PACKAGES} ./tests" .PHONY: check check: vet golangci-lint run +container-check: container-vet + ${CONTAINER_RUN} "golangci-lint run" + .PHONY: test test: go test -race -v -mod=mod -cover ${PACKAGES} +container-test: + ${CONTAINER_RUN} "go test -race -v -mod=mod -cover ${PACKAGES}" + .PHONY: cover cover: - go test -v -covermode=atomic -coverprofile=coverage.txt $(PACKAGES) + go test -v -covermode=atomic -coverprofile=coverage.txt ${PACKAGES} go tool cover -func=coverage.txt +container-cover: + ${CONTAINER_RUN} "go test -v -covermode=atomic -coverprofile=coverage.txt ${PACKAGES}" + ${CONTAINER_RUN} "go tool cover -func=coverage.txt" + # make smoke TESTS=TestPack smoke: ${GO_EXECUTABLE_PATH} test -o smoke.tests -c -race -v -cover ./tests - $(SUDO) -E NYDUS_BUILDER=${NYDUS_BUILDER} NYDUS_NYDUSD=${NYDUS_NYDUSD} ./smoke.tests -test.v -test.timeout 10m -test.parallel=8 -test.run=$(TESTS) + ${SUDO} -E NYDUS_BUILDER=${NYDUS_BUILDER} NYDUS_NYDUSD=${NYDUS_NYDUSD} ./smoke.tests -test.v -test.timeout 10m -test.parallel=8 -test.run=$(TESTS) .PHONY: integration integration: CGO_ENABLED=1 ${PROXY} GOOS=${GOOS} GOARCH=${GOARCH} go build -ldflags '-X "${PKG}/version.Version=${VERSION}" -extldflags "-static"' -race -v -o bin/containerd-nydus-grpc ./cmd/containerd-nydus-grpc - $(SUDO) DOCKER_BUILDKIT=1 docker build ${BUILD_ARG_E2E_DOWNLOADS_MIRROR} -t nydus-snapshotter-e2e:0.1 -f integration/Dockerfile . - $(SUDO) docker run --cap-add SYS_ADMIN --security-opt seccomp=unconfined --cgroup-parent=system.slice --cgroupns private --name nydus-snapshotter_e2e --rm --privileged -v /root/.docker:/root/.docker -v `go env GOMODCACHE`:/go/pkg/mod \ + ${SUDO} DOCKER_BUILDKIT=1 docker build ${BUILD_ARG_E2E_DOWNLOADS_MIRROR} -t nydus-snapshotter-e2e:0.1 -f integration/Dockerfile . + ${SUDO} docker run --cap-add SYS_ADMIN --security-opt seccomp=unconfined --cgroup-parent=system.slice --cgroupns private --name nydus-snapshotter_e2e --rm --privileged -v /root/.docker:/root/.docker -v `go env GOMODCACHE`:/go/pkg/mod \ -v `go env GOCACHE`:/root/.cache/go-build -v `pwd`:/nydus-snapshotter \ -v /usr/src/linux-headers-${KERNEL_VER}:/usr/src/linux-headers-${KERNEL_VER} \ ${ENV_TARGET_IMAGES_FILE} \ From 2dbd24fa77c8bdd8a30b7b10474955a9cd667aa9 Mon Sep 17 00:00:00 2001 From: Bin Tang Date: Thu, 24 Aug 2023 15:35:59 +0800 Subject: [PATCH 9/9] workflow: build eBPF optimizer in container Signed-off-by: Bin Tang --- .github/workflows/ci.yml | 31 +++++------------- .github/workflows/optimizer.yml | 58 ++++++++++++++++++++++++++------- .github/workflows/release.yml | 7 +--- 3 files changed, 55 insertions(+), 41 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8adcc8db95..19e48ecd38 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -12,7 +12,7 @@ env: jobs: build: name: Build and Lint - timeout-minutes: 10 + timeout-minutes: 20 runs-on: ubuntu-latest steps: - name: Set up Go @@ -33,17 +33,13 @@ jobs: go install github.com/golangci/golangci-lint/cmd/golangci-lint@v1.51.2 export PATH=$PATH:$(go env GOPATH)/bin make - make test - make check + make container-test + make container-check build-optimizer: name: Build optimizer - timeout-minutes: 10 + timeout-minutes: 20 runs-on: ubuntu-latest steps: - - name: Set up Go - uses: actions/setup-go@v3 - with: - go-version: "1.20.1" - name: Check out code uses: actions/checkout@v3 - name: cache go mod @@ -55,19 +51,12 @@ jobs: ${{ runner.os }}-go - name: Build run: | - go install github.com/golangci/golangci-lint/cmd/golangci-lint@v1.51.2 - export PATH=$PATH:$(go env GOPATH)/bin - rustup component add rustfmt clippy - make build-optimizer + make container-build-optimizer smoke: name: Smoke - timeout-minutes: 10 + timeout-minutes: 20 runs-on: ubuntu-latest steps: - - name: Set up Go - uses: actions/setup-go@v3 - with: - go-version: "1.19.6" - name: Check out code uses: actions/checkout@v3 - name: cache go mod @@ -118,14 +107,10 @@ jobs: coverage: name: Code coverage - timeout-minutes: 10 + timeout-minutes: 20 runs-on: ubuntu-latest needs: [build] steps: - - name: Install Go - uses: actions/setup-go@v3 - with: - go-version: "1.19.6" - name: Checkout code uses: actions/checkout@v3 - name: cache go mod @@ -136,7 +121,7 @@ jobs: restore-keys: | ${{ runner.os }}-go - name: Run unit tests. - run: make cover + run: make container-cover - name: Upload coverage to Codecov uses: codecov/codecov-action@v3 with: diff --git a/.github/workflows/optimizer.yml b/.github/workflows/optimizer.yml index d8d3cf266a..64cae335f9 100644 --- a/.github/workflows/optimizer.yml +++ b/.github/workflows/optimizer.yml @@ -19,12 +19,11 @@ env: jobs: run_optimizer: - runs-on: ubuntu-latest + runs-on: ubuntu-20.04 + env: + GOBCC_BUILDER_IMAGE: 'ghcr.io/sctb512/gobcc-builder:20.04' + GOBCC_BUILDER_IMAGE_FILE: '~/.image/gobcc-builder:20.04.tar' steps: - - name: Set up Go - uses: actions/setup-go@v3 - with: - go-version: "1.19.6" - name: Checkout repository uses: actions/checkout@v3 - name: cache go mod @@ -46,6 +45,44 @@ jobs: key: ${{ runner.os }}-cargo-${{ hashFiles('tools/optimizer-server/Cargo.lock') }} restore-keys: | ${{ runner.os }}-cargo + - name: cache go bcc image + uses: actions/cache@v3 + with: + path: | + ${GOBCC_BUILDER_IMAGE_FILE} + key: ${{ runner.os }}-bogcc-builder-${{ hashFiles('${GOBCC_BUILDER_IMAGE_FILE}') }} + restore-keys: | + ${{ runner.os }}-bogcc-builder + - name: load go bcc image + run: | + if [ -f ${GOBCC_BUILDER_IMAGE_FILE} ];then + sudo docker load < ${GOBCC_BUILDER_IMAGE_FILE} + else + sudo docker pull ${GOBCC_BUILDER_IMAGE} + fi + - name: Build optimizer + run: | + make container-optimizer + pwd + ldd bin/*optimizer* + ls -lh bin/*optimizer* + - name: Setup bcc for eBPF + run: | + sudo apt-get update + sudo apt install -y zip bison build-essential cmake flex git libedit-dev \ + libllvm12 llvm-12-dev libclang-12-dev python zlib1g-dev libelf-dev libfl-dev python3-setuptools \ + liblzma-dev arping netperf iperf + git clone https://github.com/iovisor/bcc.git; cd bcc + git checkout v0.24.0 + mkdir build; cd build + cmake .. + make + sudo make install + cmake -DPYTHON_CMD=python3 .. + pushd src/python/ + make + sudo make install + popd - name: containerd runc and crictl run: | sudo wget https://github.com/kubernetes-sigs/cri-tools/releases/download/v1.26.0/crictl-v1.26.0-linux-amd64.tar.gz @@ -56,6 +93,7 @@ jobs: sudo tar -zxf ./containerd-static-1.7.0-linux-amd64.tar.gz -C /usr/ sudo install -D -m 755 misc/optimizer/containerd-config.toml /etc/containerd/config.toml sudo systemctl restart containerd + systemctl status containerd --no-pager -l sudo wget https://github.com/opencontainers/runc/releases/download/v1.1.5/runc.amd64 -O /usr/bin/runc sudo chmod +x /usr/bin/runc - name: Setup CNI @@ -66,11 +104,6 @@ jobs: sudo install -D -m 755 misc/example/10-containerd-net.conflist /etc/cni/net.d/10-containerd-net.conflist - name: Build and install optimizer run: | - rustup component add rustfmt clippy - make optimizer - sudo chown -R $(id -un):$(id -gn) . ~/.cargo/ - pwd - ls -lh bin/*optimizer* sudo make install-optimizer sudo install -D -m 755 misc/example/optimizer-nri-plugin.conf /etc/nri/conf.d/02-optimizer-nri-plugin.conf sudo systemctl restart containerd @@ -87,7 +120,7 @@ jobs: echo "count: $count expected minimum value: $expected" if [ $count -lt $expected ]; then echo "failed to generate accessed files list for nginx:1.23.3" - cat misc/optimizer/script/file_list.txt + cat /opt/nri/optimizer/results/library/nginx:1.23.3 exit 1 fi cat /opt/nri/optimizer/results/library/nginx:1.23.3.csv @@ -95,6 +128,7 @@ jobs: if: failure() continue-on-error: true run: | - systemctl status containerd --no-pager -l journalctl -xeu containerd --no-pager + journalctl -xeu docker --no-pager + journalctl -t "optimizer-nri-plugin" --no-pager \ No newline at end of file diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 74ee92dda6..cf3c268024 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -15,9 +15,6 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 - - uses: actions/setup-go@v3 - with: - go-version: "1.19.6" - name: cache go mod uses: actions/cache@v3 with: @@ -39,9 +36,7 @@ jobs: ${{ runner.os }}-cargo - name: build nydus-snapshotter and optimizer run: | - go install github.com/golangci/golangci-lint/cmd/golangci-lint@v1.51.2 - export PATH=$PATH:$(go env GOPATH)/bin - make static-release + make container-static-release - name: upload artifacts uses: actions/upload-artifact@v3 with: