使用 eBPF 編寫系統調用跟蹤器
先決條件
系統調用、eBPF、C 語言、底層編程基礎。
簡介
eBPF(擴展的伯克利數據包過濾器) 是一項允許用戶在內核中運行自定義程序的技術。BPF 或 cBPF(經典 BPF) 是 eBPF 的前身, 它提供了一種簡單高效的方法來基於預定義規則過濾數據包。與內核模塊相比, eBPF 程序提供了更高的安全性、可移植性和可維護性。現有多種高級方法可用於處理 eBPF 程序, 如 Cilium 的 Go 語言庫、bpftrace、libbpf 等。
注意: 本文要求讀者對eBPF有基本瞭解。如果你不熟悉它,ebpf.io上的這篇文章是很好的參考資料。
目標
你應該已經熟悉著名的工具 strace。我們將使用 eBPF 開發類似的工具。例如:
./beetrace /bin/ls以下是該文本的地道中文翻譯：
概念
在開始編寫我們的工具之前，我們需要熟悉一些關鍵概念。
-
跟蹤點（Tracepoints）：這些是放置在 Linux 內核代碼各個部分的檢測點。它們提供了一種方法，可以在不修改內核源代碼的情況下，鉤入內核中的特定事件或代碼路徑。可用於跟蹤的事件可以在/sys/kernel/debug/tracing/events中找到。 -
SEC宏：它在目標 ELF 文件中創建一個新的段，段名與跟蹤點的名稱相同。例如，SEC(tracepoint/raw_syscalls/sys_enter)創建了一個具有這個名稱的新段。可以使用 readelf 命令查看這些段。
readelf -s --wide somefile.o映射（Maps）：這些是可以從 eBPF 程序和用戶空間運行的應用程序中訪問的共享數據結構。
編寫 eBPF 程序
由於 Linux 內核中存在大量的系統調用，我們不會編寫一個全面的工具來跟蹤所有系統調用。相反，我們將專注於跟蹤幾個常見的系統調用。爲了實現這一目標，我們將編寫兩類程序：eBPF 程序和加載器（用於將 BPF 對象加載到內核並將其附加進來）。
讓我們首先創建一些數據結構來進行初始設置:
// controller.h
// SYS_ENTER : for retrieving system call arguments
// SYS_EXIT : for retrieving the return values of syscalls
typedef enum
{
SYS_ENTER,
SYS_EXIT
} event_mode;
struct inner_syscall_info
{
union
{
struct
{
// For SYS_ENTER mode
char name[32];
int num_args;
long syscall_nr;
void *args[MAX_ARGS];
};
long retval; // For SYS_EXIT mode
};
event_mode mode;
};
struct default_syscall_info{
char name[32];
int num_args;
};
// Array for storing the name and argument count of system calls
const struct default_syscall_info syscalls[MAX_SYSCALL_NR] = {
[SYS_fork] = {"fork", 0},
[SYS_alarm] = {"alarm", 1},
[SYS_brk] = {"brk", 1},
[SYS_close] = {"close", 1},
[SYS_exit] = {"exit", 1},
[SYS_exit_group] = {"exit_group", 1},
[SYS_set_tid_address] = {"set_tid_address", 1},
[SYS_set_robust_list] = {"set_robust_list", 1},
[SYS_access] = {"access", 2},
[SYS_arch_prctl] = {"arch_prctl", 2},
[SYS_kill] = {"kill", 2},
[SYS_listen] = {"listen", 2},
[SYS_munmap] = {"sys_munmap", 2},
[SYS_open] = {"open", 2},
[SYS_stat] = {"stat", 2},
[SYS_fstat] = {"fstat", 2},
[SYS_lstat] = {"lstat", 2},
[SYS_accept] = {"accept", 3},
[SYS_connect] = {"connect", 3},
[SYS_execve] = {"execve", 3},
[SYS_ioctl] = {"ioctl", 3},
[SYS_getrandom] = {"getrandom", 3},
[SYS_lseek] = {"lseek", 3},
[SYS_poll] = {"poll", 3},
[SYS_read] = {"read", 3},
[SYS_write] = {"write", 3},
[SYS_mprotect] = {"mprotect", 3},
[SYS_openat] = {"openat", 3},
[SYS_socket] = {"socket", 3},
[SYS_newfstatat] = {"newfstatat", 4},
[SYS_pread64] = {"pread64", 4},
[SYS_prlimit64] = {"prlimit64", 4},
[SYS_rseq] = {"rseq", 4},
[SYS_sendfile] = {"sendfile", 4},
[SYS_socketpair] = {"socketpair", 4},
[SYS_mmap] = {"mmap", 6},
[SYS_recvfrom] = {"recvfrom", 6},
[SYS_sendto] = {"sendto", 6},
};加載器將讀取用戶通過命令行參數提供的待追蹤 ELF 文件的路徑。然後，加載器會創建一個子進程，並使用 execve 來運行命令行參數中指定的程序。
父進程將處理加載和附加 eBPF 程序所需的所有設置。它還執行一項關鍵任務：通過 BPF 哈希映射將子進程的 ID 發送給 eBPF 程序。
// loader.c
int main(int argc, char **argv)
{
if (argc < 2)
{
fatal_error("Usage: ./beetrace <path_to_program>");
}
const char *file_path = argv[1];
pid_t pid = fork();
if (pid == 0)
{
// Child process
int fd = open("/dev/null", O_WRONLY);
if(fd==-1){
// error
}
dup2(fd, 1); // disable stdout for the child process
sleep(2); // wait for the parent process to do the required setup for tracing
execve(file_path, NULL, NULL);
}
else{
// Parent process
}
}要追蹤系統調用，我們需要編寫由 tracepoint/raw_syscalls/sys_enter 和 tracepoint/raw_syscalls/sys_exit 跟蹤點觸發的 eBPF 程序。這些跟蹤點提供了對系統調用號和參數的訪問。對於給定的系統調用，tracepoint/raw_syscalls/sys_enter 跟蹤點總是在 tracepoint/raw_syscalls/sys_exit 跟蹤點之前觸發。我們可以使用前者獲取系統調用參數，使用後者獲取返回值。
此外，我們將使用 eBPF 映射在用戶空間程序和我們的 eBPF 程序之間共享信息。具體來說，我們將使用兩種類型的 eBPF 映射：哈希映射和環形緩衝區。
// controller.c
// Hashmap
struct
{
__uint(type, BPF_MAP_TYPE_HASH);
__uint(key_size, 10);
__uint(value_size, 4);
__uint(max_entries, 256 * 1024);
} pid_hashmap SEC(".maps");
// Ring buffer
struct
{
__uint(type, BPF_MAP_TYPE_RINGBUF);
__uint(max_entries, 256 * 1024);
} syscall_info_buffer SEC(".maps");確定了映射關係之後，我們就可以動手寫代碼了。首先，讓我們來編寫針對追蹤點 tracepoint/raw_syscalls/sys_enter 的程序代碼。
// loader.c
SEC("tracepoint/raw_syscalls/sys_enter")
int detect_syscall_enter(struct trace_event_raw_sys_enter *ctx)
{
// Retrieve the system call number
long syscall_nr = ctx->id;
const char *key = "child_pid";
int target_pid;
// Reading the process id of the child process in userland
void *value = bpf_map_lookup_elem(&pid_hashmap, key);
void *args[MAX_ARGS];
if (value)
{
target_pid = *(int *)value;
// PID of the process that executed the current system call
pid_t pid = bpf_get_current_pid_tgid() & 0xffffffff;
if (pid == target_pid && syscall_nr >= 0 && syscall_nr < MAX_SYSCALL_NR)
{
int idx = syscall_nr;
// Reserve space in the ring buffer
struct inner_syscall_info *info = bpf_ringbuf_reserve(&syscall_info_buffer, sizeof(struct inner_syscall_info), 0);
if (!info)
{
bpf_printk("bpf_ringbuf_reserve failed");
return 1;
}
// Copy the syscall name into info->name
bpf_probe_read_kernel_str(info->name, sizeof(syscalls[syscall_nr].name), syscalls[syscall_nr].name);
for (int i = 0; i < MAX_ARGS; i++)
{
info->args[i] = (void *)BPF_CORE_READ(ctx, args[i]);
}
info->num_args = syscalls[syscall_nr].num_args;
info->syscall_nr = syscall_nr;
info->mode = SYS_ENTER;
// Insert into ring buffer
bpf_ringbuf_submit(info, 0);
}
}
return 0;
}同理，我們也能編寫用於讀取返回值並將其傳遞給用戶態空間的程序代碼。
// controller.c
SEC("tracepoint/raw_syscalls/sys_exit")
int detect_syscall_exit(struct trace_event_raw_sys_exit *ctx)
{
const char *key = "child_pid";
void *value = bpf_map_lookup_elem(&pid_hashmap, key);
pid_t pid, target_pid;
if (value)
{
pid = bpf_get_current_pid_tgid() & 0xffffffff;
target_pid = *(pid_t *)value;
if (pid == target_pid)
{
struct inner_syscall_info *info = bpf_ringbuf_reserve(&syscall_info_buffer, sizeof(struct inner_syscall_info), 0);
if (!info)
{
bpf_printk("bpf_ringbuf_reserve failed");
return 1;
}
info->mode = SYS_EXIT;
info->retval = ctx->ret;
bpf_ringbuf_submit(info, 0);
}
}
return 0;
}現在，讓我們來完善加載器程序中父進程的功能部分。但在進行之前，我們需要理解幾個關鍵函數的工作原理。1、bpf_object__open: 通過打開由傳遞路徑指向的 BPF ELF 對象文件並在內存中加載它，創建一個 bpf_object 結構體實例。
LIBBPF_API struct bpf_object *bpf_object__open(const char *path);2、bpf_object__load: 將 BPF 對象加載到內核中。
LIBBPF_API int bpf_object__load(struct bpf_object *obj);3、bpf_object__find_program_by_name: 返回指向有效 BPF 程序的指針。
LIBBPF_API struct bpf_program *bpf_object__find_program_by_name(const struct bpf_object *obj, const char *name);4、bpf_program__attach: 根據自動檢測的程序類型、附加類型和適用的額外參數，將 BPF 程序附加到內核。
LIBBPF_API struct bpf_link *bpf_program__attach(const struct bpf_program *prog);5、bpf_map__update_elem: 允許在與提供的鍵對應的 BPF 映射中插入或更新值。
LIBBPF_API int bpf_map__update_elem(const struct bpf_map *map, const void *key, size_t key_sz, const void *value, size_t value_sz, __u64 flags);6、bpf_object__find_map_fd_by_name: 給定一個 BPF 映射名稱，返回該映射的文件描述符。
LIBBPF_API int bpf_object__find_map_fd_by_name(const struct bpf_object *obj, const char *name);7、ring_buffer__new: 返回指向環形緩衝區的指針。
LIBBPF_API struct ring_buffer *ring_buffer__new(int map_fd, ring_buffer_sample_fn sample_cb, void *ctx, const struct ring_buffer_opts *opts);第二個參數必須是一個可用於處理從環形緩衝區接收的數據的回調函數。
bool initialized = false;
static int syscall_logger(void *ctx, void *data, size_t len)
{
struct inner_syscall_info *info = (struct inner_syscall_info *)data;
if (!info)
{
return -1;
}
if (info->mode == SYS_ENTER)
{
initialized = true;
printf("%s(", info->name);
for (int i = 0; i < info->num_args; i++)
{
printf("%p,", info->args[i]);
}
printf("\b) = ");
}
else if (info->mode == SYS_EXIT)
{
if (initialized)
{
printf("0x%lx\n", info->retval);
}
}
return 0;
}它會打印系統調用的名稱和參數。
8、ring_buffer__consume: 此函數處理環形緩衝區中可用的事件。
LIBBPF_API int ring_buffer__consume(struct ring_buffer *rb);現在我們有了編寫加載器所需的一切要素。
// loader.c
#include <bpf/libbpf.h>
#include "controller.h"
#include <fcntl.h>
#include <stdio.h>
#include <stdlib.h>
#include <sys/wait.h>
#include <unistd.h>
void fatal_error(const char *message)
{
puts(message);
exit(1);
}
bool initialized = false;
static int syscall_logger(void *ctx, void *data, size_t len)
{
struct inner_syscall_info *info = (struct inner_syscall_info *)data;
if (!info)
{
return -1;
}
if (info->mode == SYS_ENTER)
{
initialized = true;
printf("%s(", info->name);
for (int i = 0; i < info->num_args; i++)
{
printf("%p,", info->args[i]);
}
printf("\b) = ");
}
else if (info->mode == SYS_EXIT)
{
if (initialized)
{
printf("0x%lx\n", info->retval);
}
}
return 0;
}
int main(int argc, char **argv)
{
int status;
struct bpf_object *obj;
struct bpf_program *enter_prog, *exit_prog;
struct bpf_map *syscall_map;
const char *obj_name = "controller.o";
const char *map_name = "pid_hashmap";
const char *enter_prog_name = "detect_syscall_enter";
const char *exit_prog_name = "detect_syscall_exit";
const char *syscall_info_bufname = "syscall_info_buffer";
if (argc < 2)
{
fatal_error("Usage: ./beetrace <path_to_program>");
}
const char *file_path = argv[1];
pid_t pid = fork();
if (pid == 0)
{
int fd = open("/dev/null", O_WRONLY);
if(fd==-1){
fatal_error("failed to open /dev/null");
}
dup2(fd, 1);
sleep(2);
execve(file_path, NULL, NULL);
}
else
{
printf("Spawned child process with a PID of %d\n", pid);
obj = bpf_object__open(obj_name);
if (!obj)
{
fatal_error("failed to open the BPF object");
}
if (bpf_object__load(obj))
{
fatal_error("failed to load the BPF object into kernel");
}
enter_prog = bpf_object__find_program_by_name(obj, enter_prog_name);
exit_prog = bpf_object__find_program_by_name(obj, exit_prog_name);
if (!enter_prog || !exit_prog)
{
fatal_error("failed to find the BPF program");
}
if (!bpf_program__attach(enter_prog) || !bpf_program__attach(exit_prog))
{
fatal_error("failed to attach the BPF program");
}
syscall_map = bpf_object__find_map_by_name(obj, map_name);
if (!syscall_map)
{
fatal_error("failed to find the BPF map");
}
const char *key = "child_pid";
int err = bpf_map__update_elem(syscall_map, key, 10, (void *)&pid, sizeof(pid_t), 0);
if (err)
{
printf("%d", err);
fatal_error("failed to insert child pid into the ring buffer");
}
int rbFd = bpf_object__find_map_fd_by_name(obj, syscall_info_bufname);
struct ring_buffer *rbuffer = ring_buffer__new(rbFd, syscall_logger, NULL, NULL);
if (!rbuffer)
{
fatal_error("failed to allocate ring buffer");
}
if (wait(&status) == -1)
{
fatal_error("failed to wait for the child process");
}
while (1)
{
int e = ring_buffer__consume(rbuffer);
if (!e)
{
break;
}
sleep(1);
}
}
return 0;
}以下便是 eBPF 程序的部分。所有的 C 語言源碼最終會被編譯整合成單一的對象文件。
// controller.c
#include "vmlinux.h"
#include <bpf/bpf_helpers.h>
#include <bpf/bpf_core_read.h>
#include <sys/syscall.h>
#include "controller.h"
struct
{
__uint(type, BPF_MAP_TYPE_HASH);
__uint(key_size, 10);
__uint(value_size, 4);
__uint(max_entries, 256 * 1024);
} pid_hashmap SEC(".maps");
struct
{
__uint(type, BPF_MAP_TYPE_RINGBUF);
__uint(max_entries, 256 * 1024);
} syscall_info_buffer SEC(".maps");
SEC("tracepoint/raw_syscalls/sys_enter")
int detect_syscall_enter(struct trace_event_raw_sys_enter *ctx)
{
// Retrieve the system call number
long syscall_nr = ctx->id;
const char *key = "child_pid";
int target_pid;
// Reading the process id of the child process in userland
void *value = bpf_map_lookup_elem(&pid_hashmap, key);
void *args[MAX_ARGS];
if (value)
{
target_pid = *(int *)value;
// PID of the process that executed the current system call
pid_t pid = bpf_get_current_pid_tgid() & 0xffffffff;
if (pid == target_pid && syscall_nr >= 0 && syscall_nr < MAX_SYSCALL_NR)
{
int idx = syscall_nr;
// Reserve space in the ring buffer
struct inner_syscall_info *info = bpf_ringbuf_reserve(&syscall_info_buffer, sizeof(struct inner_syscall_info), 0);
if (!info)
{
bpf_printk("bpf_ringbuf_reserve failed");
return 1;
}
// Copy the syscall name into info->name
bpf_probe_read_kernel_str(info->name, sizeof(syscalls[syscall_nr].name), syscalls[syscall_nr].name);
for (int i = 0; i < MAX_ARGS; i++)
{
info->args[i] = (void *)BPF_CORE_READ(ctx, args[i]);
}
info->num_args = syscalls[syscall_nr].num_args;
info->syscall_nr = syscall_nr;
info->mode = SYS_ENTER;
// Insert into ring buffer
bpf_ringbuf_submit(info, 0);
}
}
return 0;
}
SEC("tracepoint/raw_syscalls/sys_exit")
int detect_syscall_exit(struct trace_event_raw_sys_exit *ctx)
{
const char *key = "child_pid";
void *value = bpf_map_lookup_elem(&pid_hashmap, key);
pid_t pid, target_pid;
if (value)
{
pid = bpf_get_current_pid_tgid() & 0xffffffff;
target_pid = *(pid_t *)value;
if (pid == target_pid)
{
struct inner_syscall_info *info = bpf_ringbuf_reserve(&syscall_info_buffer, sizeof(struct inner_syscall_info), 0);
if (!info)
{
bpf_printk("bpf_ringbuf_reserve failed");
return 1;
}
info->mode = SYS_EXIT;
info->retval = ctx->ret;
bpf_ringbuf_submit(info, 0);
}
}
return 0;
}
char LICENSE[] SEC("license") = "GPL";編譯之前，我們不妨先構建一個測試程序，以便後續使用我們的工具對其進行追蹤分析。
#include<stdio.h>
int main(){
puts("tracer in action");
return 0;
}可以利用下面提供的 Makefile 來完成所有相關組件的編譯工作。
compile:
clang -O2 -g -Wall -I/usr/include -I/usr/include/bpf -o beetrace loader.c -lbpf
clang -O2 -g -target bpf -c controller.c -o controller.o整個代碼可以在以下的 GitHub 倉庫中找到：https://github.com/0xSh4dy/bee_tracer
參考鏈接：
-
https://ebpf.io/
-
https://github.com/libbpf/libbpf
原文：https://sh4dy.com/2024/08/03/beetracer/
本文由 Readfog 進行 AMP 轉碼，版權歸原作者所有。
來源：https://mp.weixin.qq.com/s/maKNeaoqoT4D0yX06wWSzw