1 // SPDX-License-Identifier: GPL-2.0
2 // Copyright (c) 2019 Facebook
3 #include <linux/sched.h>
4 #include <linux/ptrace.h>
9 #include "bpf_helpers.h"
11 #define FUNCTION_NAME_LEN 64
12 #define FILE_NAME_LEN 128
13 #define TASK_COMM_LEN 16
16 int PyThreadState_frame;
17 int PyThreadState_thread;
18 int PyFrameObject_back;
19 int PyFrameObject_code;
20 int PyFrameObject_lineno;
21 int PyCodeObject_filename;
22 int PyCodeObject_name;
28 uintptr_t current_state_addr;
29 uintptr_t tls_key_addr;
39 char name[FUNCTION_NAME_LEN];
40 char file[FILE_NAME_LEN];
46 char comm[TASK_COMM_LEN];
47 int32_t kernel_stack_id;
48 int32_t user_stack_id;
53 int32_t stack[STACK_MAX_LEN];
64 void* f_back; // PyFrameObject.f_back, previous frame
65 void* f_code; // PyFrameObject.f_code, pointer to PyCodeObject
66 void* co_filename; // PyCodeObject.co_filename
67 void* co_name; // PyCodeObject.co_name
70 static __always_inline void *get_thread_state(void *tls_base, PidData *pidData)
75 bpf_probe_read_user(&key, sizeof(key), (void*)(long)pidData->tls_key_addr);
76 bpf_probe_read_user(&thread_state, sizeof(thread_state),
77 tls_base + 0x310 + key * 0x10 + 0x08);
81 static __always_inline bool get_frame_data(void *frame_ptr, PidData *pidData,
82 FrameData *frame, Symbol *symbol)
84 // read data from PyFrameObject
85 bpf_probe_read_user(&frame->f_back,
86 sizeof(frame->f_back),
87 frame_ptr + pidData->offsets.PyFrameObject_back);
88 bpf_probe_read_user(&frame->f_code,
89 sizeof(frame->f_code),
90 frame_ptr + pidData->offsets.PyFrameObject_code);
92 // read data from PyCodeObject
95 bpf_probe_read_user(&frame->co_filename,
96 sizeof(frame->co_filename),
97 frame->f_code + pidData->offsets.PyCodeObject_filename);
98 bpf_probe_read_user(&frame->co_name,
99 sizeof(frame->co_name),
100 frame->f_code + pidData->offsets.PyCodeObject_name);
101 // read actual names into symbol
102 if (frame->co_filename)
103 bpf_probe_read_user_str(&symbol->file,
104 sizeof(symbol->file),
106 pidData->offsets.String_data);
108 bpf_probe_read_user_str(&symbol->name,
109 sizeof(symbol->name),
111 pidData->offsets.String_data);
116 __uint(type, BPF_MAP_TYPE_HASH);
117 __uint(max_entries, 1);
119 __type(value, PidData);
120 } pidmap SEC(".maps");
123 __uint(type, BPF_MAP_TYPE_HASH);
124 __uint(max_entries, 1);
126 __type(value, Event);
127 } eventmap SEC(".maps");
130 __uint(type, BPF_MAP_TYPE_HASH);
131 __uint(max_entries, 1);
134 } symbolmap SEC(".maps");
137 __uint(type, BPF_MAP_TYPE_ARRAY);
138 __uint(max_entries, 1);
140 __type(value, Stats);
141 } statsmap SEC(".maps");
144 __uint(type, BPF_MAP_TYPE_PERF_EVENT_ARRAY);
145 __uint(max_entries, 32);
146 __uint(key_size, sizeof(int));
147 __uint(value_size, sizeof(int));
148 } perfmap SEC(".maps");
151 __uint(type, BPF_MAP_TYPE_STACK_TRACE);
152 __uint(max_entries, 1000);
153 __uint(key_size, sizeof(int));
154 __uint(value_size, sizeof(long long) * 127);
155 } stackmap SEC(".maps");
157 static __always_inline int __on_event(struct pt_regs *ctx)
159 uint64_t pid_tgid = bpf_get_current_pid_tgid();
160 pid_t pid = (pid_t)(pid_tgid >> 32);
161 PidData* pidData = bpf_map_lookup_elem(&pidmap, &pid);
166 Event* event = bpf_map_lookup_elem(&eventmap, &zero);
172 event->tid = (pid_t)pid_tgid;
173 bpf_get_current_comm(&event->comm, sizeof(event->comm));
175 event->user_stack_id = bpf_get_stackid(ctx, &stackmap, BPF_F_USER_STACK);
176 event->kernel_stack_id = bpf_get_stackid(ctx, &stackmap, 0);
178 void* thread_state_current = (void*)0;
179 bpf_probe_read_user(&thread_state_current,
180 sizeof(thread_state_current),
181 (void*)(long)pidData->current_state_addr);
183 struct task_struct* task = (struct task_struct*)bpf_get_current_task();
184 void* tls_base = (void*)task;
186 void* thread_state = pidData->use_tls ? get_thread_state(tls_base, pidData)
187 : thread_state_current;
188 event->thread_current = thread_state == thread_state_current;
190 if (pidData->use_tls) {
191 uint64_t pthread_created;
192 uint64_t pthread_self;
193 bpf_probe_read_user(&pthread_self, sizeof(pthread_self),
196 bpf_probe_read_user(&pthread_created,
197 sizeof(pthread_created),
199 pidData->offsets.PyThreadState_thread);
200 event->pthread_match = pthread_created == pthread_self;
202 event->pthread_match = 1;
205 if (event->pthread_match || !pidData->use_tls) {
209 int cur_cpu = bpf_get_smp_processor_id();
211 bpf_probe_read_user(&frame_ptr,
214 pidData->offsets.PyThreadState_frame);
216 int32_t* symbol_counter = bpf_map_lookup_elem(&symbolmap, &sym);
217 if (symbol_counter == NULL)
220 #pragma clang loop unroll(disable)
222 #pragma clang loop unroll(full)
224 /* Unwind python stack */
225 for (int i = 0; i < STACK_MAX_LEN; ++i) {
226 if (frame_ptr && get_frame_data(frame_ptr, pidData, &frame, &sym)) {
227 int32_t new_symbol_id = *symbol_counter * 64 + cur_cpu;
228 int32_t *symbol_id = bpf_map_lookup_elem(&symbolmap, &sym);
230 bpf_map_update_elem(&symbolmap, &sym, &zero, 0);
231 symbol_id = bpf_map_lookup_elem(&symbolmap, &sym);
235 if (*symbol_id == new_symbol_id)
237 event->stack[i] = *symbol_id;
238 event->stack_len = i + 1;
239 frame_ptr = frame.f_back;
242 event->stack_complete = frame_ptr == NULL;
244 event->stack_complete = 1;
247 Stats* stats = bpf_map_lookup_elem(&statsmap, &zero);
252 bpf_perf_event_output(ctx, &perfmap, 0, event, offsetof(Event, metadata));
256 SEC("raw_tracepoint/kfree_skb")
257 int on_event(struct pt_regs* ctx)
260 ret |= __on_event(ctx);
261 ret |= __on_event(ctx);
262 ret |= __on_event(ctx);
263 ret |= __on_event(ctx);
264 ret |= __on_event(ctx);
268 char _license[] SEC("license") = "GPL";