#include <linux/stdarg.h>
#include <linux/kernel.h>
#include <linux/lockdep.h>
#include <linux/math.h>
#include <linux/printk.h>
#include <linux/sched/debug.h>
#include <linux/seq_file.h>
#include <linux/sprintf.h>
#include <linux/stacktrace.h>
#include <linux/string.h>
#include <trace/events/error_report.h>
#include <asm/kfence.h>
#include "kfence.h"
#ifndef ARCH_FUNC_PREFIX
#define ARCH_FUNC_PREFIX ""
#endif
__printf(2, 3)
static void seq_con_printf(struct seq_file *seq, const char *fmt, ...)
{
va_list args;
va_start(args, fmt);
if (seq)
seq_vprintf(seq, fmt, args);
else
vprintk(fmt, args);
va_end(args);
}
static int get_stack_skipnr(const unsigned long stack_entries[], int num_entries,
const enum kfence_error_type *type)
{
char buf[64];
int skipnr, fallback = 0;
if (type) {
switch (*type) {
case KFENCE_ERROR_UAF:
case KFENCE_ERROR_OOB:
case KFENCE_ERROR_INVALID:
return 0;
case KFENCE_ERROR_CORRUPTION:
case KFENCE_ERROR_INVALID_FREE:
break;
}
}
for (skipnr = 0; skipnr < num_entries; skipnr++) {
int len = scnprintf(buf, sizeof(buf), "%ps", (void *)stack_entries[skipnr]);
if (str_has_prefix(buf, ARCH_FUNC_PREFIX "kfence_") ||
str_has_prefix(buf, ARCH_FUNC_PREFIX "__kfence_") ||
str_has_prefix(buf, ARCH_FUNC_PREFIX "__kmem_cache_free") ||
!strncmp(buf, ARCH_FUNC_PREFIX "__slab_free", len)) {
fallback = skipnr + 1;
}
if (str_has_prefix(buf, ARCH_FUNC_PREFIX "kfree") ||
str_has_prefix(buf, ARCH_FUNC_PREFIX "kmem_cache_free") ||
str_has_prefix(buf, ARCH_FUNC_PREFIX "__kmalloc") ||
str_has_prefix(buf, ARCH_FUNC_PREFIX "kmem_cache_alloc"))
goto found;
}
if (fallback < num_entries)
return fallback;
found:
skipnr++;
return skipnr < num_entries ? skipnr : 0;
}
static void kfence_print_stack(struct seq_file *seq, const struct kfence_metadata *meta,
bool show_alloc)
{
const struct kfence_track *track = show_alloc ? &meta->alloc_track : &meta->free_track;
u64 ts_sec = track->ts_nsec;
unsigned long rem_nsec = do_div(ts_sec, NSEC_PER_SEC);
seq_con_printf(seq, "%s by task %d on cpu %d at %lu.%06lus:\n",
show_alloc ? "allocated" : "freed", track->pid,
track->cpu, (unsigned long)ts_sec, rem_nsec / 1000);
if (track->num_stack_entries) {
int i = get_stack_skipnr(track->stack_entries, track->num_stack_entries, NULL);
for (; i < track->num_stack_entries; i++)
seq_con_printf(seq, " %pS\n", (void *)track->stack_entries[i]);
} else {
seq_con_printf(seq, " no %s stack\n", show_alloc ? "allocation" : "deallocation");
}
}
void kfence_print_object(struct seq_file *seq, const struct kfence_metadata *meta)
{
const int size = abs(meta->size);
const unsigned long start = meta->addr;
const struct kmem_cache *const cache = meta->cache;
lockdep_assert_held(&meta->lock);
if (meta->state == KFENCE_OBJECT_UNUSED) {
seq_con_printf(seq, "kfence-#%td unused\n", meta - kfence_metadata);
return;
}
seq_con_printf(seq, "kfence-#%td: 0x%p-0x%p, size=%d, cache=%s\n\n",
meta - kfence_metadata, (void *)start, (void *)(start + size - 1),
size, (cache && cache->name) ? cache->name : "<destroyed>");
kfence_print_stack(seq, meta, true);
if (meta->state == KFENCE_OBJECT_FREED) {
seq_con_printf(seq, "\n");
kfence_print_stack(seq, meta, false);
}
}
static void print_diff_canary(unsigned long address, size_t bytes_to_show,
const struct kfence_metadata *meta)
{
const unsigned long show_until_addr = address + bytes_to_show;
const u8 *cur, *end;
end = (const u8 *)(address < meta->addr ? min(show_until_addr, meta->addr)
: min(show_until_addr, PAGE_ALIGN(address)));
pr_cont("[");
for (cur = (const u8 *)address; cur < end; cur++) {
if (*cur == KFENCE_CANARY_PATTERN_U8(cur))
pr_cont(" .");
else if (no_hash_pointers)
pr_cont(" 0x%02x", *cur);
else
pr_cont(" !");
}
pr_cont(" ]");
}
static const char *get_access_type(bool is_write)
{
return is_write ? "write" : "read";
}
void kfence_report_error(unsigned long address, bool is_write, struct pt_regs *regs,
const struct kfence_metadata *meta, enum kfence_error_type type)
{
unsigned long stack_entries[KFENCE_STACK_DEPTH] = { 0 };
const ptrdiff_t object_index = meta ? meta - kfence_metadata : -1;
int num_stack_entries;
int skipnr = 0;
if (regs) {
num_stack_entries = stack_trace_save_regs(regs, stack_entries, KFENCE_STACK_DEPTH, 0);
} else {
num_stack_entries = stack_trace_save(stack_entries, KFENCE_STACK_DEPTH, 1);
skipnr = get_stack_skipnr(stack_entries, num_stack_entries, &type);
}
if (WARN_ON(type != KFENCE_ERROR_INVALID && !meta))
return;
if (meta)
lockdep_assert_held(&meta->lock);
lockdep_off();
pr_err("==================================================================\n");
switch (type) {
case KFENCE_ERROR_OOB: {
const bool left_of_object = address < meta->addr;
pr_err("BUG: KFENCE: out-of-bounds %s in %pS\n\n", get_access_type(is_write),
(void *)stack_entries[skipnr]);
pr_err("Out-of-bounds %s at 0x%p (%luB %s of kfence-#%td):\n",
get_access_type(is_write), (void *)address,
left_of_object ? meta->addr - address : address - meta->addr,
left_of_object ? "left" : "right", object_index);
break;
}
case KFENCE_ERROR_UAF:
pr_err("BUG: KFENCE: use-after-free %s in %pS\n\n", get_access_type(is_write),
(void *)stack_entries[skipnr]);
pr_err("Use-after-free %s at 0x%p (in kfence-#%td):\n",
get_access_type(is_write), (void *)address, object_index);
break;
case KFENCE_ERROR_CORRUPTION:
pr_err("BUG: KFENCE: memory corruption in %pS\n\n", (void *)stack_entries[skipnr]);
pr_err("Corrupted memory at 0x%p ", (void *)address);
print_diff_canary(address, 16, meta);
pr_cont(" (in kfence-#%td):\n", object_index);
break;
case KFENCE_ERROR_INVALID:
pr_err("BUG: KFENCE: invalid %s in %pS\n\n", get_access_type(is_write),
(void *)stack_entries[skipnr]);
pr_err("Invalid %s at 0x%p:\n", get_access_type(is_write),
(void *)address);
break;
case KFENCE_ERROR_INVALID_FREE:
pr_err("BUG: KFENCE: invalid free in %pS\n\n", (void *)stack_entries[skipnr]);
pr_err("Invalid free of 0x%p (in kfence-#%td):\n", (void *)address,
object_index);
break;
}
stack_trace_print(stack_entries + skipnr, num_stack_entries - skipnr, 0);
if (meta) {
pr_err("\n");
kfence_print_object(NULL, meta);
}
pr_err("\n");
if (no_hash_pointers && regs)
show_regs(regs);
else
dump_stack_print_info(KERN_ERR);
trace_error_report_end(ERROR_DETECTOR_KFENCE, address);
pr_err("==================================================================\n");
lockdep_on();
check_panic_on_warn("KFENCE");
add_taint(TAINT_BAD_PAGE, LOCKDEP_STILL_OK);
}
#ifdef CONFIG_PRINTK
static void kfence_to_kp_stack(const struct kfence_track *track, void **kp_stack)
{
int i, j;
i = get_stack_skipnr(track->stack_entries, track->num_stack_entries, NULL);
for (j = 0; i < track->num_stack_entries && j < KS_ADDRS_COUNT; ++i, ++j)
kp_stack[j] = (void *)track->stack_entries[i];
if (j < KS_ADDRS_COUNT)
kp_stack[j] = NULL;
}
bool __kfence_obj_info(struct kmem_obj_info *kpp, void *object, struct slab *slab)
{
struct kfence_metadata *meta = addr_to_metadata((unsigned long)object);
unsigned long flags;
if (!meta)
return false;
kpp->kp_ptr = object;
if (WARN_ON(meta->state == KFENCE_OBJECT_UNUSED))
return true;
raw_spin_lock_irqsave(&meta->lock, flags);
kpp->kp_slab = slab;
kpp->kp_slab_cache = meta->cache;
kpp->kp_objp = (void *)meta->addr;
kfence_to_kp_stack(&meta->alloc_track, kpp->kp_stack);
if (meta->state == KFENCE_OBJECT_FREED)
kfence_to_kp_stack(&meta->free_track, kpp->kp_free_stack);
kpp->kp_ret = kpp->kp_stack[0];
raw_spin_unlock_irqrestore(&meta->lock, flags);
return true;
}
#endif