#include <linux/stop_machine.h>
#include <linux/clocksource.h>
#include <linux/sched/task.h>
#include <linux/kallsyms.h>
#include <linux/security.h>
#include <linux/seq_file.h>
#include <linux/tracefs.h>
#include <linux/hardirq.h>
#include <linux/kthread.h>
#include <linux/uaccess.h>
#include <linux/bsearch.h>
#include <linux/module.h>
#include <linux/ftrace.h>
#include <linux/sysctl.h>
#include <linux/slab.h>
#include <linux/ctype.h>
#include <linux/sort.h>
#include <linux/list.h>
#include <linux/hash.h>
#include <linux/rcupdate.h>
#include <linux/kprobes.h>
#include <trace/events/sched.h>
#include <asm/sections.h>
#include <asm/setup.h>
#include "ftrace_internal.h"
#include "trace_output.h"
#include "trace_stat.h"
#define FTRACE_NOCLEAR_FLAGS (FTRACE_FL_DISABLED | FTRACE_FL_TOUCHED | \
FTRACE_FL_MODIFIED)
#define FTRACE_INVALID_FUNCTION "__ftrace_invalid_address__"
#define FTRACE_WARN_ON(cond) \
({ \
int ___r = cond; \
if (WARN_ON(___r)) \
ftrace_kill(); \
___r; \
})
#define FTRACE_WARN_ON_ONCE(cond) \
({ \
int ___r = cond; \
if (WARN_ON_ONCE(___r)) \
ftrace_kill(); \
___r; \
})
#define FTRACE_HASH_DEFAULT_BITS 10
#define FTRACE_HASH_MAX_BITS 12
#ifdef CONFIG_DYNAMIC_FTRACE
#define INIT_OPS_HASH(opsname) \
.func_hash = &opsname.local_hash, \
.local_hash.regex_lock = __MUTEX_INITIALIZER(opsname.local_hash.regex_lock),
#else
#define INIT_OPS_HASH(opsname)
#endif
enum {
FTRACE_MODIFY_ENABLE_FL = (1 << 0),
FTRACE_MODIFY_MAY_SLEEP_FL = (1 << 1),
};
struct ftrace_ops ftrace_list_end __read_mostly = {
.func = ftrace_stub,
.flags = FTRACE_OPS_FL_STUB,
INIT_OPS_HASH(ftrace_list_end)
};
int ftrace_enabled __read_mostly;
static int __maybe_unused last_ftrace_enabled;
struct ftrace_ops *function_trace_op __read_mostly = &ftrace_list_end;
static struct ftrace_ops *set_function_trace_op;
static bool ftrace_pids_enabled(struct ftrace_ops *ops)
{
struct trace_array *tr;
if (!(ops->flags & FTRACE_OPS_FL_PID) || !ops->private)
return false;
tr = ops->private;
return tr->function_pids != NULL || tr->function_no_pids != NULL;
}
static void ftrace_update_trampoline(struct ftrace_ops *ops);
static int ftrace_disabled __read_mostly;
DEFINE_MUTEX(ftrace_lock);
struct ftrace_ops __rcu *ftrace_ops_list __read_mostly = &ftrace_list_end;
ftrace_func_t ftrace_trace_function __read_mostly = ftrace_stub;
struct ftrace_ops global_ops;
void ftrace_ops_list_func(unsigned long ip, unsigned long parent_ip,
struct ftrace_ops *op, struct ftrace_regs *fregs);
#ifdef CONFIG_DYNAMIC_FTRACE_WITH_CALL_OPS
const struct ftrace_ops ftrace_list_ops = {
.func = ftrace_ops_list_func,
.flags = FTRACE_OPS_FL_STUB,
};
static void ftrace_ops_nop_func(unsigned long ip, unsigned long parent_ip,
struct ftrace_ops *op,
struct ftrace_regs *fregs)
{
}
const struct ftrace_ops ftrace_nop_ops = {
.func = ftrace_ops_nop_func,
.flags = FTRACE_OPS_FL_STUB,
};
#endif
static inline void ftrace_ops_init(struct ftrace_ops *ops)
{
#ifdef CONFIG_DYNAMIC_FTRACE
if (!(ops->flags & FTRACE_OPS_FL_INITIALIZED)) {
mutex_init(&ops->local_hash.regex_lock);
ops->func_hash = &ops->local_hash;
ops->flags |= FTRACE_OPS_FL_INITIALIZED;
}
#endif
}
static void ftrace_pid_func(unsigned long ip, unsigned long parent_ip,
struct ftrace_ops *op, struct ftrace_regs *fregs)
{
struct trace_array *tr = op->private;
int pid;
if (tr) {
pid = this_cpu_read(tr->array_buffer.data->ftrace_ignore_pid);
if (pid == FTRACE_PID_IGNORE)
return;
if (pid != FTRACE_PID_TRACE &&
pid != current->pid)
return;
}
op->saved_func(ip, parent_ip, op, fregs);
}
static void ftrace_sync_ipi(void *data)
{
smp_rmb();
}
static ftrace_func_t ftrace_ops_get_list_func(struct ftrace_ops *ops)
{
if (ops->flags & (FTRACE_OPS_FL_DYNAMIC | FTRACE_OPS_FL_RCU) ||
FTRACE_FORCE_LIST_FUNC)
return ftrace_ops_list_func;
return ftrace_ops_get_func(ops);
}
static void update_ftrace_function(void)
{
ftrace_func_t func;
set_function_trace_op = rcu_dereference_protected(ftrace_ops_list,
lockdep_is_held(&ftrace_lock));
if (set_function_trace_op == &ftrace_list_end) {
func = ftrace_stub;
} else if (rcu_dereference_protected(ftrace_ops_list->next,
lockdep_is_held(&ftrace_lock)) == &ftrace_list_end) {
func = ftrace_ops_get_list_func(ftrace_ops_list);
} else {
set_function_trace_op = &ftrace_list_end;
func = ftrace_ops_list_func;
}
update_function_graph_func();
if (ftrace_trace_function == func)
return;
if (func == ftrace_ops_list_func) {
ftrace_trace_function = func;
return;
}
#ifndef CONFIG_DYNAMIC_FTRACE
ftrace_trace_function = ftrace_ops_list_func;
synchronize_rcu_tasks_rude();
function_trace_op = set_function_trace_op;
smp_wmb();
smp_call_function(ftrace_sync_ipi, NULL, 1);
#endif /* !CONFIG_DYNAMIC_FTRACE */
ftrace_trace_function = func;
}
static void add_ftrace_ops(struct ftrace_ops __rcu **list,
struct ftrace_ops *ops)
{
rcu_assign_pointer(ops->next, *list);
rcu_assign_pointer(*list, ops);
}
static int remove_ftrace_ops(struct ftrace_ops __rcu **list,
struct ftrace_ops *ops)
{
struct ftrace_ops **p;
if (rcu_dereference_protected(*list,
lockdep_is_held(&ftrace_lock)) == ops &&
rcu_dereference_protected(ops->next,
lockdep_is_held(&ftrace_lock)) == &ftrace_list_end) {
*list = &ftrace_list_end;
return 0;
}
for (p = list; *p != &ftrace_list_end; p = &(*p)->next)
if (*p == ops)
break;
if (*p != ops)
return -1;
*p = (*p)->next;
return 0;
}
static void ftrace_update_trampoline(struct ftrace_ops *ops);
int __register_ftrace_function(struct ftrace_ops *ops)
{
if (ops->flags & FTRACE_OPS_FL_DELETED)
return -EINVAL;
if (WARN_ON(ops->flags & FTRACE_OPS_FL_ENABLED))
return -EBUSY;
#ifndef CONFIG_DYNAMIC_FTRACE_WITH_REGS
if (ops->flags & FTRACE_OPS_FL_SAVE_REGS &&
!(ops->flags & FTRACE_OPS_FL_SAVE_REGS_IF_SUPPORTED))
return -EINVAL;
if (ops->flags & FTRACE_OPS_FL_SAVE_REGS_IF_SUPPORTED)
ops->flags |= FTRACE_OPS_FL_SAVE_REGS;
#endif
if (!ftrace_enabled && (ops->flags & FTRACE_OPS_FL_PERMANENT))
return -EBUSY;
if (!is_kernel_core_data((unsigned long)ops))
ops->flags |= FTRACE_OPS_FL_DYNAMIC;
add_ftrace_ops(&ftrace_ops_list, ops);
ops->saved_func = ops->func;
if (ftrace_pids_enabled(ops))
ops->func = ftrace_pid_func;
ftrace_update_trampoline(ops);
if (ftrace_enabled)
update_ftrace_function();
return 0;
}
int __unregister_ftrace_function(struct ftrace_ops *ops)
{
int ret;
if (WARN_ON(!(ops->flags & FTRACE_OPS_FL_ENABLED)))
return -EBUSY;
ret = remove_ftrace_ops(&ftrace_ops_list, ops);
if (ret < 0)
return ret;
if (ftrace_enabled)
update_ftrace_function();
ops->func = ops->saved_func;
return 0;
}
static void ftrace_update_pid_func(void)
{
struct ftrace_ops *op;
if (ftrace_trace_function == ftrace_stub)
return;
do_for_each_ftrace_op(op, ftrace_ops_list) {
if (op->flags & FTRACE_OPS_FL_PID) {
op->func = ftrace_pids_enabled(op) ?
ftrace_pid_func : op->saved_func;
ftrace_update_trampoline(op);
}
} while_for_each_ftrace_op(op);
update_ftrace_function();
}
#ifdef CONFIG_FUNCTION_PROFILER
struct ftrace_profile {
struct hlist_node node;
unsigned long ip;
unsigned long counter;
#ifdef CONFIG_FUNCTION_GRAPH_TRACER
unsigned long long time;
unsigned long long time_squared;
#endif
};
struct ftrace_profile_page {
struct ftrace_profile_page *next;
unsigned long index;
struct ftrace_profile records[];
};
struct ftrace_profile_stat {
atomic_t disabled;
struct hlist_head *hash;
struct ftrace_profile_page *pages;
struct ftrace_profile_page *start;
struct tracer_stat stat;
};
#define PROFILE_RECORDS_SIZE \
(PAGE_SIZE - offsetof(struct ftrace_profile_page, records))
#define PROFILES_PER_PAGE \
(PROFILE_RECORDS_SIZE / sizeof(struct ftrace_profile))
static int ftrace_profile_enabled __read_mostly;
static DEFINE_MUTEX(ftrace_profile_lock);
static DEFINE_PER_CPU(struct ftrace_profile_stat, ftrace_profile_stats);
#define FTRACE_PROFILE_HASH_BITS 10
#define FTRACE_PROFILE_HASH_SIZE (1 << FTRACE_PROFILE_HASH_BITS)
static void *
function_stat_next(void *v, int idx)
{
struct ftrace_profile *rec = v;
struct ftrace_profile_page *pg;
pg = (struct ftrace_profile_page *)((unsigned long)rec & PAGE_MASK);
again:
if (idx != 0)
rec++;
if ((void *)rec >= (void *)&pg->records[pg->index]) {
pg = pg->next;
if (!pg)
return NULL;
rec = &pg->records[0];
if (!rec->counter)
goto again;
}
return rec;
}
static void *function_stat_start(struct tracer_stat *trace)
{
struct ftrace_profile_stat *stat =
container_of(trace, struct ftrace_profile_stat, stat);
if (!stat || !stat->start)
return NULL;
return function_stat_next(&stat->start->records[0], 0);
}
#ifdef CONFIG_FUNCTION_GRAPH_TRACER
static int function_stat_cmp(const void *p1, const void *p2)
{
const struct ftrace_profile *a = p1;
const struct ftrace_profile *b = p2;
if (a->time < b->time)
return -1;
if (a->time > b->time)
return 1;
else
return 0;
}
#else
static int function_stat_cmp(const void *p1, const void *p2)
{
const struct ftrace_profile *a = p1;
const struct ftrace_profile *b = p2;
if (a->counter < b->counter)
return -1;
if (a->counter > b->counter)
return 1;
else
return 0;
}
#endif
static int function_stat_headers(struct seq_file *m)
{
#ifdef CONFIG_FUNCTION_GRAPH_TRACER
seq_puts(m, " Function "
"Hit Time Avg s^2\n"
" -------- "
"--- ---- --- ---\n");
#else
seq_puts(m, " Function Hit\n"
" -------- ---\n");
#endif
return 0;
}
static int function_stat_show(struct seq_file *m, void *v)
{
struct ftrace_profile *rec = v;
char str[KSYM_SYMBOL_LEN];
int ret = 0;
#ifdef CONFIG_FUNCTION_GRAPH_TRACER
static struct trace_seq s;
unsigned long long avg;
unsigned long long stddev;
#endif
mutex_lock(&ftrace_profile_lock);
if (unlikely(rec->counter == 0)) {
ret = -EBUSY;
goto out;
}
#ifdef CONFIG_FUNCTION_GRAPH_TRACER
avg = div64_ul(rec->time, rec->counter);
if (tracing_thresh && (avg < tracing_thresh))
goto out;
#endif
kallsyms_lookup(rec->ip, NULL, NULL, NULL, str);
seq_printf(m, " %-30.30s %10lu", str, rec->counter);
#ifdef CONFIG_FUNCTION_GRAPH_TRACER
seq_puts(m, " ");
if (rec->counter <= 1)
stddev = 0;
else {
stddev = rec->counter * rec->time_squared -
rec->time * rec->time;
stddev = div64_ul(stddev,
rec->counter * (rec->counter - 1) * 1000);
}
trace_seq_init(&s);
trace_print_graph_duration(rec->time, &s);
trace_seq_puts(&s, " ");
trace_print_graph_duration(avg, &s);
trace_seq_puts(&s, " ");
trace_print_graph_duration(stddev, &s);
trace_print_seq(m, &s);
#endif
seq_putc(m, '\n');
out:
mutex_unlock(&ftrace_profile_lock);
return ret;
}
static void ftrace_profile_reset(struct ftrace_profile_stat *stat)
{
struct ftrace_profile_page *pg;
pg = stat->pages = stat->start;
while (pg) {
memset(pg->records, 0, PROFILE_RECORDS_SIZE);
pg->index = 0;
pg = pg->next;
}
memset(stat->hash, 0,
FTRACE_PROFILE_HASH_SIZE * sizeof(struct hlist_head));
}
static int ftrace_profile_pages_init(struct ftrace_profile_stat *stat)
{
struct ftrace_profile_page *pg;
int functions;
int pages;
int i;
if (stat->pages)
return 0;
stat->pages = (void *)get_zeroed_page(GFP_KERNEL);
if (!stat->pages)
return -ENOMEM;
#ifdef CONFIG_DYNAMIC_FTRACE
functions = ftrace_update_tot_cnt;
#else
functions = 20000;
#endif
pg = stat->start = stat->pages;
pages = DIV_ROUND_UP(functions, PROFILES_PER_PAGE);
for (i = 1; i < pages; i++) {
pg->next = (void *)get_zeroed_page(GFP_KERNEL);
if (!pg->next)
goto out_free;
pg = pg->next;
}
return 0;
out_free:
pg = stat->start;
while (pg) {
unsigned long tmp = (unsigned long)pg;
pg = pg->next;
free_page(tmp);
}
stat->pages = NULL;
stat->start = NULL;
return -ENOMEM;
}
static int ftrace_profile_init_cpu(int cpu)
{
struct ftrace_profile_stat *stat;
int size;
stat = &per_cpu(ftrace_profile_stats, cpu);
if (stat->hash) {
ftrace_profile_reset(stat);
return 0;
}
size = FTRACE_PROFILE_HASH_SIZE;
stat->hash = kcalloc(size, sizeof(struct hlist_head), GFP_KERNEL);
if (!stat->hash)
return -ENOMEM;
if (ftrace_profile_pages_init(stat) < 0) {
kfree(stat->hash);
stat->hash = NULL;
return -ENOMEM;
}
return 0;
}
static int ftrace_profile_init(void)
{
int cpu;
int ret = 0;
for_each_possible_cpu(cpu) {
ret = ftrace_profile_init_cpu(cpu);
if (ret)
break;
}
return ret;
}
static struct ftrace_profile *
ftrace_find_profiled_func(struct ftrace_profile_stat *stat, unsigned long ip)
{
struct ftrace_profile *rec;
struct hlist_head *hhd;
unsigned long key;
key = hash_long(ip, FTRACE_PROFILE_HASH_BITS);
hhd = &stat->hash[key];
if (hlist_empty(hhd))
return NULL;
hlist_for_each_entry_rcu_notrace(rec, hhd, node) {
if (rec->ip == ip)
return rec;
}
return NULL;
}
static void ftrace_add_profile(struct ftrace_profile_stat *stat,
struct ftrace_profile *rec)
{
unsigned long key;
key = hash_long(rec->ip, FTRACE_PROFILE_HASH_BITS);
hlist_add_head_rcu(&rec->node, &stat->hash[key]);
}
static struct ftrace_profile *
ftrace_profile_alloc(struct ftrace_profile_stat *stat, unsigned long ip)
{
struct ftrace_profile *rec = NULL;
if (atomic_inc_return(&stat->disabled) != 1)
goto out;
rec = ftrace_find_profiled_func(stat, ip);
if (rec)
goto out;
if (stat->pages->index == PROFILES_PER_PAGE) {
if (!stat->pages->next)
goto out;
stat->pages = stat->pages->next;
}
rec = &stat->pages->records[stat->pages->index++];
rec->ip = ip;
ftrace_add_profile(stat, rec);
out:
atomic_dec(&stat->disabled);
return rec;
}
static void
function_profile_call(unsigned long ip, unsigned long parent_ip,
struct ftrace_ops *ops, struct ftrace_regs *fregs)
{
struct ftrace_profile_stat *stat;
struct ftrace_profile *rec;
unsigned long flags;
if (!ftrace_profile_enabled)
return;
local_irq_save(flags);
stat = this_cpu_ptr(&ftrace_profile_stats);
if (!stat->hash || !ftrace_profile_enabled)
goto out;
rec = ftrace_find_profiled_func(stat, ip);
if (!rec) {
rec = ftrace_profile_alloc(stat, ip);
if (!rec)
goto out;
}
rec->counter++;
out:
local_irq_restore(flags);
}
#ifdef CONFIG_FUNCTION_GRAPH_TRACER
static bool fgraph_graph_time = true;
void ftrace_graph_graph_time_control(bool enable)
{
fgraph_graph_time = enable;
}
static int profile_graph_entry(struct ftrace_graph_ent *trace)
{
struct ftrace_ret_stack *ret_stack;
function_profile_call(trace->func, 0, NULL, NULL);
if (!current->ret_stack)
return 0;
ret_stack = ftrace_graph_get_ret_stack(current, 0);
if (ret_stack)
ret_stack->subtime = 0;
return 1;
}
static void profile_graph_return(struct ftrace_graph_ret *trace)
{
struct ftrace_ret_stack *ret_stack;
struct ftrace_profile_stat *stat;
unsigned long long calltime;
struct ftrace_profile *rec;
unsigned long flags;
local_irq_save(flags);
stat = this_cpu_ptr(&ftrace_profile_stats);
if (!stat->hash || !ftrace_profile_enabled)
goto out;
if (!trace->calltime)
goto out;
calltime = trace->rettime - trace->calltime;
if (!fgraph_graph_time) {
ret_stack = ftrace_graph_get_ret_stack(current, 1);
if (ret_stack)
ret_stack->subtime += calltime;
ret_stack = ftrace_graph_get_ret_stack(current, 0);
if (ret_stack && ret_stack->subtime < calltime)
calltime -= ret_stack->subtime;
else
calltime = 0;
}
rec = ftrace_find_profiled_func(stat, trace->func);
if (rec) {
rec->time += calltime;
rec->time_squared += calltime * calltime;
}
out:
local_irq_restore(flags);
}
static struct fgraph_ops fprofiler_ops = {
.entryfunc = &profile_graph_entry,
.retfunc = &profile_graph_return,
};
static int register_ftrace_profiler(void)
{
return register_ftrace_graph(&fprofiler_ops);
}
static void unregister_ftrace_profiler(void)
{
unregister_ftrace_graph(&fprofiler_ops);
}
#else
static struct ftrace_ops ftrace_profile_ops __read_mostly = {
.func = function_profile_call,
.flags = FTRACE_OPS_FL_INITIALIZED,
INIT_OPS_HASH(ftrace_profile_ops)
};
static int register_ftrace_profiler(void)
{
return register_ftrace_function(&ftrace_profile_ops);
}
static void unregister_ftrace_profiler(void)
{
unregister_ftrace_function(&ftrace_profile_ops);
}
#endif /* CONFIG_FUNCTION_GRAPH_TRACER */
static ssize_t
ftrace_profile_write(struct file *filp, const char __user *ubuf,
size_t cnt, loff_t *ppos)
{
unsigned long val;
int ret;
ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
if (ret)
return ret;
val = !!val;
mutex_lock(&ftrace_profile_lock);
if (ftrace_profile_enabled ^ val) {
if (val) {
ret = ftrace_profile_init();
if (ret < 0) {
cnt = ret;
goto out;
}
ret = register_ftrace_profiler();
if (ret < 0) {
cnt = ret;
goto out;
}
ftrace_profile_enabled = 1;
} else {
ftrace_profile_enabled = 0;
unregister_ftrace_profiler();
}
}
out:
mutex_unlock(&ftrace_profile_lock);
*ppos += cnt;
return cnt;
}
static ssize_t
ftrace_profile_read(struct file *filp, char __user *ubuf,
size_t cnt, loff_t *ppos)
{
char buf[64];
int r;
r = sprintf(buf, "%u\n", ftrace_profile_enabled);
return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
}
static const struct file_operations ftrace_profile_fops = {
.open = tracing_open_generic,
.read = ftrace_profile_read,
.write = ftrace_profile_write,
.llseek = default_llseek,
};
static struct tracer_stat function_stats __initdata = {
.name = "functions",
.stat_start = function_stat_start,
.stat_next = function_stat_next,
.stat_cmp = function_stat_cmp,
.stat_headers = function_stat_headers,
.stat_show = function_stat_show
};
static __init void ftrace_profile_tracefs(struct dentry *d_tracer)
{
struct ftrace_profile_stat *stat;
char *name;
int ret;
int cpu;
for_each_possible_cpu(cpu) {
stat = &per_cpu(ftrace_profile_stats, cpu);
name = kasprintf(GFP_KERNEL, "function%d", cpu);
if (!name) {
WARN(1,
"Could not allocate stat file for cpu %d\n",
cpu);
return;
}
stat->stat = function_stats;
stat->stat.name = name;
ret = register_stat_tracer(&stat->stat);
if (ret) {
WARN(1,
"Could not register function stat for cpu %d\n",
cpu);
kfree(name);
return;
}
}
trace_create_file("function_profile_enabled",
TRACE_MODE_WRITE, d_tracer, NULL,
&ftrace_profile_fops);
}
#else /* CONFIG_FUNCTION_PROFILER */
static __init void ftrace_profile_tracefs(struct dentry *d_tracer)
{
}
#endif /* CONFIG_FUNCTION_PROFILER */
#ifdef CONFIG_DYNAMIC_FTRACE
static struct ftrace_ops *removed_ops;
static bool update_all_ops;
#ifndef CONFIG_FTRACE_MCOUNT_RECORD
# error Dynamic ftrace depends on MCOUNT_RECORD
#endif
struct ftrace_func_probe {
struct ftrace_probe_ops *probe_ops;
struct ftrace_ops ops;
struct trace_array *tr;
struct list_head list;
void *data;
int ref;
};
static const struct hlist_head empty_buckets[1];
static const struct ftrace_hash empty_hash = {
.buckets = (struct hlist_head *)empty_buckets,
};
#define EMPTY_HASH ((struct ftrace_hash *)&empty_hash)
struct ftrace_ops global_ops = {
.func = ftrace_stub,
.local_hash.notrace_hash = EMPTY_HASH,
.local_hash.filter_hash = EMPTY_HASH,
INIT_OPS_HASH(global_ops)
.flags = FTRACE_OPS_FL_INITIALIZED |
FTRACE_OPS_FL_PID,
};
struct ftrace_ops *ftrace_ops_trampoline(unsigned long addr)
{
struct ftrace_ops *op = NULL;
preempt_disable_notrace();
do_for_each_ftrace_op(op, ftrace_ops_list) {
if (op->trampoline && op->trampoline_size)
if (addr >= op->trampoline &&
addr < op->trampoline + op->trampoline_size) {
preempt_enable_notrace();
return op;
}
} while_for_each_ftrace_op(op);
preempt_enable_notrace();
return NULL;
}
bool is_ftrace_trampoline(unsigned long addr)
{
return ftrace_ops_trampoline(addr) != NULL;
}
struct ftrace_page {
struct ftrace_page *next;
struct dyn_ftrace *records;
int index;
int order;
};
#define ENTRY_SIZE sizeof(struct dyn_ftrace)
#define ENTRIES_PER_PAGE (PAGE_SIZE / ENTRY_SIZE)
static struct ftrace_page *ftrace_pages_start;
static struct ftrace_page *ftrace_pages;
static __always_inline unsigned long
ftrace_hash_key(struct ftrace_hash *hash, unsigned long ip)
{
if (hash->size_bits > 0)
return hash_long(ip, hash->size_bits);
return 0;
}
static __always_inline struct ftrace_func_entry *
__ftrace_lookup_ip(struct ftrace_hash *hash, unsigned long ip)
{
unsigned long key;
struct ftrace_func_entry *entry;
struct hlist_head *hhd;
key = ftrace_hash_key(hash, ip);
hhd = &hash->buckets[key];
hlist_for_each_entry_rcu_notrace(entry, hhd, hlist) {
if (entry->ip == ip)
return entry;
}
return NULL;
}
struct ftrace_func_entry *
ftrace_lookup_ip(struct ftrace_hash *hash, unsigned long ip)
{
if (ftrace_hash_empty(hash))
return NULL;
return __ftrace_lookup_ip(hash, ip);
}
static void __add_hash_entry(struct ftrace_hash *hash,
struct ftrace_func_entry *entry)
{
struct hlist_head *hhd;
unsigned long key;
key = ftrace_hash_key(hash, entry->ip);
hhd = &hash->buckets[key];
hlist_add_head(&entry->hlist, hhd);
hash->count++;
}
static int add_hash_entry(struct ftrace_hash *hash, unsigned long ip)
{
struct ftrace_func_entry *entry;
entry = kmalloc(sizeof(*entry), GFP_KERNEL);
if (!entry)
return -ENOMEM;
entry->ip = ip;
__add_hash_entry(hash, entry);
return 0;
}
static void
free_hash_entry(struct ftrace_hash *hash,
struct ftrace_func_entry *entry)
{
hlist_del(&entry->hlist);
kfree(entry);
hash->count--;
}
static void
remove_hash_entry(struct ftrace_hash *hash,
struct ftrace_func_entry *entry)
{
hlist_del_rcu(&entry->hlist);
hash->count--;
}
static void ftrace_hash_clear(struct ftrace_hash *hash)
{
struct hlist_head *hhd;
struct hlist_node *tn;
struct ftrace_func_entry *entry;
int size = 1 << hash->size_bits;
int i;
if (!hash->count)
return;
for (i = 0; i < size; i++) {
hhd = &hash->buckets[i];
hlist_for_each_entry_safe(entry, tn, hhd, hlist)
free_hash_entry(hash, entry);
}
FTRACE_WARN_ON(hash->count);
}
static void free_ftrace_mod(struct ftrace_mod_load *ftrace_mod)
{
list_del(&ftrace_mod->list);
kfree(ftrace_mod->module);
kfree(ftrace_mod->func);
kfree(ftrace_mod);
}
static void clear_ftrace_mod_list(struct list_head *head)
{
struct ftrace_mod_load *p, *n;
if (!head)
return;
mutex_lock(&ftrace_lock);
list_for_each_entry_safe(p, n, head, list)
free_ftrace_mod(p);
mutex_unlock(&ftrace_lock);
}
static void free_ftrace_hash(struct ftrace_hash *hash)
{
if (!hash || hash == EMPTY_HASH)
return;
ftrace_hash_clear(hash);
kfree(hash->buckets);
kfree(hash);
}
static void __free_ftrace_hash_rcu(struct rcu_head *rcu)
{
struct ftrace_hash *hash;
hash = container_of(rcu, struct ftrace_hash, rcu);
free_ftrace_hash(hash);
}
static void free_ftrace_hash_rcu(struct ftrace_hash *hash)
{
if (!hash || hash == EMPTY_HASH)
return;
call_rcu(&hash->rcu, __free_ftrace_hash_rcu);
}
void ftrace_free_filter(struct ftrace_ops *ops)
{
ftrace_ops_init(ops);
free_ftrace_hash(ops->func_hash->filter_hash);
free_ftrace_hash(ops->func_hash->notrace_hash);
}
EXPORT_SYMBOL_GPL(ftrace_free_filter);
static struct ftrace_hash *alloc_ftrace_hash(int size_bits)
{
struct ftrace_hash *hash;
int size;
hash = kzalloc(sizeof(*hash), GFP_KERNEL);
if (!hash)
return NULL;
size = 1 << size_bits;
hash->buckets = kcalloc(size, sizeof(*hash->buckets), GFP_KERNEL);
if (!hash->buckets) {
kfree(hash);
return NULL;
}
hash->size_bits = size_bits;
return hash;
}
static int ftrace_add_mod(struct trace_array *tr,
const char *func, const char *module,
int enable)
{
struct ftrace_mod_load *ftrace_mod;
struct list_head *mod_head = enable ? &tr->mod_trace : &tr->mod_notrace;
ftrace_mod = kzalloc(sizeof(*ftrace_mod), GFP_KERNEL);
if (!ftrace_mod)
return -ENOMEM;
INIT_LIST_HEAD(&ftrace_mod->list);
ftrace_mod->func = kstrdup(func, GFP_KERNEL);
ftrace_mod->module = kstrdup(module, GFP_KERNEL);
ftrace_mod->enable = enable;
if (!ftrace_mod->func || !ftrace_mod->module)
goto out_free;
list_add(&ftrace_mod->list, mod_head);
return 0;
out_free:
free_ftrace_mod(ftrace_mod);
return -ENOMEM;
}
static struct ftrace_hash *
alloc_and_copy_ftrace_hash(int size_bits, struct ftrace_hash *hash)
{
struct ftrace_func_entry *entry;
struct ftrace_hash *new_hash;
int size;
int ret;
int i;
new_hash = alloc_ftrace_hash(size_bits);
if (!new_hash)
return NULL;
if (hash)
new_hash->flags = hash->flags;
if (ftrace_hash_empty(hash))
return new_hash;
size = 1 << hash->size_bits;
for (i = 0; i < size; i++) {
hlist_for_each_entry(entry, &hash->buckets[i], hlist) {
ret = add_hash_entry(new_hash, entry->ip);
if (ret < 0)
goto free_hash;
}
}
FTRACE_WARN_ON(new_hash->count != hash->count);
return new_hash;
free_hash:
free_ftrace_hash(new_hash);
return NULL;
}
static void
ftrace_hash_rec_disable_modify(struct ftrace_ops *ops, int filter_hash);
static void
ftrace_hash_rec_enable_modify(struct ftrace_ops *ops, int filter_hash);
static int ftrace_hash_ipmodify_update(struct ftrace_ops *ops,
struct ftrace_hash *new_hash);
static struct ftrace_hash *dup_hash(struct ftrace_hash *src, int size)
{
struct ftrace_func_entry *entry;
struct ftrace_hash *new_hash;
struct hlist_head *hhd;
struct hlist_node *tn;
int bits = 0;
int i;
bits = fls(size / 2);
if (bits > FTRACE_HASH_MAX_BITS)
bits = FTRACE_HASH_MAX_BITS;
new_hash = alloc_ftrace_hash(bits);
if (!new_hash)
return NULL;
new_hash->flags = src->flags;
size = 1 << src->size_bits;
for (i = 0; i < size; i++) {
hhd = &src->buckets[i];
hlist_for_each_entry_safe(entry, tn, hhd, hlist) {
remove_hash_entry(src, entry);
__add_hash_entry(new_hash, entry);
}
}
return new_hash;
}
static struct ftrace_hash *
__ftrace_hash_move(struct ftrace_hash *src)
{
int size = src->count;
if (ftrace_hash_empty(src))
return EMPTY_HASH;
return dup_hash(src, size);
}
static int
ftrace_hash_move(struct ftrace_ops *ops, int enable,
struct ftrace_hash **dst, struct ftrace_hash *src)
{
struct ftrace_hash *new_hash;
int ret;
if (ops->flags & FTRACE_OPS_FL_IPMODIFY && !enable)
return -EINVAL;
new_hash = __ftrace_hash_move(src);
if (!new_hash)
return -ENOMEM;
if (enable) {
ret = ftrace_hash_ipmodify_update(ops, new_hash);
if (ret < 0) {
free_ftrace_hash(new_hash);
return ret;
}
}
ftrace_hash_rec_disable_modify(ops, enable);
rcu_assign_pointer(*dst, new_hash);
ftrace_hash_rec_enable_modify(ops, enable);
return 0;
}
static bool hash_contains_ip(unsigned long ip,
struct ftrace_ops_hash *hash)
{
return (ftrace_hash_empty(hash->filter_hash) ||
__ftrace_lookup_ip(hash->filter_hash, ip)) &&
(ftrace_hash_empty(hash->notrace_hash) ||
!__ftrace_lookup_ip(hash->notrace_hash, ip));
}
int
ftrace_ops_test(struct ftrace_ops *ops, unsigned long ip, void *regs)
{
struct ftrace_ops_hash hash;
int ret;
#ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS
if (regs == NULL && (ops->flags & FTRACE_OPS_FL_SAVE_REGS))
return 0;
#endif
rcu_assign_pointer(hash.filter_hash, ops->func_hash->filter_hash);
rcu_assign_pointer(hash.notrace_hash, ops->func_hash->notrace_hash);
if (hash_contains_ip(ip, &hash))
ret = 1;
else
ret = 0;
return ret;
}
#define do_for_each_ftrace_rec(pg, rec) \
for (pg = ftrace_pages_start; pg; pg = pg->next) { \
int _____i; \
for (_____i = 0; _____i < pg->index; _____i++) { \
rec = &pg->records[_____i];
#define while_for_each_ftrace_rec() \
} \
}
static int ftrace_cmp_recs(const void *a, const void *b)
{
const struct dyn_ftrace *key = a;
const struct dyn_ftrace *rec = b;
if (key->flags < rec->ip)
return -1;
if (key->ip >= rec->ip + MCOUNT_INSN_SIZE)
return 1;
return 0;
}
static struct dyn_ftrace *lookup_rec(unsigned long start, unsigned long end)
{
struct ftrace_page *pg;
struct dyn_ftrace *rec = NULL;
struct dyn_ftrace key;
key.ip = start;
key.flags = end;
for (pg = ftrace_pages_start; pg; pg = pg->next) {
if (pg->index == 0 ||
end < pg->records[0].ip ||
start >= (pg->records[pg->index - 1].ip + MCOUNT_INSN_SIZE))
continue;
rec = bsearch(&key, pg->records, pg->index,
sizeof(struct dyn_ftrace),
ftrace_cmp_recs);
if (rec)
break;
}
return rec;
}
unsigned long ftrace_location_range(unsigned long start, unsigned long end)
{
struct dyn_ftrace *rec;
rec = lookup_rec(start, end);
if (rec)
return rec->ip;
return 0;
}
unsigned long ftrace_location(unsigned long ip)
{
struct dyn_ftrace *rec;
unsigned long offset;
unsigned long size;
rec = lookup_rec(ip, ip);
if (!rec) {
if (!kallsyms_lookup_size_offset(ip, &size, &offset))
goto out;
if (!offset)
rec = lookup_rec(ip, ip + size - 1);
}
if (rec)
return rec->ip;
out:
return 0;
}
int ftrace_text_reserved(const void *start, const void *end)
{
unsigned long ret;
ret = ftrace_location_range((unsigned long)start,
(unsigned long)end);
return (int)!!ret;
}
static bool test_rec_ops_needs_regs(struct dyn_ftrace *rec)
{
struct ftrace_ops *ops;
bool keep_regs = false;
for (ops = ftrace_ops_list;
ops != &ftrace_list_end; ops = ops->next) {
if (ftrace_ops_test(ops, rec->ip, rec)) {
if (ops->flags & FTRACE_OPS_FL_SAVE_REGS) {
keep_regs = true;
break;
}
}
}
return keep_regs;
}
static struct ftrace_ops *
ftrace_find_tramp_ops_any(struct dyn_ftrace *rec);
static struct ftrace_ops *
ftrace_find_tramp_ops_any_other(struct dyn_ftrace *rec, struct ftrace_ops *op_exclude);
static struct ftrace_ops *
ftrace_find_tramp_ops_next(struct dyn_ftrace *rec, struct ftrace_ops *ops);
static bool skip_record(struct dyn_ftrace *rec)
{
return rec->flags & FTRACE_FL_DISABLED &&
!(rec->flags & FTRACE_FL_ENABLED);
}
static bool __ftrace_hash_rec_update(struct ftrace_ops *ops,
int filter_hash,
bool inc)
{
struct ftrace_hash *hash;
struct ftrace_hash *other_hash;
struct ftrace_page *pg;
struct dyn_ftrace *rec;
bool update = false;
int count = 0;
int all = false;
if (!(ops->flags & FTRACE_OPS_FL_ENABLED))
return false;
if (filter_hash) {
hash = ops->func_hash->filter_hash;
other_hash = ops->func_hash->notrace_hash;
if (ftrace_hash_empty(hash))
all = true;
} else {
inc = !inc;
hash = ops->func_hash->notrace_hash;
other_hash = ops->func_hash->filter_hash;
if (ftrace_hash_empty(hash))
return false;
}
do_for_each_ftrace_rec(pg, rec) {
int in_other_hash = 0;
int in_hash = 0;
int match = 0;
if (skip_record(rec))
continue;
if (all) {
if (!other_hash || !ftrace_lookup_ip(other_hash, rec->ip))
match = 1;
} else {
in_hash = !!ftrace_lookup_ip(hash, rec->ip);
in_other_hash = !!ftrace_lookup_ip(other_hash, rec->ip);
if (filter_hash && in_hash && !in_other_hash)
match = 1;
else if (!filter_hash && in_hash &&
(in_other_hash || ftrace_hash_empty(other_hash)))
match = 1;
}
if (!match)
continue;
if (inc) {
rec->flags++;
if (FTRACE_WARN_ON(ftrace_rec_count(rec) == FTRACE_REF_MAX))
return false;
if (ops->flags & FTRACE_OPS_FL_DIRECT)
rec->flags |= FTRACE_FL_DIRECT;
if (ftrace_rec_count(rec) == 1 && ops->trampoline)
rec->flags |= FTRACE_FL_TRAMP;
else
rec->flags &= ~FTRACE_FL_TRAMP;
if (ops->flags & FTRACE_OPS_FL_SAVE_REGS)
rec->flags |= FTRACE_FL_REGS;
} else {
if (FTRACE_WARN_ON(ftrace_rec_count(rec) == 0))
return false;
rec->flags--;
if (ops->flags & FTRACE_OPS_FL_DIRECT)
rec->flags &= ~FTRACE_FL_DIRECT;
if (ftrace_rec_count(rec) > 0 &&
rec->flags & FTRACE_FL_REGS &&
ops->flags & FTRACE_OPS_FL_SAVE_REGS) {
if (!test_rec_ops_needs_regs(rec))
rec->flags &= ~FTRACE_FL_REGS;
}
if (ftrace_rec_count(rec) == 1 &&
ftrace_find_tramp_ops_any_other(rec, ops))
rec->flags |= FTRACE_FL_TRAMP;
else
rec->flags &= ~FTRACE_FL_TRAMP;
}
if (IS_ENABLED(CONFIG_DYNAMIC_FTRACE_WITH_CALL_OPS) &&
ftrace_rec_count(rec) == 1 &&
ftrace_ops_get_func(ops) == ops->func)
rec->flags |= FTRACE_FL_CALL_OPS;
else
rec->flags &= ~FTRACE_FL_CALL_OPS;
count++;
update |= ftrace_test_record(rec, true) != FTRACE_UPDATE_IGNORE;
if (!all && count == hash->count)
return update;
} while_for_each_ftrace_rec();
return update;
}
static bool ftrace_hash_rec_disable(struct ftrace_ops *ops,
int filter_hash)
{
return __ftrace_hash_rec_update(ops, filter_hash, 0);
}
static bool ftrace_hash_rec_enable(struct ftrace_ops *ops,
int filter_hash)
{
return __ftrace_hash_rec_update(ops, filter_hash, 1);
}
static void ftrace_hash_rec_update_modify(struct ftrace_ops *ops,
int filter_hash, int inc)
{
struct ftrace_ops *op;
__ftrace_hash_rec_update(ops, filter_hash, inc);
if (ops->func_hash != &global_ops.local_hash)
return;
do_for_each_ftrace_op(op, ftrace_ops_list) {
if (op == ops)
continue;
if (op->func_hash == &global_ops.local_hash)
__ftrace_hash_rec_update(op, filter_hash, inc);
} while_for_each_ftrace_op(op);
}
static void ftrace_hash_rec_disable_modify(struct ftrace_ops *ops,
int filter_hash)
{
ftrace_hash_rec_update_modify(ops, filter_hash, 0);
}
static void ftrace_hash_rec_enable_modify(struct ftrace_ops *ops,
int filter_hash)
{
ftrace_hash_rec_update_modify(ops, filter_hash, 1);
}
static int __ftrace_hash_update_ipmodify(struct ftrace_ops *ops,
struct ftrace_hash *old_hash,
struct ftrace_hash *new_hash)
{
struct ftrace_page *pg;
struct dyn_ftrace *rec, *end = NULL;
int in_old, in_new;
bool is_ipmodify, is_direct;
if (!(ops->flags & FTRACE_OPS_FL_ENABLED))
return 0;
is_ipmodify = ops->flags & FTRACE_OPS_FL_IPMODIFY;
is_direct = ops->flags & FTRACE_OPS_FL_DIRECT;
if (!is_ipmodify && !is_direct)
return 0;
if (WARN_ON_ONCE(is_ipmodify && is_direct))
return 0;
if (!new_hash || !old_hash)
return -EINVAL;
do_for_each_ftrace_rec(pg, rec) {
if (rec->flags & FTRACE_FL_DISABLED)
continue;
in_old = !!ftrace_lookup_ip(old_hash, rec->ip);
in_new = !!ftrace_lookup_ip(new_hash, rec->ip);
if (in_old == in_new)
continue;
if (in_new) {
if (rec->flags & FTRACE_FL_IPMODIFY) {
int ret;
if (is_ipmodify)
goto rollback;
FTRACE_WARN_ON(rec->flags & FTRACE_FL_DIRECT);
if (!ops->ops_func)
return -EBUSY;
ret = ops->ops_func(ops, FTRACE_OPS_CMD_ENABLE_SHARE_IPMODIFY_SELF);
if (ret)
return ret;
} else if (is_ipmodify) {
rec->flags |= FTRACE_FL_IPMODIFY;
}
} else if (is_ipmodify) {
rec->flags &= ~FTRACE_FL_IPMODIFY;
}
} while_for_each_ftrace_rec();
return 0;
rollback:
end = rec;
do_for_each_ftrace_rec(pg, rec) {
if (rec->flags & FTRACE_FL_DISABLED)
continue;
if (rec == end)
goto err_out;
in_old = !!ftrace_lookup_ip(old_hash, rec->ip);
in_new = !!ftrace_lookup_ip(new_hash, rec->ip);
if (in_old == in_new)
continue;
if (in_new)
rec->flags &= ~FTRACE_FL_IPMODIFY;
else
rec->flags |= FTRACE_FL_IPMODIFY;
} while_for_each_ftrace_rec();
err_out:
return -EBUSY;
}
static int ftrace_hash_ipmodify_enable(struct ftrace_ops *ops)
{
struct ftrace_hash *hash = ops->func_hash->filter_hash;
if (ftrace_hash_empty(hash))
hash = NULL;
return __ftrace_hash_update_ipmodify(ops, EMPTY_HASH, hash);
}
static void ftrace_hash_ipmodify_disable(struct ftrace_ops *ops)
{
struct ftrace_hash *hash = ops->func_hash->filter_hash;
if (ftrace_hash_empty(hash))
hash = NULL;
__ftrace_hash_update_ipmodify(ops, hash, EMPTY_HASH);
}
static int ftrace_hash_ipmodify_update(struct ftrace_ops *ops,
struct ftrace_hash *new_hash)
{
struct ftrace_hash *old_hash = ops->func_hash->filter_hash;
if (ftrace_hash_empty(old_hash))
old_hash = NULL;
if (ftrace_hash_empty(new_hash))
new_hash = NULL;
return __ftrace_hash_update_ipmodify(ops, old_hash, new_hash);
}
static void print_ip_ins(const char *fmt, const unsigned char *p)
{
char ins[MCOUNT_INSN_SIZE];
if (copy_from_kernel_nofault(ins, p, MCOUNT_INSN_SIZE)) {
printk(KERN_CONT "%s[FAULT] %px\n", fmt, p);
return;
}
printk(KERN_CONT "%s", fmt);
pr_cont("%*phC", MCOUNT_INSN_SIZE, ins);
}
enum ftrace_bug_type ftrace_bug_type;
const void *ftrace_expected;
static void print_bug_type(void)
{
switch (ftrace_bug_type) {
case FTRACE_BUG_UNKNOWN:
break;
case FTRACE_BUG_INIT:
pr_info("Initializing ftrace call sites\n");
break;
case FTRACE_BUG_NOP:
pr_info("Setting ftrace call site to NOP\n");
break;
case FTRACE_BUG_CALL:
pr_info("Setting ftrace call site to call ftrace function\n");
break;
case FTRACE_BUG_UPDATE:
pr_info("Updating ftrace call site to call a different ftrace function\n");
break;
}
}
void ftrace_bug(int failed, struct dyn_ftrace *rec)
{
unsigned long ip = rec ? rec->ip : 0;
pr_info("------------[ ftrace bug ]------------\n");
switch (failed) {
case -EFAULT:
pr_info("ftrace faulted on modifying ");
print_ip_sym(KERN_INFO, ip);
break;
case -EINVAL:
pr_info("ftrace failed to modify ");
print_ip_sym(KERN_INFO, ip);
print_ip_ins(" actual: ", (unsigned char *)ip);
pr_cont("\n");
if (ftrace_expected) {
print_ip_ins(" expected: ", ftrace_expected);
pr_cont("\n");
}
break;
case -EPERM:
pr_info("ftrace faulted on writing ");
print_ip_sym(KERN_INFO, ip);
break;
default:
pr_info("ftrace faulted on unknown error ");
print_ip_sym(KERN_INFO, ip);
}
print_bug_type();
if (rec) {
struct ftrace_ops *ops = NULL;
pr_info("ftrace record flags: %lx\n", rec->flags);
pr_cont(" (%ld)%s%s", ftrace_rec_count(rec),
rec->flags & FTRACE_FL_REGS ? " R" : " ",
rec->flags & FTRACE_FL_CALL_OPS ? " O" : " ");
if (rec->flags & FTRACE_FL_TRAMP_EN) {
ops = ftrace_find_tramp_ops_any(rec);
if (ops) {
do {
pr_cont("\ttramp: %pS (%pS)",
(void *)ops->trampoline,
(void *)ops->func);
ops = ftrace_find_tramp_ops_next(rec, ops);
} while (ops);
} else
pr_cont("\ttramp: ERROR!");
}
ip = ftrace_get_addr_curr(rec);
pr_cont("\n expected tramp: %lx\n", ip);
}
FTRACE_WARN_ON_ONCE(1);
}
static int ftrace_check_record(struct dyn_ftrace *rec, bool enable, bool update)
{
unsigned long flag = 0UL;
ftrace_bug_type = FTRACE_BUG_UNKNOWN;
if (skip_record(rec))
return FTRACE_UPDATE_IGNORE;
if (enable && ftrace_rec_count(rec))
flag = FTRACE_FL_ENABLED;
if (flag) {
if (!(rec->flags & FTRACE_FL_REGS) !=
!(rec->flags & FTRACE_FL_REGS_EN))
flag |= FTRACE_FL_REGS;
if (!(rec->flags & FTRACE_FL_TRAMP) !=
!(rec->flags & FTRACE_FL_TRAMP_EN))
flag |= FTRACE_FL_TRAMP;
if (ftrace_rec_count(rec) == 1) {
if (!(rec->flags & FTRACE_FL_DIRECT) !=
!(rec->flags & FTRACE_FL_DIRECT_EN))
flag |= FTRACE_FL_DIRECT;
} else if (rec->flags & FTRACE_FL_DIRECT_EN) {
flag |= FTRACE_FL_DIRECT;
}
if (ftrace_rec_count(rec) == 1) {
if (!(rec->flags & FTRACE_FL_CALL_OPS) !=
!(rec->flags & FTRACE_FL_CALL_OPS_EN))
flag |= FTRACE_FL_CALL_OPS;
} else if (rec->flags & FTRACE_FL_CALL_OPS_EN) {
flag |= FTRACE_FL_CALL_OPS;
}
}
if ((rec->flags & FTRACE_FL_ENABLED) == flag)
return FTRACE_UPDATE_IGNORE;
if (flag) {
flag ^= rec->flags & FTRACE_FL_ENABLED;
if (update) {
rec->flags |= FTRACE_FL_ENABLED | FTRACE_FL_TOUCHED;
if (flag & FTRACE_FL_REGS) {
if (rec->flags & FTRACE_FL_REGS)
rec->flags |= FTRACE_FL_REGS_EN;
else
rec->flags &= ~FTRACE_FL_REGS_EN;
}
if (flag & FTRACE_FL_TRAMP) {
if (rec->flags & FTRACE_FL_TRAMP)
rec->flags |= FTRACE_FL_TRAMP_EN;
else
rec->flags &= ~FTRACE_FL_TRAMP_EN;
}
if (rec->flags & (FTRACE_FL_DIRECT | FTRACE_FL_IPMODIFY))
rec->flags |= FTRACE_FL_MODIFIED;
if (flag & FTRACE_FL_DIRECT) {
if (ftrace_rec_count(rec) == 1) {
if (rec->flags & FTRACE_FL_DIRECT)
rec->flags |= FTRACE_FL_DIRECT_EN;
else
rec->flags &= ~FTRACE_FL_DIRECT_EN;
} else {
rec->flags &= ~FTRACE_FL_DIRECT_EN;
}
}
if (flag & FTRACE_FL_CALL_OPS) {
if (ftrace_rec_count(rec) == 1) {
if (rec->flags & FTRACE_FL_CALL_OPS)
rec->flags |= FTRACE_FL_CALL_OPS_EN;
else
rec->flags &= ~FTRACE_FL_CALL_OPS_EN;
} else {
rec->flags &= ~FTRACE_FL_CALL_OPS_EN;
}
}
}
if (flag & FTRACE_FL_ENABLED) {
ftrace_bug_type = FTRACE_BUG_CALL;
return FTRACE_UPDATE_MAKE_CALL;
}
ftrace_bug_type = FTRACE_BUG_UPDATE;
return FTRACE_UPDATE_MODIFY_CALL;
}
if (update) {
if (!ftrace_rec_count(rec))
rec->flags &= FTRACE_NOCLEAR_FLAGS;
else
rec->flags &= ~(FTRACE_FL_ENABLED | FTRACE_FL_TRAMP_EN |
FTRACE_FL_REGS_EN | FTRACE_FL_DIRECT_EN |
FTRACE_FL_CALL_OPS_EN);
}
ftrace_bug_type = FTRACE_BUG_NOP;
return FTRACE_UPDATE_MAKE_NOP;
}
int ftrace_update_record(struct dyn_ftrace *rec, bool enable)
{
return ftrace_check_record(rec, enable, true);
}
int ftrace_test_record(struct dyn_ftrace *rec, bool enable)
{
return ftrace_check_record(rec, enable, false);
}
static struct ftrace_ops *
ftrace_find_tramp_ops_any(struct dyn_ftrace *rec)
{
struct ftrace_ops *op;
unsigned long ip = rec->ip;
do_for_each_ftrace_op(op, ftrace_ops_list) {
if (!op->trampoline)
continue;
if (hash_contains_ip(ip, op->func_hash))
return op;
} while_for_each_ftrace_op(op);
return NULL;
}
static struct ftrace_ops *
ftrace_find_tramp_ops_any_other(struct dyn_ftrace *rec, struct ftrace_ops *op_exclude)
{
struct ftrace_ops *op;
unsigned long ip = rec->ip;
do_for_each_ftrace_op(op, ftrace_ops_list) {
if (op == op_exclude || !op->trampoline)
continue;
if (hash_contains_ip(ip, op->func_hash))
return op;
} while_for_each_ftrace_op(op);
return NULL;
}
static struct ftrace_ops *
ftrace_find_tramp_ops_next(struct dyn_ftrace *rec,
struct ftrace_ops *op)
{
unsigned long ip = rec->ip;
while_for_each_ftrace_op(op) {
if (!op->trampoline)
continue;
if (hash_contains_ip(ip, op->func_hash))
return op;
}
return NULL;
}
static struct ftrace_ops *
ftrace_find_tramp_ops_curr(struct dyn_ftrace *rec)
{
struct ftrace_ops *op;
unsigned long ip = rec->ip;
if (removed_ops) {
if (hash_contains_ip(ip, &removed_ops->old_hash))
return removed_ops;
}
do_for_each_ftrace_op(op, ftrace_ops_list) {
if (!op->trampoline)
continue;
if (op->flags & FTRACE_OPS_FL_ADDING)
continue;
if ((op->flags & FTRACE_OPS_FL_MODIFYING) &&
hash_contains_ip(ip, &op->old_hash))
return op;
if (!(op->flags & FTRACE_OPS_FL_MODIFYING) &&
hash_contains_ip(ip, op->func_hash))
return op;
} while_for_each_ftrace_op(op);
return NULL;
}
static struct ftrace_ops *
ftrace_find_tramp_ops_new(struct dyn_ftrace *rec)
{
struct ftrace_ops *op;
unsigned long ip = rec->ip;
do_for_each_ftrace_op(op, ftrace_ops_list) {
if (hash_contains_ip(ip, op->func_hash))
return op;
} while_for_each_ftrace_op(op);
return NULL;
}
struct ftrace_ops *
ftrace_find_unique_ops(struct dyn_ftrace *rec)
{
struct ftrace_ops *op, *found = NULL;
unsigned long ip = rec->ip;
do_for_each_ftrace_op(op, ftrace_ops_list) {
if (hash_contains_ip(ip, op->func_hash)) {
if (found)
return NULL;
found = op;
}
} while_for_each_ftrace_op(op);
return found;
}
#ifdef CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS
static struct ftrace_hash *direct_functions = EMPTY_HASH;
static DEFINE_MUTEX(direct_mutex);
int ftrace_direct_func_count;
unsigned long ftrace_find_rec_direct(unsigned long ip)
{
struct ftrace_func_entry *entry;
entry = __ftrace_lookup_ip(direct_functions, ip);
if (!entry)
return 0;
return entry->direct;
}
static struct ftrace_func_entry*
ftrace_add_rec_direct(unsigned long ip, unsigned long addr,
struct ftrace_hash **free_hash)
{
struct ftrace_func_entry *entry;
if (ftrace_hash_empty(direct_functions) ||
direct_functions->count > 2 * (1 << direct_functions->size_bits)) {
struct ftrace_hash *new_hash;
int size = ftrace_hash_empty(direct_functions) ? 0 :
direct_functions->count + 1;
if (size < 32)
size = 32;
new_hash = dup_hash(direct_functions, size);
if (!new_hash)
return NULL;
*free_hash = direct_functions;
direct_functions = new_hash;
}
entry = kmalloc(sizeof(*entry), GFP_KERNEL);
if (!entry)
return NULL;
entry->ip = ip;
entry->direct = addr;
__add_hash_entry(direct_functions, entry);
return entry;
}
static void call_direct_funcs(unsigned long ip, unsigned long pip,
struct ftrace_ops *ops, struct ftrace_regs *fregs)
{
unsigned long addr = READ_ONCE(ops->direct_call);
if (!addr)
return;
arch_ftrace_set_direct_caller(fregs, addr);
}
#endif /* CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS */
unsigned long ftrace_get_addr_new(struct dyn_ftrace *rec)
{
struct ftrace_ops *ops;
unsigned long addr;
if ((rec->flags & FTRACE_FL_DIRECT) &&
(ftrace_rec_count(rec) == 1)) {
addr = ftrace_find_rec_direct(rec->ip);
if (addr)
return addr;
WARN_ON_ONCE(1);
}
if (rec->flags & FTRACE_FL_TRAMP) {
ops = ftrace_find_tramp_ops_new(rec);
if (FTRACE_WARN_ON(!ops || !ops->trampoline)) {
pr_warn("Bad trampoline accounting at: %p (%pS) (%lx)\n",
(void *)rec->ip, (void *)rec->ip, rec->flags);
return (unsigned long)FTRACE_ADDR;
}
return ops->trampoline;
}
if (rec->flags & FTRACE_FL_REGS)
return (unsigned long)FTRACE_REGS_ADDR;
else
return (unsigned long)FTRACE_ADDR;
}
unsigned long ftrace_get_addr_curr(struct dyn_ftrace *rec)
{
struct ftrace_ops *ops;
unsigned long addr;
if (rec->flags & FTRACE_FL_DIRECT_EN) {
addr = ftrace_find_rec_direct(rec->ip);
if (addr)
return addr;
WARN_ON_ONCE(1);
}
if (rec->flags & FTRACE_FL_TRAMP_EN) {
ops = ftrace_find_tramp_ops_curr(rec);
if (FTRACE_WARN_ON(!ops)) {
pr_warn("Bad trampoline accounting at: %p (%pS)\n",
(void *)rec->ip, (void *)rec->ip);
return (unsigned long)FTRACE_ADDR;
}
return ops->trampoline;
}
if (rec->flags & FTRACE_FL_REGS_EN)
return (unsigned long)FTRACE_REGS_ADDR;
else
return (unsigned long)FTRACE_ADDR;
}
static int
__ftrace_replace_code(struct dyn_ftrace *rec, bool enable)
{
unsigned long ftrace_old_addr;
unsigned long ftrace_addr;
int ret;
ftrace_addr = ftrace_get_addr_new(rec);
ftrace_old_addr = ftrace_get_addr_curr(rec);
ret = ftrace_update_record(rec, enable);
ftrace_bug_type = FTRACE_BUG_UNKNOWN;
switch (ret) {
case FTRACE_UPDATE_IGNORE:
return 0;
case FTRACE_UPDATE_MAKE_CALL:
ftrace_bug_type = FTRACE_BUG_CALL;
return ftrace_make_call(rec, ftrace_addr);
case FTRACE_UPDATE_MAKE_NOP:
ftrace_bug_type = FTRACE_BUG_NOP;
return ftrace_make_nop(NULL, rec, ftrace_old_addr);
case FTRACE_UPDATE_MODIFY_CALL:
ftrace_bug_type = FTRACE_BUG_UPDATE;
return ftrace_modify_call(rec, ftrace_old_addr, ftrace_addr);
}
return -1;
}
void __weak ftrace_replace_code(int mod_flags)
{
struct dyn_ftrace *rec;
struct ftrace_page *pg;
bool enable = mod_flags & FTRACE_MODIFY_ENABLE_FL;
int schedulable = mod_flags & FTRACE_MODIFY_MAY_SLEEP_FL;
int failed;
if (unlikely(ftrace_disabled))
return;
do_for_each_ftrace_rec(pg, rec) {
if (skip_record(rec))
continue;
failed = __ftrace_replace_code(rec, enable);
if (failed) {
ftrace_bug(failed, rec);
return;
}
if (schedulable)
cond_resched();
} while_for_each_ftrace_rec();
}
struct ftrace_rec_iter {
struct ftrace_page *pg;
int index;
};
struct ftrace_rec_iter *ftrace_rec_iter_start(void)
{
static struct ftrace_rec_iter ftrace_rec_iter;
struct ftrace_rec_iter *iter = &ftrace_rec_iter;
iter->pg = ftrace_pages_start;
iter->index = 0;
while (iter->pg && !iter->pg->index)
iter->pg = iter->pg->next;
if (!iter->pg)
return NULL;
return iter;
}
struct ftrace_rec_iter *ftrace_rec_iter_next(struct ftrace_rec_iter *iter)
{
iter->index++;
if (iter->index >= iter->pg->index) {
iter->pg = iter->pg->next;
iter->index = 0;
while (iter->pg && !iter->pg->index)
iter->pg = iter->pg->next;
}
if (!iter->pg)
return NULL;
return iter;
}
struct dyn_ftrace *ftrace_rec_iter_record(struct ftrace_rec_iter *iter)
{
return &iter->pg->records[iter->index];
}
static int
ftrace_nop_initialize(struct module *mod, struct dyn_ftrace *rec)
{
int ret;
if (unlikely(ftrace_disabled))
return 0;
ret = ftrace_init_nop(mod, rec);
if (ret) {
ftrace_bug_type = FTRACE_BUG_INIT;
ftrace_bug(ret, rec);
return 0;
}
return 1;
}
void __weak ftrace_arch_code_modify_prepare(void)
{
}
void __weak ftrace_arch_code_modify_post_process(void)
{
}
static int update_ftrace_func(ftrace_func_t func)
{
static ftrace_func_t save_func;
if (func == save_func)
return 0;
save_func = func;
return ftrace_update_ftrace_func(func);
}
void ftrace_modify_all_code(int command)
{
int update = command & FTRACE_UPDATE_TRACE_FUNC;
int mod_flags = 0;
int err = 0;
if (command & FTRACE_MAY_SLEEP)
mod_flags = FTRACE_MODIFY_MAY_SLEEP_FL;
if (update) {
err = update_ftrace_func(ftrace_ops_list_func);
if (FTRACE_WARN_ON(err))
return;
}
if (command & FTRACE_UPDATE_CALLS)
ftrace_replace_code(mod_flags | FTRACE_MODIFY_ENABLE_FL);
else if (command & FTRACE_DISABLE_CALLS)
ftrace_replace_code(mod_flags);
if (update && ftrace_trace_function != ftrace_ops_list_func) {
function_trace_op = set_function_trace_op;
smp_wmb();
if (!irqs_disabled())
smp_call_function(ftrace_sync_ipi, NULL, 1);
err = update_ftrace_func(ftrace_trace_function);
if (FTRACE_WARN_ON(err))
return;
}
if (command & FTRACE_START_FUNC_RET)
err = ftrace_enable_ftrace_graph_caller();
else if (command & FTRACE_STOP_FUNC_RET)
err = ftrace_disable_ftrace_graph_caller();
FTRACE_WARN_ON(err);
}
static int __ftrace_modify_code(void *data)
{
int *command = data;
ftrace_modify_all_code(*command);
return 0;
}
void ftrace_run_stop_machine(int command)
{
stop_machine(__ftrace_modify_code, &command, NULL);
}
void __weak arch_ftrace_update_code(int command)
{
ftrace_run_stop_machine(command);
}
static void ftrace_run_update_code(int command)
{
ftrace_arch_code_modify_prepare();
arch_ftrace_update_code(command);
ftrace_arch_code_modify_post_process();
}
static void ftrace_run_modify_code(struct ftrace_ops *ops, int command,
struct ftrace_ops_hash *old_hash)
{
ops->flags |= FTRACE_OPS_FL_MODIFYING;
ops->old_hash.filter_hash = old_hash->filter_hash;
ops->old_hash.notrace_hash = old_hash->notrace_hash;
ftrace_run_update_code(command);
ops->old_hash.filter_hash = NULL;
ops->old_hash.notrace_hash = NULL;
ops->flags &= ~FTRACE_OPS_FL_MODIFYING;
}
static ftrace_func_t saved_ftrace_func;
static int ftrace_start_up;
void __weak arch_ftrace_trampoline_free(struct ftrace_ops *ops)
{
}
static LIST_HEAD(ftrace_ops_trampoline_list);
static void ftrace_add_trampoline_to_kallsyms(struct ftrace_ops *ops)
{
lockdep_assert_held(&ftrace_lock);
list_add_rcu(&ops->list, &ftrace_ops_trampoline_list);
}
static void ftrace_remove_trampoline_from_kallsyms(struct ftrace_ops *ops)
{
lockdep_assert_held(&ftrace_lock);
list_del_rcu(&ops->list);
synchronize_rcu();
}
#define FTRACE_TRAMPOLINE_MOD "__builtin__ftrace"
#define FTRACE_TRAMPOLINE_SYM "ftrace_trampoline"
static void ftrace_trampoline_free(struct ftrace_ops *ops)
{
if (ops && (ops->flags & FTRACE_OPS_FL_ALLOC_TRAMP) &&
ops->trampoline) {
perf_event_text_poke((void *)ops->trampoline,
(void *)ops->trampoline,
ops->trampoline_size, NULL, 0);
perf_event_ksymbol(PERF_RECORD_KSYMBOL_TYPE_OOL,
ops->trampoline, ops->trampoline_size,
true, FTRACE_TRAMPOLINE_SYM);
ftrace_remove_trampoline_from_kallsyms(ops);
}
arch_ftrace_trampoline_free(ops);
}
static void ftrace_startup_enable(int command)
{
if (saved_ftrace_func != ftrace_trace_function) {
saved_ftrace_func = ftrace_trace_function;
command |= FTRACE_UPDATE_TRACE_FUNC;
}
if (!command || !ftrace_enabled)
return;
ftrace_run_update_code(command);
}
static void ftrace_startup_all(int command)
{
update_all_ops = true;
ftrace_startup_enable(command);
update_all_ops = false;
}
int ftrace_startup(struct ftrace_ops *ops, int command)
{
int ret;
if (unlikely(ftrace_disabled))
return -ENODEV;
ret = __register_ftrace_function(ops);
if (ret)
return ret;
ftrace_start_up++;
ops->flags |= FTRACE_OPS_FL_ENABLED | FTRACE_OPS_FL_ADDING;
ret = ftrace_hash_ipmodify_enable(ops);
if (ret < 0) {
__unregister_ftrace_function(ops);
ftrace_start_up--;
ops->flags &= ~FTRACE_OPS_FL_ENABLED;
if (ops->flags & FTRACE_OPS_FL_DYNAMIC)
ftrace_trampoline_free(ops);
return ret;
}
if (ftrace_hash_rec_enable(ops, 1))
command |= FTRACE_UPDATE_CALLS;
ftrace_startup_enable(command);
if (unlikely(ftrace_disabled)) {
__unregister_ftrace_function(ops);
return -ENODEV;
}
ops->flags &= ~FTRACE_OPS_FL_ADDING;
return 0;
}
int ftrace_shutdown(struct ftrace_ops *ops, int command)
{
int ret;
if (unlikely(ftrace_disabled))
return -ENODEV;
ret = __unregister_ftrace_function(ops);
if (ret)
return ret;
ftrace_start_up--;
WARN_ON_ONCE(ftrace_start_up < 0);
ftrace_hash_ipmodify_disable(ops);
if (ftrace_hash_rec_disable(ops, 1))
command |= FTRACE_UPDATE_CALLS;
ops->flags &= ~FTRACE_OPS_FL_ENABLED;
if (saved_ftrace_func != ftrace_trace_function) {
saved_ftrace_func = ftrace_trace_function;
command |= FTRACE_UPDATE_TRACE_FUNC;
}
if (!command || !ftrace_enabled)
goto out;
ops->flags |= FTRACE_OPS_FL_REMOVING;
removed_ops = ops;
ops->old_hash.filter_hash = ops->func_hash->filter_hash;
ops->old_hash.notrace_hash = ops->func_hash->notrace_hash;
ftrace_run_update_code(command);
if (rcu_dereference_protected(ftrace_ops_list,
lockdep_is_held(&ftrace_lock)) == &ftrace_list_end) {
struct ftrace_page *pg;
struct dyn_ftrace *rec;
do_for_each_ftrace_rec(pg, rec) {
if (FTRACE_WARN_ON_ONCE(rec->flags & ~FTRACE_NOCLEAR_FLAGS))
pr_warn(" %pS flags:%lx\n",
(void *)rec->ip, rec->flags);
} while_for_each_ftrace_rec();
}
ops->old_hash.filter_hash = NULL;
ops->old_hash.notrace_hash = NULL;
removed_ops = NULL;
ops->flags &= ~FTRACE_OPS_FL_REMOVING;
out:
if (ops->flags & FTRACE_OPS_FL_DYNAMIC) {
synchronize_rcu_tasks_rude();
if (IS_ENABLED(CONFIG_PREEMPTION))
synchronize_rcu_tasks();
ftrace_trampoline_free(ops);
}
return 0;
}
static u64 ftrace_update_time;
unsigned long ftrace_update_tot_cnt;
unsigned long ftrace_number_of_pages;
unsigned long ftrace_number_of_groups;
static inline int ops_traces_mod(struct ftrace_ops *ops)
{
return ftrace_hash_empty(ops->func_hash->filter_hash) &&
ftrace_hash_empty(ops->func_hash->notrace_hash);
}
static int ftrace_update_code(struct module *mod, struct ftrace_page *new_pgs)
{
bool init_nop = ftrace_need_init_nop();
struct ftrace_page *pg;
struct dyn_ftrace *p;
u64 start, stop;
unsigned long update_cnt = 0;
unsigned long rec_flags = 0;
int i;
start = ftrace_now(raw_smp_processor_id());
if (mod)
rec_flags |= FTRACE_FL_DISABLED;
for (pg = new_pgs; pg; pg = pg->next) {
for (i = 0; i < pg->index; i++) {
if (unlikely(ftrace_disabled))
return -1;
p = &pg->records[i];
p->flags = rec_flags;
if (init_nop && !ftrace_nop_initialize(mod, p))
break;
update_cnt++;
}
}
stop = ftrace_now(raw_smp_processor_id());
ftrace_update_time = stop - start;
ftrace_update_tot_cnt += update_cnt;
return 0;
}
static int ftrace_allocate_records(struct ftrace_page *pg, int count)
{
int order;
int pages;
int cnt;
if (WARN_ON(!count))
return -EINVAL;
pages = DIV_ROUND_UP(count, ENTRIES_PER_PAGE);
order = fls(pages) - 1;
again:
pg->records = (void *)__get_free_pages(GFP_KERNEL | __GFP_ZERO, order);
if (!pg->records) {
if (!order)
return -ENOMEM;
order--;
goto again;
}
ftrace_number_of_pages += 1 << order;
ftrace_number_of_groups++;
cnt = (PAGE_SIZE << order) / ENTRY_SIZE;
pg->order = order;
if (cnt > count)
cnt = count;
return cnt;
}
static void ftrace_free_pages(struct ftrace_page *pages)
{
struct ftrace_page *pg = pages;
while (pg) {
if (pg->records) {
free_pages((unsigned long)pg->records, pg->order);
ftrace_number_of_pages -= 1 << pg->order;
}
pages = pg->next;
kfree(pg);
pg = pages;
ftrace_number_of_groups--;
}
}
static struct ftrace_page *
ftrace_allocate_pages(unsigned long num_to_init)
{
struct ftrace_page *start_pg;
struct ftrace_page *pg;
int cnt;
if (!num_to_init)
return NULL;
start_pg = pg = kzalloc(sizeof(*pg), GFP_KERNEL);
if (!pg)
return NULL;
for (;;) {
cnt = ftrace_allocate_records(pg, num_to_init);
if (cnt < 0)
goto free_pages;
num_to_init -= cnt;
if (!num_to_init)
break;
pg->next = kzalloc(sizeof(*pg), GFP_KERNEL);
if (!pg->next)
goto free_pages;
pg = pg->next;
}
return start_pg;
free_pages:
ftrace_free_pages(start_pg);
pr_info("ftrace: FAILED to allocate memory for functions\n");
return NULL;
}
#define FTRACE_BUFF_MAX (KSYM_SYMBOL_LEN+4) /* room for wildcards */
struct ftrace_iterator {
loff_t pos;
loff_t func_pos;
loff_t mod_pos;
struct ftrace_page *pg;
struct dyn_ftrace *func;
struct ftrace_func_probe *probe;
struct ftrace_func_entry *probe_entry;
struct trace_parser parser;
struct ftrace_hash *hash;
struct ftrace_ops *ops;
struct trace_array *tr;
struct list_head *mod_list;
int pidx;
int idx;
unsigned flags;
};
static void *
t_probe_next(struct seq_file *m, loff_t *pos)
{
struct ftrace_iterator *iter = m->private;
struct trace_array *tr = iter->ops->private;
struct list_head *func_probes;
struct ftrace_hash *hash;
struct list_head *next;
struct hlist_node *hnd = NULL;
struct hlist_head *hhd;
int size;
(*pos)++;
iter->pos = *pos;
if (!tr)
return NULL;
func_probes = &tr->func_probes;
if (list_empty(func_probes))
return NULL;
if (!iter->probe) {
next = func_probes->next;
iter->probe = list_entry(next, struct ftrace_func_probe, list);
}
if (iter->probe_entry)
hnd = &iter->probe_entry->hlist;
hash = iter->probe->ops.func_hash->filter_hash;
if (!hash || hash == EMPTY_HASH)
return NULL;
size = 1 << hash->size_bits;
retry:
if (iter->pidx >= size) {
if (iter->probe->list.next == func_probes)
return NULL;
next = iter->probe->list.next;
iter->probe = list_entry(next, struct ftrace_func_probe, list);
hash = iter->probe->ops.func_hash->filter_hash;
size = 1 << hash->size_bits;
iter->pidx = 0;
}
hhd = &hash->buckets[iter->pidx];
if (hlist_empty(hhd)) {
iter->pidx++;
hnd = NULL;
goto retry;
}
if (!hnd)
hnd = hhd->first;
else {
hnd = hnd->next;
if (!hnd) {
iter->pidx++;
goto retry;
}
}
if (WARN_ON_ONCE(!hnd))
return NULL;
iter->probe_entry = hlist_entry(hnd, struct ftrace_func_entry, hlist);
return iter;
}
static void *t_probe_start(struct seq_file *m, loff_t *pos)
{
struct ftrace_iterator *iter = m->private;
void *p = NULL;
loff_t l;
if (!(iter->flags & FTRACE_ITER_DO_PROBES))
return NULL;
if (iter->mod_pos > *pos)
return NULL;
iter->probe = NULL;
iter->probe_entry = NULL;
iter->pidx = 0;
for (l = 0; l <= (*pos - iter->mod_pos); ) {
p = t_probe_next(m, &l);
if (!p)
break;
}
if (!p)
return NULL;
iter->flags |= FTRACE_ITER_PROBE;
return iter;
}
static int
t_probe_show(struct seq_file *m, struct ftrace_iterator *iter)
{
struct ftrace_func_entry *probe_entry;
struct ftrace_probe_ops *probe_ops;
struct ftrace_func_probe *probe;
probe = iter->probe;
probe_entry = iter->probe_entry;
if (WARN_ON_ONCE(!probe || !probe_entry))
return -EIO;
probe_ops = probe->probe_ops;
if (probe_ops->print)
return probe_ops->print(m, probe_entry->ip, probe_ops, probe->data);
seq_printf(m, "%ps:%ps\n", (void *)probe_entry->ip,
(void *)probe_ops->func);
return 0;
}
static void *
t_mod_next(struct seq_file *m, loff_t *pos)
{
struct ftrace_iterator *iter = m->private;
struct trace_array *tr = iter->tr;
(*pos)++;
iter->pos = *pos;
iter->mod_list = iter->mod_list->next;
if (iter->mod_list == &tr->mod_trace ||
iter->mod_list == &tr->mod_notrace) {
iter->flags &= ~FTRACE_ITER_MOD;
return NULL;
}
iter->mod_pos = *pos;
return iter;
}
static void *t_mod_start(struct seq_file *m, loff_t *pos)
{
struct ftrace_iterator *iter = m->private;
void *p = NULL;
loff_t l;
if (iter->func_pos > *pos)
return NULL;
iter->mod_pos = iter->func_pos;
if (!iter->tr)
return NULL;
for (l = 0; l <= (*pos - iter->func_pos); ) {
p = t_mod_next(m, &l);
if (!p)
break;
}
if (!p) {
iter->flags &= ~FTRACE_ITER_MOD;
return t_probe_start(m, pos);
}
iter->flags |= FTRACE_ITER_MOD;
return iter;
}
static int
t_mod_show(struct seq_file *m, struct ftrace_iterator *iter)
{
struct ftrace_mod_load *ftrace_mod;
struct trace_array *tr = iter->tr;
if (WARN_ON_ONCE(!iter->mod_list) ||
iter->mod_list == &tr->mod_trace ||
iter->mod_list == &tr->mod_notrace)
return -EIO;
ftrace_mod = list_entry(iter->mod_list, struct ftrace_mod_load, list);
if (ftrace_mod->func)
seq_printf(m, "%s", ftrace_mod->func);
else
seq_putc(m, '*');
seq_printf(m, ":mod:%s\n", ftrace_mod->module);
return 0;
}
static void *
t_func_next(struct seq_file *m, loff_t *pos)
{
struct ftrace_iterator *iter = m->private;
struct dyn_ftrace *rec = NULL;
(*pos)++;
retry:
if (iter->idx >= iter->pg->index) {
if (iter->pg->next) {
iter->pg = iter->pg->next;
iter->idx = 0;
goto retry;
}
} else {
rec = &iter->pg->records[iter->idx++];
if (((iter->flags & (FTRACE_ITER_FILTER | FTRACE_ITER_NOTRACE)) &&
!ftrace_lookup_ip(iter->hash, rec->ip)) ||
((iter->flags & FTRACE_ITER_ENABLED) &&
!(rec->flags & FTRACE_FL_ENABLED)) ||
((iter->flags & FTRACE_ITER_TOUCHED) &&
!(rec->flags & FTRACE_FL_TOUCHED))) {
rec = NULL;
goto retry;
}
}
if (!rec)
return NULL;
iter->pos = iter->func_pos = *pos;
iter->func = rec;
return iter;
}
static void *
t_next(struct seq_file *m, void *v, loff_t *pos)
{
struct ftrace_iterator *iter = m->private;
loff_t l = *pos;
void *ret;
if (unlikely(ftrace_disabled))
return NULL;
if (iter->flags & FTRACE_ITER_PROBE)
return t_probe_next(m, pos);
if (iter->flags & FTRACE_ITER_MOD)
return t_mod_next(m, pos);
if (iter->flags & FTRACE_ITER_PRINTALL) {
(*pos)++;
return t_mod_start(m, &l);
}
ret = t_func_next(m, pos);
if (!ret)
return t_mod_start(m, &l);
return ret;
}
static void reset_iter_read(struct ftrace_iterator *iter)
{
iter->pos = 0;
iter->func_pos = 0;
iter->flags &= ~(FTRACE_ITER_PRINTALL | FTRACE_ITER_PROBE | FTRACE_ITER_MOD);
}
static void *t_start(struct seq_file *m, loff_t *pos)
{
struct ftrace_iterator *iter = m->private;
void *p = NULL;
loff_t l;
mutex_lock(&ftrace_lock);
if (unlikely(ftrace_disabled))
return NULL;
if (*pos < iter->pos)
reset_iter_read(iter);
if ((iter->flags & (FTRACE_ITER_FILTER | FTRACE_ITER_NOTRACE)) &&
ftrace_hash_empty(iter->hash)) {
iter->func_pos = 1;
if (*pos > 0)
return t_mod_start(m, pos);
iter->flags |= FTRACE_ITER_PRINTALL;
iter->flags &= ~FTRACE_ITER_PROBE;
return iter;
}
if (iter->flags & FTRACE_ITER_MOD)
return t_mod_start(m, pos);
iter->pg = ftrace_pages_start;
iter->idx = 0;
for (l = 0; l <= *pos; ) {
p = t_func_next(m, &l);
if (!p)
break;
}
if (!p)
return t_mod_start(m, pos);
return iter;
}
static void t_stop(struct seq_file *m, void *p)
{
mutex_unlock(&ftrace_lock);
}
void * __weak
arch_ftrace_trampoline_func(struct ftrace_ops *ops, struct dyn_ftrace *rec)
{
return NULL;
}
static void add_trampoline_func(struct seq_file *m, struct ftrace_ops *ops,
struct dyn_ftrace *rec)
{
void *ptr;
ptr = arch_ftrace_trampoline_func(ops, rec);
if (ptr)
seq_printf(m, " ->%pS", ptr);
}
#ifdef FTRACE_MCOUNT_MAX_OFFSET
static int test_for_valid_rec(struct dyn_ftrace *rec)
{
char str[KSYM_SYMBOL_LEN];
unsigned long offset;
const char *ret;
ret = kallsyms_lookup(rec->ip, NULL, &offset, NULL, str);
if (!ret || offset > FTRACE_MCOUNT_MAX_OFFSET) {
rec->flags |= FTRACE_FL_DISABLED;
return 0;
}
return 1;
}
static struct workqueue_struct *ftrace_check_wq __initdata;
static struct work_struct ftrace_check_work __initdata;
static __init void ftrace_check_work_func(struct work_struct *work)
{
struct ftrace_page *pg;
struct dyn_ftrace *rec;
mutex_lock(&ftrace_lock);
do_for_each_ftrace_rec(pg, rec) {
test_for_valid_rec(rec);
} while_for_each_ftrace_rec();
mutex_unlock(&ftrace_lock);
}
static int __init ftrace_check_for_weak_functions(void)
{
INIT_WORK(&ftrace_check_work, ftrace_check_work_func);
ftrace_check_wq = alloc_workqueue("ftrace_check_wq", WQ_UNBOUND, 0);
queue_work(ftrace_check_wq, &ftrace_check_work);
return 0;
}
static int __init ftrace_check_sync(void)
{
if (ftrace_check_wq)
destroy_workqueue(ftrace_check_wq);
return 0;
}
late_initcall_sync(ftrace_check_sync);
subsys_initcall(ftrace_check_for_weak_functions);
static int print_rec(struct seq_file *m, unsigned long ip)
{
unsigned long offset;
char str[KSYM_SYMBOL_LEN];
char *modname;
const char *ret;
ret = kallsyms_lookup(ip, NULL, &offset, &modname, str);
if (!ret || offset > FTRACE_MCOUNT_MAX_OFFSET) {
snprintf(str, KSYM_SYMBOL_LEN, "%s_%ld",
FTRACE_INVALID_FUNCTION, offset);
ret = NULL;
}
seq_puts(m, str);
if (modname)
seq_printf(m, " [%s]", modname);
return ret == NULL ? -1 : 0;
}
#else
static inline int test_for_valid_rec(struct dyn_ftrace *rec)
{
return 1;
}
static inline int print_rec(struct seq_file *m, unsigned long ip)
{
seq_printf(m, "%ps", (void *)ip);
return 0;
}
#endif
static int t_show(struct seq_file *m, void *v)
{
struct ftrace_iterator *iter = m->private;
struct dyn_ftrace *rec;
if (iter->flags & FTRACE_ITER_PROBE)
return t_probe_show(m, iter);
if (iter->flags & FTRACE_ITER_MOD)
return t_mod_show(m, iter);
if (iter->flags & FTRACE_ITER_PRINTALL) {
if (iter->flags & FTRACE_ITER_NOTRACE)
seq_puts(m, "#### no functions disabled ####\n");
else
seq_puts(m, "#### all functions enabled ####\n");
return 0;
}
rec = iter->func;
if (!rec)
return 0;
if (iter->flags & FTRACE_ITER_ADDRS)
seq_printf(m, "%lx ", rec->ip);
if (print_rec(m, rec->ip)) {
WARN_ON_ONCE(!(rec->flags & FTRACE_FL_DISABLED));
seq_putc(m, '\n');
return 0;
}
if (iter->flags & (FTRACE_ITER_ENABLED | FTRACE_ITER_TOUCHED)) {
struct ftrace_ops *ops;
seq_printf(m, " (%ld)%s%s%s%s%s",
ftrace_rec_count(rec),
rec->flags & FTRACE_FL_REGS ? " R" : " ",
rec->flags & FTRACE_FL_IPMODIFY ? " I" : " ",
rec->flags & FTRACE_FL_DIRECT ? " D" : " ",
rec->flags & FTRACE_FL_CALL_OPS ? " O" : " ",
rec->flags & FTRACE_FL_MODIFIED ? " M " : " ");
if (rec->flags & FTRACE_FL_TRAMP_EN) {
ops = ftrace_find_tramp_ops_any(rec);
if (ops) {
do {
seq_printf(m, "\ttramp: %pS (%pS)",
(void *)ops->trampoline,
(void *)ops->func);
add_trampoline_func(m, ops, rec);
ops = ftrace_find_tramp_ops_next(rec, ops);
} while (ops);
} else
seq_puts(m, "\ttramp: ERROR!");
} else {
add_trampoline_func(m, NULL, rec);
}
if (rec->flags & FTRACE_FL_CALL_OPS_EN) {
ops = ftrace_find_unique_ops(rec);
if (ops) {
seq_printf(m, "\tops: %pS (%pS)",
ops, ops->func);
} else {
seq_puts(m, "\tops: ERROR!");
}
}
if (rec->flags & FTRACE_FL_DIRECT) {
unsigned long direct;
direct = ftrace_find_rec_direct(rec->ip);
if (direct)
seq_printf(m, "\n\tdirect-->%pS", (void *)direct);
}
}
seq_putc(m, '\n');
return 0;
}
static const struct seq_operations show_ftrace_seq_ops = {
.start = t_start,
.next = t_next,
.stop = t_stop,
.show = t_show,
};
static int
ftrace_avail_open(struct inode *inode, struct file *file)
{
struct ftrace_iterator *iter;
int ret;
ret = security_locked_down(LOCKDOWN_TRACEFS);
if (ret)
return ret;
if (unlikely(ftrace_disabled))
return -ENODEV;
iter = __seq_open_private(file, &show_ftrace_seq_ops, sizeof(*iter));
if (!iter)
return -ENOMEM;
iter->pg = ftrace_pages_start;
iter->ops = &global_ops;
return 0;
}
static int
ftrace_enabled_open(struct inode *inode, struct file *file)
{
struct ftrace_iterator *iter;
iter = __seq_open_private(file, &show_ftrace_seq_ops, sizeof(*iter));
if (!iter)
return -ENOMEM;
iter->pg = ftrace_pages_start;
iter->flags = FTRACE_ITER_ENABLED;
iter->ops = &global_ops;
return 0;
}
static int
ftrace_touched_open(struct inode *inode, struct file *file)
{
struct ftrace_iterator *iter;
iter = __seq_open_private(file, &show_ftrace_seq_ops, sizeof(*iter));
if (!iter)
return -ENOMEM;
iter->pg = ftrace_pages_start;
iter->flags = FTRACE_ITER_TOUCHED;
iter->ops = &global_ops;
return 0;
}
static int
ftrace_avail_addrs_open(struct inode *inode, struct file *file)
{
struct ftrace_iterator *iter;
int ret;
ret = security_locked_down(LOCKDOWN_TRACEFS);
if (ret)
return ret;
if (unlikely(ftrace_disabled))
return -ENODEV;
iter = __seq_open_private(file, &show_ftrace_seq_ops, sizeof(*iter));
if (!iter)
return -ENOMEM;
iter->pg = ftrace_pages_start;
iter->flags = FTRACE_ITER_ADDRS;
iter->ops = &global_ops;
return 0;
}
int
ftrace_regex_open(struct ftrace_ops *ops, int flag,
struct inode *inode, struct file *file)
{
struct ftrace_iterator *iter;
struct ftrace_hash *hash;
struct list_head *mod_head;
struct trace_array *tr = ops->private;
int ret = -ENOMEM;
ftrace_ops_init(ops);
if (unlikely(ftrace_disabled))
return -ENODEV;
if (tracing_check_open_get_tr(tr))
return -ENODEV;
iter = kzalloc(sizeof(*iter), GFP_KERNEL);
if (!iter)
goto out;
if (trace_parser_get_init(&iter->parser, FTRACE_BUFF_MAX))
goto out;
iter->ops = ops;
iter->flags = flag;
iter->tr = tr;
mutex_lock(&ops->func_hash->regex_lock);
if (flag & FTRACE_ITER_NOTRACE) {
hash = ops->func_hash->notrace_hash;
mod_head = tr ? &tr->mod_notrace : NULL;
} else {
hash = ops->func_hash->filter_hash;
mod_head = tr ? &tr->mod_trace : NULL;
}
iter->mod_list = mod_head;
if (file->f_mode & FMODE_WRITE) {
const int size_bits = FTRACE_HASH_DEFAULT_BITS;
if (file->f_flags & O_TRUNC) {
iter->hash = alloc_ftrace_hash(size_bits);
clear_ftrace_mod_list(mod_head);
} else {
iter->hash = alloc_and_copy_ftrace_hash(size_bits, hash);
}
if (!iter->hash) {
trace_parser_put(&iter->parser);
goto out_unlock;
}
} else
iter->hash = hash;
ret = 0;
if (file->f_mode & FMODE_READ) {
iter->pg = ftrace_pages_start;
ret = seq_open(file, &show_ftrace_seq_ops);
if (!ret) {
struct seq_file *m = file->private_data;
m->private = iter;
} else {
free_ftrace_hash(iter->hash);
trace_parser_put(&iter->parser);
}
} else
file->private_data = iter;
out_unlock:
mutex_unlock(&ops->func_hash->regex_lock);
out:
if (ret) {
kfree(iter);
if (tr)
trace_array_put(tr);
}
return ret;
}
static int
ftrace_filter_open(struct inode *inode, struct file *file)
{
struct ftrace_ops *ops = inode->i_private;
return ftrace_regex_open(ops,
FTRACE_ITER_FILTER | FTRACE_ITER_DO_PROBES,
inode, file);
}
static int
ftrace_notrace_open(struct inode *inode, struct file *file)
{
struct ftrace_ops *ops = inode->i_private;
return ftrace_regex_open(ops, FTRACE_ITER_NOTRACE,
inode, file);
}
struct ftrace_glob {
char *search;
unsigned len;
int type;
};
char * __weak arch_ftrace_match_adjust(char *str, const char *search)
{
return str;
}
static int ftrace_match(char *str, struct ftrace_glob *g)
{
int matched = 0;
int slen;
str = arch_ftrace_match_adjust(str, g->search);
switch (g->type) {
case MATCH_FULL:
if (strcmp(str, g->search) == 0)
matched = 1;
break;
case MATCH_FRONT_ONLY:
if (strncmp(str, g->search, g->len) == 0)
matched = 1;
break;
case MATCH_MIDDLE_ONLY:
if (strstr(str, g->search))
matched = 1;
break;
case MATCH_END_ONLY:
slen = strlen(str);
if (slen >= g->len &&
memcmp(str + slen - g->len, g->search, g->len) == 0)
matched = 1;
break;
case MATCH_GLOB:
if (glob_match(g->search, str))
matched = 1;
break;
}
return matched;
}
static int
enter_record(struct ftrace_hash *hash, struct dyn_ftrace *rec, int clear_filter)
{
struct ftrace_func_entry *entry;
int ret = 0;
entry = ftrace_lookup_ip(hash, rec->ip);
if (clear_filter) {
if (!entry)
return 0;
free_hash_entry(hash, entry);
} else {
if (entry)
return 0;
ret = add_hash_entry(hash, rec->ip);
}
return ret;
}
static int
add_rec_by_index(struct ftrace_hash *hash, struct ftrace_glob *func_g,
int clear_filter)
{
long index = simple_strtoul(func_g->search, NULL, 0);
struct ftrace_page *pg;
struct dyn_ftrace *rec;
if (--index < 0)
return 0;
do_for_each_ftrace_rec(pg, rec) {
if (pg->index <= index) {
index -= pg->index;
break;
}
rec = &pg->records[index];
enter_record(hash, rec, clear_filter);
return 1;
} while_for_each_ftrace_rec();
return 0;
}
#ifdef FTRACE_MCOUNT_MAX_OFFSET
static int lookup_ip(unsigned long ip, char **modname, char *str)
{
unsigned long offset;
kallsyms_lookup(ip, NULL, &offset, modname, str);
if (offset > FTRACE_MCOUNT_MAX_OFFSET)
return -1;
return 0;
}
#else
static int lookup_ip(unsigned long ip, char **modname, char *str)
{
kallsyms_lookup(ip, NULL, NULL, modname, str);
return 0;
}
#endif
static int
ftrace_match_record(struct dyn_ftrace *rec, struct ftrace_glob *func_g,
struct ftrace_glob *mod_g, int exclude_mod)
{
char str[KSYM_SYMBOL_LEN];
char *modname;
if (lookup_ip(rec->ip, &modname, str)) {
WARN_ON_ONCE(system_state == SYSTEM_RUNNING &&
!(rec->flags & FTRACE_FL_DISABLED));
return 0;
}
if (mod_g) {
int mod_matches = (modname) ? ftrace_match(modname, mod_g) : 0;
if (!mod_g->len) {
if (!exclude_mod != !modname)
goto func_match;
return 0;
}
if (!mod_matches == !exclude_mod)
return 0;
func_match:
if (!func_g->len)
return 1;
}
return ftrace_match(str, func_g);
}
static int
match_records(struct ftrace_hash *hash, char *func, int len, char *mod)
{
struct ftrace_page *pg;
struct dyn_ftrace *rec;
struct ftrace_glob func_g = { .type = MATCH_FULL };
struct ftrace_glob mod_g = { .type = MATCH_FULL };
struct ftrace_glob *mod_match = (mod) ? &mod_g : NULL;
int exclude_mod = 0;
int found = 0;
int ret;
int clear_filter = 0;
if (func) {
func_g.type = filter_parse_regex(func, len, &func_g.search,
&clear_filter);
func_g.len = strlen(func_g.search);
}
if (mod) {
mod_g.type = filter_parse_regex(mod, strlen(mod),
&mod_g.search, &exclude_mod);
mod_g.len = strlen(mod_g.search);
}
mutex_lock(&ftrace_lock);
if (unlikely(ftrace_disabled))
goto out_unlock;
if (func_g.type == MATCH_INDEX) {
found = add_rec_by_index(hash, &func_g, clear_filter);
goto out_unlock;
}
do_for_each_ftrace_rec(pg, rec) {
if (rec->flags & FTRACE_FL_DISABLED)
continue;
if (ftrace_match_record(rec, &func_g, mod_match, exclude_mod)) {
ret = enter_record(hash, rec, clear_filter);
if (ret < 0) {
found = ret;
goto out_unlock;
}
found = 1;
}
cond_resched();
} while_for_each_ftrace_rec();
out_unlock:
mutex_unlock(&ftrace_lock);
return found;
}
static int
ftrace_match_records(struct ftrace_hash *hash, char *buff, int len)
{
return match_records(hash, buff, len, NULL);
}
static void ftrace_ops_update_code(struct ftrace_ops *ops,
struct ftrace_ops_hash *old_hash)
{
struct ftrace_ops *op;
if (!ftrace_enabled)
return;
if (ops->flags & FTRACE_OPS_FL_ENABLED) {
ftrace_run_modify_code(ops, FTRACE_UPDATE_CALLS, old_hash);
return;
}
if (ops->func_hash != &global_ops.local_hash)
return;
do_for_each_ftrace_op(op, ftrace_ops_list) {
if (op->func_hash == &global_ops.local_hash &&
op->flags & FTRACE_OPS_FL_ENABLED) {
ftrace_run_modify_code(op, FTRACE_UPDATE_CALLS, old_hash);
return;
}
} while_for_each_ftrace_op(op);
}
static int ftrace_hash_move_and_update_ops(struct ftrace_ops *ops,
struct ftrace_hash **orig_hash,
struct ftrace_hash *hash,
int enable)
{
struct ftrace_ops_hash old_hash_ops;
struct ftrace_hash *old_hash;
int ret;
old_hash = *orig_hash;
old_hash_ops.filter_hash = ops->func_hash->filter_hash;
old_hash_ops.notrace_hash = ops->func_hash->notrace_hash;
ret = ftrace_hash_move(ops, enable, orig_hash, hash);
if (!ret) {
ftrace_ops_update_code(ops, &old_hash_ops);
free_ftrace_hash_rcu(old_hash);
}
return ret;
}
static bool module_exists(const char *module)
{
static const char this_mod[] = "__this_module";
char modname[MAX_PARAM_PREFIX_LEN + sizeof(this_mod) + 2];
unsigned long val;
int n;
n = snprintf(modname, sizeof(modname), "%s:%s", module, this_mod);
if (n > sizeof(modname) - 1)
return false;
val = module_kallsyms_lookup_name(modname);
return val != 0;
}
static int cache_mod(struct trace_array *tr,
const char *func, char *module, int enable)
{
struct ftrace_mod_load *ftrace_mod, *n;
struct list_head *head = enable ? &tr->mod_trace : &tr->mod_notrace;
int ret;
mutex_lock(&ftrace_lock);
if (func[0] == '!') {
func++;
ret = -EINVAL;
list_for_each_entry_safe(ftrace_mod, n, head, list) {
if (strcmp(ftrace_mod->module, module) != 0)
continue;
if (strcmp(func, "*") == 0 ||
(ftrace_mod->func &&
strcmp(ftrace_mod->func, func) == 0)) {
ret = 0;
free_ftrace_mod(ftrace_mod);
continue;
}
}
goto out;
}
ret = -EINVAL;
if (module_exists(module))
goto out;
ret = ftrace_add_mod(tr, func, module, enable);
out:
mutex_unlock(&ftrace_lock);
return ret;
}
static int
ftrace_set_regex(struct ftrace_ops *ops, unsigned char *buf, int len,
int reset, int enable);
#ifdef CONFIG_MODULES
static void process_mod_list(struct list_head *head, struct ftrace_ops *ops,
char *mod, bool enable)
{
struct ftrace_mod_load *ftrace_mod, *n;
struct ftrace_hash **orig_hash, *new_hash;
LIST_HEAD(process_mods);
char *func;
mutex_lock(&ops->func_hash->regex_lock);
if (enable)
orig_hash = &ops->func_hash->filter_hash;
else
orig_hash = &ops->func_hash->notrace_hash;
new_hash = alloc_and_copy_ftrace_hash(FTRACE_HASH_DEFAULT_BITS,
*orig_hash);
if (!new_hash)
goto out;
mutex_lock(&ftrace_lock);
list_for_each_entry_safe(ftrace_mod, n, head, list) {
if (strcmp(ftrace_mod->module, mod) != 0)
continue;
if (ftrace_mod->func)
func = kstrdup(ftrace_mod->func, GFP_KERNEL);
else
func = kstrdup("*", GFP_KERNEL);
if (!func)
continue;
list_move(&ftrace_mod->list, &process_mods);
kfree(ftrace_mod->func);
ftrace_mod->func = func;
}
mutex_unlock(&ftrace_lock);
list_for_each_entry_safe(ftrace_mod, n, &process_mods, list) {
func = ftrace_mod->func;
match_records(new_hash, func, strlen(func), mod);
free_ftrace_mod(ftrace_mod);
}
if (enable && list_empty(head))
new_hash->flags &= ~FTRACE_HASH_FL_MOD;
mutex_lock(&ftrace_lock);
ftrace_hash_move_and_update_ops(ops, orig_hash,
new_hash, enable);
mutex_unlock(&ftrace_lock);
out:
mutex_unlock(&ops->func_hash->regex_lock);
free_ftrace_hash(new_hash);
}
static void process_cached_mods(const char *mod_name)
{
struct trace_array *tr;
char *mod;
mod = kstrdup(mod_name, GFP_KERNEL);
if (!mod)
return;
mutex_lock(&trace_types_lock);
list_for_each_entry(tr, &ftrace_trace_arrays, list) {
if (!list_empty(&tr->mod_trace))
process_mod_list(&tr->mod_trace, tr->ops, mod, true);
if (!list_empty(&tr->mod_notrace))
process_mod_list(&tr->mod_notrace, tr->ops, mod, false);
}
mutex_unlock(&trace_types_lock);
kfree(mod);
}
#endif
static int
ftrace_mod_callback(struct trace_array *tr, struct ftrace_hash *hash,
char *func_orig, char *cmd, char *module, int enable)
{
char *func;
int ret;
func = kstrdup(func_orig, GFP_KERNEL);
if (!func)
return -ENOMEM;
ret = match_records(hash, func, strlen(func), module);
kfree(func);
if (!ret)
return cache_mod(tr, func_orig, module, enable);
if (ret < 0)
return ret;
return 0;
}
static struct ftrace_func_command ftrace_mod_cmd = {
.name = "mod",
.func = ftrace_mod_callback,
};
static int __init ftrace_mod_cmd_init(void)
{
return register_ftrace_command(&ftrace_mod_cmd);
}
core_initcall(ftrace_mod_cmd_init);
static void function_trace_probe_call(unsigned long ip, unsigned long parent_ip,
struct ftrace_ops *op, struct ftrace_regs *fregs)
{
struct ftrace_probe_ops *probe_ops;
struct ftrace_func_probe *probe;
probe = container_of(op, struct ftrace_func_probe, ops);
probe_ops = probe->probe_ops;
preempt_disable_notrace();
probe_ops->func(ip, parent_ip, probe->tr, probe_ops, probe->data);
preempt_enable_notrace();
}
struct ftrace_func_map {
struct ftrace_func_entry entry;
void *data;
};
struct ftrace_func_mapper {
struct ftrace_hash hash;
};
struct ftrace_func_mapper *allocate_ftrace_func_mapper(void)
{
struct ftrace_hash *hash;
hash = alloc_ftrace_hash(FTRACE_HASH_DEFAULT_BITS);
return (struct ftrace_func_mapper *)hash;
}
void **ftrace_func_mapper_find_ip(struct ftrace_func_mapper *mapper,
unsigned long ip)
{
struct ftrace_func_entry *entry;
struct ftrace_func_map *map;
entry = ftrace_lookup_ip(&mapper->hash, ip);
if (!entry)
return NULL;
map = (struct ftrace_func_map *)entry;
return &map->data;
}
int ftrace_func_mapper_add_ip(struct ftrace_func_mapper *mapper,
unsigned long ip, void *data)
{
struct ftrace_func_entry *entry;
struct ftrace_func_map *map;
entry = ftrace_lookup_ip(&mapper->hash, ip);
if (entry)
return -EBUSY;
map = kmalloc(sizeof(*map), GFP_KERNEL);
if (!map)
return -ENOMEM;
map->entry.ip = ip;
map->data = data;
__add_hash_entry(&mapper->hash, &map->entry);
return 0;
}
void *ftrace_func_mapper_remove_ip(struct ftrace_func_mapper *mapper,
unsigned long ip)
{
struct ftrace_func_entry *entry;
struct ftrace_func_map *map;
void *data;
entry = ftrace_lookup_ip(&mapper->hash, ip);
if (!entry)
return NULL;
map = (struct ftrace_func_map *)entry;
data = map->data;
remove_hash_entry(&mapper->hash, entry);
kfree(entry);
return data;
}
void free_ftrace_func_mapper(struct ftrace_func_mapper *mapper,
ftrace_mapper_func free_func)
{
struct ftrace_func_entry *entry;
struct ftrace_func_map *map;
struct hlist_head *hhd;
int size, i;
if (!mapper)
return;
if (free_func && mapper->hash.count) {
size = 1 << mapper->hash.size_bits;
for (i = 0; i < size; i++) {
hhd = &mapper->hash.buckets[i];
hlist_for_each_entry(entry, hhd, hlist) {
map = (struct ftrace_func_map *)entry;
free_func(map);
}
}
}
free_ftrace_hash(&mapper->hash);
}
static void release_probe(struct ftrace_func_probe *probe)
{
struct ftrace_probe_ops *probe_ops;
mutex_lock(&ftrace_lock);
WARN_ON(probe->ref <= 0);
probe->ref--;
if (!probe->ref) {
probe_ops = probe->probe_ops;
if (probe_ops->free)
probe_ops->free(probe_ops, probe->tr, 0, probe->data);
list_del(&probe->list);
kfree(probe);
}
mutex_unlock(&ftrace_lock);
}
static void acquire_probe_locked(struct ftrace_func_probe *probe)
{
probe->ref++;
}
int
register_ftrace_function_probe(char *glob, struct trace_array *tr,
struct ftrace_probe_ops *probe_ops,
void *data)
{
struct ftrace_func_probe *probe = NULL, *iter;
struct ftrace_func_entry *entry;
struct ftrace_hash **orig_hash;
struct ftrace_hash *old_hash;
struct ftrace_hash *hash;
int count = 0;
int size;
int ret;
int i;
if (WARN_ON(!tr))
return -EINVAL;
if (WARN_ON(glob[0] == '!'))
return -EINVAL;
mutex_lock(&ftrace_lock);
list_for_each_entry(iter, &tr->func_probes, list) {
if (iter->probe_ops == probe_ops) {
probe = iter;
break;
}
}
if (!probe) {
probe = kzalloc(sizeof(*probe), GFP_KERNEL);
if (!probe) {
mutex_unlock(&ftrace_lock);
return -ENOMEM;
}
probe->probe_ops = probe_ops;
probe->ops.func = function_trace_probe_call;
probe->tr = tr;
ftrace_ops_init(&probe->ops);
list_add(&probe->list, &tr->func_probes);
}
acquire_probe_locked(probe);
mutex_unlock(&ftrace_lock);
mutex_lock(&probe->ops.func_hash->regex_lock);
orig_hash = &probe->ops.func_hash->filter_hash;
old_hash = *orig_hash;
hash = alloc_and_copy_ftrace_hash(FTRACE_HASH_DEFAULT_BITS, old_hash);
if (!hash) {
ret = -ENOMEM;
goto out;
}
ret = ftrace_match_records(hash, glob, strlen(glob));
if (!ret)
ret = -EINVAL;
if (ret < 0)
goto out;
size = 1 << hash->size_bits;
for (i = 0; i < size; i++) {
hlist_for_each_entry(entry, &hash->buckets[i], hlist) {
if (ftrace_lookup_ip(old_hash, entry->ip))
continue;
if (probe_ops->init) {
ret = probe_ops->init(probe_ops, tr,
entry->ip, data,
&probe->data);
if (ret < 0) {
if (probe_ops->free && count)
probe_ops->free(probe_ops, tr,
0, probe->data);
probe->data = NULL;
goto out;
}
}
count++;
}
}
mutex_lock(&ftrace_lock);
if (!count) {
ret = -EINVAL;
goto out_unlock;
}
ret = ftrace_hash_move_and_update_ops(&probe->ops, orig_hash,
hash, 1);
if (ret < 0)
goto err_unlock;
probe->ref += count;
if (!(probe->ops.flags & FTRACE_OPS_FL_ENABLED))
ret = ftrace_startup(&probe->ops, 0);
out_unlock:
mutex_unlock(&ftrace_lock);
if (!ret)
ret = count;
out:
mutex_unlock(&probe->ops.func_hash->regex_lock);
free_ftrace_hash(hash);
release_probe(probe);
return ret;
err_unlock:
if (!probe_ops->free || !count)
goto out_unlock;
for (i = 0; i < size; i++) {
hlist_for_each_entry(entry, &hash->buckets[i], hlist) {
if (ftrace_lookup_ip(old_hash, entry->ip))
continue;
probe_ops->free(probe_ops, tr, entry->ip, probe->data);
}
}
goto out_unlock;
}
int
unregister_ftrace_function_probe_func(char *glob, struct trace_array *tr,
struct ftrace_probe_ops *probe_ops)
{
struct ftrace_func_probe *probe = NULL, *iter;
struct ftrace_ops_hash old_hash_ops;
struct ftrace_func_entry *entry;
struct ftrace_glob func_g;
struct ftrace_hash **orig_hash;
struct ftrace_hash *old_hash;
struct ftrace_hash *hash = NULL;
struct hlist_node *tmp;
struct hlist_head hhd;
char str[KSYM_SYMBOL_LEN];
int count = 0;
int i, ret = -ENODEV;
int size;
if (!glob || !strlen(glob) || !strcmp(glob, "*"))
func_g.search = NULL;
else {
int not;
func_g.type = filter_parse_regex(glob, strlen(glob),
&func_g.search, ¬);
func_g.len = strlen(func_g.search);
if (WARN_ON(not))
return -EINVAL;
}
mutex_lock(&ftrace_lock);
list_for_each_entry(iter, &tr->func_probes, list) {
if (iter->probe_ops == probe_ops) {
probe = iter;
break;
}
}
if (!probe)
goto err_unlock_ftrace;
ret = -EINVAL;
if (!(probe->ops.flags & FTRACE_OPS_FL_INITIALIZED))
goto err_unlock_ftrace;
acquire_probe_locked(probe);
mutex_unlock(&ftrace_lock);
mutex_lock(&probe->ops.func_hash->regex_lock);
orig_hash = &probe->ops.func_hash->filter_hash;
old_hash = *orig_hash;
if (ftrace_hash_empty(old_hash))
goto out_unlock;
old_hash_ops.filter_hash = old_hash;
old_hash_ops.notrace_hash = NULL;
ret = -ENOMEM;
hash = alloc_and_copy_ftrace_hash(FTRACE_HASH_DEFAULT_BITS, old_hash);
if (!hash)
goto out_unlock;
INIT_HLIST_HEAD(&hhd);
size = 1 << hash->size_bits;
for (i = 0; i < size; i++) {
hlist_for_each_entry_safe(entry, tmp, &hash->buckets[i], hlist) {
if (func_g.search) {
kallsyms_lookup(entry->ip, NULL, NULL,
NULL, str);
if (!ftrace_match(str, &func_g))
continue;
}
count++;
remove_hash_entry(hash, entry);
hlist_add_head(&entry->hlist, &hhd);
}
}
if (!count) {
ret = -EINVAL;
goto out_unlock;
}
mutex_lock(&ftrace_lock);
WARN_ON(probe->ref < count);
probe->ref -= count;
if (ftrace_hash_empty(hash))
ftrace_shutdown(&probe->ops, 0);
ret = ftrace_hash_move_and_update_ops(&probe->ops, orig_hash,
hash, 1);
if (ftrace_enabled && !ftrace_hash_empty(hash))
ftrace_run_modify_code(&probe->ops, FTRACE_UPDATE_CALLS,
&old_hash_ops);
synchronize_rcu();
hlist_for_each_entry_safe(entry, tmp, &hhd, hlist) {
hlist_del(&entry->hlist);
if (probe_ops->free)
probe_ops->free(probe_ops, tr, entry->ip, probe->data);
kfree(entry);
}
mutex_unlock(&ftrace_lock);
out_unlock:
mutex_unlock(&probe->ops.func_hash->regex_lock);
free_ftrace_hash(hash);
release_probe(probe);
return ret;
err_unlock_ftrace:
mutex_unlock(&ftrace_lock);
return ret;
}
void clear_ftrace_function_probes(struct trace_array *tr)
{
struct ftrace_func_probe *probe, *n;
list_for_each_entry_safe(probe, n, &tr->func_probes, list)
unregister_ftrace_function_probe_func(NULL, tr, probe->probe_ops);
}
static LIST_HEAD(ftrace_commands);
static DEFINE_MUTEX(ftrace_cmd_mutex);
__init int register_ftrace_command(struct ftrace_func_command *cmd)
{
struct ftrace_func_command *p;
int ret = 0;
mutex_lock(&ftrace_cmd_mutex);
list_for_each_entry(p, &ftrace_commands, list) {
if (strcmp(cmd->name, p->name) == 0) {
ret = -EBUSY;
goto out_unlock;
}
}
list_add(&cmd->list, &ftrace_commands);
out_unlock:
mutex_unlock(&ftrace_cmd_mutex);
return ret;
}
__init int unregister_ftrace_command(struct ftrace_func_command *cmd)
{
struct ftrace_func_command *p, *n;
int ret = -ENODEV;
mutex_lock(&ftrace_cmd_mutex);
list_for_each_entry_safe(p, n, &ftrace_commands, list) {
if (strcmp(cmd->name, p->name) == 0) {
ret = 0;
list_del_init(&p->list);
goto out_unlock;
}
}
out_unlock:
mutex_unlock(&ftrace_cmd_mutex);
return ret;
}
static int ftrace_process_regex(struct ftrace_iterator *iter,
char *buff, int len, int enable)
{
struct ftrace_hash *hash = iter->hash;
struct trace_array *tr = iter->ops->private;
char *func, *command, *next = buff;
struct ftrace_func_command *p;
int ret = -EINVAL;
func = strsep(&next, ":");
if (!next) {
ret = ftrace_match_records(hash, func, len);
if (!ret)
ret = -EINVAL;
if (ret < 0)
return ret;
return 0;
}
command = strsep(&next, ":");
mutex_lock(&ftrace_cmd_mutex);
list_for_each_entry(p, &ftrace_commands, list) {
if (strcmp(p->name, command) == 0) {
ret = p->func(tr, hash, func, command, next, enable);
goto out_unlock;
}
}
out_unlock:
mutex_unlock(&ftrace_cmd_mutex);
return ret;
}
static ssize_t
ftrace_regex_write(struct file *file, const char __user *ubuf,
size_t cnt, loff_t *ppos, int enable)
{
struct ftrace_iterator *iter;
struct trace_parser *parser;
ssize_t ret, read;
if (!cnt)
return 0;
if (file->f_mode & FMODE_READ) {
struct seq_file *m = file->private_data;
iter = m->private;
} else
iter = file->private_data;
if (unlikely(ftrace_disabled))
return -ENODEV;
parser = &iter->parser;
read = trace_get_user(parser, ubuf, cnt, ppos);
if (read >= 0 && trace_parser_loaded(parser) &&
!trace_parser_cont(parser)) {
ret = ftrace_process_regex(iter, parser->buffer,
parser->idx, enable);
trace_parser_clear(parser);
if (ret < 0)
goto out;
}
ret = read;
out:
return ret;
}
ssize_t
ftrace_filter_write(struct file *file, const char __user *ubuf,
size_t cnt, loff_t *ppos)
{
return ftrace_regex_write(file, ubuf, cnt, ppos, 1);
}
ssize_t
ftrace_notrace_write(struct file *file, const char __user *ubuf,
size_t cnt, loff_t *ppos)
{
return ftrace_regex_write(file, ubuf, cnt, ppos, 0);
}
static int
__ftrace_match_addr(struct ftrace_hash *hash, unsigned long ip, int remove)
{
struct ftrace_func_entry *entry;
ip = ftrace_location(ip);
if (!ip)
return -EINVAL;
if (remove) {
entry = ftrace_lookup_ip(hash, ip);
if (!entry)
return -ENOENT;
free_hash_entry(hash, entry);
return 0;
}
return add_hash_entry(hash, ip);
}
static int
ftrace_match_addr(struct ftrace_hash *hash, unsigned long *ips,
unsigned int cnt, int remove)
{
unsigned int i;
int err;
for (i = 0; i < cnt; i++) {
err = __ftrace_match_addr(hash, ips[i], remove);
if (err) {
return err;
}
}
return 0;
}
static int
ftrace_set_hash(struct ftrace_ops *ops, unsigned char *buf, int len,
unsigned long *ips, unsigned int cnt,
int remove, int reset, int enable)
{
struct ftrace_hash **orig_hash;
struct ftrace_hash *hash;
int ret;
if (unlikely(ftrace_disabled))
return -ENODEV;
mutex_lock(&ops->func_hash->regex_lock);
if (enable)
orig_hash = &ops->func_hash->filter_hash;
else
orig_hash = &ops->func_hash->notrace_hash;
if (reset)
hash = alloc_ftrace_hash(FTRACE_HASH_DEFAULT_BITS);
else
hash = alloc_and_copy_ftrace_hash(FTRACE_HASH_DEFAULT_BITS, *orig_hash);
if (!hash) {
ret = -ENOMEM;
goto out_regex_unlock;
}
if (buf && !ftrace_match_records(hash, buf, len)) {
ret = -EINVAL;
goto out_regex_unlock;
}
if (ips) {
ret = ftrace_match_addr(hash, ips, cnt, remove);
if (ret < 0)
goto out_regex_unlock;
}
mutex_lock(&ftrace_lock);
ret = ftrace_hash_move_and_update_ops(ops, orig_hash, hash, enable);
mutex_unlock(&ftrace_lock);
out_regex_unlock:
mutex_unlock(&ops->func_hash->regex_lock);
free_ftrace_hash(hash);
return ret;
}
static int
ftrace_set_addr(struct ftrace_ops *ops, unsigned long *ips, unsigned int cnt,
int remove, int reset, int enable)
{
return ftrace_set_hash(ops, NULL, 0, ips, cnt, remove, reset, enable);
}
#ifdef CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS
struct ftrace_direct_func {
struct list_head next;
unsigned long addr;
int count;
};
static LIST_HEAD(ftrace_direct_funcs);
static int register_ftrace_function_nolock(struct ftrace_ops *ops);
#define MULTI_FLAGS (FTRACE_OPS_FL_DIRECT | FTRACE_OPS_FL_SAVE_ARGS)
static int check_direct_multi(struct ftrace_ops *ops)
{
if (!(ops->flags & FTRACE_OPS_FL_INITIALIZED))
return -EINVAL;
if ((ops->flags & MULTI_FLAGS) != MULTI_FLAGS)
return -EINVAL;
return 0;
}
static void remove_direct_functions_hash(struct ftrace_hash *hash, unsigned long addr)
{
struct ftrace_func_entry *entry, *del;
int size, i;
size = 1 << hash->size_bits;
for (i = 0; i < size; i++) {
hlist_for_each_entry(entry, &hash->buckets[i], hlist) {
del = __ftrace_lookup_ip(direct_functions, entry->ip);
if (del && del->direct == addr) {
remove_hash_entry(direct_functions, del);
kfree(del);
}
}
}
}
int register_ftrace_direct(struct ftrace_ops *ops, unsigned long addr)
{
struct ftrace_hash *hash, *free_hash = NULL;
struct ftrace_func_entry *entry, *new;
int err = -EBUSY, size, i;
if (ops->func || ops->trampoline)
return -EINVAL;
if (!(ops->flags & FTRACE_OPS_FL_INITIALIZED))
return -EINVAL;
if (ops->flags & FTRACE_OPS_FL_ENABLED)
return -EINVAL;
hash = ops->func_hash->filter_hash;
if (ftrace_hash_empty(hash))
return -EINVAL;
mutex_lock(&direct_mutex);
size = 1 << hash->size_bits;
for (i = 0; i < size; i++) {
hlist_for_each_entry(entry, &hash->buckets[i], hlist) {
if (ftrace_find_rec_direct(entry->ip))
goto out_unlock;
}
}
err = -ENOMEM;
for (i = 0; i < size; i++) {
hlist_for_each_entry(entry, &hash->buckets[i], hlist) {
new = ftrace_add_rec_direct(entry->ip, addr, &free_hash);
if (!new)
goto out_remove;
entry->direct = addr;
}
}
ops->func = call_direct_funcs;
ops->flags = MULTI_FLAGS;
ops->trampoline = FTRACE_REGS_ADDR;
ops->direct_call = addr;
err = register_ftrace_function_nolock(ops);
out_remove:
if (err)
remove_direct_functions_hash(hash, addr);
out_unlock:
mutex_unlock(&direct_mutex);
if (free_hash) {
synchronize_rcu_tasks();
free_ftrace_hash(free_hash);
}
return err;
}
EXPORT_SYMBOL_GPL(register_ftrace_direct);
int unregister_ftrace_direct(struct ftrace_ops *ops, unsigned long addr,
bool free_filters)
{
struct ftrace_hash *hash = ops->func_hash->filter_hash;
int err;
if (check_direct_multi(ops))
return -EINVAL;
if (!(ops->flags & FTRACE_OPS_FL_ENABLED))
return -EINVAL;
mutex_lock(&direct_mutex);
err = unregister_ftrace_function(ops);
remove_direct_functions_hash(hash, addr);
mutex_unlock(&direct_mutex);
ops->func = NULL;
ops->trampoline = 0;
if (free_filters)
ftrace_free_filter(ops);
return err;
}
EXPORT_SYMBOL_GPL(unregister_ftrace_direct);
static int
__modify_ftrace_direct(struct ftrace_ops *ops, unsigned long addr)
{
struct ftrace_hash *hash;
struct ftrace_func_entry *entry, *iter;
static struct ftrace_ops tmp_ops = {
.func = ftrace_stub,
.flags = FTRACE_OPS_FL_STUB,
};
int i, size;
int err;
lockdep_assert_held_once(&direct_mutex);
ftrace_ops_init(&tmp_ops);
tmp_ops.func_hash = ops->func_hash;
tmp_ops.direct_call = addr;
err = register_ftrace_function_nolock(&tmp_ops);
if (err)
return err;
mutex_lock(&ftrace_lock);
hash = ops->func_hash->filter_hash;
size = 1 << hash->size_bits;
for (i = 0; i < size; i++) {
hlist_for_each_entry(iter, &hash->buckets[i], hlist) {
entry = __ftrace_lookup_ip(direct_functions, iter->ip);
if (!entry)
continue;
entry->direct = addr;
}
}
WRITE_ONCE(ops->direct_call, addr);
mutex_unlock(&ftrace_lock);
unregister_ftrace_function(&tmp_ops);
return err;
}
int modify_ftrace_direct_nolock(struct ftrace_ops *ops, unsigned long addr)
{
if (check_direct_multi(ops))
return -EINVAL;
if (!(ops->flags & FTRACE_OPS_FL_ENABLED))
return -EINVAL;
return __modify_ftrace_direct(ops, addr);
}
EXPORT_SYMBOL_GPL(modify_ftrace_direct_nolock);
int modify_ftrace_direct(struct ftrace_ops *ops, unsigned long addr)
{
int err;
if (check_direct_multi(ops))
return -EINVAL;
if (!(ops->flags & FTRACE_OPS_FL_ENABLED))
return -EINVAL;
mutex_lock(&direct_mutex);
err = __modify_ftrace_direct(ops, addr);
mutex_unlock(&direct_mutex);
return err;
}
EXPORT_SYMBOL_GPL(modify_ftrace_direct);
#endif /* CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS */
int ftrace_set_filter_ip(struct ftrace_ops *ops, unsigned long ip,
int remove, int reset)
{
ftrace_ops_init(ops);
return ftrace_set_addr(ops, &ip, 1, remove, reset, 1);
}
EXPORT_SYMBOL_GPL(ftrace_set_filter_ip);
int ftrace_set_filter_ips(struct ftrace_ops *ops, unsigned long *ips,
unsigned int cnt, int remove, int reset)
{
ftrace_ops_init(ops);
return ftrace_set_addr(ops, ips, cnt, remove, reset, 1);
}
EXPORT_SYMBOL_GPL(ftrace_set_filter_ips);
void ftrace_ops_set_global_filter(struct ftrace_ops *ops)
{
if (ops->flags & FTRACE_OPS_FL_INITIALIZED)
return;
ftrace_ops_init(ops);
ops->func_hash = &global_ops.local_hash;
}
EXPORT_SYMBOL_GPL(ftrace_ops_set_global_filter);
static int
ftrace_set_regex(struct ftrace_ops *ops, unsigned char *buf, int len,
int reset, int enable)
{
return ftrace_set_hash(ops, buf, len, NULL, 0, 0, reset, enable);
}
int ftrace_set_filter(struct ftrace_ops *ops, unsigned char *buf,
int len, int reset)
{
ftrace_ops_init(ops);
return ftrace_set_regex(ops, buf, len, reset, 1);
}
EXPORT_SYMBOL_GPL(ftrace_set_filter);
int ftrace_set_notrace(struct ftrace_ops *ops, unsigned char *buf,
int len, int reset)
{
ftrace_ops_init(ops);
return ftrace_set_regex(ops, buf, len, reset, 0);
}
EXPORT_SYMBOL_GPL(ftrace_set_notrace);
void ftrace_set_global_filter(unsigned char *buf, int len, int reset)
{
ftrace_set_regex(&global_ops, buf, len, reset, 1);
}
EXPORT_SYMBOL_GPL(ftrace_set_global_filter);
void ftrace_set_global_notrace(unsigned char *buf, int len, int reset)
{
ftrace_set_regex(&global_ops, buf, len, reset, 0);
}
EXPORT_SYMBOL_GPL(ftrace_set_global_notrace);
#define FTRACE_FILTER_SIZE COMMAND_LINE_SIZE
static char ftrace_notrace_buf[FTRACE_FILTER_SIZE] __initdata;
static char ftrace_filter_buf[FTRACE_FILTER_SIZE] __initdata;
bool ftrace_filter_param __initdata;
static int __init set_ftrace_notrace(char *str)
{
ftrace_filter_param = true;
strscpy(ftrace_notrace_buf, str, FTRACE_FILTER_SIZE);
return 1;
}
__setup("ftrace_notrace=", set_ftrace_notrace);
static int __init set_ftrace_filter(char *str)
{
ftrace_filter_param = true;
strscpy(ftrace_filter_buf, str, FTRACE_FILTER_SIZE);
return 1;
}
__setup("ftrace_filter=", set_ftrace_filter);
#ifdef CONFIG_FUNCTION_GRAPH_TRACER
static char ftrace_graph_buf[FTRACE_FILTER_SIZE] __initdata;
static char ftrace_graph_notrace_buf[FTRACE_FILTER_SIZE] __initdata;
static int ftrace_graph_set_hash(struct ftrace_hash *hash, char *buffer);
static int __init set_graph_function(char *str)
{
strscpy(ftrace_graph_buf, str, FTRACE_FILTER_SIZE);
return 1;
}
__setup("ftrace_graph_filter=", set_graph_function);
static int __init set_graph_notrace_function(char *str)
{
strscpy(ftrace_graph_notrace_buf, str, FTRACE_FILTER_SIZE);
return 1;
}
__setup("ftrace_graph_notrace=", set_graph_notrace_function);
static int __init set_graph_max_depth_function(char *str)
{
if (!str)
return 0;
fgraph_max_depth = simple_strtoul(str, NULL, 0);
return 1;
}
__setup("ftrace_graph_max_depth=", set_graph_max_depth_function);
static void __init set_ftrace_early_graph(char *buf, int enable)
{
int ret;
char *func;
struct ftrace_hash *hash;
hash = alloc_ftrace_hash(FTRACE_HASH_DEFAULT_BITS);
if (MEM_FAIL(!hash, "Failed to allocate hash\n"))
return;
while (buf) {
func = strsep(&buf, ",");
ret = ftrace_graph_set_hash(hash, func);
if (ret)
printk(KERN_DEBUG "ftrace: function %s not "
"traceable\n", func);
}
if (enable)
ftrace_graph_hash = hash;
else
ftrace_graph_notrace_hash = hash;
}
#endif /* CONFIG_FUNCTION_GRAPH_TRACER */
void __init
ftrace_set_early_filter(struct ftrace_ops *ops, char *buf, int enable)
{
char *func;
ftrace_ops_init(ops);
while (buf) {
func = strsep(&buf, ",");
ftrace_set_regex(ops, func, strlen(func), 0, enable);
}
}
static void __init set_ftrace_early_filters(void)
{
if (ftrace_filter_buf[0])
ftrace_set_early_filter(&global_ops, ftrace_filter_buf, 1);
if (ftrace_notrace_buf[0])
ftrace_set_early_filter(&global_ops, ftrace_notrace_buf, 0);
#ifdef CONFIG_FUNCTION_GRAPH_TRACER
if (ftrace_graph_buf[0])
set_ftrace_early_graph(ftrace_graph_buf, 1);
if (ftrace_graph_notrace_buf[0])
set_ftrace_early_graph(ftrace_graph_notrace_buf, 0);
#endif /* CONFIG_FUNCTION_GRAPH_TRACER */
}
int ftrace_regex_release(struct inode *inode, struct file *file)
{
struct seq_file *m = (struct seq_file *)file->private_data;
struct ftrace_iterator *iter;
struct ftrace_hash **orig_hash;
struct trace_parser *parser;
int filter_hash;
if (file->f_mode & FMODE_READ) {
iter = m->private;
seq_release(inode, file);
} else
iter = file->private_data;
parser = &iter->parser;
if (trace_parser_loaded(parser)) {
int enable = !(iter->flags & FTRACE_ITER_NOTRACE);
ftrace_process_regex(iter, parser->buffer,
parser->idx, enable);
}
trace_parser_put(parser);
mutex_lock(&iter->ops->func_hash->regex_lock);
if (file->f_mode & FMODE_WRITE) {
filter_hash = !!(iter->flags & FTRACE_ITER_FILTER);
if (filter_hash) {
orig_hash = &iter->ops->func_hash->filter_hash;
if (iter->tr) {
if (list_empty(&iter->tr->mod_trace))
iter->hash->flags &= ~FTRACE_HASH_FL_MOD;
else
iter->hash->flags |= FTRACE_HASH_FL_MOD;
}
} else
orig_hash = &iter->ops->func_hash->notrace_hash;
mutex_lock(&ftrace_lock);
ftrace_hash_move_and_update_ops(iter->ops, orig_hash,
iter->hash, filter_hash);
mutex_unlock(&ftrace_lock);
} else {
iter->hash = NULL;
}
mutex_unlock(&iter->ops->func_hash->regex_lock);
free_ftrace_hash(iter->hash);
if (iter->tr)
trace_array_put(iter->tr);
kfree(iter);
return 0;
}
static const struct file_operations ftrace_avail_fops = {
.open = ftrace_avail_open,
.read = seq_read,
.llseek = seq_lseek,
.release = seq_release_private,
};
static const struct file_operations ftrace_enabled_fops = {
.open = ftrace_enabled_open,
.read = seq_read,
.llseek = seq_lseek,
.release = seq_release_private,
};
static const struct file_operations ftrace_touched_fops = {
.open = ftrace_touched_open,
.read = seq_read,
.llseek = seq_lseek,
.release = seq_release_private,
};
static const struct file_operations ftrace_avail_addrs_fops = {
.open = ftrace_avail_addrs_open,
.read = seq_read,
.llseek = seq_lseek,
.release = seq_release_private,
};
static const struct file_operations ftrace_filter_fops = {
.open = ftrace_filter_open,
.read = seq_read,
.write = ftrace_filter_write,
.llseek = tracing_lseek,
.release = ftrace_regex_release,
};
static const struct file_operations ftrace_notrace_fops = {
.open = ftrace_notrace_open,
.read = seq_read,
.write = ftrace_notrace_write,
.llseek = tracing_lseek,
.release = ftrace_regex_release,
};
#ifdef CONFIG_FUNCTION_GRAPH_TRACER
static DEFINE_MUTEX(graph_lock);
struct ftrace_hash __rcu *ftrace_graph_hash = EMPTY_HASH;
struct ftrace_hash __rcu *ftrace_graph_notrace_hash = EMPTY_HASH;
enum graph_filter_type {
GRAPH_FILTER_NOTRACE = 0,
GRAPH_FILTER_FUNCTION,
};
#define FTRACE_GRAPH_EMPTY ((void *)1)
struct ftrace_graph_data {
struct ftrace_hash *hash;
struct ftrace_func_entry *entry;
int idx;
enum graph_filter_type type;
struct ftrace_hash *new_hash;
const struct seq_operations *seq_ops;
struct trace_parser parser;
};
static void *
__g_next(struct seq_file *m, loff_t *pos)
{
struct ftrace_graph_data *fgd = m->private;
struct ftrace_func_entry *entry = fgd->entry;
struct hlist_head *head;
int i, idx = fgd->idx;
if (*pos >= fgd->hash->count)
return NULL;
if (entry) {
hlist_for_each_entry_continue(entry, hlist) {
fgd->entry = entry;
return entry;
}
idx++;
}
for (i = idx; i < 1 << fgd->hash->size_bits; i++) {
head = &fgd->hash->buckets[i];
hlist_for_each_entry(entry, head, hlist) {
fgd->entry = entry;
fgd->idx = i;
return entry;
}
}
return NULL;
}
static void *
g_next(struct seq_file *m, void *v, loff_t *pos)
{
(*pos)++;
return __g_next(m, pos);
}
static void *g_start(struct seq_file *m, loff_t *pos)
{
struct ftrace_graph_data *fgd = m->private;
mutex_lock(&graph_lock);
if (fgd->type == GRAPH_FILTER_FUNCTION)
fgd->hash = rcu_dereference_protected(ftrace_graph_hash,
lockdep_is_held(&graph_lock));
else
fgd->hash = rcu_dereference_protected(ftrace_graph_notrace_hash,
lockdep_is_held(&graph_lock));
if (ftrace_hash_empty(fgd->hash) && !*pos)
return FTRACE_GRAPH_EMPTY;
fgd->idx = 0;
fgd->entry = NULL;
return __g_next(m, pos);
}
static void g_stop(struct seq_file *m, void *p)
{
mutex_unlock(&graph_lock);
}
static int g_show(struct seq_file *m, void *v)
{
struct ftrace_func_entry *entry = v;
if (!entry)
return 0;
if (entry == FTRACE_GRAPH_EMPTY) {
struct ftrace_graph_data *fgd = m->private;
if (fgd->type == GRAPH_FILTER_FUNCTION)
seq_puts(m, "#### all functions enabled ####\n");
else
seq_puts(m, "#### no functions disabled ####\n");
return 0;
}
seq_printf(m, "%ps\n", (void *)entry->ip);
return 0;
}
static const struct seq_operations ftrace_graph_seq_ops = {
.start = g_start,
.next = g_next,
.stop = g_stop,
.show = g_show,
};
static int
__ftrace_graph_open(struct inode *inode, struct file *file,
struct ftrace_graph_data *fgd)
{
int ret;
struct ftrace_hash *new_hash = NULL;
ret = security_locked_down(LOCKDOWN_TRACEFS);
if (ret)
return ret;
if (file->f_mode & FMODE_WRITE) {
const int size_bits = FTRACE_HASH_DEFAULT_BITS;
if (trace_parser_get_init(&fgd->parser, FTRACE_BUFF_MAX))
return -ENOMEM;
if (file->f_flags & O_TRUNC)
new_hash = alloc_ftrace_hash(size_bits);
else
new_hash = alloc_and_copy_ftrace_hash(size_bits,
fgd->hash);
if (!new_hash) {
ret = -ENOMEM;
goto out;
}
}
if (file->f_mode & FMODE_READ) {
ret = seq_open(file, &ftrace_graph_seq_ops);
if (!ret) {
struct seq_file *m = file->private_data;
m->private = fgd;
} else {
free_ftrace_hash(new_hash);
new_hash = NULL;
}
} else
file->private_data = fgd;
out:
if (ret < 0 && file->f_mode & FMODE_WRITE)
trace_parser_put(&fgd->parser);
fgd->new_hash = new_hash;
fgd->hash = NULL;
return ret;
}
static int
ftrace_graph_open(struct inode *inode, struct file *file)
{
struct ftrace_graph_data *fgd;
int ret;
if (unlikely(ftrace_disabled))
return -ENODEV;
fgd = kmalloc(sizeof(*fgd), GFP_KERNEL);
if (fgd == NULL)
return -ENOMEM;
mutex_lock(&graph_lock);
fgd->hash = rcu_dereference_protected(ftrace_graph_hash,
lockdep_is_held(&graph_lock));
fgd->type = GRAPH_FILTER_FUNCTION;
fgd->seq_ops = &ftrace_graph_seq_ops;
ret = __ftrace_graph_open(inode, file, fgd);
if (ret < 0)
kfree(fgd);
mutex_unlock(&graph_lock);
return ret;
}
static int
ftrace_graph_notrace_open(struct inode *inode, struct file *file)
{
struct ftrace_graph_data *fgd;
int ret;
if (unlikely(ftrace_disabled))
return -ENODEV;
fgd = kmalloc(sizeof(*fgd), GFP_KERNEL);
if (fgd == NULL)
return -ENOMEM;
mutex_lock(&graph_lock);
fgd->hash = rcu_dereference_protected(ftrace_graph_notrace_hash,
lockdep_is_held(&graph_lock));
fgd->type = GRAPH_FILTER_NOTRACE;
fgd->seq_ops = &ftrace_graph_seq_ops;
ret = __ftrace_graph_open(inode, file, fgd);
if (ret < 0)
kfree(fgd);
mutex_unlock(&graph_lock);
return ret;
}
static int
ftrace_graph_release(struct inode *inode, struct file *file)
{
struct ftrace_graph_data *fgd;
struct ftrace_hash *old_hash, *new_hash;
struct trace_parser *parser;
int ret = 0;
if (file->f_mode & FMODE_READ) {
struct seq_file *m = file->private_data;
fgd = m->private;
seq_release(inode, file);
} else {
fgd = file->private_data;
}
if (file->f_mode & FMODE_WRITE) {
parser = &fgd->parser;
if (trace_parser_loaded((parser))) {
ret = ftrace_graph_set_hash(fgd->new_hash,
parser->buffer);
}
trace_parser_put(parser);
new_hash = __ftrace_hash_move(fgd->new_hash);
if (!new_hash) {
ret = -ENOMEM;
goto out;
}
mutex_lock(&graph_lock);
if (fgd->type == GRAPH_FILTER_FUNCTION) {
old_hash = rcu_dereference_protected(ftrace_graph_hash,
lockdep_is_held(&graph_lock));
rcu_assign_pointer(ftrace_graph_hash, new_hash);
} else {
old_hash = rcu_dereference_protected(ftrace_graph_notrace_hash,
lockdep_is_held(&graph_lock));
rcu_assign_pointer(ftrace_graph_notrace_hash, new_hash);
}
mutex_unlock(&graph_lock);
if (old_hash != EMPTY_HASH)
synchronize_rcu_tasks_rude();
free_ftrace_hash(old_hash);
}
out:
free_ftrace_hash(fgd->new_hash);
kfree(fgd);
return ret;
}
static int
ftrace_graph_set_hash(struct ftrace_hash *hash, char *buffer)
{
struct ftrace_glob func_g;
struct dyn_ftrace *rec;
struct ftrace_page *pg;
struct ftrace_func_entry *entry;
int fail = 1;
int not;
func_g.type = filter_parse_regex(buffer, strlen(buffer),
&func_g.search, ¬);
func_g.len = strlen(func_g.search);
mutex_lock(&ftrace_lock);
if (unlikely(ftrace_disabled)) {
mutex_unlock(&ftrace_lock);
return -ENODEV;
}
do_for_each_ftrace_rec(pg, rec) {
if (rec->flags & FTRACE_FL_DISABLED)
continue;
if (ftrace_match_record(rec, &func_g, NULL, 0)) {
entry = ftrace_lookup_ip(hash, rec->ip);
if (!not) {
fail = 0;
if (entry)
continue;
if (add_hash_entry(hash, rec->ip) < 0)
goto out;
} else {
if (entry) {
free_hash_entry(hash, entry);
fail = 0;
}
}
}
} while_for_each_ftrace_rec();
out:
mutex_unlock(&ftrace_lock);
if (fail)
return -EINVAL;
return 0;
}
static ssize_t
ftrace_graph_write(struct file *file, const char __user *ubuf,
size_t cnt, loff_t *ppos)
{
ssize_t read, ret = 0;
struct ftrace_graph_data *fgd = file->private_data;
struct trace_parser *parser;
if (!cnt)
return 0;
if (file->f_mode & FMODE_READ) {
struct seq_file *m = file->private_data;
fgd = m->private;
}
parser = &fgd->parser;
read = trace_get_user(parser, ubuf, cnt, ppos);
if (read >= 0 && trace_parser_loaded(parser) &&
!trace_parser_cont(parser)) {
ret = ftrace_graph_set_hash(fgd->new_hash,
parser->buffer);
trace_parser_clear(parser);
}
if (!ret)
ret = read;
return ret;
}
static const struct file_operations ftrace_graph_fops = {
.open = ftrace_graph_open,
.read = seq_read,
.write = ftrace_graph_write,
.llseek = tracing_lseek,
.release = ftrace_graph_release,
};
static const struct file_operations ftrace_graph_notrace_fops = {
.open = ftrace_graph_notrace_open,
.read = seq_read,
.write = ftrace_graph_write,
.llseek = tracing_lseek,
.release = ftrace_graph_release,
};
#endif /* CONFIG_FUNCTION_GRAPH_TRACER */
void ftrace_create_filter_files(struct ftrace_ops *ops,
struct dentry *parent)
{
trace_create_file("set_ftrace_filter", TRACE_MODE_WRITE, parent,
ops, &ftrace_filter_fops);
trace_create_file("set_ftrace_notrace", TRACE_MODE_WRITE, parent,
ops, &ftrace_notrace_fops);
}
void ftrace_destroy_filter_files(struct ftrace_ops *ops)
{
mutex_lock(&ftrace_lock);
if (ops->flags & FTRACE_OPS_FL_ENABLED)
ftrace_shutdown(ops, 0);
ops->flags |= FTRACE_OPS_FL_DELETED;
ftrace_free_filter(ops);
mutex_unlock(&ftrace_lock);
}
static __init int ftrace_init_dyn_tracefs(struct dentry *d_tracer)
{
trace_create_file("available_filter_functions", TRACE_MODE_READ,
d_tracer, NULL, &ftrace_avail_fops);
trace_create_file("available_filter_functions_addrs", TRACE_MODE_READ,
d_tracer, NULL, &ftrace_avail_addrs_fops);
trace_create_file("enabled_functions", TRACE_MODE_READ,
d_tracer, NULL, &ftrace_enabled_fops);
trace_create_file("touched_functions", TRACE_MODE_READ,
d_tracer, NULL, &ftrace_touched_fops);
ftrace_create_filter_files(&global_ops, d_tracer);
#ifdef CONFIG_FUNCTION_GRAPH_TRACER
trace_create_file("set_graph_function", TRACE_MODE_WRITE, d_tracer,
NULL,
&ftrace_graph_fops);
trace_create_file("set_graph_notrace", TRACE_MODE_WRITE, d_tracer,
NULL,
&ftrace_graph_notrace_fops);
#endif /* CONFIG_FUNCTION_GRAPH_TRACER */
return 0;
}
static int ftrace_cmp_ips(const void *a, const void *b)
{
const unsigned long *ipa = a;
const unsigned long *ipb = b;
if (*ipa > *ipb)
return 1;
if (*ipa < *ipb)
return -1;
return 0;
}
#ifdef CONFIG_FTRACE_SORT_STARTUP_TEST
static void test_is_sorted(unsigned long *start, unsigned long count)
{
int i;
for (i = 1; i < count; i++) {
if (WARN(start[i - 1] > start[i],
"[%d] %pS at %lx is not sorted with %pS at %lx\n", i,
(void *)start[i - 1], start[i - 1],
(void *)start[i], start[i]))
break;
}
if (i == count)
pr_info("ftrace section at %px sorted properly\n", start);
}
#else
static void test_is_sorted(unsigned long *start, unsigned long count)
{
}
#endif
static int ftrace_process_locs(struct module *mod,
unsigned long *start,
unsigned long *end)
{
struct ftrace_page *pg_unuse = NULL;
struct ftrace_page *start_pg;
struct ftrace_page *pg;
struct dyn_ftrace *rec;
unsigned long skipped = 0;
unsigned long count;
unsigned long *p;
unsigned long addr;
unsigned long flags = 0;
int ret = -ENOMEM;
count = end - start;
if (!count)
return 0;
if (!IS_ENABLED(CONFIG_BUILDTIME_MCOUNT_SORT) || mod) {
sort(start, count, sizeof(*start),
ftrace_cmp_ips, NULL);
} else {
test_is_sorted(start, count);
}
start_pg = ftrace_allocate_pages(count);
if (!start_pg)
return -ENOMEM;
mutex_lock(&ftrace_lock);
if (!mod) {
WARN_ON(ftrace_pages || ftrace_pages_start);
ftrace_pages = ftrace_pages_start = start_pg;
} else {
if (!ftrace_pages)
goto out;
if (WARN_ON(ftrace_pages->next)) {
while (ftrace_pages->next)
ftrace_pages = ftrace_pages->next;
}
ftrace_pages->next = start_pg;
}
p = start;
pg = start_pg;
while (p < end) {
unsigned long end_offset;
addr = ftrace_call_adjust(*p++);
if (!addr) {
skipped++;
continue;
}
end_offset = (pg->index+1) * sizeof(pg->records[0]);
if (end_offset > PAGE_SIZE << pg->order) {
if (WARN_ON(!pg->next))
break;
pg = pg->next;
}
rec = &pg->records[pg->index++];
rec->ip = addr;
}
if (pg->next) {
pg_unuse = pg->next;
pg->next = NULL;
}
ftrace_pages = pg;
if (!mod)
local_irq_save(flags);
ftrace_update_code(mod, start_pg);
if (!mod)
local_irq_restore(flags);
ret = 0;
out:
mutex_unlock(&ftrace_lock);
if (pg_unuse) {
WARN_ON(!skipped);
ftrace_free_pages(pg_unuse);
}
return ret;
}
struct ftrace_mod_func {
struct list_head list;
char *name;
unsigned long ip;
unsigned int size;
};
struct ftrace_mod_map {
struct rcu_head rcu;
struct list_head list;
struct module *mod;
unsigned long start_addr;
unsigned long end_addr;
struct list_head funcs;
unsigned int num_funcs;
};
static int ftrace_get_trampoline_kallsym(unsigned int symnum,
unsigned long *value, char *type,
char *name, char *module_name,
int *exported)
{
struct ftrace_ops *op;
list_for_each_entry_rcu(op, &ftrace_ops_trampoline_list, list) {
if (!op->trampoline || symnum--)
continue;
*value = op->trampoline;
*type = 't';
strscpy(name, FTRACE_TRAMPOLINE_SYM, KSYM_NAME_LEN);
strscpy(module_name, FTRACE_TRAMPOLINE_MOD, MODULE_NAME_LEN);
*exported = 0;
return 0;
}
return -ERANGE;
}
#if defined(CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS) || defined(CONFIG_MODULES)
static bool
ops_references_ip(struct ftrace_ops *ops, unsigned long ip)
{
if (!(ops->flags & FTRACE_OPS_FL_ENABLED))
return false;
if (ops_traces_mod(ops))
return true;
if (!ftrace_hash_empty(ops->func_hash->filter_hash) &&
!__ftrace_lookup_ip(ops->func_hash->filter_hash, ip))
return false;
if (ftrace_lookup_ip(ops->func_hash->notrace_hash, ip))
return false;
return true;
}
#endif
#ifdef CONFIG_MODULES
#define next_to_ftrace_page(p) container_of(p, struct ftrace_page, next)
static LIST_HEAD(ftrace_mod_maps);
static int referenced_filters(struct dyn_ftrace *rec)
{
struct ftrace_ops *ops;
int cnt = 0;
for (ops = ftrace_ops_list; ops != &ftrace_list_end; ops = ops->next) {
if (ops_references_ip(ops, rec->ip)) {
if (WARN_ON_ONCE(ops->flags & FTRACE_OPS_FL_DIRECT))
continue;
if (WARN_ON_ONCE(ops->flags & FTRACE_OPS_FL_IPMODIFY))
continue;
cnt++;
if (ops->flags & FTRACE_OPS_FL_SAVE_REGS)
rec->flags |= FTRACE_FL_REGS;
if (cnt == 1 && ops->trampoline)
rec->flags |= FTRACE_FL_TRAMP;
else
rec->flags &= ~FTRACE_FL_TRAMP;
}
}
return cnt;
}
static void
clear_mod_from_hash(struct ftrace_page *pg, struct ftrace_hash *hash)
{
struct ftrace_func_entry *entry;
struct dyn_ftrace *rec;
int i;
if (ftrace_hash_empty(hash))
return;
for (i = 0; i < pg->index; i++) {
rec = &pg->records[i];
entry = __ftrace_lookup_ip(hash, rec->ip);
if (entry)
entry->ip = 0;
}
}
static void clear_mod_from_hashes(struct ftrace_page *pg)
{
struct trace_array *tr;
mutex_lock(&trace_types_lock);
list_for_each_entry(tr, &ftrace_trace_arrays, list) {
if (!tr->ops || !tr->ops->func_hash)
continue;
mutex_lock(&tr->ops->func_hash->regex_lock);
clear_mod_from_hash(pg, tr->ops->func_hash->filter_hash);
clear_mod_from_hash(pg, tr->ops->func_hash->notrace_hash);
mutex_unlock(&tr->ops->func_hash->regex_lock);
}
mutex_unlock(&trace_types_lock);
}
static void ftrace_free_mod_map(struct rcu_head *rcu)
{
struct ftrace_mod_map *mod_map = container_of(rcu, struct ftrace_mod_map, rcu);
struct ftrace_mod_func *mod_func;
struct ftrace_mod_func *n;
list_for_each_entry_safe(mod_func, n, &mod_map->funcs, list) {
kfree(mod_func->name);
list_del(&mod_func->list);
kfree(mod_func);
}
kfree(mod_map);
}
void ftrace_release_mod(struct module *mod)
{
struct ftrace_mod_map *mod_map;
struct ftrace_mod_map *n;
struct dyn_ftrace *rec;
struct ftrace_page **last_pg;
struct ftrace_page *tmp_page = NULL;
struct ftrace_page *pg;
mutex_lock(&ftrace_lock);
if (ftrace_disabled)
goto out_unlock;
list_for_each_entry_safe(mod_map, n, &ftrace_mod_maps, list) {
if (mod_map->mod == mod) {
list_del_rcu(&mod_map->list);
call_rcu(&mod_map->rcu, ftrace_free_mod_map);
break;
}
}
last_pg = &ftrace_pages_start;
for (pg = ftrace_pages_start; pg; pg = *last_pg) {
rec = &pg->records[0];
if (within_module(rec->ip, mod)) {
if (WARN_ON(pg == ftrace_pages_start))
goto out_unlock;
if (pg == ftrace_pages)
ftrace_pages = next_to_ftrace_page(last_pg);
ftrace_update_tot_cnt -= pg->index;
*last_pg = pg->next;
pg->next = tmp_page;
tmp_page = pg;
} else
last_pg = &pg->next;
}
out_unlock:
mutex_unlock(&ftrace_lock);
for (pg = tmp_page; pg; pg = tmp_page) {
clear_mod_from_hashes(pg);
if (pg->records) {
free_pages((unsigned long)pg->records, pg->order);
ftrace_number_of_pages -= 1 << pg->order;
}
tmp_page = pg->next;
kfree(pg);
ftrace_number_of_groups--;
}
}
void ftrace_module_enable(struct module *mod)
{
struct dyn_ftrace *rec;
struct ftrace_page *pg;
mutex_lock(&ftrace_lock);
if (ftrace_disabled)
goto out_unlock;
if (ftrace_start_up)
ftrace_arch_code_modify_prepare();
do_for_each_ftrace_rec(pg, rec) {
int cnt;
if (!within_module(rec->ip, mod))
break;
if (!test_for_valid_rec(rec)) {
rec->flags = FTRACE_FL_DISABLED;
continue;
}
cnt = 0;
if (ftrace_start_up)
cnt += referenced_filters(rec);
rec->flags &= ~FTRACE_FL_DISABLED;
rec->flags += cnt;
if (ftrace_start_up && cnt) {
int failed = __ftrace_replace_code(rec, 1);
if (failed) {
ftrace_bug(failed, rec);
goto out_loop;
}
}
} while_for_each_ftrace_rec();
out_loop:
if (ftrace_start_up)
ftrace_arch_code_modify_post_process();
out_unlock:
mutex_unlock(&ftrace_lock);
process_cached_mods(mod->name);
}
void ftrace_module_init(struct module *mod)
{
int ret;
if (ftrace_disabled || !mod->num_ftrace_callsites)
return;
ret = ftrace_process_locs(mod, mod->ftrace_callsites,
mod->ftrace_callsites + mod->num_ftrace_callsites);
if (ret)
pr_warn("ftrace: failed to allocate entries for module '%s' functions\n",
mod->name);
}
static void save_ftrace_mod_rec(struct ftrace_mod_map *mod_map,
struct dyn_ftrace *rec)
{
struct ftrace_mod_func *mod_func;
unsigned long symsize;
unsigned long offset;
char str[KSYM_SYMBOL_LEN];
char *modname;
const char *ret;
ret = kallsyms_lookup(rec->ip, &symsize, &offset, &modname, str);
if (!ret)
return;
mod_func = kmalloc(sizeof(*mod_func), GFP_KERNEL);
if (!mod_func)
return;
mod_func->name = kstrdup(str, GFP_KERNEL);
if (!mod_func->name) {
kfree(mod_func);
return;
}
mod_func->ip = rec->ip - offset;
mod_func->size = symsize;
mod_map->num_funcs++;
list_add_rcu(&mod_func->list, &mod_map->funcs);
}
static struct ftrace_mod_map *
allocate_ftrace_mod_map(struct module *mod,
unsigned long start, unsigned long end)
{
struct ftrace_mod_map *mod_map;
mod_map = kmalloc(sizeof(*mod_map), GFP_KERNEL);
if (!mod_map)
return NULL;
mod_map->mod = mod;
mod_map->start_addr = start;
mod_map->end_addr = end;
mod_map->num_funcs = 0;
INIT_LIST_HEAD_RCU(&mod_map->funcs);
list_add_rcu(&mod_map->list, &ftrace_mod_maps);
return mod_map;
}
static const char *
ftrace_func_address_lookup(struct ftrace_mod_map *mod_map,
unsigned long addr, unsigned long *size,
unsigned long *off, char *sym)
{
struct ftrace_mod_func *found_func = NULL;
struct ftrace_mod_func *mod_func;
list_for_each_entry_rcu(mod_func, &mod_map->funcs, list) {
if (addr >= mod_func->ip &&
addr < mod_func->ip + mod_func->size) {
found_func = mod_func;
break;
}
}
if (found_func) {
if (size)
*size = found_func->size;
if (off)
*off = addr - found_func->ip;
if (sym)
strscpy(sym, found_func->name, KSYM_NAME_LEN);
return found_func->name;
}
return NULL;
}
const char *
ftrace_mod_address_lookup(unsigned long addr, unsigned long *size,
unsigned long *off, char **modname, char *sym)
{
struct ftrace_mod_map *mod_map;
const char *ret = NULL;
preempt_disable();
list_for_each_entry_rcu(mod_map, &ftrace_mod_maps, list) {
ret = ftrace_func_address_lookup(mod_map, addr, size, off, sym);
if (ret) {
if (modname)
*modname = mod_map->mod->name;
break;
}
}
preempt_enable();
return ret;
}
int ftrace_mod_get_kallsym(unsigned int symnum, unsigned long *value,
char *type, char *name,
char *module_name, int *exported)
{
struct ftrace_mod_map *mod_map;
struct ftrace_mod_func *mod_func;
int ret;
preempt_disable();
list_for_each_entry_rcu(mod_map, &ftrace_mod_maps, list) {
if (symnum >= mod_map->num_funcs) {
symnum -= mod_map->num_funcs;
continue;
}
list_for_each_entry_rcu(mod_func, &mod_map->funcs, list) {
if (symnum > 1) {
symnum--;
continue;
}
*value = mod_func->ip;
*type = 'T';
strscpy(name, mod_func->name, KSYM_NAME_LEN);
strscpy(module_name, mod_map->mod->name, MODULE_NAME_LEN);
*exported = 1;
preempt_enable();
return 0;
}
WARN_ON(1);
break;
}
ret = ftrace_get_trampoline_kallsym(symnum, value, type, name,
module_name, exported);
preempt_enable();
return ret;
}
#else
static void save_ftrace_mod_rec(struct ftrace_mod_map *mod_map,
struct dyn_ftrace *rec) { }
static inline struct ftrace_mod_map *
allocate_ftrace_mod_map(struct module *mod,
unsigned long start, unsigned long end)
{
return NULL;
}
int ftrace_mod_get_kallsym(unsigned int symnum, unsigned long *value,
char *type, char *name, char *module_name,
int *exported)
{
int ret;
preempt_disable();
ret = ftrace_get_trampoline_kallsym(symnum, value, type, name,
module_name, exported);
preempt_enable();
return ret;
}
#endif /* CONFIG_MODULES */
struct ftrace_init_func {
struct list_head list;
unsigned long ip;
};
static void
clear_func_from_hash(struct ftrace_init_func *func, struct ftrace_hash *hash)
{
struct ftrace_func_entry *entry;
entry = ftrace_lookup_ip(hash, func->ip);
if (entry)
entry->ip = 0;
}
static void
clear_func_from_hashes(struct ftrace_init_func *func)
{
struct trace_array *tr;
mutex_lock(&trace_types_lock);
list_for_each_entry(tr, &ftrace_trace_arrays, list) {
if (!tr->ops || !tr->ops->func_hash)
continue;
mutex_lock(&tr->ops->func_hash->regex_lock);
clear_func_from_hash(func, tr->ops->func_hash->filter_hash);
clear_func_from_hash(func, tr->ops->func_hash->notrace_hash);
mutex_unlock(&tr->ops->func_hash->regex_lock);
}
mutex_unlock(&trace_types_lock);
}
static void add_to_clear_hash_list(struct list_head *clear_list,
struct dyn_ftrace *rec)
{
struct ftrace_init_func *func;
func = kmalloc(sizeof(*func), GFP_KERNEL);
if (!func) {
MEM_FAIL(1, "alloc failure, ftrace filter could be stale\n");
return;
}
func->ip = rec->ip;
list_add(&func->list, clear_list);
}
void ftrace_free_mem(struct module *mod, void *start_ptr, void *end_ptr)
{
unsigned long start = (unsigned long)(start_ptr);
unsigned long end = (unsigned long)(end_ptr);
struct ftrace_page **last_pg = &ftrace_pages_start;
struct ftrace_page *pg;
struct dyn_ftrace *rec;
struct dyn_ftrace key;
struct ftrace_mod_map *mod_map = NULL;
struct ftrace_init_func *func, *func_next;
LIST_HEAD(clear_hash);
key.ip = start;
key.flags = end;
mutex_lock(&ftrace_lock);
if (mod && ftrace_ops_list != &ftrace_list_end)
mod_map = allocate_ftrace_mod_map(mod, start, end);
for (pg = ftrace_pages_start; pg; last_pg = &pg->next, pg = *last_pg) {
if (end < pg->records[0].ip ||
start >= (pg->records[pg->index - 1].ip + MCOUNT_INSN_SIZE))
continue;
again:
rec = bsearch(&key, pg->records, pg->index,
sizeof(struct dyn_ftrace),
ftrace_cmp_recs);
if (!rec)
continue;
add_to_clear_hash_list(&clear_hash, rec);
if (mod_map)
save_ftrace_mod_rec(mod_map, rec);
pg->index--;
ftrace_update_tot_cnt--;
if (!pg->index) {
*last_pg = pg->next;
if (pg->records) {
free_pages((unsigned long)pg->records, pg->order);
ftrace_number_of_pages -= 1 << pg->order;
}
ftrace_number_of_groups--;
kfree(pg);
pg = container_of(last_pg, struct ftrace_page, next);
if (!(*last_pg))
ftrace_pages = pg;
continue;
}
memmove(rec, rec + 1,
(pg->index - (rec - pg->records)) * sizeof(*rec));
goto again;
}
mutex_unlock(&ftrace_lock);
list_for_each_entry_safe(func, func_next, &clear_hash, list) {
clear_func_from_hashes(func);
kfree(func);
}
}
void __init ftrace_free_init_mem(void)
{
void *start = (void *)(&__init_begin);
void *end = (void *)(&__init_end);
ftrace_boot_snapshot();
ftrace_free_mem(NULL, start, end);
}
int __init __weak ftrace_dyn_arch_init(void)
{
return 0;
}
void __init ftrace_init(void)
{
extern unsigned long __start_mcount_loc[];
extern unsigned long __stop_mcount_loc[];
unsigned long count, flags;
int ret;
local_irq_save(flags);
ret = ftrace_dyn_arch_init();
local_irq_restore(flags);
if (ret)
goto failed;
count = __stop_mcount_loc - __start_mcount_loc;
if (!count) {
pr_info("ftrace: No functions to be traced?\n");
goto failed;
}
pr_info("ftrace: allocating %ld entries in %ld pages\n",
count, DIV_ROUND_UP(count, ENTRIES_PER_PAGE));
ret = ftrace_process_locs(NULL,
__start_mcount_loc,
__stop_mcount_loc);
if (ret) {
pr_warn("ftrace: failed to allocate entries for functions\n");
goto failed;
}
pr_info("ftrace: allocated %ld pages with %ld groups\n",
ftrace_number_of_pages, ftrace_number_of_groups);
last_ftrace_enabled = ftrace_enabled = 1;
set_ftrace_early_filters();
return;
failed:
ftrace_disabled = 1;
}
void __weak arch_ftrace_update_trampoline(struct ftrace_ops *ops)
{
}
static void ftrace_update_trampoline(struct ftrace_ops *ops)
{
unsigned long trampoline = ops->trampoline;
arch_ftrace_update_trampoline(ops);
if (ops->trampoline && ops->trampoline != trampoline &&
(ops->flags & FTRACE_OPS_FL_ALLOC_TRAMP)) {
ftrace_add_trampoline_to_kallsyms(ops);
perf_event_ksymbol(PERF_RECORD_KSYMBOL_TYPE_OOL,
ops->trampoline, ops->trampoline_size, false,
FTRACE_TRAMPOLINE_SYM);
perf_event_text_poke((void *)ops->trampoline, NULL, 0,
(void *)ops->trampoline,
ops->trampoline_size);
}
}
void ftrace_init_trace_array(struct trace_array *tr)
{
INIT_LIST_HEAD(&tr->func_probes);
INIT_LIST_HEAD(&tr->mod_trace);
INIT_LIST_HEAD(&tr->mod_notrace);
}
#else
struct ftrace_ops global_ops = {
.func = ftrace_stub,
.flags = FTRACE_OPS_FL_INITIALIZED |
FTRACE_OPS_FL_PID,
};
static int __init ftrace_nodyn_init(void)
{
ftrace_enabled = 1;
return 0;
}
core_initcall(ftrace_nodyn_init);
static inline int ftrace_init_dyn_tracefs(struct dentry *d_tracer) { return 0; }
static inline void ftrace_startup_all(int command) { }
static void ftrace_update_trampoline(struct ftrace_ops *ops)
{
}
#endif /* CONFIG_DYNAMIC_FTRACE */
__init void ftrace_init_global_array_ops(struct trace_array *tr)
{
tr->ops = &global_ops;
tr->ops->private = tr;
ftrace_init_trace_array(tr);
}
void ftrace_init_array_ops(struct trace_array *tr, ftrace_func_t func)
{
if (tr->flags & TRACE_ARRAY_FL_GLOBAL) {
if (WARN_ON(tr->ops->func != ftrace_stub))
printk("ftrace ops had %pS for function\n",
tr->ops->func);
}
tr->ops->func = func;
tr->ops->private = tr;
}
void ftrace_reset_array_ops(struct trace_array *tr)
{
tr->ops->func = ftrace_stub;
}
static nokprobe_inline void
__ftrace_ops_list_func(unsigned long ip, unsigned long parent_ip,
struct ftrace_ops *ignored, struct ftrace_regs *fregs)
{
struct pt_regs *regs = ftrace_get_regs(fregs);
struct ftrace_ops *op;
int bit;
bit = trace_test_and_set_recursion(ip, parent_ip, TRACE_LIST_START);
if (bit < 0)
return;
do_for_each_ftrace_op(op, ftrace_ops_list) {
if (op->flags & FTRACE_OPS_FL_STUB)
continue;
if ((!(op->flags & FTRACE_OPS_FL_RCU) || rcu_is_watching()) &&
ftrace_ops_test(op, ip, regs)) {
if (FTRACE_WARN_ON(!op->func)) {
pr_warn("op=%p %pS\n", op, op);
goto out;
}
op->func(ip, parent_ip, op, fregs);
}
} while_for_each_ftrace_op(op);
out:
trace_clear_recursion(bit);
}
#if ARCH_SUPPORTS_FTRACE_OPS
void arch_ftrace_ops_list_func(unsigned long ip, unsigned long parent_ip,
struct ftrace_ops *op, struct ftrace_regs *fregs)
{
__ftrace_ops_list_func(ip, parent_ip, NULL, fregs);
}
#else
void arch_ftrace_ops_list_func(unsigned long ip, unsigned long parent_ip)
{
__ftrace_ops_list_func(ip, parent_ip, NULL, NULL);
}
#endif
NOKPROBE_SYMBOL(arch_ftrace_ops_list_func);
static void ftrace_ops_assist_func(unsigned long ip, unsigned long parent_ip,
struct ftrace_ops *op, struct ftrace_regs *fregs)
{
int bit;
bit = trace_test_and_set_recursion(ip, parent_ip, TRACE_LIST_START);
if (bit < 0)
return;
if (!(op->flags & FTRACE_OPS_FL_RCU) || rcu_is_watching())
op->func(ip, parent_ip, op, fregs);
trace_clear_recursion(bit);
}
NOKPROBE_SYMBOL(ftrace_ops_assist_func);
ftrace_func_t ftrace_ops_get_func(struct ftrace_ops *ops)
{
if (ops->flags & (FTRACE_OPS_FL_RECURSION |
FTRACE_OPS_FL_RCU))
return ftrace_ops_assist_func;
return ops->func;
}
static void
ftrace_filter_pid_sched_switch_probe(void *data, bool preempt,
struct task_struct *prev,
struct task_struct *next,
unsigned int prev_state)
{
struct trace_array *tr = data;
struct trace_pid_list *pid_list;
struct trace_pid_list *no_pid_list;
pid_list = rcu_dereference_sched(tr->function_pids);
no_pid_list = rcu_dereference_sched(tr->function_no_pids);
if (trace_ignore_this_task(pid_list, no_pid_list, next))
this_cpu_write(tr->array_buffer.data->ftrace_ignore_pid,
FTRACE_PID_IGNORE);
else
this_cpu_write(tr->array_buffer.data->ftrace_ignore_pid,
next->pid);
}
static void
ftrace_pid_follow_sched_process_fork(void *data,
struct task_struct *self,
struct task_struct *task)
{
struct trace_pid_list *pid_list;
struct trace_array *tr = data;
pid_list = rcu_dereference_sched(tr->function_pids);
trace_filter_add_remove_task(pid_list, self, task);
pid_list = rcu_dereference_sched(tr->function_no_pids);
trace_filter_add_remove_task(pid_list, self, task);
}
static void
ftrace_pid_follow_sched_process_exit(void *data, struct task_struct *task)
{
struct trace_pid_list *pid_list;
struct trace_array *tr = data;
pid_list = rcu_dereference_sched(tr->function_pids);
trace_filter_add_remove_task(pid_list, NULL, task);
pid_list = rcu_dereference_sched(tr->function_no_pids);
trace_filter_add_remove_task(pid_list, NULL, task);
}
void ftrace_pid_follow_fork(struct trace_array *tr, bool enable)
{
if (enable) {
register_trace_sched_process_fork(ftrace_pid_follow_sched_process_fork,
tr);
register_trace_sched_process_free(ftrace_pid_follow_sched_process_exit,
tr);
} else {
unregister_trace_sched_process_fork(ftrace_pid_follow_sched_process_fork,
tr);
unregister_trace_sched_process_free(ftrace_pid_follow_sched_process_exit,
tr);
}
}
static void clear_ftrace_pids(struct trace_array *tr, int type)
{
struct trace_pid_list *pid_list;
struct trace_pid_list *no_pid_list;
int cpu;
pid_list = rcu_dereference_protected(tr->function_pids,
lockdep_is_held(&ftrace_lock));
no_pid_list = rcu_dereference_protected(tr->function_no_pids,
lockdep_is_held(&ftrace_lock));
if (!pid_type_enabled(type, pid_list, no_pid_list))
return;
if (!still_need_pid_events(type, pid_list, no_pid_list)) {
unregister_trace_sched_switch(ftrace_filter_pid_sched_switch_probe, tr);
for_each_possible_cpu(cpu)
per_cpu_ptr(tr->array_buffer.data, cpu)->ftrace_ignore_pid = FTRACE_PID_TRACE;
}
if (type & TRACE_PIDS)
rcu_assign_pointer(tr->function_pids, NULL);
if (type & TRACE_NO_PIDS)
rcu_assign_pointer(tr->function_no_pids, NULL);
synchronize_rcu();
if ((type & TRACE_PIDS) && pid_list)
trace_pid_list_free(pid_list);
if ((type & TRACE_NO_PIDS) && no_pid_list)
trace_pid_list_free(no_pid_list);
}
void ftrace_clear_pids(struct trace_array *tr)
{
mutex_lock(&ftrace_lock);
clear_ftrace_pids(tr, TRACE_PIDS | TRACE_NO_PIDS);
mutex_unlock(&ftrace_lock);
}
static void ftrace_pid_reset(struct trace_array *tr, int type)
{
mutex_lock(&ftrace_lock);
clear_ftrace_pids(tr, type);
ftrace_update_pid_func();
ftrace_startup_all(0);
mutex_unlock(&ftrace_lock);
}
#define FTRACE_NO_PIDS (void *)(PID_MAX_LIMIT + 1)
static void *fpid_start(struct seq_file *m, loff_t *pos)
__acquires(RCU)
{
struct trace_pid_list *pid_list;
struct trace_array *tr = m->private;
mutex_lock(&ftrace_lock);
rcu_read_lock_sched();
pid_list = rcu_dereference_sched(tr->function_pids);
if (!pid_list)
return !(*pos) ? FTRACE_NO_PIDS : NULL;
return trace_pid_start(pid_list, pos);
}
static void *fpid_next(struct seq_file *m, void *v, loff_t *pos)
{
struct trace_array *tr = m->private;
struct trace_pid_list *pid_list = rcu_dereference_sched(tr->function_pids);
if (v == FTRACE_NO_PIDS) {
(*pos)++;
return NULL;
}
return trace_pid_next(pid_list, v, pos);
}
static void fpid_stop(struct seq_file *m, void *p)
__releases(RCU)
{
rcu_read_unlock_sched();
mutex_unlock(&ftrace_lock);
}
static int fpid_show(struct seq_file *m, void *v)
{
if (v == FTRACE_NO_PIDS) {
seq_puts(m, "no pid\n");
return 0;
}
return trace_pid_show(m, v);
}
static const struct seq_operations ftrace_pid_sops = {
.start = fpid_start,
.next = fpid_next,
.stop = fpid_stop,
.show = fpid_show,
};
static void *fnpid_start(struct seq_file *m, loff_t *pos)
__acquires(RCU)
{
struct trace_pid_list *pid_list;
struct trace_array *tr = m->private;
mutex_lock(&ftrace_lock);
rcu_read_lock_sched();
pid_list = rcu_dereference_sched(tr->function_no_pids);
if (!pid_list)
return !(*pos) ? FTRACE_NO_PIDS : NULL;
return trace_pid_start(pid_list, pos);
}
static void *fnpid_next(struct seq_file *m, void *v, loff_t *pos)
{
struct trace_array *tr = m->private;
struct trace_pid_list *pid_list = rcu_dereference_sched(tr->function_no_pids);
if (v == FTRACE_NO_PIDS) {
(*pos)++;
return NULL;
}
return trace_pid_next(pid_list, v, pos);
}
static const struct seq_operations ftrace_no_pid_sops = {
.start = fnpid_start,
.next = fnpid_next,
.stop = fpid_stop,
.show = fpid_show,
};
static int pid_open(struct inode *inode, struct file *file, int type)
{
const struct seq_operations *seq_ops;
struct trace_array *tr = inode->i_private;
struct seq_file *m;
int ret = 0;
ret = tracing_check_open_get_tr(tr);
if (ret)
return ret;
if ((file->f_mode & FMODE_WRITE) &&
(file->f_flags & O_TRUNC))
ftrace_pid_reset(tr, type);
switch (type) {
case TRACE_PIDS:
seq_ops = &ftrace_pid_sops;
break;
case TRACE_NO_PIDS:
seq_ops = &ftrace_no_pid_sops;
break;
default:
trace_array_put(tr);
WARN_ON_ONCE(1);
return -EINVAL;
}
ret = seq_open(file, seq_ops);
if (ret < 0) {
trace_array_put(tr);
} else {
m = file->private_data;
m->private = tr;
}
return ret;
}
static int
ftrace_pid_open(struct inode *inode, struct file *file)
{
return pid_open(inode, file, TRACE_PIDS);
}
static int
ftrace_no_pid_open(struct inode *inode, struct file *file)
{
return pid_open(inode, file, TRACE_NO_PIDS);
}
static void ignore_task_cpu(void *data)
{
struct trace_array *tr = data;
struct trace_pid_list *pid_list;
struct trace_pid_list *no_pid_list;
pid_list = rcu_dereference_protected(tr->function_pids,
mutex_is_locked(&ftrace_lock));
no_pid_list = rcu_dereference_protected(tr->function_no_pids,
mutex_is_locked(&ftrace_lock));
if (trace_ignore_this_task(pid_list, no_pid_list, current))
this_cpu_write(tr->array_buffer.data->ftrace_ignore_pid,
FTRACE_PID_IGNORE);
else
this_cpu_write(tr->array_buffer.data->ftrace_ignore_pid,
current->pid);
}
static ssize_t
pid_write(struct file *filp, const char __user *ubuf,
size_t cnt, loff_t *ppos, int type)
{
struct seq_file *m = filp->private_data;
struct trace_array *tr = m->private;
struct trace_pid_list *filtered_pids;
struct trace_pid_list *other_pids;
struct trace_pid_list *pid_list;
ssize_t ret;
if (!cnt)
return 0;
mutex_lock(&ftrace_lock);
switch (type) {
case TRACE_PIDS:
filtered_pids = rcu_dereference_protected(tr->function_pids,
lockdep_is_held(&ftrace_lock));
other_pids = rcu_dereference_protected(tr->function_no_pids,
lockdep_is_held(&ftrace_lock));
break;
case TRACE_NO_PIDS:
filtered_pids = rcu_dereference_protected(tr->function_no_pids,
lockdep_is_held(&ftrace_lock));
other_pids = rcu_dereference_protected(tr->function_pids,
lockdep_is_held(&ftrace_lock));
break;
default:
ret = -EINVAL;
WARN_ON_ONCE(1);
goto out;
}
ret = trace_pid_write(filtered_pids, &pid_list, ubuf, cnt);
if (ret < 0)
goto out;
switch (type) {
case TRACE_PIDS:
rcu_assign_pointer(tr->function_pids, pid_list);
break;
case TRACE_NO_PIDS:
rcu_assign_pointer(tr->function_no_pids, pid_list);
break;
}
if (filtered_pids) {
synchronize_rcu();
trace_pid_list_free(filtered_pids);
} else if (pid_list && !other_pids) {
register_trace_sched_switch(ftrace_filter_pid_sched_switch_probe, tr);
}
on_each_cpu(ignore_task_cpu, tr, 1);
ftrace_update_pid_func();
ftrace_startup_all(0);
out:
mutex_unlock(&ftrace_lock);
if (ret > 0)
*ppos += ret;
return ret;
}
static ssize_t
ftrace_pid_write(struct file *filp, const char __user *ubuf,
size_t cnt, loff_t *ppos)
{
return pid_write(filp, ubuf, cnt, ppos, TRACE_PIDS);
}
static ssize_t
ftrace_no_pid_write(struct file *filp, const char __user *ubuf,
size_t cnt, loff_t *ppos)
{
return pid_write(filp, ubuf, cnt, ppos, TRACE_NO_PIDS);
}
static int
ftrace_pid_release(struct inode *inode, struct file *file)
{
struct trace_array *tr = inode->i_private;
trace_array_put(tr);
return seq_release(inode, file);
}
static const struct file_operations ftrace_pid_fops = {
.open = ftrace_pid_open,
.write = ftrace_pid_write,
.read = seq_read,
.llseek = tracing_lseek,
.release = ftrace_pid_release,
};
static const