#include <linux/linkage.h>
#include <linux/sys.h>
#include <linux/cache.h>
#include <linux/syscalls.h>
#include <linux/entry-common.h>
#include <linux/nospec.h>
#include <asm/syscall.h>
#define __SYSCALL(nr, sym) extern long __x64_##sym(const struct pt_regs *);
#define __SYSCALL_NORETURN(nr, sym) extern long __noreturn __x64_##sym(const struct pt_regs *);
#include <asm/syscalls_64.h>
#ifdef CONFIG_X86_X32_ABI
#include <asm/syscalls_x32.h>
#endif
#undef __SYSCALL
#undef __SYSCALL_NORETURN
#define __SYSCALL_NORETURN __SYSCALL
#define __SYSCALL(nr, sym) __x64_##sym,
const sys_call_ptr_t sys_call_table[] = {
#include <asm/syscalls_64.h>
};
#undef __SYSCALL
#define __SYSCALL(nr, sym) case nr: return __x64_##sym(regs);
static noinline long x64_sys_call(const struct pt_regs *regs, unsigned int nr)
{
switch (nr) {
#include <asm/syscalls_64.h>
default: return __x64_sys_ni_syscall(regs);
}
}
static noinline long x32_sys_call(const struct pt_regs *regs, unsigned int nr)
{
#ifdef CONFIG_X86_X32_ABI
switch (nr) {
#include <asm/syscalls_x32.h>
default: return __x64_sys_ni_syscall(regs);
}
#else
return -ENOSYS;
#endif
}
static __always_inline bool do_syscall_x64(struct pt_regs *regs, unsigned long nr)
{
if (likely(nr < NR_syscalls)) {
nr = array_index_nospec(nr, NR_syscalls);
regs->ax = x64_sys_call(regs, (unsigned int)nr);
return true;
}
return false;
}
static __always_inline bool do_syscall_x32(struct pt_regs *regs, unsigned long nr)
{
if (!IS_ENABLED(CONFIG_X86_X32_ABI))
return false;
nr -= __X32_SYSCALL_BIT;
if (likely(nr < X32_NR_syscalls)) {
nr = array_index_nospec(nr, X32_NR_syscalls);
regs->ax = x32_sys_call(regs, (unsigned int)nr);
return true;
}
return false;
}
__visible noinstr bool do_syscall_64(struct pt_regs *regs, long nr)
{
if (likely(syscall_enter_from_user_mode_randomize_stack(regs, &nr))) {
instrumentation_begin();
if (unlikely(!do_syscall_x64(regs, nr))) {
if (!do_syscall_x32(regs, nr) && nr != -1L)
regs->ax = -ENOSYS;
}
instrumentation_end();
}
syscall_exit_to_user_mode(regs);
if (cpu_feature_enabled(X86_FEATURE_XENPV))
return false;
if (unlikely(regs->cx != regs->ip || regs->r11 != regs->flags))
return false;
if (unlikely(regs->cs != __USER_CS || regs->ss != __USER_DS))
return false;
if (unlikely(regs->ip >= TASK_SIZE_MAX))
return false;
if (unlikely(regs->flags & (X86_EFLAGS_RF | X86_EFLAGS_TF)))
return false;
return true;
}