#include <linux/kernel.h>
#include <linux/mm.h>
#include <linux/page-flags.h>
#include <linux/kernel-page-flags.h>
#include <linux/sched/signal.h>
#include <linux/sched/task.h>
#include <linux/dax.h>
#include <linux/ksm.h>
#include <linux/rmap.h>
#include <linux/export.h>
#include <linux/pagemap.h>
#include <linux/swap.h>
#include <linux/backing-dev.h>
#include <linux/migrate.h>
#include <linux/suspend.h>
#include <linux/slab.h>
#include <linux/swapops.h>
#include <linux/hugetlb.h>
#include <linux/memory_hotplug.h>
#include <linux/mm_inline.h>
#include <linux/memremap.h>
#include <linux/kfifo.h>
#include <linux/ratelimit.h>
#include <linux/page-isolation.h>
#include <linux/pagewalk.h>
#include <linux/shmem_fs.h>
#include "internal.h"
#include "ras/ras_event.h"
int sysctl_memory_failure_early_kill __read_mostly = 0;
int sysctl_memory_failure_recovery __read_mostly = 1;
atomic_long_t num_poisoned_pages __read_mostly = ATOMIC_LONG_INIT(0);
static bool __page_handle_poison(struct page *page)
{
int ret;
zone_pcp_disable(page_zone(page));
ret = dissolve_free_huge_page(page);
if (!ret)
ret = take_page_off_buddy(page);
zone_pcp_enable(page_zone(page));
return ret > 0;
}
static bool page_handle_poison(struct page *page, bool hugepage_or_freepage, bool release)
{
if (hugepage_or_freepage) {
if (!__page_handle_poison(page))
return false;
}
SetPageHWPoison(page);
if (release)
put_page(page);
page_ref_inc(page);
num_poisoned_pages_inc();
return true;
}
#if defined(CONFIG_HWPOISON_INJECT) || defined(CONFIG_HWPOISON_INJECT_MODULE)
u32 hwpoison_filter_enable = 0;
u32 hwpoison_filter_dev_major = ~0U;
u32 hwpoison_filter_dev_minor = ~0U;
u64 hwpoison_filter_flags_mask;
u64 hwpoison_filter_flags_value;
EXPORT_SYMBOL_GPL(hwpoison_filter_enable);
EXPORT_SYMBOL_GPL(hwpoison_filter_dev_major);
EXPORT_SYMBOL_GPL(hwpoison_filter_dev_minor);
EXPORT_SYMBOL_GPL(hwpoison_filter_flags_mask);
EXPORT_SYMBOL_GPL(hwpoison_filter_flags_value);
static int hwpoison_filter_dev(struct page *p)
{
struct address_space *mapping;
dev_t dev;
if (hwpoison_filter_dev_major == ~0U &&
hwpoison_filter_dev_minor == ~0U)
return 0;
mapping = page_mapping(p);
if (mapping == NULL || mapping->host == NULL)
return -EINVAL;
dev = mapping->host->i_sb->s_dev;
if (hwpoison_filter_dev_major != ~0U &&
hwpoison_filter_dev_major != MAJOR(dev))
return -EINVAL;
if (hwpoison_filter_dev_minor != ~0U &&
hwpoison_filter_dev_minor != MINOR(dev))
return -EINVAL;
return 0;
}
static int hwpoison_filter_flags(struct page *p)
{
if (!hwpoison_filter_flags_mask)
return 0;
if ((stable_page_flags(p) & hwpoison_filter_flags_mask) ==
hwpoison_filter_flags_value)
return 0;
else
return -EINVAL;
}
#ifdef CONFIG_MEMCG
u64 hwpoison_filter_memcg;
EXPORT_SYMBOL_GPL(hwpoison_filter_memcg);
static int hwpoison_filter_task(struct page *p)
{
if (!hwpoison_filter_memcg)
return 0;
if (page_cgroup_ino(p) != hwpoison_filter_memcg)
return -EINVAL;
return 0;
}
#else
static int hwpoison_filter_task(struct page *p) { return 0; }
#endif
int hwpoison_filter(struct page *p)
{
if (!hwpoison_filter_enable)
return 0;
if (hwpoison_filter_dev(p))
return -EINVAL;
if (hwpoison_filter_flags(p))
return -EINVAL;
if (hwpoison_filter_task(p))
return -EINVAL;
return 0;
}
#else
int hwpoison_filter(struct page *p)
{
return 0;
}
#endif
EXPORT_SYMBOL_GPL(hwpoison_filter);
struct to_kill {
struct list_head nd;
struct task_struct *tsk;
unsigned long addr;
short size_shift;
};
static int kill_proc(struct to_kill *tk, unsigned long pfn, int flags)
{
struct task_struct *t = tk->tsk;
short addr_lsb = tk->size_shift;
int ret = 0;
pr_err("Memory failure: %#lx: Sending SIGBUS to %s:%d due to hardware memory corruption\n",
pfn, t->comm, t->pid);
if ((flags & MF_ACTION_REQUIRED) && (t == current))
ret = force_sig_mceerr(BUS_MCEERR_AR,
(void __user *)tk->addr, addr_lsb);
else
ret = send_sig_mceerr(BUS_MCEERR_AO, (void __user *)tk->addr,
addr_lsb, t);
if (ret < 0)
pr_info("Memory failure: Error sending signal to %s:%d: %d\n",
t->comm, t->pid, ret);
return ret;
}
void shake_page(struct page *p)
{
if (PageHuge(p))
return;
if (!PageSlab(p)) {
lru_add_drain_all();
if (PageLRU(p) || is_free_buddy_page(p))
return;
}
}
EXPORT_SYMBOL_GPL(shake_page);
static unsigned long dev_pagemap_mapping_shift(struct page *page,
struct vm_area_struct *vma)
{
unsigned long address = vma_address(page, vma);
unsigned long ret = 0;
pgd_t *pgd;
p4d_t *p4d;
pud_t *pud;
pmd_t *pmd;
pte_t *pte;
VM_BUG_ON_VMA(address == -EFAULT, vma);
pgd = pgd_offset(vma->vm_mm, address);
if (!pgd_present(*pgd))
return 0;
p4d = p4d_offset(pgd, address);
if (!p4d_present(*p4d))
return 0;
pud = pud_offset(p4d, address);
if (!pud_present(*pud))
return 0;
if (pud_devmap(*pud))
return PUD_SHIFT;
pmd = pmd_offset(pud, address);
if (!pmd_present(*pmd))
return 0;
if (pmd_devmap(*pmd))
return PMD_SHIFT;
pte = pte_offset_map(pmd, address);
if (pte_present(*pte) && pte_devmap(*pte))
ret = PAGE_SHIFT;
pte_unmap(pte);
return ret;
}
static void add_to_kill(struct task_struct *tsk, struct page *p,
struct vm_area_struct *vma,
struct list_head *to_kill)
{
struct to_kill *tk;
tk = kmalloc(sizeof(struct to_kill), GFP_ATOMIC);
if (!tk) {
pr_err("Memory failure: Out of memory while machine check handling\n");
return;
}
tk->addr = page_address_in_vma(p, vma);
if (is_zone_device_page(p))
tk->size_shift = dev_pagemap_mapping_shift(p, vma);
else
tk->size_shift = page_shift(compound_head(p));
if (tk->addr == -EFAULT) {
pr_info("Memory failure: Unable to find user space address %lx in %s\n",
page_to_pfn(p), tsk->comm);
} else if (tk->size_shift == 0) {
kfree(tk);
return;
}
get_task_struct(tsk);
tk->tsk = tsk;
list_add_tail(&tk->nd, to_kill);
}
static void kill_procs(struct list_head *to_kill, int forcekill, bool fail,
unsigned long pfn, int flags)
{
struct to_kill *tk, *next;
list_for_each_entry_safe (tk, next, to_kill, nd) {
if (forcekill) {
if (fail || tk->addr == -EFAULT) {
pr_err("Memory failure: %#lx: forcibly killing %s:%d because of failure to unmap corrupted page\n",
pfn, tk->tsk->comm, tk->tsk->pid);
do_send_sig_info(SIGKILL, SEND_SIG_PRIV,
tk->tsk, PIDTYPE_PID);
}
else if (kill_proc(tk, pfn, flags) < 0)
pr_err("Memory failure: %#lx: Cannot send advisory machine check signal to %s:%d\n",
pfn, tk->tsk->comm, tk->tsk->pid);
}
put_task_struct(tk->tsk);
kfree(tk);
}
}
static struct task_struct *find_early_kill_thread(struct task_struct *tsk)
{
struct task_struct *t;
for_each_thread(tsk, t) {
if (t->flags & PF_MCE_PROCESS) {
if (t->flags & PF_MCE_EARLY)
return t;
} else {
if (sysctl_memory_failure_early_kill)
return t;
}
}
return NULL;
}
static struct task_struct *task_early_kill(struct task_struct *tsk,
int force_early)
{
if (!tsk->mm)
return NULL;
if (force_early && tsk->mm == current->mm)
return current;
return find_early_kill_thread(tsk);
}
static void collect_procs_anon(struct page *page, struct list_head *to_kill,
int force_early)
{
struct folio *folio = page_folio(page);
struct vm_area_struct *vma;
struct task_struct *tsk;
struct anon_vma *av;
pgoff_t pgoff;
av = folio_lock_anon_vma_read(folio);
if (av == NULL)
return;
pgoff = page_to_pgoff(page);
read_lock(&tasklist_lock);
for_each_process (tsk) {
struct anon_vma_chain *vmac;
struct task_struct *t = task_early_kill(tsk, force_early);
if (!t)
continue;
anon_vma_interval_tree_foreach(vmac, &av->rb_root,
pgoff, pgoff) {
vma = vmac->vma;
if (!page_mapped_in_vma(page, vma))
continue;
if (vma->vm_mm == t->mm)
add_to_kill(t, page, vma, to_kill);
}
}
read_unlock(&tasklist_lock);
page_unlock_anon_vma_read(av);
}
static void collect_procs_file(struct page *page, struct list_head *to_kill,
int force_early)
{
struct vm_area_struct *vma;
struct task_struct *tsk;
struct address_space *mapping = page->mapping;
pgoff_t pgoff;
i_mmap_lock_read(mapping);
read_lock(&tasklist_lock);
pgoff = page_to_pgoff(page);
for_each_process(tsk) {
struct task_struct *t = task_early_kill(tsk, force_early);
if (!t)
continue;
vma_interval_tree_foreach(vma, &mapping->i_mmap, pgoff,
pgoff) {
if (vma->vm_mm == t->mm)
add_to_kill(t, page, vma, to_kill);
}
}
read_unlock(&tasklist_lock);
i_mmap_unlock_read(mapping);
}
static void collect_procs(struct page *page, struct list_head *tokill,
int force_early)
{
if (!page->mapping)
return;
if (PageAnon(page))
collect_procs_anon(page, tokill, force_early);
else
collect_procs_file(page, tokill, force_early);
}
struct hwp_walk {
struct to_kill tk;
unsigned long pfn;
int flags;
};
static void set_to_kill(struct to_kill *tk, unsigned long addr, short shift)
{
tk->addr = addr;
tk->size_shift = shift;
}
static int check_hwpoisoned_entry(pte_t pte, unsigned long addr, short shift,
unsigned long poisoned_pfn, struct to_kill *tk)
{
unsigned long pfn = 0;
if (pte_present(pte)) {
pfn = pte_pfn(pte);
} else {
swp_entry_t swp = pte_to_swp_entry(pte);
if (is_hwpoison_entry(swp))
pfn = hwpoison_entry_to_pfn(swp);
}
if (!pfn || pfn != poisoned_pfn)
return 0;
set_to_kill(tk, addr, shift);
return 1;
}
#ifdef CONFIG_TRANSPARENT_HUGEPAGE
static int check_hwpoisoned_pmd_entry(pmd_t *pmdp, unsigned long addr,
struct hwp_walk *hwp)
{
pmd_t pmd = *pmdp;
unsigned long pfn;
unsigned long hwpoison_vaddr;
if (!pmd_present(pmd))
return 0;
pfn = pmd_pfn(pmd);
if (pfn <= hwp->pfn && hwp->pfn < pfn + HPAGE_PMD_NR) {
hwpoison_vaddr = addr + ((hwp->pfn - pfn) << PAGE_SHIFT);
set_to_kill(&hwp->tk, hwpoison_vaddr, PAGE_SHIFT);
return 1;
}
return 0;
}
#else
static int check_hwpoisoned_pmd_entry(pmd_t *pmdp, unsigned long addr,
struct hwp_walk *hwp)
{
return 0;
}
#endif
static int hwpoison_pte_range(pmd_t *pmdp, unsigned long addr,
unsigned long end, struct mm_walk *walk)
{
struct hwp_walk *hwp = (struct hwp_walk *)walk->private;
int ret = 0;
pte_t *ptep, *mapped_pte;
spinlock_t *ptl;
ptl = pmd_trans_huge_lock(pmdp, walk->vma);
if (ptl) {
ret = check_hwpoisoned_pmd_entry(pmdp, addr, hwp);
spin_unlock(ptl);
goto out;
}
if (pmd_trans_unstable(pmdp))
goto out;
mapped_pte = ptep = pte_offset_map_lock(walk->vma->vm_mm, pmdp,
addr, &ptl);
for (; addr != end; ptep++, addr += PAGE_SIZE) {
ret = check_hwpoisoned_entry(*ptep, addr, PAGE_SHIFT,
hwp->pfn, &hwp->tk);
if (ret == 1)
break;
}
pte_unmap_unlock(mapped_pte, ptl);
out:
cond_resched();
return ret;
}
#ifdef CONFIG_HUGETLB_PAGE
static int hwpoison_hugetlb_range(pte_t *ptep, unsigned long hmask,
unsigned long addr, unsigned long end,
struct mm_walk *walk)
{
struct hwp_walk *hwp = (struct hwp_walk *)walk->private;
pte_t pte = huge_ptep_get(ptep);
struct hstate *h = hstate_vma(walk->vma);
return check_hwpoisoned_entry(pte, addr, huge_page_shift(h),
hwp->pfn, &hwp->tk);
}
#else
#define hwpoison_hugetlb_range NULL
#endif
static const struct mm_walk_ops hwp_walk_ops = {
.pmd_entry = hwpoison_pte_range,
.hugetlb_entry = hwpoison_hugetlb_range,
};
static int kill_accessing_process(struct task_struct *p, unsigned long pfn,
int flags)
{
int ret;
struct hwp_walk priv = {
.pfn = pfn,
};
priv.tk.tsk = p;
mmap_read_lock(p->mm);
ret = walk_page_range(p->mm, 0, TASK_SIZE, &hwp_walk_ops,
(void *)&priv);
if (ret == 1 && priv.tk.addr)
kill_proc(&priv.tk, pfn, flags);
else
ret = 0;
mmap_read_unlock(p->mm);
return ret > 0 ? -EHWPOISON : -EFAULT;
}
static const char *action_name[] = {
[MF_IGNORED] = "Ignored",
[MF_FAILED] = "Failed",
[MF_DELAYED] = "Delayed",
[MF_RECOVERED] = "Recovered",
};
static const char * const action_page_types[] = {
[MF_MSG_KERNEL] = "reserved kernel page",
[MF_MSG_KERNEL_HIGH_ORDER] = "high-order kernel page",
[MF_MSG_SLAB] = "kernel slab page",
[MF_MSG_DIFFERENT_COMPOUND] = "different compound page after locking",
[MF_MSG_HUGE] = "huge page",
[MF_MSG_FREE_HUGE] = "free huge page",
[MF_MSG_NON_PMD_HUGE] = "non-pmd-sized huge page",
[MF_MSG_UNMAP_FAILED] = "unmapping failed page",
[MF_MSG_DIRTY_SWAPCACHE] = "dirty swapcache page",
[MF_MSG_CLEAN_SWAPCACHE] = "clean swapcache page",
[MF_MSG_DIRTY_MLOCKED_LRU] = "dirty mlocked LRU page",
[MF_MSG_CLEAN_MLOCKED_LRU] = "clean mlocked LRU page",
[MF_MSG_DIRTY_UNEVICTABLE_LRU] = "dirty unevictable LRU page",
[MF_MSG_CLEAN_UNEVICTABLE_LRU] = "clean unevictable LRU page",
[MF_MSG_DIRTY_LRU] = "dirty LRU page",
[MF_MSG_CLEAN_LRU] = "clean LRU page",
[MF_MSG_TRUNCATED_LRU] = "already truncated LRU page",
[MF_MSG_BUDDY] = "free buddy page",
[MF_MSG_DAX] = "dax page",
[MF_MSG_UNSPLIT_THP] = "unsplit thp",
[MF_MSG_DIFFERENT_PAGE_SIZE] = "different page size",
[MF_MSG_UNKNOWN] = "unknown page",
};
static int delete_from_lru_cache(struct page *p)
{
if (!isolate_lru_page(p)) {
ClearPageActive(p);
ClearPageUnevictable(p);
mem_cgroup_uncharge(page_folio(p));
put_page(p);
return 0;
}
return -EIO;
}
static int truncate_error_page(struct page *p, unsigned long pfn,
struct address_space *mapping)
{
int ret = MF_FAILED;
if (mapping->a_ops->error_remove_page) {
int err = mapping->a_ops->error_remove_page(mapping, p);
if (err != 0) {
pr_info("Memory failure: %#lx: Failed to punch page: %d\n",
pfn, err);
} else if (page_has_private(p) &&
!try_to_release_page(p, GFP_NOIO)) {
pr_info("Memory failure: %#lx: failed to release buffers\n",
pfn);
} else {
ret = MF_RECOVERED;
}
} else {
if (invalidate_inode_page(p))
ret = MF_RECOVERED;
else
pr_info("Memory failure: %#lx: Failed to invalidate\n",
pfn);
}
return ret;
}
struct page_state {
unsigned long mask;
unsigned long res;
enum mf_action_page_type type;
int (*action)(struct page_state *ps, struct page *p);
};
static bool has_extra_refcount(struct page_state *ps, struct page *p,
bool extra_pins)
{
int count = page_count(p) - 1;
if (extra_pins)
count -= 1;
if (count > 0) {
pr_err("Memory failure: %#lx: %s still referenced by %d users\n",
page_to_pfn(p), action_page_types[ps->type], count);
return true;
}
return false;
}
static int me_kernel(struct page_state *ps, struct page *p)
{
unlock_page(p);
return MF_IGNORED;
}
static int me_unknown(struct page_state *ps, struct page *p)
{
pr_err("Memory failure: %#lx: Unknown page state\n", page_to_pfn(p));
unlock_page(p);
return MF_FAILED;
}
static int me_pagecache_clean(struct page_state *ps, struct page *p)
{
int ret;
struct address_space *mapping;
bool extra_pins;
delete_from_lru_cache(p);
if (PageAnon(p)) {
ret = MF_RECOVERED;
goto out;
}
mapping = page_mapping(p);
if (!mapping) {
ret = MF_FAILED;
goto out;
}
extra_pins = shmem_mapping(mapping);
ret = truncate_error_page(p, page_to_pfn(p), mapping);
if (has_extra_refcount(ps, p, extra_pins))
ret = MF_FAILED;
out:
unlock_page(p);
return ret;
}
static int me_pagecache_dirty(struct page_state *ps, struct page *p)
{
struct address_space *mapping = page_mapping(p);
SetPageError(p);
if (mapping) {
mapping_set_error(mapping, -EIO);
}
return me_pagecache_clean(ps, p);
}
static int me_swapcache_dirty(struct page_state *ps, struct page *p)
{
int ret;
bool extra_pins = false;
ClearPageDirty(p);
ClearPageUptodate(p);
ret = delete_from_lru_cache(p) ? MF_FAILED : MF_DELAYED;
unlock_page(p);
if (ret == MF_DELAYED)
extra_pins = true;
if (has_extra_refcount(ps, p, extra_pins))
ret = MF_FAILED;
return ret;
}
static int me_swapcache_clean(struct page_state *ps, struct page *p)
{
int ret;
delete_from_swap_cache(p);
ret = delete_from_lru_cache(p) ? MF_FAILED : MF_RECOVERED;
unlock_page(p);
if (has_extra_refcount(ps, p, false))
ret = MF_FAILED;
return ret;
}
static int me_huge_page(struct page_state *ps, struct page *p)
{
int res;
struct page *hpage = compound_head(p);
struct address_space *mapping;
if (!PageHuge(hpage))
return MF_DELAYED;
mapping = page_mapping(hpage);
if (mapping) {
res = truncate_error_page(hpage, page_to_pfn(p), mapping);
unlock_page(hpage);
} else {
res = MF_FAILED;
unlock_page(hpage);
if (PageAnon(hpage))
put_page(hpage);
if (__page_handle_poison(p)) {
page_ref_inc(p);
res = MF_RECOVERED;
}
}
if (has_extra_refcount(ps, p, false))
res = MF_FAILED;
return res;
}
#define dirty (1UL << PG_dirty)
#define sc ((1UL << PG_swapcache) | (1UL << PG_swapbacked))
#define unevict (1UL << PG_unevictable)
#define mlock (1UL << PG_mlocked)
#define lru (1UL << PG_lru)
#define head (1UL << PG_head)
#define slab (1UL << PG_slab)
#define reserved (1UL << PG_reserved)
static struct page_state error_states[] = {
{ reserved, reserved, MF_MSG_KERNEL, me_kernel },
{ slab, slab, MF_MSG_SLAB, me_kernel },
{ head, head, MF_MSG_HUGE, me_huge_page },
{ sc|dirty, sc|dirty, MF_MSG_DIRTY_SWAPCACHE, me_swapcache_dirty },
{ sc|dirty, sc, MF_MSG_CLEAN_SWAPCACHE, me_swapcache_clean },
{ mlock|dirty, mlock|dirty, MF_MSG_DIRTY_MLOCKED_LRU, me_pagecache_dirty },
{ mlock|dirty, mlock, MF_MSG_CLEAN_MLOCKED_LRU, me_pagecache_clean },
{ unevict|dirty, unevict|dirty, MF_MSG_DIRTY_UNEVICTABLE_LRU, me_pagecache_dirty },
{ unevict|dirty, unevict, MF_MSG_CLEAN_UNEVICTABLE_LRU, me_pagecache_clean },
{ lru|dirty, lru|dirty, MF_MSG_DIRTY_LRU, me_pagecache_dirty },
{ lru|dirty, lru, MF_MSG_CLEAN_LRU, me_pagecache_clean },
{ 0, 0, MF_MSG_UNKNOWN, me_unknown },
};
#undef dirty
#undef sc
#undef unevict
#undef mlock
#undef lru
#undef head
#undef slab
#undef reserved
static void action_result(unsigned long pfn, enum mf_action_page_type type,
enum mf_result result)
{
trace_memory_failure_event(pfn, type, result);
pr_err("Memory failure: %#lx: recovery action for %s: %s\n",
pfn, action_page_types[type], action_name[result]);
}
static int page_action(struct page_state *ps, struct page *p,
unsigned long pfn)
{
int result;
result = ps->action(ps, p);
action_result(pfn, ps->type, result);
return (result == MF_RECOVERED || result == MF_DELAYED) ? 0 : -EBUSY;
}
static inline bool PageHWPoisonTakenOff(struct page *page)
{
return PageHWPoison(page) && page_private(page) == MAGIC_HWPOISON;
}
void SetPageHWPoisonTakenOff(struct page *page)
{
set_page_private(page, MAGIC_HWPOISON);
}
void ClearPageHWPoisonTakenOff(struct page *page)
{
if (PageHWPoison(page))
set_page_private(page, 0);
}
static inline bool HWPoisonHandlable(struct page *page, unsigned long flags)
{
bool movable = false;
if ((flags & MF_SOFT_OFFLINE) && __PageMovable(page))
movable = true;
return movable || PageLRU(page) || is_free_buddy_page(page);
}
static int __get_hwpoison_page(struct page *page, unsigned long flags)
{
struct page *head = compound_head(page);
int ret = 0;
bool hugetlb = false;
ret = get_hwpoison_huge_page(head, &hugetlb);
if (hugetlb)
return ret;
if (!HWPoisonHandlable(head, flags))
return -EBUSY;
if (get_page_unless_zero(head)) {
if (head == compound_head(page))
return 1;
pr_info("Memory failure: %#lx cannot catch tail\n",
page_to_pfn(page));
put_page(head);
}
return 0;
}
static int get_any_page(struct page *p, unsigned long flags)
{
int ret = 0, pass = 0;
bool count_increased = false;
if (flags & MF_COUNT_INCREASED)
count_increased = true;
try_again:
if (!count_increased) {
ret = __get_hwpoison_page(p, flags);
if (!ret) {
if (page_count(p)) {
if (pass++ < 3)
goto try_again;
ret = -EBUSY;
} else if (!PageHuge(p) && !is_free_buddy_page(p)) {
if (pass++ < 3)
goto try_again;
ret = -EIO;
}
goto out;
} else if (ret == -EBUSY) {
if (pass++ < 3) {
shake_page(p);
goto try_again;
}
ret = -EIO;
goto out;
}
}
if (PageHuge(p) || HWPoisonHandlable(p, flags)) {
ret = 1;
} else {
if (pass++ < 3) {
put_page(p);
shake_page(p);
count_increased = false;
goto try_again;
}
put_page(p);
ret = -EIO;
}
out:
if (ret == -EIO)
pr_err("Memory failure: %#lx: unhandlable page.\n", page_to_pfn(p));
return ret;
}
static int __get_unpoison_page(struct page *page)
{
struct page *head = compound_head(page);
int ret = 0;
bool hugetlb = false;
ret = get_hwpoison_huge_page(head, &hugetlb);
if (hugetlb)
return ret;
if (PageHWPoisonTakenOff(page))
return -EHWPOISON;
return get_page_unless_zero(page) ? 1 : 0;
}
static int get_hwpoison_page(struct page *p, unsigned long flags)
{
int ret;
zone_pcp_disable(page_zone(p));
if (flags & MF_UNPOISON)
ret = __get_unpoison_page(p);
else
ret = get_any_page(p, flags);
zone_pcp_enable(page_zone(p));
return ret;
}
static bool hwpoison_user_mappings(struct page *p, unsigned long pfn,
int flags, struct page *hpage)
{
struct folio *folio = page_folio(hpage);
enum ttu_flags ttu = TTU_IGNORE_MLOCK | TTU_SYNC;
struct address_space *mapping;
LIST_HEAD(tokill);
bool unmap_success;
int kill = 1, forcekill;
bool mlocked = PageMlocked(hpage);
if (PageReserved(p) || PageSlab(p))
return true;
if (!(PageLRU(hpage) || PageHuge(p)))
return true;
if (!page_mapped(hpage))
return true;
if (PageKsm(p)) {
pr_err("Memory failure: %#lx: can't handle KSM pages.\n", pfn);
return false;
}
if (PageSwapCache(p)) {
pr_err("Memory failure: %#lx: keeping poisoned page in swap cache\n",
pfn);
ttu |= TTU_IGNORE_HWPOISON;
}
mapping = page_mapping(hpage);
if (!(flags & MF_MUST_KILL) && !PageDirty(hpage) && mapping &&
mapping_can_writeback(mapping)) {
if (page_mkclean(hpage)) {
SetPageDirty(hpage);
} else {
kill = 0;
ttu |= TTU_IGNORE_HWPOISON;
pr_info("Memory failure: %#lx: corrupted page was clean: dropped without side effects\n",
pfn);
}
}
if (kill)
collect_procs(hpage, &tokill, flags & MF_ACTION_REQUIRED);
if (PageHuge(hpage) && !PageAnon(hpage)) {
mapping = hugetlb_page_mapping_lock_write(hpage);
if (mapping) {
try_to_unmap(folio, ttu|TTU_RMAP_LOCKED);
i_mmap_unlock_write(mapping);
} else
pr_info("Memory failure: %#lx: could not lock mapping for mapped huge page\n", pfn);
} else {
try_to_unmap(folio, ttu);
}
unmap_success = !page_mapped(hpage);
if (!unmap_success)
pr_err("Memory failure: %#lx: failed to unmap page (mapcount=%d)\n",
pfn, page_mapcount(hpage));
if (mlocked)
shake_page(hpage);
forcekill = PageDirty(hpage) || (flags & MF_MUST_KILL);
kill_procs(&tokill, forcekill, !unmap_success, pfn, flags);
return unmap_success;
}
static int identify_page_state(unsigned long pfn, struct page *p,
unsigned long page_flags)
{
struct page_state *ps;
for (ps = error_states;; ps++)
if ((p->flags & ps->mask) == ps->res)
break;
page_flags |= (p->flags & (1UL << PG_dirty));
if (!ps->mask)
for (ps = error_states;; ps++)
if ((page_flags & ps->mask) == ps->res)
break;
return page_action(ps, p, pfn);
}
static int try_to_split_thp_page(struct page *page, const char *msg)
{
lock_page(page);
if (unlikely(split_huge_page(page))) {
unsigned long pfn = page_to_pfn(page);
unlock_page(page);
pr_info("%s: %#lx: thp split failed\n", msg, pfn);
put_page(page);
return -EBUSY;
}
unlock_page(page);
return 0;
}
int __get_huge_page_for_hwpoison(unsigned long pfn, int flags)
{
struct page *page = pfn_to_page(pfn);
struct page *head = compound_head(page);
int ret = 2;
bool count_increased = false;
if (!PageHeadHuge(head))
goto out;
if (flags & MF_COUNT_INCREASED) {
ret = 1;
count_increased = true;
} else if (HPageFreed(head) || HPageMigratable(head)) {
ret = get_page_unless_zero(head);
if (ret)
count_increased = true;
} else {
ret = -EBUSY;
goto out;
}
if (TestSetPageHWPoison(head)) {
ret = -EHWPOISON;
goto out;
}
return ret;
out:
if (count_increased)
put_page(head);
return ret;
}
#ifdef CONFIG_HUGETLB_PAGE
static int try_memory_failure_hugetlb(unsigned long pfn, int flags, int *hugetlb)
{
int res;
struct page *p = pfn_to_page(pfn);
struct page *head;
unsigned long page_flags;
bool retry = true;
*hugetlb = 1;
retry:
res = get_huge_page_for_hwpoison(pfn, flags);
if (res == 2) {
*hugetlb = 0;
return 0;
} else if (res == -EHWPOISON) {
pr_err("Memory failure: %#lx: already hardware poisoned\n", pfn);
if (flags & MF_ACTION_REQUIRED) {
head = compound_head(p);
res = kill_accessing_process(current, page_to_pfn(head), flags);
}
return res;
} else if (res == -EBUSY) {
if (retry) {
retry = false;
goto retry;
}
action_result(pfn, MF_MSG_UNKNOWN, MF_IGNORED);
return res;
}
head = compound_head(p);
lock_page(head);
if (hwpoison_filter(p)) {
ClearPageHWPoison(head);
res = -EOPNOTSUPP;
goto out;
}
num_poisoned_pages_inc();
if (res == 0) {
unlock_page(head);
res = MF_FAILED;
if (__page_handle_poison(p)) {
page_ref_inc(p);
res = MF_RECOVERED;
}
action_result(pfn, MF_MSG_FREE_HUGE, res);
return res == MF_RECOVERED ? 0 : -EBUSY;
}
if (!PageHuge(p) || compound_head(p) != head) {
action_result(pfn, MF_MSG_DIFFERENT_PAGE_SIZE, MF_IGNORED);
res = -EBUSY;
goto out;
}
page_flags = head->flags;
if (huge_page_size(page_hstate(head)) > PMD_SIZE) {
action_result(pfn, MF_MSG_NON_PMD_HUGE, MF_IGNORED);
res = -EBUSY;
goto out;
}
if (!hwpoison_user_mappings(p, pfn, flags, head)) {
action_result(pfn, MF_MSG_UNMAP_FAILED, MF_IGNORED);
res = -EBUSY;
goto out;
}
return identify_page_state(pfn, p, page_flags);
out:
unlock_page(head);
return res;
}
#else
static inline int try_memory_failure_hugetlb(unsigned long pfn, int flags, int *hugetlb)
{
return 0;
}
#endif
static int memory_failure_dev_pagemap(unsigned long pfn, int flags,
struct dev_pagemap *pgmap)
{
struct page *page = pfn_to_page(pfn);
unsigned long size = 0;
struct to_kill *tk;
LIST_HEAD(tokill);
int rc = -EBUSY;
loff_t start;
dax_entry_t cookie;
if (flags & MF_COUNT_INCREASED)
put_page(page);
if (!pgmap_pfn_valid(pgmap, pfn)) {
rc = -ENXIO;
goto out;
}
page = compound_head(page);
cookie = dax_lock_page(page);
if (!cookie)
goto out;
if (hwpoison_filter(page)) {
rc = -EOPNOTSUPP;
goto unlock;
}
if (pgmap->type == MEMORY_DEVICE_PRIVATE) {
goto unlock;
}
SetPageHWPoison(page);
flags |= MF_ACTION_REQUIRED | MF_MUST_KILL;
collect_procs(page, &tokill, true);
list_for_each_entry(tk, &tokill, nd)
if (tk->size_shift)
size = max(size, 1UL << tk->size_shift);
if (size) {
start = (page->index << PAGE_SHIFT) & ~(size - 1);
unmap_mapping_range(page->mapping, start, size, 0);
}
kill_procs(&tokill, true, false, pfn, flags);
rc = 0;
unlock:
dax_unlock_page(page, cookie);
out:
put_dev_pagemap(pgmap);
action_result(pfn, MF_MSG_DAX, rc ? MF_FAILED : MF_RECOVERED);
return rc;
}
static DEFINE_MUTEX(mf_mutex);
int memory_failure(unsigned long pfn, int flags)
{
struct page *p;
struct page *hpage;
struct dev_pagemap *pgmap;
int res = 0;
unsigned long page_flags;
bool retry = true;
int hugetlb = 0;
if (!sysctl_memory_failure_recovery)
panic("Memory failure on page %lx", pfn);
mutex_lock(&mf_mutex);
p = pfn_to_online_page(pfn);
if (!p) {
res = arch_memory_failure(pfn, flags);
if (res == 0)
goto unlock_mutex;
if (pfn_valid(pfn)) {
pgmap = get_dev_pagemap(pfn, NULL);
if (pgmap) {
res = memory_failure_dev_pagemap(pfn, flags,
pgmap);
goto unlock_mutex;
}
}
pr_err("Memory failure: %#lx: memory outside kernel control\n",
pfn);
res = -ENXIO;
goto unlock_mutex;
}
try_again:
res = try_memory_failure_hugetlb(pfn, flags, &hugetlb);
if (hugetlb)
goto unlock_mutex;
if (TestSetPageHWPoison(p)) {
pr_err("Memory failure: %#lx: already hardware poisoned\n",
pfn);
res = -EHWPOISON;
if (flags & MF_ACTION_REQUIRED)
res = kill_accessing_process(current, pfn, flags);
goto unlock_mutex;
}
hpage = compound_head(p);
num_poisoned_pages_inc();
if (!(flags & MF_COUNT_INCREASED)) {
res = get_hwpoison_page(p, flags);
if (!res) {
if (is_free_buddy_page(p)) {
if (take_page_off_buddy(p)) {
page_ref_inc(p);
res = MF_RECOVERED;
} else {
if (retry) {
ClearPageHWPoison(p);
num_poisoned_pages_dec();
retry = false;
goto try_again;
}
res = MF_FAILED;
}
action_result(pfn, MF_MSG_BUDDY, res);
res = res == MF_RECOVERED ? 0 : -EBUSY;
} else {
action_result(pfn, MF_MSG_KERNEL_HIGH_ORDER, MF_IGNORED);
res = -EBUSY;
}
goto unlock_mutex;
} else if (res < 0) {
action_result(pfn, MF_MSG_UNKNOWN, MF_IGNORED);
res = -EBUSY;
goto unlock_mutex;
}
}
if (PageTransHuge(hpage)) {
SetPageHasHWPoisoned(hpage);
if (try_to_split_thp_page(p, "Memory Failure") < 0) {
action_result(pfn, MF_MSG_UNSPLIT_THP, MF_IGNORED);
res = -EBUSY;
goto unlock_mutex;
}
VM_BUG_ON_PAGE(!page_count(p), p);
}
shake_page(p);
lock_page(p);
if (PageCompound(p)) {
if (retry) {
if (TestClearPageHWPoison(p))
num_poisoned_pages_dec();
unlock_page(p);
put_page(p);
flags &= ~MF_COUNT_INCREASED;
retry = false;
goto try_again;
}
action_result(pfn, MF_MSG_DIFFERENT_COMPOUND, MF_IGNORED);
res = -EBUSY;
goto unlock_page;
}
page_flags = p->flags;
if (hwpoison_filter(p)) {
if (TestClearPageHWPoison(p))
num_poisoned_pages_dec();
unlock_page(p);
put_page(p);
res = -EOPNOTSUPP;
goto unlock_mutex;
}
if (!PageLRU(p) && !PageWriteback(p))
goto identify_page_state;
wait_on_page_writeback(p);
if (!hwpoison_user_mappings(p, pfn, flags, p)) {
action_result(pfn, MF_MSG_UNMAP_FAILED, MF_IGNORED);
res = -EBUSY;
goto unlock_page;
}
if (PageLRU(p) && !PageSwapCache(p) && p->mapping == NULL) {
action_result(pfn, MF_MSG_TRUNCATED_LRU, MF_IGNORED);
res = -EBUSY;
goto unlock_page;
}
identify_page_state:
res = identify_page_state(pfn, p, page_flags);
mutex_unlock(&mf_mutex);
return res;
unlock_page:
unlock_page(p);
unlock_mutex:
mutex_unlock(&mf_mutex);
return res;
}
EXPORT_SYMBOL_GPL(memory_failure);
#define MEMORY_FAILURE_FIFO_ORDER 4
#define MEMORY_FAILURE_FIFO_SIZE (1 << MEMORY_FAILURE_FIFO_ORDER)
struct memory_failure_entry {
unsigned long pfn;
int flags;
};
struct memory_failure_cpu {
DECLARE_KFIFO(fifo, struct memory_failure_entry,
MEMORY_FAILURE_FIFO_SIZE);
spinlock_t lock;
struct work_struct work;
};
static DEFINE_PER_CPU(struct memory_failure_cpu, memory_failure_cpu);
void memory_failure_queue(unsigned long pfn, int flags)
{
struct memory_failure_cpu *mf_cpu;
unsigned long proc_flags;
struct memory_failure_entry entry = {
.pfn = pfn,
.flags = flags,
};
mf_cpu = &get_cpu_var(memory_failure_cpu);
spin_lock_irqsave(&mf_cpu->lock, proc_flags);
if (kfifo_put(&mf_cpu->fifo, entry))
schedule_work_on(smp_processor_id(), &mf_cpu->work);
else
pr_err("Memory failure: buffer overflow when queuing memory failure at %#lx\n",
pfn);
spin_unlock_irqrestore(&mf_cpu->lock, proc_flags);
put_cpu_var(memory_failure_cpu);
}
EXPORT_SYMBOL_GPL(memory_failure_queue);
static void memory_failure_work_func(struct work_struct *work)
{
struct memory_failure_cpu *mf_cpu;
struct memory_failure_entry entry = { 0, };
unsigned long proc_flags;
int gotten;
mf_cpu = container_of(work, struct memory_failure_cpu, work);
for (;;) {
spin_lock_irqsave(&mf_cpu->lock, proc_flags);
gotten = kfifo_get(&mf_cpu->fifo, &entry);
spin_unlock_irqrestore(&mf_cpu->lock, proc_flags);
if (!gotten)
break;
if (entry.flags & MF_SOFT_OFFLINE)
soft_offline_page(entry.pfn, entry.flags);
else
memory_failure(entry.pfn, entry.flags);
}
}
void memory_failure_queue_kick(int cpu)
{
struct memory_failure_cpu *mf_cpu;
mf_cpu = &per_cpu(memory_failure_cpu, cpu);
cancel_work_sync(&mf_cpu->work);
memory_failure_work_func(&mf_cpu->work);
}
static int __init memory_failure_init(void)
{
struct memory_failure_cpu *mf_cpu;
int cpu;
for_each_possible_cpu(cpu) {
mf_cpu = &per_cpu(memory_failure_cpu, cpu);
spin_lock_init(&mf_cpu->lock);
INIT_KFIFO(mf_cpu->fifo);
INIT_WORK(&mf_cpu->work, memory_failure_work_func);
}
return 0;
}
core_initcall(memory_failure_init);
#define unpoison_pr_info(fmt, pfn, rs) \
({ \
if (__ratelimit(rs)) \
pr_info(fmt, pfn); \
})
static inline int clear_page_hwpoison(struct ratelimit_state *rs, struct page *p)
{
if (TestClearPageHWPoison(p)) {
unpoison_pr_info("Unpoison: Software-unpoisoned page %#lx\n",
page_to_pfn(p), rs);
num_poisoned_pages_dec();
return 1;
}
return 0;
}
static inline int unpoison_taken_off_page(struct ratelimit_state *rs,
struct page *p)
{
if (put_page_back_buddy(p)) {
unpoison_pr_info("Unpoison: Software-unpoisoned page %#lx\n",
page_to_pfn(p), rs);
return 0;
}
return -EBUSY;
}
int unpoison_memory(unsigned long pfn)
{
struct page *page;
struct page *p;
int ret = -EBUSY;
static DEFINE_RATELIMIT_STATE(unpoison_rs, DEFAULT_RATELIMIT_INTERVAL,
DEFAULT_RATELIMIT_BURST);
if (!pfn_valid(pfn))
return -ENXIO;
p = pfn_to_page(pfn);
page = compound_head(p);
mutex_lock(&mf_mutex);
if (!PageHWPoison(p)) {
unpoison_pr_info("Unpoison: Page was already unpoisoned %#lx\n",
pfn, &unpoison_rs);
goto unlock_mutex;
}
if (page_count(page) > 1) {
unpoison_pr_info("Unpoison: Someone grabs the hwpoison page %#lx\n",
pfn, &unpoison_rs);
goto unlock_mutex;
}
if (page_mapped(page)) {
unpoison_pr_info("Unpoison: Someone maps the hwpoison page %#lx\n",
pfn, &unpoison_rs);
goto unlock_mutex;
}
if (page_mapping(page)) {
unpoison_pr_info("Unpoison: the hwpoison page has non-NULL mapping %#lx\n",
pfn, &unpoison_rs);
goto unlock_mutex;
}
if (PageSlab(page) || PageTable(page))
goto unlock_mutex;
ret = get_hwpoison_page(p, MF_UNPOISON);
if (!ret) {
if (clear_page_hwpoison(&unpoison_rs, page))
ret = 0;
else
ret = -EBUSY;
} else if (ret < 0) {
if (ret == -EHWPOISON) {
ret = unpoison_taken_off_page(&unpoison_rs, p);
} else
unpoison_pr_info("Unpoison: failed to grab page %#lx\n",
pfn, &unpoison_rs);
} else {
int freeit = clear_page_hwpoison(&unpoison_rs, p);
put_page(page);
if (freeit && !(pfn == my_zero_pfn(0) && page_count(p) == 1)) {
put_page(page);
ret = 0;
}
}
unlock_mutex:
mutex_unlock(&mf_mutex);
return ret;
}
EXPORT_SYMBOL(unpoison_memory);
static bool isolate_page(struct page *page, struct list_head *pagelist)
{
bool isolated = false;
bool lru = PageLRU(page);
if (PageHuge(page)) {
isolated = isolate_huge_page(page, pagelist);
} else {
if (lru)
isolated = !isolate_lru_page(page);
else
isolated = !isolate_movable_page(page, ISOLATE_UNEVICTABLE);
if (isolated)
list_add(&page->lru, pagelist);
}
if (isolated && lru)
inc_node_page_state(page, NR_ISOLATED_ANON +
page_is_file_lru(page));
put_page(page);
return isolated;
}
static int __soft_offline_page(struct page *page)
{
long ret = 0;
unsigned long pfn = page_to_pfn(page);
struct page *hpage = compound_head(page);
char const *msg_page[] = {"page", "hugepage"};
bool huge = PageHuge(page);
LIST_HEAD(pagelist);
struct migration_target_control mtc = {
.nid = NUMA_NO_NODE,
.gfp_mask = GFP_USER | __GFP_MOVABLE | __GFP_RETRY_MAYFAIL,
};
lock_page(page);
if (!PageHuge(page))
wait_on_page_writeback(page);
if (PageHWPoison(page)) {
unlock_page(page);
put_page(page);
pr_info("soft offline: %#lx page already poisoned\n", pfn);
return 0;
}
if (!PageHuge(page) && PageLRU(page) && !PageSwapCache(page))
ret = invalidate_inode_page(page);
unlock_page(page);
if (ret) {
pr_info("soft_offline: %#lx: invalidated\n", pfn);
page_handle_poison(page, false, true);
return 0;
}
if (isolate_page(hpage, &pagelist)) {
ret = migrate_pages(&pagelist, alloc_migration_target, NULL,
(unsigned long)&mtc, MIGRATE_SYNC, MR_MEMORY_FAILURE, NULL);
if (!ret) {
bool release = !huge;
if (!page_handle_poison(page, huge, release))
ret = -EBUSY;
} else {
if (!list_empty(&pagelist))
putback_movable_pages(&pagelist);
pr_info("soft offline: %#lx: %s migration failed %ld, type %pGp\n",
pfn, msg_page[huge], ret, &page->flags);
if (ret > 0)
ret = -EBUSY;
}
} else {
pr_info("soft offline: %#lx: %s isolation failed, page count %d, type %pGp\n",
pfn, msg_page[huge], page_count(page), &page->flags);
ret = -EBUSY;
}
return ret;
}
static int soft_offline_in_use_page(struct page *page)
{
struct page *hpage = compound_head(page);
if (!PageHuge(page) && PageTransHuge(hpage))
if (try_to_split_thp_page(page, "soft offline") < 0)
return -EBUSY;
return __soft_offline_page(page);
}
static int soft_offline_free_page(struct page *page)
{
int rc = 0;
if (!page_handle_poison(page, true, false))
rc = -EBUSY;
return rc;
}
static void put_ref_page(struct page *page)
{
if (page)
put_page(page);
}
int soft_offline_page(unsigned long pfn, int flags)
{
int ret;
bool try_again = true;
struct page *page, *ref_page = NULL;
WARN_ON_ONCE(!pfn_valid(pfn) && (flags & MF_COUNT_INCREASED));
if (!pfn_valid(pfn))
return -ENXIO;
if (flags & MF_COUNT_INCREASED)
ref_page = pfn_to_page(pfn);
page = pfn_to_online_page(pfn);
if (!page) {
put_ref_page(ref_page);
return -EIO;
}
mutex_lock(&mf_mutex);
if (PageHWPoison(page)) {
pr_info("%s: %#lx page already poisoned\n", __func__, pfn);
put_ref_page(ref_page);
mutex_unlock(&mf_mutex);
return 0;
}
retry:
get_online_mems();
ret = get_hwpoison_page(page, flags | MF_SOFT_OFFLINE);
put_online_mems();
if (ret > 0) {
ret = soft_offline_in_use_page(page);
} else if (ret == 0) {
if (soft_offline_free_page(page) && try_again) {
try_again = false;
flags &= ~MF_COUNT_INCREASED;
goto retry;
}
}
mutex_unlock(&mf_mutex);
return ret;
}