#define pr_fmt(fmt) "HugeTLB: " fmt
#include <linux/pgtable.h>
#include <linux/moduleparam.h>
#include <linux/mmdebug.h>
#include <linux/pagewalk.h>
#include <linux/pgalloc.h>
#include <linux/vmemmap-optimization.h>
#include <asm/tlbflush.h>
#include "hugetlb_vmemmap.h"
struct vmemmap_remap_walk {
void (*remap_pte)(pte_t *pte, unsigned long addr,
struct vmemmap_remap_walk *walk);
unsigned long nr_walked;
struct page *vmemmap_head;
struct page *vmemmap_tail;
struct list_head *vmemmap_pages;
#define VMEMMAP_SPLIT_NO_TLB_FLUSH BIT(0)
#define VMEMMAP_REMAP_NO_TLB_FLUSH BIT(1)
unsigned long flags;
};
static int vmemmap_split_pmd(pmd_t *pmd, struct page *head, unsigned long start,
struct vmemmap_remap_walk *walk)
{
pmd_t __pmd;
int i;
unsigned long addr = start;
pte_t *pgtable;
pgtable = pte_alloc_one_kernel(&init_mm);
if (!pgtable)
return -ENOMEM;
pmd_populate_kernel(&init_mm, &__pmd, pgtable);
for (i = 0; i < PTRS_PER_PTE; i++, addr += PAGE_SIZE) {
pte_t entry, *pte;
pgprot_t pgprot = PAGE_KERNEL;
entry = mk_pte(head + i, pgprot);
pte = pte_offset_kernel(&__pmd, addr);
set_pte_at(&init_mm, addr, pte, entry);
}
spin_lock(&init_mm.page_table_lock);
if (likely(pmd_leaf(*pmd))) {
if (!PageReserved(head))
split_page(head, get_order(PMD_SIZE));
smp_wmb();
pmd_populate_kernel(&init_mm, pmd, pgtable);
if (!(walk->flags & VMEMMAP_SPLIT_NO_TLB_FLUSH))
flush_tlb_kernel_range(start, start + PMD_SIZE);
} else {
pte_free_kernel(&init_mm, pgtable);
}
spin_unlock(&init_mm.page_table_lock);
return 0;
}
static int vmemmap_pmd_entry(pmd_t *pmd, unsigned long addr,
unsigned long next, struct mm_walk *walk)
{
int ret = 0;
struct page *head;
struct vmemmap_remap_walk *vmemmap_walk = walk->private;
if (!vmemmap_walk->remap_pte)
walk->action = ACTION_CONTINUE;
spin_lock(&init_mm.page_table_lock);
head = pmd_leaf(*pmd) ? pmd_page(*pmd) : NULL;
if (IS_ENABLED(CONFIG_MEMORY_HOTPLUG) && unlikely(!vmemmap_walk->nr_walked)) {
struct page *page = head ? head + pte_index(addr) :
pte_page(ptep_get(pte_offset_kernel(pmd, addr)));
if (PageVmemmapSelfHosted(page))
ret = -ENOTSUPP;
}
spin_unlock(&init_mm.page_table_lock);
if (!head || ret)
return ret;
return vmemmap_split_pmd(pmd, head, addr & PMD_MASK, vmemmap_walk);
}
static int vmemmap_pte_entry(pte_t *pte, unsigned long addr,
unsigned long next, struct mm_walk *walk)
{
struct vmemmap_remap_walk *vmemmap_walk = walk->private;
vmemmap_walk->remap_pte(pte, addr, vmemmap_walk);
vmemmap_walk->nr_walked++;
return 0;
}
static const struct mm_walk_ops vmemmap_remap_ops = {
.pmd_entry = vmemmap_pmd_entry,
.pte_entry = vmemmap_pte_entry,
};
static int vmemmap_remap_range(unsigned long start, unsigned long end,
struct vmemmap_remap_walk *walk)
{
int ret;
VM_BUG_ON(!PAGE_ALIGNED(start | end));
mmap_read_lock(&init_mm);
ret = walk_kernel_page_table_range(start, end, &vmemmap_remap_ops,
NULL, walk);
mmap_read_unlock(&init_mm);
if (ret)
return ret;
if (walk->remap_pte && !(walk->flags & VMEMMAP_REMAP_NO_TLB_FLUSH))
flush_tlb_kernel_range(start, end);
return 0;
}
static inline void free_vmemmap_page(struct page *page)
{
if (PageReserved(page)) {
memmap_boot_pages_add(-1);
free_reserved_page(page);
} else {
memmap_pages_add(-1);
__free_page(page);
}
}
static void free_vmemmap_page_list(struct list_head *list)
{
struct page *page, *next;
list_for_each_entry_safe(page, next, list, lru)
free_vmemmap_page(page);
}
static void vmemmap_remap_pte(pte_t *pte, unsigned long addr,
struct vmemmap_remap_walk *walk)
{
struct page *page = pte_page(ptep_get(pte));
pte_t entry;
if (unlikely(walk->nr_walked == 0 && walk->vmemmap_head)) {
VM_WARN_ON_ONCE(!PageHead((const struct page *)addr));
list_del(&walk->vmemmap_head->lru);
smp_wmb();
entry = mk_pte(walk->vmemmap_head, PAGE_KERNEL);
} else {
VM_WARN_ON_ONCE(!PageTail((const struct page *)addr));
entry = mk_pte(walk->vmemmap_tail, PAGE_KERNEL);
}
list_add(&page->lru, walk->vmemmap_pages);
set_pte_at(&init_mm, addr, pte, entry);
}
static void vmemmap_restore_pte(pte_t *pte, unsigned long addr,
struct vmemmap_remap_walk *walk)
{
struct page *src = pte_page(ptep_get(pte)), *dst;
if (walk->vmemmap_tail && walk->vmemmap_tail != src)
return;
VM_WARN_ON_ONCE(PageHead((const struct page *)addr));
dst = list_first_entry(walk->vmemmap_pages, struct page, lru);
list_del(&dst->lru);
copy_page(page_to_virt(dst), page_to_virt(src));
smp_wmb();
set_pte_at(&init_mm, addr, pte, mk_pte(dst, PAGE_KERNEL));
}
static int vmemmap_remap_split(unsigned long start, unsigned long end)
{
struct vmemmap_remap_walk walk = {
.remap_pte = NULL,
.flags = VMEMMAP_SPLIT_NO_TLB_FLUSH,
};
return vmemmap_remap_range(start, end, &walk);
}
static int vmemmap_remap_free(unsigned long start, unsigned long end,
struct page *vmemmap_head,
struct page *vmemmap_tail,
struct list_head *vmemmap_pages,
unsigned long flags)
{
int ret;
struct vmemmap_remap_walk walk = {
.remap_pte = vmemmap_remap_pte,
.vmemmap_head = vmemmap_head,
.vmemmap_tail = vmemmap_tail,
.vmemmap_pages = vmemmap_pages,
.flags = flags,
};
ret = vmemmap_remap_range(start, end, &walk);
if (!ret || !walk.nr_walked)
return ret;
end = start + walk.nr_walked * PAGE_SIZE;
walk = (struct vmemmap_remap_walk) {
.remap_pte = vmemmap_restore_pte,
.vmemmap_tail = vmemmap_tail,
.vmemmap_pages = vmemmap_pages,
.flags = 0,
};
vmemmap_remap_range(start, end, &walk);
return ret;
}
static int alloc_vmemmap_page_list(unsigned long start, unsigned long end,
struct list_head *list)
{
gfp_t gfp_mask = GFP_KERNEL | __GFP_RETRY_MAYFAIL;
unsigned long nr_pages = (end - start) >> PAGE_SHIFT;
int nid = page_to_nid((struct page *)start);
struct page *page, *next;
int i;
for (i = 0; i < nr_pages; i++) {
page = alloc_pages_node(nid, gfp_mask, 0);
if (!page)
goto out;
list_add(&page->lru, list);
}
memmap_pages_add(nr_pages);
return 0;
out:
list_for_each_entry_safe(page, next, list, lru)
__free_page(page);
return -ENOMEM;
}
static int vmemmap_remap_alloc(unsigned long start, unsigned long end,
unsigned long flags)
{
LIST_HEAD(vmemmap_pages);
struct vmemmap_remap_walk walk = {
.remap_pte = vmemmap_restore_pte,
.vmemmap_pages = &vmemmap_pages,
.flags = flags,
};
if (alloc_vmemmap_page_list(start, end, &vmemmap_pages))
return -ENOMEM;
return vmemmap_remap_range(start, end, &walk);
}
static bool vmemmap_optimize_enabled = IS_ENABLED(CONFIG_HUGETLB_PAGE_OPTIMIZE_VMEMMAP_DEFAULT_ON);
static int __init hugetlb_vmemmap_optimize_param(char *buf)
{
return kstrtobool(buf, &vmemmap_optimize_enabled);
}
early_param("hugetlb_free_vmemmap", hugetlb_vmemmap_optimize_param);
static int __hugetlb_vmemmap_restore_folio(const struct hstate *h,
struct folio *folio, unsigned long flags)
{
int ret;
unsigned long vmemmap_start, vmemmap_end;
VM_WARN_ON_ONCE_FOLIO(!folio_test_hugetlb(folio), folio);
VM_WARN_ON_ONCE_FOLIO(folio_ref_count(folio), folio);
if (!folio_test_hugetlb_vmemmap_optimized(folio))
return 0;
vmemmap_start = (unsigned long)&folio->page;
vmemmap_end = vmemmap_start + hugetlb_vmemmap_size(h);
vmemmap_start += HUGETLB_VMEMMAP_RESERVE_SIZE;
ret = vmemmap_remap_alloc(vmemmap_start, vmemmap_end, flags);
if (!ret)
folio_clear_hugetlb_vmemmap_optimized(folio);
return ret;
}
int hugetlb_vmemmap_restore_folio(const struct hstate *h, struct folio *folio)
{
return __hugetlb_vmemmap_restore_folio(h, folio, 0);
}
long hugetlb_vmemmap_restore_folios(const struct hstate *h,
struct list_head *folio_list,
struct list_head *non_hvo_folios)
{
struct folio *folio, *t_folio;
long restored = 0;
long ret = 0;
unsigned long flags = VMEMMAP_REMAP_NO_TLB_FLUSH;
list_for_each_entry_safe(folio, t_folio, folio_list, lru) {
if (folio_test_hugetlb_vmemmap_optimized(folio)) {
ret = __hugetlb_vmemmap_restore_folio(h, folio, flags);
if (ret)
break;
restored++;
}
list_move(&folio->lru, non_hvo_folios);
}
if (restored)
flush_tlb_all();
if (!ret)
ret = restored;
return ret;
}
static bool vmemmap_should_optimize_folio(const struct hstate *h, struct folio *folio)
{
if (folio_test_hugetlb_vmemmap_optimized(folio))
return false;
if (!READ_ONCE(vmemmap_optimize_enabled))
return false;
if (!hugetlb_vmemmap_optimizable(h))
return false;
return true;
}
static int __hugetlb_vmemmap_optimize_folio(const struct hstate *h,
struct folio *folio,
struct list_head *vmemmap_pages,
unsigned long flags)
{
unsigned long vmemmap_start, vmemmap_end;
struct page *vmemmap_head, *vmemmap_tail;
int nid, ret = 0;
VM_WARN_ON_ONCE_FOLIO(!folio_test_hugetlb(folio), folio);
VM_WARN_ON_ONCE_FOLIO(folio_ref_count(folio), folio);
if (!vmemmap_should_optimize_folio(h, folio))
return ret;
nid = folio_nid(folio);
vmemmap_tail = vmemmap_shared_tail_page(h->order, folio_zone(folio));
if (!vmemmap_tail)
return -ENOMEM;
folio_set_hugetlb_vmemmap_optimized(folio);
vmemmap_head = alloc_pages_node(nid, GFP_KERNEL, 0);
if (!vmemmap_head) {
ret = -ENOMEM;
goto out;
}
copy_page(page_to_virt(vmemmap_head), folio);
list_add(&vmemmap_head->lru, vmemmap_pages);
memmap_pages_add(1);
vmemmap_start = (unsigned long)&folio->page;
vmemmap_end = vmemmap_start + hugetlb_vmemmap_size(h);
ret = vmemmap_remap_free(vmemmap_start, vmemmap_end,
vmemmap_head, vmemmap_tail,
vmemmap_pages, flags);
out:
if (ret)
folio_clear_hugetlb_vmemmap_optimized(folio);
return ret;
}
void hugetlb_vmemmap_optimize_folio(const struct hstate *h, struct folio *folio)
{
LIST_HEAD(vmemmap_pages);
__hugetlb_vmemmap_optimize_folio(h, folio, &vmemmap_pages, 0);
free_vmemmap_page_list(&vmemmap_pages);
}
static int hugetlb_vmemmap_split_folio(const struct hstate *h, struct folio *folio)
{
unsigned long vmemmap_start, vmemmap_end;
if (!vmemmap_should_optimize_folio(h, folio))
return 0;
vmemmap_start = (unsigned long)&folio->page;
vmemmap_end = vmemmap_start + hugetlb_vmemmap_size(h);
return vmemmap_remap_split(vmemmap_start, vmemmap_end);
}
static void __hugetlb_vmemmap_optimize_folios(struct hstate *h,
struct list_head *folio_list,
bool boot)
{
struct folio *folio;
int nr_to_optimize;
LIST_HEAD(vmemmap_pages);
unsigned long flags = VMEMMAP_REMAP_NO_TLB_FLUSH;
if (list_empty(folio_list))
return;
nr_to_optimize = 0;
list_for_each_entry(folio, folio_list, lru) {
int ret;
unsigned long spfn, epfn;
if (boot && folio_test_hugetlb_vmemmap_optimized(folio)) {
spfn = (unsigned long)&folio->page;
epfn = spfn + hugetlb_vmemmap_size(h);
vmemmap_wrprotect_hvo(spfn, epfn, folio_nid(folio),
HUGETLB_VMEMMAP_RESERVE_SIZE);
continue;
}
nr_to_optimize++;
ret = hugetlb_vmemmap_split_folio(h, folio);
if (ret == -ENOMEM)
break;
}
if (!nr_to_optimize)
goto out;
flush_tlb_all();
list_for_each_entry(folio, folio_list, lru) {
int ret;
ret = __hugetlb_vmemmap_optimize_folio(h, folio, &vmemmap_pages, flags);
if (ret == -ENOMEM && !list_empty(&vmemmap_pages)) {
flush_tlb_all();
free_vmemmap_page_list(&vmemmap_pages);
INIT_LIST_HEAD(&vmemmap_pages);
__hugetlb_vmemmap_optimize_folio(h, folio, &vmemmap_pages, flags);
}
}
out:
flush_tlb_all();
free_vmemmap_page_list(&vmemmap_pages);
}
void hugetlb_vmemmap_optimize_folios(struct hstate *h, struct list_head *folio_list)
{
__hugetlb_vmemmap_optimize_folios(h, folio_list, false);
}
void hugetlb_vmemmap_optimize_bootmem_folios(struct hstate *h, struct list_head *folio_list)
{
__hugetlb_vmemmap_optimize_folios(h, folio_list, true);
}
void __init hugetlb_vmemmap_optimize_bootmem_page(unsigned long pfn, unsigned int order)
{
if (!READ_ONCE(vmemmap_optimize_enabled))
return;
section_set_compound_order_range(pfn, 1UL << order, order);
}
static const struct ctl_table hugetlb_vmemmap_sysctls[] = {
{
.procname = "hugetlb_optimize_vmemmap",
.data = &vmemmap_optimize_enabled,
.maxlen = sizeof(vmemmap_optimize_enabled),
.mode = 0644,
.proc_handler = proc_dobool,
},
};
static int __init hugetlb_vmemmap_init(void)
{
const struct hstate *h;
BUILD_BUG_ON(__NR_USED_SUBPAGE > HUGETLB_VMEMMAP_RESERVE_PAGES);
for_each_hstate(h) {
if (hugetlb_vmemmap_optimizable(h)) {
register_sysctl_init("vm", hugetlb_vmemmap_sysctls);
break;
}
}
return 0;
}
late_initcall