/* SPDX-License-Identifier: GPL-2.0 */ /* * Copyright (C) 2020-2022 Loongson Technology Corporation Limited */ #include <asm/asm.h> #include <asm/loongarch.h> #include <asm/page.h> #include <asm/pgtable.h> #include <asm/regdef.h> #include <asm/stackframe.h> #define INVTLB_ADDR_GFALSE_AND_ASID 5 #define PTRS_PER_PGD_BITS (PAGE_SHIFT - 3) #define PTRS_PER_PUD_BITS (PAGE_SHIFT - 3) #define PTRS_PER_PMD_BITS (PAGE_SHIFT - 3) #define PTRS_PER_PTE_BITS (PAGE_SHIFT - 3) .macro tlb_do_page_fault, write SYM_CODE_START(tlb_do_page_fault_\write) SAVE_ALL csrrd a2, LOONGARCH_CSR_BADV move a0, sp REG_S a2, sp, PT_BVADDR li.w a1, \write bl do_page_fault RESTORE_ALL_AND_RET SYM_CODE_END(tlb_do_page_fault_\write) .endm tlb_do_page_fault 0 tlb_do_page_fault 1 SYM_CODE_START(handle_tlb_protect) BACKUP_T0T1 SAVE_ALL move a0, sp move a1, zero csrrd a2, LOONGARCH_CSR_BADV REG_S a2, sp, PT_BVADDR la_abs t0, do_page_fault jirl ra, t0, 0 RESTORE_ALL_AND_RET SYM_CODE_END(handle_tlb_protect) SYM_CODE_START(handle_tlb_load) csrwr t0, EXCEPTION_KS0 csrwr t1, EXCEPTION_KS1 csrwr ra, EXCEPTION_KS2 /* * The vmalloc handling is not in the hotpath. */ csrrd t0, LOONGARCH_CSR_BADV bltz t0, vmalloc_load csrrd t1, LOONGARCH_CSR_PGDL vmalloc_done_load: /* Get PGD offset in bytes */ bstrpick.d ra, t0, PTRS_PER_PGD_BITS + PGDIR_SHIFT - 1, PGDIR_SHIFT alsl.d t1, ra, t1, 3 #if CONFIG_PGTABLE_LEVELS > 3 ld.d t1, t1, 0 bstrpick.d ra, t0, PTRS_PER_PUD_BITS + PUD_SHIFT - 1, PUD_SHIFT alsl.d t1, ra, t1, 3 #endif #if CONFIG_PGTABLE_LEVELS > 2 ld.d t1, t1, 0 bstrpick.d ra, t0, PTRS_PER_PMD_BITS + PMD_SHIFT - 1, PMD_SHIFT alsl.d t1, ra, t1, 3 #endif ld.d ra, t1, 0 /* * For huge tlb entries, pmde doesn't contain an address but * instead contains the tlb pte. Check the PAGE_HUGE bit and * see if we need to jump to huge tlb processing. */ rotri.d ra, ra, _PAGE_HUGE_SHIFT + 1 bltz ra, tlb_huge_update_load rotri.d ra, ra, 64 - (_PAGE_HUGE_SHIFT + 1) bstrpick.d t0, t0, PTRS_PER_PTE_BITS + PAGE_SHIFT - 1, PAGE_SHIFT alsl.d t1, t0, ra, _PTE_T_LOG2 #ifdef CONFIG_SMP smp_pgtable_change_load: ll.d t0, t1, 0 #else ld.d t0, t1, 0 #endif andi ra, t0, _PAGE_PRESENT beqz ra, nopage_tlb_load ori t0, t0, _PAGE_VALID #ifdef CONFIG_SMP sc.d t0, t1, 0 beqz t0, smp_pgtable_change_load #else st.d t0, t1, 0 #endif tlbsrch bstrins.d t1, zero, 3, 3 ld.d t0, t1, 0 ld.d t1, t1, 8 csrwr t0, LOONGARCH_CSR_TLBELO0 csrwr t1, LOONGARCH_CSR_TLBELO1 tlbwr csrrd t0, EXCEPTION_KS0 csrrd t1, EXCEPTION_KS1 csrrd ra, EXCEPTION_KS2 ertn #ifdef CONFIG_64BIT vmalloc_load: la_abs t1, swapper_pg_dir b vmalloc_done_load #endif /* This is the entry point of a huge page. */ tlb_huge_update_load: #ifdef CONFIG_SMP ll.d ra, t1, 0 #endif andi t0, ra, _PAGE_PRESENT beqz t0, nopage_tlb_load #ifdef CONFIG_SMP ori t0, ra, _PAGE_VALID sc.d t0, t1, 0 beqz t0, tlb_huge_update_load ori t0, ra, _PAGE_VALID #else rotri.d ra, ra, 64 - (_PAGE_HUGE_SHIFT + 1) ori t0, ra, _PAGE_VALID st.d t0, t1, 0 #endif csrrd ra, LOONGARCH_CSR_ASID csrrd t1, LOONGARCH_CSR_BADV andi ra, ra, CSR_ASID_ASID invtlb INVTLB_ADDR_GFALSE_AND_ASID, ra, t1 /* * A huge PTE describes an area the size of the * configured huge page size. This is twice the * of the large TLB entry size we intend to use. * A TLB entry half the size of the configured * huge page size is configured into entrylo0 * and entrylo1 to cover the contiguous huge PTE * address space. */ /* Huge page: Move Global bit */ xori t0, t0, _PAGE_HUGE lu12i.w t1, _PAGE_HGLOBAL >> 12 and t1, t0, t1 srli.d t1, t1, (_PAGE_HGLOBAL_SHIFT - _PAGE_GLOBAL_SHIFT) or t0, t0, t1 move ra, t0 csrwr ra, LOONGARCH_CSR_TLBELO0 /* Convert to entrylo1 */ addi.d t1, zero, 1 slli.d t1, t1, (HPAGE_SHIFT - 1) add.d t0, t0, t1 csrwr t0, LOONGARCH_CSR_TLBELO1 /* Set huge page tlb entry size */ addu16i.d t0, zero, (CSR_TLBIDX_PS >> 16) addu16i.d t1, zero, (PS_HUGE_SIZE << (CSR_TLBIDX_PS_SHIFT - 16)) csrxchg t1, t0, LOONGARCH_CSR_TLBIDX tlbfill addu16i.d t0, zero, (CSR_TLBIDX_PS >> 16) addu16i.d t1, zero, (PS_DEFAULT_SIZE << (CSR_TLBIDX_PS_SHIFT - 16)) csrxchg t1, t0, LOONGARCH_CSR_TLBIDX csrrd t0, EXCEPTION_KS0 csrrd t1, EXCEPTION_KS1 csrrd ra, EXCEPTION_KS2 ertn nopage_tlb_load: dbar 0x700 csrrd ra, EXCEPTION_KS2 la_abs t0, tlb_do_page_fault_0 jr t0 SYM_CODE_END(handle_tlb_load) SYM_CODE_START(handle_tlb_load_ptw) csrwr t0, LOONGARCH_CSR_KS0 csrwr t1, LOONGARCH_CSR_KS1 la_abs t0, tlb_do_page_fault_0 jr t0 SYM_CODE_END(handle_tlb_load_ptw) SYM_CODE_START(handle_tlb_store) csrwr t0, EXCEPTION_KS0 csrwr t1, EXCEPTION_KS1 csrwr ra, EXCEPTION_KS2 /* * The vmalloc handling is not in the hotpath. */ csrrd t0, LOONGARCH_CSR_BADV bltz t0, vmalloc_store csrrd t1, LOONGARCH_CSR_PGDL vmalloc_done_store: /* Get PGD offset in bytes */ bstrpick.d ra, t0, PTRS_PER_PGD_BITS + PGDIR_SHIFT - 1, PGDIR_SHIFT alsl.d t1, ra, t1, 3 #if CONFIG_PGTABLE_LEVELS > 3 ld.d t1, t1, 0 bstrpick.d ra, t0, PTRS_PER_PUD_BITS + PUD_SHIFT - 1, PUD_SHIFT alsl.d t1, ra, t1, 3 #endif #if CONFIG_PGTABLE_LEVELS > 2 ld.d t1, t1, 0 bstrpick.d ra, t0, PTRS_PER_PMD_BITS + PMD_SHIFT - 1, PMD_SHIFT alsl.d t1, ra, t1, 3 #endif ld.d ra, t1, 0 /* * For huge tlb entries, pmde doesn't contain an address but * instead contains the tlb pte. Check the PAGE_HUGE bit and * see if we need to jump to huge tlb processing. */ rotri.d ra, ra, _PAGE_HUGE_SHIFT + 1 bltz ra, tlb_huge_update_store rotri.d ra, ra, 64 - (_PAGE_HUGE_SHIFT + 1) bstrpick.d t0, t0, PTRS_PER_PTE_BITS + PAGE_SHIFT - 1, PAGE_SHIFT alsl.d t1, t0, ra, _PTE_T_LOG2 #ifdef CONFIG_SMP smp_pgtable_change_store: ll.d t0, t1, 0 #else ld.d t0, t1, 0 #endif andi ra, t0, _PAGE_PRESENT | _PAGE_WRITE xori ra, ra, _PAGE_PRESENT | _PAGE_WRITE bnez ra, nopage_tlb_store ori t0, t0, (_PAGE_VALID | _PAGE_DIRTY | _PAGE_MODIFIED) #ifdef CONFIG_SMP sc.d t0, t1, 0 beqz t0, smp_pgtable_change_store #else st.d t0, t1, 0 #endif tlbsrch bstrins.d t1, zero, 3, 3 ld.d t0, t1, 0 ld.d t1, t1, 8 csrwr t0, LOONGARCH_CSR_TLBELO0 csrwr t1, LOONGARCH_CSR_TLBELO1 tlbwr csrrd t0, EXCEPTION_KS0 csrrd t1, EXCEPTION_KS1 csrrd ra, EXCEPTION_KS2 ertn #ifdef CONFIG_64BIT vmalloc_store: la_abs t1, swapper_pg_dir b vmalloc_done_store #endif /* This is the entry point of a huge page. */ tlb_huge_update_store: #ifdef CONFIG_SMP ll.d ra, t1, 0 #endif andi t0, ra, _PAGE_PRESENT | _PAGE_WRITE xori t0, t0, _PAGE_PRESENT | _PAGE_WRITE bnez t0, nopage_tlb_store #ifdef CONFIG_SMP ori t0, ra, (_PAGE_VALID | _PAGE_DIRTY | _PAGE_MODIFIED) sc.d t0, t1, 0 beqz t0, tlb_huge_update_store ori t0, ra, (_PAGE_VALID | _PAGE_DIRTY | _PAGE_MODIFIED) #else rotri.d ra, ra, 64 - (_PAGE_HUGE_SHIFT + 1) ori t0, ra, (_PAGE_VALID | _PAGE_DIRTY | _PAGE_MODIFIED) st.d t0, t1, 0 #endif csrrd ra, LOONGARCH_CSR_ASID csrrd t1, LOONGARCH_CSR_BADV andi ra, ra, CSR_ASID_ASID invtlb INVTLB_ADDR_GFALSE_AND_ASID, ra, t1 /* * A huge PTE describes an area the size of the * configured huge page size. This is twice the * of the large TLB entry size we intend to use. * A TLB entry half the size of the configured * huge page size is configured into entrylo0 * and entrylo1 to cover the contiguous huge PTE * address space. */ /* Huge page: Move Global bit */ xori t0, t0, _PAGE_HUGE lu12i.w t1, _PAGE_HGLOBAL >> 12 and t1, t0, t1 srli.d t1, t1, (_PAGE_HGLOBAL_SHIFT - _PAGE_GLOBAL_SHIFT) or t0, t0, t1 move ra, t0 csrwr ra, LOONGARCH_CSR_TLBELO0 /* Convert to entrylo1 */ addi.d t1, zero, 1 slli.d t1, t1, (HPAGE_SHIFT - 1) add.d t0, t0, t1 csrwr t0, LOONGARCH_CSR_TLBELO1 /* Set huge page tlb entry size */ addu16i.d t0, zero, (CSR_TLBIDX_PS >> 16) addu16i.d t1, zero, (PS_HUGE_SIZE << (CSR_TLBIDX_PS_SHIFT - 16)) csrxchg t1, t0, LOONGARCH_CSR_TLBIDX tlbfill /* Reset default page size */ addu16i.d t0, zero, (CSR_TLBIDX_PS >> 16) addu16i.d t1, zero, (PS_DEFAULT_SIZE << (CSR_TLBIDX_PS_SHIFT - 16)) csrxchg t1, t0, LOONGARCH_CSR_TLBIDX csrrd t0, EXCEPTION_KS0 csrrd t1, EXCEPTION_KS1 csrrd ra, EXCEPTION_KS2 ertn nopage_tlb_store: dbar 0x700 csrrd ra, EXCEPTION_KS2 la_abs t0, tlb_do_page_fault_1 jr t0 SYM_CODE_END(handle_tlb_store) SYM_CODE_START(handle_tlb_store_ptw) csrwr t0, LOONGARCH_CSR_KS0 csrwr t1, LOONGARCH_CSR_KS1 la_abs t0, tlb_do_page_fault_1 jr t0 SYM_CODE_END(handle_tlb_store_ptw) SYM_CODE_START(handle_tlb_modify) csrwr t0, EXCEPTION_KS0 csrwr t1, EXCEPTION_KS1 csrwr ra, EXCEPTION_KS2 /* * The vmalloc handling is not in the hotpath. */ csrrd t0, LOONGARCH_CSR_BADV bltz t0, vmalloc_modify csrrd t1, LOONGARCH_CSR_PGDL vmalloc_done_modify: /* Get PGD offset in bytes */ bstrpick.d ra, t0, PTRS_PER_PGD_BITS + PGDIR_SHIFT - 1, PGDIR_SHIFT alsl.d t1, ra, t1, 3 #if CONFIG_PGTABLE_LEVELS > 3 ld.d t1, t1, 0 bstrpick.d ra, t0, PTRS_PER_PUD_BITS + PUD_SHIFT - 1, PUD_SHIFT alsl.d t1, ra, t1, 3 #endif #if CONFIG_PGTABLE_LEVELS > 2 ld.d t1, t1, 0 bstrpick.d ra, t0, PTRS_PER_PMD_BITS + PMD_SHIFT - 1, PMD_SHIFT alsl.d t1, ra, t1, 3 #endif ld.d ra, t1, 0 /* * For huge tlb entries, pmde doesn't contain an address but * instead contains the tlb pte. Check the PAGE_HUGE bit and * see if we need to jump to huge tlb processing. */ rotri.d ra, ra, _PAGE_HUGE_SHIFT + 1 bltz ra, tlb_huge_update_modify rotri.d ra, ra, 64 - (_PAGE_HUGE_SHIFT + 1) bstrpick.d t0, t0, PTRS_PER_PTE_BITS + PAGE_SHIFT - 1, PAGE_SHIFT alsl.d t1, t0, ra, _PTE_T_LOG2 #ifdef CONFIG_SMP smp_pgtable_change_modify: ll.d t0, t1, 0 #else ld.d t0, t1, 0 #endif andi ra, t0, _PAGE_WRITE beqz ra, nopage_tlb_modify ori t0, t0, (_PAGE_VALID | _PAGE_DIRTY | _PAGE_MODIFIED) #ifdef CONFIG_SMP sc.d t0, t1, 0 beqz t0, smp_pgtable_change_modify #else st.d t0, t1, 0 #endif tlbsrch bstrins.d t1, zero, 3, 3 ld.d t0, t1, 0 ld.d t1, t1, 8 csrwr t0, LOONGARCH_CSR_TLBELO0 csrwr t1, LOONGARCH_CSR_TLBELO1 tlbwr csrrd t0, EXCEPTION_KS0 csrrd t1, EXCEPTION_KS1 csrrd ra, EXCEPTION_KS2 ertn #ifdef CONFIG_64BIT vmalloc_modify: la_abs t1, swapper_pg_dir b vmalloc_done_modify #endif /* This is the entry point of a huge page. */ tlb_huge_update_modify: #ifdef CONFIG_SMP ll.d ra, t1, 0 #endif andi t0, ra, _PAGE_WRITE beqz t0, nopage_tlb_modify #ifdef CONFIG_SMP ori t0, ra, (_PAGE_VALID | _PAGE_DIRTY | _PAGE_MODIFIED) sc.d t0, t1, 0 beqz t0, tlb_huge_update_modify ori t0, ra, (_PAGE_VALID | _PAGE_DIRTY | _PAGE_MODIFIED) #else rotri.d ra, ra, 64 - (_PAGE_HUGE_SHIFT + 1) ori t0, ra, (_PAGE_VALID | _PAGE_DIRTY | _PAGE_MODIFIED) st.d t0, t1, 0 #endif csrrd ra, LOONGARCH_CSR_ASID csrrd t1, LOONGARCH_CSR_BADV andi ra, ra, CSR_ASID_ASID invtlb INVTLB_ADDR_GFALSE_AND_ASID, ra, t1 /* * A huge PTE describes an area the size of the * configured huge page size. This is twice the * of the large TLB entry size we intend to use. * A TLB entry half the size of the configured * huge page size is configured into entrylo0 * and entrylo1 to cover the contiguous huge PTE * address space. */ /* Huge page: Move Global bit */ xori t0, t0, _PAGE_HUGE lu12i.w t1, _PAGE_HGLOBAL >> 12 and t1, t0, t1 srli.d t1, t1, (_PAGE_HGLOBAL_SHIFT - _PAGE_GLOBAL_SHIFT) or t0, t0, t1 move ra, t0 csrwr ra, LOONGARCH_CSR_TLBELO0 /* Convert to entrylo1 */ addi.d t1, zero, 1 slli.d t1, t1, (HPAGE_SHIFT - 1) add.d t0, t0, t1 csrwr t0, LOONGARCH_CSR_TLBELO1 /* Set huge page tlb entry size */ addu16i.d t0, zero, (CSR_TLBIDX_PS >> 16) addu16i.d t1, zero, (PS_HUGE_SIZE << (CSR_TLBIDX_PS_SHIFT - 16)) csrxchg t1, t0, LOONGARCH_CSR_TLBIDX tlbfill /* Reset default page size */ addu16i.d t0, zero, (CSR_TLBIDX_PS >> 16) addu16i.d t1, zero, (PS_DEFAULT_SIZE << (CSR_TLBIDX_PS_SHIFT - 16)) csrxchg t1, t0, LOONGARCH_CSR_TLBIDX csrrd t0, EXCEPTION_KS0 csrrd t1, EXCEPTION_KS1 csrrd ra, EXCEPTION_KS2 ertn nopage_tlb_modify: dbar 0x700 csrrd ra, EXCEPTION_KS2 la_abs t0, tlb_do_page_fault_1 jr t0 SYM_CODE_END(handle_tlb_modify) SYM_CODE_START(handle_tlb_modify_ptw) csrwr t0, LOONGARCH_CSR_KS0 csrwr t1, LOONGARCH_CSR_KS1 la_abs t0, tlb_do_page_fault_1 jr t0 SYM_CODE_END(handle_tlb_modify_ptw) SYM_CODE_START(handle_tlb_refill) csrwr t0, LOONGARCH_CSR_TLBRSAVE csrrd t0, LOONGARCH_CSR_PGD lddir t0, t0, 3 #if CONFIG_PGTABLE_LEVELS > 3 lddir t0, t0, 2 #endif #if CONFIG_PGTABLE_LEVELS > 2 lddir t0, t0, 1 #endif ldpte t0, 0 ldpte t0, 1 tlbfill csrrd t0, LOONGARCH_CSR_TLBRSAVE ertn SYM_CODE_END(handle_tlb_refill)