/* SPDX-License-Identifier: GPL-2.0 */ /* sun4v_tlb_miss.S: Sun4v TLB miss handlers. * * Copyright (C) 2006 <davem@davemloft.net> */ .text .align 32 /* Load ITLB fault information into VADDR and CTX, using BASE. */ #define LOAD_ITLB_INFO(BASE, VADDR, CTX) \ ldx [BASE + HV_FAULT_I_ADDR_OFFSET], VADDR; \ ldx [BASE + HV_FAULT_I_CTX_OFFSET], CTX; /* Load DTLB fault information into VADDR and CTX, using BASE. */ #define LOAD_DTLB_INFO(BASE, VADDR, CTX) \ ldx [BASE + HV_FAULT_D_ADDR_OFFSET], VADDR; \ ldx [BASE + HV_FAULT_D_CTX_OFFSET], CTX; /* DEST = (VADDR >> 22) * * Branch to ZERO_CTX_LABEL if context is zero. */ #define COMPUTE_TAG_TARGET(DEST, VADDR, CTX, ZERO_CTX_LABEL) \ srlx VADDR, 22, DEST; \ brz,pn CTX, ZERO_CTX_LABEL; \ nop; /* Create TSB pointer. This is something like: * * index_mask = (512 << (tsb_reg & 0x7UL)) - 1UL; * tsb_base = tsb_reg & ~0x7UL; * tsb_index = ((vaddr >> HASH_SHIFT) & tsb_mask); * tsb_ptr = tsb_base + (tsb_index * 16); */ #define COMPUTE_TSB_PTR(TSB_PTR, VADDR, HASH_SHIFT, TMP1, TMP2) \ and TSB_PTR, 0x7, TMP1; \ mov 512, TMP2; \ andn TSB_PTR, 0x7, TSB_PTR; \ sllx TMP2, TMP1, TMP2; \ srlx VADDR, HASH_SHIFT, TMP1; \ sub TMP2, 1, TMP2; \ and TMP1, TMP2, TMP1; \ sllx TMP1, 4, TMP1; \ add TSB_PTR, TMP1, TSB_PTR; sun4v_itlb_miss: /* Load MMU Miss base into %g2. */ ldxa [%g0] ASI_SCRATCHPAD, %g2 /* Load UTSB reg into %g1. */ mov SCRATCHPAD_UTSBREG1, %g1 ldxa [%g1] ASI_SCRATCHPAD, %g1 LOAD_ITLB_INFO(%g2, %g4, %g5) COMPUTE_TAG_TARGET(%g6, %g4, %g5, kvmap_itlb_4v) COMPUTE_TSB_PTR(%g1, %g4, PAGE_SHIFT, %g3, %g7) /* Load TSB tag/pte into %g2/%g3 and compare the tag. */ ldda [%g1] ASI_QUAD_LDD_PHYS_4V, %g2 cmp %g2, %g6 bne,a,pn %xcc, tsb_miss_page_table_walk mov FAULT_CODE_ITLB, %g3 andcc %g3, _PAGE_EXEC_4V, %g0 be,a,pn %xcc, tsb_do_fault mov FAULT_CODE_ITLB, %g3 /* We have a valid entry, make hypervisor call to load * I-TLB and return from trap. * * %g3: PTE * %g4: vaddr */ sun4v_itlb_load: ldxa [%g0] ASI_SCRATCHPAD, %g6 mov %o0, %g1 ! save %o0 mov %o1, %g2 ! save %o1 mov %o2, %g5 ! save %o2 mov %o3, %g7 ! save %o3 mov %g4, %o0 ! vaddr ldx [%g6 + HV_FAULT_I_CTX_OFFSET], %o1 ! ctx mov %g3, %o2 ! PTE mov HV_MMU_IMMU, %o3 ! flags ta HV_MMU_MAP_ADDR_TRAP brnz,pn %o0, sun4v_itlb_error mov %g2, %o1 ! restore %o1 mov %g1, %o0 ! restore %o0 mov %g5, %o2 ! restore %o2 mov %g7, %o3 ! restore %o3 retry sun4v_dtlb_miss: /* Load MMU Miss base into %g2. */ ldxa [%g0] ASI_SCRATCHPAD, %g2 /* Load UTSB reg into %g1. */ mov SCRATCHPAD_UTSBREG1, %g1 ldxa [%g1] ASI_SCRATCHPAD, %g1 LOAD_DTLB_INFO(%g2, %g4, %g5) COMPUTE_TAG_TARGET(%g6, %g4, %g5, kvmap_dtlb_4v) COMPUTE_TSB_PTR(%g1, %g4, PAGE_SHIFT, %g3, %g7) /* Load TSB tag/pte into %g2/%g3 and compare the tag. */ ldda [%g1] ASI_QUAD_LDD_PHYS_4V, %g2 cmp %g2, %g6 bne,a,pn %xcc, tsb_miss_page_table_walk mov FAULT_CODE_DTLB, %g3 /* We have a valid entry, make hypervisor call to load * D-TLB and return from trap. * * %g3: PTE * %g4: vaddr */ sun4v_dtlb_load: ldxa [%g0] ASI_SCRATCHPAD, %g6 mov %o0, %g1 ! save %o0 mov %o1, %g2 ! save %o1 mov %o2, %g5 ! save %o2 mov %o3, %g7 ! save %o3 mov %g4, %o0 ! vaddr ldx [%g6 + HV_FAULT_D_CTX_OFFSET], %o1 ! ctx mov %g3, %o2 ! PTE mov HV_MMU_DMMU, %o3 ! flags ta HV_MMU_MAP_ADDR_TRAP brnz,pn %o0, sun4v_dtlb_error mov %g2, %o1 ! restore %o1 mov %g1, %o0 ! restore %o0 mov %g5, %o2 ! restore %o2 mov %g7, %o3 ! restore %o3 retry sun4v_dtlb_prot: SET_GL(1) /* Load MMU Miss base into %g5. */ ldxa [%g0] ASI_SCRATCHPAD, %g5 ldx [%g5 + HV_FAULT_D_ADDR_OFFSET], %g5 rdpr %tl, %g1 cmp %g1, 1 bgu,pn %xcc, winfix_trampoline mov FAULT_CODE_DTLB | FAULT_CODE_WRITE, %g4 ba,pt %xcc, sparc64_realfault_common nop /* Called from trap table: * %g4: vaddr * %g5: context * %g6: TAG TARGET */ sun4v_itsb_miss: mov SCRATCHPAD_UTSBREG1, %g1 ldxa [%g1] ASI_SCRATCHPAD, %g1 brz,pn %g5, kvmap_itlb_4v mov FAULT_CODE_ITLB, %g3 ba,a,pt %xcc, sun4v_tsb_miss_common /* Called from trap table: * %g4: vaddr * %g5: context * %g6: TAG TARGET */ sun4v_dtsb_miss: mov SCRATCHPAD_UTSBREG1, %g1 ldxa [%g1] ASI_SCRATCHPAD, %g1 brz,pn %g5, kvmap_dtlb_4v mov FAULT_CODE_DTLB, %g3 /* fallthrough */ sun4v_tsb_miss_common: COMPUTE_TSB_PTR(%g1, %g4, PAGE_SHIFT, %g5, %g7) sub %g2, TRAP_PER_CPU_FAULT_INFO, %g2 #if defined(CONFIG_HUGETLB_PAGE) || defined(CONFIG_TRANSPARENT_HUGEPAGE) mov SCRATCHPAD_UTSBREG2, %g5 ldxa [%g5] ASI_SCRATCHPAD, %g5 cmp %g5, -1 be,pt %xcc, 80f nop COMPUTE_TSB_PTR(%g5, %g4, REAL_HPAGE_SHIFT, %g2, %g7) /* That clobbered %g2, reload it. */ ldxa [%g0] ASI_SCRATCHPAD, %g2 sub %g2, TRAP_PER_CPU_FAULT_INFO, %g2 80: stx %g5, [%g2 + TRAP_PER_CPU_TSB_HUGE_TEMP] #endif ba,pt %xcc, tsb_miss_page_table_walk_sun4v_fastpath ldx [%g2 + TRAP_PER_CPU_PGD_PADDR], %g7 sun4v_itlb_error: rdpr %tl, %g1 cmp %g1, 1 ble,pt %icc, sun4v_bad_ra or %g0, FAULT_CODE_BAD_RA | FAULT_CODE_ITLB, %g1 sethi %hi(sun4v_err_itlb_vaddr), %g1 stx %g4, [%g1 + %lo(sun4v_err_itlb_vaddr)] sethi %hi(sun4v_err_itlb_ctx), %g1 ldxa [%g0] ASI_SCRATCHPAD, %g6 ldx [%g6 + HV_FAULT_I_CTX_OFFSET], %o1 stx %o1, [%g1 + %lo(sun4v_err_itlb_ctx)] sethi %hi(sun4v_err_itlb_pte), %g1 stx %g3, [%g1 + %lo(sun4v_err_itlb_pte)] sethi %hi(sun4v_err_itlb_error), %g1 stx %o0, [%g1 + %lo(sun4v_err_itlb_error)] sethi %hi(1f), %g7 rdpr %tl, %g4 ba,pt %xcc, etraptl1 1: or %g7, %lo(1f), %g7 mov %l4, %o1 call sun4v_itlb_error_report add %sp, PTREGS_OFF, %o0 /* NOTREACHED */ sun4v_dtlb_error: rdpr %tl, %g1 cmp %g1, 1 ble,pt %icc, sun4v_bad_ra or %g0, FAULT_CODE_BAD_RA | FAULT_CODE_DTLB, %g1 sethi %hi(sun4v_err_dtlb_vaddr), %g1 stx %g4, [%g1 + %lo(sun4v_err_dtlb_vaddr)] sethi %hi(sun4v_err_dtlb_ctx), %g1 ldxa [%g0] ASI_SCRATCHPAD, %g6 ldx [%g6 + HV_FAULT_D_CTX_OFFSET], %o1 stx %o1, [%g1 + %lo(sun4v_err_dtlb_ctx)] sethi %hi(sun4v_err_dtlb_pte), %g1 stx %g3, [%g1 + %lo(sun4v_err_dtlb_pte)] sethi %hi(sun4v_err_dtlb_error), %g1 stx %o0, [%g1 + %lo(sun4v_err_dtlb_error)] sethi %hi(1f), %g7 rdpr %tl, %g4 ba,pt %xcc, etraptl1 1: or %g7, %lo(1f), %g7 mov %l4, %o1 call sun4v_dtlb_error_report add %sp, PTREGS_OFF, %o0 /* NOTREACHED */ sun4v_bad_ra: or %g0, %g4, %g5 ba,pt %xcc, sparc64_realfault_common or %g1, %g0, %g4 /* NOTREACHED */ /* Instruction Access Exception, tl0. */ sun4v_iacc: ldxa [%g0] ASI_SCRATCHPAD, %g2 ldx [%g2 + HV_FAULT_I_TYPE_OFFSET], %g3 ldx [%g2 + HV_FAULT_I_ADDR_OFFSET], %g4 ldx [%g2 + HV_FAULT_I_CTX_OFFSET], %g5 sllx %g3, 16, %g3 or %g5, %g3, %g5 ba,pt %xcc, etrap rd %pc, %g7 mov %l4, %o1 mov %l5, %o2 call sun4v_insn_access_exception add %sp, PTREGS_OFF, %o0 ba,a,pt %xcc, rtrap /* Instruction Access Exception, tl1. */ sun4v_iacc_tl1: ldxa [%g0] ASI_SCRATCHPAD, %g2 ldx [%g2 + HV_FAULT_I_TYPE_OFFSET], %g3 ldx [%g2 + HV_FAULT_I_ADDR_OFFSET], %g4 ldx [%g2 + HV_FAULT_I_CTX_OFFSET], %g5 sllx %g3, 16, %g3 or %g5, %g3, %g5 ba,pt %xcc, etraptl1 rd %pc, %g7 mov %l4, %o1 mov %l5, %o2 call sun4v_insn_access_exception_tl1 add %sp, PTREGS_OFF, %o0 ba,a,pt %xcc, rtrap /* Data Access Exception, tl0. */ sun4v_dacc: ldxa [%g0] ASI_SCRATCHPAD, %g2 ldx [%g2 + HV_FAULT_D_TYPE_OFFSET], %g3 ldx [%g2 + HV_FAULT_D_ADDR_OFFSET], %g4 ldx [%g2 + HV_FAULT_D_CTX_OFFSET], %g5 sllx %g3, 16, %g3 or %g5, %g3, %g5 ba,pt %xcc, etrap rd %pc, %g7 mov %l4, %o1 mov %l5, %o2 call sun4v_data_access_exception add %sp, PTREGS_OFF, %o0 ba,a,pt %xcc, rtrap /* Data Access Exception, tl1. */ sun4v_dacc_tl1: ldxa [%g0] ASI_SCRATCHPAD, %g2 ldx [%g2 + HV_FAULT_D_TYPE_OFFSET], %g3 ldx [%g2 + HV_FAULT_D_ADDR_OFFSET], %g4 ldx [%g2 + HV_FAULT_D_CTX_OFFSET], %g5 sllx %g3, 16, %g3 or %g5, %g3, %g5 ba,pt %xcc, etraptl1 rd %pc, %g7 mov %l4, %o1 mov %l5, %o2 call sun4v_data_access_exception_tl1 add %sp, PTREGS_OFF, %o0 ba,a,pt %xcc, rtrap /* Memory Address Unaligned. */ sun4v_mna: /* Window fixup? */ rdpr %tl, %g2 cmp %g2, 1 ble,pt %icc, 1f nop SET_GL(1) ldxa [%g0] ASI_SCRATCHPAD, %g2 ldx [%g2 + HV_FAULT_D_ADDR_OFFSET], %g5 mov HV_FAULT_TYPE_UNALIGNED, %g3 ldx [%g2 + HV_FAULT_D_CTX_OFFSET], %g4 sllx %g3, 16, %g3 or %g4, %g3, %g4 ba,pt %xcc, winfix_mna rdpr %tpc, %g3 /* not reached */ 1: ldxa [%g0] ASI_SCRATCHPAD, %g2 mov HV_FAULT_TYPE_UNALIGNED, %g3 ldx [%g2 + HV_FAULT_D_ADDR_OFFSET], %g4 ldx [%g2 + HV_FAULT_D_CTX_OFFSET], %g5 sllx %g3, 16, %g3 or %g5, %g3, %g5 ba,pt %xcc, etrap rd %pc, %g7 mov %l4, %o1 mov %l5, %o2 call sun4v_do_mna add %sp, PTREGS_OFF, %o0 ba,a,pt %xcc, rtrap nop /* Privileged Action. */ sun4v_privact: ba,pt %xcc, etrap rd %pc, %g7 call do_privact add %sp, PTREGS_OFF, %o0 ba,a,pt %xcc, rtrap /* Unaligned ldd float, tl0. */ sun4v_lddfmna: ldxa [%g0] ASI_SCRATCHPAD, %g2 ldx [%g2 + HV_FAULT_D_TYPE_OFFSET], %g3 ldx [%g2 + HV_FAULT_D_ADDR_OFFSET], %g4 ldx [%g2 + HV_FAULT_D_CTX_OFFSET], %g5 sllx %g3, 16, %g3 or %g5, %g3, %g5 ba,pt %xcc, etrap rd %pc, %g7 mov %l4, %o1 mov %l5, %o2 call handle_lddfmna add %sp, PTREGS_OFF, %o0 ba,a,pt %xcc, rtrap /* Unaligned std float, tl0. */ sun4v_stdfmna: ldxa [%g0] ASI_SCRATCHPAD, %g2 ldx [%g2 + HV_FAULT_D_TYPE_OFFSET], %g3 ldx [%g2 + HV_FAULT_D_ADDR_OFFSET], %g4 ldx [%g2 + HV_FAULT_D_CTX_OFFSET], %g5 sllx %g3, 16, %g3 or %g5, %g3, %g5 ba,pt %xcc, etrap rd %pc, %g7 mov %l4, %o1 mov %l5, %o2 call handle_stdfmna add %sp, PTREGS_OFF, %o0 ba,a,pt %xcc, rtrap #define BRANCH_ALWAYS 0x10680000 #define NOP 0x01000000 #define SUN4V_DO_PATCH(OLD, NEW) \ sethi %hi(NEW), %g1; \ or %g1, %lo(NEW), %g1; \ sethi %hi(OLD), %g2; \ or %g2, %lo(OLD), %g2; \ sub %g1, %g2, %g1; \ sethi %hi(BRANCH_ALWAYS), %g3; \ sll %g1, 11, %g1; \ srl %g1, 11 + 2, %g1; \ or %g3, %lo(BRANCH_ALWAYS), %g3; \ or %g3, %g1, %g3; \ stw %g3, [%g2]; \ sethi %hi(NOP), %g3; \ or %g3, %lo(NOP), %g3; \ stw %g3, [%g2 + 0x4]; \ flush %g2; .globl sun4v_patch_tlb_handlers .type sun4v_patch_tlb_handlers,#function sun4v_patch_tlb_handlers: SUN4V_DO_PATCH(tl0_iamiss, sun4v_itlb_miss) SUN4V_DO_PATCH(tl1_iamiss, sun4v_itlb_miss) SUN4V_DO_PATCH(tl0_damiss, sun4v_dtlb_miss) SUN4V_DO_PATCH(tl1_damiss, sun4v_dtlb_miss) SUN4V_DO_PATCH(tl0_daprot, sun4v_dtlb_prot) SUN4V_DO_PATCH(tl1_daprot, sun4v_dtlb_prot) SUN4V_DO_PATCH(tl0_iax, sun4v_iacc) SUN4V_DO_PATCH(tl1_iax, sun4v_iacc_tl1) SUN4V_DO_PATCH(tl0_dax, sun4v_dacc) SUN4V_DO_PATCH(tl1_dax, sun4v_dacc_tl1) SUN4V_DO_PATCH(tl0_mna, sun4v_mna) SUN4V_DO_PATCH(tl1_mna, sun4v_mna) SUN4V_DO_PATCH(tl0_lddfmna, sun4v_lddfmna) SUN4V_DO_PATCH(tl0_stdfmna, sun4v_stdfmna) SUN4V_DO_PATCH(tl0_privact, sun4v_privact) retl nop .size sun4v_patch_tlb_handlers,.-sun4v_patch_tlb_handlers