/* SPDX-License-Identifier: GPL-2.0-or-later */ /* * This file contains low-level functions for performing various * types of TLB invalidations on various processors with no hash * table. * * This file implements the following functions for all no-hash * processors. Some aren't implemented for some variants. Some * are inline in tlbflush.h * * - tlbil_va * - tlbil_pid * - tlbil_all * - tlbivax_bcast * * Code mostly moved over from misc_32.S * * Copyright (C) 1995-1996 Gary Thomas (gdt@linuxppc.org) * * Partially rewritten by Cort Dougan (cort@cs.nmt.edu) * Paul Mackerras, Kumar Gala and Benjamin Herrenschmidt. */ #include <asm/reg.h> #include <asm/page.h> #include <asm/cputable.h> #include <asm/mmu.h> #include <asm/ppc_asm.h> #include <asm/asm-offsets.h> #include <asm/processor.h> #include <asm/bug.h> #include <asm/asm-compat.h> #include <asm/feature-fixups.h> #if defined(CONFIG_40x) /* * 40x implementation needs only tlbil_va */ _GLOBAL(__tlbil_va) /* We run the search with interrupts disabled because we have to change * the PID and I don't want to preempt when that happens. */ mfmsr r5 mfspr r6,SPRN_PID wrteei 0 mtspr SPRN_PID,r4 tlbsx. r3, 0, r3 mtspr SPRN_PID,r6 wrtee r5 bne 1f sync /* There are only 64 TLB entries, so r3 < 64, which means bit 25 is * clear. Since 25 is the V bit in the TLB_TAG, loading this value * will invalidate the TLB entry. */ tlbwe r3, r3, TLB_TAG isync 1: blr #elif defined(CONFIG_PPC_8xx) /* * Nothing to do for 8xx, everything is inline */ #elif defined(CONFIG_44x) /* Includes 47x */ /* * 440 implementation uses tlbsx/we for tlbil_va and a full sweep * of the TLB for everything else. */ _GLOBAL(__tlbil_va) mfspr r5,SPRN_MMUCR mfmsr r10 /* * We write 16 bits of STID since 47x supports that much, we * will never be passed out of bounds values on 440 (hopefully) */ rlwimi r5,r4,0,16,31 /* We have to run the search with interrupts disabled, otherwise * an interrupt which causes a TLB miss can clobber the MMUCR * between the mtspr and the tlbsx. * * Critical and Machine Check interrupts take care of saving * and restoring MMUCR, so only normal interrupts have to be * taken care of. */ wrteei 0 mtspr SPRN_MMUCR,r5 tlbsx. r6,0,r3 bne 10f sync #ifndef CONFIG_PPC_47x /* On 440 There are only 64 TLB entries, so r3 < 64, which means bit * 22, is clear. Since 22 is the V bit in the TLB_PAGEID, loading this * value will invalidate the TLB entry. */ tlbwe r6,r6,PPC44x_TLB_PAGEID #else oris r7,r6,0x8000 /* specify way explicitly */ clrrwi r4,r3,12 /* get an EPN for the hashing with V = 0 */ ori r4,r4,PPC47x_TLBE_SIZE tlbwe r4,r7,0 /* write it */ #endif /* !CONFIG_PPC_47x */ isync 10: wrtee r10 blr _GLOBAL(_tlbil_all) _GLOBAL(_tlbil_pid) #ifndef CONFIG_PPC_47x li r3,0 sync /* Load high watermark */ lis r4,tlb_44x_hwater@ha lwz r5,tlb_44x_hwater@l(r4) 1: tlbwe r3,r3,PPC44x_TLB_PAGEID addi r3,r3,1 cmpw 0,r3,r5 ble 1b isync blr #else /* 476 variant. There's not simple way to do this, hopefully we'll * try to limit the amount of such full invalidates */ mfmsr r11 /* Interrupts off */ wrteei 0 li r3,-1 /* Current set */ lis r10,tlb_47x_boltmap@h ori r10,r10,tlb_47x_boltmap@l lis r7,0x8000 /* Specify way explicitly */ b 9f /* For each set */ 1: li r9,4 /* Number of ways */ li r4,0 /* Current way */ li r6,0 /* Default entry value 0 */ andi. r0,r8,1 /* Check if way 0 is bolted */ mtctr r9 /* Load way counter */ bne- 3f /* Bolted, skip loading it */ 2: /* For each way */ or r5,r3,r4 /* Make way|index for tlbre */ rlwimi r5,r5,16,8,15 /* Copy index into position */ tlbre r6,r5,0 /* Read entry */ 3: addis r4,r4,0x2000 /* Next way */ andi. r0,r6,PPC47x_TLB0_VALID /* Valid entry ? */ beq 4f /* Nope, skip it */ rlwimi r7,r5,0,1,2 /* Insert way number */ rlwinm r6,r6,0,21,19 /* Clear V */ tlbwe r6,r7,0 /* Write it */ 4: bdnz 2b /* Loop for each way */ srwi r8,r8,1 /* Next boltmap bit */ 9: cmpwi cr1,r3,255 /* Last set done ? */ addi r3,r3,1 /* Next set */ beq cr1,1f /* End of loop */ andi. r0,r3,0x1f /* Need to load a new boltmap word ? */ bne 1b /* No, loop */ lwz r8,0(r10) /* Load boltmap entry */ addi r10,r10,4 /* Next word */ b 1b /* Then loop */ 1: isync /* Sync shadows */ wrtee r11 blr #endif /* !CONFIG_PPC_47x */ #ifdef CONFIG_PPC_47x /* * _tlbivax_bcast is only on 47x. We don't bother doing a runtime * check though, it will blow up soon enough if we mistakenly try * to use it on a 440. */ _GLOBAL(_tlbivax_bcast) mfspr r5,SPRN_MMUCR mfmsr r10 rlwimi r5,r4,0,16,31 wrteei 0 mtspr SPRN_MMUCR,r5 isync PPC_TLBIVAX(0, R3) isync mbar tlbsync BEGIN_FTR_SECTION b 1f END_FTR_SECTION_IFSET(CPU_FTR_476_DD2) sync wrtee r10 blr /* * DD2 HW could hang if in instruction fetch happens before msync completes. * Touch enough instruction cache lines to ensure cache hits */ 1: mflr r9 bcl 20,31,$+4 2: mflr r6 li r7,32 PPC_ICBT(0,R6,R7) /* touch next cache line */ add r6,r6,r7 PPC_ICBT(0,R6,R7) /* touch next cache line */ add r6,r6,r7 PPC_ICBT(0,R6,R7) /* touch next cache line */ sync nop nop nop nop nop nop nop nop mtlr r9 wrtee r10 blr #endif /* CONFIG_PPC_47x */ #elif defined(CONFIG_PPC_85xx) /* * FSL BookE implementations. * * Since feature sections are using _SECTION_ELSE we need * to have the larger code path before the _SECTION_ELSE */ /* * Flush MMU TLB on the local processor */ _GLOBAL(_tlbil_all) BEGIN_MMU_FTR_SECTION li r3,(MMUCSR0_TLBFI)@l mtspr SPRN_MMUCSR0, r3 1: mfspr r3,SPRN_MMUCSR0 andi. r3,r3,MMUCSR0_TLBFI@l bne 1b MMU_FTR_SECTION_ELSE PPC_TLBILX_ALL(0,R0) ALT_MMU_FTR_SECTION_END_IFCLR(MMU_FTR_USE_TLBILX) msync isync blr _GLOBAL(_tlbil_pid) BEGIN_MMU_FTR_SECTION slwi r3,r3,16 mfmsr r10 wrteei 0 mfspr r4,SPRN_MAS6 /* save MAS6 */ mtspr SPRN_MAS6,r3 PPC_TLBILX_PID(0,R0) mtspr SPRN_MAS6,r4 /* restore MAS6 */ wrtee r10 MMU_FTR_SECTION_ELSE li r3,(MMUCSR0_TLBFI)@l mtspr SPRN_MMUCSR0, r3 1: mfspr r3,SPRN_MMUCSR0 andi. r3,r3,MMUCSR0_TLBFI@l bne 1b ALT_MMU_FTR_SECTION_END_IFSET(MMU_FTR_USE_TLBILX) msync isync blr /* * Flush MMU TLB for a particular address, but only on the local processor * (no broadcast) */ _GLOBAL(__tlbil_va) mfmsr r10 wrteei 0 slwi r4,r4,16 ori r4,r4,(MAS6_ISIZE(BOOK3E_PAGESZ_4K))@l mtspr SPRN_MAS6,r4 /* assume AS=0 for now */ BEGIN_MMU_FTR_SECTION tlbsx 0,r3 mfspr r4,SPRN_MAS1 /* check valid */ andis. r3,r4,MAS1_VALID@h beq 1f rlwinm r4,r4,0,1,31 mtspr SPRN_MAS1,r4 tlbwe MMU_FTR_SECTION_ELSE PPC_TLBILX_VA(0,R3) ALT_MMU_FTR_SECTION_END_IFCLR(MMU_FTR_USE_TLBILX) msync isync 1: wrtee r10 blr #elif defined(CONFIG_PPC_BOOK3E_64) /* * New Book3E (>= 2.06) implementation * * Note: We may be able to get away without the interrupt masking stuff * if we save/restore MAS6 on exceptions that might modify it */ _GLOBAL(_tlbil_pid) slwi r4,r3,MAS6_SPID_SHIFT mfmsr r10 wrteei 0 mtspr SPRN_MAS6,r4 PPC_TLBILX_PID(0,R0) wrtee r10 msync isync blr _GLOBAL(_tlbil_pid_noind) slwi r4,r3,MAS6_SPID_SHIFT mfmsr r10 ori r4,r4,MAS6_SIND wrteei 0 mtspr SPRN_MAS6,r4 PPC_TLBILX_PID(0,R0) wrtee r10 msync isync blr _GLOBAL(_tlbil_all) PPC_TLBILX_ALL(0,R0) msync isync blr _GLOBAL(_tlbil_va) mfmsr r10 wrteei 0 cmpwi cr0,r6,0 slwi r4,r4,MAS6_SPID_SHIFT rlwimi r4,r5,MAS6_ISIZE_SHIFT,MAS6_ISIZE_MASK beq 1f rlwimi r4,r6,MAS6_SIND_SHIFT,MAS6_SIND 1: mtspr SPRN_MAS6,r4 /* assume AS=0 for now */ PPC_TLBILX_VA(0,R3) msync isync wrtee r10 blr _GLOBAL(_tlbivax_bcast) mfmsr r10 wrteei 0 cmpwi cr0,r6,0 slwi r4,r4,MAS6_SPID_SHIFT rlwimi r4,r5,MAS6_ISIZE_SHIFT,MAS6_ISIZE_MASK beq 1f rlwimi r4,r6,MAS6_SIND_SHIFT,MAS6_SIND 1: mtspr SPRN_MAS6,r4 /* assume AS=0 for now */ PPC_TLBIVAX(0,R3) mbar tlbsync sync wrtee r10 blr #else #error Unsupported processor type ! #endif #if defined(CONFIG_PPC_E500) /* * extern void loadcam_entry(unsigned int index) * * Load TLBCAM[index] entry in to the L2 CAM MMU * Must preserve r7, r8, r9, r10, r11, r12 */ _GLOBAL(loadcam_entry) mflr r5 LOAD_REG_ADDR_PIC(r4, TLBCAM) mtlr r5 mulli r5,r3,TLBCAM_SIZE add r3,r5,r4 lwz r4,TLBCAM_MAS0(r3) mtspr SPRN_MAS0,r4 lwz r4,TLBCAM_MAS1(r3) mtspr SPRN_MAS1,r4 PPC_LL r4,TLBCAM_MAS2(r3) mtspr SPRN_MAS2,r4 lwz r4,TLBCAM_MAS3(r3) mtspr SPRN_MAS3,r4 BEGIN_MMU_FTR_SECTION lwz r4,TLBCAM_MAS7(r3) mtspr SPRN_MAS7,r4 END_MMU_FTR_SECTION_IFSET(MMU_FTR_BIG_PHYS) isync tlbwe isync blr /* * Load multiple TLB entries at once, using an alternate-space * trampoline so that we don't have to care about whether the same * TLB entry maps us before and after. * * r3 = first entry to write * r4 = number of entries to write * r5 = temporary tlb entry (0 means no switch to AS1) */ _GLOBAL(loadcam_multi) mflr r8 /* Don't switch to AS=1 if already there */ mfmsr r11 andi. r11,r11,MSR_IS bne 10f mr. r12, r5 beq 10f /* * Set up temporary TLB entry that is the same as what we're * running from, but in AS=1. */ bcl 20,31,$+4 1: mflr r6 tlbsx 0,r8 mfspr r6,SPRN_MAS1 ori r6,r6,MAS1_TS mtspr SPRN_MAS1,r6 mfspr r6,SPRN_MAS0 rlwimi r6,r5,MAS0_ESEL_SHIFT,MAS0_ESEL_MASK mr r7,r5 mtspr SPRN_MAS0,r6 isync tlbwe isync /* Switch to AS=1 */ mfmsr r6 ori r6,r6,MSR_IS|MSR_DS mtmsr r6 isync 10: mr r9,r3 add r10,r3,r4 2: bl loadcam_entry addi r9,r9,1 cmpw r9,r10 mr r3,r9 blt 2b /* Don't return to AS=0 if we were in AS=1 at function start */ andi. r11,r11,MSR_IS bne 3f cmpwi r12, 0 beq 3f /* Return to AS=0 and clear the temporary entry */ mfmsr r6 rlwinm. r6,r6,0,~(MSR_IS|MSR_DS) mtmsr r6 isync li r6,0 mtspr SPRN_MAS1,r6 rlwinm r6,r7,MAS0_ESEL_SHIFT,MAS0_ESEL_MASK oris r6,r6,MAS0_TLBSEL(1)@h mtspr SPRN_MAS0,r6 isync tlbwe isync 3: mtlr r8 blr #endif