/* SPDX-License-Identifier: GPL-2.0-only */ /* * relocate_kernel.S - put the kernel image in place to boot * Copyright (C) 2002-2004 Eric Biederman <ebiederm@xmission.com> */ #include <linux/linkage.h> #include <asm/page_types.h> #include <asm/kexec.h> #include <asm/nospec-branch.h> #include <asm/processor-flags.h> /* * Must be relocatable PIC code callable as a C function, in particular * there must be a plain RET and not jump to return thunk. */ #define PTR(x) (x << 2) /* * control_page + KEXEC_CONTROL_CODE_MAX_SIZE * ~ control_page + PAGE_SIZE are used as data storage and stack for * jumping back */ #define DATA(offset) (KEXEC_CONTROL_CODE_MAX_SIZE+(offset)) /* Minimal CPU state */ #define ESP DATA(0x0) #define CR0 DATA(0x4) #define CR3 DATA(0x8) #define CR4 DATA(0xc) /* other data */ #define CP_VA_CONTROL_PAGE DATA(0x10) #define CP_PA_PGD DATA(0x14) #define CP_PA_SWAP_PAGE DATA(0x18) #define CP_PA_BACKUP_PAGES_MAP DATA(0x1c) .text SYM_CODE_START_NOALIGN(relocate_kernel) /* Save the CPU context, used for jumping back */ pushl %ebx pushl %esi pushl %edi pushl %ebp pushf movl 20+8(%esp), %ebp /* list of pages */ movl PTR(VA_CONTROL_PAGE)(%ebp), %edi movl %esp, ESP(%edi) movl %cr0, %eax movl %eax, CR0(%edi) movl %cr3, %eax movl %eax, CR3(%edi) movl %cr4, %eax movl %eax, CR4(%edi) /* read the arguments and say goodbye to the stack */ movl 20+4(%esp), %ebx /* page_list */ movl 20+8(%esp), %ebp /* list of pages */ movl 20+12(%esp), %edx /* start address */ movl 20+16(%esp), %ecx /* cpu_has_pae */ movl 20+20(%esp), %esi /* preserve_context */ /* zero out flags, and disable interrupts */ pushl $0 popfl /* save some information for jumping back */ movl PTR(VA_CONTROL_PAGE)(%ebp), %edi movl %edi, CP_VA_CONTROL_PAGE(%edi) movl PTR(PA_PGD)(%ebp), %eax movl %eax, CP_PA_PGD(%edi) movl PTR(PA_SWAP_PAGE)(%ebp), %eax movl %eax, CP_PA_SWAP_PAGE(%edi) movl %ebx, CP_PA_BACKUP_PAGES_MAP(%edi) /* * get physical address of control page now * this is impossible after page table switch */ movl PTR(PA_CONTROL_PAGE)(%ebp), %edi /* switch to new set of page tables */ movl PTR(PA_PGD)(%ebp), %eax movl %eax, %cr3 /* setup a new stack at the end of the physical control page */ lea PAGE_SIZE(%edi), %esp /* jump to identity mapped page */ movl %edi, %eax addl $(identity_mapped - relocate_kernel), %eax pushl %eax ANNOTATE_UNRET_SAFE ret int3 SYM_CODE_END(relocate_kernel) SYM_CODE_START_LOCAL_NOALIGN(identity_mapped) /* set return address to 0 if not preserving context */ pushl $0 /* store the start address on the stack */ pushl %edx /* * Set cr0 to a known state: * - Paging disabled * - Alignment check disabled * - Write protect disabled * - No task switch * - Don't do FP software emulation. * - Protected mode enabled */ movl %cr0, %eax andl $~(X86_CR0_PG | X86_CR0_AM | X86_CR0_WP | X86_CR0_TS | X86_CR0_EM), %eax orl $(X86_CR0_PE), %eax movl %eax, %cr0 /* clear cr4 if applicable */ testl %ecx, %ecx jz 1f /* * Set cr4 to a known state: * Setting everything to zero seems safe. */ xorl %eax, %eax movl %eax, %cr4 jmp 1f 1: /* Flush the TLB (needed?) */ xorl %eax, %eax movl %eax, %cr3 movl CP_PA_SWAP_PAGE(%edi), %eax pushl %eax pushl %ebx call swap_pages addl $8, %esp /* * To be certain of avoiding problems with self-modifying code * I need to execute a serializing instruction here. * So I flush the TLB, it's handy, and not processor dependent. */ xorl %eax, %eax movl %eax, %cr3 /* * set all of the registers to known values * leave %esp alone */ testl %esi, %esi jnz 1f xorl %edi, %edi xorl %eax, %eax xorl %ebx, %ebx xorl %ecx, %ecx xorl %edx, %edx xorl %esi, %esi xorl %ebp, %ebp ANNOTATE_UNRET_SAFE ret int3 1: popl %edx movl CP_PA_SWAP_PAGE(%edi), %esp addl $PAGE_SIZE, %esp 2: ANNOTATE_RETPOLINE_SAFE call *%edx /* get the re-entry point of the peer system */ movl 0(%esp), %ebp call 1f 1: popl %ebx subl $(1b - relocate_kernel), %ebx movl CP_VA_CONTROL_PAGE(%ebx), %edi lea PAGE_SIZE(%ebx), %esp movl CP_PA_SWAP_PAGE(%ebx), %eax movl CP_PA_BACKUP_PAGES_MAP(%ebx), %edx pushl %eax pushl %edx call swap_pages addl $8, %esp movl CP_PA_PGD(%ebx), %eax movl %eax, %cr3 movl %cr0, %eax orl $X86_CR0_PG, %eax movl %eax, %cr0 lea PAGE_SIZE(%edi), %esp movl %edi, %eax addl $(virtual_mapped - relocate_kernel), %eax pushl %eax ANNOTATE_UNRET_SAFE ret int3 SYM_CODE_END(identity_mapped) SYM_CODE_START_LOCAL_NOALIGN(virtual_mapped) movl CR4(%edi), %eax movl %eax, %cr4 movl CR3(%edi), %eax movl %eax, %cr3 movl CR0(%edi), %eax movl %eax, %cr0 movl ESP(%edi), %esp movl %ebp, %eax popf popl %ebp popl %edi popl %esi popl %ebx ANNOTATE_UNRET_SAFE ret int3 SYM_CODE_END(virtual_mapped) /* Do the copies */ SYM_CODE_START_LOCAL_NOALIGN(swap_pages) movl 8(%esp), %edx movl 4(%esp), %ecx pushl %ebp pushl %ebx pushl %edi pushl %esi movl %ecx, %ebx jmp 1f 0: /* top, read another word from the indirection page */ movl (%ebx), %ecx addl $4, %ebx 1: testb $0x1, %cl /* is it a destination page */ jz 2f movl %ecx, %edi andl $0xfffff000, %edi jmp 0b 2: testb $0x2, %cl /* is it an indirection page */ jz 2f movl %ecx, %ebx andl $0xfffff000, %ebx jmp 0b 2: testb $0x4, %cl /* is it the done indicator */ jz 2f jmp 3f 2: testb $0x8, %cl /* is it the source indicator */ jz 0b /* Ignore it otherwise */ movl %ecx, %esi /* For every source page do a copy */ andl $0xfffff000, %esi movl %edi, %eax movl %esi, %ebp movl %edx, %edi movl $1024, %ecx rep ; movsl movl %ebp, %edi movl %eax, %esi movl $1024, %ecx rep ; movsl movl %eax, %edi movl %edx, %esi movl $1024, %ecx rep ; movsl lea PAGE_SIZE(%ebp), %esi jmp 0b 3: popl %esi popl %edi popl %ebx popl %ebp ANNOTATE_UNRET_SAFE ret int3 SYM_CODE_END(swap_pages) .globl kexec_control_code_size .set kexec_control_code_size, . - relocate_kernel