From dac628e9563640e2de7878decc03a508b1ba319a Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Sun, 26 Jan 2025 08:46:30 +0100 Subject: x86/efistub: Merge PE and handover entrypoints The difference between the PE and handover entrypoints in the EFI stub is that the former allocates a struct boot_params whereas the latter expects one from the caller. Currently, these are two completely separate entrypoints, duplicating some logic and both relying of efi_exit() to return straight back to the firmware on an error. Simplify this by making the PE entrypoint call the handover entrypoint with NULL as the argument for the struct boot_params parameter. This makes the code easier to follow, and removes the need to support two different calling conventions in the mixed mode asm code. While at it, move the assignment of boot_params_ptr into the function that actually calls into the legacy decompressor, which is where its value is required. Acked-by: Ingo Molnar Signed-off-by: Ard Biesheuvel --- arch/x86/boot/compressed/efi_mixed.S | 16 +--------- drivers/firmware/efi/libstub/x86-stub.c | 52 ++++++++++++++++++--------------- 2 files changed, 30 insertions(+), 38 deletions(-) diff --git a/arch/x86/boot/compressed/efi_mixed.S b/arch/x86/boot/compressed/efi_mixed.S index 876fc6d46a13..d681e30c6732 100644 --- a/arch/x86/boot/compressed/efi_mixed.S +++ b/arch/x86/boot/compressed/efi_mixed.S @@ -56,22 +56,8 @@ SYM_FUNC_START(startup_64_mixed_mode) movl efi32_boot_sp(%rip), %esp andl $~7, %esp -#ifdef CONFIG_EFI_HANDOVER_PROTOCOL mov 8(%rdx), %edx // saved bootparams pointer - test %edx, %edx - jnz efi_stub_entry -#endif - /* - * efi_pe_entry uses MS calling convention, which requires 32 bytes of - * shadow space on the stack even if all arguments are passed in - * registers. We also need an additional 8 bytes for the space that - * would be occupied by the return address, and this also results in - * the correct stack alignment for entry. - */ - sub $40, %rsp - mov %rdi, %rcx // MS calling convention - mov %rsi, %rdx - jmp efi_pe_entry + call efi_stub_entry SYM_FUNC_END(startup_64_mixed_mode) SYM_FUNC_START(__efi64_thunk) diff --git a/drivers/firmware/efi/libstub/x86-stub.c b/drivers/firmware/efi/libstub/x86-stub.c index 863910e9eefc..cafc90d4caaf 100644 --- a/drivers/firmware/efi/libstub/x86-stub.c +++ b/drivers/firmware/efi/libstub/x86-stub.c @@ -397,17 +397,13 @@ static void __noreturn efi_exit(efi_handle_t handle, efi_status_t status) asm("hlt"); } -void __noreturn efi_stub_entry(efi_handle_t handle, - efi_system_table_t *sys_table_arg, - struct boot_params *boot_params); - /* * Because the x86 boot code expects to be passed a boot_params we * need to create one ourselves (usually the bootloader would create * one for us). */ -efi_status_t __efiapi efi_pe_entry(efi_handle_t handle, - efi_system_table_t *sys_table_arg) +static efi_status_t efi_allocate_bootparams(efi_handle_t handle, + struct boot_params **bp) { efi_guid_t proto = LOADED_IMAGE_PROTOCOL_GUID; struct boot_params *boot_params; @@ -416,21 +412,15 @@ efi_status_t __efiapi efi_pe_entry(efi_handle_t handle, unsigned long alloc; char *cmdline_ptr; - efi_system_table = sys_table_arg; - - /* Check if we were booted by the EFI firmware */ - if (efi_system_table->hdr.signature != EFI_SYSTEM_TABLE_SIGNATURE) - efi_exit(handle, EFI_INVALID_PARAMETER); - status = efi_bs_call(handle_protocol, handle, &proto, (void **)&image); if (status != EFI_SUCCESS) { efi_err("Failed to get handle for LOADED_IMAGE_PROTOCOL\n"); - efi_exit(handle, status); + return status; } status = efi_allocate_pages(PARAM_SIZE, &alloc, ULONG_MAX); if (status != EFI_SUCCESS) - efi_exit(handle, status); + return status; boot_params = memset((void *)alloc, 0x0, PARAM_SIZE); hdr = &boot_params->hdr; @@ -446,14 +436,14 @@ efi_status_t __efiapi efi_pe_entry(efi_handle_t handle, cmdline_ptr = efi_convert_cmdline(image); if (!cmdline_ptr) { efi_free(PARAM_SIZE, alloc); - efi_exit(handle, EFI_OUT_OF_RESOURCES); + return EFI_OUT_OF_RESOURCES; } efi_set_u64_split((unsigned long)cmdline_ptr, &hdr->cmd_line_ptr, &boot_params->ext_cmd_line_ptr); - efi_stub_entry(handle, sys_table_arg, boot_params); - /* not reached */ + *bp = boot_params; + return EFI_SUCCESS; } static void add_e820ext(struct boot_params *params, @@ -740,13 +730,16 @@ static efi_status_t parse_options(const char *cmdline) return efi_parse_options(cmdline); } -static efi_status_t efi_decompress_kernel(unsigned long *kernel_entry) +static efi_status_t efi_decompress_kernel(unsigned long *kernel_entry, + struct boot_params *boot_params) { unsigned long virt_addr = LOAD_PHYSICAL_ADDR; unsigned long addr, alloc_size, entry; efi_status_t status; u32 seed[2] = {}; + boot_params_ptr = boot_params; + /* determine the required size of the allocation */ alloc_size = ALIGN(max_t(unsigned long, output_len, kernel_total_size), MIN_KERNEL_ALIGN); @@ -777,7 +770,7 @@ static efi_status_t efi_decompress_kernel(unsigned long *kernel_entry) seed[0] = 0; } - boot_params_ptr->hdr.loadflags |= KASLR_FLAG; + boot_params->hdr.loadflags |= KASLR_FLAG; } status = efi_random_alloc(alloc_size, CONFIG_PHYSICAL_ALIGN, &addr, @@ -815,20 +808,27 @@ static void __noreturn enter_kernel(unsigned long kernel_addr, void __noreturn efi_stub_entry(efi_handle_t handle, efi_system_table_t *sys_table_arg, struct boot_params *boot_params) + { efi_guid_t guid = EFI_MEMORY_ATTRIBUTE_PROTOCOL_GUID; - struct setup_header *hdr = &boot_params->hdr; const struct linux_efi_initrd *initrd = NULL; unsigned long kernel_entry; + struct setup_header *hdr; efi_status_t status; - boot_params_ptr = boot_params; - efi_system_table = sys_table_arg; /* Check if we were booted by the EFI firmware */ if (efi_system_table->hdr.signature != EFI_SYSTEM_TABLE_SIGNATURE) efi_exit(handle, EFI_INVALID_PARAMETER); + if (!IS_ENABLED(CONFIG_EFI_HANDOVER_PROTOCOL) || !boot_params) { + status = efi_allocate_bootparams(handle, &boot_params); + if (status != EFI_SUCCESS) + efi_exit(handle, status); + } + + hdr = &boot_params->hdr; + if (have_unsupported_snp_features()) efi_exit(handle, EFI_UNSUPPORTED); @@ -870,7 +870,7 @@ void __noreturn efi_stub_entry(efi_handle_t handle, if (efi_mem_encrypt > 0) hdr->xloadflags |= XLF_MEM_ENCRYPTION; - status = efi_decompress_kernel(&kernel_entry); + status = efi_decompress_kernel(&kernel_entry, boot_params); if (status != EFI_SUCCESS) { efi_err("Failed to decompress kernel\n"); goto fail; @@ -940,6 +940,12 @@ fail: efi_exit(handle, status); } +efi_status_t __efiapi efi_pe_entry(efi_handle_t handle, + efi_system_table_t *sys_table_arg) +{ + efi_stub_entry(handle, sys_table_arg, NULL); +} + #ifdef CONFIG_EFI_HANDOVER_PROTOCOL void efi_handover_entry(efi_handle_t handle, efi_system_table_t *sys_table_arg, struct boot_params *boot_params) -- cgit v1.2.3 From eaed89559591f73ca06a3e6534c381e3bd948ee6 Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Tue, 7 Jan 2025 15:57:16 +0100 Subject: x86/efi/mixed: Check CPU compatibility without relying on verify_cpu() In order for the EFI mixed mode startup code to be reusable in a context where the legacy decompressor is not used, replace the call to verify_cpu() [which performs an elaborate set of checks] with a simple check against the 'long mode' bit in the appropriate CPUID leaf. This is reasonable, given that EFI support is implied when booting in this manner, and so there is no need to consider very old CPUs when performing this check. Acked-by: Ingo Molnar Signed-off-by: Ard Biesheuvel --- arch/x86/boot/compressed/efi_mixed.S | 22 +++++++++------------- 1 file changed, 9 insertions(+), 13 deletions(-) diff --git a/arch/x86/boot/compressed/efi_mixed.S b/arch/x86/boot/compressed/efi_mixed.S index d681e30c6732..b7886e2591fc 100644 --- a/arch/x86/boot/compressed/efi_mixed.S +++ b/arch/x86/boot/compressed/efi_mixed.S @@ -279,24 +279,20 @@ SYM_FUNC_END(efi32_entry) * efi_system_table_32_t *sys_table) */ SYM_FUNC_START(efi32_pe_entry) - pushl %ebp - movl %esp, %ebp pushl %ebx // save callee-save registers - pushl %edi - - call verify_cpu // check for long mode support - testl %eax, %eax - movl $0x80000003, %eax // EFI_UNSUPPORTED - jnz 2f - movl 8(%ebp), %ecx // image_handle - movl 12(%ebp), %edx // sys_table + /* Check whether the CPU supports long mode */ + movl $0x80000001, %eax // assume extended info support + cpuid + btl $29, %edx // check long mode bit + jnc 1f + leal 8(%esp), %esp // preserve stack alignment + movl (%esp), %ecx // image_handle + movl 4(%esp), %edx // sys_table jmp efi32_entry // pass %ecx, %edx // no other registers remain live - -2: popl %edi // restore callee-save registers +1: movl $0x80000003, %eax // EFI_UNSUPPORTED popl %ebx - leave RET SYM_FUNC_END(efi32_pe_entry) -- cgit v1.2.3 From ff38bbbac39eade57e4672f9601f602ed8d4b493 Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Sun, 12 Jan 2025 13:56:46 +0100 Subject: x86/efi/mixed: Factor out and clean up long mode entry Entering long mode involves setting the EFER_LME and CR4.PAE bits before enabling paging by setting CR0.PG bit. It also involves disabling interrupts, given that the firmware's 32-bit IDT becomes invalid as soon as the CPU transitions into long mode. Reloading the CR3 register is not necessary at boot time, given that the EFI firmware as well as the kernel's EFI stub use a 1:1 mapping of the 32-bit addressable memory in the system. Break out this code into a separate helper for clarity, and so that it can be reused in a subsequent patch. Acked-by: Ingo Molnar Signed-off-by: Ard Biesheuvel --- arch/x86/boot/compressed/efi_mixed.S | 29 +++++++++++++++-------------- 1 file changed, 15 insertions(+), 14 deletions(-) diff --git a/arch/x86/boot/compressed/efi_mixed.S b/arch/x86/boot/compressed/efi_mixed.S index b7886e2591fc..0b6b37b08f82 100644 --- a/arch/x86/boot/compressed/efi_mixed.S +++ b/arch/x86/boot/compressed/efi_mixed.S @@ -170,10 +170,6 @@ SYM_FUNC_START_LOCAL(efi_enter32) movl %edx, %gs movl %edx, %ss - /* Reload pgtables */ - movl %cr3, %eax - movl %eax, %cr3 - /* Disable paging */ movl %cr0, %eax btrl $X86_CR0_PG_BIT, %eax @@ -199,30 +195,35 @@ SYM_FUNC_START_LOCAL(efi_enter32) lidtl 16(%ebx) lgdtl (%ebx) + xorl %eax, %eax + lldt %ax + + call efi32_enable_long_mode + + pushl $__KERNEL_CS + pushl %ebp + lret +SYM_FUNC_END(efi_enter32) + +SYM_FUNC_START_LOCAL(efi32_enable_long_mode) movl %cr4, %eax btsl $(X86_CR4_PAE_BIT), %eax movl %eax, %cr4 - movl %cr3, %eax - movl %eax, %cr3 - movl $MSR_EFER, %ecx rdmsr btsl $_EFER_LME, %eax wrmsr - xorl %eax, %eax - lldt %ax - - pushl $__KERNEL_CS - pushl %ebp + /* Disable interrupts - the firmware's IDT does not work in long mode */ + cli /* Enable paging */ movl %cr0, %eax btsl $X86_CR0_PG_BIT, %eax movl %eax, %cr0 - lret -SYM_FUNC_END(efi_enter32) + ret +SYM_FUNC_END(efi32_enable_long_mode) /* * This is the common EFI stub entry point for mixed mode. -- cgit v1.2.3 From d545e182a8bb37bce3c00d89ab88e33414add1c3 Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Wed, 5 Feb 2025 12:23:21 +0100 Subject: x86/efi/mixed: Set up 1:1 mapping of lower 4GiB in the stub In preparation for dropping the dependency on startup_32 entirely in the next patch, add the code that sets up the 1:1 mapping of the lower 4 GiB of system RAM to the mixed mode stub. The reload of CR3 after the long mode switch will be removed in a subsequent patch, when it is no longer needed. Acked-by: Ingo Molnar Signed-off-by: Ard Biesheuvel --- arch/x86/boot/compressed/efi_mixed.S | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/arch/x86/boot/compressed/efi_mixed.S b/arch/x86/boot/compressed/efi_mixed.S index 0b6b37b08f82..dca916c3e6f0 100644 --- a/arch/x86/boot/compressed/efi_mixed.S +++ b/arch/x86/boot/compressed/efi_mixed.S @@ -18,6 +18,7 @@ #include #include #include +#include #include #include #include @@ -52,6 +53,9 @@ SYM_FUNC_START(startup_64_mixed_mode) mov 0(%rdx), %edi mov 4(%rdx), %esi + leaq (pte + 5 * PAGE_SIZE)(%rip), %rax + movq %rax, %cr3 // reload after startup_32 + /* Switch to the firmware's stack */ movl efi32_boot_sp(%rip), %esp andl $~7, %esp @@ -267,11 +271,32 @@ SYM_FUNC_START_LOCAL(efi32_entry) movl $_end - 1b, BP_init_size(%esi) subl $startup_32 - 1b, BP_init_size(%esi) + call 1f +1: pop %edi + /* Disable paging */ movl %cr0, %eax btrl $X86_CR0_PG_BIT, %eax movl %eax, %cr0 + /* Set up 1:1 mapping */ + leal (pte - 1b)(%edi), %eax + movl $_PAGE_PRESENT | _PAGE_RW | _PAGE_PSE, %ecx + leal (_PAGE_PRESENT | _PAGE_RW)(%eax), %edx +2: movl %ecx, (%eax) + addl $8, %eax + addl $PMD_SIZE, %ecx + jnc 2b + + movl $PAGE_SIZE, %ecx + .irpc l, 0123 + movl %edx, \l * 8(%eax) + addl %ecx, %edx + .endr + addl %ecx, %eax + movl %edx, (%eax) + movl %eax, %cr3 + jmp startup_32 SYM_FUNC_END(efi32_entry) @@ -322,3 +347,7 @@ SYM_DATA_LOCAL(efi32_boot_ds, .word 0) SYM_DATA_LOCAL(efi32_boot_sp, .long 0) SYM_DATA_LOCAL(efi32_boot_args, .long 0, 0, 0) SYM_DATA(efi_is64, .byte 1) + + .bss + .balign PAGE_SIZE +SYM_DATA_LOCAL(pte, .fill 6 * PAGE_SIZE, 1, 0) -- cgit v1.2.3 From 6e2da8d87c9c1133d8d6e1f93a0bc0c416dbc8ee Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Tue, 7 Jan 2025 17:55:07 +0100 Subject: x86/efi/mixed: Remove dependency on legacy startup_32 code The EFI mixed mode startup code calls into startup_32 in the legacy decompressor with a mocked up boot_params struct, only to get it to set up the 1:1 mapping of the lower 4 GiB of memory and switch to a GDT that supports 64-bit mode. In order to be able to reuse the EFI mixed mode startup code in EFI zboot images, which do not incorporate the legacy decompressor code, decouple it, by dealing with the GDT and IDT directly. Doing so makes it possible to construct a GDT that is compatible with the one the firmware uses, with one additional entry for a 64-bit mode code segment appended. This removes the need entirely to switch between GDTs and IDTs or data segment selector values and all of this code can be removed. Acked-by: Ingo Molnar Signed-off-by: Ard Biesheuvel --- arch/x86/boot/compressed/efi_mixed.S | 227 ++++++++++------------------------- arch/x86/boot/compressed/head_64.S | 7 -- 2 files changed, 65 insertions(+), 169 deletions(-) diff --git a/arch/x86/boot/compressed/efi_mixed.S b/arch/x86/boot/compressed/efi_mixed.S index dca916c3e6f0..984956931ed7 100644 --- a/arch/x86/boot/compressed/efi_mixed.S +++ b/arch/x86/boot/compressed/efi_mixed.S @@ -15,70 +15,23 @@ */ #include -#include +#include #include #include #include #include #include -#include .code64 .text -/* - * When booting in 64-bit mode on 32-bit EFI firmware, startup_64_mixed_mode() - * is the first thing that runs after switching to long mode. Depending on - * whether the EFI handover protocol or the compat entry point was used to - * enter the kernel, it will either branch to the common 64-bit EFI stub - * entrypoint efi_stub_entry() directly, or via the 64-bit EFI PE/COFF - * entrypoint efi_pe_entry(). In the former case, the bootloader must provide a - * struct bootparams pointer as the third argument, so the presence of such a - * pointer is used to disambiguate. - * - * +--------------+ - * +------------------+ +------------+ +------>| efi_pe_entry | - * | efi32_pe_entry |---->| | | +-----------+--+ - * +------------------+ | | +------+----------------+ | - * | startup_32 |---->| startup_64_mixed_mode | | - * +------------------+ | | +------+----------------+ | - * | efi32_stub_entry |---->| | | | - * +------------------+ +------------+ | | - * V | - * +------------+ +----------------+ | - * | startup_64 |<----| efi_stub_entry |<--------+ - * +------------+ +----------------+ - */ -SYM_FUNC_START(startup_64_mixed_mode) - lea efi32_boot_args(%rip), %rdx - mov 0(%rdx), %edi - mov 4(%rdx), %esi - - leaq (pte + 5 * PAGE_SIZE)(%rip), %rax - movq %rax, %cr3 // reload after startup_32 - - /* Switch to the firmware's stack */ - movl efi32_boot_sp(%rip), %esp - andl $~7, %esp - - mov 8(%rdx), %edx // saved bootparams pointer - call efi_stub_entry -SYM_FUNC_END(startup_64_mixed_mode) - SYM_FUNC_START(__efi64_thunk) push %rbp push %rbx - movl %ds, %eax - push %rax - movl %es, %eax - push %rax - movl %ss, %eax - push %rax - /* Copy args passed on stack */ - movq 0x30(%rsp), %rbp - movq 0x38(%rsp), %rbx - movq 0x40(%rsp), %rax + movq 0x18(%rsp), %rbp + movq 0x20(%rsp), %rbx + movq 0x28(%rsp), %rax /* * Convert x86-64 ABI params to i386 ABI @@ -93,44 +46,14 @@ SYM_FUNC_START(__efi64_thunk) movl %ebx, 0x18(%rsp) movl %eax, 0x1c(%rsp) - leaq 0x20(%rsp), %rbx - sgdt (%rbx) - sidt 16(%rbx) - leaq 1f(%rip), %rbp + movl %cs, %ebx - /* - * Switch to IDT and GDT with 32-bit segments. These are the firmware - * GDT and IDT that were installed when the kernel started executing. - * The pointers were saved by the efi32_entry() routine below. - * - * Pass the saved DS selector to the 32-bit code, and use far return to - * restore the saved CS selector. - */ - lidt efi32_boot_idt(%rip) - lgdt efi32_boot_gdt(%rip) - - movzwl efi32_boot_ds(%rip), %edx - movzwq efi32_boot_cs(%rip), %rax - pushq %rax - leaq efi_enter32(%rip), %rax - pushq %rax - lretq + ljmpl *efi32_call(%rip) 1: addq $64, %rsp movq %rdi, %rax - pop %rbx - movl %ebx, %ss - pop %rbx - movl %ebx, %es - pop %rbx - movl %ebx, %ds - /* Clear out 32-bit selector from FS and GS */ - xorl %ebx, %ebx - movl %ebx, %fs - movl %ebx, %gs - pop %rbx pop %rbp RET @@ -141,7 +64,6 @@ SYM_FUNC_END(__efi64_thunk) SYM_FUNC_START(efi32_stub_entry) call 1f 1: popl %ecx - leal (efi32_boot_args - 1b)(%ecx), %ebx /* Clear BSS */ xorl %eax, %eax @@ -153,11 +75,8 @@ SYM_FUNC_START(efi32_stub_entry) rep stosl add $0x4, %esp /* Discard return address */ - popl %ecx - popl %edx - popl %esi - movl %esi, 8(%ebx) - jmp efi32_entry + movl 8(%esp), %ebx /* struct boot_params pointer */ + jmp efi32_startup SYM_FUNC_END(efi32_stub_entry) #endif @@ -167,13 +86,6 @@ SYM_FUNC_END(efi32_stub_entry) * The stack should represent the 32-bit calling convention. */ SYM_FUNC_START_LOCAL(efi_enter32) - /* Load firmware selector into data and stack segment registers */ - movl %edx, %ds - movl %edx, %es - movl %edx, %fs - movl %edx, %gs - movl %edx, %ss - /* Disable paging */ movl %cr0, %eax btrl $X86_CR0_PG_BIT, %eax @@ -190,21 +102,9 @@ SYM_FUNC_START_LOCAL(efi_enter32) /* We must preserve return value */ movl %eax, %edi - /* - * Some firmware will return with interrupts enabled. Be sure to - * disable them before we switch GDTs and IDTs. - */ - cli - - lidtl 16(%ebx) - lgdtl (%ebx) - - xorl %eax, %eax - lldt %ax - call efi32_enable_long_mode - pushl $__KERNEL_CS + pushl %ebx pushl %ebp lret SYM_FUNC_END(efi_enter32) @@ -230,50 +130,56 @@ SYM_FUNC_START_LOCAL(efi32_enable_long_mode) SYM_FUNC_END(efi32_enable_long_mode) /* - * This is the common EFI stub entry point for mixed mode. + * This is the common EFI stub entry point for mixed mode. It sets up the GDT + * and page tables needed for 64-bit execution, after which it calls the + * common 64-bit EFI entrypoint efi_stub_entry(). * - * Arguments: %ecx image handle - * %edx EFI system table pointer + * Arguments: 0(%esp) image handle + * 4(%esp) EFI system table pointer + * %ebx struct boot_params pointer (or NULL) * * Since this is the point of no return for ordinary execution, no registers * are considered live except for the function parameters. [Note that the EFI * stub may still exit and return to the firmware using the Exit() EFI boot * service.] */ -SYM_FUNC_START_LOCAL(efi32_entry) - call 1f -1: pop %ebx - - /* Save firmware GDTR and code/data selectors */ - sgdtl (efi32_boot_gdt - 1b)(%ebx) - movw %cs, (efi32_boot_cs - 1b)(%ebx) - movw %ds, (efi32_boot_ds - 1b)(%ebx) - - /* Store firmware IDT descriptor */ - sidtl (efi32_boot_idt - 1b)(%ebx) - - /* Store firmware stack pointer */ - movl %esp, (efi32_boot_sp - 1b)(%ebx) - - /* Store boot arguments */ - leal (efi32_boot_args - 1b)(%ebx), %ebx - movl %ecx, 0(%ebx) - movl %edx, 4(%ebx) - movb $0x0, 12(%ebx) // efi_is64 - - /* - * Allocate some memory for a temporary struct boot_params, which only - * needs the minimal pieces that startup_32() relies on. - */ - subl $PARAM_SIZE, %esp - movl %esp, %esi - movl $PAGE_SIZE, BP_kernel_alignment(%esi) - movl $_end - 1b, BP_init_size(%esi) - subl $startup_32 - 1b, BP_init_size(%esi) +SYM_FUNC_START_LOCAL(efi32_startup) + movl %esp, %ebp + + subl $8, %esp + sgdtl (%esp) /* Save GDT descriptor to the stack */ + movl 2(%esp), %esi /* Existing GDT pointer */ + movzwl (%esp), %ecx /* Existing GDT limit */ + inc %ecx /* Existing GDT size */ + andl $~7, %ecx /* Ensure size is multiple of 8 */ + + subl %ecx, %esp /* Allocate new GDT */ + andl $~15, %esp /* Realign the stack */ + movl %esp, %edi /* New GDT address */ + leal 7(%ecx), %eax /* New GDT limit */ + pushw %cx /* Push 64-bit CS (for LJMP below) */ + pushl %edi /* Push new GDT address */ + pushw %ax /* Push new GDT limit */ + + /* Copy GDT to the stack and add a 64-bit code segment at the end */ + movl $GDT_ENTRY(DESC_CODE64, 0, 0xfffff) & 0xffffffff, (%edi,%ecx) + movl $GDT_ENTRY(DESC_CODE64, 0, 0xfffff) >> 32, 4(%edi,%ecx) + shrl $2, %ecx + cld + rep movsl /* Copy the firmware GDT */ + lgdtl (%esp) /* Switch to the new GDT */ call 1f 1: pop %edi + /* Record mixed mode entry */ + movb $0x0, (efi_is64 - 1b)(%edi) + + /* Set up indirect far call to re-enter 32-bit mode */ + leal (efi32_call - 1b)(%edi), %eax + addl %eax, (%eax) + movw %cs, 4(%eax) + /* Disable paging */ movl %cr0, %eax btrl $X86_CR0_PG_BIT, %eax @@ -297,8 +203,17 @@ SYM_FUNC_START_LOCAL(efi32_entry) movl %edx, (%eax) movl %eax, %cr3 - jmp startup_32 -SYM_FUNC_END(efi32_entry) + call efi32_enable_long_mode + + /* Set up far jump to 64-bit mode (CS is already on the stack) */ + leal (efi_stub_entry - 1b)(%edi), %eax + movl %eax, 2(%esp) + + movl 0(%ebp), %edi + movl 4(%ebp), %esi + movl %ebx, %edx + ljmpl *2(%esp) +SYM_FUNC_END(efi32_startup) /* * efi_status_t efi32_pe_entry(efi_handle_t image_handle, @@ -313,10 +228,8 @@ SYM_FUNC_START(efi32_pe_entry) btl $29, %edx // check long mode bit jnc 1f leal 8(%esp), %esp // preserve stack alignment - movl (%esp), %ecx // image_handle - movl 4(%esp), %edx // sys_table - jmp efi32_entry // pass %ecx, %edx - // no other registers remain live + xor %ebx, %ebx // no struct boot_params pointer + jmp efi32_startup // only ESP and EBX remain live 1: movl $0x80000003, %eax // EFI_UNSUPPORTED popl %ebx RET @@ -332,20 +245,10 @@ SYM_FUNC_END(efi64_stub_entry) .data .balign 8 -SYM_DATA_START_LOCAL(efi32_boot_gdt) - .word 0 - .quad 0 -SYM_DATA_END(efi32_boot_gdt) - -SYM_DATA_START_LOCAL(efi32_boot_idt) - .word 0 - .quad 0 -SYM_DATA_END(efi32_boot_idt) - -SYM_DATA_LOCAL(efi32_boot_cs, .word 0) -SYM_DATA_LOCAL(efi32_boot_ds, .word 0) -SYM_DATA_LOCAL(efi32_boot_sp, .long 0) -SYM_DATA_LOCAL(efi32_boot_args, .long 0, 0, 0) +SYM_DATA_START_LOCAL(efi32_call) + .long efi_enter32 - . + .word 0x0 +SYM_DATA_END(efi32_call) SYM_DATA(efi_is64, .byte 1) .bss diff --git a/arch/x86/boot/compressed/head_64.S b/arch/x86/boot/compressed/head_64.S index 1dcb794c5479..5db6495a3bb9 100644 --- a/arch/x86/boot/compressed/head_64.S +++ b/arch/x86/boot/compressed/head_64.S @@ -263,13 +263,6 @@ SYM_FUNC_START(startup_32) * used to perform that far jump. */ leal rva(startup_64)(%ebp), %eax -#ifdef CONFIG_EFI_MIXED - cmpb $1, rva(efi_is64)(%ebp) - je 1f - leal rva(startup_64_mixed_mode)(%ebp), %eax -1: -#endif - pushl $__KERNEL_CS pushl %eax -- cgit v1.2.3 From b891e4209c9f96e25a4e90b86e460f515e902c37 Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Mon, 6 Jan 2025 14:20:35 +0100 Subject: x86/efi/mixed: Simplify and document thunking logic Now that the GDT/IDT and data segment selector preserve/restore logic has been removed from the boot-time EFI mixed mode thunking routines, the remaining logic to handle the function arguments can be simplified: the setup of the arguments on the stack can be moved into the 32-bit callee, which is able to use a more idiomatic sequence of PUSH instructions. This, in turn, allows the far call and far return to be issued using plain LCALL and LRET instructions, removing the need to set up the return explicitly. Acked-by: Ingo Molnar Signed-off-by: Ard Biesheuvel --- arch/x86/boot/compressed/efi_mixed.S | 77 +++++++++++++++++------------------- 1 file changed, 37 insertions(+), 40 deletions(-) diff --git a/arch/x86/boot/compressed/efi_mixed.S b/arch/x86/boot/compressed/efi_mixed.S index 984956931ed7..e04ed99bc449 100644 --- a/arch/x86/boot/compressed/efi_mixed.S +++ b/arch/x86/boot/compressed/efi_mixed.S @@ -22,43 +22,7 @@ #include #include - .code64 .text -SYM_FUNC_START(__efi64_thunk) - push %rbp - push %rbx - - /* Copy args passed on stack */ - movq 0x18(%rsp), %rbp - movq 0x20(%rsp), %rbx - movq 0x28(%rsp), %rax - - /* - * Convert x86-64 ABI params to i386 ABI - */ - subq $64, %rsp - movl %esi, 0x0(%rsp) - movl %edx, 0x4(%rsp) - movl %ecx, 0x8(%rsp) - movl %r8d, 0xc(%rsp) - movl %r9d, 0x10(%rsp) - movl %ebp, 0x14(%rsp) - movl %ebx, 0x18(%rsp) - movl %eax, 0x1c(%rsp) - - leaq 1f(%rip), %rbp - movl %cs, %ebx - - ljmpl *efi32_call(%rip) - -1: addq $64, %rsp - movq %rdi, %rax - - pop %rbx - pop %rbp - RET -SYM_FUNC_END(__efi64_thunk) - .code32 #ifdef CONFIG_EFI_HANDOVER_PROTOCOL SYM_FUNC_START(efi32_stub_entry) @@ -81,11 +45,26 @@ SYM_FUNC_END(efi32_stub_entry) #endif /* - * EFI service pointer must be in %edi. + * Called using a far call from __efi64_thunk() below, using the x86_64 SysV + * ABI (except for R8/R9 which are inaccessible to 32-bit code - EAX/EBX are + * used instead). EBP+16 points to the arguments passed via the stack. * - * The stack should represent the 32-bit calling convention. + * The first argument (EDI) is a pointer to the boot service or protocol, to + * which the remaining arguments are passed, each truncated to 32 bits. */ SYM_FUNC_START_LOCAL(efi_enter32) + /* + * Convert x86-64 SysV ABI params to i386 ABI + */ + pushl 32(%ebp) /* Up to 3 args passed via the stack */ + pushl 24(%ebp) + pushl 16(%ebp) + pushl %ebx /* R9 */ + pushl %eax /* R8 */ + pushl %ecx + pushl %edx + pushl %esi + /* Disable paging */ movl %cr0, %eax btrl $X86_CR0_PG_BIT, %eax @@ -104,11 +83,29 @@ SYM_FUNC_START_LOCAL(efi_enter32) call efi32_enable_long_mode - pushl %ebx - pushl %ebp + addl $32, %esp + movl %edi, %eax lret SYM_FUNC_END(efi_enter32) + .code64 +SYM_FUNC_START(__efi64_thunk) + push %rbp + movl %esp, %ebp + push %rbx + + /* Move args #5 and #6 into 32-bit accessible registers */ + movl %r8d, %eax + movl %r9d, %ebx + + lcalll *efi32_call(%rip) + + pop %rbx + pop %rbp + RET +SYM_FUNC_END(__efi64_thunk) + + .code32 SYM_FUNC_START_LOCAL(efi32_enable_long_mode) movl %cr4, %eax btsl $(X86_CR4_PAE_BIT), %eax -- cgit v1.2.3 From fb84cefd4ce77c3d63cd3d23adaa7faaef3737cc Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Tue, 7 Jan 2025 18:16:59 +0100 Subject: x86/efi/mixed: Move mixed mode startup code into libstub The EFI mixed mode code has been decoupled from the legacy decompressor, in order to be able to reuse it with generic EFI zboot images for x86. Move the source file into the libstub source directory to facilitate this. Acked-by: Ingo Molnar Signed-off-by: Ard Biesheuvel --- arch/x86/boot/compressed/Makefile | 1 - arch/x86/boot/compressed/efi_mixed.S | 253 ------------------------------- drivers/firmware/efi/libstub/Makefile | 3 + drivers/firmware/efi/libstub/x86-mixed.S | 253 +++++++++++++++++++++++++++++++ 4 files changed, 256 insertions(+), 254 deletions(-) delete mode 100644 arch/x86/boot/compressed/efi_mixed.S create mode 100644 drivers/firmware/efi/libstub/x86-mixed.S diff --git a/arch/x86/boot/compressed/Makefile b/arch/x86/boot/compressed/Makefile index f2051644de94..fc5563704466 100644 --- a/arch/x86/boot/compressed/Makefile +++ b/arch/x86/boot/compressed/Makefile @@ -104,7 +104,6 @@ vmlinux-objs-$(CONFIG_INTEL_TDX_GUEST) += $(obj)/tdx.o $(obj)/tdcall.o $(obj)/td vmlinux-objs-$(CONFIG_UNACCEPTED_MEMORY) += $(obj)/mem.o vmlinux-objs-$(CONFIG_EFI) += $(obj)/efi.o -vmlinux-objs-$(CONFIG_EFI_MIXED) += $(obj)/efi_mixed.o vmlinux-libs-$(CONFIG_EFI_STUB) += $(objtree)/drivers/firmware/efi/libstub/lib.a $(obj)/vmlinux: $(vmlinux-objs-y) $(vmlinux-libs-y) FORCE diff --git a/arch/x86/boot/compressed/efi_mixed.S b/arch/x86/boot/compressed/efi_mixed.S deleted file mode 100644 index e04ed99bc449..000000000000 --- a/arch/x86/boot/compressed/efi_mixed.S +++ /dev/null @@ -1,253 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -/* - * Copyright (C) 2014, 2015 Intel Corporation; author Matt Fleming - * - * Early support for invoking 32-bit EFI services from a 64-bit kernel. - * - * Because this thunking occurs before ExitBootServices() we have to - * restore the firmware's 32-bit GDT and IDT before we make EFI service - * calls. - * - * On the plus side, we don't have to worry about mangling 64-bit - * addresses into 32-bits because we're executing with an identity - * mapped pagetable and haven't transitioned to 64-bit virtual addresses - * yet. - */ - -#include -#include -#include -#include -#include -#include -#include - - .text - .code32 -#ifdef CONFIG_EFI_HANDOVER_PROTOCOL -SYM_FUNC_START(efi32_stub_entry) - call 1f -1: popl %ecx - - /* Clear BSS */ - xorl %eax, %eax - leal (_bss - 1b)(%ecx), %edi - leal (_ebss - 1b)(%ecx), %ecx - subl %edi, %ecx - shrl $2, %ecx - cld - rep stosl - - add $0x4, %esp /* Discard return address */ - movl 8(%esp), %ebx /* struct boot_params pointer */ - jmp efi32_startup -SYM_FUNC_END(efi32_stub_entry) -#endif - -/* - * Called using a far call from __efi64_thunk() below, using the x86_64 SysV - * ABI (except for R8/R9 which are inaccessible to 32-bit code - EAX/EBX are - * used instead). EBP+16 points to the arguments passed via the stack. - * - * The first argument (EDI) is a pointer to the boot service or protocol, to - * which the remaining arguments are passed, each truncated to 32 bits. - */ -SYM_FUNC_START_LOCAL(efi_enter32) - /* - * Convert x86-64 SysV ABI params to i386 ABI - */ - pushl 32(%ebp) /* Up to 3 args passed via the stack */ - pushl 24(%ebp) - pushl 16(%ebp) - pushl %ebx /* R9 */ - pushl %eax /* R8 */ - pushl %ecx - pushl %edx - pushl %esi - - /* Disable paging */ - movl %cr0, %eax - btrl $X86_CR0_PG_BIT, %eax - movl %eax, %cr0 - - /* Disable long mode via EFER */ - movl $MSR_EFER, %ecx - rdmsr - btrl $_EFER_LME, %eax - wrmsr - - call *%edi - - /* We must preserve return value */ - movl %eax, %edi - - call efi32_enable_long_mode - - addl $32, %esp - movl %edi, %eax - lret -SYM_FUNC_END(efi_enter32) - - .code64 -SYM_FUNC_START(__efi64_thunk) - push %rbp - movl %esp, %ebp - push %rbx - - /* Move args #5 and #6 into 32-bit accessible registers */ - movl %r8d, %eax - movl %r9d, %ebx - - lcalll *efi32_call(%rip) - - pop %rbx - pop %rbp - RET -SYM_FUNC_END(__efi64_thunk) - - .code32 -SYM_FUNC_START_LOCAL(efi32_enable_long_mode) - movl %cr4, %eax - btsl $(X86_CR4_PAE_BIT), %eax - movl %eax, %cr4 - - movl $MSR_EFER, %ecx - rdmsr - btsl $_EFER_LME, %eax - wrmsr - - /* Disable interrupts - the firmware's IDT does not work in long mode */ - cli - - /* Enable paging */ - movl %cr0, %eax - btsl $X86_CR0_PG_BIT, %eax - movl %eax, %cr0 - ret -SYM_FUNC_END(efi32_enable_long_mode) - -/* - * This is the common EFI stub entry point for mixed mode. It sets up the GDT - * and page tables needed for 64-bit execution, after which it calls the - * common 64-bit EFI entrypoint efi_stub_entry(). - * - * Arguments: 0(%esp) image handle - * 4(%esp) EFI system table pointer - * %ebx struct boot_params pointer (or NULL) - * - * Since this is the point of no return for ordinary execution, no registers - * are considered live except for the function parameters. [Note that the EFI - * stub may still exit and return to the firmware using the Exit() EFI boot - * service.] - */ -SYM_FUNC_START_LOCAL(efi32_startup) - movl %esp, %ebp - - subl $8, %esp - sgdtl (%esp) /* Save GDT descriptor to the stack */ - movl 2(%esp), %esi /* Existing GDT pointer */ - movzwl (%esp), %ecx /* Existing GDT limit */ - inc %ecx /* Existing GDT size */ - andl $~7, %ecx /* Ensure size is multiple of 8 */ - - subl %ecx, %esp /* Allocate new GDT */ - andl $~15, %esp /* Realign the stack */ - movl %esp, %edi /* New GDT address */ - leal 7(%ecx), %eax /* New GDT limit */ - pushw %cx /* Push 64-bit CS (for LJMP below) */ - pushl %edi /* Push new GDT address */ - pushw %ax /* Push new GDT limit */ - - /* Copy GDT to the stack and add a 64-bit code segment at the end */ - movl $GDT_ENTRY(DESC_CODE64, 0, 0xfffff) & 0xffffffff, (%edi,%ecx) - movl $GDT_ENTRY(DESC_CODE64, 0, 0xfffff) >> 32, 4(%edi,%ecx) - shrl $2, %ecx - cld - rep movsl /* Copy the firmware GDT */ - lgdtl (%esp) /* Switch to the new GDT */ - - call 1f -1: pop %edi - - /* Record mixed mode entry */ - movb $0x0, (efi_is64 - 1b)(%edi) - - /* Set up indirect far call to re-enter 32-bit mode */ - leal (efi32_call - 1b)(%edi), %eax - addl %eax, (%eax) - movw %cs, 4(%eax) - - /* Disable paging */ - movl %cr0, %eax - btrl $X86_CR0_PG_BIT, %eax - movl %eax, %cr0 - - /* Set up 1:1 mapping */ - leal (pte - 1b)(%edi), %eax - movl $_PAGE_PRESENT | _PAGE_RW | _PAGE_PSE, %ecx - leal (_PAGE_PRESENT | _PAGE_RW)(%eax), %edx -2: movl %ecx, (%eax) - addl $8, %eax - addl $PMD_SIZE, %ecx - jnc 2b - - movl $PAGE_SIZE, %ecx - .irpc l, 0123 - movl %edx, \l * 8(%eax) - addl %ecx, %edx - .endr - addl %ecx, %eax - movl %edx, (%eax) - movl %eax, %cr3 - - call efi32_enable_long_mode - - /* Set up far jump to 64-bit mode (CS is already on the stack) */ - leal (efi_stub_entry - 1b)(%edi), %eax - movl %eax, 2(%esp) - - movl 0(%ebp), %edi - movl 4(%ebp), %esi - movl %ebx, %edx - ljmpl *2(%esp) -SYM_FUNC_END(efi32_startup) - -/* - * efi_status_t efi32_pe_entry(efi_handle_t image_handle, - * efi_system_table_32_t *sys_table) - */ -SYM_FUNC_START(efi32_pe_entry) - pushl %ebx // save callee-save registers - - /* Check whether the CPU supports long mode */ - movl $0x80000001, %eax // assume extended info support - cpuid - btl $29, %edx // check long mode bit - jnc 1f - leal 8(%esp), %esp // preserve stack alignment - xor %ebx, %ebx // no struct boot_params pointer - jmp efi32_startup // only ESP and EBX remain live -1: movl $0x80000003, %eax // EFI_UNSUPPORTED - popl %ebx - RET -SYM_FUNC_END(efi32_pe_entry) - -#ifdef CONFIG_EFI_HANDOVER_PROTOCOL - .org efi32_stub_entry + 0x200 - .code64 -SYM_FUNC_START_NOALIGN(efi64_stub_entry) - jmp efi_handover_entry -SYM_FUNC_END(efi64_stub_entry) -#endif - - .data - .balign 8 -SYM_DATA_START_LOCAL(efi32_call) - .long efi_enter32 - . - .word 0x0 -SYM_DATA_END(efi32_call) -SYM_DATA(efi_is64, .byte 1) - - .bss - .balign PAGE_SIZE -SYM_DATA_LOCAL(pte, .fill 6 * PAGE_SIZE, 1, 0) diff --git a/drivers/firmware/efi/libstub/Makefile b/drivers/firmware/efi/libstub/Makefile index 1141cd06011f..903afd2d3d58 100644 --- a/drivers/firmware/efi/libstub/Makefile +++ b/drivers/firmware/efi/libstub/Makefile @@ -62,6 +62,8 @@ KBUILD_CFLAGS := $(filter-out $(CC_FLAGS_LTO), $(KBUILD_CFLAGS)) # `-fdata-sections` flag from KBUILD_CFLAGS_KERNEL KBUILD_CFLAGS_KERNEL := $(filter-out -fdata-sections, $(KBUILD_CFLAGS_KERNEL)) +KBUILD_AFLAGS := $(KBUILD_CFLAGS) -D__ASSEMBLY__ + lib-y := efi-stub-helper.o gop.o secureboot.o tpm.o \ file.o mem.o random.o randomalloc.o pci.o \ skip_spaces.o lib-cmdline.o lib-ctype.o \ @@ -83,6 +85,7 @@ lib-$(CONFIG_EFI_GENERIC_STUB) += efi-stub.o string.o intrinsics.o systable.o \ lib-$(CONFIG_ARM) += arm32-stub.o lib-$(CONFIG_ARM64) += kaslr.o arm64.o arm64-stub.o smbios.o lib-$(CONFIG_X86) += x86-stub.o smbios.o +lib-$(CONFIG_EFI_MIXED) += x86-mixed.o lib-$(CONFIG_X86_64) += x86-5lvl.o lib-$(CONFIG_RISCV) += kaslr.o riscv.o riscv-stub.o lib-$(CONFIG_LOONGARCH) += loongarch.o loongarch-stub.o diff --git a/drivers/firmware/efi/libstub/x86-mixed.S b/drivers/firmware/efi/libstub/x86-mixed.S new file mode 100644 index 000000000000..e04ed99bc449 --- /dev/null +++ b/drivers/firmware/efi/libstub/x86-mixed.S @@ -0,0 +1,253 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2014, 2015 Intel Corporation; author Matt Fleming + * + * Early support for invoking 32-bit EFI services from a 64-bit kernel. + * + * Because this thunking occurs before ExitBootServices() we have to + * restore the firmware's 32-bit GDT and IDT before we make EFI service + * calls. + * + * On the plus side, we don't have to worry about mangling 64-bit + * addresses into 32-bits because we're executing with an identity + * mapped pagetable and haven't transitioned to 64-bit virtual addresses + * yet. + */ + +#include +#include +#include +#include +#include +#include +#include + + .text + .code32 +#ifdef CONFIG_EFI_HANDOVER_PROTOCOL +SYM_FUNC_START(efi32_stub_entry) + call 1f +1: popl %ecx + + /* Clear BSS */ + xorl %eax, %eax + leal (_bss - 1b)(%ecx), %edi + leal (_ebss - 1b)(%ecx), %ecx + subl %edi, %ecx + shrl $2, %ecx + cld + rep stosl + + add $0x4, %esp /* Discard return address */ + movl 8(%esp), %ebx /* struct boot_params pointer */ + jmp efi32_startup +SYM_FUNC_END(efi32_stub_entry) +#endif + +/* + * Called using a far call from __efi64_thunk() below, using the x86_64 SysV + * ABI (except for R8/R9 which are inaccessible to 32-bit code - EAX/EBX are + * used instead). EBP+16 points to the arguments passed via the stack. + * + * The first argument (EDI) is a pointer to the boot service or protocol, to + * which the remaining arguments are passed, each truncated to 32 bits. + */ +SYM_FUNC_START_LOCAL(efi_enter32) + /* + * Convert x86-64 SysV ABI params to i386 ABI + */ + pushl 32(%ebp) /* Up to 3 args passed via the stack */ + pushl 24(%ebp) + pushl 16(%ebp) + pushl %ebx /* R9 */ + pushl %eax /* R8 */ + pushl %ecx + pushl %edx + pushl %esi + + /* Disable paging */ + movl %cr0, %eax + btrl $X86_CR0_PG_BIT, %eax + movl %eax, %cr0 + + /* Disable long mode via EFER */ + movl $MSR_EFER, %ecx + rdmsr + btrl $_EFER_LME, %eax + wrmsr + + call *%edi + + /* We must preserve return value */ + movl %eax, %edi + + call efi32_enable_long_mode + + addl $32, %esp + movl %edi, %eax + lret +SYM_FUNC_END(efi_enter32) + + .code64 +SYM_FUNC_START(__efi64_thunk) + push %rbp + movl %esp, %ebp + push %rbx + + /* Move args #5 and #6 into 32-bit accessible registers */ + movl %r8d, %eax + movl %r9d, %ebx + + lcalll *efi32_call(%rip) + + pop %rbx + pop %rbp + RET +SYM_FUNC_END(__efi64_thunk) + + .code32 +SYM_FUNC_START_LOCAL(efi32_enable_long_mode) + movl %cr4, %eax + btsl $(X86_CR4_PAE_BIT), %eax + movl %eax, %cr4 + + movl $MSR_EFER, %ecx + rdmsr + btsl $_EFER_LME, %eax + wrmsr + + /* Disable interrupts - the firmware's IDT does not work in long mode */ + cli + + /* Enable paging */ + movl %cr0, %eax + btsl $X86_CR0_PG_BIT, %eax + movl %eax, %cr0 + ret +SYM_FUNC_END(efi32_enable_long_mode) + +/* + * This is the common EFI stub entry point for mixed mode. It sets up the GDT + * and page tables needed for 64-bit execution, after which it calls the + * common 64-bit EFI entrypoint efi_stub_entry(). + * + * Arguments: 0(%esp) image handle + * 4(%esp) EFI system table pointer + * %ebx struct boot_params pointer (or NULL) + * + * Since this is the point of no return for ordinary execution, no registers + * are considered live except for the function parameters. [Note that the EFI + * stub may still exit and return to the firmware using the Exit() EFI boot + * service.] + */ +SYM_FUNC_START_LOCAL(efi32_startup) + movl %esp, %ebp + + subl $8, %esp + sgdtl (%esp) /* Save GDT descriptor to the stack */ + movl 2(%esp), %esi /* Existing GDT pointer */ + movzwl (%esp), %ecx /* Existing GDT limit */ + inc %ecx /* Existing GDT size */ + andl $~7, %ecx /* Ensure size is multiple of 8 */ + + subl %ecx, %esp /* Allocate new GDT */ + andl $~15, %esp /* Realign the stack */ + movl %esp, %edi /* New GDT address */ + leal 7(%ecx), %eax /* New GDT limit */ + pushw %cx /* Push 64-bit CS (for LJMP below) */ + pushl %edi /* Push new GDT address */ + pushw %ax /* Push new GDT limit */ + + /* Copy GDT to the stack and add a 64-bit code segment at the end */ + movl $GDT_ENTRY(DESC_CODE64, 0, 0xfffff) & 0xffffffff, (%edi,%ecx) + movl $GDT_ENTRY(DESC_CODE64, 0, 0xfffff) >> 32, 4(%edi,%ecx) + shrl $2, %ecx + cld + rep movsl /* Copy the firmware GDT */ + lgdtl (%esp) /* Switch to the new GDT */ + + call 1f +1: pop %edi + + /* Record mixed mode entry */ + movb $0x0, (efi_is64 - 1b)(%edi) + + /* Set up indirect far call to re-enter 32-bit mode */ + leal (efi32_call - 1b)(%edi), %eax + addl %eax, (%eax) + movw %cs, 4(%eax) + + /* Disable paging */ + movl %cr0, %eax + btrl $X86_CR0_PG_BIT, %eax + movl %eax, %cr0 + + /* Set up 1:1 mapping */ + leal (pte - 1b)(%edi), %eax + movl $_PAGE_PRESENT | _PAGE_RW | _PAGE_PSE, %ecx + leal (_PAGE_PRESENT | _PAGE_RW)(%eax), %edx +2: movl %ecx, (%eax) + addl $8, %eax + addl $PMD_SIZE, %ecx + jnc 2b + + movl $PAGE_SIZE, %ecx + .irpc l, 0123 + movl %edx, \l * 8(%eax) + addl %ecx, %edx + .endr + addl %ecx, %eax + movl %edx, (%eax) + movl %eax, %cr3 + + call efi32_enable_long_mode + + /* Set up far jump to 64-bit mode (CS is already on the stack) */ + leal (efi_stub_entry - 1b)(%edi), %eax + movl %eax, 2(%esp) + + movl 0(%ebp), %edi + movl 4(%ebp), %esi + movl %ebx, %edx + ljmpl *2(%esp) +SYM_FUNC_END(efi32_startup) + +/* + * efi_status_t efi32_pe_entry(efi_handle_t image_handle, + * efi_system_table_32_t *sys_table) + */ +SYM_FUNC_START(efi32_pe_entry) + pushl %ebx // save callee-save registers + + /* Check whether the CPU supports long mode */ + movl $0x80000001, %eax // assume extended info support + cpuid + btl $29, %edx // check long mode bit + jnc 1f + leal 8(%esp), %esp // preserve stack alignment + xor %ebx, %ebx // no struct boot_params pointer + jmp efi32_startup // only ESP and EBX remain live +1: movl $0x80000003, %eax // EFI_UNSUPPORTED + popl %ebx + RET +SYM_FUNC_END(efi32_pe_entry) + +#ifdef CONFIG_EFI_HANDOVER_PROTOCOL + .org efi32_stub_entry + 0x200 + .code64 +SYM_FUNC_START_NOALIGN(efi64_stub_entry) + jmp efi_handover_entry +SYM_FUNC_END(efi64_stub_entry) +#endif + + .data + .balign 8 +SYM_DATA_START_LOCAL(efi32_call) + .long efi_enter32 - . + .word 0x0 +SYM_DATA_END(efi32_call) +SYM_DATA(efi_is64, .byte 1) + + .bss + .balign PAGE_SIZE +SYM_DATA_LOCAL(pte, .fill 6 * PAGE_SIZE, 1, 0) -- cgit v1.2.3 From b6b227e36b5ad878260f5a3a1838f2d79d5e68e9 Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Thu, 6 Mar 2025 14:08:27 +0100 Subject: efivarfs: Revert "allow creation of zero length files" As agreed with the fwupd/LVFS maintainer, this reverts commit fc20737d8b85691ecabab3739ed7d06c9b7bc00f again for the v6.15 cycle, leaving them sufficient time to roll out a fix for the issue that the reverted commit works around. Link: https://lore.kernel.org/all/63837c36eceaf8cf2af7933dccca54ff4dd9f30d.camel@HansenPartnership.com/ Signed-off-by: Ard Biesheuvel --- fs/efivarfs/file.c | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/fs/efivarfs/file.c b/fs/efivarfs/file.c index c294a8fc566d..cb1b6d0c3454 100644 --- a/fs/efivarfs/file.c +++ b/fs/efivarfs/file.c @@ -57,11 +57,10 @@ static ssize_t efivarfs_file_write(struct file *file, if (bytes == -ENOENT) { /* - * FIXME: temporary workaround for fwupdate, signal - * failed write with a 1 to keep created but not - * written files + * zero size signals to release that the write deleted + * the variable */ - i_size_write(inode, 1); + i_size_write(inode, 0); } else { i_size_write(inode, datasize + sizeof(attributes)); inode_set_mtime_to_ts(inode, inode_set_ctime_current(inode)); @@ -125,8 +124,7 @@ static int efivarfs_file_release(struct inode *inode, struct file *file) struct efivar_entry *var = inode->i_private; inode_lock(inode); - /* FIXME: temporary work around for fwupdate */ - var->removed = (--var->open_count == 0 && i_size_read(inode) == 1); + var->removed = (--var->open_count == 0 && i_size_read(inode) == 0); inode_unlock(inode); if (var->removed) -- cgit v1.2.3 From ac2efaa8455021ce1e6216457684d60a9e2c77fd Mon Sep 17 00:00:00 2001 From: Ethan Carter Edwards Date: Sat, 8 Mar 2025 20:27:41 -0500 Subject: efi: efibc: change kmalloc(size * count, ...) to kmalloc_array() Open coded arithmetic in allocator arguments is discouraged. Helper functions like kcalloc or, in this case, kmalloc_array are preferred. Link: https://www.kernel.org/doc/html/latest/process/deprecated.html#open-coded-arithmetic-in-allocator-arguments Signed-off-by: Ethan Carter Edwards Signed-off-by: Ard Biesheuvel --- drivers/firmware/efi/efibc.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/firmware/efi/efibc.c b/drivers/firmware/efi/efibc.c index 4f9fb086eab7..0a7c764dcc61 100644 --- a/drivers/firmware/efi/efibc.c +++ b/drivers/firmware/efi/efibc.c @@ -47,7 +47,7 @@ static int efibc_reboot_notifier_call(struct notifier_block *notifier, if (ret || !data) return NOTIFY_DONE; - wdata = kmalloc(MAX_DATA_LEN * sizeof(efi_char16_t), GFP_KERNEL); + wdata = kmalloc_array(MAX_DATA_LEN, sizeof(efi_char16_t), GFP_KERNEL); if (!wdata) return NOTIFY_DONE; -- cgit v1.2.3 From 74d613e046e418ed512b265aa2ef8a27a761fb4d Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Thu, 27 Feb 2025 18:35:11 +0100 Subject: efi/libstub: Avoid CopyMem/SetMem EFI services after ExitBootServices Given that memset/memcpy are intrinsics, the compiler might insert calls to these routines unexpectedly, including in code that executes after ExitBootServices(). In this case, the respective boot services are no longer accessible, and calling them will cause a crash. So fall back to a bytewise copy/store if this happens to occur, even though no such occurrences are known to exist in the kernel currently. Signed-off-by: Ard Biesheuvel --- drivers/firmware/efi/libstub/intrinsics.c | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/drivers/firmware/efi/libstub/intrinsics.c b/drivers/firmware/efi/libstub/intrinsics.c index 965e734f6f98..418cd2e6dccc 100644 --- a/drivers/firmware/efi/libstub/intrinsics.c +++ b/drivers/firmware/efi/libstub/intrinsics.c @@ -15,8 +15,31 @@ void *__memmove(void *__dest, const void *__src, size_t count) __alias(memmove); void *__memset(void *s, int c, size_t count) __alias(memset); #endif +static void *efistub_memmove(u8 *dst, const u8 *src, size_t len) +{ + if (src > dst || dst >= (src + len)) + for (size_t i = 0; i < len; i++) + dst[i] = src[i]; + else + for (ssize_t i = len - 1; i >= 0; i--) + dst[i] = src[i]; + + return dst; +} + +static void *efistub_memset(void *dst, int c, size_t len) +{ + for (u8 *d = dst; len--; d++) + *d = c; + + return dst; +} + void *memcpy(void *dst, const void *src, size_t len) { + if (efi_table_attr(efi_system_table, boottime) == NULL) + return efistub_memmove(dst, src, len); + efi_bs_call(copy_mem, dst, src, len); return dst; } @@ -25,6 +48,9 @@ extern void *memmove(void *dst, const void *src, size_t len) __alias(memcpy); void *memset(void *dst, int c, size_t len) { + if (efi_table_attr(efi_system_table, boottime) == NULL) + return efistub_memset(dst, c, len); + efi_bs_call(set_mem, dst, len, c & U8_MAX); return dst; } -- cgit v1.2.3 From 0dc1754e16b4c14ae42f6cf59f319331d885f0f6 Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Wed, 20 Nov 2024 20:36:03 +0100 Subject: efi/libstub: Avoid legacy decompressor zlib/zstd wrappers Remove EFI zboot's dependency on the decompression wrappers used by the legacy decompressor boot code, which can only process the input in one go, and this will not work for upcoming support for embedded ELF images. They also do some odd things like providing a barebones malloc() implementation, which is not needed in a hosted environment such as the EFI boot services. So instead, implement GZIP deflate and ZSTD decompression in terms of the underlying libraries. Support for other compression algoritms has already been dropped. Signed-off-by: Ard Biesheuvel --- drivers/firmware/efi/libstub/Makefile | 7 ++- drivers/firmware/efi/libstub/efistub.h | 3 + .../firmware/efi/libstub/zboot-decompress-gzip.c | 68 ++++++++++++++++++++++ .../firmware/efi/libstub/zboot-decompress-zstd.c | 49 ++++++++++++++++ drivers/firmware/efi/libstub/zboot.c | 65 +++------------------ drivers/firmware/efi/libstub/zboot.lds | 1 + 6 files changed, 136 insertions(+), 57 deletions(-) create mode 100644 drivers/firmware/efi/libstub/zboot-decompress-gzip.c create mode 100644 drivers/firmware/efi/libstub/zboot-decompress-zstd.c diff --git a/drivers/firmware/efi/libstub/Makefile b/drivers/firmware/efi/libstub/Makefile index 903afd2d3d58..d23a1b9fed75 100644 --- a/drivers/firmware/efi/libstub/Makefile +++ b/drivers/firmware/efi/libstub/Makefile @@ -92,7 +92,12 @@ lib-$(CONFIG_LOONGARCH) += loongarch.o loongarch-stub.o CFLAGS_arm32-stub.o := -DTEXT_OFFSET=$(TEXT_OFFSET) -zboot-obj-$(CONFIG_RISCV) := lib-clz_ctz.o lib-ashldi3.o +zboot-obj-y := zboot-decompress-gzip.o +CFLAGS_zboot-decompress-gzip.o += -I$(srctree)/lib/zlib_inflate +zboot-obj-$(CONFIG_KERNEL_ZSTD) := zboot-decompress-zstd.o lib-xxhash.o +CFLAGS_zboot-decompress-zstd.o += -I$(srctree)/lib/zstd + +zboot-obj-$(CONFIG_RISCV) += lib-clz_ctz.o lib-ashldi3.o lib-$(CONFIG_EFI_ZBOOT) += zboot.o $(zboot-obj-y) lib-$(CONFIG_UNACCEPTED_MEMORY) += unaccepted_memory.o bitmap.o find.o diff --git a/drivers/firmware/efi/libstub/efistub.h b/drivers/firmware/efi/libstub/efistub.h index d96d4494070d..f5ba032863a9 100644 --- a/drivers/firmware/efi/libstub/efistub.h +++ b/drivers/firmware/efi/libstub/efistub.h @@ -1234,4 +1234,7 @@ void process_unaccepted_memory(u64 start, u64 end); void accept_memory(phys_addr_t start, unsigned long size); void arch_accept_memory(phys_addr_t start, phys_addr_t end); +efi_status_t efi_zboot_decompress_init(unsigned long *alloc_size); +efi_status_t efi_zboot_decompress(u8 *out, unsigned long outlen); + #endif diff --git a/drivers/firmware/efi/libstub/zboot-decompress-gzip.c b/drivers/firmware/efi/libstub/zboot-decompress-gzip.c new file mode 100644 index 000000000000..e97a7e9d3c98 --- /dev/null +++ b/drivers/firmware/efi/libstub/zboot-decompress-gzip.c @@ -0,0 +1,68 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include +#include + +#include + +#include "efistub.h" + +#include "inftrees.c" +#include "inffast.c" +#include "inflate.c" + +extern unsigned char _gzdata_start[], _gzdata_end[]; +extern u32 __aligned(1) payload_size; + +static struct z_stream_s stream; + +efi_status_t efi_zboot_decompress_init(unsigned long *alloc_size) +{ + efi_status_t status; + int rc; + + /* skip the 10 byte header, assume no recorded filename */ + stream.next_in = _gzdata_start + 10; + stream.avail_in = _gzdata_end - stream.next_in; + + status = efi_allocate_pages(zlib_inflate_workspacesize(), + (unsigned long *)&stream.workspace, + ULONG_MAX); + if (status != EFI_SUCCESS) + return status; + + rc = zlib_inflateInit2(&stream, -MAX_WBITS); + if (rc != Z_OK) { + efi_err("failed to initialize GZIP decompressor: %d\n", rc); + status = EFI_LOAD_ERROR; + goto out; + } + + *alloc_size = payload_size; + return EFI_SUCCESS; +out: + efi_free(zlib_inflate_workspacesize(), (unsigned long)stream.workspace); + return status; +} + +efi_status_t efi_zboot_decompress(u8 *out, unsigned long outlen) +{ + int rc; + + stream.next_out = out; + stream.avail_out = outlen; + + rc = zlib_inflate(&stream, 0); + zlib_inflateEnd(&stream); + + efi_free(zlib_inflate_workspacesize(), (unsigned long)stream.workspace); + + if (rc != Z_STREAM_END) { + efi_err("GZIP decompression failed with status %d\n", rc); + return EFI_LOAD_ERROR; + } + + efi_cache_sync_image((unsigned long)out, outlen); + + return EFI_SUCCESS; +} diff --git a/drivers/firmware/efi/libstub/zboot-decompress-zstd.c b/drivers/firmware/efi/libstub/zboot-decompress-zstd.c new file mode 100644 index 000000000000..bde9d94dd2e3 --- /dev/null +++ b/drivers/firmware/efi/libstub/zboot-decompress-zstd.c @@ -0,0 +1,49 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include +#include + +#include + +#include "decompress_sources.h" +#include "efistub.h" + +extern unsigned char _gzdata_start[], _gzdata_end[]; +extern u32 __aligned(1) payload_size; + +static size_t wksp_size; +static void *wksp; + +efi_status_t efi_zboot_decompress_init(unsigned long *alloc_size) +{ + efi_status_t status; + + wksp_size = zstd_dctx_workspace_bound(); + status = efi_allocate_pages(wksp_size, (unsigned long *)&wksp, ULONG_MAX); + if (status != EFI_SUCCESS) + return status; + + *alloc_size = payload_size; + return EFI_SUCCESS; +} + +efi_status_t efi_zboot_decompress(u8 *out, unsigned long outlen) +{ + zstd_dctx *dctx = zstd_init_dctx(wksp, wksp_size); + size_t ret; + int retval; + + ret = zstd_decompress_dctx(dctx, out, outlen, _gzdata_start, + _gzdata_end - _gzdata_start - 4); + efi_free(wksp_size, (unsigned long)wksp); + + retval = zstd_get_error_code(ret); + if (retval) { + efi_err("ZSTD-decompression failed with status %d\n", retval); + return EFI_LOAD_ERROR; + } + + efi_cache_sync_image((unsigned long)out, outlen); + + return EFI_SUCCESS; +} diff --git a/drivers/firmware/efi/libstub/zboot.c b/drivers/firmware/efi/libstub/zboot.c index af23b3c50228..c47ace06f010 100644 --- a/drivers/firmware/efi/libstub/zboot.c +++ b/drivers/firmware/efi/libstub/zboot.c @@ -7,36 +7,6 @@ #include "efistub.h" -static unsigned char zboot_heap[SZ_256K] __aligned(64); -static unsigned long free_mem_ptr, free_mem_end_ptr; - -#define STATIC static -#if defined(CONFIG_KERNEL_GZIP) -#include "../../../../lib/decompress_inflate.c" -#elif defined(CONFIG_KERNEL_LZ4) -#include "../../../../lib/decompress_unlz4.c" -#elif defined(CONFIG_KERNEL_LZMA) -#include "../../../../lib/decompress_unlzma.c" -#elif defined(CONFIG_KERNEL_LZO) -#include "../../../../lib/decompress_unlzo.c" -#elif defined(CONFIG_KERNEL_XZ) -#undef memcpy -#define memcpy memcpy -#undef memmove -#define memmove memmove -#include "../../../../lib/decompress_unxz.c" -#elif defined(CONFIG_KERNEL_ZSTD) -#include "../../../../lib/decompress_unzstd.c" -#endif - -extern char efi_zboot_header[]; -extern char _gzdata_start[], _gzdata_end[]; - -static void error(char *x) -{ - efi_err("EFI decompressor: %s\n", x); -} - static unsigned long alloc_preferred_address(unsigned long alloc_size) { #ifdef EFI_KIMG_PREFERRED_ADDRESS @@ -64,22 +34,17 @@ struct screen_info *alloc_screen_info(void) asmlinkage efi_status_t __efiapi efi_zboot_entry(efi_handle_t handle, efi_system_table_t *systab) { - unsigned long compressed_size = _gzdata_end - _gzdata_start; + char *cmdline_ptr __free(efi_pool) = NULL; unsigned long image_base, alloc_size; efi_loaded_image_t *image; efi_status_t status; - char *cmdline_ptr; - int ret; WRITE_ONCE(efi_system_table, systab); - free_mem_ptr = (unsigned long)&zboot_heap; - free_mem_end_ptr = free_mem_ptr + sizeof(zboot_heap); - status = efi_bs_call(handle_protocol, handle, &LOADED_IMAGE_PROTOCOL_GUID, (void **)&image); if (status != EFI_SUCCESS) { - error("Failed to locate parent's loaded image protocol"); + efi_err("Failed to locate parent's loaded image protocol\n"); return status; } @@ -89,9 +54,9 @@ efi_zboot_entry(efi_handle_t handle, efi_system_table_t *systab) efi_info("Decompressing Linux Kernel...\n"); - // SizeOfImage from the compressee's PE/COFF header - alloc_size = round_up(get_unaligned_le32(_gzdata_end - 4), - EFI_ALLOC_ALIGN); + status = efi_zboot_decompress_init(&alloc_size); + if (status != EFI_SUCCESS) + return status; // If the architecture has a preferred address for the image, // try that first. @@ -122,26 +87,14 @@ efi_zboot_entry(efi_handle_t handle, efi_system_table_t *systab) seed, EFI_LOADER_CODE, 0, EFI_ALLOC_LIMIT); if (status != EFI_SUCCESS) { efi_err("Failed to allocate memory\n"); - goto free_cmdline; + return status; } } - // Decompress the payload into the newly allocated buffer. - ret = __decompress(_gzdata_start, compressed_size, NULL, NULL, - (void *)image_base, alloc_size, NULL, error); - if (ret < 0) { - error("Decompression failed"); - status = EFI_DEVICE_ERROR; - goto free_image; - } - - efi_cache_sync_image(image_base, alloc_size); - - status = efi_stub_common(handle, image, image_base, cmdline_ptr); + // Decompress the payload into the newly allocated buffer + status = efi_zboot_decompress((void *)image_base, alloc_size) ?: + efi_stub_common(handle, image, image_base, cmdline_ptr); -free_image: efi_free(alloc_size, image_base); -free_cmdline: - efi_bs_call(free_pool, cmdline_ptr); return status; } diff --git a/drivers/firmware/efi/libstub/zboot.lds b/drivers/firmware/efi/libstub/zboot.lds index af2c82f7bd90..9ecc57ff5b45 100644 --- a/drivers/firmware/efi/libstub/zboot.lds +++ b/drivers/firmware/efi/libstub/zboot.lds @@ -17,6 +17,7 @@ SECTIONS .rodata : ALIGN(8) { __efistub__gzdata_start = .; *(.gzdata) + __efistub_payload_size = . - 4; __efistub__gzdata_end = .; *(.rodata* .init.rodata* .srodata*) -- cgit v1.2.3