/* SPDX-License-Identifier: GPL-2.0-only */ /* * Accelerated CRC32(C) using AArch64 CRC instructions * * Copyright (C) 2016 - 2018 Linaro Ltd <ard.biesheuvel@linaro.org> */ #include <linux/linkage.h> #include <asm/alternative.h> #include <asm/assembler.h> .arch armv8-a+crc .macro byteorder, reg, be .if \be CPU_LE( rev \reg, \reg ) .else CPU_BE( rev \reg, \reg ) .endif .endm .macro byteorder16, reg, be .if \be CPU_LE( rev16 \reg, \reg ) .else CPU_BE( rev16 \reg, \reg ) .endif .endm .macro bitorder, reg, be .if \be rbit \reg, \reg .endif .endm .macro bitorder16, reg, be .if \be rbit \reg, \reg lsr \reg, \reg, #16 .endif .endm .macro bitorder8, reg, be .if \be rbit \reg, \reg lsr \reg, \reg, #24 .endif .endm .macro __crc32, c, be=0 bitorder w0, \be cmp x2, #16 b.lt 8f // less than 16 bytes and x7, x2, #0x1f and x2, x2, #~0x1f cbz x7, 32f // multiple of 32 bytes and x8, x7, #0xf ldp x3, x4, [x1] add x8, x8, x1 add x1, x1, x7 ldp x5, x6, [x8] byteorder x3, \be byteorder x4, \be byteorder x5, \be byteorder x6, \be bitorder x3, \be bitorder x4, \be bitorder x5, \be bitorder x6, \be tst x7, #8 crc32\c\()x w8, w0, x3 csel x3, x3, x4, eq csel w0, w0, w8, eq tst x7, #4 lsr x4, x3, #32 crc32\c\()w w8, w0, w3 csel x3, x3, x4, eq csel w0, w0, w8, eq tst x7, #2 lsr w4, w3, #16 crc32\c\()h w8, w0, w3 csel w3, w3, w4, eq csel w0, w0, w8, eq tst x7, #1 crc32\c\()b w8, w0, w3 csel w0, w0, w8, eq tst x7, #16 crc32\c\()x w8, w0, x5 crc32\c\()x w8, w8, x6 csel w0, w0, w8, eq cbz x2, 0f 32: ldp x3, x4, [x1], #32 sub x2, x2, #32 ldp x5, x6, [x1, #-16] byteorder x3, \be byteorder x4, \be byteorder x5, \be byteorder x6, \be bitorder x3, \be bitorder x4, \be bitorder x5, \be bitorder x6, \be crc32\c\()x w0, w0, x3 crc32\c\()x w0, w0, x4 crc32\c\()x w0, w0, x5 crc32\c\()x w0, w0, x6 cbnz x2, 32b 0: bitorder w0, \be ret 8: tbz x2, #3, 4f ldr x3, [x1], #8 byteorder x3, \be bitorder x3, \be crc32\c\()x w0, w0, x3 4: tbz x2, #2, 2f ldr w3, [x1], #4 byteorder w3, \be bitorder w3, \be crc32\c\()w w0, w0, w3 2: tbz x2, #1, 1f ldrh w3, [x1], #2 byteorder16 w3, \be bitorder16 w3, \be crc32\c\()h w0, w0, w3 1: tbz x2, #0, 0f ldrb w3, [x1] bitorder8 w3, \be crc32\c\()b w0, w0, w3 0: bitorder w0, \be ret .endm .align 5 SYM_FUNC_START(crc32_le) alternative_if_not ARM64_HAS_CRC32 b crc32_le_base alternative_else_nop_endif __crc32 SYM_FUNC_END(crc32_le) .align 5 SYM_FUNC_START(__crc32c_le) alternative_if_not ARM64_HAS_CRC32 b __crc32c_le_base alternative_else_nop_endif __crc32 c SYM_FUNC_END(__crc32c_le) .align 5 SYM_FUNC_START(crc32_be) alternative_if_not ARM64_HAS_CRC32 b crc32_be_base alternative_else_nop_endif __crc32 be=1 SYM_FUNC_END(crc32_be)