/* SPDX-License-Identifier: GPL-2.0-only */
/*
* Accelerated GHASH implementation with ARMv8 PMULL instructions.
*
* Copyright (C) 2014 - 2018 Linaro Ltd. <ard.biesheuvel@linaro.org>
*/
#include <linux/linkage.h>
#include <linux/cfi_types.h>
#include <asm/assembler.h>
SHASH .req v0
SHASH2 .req v1
T1 .req v2
T2 .req v3
MASK .req v4
XM .req v5
XL .req v6
XH .req v7
IN1 .req v7
k00_16 .req v8
k32_48 .req v9
t3 .req v10
t4 .req v11
t5 .req v12
t6 .req v13
t7 .req v14
t8 .req v15
t9 .req v16
perm1 .req v17
perm2 .req v18
perm3 .req v19
sh1 .req v20
sh2 .req v21
sh3 .req v22
sh4 .req v23
ss1 .req v24
ss2 .req v25
ss3 .req v26
ss4 .req v27
XL2 .req v8
XM2 .req v9
XH2 .req v10
XL3 .req v11
XM3 .req v12
XH3 .req v13
TT3 .req v14
TT4 .req v15
HH .req v16
HH3 .req v17
HH4 .req v18
HH34 .req v19
.text
.arch armv8-a+crypto
.macro __pmull_p64, rd, rn, rm
pmull \rd\().1q, \rn\().1d, \rm\().1d
.endm
.macro __pmull2_p64, rd, rn, rm
pmull2 \rd\().1q, \rn\().2d, \rm\().2d
.endm
.macro __pmull_p8, rq, ad, bd
ext t3.8b, \ad\().8b, \ad\().8b, #1 // A1
ext t5.8b, \ad\().8b, \ad\().8b, #2 // A2
ext t7.8b, \ad\().8b, \ad\().8b, #3 // A3
__pmull_p8_\bd \rq, \ad
.endm
.macro __pmull2_p8, rq, ad, bd
tbl t3.16b, {\ad\().16b}, perm1.16b // A1
tbl t5.16b, {\ad\().16b}, perm2.16b // A2
tbl t7.16b, {\ad\().16b}, perm3.16b // A3
__pmull2_p8_\bd \rq, \ad
.endm
.macro __pmull_p8_SHASH, rq, ad
__pmull_p8_tail \rq, \ad\().8b, SHASH.8b, 8b,, sh1, sh2, sh3, sh4
.endm
.macro __pmull_p8_SHASH2, rq, ad
__pmull_p8_tail \rq, \ad\().8b, SHASH2.8b, 8b,, ss1, ss2, ss3, ss4
.endm
.macro __pmull2_p8_SHASH, rq, ad
__pmull_p8_tail \rq, \ad\().16b, SHASH.16b, 16b, 2, sh1, sh2, sh3, sh4
.endm
.macro __pmull_p8_tail, rq, ad, bd, nb, t, b1, b2, b3, b4
pmull\t t3.8h, t3.\nb, \bd // F = A1*B
pmull\t t4.8h, \ad, \b1\().\nb // E = A*B1
pmull\t t5.8h, t5.\nb, \bd // H = A2*B
pmull\t t6.8h, \ad, \b2\().\nb // G = A*B2
pmull\t t7.8h, t7.\nb, \bd // J = A3*B
pmull\t t8.8h, \ad, \b3\().\nb // I = A*B3
pmull\t t9.8h, \ad, \b4\().\nb // K = A*B4
pmull\t \rq\().8h, \ad, \bd // D = A*B
eor t3.16b, t3.16b, t4.16b // L = E + F
eor t5.16b, t5.16b, t6.16b // M = G + H
eor t7.16b, t7.16b, t8.16b // N = I + J
uzp1 t4.2d, t3.2d, t5.2d
uzp2 t3.2d, t3.2d, t5.2d
uzp1 t6.2d, t7.2d, t9.2d
uzp2 t7.2d, t7.2d, t9.2d
// t3 = (L) (P0 + P1) << 8
// t5 = (M) (P2 + P3) << 16
eor t4.16b, t4.16b, t3.16b
and t3.16b, t3.16b, k32_48.16b
// t7 = (N) (P4 + P5) << 24
// t9 = (K) (P6 + P7) << 32
eor t6.16b, t6.16b, t7.16b
and t7.16b, t7.16b, k00_16.16b
eor t4.16b, t4.16b, t3.16b
eor t6.16b, t6.16b, t7.16b
zip2 t5.2d, t4.2d, t3.2d
zip1 t3.2d, t4.2d, t3.2d
zip2 t9.2d, t6.2d, t7.2d
zip1 t7.2d, t6.2d, t7.2d
ext t3.16b, t3.16b, t3.16b, #15
ext t5.16b, t5.16b, t5.16b, #14
ext t7.16b, t7.16b, t7.16b, #13
ext t9.16b, t9.16b, t9.16b, #12
eor t3.16b, t3.16b, t5.16b
eor t7.16b, t7.16b, t9.16b
eor \rq\().16b, \rq\().16b, t3.16b
eor \rq\().16b, \rq\().16b, t7.16b
.endm
.macro __pmull_pre_p64
add x8, x3, #16
ld1 {HH.2d-HH4.2d}, [x8]
trn1 SHASH2.2d, SHASH.2d, HH.2d
trn2 T1.2d, SHASH.2d, HH.2d
eor SHASH2.16b, SHASH2.16b, T1.16b
trn1 HH34.2d, HH3.2d, HH4.2d
trn2 T1.2d, HH3.2d, HH4.2d
eor HH34.16b, HH34.16b, T1.16b
movi MASK.16b, #0xe1
shl MASK.2d, MASK.2d, #57
.endm
.macro __pmull_pre_p8
ext SHASH2.16b, SHASH.16b, SHASH.16b, #8
eor SHASH2.16b, SHASH2.16b, SHASH.16b
// k00_16 := 0x0000000000000000_000000000000ffff
// k32_48 := 0x00000000ffffffff_0000ffffffffffff
movi k32_48.2d, #0xffffffff
mov k32_48.h[2], k32_48.h[0]
ushr k00_16.2d, k32_48.2d, #32
// prepare the permutation vectors
mov_q x5, 0x080f0e0d0c0b0a09
movi T1.8b, #8
dup perm1.2d, x5
eor perm1.16b, perm1.16b, T1.16b
ushr perm2.2d, perm1.2d, #8
ushr perm3.2d, perm1.2d, #16
ushr T1.2d, perm1.2d, #24
sli perm2.2d, perm1.2d, #56
sli perm3.2d, perm1.2d, #48
sli T1.2d, perm1.2d, #40
// precompute loop invariants
tbl sh1.16b, {SHASH.16b}, perm1.16b
tbl sh2.16b, {SHASH.16b}, perm2.16b
tbl sh3.16b, {SHASH.16b}, perm3.16b
tbl sh4.16b, {SHASH.16b}, T1.16b
ext ss1.8b, SHASH2.8b, SHASH2.8b, #1
ext ss2.8b, SHASH2.8b, SHASH2.8b, #2
ext ss3.8b, SHASH2.8b, SHASH2.8b, #3
ext ss4.8b, SHASH2.8b, SHASH2.8b, #4
.endm
//
// PMULL (64x64->128) based reduction for CPUs that can do
// it in a single instruction.
//
.macro __pmull_reduce_p64
pmull T2.1q, XL.1d, MASK.1d
eor XM.16b, XM.16b, T1.16b
mov XH.d[0], XM.d[1]
mov XM.d[1], XL.d[0]
eor XL.16b, XM.16b, T2.16b
ext T2.16b, XL.16b, XL.16b, #8
pmull XL.1q, XL.1d, MASK.1d
.endm
//
// Alternative reduction for CPUs that lack support for the
// 64x64->128 PMULL instruction
//
.macro __pmull_reduce_p8
eor XM.16b, XM.16b, T1.16b
mov XL.d[1], XM.d[0]
mov XH.d[0], XM.d[1]
shl T1.2d, XL.2d, #57
shl T2.2d, XL.2d, #62
eor T2.16b, T2.16b, T1.16b
shl T1.2d, XL.2d, #63
eor T2.16b, T2.16b, T1.16b
ext T1.16b, XL.16b, XH.16b, #8
eor T2.16b, T2.16b, T1.16b
mov XL.d[1], T2.d[0]
mov XH.d[0], T2.d[1]
ushr T2.2d, XL.2d, #1
eor XH.16b, XH.16b, XL.16b
eor XL.16b, XL.16b, T2.16b
ushr T2.2d, T2.2d, #6
ushr XL.2d, XL.2d, #1
.endm
.macro __pmull_ghash, pn
ld1 {SHASH.2d}, [x3]
ld1 {XL.2d}, [x1]
__pmull_pre_\pn
/* do the head block first, if supplied */
cbz x4, 0f
ld1 {T1.2d}, [x4]
mov x4, xzr
b 3f
0: .ifc \pn, p64
tbnz w0, #0, 2f // skip until #blocks is a
tbnz w0, #1, 2f // round multiple of 4
1: ld1 {XM3.16b-TT4.16b}, [x2], #64
sub w0, w0, #4
rev64 T1.16b, XM3.16b
rev64 T2.16b, XH3.16b
rev64 TT4.16b, TT4.16b
rev64 TT3.16b, TT3.16b
ext IN1.16b, TT4.16b, TT4.16b, #8
ext XL3.16b, TT3.16b, TT3.16b, #8
eor TT4.16b, TT4.16b, IN1.16b
pmull2 XH2.1q, SHASH.2d, IN1.2d // a1 * b1
pmull XL2.1q, SHASH.1d, IN1.1d // a0 * b0
pmull XM2.1q, SHASH2.1d, TT4.1d // (a1 + a0)(b1 + b0)
eor TT3.16b, TT3.16b, XL3.16b
pmull2 XH3.1q, HH.2d, XL3.2d // a1 * b1
pmull XL3.1q, HH.1d, XL3.1d // a0 * b0
pmull2 XM3.1q, SHASH2.2d, TT3.2d // (a1 + a0)(b1 + b0)
ext IN1.16b, T2.16b, T2.16b, #8
eor XL2.16b, XL2.16b, XL3.16b
eor XH2.16b, XH2.16b, XH3.16b
eor XM2.16b, XM2.16b, XM3.16b
eor T2.16b, T2.16b, IN1.16b
pmull2 XH3.1q, HH3.2d, IN1.2d // a1 * b1
pmull XL3.1q, HH3.1d, IN1.1d // a0 * b0
pmull XM3.1q, HH34.1d, T2.1d // (a1 + a0)(b1 + b0)
eor XL2.16b, XL2.16b, XL3.16b
eor XH2.16b, XH2.16b, XH3.16b
eor XM2.16b, XM2.16b, XM3.16b
ext IN1.16b, T1.16b, T1.16b, #8
ext TT3.16b, XL.16b, XL.16b, #8
eor XL.16b, XL.16b, IN1.16b
eor T1.16b, T1.16b, TT3.16b
pmull2 XH.1q, HH4.2d, XL.2d // a1 * b1
eor T1.16b, T1.16b, XL.16b
pmull XL.1q, HH4.1d, XL.1d // a0 * b0
pmull2 XM.1q, HH34.2d, T1.2d // (a1 + a0)(b1 + b0)
eor XL.16b, XL.16b, XL2.16b
eor XH.16b, XH.16b, XH2.16b
eor XM.16b, XM.16b, XM2.16b
eor T2.16b, XL.16b, XH.16b
ext T1.16b, XL.16b, XH.16b, #8
eor XM.16b, XM.16b, T2.16b
__pmull_reduce_p64
eor T2.16b, T2.16b, XH.16b
eor XL.16b, XL.16b, T2.16b
cbz w0, 5f
b 1b
.endif
2: ld1 {T1.2d}, [x2], #16
sub w0, w0, #1
3: /* multiply XL by SHASH in GF(2^128) */
CPU_LE( rev64 T1.16b, T1.16b )
ext T2.16b, XL.16b, XL.16b, #8
ext IN1.16b, T1.16b, T1.16b, #8
eor T1.16b, T1.16b, T2.16b
eor XL.16b, XL.16b, IN1.16b
__pmull2_\pn XH, XL, SHASH // a1 * b1
eor T1.16b, T1.16b, XL.16b
__pmull_\pn XL, XL, SHASH // a0 * b0
__pmull_\pn XM, T1, SHASH2 // (a1 + a0)(b1 + b0)
4: eor T2.16b, XL.16b, XH.16b
ext T1.16b, XL.16b, XH.16b, #8
eor XM.16b, XM.16b, T2.16b
__pmull_reduce_\pn
eor T2.16b, T2.16b, XH.16b
eor XL.16b, XL.16b, T2.16b
cbnz w0, 0b
5: st1 {XL.2d}, [x1]
ret
.endm
/*
* void pmull_ghash_update(int blocks, u64 dg[], const char *src,
* struct ghash_key const *k, const char *head)
*/
SYM_TYPED_FUNC_START(pmull_ghash_update_p64)
__pmull_ghash p64
SYM_FUNC_END(pmull_ghash_update_p64)
SYM_TYPED_FUNC_START(pmull_ghash_update_p8)
__pmull_ghash p8
SYM_FUNC_END(pmull_ghash_update_p8)
KS0 .req v8
KS1 .req v9
KS2 .req v10
KS3 .req v11
INP0 .req v21
INP1 .req v22
INP2 .req v23
INP3 .req v24
K0 .req v25
K1 .req v26
K2 .req v27
K3 .req v28
K4 .req v12
K5 .req v13
K6 .req v4
K7 .req v5
K8 .req v14
K9 .req v15
KK .req v29
KL .req v30
KM .req v31
.macro load_round_keys, rounds, rk, tmp
add \tmp, \rk, #64
ld1 {K0.4s-K3.4s}, [\rk]
ld1 {K4.4s-K5.4s}, [\tmp]
add \tmp, \rk, \rounds, lsl #4
sub \tmp, \tmp, #32
ld1 {KK.4s-KM.4s}, [\tmp]
.endm
.macro enc_round, state, key
aese \state\().16b, \key\().16b
aesmc \state\().16b, \state\().16b
.endm
.macro enc_qround, s0, s1, s2, s3, key
enc_round \s0, \key
enc_round \s1, \key
enc_round \s2, \key
enc_round \s3, \key
.endm
.macro enc_block, state, rounds, rk, tmp
add \tmp, \rk, #96
ld1 {K6.4s-K7.4s}, [\tmp], #32
.irp key, K0, K1, K2, K3, K4 K5
enc_round \state, \key
.endr
tbnz \rounds, #2, .Lnot128_\@
.Lout256_\@:
enc_round \state, K6
enc_round \state, K7
.Lout192_\@:
enc_round \state, KK
aese \state\().16b, KL.16b
eor \state\().16b, \state\().16b, KM.16b
.subsection 1
.Lnot128_\@:
ld1 {K8.4s-K9.4s}, [\tmp], #32
enc_round \state, K6
enc_round \state, K7
ld1 {K6.4s-K7.4s}, [\tmp]
enc_round \state, K8
enc_round \state, K9
tbz \rounds, #1, .Lout192_\@
b .Lout256_\@
.previous
.endm
.align 6
.macro pmull_gcm_do_crypt, enc
frame_push 1
load_round_keys x7, x6, x8
ld1 {SHASH.2d}, [x3], #16
ld1 {HH.2d-HH4.2d}, [x3]
trn1 SHASH2.2d, SHASH.2d, HH.2d
trn2 T1.2d, SHASH.2d, HH.2d
eor SHASH2.16b, SHASH2.16b, T1.16b
trn1 HH34.2d, HH3.2d, HH4.2d
trn2 T1.2d, HH3.2d, HH4.2d
eor HH34.16b, HH34.16b, T1.16b
ld1 {XL.2d}, [x4]
cbz x0, 3f // tag only?
ldr w8, [x5, #12] // load lower counter
CPU_LE( rev w8, w8 )
0: mov w9, #4 // max blocks per round
add x10, x0, #0xf
lsr x10, x10, #4 // remaining blocks
subs x0, x0, #64
csel w9, w10, w9, mi
add w8, w8, w9
bmi 1f
ld1 {INP0.16b-INP3.16b}, [x2], #64
.subsection 1
/*
* Populate the four input registers right to left with up to 63 bytes
* of data, using overlapping loads to avoid branches.
*
* INP0 INP1 INP2 INP3
* 1 byte | | | |x |
* 16 bytes | | | |xxxxxxxx|
* 17 bytes | | |xxxxxxxx|x |
* 47 bytes | |xxxxxxxx|xxxxxxxx|xxxxxxx |
* etc etc
*
* Note that this code may read up to 15 bytes before the start of
* the input. It is up to the calling code to ensure this is safe if
* this happens in the first iteration of the loop (i.e., when the
* input size is < 16 bytes)
*/
1: mov x15, #16
ands x19, x0, #0xf
csel x19, x19, x15, ne
adr_l x17, .Lpermute_table + 16
sub x11, x15, x19
add x12, x17, x11
sub x17, x17, x11
ld1 {T1.16b}, [x12]
sub x10, x1, x11
sub x11, x2, x11
cmp x0, #-16
csel x14, x15, xzr, gt
cmp x0, #-32
csel x15, x15, xzr, gt
cmp x0, #-48
csel x16, x19, xzr, gt
csel x1, x1, x10, gt
csel x2, x2, x11, gt
ld1 {INP0.16b}, [x2], x14
ld1 {INP1.16b}, [x2], x15
ld1 {INP2.16b}, [x2], x16
ld1 {INP3.16b}, [x2]
tbl INP3.16b, {INP3.16b}, T1.16b
b 2f
.previous
2: .if \enc == 0
bl pmull_gcm_ghash_4x
.endif
bl pmull_gcm_enc_4x
tbnz x0, #63, 6f
st1 {INP0.16b-INP3.16b}, [x1], #64
.if \enc == 1
bl pmull_gcm_ghash_4x
.endif
bne 0b
3: ldr x10, [sp, #.Lframe_local_offset]
cbz x10, 5f // output tag?
ld1 {INP3.16b}, [x10] // load lengths[]
mov w9, #1
bl pmull_gcm_ghash_4x
mov w11, #(0x1 << 24) // BE '1U'
ld1 {KS0.16b}, [x5]
mov KS0.s[3], w11
enc_block KS0, x7, x6, x12
ext XL.16b, XL.16b, XL.16b, #8
rev64 XL.16b, XL.16b
eor XL.16b, XL.16b, KS0.16b
.if \enc == 1
st1 {XL.16b}, [x10] // store tag
.else
ldp x11, x12, [sp, #40] // load tag pointer and authsize
adr_l x17, .Lpermute_table
ld1 {KS0.16b}, [x11] // load supplied tag
add x17, x17, x12
ld1 {KS1.16b}, [x17] // load permute vector
cmeq XL.16b, XL.16b, KS0.16b // compare tags
mvn XL.16b, XL.16b // -1 for fail, 0 for pass
tbl XL.16b, {XL.16b}, KS1.16b // keep authsize bytes only
sminv b0, XL.16b // signed minimum across XL
smov w0, v0.b[0] // return b0
.endif
4: frame_pop
ret
5:
CPU_LE( rev w8, w8 )
str w8, [x5, #12] // store lower counter
st1 {XL.2d}, [x4]
b 4b
6: ld1 {T1.16b-T2.16b}, [x17], #32 // permute vectors
sub x17, x17, x19, lsl #1
cmp w9, #1
beq 7f
.subsection 1
7: ld1 {INP2.16b}, [x1]
tbx INP2.16b, {INP3.16b}, T1.16b
mov INP3.16b, INP2.16b
b 8f
.previous
st1 {INP0.16b}, [x1], x14
st1 {INP1.16b}, [x1], x15
st1 {INP2.16b}, [x1], x16
tbl INP3.16b, {INP3.16b}, T1.16b
tbx INP3.16b, {INP2.16b}, T2.16b
8: st1 {INP3.16b}, [x1]
.if \enc == 1
ld1 {T1.16b}, [x17]
tbl INP3.16b, {INP3.16b}, T1.16b // clear non-data bits
bl pmull_gcm_ghash_4x
.endif
b 3b
.endm
/*
* void pmull_gcm_encrypt(int blocks, u8 dst[], const u8 src[],
* struct ghash_key const *k, u64 dg[], u8 ctr[],
* int rounds, u8 tag)
*/
SYM_FUNC_START(pmull_gcm_encrypt)
pmull_gcm_do_crypt 1
SYM_FUNC_END(pmull_gcm_encrypt)
/*
* void pmull_gcm_decrypt(int blocks, u8 dst[], const u8 src[],
* struct ghash_key const *k, u64 dg[], u8 ctr[],
* int rounds, u8 tag)
*/
SYM_FUNC_START(pmull_gcm_decrypt)
pmull_gcm_do_crypt 0
SYM_FUNC_END(pmull_gcm_decrypt)
SYM_FUNC_START_LOCAL(pmull_gcm_ghash_4x)
movi MASK.16b, #0xe1
shl MASK.2d, MASK.2d, #57
rev64 T1.16b, INP0.16b
rev64 T2.16b, INP1.16b
rev64 TT3.16b, INP2.16b
rev64 TT4.16b, INP3.16b
ext XL.16b, XL.16b, XL.16b, #8
tbz w9, #2, 0f // <4 blocks?
.subsection 1
0: movi XH2.16b, #0
movi XM2.16b, #0
movi XL2.16b, #0
tbz w9, #0, 1f // 2 blocks?
tbz w9, #1, 2f // 1 block?
eor T2.16b, T2.16b, XL.16b
ext T1.16b, T2.16b, T2.16b, #8
b .Lgh3
1: eor TT3.16b, TT3.16b, XL.16b
ext T2.16b, TT3.16b, TT3.16b, #8
b .Lgh2
2: eor TT4.16b, TT4.16b, XL.16b
ext IN1.16b, TT4.16b, TT4.16b, #8
b .Lgh1
.previous
eor T1.16b, T1.16b, XL.16b
ext IN1.16b, T1.16b, T1.16b, #8
pmull2 XH2.1q, HH4.2d, IN1.2d // a1 * b1
eor T1.16b, T1.16b, IN1.16b
pmull XL2.1q, HH4.1d, IN1.1d // a0 * b0
pmull2 XM2.1q, HH34.2d, T1.2d // (a1 + a0)(b1 + b0)
ext T1.16b, T2.16b, T2.16b, #8
.Lgh3: eor T2.16b, T2.16b, T1.16b
pmull2 XH.1q, HH3.2d, T1.2d // a1 * b1
pmull XL.1q, HH3.1d, T1.1d // a0 * b0
pmull XM.1q, HH34.1d, T2.1d // (a1 + a0)(b1 + b0)
eor XH2.16b, XH2.16b, XH.16b
eor XL2.16b, XL2.16b, XL.16b
eor XM2.16b, XM2.16b, XM.16b
ext T2.16b, TT3.16b, TT3.16b, #8
.Lgh2: eor TT3.16b, TT3.16b, T2.16b
pmull2 XH.1q, HH.2d, T2.2d // a1 * b1
pmull XL.1q, HH.1d, T2.1d // a0 * b0
pmull2 XM.1q, SHASH2.2d, TT3.2d // (a1 + a0)(b1 + b0)
eor XH2.16b, XH2.16b, XH.16b
eor XL2.16b, XL2.16b, XL.16b
eor XM2.16b, XM2.16b, XM.16b
ext IN1.16b, TT4.16b, TT4.16b, #8
.Lgh1: eor TT4.16b, TT4.16b, IN1.16b
pmull XL.1q, SHASH.1d, IN1.1d // a0 * b0
pmull2 XH.1q, SHASH.2d, IN1.2d // a1 * b1
pmull XM.1q, SHASH2.1d, TT4.1d // (a1 + a0)(b1 + b0)
eor XH.16b, XH.16b, XH2.16b
eor XL.16b, XL.16b, XL2.16b
eor XM.16b, XM.16b, XM2.16b
eor T2.16b, XL.16b, XH.16b
ext T1.16b, XL.16b, XH.16b, #8
eor XM.16b, XM.16b, T2.16b
__pmull_reduce_p64
eor T2.16b, T2.16b, XH.16b
eor XL.16b, XL.16b, T2.16b
ret
SYM_FUNC_END(pmull_gcm_ghash_4x)
SYM_FUNC_START_LOCAL(pmull_gcm_enc_4x)
ld1 {KS0.16b}, [x5] // load upper counter
sub w10, w8, #4
sub w11, w8, #3
sub w12, w8, #2
sub w13, w8, #1
rev w10, w10
rev w11, w11
rev w12, w12
rev w13, w13
mov KS1.16b, KS0.16b
mov KS2.16b, KS0.16b
mov KS3.16b, KS0.16b
ins KS0.s[3], w10 // set lower counter
ins KS1.s[3], w11
ins KS2.s[3], w12
ins KS3.s[3], w13
add x10, x6, #96 // round key pointer
ld1 {K6.4s-K7.4s}, [x10], #32
.irp key, K0, K1, K2, K3, K4, K5
enc_qround KS0, KS1, KS2, KS3, \key
.endr
tbnz x7, #2, .Lnot128
.subsection 1
.Lnot128:
ld1 {K8.4s-K9.4s}, [x10], #32
.irp key, K6, K7
enc_qround KS0, KS1, KS2, KS3, \key
.endr
ld1 {K6.4s-K7.4s}, [x10]
.irp key, K8, K9
enc_qround KS0, KS1, KS2, KS3, \key
.endr
tbz x7, #1, .Lout192
b .Lout256
.previous
.Lout256:
.irp key, K6, K7
enc_qround KS0, KS1, KS2, KS3, \key
.endr
.Lout192:
enc_qround KS0, KS1, KS2, KS3, KK
aese KS0.16b, KL.16b
aese KS1.16b, KL.16b
aese KS2.16b, KL.16b
aese KS3.16b, KL.16b
eor KS0.16b, KS0.16b, KM.16b
eor KS1.16b, KS1.16b, KM.16b
eor KS2.16b, KS2.16b, KM.16b
eor KS3.16b, KS3.16b, KM.16b
eor INP0.16b, INP0.16b, KS0.16b
eor INP1.16b, INP1.16b, KS1.16b
eor INP2.16b, INP2.16b, KS2.16b
eor INP3.16b, INP3.16b, KS3.16b
ret
SYM_FUNC_END(pmull_gcm_enc_4x)
.section ".rodata", "a"
.align 6
.Lpermute_table:
.byte 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff
.byte 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff
.byte 0x0, 0x1, 0x2, 0x3, 0x4, 0x5, 0x6, 0x7
.byte 0x8, 0x9, 0xa, 0xb, 0xc, 0xd, 0xe, 0xf
.byte 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff
.byte 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff
.byte 0x0, 0x1, 0x2, 0x3, 0x4, 0x5, 0x6, 0x7
.byte 0x8, 0x9, 0xa, 0xb, 0xc, 0xd, 0xe, 0xf
.previous