#include <linux/linkage.h>
#ifdef __x86_64__
#define RODATA(label) label(%rip)
#define ARG0 %rdi
#define ARG1 %rsi
#define ARG2 %rdx
#define ARG2_32 %edx
#define ARG3 %rcx
#define ARG4 %r8
#define TMP %rax
#define TMP_32 %eax
#define TMP_16 %ax
#define TMP_8 %al
#else // __x86_64__
#define RODATA(label) label
#define ARG0 %eax
#define ARG1 %edx
#define ARG2 %ecx
#define ARG2_32 %ecx
#define ARG3 %edi
#define ARG4 %esi
#define TMP %ebx
#define TMP_32 %ebx
#define TMP_16 %bx
#define TMP_8 %bl
#endif // !__x86_64__
#define OFFSETOF_NROUNDS 4
#define OFFSETOF_ROUNDKEYS 16
#define OFFSETOF_INVROUNDKEYS 256
.section .rodata
.p2align 4
.Lxts_gf_poly:
.quad 0x87, 1
#ifdef __x86_64__
.Lbswap_mask:
.byte 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0
#endif
.Lexpandkey_mask:
.byte 13, 14, 15, 12, 13, 14, 15, 12, 13, 14, 15, 12, 13, 14, 15, 12
.Lrcon:
.long 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80, 0x1b, 0x36
.Lcts_permute_table:
.byte 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80
.byte 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80
.byte 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07
.byte 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f
.byte 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80
.byte 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80
.text
.macro _prologue uses_arg3=0, uses_arg4=0
#ifdef __i386__
.set ARG3_OFFSET, 4
.set ARG3_PUSHED, \uses_arg3
.set ARG4_PUSHED, \uses_arg4
.if \uses_arg3
push ARG3
.set ARG3_OFFSET, ARG3_OFFSET + 4
.if \uses_arg3 == 2
mov ARG3_OFFSET(%esp), ARG3
.endif
.endif
.if \uses_arg4
push ARG4
.set ARG3_OFFSET, ARG3_OFFSET + 4
.if \uses_arg4 == 2
mov ARG3_OFFSET+4(%esp), ARG4
.endif
.endif
push TMP
.set ARG3_OFFSET, ARG3_OFFSET + 4
#endif // __i386__
.endm
.macro _reload_arg3
#ifdef __i386__
mov ARG3_OFFSET(%esp), ARG3
#endif
.endm
.macro _epilogue
#ifdef __i386__
pop TMP
.if ARG4_PUSHED
pop ARG4
.endif
.if ARG3_PUSHED
pop ARG3
.endif
#endif
RET
.endm
.macro _aesenc enc, rndkey, data
.if \enc
aesenc \rndkey, \data
.else
aesdec \rndkey, \data
.endif
.endm
.macro _aesenclast enc, rndkeylast, data
.if \enc
aesenclast \rndkeylast, \data
.else
aesdeclast \rndkeylast, \data
.endif
.endm
.macro _prefix_sum a, tmp
movdqa \a, \tmp
pslldq $4, \a
pxor \tmp, \a
movdqa \a, \tmp
pslldq $8, \a
pxor \tmp, \a
.endm
.macro _gen_round_key a, b
movdqa \b, %xmm2
pshufb MASK, %xmm2
aesenclast RCON, %xmm2
_prefix_sum \a, tmp=%xmm3
pxor %xmm2, \a
movdqu \a, (RNDKEYS)
.endm
.macro _aes_expandkey_aesni is_aes128
.set RNDKEYS, ARG0
.set INV_RNDKEYS, ARG1
.set IN_KEY, ARG2
.set RCON_PTR, ARG3
.set COUNTER, TMP_32
.set RCON, %xmm6
.set MASK, %xmm7
_prologue uses_arg3=1
.if \is_aes128
movdqu (IN_KEY), %xmm0
movdqu %xmm0, (RNDKEYS)
.else
movdqu (IN_KEY), %xmm0
movdqu %xmm0, (RNDKEYS)
movdqu 16(IN_KEY), %xmm1
movdqu %xmm1, 16(RNDKEYS)
add $32, RNDKEYS
.endif
movdqa RODATA(.Lexpandkey_mask), MASK
.if \is_aes128
lea RODATA(.Lrcon), RCON_PTR
mov $10, COUNTER
.Lgen_next_aes128_round_key:
add $16, RNDKEYS
movd (RCON_PTR), RCON
pshufd $0x00, RCON, RCON
add $4, RCON_PTR
_gen_round_key %xmm0, %xmm0
dec COUNTER
jnz .Lgen_next_aes128_round_key
.else
pshufd $0x00, RODATA(.Lrcon), RCON
pxor %xmm5, %xmm5
mov $7, COUNTER
.Lgen_next_aes256_round_key_pair:
_gen_round_key %xmm0, %xmm1
dec COUNTER
jz .Lgen_aes256_round_keys_done
pshufd $0xff, %xmm0, %xmm2
aesenclast %xmm5, %xmm2
_prefix_sum %xmm1, tmp=%xmm3
pxor %xmm2, %xmm1
movdqu %xmm1, 16(RNDKEYS)
add $32, RNDKEYS
paddd RCON, RCON
jmp .Lgen_next_aes256_round_key_pair
.Lgen_aes256_round_keys_done:
.endif
test INV_RNDKEYS, INV_RNDKEYS
jz .Ldone\@
movdqu (RNDKEYS), %xmm0
movdqu %xmm0, (INV_RNDKEYS)
.if \is_aes128
mov $9, COUNTER
.else
mov $13, COUNTER
.endif
.Lgen_next_inv_round_key\@:
sub $16, RNDKEYS
add $16, INV_RNDKEYS
movdqu (RNDKEYS), %xmm0
aesimc %xmm0, %xmm0
movdqu %xmm0, (INV_RNDKEYS)
dec COUNTER
jnz .Lgen_next_inv_round_key\@
movdqu -16(RNDKEYS), %xmm0
movdqu %xmm0, 16(INV_RNDKEYS)
.Ldone\@:
_epilogue
.endm
SYM_FUNC_START(aes128_expandkey_aesni)
_aes_expandkey_aesni 1
SYM_FUNC_END(aes128_expandkey_aesni)
SYM_FUNC_START(aes256_expandkey_aesni)
_aes_expandkey_aesni 0
SYM_FUNC_END(aes256_expandkey_aesni)
.macro _do_aes enc, vecs:vararg
movl OFFSETOF_NROUNDS(KEY), NROUNDS
dec NROUNDS
.if \enc
.set rndkey0_offs, OFFSETOF_ROUNDKEYS
.else
.set rndkey0_offs, OFFSETOF_INVROUNDKEYS
.endif
movdqu rndkey0_offs(KEY), RNDKEY
.irp i, \vecs
pxor RNDKEY, AESDATA\i
.endr
lea rndkey0_offs+16(KEY), RNDKEY_PTR
.Lnext_round\@:
movdqu (RNDKEY_PTR), RNDKEY
add $16, RNDKEY_PTR
.irp i, \vecs
_aesenc \enc, RNDKEY, AESDATA\i
.endr
dec NROUNDS
jnz .Lnext_round\@
movdqu (RNDKEY_PTR), RNDKEY
.irp i, \vecs
_aesenclast \enc, RNDKEY, AESDATA\i
.endr
.endm
.macro _do_aes_ecb enc, vecs:vararg
.irp i, \vecs
movdqu \i*16(SRC), AESDATA\i
.endr
_do_aes \enc, \vecs
.irp i, \vecs
movdqu AESDATA\i, \i*16(DST)
.endr
.endm
.macro _aes_crypt_aesni enc
.set DST, ARG0
.set SRC, ARG1
.set KEY, ARG2
.set RNDKEY_PTR, ARG3
.set NROUNDS, TMP_32
.set AESDATA0, %xmm0
.set RNDKEY, %xmm1
_prologue uses_arg3=1
_do_aes_ecb \enc, 0
_epilogue
.endm
SYM_FUNC_START(aes_encrypt_aesni)
_aes_crypt_aesni 1
SYM_FUNC_END(aes_encrypt_aesni)
SYM_FUNC_START(aes_decrypt_aesni)
_aes_crypt_aesni 0
SYM_FUNC_END(aes_decrypt_aesni)
.macro _ecb_crypt enc
.set DST, ARG0
.set SRC, ARG1
.set NBLOCKS, ARG2
.set NBLOCKS32, ARG2_32
.set KEY, ARG3
.set RNDKEY_PTR, ARG4
.set NROUNDS, TMP_32
.set AESDATA0, %xmm0
.set AESDATA1, %xmm1
.set AESDATA2, %xmm2
.set AESDATA3, %xmm3
.set RNDKEY, %xmm4
_prologue uses_arg3=2, uses_arg4=1
sub $4, NBLOCKS
jl .Lecb_loop4_done\@
.p2align 5
.Lecb_loop4\@:
_do_aes_ecb \enc, 0,1,2,3
add $64, SRC
add $64, DST
sub $4, NBLOCKS
jge .Lecb_loop4\@
.Lecb_loop4_done\@:
add $4, NBLOCKS32
jz .Lecb_done\@
.Lecb_loop1\@:
_do_aes_ecb \enc, 0
add $16, SRC
add $16, DST
dec NBLOCKS32
jnz .Lecb_loop1\@
.Lecb_done\@:
_epilogue
.endm
SYM_FUNC_START(aes_ecb_encrypt_aesni)
_ecb_crypt 1
SYM_FUNC_END(aes_ecb_encrypt_aesni)
SYM_FUNC_START(aes_ecb_decrypt_aesni)
_ecb_crypt 0
SYM_FUNC_END(aes_ecb_decrypt_aesni)
SYM_FUNC_START(aes_cbc_encrypt_aesni)
.set DST, ARG0
.set SRC, ARG1
.set NBLOCKS, ARG2
.set IV_PTR, ARG3
.set KEY, ARG4
#ifdef __x86_64__
.set RNDKEY_PTR, %r9
#else
.set RNDKEY_PTR, IV_PTR
#endif
.set NROUNDS, TMP_32
.set AESDATA0, %xmm0
.set PTEXT, %xmm1
.set RNDKEY, %xmm2
_prologue uses_arg3=2, uses_arg4=2
movdqu (IV_PTR), AESDATA0
.p2align 5
.Lcbc_enc_loop:
movdqu (SRC), PTEXT
pxor PTEXT, AESDATA0
_do_aes 1, 0
movdqu AESDATA0, (DST)
add $16, SRC
add $16, DST
dec NBLOCKS
jnz .Lcbc_enc_loop
_reload_arg3
movdqu AESDATA0, (IV_PTR)
_epilogue
SYM_FUNC_END(aes_cbc_encrypt_aesni)
SYM_FUNC_START(aes_cbc_decrypt_aesni)
.set DST, ARG0
.set SRC, ARG1
.set NBLOCKS, ARG2
.set NBLOCKS32, ARG2_32
.set IV_PTR, ARG3
.set KEY, ARG4
#ifdef __x86_64__
.set RNDKEY_PTR, %r9
#else
.set RNDKEY_PTR, IV_PTR
#endif
.set NROUNDS, TMP_32
.set AESDATA0, %xmm0
.set AESDATA1, %xmm1
.set AESDATA2, %xmm2
.set AESDATA3, %xmm3
.set RNDKEY, %xmm4
.set IV, %xmm5
.set CTEXT0, %xmm6
.set CTEXT1, %xmm7
#ifdef __x86_64__
.set CTEXT2, %xmm8
.set CTEXT3, %xmm9
#endif
_prologue uses_arg3=2, uses_arg4=2
movdqu (IV_PTR), IV
sub $4, NBLOCKS
jl .Lcbc_dec_loop4_done
.p2align 5
.Lcbc_dec_loop4:
movdqu 0(SRC), AESDATA0
movdqu 16(SRC), AESDATA1
movdqu 32(SRC), AESDATA2
movdqu 48(SRC), AESDATA3
movdqa AESDATA0, CTEXT0
movdqa AESDATA1, CTEXT1
#ifdef __x86_64__
movdqa AESDATA2, CTEXT2
movdqa AESDATA3, CTEXT3
#endif
_do_aes 0, 0,1,2,3
pxor IV, AESDATA0
pxor CTEXT0, AESDATA1
pxor CTEXT1, AESDATA2
#ifdef __x86_64__
pxor CTEXT2, AESDATA3
movdqa CTEXT3, IV
#else
movdqu 32(SRC), CTEXT0
pxor CTEXT0, AESDATA3
movdqu 48(SRC), IV
#endif
movdqu AESDATA0, 0(DST)
movdqu AESDATA1, 16(DST)
movdqu AESDATA2, 32(DST)
movdqu AESDATA3, 48(DST)
add $64, SRC
add $64, DST
sub $4, NBLOCKS
jge .Lcbc_dec_loop4
.Lcbc_dec_loop4_done:
add $4, NBLOCKS32
jz .Lcbc_dec_done
.Lcbc_dec_loop1:
movdqu (SRC), AESDATA0
movdqa AESDATA0, CTEXT0
_do_aes 0, 0
pxor IV, AESDATA0
movdqa CTEXT0, IV
movdqu AESDATA0, (DST)
add $16, SRC
add $16, DST
dec NBLOCKS32
jnz .Lcbc_dec_loop1
.Lcbc_dec_done:
_reload_arg3
movdqu IV, (IV_PTR)
_epilogue
SYM_FUNC_END(aes_cbc_decrypt_aesni)
.macro _aes_cbc_cts_crypt enc
.set DST, ARG0
.set SRC, ARG1
.set PN_LEN, ARG2
.set IV_PTR, ARG3
.set KEY, ARG4
.set RNDKEY_PTR, IV_PTR
.set NROUNDS, TMP_32
.set RSHIFT_MASK, %xmm0
.set LSHIFT_MASK, %xmm1
.set AESDATA0, %xmm2
.set AESDATA1, %xmm3
.set RNDKEY, %xmm4
.set IV, %xmm5
_prologue uses_arg3=2, uses_arg4=2
lea RODATA(.Lcts_permute_table), TMP
movdqu (TMP,PN_LEN), LSHIFT_MASK
sub PN_LEN, TMP
movdqu 32(TMP), RSHIFT_MASK
movdqu (IV_PTR), IV
movdqu (SRC), AESDATA0
.if \enc
pxor IV, AESDATA0
.endif
_do_aes \enc, 0
movdqu (SRC,PN_LEN), AESDATA1
.if \enc
pshufb RSHIFT_MASK, AESDATA1
pxor AESDATA0, AESDATA1
_do_aes 1, 1
pshufb LSHIFT_MASK, AESDATA0
.else
movdqa AESDATA0, %xmm6
pshufb LSHIFT_MASK, AESDATA0
pxor AESDATA1, AESDATA0
pshufb RSHIFT_MASK, AESDATA1
pblendvb %xmm6, AESDATA1
_do_aes 0, 1
pxor IV, AESDATA1
.endif
movdqu AESDATA0, (DST,PN_LEN)
movdqu AESDATA1, (DST)
_epilogue
.endm
SYM_FUNC_START(aes_cbc_cts_encrypt_aesni)
_aes_cbc_cts_crypt 1
SYM_FUNC_END(aes_cbc_cts_encrypt_aesni)
SYM_FUNC_START(aes_cbc_cts_decrypt_aesni)
_aes_cbc_cts_crypt 0
SYM_FUNC_END(aes_cbc_cts_decrypt_aesni)
#ifdef __x86_64__
SYM_FUNC_START(aes_ctr64_crypt_aesni)
.set DST, ARG0
.set SRC, ARG1
.set LEN, ARG2
.set LEN32, ARG2_32
.set LE_CTR_PTR, ARG3
.set KEY, ARG4
.set AESDATA0, %xmm0
.set AESDATA1, %xmm1
.set AESDATA2, %xmm2
.set AESDATA3, %xmm3
.set LE_CTR, %xmm4
.set LE_CTR_INC, %xmm5
.set BSWAP_MASK, %xmm6
.set RNDKEY, %xmm7
.set RNDKEY_PTR, LE_CTR_PTR
.set NROUNDS, TMP_32
movdqu (LE_CTR_PTR), LE_CTR
movdqa RODATA(.Lbswap_mask), BSWAP_MASK
mov $1, TMP_32
movd TMP_32, LE_CTR_INC
sub $64, LEN
jl .Lctr_loop4_done
.p2align 5
.Lctr_loop4:
.irp i, 0,1,2,3
movdqa LE_CTR, AESDATA\i
pshufb BSWAP_MASK, AESDATA\i
paddq LE_CTR_INC, LE_CTR
.endr
_do_aes 1, 0,1,2,3
.irp i, 0,1,2,3
movdqu \i*16(SRC), RNDKEY
pxor RNDKEY, AESDATA\i
movdqu AESDATA\i, \i*16(DST)
.endr
add $64, SRC
add $64, DST
sub $64, LEN
jge .Lctr_loop4
.Lctr_loop4_done:
add $64, LEN32
jz .Lctr_done
.irp i, 0,1,2,3
movdqa LE_CTR, AESDATA\i
pshufb BSWAP_MASK, AESDATA\i
.if \i != 3
paddq LE_CTR_INC, LE_CTR
.endif
.endr
_do_aes 1, 0,1,2,3
sub $16, LEN32
jl .Lctr_xor_partial
.Lctr_xor1:
movdqu (SRC), RNDKEY
pxor RNDKEY, AESDATA0
movdqu AESDATA0, (DST)
movdqa AESDATA1, AESDATA0
movdqa AESDATA2, AESDATA1
movdqa AESDATA3, AESDATA2
add $16, SRC
add $16, DST
sub $16, LEN32
jge .Lctr_xor1
.Lctr_xor_partial:
test $8, LEN32
jz 1f
movq AESDATA0, TMP
xor (SRC), TMP
mov TMP, (DST)
add $8, SRC
add $8, DST
psrldq $8, AESDATA0
1:
test $4, LEN32
jz 2f
movd AESDATA0, TMP_32
xor (SRC), TMP_32
mov TMP_32, (DST)
add $4, SRC
add $4, DST
psrldq $4, AESDATA0
2:
test $2, LEN32
jz 3f
movd AESDATA0, TMP_32
xor (SRC), TMP_16
mov TMP_16, (DST)
add $2, SRC
add $2, DST
psrldq $2, AESDATA0
3:
test $1, LEN32
jz .Lctr_done
movd AESDATA0, TMP_32
xor (SRC), TMP_8
mov TMP_8, (DST)
.Lctr_done:
RET
SYM_FUNC_END(aes_ctr64_crypt_aesni)
#endif // __x86_64__
.macro _next_tweak tweak, tmp
pshufd $0x13, \tweak, \tmp
paddq \tweak, \tweak
psrad $31, \tmp
pand GF_POLY, \tmp
pxor \tmp, \tweak
.endm
.macro _aes_xts_crypt enc
.set DST, ARG0
.set SRC, ARG1
.set NBLOCKS, ARG2
.set NBLOCKS32, ARG2_32
.set TWEAK_PTR, ARG3
.set KEY, ARG4
#ifdef __x86_64__
.set RNDKEY_PTR, %r9
#else
.set RNDKEY_PTR, TWEAK_PTR
#endif
.set NROUNDS, TMP_32
.set AESDATA0, %xmm0
.set AESDATA1, %xmm1
.set AESDATA2, %xmm2
.set AESDATA3, %xmm3
.set GF_POLY, %xmm4
.set RNDKEY, %xmm5
.set TWEAK, %xmm6
.set SAVED_TWEAK0, %xmm7
#ifdef __x86_64__
.set SAVED_TWEAK1, %xmm8
.set SAVED_TWEAK2, %xmm9
#endif
_prologue uses_arg3=2, uses_arg4=2
#ifdef __i386__
push %ebp
mov %esp, %ebp
and $~15, %esp
sub $32, %esp
#endif
movdqu (TWEAK_PTR), TWEAK
movdqa RODATA(.Lxts_gf_poly), GF_POLY
sub $4, NBLOCKS
jl .Lxts_loop4_done\@
.p2align 5
.Lxts_loop4\@:
.irp i, 0,1,2,3
movdqu \i*16(SRC), AESDATA\i
pxor TWEAK, AESDATA\i
.if \i != 3
#ifdef __x86_64__
movdqa TWEAK, SAVED_TWEAK\i
#else
.if \i == 0
movdqa TWEAK, SAVED_TWEAK0
.else
movdqa TWEAK, (\i-1)*16(%esp)
.endif
#endif
_next_tweak TWEAK, RNDKEY
.endif
.endr
_do_aes \enc, 0,1,2,3
pxor SAVED_TWEAK0, AESDATA0
#ifdef __x86_64__
pxor SAVED_TWEAK1, AESDATA1
pxor SAVED_TWEAK2, AESDATA2
#else
pxor (%esp), AESDATA1
pxor 16(%esp), AESDATA2
#endif
pxor TWEAK, AESDATA3
.irp i, 0,1,2,3
movdqu AESDATA\i, \i*16(DST)
.endr
_next_tweak TWEAK, RNDKEY
add $64, SRC
add $64, DST
sub $4, NBLOCKS
jge .Lxts_loop4\@
.Lxts_loop4_done\@:
add $4, NBLOCKS32
jz .Lxts_done\@
.Lxts_loop1\@:
movdqu (SRC), AESDATA0
pxor TWEAK, AESDATA0
_do_aes \enc, 0
pxor TWEAK, AESDATA0
movdqu AESDATA0, (DST)
_next_tweak TWEAK, RNDKEY
add $16, SRC
add $16, DST
dec NBLOCKS32
jnz .Lxts_loop1\@
.Lxts_done\@:
#ifdef __i386__
pxor AESDATA0, AESDATA0
movdqa AESDATA0, (%esp)
movdqa AESDATA0, 16(%esp)
mov %ebp, %esp
pop %ebp
#endif
_reload_arg3
movdqu TWEAK, (TWEAK_PTR)
_epilogue
.endm
SYM_FUNC_START(aes_xts_encrypt_aesni)
_aes_xts_crypt 1
SYM_FUNC_END(aes_xts_encrypt_aesni)
SYM_FUNC_START(aes_xts_decrypt_aesni)
_aes_xts_crypt 0
SYM_FUNC_END