#include <asm/fpu/api.h>
static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_aesni);
static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_aesni_avx);
static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_vaes_avx2);
static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_vaes_avx512);
static_assert(offsetof(struct aes_enckey, len) == 0);
static_assert(offsetof(struct aes_enckey, nrounds) == 4);
static_assert(offsetof(struct aes_enckey, k.rndkeys) == 16);
static_assert(offsetof(struct aes_key, len) == 0);
static_assert(offsetof(struct aes_key, nrounds) == 4);
static_assert(offsetof(struct aes_key, k.rndkeys) == 16);
static_assert(offsetof(struct aes_key, inv_k.inv_rndkeys) == 256);
void aes128_expandkey_aesni(u32 rndkeys[], u32 *inv_rndkeys,
const u8 in_key[AES_KEYSIZE_128]);
void aes256_expandkey_aesni(u32 rndkeys[], u32 *inv_rndkeys,
const u8 in_key[AES_KEYSIZE_256]);
void aes_encrypt_aesni(u8 dst[AES_BLOCK_SIZE], const u8 src[AES_BLOCK_SIZE],
const struct aes_enckey *key);
void aes_decrypt_aesni(u8 dst[AES_BLOCK_SIZE], const u8 src[AES_BLOCK_SIZE],
const struct aes_key *key);
static void aes_preparekey_arch(union aes_enckey_arch *k,
union aes_invkey_arch *inv_k,
const u8 *in_key, int key_len, int nrounds)
{
u32 *rndkeys = k->rndkeys;
u32 *inv_rndkeys = inv_k ? inv_k->inv_rndkeys : NULL;
if (static_branch_likely(&have_aesni) && key_len != AES_KEYSIZE_192 &&
irq_fpu_usable()) {
kernel_fpu_begin();
if (key_len == AES_KEYSIZE_128)
aes128_expandkey_aesni(rndkeys, inv_rndkeys, in_key);
else
aes256_expandkey_aesni(rndkeys, inv_rndkeys, in_key);
kernel_fpu_end();
} else {
aes_expandkey_generic(rndkeys, inv_rndkeys, in_key, key_len);
}
}
static void aes_encrypt_arch(const struct aes_enckey *key,
u8 out[AES_BLOCK_SIZE],
const u8 in[AES_BLOCK_SIZE])
{
if (static_branch_likely(&have_aesni) && irq_fpu_usable()) {
kernel_fpu_begin();
aes_encrypt_aesni(out, in, key);
kernel_fpu_end();
} else {
aes_encrypt_generic(key->k.rndkeys, key->nrounds, out, in);
}
}
static void aes_decrypt_arch(const struct aes_key *key,
u8 out[AES_BLOCK_SIZE],
const u8 in[AES_BLOCK_SIZE])
{
if (static_branch_likely(&have_aesni) && irq_fpu_usable()) {
kernel_fpu_begin();
aes_decrypt_aesni(out, in, key);
kernel_fpu_end();
} else {
aes_decrypt_generic(key->inv_k.inv_rndkeys, key->nrounds,
out, in);
}
}
#if IS_ENABLED(CONFIG_CRYPTO_LIB_AES_ECB)
void aes_ecb_encrypt_aesni(u8 *dst, const u8 *src, long nblocks,
const struct aes_enckey *key);
void aes_ecb_decrypt_aesni(u8 *dst, const u8 *src, long nblocks,
const struct aes_key *key);
#define aes_ecb_encrypt_arch aes_ecb_encrypt_arch
static bool aes_ecb_encrypt_arch(u8 *dst, const u8 *src, size_t len,
const struct aes_enckey *key)
{
if (!static_branch_likely(&have_aesni) || unlikely(!irq_fpu_usable()))
return false;
kernel_fpu_begin();
aes_ecb_encrypt_aesni(dst, src, len / AES_BLOCK_SIZE, key);
kernel_fpu_end();
return true;
}
#define aes_ecb_decrypt_arch aes_ecb_decrypt_arch
static bool aes_ecb_decrypt_arch(u8 *dst, const u8 *src, size_t len,
const struct aes_key *key)
{
if (!static_branch_likely(&have_aesni) || unlikely(!irq_fpu_usable()))
return false;
kernel_fpu_begin();
aes_ecb_decrypt_aesni(dst, src, len / AES_BLOCK_SIZE, key);
kernel_fpu_end();
return true;
}
#endif /* CONFIG_CRYPTO_LIB_AES_ECB */
#if IS_ENABLED(CONFIG_CRYPTO_LIB_AES_CBC)
void aes_cbc_encrypt_aesni(u8 *dst, const u8 *src, long nblocks,
u8 iv[AES_BLOCK_SIZE], const struct aes_enckey *key);
void aes_cbc_decrypt_aesni(u8 *dst, const u8 *src, long nblocks,
u8 iv[AES_BLOCK_SIZE], const struct aes_key *key);
void aes_cbc_cts_encrypt_aesni(u8 *dst, const u8 *src, long pn_len,
const u8 iv[AES_BLOCK_SIZE],
const struct aes_enckey *key);
void aes_cbc_cts_decrypt_aesni(u8 *dst, const u8 *src, long pn_len,
const u8 iv[AES_BLOCK_SIZE],
const struct aes_key *key);
#define aes_cbc_encrypt_arch aes_cbc_encrypt_arch
static bool aes_cbc_encrypt_arch(u8 *dst, const u8 *src, size_t len,
u8 iv[AES_BLOCK_SIZE],
const struct aes_enckey *key)
{
if (!static_branch_likely(&have_aesni) || unlikely(!irq_fpu_usable()))
return false;
kernel_fpu_begin();
aes_cbc_encrypt_aesni(dst, src, len / AES_BLOCK_SIZE, iv, key);
kernel_fpu_end();
return true;
}
#define aes_cbc_decrypt_arch aes_cbc_decrypt_arch
static bool aes_cbc_decrypt_arch(u8 *dst, const u8 *src, size_t len,
u8 iv[AES_BLOCK_SIZE],
const struct aes_key *key)
{
if (!static_branch_likely(&have_aesni) || unlikely(!irq_fpu_usable()))
return false;
kernel_fpu_begin();
aes_cbc_decrypt_aesni(dst, src, len / AES_BLOCK_SIZE, iv, key);
kernel_fpu_end();
return true;
}
#define aes_cbc_cts_encrypt_arch aes_cbc_cts_encrypt_arch
static bool aes_cbc_cts_encrypt_arch(u8 *dst, const u8 *src, size_t len,
u8 iv[AES_BLOCK_SIZE],
const struct aes_enckey *key)
{
const size_t cbc_blocks = (len - AES_BLOCK_SIZE - 1) / AES_BLOCK_SIZE;
const size_t pn_len = ((len - 1) % AES_BLOCK_SIZE) + 1;
if (!static_branch_likely(&have_aesni) || unlikely(!irq_fpu_usable()))
return false;
kernel_fpu_begin();
if (cbc_blocks) {
aes_cbc_encrypt_aesni(dst, src, cbc_blocks, iv, key);
dst += cbc_blocks * AES_BLOCK_SIZE;
src += cbc_blocks * AES_BLOCK_SIZE;
}
aes_cbc_cts_encrypt_aesni(dst, src, pn_len, iv, key);
kernel_fpu_end();
return true;
}
#define aes_cbc_cts_decrypt_arch aes_cbc_cts_decrypt_arch
static bool aes_cbc_cts_decrypt_arch(u8 *dst, const u8 *src, size_t len,
u8 iv[AES_BLOCK_SIZE],
const struct aes_key *key)
{
const size_t cbc_blocks = (len - AES_BLOCK_SIZE - 1) / AES_BLOCK_SIZE;
const size_t pn_len = ((len - 1) % AES_BLOCK_SIZE) + 1;
if (!static_branch_likely(&have_aesni) || unlikely(!irq_fpu_usable()))
return false;
kernel_fpu_begin();
if (cbc_blocks) {
aes_cbc_decrypt_aesni(dst, src, cbc_blocks, iv, key);
dst += cbc_blocks * AES_BLOCK_SIZE;
src += cbc_blocks * AES_BLOCK_SIZE;
}
aes_cbc_cts_decrypt_aesni(dst, src, pn_len, iv, key);
kernel_fpu_end();
return true;
}
#endif /* CONFIG_CRYPTO_LIB_AES_CBC */
#if IS_ENABLED(CONFIG_CRYPTO_LIB_AES_CTR) && IS_ENABLED(CONFIG_X86_64)
void aes_ctr64_crypt_aesni(u8 *dst, const u8 *src, s64 len, const u64 le_ctr[2],
const struct aes_enckey *key);
void aes_ctr64_crypt_aesni_avx(u8 *dst, const u8 *src, s64 len,
const u64 le_ctr[2],
const struct aes_enckey *key);
void aes_ctr64_crypt_vaes_avx2(u8 *dst, const u8 *src, s64 len,
const u64 le_ctr[2],
const struct aes_enckey *key);
void aes_ctr64_crypt_vaes_avx512(u8 *dst, const u8 *src, s64 len,
const u64 le_ctr[2],
const struct aes_enckey *key);
void aes_xctr_crypt_aesni_avx(u8 *dst, const u8 *src, s64 len, u64 ctr,
const u8 iv[AES_BLOCK_SIZE],
const struct aes_enckey *key);
void aes_xctr_crypt_vaes_avx2(u8 *dst, const u8 *src, s64 len, u64 ctr,
const u8 iv[AES_BLOCK_SIZE],
const struct aes_enckey *key);
void aes_xctr_crypt_vaes_avx512(u8 *dst, const u8 *src, s64 len, u64 ctr,
const u8 iv[AES_BLOCK_SIZE],
const struct aes_enckey *key);
static void aes_ctr64_x86(u8 *dst, const u8 *src, size_t len,
const u64 le_ctr[2], const struct aes_enckey *key)
{
if (static_branch_likely(&have_vaes_avx512))
aes_ctr64_crypt_vaes_avx512(dst, src, len, le_ctr, key);
else if (static_branch_likely(&have_vaes_avx2))
aes_ctr64_crypt_vaes_avx2(dst, src, len, le_ctr, key);
else if (static_branch_likely(&have_aesni_avx))
aes_ctr64_crypt_aesni_avx(dst, src, len, le_ctr, key);
else
aes_ctr64_crypt_aesni(dst, src, len, le_ctr, key);
}
#define aes_ctr_arch aes_ctr_arch
static bool aes_ctr_arch(u8 *dst, const u8 *src, size_t len,
u8 ctr[AES_BLOCK_SIZE], const struct aes_enckey *key)
{
u64 le_ctr[2];
u64 ctr64;
size_t nblocks;
size_t part1_len;
if (!static_branch_likely(&have_aesni) || unlikely(!irq_fpu_usable()))
return false;
ctr64 = le_ctr[0] = get_unaligned_be64(&ctr[8]);
le_ctr[1] = get_unaligned_be64(&ctr[0]);
kernel_fpu_begin();
nblocks = DIV_ROUND_UP(len, AES_BLOCK_SIZE);
ctr64 += nblocks;
if (likely(ctr64 >= nblocks)) {
aes_ctr64_x86(dst, src, len, le_ctr, key);
} else {
part1_len = min(len, (nblocks - ctr64) * AES_BLOCK_SIZE);
aes_ctr64_x86(dst, src, part1_len, le_ctr, key);
le_ctr[0] = 0;
le_ctr[1]++;
aes_ctr64_x86(dst + part1_len, src + part1_len, len - part1_len,
le_ctr, key);
}
kernel_fpu_end();
put_unaligned_be64(ctr64, &ctr[8]);
put_unaligned_be64(le_ctr[1], &ctr[0]);
return true;
}
#define aes_xctr_arch aes_xctr_arch
static bool aes_xctr_arch(u8 *dst, const u8 *src, size_t len, u64 ctr,
const u8 iv[AES_BLOCK_SIZE],
const struct aes_enckey *key)
{
if (!static_branch_likely(&have_aesni_avx) ||
unlikely(!irq_fpu_usable()))
return false;
kernel_fpu_begin();
if (static_branch_likely(&have_vaes_avx512))
aes_xctr_crypt_vaes_avx512(dst, src, len, ctr, iv, key);
else if (static_branch_likely(&have_vaes_avx2))
aes_xctr_crypt_vaes_avx2(dst, src, len, ctr, iv, key);
else
aes_xctr_crypt_aesni_avx(dst, src, len, ctr, iv, key);
kernel_fpu_end();
return true;
}
#endif /* CONFIG_CRYPTO_LIB_AES_CTR && CONFIG_X86_64 */
#if IS_ENABLED(CONFIG_CRYPTO_LIB_AES_XTS)
void aes_xts_encrypt_aesni(u8 *dst, const u8 *src, long nblocks,
u8 tweak[AES_BLOCK_SIZE], const struct aes_key *key);
void aes_xts_decrypt_aesni(u8 *dst, const u8 *src, long nblocks,
u8 tweak[AES_BLOCK_SIZE], const struct aes_key *key);
void aes_xts_encrypt_iv(const struct aes_enckey *tweak_key,
u8 iv[AES_BLOCK_SIZE]);
void aes_xts_encrypt_aesni_avx(u8 *dst, const u8 *src, long nblocks,
u8 tweak[AES_BLOCK_SIZE],
const struct aes_key *key);
void aes_xts_decrypt_aesni_avx(u8 *dst, const u8 *src, long nblocks,
u8 tweak[AES_BLOCK_SIZE],
const struct aes_key *key);
void aes_xts_encrypt_vaes_avx2(u8 *dst, const u8 *src, long nblocks,
u8 tweak[AES_BLOCK_SIZE],
const struct aes_key *key);
void aes_xts_decrypt_vaes_avx2(u8 *dst, const u8 *src, long nblocks,
u8 tweak[AES_BLOCK_SIZE],
const struct aes_key *key);
void aes_xts_encrypt_vaes_avx512(u8 *dst, const u8 *src, long nblocks,
u8 tweak[AES_BLOCK_SIZE],
const struct aes_key *key);
void aes_xts_decrypt_vaes_avx512(u8 *dst, const u8 *src, long nblocks,
u8 tweak[AES_BLOCK_SIZE],
const struct aes_key *key);
static __always_inline bool
aes_xts_crypt_x86(u8 *dst, const u8 *src, size_t len, u8 tweak[AES_BLOCK_SIZE],
const struct aes_xts_key *key, bool cont, bool enc)
{
const long nblocks = len / AES_BLOCK_SIZE;
if (!static_branch_likely(&have_aesni) || unlikely(!irq_fpu_usable()))
return false;
kernel_fpu_begin();
if (IS_ENABLED(CONFIG_X86_64) &&
static_branch_likely(&have_vaes_avx512)) {
if (!cont)
aes_xts_encrypt_iv(&key->tweak_key, tweak);
if (enc)
aes_xts_encrypt_vaes_avx512(dst, src, nblocks, tweak,
&key->main_key);
else
aes_xts_decrypt_vaes_avx512(dst, src, nblocks, tweak,
&key->main_key);
} else if (IS_ENABLED(CONFIG_X86_64) &&
static_branch_likely(&have_vaes_avx2)) {
if (!cont)
aes_xts_encrypt_iv(&key->tweak_key, tweak);
if (enc)
aes_xts_encrypt_vaes_avx2(dst, src, nblocks, tweak,
&key->main_key);
else
aes_xts_decrypt_vaes_avx2(dst, src, nblocks, tweak,
&key->main_key);
} else if (IS_ENABLED(CONFIG_X86_64) &&
static_branch_likely(&have_aesni_avx)) {
if (!cont)
aes_xts_encrypt_iv(&key->tweak_key, tweak);
if (enc)
aes_xts_encrypt_aesni_avx(dst, src, nblocks, tweak,
&key->main_key);
else
aes_xts_decrypt_aesni_avx(dst, src, nblocks, tweak,
&key->main_key);
} else {
if (!cont)
aes_encrypt_aesni(tweak, tweak, &key->tweak_key);
if (enc)
aes_xts_encrypt_aesni(dst, src, nblocks, tweak,
&key->main_key);
else
aes_xts_decrypt_aesni(dst, src, nblocks, tweak,
&key->main_key);
}
kernel_fpu_end();
return true;
}
#define aes_xts_encrypt_arch aes_xts_encrypt_arch
static bool aes_xts_encrypt_arch(u8 *dst, const u8 *src, size_t len,
u8 tweak[AES_BLOCK_SIZE],
const struct aes_xts_key *key, bool cont)
{
return aes_xts_crypt_x86(dst, src, len, tweak, key, cont, true);
}
#define aes_xts_decrypt_arch aes_xts_decrypt_arch
static bool aes_xts_decrypt_arch(u8 *dst, const u8 *src, size_t len,
u8 tweak[AES_BLOCK_SIZE],
const struct aes_xts_key *key, bool cont)
{
return aes_xts_crypt_x86(dst, src, len, tweak, key, cont, false);
}
#endif /* CONFIG_CRYPTO_LIB_AES_XTS */
#define aes_mod_init_arch aes_mod_init_arch
static void aes_mod_init_arch(void)
{
if (!boot_cpu_has(X86_FEATURE_AES))
return;
static_branch_enable(&have_aesni);
if (!boot_cpu_has(X86_FEATURE_AVX) || !IS_ENABLED(CONFIG_X86_64))
return;
static_branch_enable(&have_aesni_avx);
if (!boot_cpu_has(X86_FEATURE_VAES) ||
!boot_cpu_has(X86_FEATURE_AVX2) ||
!boot_cpu_has(X86_FEATURE_VPCLMULQDQ) ||
!boot_cpu_has(X86_FEATURE_PCLMULQDQ))
return;
static_branch_enable(&have_vaes_avx2);
if (!boot_cpu_has(X86_FEATURE_AVX512BW) ||
!boot_cpu_has(X86_FEATURE_AVX512VL) ||
!boot_cpu_has(X86_FEATURE_BMI2) ||
boot_cpu_has(X86_FEATURE_PREFER_YMM))
return;
static_branch_enable(&have_vaes_avx512);
}