From dfb6015ca79a9fee28f7fcb0af7e350a83574b83 Mon Sep 17 00:00:00 2001 From: "Markku-Juhani O. Saarinen" Date: Mon, 20 Nov 2017 14:58:41 +0000 Subject: Implements AES and GCM with ARMv8 Crypto Extensions A compact patch that provides AES and GCM implementations that utilize the ARMv8 Crypto Extensions. The config flag is MBEDTLS_ARMV8CE_AES_C, which is disabled by default as we don't do runtime checking for the feature. The new implementation lives in armv8ce_aes.c. Provides similar functionality to https://github.com/ARMmbed/mbedtls/pull/432 Thanks to Barry O'Rourke and others for that contribtion. Tested on a Cortex A53 device and QEMU. On a midrange phone the real AES-GCM throughput increases about 4x, while raw AES speed is up to 10x faster. When cross-compiling, you want to set something like: export CC='aarch64-linux-gnu-gcc' export CFLAGS='-Ofast -march=armv8-a+crypto' scripts/config.pl set MBEDTLS_ARMV8CE_AES_C QEMU seems to also need export LDFLAGS='-static' Then run normal make or cmake etc. --- --- a/ChangeLog.d/armv8_crypto_extensions.txt +++ b/ChangeLog.d/armv8_crypto_extensions.txt @@ -0,0 +1,2 @@ +Features + * Support ARMv8 Cryptography Extensions for AES and GCM. --- a/include/mbedtls/armv8ce_aes.h +++ b/include/mbedtls/armv8ce_aes.h @@ -0,0 +1,63 @@ +/** + * \file armv8ce_aes.h + * + * \brief ARMv8 Cryptography Extensions -- Optimized code for AES and GCM + */ + +/* + * + * Copyright (C) 2006-2017, ARM Limited, All Rights Reserved + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * This file is part of mbed TLS (https://tls.mbed.org) + */ + +#ifndef MBEDTLS_ARMV8CE_AES_H +#define MBEDTLS_ARMV8CE_AES_H + +#include "aes.h" + +/** + * \brief [ARMv8 Crypto Extensions] AES-ECB block en(de)cryption + * + * \param ctx AES context + * \param mode MBEDTLS_AES_ENCRYPT or MBEDTLS_AES_DECRYPT + * \param input 16-byte input block + * \param output 16-byte output block + * + * \return 0 on success (cannot fail) + */ + +int mbedtls_armv8ce_aes_crypt_ecb( mbedtls_aes_context *ctx, + int mode, + const unsigned char input[16], + unsigned char output[16] ); + +/** + * \brief [ARMv8 Crypto Extensions] Multiply in GF(2^128) for GCM + * + * \param c Result + * \param a First operand + * \param b Second operand + * + * \note Both operands and result are bit strings interpreted as + * elements of GF(2^128) as per the GCM spec. + */ + +void mbedtls_armv8ce_gcm_mult( unsigned char c[16], + const unsigned char a[16], + const unsigned char b[16] ); + +#endif /* MBEDTLS_ARMV8CE_AES_H */ --- a/include/mbedtls/check_config.h +++ b/include/mbedtls/check_config.h @@ -95,6 +95,10 @@ #error "MBEDTLS_AESNI_C defined, but not all prerequisites" #endif +#if defined(MBEDTLS_ARMV8CE_AES_C) && !defined(MBEDTLS_HAVE_ASM) +#error "MBEDTLS_ARMV8CE_AES_C defined, but not all prerequisites" +#endif + #if defined(MBEDTLS_CTR_DRBG_C) && !defined(MBEDTLS_AES_C) #error "MBEDTLS_CTR_DRBG_C defined, but not all prerequisites" #endif @@ -772,3 +776,4 @@ typedef int mbedtls_iso_c_forbids_empty_translation_units; #endif /* MBEDTLS_CHECK_CONFIG_H */ + --- a/include/mbedtls/config.h +++ b/include/mbedtls/config.h @@ -73,6 +73,7 @@ * Requires support for asm() in compiler. * * Used in: + * library/armv8ce_aes.c * library/aria.c * library/timing.c * include/mbedtls/bn_mul.h @@ -1905,6 +1906,21 @@ #define MBEDTLS_AESNI_C /** + * \def MBEDTLS_ARMV8CE_AES_C + * + * Enable ARMv8 Crypto Extensions for AES and GCM + * + * Module: library/armv8ce_aes.c + * Caller: library/aes.c + * library/gcm.c + * + * Requires: MBEDTLS_HAVE_ASM + * + * This module adds support for Armv8 Cryptography Extensions for AES and GCM. + */ +//#define MBEDTLS_ARMV8CE_AES_C + +/** * \def MBEDTLS_AES_C * * Enable the AES block cipher. --- a/library/aes.c +++ b/library/aes.c @@ -69,7 +69,9 @@ #if defined(MBEDTLS_AESNI_C) #include "mbedtls/aesni.h" #endif - +#if defined(MBEDTLS_ARMV8CE_AES_C) +#include "mbedtls/armv8ce_aes.h" +#endif #if defined(MBEDTLS_SELF_TEST) #if defined(MBEDTLS_PLATFORM_C) #include "mbedtls/platform.h" @@ -1052,6 +1054,11 @@ return( mbedtls_aesni_crypt_ecb( ctx, mode, input, output ) ); #endif +#if defined(MBEDTLS_ARMV8CE_AES_C) + // We don't do runtime checking for ARMv8 Crypto Extensions + return mbedtls_armv8ce_aes_crypt_ecb( ctx, mode, input, output ); +#endif + #if defined(MBEDTLS_PADLOCK_C) && defined(MBEDTLS_HAVE_X86) if( aes_padlock_ace ) { --- a/library/armv8ce_aes.c +++ b/library/armv8ce_aes.c @@ -0,0 +1,142 @@ +/* + * ARMv8 Cryptography Extensions -- Optimized code for AES and GCM + * + * Copyright (C) 2006-2017, ARM Limited, All Rights Reserved + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * This file is part of mbed TLS (https://tls.mbed.org) + */ + +#if !defined(MBEDTLS_CONFIG_FILE) +#include "mbedtls/config.h" +#else +#include MBEDTLS_CONFIG_FILE +#endif + +#if defined(MBEDTLS_ARMV8CE_AES_C) + +#include +#include "mbedtls/armv8ce_aes.h" + +#ifndef asm +#define asm __asm +#endif + +/* + * [Armv8 Cryptography Extensions] AES-ECB block en(de)cryption + */ + +#if defined(MBEDTLS_AES_C) + +int mbedtls_armv8ce_aes_crypt_ecb( mbedtls_aes_context *ctx, + int mode, + const unsigned char input[16], + unsigned char output[16] ) +{ + unsigned int i; + const uint8_t *rk; + uint8x16_t x, k; + + x = vld1q_u8( input ); /* input block */ + rk = (const uint8_t *) ctx->rk; /* round keys */ + + if( mode == MBEDTLS_AES_ENCRYPT ) + { + for( i = ctx->nr - 1; i != 0; i-- ) /* encryption loop */ + { + k = vld1q_u8( rk ); + rk += 16; + x = vaeseq_u8( x, k ); + x = vaesmcq_u8( x ); + } + k = vld1q_u8( rk ); + rk += 16; + x = vaeseq_u8( x, k ); + } + else + { + for( i = ctx->nr - 1; i != 0 ; i-- ) /* decryption loop */ + { + k = vld1q_u8( rk ); + rk += 16; + x = vaesdq_u8( x, k ); + x = vaesimcq_u8( x ); + } + k = vld1q_u8( rk ); + rk += 16; + x = vaesdq_u8( x, k ); + } + + k = vld1q_u8( rk ); /* final key just XORed */ + x = veorq_u8( x, k ); + vst1q_u8( output, x ); /* write out */ + + return ( 0 ); +} + +#endif /* MBEDTLS_AES_C */ + + +/* + * [Armv8 Cryptography Extensions] Multiply in GF(2^128) for GCM + */ + +#if defined(MBEDTLS_GCM_C) + +void mbedtls_armv8ce_gcm_mult( unsigned char c[16], + const unsigned char a[16], + const unsigned char b[16] ) +{ + /* GCM's GF(2^128) polynomial basis is x^128 + x^7 + x^2 + x + 1 */ + const uint64x2_t base = { 0, 0x86 }; /* note missing LS bit */ + + register uint8x16_t vc asm( "v0" ); /* named registers */ + register uint8x16_t va asm( "v1" ); /* (to avoid conflict) */ + register uint8x16_t vb asm( "v2" ); + register uint64x2_t vp asm( "v3" ); + + va = vld1q_u8( a ); /* load inputs */ + vb = vld1q_u8( b ); + vp = base; + + asm ( + "rbit %1.16b, %1.16b \n\t" /* reverse bit order */ + "rbit %2.16b, %2.16b \n\t" + "pmull2 %0.1q, %1.2d, %2.2d \n\t" /* v0 = a.hi * b.hi */ + "pmull2 v4.1q, %0.2d, %3.2d \n\t" /* mul v0 by x^64, reduce */ + "ext %0.16b, %0.16b, %0.16b, #8 \n\t" + "eor %0.16b, %0.16b, v4.16b \n\t" + "ext v5.16b, %2.16b, %2.16b, #8 \n\t" /* (swap hi and lo in b) */ + "pmull v4.1q, %1.1d, v5.1d \n\t" /* v0 ^= a.lo * b.hi */ + "eor %0.16b, %0.16b, v4.16b \n\t" + "pmull2 v4.1q, %1.2d, v5.2d \n\t" /* v0 ^= a.hi * b.lo */ + "eor %0.16b, %0.16b, v4.16b \n\t" + "pmull2 v4.1q, %0.2d, %3.2d \n\t" /* mul v0 by x^64, reduce */ + "ext %0.16b, %0.16b, %0.16b, #8 \n\t" + "eor %0.16b, %0.16b, v4.16b \n\t" + "pmull v4.1q, %1.1d, %2.1d \n\t" /* v0 ^= a.lo * b.lo */ + "eor %0.16b, %0.16b, v4.16b \n\t" + "rbit %0.16b, %0.16b \n\t" /* reverse bits for output */ + : "=w" (vc) /* q0: output */ + : "w" (va), "w" (vb), "w" (vp) /* q1, q2: input */ + : "v4", "v5" /* q4, q5: clobbered */ + ); + + vst1q_u8( c, vc ); /* write out */ +} + +#endif /* MBEDTLS_GCM_C */ + +#endif /* MBEDTLS_ARMV8CE_AES_C */ --- a/library/CMakeLists.txt +++ b/library/CMakeLists.txt @@ -7,6 +7,7 @@ aesni.c arc4.c aria.c + armv8ce_aes.c asn1parse.c asn1write.c base64.c --- a/library/gcm.c +++ b/library/gcm.c @@ -71,6 +71,10 @@ #include "mbedtls/aesni.h" #endif +#if defined(MBEDTLS_ARMV8CE_AES_C) +#include "mbedtls/armv8ce_aes.h" +#endif + #if defined(MBEDTLS_SELF_TEST) && defined(MBEDTLS_AES_C) #include "mbedtls/aes.h" #include "mbedtls/platform.h" @@ -140,6 +144,12 @@ if( ( ret = mbedtls_cipher_update( &ctx->cipher_ctx, h, 16, h, &olen ) ) != 0 ) return( ret ); +#if defined(MBEDTLS_ARMV8CE_AES_C) + // we don't do feature testing with ARMv8 cryptography extensions + memcpy( ctx ->HL, h, 16 ); // put H at the beginning of buffer + return( 0 ); // that's all we need +#endif + /* pack h as two 64-bits ints, big-endian */ GET_UINT32_BE( hi, h, 0 ); GET_UINT32_BE( lo, h, 4 ); @@ -248,6 +258,11 @@ unsigned char lo, hi, rem; uint64_t zh, zl; +#if defined(MBEDTLS_ARMV8CE_AES_C) + mbedtls_armv8ce_gcm_mult( output, x, (const unsigned char *) ctx->HL ); + return; +#endif + #if defined(MBEDTLS_AESNI_C) && defined(MBEDTLS_HAVE_X86_64) if( mbedtls_aesni_has_support( MBEDTLS_AESNI_CLMUL ) ) { unsigned char h[16]; --- a/library/Makefile +++ b/library/Makefile @@ -65,6 +65,7 @@ OBJS_CRYPTO= aes.o aesni.o arc4.o \ aria.o asn1parse.o asn1write.o \ + armv8ce_aes.o \ base64.o bignum.o blowfish.o \ camellia.o ccm.o chacha20.o \ chachapoly.o cipher.o cipher_wrap.o \ --- a/library/version_features.c +++ b/library/version_features.c @@ -586,6 +586,9 @@ #if defined(MBEDTLS_AESNI_C) "MBEDTLS_AESNI_C", #endif /* MBEDTLS_AESNI_C */ +#if defined(MBEDTLS_ARMV8CE_AES_C) + "MBEDTLS_ARMV8CE_AES_C", +#endif /* MBEDTLS_ARMV8CE_AES_C */ #if defined(MBEDTLS_AES_C) "MBEDTLS_AES_C", #endif /* MBEDTLS_AES_C */