yuzu/mbedtls
1
0
Fork 0
mirror of https://github.com/yuzu-emu/mbedtls.git synced 2024-10-18 12:31:05 +00:00
Code Issues Packages Projects Releases Wiki Activity

SHA: Use SHA-NI intrinsics to speed up SHA256 hashing

Browse source
This commit is contained in:
Morph 2021-12-30 22:05:43 -05:00
parent 00d92c2989
commit 3ef96cc1ad
8 changed files with 353 additions and 9 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

36
include/mbedtls/shani.h Normal file
View file

@ -0,0 +1,36 @@
#pragma once
#include <stdint.h>
#if !defined(MBEDTLS_CONFIG_FILE)
#include "mbedtls/config.h"
#else
#include MBEDTLS_CONFIG_FILE
#endif
#if defined(MBEDTLS_HAVE_ASM) && defined(__GNUC__) && \
(defined(__amd64__) || defined(__x86_64__)) && \
!defined(MBEDTLS_HAVE_X86_64)
#define MBEDTLS_HAVE_X86_64
#endif
#if defined(_MSC_VER) && defined(_M_X64) && !defined(MBEDTLS_HAVE_X86_64)
#define MBEDTLS_HAVE_X86_64
#endif
#if defined(MBEDTLS_HAVE_X86_64)
#ifdef __cplusplus
extern "C" {
#endif
int mbedtls_shani_has_support(void);
int mbedtls_internal_sha256ni_process(uint32_t state[8],
const unsigned char data[64]);
#ifdef __cplusplus
}
#endif
#endif /* MBEDTLS_HAVE_X86_64 */

7
library/CMakeLists.txt
View file

@ -82,6 +82,7 @@ set(src_crypto
sha1.c
sha256.c
sha512.c
shani.c
threading.c
timing.c
version.c
@ -119,10 +120,16 @@ set(src_tls
if(CMAKE_COMPILER_IS_GNUCC)
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wmissing-declarations -Wmissing-prototypes")
if(CMAKE_SYSTEM_PROCESSOR MATCHES "(x64|x86_64|amd64|AMD64)")
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -msse4.2 -msha")
endif()
endif(CMAKE_COMPILER_IS_GNUCC)
if(CMAKE_COMPILER_IS_CLANG)
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wmissing-declarations -Wmissing-prototypes -Wdocumentation -Wno-documentation-deprecated-sync -Wunreachable-code")
if(CMAKE_SYSTEM_PROCESSOR MATCHES "(x64|x86_64|amd64|AMD64)")
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -msse4.2 -msha")
endif()
endif(CMAKE_COMPILER_IS_CLANG)
if(WIN32)

1
library/Makefile
View file

@ -139,6 +139,7 @@ OBJS_CRYPTO= \
sha1.o \
sha256.o \
sha512.o \
shani.o \
threading.o \
timing.o \
version.o \

43
library/sha256.c
View file

@ -27,6 +27,7 @@
#if defined(MBEDTLS_SHA256_C)
#include "mbedtls/sha256.h"
#include "mbedtls/shani.h"
#include "mbedtls/platform_util.h"
#include "mbedtls/error.h"
@ -262,7 +263,11 @@ int mbedtls_internal_sha256_process( mbedtls_sha256_context *ctx,
void mbedtls_sha256_process( mbedtls_sha256_context *ctx,
const unsigned char data[64] )
{
mbedtls_internal_sha256_process( ctx, data );
if (mbedtls_shani_has_support() != 0) {
mbedtls_internal_sha256ni_process(ctx->state, data);
} else {
mbedtls_internal_sha256_process( ctx, data );
}
}
#endif
#endif /* !MBEDTLS_SHA256_PROCESS_ALT */
@ -297,8 +302,13 @@ int mbedtls_sha256_update_ret( mbedtls_sha256_context *ctx,
{
memcpy( (void *) (ctx->buffer + left), input, fill );
if( ( ret = mbedtls_internal_sha256_process( ctx, ctx->buffer ) ) != 0 )
return( ret );
if (mbedtls_shani_has_support() != 0) {
if( ( ret = mbedtls_internal_sha256ni_process( ctx->state, ctx->buffer ) ) != 0 )
return( ret );
} else {
if( ( ret = mbedtls_internal_sha256_process( ctx, ctx->buffer ) ) != 0 )
return( ret );
}
input += fill;
ilen -= fill;
@ -307,8 +317,13 @@ int mbedtls_sha256_update_ret( mbedtls_sha256_context *ctx,
while( ilen >= 64 )
{
if( ( ret = mbedtls_internal_sha256_process( ctx, input ) ) != 0 )
return( ret );
if (mbedtls_shani_has_support() != 0) {
if( ( ret = mbedtls_internal_sha256ni_process( ctx->state, input ) ) != 0 )
return( ret );
} else {
if( ( ret = mbedtls_internal_sha256_process( ctx, input ) ) != 0 )
return( ret );
}
input += 64;
ilen -= 64;
@ -359,8 +374,13 @@ int mbedtls_sha256_finish_ret( mbedtls_sha256_context *ctx,
/* We'll need an extra block */
memset( ctx->buffer + used, 0, 64 - used );
if( ( ret = mbedtls_internal_sha256_process( ctx, ctx->buffer ) ) != 0 )
return( ret );
if (mbedtls_shani_has_support() != 0) {
if( ( ret = mbedtls_internal_sha256ni_process( ctx->state, ctx->buffer ) ) != 0 )
return( ret );
} else {
if( ( ret = mbedtls_internal_sha256_process( ctx, ctx->buffer ) ) != 0 )
return( ret );
}
memset( ctx->buffer, 0, 56 );
}
@ -375,8 +395,13 @@ int mbedtls_sha256_finish_ret( mbedtls_sha256_context *ctx,
MBEDTLS_PUT_UINT32_BE( high, ctx->buffer, 56 );
MBEDTLS_PUT_UINT32_BE( low, ctx->buffer, 60 );
if( ( ret = mbedtls_internal_sha256_process( ctx, ctx->buffer ) ) != 0 )
return( ret );
if (mbedtls_shani_has_support() != 0) {
if( ( ret = mbedtls_internal_sha256ni_process( ctx->state, ctx->buffer ) ) != 0 )
return( ret );
} else {
if( ( ret = mbedtls_internal_sha256_process( ctx, ctx->buffer ) ) != 0 )
return( ret );
}
/*
* Output final state

271
library/shani.c Normal file
View file

@ -0,0 +1,271 @@
#include "mbedtls/shani.h"
#if defined(MBEDTLS_HAVE_X86_64)
#ifdef _MSC_VER
#include <intrin.h>
#else
#include <x86intrin.h>
#ifndef asm
#define asm __asm
#endif
#if defined(__DragonFly__) || defined(__FreeBSD__)
// clang-format off
#include <sys/types.h>
#include <machine/cpufunc.h>
// clang-format on
#endif
static inline void __cpuidex(int info[4], int function_id, int subfunction_id) {
#if defined(__DragonFly__) || defined(__FreeBSD__)
// Despite the name, this is just do_cpuid() with ECX as second input.
cpuid_count((u_int)function_id, (u_int)subfunction_id, (u_int *)info);
#else
info[0] = function_id; // eax
info[2] = subfunction_id; // ecx
asm("cpuid"
: "=a"(info[0]), "=b"(info[1]), "=c"(info[2]), "=d"(info[3])
: "a"(function_id), "c"(subfunction_id));
#endif
}
#endif
/*
* SHA-NI support detection routine
*/
int mbedtls_shani_has_support(void) {
static int done = 0;
static unsigned int b = 0;
if (done != 1) {
int regs[4]; // eax, ebx, ecx, edx
__cpuidex(regs, 7, 0);
b = regs[1];
}
done = 1;
return ((b & (1 << 29)) != 0);
}
int mbedtls_internal_sha256ni_process(uint32_t state[8],
const unsigned char data[64]) {
// Implementation based from
// https://www.intel.com/content/www/us/en/developer/articles/technical/intel-sha-extensions.html
__m128i STATE0, STATE1;
__m128i ABEF_SAVE, CDGH_SAVE;
__m128i MSG, MSGTEMP;
__m128i MSGTEMP0, MSGTEMP1, MSGTEMP2, MSGTEMP3;
__m128i SHUF_MASK;
// Load initial digest
STATE0 = _mm_loadu_si128((__m128i *)&state[0]);
STATE1 = _mm_loadu_si128((__m128i *)&state[4]);
// CDAB
MSGTEMP = _mm_shuffle_epi32(STATE0, 0xB1);
// EFGH
STATE1 = _mm_shuffle_epi32(STATE1, 0x1B);
// ABEF
STATE0 = _mm_alignr_epi8(MSGTEMP, STATE1, 8);
// CDGH
STATE1 = _mm_blend_epi16(STATE1, MSGTEMP, 0xF0);
SHUF_MASK = _mm_set_epi8(0xC, 0xD, 0xE, 0xF, 0x8, 0x9, 0xA, 0xB,
0x4, 0x5, 0x6, 0x7, 0x0, 0x1, 0x2, 0x3);
// Save digests
ABEF_SAVE = STATE0;
CDGH_SAVE = STATE1;
// Rounds 0-3
MSG = _mm_loadu_si128((__m128i *)&data[0]);
MSGTEMP0 = _mm_shuffle_epi8(MSG, SHUF_MASK);
MSG = _mm_add_epi32(MSGTEMP0,
_mm_set_epi64x(0xE9B5DBA5B5C0FBCF, 0x71374491428A2F98));
STATE1 = _mm_sha256rnds2_epu32(STATE1, STATE0, MSG);
MSG = _mm_shuffle_epi32(MSG, 0x0E);
STATE0 = _mm_sha256rnds2_epu32(STATE0, STATE1, MSG);
// Rounds 4-7
MSGTEMP1 = _mm_loadu_si128((const __m128i *)(data + 16));
MSGTEMP1 = _mm_shuffle_epi8(MSGTEMP1, SHUF_MASK);
MSG = _mm_add_epi32(MSGTEMP1,
_mm_set_epi64x(0xAB1C5ED5923F82A4, 0x59F111F13956C25B));
STATE1 = _mm_sha256rnds2_epu32(STATE1, STATE0, MSG);
MSG = _mm_shuffle_epi32(MSG, 0x0E);
STATE0 = _mm_sha256rnds2_epu32(STATE0, STATE1, MSG);
MSGTEMP0 = _mm_sha256msg1_epu32(MSGTEMP0, MSGTEMP1);
// Rounds 8-11
MSGTEMP2 = _mm_loadu_si128((const __m128i *)(data + 32));
MSGTEMP2 = _mm_shuffle_epi8(MSGTEMP2, SHUF_MASK);
MSG = _mm_add_epi32(MSGTEMP2,
_mm_set_epi64x(0x550C7DC3243185BE, 0x12835B01D807AA98));
STATE1 = _mm_sha256rnds2_epu32(STATE1, STATE0, MSG);
MSG = _mm_shuffle_epi32(MSG, 0x0E);
STATE0 = _mm_sha256rnds2_epu32(STATE0, STATE1, MSG);
MSGTEMP1 = _mm_sha256msg1_epu32(MSGTEMP1, MSGTEMP2);
// Rounds 12-15
MSGTEMP3 = _mm_loadu_si128((const __m128i *)(data + 48));
MSGTEMP3 = _mm_shuffle_epi8(MSGTEMP3, SHUF_MASK);
MSG = _mm_add_epi32(MSGTEMP3,
_mm_set_epi64x(0xC19BF1749BDC06A7, 0x80DEB1FE72BE5D74));
STATE1 = _mm_sha256rnds2_epu32(STATE1, STATE0, MSG);
MSGTEMP = _mm_alignr_epi8(MSGTEMP3, MSGTEMP2, 4);
MSGTEMP0 = _mm_add_epi32(MSGTEMP0, MSGTEMP);
MSGTEMP0 = _mm_sha256msg2_epu32(MSGTEMP0, MSGTEMP3);
MSG = _mm_shuffle_epi32(MSG, 0x0E);
STATE0 = _mm_sha256rnds2_epu32(STATE0, STATE1, MSG);
MSGTEMP2 = _mm_sha256msg1_epu32(MSGTEMP2, MSGTEMP3);
// Rounds 16-19
MSG = _mm_add_epi32(MSGTEMP0,
_mm_set_epi64x(0x240CA1CC0FC19DC6, 0xEFBE4786E49B69C1));
STATE1 = _mm_sha256rnds2_epu32(STATE1, STATE0, MSG);
MSGTEMP = _mm_alignr_epi8(MSGTEMP0, MSGTEMP3, 4);
MSGTEMP1 = _mm_add_epi32(MSGTEMP1, MSGTEMP);
MSGTEMP1 = _mm_sha256msg2_epu32(MSGTEMP1, MSGTEMP0);
MSG = _mm_shuffle_epi32(MSG, 0x0E);
STATE0 = _mm_sha256rnds2_epu32(STATE0, STATE1, MSG);
MSGTEMP3 = _mm_sha256msg1_epu32(MSGTEMP3, MSGTEMP0);
// Rounds 20-23
MSG = _mm_add_epi32(MSGTEMP1,
_mm_set_epi64x(0x76F988DA5CB0A9DC, 0x4A7484AA2DE92C6F));
STATE1 = _mm_sha256rnds2_epu32(STATE1, STATE0, MSG);
MSGTEMP = _mm_alignr_epi8(MSGTEMP1, MSGTEMP0, 4);
MSGTEMP2 = _mm_add_epi32(MSGTEMP2, MSGTEMP);
MSGTEMP2 = _mm_sha256msg2_epu32(MSGTEMP2, MSGTEMP1);
MSG = _mm_shuffle_epi32(MSG, 0x0E);
STATE0 = _mm_sha256rnds2_epu32(STATE0, STATE1, MSG);
MSGTEMP0 = _mm_sha256msg1_epu32(MSGTEMP0, MSGTEMP1);
// Rounds 24-27
MSG = _mm_add_epi32(MSGTEMP2,
_mm_set_epi64x(0xBF597FC7B00327C8, 0xA831C66D983E5152));
STATE1 = _mm_sha256rnds2_epu32(STATE1, STATE0, MSG);
MSGTEMP = _mm_alignr_epi8(MSGTEMP2, MSGTEMP1, 4);
MSGTEMP3 = _mm_add_epi32(MSGTEMP3, MSGTEMP);
MSGTEMP3 = _mm_sha256msg2_epu32(MSGTEMP3, MSGTEMP2);
MSG = _mm_shuffle_epi32(MSG, 0x0E);
STATE0 = _mm_sha256rnds2_epu32(STATE0, STATE1, MSG);
MSGTEMP1 = _mm_sha256msg1_epu32(MSGTEMP1, MSGTEMP2);
// Rounds 28-31
MSG = _mm_add_epi32(MSGTEMP3,
_mm_set_epi64x(0x1429296706CA6351, 0xD5A79147C6E00BF3));
STATE1 = _mm_sha256rnds2_epu32(STATE1, STATE0, MSG);
MSGTEMP = _mm_alignr_epi8(MSGTEMP3, MSGTEMP2, 4);
MSGTEMP0 = _mm_add_epi32(MSGTEMP0, MSGTEMP);
MSGTEMP0 = _mm_sha256msg2_epu32(MSGTEMP0, MSGTEMP3);
MSG = _mm_shuffle_epi32(MSG, 0x0E);
STATE0 = _mm_sha256rnds2_epu32(STATE0, STATE1, MSG);
MSGTEMP2 = _mm_sha256msg1_epu32(MSGTEMP2, MSGTEMP3);
// Rounds 32-35
MSG = _mm_add_epi32(MSGTEMP0,
_mm_set_epi64x(0x53380D134D2C6DFC, 0x2E1B213827B70A85));
STATE1 = _mm_sha256rnds2_epu32(STATE1, STATE0, MSG);
MSGTEMP = _mm_alignr_epi8(MSGTEMP0, MSGTEMP3, 4);
MSGTEMP1 = _mm_add_epi32(MSGTEMP1, MSGTEMP);
MSGTEMP1 = _mm_sha256msg2_epu32(MSGTEMP1, MSGTEMP0);
MSG = _mm_shuffle_epi32(MSG, 0x0E);
STATE0 = _mm_sha256rnds2_epu32(STATE0, STATE1, MSG);
MSGTEMP3 = _mm_sha256msg1_epu32(MSGTEMP3, MSGTEMP0);
// Rounds 36-39
MSG = _mm_add_epi32(MSGTEMP1,
_mm_set_epi64x(0x92722C8581C2C92E, 0x766A0ABB650A7354));
STATE1 = _mm_sha256rnds2_epu32(STATE1, STATE0, MSG);
MSGTEMP = _mm_alignr_epi8(MSGTEMP1, MSGTEMP0, 4);
MSGTEMP2 = _mm_add_epi32(MSGTEMP2, MSGTEMP);
MSGTEMP2 = _mm_sha256msg2_epu32(MSGTEMP2, MSGTEMP1);
MSG = _mm_shuffle_epi32(MSG, 0x0E);
STATE0 = _mm_sha256rnds2_epu32(STATE0, STATE1, MSG);
MSGTEMP0 = _mm_sha256msg1_epu32(MSGTEMP0, MSGTEMP1);
// Rounds 40-43
MSG = _mm_add_epi32(MSGTEMP2,
_mm_set_epi64x(0xC76C51A3C24B8B70, 0xA81A664BA2BFE8A1));
STATE1 = _mm_sha256rnds2_epu32(STATE1, STATE0, MSG);
MSGTEMP = _mm_alignr_epi8(MSGTEMP2, MSGTEMP1, 4);
MSGTEMP3 = _mm_add_epi32(MSGTEMP3, MSGTEMP);
MSGTEMP3 = _mm_sha256msg2_epu32(MSGTEMP3, MSGTEMP2);
MSG = _mm_shuffle_epi32(MSG, 0x0E);
STATE0 = _mm_sha256rnds2_epu32(STATE0, STATE1, MSG);
MSGTEMP1 = _mm_sha256msg1_epu32(MSGTEMP1, MSGTEMP2);
// Rounds 44-47
MSG = _mm_add_epi32(MSGTEMP3,
_mm_set_epi64x(0x106AA070F40E3585, 0xD6990624D192E819));
STATE1 = _mm_sha256rnds2_epu32(STATE1, STATE0, MSG);
MSGTEMP = _mm_alignr_epi8(MSGTEMP3, MSGTEMP2, 4);
MSGTEMP0 = _mm_add_epi32(MSGTEMP0, MSGTEMP);
MSGTEMP0 = _mm_sha256msg2_epu32(MSGTEMP0, MSGTEMP3);
MSG = _mm_shuffle_epi32(MSG, 0x0E);
STATE0 = _mm_sha256rnds2_epu32(STATE0, STATE1, MSG);
MSGTEMP2 = _mm_sha256msg1_epu32(MSGTEMP2, MSGTEMP3);
// Rounds 48-51
MSG = _mm_add_epi32(MSGTEMP0,
_mm_set_epi64x(0x34B0BCB52748774C, 0x1E376C0819A4C116));
STATE1 = _mm_sha256rnds2_epu32(STATE1, STATE0, MSG);
MSGTEMP = _mm_alignr_epi8(MSGTEMP0, MSGTEMP3, 4);
MSGTEMP1 = _mm_add_epi32(MSGTEMP1, MSGTEMP);
MSGTEMP1 = _mm_sha256msg2_epu32(MSGTEMP1, MSGTEMP0);
MSG = _mm_shuffle_epi32(MSG, 0x0E);
STATE0 = _mm_sha256rnds2_epu32(STATE0, STATE1, MSG);
MSGTEMP3 = _mm_sha256msg1_epu32(MSGTEMP3, MSGTEMP0);
// Rounds 52-55
MSG = _mm_add_epi32(MSGTEMP1,
_mm_set_epi64x(0x682E6FF35B9CCA4F, 0x4ED8AA4A391C0CB3));
STATE1 = _mm_sha256rnds2_epu32(STATE1, STATE0, MSG);
MSGTEMP = _mm_alignr_epi8(MSGTEMP1, MSGTEMP0, 4);
MSGTEMP2 = _mm_add_epi32(MSGTEMP2, MSGTEMP);
MSGTEMP2 = _mm_sha256msg2_epu32(MSGTEMP2, MSGTEMP1);
MSG = _mm_shuffle_epi32(MSG, 0x0E);
STATE0 = _mm_sha256rnds2_epu32(STATE0, STATE1, MSG);
// Rounds 56-59
MSG = _mm_add_epi32(MSGTEMP2,
_mm_set_epi64x(0x8CC7020884C87814, 0x78A5636F748F82EE));
STATE1 = _mm_sha256rnds2_epu32(STATE1, STATE0, MSG);
MSGTEMP = _mm_alignr_epi8(MSGTEMP2, MSGTEMP1, 4);
MSGTEMP3 = _mm_add_epi32(MSGTEMP3, MSGTEMP);
MSGTEMP3 = _mm_sha256msg2_epu32(MSGTEMP3, MSGTEMP2);
MSG = _mm_shuffle_epi32(MSG, 0x0E);
STATE0 = _mm_sha256rnds2_epu32(STATE0, STATE1, MSG);
// Rounds 60-63
MSG = _mm_add_epi32(MSGTEMP3,
_mm_set_epi64x(0xC67178F2BEF9A3F7, 0xA4506CEB90BEFFFA));
STATE1 = _mm_sha256rnds2_epu32(STATE1, STATE0, MSG);
MSG = _mm_shuffle_epi32(MSG, 0x0E);
STATE0 = _mm_sha256rnds2_epu32(STATE0, STATE1, MSG);
STATE0 = _mm_add_epi32(STATE0, ABEF_SAVE);
STATE1 = _mm_add_epi32(STATE1, CDGH_SAVE);
// FEBA
MSGTEMP = _mm_shuffle_epi32(STATE0, 0x1B);
// DCHG
STATE1 = _mm_shuffle_epi32(STATE1, 0xB1);
// DCBA
STATE0 = _mm_blend_epi16(MSGTEMP, STATE1, 0xF0);
// ABEF
STATE1 = _mm_alignr_epi8(STATE1, MSGTEMP, 8);
// Store digests
_mm_storeu_si128((__m128i *)&state[0], STATE0);
_mm_storeu_si128((__m128i *)&state[4], STATE1);
return 0;
}
#endif // MBEDTLS_HAVE_X86_64

1
programs/test/query_config.c
View file

@ -91,6 +91,7 @@
#include "mbedtls/sha1.h"
#include "mbedtls/sha256.h"
#include "mbedtls/sha512.h"
#include "mbedtls/shani.h"
#include "mbedtls/ssl.h"
#include "mbedtls/ssl_cache.h"
#include "mbedtls/ssl_ciphersuites.h"

1
scripts/data_files/query_config.fmt
View file

@ -91,6 +91,7 @@
#include "mbedtls/sha1.h"
#include "mbedtls/sha256.h"
#include "mbedtls/sha512.h"
#include "mbedtls/shani.h"
#include "mbedtls/ssl.h"
#include "mbedtls/ssl_cache.h"
#include "mbedtls/ssl_ciphersuites.h"

2
visualc/VS2010/mbedTLS.vcxproj
View file

@ -208,6 +208,7 @@
<ClInclude Include="..\..\include\mbedtls\sha1.h" />
<ClInclude Include="..\..\include\mbedtls\sha256.h" />
<ClInclude Include="..\..\include\mbedtls\sha512.h" />
<ClInclude Include="..\..\include\mbedtls\shani.h" />
<ClInclude Include="..\..\include\mbedtls\ssl.h" />
<ClInclude Include="..\..\include\mbedtls\ssl_cache.h" />
<ClInclude Include="..\..\include\mbedtls\ssl_ciphersuites.h" />
@ -361,6 +362,7 @@
<ClCompile Include="..\..\library\sha1.c" />
<ClCompile Include="..\..\library\sha256.c" />
<ClCompile Include="..\..\library\sha512.c" />
<ClCompile Include="..\..\library\shani.c" />
<ClCompile Include="..\..\library\ssl_cache.c" />
<ClCompile Include="..\..\library\ssl_ciphersuites.c" />
<ClCompile Include="..\..\library\ssl_cli.c" />