31 #if !defined(POLARSSL_CONFIG_FILE) 34 #include POLARSSL_CONFIG_FILE 37 #if defined(POLARSSL_AESNI_C) 42 #if defined(POLARSSL_HAVE_X86_64) 47 int aesni_supports(
unsigned int what )
50 static unsigned int c = 0;
54 asm(
"movl $1, %%eax \n\t" 58 :
"eax",
"ebx",
"edx" );
62 return( ( c & what ) != 0 );
75 #define AESDEC ".byte 0x66,0x0F,0x38,0xDE," 76 #define AESDECLAST ".byte 0x66,0x0F,0x38,0xDF," 77 #define AESENC ".byte 0x66,0x0F,0x38,0xDC," 78 #define AESENCLAST ".byte 0x66,0x0F,0x38,0xDD," 79 #define AESIMC ".byte 0x66,0x0F,0x38,0xDB," 80 #define AESKEYGENA ".byte 0x66,0x0F,0x3A,0xDF," 81 #define PCLMULQDQ ".byte 0x66,0x0F,0x3A,0x44," 83 #define xmm0_xmm0 "0xC0" 84 #define xmm0_xmm1 "0xC8" 85 #define xmm0_xmm2 "0xD0" 86 #define xmm0_xmm3 "0xD8" 87 #define xmm0_xmm4 "0xE0" 88 #define xmm1_xmm0 "0xC1" 89 #define xmm1_xmm2 "0xD1" 96 const unsigned char input[16],
97 unsigned char output[16] )
99 asm(
"movdqu (%3), %%xmm0 \n\t" 100 "movdqu (%1), %%xmm1 \n\t" 101 "pxor %%xmm1, %%xmm0 \n\t" 108 "movdqu (%1), %%xmm1 \n\t" 109 AESENC xmm1_xmm0
"\n\t" 113 "movdqu (%1), %%xmm1 \n\t" 114 AESENCLAST xmm1_xmm0
"\n\t" 118 "movdqu (%1), %%xmm1 \n\t" 119 AESDEC xmm1_xmm0
"\n\t" 123 "movdqu (%1), %%xmm1 \n\t" 124 AESDECLAST xmm1_xmm0
"\n\t" 127 "movdqu %%xmm0, (%4) \n\t" 129 :
"r" (ctx->
nr),
"r" (ctx->
rk),
"r" (mode),
"r" (input),
"r" (output)
130 :
"memory",
"cc",
"xmm0",
"xmm1" );
140 void aesni_gcm_mult(
unsigned char c[16],
141 const unsigned char a[16],
142 const unsigned char b[16] )
144 unsigned char aa[16], bb[16], cc[16];
148 for( i = 0; i < 16; i++ )
154 asm(
"movdqu (%0), %%xmm0 \n\t" 155 "movdqu (%1), %%xmm1 \n\t" 161 "movdqa %%xmm1, %%xmm2 \n\t" 162 "movdqa %%xmm1, %%xmm3 \n\t" 163 "movdqa %%xmm1, %%xmm4 \n\t" 164 PCLMULQDQ xmm0_xmm1
",0x00 \n\t" 165 PCLMULQDQ xmm0_xmm2
",0x11 \n\t" 166 PCLMULQDQ xmm0_xmm3
",0x10 \n\t" 167 PCLMULQDQ xmm0_xmm4
",0x01 \n\t" 168 "pxor %%xmm3, %%xmm4 \n\t" 169 "movdqa %%xmm4, %%xmm3 \n\t" 170 "psrldq $8, %%xmm4 \n\t" 171 "pslldq $8, %%xmm3 \n\t" 172 "pxor %%xmm4, %%xmm2 \n\t" 173 "pxor %%xmm3, %%xmm1 \n\t" 179 "movdqa %%xmm1, %%xmm3 \n\t" 180 "movdqa %%xmm2, %%xmm4 \n\t" 181 "psllq $1, %%xmm1 \n\t" 182 "psllq $1, %%xmm2 \n\t" 183 "psrlq $63, %%xmm3 \n\t" 184 "psrlq $63, %%xmm4 \n\t" 185 "movdqa %%xmm3, %%xmm5 \n\t" 186 "pslldq $8, %%xmm3 \n\t" 187 "pslldq $8, %%xmm4 \n\t" 188 "psrldq $8, %%xmm5 \n\t" 189 "por %%xmm3, %%xmm1 \n\t" 190 "por %%xmm4, %%xmm2 \n\t" 191 "por %%xmm5, %%xmm2 \n\t" 199 "movdqa %%xmm1, %%xmm3 \n\t" 200 "movdqa %%xmm1, %%xmm4 \n\t" 201 "movdqa %%xmm1, %%xmm5 \n\t" 202 "psllq $63, %%xmm3 \n\t" 203 "psllq $62, %%xmm4 \n\t" 204 "psllq $57, %%xmm5 \n\t" 207 "pxor %%xmm4, %%xmm3 \n\t" 208 "pxor %%xmm5, %%xmm3 \n\t" 209 "pslldq $8, %%xmm3 \n\t" 210 "pxor %%xmm3, %%xmm1 \n\t" 213 "movdqa %%xmm1,%%xmm0 \n\t" 214 "movdqa %%xmm1,%%xmm4 \n\t" 215 "movdqa %%xmm1,%%xmm5 \n\t" 216 "psrlq $1, %%xmm0 \n\t" 217 "psrlq $2, %%xmm4 \n\t" 218 "psrlq $7, %%xmm5 \n\t" 219 "pxor %%xmm4, %%xmm0 \n\t" 220 "pxor %%xmm5, %%xmm0 \n\t" 223 "movdqa %%xmm1,%%xmm3 \n\t" 224 "movdqa %%xmm1,%%xmm4 \n\t" 225 "movdqa %%xmm1,%%xmm5 \n\t" 226 "psllq $63, %%xmm3 \n\t" 227 "psllq $62, %%xmm4 \n\t" 228 "psllq $57, %%xmm5 \n\t" 229 "pxor %%xmm4, %%xmm3 \n\t" 230 "pxor %%xmm5, %%xmm3 \n\t" 231 "psrldq $8, %%xmm3 \n\t" 232 "pxor %%xmm3, %%xmm0 \n\t" 233 "pxor %%xmm1, %%xmm0 \n\t" 234 "pxor %%xmm2, %%xmm0 \n\t" 236 "movdqu %%xmm0, (%2) \n\t" 238 :
"r" (aa),
"r" (bb),
"r" (cc)
239 :
"memory",
"cc",
"xmm0",
"xmm1",
"xmm2",
"xmm3",
"xmm4",
"xmm5" );
242 for( i = 0; i < 16; i++ )
251 void aesni_inverse_key(
unsigned char *invkey,
252 const unsigned char *fwdkey,
int nr )
254 unsigned char *ik = invkey;
255 const unsigned char *fk = fwdkey + 16 * nr;
257 memcpy( ik, fk, 16 );
259 for( fk -= 16, ik += 16; fk > fwdkey; fk -= 16, ik += 16 )
260 asm(
"movdqu (%0), %%xmm0 \n\t" 261 AESIMC xmm0_xmm0
"\n\t" 262 "movdqu %%xmm0, (%1) \n\t" 265 :
"memory",
"xmm0" );
267 memcpy( ik, fk, 16 );
273 static void aesni_setkey_enc_128(
unsigned char *rk,
274 const unsigned char *key )
276 asm(
"movdqu (%1), %%xmm0 \n\t" 277 "movdqu %%xmm0, (%0) \n\t" 291 "pshufd $0xff, %%xmm1, %%xmm1 \n\t" 292 "pxor %%xmm0, %%xmm1 \n\t" 293 "pslldq $4, %%xmm0 \n\t" 294 "pxor %%xmm0, %%xmm1 \n\t" 295 "pslldq $4, %%xmm0 \n\t" 296 "pxor %%xmm0, %%xmm1 \n\t" 297 "pslldq $4, %%xmm0 \n\t" 298 "pxor %%xmm1, %%xmm0 \n\t" 300 "movdqu %%xmm0, (%0) \n\t" 305 AESKEYGENA xmm0_xmm1
",0x01 \n\tcall 1b \n\t" 306 AESKEYGENA xmm0_xmm1
",0x02 \n\tcall 1b \n\t" 307 AESKEYGENA xmm0_xmm1
",0x04 \n\tcall 1b \n\t" 308 AESKEYGENA xmm0_xmm1
",0x08 \n\tcall 1b \n\t" 309 AESKEYGENA xmm0_xmm1
",0x10 \n\tcall 1b \n\t" 310 AESKEYGENA xmm0_xmm1
",0x20 \n\tcall 1b \n\t" 311 AESKEYGENA xmm0_xmm1
",0x40 \n\tcall 1b \n\t" 312 AESKEYGENA xmm0_xmm1
",0x80 \n\tcall 1b \n\t" 313 AESKEYGENA xmm0_xmm1
",0x1B \n\tcall 1b \n\t" 314 AESKEYGENA xmm0_xmm1
",0x36 \n\tcall 1b \n\t" 316 :
"r" (rk),
"r" (key)
317 :
"memory",
"cc",
"0" );
323 static void aesni_setkey_enc_192(
unsigned char *rk,
324 const unsigned char *key )
326 asm(
"movdqu (%1), %%xmm0 \n\t" 327 "movdqu %%xmm0, (%0) \n\t" 329 "movq 16(%1), %%xmm1 \n\t" 330 "movq %%xmm1, (%0) \n\t" 344 "pshufd $0x55, %%xmm2, %%xmm2 \n\t" 345 "pxor %%xmm0, %%xmm2 \n\t" 346 "pslldq $4, %%xmm0 \n\t" 347 "pxor %%xmm0, %%xmm2 \n\t" 348 "pslldq $4, %%xmm0 \n\t" 349 "pxor %%xmm0, %%xmm2 \n\t" 350 "pslldq $4, %%xmm0 \n\t" 351 "pxor %%xmm2, %%xmm0 \n\t" 352 "movdqu %%xmm0, (%0) \n\t" 354 "pshufd $0xff, %%xmm0, %%xmm2 \n\t" 355 "pxor %%xmm1, %%xmm2 \n\t" 356 "pslldq $4, %%xmm1 \n\t" 357 "pxor %%xmm2, %%xmm1 \n\t" 358 "movq %%xmm1, (%0) \n\t" 363 AESKEYGENA xmm1_xmm2
",0x01 \n\tcall 1b \n\t" 364 AESKEYGENA xmm1_xmm2
",0x02 \n\tcall 1b \n\t" 365 AESKEYGENA xmm1_xmm2
",0x04 \n\tcall 1b \n\t" 366 AESKEYGENA xmm1_xmm2
",0x08 \n\tcall 1b \n\t" 367 AESKEYGENA xmm1_xmm2
",0x10 \n\tcall 1b \n\t" 368 AESKEYGENA xmm1_xmm2
",0x20 \n\tcall 1b \n\t" 369 AESKEYGENA xmm1_xmm2
",0x40 \n\tcall 1b \n\t" 370 AESKEYGENA xmm1_xmm2
",0x80 \n\tcall 1b \n\t" 373 :
"r" (rk),
"r" (key)
374 :
"memory",
"cc",
"0" );
380 static void aesni_setkey_enc_256(
unsigned char *rk,
381 const unsigned char *key )
383 asm(
"movdqu (%1), %%xmm0 \n\t" 384 "movdqu %%xmm0, (%0) \n\t" 386 "movdqu 16(%1), %%xmm1 \n\t" 387 "movdqu %%xmm1, (%0) \n\t" 400 "pshufd $0xff, %%xmm2, %%xmm2 \n\t" 401 "pxor %%xmm0, %%xmm2 \n\t" 402 "pslldq $4, %%xmm0 \n\t" 403 "pxor %%xmm0, %%xmm2 \n\t" 404 "pslldq $4, %%xmm0 \n\t" 405 "pxor %%xmm0, %%xmm2 \n\t" 406 "pslldq $4, %%xmm0 \n\t" 407 "pxor %%xmm2, %%xmm0 \n\t" 409 "movdqu %%xmm0, (%0) \n\t" 413 AESKEYGENA xmm0_xmm2
",0x00 \n\t" 414 "pshufd $0xaa, %%xmm2, %%xmm2 \n\t" 415 "pxor %%xmm1, %%xmm2 \n\t" 416 "pslldq $4, %%xmm1 \n\t" 417 "pxor %%xmm1, %%xmm2 \n\t" 418 "pslldq $4, %%xmm1 \n\t" 419 "pxor %%xmm1, %%xmm2 \n\t" 420 "pslldq $4, %%xmm1 \n\t" 421 "pxor %%xmm2, %%xmm1 \n\t" 423 "movdqu %%xmm1, (%0) \n\t" 431 AESKEYGENA xmm1_xmm2
",0x01 \n\tcall 1b \n\t" 432 AESKEYGENA xmm1_xmm2
",0x02 \n\tcall 1b \n\t" 433 AESKEYGENA xmm1_xmm2
",0x04 \n\tcall 1b \n\t" 434 AESKEYGENA xmm1_xmm2
",0x08 \n\tcall 1b \n\t" 435 AESKEYGENA xmm1_xmm2
",0x10 \n\tcall 1b \n\t" 436 AESKEYGENA xmm1_xmm2
",0x20 \n\tcall 1b \n\t" 437 AESKEYGENA xmm1_xmm2
",0x40 \n\tcall 1b \n\t" 439 :
"r" (rk),
"r" (key)
440 :
"memory",
"cc",
"0" );
446 int aesni_setkey_enc(
unsigned char *rk,
447 const unsigned char *key,
452 case 128: aesni_setkey_enc_128( rk, key );
break;
453 case 192: aesni_setkey_enc_192( rk, key );
break;
454 case 256: aesni_setkey_enc_256( rk, key );
break;
Configuration options (set of defines)
AES-NI for hardware AES acceleration on some Intel processors.
#define POLARSSL_ERR_AES_INVALID_KEY_LENGTH
Invalid key length.