31 #if !defined(POLARSSL_CONFIG_FILE)
34 #include POLARSSL_CONFIG_FILE
37 #if defined(POLARSSL_AESNI_C)
42 #if defined(POLARSSL_HAVE_X86_64)
47 int aesni_supports(
unsigned int what )
50 static unsigned int c = 0;
54 asm(
"movl $1, %%eax \n\t"
58 :
"eax",
"ebx",
"edx" );
62 return( ( c & what ) != 0 );
75 #define AESDEC ".byte 0x66,0x0F,0x38,0xDE,"
76 #define AESDECLAST ".byte 0x66,0x0F,0x38,0xDF,"
77 #define AESENC ".byte 0x66,0x0F,0x38,0xDC,"
78 #define AESENCLAST ".byte 0x66,0x0F,0x38,0xDD,"
79 #define AESIMC ".byte 0x66,0x0F,0x38,0xDB,"
80 #define AESKEYGENA ".byte 0x66,0x0F,0x3A,0xDF,"
81 #define PCLMULQDQ ".byte 0x66,0x0F,0x3A,0x44,"
83 #define xmm0_xmm0 "0xC0"
84 #define xmm0_xmm1 "0xC8"
85 #define xmm0_xmm2 "0xD0"
86 #define xmm0_xmm3 "0xD8"
87 #define xmm0_xmm4 "0xE0"
88 #define xmm1_xmm0 "0xC1"
89 #define xmm1_xmm2 "0xD1"
96 const unsigned char input[16],
97 unsigned char output[16] )
99 asm(
"movdqu (%3), %%xmm0 \n\t"
100 "movdqu (%1), %%xmm1 \n\t"
101 "pxor %%xmm1, %%xmm0 \n\t"
108 "movdqu (%1), %%xmm1 \n\t"
109 AESENC xmm1_xmm0
"\n\t"
113 "movdqu (%1), %%xmm1 \n\t"
114 AESENCLAST xmm1_xmm0
"\n\t"
118 "movdqu (%1), %%xmm1 \n\t"
119 AESDEC xmm1_xmm0
"\n\t"
123 "movdqu (%1), %%xmm1 \n\t"
124 AESDECLAST xmm1_xmm0
"\n\t"
127 "movdqu %%xmm0, (%4) \n\t"
129 :
"r" (ctx->
nr),
"r" (ctx->
rk),
"r" (mode),
"r" (input),
"r" (output)
130 :
"memory",
"cc",
"xmm0",
"xmm1" );
140 void aesni_gcm_mult(
unsigned char c[16],
141 const unsigned char a[16],
142 const unsigned char b[16] )
144 unsigned char aa[16], bb[16], cc[16];
148 for( i = 0; i < 16; i++ )
154 asm(
"movdqu (%0), %%xmm0 \n\t"
155 "movdqu (%1), %%xmm1 \n\t"
161 "movdqa %%xmm1, %%xmm2 \n\t"
162 "movdqa %%xmm1, %%xmm3 \n\t"
163 "movdqa %%xmm1, %%xmm4 \n\t"
164 PCLMULQDQ xmm0_xmm1
",0x00 \n\t"
165 PCLMULQDQ xmm0_xmm2
",0x11 \n\t"
166 PCLMULQDQ xmm0_xmm3
",0x10 \n\t"
167 PCLMULQDQ xmm0_xmm4
",0x01 \n\t"
168 "pxor %%xmm3, %%xmm4 \n\t"
169 "movdqa %%xmm4, %%xmm3 \n\t"
170 "psrldq $8, %%xmm4 \n\t"
171 "pslldq $8, %%xmm3 \n\t"
172 "pxor %%xmm4, %%xmm2 \n\t"
173 "pxor %%xmm3, %%xmm1 \n\t"
179 "movdqa %%xmm1, %%xmm3 \n\t"
180 "movdqa %%xmm2, %%xmm4 \n\t"
181 "psllq $1, %%xmm1 \n\t"
182 "psllq $1, %%xmm2 \n\t"
183 "psrlq $63, %%xmm3 \n\t"
184 "psrlq $63, %%xmm4 \n\t"
185 "movdqa %%xmm3, %%xmm5 \n\t"
186 "pslldq $8, %%xmm3 \n\t"
187 "pslldq $8, %%xmm4 \n\t"
188 "psrldq $8, %%xmm5 \n\t"
189 "por %%xmm3, %%xmm1 \n\t"
190 "por %%xmm4, %%xmm2 \n\t"
191 "por %%xmm5, %%xmm2 \n\t"
199 "movdqa %%xmm1, %%xmm3 \n\t"
200 "movdqa %%xmm1, %%xmm4 \n\t"
201 "movdqa %%xmm1, %%xmm5 \n\t"
202 "psllq $63, %%xmm3 \n\t"
203 "psllq $62, %%xmm4 \n\t"
204 "psllq $57, %%xmm5 \n\t"
207 "pxor %%xmm4, %%xmm3 \n\t"
208 "pxor %%xmm5, %%xmm3 \n\t"
209 "pslldq $8, %%xmm3 \n\t"
210 "pxor %%xmm3, %%xmm1 \n\t"
213 "movdqa %%xmm1,%%xmm0 \n\t"
214 "movdqa %%xmm1,%%xmm4 \n\t"
215 "movdqa %%xmm1,%%xmm5 \n\t"
216 "psrlq $1, %%xmm0 \n\t"
217 "psrlq $2, %%xmm4 \n\t"
218 "psrlq $7, %%xmm5 \n\t"
219 "pxor %%xmm4, %%xmm0 \n\t"
220 "pxor %%xmm5, %%xmm0 \n\t"
223 "movdqa %%xmm1,%%xmm3 \n\t"
224 "movdqa %%xmm1,%%xmm4 \n\t"
225 "movdqa %%xmm1,%%xmm5 \n\t"
226 "psllq $63, %%xmm3 \n\t"
227 "psllq $62, %%xmm4 \n\t"
228 "psllq $57, %%xmm5 \n\t"
229 "pxor %%xmm4, %%xmm3 \n\t"
230 "pxor %%xmm5, %%xmm3 \n\t"
231 "psrldq $8, %%xmm3 \n\t"
232 "pxor %%xmm3, %%xmm0 \n\t"
233 "pxor %%xmm1, %%xmm0 \n\t"
234 "pxor %%xmm2, %%xmm0 \n\t"
236 "movdqu %%xmm0, (%2) \n\t"
238 :
"r" (aa),
"r" (bb),
"r" (cc)
239 :
"memory",
"cc",
"xmm0",
"xmm1",
"xmm2",
"xmm3",
"xmm4",
"xmm5" );
242 for( i = 0; i < 16; i++ )
251 void aesni_inverse_key(
unsigned char *invkey,
252 const unsigned char *fwdkey,
int nr )
254 unsigned char *ik = invkey;
255 const unsigned char *fk = fwdkey + 16 * nr;
257 memcpy( ik, fk, 16 );
259 for( fk -= 16, ik += 16; fk > fwdkey; fk -= 16, ik += 16 )
260 asm(
"movdqu (%0), %%xmm0 \n\t"
261 AESIMC xmm0_xmm0
"\n\t"
262 "movdqu %%xmm0, (%1) \n\t"
265 :
"memory",
"xmm0" );
267 memcpy( ik, fk, 16 );
273 static void aesni_setkey_enc_128(
unsigned char *rk,
274 const unsigned char *key )
276 asm(
"movdqu (%1), %%xmm0 \n\t"
277 "movdqu %%xmm0, (%0) \n\t"
291 "pshufd $0xff, %%xmm1, %%xmm1 \n\t"
292 "pxor %%xmm0, %%xmm1 \n\t"
293 "pslldq $4, %%xmm0 \n\t"
294 "pxor %%xmm0, %%xmm1 \n\t"
295 "pslldq $4, %%xmm0 \n\t"
296 "pxor %%xmm0, %%xmm1 \n\t"
297 "pslldq $4, %%xmm0 \n\t"
298 "pxor %%xmm1, %%xmm0 \n\t"
300 "movdqu %%xmm0, (%0) \n\t"
305 AESKEYGENA xmm0_xmm1
",0x01 \n\tcall 1b \n\t"
306 AESKEYGENA xmm0_xmm1
",0x02 \n\tcall 1b \n\t"
307 AESKEYGENA xmm0_xmm1
",0x04 \n\tcall 1b \n\t"
308 AESKEYGENA xmm0_xmm1
",0x08 \n\tcall 1b \n\t"
309 AESKEYGENA xmm0_xmm1
",0x10 \n\tcall 1b \n\t"
310 AESKEYGENA xmm0_xmm1
",0x20 \n\tcall 1b \n\t"
311 AESKEYGENA xmm0_xmm1
",0x40 \n\tcall 1b \n\t"
312 AESKEYGENA xmm0_xmm1
",0x80 \n\tcall 1b \n\t"
313 AESKEYGENA xmm0_xmm1
",0x1B \n\tcall 1b \n\t"
314 AESKEYGENA xmm0_xmm1
",0x36 \n\tcall 1b \n\t"
316 :
"r" (rk),
"r" (key)
317 :
"memory",
"cc",
"0" );
323 static void aesni_setkey_enc_192(
unsigned char *rk,
324 const unsigned char *key )
326 asm(
"movdqu (%1), %%xmm0 \n\t"
327 "movdqu %%xmm0, (%0) \n\t"
329 "movq 16(%1), %%xmm1 \n\t"
330 "movq %%xmm1, (%0) \n\t"
344 "pshufd $0x55, %%xmm2, %%xmm2 \n\t"
345 "pxor %%xmm0, %%xmm2 \n\t"
346 "pslldq $4, %%xmm0 \n\t"
347 "pxor %%xmm0, %%xmm2 \n\t"
348 "pslldq $4, %%xmm0 \n\t"
349 "pxor %%xmm0, %%xmm2 \n\t"
350 "pslldq $4, %%xmm0 \n\t"
351 "pxor %%xmm2, %%xmm0 \n\t"
352 "movdqu %%xmm0, (%0) \n\t"
354 "pshufd $0xff, %%xmm0, %%xmm2 \n\t"
355 "pxor %%xmm1, %%xmm2 \n\t"
356 "pslldq $4, %%xmm1 \n\t"
357 "pxor %%xmm2, %%xmm1 \n\t"
358 "movq %%xmm1, (%0) \n\t"
363 AESKEYGENA xmm1_xmm2
",0x01 \n\tcall 1b \n\t"
364 AESKEYGENA xmm1_xmm2
",0x02 \n\tcall 1b \n\t"
365 AESKEYGENA xmm1_xmm2
",0x04 \n\tcall 1b \n\t"
366 AESKEYGENA xmm1_xmm2
",0x08 \n\tcall 1b \n\t"
367 AESKEYGENA xmm1_xmm2
",0x10 \n\tcall 1b \n\t"
368 AESKEYGENA xmm1_xmm2
",0x20 \n\tcall 1b \n\t"
369 AESKEYGENA xmm1_xmm2
",0x40 \n\tcall 1b \n\t"
370 AESKEYGENA xmm1_xmm2
",0x80 \n\tcall 1b \n\t"
373 :
"r" (rk),
"r" (key)
374 :
"memory",
"cc",
"0" );
380 static void aesni_setkey_enc_256(
unsigned char *rk,
381 const unsigned char *key )
383 asm(
"movdqu (%1), %%xmm0 \n\t"
384 "movdqu %%xmm0, (%0) \n\t"
386 "movdqu 16(%1), %%xmm1 \n\t"
387 "movdqu %%xmm1, (%0) \n\t"
400 "pshufd $0xff, %%xmm2, %%xmm2 \n\t"
401 "pxor %%xmm0, %%xmm2 \n\t"
402 "pslldq $4, %%xmm0 \n\t"
403 "pxor %%xmm0, %%xmm2 \n\t"
404 "pslldq $4, %%xmm0 \n\t"
405 "pxor %%xmm0, %%xmm2 \n\t"
406 "pslldq $4, %%xmm0 \n\t"
407 "pxor %%xmm2, %%xmm0 \n\t"
409 "movdqu %%xmm0, (%0) \n\t"
413 AESKEYGENA xmm0_xmm2
",0x00 \n\t"
414 "pshufd $0xaa, %%xmm2, %%xmm2 \n\t"
415 "pxor %%xmm1, %%xmm2 \n\t"
416 "pslldq $4, %%xmm1 \n\t"
417 "pxor %%xmm1, %%xmm2 \n\t"
418 "pslldq $4, %%xmm1 \n\t"
419 "pxor %%xmm1, %%xmm2 \n\t"
420 "pslldq $4, %%xmm1 \n\t"
421 "pxor %%xmm2, %%xmm1 \n\t"
423 "movdqu %%xmm1, (%0) \n\t"
431 AESKEYGENA xmm1_xmm2
",0x01 \n\tcall 1b \n\t"
432 AESKEYGENA xmm1_xmm2
",0x02 \n\tcall 1b \n\t"
433 AESKEYGENA xmm1_xmm2
",0x04 \n\tcall 1b \n\t"
434 AESKEYGENA xmm1_xmm2
",0x08 \n\tcall 1b \n\t"
435 AESKEYGENA xmm1_xmm2
",0x10 \n\tcall 1b \n\t"
436 AESKEYGENA xmm1_xmm2
",0x20 \n\tcall 1b \n\t"
437 AESKEYGENA xmm1_xmm2
",0x40 \n\tcall 1b \n\t"
439 :
"r" (rk),
"r" (key)
440 :
"memory",
"cc",
"0" );
446 int aesni_setkey_enc(
unsigned char *rk,
447 const unsigned char *key,
452 case 128: aesni_setkey_enc_128( rk, key );
break;
453 case 192: aesni_setkey_enc_192( rk, key );
break;
454 case 256: aesni_setkey_enc_256( rk, key );
break;