3 # ====================================================================
4 # Written by Andy Polyakov <appro@fy.chalmers.se> for the OpenSSL
5 # project. Rights for redistribution and usage in source and binary
6 # forms are granted according to the OpenSSL license.
7 # ====================================================================
9 # SHA512_Transform_SSE2.
11 # As the name suggests, this is an IA-32 SSE2 implementation of
12 # SHA512_Transform. Motivating factor for the undertaken effort was that
13 # SHA512 was observed to *consistently* perform *significantly* poorer
14 # than SHA256 [2x and slower is common] on 32-bit platforms. On 64-bit
15 # platforms on the other hand SHA512 tend to outperform SHA256 [~50%
16 # seem to be common improvement factor]. All this is perfectly natural,
17 # as SHA512 is a 64-bit algorithm. But isn't IA-32 SSE2 essentially
18 # a 64-bit instruction set? Is it rich enough to implement SHA512?
19 # If answer was "no," then you wouldn't have been reading this...
21 # Throughput performance in MBps (larger is better):
23 # 2.4GHz P4 1.4GHz AMD32 1.4GHz AMD64(*)
24 # SHA256/gcc(*) 54 43 59
26 # SHA512/sse2 61(**) 57(**)
30 # (*) AMD64 and SHA256 numbers are presented mostly for amusement or
32 # (**) I.e. it gives ~2-3x speed-up if compared with compiler generated
33 # code. One can argue that hand-coded *non*-SSE2 implementation
34 # would perform better than compiler generated one as well, and
35 # that comparison is therefore not exactly fair. Well, as SHA512
36 # puts enormous pressure on IA-32 GP register bank, I reckon that
37 # hand-coded version wouldn't perform significantly better than
38 # one compiled with icc, ~20% perhaps... So that this code would
39 # still outperform it with distinguishing marginal. But feel free
40 # to prove me wrong:-)
41 # <appro@fy.chalmers.se>
42 push(@INC,"perlasm","../../perlasm");
45 &asm_init
($ARGV[0],"sha512-sse2.pl",$ARGV[$#ARGV] eq "386");
47 $K512="esi"; # K512[80] table, found at the end...
48 #$W512="esp"; # $W512 is not just W512[16]: it comprises *two* copies
49 # of W512[16] and a copy of A-H variables...
50 $W512_SZ=8*(16+16+8); # see above...
51 #$Kidx="ebx"; # index in K512 table, advances from 0 to 80...
52 $Widx="edx"; # index in W512, wraps around at 16...
53 $data="edi"; # 16 qwords of input data...
55 $E="mm1"; # F-H are allocated dynamically...
56 $Aoff=256+0; # A-H offsets relative to $W512...
66 { local ($kidx,$widx)=@_;
68 # One can argue that one could reorder instructions for better
69 # performance. Well, I tried and it doesn't seem to make any
70 # noticeable difference. Modern out-of-order execution cores
71 # reorder instructions to their liking in either case and they
72 # apparently do decent job. So we can keep the code more
73 # readable/regular/comprehensible:-)
75 # I adhere to 64-bit %mmX registers in order to avoid/not care
76 # about #GP exceptions on misaligned 128-bit access, most
77 # notably in paddq with memory operand. Not to mention that
78 # SSE2 intructions operating on %mmX can be scheduled every
79 # cycle [and not every second one if operating on %xmmN].
81 &movq
("mm4",&QWP
($Foff,$W512)); # load f
82 &movq
("mm5",&QWP
($Goff,$W512)); # load g
83 &movq
("mm6",&QWP
($Hoff,$W512)); # load h
85 &movq
("mm2",$E); # %mm2 is sliding right
86 &movq
("mm3",$E); # %mm3 is sliding left
89 &movq
("mm7","mm2"); # %mm7 is T1
98 &pxor
("mm7","mm3"); # T1=Sigma1_512(e)
100 &movq
(&QWP
($Foff,$W512),$E); # f = e
101 &movq
(&QWP
($Goff,$W512),"mm4"); # g = f
102 &movq
(&QWP
($Hoff,$W512),"mm5"); # h = g
104 &pxor
("mm4","mm5"); # f^=g
105 &pand
("mm4",$E); # f&=e
106 &pxor
("mm4","mm5"); # f^=g
107 &paddq
("mm7","mm4"); # T1+=Ch(e,f,g)
109 &movq
("mm2",&QWP
($Boff,$W512)); # load b
110 &movq
("mm3",&QWP
($Coff,$W512)); # load c
111 &movq
($E,&QWP
($Doff,$W512)); # e = d
113 &paddq
("mm7","mm6"); # T1+=h
114 &paddq
("mm7",&QWP
(0,$K512,$kidx,8)); # T1+=K512[i]
115 &paddq
("mm7",&QWP
(0,$W512,$widx,8)); # T1+=W512[i]
116 &paddq
($E,"mm7"); # e += T1
118 &movq
("mm4",$A); # %mm4 is sliding right
119 &movq
("mm5",$A); # %mm5 is sliding left
122 &movq
("mm6","mm4"); # %mm6 is T2
131 &pxor
("mm6","mm5"); # T2=Sigma0_512(a)
133 &movq
(&QWP
($Boff,$W512),$A); # b = a
134 &movq
(&QWP
($Coff,$W512),"mm2"); # c = b
135 &movq
(&QWP
($Doff,$W512),"mm3"); # d = c
137 &movq
("mm4",$A); # %mm4=a
138 &por
($A,"mm3"); # a=a|c
139 &pand
("mm4","mm3"); # %mm4=a&c
140 &pand
($A,"mm2"); # a=(a|c)&b
141 &por
("mm4",$A); # %mm4=(a&c)|((a|c)&b)
142 &paddq
("mm6","mm4"); # T2+=Maj(a,b,c)
144 &movq
($A,"mm7"); # a=T1
145 &paddq
($A,"mm6"); # a+=T2
148 $func="sha512_block_sse2";
150 &function_begin_B
($func);
151 if (0) {# Caller is expected to check if it's appropriate to
152 # call this routine. Below 3 lines are retained for
153 # debugging purposes...
154 &picmeup
("eax","OPENSSL_ia32cap");
155 &bt
(&DWP
(0,"eax"),26);
156 &jnc
("SHA512_Transform");
165 &mov
($Widx,&DWP
(8,"ebp")); # A-H state, 1st arg
166 &mov
($data,&DWP
(12,"ebp")); # input data, 2nd arg
167 &call
(&label
("pic_point")); # make it PIC!
168 &set_label
("pic_point");
170 &lea
($K512,&DWP
(&label
("K512")."-".&label
("pic_point"),$K512));
172 $W512 = "esp"; # start using %esp as W512
173 &sub ($W512,$W512_SZ);
174 &and ($W512,-16); # ensure 128-bit alignment
176 # make private copy of A-H
177 # v assume the worst and stick to unaligned load
178 &movdqu
("xmm0",&QWP
(0,$Widx));
179 &movdqu
("xmm1",&QWP
(16,$Widx));
180 &movdqu
("xmm2",&QWP
(32,$Widx));
181 &movdqu
("xmm3",&QWP
(48,$Widx));
184 &set_label
("_chunk_loop");
186 &movdqa
(&QWP
($Aoff,$W512),"xmm0"); # a,b
187 &movdqa
(&QWP
($Coff,$W512),"xmm1"); # c,d
188 &movdqa
(&QWP
($Eoff,$W512),"xmm2"); # e,f
189 &movdqa
(&QWP
($Goff,$W512),"xmm3"); # g,h
193 &movdq2q
($A,"xmm0"); # load a
194 &movdq2q
($E,"xmm2"); # load e
196 # Why aren't loops unrolled? It makes sense to unroll if
197 # execution time for loop body is comparable with branch
198 # penalties and/or if whole data-set resides in register bank.
199 # Neither is case here... Well, it would be possible to
200 # eliminate few store operations, but it would hardly affect
201 # so to say stop-watch performance, as there is a lot of
202 # available memory slots to fill. It will only relieve some
203 # pressure off memory bus...
205 # flip input stream byte order...
206 &mov
("eax",&DWP
(0,$data,$Widx,8));
207 &mov
("ebx",&DWP
(4,$data,$Widx,8));
210 &mov
(&DWP
(0,$W512,$Widx,8),"ebx"); # W512[i]
211 &mov
(&DWP
(4,$W512,$Widx,8),"eax");
212 &mov
(&DWP
(128+0,$W512,$Widx,8),"ebx"); # copy of W512[i]
213 &mov
(&DWP
(128+4,$W512,$Widx,8),"eax");
216 &set_label
("_1st_loop"); # 0-15
217 # flip input stream byte order...
218 &mov
("eax",&DWP
(0+8,$data,$Widx,8));
219 &mov
("ebx",&DWP
(4+8,$data,$Widx,8));
222 &mov
(&DWP
(0+8,$W512,$Widx,8),"ebx"); # W512[i]
223 &mov
(&DWP
(4+8,$W512,$Widx,8),"eax");
224 &mov
(&DWP
(128+0+8,$W512,$Widx,8),"ebx"); # copy of W512[i]
225 &mov
(&DWP
(128+4+8,$W512,$Widx,8),"eax");
226 &set_label
("_1st_looplet");
227 &SHA2_ROUND
($Widx,$Widx); &inc
($Widx);
230 &jl
(&label
("_1st_loop"));
231 &je
(&label
("_1st_looplet")); # playing similar trick on 2nd loop
232 # does not improve performance...
234 $Kidx = "ebx"; # start using %ebx as Kidx
238 &set_label
("_2nd_loop"); # 16-79
241 # 128-bit fragment! I update W512[i] and W512[i+1] in
242 # parallel:-) Note that I refer to W512[(i&0xf)+N] and not to
243 # W512[(i+N)&0xf]! This is exactly what I maintain the second
244 # copy of W512[16] for...
245 &movdqu
("xmm0",&QWP
(8*1,$W512,$Widx,8)); # s0=W512[i+1]
246 &movdqa
("xmm2","xmm0"); # %xmm2 is sliding right
247 &movdqa
("xmm3","xmm0"); # %xmm3 is sliding left
250 &movdqa
("xmm0","xmm2");
251 &pxor
("xmm0","xmm3");
254 &pxor
("xmm0","xmm2");
255 &pxor
("xmm0","xmm3");
257 &pxor
("xmm0","xmm2"); # s0 = sigma0_512(s0);
259 &movdqa
("xmm1",&QWP
(8*14,$W512,$Widx,8)); # s1=W512[i+14]
260 &movdqa
("xmm4","xmm1"); # %xmm4 is sliding right
261 &movdqa
("xmm5","xmm1"); # %xmm5 is sliding left
264 &movdqa
("xmm1","xmm4");
265 &pxor
("xmm1","xmm5");
268 &pxor
("xmm1","xmm4");
269 &pxor
("xmm1","xmm5");
271 &pxor
("xmm1","xmm4"); # s1 = sigma1_512(s1);
273 # + have to explictly load W512[i+9] as it's not 128-bit
274 # v aligned and paddq would throw an exception...
275 &movdqu
("xmm6",&QWP
(8*9,$W512,$Widx,8));
276 &paddq
("xmm0","xmm1"); # s0 += s1
277 &paddq
("xmm0","xmm6"); # s0 += W512[i+9]
278 &paddq
("xmm0",&QWP
(0,$W512,$Widx,8)); # s0 += W512[i]
280 &movdqa
(&QWP
(0,$W512,$Widx,8),"xmm0"); # W512[i] = s0
281 &movdqa
(&QWP
(16*8,$W512,$Widx,8),"xmm0"); # copy of W512[i]
283 # as the above fragment was 128-bit, we "owe" 2 rounds...
284 &SHA2_ROUND
($Kidx,$Widx); &inc
($Kidx); &inc
($Widx);
285 &SHA2_ROUND
($Kidx,$Widx); &inc
($Kidx); &inc
($Widx);
288 &jl
(&label
("_2nd_loop"));
291 &mov
($Widx,&DWP
(8,"ebp")); # A-H state, 1st arg
292 &movq
(&QWP
($Aoff,$W512),$A); # write out a
293 &movq
(&QWP
($Eoff,$W512),$E); # write out e
294 &movdqu
("xmm0",&QWP
(0,$Widx));
295 &movdqu
("xmm1",&QWP
(16,$Widx));
296 &movdqu
("xmm2",&QWP
(32,$Widx));
297 &movdqu
("xmm3",&QWP
(48,$Widx));
298 &paddq
("xmm0",&QWP
($Aoff,$W512)); # 128-bit additions...
299 &paddq
("xmm1",&QWP
($Coff,$W512));
300 &paddq
("xmm2",&QWP
($Eoff,$W512));
301 &paddq
("xmm3",&QWP
($Goff,$W512));
302 &movdqu
(&QWP
(0,$Widx),"xmm0");
303 &movdqu
(&QWP
(16,$Widx),"xmm1");
304 &movdqu
(&QWP
(32,$Widx),"xmm2");
305 &movdqu
(&QWP
(48,$Widx),"xmm3");
307 &add
($data,16*8); # advance input data pointer
308 &dec
(&DWP
(16,"ebp")); # decrement 3rd arg
309 &jnz
(&label
("_chunk_loop"));
312 &emms
(); # required for at least ELF and Win32 ABIs
313 &mov
("edi",&DWP
(-12,"ebp"));
314 &mov
("esi",&DWP
(-8,"ebp"));
315 &mov
("ebx",&DWP
(-4,"ebp"));
320 &set_label
("K512"); # Yes! I keep it in the code segment!
321 &data_word
(0xd728ae22,0x428a2f98); # u64
322 &data_word
(0x23ef65cd,0x71374491); # u64
323 &data_word
(0xec4d3b2f,0xb5c0fbcf); # u64
324 &data_word
(0x8189dbbc,0xe9b5dba5); # u64
325 &data_word
(0xf348b538,0x3956c25b); # u64
326 &data_word
(0xb605d019,0x59f111f1); # u64
327 &data_word
(0xaf194f9b,0x923f82a4); # u64
328 &data_word
(0xda6d8118,0xab1c5ed5); # u64
329 &data_word
(0xa3030242,0xd807aa98); # u64
330 &data_word
(0x45706fbe,0x12835b01); # u64
331 &data_word
(0x4ee4b28c,0x243185be); # u64
332 &data_word
(0xd5ffb4e2,0x550c7dc3); # u64
333 &data_word
(0xf27b896f,0x72be5d74); # u64
334 &data_word
(0x3b1696b1,0x80deb1fe); # u64
335 &data_word
(0x25c71235,0x9bdc06a7); # u64
336 &data_word
(0xcf692694,0xc19bf174); # u64
337 &data_word
(0x9ef14ad2,0xe49b69c1); # u64
338 &data_word
(0x384f25e3,0xefbe4786); # u64
339 &data_word
(0x8b8cd5b5,0x0fc19dc6); # u64
340 &data_word
(0x77ac9c65,0x240ca1cc); # u64
341 &data_word
(0x592b0275,0x2de92c6f); # u64
342 &data_word
(0x6ea6e483,0x4a7484aa); # u64
343 &data_word
(0xbd41fbd4,0x5cb0a9dc); # u64
344 &data_word
(0x831153b5,0x76f988da); # u64
345 &data_word
(0xee66dfab,0x983e5152); # u64
346 &data_word
(0x2db43210,0xa831c66d); # u64
347 &data_word
(0x98fb213f,0xb00327c8); # u64
348 &data_word
(0xbeef0ee4,0xbf597fc7); # u64
349 &data_word
(0x3da88fc2,0xc6e00bf3); # u64
350 &data_word
(0x930aa725,0xd5a79147); # u64
351 &data_word
(0xe003826f,0x06ca6351); # u64
352 &data_word
(0x0a0e6e70,0x14292967); # u64
353 &data_word
(0x46d22ffc,0x27b70a85); # u64
354 &data_word
(0x5c26c926,0x2e1b2138); # u64
355 &data_word
(0x5ac42aed,0x4d2c6dfc); # u64
356 &data_word
(0x9d95b3df,0x53380d13); # u64
357 &data_word
(0x8baf63de,0x650a7354); # u64
358 &data_word
(0x3c77b2a8,0x766a0abb); # u64
359 &data_word
(0x47edaee6,0x81c2c92e); # u64
360 &data_word
(0x1482353b,0x92722c85); # u64
361 &data_word
(0x4cf10364,0xa2bfe8a1); # u64
362 &data_word
(0xbc423001,0xa81a664b); # u64
363 &data_word
(0xd0f89791,0xc24b8b70); # u64
364 &data_word
(0x0654be30,0xc76c51a3); # u64
365 &data_word
(0xd6ef5218,0xd192e819); # u64
366 &data_word
(0x5565a910,0xd6990624); # u64
367 &data_word
(0x5771202a,0xf40e3585); # u64
368 &data_word
(0x32bbd1b8,0x106aa070); # u64
369 &data_word
(0xb8d2d0c8,0x19a4c116); # u64
370 &data_word
(0x5141ab53,0x1e376c08); # u64
371 &data_word
(0xdf8eeb99,0x2748774c); # u64
372 &data_word
(0xe19b48a8,0x34b0bcb5); # u64
373 &data_word
(0xc5c95a63,0x391c0cb3); # u64
374 &data_word
(0xe3418acb,0x4ed8aa4a); # u64
375 &data_word
(0x7763e373,0x5b9cca4f); # u64
376 &data_word
(0xd6b2b8a3,0x682e6ff3); # u64
377 &data_word
(0x5defb2fc,0x748f82ee); # u64
378 &data_word
(0x43172f60,0x78a5636f); # u64
379 &data_word
(0xa1f0ab72,0x84c87814); # u64
380 &data_word
(0x1a6439ec,0x8cc70208); # u64
381 &data_word
(0x23631e28,0x90befffa); # u64
382 &data_word
(0xde82bde9,0xa4506ceb); # u64
383 &data_word
(0xb2c67915,0xbef9a3f7); # u64
384 &data_word
(0xe372532b,0xc67178f2); # u64
385 &data_word
(0xea26619c,0xca273ece); # u64
386 &data_word
(0x21c0c207,0xd186b8c7); # u64
387 &data_word
(0xcde0eb1e,0xeada7dd6); # u64
388 &data_word
(0xee6ed178,0xf57d4f7f); # u64
389 &data_word
(0x72176fba,0x06f067aa); # u64
390 &data_word
(0xa2c898a6,0x0a637dc5); # u64
391 &data_word
(0xbef90dae,0x113f9804); # u64
392 &data_word
(0x131c471b,0x1b710b35); # u64
393 &data_word
(0x23047d84,0x28db77f5); # u64
394 &data_word
(0x40c72493,0x32caab7b); # u64
395 &data_word
(0x15c9bebc,0x3c9ebe0a); # u64
396 &data_word
(0x9c100d4c,0x431d67c4); # u64
397 &data_word
(0xcb3e42b6,0x4cc5d4be); # u64
398 &data_word
(0xfc657e2a,0x597f299c); # u64
399 &data_word
(0x3ad6faec,0x5fcb6fab); # u64
400 &data_word
(0x4a475817,0x6c44198c); # u64
402 &function_end_B
($func);