1 // Code generated by command: go run chacha20poly1305_amd64_asm.go -out ../chacha20poly1305_amd64.s -pkg chacha20poly1305. DO NOT EDIT.
2
3 //go:build gc && !purego
4
5 #include "textflag.h"
6
7 // func polyHashADInternal<>()
8 TEXT polyHashADInternal<>(SB), NOSPLIT, $0
9 XORQ R10, R10
10 XORQ R11, R11
11 XORQ R12, R12
12 CMPQ R9, $0x0d
13 JNE hashADLoop
14 MOVQ (CX), R10
15 MOVQ 5(CX), R11
16 SHRQ $0x18, R11
17 MOVQ $0x00000001, R12
18 MOVQ (BP), AX
19 MOVQ AX, R15
20 MULQ R10
21 MOVQ AX, R13
22 MOVQ DX, R14
23 MOVQ (BP), AX
24 MULQ R11
25 IMULQ R12, R15
26 ADDQ AX, R14
27 ADCQ DX, R15
28 MOVQ 8(BP), AX
29 MOVQ AX, R8
30 MULQ R10
31 ADDQ AX, R14
32 ADCQ $0x00, DX
33 MOVQ DX, R10
34 MOVQ 8(BP), AX
35 MULQ R11
36 ADDQ AX, R15
37 ADCQ $0x00, DX
38 IMULQ R12, R8
39 ADDQ R10, R15
40 ADCQ DX, R8
41 MOVQ R13, R10
42 MOVQ R14, R11
43 MOVQ R15, R12
44 ANDQ $0x03, R12
45 MOVQ R15, R13
46 ANDQ $-4, R13
47 MOVQ R8, R14
48 SHRQ $0x02, R8, R15
49 SHRQ $0x02, R8
50 ADDQ R13, R10
51 ADCQ R14, R11
52 ADCQ $0x00, R12
53 ADDQ R15, R10
54 ADCQ R8, R11
55 ADCQ $0x00, R12
56 RET
57
58 hashADLoop:
59 // Hash in 16 byte chunks
60 CMPQ R9, $0x10
61 JB hashADTail
62 ADDQ (CX), R10
63 ADCQ 8(CX), R11
64 ADCQ $0x01, R12
65 LEAQ 16(CX), CX
66 SUBQ $0x10, R9
67 MOVQ (BP), AX
68 MOVQ AX, R15
69 MULQ R10
70 MOVQ AX, R13
71 MOVQ DX, R14
72 MOVQ (BP), AX
73 MULQ R11
74 IMULQ R12, R15
75 ADDQ AX, R14
76 ADCQ DX, R15
77 MOVQ 8(BP), AX
78 MOVQ AX, R8
79 MULQ R10
80 ADDQ AX, R14
81 ADCQ $0x00, DX
82 MOVQ DX, R10
83 MOVQ 8(BP), AX
84 MULQ R11
85 ADDQ AX, R15
86 ADCQ $0x00, DX
87 IMULQ R12, R8
88 ADDQ R10, R15
89 ADCQ DX, R8
90 MOVQ R13, R10
91 MOVQ R14, R11
92 MOVQ R15, R12
93 ANDQ $0x03, R12
94 MOVQ R15, R13
95 ANDQ $-4, R13
96 MOVQ R8, R14
97 SHRQ $0x02, R8, R15
98 SHRQ $0x02, R8
99 ADDQ R13, R10
100 ADCQ R14, R11
101 ADCQ $0x00, R12
102 ADDQ R15, R10
103 ADCQ R8, R11
104 ADCQ $0x00, R12
105 JMP hashADLoop
106
107 hashADTail:
108 CMPQ R9, $0x00
109 JE hashADDone
110
111 // Hash last < 16 byte tail
112 XORQ R13, R13
113 XORQ R14, R14
114 XORQ R15, R15
115 ADDQ R9, CX
116
117 hashADTailLoop:
118 SHLQ $0x08, R13, R14
119 SHLQ $0x08, R13
120 MOVB -1(CX), R15
121 XORQ R15, R13
122 DECQ CX
123 DECQ R9
124 JNE hashADTailLoop
125 ADDQ R13, R10
126 ADCQ R14, R11
127 ADCQ $0x01, R12
128 MOVQ (BP), AX
129 MOVQ AX, R15
130 MULQ R10
131 MOVQ AX, R13
132 MOVQ DX, R14
133 MOVQ (BP), AX
134 MULQ R11
135 IMULQ R12, R15
136 ADDQ AX, R14
137 ADCQ DX, R15
138 MOVQ 8(BP), AX
139 MOVQ AX, R8
140 MULQ R10
141 ADDQ AX, R14
142 ADCQ $0x00, DX
143 MOVQ DX, R10
144 MOVQ 8(BP), AX
145 MULQ R11
146 ADDQ AX, R15
147 ADCQ $0x00, DX
148 IMULQ R12, R8
149 ADDQ R10, R15
150 ADCQ DX, R8
151 MOVQ R13, R10
152 MOVQ R14, R11
153 MOVQ R15, R12
154 ANDQ $0x03, R12
155 MOVQ R15, R13
156 ANDQ $-4, R13
157 MOVQ R8, R14
158 SHRQ $0x02, R8, R15
159 SHRQ $0x02, R8
160 ADDQ R13, R10
161 ADCQ R14, R11
162 ADCQ $0x00, R12
163 ADDQ R15, R10
164 ADCQ R8, R11
165 ADCQ $0x00, R12
166
167 hashADDone:
168 RET
169
170 // func chacha20Poly1305Open(dst []byte, key []uint32, src []byte, ad []byte) bool
171 // Requires: AVX, AVX2, BMI2, CMOV, SSE2
172 TEXT ·chacha20Poly1305Open(SB), $288-97
173 // For aligned stack access
174 MOVQ SP, BP
175 ADDQ $0x20, BP
176 ANDQ $-32, BP
177 MOVQ dst_base+0(FP), DI
178 MOVQ key_base+24(FP), R8
179 MOVQ src_base+48(FP), SI
180 MOVQ src_len+56(FP), BX
181 MOVQ ad_base+72(FP), CX
182 VZEROUPPER
183 VMOVDQU ·chacha20Constants<>+0(SB), Y0
184 VBROADCASTI128 16(R8), Y14
185 VBROADCASTI128 32(R8), Y12
186 VBROADCASTI128 48(R8), Y4
187 VPADDD ·avx2InitMask<>+0(SB), Y4, Y4
188
189 // Special optimization, for very short buffers
190 CMPQ BX, $0xc0
191 JBE openAVX2192
192 CMPQ BX, $0x00000140
193 JBE openAVX2320
194
195 // For the general key prepare the key first - as a byproduct we have 64 bytes of cipher stream
196 VMOVDQA Y14, 32(BP)
197 VMOVDQA Y12, 64(BP)
198 VMOVDQA Y4, 192(BP)
199 MOVQ $0x0000000a, R9
200
201 openAVX2PreparePolyKey:
202 VPADDD Y14, Y0, Y0
203 VPXOR Y0, Y4, Y4
204 VPSHUFB ·rol16<>+0(SB), Y4, Y4
205 VPADDD Y4, Y12, Y12
206 VPXOR Y12, Y14, Y14
207 VPSLLD $0x0c, Y14, Y3
208 VPSRLD $0x14, Y14, Y14
209 VPXOR Y3, Y14, Y14
210 VPADDD Y14, Y0, Y0
211 VPXOR Y0, Y4, Y4
212 VPSHUFB ·rol8<>+0(SB), Y4, Y4
213 VPADDD Y4, Y12, Y12
214 VPXOR Y12, Y14, Y14
215 VPSLLD $0x07, Y14, Y3
216 VPSRLD $0x19, Y14, Y14
217 VPXOR Y3, Y14, Y14
218 VPALIGNR $0x04, Y14, Y14, Y14
219 VPALIGNR $0x08, Y12, Y12, Y12
220 VPALIGNR $0x0c, Y4, Y4, Y4
221 VPADDD Y14, Y0, Y0
222 VPXOR Y0, Y4, Y4
223 VPSHUFB ·rol16<>+0(SB), Y4, Y4
224 VPADDD Y4, Y12, Y12
225 VPXOR Y12, Y14, Y14
226 VPSLLD $0x0c, Y14, Y3
227 VPSRLD $0x14, Y14, Y14
228 VPXOR Y3, Y14, Y14
229 VPADDD Y14, Y0, Y0
230 VPXOR Y0, Y4, Y4
231 VPSHUFB ·rol8<>+0(SB), Y4, Y4
232 VPADDD Y4, Y12, Y12
233 VPXOR Y12, Y14, Y14
234 VPSLLD $0x07, Y14, Y3
235 VPSRLD $0x19, Y14, Y14
236 VPXOR Y3, Y14, Y14
237 VPALIGNR $0x0c, Y14, Y14, Y14
238 VPALIGNR $0x08, Y12, Y12, Y12
239 VPALIGNR $0x04, Y4, Y4, Y4
240 DECQ R9
241 JNE openAVX2PreparePolyKey
242 VPADDD ·chacha20Constants<>+0(SB), Y0, Y0
243 VPADDD 32(BP), Y14, Y14
244 VPADDD 64(BP), Y12, Y12
245 VPADDD 192(BP), Y4, Y4
246 VPERM2I128 $0x02, Y0, Y14, Y3
247
248 // Clamp and store poly key
249 VPAND ·polyClampMask<>+0(SB), Y3, Y3
250 VMOVDQA Y3, (BP)
251
252 // Stream for the first 64 bytes
253 VPERM2I128 $0x13, Y0, Y14, Y0
254 VPERM2I128 $0x13, Y12, Y4, Y14
255
256 // Hash AD + first 64 bytes
257 MOVQ ad_len+80(FP), R9
258 CALL polyHashADInternal<>(SB)
259 XORQ CX, CX
260
261 openAVX2InitialHash64:
262 ADDQ (SI)(CX*1), R10
263 ADCQ 8(SI)(CX*1), R11
264 ADCQ $0x01, R12
265 MOVQ (BP), DX
266 MOVQ DX, R15
267 MULXQ R10, R13, R14
268 IMULQ R12, R15
269 MULXQ R11, AX, DX
270 ADDQ AX, R14
271 ADCQ DX, R15
272 MOVQ 8(BP), DX
273 MULXQ R10, R10, AX
274 ADDQ R10, R14
275 MULXQ R11, R11, R8
276 ADCQ R11, R15
277 ADCQ $0x00, R8
278 IMULQ R12, DX
279 ADDQ AX, R15
280 ADCQ DX, R8
281 MOVQ R13, R10
282 MOVQ R14, R11
283 MOVQ R15, R12
284 ANDQ $0x03, R12
285 MOVQ R15, R13
286 ANDQ $-4, R13
287 MOVQ R8, R14
288 SHRQ $0x02, R8, R15
289 SHRQ $0x02, R8
290 ADDQ R13, R10
291 ADCQ R14, R11
292 ADCQ $0x00, R12
293 ADDQ R15, R10
294 ADCQ R8, R11
295 ADCQ $0x00, R12
296 ADDQ $0x10, CX
297 CMPQ CX, $0x40
298 JNE openAVX2InitialHash64
299
300 // Decrypt the first 64 bytes
301 VPXOR (SI), Y0, Y0
302 VPXOR 32(SI), Y14, Y14
303 VMOVDQU Y0, (DI)
304 VMOVDQU Y14, 32(DI)
305 LEAQ 64(SI), SI
306 LEAQ 64(DI), DI
307 SUBQ $0x40, BX
308
309 openAVX2MainLoop:
310 CMPQ BX, $0x00000200
311 JB openAVX2MainLoopDone
312
313 // Load state, increment counter blocks, store the incremented counters
314 VMOVDQU ·chacha20Constants<>+0(SB), Y0
315 VMOVDQA Y0, Y5
316 VMOVDQA Y0, Y6
317 VMOVDQA Y0, Y7
318 VMOVDQA 32(BP), Y14
319 VMOVDQA Y14, Y9
320 VMOVDQA Y14, Y10
321 VMOVDQA Y14, Y11
322 VMOVDQA 64(BP), Y12
323 VMOVDQA Y12, Y13
324 VMOVDQA Y12, Y8
325 VMOVDQA Y12, Y15
326 VMOVDQA 192(BP), Y4
327 VPADDD ·avx2IncMask<>+0(SB), Y4, Y4
328 VPADDD ·avx2IncMask<>+0(SB), Y4, Y1
329 VPADDD ·avx2IncMask<>+0(SB), Y1, Y2
330 VPADDD ·avx2IncMask<>+0(SB), Y2, Y3
331 VMOVDQA Y4, 96(BP)
332 VMOVDQA Y1, 128(BP)
333 VMOVDQA Y2, 160(BP)
334 VMOVDQA Y3, 192(BP)
335 XORQ CX, CX
336
337 openAVX2InternalLoop:
338 ADDQ (SI)(CX*1), R10
339 ADCQ 8(SI)(CX*1), R11
340 ADCQ $0x01, R12
341 VPADDD Y14, Y0, Y0
342 VPADDD Y9, Y5, Y5
343 VPADDD Y10, Y6, Y6
344 VPADDD Y11, Y7, Y7
345 MOVQ (BP), DX
346 MOVQ DX, R15
347 MULXQ R10, R13, R14
348 IMULQ R12, R15
349 MULXQ R11, AX, DX
350 ADDQ AX, R14
351 ADCQ DX, R15
352 VPXOR Y0, Y4, Y4
353 VPXOR Y5, Y1, Y1
354 VPXOR Y6, Y2, Y2
355 VPXOR Y7, Y3, Y3
356 VPSHUFB ·rol16<>+0(SB), Y4, Y4
357 VPSHUFB ·rol16<>+0(SB), Y1, Y1
358 VPSHUFB ·rol16<>+0(SB), Y2, Y2
359 VPSHUFB ·rol16<>+0(SB), Y3, Y3
360 MOVQ 8(BP), DX
361 MULXQ R10, R10, AX
362 ADDQ R10, R14
363 MULXQ R11, R11, R8
364 ADCQ R11, R15
365 ADCQ $0x00, R8
366 VPADDD Y4, Y12, Y12
367 VPADDD Y1, Y13, Y13
368 VPADDD Y2, Y8, Y8
369 VPADDD Y3, Y15, Y15
370 VPXOR Y12, Y14, Y14
371 VPXOR Y13, Y9, Y9
372 VPXOR Y8, Y10, Y10
373 VPXOR Y15, Y11, Y11
374 IMULQ R12, DX
375 ADDQ AX, R15
376 ADCQ DX, R8
377 VMOVDQA Y15, 224(BP)
378 VPSLLD $0x0c, Y14, Y15
379 VPSRLD $0x14, Y14, Y14
380 VPXOR Y15, Y14, Y14
381 VPSLLD $0x0c, Y9, Y15
382 VPSRLD $0x14, Y9, Y9
383 VPXOR Y15, Y9, Y9
384 VPSLLD $0x0c, Y10, Y15
385 VPSRLD $0x14, Y10, Y10
386 VPXOR Y15, Y10, Y10
387 VPSLLD $0x0c, Y11, Y15
388 VPSRLD $0x14, Y11, Y11
389 VPXOR Y15, Y11, Y11
390 VMOVDQA 224(BP), Y15
391 MOVQ R13, R10
392 MOVQ R14, R11
393 MOVQ R15, R12
394 ANDQ $0x03, R12
395 MOVQ R15, R13
396 ANDQ $-4, R13
397 MOVQ R8, R14
398 SHRQ $0x02, R8, R15
399 SHRQ $0x02, R8
400 ADDQ R13, R10
401 ADCQ R14, R11
402 ADCQ $0x00, R12
403 ADDQ R15, R10
404 ADCQ R8, R11
405 ADCQ $0x00, R12
406 VPADDD Y14, Y0, Y0
407 VPADDD Y9, Y5, Y5
408 VPADDD Y10, Y6, Y6
409 VPADDD Y11, Y7, Y7
410 VPXOR Y0, Y4, Y4
411 VPXOR Y5, Y1, Y1
412 VPXOR Y6, Y2, Y2
413 VPXOR Y7, Y3, Y3
414 VPSHUFB ·rol8<>+0(SB), Y4, Y4
415 VPSHUFB ·rol8<>+0(SB), Y1, Y1
416 VPSHUFB ·rol8<>+0(SB), Y2, Y2
417 VPSHUFB ·rol8<>+0(SB), Y3, Y3
418 ADDQ 16(SI)(CX*1), R10
419 ADCQ 24(SI)(CX*1), R11
420 ADCQ $0x01, R12
421 VPADDD Y4, Y12, Y12
422 VPADDD Y1, Y13, Y13
423 VPADDD Y2, Y8, Y8
424 VPADDD Y3, Y15, Y15
425 MOVQ (BP), DX
426 MOVQ DX, R15
427 MULXQ R10, R13, R14
428 IMULQ R12, R15
429 MULXQ R11, AX, DX
430 ADDQ AX, R14
431 ADCQ DX, R15
432 VPXOR Y12, Y14, Y14
433 VPXOR Y13, Y9, Y9
434 VPXOR Y8, Y10, Y10
435 VPXOR Y15, Y11, Y11
436 VMOVDQA Y15, 224(BP)
437 VPSLLD $0x07, Y14, Y15
438 VPSRLD $0x19, Y14, Y14
439 VPXOR Y15, Y14, Y14
440 VPSLLD $0x07, Y9, Y15
441 VPSRLD $0x19, Y9, Y9
442 VPXOR Y15, Y9, Y9
443 VPSLLD $0x07, Y10, Y15
444 VPSRLD $0x19, Y10, Y10
445 VPXOR Y15, Y10, Y10
446 VPSLLD $0x07, Y11, Y15
447 VPSRLD $0x19, Y11, Y11
448 VPXOR Y15, Y11, Y11
449 VMOVDQA 224(BP), Y15
450 MOVQ 8(BP), DX
451 MULXQ R10, R10, AX
452 ADDQ R10, R14
453 MULXQ R11, R11, R8
454 ADCQ R11, R15
455 ADCQ $0x00, R8
456 VPALIGNR $0x04, Y14, Y14, Y14
457 VPALIGNR $0x04, Y9, Y9, Y9
458 VPALIGNR $0x04, Y10, Y10, Y10
459 VPALIGNR $0x04, Y11, Y11, Y11
460 VPALIGNR $0x08, Y12, Y12, Y12
461 VPALIGNR $0x08, Y13, Y13, Y13
462 VPALIGNR $0x08, Y8, Y8, Y8
463 VPALIGNR $0x08, Y15, Y15, Y15
464 VPALIGNR $0x0c, Y4, Y4, Y4
465 VPALIGNR $0x0c, Y1, Y1, Y1
466 VPALIGNR $0x0c, Y2, Y2, Y2
467 VPALIGNR $0x0c, Y3, Y3, Y3
468 VPADDD Y14, Y0, Y0
469 VPADDD Y9, Y5, Y5
470 VPADDD Y10, Y6, Y6
471 VPADDD Y11, Y7, Y7
472 IMULQ R12, DX
473 ADDQ AX, R15
474 ADCQ DX, R8
475 VPXOR Y0, Y4, Y4
476 VPXOR Y5, Y1, Y1
477 VPXOR Y6, Y2, Y2
478 VPXOR Y7, Y3, Y3
479 VPSHUFB ·rol16<>+0(SB), Y4, Y4
480 VPSHUFB ·rol16<>+0(SB), Y1, Y1
481 VPSHUFB ·rol16<>+0(SB), Y2, Y2
482 VPSHUFB ·rol16<>+0(SB), Y3, Y3
483 MOVQ R13, R10
484 MOVQ R14, R11
485 MOVQ R15, R12
486 ANDQ $0x03, R12
487 MOVQ R15, R13
488 ANDQ $-4, R13
489 MOVQ R8, R14
490 SHRQ $0x02, R8, R15
491 SHRQ $0x02, R8
492 ADDQ R13, R10
493 ADCQ R14, R11
494 ADCQ $0x00, R12
495 ADDQ R15, R10
496 ADCQ R8, R11
497 ADCQ $0x00, R12
498 VPADDD Y4, Y12, Y12
499 VPADDD Y1, Y13, Y13
500 VPADDD Y2, Y8, Y8
501 VPADDD Y3, Y15, Y15
502 VPXOR Y12, Y14, Y14
503 VPXOR Y13, Y9, Y9
504 VPXOR Y8, Y10, Y10
505 VPXOR Y15, Y11, Y11
506 ADDQ 32(SI)(CX*1), R10
507 ADCQ 40(SI)(CX*1), R11
508 ADCQ $0x01, R12
509 LEAQ 48(CX), CX
510 VMOVDQA Y15, 224(BP)
511 VPSLLD $0x0c, Y14, Y15
512 VPSRLD $0x14, Y14, Y14
513 VPXOR Y15, Y14, Y14
514 VPSLLD $0x0c, Y9, Y15
515 VPSRLD $0x14, Y9, Y9
516 VPXOR Y15, Y9, Y9
517 VPSLLD $0x0c, Y10, Y15
518 VPSRLD $0x14, Y10, Y10
519 VPXOR Y15, Y10, Y10
520 VPSLLD $0x0c, Y11, Y15
521 VPSRLD $0x14, Y11, Y11
522 VPXOR Y15, Y11, Y11
523 VMOVDQA 224(BP), Y15
524 MOVQ (BP), DX
525 MOVQ DX, R15
526 MULXQ R10, R13, R14
527 IMULQ R12, R15
528 MULXQ R11, AX, DX
529 ADDQ AX, R14
530 ADCQ DX, R15
531 VPADDD Y14, Y0, Y0
532 VPADDD Y9, Y5, Y5
533 VPADDD Y10, Y6, Y6
534 VPADDD Y11, Y7, Y7
535 VPXOR Y0, Y4, Y4
536 VPXOR Y5, Y1, Y1
537 VPXOR Y6, Y2, Y2
538 VPXOR Y7, Y3, Y3
539 MOVQ 8(BP), DX
540 MULXQ R10, R10, AX
541 ADDQ R10, R14
542 MULXQ R11, R11, R8
543 ADCQ R11, R15
544 ADCQ $0x00, R8
545 VPSHUFB ·rol8<>+0(SB), Y4, Y4
546 VPSHUFB ·rol8<>+0(SB), Y1, Y1
547 VPSHUFB ·rol8<>+0(SB), Y2, Y2
548 VPSHUFB ·rol8<>+0(SB), Y3, Y3
549 VPADDD Y4, Y12, Y12
550 VPADDD Y1, Y13, Y13
551 VPADDD Y2, Y8, Y8
552 VPADDD Y3, Y15, Y15
553 IMULQ R12, DX
554 ADDQ AX, R15
555 ADCQ DX, R8
556 VPXOR Y12, Y14, Y14
557 VPXOR Y13, Y9, Y9
558 VPXOR Y8, Y10, Y10
559 VPXOR Y15, Y11, Y11
560 VMOVDQA Y15, 224(BP)
561 VPSLLD $0x07, Y14, Y15
562 VPSRLD $0x19, Y14, Y14
563 VPXOR Y15, Y14, Y14
564 VPSLLD $0x07, Y9, Y15
565 VPSRLD $0x19, Y9, Y9
566 VPXOR Y15, Y9, Y9
567 VPSLLD $0x07, Y10, Y15
568 VPSRLD $0x19, Y10, Y10
569 VPXOR Y15, Y10, Y10
570 VPSLLD $0x07, Y11, Y15
571 VPSRLD $0x19, Y11, Y11
572 VPXOR Y15, Y11, Y11
573 VMOVDQA 224(BP), Y15
574 MOVQ R13, R10
575 MOVQ R14, R11
576 MOVQ R15, R12
577 ANDQ $0x03, R12
578 MOVQ R15, R13
579 ANDQ $-4, R13
580 MOVQ R8, R14
581 SHRQ $0x02, R8, R15
582 SHRQ $0x02, R8
583 ADDQ R13, R10
584 ADCQ R14, R11
585 ADCQ $0x00, R12
586 ADDQ R15, R10
587 ADCQ R8, R11
588 ADCQ $0x00, R12
589 VPALIGNR $0x0c, Y14, Y14, Y14
590 VPALIGNR $0x0c, Y9, Y9, Y9
591 VPALIGNR $0x0c, Y10, Y10, Y10
592 VPALIGNR $0x0c, Y11, Y11, Y11
593 VPALIGNR $0x08, Y12, Y12, Y12
594 VPALIGNR $0x08, Y13, Y13, Y13
595 VPALIGNR $0x08, Y8, Y8, Y8
596 VPALIGNR $0x08, Y15, Y15, Y15
597 VPALIGNR $0x04, Y4, Y4, Y4
598 VPALIGNR $0x04, Y1, Y1, Y1
599 VPALIGNR $0x04, Y2, Y2, Y2
600 VPALIGNR $0x04, Y3, Y3, Y3
601 CMPQ CX, $0x000001e0
602 JNE openAVX2InternalLoop
603 VPADDD ·chacha20Constants<>+0(SB), Y0, Y0
604 VPADDD ·chacha20Constants<>+0(SB), Y5, Y5
605 VPADDD ·chacha20Constants<>+0(SB), Y6, Y6
606 VPADDD ·chacha20Constants<>+0(SB), Y7, Y7
607 VPADDD 32(BP), Y14, Y14
608 VPADDD 32(BP), Y9, Y9
609 VPADDD 32(BP), Y10, Y10
610 VPADDD 32(BP), Y11, Y11
611 VPADDD 64(BP), Y12, Y12
612 VPADDD 64(BP), Y13, Y13
613 VPADDD 64(BP), Y8, Y8
614 VPADDD 64(BP), Y15, Y15
615 VPADDD 96(BP), Y4, Y4
616 VPADDD 128(BP), Y1, Y1
617 VPADDD 160(BP), Y2, Y2
618 VPADDD 192(BP), Y3, Y3
619 VMOVDQA Y15, 224(BP)
620
621 // We only hashed 480 of the 512 bytes available - hash the remaining 32 here
622 ADDQ 480(SI), R10
623 ADCQ 488(SI), R11
624 ADCQ $0x01, R12
625 MOVQ (BP), DX
626 MOVQ DX, R15
627 MULXQ R10, R13, R14
628 IMULQ R12, R15
629 MULXQ R11, AX, DX
630 ADDQ AX, R14
631 ADCQ DX, R15
632 MOVQ 8(BP), DX
633 MULXQ R10, R10, AX
634 ADDQ R10, R14
635 MULXQ R11, R11, R8
636 ADCQ R11, R15
637 ADCQ $0x00, R8
638 IMULQ R12, DX
639 ADDQ AX, R15
640 ADCQ DX, R8
641 MOVQ R13, R10
642 MOVQ R14, R11
643 MOVQ R15, R12
644 ANDQ $0x03, R12
645 MOVQ R15, R13
646 ANDQ $-4, R13
647 MOVQ R8, R14
648 SHRQ $0x02, R8, R15
649 SHRQ $0x02, R8
650 ADDQ R13, R10
651 ADCQ R14, R11
652 ADCQ $0x00, R12
653 ADDQ R15, R10
654 ADCQ R8, R11
655 ADCQ $0x00, R12
656 VPERM2I128 $0x02, Y0, Y14, Y15
657 VPERM2I128 $0x13, Y0, Y14, Y14
658 VPERM2I128 $0x02, Y12, Y4, Y0
659 VPERM2I128 $0x13, Y12, Y4, Y12
660 VPXOR (SI), Y15, Y15
661 VPXOR 32(SI), Y0, Y0
662 VPXOR 64(SI), Y14, Y14
663 VPXOR 96(SI), Y12, Y12
664 VMOVDQU Y15, (DI)
665 VMOVDQU Y0, 32(DI)
666 VMOVDQU Y14, 64(DI)
667 VMOVDQU Y12, 96(DI)
668 VPERM2I128 $0x02, Y5, Y9, Y0
669 VPERM2I128 $0x02, Y13, Y1, Y14
670 VPERM2I128 $0x13, Y5, Y9, Y12
671 VPERM2I128 $0x13, Y13, Y1, Y4
672 VPXOR 128(SI), Y0, Y0
673 VPXOR 160(SI), Y14, Y14
674 VPXOR 192(SI), Y12, Y12
675 VPXOR 224(SI), Y4, Y4
676 VMOVDQU Y0, 128(DI)
677 VMOVDQU Y14, 160(DI)
678 VMOVDQU Y12, 192(DI)
679 VMOVDQU Y4, 224(DI)
680
681 // and here
682 ADDQ 496(SI), R10
683 ADCQ 504(SI), R11
684 ADCQ $0x01, R12
685 MOVQ (BP), DX
686 MOVQ DX, R15
687 MULXQ R10, R13, R14
688 IMULQ R12, R15
689 MULXQ R11, AX, DX
690 ADDQ AX, R14
691 ADCQ DX, R15
692 MOVQ 8(BP), DX
693 MULXQ R10, R10, AX
694 ADDQ R10, R14
695 MULXQ R11, R11, R8
696 ADCQ R11, R15
697 ADCQ $0x00, R8
698 IMULQ R12, DX
699 ADDQ AX, R15
700 ADCQ DX, R8
701 MOVQ R13, R10
702 MOVQ R14, R11
703 MOVQ R15, R12
704 ANDQ $0x03, R12
705 MOVQ R15, R13
706 ANDQ $-4, R13
707 MOVQ R8, R14
708 SHRQ $0x02, R8, R15
709 SHRQ $0x02, R8
710 ADDQ R13, R10
711 ADCQ R14, R11
712 ADCQ $0x00, R12
713 ADDQ R15, R10
714 ADCQ R8, R11
715 ADCQ $0x00, R12
716 VPERM2I128 $0x02, Y6, Y10, Y0
717 VPERM2I128 $0x02, Y8, Y2, Y14
718 VPERM2I128 $0x13, Y6, Y10, Y12
719 VPERM2I128 $0x13, Y8, Y2, Y4
720 VPXOR 256(SI), Y0, Y0
721 VPXOR 288(SI), Y14, Y14
722 VPXOR 320(SI), Y12, Y12
723 VPXOR 352(SI), Y4, Y4
724 VMOVDQU Y0, 256(DI)
725 VMOVDQU Y14, 288(DI)
726 VMOVDQU Y12, 320(DI)
727 VMOVDQU Y4, 352(DI)
728 VPERM2I128 $0x02, Y7, Y11, Y0
729 VPERM2I128 $0x02, 224(BP), Y3, Y14
730 VPERM2I128 $0x13, Y7, Y11, Y12
731 VPERM2I128 $0x13, 224(BP), Y3, Y4
732 VPXOR 384(SI), Y0, Y0
733 VPXOR 416(SI), Y14, Y14
734 VPXOR 448(SI), Y12, Y12
735 VPXOR 480(SI), Y4, Y4
736 VMOVDQU Y0, 384(DI)
737 VMOVDQU Y14, 416(DI)
738 VMOVDQU Y12, 448(DI)
739 VMOVDQU Y4, 480(DI)
740 LEAQ 512(SI), SI
741 LEAQ 512(DI), DI
742 SUBQ $0x00000200, BX
743 JMP openAVX2MainLoop
744
745 openAVX2MainLoopDone:
746 // Handle the various tail sizes efficiently
747 TESTQ BX, BX
748 JE openSSEFinalize
749 CMPQ BX, $0x80
750 JBE openAVX2Tail128
751 CMPQ BX, $0x00000100
752 JBE openAVX2Tail256
753 CMPQ BX, $0x00000180
754 JBE openAVX2Tail384
755 JMP openAVX2Tail512
756
757 openSSEFinalize:
758 // Hash in the PT, AAD lengths
759 ADDQ ad_len+80(FP), R10
760 ADCQ src_len+56(FP), R11
761 ADCQ $0x01, R12
762 MOVQ (BP), AX
763 MOVQ AX, R15
764 MULQ R10
765 MOVQ AX, R13
766 MOVQ DX, R14
767 MOVQ (BP), AX
768 MULQ R11
769 IMULQ R12, R15
770 ADDQ AX, R14
771 ADCQ DX, R15
772 MOVQ 8(BP), AX
773 MOVQ AX, R8
774 MULQ R10
775 ADDQ AX, R14
776 ADCQ $0x00, DX
777 MOVQ DX, R10
778 MOVQ 8(BP), AX
779 MULQ R11
780 ADDQ AX, R15
781 ADCQ $0x00, DX
782 IMULQ R12, R8
783 ADDQ R10, R15
784 ADCQ DX, R8
785 MOVQ R13, R10
786 MOVQ R14, R11
787 MOVQ R15, R12
788 ANDQ $0x03, R12
789 MOVQ R15, R13
790 ANDQ $-4, R13
791 MOVQ R8, R14
792 SHRQ $0x02, R8, R15
793 SHRQ $0x02, R8
794 ADDQ R13, R10
795 ADCQ R14, R11
796 ADCQ $0x00, R12
797 ADDQ R15, R10
798 ADCQ R8, R11
799 ADCQ $0x00, R12
800
801 // Final reduce
802 MOVQ R10, R13
803 MOVQ R11, R14
804 MOVQ R12, R15
805 SUBQ $-5, R10
806 SBBQ $-1, R11
807 SBBQ $0x03, R12
808 CMOVQCS R13, R10
809 CMOVQCS R14, R11
810 CMOVQCS R15, R12
811
812 // Add in the "s" part of the key
813 ADDQ 16(BP), R10
814 ADCQ 24(BP), R11
815
816 // Finally, constant time compare to the tag at the end of the message
817 XORQ AX, AX
818 MOVQ $0x00000001, DX
819 XORQ (SI), R10
820 XORQ 8(SI), R11
821 ORQ R11, R10
822 CMOVQEQ DX, AX
823
824 // Return true iff tags are equal
825 MOVB AX, ret+96(FP)
826 RET
827
828 openSSETail16:
829 TESTQ BX, BX
830 JE openSSEFinalize
831
832 // We can safely load the CT from the end, because it is padded with the MAC
833 MOVQ BX, R9
834 SHLQ $0x04, R9
835 LEAQ ·andMask<>+0(SB), R13
836 MOVOU (SI), X12
837 ADDQ BX, SI
838 PAND -16(R13)(R9*1), X12
839 MOVO X12, 64(BP)
840 MOVQ X12, R13
841 MOVQ 72(BP), R14
842 PXOR X1, X12
843
844 // We can only store one byte at a time, since plaintext can be shorter than 16 bytes
845 openSSETail16Store:
846 MOVQ X12, R8
847 MOVB R8, (DI)
848 PSRLDQ $0x01, X12
849 INCQ DI
850 DECQ BX
851 JNE openSSETail16Store
852 ADDQ R13, R10
853 ADCQ R14, R11
854 ADCQ $0x01, R12
855 MOVQ (BP), AX
856 MOVQ AX, R15
857 MULQ R10
858 MOVQ AX, R13
859 MOVQ DX, R14
860 MOVQ (BP), AX
861 MULQ R11
862 IMULQ R12, R15
863 ADDQ AX, R14
864 ADCQ DX, R15
865 MOVQ 8(BP), AX
866 MOVQ AX, R8
867 MULQ R10
868 ADDQ AX, R14
869 ADCQ $0x00, DX
870 MOVQ DX, R10
871 MOVQ 8(BP), AX
872 MULQ R11
873 ADDQ AX, R15
874 ADCQ $0x00, DX
875 IMULQ R12, R8
876 ADDQ R10, R15
877 ADCQ DX, R8
878 MOVQ R13, R10
879 MOVQ R14, R11
880 MOVQ R15, R12
881 ANDQ $0x03, R12
882 MOVQ R15, R13
883 ANDQ $-4, R13
884 MOVQ R8, R14
885 SHRQ $0x02, R8, R15
886 SHRQ $0x02, R8
887 ADDQ R13, R10
888 ADCQ R14, R11
889 ADCQ $0x00, R12
890 ADDQ R15, R10
891 ADCQ R8, R11
892 ADCQ $0x00, R12
893 JMP openSSEFinalize
894
895 openAVX2192:
896 VMOVDQA Y0, Y5
897 VMOVDQA Y14, Y9
898 VMOVDQA Y12, Y13
899 VPADDD ·avx2IncMask<>+0(SB), Y4, Y1
900 VMOVDQA Y0, Y6
901 VMOVDQA Y14, Y10
902 VMOVDQA Y12, Y8
903 VMOVDQA Y4, Y2
904 VMOVDQA Y1, Y15
905 MOVQ $0x0000000a, R9
906
907 openAVX2192InnerCipherLoop:
908 VPADDD Y14, Y0, Y0
909 VPXOR Y0, Y4, Y4
910 VPSHUFB ·rol16<>+0(SB), Y4, Y4
911 VPADDD Y4, Y12, Y12
912 VPXOR Y12, Y14, Y14
913 VPSLLD $0x0c, Y14, Y3
914 VPSRLD $0x14, Y14, Y14
915 VPXOR Y3, Y14, Y14
916 VPADDD Y14, Y0, Y0
917 VPXOR Y0, Y4, Y4
918 VPSHUFB ·rol8<>+0(SB), Y4, Y4
919 VPADDD Y4, Y12, Y12
920 VPXOR Y12, Y14, Y14
921 VPSLLD $0x07, Y14, Y3
922 VPSRLD $0x19, Y14, Y14
923 VPXOR Y3, Y14, Y14
924 VPADDD Y9, Y5, Y5
925 VPXOR Y5, Y1, Y1
926 VPSHUFB ·rol16<>+0(SB), Y1, Y1
927 VPADDD Y1, Y13, Y13
928 VPXOR Y13, Y9, Y9
929 VPSLLD $0x0c, Y9, Y3
930 VPSRLD $0x14, Y9, Y9
931 VPXOR Y3, Y9, Y9
932 VPADDD Y9, Y5, Y5
933 VPXOR Y5, Y1, Y1
934 VPSHUFB ·rol8<>+0(SB), Y1, Y1
935 VPADDD Y1, Y13, Y13
936 VPXOR Y13, Y9, Y9
937 VPSLLD $0x07, Y9, Y3
938 VPSRLD $0x19, Y9, Y9
939 VPXOR Y3, Y9, Y9
940 VPALIGNR $0x04, Y14, Y14, Y14
941 VPALIGNR $0x04, Y9, Y9, Y9
942 VPALIGNR $0x08, Y12, Y12, Y12
943 VPALIGNR $0x08, Y13, Y13, Y13
944 VPALIGNR $0x0c, Y4, Y4, Y4
945 VPALIGNR $0x0c, Y1, Y1, Y1
946 VPADDD Y14, Y0, Y0
947 VPXOR Y0, Y4, Y4
948 VPSHUFB ·rol16<>+0(SB), Y4, Y4
949 VPADDD Y4, Y12, Y12
950 VPXOR Y12, Y14, Y14
951 VPSLLD $0x0c, Y14, Y3
952 VPSRLD $0x14, Y14, Y14
953 VPXOR Y3, Y14, Y14
954 VPADDD Y14, Y0, Y0
955 VPXOR Y0, Y4, Y4
956 VPSHUFB ·rol8<>+0(SB), Y4, Y4
957 VPADDD Y4, Y12, Y12
958 VPXOR Y12, Y14, Y14
959 VPSLLD $0x07, Y14, Y3
960 VPSRLD $0x19, Y14, Y14
961 VPXOR Y3, Y14, Y14
962 VPADDD Y9, Y5, Y5
963 VPXOR Y5, Y1, Y1
964 VPSHUFB ·rol16<>+0(SB), Y1, Y1
965 VPADDD Y1, Y13, Y13
966 VPXOR Y13, Y9, Y9
967 VPSLLD $0x0c, Y9, Y3
968 VPSRLD $0x14, Y9, Y9
969 VPXOR Y3, Y9, Y9
970 VPADDD Y9, Y5, Y5
971 VPXOR Y5, Y1, Y1
972 VPSHUFB ·rol8<>+0(SB), Y1, Y1
973 VPADDD Y1, Y13, Y13
974 VPXOR Y13, Y9, Y9
975 VPSLLD $0x07, Y9, Y3
976 VPSRLD $0x19, Y9, Y9
977 VPXOR Y3, Y9, Y9
978 VPALIGNR $0x0c, Y14, Y14, Y14
979 VPALIGNR $0x0c, Y9, Y9, Y9
980 VPALIGNR $0x08, Y12, Y12, Y12
981 VPALIGNR $0x08, Y13, Y13, Y13
982 VPALIGNR $0x04, Y4, Y4, Y4
983 VPALIGNR $0x04, Y1, Y1, Y1
984 DECQ R9
985 JNE openAVX2192InnerCipherLoop
986 VPADDD Y6, Y0, Y0
987 VPADDD Y6, Y5, Y5
988 VPADDD Y10, Y14, Y14
989 VPADDD Y10, Y9, Y9
990 VPADDD Y8, Y12, Y12
991 VPADDD Y8, Y13, Y13
992 VPADDD Y2, Y4, Y4
993 VPADDD Y15, Y1, Y1
994 VPERM2I128 $0x02, Y0, Y14, Y3
995
996 // Clamp and store poly key
997 VPAND ·polyClampMask<>+0(SB), Y3, Y3
998 VMOVDQA Y3, (BP)
999
1000 // Stream for up to 192 bytes
1001 VPERM2I128 $0x13, Y0, Y14, Y0
1002 VPERM2I128 $0x13, Y12, Y4, Y14
1003 VPERM2I128 $0x02, Y5, Y9, Y12
1004 VPERM2I128 $0x02, Y13, Y1, Y4
1005 VPERM2I128 $0x13, Y5, Y9, Y5
1006 VPERM2I128 $0x13, Y13, Y1, Y9
1007
1008 openAVX2ShortOpen:
1009 // Hash
1010 MOVQ ad_len+80(FP), R9
1011 CALL polyHashADInternal<>(SB)
1012
1013 openAVX2ShortOpenLoop:
1014 CMPQ BX, $0x20
1015 JB openAVX2ShortTail32
1016 SUBQ $0x20, BX
1017
1018 // Load for hashing
1019 ADDQ (SI), R10
1020 ADCQ 8(SI), R11
1021 ADCQ $0x01, R12
1022 MOVQ (BP), DX
1023 MOVQ DX, R15
1024 MULXQ R10, R13, R14
1025 IMULQ R12, R15
1026 MULXQ R11, AX, DX
1027 ADDQ AX, R14
1028 ADCQ DX, R15
1029 MOVQ 8(BP), DX
1030 MULXQ R10, R10, AX
1031 ADDQ R10, R14
1032 MULXQ R11, R11, R8
1033 ADCQ R11, R15
1034 ADCQ $0x00, R8
1035 IMULQ R12, DX
1036 ADDQ AX, R15
1037 ADCQ DX, R8
1038 MOVQ R13, R10
1039 MOVQ R14, R11
1040 MOVQ R15, R12
1041 ANDQ $0x03, R12
1042 MOVQ R15, R13
1043 ANDQ $-4, R13
1044 MOVQ R8, R14
1045 SHRQ $0x02, R8, R15
1046 SHRQ $0x02, R8
1047 ADDQ R13, R10
1048 ADCQ R14, R11
1049 ADCQ $0x00, R12
1050 ADDQ R15, R10
1051 ADCQ R8, R11
1052 ADCQ $0x00, R12
1053 ADDQ 16(SI), R10
1054 ADCQ 24(SI), R11
1055 ADCQ $0x01, R12
1056 MOVQ (BP), DX
1057 MOVQ DX, R15
1058 MULXQ R10, R13, R14
1059 IMULQ R12, R15
1060 MULXQ R11, AX, DX
1061 ADDQ AX, R14
1062 ADCQ DX, R15
1063 MOVQ 8(BP), DX
1064 MULXQ R10, R10, AX
1065 ADDQ R10, R14
1066 MULXQ R11, R11, R8
1067 ADCQ R11, R15
1068 ADCQ $0x00, R8
1069 IMULQ R12, DX
1070 ADDQ AX, R15
1071 ADCQ DX, R8
1072 MOVQ R13, R10
1073 MOVQ R14, R11
1074 MOVQ R15, R12
1075 ANDQ $0x03, R12
1076 MOVQ R15, R13
1077 ANDQ $-4, R13
1078 MOVQ R8, R14
1079 SHRQ $0x02, R8, R15
1080 SHRQ $0x02, R8
1081 ADDQ R13, R10
1082 ADCQ R14, R11
1083 ADCQ $0x00, R12
1084 ADDQ R15, R10
1085 ADCQ R8, R11
1086 ADCQ $0x00, R12
1087
1088 // Load for decryption
1089 VPXOR (SI), Y0, Y0
1090 VMOVDQU Y0, (DI)
1091 LEAQ 32(SI), SI
1092 LEAQ 32(DI), DI
1093
1094 // Shift stream left
1095 VMOVDQA Y14, Y0
1096 VMOVDQA Y12, Y14
1097 VMOVDQA Y4, Y12
1098 VMOVDQA Y5, Y4
1099 VMOVDQA Y9, Y5
1100 VMOVDQA Y13, Y9
1101 VMOVDQA Y1, Y13
1102 VMOVDQA Y6, Y1
1103 VMOVDQA Y10, Y6
1104 JMP openAVX2ShortOpenLoop
1105
1106 openAVX2ShortTail32:
1107 CMPQ BX, $0x10
1108 VMOVDQA X0, X1
1109 JB openAVX2ShortDone
1110 SUBQ $0x10, BX
1111
1112 // Load for hashing
1113 ADDQ (SI), R10
1114 ADCQ 8(SI), R11
1115 ADCQ $0x01, R12
1116 MOVQ (BP), DX
1117 MOVQ DX, R15
1118 MULXQ R10, R13, R14
1119 IMULQ R12, R15
1120 MULXQ R11, AX, DX
1121 ADDQ AX, R14
1122 ADCQ DX, R15
1123 MOVQ 8(BP), DX
1124 MULXQ R10, R10, AX
1125 ADDQ R10, R14
1126 MULXQ R11, R11, R8
1127 ADCQ R11, R15
1128 ADCQ $0x00, R8
1129 IMULQ R12, DX
1130 ADDQ AX, R15
1131 ADCQ DX, R8
1132 MOVQ R13, R10
1133 MOVQ R14, R11
1134 MOVQ R15, R12
1135 ANDQ $0x03, R12
1136 MOVQ R15, R13
1137 ANDQ $-4, R13
1138 MOVQ R8, R14
1139 SHRQ $0x02, R8, R15
1140 SHRQ $0x02, R8
1141 ADDQ R13, R10
1142 ADCQ R14, R11
1143 ADCQ $0x00, R12
1144 ADDQ R15, R10
1145 ADCQ R8, R11
1146 ADCQ $0x00, R12
1147
1148 // Load for decryption
1149 VPXOR (SI), X0, X12
1150 VMOVDQU X12, (DI)
1151 LEAQ 16(SI), SI
1152 LEAQ 16(DI), DI
1153 VPERM2I128 $0x11, Y0, Y0, Y0
1154 VMOVDQA X0, X1
1155
1156 openAVX2ShortDone:
1157 VZEROUPPER
1158 JMP openSSETail16
1159
1160 openAVX2320:
1161 VMOVDQA Y0, Y5
1162 VMOVDQA Y14, Y9
1163 VMOVDQA Y12, Y13
1164 VPADDD ·avx2IncMask<>+0(SB), Y4, Y1
1165 VMOVDQA Y0, Y6
1166 VMOVDQA Y14, Y10
1167 VMOVDQA Y12, Y8
1168 VPADDD ·avx2IncMask<>+0(SB), Y1, Y2
1169 VMOVDQA Y14, Y7
1170 VMOVDQA Y12, Y11
1171 VMOVDQA Y4, Y15
1172 MOVQ $0x0000000a, R9
1173
1174 openAVX2320InnerCipherLoop:
1175 VPADDD Y14, Y0, Y0
1176 VPXOR Y0, Y4, Y4
1177 VPSHUFB ·rol16<>+0(SB), Y4, Y4
1178 VPADDD Y4, Y12, Y12
1179 VPXOR Y12, Y14, Y14
1180 VPSLLD $0x0c, Y14, Y3
1181 VPSRLD $0x14, Y14, Y14
1182 VPXOR Y3, Y14, Y14
1183 VPADDD Y14, Y0, Y0
1184 VPXOR Y0, Y4, Y4
1185 VPSHUFB ·rol8<>+0(SB), Y4, Y4
1186 VPADDD Y4, Y12, Y12
1187 VPXOR Y12, Y14, Y14
1188 VPSLLD $0x07, Y14, Y3
1189 VPSRLD $0x19, Y14, Y14
1190 VPXOR Y3, Y14, Y14
1191 VPADDD Y9, Y5, Y5
1192 VPXOR Y5, Y1, Y1
1193 VPSHUFB ·rol16<>+0(SB), Y1, Y1
1194 VPADDD Y1, Y13, Y13
1195 VPXOR Y13, Y9, Y9
1196 VPSLLD $0x0c, Y9, Y3
1197 VPSRLD $0x14, Y9, Y9
1198 VPXOR Y3, Y9, Y9
1199 VPADDD Y9, Y5, Y5
1200 VPXOR Y5, Y1, Y1
1201 VPSHUFB ·rol8<>+0(SB), Y1, Y1
1202 VPADDD Y1, Y13, Y13
1203 VPXOR Y13, Y9, Y9
1204 VPSLLD $0x07, Y9, Y3
1205 VPSRLD $0x19, Y9, Y9
1206 VPXOR Y3, Y9, Y9
1207 VPADDD Y10, Y6, Y6
1208 VPXOR Y6, Y2, Y2
1209 VPSHUFB ·rol16<>+0(SB), Y2, Y2
1210 VPADDD Y2, Y8, Y8
1211 VPXOR Y8, Y10, Y10
1212 VPSLLD $0x0c, Y10, Y3
1213 VPSRLD $0x14, Y10, Y10
1214 VPXOR Y3, Y10, Y10
1215 VPADDD Y10, Y6, Y6
1216 VPXOR Y6, Y2, Y2
1217 VPSHUFB ·rol8<>+0(SB), Y2, Y2
1218 VPADDD Y2, Y8, Y8
1219 VPXOR Y8, Y10, Y10
1220 VPSLLD $0x07, Y10, Y3
1221 VPSRLD $0x19, Y10, Y10
1222 VPXOR Y3, Y10, Y10
1223 VPALIGNR $0x04, Y14, Y14, Y14
1224 VPALIGNR $0x04, Y9, Y9, Y9
1225 VPALIGNR $0x04, Y10, Y10, Y10
1226 VPALIGNR $0x08, Y12, Y12, Y12
1227 VPALIGNR $0x08, Y13, Y13, Y13
1228 VPALIGNR $0x08, Y8, Y8, Y8
1229 VPALIGNR $0x0c, Y4, Y4, Y4
1230 VPALIGNR $0x0c, Y1, Y1, Y1
1231 VPALIGNR $0x0c, Y2, Y2, Y2
1232 VPADDD Y14, Y0, Y0
1233 VPXOR Y0, Y4, Y4
1234 VPSHUFB ·rol16<>+0(SB), Y4, Y4
1235 VPADDD Y4, Y12, Y12
1236 VPXOR Y12, Y14, Y14
1237 VPSLLD $0x0c, Y14, Y3
1238 VPSRLD $0x14, Y14, Y14
1239 VPXOR Y3, Y14, Y14
1240 VPADDD Y14, Y0, Y0
1241 VPXOR Y0, Y4, Y4
1242 VPSHUFB ·rol8<>+0(SB), Y4, Y4
1243 VPADDD Y4, Y12, Y12
1244 VPXOR Y12, Y14, Y14
1245 VPSLLD $0x07, Y14, Y3
1246 VPSRLD $0x19, Y14, Y14
1247 VPXOR Y3, Y14, Y14
1248 VPADDD Y9, Y5, Y5
1249 VPXOR Y5, Y1, Y1
1250 VPSHUFB ·rol16<>+0(SB), Y1, Y1
1251 VPADDD Y1, Y13, Y13
1252 VPXOR Y13, Y9, Y9
1253 VPSLLD $0x0c, Y9, Y3
1254 VPSRLD $0x14, Y9, Y9
1255 VPXOR Y3, Y9, Y9
1256 VPADDD Y9, Y5, Y5
1257 VPXOR Y5, Y1, Y1
1258 VPSHUFB ·rol8<>+0(SB), Y1, Y1
1259 VPADDD Y1, Y13, Y13
1260 VPXOR Y13, Y9, Y9
1261 VPSLLD $0x07, Y9, Y3
1262 VPSRLD $0x19, Y9, Y9
1263 VPXOR Y3, Y9, Y9
1264 VPADDD Y10, Y6, Y6
1265 VPXOR Y6, Y2, Y2
1266 VPSHUFB ·rol16<>+0(SB), Y2, Y2
1267 VPADDD Y2, Y8, Y8
1268 VPXOR Y8, Y10, Y10
1269 VPSLLD $0x0c, Y10, Y3
1270 VPSRLD $0x14, Y10, Y10
1271 VPXOR Y3, Y10, Y10
1272 VPADDD Y10, Y6, Y6
1273 VPXOR Y6, Y2, Y2
1274 VPSHUFB ·rol8<>+0(SB), Y2, Y2
1275 VPADDD Y2, Y8, Y8
1276 VPXOR Y8, Y10, Y10
1277 VPSLLD $0x07, Y10, Y3
1278 VPSRLD $0x19, Y10, Y10
1279 VPXOR Y3, Y10, Y10
1280 VPALIGNR $0x0c, Y14, Y14, Y14
1281 VPALIGNR $0x0c, Y9, Y9, Y9
1282 VPALIGNR $0x0c, Y10, Y10, Y10
1283 VPALIGNR $0x08, Y12, Y12, Y12
1284 VPALIGNR $0x08, Y13, Y13, Y13
1285 VPALIGNR $0x08, Y8, Y8, Y8
1286 VPALIGNR $0x04, Y4, Y4, Y4
1287 VPALIGNR $0x04, Y1, Y1, Y1
1288 VPALIGNR $0x04, Y2, Y2, Y2
1289 DECQ R9
1290 JNE openAVX2320InnerCipherLoop
1291 VMOVDQA ·chacha20Constants<>+0(SB), Y3
1292 VPADDD Y3, Y0, Y0
1293 VPADDD Y3, Y5, Y5
1294 VPADDD Y3, Y6, Y6
1295 VPADDD Y7, Y14, Y14
1296 VPADDD Y7, Y9, Y9
1297 VPADDD Y7, Y10, Y10
1298 VPADDD Y11, Y12, Y12
1299 VPADDD Y11, Y13, Y13
1300 VPADDD Y11, Y8, Y8
1301 VMOVDQA ·avx2IncMask<>+0(SB), Y3
1302 VPADDD Y15, Y4, Y4
1303 VPADDD Y3, Y15, Y15
1304 VPADDD Y15, Y1, Y1
1305 VPADDD Y3, Y15, Y15
1306 VPADDD Y15, Y2, Y2
1307
1308 // Clamp and store poly key
1309 VPERM2I128 $0x02, Y0, Y14, Y3
1310 VPAND ·polyClampMask<>+0(SB), Y3, Y3
1311 VMOVDQA Y3, (BP)
1312
1313 // Stream for up to 320 bytes
1314 VPERM2I128 $0x13, Y0, Y14, Y0
1315 VPERM2I128 $0x13, Y12, Y4, Y14
1316 VPERM2I128 $0x02, Y5, Y9, Y12
1317 VPERM2I128 $0x02, Y13, Y1, Y4
1318 VPERM2I128 $0x13, Y5, Y9, Y5
1319 VPERM2I128 $0x13, Y13, Y1, Y9
1320 VPERM2I128 $0x02, Y6, Y10, Y13
1321 VPERM2I128 $0x02, Y8, Y2, Y1
1322 VPERM2I128 $0x13, Y6, Y10, Y6
1323 VPERM2I128 $0x13, Y8, Y2, Y10
1324 JMP openAVX2ShortOpen
1325
1326 openAVX2Tail128:
1327 // Need to decrypt up to 128 bytes - prepare two blocks
1328 VMOVDQA ·chacha20Constants<>+0(SB), Y5
1329 VMOVDQA 32(BP), Y9
1330 VMOVDQA 64(BP), Y13
1331 VMOVDQA 192(BP), Y1
1332 VPADDD ·avx2IncMask<>+0(SB), Y1, Y1
1333 VMOVDQA Y1, Y4
1334 XORQ R9, R9
1335 MOVQ BX, CX
1336 ANDQ $-16, CX
1337 TESTQ CX, CX
1338 JE openAVX2Tail128LoopB
1339
1340 openAVX2Tail128LoopA:
1341 ADDQ (SI)(R9*1), R10
1342 ADCQ 8(SI)(R9*1), R11
1343 ADCQ $0x01, R12
1344 MOVQ (BP), DX
1345 MOVQ DX, R15
1346 MULXQ R10, R13, R14
1347 IMULQ R12, R15
1348 MULXQ R11, AX, DX
1349 ADDQ AX, R14
1350 ADCQ DX, R15
1351 MOVQ 8(BP), DX
1352 MULXQ R10, R10, AX
1353 ADDQ R10, R14
1354 MULXQ R11, R11, R8
1355 ADCQ R11, R15
1356 ADCQ $0x00, R8
1357 IMULQ R12, DX
1358 ADDQ AX, R15
1359 ADCQ DX, R8
1360 MOVQ R13, R10
1361 MOVQ R14, R11
1362 MOVQ R15, R12
1363 ANDQ $0x03, R12
1364 MOVQ R15, R13
1365 ANDQ $-4, R13
1366 MOVQ R8, R14
1367 SHRQ $0x02, R8, R15
1368 SHRQ $0x02, R8
1369 ADDQ R13, R10
1370 ADCQ R14, R11
1371 ADCQ $0x00, R12
1372 ADDQ R15, R10
1373 ADCQ R8, R11
1374 ADCQ $0x00, R12
1375
1376 openAVX2Tail128LoopB:
1377 ADDQ $0x10, R9
1378 VPADDD Y9, Y5, Y5
1379 VPXOR Y5, Y1, Y1
1380 VPSHUFB ·rol16<>+0(SB), Y1, Y1
1381 VPADDD Y1, Y13, Y13
1382 VPXOR Y13, Y9, Y9
1383 VPSLLD $0x0c, Y9, Y3
1384 VPSRLD $0x14, Y9, Y9
1385 VPXOR Y3, Y9, Y9
1386 VPADDD Y9, Y5, Y5
1387 VPXOR Y5, Y1, Y1
1388 VPSHUFB ·rol8<>+0(SB), Y1, Y1
1389 VPADDD Y1, Y13, Y13
1390 VPXOR Y13, Y9, Y9
1391 VPSLLD $0x07, Y9, Y3
1392 VPSRLD $0x19, Y9, Y9
1393 VPXOR Y3, Y9, Y9
1394 VPALIGNR $0x04, Y9, Y9, Y9
1395 VPALIGNR $0x08, Y13, Y13, Y13
1396 VPALIGNR $0x0c, Y1, Y1, Y1
1397 VPADDD Y9, Y5, Y5
1398 VPXOR Y5, Y1, Y1
1399 VPSHUFB ·rol16<>+0(SB), Y1, Y1
1400 VPADDD Y1, Y13, Y13
1401 VPXOR Y13, Y9, Y9
1402 VPSLLD $0x0c, Y9, Y3
1403 VPSRLD $0x14, Y9, Y9
1404 VPXOR Y3, Y9, Y9
1405 VPADDD Y9, Y5, Y5
1406 VPXOR Y5, Y1, Y1
1407 VPSHUFB ·rol8<>+0(SB), Y1, Y1
1408 VPADDD Y1, Y13, Y13
1409 VPXOR Y13, Y9, Y9
1410 VPSLLD $0x07, Y9, Y3
1411 VPSRLD $0x19, Y9, Y9
1412 VPXOR Y3, Y9, Y9
1413 VPALIGNR $0x0c, Y9, Y9, Y9
1414 VPALIGNR $0x08, Y13, Y13, Y13
1415 VPALIGNR $0x04, Y1, Y1, Y1
1416 CMPQ R9, CX
1417 JB openAVX2Tail128LoopA
1418 CMPQ R9, $0xa0
1419 JNE openAVX2Tail128LoopB
1420 VPADDD ·chacha20Constants<>+0(SB), Y5, Y5
1421 VPADDD 32(BP), Y9, Y9
1422 VPADDD 64(BP), Y13, Y13
1423 VPADDD Y4, Y1, Y1
1424 VPERM2I128 $0x02, Y5, Y9, Y0
1425 VPERM2I128 $0x02, Y13, Y1, Y14
1426 VPERM2I128 $0x13, Y5, Y9, Y12
1427 VPERM2I128 $0x13, Y13, Y1, Y4
1428
1429 openAVX2TailLoop:
1430 CMPQ BX, $0x20
1431 JB openAVX2Tail
1432 SUBQ $0x20, BX
1433
1434 // Load for decryption
1435 VPXOR (SI), Y0, Y0
1436 VMOVDQU Y0, (DI)
1437 LEAQ 32(SI), SI
1438 LEAQ 32(DI), DI
1439 VMOVDQA Y14, Y0
1440 VMOVDQA Y12, Y14
1441 VMOVDQA Y4, Y12
1442 JMP openAVX2TailLoop
1443
1444 openAVX2Tail:
1445 CMPQ BX, $0x10
1446 VMOVDQA X0, X1
1447 JB openAVX2TailDone
1448 SUBQ $0x10, BX
1449
1450 // Load for decryption
1451 VPXOR (SI), X0, X12
1452 VMOVDQU X12, (DI)
1453 LEAQ 16(SI), SI
1454 LEAQ 16(DI), DI
1455 VPERM2I128 $0x11, Y0, Y0, Y0
1456 VMOVDQA X0, X1
1457
1458 openAVX2TailDone:
1459 VZEROUPPER
1460 JMP openSSETail16
1461
1462 openAVX2Tail256:
1463 VMOVDQA ·chacha20Constants<>+0(SB), Y0
1464 VMOVDQA Y0, Y5
1465 VMOVDQA 32(BP), Y14
1466 VMOVDQA Y14, Y9
1467 VMOVDQA 64(BP), Y12
1468 VMOVDQA Y12, Y13
1469 VMOVDQA 192(BP), Y4
1470 VPADDD ·avx2IncMask<>+0(SB), Y4, Y4
1471 VPADDD ·avx2IncMask<>+0(SB), Y4, Y1
1472 VMOVDQA Y4, Y7
1473 VMOVDQA Y1, Y11
1474
1475 // Compute the number of iterations that will hash data
1476 MOVQ BX, 224(BP)
1477 MOVQ BX, CX
1478 SUBQ $0x80, CX
1479 SHRQ $0x04, CX
1480 MOVQ $0x0000000a, R9
1481 CMPQ CX, $0x0a
1482 CMOVQGT R9, CX
1483 MOVQ SI, BX
1484 XORQ R9, R9
1485
1486 openAVX2Tail256LoopA:
1487 ADDQ (BX), R10
1488 ADCQ 8(BX), R11
1489 ADCQ $0x01, R12
1490 MOVQ (BP), DX
1491 MOVQ DX, R15
1492 MULXQ R10, R13, R14
1493 IMULQ R12, R15
1494 MULXQ R11, AX, DX
1495 ADDQ AX, R14
1496 ADCQ DX, R15
1497 MOVQ 8(BP), DX
1498 MULXQ R10, R10, AX
1499 ADDQ R10, R14
1500 MULXQ R11, R11, R8
1501 ADCQ R11, R15
1502 ADCQ $0x00, R8
1503 IMULQ R12, DX
1504 ADDQ AX, R15
1505 ADCQ DX, R8
1506 MOVQ R13, R10
1507 MOVQ R14, R11
1508 MOVQ R15, R12
1509 ANDQ $0x03, R12
1510 MOVQ R15, R13
1511 ANDQ $-4, R13
1512 MOVQ R8, R14
1513 SHRQ $0x02, R8, R15
1514 SHRQ $0x02, R8
1515 ADDQ R13, R10
1516 ADCQ R14, R11
1517 ADCQ $0x00, R12
1518 ADDQ R15, R10
1519 ADCQ R8, R11
1520 ADCQ $0x00, R12
1521 LEAQ 16(BX), BX
1522
1523 openAVX2Tail256LoopB:
1524 VPADDD Y14, Y0, Y0
1525 VPXOR Y0, Y4, Y4
1526 VPSHUFB ·rol16<>+0(SB), Y4, Y4
1527 VPADDD Y4, Y12, Y12
1528 VPXOR Y12, Y14, Y14
1529 VPSLLD $0x0c, Y14, Y3
1530 VPSRLD $0x14, Y14, Y14
1531 VPXOR Y3, Y14, Y14
1532 VPADDD Y14, Y0, Y0
1533 VPXOR Y0, Y4, Y4
1534 VPSHUFB ·rol8<>+0(SB), Y4, Y4
1535 VPADDD Y4, Y12, Y12
1536 VPXOR Y12, Y14, Y14
1537 VPSLLD $0x07, Y14, Y3
1538 VPSRLD $0x19, Y14, Y14
1539 VPXOR Y3, Y14, Y14
1540 VPADDD Y9, Y5, Y5
1541 VPXOR Y5, Y1, Y1
1542 VPSHUFB ·rol16<>+0(SB), Y1, Y1
1543 VPADDD Y1, Y13, Y13
1544 VPXOR Y13, Y9, Y9
1545 VPSLLD $0x0c, Y9, Y3
1546 VPSRLD $0x14, Y9, Y9
1547 VPXOR Y3, Y9, Y9
1548 VPADDD Y9, Y5, Y5
1549 VPXOR Y5, Y1, Y1
1550 VPSHUFB ·rol8<>+0(SB), Y1, Y1
1551 VPADDD Y1, Y13, Y13
1552 VPXOR Y13, Y9, Y9
1553 VPSLLD $0x07, Y9, Y3
1554 VPSRLD $0x19, Y9, Y9
1555 VPXOR Y3, Y9, Y9
1556 VPALIGNR $0x04, Y14, Y14, Y14
1557 VPALIGNR $0x04, Y9, Y9, Y9
1558 VPALIGNR $0x08, Y12, Y12, Y12
1559 VPALIGNR $0x08, Y13, Y13, Y13
1560 VPALIGNR $0x0c, Y4, Y4, Y4
1561 VPALIGNR $0x0c, Y1, Y1, Y1
1562 INCQ R9
1563 VPADDD Y14, Y0, Y0
1564 VPXOR Y0, Y4, Y4
1565 VPSHUFB ·rol16<>+0(SB), Y4, Y4
1566 VPADDD Y4, Y12, Y12
1567 VPXOR Y12, Y14, Y14
1568 VPSLLD $0x0c, Y14, Y3
1569 VPSRLD $0x14, Y14, Y14
1570 VPXOR Y3, Y14, Y14
1571 VPADDD Y14, Y0, Y0
1572 VPXOR Y0, Y4, Y4
1573 VPSHUFB ·rol8<>+0(SB), Y4, Y4
1574 VPADDD Y4, Y12, Y12
1575 VPXOR Y12, Y14, Y14
1576 VPSLLD $0x07, Y14, Y3
1577 VPSRLD $0x19, Y14, Y14
1578 VPXOR Y3, Y14, Y14
1579 VPADDD Y9, Y5, Y5
1580 VPXOR Y5, Y1, Y1
1581 VPSHUFB ·rol16<>+0(SB), Y1, Y1
1582 VPADDD Y1, Y13, Y13
1583 VPXOR Y13, Y9, Y9
1584 VPSLLD $0x0c, Y9, Y3
1585 VPSRLD $0x14, Y9, Y9
1586 VPXOR Y3, Y9, Y9
1587 VPADDD Y9, Y5, Y5
1588 VPXOR Y5, Y1, Y1
1589 VPSHUFB ·rol8<>+0(SB), Y1, Y1
1590 VPADDD Y1, Y13, Y13
1591 VPXOR Y13, Y9, Y9
1592 VPSLLD $0x07, Y9, Y3
1593 VPSRLD $0x19, Y9, Y9
1594 VPXOR Y3, Y9, Y9
1595 VPALIGNR $0x0c, Y14, Y14, Y14
1596 VPALIGNR $0x0c, Y9, Y9, Y9
1597 VPALIGNR $0x08, Y12, Y12, Y12
1598 VPALIGNR $0x08, Y13, Y13, Y13
1599 VPALIGNR $0x04, Y4, Y4, Y4
1600 VPALIGNR $0x04, Y1, Y1, Y1
1601 CMPQ R9, CX
1602 JB openAVX2Tail256LoopA
1603 CMPQ R9, $0x0a
1604 JNE openAVX2Tail256LoopB
1605 MOVQ BX, R9
1606 SUBQ SI, BX
1607 MOVQ BX, CX
1608 MOVQ 224(BP), BX
1609
1610 openAVX2Tail256Hash:
1611 ADDQ $0x10, CX
1612 CMPQ CX, BX
1613 JGT openAVX2Tail256HashEnd
1614 ADDQ (R9), R10
1615 ADCQ 8(R9), R11
1616 ADCQ $0x01, R12
1617 MOVQ (BP), DX
1618 MOVQ DX, R15
1619 MULXQ R10, R13, R14
1620 IMULQ R12, R15
1621 MULXQ R11, AX, DX
1622 ADDQ AX, R14
1623 ADCQ DX, R15
1624 MOVQ 8(BP), DX
1625 MULXQ R10, R10, AX
1626 ADDQ R10, R14
1627 MULXQ R11, R11, R8
1628 ADCQ R11, R15
1629 ADCQ $0x00, R8
1630 IMULQ R12, DX
1631 ADDQ AX, R15
1632 ADCQ DX, R8
1633 MOVQ R13, R10
1634 MOVQ R14, R11
1635 MOVQ R15, R12
1636 ANDQ $0x03, R12
1637 MOVQ R15, R13
1638 ANDQ $-4, R13
1639 MOVQ R8, R14
1640 SHRQ $0x02, R8, R15
1641 SHRQ $0x02, R8
1642 ADDQ R13, R10
1643 ADCQ R14, R11
1644 ADCQ $0x00, R12
1645 ADDQ R15, R10
1646 ADCQ R8, R11
1647 ADCQ $0x00, R12
1648 LEAQ 16(R9), R9
1649 JMP openAVX2Tail256Hash
1650
1651 openAVX2Tail256HashEnd:
1652 VPADDD ·chacha20Constants<>+0(SB), Y0, Y0
1653 VPADDD ·chacha20Constants<>+0(SB), Y5, Y5
1654 VPADDD 32(BP), Y14, Y14
1655 VPADDD 32(BP), Y9, Y9
1656 VPADDD 64(BP), Y12, Y12
1657 VPADDD 64(BP), Y13, Y13
1658 VPADDD Y7, Y4, Y4
1659 VPADDD Y11, Y1, Y1
1660 VPERM2I128 $0x02, Y0, Y14, Y6
1661 VPERM2I128 $0x02, Y12, Y4, Y10
1662 VPERM2I128 $0x13, Y0, Y14, Y8
1663 VPERM2I128 $0x13, Y12, Y4, Y2
1664 VPERM2I128 $0x02, Y5, Y9, Y0
1665 VPERM2I128 $0x02, Y13, Y1, Y14
1666 VPERM2I128 $0x13, Y5, Y9, Y12
1667 VPERM2I128 $0x13, Y13, Y1, Y4
1668 VPXOR (SI), Y6, Y6
1669 VPXOR 32(SI), Y10, Y10
1670 VPXOR 64(SI), Y8, Y8
1671 VPXOR 96(SI), Y2, Y2
1672 VMOVDQU Y6, (DI)
1673 VMOVDQU Y10, 32(DI)
1674 VMOVDQU Y8, 64(DI)
1675 VMOVDQU Y2, 96(DI)
1676 LEAQ 128(SI), SI
1677 LEAQ 128(DI), DI
1678 SUBQ $0x80, BX
1679 JMP openAVX2TailLoop
1680
1681 openAVX2Tail384:
1682 // Need to decrypt up to 384 bytes - prepare six blocks
1683 VMOVDQA ·chacha20Constants<>+0(SB), Y0
1684 VMOVDQA Y0, Y5
1685 VMOVDQA Y0, Y6
1686 VMOVDQA 32(BP), Y14
1687 VMOVDQA Y14, Y9
1688 VMOVDQA Y14, Y10
1689 VMOVDQA 64(BP), Y12
1690 VMOVDQA Y12, Y13
1691 VMOVDQA Y12, Y8
1692 VMOVDQA 192(BP), Y4
1693 VPADDD ·avx2IncMask<>+0(SB), Y4, Y4
1694 VPADDD ·avx2IncMask<>+0(SB), Y4, Y1
1695 VPADDD ·avx2IncMask<>+0(SB), Y1, Y2
1696 VMOVDQA Y4, 96(BP)
1697 VMOVDQA Y1, 128(BP)
1698 VMOVDQA Y2, 160(BP)
1699
1700 // Compute the number of iterations that will hash two blocks of data
1701 MOVQ BX, 224(BP)
1702 MOVQ BX, CX
1703 SUBQ $0x00000100, CX
1704 SHRQ $0x04, CX
1705 ADDQ $0x06, CX
1706 MOVQ $0x0000000a, R9
1707 CMPQ CX, $0x0a
1708 CMOVQGT R9, CX
1709 MOVQ SI, BX
1710 XORQ R9, R9
1711
1712 openAVX2Tail384LoopB:
1713 ADDQ (BX), R10
1714 ADCQ 8(BX), R11
1715 ADCQ $0x01, R12
1716 MOVQ (BP), DX
1717 MOVQ DX, R15
1718 MULXQ R10, R13, R14
1719 IMULQ R12, R15
1720 MULXQ R11, AX, DX
1721 ADDQ AX, R14
1722 ADCQ DX, R15
1723 MOVQ 8(BP), DX
1724 MULXQ R10, R10, AX
1725 ADDQ R10, R14
1726 MULXQ R11, R11, R8
1727 ADCQ R11, R15
1728 ADCQ $0x00, R8
1729 IMULQ R12, DX
1730 ADDQ AX, R15
1731 ADCQ DX, R8
1732 MOVQ R13, R10
1733 MOVQ R14, R11
1734 MOVQ R15, R12
1735 ANDQ $0x03, R12
1736 MOVQ R15, R13
1737 ANDQ $-4, R13
1738 MOVQ R8, R14
1739 SHRQ $0x02, R8, R15
1740 SHRQ $0x02, R8
1741 ADDQ R13, R10
1742 ADCQ R14, R11
1743 ADCQ $0x00, R12
1744 ADDQ R15, R10
1745 ADCQ R8, R11
1746 ADCQ $0x00, R12
1747 LEAQ 16(BX), BX
1748
1749 openAVX2Tail384LoopA:
1750 VPADDD Y14, Y0, Y0
1751 VPXOR Y0, Y4, Y4
1752 VPSHUFB ·rol16<>+0(SB), Y4, Y4
1753 VPADDD Y4, Y12, Y12
1754 VPXOR Y12, Y14, Y14
1755 VPSLLD $0x0c, Y14, Y3
1756 VPSRLD $0x14, Y14, Y14
1757 VPXOR Y3, Y14, Y14
1758 VPADDD Y14, Y0, Y0
1759 VPXOR Y0, Y4, Y4
1760 VPSHUFB ·rol8<>+0(SB), Y4, Y4
1761 VPADDD Y4, Y12, Y12
1762 VPXOR Y12, Y14, Y14
1763 VPSLLD $0x07, Y14, Y3
1764 VPSRLD $0x19, Y14, Y14
1765 VPXOR Y3, Y14, Y14
1766 VPADDD Y9, Y5, Y5
1767 VPXOR Y5, Y1, Y1
1768 VPSHUFB ·rol16<>+0(SB), Y1, Y1
1769 VPADDD Y1, Y13, Y13
1770 VPXOR Y13, Y9, Y9
1771 VPSLLD $0x0c, Y9, Y3
1772 VPSRLD $0x14, Y9, Y9
1773 VPXOR Y3, Y9, Y9
1774 VPADDD Y9, Y5, Y5
1775 VPXOR Y5, Y1, Y1
1776 VPSHUFB ·rol8<>+0(SB), Y1, Y1
1777 VPADDD Y1, Y13, Y13
1778 VPXOR Y13, Y9, Y9
1779 VPSLLD $0x07, Y9, Y3
1780 VPSRLD $0x19, Y9, Y9
1781 VPXOR Y3, Y9, Y9
1782 VPADDD Y10, Y6, Y6
1783 VPXOR Y6, Y2, Y2
1784 VPSHUFB ·rol16<>+0(SB), Y2, Y2
1785 VPADDD Y2, Y8, Y8
1786 VPXOR Y8, Y10, Y10
1787 VPSLLD $0x0c, Y10, Y3
1788 VPSRLD $0x14, Y10, Y10
1789 VPXOR Y3, Y10, Y10
1790 VPADDD Y10, Y6, Y6
1791 VPXOR Y6, Y2, Y2
1792 VPSHUFB ·rol8<>+0(SB), Y2, Y2
1793 VPADDD Y2, Y8, Y8
1794 VPXOR Y8, Y10, Y10
1795 VPSLLD $0x07, Y10, Y3
1796 VPSRLD $0x19, Y10, Y10
1797 VPXOR Y3, Y10, Y10
1798 VPALIGNR $0x04, Y14, Y14, Y14
1799 VPALIGNR $0x04, Y9, Y9, Y9
1800 VPALIGNR $0x04, Y10, Y10, Y10
1801 VPALIGNR $0x08, Y12, Y12, Y12
1802 VPALIGNR $0x08, Y13, Y13, Y13
1803 VPALIGNR $0x08, Y8, Y8, Y8
1804 VPALIGNR $0x0c, Y4, Y4, Y4
1805 VPALIGNR $0x0c, Y1, Y1, Y1
1806 VPALIGNR $0x0c, Y2, Y2, Y2
1807 ADDQ (BX), R10
1808 ADCQ 8(BX), R11
1809 ADCQ $0x01, R12
1810 MOVQ (BP), DX
1811 MOVQ DX, R15
1812 MULXQ R10, R13, R14
1813 IMULQ R12, R15
1814 MULXQ R11, AX, DX
1815 ADDQ AX, R14
1816 ADCQ DX, R15
1817 MOVQ 8(BP), DX
1818 MULXQ R10, R10, AX
1819 ADDQ R10, R14
1820 MULXQ R11, R11, R8
1821 ADCQ R11, R15
1822 ADCQ $0x00, R8
1823 IMULQ R12, DX
1824 ADDQ AX, R15
1825 ADCQ DX, R8
1826 MOVQ R13, R10
1827 MOVQ R14, R11
1828 MOVQ R15, R12
1829 ANDQ $0x03, R12
1830 MOVQ R15, R13
1831 ANDQ $-4, R13
1832 MOVQ R8, R14
1833 SHRQ $0x02, R8, R15
1834 SHRQ $0x02, R8
1835 ADDQ R13, R10
1836 ADCQ R14, R11
1837 ADCQ $0x00, R12
1838 ADDQ R15, R10
1839 ADCQ R8, R11
1840 ADCQ $0x00, R12
1841 LEAQ 16(BX), BX
1842 INCQ R9
1843 VPADDD Y14, Y0, Y0
1844 VPXOR Y0, Y4, Y4
1845 VPSHUFB ·rol16<>+0(SB), Y4, Y4
1846 VPADDD Y4, Y12, Y12
1847 VPXOR Y12, Y14, Y14
1848 VPSLLD $0x0c, Y14, Y3
1849 VPSRLD $0x14, Y14, Y14
1850 VPXOR Y3, Y14, Y14
1851 VPADDD Y14, Y0, Y0
1852 VPXOR Y0, Y4, Y4
1853 VPSHUFB ·rol8<>+0(SB), Y4, Y4
1854 VPADDD Y4, Y12, Y12
1855 VPXOR Y12, Y14, Y14
1856 VPSLLD $0x07, Y14, Y3
1857 VPSRLD $0x19, Y14, Y14
1858 VPXOR Y3, Y14, Y14
1859 VPADDD Y9, Y5, Y5
1860 VPXOR Y5, Y1, Y1
1861 VPSHUFB ·rol16<>+0(SB), Y1, Y1
1862 VPADDD Y1, Y13, Y13
1863 VPXOR Y13, Y9, Y9
1864 VPSLLD $0x0c, Y9, Y3
1865 VPSRLD $0x14, Y9, Y9
1866 VPXOR Y3, Y9, Y9
1867 VPADDD Y9, Y5, Y5
1868 VPXOR Y5, Y1, Y1
1869 VPSHUFB ·rol8<>+0(SB), Y1, Y1
1870 VPADDD Y1, Y13, Y13
1871 VPXOR Y13, Y9, Y9
1872 VPSLLD $0x07, Y9, Y3
1873 VPSRLD $0x19, Y9, Y9
1874 VPXOR Y3, Y9, Y9
1875 VPADDD Y10, Y6, Y6
1876 VPXOR Y6, Y2, Y2
1877 VPSHUFB ·rol16<>+0(SB), Y2, Y2
1878 VPADDD Y2, Y8, Y8
1879 VPXOR Y8, Y10, Y10
1880 VPSLLD $0x0c, Y10, Y3
1881 VPSRLD $0x14, Y10, Y10
1882 VPXOR Y3, Y10, Y10
1883 VPADDD Y10, Y6, Y6
1884 VPXOR Y6, Y2, Y2
1885 VPSHUFB ·rol8<>+0(SB), Y2, Y2
1886 VPADDD Y2, Y8, Y8
1887 VPXOR Y8, Y10, Y10
1888 VPSLLD $0x07, Y10, Y3
1889 VPSRLD $0x19, Y10, Y10
1890 VPXOR Y3, Y10, Y10
1891 VPALIGNR $0x0c, Y14, Y14, Y14
1892 VPALIGNR $0x0c, Y9, Y9, Y9
1893 VPALIGNR $0x0c, Y10, Y10, Y10
1894 VPALIGNR $0x08, Y12, Y12, Y12
1895 VPALIGNR $0x08, Y13, Y13, Y13
1896 VPALIGNR $0x08, Y8, Y8, Y8
1897 VPALIGNR $0x04, Y4, Y4, Y4
1898 VPALIGNR $0x04, Y1, Y1, Y1
1899 VPALIGNR $0x04, Y2, Y2, Y2
1900 CMPQ R9, CX
1901 JB openAVX2Tail384LoopB
1902 CMPQ R9, $0x0a
1903 JNE openAVX2Tail384LoopA
1904 MOVQ BX, R9
1905 SUBQ SI, BX
1906 MOVQ BX, CX
1907 MOVQ 224(BP), BX
1908
1909 openAVX2Tail384Hash:
1910 ADDQ $0x10, CX
1911 CMPQ CX, BX
1912 JGT openAVX2Tail384HashEnd
1913 ADDQ (R9), R10
1914 ADCQ 8(R9), R11
1915 ADCQ $0x01, R12
1916 MOVQ (BP), DX
1917 MOVQ DX, R15
1918 MULXQ R10, R13, R14
1919 IMULQ R12, R15
1920 MULXQ R11, AX, DX
1921 ADDQ AX, R14
1922 ADCQ DX, R15
1923 MOVQ 8(BP), DX
1924 MULXQ R10, R10, AX
1925 ADDQ R10, R14
1926 MULXQ R11, R11, R8
1927 ADCQ R11, R15
1928 ADCQ $0x00, R8
1929 IMULQ R12, DX
1930 ADDQ AX, R15
1931 ADCQ DX, R8
1932 MOVQ R13, R10
1933 MOVQ R14, R11
1934 MOVQ R15, R12
1935 ANDQ $0x03, R12
1936 MOVQ R15, R13
1937 ANDQ $-4, R13
1938 MOVQ R8, R14
1939 SHRQ $0x02, R8, R15
1940 SHRQ $0x02, R8
1941 ADDQ R13, R10
1942 ADCQ R14, R11
1943 ADCQ $0x00, R12
1944 ADDQ R15, R10
1945 ADCQ R8, R11
1946 ADCQ $0x00, R12
1947 LEAQ 16(R9), R9
1948 JMP openAVX2Tail384Hash
1949
1950 openAVX2Tail384HashEnd:
1951 VPADDD ·chacha20Constants<>+0(SB), Y0, Y0
1952 VPADDD ·chacha20Constants<>+0(SB), Y5, Y5
1953 VPADDD ·chacha20Constants<>+0(SB), Y6, Y6
1954 VPADDD 32(BP), Y14, Y14
1955 VPADDD 32(BP), Y9, Y9
1956 VPADDD 32(BP), Y10, Y10
1957 VPADDD 64(BP), Y12, Y12
1958 VPADDD 64(BP), Y13, Y13
1959 VPADDD 64(BP), Y8, Y8
1960 VPADDD 96(BP), Y4, Y4
1961 VPADDD 128(BP), Y1, Y1
1962 VPADDD 160(BP), Y2, Y2
1963 VPERM2I128 $0x02, Y0, Y14, Y3
1964 VPERM2I128 $0x02, Y12, Y4, Y7
1965 VPERM2I128 $0x13, Y0, Y14, Y11
1966 VPERM2I128 $0x13, Y12, Y4, Y15
1967 VPXOR (SI), Y3, Y3
1968 VPXOR 32(SI), Y7, Y7
1969 VPXOR 64(SI), Y11, Y11
1970 VPXOR 96(SI), Y15, Y15
1971 VMOVDQU Y3, (DI)
1972 VMOVDQU Y7, 32(DI)
1973 VMOVDQU Y11, 64(DI)
1974 VMOVDQU Y15, 96(DI)
1975 VPERM2I128 $0x02, Y5, Y9, Y3
1976 VPERM2I128 $0x02, Y13, Y1, Y7
1977 VPERM2I128 $0x13, Y5, Y9, Y11
1978 VPERM2I128 $0x13, Y13, Y1, Y15
1979 VPXOR 128(SI), Y3, Y3
1980 VPXOR 160(SI), Y7, Y7
1981 VPXOR 192(SI), Y11, Y11
1982 VPXOR 224(SI), Y15, Y15
1983 VMOVDQU Y3, 128(DI)
1984 VMOVDQU Y7, 160(DI)
1985 VMOVDQU Y11, 192(DI)
1986 VMOVDQU Y15, 224(DI)
1987 VPERM2I128 $0x02, Y6, Y10, Y0
1988 VPERM2I128 $0x02, Y8, Y2, Y14
1989 VPERM2I128 $0x13, Y6, Y10, Y12
1990 VPERM2I128 $0x13, Y8, Y2, Y4
1991 LEAQ 256(SI), SI
1992 LEAQ 256(DI), DI
1993 SUBQ $0x00000100, BX
1994 JMP openAVX2TailLoop
1995
1996 openAVX2Tail512:
1997 VMOVDQU ·chacha20Constants<>+0(SB), Y0
1998 VMOVDQA Y0, Y5
1999 VMOVDQA Y0, Y6
2000 VMOVDQA Y0, Y7
2001 VMOVDQA 32(BP), Y14
2002 VMOVDQA Y14, Y9
2003 VMOVDQA Y14, Y10
2004 VMOVDQA Y14, Y11
2005 VMOVDQA 64(BP), Y12
2006 VMOVDQA Y12, Y13
2007 VMOVDQA Y12, Y8
2008 VMOVDQA Y12, Y15
2009 VMOVDQA 192(BP), Y4
2010 VPADDD ·avx2IncMask<>+0(SB), Y4, Y4
2011 VPADDD ·avx2IncMask<>+0(SB), Y4, Y1
2012 VPADDD ·avx2IncMask<>+0(SB), Y1, Y2
2013 VPADDD ·avx2IncMask<>+0(SB), Y2, Y3
2014 VMOVDQA Y4, 96(BP)
2015 VMOVDQA Y1, 128(BP)
2016 VMOVDQA Y2, 160(BP)
2017 VMOVDQA Y3, 192(BP)
2018 XORQ CX, CX
2019 MOVQ SI, R9
2020
2021 openAVX2Tail512LoopB:
2022 ADDQ (R9), R10
2023 ADCQ 8(R9), R11
2024 ADCQ $0x01, R12
2025 MOVQ (BP), DX
2026 MOVQ DX, R15
2027 MULXQ R10, R13, R14
2028 IMULQ R12, R15
2029 MULXQ R11, AX, DX
2030 ADDQ AX, R14
2031 ADCQ DX, R15
2032 MOVQ 8(BP), DX
2033 MULXQ R10, R10, AX
2034 ADDQ R10, R14
2035 MULXQ R11, R11, R8
2036 ADCQ R11, R15
2037 ADCQ $0x00, R8
2038 IMULQ R12, DX
2039 ADDQ AX, R15
2040 ADCQ DX, R8
2041 MOVQ R13, R10
2042 MOVQ R14, R11
2043 MOVQ R15, R12
2044 ANDQ $0x03, R12
2045 MOVQ R15, R13
2046 ANDQ $-4, R13
2047 MOVQ R8, R14
2048 SHRQ $0x02, R8, R15
2049 SHRQ $0x02, R8
2050 ADDQ R13, R10
2051 ADCQ R14, R11
2052 ADCQ $0x00, R12
2053 ADDQ R15, R10
2054 ADCQ R8, R11
2055 ADCQ $0x00, R12
2056 LEAQ 16(R9), R9
2057
2058 openAVX2Tail512LoopA:
2059 VPADDD Y14, Y0, Y0
2060 VPADDD Y9, Y5, Y5
2061 VPADDD Y10, Y6, Y6
2062 VPADDD Y11, Y7, Y7
2063 VPXOR Y0, Y4, Y4
2064 VPXOR Y5, Y1, Y1
2065 VPXOR Y6, Y2, Y2
2066 VPXOR Y7, Y3, Y3
2067 VPSHUFB ·rol16<>+0(SB), Y4, Y4
2068 VPSHUFB ·rol16<>+0(SB), Y1, Y1
2069 VPSHUFB ·rol16<>+0(SB), Y2, Y2
2070 VPSHUFB ·rol16<>+0(SB), Y3, Y3
2071 VPADDD Y4, Y12, Y12
2072 VPADDD Y1, Y13, Y13
2073 VPADDD Y2, Y8, Y8
2074 VPADDD Y3, Y15, Y15
2075 VPXOR Y12, Y14, Y14
2076 VPXOR Y13, Y9, Y9
2077 VPXOR Y8, Y10, Y10
2078 VPXOR Y15, Y11, Y11
2079 VMOVDQA Y15, 224(BP)
2080 VPSLLD $0x0c, Y14, Y15
2081 VPSRLD $0x14, Y14, Y14
2082 VPXOR Y15, Y14, Y14
2083 VPSLLD $0x0c, Y9, Y15
2084 VPSRLD $0x14, Y9, Y9
2085 VPXOR Y15, Y9, Y9
2086 VPSLLD $0x0c, Y10, Y15
2087 VPSRLD $0x14, Y10, Y10
2088 VPXOR Y15, Y10, Y10
2089 VPSLLD $0x0c, Y11, Y15
2090 VPSRLD $0x14, Y11, Y11
2091 VPXOR Y15, Y11, Y11
2092 VMOVDQA 224(BP), Y15
2093 ADDQ (R9), R10
2094 ADCQ 8(R9), R11
2095 ADCQ $0x01, R12
2096 MOVQ (BP), DX
2097 MOVQ DX, R15
2098 MULXQ R10, R13, R14
2099 IMULQ R12, R15
2100 MULXQ R11, AX, DX
2101 ADDQ AX, R14
2102 ADCQ DX, R15
2103 MOVQ 8(BP), DX
2104 MULXQ R10, R10, AX
2105 ADDQ R10, R14
2106 MULXQ R11, R11, R8
2107 ADCQ R11, R15
2108 ADCQ $0x00, R8
2109 IMULQ R12, DX
2110 ADDQ AX, R15
2111 ADCQ DX, R8
2112 MOVQ R13, R10
2113 MOVQ R14, R11
2114 MOVQ R15, R12
2115 ANDQ $0x03, R12
2116 MOVQ R15, R13
2117 ANDQ $-4, R13
2118 MOVQ R8, R14
2119 SHRQ $0x02, R8, R15
2120 SHRQ $0x02, R8
2121 ADDQ R13, R10
2122 ADCQ R14, R11
2123 ADCQ $0x00, R12
2124 ADDQ R15, R10
2125 ADCQ R8, R11
2126 ADCQ $0x00, R12
2127 VPADDD Y14, Y0, Y0
2128 VPADDD Y9, Y5, Y5
2129 VPADDD Y10, Y6, Y6
2130 VPADDD Y11, Y7, Y7
2131 VPXOR Y0, Y4, Y4
2132 VPXOR Y5, Y1, Y1
2133 VPXOR Y6, Y2, Y2
2134 VPXOR Y7, Y3, Y3
2135 VPSHUFB ·rol8<>+0(SB), Y4, Y4
2136 VPSHUFB ·rol8<>+0(SB), Y1, Y1
2137 VPSHUFB ·rol8<>+0(SB), Y2, Y2
2138 VPSHUFB ·rol8<>+0(SB), Y3, Y3
2139 VPADDD Y4, Y12, Y12
2140 VPADDD Y1, Y13, Y13
2141 VPADDD Y2, Y8, Y8
2142 VPADDD Y3, Y15, Y15
2143 VPXOR Y12, Y14, Y14
2144 VPXOR Y13, Y9, Y9
2145 VPXOR Y8, Y10, Y10
2146 VPXOR Y15, Y11, Y11
2147 VMOVDQA Y15, 224(BP)
2148 VPSLLD $0x07, Y14, Y15
2149 VPSRLD $0x19, Y14, Y14
2150 VPXOR Y15, Y14, Y14
2151 VPSLLD $0x07, Y9, Y15
2152 VPSRLD $0x19, Y9, Y9
2153 VPXOR Y15, Y9, Y9
2154 VPSLLD $0x07, Y10, Y15
2155 VPSRLD $0x19, Y10, Y10
2156 VPXOR Y15, Y10, Y10
2157 VPSLLD $0x07, Y11, Y15
2158 VPSRLD $0x19, Y11, Y11
2159 VPXOR Y15, Y11, Y11
2160 VMOVDQA 224(BP), Y15
2161 VPALIGNR $0x04, Y14, Y14, Y14
2162 VPALIGNR $0x04, Y9, Y9, Y9
2163 VPALIGNR $0x04, Y10, Y10, Y10
2164 VPALIGNR $0x04, Y11, Y11, Y11
2165 VPALIGNR $0x08, Y12, Y12, Y12
2166 VPALIGNR $0x08, Y13, Y13, Y13
2167 VPALIGNR $0x08, Y8, Y8, Y8
2168 VPALIGNR $0x08, Y15, Y15, Y15
2169 VPALIGNR $0x0c, Y4, Y4, Y4
2170 VPALIGNR $0x0c, Y1, Y1, Y1
2171 VPALIGNR $0x0c, Y2, Y2, Y2
2172 VPALIGNR $0x0c, Y3, Y3, Y3
2173 VPADDD Y14, Y0, Y0
2174 VPADDD Y9, Y5, Y5
2175 VPADDD Y10, Y6, Y6
2176 VPADDD Y11, Y7, Y7
2177 VPXOR Y0, Y4, Y4
2178 VPXOR Y5, Y1, Y1
2179 VPXOR Y6, Y2, Y2
2180 VPXOR Y7, Y3, Y3
2181 VPSHUFB ·rol16<>+0(SB), Y4, Y4
2182 VPSHUFB ·rol16<>+0(SB), Y1, Y1
2183 VPSHUFB ·rol16<>+0(SB), Y2, Y2
2184 VPSHUFB ·rol16<>+0(SB), Y3, Y3
2185 VPADDD Y4, Y12, Y12
2186 VPADDD Y1, Y13, Y13
2187 VPADDD Y2, Y8, Y8
2188 VPADDD Y3, Y15, Y15
2189 VPXOR Y12, Y14, Y14
2190 VPXOR Y13, Y9, Y9
2191 VPXOR Y8, Y10, Y10
2192 VPXOR Y15, Y11, Y11
2193 ADDQ 16(R9), R10
2194 ADCQ 24(R9), R11
2195 ADCQ $0x01, R12
2196 MOVQ (BP), DX
2197 MOVQ DX, R15
2198 MULXQ R10, R13, R14
2199 IMULQ R12, R15
2200 MULXQ R11, AX, DX
2201 ADDQ AX, R14
2202 ADCQ DX, R15
2203 MOVQ 8(BP), DX
2204 MULXQ R10, R10, AX
2205 ADDQ R10, R14
2206 MULXQ R11, R11, R8
2207 ADCQ R11, R15
2208 ADCQ $0x00, R8
2209 IMULQ R12, DX
2210 ADDQ AX, R15
2211 ADCQ DX, R8
2212 MOVQ R13, R10
2213 MOVQ R14, R11
2214 MOVQ R15, R12
2215 ANDQ $0x03, R12
2216 MOVQ R15, R13
2217 ANDQ $-4, R13
2218 MOVQ R8, R14
2219 SHRQ $0x02, R8, R15
2220 SHRQ $0x02, R8
2221 ADDQ R13, R10
2222 ADCQ R14, R11
2223 ADCQ $0x00, R12
2224 ADDQ R15, R10
2225 ADCQ R8, R11
2226 ADCQ $0x00, R12
2227 LEAQ 32(R9), R9
2228 VMOVDQA Y15, 224(BP)
2229 VPSLLD $0x0c, Y14, Y15
2230 VPSRLD $0x14, Y14, Y14
2231 VPXOR Y15, Y14, Y14
2232 VPSLLD $0x0c, Y9, Y15
2233 VPSRLD $0x14, Y9, Y9
2234 VPXOR Y15, Y9, Y9
2235 VPSLLD $0x0c, Y10, Y15
2236 VPSRLD $0x14, Y10, Y10
2237 VPXOR Y15, Y10, Y10
2238 VPSLLD $0x0c, Y11, Y15
2239 VPSRLD $0x14, Y11, Y11
2240 VPXOR Y15, Y11, Y11
2241 VMOVDQA 224(BP), Y15
2242 VPADDD Y14, Y0, Y0
2243 VPADDD Y9, Y5, Y5
2244 VPADDD Y10, Y6, Y6
2245 VPADDD Y11, Y7, Y7
2246 VPXOR Y0, Y4, Y4
2247 VPXOR Y5, Y1, Y1
2248 VPXOR Y6, Y2, Y2
2249 VPXOR Y7, Y3, Y3
2250 VPSHUFB ·rol8<>+0(SB), Y4, Y4
2251 VPSHUFB ·rol8<>+0(SB), Y1, Y1
2252 VPSHUFB ·rol8<>+0(SB), Y2, Y2
2253 VPSHUFB ·rol8<>+0(SB), Y3, Y3
2254 VPADDD Y4, Y12, Y12
2255 VPADDD Y1, Y13, Y13
2256 VPADDD Y2, Y8, Y8
2257 VPADDD Y3, Y15, Y15
2258 VPXOR Y12, Y14, Y14
2259 VPXOR Y13, Y9, Y9
2260 VPXOR Y8, Y10, Y10
2261 VPXOR Y15, Y11, Y11
2262 VMOVDQA Y15, 224(BP)
2263 VPSLLD $0x07, Y14, Y15
2264 VPSRLD $0x19, Y14, Y14
2265 VPXOR Y15, Y14, Y14
2266 VPSLLD $0x07, Y9, Y15
2267 VPSRLD $0x19, Y9, Y9
2268 VPXOR Y15, Y9, Y9
2269 VPSLLD $0x07, Y10, Y15
2270 VPSRLD $0x19, Y10, Y10
2271 VPXOR Y15, Y10, Y10
2272 VPSLLD $0x07, Y11, Y15
2273 VPSRLD $0x19, Y11, Y11
2274 VPXOR Y15, Y11, Y11
2275 VMOVDQA 224(BP), Y15
2276 VPALIGNR $0x0c, Y14, Y14, Y14
2277 VPALIGNR $0x0c, Y9, Y9, Y9
2278 VPALIGNR $0x0c, Y10, Y10, Y10
2279 VPALIGNR $0x0c, Y11, Y11, Y11
2280 VPALIGNR $0x08, Y12, Y12, Y12
2281 VPALIGNR $0x08, Y13, Y13, Y13
2282 VPALIGNR $0x08, Y8, Y8, Y8
2283 VPALIGNR $0x08, Y15, Y15, Y15
2284 VPALIGNR $0x04, Y4, Y4, Y4
2285 VPALIGNR $0x04, Y1, Y1, Y1
2286 VPALIGNR $0x04, Y2, Y2, Y2
2287 VPALIGNR $0x04, Y3, Y3, Y3
2288 INCQ CX
2289 CMPQ CX, $0x04
2290 JLT openAVX2Tail512LoopB
2291 CMPQ CX, $0x0a
2292 JNE openAVX2Tail512LoopA
2293 MOVQ BX, CX
2294 SUBQ $0x00000180, CX
2295 ANDQ $-16, CX
2296
2297 openAVX2Tail512HashLoop:
2298 TESTQ CX, CX
2299 JE openAVX2Tail512HashEnd
2300 ADDQ (R9), R10
2301 ADCQ 8(R9), R11
2302 ADCQ $0x01, R12
2303 MOVQ (BP), DX
2304 MOVQ DX, R15
2305 MULXQ R10, R13, R14
2306 IMULQ R12, R15
2307 MULXQ R11, AX, DX
2308 ADDQ AX, R14
2309 ADCQ DX, R15
2310 MOVQ 8(BP), DX
2311 MULXQ R10, R10, AX
2312 ADDQ R10, R14
2313 MULXQ R11, R11, R8
2314 ADCQ R11, R15
2315 ADCQ $0x00, R8
2316 IMULQ R12, DX
2317 ADDQ AX, R15
2318 ADCQ DX, R8
2319 MOVQ R13, R10
2320 MOVQ R14, R11
2321 MOVQ R15, R12
2322 ANDQ $0x03, R12
2323 MOVQ R15, R13
2324 ANDQ $-4, R13
2325 MOVQ R8, R14
2326 SHRQ $0x02, R8, R15
2327 SHRQ $0x02, R8
2328 ADDQ R13, R10
2329 ADCQ R14, R11
2330 ADCQ $0x00, R12
2331 ADDQ R15, R10
2332 ADCQ R8, R11
2333 ADCQ $0x00, R12
2334 LEAQ 16(R9), R9
2335 SUBQ $0x10, CX
2336 JMP openAVX2Tail512HashLoop
2337
2338 openAVX2Tail512HashEnd:
2339 VPADDD ·chacha20Constants<>+0(SB), Y0, Y0
2340 VPADDD ·chacha20Constants<>+0(SB), Y5, Y5
2341 VPADDD ·chacha20Constants<>+0(SB), Y6, Y6
2342 VPADDD ·chacha20Constants<>+0(SB), Y7, Y7
2343 VPADDD 32(BP), Y14, Y14
2344 VPADDD 32(BP), Y9, Y9
2345 VPADDD 32(BP), Y10, Y10
2346 VPADDD 32(BP), Y11, Y11
2347 VPADDD 64(BP), Y12, Y12
2348 VPADDD 64(BP), Y13, Y13
2349 VPADDD 64(BP), Y8, Y8
2350 VPADDD 64(BP), Y15, Y15
2351 VPADDD 96(BP), Y4, Y4
2352 VPADDD 128(BP), Y1, Y1
2353 VPADDD 160(BP), Y2, Y2
2354 VPADDD 192(BP), Y3, Y3
2355 VMOVDQA Y15, 224(BP)
2356 VPERM2I128 $0x02, Y0, Y14, Y15
2357 VPERM2I128 $0x13, Y0, Y14, Y14
2358 VPERM2I128 $0x02, Y12, Y4, Y0
2359 VPERM2I128 $0x13, Y12, Y4, Y12
2360 VPXOR (SI), Y15, Y15
2361 VPXOR 32(SI), Y0, Y0
2362 VPXOR 64(SI), Y14, Y14
2363 VPXOR 96(SI), Y12, Y12
2364 VMOVDQU Y15, (DI)
2365 VMOVDQU Y0, 32(DI)
2366 VMOVDQU Y14, 64(DI)
2367 VMOVDQU Y12, 96(DI)
2368 VPERM2I128 $0x02, Y5, Y9, Y0
2369 VPERM2I128 $0x02, Y13, Y1, Y14
2370 VPERM2I128 $0x13, Y5, Y9, Y12
2371 VPERM2I128 $0x13, Y13, Y1, Y4
2372 VPXOR 128(SI), Y0, Y0
2373 VPXOR 160(SI), Y14, Y14
2374 VPXOR 192(SI), Y12, Y12
2375 VPXOR 224(SI), Y4, Y4
2376 VMOVDQU Y0, 128(DI)
2377 VMOVDQU Y14, 160(DI)
2378 VMOVDQU Y12, 192(DI)
2379 VMOVDQU Y4, 224(DI)
2380 VPERM2I128 $0x02, Y6, Y10, Y0
2381 VPERM2I128 $0x02, Y8, Y2, Y14
2382 VPERM2I128 $0x13, Y6, Y10, Y12
2383 VPERM2I128 $0x13, Y8, Y2, Y4
2384 VPXOR 256(SI), Y0, Y0
2385 VPXOR 288(SI), Y14, Y14
2386 VPXOR 320(SI), Y12, Y12
2387 VPXOR 352(SI), Y4, Y4
2388 VMOVDQU Y0, 256(DI)
2389 VMOVDQU Y14, 288(DI)
2390 VMOVDQU Y12, 320(DI)
2391 VMOVDQU Y4, 352(DI)
2392 VPERM2I128 $0x02, Y7, Y11, Y0
2393 VPERM2I128 $0x02, 224(BP), Y3, Y14
2394 VPERM2I128 $0x13, Y7, Y11, Y12
2395 VPERM2I128 $0x13, 224(BP), Y3, Y4
2396 LEAQ 384(SI), SI
2397 LEAQ 384(DI), DI
2398 SUBQ $0x00000180, BX
2399 JMP openAVX2TailLoop
2400
2401 DATA ·chacha20Constants<>+0(SB)/4, $0x61707865
2402 DATA ·chacha20Constants<>+4(SB)/4, $0x3320646e
2403 DATA ·chacha20Constants<>+8(SB)/4, $0x79622d32
2404 DATA ·chacha20Constants<>+12(SB)/4, $0x6b206574
2405 DATA ·chacha20Constants<>+16(SB)/4, $0x61707865
2406 DATA ·chacha20Constants<>+20(SB)/4, $0x3320646e
2407 DATA ·chacha20Constants<>+24(SB)/4, $0x79622d32
2408 DATA ·chacha20Constants<>+28(SB)/4, $0x6b206574
2409 GLOBL ·chacha20Constants<>(SB), RODATA|NOPTR, $32
2410
2411 DATA ·avx2InitMask<>+0(SB)/8, $0x0000000000000000
2412 DATA ·avx2InitMask<>+8(SB)/8, $0x0000000000000000
2413 DATA ·avx2InitMask<>+16(SB)/8, $0x0000000000000001
2414 DATA ·avx2InitMask<>+24(SB)/8, $0x0000000000000000
2415 GLOBL ·avx2InitMask<>(SB), RODATA|NOPTR, $32
2416
2417 DATA ·rol16<>+0(SB)/8, $0x0504070601000302
2418 DATA ·rol16<>+8(SB)/8, $0x0d0c0f0e09080b0a
2419 DATA ·rol16<>+16(SB)/8, $0x0504070601000302
2420 DATA ·rol16<>+24(SB)/8, $0x0d0c0f0e09080b0a
2421 GLOBL ·rol16<>(SB), RODATA|NOPTR, $32
2422
2423 DATA ·rol8<>+0(SB)/8, $0x0605040702010003
2424 DATA ·rol8<>+8(SB)/8, $0x0e0d0c0f0a09080b
2425 DATA ·rol8<>+16(SB)/8, $0x0605040702010003
2426 DATA ·rol8<>+24(SB)/8, $0x0e0d0c0f0a09080b
2427 GLOBL ·rol8<>(SB), RODATA|NOPTR, $32
2428
2429 DATA ·polyClampMask<>+0(SB)/8, $0x0ffffffc0fffffff
2430 DATA ·polyClampMask<>+8(SB)/8, $0x0ffffffc0ffffffc
2431 DATA ·polyClampMask<>+16(SB)/8, $0xffffffffffffffff
2432 DATA ·polyClampMask<>+24(SB)/8, $0xffffffffffffffff
2433 GLOBL ·polyClampMask<>(SB), RODATA|NOPTR, $32
2434
2435 DATA ·avx2IncMask<>+0(SB)/8, $0x0000000000000002
2436 DATA ·avx2IncMask<>+8(SB)/8, $0x0000000000000000
2437 DATA ·avx2IncMask<>+16(SB)/8, $0x0000000000000002
2438 DATA ·avx2IncMask<>+24(SB)/8, $0x0000000000000000
2439 GLOBL ·avx2IncMask<>(SB), RODATA|NOPTR, $32
2440
2441 DATA ·andMask<>+0(SB)/8, $0x00000000000000ff
2442 DATA ·andMask<>+8(SB)/8, $0x0000000000000000
2443 DATA ·andMask<>+16(SB)/8, $0x000000000000ffff
2444 DATA ·andMask<>+24(SB)/8, $0x0000000000000000
2445 DATA ·andMask<>+32(SB)/8, $0x0000000000ffffff
2446 DATA ·andMask<>+40(SB)/8, $0x0000000000000000
2447 DATA ·andMask<>+48(SB)/8, $0x00000000ffffffff
2448 DATA ·andMask<>+56(SB)/8, $0x0000000000000000
2449 DATA ·andMask<>+64(SB)/8, $0x000000ffffffffff
2450 DATA ·andMask<>+72(SB)/8, $0x0000000000000000
2451 DATA ·andMask<>+80(SB)/8, $0x0000ffffffffffff
2452 DATA ·andMask<>+88(SB)/8, $0x0000000000000000
2453 DATA ·andMask<>+96(SB)/8, $0x00ffffffffffffff
2454 DATA ·andMask<>+104(SB)/8, $0x0000000000000000
2455 DATA ·andMask<>+112(SB)/8, $0xffffffffffffffff
2456 DATA ·andMask<>+120(SB)/8, $0x0000000000000000
2457 DATA ·andMask<>+128(SB)/8, $0xffffffffffffffff
2458 DATA ·andMask<>+136(SB)/8, $0x00000000000000ff
2459 DATA ·andMask<>+144(SB)/8, $0xffffffffffffffff
2460 DATA ·andMask<>+152(SB)/8, $0x000000000000ffff
2461 DATA ·andMask<>+160(SB)/8, $0xffffffffffffffff
2462 DATA ·andMask<>+168(SB)/8, $0x0000000000ffffff
2463 DATA ·andMask<>+176(SB)/8, $0xffffffffffffffff
2464 DATA ·andMask<>+184(SB)/8, $0x00000000ffffffff
2465 DATA ·andMask<>+192(SB)/8, $0xffffffffffffffff
2466 DATA ·andMask<>+200(SB)/8, $0x000000ffffffffff
2467 DATA ·andMask<>+208(SB)/8, $0xffffffffffffffff
2468 DATA ·andMask<>+216(SB)/8, $0x0000ffffffffffff
2469 DATA ·andMask<>+224(SB)/8, $0xffffffffffffffff
2470 DATA ·andMask<>+232(SB)/8, $0x00ffffffffffffff
2471 GLOBL ·andMask<>(SB), RODATA|NOPTR, $240
2472
2473 // func chacha20Poly1305Seal(dst []byte, key []uint32, src []byte, ad []byte)
2474 // Requires: AVX, AVX2, BMI2, CMOV, SSE2
2475 TEXT ·chacha20Poly1305Seal(SB), $288-96
2476 MOVQ SP, BP
2477 ADDQ $0x20, BP
2478 ANDQ $-32, BP
2479 MOVQ dst_base+0(FP), DI
2480 MOVQ key_base+24(FP), R8
2481 MOVQ src_base+48(FP), SI
2482 MOVQ src_len+56(FP), BX
2483 MOVQ ad_base+72(FP), CX
2484 VZEROUPPER
2485 VMOVDQU ·chacha20Constants<>+0(SB), Y0
2486 VBROADCASTI128 16(R8), Y14
2487 VBROADCASTI128 32(R8), Y12
2488 VBROADCASTI128 48(R8), Y4
2489 VPADDD ·avx2InitMask<>+0(SB), Y4, Y4
2490
2491 // Special optimizations, for very short buffers
2492 CMPQ BX, $0x000000c0
2493 JBE seal192AVX2
2494 CMPQ BX, $0x00000140
2495 JBE seal320AVX2
2496
2497 // For the general key prepare the key first - as a byproduct we have 64 bytes of cipher stream
2498 VMOVDQA Y0, Y5
2499 VMOVDQA Y0, Y6
2500 VMOVDQA Y0, Y7
2501 VMOVDQA Y14, Y9
2502 VMOVDQA Y14, Y10
2503 VMOVDQA Y14, Y11
2504 VMOVDQA Y14, 32(BP)
2505 VMOVDQA Y12, Y13
2506 VMOVDQA Y12, Y8
2507 VMOVDQA Y12, Y15
2508 VMOVDQA Y12, 64(BP)
2509 VPADDD ·avx2IncMask<>+0(SB), Y4, Y1
2510 VMOVDQA Y4, 96(BP)
2511 VPADDD ·avx2IncMask<>+0(SB), Y1, Y2
2512 VMOVDQA Y1, 128(BP)
2513 VPADDD ·avx2IncMask<>+0(SB), Y2, Y3
2514 VMOVDQA Y2, 160(BP)
2515 VMOVDQA Y3, 192(BP)
2516 MOVQ $0x0000000a, R9
2517
2518 sealAVX2IntroLoop:
2519 VMOVDQA Y15, 224(BP)
2520 VPADDD Y14, Y0, Y0
2521 VPXOR Y0, Y4, Y4
2522 VPSHUFB ·rol16<>+0(SB), Y4, Y4
2523 VPADDD Y4, Y12, Y12
2524 VPXOR Y12, Y14, Y14
2525 VPSLLD $0x0c, Y14, Y15
2526 VPSRLD $0x14, Y14, Y14
2527 VPXOR Y15, Y14, Y14
2528 VPADDD Y14, Y0, Y0
2529 VPXOR Y0, Y4, Y4
2530 VPSHUFB ·rol8<>+0(SB), Y4, Y4
2531 VPADDD Y4, Y12, Y12
2532 VPXOR Y12, Y14, Y14
2533 VPSLLD $0x07, Y14, Y15
2534 VPSRLD $0x19, Y14, Y14
2535 VPXOR Y15, Y14, Y14
2536 VPADDD Y9, Y5, Y5
2537 VPXOR Y5, Y1, Y1
2538 VPSHUFB ·rol16<>+0(SB), Y1, Y1
2539 VPADDD Y1, Y13, Y13
2540 VPXOR Y13, Y9, Y9
2541 VPSLLD $0x0c, Y9, Y15
2542 VPSRLD $0x14, Y9, Y9
2543 VPXOR Y15, Y9, Y9
2544 VPADDD Y9, Y5, Y5
2545 VPXOR Y5, Y1, Y1
2546 VPSHUFB ·rol8<>+0(SB), Y1, Y1
2547 VPADDD Y1, Y13, Y13
2548 VPXOR Y13, Y9, Y9
2549 VPSLLD $0x07, Y9, Y15
2550 VPSRLD $0x19, Y9, Y9
2551 VPXOR Y15, Y9, Y9
2552 VPADDD Y10, Y6, Y6
2553 VPXOR Y6, Y2, Y2
2554 VPSHUFB ·rol16<>+0(SB), Y2, Y2
2555 VPADDD Y2, Y8, Y8
2556 VPXOR Y8, Y10, Y10
2557 VPSLLD $0x0c, Y10, Y15
2558 VPSRLD $0x14, Y10, Y10
2559 VPXOR Y15, Y10, Y10
2560 VPADDD Y10, Y6, Y6
2561 VPXOR Y6, Y2, Y2
2562 VPSHUFB ·rol8<>+0(SB), Y2, Y2
2563 VPADDD Y2, Y8, Y8
2564 VPXOR Y8, Y10, Y10
2565 VPSLLD $0x07, Y10, Y15
2566 VPSRLD $0x19, Y10, Y10
2567 VPXOR Y15, Y10, Y10
2568 VMOVDQA 224(BP), Y15
2569 VMOVDQA Y13, 224(BP)
2570 VPADDD Y11, Y7, Y7
2571 VPXOR Y7, Y3, Y3
2572 VPSHUFB ·rol16<>+0(SB), Y3, Y3
2573 VPADDD Y3, Y15, Y15
2574 VPXOR Y15, Y11, Y11
2575 VPSLLD $0x0c, Y11, Y13
2576 VPSRLD $0x14, Y11, Y11
2577 VPXOR Y13, Y11, Y11
2578 VPADDD Y11, Y7, Y7
2579 VPXOR Y7, Y3, Y3
2580 VPSHUFB ·rol8<>+0(SB), Y3, Y3
2581 VPADDD Y3, Y15, Y15
2582 VPXOR Y15, Y11, Y11
2583 VPSLLD $0x07, Y11, Y13
2584 VPSRLD $0x19, Y11, Y11
2585 VPXOR Y13, Y11, Y11
2586 VMOVDQA 224(BP), Y13
2587 VPALIGNR $0x04, Y14, Y14, Y14
2588 VPALIGNR $0x08, Y12, Y12, Y12
2589 VPALIGNR $0x0c, Y4, Y4, Y4
2590 VPALIGNR $0x04, Y9, Y9, Y9
2591 VPALIGNR $0x08, Y13, Y13, Y13
2592 VPALIGNR $0x0c, Y1, Y1, Y1
2593 VPALIGNR $0x04, Y10, Y10, Y10
2594 VPALIGNR $0x08, Y8, Y8, Y8
2595 VPALIGNR $0x0c, Y2, Y2, Y2
2596 VPALIGNR $0x04, Y11, Y11, Y11
2597 VPALIGNR $0x08, Y15, Y15, Y15
2598 VPALIGNR $0x0c, Y3, Y3, Y3
2599 VMOVDQA Y15, 224(BP)
2600 VPADDD Y14, Y0, Y0
2601 VPXOR Y0, Y4, Y4
2602 VPSHUFB ·rol16<>+0(SB), Y4, Y4
2603 VPADDD Y4, Y12, Y12
2604 VPXOR Y12, Y14, Y14
2605 VPSLLD $0x0c, Y14, Y15
2606 VPSRLD $0x14, Y14, Y14
2607 VPXOR Y15, Y14, Y14
2608 VPADDD Y14, Y0, Y0
2609 VPXOR Y0, Y4, Y4
2610 VPSHUFB ·rol8<>+0(SB), Y4, Y4
2611 VPADDD Y4, Y12, Y12
2612 VPXOR Y12, Y14, Y14
2613 VPSLLD $0x07, Y14, Y15
2614 VPSRLD $0x19, Y14, Y14
2615 VPXOR Y15, Y14, Y14
2616 VPADDD Y9, Y5, Y5
2617 VPXOR Y5, Y1, Y1
2618 VPSHUFB ·rol16<>+0(SB), Y1, Y1
2619 VPADDD Y1, Y13, Y13
2620 VPXOR Y13, Y9, Y9
2621 VPSLLD $0x0c, Y9, Y15
2622 VPSRLD $0x14, Y9, Y9
2623 VPXOR Y15, Y9, Y9
2624 VPADDD Y9, Y5, Y5
2625 VPXOR Y5, Y1, Y1
2626 VPSHUFB ·rol8<>+0(SB), Y1, Y1
2627 VPADDD Y1, Y13, Y13
2628 VPXOR Y13, Y9, Y9
2629 VPSLLD $0x07, Y9, Y15
2630 VPSRLD $0x19, Y9, Y9
2631 VPXOR Y15, Y9, Y9
2632 VPADDD Y10, Y6, Y6
2633 VPXOR Y6, Y2, Y2
2634 VPSHUFB ·rol16<>+0(SB), Y2, Y2
2635 VPADDD Y2, Y8, Y8
2636 VPXOR Y8, Y10, Y10
2637 VPSLLD $0x0c, Y10, Y15
2638 VPSRLD $0x14, Y10, Y10
2639 VPXOR Y15, Y10, Y10
2640 VPADDD Y10, Y6, Y6
2641 VPXOR Y6, Y2, Y2
2642 VPSHUFB ·rol8<>+0(SB), Y2, Y2
2643 VPADDD Y2, Y8, Y8
2644 VPXOR Y8, Y10, Y10
2645 VPSLLD $0x07, Y10, Y15
2646 VPSRLD $0x19, Y10, Y10
2647 VPXOR Y15, Y10, Y10
2648 VMOVDQA 224(BP), Y15
2649 VMOVDQA Y13, 224(BP)
2650 VPADDD Y11, Y7, Y7
2651 VPXOR Y7, Y3, Y3
2652 VPSHUFB ·rol16<>+0(SB), Y3, Y3
2653 VPADDD Y3, Y15, Y15
2654 VPXOR Y15, Y11, Y11
2655 VPSLLD $0x0c, Y11, Y13
2656 VPSRLD $0x14, Y11, Y11
2657 VPXOR Y13, Y11, Y11
2658 VPADDD Y11, Y7, Y7
2659 VPXOR Y7, Y3, Y3
2660 VPSHUFB ·rol8<>+0(SB), Y3, Y3
2661 VPADDD Y3, Y15, Y15
2662 VPXOR Y15, Y11, Y11
2663 VPSLLD $0x07, Y11, Y13
2664 VPSRLD $0x19, Y11, Y11
2665 VPXOR Y13, Y11, Y11
2666 VMOVDQA 224(BP), Y13
2667 VPALIGNR $0x0c, Y14, Y14, Y14
2668 VPALIGNR $0x08, Y12, Y12, Y12
2669 VPALIGNR $0x04, Y4, Y4, Y4
2670 VPALIGNR $0x0c, Y9, Y9, Y9
2671 VPALIGNR $0x08, Y13, Y13, Y13
2672 VPALIGNR $0x04, Y1, Y1, Y1
2673 VPALIGNR $0x0c, Y10, Y10, Y10
2674 VPALIGNR $0x08, Y8, Y8, Y8
2675 VPALIGNR $0x04, Y2, Y2, Y2
2676 VPALIGNR $0x0c, Y11, Y11, Y11
2677 VPALIGNR $0x08, Y15, Y15, Y15
2678 VPALIGNR $0x04, Y3, Y3, Y3
2679 DECQ R9
2680 JNE sealAVX2IntroLoop
2681 VPADDD ·chacha20Constants<>+0(SB), Y0, Y0
2682 VPADDD ·chacha20Constants<>+0(SB), Y5, Y5
2683 VPADDD ·chacha20Constants<>+0(SB), Y6, Y6
2684 VPADDD ·chacha20Constants<>+0(SB), Y7, Y7
2685 VPADDD 32(BP), Y14, Y14
2686 VPADDD 32(BP), Y9, Y9
2687 VPADDD 32(BP), Y10, Y10
2688 VPADDD 32(BP), Y11, Y11
2689 VPADDD 64(BP), Y12, Y12
2690 VPADDD 64(BP), Y13, Y13
2691 VPADDD 64(BP), Y8, Y8
2692 VPADDD 64(BP), Y15, Y15
2693 VPADDD 96(BP), Y4, Y4
2694 VPADDD 128(BP), Y1, Y1
2695 VPADDD 160(BP), Y2, Y2
2696 VPADDD 192(BP), Y3, Y3
2697 VPERM2I128 $0x13, Y12, Y4, Y12
2698 VPERM2I128 $0x02, Y0, Y14, Y4
2699 VPERM2I128 $0x13, Y0, Y14, Y0
2700
2701 // Clamp and store poly key
2702 VPAND ·polyClampMask<>+0(SB), Y4, Y4
2703 VMOVDQA Y4, (BP)
2704
2705 // Hash AD
2706 MOVQ ad_len+80(FP), R9
2707 CALL polyHashADInternal<>(SB)
2708
2709 // Can store at least 320 bytes
2710 VPXOR (SI), Y0, Y0
2711 VPXOR 32(SI), Y12, Y12
2712 VMOVDQU Y0, (DI)
2713 VMOVDQU Y12, 32(DI)
2714 VPERM2I128 $0x02, Y5, Y9, Y0
2715 VPERM2I128 $0x02, Y13, Y1, Y14
2716 VPERM2I128 $0x13, Y5, Y9, Y12
2717 VPERM2I128 $0x13, Y13, Y1, Y4
2718 VPXOR 64(SI), Y0, Y0
2719 VPXOR 96(SI), Y14, Y14
2720 VPXOR 128(SI), Y12, Y12
2721 VPXOR 160(SI), Y4, Y4
2722 VMOVDQU Y0, 64(DI)
2723 VMOVDQU Y14, 96(DI)
2724 VMOVDQU Y12, 128(DI)
2725 VMOVDQU Y4, 160(DI)
2726 VPERM2I128 $0x02, Y6, Y10, Y0
2727 VPERM2I128 $0x02, Y8, Y2, Y14
2728 VPERM2I128 $0x13, Y6, Y10, Y12
2729 VPERM2I128 $0x13, Y8, Y2, Y4
2730 VPXOR 192(SI), Y0, Y0
2731 VPXOR 224(SI), Y14, Y14
2732 VPXOR 256(SI), Y12, Y12
2733 VPXOR 288(SI), Y4, Y4
2734 VMOVDQU Y0, 192(DI)
2735 VMOVDQU Y14, 224(DI)
2736 VMOVDQU Y12, 256(DI)
2737 VMOVDQU Y4, 288(DI)
2738 MOVQ $0x00000140, CX
2739 SUBQ $0x00000140, BX
2740 LEAQ 320(SI), SI
2741 VPERM2I128 $0x02, Y7, Y11, Y0
2742 VPERM2I128 $0x02, Y15, Y3, Y14
2743 VPERM2I128 $0x13, Y7, Y11, Y12
2744 VPERM2I128 $0x13, Y15, Y3, Y4
2745 CMPQ BX, $0x80
2746 JBE sealAVX2SealHash
2747 VPXOR (SI), Y0, Y0
2748 VPXOR 32(SI), Y14, Y14
2749 VPXOR 64(SI), Y12, Y12
2750 VPXOR 96(SI), Y4, Y4
2751 VMOVDQU Y0, 320(DI)
2752 VMOVDQU Y14, 352(DI)
2753 VMOVDQU Y12, 384(DI)
2754 VMOVDQU Y4, 416(DI)
2755 SUBQ $0x80, BX
2756 LEAQ 128(SI), SI
2757 MOVQ $0x00000008, CX
2758 MOVQ $0x00000002, R9
2759 CMPQ BX, $0x80
2760 JBE sealAVX2Tail128
2761 CMPQ BX, $0x00000100
2762 JBE sealAVX2Tail256
2763 CMPQ BX, $0x00000180
2764 JBE sealAVX2Tail384
2765 CMPQ BX, $0x00000200
2766 JBE sealAVX2Tail512
2767
2768 // We have 448 bytes to hash, but main loop hashes 512 bytes at a time - perform some rounds, before the main loop
2769 VMOVDQA ·chacha20Constants<>+0(SB), Y0
2770 VMOVDQA Y0, Y5
2771 VMOVDQA Y0, Y6
2772 VMOVDQA Y0, Y7
2773 VMOVDQA 32(BP), Y14
2774 VMOVDQA Y14, Y9
2775 VMOVDQA Y14, Y10
2776 VMOVDQA Y14, Y11
2777 VMOVDQA 64(BP), Y12
2778 VMOVDQA Y12, Y13
2779 VMOVDQA Y12, Y8
2780 VMOVDQA Y12, Y15
2781 VMOVDQA 192(BP), Y4
2782 VPADDD ·avx2IncMask<>+0(SB), Y4, Y4
2783 VPADDD ·avx2IncMask<>+0(SB), Y4, Y1
2784 VPADDD ·avx2IncMask<>+0(SB), Y1, Y2
2785 VPADDD ·avx2IncMask<>+0(SB), Y2, Y3
2786 VMOVDQA Y4, 96(BP)
2787 VMOVDQA Y1, 128(BP)
2788 VMOVDQA Y2, 160(BP)
2789 VMOVDQA Y3, 192(BP)
2790 VMOVDQA Y15, 224(BP)
2791 VPADDD Y14, Y0, Y0
2792 VPXOR Y0, Y4, Y4
2793 VPSHUFB ·rol16<>+0(SB), Y4, Y4
2794 VPADDD Y4, Y12, Y12
2795 VPXOR Y12, Y14, Y14
2796 VPSLLD $0x0c, Y14, Y15
2797 VPSRLD $0x14, Y14, Y14
2798 VPXOR Y15, Y14, Y14
2799 VPADDD Y14, Y0, Y0
2800 VPXOR Y0, Y4, Y4
2801 VPSHUFB ·rol8<>+0(SB), Y4, Y4
2802 VPADDD Y4, Y12, Y12
2803 VPXOR Y12, Y14, Y14
2804 VPSLLD $0x07, Y14, Y15
2805 VPSRLD $0x19, Y14, Y14
2806 VPXOR Y15, Y14, Y14
2807 VPADDD Y9, Y5, Y5
2808 VPXOR Y5, Y1, Y1
2809 VPSHUFB ·rol16<>+0(SB), Y1, Y1
2810 VPADDD Y1, Y13, Y13
2811 VPXOR Y13, Y9, Y9
2812 VPSLLD $0x0c, Y9, Y15
2813 VPSRLD $0x14, Y9, Y9
2814 VPXOR Y15, Y9, Y9
2815 VPADDD Y9, Y5, Y5
2816 VPXOR Y5, Y1, Y1
2817 VPSHUFB ·rol8<>+0(SB), Y1, Y1
2818 VPADDD Y1, Y13, Y13
2819 VPXOR Y13, Y9, Y9
2820 VPSLLD $0x07, Y9, Y15
2821 VPSRLD $0x19, Y9, Y9
2822 VPXOR Y15, Y9, Y9
2823 VPADDD Y10, Y6, Y6
2824 VPXOR Y6, Y2, Y2
2825 VPSHUFB ·rol16<>+0(SB), Y2, Y2
2826 VPADDD Y2, Y8, Y8
2827 VPXOR Y8, Y10, Y10
2828 VPSLLD $0x0c, Y10, Y15
2829 VPSRLD $0x14, Y10, Y10
2830 VPXOR Y15, Y10, Y10
2831 VPADDD Y10, Y6, Y6
2832 VPXOR Y6, Y2, Y2
2833 VPSHUFB ·rol8<>+0(SB), Y2, Y2
2834 VPADDD Y2, Y8, Y8
2835 VPXOR Y8, Y10, Y10
2836 VPSLLD $0x07, Y10, Y15
2837 VPSRLD $0x19, Y10, Y10
2838 VPXOR Y15, Y10, Y10
2839 VMOVDQA 224(BP), Y15
2840 VMOVDQA Y13, 224(BP)
2841 VPADDD Y11, Y7, Y7
2842 VPXOR Y7, Y3, Y3
2843 VPSHUFB ·rol16<>+0(SB), Y3, Y3
2844 VPADDD Y3, Y15, Y15
2845 VPXOR Y15, Y11, Y11
2846 VPSLLD $0x0c, Y11, Y13
2847 VPSRLD $0x14, Y11, Y11
2848 VPXOR Y13, Y11, Y11
2849 VPADDD Y11, Y7, Y7
2850 VPXOR Y7, Y3, Y3
2851 VPSHUFB ·rol8<>+0(SB), Y3, Y3
2852 VPADDD Y3, Y15, Y15
2853 VPXOR Y15, Y11, Y11
2854 VPSLLD $0x07, Y11, Y13
2855 VPSRLD $0x19, Y11, Y11
2856 VPXOR Y13, Y11, Y11
2857 VMOVDQA 224(BP), Y13
2858 VPALIGNR $0x04, Y14, Y14, Y14
2859 VPALIGNR $0x08, Y12, Y12, Y12
2860 VPALIGNR $0x0c, Y4, Y4, Y4
2861 VPALIGNR $0x04, Y9, Y9, Y9
2862 VPALIGNR $0x08, Y13, Y13, Y13
2863 VPALIGNR $0x0c, Y1, Y1, Y1
2864 VPALIGNR $0x04, Y10, Y10, Y10
2865 VPALIGNR $0x08, Y8, Y8, Y8
2866 VPALIGNR $0x0c, Y2, Y2, Y2
2867 VPALIGNR $0x04, Y11, Y11, Y11
2868 VPALIGNR $0x08, Y15, Y15, Y15
2869 VPALIGNR $0x0c, Y3, Y3, Y3
2870 VMOVDQA Y15, 224(BP)
2871 VPADDD Y14, Y0, Y0
2872 VPXOR Y0, Y4, Y4
2873 VPSHUFB ·rol16<>+0(SB), Y4, Y4
2874 VPADDD Y4, Y12, Y12
2875 VPXOR Y12, Y14, Y14
2876 VPSLLD $0x0c, Y14, Y15
2877 VPSRLD $0x14, Y14, Y14
2878 VPXOR Y15, Y14, Y14
2879 VPADDD Y14, Y0, Y0
2880 VPXOR Y0, Y4, Y4
2881 VPSHUFB ·rol8<>+0(SB), Y4, Y4
2882 VPADDD Y4, Y12, Y12
2883 VPXOR Y12, Y14, Y14
2884 VPSLLD $0x07, Y14, Y15
2885 VPSRLD $0x19, Y14, Y14
2886 VPXOR Y15, Y14, Y14
2887 VPADDD Y9, Y5, Y5
2888 VPXOR Y5, Y1, Y1
2889 VPSHUFB ·rol16<>+0(SB), Y1, Y1
2890 VPADDD Y1, Y13, Y13
2891 VPXOR Y13, Y9, Y9
2892 VPSLLD $0x0c, Y9, Y15
2893 VPSRLD $0x14, Y9, Y9
2894 VPXOR Y15, Y9, Y9
2895 VPADDD Y9, Y5, Y5
2896 VPXOR Y5, Y1, Y1
2897 VPSHUFB ·rol8<>+0(SB), Y1, Y1
2898 VPADDD Y1, Y13, Y13
2899 VPXOR Y13, Y9, Y9
2900 VPSLLD $0x07, Y9, Y15
2901 VPSRLD $0x19, Y9, Y9
2902 VPXOR Y15, Y9, Y9
2903 VPADDD Y10, Y6, Y6
2904 VPXOR Y6, Y2, Y2
2905 VPSHUFB ·rol16<>+0(SB), Y2, Y2
2906 VPADDD Y2, Y8, Y8
2907 VPXOR Y8, Y10, Y10
2908 VPSLLD $0x0c, Y10, Y15
2909 VPSRLD $0x14, Y10, Y10
2910 VPXOR Y15, Y10, Y10
2911 VPADDD Y10, Y6, Y6
2912 VPXOR Y6, Y2, Y2
2913 VPSHUFB ·rol8<>+0(SB), Y2, Y2
2914 VPADDD Y2, Y8, Y8
2915 VPXOR Y8, Y10, Y10
2916 VPSLLD $0x07, Y10, Y15
2917 VPSRLD $0x19, Y10, Y10
2918 VPXOR Y15, Y10, Y10
2919 VMOVDQA 224(BP), Y15
2920 VMOVDQA Y13, 224(BP)
2921 VPADDD Y11, Y7, Y7
2922 VPXOR Y7, Y3, Y3
2923 VPSHUFB ·rol16<>+0(SB), Y3, Y3
2924 VPADDD Y3, Y15, Y15
2925 VPXOR Y15, Y11, Y11
2926 VPSLLD $0x0c, Y11, Y13
2927 VPSRLD $0x14, Y11, Y11
2928 VPXOR Y13, Y11, Y11
2929 VPADDD Y11, Y7, Y7
2930 VPXOR Y7, Y3, Y3
2931 VPSHUFB ·rol8<>+0(SB), Y3, Y3
2932 VPADDD Y3, Y15, Y15
2933 VPXOR Y15, Y11, Y11
2934 VPSLLD $0x07, Y11, Y13
2935 VPSRLD $0x19, Y11, Y11
2936 VPXOR Y13, Y11, Y11
2937 VMOVDQA 224(BP), Y13
2938 VPALIGNR $0x0c, Y14, Y14, Y14
2939 VPALIGNR $0x08, Y12, Y12, Y12
2940 VPALIGNR $0x04, Y4, Y4, Y4
2941 VPALIGNR $0x0c, Y9, Y9, Y9
2942 VPALIGNR $0x08, Y13, Y13, Y13
2943 VPALIGNR $0x04, Y1, Y1, Y1
2944 VPALIGNR $0x0c, Y10, Y10, Y10
2945 VPALIGNR $0x08, Y8, Y8, Y8
2946 VPALIGNR $0x04, Y2, Y2, Y2
2947 VPALIGNR $0x0c, Y11, Y11, Y11
2948 VPALIGNR $0x08, Y15, Y15, Y15
2949 VPALIGNR $0x04, Y3, Y3, Y3
2950 VPADDD Y14, Y0, Y0
2951 VPADDD Y9, Y5, Y5
2952 VPADDD Y10, Y6, Y6
2953 VPADDD Y11, Y7, Y7
2954 VPXOR Y0, Y4, Y4
2955 VPXOR Y5, Y1, Y1
2956 VPXOR Y6, Y2, Y2
2957 VPXOR Y7, Y3, Y3
2958 VPSHUFB ·rol16<>+0(SB), Y4, Y4
2959 VPSHUFB ·rol16<>+0(SB), Y1, Y1
2960 VPSHUFB ·rol16<>+0(SB), Y2, Y2
2961 VPSHUFB ·rol16<>+0(SB), Y3, Y3
2962 VPADDD Y4, Y12, Y12
2963 VPADDD Y1, Y13, Y13
2964 VPADDD Y2, Y8, Y8
2965 VPADDD Y3, Y15, Y15
2966 VPXOR Y12, Y14, Y14
2967 VPXOR Y13, Y9, Y9
2968 VPXOR Y8, Y10, Y10
2969 VPXOR Y15, Y11, Y11
2970 VMOVDQA Y15, 224(BP)
2971 VPSLLD $0x0c, Y14, Y15
2972 VPSRLD $0x14, Y14, Y14
2973 VPXOR Y15, Y14, Y14
2974 VPSLLD $0x0c, Y9, Y15
2975 VPSRLD $0x14, Y9, Y9
2976 VPXOR Y15, Y9, Y9
2977 VPSLLD $0x0c, Y10, Y15
2978 VPSRLD $0x14, Y10, Y10
2979 VPXOR Y15, Y10, Y10
2980 VPSLLD $0x0c, Y11, Y15
2981 VPSRLD $0x14, Y11, Y11
2982 VPXOR Y15, Y11, Y11
2983 VMOVDQA 224(BP), Y15
2984 SUBQ $0x10, DI
2985 MOVQ $0x00000009, CX
2986 JMP sealAVX2InternalLoopStart
2987
2988 sealAVX2MainLoop:
2989 VMOVDQU ·chacha20Constants<>+0(SB), Y0
2990 VMOVDQA Y0, Y5
2991 VMOVDQA Y0, Y6
2992 VMOVDQA Y0, Y7
2993 VMOVDQA 32(BP), Y14
2994 VMOVDQA Y14, Y9
2995 VMOVDQA Y14, Y10
2996 VMOVDQA Y14, Y11
2997 VMOVDQA 64(BP), Y12
2998 VMOVDQA Y12, Y13
2999 VMOVDQA Y12, Y8
3000 VMOVDQA Y12, Y15
3001 VMOVDQA 192(BP), Y4
3002 VPADDD ·avx2IncMask<>+0(SB), Y4, Y4
3003 VPADDD ·avx2IncMask<>+0(SB), Y4, Y1
3004 VPADDD ·avx2IncMask<>+0(SB), Y1, Y2
3005 VPADDD ·avx2IncMask<>+0(SB), Y2, Y3
3006 VMOVDQA Y4, 96(BP)
3007 VMOVDQA Y1, 128(BP)
3008 VMOVDQA Y2, 160(BP)
3009 VMOVDQA Y3, 192(BP)
3010 MOVQ $0x0000000a, CX
3011
3012 sealAVX2InternalLoop:
3013 ADDQ (DI), R10
3014 ADCQ 8(DI), R11
3015 ADCQ $0x01, R12
3016 VPADDD Y14, Y0, Y0
3017 VPADDD Y9, Y5, Y5
3018 VPADDD Y10, Y6, Y6
3019 VPADDD Y11, Y7, Y7
3020 MOVQ (BP), DX
3021 MOVQ DX, R15
3022 MULXQ R10, R13, R14
3023 IMULQ R12, R15
3024 MULXQ R11, AX, DX
3025 ADDQ AX, R14
3026 ADCQ DX, R15
3027 VPXOR Y0, Y4, Y4
3028 VPXOR Y5, Y1, Y1
3029 VPXOR Y6, Y2, Y2
3030 VPXOR Y7, Y3, Y3
3031 VPSHUFB ·rol16<>+0(SB), Y4, Y4
3032 VPSHUFB ·rol16<>+0(SB), Y1, Y1
3033 VPSHUFB ·rol16<>+0(SB), Y2, Y2
3034 VPSHUFB ·rol16<>+0(SB), Y3, Y3
3035 MOVQ 8(BP), DX
3036 MULXQ R10, R10, AX
3037 ADDQ R10, R14
3038 MULXQ R11, R11, R8
3039 ADCQ R11, R15
3040 ADCQ $0x00, R8
3041 VPADDD Y4, Y12, Y12
3042 VPADDD Y1, Y13, Y13
3043 VPADDD Y2, Y8, Y8
3044 VPADDD Y3, Y15, Y15
3045 VPXOR Y12, Y14, Y14
3046 VPXOR Y13, Y9, Y9
3047 VPXOR Y8, Y10, Y10
3048 VPXOR Y15, Y11, Y11
3049 IMULQ R12, DX
3050 ADDQ AX, R15
3051 ADCQ DX, R8
3052 VMOVDQA Y15, 224(BP)
3053 VPSLLD $0x0c, Y14, Y15
3054 VPSRLD $0x14, Y14, Y14
3055 VPXOR Y15, Y14, Y14
3056 VPSLLD $0x0c, Y9, Y15
3057 VPSRLD $0x14, Y9, Y9
3058 VPXOR Y15, Y9, Y9
3059 VPSLLD $0x0c, Y10, Y15
3060 VPSRLD $0x14, Y10, Y10
3061 VPXOR Y15, Y10, Y10
3062 VPSLLD $0x0c, Y11, Y15
3063 VPSRLD $0x14, Y11, Y11
3064 VPXOR Y15, Y11, Y11
3065 VMOVDQA 224(BP), Y15
3066 MOVQ R13, R10
3067 MOVQ R14, R11
3068 MOVQ R15, R12
3069 ANDQ $0x03, R12
3070 MOVQ R15, R13
3071 ANDQ $-4, R13
3072 MOVQ R8, R14
3073 SHRQ $0x02, R8, R15
3074 SHRQ $0x02, R8
3075 ADDQ R13, R10
3076 ADCQ R14, R11
3077 ADCQ $0x00, R12
3078 ADDQ R15, R10
3079 ADCQ R8, R11
3080 ADCQ $0x00, R12
3081
3082 sealAVX2InternalLoopStart:
3083 VPADDD Y14, Y0, Y0
3084 VPADDD Y9, Y5, Y5
3085 VPADDD Y10, Y6, Y6
3086 VPADDD Y11, Y7, Y7
3087 VPXOR Y0, Y4, Y4
3088 VPXOR Y5, Y1, Y1
3089 VPXOR Y6, Y2, Y2
3090 VPXOR Y7, Y3, Y3
3091 VPSHUFB ·rol8<>+0(SB), Y4, Y4
3092 VPSHUFB ·rol8<>+0(SB), Y1, Y1
3093 VPSHUFB ·rol8<>+0(SB), Y2, Y2
3094 VPSHUFB ·rol8<>+0(SB), Y3, Y3
3095 ADDQ 16(DI), R10
3096 ADCQ 24(DI), R11
3097 ADCQ $0x01, R12
3098 VPADDD Y4, Y12, Y12
3099 VPADDD Y1, Y13, Y13
3100 VPADDD Y2, Y8, Y8
3101 VPADDD Y3, Y15, Y15
3102 MOVQ (BP), DX
3103 MOVQ DX, R15
3104 MULXQ R10, R13, R14
3105 IMULQ R12, R15
3106 MULXQ R11, AX, DX
3107 ADDQ AX, R14
3108 ADCQ DX, R15
3109 VPXOR Y12, Y14, Y14
3110 VPXOR Y13, Y9, Y9
3111 VPXOR Y8, Y10, Y10
3112 VPXOR Y15, Y11, Y11
3113 VMOVDQA Y15, 224(BP)
3114 VPSLLD $0x07, Y14, Y15
3115 VPSRLD $0x19, Y14, Y14
3116 VPXOR Y15, Y14, Y14
3117 VPSLLD $0x07, Y9, Y15
3118 VPSRLD $0x19, Y9, Y9
3119 VPXOR Y15, Y9, Y9
3120 VPSLLD $0x07, Y10, Y15
3121 VPSRLD $0x19, Y10, Y10
3122 VPXOR Y15, Y10, Y10
3123 VPSLLD $0x07, Y11, Y15
3124 VPSRLD $0x19, Y11, Y11
3125 VPXOR Y15, Y11, Y11
3126 VMOVDQA 224(BP), Y15
3127 MOVQ 8(BP), DX
3128 MULXQ R10, R10, AX
3129 ADDQ R10, R14
3130 MULXQ R11, R11, R8
3131 ADCQ R11, R15
3132 ADCQ $0x00, R8
3133 VPALIGNR $0x04, Y14, Y14, Y14
3134 VPALIGNR $0x04, Y9, Y9, Y9
3135 VPALIGNR $0x04, Y10, Y10, Y10
3136 VPALIGNR $0x04, Y11, Y11, Y11
3137 VPALIGNR $0x08, Y12, Y12, Y12
3138 VPALIGNR $0x08, Y13, Y13, Y13
3139 VPALIGNR $0x08, Y8, Y8, Y8
3140 VPALIGNR $0x08, Y15, Y15, Y15
3141 VPALIGNR $0x0c, Y4, Y4, Y4
3142 VPALIGNR $0x0c, Y1, Y1, Y1
3143 VPALIGNR $0x0c, Y2, Y2, Y2
3144 VPALIGNR $0x0c, Y3, Y3, Y3
3145 VPADDD Y14, Y0, Y0
3146 VPADDD Y9, Y5, Y5
3147 VPADDD Y10, Y6, Y6
3148 VPADDD Y11, Y7, Y7
3149 IMULQ R12, DX
3150 ADDQ AX, R15
3151 ADCQ DX, R8
3152 VPXOR Y0, Y4, Y4
3153 VPXOR Y5, Y1, Y1
3154 VPXOR Y6, Y2, Y2
3155 VPXOR Y7, Y3, Y3
3156 VPSHUFB ·rol16<>+0(SB), Y4, Y4
3157 VPSHUFB ·rol16<>+0(SB), Y1, Y1
3158 VPSHUFB ·rol16<>+0(SB), Y2, Y2
3159 VPSHUFB ·rol16<>+0(SB), Y3, Y3
3160 MOVQ R13, R10
3161 MOVQ R14, R11
3162 MOVQ R15, R12
3163 ANDQ $0x03, R12
3164 MOVQ R15, R13
3165 ANDQ $-4, R13
3166 MOVQ R8, R14
3167 SHRQ $0x02, R8, R15
3168 SHRQ $0x02, R8
3169 ADDQ R13, R10
3170 ADCQ R14, R11
3171 ADCQ $0x00, R12
3172 ADDQ R15, R10
3173 ADCQ R8, R11
3174 ADCQ $0x00, R12
3175 VPADDD Y4, Y12, Y12
3176 VPADDD Y1, Y13, Y13
3177 VPADDD Y2, Y8, Y8
3178 VPADDD Y3, Y15, Y15
3179 VPXOR Y12, Y14, Y14
3180 VPXOR Y13, Y9, Y9
3181 VPXOR Y8, Y10, Y10
3182 VPXOR Y15, Y11, Y11
3183 ADDQ 32(DI), R10
3184 ADCQ 40(DI), R11
3185 ADCQ $0x01, R12
3186 LEAQ 48(DI), DI
3187 VMOVDQA Y15, 224(BP)
3188 VPSLLD $0x0c, Y14, Y15
3189 VPSRLD $0x14, Y14, Y14
3190 VPXOR Y15, Y14, Y14
3191 VPSLLD $0x0c, Y9, Y15
3192 VPSRLD $0x14, Y9, Y9
3193 VPXOR Y15, Y9, Y9
3194 VPSLLD $0x0c, Y10, Y15
3195 VPSRLD $0x14, Y10, Y10
3196 VPXOR Y15, Y10, Y10
3197 VPSLLD $0x0c, Y11, Y15
3198 VPSRLD $0x14, Y11, Y11
3199 VPXOR Y15, Y11, Y11
3200 VMOVDQA 224(BP), Y15
3201 MOVQ (BP), DX
3202 MOVQ DX, R15
3203 MULXQ R10, R13, R14
3204 IMULQ R12, R15
3205 MULXQ R11, AX, DX
3206 ADDQ AX, R14
3207 ADCQ DX, R15
3208 VPADDD Y14, Y0, Y0
3209 VPADDD Y9, Y5, Y5
3210 VPADDD Y10, Y6, Y6
3211 VPADDD Y11, Y7, Y7
3212 VPXOR Y0, Y4, Y4
3213 VPXOR Y5, Y1, Y1
3214 VPXOR Y6, Y2, Y2
3215 VPXOR Y7, Y3, Y3
3216 MOVQ 8(BP), DX
3217 MULXQ R10, R10, AX
3218 ADDQ R10, R14
3219 MULXQ R11, R11, R8
3220 ADCQ R11, R15
3221 ADCQ $0x00, R8
3222 VPSHUFB ·rol8<>+0(SB), Y4, Y4
3223 VPSHUFB ·rol8<>+0(SB), Y1, Y1
3224 VPSHUFB ·rol8<>+0(SB), Y2, Y2
3225 VPSHUFB ·rol8<>+0(SB), Y3, Y3
3226 VPADDD Y4, Y12, Y12
3227 VPADDD Y1, Y13, Y13
3228 VPADDD Y2, Y8, Y8
3229 VPADDD Y3, Y15, Y15
3230 IMULQ R12, DX
3231 ADDQ AX, R15
3232 ADCQ DX, R8
3233 VPXOR Y12, Y14, Y14
3234 VPXOR Y13, Y9, Y9
3235 VPXOR Y8, Y10, Y10
3236 VPXOR Y15, Y11, Y11
3237 VMOVDQA Y15, 224(BP)
3238 VPSLLD $0x07, Y14, Y15
3239 VPSRLD $0x19, Y14, Y14
3240 VPXOR Y15, Y14, Y14
3241 VPSLLD $0x07, Y9, Y15
3242 VPSRLD $0x19, Y9, Y9
3243 VPXOR Y15, Y9, Y9
3244 VPSLLD $0x07, Y10, Y15
3245 VPSRLD $0x19, Y10, Y10
3246 VPXOR Y15, Y10, Y10
3247 VPSLLD $0x07, Y11, Y15
3248 VPSRLD $0x19, Y11, Y11
3249 VPXOR Y15, Y11, Y11
3250 VMOVDQA 224(BP), Y15
3251 MOVQ R13, R10
3252 MOVQ R14, R11
3253 MOVQ R15, R12
3254 ANDQ $0x03, R12
3255 MOVQ R15, R13
3256 ANDQ $-4, R13
3257 MOVQ R8, R14
3258 SHRQ $0x02, R8, R15
3259 SHRQ $0x02, R8
3260 ADDQ R13, R10
3261 ADCQ R14, R11
3262 ADCQ $0x00, R12
3263 ADDQ R15, R10
3264 ADCQ R8, R11
3265 ADCQ $0x00, R12
3266 VPALIGNR $0x0c, Y14, Y14, Y14
3267 VPALIGNR $0x0c, Y9, Y9, Y9
3268 VPALIGNR $0x0c, Y10, Y10, Y10
3269 VPALIGNR $0x0c, Y11, Y11, Y11
3270 VPALIGNR $0x08, Y12, Y12, Y12
3271 VPALIGNR $0x08, Y13, Y13, Y13
3272 VPALIGNR $0x08, Y8, Y8, Y8
3273 VPALIGNR $0x08, Y15, Y15, Y15
3274 VPALIGNR $0x04, Y4, Y4, Y4
3275 VPALIGNR $0x04, Y1, Y1, Y1
3276 VPALIGNR $0x04, Y2, Y2, Y2
3277 VPALIGNR $0x04, Y3, Y3, Y3
3278 DECQ CX
3279 JNE sealAVX2InternalLoop
3280 VPADDD ·chacha20Constants<>+0(SB), Y0, Y0
3281 VPADDD ·chacha20Constants<>+0(SB), Y5, Y5
3282 VPADDD ·chacha20Constants<>+0(SB), Y6, Y6
3283 VPADDD ·chacha20Constants<>+0(SB), Y7, Y7
3284 VPADDD 32(BP), Y14, Y14
3285 VPADDD 32(BP), Y9, Y9
3286 VPADDD 32(BP), Y10, Y10
3287 VPADDD 32(BP), Y11, Y11
3288 VPADDD 64(BP), Y12, Y12
3289 VPADDD 64(BP), Y13, Y13
3290 VPADDD 64(BP), Y8, Y8
3291 VPADDD 64(BP), Y15, Y15
3292 VPADDD 96(BP), Y4, Y4
3293 VPADDD 128(BP), Y1, Y1
3294 VPADDD 160(BP), Y2, Y2
3295 VPADDD 192(BP), Y3, Y3
3296 VMOVDQA Y15, 224(BP)
3297
3298 // We only hashed 480 of the 512 bytes available - hash the remaining 32 here
3299 ADDQ (DI), R10
3300 ADCQ 8(DI), R11
3301 ADCQ $0x01, R12
3302 MOVQ (BP), DX
3303 MOVQ DX, R15
3304 MULXQ R10, R13, R14
3305 IMULQ R12, R15
3306 MULXQ R11, AX, DX
3307 ADDQ AX, R14
3308 ADCQ DX, R15
3309 MOVQ 8(BP), DX
3310 MULXQ R10, R10, AX
3311 ADDQ R10, R14
3312 MULXQ R11, R11, R8
3313 ADCQ R11, R15
3314 ADCQ $0x00, R8
3315 IMULQ R12, DX
3316 ADDQ AX, R15
3317 ADCQ DX, R8
3318 MOVQ R13, R10
3319 MOVQ R14, R11
3320 MOVQ R15, R12
3321 ANDQ $0x03, R12
3322 MOVQ R15, R13
3323 ANDQ $-4, R13
3324 MOVQ R8, R14
3325 SHRQ $0x02, R8, R15
3326 SHRQ $0x02, R8
3327 ADDQ R13, R10
3328 ADCQ R14, R11
3329 ADCQ $0x00, R12
3330 ADDQ R15, R10
3331 ADCQ R8, R11
3332 ADCQ $0x00, R12
3333 LEAQ 32(DI), DI
3334 VPERM2I128 $0x02, Y0, Y14, Y15
3335 VPERM2I128 $0x13, Y0, Y14, Y14
3336 VPERM2I128 $0x02, Y12, Y4, Y0
3337 VPERM2I128 $0x13, Y12, Y4, Y12
3338 VPXOR (SI), Y15, Y15
3339 VPXOR 32(SI), Y0, Y0
3340 VPXOR 64(SI), Y14, Y14
3341 VPXOR 96(SI), Y12, Y12
3342 VMOVDQU Y15, (DI)
3343 VMOVDQU Y0, 32(DI)
3344 VMOVDQU Y14, 64(DI)
3345 VMOVDQU Y12, 96(DI)
3346 VPERM2I128 $0x02, Y5, Y9, Y0
3347 VPERM2I128 $0x02, Y13, Y1, Y14
3348 VPERM2I128 $0x13, Y5, Y9, Y12
3349 VPERM2I128 $0x13, Y13, Y1, Y4
3350 VPXOR 128(SI), Y0, Y0
3351 VPXOR 160(SI), Y14, Y14
3352 VPXOR 192(SI), Y12, Y12
3353 VPXOR 224(SI), Y4, Y4
3354 VMOVDQU Y0, 128(DI)
3355 VMOVDQU Y14, 160(DI)
3356 VMOVDQU Y12, 192(DI)
3357 VMOVDQU Y4, 224(DI)
3358
3359 // and here
3360 ADDQ -16(DI), R10
3361 ADCQ -8(DI), R11
3362 ADCQ $0x01, R12
3363 MOVQ (BP), DX
3364 MOVQ DX, R15
3365 MULXQ R10, R13, R14
3366 IMULQ R12, R15
3367 MULXQ R11, AX, DX
3368 ADDQ AX, R14
3369 ADCQ DX, R15
3370 MOVQ 8(BP), DX
3371 MULXQ R10, R10, AX
3372 ADDQ R10, R14
3373 MULXQ R11, R11, R8
3374 ADCQ R11, R15
3375 ADCQ $0x00, R8
3376 IMULQ R12, DX
3377 ADDQ AX, R15
3378 ADCQ DX, R8
3379 MOVQ R13, R10
3380 MOVQ R14, R11
3381 MOVQ R15, R12
3382 ANDQ $0x03, R12
3383 MOVQ R15, R13
3384 ANDQ $-4, R13
3385 MOVQ R8, R14
3386 SHRQ $0x02, R8, R15
3387 SHRQ $0x02, R8
3388 ADDQ R13, R10
3389 ADCQ R14, R11
3390 ADCQ $0x00, R12
3391 ADDQ R15, R10
3392 ADCQ R8, R11
3393 ADCQ $0x00, R12
3394 VPERM2I128 $0x02, Y6, Y10, Y0
3395 VPERM2I128 $0x02, Y8, Y2, Y14
3396 VPERM2I128 $0x13, Y6, Y10, Y12
3397 VPERM2I128 $0x13, Y8, Y2, Y4
3398 VPXOR 256(SI), Y0, Y0
3399 VPXOR 288(SI), Y14, Y14
3400 VPXOR 320(SI), Y12, Y12
3401 VPXOR 352(SI), Y4, Y4
3402 VMOVDQU Y0, 256(DI)
3403 VMOVDQU Y14, 288(DI)
3404 VMOVDQU Y12, 320(DI)
3405 VMOVDQU Y4, 352(DI)
3406 VPERM2I128 $0x02, Y7, Y11, Y0
3407 VPERM2I128 $0x02, 224(BP), Y3, Y14
3408 VPERM2I128 $0x13, Y7, Y11, Y12
3409 VPERM2I128 $0x13, 224(BP), Y3, Y4
3410 VPXOR 384(SI), Y0, Y0
3411 VPXOR 416(SI), Y14, Y14
3412 VPXOR 448(SI), Y12, Y12
3413 VPXOR 480(SI), Y4, Y4
3414 VMOVDQU Y0, 384(DI)
3415 VMOVDQU Y14, 416(DI)
3416 VMOVDQU Y12, 448(DI)
3417 VMOVDQU Y4, 480(DI)
3418 LEAQ 512(SI), SI
3419 SUBQ $0x00000200, BX
3420 CMPQ BX, $0x00000200
3421 JG sealAVX2MainLoop
3422
3423 // Tail can only hash 480 bytes
3424 ADDQ (DI), R10
3425 ADCQ 8(DI), R11
3426 ADCQ $0x01, R12
3427 MOVQ (BP), DX
3428 MOVQ DX, R15
3429 MULXQ R10, R13, R14
3430 IMULQ R12, R15
3431 MULXQ R11, AX, DX
3432 ADDQ AX, R14
3433 ADCQ DX, R15
3434 MOVQ 8(BP), DX
3435 MULXQ R10, R10, AX
3436 ADDQ R10, R14
3437 MULXQ R11, R11, R8
3438 ADCQ R11, R15
3439 ADCQ $0x00, R8
3440 IMULQ R12, DX
3441 ADDQ AX, R15
3442 ADCQ DX, R8
3443 MOVQ R13, R10
3444 MOVQ R14, R11
3445 MOVQ R15, R12
3446 ANDQ $0x03, R12
3447 MOVQ R15, R13
3448 ANDQ $-4, R13
3449 MOVQ R8, R14
3450 SHRQ $0x02, R8, R15
3451 SHRQ $0x02, R8
3452 ADDQ R13, R10
3453 ADCQ R14, R11
3454 ADCQ $0x00, R12
3455 ADDQ R15, R10
3456 ADCQ R8, R11
3457 ADCQ $0x00, R12
3458 ADDQ 16(DI), R10
3459 ADCQ 24(DI), R11
3460 ADCQ $0x01, R12
3461 MOVQ (BP), DX
3462 MOVQ DX, R15
3463 MULXQ R10, R13, R14
3464 IMULQ R12, R15
3465 MULXQ R11, AX, DX
3466 ADDQ AX, R14
3467 ADCQ DX, R15
3468 MOVQ 8(BP), DX
3469 MULXQ R10, R10, AX
3470 ADDQ R10, R14
3471 MULXQ R11, R11, R8
3472 ADCQ R11, R15
3473 ADCQ $0x00, R8
3474 IMULQ R12, DX
3475 ADDQ AX, R15
3476 ADCQ DX, R8
3477 MOVQ R13, R10
3478 MOVQ R14, R11
3479 MOVQ R15, R12
3480 ANDQ $0x03, R12
3481 MOVQ R15, R13
3482 ANDQ $-4, R13
3483 MOVQ R8, R14
3484 SHRQ $0x02, R8, R15
3485 SHRQ $0x02, R8
3486 ADDQ R13, R10
3487 ADCQ R14, R11
3488 ADCQ $0x00, R12
3489 ADDQ R15, R10
3490 ADCQ R8, R11
3491 ADCQ $0x00, R12
3492 LEAQ 32(DI), DI
3493 MOVQ $0x0000000a, CX
3494 MOVQ $0x00000000, R9
3495 CMPQ BX, $0x80
3496 JBE sealAVX2Tail128
3497 CMPQ BX, $0x00000100
3498 JBE sealAVX2Tail256
3499 CMPQ BX, $0x00000180
3500 JBE sealAVX2Tail384
3501 JMP sealAVX2Tail512
3502
3503 sealSSETail:
3504 TESTQ BX, BX
3505 JE sealSSEFinalize
3506
3507 // We can only load the PT one byte at a time to avoid read after end of buffer
3508 MOVQ BX, R9
3509 SHLQ $0x04, R9
3510 LEAQ ·andMask<>+0(SB), R13
3511 MOVQ BX, CX
3512 LEAQ -1(SI)(BX*1), SI
3513 XORQ R15, R15
3514 XORQ R8, R8
3515 XORQ AX, AX
3516
3517 sealSSETailLoadLoop:
3518 SHLQ $0x08, R15, R8
3519 SHLQ $0x08, R15
3520 MOVB (SI), AX
3521 XORQ AX, R15
3522 LEAQ -1(SI), SI
3523 DECQ CX
3524 JNE sealSSETailLoadLoop
3525 MOVQ R15, 64(BP)
3526 MOVQ R8, 72(BP)
3527 PXOR 64(BP), X1
3528 MOVOU X1, (DI)
3529 MOVOU -16(R13)(R9*1), X12
3530 PAND X12, X1
3531 MOVQ X1, R13
3532 PSRLDQ $0x08, X1
3533 MOVQ X1, R14
3534 ADDQ R13, R10
3535 ADCQ R14, R11
3536 ADCQ $0x01, R12
3537 MOVQ (BP), AX
3538 MOVQ AX, R15
3539 MULQ R10
3540 MOVQ AX, R13
3541 MOVQ DX, R14
3542 MOVQ (BP), AX
3543 MULQ R11
3544 IMULQ R12, R15
3545 ADDQ AX, R14
3546 ADCQ DX, R15
3547 MOVQ 8(BP), AX
3548 MOVQ AX, R8
3549 MULQ R10
3550 ADDQ AX, R14
3551 ADCQ $0x00, DX
3552 MOVQ DX, R10
3553 MOVQ 8(BP), AX
3554 MULQ R11
3555 ADDQ AX, R15
3556 ADCQ $0x00, DX
3557 IMULQ R12, R8
3558 ADDQ R10, R15
3559 ADCQ DX, R8
3560 MOVQ R13, R10
3561 MOVQ R14, R11
3562 MOVQ R15, R12
3563 ANDQ $0x03, R12
3564 MOVQ R15, R13
3565 ANDQ $-4, R13
3566 MOVQ R8, R14
3567 SHRQ $0x02, R8, R15
3568 SHRQ $0x02, R8
3569 ADDQ R13, R10
3570 ADCQ R14, R11
3571 ADCQ $0x00, R12
3572 ADDQ R15, R10
3573 ADCQ R8, R11
3574 ADCQ $0x00, R12
3575 ADDQ BX, DI
3576
3577 sealSSEFinalize:
3578 // Hash in the buffer lengths
3579 ADDQ ad_len+80(FP), R10
3580 ADCQ src_len+56(FP), R11
3581 ADCQ $0x01, R12
3582 MOVQ (BP), AX
3583 MOVQ AX, R15
3584 MULQ R10
3585 MOVQ AX, R13
3586 MOVQ DX, R14
3587 MOVQ (BP), AX
3588 MULQ R11
3589 IMULQ R12, R15
3590 ADDQ AX, R14
3591 ADCQ DX, R15
3592 MOVQ 8(BP), AX
3593 MOVQ AX, R8
3594 MULQ R10
3595 ADDQ AX, R14
3596 ADCQ $0x00, DX
3597 MOVQ DX, R10
3598 MOVQ 8(BP), AX
3599 MULQ R11
3600 ADDQ AX, R15
3601 ADCQ $0x00, DX
3602 IMULQ R12, R8
3603 ADDQ R10, R15
3604 ADCQ DX, R8
3605 MOVQ R13, R10
3606 MOVQ R14, R11
3607 MOVQ R15, R12
3608 ANDQ $0x03, R12
3609 MOVQ R15, R13
3610 ANDQ $-4, R13
3611 MOVQ R8, R14
3612 SHRQ $0x02, R8, R15
3613 SHRQ $0x02, R8
3614 ADDQ R13, R10
3615 ADCQ R14, R11
3616 ADCQ $0x00, R12
3617 ADDQ R15, R10
3618 ADCQ R8, R11
3619 ADCQ $0x00, R12
3620
3621 // Final reduce
3622 MOVQ R10, R13
3623 MOVQ R11, R14
3624 MOVQ R12, R15
3625 SUBQ $-5, R10
3626 SBBQ $-1, R11
3627 SBBQ $0x03, R12
3628 CMOVQCS R13, R10
3629 CMOVQCS R14, R11
3630 CMOVQCS R15, R12
3631
3632 // Add in the "s" part of the key
3633 ADDQ 16(BP), R10
3634 ADCQ 24(BP), R11
3635
3636 // Finally store the tag at the end of the message
3637 MOVQ R10, (DI)
3638 MOVQ R11, 8(DI)
3639 RET
3640
3641 seal192AVX2:
3642 VMOVDQA Y0, Y5
3643 VMOVDQA Y14, Y9
3644 VMOVDQA Y12, Y13
3645 VPADDD ·avx2IncMask<>+0(SB), Y4, Y1
3646 VMOVDQA Y0, Y6
3647 VMOVDQA Y14, Y10
3648 VMOVDQA Y12, Y8
3649 VMOVDQA Y4, Y2
3650 VMOVDQA Y1, Y15
3651 MOVQ $0x0000000a, R9
3652
3653 sealAVX2192InnerCipherLoop:
3654 VPADDD Y14, Y0, Y0
3655 VPXOR Y0, Y4, Y4
3656 VPSHUFB ·rol16<>+0(SB), Y4, Y4
3657 VPADDD Y4, Y12, Y12
3658 VPXOR Y12, Y14, Y14
3659 VPSLLD $0x0c, Y14, Y3
3660 VPSRLD $0x14, Y14, Y14
3661 VPXOR Y3, Y14, Y14
3662 VPADDD Y14, Y0, Y0
3663 VPXOR Y0, Y4, Y4
3664 VPSHUFB ·rol8<>+0(SB), Y4, Y4
3665 VPADDD Y4, Y12, Y12
3666 VPXOR Y12, Y14, Y14
3667 VPSLLD $0x07, Y14, Y3
3668 VPSRLD $0x19, Y14, Y14
3669 VPXOR Y3, Y14, Y14
3670 VPADDD Y9, Y5, Y5
3671 VPXOR Y5, Y1, Y1
3672 VPSHUFB ·rol16<>+0(SB), Y1, Y1
3673 VPADDD Y1, Y13, Y13
3674 VPXOR Y13, Y9, Y9
3675 VPSLLD $0x0c, Y9, Y3
3676 VPSRLD $0x14, Y9, Y9
3677 VPXOR Y3, Y9, Y9
3678 VPADDD Y9, Y5, Y5
3679 VPXOR Y5, Y1, Y1
3680 VPSHUFB ·rol8<>+0(SB), Y1, Y1
3681 VPADDD Y1, Y13, Y13
3682 VPXOR Y13, Y9, Y9
3683 VPSLLD $0x07, Y9, Y3
3684 VPSRLD $0x19, Y9, Y9
3685 VPXOR Y3, Y9, Y9
3686 VPALIGNR $0x04, Y14, Y14, Y14
3687 VPALIGNR $0x04, Y9, Y9, Y9
3688 VPALIGNR $0x08, Y12, Y12, Y12
3689 VPALIGNR $0x08, Y13, Y13, Y13
3690 VPALIGNR $0x0c, Y4, Y4, Y4
3691 VPALIGNR $0x0c, Y1, Y1, Y1
3692 VPADDD Y14, Y0, Y0
3693 VPXOR Y0, Y4, Y4
3694 VPSHUFB ·rol16<>+0(SB), Y4, Y4
3695 VPADDD Y4, Y12, Y12
3696 VPXOR Y12, Y14, Y14
3697 VPSLLD $0x0c, Y14, Y3
3698 VPSRLD $0x14, Y14, Y14
3699 VPXOR Y3, Y14, Y14
3700 VPADDD Y14, Y0, Y0
3701 VPXOR Y0, Y4, Y4
3702 VPSHUFB ·rol8<>+0(SB), Y4, Y4
3703 VPADDD Y4, Y12, Y12
3704 VPXOR Y12, Y14, Y14
3705 VPSLLD $0x07, Y14, Y3
3706 VPSRLD $0x19, Y14, Y14
3707 VPXOR Y3, Y14, Y14
3708 VPADDD Y9, Y5, Y5
3709 VPXOR Y5, Y1, Y1
3710 VPSHUFB ·rol16<>+0(SB), Y1, Y1
3711 VPADDD Y1, Y13, Y13
3712 VPXOR Y13, Y9, Y9
3713 VPSLLD $0x0c, Y9, Y3
3714 VPSRLD $0x14, Y9, Y9
3715 VPXOR Y3, Y9, Y9
3716 VPADDD Y9, Y5, Y5
3717 VPXOR Y5, Y1, Y1
3718 VPSHUFB ·rol8<>+0(SB), Y1, Y1
3719 VPADDD Y1, Y13, Y13
3720 VPXOR Y13, Y9, Y9
3721 VPSLLD $0x07, Y9, Y3
3722 VPSRLD $0x19, Y9, Y9
3723 VPXOR Y3, Y9, Y9
3724 VPALIGNR $0x0c, Y14, Y14, Y14
3725 VPALIGNR $0x0c, Y9, Y9, Y9
3726 VPALIGNR $0x08, Y12, Y12, Y12
3727 VPALIGNR $0x08, Y13, Y13, Y13
3728 VPALIGNR $0x04, Y4, Y4, Y4
3729 VPALIGNR $0x04, Y1, Y1, Y1
3730 DECQ R9
3731 JNE sealAVX2192InnerCipherLoop
3732 VPADDD Y6, Y0, Y0
3733 VPADDD Y6, Y5, Y5
3734 VPADDD Y10, Y14, Y14
3735 VPADDD Y10, Y9, Y9
3736 VPADDD Y8, Y12, Y12
3737 VPADDD Y8, Y13, Y13
3738 VPADDD Y2, Y4, Y4
3739 VPADDD Y15, Y1, Y1
3740 VPERM2I128 $0x02, Y0, Y14, Y3
3741
3742 // Clamp and store poly key
3743 VPAND ·polyClampMask<>+0(SB), Y3, Y3
3744 VMOVDQA Y3, (BP)
3745
3746 // Stream for up to 192 bytes
3747 VPERM2I128 $0x13, Y0, Y14, Y0
3748 VPERM2I128 $0x13, Y12, Y4, Y14
3749 VPERM2I128 $0x02, Y5, Y9, Y12
3750 VPERM2I128 $0x02, Y13, Y1, Y4
3751 VPERM2I128 $0x13, Y5, Y9, Y5
3752 VPERM2I128 $0x13, Y13, Y1, Y9
3753
3754 sealAVX2ShortSeal:
3755 // Hash aad
3756 MOVQ ad_len+80(FP), R9
3757 CALL polyHashADInternal<>(SB)
3758 XORQ CX, CX
3759
3760 sealAVX2SealHash:
3761 // itr1 holds the number of bytes encrypted but not yet hashed
3762 CMPQ CX, $0x10
3763 JB sealAVX2ShortSealLoop
3764 ADDQ (DI), R10
3765 ADCQ 8(DI), R11
3766 ADCQ $0x01, R12
3767 MOVQ (BP), AX
3768 MOVQ AX, R15
3769 MULQ R10
3770 MOVQ AX, R13
3771 MOVQ DX, R14
3772 MOVQ (BP), AX
3773 MULQ R11
3774 IMULQ R12, R15
3775 ADDQ AX, R14
3776 ADCQ DX, R15
3777 MOVQ 8(BP), AX
3778 MOVQ AX, R8
3779 MULQ R10
3780 ADDQ AX, R14
3781 ADCQ $0x00, DX
3782 MOVQ DX, R10
3783 MOVQ 8(BP), AX
3784 MULQ R11
3785 ADDQ AX, R15
3786 ADCQ $0x00, DX
3787 IMULQ R12, R8
3788 ADDQ R10, R15
3789 ADCQ DX, R8
3790 MOVQ R13, R10
3791 MOVQ R14, R11
3792 MOVQ R15, R12
3793 ANDQ $0x03, R12
3794 MOVQ R15, R13
3795 ANDQ $-4, R13
3796 MOVQ R8, R14
3797 SHRQ $0x02, R8, R15
3798 SHRQ $0x02, R8
3799 ADDQ R13, R10
3800 ADCQ R14, R11
3801 ADCQ $0x00, R12
3802 ADDQ R15, R10
3803 ADCQ R8, R11
3804 ADCQ $0x00, R12
3805 SUBQ $0x10, CX
3806 ADDQ $0x10, DI
3807 JMP sealAVX2SealHash
3808
3809 sealAVX2ShortSealLoop:
3810 CMPQ BX, $0x20
3811 JB sealAVX2ShortTail32
3812 SUBQ $0x20, BX
3813
3814 // Load for encryption
3815 VPXOR (SI), Y0, Y0
3816 VMOVDQU Y0, (DI)
3817 LEAQ 32(SI), SI
3818
3819 // Now can hash
3820 ADDQ (DI), R10
3821 ADCQ 8(DI), R11
3822 ADCQ $0x01, R12
3823 MOVQ (BP), DX
3824 MOVQ DX, R15
3825 MULXQ R10, R13, R14
3826 IMULQ R12, R15
3827 MULXQ R11, AX, DX
3828 ADDQ AX, R14
3829 ADCQ DX, R15
3830 MOVQ 8(BP), DX
3831 MULXQ R10, R10, AX
3832 ADDQ R10, R14
3833 MULXQ R11, R11, R8
3834 ADCQ R11, R15
3835 ADCQ $0x00, R8
3836 IMULQ R12, DX
3837 ADDQ AX, R15
3838 ADCQ DX, R8
3839 MOVQ R13, R10
3840 MOVQ R14, R11
3841 MOVQ R15, R12
3842 ANDQ $0x03, R12
3843 MOVQ R15, R13
3844 ANDQ $-4, R13
3845 MOVQ R8, R14
3846 SHRQ $0x02, R8, R15
3847 SHRQ $0x02, R8
3848 ADDQ R13, R10
3849 ADCQ R14, R11
3850 ADCQ $0x00, R12
3851 ADDQ R15, R10
3852 ADCQ R8, R11
3853 ADCQ $0x00, R12
3854 ADDQ 16(DI), R10
3855 ADCQ 24(DI), R11
3856 ADCQ $0x01, R12
3857 MOVQ (BP), DX
3858 MOVQ DX, R15
3859 MULXQ R10, R13, R14
3860 IMULQ R12, R15
3861 MULXQ R11, AX, DX
3862 ADDQ AX, R14
3863 ADCQ DX, R15
3864 MOVQ 8(BP), DX
3865 MULXQ R10, R10, AX
3866 ADDQ R10, R14
3867 MULXQ R11, R11, R8
3868 ADCQ R11, R15
3869 ADCQ $0x00, R8
3870 IMULQ R12, DX
3871 ADDQ AX, R15
3872 ADCQ DX, R8
3873 MOVQ R13, R10
3874 MOVQ R14, R11
3875 MOVQ R15, R12
3876 ANDQ $0x03, R12
3877 MOVQ R15, R13
3878 ANDQ $-4, R13
3879 MOVQ R8, R14
3880 SHRQ $0x02, R8, R15
3881 SHRQ $0x02, R8
3882 ADDQ R13, R10
3883 ADCQ R14, R11
3884 ADCQ $0x00, R12
3885 ADDQ R15, R10
3886 ADCQ R8, R11
3887 ADCQ $0x00, R12
3888 LEAQ 32(DI), DI
3889
3890 // Shift stream left
3891 VMOVDQA Y14, Y0
3892 VMOVDQA Y12, Y14
3893 VMOVDQA Y4, Y12
3894 VMOVDQA Y5, Y4
3895 VMOVDQA Y9, Y5
3896 VMOVDQA Y13, Y9
3897 VMOVDQA Y1, Y13
3898 VMOVDQA Y6, Y1
3899 VMOVDQA Y10, Y6
3900 JMP sealAVX2ShortSealLoop
3901
3902 sealAVX2ShortTail32:
3903 CMPQ BX, $0x10
3904 VMOVDQA X0, X1
3905 JB sealAVX2ShortDone
3906 SUBQ $0x10, BX
3907
3908 // Load for encryption
3909 VPXOR (SI), X0, X12
3910 VMOVDQU X12, (DI)
3911 LEAQ 16(SI), SI
3912
3913 // Hash
3914 ADDQ (DI), R10
3915 ADCQ 8(DI), R11
3916 ADCQ $0x01, R12
3917 MOVQ (BP), DX
3918 MOVQ DX, R15
3919 MULXQ R10, R13, R14
3920 IMULQ R12, R15
3921 MULXQ R11, AX, DX
3922 ADDQ AX, R14
3923 ADCQ DX, R15
3924 MOVQ 8(BP), DX
3925 MULXQ R10, R10, AX
3926 ADDQ R10, R14
3927 MULXQ R11, R11, R8
3928 ADCQ R11, R15
3929 ADCQ $0x00, R8
3930 IMULQ R12, DX
3931 ADDQ AX, R15
3932 ADCQ DX, R8
3933 MOVQ R13, R10
3934 MOVQ R14, R11
3935 MOVQ R15, R12
3936 ANDQ $0x03, R12
3937 MOVQ R15, R13
3938 ANDQ $-4, R13
3939 MOVQ R8, R14
3940 SHRQ $0x02, R8, R15
3941 SHRQ $0x02, R8
3942 ADDQ R13, R10
3943 ADCQ R14, R11
3944 ADCQ $0x00, R12
3945 ADDQ R15, R10
3946 ADCQ R8, R11
3947 ADCQ $0x00, R12
3948 LEAQ 16(DI), DI
3949 VPERM2I128 $0x11, Y0, Y0, Y0
3950 VMOVDQA X0, X1
3951
3952 sealAVX2ShortDone:
3953 VZEROUPPER
3954 JMP sealSSETail
3955
3956 seal320AVX2:
3957 VMOVDQA Y0, Y5
3958 VMOVDQA Y14, Y9
3959 VMOVDQA Y12, Y13
3960 VPADDD ·avx2IncMask<>+0(SB), Y4, Y1
3961 VMOVDQA Y0, Y6
3962 VMOVDQA Y14, Y10
3963 VMOVDQA Y12, Y8
3964 VPADDD ·avx2IncMask<>+0(SB), Y1, Y2
3965 VMOVDQA Y14, Y7
3966 VMOVDQA Y12, Y11
3967 VMOVDQA Y4, Y15
3968 MOVQ $0x0000000a, R9
3969
3970 sealAVX2320InnerCipherLoop:
3971 VPADDD Y14, Y0, Y0
3972 VPXOR Y0, Y4, Y4
3973 VPSHUFB ·rol16<>+0(SB), Y4, Y4
3974 VPADDD Y4, Y12, Y12
3975 VPXOR Y12, Y14, Y14
3976 VPSLLD $0x0c, Y14, Y3
3977 VPSRLD $0x14, Y14, Y14
3978 VPXOR Y3, Y14, Y14
3979 VPADDD Y14, Y0, Y0
3980 VPXOR Y0, Y4, Y4
3981 VPSHUFB ·rol8<>+0(SB), Y4, Y4
3982 VPADDD Y4, Y12, Y12
3983 VPXOR Y12, Y14, Y14
3984 VPSLLD $0x07, Y14, Y3
3985 VPSRLD $0x19, Y14, Y14
3986 VPXOR Y3, Y14, Y14
3987 VPADDD Y9, Y5, Y5
3988 VPXOR Y5, Y1, Y1
3989 VPSHUFB ·rol16<>+0(SB), Y1, Y1
3990 VPADDD Y1, Y13, Y13
3991 VPXOR Y13, Y9, Y9
3992 VPSLLD $0x0c, Y9, Y3
3993 VPSRLD $0x14, Y9, Y9
3994 VPXOR Y3, Y9, Y9
3995 VPADDD Y9, Y5, Y5
3996 VPXOR Y5, Y1, Y1
3997 VPSHUFB ·rol8<>+0(SB), Y1, Y1
3998 VPADDD Y1, Y13, Y13
3999 VPXOR Y13, Y9, Y9
4000 VPSLLD $0x07, Y9, Y3
4001 VPSRLD $0x19, Y9, Y9
4002 VPXOR Y3, Y9, Y9
4003 VPADDD Y10, Y6, Y6
4004 VPXOR Y6, Y2, Y2
4005 VPSHUFB ·rol16<>+0(SB), Y2, Y2
4006 VPADDD Y2, Y8, Y8
4007 VPXOR Y8, Y10, Y10
4008 VPSLLD $0x0c, Y10, Y3
4009 VPSRLD $0x14, Y10, Y10
4010 VPXOR Y3, Y10, Y10
4011 VPADDD Y10, Y6, Y6
4012 VPXOR Y6, Y2, Y2
4013 VPSHUFB ·rol8<>+0(SB), Y2, Y2
4014 VPADDD Y2, Y8, Y8
4015 VPXOR Y8, Y10, Y10
4016 VPSLLD $0x07, Y10, Y3
4017 VPSRLD $0x19, Y10, Y10
4018 VPXOR Y3, Y10, Y10
4019 VPALIGNR $0x04, Y14, Y14, Y14
4020 VPALIGNR $0x04, Y9, Y9, Y9
4021 VPALIGNR $0x04, Y10, Y10, Y10
4022 VPALIGNR $0x08, Y12, Y12, Y12
4023 VPALIGNR $0x08, Y13, Y13, Y13
4024 VPALIGNR $0x08, Y8, Y8, Y8
4025 VPALIGNR $0x0c, Y4, Y4, Y4
4026 VPALIGNR $0x0c, Y1, Y1, Y1
4027 VPALIGNR $0x0c, Y2, Y2, Y2
4028 VPADDD Y14, Y0, Y0
4029 VPXOR Y0, Y4, Y4
4030 VPSHUFB ·rol16<>+0(SB), Y4, Y4
4031 VPADDD Y4, Y12, Y12
4032 VPXOR Y12, Y14, Y14
4033 VPSLLD $0x0c, Y14, Y3
4034 VPSRLD $0x14, Y14, Y14
4035 VPXOR Y3, Y14, Y14
4036 VPADDD Y14, Y0, Y0
4037 VPXOR Y0, Y4, Y4
4038 VPSHUFB ·rol8<>+0(SB), Y4, Y4
4039 VPADDD Y4, Y12, Y12
4040 VPXOR Y12, Y14, Y14
4041 VPSLLD $0x07, Y14, Y3
4042 VPSRLD $0x19, Y14, Y14
4043 VPXOR Y3, Y14, Y14
4044 VPADDD Y9, Y5, Y5
4045 VPXOR Y5, Y1, Y1
4046 VPSHUFB ·rol16<>+0(SB), Y1, Y1
4047 VPADDD Y1, Y13, Y13
4048 VPXOR Y13, Y9, Y9
4049 VPSLLD $0x0c, Y9, Y3
4050 VPSRLD $0x14, Y9, Y9
4051 VPXOR Y3, Y9, Y9
4052 VPADDD Y9, Y5, Y5
4053 VPXOR Y5, Y1, Y1
4054 VPSHUFB ·rol8<>+0(SB), Y1, Y1
4055 VPADDD Y1, Y13, Y13
4056 VPXOR Y13, Y9, Y9
4057 VPSLLD $0x07, Y9, Y3
4058 VPSRLD $0x19, Y9, Y9
4059 VPXOR Y3, Y9, Y9
4060 VPADDD Y10, Y6, Y6
4061 VPXOR Y6, Y2, Y2
4062 VPSHUFB ·rol16<>+0(SB), Y2, Y2
4063 VPADDD Y2, Y8, Y8
4064 VPXOR Y8, Y10, Y10
4065 VPSLLD $0x0c, Y10, Y3
4066 VPSRLD $0x14, Y10, Y10
4067 VPXOR Y3, Y10, Y10
4068 VPADDD Y10, Y6, Y6
4069 VPXOR Y6, Y2, Y2
4070 VPSHUFB ·rol8<>+0(SB), Y2, Y2
4071 VPADDD Y2, Y8, Y8
4072 VPXOR Y8, Y10, Y10
4073 VPSLLD $0x07, Y10, Y3
4074 VPSRLD $0x19, Y10, Y10
4075 VPXOR Y3, Y10, Y10
4076 VPALIGNR $0x0c, Y14, Y14, Y14
4077 VPALIGNR $0x0c, Y9, Y9, Y9
4078 VPALIGNR $0x0c, Y10, Y10, Y10
4079 VPALIGNR $0x08, Y12, Y12, Y12
4080 VPALIGNR $0x08, Y13, Y13, Y13
4081 VPALIGNR $0x08, Y8, Y8, Y8
4082 VPALIGNR $0x04, Y4, Y4, Y4
4083 VPALIGNR $0x04, Y1, Y1, Y1
4084 VPALIGNR $0x04, Y2, Y2, Y2
4085 DECQ R9
4086 JNE sealAVX2320InnerCipherLoop
4087 VMOVDQA ·chacha20Constants<>+0(SB), Y3
4088 VPADDD Y3, Y0, Y0
4089 VPADDD Y3, Y5, Y5
4090 VPADDD Y3, Y6, Y6
4091 VPADDD Y7, Y14, Y14
4092 VPADDD Y7, Y9, Y9
4093 VPADDD Y7, Y10, Y10
4094 VPADDD Y11, Y12, Y12
4095 VPADDD Y11, Y13, Y13
4096 VPADDD Y11, Y8, Y8
4097 VMOVDQA ·avx2IncMask<>+0(SB), Y3
4098 VPADDD Y15, Y4, Y4
4099 VPADDD Y3, Y15, Y15
4100 VPADDD Y15, Y1, Y1
4101 VPADDD Y3, Y15, Y15
4102 VPADDD Y15, Y2, Y2
4103
4104 // Clamp and store poly key
4105 VPERM2I128 $0x02, Y0, Y14, Y3
4106 VPAND ·polyClampMask<>+0(SB), Y3, Y3
4107 VMOVDQA Y3, (BP)
4108
4109 // Stream for up to 320 bytes
4110 VPERM2I128 $0x13, Y0, Y14, Y0
4111 VPERM2I128 $0x13, Y12, Y4, Y14
4112 VPERM2I128 $0x02, Y5, Y9, Y12
4113 VPERM2I128 $0x02, Y13, Y1, Y4
4114 VPERM2I128 $0x13, Y5, Y9, Y5
4115 VPERM2I128 $0x13, Y13, Y1, Y9
4116 VPERM2I128 $0x02, Y6, Y10, Y13
4117 VPERM2I128 $0x02, Y8, Y2, Y1
4118 VPERM2I128 $0x13, Y6, Y10, Y6
4119 VPERM2I128 $0x13, Y8, Y2, Y10
4120 JMP sealAVX2ShortSeal
4121
4122 sealAVX2Tail128:
4123 VMOVDQA ·chacha20Constants<>+0(SB), Y0
4124 VMOVDQA 32(BP), Y14
4125 VMOVDQA 64(BP), Y12
4126 VMOVDQA 192(BP), Y4
4127 VPADDD ·avx2IncMask<>+0(SB), Y4, Y4
4128 VMOVDQA Y4, Y1
4129
4130 sealAVX2Tail128LoopA:
4131 ADDQ (DI), R10
4132 ADCQ 8(DI), R11
4133 ADCQ $0x01, R12
4134 MOVQ (BP), AX
4135 MOVQ AX, R15
4136 MULQ R10
4137 MOVQ AX, R13
4138 MOVQ DX, R14
4139 MOVQ (BP), AX
4140 MULQ R11
4141 IMULQ R12, R15
4142 ADDQ AX, R14
4143 ADCQ DX, R15
4144 MOVQ 8(BP), AX
4145 MOVQ AX, R8
4146 MULQ R10
4147 ADDQ AX, R14
4148 ADCQ $0x00, DX
4149 MOVQ DX, R10
4150 MOVQ 8(BP), AX
4151 MULQ R11
4152 ADDQ AX, R15
4153 ADCQ $0x00, DX
4154 IMULQ R12, R8
4155 ADDQ R10, R15
4156 ADCQ DX, R8
4157 MOVQ R13, R10
4158 MOVQ R14, R11
4159 MOVQ R15, R12
4160 ANDQ $0x03, R12
4161 MOVQ R15, R13
4162 ANDQ $-4, R13
4163 MOVQ R8, R14
4164 SHRQ $0x02, R8, R15
4165 SHRQ $0x02, R8
4166 ADDQ R13, R10
4167 ADCQ R14, R11
4168 ADCQ $0x00, R12
4169 ADDQ R15, R10
4170 ADCQ R8, R11
4171 ADCQ $0x00, R12
4172 LEAQ 16(DI), DI
4173
4174 sealAVX2Tail128LoopB:
4175 VPADDD Y14, Y0, Y0
4176 VPXOR Y0, Y4, Y4
4177 VPSHUFB ·rol16<>+0(SB), Y4, Y4
4178 VPADDD Y4, Y12, Y12
4179 VPXOR Y12, Y14, Y14
4180 VPSLLD $0x0c, Y14, Y3
4181 VPSRLD $0x14, Y14, Y14
4182 VPXOR Y3, Y14, Y14
4183 VPADDD Y14, Y0, Y0
4184 VPXOR Y0, Y4, Y4
4185 VPSHUFB ·rol8<>+0(SB), Y4, Y4
4186 VPADDD Y4, Y12, Y12
4187 VPXOR Y12, Y14, Y14
4188 VPSLLD $0x07, Y14, Y3
4189 VPSRLD $0x19, Y14, Y14
4190 VPXOR Y3, Y14, Y14
4191 ADDQ (DI), R10
4192 ADCQ 8(DI), R11
4193 ADCQ $0x01, R12
4194 MOVQ (BP), AX
4195 MOVQ AX, R15
4196 MULQ R10
4197 MOVQ AX, R13
4198 MOVQ DX, R14
4199 MOVQ (BP), AX
4200 MULQ R11
4201 IMULQ R12, R15
4202 ADDQ AX, R14
4203 ADCQ DX, R15
4204 MOVQ 8(BP), AX
4205 MOVQ AX, R8
4206 MULQ R10
4207 ADDQ AX, R14
4208 ADCQ $0x00, DX
4209 MOVQ DX, R10
4210 MOVQ 8(BP), AX
4211 MULQ R11
4212 ADDQ AX, R15
4213 ADCQ $0x00, DX
4214 IMULQ R12, R8
4215 ADDQ R10, R15
4216 ADCQ DX, R8
4217 MOVQ R13, R10
4218 MOVQ R14, R11
4219 MOVQ R15, R12
4220 ANDQ $0x03, R12
4221 MOVQ R15, R13
4222 ANDQ $-4, R13
4223 MOVQ R8, R14
4224 SHRQ $0x02, R8, R15
4225 SHRQ $0x02, R8
4226 ADDQ R13, R10
4227 ADCQ R14, R11
4228 ADCQ $0x00, R12
4229 ADDQ R15, R10
4230 ADCQ R8, R11
4231 ADCQ $0x00, R12
4232 VPALIGNR $0x04, Y14, Y14, Y14
4233 VPALIGNR $0x08, Y12, Y12, Y12
4234 VPALIGNR $0x0c, Y4, Y4, Y4
4235 VPADDD Y14, Y0, Y0
4236 VPXOR Y0, Y4, Y4
4237 VPSHUFB ·rol16<>+0(SB), Y4, Y4
4238 VPADDD Y4, Y12, Y12
4239 VPXOR Y12, Y14, Y14
4240 VPSLLD $0x0c, Y14, Y3
4241 VPSRLD $0x14, Y14, Y14
4242 VPXOR Y3, Y14, Y14
4243 VPADDD Y14, Y0, Y0
4244 VPXOR Y0, Y4, Y4
4245 VPSHUFB ·rol8<>+0(SB), Y4, Y4
4246 VPADDD Y4, Y12, Y12
4247 VPXOR Y12, Y14, Y14
4248 VPSLLD $0x07, Y14, Y3
4249 VPSRLD $0x19, Y14, Y14
4250 VPXOR Y3, Y14, Y14
4251 ADDQ 16(DI), R10
4252 ADCQ 24(DI), R11
4253 ADCQ $0x01, R12
4254 MOVQ (BP), AX
4255 MOVQ AX, R15
4256 MULQ R10
4257 MOVQ AX, R13
4258 MOVQ DX, R14
4259 MOVQ (BP), AX
4260 MULQ R11
4261 IMULQ R12, R15
4262 ADDQ AX, R14
4263 ADCQ DX, R15
4264 MOVQ 8(BP), AX
4265 MOVQ AX, R8
4266 MULQ R10
4267 ADDQ AX, R14
4268 ADCQ $0x00, DX
4269 MOVQ DX, R10
4270 MOVQ 8(BP), AX
4271 MULQ R11
4272 ADDQ AX, R15
4273 ADCQ $0x00, DX
4274 IMULQ R12, R8
4275 ADDQ R10, R15
4276 ADCQ DX, R8
4277 MOVQ R13, R10
4278 MOVQ R14, R11
4279 MOVQ R15, R12
4280 ANDQ $0x03, R12
4281 MOVQ R15, R13
4282 ANDQ $-4, R13
4283 MOVQ R8, R14
4284 SHRQ $0x02, R8, R15
4285 SHRQ $0x02, R8
4286 ADDQ R13, R10
4287 ADCQ R14, R11
4288 ADCQ $0x00, R12
4289 ADDQ R15, R10
4290 ADCQ R8, R11
4291 ADCQ $0x00, R12
4292 LEAQ 32(DI), DI
4293 VPALIGNR $0x0c, Y14, Y14, Y14
4294 VPALIGNR $0x08, Y12, Y12, Y12
4295 VPALIGNR $0x04, Y4, Y4, Y4
4296 DECQ CX
4297 JG sealAVX2Tail128LoopA
4298 DECQ R9
4299 JGE sealAVX2Tail128LoopB
4300 VPADDD ·chacha20Constants<>+0(SB), Y0, Y5
4301 VPADDD 32(BP), Y14, Y9
4302 VPADDD 64(BP), Y12, Y13
4303 VPADDD Y1, Y4, Y1
4304 VPERM2I128 $0x02, Y5, Y9, Y0
4305 VPERM2I128 $0x02, Y13, Y1, Y14
4306 VPERM2I128 $0x13, Y5, Y9, Y12
4307 VPERM2I128 $0x13, Y13, Y1, Y4
4308 JMP sealAVX2ShortSealLoop
4309
4310 sealAVX2Tail256:
4311 VMOVDQA ·chacha20Constants<>+0(SB), Y0
4312 VMOVDQA ·chacha20Constants<>+0(SB), Y5
4313 VMOVDQA 32(BP), Y14
4314 VMOVDQA 32(BP), Y9
4315 VMOVDQA 64(BP), Y12
4316 VMOVDQA 64(BP), Y13
4317 VMOVDQA 192(BP), Y4
4318 VPADDD ·avx2IncMask<>+0(SB), Y4, Y4
4319 VPADDD ·avx2IncMask<>+0(SB), Y4, Y1
4320 VMOVDQA Y4, Y7
4321 VMOVDQA Y1, Y11
4322
4323 sealAVX2Tail256LoopA:
4324 ADDQ (DI), R10
4325 ADCQ 8(DI), R11
4326 ADCQ $0x01, R12
4327 MOVQ (BP), AX
4328 MOVQ AX, R15
4329 MULQ R10
4330 MOVQ AX, R13
4331 MOVQ DX, R14
4332 MOVQ (BP), AX
4333 MULQ R11
4334 IMULQ R12, R15
4335 ADDQ AX, R14
4336 ADCQ DX, R15
4337 MOVQ 8(BP), AX
4338 MOVQ AX, R8
4339 MULQ R10
4340 ADDQ AX, R14
4341 ADCQ $0x00, DX
4342 MOVQ DX, R10
4343 MOVQ 8(BP), AX
4344 MULQ R11
4345 ADDQ AX, R15
4346 ADCQ $0x00, DX
4347 IMULQ R12, R8
4348 ADDQ R10, R15
4349 ADCQ DX, R8
4350 MOVQ R13, R10
4351 MOVQ R14, R11
4352 MOVQ R15, R12
4353 ANDQ $0x03, R12
4354 MOVQ R15, R13
4355 ANDQ $-4, R13
4356 MOVQ R8, R14
4357 SHRQ $0x02, R8, R15
4358 SHRQ $0x02, R8
4359 ADDQ R13, R10
4360 ADCQ R14, R11
4361 ADCQ $0x00, R12
4362 ADDQ R15, R10
4363 ADCQ R8, R11
4364 ADCQ $0x00, R12
4365 LEAQ 16(DI), DI
4366
4367 sealAVX2Tail256LoopB:
4368 VPADDD Y14, Y0, Y0
4369 VPXOR Y0, Y4, Y4
4370 VPSHUFB ·rol16<>+0(SB), Y4, Y4
4371 VPADDD Y4, Y12, Y12
4372 VPXOR Y12, Y14, Y14
4373 VPSLLD $0x0c, Y14, Y3
4374 VPSRLD $0x14, Y14, Y14
4375 VPXOR Y3, Y14, Y14
4376 VPADDD Y14, Y0, Y0
4377 VPXOR Y0, Y4, Y4
4378 VPSHUFB ·rol8<>+0(SB), Y4, Y4
4379 VPADDD Y4, Y12, Y12
4380 VPXOR Y12, Y14, Y14
4381 VPSLLD $0x07, Y14, Y3
4382 VPSRLD $0x19, Y14, Y14
4383 VPXOR Y3, Y14, Y14
4384 VPADDD Y9, Y5, Y5
4385 VPXOR Y5, Y1, Y1
4386 VPSHUFB ·rol16<>+0(SB), Y1, Y1
4387 VPADDD Y1, Y13, Y13
4388 VPXOR Y13, Y9, Y9
4389 VPSLLD $0x0c, Y9, Y3
4390 VPSRLD $0x14, Y9, Y9
4391 VPXOR Y3, Y9, Y9
4392 VPADDD Y9, Y5, Y5
4393 VPXOR Y5, Y1, Y1
4394 VPSHUFB ·rol8<>+0(SB), Y1, Y1
4395 VPADDD Y1, Y13, Y13
4396 VPXOR Y13, Y9, Y9
4397 VPSLLD $0x07, Y9, Y3
4398 VPSRLD $0x19, Y9, Y9
4399 VPXOR Y3, Y9, Y9
4400 ADDQ (DI), R10
4401 ADCQ 8(DI), R11
4402 ADCQ $0x01, R12
4403 MOVQ (BP), AX
4404 MOVQ AX, R15
4405 MULQ R10
4406 MOVQ AX, R13
4407 MOVQ DX, R14
4408 MOVQ (BP), AX
4409 MULQ R11
4410 IMULQ R12, R15
4411 ADDQ AX, R14
4412 ADCQ DX, R15
4413 MOVQ 8(BP), AX
4414 MOVQ AX, R8
4415 MULQ R10
4416 ADDQ AX, R14
4417 ADCQ $0x00, DX
4418 MOVQ DX, R10
4419 MOVQ 8(BP), AX
4420 MULQ R11
4421 ADDQ AX, R15
4422 ADCQ $0x00, DX
4423 IMULQ R12, R8
4424 ADDQ R10, R15
4425 ADCQ DX, R8
4426 MOVQ R13, R10
4427 MOVQ R14, R11
4428 MOVQ R15, R12
4429 ANDQ $0x03, R12
4430 MOVQ R15, R13
4431 ANDQ $-4, R13
4432 MOVQ R8, R14
4433 SHRQ $0x02, R8, R15
4434 SHRQ $0x02, R8
4435 ADDQ R13, R10
4436 ADCQ R14, R11
4437 ADCQ $0x00, R12
4438 ADDQ R15, R10
4439 ADCQ R8, R11
4440 ADCQ $0x00, R12
4441 VPALIGNR $0x04, Y14, Y14, Y14
4442 VPALIGNR $0x04, Y9, Y9, Y9
4443 VPALIGNR $0x08, Y12, Y12, Y12
4444 VPALIGNR $0x08, Y13, Y13, Y13
4445 VPALIGNR $0x0c, Y4, Y4, Y4
4446 VPALIGNR $0x0c, Y1, Y1, Y1
4447 VPADDD Y14, Y0, Y0
4448 VPXOR Y0, Y4, Y4
4449 VPSHUFB ·rol16<>+0(SB), Y4, Y4
4450 VPADDD Y4, Y12, Y12
4451 VPXOR Y12, Y14, Y14
4452 VPSLLD $0x0c, Y14, Y3
4453 VPSRLD $0x14, Y14, Y14
4454 VPXOR Y3, Y14, Y14
4455 VPADDD Y14, Y0, Y0
4456 VPXOR Y0, Y4, Y4
4457 VPSHUFB ·rol8<>+0(SB), Y4, Y4
4458 VPADDD Y4, Y12, Y12
4459 VPXOR Y12, Y14, Y14
4460 VPSLLD $0x07, Y14, Y3
4461 VPSRLD $0x19, Y14, Y14
4462 VPXOR Y3, Y14, Y14
4463 VPADDD Y9, Y5, Y5
4464 VPXOR Y5, Y1, Y1
4465 VPSHUFB ·rol16<>+0(SB), Y1, Y1
4466 VPADDD Y1, Y13, Y13
4467 VPXOR Y13, Y9, Y9
4468 VPSLLD $0x0c, Y9, Y3
4469 VPSRLD $0x14, Y9, Y9
4470 VPXOR Y3, Y9, Y9
4471 VPADDD Y9, Y5, Y5
4472 VPXOR Y5, Y1, Y1
4473 VPSHUFB ·rol8<>+0(SB), Y1, Y1
4474 VPADDD Y1, Y13, Y13
4475 VPXOR Y13, Y9, Y9
4476 VPSLLD $0x07, Y9, Y3
4477 VPSRLD $0x19, Y9, Y9
4478 VPXOR Y3, Y9, Y9
4479 ADDQ 16(DI), R10
4480 ADCQ 24(DI), R11
4481 ADCQ $0x01, R12
4482 MOVQ (BP), AX
4483 MOVQ AX, R15
4484 MULQ R10
4485 MOVQ AX, R13
4486 MOVQ DX, R14
4487 MOVQ (BP), AX
4488 MULQ R11
4489 IMULQ R12, R15
4490 ADDQ AX, R14
4491 ADCQ DX, R15
4492 MOVQ 8(BP), AX
4493 MOVQ AX, R8
4494 MULQ R10
4495 ADDQ AX, R14
4496 ADCQ $0x00, DX
4497 MOVQ DX, R10
4498 MOVQ 8(BP), AX
4499 MULQ R11
4500 ADDQ AX, R15
4501 ADCQ $0x00, DX
4502 IMULQ R12, R8
4503 ADDQ R10, R15
4504 ADCQ DX, R8
4505 MOVQ R13, R10
4506 MOVQ R14, R11
4507 MOVQ R15, R12
4508 ANDQ $0x03, R12
4509 MOVQ R15, R13
4510 ANDQ $-4, R13
4511 MOVQ R8, R14
4512 SHRQ $0x02, R8, R15
4513 SHRQ $0x02, R8
4514 ADDQ R13, R10
4515 ADCQ R14, R11
4516 ADCQ $0x00, R12
4517 ADDQ R15, R10
4518 ADCQ R8, R11
4519 ADCQ $0x00, R12
4520 LEAQ 32(DI), DI
4521 VPALIGNR $0x0c, Y14, Y14, Y14
4522 VPALIGNR $0x0c, Y9, Y9, Y9
4523 VPALIGNR $0x08, Y12, Y12, Y12
4524 VPALIGNR $0x08, Y13, Y13, Y13
4525 VPALIGNR $0x04, Y4, Y4, Y4
4526 VPALIGNR $0x04, Y1, Y1, Y1
4527 DECQ CX
4528 JG sealAVX2Tail256LoopA
4529 DECQ R9
4530 JGE sealAVX2Tail256LoopB
4531 VPADDD ·chacha20Constants<>+0(SB), Y0, Y0
4532 VPADDD ·chacha20Constants<>+0(SB), Y5, Y5
4533 VPADDD 32(BP), Y14, Y14
4534 VPADDD 32(BP), Y9, Y9
4535 VPADDD 64(BP), Y12, Y12
4536 VPADDD 64(BP), Y13, Y13
4537 VPADDD Y7, Y4, Y4
4538 VPADDD Y11, Y1, Y1
4539 VPERM2I128 $0x02, Y0, Y14, Y3
4540 VPERM2I128 $0x02, Y12, Y4, Y7
4541 VPERM2I128 $0x13, Y0, Y14, Y11
4542 VPERM2I128 $0x13, Y12, Y4, Y15
4543 VPXOR (SI), Y3, Y3
4544 VPXOR 32(SI), Y7, Y7
4545 VPXOR 64(SI), Y11, Y11
4546 VPXOR 96(SI), Y15, Y15
4547 VMOVDQU Y3, (DI)
4548 VMOVDQU Y7, 32(DI)
4549 VMOVDQU Y11, 64(DI)
4550 VMOVDQU Y15, 96(DI)
4551 MOVQ $0x00000080, CX
4552 LEAQ 128(SI), SI
4553 SUBQ $0x80, BX
4554 VPERM2I128 $0x02, Y5, Y9, Y0
4555 VPERM2I128 $0x02, Y13, Y1, Y14
4556 VPERM2I128 $0x13, Y5, Y9, Y12
4557 VPERM2I128 $0x13, Y13, Y1, Y4
4558 JMP sealAVX2SealHash
4559
4560 sealAVX2Tail384:
4561 VMOVDQA ·chacha20Constants<>+0(SB), Y0
4562 VMOVDQA Y0, Y5
4563 VMOVDQA Y0, Y6
4564 VMOVDQA 32(BP), Y14
4565 VMOVDQA Y14, Y9
4566 VMOVDQA Y14, Y10
4567 VMOVDQA 64(BP), Y12
4568 VMOVDQA Y12, Y13
4569 VMOVDQA Y12, Y8
4570 VMOVDQA 192(BP), Y4
4571 VPADDD ·avx2IncMask<>+0(SB), Y4, Y4
4572 VPADDD ·avx2IncMask<>+0(SB), Y4, Y1
4573 VPADDD ·avx2IncMask<>+0(SB), Y1, Y2
4574 VMOVDQA Y4, Y7
4575 VMOVDQA Y1, Y11
4576 VMOVDQA Y2, Y15
4577
4578 sealAVX2Tail384LoopA:
4579 ADDQ (DI), R10
4580 ADCQ 8(DI), R11
4581 ADCQ $0x01, R12
4582 MOVQ (BP), AX
4583 MOVQ AX, R15
4584 MULQ R10
4585 MOVQ AX, R13
4586 MOVQ DX, R14
4587 MOVQ (BP), AX
4588 MULQ R11
4589 IMULQ R12, R15
4590 ADDQ AX, R14
4591 ADCQ DX, R15
4592 MOVQ 8(BP), AX
4593 MOVQ AX, R8
4594 MULQ R10
4595 ADDQ AX, R14
4596 ADCQ $0x00, DX
4597 MOVQ DX, R10
4598 MOVQ 8(BP), AX
4599 MULQ R11
4600 ADDQ AX, R15
4601 ADCQ $0x00, DX
4602 IMULQ R12, R8
4603 ADDQ R10, R15
4604 ADCQ DX, R8
4605 MOVQ R13, R10
4606 MOVQ R14, R11
4607 MOVQ R15, R12
4608 ANDQ $0x03, R12
4609 MOVQ R15, R13
4610 ANDQ $-4, R13
4611 MOVQ R8, R14
4612 SHRQ $0x02, R8, R15
4613 SHRQ $0x02, R8
4614 ADDQ R13, R10
4615 ADCQ R14, R11
4616 ADCQ $0x00, R12
4617 ADDQ R15, R10
4618 ADCQ R8, R11
4619 ADCQ $0x00, R12
4620 LEAQ 16(DI), DI
4621
4622 sealAVX2Tail384LoopB:
4623 VPADDD Y14, Y0, Y0
4624 VPXOR Y0, Y4, Y4
4625 VPSHUFB ·rol16<>+0(SB), Y4, Y4
4626 VPADDD Y4, Y12, Y12
4627 VPXOR Y12, Y14, Y14
4628 VPSLLD $0x0c, Y14, Y3
4629 VPSRLD $0x14, Y14, Y14
4630 VPXOR Y3, Y14, Y14
4631 VPADDD Y14, Y0, Y0
4632 VPXOR Y0, Y4, Y4
4633 VPSHUFB ·rol8<>+0(SB), Y4, Y4
4634 VPADDD Y4, Y12, Y12
4635 VPXOR Y12, Y14, Y14
4636 VPSLLD $0x07, Y14, Y3
4637 VPSRLD $0x19, Y14, Y14
4638 VPXOR Y3, Y14, Y14
4639 VPADDD Y9, Y5, Y5
4640 VPXOR Y5, Y1, Y1
4641 VPSHUFB ·rol16<>+0(SB), Y1, Y1
4642 VPADDD Y1, Y13, Y13
4643 VPXOR Y13, Y9, Y9
4644 VPSLLD $0x0c, Y9, Y3
4645 VPSRLD $0x14, Y9, Y9
4646 VPXOR Y3, Y9, Y9
4647 VPADDD Y9, Y5, Y5
4648 VPXOR Y5, Y1, Y1
4649 VPSHUFB ·rol8<>+0(SB), Y1, Y1
4650 VPADDD Y1, Y13, Y13
4651 VPXOR Y13, Y9, Y9
4652 VPSLLD $0x07, Y9, Y3
4653 VPSRLD $0x19, Y9, Y9
4654 VPXOR Y3, Y9, Y9
4655 VPADDD Y10, Y6, Y6
4656 VPXOR Y6, Y2, Y2
4657 VPSHUFB ·rol16<>+0(SB), Y2, Y2
4658 VPADDD Y2, Y8, Y8
4659 VPXOR Y8, Y10, Y10
4660 VPSLLD $0x0c, Y10, Y3
4661 VPSRLD $0x14, Y10, Y10
4662 VPXOR Y3, Y10, Y10
4663 VPADDD Y10, Y6, Y6
4664 VPXOR Y6, Y2, Y2
4665 VPSHUFB ·rol8<>+0(SB), Y2, Y2
4666 VPADDD Y2, Y8, Y8
4667 VPXOR Y8, Y10, Y10
4668 VPSLLD $0x07, Y10, Y3
4669 VPSRLD $0x19, Y10, Y10
4670 VPXOR Y3, Y10, Y10
4671 ADDQ (DI), R10
4672 ADCQ 8(DI), R11
4673 ADCQ $0x01, R12
4674 MOVQ (BP), AX
4675 MOVQ AX, R15
4676 MULQ R10
4677 MOVQ AX, R13
4678 MOVQ DX, R14
4679 MOVQ (BP), AX
4680 MULQ R11
4681 IMULQ R12, R15
4682 ADDQ AX, R14
4683 ADCQ DX, R15
4684 MOVQ 8(BP), AX
4685 MOVQ AX, R8
4686 MULQ R10
4687 ADDQ AX, R14
4688 ADCQ $0x00, DX
4689 MOVQ DX, R10
4690 MOVQ 8(BP), AX
4691 MULQ R11
4692 ADDQ AX, R15
4693 ADCQ $0x00, DX
4694 IMULQ R12, R8
4695 ADDQ R10, R15
4696 ADCQ DX, R8
4697 MOVQ R13, R10
4698 MOVQ R14, R11
4699 MOVQ R15, R12
4700 ANDQ $0x03, R12
4701 MOVQ R15, R13
4702 ANDQ $-4, R13
4703 MOVQ R8, R14
4704 SHRQ $0x02, R8, R15
4705 SHRQ $0x02, R8
4706 ADDQ R13, R10
4707 ADCQ R14, R11
4708 ADCQ $0x00, R12
4709 ADDQ R15, R10
4710 ADCQ R8, R11
4711 ADCQ $0x00, R12
4712 VPALIGNR $0x04, Y14, Y14, Y14
4713 VPALIGNR $0x04, Y9, Y9, Y9
4714 VPALIGNR $0x04, Y10, Y10, Y10
4715 VPALIGNR $0x08, Y12, Y12, Y12
4716 VPALIGNR $0x08, Y13, Y13, Y13
4717 VPALIGNR $0x08, Y8, Y8, Y8
4718 VPALIGNR $0x0c, Y4, Y4, Y4
4719 VPALIGNR $0x0c, Y1, Y1, Y1
4720 VPALIGNR $0x0c, Y2, Y2, Y2
4721 VPADDD Y14, Y0, Y0
4722 VPXOR Y0, Y4, Y4
4723 VPSHUFB ·rol16<>+0(SB), Y4, Y4
4724 VPADDD Y4, Y12, Y12
4725 VPXOR Y12, Y14, Y14
4726 VPSLLD $0x0c, Y14, Y3
4727 VPSRLD $0x14, Y14, Y14
4728 VPXOR Y3, Y14, Y14
4729 VPADDD Y14, Y0, Y0
4730 VPXOR Y0, Y4, Y4
4731 VPSHUFB ·rol8<>+0(SB), Y4, Y4
4732 VPADDD Y4, Y12, Y12
4733 VPXOR Y12, Y14, Y14
4734 VPSLLD $0x07, Y14, Y3
4735 VPSRLD $0x19, Y14, Y14
4736 VPXOR Y3, Y14, Y14
4737 VPADDD Y9, Y5, Y5
4738 VPXOR Y5, Y1, Y1
4739 VPSHUFB ·rol16<>+0(SB), Y1, Y1
4740 VPADDD Y1, Y13, Y13
4741 VPXOR Y13, Y9, Y9
4742 VPSLLD $0x0c, Y9, Y3
4743 VPSRLD $0x14, Y9, Y9
4744 VPXOR Y3, Y9, Y9
4745 VPADDD Y9, Y5, Y5
4746 VPXOR Y5, Y1, Y1
4747 VPSHUFB ·rol8<>+0(SB), Y1, Y1
4748 VPADDD Y1, Y13, Y13
4749 VPXOR Y13, Y9, Y9
4750 VPSLLD $0x07, Y9, Y3
4751 VPSRLD $0x19, Y9, Y9
4752 VPXOR Y3, Y9, Y9
4753 VPADDD Y10, Y6, Y6
4754 VPXOR Y6, Y2, Y2
4755 VPSHUFB ·rol16<>+0(SB), Y2, Y2
4756 VPADDD Y2, Y8, Y8
4757 VPXOR Y8, Y10, Y10
4758 VPSLLD $0x0c, Y10, Y3
4759 VPSRLD $0x14, Y10, Y10
4760 VPXOR Y3, Y10, Y10
4761 VPADDD Y10, Y6, Y6
4762 VPXOR Y6, Y2, Y2
4763 VPSHUFB ·rol8<>+0(SB), Y2, Y2
4764 VPADDD Y2, Y8, Y8
4765 VPXOR Y8, Y10, Y10
4766 VPSLLD $0x07, Y10, Y3
4767 VPSRLD $0x19, Y10, Y10
4768 VPXOR Y3, Y10, Y10
4769 ADDQ 16(DI), R10
4770 ADCQ 24(DI), R11
4771 ADCQ $0x01, R12
4772 MOVQ (BP), AX
4773 MOVQ AX, R15
4774 MULQ R10
4775 MOVQ AX, R13
4776 MOVQ DX, R14
4777 MOVQ (BP), AX
4778 MULQ R11
4779 IMULQ R12, R15
4780 ADDQ AX, R14
4781 ADCQ DX, R15
4782 MOVQ 8(BP), AX
4783 MOVQ AX, R8
4784 MULQ R10
4785 ADDQ AX, R14
4786 ADCQ $0x00, DX
4787 MOVQ DX, R10
4788 MOVQ 8(BP), AX
4789 MULQ R11
4790 ADDQ AX, R15
4791 ADCQ $0x00, DX
4792 IMULQ R12, R8
4793 ADDQ R10, R15
4794 ADCQ DX, R8
4795 MOVQ R13, R10
4796 MOVQ R14, R11
4797 MOVQ R15, R12
4798 ANDQ $0x03, R12
4799 MOVQ R15, R13
4800 ANDQ $-4, R13
4801 MOVQ R8, R14
4802 SHRQ $0x02, R8, R15
4803 SHRQ $0x02, R8
4804 ADDQ R13, R10
4805 ADCQ R14, R11
4806 ADCQ $0x00, R12
4807 ADDQ R15, R10
4808 ADCQ R8, R11
4809 ADCQ $0x00, R12
4810 LEAQ 32(DI), DI
4811 VPALIGNR $0x0c, Y14, Y14, Y14
4812 VPALIGNR $0x0c, Y9, Y9, Y9
4813 VPALIGNR $0x0c, Y10, Y10, Y10
4814 VPALIGNR $0x08, Y12, Y12, Y12
4815 VPALIGNR $0x08, Y13, Y13, Y13
4816 VPALIGNR $0x08, Y8, Y8, Y8
4817 VPALIGNR $0x04, Y4, Y4, Y4
4818 VPALIGNR $0x04, Y1, Y1, Y1
4819 VPALIGNR $0x04, Y2, Y2, Y2
4820 DECQ CX
4821 JG sealAVX2Tail384LoopA
4822 DECQ R9
4823 JGE sealAVX2Tail384LoopB
4824 VPADDD ·chacha20Constants<>+0(SB), Y0, Y0
4825 VPADDD ·chacha20Constants<>+0(SB), Y5, Y5
4826 VPADDD ·chacha20Constants<>+0(SB), Y6, Y6
4827 VPADDD 32(BP), Y14, Y14
4828 VPADDD 32(BP), Y9, Y9
4829 VPADDD 32(BP), Y10, Y10
4830 VPADDD 64(BP), Y12, Y12
4831 VPADDD 64(BP), Y13, Y13
4832 VPADDD 64(BP), Y8, Y8
4833 VPADDD Y7, Y4, Y4
4834 VPADDD Y11, Y1, Y1
4835 VPADDD Y15, Y2, Y2
4836 VPERM2I128 $0x02, Y0, Y14, Y3
4837 VPERM2I128 $0x02, Y12, Y4, Y7
4838 VPERM2I128 $0x13, Y0, Y14, Y11
4839 VPERM2I128 $0x13, Y12, Y4, Y15
4840 VPXOR (SI), Y3, Y3
4841 VPXOR 32(SI), Y7, Y7
4842 VPXOR 64(SI), Y11, Y11
4843 VPXOR 96(SI), Y15, Y15
4844 VMOVDQU Y3, (DI)
4845 VMOVDQU Y7, 32(DI)
4846 VMOVDQU Y11, 64(DI)
4847 VMOVDQU Y15, 96(DI)
4848 VPERM2I128 $0x02, Y5, Y9, Y3
4849 VPERM2I128 $0x02, Y13, Y1, Y7
4850 VPERM2I128 $0x13, Y5, Y9, Y11
4851 VPERM2I128 $0x13, Y13, Y1, Y15
4852 VPXOR 128(SI), Y3, Y3
4853 VPXOR 160(SI), Y7, Y7
4854 VPXOR 192(SI), Y11, Y11
4855 VPXOR 224(SI), Y15, Y15
4856 VMOVDQU Y3, 128(DI)
4857 VMOVDQU Y7, 160(DI)
4858 VMOVDQU Y11, 192(DI)
4859 VMOVDQU Y15, 224(DI)
4860 MOVQ $0x00000100, CX
4861 LEAQ 256(SI), SI
4862 SUBQ $0x00000100, BX
4863 VPERM2I128 $0x02, Y6, Y10, Y0
4864 VPERM2I128 $0x02, Y8, Y2, Y14
4865 VPERM2I128 $0x13, Y6, Y10, Y12
4866 VPERM2I128 $0x13, Y8, Y2, Y4
4867 JMP sealAVX2SealHash
4868
4869 sealAVX2Tail512:
4870 VMOVDQA ·chacha20Constants<>+0(SB), Y0
4871 VMOVDQA Y0, Y5
4872 VMOVDQA Y0, Y6
4873 VMOVDQA Y0, Y7
4874 VMOVDQA 32(BP), Y14
4875 VMOVDQA Y14, Y9
4876 VMOVDQA Y14, Y10
4877 VMOVDQA Y14, Y11
4878 VMOVDQA 64(BP), Y12
4879 VMOVDQA Y12, Y13
4880 VMOVDQA Y12, Y8
4881 VMOVDQA Y12, Y15
4882 VMOVDQA 192(BP), Y4
4883 VPADDD ·avx2IncMask<>+0(SB), Y4, Y4
4884 VPADDD ·avx2IncMask<>+0(SB), Y4, Y1
4885 VPADDD ·avx2IncMask<>+0(SB), Y1, Y2
4886 VPADDD ·avx2IncMask<>+0(SB), Y2, Y3
4887 VMOVDQA Y4, 96(BP)
4888 VMOVDQA Y1, 128(BP)
4889 VMOVDQA Y2, 160(BP)
4890 VMOVDQA Y3, 192(BP)
4891
4892 sealAVX2Tail512LoopA:
4893 ADDQ (DI), R10
4894 ADCQ 8(DI), R11
4895 ADCQ $0x01, R12
4896 MOVQ (BP), AX
4897 MOVQ AX, R15
4898 MULQ R10
4899 MOVQ AX, R13
4900 MOVQ DX, R14
4901 MOVQ (BP), AX
4902 MULQ R11
4903 IMULQ R12, R15
4904 ADDQ AX, R14
4905 ADCQ DX, R15
4906 MOVQ 8(BP), AX
4907 MOVQ AX, R8
4908 MULQ R10
4909 ADDQ AX, R14
4910 ADCQ $0x00, DX
4911 MOVQ DX, R10
4912 MOVQ 8(BP), AX
4913 MULQ R11
4914 ADDQ AX, R15
4915 ADCQ $0x00, DX
4916 IMULQ R12, R8
4917 ADDQ R10, R15
4918 ADCQ DX, R8
4919 MOVQ R13, R10
4920 MOVQ R14, R11
4921 MOVQ R15, R12
4922 ANDQ $0x03, R12
4923 MOVQ R15, R13
4924 ANDQ $-4, R13
4925 MOVQ R8, R14
4926 SHRQ $0x02, R8, R15
4927 SHRQ $0x02, R8
4928 ADDQ R13, R10
4929 ADCQ R14, R11
4930 ADCQ $0x00, R12
4931 ADDQ R15, R10
4932 ADCQ R8, R11
4933 ADCQ $0x00, R12
4934 LEAQ 16(DI), DI
4935
4936 sealAVX2Tail512LoopB:
4937 VPADDD Y14, Y0, Y0
4938 VPADDD Y9, Y5, Y5
4939 VPADDD Y10, Y6, Y6
4940 VPADDD Y11, Y7, Y7
4941 VPXOR Y0, Y4, Y4
4942 VPXOR Y5, Y1, Y1
4943 VPXOR Y6, Y2, Y2
4944 VPXOR Y7, Y3, Y3
4945 VPSHUFB ·rol16<>+0(SB), Y4, Y4
4946 VPSHUFB ·rol16<>+0(SB), Y1, Y1
4947 VPSHUFB ·rol16<>+0(SB), Y2, Y2
4948 VPSHUFB ·rol16<>+0(SB), Y3, Y3
4949 VPADDD Y4, Y12, Y12
4950 VPADDD Y1, Y13, Y13
4951 VPADDD Y2, Y8, Y8
4952 VPADDD Y3, Y15, Y15
4953 VPXOR Y12, Y14, Y14
4954 VPXOR Y13, Y9, Y9
4955 VPXOR Y8, Y10, Y10
4956 VPXOR Y15, Y11, Y11
4957 VMOVDQA Y15, 224(BP)
4958 VPSLLD $0x0c, Y14, Y15
4959 VPSRLD $0x14, Y14, Y14
4960 VPXOR Y15, Y14, Y14
4961 VPSLLD $0x0c, Y9, Y15
4962 VPSRLD $0x14, Y9, Y9
4963 VPXOR Y15, Y9, Y9
4964 VPSLLD $0x0c, Y10, Y15
4965 VPSRLD $0x14, Y10, Y10
4966 VPXOR Y15, Y10, Y10
4967 VPSLLD $0x0c, Y11, Y15
4968 VPSRLD $0x14, Y11, Y11
4969 VPXOR Y15, Y11, Y11
4970 VMOVDQA 224(BP), Y15
4971 ADDQ (DI), R10
4972 ADCQ 8(DI), R11
4973 ADCQ $0x01, R12
4974 MOVQ (BP), DX
4975 MOVQ DX, R15
4976 MULXQ R10, R13, R14
4977 IMULQ R12, R15
4978 MULXQ R11, AX, DX
4979 ADDQ AX, R14
4980 ADCQ DX, R15
4981 MOVQ 8(BP), DX
4982 MULXQ R10, R10, AX
4983 ADDQ R10, R14
4984 MULXQ R11, R11, R8
4985 ADCQ R11, R15
4986 ADCQ $0x00, R8
4987 IMULQ R12, DX
4988 ADDQ AX, R15
4989 ADCQ DX, R8
4990 MOVQ R13, R10
4991 MOVQ R14, R11
4992 MOVQ R15, R12
4993 ANDQ $0x03, R12
4994 MOVQ R15, R13
4995 ANDQ $-4, R13
4996 MOVQ R8, R14
4997 SHRQ $0x02, R8, R15
4998 SHRQ $0x02, R8
4999 ADDQ R13, R10
5000 ADCQ R14, R11
5001 ADCQ $0x00, R12
5002 ADDQ R15, R10
5003 ADCQ R8, R11
5004 ADCQ $0x00, R12
5005 VPADDD Y14, Y0, Y0
5006 VPADDD Y9, Y5, Y5
5007 VPADDD Y10, Y6, Y6
5008 VPADDD Y11, Y7, Y7
5009 VPXOR Y0, Y4, Y4
5010 VPXOR Y5, Y1, Y1
5011 VPXOR Y6, Y2, Y2
5012 VPXOR Y7, Y3, Y3
5013 VPSHUFB ·rol8<>+0(SB), Y4, Y4
5014 VPSHUFB ·rol8<>+0(SB), Y1, Y1
5015 VPSHUFB ·rol8<>+0(SB), Y2, Y2
5016 VPSHUFB ·rol8<>+0(SB), Y3, Y3
5017 VPADDD Y4, Y12, Y12
5018 VPADDD Y1, Y13, Y13
5019 VPADDD Y2, Y8, Y8
5020 VPADDD Y3, Y15, Y15
5021 VPXOR Y12, Y14, Y14
5022 VPXOR Y13, Y9, Y9
5023 VPXOR Y8, Y10, Y10
5024 VPXOR Y15, Y11, Y11
5025 VMOVDQA Y15, 224(BP)
5026 VPSLLD $0x07, Y14, Y15
5027 VPSRLD $0x19, Y14, Y14
5028 VPXOR Y15, Y14, Y14
5029 VPSLLD $0x07, Y9, Y15
5030 VPSRLD $0x19, Y9, Y9
5031 VPXOR Y15, Y9, Y9
5032 VPSLLD $0x07, Y10, Y15
5033 VPSRLD $0x19, Y10, Y10
5034 VPXOR Y15, Y10, Y10
5035 VPSLLD $0x07, Y11, Y15
5036 VPSRLD $0x19, Y11, Y11
5037 VPXOR Y15, Y11, Y11
5038 VMOVDQA 224(BP), Y15
5039 VPALIGNR $0x04, Y14, Y14, Y14
5040 VPALIGNR $0x04, Y9, Y9, Y9
5041 VPALIGNR $0x04, Y10, Y10, Y10
5042 VPALIGNR $0x04, Y11, Y11, Y11
5043 VPALIGNR $0x08, Y12, Y12, Y12
5044 VPALIGNR $0x08, Y13, Y13, Y13
5045 VPALIGNR $0x08, Y8, Y8, Y8
5046 VPALIGNR $0x08, Y15, Y15, Y15
5047 VPALIGNR $0x0c, Y4, Y4, Y4
5048 VPALIGNR $0x0c, Y1, Y1, Y1
5049 VPALIGNR $0x0c, Y2, Y2, Y2
5050 VPALIGNR $0x0c, Y3, Y3, Y3
5051 VPADDD Y14, Y0, Y0
5052 VPADDD Y9, Y5, Y5
5053 VPADDD Y10, Y6, Y6
5054 VPADDD Y11, Y7, Y7
5055 VPXOR Y0, Y4, Y4
5056 VPXOR Y5, Y1, Y1
5057 VPXOR Y6, Y2, Y2
5058 VPXOR Y7, Y3, Y3
5059 VPSHUFB ·rol16<>+0(SB), Y4, Y4
5060 VPSHUFB ·rol16<>+0(SB), Y1, Y1
5061 VPSHUFB ·rol16<>+0(SB), Y2, Y2
5062 VPSHUFB ·rol16<>+0(SB), Y3, Y3
5063 VPADDD Y4, Y12, Y12
5064 VPADDD Y1, Y13, Y13
5065 VPADDD Y2, Y8, Y8
5066 VPADDD Y3, Y15, Y15
5067 VPXOR Y12, Y14, Y14
5068 VPXOR Y13, Y9, Y9
5069 VPXOR Y8, Y10, Y10
5070 VPXOR Y15, Y11, Y11
5071 ADDQ 16(DI), R10
5072 ADCQ 24(DI), R11
5073 ADCQ $0x01, R12
5074 MOVQ (BP), DX
5075 MOVQ DX, R15
5076 MULXQ R10, R13, R14
5077 IMULQ R12, R15
5078 MULXQ R11, AX, DX
5079 ADDQ AX, R14
5080 ADCQ DX, R15
5081 MOVQ 8(BP), DX
5082 MULXQ R10, R10, AX
5083 ADDQ R10, R14
5084 MULXQ R11, R11, R8
5085 ADCQ R11, R15
5086 ADCQ $0x00, R8
5087 IMULQ R12, DX
5088 ADDQ AX, R15
5089 ADCQ DX, R8
5090 MOVQ R13, R10
5091 MOVQ R14, R11
5092 MOVQ R15, R12
5093 ANDQ $0x03, R12
5094 MOVQ R15, R13
5095 ANDQ $-4, R13
5096 MOVQ R8, R14
5097 SHRQ $0x02, R8, R15
5098 SHRQ $0x02, R8
5099 ADDQ R13, R10
5100 ADCQ R14, R11
5101 ADCQ $0x00, R12
5102 ADDQ R15, R10
5103 ADCQ R8, R11
5104 ADCQ $0x00, R12
5105 LEAQ 32(DI), DI
5106 VMOVDQA Y15, 224(BP)
5107 VPSLLD $0x0c, Y14, Y15
5108 VPSRLD $0x14, Y14, Y14
5109 VPXOR Y15, Y14, Y14
5110 VPSLLD $0x0c, Y9, Y15
5111 VPSRLD $0x14, Y9, Y9
5112 VPXOR Y15, Y9, Y9
5113 VPSLLD $0x0c, Y10, Y15
5114 VPSRLD $0x14, Y10, Y10
5115 VPXOR Y15, Y10, Y10
5116 VPSLLD $0x0c, Y11, Y15
5117 VPSRLD $0x14, Y11, Y11
5118 VPXOR Y15, Y11, Y11
5119 VMOVDQA 224(BP), Y15
5120 VPADDD Y14, Y0, Y0
5121 VPADDD Y9, Y5, Y5
5122 VPADDD Y10, Y6, Y6
5123 VPADDD Y11, Y7, Y7
5124 VPXOR Y0, Y4, Y4
5125 VPXOR Y5, Y1, Y1
5126 VPXOR Y6, Y2, Y2
5127 VPXOR Y7, Y3, Y3
5128 VPSHUFB ·rol8<>+0(SB), Y4, Y4
5129 VPSHUFB ·rol8<>+0(SB), Y1, Y1
5130 VPSHUFB ·rol8<>+0(SB), Y2, Y2
5131 VPSHUFB ·rol8<>+0(SB), Y3, Y3
5132 VPADDD Y4, Y12, Y12
5133 VPADDD Y1, Y13, Y13
5134 VPADDD Y2, Y8, Y8
5135 VPADDD Y3, Y15, Y15
5136 VPXOR Y12, Y14, Y14
5137 VPXOR Y13, Y9, Y9
5138 VPXOR Y8, Y10, Y10
5139 VPXOR Y15, Y11, Y11
5140 VMOVDQA Y15, 224(BP)
5141 VPSLLD $0x07, Y14, Y15
5142 VPSRLD $0x19, Y14, Y14
5143 VPXOR Y15, Y14, Y14
5144 VPSLLD $0x07, Y9, Y15
5145 VPSRLD $0x19, Y9, Y9
5146 VPXOR Y15, Y9, Y9
5147 VPSLLD $0x07, Y10, Y15
5148 VPSRLD $0x19, Y10, Y10
5149 VPXOR Y15, Y10, Y10
5150 VPSLLD $0x07, Y11, Y15
5151 VPSRLD $0x19, Y11, Y11
5152 VPXOR Y15, Y11, Y11
5153 VMOVDQA 224(BP), Y15
5154 VPALIGNR $0x0c, Y14, Y14, Y14
5155 VPALIGNR $0x0c, Y9, Y9, Y9
5156 VPALIGNR $0x0c, Y10, Y10, Y10
5157 VPALIGNR $0x0c, Y11, Y11, Y11
5158 VPALIGNR $0x08, Y12, Y12, Y12
5159 VPALIGNR $0x08, Y13, Y13, Y13
5160 VPALIGNR $0x08, Y8, Y8, Y8
5161 VPALIGNR $0x08, Y15, Y15, Y15
5162 VPALIGNR $0x04, Y4, Y4, Y4
5163 VPALIGNR $0x04, Y1, Y1, Y1
5164 VPALIGNR $0x04, Y2, Y2, Y2
5165 VPALIGNR $0x04, Y3, Y3, Y3
5166 DECQ CX
5167 JG sealAVX2Tail512LoopA
5168 DECQ R9
5169 JGE sealAVX2Tail512LoopB
5170 VPADDD ·chacha20Constants<>+0(SB), Y0, Y0
5171 VPADDD ·chacha20Constants<>+0(SB), Y5, Y5
5172 VPADDD ·chacha20Constants<>+0(SB), Y6, Y6
5173 VPADDD ·chacha20Constants<>+0(SB), Y7, Y7
5174 VPADDD 32(BP), Y14, Y14
5175 VPADDD 32(BP), Y9, Y9
5176 VPADDD 32(BP), Y10, Y10
5177 VPADDD 32(BP), Y11, Y11
5178 VPADDD 64(BP), Y12, Y12
5179 VPADDD 64(BP), Y13, Y13
5180 VPADDD 64(BP), Y8, Y8
5181 VPADDD 64(BP), Y15, Y15
5182 VPADDD 96(BP), Y4, Y4
5183 VPADDD 128(BP), Y1, Y1
5184 VPADDD 160(BP), Y2, Y2
5185 VPADDD 192(BP), Y3, Y3
5186 VMOVDQA Y15, 224(BP)
5187 VPERM2I128 $0x02, Y0, Y14, Y15
5188 VPXOR (SI), Y15, Y15
5189 VMOVDQU Y15, (DI)
5190 VPERM2I128 $0x02, Y12, Y4, Y15
5191 VPXOR 32(SI), Y15, Y15
5192 VMOVDQU Y15, 32(DI)
5193 VPERM2I128 $0x13, Y0, Y14, Y15
5194 VPXOR 64(SI), Y15, Y15
5195 VMOVDQU Y15, 64(DI)
5196 VPERM2I128 $0x13, Y12, Y4, Y15
5197 VPXOR 96(SI), Y15, Y15
5198 VMOVDQU Y15, 96(DI)
5199 VPERM2I128 $0x02, Y5, Y9, Y0
5200 VPERM2I128 $0x02, Y13, Y1, Y14
5201 VPERM2I128 $0x13, Y5, Y9, Y12
5202 VPERM2I128 $0x13, Y13, Y1, Y4
5203 VPXOR 128(SI), Y0, Y0
5204 VPXOR 160(SI), Y14, Y14
5205 VPXOR 192(SI), Y12, Y12
5206 VPXOR 224(SI), Y4, Y4
5207 VMOVDQU Y0, 128(DI)
5208 VMOVDQU Y14, 160(DI)
5209 VMOVDQU Y12, 192(DI)
5210 VMOVDQU Y4, 224(DI)
5211 VPERM2I128 $0x02, Y6, Y10, Y0
5212 VPERM2I128 $0x02, Y8, Y2, Y14
5213 VPERM2I128 $0x13, Y6, Y10, Y12
5214 VPERM2I128 $0x13, Y8, Y2, Y4
5215 VPXOR 256(SI), Y0, Y0
5216 VPXOR 288(SI), Y14, Y14
5217 VPXOR 320(SI), Y12, Y12
5218 VPXOR 352(SI), Y4, Y4
5219 VMOVDQU Y0, 256(DI)
5220 VMOVDQU Y14, 288(DI)
5221 VMOVDQU Y12, 320(DI)
5222 VMOVDQU Y4, 352(DI)
5223 MOVQ $0x00000180, CX
5224 LEAQ 384(SI), SI
5225 SUBQ $0x00000180, BX
5226 VPERM2I128 $0x02, Y7, Y11, Y0
5227 VPERM2I128 $0x02, 224(BP), Y3, Y14
5228 VPERM2I128 $0x13, Y7, Y11, Y12
5229 VPERM2I128 $0x13, 224(BP), Y3, Y4
5230 JMP sealAVX2SealHash
5231
View as plain text