; Copyright 2018-2025 - Mersenne Research, Inc.  All rights reserved
; Author:  George Woltman
; Email: woltman@alum.mit.edu
;

;;
;; All new macros for version 29 of gwnum.  Do a radix-12 step in an FFT.
;; The forward FFT macros multiply by the sin/cos values at the end of the macro
;; and the inverse FFTs multiply by the sin/cos values at the start of the macro.
;; We also implement the Daniel J. Bernstein (DJB) "exponent-1" idea to save sin/cos
;; memory.
;;



;;
;; ************************************* twelve-complex-djbfft variants ******************************************
;;

;; The standard version
zr12_twelve_complex_djbfft_preload MACRO
	zr12_12c_djbfft_cmn_preload
	ENDM
zr12_twelve_complex_djbfft MACRO srcreg,srcinc,d1,d2,d4,screg,scinc,maxrpt,L1pt,L1pd
	zr12_12c_djbfft_cmn srcreg,0,srcinc,d1,d2,d4,noexec,screg,scinc,maxrpt,L1pt,L1pd
	ENDM

;; Like the standard version but uses optional [rbx] source addressing for first levels of pass 2
zr12f_twelve_complex_djbfft_preload MACRO
	zr12_12c_djbfft_cmn_preload
	ENDM
zr12f_twelve_complex_djbfft MACRO srcreg,srcinc,d1,d2,d4,screg,scinc,maxrpt,L1pt,L1pd
	zr12_12c_djbfft_cmn srcreg,rbx,srcinc,d1,d2,d4,noexec,screg,scinc,maxrpt,L1pt,L1pd
	ENDM

; Like the standard version except vbroadcastsd is used to reduce sin/cos data
zr12b_twelve_complex_djbfft_preload MACRO
	zr12_12c_djbfft_cmn_preload
	ENDM
zr12b_twelve_complex_djbfft MACRO srcreg,srcinc,d1,d2,d4,screg,scinc,maxrpt,L1pt,L1pd
	zr12_12c_djbfft_cmn srcreg,0,srcinc,d1,d2,d4,exec,screg,scinc,maxrpt,L1pt,L1pd
	ENDM


;; Common macro to operate on 12 complex values doing 3.585 levels of the FFT, applying
;; the sin/cos multipliers afterwards.

;; To calculate a 12-complex FFT in a brute force way (using a shorthand notation):
;; c1 + c2 + ... + c12 * w^000000000000
;; c1 + c2 + ... + c12 * w^0123456789AB
;; c1 + c2 + ... + c12 * w^02468A02468A
;; ...
;; c1 + c2 + ... + c12 * w^0A86420A8642
;; c1 + c2 + ... + c12 * w^0BA987654231
;;
;; The sin/cos values (w = 12th root of unity) are:
;; w^1 =  .866 + .5i
;; w^2 =  .5   + .866i
;; w^3 =  0    + 1i
;; w^4 = -.5   + .866i
;; w^5 = -.866 + .5i
;; w^6 = -1

;; Applying the sin/cos values above:
;; reals:
;; r1     +r2     +r3 +r4     +r5     +r6 + r7    + r8    + r9 +r10    + r11     +r12
;; r1 +.866r2 +.500r3     -.500r5 -.866r6 - r7 -.866r8 -.500r9      +.500r11 +.866r12 -.500i2 -.866i3 -i4 -.866i5 -.500i6 +.500i8 +.866i9 +i10 +.866i11 +.500i12
;; r1 +.500r2 -.500r3 -r4 -.500r5 +.500r6 + r7 +.500r8 -.500r9 -r10 -.500r11 +.500r12 -.866i2 -.866i3     +.866i5 +.866i6 -.866i8 -.866i9      +.866i11 +.866i12
;; r1             -r3         +r5         - r7             +r9          -r11              -i2         +i4             -i6     +i8         -i10              +i12
;; r1 -.500r2 -.500r3 +r4 -.500r5 -.500r6 + r7 -.500r8 -.500r9 +r10 -.500r11 -.500r12 -.866i2 +.866i3     -.866i5 +.866i6 -.866i8 +.866i9      -.866i11 +.866i12
;; r1 -.866r2 +.500r3     -.500r5 +.866r6 - r7 +.866r8 -.500r9      +.500r11 -.866r12 -.500i2 +.866i3 -i4 +.866i5 -.500i6 +.500i8 -.866i9 +i10 -.866i11 +.500i12
;; r1     -r2     +r3 -r4     +r5     -r6 + r7    - r8    + r9 -r10    + r11     -r12
;; r1 -.866r2 +.500r3     -.500r5 +.866r6 - r7 +.866r8 -.500r9      +.500r11 -.866r12 +.500i2 -.866i3 +i4 -.866i5 +.500i6 -.500i8 +.866i9 -i10 +.866i11 -.500i12
;; r1 -.500r2 -.500r3 +r4 -.500r5 -.500r6 + r7 -.500r8 -.500r9 +r10 -.500r11 -.500r12 +.866i2 -.866i3     +.866i5 -.866i6 +.866i8 -.866i9      +.866i11 -.866i12
;; r1             -r3         +r5         - r7             +r9         - r11              +i2         -i4             +i6     -i8         +i10              -i12
;; r1 +.500r2 -.500r3 -r4 -.500r5 +.500r6 + r7 +.500r8 -.500r9 -r10 -.500r11 +.500r12 +.866i2 +.866i3     -.866i5 -.866i6 +.866i8 +.866i9      -.866i11 -.866i12
;; r1 +.866r2 +.500r3     -.500r5 -.866r6 - r7 -.866r8 -.500r9      +.500r11 +.866r12 +.500i2 +.866i3 +i4 +.866i5 +.500i6 -.500i8 -.866i9 -i10 -.866i11 -.500i12

;; imaginarys:
;;                                                                            +i1     +i2     +i3 +i4     +i5     +i6  +i7     +i8     +i9 +i10     +i11     +i12
;; +.500r2 +.866r3 +r4 +.866r5 +.500r6 -.500r8 -.866r9 -r10 -.866r11 -.500r12 +i1 +.866i2 +.500i3     -.500i5 -.866i6  -i7 -.866i8 -.500i9      +.500i11 +.866i12
;; +.866r2 +.866r3     -.866r5 -.866r6 +.866r8 +.866r9      -.866r11 -.866r12 +i1 +.500i2 -.500i3 -i4 -.500i5 +.500i6  +i7 +.500i8 -.500i9 -i10 -.500i11 +.500i12
;;     +r2         -r4             +r6     -r8         +r10              -r12 +i1             -i3         +i5          -i7             +i9          -i11
;; +.866r2 -.866r3     +.866r5 -.866r6 +.866r8 -.866r9      +.866r11 -.866r12 +i1 -.500i2 -.500i3 +i4 -.500i5 -.500i6  +i7 -.500i8 -.500i9 +i10 -.500i11 -.500i12
;; +.500r2 -.866r3 +r4 -.866r5 +.500r6 -.500r8 +.866r9 -r10 +.866r11 -.500r12 +i1 -.866i2 +.500i3     -.500i5 +.866i6  -i7 +.866i8 -.500i9      +.500i11 -.866i12
;;                                                                            +i1     -i2     +i3 -i4     +i5     -i6  +i7     -i8     +i9 -i10     +i11     -i12
;; -.500r2 +.866r3 -r4 +.866r5 -.500r6 +.500r8 -.866r9 +r10 -.866r11 +.500r12 +i1 -.866i2 +.500i3     -.500i5 +.866i6  -i7 +.866i8 -.500i9      +.500i11 -.866i12
;; -.866r2 +.866r3     -.866r5 +.866r6 -.866r8 +.866r9      -.866r11 +.866r12 +i1 -.500i2 -.500i3 +i4 -.500i5 -.500i6  +i7 -.500i8 -.500i9 +i10 -.500i11 -.500i12
;;     -r2         +r4             -r6     +r8         -r10              +r12 +i1             -i3         +i5          -i7             +i9          -i11
;; -.866r2 -.866r3     +.866r5 +.866r6 -.866r8 -.866r9      +.866r11 +.866r12 +i1 +.500i2 -.500i3 -i4 -.500i5 +.500i6  +i7 +.500i8 -.500i9 -i10 -.500i11 +.500i12
;; -.500r2 -.866r3 -r4 -.866r5 -.500r6 +.500r8 +.866r9 +r10 +.866r11 +.500r12 +i1 +.866i2 +.500i3     -.500i5 -.866i6  -i7 -.866i8 -.500i9      +.500i11 +.866i12

;; Massive rearranging, we get:
;; R1 = (r1+r7)+(r4+r10)     +(((r3+r9)+(r5+r11))+((r2+r8)+(r6+r12)))
;; R7 = (r1+r7)-(r4+r10)     +(((r3+r9)+(r5+r11))-((r2+r8)+(r6+r12)))
;; R5 = (r1+r7)+(r4+r10) -.500(((r3+r9)+(r5+r11))+((r2+r8)+(r6+r12))) +.866(((i3+i9)-(i5+i11))-((i2+i8)-(i6+i12)))
;; R9 = (r1+r7)+(r4+r10) -.500(((r3+r9)+(r5+r11))+((r2+r8)+(r6+r12))) -.866(((i3+i9)-(i5+i11))-((i2+i8)-(i6+i12)))
;; R3 = (r1+r7)-(r4+r10) -.500(((r3+r9)+(r5+r11))-((r2+r8)+(r6+r12))) -.866(((i3+i9)-(i5+i11))+((i2+i8)-(i6+i12)))
;; R11= (r1+r7)-(r4+r10) -.500(((r3+r9)+(r5+r11))-((r2+r8)+(r6+r12))) +.866(((i3+i9)-(i5+i11))+((i2+i8)-(i6+i12)))

;; I1 = (i1+i7)+(i4+i10)     +(((i3+i9)+(i5+i11))+((i2+i8)+(i6+i12)))
;; I7 = (i1+i7)-(i4+i10)     +(((i3+i9)+(i5+i11))-((i2+i8)+(i6+i12)))
;; I5 = (i1+i7)+(i4+i10) -.500(((i3+i9)+(i5+i11))+((i2+i8)+(i6+i12))) -.866(((r3+r9)-(r5+r11))-((r2+r8)-(r6+r12)))
;; I9 = (i1+i7)+(i4+i10) -.500(((i3+i9)+(i5+i11))+((i2+i8)+(i6+i12))) +.866(((r3+r9)-(r5+r11))-((r2+r8)-(r6+r12)))
;; I3 = (i1+i7)-(i4+i10) -.500(((i3+i9)+(i5+i11))-((i2+i8)+(i6+i12))) +.866(((r3+r9)-(r5+r11))+((r2+r8)-(r6+r12)))
;; I11= (i1+i7)-(i4+i10) -.500(((i3+i9)+(i5+i11))-((i2+i8)+(i6+i12))) -.866(((r3+r9)-(r5+r11))+((r2+r8)-(r6+r12)))

;; R4 = (r1-r7)     -((r3-r9)-(r5-r11))				+(i4-i10)     -((i2-i8)+(i6-i12))
;; R10= (r1-r7)     -((r3-r9)-(r5-r11))				-(i4-i10)     +((i2-i8)+(i6-i12))
;; R2 = (r1-r7) +.500((r3-r9)-(r5-r11)) +.866((r2-r8)-(r6-r12))	-(i4-i10) -.500((i2-i8)+(i6-i12)) -.866((i3-i9)+(i5-i11))
;; R12= (r1-r7) +.500((r3-r9)-(r5-r11)) +.866((r2-r8)-(r6-r12))	+(i4-i10) +.500((i2-i8)+(i6-i12)) +.866((i3-i9)+(i5-i11))
;; R6 = (r1-r7) +.500((r3-r9)-(r5-r11)) -.866((r2-r8)-(r6-r12))	-(i4-i10) -.500((i2-i8)+(i6-i12)) +.866((i3-i9)+(i5-i11))
;; R8 = (r1-r7) +.500((r3-r9)-(r5-r11)) -.866((r2-r8)-(r6-r12))	+(i4-i10) +.500((i2-i8)+(i6-i12)) -.866((i3-i9)+(i5-i11))

;; I4 = (i1-i7)     -((i3-i9)-(i5-i11))                         -(r4-r10)     +((r2-r8)+(r6-r12))
;; I10= (i1-i7)     -((i3-i9)-(i5-i11))                         +(r4-r10)     -((r2-r8)+(r6-r12))
;; I2 = (i1-i7) +.500((i3-i9)-(i5-i11)) +.866((i2-i8)-(i6-i12))	+(r4-r10) +.500((r2-r8)+(r6-r12)) +.866((r3-r9)+(r5-r11))
;; I12= (i1-i7) +.500((i3-i9)-(i5-i11)) +.866((i2-i8)-(i6-i12))	-(r4-r10) -.500((r2-r8)+(r6-r12)) -.866((r3-r9)+(r5-r11))
;; I6 = (i1-i7) +.500((i3-i9)-(i5-i11)) -.866((i2-i8)-(i6-i12))	+(r4-r10) +.500((r2-r8)+(r6-r12)) -.866((r3-r9)+(r5-r11))
;; I8 = (i1-i7) +.500((i3-i9)-(i5-i11)) -.866((i2-i8)-(i6-i12))	-(r4-r10) -.500((r2-r8)+(r6-r12)) +.866((r3-r9)+(r5-r11))

zr12_12c_djbfft_cmn_preload MACRO
	vbroadcastsd zmm31, ZMM_HALF
	vbroadcastsd zmm30, ZMM_P866
	ENDM

zr12_12c_djbfft_cmn MACRO srcreg,srcoff,srcinc,d1,d2,d4,bcast,screg,scinc,maxrpt,L1pt,L1pd
	vmovapd	zmm2, [srcreg+srcoff+d2]	;; R3
	vmovapd	zmm8, [srcreg+srcoff+2*d4]	;; R9
	vaddpd	zmm24, zmm2, zmm8		;; R3+R9						; 1-4
	vsubpd	zmm2, zmm2, zmm8		;; R3-R9						; 1-4

	vmovapd	zmm4, [srcreg+srcoff+d4]	;; R5
	vmovapd	zmm10, [srcreg+srcoff+2*d4+d2]	;; R11
	vaddpd	zmm8, zmm4, zmm10		;; R5+R11						; 2-5
	vsubpd	zmm4, zmm4, zmm10		;; R5-R11						; 2-5

	vmovapd	zmm1, [srcreg+srcoff+d1]	;; R2
	vmovapd	zmm7, [srcreg+srcoff+d4+d2+d1]	;; R8
	vaddpd	zmm10, zmm1, zmm7		;; R2+R8						; 3-6
	vsubpd	zmm1, zmm1, zmm7		;; R2-R8						; 3-6

	vmovapd	zmm5, [srcreg+srcoff+d4+d1]	;; R6
	vmovapd	zmm11, [srcreg+srcoff+2*d4+d2+d1] ;; R12
	vaddpd	zmm7, zmm5, zmm11		;; R6+R12						; 4-7
	vsubpd	zmm5, zmm5, zmm11		;; R6-R12						; 4-7

	vmovapd	zmm14, [srcreg+srcoff+d2+64]	;; I3
	vmovapd	zmm20, [srcreg+srcoff+2*d4+64]	;; I9
	vaddpd	zmm11, zmm14, zmm20		;; I3+I9						; 5-8
	vsubpd	zmm14, zmm14, zmm20		;; I3-I9						; 5-8

	vmovapd	zmm16, [srcreg+srcoff+d4+64]	;; I5
	vmovapd	zmm22, [srcreg+srcoff+2*d4+d2+64] ;; I11
	vaddpd	zmm20, zmm16, zmm22		;; I5+I11						; 6-9
	vsubpd	zmm16, zmm16, zmm22		;; I5-I11						; 6-9

	vmovapd	zmm13, [srcreg+srcoff+d1+64]	;; I2
	vmovapd	zmm19, [srcreg+srcoff+d4+d2+d1+64] ;; I8
	vaddpd	zmm22, zmm13, zmm19		;; I2+I8						; 7-10
	vsubpd	zmm13, zmm13, zmm19		;; I2-I8						; 7-10

	vmovapd	zmm17, [srcreg+srcoff+d4+d1+64]	;; I6
	vmovapd	zmm23, [srcreg+srcoff+2*d4+d2+d1+64] ;; I12
	vaddpd	zmm19, zmm17, zmm23		;; I6+I12						; 8-11
	vsubpd	zmm17, zmm17, zmm23		;; I6-I12						; 8-11

	vmovapd	zmm0, [srcreg+srcoff]		;; R1
	vmovapd	zmm6, [srcreg+srcoff+d4+d2]	;; R7
	vaddpd	zmm23, zmm0, zmm6		;; R1+R7						; 9-12
	vsubpd	zmm0, zmm0, zmm6		;; R1-R7						; 9-12

	vmovapd	zmm3, [srcreg+srcoff+d2+d1]	;; R4
	vmovapd	zmm9, [srcreg+srcoff+2*d4+d1]	;; R10
	vaddpd	zmm6, zmm3, zmm9		;; R4+R10						; 10-13
	vsubpd	zmm3, zmm3, zmm9		;; R4-R10						; 10-13

	vmovapd	zmm12, [srcreg+srcoff+64]	;; I1
	vmovapd	zmm18, [srcreg+srcoff+d4+d2+64]	;; I7
	vaddpd	zmm9, zmm12, zmm18		;; I1+I7						; 11-14
	vsubpd	zmm12, zmm12, zmm18		;; I1-I7						; 11-14

	vmovapd	zmm15, [srcreg+srcoff+d2+d1+64]	;; I4
	vmovapd	zmm21, [srcreg+srcoff+2*d4+d1+64] ;; I10
	vaddpd	zmm18, zmm15, zmm21		;; I4+I10						; 12-15
	vsubpd	zmm15, zmm15, zmm21		;; I4-I10						; 12-15

no bcast vmovapd zmm29, [screg+5*128+64]	;; cosine/sine for R7/I7 (w^6)
bcast	vbroadcastsd zmm29, Q [screg+5*16+8]	;; cosine/sine for R7/I7
	vaddpd	zmm21, zmm24, zmm8		;; r3++ = (r3+r9) + (r5+r11)				; 13-16		n 20
	vsubpd	zmm24, zmm24, zmm8		;; r3+- = (r3+r9) - (r5+r11)				; 13-16		n 23

no bcast vmovapd zmm28, [screg+1*128]		;; sine for R3/I3 and R11/I11 (w^2)
bcast	vbroadcastsd zmm28, Q [screg+1*16]	;; sine for R3/I3 and R11/I11
	vaddpd	zmm8, zmm10, zmm7		;; r2++ = (r2+r8) + (r6+r12)				; 14-17		n 20
	vsubpd	zmm10, zmm10, zmm7		;; r2+- = (r2+r8) - (r6+r12)				; 14-17		n 23

no bcast vmovapd zmm27, [screg+3*128]		;; sine for R5/I5 and R9/I9 (w^4)
bcast	vbroadcastsd zmm27, Q [screg+3*16]	;; sine for R5/I5 and R9/I9
	vaddpd	zmm7, zmm11, zmm20		;; i3++ = (i3+i9) + (i5+i11)				; 15-18		n 21
	vsubpd	zmm11, zmm11, zmm20		;; i3+- = (i3+i9) - (i5+i11)				; 15-18		n 22

	L1prefetchw srcreg+L1pd, L1pt
	vaddpd	zmm20, zmm22, zmm19		;; i2++ = (i2+i8) + (i6+i12)				; 16-19		n 21
	vsubpd	zmm22, zmm22, zmm19		;; i2+- = (i2+i8) - (i6+i12)				; 16-19		n 22

	L1prefetchw srcreg+64+L1pd, L1pt
	vaddpd	zmm19, zmm23, zmm6		;; r1++ = (r1+r7) + (r4+r10)				; 17-20		n 27
	vsubpd	zmm23, zmm23, zmm6		;; r1+- = (r1+r7) - (r4+r10)				; 17-20		n 25

	L1prefetchw srcreg+d1+L1pd, L1pt
	vaddpd	zmm6, zmm9, zmm18		;; i1++ = (i1+i7) + (i4+i10)				; 18-21		n 28
	vsubpd	zmm9, zmm9, zmm18		;; i1+- = (i1+i7) - (i4+i10)				; 18-21		n 26

	L1prefetchw srcreg+d1+64+L1pd, L1pt
	vaddpd	zmm18, zmm2, zmm4		;;  r3-+ = (r3-r9) + (r5-r11)				; 19-22		n 50
	vsubpd	zmm2, zmm2, zmm4		;;  r3-- = (r3-r9) - (r5-r11)				; 19-22		n 35

	L1prefetchw srcreg+d2+L1pd, L1pt
	vaddpd	zmm4, zmm21, zmm8		;; r3+++ = (r3++) + (r2++)				; 20-23		n 27
	vsubpd	zmm21, zmm21, zmm8		;; r3++- = (r3++) - (r2++)				; 20-23		n 25

	L1prefetchw srcreg+d2+64+L1pd, L1pt
	vaddpd	zmm8, zmm7, zmm20		;; i3+++ = (i3++) + (i2++)				; 21-24		n 28
	vsubpd	zmm7, zmm7, zmm20		;; i3++- = (i3++) - (i2++)				; 21-24		n 26

	L1prefetchw srcreg+d2+d1+L1pd, L1pt
	vaddpd	zmm20, zmm11, zmm22		;; i3+-+ = (i3+-) + (i2+-)				; 22-25		n 37
	vsubpd	zmm11, zmm11, zmm22		;; i3+-- = (i3+-) - (i2+-)				; 22-25		n 39

	L1prefetchw srcreg+d2+d1+64+L1pd, L1pt
	vaddpd	zmm22, zmm24, zmm10		;; r3+-+ = (r3+-) + (r2+-)				; 23-26		n 38
	vsubpd	zmm24, zmm24, zmm10		;; r3+-- = (r3+-) - (r2+-)				; 23-26		n 40

	L1prefetchw srcreg+d4+L1pd, L1pt
	vaddpd	zmm10, zmm13, zmm17		;;  i2-+ = (i2-i8) + (i6-i12)				; 24-27		n 42
	vsubpd	zmm13, zmm13, zmm17		;;  i2-- = (i2-i8) - (i6-i12)				; 24-27		n 48

	L1prefetchw srcreg+d4+64+L1pd, L1pt
	vaddpd	zmm17, zmm23, zmm21		;; R7 = (r1+-) + (r3++-)				; 25-28		n 31
	zfnmaddpd zmm21, zmm21, zmm31, zmm23	;; R3B = (r1+-) - .500(r3++-)				; 25-28		n 32

	L1prefetchw srcreg+d4+d1+L1pd, L1pt
	vaddpd	zmm23, zmm9, zmm7		;; I7 = (i1+-) + (i3++-)				; 26-29		n 31
	zfnmaddpd zmm7, zmm7, zmm31, zmm9	;; I3B = (i1+-) - .500(i3++-)				; 26-29		n 33

	L1prefetchw srcreg+d4+d1+64+L1pd, L1pt
	vaddpd	zmm9, zmm19, zmm4		;; R1 = (r1++) + (r3+++)				; 27-30
	zfnmaddpd zmm4, zmm4, zmm31, zmm19	;; R59 = (r1++) - .500(r3+++)				; 27-30		n 34

	L1prefetchw srcreg+d4+d2+L1pd, L1pt
	vaddpd	zmm19, zmm6, zmm8		;; I1 = (i1++) + (i3+++)				; 28-31
	zfnmaddpd zmm8, zmm8, zmm31, zmm6	;; I59 = (i1++) - .500(i3+++)				; 28-31		n 34

	L1prefetchw srcreg+d4+d2+64+L1pd, L1pt
	vaddpd	zmm6, zmm14, zmm16		;;  i3-+ = (i3-i9) + (i5-i11)				; 29-32		n 49
	vsubpd	zmm14, zmm14, zmm16		;;  i3-- = (i3-i9) - (i5-i11)				; 29-32		n 41

	L1prefetchw srcreg+d4+d2+d1+L1pd, L1pt
	vaddpd	zmm16, zmm1, zmm5		;;  r2-+ = (r2-r8) + (r6-r12)				; 30-33		n 45
	vsubpd	zmm1, zmm1, zmm5		;;  r2-- = (r2-r8) - (r6-r12)				; 30-33		n 47

	L1prefetchw srcreg+d4+d2+d1+64+L1pd, L1pt
	zfmsubpd zmm5, zmm17, zmm29, zmm23	;; A7 = R7 * cosine/sine - I7				; 31-34		n 36
	zfmaddpd zmm23, zmm23, zmm29, zmm17	;; B7 = I7 * cosine/sine + R7				; 31-34		n 36
	zstore	[srcreg], zmm9			;; Save R1						; 31

no bcast vmovapd zmm29, [screg+5*128]		;; sine for R7/I7 (w^6)
bcast	vbroadcastsd zmm29, Q [screg+5*16]	;; sine for R7/I7
	vmulpd	zmm25, zmm30, zmm28		;; .866s = .866 * sine3B				; 32-35		n 37
	vmulpd	zmm21, zmm21, zmm28		;; R3Bs = R3B * sine3B					; 32-35		n 37
	zstore	[srcreg+64], zmm19		;; Save I1						; 32

no bcast vmovapd zmm9, [screg+1*128+64]		;; cosine/sine for R3/I3 and R11/I11 (w^2)
bcast	vbroadcastsd zmm9, Q [screg+1*16+8]	;; cosine/sine for R3/I3 and R11/I11
	vmulpd	zmm7, zmm7, zmm28		;; I3Bs = I3B * sine3B					; 33-36		n 38
	vmulpd	zmm26, zmm30, zmm27		;; .866s = .866 * sine59				; 33-36		n 39

no bcast vmovapd zmm28, [screg+3*128+64]	;; cosine/sine for R5/I5 and R9/I9 (w^4)
bcast	vbroadcastsd zmm28, Q [screg+3*16+8]	;; cosine/sine for R5/I5 and R9/I9
	vmulpd	zmm4, zmm4, zmm27		;; R59s = R59 * sine59					; 34-37		n 39
	vmulpd	zmm8, zmm8, zmm27		;; I59s = I59 * sine59					; 34-37		n 40

no bcast vmovapd zmm27, [screg+2*128]		;; sine for R4/I4 and R10/I10 (w^3)
bcast	vbroadcastsd zmm27, Q [screg+2*16]	;; sine for R4/I4 and R10/I10
	vsubpd	zmm17, zmm0, zmm2		;;  r1-- = (r1-r7) - (r3--)				; 35-38		n 51
	zfmaddpd zmm2, zmm2, zmm31, zmm0	;;  r1-+ = (r1-r7) + .500(r3--)				; 35-38		n 47

no bcast vmovapd zmm19, [screg+0*128]		;; sine for R2/I2 and R12/I12 (w^1)
bcast	vbroadcastsd zmm19, Q [screg+0*16]	;; sine for R2/I2 and R12/I12
	vmulpd	zmm5, zmm5, zmm29		;; A7 = A7 * sine (final R7)				; 36-39
	vmulpd	zmm23, zmm23, zmm29		;; B7 = B7 * sine (final I7)				; 36-39

no bcast vmovapd zmm29, [screg+4*128]		;; sine for R6/I6 and R8/I8 (w^5)
bcast	vbroadcastsd zmm29, Q [screg+4*16]	;; sine for R6/I6 and R8/I8
	zfnmaddpd zmm0, zmm20, zmm25, zmm21	;; R3s = R3Bs - .866s(i3+-+)				; 37-40		n 43
	zfmaddpd zmm20, zmm20, zmm25, zmm21	;; R11s = R3Bs + .866s(i3+-+)				; 37-40		n 44

	L1prefetchw srcreg+2*d4+L1pd, L1pt
	zfmaddpd zmm21, zmm22, zmm25, zmm7	;; I3s = I3Bs + .866s(r3+-+)				; 38-41		n 43
	zfnmaddpd zmm22, zmm22, zmm25, zmm7	;; I11s = I3Bs - .866s(r3+-+)				; 38-41		n 44

no bcast vmovapd zmm25, [screg+2*128+64]	;; cosine/sine for R4/I4 and R10/I10 (w^3)
bcast	vbroadcastsd zmm25, Q [screg+2*16+8]	;; cosine/sine for R4/I4 and R10/I10
	zfmaddpd zmm7, zmm11, zmm26, zmm4	;; R5s = R59s + .866s(i3+--)				; 39-42		n 46
	zfnmaddpd zmm11, zmm11, zmm26, zmm4	;; R9s = R59s - .866s(i3+--)				; 39-42		n 54

	L1prefetchw srcreg+2*d4+64+L1pd, L1pt
	zfnmaddpd zmm4, zmm24, zmm26, zmm8	;; I5s = I59s - .866s(r3+--)				; 40-43		n 46
	zfmaddpd zmm24, zmm24, zmm26, zmm8	;; I9s = I59s + .866s(r3+--)				; 40-43		n 54
	zstore	[srcreg+d4+d2], zmm5		;; Save R7						; 40

no bcast vmovapd zmm26, [screg+0*128+64]	;; cosine/sine for R2/I2 and R12/I12 (w^1)
bcast	vbroadcastsd zmm26, Q [screg+0*16+8]	;; cosine/sine for R2/I2 and R12/I12
	vsubpd	zmm8, zmm12, zmm14		;;  i1-- = (i1-i7) - (i3--)				; 41-44		n 51
	zfmaddpd zmm14, zmm14, zmm31, zmm12	;;  i1-+ = (i1-i7) + .500(i3--)				; 41-44		n 48
	zstore	[srcreg+d4+d2+64], zmm23	;; Save I7						; 40+1

no bcast vmovapd zmm5, [screg+4*128+64]		;; cosine/sine for R6/I6 and R8/I8 (w^5)
bcast	vbroadcastsd zmm5, Q [screg+4*16+8]	;; cosine/sine for R6/I6 and R8/I8
	vsubpd	zmm12, zmm15, zmm10		;;  i4-- = (i4-i10) - (i2-+)				; 42-45		n 55
	zfmaddpd zmm10, zmm10, zmm31, zmm15	;;  i4-+ = (i4-i10) + .500(i2-+)			; 42-45		n 49
	bump	screg, scinc

	L1prefetchw srcreg+2*d4+d1+L1pd, L1pt
	zfmsubpd zmm15, zmm0, zmm9, zmm21	;; R3s * cosine/sine - I3s (final R3)			; 43-46
	zfmaddpd zmm21, zmm21, zmm9, zmm0	;; I3s * cosine/sine + R3s (final I3)			; 43-46

	L1prefetchw srcreg+2*d4+d1+64+L1pd, L1pt
	zfmaddpd zmm0, zmm20, zmm9, zmm22	;; A11 = R11s * cosine/sine + I11s (final R11)		; 44-47
	zfmsubpd zmm22, zmm22, zmm9, zmm20	;; B11 = I11s * cosine/sine - R11s (final I11)		; 44-47

	L1prefetchw srcreg+2*d4+d2+L1pd, L1pt
	vsubpd	zmm20, zmm3, zmm16		;;  r4-- = (r4-r10) - (r2-+)				; 45-48		n 56
	zfmaddpd zmm16, zmm16, zmm31, zmm3	;;  r4-+ = (r4-r10) + .500(r2-+)			; 45-48		n 50

	L1prefetchw srcreg+2*d4+d2+64+L1pd, L1pt
	zfmsubpd zmm3, zmm7, zmm28, zmm4	;; R5s * cosine/sine - I5s (final R5)			; 46-49
	zfmaddpd zmm4, zmm4, zmm28, zmm7	;; I5s * cosine/sine + R5s (final I5)			; 46-49

	L1prefetchw srcreg+2*d4+d2+d1+L1pd, L1pt
	zfmaddpd zmm7, zmm1, zmm30, zmm2	;; r1-++ = (r1-+) + .866(r2--)				; 47-50		n 52
	zfnmaddpd zmm1, zmm1, zmm30, zmm2	;; r1-+- = (r1-+) - .866(r2--)				; 47-50		n 53
	zstore	[srcreg+d2], zmm15		;; Save R3						; 47

	L1prefetchw srcreg+2*d4+d2+d1+64+L1pd, L1pt
	zfmaddpd zmm2, zmm13, zmm30, zmm14	;; i1-++ = (i1-+) + .866(i2--)				; 48-51		n 52
	zfnmaddpd zmm13, zmm13, zmm30, zmm14	;; i1-+- = (i1-+) - .866(i2--)				; 48-51		n 53
	zstore	[srcreg+d2+64], zmm21		;; Save I3						; 47+1

	zfmaddpd zmm14, zmm6, zmm30, zmm10	;; i4-++ = (i4-+) + .866(i3-+)				; 49-52		n 57
	zfnmaddpd zmm6, zmm6, zmm30, zmm10	;; i4-+- = (i4-+) - .866(i3-+)				; 49-52		n 59
	zstore	[srcreg+2*d4+d2], zmm0		;; Save R11						; 48+1

	zfmaddpd zmm10, zmm18, zmm30, zmm16	;; r4-++ = (r4-+) + .866(r3-+)				; 50-53		n 58
	zfnmaddpd zmm18, zmm18, zmm30, zmm16	;; r4-+- = (r4-+) - .866(r3-+)				; 50-53		n 60
	zstore	[srcreg+2*d4+d2+64], zmm22	;; Save I11						; 48+2

	vmulpd	zmm17, zmm17, zmm27		;; r1--s = r1-- * sine4A				; 51-54		n 55
	vmulpd	zmm8, zmm8, zmm27		;; i1--s = i1-- * sine4A				; 51-54		n 56
	zstore	[srcreg+d4], zmm3		;; Save R5						; 50+1

	vmulpd	zmm7, zmm7, zmm19		;; r1-++s = r1-++ * sine2C				; 52-55		n 57
	vmulpd	zmm2, zmm2, zmm19		;; i1-++s = i1-++ * sine2C				; 52-55		n 58
	zstore	[srcreg+d4+64], zmm4		;; Save I5						; 50+2

	vmulpd	zmm1, zmm1, zmm29		;; r1-+-s = r1-+- * sine68				; 53-56		n 59
	vmulpd	zmm13, zmm13, zmm29		;; i1-+-s = i1-+- * sine68				; 53-56		n 60

	zfmaddpd zmm16, zmm11, zmm28, zmm24	;; R9s * cosine/sine + I9s (final R9)			; 54-57
	zfmsubpd zmm24, zmm24, zmm28, zmm11	;; I9s * cosine/sine - R9s (final I9)			; 54-57

	zfmaddpd zmm11, zmm12, zmm27, zmm17	;; R4s = (r1--s) + sine4A(i4--)				; 55-58		n 61
	zfnmaddpd zmm12, zmm12, zmm27, zmm17	;; R10s= (r1--s) - sine4A(i4--)				; 55-58		n 62

	zfnmaddpd zmm17, zmm20, zmm27, zmm8	;; I4s = i1--s - sine4A(r4--)				; 56-59		n 61
	zfmaddpd zmm20, zmm20, zmm27, zmm8	;; I10s= i1--s + sine4A(r4--)				; 56-59		n 62

	zfnmaddpd zmm8, zmm14, zmm19, zmm7	;; R2s = r1-++s - sine2C(i4-++)				; 57-60		n 63
	zfmaddpd zmm14, zmm14, zmm19, zmm7	;; R12s= r1-++s + sine2C(i4-++)				; 57-60		n 64

	zfmaddpd zmm7, zmm10, zmm19, zmm2	;; I2s = i1-++s + sine2C(r4-++)				; 58-61		n 63
	zfnmaddpd zmm10, zmm10, zmm19, zmm2	;; I12s= i1-++s - sine2C(r4-++)				; 58-61		n 64
	zstore	[srcreg+2*d4], zmm16		;; Save R9						; 58

	zfnmaddpd zmm2, zmm6, zmm29, zmm1	;; R6s = r1-+-s - sine68(i4-+-)				; 59-62		n 65
	zfmaddpd zmm6, zmm6, zmm29, zmm1	;; R8s = r1-+-s + sine68(i4-+-)				; 59-62		n 66
	zstore	[srcreg+2*d4+64], zmm24		;; Save I9						; 58+1

	zfmaddpd zmm1, zmm18, zmm29, zmm13	;; I6s = i1-+-s + sine68(r4-+-)				; 60-63		n 65
	zfnmaddpd zmm18, zmm18, zmm29, zmm13	;; I8s = i1-+-s - sine68(r4-+-)				; 60-63		n 66

	zfmsubpd zmm13, zmm11, zmm25, zmm17	;; R4s * cosine/sine - I4s (final R4)			; 61-64
	zfmaddpd zmm17, zmm17, zmm25, zmm11	;; I4s * cosine/sine + R4s (final I4)			; 61-64

	zfmaddpd zmm11, zmm12, zmm25, zmm20	;; R10s * cosine/sine + I10s (final R10)		; 62-65
	zfmsubpd zmm20, zmm20, zmm25, zmm12	;; I10s * cosine/sine - R10s (final I10)		; 62-65

	zfmsubpd zmm12, zmm8, zmm26, zmm7	;; R2s * cosine/sine - I2s (final R2)			; 63-66
	zfmaddpd zmm7, zmm7, zmm26, zmm8	;; I2s * cosine/sine + R2s (final I2)			; 63-66

	zfmaddpd zmm8, zmm14, zmm26, zmm10	;; R12s * cosine/sine + I12s (final R12)		; 64-67
	zfmsubpd zmm10, zmm10, zmm26, zmm14	;; I12s * cosine/sine - R12s (final I12)		; 64-67

	zfmsubpd zmm14, zmm2, zmm5, zmm1	;; R6s * cosine/sine - I6s (final R6)			; 65-68
	zfmaddpd zmm1, zmm1, zmm5, zmm2		;; I6s * cosine/sine + R6s (final I6)			; 65-68

	zfmaddpd zmm2, zmm6, zmm5, zmm18	;; R8s * cosine/sine + I8s (final R8)			; 66-69
	zfmsubpd zmm18, zmm18, zmm5, zmm6	;; I8s * cosine/sine - R8s (final I8)			; 66-69

	zstore	[srcreg+d2+d1], zmm13		;; Save R4						; 65
	zstore	[srcreg+d2+d1+64], zmm17	;; Save I4						; 65+1
	zstore	[srcreg+2*d4+d1], zmm11		;; Save R10						; 66+1
	zstore	[srcreg+2*d4+d1+64], zmm20	;; Save I10						; 66+2
	zstore	[srcreg+d1], zmm12		;; Save R2						; 67+2
	zstore	[srcreg+d1+64], zmm7		;; Save I2						; 67+3
	zstore	[srcreg+2*d4+d2+d1], zmm8	;; Save R12						; 68+3
	zstore	[srcreg+2*d4+d2+d1+64], zmm10	;; Save I12						; 68+4
	zstore	[srcreg+d4+d1], zmm14		;; Save R6						; 69+4
	zstore	[srcreg+d4+d1+64], zmm1		;; Save I6						; 69+5
	zstore	[srcreg+d4+d2+d1], zmm2		;; Save R8						; 70+5
	zstore	[srcreg+d4+d2+d1+64], zmm18	;; Save I8						; 70+6
	bump	srcreg, srcinc
	ENDM


;; This code applies the roots-of-minus-1 premultipliers in a negacyclic FFT as well as
;; the group weight multipliers (the column weights are applied later).
;; It also applies the eleven sin/cos multipliers after the first radix-12 butterfly.
;; We save memory by splitting the roots-of-minus-1 premultipliers such that every
;; macro uses the same premultiplier data.  The rest of the premultiplier is applied
;; at the end of pass 1.

;; The sin/cos and premultiplier data is combined in one table.  The premultiplier data
;; is at screg the sin/cos data is at screg+12*64.  Note that the sine value for the
;; premultiplier has been combined with the group multiplier.

zr12_csc_wpn_twelve_complex_first_djbfft_preload MACRO
	vbroadcastsd zmm31, ZMM_HALF
	vbroadcastsd zmm30, ZMM_P866
	vbroadcastsd zmm29, ZMM_ONE_OVER_B
	ENDM
zr12_csc_wpn_twelve_complex_first_djbfft MACRO srcreg,srcinc,d1,d2,d4,screg,scinc,maskreg,maskinc,grpreg,grpinc,maxrpt,L1pt,L1pd
	mov	bl, [maskreg+0*1]		;; Load index into compressed fudges
	mov	r14, [r13+0*8]			;; Load the xor mask
	xor	r14, [r12+rbx*8]		;; Apply the xor mask to 64 bits of fudge flags

	kmovw	k1, r14d			;; Load R1 and I1 fudge factor mask			; 1
	kshiftrw k2, k1, 8			;; I1's fudge						; 2
	vmovapd	zmm1, [srcreg]			;; R1
	vmulpd	zmm1{k1}, zmm1, zmm29		;; apply fudges to R1					; 2-5		n 34
	vmovapd	zmm13, [srcreg+64]		;; I1
	vmulpd	zmm13{k2}, zmm13, zmm29		;; apply fudges to I1					; 3-6		n 25

	shr	r14, 16				;; Next 16 bits of fudge flags
	kmovw	k1, r14d			;; Load R2 and I2 fudge factor mask			; 3
	kshiftrw k2, k1, 8			;; I2's fudge						; 4
	vmovapd	zmm2, [srcreg+d1]		;; R2
	vmulpd	zmm2{k1}, zmm2, zmm29		;; apply fudges to R2					; 4-7		n 35
	vmovapd	zmm14, [srcreg+d1+64]		;; I2
	vmulpd	zmm14{k2}, zmm14, zmm29		;; apply fudges to I2					; 5-8		n 26

	shr	r14, 16				;; Next 16 bits of fudge flags
	kmovw	k1, r14d			;; Load R3 and I3 fudge factor mask			; 5
	kshiftrw k2, k1, 8			;; I3's fudge						; 6
	vmovapd	zmm3, [srcreg+d2]		;; R3
	vmulpd	zmm3{k1}, zmm3, zmm29		;; apply fudges to R3					; 6-9		n 36
	vmovapd	zmm15, [srcreg+d2+64]		;; I3
	vmulpd	zmm15{k2}, zmm15, zmm29		;; apply fudges to I3					; 7-10		n 26

	shr	r14, 16				;; Next 16 bits of fudge flags
	kmovw	k1, r14d			;; Load R4 and I4 fudge factor mask			; 7
	kshiftrw k2, k1, 8			;; I4's fudge						; 8
	vmovapd	zmm4, [srcreg+d2+d1]		;; R4
	vmulpd	zmm4{k1}, zmm4, zmm29		;; apply fudges to R4					; 8-11		n 37
	vmovapd	zmm16, [srcreg+d2+d1+64]	;; I4
	vmulpd	zmm16{k2}, zmm16, zmm29		;; apply fudges to I4					; 9-12		n 27

	mov	bl, [maskreg+1*1]		;; Load index into compressed fudges
	mov	r14, [r13+1*8]			;; Load the xor mask
	xor	r14, [r12+rbx*8]		;; Apply the xor mask to 64 bits of fudge flags

	kmovw	k1, r14d			;; Load R5 and I5 fudge factor mask			; 9
	kshiftrw k2, k1, 8			;; I5's fudge						; 10
	vmovapd	zmm5, [srcreg+d4]		;; R5
	vmulpd	zmm5{k1}, zmm5, zmm29		;; apply fudges to R5					; 10-13		n 38
	vmovapd	zmm17, [srcreg+d4+64]		;; I5
	vmulpd	zmm17{k2}, zmm17, zmm29		;; apply fudges to I5					; 11-14		n 27

	shr	r14, 16				;; Next 16 bits of fudge flags
	kmovw	k1, r14d			;; Load R6 and I6 fudge factor mask			; 11
	kshiftrw k2, k1, 8			;; I6's fudge						; 12
	vmovapd	zmm6, [srcreg+d4+d1]		;; R6
	vmulpd	zmm6{k1}, zmm6, zmm29		;; apply fudges to R6					; 12-15		n 39
	vmovapd	zmm18, [srcreg+d4+d1+64]	;; I6
	vmulpd	zmm18{k2}, zmm18, zmm29		;; apply fudges to I6					; 13-16		n 28

	shr	r14, 16				;; Next 16 bits of fudge flags
	kmovw	k1, r14d			;; Load R7 and I7 fudge factor mask			; 13
	kshiftrw k2, k1, 8			;; I7's fudge						; 14
	vmovapd	zmm7, [srcreg+d4+d2]		;; R7
	vmulpd	zmm7{k1}, zmm7, zmm29		;; apply fudges to R7					; 14-17		n 32
	vmovapd	zmm19, [srcreg+d4+d2+64]	;; I7
	vmulpd	zmm19{k2}, zmm19, zmm29		;; apply fudges to I7					; 15-18		n 33

	shr	r14, 16				;; Next 16 bits of fudge flags
	kmovw	k1, r14d			;; Load R8 and I8 fudge factor mask			; 15
	kshiftrw k2, k1, 8			;; I8's fudge						; 16
	vmovapd	zmm8, [srcreg+d4+d2+d1]		;; R8
	vmulpd	zmm8{k1}, zmm8, zmm29		;; apply fudges to R8					; 16-19		n 28
	vmovapd	zmm20, [srcreg+d4+d2+d1+64]	;; I8
	vmulpd	zmm20{k2}, zmm20, zmm29		;; apply fudges to I8					; 17-20		n 29

	mov	bl, [maskreg+2*1]		;; Load index into compressed fudges
	mov	r14, [r13+2*8]			;; Load the xor mask
	xor	r14, [r12+rbx*8]		;; Apply the xor mask to 64 bits of fudge flags

	kmovw	k1, r14d			;; Load R9 and I9 fudge factor mask			; 17
	kshiftrw k2, k1, 8			;; I9's fudge						; 18
	vmovapd	zmm9, [srcreg+2*d4]		;; R9
	vmulpd	zmm9{k1}, zmm9, zmm29		;; apply fudges to R9					; 18-21		n 29
	vmovapd	zmm21, [srcreg+2*d4+64]		;; I9
	vmulpd	zmm21{k2}, zmm21, zmm29		;; apply fudges to I9					; 19-22		n 30

	shr	r14, 16				;; Next 16 bits of fudge flags
	kmovw	k1, r14d			;; Load R10 and I10 fudge factor mask			; 19
	kshiftrw k2, k1, 8			;; I10's fudge						; 20
	vmovapd	zmm10, [srcreg+2*d4+d1]		;; R10
	vmulpd	zmm10{k1}, zmm10, zmm29		;; apply fudges to R10					; 20-23		n 33
	vmovapd	zmm22, [srcreg+2*d4+d1+64]	;; I10
	vmulpd	zmm22{k2}, zmm22, zmm29		;; apply fudges to I10					; 21-24		n 34

	shr	r14, 16				;; Next 16 bits of fudge flags
	kmovw	k1, r14d			;; Load R11 and I11 fudge factor mask			; 21
	kshiftrw k2, k1, 8			;; I11's fudge						; 22
	vmovapd	zmm11, [srcreg+2*d4+d2]		;; R11
	vmulpd	zmm11{k1}, zmm11, zmm29		;; apply fudges to R11					; 22-25		n 30
	vmovapd	zmm23, [srcreg+2*d4+d2+64]	;; I11
	vmulpd	zmm23{k2}, zmm23, zmm29		;; apply fudges to I11					; 23-26		n 31

	shr	r14, 16				;; Next 16 bits of fudge flags
	kmovw	k1, r14d			;; Load R12 and I12 fudge factor mask			; 23
	kshiftrw k2, k1, 8			;; I12's fudge						; 24
	vmovapd	zmm12, [srcreg+2*d4+d2+d1]	;; R12
	vmulpd	zmm12{k1}, zmm12, zmm29		;; apply fudges to R12					; 24-27		n 31
	vmovapd	zmm24, [srcreg+2*d4+d2+d1+64]	;; I12
	vmulpd	zmm24{k2}, zmm24, zmm29		;; apply fudges to I12					; 25-28		n 32

	bump	maskreg, maskinc

	vmulpd	zmm13, zmm13, [grpreg+1*64]	;; apply group multiplier I1 over R1 to I1		; 25-28		n 34
	vmulpd	zmm14, zmm14, [grpreg+3*64]	;; apply group multiplier I2 over R2 to I2		; 26-29		n 35
	vmulpd	zmm15, zmm15, [grpreg+5*64]	;; apply group multiplier I3 over R3 to I3		; 26-29		n 36
	vmulpd	zmm16, zmm16, [grpreg+7*64]	;; apply group multiplier I4 over R4 to I4		; 27-30		n 37
	vmulpd	zmm17, zmm17, [grpreg+9*64]	;; apply group multiplier I5 over R5 to I5		; 27-30		n 38
	vmulpd	zmm18, zmm18, [grpreg+11*64]	;; apply group multiplier I6 over R6 to I6		; 28-31		n 39
	vmulpd	zmm8, zmm8, [grpreg+14*64]	;; apply group multiplier to R8				; 28-31		n 40
	vmulpd	zmm20, zmm20, [grpreg+15*64]	;; apply group multiplier to I8				; 29-32		n 40
	vmulpd	zmm9, zmm9, [grpreg+16*64]	;; apply group multiplier to R9				; 29-32		n 41
	vmulpd	zmm21, zmm21, [grpreg+17*64]	;; apply group multiplier to I9				; 30-33		n 41
	vmulpd	zmm11, zmm11, [grpreg+20*64]	;; apply group multiplier to R11			; 30-33		n 42
	vmulpd	zmm23, zmm23, [grpreg+21*64]	;; apply group multiplier to I11			; 31-34		n 42
	vmulpd	zmm12, zmm12, [grpreg+22*64]	;; apply group multiplier to R12			; 31-34		n 43
	vmulpd	zmm24, zmm24, [grpreg+23*64]	;; apply group multiplier to I12			; 32-35		n 43
	vmulpd	zmm7, zmm7, [grpreg+12*64]	;; apply group multiplier to R7				; 32-35		n 44
	vmulpd	zmm19, zmm19, [grpreg+13*64]	;; apply group multiplier to I7				; 33-36		n 44
	vmulpd	zmm10, zmm10, [grpreg+18*64]	;; apply group multiplier to R10			; 33-36		n 45
	vmulpd	zmm22, zmm22, [grpreg+19*64]	;; apply group multiplier to I10			; 34-37		n 45

;; Apply the complex premultipliers (the cosine/sine part, the sine part was pre-applied to the group multiplier)

	vmovapd zmm28, [screg+0*64]		;; premultiplier cosine/sine for R1/I1
	zfmsubpd zmm0, zmm1, zmm28, zmm13	;; A1 = R1 * cosine - I1				; 34-37		n 54
	zfmaddpd zmm13, zmm13, zmm28, zmm1	;; B1 = I1 * cosine + R1				; 35-38		n 56

	vmovapd zmm28, [screg+1*64]		;; premultiplier cosine/sine for R2/I2
	zfmsubpd zmm1, zmm2, zmm28, zmm14	;; A2 = R2 * cosine - I2				; 35-38		n 48
	zfmaddpd zmm14, zmm14, zmm28, zmm2	;; B2 = I2 * cosine + R2				; 36-39		n 52

	vmovapd zmm28, [screg+2*64]		;; premultiplier cosine/sine for R3/I3
	zfmsubpd zmm2, zmm3, zmm28, zmm15	;; A3 = R3 * cosine - I3				; 36-39		n 46
	zfmaddpd zmm15, zmm15, zmm28, zmm3	;; B3 = I3 * cosine + R3				; 37-40		n 50

	vmovapd zmm28, [screg+3*64]		;; premultiplier cosine/sine for R4/I4
	zfmsubpd zmm3, zmm4, zmm28, zmm16	;; A4 = R4 * cosine - I4				; 37-40		n 55
	zfmaddpd zmm16, zmm16, zmm28, zmm4	;; B4 = I4 * cosine + R4				; 38-41		n 57

	vmovapd zmm28, [screg+4*64]		;; premultiplier cosine/sine for R5/I5
	zfmsubpd zmm4, zmm5, zmm28, zmm17	;; A5 = R5 * cosine - I5				; 38-41		n 47
	zfmaddpd zmm17, zmm17, zmm28, zmm5	;; B5 = I5 * cosine + R5				; 39-42		n 51

	vmovapd zmm28, [screg+5*64]		;; premultiplier cosine/sine for R6/I6
	zfmsubpd zmm5, zmm6, zmm28, zmm18	;; A6 = R6 * cosine - I6				; 39-42		n 49
	zfmaddpd zmm18, zmm18, zmm28, zmm6	;; B6 = I6 * cosine + R6				; 40-43		n 53

	vmovapd zmm28, [screg+7*64]		;; premultiplier cosine/sine for R8/I8
	zfmsubpd zmm6, zmm8, zmm28, zmm20	;; A8 = R8 * cosine - I8				; 40-43		n 48
	zfmaddpd zmm20, zmm20, zmm28, zmm8	;; B8 = I8 * cosine + R8				; 41-44		n 52

	vmovapd zmm28, [screg+8*64]		;; premultiplier cosine/sine for R9/I9
	zfmsubpd zmm8, zmm9, zmm28, zmm21	;; A9 = R9 * cosine - I9				; 41-44		n 46
	zfmaddpd zmm21, zmm21, zmm28, zmm9	;; B9 = I9 * cosine + R9				; 42-45		n 50

	vmovapd zmm28, [screg+10*64]		;; premultiplier cosine/sine for R11/I11
	zfmsubpd zmm9, zmm11, zmm28, zmm23	;; A11 = R11 * cosine - I11				; 42-45		n 47
	zfmaddpd zmm23, zmm23, zmm28, zmm11	;; B11 = I11 * cosine + R11				; 43-46		n 51

	vmovapd zmm28, [screg+11*64]		;; premultiplier cosine/sine for R12/I12
	zfmsubpd zmm11, zmm12, zmm28, zmm24	;; A12 = R12 * cosine - I12				; 43-46		n 49
	zfmaddpd zmm24, zmm24, zmm28, zmm12	;; B12 = I12 * cosine + R12				; 44-47		n 53

	vmovapd zmm28, [screg+6*64]		;; premultiplier cosine/sine for R7/I7
	zfmsubpd zmm12, zmm7, zmm28, zmm19	;; A7 = R7 * cosine - I7				; 44-47		n 54
	zfmaddpd zmm19, zmm19, zmm28, zmm7	;; B7 = I7 * cosine + R7				; 45-48		n 56

	vmovapd zmm28, [screg+9*64]		;; premultiplier cosine/sine for R10/I10
	zfmsubpd zmm7, zmm10, zmm28, zmm22	;; A10 = R10 * cosine - I10				; 45-48		n 55
	zfmaddpd zmm22, zmm22, zmm28, zmm10	;; B10 = I10 * cosine + R10				; 46-49		n 57

; Now a standard twelve complex DJB FFT

	vmovapd zmm28, [grpreg+4*64]		;; group multiplier for R3
	zfmaddpd zmm10, zmm2, zmm28, zmm8	;; R3*grpmult3+R9					; 46-49		n 58
	zfmsubpd zmm2, zmm2, zmm28, zmm8	;; R3*grpmult3-R9					; 47-50		n 64

	vmovapd zmm27, [grpreg+8*64]		;; group multiplier for R5
	zfmaddpd zmm8, zmm4, zmm27, zmm9	;; R5*grpmult5+R11					; 47-50		n 58
	zfmsubpd zmm4, zmm4, zmm27, zmm9	;; R5*grpmult5-R11					; 48-51		n 64

	vmovapd zmm26, [grpreg+2*64]		;; group multiplier for R2
	zfmaddpd zmm9, zmm1, zmm26, zmm6	;; R2*grpmult2+R8					; 48-51		n 59
	zfmsubpd zmm1, zmm1, zmm26, zmm6	;; R2*grpmult2-R8					; 49-52		n 75

	vmovapd zmm25, [grpreg+10*64]		;; group multiplier for R6
	zfmaddpd zmm6, zmm5, zmm25, zmm11	;; R6*grpmult6+R12					; 49-52		n 59
	zfmsubpd zmm5, zmm5, zmm25, zmm11	;; R6*grpmult6-R12					; 50-53		n 75

	L1prefetchw srcreg+L1pd, L1pt
	zfmaddpd zmm11, zmm15, zmm28, zmm21	;; I3*grpmult3+I9					; 50-53		n 60
	zfmsubpd zmm15, zmm15, zmm28, zmm21	;; I3*grpmult3-I9					; 51-54		n 74

	vmovapd zmm28, [grpreg+0*64]		;; group multiplier for R1
	zfmaddpd zmm21, zmm17, zmm27, zmm23	;; I5*grpmult5+I11					; 51-54		n 60
	zfmsubpd zmm17, zmm17, zmm27, zmm23	;; I5*grpmult5-I11					; 52-55		n 74

	vmovapd zmm27, [grpreg+6*64]		;; group multiplier for R4
	zfmaddpd zmm23, zmm14, zmm26, zmm20	;; I2*grpmult2+I8					; 52-55		n 61
	zfmsubpd zmm14, zmm14, zmm26, zmm20	;; I2*grpmult2-I8					; 53-56		n 69

	L1prefetchw srcreg+64+L1pd, L1pt
	zfmaddpd zmm20, zmm18, zmm25, zmm24	;; I6*grpmult6+I12					; 53-56		n 61
	zfmsubpd zmm18, zmm18, zmm25, zmm24	;; I6*grpmult6-I12					; 54-57		n 69

	L1prefetchw srcreg+d1+L1pd, L1pt
	zfmaddpd zmm24, zmm0, zmm28, zmm12	;; R1*grpmult1+R7					; 54-57		n 62
	zfmsubpd zmm0, zmm0, zmm28, zmm12	;; R1*grpmult1-R7					; 55-58		n 80

	L1prefetchw srcreg+d1+64+L1pd, L1pt
	zfmaddpd zmm12, zmm3, zmm27, zmm7	;; R4*grpmult4+R10					; 55-58		n 62
	zfmsubpd zmm3, zmm3, zmm27, zmm7	;; R4*grpmult4-R10					; 56-59		n 90

	L1prefetchw srcreg+d2+L1pd, L1pt
	zfmaddpd zmm7, zmm13, zmm28, zmm19	;; I1*grpmult1+I7					; 56-59		n 63
	zfmsubpd zmm13, zmm13, zmm28, zmm19	;; I1*grpmult1-I7					; 57-60		n 86

	vmovapd zmm28, [screg+12*64+5*128+64]	;; cosine/sine for R7/I7 (w^6)
	zfmaddpd zmm19, zmm16, zmm27, zmm22	;; I4*grpmult4+I10					; 57-60		n 63
	zfmsubpd zmm16, zmm16, zmm27, zmm22	;; I4*grpmult4-I10					; 58-61		n 87

	bump	grpreg, grpinc

	vmovapd zmm27, [screg+12*64+1*128]	;; sine for R3/I3 and R11/I11 (w^2)
	vaddpd	zmm22, zmm10, zmm8		;; r3++ = (r3+r9) + (r5+r11)				; 58-61		n 65
	vsubpd	zmm10, zmm10, zmm8		;; r3+- = (r3+r9) - (r5+r11)				; 59-62		n 68

	vmovapd zmm26, [screg+12*64+3*128]	;; sine for R5/I5 and R9/I9 (w^4)
	vaddpd	zmm8, zmm9, zmm6		;; r2++ = (r2+r8) + (r6+r12)				; 59-62		n 65
	vsubpd	zmm9, zmm9, zmm6		;; r2+- = (r2+r8) - (r6+r12)				; 60-63		n 68

	vmovapd zmm25, [screg+12*64+5*128]	;; sine for R7/I7 (w^6)
	vaddpd	zmm6, zmm11, zmm21		;; i3++ = (i3+i9) + (i5+i11)				; 60-63		n 66
	vsubpd	zmm11, zmm11, zmm21		;; i3+- = (i3+i9) - (i5+i11)				; 61-64		n 67

	L1prefetchw srcreg+d2+64+L1pd, L1pt
	vaddpd	zmm21, zmm23, zmm20		;; i2++ = (i2+i8) + (i6+i12)				; 61-64		n 66
	vsubpd	zmm23, zmm23, zmm20		;; i2+- = (i2+i8) - (i6+i12)				; 62-65		n 67

	L1prefetchw srcreg+d2+d1+L1pd, L1pt
	vaddpd	zmm20, zmm24, zmm12		;; r1++ = (r1+r7) + (r4+r10)				; 62-65		n 72
	vsubpd	zmm24, zmm24, zmm12		;; r1+- = (r1+r7) - (r4+r10)				; 63-66		n 70

	L1prefetchw srcreg+d2+d1+64+L1pd, L1pt
	vaddpd	zmm12, zmm7, zmm19		;; i1++ = (i1+i7) + (i4+i10)				; 63-66		n 73
	vsubpd	zmm7, zmm7, zmm19		;; i1+- = (i1+i7) - (i4+i10)				; 64-67		n 71

	L1prefetchw srcreg+d4+L1pd, L1pt
	vaddpd	zmm19, zmm2, zmm4		;; r3-+ = (r3-r9) + (r5-r11)				; 64-67		n 95
	vsubpd	zmm2, zmm2, zmm4		;; r3-- = (r3-r9) - (r5-r11)				; 65-68		n 80

	L1prefetchw srcreg+d4+64+L1pd, L1pt
	vsubpd	zmm4, zmm22, zmm8		;; r3++- = (r3++) - (r2++)				; 65-68		n 70
	vaddpd	zmm22, zmm22, zmm8		;; r3+++ = (r3++) + (r2++)				; 66-69		n 72

	L1prefetchw srcreg+d4+d1+L1pd, L1pt
	vsubpd	zmm8, zmm6, zmm21		;; i3++- = (i3++) - (i2++)				; 66-69		n 71
	vaddpd	zmm6, zmm6, zmm21		;; i3+++ = (i3++) + (i2++)				; 67-70		n 73

	L1prefetchw srcreg+d4+d1+64+L1pd, L1pt
	vaddpd	zmm21, zmm11, zmm23		;; i3+-+ = (i3+-) + (i2+-)				; 67-70		n 82
	vsubpd	zmm11, zmm11, zmm23		;; i3+-- = (i3+-) - (i2+-)				; 68-71		n 84

	L1prefetchw srcreg+d4+d2+L1pd, L1pt
	vaddpd	zmm23, zmm10, zmm9		;; r3+-+ = (r3+-) + (r2+-)				; 68-71		n 83
	vsubpd	zmm10, zmm10, zmm9		;; r3+-- = (r3+-) - (r2+-)				; 69-72		n 85

	L1prefetchw srcreg+d4+d2+64+L1pd, L1pt
	vaddpd	zmm9, zmm14, zmm18		;; i2-+ = (i2-i8) + (i6-i12)				; 69-72		n 87
	vsubpd	zmm14, zmm14, zmm18		;; i2-- = (i2-i8) - (i6-i12)				; 70-73		n 93

	L1prefetchw srcreg+d4+d2+d1+L1pd, L1pt
	vaddpd	zmm18, zmm24, zmm4		;; R7 = (r1+-) + (r3++-)				; 70-73		n 76
	zfnmaddpd zmm4, zmm4, zmm31, zmm24	;; R3B = (r1+-) - .500(r3++-)				; 71-74		n 77

	L1prefetchw srcreg+d4+d2+d1+64+L1pd, L1pt
	vaddpd	zmm24, zmm7, zmm8		;; I7 = (i1+-) + (i3++-)				; 71-74		n 76
	zfnmaddpd zmm8, zmm8, zmm31, zmm7	;; I3B = (i1+-) - .500(i3++-)				; 72-75		n 78

	L1prefetchw srcreg+2*d4+L1pd, L1pt
	vaddpd	zmm7, zmm20, zmm22		;; R1 = (r1++) + (r3+++)				; 72-75
	zfnmaddpd zmm22, zmm22, zmm31, zmm20	;; R59 = (r1++) - .500(r3+++)				; 73-76		n 79

	L1prefetchw srcreg+2*d4+64+L1pd, L1pt
	vaddpd	zmm20, zmm12, zmm6		;; I1 = (i1++) + (i3+++)				; 73-76
	zfnmaddpd zmm6, zmm6, zmm31, zmm12	;; I59 = (i1++) - .500(i3+++)				; 74-77		n 80

	L1prefetchw srcreg+2*d4+d1+L1pd, L1pt
	vaddpd	zmm12, zmm15, zmm17		;; i3-+ = (i3-i9) + (i5-i11)				; 74-77		n 94
	vsubpd	zmm15, zmm15, zmm17		;; i3-- = (i3-i9) - (i5-i11)				; 75-78		n 86

	L1prefetchw srcreg+2*d4+d1+64+L1pd, L1pt
	vaddpd	zmm17, zmm1, zmm5		;; r2-+ = (r2-r8) + (r6-r12)				; 75-78		n 90
	vsubpd	zmm1, zmm1, zmm5		;; r2-- = (r2-r8) - (r6-r12)				; 76-79		n 92

	L1prefetchw srcreg+2*d4+d2+L1pd, L1pt
	zfmsubpd zmm5, zmm18, zmm28, zmm24	;; A7 = R7 * cosine/sine - I7				; 76-79		n 81
	zfmaddpd zmm24, zmm24, zmm28, zmm18	;; B7 = I7 * cosine/sine + R7				; 77-80		n 81
	zstore	[srcreg], zmm7			;; Save R1						; 31

	vmovapd zmm28, [screg+12*64+1*128+64]	;; cosine/sine for R3/I3 and R11/I11 (w^2)
	vmulpd	zmm4, zmm4, zmm27		;; R3Bs = R3B * sine3B					; 77-80		n 82
	vmulpd	zmm8, zmm8, zmm27		;; I3Bs = I3B * sine3B					; 78-81		n 83
	zstore	[srcreg+64], zmm20		;; Save I1						; 32

	vmovapd zmm7, [screg+12*64+3*128+64]	;; cosine/sine for R5/I5 and R9/I9 (w^4)
	vmulpd	zmm27, zmm27, zmm30		;; .866s = .866 * sine3B				; 78-81		n 82
	vmulpd	zmm22, zmm22, zmm26		;; R59s = R59 * sine59					; 79-82		n 84

	vmovapd zmm20, [screg+12*64+2*128]	;; sine for R4/I4 and R10/I10 (w^3)
	vmulpd	zmm18, zmm26, zmm30		;; .866s = .866 * sine59				; 79-82		n 84
	vmulpd	zmm6, zmm6, zmm26		;; I59s = I59 * sine59					; 80-83		n 85

	L1prefetchw srcreg+2*d4+d2+64+L1pd, L1pt
	vsubpd	zmm26, zmm0, zmm2		;; r1-- = (r1-r7) - (r3--)				; 80-83		n 96
	zfmaddpd zmm2, zmm2, zmm31, zmm0	;; r1-+ = (r1-r7) + .500(r3--)				; 81-84		n 92

	vmulpd	zmm5, zmm5, zmm25		;; A7 = A7 * sine (final R7)				; 81-84
	vmulpd	zmm24, zmm24, zmm25		;; B7 = B7 * sine (final I7)				; 82-85

	vmovapd zmm25, [screg+12*64+0*128]	;; sine for R2/I2 and R12/I12 (w^1)
	zfnmaddpd zmm0, zmm21, zmm27, zmm4	;; R3s = R3Bs - .866s(i3+-+)				; 82-85		n 88
	zfmaddpd zmm21, zmm21, zmm27, zmm4	;; R11s = R3Bs + .866s(i3+-+)				; 83-86		n 89

	zfmaddpd zmm4, zmm23, zmm27, zmm8	;; I3s = I3Bs + .866s(r3+-+)				; 83-86		n 88
	zfnmaddpd zmm23, zmm23, zmm27, zmm8	;; I11s = I3Bs - .866s(r3+-+)				; 84-87		n 89

	vmovapd zmm27, [screg+12*64+4*128]	;; sine for R6/I6 and R8/I8 (w^5)
	zfmaddpd zmm8, zmm11, zmm18, zmm22	;; R5s = R59s + .866s(i3+--)				; 84-87		n 91
	zfnmaddpd zmm11, zmm11, zmm18, zmm22	;; R9s = R59s - .866s(i3+--)				; 85-88		n 99

	zfnmaddpd zmm22, zmm10, zmm18, zmm6	;; I5s = I59s - .866s(r3+--)				; 85-88		n 91
	zfmaddpd zmm10, zmm10, zmm18, zmm6	;; I9s = I59s + .866s(r3+--)				; 86-89		n 99
	zstore	[srcreg+d4+d2], zmm5		;; Save R7						; 40

	vmovapd zmm18, [screg+12*64+2*128+64]	;; cosine/sine for R4/I4 and R10/I10 (w^3)
	zfmaddpd zmm6, zmm15, zmm31, zmm13	;; i1-+ = (i1-i7) + .500(i3--)				; 86-89		n 93
	vsubpd	zmm13, zmm13, zmm15		;; i1-- = (i1-i7) - (i3--)				; 87-90		n 97
	zstore	[srcreg+d4+d2+64], zmm24	;; Save I7						; 40+1

	vmovapd zmm5, [screg+12*64+0*128+64]	;; cosine/sine for R2/I2 and R12/I12 (w^1)
	zfmaddpd zmm15, zmm9, zmm31, zmm16	;; i4-+ = (i4-i10) + .500(i2-+)				; 87-90		n 94
	vsubpd	zmm16, zmm16, zmm9		;; i4-- = (i4-i10) - (i2-+)				; 88-91		n 100

	vmovapd zmm24, [screg+12*64+4*128+64]	;; cosine/sine for R6/I6 and R8/I8 (w^5)
	zfmsubpd zmm9, zmm0, zmm28, zmm4	;; R3s * cosine/sine - I3s (final R3)			; 88-91
	zfmaddpd zmm4, zmm4, zmm28, zmm0	;; I3s * cosine/sine + R3s (final I3)			; 89-92

	zfmaddpd zmm0, zmm21, zmm28, zmm23	;; A11 = R11s * cosine/sine + I11s (final R11)		; 89-92
	zfmsubpd zmm23, zmm23, zmm28, zmm21	;; B11 = I11s * cosine/sine - R11s (final I11)		; 90-93
	bump	screg, scinc

	zfmaddpd zmm21, zmm17, zmm31, zmm3	;; r4-+ = (r4-r10) + .500(r2-+)				; 90-93		n 95
	vsubpd	zmm17, zmm3, zmm17		;; r4-- = (r4-r10) - (r2-+)				; 91-94		n 101

	zfmsubpd zmm3, zmm8, zmm7, zmm22	;; R5s * cosine/sine - I5s (final R5)			; 91-94
	zfmaddpd zmm22, zmm22, zmm7, zmm8	;; I5s * cosine/sine + R5s (final I5)			; 92-95

	L1prefetchw srcreg+2*d4+d2+d1+L1pd, L1pt
	zfmaddpd zmm8, zmm1, zmm30, zmm2	;; r1-++ = (r1-+) + .866(r2--)				; 92-95		n 97
	zfnmaddpd zmm1, zmm1, zmm30, zmm2	;; r1-+- = (r1-+) - .866(r2--)				; 93-96		n 98
	zstore	[srcreg+d2], zmm9		;; Save R3						; 92

	L1prefetchw srcreg+2*d4+d2+d1+64+L1pd, L1pt
	zfmaddpd zmm2, zmm14, zmm30, zmm6	;; i1-++ = (i1-+) + .866(i2--)				; 93-96		n 98
	zfnmaddpd zmm14, zmm14, zmm30, zmm6	;; i1-+- = (i1-+) - .866(i2--)				; 94-97		n 99
	zstore	[srcreg+d2+64], zmm4		;; Save I3						; 93

	zfmaddpd zmm6, zmm12, zmm30, zmm15	;; i4-++ = (i4-+) + .866(i3-+)				; 94-97		n 102
	zfnmaddpd zmm12, zmm12, zmm30, zmm15	;; i4-+- = (i4-+) - .866(i3-+)				; 95-98		n 104
	zstore	[srcreg+2*d4+d2], zmm0		;; Save R11						; 93+1

	zfmaddpd zmm15, zmm19, zmm30, zmm21	;; r4-++ = (r4-+) + .866(r3-+)				; 95-98		n 103
	zfnmaddpd zmm19, zmm19, zmm30, zmm21	;; r4-+- = (r4-+) - .866(r3-+)				; 96-99		n 105
	zstore	[srcreg+2*d4+d2+64], zmm23	;; Save I11						; 94+1

	vmulpd	zmm26, zmm26, zmm20		;; r1--s = r1-- * sine4A				; 96-99		n 100
	vmulpd	zmm13, zmm13, zmm20		;; i1--s = i1-- * sine4A				; 97-100	n 101
	zstore	[srcreg+d4], zmm3		;; Save R5						; 95+1

	vmulpd	zmm8, zmm8, zmm25		;; r1-++s = r1-++ * sine2C				; 97-100	n 102
	vmulpd	zmm2, zmm2, zmm25		;; i1-++s = i1-++ * sine2C				; 98-101	n 103
	zstore	[srcreg+d4+64], zmm22		;; Save I5						; 96+1

	vmulpd	zmm1, zmm1, zmm27		;; r1-+-s = r1-+- * sine68				; 98-101	n 104
	vmulpd	zmm14, zmm14, zmm27		;; i1-+-s = i1-+- * sine68				; 99-102	n 105

	zfmaddpd zmm21, zmm11, zmm7, zmm10	;; R9s * cosine/sine + I9s (final R9)			; 99-102
	zfmsubpd zmm10, zmm10, zmm7, zmm11	;; I9s * cosine/sine - R9s (final I9)			; 100-103

	zfmaddpd zmm11, zmm16, zmm20, zmm26	;; R4s = (r1--s) + sine4A(i4--)				; 100-103	n 106
	zfnmaddpd zmm16, zmm16, zmm20, zmm26	;; R10s= (r1--s) - sine4A(i4--)				; 101-104	n 107

	zfnmaddpd zmm26, zmm17, zmm20, zmm13	;; I4s = i1--s - sine4A(r4--)				; 101-104	n 106
	zfmaddpd zmm17, zmm17, zmm20, zmm13	;; I10s= i1--s + sine4A(r4--)				; 102-105	n 107

	zfnmaddpd zmm13, zmm6, zmm25, zmm8	;; R2s = r1-++s - sine2C(i4-++)				; 102-105	n 108
	zfmaddpd zmm6, zmm6, zmm25, zmm8	;; R12s= r1-++s + sine2C(i4-++)				; 103-106	n 109

	zfmaddpd zmm8, zmm15, zmm25, zmm2	;; I2s = i1-++s + sine2C(r4-++)				; 103-106	n 108
	zfnmaddpd zmm15, zmm15, zmm25, zmm2	;; I12s= i1-++s - sine2C(r4-++)				; 104-107	n 109
	zstore	[srcreg+2*d4], zmm21		;; Save R9						; 103

	zfnmaddpd zmm2, zmm12, zmm27, zmm1	;; R6s = r1-+-s - sine68(i4-+-)				; 104-107	n 110
	zfmaddpd zmm12, zmm12, zmm27, zmm1	;; R8s = r1-+-s + sine68(i4-+-)				; 105-108	n 111
	zstore	[srcreg+2*d4+64], zmm10		;; Save I9						; 104

	zfmaddpd zmm1, zmm19, zmm27, zmm14	;; I6s = i1-+-s + sine68(r4-+-)				; 105-108	n 110
	zfnmaddpd zmm19, zmm19, zmm27, zmm14	;; I8s = i1-+-s - sine68(r4-+-)				; 106-109	n 111

	zfmsubpd zmm14, zmm11, zmm18, zmm26	;; R4s * cosine/sine - I4s (final R4)			; 106-109
	zfmaddpd zmm26, zmm26, zmm18, zmm11	;; I4s * cosine/sine + R4s (final I4)			; 107-110

	zfmaddpd zmm11, zmm16, zmm18, zmm17	;; R10s * cosine/sine + I10s (final R10)		; 107-110
	zfmsubpd zmm17, zmm17, zmm18, zmm16	;; I10s * cosine/sine - R10s (final I10)		; 108-111

	zfmsubpd zmm16, zmm13, zmm5, zmm8	;; R2s * cosine/sine - I2s (final R2)			; 108-111
	zfmaddpd zmm8, zmm8, zmm5, zmm13	;; I2s * cosine/sine + R2s (final I2)			; 109-112

	zfmaddpd zmm13, zmm6, zmm5, zmm15	;; R12s * cosine/sine + I12s (final R12)		; 109-112
	zfmsubpd zmm15, zmm15, zmm5, zmm6	;; I12s * cosine/sine - R12s (final I12)		; 110-113

	zfmsubpd zmm6, zmm2, zmm24, zmm1	;; R6s * cosine/sine - I6s (final R6)			; 110-113
	zfmaddpd zmm1, zmm1, zmm24, zmm2	;; I6s * cosine/sine + R6s (final I6)			; 111-114

	zfmaddpd zmm2, zmm12, zmm24, zmm19	;; R8s * cosine/sine + I8s (final R8)			; 111-114
	zfmsubpd zmm19, zmm19, zmm24, zmm12	;; I8s * cosine/sine - R8s (final I8)			; 112-115

	zstore	[srcreg+d2+d1], zmm14		;; Save R4						; 110
	zstore	[srcreg+d2+d1+64], zmm26	;; Save I4						; 111
	zstore	[srcreg+2*d4+d1], zmm11		;; Save R10						; 111+1
	zstore	[srcreg+2*d4+d1+64], zmm17	;; Save I10						; 112+1
	zstore	[srcreg+d1], zmm16		;; Save R2						; 112+2
	zstore	[srcreg+d1+64], zmm8		;; Save I2						; 113+2
	zstore	[srcreg+2*d4+d2+d1], zmm13	;; Save R12						; 113+3
	zstore	[srcreg+2*d4+d2+d1+64], zmm15	;; Save I12						; 114+3
	zstore	[srcreg+d4+d1], zmm6		;; Save R6						; 114+4
	zstore	[srcreg+d4+d1+64], zmm1		;; Save I6						; 115+4
	zstore	[srcreg+d4+d2+d1], zmm2		;; Save R8						; 115+5
	zstore	[srcreg+d4+d2+d1+64], zmm19	;; Save I8						; 116+6
	bump	srcreg, srcinc
	ENDM


;;
;; ************************************* twelve-complex-djbunfft variants ******************************************
;;

;; The standard version
zr12_twelve_complex_djbunfft_preload MACRO
	zr12_12c_djbunfft_cmn_preload
	ENDM
zr12_twelve_complex_djbunfft MACRO srcreg,srcinc,d1,d2,d4,screg,scinc,maxrpt,L1pt,L1pd
	zr12_12c_djbunfft_cmn srcreg,srcinc,d1,d2,d4,noexec,screg,scinc,maxrpt,L1pt,L1pd
	ENDM

; Like the standard version except vbroadcastsd is used to reduce sin/cos data
zr12b_twelve_complex_djbunfft_preload MACRO
	zr12_12c_djbunfft_cmn_preload
	ENDM
zr12b_twelve_complex_djbunfft MACRO srcreg,srcinc,d1,d2,d4,screg,scinc,maxrpt,L1pt,L1pd
	zr12_12c_djbunfft_cmn srcreg,srcinc,d1,d2,d4,exec,screg,scinc,maxrpt,L1pt,L1pd
	ENDM

;; Common macro to operate on 12 complex values doing 3.585 levels of the inverse FFT, applying
;; the sin/cos multipliers beforehand.

;; To calculate a 12-complex inverse FFT in a brute force way (using a shorthand notation):
;; c1 + c2 + ... + c12 * w^-000000000000
;; c1 + c2 + ... + c12 * w^-0123456789AB
;; c1 + c2 + ... + c12 * w^-02468A02468A
;; ...
;; c1 + c2 + ... + c12 * w^-0A86420A8642
;; c1 + c2 + ... + c12 * w^-0BA987654321
;;
;; The sin/cos values (w = 12th root of unity) are:
;; w^-1 =  .866 - .5i
;; w^-2 =  .5   - .866i
;; w^-3 =  0    - 1i
;; w^-4 = -.5   - .866i
;; w^-5 = -.866 - .5i
;; w^-6 = -1

;; Applying the sin/cos values above:
;; reals:
;; r1     +r2     +r3 +r4     +r5     +r6 + r7    + r8    + r9 +r10    + r11     +r12
;; r1 +.866r2 +.500r3     -.500r5 -.866r6 - r7 -.866r8 -.500r9      +.500r11 +.866r12 +.500i2 +.866i3 +i4 +.866i5 +.500i6 -.500i8 -.866i9 -i10 -.866i11 -.500i12
;; r1 +.500r2 -.500r3 -r4 -.500r5 +.500r6 + r7 +.500r8 -.500r9 -r10 -.500r11 +.500r12 +.866i2 +.866i3     -.866i5 -.866i6 +.866i8 +.866i9      -.866i11 -.866i12
;; r1             -r3         +r5         - r7             +r9          -r11              +i2         -i4             +i6     -i8         +i10              -i12
;; r1 -.500r2 -.500r3 +r4 -.500r5 -.500r6 + r7 -.500r8 -.500r9 +r10 -.500r11 -.500r12 +.866i2 -.866i3     +.866i5 -.866i6 +.866i8 -.866i9      +.866i11 -.866i12
;; r1 -.866r2 +.500r3     -.500r5 +.866r6 - r7 +.866r8 -.500r9      +.500r11 -.866r12 +.500i2 -.866i3 +i4 -.866i5 +.500i6 -.500i8 +.866i9 -i10 +.866i11 -.500i12
;; r1     -r2     +r3 -r4     +r5     -r6 + r7    - r8    + r9 -r10    + r11     -r12
;; r1 -.866r2 +.500r3     -.500r5 +.866r6 - r7 +.866r8 -.500r9      +.500r11 -.866r12 -.500i2 +.866i3 -i4 +.866i5 -.500i6 +.500i8 -.866i9 +i10 -.866i11 +.500i12
;; r1 -.500r2 -.500r3 +r4 -.500r5 -.500r6 + r7 -.500r8 -.500r9 +r10 -.500r11 -.500r12 -.866i2 +.866i3     -.866i5 +.866i6 -.866i8 +.866i9      -.866i11 +.866i12
;; r1             -r3         +r5         - r7             +r9         - r11              -i2         +i4             -i6     +i8         -i10              +i12
;; r1 +.500r2 -.500r3 -r4 -.500r5 +.500r6 + r7 +.500r8 -.500r9 -r10 -.500r11 +.500r12 -.866i2 -.866i3     +.866i5 +.866i6 -.866i8 -.866i9      +.866i11 +.866i12
;; r1 +.866r2 +.500r3     -.500r5 -.866r6 - r7 -.866r8 -.500r9      +.500r11 +.866r12 -.500i2 -.866i3 -i4 -.866i5 -.500i6 +.500i8 +.866i9 +i10 +.866i11 +.500i12

;; imaginarys:
;;                                                                            +i1     +i2     +i3 +i4     +i5     +i6  +i7     +i8     +i9 +i10     +i11     +i12
;; -.500r2 -.866r3 -r4 -.866r5 -.500r6 +.500r8 +.866r9 +r10 +.866r11 +.500r12 +i1 +.866i2 +.500i3     -.500i5 -.866i6  -i7 -.866i8 -.500i9      +.500i11 +.866i12
;; -.866r2 -.866r3     +.866r5 +.866r6 -.866r8 -.866r9      +.866r11 +.866r12 +i1 +.500i2 -.500i3 -i4 -.500i5 +.500i6  +i7 +.500i8 -.500i9 -i10 -.500i11 +.500i12
;;     -r2         +r4             -r6     +r8         -r10              +r12 +i1             -i3         +i5          -i7             +i9          -i11
;; -.866r2 +.866r3     -.866r5 +.866r6 -.866r8 +.866r9      -.866r11 +.866r12 +i1 -.500i2 -.500i3 +i4 -.500i5 -.500i6  +i7 -.500i8 -.500i9 +i10 -.500i11 -.500i12
;; -.500r2 +.866r3 -r4 +.866r5 -.500r6 +.500r8 -.866r9 +r10 -.866r11 +.500r12 +i1 -.866i2 +.500i3     -.500i5 +.866i6  -i7 +.866i8 -.500i9      +.500i11 -.866i12
;;                                                                            +i1     -i2     +i3 -i4     +i5     -i6  +i7     -i8     +i9 -i10     +i11     -i12
;; +.500r2 -.866r3 +r4 -.866r5 +.500r6 -.500r8 +.866r9 -r10 +.866r11 -.500r12 +i1 -.866i2 +.500i3     -.500i5 +.866i6  -i7 +.866i8 -.500i9      +.500i11 -.866i12
;; +.866r2 -.866r3     +.866r5 -.866r6 +.866r8 -.866r9      +.866r11 -.866r12 +i1 -.500i2 -.500i3 +i4 -.500i5 -.500i6  +i7 -.500i8 -.500i9 +i10 -.500i11 -.500i12
;;     +r2         -r4             +r6     -r8         +r10              -r12 +i1             -i3         +i5          -i7             +i9          -i11
;; +.866r2 +.866r3     -.866r5 -.866r6 +.866r8 +.866r9      -.866r11 -.866r12 +i1 +.500i2 -.500i3 -i4 -.500i5 +.500i6  +i7 +.500i8 -.500i9 -i10 -.500i11 +.500i12
;; +.500r2 +.866r3 +r4 +.866r5 +.500r6 -.500r8 -.866r9 -r10 -.866r11 -.500r12 +i1 +.866i2 +.500i3     -.500i5 -.866i6  -i7 -.866i8 -.500i9      +.500i11 +.866i12

;; Massive simplifying, gets:
;; R1 = (r1+r7)+(r4+r10)     +(((r3+r9)+(r5+r11))+((r2+r8)+(r6+r12)))
;; R5 = (r1+r7)+(r4+r10) -.500(((r3+r9)+(r5+r11))+((r2+r8)+(r6+r12))) -.866(((i3+i9)-(i5+i11))-((i2+i8)-(i6+i12)))
;; R9 = (r1+r7)+(r4+r10) -.500(((r3+r9)+(r5+r11))+((r2+r8)+(r6+r12))) +.866(((i3+i9)-(i5+i11))-((i2+i8)-(i6+i12)))
;; R7 = (r1+r7)-(r4+r10)     +(((r3+r9)+(r5+r11))-((r2+r8)+(r6+r12)))
;; R3 = (r1+r7)-(r4+r10) -.500(((r3+r9)+(r5+r11))-((r2+r8)+(r6+r12))) +.866(((i3+i9)-(i5+i11))+((i2+i8)-(i6+i12)))
;; R11= (r1+r7)-(r4+r10) -.500(((r3+r9)+(r5+r11))-((r2+r8)+(r6+r12))) -.866(((i3+i9)-(i5+i11))+((i2+i8)-(i6+i12)))

;; I1 = (i1+i7)+(i4+i10)     +(((i3+i9)+(i5+i11))+((i2+i8)+(i6+i12)))
;; I5 = (i1+i7)+(i4+i10) -.500(((i3+i9)+(i5+i11))+((i2+i8)+(i6+i12))) +.866(((r3+r9)-(r5+r11))-((r2+r8)-(r6+r12)))
;; I9 = (i1+i7)+(i4+i10) -.500(((i3+i9)+(i5+i11))+((i2+i8)+(i6+i12))) -.866(((r3+r9)-(r5+r11))-((r2+r8)-(r6+r12)))
;; I7 = (i1+i7)-(i4+i10)     +(((i3+i9)+(i5+i11))-((i2+i8)+(i6+i12)))
;; I3 = (i1+i7)-(i4+i10) -.500(((i3+i9)+(i5+i11))-((i2+i8)+(i6+i12))) -.866(((r3+r9)-(r5+r11))+((r2+r8)-(r6+r12)))
;; I11= (i1+i7)-(i4+i10) -.500(((i3+i9)+(i5+i11))-((i2+i8)+(i6+i12))) +.866(((r3+r9)-(r5+r11))+((r2+r8)-(r6+r12)))

;; R4 = (r1-r7)-(i4-i10)     -(((r3-r9)-(r5-r11))-((i2-i8)+(i6-i12)))
;; R8 = (r1-r7)-(i4-i10) +.500(((r3-r9)-(r5-r11))-((i2-i8)+(i6-i12))) +.866(((i3-i9)+(i5-i11))-((r2-r8)-(r6-r12)))
;; R12= (r1-r7)-(i4-i10) +.500(((r3-r9)-(r5-r11))-((i2-i8)+(i6-i12))) -.866(((i3-i9)+(i5-i11))-((r2-r8)-(r6-r12)))
;; R10= (r1-r7)+(i4-i10)     -(((r3-r9)-(r5-r11))+((i2-i8)+(i6-i12)))
;; R2 = (r1-r7)+(i4-i10) +.500(((r3-r9)-(r5-r11))+((i2-i8)+(i6-i12))) +.866(((i3-i9)+(i5-i11))+((r2-r8)-(r6-r12)))
;; R6 = (r1-r7)+(i4-i10) +.500(((r3-r9)-(r5-r11))+((i2-i8)+(i6-i12))) -.866(((i3-i9)+(i5-i11))+((r2-r8)-(r6-r12)))

;; I4 = (i1-i7)+(r4-r10)     -(((i3-i9)-(i5-i11))+((r2-r8)+(r6-r12)))
;; I8 = (i1-i7)+(r4-r10) +.500(((i3-i9)-(i5-i11))+((r2-r8)+(r6-r12))) -.866(((r3-r9)+(r5-r11))+((i2-i8)-(i6-i12)))
;; I12= (i1-i7)+(r4-r10) +.500(((i3-i9)-(i5-i11))+((r2-r8)+(r6-r12))) +.866(((r3-r9)+(r5-r11))+((i2-i8)-(i6-i12)))
;; I10= (i1-i7)-(r4-r10)     -(((i3-i9)-(i5-i11))-((r2-r8)+(r6-r12)))
;; I2 = (i1-i7)-(r4-r10) +.500(((i3-i9)-(i5-i11))-((r2-r8)+(r6-r12))) -.866(((r3-r9)+(r5-r11))-((i2-i8)-(i6-i12)))
;; I6 = (i1-i7)-(r4-r10) +.500(((i3-i9)-(i5-i11))-((r2-r8)+(r6-r12))) +.866(((r3-r9)+(r5-r11))-((i2-i8)-(i6-i12)))

;; Rearranging for more FMA opportunities:
;; R1 = (r1+r7)+(r4+r10)     +(((r3+r11)+(r5+r9))+((r2+r12)+(r6+r8)))
;; R5 = (r1+r7)+(r4+r10) -.500(((r3+r11)+(r5+r9))+((r2+r12)+(r6+r8))) -.866(((i3-i11)-(i5-i9))-((i2-i12)-(i6-i8)))
;; R9 = (r1+r7)+(r4+r10) -.500(((r3+r11)+(r5+r9))+((r2+r12)+(r6+r8))) +.866(((i3-i11)-(i5-i9))-((i2-i12)-(i6-i8)))
;; R7 = (r1+r7)-(r4+r10)     +(((r3+r11)+(r5+r9))-((r2+r12)+(r6+r8)))
;; R3 = (r1+r7)-(r4+r10) -.500(((r3+r11)+(r5+r9))-((r2+r12)+(r6+r8))) +.866(((i3-i11)-(i5-i9))+((i2-i12)-(i6-i8)))
;; R11= (r1+r7)-(r4+r10) -.500(((r3+r11)+(r5+r9))-((r2+r12)+(r6+r8))) -.866(((i3-i11)-(i5-i9))+((i2-i12)-(i6-i8)))

;; I1 = (i1+i7)+(i4+i10)     +(((i3+i11)+(i5+i9))+((i2+i12)+(i6+i8)))
;; I5 = (i1+i7)+(i4+i10) -.500(((i3+i11)+(i5+i9))+((i2+i12)+(i6+i8))) +.866(((r3-r11)-(r5-r9))-((r2-r12)-(r6-r8)))
;; I9 = (i1+i7)+(i4+i10) -.500(((i3+i11)+(i5+i9))+((i2+i12)+(i6+i8))) -.866(((r3-r11)-(r5-r9))-((r2-r12)-(r6-r8)))
;; I7 = (i1+i7)-(i4+i10)     +(((i3+i11)+(i5+i9))-((i2+i12)+(i6+i8)))
;; I3 = (i1+i7)-(i4+i10) -.500(((i3+i11)+(i5+i9))-((i2+i12)+(i6+i8))) -.866(((r3-r11)-(r5-r9))+((r2-r12)-(r6-r8)))
;; I11= (i1+i7)-(i4+i10) -.500(((i3+i11)+(i5+i9))-((i2+i12)+(i6+i8))) +.866(((r3-r11)-(r5-r9))+((r2-r12)-(r6-r8)))

;; R4 = (r1-r7)-(i4-i10)     -(((r3+r11)-(r5+r9))-((i2-i12)+(i6-i8)))
;; R8 = (r1-r7)-(i4-i10) +.500(((r3+r11)-(r5+r9))-((i2-i12)+(i6-i8))) +.866(((i3-i11)+(i5-i9))-((r2+r12)-(r6+r8)))
;; R12= (r1-r7)-(i4-i10) +.500(((r3+r11)-(r5+r9))-((i2-i12)+(i6-i8))) -.866(((i3-i11)+(i5-i9))-((r2+r12)-(r6+r8)))
;; R10= (r1-r7)+(i4-i10)     -(((r3+r11)-(r5+r9))+((i2-i12)+(i6-i8)))
;; R2 = (r1-r7)+(i4-i10) +.500(((r3+r11)-(r5+r9))+((i2-i12)+(i6-i8))) +.866(((i3-i11)+(i5-i9))+((r2+r12)-(r6+r8)))
;; R6 = (r1-r7)+(i4-i10) +.500(((r3+r11)-(r5+r9))+((i2-i12)+(i6-i8))) -.866(((i3-i11)+(i5-i9))+((r2+r12)-(r6+r8)))

;; I4 = (i1-i7)+(r4-r10)     -(((i3+i11)-(i5+i9))+((r2-r12)+(r6-r8)))
;; I8 = (i1-i7)+(r4-r10) +.500(((i3+i11)-(i5+i9))+((r2-r12)+(r6-r8))) -.866(((r3-r11)+(r5-r9))+((i2+i12)-(i6+i8)))
;; I12= (i1-i7)+(r4-r10) +.500(((i3+i11)-(i5+i9))+((r2-r12)+(r6-r8))) +.866(((r3-r11)+(r5-r9))+((i2+i12)-(i6+i8)))
;; I10= (i1-i7)-(r4-r10)     -(((i3+i11)-(i5+i9))-((r2-r12)+(r6-r8)))
;; I2 = (i1-i7)-(r4-r10) +.500(((i3+i11)-(i5+i9))-((r2-r12)+(r6-r8))) -.866(((r3-r11)+(r5-r9))-((i2+i12)-(i6+i8)))
;; I6 = (i1-i7)-(r4-r10) +.500(((i3+i11)-(i5+i9))-((r2-r12)+(r6-r8))) +.866(((r3-r11)+(r5-r9))-((i2+i12)-(i6+i8)))

zr12_12c_djbunfft_cmn_preload MACRO
	vbroadcastsd zmm31, ZMM_HALF
	vbroadcastsd zmm30, ZMM_P866
	ENDM

zr12_12c_djbunfft_cmn MACRO srcreg,srcinc,d1,d2,d4,bcast,screg,scinc,maxrpt,L1pt,L1pd
no bcast vmovapd zmm29, [screg+1*128+64]	;; cosine/sine for R3/I3 and R11/I11 (w^2)
bcast	vbroadcastsd zmm29, Q [screg+1*16+8]	;; cosine/sine for R3/I3 and R11/I11
	vmovapd	zmm2, [srcreg+d2]		;; R3
	vmovapd	zmm14, [srcreg+d2+64]		;; I3
	zfmaddpd zmm24, zmm2, zmm29, zmm14	;; A3 = R3 * cosine/sine + I3				; 1-4		n 12
	zfmsubpd zmm14, zmm14, zmm29, zmm2	;; B3 = I3 * cosine/sine - R3				; 1-4		n 12
	vmovapd	zmm10, [srcreg+2*d4+d2]		;; R11
	vmovapd	zmm22, [srcreg+2*d4+d2+64]	;; I11
	zfmsubpd zmm2, zmm10, zmm29, zmm22	;; A11 = R11 * cosine/sine - I11			; 2-5		n 18
	zfmaddpd zmm22, zmm22, zmm29, zmm10	;; B11 = I11 * cosine/sine + R11			; 2-5		n 20

no bcast vmovapd zmm29, [screg+0*128+64]	;; cosine/sine for R2/I2 and R12/I12 (w^1)
bcast	vbroadcastsd zmm29, Q [screg+0*16+8]	;; cosine/sine for R2/I2 and R12/I12
	vmovapd	zmm1, [srcreg+d1]		;; R2
	vmovapd	zmm13, [srcreg+d1+64]		;; I2
	zfmaddpd zmm10, zmm1, zmm29, zmm13	;; A2 = R2 * cosine/sine + I2				; 3-6		n 13
	zfmsubpd zmm13, zmm13, zmm29, zmm1	;; B2 = I2 * cosine/sine - R2				; 3-6		n 13
	vmovapd	zmm11, [srcreg+2*d4+d2+d1]	;; R12
	vmovapd	zmm23, [srcreg+2*d4+d2+d1+64]	;; I12
	zfmsubpd zmm1, zmm11, zmm29, zmm23	;; A12 = R12 * cosine/sine - I12			; 4-7		n 23
	zfmaddpd zmm23, zmm23, zmm29, zmm11	;; B12 = I12 * cosine/sine + R12			; 4-7		n 25

no bcast vmovapd zmm29, [screg+5*128+64]	;; cosine/sine for R7/I7 (w^6)
bcast	vbroadcastsd zmm29, Q [screg+5*16+8]	;; cosine/sine for R7/I7
	vmovapd	zmm6, [srcreg+d4+d2]		;; R7
	vmovapd	zmm18, [srcreg+d4+d2+64]	;; I7
	zfmaddpd zmm11, zmm6, zmm29, zmm18	;; A7 = R7 * cosine/sine + I7 (new R7/sine)		; 5-8		n 14
	zfmsubpd zmm18, zmm18, zmm29, zmm6	;; B7 = I7 * cosine/sine - R7 (new I7/sine)		; 5-8		n 15

no bcast vmovapd zmm29, [screg+2*128+64]	;; cosine/sine for R4/I4 and R10/I10 (w^3)
bcast	vbroadcastsd zmm29, Q [screg+2*16+8]	;; cosine/sine for R4/I4 and R10/I10
	vmovapd	zmm3, [srcreg+d2+d1]		;; R4
	vmovapd	zmm15, [srcreg+d2+d1+64]	;; I4
	zfmaddpd zmm6, zmm3, zmm29, zmm15	;; A4 = R4 * cosine/sine + I4 (new R4/sine)		; 6-9		n 16
	zfmsubpd zmm15, zmm15, zmm29, zmm3	;; B4 = I4 * cosine/sine - R4 (new I4/sine)		; 6-9		n 17
	vmovapd	zmm9, [srcreg+2*d4+d1]		;; R10
	vmovapd	zmm21, [srcreg+2*d4+d1+64]	;; I10
	zfmsubpd zmm3, zmm9, zmm29, zmm21	;; A10 = R10 * cosine/sine - I10 (new R10/sine)		; 7-10		n 16
	zfmaddpd zmm21, zmm21, zmm29, zmm9	;; B10 = I10 * cosine/sine + R10 (new I10/sine)		; 7-10		n 17

no bcast vmovapd zmm29, [screg+3*128+64]	;; cosine/sine for R5/I5 and R9/I9 (w^4)
bcast	vbroadcastsd zmm29, Q [screg+3*16+8]	;; cosine/sine for R5/I5 and R9/I9
	vmovapd	zmm4, [srcreg+d4]		;; R5
	vmovapd	zmm16, [srcreg+d4+64]		;; I5
	zfmaddpd zmm9, zmm4, zmm29, zmm16	;; A5 = R5 * cosine/sine + I5 (new R5/sine)		; 8-11		n 19
	zfmsubpd zmm16, zmm16, zmm29, zmm4	;; B5 = I5 * cosine/sine - R5 (new I5/sine)		; 8-11		n 21
	vmovapd	zmm8, [srcreg+2*d4]		;; R9
	vmovapd	zmm20, [srcreg+2*d4+64]		;; I9
	zfmsubpd zmm4, zmm8, zmm29, zmm20	;; A9 = R9 * cosine/sine - I9 (new R9/sine)		; 9-12		n 19
	zfmaddpd zmm20, zmm20, zmm29, zmm8	;; B9 = I9 * cosine/sine + R9 (new I9/sine)		; 9-12		n 21

no bcast vmovapd zmm29, [screg+4*128+64]	;; cosine/sine for R6/I6 and R8/I8 (w^5)
bcast	vbroadcastsd zmm29, Q [screg+4*16+8]	;; cosine/sine for R6/I6 and R8/I8
	vmovapd	zmm5, [srcreg+d4+d1]		;; R6
	vmovapd	zmm17, [srcreg+d4+d1+64]	;; I6
	zfmaddpd zmm8, zmm5, zmm29, zmm17	;; A6 = R6 * cosine/sine + I6 (new R6/sine)		; 10-13		n 22
	zfmsubpd zmm17, zmm17, zmm29, zmm5	;; B6 = I6 * cosine/sine - R6 (new I6/sine)		; 10-13		n 24
	vmovapd	zmm7, [srcreg+d4+d2+d1]		;; R8
	vmovapd	zmm19, [srcreg+d4+d2+d1+64]	;; I8
	zfmsubpd zmm5, zmm7, zmm29, zmm19	;; A8 = R8 * cosine/sine - I8 (new R8/sine)		; 11-14		n 22
	zfmaddpd zmm19, zmm19, zmm29, zmm7	;; B8 = I8 * cosine/sine + R8 (new I8/sine)		; 11-14		n 24

no bcast vmovapd zmm29, [screg+1*128]		;; sine for R3/I3 and R11/I11 (w^2)
bcast	vbroadcastsd zmm29, Q [screg+1*16]	;; sine for R3/I3 and R11/I11
	vmulpd	zmm24, zmm24, zmm29		;; A3 = A3 * sine (new R3)				; 12-15		n 18
	vmulpd	zmm14, zmm14, zmm29		;; B3 = B3 * sine (new I3)				; 12-15		n 20

no bcast vmovapd zmm28, [screg+0*128]		;; sine for R2/I2 and R12/I12 (w^1)
bcast	vbroadcastsd zmm28, Q [screg+0*16]	;; sine for R2/I2 and R12/I12
	vmulpd	zmm10, zmm10, zmm28		;; A2 = A2 * sine (new R2)				; 13-16		n 23
	vmulpd	zmm13, zmm13, zmm28		;; B2 = B2 * sine (new I2)				; 13-16		n 25

no bcast vmovapd zmm27, [screg+5*128]		;; sine for R7/I7 (w^6)
bcast	vbroadcastsd zmm27, Q [screg+5*16]	;; sine for R7/I7
	vmovapd	zmm0, [srcreg]			;; R1
	zfmaddpd zmm7, zmm11, zmm27, zmm0	;; R1+R7*sine						; 14-17		n 26
	zfnmaddpd zmm11, zmm11, zmm27, zmm0	;; R1-R7*sine						; 14-17		n 36
	vmovapd	zmm12, [srcreg+64]		;; I1
	zfmaddpd zmm0, zmm18, zmm27, zmm12	;; I1+I7*sine						; 15-18		n 27
	zfnmaddpd zmm18, zmm18, zmm27, zmm12	;; I1-I7*sine						; 15-18		n 37

	L1prefetchw srcreg+L1pd, L1pt
	vaddpd	zmm12, zmm6, zmm3		;; R4+R10 / sine					; 16-19		n 26
	vsubpd	zmm6, zmm6, zmm3		;; R4-R10 / sine					; 16-19		n 37
	L1prefetchw srcreg+64+L1pd, L1pt
	vaddpd	zmm3, zmm15, zmm21		;; I4+I10 / sine					; 17-20		n 27
	vsubpd	zmm15, zmm15, zmm21		;; I4-I10 / sine					; 17-20		n 36

	L1prefetchw srcreg+d1+L1pd, L1pt
	zfmaddpd zmm21, zmm2, zmm29, zmm24	;; R3+R11*sine						; 18-21		n 28
	zfnmaddpd zmm2, zmm2, zmm29, zmm24	;; R3-R11*sine						; 18-21		n 33
	L1prefetchw srcreg+d1+64+L1pd, L1pt
	vaddpd	zmm24, zmm9, zmm4		;; R5+R9 / sine						; 19-22		n 28
	vsubpd	zmm9, zmm9, zmm4		;; R5-R9 / sine						; 19-22		n 33
	L1prefetchw srcreg+d2+L1pd, L1pt
	zfmaddpd zmm4, zmm22, zmm29, zmm14	;; I3+I11*sine						; 20-23		n 30
	zfnmaddpd zmm22, zmm22, zmm29, zmm14	;; I3-I11*sine						; 20-23		n 32
	L1prefetchw srcreg+d2+64+L1pd, L1pt
	vaddpd	zmm14, zmm16, zmm20		;; I5+I9 / sine						; 21-24		n 30
	vsubpd	zmm16, zmm16, zmm20		;; I5-I9 / sine						; 21-24		n 32
	L1prefetchw srcreg+d2+d1+L1pd, L1pt
	vaddpd	zmm20, zmm8, zmm5		;; R6+R8 / sine						; 22-25		n 29
	vsubpd	zmm8, zmm8, zmm5		;; R6-R8 / sine						; 22-25		n 35
	L1prefetchw srcreg+d2+d1+64+L1pd, L1pt
	zfmaddpd zmm5, zmm1, zmm28, zmm10	;; R2+R12*sine						; 23-26		n 29
	zfnmaddpd zmm1, zmm1, zmm28, zmm10	;; R2-R12*sine						; 23-26		n 35
	L1prefetchw srcreg+d4+L1pd, L1pt
	vaddpd	zmm10, zmm17, zmm19		;; I6+I8 / sine						; 24-27		n 31
	vsubpd	zmm17, zmm17, zmm19		;; I6-I8 / sine						; 24-27		n 34
	L1prefetchw srcreg+d4+64+L1pd, L1pt
	zfmaddpd zmm19, zmm23, zmm28, zmm13	;; I2+I12*sine						; 25-28		n 31
	zfnmaddpd zmm23, zmm23, zmm28, zmm13	;; I2-I12*sine						; 25-28		n 34

no bcast vmovapd zmm29, [screg+2*128]		;; sine for R4/I4 and R10/I10 (w^3)
bcast	vbroadcastsd zmm29, Q [screg+2*16]	;; sine for R4/I4 and R10/I10
	zfmaddpd zmm13, zmm12, zmm29, zmm7	;; r1++ = (r1+r7) + (r4+r10)*sine			; 26-29		n 42
	zfnmaddpd zmm12, zmm12, zmm29, zmm7	;; r1+- = (r1+r7) - (r4+r10)*sine			; 26-29		n 44
	zfmaddpd zmm7, zmm3, zmm29, zmm0	;; i1++ = (i1+i7) + (i4+i10)*sine			; 27-30		n 43
	zfnmaddpd zmm3, zmm3, zmm29, zmm0	;; i1+- = (i1+i7) - (i4+i10)*sine			; 27-30		n 45
no bcast vmovapd zmm28, [screg+3*128]		;; sine for R5/I5 and R9/I9 (w^4)
bcast	vbroadcastsd zmm28, Q [screg+3*16]	;; sine for R5/I5 and R9/I9
	zfmaddpd zmm0, zmm24, zmm28, zmm21	;; r3++ = (r3+r11) + (r5+r9)*sine			; 28-31		n 38
	zfnmaddpd zmm24, zmm24, zmm28, zmm21	;; r3+- = (r3+r11) - (r5+r9)*sine			; 28-31		n 50
no bcast vmovapd zmm27, [screg+4*128]		;; sine for R6/I6 and R8/I8 (w^5)
bcast	vbroadcastsd zmm27, Q [screg+4*16]	;; sine for R6/I6 and R8/I8
	zfmaddpd zmm21, zmm20, zmm27, zmm5	;; r2++ = (r2+r12) + (r6+r8)*sine			; 29-32		n 38
	zfnmaddpd zmm20, zmm20, zmm27, zmm5	;; r2+- = (r2+r12) - (r6+r8)*sine			; 29-32		n 52
	L1prefetchw srcreg+d4+d1+L1pd, L1pt
	zfmaddpd zmm5, zmm14, zmm28, zmm4	;; i3++ = (i3+i11) + (i5+i9)*sine			; 30-33		n 39
	zfnmaddpd zmm14, zmm14, zmm28, zmm4	;; i3+- = (i3+i11) - (i5+i9)*sine			; 30-33		n 51
	L1prefetchw srcreg+d4+d1+64+L1pd, L1pt
	zfmaddpd zmm4, zmm10, zmm27, zmm19	;; i2++ = (i2+i12) + (i6+i8)*sine			; 31-34		n 39
	zfnmaddpd zmm10, zmm10, zmm27, zmm19	;; i2+- = (i2+i12) - (i6+i8)*sine			; 31-34		n 53
	L1prefetchw srcreg+d4+d2+L1pd, L1pt
	zfmaddpd zmm19, zmm16, zmm28, zmm22	;; i3-+ = (i3-i11) + (i5-i9)*sine			; 32-35		n 52
	zfnmaddpd zmm16, zmm16, zmm28, zmm22	;; i3-- = (i3-i11) - (i5-i9)*sine			; 32-35		n 40
	L1prefetchw srcreg+d4+d2+64+L1pd, L1pt
	zfmaddpd zmm22, zmm9, zmm28, zmm2	;; r3-+ = (r3-r11) + (r5-r9)*sine			; 33-36		n 53
	zfnmaddpd zmm9, zmm9, zmm28, zmm2	;; r3-- = (r3-r11) - (r5-r9)*sine			; 33-36		n 41
	L1prefetchw srcreg+d4+d2+d1+L1pd, L1pt
	zfmaddpd zmm2, zmm17, zmm27, zmm23	;; i2-+ = (i2-i12) + (i6-i8)*sine			; 34-37		n 50
	zfnmaddpd zmm17, zmm17, zmm27, zmm23	;; i2-- = (i2-i12) - (i6-i8)*sine			; 34-37		n 40
	L1prefetchw srcreg+d4+d2+d1+64+L1pd, L1pt
	zfmaddpd zmm23, zmm8, zmm27, zmm1	;; r2-+ = (r2-r12) + (r6-r8)*sine			; 35-38		n 51
	zfnmaddpd zmm8, zmm8, zmm27, zmm1	;; r2-- = (r2-r12) - (r6-r8)*sine			; 35-38		n 41
	L1prefetchw srcreg+2*d4+L1pd, L1pt
	zfmaddpd zmm1, zmm15, zmm29, zmm11	;; r1-+ = (r1-r7) + (i4-i10)*sine			; 36-39		n 56
	zfnmaddpd zmm15, zmm15, zmm29, zmm11	;; r1-- = (r1-r7) - (i4-i10)*sine			; 36-39		n 54
	L1prefetchw srcreg+2*d4+64+L1pd, L1pt
	zfmaddpd zmm11, zmm6, zmm29, zmm18	;; i1-+ = (i1-i7) + (r4-r10)*sine			; 37-40		n 55
	zfnmaddpd zmm6, zmm6, zmm29, zmm18	;; i1-- = (i1-i7) - (r4-r10)*sine			; 37-40		n 57

	L1prefetchw srcreg+2*d4+d1+L1pd, L1pt
	vaddpd	zmm18, zmm0, zmm21		;; r3+++ = (r3++) + (r2++)				; 38-41		n 42
	vsubpd	zmm0, zmm0, zmm21		;; r3++- = (r3++) - (r2++)				; 38-41		n 44
	L1prefetchw srcreg+2*d4+d1+64+L1pd, L1pt
	vaddpd	zmm21, zmm5, zmm4		;; i3+++ = (i3++) + (i2++)				; 39-42		n 43
	vsubpd	zmm5, zmm5, zmm4		;; i3++- = (i3++) - (i2++)				; 39-42		n 45
	L1prefetchw srcreg+2*d4+d2+L1pd, L1pt
	vaddpd	zmm4, zmm16, zmm17		;; i3--+ = (i3--) + (i2--)				; 40-43		n 48
	vsubpd	zmm16, zmm16, zmm17		;; i3--- = (i3--) - (i2--)				; 40-43		n 46
	L1prefetchw srcreg+2*d4+d2+64+L1pd, L1pt
	vaddpd	zmm17, zmm9, zmm8		;; r3--+ = (r3--) + (r2--)				; 41-44		n 49
	vsubpd	zmm9, zmm9, zmm8		;; r3--- = (r3--) - (r2--)				; 41-44		n 47

	L1prefetchw srcreg+2*d4+d2+d1+L1pd, L1pt
	vaddpd	zmm8, zmm13, zmm18		;; R1 = (r1++) + (r3+++)				; 42-45
	zfnmaddpd zmm18, zmm18, zmm31, zmm13	;; R59= (r1++) - .500(r3+++)				; 42-45		n 46
	L1prefetchw srcreg+2*d4+d2+d1+64+L1pd, L1pt
	vaddpd	zmm13, zmm7, zmm21		;; I1 = (i1++) + (i3+++)				; 43-46
	zfnmaddpd zmm21, zmm21, zmm31, zmm7	;; I59= (i1++) - .500(i3+++)				; 43-46		n 47
	vaddpd	zmm7, zmm12, zmm0		;; R7 = (r1+-) + (r3++-)				; 44-47
	zfnmaddpd zmm0, zmm0, zmm31, zmm12	;; R3B= (r1+-) - .500(r3++-)				; 44-47		n 48
	vaddpd	zmm12, zmm3, zmm5		;; I7 = (i1+-) + (i3++-)				; 45-48
	zfnmaddpd zmm5, zmm5, zmm31, zmm3	;; I3B= (i1+-) - .500(i3++-)				; 45-48		n 49
	bump	screg, scinc

	zfnmaddpd zmm3, zmm16, zmm30, zmm18	;; R5 = R59 - .866(i3---)				; 46-49
	zfmaddpd zmm16, zmm16, zmm30, zmm18	;; R9 = R59 + .866(i3---)				; 46-49
  	zstore	[srcreg], zmm8			;; Save R1						; 46
	zfmaddpd zmm18, zmm9, zmm30, zmm21	;; I5 = I59 + .866(r3---)				; 47-50
	zfnmaddpd zmm9, zmm9, zmm30, zmm21	;; I9 = I59 - .866(r3---)				; 47-50
	zstore	[srcreg+64], zmm13		;; Save I1						; 47
	zfmaddpd zmm21, zmm4, zmm30, zmm0	;; R3 = R3B + .866(i3--+)				; 48-51
	zfnmaddpd zmm4, zmm4, zmm30, zmm0	;; R11= R3B - .866(i3--+)				; 48-51
	zstore	[srcreg+d4+d2], zmm7		;; Save R7						; 48
	zfnmaddpd zmm0, zmm17, zmm30, zmm5	;; I3 = I3B - .866(r3--+)				; 49-52
	zfmaddpd zmm17, zmm17, zmm30, zmm5	;; I11= I3B + .866(r3--+)				; 49-52
	zstore	[srcreg+d4+d2+64], zmm12	;; Save I7						; 49

	vaddpd	zmm5, zmm24, zmm2		;; r3+-+ = (r3+-) + (i2-+)				; 50-53		n 56
	vsubpd	zmm24, zmm24, zmm2		;; r3+-- = (r3+-) - (i2-+)				; 50-53		n 54
	zstore	[srcreg+d4], zmm3		;; Save R5						; 50
	vaddpd	zmm2, zmm14, zmm23		;; i3+-+ = (i3+-) + (r2-+)				; 51-54		n 55
	vsubpd	zmm14, zmm14, zmm23		;; i3+-- = (i3+-) - (r2-+)				; 51-54		n 57
	zstore	[srcreg+2*d4], zmm16		;; Save R9						; 50+1
	vaddpd	zmm23, zmm19, zmm20		;; i3-++ = (i3-+) + (r2+-)				; 52-55		n 60
	vsubpd	zmm19, zmm19, zmm20		;; i3-+- = (i3-+) - (r2+-)				; 52-55		n 58
	zstore	[srcreg+d4+64], zmm18		;; Save I5						; 51+1
	vaddpd	zmm20, zmm22, zmm10		;; r3-++ = (r3-+) + (i2+-)				; 53-56		n 59
	vsubpd	zmm22, zmm22, zmm10		;; r3-+- = (r3-+) - (i2+-)				; 53-56		n 61
	zstore	[srcreg+2*d4+64], zmm9		;; Save I9						; 51+2

	vsubpd	zmm10, zmm15, zmm24		;; R4 = (r1--) - (r3+--)				; 54-57
	zfmaddpd zmm24, zmm24, zmm31, zmm15	;; R8C= (r1--) + .500(r3+--)				; 54-57		n 58
	zstore	[srcreg+d2], zmm21		;; Save R3						; 52+2
	vsubpd	zmm15, zmm11, zmm2		;; I4 = (i1-+) - (i3+-+)				; 55-58
	zfmaddpd zmm2, zmm2, zmm31, zmm11	;; I8C= (i1-+) + .500(i3+-+)				; 55-58		n 59
	zstore	[srcreg+2*d4+d2], zmm4		;; Save R11						; 52+3
	vsubpd	zmm11, zmm1, zmm5		;; R10= (r1-+) - (r3+-+)				; 56-59
	zfmaddpd zmm5, zmm5, zmm31, zmm1	;; R26= (r1-+) + .500(r3+-+)				; 56-59		n 60
	zstore	[srcreg+d2+64], zmm0		;; Save I3						; 53+3
	vsubpd	zmm1, zmm6, zmm14		;; I10= (i1--) - (i3+--)				; 57-60
	zfmaddpd zmm14, zmm14, zmm31, zmm6	;; I26= (i1--) + .500(i3+--)				; 57-60		n 61
	zstore	[srcreg+2*d4+d2+64], zmm17	;; Save I11						; 53+4

	zfmaddpd zmm6, zmm19, zmm30, zmm24	;; R8 = R8C + .866(i3-+-)				; 58-61
	zfnmaddpd zmm19, zmm19, zmm30, zmm24	;; R12= R8C - .866(i3-+-)				; 58-61
	zstore	[srcreg+d2+d1], zmm10		;; Save R4						; 58
	zfnmaddpd zmm24, zmm20, zmm30, zmm2	;; I8 = I8C - .866(r3-++)				; 59-62
	zfmaddpd zmm20, zmm20, zmm30, zmm2	;; I12= I8C + .866(r3-++)				; 59-62
	zstore	[srcreg+d2+d1+64], zmm15	;; Save I4						; 59
	zfmaddpd zmm2, zmm23, zmm30, zmm5	;; R2 = R26 + .866(i3-++)				; 60-63
	zfnmaddpd zmm23, zmm23, zmm30, zmm5	;; R6 = R26 - .866(i3-++)				; 60-63
	zstore	[srcreg+2*d4+d1], zmm11		;; Save R10						; 60
	zfnmaddpd zmm5, zmm22, zmm30, zmm14	;; I2 = I26 - .866(r3-+-)				; 61-64
	zfmaddpd zmm22, zmm22, zmm30, zmm14	;; I6 = I26 + .866(r3-+-)				; 61-64
	zstore	[srcreg+2*d4+d1+64], zmm1	;; Save I10						; 61

	zstore	[srcreg+d4+d2+d1], zmm6		;; Save R8						; 62
	zstore	[srcreg+2*d4+d2+d1], zmm19	;; Save R12						; 62+1
	zstore	[srcreg+d4+d2+d1+64], zmm24	;; Save I8						; 63+1
	zstore	[srcreg+2*d4+d2+d1+64], zmm20	;; Save I12						; 63+2
	zstore	[srcreg+d1], zmm2		;; Save R2						; 64+2
	zstore	[srcreg+d4+d1], zmm23		;; Save R6						; 64+3
	zstore	[srcreg+d1+64], zmm5		;; Save I2						; 65+3
	zstore	[srcreg+d4+d1+64], zmm22	;; Save I6						; 65+4
	bump	srcreg, srcinc
	ENDM


;; This code applies the sin/cos multipliers before a radix-12 butterfly.
;; Then it applies the postmultipliers since the negacyclic inverse FFT is complete
;; as well as the group weight multipliers.
;; We save memory by splitting the roots-of-minus-1 premultipliers such that every
;; macro uses the same premultiplier data.  The rest of the premultiplier is applied
;; at the end of pass 1.

;; The sin/cos and premultiplier data is combined in one table.  The premultiplier data
;; is at screg the sin/cos data is at screg+12*64.  Note that the sine value for the
;; premultiplier has been combined with the group multiplier.

zr12_csc_wpn_twelve_complex_last_djbunfft_preload MACRO
	vbroadcastsd zmm31, ZMM_HALF
	vbroadcastsd zmm30, ZMM_P866
	vbroadcastsd zmm25, ZMM_B
	ENDM
zr12_csc_wpn_twelve_complex_last_djbunfft MACRO srcreg,srcinc,d1,d2,d4,screg,scinc,maskreg,maskinc,maxrpt,L1pt,L1pd
	vmovapd	zmm2, [srcreg+d2]		;; R3
	vmovapd	zmm14, [srcreg+d2+64]		;; I3
	vmovapd zmm29, [screg+12*64+1*128+64]	;; cosine/sine for R3/I3 and R11/I11 (w^2)
	zfmaddpd zmm24, zmm2, zmm29, zmm14	;; A3 = R3 * cosine/sine + I3				; 1-4		n 12
	zfmsubpd zmm14, zmm14, zmm29, zmm2	;; B3 = I3 * cosine/sine - R3				; 1-4		n 12
	vmovapd	zmm10, [srcreg+2*d4+d2]		;; R11
	vmovapd	zmm22, [srcreg+2*d4+d2+64]	;; I11
	zfmsubpd zmm2, zmm10, zmm29, zmm22	;; A11 = R11 * cosine/sine - I11			; 2-5		n 18
	zfmaddpd zmm22, zmm22, zmm29, zmm10	;; B11 = I11 * cosine/sine + R11			; 2-5		n 20

	vmovapd	zmm1, [srcreg+d1]		;; R2
	vmovapd	zmm13, [srcreg+d1+64]		;; I2
	vmovapd zmm29, [screg+12*64+0*128+64]	;; cosine/sine for R2/I2 and R12/I12 (w^1)
	zfmaddpd zmm10, zmm1, zmm29, zmm13	;; A2 = R2 * cosine/sine + I2				; 3-6		n 13
	zfmsubpd zmm13, zmm13, zmm29, zmm1	;; B2 = I2 * cosine/sine - R2				; 3-6		n 13
	vmovapd	zmm11, [srcreg+2*d4+d2+d1]	;; R12
	vmovapd	zmm23, [srcreg+2*d4+d2+d1+64]	;; I12
	zfmsubpd zmm1, zmm11, zmm29, zmm23	;; A12 = R12 * cosine/sine - I12			; 4-7		n 23
	zfmaddpd zmm23, zmm23, zmm29, zmm11	;; B12 = I12 * cosine/sine + R12			; 4-7		n 25

	vmovapd	zmm6, [srcreg+d4+d2]		;; R7
	vmovapd	zmm18, [srcreg+d4+d2+64]	;; I7
	vmovapd zmm29, [screg+12*64+5*128+64]	;; cosine/sine for R7/I7 (w^6)
	zfmaddpd zmm11, zmm6, zmm29, zmm18	;; A7 = R7 * cosine/sine + I7 (new R7/sine)		; 5-8		n 14
	zfmsubpd zmm18, zmm18, zmm29, zmm6	;; B7 = I7 * cosine/sine - R7 (new I7/sine)		; 5-8		n 15

	vmovapd	zmm3, [srcreg+d2+d1]		;; R4
	vmovapd	zmm15, [srcreg+d2+d1+64]	;; I4
	vmovapd zmm29, [screg+12*64+2*128+64]	;; cosine/sine for R4/I4 and R10/I10 (w^3)
	zfmaddpd zmm6, zmm3, zmm29, zmm15	;; A4 = R4 * cosine/sine + I4 (new R4/sine)		; 6-9		n 16
	zfmsubpd zmm15, zmm15, zmm29, zmm3	;; B4 = I4 * cosine/sine - R4 (new I4/sine)		; 6-9		n 17
	vmovapd	zmm9, [srcreg+2*d4+d1]		;; R10
	vmovapd	zmm21, [srcreg+2*d4+d1+64]	;; I10
	zfmsubpd zmm3, zmm9, zmm29, zmm21	;; A10 = R10 * cosine/sine - I10 (new R10/sine)		; 7-10		n 16
	zfmaddpd zmm21, zmm21, zmm29, zmm9	;; B10 = I10 * cosine/sine + R10 (new I10/sine)		; 7-10		n 17

	vmovapd	zmm4, [srcreg+d4]		;; R5
	vmovapd	zmm16, [srcreg+d4+64]		;; I5
	vmovapd zmm29, [screg+12*64+3*128+64]	;; cosine/sine for R5/I5 and R9/I9 (w^4)
	zfmaddpd zmm9, zmm4, zmm29, zmm16	;; A5 = R5 * cosine/sine + I5 (new R5/sine)		; 8-11		n 19
	zfmsubpd zmm16, zmm16, zmm29, zmm4	;; B5 = I5 * cosine/sine - R5 (new I5/sine)		; 8-11		n 21
	vmovapd	zmm8, [srcreg+2*d4]		;; R9
	vmovapd	zmm20, [srcreg+2*d4+64]		;; I9
	zfmsubpd zmm4, zmm8, zmm29, zmm20	;; A9 = R9 * cosine/sine - I9 (new R9/sine)		; 9-12		n 19
	zfmaddpd zmm20, zmm20, zmm29, zmm8	;; B9 = I9 * cosine/sine + R9 (new I9/sine)		; 9-12		n 21

	vmovapd	zmm5, [srcreg+d4+d1]		;; R6
	vmovapd	zmm17, [srcreg+d4+d1+64]	;; I6
	vmovapd zmm29, [screg+12*64+4*128+64]	;; cosine/sine for R6/I6 and R8/I8 (w^5)
	zfmaddpd zmm8, zmm5, zmm29, zmm17	;; A6 = R6 * cosine/sine + I6 (new R6/sine)		; 10-13		n 22
	zfmsubpd zmm17, zmm17, zmm29, zmm5	;; B6 = I6 * cosine/sine - R6 (new I6/sine)		; 10-13		n 24
	vmovapd	zmm7, [srcreg+d4+d2+d1]		;; R8
	vmovapd	zmm19, [srcreg+d4+d2+d1+64]	;; I8
	zfmsubpd zmm5, zmm7, zmm29, zmm19	;; A8 = R8 * cosine/sine - I8 (new R8/sine)		; 11-14		n 22
	zfmaddpd zmm19, zmm19, zmm29, zmm7	;; B8 = I8 * cosine/sine + R8 (new I8/sine)		; 11-14		n 24

	vmovapd zmm29, [screg+12*64+1*128]	;; sine for R3/I3 and R11/I11 (w^2)
	vmulpd	zmm24, zmm24, zmm29		;; A3 = A3 * sine (new R3)				; 12-15		n 18
	vmulpd	zmm14, zmm14, zmm29		;; B3 = B3 * sine (new I3)				; 12-15		n 20

	vmovapd zmm28, [screg+12*64+0*128]	;; sine for R2/I2 and R12/I12 (w^1)
	vmulpd	zmm10, zmm10, zmm28		;; A2 = A2 * sine (new R2)				; 13-16		n 23
	vmulpd	zmm13, zmm13, zmm28		;; B2 = B2 * sine (new I2)				; 13-16		n 25

	vmovapd	zmm0, [srcreg]			;; R1
	vmovapd zmm27, [screg+12*64+5*128]	;; sine for R7/I7 (w^6)
	zfmaddpd zmm7, zmm11, zmm27, zmm0	;; R1+R7*sine						; 14-17		n 26
	zfnmaddpd zmm11, zmm11, zmm27, zmm0	;; R1-R7*sine						; 14-17		n 36

	vmovapd	zmm12, [srcreg+64]		;; I1
	zfmaddpd zmm0, zmm18, zmm27, zmm12	;; I1+I7*sine						; 15-18		n 27
	zfnmaddpd zmm18, zmm18, zmm27, zmm12	;; I1-I7*sine						; 15-18		n 37

	vaddpd	zmm12, zmm6, zmm3		;; R4+R10 / sine					; 16-19		n 26
	vsubpd	zmm6, zmm6, zmm3		;; R4-R10 / sine					; 16-19		n 37
	vmovapd zmm26, [screg+12*64+2*128]	;; sine for R4/I4 and R10/I10 (w^3)

	vaddpd	zmm3, zmm15, zmm21		;; I4+I10 / sine					; 17-20		n 27
	vsubpd	zmm15, zmm15, zmm21		;; I4-I10 / sine					; 17-20		n 36
	mov	bl, [maskreg+0*1]		;; Load index into compressed fudges

	zfmaddpd zmm21, zmm2, zmm29, zmm24	;; R3+R11*sine						; 18-21		n 28
	zfnmaddpd zmm2, zmm2, zmm29, zmm24	;; R3-R11*sine						; 18-21		n 33
	mov	r14, [r13+0*8]			;; Load the xor mask

	vaddpd	zmm24, zmm9, zmm4		;; R5+R9 / sine						; 19-22		n 28
	vsubpd	zmm9, zmm9, zmm4		;; R5-R9 / sine						; 19-22		n 33
	xor	r14, [r12+rbx*8]		;; Apply the xor mask to 64 bits of fudge flags

	zfmaddpd zmm4, zmm22, zmm29, zmm14	;; I3+I11*sine						; 20-23		n 30
	zfnmaddpd zmm22, zmm22, zmm29, zmm14	;; I3-I11*sine						; 20-23		n 32
	vmovapd zmm29, [screg+12*64+3*128]	;; sine for R5/I5 and R9/I9 (w^4)

	vaddpd	zmm14, zmm16, zmm20		;; I5+I9 / sine						; 21-24		n 30
	vsubpd	zmm16, zmm16, zmm20		;; I5-I9 / sine						; 21-24		n 32
	vmovapd zmm27, [screg+12*64+4*128]	;; sine for R6/I6 and R8/I8 (w^5)

	vaddpd	zmm20, zmm8, zmm5		;; R6+R8 / sine						; 22-25		n 29
	vsubpd	zmm8, zmm8, zmm5		;; R6-R8 / sine						; 22-25		n 35
	mov	bl, [maskreg+1*1]		;; Load index into compressed fudges

	zfmaddpd zmm5, zmm1, zmm28, zmm10	;; R2+R12*sine						; 23-26		n 29
	zfnmaddpd zmm1, zmm1, zmm28, zmm10	;; R2-R12*sine						; 23-26		n 35
	L1prefetchw srcreg+L1pd, L1pt

	vaddpd	zmm10, zmm17, zmm19		;; I6+I8 / sine						; 24-27		n 31
	vsubpd	zmm17, zmm17, zmm19		;; I6-I8 / sine						; 24-27		n 34
	L1prefetchw srcreg+64+L1pd, L1pt

	zfmaddpd zmm19, zmm23, zmm28, zmm13	;; I2+I12*sine						; 25-28		n 31
	zfnmaddpd zmm23, zmm23, zmm28, zmm13	;; I2-I12*sine						; 25-28		n 34
	L1prefetchw srcreg+d1+L1pd, L1pt

	zfmaddpd zmm13, zmm12, zmm26, zmm7	;; r1++ = (r1+r7) + (r4+r10)*sine			; 26-29		n 42
	zfnmaddpd zmm12, zmm12, zmm26, zmm7	;; r1+- = (r1+r7) - (r4+r10)*sine			; 26-29		n 44
	L1prefetchw srcreg+d1+64+L1pd, L1pt

	zfmaddpd zmm7, zmm3, zmm26, zmm0	;; i1++ = (i1+i7) + (i4+i10)*sine			; 27-30		n 43
	zfnmaddpd zmm3, zmm3, zmm26, zmm0	;; i1+- = (i1+i7) - (i4+i10)*sine			; 27-30		n 45
	L1prefetchw srcreg+d2+L1pd, L1pt

	zfmaddpd zmm0, zmm24, zmm29, zmm21	;; r3++ = (r3+r11) + (r5+r9)*sine			; 28-31		n 38
	zfnmaddpd zmm24, zmm24, zmm29, zmm21	;; r3+- = (r3+r11) - (r5+r9)*sine			; 28-31		n 50
	L1prefetchw srcreg+d2+64+L1pd, L1pt

	zfmaddpd zmm21, zmm20, zmm27, zmm5	;; r2++ = (r2+r12) + (r6+r8)*sine			; 29-32		n 38
	zfnmaddpd zmm20, zmm20, zmm27, zmm5	;; r2+- = (r2+r12) - (r6+r8)*sine			; 29-32		n 
	L1prefetchw srcreg+d2+d1+L1pd, L1pt

	zfmaddpd zmm5, zmm14, zmm29, zmm4	;; i3++ = (i3+i11) + (i5+i9)*sine			; 30-33		n 39
	zfnmaddpd zmm14, zmm14, zmm29, zmm4	;; i3+- = (i3+i11) - (i5+i9)*sine			; 30-33		n 
	L1prefetchw srcreg+d2+d1+64+L1pd, L1pt

	zfmaddpd zmm4, zmm10, zmm27, zmm19	;; i2++ = (i2+i12) + (i6+i8)*sine			; 31-34		n 39
	zfnmaddpd zmm10, zmm10, zmm27, zmm19	;; i2+- = (i2+i12) - (i6+i8)*sine			; 31-34		n 
	L1prefetchw srcreg+d4+L1pd, L1pt

	zfmaddpd zmm19, zmm16, zmm29, zmm22	;; i3-+ = (i3-i11) + (i5-i9)*sine			; 32-35		n 
	zfnmaddpd zmm16, zmm16, zmm29, zmm22	;; i3-- = (i3-i11) - (i5-i9)*sine			; 32-35		n 40
	L1prefetchw srcreg+d4+64+L1pd, L1pt

	zfmaddpd zmm22, zmm9, zmm29, zmm2	;; r3-+ = (r3-r11) + (r5-r9)*sine			; 33-36		n 
	zfnmaddpd zmm9, zmm9, zmm29, zmm2	;; r3-- = (r3-r11) - (r5-r9)*sine			; 33-36		n 41
	L1prefetchw srcreg+d4+d1+L1pd, L1pt

	zfmaddpd zmm2, zmm17, zmm27, zmm23	;; i2-+ = (i2-i12) + (i6-i8)*sine			; 34-37		n 50
	zfnmaddpd zmm17, zmm17, zmm27, zmm23	;; i2-- = (i2-i12) - (i6-i8)*sine			; 34-37		n 40
	L1prefetchw srcreg+d4+d1+64+L1pd, L1pt

	zfmaddpd zmm23, zmm8, zmm27, zmm1	;; r2-+ = (r2-r12) + (r6-r8)*sine			; 35-38		n 51
	zfnmaddpd zmm8, zmm8, zmm27, zmm1	;; r2-- = (r2-r12) - (r6-r8)*sine			; 35-38		n 41
	L1prefetchw srcreg+d4+d2+L1pd, L1pt

	zfmaddpd zmm1, zmm15, zmm26, zmm11	;; r1-+ = (r1-r7) + (i4-i10)*sine			; 36-39		n 
	zfnmaddpd zmm15, zmm15, zmm26, zmm11	;; r1-- = (r1-r7) - (i4-i10)*sine			; 36-39		n 
	L1prefetchw srcreg+d4+d2+64+L1pd, L1pt

	zfmaddpd zmm11, zmm6, zmm26, zmm18	;; i1-+ = (i1-i7) + (r4-r10)*sine			; 37-40		n 
	zfnmaddpd zmm6, zmm6, zmm26, zmm18	;; i1-- = (i1-i7) - (r4-r10)*sine			; 37-40		n 
	vmovapd zmm29, [screg+0*64]		;; premultiplier cosine/sine for R1/I1

	vaddpd	zmm18, zmm0, zmm21		;; r3+++ = (r3++) + (r2++)				; 38-41		n 42
	vsubpd	zmm0, zmm0, zmm21		;; r3++- = (r3++) - (r2++)				; 38-41		n 44
	vmovapd zmm28, [screg+6*64]		;; premultiplier cosine/sine for R7/I7

	vaddpd	zmm21, zmm5, zmm4		;; i3+++ = (i3++) + (i2++)				; 39-42		n 43
	vsubpd	zmm5, zmm5, zmm4		;; i3++- = (i3++) - (i2++)				; 39-42		n 45
	vmovapd zmm27, [screg+4*64]		;; premultiplier cosine/sine for R5/I5

	vaddpd	zmm4, zmm16, zmm17		;; i3--+ = (i3--) + (i2--)				; 40-43		n 48
	vsubpd	zmm16, zmm16, zmm17		;; i3--- = (i3--) - (i2--)				; 40-43		n 46
	vmovapd zmm26, [screg+8*64]		;; premultiplier cosine/sine for R9/I9

	vaddpd	zmm17, zmm9, zmm8		;; r3--+ = (r3--) + (r2--)				; 41-44		n 49
	vsubpd	zmm9, zmm9, zmm8		;; r3--- = (r3--) - (r2--)				; 41-44		n 47
	L1prefetchw srcreg+d4+d2+d1+L1pd, L1pt

	vaddpd	zmm8, zmm13, zmm18		;; R1 = (r1++) + (r3+++)				; 42-45		n 65
	zfnmaddpd zmm18, zmm18, zmm31, zmm13	;; R59= (r1++) - .500(r3+++)				; 42-45		n 46
	L1prefetchw srcreg+d4+d2+d1+64+L1pd, L1pt

	vaddpd	zmm13, zmm7, zmm21		;; I1 = (i1++) + (i3+++)				; 43-46		n 65
	zfnmaddpd zmm21, zmm21, zmm31, zmm7	;; I59= (i1++) - .500(i3+++)				; 43-46		n 47
	L1prefetchw srcreg+2*d4+L1pd, L1pt

	vaddpd	zmm7, zmm12, zmm0		;; R7 = (r1+-) + (r3++-)				; 44-47		n 66
	zfnmaddpd zmm0, zmm0, zmm31, zmm12	;; R3B= (r1+-) - .500(r3++-)				; 44-47		n 48
	L1prefetchw srcreg+2*d4+64+L1pd, L1pt

	vaddpd	zmm12, zmm3, zmm5		;; I7 = (i1+-) + (i3++-)				; 45-48		n 66
	zfnmaddpd zmm5, zmm5, zmm31, zmm3	;; I3B= (i1+-) - .500(i3++-)				; 45-48		n 49
	L1prefetchw srcreg+2*d4+d1+L1pd, L1pt

	zfnmaddpd zmm3, zmm16, zmm30, zmm18	;; R5 = R59 - .866(i3---)				; 46-49		n 67
	zfmaddpd zmm16, zmm16, zmm30, zmm18	;; R9 = R59 + .866(i3---)				; 46-49		n 68
	L1prefetchw srcreg+2*d4+d1+64+L1pd, L1pt

	zfmaddpd zmm18, zmm9, zmm30, zmm21	;; I5 = I59 + .866(r3---)				; 47-50		n 67
	zfnmaddpd zmm9, zmm9, zmm30, zmm21	;; I9 = I59 - .866(r3---)				; 47-50		n 68
	L1prefetchw srcreg+2*d4+d2+L1pd, L1pt

	zfmaddpd zmm21, zmm4, zmm30, zmm0	;; R3 = R3B + .866(i3--+)				; 48-51		n 69
	zfnmaddpd zmm4, zmm4, zmm30, zmm0	;; R11= R3B - .866(i3--+)				; 48-51		n 70
	L1prefetchw srcreg+2*d4+d2+64+L1pd, L1pt

	zfnmaddpd zmm0, zmm17, zmm30, zmm5	;; I3 = I3B - .866(r3--+)				; 49-52		n 69
	zfmaddpd zmm17, zmm17, zmm30, zmm5	;; I11= I3B + .866(r3--+)				; 49-52		n 70
	L1prefetchw srcreg+2*d4+d2+d1+L1pd, L1pt

	vaddpd	zmm5, zmm24, zmm2		;; r3+-+ = (r3+-) + (i2-+)				; 50-53		n 59
	vsubpd	zmm24, zmm24, zmm2		;; r3+-- = (r3+-) - (i2-+)				; 50-53		n 56
	L1prefetchw srcreg+2*d4+d2+d1+64+L1pd, L1pt

	kmovw	k1, r14d			;; Load R1 and I1 fudge factor mask			; 51		n 77
	vaddpd	zmm2, zmm14, zmm23		;; i3+-+ = (i3+-) + (r2-+)				; 51-54		n 57
	shr	r14, 16				;; Next 16 bits of fudge flags

	kmovw	k2, r14d			;; Load R2 and I2 fudge factor mask			; 52		n 94
	vsubpd	zmm14, zmm14, zmm23		;; i3+-- = (i3+-) - (r2-+)				; 52-55		n 60
	shr	r14, 16				;; Next 16 bits of fudge flags

	kmovw	k3, r14d			;; Load R3 and I3 fudge factor mask			; 53		n 84
	vaddpd	zmm23, zmm19, zmm20		;; i3-++ = (i3-+) + (r2+-)				; 53-56		n 63
	shr	r14, 16				;; Next 16 bits of fudge flags

	kmovw	k4, r14d			;; Load R4 and I4 fudge factor mask			; 54		n 88
	vsubpd	zmm19, zmm19, zmm20		;; i3-+- = (i3-+) - (r2+-)				; 54-57		n 61
	mov	r14, [r13+1*8]			;; Load the xor mask

	vaddpd	zmm20, zmm22, zmm10		;; r3-++ = (r3-+) + (i2+-)				; 55-58		n 62
	vsubpd	zmm22, zmm22, zmm10		;; r3-+- = (r3-+) - (i2+-)				; 55-58		n 64
	xor	r14, [r12+rbx*8]		;; Apply the xor mask to 64 bits of fudge flags

	vsubpd	zmm10, zmm15, zmm24		;; R4 = (r1--) - (r3+--)				; 56-59		n 71
	zfmaddpd zmm24, zmm24, zmm31, zmm15	;; R8C= (r1--) + .500(r3+--)				; 56-59		n 61
	mov	bl, [maskreg+2*1]		;; Load index into compressed fudges
	bump	maskreg, maskinc

	kmovw	k5, r14d			;; Load R5 and I5 fudge factor mask			; 57		n 80
	vsubpd	zmm15, zmm11, zmm2		;; I4 = (i1-+) - (i3+-+)				; 57-60		n 71
	shr	r14, 16				;; Next 16 bits of fudge flags

	kmovw	k6, r14d			;; Load R6 and I6 fudge factor mask			; 58		n 95
	zfmaddpd zmm2, zmm2, zmm31, zmm11	;; I8C= (i1-+) + .500(i3+-+)				; 58-61		n 62
	shr	r14, 16				;; Next 16 bits of fudge flags

	vsubpd	zmm11, zmm1, zmm5		;; R10= (r1-+) - (r3+-+)				; 59-62		n 72
	zfmaddpd zmm5, zmm5, zmm31, zmm1	;; R26= (r1-+) + .500(r3+-+)				; 59-62		n 63

	vsubpd	zmm1, zmm6, zmm14		;; I10= (i1--) - (i3+--)				; 60-63		n 72
	zfmaddpd zmm14, zmm14, zmm31, zmm6	;; I26= (i1--) + .500(i3+--)				; 60-63		n 64

	zfmaddpd zmm6, zmm19, zmm30, zmm24	;; R8 = R8C + .866(i3-+-)				; 61-64		n 73
	zfnmaddpd zmm19, zmm19, zmm30, zmm24	;; R12= R8C - .866(i3-+-)				; 61-64		n 74

	zfnmaddpd zmm24, zmm20, zmm30, zmm2	;; I8 = I8C - .866(r3-++)				; 62-65		n 73
	zfmaddpd zmm20, zmm20, zmm30, zmm2	;; I12= I8C + .866(r3-++)				; 62-65		n 74

	zfmaddpd zmm2, zmm23, zmm30, zmm5	;; R2 = R26 + .866(i3-++)				; 63-66		n 75
	zfnmaddpd zmm23, zmm23, zmm30, zmm5	;; R6 = R26 - .866(i3-++)				; 63-66		n 76

	zfnmaddpd zmm5, zmm22, zmm30, zmm14	;; I2 = I26 - .866(r3-+-)				; 64-67		n 75
	zfmaddpd zmm22, zmm22, zmm30, zmm14	;; I6 = I26 + .866(r3-+-)				; 64-67		n 76

	kmovw	k7, r14d			;; Load R7 and I7 fudge factor mask			; 65		n 78
	zfmaddpd zmm14, zmm8, zmm29, zmm13	;; A1 = R1 * cosine + I1				; 65-68		n 77
	shr	r14, 16				;; Next 16 bits of fudge flags

	zfmsubpd zmm13, zmm13, zmm29, zmm8	;; B1 = I1 * cosine - R1				; 66-69		n 79
	zfmaddpd zmm8, zmm7, zmm28, zmm12	;; A7 = R7 * cosine + I7				; 66-69		n 78
	vmovapd zmm29, [screg+2*64]		;; premultiplier cosine/sine for R3/I3

	zfmsubpd zmm12, zmm12, zmm28, zmm7	;; B7 = I7 * cosine - R7				; 67-70		n 81
	zfmaddpd zmm7, zmm3, zmm27, zmm18	;; A5 = R5 * cosine + I5				; 67-70		n 80
	vmovapd zmm28, [screg+10*64]		;; premultiplier cosine/sine for R11/I11

	zfmsubpd zmm18, zmm18, zmm27, zmm3	;; B5 = I5 * cosine - R5				; 68-71		n 82
	zfmaddpd zmm3, zmm16, zmm26, zmm9	;; A9 = R9 * cosine + I9				; 68-71		n 83
	vmovapd zmm27, [screg+3*64]		;; premultiplier cosine/sine for R4/I4

	zfmsubpd zmm9, zmm9, zmm26, zmm16	;; B9 = I9 * cosine - R9				; 69-72		n 85
	zfmaddpd zmm16, zmm21, zmm29, zmm0	;; A3 = R3 * cosine + I3				; 69-72		n 84
	vmovapd zmm26, [screg+9*64]		;; premultiplier cosine/sine for R10/I10

	zfmsubpd zmm0, zmm0, zmm29, zmm21	;; B3 = I3 * cosine - R3				; 70-73		n 86
	zfmaddpd zmm21, zmm4, zmm28, zmm17	;; A11 = R11 * cosine + I11				; 70-73		n 87
	vmovapd zmm29, [screg+7*64]		;; premultiplier cosine/sine for R8/I8

	zfmsubpd zmm17, zmm17, zmm28, zmm4	;; B11 = I11 * cosine - R11				; 71-74		n 89
	zfmaddpd zmm4, zmm10, zmm27, zmm15	;; A4 = R4 * cosine + I4				; 71-74		n 88
	vmovapd zmm28, [screg+11*64]		;; premultiplier cosine/sine for R12/I12

	zfmsubpd zmm15, zmm15, zmm27, zmm10	;; B4 = I4 * cosine - R4				; 72-75		n 90
	zfmaddpd zmm10, zmm11, zmm26, zmm1	;; A10 = R10 * cosine + I10				; 72-75		n 90
	vmovapd zmm27, [screg+1*64]		;; premultiplier cosine/sine for R2/I2

	zfmsubpd zmm1, zmm1, zmm26, zmm11	;; B10 = I10 * cosine - R10				; 73-76		n 92
	zfmaddpd zmm11, zmm6, zmm29, zmm24	;; A8 = R8 * cosine + I8				; 73-76		n 91
	vmovapd zmm26, [screg+5*64]		;; premultiplier cosine/sine for R6/I6

	zfmsubpd zmm24, zmm24, zmm29, zmm6	;; B8 = I8 * cosine - R8				; 74-77		n 93
	zfmaddpd zmm6, zmm19, zmm28, zmm20	;; A12 = R12 * cosine + I12				; 74-77		n 93
	bump	screg, scinc

	zfmsubpd zmm20, zmm20, zmm28, zmm19	;; B12 = I12 * cosine - R12				; 75-78		n 96
	zfmaddpd zmm19, zmm2, zmm27, zmm5	;; A2 = R2 * cosine + I2				; 75-78		n 94

	zfmsubpd zmm5, zmm5, zmm27, zmm2	;; B2 = I2 * cosine - R2				; 76-79		n 97
	zfmaddpd zmm2, zmm23, zmm26, zmm22	;; A6 = R6 * cosine + I6				; 76-79		n 95

	vmulpd	zmm14{k1}, zmm14, zmm25		;; apply fudge multiplier for R1			; 77-80
	zfmsubpd zmm22, zmm22, zmm26, zmm23	;; B6 = I6 * cosine - R6				; 77-80		n 97

	kshiftrw k1, k1, 8			;; I1's fudge						; 78		n 79
	vmulpd	zmm8{k7}, zmm8, zmm25		;; apply fudge multiplier for R7			; 78-81

	kshiftrw k7, k7, 8			;; I7's fudge						; 79		n 81
	vmulpd	zmm13{k1}, zmm13, zmm25		;; apply fudge multiplier for I1			; 79-82

	kmovw	k1, r14d			;; Load R8 and I8 fudge factor mask			; 80		n 91
	vmulpd	zmm7{k5}, zmm7, zmm25		;; apply fudge multiplier for R5			; 80-83
	mov	r14, [r13+2*8]			;; Load the xor mask

	kshiftrw k5, k5, 8			;; I5's fudge						; 81		n 82
	vmulpd	zmm12{k7}, zmm12, zmm25		;; apply fudge multiplier for I7			; 81-84
	xor	r14, [r12+rbx*8]		;; Apply the xor mask to 64 bits of fudge flags
  	zstore	[srcreg], zmm14			;; Save R1						; 81

	kmovw	k7, r14d			;; Load R9 and I9 fudge factor mask			; 82		n 83
	vmulpd	zmm18{k5}, zmm18, zmm25		;; apply fudge multiplier for I5			; 82-85
	shr	r14, 16				;; Next 16 bits of fudge flags
	zstore	[srcreg+d4+d2], zmm8		;; Save R7						; 82

	kmovw	k5, r14d			;; Load R10 and I10 fudge factor mask			; 83		n 90
	vmulpd	zmm3{k7}, zmm3, zmm25		;; apply fudge multiplier for R9			; 83-86
	shr	r14, 16				;; Next 16 bits of fudge flags
	zstore	[srcreg+64], zmm13		;; Save I1						; 83

	kshiftrw k7, k7, 8			;; I9's fudge						; 84		n 85
	vmulpd	zmm16{k3}, zmm16, zmm25		;; apply fudge multiplier for R3			; 84-87
	zstore	[srcreg+d4], zmm7		;; Save R5						; 84

	kshiftrw k3, k3, 8			;; I3's fudge						; 85		n 86
	vmulpd	zmm9{k7}, zmm9, zmm25		;; apply fudge multiplier for I9			; 85-88
	zstore	[srcreg+d4+d2+64], zmm12	;; Save I7						; 85

	kmovw	k7, r14d			;; Load R11 and I11 fudge factor mask			; 86		n 87
	vmulpd	zmm0{k3}, zmm0, zmm25		;; apply fudge multiplier for I3			; 86-89
	shr	r14, 16				;; Next 16 bits of fudge flags
	zstore	[srcreg+d4+64], zmm18		;; Save I5						; 86

	kmovw	k3, r14d			;; Load R12 and I12 fudge factor mask			; 87		n 93
	vmulpd	zmm21{k7}, zmm21, zmm25		;; apply fudge multiplier for R11			; 87-90
	zstore	[srcreg+2*d4], zmm3		;; Save R9						; 87

	kshiftrw k7, k7, 8			;; I11's fudge						; 88		n 89
	vmulpd	zmm4{k4}, zmm4, zmm25		;; apply fudge multiplier for R4			; 88-91
	zstore	[srcreg+d2], zmm16		;; Save R3						; 88

	kshiftrw k4, k4, 8			;; I4's fudge						; 89		n 90
	vmulpd	zmm17{k7}, zmm17, zmm25		;; apply fudge multiplier for I11			; 89-92
	zstore	[srcreg+2*d4+64], zmm9		;; Save I9						; 89

	vmulpd	zmm15{k4}, zmm15, zmm25		;; apply fudge multiplier for I4			; 90-93
	vmulpd	zmm10{k5}, zmm10, zmm25		;; apply fudge multiplier for R10			; 90-93
	zstore	[srcreg+d2+64], zmm0		;; Save I3						; 90

	kshiftrw k5, k5, 8			;; I10's fudge						; 91		n 92
	vmulpd	zmm11{k1}, zmm11, zmm25		;; apply fudge multiplier for R8			; 91-94
	zstore	[srcreg+2*d4+d2], zmm21		;; Save R11						; 91

	kshiftrw k1, k1, 8			;; I8's fudge						; 92		n 93
	vmulpd	zmm1{k5}, zmm1, zmm25		;; apply fudge multiplier for I10			; 92-95
	zstore	[srcreg+d2+d1], zmm4		;; Save R4						; 92

	vmulpd	zmm24{k1}, zmm24, zmm25		;; apply fudge multiplier for I8			; 93-96
	vmulpd	zmm6{k3}, zmm6, zmm25		;; apply fudge multiplier for R12			; 93-96
	zstore	[srcreg+2*d4+d2+64], zmm17	;; Save I11						; 93

	kshiftrw k3, k3, 8			;; I12's fudge						; 94		n 96
	vmulpd	zmm19{k2}, zmm19, zmm25		;; apply fudge multiplier for R2			; 94-97
	zstore	[srcreg+d2+d1+64], zmm15	;; Save I4						; 94

	kshiftrw k2, k2, 8			;; I2's fudge						; 95		n 97
	vmulpd	zmm2{k6}, zmm2, zmm25		;; apply fudge multiplier for R6			; 95-98
	zstore	[srcreg+2*d4+d1], zmm10		;; Save R10						; 94+1

	kshiftrw k6, k6, 8			;; I6's fudge						; 96		n 97
	vmulpd	zmm20{k3}, zmm20, zmm25		;; apply fudge multiplier for I12			; 96-99
	zstore	[srcreg+d4+d2+d1], zmm11	;; Save R8						; 95+1

	vmulpd	zmm5{k2}, zmm5, zmm25		;; apply fudge multiplier for I2			; 97-100
	vmulpd	zmm22{k6}, zmm22, zmm25		;; apply fudge multiplier for I6			; 97-100
	zstore	[srcreg+2*d4+d1+64], zmm1	;; Save I10						; 96+1

	zstore	[srcreg+d4+d2+d1+64], zmm24	;; Save I8						; 97+1
	zstore	[srcreg+2*d4+d2+d1], zmm6	;; Save R12						; 97+2
	zstore	[srcreg+d1], zmm19		;; Save R2						; 98+2
	zstore	[srcreg+d4+d1], zmm2		;; Save R6						; 99+2
	zstore	[srcreg+2*d4+d2+d1+64], zmm20	;; Save I12						; 100+2
	zstore	[srcreg+d1+64], zmm5		;; Save I2						; 101+2
	zstore	[srcreg+d4+d1+64], zmm22	;; Save I6						; 101+3
	bump	srcreg, srcinc
	ENDM


;;
;; ************************************* 24-reals-fft variants ******************************************
;;

;; These macros operate on 24 reals doing 4.585 levels of the FFT, applying
;; the sin/cos multipliers afterwards.  The output is 2 reals and 11 complex numbers.

;; To calculate a 24-reals FFT, we calculate 24 complex values in a brute force way (using a shorthand notation):
;; r1 + r2 + ... + r24	*  w^0000000000...
;; r1 + r2 + ... + r24	*  w^0123456789A...
;; r1 + r2 + ... + r24	*  w^02468ACE....
;;    ...
;; r1 + r2 + ... + r24	*  w^...A987654321
;; Note that Hermetian symmetry means we won't need to calculate the last 12 complex values.
;;
;; The sin/cos values (w = 24th root of unity) are:
;; w^1 = .966 + .259i
;; w^2 = .866 + .5i
;; w^3 = .707 + .707i
;; w^4 = .5   + .866i
;; w^5 = .259 + .966i
;; w^6 = 0 + 1i
;; w^7 = -.259 + .966i
;; w^8 = -.5   + .866i
;; w^9 = -.707 + .707i
;; w^10 = -.866 + .5i
;; w^11 = -.966 + .259i
;; w^12 = -1

;; Applying the sin/cos values above (and noting that combining r2 and r14, r3 and r15, etc. will simplify calculations):
;; reals:
;; r1+r13     +(r2+r14)     +(r12+r24)      +(r3+r15)     +(r11+r23)     +(r4+r16)     +(r10+r22)     +(r5+r17)     +(r9+r21)     +(r6+r18)     +(r8+r20) + (r7+r19)
;; r1-r13 +.966(r2-r14) -.966(r12-r24)  +.866(r3-r15) -.866(r11-r23) +.707(r4-r16) -.707(r10-r22) +.500(r5-r17) -.500(r9-r21) +.259(r6-r18) -.259(r8-r20)
;; r1+r13 +.866(r2+r14) +.866(r12+r24)  +.500(r3+r15) +.500(r11+r23)                              -.500(r5+r17) -.500(r9+r21) -.866(r6+r18) -.866(r8+r20) - (r7+r19)
;; r1-r13 +.707(r2-r14) -.707(r12-r24)                               -.707(r4-r16) +.707(r10-r22)     -(r5-r17)     +(r9-r21) -.707(r6-r18) +.707(r8-r20)
;; r1+r13 +.500(r2+r14) +.500(r12+r24)  -.500(r3+r15) -.500(r11+r23)     -(r4+r16)     -(r10+r22) -.500(r5+r17) -.500(r9+r21) +.500(r6+r18) +.500(r8+r20) + (r7+r19)
;; r1-r13 +.259(r2-r14) -.259(r12-r24)  -.866(r3-r15) +.866(r11-r23) -.707(r4-r16) +.707(r10-r22) +.500(r5-r17) -.500(r9-r21) +.966(r6-r18) -.966(r8-r20)
;; r1+r13                                   -(r3+r15)     -(r11+r23)                                  +(r5+r17)     +(r9+r21)                             - (r7+r19)
;; r1-r13 -.259(r2-r14) +.259(r12-r24)  -.866(r3-r15) +.866(r11-r23) +.707(r4-r16) -.707(r10-r22) +.500(r5-r17) -.500(r9-r21) -.966(r6-r18) +.966(r8-r20)
;; r1+r13 -.500(r2+r14) -.500(r12+r24)  -.500(r3+r15) -.500(r11+r23)     +(r4+r16)     +(r10+r22) -.500(r5+r17) -.500(r9+r21) -.500(r6+r18) -.500(r8+r20) + (r7+r19)
;; r1-r13 -.707(r2-r14) +.707(r12-r24)                               +.707(r4-r16) -.707(r10-r22)     -(r5-r17)     +(r9-r21) +.707(r6-r18) -.707(r8-r20)
;; r1+r13 -.866(r2+r14) -.866(r12+r24)  +.500(r3+r15) +.500(r11+r23)                              -.500(r5+r17) -.500(r9+r21) +.866(r6+r18) +.866(r8+r20) - (r7+r19)
;; r1-r13 -.966(r2-r14) +.966(r12-r24)  +.866(r3-r15) -.866(r11-r23) -.707(r4-r16) +.707(r10-r22) +.500(r5-r17) -.500(r9-r21) -.259(r6-r18) +.259(r8-r20)
;; r1+r13     -(r2+r14)     -(r12+r24)      +(r3+r15)     +(r11+r23)     -(r4+r16)     -(r10+r22)     +(r5+r17)     +(r9+r21)     -(r6+r18)     -(r8+r20) + (r7+r19)
;;
;; imaginarys:
;; 0
;; +.259(r2-r14) +.259(r12-r24) +.500(r3-r15) +.500(r11-r23) +.707(r4-r16) +.707(r10-r22) +.866(r5-r17) +.866(r9-r21) +.966(r6-r18) +.966(r8-r20) + (r7-r19)
;; +.500(r2+r14) -.500(r12+r24) +.866(r3+r15) -.866(r11+r23)     +(r4+r16)     -(r10+r22) +.866(r5+r17) -.866(r9+r21) +.500(r6+r18) -.500(r8+r20)
;; +.707(r2-r14) +.707(r12-r24)     +(r3-r15)     +(r11-r23) +.707(r4-r16) +.707(r10-r22)                             -.707(r6-r18) -.707(r8-r20) - (r7-r19)
;; +.866(r2+r14) -.866(r12+r24) +.866(r3+r15) -.866(r11+r23)                              -.866(r5+r17) +.866(r9+r21) -.866(r6+r18) +.866(r8+r20)
;; +.966(r2-r14) +.966(r12-r24) +.500(r3-r15) +.500(r11-r23) -.707(r4-r16) -.707(r10-r22) -.866(r5-r17) -.866(r9-r21) +.259(r6-r18) +.259(r8-r20) + (r7-r19)
;;      (r2+r14)     -(r12+r24)                                  -(r4+r16)     +(r10+r22)                                 +(r6+r18)     -(r8+r20)
;; +.966(r2-r14) +.966(r12-r24) -.500(r3-r15) -.500(r11-r23) -.707(r4-r16) -.707(r10-r22) +.866(r5-r17) +.866(r9-r21) +.259(r6-r18) +.259(r8-r20) - (r7-r19)
;; +.866(r2+r14) -.866(r12+r24) -.866(r3+r15) +.866(r11+r23)                              +.866(r5+r17) -.866(r9+r21) -.866(r6+r18) +.866(r8+r20)
;; +.707(r2-r14) +.707(r12-r24)     -(r3-r15)     -(r11-r23) +.707(r4-r16) +.707(r10-r22)                             -.707(r6-r18) -.707(r8-r20) + (r7-r19)
;; +.500(r2+r14) -.500(r12+r24) -.866(r3+r15) +.866(r11+r23)     +(r4+r16)     -(r10+r22) -.866(r5+r17) +.866(r9+r21) +.500(r6+r18) -.500(r8+r20)
;; +.259(r2-r14) +.259(r12-r24) -.500(r3-r15) -.500(r11-r23) +.707(r4-r16) +.707(r10-r22) -.866(r5-r17) -.866(r9-r21) +.966(r6-r18) +.966(r8-r20) - (r7-r19)
;; 0

;; Simplifying and combining and rearranging to highlight the common subexpressions, we get:
;;R1 = (r1+r13)+(r7+r19)     +(((r3+r15)+(r11+r23))+((r5+r17)+(r9+r21)))   +(((r2+r14)+(r12+r24))+((r6+r18)+(r8+r20))) +((r4+r16)+(r10+r22))
;;R13= (r1+r13)+(r7+r19)     +(((r3+r15)+(r11+r23))+((r5+r17)+(r9+r21)))   -(((r2+r14)+(r12+r24))+((r6+r18)+(r8+r20))) +((r4+r16)+(r10+r22))
;;R5 = (r1+r13)+(r7+r19) -.500(((r3+r15)+(r11+r23))+((r5+r17)+(r9+r21)))   +.500(((r2+r14)+(r12+r24))+((r6+r18)+(r8+r20))) -((r4+r16)+(r10+r22))
;;R9 = (r1+r13)+(r7+r19) -.500(((r3+r15)+(r11+r23))+((r5+r17)+(r9+r21)))   -.500(((r2+r14)+(r12+r24))+((r6+r18)+(r8+r20))) +((r4+r16)+(r10+r22))
;;R7 = (r1+r13)-(r7+r19)     -(((r3+r15)+(r11+r23))-((r5+r17)+(r9+r21)))
;;R3 = (r1+r13)-(r7+r19) +.500(((r3+r15)+(r11+r23))-((r5+r17)+(r9+r21)))   +.866(((r2+r14)+(r12+r24))-((r6+r18)+(r8+r20)))
;;R11= (r1+r13)-(r7+r19) +.500(((r3+r15)+(r11+r23))-((r5+r17)+(r9+r21)))   -.866(((r2+r14)+(r12+r24))-((r6+r18)+(r8+r20)))
;;R4 = (r1-r13)     -((r5-r17)-(r9-r21))                             +.707((r2-r14)-(r12-r24)) -.707((r4-r16)-(r10-r22)) -.707((r6-r18)-(r8-r20))
;;R10= (r1-r13)     -((r5-r17)-(r9-r21))                             -.707((r2-r14)-(r12-r24)) +.707((r4-r16)-(r10-r22)) +.707((r6-r18)-(r8-r20))
;;R2 = (r1-r13) +.500((r5-r17)-(r9-r21)) +.866((r3-r15)-(r11-r23))   +.966((r2-r14)-(r12-r24)) +.707((r4-r16)-(r10-r22)) +.259((r6-r18)-(r8-r20))
;;R12= (r1-r13) +.500((r5-r17)-(r9-r21)) +.866((r3-r15)-(r11-r23))   -.966((r2-r14)-(r12-r24)) -.707((r4-r16)-(r10-r22)) -.259((r6-r18)-(r8-r20))
;;R6 = (r1-r13) +.500((r5-r17)-(r9-r21)) -.866((r3-r15)-(r11-r23))   +.259((r2-r14)-(r12-r24)) -.707((r4-r16)-(r10-r22)) +.966((r6-r18)-(r8-r20))
;;R8 = (r1-r13) +.500((r5-r17)-(r9-r21)) -.866((r3-r15)-(r11-r23))   -.259((r2-r14)-(r12-r24)) +.707((r4-r16)-(r10-r22)) -.966((r6-r18)-(r8-r20))
;;I7 =      (((r2+r14)-(r12+r24))+((r6+r18)-(r8+r20))) -((r4+r16)-(r10+r22))
;;I3 = +.500(((r2+r14)-(r12+r24))+((r6+r18)-(r8+r20))) +((r4+r16)-(r10+r22))   +.866(((r3+r15)-(r11+r23))+((r5+r17)-(r9+r21)))
;;I11= +.500(((r2+r14)-(r12+r24))+((r6+r18)-(r8+r20))) +((r4+r16)-(r10+r22))   -.866(((r3+r15)-(r11+r23))+((r5+r17)-(r9+r21)))
;;I5 = +.866(((r2+r14)-(r12+r24))-((r6+r18)-(r8+r20)))                         +.866(((r3+r15)-(r11+r23))-((r5+r17)-(r9+r21)))
;;I9 = +.866(((r2+r14)-(r12+r24))-((r6+r18)-(r8+r20)))                         -.866(((r3+r15)-(r11+r23))-((r5+r17)-(r9+r21)))
;;I10= +.707((r2-r14)+(r12-r24)) +.707((r4-r16)+(r10-r22)) -.707((r6-r18)+(r8-r20))  +(r7-r19)  -((r3-r15)+(r11-r23))
;;I4 = +.707((r2-r14)+(r12-r24)) +.707((r4-r16)+(r10-r22)) -.707((r6-r18)+(r8-r20))  -(r7-r19)  +((r3-r15)+(r11-r23))
;;I2 = +.259((r2-r14)+(r12-r24)) +.707((r4-r16)+(r10-r22)) +.966((r6-r18)+(r8-r20))  +(r7-r19) +.500((r3-r15)+(r11-r23)) +.866((r5-r17)+(r9-r21))
;;I12= +.259((r2-r14)+(r12-r24)) +.707((r4-r16)+(r10-r22)) +.966((r6-r18)+(r8-r20))  -(r7-r19) -.500((r3-r15)+(r11-r23)) -.866((r5-r17)+(r9-r21))
;;I6 = +.966((r2-r14)+(r12-r24)) -.707((r4-r16)+(r10-r22)) +.259((r6-r18)+(r8-r20))  +(r7-r19) +.500((r3-r15)+(r11-r23)) -.866((r5-r17)+(r9-r21))
;;I8 = +.966((r2-r14)+(r12-r24)) -.707((r4-r16)+(r10-r22)) +.259((r6-r18)+(r8-r20))  -(r7-r19) -.500((r3-r15)+(r11-r23)) +.866((r5-r17)+(r9-r21))

; Uses two sin/cos pointers
zr12_2sc_twentyfour_reals_fft_preload MACRO
	zr12_24r_fft_cmn_preload
	ENDM
zr12_2sc_twentyfour_reals_fft MACRO srcreg,srcinc,d1,d2,d4,screg1,scinc1,screg2,scinc2,maxrpt,L1pt,L1pd
	zr12_24r_fft_cmn srcreg,0,srcinc,d1,d2,d4,screg1,scinc1,screg2,scinc2,maxrpt,L1pt,L1pd
	ENDM

; Offsets source by rbx
zr12f_2sc_twentyfour_reals_fft_preload MACRO
	zr12_24r_fft_cmn_preload
	ENDM
zr12f_2sc_twentyfour_reals_fft MACRO srcreg,srcinc,d1,d2,d4,screg1,scinc1,screg2,scinc2,maxrpt,L1pt,L1pd
	zr12_24r_fft_cmn srcreg,rbx,srcinc,d1,d2,d4,screg1,scinc1,screg2,scinc2,maxrpt,L1pt,L1pd
	ENDM

; Combined sin/cos data
zr12_csc_twentyfour_reals_fft_preload MACRO
	zr12_24r_fft_cmn_preload
	ENDM
zr12_csc_twentyfour_reals_fft MACRO srcreg,srcinc,d1,d2,d4,screg,scinc,maxrpt,L1pt,L1pd
	zr12_24r_fft_cmn srcreg,0,srcinc,d1,d2,d4,screg+6*128,0,screg,scinc,maxrpt,L1pt,L1pd
	ENDM


zr12_24r_fft_cmn_preload MACRO
	vbroadcastsd zmm31, ZMM_HALF
	vbroadcastsd zmm30, ZMM_P866
	vbroadcastsd zmm29, ZMM_SQRTHALF
	vbroadcastsd zmm28, ZMM_P259_P707
	vbroadcastsd zmm27, ZMM_P966_P707
	ENDM

zr12_24r_fft_cmn MACRO srcreg,srcoff,srcinc,d1,d2,d4,screg1,scinc1,screg2,scinc2,maxrpt,L1pt,L1pd
	vmovapd	zmm24, [srcreg+srcoff]		;; r1+r13
	vmovapd	zmm12, [srcreg+srcoff+d4+d2]	;; r7+r19
	vaddpd	zmm19, zmm24, zmm12		;; r1++ = (r1+r13) + (r7+r19)				; 1-4
	vsubpd	zmm24, zmm24, zmm12		;; r1+- = (r1+r13) - (r7+r19)				; 1-4

	vmovapd	zmm18, [srcreg+srcoff+d2]	;; r3+r15
	vmovapd	zmm14, [srcreg+srcoff+2*d4+d2]	;; r11+r23
	vaddpd	zmm12, zmm18, zmm14		;; r3++ = (r3+r15) + (r11+r23)				; 2-5
	vsubpd	zmm18, zmm18, zmm14		;; r3+- = (r3+r15) - (r11+r23)				; 2-5

	vmovapd	zmm22, [srcreg+srcoff+d4]	;; r5+r17
	vmovapd	zmm16, [srcreg+srcoff+2*d4]	;; r9+r21
	vaddpd	zmm14, zmm22, zmm16		;; r5++ = (r5+r17) + (r9+r21)				; 3-6
	vsubpd	zmm22, zmm22, zmm16		;; r5+- = (r5+r17) - (r9+r21)				; 3-6

	vmovapd	zmm20, [srcreg+srcoff+d1]	;; r2+r14
	vmovapd	zmm13, [srcreg+srcoff+2*d4+d2+d1];; r12+r24
	vaddpd	zmm16, zmm20, zmm13		;; r2++ = (r2+r14) + (r12+r24)				; 4-7
	vsubpd	zmm20, zmm20, zmm13		;; r2+- = (r2+r14) - (r12+r24)				; 4-7

	vmovapd	zmm23, [srcreg+srcoff+d2+d1]	;; r4+r16
	vmovapd	zmm15, [srcreg+srcoff+2*d4+d1]	;; r10+r22
	vaddpd	zmm13, zmm23, zmm15		;; r4++ = (r4+r16) + (r10+r22)				; 5-8
	vsubpd	zmm23, zmm23, zmm15		;; r4+- = (r4+r16) - (r10+r22)				; 5-8

	vmovapd	zmm21, [srcreg+srcoff+d4+d1]	;; r6+r18
	vmovapd	zmm17, [srcreg+srcoff+d4+d2+d1]	;; r8+r20
	vaddpd	zmm15, zmm21, zmm17		;; r6++ = (r6+r18) + (r8+r20)				; 6-9
	vsubpd	zmm21, zmm21, zmm17		;; r6+- = (r6+r18) - (r8+r20)				; 6-9

	vmovapd	zmm2, [srcreg+srcoff+d2+64]	;; r3-r15
	vmovapd	zmm10, [srcreg+srcoff+2*d4+d2+64];; r11-r23
	vaddpd	zmm17, zmm2, zmm10		;; r3-+ = (r3-r15) + (r11-r23)				; 7-10
	vsubpd	zmm2, zmm2, zmm10		;; r3-- = (r3-r15) - (r11-r23)				; 7-10

	vmovapd	zmm4, [srcreg+srcoff+d4+64]	;; r5-r17
	vmovapd	zmm8, [srcreg+srcoff+2*d4+64]	;; r9-r21
	vaddpd	zmm10, zmm4, zmm8		;; r5-+ = (r5-r17) + (r9-r21)				; 8-11
	vsubpd	zmm4, zmm4, zmm8		;; r5-- = (r5-r17) - (r9-r21)				; 8-11

	vmovapd	zmm1, [srcreg+srcoff+d1+64]	;; r2-r14
	vmovapd	zmm11, [srcreg+srcoff+2*d4+d2+d1+64];; r12-r24
	vaddpd	zmm8, zmm1, zmm11		;; r2-+ = (r2-r14) + (r12-r24)				; 9-12
	vsubpd	zmm1, zmm1, zmm11		;; r2-- = (r2-r14) - (r12-r24)				; 9-12

	vmovapd	zmm3, [srcreg+srcoff+d2+d1+64]	;; r4-r16
	vmovapd	zmm9, [srcreg+srcoff+2*d4+d1+64];; r10-r22
	vaddpd	zmm11, zmm3, zmm9		;; r4-+ = (r4-r16) + (r10-r22)				; 10-13
	vsubpd	zmm3, zmm3, zmm9		;; r4-- = (r4-r16) - (r10-r22)				; 10-13

	vmovapd	zmm5, [srcreg+srcoff+d4+d1+64]	;; r6-r18
	vmovapd	zmm7, [srcreg+srcoff+d4+d2+d1+64];; r8-r20
	vaddpd	zmm9, zmm5, zmm7		;; r6-+ = (r6-r18) + (r8-r20)				; 11-14
	vsubpd	zmm5, zmm5, zmm7		;; r6-- = (r6-r18) - (r8-r20)				; 11-14

	vmovapd	zmm0, [srcreg+srcoff+64]	;; r1-r13
	vaddpd	zmm7, zmm12, zmm14		;; r3+++ = (r3++) + (r5++)				; 12-15
	vsubpd	zmm12, zmm12, zmm14		;; r3++- = (r3++) - (r5++)				; 12-15

	vmovapd	zmm6, [srcreg+srcoff+d4+d2+64]	;; r7-r19
	vaddpd	zmm14, zmm20, zmm21		;; r2+-+ = (r2+-) + (r6+-)				; 13-16
	vsubpd	zmm20, zmm20, zmm21		;; r2+-- = (r2+-) - (r6+-)	(I59e / .866)		; 13-16

	vmovapd zmm26, [screg2+2*128+64]	;; cosine/sine for R7/I7 (w^6 = 12-complex w^3)
	vaddpd	zmm21, zmm16, zmm15		;; r2+++ = (r2++) + (r6++)				; 14-17
	vsubpd	zmm16, zmm16, zmm15		;; r2++- = (r2++) - (r6++)	(R3Be / .866)		; 14-17

	L1prefetchw srcreg+L1pd, L1pt
	vaddpd	zmm15, zmm18, zmm22		;; r3+-+ = (r3+-) + (r5+-)	(I3Bo / .866)		; 15-18
	vsubpd	zmm18, zmm18, zmm22		;; r3+-- = (r3+-) - (r5+-)	(I59o / .866)		; 15-18

	L1prefetchw srcreg+64+L1pd, L1pt
	vsubpd	zmm22, zmm24, zmm12		;; R7 = (r1+-) - (r3++-)				; 16-19
	zfmaddpd zmm12, zmm12, zmm31, zmm24	;; R3Bo = (r1+-) + .500(r3++-)				; 16-19

	L1prefetchw srcreg+d1+L1pd, L1pt
	vsubpd	zmm24, zmm14, zmm23		;; I7 = (r2+-+) - (r4+-)				; 17-20
	zfmaddpd zmm14, zmm14, zmm31, zmm23	;; I3Be = .500(r2+-+) + (r4+-)				; 17-20

	L1prefetchw srcreg+d1+64+L1pd, L1pt
	vaddpd	zmm23, zmm19, zmm7		;; R1A = (r1++) + (r3+++)				; 18-21
	zfnmaddpd zmm7, zmm7, zmm31, zmm19	;; R59o = (r1++) - .500(r3+++)				; 18-21

	L1prefetchw srcreg+d2+L1pd, L1pt
	vaddpd	zmm19, zmm21, zmm13		;; R1B = (r2+++) + (r4++)				; 19-22
	zfmsubpd zmm21, zmm21, zmm31, zmm13	;; R59e = .500*(r2+++) - (r4++)				; 19-22

	L1prefetchw srcreg+d2+64+L1pd, L1pt
	vmulpd	zmm20, zmm20, zmm30		;; I59e = I59e * .866					; 20-23
													; empty slot, will likely get used!

	L1prefetchw srcreg+d2+d1+L1pd, L1pt
	vsubpd	zmm13, zmm0, zmm4		;; R4Ao = (r1-) - (r5--)				; 21-24
	zfmaddpd zmm4, zmm4, zmm31, zmm0	;; R2C68o = (r1-) + .500(r5--)				; 21-24

	L1prefetchw srcreg+d2+d1+64+L1pd, L1pt
	vsubpd	zmm0, zmm6, zmm17		;; IA4o = (r7-) - (r3-+)				; 22-25
	zfmaddpd zmm17, zmm17, zmm31, zmm6	;; I2C68o = (r7-) + .500(r3-+)				; 22-25

	L1prefetchw srcreg+d4+L1pd, L1pt
	vaddpd	zmm25, zmm23, zmm19		;; R1  = R1A + R1B					; 23-26
	vsubpd	zmm23, zmm23, zmm19		;; R13 = R1A - R1B					; 23-26

	L1prefetchw srcreg+d4+64+L1pd, L1pt
	zfmsubpd zmm6, zmm22, zmm26, zmm24	;; A7 = R7 * cosine/sine - I7				; 24-27
	zfmaddpd zmm24, zmm24, zmm26, zmm22	;; B7 = I7 * cosine/sine + R7				; 24-27

	vmovapd zmm26, [screg2+2*128]		;; sine for R7/I7 (w^6 = 12-complex w^3)
	zfmaddpd zmm22, zmm16, zmm30, zmm12	;; R3 = R3Bo + .866*R3Be				; 25-28
	zfnmaddpd zmm16, zmm16, zmm30, zmm12	;; R11 = R3Bo - .866*R3Be				; 25-28

	L1prefetchw srcreg+d4+d1+L1pd, L1pt
	zfmaddpd zmm12, zmm15, zmm30, zmm14	;; I3 = I3Be + .866*I3Bo				; 26-29
	zfnmaddpd zmm15, zmm15, zmm30, zmm14	;; I11 = I3Be - .866*I3Bo				; 26-29

	L1prefetchw srcreg+d4+d1+64+L1pd, L1pt
	vaddpd	zmm14, zmm7, zmm21		;; R5 = R59o + R59e					; 27-30
	vsubpd	zmm7, zmm7, zmm21		;; R9 = R59o - R59e					; 27-30
	zstore	[srcreg], zmm25			;; Save R1						; 27

	vmovapd zmm25, [screg2+0*128+64]	;; cosine/sine for R3/I3 (w^2 = 12-complex w^1)
	zfmaddpd zmm21, zmm18, zmm30, zmm20	;; I5 = I59e + .866*I59o				; 28-31
	zfnmaddpd zmm18, zmm18, zmm30, zmm20	;; I9 = I59e - .866*I59o				; 28-31
	zstore	[srcreg+64], zmm23		;; Save R13						; 27+1

	L1prefetchw srcreg+d4+d2+L1pd, L1pt
	vmulpd	zmm6, zmm6, zmm26		;; A7 = A7 * sine (final R7)				; 29-32
	vmulpd	zmm24, zmm24, zmm26		;; B7 = B7 * sine (final I7)				; 29-32

	vmovapd zmm26, [screg2+4*128+64]	;; cosine/sine for R11/I11 (w^10 = 12-complex w^5)
	vsubpd	zmm20, zmm1, zmm3		;; R4Ae = (r2--) - (r4--)				; 30-33
	zfmaddpd zmm23, zmm1, zmm27, zmm3	;; R2Ce = .966/.707(r2--) + (r4--)			; 30-33

	L1prefetchw srcreg+d4+d2+64+L1pd, L1pt
	zfmsubpd zmm1, zmm1, zmm28, zmm3	;; R68e = .259/.707(r2--) - (r4--)			; 31-34
	vaddpd	zmm3, zmm8, zmm11		;; IA4e = (r2-+) + (r4-+)				; 31-34

	L1prefetchw srcreg+d4+d2+d1+L1pd, L1pt
	zfmaddpd zmm19, zmm8, zmm28, zmm11	;; I2Ce = .259/.707(r2-+) + (r4-+)			; 32-35
	zfmsubpd zmm8, zmm8, zmm27, zmm11	;; I68e = .966/.707(r2-+) - (r4-+)			; 32-35

	L1prefetchw srcreg+d4+d2+d1+64+L1pd, L1pt
	zfmsubpd zmm11, zmm22, zmm25, zmm12	;; A3 = R3 * cosine/sine - I3				; 33-36
	zfmaddpd zmm12, zmm12, zmm25, zmm22	;; B3 = I3 * cosine/sine + R3				; 33-36
	zstore	[srcreg+d4+d2], zmm6		;; Save R7						; 33

	vmovapd zmm25, [screg2+1*128+64]	;; cosine/sine for R5/I5 (w^4 = 12-complex w^2)
	zfmsubpd zmm22, zmm16, zmm26, zmm15	;; A11 = R11 * cosine/sine - I11			; 34-37
	zfmaddpd zmm15, zmm15, zmm26, zmm16	;; B11 = I11 * cosine/sine + R11			; 34-37
	zstore	[srcreg+d4+d2+64], zmm24	;; Save I7						; 33+1

	vmovapd zmm26, [screg2+3*128+64]	;; cosine/sine for R9/I9 (w^8 = 12-complex w^4)
	zfmsubpd zmm16, zmm14, zmm25, zmm21	;; A5 = R5 * cosine/sine - I5				; 35-38
	zfmaddpd zmm21, zmm21, zmm25, zmm14	;; B5 = I5 * cosine/sine + R5				; 35-38

	vmovapd zmm25, [screg2+0*128]		;; sine for R3/I3 (w^2 = 12-complex w^1)
	zfmsubpd zmm14, zmm7, zmm26, zmm18	;; A9 = R9 * cosine/sine - I9				; 36-39
	zfmaddpd zmm18, zmm18, zmm26, zmm7	;; B9 = I9 * cosine/sine + R9				; 36-39

	vmovapd zmm26, [screg2+4*128]		;; sine for R11/I11 (w^10 = 12-complex w^5)
	zfmaddpd zmm7, zmm2, zmm30, zmm4	;; R2Co = R2C68o + .866(r3--)				; 37-40
	zfnmaddpd zmm2, zmm2, zmm30, zmm4	;; R68o = R2C68o - .866(r3--)				; 37-40

	vmovapd zmm24, [screg2+1*128]		;; sine for R5/I5 (w^4 = 12-complex w^2)
	zfmaddpd zmm4, zmm10, zmm30, zmm17	;; I2Co = I2C68o + .866(r5-+)				; 38-41
	zfnmaddpd zmm10, zmm10, zmm30, zmm17	;; I68o = I2C68o - .866(r5-+)				; 38-41

	vmovapd zmm6, [screg2+3*128]		;; sine for R9/I9 (w^8 = 12-complex w^4)
	vsubpd	zmm20, zmm20, zmm5		;; R4Ae = R4Ae - (r6--)					; 39-42
	zfmaddpd zmm23, zmm5, zmm28, zmm23	;; R2Ce = R2Ce + .259/.707(r6--)			; 39-42

	L1prefetchw srcreg+2*d4+L1pd, L1pt
	zfmaddpd zmm1, zmm5, zmm27, zmm1	;; R68e = R68e + .966/.707(r6--)			; 40-43
	vsubpd	zmm3, zmm3, zmm9		;; IA4e = IA4e - (r6-+)					; 40-43

	L1prefetchw srcreg+2*d4+64+L1pd, L1pt
	zfmaddpd zmm19, zmm9, zmm27, zmm19	;; I2Ce = I2Ce + .966/.707(r6-+)			; 41-44
	zfmaddpd zmm8, zmm9, zmm28, zmm8	;; I68e = I68e + .259/.707(r6-+)			; 41-44

	L1prefetchw srcreg+2*d4+d1+L1pd, L1pt
	vmulpd	zmm11, zmm11, zmm25		;; A3 = A3 * sine (final R3)				; 42-45
	vmulpd	zmm12, zmm12, zmm25		;; B3 = B3 * sine (final I3)				; 42-45

	vmovapd zmm25, [screg1+1*128+64]	;; cosine/sine for R4/I4 (w^3)
	vmulpd	zmm22, zmm22, zmm26		;; A11 = A11 * sine (final R11)				; 43-46
	vmulpd	zmm15, zmm15, zmm26		;; B11 = B11 * sine (final I11)				; 43-46

	vmovapd zmm26, [screg1+4*128+64]	;; cosine/sine for R10/I10 (w^9)
	vmulpd	zmm16, zmm16, zmm24		;; A5 = A5 * sine (final R5)				; 44-47
	vmulpd	zmm21, zmm21, zmm24		;; B5 = B5 * sine (final I5)				; 44-47

	vmovapd zmm24, [screg1+0*128+64]	;; cosine/sine for R2/I2 (w^1)
	vmulpd	zmm14, zmm14, zmm6		;; A9 = A9 * sine (final R9)				; 45-48
	vmulpd	zmm18, zmm18, zmm6		;; B9 = B9 * sine (final I9)				; 45-48

	vmovapd zmm6, [screg1+5*128+64]		;; cosine/sine for R12/I12 (w^11)
	zfmaddpd zmm17, zmm20, zmm29, zmm13	;; R4 = R4Ao + .707*R4Ae				; 46-49
	zfnmaddpd zmm20, zmm20, zmm29, zmm13	;; R10 = R4Ao - .707*R4Ae				; 46-49
	zstore	[srcreg+d2], zmm11		;; Save R3						; 46

	vmovapd zmm11, [screg1+2*128+64]	;; cosine/sine for R6/I6 (w^5)
	zfmaddpd zmm13, zmm3, zmm29, zmm0	;; I10 = .707*IA4e + IA4o				; 47-50
	zfmsubpd zmm3, zmm3, zmm29, zmm0	;; I4 = .707*IA4e - IA4o				; 47-50
	zstore	[srcreg+d2+64], zmm12		;; Save I3						; 46+1

	vmovapd zmm12, [screg1+3*128+64]	;; cosine/sine for R8/I8 (w^7)
	zfmaddpd zmm0, zmm23, zmm29, zmm7	;; R2 = R2Co + .707*R2Ce				; 48-51
	zfnmaddpd zmm23, zmm23, zmm29, zmm7	;; R12 = R2Co - .707*R2Ce				; 48-51
	zstore	[srcreg+2*d4+d2], zmm22		;; Save R11						; 47+1

	vmovapd zmm22, [screg1+1*128]		;; sine for R4/I4 (w^3)
	zfmaddpd zmm7, zmm19, zmm29, zmm4	;; I2 = .707*I2Ce + I2Co				; 49-52
	zfmsubpd zmm19, zmm19, zmm29, zmm4	;; I12 = .707*I2Ce - I2Co				; 49-52
	zstore	[srcreg+2*d4+d2+64], zmm15	;; Save I11						; 47+2

	vmovapd zmm15, [screg1+4*128]		;; sine for R10/I10 (w^9)
	zfmaddpd zmm4, zmm1, zmm29, zmm2	;; R6 = R68o + .707*R68e				; 50-53
	zfnmaddpd zmm1, zmm1, zmm29, zmm2	;; R8 = R68o - .707*R68e				; 50-53
	zstore	[srcreg+d4], zmm16		;; Save R5						; 48+2

	vmovapd zmm16, [screg1+0*128]		;; sine for R2/I2 (w^1)
	zfmaddpd zmm2, zmm8, zmm29, zmm10	;; I6 = .707*I68e + I68o				; 51-54
	zfmsubpd zmm8, zmm8, zmm29, zmm10	;; I8 = .707*I68e - I68o				; 51-54
	zstore	[srcreg+d4+64], zmm21		;; Save I5						; 48+3

	vmovapd zmm21, [screg1+5*128]		;; sine for R12/I12 (w^11)
	zfmsubpd zmm10, zmm17, zmm25, zmm3	;; A4 = R4 * cosine/sine - I4				; 52-55
	zfmaddpd zmm3, zmm3, zmm25, zmm17	;; B4 = I4 * cosine/sine + R4				; 52-55
	zstore	[srcreg+2*d4], zmm14		;; Save R9						; 49+3

	vmovapd zmm14, [screg1+2*128]		;; sine for R6/I6 (w^5)
	zfmsubpd zmm17, zmm20, zmm26, zmm13	;; A10 = R10 * cosine/sine - I10			; 53-56
	zfmaddpd zmm13, zmm13, zmm26, zmm20	;; B10 = I10 * cosine/sine + R10			; 53-56
	zstore	[srcreg+2*d4+64], zmm18		;; Save I9						; 49+4

	vmovapd zmm18, [screg1+3*128]		;; sine for R8/I8 (w^7)
	zfmsubpd zmm20, zmm0, zmm24, zmm7	;; A2 = R2 * cosine/sine - I2				; 54-57
	zfmaddpd zmm7, zmm7, zmm24, zmm0	;; B2 = I2 * cosine/sine + R2				; 54-57
	bump	screg1, scinc1

	L1prefetchw srcreg+2*d4+d1+64+L1pd, L1pt
	zfmsubpd zmm0, zmm23, zmm6, zmm19	;; A12 = R12 * cosine/sine - I12			; 55-58
	zfmaddpd zmm19, zmm19, zmm6, zmm23	;; B12 = I12 * cosine/sine + R12			; 55-58
	bump	screg2, scinc2

	L1prefetchw srcreg+2*d4+d2+L1pd, L1pt
	zfmsubpd zmm23, zmm4, zmm11, zmm2	;; A6 = R6 * cosine/sine - I6				; 56-59
	zfmaddpd zmm2, zmm2, zmm11, zmm4	;; B6 = I6 * cosine/sine + R6				; 56-59

	L1prefetchw srcreg+2*d4+d2+64+L1pd, L1pt
	zfmsubpd zmm4, zmm1, zmm12, zmm8	;; A8 = R8 * cosine/sine - I8				; 57-60
	zfmaddpd zmm8, zmm8, zmm12, zmm1	;; B8 = I8 * cosine/sine + R8				; 57-60

	L1prefetchw srcreg+2*d4+d2+d1+L1pd, L1pt
	vmulpd	zmm10, zmm10, zmm22		;; A4 = A4 * sine (final R4)				; 58-61
	vmulpd	zmm3, zmm3, zmm22		;; B4 = B4 * sine (final I4)				; 58-61

	L1prefetchw srcreg+2*d4+d2+d1+64+L1pd, L1pt
	vmulpd	zmm17, zmm17, zmm15		;; A10 = A10 * sine (final R10)				; 59-62
	vmulpd	zmm13, zmm13, zmm15		;; B10 = B10 * sine (final I10)				; 59-62

	vmulpd	zmm20, zmm20, zmm16		;; A2 = A2 * sine (final R2)				; 60-63
	vmulpd	zmm7, zmm7, zmm16		;; B2 = B2 * sine (final I2)				; 60-63

	vmulpd	zmm0, zmm0, zmm21		;; A12 = A12 * sine (final R12)				; 61-64
	vmulpd	zmm19, zmm19, zmm21		;; B12 = B12 * sine (final I12)				; 61-64

	vmulpd	zmm23, zmm23, zmm14		;; A6 = A6 * sine (final R6)				; 62-65
	vmulpd	zmm2, zmm2, zmm14		;; B6 = B6 * sine (final I6)				; 62-65
	zstore	[srcreg+d2+d1], zmm10		;; Save R4						; 62

	vmulpd	zmm4, zmm4, zmm18		;; A8 = A8 * sine (final R8)				; 63-66
	vmulpd	zmm8, zmm8, zmm18		;; B8 = B8 * sine (final I8)				; 63-66

	zstore	[srcreg+d2+d1+64], zmm3		;; Save I4						; 62+1
	zstore	[srcreg+2*d4+d1], zmm17		;; Save R10						; 63+1
	zstore	[srcreg+2*d4+d1+64], zmm13	;; Save I10						; 63+2
	zstore	[srcreg+d1], zmm20		;; Save R2						; 64+2
	zstore	[srcreg+d1+64], zmm7		;; Save I2						; 64+3
	zstore	[srcreg+2*d4+d2+d1], zmm0	;; Save R12						; 65+3
	zstore	[srcreg+2*d4+d2+d1+64], zmm19	;; Save I12						; 65+4
	zstore	[srcreg+d4+d1], zmm23		;; Save R6						; 66+4
	zstore	[srcreg+d4+d1+64], zmm2		;; Save I6						; 66+5
	zstore	[srcreg+d4+d2+d1], zmm4		;; Save R8						; 67+5
	zstore	[srcreg+d4+d2+d1+64], zmm8	;; Save I8						; 67+6
	bump	srcreg, srcinc
	ENDM



;; Used in first levels of pass 1 in a two-pass FFT
zr12_wpn_twentyfour_reals_first_fft_preload MACRO
	vbroadcastsd zmm31, ZMM_HALF
	vbroadcastsd zmm30, ZMM_P866
	vbroadcastsd zmm29, ZMM_SQRTHALF
	vbroadcastsd zmm28, ZMM_P259_P707
	vbroadcastsd zmm27, ZMM_P966_P707
	vbroadcastsd zmm26, ZMM_ONE_OVER_B
	ENDM

zr12_wpn_twentyfour_reals_first_fft MACRO srcreg,srcinc,d1,d2,d4,screg,scinc,maskreg,maskinc,grpreg,grpinc,maxrpt,L1pt,L1pd
	mov	bl, [maskreg+0*1]		;; Load index into compressed fudges
	mov	r14, [r13+0*8]			;; Load the xor mask
	xor	r14, [r12+rbx*8]		;; Apply the xor mask to 64 bits of fudge flags

	kmovw	k1, r14d			;; Load R1 and R13 fudge factor mask			; 1
	kshiftrw k2, k1, 8			;; R13's fudge						; 2
	vmovapd	zmm0, [srcreg]			;; R1
	vmulpd	zmm0{k1}, zmm0, zmm26		;; apply fudges to R1					; 2-5		n 25
	vmovapd	zmm12, [srcreg+64]		;; R13
	vmulpd	zmm12{k2}, zmm12, zmm26		;; apply fudges to R13					; 3-6		n 29

	shr	r14, 16				;; Next 16 bits of fudge flags
	kmovw	k1, r14d			;; Load R2 and R14 fudge factor mask			; 3
	kshiftrw k2, k1, 8			;; R14's fudge						; 4
	vmovapd	zmm1, [srcreg+d1]		;; R2
	vmulpd	zmm1{k1}, zmm1, zmm26		;; apply fudges to R2					; 4-7		n 35
	vmovapd	zmm13, [srcreg+d1+64]		;; R14
	vmulpd	zmm13{k2}, zmm13, zmm26		;; apply fudges to R14					; 5-8		n 35

	shr	r14, 16				;; Next 16 bits of fudge flags
	kmovw	k1, r14d			;; Load R3 and R15 fudge factor mask			; 5
	kshiftrw k2, k1, 8			;; R15's fudge						; 6
	vmovapd	zmm2, [srcreg+d2]		;; R3
	vmulpd	zmm2{k1}, zmm2, zmm26		;; apply fudges to R3					; 6-9		n 31
	vmovapd	zmm14, [srcreg+d2+64]		;; R15
	vmulpd	zmm14{k2}, zmm14, zmm26		;; apply fudges to R15					; 7-10		n 31

	shr	r14, 16				;; Next 16 bits of fudge flags
	kmovw	k1, r14d			;; Load R4 and R16 fudge factor mask			; 7
	kshiftrw k2, k1, 8			;; R16's fudge						; 8
	vmovapd	zmm3, [srcreg+d2+d1]		;; R4
	vmulpd	zmm3{k1}, zmm3, zmm26		;; apply fudges to R4					; 8-11		n 37
	vmovapd	zmm15, [srcreg+d2+d1+64]	;; R16
	vmulpd	zmm15{k2}, zmm15, zmm26		;; apply fudges to R16					; 9-12		n 37

	mov	bl, [maskreg+1*1]		;; Load index into compressed fudges
	mov	r14, [r13+1*8]			;; Load the xor mask
	xor	r14, [r12+rbx*8]		;; Apply the xor mask to 64 bits of fudge flags

	kmovw	k1, r14d			;; Load R5 and R17 fudge factor mask			; 9
	kshiftrw k2, k1, 8			;; R17's fudge						; 10
	vmovapd	zmm4, [srcreg+d4]		;; R5
	vmulpd	zmm4{k1}, zmm4, zmm26		;; apply fudges to R5					; 10-13		n 33
	vmovapd	zmm16, [srcreg+d4+64]		;; R17
	vmulpd	zmm16{k2}, zmm16, zmm26		;; apply fudges to R17					; 11-14		n 33

	shr	r14, 16				;; Next 16 bits of fudge flags
	kmovw	k1, r14d			;; Load R6 and R18 fudge factor mask			; 11
	kshiftrw k2, k1, 8			;; R18's fudge						; 12
	vmovapd	zmm5, [srcreg+d4+d1]		;; R6
	vmulpd	zmm5{k1}, zmm5, zmm26		;; apply fudges to R6					; 12-15		n 39
	vmovapd	zmm17, [srcreg+d4+d1+64]	;; R18
	vmulpd	zmm17{k2}, zmm17, zmm26		;; apply fudges to R18					; 13-16		n 39

	shr	r14, 16				;; Next 16 bits of fudge flags
	kmovw	k1, r14d			;; Load R7 and R19 fudge factor mask			; 13
	kshiftrw k2, k1, 8			;; R19's fudge						; 14
	vmovapd	zmm6, [srcreg+d4+d2]		;; R7
	vmulpd	zmm6{k1}, zmm6, zmm26		;; apply fudges to R7					; 14-17		n 26
	vmovapd	zmm18, [srcreg+d4+d2+64]	;; R19
	vmulpd	zmm18{k2}, zmm18, zmm26		;; apply fudges to R19					; 15-18		n 30

	shr	r14, 16				;; Next 16 bits of fudge flags
	kmovw	k1, r14d			;; Load R8 and R20 fudge factor mask			; 15
	kshiftrw k2, k1, 8			;; R20's fudge						; 16
	vmovapd	zmm7, [srcreg+d4+d2+d1]		;; R8
	vmulpd	zmm7{k1}, zmm7, zmm26		;; apply fudges to R8					; 16-19		n 28
	vmovapd	zmm19, [srcreg+d4+d2+d1+64]	;; R20
	vmulpd	zmm19{k2}, zmm19, zmm26		;; apply fudges to R20					; 17-20		n 40

	mov	bl, [maskreg+2*1]		;; Load index into compressed fudges
	mov	r14, [r13+2*8]			;; Load the xor mask
	xor	r14, [r12+rbx*8]		;; Apply the xor mask to 64 bits of fudge flags

	kmovw	k1, r14d			;; Load R9 and R21 fudge factor mask			; 17
	kshiftrw k2, k1, 8			;; R21's fudge						; 18
	vmovapd	zmm8, [srcreg+2*d4]		;; R9
	vmulpd	zmm8{k1}, zmm8, zmm26		;; apply fudges to R9					; 18-21		n 27
	vmovapd	zmm20, [srcreg+2*d4+64]		;; R21
	vmulpd	zmm20{k2}, zmm20, zmm26		;; apply fudges to R21					; 19-22		n 34

	shr	r14, 16				;; Next 16 bits of fudge flags
	kmovw	k1, r14d			;; Load R10 and R22 fudge factor mask			; 19
	kshiftrw k2, k1, 8			;; R22's fudge						; 20
	vmovapd	zmm9, [srcreg+2*d4+d1]		;; R10
	vmulpd	zmm9{k1}, zmm9, zmm26		;; apply fudges to R10					; 20-23		n 28
	vmovapd	zmm21, [srcreg+2*d4+d1+64]	;; R22
	vmulpd	zmm21{k2}, zmm21, zmm26		;; apply fudges to R22					; 21-24		n 38

	shr	r14, 16				;; Next 16 bits of fudge flags
	kmovw	k1, r14d			;; Load R11 and R23 fudge factor mask			; 21
	kshiftrw k2, k1, 8			;; R23's fudge						; 22
	vmovapd	zmm10, [srcreg+2*d4+d2]		;; R11
	vmulpd	zmm10{k1}, zmm10, zmm26		;; apply fudges to R11					; 22-25		n 26
	vmovapd	zmm22, [srcreg+2*d4+d2+64]	;; R23
	vmulpd	zmm22{k2}, zmm22, zmm26		;; apply fudges to R23					; 23-26		n 32

	shr	r14, 16				;; Next 16 bits of fudge flags
	kmovw	k1, r14d			;; Load R12 and R24 fudge factor mask			; 23
	kshiftrw k2, k1, 8			;; R24's fudge						; 24
	vmovapd	zmm11, [srcreg+2*d4+d2+d1]	;; R12
	vmulpd	zmm11{k1}, zmm11, zmm26		;; apply fudges to R12					; 24-27		n 28
	vmovapd	zmm23, [srcreg+2*d4+d2+d1+64]	;; R24
	vmulpd	zmm23{k2}, zmm23, zmm26		;; apply fudges to R24					; 25-28		n 36

	vmulpd	zmm0, zmm0, [grpreg+0*64]	;; apply group multiplier to R1				; 25-28		n 29
	vmulpd	zmm6, zmm6, [grpreg+12*64]	;; apply group multiplier to R7				; 26-29		n 30
	vmulpd	zmm10, zmm10, [grpreg+20*64]	;; apply group multiplier to R11			; 26-29		n 32
	vmulpd	zmm8, zmm8, [grpreg+16*64]	;; apply group multiplier to R9				; 27-30		n 34
	vmulpd	zmm9, zmm9, [grpreg+18*64]	;; apply group multiplier to R10			; 27-30		n 38
	vmulpd	zmm7, zmm7, [grpreg+14*64]	;; apply group multiplier to R8				; 28-31		n 40
	vmulpd	zmm11, zmm11, [grpreg+22*64]	;; apply group multiplier to R12			; 28-31		n 36

	bump	maskreg, maskinc

	vmovapd zmm25, [grpreg+1*64]		;; group multiplier for R13
	zfmaddpd zmm24, zmm25, zmm12, zmm0	;; R1+R13*grpmult13					; 29-32		n 41
	zfnmaddpd zmm0, zmm25, zmm12, zmm0	;; R1-R13*grpmult13					; 29-32		n 60

	vmovapd zmm25, [grpreg+13*64]		;; group multiplier for R19
	zfmaddpd zmm12, zmm25, zmm18, zmm6	;; R7+R19*grpmult19					; 30-33		n 41
	zfnmaddpd zmm6, zmm25, zmm18, zmm6	;; R7-R19*grpmult19					; 30-33		n 61

	vmovapd zmm25, [grpreg+5*64]		;; group multiplier for R15 over R3
	zfmaddpd zmm18, zmm25, zmm14, zmm2	;; R3+R15*grpmult15_over_3				; 31-34		n 42
	zfnmaddpd zmm2, zmm25, zmm14, zmm2	;; R3-R15*grpmult15_over_3				; 31-34		n 43

	vmovapd zmm25, [grpreg+21*64]		;; group multiplier for R23
	zfmaddpd zmm14, zmm25, zmm22, zmm10	;; R11+R23*grpmult23					; 32-35		n 42
	zfnmaddpd zmm10, zmm25, zmm22, zmm10	;; R11-R23*grpmult23					; 32-35		n 43

	vmovapd zmm25, [grpreg+9*64]		;; group multiplier for R17 over R5
	zfmaddpd zmm22, zmm25, zmm16, zmm4	;; R5+R17*grpmult17_over_5				; 33-36		n 44
	zfnmaddpd zmm4, zmm25, zmm16, zmm4	;; R5-R17*grpmult17_over_5				; 33-36		n 45

	vmovapd zmm25, [grpreg+17*64]		;; group multiplier for R21
	zfmaddpd zmm16, zmm25, zmm20, zmm8	;; R9+R21*grpmult21					; 34-37		n 44
	zfnmaddpd zmm8, zmm25, zmm20, zmm8	;; R9-R21*grpmult21					; 34-37		n 45

	vmovapd zmm25, [grpreg+3*64]		;; group multiplier for R14 over R2
	zfmaddpd zmm20, zmm25, zmm13, zmm1	;; R2+R14*grpmult14_over_2				; 35-38		n 46
	zfnmaddpd zmm1, zmm25, zmm13, zmm1	;; R2-R14*grpmult14_over_2				; 35-38		n 47

	vmovapd zmm25, [grpreg+23*64]		;; group multiplier for R24
	zfmaddpd zmm13, zmm25, zmm23, zmm11	;; R12+R24*grpmult24					; 36-39		n 46
	zfnmaddpd zmm11, zmm25, zmm23, zmm11	;; R12-R24*grpmult24					; 36-39		n 47

	vmovapd zmm25, [grpreg+7*64]		;; group multiplier for R16 over R4
	zfmaddpd zmm23, zmm25, zmm15, zmm3	;; R4+R16*grpmult16_over_4				; 37-40		n 50
	zfnmaddpd zmm3, zmm25, zmm15, zmm3	;; R4-R16*grpmult16_over_4				; 37-40		n 54

	vmovapd zmm25, [grpreg+19*64]		;; group multiplier for R22
	zfmaddpd zmm15, zmm25, zmm21, zmm9	;; R10+R22*grpmult22					; 38-41		n 50
	zfnmaddpd zmm9, zmm25, zmm21, zmm9	;; R10-R22*grpmult22					; 38-41		n 54

	vmovapd zmm25, [grpreg+11*64]		;; group multiplier for R18 over R6
	zfmaddpd zmm21, zmm25, zmm17, zmm5	;; R6+R18*grpmult18_over_6				; 39-42		n 48
	zfnmaddpd zmm5, zmm25, zmm17, zmm5	;; R6-R18*grpmult18_over_6				; 39-42		n 49

	vmovapd zmm25, [grpreg+15*64]		;; group multiplier for R20
	zfmaddpd zmm17, zmm25, zmm19, zmm7	;; R8+R20*grpmult20					; 40-43		n 48
	zfnmaddpd zmm7, zmm25, zmm19, zmm7	;; R8-R20*grpmult20					; 40-43		n 49

	L1prefetchw srcreg+L1pd, L1pt
	vaddpd	zmm19, zmm24, zmm12		;; r1++ = (r1+r13) + (r7+r19)				; 41-44		n 58
	vsubpd	zmm24, zmm24, zmm12		;; r1+- = (r1+r13) - (r7+r19)				; 41-44		n 56

	vmovapd zmm25, [grpreg+4*64]		;; group multiplier for R3
	zfmaddpd zmm12, zmm18, zmm25, zmm14	;; r3++ = (r3+r15)*grpmult3 + (r11+r23)			; 42-45		n 51
	zfmsubpd zmm18, zmm18, zmm25, zmm14	;; r3+- = (r3+r15)*grpmult3 - (r11+r23)			; 42-45		n 55

	L1prefetchw srcreg+64+L1pd, L1pt
	zfmaddpd zmm14, zmm2, zmm25, zmm10	;; r3-+ = (r3-r15)*grpmult3 + (r11-r23)			; 43-46		n 61
	zfmsubpd zmm2, zmm2, zmm25, zmm10	;; r3-- = (r3-r15)*grpmult3 - (r11-r23)			; 43-46		n 76

	vmovapd zmm25, [grpreg+8*64]		;; group multiplier for R5
	zfmaddpd zmm10, zmm22, zmm25, zmm16	;; r5++ = (r5+r17)*grpmult5 + (r9+r21)			; 44-47		n 51
	zfmsubpd zmm22, zmm22, zmm25, zmm16	;; r5+- = (r5+r17)*grpmult5 - (r9+r21)			; 44-47		n 55

	L1prefetchw srcreg+d1+L1pd, L1pt
	zfmaddpd zmm16, zmm4, zmm25, zmm8	;; r5-+ = (r5-r17)*grpmult5 + (r9-r21)			; 45-48		n 77
	zfmsubpd zmm4, zmm4, zmm25, zmm8	;; r5-- = (r5-r17)*grpmult5 - (r9-r21)			; 45-48		n 60

	vmovapd zmm25, [grpreg+2*64]		;; group multiplier for R2
	zfmsubpd zmm8, zmm20, zmm25, zmm13	;; r2+- = (r2+r14)*grpmult2 - (r12+r24)			; 46-49		n 52
	zfmaddpd zmm20, zmm20, zmm25, zmm13	;; r2++ = (r2+r14)*grpmult2 + (r12+r24)			; 46-49		n 53

	L1prefetchw srcreg+d1+64+L1pd, L1pt
	zfmaddpd zmm13, zmm1, zmm25, zmm11	;; r2-+ = (r2-r14)*grpmult2 + (r12-r24)			; 47-50		n 71
	zfmsubpd zmm1, zmm1, zmm25, zmm11	;; r2-- = (r2-r14)*grpmult2 - (r12-r24)			; 47-50		n 69

	vmovapd zmm25, [grpreg+10*64]		;; group multiplier for R6
	zfmsubpd zmm11, zmm21, zmm25, zmm17	;; r6+- = (r6+r18)*grpmult6 - (r8+r20)			; 48-51		n 52
	zfmaddpd zmm21, zmm21, zmm25, zmm17	;; r6++ = (r6+r18)*grpmult6 + (r8+r20)			; 48-51		n 53

	L1prefetchw srcreg+d2+L1pd, L1pt
	zfmaddpd zmm17, zmm5, zmm25, zmm7	;; r6-+ = (r6-r18)*grpmult6 + (r8-r20)			; 49-52		n 80
	zfmsubpd zmm5, zmm5, zmm25, zmm7	;; r6-- = (r6-r18)*grpmult6 - (r8-r20)			; 49-52		n 78

	vmovapd zmm25, [grpreg+6*64]		;; group multiplier for R4
	zfmsubpd zmm7, zmm23, zmm25, zmm15	;; r4+- = (r4+r16)*grpmult4 - (r10+r22)			; 50-53		n 57
	zfmaddpd zmm23, zmm23, zmm25, zmm15	;; r4++ = (r4+r16)*grpmult4 + (r10+r22)			; 50-53		n 59

	bump	grpreg, grpinc

	L1prefetchw srcreg+d2+64+L1pd, L1pt
	vsubpd	zmm15, zmm12, zmm10		;; r3++- = (r3++) - (r5++)				; 51-54		n 56
	vaddpd	zmm12, zmm12, zmm10		;; r3+++ = (r3++) + (r5++)				; 51-54		n 58

	L1prefetchw srcreg+d2+d1+L1pd, L1pt
	vaddpd	zmm10, zmm8, zmm11		;; r2+-+ = (r2+-) + (r6+-)				; 52-55		n 57
	vsubpd	zmm8, zmm8, zmm11		;; r2+-- = (r2+-) - (r6+-)	(I59e / .866)		; 52-55		n 60

	L1prefetchw srcreg+d2+d1+64+L1pd, L1pt
	vaddpd	zmm11, zmm20, zmm21		;; r2+++ = (r2++) + (r6++)				; 53-56		n 59
	vsubpd	zmm20, zmm20, zmm21		;; r2++- = (r2++) - (r6++)	(R3Be / .866)		; 53-56		n 64

	L1prefetchw srcreg+d4+L1pd, L1pt
	zfmaddpd zmm21, zmm3, zmm25, zmm9	;; r4-+ = (r4-r16)*grpmult4 + (r10-r22)			; 54-57		n 71
	zfmsubpd zmm3, zmm3, zmm25, zmm9	;; r4-- = (r4-r16)*grpmult4 - (r10-r22)			; 54-57		n 69

	L1prefetchw srcreg+d4+64+L1pd, L1pt
	vaddpd	zmm9, zmm18, zmm22		;; r3+-+ = (r3+-) + (r5+-)	(I3Bo / .866)		; 55-58		n 65
	vsubpd	zmm18, zmm18, zmm22		;; r3+-- = (r3+-) - (r5+-)	(I59o / .866)		; 55-58		n 67

	L1prefetchw srcreg+d4+d1+L1pd, L1pt
	vsubpd	zmm22, zmm24, zmm15		;; R7 = (r1+-) - (r3++-)				; 56-59		n 62
	zfmaddpd zmm15, zmm15, zmm31, zmm24	;; R3Bo = (r1+-) + .500(r3++-)				; 56-59		n 64

	L1prefetchw srcreg+d4+d1+64+L1pd, L1pt
	vsubpd	zmm24, zmm10, zmm7		;; I7 = (r2+-+) - (r4+-)				; 57-60		n 62
	zfmaddpd zmm10, zmm10, zmm31, zmm7	;; I3Be = .500(r2+-+) + (r4+-)				; 57-60		n 65

	L1prefetchw srcreg+d4+d2+L1pd, L1pt
	vaddpd	zmm7, zmm19, zmm12		;; R1A = (r1++) + (r3+++)				; 58-61		n 63
	zfnmaddpd zmm12, zmm12, zmm31, zmm19	;; R59o = (r1++) - .500(r3+++)				; 58-61		n 66

	L1prefetchw srcreg+d4+d2+64+L1pd, L1pt
	vaddpd	zmm19, zmm11, zmm23		;; R1B = (r2+++) + (r4++)				; 59-62		n 63
	zfmsubpd zmm11, zmm11, zmm31, zmm23	;; R59e = .500*(r2+++) - (r4++)				; 59-62		n 66

	L1prefetchw srcreg+d4+d2+d1+L1pd, L1pt
	vmulpd	zmm8, zmm8, zmm30		;; I59e = I59e * .866					; 60-63		n 67

	L1prefetchw srcreg+d4+d2+d1+64+L1pd, L1pt
	vsubpd	zmm23, zmm0, zmm4		;; R4Ao = (r1-) - (r5--)				; 60-63		n 85
	zfmaddpd zmm4, zmm4, zmm31, zmm0	;; R2C68o = (r1-) + .500(r5--)				; 61-64		n 76

	L1prefetchw srcreg+2*d4+L1pd, L1pt
	vsubpd	zmm0, zmm6, zmm14		;; IA4o = (r7-) - (r3-+)				; 61-64		n 86
	zfmaddpd zmm14, zmm14, zmm31, zmm6	;; I2C68o = (r7-) + .500(r3-+)				; 62-65		n 77

	vmovapd zmm25, [screg+5*128+64]		;; cosine/sine for R7/I7 (w^6)
	zfmsubpd zmm6, zmm22, zmm25, zmm24	;; A7 = R7 * cosine/sine - I7				; 62-65		n 68
	zfmaddpd zmm24, zmm24, zmm25, zmm22	;; B7 = I7 * cosine/sine + R7				; 63-66		n 69

	L1prefetchw srcreg+2*d4+64+L1pd, L1pt
	vaddpd	zmm25, zmm7, zmm19		;; R1  = R1A + R1B					; 63-66
	vsubpd	zmm7, zmm7, zmm19		;; R13 = R1A - R1B					; 64-67

	L1prefetchw srcreg+2*d4+d1+L1pd, L1pt
	zfmaddpd zmm22, zmm20, zmm30, zmm15	;; R3 = R3Bo + .866*R3Be				; 64-67		n 72
	zfnmaddpd zmm20, zmm20, zmm30, zmm15	;; R11 = R3Bo - .866*R3Be				; 65-68		n 73

	L1prefetchw srcreg+2*d4+d1+64+L1pd, L1pt
	zfmaddpd zmm15, zmm9, zmm30, zmm10	;; I3 = I3Be + .866*I3Bo				; 65-68		n 72
	zfnmaddpd zmm9, zmm9, zmm30, zmm10	;; I11 = I3Be - .866*I3Bo				; 66-69		n 73

	L1prefetchw srcreg+2*d4+d2+L1pd, L1pt
	vaddpd	zmm10, zmm12, zmm11		;; R5 = R59o + R59e					; 66-69		n 74
	vsubpd	zmm12, zmm12, zmm11		;; R9 = R59o - R59e					; 67-70		n 75

	L1prefetchw srcreg+2*d4+d2+64+L1pd, L1pt
	zfmaddpd zmm11, zmm18, zmm30, zmm8	;; I5 = I59e + .866*I59o				; 67-70		n 74
	zfnmaddpd zmm18, zmm18, zmm30, zmm8	;; I9 = I59e - .866*I59o				; 68-71		n 75
	zstore	[srcreg], zmm25			;; Save R1						; 67

	vmovapd zmm25, [screg+5*128]		;; sine for R7/I7 (w^6)
	vmulpd	zmm6, zmm6, zmm25		;; A7 = A7 * sine (final R7)				; 68-71
	vmulpd	zmm24, zmm24, zmm25		;; B7 = B7 * sine (final I7)				; 69-72
	zstore	[srcreg+64], zmm7		;; Save R13						; 68

	L1prefetchw srcreg+2*d4+d2+d1+L1pd, L1pt
	vsubpd	zmm8, zmm1, zmm3		;; R4Ae = (r2--) - (r4--)				; 69-72		n 78
	zfmaddpd zmm7, zmm1, zmm27, zmm3	;; R2Ce = .966/.707(r2--) + (r4--)			; 70-73		n 79

	L1prefetchw srcreg+2*d4+d2+d1+64+L1pd, L1pt
	zfmsubpd zmm1, zmm1, zmm28, zmm3	;; R68e = .259/.707(r2--) - (r4--)			; 70-73		n 79
	vaddpd	zmm3, zmm13, zmm21		;; IA4e = (r2-+) + (r4-+)				; 71-74		n 80

	zfmaddpd zmm19, zmm13, zmm28, zmm21	;; I2Ce = .259/.707(r2-+) + (r4-+)			; 71-74		n 80
	zfmsubpd zmm13, zmm13, zmm27, zmm21	;; I68e = .966/.707(r2-+) - (r4-+)			; 72-75		n 81

	vmovapd zmm25, [screg+1*128+64]		;; cosine/sine for R3/I3 (w^2)
	zfmsubpd zmm21, zmm22, zmm25, zmm15	;; A3 = R3 * cosine/sine - I3				; 72-75		n 81
	zfmaddpd zmm15, zmm15, zmm25, zmm22	;; B3 = I3 * cosine/sine + R3				; 73-76		n 82
	zstore	[srcreg+d4+d2], zmm6		;; Save R7						; 72

	vmovapd zmm25, [screg+9*128+64]		;; cosine/sine for R11/I11 (w^10)
	zfmsubpd zmm22, zmm20, zmm25, zmm9	;; A11 = R11 * cosine/sine - I11			; 73-76		n 82
	zfmaddpd zmm9, zmm9, zmm25, zmm20	;; B11 = I11 * cosine/sine + R11			; 74-77		n 83
	zstore	[srcreg+d4+d2+64], zmm24	;; Save I7						; 73

	vmovapd zmm25, [screg+3*128+64]		;; cosine/sine for R5/I5 (w^4)
	zfmsubpd zmm20, zmm10, zmm25, zmm11	;; A5 = R5 * cosine/sine - I5				; 74-77		n 83
	zfmaddpd zmm11, zmm11, zmm25, zmm10	;; B5 = I5 * cosine/sine + R5				; 75-78		n 84

	vmovapd zmm25, [screg+7*128+64]		;; cosine/sine for R9/I9 (w^8)
	zfmsubpd zmm10, zmm12, zmm25, zmm18	;; A9 = R9 * cosine/sine - I9				; 75-78		n 84
	zfmaddpd zmm18, zmm18, zmm25, zmm12	;; B9 = I9 * cosine/sine + R9				; 76-79		n 85

	zfmaddpd zmm12, zmm2, zmm30, zmm4	;; R2Co = R2C68o + .866(r3--)				; 76-79		n 87
	zfnmaddpd zmm2, zmm2, zmm30, zmm4	;; R68o = R2C68o - .866(r3--)				; 77-80		n 89
	zfmaddpd zmm4, zmm16, zmm30, zmm14	;; I2Co = I2C68o + .866(r5-+)				; 77-80		n 88
	zfnmaddpd zmm16, zmm16, zmm30, zmm14	;; I68o = I2C68o - .866(r5-+)				; 78-81		n 90

	vsubpd	zmm8, zmm8, zmm5		;; R4Ae = R4Ae - (r6--)					; 78-81		n 85
	zfmaddpd zmm7, zmm5, zmm28, zmm7	;; R2Ce = R2Ce + .259/.707(r6--)			; 79-82		n 87
	zfmaddpd zmm1, zmm5, zmm27, zmm1	;; R68e = R68e + .966/.707(r6--)			; 79-82		n 89
	vsubpd	zmm3, zmm3, zmm17		;; IA4e = IA4e - (r6-+)					; 80-83		n 86
	zfmaddpd zmm19, zmm17, zmm27, zmm19	;; I2Ce = I2Ce + .966/.707(r6-+)			; 80-83		n 88
	zfmaddpd zmm13, zmm17, zmm28, zmm13	;; I68e = I68e + .259/.707(r6-+)			; 81-84		n 90

	vmovapd zmm25, [screg+1*128]		;; sine for R3/I3 (w^2)
	vmulpd	zmm21, zmm21, zmm25		;; A3 = A3 * sine (final R3)				; 81-84
	vmulpd	zmm15, zmm15, zmm25		;; B3 = B3 * sine (final I3)				; 82-85

	vmovapd zmm25, [screg+9*128]		;; sine for R11/I11 (w^10)
	vmulpd	zmm22, zmm22, zmm25		;; A11 = A11 * sine (final R11)				; 82-85
	vmulpd	zmm9, zmm9, zmm25		;; B11 = B11 * sine (final I11)				; 83-86

	vmovapd zmm25, [screg+3*128]		;; sine for R5/I5 (w^4)
	vmulpd	zmm20, zmm20, zmm25		;; A5 = A5 * sine (final R5)				; 83-86
	vmulpd	zmm11, zmm11, zmm25		;; B5 = B5 * sine (final I5)				; 84-87

	vmovapd zmm25, [screg+7*128]		;; sine for R9/I9 (w^8)
	vmulpd	zmm10, zmm10, zmm25		;; A9 = A9 * sine (final R9)				; 84-87
	vmulpd	zmm18, zmm18, zmm25		;; B9 = B9 * sine (final I9)				; 85-88

	zfmaddpd zmm17, zmm8, zmm29, zmm23	;; R4 = R4Ao + .707*R4Ae				; 85-88		n 91
	zfnmaddpd zmm8, zmm8, zmm29, zmm23	;; R10 = R4Ao - .707*R4Ae				; 86-89		n 92
	zstore	[srcreg+d2], zmm21		;; Save R3						; 85

	zfmsubpd zmm23, zmm3, zmm29, zmm0	;; I4 = .707*IA4e - IA4o				; 86-89		n 91
	zfmaddpd zmm3, zmm3, zmm29, zmm0	;; I10 = .707*IA4e + IA4o				; 87-90		n 92
	zstore	[srcreg+d2+64], zmm15		;; Save I3						; 86

	zfmaddpd zmm0, zmm7, zmm29, zmm12	;; R2 = R2Co + .707*R2Ce				; 87-90		n 93
	zfnmaddpd zmm7, zmm7, zmm29, zmm12	;; R12 = R2Co - .707*R2Ce				; 88-91		n 94
	zstore	[srcreg+2*d4+d2], zmm22		;; Save R11						; 86+1

	zfmaddpd zmm12, zmm19, zmm29, zmm4	;; I2 = .707*I2Ce + I2Co				; 88-91		n 93
	zfmsubpd zmm19, zmm19, zmm29, zmm4	;; I12 = .707*I2Ce - I2Co				; 89-92		n 94
	zstore	[srcreg+2*d4+d2+64], zmm9	;; Save I11						; 87+1

	zfmaddpd zmm4, zmm1, zmm29, zmm2	;; R6 = R68o + .707*R68e				; 89-92		n 95
	zfnmaddpd zmm1, zmm1, zmm29, zmm2	;; R8 = R68o - .707*R68e				; 90-93		n 96
	zstore	[srcreg+d4], zmm20		;; Save R5						; 87+2

	zfmaddpd zmm2, zmm13, zmm29, zmm16	;; I6 = .707*I68e + I68o				; 90-93		n 95
	zfmsubpd zmm13, zmm13, zmm29, zmm16	;; I8 = .707*I68e - I68o				; 91-94		n 96
	zstore	[srcreg+d4+64], zmm11		;; Save I5						; 88+2

	vmovapd zmm25, [screg+2*128+64]		;; cosine/sine for R4/I4 (w^3)
	zfmsubpd zmm16, zmm17, zmm25, zmm23	;; A4 = R4 * cosine/sine - I4				; 91-94		n 97
	zfmaddpd zmm23, zmm23, zmm25, zmm17	;; B4 = I4 * cosine/sine + R4				; 92-95		n 98
	zstore	[srcreg+2*d4], zmm10		;; Save R9						; 88+3

	vmovapd zmm25, [screg+8*128+64]		;; cosine/sine for R10/I10 (w^9)
	zfmsubpd zmm17, zmm8, zmm25, zmm3	;; A10 = R10 * cosine/sine - I10			; 92-95		n 98
	zfmaddpd zmm3, zmm3, zmm25, zmm8	;; B10 = I10 * cosine/sine + R10			; 93-96		n 99
	zstore	[srcreg+2*d4+64], zmm18		;; Save I9						; 89+3

	vmovapd zmm25, [screg+0*128+64]		;; cosine/sine for R2/I2 (w^1)
	zfmsubpd zmm8, zmm0, zmm25, zmm12	;; A2 = R2 * cosine/sine - I2				; 93-96		n 99
	zfmaddpd zmm12, zmm12, zmm25, zmm0	;; B2 = I2 * cosine/sine + R2				; 94-97		n 100

	vmovapd zmm25, [screg+10*128+64]	;; cosine/sine for R12/I12 (w^11)
	zfmsubpd zmm0, zmm7, zmm25, zmm19	;; A12 = R12 * cosine/sine - I12			; 94-97		n 100
	zfmaddpd zmm19, zmm19, zmm25, zmm7	;; B12 = I12 * cosine/sine + R12			; 95-98		n 101

	vmovapd zmm25, [screg+4*128+64]		;; cosine/sine for R6/I6 (w^5)
	zfmsubpd zmm7, zmm4, zmm25, zmm2	;; A6 = R6 * cosine/sine - I6				; 95-98		n 101
	zfmaddpd zmm2, zmm2, zmm25, zmm4	;; B6 = I6 * cosine/sine + R6				; 96-99		n 102

	vmovapd zmm25, [screg+6*128+64]		;; cosine/sine for R8/I8 (w^7)
	zfmsubpd zmm4, zmm1, zmm25, zmm13	;; A8 = R8 * cosine/sine - I8				; 96-99		n 102
	zfmaddpd zmm13, zmm13, zmm25, zmm1	;; B8 = I8 * cosine/sine + R8				; 97-100	n 103

	vmovapd zmm25, [screg+2*128]		;; sine for R4/I4 (w^3)
	vmulpd	zmm16, zmm16, zmm25		;; A4 = A4 * sine (final R4)				; 97-100
	vmulpd	zmm23, zmm23, zmm25		;; B4 = B4 * sine (final I4)				; 98-101

	vmovapd zmm25, [screg+8*128]		;; sine for R10/I10 (w^9)
	vmulpd	zmm17, zmm17, zmm25		;; A10 = A10 * sine (final R10)				; 98-101
	vmulpd	zmm3, zmm3, zmm25		;; B10 = B10 * sine (final I10)				; 99-102

	vmovapd zmm25, [screg+0*128]		;; sine for R2/I2 (w^1)
	vmulpd	zmm8, zmm8, zmm25		;; A2 = A2 * sine (final R2)				; 99-102
	vmulpd	zmm12, zmm12, zmm25		;; B2 = B2 * sine (final I2)				; 100-103

	vmovapd zmm25, [screg+10*128]		;; sine for R12/I12 (w^11)
	vmulpd	zmm0, zmm0, zmm25		;; A12 = A12 * sine (final R12)				; 100-103
	vmulpd	zmm19, zmm19, zmm25		;; B12 = B12 * sine (final I12)				; 101-104
	zstore	[srcreg+d2+d1], zmm16		;; Save R4						; 101

	vmovapd zmm25, [screg+4*128]		;; sine for R6/I6 (w^5)
	vmulpd	zmm7, zmm7, zmm25		;; A6 = A6 * sine (final R6)				; 101-104
	vmulpd	zmm2, zmm2, zmm25		;; B6 = B6 * sine (final I6)				; 102-105
	zstore	[srcreg+d2+d1+64], zmm23	;; Save I4						; 102

	vmovapd zmm25, [screg+6*128]		;; sine for R8/I8 (w^7)
	vmulpd	zmm4, zmm4, zmm25		;; A8 = A8 * sine (final R8)				; 102-105
	vmulpd	zmm13, zmm13, zmm25		;; B8 = B8 * sine (final I8)				; 103-106
	zstore	[srcreg+2*d4+d1], zmm17		;; Save R10						; 102+1

	bump	screg, scinc
	zstore	[srcreg+2*d4+d1+64], zmm3	;; Save I10						; 103+1
	zstore	[srcreg+d1], zmm8		;; Save R2						; 103+2
	zstore	[srcreg+d1+64], zmm12		;; Save I2						; 104+2
	zstore	[srcreg+2*d4+d2+d1], zmm0	;; Save R12						; 104+3
	zstore	[srcreg+2*d4+d2+d1+64], zmm19	;; Save I12						; 105+3
	zstore	[srcreg+d4+d1], zmm7		;; Save R6						; 105+4
	zstore	[srcreg+d4+d1+64], zmm2		;; Save I6						; 106+4
	zstore	[srcreg+d4+d2+d1], zmm4		;; Save R8						; 106+5
	zstore	[srcreg+d4+d2+d1+64], zmm13	;; Save I8						; 107+5
	bump	srcreg, srcinc
	ENDM

;;
;; ************************************* 24-reals-unfft variants ******************************************
;;

;; These macros produce 24 reals after doing 4.585 levels of the inverse FFT applying
;; the sin/cos multipliers beforehand.  The input is 2 real and 11 complex numbers.

;; To calculate a 24-reals inverse FFT, we calculate 24 real values from 24 complex inputs in a brute force way.
;; First we note that the 24 complex values are computed from the 11 complex and 2 real inputs using Hermetian symmetry, thusly:
;; c1 = r1A + 0*i
;; c2 = r2 + i2*i
;; ...
;; c12 = r12 + i12*i
;; c13 = r1B + 0*i
;; c14 = r12 - i12*i
;; ...
;; c24 = r2 - i2*i 
;;
;; The brute force calculations are:
;;
;; c1 + c2 + ... + c24	*  w^-0000000000...
;; c1 + c2 + ... + c24	*  w^-0123456789A...
;; c1 + c2 + ... + c24	*  w^-02468ACE....
;;    ...
;; c1 + c2 + ... + c24	*  w^-...A987654321
;;
;; The sin/cos values (w = 24th root of unity) are:
;; w^-1 = .966 - .259i
;; w^-2 = .866 - .5i
;; w^-3 = .707 - .707i
;; w^-4 = .5 - .866i
;; w^-5 = .259 - .966i
;; w^-6 = 0 - 1i
;; w^-7 = -.259 - .966i
;; w^-8 = -.5 - .866i
;; w^-9 = -.707 - .707i
;; w^-10 = -.866 - .5i
;; w^-11 = -.966 - .259i
;; w^-12 = -1

;;
;; Applying the sin/cos values above, taking advantage of symmetry, and ignoring a lot of multiplies by 2:
;; r1A     +(r2+r12)     +(r3+r11)     +(r4+r10)     +(r5+r9)     +(r6+r8) + r7 + r1B
;; r1A +.966(r2-r12) +.866(r3-r11) +.707(r4-r10) +.500(r5-r9) +.259(r6-r8)      - r1B +.259(i2+i12) +.500(i3+i11) +.707(i4+i10) +.866(i5+i9) +.966(i6+i8) + i7
;; r1A +.866(r2+r12) +.500(r3+r11)               -.500(r5+r9) -.866(r6+r8) - r7 + r1B +.500(i2-i12) +.866(i3-i11)     +(i4-i10) +.866(i5-i9) +.500(i6-i8)
;; r1A +.707(r2-r12)               -.707(r4-r10)     -(r5-r9) -.707(r6-r8)      - r1B +.707(i2+i12)     +(i3+i11) +.707(i4+i10)              -.707(i6+i8) - i7
;; r1A +.500(r2+r12) -.500(r3+r11)     -(r4+r10) -.500(r5+r9) +.500(r6+r8) + r7 + r1B +.866(i2-i12) +.866(i3-i11)               -.866(i5-i9) -.866(i6-i8)
;; r1A +.259(r2-r12) -.866(r3-r11) -.707(r4-r10) +.500(r5-r9) +.966(r6-r8)      - r1B +.966(i2+i12) +.500(i3+i11) -.707(i4+i10) -.866(i5+i9) +.259(i6+i8) + i7
;; r1A                   -(r3+r11)                   +(r5+r9)              - r7 + r1B     +(i2-i12)                   -(i4-i10)                  +(i6-i8)
;; r1A -.259(r2-r12) -.866(r3-r11) +.707(r4-r10) +.500(r5-r9) -.966(r6-r8)      - r1B +.966(i2+i12) -.500(i3+i11) -.707(i4+i10) +.866(i5+i9) +.259(i6+i8) - i7
;; r1A -.500(r2+r12) -.500(r3+r11)     +(r4+r10) -.500(r5+r9) -.500(r6+r8) + r7 + r1B +.866(i2-i12) -.866(i3-i11)               +.866(i5-i9) -.866(i6-i8)
;; r1A -.707(r2-r12)               +.707(r4-r10)     -(r5-r9) +.707(r6-r8)      - r1B +.707(i2+i12)     -(i3+i11) +.707(i4+i10)              -.707(i6+i8) + i7
;; r1A -.866(r2+r12) +.500(r3+r11)               -.500(r5+r9) +.866(r6+r8) - r7 + r1B +.500(i2-i12) -.866(i3-i11)     +(i4-i10) -.866(i5-i9) +.500(i6-i8)
;; r1A -.966(r2-r12) +.866(r3-r11) -.707(r4-r10) +.500(r5-r9) -.259(r6-r8)      - r1B +.259(i2+i12) -.500(i3+i11) +.707(i4+i10) -.866(i5+i9) +.966(i6+i8) - i7
;; r1A     -(r2+r12)     +(r3+r11)     -(r4+r10)     +(r5+r9)     -(r6+r8) + r7 + r1B
;; ... r14 thru r24 are the same as r12 through r2 but with the sign of the imaginary component changed.
;;
;; Also remember that due to the funny way we do things, input #1 is r1A+r1B and input #13 is r1A-r1B

;; Simplifying yields:
;; R1 = (r1A+r1B)+r7 +(r4+r10)     +(((r3+r11)+(r5+r9))+((r2+r12)+(r6+r8)))  
;; R9 = (r1A+r1B)+r7 +(r4+r10) -.500(((r3+r11)+(r5+r9))+((r2+r12)+(r6+r8))) -.866(((i3-i11)-(i5-i9))-((i2-i12)-(i6-i8)))
;; R17= (r1A+r1B)+r7 +(r4+r10) -.500(((r3+r11)+(r5+r9))+((r2+r12)+(r6+r8))) +.866(((i3-i11)-(i5-i9))-((i2-i12)-(i6-i8)))
;; R13= (r1A+r1B)+r7 -(r4+r10)     +(((r3+r11)+(r5+r9))-((r2+r12)+(r6+r8)))  
;; R5 = (r1A+r1B)+r7 -(r4+r10) -.500(((r3+r11)+(r5+r9))-((r2+r12)+(r6+r8))) +.866(((i3-i11)-(i5-i9))+((i2-i12)-(i6-i8)))
;; R21= (r1A+r1B)+r7 -(r4+r10) -.500(((r3+r11)+(r5+r9))-((r2+r12)+(r6+r8))) -.866(((i3-i11)-(i5-i9))+((i2-i12)-(i6-i8)))

;; R19= (r1A+r1B)-r7 +(i4-i10)     -(((r3+r11)-(r5+r9))+((i2-i12)+(i6-i8)))
;; R3 = (r1A+r1B)-r7 +(i4-i10) +.500(((r3+r11)-(r5+r9))+((i2-i12)+(i6-i8))) +.866(((i3-i11)+(i5-i9))+((r2+r12)-(r6+r8)))
;; R11= (r1A+r1B)-r7 +(i4-i10) +.500(((r3+r11)-(r5+r9))+((i2-i12)+(i6-i8))) -.866(((i3-i11)+(i5-i9))+((r2+r12)-(r6+r8)))
;; R7 = (r1A+r1B)-r7 -(i4-i10)     -(((r3+r11)-(r5+r9))-((i2-i12)+(i6-i8)))
;; R15= (r1A+r1B)-r7 -(i4-i10) +.500(((r3+r11)-(r5+r9))-((i2-i12)+(i6-i8))) +.866(((i3-i11)+(i5-i9))-((r2+r12)-(r6+r8)))
;; R23= (r1A+r1B)-r7 -(i4-i10) +.500(((r3+r11)-(r5+r9))-((i2-i12)+(i6-i8))) -.866(((i3-i11)+(i5-i9))-((r2+r12)-(r6+r8)))

;; R4 = (r1A-r1B)-i7     +((i3+i11)-(r5-r9))                           -.707((r4-r10)-(i4+i10)) +.707((i2+i12)-(r6-r8)) +.707((r2-r12)-(i6+i8))
;; R16= (r1A-r1B)-i7     +((i3+i11)-(r5-r9))                           +.707((r4-r10)-(i4+i10)) -.707((i2+i12)-(r6-r8)) -.707((r2-r12)-(i6+i8))
;; R12= (r1A-r1B)-i7 -.500((i3+i11)-(r5-r9)) +.866((r3-r11)-(i5+i9))   -.707((r4-r10)-(i4+i10)) +.259((i2+i12)-(r6-r8)) -.966((r2-r12)-(i6+i8))
;; R24= (r1A-r1B)-i7 -.500((i3+i11)-(r5-r9)) +.866((r3-r11)-(i5+i9))   +.707((r4-r10)-(i4+i10)) -.259((i2+i12)-(r6-r8)) +.966((r2-r12)-(i6+i8))
;; R8 = (r1A-r1B)-i7 -.500((i3+i11)-(r5-r9)) -.866((r3-r11)-(i5+i9))   +.707((r4-r10)-(i4+i10)) +.966((i2+i12)-(r6-r8)) -.259((r2-r12)-(i6+i8))
;; R20= (r1A-r1B)-i7 -.500((i3+i11)-(r5-r9)) -.866((r3-r11)-(i5+i9))   -.707((r4-r10)-(i4+i10)) -.966((i2+i12)-(r6-r8)) +.259((r2-r12)-(i6+i8))

;; R10= (r1A-r1B)+i7     -((i3+i11)+(r5-r9))                           +.707((r4-r10)+(i4+i10)) +.707((i2+i12)+(r6-r8)) -.707((r2-r12)+(i6+i8))
;; R22= (r1A-r1B)+i7     -((i3+i11)+(r5-r9))                           -.707((r4-r10)+(i4+i10)) -.707((i2+i12)+(r6-r8)) +.707((r2-r12)+(i6+i8))
;; R6 = (r1A-r1B)+i7 +.500((i3+i11)+(r5-r9)) -.866((r3-r11)+(i5+i9))   -.707((r4-r10)+(i4+i10)) +.966((i2+i12)+(r6-r8)) +.259((r2-r12)+(i6+i8))
;; R18= (r1A-r1B)+i7 +.500((i3+i11)+(r5-r9)) -.866((r3-r11)+(i5+i9))   +.707((r4-r10)+(i4+i10)) -.966((i2+i12)+(r6-r8)) -.259((r2-r12)+(i6+i8))
;; R2 = (r1A-r1B)+i7 +.500((i3+i11)+(r5-r9)) +.866((r3-r11)+(i5+i9))   +.707((r4-r10)+(i4+i10)) +.259((i2+i12)+(r6-r8)) +.966((r2-r12)+(i6+i8))
;; R14= (r1A-r1B)+i7 +.500((i3+i11)+(r5-r9)) +.866((r3-r11)+(i5+i9))   -.707((r4-r10)+(i4+i10)) -.259((i2+i12)+(r6-r8)) -.966((r2-r12)+(i6+i8))

;; Uses two sin/cos ptrs
zr12_2sc_twentyfour_reals_unfft_preload MACRO
	zr12_24r_unfft_cmn_preload
	ENDM
zr12_2sc_twentyfour_reals_unfft MACRO srcreg,srcinc,d1,d2,d4,screg1,scinc1,screg2,scinc2,maxrpt,L1pt,L1pd
	zr12_24r_unfft_cmn srcreg,srcinc,d1,d2,d4,screg1,scinc1,screg2,scinc2,maxrpt,L1pt,L1pd
	ENDM

;; Combined sin/cos data
zr12_csc_twentyfour_reals_unfft_preload MACRO
	zr12_24r_unfft_cmn_preload
	ENDM
zr12_csc_twentyfour_reals_unfft MACRO srcreg,srcinc,d1,d2,d4,screg,scinc,maxrpt,L1pt,L1pd
	zr12_24r_unfft_cmn srcreg,srcinc,d1,d2,d4,screg+6*128,0,screg,scinc,maxrpt,L1pt,L1pd
	ENDM


zr12_24r_unfft_cmn_preload MACRO
	vbroadcastsd zmm31, ZMM_HALF
	vbroadcastsd zmm30, ZMM_P866
	vbroadcastsd zmm29, ZMM_SQRTHALF
	vbroadcastsd zmm28, ZMM_P259_P707
	vbroadcastsd zmm27, ZMM_P966_P707
	ENDM
zr12_24r_unfft_cmn MACRO srcreg,srcinc,d1,d2,d4,screg1,scinc1,screg2,scinc2,maxrpt,L1pt,L1pd
	vmovapd zmm26, [screg2+0*128+64]	;; cosine/sine for R3/I3 (w^2 = complex w^1)
	vmovapd	zmm2, [srcreg+d2]		;; R3
	vmovapd	zmm14, [srcreg+d2+64]		;; I3
	zfmaddpd zmm24, zmm2, zmm26, zmm14	;; A3 = R3 * cosine/sine + I3				; 1-4
	zfmsubpd zmm14, zmm14, zmm26, zmm2	;; B3 = I3 * cosine/sine - R3				; 1-4

	vmovapd zmm26, [screg2+1*128+64]	;; cosine/sine for R5/I5 (w^4 = complex w^2)
	vmovapd	zmm4, [srcreg+d4]		;; R5
	vmovapd	zmm16, [srcreg+d4+64]		;; I5
	zfmaddpd zmm2, zmm4, zmm26, zmm16	;; A5 = R5 * cosine/sine + I5				; 2-5
	zfmsubpd zmm16, zmm16, zmm26, zmm4	;; B5 = I5 * cosine/sine - R5				; 2-5

	vmovapd zmm26, [screg1+0*128+64]	;; cosine/sine for R2/I2 (w^1)
	vmovapd	zmm1, [srcreg+d1]		;; R2
	vmovapd	zmm13, [srcreg+d1+64]		;; I2
	zfmaddpd zmm4, zmm1, zmm26, zmm13	;; A2 = R2 * cosine/sine + I2				; 3-6
	zfmsubpd zmm13, zmm13, zmm26, zmm1	;; B2 = I2 * cosine/sine - R2				; 3-6

	vmovapd zmm26, [screg1+2*128+64]	;; cosine/sine for R6/I6 (w^5)
	vmovapd	zmm5, [srcreg+d4+d1]		;; R6
	vmovapd	zmm17, [srcreg+d4+d1+64]	;; I6
	zfmaddpd zmm1, zmm5, zmm26, zmm17	;; A6 = R6 * cosine/sine + I6				; 4-7
	zfmsubpd zmm17, zmm17, zmm26, zmm5	;; B6 = I6 * cosine/sine - R6				; 4-7

	vmovapd zmm26, [screg1+1*128+64]	;; cosine/sine for R4/I4 (w^3)
	vmovapd	zmm3, [srcreg+d2+d1]		;; R4
	vmovapd	zmm15, [srcreg+d2+d1+64]	;; I4
	zfmaddpd zmm5, zmm3, zmm26, zmm15	;; A4 = R4 * cosine/sine + I4				; 5-8
	zfmsubpd zmm15, zmm15, zmm26, zmm3	;; B4 = I4 * cosine/sine - R4				; 5-8

	vmovapd zmm26, [screg2+4*128+64]	;; cosine/sine for R11/I11 (w^10 = complex w^5)
	vmovapd	zmm10, [srcreg+2*d4+d2]		;; R11
	vmovapd	zmm22, [srcreg+2*d4+d2+64]	;; I11
	zfmaddpd zmm3, zmm10, zmm26, zmm22	;; A11 = R11 * cosine/sine + I11 (new R11/sine)		; 6-9
	zfmsubpd zmm22, zmm22, zmm26, zmm10	;; B11 = I11 * cosine/sine - R11 (new I11/sine)		; 6-9

	vmovapd zmm26, [screg2+0*128]		;; sine for R3/I3 (w^2 = complex w^1)
	vmulpd	zmm24, zmm24, zmm26		;; A3 = A3 * sine (new R3)				; 7-10
	vmulpd	zmm14, zmm14, zmm26		;; B3 = B3 * sine (new I3)				; 7-10

	vmovapd zmm26, [screg2+3*128+64]	;; cosine/sine for R9/I9 (w^8 = complex w^4)
	vmovapd	zmm8, [srcreg+2*d4]		;; R9
	vmovapd	zmm20, [srcreg+2*d4+64]		;; I9
	zfmaddpd zmm10, zmm8, zmm26, zmm20	;; A9 = R9 * cosine/sine + I9 (new R9/sine)		; 8-11
	zfmsubpd zmm20, zmm20, zmm26, zmm8	;; B9 = I9 * cosine/sine - R9 (new I9/sine)		; 8-11

	vmovapd zmm26, [screg2+1*128]		;; sine for R5/I5 (w^4 = complex w^2)
	vmulpd	zmm2, zmm2, zmm26		;; A5 = A5 * sine (new R5)				; 9-12
	vmulpd	zmm16, zmm16, zmm26		;; B5 = B5 * sine (new I5)				; 9-12

	vmovapd zmm26, [screg1+5*128+64]	;; cosine/sine for R12/I12 (w^11)
	vmovapd	zmm11, [srcreg+2*d4+d2+d1]	;; R12
	vmovapd	zmm23, [srcreg+2*d4+d2+d1+64]	;; I12
	zfmaddpd zmm8, zmm11, zmm26, zmm23	;; A12 = R12 * cosine/sine + I12 (new R12/sine)		; 10-13
	zfmsubpd zmm23, zmm23, zmm26, zmm11	;; B12 = I12 * cosine/sine - R12 (new I12/sine)		; 10-13

	vmovapd zmm26, [screg1+0*128]		;; sine for R2/I2 (w^1)
	vmulpd	zmm4, zmm4, zmm26		;; A2 = A2 * sine (new R2)				; 11-14
	vmulpd	zmm13, zmm13, zmm26		;; B2 = B2 * sine (new I2)				; 11-14

	vmovapd zmm26, [screg1+3*128+64]	;; cosine/sine for R8/I8 (w^7)
	vmovapd	zmm7, [srcreg+d4+d2+d1]		;; R8
	vmovapd	zmm19, [srcreg+d4+d2+d1+64]	;; I8
	zfmaddpd zmm11, zmm7, zmm26, zmm19	;; A8 = R8 * cosine/sine + I8 (new R8/sine)		; 12-15
	zfmsubpd zmm19, zmm19, zmm26, zmm7	;; B8 = I8 * cosine/sine - R8 (new I8/sine)		; 12-15

	vmovapd zmm26, [screg1+2*128]		;; sine for R6/I6 (w^5)
	vmulpd	zmm1, zmm1, zmm26		;; A6 = A6 * sine (new R6)				; 13-16
	vmulpd	zmm17, zmm17, zmm26		;; B6 = B6 * sine (new I6)				; 13-16

	vmovapd zmm26, [screg2+2*128+64]	;; cosine/sine for R7/I7 (w^6 = complex w^3)
	vmovapd	zmm6, [srcreg+d4+d2]		;; R7
	vmovapd	zmm18, [srcreg+d4+d2+64]	;; I7
	zfmaddpd zmm7, zmm6, zmm26, zmm18	;; A7 = R7 * cosine/sine + I7 (new R7/sine)		; 14-17
	zfmsubpd zmm18, zmm18, zmm26, zmm6	;; B7 = I7 * cosine/sine - R7 (new I7/sine)		; 14-17

	vmovapd zmm26, [screg1+4*128+64]	;; cosine/sine for R10/I10 (w^9)
	vmovapd	zmm9, [srcreg+2*d4+d1]		;; R10
	vmovapd	zmm21, [srcreg+2*d4+d1+64]	;; I10
	zfmaddpd zmm6, zmm9, zmm26, zmm21	;; A10 = R10 * cosine/sine + I10 (new R10/sine)		; 15-18
	zfmsubpd zmm21, zmm21, zmm26, zmm9	;; B10 = I10 * cosine/sine - R10 (new I10/sine)		; 15-18

	vmovapd zmm26, [screg1+1*128]		;; sine for R4/I4 (w^3)
	vmulpd	zmm5, zmm5, zmm26		;; A4 = A4 * sine (new R4)				; 16-19
	vmulpd	zmm15, zmm15, zmm26		;; B4 = B4 * sine (new I4)				; 16-19

	vmovapd zmm26, [screg2+4*128]		;; sine for R11/I11 (w^10 = complex w^5)
	zfmaddpd zmm9, zmm3, zmm26, zmm24	;; r3+ = R3+R11*sine					; 17-20
	zfnmaddpd zmm3, zmm3, zmm26, zmm24	;; r3- = R3-R11*sine					; 17-20
	zfmaddpd zmm24, zmm22, zmm26, zmm14	;; i3+ = I3+I11*sine					; 18-21
	zfnmaddpd zmm22, zmm22, zmm26, zmm14	;; i3- = I3-I11*sine					; 18-21
	vmovapd zmm26, [screg2+3*128]		;; sine for R9/I9 (w^8 = complex w^4)
	zfmaddpd zmm14, zmm10, zmm26, zmm2	;; r5+ = R5+R9*sine					; 19-22
	zfnmaddpd zmm10, zmm10, zmm26, zmm2	;; r5- = R5-R9*sine					; 19-22
	zfmaddpd zmm2, zmm20, zmm26, zmm16	;; i5+ = I5+I9*sine					; 20-23
	zfnmaddpd zmm20, zmm20, zmm26, zmm16	;; i5- = I5-I9*sine					; 20-23
	vmovapd zmm26, [screg1+5*128]		;; sine for R12/I12 (w^11)
	zfmaddpd zmm16, zmm8, zmm26, zmm4	;; r2+ = R2+R12*sine					; 21-24
	zfnmaddpd zmm8, zmm8, zmm26, zmm4	;; r2- = R2-R12*sine					; 21-24
	zfmaddpd zmm4, zmm23, zmm26, zmm13	;; i2+ = I2+I12*sine					; 22-25
	zfnmaddpd zmm23, zmm23, zmm26, zmm13	;; i2- = I2-I12*sine					; 22-25
	vmovapd zmm26, [screg1+3*128]		;; sine for R8/I8 (w^7)
	zfmaddpd zmm13, zmm11, zmm26, zmm1	;; r6+ = R6+R8*sine					; 23-26
	zfnmaddpd zmm11, zmm11, zmm26, zmm1	;; r6- = R6-R8*sine					; 23-26
	zfmaddpd zmm1, zmm19, zmm26, zmm17	;; i6+ = I6+I8*sine					; 24-27
	zfnmaddpd zmm19, zmm19, zmm26, zmm17	;; i6- = I6-I8*sine					; 24-27
	vmovapd zmm26, [screg2+2*128]		;; sine for R7/I7 (w^6 = complex w^3)
	vmovapd	zmm0, [srcreg]			;; r1A+r1B
	zfmaddpd zmm17, zmm7, zmm26, zmm0	;; r1++ = (r1A+r1B)+R7*sine				; 25-28
	zfnmaddpd zmm7, zmm7, zmm26, zmm0	;; r1+- = (r1A+r1B)-R7*sine				; 25-28
	vmovapd	zmm12, [srcreg+64]		;; r1A-r1B
	zfmaddpd zmm0, zmm18, zmm26, zmm12	;; r1-+ = (r1A-r1B)+I7*sine				; 26-29
	zfnmaddpd zmm18, zmm18, zmm26, zmm12	;; r1-- = (r1A-r1B)-I7*sine				; 26-29
	vmovapd zmm26, [screg1+4*128]		;; sine for R10/I10 (w^9)
	zfmaddpd zmm12, zmm6, zmm26, zmm5	;; r4+ = R4+R10*sine					; 27-30
	zfnmaddpd zmm6, zmm6, zmm26, zmm5	;; r4- = R4-R10*sine					; 27-30
	zfmaddpd zmm5, zmm21, zmm26, zmm15	;; i4+ = I4+I10*sine					; 28-31
	zfnmaddpd zmm21, zmm21, zmm26, zmm15	;; i4- = I4-I10*sine					; 28-31

	vaddpd	zmm15, zmm9, zmm14		;; r3++ = (r3+r11) + (r5+r9)				; 29-32
	vsubpd	zmm9, zmm9, zmm14		;; r3+- = (r3+r11) - (r5+r9)				; 29-32

	vaddpd	zmm14, zmm16, zmm13		;; r2++ = (r2+r12) + (r6+r8)				; 30-33
	vsubpd	zmm16, zmm16, zmm13		;; r2+- = (r2+r12) - (r6+r8)				; 30-33

	L1prefetchw srcreg+L1pd, L1pt
	vaddpd	zmm13, zmm23, zmm19		;; i2-+ = (i2-i12) + (i6-i8)				; 31-34
	vsubpd	zmm23, zmm23, zmm19		;; i2-- = (i2-i12) - (i6-i8)				; 31-34

	L1prefetchw srcreg+64+L1pd, L1pt
	vaddpd	zmm19, zmm22, zmm20		;; i3-+ = (i3-i11) + (i5-i9)				; 32-35
	vsubpd	zmm22, zmm22, zmm20		;; i3-- = (i3-i11) - (i5-i9)				; 32-35

	L1prefetchw srcreg+d1+L1pd, L1pt
	vaddpd	zmm20, zmm17, zmm12		;; r1+++ = (r1++) + (r4+r10)				; 33-36
	vsubpd	zmm17, zmm17, zmm12		;; r1++- = (r1++) - (r4+r10)				; 33-36

	L1prefetchw srcreg+d1+64+L1pd, L1pt
	vaddpd	zmm12, zmm7, zmm21		;; r1+-+ = (r1+-) + (i4-i10)				; 34-37
	vsubpd	zmm7, zmm7, zmm21		;; r1+-- = (r1+-) - (i4+i10)				; 34-37

	L1prefetchw srcreg+d2+L1pd, L1pt
	vaddpd	zmm21, zmm15, zmm14		;; r3+++ = (r3++) + (r2++)				; 35-38
	vsubpd	zmm15, zmm15, zmm14		;; r3++- = (r3++) - (r2++)				; 35-38

	L1prefetchw srcreg+d2+64+L1pd, L1pt
	vaddpd	zmm14, zmm9, zmm13		;; r3+-+ = (r3+-) + (i2-+)				; 36-39
	vsubpd	zmm9, zmm9, zmm13		;; r3+-- = (r3+-) - (i2-+)				; 36-39

	L1prefetchw srcreg+d2+d1+L1pd, L1pt
	vaddpd	zmm13, zmm22, zmm23		;; i3--+ = (i3--) + (i2--)				; 37-40
	vsubpd	zmm22, zmm22, zmm23		;; i3--- = (i3--) - (i2--)				; 37-40

	L1prefetchw srcreg+d2+d1+64+L1pd, L1pt
	vaddpd	zmm23, zmm19, zmm16		;; i3-++ = (i3-+) + (r2+-)				; 38-41
	vsubpd	zmm19, zmm19, zmm16		;; i3-+- = (i3-+) - (r2+-)				; 38-41

	L1prefetchw srcreg+d4+L1pd, L1pt
	vaddpd	zmm16, zmm4, zmm11		;; i2++ = (i2+i12) + (r6-r8)				; 39-42
	vsubpd	zmm4, zmm4, zmm11		;; i2+- = (i2+i12) - (r6-r8)				; 39-42

	L1prefetchw srcreg+d4+64+L1pd, L1pt
	vaddpd	zmm11, zmm6, zmm5		;; r4-+ = (r4-r10) + (i4+i10)				; 40-43
	vsubpd	zmm6, zmm6, zmm5		;; r4-- = (r4-r10) - (i4+i10)				; 40-43

	L1prefetchw srcreg+d4+d1+L1pd, L1pt
	vaddpd	zmm5, zmm24, zmm10		;; i3++ = (i3+i11) + (r5-r9)				; 41-44
	vsubpd	zmm24, zmm24, zmm10		;; i3+- = (i3+i11) - (r5-r9)				; 41-44

	L1prefetchw srcreg+d4+d1+64+L1pd, L1pt
	vaddpd	zmm10, zmm8, zmm1		;; r2-+ = (r2-r12) + (i6+i8)				; 42-45
	vsubpd	zmm8, zmm8, zmm1		;; r2-- = (r2-r12) - (i6+i8)				; 42-45

	L1prefetchw srcreg+d4+d2+L1pd, L1pt
	vaddpd	zmm1, zmm3, zmm2		;; r3-+ = (r3-r11) + (i5+i9)				; 43-46
	vsubpd	zmm3, zmm3, zmm2		;; r3-- = (r3-r11) - (i5+i9)				; 43-46

	L1prefetchw srcreg+d4+d2+64+L1pd, L1pt
	vaddpd	zmm2, zmm20, zmm21		;; R1   = (r1+++) + (r3+++)				; 44-47
	zfnmaddpd zmm21, zmm21, zmm31, zmm20	;; R9_17= (r1+++) - .500(r3+++)				; 44-47

	L1prefetchw srcreg+d4+d2+d1+L1pd, L1pt
	vaddpd	zmm20, zmm17, zmm15		;; R13  = (r1++-) + (r3++-)				; 45-48
	zfnmaddpd zmm15, zmm15, zmm31, zmm17	;; R5_21= (r1++-) - .500(r3++-)				; 45-48

	L1prefetchw srcreg+d4+d2+d1+64+L1pd, L1pt
	vsubpd	zmm17, zmm12, zmm14		;; R19   = (r1+-+) - (r3+-+)				; 46-49
	zfmaddpd zmm14, zmm14, zmm31, zmm12	;; R3_11 = (r1+-+) + .500(r3+-+)			; 46-49

	L1prefetchw srcreg+2*d4+L1pd, L1pt
	vsubpd	zmm12, zmm7, zmm9		;; R7    = (r1+--) - (r3+--)				; 47-50
	zfmaddpd zmm9, zmm9, zmm31, zmm7	;; R15_23= (r1+--) + .500(r3+--)			; 47-50

	L1prefetchw srcreg+2*d4+64+L1pd, L1pt
	zfnmaddpd zmm7, zmm22, zmm30, zmm21	;; R9 = R9_17 - .866(i3---)				; 48-51
	zfmaddpd zmm22, zmm22, zmm30, zmm21	;; R17= R9_17 + .866(i3---)				; 48-51
  	zstore	[srcreg], zmm2			;; Save R1						; 48

	L1prefetchw srcreg+2*d4+d1+L1pd, L1pt
	zfmaddpd zmm21, zmm13, zmm30, zmm15	;; R5 = R5_21 + .866(i3--+)				; 49-52
	zfnmaddpd zmm13, zmm13, zmm30, zmm15	;; R21= R5_21 - .866(i3--+)				; 49-52
	zstore	[srcreg+64], zmm20		;; Save R13						; 49

	L1prefetchw srcreg+2*d4+d1+64+L1pd, L1pt
	zfmaddpd zmm15, zmm23, zmm30, zmm14	;; R3 = R3_11 + .866(i3-++)				; 50-53
	zfnmaddpd zmm23, zmm23, zmm30, zmm14	;; R11= R3_11 - .866(i3-++)				; 50-53
	zstore	[srcreg+d4+d2+64], zmm17	;; Save R19						; 50

	L1prefetchw srcreg+2*d4+d2+L1pd, L1pt
	zfmaddpd zmm14, zmm19, zmm30, zmm9	;; R15= R15_23 + .866(i3-+-)				; 51-54
	zfnmaddpd zmm19, zmm19, zmm30, zmm9	;; R23= R15_23 - .866(i3-+-)				; 51-54
	zstore	[srcreg+d4+d2], zmm12		;; Save R7						; 51

	L1prefetchw srcreg+2*d4+d2+64+L1pd, L1pt
	vsubpd	zmm9, zmm6, zmm4		;; R4_16e = (r4--) - (i2+-)				; 52-55
	zfnmaddpd zmm2, zmm4, zmm28, zmm6	;; R12_24e= (r4--) - .259/.707(i2+-)			; 52-55
	zstore	[srcreg+2*d4], zmm7		;; Save R9						; 52

	L1prefetchw srcreg+2*d4+d2+d1+L1pd, L1pt
	zfmaddpd zmm4, zmm4, zmm27, zmm6	;; R8_20e = (r4--) + .966/.707(i2+-)			; 53-56
	vaddpd	zmm6, zmm18, zmm24		;; R4_16o      = (r1--) + (i3+-)			; 53-56
	zstore	[srcreg+d4+64], zmm22		;; Save R17						; 52+1

	L1prefetchw srcreg+2*d4+d2+d1+64+L1pd, L1pt
	zfnmaddpd zmm24, zmm24, zmm31, zmm18	;; R12_24_8_20o= (r1--) - .500(i3+-)			; 54-57
	vaddpd	zmm18, zmm11, zmm16		;; R10_22e= (r4-+) + (i2++)				; 54-57
	zstore	[srcreg+d4], zmm21		;; Save R5						; 53+1

	zfnmaddpd zmm20, zmm16, zmm27, zmm11	;; R6_18e = (r4-+) - .966/.707(i2++)			; 55-58
	zfmaddpd zmm16, zmm16, zmm28, zmm11	;; R2_14e = (r4-+) + .259/.707(i2++)			; 55-58
	zstore	[srcreg+2*d4+64], zmm13		;; Save R21						; 53+2

	vsubpd	zmm11, zmm0, zmm5		;; R10_22o     = (r1-+) - (i3++)			; 56-59
	zfmaddpd zmm5, zmm5, zmm31, zmm0	;; R6_18_2_14o = (r1-+) + .500(i3++)			; 56-59
	zstore	[srcreg+d2], zmm15		;; Save R3						; 54+2

	vsubpd	zmm9, zmm9, zmm8		;; R4_16e = R4_16e - (r2--)				; 57-60
	zfmaddpd zmm2, zmm8, zmm27, zmm2	;; R12_24e= R12_24e + .966/.707(r2--)			; 57-60
	zstore	[srcreg+2*d4+d2], zmm23		;; Save R11						; 54+3

	zfnmaddpd zmm4, zmm8, zmm28, zmm4	;; R8_20e = R8_20e - .259/.707(r2--)			; 58-61
	zfmaddpd zmm0, zmm3, zmm30, zmm24	;; R12_24o= R12_24_8_20o + .866(r3--)			; 58-61
	zstore	[srcreg+d2+64], zmm14		;; Save R15						; 55+3

	zfnmaddpd zmm3, zmm3, zmm30, zmm24	;; R8_20o = R12_24_8_20o - .866(r3--)			; 59-62
	vsubpd	zmm18, zmm18, zmm10		;; R10_22e= R10_22e - (r2-+)				; 59-62
	zstore	[srcreg+2*d4+d2+64], zmm19	;; Save R23						; 55+4

	zfnmaddpd zmm20, zmm10, zmm28, zmm20	;; R6_18e = R6_18e - .259/.707(r2-+)			; 60-63
	zfmaddpd zmm16, zmm10, zmm27, zmm16	;; R2_14e = R2_14e + .966/.707(r2-+)			; 60-63
	bump	screg1, scinc1

	zfnmaddpd zmm24, zmm1, zmm30, zmm5	;; R6_18o = R6_18_2_14o - .866(r3-+)			; 61-64
	zfmaddpd zmm1, zmm1, zmm30, zmm5	;; R2_14o = R6_18_2_14o + .866(r3-+)			; 61-64
	bump	screg2, scinc2

	zfnmaddpd zmm5, zmm9, zmm29, zmm6	;; R4 = R4_16o - .707*R4_16e				; 62-65
	zfmaddpd zmm9, zmm9, zmm29, zmm6	;; R16= R4_16o + .707*R4_16e				; 62-65
	zfnmaddpd zmm6, zmm2, zmm29, zmm0	;; R12= R12_24o - .707*R12_24e				; 63-66
	zfmaddpd zmm2, zmm2, zmm29, zmm0	;; R24= R12_24o + .707*R12_24e				; 63-66
	zfmaddpd zmm0, zmm4, zmm29, zmm3	;; R8 = R8_20o + .707*R8_20e				; 64-67
	zfnmaddpd zmm4, zmm4, zmm29, zmm3	;; R20= R8_20o - .707*R8_20e				; 64-67
	zfmaddpd zmm3, zmm18, zmm29, zmm11	;; R10= R10_22o + .707*R10_22e				; 65-68
	zfnmaddpd zmm18, zmm18, zmm29, zmm11	;; R22= R10_22o - .707*R10_22e				; 65-68
	zfnmaddpd zmm11, zmm20, zmm29, zmm24	;; R6 = R6_18o - .707*R6_18e				; 66-69
	zfmaddpd zmm20, zmm20, zmm29, zmm24	;; R18= R6_18o + .707*R6_18e				; 66-69
	zfmaddpd zmm24, zmm16, zmm29, zmm1	;; R2 = R2_14o + .707*R2_14e				; 67-70
	zfnmaddpd zmm16, zmm16, zmm29, zmm1	;; R14= R2_14o - .707*R2_14e				; 67-70

	zstore	[srcreg+d2+d1], zmm5		;; Save R4						; 66
	zstore	[srcreg+d2+d1+64], zmm9		;; Save R16						; 66+1
	zstore	[srcreg+2*d4+d2+d1], zmm6	;; Save R12						; 67+1
	zstore	[srcreg+2*d4+d2+d1+64], zmm2	;; Save R24						; 67+2
	zstore	[srcreg+d4+d2+d1], zmm0		;; Save R8						; 68+2
	zstore	[srcreg+d4+d2+d1+64], zmm4	;; Save R20						; 68+3
	zstore	[srcreg+2*d4+d1], zmm3		;; Save R10						; 69+3
	zstore	[srcreg+2*d4+d1+64], zmm18	;; Save R22						; 69+4
	zstore	[srcreg+d4+d1], zmm11		;; Save R6						; 70+4
	zstore	[srcreg+d4+d1+64], zmm20	;; Save R18						; 70+5
	zstore	[srcreg+d1], zmm24		;; Save R2						; 71+5
	zstore	[srcreg+d1+64], zmm16		;; Save R14						; 71+6
	bump	srcreg, srcinc
	ENDM


;; Used in last levels of pass 1 in a two-pass FFT
zr12_wpn_twentyfour_reals_last_unfft_preload MACRO
	vbroadcastsd zmm31, ZMM_HALF
	vbroadcastsd zmm30, ZMM_P866
	vbroadcastsd zmm29, ZMM_SQRTHALF
	vbroadcastsd zmm28, ZMM_P259_P707
	vbroadcastsd zmm27, ZMM_P966_P707
	vbroadcastsd zmm26, ZMM_B
	ENDM

zr12_wpn_twentyfour_reals_last_unfft MACRO srcreg,srcinc,d1,d2,d4,screg,scinc,maskreg,maskinc,maxrpt,L1pt,L1pd
	vmovapd	zmm2, [srcreg+d2]		;; R3
	vmovapd	zmm14, [srcreg+d2+64]		;; I3
	vmovapd zmm25, [screg+1*128+64]		;; cosine/sine for R3/I3 (w^2)
	zfmaddpd zmm24, zmm2, zmm25, zmm14	;; A3 = R3 * cosine/sine + I3				; 1-4		n 7
	zfmsubpd zmm14, zmm14, zmm25, zmm2	;; B3 = I3 * cosine/sine - R3				; 1-4		n 7

	vmovapd	zmm4, [srcreg+d4]		;; R5
	vmovapd	zmm16, [srcreg+d4+64]		;; I5
	vmovapd zmm25, [screg+3*128+64]		;; cosine/sine for R5/I5 (w^4)
	zfmaddpd zmm2, zmm4, zmm25, zmm16	;; A5 = R5 * cosine/sine + I5				; 2-5		n 9
	zfmsubpd zmm16, zmm16, zmm25, zmm4	;; B5 = I5 * cosine/sine - R5				; 2-5		n 9

	vmovapd	zmm1, [srcreg+d1]		;; R2
	vmovapd	zmm13, [srcreg+d1+64]		;; I2
	vmovapd zmm25, [screg+0*128+64]		;; cosine/sine for R2/I2 (w^1)
	zfmaddpd zmm4, zmm1, zmm25, zmm13	;; A2 = R2 * cosine/sine + I2				; 3-6		n 11
	zfmsubpd zmm13, zmm13, zmm25, zmm1	;; B2 = I2 * cosine/sine - R2				; 3-6		n 11

	vmovapd	zmm5, [srcreg+d4+d1]		;; R6
	vmovapd	zmm17, [srcreg+d4+d1+64]	;; I6
	vmovapd zmm25, [screg+4*128+64]		;; cosine/sine for R6/I6 (w^5)
	zfmaddpd zmm1, zmm5, zmm25, zmm17	;; A6 = R6 * cosine/sine + I6				; 4-7		n 13
	zfmsubpd zmm17, zmm17, zmm25, zmm5	;; B6 = I6 * cosine/sine - R6				; 4-7		n 13

	vmovapd	zmm3, [srcreg+d2+d1]		;; R4
	vmovapd	zmm15, [srcreg+d2+d1+64]	;; I4
	vmovapd zmm25, [screg+2*128+64]		;; cosine/sine for R4/I4 (w^3)
	zfmaddpd zmm5, zmm3, zmm25, zmm15	;; A4 = R4 * cosine/sine + I4				; 5-8		n 16
	zfmsubpd zmm15, zmm15, zmm25, zmm3	;; B4 = I4 * cosine/sine - R4				; 5-8		n 16

	vmovapd	zmm10, [srcreg+2*d4+d2]		;; R11
	vmovapd	zmm22, [srcreg+2*d4+d2+64]	;; I11
	vmovapd zmm25, [screg+9*128+64]		;; cosine/sine for R11/I11 (w^10)
	zfmaddpd zmm3, zmm10, zmm25, zmm22	;; A11 = R11 * cosine/sine + I11 (new R11/sine)		; 6-9		n 17
	zfmsubpd zmm22, zmm22, zmm25, zmm10	;; B11 = I11 * cosine/sine - R11 (new I11/sine)		; 6-9		n 18

	vmovapd zmm25, [screg+1*128]		;; sine for R3/I3 (w^2)
	vmulpd	zmm24, zmm24, zmm25		;; A3 = A3 * sine (new R3)				; 7-10		n 17
	vmulpd	zmm14, zmm14, zmm25		;; B3 = B3 * sine (new I3)				; 7-10		n 18

	vmovapd	zmm8, [srcreg+2*d4]		;; R9
	vmovapd	zmm20, [srcreg+2*d4+64]		;; I9
	vmovapd zmm25, [screg+7*128+64]		;; cosine/sine for R9/I9 (w^8)
	zfmaddpd zmm10, zmm8, zmm25, zmm20	;; A9 = R9 * cosine/sine + I9 (new R9/sine)		; 8-11		n 19
	zfmsubpd zmm20, zmm20, zmm25, zmm8	;; B9 = I9 * cosine/sine - R9 (new I9/sine)		; 8-11		n 20

	vmovapd zmm25, [screg+3*128]		;; sine for R5/I5 (w^4)
	vmulpd	zmm2, zmm2, zmm25		;; A5 = A5 * sine (new R5)				; 9-12		n 19
	vmulpd	zmm16, zmm16, zmm25		;; B5 = B5 * sine (new I5)				; 9-12		n 20

	vmovapd	zmm11, [srcreg+2*d4+d2+d1]	;; R12
	vmovapd	zmm23, [srcreg+2*d4+d2+d1+64]	;; I12
	vmovapd zmm25, [screg+10*128+64]	;; cosine/sine for R12/I12 (w^11)
	zfmaddpd zmm8, zmm11, zmm25, zmm23	;; A12 = R12 * cosine/sine + I12 (new R12/sine)		; 10-13		n 21
	zfmsubpd zmm23, zmm23, zmm25, zmm11	;; B12 = I12 * cosine/sine - R12 (new I12/sine)		; 10-13		n 22

	vmovapd zmm25, [screg+0*128]		;; sine for R2/I2 (w^1)
	vmulpd	zmm4, zmm4, zmm25		;; A2 = A2 * sine (new R2)				; 11-14		n 21
	vmulpd	zmm13, zmm13, zmm25		;; B2 = B2 * sine (new I2)				; 11-14		n 22

	vmovapd	zmm7, [srcreg+d4+d2+d1]		;; R8
	vmovapd	zmm19, [srcreg+d4+d2+d1+64]	;; I8
	vmovapd zmm25, [screg+6*128+64]		;; cosine/sine for R8/I8 (w^7)
	zfmaddpd zmm11, zmm7, zmm25, zmm19	;; A8 = R8 * cosine/sine + I8 (new R8/sine)		; 12-15		n 23
	zfmsubpd zmm19, zmm19, zmm25, zmm7	;; B8 = I8 * cosine/sine - R8 (new I8/sine)		; 12-15		n 24

	vmovapd zmm25, [screg+4*128]		;; sine for R6/I6 (w^5)
	vmulpd	zmm1, zmm1, zmm25		;; A6 = A6 * sine (new R6)				; 13-16		n 23
	vmulpd	zmm17, zmm17, zmm25		;; B6 = B6 * sine (new I6)				; 13-16		n 24

	vmovapd	zmm6, [srcreg+d4+d2]		;; R7
	vmovapd	zmm18, [srcreg+d4+d2+64]	;; I7
	vmovapd zmm25, [screg+5*128+64]		;; cosine/sine for R7/I7 (w^6)
	zfmaddpd zmm7, zmm6, zmm25, zmm18	;; A7 = R7 * cosine/sine + I7 (new R7/sine)		; 14-17		n 25
	zfmsubpd zmm18, zmm18, zmm25, zmm6	;; B7 = I7 * cosine/sine - R7 (new I7/sine)		; 14-17		n 26

	vmovapd	zmm9, [srcreg+2*d4+d1]		;; R10
	vmovapd	zmm21, [srcreg+2*d4+d1+64]	;; I10
	vmovapd zmm25, [screg+8*128+64]		;; cosine/sine for R10/I10 (w^9)
	zfmaddpd zmm6, zmm9, zmm25, zmm21	;; A10 = R10 * cosine/sine + I10 (new R10/sine)		; 15-18		n 27
	zfmsubpd zmm21, zmm21, zmm25, zmm9	;; B10 = I10 * cosine/sine - R10 (new I10/sine)		; 15-18		n 28

	vmovapd zmm25, [screg+2*128]		;; sine for R4/I4 (w^3)
	vmulpd	zmm5, zmm5, zmm25		;; A4 = A4 * sine (new R4)				; 16-19		n 27
	vmulpd	zmm15, zmm15, zmm25		;; B4 = B4 * sine (new I4)				; 16-19		n 28

	vmovapd zmm25, [screg+9*128]		;; sine for R11/I11 (w^10)
	zfmaddpd zmm9, zmm3, zmm25, zmm24	;; r3+ = R3+R11*sine					; 17-20		n 29
	zfnmaddpd zmm3, zmm3, zmm25, zmm24	;; r3- = R3-R11*sine					; 17-20		n 66
	zfmaddpd zmm24, zmm22, zmm25, zmm14	;; i3+ = I3+I11*sine					; 18-21		n 64
	zfnmaddpd zmm22, zmm22, zmm25, zmm14	;; i3- = I3-I11*sine					; 18-21		n 32
	vmovapd zmm25, [screg+7*128]		;; sine for R9/I9 (w^8)
	zfmaddpd zmm14, zmm10, zmm25, zmm2	;; r5+ = R5+R9*sine					; 19-22		n 29
	zfnmaddpd zmm10, zmm10, zmm25, zmm2	;; r5- = R5-R9*sine					; 19-22		n 64
	zfmaddpd zmm2, zmm20, zmm25, zmm16	;; i5+ = I5+I9*sine					; 20-23		n 66
	zfnmaddpd zmm20, zmm20, zmm25, zmm16	;; i5- = I5-I9*sine					; 20-23		n 32
	vmovapd zmm25, [screg+10*128]		;; sine for R12/I12 (w^11)
	zfmaddpd zmm16, zmm8, zmm25, zmm4	;; r2+ = R2+R12*sine					; 21-24		n 30
	zfnmaddpd zmm8, zmm8, zmm25, zmm4	;; r2- = R2-R12*sine					; 21-24		n 65
	zfmaddpd zmm4, zmm23, zmm25, zmm13	;; i2+ = I2+I12*sine					; 22-25		n 61
	zfnmaddpd zmm23, zmm23, zmm25, zmm13	;; i2- = I2-I12*sine					; 22-25		n 31
	vmovapd zmm25, [screg+6*128]		;; sine for R8/I8 (w^7)
	zfmaddpd zmm13, zmm11, zmm25, zmm1	;; r6+ = R6+R8*sine					; 23-26		n 30
	zfnmaddpd zmm11, zmm11, zmm25, zmm1	;; r6- = R6-R8*sine					; 23-26		n 61
	zfmaddpd zmm1, zmm19, zmm25, zmm17	;; i6+ = I6+I8*sine					; 24-27		n 65
	zfnmaddpd zmm19, zmm19, zmm25, zmm17	;; i6- = I6-I8*sine					; 24-27		n 31
	vmovapd	zmm0, [srcreg]			;; r1A+r1B
	vmovapd zmm25, [screg+5*128]		;; sine for R7/I7 (w^6)
	zfmaddpd zmm17, zmm7, zmm25, zmm0	;; r1++ = (r1A+r1B)+R7*sine				; 25-28		n 34
	zfnmaddpd zmm7, zmm7, zmm25, zmm0	;; r1+- = (r1A+r1B)-R7*sine				; 25-28		n 36
	vmovapd	zmm12, [srcreg+64]		;; r1A-r1B
	zfmaddpd zmm0, zmm18, zmm25, zmm12	;; r1-+ = (r1A-r1B)+I7*sine				; 26-29		n 71
	zfnmaddpd zmm18, zmm18, zmm25, zmm12	;; r1-- = (r1A-r1B)-I7*sine				; 26-29		n 68
	vmovapd zmm25, [screg+8*128]		;; sine for R10/I10 (w^9)
	zfmaddpd zmm12, zmm6, zmm25, zmm5	;; r4+ = R4+R10*sine					; 27-30		n 34
	zfnmaddpd zmm6, zmm6, zmm25, zmm5	;; r4- = R4-R10*sine					; 27-30		n 63
	zfmaddpd zmm5, zmm21, zmm25, zmm15	;; i4+ = I4+I10*sine					; 28-31		n 63
	zfnmaddpd zmm21, zmm21, zmm25, zmm15	;; i4- = I4-I10*sine					; 28-31		n 36
	mov	bl, [maskreg+0*1]		;; Load index into compressed fudges
	bump	screg, scinc

	vaddpd	zmm15, zmm9, zmm14		;; r3++ = (r3+r11) + (r5+r9)				; 29-32		n 37
	vsubpd	zmm9, zmm9, zmm14		;; r3+- = (r3+r11) - (r5+r9)				; 29-32		n 38
	mov	r14, [r13+0*8]			;; Load the xor mask

	vaddpd	zmm14, zmm16, zmm13		;; r2++ = (r2+r12) + (r6+r8)				; 30-33		n 37
	vsubpd	zmm16, zmm16, zmm13		;; r2+- = (r2+r12) - (r6+r8)				; 30-33		n 41
	xor	r14, [r12+rbx*8]		;; Apply the xor mask to 64 bits of fudge flags

	vaddpd	zmm13, zmm23, zmm19		;; i2-+ = (i2-i12) + (i6-i8)				; 31-34		n 38
	vsubpd	zmm23, zmm23, zmm19		;; i2-- = (i2-i12) - (i6-i8)				; 31-34		n 40
	mov	bl, [maskreg+1*1]		;; Load index into compressed fudges

	kmovw	k1, r14d			;; Load R1 and R13 fudge factor mask			; 32		n 46
	vaddpd	zmm19, zmm22, zmm20		;; i3-+ = (i3-i11) + (i5-i9)				; 32-35		n 41
	shr	r14, 16				;; Next 16 bits of fudge flags

	kmovw	k2, r14d			;; Load R2 and R14 fudge factor mask			; 33		n 89
	vsubpd	zmm22, zmm22, zmm20		;; i3-- = (i3-i11) - (i5-i9)				; 33-36		n 40
	shr	r14, 16				;; Next 16 bits of fudge flags

	kmovw	k3, r14d			;; Load R3 and R15 fudge factor mask			; 34		n 56
	vaddpd	zmm20, zmm17, zmm12		;; r1+++ = (r1++) + (r4+r10)				; 34-37		n 42
	shr	r14, 16				;; Next 16 bits of fudge flags

	kmovw	k4, r14d			;; Load R4 and R16 fudge factor mask			; 35		n 81
	vsubpd	zmm17, zmm17, zmm12		;; r1++- = (r1++) - (r4+r10)				; 35-38		n 43
	mov	r14, [r13+1*8]			;; Load the xor mask

	vaddpd	zmm12, zmm7, zmm21		;; r1+-+ = (r1+-) + (i4-i10)				; 36-39		n 45
	vsubpd	zmm7, zmm7, zmm21		;; r1+-- = (r1+-) - (i4-i10)				; 36-39		n 44
	xor	r14, [r12+rbx*8]		;; Apply the xor mask to 64 bits of fudge flags

	kmovw	k5, r14d			;; Load R5 and R17 fudge factor mask			; 37		n 54
	vaddpd	zmm21, zmm15, zmm14		;; r3+++ = (r3++) + (r2++)				; 37-40		n 42
	shr	r14, 16				;; Next 16 bits of fudge flags

	vsubpd	zmm15, zmm15, zmm14		;; r3++- = (r3++) - (r2++)				; 38-41		n 43
	vaddpd	zmm14, zmm9, zmm13		;; r3+-+ = (r3+-) + (i2-+)				; 38-41		n 45
	mov	bl, [maskreg+2*1]		;; Load index into compressed fudges

	kmovw	k6, r14d			;; Load R6 and R18 fudge factor mask			; 39		n 88
	vsubpd	zmm9, zmm9, zmm13		;; r3+-- = (r3+-) - (i2-+)				; 39-42		n 44
	shr	r14, 16				;; Next 16 bits of fudge flags

	kmovw	k7, r14d			;; Load R7 and R19 fudge factor mask			; 40		n 49
	vaddpd	zmm13, zmm22, zmm23		;; i3--+ = (i3--) + (i2--)				; 40-43		n 50
	shr	r14, 16				;; Next 16 bits of fudge flags

	vsubpd	zmm22, zmm22, zmm23		;; i3--- = (i3--) - (i2--)				; 41-44		n 47
	vaddpd	zmm23, zmm19, zmm16		;; i3-++ = (i3-+) + (r2+-)				; 41-44		n 52
	L1prefetchw srcreg+L1pd, L1pt

	vsubpd	zmm19, zmm19, zmm16		;; i3-+- = (i3-+) - (r2+-)				; 42-45		n 55
	vaddpd	zmm16, zmm20, zmm21		;; R1   = (r1+++) + (r3+++)				; 42-45		n 46
	L1prefetchw srcreg+64+L1pd, L1pt

	zfnmaddpd zmm21, zmm21, zmm31, zmm20	;; R9_17= (r1+++) - .500(r3+++)				; 43-46		n 47
	vaddpd	zmm20, zmm17, zmm15		;; R13  = (r1++-) + (r3++-)				; 43-46		n 48
	L1prefetchw srcreg+d1+L1pd, L1pt

	zfnmaddpd zmm15, zmm15, zmm31, zmm17	;; R5_21= (r1++-) - .500(r3++-)				; 44-47		n 50
	vsubpd	zmm17, zmm7, zmm9		;; R7    = (r1+--) - (r3+--)				; 44-47		n 49
	L1prefetchw srcreg+d1+64+L1pd, L1pt

	zfmaddpd zmm9, zmm9, zmm31, zmm7	;; R15_23= (r1+--) + .500(r3+--)			; 45-48		n 55
	vsubpd	zmm7, zmm12, zmm14		;; R19   = (r1+-+) - (r3+-+)				; 45-48		n 51
	L1prefetchw srcreg+d2+L1pd, L1pt

	zfmaddpd zmm14, zmm14, zmm31, zmm12	;; R3_11 = (r1+-+) + .500(r3+-+)			; 46-49		n 52
	vmulpd	zmm16{k1}, zmm16, zmm26		;; apply fudge multiplier for R1			; 46-49
	L1prefetchw srcreg+d2+64+L1pd, L1pt

	kshiftrw k1, k1, 8			;; R13's fudge						; 47		n 48
	zfnmaddpd zmm12, zmm22, zmm30, zmm21	;; R9 = R9_17 - .866(i3---)				; 47-50		n 53
	L1prefetchw srcreg+d2+d1+L1pd, L1pt

	vmulpd	zmm20{k1}, zmm20, zmm26		;; apply fudge multiplier for R13			; 48-51
	zfmaddpd zmm22, zmm22, zmm30, zmm21	;; R17= R9_17 + .866(i3---)				; 48-51		n 56

	kmovw	k1, r14d			;; Load R8 and R20 fudge factor mask			; 49		n 83
	vmulpd	zmm17{k7}, zmm17, zmm26		;; apply fudge multiplier for R7			; 49-52
	mov	r14, [r13+2*8]			;; Load the xor mask

	kshiftrw k7, k7, 8			;; R19's fudge						; 50		n 51
	zfmaddpd zmm21, zmm13, zmm30, zmm15	;; R5 = R5_21 + .866(i3--+)				; 50-53		n 54
  	zstore	[srcreg], zmm16			;; Save R1						; 50
	xor	r14, [r12+rbx*8]		;; Apply the xor mask to 64 bits of fudge flags

	vmulpd	zmm7{k7}, zmm7, zmm26		;; apply fudge multiplier for R19			; 51-54
	zfnmaddpd zmm13, zmm13, zmm30, zmm15	;; R21= R5_21 - .866(i3--+)				; 51-54		n 57
	bump	maskreg, maskinc

	kmovw	k7, r14d			;; Load R9 and R21 fudge factor mask			; 52		n 53
	zfmaddpd zmm15, zmm23, zmm30, zmm14	;; R3 = R3_11 + .866(i3-++)				; 52-55		n 56
	zstore	[srcreg+64], zmm20		;; Save R13						; 52
	shr	r14, 16				;; Next 16 bits of fudge flags

	vmulpd	zmm12{k7}, zmm12, zmm26		;; apply fudge multiplier for R9			; 53-56
	zfnmaddpd zmm23, zmm23, zmm30, zmm14	;; R11= R3_11 - .866(i3-++)				; 53-56		n 59
	zstore	[srcreg+d4+d2], zmm17		;; Save R7						; 53
	L1prefetchw srcreg+d2+d1+64+L1pd, L1pt

	kshiftrw k7, k7, 8			;; R21's fudge						; 54		n 57
	vmulpd	zmm21{k5}, zmm21, zmm26		;; apply fudge multiplier for R5			; 54-57
	L1prefetchw srcreg+d4+L1pd, L1pt

	kshiftrw k5, k5, 8			;; R17's fudge						; 55		n 56
	zfmaddpd zmm14, zmm19, zmm30, zmm9	;; R15= R15_23 + .866(i3-+-)				; 55-58		n 60
	zstore	[srcreg+d4+d2+64], zmm7		;; Save R19						; 55
	L1prefetchw srcreg+d4+64+L1pd, L1pt

	vmulpd	zmm15{k3}, zmm15, zmm26		;; apply fudge multiplier for R3			; 56-59
	vmulpd	zmm22{k5}, zmm22, zmm26		;; apply fudge multiplier for R17			; 56-59
	L1prefetchw srcreg+d4+d1+L1pd, L1pt

	kmovw	k5, r14d			;; Load R10 and R22 fudge factor mask			; 57		n 86
	vmulpd	zmm13{k7}, zmm13, zmm26		;; apply fudge multiplier for R21			; 57-60
	zstore	[srcreg+2*d4], zmm12		;; Save R9						; 57
	shr	r14, 16				;; Next 16 bits of fudge flags

	kmovw	k7, r14d			;; Load R11 and R23 fudge factor mask			; 58		n 59
	zfnmaddpd zmm19, zmm19, zmm30, zmm9	;; R23= R15_23 - .866(i3-+-)				; 58-61		n 62
	zstore	[srcreg+d4], zmm21		;; Save R5						; 58
	shr	r14, 16				;; Next 16 bits of fudge flags

	kshiftrw k3, k3, 8			;; R15's fudge						; 59		n 60
	vmulpd	zmm23{k7}, zmm23, zmm26		;; apply fudge multiplier for R11			; 59-62
	L1prefetchw srcreg+d4+d1+64+L1pd, L1pt

	kshiftrw k7, k7, 8			;; R23's fudge						; 60		n 61
	vmulpd	zmm14{k3}, zmm14, zmm26		;; apply fudge multiplier for R15			; 60-63
	zstore	[srcreg+d2], zmm15		;; Save R3						; 60
	L1prefetchw srcreg+d4+d2+L1pd, L1pt

	kmovw	k3, r14d			;; Load R12 and R24 fudge factor mask			; 61		n 82
	vaddpd	zmm9, zmm4, zmm11		;; i2++ = (i2+i12) + (r6-r8)				; 61-64		n 69
	zstore	[srcreg+d4+64], zmm22		;; Save R17						; 60+1
	L1prefetchw srcreg+d4+d2+64+L1pd, L1pt

	vsubpd	zmm4, zmm4, zmm11		;; i2+- = (i2+i12) - (r6-r8)				; 62-65		n 67
	vmulpd	zmm19{k7}, zmm19, zmm26		;; apply fudge multiplier for R23			; 62-65
	zstore	[srcreg+2*d4+64], zmm13		;; Save R21						; 61+1
	L1prefetchw srcreg+d4+d2+d1+L1pd, L1pt

	vaddpd	zmm11, zmm6, zmm5		;; r4-+ = (r4-r10) + (i4+i10)				; 63-66		n 69
	vsubpd	zmm6, zmm6, zmm5		;; r4-- = (r4-r10) - (i4+i10)				; 63-66		n 67
	zstore	[srcreg+2*d4+d2], zmm23		;; Save R11						; 63
	L1prefetchw srcreg+d4+d2+d1+64+L1pd, L1pt

	vaddpd	zmm5, zmm24, zmm10		;; i3++ = (i3+i11) + (r5-r9)				; 64-67		n 71
	vsubpd	zmm24, zmm24, zmm10		;; i3+- = (i3+i11) - (r5-r9)				; 64-67		n 68
	zstore	[srcreg+d2+64], zmm14		;; Save R15						; 64
	L1prefetchw srcreg+2*d4+L1pd, L1pt

	vaddpd	zmm10, zmm8, zmm1		;; r2-+ = (r2-r12) + (i6+i8)				; 65-68		n 74
	vsubpd	zmm8, zmm8, zmm1		;; r2-- = (r2-r12) - (i6+i8)				; 65-68		n 72
	L1prefetchw srcreg+2*d4+64+L1pd, L1pt

	vaddpd	zmm1, zmm3, zmm2		;; r3-+ = (r3-r11) + (i5+i9)				; 66-69		n 76
	vsubpd	zmm3, zmm3, zmm2		;; r3-- = (r3-r11) - (i5+i9)				; 66-69		n 73
	zstore	[srcreg+2*d4+d2+64], zmm19	;; Save R23						; 66
	L1prefetchw srcreg+2*d4+d1+L1pd, L1pt

	vsubpd	zmm16, zmm6, zmm4		;; R4_16e = (r4--) - (i2+-)				; 67-70		n 72
	zfnmaddpd zmm2, zmm4, zmm28, zmm6	;; R12_24e= (r4--) - .259/.707(i2+-)			; 67-70		n 72
	L1prefetchw srcreg+2*d4+d1+64+L1pd, L1pt

	zfmaddpd zmm4, zmm4, zmm27, zmm6	;; R8_20e = (r4--) + .966/.707(i2+-)			; 68-71		n 73
	vaddpd	zmm6, zmm18, zmm24		;; R4_16o = (r1--) + (i3+-)				; 68-71		n 77
	L1prefetchw srcreg+2*d4+d2+L1pd, L1pt

	zfnmaddpd zmm24, zmm24, zmm31, zmm18	;; R12_24_8_20o= (r1--) - .500(i3+-)			; 69-72		n 73
	vaddpd	zmm18, zmm11, zmm9		;; R10_22e= (r4-+) + (i2++)				; 69-72		n 74
	L1prefetchw srcreg+2*d4+d2+64+L1pd, L1pt

	zfnmaddpd zmm20, zmm9, zmm27, zmm11	;; R6_18e = (r4-+) - .966/.707(i2++)			; 70-73		n 75
	zfmaddpd zmm9, zmm9, zmm28, zmm11	;; R2_14e = (r4-+) + .259/.707(i2++)			; 70-73		n 75
	L1prefetchw srcreg+2*d4+d2+d1+L1pd, L1pt

	vsubpd	zmm11, zmm0, zmm5		;; R10_22o     = (r1-+) - (i3++)			; 71-74		n 80
	zfmaddpd zmm5, zmm5, zmm31, zmm0	;; R6_18_2_14o = (r1-+) + .500(i3++)			; 71-74		n 76
	L1prefetchw srcreg+2*d4+d2+d1+64+L1pd, L1pt

	vsubpd	zmm16, zmm16, zmm8		;; R4_16e = R4_16e - (r2--)				; 72-75		n 77
	zfmaddpd zmm2, zmm8, zmm27, zmm2	;; R12_24e= R12_24e + .966/.707(r2--)			; 72-75		n 78

	zfnmaddpd zmm4, zmm8, zmm28, zmm4	;; R8_20e = R8_20e - .259/.707(r2--)			; 73-76		n 79
	zfmaddpd zmm0, zmm3, zmm30, zmm24	;; R12_24o= R12_24_8_20o + .866(r3--)			; 73-76		n 78

	zfnmaddpd zmm3, zmm3, zmm30, zmm24	;; R8_20o = R12_24_8_20o - .866(r3--)			; 74-77		n 79
	vsubpd	zmm18, zmm18, zmm10		;; R10_22e= R10_22e - (r2-+)				; 74-77		n 80

	zfnmaddpd zmm20, zmm10, zmm28, zmm20	;; R6_18e = R6_18e - .259/.707(r2-+)			; 75-78		n 81
	zfmaddpd zmm9, zmm10, zmm27, zmm9	;; R2_14e = R2_14e + .966/.707(r2-+)			; 75-78		n 85

	zfnmaddpd zmm24, zmm1, zmm30, zmm5	;; R6_18o = R6_18_2_14o - .866(r3-+)			; 76-79		n 81
	zfmaddpd zmm1, zmm1, zmm30, zmm5	;; R2_14o = R6_18_2_14o + .866(r3-+)			; 76-79		n 85

	zfnmaddpd zmm5, zmm16, zmm29, zmm6	;; R4 = R4_16o - .707*R4_16e				; 77-80		n 81
	zfmaddpd zmm16, zmm16, zmm29, zmm6	;; R16= R4_16o + .707*R4_16e				; 77-80		n 84

	zfnmaddpd zmm6, zmm2, zmm29, zmm0	;; R12= R12_24o - .707*R12_24e				; 78-81		n 82
	zfmaddpd zmm2, zmm2, zmm29, zmm0	;; R24= R12_24o + .707*R12_24e				; 78-81		n 85

	zfmaddpd zmm0, zmm4, zmm29, zmm3	;; R8 = R8_20o + .707*R8_20e				; 79-82		n 83
	zfnmaddpd zmm4, zmm4, zmm29, zmm3	;; R20= R8_20o - .707*R8_20e				; 79-82		n 87

	zfmaddpd zmm3, zmm18, zmm29, zmm11	;; R10= R10_22o + .707*R10_22e				; 80-83		n 86
	zfnmaddpd zmm18, zmm18, zmm29, zmm11	;; R22= R10_22o - .707*R10_22e				; 80-83		n 90

	vmulpd	zmm5{k4}, zmm5, zmm26		;; apply fudge multiplier for R4			; 81-84
	zfnmaddpd zmm11, zmm20, zmm29, zmm24	;; R6 = R6_18o - .707*R6_18e				; 81-84		n 88

	vmulpd	zmm6{k3}, zmm6, zmm26		;; apply fudge multiplier for R12			; 82-85
	zfmaddpd zmm20, zmm20, zmm29, zmm24	;; R18= R6_18o + .707*R6_18e				; 82-85		n 91

	kshiftrw k4, k4, 8			;; R16's fudge						; 83		n 84
	vmulpd	zmm0{k1}, zmm0, zmm26		;; apply fudge multiplier for R8			; 83-86

	kshiftrw k3, k3, 8			;; R24's fudge						; 84		n 85
	vmulpd	zmm16{k4}, zmm16, zmm26		;; apply fudge multiplier for R16			; 84-87

	zfmaddpd zmm24, zmm9, zmm29, zmm1	;; R2 = R2_14o + .707*R2_14e				; 85-88		n 89
	vmulpd	zmm2{k3}, zmm2, zmm26		;; apply fudge multiplier for R24			; 85-88
	zstore	[srcreg+d2+d1], zmm5		;; Save R4						; 85

	kshiftrw k1, k1, 8			;; R20's fudge						; 86		n 87
	vmulpd	zmm3{k5}, zmm3, zmm26		;; apply fudge multiplier for R10			; 86-89
	zstore	[srcreg+2*d4+d2+d1], zmm6	;; Save R12						; 86

	zfnmaddpd zmm9, zmm9, zmm29, zmm1	;; R14= R2_14o - .707*R2_14e				; 87-90		n 91
	vmulpd	zmm4{k1}, zmm4, zmm26		;; apply fudge multiplier for R20			; 87-90
	zstore	[srcreg+d4+d2+d1], zmm0		;; Save R8						; 87

	kshiftrw k5, k5, 8			;; R22's fudge						; 88		n 90
	vmulpd	zmm11{k6}, zmm11, zmm26		;; apply fudge multiplier for R6			; 88-91
	zstore	[srcreg+d2+d1+64], zmm16	;; Save R16						; 88

	kshiftrw k6, k6, 8			;; R18's fudge						; 89		n 91
	vmulpd	zmm24{k2}, zmm24, zmm26		;; apply fudge multiplier for R2			; 89-92
	zstore	[srcreg+2*d4+d2+d1+64], zmm2	;; Save R24						; 89

	kshiftrw k2, k2, 8			;; R14's fudge						; 90		n 91
	vmulpd	zmm18{k5}, zmm18, zmm26		;; apply fudge multiplier for R22			; 90-93
	zstore	[srcreg+2*d4+d1], zmm3		;; Save R10						; 90

	vmulpd	zmm20{k6}, zmm20, zmm26		;; apply fudge multiplier for R18			; 91-94
	vmulpd	zmm9{k2}, zmm9, zmm26		;; apply fudge multiplier for R14			; 91-94
	zstore	[srcreg+d4+d2+d1+64], zmm4	;; Save R20						; 91

	zstore	[srcreg+d4+d1], zmm11		;; Save R6						; 92
	zstore	[srcreg+d1], zmm24		;; Save R2						; 93
	zstore	[srcreg+2*d4+d1+64], zmm18	;; Save R22						; 94
	zstore	[srcreg+d4+d1+64], zmm20	;; Save R18						; 95
	zstore	[srcreg+d1+64], zmm9		;; Save R14						; 95+1
	bump	srcreg, srcinc
	ENDM


;;
;; ************************************* One-pass variant of twelve-complex-djbfft ******************************************
;; Implements twelve complex in three and four complex chunks.  Uses a three and four complex sin/cos table.
;; This lets one pass FFTs get much of the benefits of 12-complex without writing the difficult twelve_complex_twenty_four_reals macros
;;

;; The standard version
zr34_twelve_complex_djbfft_preload MACRO
	zr34_12c_djbfft_cmn_preload
	ENDM
zr34_twelve_complex_djbfft MACRO srcreg,srcinc,d1,sc3reg,sc3gap,sc3inc,sc4reg,sc4gap,sc4inc,maxrpt,L1pt,L1pd
	zr34_12c_djbfft_cmn srcreg,srcinc,d1,noexec,0,0,0,sc3reg,sc3gap,sc3inc,sc4reg,sc4gap,sc4inc,maxrpt,L1pt,L1pd
	ENDM

; Like the standard version except vbroadcastsd is used to reduce sin/cos data
zr34b_twelve_complex_djbfft_preload MACRO
	zr34_12c_djbfft_cmn_preload
	ENDM
zr34b_twelve_complex_djbfft MACRO srcreg,srcinc,d1,sc3reg,sc3gap,sc3inc,sc4reg,sc4gap,sc4inc,maxrpt,L1pt,L1pd
	zr34_12c_djbfft_cmn srcreg,srcinc,d1,exec,sc3reg,sc4reg,16,sc3reg,sc3gap,sc3inc,sc4reg,sc4gap,sc4inc,maxrpt,L1pt,L1pd
	ENDM

; Like the zr34b version except sin/cos data is loaded from a larger real sin/cos table
zr34rb_twelve_complex_djbfft_preload MACRO
	zr34_12c_djbfft_cmn_preload
	ENDM
zr34rb_twelve_complex_djbfft MACRO srcreg,srcinc,d1,sc3reg,sc3gap,sc3inc,sc4reg,sc4gap,sc4inc,maxrpt,L1pt,L1pd
	zr34_12c_djbfft_cmn srcreg,srcinc,d1,exec,sc3reg+8,sc4reg+8,128,sc3reg,sc3gap,sc3inc,sc4reg,sc4gap,sc4inc,maxrpt,L1pt,L1pd
	ENDM

zr34_12c_djbfft_cmn_preload MACRO
	vbroadcastsd zmm31, ZMM_P866
	vbroadcastsd zmm30, ZMM_HALF
	ENDM
zr34_12c_djbfft_cmn MACRO srcreg,srcinc,d1,bcast,bc3reg,bc4reg,bcsz,sc3reg,sc3gap,sc3inc,sc4reg,sc4gap,sc4inc,maxrpt,L1pt,L1pd
	IF sc4gap NE 0
	need_code_for_non_zero_sc4gap
	ENDIF
	vmovapd	zmm1, [srcreg+0*d1+4*d1]	;;30 R2
	vmovapd	zmm2, [srcreg+0*d1+8*d1]	;;30 R3
	vaddpd	zmm0, zmm1, zmm2		;;30 r2+ = R2 + R3				; 1-4		n 5
	vsubpd	zmm1, zmm1, zmm2		;;30 r2- = R2 - R3				; 1-4		n 21

	vmovapd	zmm3, [srcreg+0*d1+4*d1+64]	;;30 I2
	vmovapd	zmm4, [srcreg+0*d1+8*d1+64]	;;30 I3
	vaddpd	zmm2, zmm3, zmm4		;;30 i2+ = I2 + I3				; 2-5		n 6
	vsubpd	zmm3, zmm3, zmm4		;;30 i2- = I2 - I3				; 2-5		n 20

	vmovapd	zmm5, [srcreg+1*d1+4*d1]	;;31 R2
	vmovapd	zmm6, [srcreg+1*d1+8*d1]	;;31 R3
	vaddpd	zmm4, zmm5, zmm6		;;31 r2+ = R2 + R3				; 3-6		n 7
	vsubpd	zmm5, zmm5, zmm6		;;31 r2- = R2 - R3				; 3-6		n 23

	vmovapd	zmm7, [srcreg+1*d1+4*d1+64]	;;31 I2
	vmovapd	zmm8, [srcreg+1*d1+8*d1+64]	;;31 I3
	vaddpd	zmm6, zmm7, zmm8		;;31 i2+ = I2 + I3				; 4-7		n 8
	vsubpd	zmm7, zmm7, zmm8		;;31 i2- = I2 - I3				; 4-7		n 22

	vmovapd	zmm9, [srcreg+0*d1]		;;30 R1
	vaddpd	zmm8, zmm9, zmm0		;;30 R1 = R1 + r2+ (R1 in 40)			; 5-8		n 36
	zfnmaddpd zmm0, zmm0, zmm30, zmm9	;;30 r1- = R1 - .5r2+				; 5-8		n 13

	vmovapd	zmm10, [srcreg+0*d1+64]		;;30 I1
	vaddpd	zmm9, zmm10, zmm2		;;30 I1 = I1 + i2+ (I1 in 40)			; 6-9		n 37
	zfnmaddpd zmm2, zmm2, zmm30, zmm10	;;30 i1- = I1 - .5i2+				; 6-9		n 14

	vmovapd	zmm11, [srcreg+1*d1]		;;31 R1
	vaddpd	zmm10, zmm11, zmm4		;;31 R1 = R1 + r2+ (R2 in 40)			; 7-10		n 38
	zfnmaddpd zmm4, zmm4, zmm30, zmm11	;;31 r1- = R1 - .5r2+				; 7-10		n 15

	vmovapd	zmm12, [srcreg+1*d1+64]		;;31 I1
	vaddpd	zmm11, zmm12, zmm6		;;31 I1 = I1 + i2+ (I2 in 40)			; 8-11		n 39
	zfnmaddpd zmm6, zmm6, zmm30, zmm12	;;31 i1- = I1 - .5i2+				; 8-11		n 15

	vmovapd	zmm13, [srcreg+2*d1+4*d1]	;;32 R2
	vmovapd	zmm14, [srcreg+2*d1+8*d1]	;;32 R3
	vaddpd	zmm12, zmm13, zmm14		;;32 r2+ = R2 + R3				; 9-12		n 16
	vsubpd	zmm13, zmm13, zmm14		;;32 r2- = R2 - R3				; 9-12		n 25

	vmovapd	zmm15, [srcreg+2*d1+4*d1+64]	;;32 I2
	vmovapd	zmm16, [srcreg+2*d1+8*d1+64]	;;32 I3
	vaddpd	zmm14, zmm15, zmm16		;;32 i2+ = I2 + I3				; 10-13		n 17
	vsubpd	zmm15, zmm15, zmm16		;;32 i2- = I2 - I3				; 10-13		n 24

	vmovapd	zmm17, [srcreg+3*d1+4*d1]	;;33 R2
	vmovapd	zmm18, [srcreg+3*d1+8*d1]	;;33 R3
	vaddpd	zmm16, zmm17, zmm18		;;33 r2+ = R2 + R3				; 11-14		n 18
	vsubpd	zmm17, zmm17, zmm18		;;33 r2- = R2 - R3				; 11-14		n 27

	vmovapd	zmm19, [srcreg+3*d1+4*d1+64]	;;33 I2
	vmovapd	zmm20, [srcreg+3*d1+8*d1+64]	;;33 I3
	vaddpd	zmm18, zmm19, zmm20		;;33 i2+ = I2 + I3				; 12-15		n 19
	vsubpd	zmm19, zmm19, zmm20		;;33 i2- = I2 - I3				; 12-15		n 26

no bcast vmovapd zmm29, [sc3reg+0*sc3gap]	;;30 sine
bcast	vbroadcastsd zmm29, Q [bc3reg+0*sc3gap]	;;30 sine
	vmulpd	zmm28, zmm31, zmm29		;;30 .866s = .866 * sine			; 13-16		n 20
	vmulpd	zmm0, zmm0, zmm29		;;30 r1-s = r1- * sine				; 13-16		n 20
	vmulpd	zmm2, zmm2, zmm29		;;30 i1-s = i1- * sine				; 14-17		n 21

no bcast vmovapd zmm29, [sc3reg+1*sc3gap]	;;31 sine
bcast	vbroadcastsd zmm29, Q [bc3reg+1*sc3gap]	;;31 sine
	vmulpd	zmm27, zmm31, zmm29		;;31 .866s = .866 * sine			; 14-17		n 22
	vmulpd	zmm4, zmm4, zmm29		;;31 r1-s = r1- * sine				; 15-18		n 22
	vmulpd	zmm6, zmm6, zmm29		;;31 i1-s = i1- * sine				; 15-18		n 23

	vmovapd	zmm21, [srcreg+2*d1]		;;32 R1
	vaddpd	zmm20, zmm21, zmm12		;;32 R1 = R1 + r2+ (R3 in 40)			; 16-19		n 36
	zfnmaddpd zmm12, zmm12, zmm30, zmm21	;;32 r1- = R1 - .5r2+				; 16-19		n 24

	vmovapd	zmm22, [srcreg+2*d1+64]		;;32 I1
	vaddpd	zmm21, zmm22, zmm14		;;32 I1 = I1 + i2+ (I3 in 40)			; 17-20		n 37
	zfnmaddpd zmm14, zmm14, zmm30, zmm22	;;32 i1- = I1 - .5i2+				; 17-20		n 25

	vmovapd	zmm23, [srcreg+3*d1]		;;33 R1
	vaddpd	zmm22, zmm23, zmm16		;;33 R1 = R1 + r2+ (R4 in 40)			; 18-21		n 38
	zfnmaddpd zmm16, zmm16, zmm30, zmm23	;;33 r1- = R1 - .5r2+				; 18-21		n 26

	vmovapd	zmm24, [srcreg+3*d1+64]		;;33 I1
	vaddpd	zmm23, zmm24, zmm18		;;33 I1 = I1 + i2+ (I4 in 40)			; 19-22		n 39
	zfnmaddpd zmm18, zmm18, zmm30, zmm24	;;33 i1- = I1 - .5i2+				; 19-22		n 27

no bcast vmovapd zmm26, [sc3reg+0*sc3gap+64]	;;30 cosine/sine
bcast	vbroadcastsd zmm26, Q [bc3reg+0*sc3gap+bcsz/2] ;;30 cosine/sine						n 28
	zfnmaddpd zmm24, zmm3, zmm28, zmm0	;;30 R2s = r1-s - .866s*i2-			; 20-23		n 28
	zfmaddpd zmm3, zmm3, zmm28, zmm0	;;30 R3s = r1-s + .866s*i2-			; 20-23		n 29

no bcast vmovapd zmm25, [sc3reg+1*sc3gap+64]	;;31 cosine/sine
bcast	vbroadcastsd zmm25, Q [bc3reg+1*sc3gap+bcsz/2] ;;31 cosine/sine						n 30
	zfmaddpd zmm0, zmm1, zmm28, zmm2	;;30 I2s = i1-s + .866s*r2-			; 21-24		n 28
	zfnmaddpd zmm1, zmm1, zmm28, zmm2	;;30 I3s = i1-s - .866s*r2-			; 21-24		n 29

no bcast vmovapd zmm29, [sc3reg+2*sc3gap+64]	;;32 cosine/sine
bcast	vbroadcastsd zmm29, Q [bc3reg+2*sc3gap+bcsz/2] ;;32 cosine/sine						n 32
	zfnmaddpd zmm2, zmm7, zmm27, zmm4	;;31 R2s = r1-s - .866s*i2-			; 22-25		n 30
	zfmaddpd zmm7, zmm7, zmm27, zmm4	;;31 R3s = r1-s + .866s*i2-			; 22-25		n 31

no bcast vmovapd zmm28, [sc3reg+3*sc3gap+64]	;;33 cosine/sine
bcast	vbroadcastsd zmm28, Q [bc3reg+3*sc3gap+bcsz/2] ;;33 cosine/sine						n 34
	zfmaddpd zmm4, zmm5, zmm27, zmm6	;;31 I2s = i1-s + .866s*r2-			; 23-26		n 30
	zfnmaddpd zmm5, zmm5, zmm27, zmm6	;;31 I3s = i1-s - .866s*r2-			; 23-26		n 31

no bcast vmovapd zmm27, [sc3reg+2*sc3gap]	;;32 sine
bcast	vbroadcastsd zmm27, Q [bc3reg+2*sc3gap]	;;32 sine for R3/I3 in 41 and 42				n 40
	zfnmaddpd zmm6, zmm15, zmm31, zmm12	;;32 R2 = r1- - .866*i2-			; 24-27		n 32
	zfmaddpd zmm15, zmm15, zmm31, zmm12	;;32 R3 = r1- + .866*i2-			; 24-27		n 33

	L1prefetchw srcreg+srcinc+0*d1+4*d1, L1pt
	zfmaddpd zmm12, zmm13, zmm31, zmm14	;;32 I2 = i1- + .866*r2-			; 25-28		n 32
	zfnmaddpd zmm13, zmm13, zmm31, zmm14	;;32 I3 = i1- - .866*r2-			; 25-28		n 33

	L1prefetchw srcreg+srcinc+0*d1+8*d1, L1pt
	zfnmaddpd zmm14, zmm19, zmm31, zmm16	;;33 R2 = r1- - .866*i2-			; 26-29		n 34
	zfmaddpd zmm19, zmm19, zmm31, zmm16	;;33 R3 = r1- + .866*i2-			; 26-29		n 35

	L1prefetchw srcreg+srcinc+0*d1+4*d1+64, L1pt
	zfmaddpd zmm16, zmm17, zmm31, zmm18	;;33 I2 = i1- + .866*r2-			; 27-30		n 34
	zfnmaddpd zmm17, zmm17, zmm31, zmm18	;;33 I3 = i1- - .866*r2-			; 27-30		n 35

	L1prefetchw srcreg+srcinc+0*d1+8*d1+64, L1pt
	zfmsubpd zmm18, zmm24, zmm26, zmm0	;;30 R2 = R2s * cosine/sine - I2s (R1 in 41)	; 28-31		n 40
	zfmaddpd zmm0, zmm0, zmm26, zmm24	;;30 I2 = I2s * cosine/sine + R2s (I1 in 41)	; 28-31		n 41

	L1prefetchw srcreg+srcinc+0*d1, L1pt
	zfmaddpd zmm24, zmm3, zmm26, zmm1	;;30 R3 = R3s * cosine/sine + I3s (R1 in 42)	; 29-32		n 50
	zfmsubpd zmm1, zmm1, zmm26, zmm3	;;30 I3 = I3s * cosine/sine - R3s (I1 in 42)	; 29-32		n 51

no bcast vmovapd zmm26, [sc4reg+0*128]		;;4x Sine
bcast	vbroadcastsd zmm26, Q [bc4reg+0*bcsz]	;;4x Sine							n 42
	zfmsubpd zmm3, zmm2, zmm25, zmm4	;;31 R2 = R2s * cosine/sine - I2s (R2 in 41)	; 30-33		n 45
	zfmaddpd zmm4, zmm4, zmm25, zmm2	;;31 I2 = I2s * cosine/sine + R2s (I2 in 41)	; 30-33		n 46

	L1prefetchw srcreg+srcinc+0*d1+64, L1pt
	zfmaddpd zmm2, zmm7, zmm25, zmm5	;;31 R3 = R3s * cosine/sine + I3s (R2 in 42)	; 31-34		n 52
	zfmsubpd zmm5, zmm5, zmm25, zmm7	;;31 I3 = I3s * cosine/sine - R3s (I2 in 42)	; 31-34		n 53

no bcast vmovapd zmm25, [sc3reg+3*sc3gap]	;;33 sine
bcast	vbroadcastsd zmm25, Q [bc3reg+3*sc3gap]	;;33 sine for R4/I4 in 41 and 42				n 45
	zfmsubpd zmm7, zmm6, zmm29, zmm12	;;32 R2 = R2 * cosine/sine - I2 (R3/sine in 41)	; 32-35		n 40
	zfmaddpd zmm12, zmm12, zmm29, zmm6	;;32 I2 = I2 * cosine/sine + R2 (I3/sine in 41)	; 32-35		n 41
	bump	sc3reg, sc3inc

	L1prefetchw srcreg+srcinc+1*d1+4*d1, L1pt
	zfmaddpd zmm6, zmm15, zmm29, zmm13	;;32 R3 = R3 * cosine/sine + I3 (R3/sine in 42)	; 33-36		n 50
	zfmsubpd zmm13, zmm13, zmm29, zmm15	;;32 I3 = I3 * cosine/sine - R3 (I3/sine in 42)	; 33-36		n 51

no bcast vmovapd zmm29, [sc4reg+1*128+64]	;;4x cosine/sine
bcast	vbroadcastsd zmm29, Q [bc4reg+1*bcsz+bcsz/2] ;;4x cosine/sine						n 49
	zfmsubpd zmm15, zmm14, zmm28, zmm16	;;33 R2 = R2 * cosine/sine - I2 (R4/sine in 41)	; 34-37		n 45
	zfmaddpd zmm16, zmm16, zmm28, zmm14	;;33 I2 = I2 * cosine/sine + R2 (I4/sine in 41)	; 34-37		n 46

	L1prefetchw srcreg+srcinc+1*d1+8*d1, L1pt
	zfmaddpd zmm14, zmm19, zmm28, zmm17	;;33 R3 = R3 * cosine/sine + I3 (R4/sine in 42)	; 35-38		n 52
	zfmsubpd zmm17, zmm17, zmm28, zmm19	;;33 I3 = I3 * cosine/sine - R3 (I4/sine in 42)	; 35-38		n 53

no bcast vmovapd zmm28, [sc4reg+0*128+64]	;;4x cosine/sine
bcast	vbroadcastsd zmm28, Q [bc4reg+0*bcsz+bcsz/2] ;;4x cosine/sine						n 54
	vaddpd	zmm19, zmm8, zmm20		;;40 R1 + R3 (new R1)				; 36-39		n 43
	vsubpd	zmm8, zmm8, zmm20		;;40 R1 - R3 (new R3)				; 36-39		n 42

	L1prefetchw srcreg+srcinc+1*d1+4*d1+64, L1pt
	vaddpd	zmm20, zmm9, zmm21		;;40 I1 + I3 (new I1)				; 37-40		n 44
	vsubpd	zmm9, zmm9, zmm21		;;40 I1 - I3 (new I3)				; 37-40		n 42

	L1prefetchw srcreg+srcinc+1*d1+8*d1+64, L1pt
	vaddpd	zmm21, zmm10, zmm22		;;40 R2 + R4 (new R2)				; 38-41		n 43
	vsubpd	zmm10, zmm10, zmm22		;;40 R2 - R4 (new R4)				; 38-41		n 48

	L1prefetchw srcreg+srcinc+1*d1, L1pt
	vaddpd	zmm22, zmm11, zmm23		;;40 I2 + I4 (new I2)				; 39-42		n 44
	vsubpd	zmm11, zmm11, zmm23		;;40 I2 - I4 (new I4)				; 39-42		n 47

	L1prefetchw srcreg+srcinc+1*d1+64, L1pt
	zfmaddpd zmm23, zmm7, zmm27, zmm18	;;41 R1 + R3*sine (new R1)			; 40-43		n 58
	zfnmaddpd zmm7, zmm7, zmm27, zmm18	;;41 R1 - R3*sine (new R3)			; 40-43		n 57

	L1prefetchw srcreg+srcinc+2*d1+4*d1, L1pt
	zfmaddpd zmm18, zmm12, zmm27, zmm0	;;41 I1 + I3*sine (new I1)			; 41-44		n 59
	zfnmaddpd zmm12, zmm12, zmm27, zmm0	;;41 I1 - I3*sine (new I3)			; 41-44		n 57

	L1prefetchw srcreg+srcinc+2*d1+8*d1, L1pt
	vmulpd	zmm8, zmm8, zmm26		;;40 R3s = R3 * sine				; 42-45		n 47
	vmulpd	zmm9, zmm9, zmm26		;;40 I3s = I3 * sine				; 42-45		n 48

	L1prefetchw srcreg+srcinc+2*d1+4*d1+64, L1pt
	vaddpd	zmm0, zmm19, zmm21		;;40 R1 + R2 (final R1)				; 43-46
	vsubpd	zmm19, zmm19, zmm21		;;40 R1 - R2 (new R2)				; 43-46		n 49

	L1prefetchw srcreg+srcinc+2*d1+8*d1+64, L1pt
	vaddpd	zmm21, zmm20, zmm22		;;40 I1 + I2 (final I1)				; 44-47
	vsubpd	zmm20, zmm20, zmm22		;;40 I1 - I2 (new I2)				; 44-47		n 49

	L1prefetchw srcreg+srcinc+2*d1, L1pt
	zfmaddpd zmm22, zmm15, zmm25, zmm3	;;41 R2 + R4*sine (new R2)			; 45-48		n 58
	zfnmaddpd zmm15, zmm15, zmm25, zmm3	;;41 R2 - R4*sine (new R4)			; 45-48		n 62

	L1prefetchw srcreg+srcinc+2*d1+64, L1pt
	zfmaddpd zmm3, zmm16, zmm25, zmm4	;;41 I2 + I4*sine (new I2)			; 46-49		n 59
	zfnmaddpd zmm16, zmm16, zmm25, zmm4	;;41 I2 - I4*sine (new I4)			; 46-49		n 61

	L1prefetchw srcreg+srcinc+3*d1+4*d1, L1pt
	zfnmaddpd zmm4, zmm11, zmm26, zmm8	;;40 R3s - I4*sine (new R3s)			; 47-50		n 54
	zfmaddpd zmm11, zmm11, zmm26, zmm8	;;40 R3s + I4*sine (new R4s)			; 47-50		n 55
	zstore	[srcreg+0*d1], zmm0		;;40 Save R1					; 47

no bcast vmovapd zmm0, [sc4reg+1*128]		;;4x Sine
bcast	vbroadcastsd zmm0, Q [bc4reg+1*bcsz]	;;4x Sine							n 56
	zfmaddpd zmm8, zmm10, zmm26, zmm9	;;40 I3s + R4*sine (new I3s)			; 48-51		n 54
	zfnmaddpd zmm10, zmm10, zmm26, zmm9	;;40 I3s - R4*sine (new I4s)			; 48-51		n 55
	zstore	[srcreg+0*d1+64], zmm21		;;40 Save I1					; 48

	L1prefetchw srcreg+srcinc+3*d1+8*d1, L1pt
	zfmsubpd zmm9, zmm19, zmm29, zmm20	;;40 A2 = R2 * cosine/sine - I2			; 49-52		n 56
	zfmaddpd zmm20, zmm20, zmm29, zmm19	;;40 B2 = I2 * cosine/sine + R2			; 49-52		n 56
	bump	sc4reg, sc4inc

	L1prefetchw srcreg+srcinc+3*d1+4*d1+64, L1pt
	zfmaddpd zmm19, zmm6, zmm27, zmm24	;;42 R1 + R3*sine (new R1)			; 50-53		n 64
	zfnmaddpd zmm6, zmm6, zmm27, zmm24	;;42 R1 - R3*sine (new R3)			; 50-53		n 60

	L1prefetchw srcreg+srcinc+3*d1+8*d1+64, L1pt
	zfmaddpd zmm24, zmm13, zmm27, zmm1	;;42 I1 + I3*sine (new I1)			; 51-54		n 65
	zfnmaddpd zmm13, zmm13, zmm27, zmm1	;;42 I1 - I3*sine (new I3)			; 51-54		n 60

	L1prefetchw srcreg+srcinc+3*d1, L1pt
	zfmaddpd zmm1, zmm14, zmm25, zmm2	;;42 R2 + R4*sine (new R2)			; 52-55		n 64
	zfnmaddpd zmm14, zmm14, zmm25, zmm2	;;42 R2 - R4*sine (new R4)			; 52-55		n 68

	L1prefetchw srcreg+srcinc+3*d1+64, L1pt
	zfmaddpd zmm2, zmm17, zmm25, zmm5	;;42 I2 + I4*sine (new I2)			; 53-56		n 65
	zfnmaddpd zmm17, zmm17, zmm25, zmm5	;;42 I2 - I4*sine (new I4)			; 53-56		n 66

	zfmsubpd zmm5, zmm4, zmm28, zmm8	;;40 R3s * cosine/sine - I3s (final R3)		; 54-57
	zfmaddpd zmm8, zmm8, zmm28, zmm4	;;40 I3s * cosine/sine + R3s (final I3)		; 54-57

	zfmaddpd zmm4, zmm11, zmm28, zmm10	;;40 R4s * cosine/sine + I4s (final R4)		; 55-58
	zfmsubpd zmm10, zmm10, zmm28, zmm11	;;40 I4s * cosine/sine - R4s (final I4)		; 55-58

	vmulpd	zmm9, zmm9, zmm0		;;40 A2 = A2 * sine (final R2)			; 56-59
	vmulpd	zmm20, zmm20, zmm0		;;40 B2 = B2 * sine (final I2)			; 56-59

	vmulpd	zmm7, zmm7, zmm26		;;41 R3s = R3 * sine				; 57-60		n 61
	vmulpd	zmm12, zmm12, zmm26		;;41 I3s = I3 * sine				; 57-60		n 62

	vaddpd	zmm11, zmm23, zmm22		;;41 R1 + R2 (final R1)				; 58-61
	vsubpd	zmm23, zmm23, zmm22		;;41 R1 - R2 (new R2)				; 58-61		n 63
	zstore	[srcreg+0*d1+2*d1], zmm5	;;40 Save R3					; 58

	vaddpd	zmm22, zmm18, zmm3		;;41 I1 + I2 (final I1)				; 59-62
	vsubpd	zmm18, zmm18, zmm3		;;41 I1 - I2 (new I2)				; 59-62		n 63
	zstore	[srcreg+0*d1+2*d1+64], zmm8	;;40 Save I3					; 59

	vmulpd	zmm6, zmm6, zmm26		;;42 R3s = R3 * sine				; 60-63		n 66
	vmulpd	zmm13, zmm13, zmm26		;;42 I3s = I3 * sine				; 60-63		n 68
	zstore	[srcreg+0*d1+3*d1], zmm4	;;40 Save R4					; 60

	zfnmaddpd zmm3, zmm16, zmm26, zmm7	;;41 R3s - I4*sine (new R3s)			; 61-64		n 67
	zfmaddpd zmm16, zmm16, zmm26, zmm7	;;41 R3s + I4*sine (new R4s)			; 61-64		n 69
	zstore	[srcreg+0*d1+3*d1+64], zmm10	;;40 Save I4					; 61

	zfmaddpd zmm7, zmm15, zmm26, zmm12	;;41 I3s + R4*sine (new I3s)			; 62-65		n 67
	zfnmaddpd zmm15, zmm15, zmm26, zmm12	;;41 I3s - R4*sine (new I4s)			; 62-65		n 69
	zstore	[srcreg+0*d1+d1], zmm9		;;40 Save R2					; 62

	zfmsubpd zmm12, zmm23, zmm29, zmm18	;;41 A2 = R2 * cosine/sine - I2			; 63-66		n 71
	zfmaddpd zmm18, zmm18, zmm29, zmm23	;;41 B2 = I2 * cosine/sine + R2			; 63-66		n 71
	zstore	[srcreg+0*d1+d1+64], zmm20	;;40 Save I2					; 63

	vaddpd	zmm23, zmm19, zmm1		;;42 R1 + R2 (final R1)				; 64-67
	vsubpd	zmm19, zmm19, zmm1		;;42 R1 - R2 (new R2)				; 64-67		n 70
	zstore	[srcreg+4*d1], zmm11		;;41 Save R1					; 64

	vaddpd	zmm1, zmm24, zmm2		;;42 I1 + I2 (final I1)				; 65-68
	vsubpd	zmm24, zmm24, zmm2		;;42 I1 - I2 (new I2)				; 65-68		n 70
	zstore	[srcreg+4*d1+64], zmm22		;;41 Save I1					; 65

	zfnmaddpd zmm2, zmm17, zmm26, zmm6	;;42 R3s - I4*sine (new R3s)			; 66-69		n 72
	zfmaddpd zmm17, zmm17, zmm26, zmm6	;;42 R3s + I4*sine (new R4s)			; 66-69		n 73

	zfmsubpd zmm6, zmm3, zmm28, zmm7	;;41 R3s * cosine/sine - I3s (final R3)		; 67-70
	zfmaddpd zmm7, zmm7, zmm28, zmm3	;;41 I3s * cosine/sine + R3s (final I3)		; 67-70

	zfmaddpd zmm3, zmm14, zmm26, zmm13	;;42 I3s + R4*sine (new I3s)			; 68-71		n 72
	zfnmaddpd zmm14, zmm14, zmm26, zmm13	;;42 I3s - R4*sine (new I4s)			; 68-71		n 73
	zstore	[srcreg+8*d1], zmm23		;;42 Save R1					; 68

	zfmaddpd zmm13, zmm16, zmm28, zmm15	;;41 R4s * cosine/sine + I4s (final R4)		; 69-72
	zfmsubpd zmm15, zmm15, zmm28, zmm16	;;41 I4s * cosine/sine - R4s (final I4)		; 69-72
	zstore	[srcreg+8*d1+64], zmm1		;;42 Save I1					; 69

	zfmsubpd zmm16, zmm19, zmm29, zmm24	;;42 A2 = R2 * cosine/sine - I2			; 70-73		n 74
	zfmaddpd zmm24, zmm24, zmm29, zmm19	;;42 B2 = I2 * cosine/sine + R2			; 70-73		n 74

	vmulpd	zmm12, zmm12, zmm0		;;41 A2 = A2 * sine (final R2)			; 71-74
	vmulpd	zmm18, zmm18, zmm0		;;41 B2 = B2 * sine (final I2)			; 71-74
	zstore	[srcreg+4*d1+2*d1], zmm6	;;41 Save R3					; 71

	zfmsubpd zmm19, zmm2, zmm28, zmm3	;;42 R3s * cosine/sine - I3s (final R3)		; 72-75
	zfmaddpd zmm3, zmm3, zmm28, zmm2	;;42 I3s * cosine/sine + R3s (final I3)		; 72-75
	zstore	[srcreg+4*d1+2*d1+64], zmm7	;;41 Save I3					; 72

	zfmaddpd zmm2, zmm17, zmm28, zmm14	;;42 R4s * cosine/sine + I4s (final R4)		; 73-76
	zfmsubpd zmm14, zmm14, zmm28, zmm17	;;42 I4s * cosine/sine - R4s (final I4)		; 73-76
	zstore	[srcreg+4*d1+3*d1], zmm13	;;41 Save R4					; 73

	vmulpd	zmm16, zmm16, zmm0		;;42 A2 = A2 * sine (final R2)			; 74-77
	vmulpd	zmm24, zmm24, zmm0		;;42 B2 = B2 * sine (final I2)			; 74-77
	zstore	[srcreg+4*d1+3*d1+64], zmm15	;;41 Save I4					; 74

	zstore	[srcreg+4*d1+d1], zmm12		;;41 Save R2
	zstore	[srcreg+4*d1+d1+64], zmm18	;;41 Save I2
	zstore	[srcreg+8*d1+2*d1], zmm19	;;42 Save R3
	zstore	[srcreg+8*d1+2*d1+64], zmm3	;;42 Save I3
	zstore	[srcreg+8*d1+3*d1], zmm2	;;42 Save R4
	zstore	[srcreg+8*d1+3*d1+64], zmm14	;;42 Save I4
	zstore	[srcreg+8*d1+d1], zmm16		;;42 Save R2
	zstore	[srcreg+8*d1+d1+64], zmm24	;;42 Save I2
	bump	srcreg, srcinc
	ENDM

;;
;; ************************************* One-pass variant of twelve-complex-djbunfft ******************************************
;; Implements twelve complex in three and four complex chunks.  Uses a three and four complex sin/cos table.
;; This lets one pass FFTs get much of the benefits of 12-complex without writing the difficult twelve_complex_twenty_four_reals macros
;;

;; The standard version
zr34_twelve_complex_djbunfft_preload MACRO
	zr34_12c_djbunfft_cmn_preload
	ENDM
zr34_twelve_complex_djbunfft MACRO srcreg,srcinc,d1,sc4reg,sc4gap,sc4inc,sc3reg,sc3gap,sc3inc,maxrpt,L1pt,L1pd
	zr34_12c_djbunfft_cmn srcreg,srcinc,d1,noexec,0,0,0,sc4reg,sc4gap,sc4inc,sc3reg,sc3gap,sc3inc,maxrpt,L1pt,L1pd
	ENDM

; Like the standard version except vbroadcastsd is used to reduce sin/cos data
zr34b_twelve_complex_djbunfft_preload MACRO
	zr34_12c_djbunfft_cmn_preload
	ENDM
zr34b_twelve_complex_djbunfft MACRO srcreg,srcinc,d1,sc4reg,sc4gap,sc4inc,sc3reg,sc3gap,sc3inc,maxrpt,L1pt,L1pd
	zr34_12c_djbunfft_cmn srcreg,srcinc,d1,exec,sc4reg,sc3reg,16,sc4reg,sc4gap,sc4inc,sc3reg,sc3gap,sc3inc,maxrpt,L1pt,L1pd
	ENDM

; Like the zr34b version except sin/cos data is loaded from a larger real sin/cos table
zr34rb_twelve_complex_djbunfft_preload MACRO
	zr34_12c_djbunfft_cmn_preload
	ENDM
zr34rb_twelve_complex_djbunfft MACRO srcreg,srcinc,d1,sc4reg,sc4gap,sc4inc,sc3reg,sc3gap,sc3inc,maxrpt,L1pt,L1pd
	zr34_12c_djbunfft_cmn srcreg,srcinc,d1,exec,sc4reg+8,sc3reg+8,128,sc4reg,sc4gap,sc4inc,sc3reg,sc3gap,sc3inc,maxrpt,L1pt,L1pd
	ENDM

zr34_12c_djbunfft_cmn_preload MACRO
	vbroadcastsd zmm31, ZMM_P866
	vbroadcastsd zmm30, ZMM_HALF
	ENDM
zr34_12c_djbunfft_cmn MACRO srcreg,srcinc,d1,bcast,bc4reg,bc3reg,bcsz,sc4reg,sc4gap,sc4inc,sc3reg,sc3gap,sc3inc,maxrpt,L1pt,L1pd
	IF sc4gap NE 0
	need_code_for_non_zero_sc4gap
	ENDIF
no bcast vmovapd zmm29, [sc4reg+1*128+64]	;;4x cosine/sine
bcast	vbroadcastsd zmm29, Q [bc4reg+1*bcsz++bcsz/2] ;;4x cosine/sine
	vmovapd	zmm2, [srcreg+4*d1+d1]		;;41 R2
	vmovapd	zmm1, [srcreg+4*d1+d1+64]	;;41 I2
	zfmaddpd zmm0, zmm2, zmm29, zmm1	;;41 A2 = R2 * cosine/sine + I2		; 1-4		n 5
	zfmsubpd zmm1, zmm1, zmm29, zmm2	;;41 B2 = I2 * cosine/sine - R2		; 1-4		n 6

	vmovapd	zmm4, [srcreg+8*d1+d1]		;;42 R2
	vmovapd	zmm3, [srcreg+8*d1+d1+64]	;;42 I2
	zfmaddpd zmm2, zmm4, zmm29, zmm3	;;42 A2 = R2 * cosine/sine + I2		; 2-5		n 7
	zfmsubpd zmm3, zmm3, zmm29, zmm4	;;42 B2 = I2 * cosine/sine - R2		; 2-5		n 8

no bcast vmovapd zmm28, [sc4reg+0*128+64]	;;4x cosine/sine
bcast	vbroadcastsd zmm28, Q [bc4reg+0*bcsz++bcsz/2] ;;4x cosine/sine
	vmovapd	zmm6, [srcreg+4*d1+2*d1]	;;41 R3
	vmovapd	zmm5, [srcreg+4*d1+2*d1+64]	;;41 I3
	zfmaddpd zmm4, zmm6, zmm28, zmm5	;;41 A3 = R3 * cosine/sine + I3		; 3-6		n 9
	zfmsubpd zmm5, zmm5, zmm28, zmm6	;;41 B3 = I3 * cosine/sine - R3		; 3-6		n 10

	vmovapd	zmm8, [srcreg+4*d1+3*d1]	;;41 R4
	vmovapd	zmm7, [srcreg+4*d1+3*d1+64]	;;41 I4
	zfmsubpd zmm6, zmm8, zmm28, zmm7	;;41 A4 = R4 * cosine/sine - I4		; 4-7		n 9
	zfmaddpd zmm7, zmm7, zmm28, zmm8	;;41 B4 = I4 * cosine/sine + R4		; 4-7		n 10

no bcast vmovapd zmm27, [sc4reg+1*128]		;;4x Sine
bcast	vbroadcastsd zmm27, Q [bc4reg+1*bcsz]	;;4x Sine
	vmovapd	zmm9, [srcreg+4*d1]		;;41 R1
	zfmaddpd zmm8, zmm0, zmm27, zmm9	;;41 R1 + R2*sine (new R1)		; 5-8		n 13
	zfnmaddpd zmm0, zmm0, zmm27, zmm9	;;41 R1 - R2*sine (new R2)		; 5-8		n 14

	vmovapd	zmm10, [srcreg+4*d1+64]		;;41 I1
	zfnmaddpd zmm9, zmm1, zmm27, zmm10	;;41 I1 - I2*sine (new I2)		; 6-9		n 15
	zfmaddpd zmm1, zmm1, zmm27, zmm10	;;41 I1 + I2*sine (new I1)		; 6-9		n 16

	vmovapd	zmm11, [srcreg+8*d1]		;;42 R1
	zfmaddpd zmm10, zmm2, zmm27, zmm11	;;42 R1 + R2*sine (new R1)		; 7-10		n 26
	zfnmaddpd zmm2, zmm2, zmm27, zmm11	;;42 R1 - R2*sine (new R2)		; 7-10		n 27

	vmovapd	zmm12, [srcreg+8*d1+64]		;;42 I1
	zfnmaddpd zmm11, zmm3, zmm27, zmm12	;;42 I1 - I2*sine (new I2)		; 8-11		n 28
	zfmaddpd zmm3, zmm3, zmm27, zmm12	;;42 I1 + I2*sine (new I1)		; 8-11		n 29

	vmovapd	zmm14, [srcreg+8*d1+2*d1]	;;42 R3							n 11
	vaddpd	zmm12, zmm6, zmm4		;;41 R4 + R3 (new R3)			; 9-12		n 13
	vsubpd	zmm6, zmm6, zmm4		;;41 R4 - R3 (new I4)			; 9-12		n 15

	vmovapd	zmm13, [srcreg+8*d1+2*d1+64]	;;42 I3							n 11
	vsubpd	zmm4, zmm5, zmm7		;;41 I3 - I4 (new R4)			; 10-13		n 14
	vaddpd	zmm5, zmm5, zmm7		;;41 I3 + I4 (new I3)			; 10-13		n 16

	vmovapd	zmm16, [srcreg+8*d1+3*d1]	;;42 R4							n 12
	zfmaddpd zmm7, zmm14, zmm28, zmm13	;;42 A3 = R3 * cosine/sine + I3		; 11-14		n 20
	zfmsubpd zmm13, zmm13, zmm28, zmm14	;;42 B3 = I3 * cosine/sine - R3		; 11-14		n 21

	vmovapd	zmm15, [srcreg+8*d1+3*d1+64]	;;42 I4							n 12
	zfmsubpd zmm14, zmm16, zmm28, zmm15	;;42 A4 = R4 * cosine/sine - I4		; 12-15		n 20
	zfmaddpd zmm15, zmm15, zmm28, zmm16	;;42 B4 = I4 * cosine/sine + R4		; 12-15		n 21

no bcast vmovapd zmm26, [sc4reg+0*128]		;;4x Sine
bcast	vbroadcastsd zmm26, Q [bc4reg+0*bcsz]	;;4x Sine						n 13
	zfmaddpd zmm16, zmm12, zmm26, zmm8	;;41 R1 + R3*sine (R2 in 30)		; 13-16		n 34
	zfnmaddpd zmm12, zmm12, zmm26, zmm8	;;41 R1 - R3*sine (R2 in 32)		; 13-16		n 52
	bump	sc4reg, sc4inc

	vmovapd	zmm18, [srcreg+d1]		;;40 R2							n 17
	zfmaddpd zmm8, zmm4, zmm26, zmm0	;;41 R2 + R4*sine (R2 in 31)		; 14-17		n 36
	zfnmaddpd zmm4, zmm4, zmm26, zmm0	;;41 R2 - R4*sine (R2 in 33)		; 14-17		n 54

	vmovapd	zmm17, [srcreg+d1+64]		;;40 I2							n 17
	zfmaddpd zmm0, zmm6, zmm26, zmm9	;;41 I2 + I4*sine (I2 in 31)		; 15-18		n 36
	zfnmaddpd zmm6, zmm6, zmm26, zmm9	;;41 I2 - I4*sine (I2 in 33)		; 15-18		n 54

	vmovapd	zmm20, [srcreg+2*d1]		;;40 R3							n 18
	zfmaddpd zmm9, zmm5, zmm26, zmm1	;;41 I1 + I3*sine (I2 in 30)		; 16-19		n 34
	zfnmaddpd zmm5, zmm5, zmm26, zmm1	;;41 I1 - I3*sine (I2 in 32)		; 16-19		n 52

	vmovapd	zmm19, [srcreg+2*d1+64]		;;40 I3							n 18
	zfmaddpd zmm1, zmm18, zmm29, zmm17	;;40 A2 = R2 * cosine/sine + I2		; 17-20		n 22
	zfmsubpd zmm17, zmm17, zmm29, zmm18	;;40 B2 = I2 * cosine/sine - R2		; 17-20		n 23

	vmovapd	zmm22, [srcreg+3*d1]		;;40 R4							n 19
	zfmaddpd zmm18, zmm20, zmm28, zmm19	;;40 A3 = R3 * cosine/sine + I3		; 18-21		n 24
	zfmsubpd zmm19, zmm19, zmm28, zmm20	;;40 B3 = I3 * cosine/sine - R3		; 18-21		n 25

	vmovapd	zmm21, [srcreg+3*d1+64]		;;40 I4							n 19
	zfmsubpd zmm20, zmm22, zmm28, zmm21	;;40 A4 = R4 * cosine/sine - I4		; 19-22		n 24
	zfmaddpd zmm21, zmm21, zmm28, zmm22	;;40 B4 = I4 * cosine/sine + R4		; 19-22		n 25

	vmovapd	zmm23, [srcreg]			;;40 R1							n 22
	vaddpd	zmm22, zmm14, zmm7		;;42 R4 + R3 (new R3)			; 20-23		n 26
	vsubpd	zmm14, zmm14, zmm7		;;42 R4 - R3 (new I4)			; 20-23		n 28

	vmovapd	zmm24, [srcreg+64]		;;40 I1							n 23
	vsubpd	zmm7, zmm13, zmm15		;;42 I3 - I4 (new R4)			; 21-24		n 27
	vaddpd	zmm13, zmm13, zmm15		;;42 I3 + I4 (new I3)			; 21-24		n 29

no bcast vmovapd zmm25, [sc3reg+0*sc3gap+64]	;;30 cosine/sine
bcast	vbroadcastsd zmm25, Q [bc3reg+0*sc3gap+bcsz/2] ;;30 cosine/sine					n 34
	zfmaddpd zmm15, zmm1, zmm27, zmm23	;;40 R1 + R2*sine (new R1)		; 22-25		n 30
	zfnmaddpd zmm1, zmm1, zmm27, zmm23	;;40 R1 - R2*sine (new R2)		; 22-25		n 31

no bcast vmovapd zmm29, [sc3reg+1*sc3gap+64]	;;31 cosine/sine
bcast	vbroadcastsd zmm29, Q [bc3reg+1*sc3gap+bcsz/2] ;;31 cosine/sine					n 36
	zfnmaddpd zmm23, zmm17, zmm27, zmm24	;;40 I1 - I2*sine (new I2)		; 23-26		n 32
	zfmaddpd zmm17, zmm17, zmm27, zmm24	;;40 I1 + I2*sine (new I1)		; 23-26		n 33

no bcast vmovapd zmm28, [sc3reg+0*sc3gap]	;;30 sine
bcast	vbroadcastsd zmm28, Q [bc3reg+0*sc3gap]	;;30 sine						n 38
	vaddpd	zmm24, zmm20, zmm18		;;40 R4 + R3 (new R3)			; 24-27		n 30
	vsubpd	zmm20, zmm20, zmm18		;;40 R4 - R3 (new I4)			; 24-27		n 32

no bcast vmovapd zmm27, [sc3reg+1*sc3gap]	;;31 sine
bcast	vbroadcastsd zmm27, Q [bc3reg+1*sc3gap]	;;31 sine						n 41
	vsubpd	zmm18, zmm19, zmm21		;;40 I3 - I4 (new R4)			; 25-28		n 31
	vaddpd	zmm19, zmm19, zmm21		;;40 I3 + I4 (new I3)			; 25-28		n 33

	L1prefetchw srcreg+srcinc+4*d1+d1, L1pt
	zfmaddpd zmm21, zmm22, zmm26, zmm10	;;42 R1 + R3*sine (R3 in 30)		; 26-29		n 35
	zfnmaddpd zmm22, zmm22, zmm26, zmm10	;;42 R1 - R3*sine (R3 in 32)		; 26-29		n 53

	L1prefetchw srcreg+srcinc+4*d1+d1+64, L1pt
	zfmaddpd zmm10, zmm7, zmm26, zmm2	;;42 R2 + R4*sine (R3 in 31)		; 27-30		n 37
	zfnmaddpd zmm7, zmm7, zmm26, zmm2	;;42 R2 - R4*sine (R3 in 33)		; 27-30		n 54

	L1prefetchw srcreg+srcinc+8*d1+d1, L1pt
	zfmaddpd zmm2, zmm14, zmm26, zmm11	;;42 I2 + I4*sine (I3 in 31)		; 28-31		n 37
	zfnmaddpd zmm14, zmm14, zmm26, zmm11	;;42 I2 - I4*sine (I3 in 33)		; 28-31		n 54

	L1prefetchw srcreg+srcinc+8*d1+d1+64, L1pt
	zfmaddpd zmm11, zmm13, zmm26, zmm3	;;42 I1 + I3*sine (I3 in 30)		; 29-32		n 35
	zfnmaddpd zmm13, zmm13, zmm26, zmm3	;;42 I1 - I3*sine (I3 in 32)		; 29-32		n 53

	L1prefetchw srcreg+srcinc+4*d1+2*d1, L1pt
	zfmaddpd zmm3, zmm24, zmm26, zmm15	;;40 R1 + R3*sine (R1 in 30)		; 30-33		n 44
	zfnmaddpd zmm24, zmm24, zmm26, zmm15	;;40 R1 - R3*sine (R1 in 32)		; 30-33		n 62

	L1prefetchw srcreg+srcinc+4*d1+2*d1+64, L1pt
	zfmaddpd zmm15, zmm18, zmm26, zmm1	;;40 R2 + R4*sine (R1 in 31)		; 31-34		n 46
	zfnmaddpd zmm18, zmm18, zmm26, zmm1	;;40 R2 - R4*sine (R1 in 33)		; 31-34		n 64

	L1prefetchw srcreg+srcinc+4*d1+3*d1, L1pt
	zfmaddpd zmm1, zmm20, zmm26, zmm23	;;40 I2 + I4*sine (I1 in 31)		; 32-35		n 47
	zfnmaddpd zmm20, zmm20, zmm26, zmm23	;;40 I2 - I4*sine (I1 in 33)		; 32-35		n 65

	L1prefetchw srcreg+srcinc+4*d1+3*d1+64, L1pt
	zfmaddpd zmm23, zmm19, zmm26, zmm17	;;40 I1 + I3*sine (I1 in 30)		; 33-36		n 45
	zfnmaddpd zmm19, zmm19, zmm26, zmm17	;;40 I1 - I3*sine (I1 in 32)		; 33-36		n 63

	L1prefetchw srcreg+srcinc+4*d1, L1pt
	zfmaddpd zmm17, zmm16, zmm25, zmm9	;;30 A2 = R2 * cosine/sine + I2		; 34-37		n 39
	zfmsubpd zmm9, zmm9, zmm25, zmm16	;;30 B2 = I2 * cosine/sine - R2		; 34-37		n 40

	L1prefetchw srcreg+srcinc+4*d1+64, L1pt
	zfmsubpd zmm16, zmm21, zmm25, zmm11	;;30 A3 = R3 * cosine/sine - I3		; 35-38		n 39
	zfmaddpd zmm11, zmm11, zmm25, zmm21	;;30 B3 = I3 * cosine/sine + R3		; 35-38		n 40

	L1prefetchw srcreg+srcinc+8*d1, L1pt
	zfmaddpd zmm21, zmm8, zmm29, zmm0	;;31 A2 = R2 * cosine/sine + I2		; 36-39		n 42
	zfmsubpd zmm0, zmm0, zmm29, zmm8	;;31 B2 = I2 * cosine/sine - R2		; 36-39		n 43

	L1prefetchw srcreg+srcinc+8*d1+64, L1pt
	zfmsubpd zmm8, zmm10, zmm29, zmm2	;;31 A3 = R3 * cosine/sine - I3		; 37-40		n 42
	zfmaddpd zmm2, zmm2, zmm29, zmm10	;;31 B3 = I3 * cosine/sine + R3		; 37-40		n 43

	L1prefetchw srcreg+srcinc+8*d1+2*d1, L1pt
	vmulpd	zmm29, zmm30, zmm28		;;30 500s = .5*sine			; 38-41		n 44
	vmulpd	zmm25, zmm31, zmm28		;;30 866s = .866*sine			; 38-41		n 48

	L1prefetchw srcreg+srcinc+8*d1+2*d1+64, L1pt
	vaddpd	zmm10, zmm17, zmm16		;;30 r2+ = R2 + R3			; 39-42		n 44
	vsubpd	zmm17, zmm17, zmm16		;;30 r2- = R2 - R3			; 39-42		n 49

	L1prefetchw srcreg+srcinc+8*d1+3*d1, L1pt
	vaddpd	zmm16, zmm9, zmm11		;;30 i2+ = I2 + I3			; 40-43		n 45
	vsubpd	zmm9, zmm9, zmm11		;;30 i2- = I2 - I3			; 40-43		n 48

	L1prefetchw srcreg+srcinc+8*d1+3*d1+64, L1pt
	vmulpd	zmm26, zmm30, zmm27		;;31 500s = .5*sine			; 41-44		n 46
	vmulpd	zmm31, zmm31, zmm27		;;31 866s = .866*sine			; 41-44		n 50

	L1prefetchw srcreg+srcinc+d1, L1pt
	vaddpd	zmm11, zmm21, zmm8		;;31 r2+ = R2 + R3			; 42-45		n 46
	vsubpd	zmm21, zmm21, zmm8		;;31 r2- = R2 - R3			; 42-45		n 51

	L1prefetchw srcreg+srcinc+d1+64, L1pt
	vaddpd	zmm8, zmm0, zmm2		;;31 i2+ = I2 + I3			; 43-46		n 47
	vsubpd	zmm0, zmm0, zmm2		;;31 i2- = I2 - I3			; 43-46		n 50

	L1prefetchw srcreg+srcinc+2*d1, L1pt
	zfmaddpd zmm2, zmm10, zmm28, zmm3	;;30 R1 = R1 + sine*r2+			; 44-47
	zfnmaddpd zmm10, zmm10, zmm29, zmm3	;;30 r1- = R1 - 500s*r2+		; 44-47		n 48

	L1prefetchw srcreg+srcinc+2*d1+64, L1pt
	zfmaddpd zmm3, zmm16, zmm28, zmm23	;;30 I1 = I1 + sine*i2+			; 45-48
	zfnmaddpd zmm16, zmm16, zmm29, zmm23	;;30 i1- = I1 - 500s*i2+		; 45-48		n 49

no bcast vmovapd zmm28, [sc3reg+2*sc3gap+64]	;;32 cosine/sine
bcast	vbroadcastsd zmm28, Q [bc3reg+2*sc3gap+bcsz/2] ;;32 cosine/sine					n 52
	zfmaddpd zmm23, zmm11, zmm27, zmm15	;;31 R1 = R1 + sine*r2+			; 46-49
	zfnmaddpd zmm11, zmm11, zmm26, zmm15	;;31 r1- = R1 - 500s*r2+		; 46-49		n 50

no bcast vmovapd zmm29, [sc3reg+3*sc3gap+64]	;;33 cosine/sine
bcast	vbroadcastsd zmm29, Q [bc3reg+3*sc3gap+bcsz/2] ;;33 cosine/sine					n 54
	zfmaddpd zmm15, zmm8, zmm27, zmm1	;;31 I1 = I1 + sine*i2+			; 47-50
	zfnmaddpd zmm8, zmm8, zmm26, zmm1	;;31 i1- = I1 - 500s*i2+		; 47-50		n 51

no bcast vmovapd zmm27, [sc3reg+2*sc3gap]	;;32 sine
bcast	vbroadcastsd zmm27, Q [bc3reg+2*sc3gap]	;;32 sine						n 56
	zfmaddpd zmm1, zmm9, zmm25, zmm10	;;30 R2 = r1- + 866s*i2-		; 48-51
	zfnmaddpd zmm9, zmm9, zmm25, zmm10	;;30 R3 = r1- - 866s*i2-		; 48-51
	zstore	[srcreg], zmm2			;;30 Save R1				; 48

no bcast vmovapd zmm26, [sc3reg+3*sc3gap]	;;33 sine
bcast	vbroadcastsd zmm26, Q [bc3reg+3*sc3gap]	;;33 sine						n 59
	zfnmaddpd zmm10, zmm17, zmm25, zmm16	;;30 I2 = i1- - 866s*r2-		; 49-52
	zfmaddpd zmm17, zmm17, zmm25, zmm16	;;30 I3 = i1- + 866s*r2-		; 49-52
	zstore	[srcreg+64], zmm3		;;30 Save I1				; 49
	bump	sc3reg, sc3inc

	L1prefetchw srcreg+srcinc+3*d1, L1pt
	zfmaddpd zmm16, zmm0, zmm31, zmm11	;;31 R2 = r1- + 866s*i2-		; 50-53
	zfnmaddpd zmm0, zmm0, zmm31, zmm11	;;31 R3 = r1- - 866s*i2-		; 50-53
	zstore	[srcreg+d1], zmm23		;;31 Save R1				; 50

	L1prefetchw srcreg+srcinc+3*d1+64, L1pt
	zfnmaddpd zmm11, zmm21, zmm31, zmm8	;;31 I2 = i1- - 866s*r2-		; 51-54
	zfmaddpd zmm21, zmm21, zmm31, zmm8	;;31 I3 = i1- + 866s*r2-		; 51-54
	zstore	[srcreg+d1+64], zmm15		;;31 Save I1				; 51

	L1prefetchw srcreg+srcinc, L1pt
	zfmaddpd zmm8, zmm12, zmm28, zmm5	;;32 A2 = R2 * cosine/sine + I2		; 52-55		n 57
	zfmsubpd zmm5, zmm5, zmm28, zmm12	;;32 B2 = I2 * cosine/sine - R2		; 52-55		n 58
	zstore	[srcreg+4*d1], zmm1		;;30 Save R2				; 52

	L1prefetchw srcreg+srcinc+64, L1pt
	zfmsubpd zmm12, zmm22, zmm28, zmm13	;;32 A3 = R3 * cosine/sine - I3		; 53-56		n 57
	zfmaddpd zmm13, zmm13, zmm28, zmm22	;;32 B3 = I3 * cosine/sine + R3		; 53-56		n 58
	zstore	[srcreg+8*d1], zmm9		;;30 Save R3				; 53

	zfmaddpd zmm22, zmm4, zmm29, zmm6	;;33 A2 = R2 * cosine/sine + I2		; 54-57		n 60
	zfmsubpd zmm6, zmm6, zmm29, zmm4	;;33 B2 = I2 * cosine/sine - R2		; 54-57		n 61
	zstore	[srcreg+4*d1+64], zmm10		;;30 Save I2				; 54

	zfmsubpd zmm4, zmm7, zmm29, zmm14	;;33 A3 = R3 * cosine/sine - I3		; 55-58		n 60
	zfmaddpd zmm14, zmm14, zmm29, zmm7	;;33 B3 = I3 * cosine/sine + R3		; 55-58		n 61
	zstore	[srcreg+8*d1+64], zmm17		;;30 Save I3				; 55

	vbroadcastsd zmm31, ZMM_P866		;;3x Reload .866					n 56
	vmulpd	zmm29, zmm30, zmm27		;;32 500s = .5*sine			; 56-59		n 62
	vmulpd	zmm28, zmm31, zmm27		;;32 866s = .866*sine			; 56-59		n 66
	zstore	[srcreg+d1+4*d1], zmm16		;;31 Save R2				; 56

	vaddpd	zmm7, zmm8, zmm12		;;32 r2+ = R2 + R3			; 57-60		n 62
	vsubpd	zmm8, zmm8, zmm12		;;32 r2- = R2 - R3			; 57-60		n 67
	zstore	[srcreg+d1+8*d1], zmm0		;;31 Save R3				; 57

	vaddpd	zmm12, zmm5, zmm13		;;32 i2+ = I2 + I3			; 58-61		n 63
	vsubpd	zmm5, zmm5, zmm13		;;32 i2- = I2 - I3			; 58-61		n 66
	zstore	[srcreg+d1+4*d1+64], zmm11	;;31 Save I2				; 58

	vmulpd	zmm25, zmm30, zmm26		;;33 500s = .5*sine			; 59-62		n 64
	vmulpd	zmm2, zmm31, zmm26		;;33 866s = .866*sine			; 59-62		n 68
	zstore	[srcreg+d1+8*d1+64], zmm21	;;31 Save I3				; 59

	vaddpd	zmm13, zmm22, zmm4		;;33 r2+ = R2 + R3			; 60-63		n 64
	vsubpd	zmm22, zmm22, zmm4		;;33 r2- = R2 - R3			; 60-63		n 69

	vaddpd	zmm4, zmm6, zmm14		;;33 i2+ = I2 + I3			; 61-64		n 65
	vsubpd	zmm6, zmm6, zmm14		;;33 i2- = I2 - I3			; 61-64		n 68

	zfmaddpd zmm14, zmm7, zmm27, zmm24	;;32 R1 = R1 + sine*r2+			; 62-65
	zfnmaddpd zmm7, zmm7, zmm29, zmm24	;;32 r1- = R1 - 500s*r2+		; 62-65		n 66

	zfmaddpd zmm24, zmm12, zmm27, zmm19	;;32 I1 = I1 + sine*i2+			; 63-66
	zfnmaddpd zmm12, zmm12, zmm29, zmm19	;;32 i1- = I1 - 500s*i2+		; 63-66		n 67

	zfmaddpd zmm19, zmm13, zmm26, zmm18	;;33 R1 = R1 + sine*r2+			; 64-67
	zfnmaddpd zmm13, zmm13, zmm25, zmm18	;;33 r1- = R1 - 500s*r2+		; 64-67		n 68

	zfmaddpd zmm18, zmm4, zmm26, zmm20	;;33 I1 = I1 + sine*i2+			; 65-68
	zfnmaddpd zmm4, zmm4, zmm25, zmm20	;;33 i1- = I1 - 500s*i2+		; 65-68		n 69

	zfmaddpd zmm20, zmm5, zmm28, zmm7	;;32 R2 = r1- + 866s*i2-		; 66-69
	zfnmaddpd zmm5, zmm5, zmm28, zmm7	;;32 R3 = r1- - 866s*i2-		; 66-69
	zstore	[srcreg+2*d1], zmm14		;;32 Save R1				; 66

	zfnmaddpd zmm7, zmm8, zmm28, zmm12	;;32 I2 = i1- - 866s*r2-		; 67-70
	zfmaddpd zmm8, zmm8, zmm28, zmm12	;;32 I3 = i1- + 866s*r2-		; 67-70
	zstore	[srcreg+2*d1+64], zmm24		;;32 Save I1				; 67

	zfmaddpd zmm12, zmm6, zmm2, zmm13	;;33 R2 = r1- + 866s*i2-		; 68-71
	zfnmaddpd zmm6, zmm6, zmm2, zmm13	;;33 R3 = r1- - 866s*i2-		; 68-71
	zstore	[srcreg+3*d1], zmm19		;;33 Save R1				; 68

	zfnmaddpd zmm13, zmm22, zmm2, zmm4	;;33 I2 = i1- - 866s*r2-		; 69-72
	zfmaddpd zmm22, zmm22, zmm2, zmm4	;;33 I3 = i1- + 866s*r2-		; 69-72
	zstore	[srcreg+3*d1+64], zmm18		;;33 Save I1				; 69

	zstore	[srcreg+2*d1+4*d1], zmm20	;;32 Save R2
	zstore	[srcreg+2*d1+8*d1], zmm5	;;32 Save R3
	zstore	[srcreg+2*d1+4*d1+64], zmm7	;;32 Save I2
	zstore	[srcreg+2*d1+8*d1+64], zmm8	;;32 Save I3
	zstore	[srcreg+3*d1+4*d1], zmm12	;;33 Save R2
	zstore	[srcreg+3*d1+8*d1], zmm6	;;33 Save R3
	zstore	[srcreg+3*d1+4*d1+64], zmm13	;;33 Save I2
	zstore	[srcreg+3*d1+8*d1+64], zmm22	;;33 Save I3
	bump	srcreg, srcinc
	ENDM

;;
;; ************************************* One-pass variants of twentyfour_reals_twelve_complex fft and unfft ******************************************
;;

zr34_twentyfour_reals_twelve_complex_djbfft_preload MACRO
	mov	r9d, 11111110b			;; We're pretty sure r9 is safe to use
	kmovw	k7, r9d				;; Set k7 to 11111110b
	knotw	k6, k7				;; Set k6 to 00000001b
	vbroadcastsd zmm31, ZMM_ONE
	vzero	zmm0
	vsubpd	zmm31 {k6}, zmm0, zmm31				;; 1, 1, 1, 1, 1, 1, 1		-1
	vblendmpd zmm30 {k6}, zmm31, ZMM_SQRTHALF {1to8}	;; 1, 1, 1, 1, 1, 1, 1		SQRTHALF
	vmulpd	zmm29, zmm31, zmm30				;; 1, 1, 1, 1, 1, 1, 1		-SQRTHALF
	vmulpd	zmm28, zmm31, ZMM_HALF{1to8}			;; 0.5 (7 times)		-0.5
	ENDM
zr34_twentyfour_reals_twelve_complex_djbfft MACRO srcreg,srcinc,d1,sc3reg,sc3gap,sc3inc,sc4reg,sc4gap,sc4inc,maxrpt,L1pt,L1pd
	IF sc4gap NE 0
	need_code_for_non_zero_sc4gap
	ENDIF
						;;30 Three complex comments	Six-reals comments
	vmovapd	zmm2, [srcreg+0*d1+4*d1+64]	;;30 I2				R2-R5
	vmovapd	zmm1, [srcreg+0*d1+8*d1+64]	;;30 I3				R3-R6
	zfmaddpd zmm0, zmm1, zmm31, zmm2	;;30 i2+ = I2 + 1*I3		r2-- = R2-R5 + -1*(R3-R6)	; 1-4		n 13
	zfnmaddpd zmm1, zmm1, zmm31, zmm2	;;30 i2- = I2 - 1*I3		r2-+ = R2-R5 - -1*(R3-R6)	; 1-4		n 9

	vmovapd	zmm3, [srcreg+0*d1+4*d1]	;;30 R2				R2+R5
	vmovapd	zmm4, [srcreg+0*d1+8*d1]	;;30 R3				R3+R6
	vaddpd	zmm2, zmm3, zmm4		;;30 r2+ = R2 + R3		r2++ = R2+R5 + (R3+R6)		; 2-5		n 14
	vsubpd	zmm3, zmm3, zmm4		;;30 r2- = R2 - R3		r2+- = R2+R5 - (R3+R6)		; 2-5		n 9

	vmovapd	zmm6, [srcreg+1*d1+4*d1+64]	;;31 I2				R2-R5
	vmovapd	zmm5, [srcreg+1*d1+8*d1+64]	;;31 I3				R3-R6
	zfmaddpd zmm4, zmm5, zmm31, zmm6	;;31 i2+ = I2 + 1*I3		r2-- = R2-R5 + -1*(R3-R6)	; 3-6		n 15
	zfnmaddpd zmm5, zmm5, zmm31, zmm6	;;31 i2- = I2 - 1*I3		r2-+ = R2-R5 - -1*(R3-R6)	; 3-6		n 10

	vmovapd	zmm7, [srcreg+1*d1+4*d1]	;;31 R2				R2+R5
	vmovapd	zmm8, [srcreg+1*d1+8*d1]	;;31 R3				R3+R6
	vaddpd	zmm6, zmm7, zmm8		;;31 r2+ = R2 + R3		r2++ = R2+R5 + (R3+R6)		; 4-7		n 16
	vsubpd	zmm7, zmm7, zmm8		;;31 r2- = R2 - R3		r2+- = R2+R5 - (R3+R6)		; 4-7		n 10

	vmovapd	zmm10, [srcreg+2*d1+4*d1+64]	;;32 I2				R2-R5
	vmovapd	zmm9, [srcreg+2*d1+8*d1+64]	;;32 I3				R3-R6
	zfmaddpd zmm8, zmm9, zmm31, zmm10	;;32 i2+ = I2 + 1*I3		r2-- = R2-R5 + -1*(R3-R6)	; 5-8		n 17
	zfnmaddpd zmm9, zmm9, zmm31, zmm10	;;32 i2- = I2 - 1*I3		r2-+ = R2-R5 - -1*(R3-R6)	; 5-8		n 11

	vmovapd	zmm11, [srcreg+2*d1+4*d1]	;;32 R2				R2+R5
	vmovapd	zmm12, [srcreg+2*d1+8*d1]	;;32 R3				R3+R6
	vaddpd	zmm10, zmm11, zmm12		;;32 r2+ = R2 + R3		r2++ = R2+R5 + (R3+R6)		; 6-9		n 18
	vsubpd	zmm11, zmm11, zmm12		;;32 r2- = R2 - R3		r2+- = R2+R5 - (R3+R6)		; 6-9		n 11

	vmovapd	zmm14, [srcreg+3*d1+4*d1+64]	;;33 I2				R2-R5
	vmovapd	zmm13, [srcreg+3*d1+8*d1+64]	;;33 I3				R3-R6
	zfmaddpd zmm12, zmm13, zmm31, zmm14	;;33 i2+ = I2 + 1*I3		r2-- = R2-R5 + -1*(R3-R6)	; 7-10		n 19
	zfnmaddpd zmm13, zmm13, zmm31, zmm14	;;33 i2- = I2 - 1*I3		r2-+ = R2-R5 - -1*(R3-R6)	; 7-10		n 12

	vmovapd	zmm15, [srcreg+3*d1+4*d1]	;;33 R2				R2+R5
	vmovapd	zmm16, [srcreg+3*d1+8*d1]	;;33 R3				R3+R6
	vaddpd	zmm14, zmm15, zmm16		;;33 r2+ = R2 + R3		r2++ = R2+R5 + (R3+R6)		; 8-11		n 20
	vsubpd	zmm15, zmm15, zmm16		;;33 r2- = R2 - R3		r2+- = R2+R5 - (R3+R6)		; 8-11		n 12

	vbroadcastsd zmm27, ZMM_P866
	vmulpd	zmm1, zmm1, zmm27		;;30 i2- = .866*i2-		I2 = .866 * r2-+		; 9-12		n 21
	vmulpd	zmm3, zmm3, zmm27		;;30 r2- = .866*r2-		I3 = .866 * r2+-		; 9-12		n 21

	vmovapd	zmm17, [srcreg+0*d1+64]		;;30 I1				R1-R4						n 13
	vmulpd	zmm5, zmm5, zmm27		;;31 i2- = .866*i2-		I2 = .866 * r2-+		; 10-13		n 22
	vmulpd	zmm7, zmm7, zmm27		;;31 r2- = .866*r2-		I3 = .866 * r2+-		; 10-13		n 22

	vmovapd	zmm18, [srcreg+0*d1]		;;30 R1				R1+R4						n 14
	vmulpd	zmm9, zmm9, zmm27		;;32 i2- = .866*i2-		I2 = .866 * r2-+		; 11-14		n 23
	vmulpd	zmm11, zmm11, zmm27		;;32 r2- = .866*r2-		I3 = .866 * r2+-		; 11-14		n 23

	vbroadcastsd zmm26, ZMM_HALF		;;.5										n 14
	vmulpd	zmm13, zmm13, zmm27		;;33 i2- = .866*i2-		I2 = .866 * r2-+		; 12-15		n 24
	vmulpd	zmm15, zmm15, zmm27		;;33 r2- = .866*r2-		I3 = .866 * r2+-		; 12-15		n 24

	vmovapd	zmm19, [srcreg+1*d1+64]		;;31 I1				R1-R4						n 15
	zfmaddpd zmm16, zmm0, zmm31, zmm17	;;30 I1 + 1*(I2+I3) (I1 in 40)	R1-R4 + -1*r2-- (final R1b)	; 13-16		n 63
	zfnmaddpd zmm0, zmm0, zmm28, zmm17	;;30 i1- = I1 - .5(I2+I3)	R1-R4 - -0.5 * r2-- (R2)	; 13-16		n 21

	vmovapd	zmm20, [srcreg+1*d1]		;;31 R1				R1+R4						n 16
	vaddpd	zmm17, zmm18, zmm2		;;30 R1 + (R2+R3) (R1 in 40)	R1+R4 + r2++ (final R1a)	; 14-17		n 64
	zfnmaddpd zmm2, zmm2, zmm26, zmm18	;;30 r1- = R1 - .5(R2+R3)	R1+R4 - 0.5 * r2++ (R3)		; 14-17		n 22

	vmovapd	zmm21, [srcreg+2*d1+64]		;;32 I1				R1-R4						n 17
	zfmaddpd zmm18, zmm4, zmm31, zmm19	;;31 I1 + 1*(I2+I3) (I2 in 40)	R1-R4 + -1*r2-- (final R1b)	; 15-18		n 59
	zfnmaddpd zmm4, zmm4, zmm28, zmm19	;;31 i1- = I1 - .5(I2+I3)	R1-R4 - -0.5 * r2-- (R2)	; 15-18		n 23

	vmovapd	zmm22, [srcreg+2*d1]		;;32 R1				R1+R4						n 18
	vaddpd	zmm19, zmm20, zmm6		;;31 R1 + (R2+R3) (R2 in 40)	R1+R4 + r2++ (final R1a)	; 16-19		n 65
	zfnmaddpd zmm6, zmm6, zmm26, zmm20	;;31 r1- = R1 - .5(R2+R3)	R1+R4 - 0.5 * r2++ (R3)		; 16-19		n 24

	vmovapd	zmm23, [srcreg+3*d1+64]		;;33 I1				R1-R4						n 19
	zfmaddpd zmm20, zmm8, zmm31, zmm21	;;32 I1 + 1*(I2+I3) (I3 in 40)	R1-R4 + -1*r2-- (final R1b)	; 17-20		n 63
	zfnmaddpd zmm8, zmm8, zmm28, zmm21	;;32 i1- = I1 - .5(I2+I3)	R1-R4 - -0.5 * r2-- (R2)	; 17-20		n 25

	vmovapd	zmm24, [srcreg+3*d1]		;;33 R1				R1+R4						n 20
	vaddpd	zmm21, zmm22, zmm10		;;32 R1 + (R2+R3) (R3 in 40)	R1+R4 + r2++ (final R1a)	; 18-21		n 64
	zfnmaddpd zmm10, zmm10, zmm26, zmm22	;;32 r1- = R1 - .5(R2+R3)	R1+R4 - 0.5 * r2++ (R3)		; 18-21		n 26

	vmovapd	zmm25, [sc3reg+0*sc3gap+64]			;;30 cosine/sine						n 29
	zfmaddpd zmm22, zmm12, zmm31, zmm23	;;33 I1 + 1*(I2+I3) (I4 in 40)	R1-R4 + -1*r2-- (final R1b)	; 19-22		n 59
	zfnmaddpd zmm12, zmm12, zmm28, zmm23	;;33 i1- = I1 - .5(I2+I3)	R1-R4 - -0.5 * r2-- (R2)	; 19-22		n 27

	vmovapd	zmm27, [sc3reg+1*sc3gap+64]			;;31 cosine/sine						n 30
	vaddpd	zmm23, zmm24, zmm14		;;33 R1 + (R2+R3) (R4 in 40)	R1+R4 + r2++ (final R1a)	; 20-23		n 65
	zfnmaddpd zmm14, zmm14, zmm26, zmm24	;;33 r1- = R1 - .5(R2+R3)	R1+R4 - 0.5 * r2++ (R3)		; 20-23		n 28

	vmovapd	zmm26, [sc3reg+0*sc3gap+128+64]			;;30 cosine/sine						n 31
	vmovapd zmm24, zmm1			;;30 not important		I2
	vaddpd	zmm24 {k7}, zmm0, zmm3		;;30 I2 = i1- + r2-		blend in I2			; 21-24		n 29
	vsubpd	zmm3 {k7}, zmm0, zmm3		;;30 I3 = i1- - r2-		blend in I3			; 21-24		n 31

	L1prefetchw srcreg+srcinc+0*d1+4*d1+64, L1pt
	vsubpd	zmm0 {k7}, zmm2, zmm1		;;30 R2 = r1- - i2-		blend in R2			; 22-25		n 29
	vaddpd	zmm2 {k7}, zmm2, zmm1		;;30 R3 = r1- + i2-		blend in R3			; 22-25		n 31

	L1prefetchw srcreg+srcinc+0*d1+8*d1+64, L1pt
	vmovapd zmm1, zmm5			;;31 not important		I2
	vaddpd	zmm1 {k7}, zmm4, zmm7		;;31 I2 = i1- + r2-		blend in I2			; 23-26		n 30
	vsubpd	zmm7 {k7}, zmm4, zmm7		;;31 I3 = i1- - r2-		blend in I3			; 23-26		n 32

	L1prefetchw srcreg+srcinc+0*d1+4*d1, L1pt
	vsubpd	zmm4 {k7}, zmm6, zmm5		;;31 R2 = r1- - i2-		blend in R2			; 24-27		n 30
	vaddpd	zmm6 {k7}, zmm6, zmm5		;;31 R3 = r1- + i2-		blend in R3			; 24-27		n 32

	L1prefetchw srcreg+srcinc+0*d1+8*d1, L1pt
	vmovapd zmm5, zmm9			;;32 not important		I2
	vaddpd	zmm5 {k7}, zmm8, zmm11		;;32 I2 = i1- + r2-		blend in I2			; 25-28		n 35
	vsubpd	zmm11 {k7}, zmm8, zmm11		;;32 I3 = i1- - r2-		blend in I3			; 25-28		n 39

	L1prefetchw srcreg+srcinc+1*d1+4*d1+64, L1pt
	vsubpd	zmm8 {k7}, zmm10, zmm9		;;32 R2 = r1- - i2-		blend in R2			; 26-29		n 35
	vaddpd	zmm10 {k7}, zmm10, zmm9		;;32 R3 = r1- + i2-		blend in R3			; 26-29		n 39

	L1prefetchw srcreg+srcinc+1*d1+8*d1+64, L1pt
	vmovapd zmm9, zmm13			;;33 not important		I2
	vaddpd	zmm9 {k7}, zmm12, zmm15		;;33 I2 = i1- + r2-		blend in I2			; 27-30		n 36
	vsubpd	zmm15 {k7}, zmm12, zmm15	;;33 I3 = i1- - r2-		blend in I3			; 27-30		n 40

	L1prefetchw srcreg+srcinc+1*d1+4*d1, L1pt
	vsubpd	zmm12 {k7}, zmm14, zmm13	;;33 R2 = r1- - i2-		blend in R2			; 28-31		n 36
	vaddpd	zmm14 {k7}, zmm14, zmm13	;;33 R3 = r1- + i2-		blend in R3			; 28-31		n 40

	L1prefetchw srcreg+srcinc+1*d1+8*d1, L1pt
	zfmsubpd zmm13, zmm0, zmm25, zmm24			;;30 A2 = R2 * cosine/sine - I2			; 29-32		n 33
	zfmaddpd zmm24, zmm24, zmm25, zmm0			;;30 B2 = I2 * cosine/sine + R2			; 29-32		n 33

	vmovapd	zmm25, [sc3reg+1*sc3gap+128+64]			;;31 cosine/sine						n 32
	zfmsubpd zmm0, zmm4, zmm27, zmm1			;;31 A2 = R2 * cosine/sine - I2			; 30-33		n 34
	zfmaddpd zmm1, zmm1, zmm27, zmm4			;;31 B2 = I2 * cosine/sine + R2			; 30-33		n 34

	vmovapd	zmm27, [sc3reg+0*sc3gap]			;;30 sine							n 33
	zfmsubpd zmm4, zmm2, zmm26, zmm3			;;30 A3 = R3 * cosine/sine - I3			; 31-34		n 37
	zfmaddpd zmm3, zmm3, zmm26, zmm2			;;30 B3 = I3 * cosine/sine + R3			; 31-34		n 37

	vmovapd	zmm26, [sc3reg+1*sc3gap]			;;31 sine							n 34
	zfmsubpd zmm2, zmm6, zmm25, zmm7			;;31 A3 = R3 * cosine/sine - I3			; 32-35		n 38
	zfmaddpd zmm7, zmm7, zmm25, zmm6			;;31 B3 = I3 * cosine/sine + R3			; 32-35		n 38

	vmovapd	zmm25, [sc3reg+2*sc3gap+64]			;;32 cosine/sine						n 35
	vmulpd	zmm13, zmm13, zmm27				;;30 A2 = A2 * sine (R1 in 41)			; 33-36		n 41
	vmulpd	zmm24, zmm24, zmm27				;;30 B2 = B2 * sine (I1 in 41)			; 33-36		n 42

	vmovapd	zmm27, [sc3reg+3*sc3gap+64]			;;33 cosine/sine						n 36
	vmulpd	zmm0, zmm0, zmm26				;;31 A2 = A2 * sine (R2 in 41)			; 34-37		n 43
	vmulpd	zmm1, zmm1, zmm26				;;31 B2 = B2 * sine (I2 in 41)			; 34-37		n 44

	vmovapd	zmm26, [sc3reg+0*sc3gap+128]			;;30 sine							n 37
	zfmsubpd zmm6, zmm8, zmm25, zmm5			;;32 R2 * cosine/sine - I2 (R3/sine in 41)	; 35-38		n 41
	zfmaddpd zmm5, zmm5, zmm25, zmm8			;;32 I2 * cosine/sine + R2 (I3/sine in 41)	; 35-38		n 42

	vmovapd	zmm25, [sc3reg+1*sc3gap+128]			;;31 sine							n 38
	zfmsubpd zmm8, zmm12, zmm27, zmm9			;;33 R2 * cosine/sine - I2 (R4/sine in 41)	; 36-39		n 43
	zfmaddpd zmm9, zmm9, zmm27, zmm12			;;33 I2 * cosine/sine + R2 (I4/sine in 41)	; 36-39		n 44

	vmovapd	zmm27, [sc3reg+2*sc3gap+128+64]			;;32 cosine/sine						n 39
	vmulpd	zmm4, zmm4, zmm26				;;30 A3 = A3 * sine (R1 in 42)			; 37-40		n 45
	vmulpd	zmm3, zmm3, zmm26				;;30 B3 = B3 * sine (I1 in 42)			; 37-40		n 46

	vmovapd	zmm26, [sc3reg+3*sc3gap+128+64]			;;33 cosine/sine						n 40
	vmulpd	zmm2, zmm2, zmm25				;;31 A3 = A3 * sine (R2 in 42)			; 38-41		n 47
	vmulpd	zmm7, zmm7, zmm25				;;31 B3 = B3 * sine (I2 in 42)			; 38-41		n 48

	vmovapd	zmm25, [sc3reg+2*sc3gap]			;;32 sine for R3/I3 in 41					n 41
	zfmsubpd zmm12, zmm10, zmm27, zmm11			;;32 R3 * cosine/sine - I3 (R3/sine in 42)	; 39-42		n 45
	zfmaddpd zmm11, zmm11, zmm27, zmm10			;;32 I3 * cosine/sine + R3 (I3/sine in 42)	; 39-42		n 46

	vmovapd	zmm27, [sc3reg+3*sc3gap]			;;33 sine for R4/I4 in 41					n 43
	zfmsubpd zmm10, zmm14, zmm26, zmm15			;;33 R3 * cosine/sine - I3 (R4/sine in 42)	; 40-43		n 47
	zfmaddpd zmm15, zmm15, zmm26, zmm14			;;33 I3 * cosine/sine + R3 (I4/sine in 42)	; 40-43		n 48

	vmovapd	zmm26, [sc3reg+2*sc3gap+128]			;;32 sine for R3/I3 in 42					n 45
	zfmaddpd zmm14, zmm6, zmm25, zmm13			;;41 R1 + R3*sine (new R1)			; 41-44		n 51
	zfnmaddpd zmm6, zmm6, zmm25, zmm13			;;41 R1 - R3*sine (new R3)			; 41-44		n 49

	L1prefetchw srcreg+srcinc+2*d1+4*d1+64, L1pt
	zfmaddpd zmm13, zmm5, zmm25, zmm24			;;41 I1 + I3*sine (new I1)			; 42-45		n 52
	zfnmaddpd zmm5, zmm5, zmm25, zmm24			;;41 I1 - I3*sine (new I3)			; 42-45		n 49

	vmovapd	zmm25, [sc3reg+3*sc3gap+128]			;;33 sine for R4/I4 in 42					n 47
	zfmaddpd zmm24, zmm8, zmm27, zmm0			;;41 R2 + R4*sine (new R2)			; 43-46		n 51
	zfnmaddpd zmm8, zmm8, zmm27, zmm0			;;41 R2 - R4*sine (new R4)			; 43-46		n 56
	bump	sc3reg, sc3inc

	L1prefetchw srcreg+srcinc+2*d1+8*d1+64, L1pt
	zfmaddpd zmm0, zmm9, zmm27, zmm1			;;41 I2 + I4*sine (new I2)			; 44-47		n 52
	zfnmaddpd zmm9, zmm9, zmm27, zmm1			;;41 I2 - I4*sine (new I4)			; 44-47		n 55

	vbroadcastsd zmm27, Q [sc4reg+8]			;;4x Sine							n 49
	zfmaddpd zmm1, zmm12, zmm26, zmm4			;;42 R1 + R3*sine (new R1)			; 45-48		n 53
	zfnmaddpd zmm12, zmm12, zmm26, zmm4			;;42 R1 - R3*sine (new R3)			; 45-48		n 50

	L1prefetchw srcreg+srcinc+2*d1+4*d1, L1pt
	zfmaddpd zmm4, zmm11, zmm26, zmm3			;;42 I1 + I3*sine (new I1)			; 46-49		n 54
	zfnmaddpd zmm11, zmm11, zmm26, zmm3			;;42 I1 - I3*sine (new I3)			; 46-49		n 50

	vbroadcastsd zmm26, Q [sc4reg+8+128+64]			;;4x cosine/sine						n 60
	zfmaddpd zmm3, zmm10, zmm25, zmm2			;;42 R2 + R4*sine (new R2)			; 47-50		n 53
	zfnmaddpd zmm10, zmm10, zmm25, zmm2			;;42 R2 - R4*sine (new R4)			; 47-50		n 57

	L1prefetchw srcreg+srcinc+2*d1+8*d1, L1pt
	zfmaddpd zmm2, zmm15, zmm25, zmm7			;;42 I2 + I4*sine (new I2)			; 48-51		n 54
	zfnmaddpd zmm15, zmm15, zmm25, zmm7			;;42 I2 - I4*sine (new I4)			; 48-51		n 58

	vbroadcastsd zmm25, Q [sc4reg+8+64]			;;4x cosine/sine						n 61
	vmulpd	zmm6, zmm6, zmm27				;;41 R3s = R3 * sine				; 49-52		n 55
	vmulpd	zmm5, zmm5, zmm27				;;41 I3s = I3 * sine				; 49-52		n 56

	L1prefetchw srcreg+srcinc+3*d1+4*d1+64, L1pt
	vmulpd	zmm12, zmm12, zmm27				;;42 R3s = R3 * sine				; 50-53		n 57
	vmulpd	zmm11, zmm11, zmm27				;;42 I3s = I3 * sine				; 50-53		n 58

	L1prefetchw srcreg+srcinc+3*d1+8*d1+64, L1pt
	vsubpd	zmm7, zmm14, zmm24				;;41 R1 - R2 (new R2)				; 51-54		n 60
	vaddpd	zmm14, zmm14, zmm24				;;41 R1 + R2 (final R1)				; 51-54

	L1prefetchw srcreg+srcinc+3*d1+4*d1, L1pt
	vsubpd	zmm24, zmm13, zmm0				;;41 I1 - I2 (new I2)				; 52-55		n 60
	vaddpd	zmm13, zmm13, zmm0				;;41 I1 + I2 (final I1)				; 52-55

	L1prefetchw srcreg+srcinc+3*d1+8*d1, L1pt
	vsubpd	zmm0, zmm1, zmm3				;;42 R1 - R2 (new R2)				; 53-56		n 72
	vaddpd	zmm1, zmm1, zmm3				;;42 R1 + R2 (final R1)				; 53-56

	L1prefetchw srcreg+srcinc+0*d1+64, L1pt
	vsubpd	zmm3, zmm4, zmm2				;;42 I1 - I2 (new I2)				; 54-57		n 72
	vaddpd	zmm4, zmm4, zmm2				;;42 I1 + I2 (final I1)				; 54-57

	L1prefetchw srcreg+srcinc+0*d1, L1pt
	zfnmaddpd zmm2, zmm9, zmm27, zmm6			;;41 R3s - I4*sine (new R3s)			; 55-58		n 61
	zfmaddpd zmm9, zmm9, zmm27, zmm6			;;41 R3s + I4*sine (new R4s)			; 55-58		n 62
	zstore	[srcreg+4*d1], zmm14				;;41 Save R1					; 55

	vbroadcastsd zmm14, Q [sc4reg+8+128]			;;4x Sine							n 68
	zfmaddpd zmm6, zmm8, zmm27, zmm5			;;41 I3s + R4*sine (new I3s)			; 56-59		n 61
	zfnmaddpd zmm8, zmm8, zmm27, zmm5			;;41 I3s - R4*sine (new I4s)			; 56-59		n 62
	zstore	[srcreg+4*d1+64], zmm13				;;41 Save I1					; 56

	vmovapd	zmm13, [sc4reg+0*128+64]			;;40 cosine/sine						n 75
	zfnmaddpd zmm5, zmm15, zmm27, zmm12			;;42 R3s - I4*sine (new R3s)			; 57-60		n 66
	zfmaddpd zmm15, zmm15, zmm27, zmm12			;;42 R3s + I4*sine (new R4s)			; 57-60		n 67
	zstore	[srcreg+8*d1], zmm1				;;42 Save R1					; 57

	L1prefetchw srcreg+srcinc+1*d1+64, L1pt
	zfmaddpd zmm12, zmm10, zmm27, zmm11			;;42 I3s + R4*sine (new I3s)			; 58-61		n 66
	zfnmaddpd zmm10, zmm10, zmm27, zmm11			;;42 I3s - R4*sine (new I4s)			; 58-61		n 67
	zstore	[srcreg+8*d1+64], zmm4				;;42 Save I1					; 58

						;;40 Four-complex comments	Eight-reals comments
	L1prefetchw srcreg+srcinc+1*d1, L1pt
	vaddpd	zmm11, zmm18, zmm22		;;40 i2+ = I2 + I4		i34tmp = r2- + r4-		; 59-62		n 65
	vsubpd	zmm18, zmm18, zmm22		;;40 r34tmp = I2 - I4		r34tmp = r2- - r4-		; 59-62		n 70

	L1prefetchw srcreg+srcinc+2*d1+64, L1pt
	zfmsubpd zmm22, zmm7, zmm26, zmm24			;;41 A2 = R2 * cosine/sine - I2			; 60-63		n 68
	zfmaddpd zmm24, zmm24, zmm26, zmm7			;;41 B2 = I2 * cosine/sine + R2			; 60-63		n 68

	L1prefetchw srcreg+srcinc+2*d1, L1pt
	zfmsubpd zmm7, zmm2, zmm25, zmm6			;;41 R3s * cosine/sine - I3s (final R3)		; 61-64
	zfmaddpd zmm6, zmm6, zmm25, zmm2			;;41 I3s * cosine/sine + R3s (final I3)		; 61-64

	L1prefetchw srcreg+srcinc+3*d1+64, L1pt
	zfmaddpd zmm2, zmm9, zmm25, zmm8			;;41 R4s * cosine/sine + I4s (final R4)		; 62-65
	zfmsubpd zmm8, zmm8, zmm25, zmm9			;;41 I4s * cosine/sine - R4s (final I4)		; 62-65

	L1prefetchw srcreg+srcinc+3*d1, L1pt
	vaddpd	zmm9 {k7}{z}, zmm16, zmm20	;;40 i1+ = I1 + I3		0				; 63-66		n 69
	vsubpd	zmm20 {k7}, zmm16, zmm20	;;40 I34 = I1 - I3		blend in I34			; 63-66		n 71

	vaddpd	zmm4, zmm17, zmm21		;;40 r1+ = R1 + R3		r1++ = r1+ + r3+		; 64-67   	n 69
	vsubpd	zmm16 {k7}, zmm17, zmm21	;;40 R34 = R1 - R3		blend in R34			; 64-67		n 70

	vmovapd	zmm27, zmm11			;;40 not important		i34tmp
	vaddpd	zmm1, zmm19, zmm23		;;40 r2+ = R2 + R4		r2++ = r2+ + r4+		; 65-68		n 69
	vsubpd	zmm27 {k7}, zmm19, zmm23	;;40 i34tmp = R2 - R4		blend in i34tmp			; 65-68		n 71
	zstore	[srcreg+4*d1+2*d1], zmm7			;;41 Save R3					; 65

	zfmsubpd zmm7, zmm5, zmm25, zmm12			;;42 R3s * cosine/sine - I3s (final R3)		; 66-69
	zfmaddpd zmm12, zmm12, zmm25, zmm5			;;42 I3s * cosine/sine + R3s (final I3)		; 66-69
	zstore	[srcreg+4*d1+2*d1+64], zmm6			;;41 Save I3					; 66

	vmovapd	zmm6, [sc4reg+2*128+64]				;;40 cosine/sine						n 76
	zfmaddpd zmm5, zmm15, zmm25, zmm10			;;42 R4s * cosine/sine + I4s (final R4)		; 67-70
	zfmsubpd zmm10, zmm10, zmm25, zmm15			;;42 I4s * cosine/sine - R4s (final I4)		; 67-70
	zstore	[srcreg+4*d1+3*d1], zmm2			;;41 Save R4					; 67

	vmovapd	zmm2, [sc4reg+1*128+64]				;;40 cosine/sine						n 77
	vmulpd	zmm22, zmm22, zmm14				;;41 A2 = A2 * sine (final R2)			; 68-71
	vmulpd	zmm24, zmm24, zmm14				;;41 B2 = B2 * sine (final I2)			; 68-71
	zstore	[srcreg+4*d1+3*d1+64], zmm8			;;41 Save I4					; 68

	vsubpd	zmm15, zmm4, zmm1		;;40 R2 = r1+ - r2+		R1b = r1++ - r2++		; 69-72		n 73
	vsubpd	zmm25 {k7}{z}, zmm9, zmm11	;;40 I2 = i1+ - i2+		0				; 69-72		n 73

	zfnmaddpd zmm8, zmm18, zmm29, zmm16	;;40 R3 = R34 - r34tmp*1	R3 = R34 - r34tmp*-SQRTHALF	; 70-73		n 75
	zfmaddpd zmm18, zmm18, zmm29, zmm16	;;40 R4 = R34 + r34tmp*1	R4 = R34 + r34tmp*-SQRTHALF	; 70-73		n 76
	zstore	[srcreg+8*d1+2*d1], zmm7			;;42 Save R3					; 70

	vmovapd	zmm7, [sc4reg+0*128]				;;40 Sine							n 79
	zfmaddpd zmm16, zmm27,zmm30, zmm20	;;40 I3 = I34 + i34tmp*1	I3 = I34 + i34tmp*SQRTHALF	; 71-74		n 75
	zfnmaddpd zmm27, zmm27, zmm30, zmm20	;;40 I4 = I34 - i34tmp*1	I4 = I34 - i34tmp*SQRTHALF	; 71-74		n 76
	zstore	[srcreg+8*d1+2*d1+64], zmm12			;;42 Save I3					; 71

	vmovapd	zmm12, [sc4reg+2*128]				;;40 Sine							n 80
	zfmsubpd zmm20, zmm0, zmm26, zmm3			;;42 A2 = R2 * cosine/sine - I2			; 72-75		n 78
	zfmaddpd zmm3, zmm3, zmm26, zmm0			;;42 B2 = I2 * cosine/sine + R2			; 72-75		n 78
	zstore	[srcreg+8*d1+3*d1], zmm5			;;42 Save R4					; 72

	vmovapd	zmm5, [sc4reg+1*128]				;;40 Sine							n 81
	vmovapd	zmm0, zmm15			;;40 R2			not important
	vsubpd	zmm0 {k6}, zmm17, zmm21		;;40 blend in R2		R2 = r1+ - r3+			; 73-76		n 77
	vsubpd	zmm25 {k6}, zmm19, zmm23	;;40 blend in I2		I2 = r2+ - r4+			; 73-76		n 77
	zstore	[srcreg+8*d1+3*d1+64], zmm10			;;42 Save I4					; 73

	vaddpd	zmm4, zmm4, zmm1		;;40 R1 = r1+ + r2+		R1a = r1++ + r2++		; 74-77
	vaddpd	zmm15 {k7}, zmm9, zmm11		;;40 I1 = i1+ + i2+		blend in R1b			; 74-77
	zstore	[srcreg+4*d1+d1], zmm22				;;41 Save R2					; 74

	zfmsubpd zmm11, zmm8, zmm13, zmm16			;;40 A3 = R3 * cosine/sine - I3			; 75-78		n 79
	zfmaddpd zmm16, zmm16, zmm13, zmm8			;;40 B3 = I3 * cosine/sine + R3			; 75-78		n 79
	zstore	[srcreg+4*d1+d1+64], zmm24			;;41 Save I2					; 75

	zfmsubpd zmm8, zmm18, zmm6, zmm27			;;40 A4 = R4 * cosine/sine - I4			; 76-79		n 80
	zfmaddpd zmm27, zmm27, zmm6, zmm18			;;40 B4 = I4 * cosine/sine + R4			; 76-79		n 80
	bump	sc4reg, sc4inc

	zfmsubpd zmm18, zmm0, zmm2, zmm25			;;40 A2 = R2 * cosine/sine - I2			; 77-80		n 81
	zfmaddpd zmm25, zmm25, zmm2, zmm0			;;40 B2 = I2 * cosine/sine + R2			; 77-80		n 81

	vmulpd	zmm20, zmm20, zmm14				;;42 A2 = A2 * sine (final R2)			; 78-81
	vmulpd	zmm3, zmm3, zmm14				;;42 B2 = B2 * sine (final I2)			; 78-81
	zstore	[srcreg+0*d1], zmm4		;;40 Save R1							; 78

	vmulpd	zmm11, zmm11, zmm7				;;40 A3 = A3 * sine (final R3)			; 79-82
	vmulpd	zmm16, zmm16, zmm7				;;40 B3 = B3 * sine (final I3)			; 79-82
	zstore	[srcreg+0*d1+64], zmm15		;;40 Save I1							; 79

	vmulpd	zmm8, zmm8, zmm12				;;40 A4 = A4 * sine (final R4)			; 80-83
	vmulpd	zmm27, zmm27, zmm12				;;40 B4 = B4 * sine (final I4)			; 80-83

	vmulpd	zmm18, zmm18, zmm5				;;40 A2 = A2 * sine (final R2)			; 81-84
	vmulpd	zmm25, zmm25, zmm5				;;40 B2 = B2 * sine (final I2)			; 81-84

	zstore	[srcreg+8*d1+d1], zmm20				;;42 Save R2
	zstore	[srcreg+8*d1+d1+64], zmm3			;;42 Save I2
	zstore	[srcreg+0*d1+2*d1], zmm11			;;40 Save R3
	zstore	[srcreg+0*d1+2*d1+64], zmm16			;;40 Save I3
	zstore	[srcreg+0*d1+3*d1], zmm8			;;40 Save R4
	zstore	[srcreg+0*d1+3*d1+64], zmm27			;;40 Save I4
	zstore	[srcreg+0*d1+d1], zmm18				;;40 Save R2
	zstore	[srcreg+0*d1+d1+64], zmm25			;;40 Save I2
	bump	srcreg, srcinc
	ENDM


zr34_twentyfour_reals_twelve_complex_djbunfft_preload MACRO
	mov	r9d, 11111110b			;; We're pretty sure r9 is safe to use
	kmovw	k7, r9d				;; Set k7 to 11111110b
	knotw	k6, k7				;; Set k6 to 00000001b
	vbroadcastsd zmm0, ZMM_SQRTHALF
	vbroadcastsd zmm31, ZMM_ONE
	vzero	zmm29
	vblendmpd zmm30 {k6}, zmm31, zmm0	;; 1, 1, 1, 1, 1, 1, 1		SQRTHALF
	vsubpd	zmm31 {k6}, zmm29, zmm0		;; 1, 1, 1, 1, 1, 1, 1		-SQRTHALF
	ENDM
zr34_twentyfour_reals_twelve_complex_djbunfft MACRO srcreg,srcinc,d1,sc4reg,sc4gap,sc4inc,sc3reg,sc3gap,sc3inc,maxrpt,L1pt,L1pd
	IF sc4gap NE 0
	need_code_for_non_zero_sc4gap
	ENDIF
	vbroadcastsd zmm28, Q [sc4reg+8+128+64]			;;4x cosine/sine2
	vmovapd	zmm2, [srcreg+4*d1+d1]				;;41 R2
	vmovapd	zmm1, [srcreg+4*d1+d1+64]			;;41 I2
	zfmaddpd zmm0, zmm2, zmm28, zmm1			;;41 A2 = R2 * cosine/sine + I2			; 1-4		n 7
	zfmsubpd zmm1, zmm1, zmm28, zmm2			;;41 B2 = I2 * cosine/sine - R2			; 1-4		n 8

	vmovapd	zmm4, [srcreg+8*d1+d1]				;;42 R2
	vmovapd	zmm3, [srcreg+8*d1+d1+64]			;;42 I2
	zfmaddpd zmm2, zmm4, zmm28, zmm3			;;42 A2 = R2 * cosine/sine + I2			; 2-5		n 11
	zfmsubpd zmm3, zmm3, zmm28, zmm4			;;42 B2 = I2 * cosine/sine - R2			; 2-5		n 12

	vbroadcastsd zmm28, Q [sc4reg+8+64]			;;4x cosine/sine34
	vmovapd	zmm6, [srcreg+4*d1+2*d1]			;;41 R3
	vmovapd	zmm5, [srcreg+4*d1+2*d1+64]			;;41 I3
	zfmaddpd zmm4, zmm6, zmm28, zmm5			;;41 A3 = R3 * cosine/sine + I3			; 3-6		n 9
	zfmsubpd zmm5, zmm5, zmm28, zmm6			;;41 B3 = I3 * cosine/sine - R3			; 3-6		n 10

	vmovapd	zmm8, [srcreg+4*d1+3*d1]			;;41 R4
	vmovapd	zmm7, [srcreg+4*d1+3*d1+64]			;;41 I4
	zfmsubpd zmm6, zmm8, zmm28, zmm7			;;41 A4 = R4 * cosine/sine - I4			; 4-7		n 9
	zfmaddpd zmm7, zmm7, zmm28, zmm8			;;41 B4 = I4 * cosine/sine + R4			; 4-7		n 10

	vmovapd	zmm10, [srcreg+8*d1+2*d1]			;;42 R3
	vmovapd	zmm9, [srcreg+8*d1+2*d1+64]			;;42 I3
	zfmaddpd zmm8, zmm10, zmm28, zmm9			;;42 A3 = R3 * cosine/sine + I3			; 5-8		n 13
	zfmsubpd zmm9, zmm9, zmm28, zmm10			;;42 B3 = I3 * cosine/sine - R3			; 5-8		n 14

	vmovapd	zmm12, [srcreg+8*d1+3*d1]			;;42 R4
	vmovapd	zmm11, [srcreg+8*d1+3*d1+64]			;;42 I4
	zfmsubpd zmm10, zmm12, zmm28, zmm11			;;42 A4 = R4 * cosine/sine - I4			; 6-9		n 13
	zfmaddpd zmm11, zmm11, zmm28, zmm12			;;42 B4 = I4 * cosine/sine + R4			; 6-9		n 14

	vbroadcastsd zmm28, Q [sc4reg+8+128]			;;4x Sine2
	vmovapd	zmm13, [srcreg+4*d1]				;;41 R1
	zfmaddpd zmm12, zmm0, zmm28, zmm13			;;41 R1 + R2*sine (new R1)			; 7-10		n 15
	zfnmaddpd zmm0, zmm0, zmm28, zmm13			;;41 R1 - R2*sine (new R2)			; 7-10		n 16

	vmovapd	zmm14, [srcreg+4*d1+64]				;;41 I1
	zfmaddpd zmm13, zmm1, zmm28, zmm14			;;41 I1 + I2*sine (new I1)			; 8-11		n 17
	zfnmaddpd zmm1, zmm1, zmm28, zmm14			;;41 I1 - I2*sine (new I2)			; 8-11		n 18

	vmovapd	zmm15, [srcreg+8*d1]				;;42 R1								n 11
	vaddpd	zmm14, zmm6, zmm4				;;41 R4 + R3 (new R3)				; 9-12		n 15
	vsubpd	zmm6, zmm6, zmm4				;;41 R4 - R3 (new I4)				; 9-12		n 18

	vmovapd	zmm16, [srcreg+8*d1+64]				;;42 I1								n 12
	vaddpd	zmm4, zmm5, zmm7				;;41 I3 + I4 (new I3)				; 10-13		n 17
	vsubpd	zmm5, zmm5, zmm7				;;41 I3 - I4 (new R4)				; 10-13		n 16

	vbroadcastsd zmm27, Q [sc4reg+8]			;;4x Sine34							n 15
	zfmaddpd zmm7, zmm2, zmm28, zmm15			;;42 R1 + R2*sine (new R1)			; 11-14		n 19
	zfnmaddpd zmm2, zmm2, zmm28, zmm15			;;42 R1 - R2*sine (new R2)			; 11-14		n 20

	vmovapd	zmm26, [sc3reg+0*sc3gap+64]			;;30 cosine2/sine2						n 23
	zfmaddpd zmm15, zmm3, zmm28, zmm16			;;42 I1 + I2*sine (new I1)			; 12-15		n 21
	zfnmaddpd zmm3, zmm3, zmm28, zmm16			;;42 I1 - I2*sine (new I2)			; 12-15		n 22

	vmovapd	zmm25, [sc3reg+1*sc3gap+64]			;;31 cosine2/sine2						n 24
	vaddpd	zmm16, zmm10, zmm8				;;42 R4 + R3 (new R3)				; 13-16		n 19
	vsubpd	zmm10, zmm10, zmm8				;;42 R4 - R3 (new I4)				; 13-16		n 22

	vmovapd	zmm24, [sc3reg+2*sc3gap+64]			;;32 cosine2/sine2						n 25
	vaddpd	zmm8, zmm9, zmm11				;;42 I3 + I4 (new I3)				; 14-17		n 21
	vsubpd	zmm9, zmm9, zmm11				;;42 I3 - I4 (new R4)				; 14-17		n 20

	vmovapd	zmm23, [sc3reg+3*sc3gap+64]			;;33 cosine2/sine2						n 26
	zfmaddpd zmm11, zmm14, zmm27, zmm12			;;41 R1 + R3*sine (R2 in 30)			; 15-18		n 23
	zfnmaddpd zmm14, zmm14, zmm27, zmm12			;;41 R1 - R3*sine (R2 in 32)			; 15-18		n 25

	vmovapd	zmm22, [sc3reg+0*sc3gap+128+64]			;;30 cosine3/sine3						n 27
	zfmaddpd zmm12, zmm5, zmm27, zmm0			;;41 R2 + R4*sine (R2 in 31)			; 16-19		n 24
	zfnmaddpd zmm5, zmm5, zmm27, zmm0			;;41 R2 - R4*sine (R2 in 33)			; 16-19		n 26

	vmovapd	zmm21, [sc3reg+1*sc3gap+128+64]			;;31 cosine3/sine3						n 28
	zfmaddpd zmm0, zmm4, zmm27, zmm13			;;41 I1 + I3*sine (I2 in 30)			; 17-20		n 23
	zfnmaddpd zmm4, zmm4, zmm27, zmm13			;;41 I1 - I3*sine (I2 in 32)			; 17-20		n 25

	vmovapd	zmm20, [sc3reg+2*sc3gap+128+64]			;;32 cosine3/sine3						n 29
	zfmaddpd zmm13, zmm6, zmm27, zmm1			;;41 I2 + I4*sine (I2 in 31)			; 18-21		n 24
	zfnmaddpd zmm6, zmm6, zmm27, zmm1			;;41 I2 - I4*sine (I2 in 33)			; 18-21		n 26

	vmovapd	zmm19, [sc3reg+3*sc3gap+128+64]			;;33 cosine3/sine3						n 30
	zfmaddpd zmm1, zmm16, zmm27, zmm7			;;42 R1 + R3*sine (R3 in 30)			; 19-22		n 27
	zfnmaddpd zmm16, zmm16, zmm27, zmm7			;;42 R1 - R3*sine (R3 in 32)			; 19-22		n 29

	vmovapd	zmm18, [sc3reg+0*sc3gap]			;;30 sine2							n 31
	zfmaddpd zmm7, zmm9, zmm27, zmm2			;;42 R2 + R4*sine (R3 in 31)			; 20-23		n 28
	zfnmaddpd zmm9, zmm9, zmm27, zmm2			;;42 R2 - R4*sine (R3 in 33)			; 20-23		n 30

	vmovapd	zmm17, [sc3reg+1*sc3gap]			;;31 sine2							n 32
	zfmaddpd zmm2, zmm8, zmm27, zmm15			;;42 I1 + I3*sine (I3 in 30)			; 21-24		n 27
	zfnmaddpd zmm8, zmm8, zmm27, zmm15			;;42 I1 - I3*sine (I3 in 32)			; 21-24		n 29

	vmovapd	zmm28, [sc3reg+2*sc3gap]			;;32 sine2							n 33
	zfmaddpd zmm15, zmm10, zmm27, zmm3			;;42 I2 + I4*sine (I3 in 31)			; 22-25		n 28
	zfnmaddpd zmm10, zmm10, zmm27, zmm3			;;42 I2 - I4*sine (I3 in 33)			; 22-25		n 30

						;; Three complex comments	Six-reals comments
	vmovapd	zmm27, [sc3reg+3*sc3gap]			;;33 sine2							n 34
	zfmaddpd zmm3, zmm11, zmm26, zmm0			;;30 A2 = R2 * cosine2/sine2 + I2		; 23-26		n 31
	zfmsubpd zmm0, zmm0, zmm26, zmm11			;;30 B2 = I2 * cosine2/sine2 - R2		; 23-26		n 31

	vmovapd	zmm26, [sc3reg+0*sc3gap+128]			;;30 sine3							n 35
	zfmaddpd zmm11, zmm12, zmm25, zmm13			;;31 A2 = R2 * cosine2/sine2 + I2		; 24-27		n 32
	zfmsubpd zmm13, zmm13, zmm25, zmm12			;;31 B2 = I2 * cosine2/sine2 - R2		; 24-27		n 32

	vmovapd	zmm25, [srcreg+0*d1+2*d1]			;;40 R3								n 36
	zfmaddpd zmm12, zmm14, zmm24, zmm4			;;32 A2 = R2 * cosine2/sine2 + I2		; 25-28		n 33
	zfmsubpd zmm4, zmm4, zmm24, zmm14			;;32 B2 = I2 * cosine2/sine2 - R2		; 25-28		n 33

	vmovapd	zmm24, [srcreg+0*d1+2*d1+64]			;;40 I3								n 36
	zfmaddpd zmm14, zmm5, zmm23, zmm6			;;33 A2 = R2 * cosine2/sine2 + I2		; 26-29		n 34
	zfmsubpd zmm6, zmm6, zmm23, zmm5			;;33 B2 = I2 * cosine2/sine2 - R2		; 26-29		n 34

	vmovapd	zmm23, [sc4reg+0*128+64]			;;40 cosine/sine3						n 36
	zfmaddpd zmm5, zmm1, zmm22, zmm2			;;30 A3 = R3 * cosine3/sine3 + I3		; 27-30		n 35
	zfmsubpd zmm2, zmm2, zmm22, zmm1			;;30 B3 = I3 * cosine3/sine3 - R3		; 27-30		n 35

	vmovapd	zmm22, [srcreg+0*d1+d1]				;;40 R2								n 37
	zfmaddpd zmm1, zmm7, zmm21, zmm15			;;31 A3 = R3 * cosine3/sine3 + I3		; 28-31		n 39
	zfmsubpd zmm15, zmm15, zmm21, zmm7			;;31 B3 = I3 * cosine3/sine3 - R3		; 28-31		n 39

	vmovapd	zmm21, [srcreg+0*d1+d1+64]			;;40 I2								n 37
	zfmaddpd zmm7, zmm16, zmm20, zmm8			;;32 A3 = R3 * cosine3/sine3 + I3		; 29-32		n 42
	zfmsubpd zmm8, zmm8, zmm20, zmm16			;;32 B3 = I3 * cosine3/sine3 - R3		; 29-32		n 43

	vmovapd	zmm20, [sc4reg+1*128+64]			;;40 cosine/sine2						n 37
	zfmaddpd zmm16, zmm9, zmm19, zmm10			;;33 A3 = R3 * cosine3/sine3 + I3		; 30-33		n 43
	zfmsubpd zmm10, zmm10, zmm19, zmm9			;;33 B3 = I3 * cosine3/sine3 - R3		; 30-33		n 44

	vmovapd	zmm19, [srcreg+0*d1+3*d1]			;;40 R4								n 38
	vmulpd	zmm3, zmm3, zmm18				;;30 A2 = A2 * sine2 (R2)			; 31-34		n 47
	vmulpd	zmm0, zmm0, zmm18				;;30 B2 = B2 * sine2 (I2)			; 31-34		n 47

	vmovapd	zmm18, [srcreg+0*d1+3*d1+64]			;;40 I4								n 38
	vmulpd	zmm11, zmm11, zmm17				;;31 A2 = A2 * sine2 (R2)			; 32-35		n 52
	vmulpd	zmm13, zmm13, zmm17				;;31 B2 = B2 * sine2 (I2)			; 32-35		n 52

	vmovapd	zmm17, [sc4reg+2*128+64]			;;40 cosine/sine4						n 38
	vmulpd	zmm12, zmm12, zmm28				;;32 A2 = A2 * sine2 (R2)			; 33-36		n 65
	vmulpd	zmm4, zmm4, zmm28				;;32 B2 = B2 * sine2 (I2)			; 33-36		n 65

	vmovapd	zmm28, [sc3reg+1*sc3gap+128]			;;31 sine3							n 39
	vmulpd	zmm14, zmm14, zmm27				;;33 A2 = A2 * sine2 (R2)			; 34-37		n 66
	vmulpd	zmm6, zmm6, zmm27				;;33 B2 = B2 * sine2 (I2)			; 34-37		n 66

	vmovapd	zmm27, [sc4reg+0*128]				;;40 Sine3							n 40
	vmulpd	zmm5, zmm5, zmm26				;;30 A3 = A3 * sine3 (R3)			; 35-38		n 48
	vmulpd	zmm2, zmm2, zmm26				;;30 B3 = B3 * sine3 (I3)			; 35-38		n 47

	vmovapd	zmm26, [srcreg+0*d1+64]		;;40 I1				R1b						n 41
	zfmaddpd zmm9, zmm25, zmm23, zmm24			;;40 A3 = R3 * cosine/sine + I3			; 36-39		n 40
	zfmsubpd zmm24, zmm24, zmm23, zmm25			;;40 B3 = I3 * cosine/sine - R3			; 36-39		n 40

	vmovapd	zmm23, [sc4reg+1*128]		;;40 Sine2									n 41
	zfmaddpd zmm25, zmm22, zmm20, zmm21			;;40 A2 = R2 * cosine/sine + I2			; 37-40		n 41
	zfmsubpd zmm21, zmm21, zmm20, zmm22			;;40 B2 = I2 * cosine/sine - R2			; 37-40		n 41

	vmovapd	zmm20, [sc3reg+2*sc3gap+128]			;;32 sine3							n 42
	zfmaddpd zmm22, zmm19, zmm17, zmm18			;;40 A4 = R4 * cosine/sine + I4			; 38-41		n 44
	zfmsubpd zmm18, zmm18, zmm17, zmm19			;;40 B4 = I4 * cosine/sine - R4			; 38-41		n 45

	vmovapd	zmm17, [sc3reg+3*sc3gap+128]			;;33 sine3							n 43
	vmulpd	zmm1, zmm1, zmm28				;;31 A3 = A3 * sine3 (R3)			; 39-42		n 52
	vmulpd	zmm15, zmm15, zmm28				;;31 B3 = B3 * sine3 (I3)			; 39-42		n 53
	bump	sc3reg, sc3inc

	vmovapd	zmm19, [sc4reg+2*128]		;;40 Sine4									n 44
	vmulpd	zmm9, zmm9, zmm27				;;40 A3 = A3 * sine				; 40-43		n 44
	vmulpd	zmm24, zmm24, zmm27				;;40 B3 = B3 * sine				; 40-43		n 45
	bump	sc4reg, sc4inc

						;;40 Four complex comments	Eight reals comments
	vmovapd zmm27, zmm26			;;40 not important		R1b
	vmulpd	zmm27 {k7}, zmm25, zmm23	;;40 R2*sine			blend in R1b			; 41-44		n 46

	zfmaddpd zmm28, zmm21, zmm23, zmm26	;;40 i1+ = I1 + I2*sine		not important			; 41-44		n 51
	zfnmaddpd zmm26, zmm21, zmm23, zmm26	;;40 i1- = I1 - I2*sine		not important			; 42-45		n 50

	vmulpd	zmm7, zmm7, zmm20				;;32 A3 = A3 * sine3 (R3)			; 42-45		n 67
	vmulpd	zmm8, zmm8, zmm20				;;32 B3 = B3 * sine3 (I3)			; 43-46		n 65

	vmovapd	zmm20, [srcreg+0*d1]		;;40 R1				R1a						n 46
	vmulpd	zmm16, zmm16, zmm17				;;33 A3 = A3 * sine3 (R3)			; 43-46		n 68
	vmulpd	zmm10, zmm10, zmm17				;;33 B3 = B3 * sine3 (I3)			; 44-47		n 66

	L1prefetchw srcreg+srcinc+4*d1+d1, L1pt
	zfmaddpd zmm17, zmm22, zmm19, zmm9	;;40 r3+ = R3 + R4*sine		r3+ = R3 + R4*sine		; 44-47		n 49
	zfnmaddpd zmm22, zmm22, zmm19, zmm9	;;40 r3- = R3 - R4*sine		r3- = R3 - R4*sine		; 45-48		n 55

	L1prefetchw srcreg+srcinc+4*d1+d1+64, L1pt
	zfnmaddpd zmm9, zmm18, zmm19, zmm24	;;40 i3- = I3 - I4*sine		i3- = I3 - I4*sine		; 45-48		n 50
	zfmaddpd zmm18, zmm18, zmm19, zmm24	;;40 i3+ = I3 + I4*sine		i3+ = I3 + I4*sine		; 46-49		n 51

	L1prefetchw srcreg+srcinc+8*d1+d1, L1pt
	vaddpd	zmm24, zmm20, zmm27		;;40 r1+ = R1 + R2		r1+ = R1a + R1b			; 46-49		n 54
	vsubpd	zmm20, zmm20, zmm27		;;40 r1- = R1 - R2		r1- = R1a - R1b			; 47-50		n 56

	L1prefetchw srcreg+srcinc+8*d1+d1+64, L1pt
	vmovapd	zmm27, zmm3			;;30 not important		R2
	vaddpd	zmm27 {k7}, zmm0, zmm2		;;30 i2+ = I2 + I3		blend in R2			; 47-50		n 57
	vsubpd	zmm2 {k7}, zmm0, zmm2		;;30 i2- = I2 - I3		blend in I3			; 48-51		n 62

	L1prefetchw srcreg+srcinc+4*d1+2*d1, L1pt
	vsubpd	zmm0 {k7}, zmm3, zmm5		;;30 r2- = R2 - R3		blend in I2			; 48-51		n 61
	vaddpd	zmm5 {k7}, zmm3, zmm5		;;30 r2+ = R2 + R3		blend in R3			; 49-52		n 58

	L1prefetchw srcreg+srcinc+4*d1+2*d1+64, L1pt
	vmovapd zmm3, zmm17			;;40 r3+			not important
	vmulpd	zmm3 {k6}, zmm25, zmm23		;;40 blend in r3+		r2 = R2*sine			; 49-52		n 54
	vmulpd	zmm26 {k6}, zmm9, zmm30		;;40 blend in i1-		i3- = i3-*SQRTHALF		; 50-53		n 55
	vmulpd	zmm9 {k6}, zmm21, zmm23		;;40 blend in i3-		i2 = I2*sine			; 50-53		n 56

	vbroadcastsd zmm25, ZMM_HALF		;;.5										n 57
	vaddpd	zmm17 {k7}, zmm28, zmm18	;;40 I1 = i1+ + i3+ (I1 in 30)	blend in R5 = r3+		; 51-54		n 57
	vsubpd	zmm18 {k7}, zmm28, zmm18	;;40 I3 = i1+ - i3+ (I1 in 32)	blend in R7 = i3+		; 51-54		n 69

	vbroadcastsd zmm21, ZMM_P866		;;.866										n 61
	vmovapd	zmm28, zmm13			;;31 not important		I2
	vsubpd	zmm28 {k7}, zmm11, zmm1		;;31 r2- = R2 - R3		blend in I2			; 52-55		n 63
	vaddpd	zmm1 {k7}, zmm11, zmm1		;;31 r2+ = R2 + R3		blend in R3			; 52-55		n 60

	L1prefetchw srcreg+srcinc+4*d1+3*d1, L1pt
	vaddpd	zmm11 {k7}, zmm13, zmm15	;;31 i2+ = I2 + I3		blend in R2			; 53-56		n 59
	vsubpd	zmm15 {k7}, zmm13, zmm15	;;31 i2- = I2 - I3		blend in I3			; 53-56		n 64

	L1prefetchw srcreg+srcinc+4*d1+3*d1+64, L1pt
	vaddpd	zmm13, zmm24, zmm3		;;40 R1 = r1+ + r3+ (R1 in 30)	R1 = r1+ + r2			; 54-57		n 58
	vsubpd	zmm24, zmm24, zmm3		;;40 R3 = r1+ - r3+ (R1 in 32)	R3 = r1+ - r2			; 54-57		n 71

	L1prefetchw srcreg+srcinc+8*d1+2*d1, L1pt
	zfnmaddpd zmm3, zmm22, zmm31, zmm26	;;40 I2 = i1- - r3-*1 (I1 in 31) R6 = i3- - r3-*-SQRTHALF	; 55-58		n 59
	zfmaddpd zmm22, zmm22, zmm31, zmm26	;;40 I4 = i1- + r3-*1 (I1 in 33) R8 = i3- + r3-*-SQRTHALF	; 55-58		n 70

	L1prefetchw srcreg+srcinc+8*d1+2*d1+64, L1pt
	vaddpd	zmm26, zmm20, zmm9		;;40 R2 = r1- + i3- (R1 in 31)	R2 = r1- + i2			; 56-59		n 60
	vsubpd	zmm20, zmm20, zmm9		;;40 R4 = r1- - i3- (R1 in 33)	R4 = r1- - i2			; 56-59		n 72

	L1prefetchw srcreg+srcinc+8*d1+3*d1, L1pt
	vaddpd	zmm9, zmm17, zmm27		;;30 I1 = I1 + i2+		R4 = R1b + R2			; 57-60
	zfnmaddpd zmm27, zmm27, zmm25, zmm17	;;30 i1- = I1 - .5i2+		R1b- = R1b - .5*R2		; 57-60		n 61

	L1prefetchw srcreg+srcinc+8*d1+3*d1+64, L1pt
	vaddpd	zmm17, zmm13, zmm5		;;30 R1 = R1 + r2+		R1 = R1a + R3			; 58-61
	zfnmaddpd zmm5, zmm5, zmm25, zmm13	;;30 r1- = R1 - .5r2+		R1a- = R1a - .5*R3		; 58-61		n 62

	L1prefetchw srcreg+srcinc+4*d1, L1pt
	vaddpd	zmm13, zmm3, zmm11		;;31 I1 = I1 + i2+		R4 = R1b + R2			; 59-62
	zfnmaddpd zmm11, zmm11, zmm25, zmm3	;;31 i1- = I1 - .5i2+		R1b- = R1b - .5*R2		; 59-62		n 63

	L1prefetchw srcreg+srcinc+4*d1+64, L1pt
	vaddpd	zmm3, zmm26, zmm1		;;31 R1 = R1 + r2+		R1 = R1a + R3			; 60-63
	zfnmaddpd zmm1, zmm1, zmm25, zmm26	;;31 r1- = R1 - .5r2+		R1a- = R1a - .5*R3		; 60-63		n 64

	L1prefetchw srcreg+srcinc+8*d1, L1pt
	zfnmaddpd zmm26, zmm0, zmm21, zmm27	;;30 I2 = i1- - .866r2-		negR5 = R1b- - .866*I2		; 61-64		n 77
	zfmaddpd zmm0, zmm0, zmm21, zmm27	;;30 I3 = i1- + .866r2-		R6 = R1b- + .866*I2		; 61-64
	zstore	[srcreg+0*d1+64], zmm9		;;30 Save I1							; 61

	L1prefetchw srcreg+srcinc+8*d1+64, L1pt
	zfmaddpd zmm27, zmm2, zmm21, zmm5	;;30 R2 = r1- + .866i2-		R2 = R1a- + .866*I3		; 62-65
	zfnmaddpd zmm2, zmm2, zmm21, zmm5	;;30 R3 = r1- - .866i2-		R3 = R1a- - .866*I3		; 62-65
	zstore	[srcreg+0*d1], zmm17		;;30 Save R1							; 62

	L1prefetchw srcreg+srcinc+0*d1+2*d1, L1pt
	zfnmaddpd zmm5, zmm28, zmm21, zmm11	;;31 I2 = i1- - .866r2-		negR5 = R1b- - .866*I2		; 63-66		n 77
	zfmaddpd zmm28, zmm28, zmm21, zmm11	;;31 I3 = i1- + .866r2-		R6 = R1b- + .866*I2		; 63-66
	zstore	[srcreg+1*d1+64], zmm13		;;31 Save I1							; 63

	L1prefetchw srcreg+srcinc+0*d1+2*d1+64, L1pt
	zfmaddpd zmm11, zmm15, zmm21, zmm1	;;31 R2 = r1- + .866i2-		R2 = R1a- + .866*I3		; 64-67
	zfnmaddpd zmm15, zmm15, zmm21, zmm1	;;31 R3 = r1- - .866i2-		R3 = R1a- - .866*I3		; 64-67
	zstore	[srcreg+1*d1], zmm3		;;31 Save R1							; 64

	L1prefetchw srcreg+srcinc+0*d1+d1, L1pt
	vmovapd	zmm1, zmm12			;;32 not important		R2
	vaddpd	zmm1 {k7}, zmm4, zmm8		;;32 i2+ = I2 + I3		blend in R2			; 65-68		n 69
	vsubpd	zmm8 {k7}, zmm4, zmm8		;;32 i2- = I2 - I3		blend in I3			; 65-68		n 75
	zstore	[srcreg+0*d1+8*d1+64], zmm0	;;30 Save I3							; 65

	L1prefetchw srcreg+srcinc+0*d1+d1+64, L1pt
	vmovapd	zmm9, zmm14			;;33 not important		R2
	vaddpd	zmm9 {k7}, zmm6, zmm10		;;33 i2+ = I2 + I3		blend in R2			; 66-69		n 70
	vsubpd	zmm10 {k7}, zmm6, zmm10		;;33 i2- = I2 - I3		blend in I3			; 66-69		n 76
	zstore	[srcreg+0*d1+4*d1], zmm27	;;30 Save R2							; 66

	L1prefetchw srcreg+srcinc+0*d1+3*d1, L1pt
	vsubpd	zmm4 {k7}, zmm12, zmm7		;;32 r2- = R2 - R3		blend in I2			; 67-70		n 73
	vaddpd	zmm7 {k7}, zmm12, zmm7		;;32 r2+ = R2 + R3		blend in R3			; 67-70		n 71
	zstore	[srcreg+0*d1+8*d1], zmm2	;;30 Save R3							; 67

	L1prefetchw srcreg+srcinc+0*d1+3*d1+64, L1pt
	vsubpd	zmm6 {k7}, zmm14, zmm16		;;33 r2- = R2 - R3		blend in I2			; 68-71		n 74
	vaddpd	zmm16 {k7}, zmm14, zmm16	;;33 r2+ = R2 + R3		blend in R3			; 68-71		n 72
	zstore	[srcreg+1*d1+8*d1+64], zmm28	;;31 Save I3							; 68

	L1prefetchw srcreg+srcinc+0*d1+64, L1pt
	vaddpd	zmm14, zmm18, zmm1		;;32 I1 = I1 + i2+		R4 = R1b + R2			; 69-72
	zfnmaddpd zmm1, zmm1, zmm25, zmm18	;;32 i1- = I1 - .5i2+		R1b- = R1b - .5*R2		; 69-72		n 73
	zstore	[srcreg+1*d1+4*d1], zmm11	;;31 Save R2							; 69

	L1prefetchw srcreg+srcinc+0*d1, L1pt
	vaddpd	zmm18, zmm22, zmm9		;;33 I1 = I1 + i2+		R4 = R1b + R2			; 70-73
	zfnmaddpd zmm9, zmm9, zmm25, zmm22	;;33 i1- = I1 - .5i2+		R1b- = R1b - .5*R2		; 70-73		n 74
	zstore	[srcreg+1*d1+8*d1], zmm15	;;31 Save R3							; 70

	vaddpd	zmm22, zmm24, zmm7		;;32 R1 = R1 + r2+		R1 = R1a + R3			; 71-74
	zfnmaddpd zmm7, zmm7, zmm25, zmm24	;;32 r1- = R1 - .5r2+		R1a- = R1a - .5*R3		; 71-74		n 75
	zstore	[srcreg+2*d1+64], zmm14		;;32 Save I1							; 71

	vaddpd	zmm24, zmm20, zmm16		;;33 R1 = R1 + r2+		R1 = R1a + R3			; 72-75
	zfnmaddpd zmm16, zmm16, zmm25, zmm20	;;33 r1- = R1 - .5r2+		R1a- = R1a - .5*R3		; 72-75		n 76
	zstore	[srcreg+3*d1+64], zmm18		;;33 Save I1							; 72

	zfnmaddpd zmm20, zmm4, zmm21, zmm1	;;32 I2 = i1- - .866r2-		negR5 = R1b- - .866*I2		; 73-76		n 78
	zfmaddpd zmm4, zmm4, zmm21, zmm1	;;32 I3 = i1- + .866r2-		R6 = R1b- + .866*I2		; 73-76
	zstore	[srcreg+2*d1], zmm22		;;32 Save R1							; 73

	zfnmaddpd zmm1, zmm6, zmm21, zmm9	;;33 I2 = i1- - .866r2-		negR5 = R1b- - .866*I2		; 74-77		n 78
	zfmaddpd zmm6, zmm6, zmm21, zmm9	;;33 I3 = i1- + .866r2-		R6 = R1b- + .866*I2		; 74-77
	zstore	[srcreg+3*d1], zmm24		;;33 Save R1							; 74

	zfmaddpd zmm9, zmm8, zmm21, zmm7	;;32 R2 = r1- + .866i2-		R2 = R1a- + .866*I3		; 75-78
	zfnmaddpd zmm8, zmm8, zmm21, zmm7	;;32 R3 = r1- - .866i2-		R3 = R1a- - .866*I3		; 75-78
	zstore	[srcreg+2*d1+8*d1+64], zmm4	;;32 Save I3							; 75

	zfmaddpd zmm7, zmm10, zmm21, zmm16	;;33 R2 = r1- + .866i2-		R2 = R1a- + .866*I3		; 76-79
	zfnmaddpd zmm10, zmm10, zmm21, zmm16	;;33 R3 = r1- - .866i2-		R3 = R1a- - .866*I3		; 76-79
	zstore	[srcreg+3*d1+8*d1+64], zmm6	;;33 Save I3							; 76

	vsubpd	zmm26 {k6}, zmm29, zmm26	;;30 blend in I2		R5 = 0 - negR5			; 77-80
	vsubpd	zmm5 {k6}, zmm29, zmm5		;;31 blend in I2		R5 = 0 - negR5			; 77-80
	zstore	[srcreg+2*d1+4*d1], zmm9	;;32 Save R2							; 77

	vsubpd	zmm20 {k6}, zmm29, zmm20	;;32 blend in I2		R5 = 0 - negR5			; 78-81
	vsubpd	zmm1 {k6}, zmm29, zmm1		;;33 blend in I2		R5 = 0 - negR5			; 78-81
	zstore	[srcreg+2*d1+8*d1], zmm8	;;32 Save R3							; 78

	zstore	[srcreg+3*d1+4*d1], zmm7	;;33 Save R2
	zstore	[srcreg+3*d1+8*d1], zmm10	;;33 Save R3
	zstore	[srcreg+0*d1+4*d1+64], zmm26	;;30 Save I2
	zstore	[srcreg+1*d1+4*d1+64], zmm5	;;31 Save I2
	zstore	[srcreg+2*d1+4*d1+64], zmm20	;;32 Save I2
	zstore	[srcreg+3*d1+4*d1+64], zmm1	;;33 Save I2
	bump	srcreg, srcinc
	ENDM
