; Copyright 2017-2024 - Mersenne Research, Inc.  All rights reserved
; Author:  George Woltman
; Email: woltman@alum.mit.edu
;

;;
;; Macros for an AVX-512 radix-8 step in an FFT.  This is used in our radix-4/8 FFTs
;; whenever possible.
;;


;;
;; ************************************* eight-complex-fft variants ******************************************
;;

;; This code does an eight complex forward FFT and then applies seven sin/cos multipliers.

; Basic eight-complex DJB FFT building block
zr8_eight_complex_djbfft_preload MACRO
	zr8_8c_djbfft_cmn_preload
	ENDM
zr8_eight_complex_djbfft MACRO srcreg,srcinc,d1,d2,d4,screg,scinc,maxrpt,L1pt,L1pd
	zr8_8c_djbfft_cmn srcreg,0,srcinc,d1,d2,d4,noexec,0,0,screg,scinc,maxrpt,L1pt,L1pd
	ENDM

; Like the basic version except source data is offset by [rbx]
zr8f_eight_complex_djbfft_preload MACRO
	zr8_8c_djbfft_cmn_preload
	ENDM
zr8f_eight_complex_djbfft MACRO srcreg,srcinc,d1,d2,d4,screg,scinc,maxrpt,L1pt,L1pd
	zr8_8c_djbfft_cmn srcreg,rbx,srcinc,d1,d2,d4,noexec,0,0,screg,scinc,maxrpt,L1pt,L1pd
	ENDM

; Like the basic version except vbroadcastsd is used to reduce sin/cos data
zr8b_eight_complex_djbfft_preload MACRO
	zr8_8c_djbfft_cmn_preload
	ENDM
zr8b_eight_complex_djbfft MACRO srcreg,srcinc,d1,d2,d4,screg,scinc,maxrpt,L1pt,L1pd
	zr8_8c_djbfft_cmn srcreg,0,srcinc,d1,d2,d4,exec,screg,16,screg,scinc,maxrpt,L1pt,L1pd
	ENDM

; Like the zr8b version except data is loaded from a larger sixteen-real/eight_complex sin/cos table
zr8rb_eight_complex_djbfft_preload MACRO
	zr8_8c_djbfft_cmn_preload
	ENDM
zr8rb_eight_complex_djbfft MACRO srcreg,srcinc,d1,d2,d4,screg,scinc,maxrpt,L1pt,L1pd
	zr8_8c_djbfft_cmn srcreg,0,srcinc,d1,d2,d4,exec,screg+8,128,screg,scinc,maxrpt,L1pt,L1pd
	ENDM

;; Common macro to operate on 8 complex values doing 3 levels of the FFT, applying
;; the sin/cos multipliers afterwards.

;; To calculate a 8-complex FFT in a brute force way (using a shorthand notation):
;; c1 + c2 + ... + c8 * w^00000000
;; c1 + c2 + ... + c8 * w^01234567
;; c1 + c2 + ... + c8 * w^02468ACE
;; c1 + c2 + ... + c8 * w^0369C...
;; c1 + c2 + ... + c8 * w^048C....
;; c1 + c2 + ... + c8 * w^05A.....
;; c1 + c2 + ... + c8 * w^06C.....
;; c1 + c2 + ... + c8 * w^07E.....
;;
;; The sin/cos values (w = 8th root of unity) are:
;; w^1 = .707 + .707i
;; w^2 = 0 + 1i
;; w^3 = -.707 + .707i
;; w^4 = -1
;;
;; Applying the sin/cos values above:
;; reals:
;; r1     +r2 +r3     +r4 +r5     +r6 +r7     +r8
;; r1 +.707r2     -.707r4 -r5 -.707r6     +.707r8  -.707i2 -i3 -.707i4 +.707i6 +i7 +.707i8
;; r1         -r3         +r5         -r7              -i2         +i4     -i6         +i8
;; r1 -.707r2     +.707r4 -r5 +.707r6     -.707r8  -.707i2 +i3 -.707i4 +.707i6 -i7 +.707i8
;; r1     -r2 +r3     -r4 +r5     -r6 +r7     -r8
;; r1 -.707r2     +.707r4 -r5 +.707r6     -.707r8  +.707i2 -i3 +.707i4 -.707i6 +i7 -.707i8
;; r1         -r3         +r5         -r7              +i2         -i4     +i6         -i8
;; r1 +.707r2     -.707r4 -r5 -.707r6     +.707r8  +.707i2 +i3 +.707i4 -.707i6 -i7 -.707i8
;; imaginarys:
;;                                         +i1     +i2 +i3     +i4 +i5     +i6 +i7     +i8
;; +.707r2 +r3 +.707r4 -.707r6 -r7 -.707r8 +i1 +.707i2     -.707i4 -i5 -.707i6     +.707i8
;;     +r2         -r4     +r6         -r8 +i1         -i3         +i5         -i7
;; +.707r2 -r3 +.707r4 -.707r6 +r7 -.707r8 +i1 -.707i2     +.707i4 -i5 +.707i6     -.707i8
;;                                         +i1     -i2 +i3     -i4 +i5     -i6 +i7     -i8
;; -.707r2 +r3 -.707r4 +.707r6 -r7 +.707r8 +i1 -.707i2     +.707i4 -i5 +.707i6     -.707i8
;;     -r2         +r4     -r6         +r8 +i1         -i3         +i5         -i7
;; -.707r2 -r3 -.707r4 +.707r6 +r7 +.707r8 +i1 +.707i2     -.707i4 -i5 -.707i6     +.707i8
;;
;; Simplifying, we get:
;;R1 = ((r1+r5)+(r3+r7))      +((r2+r6)+(r4+r8))
;;R5 = ((r1+r5)+(r3+r7))      -((r2+r6)+(r4+r8))
;;R3 = ((r1+r5)-(r3+r7))                              -((i2+i6)-(i4+i8))
;;R7 = ((r1+r5)-(r3+r7))                              +((i2+i6)-(i4+i8))
;;R2 = ((r1-r5)+.707((r2-r6)-(r4-r8)))  - ((i3-i7)+.707((i2-i6)+(i4-i8)))
;;R8 = ((r1-r5)+.707((r2-r6)-(r4-r8)))  + ((i3-i7)+.707((i2-i6)+(i4-i8)))
;;R4 = ((r1-r5)-.707((r2-r6)-(r4-r8)))  + ((i3-i7)-.707((i2-i6)+(i4-i8))) 
;;R6 = ((r1-r5)-.707((r2-r6)-(r4-r8)))  - ((i3-i7)-.707((i2-i6)+(i4-i8))) 

;;I1 = ((i1+i5)+(i3+i7))      +((i2+i6)+(i4+i8))
;;I5 = ((i1+i5)+(i3+i7))      -((i2+i6)+(i4+i8))
;;I3 = ((i1+i5)-(i3+i7))                              +((r2+r6)-(r4+r8))                        
;;I7 = ((i1+i5)-(i3+i7))                              -((r2+r6)-(r4+r8))                        
;;I2 = ((i1-i5)+.707((i2-i6)-(i4-i8)))  + ((r3-r7)+.707((r2-r6)+(r4-r8)))
;;I8 = ((i1-i5)+.707((i2-i6)-(i4-i8)))  - ((r3-r7)+.707((r2-r6)+(r4-r8)))
;;I4 = ((i1-i5)-.707((i2-i6)-(i4-i8)))  - ((r3-r7)-.707((r2-r6)+(r4-r8)))
;;I6 = ((i1-i5)-.707((i2-i6)-(i4-i8)))  + ((r3-r7)-.707((r2-r6)+(r4-r8)))

zr8_8c_djbfft_cmn_preload MACRO
	vbroadcastsd zmm31, ZMM_SQRTHALF
	ENDM
zr8_8c_djbfft_cmn MACRO srcreg,srcoff,srcinc,d1,d2,d4,bcast,bcreg,bcsz,screg,scinc,maxrpt,L1pt,L1pd
	vmovapd	zmm1, [srcreg+srcoff+d1]		;; R2
	vmovapd	zmm2, [srcreg+srcoff+d4+d1]		;; R6
	vsubpd	zmm0, zmm1, zmm2			;; R2-R6				; 1-4		n 9
	vaddpd	zmm1, zmm1, zmm2			;; R2+R6				; 1-4		n 11

	vmovapd	zmm3, [srcreg+srcoff+d2+d1]		;; R4
	vmovapd	zmm4, [srcreg+srcoff+d4+d2+d1]		;; R8
	vsubpd	zmm2, zmm3, zmm4			;; R4-R8				; 2-5		n 9
	vaddpd	zmm3, zmm3, zmm4			;; R4+R8				; 2-5		n 11

	vmovapd	zmm5, [srcreg+srcoff+d2+d1+64]		;; I4
	vmovapd	zmm6, [srcreg+srcoff+d4+d2+d1+64]	;; I8
	vsubpd	zmm4, zmm5, zmm6			;; I4-I8				; 3-6		n 10
	vaddpd	zmm5, zmm5, zmm6			;; I4+I8				; 3-6		n 12

	vmovapd	zmm7, [srcreg+srcoff+d1+64]		;; I2
	vmovapd	zmm8, [srcreg+srcoff+d4+d1+64]		;; I6
	vsubpd	zmm6, zmm7, zmm8			;; I2-I6				; 4-7		n 10
	vaddpd	zmm7, zmm7, zmm8			;; I2+I6				; 4-7		n 12

	vmovapd	zmm9, [srcreg+srcoff]			;; R1
	vmovapd	zmm10, [srcreg+srcoff+d4]		;; R5
	vaddpd	zmm8, zmm9, zmm10			;; R1+R5				; 5-8		n 13
	vsubpd	zmm9, zmm9, zmm10			;; R1-R5				; 5-8		n 14

	vmovapd	zmm11, [srcreg+srcoff+d2]		;; R3
	vmovapd	zmm12, [srcreg+srcoff+d4+d2]		;; R7
	vaddpd	zmm10, zmm11, zmm12			;; R3+R7				; 6-9		n 13
	vsubpd	zmm11, zmm11, zmm12			;; R3-R7				; 6-9		n 17

	vmovapd	zmm13, [srcreg+srcoff+64]		;; I1
	vmovapd	zmm14, [srcreg+srcoff+d4+64]		;; I5
	vaddpd	zmm12, zmm13, zmm14			;; I1+I5				; 7-10		n 16
	vsubpd	zmm13, zmm13, zmm14			;; I1-I5				; 7-10		n 15

	vmovapd	zmm15, [srcreg+srcoff+d2+64]		;; I3
	vmovapd	zmm16, [srcreg+srcoff+d4+d2+64]		;; I7
	vaddpd	zmm14, zmm15, zmm16			;; I3+I7				; 8-11		n 16
	vsubpd	zmm15, zmm15, zmm16			;; I3-I7				; 8-11		n 18

	vaddpd	zmm16, zmm0, zmm2		;; r2-+ = (r2-r6) + (r4-r8)			; 9-12		n 17
	vsubpd	zmm0, zmm0, zmm2		;; r2-- = (r2-r6) - (r4-r8)			; 9-12		n 14
bcast	vbroadcastsd zmm30, [bcreg+1*bcsz]	;; sine for R3/I3 and R7/I7 (w^2)
no bcast vmovapd zmm30, [screg+1*128]		;; sine for R3/I3 and R7/I7

	vsubpd	zmm2, zmm6, zmm4		;; i2-- = (i2-i6) - (i4-i8)			; 10-13		n 15
	vaddpd	zmm6, zmm6, zmm4		;; i2-+ = (i2-i6) + (i4-i8)			; 10-13		n 18
bcast	vbroadcastsd zmm23, [bcreg+0*bcsz]	;; sine for R2/I2 and R8/I8 (w^1)
no bcast vmovapd zmm23, [screg+0*128]		;; sine for R2/I2 and R8/I8

	vsubpd	zmm4, zmm1, zmm3		;; r2+- = (r2+r6) - (r4+r8)			; 11-14		n 19
	vaddpd	zmm1, zmm1, zmm3		;; r2++ = (r2+r6) + (r4+r8)			; 11-14		n 22
bcast	vbroadcastsd zmm24, [bcreg+2*bcsz]	;; sine for R4/I4 and R6/I6 (w^3)
no bcast vmovapd zmm24, [screg+2*128]		;; sine for R4/I4 and R6/I6

	vsubpd	zmm3, zmm7, zmm5		;; i2+- = (i2+i6) - (i4+i8)			; 12-15		n 19
	vaddpd	zmm7, zmm7, zmm5		;; i2++ = (i2+i6) + (i4+i8)			; 12-15		n 23
bcast	vbroadcastsd zmm26, [bcreg+1*bcsz+bcsz/2];; cosine/sine for R3/I3 and R7/I7 (w^2)
no bcast vmovapd zmm26, [screg+1*128+64]	;; cosine/sine for R3/I3 and R7/I7

	vaddpd	zmm5, zmm8, zmm10		;; r1++ = (r1+r5) + (r3+r7)			; 13-16		n 22
	vsubpd	zmm8, zmm8, zmm10		;; r1+- = (r1+r5) - (r3+r7)			; 13-16		n 24
bcast	vbroadcastsd zmm29, [bcreg+3*bcsz+bcsz/2];; cosine/sine for R5/I5 (w^4)
no bcast vmovapd zmm29, [screg+3*128+64]	;; cosine/sine for R5/I5

	zfmaddpd zmm10, zmm0, zmm31, zmm9	;; r1-+ = (r1-r5) + .707*(r2--)			; 14-17		n 20
	zfnmaddpd zmm0, zmm0, zmm31, zmm9	;; r1-- = (r1-r5) - .707*(r2--)			; 14-17		n 21
bcast	vbroadcastsd zmm27, [bcreg+0*bcsz+bcsz/2];; cosine/sine for R2/I2 and R8/I8 (w^1)
no bcast vmovapd zmm27, [screg+0*128+64]	;; cosine/sine for R2/I2 and R8/I8

	zfmaddpd zmm9, zmm2, zmm31, zmm13	;; i1-+ = (i1-i5) + .707*(i2--)			; 15-18		n 20
	zfnmaddpd zmm2, zmm2, zmm31, zmm13	;; i1-- = (i1-i5) - .707*(i2--)			; 15-18		n 21
bcast	vbroadcastsd zmm28, [bcreg+2*bcsz+bcsz/2];; cosine/sine for R4/I4 and R6/I6 (w^3)
no bcast vmovapd zmm28, [screg+2*128+64]	;; cosine/sine for R4/I4 and R6/I6

	vaddpd	zmm13, zmm12, zmm14		;; i1++ = (i1+i5) + (i3+i7)			; 16-19		n 23
	vsubpd	zmm12, zmm12, zmm14		;; i1+- = (i1+i5) - (i3+i7)			; 16-19		n 24
bcast	vbroadcastsd zmm25, [bcreg+3*bcsz]	;; sine for R5/I5 (w^4)
no bcast vmovapd zmm25, [screg+3*128]		;; sine for R5/I5

	zfmaddpd zmm14, zmm16, zmm31, zmm11	;; r3-+ = (r3-r7) + .707*(r2-+)			; 17-20		n 26
	zfnmaddpd zmm16, zmm16, zmm31, zmm11	;; r3-- = (r3-r7) - .707*(r2-+)			; 17-20		n 30
	bump	screg, scinc
	L1prefetchw srcreg+d1+L1pd, L1pt

	zfmaddpd zmm11, zmm6, zmm31, zmm15	;; i3-+ = (i3-i7) + .707*(i2-+)			; 18-21		n 26
	zfnmaddpd zmm6, zmm6, zmm31, zmm15	;; i3-- = (i3-i7) - .707*(i2-+)			; 18-21		n 30
	L1prefetchw srcreg+d4+d1+L1pd, L1pt

	vmulpd	zmm4, zmm4, zmm30		;; r2+-s = r2+- * sine37			; 19-22		n 24
	vmulpd	zmm3, zmm3, zmm30		;; i2+-s = i2+- * sine37			; 19-22		n 24
	L1prefetchw srcreg+d2+d1+L1pd, L1pt

	vmulpd	zmm10, zmm10, zmm23		;; r1-+s = r1-+ * sine28			; 20-23		n 26
	vmulpd	zmm9, zmm9, zmm23		;; i1-+s = i1-+ * sine28			; 20-23		n 26
	L1prefetchw srcreg+d4+d2+d1+L1pd, L1pt

	vmulpd	zmm0, zmm0, zmm24		;; r1--s = r1-- * sine46			; 21-24		n 30
	vmulpd	zmm2, zmm2, zmm24		;; i1--s = i1-- * sine46			; 21-24		n 30
	L1prefetchw srcreg+d2+d1+64+L1pd, L1pt

	vsubpd	zmm15, zmm5, zmm1		;; R5 = (r1++) - (r2++)				; 22-25		n 32
	vaddpd	zmm5, zmm5, zmm1		;; R1 = (r1++) + (r2++)				; 22-25
	L1prefetchw srcreg+d4+d2+d1+64+L1pd, L1pt

	vsubpd	zmm1, zmm13, zmm7		;; I5 = (i1++) - (i2++)				; 23-26		n 32
	vaddpd	zmm13, zmm13, zmm7		;; I1 = (i1++) + (i2++)				; 23-26
	L1prefetchw srcreg+d1+64+L1pd, L1pt

	zfmsubpd zmm7, zmm8, zmm30, zmm3	;; R3s = (r1+-)*sine37 - (i2+-s)		; 24-27		n 28
	zfmaddpd zmm17, zmm12, zmm30, zmm4	;; I3s = (i1+-)*sine37 + (r2+-s)		; 24-27		n 28
	L1prefetchw srcreg+d4+d1+64+L1pd, L1pt

	zfmaddpd zmm8, zmm8, zmm30, zmm3	;; R7s = (r1+-)*sine37 + (i2+-s)		; 25-28		n 32
	zfmsubpd zmm12, zmm12, zmm30, zmm4	;; I7s = (i1+-)*sine37 - (r2+-s)		; 25-28		n 32
	L1prefetchw srcreg+L1pd, L1pt

	zfnmaddpd zmm3, zmm11, zmm23, zmm10	;; R2s = (r1-+s) - sine28*(i3-+)		; 26-29		n 33
	zfmaddpd zmm4, zmm14, zmm23, zmm9	;; I2s = (i1-+s) + sine28*(r3-+)		; 26-29		n 33
	zstore	[srcreg], zmm5			;; Store R1					; 26
	L1prefetchw srcreg+d4+L1pd, L1pt

	zfmaddpd zmm11, zmm11, zmm23, zmm10	;; R8s = (r1-+s) + sine28*(i3-+)		; 27-30		n 34
	zfnmaddpd zmm14, zmm14, zmm23, zmm9	;; I8s = (i1-+s) - sine28*(r3-+)		; 27-30		n 34
	zstore	[srcreg+64], zmm13		;; Store I1					; 27
	L1prefetchw srcreg+d2+L1pd, L1pt

	zfmsubpd zmm10, zmm7, zmm26, zmm17	;; R3s * cosine/sine - I3s (final R3)		; 28-31
	zfmaddpd zmm17, zmm17, zmm26, zmm7	;; I3s * cosine/sine + R3s (final I3)		; 28-31
	L1prefetchw srcreg+d4+d2+L1pd, L1pt

	zfmaddpd zmm9, zmm8, zmm26, zmm12	;; R7s * cosine/sine + I7s (final R7)		; 29-32
	zfmsubpd zmm12, zmm12, zmm26, zmm8	;; I7s * cosine/sine - R7s (final I7)		; 29-32
	L1prefetchw srcreg+64+L1pd, L1pt

	zfmaddpd zmm7, zmm6, zmm24, zmm0	;; R4s = (r1--s) + sine46*(i3--)		; 30-33		n 35
	zfnmaddpd zmm8, zmm16, zmm24, zmm2	;; I4s = (i1--s) - sine46*(r3--)		; 30-33		n 35
	L1prefetchw srcreg+d4+64+L1pd, L1pt

	zfnmaddpd zmm6, zmm6, zmm24, zmm0	;; R6s = (r1--s) - sine46*(i3--)		; 31-34		n 36
	zfmaddpd zmm16, zmm16, zmm24, zmm2	;; I6s = (i1--s) + sine46*(r3--)		; 31-34		n 36
	L1prefetchw srcreg+d2+64+L1pd, L1pt

	zfmsubpd zmm0, zmm15, zmm29, zmm1	;; A5 = R5 * cosine/sine - I5			; 32-35		n 37
	zfmaddpd zmm1, zmm1, zmm29, zmm15	;; B5 = I5 * cosine/sine + R5			; 32-35		n 37
	zstore	[srcreg+d2], zmm10		;; Store R3					; 32
	L1prefetchw srcreg+d4+d2+64+L1pd, L1pt

	zfmsubpd zmm2, zmm3, zmm27, zmm4	;; R2s * cosine/sine - I2s (final R2)		; 33-36
	zfmaddpd zmm4, zmm4, zmm27, zmm3	;; I2s * cosine/sine + R2s (final I2)		; 33-36
	zstore	[srcreg+d2+64], zmm17		;; Store I3					; 32+1

	zfmaddpd zmm15, zmm11, zmm27, zmm14	;; R8s * cosine/sine + I8s (final R8)		; 34-37
	zfmsubpd zmm14, zmm14, zmm27, zmm11	;; I8s * cosine/sine - R8s (final I8)		; 34-37
	zstore	[srcreg+d4+d2], zmm9		;; Store R7					; 33+1

	zfmsubpd zmm3, zmm7, zmm28, zmm8	;; R4s * cosine/sine - I4s (final R4)		; 35-38
	zfmaddpd zmm8, zmm8, zmm28, zmm7	;; I4s * cosine/sine + R4s (final I4)		; 35-38
	zstore	[srcreg+d4+d2+64], zmm12	;; Store I7					; 33+2

	zfmaddpd zmm11, zmm6, zmm28, zmm16	;; R6s * cosine/sine + I6s (final R6)		; 36-39
	zfmsubpd zmm16, zmm16, zmm28, zmm6	;; I6s * cosine/sine - R6s (final I6)		; 36-39

	vmulpd	zmm0, zmm0, zmm25		;; A5 = A5 * sine (final R5)			; 37-40
	vmulpd	zmm1, zmm1, zmm25		;; B5 = B5 * sine (final I5)			; 37-40

	zstore	[srcreg+d1], zmm2		;; Store R2					; 37
	zstore	[srcreg+d1+64], zmm4		;; Store I2					; 37+1
	zstore	[srcreg+d4+d2+d1], zmm15	;; Store R8					; 38+1
	zstore	[srcreg+d4+d2+d1+64], zmm14	;; Store I8					; 38+2
	zstore	[srcreg+d2+d1], zmm3		;; Store R4					; 39+2
	zstore	[srcreg+d2+d1+64], zmm8		;; Store I4					; 39+3
	zstore	[srcreg+d4+d1], zmm11		;; Store R6					; 40+3
	zstore	[srcreg+d4+d1+64], zmm16	;; Store I6					; 40+4
	zstore	[srcreg+d4], zmm0		;; Store R5					; 41+4
	zstore	[srcreg+d4+64], zmm1		;; Store I5					; 41+5
	bump	srcreg, srcinc
	ENDM


;; ************************************* eight-complex-djbunfft variants ******************************************

;; This code applies the 7 sin/cos postmultipliers before a radix-8 butterfly.

; Basic eight-complex DJB inverse FFT building block
zr8_eight_complex_djbunfft_preload MACRO
	zr8_8c_djbunfft_cmn_preload
	ENDM
zr8_eight_complex_djbunfft MACRO srcreg,srcinc,d1,d2,d4,screg,scinc,maxrpt,L1pt,L1pd
	zr8_8c_djbunfft_cmn srcreg,srcinc,d1,d2,d4,noexec,0,0,screg,scinc,maxrpt,L1pt,L1pd
	ENDM

; Like the basic version except vbroadcastsd is used to reduce sin/cos data
zr8b_eight_complex_djbunfft_preload MACRO
	zr8_8c_djbunfft_cmn_preload
	ENDM
zr8b_eight_complex_djbunfft MACRO srcreg,srcinc,d1,d2,d4,screg,scinc,maxrpt,L1pt,L1pd
	zr8_8c_djbunfft_cmn srcreg,srcinc,d1,d2,d4,exec,screg,16,screg,scinc,maxrpt,L1pt,L1pd
	ENDM

; Like the zr8b version except data is loaded from a larger sixteen-real/eight_complex sin/cos table
zr8rb_eight_complex_djbunfft_preload MACRO
	zr8_8c_djbunfft_cmn_preload
	ENDM
zr8rb_eight_complex_djbunfft MACRO srcreg,srcinc,d1,d2,d4,screg,scinc,maxrpt,L1pt,L1pd
	zr8_8c_djbunfft_cmn srcreg,srcinc,d1,d2,d4,exec,screg+8,128,screg,scinc,maxrpt,L1pt,L1pd
	ENDM

;; Common macro to operate on 8 complex values doing 3 levels of the inverse FFT, applying
;; the sin/cos multipliers beforehand.

;; To calculate a 8-complex inverse FFT in a brute force way (using a shorthand notation):
;; c1 + c2 + ... + c8 * w^-00000000
;; c1 + c2 + ... + c8 * w^-01234567
;; c1 + c2 + ... + c8 * w^-02468ACE
;; c1 + c2 + ... + c8 * w^-0369C...
;; c1 + c2 + ... + c8 * w^-048C....
;; c1 + c2 + ... + c8 * w^-05A.....
;; c1 + c2 + ... + c8 * w^-06C.....
;; c1 + c2 + ... + c8 * w^-07E.....
;;
;; The sin/cos values (w = 8th root of unity) are:
;; w^-1 = .707 - .707i
;; w^-2 = 0 - 1i
;; w^-3 = -.707 - .707i
;; w^-4 = -1
;;
;; Applying the sin/cos values above, you get the same results as the forward FFT except that
;; for the real results we flip the sign of the imaginary inputs and for the imaginary results
;; we flip the sign of the real inputs.
;; reals:
;; r1     +r2 +r3     +r4 +r5     +r6 +r7     +r8
;; r1 +.707r2     -.707r4 -r5 -.707r6     +.707r8  +.707i2 +i3 +.707i4 -.707i6 -i7 -.707i8
;; r1         -r3         +r5         -r7              +i2         -i4     +i6         -i8
;; r1 -.707r2     +.707r4 -r5 +.707r6     -.707r8  +.707i2 -i3 +.707i4 -.707i6 +i7 -.707i8
;; r1     -r2 +r3     -r4 +r5     -r6 +r7     -r8
;; r1 -.707r2     +.707r4 -r5 +.707r6     -.707r8  -.707i2 +i3 -.707i4 +.707i6 -i7 +.707i8
;; r1         -r3         +r5         -r7              -i2         +i4     -i6         +i8
;; r1 +.707r2     -.707r4 -r5 -.707r6     +.707r8  -.707i2 -i3 -.707i4 +.707i6 +i7 +.707i8
;; imaginarys:
;;                                         +i1     +i2 +i3     +i4 +i5     +i6 +i7     +i8
;; -.707r2 -r3 -.707r4 +.707r6 +r7 +.707r8 +i1 +.707i2     -.707i4 -i5 -.707i6     +.707i8
;;     -r2         +r4     -r6         +r8 +i1         -i3         +i5         -i7
;; -.707r2 +r3 -.707r4 +.707r6 -r7 +.707r8 +i1 -.707i2     +.707i4 -i5 +.707i6     -.707i8
;;                                         +i1     -i2 +i3     -i4 +i5     -i6 +i7     -i8
;; +.707r2 -r3 +.707r4 -.707r6 +r7 -.707r8 +i1 -.707i2     +.707i4 -i5 +.707i6     -.707i8
;;     +r2         -r4     +r6         -r8 +i1         -i3         +i5         -i7
;; +.707r2 +r3 +.707r4 -.707r6 -r7 -.707r8 +i1 +.707i2     -.707i4 -i5 -.707i6     +.707i8
;;

;; Rearranging for maximum FMA usage:
;;R1 = ((r1+r5)+(r3+r7))      +((r2+r8)+(r4+r6))
;;R5 = ((r1+r5)+(r3+r7))      -((r2+r8)+(r4+r6))
;;R3 = ((r1+r5)-(r3+r7))                        +((i2-i8)-(i4-i6))
;;R7 = ((r1+r5)-(r3+r7))                        -((i2-i8)-(i4-i6))
;;R2 = ((r1-r5)+(i3-i7)) +.707(((r2+r8)-(r4+r6))+((i2-i8)+(i4-i6)))
;;R6 = ((r1-r5)+(i3-i7)) -.707(((r2+r8)-(r4+r6))+((i2-i8)+(i4-i6)))
;;R4 = ((r1-r5)-(i3-i7)) -.707(((r2+r8)-(r4+r6))-((i2-i8)+(i4-i6)))
;;R8 = ((r1-r5)-(i3-i7)) +.707(((r2+r8)-(r4+r6))-((i2-i8)+(i4-i6)))

;;I1 = ((i1+i5)+(i3+i7))                        +((i2+i8)+(i4+i6))
;;I5 = ((i1+i5)+(i3+i7))                        -((i2+i8)+(i4+i6))
;;I3 = ((i1+i5)-(i3+i7))      -((r2-r8)-(r4-r6))                        
;;I7 = ((i1+i5)-(i3+i7))      +((r2-r8)-(r4-r6))                        
;;I2 = ((i1-i5)-(r3-r7)) -.707(((r2-r8)+(r4-r6))-((i2+i8)-(i4+i6)))
;;I6 = ((i1-i5)-(r3-r7)) +.707(((r2-r8)+(r4-r6))-((i2+i8)-(i4+i6)))
;;I4 = ((i1-i5)+(r3-r7)) -.707(((r2-r8)+(r4-r6))+((i2+i8)-(i4+i6)))
;;I8 = ((i1-i5)+(r3-r7)) +.707(((r2-r8)+(r4-r6))+((i2+i8)-(i4+i6)))

zr8_8c_djbunfft_cmn_preload MACRO
	vbroadcastsd zmm31, ZMM_SQRTHALF
	ENDM
zr8_8c_djbunfft_cmn MACRO srcreg,srcinc,d1,d2,d4,bcast,bcreg,bcsz,screg,scinc,maxrpt,L1pt,L1pd
bcast	vbroadcastsd zmm30, [bcreg+0*bcsz+bcsz/2];; cosine/sine for R2/I2 and R8/I8 (w^1)
no bcast vmovapd zmm30, [screg+0*128+64]	;; cosine/sine for R2/I2 and R8/I8
	vmovapd zmm4, [srcreg+d1]		;; Load R2
	vmovapd zmm12, [srcreg+d1+64]		;; Load I2
	zfmaddpd zmm16, zmm4, zmm30, zmm12	;; A2 = R2 * cosine/sine + I2			; 1-4		n 5
	zfmsubpd zmm12, zmm12, zmm30, zmm4	;; B2 = I2 * cosine/sine - R2			; 1-4		n 5

	vmovapd zmm7, [srcreg+d4+d2+d1]		;; Load R8
	vmovapd zmm15, [srcreg+d4+d2+d1+64]	;; Load I8
	zfmsubpd zmm4, zmm7, zmm30, zmm15	;; A8 = R8 * cosine/sine - I8			; 2-5		n 11
	zfmaddpd zmm15, zmm15, zmm30, zmm7	;; B8 = I8 * cosine/sine + R8			; 2-5		n 12

bcast	vbroadcastsd zmm30, [bcreg+2*bcsz+bcsz/2];; cosine/sine for R4/I4 and R6/I6 (w^3)
no bcast vmovapd zmm30, [screg+2*128+64]	;; cosine/sine for R4/I4 and R6/I6
	vmovapd zmm6, [srcreg+d2+d1]		;; Load R4
	vmovapd zmm14, [srcreg+d2+d1+64]	;; Load I4
	zfmaddpd zmm7, zmm6, zmm30, zmm14	;; A4 = R4 * cosine/sine + I4 (first R4/sine)	; 3-6		n 9
	zfmsubpd zmm14, zmm14, zmm30, zmm6	;; B4 = I4 * cosine/sine - R4 (first I4/sine)	; 3-6		n 10

	vmovapd zmm5, [srcreg+d4+d1]		;; Load R6
	vmovapd zmm13, [srcreg+d4+d1+64]	;; Load I6
	zfmsubpd zmm6, zmm5, zmm30, zmm13	;; A6 = R6 * cosine/sine - I6 (first R6/sine)	; 4-7		n 9
	zfmaddpd zmm13, zmm13, zmm30, zmm5	;; B6 = I6 * cosine/sine + R6 (first I6/sine)	; 4-7		n 10

bcast	vbroadcastsd zmm29, [bcreg+0*bcsz]	;; sine for R2/I2 and R8/I8 (w^1)
no bcast vmovapd zmm29, [screg+0*128]		;; sine for R2/I2 and R8/I8
	vmulpd	zmm16, zmm16, zmm29		;; A2 = A2 * sine (first R2)			; 5-8		n 11
	vmulpd	zmm12, zmm12, zmm29		;; B2 = B2 * sine (first I2)			; 5-8		n 12

bcast	vbroadcastsd zmm30, [bcreg+3*bcsz+bcsz/2];; cosine/sine for R5/I5 (w^4)
no bcast vmovapd zmm30, [screg+3*128+64]	;; cosine/sine for R5/I5
	vmovapd zmm1, [srcreg+d4]		;; Load R5
	vmovapd zmm9, [srcreg+d4+64]		;; Load I5
	zfmaddpd zmm5, zmm1, zmm30, zmm9	;; A5 = R5 * cosine/sine + I5 (first R5/sine)	; 6-9		n 13
	zfmsubpd zmm9, zmm9, zmm30, zmm1	;; B5 = I5 * cosine/sine - R5 (first I5/sine)	; 6-9		n 14

bcast	vbroadcastsd zmm30, [bcreg+1*bcsz+bcsz/2];; cosine/sine for R3/I3 and R7/I7 (w^2)
no bcast vmovapd zmm30, [screg+1*128+64]	;; cosine/sine for R3/I3 and R7/I7
	vmovapd zmm2, [srcreg+d2]		;; Load R3
	vmovapd zmm10, [srcreg+d2+64]		;; Load I3
	zfmaddpd zmm1, zmm2, zmm30, zmm10	;; A3 = R3 * cosine/sine + I3 (first R3/sine)	; 7-10		n 15
	zfmsubpd zmm10, zmm10, zmm30, zmm2	;; B3 = I3 * cosine/sine - R3 (first I3/sine)	; 7-10		n 16

	vmovapd zmm3, [srcreg+d4+d2]		;; Load R7
	vmovapd zmm11, [srcreg+d4+d2+64]	;; Load I7
	zfmsubpd zmm2, zmm3, zmm30, zmm11	;; A7 = R7 * cosine/sine - I7 (first R7/sine)	; 8-11		n 15
	zfmaddpd zmm11, zmm11, zmm30, zmm3	;; B7 = I7 * cosine/sine + R7 (first I7/sine)	; 8-11		n 16

	vmovapd zmm0, [srcreg]			;; Load R1
	vaddpd	zmm3, zmm7, zmm6		;; R4/sine + R6/sine				; 9-12		n 17
	vsubpd	zmm7, zmm7, zmm6		;; R4/sine - R6/sine				; 9-12		n 18

	vmovapd zmm8, [srcreg+64]		;; Load I1
	vaddpd	zmm6, zmm14, zmm13		;; I4/sine + I6/sine				; 10-13		n 20
	vsubpd	zmm14, zmm14, zmm13		;; I4/sine - I6/sine				; 10-13		n 19

bcast	vbroadcastsd zmm28, [bcreg+3*bcsz]	;; sine for R5/I5 (w^4)
no bcast vmovapd zmm28, [screg+3*128]		;; sine for R5/I5
	zfmaddpd zmm13, zmm4, zmm29, zmm16	;; R2 + R8 * sine				; 11-14		n 17
	zfnmaddpd zmm4, zmm4, zmm29, zmm16	;; R2 - R8 * sine				; 11-14		n 18

bcast	vbroadcastsd zmm27, [bcreg+2*bcsz]	;; sine for R4/I4 and R6/I6 (w^3)
no bcast vmovapd zmm27, [screg+2*128]		;; sine for R4/I4 and R6/I6
	zfmaddpd zmm16, zmm15, zmm29, zmm12	;; I2 + I8 * sine				; 12-15		n 20
	zfnmaddpd zmm15, zmm15, zmm29, zmm12	;; I2 - I8 * sine				; 12-15		n 19

bcast	vbroadcastsd zmm26, [bcreg+1*bcsz]	;; sine for R3/I3 and R7/I7 (w^2)
no bcast vmovapd zmm26, [screg+1*128]		;; sine for R3/I3
	zfmaddpd zmm12, zmm5, zmm28, zmm0	;; R1 + R5 * sine				; 13-16		n 21
	zfnmaddpd zmm5, zmm5, zmm28, zmm0	;; R1 - R5 * sine				; 13-16		n 23
	bump	screg, scinc

	L1prefetchw srcreg+d1+L1pd, L1pt
	zfmaddpd zmm0, zmm9, zmm28, zmm8	;; I1 + I5 * sine				; 14-17		n 22
	zfnmaddpd zmm9, zmm9, zmm28, zmm8	;; I1 - I5 * sine				; 14-17		n 24

	L1prefetchw srcreg+d1+64+L1pd, L1pt
	vaddpd	zmm8, zmm1, zmm2		;; R3/sine + R7/sine				; 15-18		n 21
	vsubpd	zmm1, zmm1, zmm2		;; R3/sine - R7/sine				; 15-18		n 24

	L1prefetchw srcreg+d4+d2+d1+L1pd, L1pt
	vaddpd	zmm2, zmm10, zmm11		;; I3/sine + I7/sine				; 16-19		n 22
	vsubpd	zmm10, zmm10, zmm11		;; I3/sine - I7/sine				; 16-19		n 23

	L1prefetchw srcreg+d4+d2+d1+64+L1pd, L1pt
	zfmaddpd zmm11, zmm3, zmm27, zmm13	;; r2++ = (r2+r8) + (r4+r6) * sine		; 17-20		n 27
	zfnmaddpd zmm3, zmm3, zmm27, zmm13	;; r2+- = (r2+r8) - (r4+r6) * sine		; 17-20		n 25

	L1prefetchw srcreg+d2+d1+L1pd, L1pt
	zfmaddpd zmm13, zmm7, zmm27, zmm4	;; r2-+ = (r2-r8) + (r4-r6) * sine		; 18-21		n 26
	zfnmaddpd zmm7, zmm7, zmm27, zmm4	;; r2-- = (r2-r8) - (r4-r6) * sine		; 18-21		n 30

	L1prefetchw srcreg+d2+d1+64+L1pd, L1pt
	zfmaddpd zmm4, zmm14, zmm27, zmm15	;; i2-+ = (i2-i8) + (i4-i6) * sine		; 19-22		n 25
	zfnmaddpd zmm14, zmm14, zmm27, zmm15	;; i2-- = (i2-i8) - (i4-i6) * sine		; 19-22		n 29

	L1prefetchw srcreg+d4+d1+L1pd, L1pt
	zfmaddpd zmm15, zmm6, zmm27, zmm16	;; i2++ = (i2+i8) + (i4+i6) * sine		; 20-23		n 28
	zfnmaddpd zmm6, zmm6, zmm27, zmm16	;; i2+- = (i2+i8) - (i4+i6) * sine		; 20-23		n 26

	L1prefetchw srcreg+d4+d1+64+L1pd, L1pt
	zfmaddpd zmm16, zmm8, zmm26, zmm12	;; r1++ = (r1+r5) + (r3+r7) * sine		; 21-24		n 27
	zfnmaddpd zmm8, zmm8, zmm26, zmm12	;; r1+- = (r1+r5) - (r3+r7) * sine		; 21-24		n 29

	L1prefetchw srcreg+d4+L1pd, L1pt
	zfmaddpd zmm12, zmm2, zmm26, zmm0	;; i1++ = (i1+i5) + (i3+i7) * sine		; 22-25		n 28
	zfnmaddpd zmm2, zmm2, zmm26, zmm0	;; i1+- = (i1+i5) - (i3+i7) * sine		; 22-25		n 30

	L1prefetchw srcreg+d4+64+L1pd, L1pt
	zfmaddpd zmm0, zmm10, zmm26, zmm5	;; r1-+ = (r1-r5) + (i3-i7) * sine		; 23-26		n 31
	zfnmaddpd zmm10, zmm10, zmm26, zmm5	;; r1-- = (r1-r5) - (i3-i7) * sine		; 23-26		n 33

	L1prefetchw srcreg+d2+L1pd, L1pt
	zfmaddpd zmm5, zmm1, zmm26, zmm9	;; i1-+ = (i1-i5) + (r3-r7) * sine		; 24-27		n 34
	zfnmaddpd zmm1, zmm1, zmm26, zmm9	;; i1-- = (i1-i5) - (r3-r7) * sine		; 24-27		n 32

	L1prefetchw srcreg+d2+64+L1pd, L1pt
	vaddpd	zmm9, zmm3, zmm4		;; r2+-+ = (r2+-) + (i2-+)			; 25-28		n 31
	vsubpd	zmm3, zmm3, zmm4		;; r2+-- = (r2+-) - (i2-+)			; 25-28		n 33

	L1prefetchw srcreg+d4+d2+L1pd, L1pt
	vaddpd	zmm4, zmm13, zmm6		;; r2-++ = (r2-+) + (i2+-)			; 26-29		n 34
	vsubpd	zmm13, zmm13, zmm6		;; r2-+- = (r2-+) - (i2+-)			; 26-29		n 32

	L1prefetchw srcreg+d4+d2+64+L1pd, L1pt
	vaddpd	zmm6, zmm16, zmm11		;; R1 = (r1++) + (r2++)				; 27-30
	vsubpd	zmm16, zmm16, zmm11		;; R5 = (r1++) - (r2++)				; 27-30

	L1prefetchw srcreg+L1pd, L1pt
	vaddpd	zmm11, zmm12, zmm15		;; I1 = (i1++) + (i2++)				; 28-31
	vsubpd	zmm12, zmm12, zmm15		;; I5 = (i1++) - (i2++)				; 28-31

	L1prefetchw srcreg+64+L1pd, L1pt
	vaddpd	zmm15, zmm8, zmm14		;; R3 = (r1+-) + (i2--)				; 29-32
	vsubpd	zmm8, zmm8, zmm14		;; R7 = (r1+-) - (i2--)				; 29-32

	vsubpd	zmm14, zmm2, zmm7		;; I3 = (i1+-) - (r2--)				; 30-33
	vaddpd	zmm2, zmm2, zmm7		;; I7 = (i1+-) + (r2--)				; 30-33

	zfmaddpd zmm7, zmm9, zmm31, zmm0	;; R2 = (r1-+) + .707*(r2+-+)			; 31-34
	zfnmaddpd zmm9, zmm9, zmm31, zmm0	;; R6 = (r1-+) - .707*(r2+-+)			; 31-34
	zstore	[srcreg], zmm6			;; Save R1					; 31

	zfnmaddpd zmm0, zmm13, zmm31, zmm1	;; I2 = (i1--) - .707*(r2-+-)			; 32-35
	zfmaddpd zmm13, zmm13, zmm31, zmm1	;; I6 = (i1--) + .707*(r2-+-)			; 32-35
	zstore	[srcreg+d4], zmm16		;; Save R5					; 31+1

	zfnmaddpd zmm1, zmm3, zmm31, zmm10	;; R4 = (r1--) - .707*(r2+--)			; 33-36
	zfmaddpd zmm3, zmm3, zmm31, zmm10	;; R8 = (r1--) + .707*(r2+--)			; 33-36
	zstore	[srcreg+64], zmm11		;; Save I1					; 32+1

	zfnmaddpd zmm10, zmm4, zmm31, zmm5	;; I4 = (i1-+) - .707*(r2-++)			; 34-37
	zfmaddpd zmm4, zmm4, zmm31, zmm5	;; I8 = (i1-+) + .707*(r2-++)			; 34-37
	zstore	[srcreg+d4+64], zmm12		;; Save I5					; 32+2

	zstore	[srcreg+d2], zmm15		;; Save R3					; 33+2
	zstore	[srcreg+d4+d2], zmm8		;; Save R7					; 33+3
	zstore	[srcreg+d2+64], zmm14		;; Save I3					; 34+3
	zstore	[srcreg+d4+d2+64], zmm2		;; Save I7					; 34+4
	zstore	[srcreg+d1], zmm7		;; Save R2					; 35+4
	zstore	[srcreg+d4+d1], zmm9		;; Save R6					; 35+5
	zstore	[srcreg+d1+64], zmm0		;; Save I2					; 36+5
	zstore	[srcreg+d4+d1+64], zmm13	;; Save I6					; 36+6
	zstore	[srcreg+d2+d1], zmm1		;; Save R4					; 37+6
	zstore	[srcreg+d4+d2+d1], zmm3		;; Save R8					; 37+7
	zstore	[srcreg+d2+d1+64], zmm10	;; Save I4					; 38+7
	zstore	[srcreg+d4+d2+d1+64], zmm4	;; Save I8					; 38+8
	bump	srcreg, srcinc
	ENDM


;;
;; ******************************* eight-complex first and last variants (two pass FFTs) ***********************************
;;

;; This code applies the roots-of-minus-1 premultipliers in a negacyclic FFT as well as
;; the group weight multipliers (the column weights are applied later).
;; It also applies the seven sin/cos multipliers after the first radix-8 butterfly.
;; We save memory by splitting the roots-of-minus-1 premultipliers such that every
;; macro uses the same premultiplier data.  The rest of the premultiplier is applied
;; at the end of pass 1.

;; The sin/cos and premultiplier data is combined in one table.  The premultiplier data
;; is at screg the sin/cos data is at screg+8*64.  The premultiplier sine value has been
;; merged into the group multipliers.

;; Macro assumes these registers:
;; rbx = register to load compressed fudge index (top 56 bits are zero)
;; r12 = pointer to compressed fudges array
;; r13 = pointer to XOR masks
;; r14 = scratch register

zr8_csc_wpn_eight_complex_first_djbfft_preload MACRO
	vbroadcastsd zmm31, ZMM_SQRTHALF
	vbroadcastsd zmm28, ZMM_ONE_OVER_B
	ENDM
zr8_csc_wpn_eight_complex_first_djbfft MACRO srcreg,srcinc,d1,d2,d4,screg,scinc,maskreg,maskinc,grpreg,grpinc,maxrpt,L1pt,L1pd
	mov	bl, [maskreg+0*1]		;; Load index into compressed fudges
	mov	r14, [r13+0*8]			;; Load the xor mask
	xor	r14, [r12+rbx*8]		;; Apply the xor mask to 64 bits of fudge flags

	kmovw	k1, r14d			;; Load R1 and I1 fudge factor mask		; 1
	kshiftrw k2, k1, 8			;; I1's fudge					; 2
	vmovapd	zmm0, [srcreg]			;; R1
	vmulpd	zmm0{k1}, zmm0, zmm28		;; apply fudges to R1				; 2-5		n 27
	vmovapd	zmm8, [srcreg+64]		;; I1
	vmulpd	zmm8{k2}, zmm8, zmm28		;; apply fudges to I1				; 3-6		n 18

	shr	r14, 16				;; Next 16 bits of fudge flags
	kmovw	k1, r14d			;; Load R2 and I2 fudge factor mask		; 3
	kshiftrw k2, k1, 8			;; I2's fudge					; 4
	vmovapd	zmm1, [srcreg+d1]		;; R2
	vmulpd	zmm1{k1}, zmm1, zmm28		;; apply fudges to R2				; 4-7		n 23
	vmovapd	zmm9, [srcreg+d1+64]		;; I2
	vmulpd	zmm9{k2}, zmm9, zmm28		;; apply fudges to I2				; 5-8		n 17

	shr	r14, 16				;; Next 16 bits of fudge flags
	kmovw	k1, r14d			;; Load R3 and I3 fudge factor mask		; 5
	kshiftrw k2, k1, 8			;; I3's fudge					; 6
	vmovapd	zmm2, [srcreg+d2]		;; R3
	vmulpd	zmm2{k1}, zmm2, zmm28		;; apply fudges to R3				; 6-9		n 28
	vmovapd	zmm10, [srcreg+d2+64]		;; I3
	vmulpd	zmm10{k2}, zmm10, zmm28		;; apply fudges to I3				; 7-10		n 19

	shr	r14, 16				;; Next 16 bits of fudge flags
	kmovw	k1, r14d			;; Load R4 and I4 fudge factor mask		; 7
	kshiftrw k2, k1, 8			;; I4's fudge					; 8
	vmovapd	zmm3, [srcreg+d2+d1]		;; R4
	vmulpd	zmm3{k1}, zmm3, zmm28		;; apply fudges to R4				; 8-11		n 24
	vmovapd	zmm11, [srcreg+d2+d1+64]	;; I4
	vmulpd	zmm11{k2}, zmm11, zmm28		;; apply fudges to I4				; 9-12		n 18

	mov	bl, [maskreg+1*1]		;; Load index into compressed fudges
	mov	r14, [r13+1*8]			;; Load the xor mask
	xor	r14, [r12+rbx*8]		;; Apply the xor mask to 64 bits of fudge flags

	kmovw	k1, r14d			;; Load R5 and I5 fudge factor mask		; 9
	kshiftrw k2, k1, 8			;; I5's fudge					; 10
	vmovapd	zmm4, [srcreg+d4]		;; R5
	vmulpd	zmm4{k1}, zmm4, zmm28		;; apply fudges to R5				; 10-13		n 20
	vmovapd	zmm12, [srcreg+d4+64]		;; I5
	vmulpd	zmm12{k2}, zmm12, zmm28		;; apply fudges to I5				; 11-14		n 21

	shr	r14, 16				;; Next 16 bits of fudge flags
	kmovw	k1, r14d			;; Load R6 and I6 fudge factor mask		; 11
	kshiftrw k2, k1, 8			;; I6's fudge					; 12
	vmovapd	zmm5, [srcreg+d4+d1]		;; R6
	vmulpd	zmm5{k1}, zmm5, zmm28		;; apply fudges to R6				; 12-15		n 19
	vmovapd	zmm13, [srcreg+d4+d1+64]	;; I6
	vmulpd	zmm13{k2}, zmm13, zmm28		;; apply fudges to I6				; 13-16		n 20

	shr	r14, 16				;; Next 16 bits of fudge flags
	kmovw	k1, r14d			;; Load R7 and I7 fudge factor mask		; 13
	kshiftrw k2, k1, 8			;; I7's fudge					; 14
	vmovapd	zmm6, [srcreg+d4+d2]		;; R7
	vmulpd	zmm6{k1}, zmm6, zmm28		;; apply fudges to R7				; 14-17		n 22
	vmovapd	zmm14, [srcreg+d4+d2+64]	;; I7
	vmulpd	zmm14{k2}, zmm14, zmm28		;; apply fudges to I7				; 15-18		n 23

	shr	r14, 16				;; Next 16 bits of fudge flags
	kmovw	k1, r14d			;; Load R8 and I8 fudge factor mask		; 15
	kshiftrw k2, k1, 8			;; I8's fudge					; 16
	vmovapd	zmm7, [srcreg+d4+d2+d1]		;; R8
	vmulpd	zmm7{k1}, zmm7, zmm28		;; apply fudges to R8				; 16-19		n 21
	vmovapd	zmm15, [srcreg+d4+d2+d1+64]	;; I8
	vmulpd	zmm15{k2}, zmm15, zmm28		;; apply fudges to I8				; 17-20		n 22

	bump	maskreg, maskinc

	vmulpd	zmm9, zmm9, [grpreg+3*64]	;; apply group multiplier I2 over R2 to I2	; 17-20		n 23
	vmulpd	zmm8, zmm8, [grpreg+1*64]	;; apply group multiplier I1 over R1 to I1	; 18-21		n 27
	vmulpd	zmm11, zmm11, [grpreg+7*64]	;; apply group multiplier I4 over R4 to I4	; 18-21		n 24
	vmulpd	zmm10, zmm10, [grpreg+5*64]	;; apply group multiplier I3 over R3 to I3	; 19-22		n 28
	vmulpd	zmm5, zmm5, [grpreg+10*64]	;; apply group multiplier to R6			; 19-22		n 25
	vmulpd	zmm13, zmm13, [grpreg+11*64]	;; apply group multiplier to I6			; 20-23		n 25
	vmulpd	zmm4, zmm4, [grpreg+8*64]	;; apply group multiplier to R5			; 20-23		n 29
	vmulpd	zmm12, zmm12, [grpreg+9*64]	;; apply group multiplier to I5			; 21-24		n 29
	vmulpd	zmm7, zmm7, [grpreg+14*64]	;; apply group multiplier to R8			; 21-24		n 26
	vmulpd	zmm15, zmm15, [grpreg+15*64]	;; apply group multiplier to I8			; 22-25		n 26
	vmulpd	zmm6, zmm6, [grpreg+12*64]	;; apply group multiplier to R7			; 22-25		n 30
	vmulpd	zmm14, zmm14, [grpreg+13*64]	;; apply group multiplier to I7			; 23-26		n 30

;; Apply the complex premultipliers

	vmovapd zmm30, [screg+1*64]		;; premultiplier cosine/sine for R2/I2
	zfmsubpd zmm16, zmm1, zmm30, zmm9	;; A2 = R2 * cosine - I2			; 23-26		n 31
	zfmaddpd zmm9, zmm9, zmm30, zmm1	;; B2 = I2 * cosine + R2			; 24-27		n 34

	vmovapd zmm30, [screg+3*64]		;; premultiplier cosine/sine for R4/I4
	zfmsubpd zmm1, zmm3, zmm30, zmm11	;; A4 = R4 * cosine - I4			; 24-27		n 32
	zfmaddpd zmm11, zmm11, zmm30, zmm3	;; B4 = I4 * cosine + R4			; 25-28		n 33

	vmovapd zmm30, [screg+5*64]		;; premultiplier cosine/sine for R6/I6
	zfmsubpd zmm3, zmm5, zmm30, zmm13	;; A6 = R6 * cosine - I6			; 25-28		n 31
	zfmaddpd zmm13, zmm13, zmm30, zmm5	;; B6 = I6 * cosine + R6			; 26-29		n 34

	vmovapd zmm30, [screg+7*64]		;; premultiplier cosine/sine for R8/I8
	zfmsubpd zmm5, zmm7, zmm30, zmm15	;; A8 = R8 * cosine - I8			; 26-29		n 32
	zfmaddpd zmm15, zmm15, zmm30, zmm7	;; B8 = I8 * cosine + R8			; 27-30		n 33

	vmovapd zmm30, [screg+0*64]		;; premultiplier cosine/sine for R1/I1
	zfmsubpd zmm7, zmm0, zmm30, zmm8	;; A1 = R1 * cosine - I1			; 27-30		n 35
	zfmaddpd zmm8, zmm8, zmm30, zmm0	;; B1 = I1 * cosine + R1			; 28-31		n 37

	vmovapd zmm30, [screg+2*64]		;; premultiplier cosine/sine for R3/I3
	zfmsubpd zmm0, zmm2, zmm30, zmm10	;; A3 = R3 * cosine - I3			; 28-31		n 36
	zfmaddpd zmm10, zmm10, zmm30, zmm2	;; B3 = I3 * cosine + R3			; 29-32		n 38

	vmovapd zmm30, [screg+4*64]		;; premultiplier cosine/sine for R5/I5
	zfmsubpd zmm2, zmm4, zmm30, zmm12	;; A5 = R5 * cosine - I5			; 29-32		n 35
	zfmaddpd zmm12, zmm12, zmm30, zmm4	;; B5 = I5 * cosine + R5			; 30-33		n 37

	vmovapd zmm30, [screg+6*64]		;; premultiplier cosine/sine for R7/I7
	zfmsubpd zmm4, zmm6, zmm30, zmm14	;; A7 = R7 * cosine - I7			; 30-33		n 36
	zfmaddpd zmm14, zmm14, zmm30, zmm6	;; B7 = I7 * cosine + R7			; 31-34		n 38

;; Copied from common 8-complex DJB FFT macro

	vmovapd zmm29, [grpreg+2*64]		;; group multiplier for R2
	zfmsubpd zmm6, zmm16, zmm29, zmm3	;; R2*grpmultR2-R6				; 31-34		n 39
	zfmaddpd zmm16, zmm16, zmm29, zmm3	;; R2*grpmultR2+R6				; 32-35		n 41

	vmovapd zmm30, [grpreg+6*64]		;; group multiplier for R4
	zfmsubpd zmm3, zmm1, zmm30, zmm5	;; R4*grpmultR4-R8				; 32-35		n 39
	zfmaddpd zmm1, zmm1, zmm30, zmm5	;; R4*grpmultR4+R8				; 33-36		n 41
	vmovapd zmm22, [grpreg+0*64]		;; group multiplier for R1

	zfmsubpd zmm5, zmm11, zmm30, zmm15	;; I4*grpmultR4-I8				; 33-36		n 40
	zfmaddpd zmm11, zmm11, zmm30, zmm15	;; I4*grpmultR4+I8				; 34-37		n 42
	vmovapd zmm23, [grpreg+4*64]		;; group multiplier for R3

	zfmsubpd zmm15, zmm9, zmm29, zmm13	;; I2*grpmultR2-I6				; 34-37		n 40
	zfmaddpd zmm9, zmm9, zmm29, zmm13	;; I2*grpmultR2+I6				; 35-38		n 42
	vmovapd zmm30, [screg+8*64+1*128]	;; sine for R3/I3 and R7/I7 (w^2)

	zfmaddpd zmm13, zmm7, zmm22, zmm2	;; R1*grpmultR1+R5				; 35-38		n 43
	zfmsubpd zmm7, zmm7, zmm22, zmm2	;; R1*grpmultR1-R5				; 36-39		n 44
	vmovapd zmm29, [screg+8*64+0*128]	;; sine for R2/I2 and R8/I8 (w^1)

	zfmaddpd zmm2, zmm0, zmm23, zmm4	;; R3*grpmultR3+R7				; 36-39		n 43
	zfmsubpd zmm0, zmm0, zmm23, zmm4	;; R3*grpmultR3-R7				; 37-40		n 47
	vmovapd zmm27, [screg+8*64+2*128]	;; sine for R4/I4 and R6/I6 (w^3)

	zfmaddpd zmm4, zmm8, zmm22, zmm12	;; I1*grpmultR1+I5				; 37-40		n 46
	zfmsubpd zmm8, zmm8, zmm22, zmm12	;; I1*grpmultR1-I5				; 38-41		n 45
	vmovapd zmm26, [screg+8*64+1*128+64]	;; cosine/sine for R3/I3 and R7/I7 (w^2)

	zfmaddpd zmm12, zmm10, zmm23, zmm14	;; I3*grpmultR3+I7				; 38-41		n 46
	zfmsubpd zmm10, zmm10, zmm23, zmm14	;; I3*grpmultR3-I7				; 39-42		n 48
	vmovapd zmm25, [screg+8*64+3*128+64]	;; cosine/sine for R5/I5 (w^4)

	bump	grpreg, grpinc

	vsubpd	zmm14, zmm6, zmm3		;; r2-- = (r2-r6) - (r4-r8)			; 39-42		n 44
	vaddpd	zmm6, zmm6, zmm3		;; r2-+ = (r2-r6) + (r4-r8)			; 40-43		n 47
	vmovapd zmm24, [screg+8*64+0*128+64]	;; cosine/sine for R2/I2 and R8/I8 (w^1)

	vsubpd	zmm3, zmm15, zmm5		;; i2-- = (i2-i6) - (i4-i8)			; 40-43		n 45
	vaddpd	zmm15, zmm15, zmm5		;; i2-+ = (i2-i6) + (i4-i8)			; 41-44		n 48
	vmovapd zmm23, [screg+8*64+2*128+64]	;; cosine/sine for R4/I4 and R6/I6 (w^3)

	vsubpd	zmm5, zmm16, zmm1		;; r2+- = (r2+r6) - (r4+r8)			; 41-44		n 49
	vaddpd	zmm16, zmm16, zmm1		;; r2++ = (r2+r6) + (r4+r8)			; 42-45		n 52
	vmovapd zmm22, [screg+8*64+3*128]	;; sine for R5/I5 (w^4)
	bump	screg, scinc

	vsubpd	zmm1, zmm9, zmm11		;; i2+- = (i2+i6) - (i4+i8)			; 42-45		n 50
	vaddpd	zmm9, zmm9, zmm11		;; i2++ = (i2+i6) + (i4+i8)			; 43-46		n 55
	L1prefetchw srcreg+L1pd, L1pt

	vaddpd	zmm11, zmm13, zmm2		;; r1++ = (r1+r5) + (r3+r7)			; 43-46		n 52
	vsubpd	zmm13, zmm13, zmm2		;; r1+- = (r1+r5) - (r3+r7)			; 44-47		n 53
	L1prefetchw srcreg+d1+L1pd, L1pt

	zfmaddpd zmm2, zmm14, zmm31, zmm7	;; r1-+ = (r1-r5) + .707*(r2--)			; 44-47		n 50
	zfnmaddpd zmm14, zmm14, zmm31, zmm7	;; r1-- = (r1-r5) - .707*(r2--)			; 45-48		n 51
	L1prefetchw srcreg+d2+L1pd, L1pt

	zfmaddpd zmm7, zmm3, zmm31, zmm8	;; i1-+ = (i1-i5) + .707*(i2--)			; 45-48		n 51
	zfnmaddpd zmm3, zmm3, zmm31, zmm8	;; i1-- = (i1-i5) - .707*(i2--)			; 46-49		n 52
	L1prefetchw srcreg+d2+d1+L1pd, L1pt

	vaddpd	zmm8, zmm4, zmm12		;; i1++ = (i1+i5) + (i3+i7)			; 46-49		n 55
	vsubpd	zmm4, zmm4, zmm12		;; i1+- = (i1+i5) - (i3+i7)			; 47-50		n 54
	L1prefetchw srcreg+d4+L1pd, L1pt

	zfmaddpd zmm12, zmm6, zmm31, zmm0	;; r3-+ = (r3-r7) + .707*(r2-+)			; 47-50		n 57
	zfnmaddpd zmm6, zmm6, zmm31, zmm0	;; r3-- = (r3-r7) - .707*(r2-+)			; 48-51		n 61
	L1prefetchw srcreg+d4+d1+L1pd, L1pt

	zfmaddpd zmm0, zmm15, zmm31, zmm10	;; i3-+ = (i3-i7) + .707*(i2-+)			; 48-51		n 56
	zfnmaddpd zmm15, zmm15, zmm31, zmm10	;; i3-- = (i3-i7) - .707*(i2-+)			; 49-52		n 60
	L1prefetchw srcreg+d4+d2+L1pd, L1pt

	vmulpd	zmm1, zmm1, zmm30		;; i2+-s = i2+- * sine37			; 49-52		n 53
	vmulpd	zmm5, zmm5, zmm30		;; r2+-s = r2+- * sine37			; 50-53		n 54
	L1prefetchw srcreg+d4+d2+d1+L1pd, L1pt

	vmulpd	zmm2, zmm2, zmm29		;; r1-+s = r1-+ * sine28			; 50-53		n 56
	vmulpd	zmm7, zmm7, zmm29		;; i1-+s = i1-+ * sine28			; 51-54		n 57
	L1prefetchw srcreg+64+L1pd, L1pt

	vmulpd	zmm14, zmm14, zmm27		;; r1--s = r1-- * sine46			; 51-54		n 60
	vmulpd	zmm3, zmm3, zmm27		;; i1--s = i1-- * sine46			; 52-55		n 61
	L1prefetchw srcreg+d1+64+L1pd, L1pt

	vsubpd	zmm10, zmm11, zmm16		;; R5 = (r1++) - (r2++)				; 52-55		n 62
	vaddpd	zmm11, zmm11, zmm16		;; R1 = (r1++) + (r2++)				; 53-56
	L1prefetchw srcreg+d2+64+L1pd, L1pt

	zfmsubpd zmm16, zmm13, zmm30, zmm1	;; R3s = (r1+-)*sine37 - (i2+-s)		; 53-56		n 58
	zfmaddpd zmm17, zmm4, zmm30, zmm5	;; I3s = (i1+-)*sine37 + (r2+-s)		; 54-57		n 58
	L1prefetchw srcreg+d2+d1+64+L1pd, L1pt

	zfmaddpd zmm13, zmm13, zmm30, zmm1	;; R7s = (r1+-)*sine37 + (i2+-s)		; 54-57		n 59
	zfmsubpd zmm4, zmm4, zmm30, zmm5	;; I7s = (i1+-)*sine37 - (r2+-s)		; 55-58		n 59
	L1prefetchw srcreg+d4+64+L1pd, L1pt

	vsubpd	zmm5, zmm8, zmm9		;; I5 = (i1++) - (i2++)				; 55-58		n 62
	vaddpd	zmm8, zmm8, zmm9		;; I1 = (i1++) + (i2++)				; 56-59
	L1prefetchw srcreg+d4+d1+64+L1pd, L1pt

	zfnmaddpd zmm1, zmm0, zmm29, zmm2	;; R2s = (r1-+s) - sine28*(i3-+)		; 56-59		n 63
	zfmaddpd zmm9, zmm12, zmm29, zmm7	;; I2s = (i1-+s) + sine28*(r3-+)		; 57-60		n 63
	L1prefetchw srcreg+d4+d2+64+L1pd, L1pt
	zstore	[srcreg], zmm11			;; Store R1					; 56

	zfmaddpd zmm0, zmm0, zmm29, zmm2	;; R8s = (r1-+s) + sine28*(i3-+)		; 57-60		n 64
	zfnmaddpd zmm12, zmm12, zmm29, zmm7	;; I8s = (i1-+s) - sine28*(r3-+)		; 58-61		n 64
	L1prefetchw srcreg+d4+d2+d1+64+L1pd, L1pt
	zstore	[srcreg+64], zmm8		;; Store I1					; 27

	zfmsubpd zmm2, zmm16, zmm26, zmm17	;; R3s * cosine/sine - I3s (final R3)		; 58-61
	zfmaddpd zmm17, zmm17, zmm26, zmm16	;; I3s * cosine/sine + R3s (final I3)		; 59-62

	zfmaddpd zmm7, zmm13, zmm26, zmm4	;; R7s * cosine/sine + I7s (final R7)		; 59-62
	zfmsubpd zmm4, zmm4, zmm26, zmm13	;; I7s * cosine/sine - R7s (final I7)		; 60-63

	zfmaddpd zmm16, zmm15, zmm27, zmm14	;; R4s = (r1--s) + sine46*(i3--)		; 60-63		n 65
	zfnmaddpd zmm13, zmm6, zmm27, zmm3	;; I4s = (i1--s) - sine46*(r3--)		; 61-64		n 65

	zfnmaddpd zmm15, zmm15, zmm27, zmm14	;; R6s = (r1--s) - sine46*(i3--)		; 61-64		n 66
	zfmaddpd zmm6, zmm6, zmm27, zmm3	;; I6s = (i1--s) + sine46*(r3--)		; 62-65		n 66

	zfmsubpd zmm14, zmm10, zmm25, zmm5	;; A5 = R5 * cosine/sine - I5			; 62-65		n 67
	zfmaddpd zmm5, zmm5, zmm25, zmm10	;; B5 = I5 * cosine/sine + R5			; 63-66		n 68
	zstore	[srcreg+d2], zmm2		;; Store R3					; 32

	zfmsubpd zmm3, zmm1, zmm24, zmm9	;; R2s * cosine/sine - I2s (final R2)		; 63-66
	zfmaddpd zmm9, zmm9, zmm24, zmm1	;; I2s * cosine/sine + R2s (final I2)		; 64-67
	zstore	[srcreg+d2+64], zmm17		;; Store I3					; 32+1

	zfmaddpd zmm10, zmm0, zmm24, zmm12	;; R8s * cosine/sine + I8s (final R8)		; 64-67
	zfmsubpd zmm12, zmm12, zmm24, zmm0	;; I8s * cosine/sine - R8s (final I8)		; 65-68
	zstore	[srcreg+d4+d2], zmm7		;; Store R7					; 33+1

	zfmsubpd zmm1, zmm16, zmm23, zmm13	;; R4s * cosine/sine - I4s (final R4)		; 65-68
	zfmaddpd zmm13, zmm13, zmm23, zmm16	;; I4s * cosine/sine + R4s (final I4)		; 66-69
	zstore	[srcreg+d4+d2+64], zmm4		;; Store I7					; 33+2

	zfmaddpd zmm0, zmm15, zmm23, zmm6	;; R6s * cosine/sine + I6s (final R6)		; 66-69
	zfmsubpd zmm6, zmm6, zmm23, zmm15	;; I6s * cosine/sine - R6s (final I6)		; 67-70

	vmulpd	zmm14, zmm14, zmm22		;; A5 = A5 * sine (final R5)			; 67-70
	vmulpd	zmm5, zmm5, zmm22		;; B5 = B5 * sine (final I5)			; 68-71

	zstore	[srcreg+d1], zmm3		;; Store R2					; 37
	zstore	[srcreg+d1+64], zmm9		;; Store I2					; 37+1
	zstore	[srcreg+d4+d2+d1], zmm10	;; Store R8					; 38+1
	zstore	[srcreg+d4+d2+d1+64], zmm12	;; Store I8					; 38+2
	zstore	[srcreg+d2+d1], zmm1		;; Store R4					; 39+2
	zstore	[srcreg+d2+d1+64], zmm13	;; Store I4					; 39+3
	zstore	[srcreg+d4+d1], zmm0		;; Store R6					; 40+3
	zstore	[srcreg+d4+d1+64], zmm6	;; Store I6					; 40+4
	zstore	[srcreg+d4], zmm14		;; Store R5					; 41+4
	zstore	[srcreg+d4+64], zmm5		;; Store I5					; 41+5
	bump	srcreg, srcinc
	ENDM

;; This code applies the sin/cos multipliers before a radix-8 butterfly.
;; Then it applies the postmultipliers since the negacyclic inverse FFT is complete
;; as well as the group weight multipliers.
;; We save memory by splitting the roots-of-minus-1 premultipliers such that every
;; macro uses the same premultiplier data.  The rest of the premultiplier is applied
;; at the end of pass 1.

;; The sin/cos and premultiplier data is combined in one table.  The premultiplier data
;; is at screg the sin/cos data is at screg+8*64.  The premultiplier sine value has been
;; merged into the group multipliers.

;; Macro assumes these registers:
;; rbx = register to load compressed fudge index (top 56 bits are zero)
;; r12 = pointer to compressed fudges array
;; r13 = pointer to XOR masks
;; r14 = scratch register

zr8_csc_wpn_eight_complex_last_djbunfft_preload MACRO
	vbroadcastsd zmm31, ZMM_SQRTHALF
	vbroadcastsd zmm28, ZMM_B
	ENDM
zr8_csc_wpn_eight_complex_last_djbunfft MACRO srcreg,srcinc,d1,d2,d4,screg,scinc,maskreg,maskinc,maxrpt,L1pt,L1pd
	vmovapd zmm30, [screg+8*64+0*128+64]	;; cosine/sine for R2/I2 and R8/I8
	vmovapd zmm4, [srcreg+d1]		;; Load R2
	vmovapd zmm12, [srcreg+d1+64]		;; Load I2
	zfmaddpd zmm16, zmm4, zmm30, zmm12	;; A2 = R2 * cosine/sine + I2			; 1-4		n 5
	zfmsubpd zmm12, zmm12, zmm30, zmm4	;; B2 = I2 * cosine/sine - R2			; 1-4		n 5

	vmovapd zmm7, [srcreg+d4+d2+d1]		;; Load R8
	vmovapd zmm15, [srcreg+d4+d2+d1+64]	;; Load I8
	zfmsubpd zmm4, zmm7, zmm30, zmm15	;; A8 = R8 * cosine/sine - I8			; 2-5		n 11
	zfmaddpd zmm15, zmm15, zmm30, zmm7	;; B8 = I8 * cosine/sine + R8			; 2-5		n 12

	vmovapd zmm30, [screg+8*64+2*128+64]	;; cosine/sine for R4/I4 and R6/I6
	vmovapd zmm6, [srcreg+d2+d1]		;; Load R4
	vmovapd zmm14, [srcreg+d2+d1+64]	;; Load I4
	zfmaddpd zmm7, zmm6, zmm30, zmm14	;; A4 = R4 * cosine/sine + I4 (first R4/sine)	; 3-6		n 9
	zfmsubpd zmm14, zmm14, zmm30, zmm6	;; B4 = I4 * cosine/sine - R4 (first I4/sine)	; 3-6		n 10

	vmovapd zmm5, [srcreg+d4+d1]		;; Load R6
	vmovapd zmm13, [srcreg+d4+d1+64]	;; Load I6
	zfmsubpd zmm6, zmm5, zmm30, zmm13	;; A6 = R6 * cosine/sine - I6 (first R6/sine)	; 4-7		n 9
	zfmaddpd zmm13, zmm13, zmm30, zmm5	;; B6 = I6 * cosine/sine + R6 (first I6/sine)	; 4-7		n 10

	vmovapd zmm29, [screg+8*64+0*128]	;; sine for R2/I2 and R8/I8
	vmulpd	zmm16, zmm16, zmm29		;; A2 = A2 * sine (first R2)			; 5-8		n 11
	vmulpd	zmm12, zmm12, zmm29		;; B2 = B2 * sine (first I2)			; 5-8		n 12

	vmovapd zmm30, [screg+8*64+3*128+64]	;; cosine/sine for R5/I5
	vmovapd zmm1, [srcreg+d4]		;; Load R5
	vmovapd zmm9, [srcreg+d4+64]		;; Load I5
	zfmaddpd zmm5, zmm1, zmm30, zmm9	;; A5 = R5 * cosine/sine + I5 (first R5/sine)	; 6-9		n 13
	zfmsubpd zmm9, zmm9, zmm30, zmm1	;; B5 = I5 * cosine/sine - R5 (first I5/sine)	; 6-9		n 14

	vmovapd zmm30, [screg+8*64+1*128+64]	;; cosine/sine for R3/I3 and R7/I7
	vmovapd zmm2, [srcreg+d2]		;; Load R3
	vmovapd zmm10, [srcreg+d2+64]		;; Load I3
	zfmaddpd zmm1, zmm2, zmm30, zmm10	;; A3 = R3 * cosine/sine + I3 (first R3/sine)	; 7-10		n 15
	zfmsubpd zmm10, zmm10, zmm30, zmm2	;; B3 = I3 * cosine/sine - R3 (first I3/sine)	; 7-10		n 16

	vmovapd zmm3, [srcreg+d4+d2]		;; Load R7
	vmovapd zmm11, [srcreg+d4+d2+64]	;; Load I7
	zfmsubpd zmm2, zmm3, zmm30, zmm11	;; A7 = R7 * cosine/sine - I7 (first R7/sine)	; 8-11		n 15
	zfmaddpd zmm11, zmm11, zmm30, zmm3	;; B7 = I7 * cosine/sine + R7 (first I7/sine)	; 8-11		n 16

	vmovapd zmm0, [srcreg]			;; Load R1
	vaddpd	zmm3, zmm7, zmm6		;; R4/sine + R6/sine				; 9-12		n 17
	vsubpd	zmm7, zmm7, zmm6		;; R4/sine - R6/sine				; 9-12		n 18

	vmovapd zmm25, [screg+8*64+3*128]	;; sine for R5/I5
	vaddpd	zmm6, zmm14, zmm13		;; I4/sine + I6/sine				; 10-13		n 22
	vsubpd	zmm14, zmm14, zmm13		;; I4/sine - I6/sine				; 10-13		n 20

	mov	bl, [maskreg+0*1]		;; Load index into compressed fudges
	zfmaddpd zmm13, zmm4, zmm29, zmm16	;; R2 + R8 * sine				; 11-14		n 17
	zfnmaddpd zmm4, zmm4, zmm29, zmm16	;; R2 - R8 * sine				; 11-14		n 18

	vmovapd zmm8, [srcreg+64]		;; Load I1
	zfmaddpd zmm16, zmm15, zmm29, zmm12	;; I2 + I8 * sine				; 12-15		n 22
	zfnmaddpd zmm15, zmm15, zmm29, zmm12	;; I2 - I8 * sine				; 12-15		n 20

	vmovapd zmm26, [screg+8*64+1*128]	;; sine for R3/I3 and R7/I7
	zfmaddpd zmm12, zmm5, zmm25, zmm0	;; R1 + R5 * sine				; 13-16		n 23
	zfnmaddpd zmm5, zmm5, zmm25, zmm0	;; R1 - R5 * sine				; 13-16		n 26

	vmovapd zmm27, [screg+8*64+2*128]	;; sine for R4/I4 and R6/I6
	zfmaddpd zmm0, zmm9, zmm25, zmm8	;; I1 + I5 * sine				; 14-17		n 24
	zfnmaddpd zmm9, zmm9, zmm25, zmm8	;; I1 - I5 * sine				; 14-17		n 27

	vaddpd	zmm8, zmm1, zmm2		;; R3/sine + R7/sine				; 15-18		n 23
	vsubpd	zmm1, zmm1, zmm2		;; R3/sine - R7/sine				; 15-18		n 27
	mov	r14, [r13+0*8]			;; Load the xor mask

	vaddpd	zmm2, zmm10, zmm11		;; I3/sine + I7/sine				; 16-19		n 24
	vsubpd	zmm10, zmm10, zmm11		;; I3/sine - I7/sine				; 16-19		n 26
	xor	r14, [r12+rbx*8]		;; Apply the xor mask to 64 bits of fudge flags

	zfmaddpd zmm11, zmm3, zmm27, zmm13	;; r2++ = (r2+r8) + (r4+r6) * sine		; 17-20		n 30
	zfnmaddpd zmm3, zmm3, zmm27, zmm13	;; r2+- = (r2+r8) - (r4+r6) * sine		; 17-20		n 28
	mov	bl, [maskreg+1*1]		;; Load index into compressed fudges
	bump	maskreg, maskinc

	kmovw	k1, r14d			;; Load R1 and I1 fudge factor mask		; 18		n 46
	zfmaddpd zmm13, zmm7, zmm27, zmm4	;; r2-+ = (r2-r8) + (r4-r6) * sine		; 18-21		n 29
	shr	r14, 16				;; Next 16 bits of fudge flags

	kmovw	k2, r14d			;; Load R2 and I2 fudge factor mask		; 19		n 52
	zfnmaddpd zmm7, zmm7, zmm27, zmm4	;; r2-- = (r2-r8) - (r4-r6) * sine		; 19-22		n 33
	shr	r14, 16				;; Next 16 bits of fudge flags

	kmovw	k3, r14d			;; Load R3 and I3 fudge factor mask		; 20		n 47
	zfmaddpd zmm4, zmm14, zmm27, zmm15	;; i2-+ = (i2-i8) + (i4-i6) * sine		; 20-23		n 28
	shr	r14, 16				;; Next 16 bits of fudge flags

	kmovw	k4, r14d			;; Load R4 and I4 fudge factor mask		; 21		n 54
	zfnmaddpd zmm14, zmm14, zmm27, zmm15	;; i2-- = (i2-i8) - (i4-i6) * sine		; 21-24		n 32
	mov	r14, [r13+1*8]			;; Load the xor mask

	zfmaddpd zmm15, zmm6, zmm27, zmm16	;; i2++ = (i2+i8) + (i4+i6) * sine		; 22-25		n 31
	zfnmaddpd zmm6, zmm6, zmm27, zmm16	;; i2+- = (i2+i8) - (i4+i6) * sine		; 22-25		n 29
	xor	r14, [r12+rbx*8]		;; Apply the xor mask to 64 bits of fudge flags

	zfmaddpd zmm16, zmm8, zmm26, zmm12	;; r1++ = (r1+r5) + (r3+r7) * sine		; 23-26		n 30
	zfnmaddpd zmm8, zmm8, zmm26, zmm12	;; r1+- = (r1+r5) - (r3+r7) * sine		; 23-26		n 32
	vmovapd zmm20, [screg+0*64]		;; premultiplier cosine/sine for R1/I1

	kmovw	k5, r14d			;; Load R5 and I5 fudge factor mask		; 24		n 49
	zfmaddpd zmm12, zmm2, zmm26, zmm0	;; i1++ = (i1+i5) + (i3+i7) * sine		; 24-27		n 31
	shr	r14, 16				;; Next 16 bits of fudge flags
	vmovapd zmm19, [screg+2*64]		;; premultiplier cosine/sine for R3/I3

	kmovw	k6, r14d			;; Load R6 and I6 fudge factor mask		; 25		n 55
	zfnmaddpd zmm2, zmm2, zmm26, zmm0	;; i1+- = (i1+i5) - (i3+i7) * sine		; 25-28		n 33
	shr	r14, 16				;; Next 16 bits of fudge flags
	vmovapd zmm18, [screg+4*64]		;; premultiplier cosine/sine for R5/I5

	kmovw	k7, r14d			;; Load R7 and I7 fudge factor mask		; 26		n 51
	zfmaddpd zmm0, zmm10, zmm26, zmm5	;; r1-+ = (r1-r5) + (i3-i7) * sine		; 26-29		n 34
	shr	r14, 16				;; Next 16 bits of fudge flags
	vmovapd zmm25, [screg+6*64]		;; premultiplier cosine/sine for R7/I7

	zfnmaddpd zmm10, zmm10, zmm26, zmm5	;; r1-- = (r1-r5) - (i3-i7) * sine		; 27-30		n 36
	zfmaddpd zmm5, zmm1, zmm26, zmm9	;; i1-+ = (i1-i5) + (r3-r7) * sine		; 27-30		n 37
	vmovapd zmm24, [screg+1*64]		;; premultiplier cosine/sine for R2/I2

	zfnmaddpd zmm1, zmm1, zmm26, zmm9	;; i1-- = (i1-i5) - (r3-r7) * sine		; 28-31		n 35
	vaddpd	zmm9, zmm3, zmm4		;; r2+-+ = (r2+-) + (i2-+)			; 28-31		n 34
	vmovapd zmm23, [screg+3*64]		;; premultiplier cosine/sine for R4/I4

	vsubpd	zmm3, zmm3, zmm4		;; r2+-- = (r2+-) - (i2-+)			; 29-32		n 36
	vaddpd	zmm4, zmm13, zmm6		;; r2-++ = (r2-+) + (i2+-)			; 29-32		n 37
	vmovapd zmm22, [screg+5*64]		;; premultiplier cosine/sine for R6/I6

	vsubpd	zmm13, zmm13, zmm6		;; r2-+- = (r2-+) - (i2+-)			; 30-33		n 35
	vaddpd	zmm6, zmm16, zmm11		;; R1 = (r1++) + (r2++)				; 30-33		n 38
	vmovapd zmm21, [screg+7*64]		;; premultiplier cosine/sine for R8/I8

	vsubpd	zmm16, zmm16, zmm11		;; R5 = (r1++) - (r2++)				; 31-34		n 40
	vaddpd	zmm11, zmm12, zmm15		;; I1 = (i1++) + (i2++)				; 31-34		n 38
	bump	screg, scinc
	L1prefetch srcreg+L1pd, L1pt

	vsubpd	zmm12, zmm12, zmm15		;; I5 = (i1++) - (i2++)				; 32-35		n 40
	vaddpd	zmm15, zmm8, zmm14		;; R3 = (r1+-) + (i2--)				; 32-35		n 39
	L1prefetch srcreg+64+L1pd, L1pt

	vsubpd	zmm8, zmm8, zmm14		;; R7 = (r1+-) - (i2--)				; 33-36		n 41
	vsubpd	zmm14, zmm2, zmm7		;; I3 = (i1+-) - (r2--)				; 33-36		n 39
	L1prefetch srcreg+d1+L1pd, L1pt

	vaddpd	zmm2, zmm2, zmm7		;; I7 = (i1+-) + (r2--)				; 34-37		n 41
	zfmaddpd zmm7, zmm9, zmm31, zmm0	;; R2 = (r1-+) + .707*(r2+-+)			; 34-37		n 42
	L1prefetch srcreg+d1+64+L1pd, L1pt

	zfnmaddpd zmm9, zmm9, zmm31, zmm0	;; R6 = (r1-+) - .707*(r2+-+)			; 35-38		n 44
	zfnmaddpd zmm0, zmm13, zmm31, zmm1	;; I2 = (i1--) - .707*(r2-+-)			; 35-38		n 42
	L1prefetch srcreg+d2+L1pd, L1pt

	zfmaddpd zmm13, zmm13, zmm31, zmm1	;; I6 = (i1--) + .707*(r2-+-)			; 36-39		n 44
	zfnmaddpd zmm1, zmm3, zmm31, zmm10	;; R4 = (r1--) - .707*(r2+--)			; 36-39		n 43
	L1prefetch srcreg+d2+64+L1pd, L1pt

	zfmaddpd zmm3, zmm3, zmm31, zmm10	;; R8 = (r1--) + .707*(r2+--)			; 37-40		n 45
	zfnmaddpd zmm10, zmm4, zmm31, zmm5	;; I4 = (i1-+) - .707*(r2-++)			; 37-40		n 43
	L1prefetch srcreg+d2+d1+L1pd, L1pt

	zfmaddpd zmm4, zmm4, zmm31, zmm5	;; I8 = (i1-+) + .707*(r2-++)			; 38-41		n 45
	zfmaddpd zmm5, zmm6, zmm20, zmm11	;; A1 = R1 * cosine + I1			; 38-41		n 46
	L1prefetch srcreg+d2+d1+64+L1pd, L1pt

	zfmsubpd zmm11, zmm11, zmm20, zmm6	;; B1 = I1 * cosine - R1			; 39-42		n 48
	zfmaddpd zmm6, zmm15, zmm19, zmm14	;; A3 = R3 * cosine + I3			; 39-42		n 47
	L1prefetch srcreg+d4+L1pd, L1pt

	zfmsubpd zmm14, zmm14, zmm19, zmm15	;; B3 = I3 * cosine - R3			; 40-43		n 50
	zfmaddpd zmm15, zmm16, zmm18, zmm12	;; A5 = R5 * cosine + I5			; 40-43		n 49
	L1prefetch srcreg+d4+64+L1pd, L1pt

	zfmsubpd zmm12, zmm12, zmm18, zmm16	;; B5 = I5 * cosine - R5			; 41-44		n 51
	zfmaddpd zmm16, zmm8, zmm25, zmm2	;; A7 = R7 * cosine + I7			; 41-44		n 51
	L1prefetch srcreg+d4+d1+L1pd, L1pt

	zfmsubpd zmm2, zmm2, zmm25, zmm8	;; B7 = I7 * cosine - R7			; 42-45		n 53
	zfmaddpd zmm8, zmm7, zmm24, zmm0	;; A2 = R2 * cosine + I2			; 42-45		n 52
	L1prefetch srcreg+d4+d1+64+L1pd, L1pt

	zfmsubpd zmm0, zmm0, zmm24, zmm7	;; B2 = I2 * cosine - R2			; 43-46		n 54
	zfmaddpd zmm7, zmm1, zmm23, zmm10	;; A4 = R4 * cosine + I4			; 43-46		n 54
	L1prefetch srcreg+d4+d2+L1pd, L1pt

	zfmsubpd zmm10, zmm10, zmm23, zmm1	;; B4 = I4 * cosine - R4			; 44-47		n 57
	zfmaddpd zmm1, zmm9, zmm22, zmm13	;; A6 = R6 * cosine + I6			; 44-47		n 55
	L1prefetch srcreg+d4+d2+64+L1pd, L1pt

	zfmsubpd zmm13, zmm13, zmm22, zmm9	;; B6 = I6 * cosine - R6			; 45-48		n 58
	zfmaddpd zmm9, zmm3, zmm21, zmm4	;; A8 = R8 * cosine + I8			; 45-48		n 56
	L1prefetch srcreg+d4+d2+d1+L1pd, L1pt

	zfmsubpd zmm4, zmm4, zmm21, zmm3	;; B8 = I8 * cosine - R8			; 46-49		n 58
	vmulpd	zmm5{k1}, zmm5, zmm28		;; apply fudge multiplier for R1		; 46-49
	L1prefetch srcreg+d4+d2+d1+64+L1pd, L1pt

	kshiftrw k1, k1, 8			;; I1's fudge					; 47		n 48
	vmulpd	zmm6{k3}, zmm6, zmm28		;; apply fudge multiplier for R3		; 47-50

	kshiftrw k3, k3, 8			;; I3's fudge					; 48		n 50
	vmulpd	zmm11{k1}, zmm11, zmm28		;; apply fudge multiplier for I1		; 48-51

	kmovw	k1, r14d			;; Load R8 and I8 fudge factor mask		; 49		n 56
	vmulpd	zmm15{k5}, zmm15, zmm28		;; apply fudge multiplier for R5		; 49-52

	kshiftrw k5, k5, 8			;; I5's fudge					; 50		n 51
	vmulpd	zmm14{k3}, zmm14, zmm28		;; apply fudge multiplier for I3		; 50-53
	zstore	[srcreg], zmm5			;; Save R1					; 50

	vmulpd	zmm16{k7}, zmm16, zmm28		;; apply fudge multiplier for R7		; 51-54
	vmulpd	zmm12{k5}, zmm12, zmm28		;; apply fudge multiplier for I5		; 51-54
	zstore	[srcreg+d2], zmm6		;; Save R3					; 51

	kshiftrw k7, k7, 8			;; I7's fudge					; 52		n 53
	vmulpd	zmm8{k2}, zmm8, zmm28		;; apply fudge multiplier for R2		; 52-55
	zstore	[srcreg+64], zmm11		;; Save I1					; 52

	kshiftrw k2, k2, 8			;; I2's fudge					; 53		n 54
	vmulpd	zmm2{k7}, zmm2, zmm28		;; apply fudge multiplier for I7		; 53-56
	zstore	[srcreg+d4], zmm15		;; Save R5					; 53

	vmulpd	zmm0{k2}, zmm0, zmm28		;; apply fudge multiplier for I2		; 54-57
	vmulpd	zmm7{k4}, zmm7, zmm28		;; apply fudge multiplier for R4		; 54-57
	zstore	[srcreg+d2+64], zmm14		;; Save I3					; 54

	kshiftrw k4, k4, 8			;; I4's fudge					; 55		n 57
	vmulpd	zmm1{k6}, zmm1, zmm28		;; apply fudge multiplier for R6		; 55-58
	zstore	[srcreg+d4+d2], zmm16		;; Save R7					; 55

	kshiftrw k6, k6, 8			;; I6's fudge					; 56		n 58
	vmulpd	zmm9{k1}, zmm9, zmm28		;; apply fudge multiplier for R8		; 56-59
	zstore	[srcreg+d4+64], zmm12		;; Save I5					; 55+1

	kshiftrw k1, k1, 8			;; I8's fudge					; 57		n 58
	vmulpd	zmm10{k4}, zmm10, zmm28		;; apply fudge multiplier for I4		; 57-60
	zstore	[srcreg+d1], zmm8		;; Save R2					; 56+1

	vmulpd	zmm13{k6}, zmm13, zmm28		;; apply fudge multiplier for I6		; 58-61
	vmulpd	zmm4{k1}, zmm4, zmm28		;; apply fudge multiplier for I8		; 58-61
	zstore	[srcreg+d4+d2+64], zmm2		;; Save I7					; 57+1

	zstore	[srcreg+d1+64], zmm0		;; Save I2					; 58+1
	zstore	[srcreg+d2+d1], zmm7		;; Save R4					; 58+2
	zstore	[srcreg+d4+d1], zmm1		;; Save R6					; 59+2
	zstore	[srcreg+d4+d2+d1], zmm9		;; Save R8					; 60+2
	zstore	[srcreg+d2+d1+64], zmm10	;; Save I4					; 61+2
	zstore	[srcreg+d4+d1+64], zmm13	;; Save I6					; 62+2
	zstore	[srcreg+d4+d2+d1+64], zmm4	;; Save I8					; 62+3
	bump	srcreg, srcinc
	ENDM


;;
;; ********************************* reduced sin/cos eight-complex-fft8 variants **************************************
;;
;; These macros are used in the last levels of pass 1.  These macros differ from the standard eight-complex macros
;; in that the sin/cos values are computed on the fly to reduce memory requirements.  To do this we cannot
;; store data as (cos/sin,sin), these macros work on (cos,sin) values. These macros also apply a column weights multiplier.

zr8_rsc_wpn_eight_complex_calc_sincos MACRO weights, src1, src2, clm
	zr8_rsc_wpn_eight_complex_calc_sincos1 weights+0*64, src1+0*ZMM_SCD1, src2+0*ZMM_SCD4, ZMM_TMPS[0*ZMM_SCD8]
	IF clm GE 2
	zr8_rsc_wpn_eight_complex_calc_sincos1 weights+1*64, src1+1*ZMM_SCD1, src2+1*ZMM_SCD4, ZMM_TMPS[1*ZMM_SCD8]
	ENDIF
	IF clm GE 4
	zr8_rsc_wpn_eight_complex_calc_sincos1 weights+2*64, src1+2*ZMM_SCD1, src2+2*ZMM_SCD4, ZMM_TMPS[2*ZMM_SCD8]
	zr8_rsc_wpn_eight_complex_calc_sincos1 weights+3*64, src1+3*ZMM_SCD1, src2+3*ZMM_SCD4, ZMM_TMPS[3*ZMM_SCD8]
	ENDIF
	ENDM

zr8_rsc_wpn_eight_complex_calc_sincos1 MACRO weights, src1, src2, dest
	vmovapd	zmm0, [src1+64]		;; cos
	vmovapd	zmm1, [src1]		;; sin
	vmulpd	zmm0, zmm0, [weights]	;; Apply the column weight		; 1-4
	vmulpd	zmm1, zmm1, [weights]	;; Apply the column weight		; 1-4
;; BUG - interleave the clm=2 and clm=4 cases here to hide this stall
	vmovapd	zmm2, [src2+0*128+64]	;; cos1
	vmulpd	zmm3, zmm0, zmm2	;; ac1 = cos * cos1			; 5-8
	vmulpd	zmm2, zmm1, zmm2	;; bc1 = sin * cos1			; 5-8
	vmovapd	zmm4, [src2+1*128+64]	;; cos2
	vmulpd	zmm5, zmm0, zmm4	;; ac2 = cos * cos2			; 6-9
	vmulpd	zmm4, zmm1, zmm4	;; bc2 = sin * cos2			; 6-9
	vmovapd	zmm6, [src2+2*128+64]	;; cos3
	vmulpd	zmm7, zmm0, zmm6	;; ac3 = cos * cos3			; 7-10
	vmulpd	zmm6, zmm1, zmm6	;; bc3 = sin * cos3			; 7-10
	vmovapd	zmm8, [src2+3*128+64]	;; cos4
	vmulpd	zmm9, zmm0, zmm8	;; ac4 = cos * cos4			; 8-11
	vmulpd	zmm8, zmm1, zmm8	;; bc4 = sin * cos4			; 8-11
	zstore	dest[0*128+64], zmm0
	zstore	dest[0*128], zmm1
	vmovapd	zmm15, [src2+0*128]	;; sin1
	zfnmaddpd zmm10, zmm1, zmm15, zmm3 ;; new cos1 = ac1 - sin * sin1	; 9-12
	zfmaddpd zmm3, zmm1, zmm15, zmm3 ;; new cos7 = ac1 + sin * sin1		; 9-12
	zfmaddpd zmm11, zmm0, zmm15, zmm2 ;; new sin1 = cos * sin1 + bc1	; 10-13
	zfnmaddpd zmm2, zmm0, zmm15, zmm2 ;; new sin7 = bc1 - cos * sin1	; 10-13
	vmovapd	zmm15, [src2+1*128]	;; sin2
	zfnmaddpd zmm12, zmm1, zmm15, zmm5 ;; new cos2 = ac2 - sin * sin2	; 11-14
	zfmaddpd zmm5, zmm1, zmm15, zmm5 ;; new cos6 = ac2 + sin * sin2		; 11-14
	zfmaddpd zmm13, zmm0, zmm15, zmm4 ;; new sin2 = cos * sin2 + bc2	; 12-15
	zfnmaddpd zmm4, zmm0, zmm15, zmm4 ;; new sin6 = bc2 - cos * sin2	; 12-15
	vmovapd	zmm15, [src2+2*128]	;; sin3
	zfnmaddpd zmm14, zmm1, zmm15, zmm7 ;; new cos3 = ac3 - sin * sin3	; 13-16
	zfmaddpd zmm7, zmm1, zmm15, zmm7 ;; new cos5 = ac3 + sin * sin3		; 13-16
	zstore	dest[1*128+64], zmm10	;; cos1
	zfmaddpd zmm10, zmm0, zmm15, zmm6 ;; new sin3 = cos * sin3 + bc3	; 14-17
	zfnmaddpd zmm6, zmm0, zmm15, zmm6 ;; new sin5 = bc3 - cos * sin3	; 14-17
	vmovapd	zmm15, [src2+3*128]	;; sin4
	zstore	dest[7*128+64], zmm3	;; cos7
	zfnmaddpd zmm9, zmm1, zmm15, zmm9 ;; new cos4 = ac4 - sin * sin4	; 15-18
	zfmaddpd zmm8, zmm0, zmm15, zmm8 ;; new sin4 = cos * sin4 + bc4		; 15-18
	zstore	dest[1*128], zmm11	;; sin1
	zstore	dest[7*128], zmm2	;; sin7
	zstore	dest[2*128+64], zmm12	;; cos2
	zstore	dest[6*128+64], zmm5	;; cos6
	zstore	dest[2*128], zmm13	;; sin2
	zstore	dest[6*128], zmm4	;; sin6
	zstore	dest[3*128], zmm10	;; sin3
	zstore	dest[3*128+64], zmm14	;; cos3
	zstore	dest[4*128], zmm8	;; sin4
	zstore	dest[4*128+64], zmm9	;; cos4
	zstore	dest[5*128], zmm6	;; sin5
	zstore	dest[5*128+64], zmm7	;; cos5
	ENDM

zr8_rsc_wpn_eight_complex_calc_sincos_simple MACRO weights, src2, clm
	zr8_rsc_wpn_eight_complex_calc_sincos_simple1 weights+0*64, src2+0*ZMM_SCD4, ZMM_TMPS[0*ZMM_SCD8]
	IF clm GE 2
	zr8_rsc_wpn_eight_complex_calc_sincos_simple1 weights+1*64, src2+1*ZMM_SCD4, ZMM_TMPS[1*ZMM_SCD8]
	ENDIF
	IF clm GE 4
	zr8_rsc_wpn_eight_complex_calc_sincos_simple1 weights+2*64, src2+2*ZMM_SCD4, ZMM_TMPS[2*ZMM_SCD8]
	zr8_rsc_wpn_eight_complex_calc_sincos_simple1 weights+3*64, src2+3*ZMM_SCD4, ZMM_TMPS[3*ZMM_SCD8]
	ENDIF
	ENDM

zr8_rsc_wpn_eight_complex_calc_sincos_simple1 MACRO weights, src2, dest
	vmovapd	zmm0, [weights]			;; cos (1) * column weight
	vpxorq	zmm1, zmm1, zmm1		;; sin (0) * column weight
	zstore	dest[0*128+64], zmm0
	zstore	dest[0*128], zmm1

	vmulpd	zmm2, zmm0, [src2+0*128+64]	;; cos * column weight
	vmulpd	zmm3, zmm0, [src2+0*128]	;; sin * column weight
	zstore	dest[1*128+64], zmm2	
	zstore	dest[1*128], zmm3
	zstore	dest[7*128+64], zmm2
	vsubpd	zmm3, zmm1, zmm3		;; - sin * column weight
	zstore	dest[7*128], zmm3

	vmulpd	zmm2, zmm0, [src2+1*128+64]	;; cos * column weight
	vmulpd	zmm3, zmm0, [src2+1*128]	;; sin * column weight
	zstore	dest[2*128+64], zmm2
	zstore	dest[2*128], zmm3
	zstore	dest[6*128+64], zmm2
	vsubpd	zmm3, zmm1, zmm3		;; - sin * column weight
	zstore	dest[6*128], zmm3

	vmulpd	zmm2, zmm0, [src2+2*128+64]	;; cos * column weight
	vmulpd	zmm3, zmm0, [src2+2*128]	;; sin * column weight
	zstore	dest[3*128+64], zmm2
	zstore	dest[3*128], zmm3
	zstore	dest[5*128+64], zmm2
	vsubpd	zmm3, zmm1, zmm3		;; - sin * column weight
	zstore	dest[5*128], zmm3

	vmulpd	zmm2, zmm0, [src2+3*128+64]	;; cos * column weight
	vmulpd	zmm3, zmm0, [src2+3*128]	;; sin * column weight
	zstore	dest[4*128+64], zmm2
	zstore	dest[4*128], zmm3
	ENDM


;; Use registers for distances between output blocks.  This lets us share pass1 code.
;; Faster, but less readable version, that integrates shuffles with computation

zr8_rsc_complex_only_preload MACRO
	mov	eax, 00110011b			;; For vblendmpd during swizzle
	kmovw	k7, eax
	vpmovzxbq zmm31, ZMM_PERMUTE1		;; zmm31 = 8+4 0+4 8+6 0+6 8+0 0+0 8+2 0+2 [msw at left]
	vpmovzxbq zmm30, ZMM_PERMUTE2		;; zmm30 = 8+5 0+5 8+7 0+7 8+1 0+1 8+3 0+3
	vbroadcastsd zmm29, ZMM_SQRTHALF
	ENDM

zr8_rsc_wpn_sgreg_eight_complex_fft8_preload MACRO
	use zr8_rsc_complex_only_preload or zr8_rsc_complex_and_real_preload
	ENDM

zr8_rsc_wpn_sgreg_eight_complex_fft8 MACRO srcreg,srcinc,d1,d2,d4,dstreg,dstinc,e1reg,e3reg,dst4reg,screg,scinc,maxrpt,L1pt,L1preg,L1p4reg
	vmovapd	zmm3, [srcreg+d2+64]		;; i3_7	i3_6 i3_5 i3_4 i3_3 i3_2 i3_1 i3_0
	vmovapd	zmm28, [srcreg+d2+d1+64]	;; i4_7	i4_6 i4_5 i4_4 i4_3 i4_2 i4_1 i4_0
	zperm2pd zmm2, zmm31, zmm3, zmm28	;; i4_4 i3_4 i4_6 i3_6 i4_0 i3_0 i4_2 i3_2		; 1-3		n 5
	zperm2pd zmm3, zmm30, zmm3, zmm28	;; i4_5 i3_5 i4_7 i3_7 i4_1 i3_1 i4_3 i3_3		; 2-4		n 7

	vmovapd	zmm0, [srcreg+64]		;; i1_7	i1_6 i1_5 i1_4 i1_3 i1_2 i1_1 i1_0
	vmovapd	zmm28, [srcreg+d1+64]		;; i2_7	i2_6 i2_5 i2_4 i2_3 i2_2 i2_1 i2_0
	vshufpd	zmm1, zmm0, zmm28, 11111111b	;; i2_7 i1_7 i2_5 i1_5 i2_3 i1_3 i2_1 i1_1		; 3		n 7
	vshufpd	zmm0, zmm0, zmm28, 00000000b	;; i2_6 i1_6 i2_4 i1_4 i2_2 i1_2 i2_0 i1_0		; 4		n 5

	vmovapd	zmm7, [srcreg+d4+d2+64]		;; i7_7	i7_6 i7_5 i7_4 i7_3 i7_2 i7_1 i7_0
	vmovapd	zmm28, [srcreg+d4+d2+d1+64]	;; i8_7	i8_6 i8_5 i8_4 i8_3 i8_2 i8_1 i8_0
	zperm2pd zmm6, zmm31, zmm7, zmm28	;; i8_4 i7_4 i8_6 i7_6 i8_0 i7_0 i8_2 i7_2		; 5-7		n 9
	vblendmpd zmm16{k7}, zmm2, zmm0		;; i4_4 i3_4 i2_4 i1_4 i4_0 i3_0 i2_0 i1_0		;  5		n 17

	zperm2pd zmm7, zmm30, zmm7, zmm28	;; i8_5 i7_5 i8_7 i7_7 i8_1 i7_1 i8_3 i7_3		; 6-8		n 11
	vblendmpd zmm0{k7}, zmm0, zmm2		;; i2_6 i1_6 i4_6 i3_6 i2_2 i1_2 i4_2 i3_2		;  6		n 19

	vmovapd	zmm5, [srcreg+d4+64]		;; i5_7	i5_6 i5_5 i5_4 i5_3 i5_2 i5_1 i5_0
	vmovapd	zmm28, [srcreg+d4+d1+64]	;; i6_7	i6_6 i6_5 i6_4 i6_3 i6_2 i6_1 i6_0
	vshufpd	zmm4, zmm5, zmm28, 00000000b	;; i6_6 i5_6 i6_4 i5_4 i6_2 i5_2 i6_0 i5_0		; 7		n 9
	vblendmpd zmm2{k7}, zmm3, zmm1		;; i4_5 i3_5 i2_5 i1_5 i4_1 i3_1 i2_1 i1_1		;  7		n 21

	vshufpd	zmm5, zmm5, zmm28, 11111111b	;; i6_7 i5_7 i6_5 i5_5 i6_3 i5_3 i6_1 i5_1		; 8		n 11
	vblendmpd zmm1{k7}, zmm1, zmm3		;; i2_7 i1_7 i4_7 i3_7 i2_3 i1_3 i4_3 i3_3		;  8		n 23

	vmovapd	zmm15, [srcreg+d4+d2]		;; r7_7	r7_6 r7_5 r7_4 r7_3 r7_2 r7_1 r7_0
	vmovapd	zmm28, [srcreg+d4+d2+d1]	;; r8_7	r8_6 r8_5 r8_4 r8_3 r8_2 r8_1 r8_0
	zperm2pd zmm14, zmm31, zmm15, zmm28	;; r8_4 r7_4 r8_6 r7_6 r8_0 r7_0 r8_2 r7_2		; 9-11		n 13
	vblendmpd zmm3{k7}, zmm6, zmm4		;; i8_4 i7_4 i6_4 i5_4 i8_0 i7_0 i6_0 i5_0		;  9		n 17

	zperm2pd zmm15, zmm30, zmm15, zmm28	;; r8_5 r7_5 r8_7 r7_7 r8_1 r7_1 r8_3 r7_3		; 10-12		n 15
	vblendmpd zmm4{k7}, zmm4, zmm6		;; i6_6 i5_6 i8_6 i7_6 i6_2 i5_2 i8_2 i7_2		;  10		n 19

	vmovapd	zmm13, [srcreg+d4]		;; r5_7	r5_6 r5_5 r5_4 r5_3 r5_2 r5_1 r5_0
	vmovapd	zmm28, [srcreg+d4+d1]		;; r6_7	r6_6 r6_5 r6_4 r6_3 r6_2 r6_1 r6_0
	vshufpd	zmm12, zmm13, zmm28, 00000000b	;; r6_6 r5_6 r6_4 r5_4 r6_2 r5_2 r6_0 r5_0		; 11		n 13
	vblendmpd zmm6{k7}, zmm7, zmm5		;; i8_5 i7_5 i6_5 i5_5 i8_1 i7_1 i6_1 i5_1		;  11		n 21

	vshufpd	zmm13, zmm13, zmm28, 11111111b	;; r6_7 r5_7 r6_5 r5_5 r6_3 r5_3 r6_1 r5_1		; 12		n 15
	vblendmpd zmm5{k7}, zmm5, zmm7		;; i6_7 i5_7 i8_7 i7_7 i6_3 i5_3 i8_3 i7_3		;  12		n 23

	vmovapd	zmm11, [srcreg+d2]		;; r3_7	r3_6 r3_5 r3_4 r3_3 r3_2 r3_1 r3_0
	vmovapd	zmm28, [srcreg+d2+d1]		;; r4_7	r4_6 r4_5 r4_4 r4_3 r4_2 r4_1 r4_0
	zperm2pd zmm10, zmm31, zmm11, zmm28	;; r4_4 r3_4 r4_6 r3_6 r4_0 r3_0 r4_2 r3_2		; 13-15		n 17
	vblendmpd zmm7{k7}, zmm14, zmm12	;; r8_4 r7_4 r6_4 r5_4 r8_0 r7_0 r6_0 r5_0		;  13		n 29

	zperm2pd zmm11, zmm30, zmm11, zmm28	;; r4_5 r3_5 r4_7 r3_7 r4_1 r3_1 r4_3 r3_3		; 14-16		n 19
	vblendmpd zmm12{k7}, zmm12, zmm14	;; r6_6 r5_6 r8_6 r7_6 r6_2 r5_2 r8_2 r7_2		;  14		n 31

	vmovapd	zmm9, [srcreg]			;; r1_7	r1_6 r1_5 r1_4 r1_3 r1_2 r1_1 r1_0
	vmovapd	zmm28, [srcreg+d1]		;; r2_7	r2_6 r2_5 r2_4 r2_3 r2_2 r2_1 r2_0
	vshufpd	zmm8, zmm9, zmm28, 00000000b	;; r2_6 r1_6 r2_4 r1_4 r2_2 r1_2 r2_0 r1_0		; 15		n 17
	vblendmpd zmm14{k7}, zmm15, zmm13	;; r8_5 r7_5 r6_5 r6_5 r8_1 r7_1 r6_1 r5_1		;  15		n 27

	vshufpd	zmm9, zmm9, zmm28, 11111111b	;; r2_7 r1_7 r2_5 r1_5 r2_3 r1_3 r2_1 r1_1		; 16		n 19
	vblendmpd zmm13{k7}, zmm13, zmm15	;; r6_7 r5_7 r8_7 r7_7 r6_3 r5_3 r8_3 r7_3		;  16		n 25
	vmovapd	zmm25, [screg+0*128+64]		;; cosine for R1/I1
	bump	srcreg, srcinc

	vshuff64x2 zmm17, zmm16, zmm3, 01000100b;; i8_0 i7_0 i6_0 i5_0 i4_0 i3_0 i2_0 i1_0		; 17-19		n 21
	vblendmpd zmm15{k7}, zmm10, zmm8	;; r4_4 r3_4 r2_4 r1_4 r4_0 r3_0 r2_0 r1_0		;  17		n 29
	vmovapd	zmm28, [screg+4*128+64]		;; cosine for R2/I2

	vshuff64x2 zmm16, zmm16, zmm3, 11101110b;; i8_4 i7_4 i6_4 i5_4 i4_4 i3_4 i2_4 i1_4		; 18-20		n 21
	vblendmpd zmm8{k7}, zmm8, zmm10		;; r2_6 r1_6 r4_6 r3_6 r2_2 r1_2 r4_2 r3_2		;  18		n 31
	vmovapd	zmm27, [screg+2*128+64]		;; cosine for R3/I3

	vshuff64x2 zmm3, zmm0, zmm4, 00010001b	;; i8_2 i7_2 i6_2 i5_2 i4_2 i3_2 i2_2 i1_2		; 19-21		n 23
	vblendmpd zmm10{k7}, zmm11, zmm9	;; r4_5 r3_5 r2_5 r1_5 r4_1 r3_1 r2_1 r1_1		;  19		n 27
	vmovapd	zmm26, [screg+6*128+64]		;; cosine for R4/I4

	vshuff64x2 zmm0, zmm0, zmm4, 10111011b	;; i8_6 i7_6 i6_6 i5_6 i4_6 i3_6 i2_6 i1_6		; 20-22		n 23
	vblendmpd zmm9{k7}, zmm9, zmm11		;; r2_7 r1_7 r4_7 r3_7 r2_3 r1_3 r4_3 r3_3		;  20		n 25
	vmovapd	zmm24, [screg+0*128]		;; sine for R1/I1

	vshuff64x2 zmm4, zmm2, zmm6, 01000100b	;; i8_1 i7_1 i6_1 i5_1 i4_1 i3_1 i2_1 i1_1		; 21-23		n 25
	vshuff64x2 zmm2, zmm2, zmm6, 11101110b	;; i8_5 i7_5 i6_5 i5_5 i4_5 i3_5 i2_5 i1_5		; 22-24		n 25
	vmovapd	zmm23, [screg+4*128]		;; sine for R2/I2

	vaddpd	zmm11, zmm17, zmm16		;; I1 + I5 (new I1)					; 21-24		n 38
	vsubpd	zmm17, zmm17, zmm16		;; I1 - I5 (new I5)					; 22-25		n 46
	vmovapd	zmm22, [screg+2*128]		;; sine for R3/I3

	vshuff64x2 zmm6, zmm1, zmm5, 00010001b	;; i8_3 i7_3 i6_3 i5_3 i4_3 i3_3 i2_3 i1_3		; 23-25		n 27
	vshuff64x2 zmm1, zmm1, zmm5, 10111011b	;; i8_7 i7_7 i6_7 i5_7 i4_7 i3_7 i2_7 i1_7		; 24-26		n 27
	vmovapd	zmm21, [screg+6*128]		;; sine for R4/I4

	vaddpd	zmm16, zmm3, zmm0		;; I3 + I7 (new I3)					; 23-26		n 38
	vsubpd	zmm3, zmm3, zmm0		;; I3 - I7 (new I7)					; 24-27		n 42
	L1prefetchw dstreg, L1pt - L1PREFETCH_DEST_NONE

	vshuff64x2 zmm5, zmm9, zmm13, 00010001b	;; r8_3 r7_3 r6_3 r5_3 r4_3 r3_3 r2_3 r1_3		; 25-27		n 29
	vshuff64x2 zmm9, zmm9, zmm13, 10111011b ;; r8_7 r7_7 r6_7 r5_7 r4_7 r3_7 r2_7 r1_7		; 26-28		n 29
	L1prefetchw dstreg+64, L1pt - L1PREFETCH_DEST_NONE

	vsubpd	zmm0, zmm4, zmm2		;; I2 - I6 (new I6)					; 25-28		n 34
	vaddpd	zmm4, zmm4, zmm2		;; I2 + I6 (new I2)					; 26-29		n 37
	L1prefetchw dstreg+e1reg, L1pt - L1PREFETCH_DEST_NONE

	vshuff64x2 zmm13, zmm10, zmm14, 01000100b ;; r8_1 r7_1 r6_1 r5_1 r4_1 r3_1 r2_1 r1_1		; 27-29		n 31
	vshuff64x2 zmm10, zmm10, zmm14, 11101110b ;; r8_5 r7_5 r6_5 r5_5 r4_5 r3_5 r2_5 r1_5		; 28-30		n 31
	L1prefetchw dstreg+e1reg+64, L1pt - L1PREFETCH_DEST_NONE

	vsubpd	zmm2, zmm6, zmm1		;; I4 - I8 (new I8)					; 27-30		n 35
	vaddpd	zmm6, zmm6, zmm1		;; I4 + I8 (new I4)					; 28-31		n 37
	L1prefetchw dstreg+2*e1reg, L1pt - L1PREFETCH_DEST_NONE

	vshuff64x2 zmm14, zmm15, zmm7, 01000100b;; r8_0 r7_0 r6_0 r5_0 r4_0 r3_0 r2_0 r1_0		; 29-31		n 33
	vshuff64x2 zmm15, zmm15, zmm7, 11101110b;; r8_4 r7_4 r6_4 r5_4 r4_4 r3_4 r2_4 r1_4		; 30-32		n 33
	L1prefetchw dstreg+2*e1reg+64, L1pt - L1PREFETCH_DEST_NONE

	vsubpd	zmm1, zmm5, zmm9		;; R4 - R8 (new R8)					; 29-32		n 34
	vaddpd	zmm5, zmm5, zmm9		;; R4 + R8 (new R4)					; 30-33		n 39
	L1prefetchw dstreg+e3reg, L1pt - L1PREFETCH_DEST_NONE

	vshuff64x2 zmm7, zmm8, zmm12, 00010001b ;; r8_2 r7_2 r6_2 r5_2 r4_2 r3_2 r2_2 r1_2		; 31-33		n 36
	vshuff64x2 zmm8, zmm8, zmm12, 10111011b ;; r8_6 r7_6 r6_6 r5_6 r4_6 r3_6 r2_6 r1_6		; 32-34		n 36
	L1prefetchw dstreg+e3reg+64, L1pt - L1PREFETCH_DEST_NONE

	vsubpd	zmm9, zmm13, zmm10		;; R2 - R6 (new R6)					; 31-34		n 35
	vaddpd	zmm13, zmm13, zmm10		;; R2 + R6 (new R2)					; 32-35		n 39
	L1prefetchw dst4reg, L1pt - L1PREFETCH_DEST_NONE

	vsubpd	zmm10, zmm14, zmm15		;; R1 - R5 (new R5)					; 33-36		n 42
	vaddpd	zmm14, zmm14, zmm15		;; R1 + R5 (new R1)					; 33-36		n 45
	L1prefetchw dst4reg+64, L1pt - L1PREFETCH_DEST_NONE

	vaddpd	zmm15, zmm0, zmm1		;; I6 + R8 (new2 I6)					; 34-37		n 41
	vsubpd	zmm0, zmm0, zmm1		;; I6 - R8 (new2 I8)					; 34-37		n 40
	L1prefetchw dst4reg+e1reg, L1pt - L1PREFETCH_DEST_NONE

	vaddpd	zmm1, zmm9, zmm2		;; R6 + I8 (new2 R8)					; 35-38		n 40
	vsubpd	zmm9, zmm9, zmm2		;; R6 - I8 (new2 R6)					; 35-38		n 41
	L1prefetchw dst4reg+e1reg+64, L1pt - L1PREFETCH_DEST_NONE

	vaddpd	zmm2, zmm7, zmm8		;; R3 + R7 (new R3)					; 36-39		n 45
	vsubpd	zmm7, zmm7, zmm8		;; R3 - R7 (new R7)					; 36-39		n 46
	L1prefetchw dst4reg+2*e1reg, L1pt - L1PREFETCH_DEST_NONE

	vaddpd	zmm8, zmm4, zmm6		;; I2 + I4 (newer I2)					; 37-40		n 43
	vsubpd	zmm4, zmm4, zmm6		;; I2 - I4 (newer I4)					; 37-40		n 51
	L1prefetchw dst4reg+2*e1reg+64, L1pt - L1PREFETCH_DEST_NONE

	vaddpd	zmm6, zmm11, zmm16		;; I1 + I3 (newer I1)					; 38-41		n 43
	vsubpd	zmm11, zmm11, zmm16		;; I1 - I3 (newer I3)					; 38-41		n 44
	L1prefetchw dst4reg+e3reg, L1pt - L1PREFETCH_DEST_NONE

	vsubpd	zmm16, zmm13, zmm5		;; R2 - R4 (newer R4)					; 39-42		n 44
	vaddpd	zmm13, zmm13, zmm5		;; R2 + R4 (newer R2)					; 39-42		n 50
	L1prefetchw dst4reg+e3reg+64, L1pt - L1PREFETCH_DEST_NONE

	vaddpd	zmm5, zmm1, zmm0		;; R8 + I8 (newer I8/SQRTHALF)				; 40-43		n 48
	vsubpd	zmm1, zmm1, zmm0		;; R8 - I8 (newer R8/SQRTHALF)				; 40-43		n 53

	vsubpd	zmm0, zmm9, zmm15		;; R6 - I6 (newer R6/SQRTHALF)				; 41-44		n 47
	vaddpd	zmm9, zmm9, zmm15		;; R6 + I6 (newer I6/SQRTHALF)				; 41-44		n 52

	vsubpd	zmm15, zmm10, zmm3		;; R5 - I7 (newer R5)					; 42-45		n 47
	vaddpd	zmm10, zmm10, zmm3		;; R5 + I7 (newer R7)					; 42-45		n 48

	vaddpd	zmm3, zmm6, zmm8		;; I1 + I2 (last I1)					; 43-46		n 49
	vsubpd	zmm6, zmm6, zmm8		;; I1 - I2 (last I2)					; 43-46		n 49

	vaddpd	zmm8, zmm11, zmm16		;; I3 + R4 (last I3)					; 44-47		n 54
	vsubpd	zmm11, zmm11, zmm16		;; I3 - R4 (last I4)					; 44-47		n 54

	vaddpd	zmm16, zmm14, zmm2		;; R1 + R3 (newer R1)					; 45-48		n 50
	vsubpd	zmm14, zmm14, zmm2		;; R1 - R3 (newer R3)					; 45-48		n 51

 	vaddpd	zmm2, zmm17, zmm7		;; I5 + R7 (newer I5)					; 46-49		n 52
	vsubpd	zmm17, zmm17, zmm7		;; I5 - R7 (newer I7)					; 46-49		n 53

	zfmaddpd zmm7, zmm0, zmm29, zmm15	;; R5 + R6 * SQRTHALF (last R5)				; 47-50		n 61
	zfnmaddpd zmm0, zmm0, zmm29, zmm15	;; R5 - R6 * SQRTHALF (last R6)				; 47-50		n 62

	zfnmaddpd zmm15, zmm5, zmm29, zmm10	;; R7 - I8 * SQRTHALF (last R7)				; 48-51		n 63
	zfmaddpd zmm5, zmm5, zmm29, zmm10	;; R7 + I8 * SQRTHALF (last R8)				; 48-51		n 64

	vmulpd	zmm12, zmm3, zmm25		;; B1 = I1 * cosine					; 49-52		n 57
	vmulpd	zmm18, zmm6, zmm28		;; B2 = I2 * cosine					; 49-52		n 57

	vaddpd	zmm10, zmm16, zmm13		;; R1 + R2 (last R1)					; 50-53		n 55
	vsubpd	zmm16, zmm16, zmm13		;; R1 - R2 (last R2)					; 50-53		n 55

	vsubpd	zmm13, zmm14, zmm4		;; R3 - I4 (last R3)					; 51-54		n 56
	vaddpd	zmm14, zmm14, zmm4		;; R3 + I4 (last R4)					; 51-54		n 56

	zfmaddpd zmm4, zmm9, zmm29, zmm2	;; I5 + I6 * SQRTHALF (last I5)				; 52-55		n 61
	zfnmaddpd zmm9, zmm9, zmm29, zmm2	;; I5 - I6 * SQRTHALF (last I6)				; 52-55		n 62

	zfmaddpd zmm2, zmm1, zmm29, zmm17	;; I7 + R8 * SQRTHALF (last I7)				; 53-56		n 63
	zfnmaddpd zmm1, zmm1, zmm29, zmm17	;; I7 - R8 * SQRTHALF (last I8)				; 53-56		n 64
	vmovapd	zmm17, [screg+1*128+64]		;; cosine for R5/I5

	vmulpd	zmm19, zmm8, zmm27		;; B3 = I3 * cosine					; 54-57		n 58
	vmulpd	zmm20, zmm11, zmm26		;; B4 = I4 * cosine					; 54-57		n 58

	vmulpd	zmm25, zmm10, zmm25		;; A1 = R1 * cosine					; 55-58		n 59
	vmulpd	zmm28, zmm16, zmm28		;; A2 = R2 * cosine					; 55-58		n 59

	vmulpd	zmm27, zmm13, zmm27		;; A3 = R3 * cosine					; 56-59		n 60
	vmulpd	zmm26, zmm14, zmm26		;; A4 = R4 * cosine					; 56-59		n 60

	zfmaddpd zmm10, zmm10, zmm24, zmm12	;; B1 + R1 * sine (final I1)				; 57-60
	zfmaddpd zmm16, zmm16, zmm23, zmm18	;; B2 + R2 * sine (final I2)				; 57-60
	vmovapd	zmm12, [screg+5*128+64]		;; cosine for R6/I6

	zfmaddpd zmm13, zmm13, zmm22, zmm19	;; B3 + R3 * sine (final I3)				; 58-61
	zfmaddpd zmm14, zmm14, zmm21, zmm20	;; B4 + R4 * sine (final I4)				; 58-61
	vmovapd	zmm18, [screg+3*128+64]		;; cosine for R7/I7

	zfnmaddpd zmm3, zmm3, zmm24, zmm25	;; A1 - I1 * sine (final R1)				; 59-62
	zfnmaddpd zmm6, zmm6, zmm23, zmm28	;; A2 - I2 * sine (final R2)				; 59-62
	vmovapd	zmm19, [screg+7*128+64]		;; cosine for R8/I8

	zfnmaddpd zmm8, zmm8, zmm22, zmm27	;; A3 - I3 * sine (final R3)				; 60-63
	zfnmaddpd zmm11, zmm11, zmm21, zmm26	;; A4 - I4 * sine (final R4)				; 60-63
	vmovapd	zmm20, [screg+1*128]		;; sine for R5/I5

	vmulpd	zmm22, zmm7, zmm17		;; A5 = R5 * cosine					; 61-64		n 65
	vmulpd	zmm17, zmm4, zmm17		;; B5 = I5 * cosine					; 61-64		n 65
	vmovapd	zmm25, [screg+5*128]		;; sine for R6/I6
	zstore	[dstreg+64], zmm10		;; Save I1						; 61

	vmulpd	zmm27, zmm0, zmm12		;; A6 = R6 * cosine					; 62-65		n 66
	vmulpd	zmm12, zmm9, zmm12		;; B6 = I6 * cosine					; 62-65		n 66
	vmovapd	zmm24, [screg+3*128]		;; sine for R7/I7
	zstore	[dstreg+e1reg+64], zmm16	;; Save I2						; 61+1

	vmulpd	zmm21, zmm15, zmm18		;; A7 = R7 * cosine					; 63-66		n 67
	vmulpd	zmm18, zmm2, zmm18		;; B7 = I7 * cosine					; 63-66		n 67
	vmovapd	zmm28, [screg+7*128]		;; sine for R8/I8
	zstore	[dstreg+2*e1reg+64], zmm13	;; Save I3						; 62+1

	vmulpd	zmm26, zmm5, zmm19		;; A8 = R8 * cosine					; 64-67		n 68
	vmulpd	zmm19, zmm1, zmm19		;; B8 = I8 * cosine					; 64-67		n 68
	bump	screg, scinc
	zstore	[dstreg+e3reg+64], zmm14	;; Save I4						; 62+2

	zfnmaddpd zmm4, zmm4, zmm20, zmm22	;; A5 - I5 * sine (final R5)				; 65-68
	zfmaddpd zmm7, zmm7, zmm20, zmm17	;; B5 + R5 * sine (final I5)				; 65-68
	zstore	[dstreg], zmm3			;; Save R1						; 63+2

	zfnmaddpd zmm9, zmm9, zmm25, zmm27	;; A6 - I6 * sine (final R6)				; 66-69
	zfmaddpd zmm0, zmm0, zmm25, zmm12	;; B6 + R6 * sine (final I6)				; 66-69
	zstore	[dstreg+e1reg], zmm6		;; Save R2						; 63+3

	zfnmaddpd zmm2, zmm2, zmm24, zmm21	;; A7 - I7 * sine (final R7)				; 67-70
	zfmaddpd zmm15, zmm15, zmm24, zmm18	;; B7 + R7 * sine (final I7)				; 67-70
	zstore	[dstreg+2*e1reg], zmm8		;; Save R3						; 64+3

	zfnmaddpd zmm1, zmm1, zmm28, zmm26	;; A8 - I8 * sine (final R8)				; 68-71
	zfmaddpd zmm5, zmm5, zmm28, zmm19	;; B8 + R8 * sine (final I8)				; 68-71
	zstore	[dstreg+e3reg], zmm11		;; Save R4						; 64+4

	zstore	[dst4reg], zmm4			;; Save R5						; 69
	zstore	[dst4reg+64], zmm7		;; Save I5						; 69+1
	zstore	[dst4reg+e1reg], zmm9		;; Save R6						; 70+1
	zstore	[dst4reg+e1reg+64], zmm0	;; Save I6						; 70+2
	zstore	[dst4reg+2*e1reg], zmm2		;; Save R7						; 71+2
	zstore	[dst4reg+2*e1reg+64], zmm15	;; Save I7						; 71+3
	zstore	[dst4reg+e3reg], zmm1		;; Save R8						; 72+3
	zstore	[dst4reg+e3reg+64], zmm5	;; Save I8						; 72+4
	bump	dstreg, dstinc
	bump	dst4reg, dstinc
	ENDM

; Use registers for distances between input blocks.  This lets us share pass1 code.

zr8_rsc_wpn_sgreg_eight_complex_unfft8_preload MACRO
	use zr8_rsc_complex_only_preload or zr8_rsc_complex_and_real_preload
	ENDM

zr8_rsc_wpn_sgreg_eight_complex_unfft8 MACRO srcreg,srcinc,d1reg,d3reg,src4reg,dstreg,dstinc,e1,e2,e4,screg,scinc,maxrpt,L1pt,L1preg,L1p4reg
	vmovapd	zmm28, [screg+1*128+64]		;; cosine for R5/I5
	vmovapd zmm4, [src4reg]			;; Load R5
	vmulpd	zmm16, zmm4, zmm28		;; A5 = R5 * cosine					; 1-4		n 6
	vmovapd zmm12, [src4reg+64]		;; Load I5
	vmulpd	zmm17, zmm12, zmm28		;; B5 = I5 * cosine					; 2-5		n 7

	vmovapd	zmm28, [screg+5*128+64]		;; cosine for R6/I6
	vmovapd zmm5, [src4reg+d1reg]		;; Load R6
	vmulpd	zmm18, zmm5, zmm28		;; A6 = R6 * cosine					; 2-5		n 7
	vmovapd zmm13, [src4reg+d1reg+64]	;; Load I6
	vmulpd	zmm19, zmm13, zmm28		;; B6 = I6 * cosine					; 3-6		n 8

	vmovapd	zmm28, [screg+3*128+64]		;; cosine for R7/I7
	vmovapd zmm14, [src4reg+2*d1reg+64]	;; Load I7
	vmulpd	zmm21, zmm14, zmm28		;; B7 = I7 * cosine					; 3-6		n 8
	vmovapd zmm6, [src4reg+2*d1reg]		;; Load R7
	vmulpd	zmm20, zmm6, zmm28		;; A7 = R7 * cosine					; 4-7		n 9

	vmovapd	zmm28, [screg+7*128+64]		;; cosine for R8/I8
	vmovapd zmm15, [src4reg+d3reg+64]	;; Load I8
	vmulpd	zmm23, zmm15, zmm28		;; B8 = I8 * cosine					; 4-7		n 9
	vmovapd zmm7, [src4reg+d3reg]		;; Load R8	
	vmulpd	zmm22, zmm7, zmm28		;; A8 = R8 * cosine					; 5-8		n 10

	vmovapd	zmm28, [screg+0*128+64]		;; cosine for R1/I1
	vmovapd zmm0, [srcreg]			;; Load R1
	vmulpd	zmm24, zmm0, zmm28		;; A1 = R1 * cosine					; 5-8		n 13
	vmovapd zmm8, [srcreg+64]		;; Load I1
	vmulpd	zmm25, zmm8, zmm28		;; B1 = I1 * cosine					; 6-9		n 14

	vmovapd	zmm28, [screg+1*128]		;; sine for R5/I5
	zfmaddpd zmm16, zmm12, zmm28, zmm16	;; A5 = A5 + I5 * sine (first R5)			; 6-9		n 17
	zfnmaddpd zmm17, zmm4, zmm28, zmm17	;; B5 = B5 - R5 * sine (first I5)			; 7-10		n 18

	vmovapd	zmm28, [screg+5*128]		;; sine for R6/I6
	zfmaddpd zmm18, zmm13, zmm28, zmm18	;; A6 = A6 + I6 * sine (first R6)			; 7-10		n 17
	zfnmaddpd zmm19, zmm5, zmm28, zmm19	;; B6 = B6 - R6 * sine (first I6)			; 8-11		n 18

	vmovapd	zmm28, [screg+3*128]		;; sine for R7/I7
	zfnmaddpd zmm21, zmm6, zmm28, zmm21	;; B7 = B7 - R7 * sine (first I7)			; 8-11		n 19
	zfmaddpd zmm20, zmm14, zmm28, zmm20	;; A7 = A7 + I7 * sine (first R7)			; 9-12		n 20

	vmovapd	zmm28, [screg+7*128]		;; sine for R8/I8
	zfnmaddpd zmm23, zmm7, zmm28, zmm23	;; B8 = B8 - R8 * sine (first I8)			; 9-12		n 19
	zfmaddpd zmm22, zmm15, zmm28, zmm22	;; A8 = A8 + I8 * sine (first R8)			; 10-13		n 20

	vmovapd	zmm28, [screg+4*128+64]		;; cosine for R2/I2
	vmovapd zmm1, [srcreg+d1reg]		;; Load R2
	vmulpd	zmm13, zmm1, zmm28		;; A2 = R2 * cosine					; 10-13		n 14
	vmovapd zmm9, [srcreg+d1reg+64]		;; Load I2
	vmulpd	zmm5, zmm9, zmm28		;; B2 = I2 * cosine					; 11-14		n 15

	vmovapd	zmm28, [screg+2*128+64]		;; cosine for R3/I3
	vmovapd zmm2, [srcreg+2*d1reg]		;; Load R3
	vmulpd	zmm14, zmm2, zmm28		;; A3 = R3 * cosine					; 11-14		n 15
	vmovapd zmm10, [srcreg+2*d1reg+64]	;; Load I3
	vmulpd	zmm6, zmm10, zmm28		;; B3 = I3 * cosine					; 12-15		n 16

	vmovapd	zmm28, [screg+6*128+64]		;; cosine for R4/I4
	vmovapd zmm3, [srcreg+d3reg]		;; Load R4
	vmulpd	zmm15, zmm3, zmm28		;; A4 = R4 * cosine					; 12-15		n 16
	vmovapd zmm11, [srcreg+d3reg+64]	;; Load I4
	vmulpd	zmm7, zmm11, zmm28		;; B4 = I4 * cosine					; 13-16		n 17

	vmovapd	zmm28, [screg+0*128]		;; sine for R1/I1
	zfmaddpd zmm24, zmm8, zmm28, zmm24	;; A1 = A1 + I1 * sine (first R1)			; 13-16		n 21
	zfnmaddpd zmm25, zmm0, zmm28, zmm25	;; B1 = B1 - R1 * sine (first I1)			; 14-17		n 26

	vmovapd	zmm28, [screg+4*128]		;; sine for R2/I2
	zfmaddpd zmm13, zmm9, zmm28, zmm13	;; A2 = A2 + I2 * sine (first R2)			; 14-17		n 21
	zfnmaddpd zmm5, zmm1, zmm28, zmm5	;; B2 = B2 - R2 * sine (first I2)			; 15-18		n 26

	vmovapd	zmm28, [screg+2*128]		;; sine for R3/I3
	zfmaddpd zmm14, zmm10, zmm28, zmm14	;; A3 = A3 + I3 * sine (first R3)			; 15-18		n 22
	zfnmaddpd zmm6, zmm2, zmm28, zmm6	;; B3 = B3 - R3 * sine (first I3)			; 16-19		n 23

	vmovapd	zmm28, [screg+6*128]		;; sine for R4/I4
	zfmaddpd zmm15, zmm11, zmm28, zmm15	;; A4 = A4 + I4 * sine (first R4)			; 16-19		n 22
	zfnmaddpd zmm7, zmm3, zmm28, zmm7	;; B4 = B4 - R4 * sine (first I4)			; 17-20		n 23

	vsubpd	zmm4, zmm16, zmm18		;; R5 - R6 (new R6)					; 17-20		n 24
	vaddpd	zmm16, zmm16, zmm18		;; R5 + R6 (new R5)					; 18-21		n 31
	L1prefetch L1preg, L1pt

	vsubpd	zmm18, zmm17, zmm19		;; I5 - I6 (new I6)					; 18-21		n 24
	vaddpd	zmm17, zmm17, zmm19		;; I5 + I6 (new I5)					; 19-22		n 28
	L1prefetch L1preg+64, L1pt

	vsubpd	zmm19, zmm21, zmm23		;; I7 - I8 (new R8)					; 19-22		n 25
	vaddpd	zmm21, zmm21, zmm23		;; I7 + I8 (new I7)					; 20-23		n 28
	L1prefetch L1preg+d1reg, L1pt

	vsubpd	zmm23, zmm22, zmm20		;; R8 - R7 (new I8)					; 20-23		n 25
	vaddpd	zmm22, zmm22, zmm20		;; R8 + R7 (new R7)					; 21-24		n 31
	L1prefetch L1preg+d1reg+64, L1pt

	vaddpd	zmm20, zmm24, zmm13		;; R1 + R2 (new R1)					; 21-24		n 27
	vsubpd	zmm24, zmm24, zmm13		;; R1 - R2 (new R2)					; 22-25		n 29
	L1prefetch L1preg+2*d1reg, L1pt

	vaddpd	zmm13, zmm15, zmm14		;; R4 + R3 (new R3)					; 22-25		n 27
	vsubpd	zmm15, zmm15, zmm14		;; R4 - R3 (new I4)					; 23-26		n 39
	L1prefetch L1preg+2*d1reg+64, L1pt

	vsubpd	zmm14, zmm6, zmm7		;; I3 - I4 (new R4)					; 23-26		n 29
	vaddpd	zmm6, zmm6, zmm7		;; I3 + I4 (new I3)					; 24-27		n 37
	L1prefetch L1preg+d3reg, L1pt

	vsubpd	zmm7, zmm18, zmm4		;; I6 - R6 (new2 I6)					; 24-27		n 30
	vaddpd	zmm4, zmm4, zmm18		;; R6 + I6 (new2 R6)					; 25-28		n 32
	L1prefetch L1preg+d3reg+64, L1pt

	vsubpd	zmm18, zmm23, zmm19		;; I8 - R8 (new2 I8)					; 25-28		n 30
	vaddpd	zmm19, zmm19, zmm23		;; R8 + I8 (new2 R8)					; 26-29		n 32
	L1prefetch L1p4reg, L1pt

	vaddpd	zmm23, zmm25, zmm5		;; I1 + I2 (new I1)					; 26-29		n 37
	vsubpd	zmm25, zmm25, zmm5		;; I1 - I2 (new I2)					; 27-30		n 39
	L1prefetch L1p4reg+64, L1pt

	vsubpd	zmm5, zmm20, zmm13		;; R1 - R3 (newer R3)					; 27-30		n 33
	vaddpd	zmm20, zmm20, zmm13		;; R1 + R3 (newer R1)					; 28-31		n 35
	L1prefetch L1p4reg+d1reg, L1pt

	vsubpd	zmm13, zmm17, zmm21		;; I5 - I7 (newer R7)					; 28-31		n 33
	vaddpd	zmm17, zmm17, zmm21		;; I5 + I7 (newer I5)					; 29-32		n 51
	L1prefetch L1p4reg+d1reg+64, L1pt

	vsubpd	zmm21, zmm24, zmm14		;; R2 - R4 (newer R4)					; 29-32		n 34
	vaddpd	zmm24, zmm24, zmm14		;; R2 + R4 (newer R2)					; 30-33		n 36
	L1prefetch L1p4reg+2*d1reg, L1pt

	vsubpd	zmm14, zmm7, zmm18		;; I6 - I8 (newer R8/SQRTHALF)				; 30-33		n 34
	vaddpd	zmm7, zmm7, zmm18		;; I6 + I8 (newer I6/SQRTHALF)				; 31-34		n 52
	L1prefetch L1p4reg+2*d1reg+64, L1pt

	vaddpd	zmm18, zmm22, zmm16		;; R7 + R5 (newer R5)					; 31-34		n 35
	vsubpd	zmm22, zmm22, zmm16		;; R7 - R5 (newer I7)					; 32-35		n 49
	L1prefetch L1p4reg+d3reg, L1pt

	vaddpd	zmm16, zmm19, zmm4		;; R8 + R6 (newer R6/SQRTHALF)				; 32-35		n 36
	vsubpd	zmm19, zmm19, zmm4		;; R8 - R6 (newer I8/SQRTHALF)				; 33-36		n 50
	L1prefetch L1p4reg+d3reg+64, L1pt

	vaddpd	zmm4, zmm5, zmm13		;; R3 + R7 (final R3)					; 33-36		n 38
	vsubpd	zmm5, zmm5, zmm13		;; R3 - R7 (final R7)					; 34-37		n 42
	bump	srcreg, srcinc

	zfmaddpd zmm13, zmm14, zmm29, zmm21	;; R4 + R8 * SQRTHALF (final R4)			; 34-37		n 38
	zfnmaddpd zmm14, zmm14, zmm29, zmm21	;; R4 - R8 * SQRTHALF (final R8)			; 35-38		n 42
	bump	src4reg, srcinc

	vaddpd	zmm21, zmm20, zmm18		;; R1 + R5 (final R1)					; 35-38		n 40
	vsubpd	zmm20, zmm20, zmm18		;; R1 - R5 (final R5)					; 36-39		n 44
	bump	screg, scinc

	zfmaddpd zmm18, zmm16, zmm29, zmm24	;; R2 + R6 * SQRTHALF (final R2)			; 36-39		n 40
	zfnmaddpd zmm16, zmm16, zmm29, zmm24	;; R2 - R6 * SQRTHALF (final R6)			; 37-40		n 44
	bump	L1preg, srcinc
	vaddpd	zmm24, zmm23, zmm6		;; I1 + I3 (newer I1)					; 37-40		n 51

	zperm2pd zmm8, zmm31, zmm4, zmm13	;; r4_4 r3_4 r4_6 r3_6 r4_0 r3_0 r4_2 r3_2		; 38-40		n 41
	zperm2pd zmm4, zmm30, zmm4, zmm13	;; r4_5 r3_5 r4_7 r3_7 r4_1 r3_1 r4_3 r3_3		; 39-41		n 43

	vsubpd	zmm23, zmm23, zmm6		;; I1 - I3 (newer I3)					;  38-41	n 49
	vaddpd	zmm6, zmm25, zmm15		;; I2 + I4 (newer I2)					;  39-42	n 52
	bump	L1p4reg, srcinc

	vshufpd	zmm13, zmm21, zmm18, 00000000b	;; r2_6 r1_6 r2_4 r1_4 r2_2 r1_2 r2_0 r1_0		; 40		n 41
	vshufpd	zmm21, zmm21, zmm18, 11111111b	;; r2_7 r1_7 r2_5 r1_5 r2_3 r1_3 r2_1 r1_1		; 41		n 43

	vsubpd	zmm25, zmm25, zmm15		;; I2 - I4 (newer I4)					;  40-43	n 50
	vblendmpd zmm9{k7}, zmm8, zmm13		;; r4_4 r3_4 r2_4 r1_4 r4_0 r3_0 r2_0 r1_0		;  41		n 46

	zperm2pd zmm18, zmm31, zmm5, zmm14	;; r8_4 r7_4 r8_6 r7_6 r8_0 r7_0 r8_2 r7_2		; 42-44		n 45
	zperm2pd zmm5, zmm30, zmm5, zmm14	;; r8_5 r7_5 r8_7 r7_7 r8_1 r7_1 r8_3 r7_3		; 43-45		n 47

	vblendmpd zmm13{k7}, zmm13, zmm8	;; r2_6 r1_6 r4_6 r3_6 r2_2 r1_2 r4_2 r3_2		;  42		n 48
	vblendmpd zmm8{k7}, zmm4, zmm21		;; r4_5 r3_5 r2_5 r1_5 r4_1 r3_1 r2_1 r1_1		;  43		n 50

	vshufpd	zmm14, zmm20, zmm16, 00000000b	;; r6_6 r5_6 r6_4 r5_4 r6_2 r5_2 r6_0 r5_0		; 44		n 45
	vshufpd	zmm20, zmm20, zmm16, 11111111b	;; r6_7 r5_7 r6_5 r5_5 r6_3 r5_3 r6_1 r5_1		; 45		n 47

	vblendmpd zmm21{k7}, zmm21, zmm4	;; r2_7 r1_7 r4_7 r3_7 r2_3 r1_3 r4_3 r3_3		;  44		n 52
	vblendmpd zmm4{k7}, zmm18, zmm14	;; r8_4 r7_4 r6_4 r5_4 r8_0 r7_0 r6_0 r5_0		;  45		n 46

	vshuff64x2 zmm2, zmm9, zmm4, 01000100b	;; r8_0 r7_0 r6_0 r5_0 r4_0 r3_0 r2_0 r1_0		; 46-48
	vshuff64x2 zmm9, zmm9, zmm4, 11101110b	;; r8_4 r7_4 r6_4 r5_4 r4_4 r3_4 r2_4 r1_4		; 47-49

	vblendmpd zmm14{k7}, zmm14, zmm18	;; r6_6 r5_6 r8_6 r7_6 r6_2 r5_2 r8_2 r7_2		;  46		n 48
	vblendmpd zmm18{k7}, zmm5, zmm20	;; r8_5 r7_5 r6_5 r6_5 r8_1 r7_1 r6_1 r5_1		;  47		n 50

	vshuff64x2 zmm4, zmm13, zmm14, 00010001b;; r8_2 r7_2 r6_2 r5_2 r4_2 r3_2 r2_2 r1_2		; 48-50
	vshuff64x2 zmm13, zmm13, zmm14, 10111011b;;r8_6 r7_6 r6_6 r5_6 r4_6 r3_6 r2_6 r1_6		; 49-51

	vblendmpd zmm20{k7}, zmm20, zmm5	;; r6_7 r5_7 r8_7 r7_7 r6_3 r5_3 r8_3 r7_3		;  48		n 52
	vaddpd	zmm15, zmm23, zmm22		;; I3 + I7 (final I3)					;  49-52	n 54
	zstore	[dstreg], zmm2										; 49

	vshuff64x2 zmm14, zmm8, zmm18, 01000100b;; r8_1 r7_1 r6_1 r5_1 r4_1 r3_1 r2_1 r1_1		; 50-52
	vshuff64x2 zmm8, zmm8, zmm18, 11101110b ;; r8_5 r7_5 r6_5 r5_5 r4_5 r3_5 r2_5 r1_5		; 51-53

	zfmaddpd zmm10, zmm19, zmm29, zmm25	;; I4 + I8 * SQRTHALF (final I4)			;  50-53	n 54
	vaddpd	zmm12, zmm24, zmm17		;; I1 + I5 (final I1)					;  51-54	n 56
	zstore	[dstreg+e4], zmm9									; 50
	zstore	[dstreg+e2], zmm4									; 51

	vshuff64x2 zmm18, zmm21, zmm20, 00010001b;;r8_3 r7_3 r6_3 r5_3 r4_3 r3_3 r2_3 r1_3		; 52-54
	vshuff64x2 zmm21, zmm21, zmm20, 10111011b;;r8_7 r7_7 r6_7 r5_7 r4_7 r3_7 r2_7 r1_7		; 53-55

	zfmaddpd zmm0, zmm7, zmm29, zmm6	;; I2 + I6 * SQRTHALF (final I2)			;  52-55	n 56
	vsubpd	zmm23, zmm23, zmm22		;; I3 - I7 (final I7)					;  53-56	n 58
	zstore	[dstreg+e4+e2], zmm13									; 52
	zstore	[dstreg+e1], zmm14									; 53

	zperm2pd zmm20, zmm31, zmm15, zmm10	;; i4_4 i3_4 i4_6 i3_6 i4_0 i3_0 i4_2 i3_2		; 54-56		n 57
	zperm2pd zmm15, zmm30, zmm15, zmm10	;; i4_5 i3_5 i4_7 i3_7 i4_1 i3_1 i4_3 i3_3		; 55-57		n 59

	zfnmaddpd zmm19, zmm19, zmm29, zmm25	;; I4 - I8 * SQRTHALF (final I8)			;  54-57	n 58
	vsubpd	zmm24, zmm24, zmm17		;; I1 - I5 (final I5)					;  55-58	n 60
	zstore	[dstreg+e4+e1], zmm8									; 54
	zstore	[dstreg+e2+e1], zmm18									; 55

	vshufpd	zmm10, zmm12, zmm0, 00000000b	;; i2_6 i1_6 i2_4 i1_4 i2_2 i1_2 i2_0 i1_0		; 56		n 57
	vshufpd	zmm12, zmm12, zmm0, 11111111b	;; i2_7 i1_7 i2_5 i1_5 i2_3 i1_3 i2_1 i1_1		; 57		n 59

	zfnmaddpd zmm7, zmm7, zmm29, zmm6	;; I2 - I6 * SQRTHALF (final I6)			;  56-59	n 60
	vblendmpd zmm11{k7}, zmm20, zmm10	;; i4_4 i3_4 i2_4 i1_4 i4_0 i3_0 i2_0 i1_0		;  57		n 62
	zstore	[dstreg+e4+e2+e1], zmm21								; 56

	zperm2pd zmm0, zmm31, zmm23, zmm19	;; i8_4 i7_4 i8_6 i7_6 i8_0 i7_0 i8_2 i7_2		; 58-60		n 61
	zperm2pd zmm23, zmm30, zmm23, zmm19	;; i8_5 i7_5 i8_7 i7_7 i8_1 i7_1 i8_3 i7_3		; 59-61		n 62

	vblendmpd zmm10{k7}, zmm10, zmm20	;; i2_6 i1_6 i4_6 i3_6 i2_2 i1_2 i4_2 i3_2		;  58		n 64
	vblendmpd zmm20{k7}, zmm15, zmm12	;; i4_5 i3_5 i2_5 i1_5 i4_1 i3_1 i2_1 i1_1		;  59		n 66

	vshufpd	zmm19, zmm24, zmm7, 00000000b	;; i6_6 i5_6 i6_4 i5_4 i6_2 i5_2 i6_0 i5_0		; 60		n 61
	vshufpd	zmm24, zmm24, zmm7, 11111111b	;; i6_7 i5_7 i6_5 i5_5 i6_3 i5_3 i6_1 i5_1		; 61		n 63

	vblendmpd zmm12{k7}, zmm12, zmm15	;; i2_7 i1_7 i4_7 i3_7 i2_3 i1_3 i4_3 i3_3		;  60		n 68
	vblendmpd zmm15{k7}, zmm0, zmm19	;; i8_4 i7_4 i6_4 i5_4 i8_0 i7_0 i6_0 i5_0		;  61		n 62

	vshuff64x2 zmm25, zmm11, zmm15, 01000100b ;; i8_0 i7_0 i6_0 i5_0 i4_0 i3_0 i2_0 i1_0		; 62-64
	vshuff64x2 zmm11, zmm11, zmm15, 11101110b ;; i8_4 i7_4 i6_4 i5_4 i4_4 i3_4 i2_4 i1_4		; 63-65

	vblendmpd zmm19{k7}, zmm19, zmm0	;; i6_6 i5_6 i8_6 i7_6 i6_2 i5_2 i8_2 i7_2		;  62		n 64
	vblendmpd zmm0{k7}, zmm23, zmm24	;; i8_5 i7_5 i6_5 i5_5 i8_1 i7_1 i6_1 i5_1		;  63		n 66

	vshuff64x2 zmm15, zmm10, zmm19, 00010001b ;; i8_2 i7_2 i6_2 i5_2 i4_2 i3_2 i2_2 i1_2		; 64-66
	vshuff64x2 zmm10, zmm10, zmm19, 10111011b;; i8_6 i7_6 i6_6 i5_6 i4_6 i3_6 i2_6 i1_6		; 65-67

	vblendmpd zmm24{k7}, zmm24, zmm23	;; i6_7 i5_7 i8_7 i7_7 i6_3 i5_3 i8_3 i7_3		;  64		n 68
	zstore	[dstreg+64], zmm25									; 65

	vshuff64x2 zmm19, zmm20, zmm0, 01000100b ;; i8_1 i7_1 i6_1 i5_1 i4_1 i3_1 i2_1 i1_1		; 66-68
	zstore	[dstreg+e4+64], zmm11									; 66

	vshuff64x2 zmm20, zmm20, zmm0, 11101110b ;; i8_5 i7_5 i6_5 i5_5 i4_5 i3_5 i2_5 i1_5		; 67-69
	zstore	[dstreg+e2+64], zmm15									; 67

	vshuff64x2 zmm0, zmm12, zmm24, 00010001b ;; i8_3 i7_3 i6_3 i5_3 i4_3 i3_3 i2_3 i1_3		; 68-70
	zstore	[dstreg+e4+e2+64], zmm10								; 68

	vshuff64x2 zmm12, zmm12, zmm24, 10111011b;; i8_7 i7_7 i6_7 i5_7 i4_7 i3_7 i2_7 i1_7		; 69-71
	zstore	[dstreg+e1+64], zmm19									; 69

	zstore	[dstreg+e4+e1+64], zmm20								; 70
	zstore	[dstreg+e2+e1+64], zmm0									; 71
	zstore	[dstreg+e4+e2+e1+64], zmm12								; 72
	bump	dstreg, dstinc
	ENDM


;;
;; ************************************* sixteen-reals-fft variants ******************************************
;;

;; These macros operate on 16 reals doing 4 levels of the FFT, applying
;; the sin/cos multipliers afterwards.  The output is 2 reals and 7 complex numbers.

;; To calculate a 16-reals FFT, we calculate 16 complex values in a brute force way (using a shorthand notation):
;; r1 + r2 + ... + r16	*  w^0000000000...
;; r1 + r2 + ... + r16	*  w^0123456789A...
;; r1 + r2 + ... + r16	*  w^02468ACE....
;;    ...
;; r1 + r2 + ... + r16	*  w^...A987654321
;; Note that Hermetian symmetry means we won't need to calculate the last 8 complex values.
;;
;; The sin/cos values (w = 16th root of unity) are:
;; w^1 = .924 + .383i
;; w^2 = .707 + .707i
;; w^3 = .383 + .924i
;; w^4 = 0 + 1i
;; w^5 = -.383 + .924i
;; w^6 = -.707 + .707i
;; w^7 = -.924 + .383i
;; w^8 = -1
;;
;; Applying the sin/cos values above (and noting that combining r2 and r10, r3 and r11, etc. will simplify calculations):
;;R1 = (r1+r9)     +(r3+r11)     +(r7+r15) + (r5+r13)     +(r4+r12)     +(r6+r14)     +(r2+r10)     +(r8+r16)
;;R2 = (r1-r9) +.707(r3-r11) -.707(r7-r15)            +.383(r4-r12) -.383(r6-r14) +.924(r2-r10) -.924(r8-r16)
;;R3 = (r1+r9)                             - (r5+r13) -.707(r4+r12) -.707(r6+r14) +.707(r2+r10) +.707(r8+r16)
;;R4 = (r1-r9) -.707(r3-r11) +.707(r7-r15)            -.924(r4-r12) +.924(r6-r14) +.383(r2-r10) -.383(r8-r16)
;;R5 = (r1+r9)     -(r3+r11)     -(r7+r15) + (r5+r13)
;;R6 = (r1-r9) -.707(r3-r11) +.707(r7-r15)            +.924(r4-r12) -.924(r6-r14) -.383(r2-r10) +.383(r8-r16)
;;R7 = (r1+r9)                             - (r5+r13) +.707(r4+r12) +.707(r6+r14) -.707(r2+r10) -.707(r8+r16)
;;R8 = (r1-r9) +.707(r3-r11) -.707(r7-r15)            -.383(r4-r12) +.383(r6-r14) -.924(r2-r10) +.924(r8-r16)
;;R9 = (r1+r9)     +(r3+r11)     +(r7+r15) + (r5+r13)     -(r4+r12)     -(r6+r14)     -(r2+r10)     -(r8+r16)
;;I2 =         +.707(r3-r11) +.707(r7-r15) + (r5-r13) +.924(r4-r12) +.924(r6-r14) +.383(r2-r10) +.383(r8-r16)
;;I3 =             +(r3+r11)     -(r7+r15)            +.707(r4+r12) -.707(r6+r14) +.707(r2+r10) -.707(r8+r16)
;;I4 =         +.707(r3-r11) +.707(r7-r15) - (r5-r13) -.383(r4-r12) -.383(r6-r14) +.924(r2-r10) +.924(r8-r16)
;;I5 =                                                    -(r4+r12)     +(r6+r14)     +(r2+r10)     -(r8+r16)
;;I6 =         -.707(r3-r11) -.707(r7-r15) + (r5-r13) -.383(r4-r12) -.383(r6-r14) +.924(r2-r10) +.924(r8-r16)
;;I7 =             -(r3+r11)     +(r7+r15)            +.707(r4+r12) -.707(r6+r14) +.707(r2+r10) -.707(r8+r16)
;;I8 =         -.707(r3-r11) -.707(r7-r15) - (r5-r13) +.924(r4-r12) +.924(r6-r14) +.383(r2-r10) +.383(r8-r16)

;; Further simplification:
;;R1 = (r1+r9) + (r5+r13)     +((r3+r11)+(r7+r15))     +(((r2+r10)+(r8+r16))+((r4+r12)+(r6+r14)))
;;R9 = (r1+r9) + (r5+r13)     +((r3+r11)+(r7+r15))     -(((r2+r10)+(r8+r16))+((r4+r12)+(r6+r14)))
;;R5 = (r1+r9) + (r5+r13)     -((r3+r11)+(r7+r15))
;;R3 = (r1+r9) - (r5+r13)                          +.707(((r2+r10)+(r8+r16))-((r4+r12)+(r6+r14)))
;;R7 = (r1+r9) - (r5+r13)                          -.707(((r2+r10)+(r8+r16))-((r4+r12)+(r6+r14)))
;;R2 = (r1-r9) +.707((r3-r11)-(r7-r15))            +.924((r2-r10)-(r8-r16)) +.383((r4-r12)-(r6-r14))
;;R8 = (r1-r9) +.707((r3-r11)-(r7-r15))            -.924((r2-r10)-(r8-r16)) -.383((r4-r12)-(r6-r14))
;;R4 = (r1-r9) -.707((r3-r11)-(r7-r15))            +.383((r2-r10)-(r8-r16)) -.924((r4-r12)-(r6-r14))
;;R6 = (r1-r9) -.707((r3-r11)-(r7-r15))            -.383((r2-r10)-(r8-r16)) +.924((r4-r12)-(r6-r14))
;;I5 =                                                 +(((r2+r10)-(r8+r16))-((r4+r12)-(r6+r14)))
;;I3 =             +((r3+r11)-(r7+r15))            +.707(((r2+r10)-(r8+r16))+((r4+r12)-(r6+r14)))
;;I7 =             -((r3+r11)-(r7+r15))            +.707(((r2+r10)-(r8+r16))+((r4+r12)-(r6+r14)))
;;I2 =         +.707((r3-r11)+(r7-r15)) + (r5-r13) +.383((r2-r10)+(r8-r16)) +.924((r4-r12)+(r6-r14))
;;I8 =         -.707((r3-r11)+(r7-r15)) - (r5-r13) +.383((r2-r10)+(r8-r16)) +.924((r4-r12)+(r6-r14))
;;I4 =         +.707((r3-r11)+(r7-r15)) - (r5-r13) +.924((r2-r10)+(r8-r16)) -.383((r4-r12)+(r6-r14))
;;I6 =         -.707((r3-r11)+(r7-r15)) + (r5-r13) +.924((r2-r10)+(r8-r16)) -.383((r4-r12)+(r6-r14))

;; NOTE: unlike the AVX versions of this macro, we do not "back up" the last 2 reals by one level.
;; Thus, the next level 16-reals will get its inputs with r1+/-r9, r2+/-r10 already calculated.
;; ALSO NOTE: this macro does not output its results in bit-reversed order.


; Uses two sin-cos pointers, one for real table (w^1,w^3,w^5,w^7), one for complex table (w^2,w^4,w^6)
; Used in middle levels of second pass in a two-pass FFT
zr8_2sc_sixteen_reals_fft_preload MACRO
	zr8_16r_fft_cmn_preload
	ENDM

zr8_2sc_sixteen_reals_fft MACRO srcreg,srcinc,d1,d2,d4,screg1,scinc1,screg2,scinc2,maxrpt,L1pt,L1pd
	zr8_16r_fft_cmn srcreg,0,srcinc,d1,d2,d4,screg1,scinc1,screg2,scinc2,maxrpt,L1pt,L1pd
	ENDM

; Offsets source by rbx
zr8f_2sc_sixteen_reals_fft_preload MACRO
	zr8_16r_fft_cmn_preload
	ENDM

zr8f_2sc_sixteen_reals_fft MACRO srcreg,srcinc,d1,d2,d4,screg1,scinc1,screg2,scinc2,maxrpt,L1pt,L1pd
	zr8_16r_fft_cmn srcreg,rbx,srcinc,d1,d2,d4,screg1,scinc1,screg2,scinc2,maxrpt,L1pt,L1pd
	ENDM

; Combined sin/cos data
; The real sin/cos values are after the complex sin/cos values.
; Used in middle levels of first pass in a two-pass FFT
zr8_csc_sixteen_reals_fft_preload MACRO
	zr8_16r_fft_cmn_preload
	ENDM

zr8_csc_sixteen_reals_fft MACRO srcreg,srcinc,d1,d2,d4,screg,scinc,maxrpt,L1pt,L1pd
	zr8_16r_fft_cmn srcreg,0,srcinc,d1,d2,d4,screg+4*128,0,screg,scinc,maxrpt,L1pt,L1pd
	ENDM

zr8_16r_fft_cmn_preload MACRO
	vbroadcastsd zmm31, ZMM_SQRTHALF
	vbroadcastsd zmm30, ZMM_P924_P383
	vbroadcastsd zmm29, ZMM_P383
	ENDM

zr8_16r_fft_cmn MACRO srcreg,srcoff,srcinc,d1,d2,d4,screg1,scinc1,screg2,scinc2,maxrpt,L1pt,L1pd
	vmovapd	zmm1, [srcreg+srcoff+d1]	;; r2+r10
	vmovapd	zmm3, [srcreg+srcoff+d4+d2+d1]	;; r8+r16
	vaddpd	zmm0, zmm1, zmm3		;; r2++ = (r2+r10)+(r8+r16)				; 1-4		n 6
	vsubpd	zmm1, zmm1, zmm3		;; r2+- = (r2+r10)-(r8+r16)				; 1-4		n 7

	vmovapd	zmm5, [srcreg+srcoff+d2+d1]	;; r4+r12
	vmovapd	zmm7, [srcreg+srcoff+d4+d1]	;; r6+r14
	vaddpd	zmm3, zmm5, zmm7		;; r4++ = (r4+r12)+(r6+r14)				; 2-5		n 6
	vsubpd	zmm5, zmm5, zmm7		;; r4+- = (r4+r12)-(r6+r14)				; 2-5		n 7

	vmovapd	zmm13, [srcreg+srcoff]		;; r1+r9
	vmovapd	zmm15, [srcreg+srcoff+d4]	;; r5+r13
	vaddpd	zmm7, zmm13, zmm15		;; r1++ = (r1+r9)+(r5+r13)				; 3-6		n 8
	vsubpd	zmm13, zmm13, zmm15		;; r1+- = (r1+r9)-(r5+r13)				; 3-6		n 11

	vmovapd	zmm9, [srcreg+srcoff+d2]	;; r3+r11
	vmovapd	zmm11, [srcreg+srcoff+d4+d2]	;; r7+r15
	vaddpd	zmm15, zmm9, zmm11		;; r3++ = (r3+r11)+(r7+r15)				; 4-7		n 8
	vsubpd	zmm9, zmm9, zmm11		;; r3+- = (r3+r11)-(r7+r15)				; 4-7		n 12

	vmovapd	zmm10, [srcreg+srcoff+d2+64]	;; r3-r11
	vmovapd	zmm12, [srcreg+srcoff+d4+d2+64]	;; r7-r15
	vaddpd	zmm11, zmm10, zmm12		;; r3-+ = (r3-r11)+(r7-r15)				; 5-8		n 16
	vsubpd	zmm10, zmm10, zmm12		;; r3-- = (r3-r11)-(r7-r15)				; 5-8		n 14

	vmovapd	zmm2, [srcreg+srcoff+d1+64]	;; r2-r10
	vaddpd	zmm12, zmm0, zmm3		;; r2+++ = (r2++) + (r4++)				; 6-9		n 13
	vsubpd	zmm0, zmm0, zmm3		;; r2++- = (r2++) - (r4++)				; 6-9		n 11

	vmovapd	zmm4, [srcreg+srcoff+d4+d2+d1+64] ;; r8-r16
	vaddpd	zmm3, zmm1, zmm5		;; r2+-+ = (r2+-) + (r4+-)				; 7-10		n 12
	vsubpd	zmm1, zmm1, zmm5		;; r2+-- = (r2+-) - (r4+-)	(final I5)		; 7-10		n 18

	vmovapd	zmm6, [srcreg+srcoff+d2+d1+64]	;; r4-r12
	vmovapd	zmm8, [srcreg+srcoff+d4+d1+64]	;; r6-r14
	vaddpd	zmm5, zmm7, zmm15		;; r1+++ = (r1++) + (r3++)				; 8-11		n 13
	vsubpd	zmm7, zmm7, zmm15		;; r1++- = (r1++) - (r3++)	(final R5)		; 8-11		n 18

	vmovapd	zmm14, [srcreg+srcoff+64]	;; r1-r9
	vmovapd	zmm16, [srcreg+srcoff+d4+64]	;; r5-r13
	vaddpd	zmm15, zmm2, zmm4		;; r2-+ = (r2-r10)+(r8-r16)				; 9-12		n 17
	vsubpd	zmm2, zmm2, zmm4		;; r2-- = (r2-r10)-(r8-r16)				; 9-12		n 15

	vmovapd	zmm28, [screg2+1*128+64]	;; cosine/sine for R5/I5 (w^4 = complex w^2)
	vaddpd	zmm4, zmm6, zmm8		;; r4-+ = (r4-r12)+(r6-r14)				; 10-13		n 17
	vsubpd	zmm6, zmm6, zmm8		;; r4-- = (r4-r12)-(r6-r14)				; 10-13		n 15

	vmovapd	zmm27, [screg2+0*128+64]	;; cosine/sine for R3/I3 (w^2 = complex w^1)
	vmovapd	zmm26, [screg2+2*128+64]	;; cosine/sine for R7/I7 (w^6 = complex w^3)
	zfmaddpd zmm8, zmm0, zmm31, zmm13	;; R3 = (r1+-) + .707(r2++-)				; 11-14		n 19
	zfnmaddpd zmm0, zmm0, zmm31, zmm13	;; R7 = (r1+-) - .707(r2++-)				; 11-14		n 20

	vmovapd	zmm25, [screg2+1*128]		;; sine for R5/I5
	zfmaddpd zmm13, zmm3, zmm31, zmm9	;; I3 = .707*(r2+-+) + (r3+-)				; 12-15		n 19
	zfmsubpd zmm3, zmm3, zmm31, zmm9	;; I7 = .707*(r2+-+) - (r3+-)				; 12-15		n 20

	vmovapd	zmm24, [screg2+0*128]		;; sine for R3/I3
	vaddpd	zmm9, zmm5, zmm12		;; R1 = (r1+++) + (r2+++)				; 13-16
	vsubpd	zmm5, zmm5, zmm12		;; R9 = (r1+++) - (r2+++)				; 13-16

	vmovapd	zmm23, [screg2+2*128]		;; sine for R7/I7
	vmovapd	zmm22, [screg1+0*128+64]	;; cosine/sine for R2/I2 (w^1)
	zfmaddpd zmm12, zmm10, zmm31, zmm14	;; r2o = (r1-r9) + .707*(r3--)				; 14-17		n 21
	zfnmaddpd zmm10, zmm10, zmm31, zmm14	;; r4o = (r1-r9) - .707*(r3--)				; 14-17		n 22

	vmovapd	zmm21, [screg1+3*128+64]	;; cosine/sine for R8/I8 (w^7)
	zfmaddpd zmm14, zmm2, zmm30, zmm6	;; r2e/.383 = .924/.383(r2--) + (r4--)			; 15-18		n 21
	zfnmaddpd zmm6, zmm6, zmm30, zmm2	;; r4e/.383 = (r2--) - .924/.383(r4--)			; 15-18		n 22

	vmovapd	zmm20, [screg1+1*128+64]	;; cosine/sine for R4/I4 (w^3)
	vmovapd	zmm19, [screg1+2*128+64]	;; cosine/sine for R6/I6 (w^5)
	zfmaddpd zmm2, zmm11, zmm31, zmm16	;; i2o = .707(r3-+) + (r5-r13)				; 16-19		n 23
	zfmsubpd zmm11, zmm11, zmm31, zmm16	;; i4o = .707(r3-+) - (r5-r13)				; 16-19		n 24

	vmovapd	zmm18, [screg1+0*128]		;; sine for R2/I2
	vmovapd	zmm17, [screg1+3*128]		;; sine for R8/I8
	zfmaddpd zmm16, zmm4, zmm30, zmm15	;; i2e/.383 = (r2-+) + .924/.383(r4-+)			; 17-20		n 23
	zfmsubpd zmm15, zmm15, zmm30, zmm4	;; i4e/.383 = .924/.383(r2-+) - (r4-+)			; 17-20		n 24
	zstore	[srcreg], zmm9			;; Store R1						; 17

	vmovapd	zmm9, [screg1+1*128]		;; sine for R4/I4
	zfmsubpd zmm4, zmm7, zmm28, zmm1	;; A5 = R5 * cosine/sine - I5				; 18-21		n 25
	zfmaddpd zmm1, zmm1, zmm28, zmm7	;; B5 = I5 * cosine/sine + R5				; 18-21		n 25
	zstore	[srcreg+64], zmm5		;; Store R9						; 17+1

	vmovapd zmm5, [screg1+2*128]		;; sine for R6/I6
	zfmsubpd zmm7, zmm8, zmm27, zmm13	;; A3 = R3 * cosine/sine - I3				; 19-22		n 26
	zfmaddpd zmm13, zmm13, zmm27, zmm8	;; B3 = I3 * cosine/sine + R3				; 19-22		n 26
	bump	screg1, scinc1

	L1prefetchw srcreg+L1pd, L1pt
	zfmsubpd zmm8, zmm0, zmm26, zmm3	;; A7 = R7 * cosine/sine - I7				; 20-23		n 27
	zfmaddpd zmm3, zmm3, zmm26, zmm0	;; B7 = I7 * cosine/sine + R7				; 20-23		n 27
	bump	screg2, scinc2

	L1prefetchw srcreg+64+L1pd, L1pt
	zfmaddpd zmm0, zmm14, zmm29, zmm12	;; R2 = r2o + .383*r2e					; 21-24		n 
	zfnmaddpd zmm14, zmm14, zmm29, zmm12	;; R8 = r2o - .383*r2e					; 21-24		n 

	L1prefetchw srcreg+d1+L1pd, L1pt
	zfmaddpd zmm12, zmm6, zmm29, zmm10	;; R4 = r4o + .383*r4e					; 22-25		n 
	zfnmaddpd zmm6, zmm6, zmm29, zmm10	;; R6 = r4o - .383*r4e					; 22-25		n 

	L1prefetchw srcreg+d1+64+L1pd, L1pt
	zfmaddpd zmm10, zmm16, zmm29, zmm2	;; I2 = .383*i2e + i2o					; 23-26		n 34
	zfmsubpd zmm16, zmm16, zmm29, zmm2	;; I8 = .383*i2e - i2o					; 23-26		n 33

	L1prefetchw srcreg+d2+L1pd, L1pt
	zfmaddpd zmm2, zmm15, zmm29, zmm11	;; I4 = .383*i4e + i4o					; 24-27		n 38
	zfmsubpd zmm15, zmm15, zmm29, zmm11	;; I6 = .383*i4e - i4o					; 24-27		n 36

	L1prefetchw srcreg+d2+64+L1pd, L1pt
	vmulpd	zmm4, zmm4, zmm25		;; A5 = A5 * sine (final R5)				; 25-28
	vmulpd	zmm1, zmm1, zmm25		;; B5 = B5 * sine (final I5)				; 25-28

	L1prefetchw srcreg+d2+d1+L1pd, L1pt
	vmulpd	zmm7, zmm7, zmm24		;; A3 = A3 * sine (final R3)				; 26-29
	vmulpd	zmm13, zmm13, zmm24		;; B3 = B3 * sine (final I3)				; 26-29

	L1prefetchw srcreg+d2+d1+64+L1pd, L1pt
	vmulpd	zmm8, zmm8, zmm23		;; A7 = A7 * sine (final R7)				; 27-30
	vmulpd	zmm3, zmm3, zmm23		;; B7 = B7 * sine (final I7)				; 27-30

	L1prefetchw srcreg+d4+L1pd, L1pt
	zfmsubpd zmm11, zmm0, zmm22, zmm10	;; A2 = R2 * cosine/sine - I2				; 28-31		n 32
	zfmaddpd zmm10, zmm10, zmm22, zmm0	;; B2 = I2 * cosine/sine + R2				; 28-31

	L1prefetchw srcreg+d4+64+L1pd, L1pt
	zfmsubpd zmm0, zmm14, zmm21, zmm16	;; A8 = R8 * cosine/sine - I8				; 29-32
	zfmaddpd zmm16, zmm16, zmm21, zmm14	;; B8 = I8 * cosine/sine + R8				; 29-32
	zstore	[srcreg+d4], zmm4		;; Store R5						; 29

	L1prefetchw srcreg+d4+d1+L1pd, L1pt
	zfmsubpd zmm14, zmm12, zmm20, zmm2	;; A4 = R4 * cosine/sine - I4				; 30-33
	zfmaddpd zmm2, zmm2, zmm20, zmm12	;; B4 = I4 * cosine/sine + R4				; 30-33
	zstore	[srcreg+d4+64], zmm1		;; Store I5						; 29+1

	L1prefetchw srcreg+d4+d1+64+L1pd, L1pt
	zfmsubpd zmm12, zmm6, zmm19, zmm15	;; A6 = R6 * cosine/sine - I6				; 31-34
	zfmaddpd zmm15, zmm15, zmm19, zmm6	;; B6 = I6 * cosine/sine + R6				; 31-34
	zstore	[srcreg+d2], zmm7		;; Store R3						; 30+1

	L1prefetchw srcreg+d4+d2+L1pd, L1pt
	vmulpd	zmm11, zmm11, zmm18		;; A2 = A2 * sine (final R2)				; 32-35
	vmulpd	zmm10, zmm10, zmm18		;; B2 = B2 * sine (final I2)				; 32-35
	zstore	[srcreg+d2+64], zmm13		;; Store I3						; 30+2

	L1prefetchw srcreg+d4+d2+64+L1pd, L1pt
	vmulpd	zmm0, zmm0, zmm17		;; A8 = A8 * sine (final R8)				; 33-36
	vmulpd	zmm16, zmm16, zmm17		;; B8 = B8 * sine (final I8)				; 33-36
	zstore	[srcreg+d4+d2], zmm8		;; Store R7						; 31+2

	L1prefetchw srcreg+d4+d2+d1+L1pd, L1pt
	vmulpd	zmm14, zmm14, zmm9		;; A4 = A4 * sine (final R4)				; 34-37
	vmulpd	zmm2, zmm2, zmm9		;; B4 = B4 * sine (final I4)				; 34-37
	zstore	[srcreg+d4+d2+64], zmm3		;; Store I7						; 31+3

	L1prefetchw srcreg+d4+d2+d1+64+L1pd, L1pt
	vmulpd	zmm12, zmm12, zmm5		;; A6 = A6 * sine (final R6)				; 35-38
	vmulpd	zmm15, zmm15, zmm5		;; B6 = B6 * sine (final I6)				; 35-38

	zstore	[srcreg+d1], zmm11		;; Store R2						; 36
	zstore	[srcreg+d1+64], zmm10		;; Store I2						; 36+1
	zstore	[srcreg+d4+d2+d1], zmm0		;; Store R8						; 37+1
	zstore	[srcreg+d4+d2+d1+64], zmm16	;; Store I8						; 37+2
	zstore	[srcreg+d2+d1], zmm14		;; Store R4						; 38+2
	zstore	[srcreg+d2+d1+64], zmm2		;; Store I4						; 38+3
	zstore	[srcreg+d4+d1], zmm12		;; Store R6						; 39+3
	zstore	[srcreg+d4+d1+64], zmm15	;; Store I6						; 39+4
	bump	srcreg, srcinc
	ENDM




;; Used in first levels of pass 1 in a two-pass FFT
;; This is the only 16-reals macro that must do the initial plus/minus on distance 8 input arguments
zr8_wpn_sixteen_reals_first_fft_preload MACRO
	vbroadcastsd zmm31, ZMM_SQRTHALF
	vbroadcastsd zmm30, ZMM_P924_P383
	vbroadcastsd zmm29, ZMM_P383
	vbroadcastsd zmm28, ZMM_ONE_OVER_B
	ENDM

zr8_wpn_sixteen_reals_first_fft MACRO srcreg,srcinc,d1,d2,d4,screg,scinc,maskreg,maskinc,grpreg,grpinc,maxrpt,L1pt,L1pd
	mov	bl, [maskreg+0*1]		;; Load index into compressed fudges
	mov	r14, [r13+0*8]			;; Load the xor mask
	xor	r14, [r12+rbx*8]		;; Apply the xor mask to 64 bits of fudge flags

	kmovw	k1, r14d			;; Load R1 and R9 fudge factor mask			; 1
	kshiftrw k2, k1, 8			;; R9's fudge						; 2
	vmovapd	zmm14, [srcreg]			;; R1
	vmulpd	zmm14{k1}, zmm14, zmm28		;; apply fudges to R1					; 2-5		n 17
	vmovapd	zmm20, [srcreg+64]		;; R9
	vmulpd	zmm20{k2}, zmm20, zmm28		;; apply fudges to R9					; 3-6		n 22

	shr	r14, 16				;; Next 16 bits of fudge flags
	kmovw	k1, r14d			;; Load R2 and R10 fudge factor mask			; 3
	kshiftrw k2, k1, 8			;; R10's fudge						; 4
	vmovapd	zmm2, [srcreg+d1]		;; R2
	vmulpd	zmm2{k1}, zmm2, zmm28		;; apply fudges to R2					; 4-7		n 19
	vmovapd	zmm21, [srcreg+d1+64]		;; R10
	vmulpd	zmm21{k2}, zmm21, zmm28		;; apply fudges to R10					; 5-8		n 19

	shr	r14, 16				;; Next 16 bits of fudge flags
	kmovw	k1, r14d			;; Load R3 and R11 fudge factor mask			; 5
	kshiftrw k2, k1, 8			;; R11's fudge						; 6
	vmovapd	zmm10, [srcreg+d2]		;; R3
	vmulpd	zmm10{k1}, zmm10, zmm28		;; apply fudges to R3					; 6-9		n 26
	vmovapd	zmm22, [srcreg+d2+64]		;; R11
	vmulpd	zmm22{k2}, zmm22, zmm28		;; apply fudges to R11					; 7-10		n 26

	shr	r14, 16				;; Next 16 bits of fudge flags
	kmovw	k1, r14d			;; Load R4 and R12 fudge factor mask			; 7
	kshiftrw k2, k1, 8			;; R12's fudge						; 8
	vmovapd	zmm6, [srcreg+d2+d1]		;; R4
	vmulpd	zmm6{k1}, zmm6, zmm28		;; apply fudges to R4					; 8-11		n 21
	vmovapd	zmm23, [srcreg+d2+d1+64]	;; R12
	vmulpd	zmm23{k2}, zmm23, zmm28		;; apply fudges to R12					; 9-12		n 21

	mov	bl, [maskreg+1*1]		;; Load index into compressed fudges
	mov	r14, [r13+1*8]			;; Load the xor mask
	xor	r14, [r12+rbx*8]		;; Apply the xor mask to 64 bits of fudge flags

	kmovw	k1, r14d			;; Load R5 and R13 fudge factor mask			; 9
	kshiftrw k2, k1, 8			;; R13's fudge						; 10
	vmovapd	zmm16, [srcreg+d4]		;; R5
	vmulpd	zmm16{k1}, zmm16, zmm28		;; apply fudges to R5					; 10-13		n 18
	vmovapd	zmm24, [srcreg+d4+64]		;; R13
	vmulpd	zmm24{k2}, zmm24, zmm28		;; apply fudges to R13					; 11-14		n 25

	shr	r14, 16				;; Next 16 bits of fudge flags
	kmovw	k1, r14d			;; Load R6 and R14 fudge factor mask			; 11
	kshiftrw k2, k1, 8			;; R14's fudge						; 12
	vmovapd	zmm8, [srcreg+d4+d1]		;; R6
	vmulpd	zmm8{k1}, zmm8, zmm28		;; apply fudges to R6					; 12-15		n 18
	vmovapd	zmm25, [srcreg+d4+d1+64]	;; R14
	vmulpd	zmm25{k2}, zmm25, zmm28		;; apply fudges to R14					; 13-16		n 23

	shr	r14, 16				;; Next 16 bits of fudge flags
	kmovw	k1, r14d			;; Load R7 and R15 fudge factor mask			; 13
	kshiftrw k2, k1, 8			;; R15's fudge						; 14
	vmovapd	zmm12, [srcreg+d4+d2]		;; R7
	vmulpd	zmm12{k1}, zmm12, zmm28		;; apply fudges to R7					; 14-17		n 20
	vmovapd	zmm26, [srcreg+d4+d2+64]	;; R15
	vmulpd	zmm26{k2}, zmm26, zmm28		;; apply fudges to R15					; 15-18		n 27

	shr	r14, 16				;; Next 16 bits of fudge flags
	kmovw	k1, r14d			;; Load R8 and R16 fudge factor mask			; 15
	kshiftrw k2, k1, 8			;; R16's fudge						; 16
	vmovapd	zmm4, [srcreg+d4+d2+d1]		;; R8
	vmulpd	zmm4{k1}, zmm4, zmm28		;; apply fudges to R8					; 16-19		n 20
	vmovapd	zmm27, [srcreg+d4+d2+d1+64]	;; R16
	vmulpd	zmm27{k2}, zmm27, zmm28		;; apply fudges to R16					; 17-20		n 24

	bump	maskreg, maskinc

	vmulpd	zmm14, zmm14, [grpreg+0*64]	;; apply group multiplier to R1				; 17-20		n 22
	vmulpd	zmm8, zmm8, [grpreg+10*64]	;; apply group multiplier to R6				; 18-21		n 23
	vmulpd	zmm16, zmm16, [grpreg+8*64]	;; apply group multiplier to R5				; 18-21		n 25

	vmovapd	zmm0, [grpreg+3*64]		;; grpmult10 / grpmult2
	zfmaddpd zmm1, zmm0, zmm21, zmm2	;; r2+r10*grpmult10_over_2				; 19-22		n 28
	zfnmaddpd zmm2, zmm0, zmm21, zmm2	;; r2-r10*grpmult10_over_2				; 19-22		n 36

	vmulpd	zmm4, zmm4, [grpreg+14*64]	;; apply group multiplier to R8				; 20-23		n 24
	vmulpd	zmm12, zmm12, [grpreg+12*64]	;; apply group multiplier to R7				; 20-23		n 27

	vmovapd	zmm0, [grpreg+7*64]		;; grpmult12 / grpmult4
	zfmaddpd zmm5, zmm0, zmm23, zmm6	;; r4+r12*grpmult12_over_4				; 21-24		n 29
	zfnmaddpd zmm6, zmm0, zmm23, zmm6	;; r4-r12*grpmult12_over_4				; 21-24		n 37

	vmovapd	zmm0, [grpreg+1*64]		;; grpmult9
	zfmaddpd zmm13, zmm0, zmm20, zmm14	;; r1+r9*grpmult9					; 22-25		n 30
	zfnmaddpd zmm14, zmm0, zmm20, zmm14	;; r1-r9*grpmult9					; 22-25		n 41

	vmovapd	zmm0, [grpreg+11*64]		;; grpmult14
	zfmaddpd zmm7, zmm0, zmm25, zmm8	;; r6+r14*grpmult14					; 23-26		n 29
	zfnmaddpd zmm8, zmm0, zmm25, zmm8	;; r6-r14*grpmult14					; 23-26		n 37

	vmovapd	zmm0, [grpreg+15*64]		;; grpmult16
	zfmaddpd zmm3, zmm0, zmm27, zmm4	;; r8+r16*grpmult16					; 24-27		n 28
	zfnmaddpd zmm4, zmm0, zmm27, zmm4	;; r8-r16*grpmult16					; 24-27		n 36

	vmovapd	zmm0, [grpreg+9*64]		;; grpmult13
	zfmaddpd zmm15, zmm0, zmm24, zmm16	;; r5+r13*grpmult13					; 25-28		n 30
	zfnmaddpd zmm16, zmm0, zmm24, zmm16	;; r5-r13*grpmult13					; 25-28		n 43

	vmovapd	zmm0, [grpreg+5*64]		;; grpmult11 / grpmult3
	zfmaddpd zmm9, zmm0, zmm22, zmm10	;; r3+r11*grpmult11_over_3				; 26-29		n 31
	zfnmaddpd zmm10, zmm0, zmm22, zmm10	;; r3-r11*grpmult11_over_3				; 26-29		n 32

	vmovapd	zmm0, [grpreg+13*64]		;; grpmult15
	zfmaddpd zmm11, zmm0, zmm26, zmm12	;; r7+r15*grpmult15					; 27-30		n 31
	zfnmaddpd zmm12, zmm0, zmm26, zmm12	;; r7-r15*grpmult15					; 27-30		n 32

	vmovapd	zmm21, [grpreg+2*64]		;; group multiplier for R2
	zfmaddpd zmm0, zmm1, zmm21, zmm3	;; r2++ = (r2+r10)*grpmult2+(r8+r16)			; 28-31		n 33
	zfmsubpd zmm1, zmm1, zmm21, zmm3	;; r2+- = (r2+r10)*grpmult2-(r8+r16)			; 28-31		n 34

	vmovapd	zmm23, [grpreg+6*64]		;; group multiplier for R4
	zfmaddpd zmm3, zmm5, zmm23, zmm7	;; r4++ = (r4+r12)*grpmult4+(r6+r14)			; 29-32		n 33
	zfmsubpd zmm5, zmm5, zmm23, zmm7	;; r4+- = (r4+r12)*grpmult4-(r6+r14)			; 29-32		n 34

	vmovapd	zmm22, [grpreg+4*64]		;; group multiplier for R3
	vaddpd	zmm7, zmm13, zmm15		;; r1++ = (r1+r9)+(r5+r13)				; 30-33		n 35
	vsubpd	zmm13, zmm13, zmm15		;; r1+- = (r1+r9)-(r5+r13)				; 30-33		n 38

	L1prefetchw srcreg+L1pd, L1pt
	zfmaddpd zmm15, zmm9, zmm22, zmm11	;; r3++ = (r3+r11)*grpmult3+(r7+r15)			; 31-34		n 35
	zfmsubpd zmm9, zmm9, zmm22, zmm11	;; r3+- = (r3+r11)*grpmult3-(r7+r15)			; 31-34		n 39
	L1prefetchw srcreg+64+L1pd, L1pt
	zfmaddpd zmm11, zmm10, zmm22, zmm12	;; r3-+ = (r3-r11)*grpmult3+(r7-r15)			; 32-35		n 43
	zfmsubpd zmm10, zmm10, zmm22, zmm12	;; r3-- = (r3-r11)*grpmult3-(r7-r15)			; 32-35		n 41

	bump	grpreg, grpinc

	L1prefetchw srcreg+d1+L1pd, L1pt
	vaddpd	zmm12, zmm0, zmm3		;; r2+++ = (r2++) + (r4++)				; 33-36		n 40
	vsubpd	zmm0, zmm0, zmm3		;; r2++- = (r2++) - (r4++)				; 33-36		n 38
	L1prefetchw srcreg+d1+64+L1pd, L1pt
	vaddpd	zmm3, zmm1, zmm5		;; r2+-+ = (r2+-) + (r4+-)				; 34-37		n 39
	vsubpd	zmm1, zmm1, zmm5		;; r2+-- = (r2+-) - (r4+-)	(final I5)		; 34-37		n 45
	L1prefetchw srcreg+d2+L1pd, L1pt
	vaddpd	zmm5, zmm7, zmm15		;; r1+++ = (r1++) + (r3++)				; 35-38		n 40
	vsubpd	zmm7, zmm7, zmm15		;; r1++- = (r1++) - (r3++)	(final R5)		; 35-38		n 45

	L1prefetchw srcreg+d2+64+L1pd, L1pt
	zfmaddpd zmm15, zmm2, zmm21, zmm4	;; r2-+ = (r2-r10)*grpmult2+(r8-r16)			; 36-39		n 44
	zfmsubpd zmm2, zmm2, zmm21, zmm4	;; r2-- = (r2-r10)*grpmult2-(r8-r16)			; 36-39		n 42
	L1prefetchw srcreg+d2+d1+L1pd, L1pt
	zfmaddpd zmm4, zmm6, zmm23, zmm8	;; r4-+ = (r4-r12)*grpmult4+(r6-r14)			; 37-40		n 44
	zfmsubpd zmm6, zmm6, zmm23, zmm8	;; r4-- = (r4-r12)*grpmult4-(r6-r14)			; 37-40		n 42

	L1prefetchw srcreg+d2+d1+64+L1pd, L1pt
	zfmaddpd zmm8, zmm0, zmm31, zmm13	;; R3 = (r1+-) + .707(r2++-)				; 38-41		n 46
	zfnmaddpd zmm0, zmm0, zmm31, zmm13	;; R7 = (r1+-) - .707(r2++-)				; 38-41		n 47

	L1prefetchw srcreg+d4+L1pd, L1pt
	zfmaddpd zmm13, zmm3, zmm31, zmm9	;; I3 = .707*(r2+-+) + (r3+-)				; 39-42		n 46
	zfmsubpd zmm3, zmm3, zmm31, zmm9	;; I7 = .707*(r2+-+) - (r3+-)				; 39-42		n 47

	L1prefetchw srcreg+d4+64+L1pd, L1pt
	vaddpd	zmm9, zmm5, zmm12		;; R1 = (r1+++) + (r2+++)				; 40-43
	vsubpd	zmm5, zmm5, zmm12		;; R9 = (r1+++) - (r2+++)				; 40-43

	L1prefetchw srcreg+d4+d1+L1pd, L1pt
	zfmaddpd zmm12, zmm10, zmm31, zmm14	;; r2o = (r1-r9) + .707*(r3--)				; 41-44		n 48
	zfnmaddpd zmm10, zmm10, zmm31, zmm14	;; r4o = (r1-r9) - .707*(r3--)				; 41-44		n 49

	L1prefetchw srcreg+d4+d1+64+L1pd, L1pt
	zfmaddpd zmm14, zmm2, zmm30, zmm6	;; r2e/.383 = .924/.383(r2--) + (r4--)			; 42-45		n 48
	zfnmaddpd zmm6, zmm6, zmm30, zmm2	;; r4e/.383 = (r2--) - .924/.383(r4--)			; 42-45		n 49

	L1prefetchw srcreg+d4+d2+L1pd, L1pt
	zfmaddpd zmm2, zmm11, zmm31, zmm16	;; i2o = .707(r3-+) + (r5-r13)				; 43-46		n 50
	zfmsubpd zmm11, zmm11, zmm31, zmm16	;; i4o = .707(r3-+) - (r5-r13)				; 43-46		n 51

	L1prefetchw srcreg+d4+d2+64+L1pd, L1pt
	zfmaddpd zmm16, zmm4, zmm30, zmm15	;; i2e/.383 = (r2-+) + .924/.383(r4-+)			; 44-47		n 50
	zfmsubpd zmm15, zmm15, zmm30, zmm4	;; i4e/.383 = .924/.383(r2-+) - (r4-+)			; 44-47		n 51
	zstore	[srcreg], zmm9			;; Store R1						; 44

	vmovapd	zmm27, [screg+3*128+64]		;; cosine/sine for R5/I5 (w^4)
	zfmsubpd zmm4, zmm7, zmm27, zmm1	;; A5 = R5 * cosine/sine - I5				; 45-48		n 52
	zfmaddpd zmm1, zmm1, zmm27, zmm7	;; B5 = I5 * cosine/sine + R5				; 45-48		n 52
	zstore	[srcreg+64], zmm5		;; Store R9						; 44+1

	vmovapd	zmm27, [screg+1*128+64]		;; cosine/sine for R3/I3 (w^2)
	zfmsubpd zmm7, zmm8, zmm27, zmm13	;; A3 = R3 * cosine/sine - I3				; 46-49		n 53
	zfmaddpd zmm13, zmm13, zmm27, zmm8	;; B3 = I3 * cosine/sine + R3				; 46-49		n 53

	vmovapd	zmm27, [screg+5*128+64]		;; cosine/sine for R7/I7 (w^6)
	zfmsubpd zmm8, zmm0, zmm27, zmm3	;; A7 = R7 * cosine/sine - I7				; 47-50		n 54
	zfmaddpd zmm3, zmm3, zmm27, zmm0	;; B7 = I7 * cosine/sine + R7				; 47-50		n 54

	L1prefetchw srcreg+d4+d2+d1+L1pd, L1pt
	zfmaddpd zmm0, zmm14, zmm29, zmm12	;; R2 = r2o + .383*r2e					; 48-51		n 55
	zfnmaddpd zmm14, zmm14, zmm29, zmm12	;; R8 = r2o - .383*r2e					; 48-51		n 56
	L1prefetchw srcreg+d4+d2+d1+64+L1pd, L1pt
	zfmaddpd zmm12, zmm6, zmm29, zmm10	;; R4 = r4o + .383*r4e					; 49-52		n 57
	zfnmaddpd zmm6, zmm6, zmm29, zmm10	;; R6 = r4o - .383*r4e					; 49-52		n 58

	zfmaddpd zmm10, zmm16, zmm29, zmm2	;; I2 = .383*i2e + i2o					; 50-53		n 55
	zfmsubpd zmm16, zmm16, zmm29, zmm2	;; I8 = .383*i2e - i2o					; 50-53		n 56
	zfmaddpd zmm2, zmm15, zmm29, zmm11	;; I4 = .383*i4e + i4o					; 51-54		n 57
	zfmsubpd zmm15, zmm15, zmm29, zmm11	;; I6 = .383*i4e - i4o					; 51-54		n 58

	vmovapd	zmm27, [screg+3*128]		;; sine for R5/I5
	vmulpd	zmm4, zmm4, zmm27		;; A5 = A5 * sine (final R5)				; 52-55
	vmulpd	zmm1, zmm1, zmm27		;; B5 = B5 * sine (final I5)				; 52-55

	vmovapd	zmm27, [screg+1*128]		;; sine for R3/I3
	vmulpd	zmm7, zmm7, zmm27		;; A3 = A3 * sine (final R3)				; 53-56
	vmulpd	zmm13, zmm13, zmm27		;; B3 = B3 * sine (final I3)				; 53-56

	vmovapd	zmm27, [screg+5*128]		;; sine for R7/I7
	vmulpd	zmm8, zmm8, zmm27		;; A7 = A7 * sine (final R7)				; 54-57
	vmulpd	zmm3, zmm3, zmm27		;; B7 = B7 * sine (final I7)				; 54-57

	vmovapd	zmm27, [screg+0*128+64]		;; cosine/sine for R2/I2 (w^1)
	zfmsubpd zmm11, zmm0, zmm27, zmm10	;; A2 = R2 * cosine/sine - I2				; 55-58		n 59
	zfmaddpd zmm10, zmm10, zmm27, zmm0	;; B2 = I2 * cosine/sine + R2				; 55-58		n 59

	vmovapd	zmm27, [screg+6*128+64]		;; cosine/sine for R8/I8 (w^7)
	zfmsubpd zmm0, zmm14, zmm27, zmm16	;; A8 = R8 * cosine/sine - I8				; 56-59		n 60
	zfmaddpd zmm16, zmm16, zmm27, zmm14	;; B8 = I8 * cosine/sine + R8				; 56-59		n 60
	zstore	[srcreg+d4], zmm4		;; Store R5						; 56

	vmovapd	zmm27, [screg+2*128+64]		;; cosine/sine for R4/I4 (w^3)
	zfmsubpd zmm14, zmm12, zmm27, zmm2	;; A4 = R4 * cosine/sine - I4				; 57-60		n 61
	zfmaddpd zmm2, zmm2, zmm27, zmm12	;; B4 = I4 * cosine/sine + R4				; 57-60		n 61
	zstore	[srcreg+d4+64], zmm1		;; Store I5						; 56+1

	vmovapd	zmm27, [screg+4*128+64]		;; cosine/sine for R6/I6 (w^5)
	zfmsubpd zmm12, zmm6, zmm27, zmm15	;; A6 = R6 * cosine/sine - I6				; 58-61		n 62
	zfmaddpd zmm15, zmm15, zmm27, zmm6	;; B6 = I6 * cosine/sine + R6				; 58-61		n 62
	zstore	[srcreg+d2], zmm7		;; Store R3						; 57+1

	vmovapd	zmm27, [screg+0*128]		;; sine for R2/I2
	vmulpd	zmm11, zmm11, zmm27		;; A2 = A2 * sine (final R2)				; 59-62
	vmulpd	zmm10, zmm10, zmm27		;; B2 = B2 * sine (final I2)				; 59-62
	zstore	[srcreg+d2+64], zmm13		;; Store I3						; 57+2

	vmovapd	zmm27, [screg+6*128]		;; sine for R8/I8
	vmulpd	zmm0, zmm0, zmm27		;; A8 = A8 * sine (final R8)				; 60-63
	vmulpd	zmm16, zmm16, zmm27		;; B8 = B8 * sine (final I8)				; 60-63
	zstore	[srcreg+d4+d2], zmm8		;; Store R7						; 58+2

	vmovapd	zmm27, [screg+2*128]		;; sine for R4/I4
	vmulpd	zmm14, zmm14, zmm27		;; A4 = A4 * sine (final R4)				; 61-64
	vmulpd	zmm2, zmm2, zmm27		;; B4 = B4 * sine (final I4)				; 61-64
	zstore	[srcreg+d4+d2+64], zmm3		;; Store I7						; 58+3

	vmovapd zmm27, [screg+4*128]		;; sine for R6/I6
	vmulpd	zmm12, zmm12, zmm27		;; A6 = A6 * sine (final R6)				; 62-65
	vmulpd	zmm15, zmm15, zmm27		;; B6 = B6 * sine (final I6)				; 62-65

	bump	screg, scinc
	zstore	[srcreg+d1], zmm11		;; Store R2						; 63
	zstore	[srcreg+d1+64], zmm10		;; Store I2						; 63+1
	zstore	[srcreg+d4+d2+d1], zmm0		;; Store R8						; 64+1
	zstore	[srcreg+d4+d2+d1+64], zmm16	;; Store I8						; 64+2
	zstore	[srcreg+d2+d1], zmm14		;; Store R4						; 65+2
	zstore	[srcreg+d2+d1+64], zmm2		;; Store I4						; 65+3
	zstore	[srcreg+d4+d1], zmm12		;; Store R6						; 66+3
	zstore	[srcreg+d4+d1+64], zmm15	;; Store I6						; 66+4
	bump	srcreg, srcinc
	ENDM

;;
;; ************************************* 16-reals-unfft variants ******************************************
;;

;; These macros produce 16 reals after doing 4 levels of the inverse FFT applying
;; the sin/cos multipliers beforehand.  The input is 2 real and 7 complex numbers.

;; To calculate a 16-reals inverse FFT, we calculate 16 real values from 16 complex inputs in a brute force way.
;; First we note that the 16 complex values are computed from the 7 complex and 2 real inputs using Hermetian symmetry, thusly:
;; c1 = r1A + 0*i
;; c2 = r2 + i2*i
;; ...
;; c8 = r8 + i8*i
;; c9 = r1B + 0*i
;; c10 = r8 - i8*i
;; ...
;; c16 = r2 - i2*i 
;;
;; The brute force calculations are:
;;
;; c1 + c2 + ... + c16	*  w^-0000000000...
;; c1 + c2 + ... + c16	*  w^-0123456789A...
;; c1 + c2 + ... + c16	*  w^-02468ACE....
;;    ...
;; c1 + c2 + ... + c16	*  w^-...A987654321
;;
;; The sin/cos values (w = 16th root of unity) are:
;; w^-1 = .924 - .383i
;; w^-2 = .707 - .707i
;; w^-3 = .383 - .924i
;; w^-4 = 0 - 1i
;; w^-5 = -.383 - .924i
;; w^-6 = -.707 - .707i
;; w^-7 = -.924 - .383i
;; w^-8 = -1

;; Applying the sin/cos values above, taking advantage of symmetry, and ignoring a lot of multiplies by 2:
;; r1     +(r2+r8)     +(r3+r7)     +(r4+r6) + r5 + r9
;; r1 +.924(r2-r8) +.707(r3-r7) +.383(r4-r6)      - r9 +.383(i2+i8) +.707(i3+i7) +.924(i4+i6) + i5
;; r1 +.707(r2+r8)              -.707(r4+r6) - r5 + r9 +.707(i2-i8)     +(i3-i7) +.707(i4-i6)
;; r1 +.383(r2-r8) -.707(r3-r7) -.924(r4-r6)      - r9 +.924(i2+i8) +.707(i3+i7) -.383(i4+i6) - i5
;; r1                  -(r3+r7)              + r5 + r9     +(i2-i8)                  -(i4-i6)
;; r1 -.383(r2-r8) -.707(r3-r7) +.924(r4-r6)      - r9 +.924(i2+i8) -.707(i3+i7) -.383(i4+i6) + i5
;; r1 -.707(r2+r8)              +.707(r4+r6) - r5 + r9 +.707(i2-i8)     -(i3-i7) +.707(i4-i6)
;; r1 -.924(r2-r8) +.707(r3-r7) -.383(r4-r6)      - r9 +.383(i2+i8) -.707(i3+i7) +.924(i4+i6) - i5
;; r1     -(r2+r8)     +(r3+r7)     -(r4+r6) + r5 + r9
;; ... r10 thru r16 are the same as r8 through r2 but with the sign of the imaginary component changed.
;;
;; Also remember that due to the funny way we do things reals input r1A = r1+r9 and r1B = r1-r9

;; Simplifying yields:
;;R1 = (r1+r9) + r5 +(r3+r7)     +((r2+r8)+(r4+r6))
;;R9 = (r1+r9) + r5 +(r3+r7)     -((r2+r8)+(r4+r6))
;;R5 = (r1+r9) + r5 -(r3+r7)                        +((i2-i8)-(i4-i6))
;;R13= (r1+r9) + r5 -(r3+r7)                        -((i2-i8)-(i4-i6))
;;R3 = (r1+r9) - r5 +(i3-i7) +.707(((r2+r8)-(r4+r6))+((i2-i8)+(i4-i6)))
;;R11= (r1+r9) - r5 +(i3-i7) -.707(((r2+r8)-(r4+r6))+((i2-i8)+(i4-i6)))
;;R7 = (r1+r9) - r5 -(i3-i7) -.707(((r2+r8)-(r4+r6))-((i2-i8)+(i4-i6)))
;;R15= (r1+r9) - r5 -(i3-i7) +.707(((r2+r8)-(r4+r6))-((i2-i8)+(i4-i6)))

;;R2 = (r1-r9) + i5 +.707((r3-r7)+(i3+i7)) +.924((r2-r8)+(i4+i6)) +.383((i2+i8)+(r4-r6))
;;R10= (r1-r9) + i5 +.707((r3-r7)+(i3+i7)) -.924((r2-r8)+(i4+i6)) -.383((i2+i8)+(r4-r6))
;;R6 = (r1-r9) + i5 -.707((r3-r7)+(i3+i7)) -.383((r2-r8)+(i4+i6)) +.924((i2+i8)+(r4-r6))
;;R14= (r1-r9) + i5 -.707((r3-r7)+(i3+i7)) +.383((r2-r8)+(i4+i6)) -.924((i2+i8)+(r4-r6))
;;R4 = (r1-r9) - i5 -.707((r3-r7)-(i3+i7)) +.383((r2-r8)-(i4+i6)) +.924((i2+i8)-(r4-r6))
;;R12= (r1-r9) - i5 -.707((r3-r7)-(i3+i7)) -.383((r2-r8)-(i4+i6)) -.924((i2+i8)-(r4-r6))
;;R8 = (r1-r9) - i5 +.707((r3-r7)-(i3+i7)) -.924((r2-r8)-(i4+i6)) +.383((i2+i8)-(r4-r6))
;;R16= (r1-r9) - i5 +.707((r3-r7)-(i3+i7)) +.924((r2-r8)-(i4+i6)) -.383((i2+i8)-(r4-r6))

;; Uses two sin/cos ptrs.  Used in last levels of 2nd pass in a two-pass FFT
zr8_2sc_sixteen_reals_unfft_preload MACRO
	zr8_16r_unfft_cmn_preload
	ENDM

zr8_2sc_sixteen_reals_unfft MACRO srcreg,srcinc,d1,d2,d4,screg1,scinc1,screg2,scinc2,maxrpt,L1pt,L1pd
	zr8_16r_unfft_cmn srcreg,srcinc,d1,d2,d4,screg1,scinc1,screg2,scinc2,maxrpt,L1pt,L1pd
	ENDM

;; Combined sin/cos data.  Used in mid-levels of 1st pass in a two-pass FFT
zr8_csc_sixteen_reals_unfft_preload MACRO
	zr8_16r_unfft_cmn_preload
	ENDM

zr8_csc_sixteen_reals_unfft MACRO srcreg,srcinc,d1,d2,d4,screg,scinc,maxrpt,L1pt,L1pd
	zr8_16r_unfft_cmn srcreg,srcinc,d1,d2,d4,screg+4*128,0,screg,scinc,maxrpt,L1pt,L1pd
	ENDM


zr8_16r_unfft_cmn_preload MACRO
	vbroadcastsd zmm31, ZMM_SQRTHALF
	vbroadcastsd zmm30, ZMM_P924_P383
	vbroadcastsd zmm29, ZMM_P383
	ENDM

zr8_16r_unfft_cmn MACRO srcreg,srcinc,d1,d2,d4,screg1,scinc1,screg2,scinc2,maxrpt,L1pt,L1pd
	vmovapd	zmm28, [screg2+0*128+64]	;; cosine/sine for R3/I3 (w^2 = complex w^1)
	vmovapd	zmm2, [srcreg+d2]		;; R3
	vmovapd	zmm10, [srcreg+d2+64]		;; I3
	zfmaddpd zmm16, zmm2, zmm28, zmm10	;; A3 = R3 * cosine/sine + I3				; 1-4		n 5
	zfmsubpd zmm10, zmm10, zmm28, zmm2	;; B3 = I3 * cosine/sine - R3				; 1-4		n 5

	vmovapd	zmm28, [screg1+0*128+64]	;; cosine/sine for R2/I2 (w^1)
	vmovapd	zmm1, [srcreg+d1]		;; R2
	vmovapd	zmm9, [srcreg+d1+64]		;; I2
	zfmaddpd zmm2, zmm1, zmm28, zmm9	;; A2 = R2 * cosine/sine + I2				; 2-5		n 7
	zfmsubpd zmm9, zmm9, zmm28, zmm1	;; B2 = I2 * cosine/sine - R2				; 2-5		n 7

	vmovapd	zmm28, [screg1+1*128+64]	;; cosine/sine for R4/I4 (w^3)
	vmovapd	zmm3, [srcreg+d2+d1]		;; R4
	vmovapd	zmm11, [srcreg+d2+d1+64]	;; I4
	zfmaddpd zmm1, zmm3, zmm28, zmm11	;; A4 = R4 * cosine/sine + I4				; 3-6		n 9
	zfmsubpd zmm11, zmm11, zmm28, zmm3	;; B4 = I4 * cosine/sine - R4				; 3-6		n 9

	vmovapd	zmm28, [screg2+2*128+64]	;; cosine/sine for R7/I7 (w^6 = complex w^3)
	vmovapd	zmm6, [srcreg+d4+d2]		;; R7
	vmovapd	zmm14, [srcreg+d4+d2+64]	;; I7
	zfmaddpd zmm3, zmm6, zmm28, zmm14	;; A7 = R7 * cosine/sine + I7				; 4-7		n 11
	zfmsubpd zmm14, zmm14, zmm28, zmm6	;; B7 = I7 * cosine/sine - R7				; 4-7		n 12

	vmovapd	zmm28, [screg2+0*128]		;; sine for R3/I3
	vmulpd	zmm16, zmm16, zmm28		;; R3 = A3 * sine					; 5-8		n 11
	vmulpd	zmm10, zmm10, zmm28		;; I3 = B3 * sine					; 5-8		n 12

	vmovapd	zmm28, [screg1+3*128+64]	;; cosine/sine for R8/I8 (w^7)
	vmovapd	zmm7, [srcreg+d4+d2+d1]		;; R8
	vmovapd	zmm15, [srcreg+d4+d2+d1+64]	;; I8
	zfmaddpd zmm6, zmm7, zmm28, zmm15	;; A8 = R8 * cosine/sine + I8				; 6-9		n 13
	zfmsubpd zmm15, zmm15, zmm28, zmm7	;; B8 = I8 * cosine/sine - R8				; 6-9		n 14

	vmovapd	zmm28, [screg1+0*128]		;; sine for R2/I2
	vmulpd	zmm2, zmm2, zmm28		;; R2 = A2 * sine					; 7-10		n 13
	vmulpd	zmm9, zmm9, zmm28		;; I2 = B2 * sine					; 7-10		n 14

	vmovapd	zmm28, [screg1+2*128+64]	;; cosine/sine for R6/I6 (w^5)
	vmovapd	zmm5, [srcreg+d4+d1]		;; R6
	vmovapd	zmm13, [srcreg+d4+d1+64]	;; I6
	zfmaddpd zmm7, zmm5, zmm28, zmm13	;; A6 = R6 * cosine/sine + I6				; 8-11		n 15
	zfmsubpd zmm13, zmm13, zmm28, zmm5	;; B6 = I6 * cosine/sine - R6				; 8-11		n 16

	vmovapd	zmm28, [screg1+1*128]		;; sine for R4/I4
	vmulpd	zmm1, zmm1, zmm28		;; R4 = A4 * sine					; 9-12		n 15
	vmulpd	zmm11, zmm11, zmm28		;; I4 = B4 * sine					; 9-12		n 16

	vmovapd	zmm28, [screg2+1*128+64]	;; cosine/sine for R5/I5 (w^4 = complex w^2)
	vmovapd	zmm4, [srcreg+d4]		;; R5
	vmovapd	zmm12, [srcreg+d4+64]		;; I5
	zfmaddpd zmm5, zmm4, zmm28, zmm12	;; A5 = R5 * cosine/sine + I5				; 10-13		n 17
	zfmsubpd zmm12, zmm12, zmm28, zmm4	;; B5 = I5 * cosine/sine - R5				; 10-13		n 18

	vmovapd	zmm28, [screg2+2*128]		;; sine for R7/I7
	zfnmaddpd zmm4, zmm3, zmm28, zmm16	;; R3-(R7 = A7 * sine)					; 11-14		n 19
	zfmaddpd zmm3, zmm3, zmm28, zmm16	;; R3+(R7 = A7 * sine)					; 11-14		n 22

	vmovapd	zmm27, [screg1+3*128]		;; sine for R8/I8
	zfmaddpd zmm16, zmm14, zmm28, zmm10	;; I3+(I7 = B7 * sine)					; 12-15		n 19
	zfnmaddpd zmm14, zmm14, zmm28, zmm10	;; I3-(I7 = B7 * sine)					; 12-15		n 23

	vmovapd	zmm26, [screg1+2*128]		;; sine for R6/I6
	zfmaddpd zmm10, zmm6, zmm27, zmm2	;; R2+(R8 = A8 * sine)					; 13-16		n 20
	zfnmaddpd zmm6, zmm6, zmm27, zmm2	;; R2-(R8 = A8 * sine)					; 13-16		n 24

	vmovapd	zmm25, [screg2+1*128]		;; sine for R5/I5
	zfnmaddpd zmm2, zmm15, zmm27, zmm9	;; I2-(I8 = B8 * sine)					; 14-17		n 21
	zfmaddpd zmm15, zmm15, zmm27, zmm9	;; I2+(I8 = B8 * sine)					; 14-17		n 25

	vmovapd	zmm0, [srcreg]			;; R1+R9
	zfmaddpd zmm9, zmm7, zmm26, zmm1	;; R4+(R6 = A6 * sine)					; 15-18		n 20
	zfnmaddpd zmm7, zmm7, zmm26, zmm1	;; R4-(R6 = A6 * sine)					; 15-18		n 25

	vmovapd	zmm8, [srcreg+64]		;; R1-R9
	zfnmaddpd zmm1, zmm13, zmm26, zmm11	;; I4-(I6 = B6 * sine)					; 16-19		n 21
	zfmaddpd zmm13, zmm13, zmm26, zmm11	;; I4+(I6 = B6 * sine)					; 16-19		n 24

	L1prefetchw srcreg+L1pd, L1pt
	zfmaddpd zmm11, zmm5, zmm25, zmm0	;; r1++ = (r1+r9) + r5*sine				; 17-20		n 22
	zfnmaddpd zmm5, zmm5, zmm25, zmm0	;; r1+- = (r1+r9) - r5*sine				; 17-20		n 23

	L1prefetchw srcreg+d1+L1pd, L1pt
	zfmaddpd zmm0, zmm12, zmm25, zmm8	;; r1-+ = (r1-r9) + i5*sine				; 18-21		n 29
	zfnmaddpd zmm12, zmm12, zmm25, zmm8	;; r1-- = (r1-r9) - i5*sine				; 18-21		n 31

	L1prefetchw srcreg+d2+L1pd, L1pt
	vaddpd	zmm8, zmm4, zmm16		;; r3-+ = (r3-r7) + (i3+i7)				; 19-22		n 29
	vsubpd	zmm4, zmm4, zmm16		;; r3-- = (r3-r7) - (i3+i7)				; 19-22		n 31

	L1prefetchw srcreg+d4+L1pd, L1pt
	vaddpd	zmm16, zmm10, zmm9		;; r2++ = (r2+r8) + (r4+r6)				; 20-23		n 26
	vsubpd	zmm10, zmm10, zmm9		;; r2+- = (r2+r8) - (r4+r6)				; 20-23		n 28

	L1prefetchw srcreg+d2+d1+L1pd, L1pt
	vsubpd	zmm9, zmm2, zmm1		;; i2-- = (i2-i8) - (i4-i6)				; 21-24		n 27
	vaddpd	zmm2, zmm2, zmm1		;; i2-+ = (i2-i8) + (i4-i6)				; 21-24		n 28

	L1prefetchw srcreg+d4+d1+L1pd, L1pt
	vaddpd	zmm1, zmm11, zmm3		;; r1+++ = (r1++) + (r3+r7)				; 22-25		n 26
	vsubpd	zmm11, zmm11, zmm3		;; r1++- = (r1++) - (r3+r7)				; 22-25		n 27

	L1prefetchw srcreg+d4+d2+L1pd, L1pt
	vaddpd	zmm3, zmm5, zmm14		;; r1+-+ = (r1+-) + (i3-i7)				; 23-26		n 33
	vsubpd	zmm5, zmm5, zmm14		;; r1+-- = (r1+-) - (i3-i7)				; 23-26		n 34

	L1prefetchw srcreg+d4+d2+d1+L1pd, L1pt
	vaddpd	zmm14, zmm6, zmm13		;; r2-+ = (r2-r8) + (i4+i6)				; 24-27		n 30
	vsubpd	zmm6, zmm6, zmm13		;; r2-- = (r2-r8) - (i4+i6)				; 24-27		n 32

	L1prefetchw srcreg+64+L1pd, L1pt
	vaddpd	zmm13, zmm15, zmm7		;; i2++ = (i2+i8) + (r4-r6)				; 25-28		n 30
	vsubpd	zmm15, zmm15, zmm7		;; i2+- = (i2+i8) - (r4-r6)				; 25-28		n 32

	L1prefetchw srcreg+d1+64+L1pd, L1pt
	vaddpd	zmm7, zmm1, zmm16		;; R1 = (r1+++) + (r2++)				; 26-29
	vsubpd	zmm1, zmm1, zmm16		;; R9 = (r1+++) - (r2++)				; 26-29

	L1prefetchw srcreg+d2+64+L1pd, L1pt
	vaddpd	zmm16, zmm11, zmm9		;; R5  = (r1++-) + (i2--)				; 27-30
	vsubpd	zmm11, zmm11, zmm9		;; R13 = (r1++-) - (i2--)				; 27-30

	L1prefetchw srcreg+d2+d1+64+L1pd, L1pt
	vaddpd	zmm9, zmm10, zmm2		;; r2+-+ = (r2+-) + (i2-+)				; 28-31		n 33
	vsubpd	zmm10, zmm10, zmm2		;; r2+-- = (r2+-) - (i2-+)				; 28-31		n 34

	L1prefetchw srcreg+d4+64+L1pd, L1pt
	zfmaddpd zmm2, zmm8, zmm31, zmm0	;; r2_10o = (r1-+) + .707(r3-+)				; 29-32		n 35
	zfnmaddpd zmm8, zmm8, zmm31, zmm0	;; r6_14o = (r1-+) - .707(r3-+)				; 29-32		n 36

	L1prefetchw srcreg+d4+d1+64+L1pd, L1pt
	zfmaddpd zmm0, zmm14, zmm30, zmm13	;; r2_10e/.383 = .924/.383(r2-+) + (i2++)		; 30-33		n 35
	zfmsubpd zmm13, zmm13, zmm30, zmm14	;; r6_14e/.383 = .924/.383(i2++) - (r2-+)		; 30-33		n 36
	zstore	[srcreg], zmm7			;; Save R1						; 30

	L1prefetchw srcreg+d4+d2+64+L1pd, L1pt
	zfnmaddpd zmm14, zmm4, zmm31, zmm12	;; r4_12o = (r1--) - .707(r3--)				; 31-34		n 37
	zfmaddpd zmm4, zmm4, zmm31, zmm12	;; r8_16o = (r1--) + .707(r3--)				; 31-34		n 38
	zstore	[srcreg+64], zmm1		;; Save R9						; 30+1

	L1prefetchw srcreg+d4+d2+d1+64+L1pd, L1pt
	zfmaddpd zmm12, zmm15, zmm30, zmm6	;; r4_12e/.383 = .924/.383(i2+-) + (r2--)		; 32-35		n 37
	zfnmaddpd zmm6, zmm6, zmm30, zmm15	;; r8_16e/.383 = (i2+-) - .924/.383(r2--)		; 32-35		n 38
	zstore	[srcreg+d4], zmm16		;; Save R5						; 31+1

	zfmaddpd zmm15, zmm9, zmm31, zmm3	;; R3  = (r1+-+) + .707(r2+-+)				; 33-36
	zfnmaddpd zmm9, zmm9, zmm31, zmm3	;; R11 = (r1+-+) - .707(r2+-+)				; 33-36
	zstore	[srcreg+d4+64], zmm11		;; Save R13						; 31+2

	zfnmaddpd zmm3, zmm10, zmm31, zmm5	;; R7  = (r1+--) - .707(r2+--)				; 34-37
	zfmaddpd zmm10, zmm10, zmm31, zmm5	;; R15 = (r1+--) + .707(r2+--)				; 34-37
	bump	screg1, scinc1

	zfmaddpd zmm5, zmm0, zmm29, zmm2	;; R2  = r2_10o + .383*r2_10e				; 35-38
	zfnmaddpd zmm0, zmm0, zmm29, zmm2	;; R10 = r2_10o - .383*r2_10e				; 35-38
	bump	screg2, scinc2

	zfmaddpd zmm2, zmm13, zmm29, zmm8	;; R6  = r6_14o + .383*r6_14e				; 36-39
	zfnmaddpd zmm13, zmm13, zmm29, zmm8	;; R14 = r6_14o - .383*r6_14e				; 36-39

	zfmaddpd zmm8, zmm12, zmm29, zmm14	;; R4  = r4_12o + .383*r4_12e				; 37-40
	zfnmaddpd zmm12, zmm12, zmm29, zmm14	;; R12 = r4_12o - .383*r4_12e				; 37-40
	zstore	[srcreg+d2], zmm15		;; Save R3						; 37

	zfmaddpd zmm14, zmm6, zmm29, zmm4	;; R8  = r8_16o + .383*r8_16e				; 38-41
	zfnmaddpd zmm6, zmm6, zmm29, zmm4	;; R16 = r8_16o - .383*r8_16e				; 38-41

	zstore	[srcreg+d2+64], zmm9		;; Save R11						; 37+1
	zstore	[srcreg+d4+d2], zmm3		;; Save R7						; 38+1
	zstore	[srcreg+d4+d2+64], zmm10	;; Save R15						; 38+2
	zstore	[srcreg+d1], zmm5		;; Save R2						; 39+2
	zstore	[srcreg+d1+64], zmm0		;; Save R10						; 39+3
	zstore	[srcreg+d4+d1], zmm2		;; Save R6						; 40+3
	zstore	[srcreg+d4+d1+64], zmm13	;; Save R14						; 40+4
	zstore	[srcreg+d2+d1], zmm8		;; Save R4						; 41+4
	zstore	[srcreg+d2+d1+64], zmm12	;; Save R12						; 41+5
	zstore	[srcreg+d4+d2+d1], zmm14	;; Save R8						; 42+5
	zstore	[srcreg+d4+d2+d1+64], zmm6	;; Save R16						; 42+6
	bump	srcreg, srcinc
	ENDM


;; Used in last levels of pass 1 in a two-pass FFT
zr8_wpn_sixteen_reals_last_unfft_preload MACRO
	vbroadcastsd zmm31, ZMM_SQRTHALF
	vbroadcastsd zmm30, ZMM_P924_P383
	vbroadcastsd zmm29, ZMM_P383
	vbroadcastsd zmm28, ZMM_B
	ENDM

zr8_wpn_sixteen_reals_last_unfft MACRO srcreg,srcinc,d1,d2,d4,screg,scinc,maskreg,maskinc,maxrpt,L1pt,L1pd
	vmovapd	zmm2, [srcreg+d2]		;; R3
	vmovapd	zmm10, [srcreg+d2+64]		;; I3
	vmovapd	zmm27, [screg+1*128+64]		;; cosine/sine for R3/I3 (w^2)
	zfmaddpd zmm16, zmm2, zmm27, zmm10	;; A3 = R3 * cosine/sine + I3				; 1-4		n 5
	zfmsubpd zmm10, zmm10, zmm27, zmm2	;; B3 = I3 * cosine/sine - R3				; 1-4		n 5

	vmovapd	zmm1, [srcreg+d1]		;; R2
	vmovapd	zmm9, [srcreg+d1+64]		;; I2
	vmovapd	zmm27, [screg+0*128+64]		;; cosine/sine for R2/I2 (w^1)
	zfmaddpd zmm2, zmm1, zmm27, zmm9	;; A2 = R2 * cosine/sine + I2				; 2-5		n 7
	zfmsubpd zmm9, zmm9, zmm27, zmm1	;; B2 = I2 * cosine/sine - R2				; 2-5		n 7

	vmovapd	zmm3, [srcreg+d2+d1]		;; R4
	vmovapd	zmm11, [srcreg+d2+d1+64]	;; I4
	vmovapd	zmm27, [screg+2*128+64]		;; cosine/sine for R4/I4 (w^3)
	zfmaddpd zmm1, zmm3, zmm27, zmm11	;; A4 = R4 * cosine/sine + I4				; 3-6		n 9
	zfmsubpd zmm11, zmm11, zmm27, zmm3	;; B4 = I4 * cosine/sine - R4				; 3-6		n 9

	vmovapd	zmm6, [srcreg+d4+d2]		;; R7
	vmovapd	zmm14, [srcreg+d4+d2+64]	;; I7
	vmovapd	zmm27, [screg+5*128+64]		;; cosine/sine for R7/I7 (w^6)
	zfmaddpd zmm3, zmm6, zmm27, zmm14	;; A7 = R7 * cosine/sine + I7				; 4-7		n 11
	zfmsubpd zmm14, zmm14, zmm27, zmm6	;; B7 = I7 * cosine/sine - R7				; 4-7		n 12

	vmovapd	zmm27, [screg+1*128]		;; sine for R3/I3
	vmulpd	zmm16, zmm16, zmm27		;; R3 = A3 * sine					; 5-8		n 11
	vmulpd	zmm10, zmm10, zmm27		;; I3 = B3 * sine					; 5-8		n 12

	vmovapd	zmm7, [srcreg+d4+d2+d1]		;; R8
	vmovapd	zmm15, [srcreg+d4+d2+d1+64]	;; I8
	vmovapd	zmm27, [screg+6*128+64]		;; cosine/sine for R8/I8 (w^7)
	zfmaddpd zmm6, zmm7, zmm27, zmm15	;; A8 = R8 * cosine/sine + I8				; 6-9		n 13
	zfmsubpd zmm15, zmm15, zmm27, zmm7	;; B8 = I8 * cosine/sine - R8				; 6-9		n 14

	vmovapd	zmm27, [screg+0*128]		;; sine for R2/I2
	vmulpd	zmm2, zmm2, zmm27		;; R2 = A2 * sine					; 7-10		n 13
	vmulpd	zmm9, zmm9, zmm27		;; I2 = B2 * sine					; 7-10		n 14

	vmovapd	zmm5, [srcreg+d4+d1]		;; R6
	vmovapd	zmm13, [srcreg+d4+d1+64]	;; I6
	vmovapd	zmm27, [screg+4*128+64]		;; cosine/sine for R6/I6 (w^5)
	zfmaddpd zmm7, zmm5, zmm27, zmm13	;; A6 = R6 * cosine/sine + I6				; 8-11		n 15
	zfmsubpd zmm13, zmm13, zmm27, zmm5	;; B6 = I6 * cosine/sine - R6				; 8-11		n 16

	vmovapd	zmm27, [screg+2*128]		;; sine for R4/I4
	vmulpd	zmm1, zmm1, zmm27		;; R4 = A4 * sine					; 9-12		n 15
	vmulpd	zmm11, zmm11, zmm27		;; I4 = B4 * sine					; 9-12		n 16

	vmovapd	zmm4, [srcreg+d4]		;; R5
	vmovapd	zmm12, [srcreg+d4+64]		;; I5
	vmovapd	zmm27, [screg+3*128+64]		;; cosine/sine for R5/I5 (w^4)
	zfmaddpd zmm5, zmm4, zmm27, zmm12	;; A5 = R5 * cosine/sine + I5				; 10-13		n 17
	zfmsubpd zmm12, zmm12, zmm27, zmm4	;; B5 = I5 * cosine/sine - R5				; 10-13		n 18

	vmovapd	zmm27, [screg+5*128]		;; sine for R7/I7
	zfnmaddpd zmm4, zmm3, zmm27, zmm16	;; R3-(R7 = A7 * sine)					; 11-14		n 19
	zfmaddpd zmm3, zmm3, zmm27, zmm16	;; R3+(R7 = A7 * sine)					; 11-14		n 22

	zfmaddpd zmm16, zmm14, zmm27, zmm10	;; I3+(I7 = B7 * sine)					; 12-15		n 19
	zfnmaddpd zmm14, zmm14, zmm27, zmm10	;; I3-(I7 = B7 * sine)					; 12-15		n 23

	vmovapd	zmm27, [screg+6*128]		;; sine for R8/I8
	zfmaddpd zmm10, zmm6, zmm27, zmm2	;; R2+(R8 = A8 * sine)					; 13-16		n 20
	zfnmaddpd zmm6, zmm6, zmm27, zmm2	;; R2-(R8 = A8 * sine)					; 13-16		n 25

	zfnmaddpd zmm2, zmm15, zmm27, zmm9	;; I2-(I8 = B8 * sine)					; 14-17		n 21
	zfmaddpd zmm15, zmm15, zmm27, zmm9	;; I2+(I8 = B8 * sine)					; 14-17		n 27

	vmovapd	zmm27, [screg+4*128]		;; sine for R6/I6
	zfmaddpd zmm9, zmm7, zmm27, zmm1	;; R4+(R6 = A6 * sine)					; 15-18		n 20
	zfnmaddpd zmm7, zmm7, zmm27, zmm1	;; R4-(R6 = A6 * sine)					; 15-18		n 27

	zfnmaddpd zmm1, zmm13, zmm27, zmm11	;; I4-(I6 = B6 * sine)					; 16-19		n 21
	zfmaddpd zmm13, zmm13, zmm27, zmm11	;; I4+(I6 = B6 * sine)					; 16-19		n 25

	vmovapd	zmm0, [srcreg]			;; R1+R9
	vmovapd	zmm27, [screg+3*128]		;; sine for R5/I5
	zfmaddpd zmm11, zmm5, zmm27, zmm0	;; r1++ = (r1+r9) + r5*sine				; 17-20		n 22
	zfnmaddpd zmm5, zmm5, zmm27, zmm0	;; r1+- = (r1+r9) - r5*sine				; 17-20		n 23

	vmovapd	zmm8, [srcreg+64]		;; R1-R9
	zfmaddpd zmm0, zmm12, zmm27, zmm8	;; r1-+ = (r1-r9) + i5*sine				; 18-21		n 33
	zfnmaddpd zmm12, zmm12, zmm27, zmm8	;; r1-- = (r1-r9) - i5*sine				; 18-21		n 37
	bump	screg, scinc

	vaddpd	zmm8, zmm4, zmm16		;; r3-+ = (r3-r7) + (i3+i7)				; 19-22		n 33
	vsubpd	zmm4, zmm4, zmm16		;; r3-- = (r3-r7) - (i3+i7)				; 19-22		n 37
	mov	bl, [maskreg+0*1]		;; Load index into compressed fudges

	vaddpd	zmm16, zmm10, zmm9		;; r2++ = (r2+r8) + (r4+r6)				; 20-23		n 28
	vsubpd	zmm10, zmm10, zmm9		;; r2+- = (r2+r8) - (r4+r6)				; 20-23		n 31
	mov	r14, [r13+0*8]			;; Load the xor mask

	vsubpd	zmm9, zmm2, zmm1		;; i2-- = (i2-i8) - (i4-i6)				; 21-24		n 30
	vaddpd	zmm2, zmm2, zmm1		;; i2-+ = (i2-i8) + (i4-i6)				; 21-24		n 31
	xor	r14, [r12+rbx*8]		;; Apply the xor mask to 64 bits of fudge flags

	vaddpd	zmm1, zmm11, zmm3		;; r1+++ = (r1++) + (r3+r7)				; 22-25		n 28
	vsubpd	zmm11, zmm11, zmm3		;; r1++- = (r1++) - (r3+r7)				; 22-25		n 30
	mov	bl, [maskreg+1*1]		;; Load index into compressed fudges
	bump	maskreg, maskinc

	kmovw	k1, r14d			;; Load R1 and R9 fudge factor mask			; 23		n 32
	vaddpd	zmm3, zmm5, zmm14		;; r1+-+ = (r1+-) + (i3-i7)				; 23-26		n 40
	shr	r14, 16				;; Next 16 bits of fudge flags

	kmovw	k2, r14d			;; Load R2 and R10 fudge factor mask			; 24		n 47
	vsubpd	zmm5, zmm5, zmm14		;; r1+-- = (r1+-) - (i3-i7)				; 24-27		n 41
	shr	r14, 16				;; Next 16 bits of fudge flags

	kmovw	k3, r14d			;; Load R3 and R11 fudge factor mask			; 25		n 44
	vaddpd	zmm14, zmm6, zmm13		;; r2-+ = (r2-r8) + (i4+i6)				; 25-28		n 35
	shr	r14, 16				;; Next 16 bits of fudge flags

	kmovw	k4, r14d			;; Load R4 and R12 fudge factor mask			; 26		n 51
	vsubpd	zmm6, zmm6, zmm13		;; r2-- = (r2-r8) - (i4+i6)				; 26-29		n 39
	mov	r14, [r13+1*8]			;; Load the xor mask

	vaddpd	zmm13, zmm15, zmm7		;; i2++ = (i2+i8) + (r4-r6)				; 27-30		n 35
	vsubpd	zmm15, zmm15, zmm7		;; i2+- = (i2+i8) - (r4-r6)				; 27-30		n 39
	xor	r14, [r12+rbx*8]		;; Apply the xor mask to 64 bits of fudge flags

	kmovw	k5, r14d			;; Load R5 and R13 fudge factor mask			; 28		n 36
	vaddpd	zmm7, zmm1, zmm16		;; R1 = (r1+++) + (r2++)				; 28-31		n 32
	shr	r14, 16				;; Next 16 bits of fudge flags
	L1prefetchw srcreg+L1pd, L1pt

	kmovw	k6, r14d			;; Load R6 and R14 fudge factor mask			; 29		n 50
	vsubpd	zmm1, zmm1, zmm16		;; R9 = (r1+++) - (r2++)				; 29-32		n 34
	shr	r14, 16				;; Next 16 bits of fudge flags
	L1prefetchw srcreg+64+L1pd, L1pt

	kmovw	k7, r14d			;; Load R7 and R15 fudge factor mask			; 30		n 45
	vaddpd	zmm16, zmm11, zmm9		;; R5  = (r1++-) + (i2--)				; 30-33		n 36
	shr	r14, 16				;; Next 16 bits of fudge flags
	L1prefetchw srcreg+d1+L1pd, L1pt

	vsubpd	zmm11, zmm11, zmm9		;; R13 = (r1++-) - (i2--)				; 31-34		n 38
	vaddpd	zmm9, zmm10, zmm2		;; r2+-+ = (r2+-) + (i2-+)				; 31-34		n 40
	L1prefetchw srcreg+d1+64+L1pd, L1pt

	vmulpd	zmm7{k1}, zmm7, zmm28		;; apply fudge multiplier for R1			; 32-35
	vsubpd	zmm10, zmm10, zmm2		;; r2+-- = (r2+-) - (i2-+)				; 32-35		n 41
	L1prefetchw srcreg+d2+L1pd, L1pt

	kshiftrw k1, k1, 8			;; R9's fudge						; 33		n 34
	zfmaddpd zmm2, zmm8, zmm31, zmm0	;; r2_10o = (r1-+) + .707(r3-+)				; 33-36		n 42
	L1prefetchw srcreg+d2+64+L1pd, L1pt

	vmulpd	zmm1{k1}, zmm1, zmm28		;; apply fudge multiplier for R9			; 34-37
	zfnmaddpd zmm8, zmm8, zmm31, zmm0	;; r6_14o = (r1-+) - .707(r3-+)				; 34-37		n 43
	L1prefetchw srcreg+d2+d1+L1pd, L1pt

	kmovw	k1, r14d			;; Load R8 and R16 fudge factor mask			; 35		n 52
	zfmaddpd zmm0, zmm14, zmm30, zmm13	;; r2_10e/.383 = .924/.383(r2-+) + (i2++)		; 35-38		n 42
	L1prefetchw srcreg+d2+d1+64+L1pd, L1pt

	vmulpd	zmm16{k5}, zmm16, zmm28		;; apply fudge multiplier for R5			; 36-39
	zfmsubpd zmm13, zmm13, zmm30, zmm14	;; r6_14e/.383 = .924/.383(i2++) - (r2-+)		; 36-39		n 43
	zstore	[srcreg], zmm7			;; Save R1						; 36
	L1prefetchw srcreg+d4+L1pd, L1pt

	kshiftrw k5, k5, 8			;; R13's fudge						; 37		n 38
	zfnmaddpd zmm14, zmm4, zmm31, zmm12	;; r4_12o = (r1--) - .707(r3--)				; 37-40		n 44
	L1prefetchw srcreg+d4+64+L1pd, L1pt

	vmulpd	zmm11{k5}, zmm11, zmm28		;; apply fudge multiplier for R13			; 38-41
	zfmaddpd zmm4, zmm4, zmm31, zmm12	;; r8_16o = (r1--) + .707(r3--)				; 38-41		n 49
	zstore	[srcreg+64], zmm1		;; Save R9						; 38
	L1prefetchw srcreg+d4+d1+L1pd, L1pt

	zfmaddpd zmm12, zmm15, zmm30, zmm6	;; r4_12e/.383 = .924/.383(i2+-) + (r2--)		; 39-42		n 44
	zfnmaddpd zmm6, zmm6, zmm30, zmm15	;; r8_16e/.383 = (i2+-) - .924/.383(r2--)		; 39-42		n 49
	L1prefetchw srcreg+d4+d1+64+L1pd, L1pt

	zfmaddpd zmm15, zmm9, zmm31, zmm3	;; R3  = (r1+-+) + .707(r2+-+)				; 40-43		n 44
	zfnmaddpd zmm9, zmm9, zmm31, zmm3	;; R11 = (r1+-+) - .707(r2+-+)				; 40-43		n 46
	zstore	[srcreg+d4], zmm16		;; Save R5						; 40
	L1prefetchw srcreg+d4+d2+L1pd, L1pt

	zfnmaddpd zmm3, zmm10, zmm31, zmm5	;; R7  = (r1+--) - .707(r2+--)				; 41-44		n 45
	zfmaddpd zmm10, zmm10, zmm31, zmm5	;; R15 = (r1+--) + .707(r2+--)				; 41-44		n 49
	L1prefetchw srcreg+d4+d2+64+L1pd, L1pt

	zfmaddpd zmm5, zmm0, zmm29, zmm2	;; R2  = r2_10o + .383*r2_10e				; 42-45		n 47
	zfnmaddpd zmm0, zmm0, zmm29, zmm2	;; R10 = r2_10o - .383*r2_10e				; 42-45		n 49
	zstore	[srcreg+d4+64], zmm11		;; Save R13						; 42
	L1prefetchw srcreg+d4+d2+d1+L1pd, L1pt

	zfmaddpd zmm2, zmm13, zmm29, zmm8	;; R6  = r6_14o + .383*r6_14e				; 43-46		n 50
	zfnmaddpd zmm13, zmm13, zmm29, zmm8	;; R14 = r6_14o - .383*r6_14e				; 43-46		n 53
	L1prefetchw srcreg+d4+d2+d1+64+L1pd, L1pt

	vmulpd	zmm15{k3}, zmm15, zmm28		;; apply fudge multiplier for R3			; 44-47
	zfmaddpd zmm8, zmm12, zmm29, zmm14	;; R4  = r4_12o + .383*r4_12e				; 44-47		n 51

	kshiftrw k3, k3, 8			;; R11's fudge						; 45		n 46
	vmulpd	zmm3{k7}, zmm3, zmm28		;; apply fudge multiplier for R7			; 45-48

	kshiftrw k7, k7, 8			;; R15's fudge						; 46		n 49
	vmulpd	zmm9{k3}, zmm9, zmm28		;; apply fudge multiplier for R11			; 46-49

	zfnmaddpd zmm12, zmm12, zmm29, zmm14	;; R12 = r4_12o - .383*r4_12e				; 47-50		n 54
	vmulpd	zmm5{k2}, zmm5, zmm28		;; apply fudge multiplier for R2			; 47-50

	kshiftrw k2, k2, 8			;; R10's fudge						; 48		n 49
	zfmaddpd zmm14, zmm6, zmm29, zmm4	;; R8  = r8_16o + .383*r8_16e				; 48-51		n 52
	zstore	[srcreg+d2], zmm15		;; Save R3						; 48

	vmulpd	zmm10{k7}, zmm10, zmm28		;; apply fudge multiplier for R15			; 49-52
	vmulpd	zmm0{k2}, zmm0, zmm28		;; apply fudge multiplier for R10			; 49-52
	zstore	[srcreg+d4+d2], zmm3		;; Save R7						; 49

	zfnmaddpd zmm6, zmm6, zmm29, zmm4	;; R16 = r8_16o - .383*r8_16e				; 50-53		n 54
	vmulpd	zmm2{k6}, zmm2, zmm28		;; apply fudge multiplier for R6			; 50-53
 	zstore	[srcreg+d2+64], zmm9		;; Save R11						; 50

	kshiftrw k6, k6, 8			;; R14's fudge						; 51		n 53
	vmulpd	zmm8{k4}, zmm8, zmm28		;; apply fudge multiplier for R4			; 51-54
	zstore	[srcreg+d1], zmm5		;; Save R2						; 51

	kshiftrw k4, k4, 8			;; R12's fudge						; 52		n 54
	vmulpd	zmm14{k1}, zmm14, zmm28		;; apply fudge multiplier for R8			; 52-55

	kshiftrw k1, k1, 8			;; R16's fudge						; 53		n 54
	vmulpd	zmm13{k6}, zmm13, zmm28		;; apply fudge multiplier for R14			; 53-56
	zstore	[srcreg+d4+d2+64], zmm10	;; Save R15						; 53

	vmulpd	zmm12{k4}, zmm12, zmm28		;; apply fudge multiplier for R12			; 54-57
	vmulpd	zmm6{k1}, zmm6, zmm28		;; apply fudge multiplier for R16			; 54-57
	zstore	[srcreg+d1+64], zmm0		;; Save R10						; 53+1

	zstore	[srcreg+d4+d1], zmm2		;; Save R6						; 54+1
	zstore	[srcreg+d2+d1], zmm8		;; Save R4						; 55+1
	zstore	[srcreg+d4+d2+d1], zmm14	;; Save R8						; 56+1
	zstore	[srcreg+d2+d1+64], zmm12	;; Save R12						; 58+1
	zstore	[srcreg+d4+d1+64], zmm13	;; Save R14						; 57+1
	zstore	[srcreg+d4+d2+d1+64], zmm6	;; Save R16						; 58+2
	bump	srcreg, srcinc
	ENDM


;;
;; ********************************* reduced sin/cos sixteen-reals-fft8 and sixteen-reals-unfft8 variants **************************************
;;

;; These versions use registers for distances between blocks.  This lets us share pass1 code.
;; Weights must be applied.  The complex s/c data referenced by screg1 has already been weighted.
;; The real s/c data referenced by screg2 has not been weighted.

zr8_rsc_complex_and_real_preload MACRO
	mov	eax, 00110011b			;; For vblendmpd during swizzle
	kmovw	k7, eax
	vpmovzxbq zmm31, ZMM_PERMUTE1		;; zmm31 = 8+4 0+4 8+6 0+6 8+0 0+0 8+2 0+2 [msw at left]
	vpmovzxbq zmm30, ZMM_PERMUTE2		;; zmm30 = 8+5 0+5 8+7 0+7 8+1 0+1 8+3 0+3
	vbroadcastsd zmm29, ZMM_SQRTHALF
	vbroadcastsd zmm28, ZMM_P924_P383
	vbroadcastsd zmm27, ZMM_P383
	ENDM

zr8_rsc_wpn_sgreg_2sc_sixteen_reals_fft8_preload MACRO
	use zr8_rsc_complex_and_real_preload
	ENDM

zr8_rsc_wpn_sgreg_2sc_sixteen_reals_fft8 MACRO srcreg,srcinc,d1,d2,d4,dstreg,dstinc,e1reg,e3reg,dst4reg,screg1,scinc1,wgtreg,wgtinc,screg2,scinc2,maxrpt,L1pt,L1preg,L1p4reg
	vmovapd	zmm15, [srcreg+d4+d2]		;; r7_7	r7_6 r7_5 r7_4 r7_3 r7_2 r7_1 r7_0
	vmovapd	zmm26, [srcreg+d4+d2+d1]	;; r8_7	r8_6 r8_5 r8_4 r8_3 r8_2 r8_1 r8_0
	zperm2pd zmm14, zmm31, zmm15, zmm26	;; r8_4 r7_4 r8_6 r7_6 r8_0 r7_0 r8_2 r7_2		; 1-3		n 5

	zperm2pd zmm15, zmm30, zmm15, zmm26	;; r8_5 r7_5 r8_7 r7_7 r8_1 r7_1 r8_3 r7_3		; 2-4		n 7

	vmovapd	zmm13, [srcreg+d4]		;; r5_7	r5_6 r5_5 r5_4 r5_3 r5_2 r5_1 r5_0
	vmovapd	zmm26, [srcreg+d4+d1]		;; r6_7	r6_6 r6_5 r6_4 r6_3 r6_2 r6_1 r6_0
	vshufpd	zmm12, zmm13, zmm26, 00000000b	;; r6_6 r5_6 r6_4 r5_4 r6_2 r5_2 r6_0 r5_0		; 3		n 5

	vshufpd	zmm13, zmm13, zmm26, 11111111b	;; r6_7 r5_7 r6_5 r5_5 r6_3 r5_3 r6_1 r5_1		; 4		n 7

	vmovapd	zmm11, [srcreg+d2]		;; r3_7	r3_6 r3_5 r3_4 r3_3 r3_2 r3_1 r3_0
	vmovapd	zmm26, [srcreg+d2+d1]		;; r4_7	r4_6 r4_5 r4_4 r4_3 r4_2 r4_1 r4_0
	zperm2pd zmm10, zmm31, zmm11, zmm26	;; r4_4 r3_4 r4_6 r3_6 r4_0 r3_0 r4_2 r3_2		; 5-7		n 9
	vblendmpd zmm17{k7}, zmm14, zmm12	;; r8_4 r7_4 r6_4 r5_4 r8_0 r7_0 r6_0 r5_0		;  5		n 21

	zperm2pd zmm11, zmm30, zmm11, zmm26	;; r4_5 r3_5 r4_7 r3_7 r4_1 r3_1 r4_3 r3_3		; 6-8		n 11
	vblendmpd zmm12{k7}, zmm12, zmm14	;; r6_6 r5_6 r8_6 r7_6 r6_2 r5_2 r8_2 r7_2		;  6		n 23

	vmovapd	zmm9, [srcreg]			;; r1_7	r1_6 r1_5 r1_4 r1_3 r1_2 r1_1 r1_0
	vmovapd	zmm26, [srcreg+d1]		;; r2_7	r2_6 r2_5 r2_4 r2_3 r2_2 r2_1 r2_0
	vshufpd	zmm8, zmm9, zmm26, 00000000b	;; r2_6 r1_6 r2_4 r1_4 r2_2 r1_2 r2_0 r1_0		; 7		n 9
	vblendmpd zmm14{k7}, zmm15, zmm13	;; r8_5 r7_5 r6_5 r6_5 r8_1 r7_1 r6_1 r5_1		;  7		n 19

	vshufpd	zmm9, zmm9, zmm26, 11111111b	;; r2_7 r1_7 r2_5 r1_5 r2_3 r1_3 r2_1 r1_1		; 8		n 11
	vblendmpd zmm13{k7}, zmm13, zmm15	;; r6_7 r5_7 r8_7 r7_7 r6_3 r5_3 r8_3 r7_3		;  8		n 17

	vmovapd	zmm3, [srcreg+d2+64]		;; r11_7 r11_6 r11_5 r11_4 r11_3 r11_2 r11_1 r11_0
	vmovapd	zmm26, [srcreg+d2+d1+64]	;; r12_7 r12_6 r12_5 r12_4 r12_3 r12_2 r12_1 r12_0
	zperm2pd zmm2, zmm31, zmm3, zmm26	;; r12_4 r11_4 r12_6 r11_6 r12_0 r11_0 r12_2 r11_2	; 9-11		n 13
	vblendmpd zmm15{k7}, zmm10, zmm8	;; r4_4 r3_4 r2_4 r1_4 r4_0 r3_0 r2_0 r1_0		;  9		n 21

	zperm2pd zmm3, zmm30, zmm3, zmm26	;; r12_5 r11_5 r12_7 r11_7 r12_1 r11_1 r12_3 r11_3	; 10-12		n 15
	vblendmpd zmm8{k7}, zmm8, zmm10		;; r2_6 r1_6 r4_6 r3_6 r2_2 r1_2 r4_2 r3_2		;  10		n 23

	vmovapd	zmm0, [srcreg+64]		;; r9_7	r9_6 r9_5 r9_4 r9_3 r9_2 r9_1 r9_0
	vmovapd	zmm26, [srcreg+d1+64]		;; r10_7 r10_6 r10_5 r10_4 r10_3 r10_2 r10_1 r10_0
	vshufpd	zmm1, zmm0, zmm26, 11111111b	;; r10_7 r9_7 r10_5 r9_5 r10_3 r9_3 r10_1 r9_1		; 11		n 15
	vblendmpd zmm10{k7}, zmm11, zmm9	;; r4_5 r3_5 r2_5 r1_5 r4_1 r3_1 r2_1 r1_1		;  11		n 19

	vshufpd	zmm0, zmm0, zmm26, 00000000b	;; r10_6 r9_6 r10_4 r9_4 r10_2 r9_2 r10_0 r9_0		; 12		n 13
	vblendmpd zmm9{k7}, zmm9, zmm11		;; r2_7 r1_7 r4_7 r3_7 r2_3 r1_3 r4_3 r3_3		;  12		n 17

	vmovapd	zmm7, [srcreg+d4+d2+64]		;; r15_7 r15_6 r15_5 r15_4 r15_3 r15_2 r15_1 r15_0
	vmovapd	zmm26, [srcreg+d4+d2+d1+64]	;; r16_7 r16_6 r16_5 r16_4 r16_3 r16_2 r16_1 r16_0
	zperm2pd zmm6, zmm31, zmm7, zmm26	;; r16_4 r15_4 r16_6 r15_6 r16_0 r15_0 r16_2 r15_2	; 13-15		n 17
	vblendmpd zmm16{k7}, zmm2, zmm0		;; r12_4 r11_4 r10_4 r9_4 r12_0 r11_0 r10_0 r9_0	;  13		n 29

	zperm2pd zmm7, zmm30, zmm7, zmm26	;; r16_5 r15_5 r16_7 r15_7 r16_1 r15_1 r16_3 r15_3	; 14-16		n 19
	vblendmpd zmm0{k7}, zmm0, zmm2		;; r10_6 r9_6 r12_6 r11_6 r10_2 r9_2 r12_2 r11_2	;  14		n 31

	vmovapd	zmm5, [srcreg+d4+64]		;; r13_7 r13_6 r13_5 r13_4 r13_3 r13_2 r13_1 r13_0
	vmovapd	zmm26, [srcreg+d4+d1+64]	;; r14_7 r14_6 r14_5 r14_4 r14_3 r14_2 r14_1 r14_0
	vshufpd	zmm4, zmm5, zmm26, 00000000b	;; r14_6 r13_6 r14_4 r13_4 r14_2 r13_2 r14_0 r13_0	; 15		n 17
	vblendmpd zmm2{k7}, zmm3, zmm1		;; r12_5 r11_5 r10_5 r9_5 r12_1 r11_1 r10_1 r9_1	;  15		n 25

	vshufpd	zmm5, zmm5, zmm26, 11111111b	;; r14_7 r13_7 r14_5 r13_5 r14_3 r13_3 r14_1 r13_1	; 16		n 19
	vblendmpd zmm1{k7}, zmm1, zmm3		;; r10_7 r9_7 r12_7 r11_7 r10_3 r9_3 r12_3 r11_3	;  16		n 27
	vmovapd	zmm26, [wgtreg]			;; Load the weights to apply to screg2 data and R1A/R1B

	vshuff64x2 zmm11, zmm9, zmm13, 00010001b;; r8_3 r7_3 r6_3 r5_3 r4_3 r3_3 r2_3 r1_3 (R4)		; 17-19		n 21
	vblendmpd zmm3{k7}, zmm6, zmm4		;; r16_4 r15_4 r14_4 r13_4 r16_0 r15_0 r14_0 r13_0	;  17		n 29
	vmovapd	zmm25, [screg1+2*128+64]	;; cosine for w^4 (8-complex w^2)

	vshuff64x2 zmm9, zmm9, zmm13, 10111011b ;; r8_7 r7_7 r6_7 r5_7 r4_7 r3_7 r2_7 r1_7 (R8)		; 18-20		n 21
	vblendmpd zmm4{k7}, zmm4, zmm6		;; r14_6 r13_6 r16_6 r15_6 r14_2 r13_2 r16_2 r15_2	;  18		n 31
	vmovapd	zmm24, [screg1+1*128+64]	;; cosine for w^2 (8-complex w^1)

	vshuff64x2 zmm13, zmm10, zmm14, 01000100b ;; r8_1 r7_1 r6_1 r5_1 r4_1 r3_1 r2_1 r1_1 (R2)	; 19-21		n 23
	vblendmpd zmm6{k7}, zmm7, zmm5		;; r16_5 r15_5 r14_5 r13_5 r16_1 r15_1 r14_1 r13_1	;  19		n 25
	vmovapd	zmm23, [screg1+5*128+64]	;; cosine for w^10 (8-complex w^5)

	vshuff64x2 zmm10, zmm10, zmm14, 11101110b ;; r8_5 r7_5 r6_5 r5_5 r4_5 r3_5 r2_5 r1_5 (R6)	; 20-22		n 23
	vblendmpd zmm5{k7}, zmm5, zmm7		;; r14_7 r13_7 r16_7 r15_7 r14_3 r13_3 r16_3 r15_3	;  20		n 27
	vmovapd	zmm22, [screg1+2*128]		;; sine for w^4 (8-complex w^2)

	vshuff64x2 zmm14, zmm15, zmm17, 01000100b;; r8_0 r7_0 r6_0 r5_0 r4_0 r3_0 r2_0 r1_0 (R1)	; 21-23		n 25
	vshuff64x2 zmm15, zmm15, zmm17, 11101110b;; r8_4 r7_4 r6_4 r5_4 r4_4 r3_4 r2_4 r1_4 (R5)	; 22-24		n 25
	vmovapd	zmm21, [screg1+1*128]		;; sine for w^2 (8-complex w^1)

	vsubpd	zmm17, zmm11, zmm9		;; R4 - R8 (newer R8)					; 21-24		n 29
	vaddpd	zmm11, zmm11, zmm9		;; R4 + R8 (newer R4)					; 22-25		n 31
	bump	srcreg, srcinc

	vshuff64x2 zmm9, zmm8, zmm12, 00010001b ;; r8_2 r7_2 r6_2 r5_2 r4_2 r3_2 r2_2 r1_2 (R3)		; 23-25		n 27
	vshuff64x2 zmm8, zmm8, zmm12, 10111011b ;; r8_6 r7_6 r6_6 r5_6 r4_6 r3_6 r2_6 r1_6 (R7)		; 24-26		n 27
	bump	wgtreg, wgtinc

	vsubpd	zmm12, zmm13, zmm10		;; R2 - R6 (newer R6)					; 23-26		n 29
	vaddpd	zmm13, zmm13, zmm10		;; R2 + R6 (newer R2)					; 24-27		n 31
	L1prefetchw dstreg, L1pt - L1PREFETCH_DEST_NONE

	vshuff64x2 zmm7, zmm2, zmm6, 01000100b	;; r16_1 r15_1 r14_1 r13_1 r12_1 r11_1 r10_1 r9_1 (R10)	; 25-27		n 36
	vshuff64x2 zmm2, zmm2, zmm6, 11101110b	;; r16_5 r15_5 r14_5 r13_5 r12_5 r11_5 r10_5 r9_5 (R14)	; 26-28		n 36
	L1prefetchw dstreg+64, L1pt - L1PREFETCH_DEST_NONE

	vaddpd	zmm10, zmm14, zmm15		;; R1 + R5 (newer R1)					; 25-28		n 33
	vsubpd	zmm14, zmm14, zmm15		;; R1 - R5 (newer R5)					; 26-29		n 34
	L1prefetchw dstreg+e1reg, L1pt - L1PREFETCH_DEST_NONE

	vshuff64x2 zmm6, zmm1, zmm5, 00010001b	;; r16_3 r15_3 r14_3 r13_3 r12_3 r11_3 r10_3 r9_3 (R12)	; 27-29		n 37
	vshuff64x2 zmm1, zmm1, zmm5, 10111011b	;; r16_7 r15_7 r14_7 r13_7 r12_7 r11_7 r10_7 r9_7 (R16)	; 28-30		n 37
	L1prefetchw dstreg+e1reg+64, L1pt - L1PREFETCH_DEST_NONE

	vaddpd	zmm15, zmm9, zmm8		;; R3 + R7 (newer R3)					; 27-30		n 33
	vsubpd	zmm9, zmm9, zmm8		;; R3 - R7 (newer R7)					; 28-31		n 35
	L1prefetchw dstreg+2*e1reg, L1pt - L1PREFETCH_DEST_NONE

	vshuff64x2 zmm5, zmm16, zmm3, 01000100b;; r16_0 r15_0 r14_0 r13_0 r12_0 r11_0 r10_0 r9_0 (R9)	; 29-31		n 45
	vshuff64x2 zmm16, zmm16, zmm3, 11101110b;; r16_4 r15_4 r14_4 r13_4 r12_4 r11_4 r10_4 r9_4 (R13)	; 30-32		n 46
	L1prefetchw dstreg+2*e1reg+64, L1pt - L1PREFETCH_DEST_NONE

						;; R6/R8 becomes newest R6/I6
						;; mul R6/I6 by w^2 = .707 + .707i
	vsubpd	zmm8, zmm12, zmm17		;; R6 - I6 (newest R6/SQRTHALF)				; 29-32		n 34
	vaddpd	zmm12, zmm12, zmm17		;; R6 + I6 (newest I6/SQRTHALF)				; 30-33		n 35
	L1prefetchw dstreg+e3reg, L1pt - L1PREFETCH_DEST_NONE

	vshuff64x2 zmm3, zmm0, zmm4, 00010001b	;; r16_2 r15_2 r14_2 r13_2 r12_2 r11_2 r10_2 r9_2 (R11)	; 31-33		n 38
	vshuff64x2 zmm0, zmm0, zmm4, 10111011b	;; r16_6 r15_6 r14_6 r13_6 r12_6 r11_6 r10_6 r9_6 (R15)	; 32-34		n 38
	L1prefetchw dstreg+e3reg+64, L1pt - L1PREFETCH_DEST_NONE

	vsubpd	zmm17, zmm13, zmm11		;; R2 - R4 (newest R4)					; 31-34		n 40
	vaddpd	zmm13, zmm13, zmm11		;; R2 + R4 (newest R2)					; 32-35		n 44
	L1prefetchw dst4reg, L1pt - L1PREFETCH_DEST_NONE


 	vaddpd	zmm11, zmm10, zmm15		;; R1 + R3 (newest R1)					; 33-36		n 37
	vsubpd	zmm10, zmm10, zmm15		;; R1 - R3 (newest R3)					; 33-36		n 40
	L1prefetchw dst4reg+64, L1pt - L1PREFETCH_DEST_NONE
						;; R3/R4 becomes last R3/I3

						;; R5/R7 becomes newest R5/I5
	zfmaddpd zmm15, zmm8, zmm29, zmm14	;; R5 + R6 * SQRTHALF (last R5)				; 34-37		n 42
	zfnmaddpd zmm8, zmm8, zmm29, zmm14	;; R5 - R6 * SQRTHALF (last R6)				; 34-37		n 43
	L1prefetchw dst4reg+e1reg, L1pt - L1PREFETCH_DEST_NONE

	zfmaddpd zmm14, zmm12, zmm29, zmm9	;; I5 + I6 * SQRTHALF (last I5)				; 35-38		n 42
	zfnmaddpd zmm12, zmm12, zmm29, zmm9	;; I5 - I6 * SQRTHALF (last I6)				; 35-38		n 43
	L1prefetchw dst4reg+e1reg+64, L1pt - L1PREFETCH_DEST_NONE

						;; R10/R14 becomes newer R10/I10
						;; mul R10/I10 by w^1 = .924 + .383i
	zfmsubpd zmm4, zmm7, zmm28, zmm2	;; R10*.924/.383 - I10 (newer R10/.383)			; 36-39		n 41
	zfmaddpd zmm2, zmm2, zmm28, zmm7	;; R10 + I10*.924/.383 (newer I10/.383)			; 36-39		n 41
	L1prefetchw dst4reg+2*e1reg, L1pt - L1PREFETCH_DEST_NONE

						;; R1/R2 becomes last & final R1A and R1B
	vmulpd	zmm18, zmm27, zmm26		;; Weight .383						; 37-40		n 41
	vmulpd	zmm11, zmm11, zmm26		;; Weight R1A						; 37-40		n 44
	L1prefetchw dst4reg+2*e1reg+64, L1pt - L1PREFETCH_DEST_NONE

						;; R12/R16 becomes newer R12/I12
						;; mul R12/I12 by w^3 = .383 + .924i
	zfnmaddpd zmm7, zmm1, zmm28, zmm6	;; R12 - I12*.924/.383 (newer R12/.383)			; 38-41		n 45
	zfmaddpd zmm6, zmm6, zmm28, zmm1	;; R12*.924/.383 + I12 (newer I12/.383)			; 38-41		n 46
	L1prefetchw dst4reg+e3reg, L1pt - L1PREFETCH_DEST_NONE

						;; R11/R15 becomes newer R11/I11
						;; mul R11/I11 by SQRTHALF + i*SQRTHALF
	vsubpd	zmm1, zmm3, zmm0		;; R11 - I11 (newer R11/SQRTHALF)			; 39-42		n 47
	vaddpd	zmm3, zmm3, zmm0		;; R11 + I11 (newer I11/SQRTHALF)			; 39-42		n 48
	L1prefetchw dst4reg+e3reg+64, L1pt - L1PREFETCH_DEST_NONE

	vmulpd	zmm9, zmm10, zmm25		;; A3 = R3 * cosine					; 40-43		n 49
	vmulpd	zmm25, zmm17, zmm25		;; B3 = I3 * cosine					; 40-43		n 49

	vmulpd	zmm4, zmm4, zmm18		;; R10 = R10 * .383*wgt					; 41-44		n 45
	vmulpd	zmm2, zmm2, zmm18		;; I10 = I10 * .383*wgt					; 41-44		n 46

	vmulpd	zmm19, zmm15, zmm24		;; A5 = R5 * cosine					; 42-45		n 50
	vmulpd	zmm24, zmm14, zmm24		;; B5 = I5 * cosine					; 42-45		n 50

	vmulpd	zmm20, zmm8, zmm23		;; A6 = R6 * cosine					; 43-46		n 51
	vmulpd	zmm23, zmm12, zmm23		;; B6 = I6 * cosine					; 43-46		n 51

	zfmaddpd zmm0, zmm13, zmm26, zmm11	;; R1A + wgt * R1B (final R1A)				; 44-47
	zfnmaddpd zmm13, zmm13, zmm26, zmm11	;; R1A - wgt * R1B (final R1B)				; 44-47

	zfmaddpd zmm11, zmm7, zmm18, zmm4	;; R10 + R12 * .383*wgt (newest R10*wgt)		; 45-48		n 52
	zfnmaddpd zmm7, zmm7, zmm18, zmm4	;; R10 - R12 * .383*wgt (newest R12*wgt)		; 45-48		n 54

	zfmaddpd zmm4, zmm6, zmm18, zmm2	;; I10 + I12 * .383*wgt (newest I10*wgt)		; 46-49		n 52
	zfnmaddpd zmm6, zmm6, zmm18, zmm2	;; I10 - I12 * .383*wgt (newest I12*wgt)		; 46-49		n 54
	vmovapd	zmm18, [screg1+5*128]		;; sine for w^10 (8-complex w^5)

						;; R9/R13 becomes newer R9/I9
	zfmaddpd zmm2, zmm1, zmm29, zmm5	;; R9 + R11 * SQRTHALF (newest R9)			; 47-50		n 52
	zfnmaddpd zmm1, zmm1, zmm29, zmm5	;; R9 - R11 * SQRTHALF (newest R11)			; 47-50		n 54

	zfmaddpd zmm5, zmm3, zmm29, zmm16	;; I9 + I11 * SQRTHALF (newest I9)			; 48-51		n 52
	zfnmaddpd zmm3, zmm3, zmm29, zmm16	;; I9 - I11 * SQRTHALF (newest I11)			; 48-51		n 54
	vmovapd	zmm16, [screg2+0*128+64]	;; cosine/sine for w^1
	zstore	[dstreg], zmm0			;; Save R1A						; 48

	zfnmaddpd zmm17, zmm17, zmm22, zmm9	;; A3 - I3 * sine (final R3)				; 49-52
	zfmaddpd zmm10, zmm10, zmm22, zmm25	;; B3 + R3 * sine (final I3)				; 49-52
	vmovapd	zmm22, [screg2+2*128+64]	;; cosine/sine for w^9
	zstore	[dstreg+64], zmm13		;; Save R1B						; 48+1

	zfnmaddpd zmm14, zmm14, zmm21, zmm19	;; A5 - I5 * sine (final R5)				; 50-53
	zfmaddpd zmm15, zmm15, zmm21, zmm24	;; B5 + R5 * sine (final I5)				; 50-53
	vmovapd	zmm25, [screg2+1*128+64]	;; cosine/sine for w^5

	zfnmaddpd zmm12, zmm12, zmm18, zmm20	;; A6 - I6 * sine (final R6)				; 51-54
	zfmaddpd zmm8, zmm8, zmm18, zmm23	;; B6 + R6 * sine (final I6)				; 51-54
	vmovapd	zmm13, [screg2+3*128+64]	;; cosine/sine for w^13

	zfmaddpd zmm9, zmm2, zmm26, zmm11	;; R9 * wgt + R10 (last R9)				; 52-55		n 56
	zfmaddpd zmm0, zmm5, zmm26, zmm4	;; I9 * wgt + I10 (last I9)				; 52-55		n 56
	vmovapd	zmm19, [screg2+0*128]		;; sine for w^1

	zfmsubpd zmm2, zmm2, zmm26, zmm11	;; R9 * wgt - R10 (last R10)				; 53-56		n 57
	zfmsubpd zmm5, zmm5, zmm26, zmm4	;; I9 * wgt - I10 (last I10)				; 53-56		n 57
	vmovapd	zmm21, [screg2+2*128]		;; sine for w^9
	zstore	[dstreg+e1reg], zmm17		;; Save R2 (final R3)					; 53

	zfmsubpd zmm11, zmm1, zmm26, zmm6	;; R11 * wgt - I12 (last R11)				; 54-57		n 58
	zfmaddpd zmm4, zmm3, zmm26, zmm7	;; I11 * wgt + R12 (last I11)				; 54-57		n 58
	vmovapd	zmm24, [screg2+1*128]		;; sine for w^5
	zstore	[dstreg+e1reg+64], zmm10	;; Save I2 (final I3)					; 53+1

	zfmaddpd zmm1, zmm1, zmm26, zmm6	;; R11 * wgt + I12 (last R12)				; 55-58		n 59
	zfmsubpd zmm3, zmm3, zmm26, zmm7	;; I11 * wgt - R12 (last I12)				; 55-58		n 59
	vmovapd	zmm20, [screg2+3*128]		;; sine for w^13
	zstore	[dstreg+2*e1reg], zmm14		;; Save R3 (final R5)					; 54+1

	zfmsubpd zmm6, zmm9, zmm16, zmm0	;; A9 = R9 * cosine/sine - I9				; 56-59		n 60
	zfmaddpd zmm0, zmm0, zmm16, zmm9	;; B9 = I9 * cosine/sine + R9				; 56-59		n 60
	bump	screg1, scinc1
	zstore	[dstreg+2*e1reg+64], zmm15	;; Save I3 (final I5)					; 54+2

	zfmsubpd zmm9, zmm2, zmm22, zmm5	;; A10 = R10 * cosine/sine - I10			; 57-60		n 61
	zfmaddpd zmm5, zmm5, zmm22, zmm2	;; B10 = I10 * cosine/sine + R10			; 57-60		n 61
	bump	screg2, scinc2
	zstore	[dstreg+e3reg], zmm12		;; Save R4 (final R6)					; 55+2

	zfmsubpd zmm2, zmm11, zmm25, zmm4	;; A11 = R11 * cosine/sine - I11			; 58-61		n 62
	zfmaddpd zmm4, zmm4, zmm25, zmm11	;; B11 = I11 * cosine/sine + R11			; 58-61		n 62
	zstore	[dstreg+e3reg+64], zmm8		;; Save I4 (final I6)					; 55+3

	zfmsubpd zmm11, zmm1, zmm13, zmm3	;; A12 = R12 * cosine/sine - I12			; 59-62		n 63
	zfmaddpd zmm3, zmm3, zmm13, zmm1	;; B12 = I12 * cosine/sine + R12			; 59-62		n 63

	vmulpd	zmm6, zmm6, zmm19		;; A9 * sine (final R9)					; 60-63
	vmulpd	zmm0, zmm0, zmm19		;; B9 * sine (final I9)					; 60-63

	vmulpd	zmm9, zmm9, zmm21		;; A10 * sine (final R10)				; 61-64
	vmulpd	zmm5, zmm5, zmm21		;; B10 * sine (final I10)				; 61-64

	vmulpd	zmm2, zmm2, zmm24		;; A11 * sine (final R11)				; 62-65
	vmulpd	zmm4, zmm4, zmm24		;; B11 * sine (final I11)				; 62-65

	vmulpd	zmm11, zmm11, zmm20		;; A12 * sine (final R12)				; 63-66
	vmulpd	zmm3, zmm3, zmm20		;; B12 * sine (final I12)				; 63-66

	zstore	[dst4reg], zmm6			;; Save R5 (final R9)					; 64
	zstore	[dst4reg+64], zmm0		;; Save I5 (final I9)					; 64+1
	zstore	[dst4reg+e1reg], zmm9		;; Save R6 (final R10)					; 65+1
	zstore	[dst4reg+e1reg+64], zmm5	;; Save I6 (final I10)					; 65+2
	zstore	[dst4reg+2*e1reg], zmm2		;; Save R7 (final R11)					; 66+2
	zstore	[dst4reg+2*e1reg+64], zmm4	;; Save I7 (final I11)					; 66+3
	zstore	[dst4reg+e3reg], zmm11		;; Save R8 (final R12)					; 67+3
	zstore	[dst4reg+e3reg+64], zmm3	;; Save I8 (final I12)					; 67+4
	bump	dstreg, dstinc
	bump	dst4reg, dstinc
	ENDM


;; These versions use registers for distances between blocks.  This lets us share pass1 code.
;; Weights must be applied.  The complex s/c data referenced by screg1 has already been weighted.
;; The real s/c data referenced by screg2 has not been weighted.

zr8_rsc_wpn_sgreg_2sc_sixteen_reals_unfft8_preload MACRO
	use zr8_rsc_complex_and_real_preload
	ENDM

zr8_rsc_wpn_sgreg_2sc_sixteen_reals_unfft8 MACRO srcreg,srcinc,d1reg,d3reg,src4reg,dstreg,dstinc,e1,e2,e4,screg1,scinc1,wgtreg,wgtinc,screg2,scinc2,maxrpt,L1pt,L1preg,L1p4reg
	vmovapd	zmm24, [screg1+1*128+64]	;; cosine for w^2 (8-complex w^1)
	vmovapd zmm2, [srcreg+2*d1reg]		;; Load R3 (aka R5)
	vmovapd zmm10, [srcreg+2*d1reg+64]	;; Load I3 (aka I5)
	vmulpd	zmm16, zmm2, zmm24		;; A5 = R5 * cosine					; 1-4		n 7
	vmulpd	zmm17, zmm10, zmm24		;; B5 = I5 * cosine					; 2-5		n 8

	vmovapd	zmm24, [screg1+5*128+64]	;; cosine for w^10 (8-complex w^5)
	vmovapd zmm3, [srcreg+d3reg]		;; Load R4 (aka R6)
	vmovapd zmm11, [srcreg+d3reg+64]	;; Load I4 (aka I6)
	vmulpd	zmm18, zmm3, zmm24		;; A6 = R6 * cosine					; 2-5		n 8
	vmulpd	zmm19, zmm11, zmm24		;; B6 = I6 * cosine					; 3-6		n 9

	vmovapd	zmm24, [screg2+0*128+64]	;; cosine/sine for w^1
	vmovapd zmm4, [src4reg]			;; Load R5 (aka R9)
	vmovapd zmm12, [src4reg+64]		;; Load I5 (aka I9)
	zfmaddpd zmm20, zmm4, zmm24, zmm12	;; A9 = R9 * cosine/sine + I9				; 3-6		n 10
	zfmsubpd zmm12, zmm12, zmm24, zmm4	;; B9 = I9 * cosine/sine - R9				; 4-7		n 11

	vmovapd	zmm24, [screg2+1*128+64]	;; cosine/sine for w^5
	vmovapd zmm6, [src4reg+2*d1reg]		;; Load R7 (aka R11)
	vmovapd zmm14, [src4reg+2*d1reg+64]	;; Load I7 (aka I11)
	zfmaddpd zmm4, zmm6, zmm24, zmm14	;; A11 = R11 * cosine/sine + I11			; 4-7		n 12
	zfmsubpd zmm14, zmm14, zmm24, zmm6	;; B11 = I11 * cosine/sine - R11			; 5-8		n 13

	vmovapd	zmm24, [screg1+2*128+64]	;; cosine for w^4 (8-complex w^2)
	vmovapd zmm1, [srcreg+d1reg]		;; Load R2 (aka R3)
	vmovapd zmm9, [srcreg+d1reg+64]		;; Load I2 (aka I3)
	vmulpd	zmm6, zmm1, zmm24		;; A3 = R3 * cosine					; 5-8		n 13
	vmulpd	zmm21, zmm9, zmm24		;; B3 = I3 * cosine					; 6-9		n 14

	vmovapd	zmm26, [wgtreg]			;; Load the weights to apply to screg2 data and R1/I1
	vmulpd	zmm25, zmm26, [screg2+1*128]	;; weighted sine for w^5				; 6-9		n 12
	vmulpd	zmm22, zmm26, [screg2+3*128]	;; weighted sine for w^13				; 7-10		n 17

	vmovapd zmm24, [screg1+1*128]		;; sine for w^2 (8-complex w^1)
	zfmaddpd zmm16, zmm10, zmm24, zmm16	;; A5 + I5 * sine (first R5)				; 7-10		n 14
	zfnmaddpd zmm17, zmm2, zmm24, zmm17	;; B5 - R5 * sine (first I5)				; 8-11		n 15

	vmovapd	zmm24, [screg1+5*128]		;; sine for w^10 (8-complex w^5)
	zfmaddpd zmm18, zmm11, zmm24, zmm18	;; A6 + I6 * sine (first R6)				; 8-11		n 14
	zfnmaddpd zmm19, zmm3, zmm24, zmm19	;; B6 - R6 * sine (first I6)				; 9-12		n 15

	vmovapd	zmm24, [screg2+2*128+64]	;; cosine/sine for w^9
	vmovapd zmm5, [src4reg+d1reg]		;; Load R6 (aka R10)
	vmovapd zmm13, [src4reg+d1reg+64]	;; Load I6 (aka I10)
	zfmaddpd zmm10, zmm5, zmm24, zmm13	;; A10 = R10 * cosine/sine + I10			; 9-12		n 16
	zfmsubpd zmm13, zmm13, zmm24, zmm5	;; B10 = I10 * cosine/sine - R10			; 10-13		n 18

	vmovapd zmm24, [screg2+0*128]		;; sine for w^1
	vmulpd	zmm20, zmm20, zmm24		;; A9 * sine (first R9/weight)				; 10-13		n 16
	vmulpd	zmm12, zmm12, zmm24		;; B9 * sine (first I9/weight)				; 11-14		n 18

	vmovapd	zmm24, [screg2+3*128+64]	;; cosine/sine for w^13
	vmovapd zmm7, [src4reg+d3reg]		;; Load R8 (aka R12)
	vmovapd zmm15, [src4reg+d3reg+64]	;; Load I8 (aka I12)
	zfmaddpd zmm2, zmm7, zmm24, zmm15	;; A12 = R12 * cosine/sine + I12			; 11-14		n 17
	zfmsubpd zmm15, zmm15, zmm24, zmm7	;; B12 = I12 * cosine/sine - R12			; 12-15		n 19

	vmulpd	zmm4, zmm4, zmm25		;; A11 * sine (first R11)				; 12-15		n 17
	vmulpd	zmm14, zmm14, zmm25		;; B11 * sine (first I11)				; 13-16		n 19

	vmovapd	zmm24, [screg1+2*128]		;; sine for w^4 (8-complex w^2)
	zfmaddpd zmm6, zmm9, zmm24, zmm6	;; A3 + I3 * sine (first R3)				; 13-16		n 20
	zfnmaddpd zmm21, zmm1, zmm24, zmm21	;; B3 - R3 * sine (first I3)				; 14-17		n 21

						;; R1/I1 becomes R1/R2
						;; R3/I3 becomes R3/R4

	vmovapd zmm23, [screg2+2*128]		;; sine for w^9
	vsubpd	zmm11, zmm16, zmm18		;; R5 - R6 (new R6)					; 14-17		n 22
	vaddpd	zmm16, zmm16, zmm18		;; R5 + R6 (new R5)					; 15-18		n 28

	vmovapd zmm0, [srcreg]			;; Load R1
	vsubpd	zmm3, zmm17, zmm19		;; I5 - I6 (new I6)					; 15-18		n 22
	vaddpd	zmm17, zmm17, zmm19		;; I5 + I6 (new I5)					; 16-19		n 26

	vmovapd zmm8, [srcreg+64]		;; Load I1
	zfmaddpd zmm5, zmm10, zmm23, zmm20	;; R9 + R10 * sine (new R9/weight)			; 16-19		n 23
	zfnmaddpd zmm10, zmm10, zmm23, zmm20	;; R9 - R10 * sine (new R10/weight)			; 17-20		n 24

	L1prefetch L1preg, L1pt
	zfmaddpd zmm7, zmm2, zmm22, zmm4	;; R12 * sine + R11 (new R11)				; 17-20		n 23
	zfmsubpd zmm2, zmm2, zmm22, zmm4	;; R12 * sine - R11 (new I12)				; 18-21		n 25

	L1prefetch L1preg+64, L1pt
	zfmaddpd zmm1, zmm15, zmm22, zmm14	;; I11 + I12 * sine (new I11)				; 18-21		n 24
	zfnmaddpd zmm15, zmm15, zmm22, zmm14	;; I11 - I12 * sine (new R12)				; 19-22		n 24

	L1prefetch L1preg+d1reg, L1pt
	zfmaddpd zmm9, zmm13, zmm23, zmm12	;; I9 + I10 * sine (new I9/weight)			; 19-22		n 24
	zfnmaddpd zmm13, zmm13, zmm23, zmm12	;; I9 - I10 * sine (new I10/weight)			; 20-23		n 25

	L1prefetch L1preg+d1reg+64, L1pt
	zfmsubpd zmm4, zmm0, zmm26, zmm6	;; R1*weight - R3 (new R3)				; 20-23		n 26
	zfmaddpd zmm0, zmm0, zmm26, zmm6	;; R1*weight + R3 (new R1)				; 21-24		n 28

	L1prefetch L1preg+2*d1reg, L1pt
	zfmsubpd zmm12, zmm8, zmm26, zmm21	;; R2*weight - R4 (new R4)				; 21-24		n 27
	zfmaddpd zmm8, zmm8, zmm26, zmm21	;; R2*weight + R4 (new R2)				; 22-25		n 29

						;; R5/I5 becomes newer R5/R7

	L1prefetch L1preg+2*d1reg+64, L1pt
						;; mul R6/I6 by w^2 = .707 - .707i
	vsubpd	zmm14, zmm3, zmm11		;; I6 / SQRTHALF = I6 - R6				; 22-25		n 27
	vaddpd	zmm3, zmm3, zmm11		;; R6 / SQRTHALF = I6 + R6				; 23-26		n 29
						;; R6/I6 becomes newer R6/R8

	L1prefetch L1preg+d3reg, L1pt
	zfmsubpd zmm6, zmm5, zmm26, zmm7	;; R9*weight - R11 (newer R11)				; 23-26		n 28
	zfmsubpd zmm11, zmm9, zmm26, zmm1	;; I9*weight - I11 (newer I11)				; 24-27		n 28

	L1prefetch L1preg+d3reg+64, L1pt
	zfmsubpd zmm18, zmm10, zmm26, zmm15	;; R10*weight - R12 (newer R12)				; 24-27		n 29
	zfmsubpd zmm19, zmm13, zmm26, zmm2	;; I10*weight - I12 (newer I12)				; 25-28		n 29

	L1prefetch L1p4reg, L1pt
	zfmaddpd zmm10, zmm10, zmm26, zmm15	;; R10*weight + R12 (newer R10)				; 25-28		n 30
	zfmaddpd zmm13, zmm13, zmm26, zmm2	;; I10*weight + I12 (newer I10)				; 26-29		n 30

	L1prefetch L1p4reg+64, L1pt
	vaddpd	zmm20, zmm4, zmm17		;; R3 + R7 (newer R3)					; 26-29		n 32
	zfmaddpd zmm21, zmm14, zmm29, zmm12	;; R4 + R8 * SQRTHALF (newer R4)			; 27-30		n 33

	L1prefetch L1p4reg+d1reg, L1pt
	zfmaddpd zmm5, zmm5, zmm26, zmm7	;; R9*weight + R11 (newer R9)				; 27-30		n 33
	vaddpd	zmm7, zmm0, zmm16		;; R1 + R5 (newer R1)					; 28-31		n 33

	L1prefetch L1p4reg+d1reg+64, L1pt
						;; mul R11/I11 by SQRTHALF - i*SQRTHALF
	vaddpd	zmm22, zmm11, zmm6		;; Twiddled R11 / SQRTHALF = I11 + R11			; 28-31		n 32
						;; mul R12/I12 by w^3 = .383 - .924i
	zfmaddpd zmm23, zmm19, zmm28, zmm18	;; Twiddled R12/.383 = R12 + I12*.924/.383		; 29-32		n 33

	L1prefetch L1p4reg+2*d1reg, L1pt
	zfmaddpd zmm15, zmm3, zmm29, zmm8	;; R2 + R6 * SQRTHALF (newer R2)			; 29-32		n 34
						;; mul R10/I10 by w^1 = .924 - .383i
	zfmaddpd zmm2, zmm10, zmm28, zmm13	;; Twiddled R10/.383 = R10*.924/.383 + I10		; 30-33		n 34

	L1prefetch L1p4reg+2*d1reg+64, L1pt
	vsubpd	zmm4, zmm4, zmm17		;; R3 - R7 (newer R7)					; 30-33		n 40
	vsubpd	zmm11, zmm11, zmm6		;; Twiddled I11 / SQRTHALF = I11 - R11			; 31-34		n 40
						;; R11/I11 becomes newer R11/R15

	L1prefetch L1p4reg+d3reg, L1pt
	zfnmaddpd zmm14, zmm14, zmm29, zmm12	;; R4 - R8 * SQRTHALF (newer R8)			; 31-34		n 41
	zfnmaddpd zmm18, zmm18, zmm28, zmm19	;; Twiddled I12/.383 = I12 - R12*.924/.383		; 32-35		n 41
						;; R12/I12 becomes newer R12/R16

	L1prefetch L1p4reg+d3reg+64, L1pt
	zfmaddpd zmm17, zmm22, zmm29, zmm20	;; R3 + R11 * SQRTHALF (last R3)			; 32-35		n 37
	zfmaddpd zmm6, zmm23, zmm27, zmm21	;; R4 + R12*.383 (last R4)				; 33-36		n 37

	vaddpd	zmm12, zmm7, zmm5		;; R1 + R9 (last R1)					; 33-36		n 39
	zfmaddpd zmm19, zmm2, zmm27, zmm15	;; R2 + R10*.383 (last R2)				; 34-37		n 39
	bump	srcreg, srcinc

	zfnmaddpd zmm22, zmm22, zmm29, zmm20	;; R3 - R11 * SQRTHALF (last R11)			; 34-37		n 41
	zfnmaddpd zmm23, zmm23, zmm27, zmm21	;; R4 - R12*.383 (last R12)				; 35-38		n 41
	bump	src4reg, srcinc

	vsubpd	zmm0, zmm0, zmm16		;; R1 - R5 (newer R5)					; 35-38		n 42
	zfmaddpd zmm9, zmm9, zmm26, zmm1	;; I9*weight + I11 (newer I9)				; 36-39		n 42
						;; R9/I9 becomes newer R9/R13

	vsubpd	zmm7, zmm7, zmm5		;; R1 - R9 (last R9)					; 36-39		n 43
	bump	wgtreg, wgtinc

	zperm2pd zmm16, zmm31, zmm17, zmm6	;; r4_4 r3_4 r4_6 r3_6 r4_0 r3_0 r4_2 r3_2		; 37-39		n 48
	zfnmaddpd zmm2, zmm2, zmm27, zmm15	;; R2 - R10*.383 (last R10)				; 37-40		n 43
	bump	screg1, scinc1

	zperm2pd zmm17, zmm30, zmm17, zmm6	;; r4_5 r3_5 r4_7 r3_7 r4_1 r3_1 r4_3 r3_3		; 38-40		n 50
	zfnmaddpd zmm3, zmm3, zmm29, zmm8	;; R2 - R6 * SQRTHALF (newer R6)			; 38-41		n 43
	bump	screg2, scinc2

	vshufpd	zmm6, zmm12, zmm19, 00000000b	;; r2_6 r1_6 r2_4 r1_4 r2_2 r1_2 r2_0 r1_0		; 39		n 48
	zfmsubpd zmm13, zmm13, zmm28, zmm10	;; Twiddled I10/.383 = I10*.924/.383 - R10		; 39-42		n 43
						;; R10/I10 becomes newer R10/R14
	bump	L1preg, srcinc

	vshufpd	zmm12, zmm12, zmm19, 11111111b	;; r2_7 r1_7 r2_5 r1_5 r2_3 r1_3 r2_1 r1_1		; 40		n 50
	zfmaddpd zmm5, zmm11, zmm29, zmm4	;; R7 + R15 * SQRTHALF (last R7)			;  40-43	n 45
	bump	L1p4reg, srcinc

	zperm2pd zmm19, zmm31, zmm22, zmm23	;; i4_4 i3_4 i4_6 i3_6 i4_0 i3_0 i4_2 i3_2		; 41-43		n 56
	zfmaddpd zmm15, zmm18, zmm27, zmm14	;; R8 + R16*.383 (last R8)				;  41-44	n 45

	zperm2pd zmm22, zmm30, zmm22, zmm23	;; i4_5 i3_5 i4_7 i3_7 i4_1 i3_1 i4_3 i3_3		; 42-44		n 58
	vaddpd	zmm8, zmm0, zmm9		;; R5 + R13 (last R5)					;  42-45	n 47

	vshufpd	zmm23, zmm7, zmm2, 00000000b	;; i2_6 i1_6 i2_4 i1_4 i2_2 i1_2 i2_0 i1_0		; 43		n 56
	zfmaddpd zmm10, zmm13, zmm27, zmm3	;; R6 + R14*.383 (last R6)				;  43-46	n 47

	vshufpd	zmm7, zmm7, zmm2, 11111111b	;; i2_7 i1_7 i2_5 i1_5 i2_3 i1_3 i2_1 i1_1		; 44		n 58
	zfnmaddpd zmm11, zmm11, zmm29, zmm4	;; R7 - R15 * SQRTHALF (last R15)			;  44-47	n 49

	zperm2pd zmm2, zmm31, zmm5, zmm15	;; r8_4 r7_4 r8_6 r7_6 r8_0 r7_0 r8_2 r7_2		; 45-47		n 52
	zfnmaddpd zmm18, zmm18, zmm27, zmm14	;; R8 - R16*.383 (last R16)				;  45-48	n 49

	zperm2pd zmm5, zmm30, zmm5, zmm15	;; r8_5 r7_5 r8_7 r7_7 r8_1 r7_1 r8_3 r7_3		; 46-48		n 54
	vsubpd	zmm0, zmm0, zmm9		;; R5 - R13 (last R13)					;  46-49	n 51

	vshufpd	zmm15, zmm8, zmm10, 00000000b	;; r6_6 r5_6 r6_4 r5_4 r6_2 r5_2 r6_0 r5_0		; 47		n 52
	zfnmaddpd zmm13, zmm13, zmm27, zmm3	;; R6 - R14*.383 (last R14)				;  47-50	n 51

	vshufpd	zmm8, zmm8, zmm10, 11111111b	;; r6_7 r5_7 r6_5 r5_5 r6_3 r5_3 r6_1 r5_1		; 48		n 54
	vblendmpd zmm9{k7}, zmm16, zmm6		;; r4_4 r3_4 r2_4 r1_4 r4_0 r3_0 r2_0 r1_0		;  48		n 53

	zperm2pd zmm10, zmm31, zmm11, zmm18	;; i8_4 i7_4 i8_6 i7_6 i8_0 i7_0 i8_2 i7_2		; 49-51		n 60
	vblendmpd zmm6{k7}, zmm6, zmm16		;; r2_6 r1_6 r4_6 r3_6 r2_2 r1_2 r4_2 r3_2		;  49		n 55

	zperm2pd zmm11, zmm30, zmm11, zmm18	;; i8_5 i7_5 i8_7 i7_7 i8_1 i7_1 i8_3 i7_3		; 50-52		n 62
	vblendmpd zmm16{k7}, zmm17, zmm12	;; r4_5 r3_5 r2_5 r1_5 r4_1 r3_1 r2_1 r1_1		;  50		n 57

	vshufpd	zmm18, zmm0, zmm13, 00000000b	;; i6_6 i5_6 i6_4 i5_4 i6_2 i5_2 i6_0 i5_0		; 51		n 60
	vblendmpd zmm12{k7}, zmm12, zmm17	;; r2_7 r1_7 r4_7 r3_7 r2_3 r1_3 r4_3 r3_3		;  51		n 59

	vshufpd	zmm0, zmm0, zmm13, 11111111b	;; i6_7 i5_7 i6_5 i5_5 i6_3 i5_3 i6_1 i5_1		; 52		n 62
 	vblendmpd zmm17{k7}, zmm2, zmm15	;; r8_4 r7_4 r6_4 r5_4 r8_0 r7_0 r6_0 r5_0		;  52		n 53

	vshuff64x2 zmm13, zmm9, zmm17, 01000100b;; r8_0 r7_0 r6_0 r5_0 r4_0 r3_0 r2_0 r1_0		; 53-55
	vblendmpd zmm15{k7}, zmm15, zmm2	;; r6_6 r5_6 r8_6 r7_6 r6_2 r5_2 r8_2 r7_2		;  53		n 55

	vshuff64x2 zmm9, zmm9, zmm17, 11101110b	;; r8_4 r7_4 r6_4 r5_4 r4_4 r3_4 r2_4 r1_4		; 54-56
	vblendmpd zmm2{k7}, zmm5, zmm8		;; r8_5 r7_5 r6_5 r6_5 r8_1 r7_1 r6_1 r5_1		;  54		n 57

	vshuff64x2 zmm17, zmm6, zmm15, 00010001b;; r8_2 r7_2 r6_2 r5_2 r4_2 r3_2 r2_2 r1_2		; 55-57
	vblendmpd zmm8{k7}, zmm8, zmm5		;; r6_7 r5_7 r8_7 r7_7 r6_3 r5_3 r8_3 r7_3		;  55		n 59

	vshuff64x2 zmm6, zmm6, zmm15, 10111011b ;; r8_6 r7_6 r6_6 r5_6 r4_6 r3_6 r2_6 r1_6		; 56-58
	vblendmpd zmm5{k7}, zmm19, zmm23	;; i4_4 i3_4 i2_4 i1_4 i4_0 i3_0 i2_0 i1_0		;  56		n 61
	zstore	[dstreg], zmm13										; 56

	vshuff64x2 zmm15, zmm16, zmm2, 01000100b;; r8_1 r7_1 r6_1 r5_1 r4_1 r3_1 r2_1 r1_1		; 57-59
	vblendmpd zmm23{k7}, zmm23, zmm19	;; i2_6 i1_6 i4_6 i3_6 i2_2 i1_2 i4_2 i3_2		;  57		n 63
	zstore	[dstreg+e4], zmm9									; 57

	vshuff64x2 zmm16, zmm16, zmm2, 11101110b;; r8_5 r7_5 r6_5 r5_5 r4_5 r3_5 r2_5 r1_5		; 58-60
	vblendmpd zmm19{k7}, zmm22, zmm7	;; i4_5 i3_5 i2_5 i1_5 i4_1 i3_1 i2_1 i1_1		;  58		n 65
	zstore	[dstreg+e2], zmm17									; 58

	vshuff64x2 zmm2, zmm12, zmm8, 00010001b	;; r8_3 r7_3 r6_3 r5_3 r4_3 r3_3 r2_3 r1_3		; 59-61
	vblendmpd zmm7{k7}, zmm7, zmm22		;; i2_7 i1_7 i4_7 i3_7 i2_3 i1_3 i4_3 i3_3		;  59		n 67
	zstore	[dstreg+e4+e2], zmm6									; 59

	vshuff64x2 zmm12, zmm12, zmm8, 10111011b;; r8_7 r7_7 r6_7 r5_7 r4_7 r3_7 r2_7 r1_7		; 60-62
	vblendmpd zmm22{k7}, zmm10, zmm18	;; i8_4 i7_4 i6_4 i5_4 i8_0 i7_0 i6_0 i5_0		;  60		n 61
	zstore	[dstreg+e1], zmm15									; 60

	vshuff64x2 zmm8, zmm5, zmm22, 01000100b ;; i8_0 i7_0 i6_0 i5_0 i4_0 i3_0 i2_0 i1_0		; 61-63
	vblendmpd zmm18{k7}, zmm18, zmm10	;; i6_6 i5_6 i8_6 i7_6 i6_2 i5_2 i8_2 i7_2		;  61		n 63
	zstore	[dstreg+e4+e1], zmm16									; 61

	vshuff64x2 zmm5, zmm5, zmm22, 11101110b;; i8_4 i7_4 i6_4 i5_4 i4_4 i3_4 i2_4 i1_4		; 62-64
	vblendmpd zmm10{k7}, zmm11, zmm0	;; i8_5 i7_5 i6_5 i5_5 i8_1 i7_1 i6_1 i5_1		;  62		n 65
	zstore	[dstreg+e2+e1], zmm2									; 62

	vshuff64x2 zmm22, zmm23, zmm18, 00010001b;; i8_2 i7_2 i6_2 i5_2 i4_2 i3_2 i2_2 i1_2		; 63-65
	vblendmpd zmm0{k7}, zmm0, zmm11		;; i6_7 i5_7 i8_7 i7_7 i6_3 i5_3 i8_3 i7_3		;  63		n 67
	zstore	[dstreg+e4+e2+e1], zmm12								; 63

	vshuff64x2 zmm23, zmm23, zmm18, 10111011b;; i8_6 i7_6 i6_6 i5_6 i4_6 i3_6 i2_6 i1_6		; 64-66
	zstore	[dstreg+64], zmm8									; 64

	vshuff64x2 zmm18, zmm19, zmm10, 01000100b;; i8_1 i7_1 i6_1 i5_1 i4_1 i3_1 i2_1 i1_1		; 65-67
	zstore	[dstreg+e4+64], zmm5									; 65

	vshuff64x2 zmm19, zmm19, zmm10, 11101110b;; i8_5 i7_5 i6_5 i5_5 i4_5 i3_5 i2_5 i1_5		; 66-68
	zstore	[dstreg+e2+64], zmm22									; 66

	vshuff64x2 zmm10, zmm7, zmm0, 00010001b ;; i8_3 i7_3 i6_3 i5_3 i4_3 i3_3 i2_3 i1_3		; 67-69
	zstore	[dstreg+e4+e2+64], zmm23									; 67

	vshuff64x2 zmm7, zmm7, zmm0, 10111011b	;; i8_7 i7_7 i6_7 i5_7 i4_7 i3_7 i2_7 i1_7		; 68-70

	zstore	[dstreg+e1+64], zmm18									; 68
	zstore	[dstreg+e4+e1+64], zmm19								; 69
	zstore	[dstreg+e2+e1+64], zmm10								; 70
	zstore	[dstreg+e4+e2+e1+64], zmm7								; 71
	bump	dstreg, dstinc
	ENDM

;;
;; ************************************* Macros used in one pass FFTs ******************************************
;;

zr8fs_sixteen_reals_first_fft_preload MACRO
	vbroadcastsd zmm31, ZMM_SQRTHALF
	vbroadcastsd zmm30, ZMM_P924_P383
	vbroadcastsd zmm29, ZMM_P383
	mov	r14d, 00110011b			;; For vblendmpd during swizzle
	kmovw	k7, r14d
	vpmovzxbq zmm28, ZMM_PERMUTE1		;; zmm28 = 8+4 0+4 8+6 0+6 8+0 0+0 8+2 0+2 [msw at left]
	vpmovzxbq zmm27, ZMM_PERMUTE2		;; zmm27 = 8+5 0+5 8+7 0+7 8+1 0+1 8+3 0+3
	ENDM

zr8fs_sixteen_reals_first_fft MACRO srcreg,srcinc,d1,d2,d4,screg,scinc,wgtreg,wgtinc,maxrpt,L1pt,L1pd
	vmovapd	zmm4, [srcreg+d4+d2+d1][rbx]	;; R8
	zfmaddpd zmm4, zmm4, [wgtreg+7*128], zmm4 ;; apply weight-1 to R8					; 1-4		n 6
	vmovapd	zmm8, [srcreg+d4+d1][rbx]	;; R6
	zfmaddpd zmm8, zmm8, [wgtreg+5*128], zmm8 ;; apply weight-1 to R6					; 1-4		n 7

	vmovapd	zmm14, [srcreg][rbx]		;; R1
	zfmaddpd zmm14, zmm14, [wgtreg], zmm14	;; apply weight-1 to R1						; 2-5		n 8
	vmovapd	zmm16, [srcreg+d4][rbx]		;; R5
	zfmaddpd zmm16, zmm16, [wgtreg+4*128], zmm16 ;; apply weights to R5					; 2-5		n 8

	vmovapd	zmm12, [srcreg+d4+d2][rbx]	;; R7
	zfmaddpd zmm12, zmm12, [wgtreg+6*128], zmm12 ;; apply weight-1 to R7					; 3-6		n 10
						;; to ease counting clocks, assume there is a 1/2 clock stall	; 3-6		but really there is no stall

	vmovapd	zmm2, [srcreg+d1][rbx]		;; R2
	vmovapd	zmm0, [srcreg+d1+64][rbx]	;; R10
	vmovapd	zmm26, [wgtreg+128+64]		;; weights10 / weights2
	zfmaddpd zmm1, zmm0, zmm26, zmm2	;; r2+r10*weights10_over_2					; 4-7		n 11
	zfnmaddpd zmm2, zmm0, zmm26, zmm2	;; r2-r10*weights10_over_2					; 4-7		n 20

	vmovapd	zmm6, [srcreg+d2+d1][rbx]	;; R4
	vmovapd	zmm0, [srcreg+d2+d1+64][rbx]	;; R12
	vmovapd	zmm26, [wgtreg+3*128+64]	;; weights12 / weights4
	zfmaddpd zmm5, zmm0, zmm26, zmm6	;; r4+r12*weights12_over_4					; 5-8		n 12
	zfnmaddpd zmm6, zmm0, zmm26, zmm6	;; r4-r12*weights12_over_4					; 5-8		n 21

	vmovapd	zmm0, [srcreg+d4+d2+d1+64][rbx]	;; R16
	vmovapd	zmm26, [wgtreg+7*128+64]	;; weights16
	zfmaddpd zmm3, zmm0, zmm26, zmm4	;; r8+r16*weights16						; 6-9		n 11
	zfnmaddpd zmm4, zmm0, zmm26, zmm4	;; r8-r16*weights16						; 6-9		n 20

	vmovapd	zmm0, [srcreg+d4+d1+64][rbx]	;; R14
	vmovapd	zmm26, [wgtreg+5*128+64]	;; weights14
	zfmaddpd zmm7, zmm0, zmm26, zmm8	;; r6+r14*weights14						; 7-10		n 12
	zfnmaddpd zmm8, zmm0, zmm26, zmm8	;; r6-r14*weights14						; 7-10		n 21

	vmovapd	zmm17, [srcreg+64][rbx]		;; R9
	vmovapd	zmm25, [wgtreg+64]		;; weights9
	zfmaddpd zmm13, zmm17, zmm25, zmm14	;; r1+r9*weights9						; 8-11		n 13
	vmovapd	zmm18, [srcreg+d4+64][rbx]	;; R13
	vmovapd	zmm24, [wgtreg+4*128+64]	;; weights13
	zfmaddpd zmm15, zmm18, zmm24, zmm16	;; r5+r13*weights13						; 8-11		n 13

	vmovapd	zmm10, [srcreg+d2][rbx]		;; R3
	vmovapd	zmm0, [srcreg+d2+64][rbx]	;; R11
	vmovapd	zmm26, [wgtreg+2*128+64]	;; weights11 / weights3
	zfmaddpd zmm9, zmm0, zmm26, zmm10	;; r3+r11*weights11_over_3					; 9-12		n 15
	zfnmaddpd zmm10, zmm0, zmm26, zmm10	;; r3-r11*weights11_over_3					; 9-12		n 16

	vmovapd	zmm0, [srcreg+d4+d2+64][rbx]	;; R15
	vmovapd	zmm26, [wgtreg+6*128+64]	;; weights15
	zfmaddpd zmm11, zmm0, zmm26, zmm12	;; r7+r15*weights15						; 10-13		n 15
	zfnmaddpd zmm12, zmm0, zmm26, zmm12	;; r7-r15*weights15						; 10-13		n 16

	vmovapd	zmm26, [wgtreg+128]		;; weights for R2
	zfmaddpd zmm0, zmm1, zmm26, zmm3	;; r2++ = (r2+r10)*weights2+(r8+r16)				; 11-14		n 17
	zfmsubpd zmm1, zmm1, zmm26, zmm3	;; r2+- = (r2+r10)*weights2-(r8+r16)				; 11-14		n 18

	vmovapd	zmm23, [wgtreg+3*128]		;; weights for R4
	zfmaddpd zmm3, zmm5, zmm23, zmm7	;; r4++ = (r4+r12)*weights4+(r6+r14)				; 12-15		n 17
	zfmsubpd zmm5, zmm5, zmm23, zmm7	;; r4+- = (r4+r12)*weights4-(r6+r14)				; 12-15		n 18

	vmovapd	zmm22, [wgtreg+2*128]		;; weights for R3
	vaddpd	zmm7, zmm13, zmm15		;; r1++ = (r1+r9)+(r5+r13)					; 13-16		n 18
	zfmaddpd zmm19, zmm9, zmm22, zmm11	;; r3++ = (r3+r11)*weights3+(r7+r15)				; 13-16		n 18
	bump	wgtreg, wgtinc

	vmovapd	zmm21, [screg+1*128+64]		;; cosine/sine for R3/I3 (w^2)							n 33
	zfnmaddpd zmm14, zmm17, zmm25, zmm14	;; r1-r9*weights9						; 14-17		n 25
	zfnmaddpd zmm16, zmm18, zmm24, zmm16	;; r5-r13*weights13						; 14-17		n 27

	vmovapd	zmm25, [screg+0*128+64]		;; cosine/sine for R2/I2 (w^1)							n 34
	vsubpd	zmm13, zmm13, zmm15		;; r1+- = (r1+r9)-(r5+r13)					; 15-18		n 21
	zfmsubpd zmm9, zmm9, zmm22, zmm11	;; r3+- = (r3+r11)*weights3-(r7+r15)				; 15-18		n 22

	vmovapd	zmm24, [screg+2*128+64]		;; cosine/sine for R4/I4 (w^3)							n 35
	zfmaddpd zmm11, zmm10, zmm22, zmm12	;; r3-+ = (r3-r11)*weights3+(r7-r15)				; 16-19		n 25
	zfmsubpd zmm10, zmm10, zmm22, zmm12	;; r3-- = (r3-r11)*weights3-(r7-r15)				; 16-19		n 23

	vmovapd	zmm22, [screg+3*128+64]		;; cosine/sine for R5/I5 (w^4)							n 36
	vsubpd	zmm12, zmm0, zmm3		;; r2++- = (r2++) - (r4++)					; 17-20		n 21
	vaddpd	zmm0, zmm0, zmm3		;; r2+++ = (r2++) + (r4++)					; 17-20		n 27

	vmovapd	zmm20, [screg+4*128+64]		;; cosine/sine for R6/I6 (w^5)							n 37
	vaddpd	zmm3, zmm1, zmm5		;; r2+-+ = (r2+-) + (r4+-)					; 18-21		n 22
	vaddpd	zmm15, zmm7, zmm19		;; r1+++ = (r1++) + (r3++)					; 18-21		n 27

	vmovapd	zmm18, [screg+0*128]		;; sine for R2/I2								n 38
	zfmaddpd zmm17, zmm2, zmm26, zmm4	;; r2-+ = (r2-r10)*weights2+(r8-r16)				; 19-22		n 26
	zfmsubpd zmm2, zmm2, zmm26, zmm4	;; r2-- = (r2-r10)*weights2-(r8-r16)				; 19-22		n 24

	vmovapd	zmm26, [screg+1*128]		;; sine for R3/I3								n 38
	zfmsubpd zmm4, zmm6, zmm23, zmm8	;; r4-- = (r4-r12)*weights4-(r6-r14)				; 20-23		n 24
	zfmaddpd zmm6, zmm6, zmm23, zmm8	;; r4-+ = (r4-r12)*weights4+(r6-r14)				; 20-23		n 26

	vmovapd	zmm23, [screg+2*128]		;; sine for R4/I4								n 39
	zfmaddpd zmm8, zmm12, zmm31, zmm13	;; R3 = (r1+-) + .707(r2++-)					; 21-24		n 33
	zfnmaddpd zmm12, zmm12, zmm31, zmm13	;; R7 = (r1+-) - .707(r2++-)					; 21-24		n 40

	zfmaddpd zmm13, zmm3, zmm31, zmm9	;; I3 = .707*(r2+-+) + (r3+-)					; 22-25		n 33
	zfmsubpd zmm3, zmm3, zmm31, zmm9	;; I7 = .707*(r2+-+) - (r3+-)					; 22-25		n 40

	zfmaddpd zmm9, zmm10, zmm31, zmm14	;; r2o = (r1-r9) + .707*(r3--)					; 23-26		n 28
	zfnmaddpd zmm10, zmm10, zmm31, zmm14	;; r4o = (r1-r9) - .707*(r3--)					; 23-26		n 29

	zfmaddpd zmm14, zmm2, zmm30, zmm4	;; r2e/.383 = .924/.383(r2--) + (r4--)				; 24-27		n 28
	zfnmaddpd zmm4, zmm4, zmm30, zmm2	;; r4e/.383 = (r2--) - .924/.383(r4--)				; 24-27		n 29

	zfmaddpd zmm2, zmm11, zmm31, zmm16	;; i2o = .707(r3-+) + (r5-r13)					; 25-28		n 30
	zfmsubpd zmm11, zmm11, zmm31, zmm16	;; i4o = .707(r3-+) - (r5-r13)					; 25-28		n 31

	zfmaddpd zmm16, zmm6, zmm30, zmm17	;; i2e/.383 = (r2-+) + .924/.383(r4-+)				; 26-29		n 30
	zfmsubpd zmm17, zmm17, zmm30, zmm6	;; i4e/.383 = .924/.383(r2-+) - (r4-+)				; 26-29		n 31

	vaddpd	zmm6, zmm15, zmm0		;; R1 = (r1+++) + (r2+++)					; 27-30		n 42
	vsubpd	zmm15, zmm15, zmm0		;; R9 = (r1+++) - (r2+++)					; 27-30		n 46

	zfmaddpd zmm0, zmm14, zmm29, zmm9	;; R2 = r2o + .383*r2e						; 28-31		n 34	r 28
	zfnmaddpd zmm14, zmm14, zmm29, zmm9	;; R8 = r2o - .383*r2e						; 28-31		n 41

	zfmaddpd zmm9, zmm4, zmm29, zmm10	;; R4 = r4o + .383*r4e						; 29-32		n 35	r 28
	zfnmaddpd zmm4, zmm4, zmm29, zmm10	;; R6 = r4o - .383*r4e						; 29-32		n 37

	zfmaddpd zmm10, zmm16, zmm29, zmm2	;; I2 = .383*i2e + i2o						; 30-33		n 34	r 30
	zfmsubpd zmm16, zmm16, zmm29, zmm2	;; I8 = .383*i2e - i2o						; 30-33		n 41

	zfmaddpd zmm2, zmm17, zmm29, zmm11	;; I4 = .383*i4e + i4o						; 31-34		n 35	r 30
	zfmsubpd zmm17, zmm17, zmm29, zmm11	;; I6 = .383*i4e - i4o						; 31-34		n 37

	vmovapd	zmm11, [screg+5*128+64]		;; cosine/sine for R7/I7 (w^6)							n 40
	vsubpd	zmm1, zmm1, zmm5		;; r2+-- = (r2+-) - (r4+-)	(I5)				; 32-35		n 36
	vsubpd	zmm7, zmm7, zmm19		;; r1++- = (r1++) - (r3++)	(R5)				; 32-35		n 36

	vmovapd	zmm19, [screg+6*128+64]		;; cosine/sine for R8/I8 (w^7)							n 41
	zfmsubpd zmm5, zmm8, zmm21, zmm13	;; A3 = R3 * cosine/sine - I3					; 33-36		n 38
	zfmaddpd zmm13, zmm13, zmm21, zmm8	;; B3 = I3 * cosine/sine + R3					; 33-36		n 42

	vmovapd	zmm21, [screg+3*128]		;; sine for R5/I5								n 44
	zfmsubpd zmm8, zmm0, zmm25, zmm10	;; A2 = R2 * cosine/sine - I2					; 34-37		n 38		r 34 c.b. 1 earlier
	zfmaddpd zmm10, zmm10, zmm25, zmm0	;; B2 = I2 * cosine/sine + R2					; 34-37		n 39

	vmovapd zmm25, [screg+4*128]		;; sine for R6/I6								n 45
	zfmsubpd zmm0, zmm9, zmm24, zmm2	;; A4 = R4 * cosine/sine - I4					; 35-38		n 39		r 35
	zfmaddpd zmm2, zmm2, zmm24, zmm9	;; B4 = I4 * cosine/sine + R4					; 35-38		n 43

	vmovapd	zmm24, [screg+5*128]		;; sine for R7/I7								n 46
	zfmsubpd zmm9, zmm7, zmm22, zmm1	;; A5 = R5 * cosine/sine - I5					; 36-39		n 44
	zfmaddpd zmm1, zmm1, zmm22, zmm7	;; B5 = I5 * cosine/sine + R5					; 36-39		n 48

	vmovapd	zmm22, [screg+6*128]		;; sine for R8/I8								n 47
	zfmsubpd zmm7, zmm4, zmm20, zmm17	;; A6 = R6 * cosine/sine - I6					; 37-40		n 45
	zfmaddpd zmm17, zmm17, zmm20, zmm4	;; B6 = I6 * cosine/sine + R6					; 37-40		n 49
	bump	screg, scinc

	L1prefetchw srcreg+srcinc+rbx+d4+d2+d1, L1pt
	vmulpd	zmm8, zmm8, zmm18		;; A2 = A2 * sine (final R2)					; 38-41		n 47		r 38
	vmulpd	zmm5, zmm5, zmm26		;; A3 = A3 * sine (final R3)					; 38-41		n 49

	L1prefetch wgtreg+7*128, L1pt
	vmulpd	zmm0, zmm0, zmm23		;; A4 = A4 * sine (final R4)					; 39-42		n 49
	vmulpd	zmm10, zmm10, zmm18		;; B2 = B2 * sine (final I2)					; 39-42		n 51

	L1prefetchw srcreg+srcinc+rbx+d4+d1, L1pt
	zfmsubpd zmm4, zmm12, zmm11, zmm3	;; A7 = R7 * cosine/sine - I7					; 40-43		n 46
	zfmaddpd zmm3, zmm3, zmm11, zmm12	;; B7 = I7 * cosine/sine + R7					; 40-43		n 50

	L1prefetch wgtreg+5*128, L1pt
	zfmsubpd zmm12, zmm14, zmm19, zmm16	;; A8 = R8 * cosine/sine - I8					; 41-44		n 47
	zfmaddpd zmm16, zmm16, zmm19, zmm14	;; B8 = I8 * cosine/sine + R8					; 41-44		n 51

	L1prefetchw srcreg+srcinc+rbx, L1pt
	vshufpd	zmm14, zmm6, zmm8, 00000000b	;; r2_6 r1_6 r2_4 r1_4 r2_2 r1_2 r2_0 r1_0			; 42		n 53
	vmulpd	zmm13, zmm13, zmm26		;; B3 = B3 * sine (final I3)					; 42-45		n 53

	L1prefetch wgtreg+0*128, L1pt
	vshufpd	zmm6, zmm6, zmm8, 11111111b	;; r2_7 r1_7 r2_5 r1_5 r2_3 r1_3 r2_1 r1_1			; 43		n 55
	vmulpd	zmm2, zmm2, zmm23		;; B4 = B4 * sine (final I4)					; 43-46		n 53

	L1prefetchw srcreg+srcinc+rbx+d4, L1pt
	zperm2pd zmm8, zmm28, zmm5, zmm0	;; r4_4 r3_4 r4_6 r3_6 r4_0 r3_0 r4_2 r3_2			; 44-46		n 53
	vmulpd	zmm9, zmm9, zmm21		;; A5 = A5 * sine (final R5)					; 44-47		n 55

	L1prefetch wgtreg+4*128, L1pt
	zperm2pd zmm5, zmm27, zmm5, zmm0	;; r4_5 r3_5 r4_7 r3_7 r4_1 r3_1 r4_3 r3_3			; 45-47		n 55
	vmulpd	zmm7, zmm7, zmm25		;; A6 = A6 * sine (final R6)					; 45-48		n 55

	L1prefetchw srcreg+srcinc+rbx+d4+d2, L1pt
	vshufpd	zmm0, zmm15, zmm10, 00000000b	;; i2_6 i1_6 i2_4 i1_4 i2_2 i1_2 i2_0 i1_0			; 46		n 61
	vmulpd	zmm4, zmm4, zmm24		;; A7 = A7 * sine (final R7)					; 46-49		n 57

	L1prefetch wgtreg+6*128, L1pt
	vshufpd	zmm15, zmm15, zmm10, 11111111b	;; i2_7 i1_7 i2_5 i1_5 i2_3 i1_3 i2_1 i1_1			; 47		n 65
	vmulpd	zmm12, zmm12, zmm22		;; A8 = A8 * sine (final R8)					; 47-50		n 57

	L1prefetchw srcreg+srcinc+rbx+d1, L1pt
	zperm2pd zmm10, zmm28, zmm13, zmm2	;; i4_4 i3_4 i4_6 i3_6 i4_0 i3_0 i4_2 i3_2			; 48-50		n 61
	vmulpd	zmm1, zmm1, zmm21		;; B5 = B5 * sine (final I5)					; 48-51		n 59

	L1prefetchw srcreg+srcinc+rbx+d1+64, L1pt
	zperm2pd zmm13, zmm27, zmm13, zmm2	;; i4_5 i3_5 i4_7 i3_7 i4_1 i3_1 i4_3 i3_3			; 49-51		n 65
	vmulpd	zmm17, zmm17, zmm25		;; B6 = B6 * sine (final I6)					; 49-52		n 59

	L1prefetch wgtreg+1*128+64, L1pt
	vshufpd	zmm2, zmm9, zmm7, 00000000b	;; r6_6 r5_6 r6_4 r5_4 r6_2 r5_2 r6_0 r5_0			; 50		n 57
	vmulpd	zmm3, zmm3, zmm24		;; B7 = B7 * sine (final I7)					; 50-53		n 61

	L1prefetchw srcreg+srcinc+rbx+d2+d1, L1pt
	vshufpd	zmm9, zmm9, zmm7, 11111111b	;; r6_7 r5_7 r6_5 r5_5 r6_3 r5_3 r6_1 r5_1			; 51		n 59
	vmulpd	zmm16, zmm16, zmm22		;; B8 = B8 * sine (final I8)					; 51-54		n 61

	L1prefetchw srcreg+srcinc+rbx+d2+d1+64, L1pt
	zperm2pd zmm7, zmm28, zmm4, zmm12	;; r8_4 r7_4 r8_6 r7_6 r8_0 r7_0 r8_2 r7_2			; 52-54		n 57

	L1prefetch wgtreg+3*128+64, L1pt
	zperm2pd zmm4, zmm27, zmm4, zmm12	;; r8_5 r7_5 r8_7 r7_7 r8_1 r7_1 r8_3 r7_3			; 53-55		n 59
	vblendmpd zmm12{k7}, zmm8, zmm14	;; r4_4 r3_4 r2_4 r1_4 r4_0 r3_0 r2_0 r1_0			; 53		n 58

	L1prefetchw srcreg+srcinc+rbx+d4+d2+d1+64, L1pt
	vshufpd	zmm18, zmm1, zmm17, 00000000b	;; i6_6 i5_6 i6_4 i5_4 i6_2 i5_2 i6_0 i5_0			; 54		n 63
	vblendmpd zmm14{k7}, zmm14, zmm8	;; r2_6 r1_6 r4_6 r3_6 r2_2 r1_2 r4_2 r3_2			; 54		n 60

	L1prefetch wgtreg+7*128+64, L1pt
	vshufpd	zmm1, zmm1, zmm17, 11111111b	;; i6_7 i5_7 i6_5 i5_5 i6_3 i5_3 i6_1 i5_1			; 55		n 67
	vblendmpd zmm17{k7}, zmm5, zmm6		;; r4_5 r3_5 r2_5 r1_5 r4_1 r3_1 r2_1 r1_1			; 55		n 62

	L1prefetchw srcreg+srcinc+rbx+d4+d1+64, L1pt
	zperm2pd zmm8, zmm28, zmm3, zmm16	;; i8_4 i7_4 i8_6 i7_6 i8_0 i7_0 i8_2 i7_2			; 56-58		n 63
	vblendmpd zmm6{k7}, zmm6, zmm5		;; r2_7 r1_7 r4_7 r3_7 r2_3 r1_3 r4_3 r3_3			; 56		n 64

	L1prefetch wgtreg+5*128+64, L1pt
	zperm2pd zmm3, zmm27, zmm3, zmm16	;; i8_5 i7_5 i8_7 i7_7 i8_1 i7_1 i8_3 i7_3			; 57-59		n 67
	vblendmpd zmm16{k7}, zmm7, zmm2		;; r8_4 r7_4 r6_4 r5_4 r8_0 r7_0 r6_0 r5_0			; 57		n 58

	L1prefetchw srcreg+srcinc+rbx+64, L1pt
	vshuff64x2 zmm5, zmm12, zmm16, 01000100b ;; r8_0 r7_0 r6_0 r5_0 r4_0 r3_0 r2_0 r1_0 (R1)		; 58-60
	vblendmpd zmm2{k7}, zmm2, zmm7		;; r6_6 r5_6 r8_6 r7_6 r6_2 r5_2 r8_2 r7_2			; 58		n 60

	L1prefetch wgtreg+0*128+64, L1pt
	vshuff64x2 zmm12, zmm12, zmm16, 11101110b ;; r8_4 r7_4 r6_4 r5_4 r4_4 r3_4 r2_4 r1_4 (R5)		; 59-61
	vblendmpd zmm16{k7}, zmm4, zmm9		;; r8_5 r7_5 r6_5 r6_5 r8_1 r7_1 r6_1 r5_1			; 59		n 62

	L1prefetchw srcreg+srcinc+rbx+d4+64, L1pt
	vshuff64x2 zmm7, zmm14, zmm2, 00010001b ;; r8_2 r7_2 r6_2 r5_2 r4_2 r3_2 r2_2 r1_2 (R3)			; 60-62
	vblendmpd zmm9{k7}, zmm9, zmm4		;; r6_7 r5_7 r8_7 r7_7 r6_3 r5_3 r8_3 r7_3			; 60		n 64

	L1prefetch wgtreg+4*128+64, L1pt
	vshuff64x2 zmm14, zmm14, zmm2, 10111011b ;; r8_6 r7_6 r6_6 r5_6 r4_6 r3_6 r2_6 r1_6 (R7)		; 61-63
	vblendmpd zmm2{k7}, zmm10, zmm0		;; i4_4 i3_4 i2_4 i1_4 i4_0 i3_0 i2_0 i1_0			; 61		n 66
	zstore	[srcreg], zmm5			;; Store R1							; 61

	L1prefetchw srcreg+srcinc+rbx+d2, L1pt
	vshuff64x2 zmm4, zmm17, zmm16, 01000100b ;; r8_1 r7_1 r6_1 r5_1 r4_1 r3_1 r2_1 r1_1 (R2)		; 62-64
	vblendmpd zmm0{k7}, zmm0, zmm10		;; i2_6 i1_6 i4_6 i3_6 i2_2 i1_2 i4_2 i3_2			; 62		n 68
	zstore	[srcreg+d4], zmm12		;; Store R5							; 62

	L1prefetchw srcreg+srcinc+rbx+d2+64, L1pt
	vshuff64x2 zmm17, zmm17, zmm16, 11101110b ;; r8_5 r7_5 r6_5 r5_5 r4_5 r3_5 r2_5 r1_5 (R6)		; 63-65
	vblendmpd zmm16{k7}, zmm8, zmm18	;; i8_4 i7_4 i6_4 i5_4 i8_0 i7_0 i6_0 i5_0			; 63		n 66
	zstore	[srcreg+d2], zmm7		;; Store R3							; 63

	L1prefetch wgtreg+2*128+64, L1pt
	vshuff64x2 zmm10, zmm6, zmm9, 00010001b ;; r8_3 r7_3 r6_3 r5_3 r4_3 r3_3 r2_3 r1_3 (R4)			; 64-66
	vblendmpd zmm18{k7}, zmm18, zmm8	;; i6_6 i5_6 i8_6 i7_6 i6_2 i5_2 i8_2 i7_2			; 64		n 68
	zstore	[srcreg+d4+d2], zmm14		;; Store R7							; 64

	L1prefetchw srcreg+srcinc+rbx+d4+d2+64, L1pt
	vshuff64x2 zmm6, zmm6, zmm9, 10111011b	;; r8_7 r7_7 r6_7 r5_7 r4_7 r3_7 r2_7 r1_7 (R8)			; 65-67
	vblendmpd zmm9{k7}, zmm13, zmm15	;; i4_5 i3_5 i2_5 i1_5 i4_1 i3_1 i2_1 i1_1			; 65		n 70
	zstore	[srcreg+d1], zmm4		;; Store R2							; 65

	L1prefetch wgtreg+6*128+64, L1pt
	vshuff64x2 zmm8, zmm2, zmm16, 01000100b ;; i8_0 i7_0 i6_0 i5_0 i4_0 i3_0 i2_0 i1_0 (I1)			; 66-68
	vblendmpd zmm15{k7}, zmm15, zmm13	;; i2_7 i1_7 i4_7 i3_7 i2_3 i1_3 i4_3 i3_3			; 66		n 72
	zstore	[srcreg+d4+d1], zmm17		;; Store R6							; 66

	L1prefetch wgtreg+1*128, L1pt
	vshuff64x2 zmm2, zmm2, zmm16, 11101110b ;; i8_4 i7_4 i6_4 i5_4 i4_4 i3_4 i2_4 i1_4 (I5)			; 67-69
	vblendmpd zmm16{k7}, zmm3, zmm1		;; i8_5 i7_5 i6_5 i5_5 i8_1 i7_1 i6_1 i5_1			; 67		n 70
	zstore	[srcreg+d2+d1], zmm10		;; Store R4							; 67

	L1prefetch wgtreg+3*128, L1pt
	vshuff64x2 zmm13, zmm0, zmm18, 00010001b ;; i8_2 i7_2 i6_2 i5_2 i4_2 i3_2 i2_2 i1_2 (I3)		; 68-70
	vblendmpd zmm1{k7}, zmm1, zmm3		;; i6_7 i5_7 i8_7 i7_7 i6_3 i5_3 i8_3 i7_3			; 68		n 72
	zstore	[srcreg+d4+d2+d1], zmm6		;; Store R8							; 68

	L1prefetch wgtreg+2*128, L1pt
	vshuff64x2 zmm0, zmm0, zmm18, 10111011b ;; i8_6 i7_6 i6_6 i5_6 i4_6 i3_6 i2_6 i1_6 (I7)			; 69-71
	zstore	[srcreg+64], zmm8		;; Store I1							; 69

	vshuff64x2 zmm18, zmm9, zmm16, 01000100b ;; i8_1 i7_1 i6_1 i5_1 i4_1 i3_1 i2_1 i1_1 (I2)		; 70-72
	zstore	[srcreg+d4+64], zmm2		;; Store I5							; 70

	vshuff64x2 zmm9, zmm9, zmm16, 11101110b ;; i8_5 i7_5 i6_5 i5_5 i4_5 i3_5 i2_5 i1_5 (I6)			; 71-73
	zstore	[srcreg+d2+64], zmm13		;; Store I3							; 71

	vshuff64x2 zmm16, zmm15, zmm1, 00010001b ;; i8_3 i7_3 i6_3 i5_3 i4_3 i3_3 i2_3 i1_3 (I4)		; 72-74
	zstore	[srcreg+d4+d2+64], zmm0		;; Store I7							; 72

	vshuff64x2 zmm15, zmm15, zmm1, 10111011b ;; i8_7 i7_7 i6_7 i5_7 i4_7 i3_7 i2_7 i1_7 (I8)		; 73-75
	zstore	[srcreg+d1+64], zmm18		;; Store I2							; 73
	zstore	[srcreg+d4+d1+64], zmm9		;; Store I6							; 74
	zstore	[srcreg+d2+d1+64], zmm16	;; Store I4							; 75
	zstore	[srcreg+d4+d2+d1+64], zmm15	;; Store I8							; 76
	bump	srcreg, srcinc
	ENDM


zr8s_sixteen_reals_last_unfft_preload MACRO
	vbroadcastsd zmm31, ZMM_SQRTHALF
	vbroadcastsd zmm30, ZMM_P924_P383
	vbroadcastsd zmm29, ZMM_P383
	mov	r14d, 11110000b
	kmovw	k6, r14d
	knotw	k7, k6
	ENDM

zr8s_sixteen_reals_last_unfft MACRO srcreg,srcinc,d1,d2,d4,screg,scinc,maxrpt,L1pt,L1pd
	vmovapd	zmm15, [srcreg]				;; d0_7	d0_6 d0_5 d0_4 d0_3 d0_2 d0_1 d0_0
	vmovapd	zmm16, [srcreg+d1]			;; d1_7	d1_6 d1_5 d1_4 d1_3 d1_2 d1_1 d1_0
	vbroadcastf64x4	zmm15{k6}, [srcreg+d2]		;; d2_3 d2_2 d2_1 d2_0 d0_3 d0_2 d0_1 d0_0		; 1		n 2
	vbroadcastf64x4	zmm16{k6}, [srcreg+d2+d1]	;; d3_3 d3_2 d3_1 d3_0 d1_3 d1_2 d1_1 d1_0		; 1		n 2

	vmovapd	zmm2, [srcreg+d4]			;; d4_7	d4_6 d4_5 d4_4 d4_3 d4_2 d4_1 d4_0
	vshufpd	zmm5, zmm15, zmm16, 00000000b		;; d3_2 d2_2 d3_0 d2_0 d1_2 d0_2 d1_0 d0_0		; 2		n 10
	vbroadcastf64x4	zmm2{k6}, [srcreg+d4+d2]	;; d6_3 d6_2 d6_1 d6_0 d4_3 d4_2 d4_1 d4_0		; 2		n 4

	vmovapd	zmm8, [srcreg+d4+d1]			;; d5_7	d5_6 d5_5 d5_4 d5_3 d5_2 d5_1 d5_0
	vshufpd	zmm16, zmm15, zmm16, 11111111b		;; d3_3 d2_3 d3_1 d2_1 d1_3 d0_3 d1_1 d0_1		; 3		n 14
	vbroadcastf64x4	zmm8{k6}, [srcreg+d4+d2+d1]	;; d7_3 d7_2 d7_1 d7_0 d5_3 d5_2 d5_1 d5_0		; 3		n 4

	vmovapd	zmm9, [srcreg+d2+64]			;; (+64 d2_7 d2_6 d2_5 d2_4 d2_3 d2_2 d2_1 d2_0)
	vmovupd	zmm17, [srcreg+32]			;; (+64 d0_3 d0_2 d0_1 d0_0) d0_7 d0_6 d0_5 d0_4
	vshufpd	zmm15, zmm2, zmm8, 00000000b		;; d7_2 d6_2 d7_0 d6_0 d5_2 d4_2 d5_0 d4_0		; 4		n 10
	vblendmpd zmm9{k6}, zmm9, zmm17			;; (+64 d0_3 d0_2 d0_1 d0_0 d2_3 d2_2 d2_1 d2_0)	; 4		n 6

	vmovapd	zmm11, [srcreg+d2+d1+64]		;; (+64 d3_7 d3_6 d3_5 d3_4 d3_3 d3_2 d3_1 d3_0)
	vmovupd	zmm18, [srcreg+d1+32]			;; (+64 d1_3 d1_2 d1_1 d1_0) d1_7 d1_6 d1_5 d1_4
	vshufpd	zmm8, zmm2, zmm8, 11111111b		;; d7_3 d6_3 d7_1 d6_1 d5_3 d4_3 d5_1 d4_1		; 5		n 14
	vblendmpd zmm11{k6}, zmm11, zmm18		;; (+64 d1_3 d1_2 d1_1 d1_0 d3_3 d3_2 d3_1 d3_0)	; 5		n 6

	vmovapd	zmm13, [srcreg+d4+d2+64]		;; (+64 d6_7 d6_6 d6_5 d6_4 d6_3 d6_2 d6_1 d6_0)
	vmovupd	zmm19, [srcreg+d4+32]			;; (+64 d4_3 d4_2 d4_1 d4_0) d4_7 d4_6 d4_5 d4_4
	vshufpd	zmm0, zmm9, zmm11, 00000000b		;; (+64 d1_2 d0_2 d1_0 d0_0 d3_2 d2_2 d3_0 d2_0)	; 6		n 12
	vblendmpd zmm13{k6}, zmm13, zmm19		;; (+64 d4_3 d4_2 d4_1 d4_0 d6_3 d6_2 d6_1 d6_0)	; 6		n 8

	vmovapd	zmm12, [srcreg+d4+d2+d1+64]		;; (+64 d7_7 d7_6 d7_5 d7_4 d7_3 d7_2 d7_1 d7_0)
	vmovupd zmm24, [srcreg+d4+d1+32]		;; (+64 d5_3 d5_2 d5_1 d5_0) d5_7 d5_6 d5_5 d5_4
	vshufpd	zmm11, zmm9, zmm11, 11111111b		;; (+64 d1_3 d0_3 d1_1 d0_1 d3_3 d2_3 d3_1 d2_1)	; 7		n 16
	vblendmpd zmm12{k6}, zmm12, zmm24		;; (+64 d5_3 d5_2 d5_1 d5_0 d7_3 d7_2 d7_1 d7_0)	; 7		n 8

	vmovapd	zmm3, [srcreg+d2]			;; d2_7	d2_6 d2_5 d2_4 d2_3 d2_2 d2_1 d2_0
	vshufpd	zmm9, zmm13, zmm12, 00000000b		;; (+64 d5_2 d4_2 d5_0 d4_0 d7_2 d6_2 d7_0 d6_0)	; 8		n 12
	vblendmpd zmm3{k6}, zmm17, zmm3			;; d2_7 d2_6 d2_5 d2_4 d0_7 d0_6 d0_5 d0_4		; 8		n 18

	vmovapd	zmm22, [srcreg+d2+d1]			;; d3_7	d3_6 d3_5 d3_4 d3_3 d3_2 d3_1 d3_0
	vshufpd	zmm12, zmm13, zmm12, 11111111b		;; (+64 d5_3 d4_3 d5_1 d4_1 d7_3 d6_3 d7_1 d6_1)	; 9		n 16
	vblendmpd zmm22{k6}, zmm18, zmm22		;; d3_7 d3_6 d3_5 d3_4 d1_7 d1_6 d1_5 d1_4		; 9		n 18

	vmovapd	zmm7, [srcreg+d4+d2]			;; d6_7	d6_6 d6_5 d6_4 d6_3 d6_2 d6_1 d6_0
	vshuff64x2 zmm17, zmm5, zmm15, 10001000b	;; d7_0 d6_0 d5_0 d4_0 d3_0 d2_0 d1_0 d0_0 (R1)		; 10-12		n 16
	vblendmpd zmm7{k6}, zmm19, zmm7			;; d6_7	d6_6 d6_5 d6_4 d4_7 d4_6 d4_5 d4_4		; 10		n 20

	vmovapd	zmm14, [srcreg+d4+d2+d1]		;; d7_7	d7_6 d7_5 d7_4 d7_3 d7_2 d7_1 d7_0
 	vshuff64x2 zmm15, zmm5, zmm15, 11011101b	;; d7_2 d6_2 d5_2 d4_2 d3_2 d2_2 d1_2 d0_2 (R3)		; 11-13		n 18
	vblendmpd zmm14{k6}, zmm24, zmm14		;; d7_7	d7_6 d7_5 d7_4 d5_7 d5_6 d5_5 d5_4		; 11		n 20

	vmovapd	zmm10, [srcreg+64]			;; (+64 d0_7 d0_6 d0_5 d0_4 d0_3 d0_2 d0_1 d0_0)
	vshuff64x2 zmm18, zmm0, zmm9, 00100010b		;; (+64 d7_0 d6_0 d5_0 d4_0 d3_0 d2_0 d1_0 d0_0)(I1)	; 12-14		n 16
	vbroadcastf64x4	zmm10{k7}, [srcreg+d2+64+32]	;; (+64 d0_7 d0_6 d0_5 d0_4 d2_7 d2_6 d2_5 d2_4)	; 12		n 22

	vmovapd	zmm23, [srcreg+d1+64]			;; (+64 d1_7 d1_6 d1_5 d1_4 d1_3 d1_2 d1_1 d1_0)
	vshuff64x2 zmm9, zmm0, zmm9, 01110111b		;; (+64 d7_2 d6_2 d5_2 d4_2 d3_2 d2_2 d1_2 d0_2)(I3)	; 13-15		n 18
	vbroadcastf64x4	zmm23{k7}, [srcreg+d2+d1+64+32]	;; (+64 d1_7 d1_6 d1_5 d1_4 d3_7 d3_6 d3_5 d3_4)	; 13		n 22

	vmovapd	zmm1, [srcreg+d4+64]			;; (+64 d4_7 d4_6 d4_5 d4_4 d4_3 d4_2 d4_1 d4_0)
	vshuff64x2 zmm5, zmm16, zmm8, 10001000b		;; d7_1 d6_1 d5_1 d4_1 d3_1 d2_1 d1_1 d0_1 (R2)		; 14-16		n 20
	vbroadcastf64x4	zmm1{k7}, [srcreg+d4+d2+64+32]	;; (+64 d4_7 d4_6 d4_5 d4_4 d6_7 d6_6 d6_5 d6_4)	; 14		n 24

	vmovapd	zmm6, [srcreg+d4+d1+64]			;; (+64 d5_7 d5_6 d5_5 d5_4 d5_3 d5_2 d5_1 d5_0)
	vshuff64x2 zmm8, zmm16, zmm8, 11011101b		;; d7_3 d6_3 d5_3 d4_3 d3_3 d2_3 d1_3 d0_3 (R4)		; 15-17		n 22
	vbroadcastf64x4	zmm6{k7}, [srcreg+d4+d2+d1+64+32];;(+64 d5_7 d5_6 d5_5 d5_4 d7_7 d7_6 d7_5 d7_4)	; 15		n 24

	vmovapd	zmm28, [screg+1*128+64]			;; cosine/sine for R3/I3 (w^2)
	vshuff64x2 zmm4, zmm11, zmm12, 00100010b	;; (+64 d7_1 d6_1 d5_1 d4_1 d3_1 d2_1 d1_1 d0_1)(I2)	; 16-18		n 20
	vaddpd	zmm16, zmm17, zmm18			;; R1+R9						; 16-19		n 42

	vmovapd	zmm27, [screg+0*128+64]			;; cosine/sine for R2/I2 (w^1)
	vshuff64x2 zmm12, zmm11, zmm12, 01110111b	;; (+64 d7_3 d6_3 d5_3 d4_3 d3_3 d2_3 d1_3 d0_3)(I4)	; 17-19		n 22
	vsubpd	zmm18, zmm17, zmm18			;; R1-R9						; 17-20		n 43

	vmovapd	zmm26, [screg+2*128+64]			;; cosine/sine for R4/I4 (w^3)
	vshufpd	zmm2, zmm3, zmm22, 00000000b		;; d3_6 d2_6 d3_4 d2_4 d1_6 d0_6 d1_4 d0_4		; 18		n 26
	zfmaddpd zmm11, zmm15, zmm28, zmm9		;; A3 = R3 * cosine/sine + I3				; 18-21		n 24

	vmovapd	zmm25, [screg+1*128]			;; sine for R3/I3
	vshufpd	zmm22, zmm3, zmm22, 11111111b		;; d3_7 d2_7 d3_5 d2_5 d1_7 d0_7 d1_5 d0_5		; 19		n 28
	zfmsubpd zmm9, zmm9, zmm28, zmm15		;; B3 = I3 * cosine/sine - R3				; 19-22		n 25

	vmovapd	zmm28, [screg+0*128]			;; sine for R2/I2
	vshufpd	zmm3, zmm7, zmm14, 00000000b		;; d7_6 d6_6 d7_4 d6_4 d5_6 d4_6 d5_4 d4_4		; 20		n 26
	zfmaddpd zmm15, zmm5, zmm27, zmm4		;; A2 = R2 * cosine/sine + I2				; 20-23		n 26

	vmovapd	zmm24, [screg+2*128]			;; sine for R4/I4
	vshufpd	zmm14, zmm7, zmm14, 11111111b		;; d7_7 d6_7 d7_5 d6_5 d5_7 d4_7 d5_5 d4_5		; 21		n 28
	zfmsubpd zmm4, zmm4, zmm27, zmm5		;; B2 = I2 * cosine/sine - R2				; 21-24		n 27

	vmovapd	zmm27, [screg+5*128+64]			;; cosine/sine for R7/I7 (w^6)
	vshufpd	zmm13, zmm10, zmm23, 00000000b		;; (+64 d1_6 d0_6 d1_4 d0_4 d3_6 d2_6 d3_4 d2_4)	; 22		n 27
	zfmaddpd zmm5, zmm8, zmm26, zmm12		;; A4 = R4 * cosine/sine + I4				; 22-25		n 28

	vmovapd	zmm21, [screg+6*128+64]			;; cosine/sine for R8/I8 (w^7)
	vshufpd	zmm23, zmm10, zmm23, 11111111b		;; (+64 d1_7 d0_7 d1_5 d0_5 d3_7 d2_7 d3_5 d2_5)	; 23		n 29
	zfmsubpd zmm12, zmm12, zmm26, zmm8		;; B4 = I4 * cosine/sine - R4				; 23-26		n 29

	vmovapd	zmm26, [screg+4*128+64]			;; cosine/sine for R6/I6 (w^5)
	vshufpd	zmm10, zmm1, zmm6, 00000000b		;; (+64 d5_6 d4_6 d5_4 d4_4 d7_6 d6_6 d7_4 d6_4)	; 24		n 27
	vmulpd	zmm11, zmm11, zmm25			;; R3 = A3 * sine					; 24-27		n 35

	vmovapd	zmm20, [screg+5*128]			;; sine for R7/I7
	vshufpd	zmm6, zmm1, zmm6, 11111111b		;; (+64 d5_7 d4_7 d5_5 d4_5 d7_7 d6_7 d7_5 d6_5)	; 25		n 29
	vmulpd	zmm9, zmm9, zmm25			;; I3 = B3 * sine					; 25-28		n 36

	vmovapd	zmm25, [screg+3*128+64]			;; cosine/sine for R5/I5 (w^4)
	vshuff64x2 zmm7, zmm2, zmm3, 11011101b		;; d7_6 d6_6 d5_6 d4_6 d3_6 d2_6 d1_6 d0_6 (R7)		; 26-28		n 30
	vmulpd	zmm15, zmm15, zmm28			;; R2 = A2 * sine					; 26-29		n 38

	vmovapd	zmm19, [screg+6*128]			;; sine for R8/I8
	vshuff64x2 zmm8, zmm13, zmm10, 01110111b	;; (+64 d7_6 d6_6 d5_6 d4_6 d3_6 d2_6 d1_6 d0_6)(I7)	; 27-29		n 30
	vmulpd	zmm4, zmm4, zmm28			;; I2 = B2 * sine					; 27-30		n 39

	vmovapd	zmm28, [screg+4*128]			;; sine for R6/I6
	vshuff64x2 zmm0, zmm22, zmm14, 11011101b	;; d7_7 d6_7 d5_7 d4_7 d3_7 d2_7 d1_7 d0_7 (R8)		; 28-30		n 32
	vmulpd	zmm5, zmm5, zmm24			;; R4 = A4 * sine					; 28-31		n 40

	vmovapd	zmm17, [screg+3*128]			;; sine for R5/I5
	vshuff64x2 zmm1, zmm23, zmm6, 01110111b		;; (+64 d7_7 d6_7 d5_7 d4_7 d3_7 d2_7 d1_7 d0_7)(I8)	; 29-31		n 32
	vmulpd	zmm12, zmm12, zmm24			;; I4 = B4 * sine					; 29-32		n 41
	bump	screg, scinc

	L1prefetchw srcreg+srcinc, L1pt
	vshuff64x2 zmm22, zmm22, zmm14, 10001000b	;; d7_5 d6_5 d5_5 d4_5 d3_5 d2_5 d1_5 d0_5 (R6)		; 30-32		n 34
	zfmaddpd zmm14, zmm7, zmm27, zmm8		;; A7 = R7 * cosine/sine + I7				; 30-33		n 35

	L1prefetchw srcreg+srcinc+d1, L1pt
	vshuff64x2 zmm6, zmm23, zmm6, 00100010b		;; (+64 d7_5 d6_5 d5_5 d4_5 d3_5 d2_5 d1_5 d0_5)(I6)	; 31-33		n 34
	zfmsubpd zmm8, zmm8, zmm27, zmm7		;; B7 = I7 * cosine/sine - R7				; 31-34		n 36

	L1prefetchw srcreg+srcinc+d2, L1pt
	vshuff64x2 zmm2, zmm2, zmm3, 10001000b		;; d7_4	d6_4 d5_4 d4_4 d3_4 d2_4 d1_4 d0_4 (R5)		; 32-34		n 37
	zfmaddpd zmm3, zmm0, zmm21, zmm1		;; A8 = R8 * cosine/sine + I8				; 32-35		n 38

	L1prefetchw srcreg+srcinc+d2+d1, L1pt
	vshuff64x2 zmm13, zmm13, zmm10, 00100010b	;; (+64 d7_4 d6_4 d5_4 d4_4 d3_4 d2_4 d1_4 d0_4)(I5)	; 33-35		n 37
	zfmsubpd zmm1, zmm1, zmm21, zmm0		;; B8 = I8 * cosine/sine - R8				; 33-36		n 39

	L1prefetchw srcreg+srcinc+d4, L1pt
	zfmaddpd zmm0, zmm22, zmm26, zmm6		;; A6 = R6 * cosine/sine + I6				; 34-37		n 40
	zfmsubpd zmm6, zmm6, zmm26, zmm22		;; B6 = I6 * cosine/sine - R6				; 34-37		n 41

	L1prefetchw srcreg+srcinc+d4+d2, L1pt
	zfnmaddpd zmm10, zmm14, zmm20, zmm11		;; R3-(R7 = A7 * sine)					; 35-38		n 44
	zfmaddpd zmm14, zmm14, zmm20, zmm11		;; R3+(R7 = A7 * sine)					; 35-38		n 47

	L1prefetchw srcreg+srcinc+d4+d1, L1pt
	zfmaddpd zmm11, zmm8, zmm20, zmm9		;; I3+(I7 = B7 * sine)					; 36-39		n 44
	zfnmaddpd zmm8, zmm8, zmm20, zmm9		;; I3-(I7 = B7 * sine)					; 36-39		n 48

	L1prefetchw srcreg+srcinc+d4+d2+d1, L1pt
	zfmaddpd zmm9, zmm2, zmm25, zmm13		;; A5 = R5 * cosine/sine + I5				; 37-40		n 42
	zfmsubpd zmm13, zmm13, zmm25, zmm2		;; B5 = I5 * cosine/sine - R5				; 37-40		n 43

	L1prefetchw srcreg+srcinc+d2+64, L1pt
	zfmaddpd zmm2, zmm3, zmm19, zmm15		;; R2+(R8 = A8 * sine)					; 38-41		n 45
	zfnmaddpd zmm3, zmm3, zmm19, zmm15		;; R2-(R8 = A8 * sine)					; 38-41		n 49

	L1prefetchw srcreg+srcinc+64, L1pt
	zfnmaddpd zmm15, zmm1, zmm19, zmm4		;; I2-(I8 = B8 * sine)					; 39-42		n 46
	zfmaddpd zmm1, zmm1, zmm19, zmm4		;; I2+(I8 = B8 * sine)					; 39-42		n 50

	L1prefetchw srcreg+srcinc+d2+d1+64, L1pt
	zfmaddpd zmm4, zmm0, zmm28, zmm5		;; R4+(R6 = A6 * sine)					; 40-43		n 45
	zfnmaddpd zmm0, zmm0, zmm28, zmm5		;; R4-(R6 = A6 * sine)					; 40-43		n 50

	L1prefetchw srcreg+srcinc+d1+64, L1pt
	zfnmaddpd zmm5, zmm6, zmm28, zmm12		;; I4-(I6 = B6 * sine)					; 41-44		n 46
	zfmaddpd zmm6, zmm6, zmm28, zmm12		;; I4+(I6 = B6 * sine)					; 41-44		n 49

	L1prefetchw srcreg+srcinc+d4+d2+64, L1pt
	zfmaddpd zmm12, zmm9, zmm17, zmm16		;; r1++ = (r1+r9) + r5*sine				; 42-45		n 47
	zfnmaddpd zmm9, zmm9, zmm17, zmm16		;; r1+- = (r1+r9) - r5*sine				; 42-45		n 48

	L1prefetchw srcreg+srcinc+d4+64, L1pt
	zfmaddpd zmm16, zmm13, zmm17, zmm18		;; r1-+ = (r1-r9) + i5*sine				; 43-46		n 54
	zfnmaddpd zmm13, zmm13, zmm17, zmm18		;; r1-- = (r1-r9) - i5*sine				; 43-46		n 56

	L1prefetchw srcreg+srcinc+d4+d2+d1+64, L1pt
	vaddpd	zmm18, zmm10, zmm11			;; r3-+ = (r3-r7) + (i3+i7)				; 44-47		n 54
	vsubpd	zmm10, zmm10, zmm11			;; r3-- = (r3-r7) - (i3+i7)				; 44-47		n 56

	L1prefetchw srcreg+srcinc+d4+d1+64, L1pt
	vaddpd	zmm11, zmm2, zmm4			;; r2++ = (r2+r8) + (r4+r6)				; 45-48		n 51
	vsubpd	zmm2, zmm2, zmm4			;; r2+- = (r2+r8) - (r4+r6)				; 45-48		n 53

;	L1prefetch screg+1*128+64, L1pt
	vsubpd	zmm4, zmm15, zmm5			;; i2-- = (i2-i8) - (i4-i6)				; 46-49		n 52
	vaddpd	zmm15, zmm15, zmm5			;; i2-+ = (i2-i8) + (i4-i6)				; 46-49		n 53

;	L1prefetch screg+0*128+64, L1pt
	vaddpd	zmm5, zmm12, zmm14			;; r1+++ = (r1++) + (r3+r7)				; 47-50		n 51
	vsubpd	zmm12, zmm12, zmm14			;; r1++- = (r1++) - (r3+r7)				; 47-50		n 52

;	L1prefetch screg+2*128+64, L1pt
	vaddpd	zmm14, zmm9, zmm8			;; r1+-+ = (r1+-) + (i3-i7)				; 48-51		n 58
	vsubpd	zmm9, zmm9, zmm8			;; r1+-- = (r1+-) - (i3-i7)				; 48-51		n 59

;	L1prefetch screg+1*128, L1pt
	vaddpd	zmm8, zmm3, zmm6			;; r2-+ = (r2-r8) + (i4+i6)				; 49-52		n 55
	vsubpd	zmm3, zmm3, zmm6			;; r2-- = (r2-r8) - (i4+i6)				; 49-52		n 57

;	L1prefetch screg+0*128, L1pt
	vaddpd	zmm6, zmm1, zmm0			;; i2++ = (i2+i8) + (r4-r6)				; 50-53		n 55
	vsubpd	zmm1, zmm1, zmm0			;; i2+- = (i2+i8) - (r4-r6)				; 50-53		n 57

;	L1prefetch screg+2*128, L1pt
	vaddpd	zmm0, zmm5, zmm11			;; R1 = (r1+++) + (r2++)				; 51-54
	vsubpd	zmm5, zmm5, zmm11			;; R9 = (r1+++) - (r2++)				; 51-54

;	L1prefetch screg+5*128+64, L1pt
	vaddpd	zmm11, zmm12, zmm4			;; R5  = (r1++-) + (i2--)				; 52-55
	vsubpd	zmm12, zmm12, zmm4			;; R13 = (r1++-) - (i2--)				; 52-55

;	L1prefetch screg+6*128+64, L1pt
	vaddpd	zmm4, zmm2, zmm15			;; r2+-+ = (r2+-) + (i2-+)				; 53-56		n 58
	vsubpd	zmm2, zmm2, zmm15			;; r2+-- = (r2+-) - (i2-+)				; 53-56		n 59

;	L1prefetch screg+4*128+64, L1pt
	zfmaddpd zmm15, zmm18, zmm31, zmm16		;; r2_10o = (r1-+) + .707(r3-+)				; 54-57		n 60
	zfnmaddpd zmm18, zmm18, zmm31, zmm16		;; r6_14o = (r1-+) - .707(r3-+)				; 54-57		n 61

;	L1prefetch screg+5*128, L1pt
	zfmaddpd zmm16, zmm8, zmm30, zmm6		;; r2_10e/.383 = .924/.383(r2-+) + (i2++)		; 55-58		n 60
	zfmsubpd zmm6, zmm6, zmm30, zmm8		;; r6_14e/.383 = .924/.383(i2++) - (r2-+)		; 55-58		n 61
	zstore	[srcreg], zmm0				;; Save R1						; 55

;	L1prefetch screg+3*128+64, L1pt
	zfnmaddpd zmm8, zmm10, zmm31, zmm13		;; r4_12o = (r1--) - .707(r3--)				; 56-59		n 62
	zfmaddpd zmm10, zmm10, zmm31, zmm13		;; r8_16o = (r1--) + .707(r3--)				; 56-59		n 63
	zstore	[srcreg+64], zmm5			;; Save R9						; 55+1

;	L1prefetch screg+6*128, L1pt
	zfmaddpd zmm13, zmm1, zmm30, zmm3		;; r4_12e/.383 = .924/.383(i2+-) + (r2--)		; 57-60		n 62
	zfnmaddpd zmm3, zmm3, zmm30, zmm1		;; r8_16e/.383 = (i2+-) - .924/.383(r2--)		; 57-60		n 63
	zstore	[srcreg+d4], zmm11			;; Save R5						; 56+1

;	L1prefetch screg+4*128, L1pt
	zfmaddpd zmm1, zmm4, zmm31, zmm14		;; R3  = (r1+-+) + .707(r2+-+)				; 58-61
	zfnmaddpd zmm4, zmm4, zmm31, zmm14		;; R11 = (r1+-+) - .707(r2+-+)				; 58-61
	zstore	[srcreg+d4+64], zmm12			;; Save R13						; 56+2

;	L1prefetch screg+3*128, L1pt
	zfnmaddpd zmm14, zmm2, zmm31, zmm9		;; R7  = (r1+--) - .707(r2+--)				; 59-62
	zfmaddpd zmm2, zmm2, zmm31, zmm9		;; R15 = (r1+--) + .707(r2+--)				; 59-62

	zfmaddpd zmm9, zmm16, zmm29, zmm15		;; R2  = r2_10o + .383*r2_10e				; 60-63
	zfnmaddpd zmm16, zmm16, zmm29, zmm15		;; R10 = r2_10o - .383*r2_10e				; 60-63

	zfmaddpd zmm15, zmm6, zmm29, zmm18		;; R6  = r6_14o + .383*r6_14e				; 61-64
	zfnmaddpd zmm6, zmm6, zmm29, zmm18		;; R14 = r6_14o - .383*r6_14e				; 61-64

	zfmaddpd zmm18, zmm13, zmm29, zmm8		;; R4  = r4_12o + .383*r4_12e				; 62-65
	zfnmaddpd zmm13, zmm13, zmm29, zmm8		;; R12 = r4_12o - .383*r4_12e				; 62-65
	zstore	[srcreg+d2], zmm1			;; Save R3						; 62

	zfmaddpd zmm8, zmm3, zmm29, zmm10		;; R8  = r8_16o + .383*r8_16e				; 63-66
	zfnmaddpd zmm3, zmm3, zmm29, zmm10		;; R16 = r8_16o - .383*r8_16e				; 63-66
 	zstore	[srcreg+d2+64], zmm4			;; Save R11						; 62+1

	zstore	[srcreg+d4+d2], zmm14			;; Save R7						; 63+1
	zstore	[srcreg+d4+d2+64], zmm2			;; Save R15						; 63+2
	zstore	[srcreg+d1], zmm9			;; Save R2						; 64+2
	zstore	[srcreg+d1+64], zmm16			;; Save R10						; 64+3
	zstore	[srcreg+d4+d1], zmm15			;; Save R6						; 65+3
	zstore	[srcreg+d4+d1+64], zmm6			;; Save R14						; 65+4
	zstore	[srcreg+d2+d1], zmm18			;; Save R4						; 66+4
	zstore	[srcreg+d2+d1+64], zmm13		;; Save R12						; 66+5
	zstore	[srcreg+d4+d2+d1], zmm8			;; Save R8						; 67+5
	zstore	[srcreg+d4+d2+d1+64], zmm3		;; Save R16						; 67+6
	bump	srcreg, srcinc
	ENDM


zr8_sixteen_reals_eight_complex_djbfft_preload MACRO
	mov	r9d, 11111110b			;; We're pretty sure r9 is safe to use
	kmovw	k7, r9d				;; Set k7 to 11111110b
	knotw	k6, k7				;; Set k6 to 00000001b
	vbroadcastsd zmm0, ZMM_SQRTHALF
	vbroadcastsd zmm28 {k7}{z}, ZMM_ONE			;; 1, 1, 1, 1, 1, 1, 1		0
	vsubpd	zmm28 {k6}, zmm28, zmm0				;; 1, 1, 1, 1, 1, 1, 1		-SQRTHALF
	vblendmpd zmm29 {k6}, zmm28, ZMM_P924_P383{1to8}	;; 1, 1, 1, 1, 1, 1, 1		.924/.383
	vblendmpd zmm30 {k6}, zmm28, zmm0			;; 1, 1, 1, 1, 1, 1, 1		SQRTHALF
	vblendmpd zmm31 {k6}, zmm0, ZMM_P383{1to8}		;; SQRTHALF, SQRTHALF, ...,	.383
	zblendmpd_preload zmm27
	ENDM
zr8_sixteen_reals_eight_complex_djbfft MACRO srcreg,srcinc,d1,d2,d4,screg,scinc,maxrpt,L1pt,L1pd
						;; Simultaneously do an 8-complex and 16-reals
						;; 8-complex			16-reals
	vmovapd	zmm9, [srcreg]			;; R1				R1+R9 (new R1)
	vmovapd	zmm0, [srcreg+d4]		;; R5				R5+R13 (new R5)
	vaddpd	zmm2, zmm9, zmm0		;;		R1 + R5 (new R1)				; 1-4		n 11

	vmovapd	zmm11, [srcreg+d2+d1]		;; R4				R4+R12 (new R4)
	vmovapd	zmm23, [srcreg+d4+d2+d1]	;; R8				R8+R16 (new R8)
	vaddpd	zmm1, zmm11, zmm23		;;		R4 + R8 (new R4)				; 1-4		n 14

						;;				R12/R16 becomes newer R12/I12
						;;				mul R12/I12 by w^3 = .383 + .924i
	vmovapd	zmm10, [srcreg+d2+d1+64]	;; I4				*R4-R12 (new R12)
	vmovapd	zmm15, [srcreg+d4+d2+d1+64]	;; I8				R8-R16 (new R16 a.k.a. I12)
	zfnmaddpd zmm6, zmm15, zmm29, zmm10	;; I4 - I8*1 (new I8)		R12/.383 = R12 - I12*.924/.383	; 2-5		n 8
	zfmaddpd zmm10, zmm10, zmm29, zmm15	;; I4*1 + I8 (new I4)		I12/.383 = R12*.924/.383 + I12	; 2-5		n 10

						;;				R11/R15 becomes newer R11/I11
						;;				mul R11/I11 by SQRTHALF + i*SQRTHALF
	vmovapd	zmm4, [srcreg+d2+64]		;; I3				R3-R11 (new R11)
	vmovapd	zmm5, [srcreg+d4+d2+64]		;; I7				R7-R15 (new R15 a.k.a. I11)
	vaddpd	zmm15, zmm4, zmm5		;; I3 + I7 (new I3)		I11/SQRTHALF = R11 + I11	; 3-6		n 9
	vsubpd	zmm4, zmm4, zmm5		;; I3 - I7 (new I7)		R11/SQRTHALF = R11 - I11	; 3-6		n 16

	vmovapd	zmm19, [srcreg+d2]		;; R3				R3+R11 (new R3)
	vmovapd	zmm24, [srcreg+d4+d2]		;; R7				R7+R15 (new R7)
	vaddpd	zmm14, zmm19, zmm24		;;		R3 + R7 (new R3)				; 4-7		n 11
	vsubpd	zmm19, zmm19, zmm24		;; 		R3 - R7 (new R7)				; 4-7		n 9

						;;				R10/R14 becomes newer R10/I10
						;;				mul R10/I10 by w^1 = .924 + .383i
	vmovapd	zmm24, [srcreg+d1+64]		;; I2				R2-R10 (new R10)
	vmovapd	zmm13, [srcreg+d4+d1+64]	;; I6				R6-R14 (new R14 a.k.a. I10)
	zfmsubpd zmm5, zmm24, zmm29, zmm13	;; I2*1 - I6 (new I6)		R10/.383 = R10*.924/.383 - I10	; 5-8		n 13
	zfmaddpd zmm24, zmm13, zmm29, zmm24	;; I2 + I6*1 (new I2)		I10/.383 = I10*.924/.383 + R10	; 5-8		n 10

	vmovapd	zmm12, [srcreg+d1]		;; R2				R2+R10 (new R2)
	vmovapd	zmm16, [srcreg+d4+d1]		;; R6				R6+R14 (new R6)
	vaddpd	zmm3, zmm12, zmm16		;;		R2 + R6 (new R2)				; 6-9		n 14
	vsubpd	zmm12, zmm12, zmm16		;;		R2 - R6 (new R6)				; 6-9		n 12

						;;				R9/R13 becomes newer R9/I9
	vmovapd	zmm16, [srcreg+64]		;; I1				R1-R9 (new R9)
	vmovapd	zmm8, [srcreg+d4+64]		;; I5				R5-R13 (new R13 a.k.a. I9)
	vaddpd	zmm13 {k7}{z}, zmm16, zmm8	;; I1 + I5 (new I1)		0				; 7-10		n 15
	vsubpd	zmm8 {k7}, zmm16, zmm8		;; I1 - I5 (new I5)		I9				; 7-10		n 17

	vsubpd	zmm16 {k7}, zmm9, zmm0		;; R1 - R5 (new R5)		blend in new R9			; 8-11		n 16
	vmovapd zmm26, [screg+3*128+64]		;; cosine/sine for R2/I2 (w^4)

	vmovapd	zmm18, zmm6			;; new I8			not important
	vsubpd	zmm18 {k6}, zmm11, zmm23	;; blend in new I8		R4 - R8 (new R8)		; 8-11		n 12
	vmovapd	zmm21, zmm6			;; not important		R12	
	vsubpd	zmm21 {k7}, zmm11, zmm23	;; R4 - R8 (new R8)		blend in R12			; 9-12		n 13
	vmovapd zmm25, [screg+1*128+64]		;; cosine/sine for R3/I3 (w^2)

						;; next FFT level
						;;				R6/R8 becomes newest R6/I6
						;;				mul R6/I6 by w^2 = .707 + .707i

	zblendmpd zmm20, k6, zmm19, zmm15	;; R7				I11/SQRTHALF			; 9-12		n 17

	vaddpd	zmm11, zmm24, zmm10		;; I2 + I4 (newer I2)		I10 + I12 (newest I10/.383)	; 10-13		n 18
	vsubpd	zmm24, zmm24, zmm10		;; I2 - I4 (newer I4)		I10 - I12 (newest I12/.383)	; 10-13		n 19
	vmovapd zmm23, [screg+5*128+64]		;; cosine/sine for R4/I4 (w^6)

	vaddpd	zmm10, zmm2, zmm14		;; R1 + R3 (newer R1)		R1 + R3 (newest R1)		; 11-14		n 19
	vsubpd	zmm2, zmm2, zmm14		;; R1 - R3 (newer R3)		R1 - R3 (newest R3)		; 11-14		n 21
	L1prefetchw srcreg+srcinc, L1pt

	vsubpd	zmm17, zmm12, zmm18		;; R6 - I8 (new2 R6)		R6 - I6 (newest R6/SQRTHALF)	; 12-15		n 18
	vaddpd	zmm12, zmm12, zmm18		;; R6 + I8 (new2 R8)		R6 + I6 (newest I6/SQRTHALF)	; 12-15		n 20
	L1prefetchw srcreg+srcinc+d4, L1pt

	vaddpd	zmm6, zmm5, zmm21		;; I6 + R8 (new2 I6)		R10 + R12 (newest R10/.383)	; 13-16		n 18
	vsubpd	zmm5, zmm5, zmm21		;; I6 - R8 (new2 I8)		R10 - R12 (newest R12/.383)	; 13-16		n 20
	vmovapd zmm21, [screg+0*128+64]		;; cosine/sine for R5/I5 (w^1)

	vaddpd	zmm14, zmm3, zmm1		;; R2 + R4 (newer R2)		R2 + R4 (newest R2)		; 14-17		n 19
	vsubpd	zmm3, zmm3, zmm1		;; R2 - R4 (newer R4)		R2 - R4 (newest R4)		; 14-17		n 21
	L1prefetchw srcreg+srcinc+d2+d1, L1pt

						;;				R5/R7 becomes newest R5/I5
	vsubpd	zmm19 {k7}, zmm13, zmm15	;; I1 - I3 (newer I3)		I5				; 15-18		n 27
	vaddpd	zmm13 {k7}{z}, zmm13, zmm15	;; I1 + I3 (newer I1)		0				; 15-18		n 23
	L1prefetchw srcreg+srcinc+d4+d2+d1, L1pt

	zfnmaddpd zmm7, zmm4, zmm28, zmm16	;; R5 - I7*1 (newer R5)		R9 - R11*-SQRTHALF (newest R9)	; 16-19		n 24
	zfmaddpd zmm4, zmm4, zmm28, zmm16	;; R5 + I7*1 (newer R7)		R9 + R11*-SQRTHALF (newest R11)	; 16-19		n 28
	L1prefetchw srcreg+srcinc+d2+d1+64, L1pt

	zfmaddpd zmm16, zmm20, zmm30, zmm8	;; I5 + R7*1 (newer I5)		I9 + I11*SQRTHALF (newest I9)	; 17-20		n 25
	zfnmaddpd zmm20, zmm20, zmm30, zmm8	;; I5 - R7*1 (newer I7)		I9 - I11*SQRTHALF (newest I11)	; 17-20		n 29
	L1prefetchw srcreg+srcinc+d4+d2+d1+64, L1pt

	vmovapd	zmm8, zmm11			;;				I10/.383
	vaddpd	zmm8 {k7}, zmm17, zmm6		;; I6 = R6 + I6 (newer I6/SQRTHALF)	I10/.383		; 18-21		n 25
	vsubpd	zmm6 {k7}, zmm17, zmm6		;; R6 = R6 - I6 (newer R6/SQRTHALF)	R10/.383		; 18-21		n 24
	L1prefetchw srcreg+srcinc+d2+64, L1pt

	vsubpd	zmm11 {k6}, zmm10, zmm14	;; I2				R1 - R2 (final R1b)		; 19-22		n 23
	zblendmpd zmm17, k6, zmm24, zmm17	;; I4				R6/SQRTHALF			; 19-22		n 26
	L1prefetchw srcreg+srcinc+d4+d2+64, L1pt

	vaddpd	zmm24 {k7}, zmm12, zmm5		;; I8 = R8 + I8 (newer I8/SQRTHALF)	I12/.383		; 20-23		n 28
	vsubpd	zmm5 {k7}, zmm12, zmm5		;; R8 = R8 - I8 (newer R8/SQRTHALF)	R12/.383		; 20-23		n 29
	L1prefetchw srcreg+srcinc+d2, L1pt

	;; last FFT level

	vmovapd zmm18, zmm2			;; R3				not important
	vsubpd	zmm18 {k6}, zmm9, zmm0		;; blend in R3			origR1 - origR5 (new R5)	; 21-24		n 26
	zblendmpd zmm12, k6, zmm3, zmm12	;; R4				I6/SQRTHALF			; 21-24		n 27
	vmovapd zmm9, [screg+4*128+64]		;; cosine/sine for R6/I6 (w^5)

						;;				R3/R4 becomes final R3/I3
	vaddpd	zmm1, zmm10, zmm14		;; R1 + R2 (final R1)		R1 + R2 (final R1a => R1)	; 22-25
	vsubpd	zmm2 {k7}, zmm10, zmm14		;; R1 - R2 (final R2)		R3 => R2			; 22-25		n 30
	zstore	[srcreg], zmm1			;; Save R1							; 26
	vmovapd zmm1, [screg+2*128+64]		;; cosine/sine for R7/I7 (w^3)

	vsubpd	zmm3 {k7}, zmm13, zmm11		;; I1 - I2 (final I2)		I3 => I2			; 23-26		n 30
	vaddpd	zmm11 {k7}, zmm13, zmm11	;; I1 + I2 (final I1)		R1b => I1			; 23-26
	zstore	[srcreg+64], zmm11		;; Save I1							; 27
	vmovapd zmm11, [screg+6*128+64]		;; cosine/sine for R8/I8 (w^7)

	zfmaddpd zmm10, zmm6, zmm31, zmm7	;; R5 + R6*SQRTHALF (final R5)	R9 + R10*.383 (final R9 => R5)	; 24-27		n 33
	zfnmaddpd zmm6, zmm6, zmm31, zmm7	;; R5 - R6*SQRTHALF (final R6)	R9 - R10*.383 (final R10 => R6)	; 24-27		n 34
	L1prefetchw srcreg+srcinc+d4+d2, L1pt

	zfmaddpd zmm7, zmm8, zmm31, zmm16	;; I5 + I6*SQRTHALF (final I5)	I9 + I10*.383 (final I9 => I5)	; 25-28		n 33
	zfnmaddpd zmm8, zmm8, zmm31, zmm16	;; I5 - I6*SQRTHALF (final I6)	I9 - I10*.383 (final I10 => I6)	; 25-28		n 34
	L1prefetchw srcreg+srcinc+d1+64, L1pt

	zfnmaddpd zmm13, zmm17, zmm28, zmm18	;; R3 - I4*1 (final R3)		R5 - R6*-SQRTHALF (final R5 => R3) ; 26-29	n 31
	zfmaddpd zmm18, zmm17, zmm28, zmm18	;; R3 + I4*1 (final R4)		R5 + R6*-SQRTHALF (final R6 => R4) ; 26-29	n 32
	vmovapd zmm17, [screg+3*128]		;; sine for R2/I2 (w^4)

	zfmaddpd zmm15, zmm12, zmm30, zmm19	;; I3 + R4*1 (final I3)		I5 + I6*SQRTHALF (final I5 => I3) ; 27-30	n 31
	zfnmaddpd zmm12, zmm12, zmm30, zmm19	;; I3 - R4*1 (final I4)		I5 - I6*SQRTHALF (final I6 => I4) ; 27-30	n 32
	vmovapd zmm19, [screg+1*128]		;; sine for R3/I3 (w^2)

	zfnmaddpd zmm14, zmm24, zmm31, zmm4	;; R7 - I8*SQRTHALF (final R7)	R11 - I12*.383 (final R11 => R7) ; 28-31	n 35
	zfmaddpd zmm24, zmm24, zmm31, zmm4	;; R7 + I8*SQRTHALF (final R8)	R11 + I12*.383 (final R12 => R8) ; 28-31	n 36
	L1prefetchw srcreg+srcinc+d4+d1+64, L1pt

	zfmaddpd zmm4, zmm5, zmm31, zmm20	;; I7 + R8*SQRTHALF (final I7)	I11 + R12*.383 (final I11 => I7) ; 29-32	n 35
	zfnmaddpd zmm5, zmm5, zmm31, zmm20	;; I7 - R8*SQRTHALF (final I8)	I11 - R12*.383 (final I12 => I8) ; 29-32	n 36
	vmovapd zmm20, [screg+5*128]		;; sine for R4/I4 (w^6)

	zfmsubpd zmm16, zmm2, zmm26, zmm3	;; A2 = R2 * cosine/sine - I2					; 30-33		n 37
	zfmaddpd zmm3, zmm3, zmm26, zmm2	;; B2 = I2 * cosine/sine + R2					; 30-33		n 37
	vmovapd zmm26, [screg+0*128]		;; sine for R5/I5 (w^1)

	zfmsubpd zmm2, zmm13, zmm25, zmm15	;; A3 = R3 * cosine/sine - I3					; 31-34		n 38
	zfmaddpd zmm15, zmm15, zmm25, zmm13	;; B3 = I3 * cosine/sine + R3					; 31-34		n 38
	vmovapd zmm25, [screg+4*128]		;; sine for R6/I6 (w^5)

	zfmsubpd zmm13, zmm18, zmm23, zmm12	;; A4 = R4 * cosine/sine - I4					; 32-35		n 39
	zfmaddpd zmm12, zmm12, zmm23, zmm18	;; B4 = I4 * cosine/sine + R4					; 32-35		n 39
	vmovapd zmm23, [screg+2*128]		;; sine for R7/I7 (w^3)

	zfmsubpd zmm18, zmm10, zmm21, zmm7	;; A5 = R5 * cosine/sine - I5					; 33-36		n 40
	zfmaddpd zmm7, zmm7, zmm21, zmm10	;; B5 = I5 * cosine/sine + R5					; 33-36		n 40
	vmovapd zmm21, [screg+6*128]		;; sine for R8/I8 (w^7)
	bump	screg, scinc

	zfmsubpd zmm10, zmm6, zmm9, zmm8	;; A6 = R6 * cosine/sine - I6					; 34-37		n 41
	zfmaddpd zmm8, zmm8, zmm9, zmm6		;; B6 = I6 * cosine/sine + R6					; 34-37		n 41
	L1prefetchw srcreg+srcinc+d1, L1pt

	zfmsubpd zmm6, zmm14, zmm1, zmm4	;; A7 = R7 * cosine/sine - I7					; 35-38		n 42
	zfmaddpd zmm4, zmm4, zmm1, zmm14	;; B7 = I7 * cosine/sine + R7					; 35-38		n 42
	L1prefetchw srcreg+srcinc+d4+d1, L1pt

	zfmsubpd zmm14, zmm24, zmm11, zmm5	;; A8 = R8 * cosine/sine - I8					; 36-39		n 43
	zfmaddpd zmm5, zmm5, zmm11, zmm24	;; B8 = I8 * cosine/sine + R8					; 36-39		n 43
	L1prefetchw srcreg+srcinc+64, L1pt

	vmulpd	zmm16, zmm16, zmm17		;; A2 = A2 * sine (final R2)					; 37-40
	vmulpd	zmm3, zmm3, zmm17		;; B2 = B2 * sine (final I2)					; 37-40
	L1prefetchw srcreg+srcinc+d4+64, L1pt

	vmulpd	zmm2, zmm2, zmm19		;; A3 = A3 * sine (final R3)					; 38-41
	vmulpd	zmm15, zmm15, zmm19		;; B3 = B3 * sine (final I3)					; 38-41

	vmulpd	zmm13, zmm13, zmm20		;; A4 = A4 * sine (final R4)					; 39-42
	vmulpd	zmm12, zmm12, zmm20		;; B4 = B4 * sine (final I4)					; 39-42

	vmulpd	zmm18, zmm18, zmm26		;; A5 = A5 * sine (final R5)					; 40-43
	vmulpd	zmm7, zmm7, zmm26		;; B5 = B5 * sine (final I5)					; 40-43

	vmulpd	zmm10, zmm10, zmm25		;; A6 = A6 * sine (final R6)					; 41-44
	vmulpd	zmm8, zmm8, zmm25		;; B6 = B6 * sine (final I6)					; 41-44
	zstore	[srcreg+d1], zmm16		;; Save R2							; 41

	vmulpd	zmm6, zmm6, zmm23		;; A7 = A7 * sine (final R7)					; 42-45
	vmulpd	zmm4, zmm4, zmm23		;; B7 = B7 * sine (final I7)					; 42-45
	zstore	[srcreg+d1+64], zmm3		;; Save I2							; 42

	vmulpd	zmm14, zmm14, zmm21		;; A8 = A8 * sine (final R8)					; 43-46
	vmulpd	zmm5, zmm5, zmm21		;; B8 = B8 * sine (final I8)					; 43-46


	zstore	[srcreg+d2], zmm2		;; Save R3
	zstore	[srcreg+d2+64], zmm15		;; Save I3
	zstore	[srcreg+d2+d1], zmm13		;; Save R4
	zstore	[srcreg+d2+d1+64], zmm12	;; Save I4
	zstore	[srcreg+d4], zmm18		;; Save R5
	zstore	[srcreg+d4+64], zmm7		;; Save I5
	zstore	[srcreg+d4+d1], zmm10		;; Save R6
	zstore	[srcreg+d4+d1+64], zmm8		;; Save I6
	zstore	[srcreg+d4+d2], zmm6		;; Save R7
	zstore	[srcreg+d4+d2+64], zmm4		;; Save I7
	zstore	[srcreg+d4+d2+d1], zmm14	;; Save R8
	zstore	[srcreg+d4+d2+d1+64], zmm5	;; Save I8
	bump	srcreg, srcinc
	ENDM


zr8_sixteen_reals_eight_complex_djbunfft_preload MACRO
	mov	r9d, 11111110b			;; We're pretty sure r9 is safe to use
	kmovw	k7, r9d				;; Set k7 to 11111110b
	knotw	k6, k7				;; Set k6 to 00000001b
	vbroadcastsd zmm0, ZMM_SQRTHALF
	vbroadcastsd zmm26, ZMM_ONE
	vblendmpd zmm30 {k6}, zmm0, zmm26		 	;; SQRTHALF (7 times)	1
	vblendmpd zmm27 {k6}, zmm0, ZMM_P383{1to8}		;; SQRTHALF (7 times)	.383
	vblendmpd zmm26 {k6}, zmm26, ZMM_P924_P383{1to8}	;; 1, 1, 1, 1, 1, 1, 1	.924/.383
	vblendmpd zmm29 {k6}, zmm26, zmm0			;; 1, 1, 1, 1, 1, 1, 1	SQRTHALF
	vbroadcastsd zmm28, ZMM_NEGSQRTHALF			;; -SQRTHALF
	zblendmpd_preload zmm31
	ENDM
zr8_sixteen_reals_eight_complex_djbunfft MACRO srcreg,srcinc,d1,d2,d4,screg,scinc,maxrpt,L1pt,L1pd
	vmovapd	zmm0, [srcreg+d1]		;; R2
	vmovapd	zmm16, [srcreg+d1+64]		;; I2
	vmovapd zmm24, [screg+3*128+64]		;; cosine/sine for R2/I2 (w^4)
	zfmaddpd zmm17, zmm0, zmm24, zmm16	;; A2 = R2 * cosine/sine + I2					; 1-4		n 11
	zfmsubpd zmm16, zmm16, zmm24, zmm0	;; B2 = I2 * cosine/sine - R2					; 1-4		n 11

	vmovapd	zmm0, [srcreg+d2]		;; R3
	vmovapd	zmm18, [srcreg+d2+64]		;; I3
	vmovapd zmm24, [screg+1*128+64]		;; cosine/sine for R3/I3 (w^2)
	zfmaddpd zmm19, zmm0, zmm24, zmm18	;; A3 = R3 * cosine/sine + I3					; 2-5		n 8
	zfmsubpd zmm18, zmm18, zmm24, zmm0	;; B3 = I3 * cosine/sine - R3					; 2-5		n 8

	vmovapd	zmm0, [srcreg+d2+d1]		;; R4
	vmovapd	zmm20, [srcreg+d2+d1+64]	;; I4
	vmovapd zmm24, [screg+5*128+64]		;; cosine/sine for R4/I4 (w^6)
	zfmaddpd zmm21, zmm0, zmm24, zmm20	;; A4 = R4 * cosine/sine + I4					; 3-6		n 12
	zfmsubpd zmm20, zmm20, zmm24, zmm0	;; B4 = I4 * cosine/sine - R4					; 3-6		n 13

	vmovapd	zmm0, [srcreg+d4]		;; R5
	vmovapd	zmm2, [srcreg+d4+64]		;; I5
	vmovapd zmm24, [screg+0*128+64]		;; cosine/sine for R5/I5 (w^1)
	zfmaddpd zmm3, zmm0, zmm24, zmm2	;; A5 = R5 * cosine/sine + I5					; 4-7		n 9
	zfmsubpd zmm2, zmm2, zmm24, zmm0	;; B5 = I5 * cosine/sine - R5					; 4-7		n 9

	vmovapd	zmm0, [srcreg+d4+d1]		;; R6
	vmovapd	zmm13, [srcreg+d4+d1+64]	;; I6
	vmovapd zmm24, [screg+4*128+64]		;; cosine/sine for R6/I6 (w^5)
	zfmaddpd zmm15, zmm0, zmm24, zmm13	;; A6 = R6 * cosine/sine + I6					; 5-8		n 14
	zfmsubpd zmm13, zmm13, zmm24, zmm0	;; B6 = I6 * cosine/sine - R6					; 5-8		n 15

	vmovapd	zmm0, [srcreg+d4+d2]		;; R7
	vmovapd	zmm1, [srcreg+d4+d2+64]		;; I7
	vmovapd zmm24, [screg+2*128+64]		;; cosine/sine for R7/I7 (w^3)
	zfmaddpd zmm9, zmm0, zmm24, zmm1	;; A7 = R7 * cosine/sine + I7					; 6-9		n 10
	zfmsubpd zmm1, zmm1, zmm24, zmm0	;; B7 = I7 * cosine/sine - R7					; 6-9		n 10

	vmovapd	zmm0, [srcreg+d4+d2+d1]		;; R8
	vmovapd	zmm7, [srcreg+d4+d2+d1+64]	;; I8
	vmovapd zmm24, [screg+6*128+64]		;; cosine/sine for R8/I8 (w^7)
	zfmaddpd zmm11, zmm0, zmm24, zmm7	;; A8 = R8 * cosine/sine + I8					; 7-10		n 16
	zfmsubpd zmm7, zmm7, zmm24, zmm0	;; B8 = I8 * cosine/sine - R8					; 7-10		n 17

	vmovapd zmm22, [screg+1*128]		;; sine for R3/I3 (w^2)
	vmulpd	zmm19, zmm19, zmm22		;; A3 = A3 * sine (final R3)					; 8-11		n 12
	vmulpd	zmm18, zmm18, zmm22		;; B3 = B3 * sine (final I3)					; 8-11		n 13

	vmovapd zmm22, [screg+0*128]		;; sine for R5/I5 (w^1)
	vmulpd	zmm3, zmm3, zmm22		;; A5 = A5 * sine (final R5)					; 9-12		n 14
	vmulpd	zmm2, zmm2, zmm22		;; B5 = B5 * sine (final I5)					; 9-12		n 15

	vmovapd zmm22, [screg+2*128]		;; sine for R7/I7 (w^3)
	vmulpd	zmm9, zmm9, zmm22		;; A7 = A7 * sine (final R7)					; 10-13		n 16
	vmulpd	zmm1, zmm1, zmm22		;; B7 = B7 * sine (final I7)					; 10-13		n 17

						;; Simultaneously do an 8-complex and 16-reals
						;; 8-complex			16-reals

						;;				R1a/R1b becomes R1/R2, R2/I2 becomes R3/R4, R3/I3 => R5/I5, etc.
	vmovapd zmm24, [screg+3*128]		;; sine for R2/I2 (w^4)
	vmovapd	zmm22, [srcreg+64]		;; I1				R1b
	vmovapd	zmm12, zmm22			;; not important		R2
	vmulpd	zmm12 {k7}, zmm17, zmm24	;; R2*sine			blend in R2			; 11-14		n 18

	vmovapd zmm25, [screg+5*128]		;; sine for R4/I4 (w^6)
	zfmaddpd zmm14, zmm16, zmm24, zmm22	;; I1 + I2*sine (new I1)	not important			; 11-14		n 31
	zfnmaddpd zmm16 {k7}, zmm16, zmm24, zmm22 ;; I1 - I2*sine (new I2)	blend in R4			; 12-15		n 20

	vmovapd zmm23, [screg+4*128]		;; sine for R6/I6 (w^5)
	zfmaddpd zmm8, zmm21, zmm25, zmm19	;; R4*sine + R3 (new R3)	R6*sine + R5 (new R5)		; 12-15		n 19
	zfmsubpd zmm21, zmm21, zmm25, zmm19	;; R4*sine - R3 (new I4)	R6*sine - R5 (new negR6)	; 13-16		n 24

	vmovapd zmm22, [screg+6*128]		;; sine for R8/I8 (w^7)
	zfnmaddpd zmm6, zmm20, zmm25, zmm18	;; I3 - I4*sine (new R4)	I5 - I6*sine (new I6)		; 13-16		n 20
	zfmaddpd zmm18, zmm20, zmm25, zmm18	;; I3 + I4*sine (new I3)	I5 + I6*sine (new I5)		; 14-17		n 30
	bump	screg, scinc

	vmovapd	zmm25, [srcreg]			;; R1				R1a
	zfnmaddpd zmm4, zmm15, zmm23, zmm3	;; R5 - R6*sine (new R6)	R9 - R10*sine (new R10)		; 14-17		n 21
	zfmaddpd zmm3, zmm15, zmm23, zmm3	;; R5 + R6*sine (new R5)	R9 + R10*sine (new R9)		; 15-18		n 22

	zfnmaddpd zmm10, zmm13, zmm23, zmm2	;; I5 - I6*sine (new I6)	I9 - I10*sine (new I10)		; 15-18		n 25
	zfmaddpd zmm2, zmm13, zmm23, zmm2	;; I5 + I6*sine (new I5)	I9 + I10*sine (new I9)		; 16-19		n 23
	L1prefetchw srcreg+srcinc+d1, L1pt

	zfmsubpd zmm0, zmm11, zmm22, zmm9	;; R8*sine - R7 (new I8)	R12*sine - R11 (new I12)	; 16-19		n 21
	zfmaddpd zmm11, zmm11, zmm22, zmm9	;; R8*sine + R7 (new R7)	R12*sine + R11 (new R11)	; 17-20		n 22
	L1prefetchw srcreg+srcinc+d1+64, L1pt

	zfnmaddpd zmm5, zmm7, zmm22, zmm1	;; I7 - I8*sine (new R8)	I11 - I12*sine (new R12)	; 17-20		n 22
	zfmaddpd zmm1, zmm7, zmm22, zmm1	;; I7 + I8*sine (new I7)	I11 + I12*sine (new I11)	; 18-21		n 23
	L1prefetchw srcreg+srcinc+d2, L1pt

	vaddpd	zmm9, zmm25, zmm12		;; R1 + R2 (new R1)		R1 + R2 (new R1)		; 18-21		n 27
	vsubpd	zmm25, zmm25, zmm12		;; R1 - R2 (new R2)		R1 - R2 (new R2)		; 19-22		n 28
	L1prefetchw srcreg+srcinc+d2+64, L1pt

;; two aparts

	vmovapd	zmm7, zmm8			;; R3				not important
	vmulpd	zmm7 {k6}, zmm17, zmm24		;; blend in R3			R3*sine				; 19-22		n 27
	vmovapd	zmm17, zmm6			;; R4				not important
	vmulpd	zmm17 {k6}, zmm16, zmm24	;; blend in R4			R4*sine				; 20-23		n 28
	L1prefetchw srcreg+srcinc+d2+d1, L1pt

	zblendmpd zmm16, k6, zmm16, zmm6	;; I2				I6				; 20-23		n 24
	vmovapd	zmm12, zmm0			;; not important		I12
	vmulpd	zmm12 {k7}, zmm4, zmm27		;; R6*SQRTHALF			blend in I12			; 21-24		n 25
	zblendmpd zmm0, k6, zmm0, zmm4		;; I8				R10				; 21-24		n 26
	vmulpd	zmm5, zmm5, zmm27		;; R8*SQRTHALF			R12*.383			; 22-25		n 26
	L1prefetchw srcreg+srcinc+d2+d1, L1pt

	vaddpd	zmm4, zmm11, zmm3		;; R7 + R5 (newer R5)		R11 + R9 (newer R9)		; 22-25		n 29
	vsubpd	zmm11, zmm11, zmm3		;; R7 - R5 (newer I7)		R11 - R9 (newer negR11)		; 23-26		n 32
	L1prefetchw srcreg+srcinc+d2+d1+64, L1pt

	vsubpd	zmm6, zmm2, zmm1		;; I5 - I7 (newer R7)		I9 - I11 (newer I11)		; 23-26		n 30
	vaddpd	zmm2, zmm2, zmm1		;; I5 + I7 (newer I5)		I9 + I11 (newer I9)		; 24-27		n 37
	L1prefetchw srcreg+srcinc+d4, L1pt

						;;				mul R6/I6 by w^2 = .707 - .707i
	vaddpd	zmm3, zmm16, zmm21		;; I2 + I4 (newer I2)		I6 + negR6 (newer I6/SQRTHALF) 	; 24-27		n 30
	vsubpd	zmm16, zmm16, zmm21		;; I2 - I4 (newer I4)		I6 - negR6 (newer R6/SQRTHALF) 	; 25-28		n 30
	L1prefetchw srcreg+srcinc+d4+64, L1pt

						;; multiply R6/I6 by SQRTHALF - i*SQRTHALF
	zfmaddpd zmm1, zmm10, zmm30, zmm12	;; I6*SQRTHALF + R6 (new2 R6)	I10*1 + I12 (newer I10)		; 25-28		n 30
	zfmsubpd zmm12, zmm10, zmm30, zmm12	;; I6*SQRTHALF - R6 (new2 I6)	I10*1 - I12 (newer I12)		; 26-29		n 31
	L1prefetchw srcreg+srcinc+d4+d1, L1pt

						;; multiply R8/I8 by SQRTHALF - i*SQRTHALF
	zfmaddpd zmm10, zmm0, zmm27, zmm5	;; I8*SQRTHALF + R8 (new2 R8)	R10*.383 + R12 (newer R10*.383)	; 26-29		n 33
	zfmsubpd zmm0, zmm0, zmm27, zmm5	;; I8*SQRTHALF - R8 (new2 I8)	R10*.383 - R12 (newer R12*.383)	; 27-30		n 34
	L1prefetchw srcreg+srcinc+d4+d1+64, L1pt

	vaddpd	zmm5, zmm9, zmm7		;; R1 + R3 (newer R1)		R1 + R3 (newer R1)		; 27-30		n 35
	vsubpd	zmm9, zmm9, zmm7		;; R1 - R3 (newer R3)		R1 - R3 (newer R3)		; 28-31		n 36
	L1prefetchw srcreg+srcinc+d4+d2, L1pt

	vaddpd	zmm7, zmm25, zmm17		;; R2 + R4 (newer R2)		R2 + R4 (newer R2)		; 28-31		n 39
	vsubpd	zmm25, zmm25, zmm17		;; R2 - R4 (newer R4)		R2 - R4 (newer R4)		; 29-32		n 40
	L1prefetchw srcreg+srcinc+d4+d2+64, L1pt

						;;				R5/I5 becomes newer R5/R7
	zblendmpd zmm8, k6, zmm4, zmm8		;; newer R5			newer R5			; 29-32		n 35
	zblendmpd zmm19, k6, zmm6, zmm18	;; newer R7			newer R7			; 30-33		n 36
	vmovapd	zmm17, zmm3			;; newer I2			not important
	vmulpd	zmm17 {k6}, zmm1, zmm27		;; blend in newer I2		I10*.383			; 30-33		n 41
	vmovapd	zmm20, zmm16			;; newer I4			not important
	vmulpd	zmm20 {k6}, zmm12, zmm27	;; blend in newer I4		I12*.383			; 31-34		n 42
	vmovapd	zmm21, zmm0			;; new2 I8			newer R12*.383
	L1prefetchw srcreg+srcinc+d4+d2+d1, L1pt

	vaddpd	zmm13 {k7}{z}, zmm14, zmm18	;; I1 + I3 (newer I1)		0				; 31-34		n 37
	vsubpd	zmm6 {k7}, zmm14, zmm18		;; I1 - I3 (newer I3)		newer I11			; 32-35		n 38
	vmulpd	zmm11 {k6}, zmm11, zmm28	;; newer I7			R11*SQRTHALF = negR11*-SQRTHALF	; 32-35		n 38
	L1prefetchw srcreg+srcinc+d4+d2+d1+64, L1pt

	vaddpd	zmm16 {k7}, zmm1, zmm10		;; R6 + R8 (newer R6)		newer R6/SQRTHALF		; 33-36		n 39
	vsubpd	zmm0 {k7}, zmm10, zmm1		;; R8 - R6 (newer I8)		newer R12*.383			; 33-36		n 42
	L1prefetchw srcreg+srcinc, L1pt

	vsubpd zmm3 {k7}, zmm12, zmm21		;; I6 - I8 (newer R8)		newer I6/SQRTHALF		; 34-37		n 40
	vaddpd zmm10 {k7}, zmm12, zmm21		;; I6 + I8 (newer I6)		newer R10*.383			; 34-37		n 41
	L1prefetchw srcreg+srcinc+64, L1pt

;; four aparts

	vaddpd	zmm1, zmm5, zmm8		;; R1 + R5 (final R1)		R1 + R5 (final R1)		; 35-38
	vsubpd	zmm5, zmm5, zmm8		;; R1 - R5 (final R5)		R1 - R5 (final R5)		; 35-38

	vaddpd	zmm8, zmm9, zmm19		;; R3 + R7 (final R3)		R3 + R7 (final R3)		; 36-39
	vsubpd	zmm9, zmm9, zmm19		;; R3 - R7 (final R7)		R3 - R7 (final R7)		; 36-39

						;;				R9/I9 becomes newer R9/R13
	vaddpd	zmm4 {k7}, zmm13, zmm2		;; I1 + I5 (final I1)		R9				; 37-40
	vsubpd	zmm2 {k7}, zmm13, zmm2		;; I1 - I5 (final I5)		R13				; 37-40

						;;				R11/I11 becomes newer R11/R15
						;;				mul R11/I11 by SQRTHALF - i*SQRTHALF
	zfmaddpd zmm12, zmm6, zmm29, zmm11	;; I3*1 + I7 (final I3)		I11*SQRTHALF + R11 (final R11)	; 38-41
	zfmsubpd zmm6, zmm6, zmm29, zmm11	;; I3*1 - I7 (final I7)		I11*SQRTHALF - R11 (final I11)	; 38-41

						;;				R6/I6 becomes new R6/R8
	zfmaddpd zmm11, zmm16, zmm29, zmm7	;; R2 + R6*1 (final R2)		R2 + R6*SQRTHALF (final R2)	; 39-42
	zfnmaddpd zmm16, zmm16, zmm29, zmm7	;; R2 - R6*1 (final R6)		R2 - R6*SQRTHALF (final R6)	; 39-42
	zstore	[srcreg], zmm1			;; Save R1							; 39

	zfmaddpd zmm7, zmm3, zmm29, zmm25	;; R4 + R8*1 (final R4)		R4 + R8*SQRTHALF (final R4)	; 40-43
	zfnmaddpd zmm3, zmm3, zmm29, zmm25	;; R4 - R8*1 (final R8)		R4 - R8*SQRTHALF (final R8)	; 40-43
	zstore	[srcreg+d4], zmm5		;; Save R5							; 39+1

						;;				mul R10/I10 by w^1 = .924 - .383i
						;;				R10/I10 becomes newer R10/R14
	zfmaddpd zmm13, zmm10, zmm26, zmm17	;; I2 + I6*1 (final I2)		I10 + R10*.924/.383 (final R10)	; 41-44
	zfmsubpd zmm17, zmm17, zmm26, zmm10	;; I2*1 - I6 (final I6)		I10*.924/.383 - R10 (final R14)	; 41-44
	zstore	[srcreg+d2], zmm8		;; Save R3							; 40+1

						;;				mul R12/I12 by w^3 = .383 - .924i
						;;				R12/I12 becomes newer R12/R16
	zfmaddpd zmm10, zmm20, zmm26, zmm0	;; I4*1 + I8 (final I4)		I12*.924/.383 + R12 (final R12)	; 42-45
	zfnmaddpd zmm0, zmm0, zmm26, zmm20	;; I4 - I8*1 (final I8)		I12 - R12*.924/.383 (final R16)	; 42-45
	zstore	[srcreg+d4+d2], zmm9		;; Save R7							; 40+2

	zstore	[srcreg+64], zmm4		;; Save I1							; 41+2
	zstore	[srcreg+d4+64], zmm2		;; Save I5							; 41+3
	zstore	[srcreg+d2+64], zmm12		;; Save I3							; 42+3
	zstore	[srcreg+d4+d2+64], zmm6		;; Save I7							; 42+4
	zstore	[srcreg+d1], zmm11		;; Save R2							; 43+4
	zstore	[srcreg+d4+d1], zmm16		;; Save R6							; 43+5
	zstore	[srcreg+d2+d1], zmm7		;; Save R4							; 44+5
	zstore	[srcreg+d4+d2+d1], zmm3		;; Save R8							; 44+6
	zstore	[srcreg+d1+64], zmm13		;; Save I2							; 45+6
	zstore	[srcreg+d4+d1+64], zmm17	;; Save I6							; 45+7
	zstore	[srcreg+d2+d1+64], zmm10	;; Save I4							; 46+7
	zstore	[srcreg+d4+d2+d1+64], zmm0	;; Save I8							; 46+8
	bump	srcreg, srcinc
	ENDM


zr8_sixteen_reals_eight_complex_fft_final_preload MACRO
	;; preload would be silly for a macro that is executed only once
	ENDM
zr8_sixteen_reals_eight_complex_fft_final MACRO srcreg,srcinc,d1,d2,d4
	vbroadcastsd zmm29, ZMM_P383
	vbroadcastsd zmm30, ZMM_P924_P383
	vbroadcastsd zmm31, ZMM_SQRTHALF
	mov	eax, 11111110b			;; We're pretty sure callers of this macro call zloop_init next
	kmovw	k7, eax				;; Set k7 to 11111110b						; 30		n 31
	knotw	k6, k7				;; Set k6 to 00000001b						; 31		n 50
	vbroadcastsd zmm22, ZMM_ONE
	vblendmpd zmm26 {k6}, zmm22, zmm30	;; 1, 1, 1, 1, 1, 1, 1		.924/.383			; 50		n 68
	vblendmpd zmm27 {k6}, zmm22, zmm31	;; 1, 1, 1, 1, 1, 1, 1		SQRTHALF			; 51		n 87
	vmovapd zmm25 {k7}{z}, zmm22		;; 1, 1, 1, 1, 1, 1, 1		0				; 52		n 53
	vsubpd	zmm25 {k6}, zmm25, zmm31	;; 1, 1, 1, 1, 1, 1, 1		-SQRTHALF			; 53		n 86
	vblendmpd zmm28 {k6}, zmm31, zmm29	;; SQRTHALF, SQRTHALF, ...,	.383				; 54		n 96

						;; Simultaneously do an 8-complex and 16-reals
						;; 8-complex			16-reals
	vmovapd	zmm9, [srcreg]			;; R1				R1+R9 (new R1)
	vmovapd	zmm0, [srcreg+d4]		;; R5				R5+R13 (new R5)
	vaddpd	zmm2, zmm9, zmm0		;;		R1 + R5 (new R1)				; 42-45		n 56
	vsubpd	zmm9, zmm9, zmm0		;;		R1 - R5 (new R5)				; 42-45		n 56

	vmovapd	zmm19, [srcreg+d2]		;; R3				R3+R11 (new R3)
	vmovapd	zmm0, [srcreg+d4+d2]		;; R7				R7+R15 (new R7)
	vaddpd	zmm14, zmm19, zmm0		;;		R3 + R7 (new R3)				; 44-47		n 56
	vsubpd	zmm19, zmm19, zmm0		;;		R3 - R7 (new R7)				; 44-47		n 56

	vmovapd	zmm12, [srcreg+d1]		;; R2				R2+R10 (new R2)
	vmovapd	zmm0, [srcreg+d4+d1]		;; R6				R6+R14 (new R6)
	vaddpd	zmm3, zmm12, zmm0		;;		R2 + R6 (new R2)				; 46-49		n 60
	vsubpd	zmm12, zmm12, zmm0		;;		R2 - R6 (new R6)				; 46-49		n 60

	vmovapd	zmm11, [srcreg+d2+d1]		;; R4				R4+R12 (new R4)
	vmovapd	zmm0, [srcreg+d4+d2+d1]		;; R8				R8+R16 (new R8)
	vaddpd	zmm1, zmm11, zmm0		;;		R4 + R8 (new R4)				; 48-51		n 60
	vsubpd	zmm11, zmm11, zmm0		;;		R4 - R8 (new R8)				; 48-51		n 60

						;;				R12/R16 becomes newer R12/I12
						;;				mul R12/I12 by w^3 = .383 + .924i
	vmovapd	zmm10, [srcreg+d2+d1+64]	;; I4				R4-R12 (new R12)
	vmovapd	zmm15, [srcreg+d4+d2+d1+64]	;; I8				R8-R16 (new R16 a.k.a. I12)
	zfnmaddpd zmm6, zmm15, zmm26, zmm10	;; I4 - I8*1 (new I8)		R12/.383 = R12 - I12*.924/.383	; 68-71		n 76
	zfmaddpd zmm10, zmm10, zmm26, zmm15	;; I4*1 + I8 (new I4)		I12/.383 = R12*.924/.383 + I12	; 69-72		n 81

						;;				R11/R15 becomes newer R11/I11
						;;				mul R11/I11 by SQRTHALF + i*SQRTHALF
	vmovapd	zmm4, [srcreg+d2+64]		;; I3				R3-R11 (new R11)
	vmovapd	zmm5, [srcreg+d4+d2+64]		;; I7				R7-R15 (new R15 a.k.a. I11)
	vaddpd	zmm15, zmm4, zmm5		;; I3 + I7 (new I3)		I11/SQRTHALF = R11 + I11	; 70-73		n 84
	vsubpd	zmm4, zmm4, zmm5		;; I3 - I7 (new I7)		R11/SQRTHALF = R11 - I11	; 71-74		n 86

						;;				R10/R14 becomes newer R10/I10
						;;				mul R10/I10 by w^1 = .924 + .383i
	vmovapd	zmm0, [srcreg+d1+64]		;; I2				R2-R10 (new R10)
	vmovapd	zmm13, [srcreg+d4+d1+64]	;; I6				R6-R14 (new R14 a.k.a. I10)
	zfmsubpd zmm5, zmm0, zmm26, zmm13	;; I2*1 - I6 (new I6)		R10/.383 = R10*.924/.383 - I10	; 72-75		n 80
	zfmaddpd zmm0, zmm13, zmm26, zmm0	;; I2 + I6*1 (new I2)		I10/.383 = I10*.924/.383 + R10	; 73-76		n 81

						;;				R9/R13 becomes newer R9/I9
	vmovapd	zmm16, [srcreg+64]		;; I1				R1-R9 (new R9)
	vmovapd	zmm8, [srcreg+d4+64]		;; I5				R5-R13 (new R13 a.k.a. I9)
	vaddpd	zmm13 {k7}{z}, zmm16, zmm8	;; I1 + I5 (new I1)		0				; 74-77		n 85
	vsubpd	zmm8 {k7}, zmm16, zmm8		;; I1 - I5 (new I5)		I9				; 75-78		n 87

						;; next FFT level
						;;				R6/R8 becomes newest R6/I6
						;;				mul R6/I6 by w^2 = .707 + .707i
	vblendmpd zmm18 {k6}, zmm6, zmm11	;; I8				I6				; 76		n 78
	vblendmpd zmm11 {k6}, zmm11, zmm6	;; R8				R12				; 77		n 80

	vsubpd	zmm17, zmm12, zmm18		;; R6 - I8 (new2 R6)		R6 - I6 (newest R6/SQRTHALF)	; 78-12		n 88
	vaddpd	zmm12, zmm12, zmm18		;; R6 + I8 (new2 R8)		R6 + I6 (newest I6/SQRTHALF)	; 79-12		n 90

	vaddpd	zmm6, zmm5, zmm11		;; I6 + R8 (new2 I6)		R10 + R12 (newest R10/.383)	; 80-13		n 88
	vsubpd	zmm5, zmm5, zmm11		;; I6 - R8 (new2 I8)		R10 - R12 (newest R12/.383)	; 80-13		n 90

	vaddpd	zmm11, zmm0, zmm10		;; I2 + I4 (newer I2)		I10 + I12 (newest I10/.383)	; 81-14		n 88
	vsubpd	zmm0, zmm0, zmm10		;; I2 - I4 (newer I4)		I10 - I12 (newest I12/.383)	; 81-14		n 89

	vaddpd	zmm10, zmm2, zmm14		;; R1 + R3 (newer R1)		R1 + R3 (newest R1)		; 82-15		n 89
	vsubpd	zmm2, zmm2, zmm14		;; R1 - R3 (newer R3)		R1 - R3 (newest R3)		; 82-15		n 91

	vaddpd	zmm14, zmm3, zmm1		;; R2 + R4 (newer R2)		R2 + R4 (newest R2)		; 83-86		n 89
	vsubpd	zmm3, zmm3, zmm1		;; R2 - R4 (newer R4)		R2 - R4 (newest R4)		; 83-16		n 91

	vblendmpd zmm18 {k6}, zmm19, zmm15	;; R7				I11/SQRTHALF			; 84		n 87
	vblendmpd zmm16 {k6}, zmm9, zmm16	;; R5				R9				; 84		n 86

						;;				R5/R7 becomes newest R5/I5
	vsubpd	zmm19 {k7}, zmm13, zmm15	;; I1 - I3 (newer I3)		I5				; 85-17		n 95
	vaddpd	zmm13 {k7}{z}, zmm13, zmm15	;; I1 + I3 (newer I1)		0				; 85-17		n 93

	zfnmaddpd zmm7, zmm4, zmm25, zmm16	;; R5 - I7*1 (newer R5)		R9 - R11*-SQRTHALF (newest R9)	; 86-18		n 96
	zfmaddpd zmm4, zmm4, zmm25, zmm16	;; R5 + I7*1 (newer R7)		R9 + R11*-SQRTHALF (newest R11)	; 86-18		n 98

	zfmaddpd zmm16, zmm18, zmm27, zmm8	;; I5 + R7*1 (newer I5)		I9 + I11*SQRTHALF (newest I9)	; 87-19		n 97
	zfnmaddpd zmm18, zmm18, zmm27, zmm8	;; I5 - R7*1 (newer I7)		I9 - I11*SQRTHALF (newest I11)	; 87-19		n 99

	vmovapd	zmm8, zmm11			;;				I10/.383
	vaddpd	zmm8 {k7}, zmm17, zmm6		;; I6 = R6 + I6 (newer I6/SQRTHALF)	I10/.383		; 88-91		n 97
	vsubpd	zmm6 {k7}, zmm17, zmm6		;; R6 = R6 - I6 (newer R6/SQRTHALF)	R10/.383		; 88-91		n 96

	vblendmpd zmm17 {k6}, zmm0, zmm17	;; I4				R6/SQRTHALF			; 89		n 94
	vsubpd	zmm11 {k6}, zmm10, zmm14	;; I2				R1 - R2 (final R1b)		; 89-92		n 93

	vaddpd	zmm0 {k7}, zmm12, zmm5		;; I8 = R8 + I8 (newer I8/SQRTHALF)	I12/.383		; 90-92		n 98
	vsubpd	zmm5 {k7}, zmm12, zmm5		;; R8 = R8 - I8 (newer R8/SQRTHALF)	R12/.383		; 90-92		n 99

	;; last FFT level

	vblendmpd zmm9 {k6}, zmm2, zmm9		;; R3				R5				; 91		n 94
	vblendmpd zmm12 {k6}, zmm3, zmm12	;; R4				I6/SQRTHALF			; 91		n 95

						;;				R3/R4 becomes final R3/I3
	vaddpd	zmm1, zmm10, zmm14		;; R1 + R2 (final R1)		R1 + R2 (final R1a => R1)	; 92-95		n 
	vsubpd	zmm2 {k7}, zmm10, zmm14		;; R1 - R2 (final R2)		R3 => R2			; 92-95		n 

	vsubpd	zmm3 {k7}, zmm13, zmm11		;; I1 - I2 (final I2)		I3 => I2			; 93-96		n 
	vaddpd	zmm11 {k7}, zmm13, zmm11	;; I1 + I2 (final I1)		R1b => I1			; 93-96		n 

	zfnmaddpd zmm13, zmm17, zmm25, zmm9	;; R3 - I4*1 (final R3)		R5 - R6*-SQRTHALF (final R5 => R3) ; 94-97	n 
	zfmaddpd zmm9, zmm17, zmm25, zmm9	;; R3 + I4*1 (final R4)		R5 + R6*-SQRTHALF (final R6 => R4) ; 94-97	n 

	zfmaddpd zmm15, zmm12, zmm27, zmm19	;; I3 + R4*1 (final I3)		I5 + I6*SQRTHALF (final I5 => I3) ; 95-98	n 
	zfnmaddpd zmm12, zmm12, zmm27, zmm19	;; I3 - R4*1 (final I4)		I5 - I6*SQRTHALF (final I6 => I4) ; 95-98	n 

	zfmaddpd zmm10, zmm6, zmm28, zmm7	;; R5 + R6*SQRTHALF (final R5)	R9 + R10*.383 (final R9 => R5)	; 96-99		n 
	zfnmaddpd zmm6, zmm6, zmm28, zmm7	;; R5 - R6*SQRTHALF (final R6)	R9 - R10*.383 (final R10 => R6)	; 96-99		n 

	zfmaddpd zmm7, zmm8, zmm28, zmm16	;; I5 + I6*SQRTHALF (final I5)	I9 + I10*.383 (final I9 => I5)	; 97-100	n 
	zfnmaddpd zmm8, zmm8, zmm28, zmm16	;; I5 - I6*SQRTHALF (final I6)	I9 - I10*.383 (final I10 => I6)	; 97-100	n 

	zfnmaddpd zmm14, zmm0, zmm28, zmm4	;; R7 - I8*SQRTHALF (final R7)	R11 - I12*.383 (final R11 => R7); 98-101	n 
	zfmaddpd zmm0, zmm0, zmm28, zmm4	;; R7 + I8*SQRTHALF (final R8)	R11 + I12*.383 (final R12 => R8); 98-101	n 

	zfmaddpd zmm4, zmm5, zmm28, zmm18	;; I7 + R8*SQRTHALF (final I7)	I11 + R12*.383 (final I11 => I7); 99-102	n 
	zfnmaddpd zmm5, zmm5, zmm28, zmm18	;; I7 - R8*SQRTHALF (final I8)	I11 - R12*.383 (final I12 => I8); 99-102	n 

	zstore	[srcreg], zmm1			;; Save R1
	zstore	[srcreg+d1], zmm2		;; Save R2
	zstore	[srcreg+d1+64], zmm3		;; Save I2
	zstore	[srcreg+64], zmm11		;; Save I1
	zstore	[srcreg+d2], zmm13		;; Save R3
	zstore	[srcreg+d2+d1], zmm9		;; Save R4
	zstore	[srcreg+d2+64], zmm15		;; Save I3
	zstore	[srcreg+d2+d1+64], zmm12	;; Save I4
	zstore	[srcreg+d4], zmm10		;; Save R5
	zstore	[srcreg+d4+d1], zmm6		;; Save R6
	zstore	[srcreg+d4+64], zmm7		;; Save I5
	zstore	[srcreg+d4+d1+64], zmm8		;; Save I6
	zstore	[srcreg+d4+d2], zmm14		;; Save R7
	zstore	[srcreg+d4+d2+d1], zmm0		;; Save R8
	zstore	[srcreg+d4+d2+64], zmm4		;; Save I7
	zstore	[srcreg+d4+d2+d1+64], zmm5	;; Save I8
	bump	srcreg, srcinc
	ENDM


zr8_sixteen_reals_eight_complex_with_square_preload MACRO
	;; preload would be silly for a macro that is executed only once
	ENDM
zr8_sixteen_reals_eight_complex_with_square MACRO srcreg,srcinc,d1,d2,d4
	LOCAL	nosave, orig, back_to_orig, muladd, muladdhard, mulsub, mulsubhard, fma_done

;GW these consts may need study/work
	vbroadcastsd zmm29, ZMM_P383
	vbroadcastsd zmm30, ZMM_P924_P383
	vbroadcastsd zmm31, ZMM_SQRTHALF
	mov	eax, 11111110b			;; We're pretty sure callers of this macro call zloop_init next
	kmovw	k7, eax				;; Set k7 to 11111110b						; 30		n 31
	knotw	k6, k7				;; Set k6 to 00000001b						; 31		n 50
	vbroadcastsd zmm22, ZMM_ONE
	vblendmpd zmm26 {k6}, zmm22, zmm30	;; 1, 1, 1, 1, 1, 1, 1		.924/.383			; 50		n 68
	vblendmpd zmm27 {k6}, zmm22, zmm31	;; 1, 1, 1, 1, 1, 1, 1		SQRTHALF			; 51		n 87
	vmovapd zmm25 {k7}{z}, zmm22		;; 1, 1, 1, 1, 1, 1, 1		0				; 52		n 53
	vsubpd	zmm25 {k6}, zmm25, zmm31	;; 1, 1, 1, 1, 1, 1, 1		-SQRTHALF			; 53		n 86
	vblendmpd zmm28 {k6}, zmm31, zmm29	;; SQRTHALF, SQRTHALF, ...,	.383				; 54		n 96


						;; Simultaneously do an 8-complex and 16-reals
						;; 8-complex			16-reals
	vmovapd	zmm9, [srcreg]			;; R1				R1+R9 (new R1)
	vmovapd	zmm0, [srcreg+d4]		;; R5				R5+R13 (new R5)
	vaddpd	zmm2, zmm9, zmm0		;;		R1 + R5 (new R1)				; 42-45		n 56
	vsubpd	zmm9, zmm9, zmm0		;;		R1 - R5 (new R5)				; 42-45		n 56

	vmovapd	zmm19, [srcreg+d2]		;; R3				R3+R11 (new R3)
	vmovapd	zmm0, [srcreg+d4+d2]		;; R7				R7+R15 (new R7)
	vaddpd	zmm14, zmm19, zmm0		;;		R3 + R7 (new R3)				; 44-47		n 56
	vsubpd	zmm19, zmm19, zmm0		;;		R3 - R7 (new R7)				; 44-47		n 56

	vmovapd	zmm12, [srcreg+d1]		;; R2				R2+R10 (new R2)
	vmovapd	zmm0, [srcreg+d4+d1]		;; R6				R6+R14 (new R6)
	vaddpd	zmm3, zmm12, zmm0		;;		R2 + R6 (new R2)				; 46-49		n 60
	vsubpd	zmm12, zmm12, zmm0		;;		R2 - R6 (new R6)				; 46-49		n 60

	vmovapd	zmm11, [srcreg+d2+d1]		;; R4				R4+R12 (new R4)
	vmovapd	zmm0, [srcreg+d4+d2+d1]		;; R8				R8+R16 (new R8)
	vaddpd	zmm1, zmm11, zmm0		;;		R4 + R8 (new R4)				; 48-51		n 60
	vsubpd	zmm11, zmm11, zmm0		;;		R4 - R8 (new R8)				; 48-51		n 60

						;;				R12/R16 becomes newer R12/I12
						;;				mul R12/I12 by w^3 = .383 + .924i
	vmovapd	zmm10, [srcreg+d2+d1+64]	;; I4				R4-R12 (new R12)
	vmovapd	zmm15, [srcreg+d4+d2+d1+64]	;; I8				R8-R16 (new R16 a.k.a. I12)
	zfnmaddpd zmm6, zmm15, zmm26, zmm10	;; I4 - I8*1 (new I8)		R12/.383 = R12 - I12*.924/.383	; 68-71		n 76
	zfmaddpd zmm10, zmm10, zmm26, zmm15	;; I4*1 + I8 (new I4)		I12/.383 = R12*.924/.383 + I12	; 69-72		n 81

						;;				R11/R15 becomes newer R11/I11
						;;				mul R11/I11 by SQRTHALF + i*SQRTHALF
	vmovapd	zmm4, [srcreg+d2+64]		;; I3				R3-R11 (new R11)
	vmovapd	zmm5, [srcreg+d4+d2+64]		;; I7				R7-R15 (new R15 a.k.a. I11)
	vaddpd	zmm15, zmm4, zmm5		;; I3 + I7 (new I3)		I11/SQRTHALF = R11 + I11	; 70-73		n 84
	vsubpd	zmm4, zmm4, zmm5		;; I3 - I7 (new I7)		R11/SQRTHALF = R11 - I11	; 71-74		n 86

						;;				R10/R14 becomes newer R10/I10
						;;				mul R10/I10 by w^1 = .924 + .383i
	vmovapd	zmm0, [srcreg+d1+64]		;; I2				R2-R10 (new R10)
	vmovapd	zmm13, [srcreg+d4+d1+64]	;; I6				R6-R14 (new R14 a.k.a. I10)
	zfmsubpd zmm5, zmm0, zmm26, zmm13	;; I2*1 - I6 (new I6)		R10/.383 = R10*.924/.383 - I10	; 72-75		n 80
	zfmaddpd zmm0, zmm13, zmm26, zmm0	;; I2 + I6*1 (new I2)		I10/.383 = I10*.924/.383 + R10	; 73-76		n 81

						;;				R9/R13 becomes newer R9/I9
	vmovapd	zmm16, [srcreg+64]		;; I1				R1-R9 (new R9)
	vmovapd	zmm8, [srcreg+d4+64]		;; I5				R5-R13 (new R13 a.k.a. I9)
	vaddpd	zmm13 {k7}{z}, zmm16, zmm8	;; I1 + I5 (new I1)		0				; 74-77		n 85
	vsubpd	zmm8 {k7}, zmm16, zmm8		;; I1 - I5 (new I5)		I9				; 75-78		n 87

						;; next FFT level
						;;				R6/R8 becomes newest R6/I6
						;;				mul R6/I6 by w^2 = .707 + .707i
	vblendmpd zmm18 {k6}, zmm6, zmm11	;; I8				I6				; 76		n 78
	vblendmpd zmm11 {k6}, zmm11, zmm6	;; R8				R12				; 77		n 80

	vsubpd	zmm17, zmm12, zmm18		;; R6 - I8 (new2 R6)		R6 - I6 (newest R6/SQRTHALF)	; 78-12		n 88
	vaddpd	zmm12, zmm12, zmm18		;; R6 + I8 (new2 R8)		R6 + I6 (newest I6/SQRTHALF)	; 79-12		n 90

	vaddpd	zmm6, zmm5, zmm11		;; I6 + R8 (new2 I6)		R10 + R12 (newest R10/.383)	; 80-13		n 88
	vsubpd	zmm5, zmm5, zmm11		;; I6 - R8 (new2 I8)		R10 - R12 (newest R12/.383)	; 80-13		n 90

	vaddpd	zmm11, zmm0, zmm10		;; I2 + I4 (newer I2)		I10 + I12 (newest I10/.383)	; 81-14		n 88
	vsubpd	zmm0, zmm0, zmm10		;; I2 - I4 (newer I4)		I10 - I12 (newest I12/.383)	; 81-14		n 89

	vaddpd	zmm10, zmm2, zmm14		;; R1 + R3 (newer R1)		R1 + R3 (newest R1)		; 82-15		n 89
	vsubpd	zmm2, zmm2, zmm14		;; R1 - R3 (newer R3)		R1 - R3 (newest R3)		; 82-15		n 91

	vaddpd	zmm14, zmm3, zmm1		;; R2 + R4 (newer R2)		R2 + R4 (newest R2)		; 83-86		n 89
	vsubpd	zmm3, zmm3, zmm1		;; R2 - R4 (newer R4)		R2 - R4 (newest R4)		; 83-16		n 91

	vblendmpd zmm18 {k6}, zmm19, zmm15	;; R7				I11/SQRTHALF			; 84		n 87
	vblendmpd zmm16 {k6}, zmm9, zmm16	;; R5				R9				; 84		n 86

						;;				R5/R7 becomes newest R5/I5
	vsubpd	zmm19 {k7}, zmm13, zmm15	;; I1 - I3 (newer I3)		I5				; 85-17		n 95
	vaddpd	zmm13 {k7}{z}, zmm13, zmm15	;; I1 + I3 (newer I1)		0				; 85-17		n 93

	zfnmaddpd zmm7, zmm4, zmm25, zmm16	;; R5 - I7*1 (newer R5)		R9 - R11*-SQRTHALF (newest R9)	; 86-18		n 96
	zfmaddpd zmm4, zmm4, zmm25, zmm16	;; R5 + I7*1 (newer R7)		R9 + R11*-SQRTHALF (newest R11)	; 86-18		n 98

	zfmaddpd zmm16, zmm18, zmm27, zmm8	;; I5 + R7*1 (newer I5)		I9 + I11*SQRTHALF (newest I9)	; 87-19		n 97
	zfnmaddpd zmm18, zmm18, zmm27, zmm8	;; I5 - R7*1 (newer I7)		I9 - I11*SQRTHALF (newest I11)	; 87-19		n 99

	vmovapd	zmm8, zmm11			;;				I10/.383
	vaddpd	zmm8 {k7}, zmm17, zmm6		;; I6 = R6 + I6 (newer I6/SQRTHALF)	I10/.383		; 88-91		n 97
	vsubpd	zmm6 {k7}, zmm17, zmm6		;; R6 = R6 - I6 (newer R6/SQRTHALF)	R10/.383		; 88-91		n 96

	vblendmpd zmm17 {k6}, zmm0, zmm17	;; I4				R6/SQRTHALF			; 89		n 94
	vsubpd	zmm11 {k6}, zmm10, zmm14	;; I2				R1 - R2 (final R1b)		; 89-92		n 93

	vaddpd	zmm0 {k7}, zmm12, zmm5		;; I8 = R8 + I8 (newer I8/SQRTHALF)	I12/.383		; 90-92		n 98
	vsubpd	zmm5 {k7}, zmm12, zmm5		;; R8 = R8 - I8 (newer R8/SQRTHALF)	R12/.383		; 90-92		n 99

	;; last FFT level

	vblendmpd zmm9 {k6}, zmm2, zmm9		;; R3				R5				; 91		n 94
	vblendmpd zmm12 {k6}, zmm3, zmm12	;; R4				I6/SQRTHALF			; 91		n 95

						;;				R3/R4 becomes final R3/I3
	vaddpd	zmm1, zmm10, zmm14		;; R1 + R2 (final R1)		R1 + R2 (final R1a => R1)	; 92-95		n 
	vsubpd	zmm2 {k7}, zmm10, zmm14		;; R1 - R2 (final R2)		R3 => R2			; 92-95		n 

	vsubpd	zmm3 {k7}, zmm13, zmm11		;; I1 - I2 (final I2)		I3 => I2			; 93-96		n 
	vaddpd	zmm11 {k7}, zmm13, zmm11	;; I1 + I2 (final I1)		R1b => I1			; 93-96		n 

	zfnmaddpd zmm13, zmm17, zmm25, zmm9	;; R3 - I4*1 (final R3)		R5 - R6*-SQRTHALF (final R5 => R3) ; 94-97	n 
	zfmaddpd zmm9, zmm17, zmm25, zmm9	;; R3 + I4*1 (final R4)		R5 + R6*-SQRTHALF (final R6 => R4) ; 94-97	n 

	zfmaddpd zmm15, zmm12, zmm27, zmm19	;; I3 + R4*1 (final I3)		I5 + I6*SQRTHALF (final I5 => I3) ; 95-98	n 
	zfnmaddpd zmm12, zmm12, zmm27, zmm19	;; I3 - R4*1 (final I4)		I5 - I6*SQRTHALF (final I6 => I4) ; 95-98	n 
	vbroadcastsd zmm27, ZMM_TWO

	zfmaddpd zmm10, zmm6, zmm28, zmm7	;; R5 + R6*SQRTHALF (final R5)	R9 + R10*.383 (final R9 => R5)	; 96-99		n 
	zfnmaddpd zmm6, zmm6, zmm28, zmm7	;; R5 - R6*SQRTHALF (final R6)	R9 - R10*.383 (final R10 => R6)	; 96-99		n 

	zfmaddpd zmm7, zmm8, zmm28, zmm16	;; I5 + I6*SQRTHALF (final I5)	I9 + I10*.383 (final I9 => I5)	; 97-100	n 
	zfnmaddpd zmm8, zmm8, zmm28, zmm16	;; I5 - I6*SQRTHALF (final I6)	I9 - I10*.383 (final I10 => I6)	; 97-100	n 

	zfnmaddpd zmm14, zmm0, zmm28, zmm4	;; R7 - I8*SQRTHALF (final R7)	R11 - I12*.383 (final R11 => R7); 98-101	n 
	zfmaddpd zmm0, zmm0, zmm28, zmm4	;; R7 + I8*SQRTHALF (final R8)	R11 + I12*.383 (final R12 => R8); 98-101	n 

	zfmaddpd zmm4, zmm5, zmm28, zmm18	;; I7 + R8*SQRTHALF (final I7)	I11 + R12*.383 (final I11 => I7); 99-102	n 
	zfnmaddpd zmm5, zmm5, zmm28, zmm18	;; I7 - R8*SQRTHALF (final I8)	I11 - R12*.383 (final I12 => I8); 99-102	n 

	mov	al, mul4_opcode			;; Load the mul4_opcode
	cmp	al, 0				;; See if we need to do more than the original type-2 FFT multiply
	je	orig				;; Jump if nothing special
	mov	r9, SRC2ARG			;; Distance to s3 arg in gwmuladd4 or gwmulsub4
	mov	r10, SRC3ARG			;; Load distance to FFT(1)
	jg	nosave				;; Jump if not saving result of the forward FFT?

	;; Store FFT results
	zstore	[srcreg], zmm1			;; R1			R1a
	zstore	[srcreg+64], zmm11		;; I1			R1b
	zstore	[srcreg+d1], zmm2		;;		R2
	zstore	[srcreg+d1+64], zmm3		;;		I2
	zstore	[srcreg+d2], zmm13		;;		R3
	zstore	[srcreg+d2+64], zmm15		;;		I3
	zstore	[srcreg+d2+d1], zmm9		;;		R4
	zstore	[srcreg+d2+d1+64], zmm12	;;		I4
	zstore	[srcreg+d4], zmm10		;;		R5
	zstore	[srcreg+d4+64], zmm7		;;		I5
	zstore	[srcreg+d4+d1], zmm6		;;		R6
	zstore	[srcreg+d4+d1+64], zmm8		;;		I6
	zstore	[srcreg+d4+d2], zmm14		;;		R7
	zstore	[srcreg+d4+d2+64], zmm4		;;		I7
	zstore	[srcreg+d4+d2+d1], zmm0		;;		R8
	zstore	[srcreg+d4+d2+d1+64], zmm5	;;		I8
	and	al, 7Fh				;; Strip "save bit" from mul4_opcode
	jz	orig				;; 0=no opcode, goto original squaring code

nosave:
	;; Square the results
	vmulpd	zmm23, zmm1, zmm1		;; A1 = R1 * R1			R1a*R1a (R1a)			; 1-4		n 5
	vmulpd	zmm17, zmm2, zmm2		;;		A2 = R2 * R2					; 1-4		n 5

	vmulpd	zmm19, zmm13, zmm13		;;		A3 = R3 * R3					; 2-5		n 8
	vmulpd	zmm21, zmm9, zmm9		;;		A4 = R4 * R4					; 2-5		n 8

	vmulpd	zmm22, zmm10, zmm10		;;		A5 = R5 * R5					; 3-6		n 9
	vmulpd	zmm18, zmm6, zmm6		;;		A6 = R6 * R6					; 3-6		n 10

	vmulpd	zmm16, zmm14, zmm14		;;		A7 = R7 * R7					; 4-7		n 12
	vmulpd	zmm20, zmm0, zmm0		;;		A8 = R8 * R8					; 4-7		n 12

	zfnmaddpd zmm23 {k7}, zmm11, zmm11, zmm23;; A1 - I1 * I1 (R1)		R1a				; 5-8		n 13
	zfnmaddpd zmm17, zmm3, zmm3, zmm17	;;		A2 - I2 * I2 (R2)				; 5-8		n 9
	vmovsd	Q [srcreg+r8-16], xmm23		;;				Save square of FFT sums		; 5

	vmulpd	zmm9, zmm12, zmm9		;;		B4 = I4 * R4 (I4/2)				; 6-9		n 10
	vmulpd	zmm0, zmm5, zmm0		;;		B8 = I8 * R8 (I8/2)				; 6-9		n 13

	vmulpd	zmm2, zmm3, zmm2		;;		I2 * R2 (I2/2)					; 7-10		n 14
	vmulpd	zmm13, zmm15, zmm13		;;		I3 * R3 (I3/2 & I5/2)				; 7-10		n 16

	zfnmaddpd zmm19, zmm15, zmm15, zmm19	;;		A3 - I3 * I3 (R3 & R5)				; 8-11		n 
	zfnmaddpd zmm21, zmm12, zmm12, zmm21	;;		A4 - I4 * I4 (R4 & R6)				; 8-11		n 

	vmulpd	zmm11 {k7}, zmm11, zmm1		;; I1*R1 (I1/2)			 R1b

	zfnmaddpd zmm22, zmm7, zmm7, zmm22	;;		A5 - I5 * I5 (R5 & R9)				; 9-12		n 
	zfnmaddpd zmm18, zmm8, zmm8, zmm18	;;		A6 - I6 * I6 (R6 & R10)				; 10-13		n 

	vmulpd	zmm7, zmm7, zmm10		;;		I5 * R5 (I5/2 & I9/2)
	vmulpd	zmm6, zmm8, zmm6 		;;		I6 * R6 (I6/2 & I10/2)				; 11-12		n 
	vmulpd	zmm14, zmm4, zmm14		;;		I7 * R7 (I7/2 & I11/2)				; 11-15		n 

	zfnmaddpd zmm16, zmm4, zmm4, zmm16	;;		A7 - I7 * I7 (R7 & R11)				; 12-18		n 
	zfnmaddpd zmm20, zmm5, zmm5, zmm20	;;		A8 - I8 * I8 (R8 & R12)				; 12-20		n 

	vbroadcastsd zmm10, ZMM_HALF
	vbroadcastsd zmm1, ZMM_TWO

	cmp	al, 4				;; Case off opcode
	je	mulsub				;; 4=mulsub
	;jb	muladd				;; 3=muladd, fall through

muladd:	test	r10, 3				;; Check if we need to multiply add-in by FFT(1)
	jz	muladdhard			;; Yes, do it the hard way

	vaddpd	zmm17, zmm17, [srcreg+r9+d1]			;; R2 = R2 + MemR2
	zfmaddpd zmm2, zmm10, [srcreg+r9+d1+64], zmm2		;; I2/2 = I2/2 + MemI2/2

	vaddpd	zmm19, zmm19, [srcreg+r9+d2]			;; R3 = R3 + MemR3
	zfmaddpd zmm13, zmm10, [srcreg+r9+d2+64], zmm13		;; I3/2 = I3/2 + MemI3/2

	vaddpd	zmm21, zmm21, [srcreg+r9+d2+d1]			;; R4 = R4 + MemR4
	zfmaddpd zmm9, zmm9, zmm1, [srcreg+r9+d2+d1+64]		;; I4 = B4 * 2 + MemI4

	vmovapd	zmm12, zmm17					;;		Copy R2
	vmulpd	zmm12 {k6}, zmm11, zmm11			;; R2				R1b * R1b (R1b)

	vaddpd	zmm23, zmm23, [srcreg+r9]			;; R1 = R1 + MemR1
	zfmaddpd zmm11, zmm10, [srcreg+r9+64], zmm11		;; I1/2 = I1/2 + MemI1/2	undefined
	vaddpd zmm12 {k6}, zmm12, [srcreg+r9+64]		;; R2				R1b = R1b + MemI1

	vaddpd	zmm22, zmm22, [srcreg+r9+d4]			;; R5 = R5 + MemR5
	zfmaddpd zmm7, zmm10, [srcreg+r9+d4+64], zmm7		;; I5/2 = I5/2 + MemI5/2

	vaddpd	zmm18, zmm18, [srcreg+r9+d4+d1]			;; R6 = R6 + MemR6
	zfmaddpd zmm6, zmm10, [srcreg+r9+d4+d1+64], zmm6	;; I6/2 = I6/2 + MemI6/2

	vaddpd	zmm16, zmm16, [srcreg+r9+d4+d2]			;; R7 = R7 + MemR7
	zfmaddpd zmm14, zmm10, [srcreg+r9+d4+d2+64], zmm14	;; I7/2 = I7/2 + MemI7/2

	vaddpd	zmm20, zmm20, [srcreg+r9+d4+d2+d1]		;; R8 = R8 + MemR8
	zfmaddpd zmm0, zmm0, zmm1, [srcreg+r9+d4+d2+d1+64]	;; I8 = B8 * 2 + MemI8
	jmp	fma_done

muladdhard:
	vmovapd	zmm3, [srcreg+r9+d1]				;; MemR2
	vmovapd	zmm4, [srcreg+r10+d1]				;; MemR2#2
	zfmaddpd zmm17, zmm3, zmm4, zmm17			;; R2 = R2 + MemR2*MemR2#2
	vmovapd	zmm5, [srcreg+r10+d1+64]			;; MemI2#2
	vmulpd	zmm3, zmm3, zmm5				;; TMP = MemR2*MemI2#2
	vmovapd	zmm8, [srcreg+r9+d1+64]				;; MemI2
	zfnmaddpd zmm17, zmm8, zmm5, zmm17			;; R2 = R2 - MemI2*MemI2#2
	zfmaddpd zmm3, zmm8, zmm4, zmm3				;; TMP += MemI2*MemR2#2
	zfmaddpd zmm2, zmm10, zmm3, zmm2			;; I2/2 = I2/2 + TMP/2

	vmovapd	zmm3, [srcreg+r9+d2]				;; MemR3
	vmovapd	zmm4, [srcreg+r10+d2]				;; MemR3#2
	zfmaddpd zmm19, zmm3, zmm4, zmm19			;; R3 = R3 + MemR3*MemR3#2
	vmovapd	zmm5, [srcreg+r10+d2+64]			;; MemI3#2
	vmulpd	zmm3, zmm3, zmm5				;; TMP = MemR3*MemI3#2
	vmovapd	zmm8, [srcreg+r9+d2+64]				;; MemI3
	zfnmaddpd zmm19, zmm8, zmm5, zmm19			;; R3 = R3 - MemI3*MemI3#2
	zfmaddpd zmm3, zmm8, zmm4, zmm3				;; TMP += MemI3*MemR3#2
	zfmaddpd zmm13, zmm10, zmm3, zmm13			;; I3/2 = I3/2 + TMP/2

	vmovapd	zmm3, [srcreg+r9+d2+d1]				;; MemR4
	vmovapd	zmm4, [srcreg+r10+d2+d1]			;; MemR4#2
	zfmaddpd zmm21, zmm3, zmm4, zmm21			;; R4 = R4 + MemR4*MemR4#2
	vmovapd	zmm5, [srcreg+r10+d2+d1+64]			;; MemI4#2
	vmulpd	zmm3, zmm3, zmm5				;; TMP = MemR4*MemI4#2
	vmovapd	zmm8, [srcreg+r9+d2+d1+64]			;; MemI4
	zfnmaddpd zmm21, zmm8, zmm5, zmm21			;; R4 = R4 - MemI4*MemI4#2
	zfmaddpd zmm3, zmm8, zmm4, zmm3				;; TMP += MemI4*MemR4#2
	zfmaddpd zmm9, zmm9, zmm1, zmm3				;; I4 = B4 * 2 + TMP

	vmovapd	zmm12, zmm17					;;		Copy R2
	vmulpd	zmm12 {k6}, zmm11, zmm11			;; R2				R1b * R1b (R1b)

	vmovapd	zmm3, [srcreg+r9]				;; MemR1
	vmovapd	zmm4, [srcreg+r10]				;; MemR1#2
	zfmaddpd zmm23, zmm3, zmm4, zmm23			;; R1 = R1 + MemR1*MemR1#2
	vmovapd	zmm5, [srcreg+r10+64]				;; MemI1#2
	vmulpd	zmm3, zmm3, zmm5				;; TMP = MemR1*MemI1#2
	vmovapd	zmm8, [srcreg+r9+64]				;; MemI1
	zfnmaddpd zmm23{k7}, zmm8, zmm5, zmm23			;; R1 = R1 - MemI1*MemI1#2	R1
	zfmaddpd zmm12{k6}, zmm8, zmm5, zmm12			;; R2				R1b = R1b + MemI1*MemI1#2
	zfmaddpd zmm3, zmm8, zmm4, zmm3				;; TMP += MemI1*MemR1#2
	zfmaddpd zmm11, zmm10, zmm3, zmm11			;; I1/2 = I1/2 + TMP/2		undefined

	vmovapd	zmm3, [srcreg+r9+d4]				;; MemR5
	vmovapd	zmm4, [srcreg+r10+d4]				;; MemR5#2
	zfmaddpd zmm22, zmm3, zmm4, zmm22			;; R5 = R5 + MemR5*MemR5#2
	vmovapd	zmm5, [srcreg+r10+d4+64]			;; MemI5#2
	vmulpd	zmm3, zmm3, zmm5				;; TMP = MemR5*MemI5#2
	vmovapd	zmm8, [srcreg+r9+d4+64]				;; MemI5
	zfnmaddpd zmm22, zmm8, zmm5, zmm22			;; R5 = R5 - MemI5*MemI5#2
	zfmaddpd zmm3, zmm8, zmm4, zmm3				;; TMP += MemI5*MemR5#2
	zfmaddpd zmm7, zmm10, zmm3, zmm7			;; I5/2 = I5/2 + TMP/2

	vmovapd	zmm3, [srcreg+r9+d4+d1]				;; MemR6
	vmovapd	zmm4, [srcreg+r10+d4+d1]			;; MemR6#2
	zfmaddpd zmm18, zmm3, zmm4, zmm18			;; R6 = R6 + MemR6*MemR6#2
	vmovapd	zmm5, [srcreg+r10+d4+d1+64]			;; MemI6#2
	vmulpd	zmm3, zmm3, zmm5				;; TMP = MemR6*MemI6#2
	vmovapd	zmm8, [srcreg+r9+d4+d1+64]			;; MemI6
	zfnmaddpd zmm18, zmm8, zmm5, zmm18			;; R6 = R6 - MemI6*MemI6#2
	zfmaddpd zmm3, zmm8, zmm4, zmm3				;; TMP += MemI6*MemR6#2
	zfmaddpd zmm6, zmm10, zmm3, zmm6			;; I6/2 = I6/2 + TMP/2

	vmovapd	zmm3, [srcreg+r9+d4+d2]				;; MemR7
	vmovapd	zmm4, [srcreg+r10+d4+d2]			;; MemR7#2
	zfmaddpd zmm16, zmm3, zmm4, zmm16			;; R7 = R7 + MemR7*MemR7#2
	vmovapd	zmm5, [srcreg+r10+d4+d2+64]			;; MemI7#2
	vmulpd	zmm3, zmm3, zmm5				;; TMP = MemR7*MemI7#2
	vmovapd	zmm8, [srcreg+r9+d4+d2+64]			;; MemI7
	zfnmaddpd zmm16, zmm8, zmm5, zmm16			;; R7 = R7 - MemI7*MemI7#2
	zfmaddpd zmm3, zmm8, zmm4, zmm3				;; TMP += MemI7*MemR7#2
	zfmaddpd zmm14, zmm10, zmm3, zmm14			;; I7/2 = I7/2 + TMP/2

	vmovapd	zmm3, [srcreg+r9+d4+d2+d1]			;; MemR8
	vmovapd	zmm4, [srcreg+r10+d4+d2+d1]			;; MemR8#2
	zfmaddpd zmm20, zmm3, zmm4, zmm20			;; R8 = R8 + MemR8*MemR8#2
	vmovapd	zmm5, [srcreg+r10+d4+d2+d1+64]			;; MemI8#2
	vmulpd	zmm3, zmm3, zmm5				;; TMP = MemR8*MemI8#2
	vmovapd	zmm8, [srcreg+r9+d4+d2+d1+64]			;; MemI8
	zfnmaddpd zmm20, zmm8, zmm5, zmm20			;; R8 = R8 - MemI8*MemI8#2
	zfmaddpd zmm3, zmm8, zmm4, zmm3				;; TMP += MemI8*MemR8#2
	zfmaddpd zmm0, zmm0, zmm1, zmm3				;; I8 = B8 * 2 + TMP
	jmp	fma_done

mulsub:	test	r10, 3				;; Check if we need to multiply add-in by FFT(1)
	jz	mulsubhard			;; Yes, do it the hard way

	vsubpd	zmm17, zmm17, [srcreg+r9+d1]			;; R2 = R2 - MemR2
	zfnmaddpd zmm2, zmm10, [srcreg+r9+d1+64], zmm2		;; I2/2 = I2/2 - MemI2/2

	vsubpd	zmm19, zmm19, [srcreg+r9+d2]			;; R3 = R3 - MemR3
	zfnmaddpd zmm13, zmm10, [srcreg+r9+d2+64], zmm13	;; I3/2 = I3/2 - MemI3/2

	vsubpd	zmm21, zmm21, [srcreg+r9+d2+d1]			;; R4 = R4 - MemR4
	zfmsubpd zmm9, zmm9, zmm1, [srcreg+r9+d2+d1+64]		;; I4 = B4 * 2 - MemI4

	vmovapd	zmm12, zmm17					;;		Copy R2
	vmulpd	zmm12 {k6}, zmm11, zmm11			;; R2				R1b * R1b (R1b)

	vsubpd	zmm23, zmm23, [srcreg+r9]			;; R1 = R1 - MemR1
	zfnmaddpd zmm11, zmm10, [srcreg+r9+64], zmm11		;; I1/2 = I1/2 - MemI1/2	undefined
	vsubpd zmm12 {k6}, zmm12, [srcreg+r9+64]		;; R2				R1b = R1b - MemI1

	vsubpd	zmm22, zmm22, [srcreg+r9+d4]			;; R5 = R5 - MemR5
	zfnmaddpd zmm7, zmm10, [srcreg+r9+d4+64], zmm7		;; I5/2 = I5/2 - MemI5/2

	vsubpd	zmm18, zmm18, [srcreg+r9+d4+d1]			;; R6 = R6 - MemR6
	zfnmaddpd zmm6, zmm10, [srcreg+r9+d4+d1+64], zmm6	;; I6/2 = I6/2 - MemI6/2

	vsubpd	zmm16, zmm16, [srcreg+r9+d4+d2]			;; R7 = R7 - MemR7
	zfnmaddpd zmm14, zmm10, [srcreg+r9+d4+d2+64], zmm14	;; I7/2 = I7/2 - MemI7/2

	vsubpd	zmm20, zmm20, [srcreg+r9+d4+d2+d1]		;; R8 = R8 - MemR8
	zfmsubpd zmm0, zmm0, zmm1, [srcreg+r9+d4+d2+d1+64]	;; I8 = B8 * 2 - MemI8
	jmp	fma_done

mulsubhard:
	vmovapd	zmm3, [srcreg+r9+d1]				;; MemR2
	vmovapd	zmm4, [srcreg+r10+d1]				;; MemR2#2
	zfnmaddpd zmm17, zmm3, zmm4, zmm17			;; R2 = R2 - MemR2*MemR2#2
	vmovapd	zmm5, [srcreg+r10+d1+64]			;; MemI2#2
	vmulpd	zmm3, zmm3, zmm5				;; TMP = MemR2*MemI2#2
	vmovapd	zmm8, [srcreg+r9+d1+64]				;; MemI2
	zfmaddpd zmm17, zmm8, zmm5, zmm17			;; R2 = R2 + MemI2*MemI2#2
	zfmaddpd zmm3, zmm8, zmm4, zmm3				;; TMP += MemI2*MemR2#2
	zfnmaddpd zmm2, zmm10, zmm3, zmm2			;; I2/2 = I2/2 - TMP/2

	vmovapd	zmm3, [srcreg+r9+d2]				;; MemR3
	vmovapd	zmm4, [srcreg+r10+d2]				;; MemR3#2
	zfnmaddpd zmm19, zmm3, zmm4, zmm19			;; R3 = R3 - MemR3*MemR3#2
	vmovapd	zmm5, [srcreg+r10+d2+64]			;; MemI3#2
	vmulpd	zmm3, zmm3, zmm5				;; TMP = MemR3*MemI3#2
	vmovapd	zmm8, [srcreg+r9+d2+64]				;; MemI3
	zfmaddpd zmm19, zmm8, zmm5, zmm19			;; R3 = R3 + MemI3*MemI3#2
	zfmaddpd zmm3, zmm8, zmm4, zmm3				;; TMP += MemI3*MemR3#2
	zfnmaddpd zmm13, zmm10, zmm3, zmm13			;; I3/2 = I3/2 - TMP/2

	vmovapd	zmm3, [srcreg+r9+d2+d1]				;; MemR4
	vmovapd	zmm4, [srcreg+r10+d2+d1]			;; MemR4#2
	zfnmaddpd zmm21, zmm3, zmm4, zmm21			;; R4 = R4 - MemR4*MemR4#2
	vmovapd	zmm5, [srcreg+r10+d2+d1+64]			;; MemI4#2
	vmulpd	zmm3, zmm3, zmm5				;; TMP = MemR4*MemI4#2
	vmovapd	zmm8, [srcreg+r9+d2+d1+64]			;; MemI4
	zfmaddpd zmm21, zmm8, zmm5, zmm21			;; R4 = R4 + MemI4*MemI4#2
	zfmaddpd zmm3, zmm8, zmm4, zmm3				;; TMP += MemI4*MemR4#2
	zfmsubpd zmm9, zmm9, zmm1, zmm3				;; I4 = B4 * 2 - TMP

	vmovapd	zmm12, zmm17					;;		Copy R2
	vmulpd	zmm12 {k6}, zmm11, zmm11			;; R2				R1b * R1b (R1b)

	vmovapd	zmm3, [srcreg+r9]				;; MemR1
	vmovapd	zmm4, [srcreg+r10]				;; MemR1#2
	zfnmaddpd zmm23, zmm3, zmm4, zmm23			;; R1 = R1 - MemR1*MemR1#2
	vmovapd	zmm5, [srcreg+r10+64]				;; MemI1#2
	vmulpd	zmm3, zmm3, zmm5				;; TMP = MemR1*MemI1#2
	vmovapd	zmm8, [srcreg+r9+64]				;; MemI1
	zfmaddpd zmm23{k7}, zmm8, zmm5, zmm23			;; R1 = R1 + MemI1*MemI1#2	R1
	zfnmaddpd zmm12{k6}, zmm8, zmm5, zmm12			;; R2				R1b = R1b - MemI1*MemI1#2
	zfmaddpd zmm3, zmm8, zmm4, zmm3				;; TMP += MemI1*MemR1#2
	zfnmaddpd zmm11, zmm10, zmm3, zmm11			;; I1/2 = I1/2 - TMP/2		undefined

	vmovapd	zmm3, [srcreg+r9+d4]				;; MemR5
	vmovapd	zmm4, [srcreg+r10+d4]				;; MemR5#2
	zfnmaddpd zmm22, zmm3, zmm4, zmm22			;; R5 = R5 - MemR5*MemR5#2
	vmovapd	zmm5, [srcreg+r10+d4+64]			;; MemI5#2
	vmulpd	zmm3, zmm3, zmm5				;; TMP = MemR5*MemI5#2
	vmovapd	zmm8, [srcreg+r9+d4+64]				;; MemI5
	zfmaddpd zmm22, zmm8, zmm5, zmm22			;; R5 = R5 + MemI5*MemI5#2
	zfmaddpd zmm3, zmm8, zmm4, zmm3				;; TMP += MemI5*MemR5#2
	zfnmaddpd zmm7, zmm10, zmm3, zmm7			;; I5/2 = I5/2 - TMP/2

	vmovapd	zmm3, [srcreg+r9+d4+d1]				;; MemR6
	vmovapd	zmm4, [srcreg+r10+d4+d1]			;; MemR6#2
	zfnmaddpd zmm18, zmm3, zmm4, zmm18			;; R6 = R6 - MemR6*MemR6#2
	vmovapd	zmm5, [srcreg+r10+d4+d1+64]			;; MemI6#2
	vmulpd	zmm3, zmm3, zmm5				;; TMP = MemR6*MemI6#2
	vmovapd	zmm8, [srcreg+r9+d4+d1+64]			;; MemI6
	zfmaddpd zmm18, zmm8, zmm5, zmm18			;; R6 = R6 + MemI6*MemI6#2
	zfmaddpd zmm3, zmm8, zmm4, zmm3				;; TMP += MemI6*MemR6#2
	zfnmaddpd zmm6, zmm10, zmm3, zmm6			;; I6/2 = I6/2 - TMP/2

	vmovapd	zmm3, [srcreg+r9+d4+d2]				;; MemR7
	vmovapd	zmm4, [srcreg+r10+d4+d2]			;; MemR7#2
	zfnmaddpd zmm16, zmm3, zmm4, zmm16			;; R7 = R7 - MemR7*MemR7#2
	vmovapd	zmm5, [srcreg+r10+d4+d2+64]			;; MemI7#2
	vmulpd	zmm3, zmm3, zmm5				;; TMP = MemR7*MemI7#2
	vmovapd	zmm8, [srcreg+r9+d4+d2+64]			;; MemI7
	zfmaddpd zmm16, zmm8, zmm5, zmm16			;; R7 = R7 + MemI7*MemI7#2
	zfmaddpd zmm3, zmm8, zmm4, zmm3				;; TMP += MemI7*MemR7#2
	zfnmaddpd zmm14, zmm10, zmm3, zmm14			;; I7/2 = I7/2 - TMP/2

	vmovapd	zmm3, [srcreg+r9+d4+d2+d1]			;; MemR8
	vmovapd	zmm4, [srcreg+r10+d4+d2+d1]			;; MemR8#2
	zfnmaddpd zmm20, zmm3, zmm4, zmm20			;; R8 = R8 - MemR8*MemR8#2
	vmovapd	zmm5, [srcreg+r10+d4+d2+d1+64]			;; MemI8#2
	vmulpd	zmm3, zmm3, zmm5				;; TMP = MemR8*MemI8#2
	vmovapd	zmm8, [srcreg+r9+d4+d2+d1+64]			;; MemI8
	zfmaddpd zmm20, zmm8, zmm5, zmm20			;; R8 = R8 + MemI8*MemI8#2
	zfmaddpd zmm3, zmm8, zmm4, zmm3				;; TMP += MemI8*MemR8#2
	zfmsubpd zmm0, zmm0, zmm1, zmm3				;; I8 = B8 * 2 - TMP

fma_done:					;;				R1a/R1b becomes R1/R2
	vaddpd	zmm3, zmm23, zmm12		;; R1 + R2 (new R1)		R1 + R2 (new R1/HALF)		; 13-19		n 26
	vsubpd	zmm23, zmm23, zmm12		;; R1 - R2 (new R2)		R1 - R2 (new R2/HALF)		; 14-20		n 28

						;;				R2/(I2/2) becomes R3/(R4/2)
	vaddpd	zmm15, zmm11, zmm2		;; I1/2 + I2/2 (new I1/2)	undefined			; 14-47		n 25
	vsubpd	zmm2 {k7}, zmm11, zmm2		;; I1/2 - I2/2 (new I2/2)	R4/2				; 15-47		n 26

	vaddpd	zmm8, zmm21, zmm19		;; R4 + R3 (new R3)		R6 + R5 (new R5)		; 15-52		n 25
	vsubpd	zmm21, zmm21, zmm19		;; R4 - R3 (new I4)		R6 - R5 (new negR6)		; 16-52		n 33

	zfmaddpd zmm11, zmm13, zmm27, zmm9	;; I3*2 + I4 (new I3)		I5*2 + I6 (new I5)		; 16-53		n 27
	zfmsubpd zmm13, zmm13, zmm27, zmm9	;; I3*2 - I4 (new R4)		I5*2 - I6 (new I6)		; 17-53		n 26

	vaddpd	zmm4, zmm22, zmm18		;; R5 + R6 (new R5)		R9 + R10 (new R9)		; 17-48		n 34
	vsubpd	zmm22, zmm22, zmm18		;; R5 - R6 (new R6)		R9 - R10 (new R10)		; 18-48		n 29

	vaddpd	zmm1, zmm7, zmm6		;; I5/2 + I6/2 (new I5/2)	I9/2 + I10/2 (new I9/2)		; 18-49		n 35
	vsubpd	zmm6, zmm7, zmm6		;; I5/2 - I6/2 (new I6/2)	I9/2 - I10/2 (new I10/2)	; 19-49		n 

	jmp	back_to_orig
orig:
	;; Square the results
	vmulpd	zmm23, zmm1, zmm1		;; A1 = R1 * R1			R1a*R1a (R1a)			; 1-4		n 5
	vmulpd	zmm17, zmm2, zmm2		;;		A2 = R2 * R2					; 1-4		n 5

	vmulpd	zmm19, zmm13, zmm13		;;		A3 = R3 * R3					; 2-5		n 8
	vmulpd	zmm21, zmm9, zmm9		;;		A4 = R4 * R4					; 2-5		n 8

	vmulpd	zmm22, zmm10, zmm10		;;		A5 = R5 * R5					; 3-6		n 9
	vmulpd	zmm18, zmm6, zmm6		;;		A6 = R6 * R6					; 3-6		n 10

	vmulpd	zmm16, zmm14, zmm14		;;		A7 = R7 * R7					; 4-7		n 12
	vmulpd	zmm20, zmm0, zmm0		;;		A8 = R8 * R8					; 4-7		n 12

	zfnmaddpd zmm23 {k7}, zmm11, zmm11, zmm23;; A1 - I1 * I1 (R1)		R1a				; 5-8		n 13
	zfnmaddpd zmm17, zmm3, zmm3, zmm17	;;		A2 - I2 * I2 (R2)				; 5-8		n 9
	vmovsd	Q [srcreg+r8-16], xmm23		;;				Save square of FFT sums		; 9

	vmulpd	zmm9, zmm12, zmm9		;;		B4 = I4 * R4					; 6-9		n 10
	vmulpd	zmm0, zmm5, zmm0		;;		B8 = I8 * R8					; 6-9		n 13

	vmulpd	zmm2, zmm3, zmm2		;;		I2 * R2 (I2/2)					; 7-10		n 14
	vmulpd	zmm13, zmm15, zmm13		;;		I3 * R3 (I3/2 & I5/2)				; 7-10		n 16

	zfnmaddpd zmm19, zmm15, zmm15, zmm19	;;		A3 - I3 * I3 (R3 & R5)				; 8-11		n 
	zfnmaddpd zmm21, zmm12, zmm12, zmm21	;;		A4 - I4 * I4 (R4 & R6)				; 8-11		n 

	vmovapd	zmm12, zmm17			;;		Copy R2
	vmulpd	zmm12 {k6}, zmm11, zmm11	;; R2				R1b * R1b (R1b)			; 9-12		n 13
	zfnmaddpd zmm22, zmm7, zmm7, zmm22	;;		A5 - I5 * I5 (R5 & R9)				; 9-12		n 

	vaddpd	zmm9, zmm9, zmm9		;;		B4 * 2 (I4 & I6)				; 10-13		n 
	zfnmaddpd zmm18, zmm8, zmm8, zmm18	;;		A6 - I6 * I6 (R6 & R10)				; 10-13		n 

	vmulpd	zmm6, zmm8, zmm6 		;;		I6 * R6 (I6/2 & I10/2)				; 11-12		n 
	vmulpd	zmm14, zmm4, zmm14		;;		I7 * R7 (I7/2 & I11/2)				; 11-15		n 

	zfnmaddpd zmm16, zmm4, zmm4, zmm16	;;		A7 - I7 * I7 (R7 & R11)				; 12-18		n 
	zfnmaddpd zmm20, zmm5, zmm5, zmm20	;;		A8 - I8 * I8 (R8 & R12)				; 12-20		n 

	vaddpd	zmm0, zmm0, zmm0		;;		B8 * 2 (I8 & I12)				; 13-17		n 
						;;				R1a/R1b becomes R1/R2
	vaddpd	zmm3, zmm23, zmm12		;; R1 + R2 (new R1)		R1 + R2 (new R1/HALF)		; 13-19		n 26
	vsubpd	zmm23, zmm23, zmm12		;; R1 - R2 (new R2)		R1 - R2 (new R2/HALF)		; 14-20		n 28

						;;				R2/(I2/2) becomes R3/(R4/2)
	zfmaddpd zmm15, zmm11, zmm1, zmm2	;; origI1*origR1 + I2 (new I1/2) undefined			; 14-47		n 25
	zfmsubpd zmm2 {k7}, zmm11, zmm1, zmm2	;; origI1*origR1 - I2 (new I2/2) R4/2				; 15-47		n 26

	vaddpd	zmm8, zmm21, zmm19		;; R4 + R3 (new R3)		R6 + R5 (new R5)		; 15-52		n 25
	vsubpd	zmm21, zmm21, zmm19		;; R4 - R3 (new I4)		R6 - R5 (new negR6)		; 16-52		n 33

	zfmaddpd zmm11, zmm13, zmm27, zmm9	;; I3*2 + I4 (new I3)		I5*2 + I6 (new I5)		; 16-53		n 27
	zfmsubpd zmm13, zmm13, zmm27, zmm9	;; I3*2 - I4 (new R4)		I5*2 - I6 (new I6)		; 17-53		n 26

	vaddpd	zmm4, zmm22, zmm18		;; R5 + R6 (new R5)		R9 + R10 (new R9)		; 17-48		n 34
	vsubpd	zmm22, zmm22, zmm18		;; R5 - R6 (new R6)		R9 - R10 (new R10)		; 18-48		n 29

	zfmaddpd zmm1, zmm7, zmm10, zmm6	;; origI5*origR5 + I6 (new I5/2) origI5*origR5 + I10 (new I9/2)	; 18-49		n 35
	zfmsubpd zmm6, zmm7, zmm10, zmm6	;; origI5*origR5 - I6 (new I6/2) origI5*origR5 - I10 (new I10/2); 19-49		n 

back_to_orig:
	vaddpd	zmm12, zmm20, zmm16		;; R8 + R7 (new R7)		R12 + R11 (new R11)		; 19-50		n 34
	vsubpd	zmm20, zmm20, zmm16		;; R8 - R7 (new I8)		R12 - R11 (new I12)		; 20-50		n 29

	zfmaddpd zmm5, zmm14, zmm27, zmm0	;; I7*2 + I8 (new I7)		I11*2 + I12 (new I11)		; 20-51		n 35
	zfmsubpd zmm14, zmm14, zmm27, zmm0	;; I7*2 - I8 (new R8)		I11*2 - I12 (new R12)		; 21-51		n 32

	vblendmpd zmm28 {k6}, zmm26, ZMM_HALF{1to8} ;; 1, 1, 1, 1, 1, 1, 1	HALF				; 25***		n 
	vblendmpd zmm0 {k6}, zmm8, zmm17	;; R3				R3				; 25		n 26

	vmovapd	zmm16, zmm13			;; R4				I6
	vaddpd zmm16 {k6}, zmm2, zmm2		;; R4				(R4/2)*2			; 18-21		n 
	vaddpd zmm13 {k7}, zmm2, zmm2		;; (I2/2)*2			I6				; 31-34		n ??***

	zfmaddpd zmm19, zmm3, zmm28, zmm0	;; R1*1 + R3 (newer R1)		R1*HALF + R3 (newer R1)		; 26-57		n 
	zfmsubpd zmm3, zmm3, zmm28, zmm0	;; R1*1 - R3 (newer R3)		R1*HALF - R3 (newer R3)		; 27-57		n 

	zfmaddpd zmm9, zmm15, zmm27, zmm11	;; I1*2 + I3 (newer I1)		undefined			; 27-59		n 38
	zfmsubpd zmm15 {k7}{z}, zmm15, zmm27, zmm11;; I1*2 - I3 (newer I3)	0				; 28-59		n 39

	zfmaddpd zmm7, zmm23, zmm28, zmm16	;; R2*1 + R4 (newer R2)		R2*HALF + R4 (newer R2)		; 28-61		n 45
	zfmsubpd zmm23, zmm23, zmm28, zmm16	;; R2*1 - R4 (newer R4)		R2*HALF - R4 (newer R4)		; 29-61		n 46

;GW these consts may need study/work
	vblendmpd zmm28 {k6}, zmm26, ZMM_P383{1to8} ;; 1, 1, 1, 1, 1, 1, 1	.383				; 25***		n 
	vmovapd zmm0, zmm22			;; R6				R10
	vmulpd	zmm0 {k6}, zmm28, zmm20		;; R6				I12*.383			; 29-61		n 
	vmulpd	zmm20 {k6}, zmm28, zmm22	;; I8				R10*.383			; 30-54		n 

						;; multiply R8/I8 by SQRTHALF - i*SQRTHALF
	zfmaddpd zmm2, zmm14, zmm28, zmm20	;; I8 + R8*1 (new2 R8/SQRTHALF)	R10*.383 + R12 (newer R10*.383)	; 32-55		n 36
	zfnmaddpd zmm20, zmm14, zmm28, zmm20	;; I8 - R8*1 (new2 I8/SQRTHALF)	R10*.383 - R12 (newer R12*.383)	; 32-55		n 37

						;; multiply R6/I6 by SQRTHALF - i*SQRTHALF
	vaddpd	zmm28, zmm28, zmm28		;; 2, 2, 2, 2, 2, 2, 2		2*.383				; 25***		n 
	zfmaddpd zmm16, zmm6, zmm28, zmm0	;; I6*2 + R6 (new2 R6/SQRTHALF)	I10*2*.383 + I12 (newer I10*.383) ; 30-54	n 36
	zfmsubpd zmm6, zmm6, zmm28, zmm0	;; I6*2 - R6 (new2 I6/SQRTHALF)	I10*2*.383 - I12 (newer I12*.383) ; 31-54	n 37

						;;				mul R6/I6 by w^2 = .707 - .707i
	vaddpd	zmm0, zmm13, zmm21		;; I2 + I4 (newer I2)		I6 + negR6 (newer I6/SQRTHALF) 	; 33-60		n 44
	vsubpd	zmm13, zmm13, zmm21		;; I2 - I4 (newer I4)		I6 - negR6 (newer R6/SQRTHALF) 	; 33-60		n 44

	vsubpd	zmm18, zmm12, zmm4		;; R7 - R5 (newer I7)		R11 - R9 (newer negR11)		; 34-58		n 38
	vaddpd	zmm12, zmm12, zmm4		;; R7 + R5 (newer R5)		R11 + R9 (newer R9)		; 34-37		n 38

	zfmaddpd zmm4, zmm1, zmm27, zmm5	;; I5*2 + I7 (newer I5)		I9*2 + I11 (newer I9)		; 35-56		n 42
	zfmsubpd zmm1, zmm1, zmm27, zmm5	;; I5*2 - I7 (newer R7)		I9*2 - I11 (newer I11)		; 35-56		n 39

						;;				mul R10/I10 by w^1 = .924 - .383i
	zfmaddpd zmm5, zmm2, zmm26, zmm16	;; R6 + R8*1 (newer R6/SQRTHALF)    I10 + R10*.924/.383 (R10)	; 36-71		n 44
	zfmsubpd zmm16, zmm16, zmm26, zmm2	;; R6*1 - R8 (newer negI8/SQRTHALF) I10*.924/.383 - R10 (I10)	; 36-71		n 

						;;				mul R12/I12 by w^3 = .383 - .924i
	zfmaddpd zmm2, zmm6, zmm26, zmm20	;; I6*1 + I8 (newer I6/SQRTHALF)  I12*.924/.383 + R12 (R12)	; 37-63		n 47
	zfnmaddpd zmm20, zmm20, zmm26, zmm6	;; I6 - I8*1 (newer R8/SQRTHALF)  I12 - R12*.924/.383 (I12)	; 37-63		n 44

						;;				R5/I5 becomes new R5/R7
	vblendmpd zmm8 {k6}, zmm12, zmm8	;; R5				R5				; 38		n 40
	vsubpd	zmm18 {k6}, zmm15, zmm18	;; I7				R11 = 0 - negR11		; 38-55		n 43

	vblendmpd zmm29 {k6}, zmm26, zmm31	;; 1, 1, 1, 1, 1, 1, 1		SQRTHALF
	vblendmpd zmm11 {k6}, zmm1, zmm11	;; R7				R7				; 39		n 41
	vmulpd	zmm15 {k6}, zmm29, zmm1		;; I3				I11*SQRTHALF			; 39-42		n 43

;; four aparts

	vaddpd	zmm22, zmm19, zmm8		;; R1 + R5 (final R1)		R1 + R5 (final R1)		; 40-66		n 
	vsubpd	zmm19, zmm19, zmm8		;; R1 - R5 (final R5)		R1 - R5 (final R5)		; 40-66		n 

	vaddpd	zmm8, zmm3, zmm11		;; R3 + R7 (final R3)		R3 + R7 (final R3)		; 41-64		n 
	vsubpd	zmm3, zmm3, zmm11		;; R3 - R7 (final R7)		R3 - R7 (final R7)		; 41-64		n 

						;;				R9/I9 becomes newer R9/R13
	vaddpd	zmm12 {k7}, zmm9, zmm4		;; I1 + I5 (final I1)		R9				; 42-67		n 
	vsubpd	zmm4 {k7}, zmm9, zmm4		;; I1 - I5 (final I5)		R13				; 42-67		n 

						;;				R11/I11 becomes newer R11/R15
						;;				mul R11/I11 by SQRTHALF - i*SQRTHALF
	zfmaddpd zmm6, zmm18, zmm29, zmm15	;; I3 + I7*1 (final I3)		I11 + R11*SQRTHALF (final R11)	; 43-65		n 
	zfnmaddpd zmm15, zmm18, zmm29, zmm15	;; I3 - I7*1 (final I7)		I11 - R11*SQRTHALF (final I11)	; 43-65		n 

						;;				R6/I6 becomes new R6/R8
	vblendmpd zmm9 {k6}, zmm5, zmm13	;; R6/SQRTHALF			R6/SQRTHALF			; 44		n 
	vblendmpd zmm21 {k6}, zmm20, zmm0	;; R8/SQRTHALF			R8/SQRTHALF			; 44		n 

	zfmaddpd zmm17, zmm9, zmm31, zmm7	;; R2 + R6*SQRTHALF (final R2)	R2 + R6 * SQRTHALF (final R2)	; 45-68		n 
	zfnmaddpd zmm9, zmm9, zmm31, zmm7	;; R2 - R6*SQRTHALF (final R6)	R2 - R6 * SQRTHALF (final R6)	; 45-68		n 

	zfmaddpd zmm7, zmm21, zmm31, zmm23	;; R4 + R8*SQRTHALF (final R4)	R4 + R8 * SQRTHALF (final R4)	; 46-70		n 
	zfnmaddpd zmm21, zmm21, zmm31, zmm23	;; R4 - R8*SQRTHALF (final R8)	R4 - R8 * SQRTHALF (final R8)	; 46-70		n 

						;;				R10/I10 becomes newer R10/R14
	vmovapd	zmm18, zmm16			;; negI8			R14
	zfmaddpd zmm5 {k7}, zmm2, zmm31, zmm0	;; I2 + I6*SQRTHALF (final I2)	R10				; 47-71		n 
	zfnmaddpd zmm16 {k7}, zmm2, zmm31, zmm0	;; I2 - I6*SQRTHALF (final I6)	R14				; 47-71		n 

						;;				R12/I12 becomes newer R12/R16
	zfmaddpd zmm20 {k7}, zmm18, zmm31, zmm13;; I4 + negI8*SQRTHALF (final I8)  R16				; 48-69		n 
	zfnmaddpd zmm2 {k7}, zmm18, zmm31, zmm13;; I4 - negI8*SQRTHALF (final I4)  R12				; 48-69		n 

	zstore	[srcreg+r8], zmm22		;; Save R1						; 30
	zstore	[srcreg+r8+64], zmm12		;; Save R9						; 30+1
	zstore	[srcreg+r8+d4], zmm19		;; Save R5						; 31+1
	zstore	[srcreg+r8+d4+64], zmm4		;; Save R13						; 31+2
	zstore	[srcreg+r8+d2], zmm8		;; Save R3						; 37
	zstore	[srcreg+r8+d2+64], zmm6		;; Save R11						; 37+1
	zstore	[srcreg+r8+d4+d2], zmm3		;; Save R7						; 38+1
	zstore	[srcreg+r8+d4+d2+64], zmm15	;; Save R15						; 38+2
	zstore	[srcreg+r8+d1], zmm17		;; Save R2						; 39+2
	zstore	[srcreg+r8+d1+64], zmm5		;; Save R10						; 39+3
	zstore	[srcreg+r8+d4+d1], zmm9		;; Save R6						; 40+3
	zstore	[srcreg+r8+d4+d1+64], zmm16	;; Save R14						; 40+4
	zstore	[srcreg+r8+d2+d1], zmm7		;; Save R4						; 41+4
	zstore	[srcreg+r8+d2+d1+64], zmm2	;; Save R12						; 41+5
	zstore	[srcreg+r8+d4+d2+d1], zmm21	;; Save R8						; 42+5
	zstore	[srcreg+r8+d4+d2+d1+64], zmm20	;; Save R16						; 42+6
	bump	srcreg, srcinc
	ENDM



zr8_sixteen_reals_eight_complex_with_mult_preload MACRO
	;; preload would be silly for a macro that is executed only once
	ENDM
zr8_sixteen_reals_eight_complex_with_mult MACRO srcreg,srcinc,d1,d2,d4
	LOCAL	orig, back_to_orig, dispatch, disptab, addmul, submul, muladd, muladdhard, mulsub, mulsubhard, unfft
	LOCAL	addmul312, addmul313, addmul132, addmul133, submul312, submul313, submul132, submul133

;GW these consts may need study/work
	vbroadcastsd zmm29, ZMM_P383
	vbroadcastsd zmm30, ZMM_P924_P383
	vbroadcastsd zmm31, ZMM_SQRTHALF
	mov	eax, 11111110b			;; We're pretty sure callers of this macro call zloop_init next
	kmovw	k7, eax				;; Set k7 to 11111110b						; 30		n 31
	knotw	k6, k7				;; Set k6 to 00000001b						; 31		n 50
	vbroadcastsd zmm22, ZMM_ONE
	vblendmpd zmm26 {k6}, zmm22, zmm30	;; 1, 1, 1, 1, 1, 1, 1		.924/.383			; 50		n 68
	vblendmpd zmm27 {k6}, zmm22, zmm31	;; 1, 1, 1, 1, 1, 1, 1		SQRTHALF			; 51		n 87
	vmovapd zmm25 {k7}{z}, zmm22		;; 1, 1, 1, 1, 1, 1, 1		0				; 52		n 53
	vsubpd	zmm25 {k6}, zmm25, zmm31	;; 1, 1, 1, 1, 1, 1, 1		-SQRTHALF			; 53		n 86
	vblendmpd zmm28 {k6}, zmm31, zmm29	;; SQRTHALF, SQRTHALF, ...,	.383				; 54		n 96


						;; Simultaneously do an 8-complex and 16-reals
						;; 8-complex			16-reals
	vmovapd	zmm9, [srcreg]			;; R1				R1+R9 (new R1)
	vmovapd	zmm0, [srcreg+d4]		;; R5				R5+R13 (new R5)
	vaddpd	zmm2, zmm9, zmm0		;;		R1 + R5 (new R1)				; 42-45		n 56
	vsubpd	zmm9, zmm9, zmm0		;;		R1 - R5 (new R5)				; 42-45		n 56

	vmovapd	zmm19, [srcreg+d2]		;; R3				R3+R11 (new R3)
	vmovapd	zmm0, [srcreg+d4+d2]		;; R7				R7+R15 (new R7)
	vaddpd	zmm14, zmm19, zmm0		;;		R3 + R7 (new R3)				; 44-47		n 56
	vsubpd	zmm19, zmm19, zmm0		;;		R3 - R7 (new R7)				; 44-47		n 56

	vmovapd	zmm12, [srcreg+d1]		;; R2				R2+R10 (new R2)
	vmovapd	zmm0, [srcreg+d4+d1]		;; R6				R6+R14 (new R6)
	vaddpd	zmm3, zmm12, zmm0		;;		R2 + R6 (new R2)				; 46-49		n 60
	vsubpd	zmm12, zmm12, zmm0		;;		R2 - R6 (new R6)				; 46-49		n 60

	vmovapd	zmm11, [srcreg+d2+d1]		;; R4				R4+R12 (new R4)
	vmovapd	zmm0, [srcreg+d4+d2+d1]		;; R8				R8+R16 (new R8)
	vaddpd	zmm1, zmm11, zmm0		;;		R4 + R8 (new R4)				; 48-51		n 60
	vsubpd	zmm11, zmm11, zmm0		;;		R4 - R8 (new R8)				; 48-51		n 60

						;;				R12/R16 becomes newer R12/I12
						;;				mul R12/I12 by w^3 = .383 + .924i
	vmovapd	zmm10, [srcreg+d2+d1+64]	;; I4				R4-R12 (new R12)
	vmovapd	zmm15, [srcreg+d4+d2+d1+64]	;; I8				R8-R16 (new R16 a.k.a. I12)
	zfnmaddpd zmm6, zmm15, zmm26, zmm10	;; I4 - I8*1 (new I8)		R12/.383 = R12 - I12*.924/.383	; 68-71		n 76
	zfmaddpd zmm10, zmm10, zmm26, zmm15	;; I4*1 + I8 (new I4)		I12/.383 = R12*.924/.383 + I12	; 69-72		n 81

						;;				R11/R15 becomes newer R11/I11
						;;				mul R11/I11 by SQRTHALF + i*SQRTHALF
	vmovapd	zmm4, [srcreg+d2+64]		;; I3				R3-R11 (new R11)
	vmovapd	zmm5, [srcreg+d4+d2+64]		;; I7				R7-R15 (new R15 a.k.a. I11)
	vaddpd	zmm15, zmm4, zmm5		;; I3 + I7 (new I3)		I11/SQRTHALF = R11 + I11	; 70-73		n 84
	vsubpd	zmm4, zmm4, zmm5		;; I3 - I7 (new I7)		R11/SQRTHALF = R11 - I11	; 71-74		n 86

						;;				R10/R14 becomes newer R10/I10
						;;				mul R10/I10 by w^1 = .924 + .383i
	vmovapd	zmm0, [srcreg+d1+64]		;; I2				R2-R10 (new R10)
	vmovapd	zmm13, [srcreg+d4+d1+64]	;; I6				R6-R14 (new R14 a.k.a. I10)
	zfmsubpd zmm5, zmm0, zmm26, zmm13	;; I2*1 - I6 (new I6)		R10/.383 = R10*.924/.383 - I10	; 72-75		n 80
	zfmaddpd zmm0, zmm13, zmm26, zmm0	;; I2 + I6*1 (new I2)		I10/.383 = I10*.924/.383 + R10	; 73-76		n 81

						;;				R9/R13 becomes newer R9/I9
	vmovapd	zmm16, [srcreg+64]		;; I1				R1-R9 (new R9)
	vmovapd	zmm8, [srcreg+d4+64]		;; I5				R5-R13 (new R13 a.k.a. I9)
	vaddpd	zmm13 {k7}{z}, zmm16, zmm8	;; I1 + I5 (new I1)		0				; 74-77		n 85
	vsubpd	zmm8 {k7}, zmm16, zmm8		;; I1 - I5 (new I5)		I9				; 75-78		n 87

						;; next FFT level
						;;				R6/R8 becomes newest R6/I6
						;;				mul R6/I6 by w^2 = .707 + .707i
	vblendmpd zmm18 {k6}, zmm6, zmm11	;; I8				I6				; 76		n 78
	vblendmpd zmm11 {k6}, zmm11, zmm6	;; R8				R12				; 77		n 80

	vsubpd	zmm17, zmm12, zmm18		;; R6 - I8 (new2 R6)		R6 - I6 (newest R6/SQRTHALF)	; 78-12		n 88
	vaddpd	zmm12, zmm12, zmm18		;; R6 + I8 (new2 R8)		R6 + I6 (newest I6/SQRTHALF)	; 79-12		n 90

	vaddpd	zmm6, zmm5, zmm11		;; I6 + R8 (new2 I6)		R10 + R12 (newest R10/.383)	; 80-13		n 88
	vsubpd	zmm5, zmm5, zmm11		;; I6 - R8 (new2 I8)		R10 - R12 (newest R12/.383)	; 80-13		n 90

	vaddpd	zmm11, zmm0, zmm10		;; I2 + I4 (newer I2)		I10 + I12 (newest I10/.383)	; 81-14		n 88
	vsubpd	zmm0, zmm0, zmm10		;; I2 - I4 (newer I4)		I10 - I12 (newest I12/.383)	; 81-14		n 89

	vaddpd	zmm10, zmm2, zmm14		;; R1 + R3 (newer R1)		R1 + R3 (newest R1)		; 82-15		n 89
	vsubpd	zmm2, zmm2, zmm14		;; R1 - R3 (newer R3)		R1 - R3 (newest R3)		; 82-15		n 91

	vaddpd	zmm14, zmm3, zmm1		;; R2 + R4 (newer R2)		R2 + R4 (newest R2)		; 83-86		n 89
	vsubpd	zmm3, zmm3, zmm1		;; R2 - R4 (newer R4)		R2 - R4 (newest R4)		; 83-16		n 91

	vblendmpd zmm18 {k6}, zmm19, zmm15	;; R7				I11/SQRTHALF			; 84		n 87
	vblendmpd zmm16 {k6}, zmm9, zmm16	;; R5				R9				; 84		n 86

						;;				R5/R7 becomes newest R5/I5
	vsubpd	zmm19 {k7}, zmm13, zmm15	;; I1 - I3 (newer I3)		I5				; 85-17		n 95
	vaddpd	zmm13 {k7}{z}, zmm13, zmm15	;; I1 + I3 (newer I1)		0				; 85-17		n 93

	zfnmaddpd zmm7, zmm4, zmm25, zmm16	;; R5 - I7*1 (newer R5)		R9 - R11*-SQRTHALF (newest R9)	; 86-18		n 96
	zfmaddpd zmm4, zmm4, zmm25, zmm16	;; R5 + I7*1 (newer R7)		R9 + R11*-SQRTHALF (newest R11)	; 86-18		n 98

	zfmaddpd zmm16, zmm18, zmm27, zmm8	;; I5 + R7*1 (newer I5)		I9 + I11*SQRTHALF (newest I9)	; 87-19		n 97
	zfnmaddpd zmm18, zmm18, zmm27, zmm8	;; I5 - R7*1 (newer I7)		I9 - I11*SQRTHALF (newest I11)	; 87-19		n 99

	vmovapd	zmm8, zmm11			;;				I10/.383
	vaddpd	zmm8 {k7}, zmm17, zmm6		;; I6 = R6 + I6 (newer I6/SQRTHALF)	I10/.383		; 88-91		n 97
	vsubpd	zmm6 {k7}, zmm17, zmm6		;; R6 = R6 - I6 (newer R6/SQRTHALF)	R10/.383		; 88-91		n 96

	vblendmpd zmm17 {k6}, zmm0, zmm17	;; I4				R6/SQRTHALF			; 89		n 94
	vsubpd	zmm11 {k6}, zmm10, zmm14	;; I2				R1 - R2 (final R1b)		; 89-92		n 93

	vaddpd	zmm0 {k7}, zmm12, zmm5		;; I8 = R8 + I8 (newer I8/SQRTHALF)	I12/.383		; 90-92		n 98
	vsubpd	zmm5 {k7}, zmm12, zmm5		;; R8 = R8 - I8 (newer R8/SQRTHALF)	R12/.383		; 90-92		n 99

	;; last FFT level

	vblendmpd zmm9 {k6}, zmm2, zmm9		;; R3				R5				; 91		n 94
	vblendmpd zmm12 {k6}, zmm3, zmm12	;; R4				I6/SQRTHALF			; 91		n 95

						;;				R3/R4 becomes final R3/I3
	vaddpd	zmm1, zmm10, zmm14		;; R1 + R2 (final R1)		R1 + R2 (final R1a => R1)	; 92-95		n 
	vsubpd	zmm2 {k7}, zmm10, zmm14		;; R1 - R2 (final R2)		R3 => R2			; 92-95		n 

	vsubpd	zmm3 {k7}, zmm13, zmm11		;; I1 - I2 (final I2)		I3 => I2			; 93-96		n 
	vaddpd	zmm11 {k7}, zmm13, zmm11	;; I1 + I2 (final I1)		R1b => I1			; 93-96		n 

	zfnmaddpd zmm13, zmm17, zmm25, zmm9	;; R3 - I4*1 (final R3)		R5 - R6*-SQRTHALF (final R5 => R3) ; 94-97	n 
	zfmaddpd zmm9, zmm17, zmm25, zmm9	;; R3 + I4*1 (final R4)		R5 + R6*-SQRTHALF (final R6 => R4) ; 94-97	n 

	zfmaddpd zmm15, zmm12, zmm27, zmm19	;; I3 + R4*1 (final I3)		I5 + I6*SQRTHALF (final I5 => I3) ; 95-98	n 
	zfnmaddpd zmm12, zmm12, zmm27, zmm19	;; I3 - R4*1 (final I4)		I5 - I6*SQRTHALF (final I6 => I4) ; 95-98	n 

	zfmaddpd zmm10, zmm6, zmm28, zmm7	;; R5 + R6*SQRTHALF (final R5)	R9 + R10*.383 (final R9 => R5)	; 96-99		n 
	zfnmaddpd zmm6, zmm6, zmm28, zmm7	;; R5 - R6*SQRTHALF (final R6)	R9 - R10*.383 (final R10 => R6)	; 96-99		n 

	zfmaddpd zmm7, zmm8, zmm28, zmm16	;; I5 + I6*SQRTHALF (final I5)	I9 + I10*.383 (final I9 => I5)	; 97-100	n 
	zfnmaddpd zmm8, zmm8, zmm28, zmm16	;; I5 - I6*SQRTHALF (final I6)	I9 - I10*.383 (final I10 => I6)	; 97-100	n 

	zfnmaddpd zmm14, zmm0, zmm28, zmm4	;; R7 - I8*SQRTHALF (final R7)	R11 - I12*.383 (final R11 => R7); 98-101	n 
	zfmaddpd zmm0, zmm0, zmm28, zmm4	;; R7 + I8*SQRTHALF (final R8)	R11 + I12*.383 (final R12 => R8); 98-101	n 

	zfmaddpd zmm4, zmm5, zmm28, zmm18	;; I7 + R8*SQRTHALF (final I7)	I11 + R12*.383 (final I11 => I7); 99-102	n 
	zfnmaddpd zmm5, zmm5, zmm28, zmm18	;; I7 - R8*SQRTHALF (final I8)	I11 - R12*.383 (final I12 => I8); 99-102	n 

	cmp	mul4_opcode, 0			;; See if we need to do more than the original type-3 FFT multiply
	je	orig				;; Jump if nothing special
	movzx	r10, mul4_opcode		;; Load opcode
	jg	dispatch			;; Jump if not saving result of the forward FFT?

	;; Store FFT results
	zstore	[srcreg], zmm1			;; R1			R1a
	zstore	[srcreg+64], zmm11		;; I1			R1b
	zstore	[srcreg+d1], zmm2		;;		R2
	zstore	[srcreg+d1+64], zmm3		;;		I2
	zstore	[srcreg+d2], zmm13		;;		R3
	zstore	[srcreg+d2+64], zmm15		;;		I3
	zstore	[srcreg+d2+d1], zmm9		;;		R4
	zstore	[srcreg+d2+d1+64], zmm12	;;		I4
	zstore	[srcreg+d4], zmm10		;;		R5
	zstore	[srcreg+d4+64], zmm7		;;		I5
	zstore	[srcreg+d4+d1], zmm6		;;		R6
	zstore	[srcreg+d4+d1+64], zmm8		;;		I6
	zstore	[srcreg+d4+d2], zmm14		;;		R7
	zstore	[srcreg+d4+d2+64], zmm4		;;		I7
	zstore	[srcreg+d4+d2+d1], zmm0		;;		R8
	zstore	[srcreg+d4+d2+d1+64], zmm5	;;		I8
	and	r10, 7Fh			;; Mask out the save-FFT-results bit
	jz	orig				;; opcode == 0

dispatch:
	mov	r9, OFFSET disptab		;; Jump table address
	mov	r10, [r9+r10*8-8]		;; Jump table offset
	mov	r9, SRC2ARG			;; Distance to s2 arg from gwaddmul4 or gwsubmul4 or s3 arg in gwmuladd4 or gwmulsub4
	jmp	r10
disptab:
	DQ	OFFSET addmul			;; 1 = addmul
	DQ	OFFSET submul			;; 2 = submul
	DQ	OFFSET muladd			;; 3 = muladd
	DQ	OFFSET mulsub			;; 4 = mulsub
	DQ	OFFSET unfft			;; 5 = unfft
	DQ	OFFSET addmul312		;; 6 = addmul FFTval mem1 mem2
	DQ	OFFSET addmul313		;; 7 = addmul FFTval mem1 FFTval
	DQ	OFFSET addmul132		;; 8 = addmul mem1 FFTval mem2
	DQ	OFFSET addmul133		;; 9 = addmul mem1 FFTval FFTval
	DQ	OFFSET submul312		;; 10 = submul FFTval mem1 mem2
	DQ	OFFSET submul313		;; 11 = submul FFTval mem1 FFTval
	DQ	OFFSET submul132		;; 12 = submul mem1 FFTval mem2
	DQ	OFFSET submul133		;; 13 = submul mem1 FFTval FFTval

addmul:	vmovapd	zmm24, [srcreg+rbp+d1]		;;		MemR2
	vaddpd	zmm24, zmm24, [srcreg+r9+d1]	;;		MemR2
	vmulpd	zmm17, zmm2, zmm24		;;		A2 = R2 * MemR2					; 1-4		n 5
	vmulpd	zmm16, zmm3, zmm24		;;		B2 = I2 * MemR2					; 1-4		n 5

	vmovapd	zmm24, [srcreg+rbp+d2]		;;		MemR3
 	vaddpd	zmm24, zmm24, [srcreg+r9+d2]	;;		MemR3
	vmulpd	zmm19, zmm13, zmm24		;;		A3 = R3 * MemR3					; 2-5		n 6
	vmulpd	zmm18, zmm15, zmm24		;;		B3 = I3 * MemR3					; 2-5		n 6

	vmovapd	zmm24, [srcreg+rbp+d2+d1]	;;		MemR4
	vaddpd	zmm24, zmm24, [srcreg+r9+d2+d1];;		MemR4
	vmulpd	zmm21, zmm9, zmm24		;;		A4 = R4 * MemR4					; 3-6		n 8
	vmulpd	zmm20, zmm12, zmm24		;;		B4 = I4 * MemR4					; 3-6		n 8

	vmovapd	zmm24, [srcreg+rbp]		;; MemR1			MemR1a
	vaddpd	zmm24, zmm24, [srcreg+r9]	;; MemR1			MemR1a
	vmulpd	zmm23, zmm1, zmm24		;; A1 = R1 * MemR1		R1a*MemR1a (R1a)		; 4-7		n 10
	vmulpd	zmm22 {k7}{z}, zmm11, zmm24	;; B1 = I1 * MemR1		0				; 4-7		n 10

	vmovapd zmm24, [srcreg+rbp+d1+64]	;;		MemI2
	vaddpd	zmm24, zmm24, [srcreg+r9+d1+64];;		MemI2
	zfnmaddpd zmm17, zmm3, zmm24, zmm17	;;		A2 - I2 * MemI2 (R2)				; 5-8		n 10
	zfmaddpd zmm16, zmm2, zmm24, zmm16	;;		B2 + R2 * MemI2 (I2)				; 5-8		n 18

	vmovapd zmm24, [srcreg+rbp+d2+64]	;;		MemI3
	vaddpd	zmm24, zmm24, [srcreg+r9+d2+64];;		MemI3
	zfnmaddpd zmm19, zmm15, zmm24, zmm19	;;		A3 - I3 * MemI3 (R3 & R5)			; 6-9		n 19
	zfmaddpd zmm18, zmm13, zmm24, zmm18	;;		B3 + R3 * MemI3 (I3 & I5)			; 6-9		n 20

	vmovapd	zmm24, [srcreg+rbp+d4]		;;		MemR5
	vaddpd	zmm24, zmm24, [srcreg+r9+d4]	;;		MemR5
	vmulpd	zmm3, zmm10, zmm24		;;		A5 = R5 * MemR5					; 7-10		n 12
	vmulpd	zmm2, zmm7, zmm24		;;		B5 = I5 * MemR5					; 7-10		n 13

	vmovapd zmm24, [srcreg+rbp+d2+d1+64]	;;		MemI4
	vaddpd	zmm24, zmm24, [srcreg+r9+d2+d1+64];;		MemI4
	zfnmaddpd zmm21, zmm12, zmm24, zmm21	;;		A4 - I4 * MemI4 (R4 & R6)			; 8-11		n 19
	zfmaddpd zmm20, zmm9, zmm24, zmm20	;;		B4 + R4 * MemI4 (I4 & I6)			; 8-11		n 20
	vmovsd	Q [srcreg+r8-16], xmm23		;;				Save product of FFT sums	; 8

	vmovapd	zmm24, [srcreg+rbp+d4+d1]	;;		MemR6
	vaddpd	zmm24, zmm24, [srcreg+r9+d4+d1];;		MemR6
	vmulpd	zmm15, zmm6, zmm24		;;		A6 = R6 * MemR6					; 9-12		n 14
	vmulpd	zmm13, zmm8, zmm24		;;		B6 = I6 * MemR6					; 9-12		n 15

	vmovapd	zmm12, zmm17			;;		Copy R2
	vmovapd zmm24, [srcreg+rbp+64]		;; MemI1			MemR1b
	vaddpd	zmm24, zmm24, [srcreg+r9+64]	;; MemI1			MemR1b
	zfnmaddpd zmm23 {k7}, zmm11, zmm24, zmm23;; A1 - I1 * MemI1 (R1)	R1a				; 10-13		n 16
	zfmaddpd zmm22 {k7}{z}, zmm1, zmm24, zmm22;; B1 + R1 * MemI1 (I1)	0				; 10-13		n 18

	vmulpd	zmm12 {k6}, zmm11, zmm24	;; R2				R1b * MemR1b (R1b)		; 11-14		n 16

	vmovapd	zmm24, [srcreg+rbp+d4+d2]	;;		MemR7
	vaddpd	zmm24, zmm24, [srcreg+r9+d4+d2];;		MemR7
	vmulpd	zmm9, zmm14, zmm24		;;		A7 = R7 * MemR7					; 11-14		n 15
	vmulpd	zmm1, zmm4, zmm24		;;		B7 = I7 * MemR7					; 12-15		n 16

	vmovapd zmm24, [srcreg+rbp+d4+64]	;;		MemI5
	vaddpd	zmm24, zmm24, [srcreg+r9+d4+64];;		MemI5
	zfnmaddpd zmm3, zmm7, zmm24, zmm3	;;		A5 - I5 * MemI5 (R5 & R9)			; 12-15		n 21
	zfmaddpd zmm2, zmm10, zmm24, zmm2	;;		B5 + R5 * MemI5 (I5 & I9)			; 13-16		n 22

	vmovapd	zmm24, [srcreg+rbp+d4+d2+d1]	;;		MemR8
	vaddpd	zmm24, zmm24, [srcreg+r9+d4+d2+d1];;		MemR8
	vmulpd	zmm11, zmm0, zmm24		;;		A8 = R8 * MemR8					; 13-16		n 17
	vmulpd	zmm7, zmm5, zmm24		;;		B8 = I8 * MemR8					; 14-17		n 18

	vmovapd zmm24, [srcreg+rbp+d4+d1+64]	;;		MemI6
	vaddpd	zmm24, zmm24, [srcreg+r9+d4+d1+64];;		MemI6
	zfnmaddpd zmm15, zmm8, zmm24, zmm15	;;		A6 - I6 * MemI6 (R6 & R10)			; 14-17		n 21
	zfmaddpd zmm13, zmm6, zmm24, zmm13	;;		B6 + R6 * MemI6 (I6 & I10)			; 15-18		n 22

	vmovapd zmm24, [srcreg+rbp+d4+d2+64]	;;		MemI7
	vaddpd	zmm24, zmm24, [srcreg+r9+d4+d2+64];;		MemI7
	zfnmaddpd zmm9, zmm4, zmm24, zmm9	;;		A7 - I7 * MemI7 (R7 & R11)			; 15-18		n 23
	zfmaddpd zmm1, zmm14, zmm24, zmm1	;;		B7 + R7 * MemI7 (I7 & I11)			; 16-19		n 24

						;;				R1a/R1b becomes R1/R2
	vaddpd	zmm10, zmm23, zmm12		;; R1 + R2 (new R1)		R1 + R2 (new R1/HALF)		; 16-19		n 26
	vsubpd	zmm23, zmm23, zmm12		;; R1 - R2 (new R2)		R1 - R2 (new R2/HALF)		; 17-20		n 28

	vmovapd zmm24, [srcreg+rbp+d4+d2+d1+64] ;;		MemI8
	vaddpd	zmm24, zmm24, [srcreg+r9+d4+d2+d1+64];;		MemI8
	zfnmaddpd zmm11, zmm5, zmm24, zmm11	;;		A8 - I8 * MemI8 (R8 & R12)			; 17-20		n 23
	zfmaddpd zmm7, zmm0, zmm24, zmm7	;;		B8 + R8 * MemI8 (I8 & I12)			; 18-21		n 24
	jmp	back_to_orig

addmul313:
addmul133:
	vaddpd	zmm24, zmm2, [srcreg+rbp+d1]	;;		R2+MemR2
	vmulpd	zmm17, zmm2, zmm24		;;		A2 = R2 * MemR2					; 1-4		n 5
	vmulpd	zmm16, zmm3, zmm24		;;		B2 = I2 * MemR2					; 1-4		n 5

	vaddpd	zmm24, zmm13, [srcreg+rbp+d2]	;;		R3+MemR3
	vmulpd	zmm19, zmm13, zmm24		;;		A3 = R3 * MemR3					; 2-5		n 6
	vmulpd	zmm18, zmm15, zmm24		;;		B3 = I3 * MemR3					; 2-5		n 6

	vaddpd	zmm24, zmm9, [srcreg+rbp+d2+d1]	;;		R4+MemR4
	vmulpd	zmm21, zmm9, zmm24		;;		A4 = R4 * MemR4					; 3-6		n 8
	vmulpd	zmm20, zmm12, zmm24		;;		B4 = I4 * MemR4					; 3-6		n 8

	vaddpd	zmm24, zmm1, [srcreg+rbp]	;; R1+MemR1			R1a+MemR1a
	vmulpd	zmm23, zmm1, zmm24		;; A1 = R1 * MemR1		R1a*MemR1a (R1a)		; 4-7		n 10
	vmulpd	zmm22 {k7}{z}, zmm11, zmm24	;; B1 = I1 * MemR1		0				; 4-7		n 10

	vaddpd	zmm24, zmm3, [srcreg+rbp+d1+64]	;;		I2+MemI2
	zfnmaddpd zmm17, zmm3, zmm24, zmm17	;;		A2 - I2 * MemI2 (R2)				; 5-8		n 10
	zfmaddpd zmm16, zmm2, zmm24, zmm16	;;		B2 + R2 * MemI2 (I2)				; 5-8		n 18

	vaddpd	zmm24, zmm15, [srcreg+rbp+d2+64];;		I3+MemI3
	zfnmaddpd zmm19, zmm15, zmm24, zmm19	;;		A3 - I3 * MemI3 (R3 & R5)			; 6-9		n 19
	zfmaddpd zmm18, zmm13, zmm24, zmm18	;;		B3 + R3 * MemI3 (I3 & I5)			; 6-9		n 20

	vaddpd	zmm24, zmm10, [srcreg+rbp+d4]	;;		R5+MemR5
	vmulpd	zmm3, zmm10, zmm24		;;		A5 = R5 * MemR5					; 7-10		n 12
	vmulpd	zmm2, zmm7, zmm24		;;		B5 = I5 * MemR5					; 7-10		n 13

	vaddpd	zmm24, zmm12, [srcreg+rbp+d2+d1+64];;		I4+MemI4
	zfnmaddpd zmm21, zmm12, zmm24, zmm21	;;		A4 - I4 * MemI4 (R4 & R6)			; 8-11		n 19
	zfmaddpd zmm20, zmm9, zmm24, zmm20	;;		B4 + R4 * MemI4 (I4 & I6)			; 8-11		n 20
	vmovsd	Q [srcreg+r8-16], xmm23		;;				Save product of FFT sums	; 8

	vaddpd	zmm24, zmm6, [srcreg+rbp+d4+d1]	;;		R6+MemR6
	vmulpd	zmm15, zmm6, zmm24		;;		A6 = R6 * MemR6					; 9-12		n 14
	vmulpd	zmm13, zmm8, zmm24		;;		B6 = I6 * MemR6					; 9-12		n 15

	vmovapd	zmm12, zmm17			;;		Copy R2
	vaddpd	zmm24, zmm11, [srcreg+rbp+64]	;; I1+MemI1			R1b+MemR1b
	zfnmaddpd zmm23 {k7}, zmm11, zmm24, zmm23;; A1 - I1 * MemI1 (R1)	R1a				; 10-13		n 16
	zfmaddpd zmm22 {k7}{z}, zmm1, zmm24, zmm22;; B1 + R1 * MemI1 (I1)	0				; 10-13		n 18

	vmulpd	zmm12 {k6}, zmm11, zmm24	;; R2				R1b * MemR1b (R1b)		; 11-14		n 16

	vaddpd	zmm24, zmm14, [srcreg+rbp+d4+d2];;		R7+MemR7
	vmulpd	zmm9, zmm14, zmm24		;;		A7 = R7 * MemR7					; 11-14		n 15
	vmulpd	zmm1, zmm4, zmm24		;;		B7 = I7 * MemR7					; 12-15		n 16

	vaddpd	zmm24, zmm7, [srcreg+rbp+d4+64]	;;		I5+MemI5
	zfnmaddpd zmm3, zmm7, zmm24, zmm3	;;		A5 - I5 * MemI5 (R5 & R9)			; 12-15		n 21
	zfmaddpd zmm2, zmm10, zmm24, zmm2	;;		B5 + R5 * MemI5 (I5 & I9)			; 13-16		n 22

	vaddpd	zmm24, zmm0, [srcreg+rbp+d4+d2+d1];;		R8+MemR8
	vmulpd	zmm11, zmm0, zmm24		;;		A8 = R8 * MemR8					; 13-16		n 17
	vmulpd	zmm7, zmm5, zmm24		;;		B8 = I8 * MemR8					; 14-17		n 18

	vaddpd	zmm24, zmm8, [srcreg+rbp+d4+d1+64];;		I6+MemI6
	zfnmaddpd zmm15, zmm8, zmm24, zmm15	;;		A6 - I6 * MemI6 (R6 & R10)			; 14-17		n 21
	zfmaddpd zmm13, zmm6, zmm24, zmm13	;;		B6 + R6 * MemI6 (I6 & I10)			; 15-18		n 22

	vaddpd	zmm24, zmm4, [srcreg+rbp+d4+d2+64];;		I7+MemI7
	zfnmaddpd zmm9, zmm4, zmm24, zmm9	;;		A7 - I7 * MemI7 (R7 & R11)			; 15-18		n 23
	zfmaddpd zmm1, zmm14, zmm24, zmm1	;;		B7 + R7 * MemI7 (I7 & I11)			; 16-19		n 24

						;;				R1a/R1b becomes R1/R2
	vaddpd	zmm10, zmm23, zmm12		;; R1 + R2 (new R1)		R1 + R2 (new R1/HALF)		; 16-19		n 26
	vsubpd	zmm23, zmm23, zmm12		;; R1 - R2 (new R2)		R1 - R2 (new R2/HALF)		; 17-20		n 28

	vaddpd	zmm24, zmm5, [srcreg+rbp+d4+d2+d1+64];;		I8+MemI8
	zfnmaddpd zmm11, zmm5, zmm24, zmm11	;;		A8 - I8 * MemI8 (R8 & R12)			; 17-20		n 23
	zfmaddpd zmm7, zmm0, zmm24, zmm7	;;		B8 + R8 * MemI8 (I8 & I12)			; 18-21		n 24
	jmp	back_to_orig

addmul132:
addmul312:
	vaddpd	zmm24, zmm2, [srcreg+rbp+d1]	;;		R2+MemR2
	vmovapd	zmm2, [srcreg+r9+d1]		;;		R2=MemR2
	vaddpd	zmm16, zmm3, [srcreg+rbp+d1+64]	;;		I2+MemI2
	vmovapd	zmm3, [srcreg+r9+d1+64]		;;		I2=MemI2
	vmulpd	zmm17, zmm2, zmm24		;;		A2 = R2 * MemR2					; 1-4		n 5
	vmulpd	zmm24, zmm3, zmm24		;;		B2 = I2 * MemR2					; 1-4		n 5
	zfnmaddpd zmm17, zmm3, zmm16, zmm17	;;		A2 - I2 * MemI2 (R2)				; 5-8		n 10
	zfmaddpd zmm16, zmm2, zmm16, zmm24	;;		B2 + R2 * MemI2 (I2)				; 5-8		n 18

	vaddpd	zmm24, zmm13, [srcreg+rbp+d2]	;;		R3+MemR3
 	vmovapd	zmm13, [srcreg+r9+d2]		;;		R3=MemR3
	vaddpd	zmm18, zmm15, [srcreg+rbp+d2+64];;		I3+MemI3
	vmovapd	zmm15, [srcreg+r9+d2+64]	;;		I3=MemI3
	vmulpd	zmm19, zmm13, zmm24		;;		A3 = R3 * MemR3					; 2-5		n 6
	vmulpd	zmm24, zmm15, zmm24		;;		B3 = I3 * MemR3					; 2-5		n 6
	zfnmaddpd zmm19, zmm15, zmm18, zmm19	;;		A3 - I3 * MemI3 (R3 & R5)			; 6-9		n 19
	zfmaddpd zmm18, zmm13, zmm18, zmm24	;;		B3 + R3 * MemI3 (I3 & I5)			; 6-9		n 20

	vaddpd	zmm24, zmm9, [srcreg+rbp+d2+d1]	;;		R4+MemR4
	vmovapd	zmm9, [srcreg+r9+d2+d1]		;;		R4=MemR4
	vaddpd	zmm20, zmm12, [srcreg+rbp+d2+d1+64];;		I4+MemI4
	vmovapd	zmm12, [srcreg+r9+d2+d1+64]	;;		I4=MemI4
	vmulpd	zmm21, zmm9, zmm24		;;		A4 = R4 * MemR4					; 3-6		n 8
	vmulpd	zmm24, zmm12, zmm24		;;		B4 = I4 * MemR4					; 3-6		n 8
	zfnmaddpd zmm21, zmm12, zmm20, zmm21	;;		A4 - I4 * MemI4 (R4 & R6)			; 8-11		n 19
	zfmaddpd zmm20, zmm9, zmm20, zmm24	;;		B4 + R4 * MemI4 (I4 & I6)			; 8-11		n 20

	vaddpd	zmm24, zmm1, [srcreg+rbp]	;; R1+MemR1			R1a+MemR1a
	vmovapd	zmm1, [srcreg+r9]		;; R1=MemR1			R1a=MemR1a
	vaddpd	zmm9, zmm11, [srcreg+rbp+64]	;; I1+MemI1			R1b+MemR1b
	vmovapd	zmm11, [srcreg+r9+64]		;; I1=MemI1			R1b=MemR1b
	vmulpd	zmm23, zmm1, zmm24		;; A1 = R1 * MemR1		R1a*MemR1a (R1a)		; 4-7		n 10
	vmulpd	zmm22 {k7}{z}, zmm11, zmm24	;; B1 = I1 * MemR1		0				; 4-7		n 10
	zfnmaddpd zmm23 {k7}, zmm11, zmm9, zmm23;; A1 - I1 * MemI1 (R1)	R1a					; 10-13		n 16
	zfmaddpd zmm22 {k7}{z}, zmm1, zmm9, zmm22;; B1 + R1 * MemI1 (I1)	0				; 10-13		n 18
	vmovapd	zmm12, zmm17			;;		Copy R2
	vmulpd	zmm12 {k6}, zmm11, zmm9		;; R2				R1b * MemR1b (R1b)		; 11-14		n 16
	vmovsd	Q [srcreg+r8-16], xmm23		;;				Save product of FFT sums	; 8

	vaddpd	zmm24, zmm10, [srcreg+rbp+d4]	;;		R5+MemR5
	vmovapd	zmm10, [srcreg+r9+d4]		;;		R5=MemR5
	vaddpd	zmm2, zmm7, [srcreg+rbp+d4+64]	;;		I5+MemI5
	vmovapd	zmm7, [srcreg+r9+d4+64]		;;		I5=MemI5
	vmulpd	zmm3, zmm10, zmm24		;;		A5 = R5 * MemR5					; 7-10		n 12
	vmulpd	zmm24, zmm7, zmm24		;;		B5 = I5 * MemR5					; 7-10		n 13
	zfnmaddpd zmm3, zmm7, zmm2, zmm3	;;		A5 - I5 * MemI5 (R5 & R9)			; 12-15		n 21
	zfmaddpd zmm2, zmm10, zmm2, zmm24	;;		B5 + R5 * MemI5 (I5 & I9)			; 13-16		n 22

	vaddpd	zmm24, zmm6, [srcreg+rbp+d4+d1]	;;		R6+MemR6
	vmovapd	zmm6, [srcreg+r9+d4+d1]		;;		R6=MemR6
	vaddpd	zmm13, zmm8, [srcreg+rbp+d4+d1+64];;		I6+MemI6
	vmovapd	zmm8, [srcreg+r9+d4+d1+64]	;;		I6=MemI6
	vmulpd	zmm15, zmm6, zmm24		;;		A6 = R6 * MemR6					; 9-12		n 14
	vmulpd	zmm24, zmm8, zmm24		;;		B6 = I6 * MemR6					; 9-12		n 15
	zfnmaddpd zmm15, zmm8, zmm13, zmm15	;;		A6 - I6 * MemI6 (R6 & R10)			; 14-17		n 21
	zfmaddpd zmm13, zmm6, zmm13, zmm24	;;		B6 + R6 * MemI6 (I6 & I10)			; 15-18		n 22

	vaddpd	zmm24, zmm14, [srcreg+rbp+d4+d2];;		R7+MemR7
	vmovapd	zmm14, [srcreg+r9+d4+d2]	;;		R7=MemR7
	vaddpd	zmm1, zmm4, [srcreg+rbp+d4+d2+64];;		I7+MemI7
	vmovapd	zmm4, [srcreg+r9+d4+d2+64]	;;		I7=MemI7
	vmulpd	zmm9, zmm14, zmm24		;;		A7 = R7 * MemR7					; 11-14		n 15
	vmulpd	zmm24, zmm4, zmm24		;;		B7 = I7 * MemR7					; 12-15		n 16
	zfnmaddpd zmm9, zmm4, zmm1, zmm9	;;		A7 - I7 * MemI7 (R7 & R11)			; 15-18		n 23
	zfmaddpd zmm1, zmm14, zmm1, zmm24	;;		B7 + R7 * MemI7 (I7 & I11)			; 16-19		n 24

	vaddpd	zmm24, zmm0, [srcreg+rbp+d4+d2+d1];;		R8+MemR8
	vmovapd	zmm0, [srcreg+r9+d4+d2+d1]	;;		R8=MemR8
	vaddpd	zmm7, zmm5, [srcreg+rbp+d4+d2+d1+64];;		I8+MemI8
	vmovapd	zmm5, [srcreg+r9+d4+d2+d1+64]	;;		I8=MemI8
	vmulpd	zmm11, zmm0, zmm24		;;		A8 = R8 * MemR8					; 13-16		n 17
	vmulpd	zmm24, zmm5, zmm24		;;		B8 = I8 * MemR8					; 14-17		n 18
	zfnmaddpd zmm11, zmm5, zmm7, zmm11	;;		A8 - I8 * MemI8 (R8 & R12)			; 17-20		n 23
	zfmaddpd zmm7, zmm0, zmm7, zmm24	;;		B8 + R8 * MemI8 (I8 & I12)			; 18-21		n 24

						;;				R1a/R1b becomes R1/R2
	vaddpd	zmm10, zmm23, zmm12		;; R1 + R2 (new R1)		R1 + R2 (new R1/HALF)		; 16-19		n 26
	vsubpd	zmm23, zmm23, zmm12		;; R1 - R2 (new R2)		R1 - R2 (new R2/HALF)		; 17-20		n 28
	jmp	back_to_orig

submul:	vmovapd	zmm24, [srcreg+rbp+d1]		;;		MemR2
	vsubpd	zmm24, zmm24, [srcreg+r9+d1]	;;		MemR2
	vmulpd	zmm17, zmm2, zmm24		;;		A2 = R2 * MemR2					; 1-4		n 5
	vmulpd	zmm16, zmm3, zmm24		;;		B2 = I2 * MemR2					; 1-4		n 5

	vmovapd	zmm24, [srcreg+rbp+d2]		;;		MemR3
	vsubpd	zmm24, zmm24, [srcreg+r9+d2]	;;		MemR3
	vmulpd	zmm19, zmm13, zmm24		;;		A3 = R3 * MemR3					; 2-5		n 6
	vmulpd	zmm18, zmm15, zmm24		;;		B3 = I3 * MemR3					; 2-5		n 6

	vmovapd	zmm24, [srcreg+rbp+d2+d1]	;;		MemR4
	vsubpd	zmm24, zmm24, [srcreg+r9+d2+d1];;		MemR4
	vmulpd	zmm21, zmm9, zmm24		;;		A4 = R4 * MemR4					; 3-6		n 8
	vmulpd	zmm20, zmm12, zmm24		;;		B4 = I4 * MemR4					; 3-6		n 8

	vmovapd	zmm24, [srcreg+rbp]		;; MemR1			MemR1a
	vsubpd	zmm24, zmm24, [srcreg+r9]	;; MemR1			MemR1a
	vmulpd	zmm23, zmm1, zmm24		;; A1 = R1 * MemR1		R1a*MemR1a (R1a)		; 4-7		n 10
	vmulpd	zmm22 {k7}{z}, zmm11, zmm24	;; B1 = I1 * MemR1		0				; 4-7		n 10

	vmovapd zmm24, [srcreg+rbp+d1+64]	;;		MemI2
	vsubpd	zmm24, zmm24, [srcreg+r9+d1+64];;		MemI2
	zfnmaddpd zmm17, zmm3, zmm24, zmm17	;;		A2 - I2 * MemI2 (R2)				; 5-8		n 10
	zfmaddpd zmm16, zmm2, zmm24, zmm16	;;		B2 + R2 * MemI2 (I2)				; 5-8		n 18

	vmovapd zmm24, [srcreg+rbp+d2+64]	;;		MemI3
	vsubpd	zmm24, zmm24, [srcreg+r9+d2+64];;		MemI3
	zfnmaddpd zmm19, zmm15, zmm24, zmm19	;;		A3 - I3 * MemI3 (R3 & R5)			; 6-9		n 19
	zfmaddpd zmm18, zmm13, zmm24, zmm18	;;		B3 + R3 * MemI3 (I3 & I5)			; 6-9		n 20

	vmovapd	zmm24, [srcreg+rbp+d4]		;;		MemR5
	vsubpd	zmm24, zmm24, [srcreg+r9+d4]	;;		MemR5
	vmulpd	zmm3, zmm10, zmm24		;;		A5 = R5 * MemR5					; 7-10		n 12
	vmulpd	zmm2, zmm7, zmm24		;;		B5 = I5 * MemR5					; 7-10		n 13

	vmovapd zmm24, [srcreg+rbp+d2+d1+64]	;;		MemI4
	vsubpd	zmm24, zmm24, [srcreg+r9+d2+d1+64];;		MemI4
	zfnmaddpd zmm21, zmm12, zmm24, zmm21	;;		A4 - I4 * MemI4 (R4 & R6)			; 8-11		n 19
	zfmaddpd zmm20, zmm9, zmm24, zmm20	;;		B4 + R4 * MemI4 (I4 & I6)			; 8-11		n 20
	vmovsd	Q [srcreg+r8-16], xmm23		;;				Save product of FFT sums	; 8

	vmovapd	zmm24, [srcreg+rbp+d4+d1]	;;		MemR6
	vsubpd	zmm24, zmm24, [srcreg+r9+d4+d1];;		MemR6
	vmulpd	zmm15, zmm6, zmm24		;;		A6 = R6 * MemR6					; 9-12		n 14
	vmulpd	zmm13, zmm8, zmm24		;;		B6 = I6 * MemR6					; 9-12		n 15

	vmovapd	zmm12, zmm17			;;		Copy R2
	vmovapd zmm24, [srcreg+rbp+64]		;; MemI1			MemR1b
	vsubpd	zmm24, zmm24, [srcreg+r9+64]	;; MemI1			MemR1b
	zfnmaddpd zmm23 {k7}, zmm11, zmm24, zmm23;; A1 - I1 * MemI1 (R1)	R1a				; 10-13		n 16
	zfmaddpd zmm22 {k7}{z}, zmm1, zmm24, zmm22;; B1 + R1 * MemI1 (I1)	0				; 10-13		n 18

	vmulpd	zmm12 {k6}, zmm11, zmm24	;; R2				R1b * MemR1b (R1b)		; 11-14		n 16

	vmovapd	zmm24, [srcreg+rbp+d4+d2]	;;		MemR7
	vsubpd	zmm24, zmm24, [srcreg+r9+d4+d2];;		MemR7
	vmulpd	zmm9, zmm14, zmm24		;;		A7 = R7 * MemR7					; 11-14		n 15
	vmulpd	zmm1, zmm4, zmm24		;;		B7 = I7 * MemR7					; 12-15		n 16

	vmovapd zmm24, [srcreg+rbp+d4+64]	;;		MemI5
	vsubpd	zmm24, zmm24, [srcreg+r9+d4+64];;		MemI5
	zfnmaddpd zmm3, zmm7, zmm24, zmm3	;;		A5 - I5 * MemI5 (R5 & R9)			; 12-15		n 21
	zfmaddpd zmm2, zmm10, zmm24, zmm2	;;		B5 + R5 * MemI5 (I5 & I9)			; 13-16		n 22

	vmovapd	zmm24, [srcreg+rbp+d4+d2+d1]	;;		MemR8
	vsubpd	zmm24, zmm24, [srcreg+r9+d4+d2+d1];;		MemR8
	vmulpd	zmm11, zmm0, zmm24		;;		A8 = R8 * MemR8					; 13-16		n 17
	vmulpd	zmm7, zmm5, zmm24		;;		B8 = I8 * MemR8					; 14-17		n 18

	vmovapd zmm24, [srcreg+rbp+d4+d1+64]	;;		MemI6
	vsubpd	zmm24, zmm24, [srcreg+r9+d4+d1+64];;		MemI6
	zfnmaddpd zmm15, zmm8, zmm24, zmm15	;;		A6 - I6 * MemI6 (R6 & R10)			; 14-17		n 21
	zfmaddpd zmm13, zmm6, zmm24, zmm13	;;		B6 + R6 * MemI6 (I6 & I10)			; 15-18		n 22

	vmovapd zmm24, [srcreg+rbp+d4+d2+64]	;;		MemI7
	vsubpd	zmm24, zmm24, [srcreg+r9+d4+d2+64];;		MemI7
	zfnmaddpd zmm9, zmm4, zmm24, zmm9	;;		A7 - I7 * MemI7 (R7 & R11)			; 15-18		n 23
	zfmaddpd zmm1, zmm14, zmm24, zmm1	;;		B7 + R7 * MemI7 (I7 & I11)			; 16-19		n 24

						;;				R1a/R1b becomes R1/R2
	vaddpd	zmm10, zmm23, zmm12		;; R1 + R2 (new R1)		R1 + R2 (new R1/HALF)		; 16-19		n 26
	vsubpd	zmm23, zmm23, zmm12		;; R1 - R2 (new R2)		R1 - R2 (new R2/HALF)		; 17-20		n 28

	vmovapd zmm24, [srcreg+rbp+d4+d2+d1+64] ;;		MemI8
	vsubpd	zmm24, zmm24, [srcreg+r9+d4+d2+d1+64];;		MemI8
	zfnmaddpd zmm11, zmm5, zmm24, zmm11	;;		A8 - I8 * MemI8 (R8 & R12)			; 17-20		n 23
	zfmaddpd zmm7, zmm0, zmm24, zmm7	;;		B8 + R8 * MemI8 (I8 & I12)			; 18-21		n 24
	jmp	back_to_orig

submul313:
	vsubpd	zmm24, zmm2, [srcreg+rbp+d1]	;;		R2-MemR2
	vmulpd	zmm17, zmm2, zmm24		;;		A2 = R2 * MemR2					; 1-4		n 5
	vmulpd	zmm16, zmm3, zmm24		;;		B2 = I2 * MemR2					; 1-4		n 5

	vsubpd	zmm24, zmm13, [srcreg+rbp+d2]	;;		R3-MemR3
	vmulpd	zmm19, zmm13, zmm24		;;		A3 = R3 * MemR3					; 2-5		n 6
	vmulpd	zmm18, zmm15, zmm24		;;		B3 = I3 * MemR3					; 2-5		n 6

	vsubpd	zmm24, zmm9, [srcreg+rbp+d2+d1]	;;		R4-MemR4
	vmulpd	zmm21, zmm9, zmm24		;;		A4 = R4 * MemR4					; 3-6		n 8
	vmulpd	zmm20, zmm12, zmm24		;;		B4 = I4 * MemR4					; 3-6		n 8

	vsubpd	zmm24, zmm1, [srcreg+rbp]	;; R1-MemR1			R1a-MemR1a
	vmulpd	zmm23, zmm1, zmm24		;; A1 = R1 * MemR1		R1a*MemR1a (R1a)		; 4-7		n 10
	vmulpd	zmm22 {k7}{z}, zmm11, zmm24	;; B1 = I1 * MemR1		0				; 4-7		n 10

	vsubpd	zmm24, zmm3, [srcreg+rbp+d1+64]	;;		I2-MemI2
	zfnmaddpd zmm17, zmm3, zmm24, zmm17	;;		A2 - I2 * MemI2 (R2)				; 5-8		n 10
	zfmaddpd zmm16, zmm2, zmm24, zmm16	;;		B2 + R2 * MemI2 (I2)				; 5-8		n 18

	vsubpd	zmm24, zmm15, [srcreg+rbp+d2+64];;		I3-MemI3
	zfnmaddpd zmm19, zmm15, zmm24, zmm19	;;		A3 - I3 * MemI3 (R3 & R5)			; 6-9		n 19
	zfmaddpd zmm18, zmm13, zmm24, zmm18	;;		B3 + R3 * MemI3 (I3 & I5)			; 6-9		n 20

	vsubpd	zmm24, zmm10, [srcreg+rbp+d4]	;;		R5-MemR5
	vmulpd	zmm3, zmm10, zmm24		;;		A5 = R5 * MemR5					; 7-10		n 12
	vmulpd	zmm2, zmm7, zmm24		;;		B5 = I5 * MemR5					; 7-10		n 13

	vsubpd	zmm24, zmm12, [srcreg+rbp+d2+d1+64];;		I4-MemI4
	zfnmaddpd zmm21, zmm12, zmm24, zmm21	;;		A4 - I4 * MemI4 (R4 & R6)			; 8-11		n 19
	zfmaddpd zmm20, zmm9, zmm24, zmm20	;;		B4 + R4 * MemI4 (I4 & I6)			; 8-11		n 20
	vmovsd	Q [srcreg+r8-16], xmm23		;;				Save product of FFT sums	; 8

	vsubpd	zmm24, zmm6, [srcreg+rbp+d4+d1];;		R6-MemR6
	vmulpd	zmm15, zmm6, zmm24		;;		A6 = R6 * MemR6					; 9-12		n 14
	vmulpd	zmm13, zmm8, zmm24		;;		B6 = I6 * MemR6					; 9-12		n 15

	vmovapd	zmm12, zmm17			;;		Copy R2
	vsubpd	zmm24, zmm11, [srcreg+rbp+64]	;; I1-MemI1			R1b-MemR1b
	zfnmaddpd zmm23 {k7}, zmm11, zmm24, zmm23;; A1 - I1 * MemI1 (R1)	R1a				; 10-13		n 16
	zfmaddpd zmm22 {k7}{z}, zmm1, zmm24, zmm22;; B1 + R1 * MemI1 (I1)	0				; 10-13		n 18

	vmulpd	zmm12 {k6}, zmm11, zmm24	;; R2				R1b * MemR1b (R1b)		; 11-14		n 16

	vsubpd	zmm24, zmm14, [srcreg+rbp+d4+d2];;		R7-MemR7
	vmulpd	zmm9, zmm14, zmm24		;;		A7 = R7 * MemR7					; 11-14		n 15
	vmulpd	zmm1, zmm4, zmm24		;;		B7 = I7 * MemR7					; 12-15		n 16

	vsubpd	zmm24, zmm7, [srcreg+rbp+d4+64];;		I5-MemI5
	zfnmaddpd zmm3, zmm7, zmm24, zmm3	;;		A5 - I5 * MemI5 (R5 & R9)			; 12-15		n 21
	zfmaddpd zmm2, zmm10, zmm24, zmm2	;;		B5 + R5 * MemI5 (I5 & I9)			; 13-16		n 22

	vsubpd	zmm24, zmm0, [srcreg+rbp+d4+d2+d1];;		R8-MemR8
	vmulpd	zmm11, zmm0, zmm24		;;		A8 = R8 * MemR8					; 13-16		n 17
	vmulpd	zmm7, zmm5, zmm24		;;		B8 = I8 * MemR8					; 14-17		n 18

	vsubpd	zmm24, zmm8, [srcreg+rbp+d4+d1+64];;		I6-MemI6
	zfnmaddpd zmm15, zmm8, zmm24, zmm15	;;		A6 - I6 * MemI6 (R6 & R10)			; 14-17		n 21
	zfmaddpd zmm13, zmm6, zmm24, zmm13	;;		B6 + R6 * MemI6 (I6 & I10)			; 15-18		n 22

	vsubpd	zmm24, zmm4, [srcreg+rbp+d4+d2+64];;		I7-MemI7
	zfnmaddpd zmm9, zmm4, zmm24, zmm9	;;		A7 - I7 * MemI7 (R7 & R11)			; 15-18		n 23
	zfmaddpd zmm1, zmm14, zmm24, zmm1	;;		B7 + R7 * MemI7 (I7 & I11)			; 16-19		n 24

						;;				R1a/R1b becomes R1/R2
	vaddpd	zmm10, zmm23, zmm12		;; R1 + R2 (new R1)		R1 + R2 (new R1/HALF)		; 16-19		n 26
	vsubpd	zmm23, zmm23, zmm12		;; R1 - R2 (new R2)		R1 - R2 (new R2/HALF)		; 17-20		n 28

	vsubpd	zmm24, zmm5, [srcreg+rbp+d4+d2+d1+64];;		I8-MemI8
	zfnmaddpd zmm11, zmm5, zmm24, zmm11	;;		A8 - I8 * MemI8 (R8 & R12)			; 17-20		n 23
	zfmaddpd zmm7, zmm0, zmm24, zmm7	;;		B8 + R8 * MemI8 (I8 & I12)			; 18-21		n 24
	jmp	back_to_orig

submul133:
	vmovapd	zmm24, [srcreg+rbp+d1]		;;		MemR2
	vsubpd	zmm24, zmm24, zmm2		;;		MemR2-R2
	vmulpd	zmm17, zmm2, zmm24		;;		A2 = R2 * MemR2					; 1-4		n 5
	vmulpd	zmm16, zmm3, zmm24		;;		B2 = I2 * MemR2					; 1-4		n 5

	vmovapd	zmm24, [srcreg+rbp+d2]		;;		MemR3
	vsubpd	zmm24, zmm24, zmm13		;;		MemR3-R3
	vmulpd	zmm19, zmm13, zmm24		;;		A3 = R3 * MemR3					; 2-5		n 6
	vmulpd	zmm18, zmm15, zmm24		;;		B3 = I3 * MemR3					; 2-5		n 6

	vmovapd	zmm24, [srcreg+rbp+d2+d1]	;;		MemR4
	vsubpd	zmm24, zmm24, zmm9		;;		MemR4-R4
	vmulpd	zmm21, zmm9, zmm24		;;		A4 = R4 * MemR4					; 3-6		n 8
	vmulpd	zmm20, zmm12, zmm24		;;		B4 = I4 * MemR4					; 3-6		n 8

	vmovapd	zmm24, [srcreg+rbp]		;; MemR1			MemR1a
	vsubpd	zmm24, zmm24, zmm1		;; MemR1-R1			MemR1a-R1a
	vmulpd	zmm23, zmm1, zmm24		;; A1 = R1 * MemR1		R1a*MemR1a (R1a)		; 4-7		n 10
	vmulpd	zmm22 {k7}{z}, zmm11, zmm24	;; B1 = I1 * MemR1		0				; 4-7		n 10

	vmovapd	zmm24, [srcreg+rbp+d1+64]	;;		MemI2
	vsubpd	zmm24, zmm24, zmm3		;;		MemI2-I2
	zfnmaddpd zmm17, zmm3, zmm24, zmm17	;;		A2 - I2 * MemI2 (R2)				; 5-8		n 10
	zfmaddpd zmm16, zmm2, zmm24, zmm16	;;		B2 + R2 * MemI2 (I2)				; 5-8		n 18

	vmovapd	zmm24, [srcreg+rbp+d2+64]	;;		MemI3
	vsubpd	zmm24, zmm24, zmm15		;;		MemI3-I3
	zfnmaddpd zmm19, zmm15, zmm24, zmm19	;;		A3 - I3 * MemI3 (R3 & R5)			; 6-9		n 19
	zfmaddpd zmm18, zmm13, zmm24, zmm18	;;		B3 + R3 * MemI3 (I3 & I5)			; 6-9		n 20

	vmovapd	zmm24, [srcreg+rbp+d4]		;;		MemR5
	vsubpd	zmm24, zmm24, zmm10		;;		MemR5-R5
	vmulpd	zmm3, zmm10, zmm24		;;		A5 = R5 * MemR5					; 7-10		n 12
	vmulpd	zmm2, zmm7, zmm24		;;		B5 = I5 * MemR5					; 7-10		n 13

	vmovapd	zmm24, [srcreg+rbp+d2+d1+64]	;;		MemI4
	vsubpd	zmm24, zmm24, zmm12		;;		MemI4-I4
	zfnmaddpd zmm21, zmm12, zmm24, zmm21	;;		A4 - I4 * MemI4 (R4 & R6)			; 8-11		n 19
	zfmaddpd zmm20, zmm9, zmm24, zmm20	;;		B4 + R4 * MemI4 (I4 & I6)			; 8-11		n 20
	vmovsd	Q [srcreg+r8-16], xmm23		;;				Save product of FFT sums	; 8

	vmovapd	zmm24, [srcreg+rbp+d4+d1]	;;		MemR6
	vsubpd	zmm24, zmm24, zmm6		;;		MemR6-R6
	vmulpd	zmm15, zmm6, zmm24		;;		A6 = R6 * MemR6					; 9-12		n 14
	vmulpd	zmm13, zmm8, zmm24		;;		B6 = I6 * MemR6					; 9-12		n 15

	vmovapd	zmm12, zmm17			;;		Copy R2
	vmovapd	zmm24, [srcreg+rbp+64]		;; MemI1			MemR1b
	vsubpd	zmm24, zmm24, zmm11		;; MemI1-I1			MemR1b-R1b
	zfnmaddpd zmm23 {k7}, zmm11, zmm24, zmm23;; A1 - I1 * MemI1 (R1)	R1a				; 10-13		n 16
	zfmaddpd zmm22 {k7}{z}, zmm1, zmm24, zmm22;; B1 + R1 * MemI1 (I1)	0				; 10-13		n 18

	vmulpd	zmm12 {k6}, zmm11, zmm24	;; R2				R1b * MemR1b (R1b)		; 11-14		n 16

	vmovapd	zmm24, [srcreg+rbp+d4+d2]	;;		MemR7
	vsubpd	zmm24, zmm24, zmm14		;;		MemR7-R7
	vmulpd	zmm9, zmm14, zmm24		;;		A7 = R7 * MemR7					; 11-14		n 15
	vmulpd	zmm1, zmm4, zmm24		;;		B7 = I7 * MemR7					; 12-15		n 16

	vmovapd	zmm24, [srcreg+rbp+d4+64]	;;		MemI5
	vsubpd	zmm24, zmm24, zmm7		;;		MemI5-I5
	zfnmaddpd zmm3, zmm7, zmm24, zmm3	;;		A5 - I5 * MemI5 (R5 & R9)			; 12-15		n 21
	zfmaddpd zmm2, zmm10, zmm24, zmm2	;;		B5 + R5 * MemI5 (I5 & I9)			; 13-16		n 22

	vmovapd	zmm24, [srcreg+rbp+d4+d2+d1]	;;		MemR8
	vsubpd	zmm24, zmm24, zmm0		;;		MemR8-R8
	vmulpd	zmm11, zmm0, zmm24		;;		A8 = R8 * MemR8					; 13-16		n 17
	vmulpd	zmm7, zmm5, zmm24		;;		B8 = I8 * MemR8					; 14-17		n 18

	vmovapd	zmm24, [srcreg+rbp+d4+d1+64]	;;		MemI6
	vsubpd	zmm24, zmm24, zmm8		;;		MemI6-I6
	zfnmaddpd zmm15, zmm8, zmm24, zmm15	;;		A6 - I6 * MemI6 (R6 & R10)			; 14-17		n 21
	zfmaddpd zmm13, zmm6, zmm24, zmm13	;;		B6 + R6 * MemI6 (I6 & I10)			; 15-18		n 22

	vmovapd	zmm24, [srcreg+rbp+d4+d2+64]	;;		MemI7
	vsubpd	zmm24, zmm24, zmm4		;;		MemI7-I7
	zfnmaddpd zmm9, zmm4, zmm24, zmm9	;;		A7 - I7 * MemI7 (R7 & R11)			; 15-18		n 23
	zfmaddpd zmm1, zmm14, zmm24, zmm1	;;		B7 + R7 * MemI7 (I7 & I11)			; 16-19		n 24

						;;				R1a/R1b becomes R1/R2
	vaddpd	zmm10, zmm23, zmm12		;; R1 + R2 (new R1)		R1 + R2 (new R1/HALF)		; 16-19		n 26
	vsubpd	zmm23, zmm23, zmm12		;; R1 - R2 (new R2)		R1 - R2 (new R2/HALF)		; 17-20		n 28

	vmovapd	zmm24, [srcreg+rbp+d4+d2+d1+64]	;;		MemI8
	vsubpd	zmm24, zmm24, zmm5		;;		MemI8-I8
	zfnmaddpd zmm11, zmm5, zmm24, zmm11	;;		A8 - I8 * MemI8 (R8 & R12)			; 17-20		n 23
	zfmaddpd zmm7, zmm0, zmm24, zmm7	;;		B8 + R8 * MemI8 (I8 & I12)			; 18-21		n 24
	jmp	back_to_orig

submul312:
	vsubpd	zmm24, zmm2, [srcreg+rbp+d1]	;;		R2-MemR2
	vmovapd	zmm2, [srcreg+r9+d1]		;;		R2=MemR2
	vsubpd	zmm16, zmm3, [srcreg+rbp+d1+64]	;;		I2-MemI2
	vmovapd	zmm3, [srcreg+r9+d1+64]		;;		I2=MemI2
	vmulpd	zmm17, zmm2, zmm24		;;		A2 = R2 * MemR2					; 1-4		n 5
	vmulpd	zmm24, zmm3, zmm24		;;		B2 = I2 * MemR2					; 1-4		n 5
	zfnmaddpd zmm17, zmm3, zmm16, zmm17	;;		A2 - I2 * MemI2 (R2)				; 5-8		n 10
	zfmaddpd zmm16, zmm2, zmm16, zmm24	;;		B2 + R2 * MemI2 (I2)				; 5-8		n 18

	vsubpd	zmm24, zmm13, [srcreg+rbp+d2]	;;		R3-MemR3
	vmovapd	zmm13, [srcreg+r9+d2]		;;		R3=MemR3
	vsubpd	zmm18, zmm15, [srcreg+rbp+d2+64];;		I3-MemI3
	vmovapd	zmm15, [srcreg+r9+d2+64]	;;		I3=MemI3
	vmulpd	zmm19, zmm13, zmm24		;;		A3 = R3 * MemR3					; 2-5		n 6
	vmulpd	zmm24, zmm15, zmm24		;;		B3 = I3 * MemR3					; 2-5		n 6
	zfnmaddpd zmm19, zmm15, zmm18, zmm19	;;		A3 - I3 * MemI3 (R3 & R5)			; 6-9		n 19
	zfmaddpd zmm18, zmm13, zmm18, zmm24	;;		B3 + R3 * MemI3 (I3 & I5)			; 6-9		n 20

	vsubpd	zmm24, zmm9, [srcreg+rbp+d2+d1]	;;		R4-MemR4
	vmovapd	zmm9, [srcreg+r9+d2+d1]		;;		R4=MemR4
	vsubpd	zmm20, zmm12, [srcreg+rbp+d2+d1+64];;		I4-MemI4
	vmovapd	zmm12, [srcreg+r9+d2+d1+64]	;;		I4=MemI4
	vmulpd	zmm21, zmm9, zmm24		;;		A4 = R4 * MemR4					; 3-6		n 8
	vmulpd	zmm24, zmm12, zmm24		;;		B4 = I4 * MemR4					; 3-6		n 8
	zfnmaddpd zmm21, zmm12, zmm20, zmm21	;;		A4 - I4 * MemI4 (R4 & R6)			; 8-11		n 19
	zfmaddpd zmm20, zmm9, zmm20, zmm24	;;		B4 + R4 * MemI4 (I4 & I6)			; 8-11		n 20

	vsubpd	zmm24, zmm1, [srcreg+rbp]	;; R1-MemR1			R1a-MemR1a
	vmovapd	zmm1, [srcreg+r9]		;; R1=MemR1			R1a=MemR1a
	vsubpd	zmm9, zmm11, [srcreg+rbp+64]	;; I1-MemI1			R1b-MemR1b
	vmovapd	zmm11, [srcreg+r9+64]		;; I1=MemI1			R1b=MemR1b
	vmulpd	zmm23, zmm1, zmm24		;; A1 = R1 * MemR1		R1a*MemR1a (R1a)		; 4-7		n 10
	vmulpd	zmm22 {k7}{z}, zmm11, zmm24	;; B1 = I1 * MemR1		0				; 4-7		n 10
	zfnmaddpd zmm23 {k7}, zmm11, zmm9, zmm23;; A1 - I1 * MemI1 (R1)	R1a					; 10-13		n 16
	zfmaddpd zmm22 {k7}{z}, zmm1, zmm9, zmm22;; B1 + R1 * MemI1 (I1)	0				; 10-13		n 18
	vmovapd	zmm12, zmm17			;;		Copy R2
	vmulpd	zmm12 {k6}, zmm11, zmm9		;; R2				R1b * MemR1b (R1b)		; 11-14		n 16
	vmovsd	Q [srcreg+r8-16], xmm23		;;				Save product of FFT sums	; 8

	vsubpd	zmm24, zmm10, [srcreg+rbp+d4]	;;		R5-MemR5
	vmovapd	zmm10, [srcreg+r9+d4]		;;		R5=MemR5
	vsubpd	zmm2, zmm7, [srcreg+rbp+d4+64];;		I5-MemI5
	vmovapd	zmm7, [srcreg+r9+d4+64]		;;		I5=MemI5
	vmulpd	zmm3, zmm10, zmm24		;;		A5 = R5 * MemR5					; 7-10		n 12
	vmulpd	zmm24, zmm7, zmm24		;;		B5 = I5 * MemR5					; 7-10		n 13
	zfnmaddpd zmm3, zmm7, zmm2, zmm3	;;		A5 - I5 * MemI5 (R5 & R9)			; 12-15		n 21
	zfmaddpd zmm2, zmm10, zmm2, zmm24	;;		B5 + R5 * MemI5 (I5 & I9)			; 13-16		n 22

	vsubpd	zmm24, zmm6, [srcreg+rbp+d4+d1];;		R6-MemR6
	vmovapd	zmm6, [srcreg+r9+d4+d1]		;;		R6=MemR6
	vsubpd	zmm13, zmm8, [srcreg+rbp+d4+d1+64];;		I6-MemI6
	vmovapd	zmm8, [srcreg+r9+d4+d1+64]	;;		I6=MemI6
	vmulpd	zmm15, zmm6, zmm24		;;		A6 = R6 * MemR6					; 9-12		n 14
	vmulpd	zmm24, zmm8, zmm24		;;		B6 = I6 * MemR6					; 9-12		n 15
	zfnmaddpd zmm15, zmm8, zmm13, zmm15	;;		A6 - I6 * MemI6 (R6 & R10)			; 14-17		n 21
	zfmaddpd zmm13, zmm6, zmm13, zmm24	;;		B6 + R6 * MemI6 (I6 & I10)			; 15-18		n 22

	vsubpd	zmm24, zmm14, [srcreg+rbp+d4+d2];;		R7-MemR7
	vmovapd	zmm14, [srcreg+r9+d4+d2]	;;		R7=MemR7
	vsubpd	zmm1, zmm4, [srcreg+rbp+d4+d2+64];;		I7-MemI7
	vmovapd	zmm4, [srcreg+r9+d4+d2+64]	;;		I7=MemI7
	vmulpd	zmm9, zmm14, zmm24		;;		A7 = R7 * MemR7					; 11-14		n 15
	vmulpd	zmm24, zmm4, zmm24		;;		B7 = I7 * MemR7					; 12-15		n 16
	zfnmaddpd zmm9, zmm4, zmm1, zmm9	;;		A7 - I7 * MemI7 (R7 & R11)			; 15-18		n 23
	zfmaddpd zmm1, zmm14, zmm1, zmm24	;;		B7 + R7 * MemI7 (I7 & I11)			; 16-19		n 24

	vsubpd	zmm24, zmm0, [srcreg+rbp+d4+d2+d1];;		R8-MemR8
	vmovapd	zmm0, [srcreg+r9+d4+d2+d1]	;;		R8=MemR8
	vsubpd	zmm7, zmm5, [srcreg+rbp+d4+d2+d1+64];;		I8-MemI8
	vmovapd	zmm5, [srcreg+r9+d4+d2+d1+64]	;;		I8=MemI8
	vmulpd	zmm11, zmm0, zmm24		;;		A8 = R8 * MemR8					; 13-16		n 17
	vmulpd	zmm24, zmm5, zmm24		;;		B8 = I8 * MemR8					; 14-17		n 18
	zfnmaddpd zmm11, zmm5, zmm7, zmm11	;;		A8 - I8 * MemI8 (R8 & R12)			; 17-20		n 23
	zfmaddpd zmm7, zmm0, zmm7, zmm24	;;		B8 + R8 * MemI8 (I8 & I12)			; 18-21		n 24

						;;				R1a/R1b becomes R1/R2
	vaddpd	zmm10, zmm23, zmm12		;; R1 + R2 (new R1)		R1 + R2 (new R1/HALF)		; 16-19		n 26
	vsubpd	zmm23, zmm23, zmm12		;; R1 - R2 (new R2)		R1 - R2 (new R2/HALF)		; 17-20		n 28
	jmp	back_to_orig

submul132:
	vmovapd	zmm24, [srcreg+rbp+d1]		;;		MemR2
	vsubpd	zmm24, zmm24, zmm2		;;		MemR2-R2
	vmovapd	zmm2, [srcreg+r9+d1]		;;		R2=MemR2
	vmovapd	zmm16, [srcreg+rbp+d1+64]	;;		MemI2
	vsubpd	zmm16, zmm16, zmm3		;;		MemI2-I2
	vmovapd	zmm3, [srcreg+r9+d1+64]		;;		I2=MemI2
	vmulpd	zmm17, zmm2, zmm24		;;		A2 = R2 * MemR2					; 1-4		n 5
	vmulpd	zmm24, zmm3, zmm24		;;		B2 = I2 * MemR2					; 1-4		n 5
	zfnmaddpd zmm17, zmm3, zmm16, zmm17	;;		A2 - I2 * MemI2 (R2)				; 5-8		n 10
	zfmaddpd zmm16, zmm2, zmm16, zmm24	;;		B2 + R2 * MemI2 (I2)				; 5-8		n 18

	vmovapd	zmm24, [srcreg+rbp+d2]		;;		MemR3
	vsubpd	zmm24, zmm24, zmm13		;;		MemR3-R3
	vmovapd	zmm13, [srcreg+r9+d2]		;;		R3=MemR3
	vmovapd	zmm18, [srcreg+rbp+d2+64]	;;		MemI3
	vsubpd	zmm18, zmm18, zmm15		;;		MemI3-I3
	vmovapd	zmm15, [srcreg+r9+d2+64]	;;		I3=MemI3
	vmulpd	zmm19, zmm13, zmm24		;;		A3 = R3 * MemR3					; 2-5		n 6
	vmulpd	zmm24, zmm15, zmm24		;;		B3 = I3 * MemR3					; 2-5		n 6
	zfnmaddpd zmm19, zmm15, zmm18, zmm19	;;		A3 - I3 * MemI3 (R3 & R5)			; 6-9		n 19
	zfmaddpd zmm18, zmm13, zmm18, zmm24	;;		B3 + R3 * MemI3 (I3 & I5)			; 6-9		n 20

	vmovapd	zmm24, [srcreg+rbp+d2+d1]	;;		MemR4
	vsubpd	zmm24, zmm24, zmm9		;;		MemR4-R4
	vmovapd	zmm9, [srcreg+r9+d2+d1]		;;		R4=MemR4
	vmovapd	zmm20, [srcreg+rbp+d2+d1+64]	;;		MemI4
	vsubpd	zmm20, zmm20, zmm12		;;		MemI4-I4
	vmovapd	zmm12, [srcreg+r9+d2+d1+64]	;;		I4=MemI4
	vmulpd	zmm21, zmm9, zmm24		;;		A4 = R4 * MemR4					; 3-6		n 8
	vmulpd	zmm24, zmm12, zmm24		;;		B4 = I4 * MemR4					; 3-6		n 8
	zfnmaddpd zmm21, zmm12, zmm20, zmm21	;;		A4 - I4 * MemI4 (R4 & R6)			; 8-11		n 19
	zfmaddpd zmm20, zmm9, zmm20, zmm24	;;		B4 + R4 * MemI4 (I4 & I6)			; 8-11		n 20

	vmovapd	zmm24, [srcreg+rbp]		;; MemR1			MemR1a
	vsubpd	zmm24, zmm24, zmm1		;; MemR1-R1			MemR1a-R1a
	vmovapd	zmm1, [srcreg+r9]		;; R1=MemR1			R1a=MemR1a
	vmovapd	zmm9, [srcreg+rbp+64]		;; MemI1			MemR1b
	vsubpd	zmm9, zmm9, zmm11		;; MemI1-I1			MemR1b-R1b
	vmovapd	zmm11, [srcreg+r9+64]		;; I1=MemI1			R1b=MemR1b
	vmulpd	zmm23, zmm1, zmm24		;; A1 = R1 * MemR1		R1a*MemR1a (R1a)		; 4-7		n 10
	vmulpd	zmm22 {k7}{z}, zmm11, zmm24	;; B1 = I1 * MemR1		0				; 4-7		n 10
	zfnmaddpd zmm23 {k7}, zmm11, zmm9, zmm23;; A1 - I1 * MemI1 (R1)	R1a					; 10-13		n 16
	zfmaddpd zmm22 {k7}{z}, zmm1, zmm9, zmm22;; B1 + R1 * MemI1 (I1)	0				; 10-13		n 18
	vmovapd	zmm12, zmm17			;;		Copy R2
	vmulpd	zmm12 {k6}, zmm11, zmm9		;; R2				R1b * MemR1b (R1b)		; 11-14		n 16
	vmovsd	Q [srcreg+r8-16], xmm23		;;				Save product of FFT sums	; 8

	vmovapd	zmm24, [srcreg+rbp+d4]		;;		MemR5
	vsubpd	zmm24, zmm24, zmm10		;;		MemR5-R5
	vmovapd	zmm10, [srcreg+r9+d4]		;;		R5=MemR5
	vmovapd	zmm2, [srcreg+rbp+d4+64]	;;		MemI5
	vsubpd	zmm2, zmm2, zmm7		;;		MemI5-I5
	vmovapd	zmm7, [srcreg+r9+d4+64]		;;		I5=MemI5
	vmulpd	zmm3, zmm10, zmm24		;;		A5 = R5 * MemR5					; 7-10		n 12
	vmulpd	zmm24, zmm7, zmm24		;;		B5 = I5 * MemR5					; 7-10		n 13
	zfnmaddpd zmm3, zmm7, zmm2, zmm3	;;		A5 - I5 * MemI5 (R5 & R9)			; 12-15		n 21
	zfmaddpd zmm2, zmm10, zmm2, zmm24	;;		B5 + R5 * MemI5 (I5 & I9)			; 13-16		n 22

	vmovapd	zmm24, [srcreg+rbp+d4+d1]	;;		MemR6
	vsubpd	zmm24, zmm24, zmm6		;;		MemR6-R6
	vmovapd	zmm6, [srcreg+r9+d4+d1]		;;		R6=MemR6
	vmovapd	zmm13, [srcreg+rbp+d4+d1+64]	;;		MemI6
	vsubpd	zmm13, zmm13, zmm8		;;		MemI6-I6
	vmovapd	zmm8, [srcreg+r9+d4+d1+64]	;;		I6=MemI6
	vmulpd	zmm15, zmm6, zmm24		;;		A6 = R6 * MemR6					; 9-12		n 14
	vmulpd	zmm24, zmm8, zmm24		;;		B6 = I6 * MemR6					; 9-12		n 15
	zfnmaddpd zmm15, zmm8, zmm13, zmm15	;;		A6 - I6 * MemI6 (R6 & R10)			; 14-17		n 21
	zfmaddpd zmm13, zmm6, zmm13, zmm24	;;		B6 + R6 * MemI6 (I6 & I10)			; 15-18		n 22

	vmovapd	zmm24, [srcreg+rbp+d4+d2]	;;		MemR7
	vsubpd	zmm24, zmm24, zmm14		;;		MemR7-R7
	vmovapd	zmm14, [srcreg+r9+d4+d2]	;;		R7=MemR7
	vmovapd	zmm1, [srcreg+rbp+d4+d2+64]	;;		MemI7
	vsubpd	zmm1, zmm1, zmm4		;;		MemI7-I7
	vmovapd	zmm4, [srcreg+r9+d4+d2+64]	;;		I7=MemI7
	vmulpd	zmm9, zmm14, zmm24		;;		A7 = R7 * MemR7					; 11-14		n 15
	vmulpd	zmm24, zmm4, zmm24		;;		B7 = I7 * MemR7					; 12-15		n 16
	zfnmaddpd zmm9, zmm4, zmm1, zmm9	;;		A7 - I7 * MemI7 (R7 & R11)			; 15-18		n 23
	zfmaddpd zmm1, zmm14, zmm1, zmm24	;;		B7 + R7 * MemI7 (I7 & I11)			; 16-19		n 24

	vmovapd	zmm24, [srcreg+rbp+d4+d2+d1]	;;		MemR8
	vsubpd	zmm24, zmm24, zmm0		;;		MemR8-R8
	vmovapd	zmm0, [srcreg+r9+d4+d2+d1]	;;		R8=MemR8
	vmovapd	zmm7, [srcreg+rbp+d4+d2+d1+64]	;;		MemI8
	vsubpd	zmm7, zmm7, zmm5		;;		MemI8-I8
	vmovapd	zmm5, [srcreg+r9+d4+d2+d1+64]	;;		I8=MemI8
	vmulpd	zmm11, zmm0, zmm24		;;		A8 = R8 * MemR8					; 13-16		n 17
	vmulpd	zmm24, zmm5, zmm24		;;		B8 = I8 * MemR8					; 14-17		n 18
	zfnmaddpd zmm11, zmm5, zmm7, zmm11	;;		A8 - I8 * MemI8 (R8 & R12)			; 17-20		n 23
	zfmaddpd zmm7, zmm0, zmm7, zmm24	;;		B8 + R8 * MemI8 (I8 & I12)			; 18-21		n 24

						;;				R1a/R1b becomes R1/R2
	vaddpd	zmm10, zmm23, zmm12		;; R1 + R2 (new R1)		R1 + R2 (new R1/HALF)		; 16-19		n 26
	vsubpd	zmm23, zmm23, zmm12		;; R1 - R2 (new R2)		R1 - R2 (new R2/HALF)		; 17-20		n 28
	jmp	back_to_orig

unfft:	vmovapd	zmm23, zmm1			;; R1				R1a
	vmovapd	zmm22 {k7}{z}, zmm11		;; I1				0
	vmovsd	Q [srcreg+r8-16], xmm23		;;				Save product of FFT sums

	vmovapd	zmm17, zmm2			;;		R2
	vmovapd	zmm2 {k6}, zmm11		;; R2				R1b

	vmovapd	zmm16, zmm3			;;		I2
	vmovapd	zmm3, zmm10			;;		R5 & R9

						;;				R1a/R1b becomes R1/R2
	vaddpd	zmm10, zmm23, zmm2		;; R1 + R2 (new R1)		R1 + R2 (new R1/HALF)		; 16-19		n 26
	vsubpd	zmm23, zmm23, zmm2		;; R1 - R2 (new R2)		R1 - R2 (new R2/HALF)		; 17-20		n 28

	vmovapd	zmm20, zmm12			;;		I4 & I6
	vmovapd	zmm1, zmm4			;;		I7 & I11
	vmovapd	zmm18, zmm15			;;		I3 & I5
	vmovapd	zmm15, zmm6			;;		R6 & R10
	vmovapd	zmm19, zmm13			;;		R3 & R5
	vmovapd	zmm13, zmm8			;;		I6 & I10
	vmovapd	zmm21, zmm9			;;		R4 & R6
	vmovapd	zmm9, zmm14			;;		R7 & R11
	vmovapd	zmm11, zmm0			;;		R8 & R12
	vmovapd	zmm2, zmm7			;;		I5 & I9
	vmovapd	zmm7, zmm5			;;		I8 & I12
	jmp	back_to_orig

muladd:	vmovapd	zmm24, [srcreg+rbp+d1]		;;		MemR2
	vmulpd	zmm17, zmm2, zmm24		;;		A2 = R2 * MemR2					; 1-4		n 5
	vmulpd	zmm16, zmm3, zmm24		;;		B2 = I2 * MemR2					; 1-4		n 5

 	vmovapd	zmm24, [srcreg+rbp+d2]		;;		MemR3
	vmulpd	zmm19, zmm13, zmm24		;;		A3 = R3 * MemR3					; 2-5		n 6
	vmulpd	zmm18, zmm15, zmm24		;;		B3 = I3 * MemR3					; 2-5		n 6

	vmovapd	zmm24, [srcreg+rbp+d2+d1]	;;		MemR4
	vmulpd	zmm21, zmm9, zmm24		;;		A4 = R4 * MemR4					; 3-6		n 8
	vmulpd	zmm20, zmm12, zmm24		;;		B4 = I4 * MemR4					; 3-6		n 8

	vmovapd	zmm24, [srcreg+rbp]		;; MemR1			MemR1a
	vmulpd	zmm23, zmm1, zmm24		;; A1 = R1 * MemR1		R1a*MemR1a (R1a)		; 4-7		n 10
	vmulpd	zmm22 {k7}{z}, zmm11, zmm24	;; B1 = I1 * MemR1		0				; 4-7		n 10

	vmovapd zmm24, [srcreg+rbp+d1+64]	;;		MemI2
	zfnmaddpd zmm17, zmm3, zmm24, zmm17	;;		A2 - I2 * MemI2 (R2)				; 5-8		n 10
	zfmaddpd zmm16, zmm2, zmm24, zmm16	;;		B2 + R2 * MemI2 (I2)				; 5-8		n 18

	vmovapd zmm24, [srcreg+rbp+d2+64]	;;		MemI3
	zfnmaddpd zmm19, zmm15, zmm24, zmm19	;;		A3 - I3 * MemI3 (R3 & R5)			; 6-9		n 19
	zfmaddpd zmm18, zmm13, zmm24, zmm18	;;		B3 + R3 * MemI3 (I3 & I5)			; 6-9		n 20

	vmovapd	zmm24, [srcreg+rbp+d4]		;;		MemR5
	vmulpd	zmm3, zmm10, zmm24		;;		A5 = R5 * MemR5					; 7-10		n 12
	vmulpd	zmm2, zmm7, zmm24		;;		B5 = I5 * MemR5					; 7-10		n 13

	vmovapd zmm24, [srcreg+rbp+d2+d1+64]	;;		MemI4
	zfnmaddpd zmm21, zmm12, zmm24, zmm21	;;		A4 - I4 * MemI4 (R4 & R6)			; 8-11		n 19
	zfmaddpd zmm20, zmm9, zmm24, zmm20	;;		B4 + R4 * MemI4 (I4 & I6)			; 8-11		n 20
	vmovsd	Q [srcreg+r8-16], xmm23		;;				Save product of FFT sums	; 8

	vmovapd	zmm24, [srcreg+rbp+d4+d1]	;;		MemR6
	vmulpd	zmm15, zmm6, zmm24		;;		A6 = R6 * MemR6					; 9-12		n 14
	vmulpd	zmm13, zmm8, zmm24		;;		B6 = I6 * MemR6					; 9-12		n 15

	mov	r10, SRC3ARG			;; Load distance to FFT(1)
	test	r10, 3				;; Check if we need to multiply add-in by FFT(1)
	jz	muladdhard			;; Yes, do it the hard way

	vaddpd	zmm17, zmm17, [srcreg+r9+d1]		;; R2 = R2 + MemR2
	vaddpd	zmm16, zmm16, [srcreg+r9+d1+64]		;; I2 = I2 + MemI2
	vaddpd	zmm19, zmm19, [srcreg+r9+d2]		;; R3 = R3 + MemR3
	vaddpd	zmm18, zmm18, [srcreg+r9+d2+64]		;; I3 = I3 + MemI3
	vaddpd	zmm21, zmm21, [srcreg+r9+d2+d1]		;; R4 = R4 + MemR4
	vaddpd	zmm20, zmm20, [srcreg+r9+d2+d1+64]	;; I4 = I4 + MemI4

	vmovapd	zmm12, zmm17			;;		Copy R2
	vmovapd zmm24, [srcreg+rbp+64]		;; MemI1			MemR1b
	zfnmaddpd zmm23 {k7}, zmm11, zmm24, zmm23;; A1 - I1 * MemI1 (R1)	R1a				; 10-13		n 16
	zfmaddpd zmm22 {k7}{z}, zmm1, zmm24, zmm22;; B1 + R1 * MemI1 (I1)	0				; 10-13		n 18

	vmulpd	zmm12 {k6}, zmm11, zmm24	;; R2				R1b * MemR1b (R1b)		; 11-14		n 16

	vmovapd	zmm24, [srcreg+rbp+d4+d2]	;;		MemR7
	vmulpd	zmm9, zmm14, zmm24		;;		A7 = R7 * MemR7					; 11-14		n 15
	vmulpd	zmm1, zmm4, zmm24		;;		B7 = I7 * MemR7					; 12-15		n 16

	vmovapd zmm24, [srcreg+rbp+d4+64]	;;		MemI5
	zfnmaddpd zmm3, zmm7, zmm24, zmm3	;;		A5 - I5 * MemI5 (R5 & R9)			; 12-15		n 21
	zfmaddpd zmm2, zmm10, zmm24, zmm2	;;		B5 + R5 * MemI5 (I5 & I9)			; 13-16		n 22

	vmovapd	zmm24, [srcreg+rbp+d4+d2+d1]	;;		MemR8
	vmulpd	zmm11, zmm0, zmm24		;;		A8 = R8 * MemR8					; 13-16		n 17
	vmulpd	zmm7, zmm5, zmm24		;;		B8 = I8 * MemR8					; 14-17		n 18

	vmovapd zmm24, [srcreg+rbp+d4+d1+64]	;;		MemI6
	zfnmaddpd zmm15, zmm8, zmm24, zmm15	;;		A6 - I6 * MemI6 (R6 & R10)			; 14-17		n 21
	zfmaddpd zmm13, zmm6, zmm24, zmm13	;;		B6 + R6 * MemI6 (I6 & I10)			; 15-18		n 22

	vmovapd zmm24, [srcreg+rbp+d4+d2+64]	;;		MemI7
	zfnmaddpd zmm9, zmm4, zmm24, zmm9	;;		A7 - I7 * MemI7 (R7 & R11)			; 15-18		n 23
	zfmaddpd zmm1, zmm14, zmm24, zmm1	;;		B7 + R7 * MemI7 (I7 & I11)			; 16-19		n 24

	vaddpd	zmm23, zmm23, [srcreg+r9]		;; R1 = R1 + MemR1
	vaddpd	zmm12{k6}, zmm12, [srcreg+r9+64]	;; R1b = R1b + MemI1
	vaddpd	zmm22{k7}{z}, zmm22, [srcreg+r9+64]	;; I1 = I1 + MemI1
	vaddpd	zmm3, zmm3, [srcreg+r9+d4]		;; R5 = R5 + MemR5
	vaddpd	zmm2, zmm2, [srcreg+r9+d4+64]		;; I5 = I5 + MemI5

						;;				R1a/R1b becomes R1/R2
	vaddpd	zmm10, zmm23, zmm12		;; R1 + R2 (new R1)		R1 + R2 (new R1/HALF)		; 16-19		n 26
	vsubpd	zmm23, zmm23, zmm12		;; R1 - R2 (new R2)		R1 - R2 (new R2/HALF)		; 17-20		n 28

	vmovapd zmm24, [srcreg+rbp+d4+d2+d1+64] ;;		MemI8
	zfnmaddpd zmm11, zmm5, zmm24, zmm11	;;		A8 - I8 * MemI8 (R8 & R12)			; 17-20		n 23
	zfmaddpd zmm7, zmm0, zmm24, zmm7	;;		B8 + R8 * MemI8 (I8 & I12)			; 18-21		n 24

	vaddpd	zmm15, zmm15, [srcreg+r9+d4+d1]		;; R6 = R6 + MemR6
	vaddpd	zmm13, zmm13, [srcreg+r9+d4+d1+64]	;; I6 = I6 + MemI6
	vaddpd	zmm9, zmm9, [srcreg+r9+d4+d2]		;; R7 = R7 + MemR7
	vaddpd	zmm1, zmm1, [srcreg+r9+d4+d2+64]	;; I7 = I7 + MemI7
	vaddpd	zmm11, zmm11, [srcreg+r9+d4+d2+d1]	;; R8 = R8 + MemR8
	vaddpd	zmm7, zmm7, [srcreg+r9+d4+d2+d1+64]	;; I8 = I8 + MemI8
	jmp	back_to_orig

muladdhard:
	vmovapd	zmm9, [srcreg+r9+d1]			;; MemR2
	vmovapd	zmm12, [srcreg+r10+d1]			;; MemR2#2
	vmovapd	zmm24, [srcreg+r10+d1+64]		;; MemI2#2
	zfmaddpd zmm17, zmm9, zmm12, zmm17		;; R2 = R2 + MemR2*MemR2#2
	zfmaddpd zmm16, zmm9, zmm24, zmm16		;; I2 = I2 + MemR2*MemI2#2
	vmovapd	zmm9, [srcreg+r9+d1+64]			;; MemI2
	zfnmaddpd zmm17, zmm9, zmm24, zmm17		;; R2 = R2 - MemI2*MemI2#2
	zfmaddpd zmm16, zmm9, zmm12, zmm16		;; I2 = I2 + MemI2*MemR2#2

	vmovapd	zmm9, [srcreg+r9+d2]			;; MemR3
	vmovapd	zmm12, [srcreg+r10+d2]			;; MemR3#2
	vmovapd	zmm24, [srcreg+r10+d2+64]		;; MemI3#2
	zfmaddpd zmm19, zmm9, zmm12, zmm19		;; R3 = R3 + MemR3*MemR3#2
	zfmaddpd zmm18, zmm9, zmm24, zmm18		;; I3 = I3 + MemR3*MemI3#2
	vmovapd	zmm9, [srcreg+r9+d2+64]			;; MemI3
	zfnmaddpd zmm19, zmm9, zmm24, zmm19		;; R3 = R3 - MemI3*MemI3#2
	zfmaddpd zmm18, zmm9, zmm12, zmm18		;; I3 = I3 + MemI3*MemR3#2

	vmovapd	zmm9, [srcreg+r9+d2+d1]			;; MemR4
	vmovapd	zmm12, [srcreg+r10+d2+d1]		;; MemR4#2
	vmovapd	zmm24, [srcreg+r10+d2+d1+64]		;; MemI4#2
	zfmaddpd zmm21, zmm9, zmm12, zmm21		;; R4 = R4 + MemR4*MemR4#2
	zfmaddpd zmm20, zmm9, zmm24, zmm20		;; I4 = I4 + MemR4*MemI4#2
	vmovapd	zmm9, [srcreg+r9+d2+d1+64]		;; MemI4
	zfnmaddpd zmm21, zmm9, zmm24, zmm21		;; R4 = R4 - MemI4*MemI4#2
	zfmaddpd zmm20, zmm9, zmm12, zmm20		;; I4 = I4 + MemI4*MemR4#2

	vmovapd	zmm12, zmm17			;;		Copy R2
	vmovapd zmm24, [srcreg+rbp+64]		;; MemI1			MemR1b
	zfnmaddpd zmm23 {k7}, zmm11, zmm24, zmm23;; A1 - I1 * MemI1 (R1)	R1a				; 10-13		n 16
	zfmaddpd zmm22 {k7}{z}, zmm1, zmm24, zmm22;; B1 + R1 * MemI1 (I1)	0				; 10-13		n 18

	vmulpd	zmm12 {k6}, zmm11, zmm24	;; R2				R1b * MemR1b (R1b)		; 11-14		n 16

	vmovapd	zmm24, [srcreg+rbp+d4+d2]	;;		MemR7
	vmulpd	zmm9, zmm14, zmm24		;;		A7 = R7 * MemR7					; 11-14		n 15
	vmulpd	zmm1, zmm4, zmm24		;;		B7 = I7 * MemR7					; 12-15		n 16

	vmovapd zmm24, [srcreg+rbp+d4+64]	;;		MemI5
	zfnmaddpd zmm3, zmm7, zmm24, zmm3	;;		A5 - I5 * MemI5 (R5 & R9)			; 12-15		n 21
	zfmaddpd zmm2, zmm10, zmm24, zmm2	;;		B5 + R5 * MemI5 (I5 & I9)			; 13-16		n 22

	vmovapd	zmm24, [srcreg+rbp+d4+d2+d1]	;;		MemR8
	vmulpd	zmm11, zmm0, zmm24		;;		A8 = R8 * MemR8					; 13-16		n 17
	vmulpd	zmm7, zmm5, zmm24		;;		B8 = I8 * MemR8					; 14-17		n 18

	vmovapd zmm24, [srcreg+rbp+d4+d1+64]	;;		MemI6
	zfnmaddpd zmm15, zmm8, zmm24, zmm15	;;		A6 - I6 * MemI6 (R6 & R10)			; 14-17		n 21
	zfmaddpd zmm13, zmm6, zmm24, zmm13	;;		B6 + R6 * MemI6 (I6 & I10)			; 15-18		n 22

	vmovapd zmm24, [srcreg+rbp+d4+d2+64]	;;		MemI7
	zfnmaddpd zmm9, zmm4, zmm24, zmm9	;;		A7 - I7 * MemI7 (R7 & R11)			; 15-18		n 23
	zfmaddpd zmm1, zmm14, zmm24, zmm1	;;		B7 + R7 * MemI7 (I7 & I11)			; 16-19		n 24

	vmovapd	zmm10, [srcreg+r9]			;; MemR1
	vmovapd	zmm6, [srcreg+r10]			;; MemR1#2
	vmovapd	zmm8, [srcreg+r10+64]			;; MemI1#2
	zfmaddpd zmm23, zmm10, zmm6, zmm23		;; R1 = R1 + MemR1*MemR1#2
	zfmaddpd zmm22{k7}{z}, zmm10, zmm8, zmm22	;; I1 = I1 + MemR1*MemI1#2
	vmovapd	zmm10, [srcreg+r9+64]			;; MemI1
	zfnmaddpd zmm23{k7}, zmm10, zmm8, zmm23		;; R1 = R1 - MemI1*MemI1#2
	zfmaddpd zmm12{k6}, zmm10, zmm8, zmm12		;; R1b = R1b + MemI1*MemI1#2
	zfmaddpd zmm22{k7}{z}, zmm10, zmm6, zmm22	;; I1 = I1 + MemI1*MemR1#2

	vmovapd	zmm10, [srcreg+r9+d4]			;; MemR5
	vmovapd	zmm6, [srcreg+r10+d4]			;; MemR5#2
	vmovapd	zmm8, [srcreg+r10+d4+64]		;; MemI5#2
	zfmaddpd zmm3, zmm10, zmm6, zmm3		;; R5 = R5 + MemR5*MemR5#2
	zfmaddpd zmm2, zmm10, zmm8, zmm2		;; I5 = I5 + MemR5*MemI5#2
	vmovapd	zmm10, [srcreg+r9+d4+64]		;; MemI5
	zfnmaddpd zmm3, zmm10, zmm8, zmm3		;; R5 = R5 - MemI5*MemI5#2
	zfmaddpd zmm2, zmm10, zmm6, zmm2		;; I5 = I5 + MemI5*MemR5#2

						;;				R1a/R1b becomes R1/R2
	vaddpd	zmm10, zmm23, zmm12		;; R1 + R2 (new R1)		R1 + R2 (new R1/HALF)		; 16-19		n 26
	vsubpd	zmm23, zmm23, zmm12		;; R1 - R2 (new R2)		R1 - R2 (new R2/HALF)		; 17-20		n 28

	vmovapd zmm24, [srcreg+rbp+d4+d2+d1+64] ;;		MemI8
	zfnmaddpd zmm11, zmm5, zmm24, zmm11	;;		A8 - I8 * MemI8 (R8 & R12)			; 17-20		n 23
	zfmaddpd zmm7, zmm0, zmm24, zmm7	;;		B8 + R8 * MemI8 (I8 & I12)			; 18-21		n 24

	vmovapd	zmm4, [srcreg+r9+d4+d1]			;; MemR6
	vmovapd	zmm6, [srcreg+r10+d4+d1]		;; MemR6#2
	vmovapd	zmm8, [srcreg+r10+d4+d1+64]		;; MemI6#2
	zfmaddpd zmm15, zmm4, zmm6, zmm15		;; R6 = R6 + MemR6*MemR6#2
	zfmaddpd zmm13, zmm4, zmm8, zmm13		;; I6 = I6 + MemR6*MemI6#2
	vmovapd	zmm4, [srcreg+r9+d4+d1+64]		;; MemI6
	zfnmaddpd zmm15, zmm4, zmm8, zmm15		;; R6 = R6 - MemI6*MemI6#2
	zfmaddpd zmm13, zmm4, zmm6, zmm13		;; I6 = I6 + MemI6*MemR6#2

	vmovapd	zmm4, [srcreg+r9+d4+d2]			;; MemR7
	vmovapd	zmm6, [srcreg+r10+d4+d2]		;; MemR7#2
	vmovapd	zmm8, [srcreg+r10+d4+d2+64]		;; MemI7#2
	zfmaddpd zmm9, zmm4, zmm6, zmm9			;; R7 = R7 + MemR7*MemR7#2
	zfmaddpd zmm1, zmm4, zmm8, zmm1			;; I7 = I7 + MemR7*MemI7#2
	vmovapd	zmm4, [srcreg+r9+d4+d2+64]		;; MemI7
	zfnmaddpd zmm9, zmm4, zmm8, zmm9		;; R7 = R7 - MemI7*MemI7#2
	zfmaddpd zmm1, zmm4, zmm6, zmm1			;; I7 = I7 + MemI7*MemR7#2

	vmovapd	zmm4, [srcreg+r9+d4+d2+d1]		;; MemR8
	vmovapd	zmm6, [srcreg+r10+d4+d2+d1]		;; MemR8#2
	vmovapd	zmm8, [srcreg+r10+d4+d2+d1+64]		;; MemI8#2
	zfmaddpd zmm11, zmm4, zmm6, zmm11		;; R8 = R8 + MemR8*MemR8#2
	zfmaddpd zmm7, zmm4, zmm8, zmm7			;; I8 = I8 + MemR8*MemI8#2
	vmovapd	zmm4, [srcreg+r9+d4+d2+d1+64]		;; MemI8
	zfnmaddpd zmm11, zmm4, zmm8, zmm11		;; R8 = R8 - MemI8*MemI8#2
	zfmaddpd zmm7, zmm4, zmm6, zmm7			;; I8 = I8 + MemI8*MemR8#2
	jmp	back_to_orig

mulsub:	vmovapd	zmm24, [srcreg+rbp+d1]		;;		MemR2
	vmulpd	zmm17, zmm2, zmm24		;;		A2 = R2 * MemR2					; 1-4		n 5
	vmulpd	zmm16, zmm3, zmm24		;;		B2 = I2 * MemR2					; 1-4		n 5

 	vmovapd	zmm24, [srcreg+rbp+d2]		;;		MemR3
	vmulpd	zmm19, zmm13, zmm24		;;		A3 = R3 * MemR3					; 2-5		n 6
	vmulpd	zmm18, zmm15, zmm24		;;		B3 = I3 * MemR3					; 2-5		n 6

	vmovapd	zmm24, [srcreg+rbp+d2+d1]	;;		MemR4
	vmulpd	zmm21, zmm9, zmm24		;;		A4 = R4 * MemR4					; 3-6		n 8
	vmulpd	zmm20, zmm12, zmm24		;;		B4 = I4 * MemR4					; 3-6		n 8

	vmovapd	zmm24, [srcreg+rbp]		;; MemR1			MemR1a
	vmulpd	zmm23, zmm1, zmm24		;; A1 = R1 * MemR1		R1a*MemR1a (R1a)		; 4-7		n 10
	vmulpd	zmm22 {k7}{z}, zmm11, zmm24	;; B1 = I1 * MemR1		0				; 4-7		n 10

	vmovapd zmm24, [srcreg+rbp+d1+64]	;;		MemI2
	zfnmaddpd zmm17, zmm3, zmm24, zmm17	;;		A2 - I2 * MemI2 (R2)				; 5-8		n 10
	zfmaddpd zmm16, zmm2, zmm24, zmm16	;;		B2 + R2 * MemI2 (I2)				; 5-8		n 18

	vmovapd zmm24, [srcreg+rbp+d2+64]	;;		MemI3
	zfnmaddpd zmm19, zmm15, zmm24, zmm19	;;		A3 - I3 * MemI3 (R3 & R5)			; 6-9		n 19
	zfmaddpd zmm18, zmm13, zmm24, zmm18	;;		B3 + R3 * MemI3 (I3 & I5)			; 6-9		n 20

	vmovapd	zmm24, [srcreg+rbp+d4]		;;		MemR5
	vmulpd	zmm3, zmm10, zmm24		;;		A5 = R5 * MemR5					; 7-10		n 12
	vmulpd	zmm2, zmm7, zmm24		;;		B5 = I5 * MemR5					; 7-10		n 13

	vmovapd zmm24, [srcreg+rbp+d2+d1+64]	;;		MemI4
	zfnmaddpd zmm21, zmm12, zmm24, zmm21	;;		A4 - I4 * MemI4 (R4 & R6)			; 8-11		n 19
	zfmaddpd zmm20, zmm9, zmm24, zmm20	;;		B4 + R4 * MemI4 (I4 & I6)			; 8-11		n 20
	vmovsd	Q [srcreg+r8-16], xmm23		;;				Save product of FFT sums	; 8

	vmovapd	zmm24, [srcreg+rbp+d4+d1]	;;		MemR6
	vmulpd	zmm15, zmm6, zmm24		;;		A6 = R6 * MemR6					; 9-12		n 14
	vmulpd	zmm13, zmm8, zmm24		;;		B6 = I6 * MemR6					; 9-12		n 15

	mov	r10, SRC3ARG			;; Load distance to FFT(1)
	test	r10, 3				;; Check if we need to multiply add-in by FFT(1)
	jz	mulsubhard			;; Yes, do it the hard way

	vsubpd	zmm17, zmm17, [srcreg+r9+d1]		;; R2 = R2 - MemR2
	vsubpd	zmm16, zmm16, [srcreg+r9+d1+64]		;; I2 = I2 - MemI2
	vsubpd	zmm19, zmm19, [srcreg+r9+d2]		;; R3 = R3 - MemR3
	vsubpd	zmm18, zmm18, [srcreg+r9+d2+64]		;; I3 = I3 - MemI3
	vsubpd	zmm21, zmm21, [srcreg+r9+d2+d1]		;; R4 = R4 - MemR4
	vsubpd	zmm20, zmm20, [srcreg+r9+d2+d1+64]	;; I4 = I4 - MemI4

	vmovapd	zmm12, zmm17			;;		Copy R2
	vmovapd zmm24, [srcreg+rbp+64]		;; MemI1			MemR1b
	zfnmaddpd zmm23 {k7}, zmm11, zmm24, zmm23;; A1 - I1 * MemI1 (R1)	R1a				; 10-13		n 16
	zfmaddpd zmm22 {k7}{z}, zmm1, zmm24, zmm22;; B1 + R1 * MemI1 (I1)	0				; 10-13		n 18

	vmulpd	zmm12 {k6}, zmm11, zmm24	;; R2				R1b * MemR1b (R1b)		; 11-14		n 16

	vmovapd	zmm24, [srcreg+rbp+d4+d2]	;;		MemR7
	vmulpd	zmm9, zmm14, zmm24		;;		A7 = R7 * MemR7					; 11-14		n 15
	vmulpd	zmm1, zmm4, zmm24		;;		B7 = I7 * MemR7					; 12-15		n 16

	vmovapd zmm24, [srcreg+rbp+d4+64]	;;		MemI5
	zfnmaddpd zmm3, zmm7, zmm24, zmm3	;;		A5 - I5 * MemI5 (R5 & R9)			; 12-15		n 21
	zfmaddpd zmm2, zmm10, zmm24, zmm2	;;		B5 + R5 * MemI5 (I5 & I9)			; 13-16		n 22

	vmovapd	zmm24, [srcreg+rbp+d4+d2+d1]	;;		MemR8
	vmulpd	zmm11, zmm0, zmm24		;;		A8 = R8 * MemR8					; 13-16		n 17
	vmulpd	zmm7, zmm5, zmm24		;;		B8 = I8 * MemR8					; 14-17		n 18

	vmovapd zmm24, [srcreg+rbp+d4+d1+64]	;;		MemI6
	zfnmaddpd zmm15, zmm8, zmm24, zmm15	;;		A6 - I6 * MemI6 (R6 & R10)			; 14-17		n 21
	zfmaddpd zmm13, zmm6, zmm24, zmm13	;;		B6 + R6 * MemI6 (I6 & I10)			; 15-18		n 22

	vmovapd zmm24, [srcreg+rbp+d4+d2+64]	;;		MemI7
	zfnmaddpd zmm9, zmm4, zmm24, zmm9	;;		A7 - I7 * MemI7 (R7 & R11)			; 15-18		n 23
	zfmaddpd zmm1, zmm14, zmm24, zmm1	;;		B7 + R7 * MemI7 (I7 & I11)			; 16-19		n 24

	vsubpd	zmm23, zmm23, [srcreg+r9]		;; R1 = R1 - MemR1
	vsubpd	zmm12{k6}, zmm12, [srcreg+r9+64]	;; R1b = R1b - MemI1
	vsubpd	zmm22{k7}, zmm22, [srcreg+r9+64]	;; I1 = I1 - MemI1
	vsubpd	zmm3, zmm3, [srcreg+r9+d4]		;; R5 = R5 - MemR5
	vsubpd	zmm2, zmm2, [srcreg+r9+d4+64]		;; I5 = I5 - MemI5

						;;				R1a/R1b becomes R1/R2
	vaddpd	zmm10, zmm23, zmm12		;; R1 + R2 (new R1)		R1 + R2 (new R1/HALF)		; 16-19		n 26
	vsubpd	zmm23, zmm23, zmm12		;; R1 - R2 (new R2)		R1 - R2 (new R2/HALF)		; 17-20		n 28

	vmovapd zmm24, [srcreg+rbp+d4+d2+d1+64] ;;		MemI8
	zfnmaddpd zmm11, zmm5, zmm24, zmm11	;;		A8 - I8 * MemI8 (R8 & R12)			; 17-20		n 23
	zfmaddpd zmm7, zmm0, zmm24, zmm7	;;		B8 + R8 * MemI8 (I8 & I12)			; 18-21		n 24

	vsubpd	zmm15, zmm15, [srcreg+r9+d4+d1]		;; R6 = R6 - MemR6
	vsubpd	zmm13, zmm13, [srcreg+r9+d4+d1+64]	;; I6 = I6 - MemI6
	vsubpd	zmm9, zmm9, [srcreg+r9+d4+d2]		;; R7 = R7 - MemR7
	vsubpd	zmm1, zmm1, [srcreg+r9+d4+d2+64]	;; I7 = I7 - MemI7
	vsubpd	zmm11, zmm11, [srcreg+r9+d4+d2+d1]	;; R8 = R8 - MemR8
	vsubpd	zmm7, zmm7, [srcreg+r9+d4+d2+d1+64]	;; I8 = I8 - MemI8
	jmp	back_to_orig

mulsubhard:
	vmovapd	zmm9, [srcreg+r9+d1]			;; MemR2
	vmovapd	zmm12, [srcreg+r10+d1]			;; MemR2#2
	vmovapd	zmm24, [srcreg+r10+d1+64]		;; MemI2#2
	zfnmaddpd zmm17, zmm9, zmm12, zmm17		;; R2 = R2 - MemR2*MemR2#2
	zfnmaddpd zmm16, zmm9, zmm24, zmm16		;; I2 = I2 - MemR2*MemI2#2
	vmovapd	zmm9, [srcreg+r9+d1+64]			;; MemI2
	zfmaddpd zmm17, zmm9, zmm24, zmm17		;; R2 = R2 + MemI2*MemI2#2
	zfnmaddpd zmm16, zmm9, zmm12, zmm16		;; I2 = I2 - MemI2*MemR2#2

	vmovapd	zmm9, [srcreg+r9+d2]			;; MemR3
	vmovapd	zmm12, [srcreg+r10+d2]			;; MemR3#2
	vmovapd	zmm24, [srcreg+r10+d2+64]		;; MemI3#2
	zfnmaddpd zmm19, zmm9, zmm12, zmm19		;; R3 = R3 - MemR3*MemR3#2
	zfnmaddpd zmm18, zmm9, zmm24, zmm18		;; I3 = I3 - MemR3*MemI3#2
	vmovapd	zmm9, [srcreg+r9+d2+64]			;; MemI3
	zfmaddpd zmm19, zmm9, zmm24, zmm19		;; R3 = R3 + MemI3*MemI3#2
	zfnmaddpd zmm18, zmm9, zmm12, zmm18		;; I3 = I3 - MemI3*MemR3#2

	vmovapd	zmm9, [srcreg+r9+d2+d1]			;; MemR4
	vmovapd	zmm12, [srcreg+r10+d2+d1]		;; MemR4#2
	vmovapd	zmm24, [srcreg+r10+d2+d1+64]		;; MemI4#2
	zfnmaddpd zmm21, zmm9, zmm12, zmm21		;; R4 = R4 - MemR4*MemR4#2
	zfnmaddpd zmm20, zmm9, zmm24, zmm20		;; I4 = I4 - MemR4*MemI4#2
	vmovapd	zmm9, [srcreg+r9+d2+d1+64]		;; MemI4
	zfmaddpd zmm21, zmm9, zmm24, zmm21		;; R4 = R4 + MemI4*MemI4#2
	zfnmaddpd zmm20, zmm9, zmm12, zmm20		;; I4 = I4 - MemI4*MemR4#2

	vmovapd	zmm12, zmm17			;;		Copy R2
	vmovapd zmm24, [srcreg+rbp+64]		;; MemI1			MemR1b
	zfnmaddpd zmm23 {k7}, zmm11, zmm24, zmm23;; A1 - I1 * MemI1 (R1)	R1a				; 10-13		n 16
	zfmaddpd zmm22 {k7}{z}, zmm1, zmm24, zmm22;; B1 + R1 * MemI1 (I1)	0				; 10-13		n 18

	vmulpd	zmm12 {k6}, zmm11, zmm24	;; R2				R1b * MemR1b (R1b)		; 11-14		n 16

	vmovapd	zmm24, [srcreg+rbp+d4+d2]	;;		MemR7
	vmulpd	zmm9, zmm14, zmm24		;;		A7 = R7 * MemR7					; 11-14		n 15
	vmulpd	zmm1, zmm4, zmm24		;;		B7 = I7 * MemR7					; 12-15		n 16

	vmovapd zmm24, [srcreg+rbp+d4+64]	;;		MemI5
	zfnmaddpd zmm3, zmm7, zmm24, zmm3	;;		A5 - I5 * MemI5 (R5 & R9)			; 12-15		n 21
	zfmaddpd zmm2, zmm10, zmm24, zmm2	;;		B5 + R5 * MemI5 (I5 & I9)			; 13-16		n 22

	vmovapd	zmm24, [srcreg+rbp+d4+d2+d1]	;;		MemR8
	vmulpd	zmm11, zmm0, zmm24		;;		A8 = R8 * MemR8					; 13-16		n 17
	vmulpd	zmm7, zmm5, zmm24		;;		B8 = I8 * MemR8					; 14-17		n 18

	vmovapd zmm24, [srcreg+rbp+d4+d1+64]	;;		MemI6
	zfnmaddpd zmm15, zmm8, zmm24, zmm15	;;		A6 - I6 * MemI6 (R6 & R10)			; 14-17		n 21
	zfmaddpd zmm13, zmm6, zmm24, zmm13	;;		B6 + R6 * MemI6 (I6 & I10)			; 15-18		n 22

	vmovapd zmm24, [srcreg+rbp+d4+d2+64]	;;		MemI7
	zfnmaddpd zmm9, zmm4, zmm24, zmm9	;;		A7 - I7 * MemI7 (R7 & R11)			; 15-18		n 23
	zfmaddpd zmm1, zmm14, zmm24, zmm1	;;		B7 + R7 * MemI7 (I7 & I11)			; 16-19		n 24

	vmovapd	zmm10, [srcreg+r9]			;; MemR1
	vmovapd	zmm6, [srcreg+r10]			;; MemR1#2
	vmovapd	zmm8, [srcreg+r10+64]			;; MemI1#2
	zfnmaddpd zmm23, zmm10, zmm6, zmm23		;; R1 = R1 - MemR1*MemR1#2
	zfnmaddpd zmm22{k7}{z}, zmm10, zmm8, zmm22	;; I1 = I1 - MemR1*MemI1#2
	vmovapd	zmm10, [srcreg+r9+64]			;; MemI1
	zfmaddpd zmm23{k7}, zmm10, zmm8, zmm23		;; R1 = R1 + MemI1*MemI1#2
	zfnmaddpd zmm12{k6}, zmm10, zmm8, zmm12		;; R1b = R1b - MemI1*MemI1#2
	zfnmaddpd zmm22{k7}{z}, zmm10, zmm6, zmm22	;; I1 = I1 - MemI1*MemR1#2

	vmovapd	zmm10, [srcreg+r9+d4]			;; MemR5
	vmovapd	zmm6, [srcreg+r10+d4]			;; MemR5#2
	vmovapd	zmm8, [srcreg+r10+d4+64]		;; MemI5#2
	zfnmaddpd zmm3, zmm10, zmm6, zmm3		;; R5 = R5 - MemR5*MemR5#2
	zfnmaddpd zmm2, zmm10, zmm8, zmm2		;; I5 = I5 - MemR5*MemI5#2
	vmovapd	zmm10, [srcreg+r9+d4+64]		;; MemI5
	zfmaddpd zmm3, zmm10, zmm8, zmm3		;; R5 = R5 + MemI5*MemI5#2
	zfnmaddpd zmm2, zmm10, zmm6, zmm2		;; I5 = I5 - MemI5*MemR5#2

						;;				R1a/R1b becomes R1/R2
	vaddpd	zmm10, zmm23, zmm12		;; R1 + R2 (new R1)		R1 + R2 (new R1/HALF)		; 16-19		n 26
	vsubpd	zmm23, zmm23, zmm12		;; R1 - R2 (new R2)		R1 - R2 (new R2/HALF)		; 17-20		n 28

	vmovapd zmm24, [srcreg+rbp+d4+d2+d1+64] ;;		MemI8
	zfnmaddpd zmm11, zmm5, zmm24, zmm11	;;		A8 - I8 * MemI8 (R8 & R12)			; 17-20		n 23
	zfmaddpd zmm7, zmm0, zmm24, zmm7	;;		B8 + R8 * MemI8 (I8 & I12)			; 18-21		n 24

	vmovapd	zmm4, [srcreg+r9+d4+d1]			;; MemR6
	vmovapd	zmm6, [srcreg+r10+d4+d1]		;; MemR6#2
	vmovapd	zmm8, [srcreg+r10+d4+d1+64]		;; MemI6#2
	zfnmaddpd zmm15, zmm4, zmm6, zmm15		;; R6 = R6 - MemR6*MemR6#2
	zfnmaddpd zmm13, zmm4, zmm8, zmm13		;; I6 = I6 - MemR6*MemI6#2
	vmovapd	zmm4, [srcreg+r9+d4+d1+64]		;; MemI6
	zfmaddpd zmm15, zmm4, zmm8, zmm15		;; R6 = R6 + MemI6*MemI6#2
	zfnmaddpd zmm13, zmm4, zmm6, zmm13		;; I6 = I6 - MemI6*MemR6#2

	vmovapd	zmm4, [srcreg+r9+d4+d2]			;; MemR7
	vmovapd	zmm6, [srcreg+r10+d4+d2]		;; MemR7#2
	vmovapd	zmm8, [srcreg+r10+d4+d2+64]		;; MemI7#2
	zfnmaddpd zmm9, zmm4, zmm6, zmm9		;; R7 = R7 - MemR7*MemR7#2
	zfnmaddpd zmm1, zmm4, zmm8, zmm1		;; I7 = I7 - MemR7*MemI7#2
	vmovapd	zmm4, [srcreg+r9+d4+d2+64]		;; MemI7
	zfmaddpd zmm9, zmm4, zmm8, zmm9			;; R7 = R7 + MemI7*MemI7#2
	zfnmaddpd zmm1, zmm4, zmm6, zmm1		;; I7 = I7 - MemI7*MemR7#2

	vmovapd	zmm4, [srcreg+r9+d4+d2+d1]		;; MemR8
	vmovapd	zmm6, [srcreg+r10+d4+d2+d1]		;; MemR8#2
	vmovapd	zmm8, [srcreg+r10+d4+d2+d1+64]		;; MemI8#2
	zfnmaddpd zmm11, zmm4, zmm6, zmm11		;; R8 = R8 - MemR8*MemR8#2
	zfnmaddpd zmm7, zmm4, zmm8, zmm7		;; I8 = I8 - MemR8*MemI8#2
	vmovapd	zmm4, [srcreg+r9+d4+d2+d1+64]		;; MemI8
	zfmaddpd zmm11, zmm4, zmm8, zmm11		;; R8 = R8 + MemI8*MemI8#2
	zfnmaddpd zmm7, zmm4, zmm6, zmm7		;; I8 = I8 - MemI8*MemR8#2
	jmp	back_to_orig

orig:
	;; Multiply the results
	vmovapd	zmm24, [srcreg+rbp+d1]		;;		MemR2
	vmulpd	zmm17, zmm2, zmm24		;;		A2 = R2 * MemR2					; 1-4		n 5
	vmulpd	zmm16, zmm3, zmm24		;;		B2 = I2 * MemR2					; 1-4		n 5

 	vmovapd	zmm24, [srcreg+rbp+d2]		;;		MemR3
	vmulpd	zmm19, zmm13, zmm24		;;		A3 = R3 * MemR3					; 2-5		n 6
	vmulpd	zmm18, zmm15, zmm24		;;		B3 = I3 * MemR3					; 2-5		n 6

	vmovapd	zmm24, [srcreg+rbp+d2+d1]	;;		MemR4
	vmulpd	zmm21, zmm9, zmm24		;;		A4 = R4 * MemR4					; 3-6		n 8
	vmulpd	zmm20, zmm12, zmm24		;;		B4 = I4 * MemR4					; 3-6		n 8

	vmovapd	zmm24, [srcreg+rbp]		;; MemR1			MemR1a
	vmulpd	zmm23, zmm1, zmm24		;; A1 = R1 * MemR1		R1a*MemR1a (R1a)		; 4-7		n 10
	vmulpd	zmm22 {k7}{z}, zmm11, zmm24	;; B1 = I1 * MemR1		0				; 4-7		n 10

	vmovapd zmm24, [srcreg+rbp+d1+64]	;;		MemI2
	zfnmaddpd zmm17, zmm3, zmm24, zmm17	;;		A2 - I2 * MemI2 (R2)				; 5-8		n 10
	zfmaddpd zmm16, zmm2, zmm24, zmm16	;;		B2 + R2 * MemI2 (I2)				; 5-8		n 18

	vmovapd zmm24, [srcreg+rbp+d2+64]	;;		MemI3
	zfnmaddpd zmm19, zmm15, zmm24, zmm19	;;		A3 - I3 * MemI3 (R3 & R5)			; 6-9		n 19
	zfmaddpd zmm18, zmm13, zmm24, zmm18	;;		B3 + R3 * MemI3 (I3 & I5)			; 6-9		n 20

	vmovapd	zmm24, [srcreg+rbp+d4]		;;		MemR5
	vmulpd	zmm3, zmm10, zmm24		;;		A5 = R5 * MemR5					; 7-10		n 12
	vmulpd	zmm2, zmm7, zmm24		;;		B5 = I5 * MemR5					; 7-10		n 13

	vmovapd zmm24, [srcreg+rbp+d2+d1+64]	;;		MemI4
	zfnmaddpd zmm21, zmm12, zmm24, zmm21	;;		A4 - I4 * MemI4 (R4 & R6)			; 8-11		n 19
	zfmaddpd zmm20, zmm9, zmm24, zmm20	;;		B4 + R4 * MemI4 (I4 & I6)			; 8-11		n 20
	vmovsd	Q [srcreg+r8-16], xmm23		;;				Save product of FFT sums	; 8

	vmovapd	zmm24, [srcreg+rbp+d4+d1]	;;		MemR6
	vmulpd	zmm15, zmm6, zmm24		;;		A6 = R6 * MemR6					; 9-12		n 14
	vmulpd	zmm13, zmm8, zmm24		;;		B6 = I6 * MemR6					; 9-12		n 15

	vmovapd	zmm12, zmm17			;;		Copy R2
	vmovapd zmm24, [srcreg+rbp+64]		;; MemI1			MemR1b
	zfnmaddpd zmm23 {k7}, zmm11, zmm24, zmm23;; A1 - I1 * MemI1 (R1)	R1a				; 10-13		n 16
	zfmaddpd zmm22 {k7}{z}, zmm1, zmm24, zmm22;; B1 + R1 * MemI1 (I1)	0				; 10-13		n 18

	vmulpd	zmm12 {k6}, zmm11, zmm24	;; R2				R1b * MemR1b (R1b)		; 11-14		n 16

	vmovapd	zmm24, [srcreg+rbp+d4+d2]	;;		MemR7
	vmulpd	zmm9, zmm14, zmm24		;;		A7 = R7 * MemR7					; 11-14		n 15
	vmulpd	zmm1, zmm4, zmm24		;;		B7 = I7 * MemR7					; 12-15		n 16

	vmovapd zmm24, [srcreg+rbp+d4+64]	;;		MemI5
	zfnmaddpd zmm3, zmm7, zmm24, zmm3	;;		A5 - I5 * MemI5 (R5 & R9)			; 12-15		n 21
	zfmaddpd zmm2, zmm10, zmm24, zmm2	;;		B5 + R5 * MemI5 (I5 & I9)			; 13-16		n 22

	vmovapd	zmm24, [srcreg+rbp+d4+d2+d1]	;;		MemR8
	vmulpd	zmm11, zmm0, zmm24		;;		A8 = R8 * MemR8					; 13-16		n 17
	vmulpd	zmm7, zmm5, zmm24		;;		B8 = I8 * MemR8					; 14-17		n 18

	vmovapd zmm24, [srcreg+rbp+d4+d1+64]	;;		MemI6
	zfnmaddpd zmm15, zmm8, zmm24, zmm15	;;		A6 - I6 * MemI6 (R6 & R10)			; 14-17		n 21
	zfmaddpd zmm13, zmm6, zmm24, zmm13	;;		B6 + R6 * MemI6 (I6 & I10)			; 15-18		n 22

	vmovapd zmm24, [srcreg+rbp+d4+d2+64]	;;		MemI7
	zfnmaddpd zmm9, zmm4, zmm24, zmm9	;;		A7 - I7 * MemI7 (R7 & R11)			; 15-18		n 23
	zfmaddpd zmm1, zmm14, zmm24, zmm1	;;		B7 + R7 * MemI7 (I7 & I11)			; 16-19		n 24

						;;				R1a/R1b becomes R1/R2
	vaddpd	zmm10, zmm23, zmm12		;; R1 + R2 (new R1)		R1 + R2 (new R1/HALF)		; 16-19		n 26
	vsubpd	zmm23, zmm23, zmm12		;; R1 - R2 (new R2)		R1 - R2 (new R2/HALF)		; 17-20		n 28

	vmovapd zmm24, [srcreg+rbp+d4+d2+d1+64] ;;		MemI8
	zfnmaddpd zmm11, zmm5, zmm24, zmm11	;;		A8 - I8 * MemI8 (R8 & R12)			; 17-20		n 23
	zfmaddpd zmm7, zmm0, zmm24, zmm7	;;		B8 + R8 * MemI8 (I8 & I12)			; 18-21		n 24

back_to_orig:					;;				R2/I2 becomes R3/R4
	vaddpd	zmm17 {k7}, zmm22, zmm16	;; I1 + I2 (new I1)		R3				; 18-47		n 25
	vsubpd	zmm16 {k7}, zmm22, zmm16	;; I1 - I2 (new I2)		R4				; 19-47		n 26

	vaddpd	zmm8, zmm21, zmm19		;; R4 + R3 (new R3)		R6 + R5 (new R5)		; 19-52		n 25
	vsubpd	zmm21, zmm21, zmm19		;; R4 - R3 (new I4)		R6 - R5 (new negR6)		; 20-52		n 33

	vaddpd	zmm6, zmm18, zmm20		;; I3 + I4 (new I3)		I5 + I6 (new I5)		; 20-53		n 27
	vsubpd	zmm18, zmm18, zmm20		;; I3 - I4 (new R4)		I5 - I6 (new I6)		; 21-53		n 26

	vaddpd	zmm4, zmm3, zmm15		;; R5 + R6 (new R5)		R9 + R10 (new R9)		; 21-48		n 34
	vsubpd	zmm3, zmm3, zmm15		;; R5 - R6 (new R6)		R9 - R10 (new R10)		; 22-48		n 29

	vaddpd	zmm14, zmm2, zmm13		;; I5 + I6 (new I5)		I9 + I10 (new I9)		; 22-49		n 35
	vsubpd	zmm2, zmm2, zmm13		;; I5 - I6 (new I6)		I9 - I10 (new I10)		; 23-49		n 

	vaddpd	zmm12, zmm11, zmm9		;; R8 + R7 (new R7)		R12 + R11 (new R11)		; 23-50		n 34
	vsubpd	zmm11, zmm11, zmm9		;; R8 - R7 (new I8)		R12 - R11 (new I12)		; 24-50		n 29

	vaddpd	zmm5, zmm1, zmm7		;; I7 + I8 (new I7)		I11 + I12 (new I11)		; 24-51		n 35
	vsubpd	zmm1, zmm1, zmm7		;; I7 - I8 (new R8)		I11 - I12 (new R12)		; 25-51		n 32

	vblendmpd zmm28 {k6}, zmm26, ZMM_HALF{1to8} ;; 1, 1, 1, 1, 1, 1, 1	HALF				; 25***		n 
	vblendmpd zmm0 {k6}, zmm8, zmm17	;; R3				R3				; 25		n 26
	vblendmpd zmm15 {k6}, zmm18, zmm16	;; R4				R4				; 26		n 28

	zfmaddpd zmm13, zmm10, zmm28, zmm0	;; R1*1 + R3 (newer R1)		R1*HALF + R3 (newer R1)		; 26-57		n 
	zfmsubpd zmm10, zmm10, zmm28, zmm0	;; R1*1 - R3 (newer R3)		R1*HALF - R3 (newer R3)		; 27-57		n 


	vaddpd	zmm9 {k7}{z}, zmm17, zmm6	;; I1 + I3 (newer I1)		0				; 27-59		n 38
	vsubpd	zmm17 {k7}{z}, zmm17, zmm6	;; I1 - I3 (newer I3)		0				; 28-59		n 39

	zfmaddpd zmm7, zmm23, zmm28, zmm15	;; R2*1 + R4 (newer R2)		R2*HALF + R4 (newer R2)		; 28-61		n 45
	zfmsubpd zmm23, zmm23, zmm28, zmm15	;; R2*1 - R4 (newer R4)		R2*HALF - R4 (newer R4)		; 29-61		n 46

	vblendmpd zmm27 {k6}, zmm26, ZMM_P383{1to8} ;; 1, 1, 1, 1, 1, 1, 1	.383				; 25***		n 
	vmovapd zmm0, zmm3			;; R6				R10
	vmulpd	zmm0 {k6}, zmm27, zmm11		;; R6				I12*.383			; 29-61		n 
	vmulpd	zmm11 {k6}, zmm27, zmm3		;; I8				R10*.383			; 30-54		n 

						;; multiply R6/I6 by SQRTHALF - i*SQRTHALF
	zfmaddpd zmm15, zmm2, zmm27, zmm0	;; I6*1 + R6 (new2 R6/SQRTHALF)	I10*.383 + I12 (newer I10*.383)	; 30-54		n 36
	zfmsubpd zmm2, zmm2, zmm27, zmm0	;; I6*1 - R6 (new2 I6/SQRTHALF)	I10*.383 - I12 (newer I12*.383)	; 31-54		n 37

	vblendmpd zmm16 {k6}, zmm16, zmm18	;; I2				I6				; 31		n 

						;; multiply R8/I8 by SQRTHALF - i*SQRTHALF
	zfmaddpd zmm3, zmm1, zmm27, zmm11	;; I8 + R8*1 (new2 R8/SQRTHALF)	R10 + R12*.383 (newer R10*.383)	; 32-55		n 36
	zfnmaddpd zmm11, zmm1, zmm27, zmm11	;; I8 - R8*1 (new2 I8/SQRTHALF)	R10 - R12*.383 (newer R12*.383)	; 32-55		n 37

						;;				mul R6/I6 by w^2 = .707 - .707i
	vaddpd	zmm0, zmm16, zmm21		;; I2 + I4 (newer I2)		I6 + negR6 (newer I6/SQRTHALF) 	; 33-60		n 44
	vsubpd	zmm16, zmm16, zmm21		;; I2 - I4 (newer I4)		I6 - negR6 (newer R6/SQRTHALF) 	; 33-60		n 44

	vsubpd	zmm1, zmm12, zmm4		;; R7 - R5 (newer I7)		R11 - R9 (newer negR11)		; 34-58		n 38
	vaddpd	zmm12, zmm12, zmm4		;; R7 + R5 (newer R5)		R11 + R9 (newer R9)		; 34-37		n 38

	vaddpd	zmm4, zmm14, zmm5		;; I5 + I7 (newer I5)		I9 + I11 (newer I9)		; 35-56		n 42
	vsubpd	zmm14, zmm14, zmm5		;; I5 - I7 (newer R7)		I9 - I11 (newer I11)		; 35-56		n 39

						;;				mul R10/I10 by w^1 = .924 - .383i
	zfmaddpd zmm5, zmm3, zmm26, zmm15	;; R6 + R8*1 (newer R6/SQRTHALF)    I10 + R10*.924/.383 (R10)	; 36-71		n 44
	zfmsubpd zmm15, zmm15, zmm26, zmm3	;; R6*1 - R8 (newer negI8/SQRTHALF) I10*.924/.383 - R10 (I10)	; 36-71		n 

						;;				mul R12/I12 by w^3 = .383 - .924i
	zfmaddpd zmm3, zmm2, zmm26, zmm11	;; I6*1 + I8 (newer I6/SQRTHALF)  I12*.924/.383 + R12 (R12)	; 37-63		n 47
	zfnmaddpd zmm11, zmm11, zmm26, zmm2	;; I6 - I8*1 (newer R8/SQRTHALF)  I12 - R12*.924/.383 (I12)	; 37-63		n 44

						;;				R5/I5 becomes new R5/R7
	vblendmpd zmm8 {k6}, zmm12, zmm8	;; R5				R5				; 38		n 40
	vsubpd	zmm1 {k6}, zmm9, zmm1		;; I7				R11 = 0 - negR11		; 38-55		n 43

	vblendmpd zmm29 {k6}, zmm26, zmm31	;; 1, 1, 1, 1, 1, 1, 1		SQRTHALF
	vblendmpd zmm6 {k6}, zmm14, zmm6	;; R7				R7				; 39		n 41
	vmulpd	zmm17 {k6}, zmm29, zmm14	;; I3				I11*SQRTHALF			; 39-42		n 43

;; four aparts

	vaddpd	zmm2, zmm13, zmm8		;; R1 + R5 (final R1)		R1 + R5 (final R1)		; 40-66		n 
	vsubpd	zmm13, zmm13, zmm8		;; R1 - R5 (final R5)		R1 - R5 (final R5)		; 40-66		n 

	vaddpd	zmm8, zmm10, zmm6		;; R3 + R7 (final R3)		R3 + R7 (final R3)		; 41-64		n 
	vsubpd	zmm10, zmm10, zmm6		;; R3 - R7 (final R7)		R3 - R7 (final R7)		; 41-64		n 

						;;				R9/I9 becomes newer R9/R13
	vaddpd	zmm12 {k7}, zmm9, zmm4		;; I1 + I5 (final I1)		R9				; 42-67		n 
	vsubpd	zmm4 {k7}, zmm9, zmm4		;; I1 - I5 (final I5)		R13				; 42-67		n 

						;;				R11/I11 becomes newer R11/R15
						;;				mul R11/I11 by SQRTHALF - i*SQRTHALF
	zfmaddpd zmm6, zmm1, zmm29, zmm17	;; I3 + I7*1 (final I3)		I11 + R11*SQRTHALF (final R11)	; 43-65		n 
	zfnmaddpd zmm17, zmm1, zmm29, zmm17	;; I3 - I7*1 (final I7)		I11 - R11*SQRTHALF (final I11)	; 43-65		n 

						;;				R6/I6 becomes new R6/R8
	vblendmpd zmm9 {k6}, zmm5, zmm16	;; R6/SQRTHALF			R6/SQRTHALF			; 44		n 
	vblendmpd zmm1 {k6}, zmm11, zmm0	;; R8/SQRTHALF			R8/SQRTHALF			; 44		n 

	zfmaddpd zmm14, zmm9, zmm31, zmm7	;; R2 + R6*SQRTHALF (final R2)	R2 + R6 * SQRTHALF (final R2)	; 45-68		n 
	zfnmaddpd zmm9, zmm9, zmm31, zmm7	;; R2 - R6*SQRTHALF (final R6)	R2 - R6 * SQRTHALF (final R6)	; 45-68		n 

	zfmaddpd zmm7, zmm1, zmm31, zmm23	;; R4 + R8*SQRTHALF (final R4)	R4 + R8 * SQRTHALF (final R4)	; 46-70		n 
	zfnmaddpd zmm1, zmm1, zmm31, zmm23	;; R4 - R8*SQRTHALF (final R8)	R4 - R8 * SQRTHALF (final R8)	; 46-70		n 

						;;				R10/I10 becomes newer R10/R14
	vmovapd	zmm18, zmm15			;; negI8			R14
;; bug - hidden vmovapd uops
	zfmaddpd zmm5 {k7}, zmm3, zmm31, zmm0	;; I2 + I6*SQRTHALF (final I2)	R10				; 47-71		n 
	zfnmaddpd zmm15 {k7}, zmm3, zmm31, zmm0	;; I2 - I6*SQRTHALF (final I6)	R14				; 47-71		n 

						;;				R12/I12 becomes newer R12/R16
;; bug - hidden vmovapd uops
	zfmaddpd zmm11 {k7}, zmm18, zmm31, zmm16;; I4 + negI8*SQRTHALF (final I8)  R16				; 48-69		n 
	zfnmaddpd zmm3 {k7}, zmm18, zmm31, zmm16;; I4 - negI8*SQRTHALF (final I4)  R12				; 48-69		n 

	zstore	[srcreg+r8], zmm2		;; Save R1						; 30
	zstore	[srcreg+r8+64], zmm12		;; Save R9						; 30+1
	zstore	[srcreg+r8+d4], zmm13		;; Save R5						; 31+1
	zstore	[srcreg+r8+d4+64], zmm4		;; Save R13						; 31+2
	zstore	[srcreg+r8+d2], zmm8		;; Save R3						; 37
	zstore	[srcreg+r8+d2+64], zmm6		;; Save R11						; 37+1
	zstore	[srcreg+r8+d4+d2], zmm10	;; Save R7						; 38+1
	zstore	[srcreg+r8+d4+d2+64], zmm17	;; Save R15						; 38+2
	zstore	[srcreg+r8+d1], zmm14		;; Save R2						; 39+2
	zstore	[srcreg+r8+d1+64], zmm5		;; Save R10						; 39+3
	zstore	[srcreg+r8+d4+d1], zmm9		;; Save R6						; 40+3
	zstore	[srcreg+r8+d4+d1+64], zmm15	;; Save R14						; 40+4
	zstore	[srcreg+r8+d2+d1], zmm7		;; Save R4						; 41+4
	zstore	[srcreg+r8+d2+d1+64], zmm3	;; Save R12						; 41+5
	zstore	[srcreg+r8+d4+d2+d1], zmm1	;; Save R8						; 42+5
	zstore	[srcreg+r8+d4+d2+d1+64], zmm11	;; Save R16						; 42+6
	bump	srcreg, srcinc
	ENDM

zr8_sixteen_reals_eight_complex_with_mulf_preload MACRO
	;; preload would be silly for a macro that is executed only once
	ENDM
zr8_sixteen_reals_eight_complex_with_mulf MACRO srcreg,srcinc,d1,d2,d4
	LOCAL	orig, back_to_orig, fma_or_unfft, fma, addmul, submul, muladd, muladdhard, mulsub, mulsubhard, unfft

;GW these consts may need study/work
	vbroadcastsd zmm29, ZMM_P383
	vbroadcastsd zmm30, ZMM_P924_P383
	vbroadcastsd zmm31, ZMM_SQRTHALF
	mov	eax, 11111110b			;; We're pretty sure callers of this macro call zloop_init next
	kmovw	k7, eax				;; Set k7 to 11111110b						; 30		n 31
	knotw	k6, k7				;; Set k6 to 00000001b						; 31		n 50
	vbroadcastsd zmm22, ZMM_ONE
	vblendmpd zmm26 {k6}, zmm22, zmm30	;; 1, 1, 1, 1, 1, 1, 1		.924/.383			; 50		n 68
	vblendmpd zmm27 {k6}, zmm22, zmm29	;; 1, 1, 1, 1, 1, 1, 1		.383				; 51		n 87
	vmovapd zmm25 {k7}{z}, zmm22		;; 1, 1, 1, 1, 1, 1, 1		0				; 52		n 53
	vsubpd	zmm25 {k6}, zmm25, zmm31	;; 1, 1, 1, 1, 1, 1, 1		-SQRTHALF			; 53		n 86
	vblendmpd zmm28 {k6}, zmm31, zmm29	;; SQRTHALF, SQRTHALF, ...,	.383				; 54		n 96

	mov	al, mul4_opcode			;; Load the mul4_opcode
	cmp	al, 0				;; See if we need to do more than the original type-4 FFT multiply
	je	orig
	mov	r9, SRC2ARG			;; Distance to s2 arg from gwaddmul4 or gwsubmul4 or s3 arg in gwmuladd4 or gwmulsub4
	cmp	al, 2				;; Case off opcode
	jg	fma_or_unfft			;; 3,4,5=muladd,mulsub,unfft
	je	submul				;; 2=submul
	;jl	addmul				;; 1=addmul, fall through

addmul:	vmovapd	zmm2, [srcreg+d1][rbx]		;;		R2
	vmovapd	zmm24, [srcreg+rbp+d1]		;;		MemR2
	vaddpd	zmm24, zmm24, [srcreg+r9+d1]	;;		MemR2
	vmulpd	zmm17, zmm2, zmm24		;;		A2 = R2 * MemR2					; 1-4		n 5
	vmovapd	zmm3, [srcreg+d1+64][rbx]	;;		I2
	vmulpd	zmm16, zmm3, zmm24		;;		B2 = I2 * MemR2					; 1-4		n 5

	vmovapd	zmm13, [srcreg+d2][rbx]		;;		R3
	vmovapd	zmm24, [srcreg+rbp+d2]		;;		MemR3
	vaddpd	zmm24, zmm24, [srcreg+r9+d2]	;;		MemR3
	vmulpd	zmm19, zmm13, zmm24		;;		A3 = R3 * MemR3					; 2-5		n 6
	vmovapd	zmm15, [srcreg+d2+64][rbx]	;;		I3
	vmulpd	zmm18, zmm15, zmm24		;;		B3 = I3 * MemR3					; 2-5		n 6

	vmovapd	zmm9, [srcreg+d2+d1][rbx]	;;		R4
	vmovapd	zmm24, [srcreg+rbp+d2+d1]	;;		MemR4
	vaddpd	zmm24, zmm24, [srcreg+r9+d2+d1];;		MemR4
	vmulpd	zmm21, zmm9, zmm24		;;		A4 = R4 * MemR4					; 3-6		n 8
	vmovapd	zmm12, [srcreg+d2+d1+64][rbx]	;;		I4
	vmulpd	zmm20, zmm12, zmm24		;;		B4 = I4 * MemR4					; 3-6		n 8

	vmovapd	zmm1, [srcreg][rbx]		;; R1				R1a
	vmovapd	zmm24, [srcreg+rbp]		;; MemR1			MemR1a
	vaddpd	zmm24, zmm24, [srcreg+r9]	;; MemR1			MemR1a
	vmulpd	zmm23, zmm1, zmm24		;; A1 = R1 * MemR1		R1a*MemR1a (R1a)		; 4-7		n 10
	vmovapd	zmm11, [srcreg+64][rbx]		;; I1				R1b
	vmulpd	zmm22 {k7}{z}, zmm11, zmm24	;; B1 = I1 * MemR1		0				; 4-7		n 10

	vmovapd zmm24, [srcreg+rbp+d1+64]	;;		MemI2
	vaddpd	zmm24, zmm24, [srcreg+r9+d1+64];;		MemI2
	zfnmaddpd zmm17, zmm3, zmm24, zmm17	;;		A2 - I2 * MemI2 (R2)				; 5-8		n 10
	zfmaddpd zmm16, zmm2, zmm24, zmm16	;;		B2 + R2 * MemI2 (I2)				; 5-8		n 18

	vmovapd zmm24, [srcreg+rbp+d2+64]	;;		MemI3
	vaddpd	zmm24, zmm24, [srcreg+r9+d2+64];;		MemI3
	zfnmaddpd zmm19, zmm15, zmm24, zmm19	;;		A3 - I3 * MemI3 (R3 & R5)			; 6-9		n 19
	zfmaddpd zmm18, zmm13, zmm24, zmm18	;;		B3 + R3 * MemI3 (I3 & I5)			; 6-9		n 20

	vmovapd	zmm10, [srcreg+d4][rbx]		;;		R5
	vmovapd	zmm24, [srcreg+rbp+d4]		;;		MemR5
	vaddpd	zmm24, zmm24, [srcreg+r9+d4]	;;		MemR5
	vmulpd	zmm3, zmm10, zmm24		;;		A5 = R5 * MemR5					; 7-10		n 12
	vmovapd	zmm7, [srcreg+d4+64][rbx]	;;		I5
	vmulpd	zmm2, zmm7, zmm24		;;		B5 = I5 * MemR5					; 7-10		n 13

	vmovapd zmm24, [srcreg+rbp+d2+d1+64]	;;		MemI4
	vaddpd	zmm24, zmm24, [srcreg+r9+d2+d1+64];;		MemI4
	zfnmaddpd zmm21, zmm12, zmm24, zmm21	;;		A4 - I4 * MemI4 (R4 & R6)			; 8-11		n 19
	zfmaddpd zmm20, zmm9, zmm24, zmm20	;;		B4 + R4 * MemI4 (I4 & I6)			; 8-11		n 20
	vmovsd	Q [srcreg-16], xmm23		;;				Save product of FFT sums	; 8

	vmovapd	zmm6, [srcreg+d4+d1][rbx]	;;		R6
	vmovapd	zmm24, [srcreg+rbp+d4+d1]	;;		MemR6
	vaddpd	zmm24, zmm24, [srcreg+r9+d4+d1];;		MemR6
	vmulpd	zmm15, zmm6, zmm24		;;		A6 = R6 * MemR6					; 9-12		n 14
	vmovapd	zmm8, [srcreg+d4+d1+64][rbx]	;;		I6
	vmulpd	zmm13, zmm8, zmm24		;;		B6 = I6 * MemR6					; 9-12		n 15

	vmovapd	zmm12, zmm17			;;		Copy R2
	vmovapd zmm24, [srcreg+rbp+64]		;; MemI1			MemR1b
	vaddpd	zmm24, zmm24, [srcreg+r9+64]	;; MemI1			MemR1b
	zfnmaddpd zmm23 {k7}, zmm11, zmm24, zmm23;; A1 - I1 * MemI1 (R1)	R1a				; 10-13		n 16
	zfmaddpd zmm22 {k7}{z}, zmm1, zmm24, zmm22;; B1 + R1 * MemI1 (I1)	0				; 10-13		n 18

	vmulpd	zmm12 {k6}, zmm11, zmm24	;; R2				R1b * MemR1b (R1b)		; 11-14		n 16

	vmovapd	zmm14, [srcreg+d4+d2][rbx]	;;		R7
	vmovapd	zmm24, [srcreg+rbp+d4+d2]	;;		MemR7
	vaddpd	zmm24, zmm24, [srcreg+r9+d4+d2];;		MemR7
	vmulpd	zmm9, zmm14, zmm24		;;		A7 = R7 * MemR7					; 11-14		n 15
	vmovapd	zmm4, [srcreg+d4+d2+64][rbx]	;;		I7
	vmulpd	zmm1, zmm4, zmm24		;;		B7 = I7 * MemR7					; 12-15		n 16

	vmovapd zmm24, [srcreg+rbp+d4+64]	;;		MemI5
	vaddpd	zmm24, zmm24, [srcreg+r9+d4+64];;		MemI5
	zfnmaddpd zmm3, zmm7, zmm24, zmm3	;;		A5 - I5 * MemI5 (R5 & R9)			; 12-15		n 21
	zfmaddpd zmm2, zmm10, zmm24, zmm2	;;		B5 + R5 * MemI5 (I5 & I9)			; 13-16		n 22

	vmovapd	zmm0, [srcreg+d4+d2+d1][rbx]	;;		R8
	vmovapd	zmm24, [srcreg+rbp+d4+d2+d1]	;;		MemR8
	vaddpd	zmm24, zmm24, [srcreg+r9+d4+d2+d1];;		MemR8
	vmulpd	zmm11, zmm0, zmm24		;;		A8 = R8 * MemR8					; 13-16		n 17
	vmovapd	zmm5, [srcreg+d4+d2+d1+64][rbx] ;;		I8
	vmulpd	zmm7, zmm5, zmm24		;;		B8 = I8 * MemR8					; 14-17		n 18

	vmovapd zmm24, [srcreg+rbp+d4+d1+64]	;;		MemI6
	vaddpd	zmm24, zmm24, [srcreg+r9+d4+d1+64];;		MemI6
	zfnmaddpd zmm15, zmm8, zmm24, zmm15	;;		A6 - I6 * MemI6 (R6 & R10)			; 14-17		n 21
	zfmaddpd zmm13, zmm6, zmm24, zmm13	;;		B6 + R6 * MemI6 (I6 & I10)			; 15-18		n 22

	vmovapd zmm24, [srcreg+rbp+d4+d2+64]	;;		MemI7
	vaddpd	zmm24, zmm24, [srcreg+r9+d4+d2+64];;		MemI7
	zfnmaddpd zmm9, zmm4, zmm24, zmm9	;;		A7 - I7 * MemI7 (R7 & R11)			; 15-18		n 23
	zfmaddpd zmm1, zmm14, zmm24, zmm1	;;		B7 + R7 * MemI7 (I7 & I11)			; 16-19		n 24

						;;				R1a/R1b becomes R1/R2
	vaddpd	zmm10, zmm23, zmm12		;; R1 + R2 (new R1)		R1 + R2 (new R1/HALF)		; 16-19		n 26
	vsubpd	zmm23, zmm23, zmm12		;; R1 - R2 (new R2)		R1 - R2 (new R2/HALF)		; 17-20		n 28

	vmovapd zmm24, [srcreg+rbp+d4+d2+d1+64] ;;		MemI8
	vaddpd	zmm24, zmm24, [srcreg+r9+d4+d2+d1+64];;		MemI8
	zfnmaddpd zmm11, zmm5, zmm24, zmm11	;;		A8 - I8 * MemI8 (R8 & R12)			; 17-20		n 23
	zfmaddpd zmm7, zmm0, zmm24, zmm7	;;		B8 + R8 * MemI8 (I8 & I12)			; 18-21		n 24

	jmp	back_to_orig

submul:	vmovapd	zmm2, [srcreg+d1][rbx]		;;		R2
	vmovapd	zmm24, [srcreg+rbp+d1]		;;		MemR2
	vsubpd	zmm24, zmm24, [srcreg+r9+d1]	;;		MemR2
	vmulpd	zmm17, zmm2, zmm24		;;		A2 = R2 * MemR2					; 1-4		n 5
	vmovapd	zmm3, [srcreg+d1+64][rbx]	;;		I2
	vmulpd	zmm16, zmm3, zmm24		;;		B2 = I2 * MemR2					; 1-4		n 5

	vmovapd	zmm13, [srcreg+d2][rbx]		;;		R3
	vmovapd	zmm24, [srcreg+rbp+d2]		;;		MemR3
	vsubpd	zmm24, zmm24, [srcreg+r9+d2]	;;		MemR3
	vmulpd	zmm19, zmm13, zmm24		;;		A3 = R3 * MemR3					; 2-5		n 6
	vmovapd	zmm15, [srcreg+d2+64][rbx]	;;		I3
	vmulpd	zmm18, zmm15, zmm24		;;		B3 = I3 * MemR3					; 2-5		n 6

	vmovapd	zmm9, [srcreg+d2+d1][rbx]	;;		R4
	vmovapd	zmm24, [srcreg+rbp+d2+d1]	;;		MemR4
	vsubpd	zmm24, zmm24, [srcreg+r9+d2+d1];;		MemR4
	vmulpd	zmm21, zmm9, zmm24		;;		A4 = R4 * MemR4					; 3-6		n 8
	vmovapd	zmm12, [srcreg+d2+d1+64][rbx]	;;		I4
	vmulpd	zmm20, zmm12, zmm24		;;		B4 = I4 * MemR4					; 3-6		n 8

	vmovapd	zmm1, [srcreg][rbx]		;; R1				R1a
	vmovapd	zmm24, [srcreg+rbp]		;; MemR1			MemR1a
	vsubpd	zmm24, zmm24, [srcreg+r9]	;; MemR1			MemR1a
	vmulpd	zmm23, zmm1, zmm24		;; A1 = R1 * MemR1		R1a*MemR1a (R1a)		; 4-7		n 10
	vmovapd	zmm11, [srcreg+64][rbx]		;; I1				R1b
	vmulpd	zmm22 {k7}{z}, zmm11, zmm24	;; B1 = I1 * MemR1		0				; 4-7		n 10

	vmovapd zmm24, [srcreg+rbp+d1+64]	;;		MemI2
	vsubpd	zmm24, zmm24, [srcreg+r9+d1+64];;		MemI2
	zfnmaddpd zmm17, zmm3, zmm24, zmm17	;;		A2 - I2 * MemI2 (R2)				; 5-8		n 10
	zfmaddpd zmm16, zmm2, zmm24, zmm16	;;		B2 + R2 * MemI2 (I2)				; 5-8		n 18

	vmovapd zmm24, [srcreg+rbp+d2+64]	;;		MemI3
	vsubpd	zmm24, zmm24, [srcreg+r9+d2+64];;		MemI3
	zfnmaddpd zmm19, zmm15, zmm24, zmm19	;;		A3 - I3 * MemI3 (R3 & R5)			; 6-9		n 19
	zfmaddpd zmm18, zmm13, zmm24, zmm18	;;		B3 + R3 * MemI3 (I3 & I5)			; 6-9		n 20

	vmovapd	zmm10, [srcreg+d4][rbx]		;;		R5
	vmovapd	zmm24, [srcreg+rbp+d4]		;;		MemR5
	vsubpd	zmm24, zmm24, [srcreg+r9+d4]	;;		MemR5
	vmulpd	zmm3, zmm10, zmm24		;;		A5 = R5 * MemR5					; 7-10		n 12
	vmovapd	zmm7, [srcreg+d4+64][rbx]	;;		I5
	vmulpd	zmm2, zmm7, zmm24		;;		B5 = I5 * MemR5					; 7-10		n 13

	vmovapd zmm24, [srcreg+rbp+d2+d1+64]	;;		MemI4
	vsubpd	zmm24, zmm24, [srcreg+r9+d2+d1+64];;		MemI4
	zfnmaddpd zmm21, zmm12, zmm24, zmm21	;;		A4 - I4 * MemI4 (R4 & R6)			; 8-11		n 19
	zfmaddpd zmm20, zmm9, zmm24, zmm20	;;		B4 + R4 * MemI4 (I4 & I6)			; 8-11		n 20
	vmovsd	Q [srcreg-16], xmm23		;;				Save product of FFT sums	; 8

	vmovapd	zmm6, [srcreg+d4+d1][rbx]	;;		R6
	vmovapd	zmm24, [srcreg+rbp+d4+d1]	;;		MemR6
	vsubpd	zmm24, zmm24, [srcreg+r9+d4+d1];;		MemR6
	vmulpd	zmm15, zmm6, zmm24		;;		A6 = R6 * MemR6					; 9-12		n 14
	vmovapd	zmm8, [srcreg+d4+d1+64][rbx]	;;		I6
	vmulpd	zmm13, zmm8, zmm24		;;		B6 = I6 * MemR6					; 9-12		n 15

	vmovapd	zmm12, zmm17			;;		Copy R2
	vmovapd zmm24, [srcreg+rbp+64]		;; MemI1			MemR1b
	vsubpd	zmm24, zmm24, [srcreg+r9+64]	;; MemI1			MemR1b
	zfnmaddpd zmm23 {k7}, zmm11, zmm24, zmm23;; A1 - I1 * MemI1 (R1)	R1a				; 10-13		n 16
	zfmaddpd zmm22 {k7}{z}, zmm1, zmm24, zmm22;; B1 + R1 * MemI1 (I1)	0				; 10-13		n 18

	vmulpd	zmm12 {k6}, zmm11, zmm24	;; R2				R1b * MemR1b (R1b)		; 11-14		n 16

	vmovapd	zmm14, [srcreg+d4+d2][rbx]	;;		R7
	vmovapd	zmm24, [srcreg+rbp+d4+d2]	;;		MemR7
	vsubpd	zmm24, zmm24, [srcreg+r9+d4+d2];;		MemR7
	vmulpd	zmm9, zmm14, zmm24		;;		A7 = R7 * MemR7					; 11-14		n 15
	vmovapd	zmm4, [srcreg+d4+d2+64][rbx]	;;		I7
	vmulpd	zmm1, zmm4, zmm24		;;		B7 = I7 * MemR7					; 12-15		n 16

	vmovapd zmm24, [srcreg+rbp+d4+64]	;;		MemI5
	vsubpd	zmm24, zmm24, [srcreg+r9+d4+64];;		MemI5
	zfnmaddpd zmm3, zmm7, zmm24, zmm3	;;		A5 - I5 * MemI5 (R5 & R9)			; 12-15		n 21
	zfmaddpd zmm2, zmm10, zmm24, zmm2	;;		B5 + R5 * MemI5 (I5 & I9)			; 13-16		n 22

	vmovapd	zmm0, [srcreg+d4+d2+d1][rbx]	;;		R8
	vmovapd	zmm24, [srcreg+rbp+d4+d2+d1]	;;		MemR8
	vsubpd	zmm24, zmm24, [srcreg+r9+d4+d2+d1];;		MemR8
	vmulpd	zmm11, zmm0, zmm24		;;		A8 = R8 * MemR8					; 13-16		n 17
	vmovapd	zmm5, [srcreg+d4+d2+d1+64][rbx] ;;		I8
	vmulpd	zmm7, zmm5, zmm24		;;		B8 = I8 * MemR8					; 14-17		n 18

	vmovapd zmm24, [srcreg+rbp+d4+d1+64]	;;		MemI6
	vsubpd	zmm24, zmm24, [srcreg+r9+d4+d1+64];;		MemI6
	zfnmaddpd zmm15, zmm8, zmm24, zmm15	;;		A6 - I6 * MemI6 (R6 & R10)			; 14-17		n 21
	zfmaddpd zmm13, zmm6, zmm24, zmm13	;;		B6 + R6 * MemI6 (I6 & I10)			; 15-18		n 22

	vmovapd zmm24, [srcreg+rbp+d4+d2+64]	;;		MemI7
	vsubpd	zmm24, zmm24, [srcreg+r9+d4+d2+64];;		MemI7
	zfnmaddpd zmm9, zmm4, zmm24, zmm9	;;		A7 - I7 * MemI7 (R7 & R11)			; 15-18		n 23
	zfmaddpd zmm1, zmm14, zmm24, zmm1	;;		B7 + R7 * MemI7 (I7 & I11)			; 16-19		n 24

						;;				R1a/R1b becomes R1/R2
	vaddpd	zmm10, zmm23, zmm12		;; R1 + R2 (new R1)		R1 + R2 (new R1/HALF)		; 16-19		n 26
	vsubpd	zmm23, zmm23, zmm12		;; R1 - R2 (new R2)		R1 - R2 (new R2/HALF)		; 17-20		n 28

	vmovapd zmm24, [srcreg+rbp+d4+d2+d1+64] ;;		MemI8
	vsubpd	zmm24, zmm24, [srcreg+r9+d4+d2+d1+64];;		MemI8
	zfnmaddpd zmm11, zmm5, zmm24, zmm11	;;		A8 - I8 * MemI8 (R8 & R12)			; 17-20		n 23
	zfmaddpd zmm7, zmm0, zmm24, zmm7	;;		B8 + R8 * MemI8 (I8 & I12)			; 18-21		n 24

	jmp	back_to_orig

fma_or_unfft:
	cmp	al, 4				;; Case off opcode
	jbe	fma				;; 3=muladd, 4=mulsub, fma MUST NOT ALTER CONDITION CODES!
	;;ja	unfft				;; 5=unfft

unfft:	vmovapd	zmm23, [srcreg][rbx]		;; R1				R1a
	vmovapd	zmm11, [srcreg+64][rbx]		;; I1				R1b
	vmovapd	zmm22 {k7}{z}, zmm11		;; I1				0
	vmovsd	Q [srcreg-16], xmm23		;;				Save product of FFT sums

	vmovapd	zmm17, [srcreg+d1][rbx]		;;		R2
	vblendmpd zmm12 {k6}, zmm17, zmm11	;; R2				R1b

						;;				R1a/R1b becomes R1/R2
	vaddpd	zmm10, zmm23, zmm12		;; R1 + R2 (new R1)		R1 + R2 (new R1/HALF)		; 16-19		n 26
	vsubpd	zmm23, zmm23, zmm12		;; R1 - R2 (new R2)		R1 - R2 (new R2/HALF)		; 17-20		n 28

	vmovapd	zmm16, [srcreg+d1+64][rbx]	;;		I2
	vmovapd	zmm19, [srcreg+d2][rbx]		;;		R3 & R5
	vmovapd	zmm18, [srcreg+d2+64][rbx]	;;		I3 & I5
	vmovapd	zmm21, [srcreg+d2+d1][rbx]	;;		R4 & R6
	vmovapd	zmm20, [srcreg+d2+d1+64][rbx]	;;		I4 & I6
	vmovapd	zmm3, [srcreg+d4][rbx]		;;		R5 & R9
	vmovapd	zmm2, [srcreg+d4+64][rbx]	;;		I5 & I9
	vmovapd	zmm15, [srcreg+d4+d1][rbx]	;;		R6 & R10
	vmovapd	zmm13, [srcreg+d4+d1+64][rbx]	;;		I6 & I10
	vmovapd	zmm9, [srcreg+d4+d2][rbx]	;;		R7 & R11
	vmovapd	zmm1, [srcreg+d4+d2+64][rbx]	;;		I7 & I11
	vmovapd	zmm11, [srcreg+d4+d2+d1][rbx]	;;		R8 & R12
	vmovapd	zmm7, [srcreg+d4+d2+d1+64][rbx] ;;		I8 & I12
	jmp	back_to_orig

fma:	vmovapd	zmm2, [srcreg+d1][rbx]		;;		R2
	vmovapd	zmm24, [srcreg+rbp+d1]		;;		MemR2
	vmulpd	zmm17, zmm2, zmm24		;;		A2 = R2 * MemR2					; 1-4		n 5
	vmovapd	zmm3, [srcreg+d1+64][rbx]	;;		I2
	vmulpd	zmm16, zmm3, zmm24		;;		B2 = I2 * MemR2					; 1-4		n 5

	vmovapd	zmm13, [srcreg+d2][rbx]		;;		R3
	vmovapd	zmm24, [srcreg+rbp+d2]		;;		MemR3
	vmulpd	zmm19, zmm13, zmm24		;;		A3 = R3 * MemR3					; 2-5		n 6
	vmovapd	zmm15, [srcreg+d2+64][rbx]	;;		I3
	vmulpd	zmm18, zmm15, zmm24		;;		B3 = I3 * MemR3					; 2-5		n 6

	vmovapd	zmm9, [srcreg+d2+d1][rbx]	;;		R4
	vmovapd	zmm24, [srcreg+rbp+d2+d1]	;;		MemR4
	vmulpd	zmm21, zmm9, zmm24		;;		A4 = R4 * MemR4					; 3-6		n 8
	vmovapd	zmm12, [srcreg+d2+d1+64][rbx]	;;		I4
	vmulpd	zmm20, zmm12, zmm24		;;		B4 = I4 * MemR4					; 3-6		n 8

	vmovapd	zmm1, [srcreg][rbx]		;; R1				R1a
	vmovapd	zmm24, [srcreg+rbp]		;; MemR1			MemR1a
	vmulpd	zmm23, zmm1, zmm24		;; A1 = R1 * MemR1		R1a*MemR1a (R1a)		; 4-7		n 10
	vmovapd	zmm11, [srcreg+64][rbx]		;; I1				R1b
	vmulpd	zmm22 {k7}{z}, zmm11, zmm24	;; B1 = I1 * MemR1		0				; 4-7		n 10

	vmovapd zmm24, [srcreg+rbp+d1+64]	;;		MemI2
	zfnmaddpd zmm17, zmm3, zmm24, zmm17	;;		A2 - I2 * MemI2 (R2)				; 5-8		n 10
	zfmaddpd zmm16, zmm2, zmm24, zmm16	;;		B2 + R2 * MemI2 (I2)				; 5-8		n 18

	vmovapd zmm24, [srcreg+rbp+d2+64]	;;		MemI3
	zfnmaddpd zmm19, zmm15, zmm24, zmm19	;;		A3 - I3 * MemI3 (R3 & R5)			; 6-9		n 19
	zfmaddpd zmm18, zmm13, zmm24, zmm18	;;		B3 + R3 * MemI3 (I3 & I5)			; 6-9		n 20

	vmovapd	zmm10, [srcreg+d4][rbx]		;;		R5
	vmovapd	zmm24, [srcreg+rbp+d4]		;;		MemR5
	vmulpd	zmm3, zmm10, zmm24		;;		A5 = R5 * MemR5					; 7-10		n 12
	vmovapd	zmm7, [srcreg+d4+64][rbx]	;;		I5
	vmulpd	zmm2, zmm7, zmm24		;;		B5 = I5 * MemR5					; 7-10		n 13

	vmovapd zmm24, [srcreg+rbp+d2+d1+64]	;;		MemI4
	zfnmaddpd zmm21, zmm12, zmm24, zmm21	;;		A4 - I4 * MemI4 (R4 & R6)			; 8-11		n 19
	zfmaddpd zmm20, zmm9, zmm24, zmm20	;;		B4 + R4 * MemI4 (I4 & I6)			; 8-11		n 20
	vmovsd	Q [srcreg-16], xmm23		;;				Save product of FFT sums	; 8

	vmovapd	zmm6, [srcreg+d4+d1][rbx]	;;		R6
	vmovapd	zmm24, [srcreg+rbp+d4+d1]	;;		MemR6
	vmulpd	zmm15, zmm6, zmm24		;;		A6 = R6 * MemR6					; 9-12		n 14
	vmovapd	zmm8, [srcreg+d4+d1+64][rbx]	;;		I6
	vmulpd	zmm13, zmm8, zmm24		;;		B6 = I6 * MemR6					; 9-12		n 15

	mov	r10, SRC3ARG			;; Load distance to FFT(1)
	je	mulsub				;; 4=mulsub
	;jb	muladd				;; 3=muladd, fall through

muladd:	test	r10, 3				;; Check if we need to multiply add-in by FFT(1)
	jz	muladdhard			;; Yes, do it the hard way

	vaddpd	zmm17, zmm17, [srcreg+r9+d1]		;; R2 = R2 + MemR2
	vaddpd	zmm16, zmm16, [srcreg+r9+d1+64]		;; I2 = I2 + MemI2
	vaddpd	zmm19, zmm19, [srcreg+r9+d2]		;; R3 = R3 + MemR3
	vaddpd	zmm18, zmm18, [srcreg+r9+d2+64]		;; I3 = I3 + MemI3
	vaddpd	zmm21, zmm21, [srcreg+r9+d2+d1]		;; R4 = R4 + MemR4
	vaddpd	zmm20, zmm20, [srcreg+r9+d2+d1+64]	;; I4 = I4 + MemI4

	vmovapd	zmm12, zmm17			;;		Copy R2
	vmovapd zmm24, [srcreg+rbp+64]		;; MemI1			MemR1b
	zfnmaddpd zmm23 {k7}, zmm11, zmm24, zmm23;; A1 - I1 * MemI1 (R1)	R1a				; 10-13		n 16
	zfmaddpd zmm22 {k7}{z}, zmm1, zmm24, zmm22;; B1 + R1 * MemI1 (I1)	0				; 10-13		n 18

	vmulpd	zmm12 {k6}, zmm11, zmm24	;; R2				R1b * MemR1b (R1b)		; 11-14		n 16

	vmovapd	zmm14, [srcreg+d4+d2][rbx]	;;		R7
	vmovapd	zmm24, [srcreg+rbp+d4+d2]	;;		MemR7
	vmulpd	zmm9, zmm14, zmm24		;;		A7 = R7 * MemR7					; 11-14		n 15
	vmovapd	zmm4, [srcreg+d4+d2+64][rbx]	;;		I7
	vmulpd	zmm1, zmm4, zmm24		;;		B7 = I7 * MemR7					; 12-15		n 16

	vmovapd zmm24, [srcreg+rbp+d4+64]	;;		MemI5
	zfnmaddpd zmm3, zmm7, zmm24, zmm3	;;		A5 - I5 * MemI5 (R5 & R9)			; 12-15		n 21
	zfmaddpd zmm2, zmm10, zmm24, zmm2	;;		B5 + R5 * MemI5 (I5 & I9)			; 13-16		n 22

	vmovapd	zmm0, [srcreg+d4+d2+d1][rbx]	;;		R8
	vmovapd	zmm24, [srcreg+rbp+d4+d2+d1]	;;		MemR8
	vmulpd	zmm11, zmm0, zmm24		;;		A8 = R8 * MemR8					; 13-16		n 17
	vmovapd	zmm5, [srcreg+d4+d2+d1+64][rbx] ;;		I8
	vmulpd	zmm7, zmm5, zmm24		;;		B8 = I8 * MemR8					; 14-17		n 18

	vmovapd zmm24, [srcreg+rbp+d4+d1+64]	;;		MemI6
	zfnmaddpd zmm15, zmm8, zmm24, zmm15	;;		A6 - I6 * MemI6 (R6 & R10)			; 14-17		n 21
	zfmaddpd zmm13, zmm6, zmm24, zmm13	;;		B6 + R6 * MemI6 (I6 & I10)			; 15-18		n 22

	vmovapd zmm24, [srcreg+rbp+d4+d2+64]	;;		MemI7
	zfnmaddpd zmm9, zmm4, zmm24, zmm9	;;		A7 - I7 * MemI7 (R7 & R11)			; 15-18		n 23
	zfmaddpd zmm1, zmm14, zmm24, zmm1	;;		B7 + R7 * MemI7 (I7 & I11)			; 16-19		n 24

	vaddpd	zmm23, zmm23, [srcreg+r9]		;; R1 = R1 + MemR1
	vaddpd	zmm12{k6}, zmm12, [srcreg+r9+64]	;; R1b = R1b + MemI1
	vaddpd	zmm22{k7}, zmm22, [srcreg+r9+64]	;; I1 = I1 + MemI1
	vaddpd	zmm3, zmm3, [srcreg+r9+d4]		;; R5 = R5 + MemR5
	vaddpd	zmm2, zmm2, [srcreg+r9+d4+64]		;; I5 = I5 + MemI5

						;;				R1a/R1b becomes R1/R2
	vaddpd	zmm10, zmm23, zmm12		;; R1 + R2 (new R1)		R1 + R2 (new R1/HALF)		; 16-19		n 26
	vsubpd	zmm23, zmm23, zmm12		;; R1 - R2 (new R2)		R1 - R2 (new R2/HALF)		; 17-20		n 28

	vmovapd zmm24, [srcreg+rbp+d4+d2+d1+64] ;;		MemI8
	zfnmaddpd zmm11, zmm5, zmm24, zmm11	;;		A8 - I8 * MemI8 (R8 & R12)			; 17-20		n 23
	zfmaddpd zmm7, zmm0, zmm24, zmm7	;;		B8 + R8 * MemI8 (I8 & I12)			; 18-21		n 24

	vaddpd	zmm15, zmm15, [srcreg+r9+d4+d1]		;; R6 = R6 + MemR6
	vaddpd	zmm13, zmm13, [srcreg+r9+d4+d1+64]	;; I6 = I6 + MemI6
	vaddpd	zmm9, zmm9, [srcreg+r9+d4+d2]		;; R7 = R7 + MemR7
	vaddpd	zmm1, zmm1, [srcreg+r9+d4+d2+64]	;; I7 = I7 + MemI7
	vaddpd	zmm11, zmm11, [srcreg+r9+d4+d2+d1]	;; R8 = R8 + MemR8
	vaddpd	zmm7, zmm7, [srcreg+r9+d4+d2+d1+64]	;; I8 = I8 + MemI8

	jmp	back_to_orig

muladdhard:
	vmovapd	zmm9, [srcreg+r9+d1]			;; MemR2
	vmovapd	zmm12, [srcreg+r10+d1]			;; MemR2#2
	vmovapd	zmm24, [srcreg+r10+d1+64]		;; MemI2#2
	zfmaddpd zmm17, zmm9, zmm12, zmm17		;; R2 = R2 + MemR2*MemR2#2
	zfmaddpd zmm16, zmm9, zmm24, zmm16		;; I2 = I2 + MemR2*MemI2#2
	vmovapd	zmm9, [srcreg+r9+d1+64]			;; MemI2
	zfnmaddpd zmm17, zmm9, zmm24, zmm17		;; R2 = R2 - MemI2*MemI2#2
	zfmaddpd zmm16, zmm9, zmm12, zmm16		;; I2 = I2 + MemI2*MemR2#2

	vmovapd	zmm9, [srcreg+r9+d2]			;; MemR3
	vmovapd	zmm12, [srcreg+r10+d2]			;; MemR3#2
	vmovapd	zmm24, [srcreg+r10+d2+64]		;; MemI3#2
	zfmaddpd zmm19, zmm9, zmm12, zmm19		;; R3 = R3 + MemR3*MemR3#2
	zfmaddpd zmm18, zmm9, zmm24, zmm18		;; I3 = I3 + MemR3*MemI3#2
	vmovapd	zmm9, [srcreg+r9+d2+64]			;; MemI3
	zfnmaddpd zmm19, zmm9, zmm24, zmm19		;; R3 = R3 - MemI3*MemI3#2
	zfmaddpd zmm18, zmm9, zmm12, zmm18		;; I3 = I3 + MemI3*MemR3#2

	vmovapd	zmm9, [srcreg+r9+d2+d1]			;; MemR4
	vmovapd	zmm12, [srcreg+r10+d2+d1]		;; MemR4#2
	vmovapd	zmm24, [srcreg+r10+d2+d1+64]		;; MemI4#2
	zfmaddpd zmm21, zmm9, zmm12, zmm21		;; R4 = R4 + MemR4*MemR4#2
	zfmaddpd zmm20, zmm9, zmm24, zmm20		;; I4 = I4 + MemR4*MemI4#2
	vmovapd	zmm9, [srcreg+r9+d2+d1+64]		;; MemI4
	zfnmaddpd zmm21, zmm9, zmm24, zmm21		;; R4 = R4 - MemI4*MemI4#2
	zfmaddpd zmm20, zmm9, zmm12, zmm20		;; I4 = I4 + MemI4*MemR4#2

	vmovapd	zmm12, zmm17			;;		Copy R2
	vmovapd zmm24, [srcreg+rbp+64]		;; MemI1			MemR1b
	zfnmaddpd zmm23 {k7}, zmm11, zmm24, zmm23;; A1 - I1 * MemI1 (R1)	R1a				; 10-13		n 16
	zfmaddpd zmm22 {k7}{z}, zmm1, zmm24, zmm22;; B1 + R1 * MemI1 (I1)	0				; 10-13		n 18

	vmulpd	zmm12 {k6}, zmm11, zmm24	;; R2				R1b * MemR1b (R1b)		; 11-14		n 16

	vmovapd	zmm14, [srcreg+d4+d2][rbx]	;;		R7
	vmovapd	zmm24, [srcreg+rbp+d4+d2]	;;		MemR7
	vmulpd	zmm9, zmm14, zmm24		;;		A7 = R7 * MemR7					; 11-14		n 15
	vmovapd	zmm4, [srcreg+d4+d2+64][rbx]	;;		I7
	vmulpd	zmm1, zmm4, zmm24		;;		B7 = I7 * MemR7					; 12-15		n 16

	vmovapd zmm24, [srcreg+rbp+d4+64]	;;		MemI5
	zfnmaddpd zmm3, zmm7, zmm24, zmm3	;;		A5 - I5 * MemI5 (R5 & R9)			; 12-15		n 21
	zfmaddpd zmm2, zmm10, zmm24, zmm2	;;		B5 + R5 * MemI5 (I5 & I9)			; 13-16		n 22

	vmovapd	zmm0, [srcreg+d4+d2+d1][rbx]	;;		R8
	vmovapd	zmm24, [srcreg+rbp+d4+d2+d1]	;;		MemR8
	vmulpd	zmm11, zmm0, zmm24		;;		A8 = R8 * MemR8					; 13-16		n 17
	vmovapd	zmm5, [srcreg+d4+d2+d1+64][rbx] ;;		I8
	vmulpd	zmm7, zmm5, zmm24		;;		B8 = I8 * MemR8					; 14-17		n 18

	vmovapd zmm24, [srcreg+rbp+d4+d1+64]	;;		MemI6
	zfnmaddpd zmm15, zmm8, zmm24, zmm15	;;		A6 - I6 * MemI6 (R6 & R10)			; 14-17		n 21
	zfmaddpd zmm13, zmm6, zmm24, zmm13	;;		B6 + R6 * MemI6 (I6 & I10)			; 15-18		n 22

	vmovapd zmm24, [srcreg+rbp+d4+d2+64]	;;		MemI7
	zfnmaddpd zmm9, zmm4, zmm24, zmm9	;;		A7 - I7 * MemI7 (R7 & R11)			; 15-18		n 23
	zfmaddpd zmm1, zmm14, zmm24, zmm1	;;		B7 + R7 * MemI7 (I7 & I11)			; 16-19		n 24

	vmovapd	zmm10, [srcreg+r9]			;; MemR1
	vmovapd	zmm6, [srcreg+r10]			;; MemR1#2
	vmovapd	zmm8, [srcreg+r10+64]			;; MemI1#2
	zfmaddpd zmm23, zmm10, zmm6, zmm23		;; R1 = R1 + MemR1*MemR1#2
	zfmaddpd zmm22{k7}{z}, zmm10, zmm8, zmm22	;; I1 = I1 + MemR1*MemI1#2
	vmovapd	zmm10, [srcreg+r9+64]			;; MemI1
	zfnmaddpd zmm23{k7}, zmm10, zmm8, zmm23		;; R1 = R1 - MemI1*MemI1#2
	zfmaddpd zmm12{k6}, zmm10, zmm8, zmm12		;; R1b = R1b + MemI1*MemI1#2
	zfmaddpd zmm22{k7}{z}, zmm10, zmm6, zmm22	;; I1 = I1 + MemI1*MemR1#2

	vmovapd	zmm10, [srcreg+r9+d4]			;; MemR5
	vmovapd	zmm6, [srcreg+r10+d4]			;; MemR5#2
	vmovapd	zmm8, [srcreg+r10+d4+64]		;; MemI5#2
	zfmaddpd zmm3, zmm10, zmm6, zmm3		;; R5 = R5 + MemR5*MemR5#2
	zfmaddpd zmm2, zmm10, zmm8, zmm2		;; I5 = I5 + MemR5*MemI5#2
	vmovapd	zmm10, [srcreg+r9+d4+64]		;; MemI5
	zfnmaddpd zmm3, zmm10, zmm8, zmm3		;; R5 = R5 - MemI5*MemI5#2
	zfmaddpd zmm2, zmm10, zmm6, zmm2		;; I5 = I5 + MemI5*MemR5#2

						;;				R1a/R1b becomes R1/R2
	vaddpd	zmm10, zmm23, zmm12		;; R1 + R2 (new R1)		R1 + R2 (new R1/HALF)		; 16-19		n 26
	vsubpd	zmm23, zmm23, zmm12		;; R1 - R2 (new R2)		R1 - R2 (new R2/HALF)		; 17-20		n 28

	vmovapd zmm24, [srcreg+rbp+d4+d2+d1+64] ;;		MemI8
	zfnmaddpd zmm11, zmm5, zmm24, zmm11	;;		A8 - I8 * MemI8 (R8 & R12)			; 17-20		n 23
	zfmaddpd zmm7, zmm0, zmm24, zmm7	;;		B8 + R8 * MemI8 (I8 & I12)			; 18-21		n 24

	vmovapd	zmm4, [srcreg+r9+d4+d1]			;; MemR6
	vmovapd	zmm6, [srcreg+r10+d4+d1]		;; MemR6#2
	vmovapd	zmm8, [srcreg+r10+d4+d1+64]		;; MemI6#2
	zfmaddpd zmm15, zmm4, zmm6, zmm15		;; R6 = R6 + MemR6*MemR6#2
	zfmaddpd zmm13, zmm4, zmm8, zmm13		;; I6 = I6 + MemR6*MemI6#2
	vmovapd	zmm4, [srcreg+r9+d4+d1+64]		;; MemI6
	zfnmaddpd zmm15, zmm4, zmm8, zmm15		;; R6 = R6 - MemI6*MemI6#2
	zfmaddpd zmm13, zmm4, zmm6, zmm13		;; I6 = I6 + MemI6*MemR6#2

	vmovapd	zmm4, [srcreg+r9+d4+d2]			;; MemR7
	vmovapd	zmm6, [srcreg+r10+d4+d2]		;; MemR7#2
	vmovapd	zmm8, [srcreg+r10+d4+d2+64]		;; MemI7#2
	zfmaddpd zmm9, zmm4, zmm6, zmm9			;; R7 = R7 + MemR7*MemR7#2
	zfmaddpd zmm1, zmm4, zmm8, zmm1			;; I7 = I7 + MemR7*MemI7#2
	vmovapd	zmm4, [srcreg+r9+d4+d2+64]		;; MemI7
	zfnmaddpd zmm9, zmm4, zmm8, zmm9		;; R7 = R7 - MemI7*MemI7#2
	zfmaddpd zmm1, zmm4, zmm6, zmm1			;; I7 = I7 + MemI7*MemR7#2

	vmovapd	zmm4, [srcreg+r9+d4+d2+d1]		;; MemR8
	vmovapd	zmm6, [srcreg+r10+d4+d2+d1]		;; MemR8#2
	vmovapd	zmm8, [srcreg+r10+d4+d2+d1+64]		;; MemI8#2
	zfmaddpd zmm11, zmm4, zmm6, zmm11		;; R8 = R8 + MemR8*MemR8#2
	zfmaddpd zmm7, zmm4, zmm8, zmm7			;; I8 = I8 + MemR8*MemI8#2
	vmovapd	zmm4, [srcreg+r9+d4+d2+d1+64]		;; MemI8
	zfnmaddpd zmm11, zmm4, zmm8, zmm11		;; R8 = R8 - MemI8*MemI8#2
	zfmaddpd zmm7, zmm4, zmm6, zmm7			;; I8 = I8 + MemI8*MemR8#2

	jmp	back_to_orig

mulsub:	test	r10, 3				;; Check if we need to multiply add-in by FFT(1)
	jz	mulsubhard			;; Yes, do it the hard way

	vsubpd	zmm17, zmm17, [srcreg+r9+d1]		;; R2 = R2 - MemR2
	vsubpd	zmm16, zmm16, [srcreg+r9+d1+64]		;; I2 = I2 - MemI2
	vsubpd	zmm19, zmm19, [srcreg+r9+d2]		;; R3 = R3 - MemR3
	vsubpd	zmm18, zmm18, [srcreg+r9+d2+64]		;; I3 = I3 - MemI3
	vsubpd	zmm21, zmm21, [srcreg+r9+d2+d1]		;; R4 = R4 - MemR4
	vsubpd	zmm20, zmm20, [srcreg+r9+d2+d1+64]	;; I4 = I4 - MemI4

	vmovapd	zmm12, zmm17			;;		Copy R2
	vmovapd zmm24, [srcreg+rbp+64]		;; MemI1			MemR1b
	zfnmaddpd zmm23 {k7}, zmm11, zmm24, zmm23;; A1 - I1 * MemI1 (R1)	R1a				; 10-13		n 16
	zfmaddpd zmm22 {k7}{z}, zmm1, zmm24, zmm22;; B1 + R1 * MemI1 (I1)	0				; 10-13		n 18

	vmulpd	zmm12 {k6}, zmm11, zmm24	;; R2				R1b * MemR1b (R1b)		; 11-14		n 16

	vmovapd	zmm14, [srcreg+d4+d2][rbx]	;;		R7
	vmovapd	zmm24, [srcreg+rbp+d4+d2]	;;		MemR7
	vmulpd	zmm9, zmm14, zmm24		;;		A7 = R7 * MemR7					; 11-14		n 15
	vmovapd	zmm4, [srcreg+d4+d2+64][rbx]	;;		I7
	vmulpd	zmm1, zmm4, zmm24		;;		B7 = I7 * MemR7					; 12-15		n 16

	vmovapd zmm24, [srcreg+rbp+d4+64]	;;		MemI5
	zfnmaddpd zmm3, zmm7, zmm24, zmm3	;;		A5 - I5 * MemI5 (R5 & R9)			; 12-15		n 21
	zfmaddpd zmm2, zmm10, zmm24, zmm2	;;		B5 + R5 * MemI5 (I5 & I9)			; 13-16		n 22

	vmovapd	zmm0, [srcreg+d4+d2+d1][rbx]	;;		R8
	vmovapd	zmm24, [srcreg+rbp+d4+d2+d1]	;;		MemR8
	vmulpd	zmm11, zmm0, zmm24		;;		A8 = R8 * MemR8					; 13-16		n 17
	vmovapd	zmm5, [srcreg+d4+d2+d1+64][rbx] ;;		I8
	vmulpd	zmm7, zmm5, zmm24		;;		B8 = I8 * MemR8					; 14-17		n 18

	vmovapd zmm24, [srcreg+rbp+d4+d1+64]	;;		MemI6
	zfnmaddpd zmm15, zmm8, zmm24, zmm15	;;		A6 - I6 * MemI6 (R6 & R10)			; 14-17		n 21
	zfmaddpd zmm13, zmm6, zmm24, zmm13	;;		B6 + R6 * MemI6 (I6 & I10)			; 15-18		n 22

	vmovapd zmm24, [srcreg+rbp+d4+d2+64]	;;		MemI7
	zfnmaddpd zmm9, zmm4, zmm24, zmm9	;;		A7 - I7 * MemI7 (R7 & R11)			; 15-18		n 23
	zfmaddpd zmm1, zmm14, zmm24, zmm1	;;		B7 + R7 * MemI7 (I7 & I11)			; 16-19		n 24

	vsubpd	zmm23, zmm23, [srcreg+r9]		;; R1 = R1 - MemR1
	vsubpd	zmm12{k6}, zmm12, [srcreg+r9+64]	;; R1b = R1b - MemI1
	vsubpd	zmm22{k7}, zmm22, [srcreg+r9+64]	;; I1 = I1 - MemI1
	vsubpd	zmm3, zmm3, [srcreg+r9+d4]		;; R5 = R5 - MemR5
	vsubpd	zmm2, zmm2, [srcreg+r9+d4+64]		;; I5 = I5 - MemI5

						;;				R1a/R1b becomes R1/R2
	vaddpd	zmm10, zmm23, zmm12		;; R1 + R2 (new R1)		R1 + R2 (new R1/HALF)		; 16-19		n 26
	vsubpd	zmm23, zmm23, zmm12		;; R1 - R2 (new R2)		R1 - R2 (new R2/HALF)		; 17-20		n 28

	vmovapd zmm24, [srcreg+rbp+d4+d2+d1+64] ;;		MemI8
	zfnmaddpd zmm11, zmm5, zmm24, zmm11	;;		A8 - I8 * MemI8 (R8 & R12)			; 17-20		n 23
	zfmaddpd zmm7, zmm0, zmm24, zmm7	;;		B8 + R8 * MemI8 (I8 & I12)			; 18-21		n 24

	vsubpd	zmm15, zmm15, [srcreg+r9+d4+d1]		;; R6 = R6 - MemR6
	vsubpd	zmm13, zmm13, [srcreg+r9+d4+d1+64]	;; I6 = I6 - MemI6
	vsubpd	zmm9, zmm9, [srcreg+r9+d4+d2]		;; R7 = R7 - MemR7
	vsubpd	zmm1, zmm1, [srcreg+r9+d4+d2+64]	;; I7 = I7 - MemI7
	vsubpd	zmm11, zmm11, [srcreg+r9+d4+d2+d1]	;; R8 = R8 - MemR8
	vsubpd	zmm7, zmm7, [srcreg+r9+d4+d2+d1+64]	;; I8 = I8 - MemI8

	jmp	back_to_orig

mulsubhard:
	vmovapd	zmm9, [srcreg+r9+d1]			;; MemR2
	vmovapd	zmm12, [srcreg+r10+d1]			;; MemR2#2
	vmovapd	zmm24, [srcreg+r10+d1+64]		;; MemI2#2
	zfnmaddpd zmm17, zmm9, zmm12, zmm17		;; R2 = R2 - MemR2*MemR2#2
	zfnmaddpd zmm16, zmm9, zmm24, zmm16		;; I2 = I2 - MemR2*MemI2#2
	vmovapd	zmm9, [srcreg+r9+d1+64]			;; MemI2
	zfmaddpd zmm17, zmm9, zmm24, zmm17		;; R2 = R2 + MemI2*MemI2#2
	zfnmaddpd zmm16, zmm9, zmm12, zmm16		;; I2 = I2 - MemI2*MemR2#2

	vmovapd	zmm9, [srcreg+r9+d2]			;; MemR3
	vmovapd	zmm12, [srcreg+r10+d2]			;; MemR3#2
	vmovapd	zmm24, [srcreg+r10+d2+64]		;; MemI3#2
	zfnmaddpd zmm19, zmm9, zmm12, zmm19		;; R3 = R3 - MemR3*MemR3#2
	zfnmaddpd zmm18, zmm9, zmm24, zmm18		;; I3 = I3 - MemR3*MemI3#2
	vmovapd	zmm9, [srcreg+r9+d2+64]			;; MemI3
	zfmaddpd zmm19, zmm9, zmm24, zmm19		;; R3 = R3 + MemI3*MemI3#2
	zfnmaddpd zmm18, zmm9, zmm12, zmm18		;; I3 = I3 - MemI3*MemR3#2

	vmovapd	zmm9, [srcreg+r9+d2+d1]			;; MemR4
	vmovapd	zmm12, [srcreg+r10+d2+d1]		;; MemR4#2
	vmovapd	zmm24, [srcreg+r10+d2+d1+64]		;; MemI4#2
	zfnmaddpd zmm21, zmm9, zmm12, zmm21		;; R4 = R4 - MemR4*MemR4#2
	zfnmaddpd zmm20, zmm9, zmm24, zmm20		;; I4 = I4 - MemR4*MemI4#2
	vmovapd	zmm9, [srcreg+r9+d2+d1+64]		;; MemI4
	zfmaddpd zmm21, zmm9, zmm24, zmm21		;; R4 = R4 + MemI4*MemI4#2
	zfnmaddpd zmm20, zmm9, zmm12, zmm20		;; I4 = I4 - MemI4*MemR4#2

	vmovapd	zmm12, zmm17			;;		Copy R2
	vmovapd zmm24, [srcreg+rbp+64]		;; MemI1			MemR1b
	zfnmaddpd zmm23 {k7}, zmm11, zmm24, zmm23;; A1 - I1 * MemI1 (R1)	R1a				; 10-13		n 16
	zfmaddpd zmm22 {k7}{z}, zmm1, zmm24, zmm22;; B1 + R1 * MemI1 (I1)	0				; 10-13		n 18

	vmulpd	zmm12 {k6}, zmm11, zmm24	;; R2				R1b * MemR1b (R1b)		; 11-14		n 16

	vmovapd	zmm14, [srcreg+d4+d2][rbx]	;;		R7
	vmovapd	zmm24, [srcreg+rbp+d4+d2]	;;		MemR7
	vmulpd	zmm9, zmm14, zmm24		;;		A7 = R7 * MemR7					; 11-14		n 15
	vmovapd	zmm4, [srcreg+d4+d2+64][rbx]	;;		I7
	vmulpd	zmm1, zmm4, zmm24		;;		B7 = I7 * MemR7					; 12-15		n 16

	vmovapd zmm24, [srcreg+rbp+d4+64]	;;		MemI5
	zfnmaddpd zmm3, zmm7, zmm24, zmm3	;;		A5 - I5 * MemI5 (R5 & R9)			; 12-15		n 21
	zfmaddpd zmm2, zmm10, zmm24, zmm2	;;		B5 + R5 * MemI5 (I5 & I9)			; 13-16		n 22

	vmovapd	zmm0, [srcreg+d4+d2+d1][rbx]	;;		R8
	vmovapd	zmm24, [srcreg+rbp+d4+d2+d1]	;;		MemR8
	vmulpd	zmm11, zmm0, zmm24		;;		A8 = R8 * MemR8					; 13-16		n 17
	vmovapd	zmm5, [srcreg+d4+d2+d1+64][rbx] ;;		I8
	vmulpd	zmm7, zmm5, zmm24		;;		B8 = I8 * MemR8					; 14-17		n 18

	vmovapd zmm24, [srcreg+rbp+d4+d1+64]	;;		MemI6
	zfnmaddpd zmm15, zmm8, zmm24, zmm15	;;		A6 - I6 * MemI6 (R6 & R10)			; 14-17		n 21
	zfmaddpd zmm13, zmm6, zmm24, zmm13	;;		B6 + R6 * MemI6 (I6 & I10)			; 15-18		n 22

	vmovapd zmm24, [srcreg+rbp+d4+d2+64]	;;		MemI7
	zfnmaddpd zmm9, zmm4, zmm24, zmm9	;;		A7 - I7 * MemI7 (R7 & R11)			; 15-18		n 23
	zfmaddpd zmm1, zmm14, zmm24, zmm1	;;		B7 + R7 * MemI7 (I7 & I11)			; 16-19		n 24

	vmovapd	zmm10, [srcreg+r9]			;; MemR1
	vmovapd	zmm6, [srcreg+r10]			;; MemR1#2
	vmovapd	zmm8, [srcreg+r10+64]			;; MemI1#2
	zfnmaddpd zmm23, zmm10, zmm6, zmm23		;; R1 = R1 - MemR1*MemR1#2
	zfnmaddpd zmm22{k7}{z}, zmm10, zmm8, zmm22	;; I1 = I1 - MemR1*MemI1#2
	vmovapd	zmm10, [srcreg+r9+64]			;; MemI1
	zfmaddpd zmm23{k7}, zmm10, zmm8, zmm23		;; R1 = R1 + MemI1*MemI1#2
	zfnmaddpd zmm12{k6}, zmm10, zmm8, zmm12		;; R1b = R1b - MemI1*MemI1#2
	zfnmaddpd zmm22{k7}{z}, zmm10, zmm6, zmm22	;; I1 = I1 - MemI1*MemR1#2

	vmovapd	zmm10, [srcreg+r9+d4]			;; MemR5
	vmovapd	zmm6, [srcreg+r10+d4]			;; MemR5#2
	vmovapd	zmm8, [srcreg+r10+d4+64]		;; MemI5#2
	zfnmaddpd zmm3, zmm10, zmm6, zmm3		;; R5 = R5 - MemR5*MemR5#2
	zfnmaddpd zmm2, zmm10, zmm8, zmm2		;; I5 = I5 - MemR5*MemI5#2
	vmovapd	zmm10, [srcreg+r9+d4+64]		;; MemI5
	zfmaddpd zmm3, zmm10, zmm8, zmm3		;; R5 = R5 + MemI5*MemI5#2
	zfnmaddpd zmm2, zmm10, zmm6, zmm2		;; I5 = I5 - MemI5*MemR5#2

						;;				R1a/R1b becomes R1/R2
	vaddpd	zmm10, zmm23, zmm12		;; R1 + R2 (new R1)		R1 + R2 (new R1/HALF)		; 16-19		n 26
	vsubpd	zmm23, zmm23, zmm12		;; R1 - R2 (new R2)		R1 - R2 (new R2/HALF)		; 17-20		n 28

	vmovapd zmm24, [srcreg+rbp+d4+d2+d1+64] ;;		MemI8
	zfnmaddpd zmm11, zmm5, zmm24, zmm11	;;		A8 - I8 * MemI8 (R8 & R12)			; 17-20		n 23
	zfmaddpd zmm7, zmm0, zmm24, zmm7	;;		B8 + R8 * MemI8 (I8 & I12)			; 18-21		n 24

	vmovapd	zmm4, [srcreg+r9+d4+d1]			;; MemR6
	vmovapd	zmm6, [srcreg+r10+d4+d1]		;; MemR6#2
	vmovapd	zmm8, [srcreg+r10+d4+d1+64]		;; MemI6#2
	zfnmaddpd zmm15, zmm4, zmm6, zmm15		;; R6 = R6 - MemR6*MemR6#2
	zfnmaddpd zmm13, zmm4, zmm8, zmm13		;; I6 = I6 - MemR6*MemI6#2
	vmovapd	zmm4, [srcreg+r9+d4+d1+64]		;; MemI6
	zfmaddpd zmm15, zmm4, zmm8, zmm15		;; R6 = R6 + MemI6*MemI6#2
	zfnmaddpd zmm13, zmm4, zmm6, zmm13		;; I6 = I6 - MemI6*MemR6#2

	vmovapd	zmm4, [srcreg+r9+d4+d2]			;; MemR7
	vmovapd	zmm6, [srcreg+r10+d4+d2]		;; MemR7#2
	vmovapd	zmm8, [srcreg+r10+d4+d2+64]		;; MemI7#2
	zfnmaddpd zmm9, zmm4, zmm6, zmm9		;; R7 = R7 - MemR7*MemR7#2
	zfnmaddpd zmm1, zmm4, zmm8, zmm1		;; I7 = I7 - MemR7*MemI7#2
	vmovapd	zmm4, [srcreg+r9+d4+d2+64]		;; MemI7
	zfmaddpd zmm9, zmm4, zmm8, zmm9			;; R7 = R7 + MemI7*MemI7#2
	zfnmaddpd zmm1, zmm4, zmm6, zmm1		;; I7 = I7 - MemI7*MemR7#2

	vmovapd	zmm4, [srcreg+r9+d4+d2+d1]		;; MemR8
	vmovapd	zmm6, [srcreg+r10+d4+d2+d1]		;; MemR8#2
	vmovapd	zmm8, [srcreg+r10+d4+d2+d1+64]		;; MemI8#2
	zfnmaddpd zmm11, zmm4, zmm6, zmm11		;; R8 = R8 - MemR8*MemR8#2
	zfnmaddpd zmm7, zmm4, zmm8, zmm7		;; I8 = I8 - MemR8*MemI8#2
	vmovapd	zmm4, [srcreg+r9+d4+d2+d1+64]		;; MemI8
	zfmaddpd zmm11, zmm4, zmm8, zmm11		;; R8 = R8 + MemI8*MemI8#2
	zfnmaddpd zmm7, zmm4, zmm6, zmm7		;; I8 = I8 - MemI8*MemR8#2

	jmp	back_to_orig

orig:

	vmovapd	zmm2, [srcreg+d1][rbx]		;;		R2
	vmovapd	zmm24, [srcreg+rbp+d1]		;;		MemR2
	vmulpd	zmm17, zmm2, zmm24		;;		A2 = R2 * MemR2					; 1-4		n 5
	vmovapd	zmm3, [srcreg+d1+64][rbx]	;;		I2
	vmulpd	zmm16, zmm3, zmm24		;;		B2 = I2 * MemR2					; 1-4		n 5

	vmovapd	zmm13, [srcreg+d2][rbx]		;;		R3
	vmovapd	zmm24, [srcreg+rbp+d2]		;;		MemR3
	vmulpd	zmm19, zmm13, zmm24		;;		A3 = R3 * MemR3					; 2-5		n 6
	vmovapd	zmm15, [srcreg+d2+64][rbx]	;;		I3
	vmulpd	zmm18, zmm15, zmm24		;;		B3 = I3 * MemR3					; 2-5		n 6

	vmovapd	zmm9, [srcreg+d2+d1][rbx]	;;		R4
	vmovapd	zmm24, [srcreg+rbp+d2+d1]	;;		MemR4
	vmulpd	zmm21, zmm9, zmm24		;;		A4 = R4 * MemR4					; 3-6		n 8
	vmovapd	zmm12, [srcreg+d2+d1+64][rbx]	;;		I4
	vmulpd	zmm20, zmm12, zmm24		;;		B4 = I4 * MemR4					; 3-6		n 8

	vmovapd	zmm1, [srcreg][rbx]		;; R1				R1a
	vmovapd	zmm24, [srcreg+rbp]		;; MemR1			MemR1a
	vmulpd	zmm23, zmm1, zmm24		;; A1 = R1 * MemR1		R1a*MemR1a (R1a)		; 4-7		n 10
	vmovapd	zmm11, [srcreg+64][rbx]		;; I1				R1b
	vmulpd	zmm22 {k7}{z}, zmm11, zmm24	;; B1 = I1 * MemR1		0				; 4-7		n 10

	vmovapd zmm24, [srcreg+rbp+d1+64]	;;		MemI2
	zfnmaddpd zmm17, zmm3, zmm24, zmm17	;;		A2 - I2 * MemI2 (R2)				; 5-8		n 10
	zfmaddpd zmm16, zmm2, zmm24, zmm16	;;		B2 + R2 * MemI2 (I2)				; 5-8		n 18

	vmovapd zmm24, [srcreg+rbp+d2+64]	;;		MemI3
	zfnmaddpd zmm19, zmm15, zmm24, zmm19	;;		A3 - I3 * MemI3 (R3 & R5)			; 6-9		n 19
	zfmaddpd zmm18, zmm13, zmm24, zmm18	;;		B3 + R3 * MemI3 (I3 & I5)			; 6-9		n 20

	vmovapd	zmm10, [srcreg+d4][rbx]		;;		R5
	vmovapd	zmm24, [srcreg+rbp+d4]		;;		MemR5
	vmulpd	zmm3, zmm10, zmm24		;;		A5 = R5 * MemR5					; 7-10		n 12
	vmovapd	zmm7, [srcreg+d4+64][rbx]	;;		I5
	vmulpd	zmm2, zmm7, zmm24		;;		B5 = I5 * MemR5					; 7-10		n 13

	vmovapd zmm24, [srcreg+rbp+d2+d1+64]	;;		MemI4
	zfnmaddpd zmm21, zmm12, zmm24, zmm21	;;		A4 - I4 * MemI4 (R4 & R6)			; 8-11		n 19
	zfmaddpd zmm20, zmm9, zmm24, zmm20	;;		B4 + R4 * MemI4 (I4 & I6)			; 8-11		n 20
	vmovsd	Q [srcreg-16], xmm23		;;				Save product of FFT sums	; 8

	vmovapd	zmm6, [srcreg+d4+d1][rbx]	;;		R6
	vmovapd	zmm24, [srcreg+rbp+d4+d1]	;;		MemR6
	vmulpd	zmm15, zmm6, zmm24		;;		A6 = R6 * MemR6					; 9-12		n 14
	vmovapd	zmm8, [srcreg+d4+d1+64][rbx]	;;		I6
	vmulpd	zmm13, zmm8, zmm24		;;		B6 = I6 * MemR6					; 9-12		n 15

	vmovapd	zmm12, zmm17			;;		Copy R2
	vmovapd zmm24, [srcreg+rbp+64]		;; MemI1			MemR1b
	zfnmaddpd zmm23 {k7}, zmm11, zmm24, zmm23;; A1 - I1 * MemI1 (R1)	R1a				; 10-13		n 16
	zfmaddpd zmm22 {k7}{z}, zmm1, zmm24, zmm22;; B1 + R1 * MemI1 (I1)	0				; 10-13		n 18

	vmulpd	zmm12 {k6}, zmm11, zmm24	;; R2				R1b * MemR1b (R1b)		; 11-14		n 16

	vmovapd	zmm14, [srcreg+d4+d2][rbx]	;;		R7
	vmovapd	zmm24, [srcreg+rbp+d4+d2]	;;		MemR7
	vmulpd	zmm9, zmm14, zmm24		;;		A7 = R7 * MemR7					; 11-14		n 15
	vmovapd	zmm4, [srcreg+d4+d2+64][rbx]	;;		I7
	vmulpd	zmm1, zmm4, zmm24		;;		B7 = I7 * MemR7					; 12-15		n 16

	vmovapd zmm24, [srcreg+rbp+d4+64]	;;		MemI5
	zfnmaddpd zmm3, zmm7, zmm24, zmm3	;;		A5 - I5 * MemI5 (R5 & R9)			; 12-15		n 21
	zfmaddpd zmm2, zmm10, zmm24, zmm2	;;		B5 + R5 * MemI5 (I5 & I9)			; 13-16		n 22

	vmovapd	zmm0, [srcreg+d4+d2+d1][rbx]	;;		R8
	vmovapd	zmm24, [srcreg+rbp+d4+d2+d1]	;;		MemR8
	vmulpd	zmm11, zmm0, zmm24		;;		A8 = R8 * MemR8					; 13-16		n 17
	vmovapd	zmm5, [srcreg+d4+d2+d1+64][rbx] ;;		I8
	vmulpd	zmm7, zmm5, zmm24		;;		B8 = I8 * MemR8					; 14-17		n 18

	vmovapd zmm24, [srcreg+rbp+d4+d1+64]	;;		MemI6
	zfnmaddpd zmm15, zmm8, zmm24, zmm15	;;		A6 - I6 * MemI6 (R6 & R10)			; 14-17		n 21
	zfmaddpd zmm13, zmm6, zmm24, zmm13	;;		B6 + R6 * MemI6 (I6 & I10)			; 15-18		n 22

	vmovapd zmm24, [srcreg+rbp+d4+d2+64]	;;		MemI7
	zfnmaddpd zmm9, zmm4, zmm24, zmm9	;;		A7 - I7 * MemI7 (R7 & R11)			; 15-18		n 23
	zfmaddpd zmm1, zmm14, zmm24, zmm1	;;		B7 + R7 * MemI7 (I7 & I11)			; 16-19		n 24

						;;				R1a/R1b becomes R1/R2
	vaddpd	zmm10, zmm23, zmm12		;; R1 + R2 (new R1)		R1 + R2 (new R1/HALF)		; 16-19		n 26
	vsubpd	zmm23, zmm23, zmm12		;; R1 - R2 (new R2)		R1 - R2 (new R2/HALF)		; 17-20		n 28

	vmovapd zmm24, [srcreg+rbp+d4+d2+d1+64] ;;		MemI8
	zfnmaddpd zmm11, zmm5, zmm24, zmm11	;;		A8 - I8 * MemI8 (R8 & R12)			; 17-20		n 23
	zfmaddpd zmm7, zmm0, zmm24, zmm7	;;		B8 + R8 * MemI8 (I8 & I12)			; 18-21		n 24

back_to_orig:					;;				R2/I2 becomes R3/R4
	vaddpd	zmm17 {k7}, zmm22, zmm16	;; I1 + I2 (new I1)		R3				; 18-47		n 25
	vsubpd	zmm16 {k7}, zmm22, zmm16	;; I1 - I2 (new I2)		R4				; 19-47		n 26
	vbroadcastsd zmm30, ZMM_P924_P383

	vaddpd	zmm8, zmm21, zmm19		;; R4 + R3 (new R3)		R6 + R5 (new R5)		; 19-52		n 25
	vsubpd	zmm21, zmm21, zmm19		;; R4 - R3 (new I4)		R6 - R5 (new negR6)		; 20-52		n 33
	vbroadcastsd zmm31, ZMM_SQRTHALF

	vaddpd	zmm6, zmm18, zmm20		;; I3 + I4 (new I3)		I5 + I6 (new I5)		; 20-53		n 27
	vsubpd	zmm18, zmm18, zmm20		;; I3 - I4 (new R4)		I5 - I6 (new I6)		; 21-53		n 26

	vaddpd	zmm4, zmm3, zmm15		;; R5 + R6 (new R5)		R9 + R10 (new R9)		; 21-48		n 34
	vsubpd	zmm3, zmm3, zmm15		;; R5 - R6 (new R6)		R9 - R10 (new R10)		; 22-48		n 29

	vaddpd	zmm14, zmm2, zmm13		;; I5 + I6 (new I5)		I9 + I10 (new I9)		; 22-49		n 35
	vsubpd	zmm2, zmm2, zmm13		;; I5 - I6 (new I6)		I9 - I10 (new I10)		; 23-49		n 

	vaddpd	zmm12, zmm11, zmm9		;; R8 + R7 (new R7)		R12 + R11 (new R11)		; 23-50		n 34
	vsubpd	zmm11, zmm11, zmm9		;; R8 - R7 (new I8)		R12 - R11 (new I12)		; 24-50		n 29

	vaddpd	zmm5, zmm1, zmm7		;; I7 + I8 (new I7)		I11 + I12 (new I11)		; 24-51		n 35
	vsubpd	zmm1, zmm1, zmm7		;; I7 - I8 (new R8)		I11 - I12 (new R12)		; 25-51		n 32

	vblendmpd zmm26 {k7}, zmm30, ZMM_ONE{1to8} ;; 1, 1, 1, 1, 1, 1, 1	.924/.383			; ?		n 
	vblendmpd zmm28 {k6}, zmm26, ZMM_HALF{1to8} ;; 1, 1, 1, 1, 1, 1, 1	HALF				; 25***		n 
	vblendmpd zmm0 {k6}, zmm8, zmm17	;; R3				R3				; 25		n 26
	vblendmpd zmm15 {k6}, zmm18, zmm16	;; R4				R4				; 26		n 28

	zfmaddpd zmm13, zmm10, zmm28, zmm0	;; R1*1 + R3 (newer R1)		R1*HALF + R3 (newer R1)		; 26-57		n 
	zfmsubpd zmm10, zmm10, zmm28, zmm0	;; R1*1 - R3 (newer R3)		R1*HALF - R3 (newer R3)		; 27-57		n 

	vaddpd	zmm9 {k7}{z}, zmm17, zmm6	;; I1 + I3 (newer I1)		0				; 27-59		n 38
	vsubpd	zmm17 {k7}{z}, zmm17, zmm6	;; I1 - I3 (newer I3)		0				; 28-59		n 39

	zfmaddpd zmm7, zmm23, zmm28, zmm15	;; R2*1 + R4 (newer R2)		R2*HALF + R4 (newer R2)		; 28-61		n 45
	zfmsubpd zmm23, zmm23, zmm28, zmm15	;; R2*1 - R4 (newer R4)		R2*HALF - R4 (newer R4)		; 29-61		n 46

	vmovapd zmm0, zmm3			;; R6				R10
	vmulpd	zmm0 {k6}, zmm27, zmm11		;; R6				I12*.383			; 29-61		n 
	vmulpd	zmm11 {k6}, zmm27, zmm3		;; I8				R10*.383			; 30-54		n 

						;; multiply R6/I6 by SQRTHALF - i*SQRTHALF
	zfmaddpd zmm15, zmm2, zmm27, zmm0	;; I6*1 + R6 (new2 R6/SQRTHALF)	I10*.383 + I12 (newer I10*.383)	; 30-54		n 36
	zfmsubpd zmm2, zmm2, zmm27, zmm0	;; I6*1 - R6 (new2 I6/SQRTHALF)	I10*.383 - I12 (newer I12*.383)	; 31-54		n 37

	vblendmpd zmm16 {k6}, zmm16, zmm18	;; I2				I6				; 31		n 

						;; multiply R8/I8 by SQRTHALF - i*SQRTHALF
	zfmaddpd zmm3, zmm1, zmm27, zmm11	;; I8 + R8*1 (new2 R8/SQRTHALF)	R10 + R12*.383 (newer R10*.383)	; 32-55		n 36
	zfnmaddpd zmm11, zmm1, zmm27, zmm11	;; I8 - R8*1 (new2 I8/SQRTHALF)	R10 - R12*.383 (newer R12*.383)	; 32-55		n 37

						;;				mul R6/I6 by w^2 = .707 - .707i
	vaddpd	zmm0, zmm16, zmm21		;; I2 + I4 (newer I2)		I6 + negR6 (newer I6/SQRTHALF) 	; 33-60		n 44
	vsubpd	zmm16, zmm16, zmm21		;; I2 - I4 (newer I4)		I6 - negR6 (newer R6/SQRTHALF) 	; 33-60		n 44

	vsubpd	zmm1, zmm12, zmm4		;; R7 - R5 (newer I7)		R11 - R9 (newer negR11)		; 34-58		n 38
	vaddpd	zmm12, zmm12, zmm4		;; R7 + R5 (newer R5)		R11 + R9 (newer R9)		; 34-37		n 38

	vaddpd	zmm4, zmm14, zmm5		;; I5 + I7 (newer I5)		I9 + I11 (newer I9)		; 35-56		n 42
	vsubpd	zmm14, zmm14, zmm5		;; I5 - I7 (newer R7)		I9 - I11 (newer I11)		; 35-56		n 39

						;;				mul R10/I10 by w^1 = .924 - .383i
	zfmaddpd zmm5, zmm3, zmm26, zmm15	;; R6 + R8*1 (newer R6/SQRTHALF)    I10 + R10*.924/.383 (R10)	; 36-71		n 44
	zfmsubpd zmm15, zmm15, zmm26, zmm3	;; R6*1 - R8 (newer negI8/SQRTHALF) I10*.924/.383 - R10 (I10)	; 36-71		n 

						;;				mul R12/I12 by w^3 = .383 - .924i
	zfmaddpd zmm3, zmm2, zmm26, zmm11	;; I6*1 + I8 (newer I6/SQRTHALF)  I12*.924/.383 + R12 (R12)	; 37-63		n 47
	zfnmaddpd zmm11, zmm11, zmm26, zmm2	;; I6 - I8*1 (newer R8/SQRTHALF)  I12 - R12*.924/.383 (I12)	; 37-63		n 44

						;;				R5/I5 becomes new R5/R7
	vblendmpd zmm8 {k6}, zmm12, zmm8	;; R5				R5				; 38		n 40
	vsubpd	zmm1 {k6}, zmm9, zmm1		;; I7				R11 = 0 - negR11		; 38-55		n 43

	vblendmpd zmm29 {k6}, zmm26, zmm31	;; 1, 1, 1, 1, 1, 1, 1		SQRTHALF
	vblendmpd zmm6 {k6}, zmm14, zmm6	;; R7				R7				; 39		n 41
	vmulpd	zmm17 {k6}, zmm29, zmm14	;; I3				I11*SQRTHALF			; 39-42		n 43

;; four aparts

	vaddpd	zmm2, zmm13, zmm8		;; R1 + R5 (final R1)		R1 + R5 (final R1)		; 40-66		n 
	vsubpd	zmm13, zmm13, zmm8		;; R1 - R5 (final R5)		R1 - R5 (final R5)		; 40-66		n 

	vaddpd	zmm8, zmm10, zmm6		;; R3 + R7 (final R3)		R3 + R7 (final R3)		; 41-64		n 
	vsubpd	zmm10, zmm10, zmm6		;; R3 - R7 (final R7)		R3 - R7 (final R7)		; 41-64		n 

						;;				R9/I9 becomes newer R9/R13
	vaddpd	zmm12 {k7}, zmm9, zmm4		;; I1 + I5 (final I1)		R9				; 42-67		n 
	vsubpd	zmm4 {k7}, zmm9, zmm4		;; I1 - I5 (final I5)		R13				; 42-67		n 

						;;				R11/I11 becomes newer R11/R15
						;;				mul R11/I11 by SQRTHALF - i*SQRTHALF
	zfmaddpd zmm6, zmm1, zmm29, zmm17	;; I3 + I7*1 (final I3)		I11 + R11*SQRTHALF (final R11)	; 43-65		n 
	zfnmaddpd zmm17, zmm1, zmm29, zmm17	;; I3 - I7*1 (final I7)		I11 - R11*SQRTHALF (final I11)	; 43-65		n 

						;;				R6/I6 becomes new R6/R8
	vblendmpd zmm9 {k6}, zmm5, zmm16	;; R6/SQRTHALF			R6/SQRTHALF			; 44		n 
	vblendmpd zmm1 {k6}, zmm11, zmm0	;; R8/SQRTHALF			R8/SQRTHALF			; 44		n 

	zfmaddpd zmm14, zmm9, zmm31, zmm7	;; R2 + R6*SQRTHALF (final R2)	R2 + R6 * SQRTHALF (final R2)	; 45-68		n 
	zfnmaddpd zmm9, zmm9, zmm31, zmm7	;; R2 - R6*SQRTHALF (final R6)	R2 - R6 * SQRTHALF (final R6)	; 45-68		n 

	zfmaddpd zmm7, zmm1, zmm31, zmm23	;; R4 + R8*SQRTHALF (final R4)	R4 + R8 * SQRTHALF (final R4)	; 46-70		n 
	zfnmaddpd zmm1, zmm1, zmm31, zmm23	;; R4 - R8*SQRTHALF (final R8)	R4 - R8 * SQRTHALF (final R8)	; 46-70		n 

						;;				R10/I10 becomes newer R10/R14
	vmovapd	zmm18, zmm15			;; negI8			R14
;; bug - hidden vmovapd uops
	zfmaddpd zmm5 {k7}, zmm3, zmm31, zmm0	;; I2 + I6*SQRTHALF (final I2)	R10				; 47-71		n 
	zfnmaddpd zmm15 {k7}, zmm3, zmm31, zmm0	;; I2 - I6*SQRTHALF (final I6)	R14				; 47-71		n 

						;;				R12/I12 becomes newer R12/R16
;; bug - hidden vmovapd uops
	zfmaddpd zmm11 {k7}, zmm18, zmm31, zmm16;; I4 + negI8*SQRTHALF (final I8)  R16				; 48-69		n 
	zfnmaddpd zmm3 {k7}, zmm18, zmm31, zmm16;; I4 - negI8*SQRTHALF (final I4)  R12				; 48-69		n 

	zstore	[srcreg], zmm2			;; Save R1						; 30
	zstore	[srcreg+64], zmm12		;; Save R9						; 30+1
	zstore	[srcreg+d4], zmm13		;; Save R5						; 31+1
	zstore	[srcreg+d4+64], zmm4		;; Save R13						; 31+2
	zstore	[srcreg+d2], zmm8		;; Save R3						; 37
	zstore	[srcreg+d2+64], zmm6		;; Save R11						; 37+1
	zstore	[srcreg+d4+d2], zmm10		;; Save R7						; 38+1
	zstore	[srcreg+d4+d2+64], zmm17	;; Save R15						; 38+2
	zstore	[srcreg+d1], zmm14		;; Save R2						; 39+2
	zstore	[srcreg+d1+64], zmm5		;; Save R10						; 39+3
	zstore	[srcreg+d4+d1], zmm9		;; Save R6						; 40+3
	zstore	[srcreg+d4+d1+64], zmm15	;; Save R14						; 40+4
	zstore	[srcreg+d2+d1], zmm7		;; Save R4						; 41+4
	zstore	[srcreg+d2+d1+64], zmm3		;; Save R12						; 41+5
	zstore	[srcreg+d4+d2+d1], zmm1		;; Save R8						; 42+5
	zstore	[srcreg+d4+d2+d1+64], zmm11	;; Save R16						; 42+6
	bump	srcreg, srcinc
	ENDM


zr8fs_eight_complex_first_fft_preload MACRO
	vbroadcastsd zmm31, ZMM_SQRTHALF
	vpmovzxbq zmm30, ZMM_PERMUTE1		;; zmm30 = 8+4 0+4 8+6 0+6 8+0 0+0 8+2 0+2 [msw at left]
	vpmovzxbq zmm29, ZMM_PERMUTE2		;; zmm29 = 8+5 0+5 8+7 0+7 8+1 0+1 8+3 0+3
	mov	r14d, 00110011b			;; For vblendmpd during swizzle
	kmovw	k7, r14d
	ENDM

zr8fs_eight_complex_first_fft MACRO srcreg,srcinc,d1,d2,d4,screg,scinc,wgtreg,wgtinc,maxrpt,L1pt,L1pd
	vmovapd	zmm9, [srcreg+d1+64][rbx]	;; I2
	vmulpd	zmm9, zmm9, [wgtreg+1*128+64]	;; apply weight I2 over R2 to I2		; 1-4		n 7
	vmovapd	zmm11, [srcreg+d2+d1+64][rbx]	;; I4
	vmulpd	zmm11, zmm11, [wgtreg+3*128+64]	;; apply weight I4 over R4 to I4		; 1-4		n 8

	vmovapd	zmm5, [srcreg+d4+d1][rbx]	;; R6
	vmulpd	zmm5, zmm5, [wgtreg+5*128]	;; apply weight to R6				; 2-5		n 9
	vmovapd	zmm13, [srcreg+d4+d1+64][rbx]	;; I6
	vmulpd	zmm13, zmm13, [wgtreg+5*128+64]	;; apply weight to I6				; 2-5		n 9

	vmovapd	zmm8, [srcreg+64][rbx]		;; I1
	vmulpd	zmm8, zmm8, [wgtreg+64]		;; apply weight I1 over R1 to I1		; 3-6		n 11
	vmovapd	zmm10, [srcreg+d2+64][rbx]	;; I3
	vmulpd	zmm10, zmm10, [wgtreg+2*128+64]	;; apply weight I3 over R3 to I3		; 3-6		n 12

	vmovapd	zmm4, [srcreg+d4][rbx]		;; R5
	vmulpd	zmm4, zmm4, [wgtreg+4*128]	;; apply weight to R5				; 4-7		n 13
	vmovapd	zmm12, [srcreg+d4+64][rbx]	;; I5
	vmulpd	zmm12, zmm12, [wgtreg+4*128+64]	;; apply weight to I5				; 4-7		n 13

	vmovapd	zmm7, [srcreg+d4+d2+d1][rbx]	;; R8
	vmulpd	zmm7, zmm7, [wgtreg+7*128]	;; apply weight to R8				; 5-8		n 10
	vmovapd	zmm15, [srcreg+d4+d2+d1+64][rbx]	;; I8
	vmulpd	zmm15, zmm15, [wgtreg+7*128+64]	;; apply weight to I8				; 5-8		n 10

	vmovapd	zmm6, [srcreg+d4+d2][rbx]	;; R7
	vmulpd	zmm6, zmm6, [wgtreg+6*128]	;; apply weight to R7				; 6-9		n 14
	vmovapd	zmm14, [srcreg+d4+d2+64][rbx]	;; I7
	vmulpd	zmm14, zmm14, [wgtreg+6*128+64]	;; apply weight to I7				; 6-9		n 14

;; Apply the complex premultipliers

	vmovapd	zmm1, [srcreg+d1][rbx]		;; R2
	vmovapd zmm28, [screg+1*64]		;; premultiplier cosine/sine for R2/I2
	zfmsubpd zmm16, zmm1, zmm28, zmm9	;; A2 = R2 * cosine - I2			; 7-10		n 15
	zfmaddpd zmm9, zmm9, zmm28, zmm1	;; B2 = I2 * cosine + R2			; 7-10		n 18

	vmovapd	zmm3, [srcreg+d2+d1][rbx]	;; R4
	vmovapd zmm28, [screg+3*64]		;; premultiplier cosine/sine for R4/I4
	zfmsubpd zmm1, zmm3, zmm28, zmm11	;; A4 = R4 * cosine - I4			; 8-11		n 16
	zfmaddpd zmm11, zmm11, zmm28, zmm3	;; B4 = I4 * cosine + R4			; 8-11		n 17

	vmovapd zmm28, [screg+5*64]		;; premultiplier cosine/sine for R6/I6
	zfmsubpd zmm3, zmm5, zmm28, zmm13	;; A6 = R6 * cosine - I6			; 9-12		n 15
	zfmaddpd zmm13, zmm13, zmm28, zmm5	;; B6 = I6 * cosine + R6			; 9-12		n 18

	vmovapd zmm28, [screg+7*64]		;; premultiplier cosine/sine for R8/I8
	zfmsubpd zmm5, zmm7, zmm28, zmm15	;; A8 = R8 * cosine - I8			; 10-13		n 16
	zfmaddpd zmm15, zmm15, zmm28, zmm7	;; B8 = I8 * cosine + R8			; 10-13		n 17

	vmovapd	zmm0, [srcreg][rbx]		;; R1
	vmovapd zmm28, [screg+0*64]		;; premultiplier cosine/sine for R1/I1
	zfmsubpd zmm7, zmm0, zmm28, zmm8	;; A1 = R1 * cosine - I1			; 11-14		n 19
	zfmaddpd zmm8, zmm8, zmm28, zmm0	;; B1 = I1 * cosine + R1			; 11-14		n 21

	vmovapd	zmm2, [srcreg+d2][rbx]		;; R3
	vmovapd zmm28, [screg+2*64]		;; premultiplier cosine/sine for R3/I3
	zfmsubpd zmm0, zmm2, zmm28, zmm10	;; A3 = R3 * cosine - I3			; 12-15		n 20
	zfmaddpd zmm10, zmm10, zmm28, zmm2	;; B3 = I3 * cosine + R3			; 12-15		n 22

	vmovapd zmm28, [screg+4*64]		;; premultiplier cosine/sine for R5/I5
	zfmsubpd zmm2, zmm4, zmm28, zmm12	;; A5 = R5 * cosine - I5			; 13-16		n 19
	zfmaddpd zmm12, zmm12, zmm28, zmm4	;; B5 = I5 * cosine + R5			; 13-16		n 21

	vmovapd zmm28, [screg+6*64]		;; premultiplier cosine/sine for R7/I7
	zfmsubpd zmm4, zmm6, zmm28, zmm14	;; A7 = R7 * cosine - I7			; 14-17		n 20
	zfmaddpd zmm14, zmm14, zmm28, zmm6	;; B7 = I7 * cosine + R7			; 14-17		n 22

;; Copied from common 8-complex DJB FFT macro

	vmovapd zmm28, [wgtreg+1*128]		;; weight for R2
	zfmsubpd zmm6, zmm16, zmm28, zmm3	;; R2*weightR2-R6				; 15-18		n 23
	zfmaddpd zmm16, zmm16, zmm28, zmm3	;; R2*weightR2+R6				; 15-18		n 25

	vmovapd zmm27, [wgtreg+3*128]		;; weight for R4
	zfmsubpd zmm3, zmm1, zmm27, zmm5	;; R4*weightR4-R8				; 16-19		n 23
	zfmaddpd zmm1, zmm1, zmm27, zmm5	;; R4*weightR4+R8				; 16-19		n 25

	vmovapd zmm26, [wgtreg+0*128]		;; weight for R1
	zfmsubpd zmm5, zmm11, zmm27, zmm15	;; I4*weightR4-I8				; 17-20		n 24
	zfmaddpd zmm11, zmm11, zmm27, zmm15	;; I4*weightR4+I8				; 17-20		n 26

	vmovapd zmm27, [wgtreg+2*128]		;; weight for R3
	zfmsubpd zmm15, zmm9, zmm28, zmm13	;; I2*weightR2-I6				; 18-21		n 24
	zfmaddpd zmm9, zmm9, zmm28, zmm13	;; I2*weightR2+I6				; 18-21		n 26
	bump	wgtreg, wgtinc

	vmovapd zmm28, [screg+8*64+1*128]	;; sine for R3/I3 and R7/I7 (w^2)
	zfmaddpd zmm13, zmm7, zmm26, zmm2	;; R1*weightR1+R5				; 19-22		n 27
	zfmsubpd zmm7, zmm7, zmm26, zmm2	;; R1*weightR1-R5				; 19-22		n 28

	vmovapd zmm25, [screg+8*64+0*128]	;; sine for R2/I2 and R8/I8 (w^1)
	zfmaddpd zmm2, zmm0, zmm27, zmm4	;; R3*weightR3+R7				; 20-23		n 27
	zfmsubpd zmm0, zmm0, zmm27, zmm4	;; R3*weightR3-R7				; 20-23		n 31

	vmovapd zmm24, [screg+8*64+2*128]	;; sine for R4/I4 and R6/I6 (w^3)
	zfmaddpd zmm4, zmm8, zmm26, zmm12	;; I1*weightR1+I5				; 21-24		n 30
	zfmsubpd zmm8, zmm8, zmm26, zmm12	;; I1*weightR1-I5				; 21-24		n 29

	vmovapd zmm26, [screg+8*64+1*128+64]	;; cosine/sine for R3/I3 and R7/I7 (w^2)
	zfmaddpd zmm12, zmm10, zmm27, zmm14	;; I3*weightR3+I7				; 22-25		n 46
	zfmsubpd zmm10, zmm10, zmm27, zmm14	;; I3*weightR3-I7				; 22-25		n 48

	vmovapd zmm27, [screg+8*64+3*128+64]	;; cosine/sine for R5/I5 (w^4)
	vsubpd	zmm14, zmm6, zmm3		;; r2-- = (r2-r6) - (r4-r8)			; 23-26		n 28
	vaddpd	zmm6, zmm6, zmm3		;; r2-+ = (r2-r6) + (r4-r8)			; 23-26		n 31

	vmovapd zmm23, [screg+8*64+0*128+64]	;; cosine/sine for R2/I2 and R8/I8 (w^1)
	vsubpd	zmm3, zmm15, zmm5		;; i2-- = (i2-i6) - (i4-i8)			; 24-27		n 29
	vaddpd	zmm15, zmm15, zmm5		;; i2-+ = (i2-i6) + (i4-i8)			; 24-27		n 32

	vmovapd zmm22, [screg+8*64+2*128+64]	;; cosine/sine for R4/I4 and R6/I6 (w^3)
	vsubpd	zmm5, zmm16, zmm1		;; r2+- = (r2+r6) - (r4+r8)			; 25-28		n 33
	vaddpd	zmm16, zmm16, zmm1		;; r2++ = (r2+r6) + (r4+r8)			; 25-28		n 36

	vmovapd zmm21, [screg+8*64+3*128]	;; sine for R5/I5 (w^4)
	vsubpd	zmm1, zmm9, zmm11		;; i2+- = (i2+i6) - (i4+i8)			; 26-29		n 33
	vaddpd	zmm9, zmm9, zmm11		;; i2++ = (i2+i6) + (i4+i8)			; 26-29		n 39
	bump	screg, scinc

	L1prefetchw srcreg+srcinc+rbx+d1+64, L1pt
	vaddpd	zmm11, zmm13, zmm2		;; r1++ = (r1+r5) + (r3+r7)			; 27-30		n 36
	vsubpd	zmm13, zmm13, zmm2		;; r1+- = (r1+r5) - (r3+r7)			; 27-30		n 37

	L1prefetch wgtreg+1*128+64, L1pt
	zfmaddpd zmm2, zmm14, zmm31, zmm7	;; r1-+ = (r1-r5) + .707*(r2--)			; 28-31		n 34
	zfnmaddpd zmm14, zmm14, zmm31, zmm7	;; r1-- = (r1-r5) - .707*(r2--)			; 28-31		n 35

	L1prefetchw srcreg+srcinc+rbx+d2+d1+64, L1pt
	zfmaddpd zmm7, zmm3, zmm31, zmm8	;; i1-+ = (i1-i5) + .707*(i2--)			; 29-32		n 34
	zfnmaddpd zmm3, zmm3, zmm31, zmm8	;; i1-- = (i1-i5) - .707*(i2--)			; 29-32		n 35

	L1prefetch wgtreg+3*128+64, L1pt
	vaddpd	zmm8, zmm4, zmm12		;; i1++ = (i1+i5) + (i3+i7)			; 30-33		n 39
	vsubpd	zmm4, zmm4, zmm12		;; i1+- = (i1+i5) - (i3+i7)			; 30-33		n 38

	L1prefetchw srcreg+srcinc+rbx+d4+d1, L1pt
	zfmaddpd zmm12, zmm6, zmm31, zmm0	;; r3-+ = (r3-r7) + .707*(r2-+)			; 31-34		n 40
	zfnmaddpd zmm6, zmm6, zmm31, zmm0	;; r3-- = (r3-r7) - .707*(r2-+)			; 31-34		n 44

	L1prefetch wgtreg+5*128, L1pt
	zfmaddpd zmm0, zmm15, zmm31, zmm10	;; i3-+ = (i3-i7) + .707*(i2-+)			; 32-35		n 40
	zfnmaddpd zmm15, zmm15, zmm31, zmm10	;; i3-- = (i3-i7) - .707*(i2-+)			; 32-35		n 44

	L1prefetchw srcreg+srcinc+rbx+d4+d1+64, L1pt
	vmulpd	zmm1, zmm1, zmm28		;; i2+-s = i2+- * sine37			; 33-36		n 37
	vmulpd	zmm5, zmm5, zmm28		;; r2+-s = r2+- * sine37			; 33-36		n 37

	L1prefetch wgtreg+5*128+64, L1pt
	vmulpd	zmm2, zmm2, zmm25		;; r1-+s = r1-+ * sine28			; 34-37		n 40
	vmulpd	zmm7, zmm7, zmm25		;; i1-+s = i1-+ * sine28			; 34-37		n 40

	L1prefetchw srcreg+srcinc+rbx+64, L1pt
	vmulpd	zmm14, zmm14, zmm24		;; r1--s = r1-- * sine46			; 35-38		n 44
	vmulpd	zmm3, zmm3, zmm24		;; i1--s = i1-- * sine46			; 35-38		n 44

	L1prefetch wgtreg+0*128+64, L1pt
	vsubpd	zmm10, zmm11, zmm16		;; R5 = (r1++) - (r2++)				; 36-39		n 46
	vaddpd	zmm11, zmm11, zmm16		;; R1 = (r1++) + (r2++)				; 36-39		n 

	L1prefetchw srcreg+srcinc+rbx+d2+64, L1pt
	zfmsubpd zmm16, zmm13, zmm28, zmm1	;; R3s = (r1+-)*sine37 - (i2+-s)		; 37-40		n 42
	zfmaddpd zmm17, zmm4, zmm28, zmm5	;; I3s = (i1+-)*sine37 + (r2+-s)		; 37-40		n 42

	L1prefetch wgtreg+2*128+64, L1pt
	zfmaddpd zmm13, zmm13, zmm28, zmm1	;; R7s = (r1+-)*sine37 + (i2+-s)		; 38-41		n 43
	zfmsubpd zmm4, zmm4, zmm28, zmm5	;; I7s = (i1+-)*sine37 - (r2+-s)		; 38-41		n 43

	L1prefetchw srcreg+srcinc+rbx+d4, L1pt
	vsubpd	zmm5, zmm8, zmm9		;; I5 = (i1++) - (i2++)				; 39-42		n 46
	vaddpd	zmm8, zmm8, zmm9		;; I1 = (i1++) + (i2++)				; 39-42		n 

	L1prefetch wgtreg+4*128, L1pt
	zfnmaddpd zmm1, zmm0, zmm25, zmm2	;; R2s = (r1-+s) - sine28*(i3-+)		; 40-43		n 47
	zfmaddpd zmm9, zmm12, zmm25, zmm7	;; I2s = (i1-+s) + sine28*(r3-+)		; 40-43		n 47

	L1prefetchw srcreg+srcinc+rbx+d4+64, L1pt
	zfmaddpd zmm0, zmm0, zmm25, zmm2	;; R8s = (r1-+s) + sine28*(i3-+)		; 41-44		n 48
	zfnmaddpd zmm12, zmm12, zmm25, zmm7	;; I8s = (i1-+s) - sine28*(r3-+)		; 41-44		n 48

	L1prefetch wgtreg+4*128+64, L1pt
	zfmsubpd zmm2, zmm16, zmm26, zmm17	;; R3s * cosine/sine - I3s (final R3)		; 42-45		n 
	zfmaddpd zmm17, zmm17, zmm26, zmm16	;; I3s * cosine/sine + R3s (final I3)		; 42-45		n 

	L1prefetchw srcreg+srcinc+rbx+d4+d2+d1, L1pt
	zfmaddpd zmm7, zmm13, zmm26, zmm4	;; R7s * cosine/sine + I7s (final R7)		; 43-46		n 
	zfmsubpd zmm4, zmm4, zmm26, zmm13	;; I7s * cosine/sine - R7s (final I7)		; 43-46		n 

	L1prefetch wgtreg+7*128, L1pt
	zfmaddpd zmm16, zmm15, zmm24, zmm14	;; R4s = (r1--s) + sine46*(i3--)		; 44-47		n 49
	zfnmaddpd zmm13, zmm6, zmm24, zmm3	;; I4s = (i1--s) - sine46*(r3--)		; 44-47		n 49

	L1prefetchw srcreg+srcinc+rbx+d4+d2+d1+64, L1pt
	zfnmaddpd zmm15, zmm15, zmm24, zmm14	;; R6s = (r1--s) - sine46*(i3--)		; 45-48		n 50
	zfmaddpd zmm6, zmm6, zmm24, zmm3	;; I6s = (i1--s) + sine46*(r3--)		; 45-48		n 50

	L1prefetch wgtreg+7*128+64, L1pt
	zfmsubpd zmm14, zmm10, zmm27, zmm5	;; A5 = R5 * cosine/sine - I5			; 46-49		n 51
	zfmaddpd zmm5, zmm5, zmm27, zmm10	;; B5 = I5 * cosine/sine + R5			; 46-49		n 51

	L1prefetchw srcreg+srcinc+rbx+d4+d2, L1pt
	zfmsubpd zmm3, zmm1, zmm23, zmm9	;; R2s * cosine/sine - I2s (final R2)		; 47-50		n 
	zfmaddpd zmm9, zmm9, zmm23, zmm1	;; I2s * cosine/sine + R2s (final I2)		; 47-50		n 

	L1prefetch wgtreg+6*128, L1pt
	zfmaddpd zmm10, zmm0, zmm23, zmm12	;; R8s * cosine/sine + I8s (final R8)		; 48-51		n 
	zfmsubpd zmm12, zmm12, zmm23, zmm0	;; I8s * cosine/sine - R8s (final I8)		; 48-51		n 

	L1prefetchw srcreg+srcinc+rbx+d4+d2+64, L1pt
	zfmsubpd zmm1, zmm16, zmm22, zmm13	;; R4s * cosine/sine - I4s (final R4)		; 49-52		n 
	zfmaddpd zmm13, zmm13, zmm22, zmm16	;; I4s * cosine/sine + R4s (final I4)		; 49-52		n 

	L1prefetch wgtreg+6*128+64, L1pt
	zfmaddpd zmm0, zmm15, zmm22, zmm6	;; R6s * cosine/sine + I6s (final R6)		; 50-53		n 
	zfmsubpd zmm6, zmm6, zmm22, zmm15	;; I6s * cosine/sine - R6s (final I6)		; 50-53		n 

	L1prefetchw srcreg+srcinc+rbx+d1, L1pt
	vmulpd	zmm14, zmm14, zmm21		;; A5 = A5 * sine (final R5)			; 51-54		n 
	vmulpd	zmm5, zmm5, zmm21		;; B5 = B5 * sine (final I5)			; 51-54		n 

;bug/optimize move these port 5 only shuffles forward
	L1prefetchw srcreg+srcinc+rbx+d2+d1, L1pt
	vshufpd	zmm15, zmm11, zmm3, 00000000b	;; r2_6 r1_6 r2_4 r1_4 r2_2 r1_2 r2_0 r1_0		; 29		n 39
	vshufpd	zmm11, zmm11, zmm3, 11111111b	;; r2_7 r1_7 r2_5 r1_5 r2_3 r1_3 r2_1 r1_1		; 30		n 41

	L1prefetchw srcreg+srcinc+rbx, L1pt
	zperm2pd zmm3, zmm30, zmm2, zmm1	;; r4_4 r3_4 r4_6 r3_6 r4_0 r3_0 r4_2 r3_2		; 31-33		n 39
	zperm2pd zmm2, zmm29, zmm2, zmm1	;; r4_5 r3_5 r4_7 r3_7 r4_1 r3_1 r4_3 r3_3		; 32-34		n 41

	L1prefetchw srcreg+srcinc+rbx+d2, L1pt
	vshufpd	zmm1, zmm8, zmm9, 00000000b	;; i2_6 i1_6 i2_4 i1_4 i2_2 i1_2 i2_0 i1_0		; 33		n 47
	vshufpd	zmm8, zmm8, zmm9, 11111111b	;; i2_7 i1_7 i2_5 i1_5 i2_3 i1_3 i2_1 i1_1		; 34		n 51

	L1prefetch wgtreg+1*128, L1pt
	zperm2pd zmm9, zmm30, zmm17, zmm13	;; i4_4 i3_4 i4_6 i3_6 i4_0 i3_0 i4_2 i3_2		; 35-37		n 47
	zperm2pd zmm17, zmm29, zmm17, zmm13	;; i4_5 i3_5 i4_7 i3_7 i4_1 i3_1 i4_3 i3_3		; 36-38		n 51

	L1prefetch wgtreg+3*128, L1pt
	vshufpd	zmm13, zmm14, zmm0, 00000000b	;; r6_6 r5_6 r6_4 r5_4 r6_2 r5_2 r6_0 r5_0		; 37		n 43
	vshufpd	zmm14, zmm14, zmm0, 11111111b	;; r6_7 r5_7 r6_5 r5_5 r6_3 r5_3 r6_1 r5_1		; 38		n 45

	L1prefetch wgtreg+0*128, L1pt
	zperm2pd zmm0, zmm30, zmm7, zmm10	;; r8_4 r7_4 r8_6 r7_6 r8_0 r7_0 r8_2 r7_2		; 39-41		n 43
	zperm2pd zmm7, zmm29, zmm7, zmm10	;; r8_5 r7_5 r8_7 r7_7 r8_1 r7_1 r8_3 r7_3		; 40-42		n 45

	L1prefetch wgtreg+2*128, L1pt
	vblendmpd zmm10{k7}, zmm3, zmm15	;; r4_4 r3_4 r2_4 r1_4 r4_0 r3_0 r2_0 r1_0		; 39		n 45
	vblendmpd zmm15{k7}, zmm15, zmm3	;; r2_6 r1_6 r4_6 r3_6 r2_2 r1_2 r4_2 r3_2		; 40		n 47

;	L1prefetch screg+1*64, L1pt
	vshufpd	zmm3, zmm5, zmm6, 00000000b	;; i6_6 i5_6 i6_4 i5_4 i6_2 i5_2 i6_0 i5_0		; 41		n 49
	vshufpd	zmm5, zmm5, zmm6, 11111111b	;; i6_7 i5_7 i6_5 i5_5 i6_3 i5_3 i6_1 i5_1		; 42		n 53

;	L1prefetch screg+3*64, L1pt
	vblendmpd zmm6{k7}, zmm2, zmm11		;; r4_5 r3_5 r2_5 r1_5 r4_1 r3_1 r2_1 r1_1		; 41		n 49
	vblendmpd zmm11{k7}, zmm11, zmm2	;; r2_7 r1_7 r4_7 r3_7 r2_3 r1_3 r4_3 r3_3		; 42		n 51

;	L1prefetch screg+5*64, L1pt
	zperm2pd zmm2, zmm30, zmm4, zmm12	;; i8_4 i7_4 i8_6 i7_6 i8_0 i7_0 i8_2 i7_2		; 43-45		n 49
	zperm2pd zmm4, zmm29, zmm4, zmm12	;; i8_5 i7_5 i8_7 i7_7 i8_1 i7_1 i8_3 i7_3		; 44-46		n 59

;	L1prefetch screg+7*64, L1pt
	vblendmpd zmm12{k7}, zmm0, zmm13	;; r8_4 r7_4 r6_4 r5_4 r8_0 r7_0 r6_0 r5_0		; 43		n 45
	vblendmpd zmm13{k7}, zmm13, zmm0	;; r6_6 r5_6 r8_6 r7_6 r6_2 r5_2 r8_2 r7_2		; 44		n 47

;	L1prefetch screg+0*64, L1pt
	vshuff64x2 zmm0, zmm10, zmm12, 01000100b;; r8_0 r7_0 r6_0 r5_0 r4_0 r3_0 r2_0 r1_0 (R1)		; 45-47
	vshuff64x2 zmm10, zmm10, zmm12, 11101110b;;r8_4 r7_4 r6_4 r5_4 r4_4 r3_4 r2_4 r1_4 (R5)		; 46-48

;	L1prefetch screg+2*64, L1pt
	vblendmpd zmm12{k7}, zmm7, zmm14	;; r8_5 r7_5 r6_5 r6_5 r8_1 r7_1 r6_1 r5_1		; 45		n 49
	vblendmpd zmm14{k7}, zmm14, zmm7	;; r6_7 r5_7 r8_7 r7_7 r6_3 r5_3 r8_3 r7_3		; 46		n 51

;	L1prefetch screg+4*64, L1pt
	vshuff64x2 zmm7, zmm15, zmm13, 00010001b ;;r8_2 r7_2 r6_2 r5_2 r4_2 r3_2 r2_2 r1_2 (R3)		; 47-49
	vshuff64x2 zmm15, zmm15, zmm13, 10111011b;;r8_6 r7_6 r6_6 r5_6 r4_6 r3_6 r2_6 r1_6 (R7)		; 48-50

;	L1prefetch screg+6*64, L1pt
	vblendmpd zmm13{k7}, zmm9, zmm1		;; i4_4 i3_4 i2_4 i1_4 i4_0 i3_0 i2_0 i1_0		; 47		n 53
	vblendmpd zmm1{k7}, zmm1, zmm9		;; i2_6 i1_6 i4_6 i3_6 i2_2 i1_2 i4_2 i3_2		; 48		n 55
	zstore	[srcreg], zmm0			;; Store R1						; 48

;	L1prefetch screg+8*64+1*128, L1pt
	vshuff64x2 zmm9, zmm6, zmm12, 01000100b ;; r8_1 r7_1 r6_1 r5_1 r4_1 r3_1 r2_1 r1_1 (R2)		; 49-51
	vshuff64x2 zmm6, zmm6, zmm12, 11101110b ;; r8_5 r7_5 r6_5 r5_5 r4_5 r3_5 r2_5 r1_5 (R6)		; 50-52
	zstore	[srcreg+d4], zmm10		;; Store R5						; 49

;	L1prefetch screg+8*64+0*128, L1pt
	vblendmpd zmm12{k7}, zmm2, zmm3		;; i8_4 i7_4 i6_4 i5_4 i8_0 i7_0 i6_0 i5_0		; 49		n 53
	vblendmpd zmm3{k7}, zmm3, zmm2		;; i6_6 i5_6 i8_6 i7_6 i6_2 i5_2 i8_2 i7_2		; 50		n 55
	zstore	[srcreg+d2], zmm7		;; Store R3						; 50
	zstore	[srcreg+d4+d2], zmm15		;; Store R7						; 51

;	L1prefetch screg+8*64+2*128, L1pt
	vshuff64x2 zmm2, zmm11, zmm14, 00010001b;; r8_3 r7_3 r6_3 r5_3 r4_3 r3_3 r2_3 r1_3 (R4)		; 51-53
	vshuff64x2 zmm11, zmm11, zmm14, 10111011b;;r8_7 r7_7 r6_7 r5_7 r4_7 r3_7 r2_7 r1_7 (R8)		; 52-54

;	L1prefetch screg+8*64+1*128+64, L1pt
	vblendmpd zmm14{k7}, zmm17, zmm8	;; i4_5 i3_5 i2_5 i1_5 i4_1 i3_1 i2_1 i1_1		; 51		n 57
	vblendmpd zmm8{k7}, zmm8, zmm17		;; i2_7 i1_7 i4_7 i3_7 i2_3 i1_3 i4_3 i3_3		; 52		n 59
	zstore	[srcreg+d1], zmm9		;; Store R2						; 52
	zstore	[srcreg+d4+d1], zmm6		;; Store R6						; 53

;	L1prefetch screg+8*64+3*128+64, L1pt
	vshuff64x2 zmm17, zmm13, zmm12, 01000100b;;i8_0 i7_0 i6_0 i5_0 i4_0 i3_0 i2_0 i1_0 (I1)		; 53-55
	vshuff64x2 zmm13, zmm13, zmm12, 11101110b;;i8_4 i7_4 i6_4 i5_4 i4_4 i3_4 i2_4 i1_4 (I5)		; 54-56

;	L1prefetch screg+8*64+0*128+64, L1pt
	vblendmpd zmm12{k7}, zmm4, zmm5		;; i8_5 i7_5 i6_5 i5_5 i8_1 i7_1 i6_1 i5_1		; 53		n 57
	vblendmpd zmm5{k7}, zmm5, zmm4		;; i6_7 i5_7 i8_7 i7_7 i6_3 i5_3 i8_3 i7_3		; 54		n 59
	zstore	[srcreg+d2+d1], zmm2		;; Store R4						; 54
	zstore	[srcreg+d4+d2+d1], zmm11	;; Store R8						; 55

;	L1prefetch screg+8*64+2*128+64, L1pt
	vshuff64x2 zmm4, zmm1, zmm3, 00010001b	;; i8_2 i7_2 i6_2 i5_2 i4_2 i3_2 i2_2 i1_2 (I3)		; 55-57
	vshuff64x2 zmm1, zmm1, zmm3, 10111011b	;; i8_6 i7_6 i6_6 i5_6 i4_6 i3_6 i2_6 i1_6 (I7)		; 56-58
	zstore	[srcreg+64], zmm17		;; Store R9						; 56

;	L1prefetch screg+8*64+3*128, L1pt
	vshuff64x2 zmm3, zmm14, zmm12, 01000100b;; i8_1 i7_1 i6_1 i5_1 i4_1 i3_1 i2_1 i1_1 (I2)		; 57-59
	vshuff64x2 zmm14, zmm14, zmm12, 11101110b;;i8_5 i7_5 i6_5 i5_5 i4_5 i3_5 i2_5 i1_5 (I6)		; 58-60
	zstore	[srcreg+d4+64], zmm13		;; Store I5						; 57

	vshuff64x2 zmm12, zmm8, zmm5, 00010001b ;; i8_3 i7_3 i6_3 i5_3 i4_3 i3_3 i2_3 i1_3 (I4)		; 59-61
	vshuff64x2 zmm8, zmm8, zmm5, 10111011b	;; i8_7 i7_7 i6_7 i5_7 i4_7 i3_7 i2_7 i1_7 (I8)		; 60-62

	zstore	[srcreg+d2+64], zmm4		;; Store I3					; 58
	zstore	[srcreg+d4+d2+64], zmm1		;; Store I7					; 59
	zstore	[srcreg+d1+64], zmm3		;; Store I2					; 60
	zstore	[srcreg+d4+d1+64], zmm14	;; Store I6					; 61
	zstore	[srcreg+d2+d1+64], zmm12	;; Store I4					; 62
	zstore	[srcreg+d4+d2+d1+64], zmm8	;; Store I8					; 63
	bump	srcreg, srcinc
	ENDM


zr8s_eight_complex_last_unfft_preload MACRO
	vbroadcastsd zmm31, ZMM_SQRTHALF
	mov	r14d, 11110000b
	kmovw	k6, r14d
	knotw	k7, k6
	ENDM

zr8s_eight_complex_last_unfft MACRO srcreg,srcinc,d1,d2,d4,screg,scinc,maxrpt,L1pt,L1pd
	vmovapd	zmm0, [srcreg+d2]			;; d2_7	d2_6 d2_5 d2_4 d2_3 d2_2 d2_1 d2_0
	vmovupd	zmm17, [srcreg+32]			;; (+64 d0_3 d0_2 d0_1 d0_0) d0_7 d0_6 d0_5 d0_4
	vmovapd	zmm1, [srcreg+d2+d1]			;; d3_7	d3_6 d3_5 d3_4 d3_3 d3_2 d3_1 d3_0
	vmovupd	zmm18, [srcreg+d1+32]			;; (+64 d1_3 d1_2 d1_1 d1_0) d1_7 d1_6 d1_5 d1_4
	vblendmpd zmm0{k6}, zmm17, zmm0			;; d2_7 d2_6 d2_5 d2_4 d0_7 d0_6 d0_5 d0_4		; 1		n 2	r5-8
	vblendmpd zmm1{k6}, zmm18, zmm1			;; d3_7 d3_6 d3_5 d3_4 d1_7 d1_6 d1_5 d1_4		; 1		n 2	r5-8

	vmovapd	zmm2, [srcreg+d4+d2]			;; d6_7	d6_6 d6_5 d6_4 d6_3 d6_2 d6_1 d6_0
	vmovupd	zmm19, [srcreg+d4+32]			;; (+64 d4_3 d4_2 d4_1 d4_0) d4_7 d4_6 d4_5 d4_4
	vshufpd	zmm4, zmm0, zmm1, 00000000b		;; d3_6 d2_6 d3_4 d2_4 d1_6 d0_6 d1_4 d0_4		; 2		n 14	*R5/R7
	vblendmpd zmm2{k6}, zmm19, zmm2			;; d6_7	d6_6 d6_5 d6_4 d4_7 d4_6 d4_5 d4_4		; 2		n 4	r5-8

	vmovapd	zmm3, [srcreg+d4+d2+d1]			;; d7_7	d7_6 d7_5 d7_4 d7_3 d7_2 d7_1 d7_0
	vmovupd zmm20, [srcreg+d4+d1+32]		;; (+64 d5_3 d5_2 d5_1 d5_0) d5_7 d5_6 d5_5 d5_4
	vshufpd	zmm0, zmm0, zmm1, 11111111b		;; d3_7 d2_7 d3_5 d2_5 d1_7 d0_7 d1_5 d0_5		; 3		n 10	*R6/R8
	vblendmpd zmm3{k6}, zmm20, zmm3			;; d7_7	d7_6 d7_5 d7_4 d5_7 d5_6 d5_5 d5_4		; 3		n 4	r5-8

	vmovapd	zmm6, [srcreg+64]			;; (+64 d0_7 d0_6 d0_5 d0_4 d0_3 d0_2 d0_1 d0_0)
	vshufpd	zmm5, zmm2, zmm3, 00000000b		;; d7_6 d6_6 d7_4 d6_4 d5_6 d4_6 d5_4 d4_4		; 4		n 14	*R5/R7
	vbroadcastf64x4	zmm6{k7}, [srcreg+d2+64+32]	;; (+64 d0_7 d0_6 d0_5 d0_4 d2_7 d2_6 d2_5 d2_4)	; 4		n 6	i5-8

	vmovapd	zmm7, [srcreg+d1+64]			;; (+64 d1_7 d1_6 d1_5 d1_4 d1_3 d1_2 d1_1 d1_0)
	vshufpd	zmm2, zmm2, zmm3, 11111111b		;; d7_7 d6_7 d7_5 d6_5 d5_7 d4_7 d5_5 d4_5		; 5		n 10	*R6/R8
	vbroadcastf64x4	zmm7{k7}, [srcreg+d2+d1+64+32]	;; (+64 d1_7 d1_6 d1_5 d1_4 d3_7 d3_6 d3_5 d3_4)	; 5		n 6	i5-8

	vmovapd	zmm8, [srcreg+d4+64]			;; (+64 d4_7 d4_6 d4_5 d4_4 d4_3 d4_2 d4_1 d4_0)
	vshufpd	zmm10, zmm6, zmm7, 00000000b		;; (+64 d1_6 d0_6 d1_4 d0_4 d3_6 d2_6 d3_4 d2_4)	; 6		n 16	I5/I7
	vbroadcastf64x4	zmm8{k7}, [srcreg+d4+d2+64+32]	;; (+64 d4_7 d4_6 d4_5 d4_4 d6_7 d6_6 d6_5 d6_4)	; 6		n 8	i5-8

	vmovapd	zmm9, [srcreg+d4+d1+64]			;; (+64 d5_7 d5_6 d5_5 d5_4 d5_3 d5_2 d5_1 d5_0)
	vshufpd	zmm6, zmm6, zmm7, 11111111b		;; (+64 d1_7 d0_7 d1_5 d0_5 d3_7 d2_7 d3_5 d2_5)	; 7		n 12	I6/I8
	vbroadcastf64x4	zmm9{k7}, [srcreg+d4+d2+d1+64+32];;(+64 d5_7 d5_6 d5_5 d5_4 d7_7 d7_6 d7_5 d7_4)	; 7		n 8	i5-8

	vmovapd	zmm12, [srcreg]				;; d0_7	d0_6 d0_5 d0_4 d0_3 d0_2 d0_1 d0_0
	vshufpd	zmm11, zmm8, zmm9, 00000000b		;; (+64 d5_6 d4_6 d5_4 d4_4 d7_6 d6_6 d7_4 d6_4)	; 8		n 16	I5/I7
	vbroadcastf64x4	zmm12{k6}, [srcreg+d2]		;; d2_3 d2_2 d2_1 d2_0 d0_3 d0_2 d0_1 d0_0		; 8		n 18		r1-4

	vmovapd	zmm13, [srcreg+d1]			;; d1_7	d1_6 d1_5 d1_4 d1_3 d1_2 d1_1 d1_0
	vshufpd	zmm8, zmm8, zmm9, 11111111b		;; (+64 d5_7 d4_7 d5_5 d4_5 d7_7 d6_7 d7_5 d6_5)	; 9		n 12	I6/I8
	vbroadcastf64x4	zmm13{k6}, [srcreg+d2+d1]	;; d3_3 d3_2 d3_1 d3_0 d1_3 d1_2 d1_1 d1_0		; 9		n 18		r1-4

	vmovapd	zmm14, [srcreg+d4]			;; d4_7	d4_6 d4_5 d4_4 d4_3 d4_2 d4_1 d4_0
	vshuff64x2 zmm1, zmm0, zmm2, 11011101b		;; d7_7 d6_7 d5_7 d4_7 d3_7 d2_7 d1_7 d0_7 (R8)		; 10-12		n 16	R8
	vbroadcastf64x4	zmm14{k6}, [srcreg+d4+d2]	;; d6_3 d6_2 d6_1 d6_0 d4_3 d4_2 d4_1 d4_0		; 10		n 19		r1-4

	vmovapd	zmm15, [srcreg+d4+d1]			;; d5_7	d5_6 d5_5 d5_4 d5_3 d5_2 d5_1 d5_0
	vshuff64x2 zmm0, zmm0, zmm2, 10001000b		;; d7_5 d6_5 d5_5 d4_5 d3_5 d2_5 d1_5 d0_5 (R6)		; 11-13		n 18	R6
	vbroadcastf64x4	zmm15{k6}, [srcreg+d4+d2+d1]	;; d7_3 d7_2 d7_1 d7_0 d5_3 d5_2 d5_1 d5_0		; 11		n 19		r1-4

	vmovapd	zmm16, [srcreg+d2+64]			;; (+64 d2_7 d2_6 d2_5 d2_4 d2_3 d2_2 d2_1 d2_0)
	vshuff64x2 zmm3, zmm6, zmm8, 01110111b		;; (+64 d7_7 d6_7 d5_7 d4_7 d3_7 d2_7 d1_7 d0_7)(I8)	; 12-14		n 16	I8
	vblendmpd zmm16{k6}, zmm16, zmm17		;; (+64 d0_3 d0_2 d0_1 d0_0 d2_3 d2_2 d2_1 d2_0)	; 12		n 20		i1-4

	vmovapd	zmm17, [srcreg+d2+d1+64]		;; (+64 d3_7 d3_6 d3_5 d3_4 d3_3 d3_2 d3_1 d3_0)
	vshuff64x2 zmm6, zmm6, zmm8, 00100010b		;; (+64 d7_5 d6_5 d5_5 d4_5 d3_5 d2_5 d1_5 d0_5)(I6)	; 13-15		n 18	I6
	vblendmpd zmm17{k6}, zmm17, zmm18		;; (+64 d1_3 d1_2 d1_1 d1_0 d3_3 d3_2 d3_1 d3_0)	; 13		n 20		i1-4

	vmovapd	zmm18, [srcreg+d4+d2+64]		;; (+64 d6_7 d6_6 d6_5 d6_4 d6_3 d6_2 d6_1 d6_0)
	vshuff64x2 zmm7, zmm4, zmm5, 11011101b		;; d7_6 d6_6 d5_6 d4_6 d3_6 d2_6 d1_6 d0_6 (R7)		; 14-16		n 20	R7
	vblendmpd zmm18{k6}, zmm18, zmm19		;; (+64 d4_3 d4_2 d4_1 d4_0 d6_3 d6_2 d6_1 d6_0)	; 14		n 21		i1-4

	vmovapd	zmm19, [srcreg+d4+d2+d1+64]		;; (+64 d7_7 d7_6 d7_5 d7_4 d7_3 d7_2 d7_1 d7_0)
	vshuff64x2 zmm4, zmm4, zmm5, 10001000b		;; d7_4	d6_4 d5_4 d4_4 d3_4 d2_4 d1_4 d0_4 (R5)		; 15-17		n 22	R5
	vblendmpd zmm19{k6}, zmm19, zmm20		;; (+64 d5_3 d5_2 d5_1 d5_0 d7_3 d7_2 d7_1 d7_0)	; 15		n 21		i1-4

	vmovapd zmm30, [screg+8*64+0*128+64]		;; cosine/sine for R2/I2 and R8/I8
	vshuff64x2 zmm9, zmm10, zmm11, 01110111b	;; (+64 d7_6 d6_6 d5_6 d4_6 d3_6 d2_6 d1_6 d0_6)(I7)	; 16-18		n 20	I7
	zfmsubpd zmm22, zmm1, zmm30, zmm3		;; A8 = R8 * cosine/sine - I8				; 16-19		n 24

	vmovapd zmm28, [screg+8*64+2*128+64]		;; cosine/sine for R4/I4 and R6/I6
	vshuff64x2 zmm10, zmm10, zmm11, 00100010b	;; (+64 d7_4 d6_4 d5_4 d4_4 d3_4 d2_4 d1_4 d0_4)(I5)	; 17-19		n 22	I5
	zfmaddpd zmm3, zmm3, zmm30, zmm1		;; B8 = I8 * cosine/sine + R8				; 17-20		n 25

	vmovapd zmm26, [screg+8*64+1*128+64]		;; cosine/sine for R3/I3 and R7/I7
	vshufpd	zmm2, zmm12, zmm13, 11111111b		;; d3_3 d2_3 d3_1 d2_1 d1_3 d0_3 d1_1 d0_1		; 18		n 22		*R2/R4
	zfmsubpd zmm1, zmm0, zmm28, zmm6		;; A6 = R6 * cosine/sine - I6 (first R6/sine)		; 18-21		n 30

	vmovapd zmm27, [screg+8*64+3*128+64]		;; cosine/sine for R5/I5
	vshufpd	zmm20, zmm14, zmm15, 11111111b		;; d7_3 d6_3 d7_1 d6_1 d5_3 d4_3 d5_1 d4_1		; 19		n 22		*R2/R4
	zfmaddpd zmm6, zmm6, zmm28, zmm0		;; B6 = I6 * cosine/sine + R6 (first I6/sine)		; 19-22		n 32

	vmovapd zmm29, [screg+8*64+0*128]		;; sine for R2/I2 and R8/I8
	vshufpd	zmm5, zmm16, zmm17, 11111111b		;; (+64 d1_3 d0_3 d1_1 d0_1 d3_3 d2_3 d3_1 d2_1)	; 20		n 23		I2/I4
	zfmsubpd zmm0, zmm7, zmm26, zmm9		;; A7 = R7 * cosine/sine - I7 (first R7/sine)		; 20-23		n 39

	vmovapd zmm25, [screg+8*64+3*128]		;; sine for R5/I5
	vshufpd	zmm21, zmm18, zmm19, 11111111b		;; (+64 d5_3 d4_3 d5_1 d4_1 d7_3 d6_3 d7_1 d6_1)	; 21		n 23		I2/I4
	zfmaddpd zmm9, zmm9, zmm26, zmm7		;; B7 = I7 * cosine/sine + R7 (first I7/sine)		; 21-24		n 40

	vshuff64x2 zmm8, zmm2, zmm20, 11011101b		;; d7_3 d6_3 d5_3 d4_3 d3_3 d2_3 d1_3 d0_3 (R4)		; 22-24		n 26		R4
	zfmaddpd zmm7, zmm4, zmm27, zmm10		;; A5 = R5 * cosine/sine + I5 (first R5/sine)		; 22-25		n 37

	vshuff64x2 zmm11, zmm5, zmm21, 01110111b	;; (+64 d7_3 d6_3 d5_3 d4_3 d3_3 d2_3 d1_3 d0_3)(I4)	; 23-25		n 26		I4
	zfmsubpd zmm10, zmm10, zmm27, zmm4		;; B5 = I5 * cosine/sine - R5 (first I5/sine)		; 23-26		n 38

	vmovapd zmm27, [screg+8*64+2*128]		;; sine for R4/I4 and R6/I6
	vshuff64x2 zmm2, zmm2, zmm20, 10001000b		;; d7_1 d6_1 d5_1 d4_1 d3_1 d2_1 d1_1 d0_1 (R2)		; 24-26		n 28		R2
	vmulpd	zmm22, zmm22, zmm29			;; A8 = A8 * sine (first R8)				; 24-27		n 35

	vshuff64x2 zmm5, zmm5, zmm21, 00100010b		;; (+64 d7_1 d6_1 d5_1 d4_1 d3_1 d2_1 d1_1 d0_1)(I2)	; 25-27		n 28		I2
	vmulpd	zmm3, zmm3, zmm29			;; B8 = B8 * sine (first I8)				; 25-28		n 36

	vshufpd	zmm12, zmm12, zmm13, 00000000b		;; d3_2 d2_2 d3_0 d2_0 d1_2 d0_2 d1_0 d0_0		; 26		n 30		*R1/R3
	zfmaddpd zmm4, zmm8, zmm28, zmm11		;; A4 = R4 * cosine/sine + I4 (first R4/sine)		; 26-29		n 30

	vshufpd	zmm14, zmm14, zmm15, 00000000b		;; d7_2 d6_2 d7_0 d6_0 d5_2 d4_2 d5_0 d4_0		; 27		n 30		*R1/R3
	zfmsubpd zmm11, zmm11, zmm28, zmm8		;; B4 = I4 * cosine/sine - R4 (first I4/sine)		; 27-30		n 32

	vshufpd	zmm16, zmm16, zmm17, 00000000b		;; (+64 d1_2 d0_2 d1_0 d0_0 d3_2 d2_2 d3_0 d2_0)	; 28		n 31		I1/I3
	zfmaddpd zmm8, zmm2, zmm30, zmm5		;; A2 = R2 * cosine/sine + I2				; 28-31		n 35

	vshufpd	zmm18, zmm18, zmm19, 00000000b		;; (+64 d5_2 d4_2 d5_0 d4_0 d7_2 d6_2 d7_0 d6_0)	; 29		n 31		I1/I3
	zfmsubpd zmm5, zmm5, zmm30, zmm2		;; B2 = I2 * cosine/sine - R2				; 29-32		n 36

	vshuff64x2 zmm13, zmm12, zmm14, 11011101b	;; d7_2 d6_2 d5_2 d4_2 d3_2 d2_2 d1_2 d0_2 (R3)		; 30-32		n 34		R3
	vaddpd	zmm2, zmm4, zmm1			;; R4/sine + R6/sine					; 30-33		n 41

	vshuff64x2 zmm15, zmm16, zmm18, 01110111b	;; (+64 d7_2 d6_2 d5_2 d4_2 d3_2 d2_2 d1_2 d0_2)(I3)	; 31-33		n 34		I3
	vsubpd	zmm4, zmm4, zmm1			;; R4/sine - R6/sine					; 31-34		n 42

	vshuff64x2 zmm12, zmm12, zmm14, 10001000b	;; d7_0 d6_0 d5_0 d4_0 d3_0 d2_0 d1_0 d0_0 (R1)		; 32-34		n 37		R1
	vaddpd	zmm1, zmm11, zmm6			;; I4/sine + I6/sine					; 32-35		n 44

	vshuff64x2 zmm16, zmm16, zmm18, 00100010b	;; (+64 d7_0 d6_0 d5_0 d4_0 d3_0 d2_0 d1_0 d0_0)(I1)	; 33-35		n 38		I1
	vsubpd	zmm11, zmm11, zmm6			;; I4/sine - I6/sine					; 33-36		n 43

	zfmaddpd zmm6, zmm13, zmm26, zmm15		;; A3 = R3 * cosine/sine + I3 (first R3/sine)		; 34-37		n 39
	zfmsubpd zmm15, zmm15, zmm26, zmm13		;; B3 = I3 * cosine/sine - R3 (first I3/sine)		; 34-37		n 40

	vmovapd zmm26, [screg+8*64+1*128]		;; sine for R3/I3 and R7/I7
	zfmaddpd zmm13, zmm8, zmm29, zmm22		;; R2 * sine + R8					; 35-38		n 41
	zfmsubpd zmm8, zmm8, zmm29, zmm22		;; R2 * sine - R8					; 35-38		n 42

	vmovapd zmm20, [screg+0*64]			;; premultiplier cosine/sine for R1/I1
	zfmaddpd zmm14, zmm5, zmm29, zmm3		;; I2 * sine + I8					; 36-39		n 44
	zfmsubpd zmm5, zmm5, zmm29, zmm3		;; I2 * sine - I8					; 36-39		n 43

	vmovapd zmm18, [screg+4*64]			;; premultiplier cosine/sine for R5/I5
	zfmaddpd zmm3, zmm7, zmm25, zmm12		;; R1 + R5 * sine					; 37-40		n 45
	zfnmaddpd zmm7, zmm7, zmm25, zmm12		;; R1 - R5 * sine					; 37-40		n 47

	vmovapd zmm19, [screg+2*64]			;; premultiplier cosine/sine for R3/I3
	zfmaddpd zmm12, zmm10, zmm25, zmm16		;; I1 + I5 * sine					; 38-41		n 46
	zfnmaddpd zmm10, zmm10, zmm25, zmm16		;; I1 - I5 * sine					; 38-41		n 48

	vmovapd zmm25, [screg+6*64]			;; premultiplier cosine/sine for R7/I7
	vaddpd	zmm16, zmm6, zmm0			;; R3/sine + R7/sine					; 39-42		n 45
	vsubpd	zmm6, zmm6, zmm0			;; R3/sine - R7/sine					; 39-42		n 48

	vmovapd zmm24, [screg+1*64]			;; premultiplier cosine/sine for R2/I2
	vaddpd	zmm0, zmm15, zmm9			;; I3/sine + I7/sine					; 40-43		n 46
	vsubpd	zmm15, zmm15, zmm9			;; I3/sine - I7/sine					; 40-43		n 47

	vmovapd zmm22, [screg+5*64]			;; premultiplier cosine/sine for R6/I6
	zfmaddpd zmm9, zmm2, zmm27, zmm13		;; r2++ = (r2+r8) + (r4+r6) * sine			; 41-44		n 51
	zfnmaddpd zmm2, zmm2, zmm27, zmm13		;; r2+- = (r2+r8) - (r4+r6) * sine			; 41-44		n 49

	vmovapd zmm23, [screg+3*64]			;; premultiplier cosine/sine for R4/I4
	zfmaddpd zmm13, zmm4, zmm27, zmm8		;; r2-+ = (r2-r8) + (r4-r6) * sine			; 42-45		n 50
	zfnmaddpd zmm4, zmm4, zmm27, zmm8		;; r2-- = (r2-r8) - (r4-r6) * sine			; 42-45		n 54

	vmovapd zmm21, [screg+7*64]			;; premultiplier cosine/sine for R8/I8
	zfmaddpd zmm8, zmm11, zmm27, zmm5		;; i2-+ = (i2-i8) + (i4-i6) * sine			; 43-46		n 49
	zfnmaddpd zmm11, zmm11, zmm27, zmm5		;; i2-- = (i2-i8) - (i4-i6) * sine			; 43-46		n 53
	bump	screg, scinc

	L1prefetchw srcreg+srcinc+d2, L1pt
	zfmaddpd zmm5, zmm1, zmm27, zmm14		;; i2++ = (i2+i8) + (i4+i6) * sine			; 44-47		n 52
	zfnmaddpd zmm1, zmm1, zmm27, zmm14		;; i2+- = (i2+i8) - (i4+i6) * sine			; 44-47		n 50

	L1prefetchw srcreg+srcinc, L1pt
	zfmaddpd zmm14, zmm16, zmm26, zmm3		;; r1++ = (r1+r5) + (r3+r7) * sine			; 45-48		n 51
	zfnmaddpd zmm16, zmm16, zmm26, zmm3		;; r1+- = (r1+r5) - (r3+r7) * sine			; 45-48		n 53

	L1prefetchw srcreg+srcinc+64, L1pt
	zfmaddpd zmm3, zmm0, zmm26, zmm12		;; i1++ = (i1+i5) + (i3+i7) * sine			; 46-49		n 52
	zfnmaddpd zmm0, zmm0, zmm26, zmm12		;; i1+- = (i1+i5) - (i3+i7) * sine			; 46-49		n 54

	L1prefetchw srcreg+srcinc+d2+d1, L1pt
	zfmaddpd zmm12, zmm15, zmm26, zmm7		;; r1-+ = (r1-r5) + (i3-i7) * sine			; 47-50		n 55
	zfnmaddpd zmm15, zmm15, zmm26, zmm7		;; r1-- = (r1-r5) - (i3-i7) * sine			; 47-50		n 59

	L1prefetchw srcreg+srcinc+d1, L1pt
	zfmaddpd zmm7, zmm6, zmm26, zmm10		;; i1-+ = (i1-i5) + (r3-r7) * sine			; 48-51		n 61
	zfnmaddpd zmm6, zmm6, zmm26, zmm10		;; i1-- = (i1-i5) - (r3-r7) * sine			; 48-51		n 57

	L1prefetchw srcreg+srcinc+d1+64, L1pt
	vaddpd	zmm10, zmm2, zmm8			;; r2+-+ = (r2+-) + (i2-+)				; 49-52		n 55
	vsubpd	zmm2, zmm2, zmm8			;; r2+-- = (r2+-) - (i2-+)				; 49-52		n 59

	L1prefetchw srcreg+srcinc+d4+d2, L1pt
	vaddpd	zmm8, zmm13, zmm1			;; r2-++ = (r2-+) + (i2+-)				; 50-53		n 61
	vsubpd	zmm13, zmm13, zmm1			;; r2-+- = (r2-+) - (i2+-)				; 50-53		n 57

	L1prefetchw srcreg+srcinc+d4, L1pt
	vaddpd	zmm1, zmm14, zmm9			;; R1 = (r1++) + (r2++)					; 51-54		n 56
	vsubpd	zmm14, zmm14, zmm9			;; R5 = (r1++) - (r2++)					; 51-54		n 58

	L1prefetchw srcreg+srcinc+d4+64, L1pt
	vaddpd	zmm9, zmm3, zmm5			;; I1 = (i1++) + (i2++)					; 52-55		n 56
	vsubpd	zmm3, zmm3, zmm5			;; I5 = (i1++) - (i2++)					; 52-55		n 58

	L1prefetchw srcreg+srcinc+d4+d2+d1, L1pt
	vaddpd	zmm5, zmm16, zmm11			;; R3 = (r1+-) + (i2--)					; 53-56		n 60
	vsubpd	zmm16, zmm16, zmm11			;; R7 = (r1+-) - (i2--)					; 53-56		n 62

	L1prefetchw srcreg+srcinc+d4+d1, L1pt
	vsubpd	zmm11, zmm0, zmm4			;; I3 = (i1+-) - (r2--)					; 54-57		n 60
	vaddpd	zmm0, zmm0, zmm4			;; I7 = (i1+-) + (r2--)					; 54-57		n 62

	L1prefetchw srcreg+srcinc+d4+d1+64, L1pt
	zfmaddpd zmm4, zmm10, zmm31, zmm12		;; R2 = (r1-+) + .707*(r2+-+)				; 55-58		n 63
	zfnmaddpd zmm10, zmm10, zmm31, zmm12		;; R6 = (r1-+) - .707*(r2+-+)				; 55-58		n 64

	L1prefetchw srcreg+srcinc+d2+64, L1pt
	zfmaddpd zmm12, zmm1, zmm20, zmm9		;; A1 = R1 * cosine + I1				; 56-59
	zfmsubpd zmm9, zmm9, zmm20, zmm1		;; B1 = I1 * cosine - R1				; 56-59

	L1prefetchw srcreg+srcinc+d2+d1+64, L1pt
	zfnmaddpd zmm1, zmm13, zmm31, zmm6		;; I2 = (i1--) - .707*(r2-+-)				; 57-60		n 63
	zfmaddpd zmm13, zmm13, zmm31, zmm6		;; I6 = (i1--) + .707*(r2-+-)				; 57-60		n 64

	L1prefetchw srcreg+srcinc+d4+d2+64, L1pt
	zfmaddpd zmm6, zmm14, zmm18, zmm3		;; A5 = R5 * cosine + I5				; 58-61
	zfmsubpd zmm3, zmm3, zmm18, zmm14		;; B5 = I5 * cosine - R5				; 58-61

	L1prefetchw srcreg+srcinc+d4+d2+d1+64, L1pt
	zfnmaddpd zmm14, zmm2, zmm31, zmm15		;; R4 = (r1--) - .707*(r2+--)				; 59-62		n 65
	zfmaddpd zmm2, zmm2, zmm31, zmm15		;; R8 = (r1--) + .707*(r2+--)				; 59-62		n 66

	zfmaddpd zmm15, zmm5, zmm19, zmm11		;; A3 = R3 * cosine + I3				; 60-63
	zfmsubpd zmm11, zmm11, zmm19, zmm5		;; B3 = I3 * cosine - R3				; 60-63
	zstore	[srcreg], zmm12				;; Save R1						; 60

	zfnmaddpd zmm5, zmm8, zmm31, zmm7		;; I4 = (i1-+) - .707*(r2-++)				; 61-64		n 65
	zfmaddpd zmm8, zmm8, zmm31, zmm7		;; I8 = (i1-+) + .707*(r2-++)				; 61-64		n 66
	zstore	[srcreg+64], zmm9			;; Save I1						; 60+1

	zfmaddpd zmm7, zmm16, zmm25, zmm0		;; A7 = R7 * cosine + I7				; 62-65
	zfmsubpd zmm0, zmm0, zmm25, zmm16		;; B7 = I7 * cosine - R7				; 62-65
	zstore	[srcreg+d4], zmm6			;; Save R5						; 62

	zfmaddpd zmm16, zmm4, zmm24, zmm1		;; A2 = R2 * cosine + I2				; 63-66
	zfmsubpd zmm1, zmm1, zmm24, zmm4		;; B2 = I2 * cosine - R2				; 63-66
	zstore	[srcreg+d4+64], zmm3			;; Save I5						; 62+1

	zfmaddpd zmm4, zmm10, zmm22, zmm13		;; A6 = R6 * cosine + I6				; 64-67
	zfmsubpd zmm13, zmm13, zmm22, zmm10		;; B6 = I6 * cosine - R6				; 64-67
	zstore	[srcreg+d2], zmm15			;; Save R3						; 64

	zfmaddpd zmm10, zmm14, zmm23, zmm5		;; A4 = R4 * cosine + I4				; 65-68
	zfmsubpd zmm5, zmm5, zmm23, zmm14		;; B4 = I4 * cosine - R4				; 65-68
	zstore	[srcreg+d2+64], zmm11			;; Save I3						; 64+1

	zfmaddpd zmm14, zmm2, zmm21, zmm8		;; A8 = R8 * cosine + I8				; 66-69
	zfmsubpd zmm8, zmm8, zmm21, zmm2		;; B8 = I8 * cosine - R8				; 66-69

	zstore	[srcreg+d4+d2], zmm7			;; Save R7						; 66
	zstore	[srcreg+d4+d2+64], zmm0			;; Save I7						; 66+1
	zstore	[srcreg+d1], zmm16			;; Save R2						; 67+1
	zstore	[srcreg+d1+64], zmm1			;; Save I2						; 67+2
	zstore	[srcreg+d4+d1], zmm4			;; Save R6						; 68+2
	zstore	[srcreg+d4+d1+64], zmm13		;; Save I6						; 68+3
	zstore	[srcreg+d2+d1], zmm10			;; Save R4						; 69+3
	zstore	[srcreg+d2+d1+64], zmm5			;; Save I4						; 69+4
	zstore	[srcreg+d4+d2+d1], zmm14		;; Save R8						; 70+4
	zstore	[srcreg+d4+d2+d1+64], zmm8		;; Save I8						; 70+5
	bump	srcreg, srcinc
	ENDM


;; Merge macros zr8fs_eight_complex_first_fft and zr8b_eight_complex_djbfft into one more efficient macro

zr8fs_sixtyfour_complex_first_fft_preload MACRO
	vbroadcastsd zmm31, ZMM_SQRTHALF
	vpmovzxbq zmm30, ZMM_PERMUTE1		;; zmm30 = 8+4 0+4 8+6 0+6 8+0 0+0 8+2 0+2 [msw at left]
	vpmovzxbq zmm29, ZMM_PERMUTE2		;; zmm29 = 8+5 0+5 8+7 0+7 8+1 0+1 8+3 0+3
	mov	r14d, 00110011b			;; For vblendmpd during swizzle
	kmovw	k7, r14d
	ENDM

zr8fs_sixtyfour_complex_first_fft MACRO srcreg,srcinc,d1,d2,d4,screg,scinc,sc2reg,sc2inc,wgtreg,wgtinc,maxrpt,L1pt,L1pd
	vmovapd	zmm9, [srcreg+d1+64][rbx]	;; I2
	vmulpd	zmm9, zmm9, [wgtreg+1*128+64]	;; apply weight I2 over R2 to I2		; 1-4		n 7
	vmovapd	zmm11, [srcreg+d2+d1+64][rbx]	;; I4
	vmulpd	zmm11, zmm11, [wgtreg+3*128+64]	;; apply weight I4 over R4 to I4		; 1-4		n 8

	vmovapd	zmm5, [srcreg+d4+d1][rbx]	;; R6
	vmulpd	zmm5, zmm5, [wgtreg+5*128]	;; apply weight to R6				; 2-5		n 9
	vmovapd	zmm13, [srcreg+d4+d1+64][rbx]	;; I6
	vmulpd	zmm13, zmm13, [wgtreg+5*128+64]	;; apply weight to I6				; 2-5		n 9

	vmovapd	zmm8, [srcreg+64][rbx]		;; I1
	vmulpd	zmm8, zmm8, [wgtreg+64]		;; apply weight I1 over R1 to I1		; 3-6		n 11
	vmovapd	zmm10, [srcreg+d2+64][rbx]	;; I3
	vmulpd	zmm10, zmm10, [wgtreg+2*128+64]	;; apply weight I3 over R3 to I3		; 3-6		n 12

	vmovapd	zmm4, [srcreg+d4][rbx]		;; R5
	vmulpd	zmm4, zmm4, [wgtreg+4*128]	;; apply weight to R5				; 4-7		n 13
	vmovapd	zmm12, [srcreg+d4+64][rbx]	;; I5
	vmulpd	zmm12, zmm12, [wgtreg+4*128+64]	;; apply weight to I5				; 4-7		n 13

	vmovapd	zmm7, [srcreg+d4+d2+d1][rbx]	;; R8
	vmulpd	zmm7, zmm7, [wgtreg+7*128]	;; apply weight to R8				; 5-8		n 10
	vmovapd	zmm15, [srcreg+d4+d2+d1+64][rbx];; I8
	vmulpd	zmm15, zmm15, [wgtreg+7*128+64]	;; apply weight to I8				; 5-8		n 10

	vmovapd	zmm6, [srcreg+d4+d2][rbx]	;; R7
	vmulpd	zmm6, zmm6, [wgtreg+6*128]	;; apply weight to R7				; 6-9		n 14
	vmovapd	zmm14, [srcreg+d4+d2+64][rbx]	;; I7
	vmulpd	zmm14, zmm14, [wgtreg+6*128+64]	;; apply weight to I7				; 6-9		n 14

;; Apply the complex premultipliers

	vmovapd	zmm1, [srcreg+d1][rbx]		;; R2
	vmovapd zmm28, [screg+1*64]		;; premultiplier cosine/sine for R2/I2
	zfmsubpd zmm16, zmm1, zmm28, zmm9	;; A2 = R2 * cosine - I2			; 7-10		n 15
	zfmaddpd zmm9, zmm9, zmm28, zmm1	;; B2 = I2 * cosine + R2			; 7-10		n 18

	vmovapd	zmm3, [srcreg+d2+d1][rbx]	;; R4
	vmovapd zmm28, [screg+3*64]		;; premultiplier cosine/sine for R4/I4
	zfmsubpd zmm1, zmm3, zmm28, zmm11	;; A4 = R4 * cosine - I4			; 8-11		n 16
	zfmaddpd zmm11, zmm11, zmm28, zmm3	;; B4 = I4 * cosine + R4			; 8-11		n 17

	vmovapd zmm28, [screg+5*64]		;; premultiplier cosine/sine for R6/I6
	zfmsubpd zmm3, zmm5, zmm28, zmm13	;; A6 = R6 * cosine - I6			; 9-12		n 15
	zfmaddpd zmm13, zmm13, zmm28, zmm5	;; B6 = I6 * cosine + R6			; 9-12		n 18

	vmovapd zmm28, [screg+7*64]		;; premultiplier cosine/sine for R8/I8
	zfmsubpd zmm5, zmm7, zmm28, zmm15	;; A8 = R8 * cosine - I8			; 10-13		n 16
	zfmaddpd zmm15, zmm15, zmm28, zmm7	;; B8 = I8 * cosine + R8			; 10-13		n 17

	vmovapd	zmm0, [srcreg][rbx]		;; R1
	vmovapd zmm28, [screg+0*64]		;; premultiplier cosine/sine for R1/I1
	zfmsubpd zmm7, zmm0, zmm28, zmm8	;; A1 = R1 * cosine - I1			; 11-14		n 19
	zfmaddpd zmm8, zmm8, zmm28, zmm0	;; B1 = I1 * cosine + R1			; 11-14		n 21

	vmovapd	zmm2, [srcreg+d2][rbx]		;; R3
	vmovapd zmm28, [screg+2*64]		;; premultiplier cosine/sine for R3/I3
	zfmsubpd zmm0, zmm2, zmm28, zmm10	;; A3 = R3 * cosine - I3			; 12-15		n 20
	zfmaddpd zmm10, zmm10, zmm28, zmm2	;; B3 = I3 * cosine + R3			; 12-15		n 22

	vmovapd zmm28, [screg+4*64]		;; premultiplier cosine/sine for R5/I5
	zfmsubpd zmm2, zmm4, zmm28, zmm12	;; A5 = R5 * cosine - I5			; 13-16		n 19
	zfmaddpd zmm12, zmm12, zmm28, zmm4	;; B5 = I5 * cosine + R5			; 13-16		n 21

	vmovapd zmm28, [screg+6*64]		;; premultiplier cosine/sine for R7/I7
	zfmsubpd zmm4, zmm6, zmm28, zmm14	;; A7 = R7 * cosine - I7			; 14-17		n 20
	zfmaddpd zmm14, zmm14, zmm28, zmm6	;; B7 = I7 * cosine + R7			; 14-17		n 22

;; Copied from common 8-complex DJB FFT macro

	vmovapd zmm28, [wgtreg+1*128]		;; weight for R2
	zfmsubpd zmm6, zmm16, zmm28, zmm3	;; R2*weightR2-R6				; 15-18		n 23
	zfmaddpd zmm16, zmm16, zmm28, zmm3	;; R2*weightR2+R6				; 15-18		n 25

	vmovapd zmm27, [wgtreg+3*128]		;; weight for R4
	zfmsubpd zmm3, zmm1, zmm27, zmm5	;; R4*weightR4-R8				; 16-19		n 23
	zfmaddpd zmm1, zmm1, zmm27, zmm5	;; R4*weightR4+R8				; 16-19		n 25

	vmovapd zmm26, [wgtreg+0*128]		;; weight for R1
	zfmsubpd zmm5, zmm11, zmm27, zmm15	;; I4*weightR4-I8				; 17-20		n 24
	zfmaddpd zmm11, zmm11, zmm27, zmm15	;; I4*weightR4+I8				; 17-20		n 26

	vmovapd zmm27, [wgtreg+2*128]		;; weight for R3
	zfmsubpd zmm15, zmm9, zmm28, zmm13	;; I2*weightR2-I6				; 18-21		n 24
	zfmaddpd zmm9, zmm9, zmm28, zmm13	;; I2*weightR2+I6				; 18-21		n 26
	bump	wgtreg, wgtinc

	vmovapd zmm28, [screg+8*64+1*128]	;; sine for R3/I3 and R7/I7 (w^2)		;		n 33
	zfmaddpd zmm13, zmm7, zmm26, zmm2	;; R1*weightR1+R5				; 19-22		n 27
	zfmsubpd zmm7, zmm7, zmm26, zmm2	;; R1*weightR1-R5				; 19-22		n 28

	vmovapd zmm25, [screg+8*64+0*128]	;; sine for R2/I2 and R8/I8 (w^1)		;		n 34
	zfmaddpd zmm2, zmm0, zmm27, zmm4	;; R3*weightR3+R7				; 20-23		n 27
	zfmsubpd zmm0, zmm0, zmm27, zmm4	;; R3*weightR3-R7				; 20-23		n 31

	vmovapd zmm24, [screg+8*64+2*128]	;; sine for R4/I4 and R6/I6 (w^3)		;		n 35
	zfmaddpd zmm4, zmm8, zmm26, zmm12	;; I1*weightR1+I5				; 21-24		n 30
	zfmsubpd zmm8, zmm8, zmm26, zmm12	;; I1*weightR1-I5				; 21-24		n 29

	vmovapd zmm23, [screg+8*64+0*128+64]	;; cosine/sine for R2/I2 and R8/I8 (w^1)	;		n 44
	zfmaddpd zmm12, zmm10, zmm27, zmm14	;; I3*weightR3+I7				; 22-25		n 30
	zfmsubpd zmm10, zmm10, zmm27, zmm14	;; I3*weightR3-I7				; 22-25		n 32

	vmovapd zmm26, [screg+8*64+1*128+64]	;; cosine/sine for R3/I3 and R7/I7 (w^2)	;		n 45
	vsubpd	zmm14, zmm6, zmm3		;; r2-- = (r2-r6) - (r4-r8)			; 23-26		n 28
	vaddpd	zmm6, zmm6, zmm3		;; r2-+ = (r2-r6) + (r4-r8)			; 23-26		n 31

	vmovapd zmm22, [screg+8*64+2*128+64]	;; cosine/sine for R4/I4 and R6/I6 (w^3)	;		n 46
	vsubpd	zmm3, zmm15, zmm5		;; i2-- = (i2-i6) - (i4-i8)			; 24-27		n 29
	vaddpd	zmm15, zmm15, zmm5		;; i2-+ = (i2-i6) + (i4-i8)			; 24-27		n 32

	vmovapd zmm27, [screg+8*64+3*128+64]	;; cosine/sine for R5/I5 (w^4)			;		n 47
	vsubpd	zmm5, zmm16, zmm1		;; r2+- = (r2+r6) - (r4+r8)			; 25-28		n 33
	vaddpd	zmm16, zmm16, zmm1		;; r2++ = (r2+r6) + (r4+r8)			; 25-28		n 36

	vmovapd zmm21, [screg+8*64+3*128]	;; sine for R5/I5 (w^4)				;		n 54
	vsubpd	zmm1, zmm9, zmm11		;; i2+- = (i2+i6) - (i4+i8)			; 26-29		n 33
	vaddpd	zmm9, zmm9, zmm11		;; i2++ = (i2+i6) + (i4+i8)			; 26-29		n 41
	bump	screg, scinc

	vbroadcastsd zmm20, [sc2reg+1*16]	;; sine for R3/I3 and R7/I7 (w^2)		;		n 94
	vaddpd	zmm11, zmm13, zmm2		;; r1++ = (r1+r5) + (r3+r7)			; 27-30		n 36
	vsubpd	zmm13, zmm13, zmm2		;; r1+- = (r1+r5) - (r3+r7)			; 27-30		n 37

	vbroadcastsd zmm19, [sc2reg+0*16]	;; sine for R2/I2 and R8/I8 (w^1)		;		n 95
	zfmaddpd zmm2, zmm14, zmm31, zmm7	;; r1-+ = (r1-r5) + .707*(r2--)			; 28-31		n 34
	zfnmaddpd zmm14, zmm14, zmm31, zmm7	;; r1-- = (r1-r5) - .707*(r2--)			; 28-31		n 35

	vbroadcastsd zmm18, [sc2reg+2*16]	;; sine for R4/I4 and R6/I6 (w^3)		;		n 96
	zfmaddpd zmm7, zmm3, zmm31, zmm8	;; i1-+ = (i1-i5) + .707*(i2--)			; 29-32		n 34
	zfnmaddpd zmm3, zmm3, zmm31, zmm8	;; i1-- = (i1-i5) - .707*(i2--)			; 29-32		n 35

	L1prefetchw srcreg+srcinc+rbx+d1+64, L1pt
	vaddpd	zmm8, zmm4, zmm12		;; i1++ = (i1+i5) + (i3+i7)			; 30-33		n 41
	vsubpd	zmm4, zmm4, zmm12		;; i1+- = (i1+i5) - (i3+i7)			; 30-33		n 37

	L1prefetch wgtreg+1*128+64, L1pt
	zfmaddpd zmm12, zmm6, zmm31, zmm0	;; r3-+ = (r3-r7) + .707*(r2-+)			; 31-34		n 39
	zfnmaddpd zmm6, zmm6, zmm31, zmm0	;; r3-- = (r3-r7) - .707*(r2-+)			; 31-34		n 42

	L1prefetchw srcreg+srcinc+rbx+d2+d1+64, L1pt
	zfmaddpd zmm0, zmm15, zmm31, zmm10	;; i3-+ = (i3-i7) + .707*(i2-+)			; 32-35		n 39
	zfnmaddpd zmm15, zmm15, zmm31, zmm10	;; i3-- = (i3-i7) - .707*(i2-+)			; 32-35		n 42

	L1prefetch wgtreg+3*128+64, L1pt
	vmulpd	zmm1, zmm1, zmm28		;; i2+-s = i2+- * sine37			; 33-36		n 37
	vmulpd	zmm5, zmm5, zmm28		;; r2+-s = r2+- * sine37			; 33-36		n 37

	L1prefetchw srcreg+srcinc+rbx+d4+d1, L1pt
	vmulpd	zmm2, zmm2, zmm25		;; r1-+s = r1-+ * sine28			; 34-37		n 39
	vmulpd	zmm7, zmm7, zmm25		;; i1-+s = i1-+ * sine28			; 34-37		n 39

	L1prefetch wgtreg+5*128, L1pt
	vmulpd	zmm14, zmm14, zmm24		;; r1--s = r1-- * sine46			; 35-38		n 42
	vmulpd	zmm3, zmm3, zmm24		;; i1--s = i1-- * sine46			; 35-38		n 42

	L1prefetchw srcreg+srcinc+rbx+d4+d1+64, L1pt
	vsubpd	zmm10, zmm11, zmm16		;; R5 = (r1++) - (r2++)				; 36-39		n 47
	vaddpd	zmm11, zmm11, zmm16		;; R1 = (r1++) + (r2++)				; 36-39		n 48

	L1prefetch wgtreg+5*128+64, L1pt
	zfmsubpd zmm16, zmm13, zmm28, zmm1	;; R3s = (r1+-)*sine37 - (i2+-s)		; 37-40		n 45
	zfmaddpd zmm17, zmm4, zmm28, zmm5	;; I3s = (i1+-)*sine37 + (r2+-s)		; 37-40		n 45

	L1prefetchw srcreg+srcinc+rbx+64, L1pt
	zfmaddpd zmm13, zmm13, zmm28, zmm1	;; R7s = (r1+-)*sine37 + (i2+-s)		; 38-41		n 48
	zfmsubpd zmm4, zmm4, zmm28, zmm5	;; I7s = (i1+-)*sine37 - (r2+-s)		; 38-41		n 48

	vbroadcastsd zmm28, [sc2reg+1*16+8]	;; cosine/sine for R3/I3 and R7/I7 (w^2)	;		n 103
	zfnmaddpd zmm1, zmm0, zmm25, zmm2	;; R2s = (r1-+s) - sine28*(i3-+)		; 39-42		n 44
	zfmaddpd zmm5, zmm12, zmm25, zmm7	;; I2s = (i1-+s) + sine28*(r3-+)		; 39-42		n 44

	L1prefetch wgtreg+0*128+64, L1pt
	zfmaddpd zmm0, zmm0, zmm25, zmm2	;; R8s = (r1-+s) + sine28*(i3-+)		; 40-43		n 50
	zfnmaddpd zmm12, zmm12, zmm25, zmm7	;; I8s = (i1-+s) - sine28*(r3-+)		; 40-43		n 50

	vbroadcastsd zmm25, [sc2reg+3*16+8]	;; cosine/sine for R5/I5 (w^4)			;		n 107
	vsubpd	zmm7, zmm8, zmm9		;; I5 = (i1++) - (i2++)				; 41-44		n 47
	vaddpd	zmm8, zmm8, zmm9		;; I1 = (i1++) + (i2++)				; 41-44		n 50

	L1prefetchw srcreg+srcinc+rbx+d2+64, L1pt
	zfmaddpd zmm9, zmm15, zmm24, zmm14	;; R4s = (r1--s) + sine46*(i3--)		; 42-45		n 46
	zfnmaddpd zmm2, zmm6, zmm24, zmm3	;; I4s = (i1--s) - sine46*(r3--)		; 42-45		n 46

	L1prefetch wgtreg+2*128+64, L1pt
	zfnmaddpd zmm15, zmm15, zmm24, zmm14	;; R6s = (r1--s) - sine46*(i3--)		; 43-46		n 52
	zfmaddpd zmm6, zmm6, zmm24, zmm3	;; I6s = (i1--s) + sine46*(r3--)		; 43-46		n 52

	vbroadcastsd zmm24, [sc2reg+0*16+8]	;; cosine/sine for R2/I2 and R8/I8 (w^1)	;		n 108
	zfmsubpd zmm3, zmm1, zmm23, zmm5	;; R2s * cosine/sine - I2s (final R2)		; 44-47		n 48
	zfmaddpd zmm5, zmm5, zmm23, zmm1	;; I2s * cosine/sine + R2s (final I2)		; 44-47		n 50

	L1prefetchw srcreg+srcinc+rbx+d4, L1pt
	zfmsubpd zmm1, zmm16, zmm26, zmm17	;; R3s * cosine/sine - I3s (final R3)		; 45-48		n 52
	zfmaddpd zmm17, zmm17, zmm26, zmm16	;; I3s * cosine/sine + R3s (final I3)		; 45-48		n 54

	L1prefetch wgtreg+4*128, L1pt
	zfmsubpd zmm16, zmm9, zmm22, zmm2	;; R4s * cosine/sine - I4s (final R4)		; 46-49		n 52
	zfmaddpd zmm2, zmm2, zmm22, zmm9	;; I4s * cosine/sine + R4s (final I4)		; 46-49		n 54

	L1prefetchw srcreg+srcinc+rbx+d4+64, L1pt
	zfmsubpd zmm9, zmm10, zmm27, zmm7	;; A5 = R5 * cosine/sine - I5			; 47-50		n 54
	zfmaddpd zmm7, zmm7, zmm27, zmm10	;; B5 = I5 * cosine/sine + R5			; 47-50		n 55

	vbroadcastsd zmm27, [sc2reg+2*16+8]	;; cosine/sine for R4/I4 and R6/I6 (w^3)	;		n 110
	vshufpd	zmm10, zmm11, zmm3, 00000000b	;; r2_6 r1_6 r2_4 r1_4 r2_2 r1_2 r2_0 r1_0	; 48		n 56
	zfmaddpd zmm14, zmm13, zmm26, zmm4	;; R7s * cosine/sine + I7s (final R7)		; 48-51		n 56

	L1prefetch wgtreg+4*128+64, L1pt
	vshufpd	zmm11, zmm11, zmm3, 11111111b	;; r2_7 r1_7 r2_5 r1_5 r2_3 r1_3 r2_1 r1_1	; 49		n 58
	zfmsubpd zmm4, zmm4, zmm26, zmm13	;; I7s * cosine/sine - R7s (final I7)		; 49-52		n 58

	vbroadcastsd zmm26, [sc2reg+3*16]	;; sine for R5/I5 (w^4)				;		n 112
	vshufpd	zmm13, zmm8, zmm5, 00000000b	;; i2_6 i1_6 i2_4 i1_4 i2_2 i1_2 i2_0 i1_0	; 50		n 60
	zfmaddpd zmm3, zmm0, zmm23, zmm12	;; R8s * cosine/sine + I8s (final R8)		; 50-53		n 56
	bump	sc2reg, sc2inc

	L1prefetchw srcreg+srcinc+rbx+d4+d2+d1, L1pt
	vshufpd	zmm8, zmm8, zmm5, 11111111b	;; i2_7 i1_7 i2_5 i1_5 i2_3 i1_3 i2_1 i1_1	; 51		n 64
	zfmsubpd zmm12, zmm12, zmm23, zmm0	;; I8s * cosine/sine - R8s (final I8)		; 51-54		n 58

	L1prefetch wgtreg+7*128, L1pt
	zperm2pd zmm0, zmm30, zmm1, zmm16	;; r4_4 r3_4 r4_6 r3_6 r4_0 r3_0 r4_2 r3_2	; 52-54		n 56			
	zfmaddpd zmm5, zmm15, zmm22, zmm6	;; R6s * cosine/sine + I6s (final R6)		; 52-55		n 60

	L1prefetchw srcreg+srcinc+rbx+d4+d2+d1+64, L1pt
	zperm2pd zmm1, zmm29, zmm1, zmm16	;; r4_5 r3_5 r4_7 r3_7 r4_1 r3_1 r4_3 r3_3	; 53-55		n 58
	zfmsubpd zmm6, zmm6, zmm22, zmm15	;; I6s * cosine/sine - R6s (final I6)		; 53-56		n 62

	L1prefetch wgtreg+7*128+64, L1pt
	zperm2pd zmm15, zmm30, zmm17, zmm2	;; i4_4 i3_4 i4_6 i3_6 i4_0 i3_0 i4_2 i3_2	; 54-56		n 60
	vmulpd	zmm9, zmm9, zmm21		;; A5 = A5 * sine (final R5)			; 54-57		n 60

	L1prefetchw srcreg+srcinc+rbx+d4+d2, L1pt
	zperm2pd zmm17, zmm29, zmm17, zmm2	;; i4_5 i3_5 i4_7 i3_7 i4_1 i3_1 i4_3 i3_3	; 55-57		n 64
	vmulpd	zmm7, zmm7, zmm21		;; B5 = B5 * sine (final I5)			; 55-58		n 62

	L1prefetch wgtreg+6*128, L1pt
	zperm2pd zmm2, zmm30, zmm14, zmm3	;; r8_4 r7_4 r8_6 r7_6 r8_0 r7_0 r8_2 r7_2	; 56-58		n 62
	vblendmpd zmm16{k7}, zmm0, zmm10	;; r4_4 r3_4 r2_4 r1_4 r4_0 r3_0 r2_0 r1_0	; 56		n 64

	L1prefetchw srcreg+srcinc+rbx+d4+d2+64, L1pt
	zperm2pd zmm14, zmm29, zmm14, zmm3	;; r8_5 r7_5 r8_7 r7_7 r8_1 r7_1 r8_3 r7_3	; 57-59		n 66
	vblendmpd zmm10{k7}, zmm10, zmm0	;; r2_6 r1_6 r4_6 r3_6 r2_2 r1_2 r4_2 r3_2	; 57		n 66

	L1prefetch wgtreg+6*128+64, L1pt
	zperm2pd zmm0, zmm30, zmm4, zmm12	;; i8_4 i7_4 i8_6 i7_6 i8_0 i7_0 i8_2 i7_2	; 58-60		n 68
	vblendmpd zmm3{k7}, zmm1, zmm11		;; r4_5 r3_5 r2_5 r1_5 r4_1 r3_1 r2_1 r1_1	; 58		n 68

	L1prefetchw srcreg+srcinc+rbx+d1, L1pt
	zperm2pd zmm4, zmm29, zmm4, zmm12	;; i8_5 i7_5 i8_7 i7_7 i8_1 i7_1 i8_3 i7_3	; 59-61		n 70
	vblendmpd zmm11{k7}, zmm11, zmm1	;; r2_7 r1_7 r4_7 r3_7 r2_3 r1_3 r4_3 r3_3	; 59		n 70

	L1prefetch screg+1*64, L1pt
	vshufpd	zmm1, zmm9, zmm5, 00000000b	;; r6_6 r5_6 r6_4 r5_4 r6_2 r5_2 r6_0 r5_0	; 60		n 62
	vblendmpd zmm12{k7}, zmm15, zmm13	;; i4_4 i3_4 i2_4 i1_4 i4_0 i3_0 i2_0 i1_0	; 60		n 76

	L1prefetchw srcreg+srcinc+rbx+d2+d1, L1pt
	vshufpd	zmm9, zmm9, zmm5, 11111111b	;; r6_7 r5_7 r6_5 r5_5 r6_3 r5_3 r6_1 r5_1	; 61		n 66
	vblendmpd zmm13{k7}, zmm13, zmm15	;; i2_6 i1_6 i4_6 i3_6 i2_2 i1_2 i4_2 i3_2	; 61		n 78

	L1prefetch screg+3*64, L1pt
	vshufpd	zmm15, zmm7, zmm6, 00000000b	;; i6_6 i5_6 i6_4 i5_4 i6_2 i5_2 i6_0 i5_0	; 62		n 68
	vblendmpd zmm5{k7}, zmm2, zmm1		;; r8_4 r7_4 r6_4 r5_4 r8_0 r7_0 r6_0 r5_0	; 62		n 64

	L1prefetch screg+5*64, L1pt
	vshufpd	zmm7, zmm7, zmm6, 11111111b	;; i6_7 i5_7 i6_5 i5_5 i6_3 i5_3 i6_1 i5_1	; 63		n 70
	vblendmpd zmm1{k7}, zmm1, zmm2		;; r6_6 r5_6 r8_6 r7_6 r6_2 r5_2 r8_2 r7_2	; 63		n 66

	L1prefetch screg+7*64, L1pt
	vshuff64x2 zmm2, zmm16, zmm5, 01000100b	;; r8_0 r7_0 r6_0 r5_0 r4_0 r3_0 r2_0 r1_0 (R1)	; 64-66		n 72
	vblendmpd zmm6{k7}, zmm17, zmm8		;; i4_5 i3_5 i2_5 i1_5 i4_1 i3_1 i2_1 i1_1	; 64		n 72

	L1prefetchw srcreg+srcinc+rbx, L1pt
	vshuff64x2 zmm16, zmm16, zmm5, 11101110b;;r8_4 r7_4 r6_4 r5_4 r4_4 r3_4 r2_4 r1_4 (R5)	; 65-67		n 72
	vblendmpd zmm8{k7}, zmm8, zmm17		;; i2_7 i1_7 i4_7 i3_7 i2_3 i1_3 i4_3 i3_3	; 65		n 74

	L1prefetch screg+0*64, L1pt
	vshuff64x2 zmm17, zmm10, zmm1, 00010001b;; r8_2 r7_2 r6_2 r5_2 r4_2 r3_2 r2_2 r1_2 (R3)	; 66-68		n 74
	vblendmpd zmm5{k7}, zmm14, zmm9		;; r8_5 r7_5 r6_5 r6_5 r8_1 r7_1 r6_1 r5_1	; 66		n 68

	L1prefetchw srcreg+srcinc+rbx+d2, L1pt
	vshuff64x2 zmm10, zmm10, zmm1, 10111011b;; r8_6 r7_6 r6_6 r5_6 r4_6 r3_6 r2_6 r1_6 (R7)	; 67-69		n 74
	vblendmpd zmm9{k7}, zmm9, zmm14		;; r6_7 r5_7 r8_7 r7_7 r6_3 r5_3 r8_3 r7_3	; 67		n 70

	L1prefetch screg+2*64, L1pt
	vshuff64x2 zmm14, zmm3, zmm5, 01000100b ;; r8_1 r7_1 r6_1 r5_1 r4_1 r3_1 r2_1 r1_1 (R2)	; 68-70		n 76
	vblendmpd zmm1{k7}, zmm0, zmm15		;; i8_4 i7_4 i6_4 i5_4 i8_0 i7_0 i6_0 i5_0	; 68		n 76

	L1prefetch screg+4*64, L1pt
	vshuff64x2 zmm3, zmm3, zmm5, 11101110b	;; r8_5 r7_5 r6_5 r5_5 r4_5 r3_5 r2_5 r1_5 (R6)	; 69-71		n 76
	vblendmpd zmm15{k7}, zmm15, zmm0	;; i6_6 i5_6 i8_6 i7_6 i6_2 i5_2 i8_2 i7_2	; 69		n 78

	L1prefetch screg+6*64, L1pt
	vshuff64x2 zmm0, zmm11, zmm9, 00010001b	;; r8_3 r7_3 r6_3 r5_3 r4_3 r3_3 r2_3 r1_3 (R4)	; 70-72		n 78
	vblendmpd zmm5{k7}, zmm4, zmm7		;; i8_5 i7_5 i6_5 i5_5 i8_1 i7_1 i6_1 i5_1	; 70		n 72

	L1prefetch wgtreg+1*128, L1pt
	vshuff64x2 zmm11, zmm11, zmm9, 10111011b;; r8_7 r7_7 r6_7 r5_7 r4_7 r3_7 r2_7 r1_7 (R8)	; 71-73		n 78
	vblendmpd zmm7{k7}, zmm7, zmm4		;; i6_7 i5_7 i8_7 i7_7 i6_3 i5_3 i8_3 i7_3	; 71		n 74

	L1prefetch wgtreg+3*128, L1pt
	vshuff64x2 zmm4, zmm6, zmm5, 01000100b	;; i8_1 i7_1 i6_1 i5_1 i4_1 i3_1 i2_1 i1_1 (I2)	; 72-74		n 80
	vaddpd	zmm9, zmm2, zmm16		;; R1+R5					; 72-75		n 88

	L1prefetch wgtreg+0*128, L1pt
	vshuff64x2 zmm6, zmm6, zmm5, 11101110b	;; i8_5 i7_5 i6_5 i5_5 i4_5 i3_5 i2_5 i1_5 (I6)	; 73-75		n 80
	vsubpd	zmm2, zmm2, zmm16		;; R1-R5					; 73-76		n 89

	L1prefetch wgtreg+2*128, L1pt		;; Remaining screg values are preloaded well in advance of their use
	vshuff64x2 zmm16, zmm8, zmm7, 00010001b ;; i8_3 i7_3 i6_3 i5_3 i4_3 i3_3 i2_3 i1_3 (I4)	; 74-76		n 81
	vaddpd	zmm5, zmm17, zmm10		;; R3+R7					; 74-77		n 88

	vshuff64x2 zmm8, zmm8, zmm7, 10111011b	;; i8_7 i7_7 i6_7 i5_7 i4_7 i3_7 i2_7 i1_7 (I8)	; 75-77		n 81
	vsubpd	zmm17, zmm17, zmm10		;; R3-R7					; 75-78		n 92

	vshuff64x2 zmm10, zmm12, zmm1, 01000100b;; i8_0 i7_0 i6_0 i5_0 i4_0 i3_0 i2_0 i1_0 (I1)	; 76-78		n 82
	vsubpd	zmm7, zmm14, zmm3		;; R2-R6					; 76-79		n 84

	vshuff64x2 zmm12, zmm12, zmm1, 11101110b;; i8_4 i7_4 i6_4 i5_4 i4_4 i3_4 i2_4 i1_4 (I5)	; 77-79		n 82
	vaddpd	zmm14, zmm14, zmm3		;; R2+R6					; 77-80		n 86

	vshuff64x2 zmm3, zmm13, zmm15, 00010001b;; i8_2 i7_2 i6_2 i5_2 i4_2 i3_2 i2_2 i1_2 (I3)	; 78-80		n 83
	vsubpd	zmm1, zmm0, zmm11		;; R4-R8					; 78-81		n 84

	vshuff64x2 zmm13, zmm13, zmm15, 10111011b;;i8_6 i7_6 i6_6 i5_6 i4_6 i3_6 i2_6 i1_6 (I7)	; 79-81		n 83
	vaddpd	zmm0, zmm0, zmm11		;; R4+R8					; 79-82		n 86

; Code copied from zr8b_eight_complex_djbfft

	vsubpd	zmm11, zmm4, zmm6		;; I2-I6					; 80-83		n 85
	vaddpd	zmm4, zmm4, zmm6		;; I2+I6					; 80-83		n 87

	vsubpd	zmm6, zmm16, zmm8		;; I4-I8					; 81-84		n 85
	vaddpd	zmm16, zmm16, zmm8		;; I4+I8					; 81-84		n 87

	vaddpd	zmm8, zmm10, zmm12		;; I1+I5					; 82-85		n 91
	vsubpd	zmm10, zmm10, zmm12		;; I1-I5					; 82-85		n 90

	vaddpd	zmm12, zmm3, zmm13		;; I3+I7					; 83-86		n 91
	vsubpd	zmm3, zmm3, zmm13		;; I3-I7					; 83-86		n 93

	vaddpd	zmm13, zmm7, zmm1		;; r2-+ = (r2-r6) + (r4-r8)			; 84-87		n 92
	vsubpd	zmm7, zmm7, zmm1		;; r2-- = (r2-r6) - (r4-r8)			; 84-87		n 89

	vsubpd	zmm1, zmm11, zmm6		;; i2-- = (i2-i6) - (i4-i8)			; 85-88		n 90
	vaddpd	zmm11, zmm11, zmm6		;; i2-+ = (i2-i6) + (i4-i8)			; 85-88		n 93

	vsubpd	zmm6, zmm14, zmm0		;; r2+- = (r2+r6) - (r4+r8)			; 86-89		n 94
	vaddpd	zmm14, zmm14, zmm0		;; r2++ = (r2+r6) + (r4+r8)			; 86-89		n 97

	vsubpd	zmm0, zmm4, zmm16		;; i2+- = (i2+i6) - (i4+i8)			; 87-90		n 94
	vaddpd	zmm4, zmm4, zmm16		;; i2++ = (i2+i6) + (i4+i8)			; 87-90		n 98

	vaddpd	zmm16, zmm9, zmm5		;; r1++ = (r1+r5) + (r3+r7)			; 88-91		n 97
	vsubpd	zmm9, zmm9, zmm5		;; r1+- = (r1+r5) - (r3+r7)			; 88-91		n 99

	zfmaddpd zmm5, zmm7, zmm31, zmm2	;; r1-+ = (r1-r5) + .707*(r2--)			; 89-92		n 95
	zfnmaddpd zmm7, zmm7, zmm31, zmm2	;; r1-- = (r1-r5) - .707*(r2--)			; 89-92		n 96

	zfmaddpd zmm2, zmm1, zmm31, zmm10	;; i1-+ = (i1-i5) + .707*(i2--)			; 90-93		n 95
	zfnmaddpd zmm1, zmm1, zmm31, zmm10	;; i1-- = (i1-i5) - .707*(i2--)			; 90-93		n 96

	vaddpd	zmm10, zmm8, zmm12		;; i1++ = (i1+i5) + (i3+i7)			; 91-94		n 98
	vsubpd	zmm8, zmm8, zmm12		;; i1+- = (i1+i5) - (i3+i7)			; 91-94		n 99

	zfmaddpd zmm12, zmm13, zmm31, zmm17	;; r3-+ = (r3-r7) + .707*(r2-+)			; 92-95		n 101
	zfnmaddpd zmm13, zmm13, zmm31, zmm17	;; r3-- = (r3-r7) - .707*(r2-+)			; 92-95		n 105

	zfmaddpd zmm17, zmm11, zmm31, zmm3	;; i3-+ = (i3-i7) + .707*(i2-+)			; 93-96		n 101
	zfnmaddpd zmm11, zmm11, zmm31, zmm3	;; i3-- = (i3-i7) - .707*(i2-+)			; 93-96		n 105

	vmulpd	zmm6, zmm6, zmm20		;; r2+-s = r2+- * sine37			; 94-97		n 99
	vmulpd	zmm0, zmm0, zmm20		;; i2+-s = i2+- * sine37			; 94-97		n 99

	vmulpd	zmm5, zmm5, zmm19		;; r1-+s = r1-+ * sine28			; 95-98		n 101
	vmulpd	zmm2, zmm2, zmm19		;; i1-+s = i1-+ * sine28			; 95-98		n 101

	vmulpd	zmm7, zmm7, zmm18		;; r1--s = r1-- * sine46			; 96-99		n 105
	vmulpd	zmm1, zmm1, zmm18		;; i1--s = i1-- * sine46			; 96-99		n 105

	vsubpd	zmm3, zmm16, zmm14		;; R5 = (r1++) - (r2++)				; 97-100	n 107
	vaddpd	zmm16, zmm16, zmm14		;; R1 = (r1++) + (r2++)				; 97-100

	vsubpd	zmm14, zmm10, zmm4		;; I5 = (i1++) - (i2++)				; 98-101	n 107
	vaddpd	zmm10, zmm10, zmm4		;; I1 = (i1++) + (i2++)				; 98-101

	zfmsubpd zmm4, zmm9, zmm20, zmm0	;; R3s = (r1+-)*sine37 - (i2+-s)		; 99-102	n 103
	zfmaddpd zmm15, zmm8, zmm20, zmm6	;; I3s = (i1+-)*sine37 + (r2+-s)		; 99-102	n 103

	zfmaddpd zmm9, zmm9, zmm20, zmm0	;; R7s = (r1+-)*sine37 + (i2+-s)		; 100-103	n 107
	zfmsubpd zmm8, zmm8, zmm20, zmm6	;; I7s = (i1+-)*sine37 - (r2+-s)		; 100-103	n 107

	zfnmaddpd zmm6, zmm17, zmm19, zmm5	;; R2s = (r1-+s) - sine28*(i3-+)		; 101-104	n 108
	zfmaddpd zmm0, zmm12, zmm19, zmm2	;; I2s = (i1-+s) + sine28*(r3-+)		; 101-104	n 108
	zstore	[srcreg], zmm16			;; Store R1					; 101

	zfmaddpd zmm17, zmm17, zmm19, zmm5	;; R8s = (r1-+s) + sine28*(i3-+)		; 102-105	n 109
	zfnmaddpd zmm12, zmm12, zmm19, zmm2	;; I8s = (i1-+s) - sine28*(r3-+)		; 102-105	n 109
	zstore	[srcreg+64], zmm10		;; Store I1					; 102

	zfmsubpd zmm2, zmm4, zmm28, zmm15	;; R3s * cosine/sine - I3s (final R3)		; 103-106
	zfmaddpd zmm15, zmm15, zmm28, zmm4	;; I3s * cosine/sine + R3s (final I3)		; 103-106

	zfmaddpd zmm4, zmm9, zmm28, zmm8	;; R7s * cosine/sine + I7s (final R7)		; 104-107
	zfmsubpd zmm8, zmm8, zmm28, zmm9	;; I7s * cosine/sine - R7s (final I7)		; 104-107

	zfmaddpd zmm9, zmm11, zmm18, zmm7	;; R4s = (r1--s) + sine46*(i3--)		; 105-108	n 110
	zfnmaddpd zmm5, zmm13, zmm18, zmm1	;; I4s = (i1--s) - sine46*(r3--)		; 105-108	n 110

	zfnmaddpd zmm11, zmm11, zmm18, zmm7	;; R6s = (r1--s) - sine46*(i3--)		; 106-109	n 111
	zfmaddpd zmm13, zmm13, zmm18, zmm1	;; I6s = (i1--s) + sine46*(r3--)		; 106-109	n 111

	zfmsubpd zmm1, zmm3, zmm25, zmm14	;; A5 = R5 * cosine/sine - I5			; 107-110	n 112
	zfmaddpd zmm14, zmm14, zmm25, zmm3	;; B5 = I5 * cosine/sine + R5			; 107-110	n 112
	zstore	[srcreg+d2], zmm2		;; Store R3					; 107

	zfmsubpd zmm3, zmm6, zmm24, zmm0	;; R2s * cosine/sine - I2s (final R2)		; 108-111
	zfmaddpd zmm0, zmm0, zmm24, zmm6	;; I2s * cosine/sine + R2s (final I2)		; 108-111
	zstore	[srcreg+d2+64], zmm15		;; Store I3					; 107+1

	zfmaddpd zmm6, zmm17, zmm24, zmm12	;; R8s * cosine/sine + I8s (final R8)		; 109-112
	zfmsubpd zmm12, zmm12, zmm24, zmm17	;; I8s * cosine/sine - R8s (final I8)		; 109-112
	zstore	[srcreg+d4+d2], zmm4		;; Store R7					; 108+1

	zfmsubpd zmm17, zmm9, zmm27, zmm5	;; R4s * cosine/sine - I4s (final R4)		; 110-113
	zfmaddpd zmm5, zmm5, zmm27, zmm9	;; I4s * cosine/sine + R4s (final I4)		; 110-113
	zstore	[srcreg+d4+d2+64], zmm8		;; Store I7					; 108+2

	zfmaddpd zmm9, zmm11, zmm27, zmm13	;; R6s * cosine/sine + I6s (final R6)		; 111-114
	zfmsubpd zmm13, zmm13, zmm27, zmm11	;; I6s * cosine/sine - R6s (final I6)		; 111-114

	vmulpd	zmm1, zmm1, zmm26		;; A5 = A5 * sine (final R5)			; 112-115
	vmulpd	zmm14, zmm14, zmm26		;; B5 = B5 * sine (final I5)			; 112-115
	zstore	[srcreg+d1], zmm3		;; Store R2					; 112

	zstore	[srcreg+d1+64], zmm0		;; Store I2					; 112+1
	zstore	[srcreg+d4+d2+d1], zmm6		;; Store R8					; 113+1
	zstore	[srcreg+d4+d2+d1+64], zmm12	;; Store I8					; 113+2
	zstore	[srcreg+d2+d1], zmm17		;; Store R4					; 114+2
	zstore	[srcreg+d2+d1+64], zmm5		;; Store I4					; 114+3
	zstore	[srcreg+d4+d1], zmm9		;; Store R6					; 115+3
	zstore	[srcreg+d4+d1+64], zmm13	;; Store I6					; 115+4
	zstore	[srcreg+d4], zmm1		;; Store R5					; 116+4
	zstore	[srcreg+d4+64], zmm14		;; Store I5					; 116+5
	bump	srcreg, srcinc
	ENDM


;; Merge macros zr8b_eight_complex_djbunfft and zr8s_eight_complex_last_unfft into one more efficient macro

zr8s_sixtyfour_complex_last_unfft_preload MACRO
	vbroadcastsd zmm31, ZMM_SQRTHALF
	vpmovzxbq zmm30, ZMM_PERMUTE5			;; zmm30 = 0+4 0+5 8+0 8+1 8+4 8+5 0+0 0+1
	vpmovzxbq zmm29, ZMM_PERMUTE6			;; zmm29 = 0+6 0+7 8+2 8+3 8+6 8+7 0+2 0+3
	mov	r14d, 10101010b				;; For vblendmpd during swizzle
	kmovw	k6, r14d
	ENDM

zr8s_sixtyfour_complex_last_unfft MACRO srcreg,srcinc,d1,d2,d4,screg,scinc,sc2reg,sc2inc,maxrpt,L1pt,L1pd

	vbroadcastsd zmm28, [screg+0*16+8]		;; cosine/sine for R2/I2 and R8/I8 (w^1)
	vmovapd zmm4, [srcreg+d1]			;; Load R2
	vmovapd zmm12, [srcreg+d1+64]			;; Load I2
	zfmaddpd zmm16, zmm4, zmm28, zmm12		;; A2 = R2 * cosine/sine + I2				; 1-4		n 5
	zfmsubpd zmm12, zmm12, zmm28, zmm4		;; B2 = I2 * cosine/sine - R2				; 1-4		n 5

	vmovapd zmm7, [srcreg+d4+d2+d1]			;; Load R8
	vmovapd zmm15, [srcreg+d4+d2+d1+64]		;; Load I8
	zfmsubpd zmm4, zmm7, zmm28, zmm15		;; R8/sine = R8 * cosine/sine - I8			; 2-5		n 11
	zfmaddpd zmm15, zmm15, zmm28, zmm7		;; I8/sine = I8 * cosine/sine + R8			; 2-5		n 12

	vbroadcastsd zmm28, [screg+2*16+8]		;; cosine/sine for R4/I4 and R6/I6 (w^3)
	vmovapd zmm6, [srcreg+d2+d1]			;; Load R4
	vmovapd zmm14, [srcreg+d2+d1+64]		;; Load I4
	zfmaddpd zmm7, zmm6, zmm28, zmm14		;; R4/sine = R4 * cosine/sine + I4			; 3-6		n 9
	zfmsubpd zmm14, zmm14, zmm28, zmm6		;; I4/sine = I4 * cosine/sine - R4			; 3-6		n 10

	vmovapd zmm5, [srcreg+d4+d1]			;; Load R6
	vmovapd zmm13, [srcreg+d4+d1+64]		;; Load I6
	zfmsubpd zmm6, zmm5, zmm28, zmm13		;; R6/sine = R6 * cosine/sine - I6			; 4-7		n 9
	zfmaddpd zmm13, zmm13, zmm28, zmm5		;; I6/sine = I6 * cosine/sine + R6			; 4-7		n 10

	vbroadcastsd zmm27, [screg+0*16]		;; sine for R2/I2 and R8/I8 (w^1)
	vmulpd	zmm16, zmm16, zmm27			;; R2 = A2 * sine					; 5-8		n 11
	vmulpd	zmm12, zmm12, zmm27			;; I2 = B2 * sine					; 5-8		n 12

	vbroadcastsd zmm28, [screg+3*16+8]		;; cosine/sine for R5/I5 (w^4)
	vmovapd zmm1, [srcreg+d4]			;; Load R5
	vmovapd zmm9, [srcreg+d4+64]			;; Load I5
	zfmaddpd zmm5, zmm1, zmm28, zmm9		;; R5/sine = R5 * cosine/sine + I5			; 6-9		n 13
	zfmsubpd zmm9, zmm9, zmm28, zmm1		;; I5/sine = I5 * cosine/sine - R5			; 6-9		n 16

	vbroadcastsd zmm28, [screg+1*16+8]		;; cosine/sine for R3/I3 and R7/I7 (w^2)
	vmovapd zmm2, [srcreg+d2]			;; Load R3
	vmovapd zmm10, [srcreg+d2+64]			;; Load I3
	zfmaddpd zmm1, zmm2, zmm28, zmm10		;; R3/sine = R3 * cosine/sine + I3			; 7-10		n 14
	zfmsubpd zmm10, zmm10, zmm28, zmm2		;; I3/sine = I3 * cosine/sine - R3			; 7-10		n 15

	vmovapd zmm3, [srcreg+d4+d2]			;; Load R7
	vmovapd zmm11, [srcreg+d4+d2+64]		;; Load I7
	zfmsubpd zmm2, zmm3, zmm28, zmm11		;; R7/sine = R7 * cosine/sine - I7			; 8-11		n 14
	zfmaddpd zmm11, zmm11, zmm28, zmm3		;; I7/sine = I7 * cosine/sine + R7			; 8-11		n 15

	vmovapd zmm0, [srcreg]				;; Load R1
	vaddpd	zmm3, zmm7, zmm6			;; R4/sine + R6/sine					; 9-12		n 17
	vsubpd	zmm7, zmm7, zmm6			;; R4/sine - R6/sine					; 9-12		n 20

	vbroadcastsd zmm28, [screg+3*16]		;; sine for R5/I5 (w^4)
	vaddpd	zmm6, zmm14, zmm13			;; I4/sine + I6/sine					; 10-13		n 21
	vsubpd	zmm14, zmm14, zmm13			;; I4/sine - I6/sine					; 10-13		n 18

	vmovapd zmm8, [srcreg+64]			;; Load I1
	zfmaddpd zmm13, zmm4, zmm27, zmm16		;; R2 + R8/sine * sine					; 11-14		n 17
	zfnmaddpd zmm4, zmm4, zmm27, zmm16		;; R2 - R8/sine * sine					; 11-14		n 20

	vbroadcastsd zmm26, [screg+2*16]		;; sine for R4/I4 and R6/I6 (w^3)
	zfmaddpd zmm16, zmm15, zmm27, zmm12		;; I2 + I8/sine * sine					; 12-15		n 21
	zfnmaddpd zmm15, zmm15, zmm27, zmm12		;; I2 - I8/sine * sine					; 12-15		n 18

	vbroadcastsd zmm27, [screg+1*16]		;; sine for R3/I3 and R7/I7 (w^2)
	zfmaddpd zmm12, zmm5, zmm28, zmm0		;; R1 + R5/sine * sine					; 13-16		n 19
	zfnmaddpd zmm5, zmm5, zmm28, zmm0		;; R1 - R5/sine * sine					; 13-16		n 22
	bump	screg, scinc

	vmovapd zmm25, [sc2reg+8*64+0*128+64]		;; cosine/sine for R2/I2 and R8/I8
	vaddpd	zmm0, zmm1, zmm2			;; R3/sine + R7/sine					; 14-17		n 19
	vsubpd	zmm1, zmm1, zmm2			;; R3/sine - R7/sine					; 14-17		n 27

	vmovapd zmm24, [sc2reg+8*64+1*128+64]		;; cosine/sine for R3/I3 and R7/I7
	vaddpd	zmm2, zmm10, zmm11			;; I3/sine + I7/sine					; 15-18		n 25
	vsubpd	zmm10, zmm10, zmm11			;; I3/sine - I7/sine					; 15-18		n 22

	vmovapd zmm23, [sc2reg+8*64+2*128+64]		;; cosine/sine for R4/I4 and R6/I6
	zfmaddpd zmm11, zmm9, zmm28, zmm8		;; I1 + I5/sine * sine					; 16-19		n 25
	zfnmaddpd zmm9, zmm9, zmm28, zmm8		;; I1 - I5/sine * sine					; 16-19		n 27

	vmovapd zmm28, [sc2reg+8*64+3*128+64]		;; cosine/sine for R5/I5
	zfmaddpd zmm8, zmm3, zmm26, zmm13		;; r2++ = (r2+r8) + (r4+r6)/sine * sine			; 17-20		n 23
	zfnmaddpd zmm3, zmm3, zmm26, zmm13		;; r2+- = (r2+r8) - (r4+r6)/sine * sine			; 17-20		n 23

	vmovapd zmm22, [sc2reg+8*64+0*128]		;; sine for R2/I2 and R8/I8
	zfmaddpd zmm13, zmm14, zmm26, zmm15		;; i2-+ = (i2-i8) + (i4-i6)/sine * sine			; 18-21		n 23
	zfnmaddpd zmm14, zmm14, zmm26, zmm15		;; i2-- = (i2-i8) - (i4-i6)/sine * sine			; 18-21		n 23

	vmovapd zmm21, [sc2reg+8*64+3*128]		;; sine for R5/I5
	zfmaddpd zmm15, zmm0, zmm27, zmm12		;; r1++ = (r1+r5) + (r3+r7)/sine * sine			; 19-22		n 25
	zfnmaddpd zmm0, zmm0, zmm27, zmm12		;; r1+- = (r1+r5) - (r3+r7)/sine * sine			; 19-22		n 25

	zfmaddpd zmm12, zmm7, zmm26, zmm4		;; r2-+ = (r2-r8) + (r4-r6)/sine * sine			; 20-23		n 29
	zfnmaddpd zmm7, zmm7, zmm26, zmm4		;; r2-- = (r2-r8) - (r4-r6)/sine * sine			; 20-23		n 31

	zfmaddpd zmm4, zmm6, zmm26, zmm16		;; i2++ = (i2+i8) + (i4+i6)/sine * sine			; 21-24		n 31
	zfnmaddpd zmm6, zmm6, zmm26, zmm16		;; i2+- = (i2+i8) - (i4+i6)/sine * sine			; 21-24		n 29

	vmovapd zmm26, [sc2reg+8*64+2*128]		;; sine for R4/I4 and R6/I6
	zfmaddpd zmm16, zmm10, zmm27, zmm5		;; r1-+ = (r1-r5) + (i3-i7)/sine * sine			; 22-25		n 27
	zfnmaddpd zmm10, zmm10, zmm27, zmm5		;; r1-- = (r1-r5) - (i3-i7)/sine * sine			; 22-25		n 27

	vshuff64x2 zmm5, zmm8, zmm14, 11101110b		;; i2--7654 r2++7654					; 23-25		n 31
	vaddpd	zmm17, zmm3, zmm13			;; r2+-+ = (r2+-) + (i2-+)				; 23-26		n 29

	vshuff64x2 zmm8, zmm8, zmm14, 01000100b		;; i2--3210 r2++3210					; 24-26		n 39
	vsubpd	zmm3, zmm3, zmm13			;; r2+-- = (r2+-) - (i2-+)				; 24-27		n 29

	vshuff64x2 zmm14, zmm15, zmm0, 11101110b	;; r1+-7654 r1++7654					; 25-27		n 31
	zfmaddpd zmm13, zmm2, zmm27, zmm11		;; i1++ = (i1+i5) + (i3+i7)/sine * sine			; 25-28		n 32

	vshuff64x2 zmm15, zmm15, zmm0, 01000100b	;; r1+-3210 r1++3210					; 26-28		n 39
	zfnmaddpd zmm2, zmm2, zmm27, zmm11		;; i1+- = (i1+i5) - (i3+i7)/sine * sine			; 26-29		n 32

	vshuff64x2 zmm0, zmm16, zmm10, 11101110b	;; r1--7654 r1-+7654					; 27-29		n 33
	zfmaddpd zmm11, zmm1, zmm27, zmm9		;; i1-+ = (i1-i5) + (r3-r7)/sine * sine			; 27-30		n 33

	vshuff64x2 zmm16, zmm16, zmm10, 01000100b	;; r1--3210 r1-+3210					; 28-30		n 41
	zfnmaddpd zmm1, zmm1, zmm27, zmm9		;; i1-- = (i1-i5) - (r3-r7)/sine * sine			; 28-31		n 33

	vmovapd zmm27, [sc2reg+8*64+1*128]		;; sine for R3/I3 and R7/I7
	vshuff64x2 zmm10, zmm17, zmm3, 11101110b	;; r2+--7654 r2+-+7654					; 29-31		n 33
	vaddpd	zmm9, zmm12, zmm6			;; r2-++ = (r2-+) + (i2+-)				; 29-32		n 34

	vshuff64x2 zmm17, zmm17, zmm3, 01000100b	;; r2+--3210 r2+-+3210					; 30-32		n 41
	vsubpd	zmm12, zmm12, zmm6			;; r2-+- = (r2-+) - (i2+-)				; 30-33		n 34

	vshuff64x2 zmm3, zmm4, zmm7, 11101110b		;; r2--7654 i2++7654					; 31-33		n 35
	vaddpd	zmm6, zmm14, zmm5			;; R3H = (r1+-) + (i2--) R1H = (r1++) + (r2++)		; 31-34		n 39

	vshuff64x2 zmm18, zmm13, zmm2, 11101110b	;; i1+-7654 i1++7654					; 32-34		n 35
	vsubpd	zmm14, zmm14, zmm5			;; R7H = (r1+-) - (i2--) R5H = (r1++) - (r2++)		; 32-35		n 39

	vshuff64x2 zmm5, zmm1, zmm11, 11101110b		;; i1-+7654 i1--7654					; 33-35		n 37
	zfmaddpd zmm19, zmm10, zmm31, zmm0		;; R8H = r1-- + .707*r2+-- R2H = r1-+ + .707*r2+-+	; 33-36		n 41

	vshuff64x2 zmm20, zmm12, zmm9, 11101110b	;; r2-++7654 r2-+-7654					; 34-36		n 37
	zfnmaddpd zmm10, zmm10, zmm31, zmm0		;; R4H = r1-- - .707*r2+-- R6H = r1-+ - .707*r2+-+	; 34-37		n 41

	vmovapd zmm0, [sc2reg+0*64]			;; premultiplier cosine/sine for R1/I1
	vshuff64x2 zmm4, zmm4, zmm7, 01000100b		;; r2--3210 i2++3210					; 28-30		n 43
	vaddpd	zmm7, zmm18, zmm3			;; I7H = (i1+-) + (r2--) I1H = (i1++) + (i2++)		; 35-38		n 43

	vshuff64x2 zmm13, zmm13, zmm2, 01000100b	;; i1+-3210 i1++3210					; 30-32		n 43
	vsubpd	zmm18, zmm18, zmm3			;; I3H = (i1+-) - (r2--) I5H = (i1++) - (i2++)		; 36-39		n 43

	vmovapd zmm2, [sc2reg+4*64]			;; premultiplier cosine/sine for R5/I5
	vshuff64x2 zmm1, zmm1, zmm11, 01000100b		;; i1-+3210 i1--3210					; 37-39		n 45
	zfnmaddpd zmm11, zmm20, zmm31, zmm5		;; I4H = i1-+ - .707*r2-++ I2H = i1-- - .707*r2-+-	; 37-40		n 45

	vmovapd zmm3, [sc2reg+2*64]			;; premultiplier cosine/sine for R3/I3
	vshuff64x2 zmm12, zmm12, zmm9, 01000100b	;; r2-++3210 r2-+-3210					; 38-40		n 45
	zfmaddpd zmm20, zmm20, zmm31, zmm5		;; I8H = i1-+ + .707*r2-++ I6H = i1-- + .707*r2-+-	; 38-41		n 45

	;; Code below inspired by zr8s_eight_complex_last_unfft
	;; R31H = R3H,R1H =				;; r3_7 r3_6 r3_5 r3_4 r1_7 r1_6 r1_5 r1_4
	;; R31L = R3L,R1L =				;; r3_3 r3_2 r3_1 r3_0 r1_3 r1_2 r1_1 r1_0
	;; R75H = R7H,R5H =				;; r7_7	r7_6 r7_5 r7_4 r5_7 r5_6 r5_5 r5_4
	;; R75L = R7L,R5L =				;; r7_3 r7_2 r7_1 r7_0 r5_3 r5_2 r5_1 r5_0, etc.

	L1prefetch screg, L1pt				;; The sc2reg values are preloaded well in advance, so prefetching those is not necessary.
	vshuff64x2 zmm5, zmm6, zmm14, 11011101b		;; r7_7 r7_6 r5_7 r5_6 r3_7 r3_6 r1_7 r1_6		; 39-41		n 47
	vaddpd	zmm9, zmm15, zmm8			;; R3L = (r1+-) + (i2--) R1L = (r1++) + (r2++)		; 39-42		n 53

	L1prefetchw srcreg+srcinc+d1, L1pt
	vshuff64x2 zmm6, zmm6, zmm14, 10001000b		;; r7_5 r7_4 r5_5 r5_4 r3_5 r3_4 r1_5 r1_4		; 40-42		n 48
	vsubpd	zmm15, zmm15, zmm8			;; R7L = (r1+-) - (i2--) R5L = (r1++) - (r2++)		; 40-43		n 53

	L1prefetchw srcreg+srcinc+d1+64, L1pt
	zperm2pd zmm8, zmm29, zmm19, zmm10		;; r8_6 r8_7 r6_6 r6_7 r4_6 r4_7 r2_6 r2_7		; 41-43		n 47
	zfmaddpd zmm14, zmm17, zmm31, zmm16		;; R8L = r1-- + .707*r2+-- R2L = r1-+ + .707*r2+-+	; 41-44		n 54

	L1prefetchw srcreg+srcinc+d4+d2+d1, L1pt
	zperm2pd zmm19, zmm30, zmm19, zmm10		;; r8_4 r8_5 r6_4 r6_5 r4_4 r4_5 r2_4 r2_5		; 42-44		n 48
	zfnmaddpd zmm17, zmm17, zmm31, zmm16		;; R4L = r1-- - .707*r2+-- R6L = r1-+ - .707*r2+-+	; 42-45		n 54

	L1prefetchw srcreg+srcinc+d4+d2+d1+64, L1pt
	zperm2pd zmm10, zmm29, zmm7, zmm18		;; i7_6 i7_7 i5_6 i5_7 i3_6 i3_7 i1_6 i1_7		; 43-45		n 49
	vaddpd	zmm16, zmm13, zmm4			;; I7L = (i1+-) + (r2--) I1L = (i1++) + (i2++)		; 43-46		n 55

	L1prefetchw srcreg+srcinc+d2+d1, L1pt
	zperm2pd zmm7, zmm30, zmm7, zmm18		;; i7_4 i7_5 i5_4 i5_5 i3_4 i3_5 i1_4 i1_5		; 44-46		n 50
	vsubpd	zmm13, zmm13, zmm4			;; I3L = (i1+-) - (r2--) I5L = (i1++) - (i2++)		; 44-47		n 55

	L1prefetchw srcreg+srcinc+d2+d1+64, L1pt
	vshuff64x2 zmm18, zmm11, zmm20, 11011101b	;; i8_7 i8_6 i6_7 i6_6 i4_7 i4_6 i2_7 i2_6		; 45-47		n 49
	zfnmaddpd zmm4, zmm12, zmm31, zmm1		;; I4L = i1-+ - .707*r2-++ I2L = i1-- - .707*r2-+-	; 45-48		n 56

	L1prefetchw srcreg+srcinc+d4+d1, L1pt
	vshuff64x2 zmm11, zmm11, zmm20, 10001000b	;; i8_5 i8_4 i6_5 i6_4 i4_5 i4_4 i2_5 i2_4		; 46-48		n 50
	zfmaddpd zmm12, zmm12, zmm31, zmm1		;; I8L = i1-+ + .707*r2-++ I6L = i1-- + .707*r2-+-	; 46-49		n 56

	L1prefetchw srcreg+srcinc+d4+d1+64, L1pt
	vshufpd	zmm1, zmm5, zmm8, 01010101b		;; r8_7 r7_7 r6_7 r5_7 r4_7 r3_7 r2_7 r1_7 (R8)		; 47		n 51
	vblendmpd zmm5 {k6}, zmm5, zmm8			;; r8_6 r7_6 r6_6 r5_6 r4_6 r3_6 r2_6 r1_6 (R7)		; 47		n 53

	L1prefetchw srcreg+srcinc+d4, L1pt
	vshufpd	zmm8, zmm6, zmm19, 01010101b		;; r8_5 r7_5 r6_5 r5_5 r4_5 r3_5 r2_5 r1_5 (R6)		; 48		n 55
	vblendmpd zmm6 {k6}, zmm6, zmm19		;; r8_4	r7_4 r6_4 r5_4 r4_4 r3_4 r2_4 r1_4 (R5)		; 48		n 57

	L1prefetchw srcreg+srcinc+d4+64, L1pt
	vshufpd	zmm19, zmm10, zmm18, 01010101b		;; i8_6 i7_6 i6_6 i5_6 i4_6 i3_6 i2_6 i1_6 (I7)		; 49		n 53
	vblendmpd zmm10 {k6}, zmm10, zmm18		;; i8_7 i7_7 i6_7 i5_7 i4_7 i3_7 i2_7 i1_7 (I8)		; 49		n 51

	L1prefetchw srcreg+srcinc+d2, L1pt
	vshufpd	zmm18, zmm7, zmm11, 01010101b		;; i8_4	i7_4 i6_4 i5_4 i4_4 i3_4 i2_4 i1_4 (I5)		; 50		n 57
	vblendmpd zmm7 {k6}, zmm7, zmm11		;; i8_5 i7_5 i6_5 i5_5 i4_5 i3_5 i2_5 i1_5 (I6)		; 50		n 55

	L1prefetchw srcreg+srcinc+d2+64, L1pt
	vshuff64x2 zmm20, zmm9, zmm15, 11011101b	;; r7_3 r7_2 r5_3 r5_2 r3_3 r3_2 r1_3 r1_2		; 51-53		n 59
	zfmsubpd zmm11, zmm1, zmm25, zmm10		;; A8 = R8 * cosine/sine - I8				; 51-54		n 66

	L1prefetchw srcreg+srcinc+d4+d2, L1pt
	vshuff64x2 zmm9, zmm9, zmm15, 10001000b		;; r7_1 r7_0 r5_1 r5_0 r3_1 r3_0 r1_1 r1_0		; 52-54		n 60
	zfmaddpd zmm10, zmm10, zmm25, zmm1		;; B8 = I8 * cosine/sine + R8				; 52-55		n 66

	L1prefetchw srcreg+srcinc+d4+d2+64, L1pt
	zperm2pd zmm15, zmm29, zmm14, zmm17		;; r8_2 r8_3 r6_2 r6_3 r4_2 r4_3 r2_2 r2_3		; 53-55		n 59
	zfmsubpd zmm1, zmm5, zmm24, zmm19		;; R7/sine = R7 * cosine/sine - I7			; 53-56		n 69

	L1prefetchw srcreg+srcinc, L1pt
	zperm2pd zmm14, zmm30, zmm14, zmm17		;; r8_0 r8_1 r6_0 r6_1 r4_0 r4_1 r2_0 r2_1		; 54-56		n 60
	zfmaddpd zmm19, zmm19, zmm24, zmm5		;; I7/sine = I7 * cosine/sine + R7			; 54-57		n 70

	L1prefetchw srcreg+srcinc+64, L1pt
	zperm2pd zmm17, zmm29, zmm16, zmm13		;; i7_2 i7_3 i5_2 i5_3 i3_2 i3_3 i1_2 i1_3		; 55-57		n 61
	zfmsubpd zmm5, zmm8, zmm23, zmm7		;; R6/sine = R6 * cosine/sine - I6			; 55-58		n 67

	zperm2pd zmm16, zmm30, zmm16, zmm13		;; i7_0 i7_1 i5_0 i5_1 i3_0 i3_1 i1_0 i1_1		; 56-58		n 62
	zfmaddpd zmm7, zmm7, zmm23, zmm8		;; I6/sine = I6 * cosine/sine + R6			; 56-59		n 68

	vshuff64x2 zmm13, zmm4, zmm12, 11011101b	;; i8_3 i8_2 i6_3 i6_2 i4_3 i4_2 i2_3 i2_2		; 57-59		n 61
	zfmaddpd zmm8, zmm6, zmm28, zmm18		;; R5/sine = R5 * cosine/sine + I5			; 57-60		n 73

	vshuff64x2 zmm4, zmm4, zmm12, 10001000b		;; i8_1 i8_0 i6_1 i6_0 i4_1 i4_0 i2_1 i2_0		; 58-60		n 62
	zfmsubpd zmm18, zmm18, zmm28, zmm6		;; I5/sine = I5 * cosine/sine - R5			; 58-61		n 74

	vmovapd zmm12, [sc2reg+6*64]			;; premultiplier cosine/sine for R7/I7
	vshufpd	zmm6, zmm20, zmm15, 01010101b		;; r8_3 r7_3 r6_3 r5_3 r4_3 r3_3 r2_3 r1_3 (R4)		; 59		n 63
	vblendmpd zmm20 {k6}, zmm20, zmm15		;; r8_2 r7_2 r6_2 r5_2 r4_2 r3_2 r2_2 r1_2 (R3)		; 59		n 64

	vshufpd	zmm15, zmm9, zmm14, 01010101b		;; r8_1 r7_1 r6_1 r5_1 r4_1 r3_1 r2_1 r1_1 (R2)		; 60		n 65
	vblendmpd zmm9 {k6}, zmm9, zmm14		;; r8_0	r7_0 r6_0 r5_0 r4_0 r3_0 r2_0 r1_0 (R1)		; 60		n 73

	vshufpd	zmm14, zmm17, zmm13, 01010101b		;; i8_2 i7_2 i6_2 i5_2 i4_2 i3_2 i2_2 i1_2 (I3)		; 61		n 64
	vblendmpd zmm17 {k6}, zmm17, zmm13		;; i8_3 i7_3 i6_3 i5_3 i4_3 i3_3 i2_3 i1_3 (I4)		; 61		n 63

	vshufpd	zmm13, zmm16, zmm4, 01010101b		;; i8_0	i7_0 i6_0 i5_0 i4_0 i3_0 i2_0 i1_0 (I1)		; 62		n 74
	vblendmpd zmm16 {k6}, zmm16, zmm4		;; i8_1 i7_1 i6_1 i5_1 i4_1 i3_1 i2_1 i1_1 (I2)		; 62		n 65

	zfmaddpd zmm4, zmm6, zmm23, zmm17		;; R4/sine = R4 * cosine/sine + I4			; 63-66		n 67
	zfmsubpd zmm17, zmm17, zmm23, zmm6		;; I4/sine = I4 * cosine/sine - R4			; 63-66		n 68

	vmovapd zmm23, [sc2reg+1*64]			;; premultiplier cosine/sine for R2/I2
	zfmaddpd zmm6, zmm20, zmm24, zmm14		;; R3/sine = R3 * cosine/sine + I3			; 64-67		n 69
	zfmsubpd zmm14, zmm14, zmm24, zmm20		;; I3/sine = I3 * cosine/sine - R3			; 64-67		n 70

	vmovapd zmm24, [sc2reg+5*64]			;; premultiplier cosine/sine for R6/I6
	zfmaddpd zmm20, zmm15, zmm25, zmm16		;; R2/sine = R2 * cosine/sine + I2			; 65-68		n 71
	zfmsubpd zmm16, zmm16, zmm25, zmm15		;; I2/sine = I2 * cosine/sine - R2			; 65-68		n 72

	vmovapd zmm25, [sc2reg+3*64]			;; premultiplier cosine/sine for R4/I4
	vmulpd	zmm11, zmm11, zmm22			;; R8 = A8 * sine					; 66-69		n 71
	vmulpd	zmm10, zmm10, zmm22			;; I8 = B8 * sine					; 66-69		n 72

	vaddpd	zmm15, zmm4, zmm5			;; R4/sine + R6/sine					; 67-70		n 75
	vsubpd	zmm4, zmm4, zmm5			;; R4/sine - R6/sine					; 67-70		n 76

	vaddpd	zmm5, zmm17, zmm7			;; I4/sine + I6/sine					; 68-71		n 78
	vsubpd	zmm17, zmm17, zmm7			;; I4/sine - I6/sine					; 68-71		n 77

	vaddpd	zmm7, zmm6, zmm1			;; R3/sine + R7/sine					; 69-72		n 79
	vsubpd	zmm6, zmm6, zmm1			;; R3/sine - R7/sine					; 69-72		n 82

	vaddpd	zmm1, zmm14, zmm19			;; I3/sine + I7/sine					; 70-73		n 80
	vsubpd	zmm14, zmm14, zmm19			;; I3/sine - I7/sine					; 70-73		n 81

	zfmaddpd zmm19, zmm20, zmm22, zmm11		;; (R2/sine * sine) + R8				; 71-74		n 75
	zfmsubpd zmm20, zmm20, zmm22, zmm11		;; (R2/sine * sine) - R8				; 71-74		n 76

	zfmaddpd zmm11, zmm16, zmm22, zmm10		;; (I2/sine * sine) + I8				; 72-75		n 78
	zfmsubpd zmm16, zmm16, zmm22, zmm10		;; (I2/sine * sine) - I8				; 72-75		n 77

	vmovapd zmm22, [sc2reg+7*64]			;; premultiplier cosine/sine for R8/I8
	zfmaddpd zmm10, zmm8, zmm21, zmm9		;; R1 + (R5/sine * sine)				; 73-76		n 79
	zfnmaddpd zmm8, zmm8, zmm21, zmm9		;; R1 - (R5/sine * sine)				; 73-76		n 81
	bump	sc2reg, sc2inc

	zfmaddpd zmm9, zmm18, zmm21, zmm13		;; I1 + (I5/sine * sine)				; 74-77		n 80
	zfnmaddpd zmm18, zmm18, zmm21, zmm13		;; I1 - (I5/sine * sine)				; 74-77		n 82

	zfmaddpd zmm13, zmm15, zmm26, zmm19		;; r2++ = (r2+r8) + (r4+r6)/sine * sine			; 75-78		n 85
	zfnmaddpd zmm15, zmm15, zmm26, zmm19		;; r2+- = (r2+r8) - (r4+r6)/sine * sine			; 75-78		n 83

	zfmaddpd zmm19, zmm4, zmm26, zmm20		;; r2-+ = (r2-r8) + (r4-r6)/sine * sine			; 76-79		n 84
	zfnmaddpd zmm4, zmm4, zmm26, zmm20		;; r2-- = (r2-r8) - (r4-r6)/sine * sine			; 76-79		n 88

	zfmaddpd zmm20, zmm17, zmm26, zmm16		;; i2-+ = (i2-i8) + (i4-i6)/sine * sine			; 77-80		n 83
	zfnmaddpd zmm17, zmm17, zmm26, zmm16		;; i2-- = (i2-i8) - (i4-i6)/sine * sine			; 77-80		n 87

	zfmaddpd zmm16, zmm5, zmm26, zmm11		;; i2++ = (i2+i8) + (i4+i6)/sine * sine			; 78-81		n 86
	zfnmaddpd zmm5, zmm5, zmm26, zmm11		;; i2+- = (i2+i8) - (i4+i6)/sine * sine			; 78-81		n 84

	zfmaddpd zmm11, zmm7, zmm27, zmm10		;; r1++ = (r1+r5) + (r3+r7)/sine * sine			; 79-82		n 85
	zfnmaddpd zmm7, zmm7, zmm27, zmm10		;; r1+- = (r1+r5) - (r3+r7)/sine * sine			; 79-82		n 87

	zfmaddpd zmm10, zmm1, zmm27, zmm9		;; i1++ = (i1+i5) + (i3+i7)/sine * sine			; 80-83		n 86
	zfnmaddpd zmm1, zmm1, zmm27, zmm9		;; i1+- = (i1+i5) - (i3+i7)/sine * sine			; 80-83		n 88

	zfmaddpd zmm9, zmm14, zmm27, zmm8		;; r1-+ = (r1-r5) + (i3-i7)/sine * sine			; 81-84		n 89
	zfnmaddpd zmm14, zmm14, zmm27, zmm8		;; r1-- = (r1-r5) - (i3-i7)/sine * sine			; 81-84		n 93

	zfmaddpd zmm8, zmm6, zmm27, zmm18		;; i1-+ = (i1-i5) + (r3-r7)/sine * sine			; 82-85		n 95
	zfnmaddpd zmm6, zmm6, zmm27, zmm18		;; i1-- = (i1-i5) - (r3-r7)/sine * sine			; 82-85		n 91

	vaddpd	zmm18, zmm15, zmm20			;; r2+-+ = (r2+-) + (i2-+)				; 83-86		n 89
	vsubpd	zmm15, zmm15, zmm20			;; r2+-- = (r2+-) - (i2-+)				; 83-86		n 93

	vaddpd	zmm20, zmm19, zmm5			;; r2-++ = (r2-+) + (i2+-)				; 84-87		n 95
	vsubpd	zmm19, zmm19, zmm5			;; r2-+- = (r2-+) - (i2+-)				; 84-87		n 91

	vaddpd	zmm5, zmm11, zmm13			;; R1 = (r1++) + (r2++)					; 85-88		n 90
	vsubpd	zmm11, zmm11, zmm13			;; R5 = (r1++) - (r2++)					; 85-88		n 92

	vaddpd	zmm13, zmm10, zmm16			;; I1 = (i1++) + (i2++)					; 86-89		n 90
	vsubpd	zmm10, zmm10, zmm16			;; I5 = (i1++) - (i2++)					; 86-89		n 92

	vaddpd	zmm16, zmm7, zmm17			;; R3 = (r1+-) + (i2--)					; 87-90		n 94
	vsubpd	zmm7, zmm7, zmm17			;; R7 = (r1+-) - (i2--)					; 87-90		n 96

	vsubpd	zmm17, zmm1, zmm4			;; I3 = (i1+-) - (r2--)					; 88-91		n 94
	vaddpd	zmm1, zmm1, zmm4			;; I7 = (i1+-) + (r2--)					; 88-91		n 96

	zfmaddpd zmm4, zmm18, zmm31, zmm9		;; R2 = (r1-+) + .707*(r2+-+)				; 89-92		n 97
	zfnmaddpd zmm18, zmm18, zmm31, zmm9		;; R6 = (r1-+) - .707*(r2+-+)				; 89-92		n 98

	zfmaddpd zmm9, zmm5, zmm0, zmm13		;; R1/sine = R1 * cosine + I1				; 90-93
	zfmsubpd zmm13, zmm13, zmm0, zmm5		;; I1/sine = I1 * cosine - R1				; 90-93

	zfnmaddpd zmm5, zmm19, zmm31, zmm6		;; I2 = (i1--) - .707*(r2-+-)				; 91-94		n 97
	zfmaddpd zmm19, zmm19, zmm31, zmm6		;; I6 = (i1--) + .707*(r2-+-)				; 91-94		n 98

	zfmaddpd zmm6, zmm11, zmm2, zmm10		;; R5/sine = R5 * cosine + I5				; 92-95
	zfmsubpd zmm10, zmm10, zmm2, zmm11		;; I5/sine = I5 * cosine - R5				; 92-95

	zfnmaddpd zmm11, zmm15, zmm31, zmm14		;; R4 = (r1--) - .707*(r2+--)				; 93-96		n 99
	zfmaddpd zmm15, zmm15, zmm31, zmm14		;; R8 = (r1--) + .707*(r2+--)				; 93-96		n 100

	zfmaddpd zmm14, zmm16, zmm3, zmm17		;; R3/sine = R3 * cosine + I3				; 94-97
	zfmsubpd zmm17, zmm17, zmm3, zmm16		;; I3/sine = I3 * cosine - R3				; 94-97
	zstore	[srcreg], zmm9				;; Save R1/sine						; 94

	zfnmaddpd zmm16, zmm20, zmm31, zmm8		;; I4 = (i1-+) - .707*(r2-++)				; 95-98		n 99
	zfmaddpd zmm20, zmm20, zmm31, zmm8		;; I8 = (i1-+) + .707*(r2-++)				; 95-98		n 100
	zstore	[srcreg+64], zmm13			;; Save I1/sine						; 94+1

	zfmaddpd zmm8, zmm7, zmm12, zmm1		;; R7/sine = R7 * cosine + I7				; 96-99
	zfmsubpd zmm1, zmm1, zmm12, zmm7		;; I7/sine = I7 * cosine - R7				; 96-99
	zstore	[srcreg+d4], zmm6			;; Save R5/sine						; 96

	zfmaddpd zmm7, zmm4, zmm23, zmm5		;; R2/sine = R2 * cosine + I2				; 97-100
	zfmsubpd zmm5, zmm5, zmm23, zmm4		;; I2/sine = I2 * cosine - R2				; 97-100
	zstore	[srcreg+d4+64], zmm10			;; Save I5/sine						; 96+1

	zfmaddpd zmm4, zmm18, zmm24, zmm19		;; R6/sine = R6 * cosine + I6				; 98-101
	zfmsubpd zmm19, zmm19, zmm24, zmm18		;; I6/sine = I6 * cosine - R6				; 98-101
	zstore	[srcreg+d2], zmm14			;; Save R3/sine						; 98

	zfmaddpd zmm18, zmm11, zmm25, zmm16		;; R4/sine = R4 * cosine + I4				; 99-102
	zfmsubpd zmm16, zmm16, zmm25, zmm11		;; I4/sine = I4 * cosine - R4				; 99-102
	zstore	[srcreg+d2+64], zmm17			;; Save I3/sine						; 98+1

	zfmaddpd zmm11, zmm15, zmm22, zmm20		;; R8/sine = R8 * cosine + I8				; 100-103
	zfmsubpd zmm20, zmm20, zmm22, zmm15		;; I8/sine = I8 * cosine - R8				; 100-103

	zstore	[srcreg+d4+d2], zmm8			;; Save R7/sine						; 100
	zstore	[srcreg+d4+d2+64], zmm1			;; Save I7/sine						; 100+1
	zstore	[srcreg+d1], zmm7			;; Save R2/sine						; 101+1
	zstore	[srcreg+d1+64], zmm5			;; Save I2/sine						; 101+2
	zstore	[srcreg+d4+d1], zmm4			;; Save R6/sine						; 102+2
	zstore	[srcreg+d4+d1+64], zmm19		;; Save I6/sine						; 102+3
	zstore	[srcreg+d2+d1], zmm18			;; Save R4/sine						; 103+3
	zstore	[srcreg+d2+d1+64], zmm16		;; Save I4/sine						; 103+4
	zstore	[srcreg+d4+d2+d1], zmm11		;; Save R8/sine						; 104+4
	zstore	[srcreg+d4+d2+d1+64], zmm20		;; Save I8/sine						; 104+5
	bump	srcreg, srcinc
	ENDM


;; Merge macros zr8fs_sixteen_reals_first_fft and zr8_sixteen_reals_eight_complex_djbfft into one more efficient macro

zr8fs_hundredtwentyeight_reals_first_fft_preload MACRO
	;; Constants from zr8fs_sixteen_reals_first_fft_preload
	mov	r14d, 00110011b			;; We're pretty sure r14 is safe to use
	kmovw	k5, r14d			;; For vblendmpd during swizzle
	vpmovzxbq zmm26, ZMM_PERMUTE1		;; zmm26 = 8+4 0+4 8+6 0+6 8+0 0+0 8+2 0+2 [msw at left]
	vpmovzxbq zmm27, ZMM_PERMUTE2		;; zmm27 = 8+5 0+5 8+7 0+7 8+1 0+1 8+3 0+3
	;; Constants from zr8_sixteen_reals_eight_complex_djbfft_preload
	mov	r14d, 11111110b			;; For joint complex/real operations
	kmovw	k7, r14d			;; Set k7 to 11111110b
	knotw	k6, k7				;; Set k6 to 00000001b
	vbroadcastsd zmm0, ZMM_SQRTHALF
	vbroadcastsd zmm28 {k7}{z}, ZMM_ONE			;; 1, 1, 1, 1, 1, 1, 1		0
	vsubpd	zmm28 {k6}, zmm28, zmm0				;; 1, 1, 1, 1, 1, 1, 1		-SQRTHALF
	vblendmpd zmm29 {k6}, zmm28, ZMM_P924_P383{1to8}	;; 1, 1, 1, 1, 1, 1, 1		.924/.383
	vblendmpd zmm30 {k6}, zmm28, zmm0			;; 1, 1, 1, 1, 1, 1, 1		SQRTHALF
	vblendmpd zmm31 {k6}, zmm0, ZMM_P383{1to8}		;; SQRTHALF, SQRTHALF, ...,	.383
	ENDM

zr8fs_hundredtwentyeight_reals_first_fft MACRO srcreg,srcinc,d1,d2,d4,screg,scinc,sc2reg,sc2inc,wgtreg,wgtinc,maxrpt,L1pt,L1pd
	vmovapd	zmm4, [srcreg+d4+d2+d1][rbx]	;; R8
	zfmaddpd zmm4, zmm4, [wgtreg+7*128], zmm4 ;; apply weight-1 to R8					; 1-4		n 6
	vmovapd	zmm8, [srcreg+d4+d1][rbx]	;; R6
	zfmaddpd zmm8, zmm8, [wgtreg+5*128], zmm8 ;; apply weight-1 to R6					; 1-4		n 7

	vmovapd	zmm14, [srcreg][rbx]		;; R1
	zfmaddpd zmm14, zmm14, [wgtreg], zmm14	;; apply weight-1 to R1						; 2-5		n 8
	vmovapd	zmm16, [srcreg+d4][rbx]		;; R5
	zfmaddpd zmm16, zmm16, [wgtreg+4*128], zmm16 ;; apply weight-1 to R5					; 2-5		n 8

	vmovapd	zmm12, [srcreg+d4+d2][rbx]	;; R7
	zfmaddpd zmm12, zmm12, [wgtreg+6*128], zmm12 ;; apply weight-1 to R7					; 3-6		n 10
						;; to ease counting clocks, assume there is a 1/2 clock stall	; 3-6		but really there is no stall

	vmovapd	zmm2, [srcreg+d1][rbx]		;; R2
	vmovapd	zmm0, [srcreg+d1+64][rbx]	;; R10
	vmovapd	zmm25, [wgtreg+128+64]		;; weights10 / weights2
	zfmaddpd zmm1, zmm0, zmm25, zmm2	;; r2+r10*weights10_over_2					; 4-7		n 11
	zfnmaddpd zmm2, zmm0, zmm25, zmm2	;; r2-r10*weights10_over_2					; 4-7		n 20

	vmovapd	zmm6, [srcreg+d2+d1][rbx]	;; R4
	vmovapd	zmm0, [srcreg+d2+d1+64][rbx]	;; R12
	vmovapd	zmm25, [wgtreg+3*128+64]	;; weights12 / weights4
	zfmaddpd zmm5, zmm0, zmm25, zmm6	;; r4+r12*weights12_over_4					; 5-8		n 12
	zfnmaddpd zmm6, zmm0, zmm25, zmm6	;; r4-r12*weights12_over_4					; 5-8		n 21

	vmovapd	zmm0, [srcreg+d4+d2+d1+64][rbx]	;; R16
	vmovapd	zmm25, [wgtreg+7*128+64]	;; weights16
	zfmaddpd zmm3, zmm0, zmm25, zmm4	;; r8+r16*weights16						; 6-9		n 11
	zfnmaddpd zmm4, zmm0, zmm25, zmm4	;; r8-r16*weights16						; 6-9		n 20

	vmovapd	zmm0, [srcreg+d4+d1+64][rbx]	;; R14
	vmovapd	zmm25, [wgtreg+5*128+64]	;; weights14
	zfmaddpd zmm7, zmm0, zmm25, zmm8	;; r6+r14*weights14						; 7-10		n 12
	zfnmaddpd zmm8, zmm0, zmm25, zmm8	;; r6-r14*weights14						; 7-10		n 21

	vmovapd	zmm17, [srcreg+64][rbx]		;; R9
	vmovapd	zmm25, [wgtreg+64]		;; weights9
	zfmaddpd zmm13, zmm17, zmm25, zmm14	;; r1+r9*weights9						; 8-11		n 13
	vmovapd	zmm18, [srcreg+d4+64][rbx]	;; R13
	vmovapd	zmm24, [wgtreg+4*128+64]	;; weights13
	zfmaddpd zmm15, zmm18, zmm24, zmm16	;; r5+r13*weights13						; 8-11		n 13

	vmovapd	zmm10, [srcreg+d2][rbx]		;; R3
	vmovapd	zmm0, [srcreg+d2+64][rbx]	;; R11
	vmovapd	zmm23, [wgtreg+2*128+64]	;; weights11 / weights3
	zfmaddpd zmm9, zmm0, zmm23, zmm10	;; r3+r11*weights11_over_3					; 9-12		n 15
	zfnmaddpd zmm10, zmm0, zmm23, zmm10	;; r3-r11*weights11_over_3					; 9-12		n 16

	vmovapd	zmm0, [srcreg+d4+d2+64][rbx]	;; R15
	vmovapd	zmm23, [wgtreg+6*128+64]	;; weights15
	zfmaddpd zmm11, zmm0, zmm23, zmm12	;; r7+r15*weights15						; 10-13		n 15
	zfnmaddpd zmm12, zmm0, zmm23, zmm12	;; r7-r15*weights15						; 10-13		n 16

	vmovapd	zmm23, [wgtreg+128]		;; weights for R2
	zfmaddpd zmm0, zmm1, zmm23, zmm3	;; r2++ = (r2+r10)*weights2+(r8+r16)				; 11-14		n 17
	zfmsubpd zmm1, zmm1, zmm23, zmm3	;; r2+- = (r2+r10)*weights2-(r8+r16)				; 11-14		n 18

	vmovapd	zmm22, [wgtreg+3*128]		;; weights for R4
	zfmaddpd zmm3, zmm5, zmm22, zmm7	;; r4++ = (r4+r12)*weights4+(r6+r14)				; 12-15		n 17
	zfmsubpd zmm5, zmm5, zmm22, zmm7	;; r4+- = (r4+r12)*weights4-(r6+r14)				; 12-15		n 18

	vmovapd	zmm21, [wgtreg+2*128]		;; weights for R3
	vaddpd	zmm7, zmm13, zmm15		;; r1++ = (r1+r9)+(r5+r13)					; 13-16		n 18
	zfmaddpd zmm19, zmm9, zmm21, zmm11	;; r3++ = (r3+r11)*weights3+(r7+r15)				; 13-16		n 18
	bump	wgtreg, wgtinc

	vbroadcastsd zmm20, ZMM_SQRTHALF
	zfnmaddpd zmm14, zmm17, zmm25, zmm14	;; r1-r9*weights9						; 14-17		n 25
	zfnmaddpd zmm16, zmm18, zmm24, zmm16	;; r5-r13*weights13						; 14-17		n 27

	vbroadcastsd zmm25, ZMM_P924_P383
	vsubpd	zmm13, zmm13, zmm15		;; r1+- = (r1+r9)-(r5+r13)					; 15-18		n 21
	zfmsubpd zmm9, zmm9, zmm21, zmm11	;; r3+- = (r3+r11)*weights3-(r7+r15)				; 15-18		n 22

	vbroadcastsd zmm24, ZMM_P383
	zfmaddpd zmm11, zmm10, zmm21, zmm12	;; r3-+ = (r3-r11)*weights3+(r7-r15)				; 16-19		n 25
	zfmsubpd zmm10, zmm10, zmm21, zmm12	;; r3-- = (r3-r11)*weights3-(r7-r15)				; 16-19		n 23

	vmovapd	zmm21, [screg+1*128+64]		;; cosine/sine for R3/I3 (w^2)							n 33
	vsubpd	zmm12, zmm0, zmm3		;; r2++- = (r2++) - (r4++)					; 17-20		n 21
	vaddpd	zmm0, zmm0, zmm3		;; r2+++ = (r2++) + (r4++)					; 17-20		n 27

	L1prefetchw srcreg+srcinc+rbx+d4+d2+d1, L1pt
	vmovapd	zmm18, [screg+0*128+64]		;; cosine/sine for R2/I2 (w^1)							n 34
	vaddpd	zmm3, zmm1, zmm5		;; r2+-+ = (r2+-) + (r4+-)					; 18-21		n 22
	vaddpd	zmm15, zmm7, zmm19		;; r1+++ = (r1++) + (r3++)					; 18-21		n 27

	zfmaddpd zmm17, zmm2, zmm23, zmm4	;; r2-+ = (r2-r10)*weights2+(r8-r16)				; 19-22		n 26
	zfmsubpd zmm2, zmm2, zmm23, zmm4	;; r2-- = (r2-r10)*weights2-(r8-r16)				; 19-22		n 24

	vmovapd	zmm23, [screg+2*128+64]		;; cosine/sine for R4/I4 (w^3)							n 35
	zfmsubpd zmm4, zmm6, zmm22, zmm8	;; r4-- = (r4-r12)*weights4-(r6-r14)				; 20-23		n 24
	zfmaddpd zmm6, zmm6, zmm22, zmm8	;; r4-+ = (r4-r12)*weights4+(r6-r14)				; 20-23		n 26

	vmovapd	zmm22, [screg+3*128+64]		;; cosine/sine for R5/I5 (w^4)							n 36
	zfmaddpd zmm8, zmm12, zmm20, zmm13	;; R3 = (r1+-) + .707(r2++-)					; 21-24		n 33
	zfnmaddpd zmm12, zmm12, zmm20, zmm13	;; R7 = (r1+-) - .707(r2++-)					; 21-24		n 40

	zfmaddpd zmm13, zmm3, zmm20, zmm9	;; I3 = .707*(r2+-+) + (r3+-)					; 22-25		n 33
	zfmsubpd zmm3, zmm3, zmm20, zmm9	;; I7 = .707*(r2+-+) - (r3+-)					; 22-25		n 40

	L1prefetch wgtreg+7*128, L1pt
	zfmaddpd zmm9, zmm10, zmm20, zmm14	;; r2o = (r1-r9) + .707*(r3--)					; 23-26		n 28
	zfnmaddpd zmm10, zmm10, zmm20, zmm14	;; r4o = (r1-r9) - .707*(r3--)					; 23-26		n 29

	zfmaddpd zmm14, zmm2, zmm25, zmm4	;; r2e/.383 = .924/.383(r2--) + (r4--)				; 24-27		n 28
	zfnmaddpd zmm4, zmm4, zmm25, zmm2	;; r4e/.383 = (r2--) - .924/.383(r4--)				; 24-27		n 29

	L1prefetchw srcreg+srcinc+rbx+d4+d1, L1pt
	zfmaddpd zmm2, zmm11, zmm20, zmm16	;; i2o = .707(r3-+) + (r5-r13)					; 25-28		n 30
	zfmsubpd zmm11, zmm11, zmm20, zmm16	;; i4o = .707(r3-+) - (r5-r13)					; 25-28		n 31

	vmovapd	zmm20, [screg+4*128+64]		;; cosine/sine for R6/I6 (w^5)							n 37
	zfmaddpd zmm16, zmm6, zmm25, zmm17	;; i2e/.383 = (r2-+) + .924/.383(r4-+)				; 26-29		n 30
	zfmsubpd zmm17, zmm17, zmm25, zmm6	;; i4e/.383 = .924/.383(r2-+) - (r4-+)				; 26-29		n 31

	vmovapd	zmm25, [screg+0*128]		;; sine for R2/I2								n 38
	vaddpd	zmm6, zmm15, zmm0		;; R1 = (r1+++) + (r2+++)					; 27-30		n 42
	vsubpd	zmm15, zmm15, zmm0		;; R9 = (r1+++) - (r2+++)					; 27-30		n 46

	zfmaddpd zmm0, zmm14, zmm24, zmm9	;; R2 = r2o + .383*r2e						; 28-31		n 34	r 28
	zfnmaddpd zmm14, zmm14, zmm24, zmm9	;; R8 = r2o - .383*r2e						; 28-31		n 41

	L1prefetch wgtreg+5*128, L1pt
	zfmaddpd zmm9, zmm4, zmm24, zmm10	;; R4 = r4o + .383*r4e						; 29-32		n 35	r 28
	zfnmaddpd zmm4, zmm4, zmm24, zmm10	;; R6 = r4o - .383*r4e						; 29-32		n 37

	zfmaddpd zmm10, zmm16, zmm24, zmm2	;; I2 = .383*i2e + i2o						; 30-33		n 34	r 30
	zfmsubpd zmm16, zmm16, zmm24, zmm2	;; I8 = .383*i2e - i2o						; 30-33		n 41

	L1prefetchw srcreg+srcinc+rbx, L1pt
	zfmaddpd zmm2, zmm17, zmm24, zmm11	;; I4 = .383*i4e + i4o						; 31-34		n 35	r 30
	zfmsubpd zmm17, zmm17, zmm24, zmm11	;; I6 = .383*i4e - i4o						; 31-34		n 37

	vmovapd	zmm24, [screg+1*128]		;; sine for R3/I3								n 38
	vsubpd	zmm1, zmm1, zmm5		;; r2+-- = (r2+-) - (r4+-)	(I5)				; 32-35		n 36
	vsubpd	zmm7, zmm7, zmm19		;; r1++- = (r1++) - (r3++)	(R5)				; 32-35		n 36

	vmovapd	zmm11, [screg+2*128]		;; sine for R4/I4								n 39
	zfmsubpd zmm5, zmm8, zmm21, zmm13	;; A3 = R3 * cosine/sine - I3					; 33-36		n 38
	zfmaddpd zmm13, zmm13, zmm21, zmm8	;; B3 = I3 * cosine/sine + R3					; 33-36		n 42

	vmovapd	zmm19, [screg+5*128+64]		;; cosine/sine for R7/I7 (w^6)							n 40
	zfmsubpd zmm8, zmm0, zmm18, zmm10	;; A2 = R2 * cosine/sine - I2					; 34-37		n 38		r 34 c.b. 1 earlier
	zfmaddpd zmm10, zmm10, zmm18, zmm0	;; B2 = I2 * cosine/sine + R2					; 34-37		n 39

	vmovapd	zmm21, [screg+6*128+64]		;; cosine/sine for R8/I8 (w^7)							n 41
	zfmsubpd zmm0, zmm9, zmm23, zmm2	;; A4 = R4 * cosine/sine - I4					; 35-38		n 39		r 35
	zfmaddpd zmm2, zmm2, zmm23, zmm9	;; B4 = I4 * cosine/sine + R4					; 35-38		n 43

	vmovapd	zmm18, [screg+3*128]		;; sine for R5/I5								n 44
	zfmsubpd zmm9, zmm7, zmm22, zmm1	;; A5 = R5 * cosine/sine - I5					; 36-39		n 44
	zfmaddpd zmm1, zmm1, zmm22, zmm7	;; B5 = I5 * cosine/sine + R5					; 36-39		n 48

	vmovapd zmm23, [screg+4*128]		;; sine for R6/I6								n 45
	zfmsubpd zmm7, zmm4, zmm20, zmm17	;; A6 = R6 * cosine/sine - I6					; 37-40		n 45
	zfmaddpd zmm17, zmm17, zmm20, zmm4	;; B6 = I6 * cosine/sine + R6					; 37-40		n 49

	vmovapd	zmm22, [screg+5*128]		;; sine for R7/I7								n 46
	vmulpd	zmm8, zmm8, zmm25		;; A2 = A2 * sine (final R2)					; 38-41		n 47		r 38
	vmulpd	zmm5, zmm5, zmm24		;; A3 = A3 * sine (final R3)					; 38-41		n 49

	vmovapd	zmm20, [screg+6*128]		;; sine for R8/I8								n 47
	vmulpd	zmm0, zmm0, zmm11		;; A4 = A4 * sine (final R4)					; 39-42		n 49
	vmulpd	zmm10, zmm10, zmm25		;; B2 = B2 * sine (final I2)					; 39-42		n 51
	bump	screg, scinc

	L1prefetch wgtreg+0*128, L1pt
	zfmsubpd zmm4, zmm12, zmm19, zmm3	;; A7 = R7 * cosine/sine - I7					; 40-43		n 46
	zfmaddpd zmm3, zmm3, zmm19, zmm12	;; B7 = I7 * cosine/sine + R7					; 40-43		n 50

	L1prefetchw srcreg+srcinc+rbx+d4, L1pt
	zfmsubpd zmm12, zmm14, zmm21, zmm16	;; A8 = R8 * cosine/sine - I8					; 41-44		n 47
	zfmaddpd zmm16, zmm16, zmm21, zmm14	;; B8 = I8 * cosine/sine + R8					; 41-44		n 51

	L1prefetch wgtreg+4*128, L1pt
	vshufpd	zmm14, zmm6, zmm8, 00000000b	;; r2_6 r1_6 r2_4 r1_4 r2_2 r1_2 r2_0 r1_0			; 42		n 53
	vmulpd	zmm13, zmm13, zmm24		;; B3 = B3 * sine (final I3)					; 42-45		n 53

	L1prefetchw srcreg+srcinc+rbx+d4+d2, L1pt
	vshufpd	zmm6, zmm6, zmm8, 11111111b	;; r2_7 r1_7 r2_5 r1_5 r2_3 r1_3 r2_1 r1_1			; 43		n 55
	vmulpd	zmm2, zmm2, zmm11		;; B4 = B4 * sine (final I4)					; 43-46		n 53

	L1prefetch wgtreg+6*128, L1pt
	zperm2pd zmm8, zmm26, zmm5, zmm0	;; r4_4 r3_4 r4_6 r3_6 r4_0 r3_0 r4_2 r3_2			; 44-46		n 53
	vmulpd	zmm9, zmm9, zmm18		;; A5 = A5 * sine (final R5)					; 44-47		n 55

	L1prefetchw srcreg+srcinc+rbx+d1, L1pt
	zperm2pd zmm5, zmm27, zmm5, zmm0	;; r4_5 r3_5 r4_7 r3_7 r4_1 r3_1 r4_3 r3_3			; 45-47		n 55
	vmulpd	zmm7, zmm7, zmm23		;; A6 = A6 * sine (final R6)					; 45-48		n 55

	vshufpd	zmm0, zmm15, zmm10, 00000000b	;; i2_6 i1_6 i2_4 i1_4 i2_2 i1_2 i2_0 i1_0			; 46		n 61
	vmulpd	zmm4, zmm4, zmm22		;; A7 = A7 * sine (final R7)					; 46-49		n 57

	L1prefetchw srcreg+srcinc+rbx+d1+64, L1pt
	vshufpd	zmm15, zmm15, zmm10, 11111111b	;; i2_7 i1_7 i2_5 i1_5 i2_3 i1_3 i2_1 i1_1			; 47		n 65
	vmulpd	zmm12, zmm12, zmm20		;; A8 = A8 * sine (final R8)					; 47-50		n 57

	L1prefetch wgtreg+1*128+64, L1pt
	zperm2pd zmm10, zmm26, zmm13, zmm2	;; i4_4 i3_4 i4_6 i3_6 i4_0 i3_0 i4_2 i3_2			; 48-50		n 61
	vmulpd	zmm1, zmm1, zmm18		;; B5 = B5 * sine (final I5)					; 48-51		n 59

	zperm2pd zmm13, zmm27, zmm13, zmm2	;; i4_5 i3_5 i4_7 i3_7 i4_1 i3_1 i4_3 i3_3			; 49-51		n 65
	vmulpd	zmm17, zmm17, zmm23		;; B6 = B6 * sine (final I6)					; 49-52		n 59

	vshufpd	zmm2, zmm9, zmm7, 00000000b	;; r6_6 r5_6 r6_4 r5_4 r6_2 r5_2 r6_0 r5_0			; 50		n 57
	vmulpd	zmm3, zmm3, zmm22		;; B7 = B7 * sine (final I7)					; 50-53		n 61

	L1prefetchw srcreg+srcinc+rbx+d2+d1, L1pt
	vshufpd	zmm9, zmm9, zmm7, 11111111b	;; r6_7 r5_7 r6_5 r5_5 r6_3 r5_3 r6_1 r5_1			; 51		n 59
	vmulpd	zmm16, zmm16, zmm20		;; B8 = B8 * sine (final I8)					; 51-54		n 61

	L1prefetchw srcreg+srcinc+rbx+d2+d1+64, L1pt
	zperm2pd zmm7, zmm26, zmm4, zmm12	;; r8_4 r7_4 r8_6 r7_6 r8_0 r7_0 r8_2 r7_2			; 52-54		n 57

	L1prefetch wgtreg+3*128+64, L1pt
	zperm2pd zmm4, zmm27, zmm4, zmm12	;; r8_5 r7_5 r8_7 r7_7 r8_1 r7_1 r8_3 r7_3			; 53-55		n 59
	vblendmpd zmm12{k5}, zmm8, zmm14	;; r4_4 r3_4 r2_4 r1_4 r4_0 r3_0 r2_0 r1_0			; 53		n 58

	L1prefetchw srcreg+srcinc+rbx+d4+d2+d1+64, L1pt
	vshufpd	zmm18, zmm1, zmm17, 00000000b	;; i6_6 i5_6 i6_4 i5_4 i6_2 i5_2 i6_0 i5_0			; 54		n 63
	vblendmpd zmm14{k5}, zmm14, zmm8	;; r2_6 r1_6 r4_6 r3_6 r2_2 r1_2 r4_2 r3_2			; 54		n 60

	L1prefetch wgtreg+7*128+64, L1pt
	vshufpd	zmm1, zmm1, zmm17, 11111111b	;; i6_7 i5_7 i6_5 i5_5 i6_3 i5_3 i6_1 i5_1			; 55		n 67
	vblendmpd zmm17{k5}, zmm5, zmm6		;; r4_5 r3_5 r2_5 r1_5 r4_1 r3_1 r2_1 r1_1			; 55		n 62

	L1prefetchw srcreg+srcinc+rbx+d4+d1+64, L1pt
	zperm2pd zmm8, zmm26, zmm3, zmm16	;; i8_4 i7_4 i8_6 i7_6 i8_0 i7_0 i8_2 i7_2			; 56-58		n 63
	vblendmpd zmm6{k5}, zmm6, zmm5		;; r2_7 r1_7 r4_7 r3_7 r2_3 r1_3 r4_3 r3_3			; 56		n 64

	L1prefetch wgtreg+5*128+64, L1pt
	zperm2pd zmm3, zmm27, zmm3, zmm16	;; i8_5 i7_5 i8_7 i7_7 i8_1 i7_1 i8_3 i7_3			; 57-59		n 67
	vblendmpd zmm16{k5}, zmm7, zmm2		;; r8_4 r7_4 r6_4 r5_4 r8_0 r7_0 r6_0 r5_0			; 57		n 58

	L1prefetchw srcreg+srcinc+rbx+64, L1pt
	vshuff64x2 zmm5, zmm12, zmm16, 01000100b ;; r8_0 r7_0 r6_0 r5_0 r4_0 r3_0 r2_0 r1_0 (R1)		; 58-60
	vblendmpd zmm2{k5}, zmm2, zmm7		;; r6_6 r5_6 r8_6 r7_6 r6_2 r5_2 r8_2 r7_2			; 58		n 60

	L1prefetch wgtreg+0*128+64, L1pt
	vshuff64x2 zmm12, zmm12, zmm16, 11101110b ;; r8_4 r7_4 r6_4 r5_4 r4_4 r3_4 r2_4 r1_4 (R5)		; 59-61
	vblendmpd zmm16{k5}, zmm4, zmm9		;; r8_5 r7_5 r6_5 r6_5 r8_1 r7_1 r6_1 r5_1			; 59		n 62

	L1prefetchw srcreg+srcinc+rbx+d4+64, L1pt
	vshuff64x2 zmm7, zmm14, zmm2, 00010001b ;; r8_2 r7_2 r6_2 r5_2 r4_2 r3_2 r2_2 r1_2 (R3)			; 60-62
	vblendmpd zmm9{k5}, zmm9, zmm4		;; r6_7 r5_7 r8_7 r7_7 r6_3 r5_3 r8_3 r7_3			; 60		n 64

	L1prefetch wgtreg+4*128+64, L1pt
	vshuff64x2 zmm14, zmm14, zmm2, 10111011b ;; r8_6 r7_6 r6_6 r5_6 r4_6 r3_6 r2_6 r1_6 (R7)		; 61-63
	vblendmpd zmm2{k5}, zmm10, zmm0		;; i4_4 i3_4 i2_4 i1_4 i4_0 i3_0 i2_0 i1_0			; 61		n 66

	L1prefetchw srcreg+srcinc+rbx+d2, L1pt
	vshuff64x2 zmm4, zmm17, zmm16, 01000100b ;; r8_1 r7_1 r6_1 r5_1 r4_1 r3_1 r2_1 r1_1 (R2)		; 62-64
	vblendmpd zmm0{k5}, zmm0, zmm10		;; i2_6 i1_6 i4_6 i3_6 i2_2 i1_2 i4_2 i3_2			; 62		n 68

	L1prefetchw srcreg+srcinc+rbx+d2+64, L1pt
	vshuff64x2 zmm17, zmm17, zmm16, 11101110b ;; r8_5 r7_5 r6_5 r5_5 r4_5 r3_5 r2_5 r1_5 (R6)		; 63-65
	vblendmpd zmm16{k5}, zmm8, zmm18	;; i8_4 i7_4 i6_4 i5_4 i8_0 i7_0 i6_0 i5_0			; 63		n 66

	L1prefetch wgtreg+2*128+64, L1pt
	vshuff64x2 zmm10, zmm6, zmm9, 00010001b ;; r8_3 r7_3 r6_3 r5_3 r4_3 r3_3 r2_3 r1_3 (R4)			; 64-66
	vblendmpd zmm18{k5}, zmm18, zmm8	;; i6_6 i5_6 i8_6 i7_6 i6_2 i5_2 i8_2 i7_2			; 64		n 68

	L1prefetchw srcreg+srcinc+rbx+d4+d2+64, L1pt
	vshuff64x2 zmm6, zmm6, zmm9, 10111011b	;; r8_7 r7_7 r6_7 r5_7 r4_7 r3_7 r2_7 r1_7 (R8)			; 65-67
	vblendmpd zmm9{k5}, zmm13, zmm15	;; i4_5 i3_5 i2_5 i1_5 i4_1 i3_1 i2_1 i1_1			; 65		n 70

	L1prefetch wgtreg+6*128+64, L1pt
	vshuff64x2 zmm8, zmm2, zmm16, 01000100b ;; i8_0 i7_0 i6_0 i5_0 i4_0 i3_0 i2_0 i1_0 (I1)			; 66-68
	vblendmpd zmm15{k5}, zmm15, zmm13	;; i2_7 i1_7 i4_7 i3_7 i2_3 i1_3 i4_3 i3_3			; 66		n 72

	L1prefetch wgtreg+1*128, L1pt
	vshuff64x2 zmm2, zmm2, zmm16, 11101110b ;; i8_4 i7_4 i6_4 i5_4 i4_4 i3_4 i2_4 i1_4 (I5)			; 67-69
	vblendmpd zmm16{k5}, zmm3, zmm1		;; i8_5 i7_5 i6_5 i5_5 i8_1 i7_1 i6_1 i5_1			; 67		n 70

	L1prefetch wgtreg+3*128, L1pt
	vshuff64x2 zmm13, zmm0, zmm18, 00010001b ;; i8_2 i7_2 i6_2 i5_2 i4_2 i3_2 i2_2 i1_2 (I3)		; 68-70
	vblendmpd zmm1{k5}, zmm1, zmm3		;; i6_7 i5_7 i8_7 i7_7 i6_3 i5_3 i8_3 i7_3			; 68		n 72

	L1prefetch wgtreg+2*128, L1pt
	vshuff64x2 zmm0, zmm0, zmm18, 10111011b ;; i8_6 i7_6 i6_6 i5_6 i4_6 i3_6 i2_6 i1_6 (I7)			; 69-71

	vshuff64x2 zmm18, zmm9, zmm16, 01000100b ;; i8_1 i7_1 i6_1 i5_1 i4_1 i3_1 i2_1 i1_1 (I2)		; 70-72

	vshuff64x2 zmm9, zmm9, zmm16, 11101110b ;; i8_5 i7_5 i6_5 i5_5 i4_5 i3_5 i2_5 i1_5 (I6)			; 71-73

	vshuff64x2 zmm16, zmm15, zmm1, 00010001b ;; i8_3 i7_3 i6_3 i5_3 i4_3 i3_3 i2_3 i1_3 (I4)		; 72-74

	vshuff64x2 zmm15, zmm15, zmm1, 10111011b ;; i8_7 i7_7 i6_7 i5_7 i4_7 i3_7 i2_7 i1_7 (I8)		; 73-75

	;; Code copied from zr8_sixteen_reals_eight_complex_djbfft

						;; Simultaneously do an 8-complex and 16-reals
						;; 8-complex			16-reals
	;; R1 above maps to			;; R1				R1+R9 (new R1)
	;; I1 above maps to			;; I1				R1-R9 (new R9), etc.

	vaddpd	zmm19, zmm5, zmm12		;;		R1 + R5 (new R1)				; 1-4		n 11
	vaddpd	zmm20, zmm10, zmm6		;;		R4 + R8 (new R4)				; 1-4		n 14

						;;				R12/R16 becomes newer R12/I12
						;;				mul R12/I12 by w^3 = .383 + .924i
	zfnmaddpd zmm1, zmm15, zmm29, zmm16	;; I4 - I8*1 (new I8)		R12/.383 = R12 - I12*.924/.383	; 2-5		n 8
	zfmaddpd zmm16, zmm16, zmm29, zmm15	;; I4*1 + I8 (new I4)		I12/.383 = R12*.924/.383 + I12	; 2-5		n 10

						;;				R11/R15 becomes newer R11/I11
						;;				mul R11/I11 by SQRTHALF + i*SQRTHALF
	vaddpd	zmm15, zmm13, zmm0		;; I3 + I7 (new I3)		I11/SQRTHALF = R11 + I11	; 3-6		n 9
	vsubpd	zmm13, zmm13, zmm0		;; I3 - I7 (new I7)		R11/SQRTHALF = R11 - I11	; 3-6		n 16

	vaddpd	zmm3, zmm7, zmm14		;;		R3 + R7 (new R3)				; 4-7		n 11
	vsubpd	zmm7, zmm7, zmm14		;; 		R3 - R7 (new R7)				; 4-7		n 9
						;;				R10/R14 becomes newer R10/I10
						;;				mul R10/I10 by w^1 = .924 + .383i
	zfmsubpd zmm0, zmm18, zmm29, zmm9	;; I2*1 - I6 (new I6)		R10/.383 = R10*.924/.383 - I10	; 5-8		n 13
	zfmaddpd zmm18, zmm9, zmm29, zmm18	;; I2 + I6*1 (new I2)		I10/.383 = I10*.924/.383 + R10	; 5-8		n 10

	vaddpd	zmm14, zmm4, zmm17		;;		R2 + R6 (new R2)				; 6-9		n 14
	vsubpd	zmm4, zmm4, zmm17		;;		R2 - R6 (new R6)				; 6-9		n 12

						;;				R9/R13 becomes newer R9/I9
	vaddpd	zmm9 {k7}{z}, zmm8, zmm2	;; I1 + I5 (new I1)		0				; 7-10		n 15
	vsubpd	zmm2 {k7}, zmm8, zmm2		;; I1 - I5 (new I5)		I9				; 7-10		n 17

	vsubpd	zmm8 {k7}, zmm5, zmm12		;; R1 - R5 (new R5)		blend in new R9			; 8-11		n 16
	vmovapd zmm25, [sc2reg+3*128+64]	;; cosine/sine for R2/I2 (w^4)							n 30

	vmovapd	zmm17, zmm1			;; new I8			not important
	vsubpd	zmm17 {k6}, zmm10, zmm6		;; blend in new I8		R4 - R8 (new R8)		; 8-11		n 12
	vmovapd	zmm21, zmm1			;; not important		R12	
	vsubpd	zmm21 {k7}, zmm10, zmm6		;; R4 - R8 (new R8)		blend in R12			; 9-12		n 13
	vmovapd zmm24, [sc2reg+1*128+64]	;; cosine/sine for R3/I3 (w^2)							n 31

						;; next FFT level
						;;				R6/R8 becomes newest R6/I6
						;;				mul R6/I6 by w^2 = .707 + .707i

	vblendmpd zmm22 {k6}, zmm7, zmm15	;; R7				I11/SQRTHALF			; 9		n 17

	vaddpd	zmm10, zmm18, zmm16		;; I2 + I4 (newer I2)		I10 + I12 (newest I10/.383)	; 10-13		n 18
	vsubpd	zmm18, zmm18, zmm16		;; I2 - I4 (newer I4)		I10 - I12 (newest I12/.383)	; 10-13		n 19
	vmovapd zmm6, [sc2reg+5*128+64]		;; cosine/sine for R4/I4 (w^6)							n 32

	vaddpd	zmm16, zmm19, zmm3		;; R1 + R3 (newer R1)		R1 + R3 (newest R1)		; 11-14		n 19
	vsubpd	zmm19, zmm19, zmm3		;; R1 - R3 (newer R3)		R1 - R3 (newest R3)		; 11-14		n 21

	vsubpd	zmm23, zmm4, zmm17		;; R6 - I8 (new2 R6)		R6 - I6 (newest R6/SQRTHALF)	; 12-15		n 18
	vaddpd	zmm4, zmm4, zmm17		;; R6 + I8 (new2 R8)		R6 + I6 (newest I6/SQRTHALF)	; 12-15		n 20

	vaddpd	zmm1, zmm0, zmm21		;; I6 + R8 (new2 I6)		R10 + R12 (newest R10/.383)	; 13-16		n 18
	vsubpd	zmm0, zmm0, zmm21		;; I6 - R8 (new2 I8)		R10 - R12 (newest R12/.383)	; 13-16		n 20
	vmovapd zmm21, [sc2reg+0*128+64]	;; cosine/sine for R5/I5 (w^1)							n 33

	vaddpd	zmm3, zmm14, zmm20		;; R2 + R4 (newer R2)		R2 + R4 (newest R2)		; 14-17		n 19
	vsubpd	zmm14, zmm14, zmm20		;; R2 - R4 (newer R4)		R2 - R4 (newest R4)		; 14-17		n 21

						;;				R5/R7 becomes newest R5/I5
	vsubpd	zmm7 {k7}, zmm9, zmm15		;; I1 - I3 (newer I3)		I5				; 15-18		n 27
	vaddpd	zmm9 {k7}{z}, zmm9, zmm15	;; I1 + I3 (newer I1)		0				; 15-18		n 23

	zfnmaddpd zmm11, zmm13, zmm28, zmm8	;; R5 - I7*1 (newer R5)		R9 - R11*-SQRTHALF (newest R9)	; 16-19		n 24
	zfmaddpd zmm13, zmm13, zmm28, zmm8	;; R5 + I7*1 (newer R7)		R9 + R11*-SQRTHALF (newest R11)	; 16-19		n 28

	zfmaddpd zmm8, zmm22, zmm30, zmm2	;; I5 + R7*1 (newer I5)		I9 + I11*SQRTHALF (newest I9)	; 17-20		n 25
	zfnmaddpd zmm22, zmm22, zmm30, zmm2	;; I5 - R7*1 (newer I7)		I9 - I11*SQRTHALF (newest I11)	; 17-20		n 29

	vmovapd	zmm2, zmm10			;;				I10/.383
	vaddpd	zmm2 {k7}, zmm23, zmm1		;; I6 = R6 + I6 (newer I6/SQRTHALF)	I10/.383		; 18-21		n 25
	vsubpd	zmm1 {k7}, zmm23, zmm1		;; R6 = R6 - I6 (newer R6/SQRTHALF)	R10/.383		; 18-21		n 24

	vsubpd	zmm10 {k6}, zmm16, zmm3		;; I2				R1 - R2 (final R1b)		; 19-22		n 23
	vblendmpd zmm23 {k6}, zmm18, zmm23	;; I4				R6/SQRTHALF			; 19		n 26

	vaddpd	zmm18 {k7}, zmm4, zmm0		;; I8 = R8 + I8 (newer I8/SQRTHALF)	I12/.383		; 20-23		n 28
	vsubpd	zmm0 {k7}, zmm4, zmm0		;; R8 = R8 - I8 (newer R8/SQRTHALF)	R12/.383		; 20-23		n 29

	;; last FFT level

	vmovapd zmm17, zmm19			;; R3				not important
	vsubpd	zmm17 {k6}, zmm5, zmm12		;; blend in R3			origR1 - origR5 (new R5)	; 21-24		n 26
	vblendmpd zmm4 {k6}, zmm14, zmm4	;; R4				I6/SQRTHALF			; 21		n 27
	vmovapd zmm5, [sc2reg+4*128+64]		;; cosine/sine for R6/I6 (w^5)							n 34

						;;				R3/R4 becomes final R3/I3
	vaddpd	zmm20, zmm16, zmm3		;; R1 + R2 (final R1)		R1 + R2 (final R1a => R1)	; 22-25
	vsubpd	zmm19 {k7}, zmm16, zmm3		;; R1 - R2 (final R2)		R3 => R2			; 22-25		n 30
	zstore	[srcreg], zmm20			;; Save R1							; 26
	vmovapd zmm20, [sc2reg+2*128+64]	;; cosine/sine for R7/I7 (w^3)							n 35

	vsubpd	zmm14 {k7}, zmm9, zmm10		;; I1 - I2 (final I2)		I3 => I2			; 23-26		n 30
	vaddpd	zmm10 {k7}, zmm9, zmm10		;; I1 + I2 (final I1)		R1b => I1			; 23-26
	zstore	[srcreg+64], zmm10		;; Save I1							; 27
	vmovapd zmm10, [sc2reg+6*128+64]	;; cosine/sine for R8/I8 (w^7)							n 36

	zfmaddpd zmm16, zmm1, zmm31, zmm11	;; R5 + R6*SQRTHALF (final R5)	R9 + R10*.383 (final R9 => R5)	; 24-27		n 33
	zfnmaddpd zmm1, zmm1, zmm31, zmm11	;; R5 - R6*SQRTHALF (final R6)	R9 - R10*.383 (final R10 => R6)	; 24-27		n 34

	zfmaddpd zmm11, zmm2, zmm31, zmm8	;; I5 + I6*SQRTHALF (final I5)	I9 + I10*.383 (final I9 => I5)	; 25-28		n 33
	zfnmaddpd zmm2, zmm2, zmm31, zmm8	;; I5 - I6*SQRTHALF (final I6)	I9 - I10*.383 (final I10 => I6)	; 25-28		n 34

	zfnmaddpd zmm9, zmm23, zmm28, zmm17	;; R3 - I4*1 (final R3)		R5 - R6*-SQRTHALF (final R5 => R3) ; 26-29	n 31
	zfmaddpd zmm17, zmm23, zmm28, zmm17	;; R3 + I4*1 (final R4)		R5 + R6*-SQRTHALF (final R6 => R4) ; 26-29	n 32
	vmovapd zmm23, [sc2reg+3*128]		;; sine for R2/I2 (w^4)								n 37

	zfmaddpd zmm15, zmm4, zmm30, zmm7	;; I3 + R4*1 (final I3)		I5 + I6*SQRTHALF (final I5 => I3) ; 27-30	n 31
	zfnmaddpd zmm4, zmm4, zmm30, zmm7	;; I3 - R4*1 (final I4)		I5 - I6*SQRTHALF (final I6 => I4) ; 27-30	n 32
	vmovapd zmm7, [sc2reg+1*128]		;; sine for R3/I3 (w^2)								n 38

	zfnmaddpd zmm3, zmm18, zmm31, zmm13	;; R7 - I8*SQRTHALF (final R7)	R11 - I12*.383 (final R11 => R7) ; 28-31	n 35
	zfmaddpd zmm18, zmm18, zmm31, zmm13	;; R7 + I8*SQRTHALF (final R8)	R11 + I12*.383 (final R12 => R8) ; 28-31	n 36

	zfmaddpd zmm13, zmm0, zmm31, zmm22	;; I7 + R8*SQRTHALF (final I7)	I11 + R12*.383 (final I11 => I7) ; 29-32	n 35
	zfnmaddpd zmm0, zmm0, zmm31, zmm22	;; I7 - R8*SQRTHALF (final I8)	I11 - R12*.383 (final I12 => I8) ; 29-32	n 36
	vmovapd zmm22, [sc2reg+5*128]		;; sine for R4/I4 (w^6)								n 39

	zfmsubpd zmm8, zmm19, zmm25, zmm14	;; A2 = R2 * cosine/sine - I2					; 30-33		n 37
	zfmaddpd zmm14, zmm14, zmm25, zmm19	;; B2 = I2 * cosine/sine + R2					; 30-33		n 37
	vmovapd zmm25, [sc2reg+0*128]		;; sine for R5/I5 (w^1)								n 40

	zfmsubpd zmm19, zmm9, zmm24, zmm15	;; A3 = R3 * cosine/sine - I3					; 31-34		n 38
	zfmaddpd zmm15, zmm15, zmm24, zmm9	;; B3 = I3 * cosine/sine + R3					; 31-34		n 38
	vmovapd zmm24, [sc2reg+4*128]		;; sine for R6/I6 (w^5)								n 41

	zfmsubpd zmm9, zmm17, zmm6, zmm4	;; A4 = R4 * cosine/sine - I4					; 32-35		n 39
	zfmaddpd zmm4, zmm4, zmm6, zmm17	;; B4 = I4 * cosine/sine + R4					; 32-35		n 39
	vmovapd zmm6, [sc2reg+2*128]		;; sine for R7/I7 (w^3)								n 42

	zfmsubpd zmm17, zmm16, zmm21, zmm11	;; A5 = R5 * cosine/sine - I5					; 33-36		n 40
	zfmaddpd zmm11, zmm11, zmm21, zmm16	;; B5 = I5 * cosine/sine + R5					; 33-36		n 40
	vmovapd zmm21, [sc2reg+6*128]		;; sine for R8/I8 (w^7)								n 43

	zfmsubpd zmm16, zmm1, zmm5, zmm2	;; A6 = R6 * cosine/sine - I6					; 34-37		n 41
	zfmaddpd zmm2, zmm2, zmm5, zmm1		;; B6 = I6 * cosine/sine + R6					; 34-37		n 41
	bump	sc2reg, sc2inc

	zfmsubpd zmm1, zmm3, zmm20, zmm13	;; A7 = R7 * cosine/sine - I7					; 35-38		n 42
	zfmaddpd zmm13, zmm13, zmm20, zmm3	;; B7 = I7 * cosine/sine + R7					; 35-38		n 42

	zfmsubpd zmm3, zmm18, zmm10, zmm0	;; A8 = R8 * cosine/sine - I8					; 36-39		n 43
	zfmaddpd zmm0, zmm0, zmm10, zmm18	;; B8 = I8 * cosine/sine + R8					; 36-39		n 43

	vmulpd	zmm8, zmm8, zmm23		;; A2 = A2 * sine (final R2)					; 37-40
	vmulpd	zmm14, zmm14, zmm23		;; B2 = B2 * sine (final I2)					; 37-40

	vmulpd	zmm19, zmm19, zmm7		;; A3 = A3 * sine (final R3)					; 38-41
	vmulpd	zmm15, zmm15, zmm7		;; B3 = B3 * sine (final I3)					; 38-41

	vmulpd	zmm9, zmm9, zmm22		;; A4 = A4 * sine (final R4)					; 39-42
	vmulpd	zmm4, zmm4, zmm22		;; B4 = B4 * sine (final I4)					; 39-42

	vmulpd	zmm17, zmm17, zmm25		;; A5 = A5 * sine (final R5)					; 40-43
	vmulpd	zmm11, zmm11, zmm25		;; B5 = B5 * sine (final I5)					; 40-43

	vmulpd	zmm16, zmm16, zmm24		;; A6 = A6 * sine (final R6)					; 41-44
	vmulpd	zmm2, zmm2, zmm24		;; B6 = B6 * sine (final I6)					; 41-44
	zstore	[srcreg+d1], zmm8		;; Save R2							; 41

	vmulpd	zmm1, zmm1, zmm6		;; A7 = A7 * sine (final R7)					; 42-45
	vmulpd	zmm13, zmm13, zmm6		;; B7 = B7 * sine (final I7)					; 42-45
	zstore	[srcreg+d1+64], zmm14		;; Save I2							; 42

	vmulpd	zmm3, zmm3, zmm21		;; A8 = A8 * sine (final R8)					; 43-46
	vmulpd	zmm0, zmm0, zmm21		;; B8 = B8 * sine (final I8)					; 43-46

	zstore	[srcreg+d2], zmm19		;; Save R3
	zstore	[srcreg+d2+64], zmm15		;; Save I3
	zstore	[srcreg+d2+d1], zmm9		;; Save R4
	zstore	[srcreg+d2+d1+64], zmm4		;; Save I4
	zstore	[srcreg+d4], zmm17		;; Save R5
	zstore	[srcreg+d4+64], zmm11		;; Save I5
	zstore	[srcreg+d4+d1], zmm16		;; Save R6
	zstore	[srcreg+d4+d1+64], zmm2		;; Save I6
	zstore	[srcreg+d4+d2], zmm1		;; Save R7
	zstore	[srcreg+d4+d2+64], zmm13	;; Save I7
	zstore	[srcreg+d4+d2+d1], zmm3		;; Save R8
	zstore	[srcreg+d4+d2+d1+64], zmm0	;; Save I8
	bump	srcreg, srcinc
	ENDM

zr8s_hundredtwentyeight_reals_last_unfft_preload MACRO
	mov	r14d, 11111110b				;; We're pretty sure r14 is safe to use
	kmovw	k7, r14d				;; Set k7 to 11111110b
	knotw	k6, k7					;; Set k6 to 00000001b
	vpmovzxbq zmm31, ZMM_PERMUTE7			;; zmm31 = 8+4 8+5 8+0 8+1 0+4 0+5 0+0 0+1
	vpmovzxbq zmm30, ZMM_PERMUTE8			;; zmm30 = 8+6 8+7 8+2 8+3 0+6 0+7 0+2 0+3
	vbroadcastsd zmm0, ZMM_SQRTHALF
	vbroadcastsd zmm1, ZMM_ONE
	vblendmpd zmm29 {k6}, zmm0, zmm1		;; SQRTHALF (7 times) ONE
	vblendmpd zmm28 {k6}, zmm0, ZMM_P383{1to8}	;; SQRTHALF (7 times) .383
	vblendmpd zmm27 {k6}, zmm1, ZMM_P924_P383{1to8}	;;      ONE (7 times) .924/.383
	vblendmpd zmm26 {k6}, zmm1, zmm0		;;	ONE (7 times) SQRTHALF
	vshuff64x2 zmm26, zmm26, zmm26, 01000100b	;; 1, 1, 1, SQRTHALF, 1, 1, 1, SQRTHALF
	mov	r14d, 10101010b				;; For vblendmpd during swizzle
	kmovw	k5, r14d
	ENDM

zr8s_hundredtwentyeight_reals_last_unfft MACRO srcreg,srcinc,d1,d2,d4,screg,scinc,sc2reg,sc2inc,maxrpt,L1pt,L1pd
	vmovapd	zmm2, [srcreg+d1]			;; R2
	vmovapd	zmm1, [srcreg+d1+64]			;; I2
	vmovapd zmm25, [screg+3*128+64]			;; cosine/sine for R2/I2 (w^4)
	zfmaddpd zmm0, zmm2, zmm25, zmm1		;; A2 = R2 * cosine/sine + I2					; 1-4		n 11
	zfmsubpd zmm1, zmm1, zmm25, zmm2		;; B2 = I2 * cosine/sine - R2					; 1-4		n 11

	vmovapd	zmm4, [srcreg+d2]			;; R3
	vmovapd	zmm3, [srcreg+d2+64]			;; I3
	vmovapd zmm25, [screg+1*128+64]			;; cosine/sine for R3/I3 (w^2)
	zfmaddpd zmm2, zmm4, zmm25, zmm3		;; A3 = R3 * cosine/sine + I3					; 2-5		n 8
	zfmsubpd zmm3, zmm3, zmm25, zmm4		;; B3 = I3 * cosine/sine - R3					; 2-5		n 8

	vmovapd	zmm6, [srcreg+d2+d1]			;; R4
	vmovapd	zmm5, [srcreg+d2+d1+64]			;; I4
	vmovapd zmm25, [screg+5*128+64]			;; cosine/sine for R4/I4 (w^6)
	zfmaddpd zmm4, zmm6, zmm25, zmm5		;; A4 = R4 * cosine/sine + I4					; 3-6		n 12
	zfmsubpd zmm5, zmm5, zmm25, zmm6		;; B4 = I4 * cosine/sine - R4					; 3-6		n 13

	vmovapd	zmm8, [srcreg+d4]			;; R5
	vmovapd	zmm7, [srcreg+d4+64]			;; I5
	vmovapd zmm25, [screg+0*128+64]			;; cosine/sine for R5/I5 (w^1)
	zfmaddpd zmm6, zmm8, zmm25, zmm7		;; A5 = R5 * cosine/sine + I5					; 4-7		n 9
	zfmsubpd zmm7, zmm7, zmm25, zmm8		;; B5 = I5 * cosine/sine - R5					; 4-7		n 9

	vmovapd	zmm10, [srcreg+d4+d1]			;; R6
	vmovapd	zmm9, [srcreg+d4+d1+64]			;; I6
	vmovapd zmm25, [screg+4*128+64]			;; cosine/sine for R6/I6 (w^5)
	zfmaddpd zmm8, zmm10, zmm25, zmm9		;; A6 = R6 * cosine/sine + I6					; 5-8		n 14
	zfmsubpd zmm9, zmm9, zmm25, zmm10		;; B6 = I6 * cosine/sine - R6					; 5-8		n 15

	vmovapd	zmm12, [srcreg+d4+d2]			;; R7
	vmovapd	zmm11, [srcreg+d4+d2+64]		;; I7
	vmovapd zmm25, [screg+2*128+64]			;; cosine/sine for R7/I7 (w^3)
	zfmaddpd zmm10, zmm12, zmm25, zmm11		;; A7 = R7 * cosine/sine + I7					; 6-9		n 10
	zfmsubpd zmm11, zmm11, zmm25, zmm12		;; B7 = I7 * cosine/sine - R7					; 6-9		n 10

	vmovapd	zmm14, [srcreg+d4+d2+d1]		;; R8
	vmovapd	zmm13, [srcreg+d4+d2+d1+64]		;; I8
	vmovapd zmm25, [screg+6*128+64]			;; cosine/sine for R8/I8 (w^7)
	zfmaddpd zmm12, zmm14, zmm25, zmm13		;; A8 = R8 * cosine/sine + I8					; 7-10		n 16
	zfmsubpd zmm13, zmm13, zmm25, zmm14		;; B8 = I8 * cosine/sine - R8					; 7-10		n 17

	vmovapd zmm25, [screg+1*128]			;; sine for R3/I3 (w^2)
	vmulpd	zmm2, zmm2, zmm25			;; A3 = A3 * sine (final R3)					; 8-11		n 12
	vmulpd	zmm3, zmm3, zmm25			;; B3 = B3 * sine (final I3)					; 8-11		n 13

	vmovapd zmm25, [screg+0*128]			;; sine for R5/I5 (w^1)
	vmulpd	zmm6, zmm6, zmm25			;; A5 = A5 * sine (final R5)					; 9-12		n 14
	vmulpd	zmm7, zmm7, zmm25			;; B5 = B5 * sine (final I5)					; 9-12		n 15

	vmovapd zmm25, [screg+2*128]			;; sine for R7/I7 (w^3)
	vmulpd	zmm10, zmm10, zmm25			;; A7 = A7 * sine (final R7)					; 10-13		n 16
	vmulpd	zmm11, zmm11, zmm25			;; B7 = B7 * sine (final I7)					; 10-13		n 17

							;; Simultaneously do an 8-complex and 16-reals
							;; 8-complex			16-reals

							;;				R1a/R1b becomes R1/R2, R2/I2 becomes R3/R4, R3/I3 => R5/I5, etc.
	vmovapd zmm25, [screg+3*128]			;; sine for R2/I2 (w^4)
	vmovapd	zmm14, [srcreg+64]			;; I1				R1b
	vmovapd	zmm15, zmm14				;; not important		R2
	vmulpd	zmm15 {k7}, zmm0, zmm25			;; R2*sine			blend in R2			; 11-14		n 18

	vmovapd zmm24, [screg+5*128]			;; sine for R4/I4 (w^6)								n 12
	zfmaddpd zmm16, zmm1, zmm25, zmm14		;; I1 + I2*sine (new I1)	not important			; 11-14		n 31
	zfnmaddpd zmm1 {k7}, zmm1, zmm25, zmm14		;; I1 - I2*sine (new I2)	blend in R4			; 12-15		n 20

	vmovapd zmm23, [screg+4*128]			;; sine for R6/I6 (w^5)								n 14
	zfmaddpd zmm14, zmm4, zmm24, zmm2		;; R4*sine + R3 (new R3)	R6*sine + R5 (new R5)		; 12-15		n 19
	zfmsubpd zmm4, zmm4, zmm24, zmm2		;; R4*sine - R3 (new I4)	R6*sine - R5 (new negR6)	; 13-16		n 24

	vmovapd zmm22, [screg+6*128]			;; sine for R8/I8 (w^7)								n 16
	zfnmaddpd zmm2, zmm5, zmm24, zmm3		;; I3 - I4*sine (new R4)	I5 - I6*sine (new I6)		; 13-16		n 20
	zfmaddpd zmm5, zmm5, zmm24, zmm3		;; I3 + I4*sine (new I3)	I5 + I6*sine (new I5)		; 14-17		n 30
	bump	screg, scinc

	vmovapd	zmm17, [srcreg]				;; R1				R1a						n 18
	zfnmaddpd zmm3, zmm8, zmm23, zmm6		;; R5 - R6*sine (new R6)	R9 - R10*sine (new R10)		; 14-17		n 21
	zfmaddpd zmm8, zmm8, zmm23, zmm6		;; R5 + R6*sine (new R5)	R9 + R10*sine (new R9)		; 15-18		n 22

	vbroadcastsd zmm24, ZMM_NEGSQRTHALF										;		n 32
	zfnmaddpd zmm6, zmm9, zmm23, zmm7		;; I5 - I6*sine (new I6)	I9 - I10*sine (new I10)		; 15-18		n 25
	zfmaddpd zmm9, zmm9, zmm23, zmm7		;; I5 + I6*sine (new I5)	I9 + I10*sine (new I9)		; 16-19		n 23

	vmovapd	zmm23, [sc2reg+5*128]			;; sine for R7/I7								n 55
	zfmsubpd zmm7, zmm12, zmm22, zmm10		;; R8*sine - R7 (new I8)	R12*sine - R11 (new I12)	; 16-19		n 21
	zfmaddpd zmm12, zmm12, zmm22, zmm10		;; R8*sine + R7 (new R7)	R12*sine + R11 (new R11)	; 17-20		n 22

	zfnmaddpd zmm10, zmm13, zmm22, zmm11		;; I7 - I8*sine (new R8)	I11 - I12*sine (new R12)	; 17-20		n 22
	zfmaddpd zmm13, zmm13, zmm22, zmm11		;; I7 + I8*sine (new I7)	I11 + I12*sine (new I11)	; 18-21		n 23

	vmovapd	zmm22, [sc2reg+6*128]			;; sine for R8/I8								n 59
	vaddpd	zmm11, zmm17, zmm15			;; R1 + R2 (new R1)		R1 + R2 (new R1)		; 18-21		n 27
	vsubpd	zmm17, zmm17, zmm15			;; R1 - R2 (new R2)		R1 - R2 (new R2)		; 19-22		n 28

;; two aparts

	vmovapd	zmm15, zmm14				;; R3				not important
	vmulpd	zmm15 {k6}, zmm0, zmm25			;; blend in R3			R3*sine				; 19-22		n 27
	vmovapd	zmm0, zmm2				;; R4				not important
	vmulpd	zmm0 {k6}, zmm1, zmm25			;; blend in R4			R4*sine				; 20-23		n 28

	vmovapd	zmm25, [sc2reg+4*128+64]		;; cosine/sine for R6/I6 (w^5)							n 61
	vblendmpd zmm1 {k6}, zmm1, zmm2			;; I2				I6				; 20-23		n 24
	vmovapd	zmm2, zmm7				;; not important		I12
	vmulpd	zmm2 {k7}, zmm3, zmm28			;; R6*SQRTHALF			blend in I12			; 21-24		n 25
	vblendmpd zmm7 {k6}, zmm7, zmm3			;; I8				R10				; 21-24		n 26
	vmulpd	zmm10, zmm10, zmm28			;; R8*SQRTHALF			R12*.383			; 22-25		n 26

	vaddpd	zmm3, zmm12, zmm8			;; R7 + R5 (newer R5)		R11 + R9 (newer R9)		; 22-25		n 29
	vsubpd	zmm12, zmm12, zmm8			;; R7 - R5 (newer I7)		R11 - R9 (newer negR11)		; 23-26		n 32
	L1prefetchw srcreg+srcinc+d1, L1pt

	vsubpd	zmm8, zmm9, zmm13			;; I5 - I7 (newer R7)		I9 - I11 (newer I11)		; 23-26		n 30
	vaddpd	zmm9, zmm9, zmm13			;; I5 + I7 (newer I5)		I9 + I11 (newer I9)		; 24-27		n 39
	L1prefetchw srcreg+srcinc+d1+64, L1pt

							;;				mul R6/I6 by w^2 = .707 - .707i
	vaddpd	zmm13, zmm1, zmm4			;; I2 + I4 (newer I2)		I6 + negR6 (newer I6/SQRTHALF) 	; 24-27		n 30
	vsubpd	zmm1, zmm1, zmm4			;; I2 - I4 (newer I4)		I6 - negR6 (newer R6/SQRTHALF) 	; 25-28		n 31
	L1prefetch screg+3*128+64, L1pt

							;; multiply R6/I6 by SQRTHALF - i*SQRTHALF
	zfmaddpd zmm4, zmm6, zmm29, zmm2		;; I6*SQRTHALF + R6 (new2 R6)	I10*1 + I12 (newer I10)		; 25-28		n 30
	zfmsubpd zmm6, zmm6, zmm29, zmm2		;; I6*SQRTHALF - R6 (new2 I6)	I10*1 - I12 (newer I12)		; 26-29		n 31
	L1prefetchw srcreg+srcinc+d2, L1pt

							;; multiply R8/I8 by SQRTHALF - i*SQRTHALF
	zfmaddpd zmm2, zmm7, zmm28, zmm10		;; I8*SQRTHALF + R8 (new2 R8)	R10*.383 + R12 (newer R10*.383)	; 26-29		n 33
	zfmsubpd zmm7, zmm7, zmm28, zmm10		;; I8*SQRTHALF - R8 (new2 I8)	R10*.383 - R12 (newer R12*.383)	; 27-30		n 33
	L1prefetchw srcreg+srcinc+d2+64, L1pt

	vaddpd	zmm10, zmm11, zmm15			;; R1 + R3 (newer R1)		R1 + R3 (newer R1)		; 27-30		n 33
	vsubpd	zmm11, zmm11, zmm15			;; R1 - R3 (newer R3)		R1 - R3 (newer R3)		; 28-31		n 33
	L1prefetch screg+1*128+64, L1pt

	vaddpd	zmm15, zmm17, zmm0			;; R2 + R4 (newer R2)		R2 + R4 (newer R2)		; 28-31		n 41
	vsubpd	zmm17, zmm17, zmm0			;; R2 - R4 (newer R4)		R2 - R4 (newer R4)		; 29-32		n 41
	L1prefetchw srcreg+srcinc+d2+d1, L1pt

							;;				R5/I5 becomes newer R5/R7
	vblendmpd zmm14 {k6}, zmm3, zmm14		;; newer R5			newer R5			; 29-32		n 35
	vblendmpd zmm0 {k6}, zmm8, zmm5			;; newer R7			newer R7			; 30-33		n 35
	vmovapd	zmm18, zmm13				;; newer I2			not important
	vmulpd	zmm18 {k6}, zmm4, zmm28			;; blend in newer I2		I10*.383			; 30-33		n 39
	vmovapd	zmm19, zmm1				;; newer I4			not important
	vmulpd	zmm19 {k6}, zmm6, zmm28			;; blend in newer I4		I12*.383			; 31-34		n 37
	L1prefetchw srcreg+srcinc+d2+d1+64, L1pt

	vaddpd	zmm20 {k7}{z}, zmm16, zmm5		;; I1 + I3 (newer I1)		0				; 31-34		n 37
	vsubpd	zmm8 {k7}, zmm16, zmm5			;; I1 - I3 (newer I3)		blend in newer I11		; 32-35		n 37
	vmulpd	zmm12 {k6}, zmm12, zmm24		;; blend in newer I7		R11*SQRTHALF = negR11*-SQRTHALF	; 32-35		n 39

	vmovapd	zmm24, [sc2reg+3*128+64]		;; cosine/sine for R5/I5 (w^4)							n 63
	vshuff64x2 zmm5, zmm10, zmm11, 11101110b	;; R31H = r3_7654 r1_7654					; 33-35		n 41
	vmovapd	zmm16, zmm7				;; not important		newer R12*.383
	vsubpd	zmm16 {k7}, zmm2, zmm4			;; R8 - R6 (newer I8)		blend in newer R12*.383		; 33-36   	n 37

	vshuff64x2 zmm10, zmm10, zmm11, 01000100b	;; R31L = r3_3210 r1_3210					; 34-36		n 47
	vaddpd	zmm1 {k7}, zmm4, zmm2			;; R6 + R8 (newer R6)		blend in newer R6/SQRTHALF	; 34-37   	n 42
	L1prefetch screg+5*128+64, L1pt

	vshuff64x2 zmm4, zmm14, zmm0, 11101110b		;; R75H = r7_7654 r5_7654					; 35-37		n 41
	vaddpd zmm2 {k7}, zmm6, zmm7			;; I6 + I8 (newer I6)		blend in newer R10*.383		; 35-38   	n 39
	L1prefetchw srcreg+srcinc+d4, L1pt

	vshuff64x2 zmm14, zmm14, zmm0, 01000100b	;; R75L = r7_3210 r5_3210					; 36-38		n 47
	vsubpd zmm13 {k7}, zmm6, zmm7			;; I6 - I8 (newer R8)		blend in newer I6/SQRTHALF	; 36-39   	n 42
	L1prefetchw srcreg+srcinc+d4+64, L1pt

;; four aparts

							;;				R9/I9 becomes newer R9/R13
							;;				R11/I11 becomes newer R11/R15
	vshuff64x2 zmm7, zmm20, zmm8, 11101110b		;; I31H = i3_7654 i1_7654					; 37-39		n 43
							;;				mul R12/I12 by w^3 = .383 - .924i
							;;				R12/I12 becomes newer R12/R16
	zfmaddpd zmm6, zmm19, zmm27, zmm16		;; I4*1 + I8 (final I4)		I12*.924/.383 + R12 (final R12)	; 37-40		n 45
	L1prefetch screg+0*128+64, L1pt

	vshuff64x2 zmm3 {k7}, zmm20, zmm8, 01000100b	;; I31L = i3_3210 i1_3210	R15, blend in R9		; 38-40		n 51
	zfnmaddpd zmm16, zmm16, zmm27, zmm19		;; I4 - I8*1 (final I8)		I12 - R12*.924/.383 (final R16)	; 38-41		n 46
	L1prefetchw srcreg+srcinc+d4+d1, L1pt

	vshuff64x2 zmm19, zmm9, zmm12, 11101110b	;; I75H = i7_7654 i5_7654					; 39-41		n 43
							;;				mul R10/I10 by w^1 = .924 - .383i
							;;				R10/I10 becomes newer R10/R14
	zfmaddpd zmm8, zmm2, zmm27, zmm18		;; I2 + I6*1 (final I2)		I10 + R10*.924/.383 (final R10)	; 39-42		n 45
	L1prefetchw srcreg+srcinc+d4+d1+64, L1pt

	vshuff64x2 zmm9 {k7}, zmm9, zmm12, 01000100b	;; I75L = i7_3210 i5_3210	R11, blend in R13		; 40-42		n 51
	zfmsubpd zmm18, zmm18, zmm27, zmm2		;; I2*1 - I6 (final I6)		I10*.924/.383 - R10 (final R14)	; 40-43		n 46
	L1prefetch screg+4*128+64, L1pt

							;;				R6/I6 becomes new R6/R8
	vshuff64x2 zmm2, zmm15, zmm17, 11101110b	;; R42H = r4_7654 r2_7654					; 41-43		n 45
	vaddpd	zmm12, zmm5, zmm4			;; final R31H = R31H + R75H					; 41-44		n 47
	L1prefetchw srcreg+srcinc+d4+d2, L1pt

	vshuff64x2 zmm20, zmm1, zmm13, 11101110b	;; R86H = r8_7654 r6_7654					; 42-44		n 45
	vsubpd	zmm5, zmm5, zmm4			;; final R75H = R31H - R75H					; 42-45		n 47
	L1prefetchw srcreg+srcinc+d4+d2+64, L1pt

	vshuff64x2 zmm15, zmm15, zmm17, 01000100b	;; R42L = r4_3210 r2_3210					; 43-45		n 49
	vaddpd	zmm17, zmm7, zmm19			;; final I31H = I31H + I75H					; 43-46		n 49
	L1prefetch screg+2*128+64, L1pt

	vshuff64x2 zmm1, zmm1, zmm13, 01000100b		;; R86L = r8_3210 r6_3210	R8/SQRTHALF, R6/SQRTHALF	; 44-46		n 49
	vsubpd	zmm7, zmm7, zmm19			;; final I75H = I31H - I75H					; 44-47		n 49
	L1prefetchw srcreg+srcinc+d4+d2+d1, L1pt

	vshuff64x2 zmm19, zmm8, zmm6, 11101110b		;; final I42H							; 45-47		n 50
	vaddpd	zmm13, zmm2, zmm20			;; final R42H = R42H + R86H					; 45-48		n 51
	L1prefetchw srcreg+srcinc+d4+d2+d1+64, L1pt

	vshuff64x2 zmm4, zmm18, zmm16, 11101110b	;; final I86H							; 46-48		n 50
	vsubpd	zmm2, zmm2, zmm20			;; final R86H = R42H - R86H					; 46-49		n 54
	L1prefetch screg+6*128+64, L1pt

	vshuff64x2 zmm20, zmm12, zmm5, 11011101b	;; r7_7 r7_6 r5_7 r5_6 r3_7 r3_6 r1_7 r1_6			; 47-49		n 54
	vaddpd	zmm0, zmm10, zmm14			;; final R31L = R31L + R75L					; 47-50		n 61
	L1prefetch screg+1*128, L1pt

	vshuff64x2 zmm12, zmm12, zmm5, 10001000b	;; r7_5 r7_4 r5_5 r5_4 r3_5 r3_4 r1_5 r1_4			; 48-50		n 58
	vsubpd	zmm10, zmm10, zmm14			;; final R75L = R31L - R75L					; 48-51		n 61
	L1prefetch screg+0*128, L1pt

	zperm2pd zmm14, zmm30, zmm17, zmm7		;; i7_6 i7_7 i5_6 i5_7 i3_6 i3_7 i1_6 i1_7			; 49-51		n 53
	zfmaddpd zmm5, zmm1, zmm26, zmm15		;; final R42L = R42L + R86L*1	(R4,R2) + (R8,R6)*SQRTHALF	; 49-52		n 63
	L1prefetch screg+2*128, L1pt

	vshuff64x2 zmm11, zmm19, zmm4, 11011101b	;; i8_7 i8_6 i6_7 i6_6 i4_7 i4_6 i2_7 i2_6			; 50-52		n 53
	zfnmaddpd zmm1, zmm1, zmm26, zmm15		;; final R86L = R42L - R86L*1	(R4,R2) - (R8,R6)*SQRTHALF	; 50-53		n 63
	L1prefetch screg+3*128, L1pt

	zperm2pd zmm15, zmm30, zmm13, zmm2		;; r8_6 r8_7 r6_6 r6_7 r4_6 r4_7 r2_6 r2_7			; 51-53		n 54
							;;				mul R11/I11 by SQRTHALF - i*SQRTHALF
	vmovapd zmm21, zmm3				;; Copy I31L
	zfmaddpd zmm3 {k7}, zmm3, zmm26, zmm9		;; final I31L = I31L*1 + I75L	I11*SQRTHALF + R11, BLEND=R9	; 51-54		n 65
	L1prefetchw srcreg+srcinc+64, L1pt

	zperm2pd zmm17, zmm31, zmm17, zmm7		;; i7_4 i7_5 i5_4 i5_5 i3_4 i3_5 i1_4 i1_5			; 52-54		n 57
	zfmsubpd zmm9 {k7}, zmm21, zmm26, zmm9		;; final I75L = I31L*1 - I75L	I11*SQRTHALF - R11, BLEND=R13	; 52-55		n 65

	vmovapd	zmm7, [sc2reg+5*128+64]			;; cosine/sine for R7/I7 (w^6)							n 65
	vshuff64x2 zmm19, zmm19, zmm4, 10001000b	;; i8_5 i8_4 i6_5 i6_4 i4_5 i4_4 i2_5 i2_4			; 53-55		n 57
	vblendmpd zmm4 {k5}, zmm14, zmm11		;; i8_7 i7_7 i6_7 i5_7 i4_7 i3_7 i2_7 i1_7 (I8)			; 53		n 60

	vmovapd	zmm21, [sc2reg+4*128]			;; sine for R6/I6								n 67
	zperm2pd zmm13, zmm31, zmm13, zmm2		;; r8_4 r8_5 r6_4 r6_5 r4_4 r4_5 r2_4 r2_5			; 54-56		n 58
	vblendmpd zmm2 {k5}, zmm20, zmm15		;; r8_6 r7_6 r6_6 r5_6 r4_6 r3_6 r2_6 r1_6 (R7)			; 54		n 57

	vshufpd	zmm14, zmm14, zmm11, 01010101b		;; i8_6 i7_6 i6_6 i5_6 i4_6 i3_6 i2_6 i1_6 (I7)			; 55		n 56
	vmulpd	zmm2, zmm2, zmm23			;; R7s = R7 * sine						; 55-58		n 65

	vmovapd	zmm11, [sc2reg+6*128+64]		;; cosine/sine for R8/I8 (w^7)							n 69
	vshufpd	zmm20, zmm20, zmm15, 01010101b		;; r8_7 r7_7 r6_7 r5_7 r4_7 r3_7 r2_7 r1_7 (R8)			; 56		n 59
	vmulpd	zmm14, zmm14, zmm23			;; I7s = I7 * sine						; 56-59		n 65

	vmovapd	zmm23, [sc2reg+1*128+64]		;; cosine/sine for R3/I3 (w^2)							n 74
	vshufpd	zmm15, zmm17, zmm19, 01010101b		;; i8_4	i7_4 i6_4 i5_4 i4_4 i3_4 i2_4 i1_4 (I5)			; 57		n 63
	vblendmpd zmm17 {k5}, zmm17, zmm19		;; i8_5 i7_5 i6_5 i5_5 i4_5 i3_5 i2_5 i1_5 (I6)			; 57		n 61

	vshufpd	zmm19, zmm12, zmm13, 01010101b		;; r8_5 r7_5 r6_5 r5_5 r4_5 r3_5 r2_5 r1_5 (R6)			; 58		n 61
	vblendmpd zmm12 {k5}, zmm12, zmm13		;; r8_4	r7_4 r6_4 r5_4 r4_4 r3_4 r2_4 r1_4 (R5)			; 58		n 63

	vmovapd	zmm13, [sc2reg+0*128+64]		;; cosine/sine for R2/I2 (w^1)							n 75
	vshuff64x2 zmm8, zmm8, zmm6, 01000100b		;; I42L								; 59-61		n 67
	vmulpd	zmm20, zmm20, zmm22			;; R8s = R8 * sine						; 59-62		n 69

	vshuff64x2 zmm18, zmm18, zmm16, 01000100b	;; I86L								; 60-62		n 67
	vmulpd	zmm4, zmm4, zmm22			;; I8s = I8 * sine						; 60-63		n 69

	vmovapd	zmm22, [sc2reg+2*128+64]		;; cosine/sine for R4/I4 (w^3)							n 76
	vshuff64x2 zmm16, zmm0, zmm10, 11011101b	;; r7_3 r7_2 r5_3 r5_2 r3_3 r3_2 r1_3 r1_2			; 61-63		n 70
	zfmaddpd zmm6, zmm19, zmm25, zmm17		;; A6 = R6 * cosine/sine + I6					; 61-64		n 67

	vshuff64x2 zmm0, zmm0, zmm10, 10001000b		;; r7_1 r7_0 r5_1 r5_0 r3_1 r3_0 r1_1 r1_0			; 62-64		n 71
	zfmsubpd zmm17, zmm17, zmm25, zmm19		;; B6 = I6 * cosine/sine - R6					; 62-65		n 68

	vmovapd	zmm25, [sc2reg+1*128]			;; sine for R3/I3								n 78
	zperm2pd zmm19, zmm30, zmm5, zmm1		;; r8_2 r8_3 r6_2 r6_3 r4_2 r4_3 r2_2 r2_3			; 63-65		n 70
	zfmaddpd zmm10, zmm12, zmm24, zmm15		;; R5/sine = R5 * cosine/sine + I5				; 63-66		n 84

	zperm2pd zmm5, zmm31, zmm5, zmm1		;; r8_0 r8_1 r6_0 r6_1 r4_0 r4_1 r2_0 r2_1			; 64-66		n 71
	zfmsubpd zmm15, zmm15, zmm24, zmm12		;; I5/sine = I5 * cosine/sine - R5				; 64-67		n 85

	vmovapd	zmm24, [sc2reg+0*128]			;; sine for R2/I2								n 80
	zperm2pd zmm12, zmm30, zmm3, zmm9		;; i7_2 i7_3 i5_2 i5_3 i3_2 i3_3 i1_2 i1_3			; 65-67		n 72
	zfmaddpd zmm1, zmm2, zmm7, zmm14		;; R7 = R7s * cosine/sine + I7s					; 65-68		n 78

	zperm2pd zmm3, zmm31, zmm3, zmm9		;; i7_0 i7_1 i5_0 i5_1 i3_0 i3_1 i1_0 i1_1			; 66-68		n 73
	zfmsubpd zmm14, zmm14, zmm7, zmm2		;; I7 = I7s * cosine/sine - R7s					; 66-69		n 79

	vmovapd	zmm9, [sc2reg+2*128]			;; sine for R4/I4								n 82
	vshuff64x2 zmm2, zmm8, zmm18, 11011101b		;; i8_3 i8_2 i6_3 i6_2 i4_3 i4_2 i2_3 i2_2			; 67-69		n 72
	vmulpd	zmm6, zmm6, zmm21			;; R6 = A6 * sine						; 67-70		n 82

	vmovapd	zmm7, [sc2reg+3*128]			;; sine for R5/I5								n 84
	vshuff64x2 zmm8, zmm8, zmm18, 10001000b		;; i8_1 i8_0 i6_1 i6_0 i4_1 i4_0 i2_1 i2_0			; 68-70		n 73
	vmulpd	zmm17, zmm17, zmm21			;; I6 = B6 * sine						; 68-71		n 83
	bump	sc2reg, sc2inc

	vbroadcastsd zmm21, ZMM_SQRTHALF										;		n 96
	zfmaddpd zmm18, zmm20, zmm11, zmm4		;; R8 = R8s * cosine/sine + I8s					; 69-72		n 80
	zfmsubpd zmm4, zmm4, zmm11, zmm20		;; I8 = I8s * cosine/sine - R8s					; 69-72		n 81

	vbroadcastsd zmm11, ZMM_P924_P383										;		n 97
	vshufpd	zmm20, zmm16, zmm19, 01010101b		;; r8_3 r7_3 r6_3 r5_3 r4_3 r3_3 r2_3 r1_3 (R4)			; 70		n 76
	vblendmpd zmm16 {k5}, zmm16, zmm19		;; r8_2 r7_2 r6_2 r5_2 r4_2 r3_2 r2_2 r1_2 (R3)			; 70		n 74

	vshufpd	zmm19, zmm0, zmm5, 01010101b		;; r8_1 r7_1 r6_1 r5_1 r4_1 r3_1 r2_1 r1_1 (R2)			; 71		n 75
	vblendmpd zmm0 {k5}, zmm0, zmm5			;; r8_0	r7_0 r6_0 r5_0 r4_0 r3_0 r2_0 r1_0 (R1)			; 71		n 77
	L1prefetch screg+5*128, L1pt

	vshufpd	zmm5, zmm12, zmm2, 01010101b		;; i8_2 i7_2 i6_2 i5_2 i4_2 i3_2 i2_2 i1_2 (I3)			; 72		n 74
	vblendmpd zmm12 {k5}, zmm12, zmm2		;; i8_3 i7_3 i6_3 i5_3 i4_3 i3_3 i2_3 i1_3 (I4)			; 72		n 76
	L1prefetch screg+4*128, L1pt

	vshufpd	zmm2, zmm3, zmm8, 01010101b		;; i8_0	i7_0 i6_0 i5_0 i4_0 i3_0 i2_0 i1_0 (I1)			; 73		n 77
	vblendmpd zmm3 {k5}, zmm3, zmm8			;; i8_1 i7_1 i6_1 i5_1 i4_1 i3_1 i2_1 i1_1 (I2)			; 73		n 75
	L1prefetch screg+6*128, L1pt

	zfmaddpd zmm8, zmm16, zmm23, zmm5		;; R3/sine = R3 * cosine/sine + I3				; 74-77		n 78
	zfmsubpd zmm5, zmm5, zmm23, zmm16		;; I3/sine = I3 * cosine/sine - R3				; 74-77		n 79

	vbroadcastsd zmm23, ZMM_P383											;		n 102
	zfmaddpd zmm16, zmm19, zmm13, zmm3		;; R2/sine = R2 * cosine/sine + I2				; 75-78		n 80
	zfmsubpd zmm3, zmm3, zmm13, zmm19		;; I2/sine = I2 * cosine/sine - R2				; 75-78		n 81

	zfmaddpd zmm19, zmm20, zmm22, zmm12		;; R4/sine = R4 * cosine/sine + I4				; 76-79		n 82
	zfmsubpd zmm12, zmm12, zmm22, zmm20		;; I4/sine = I4 * cosine/sine - R4				; 76-79		n 83
	L1prefetchw srcreg+srcinc, L1pt

	vaddpd	zmm20, zmm0, zmm2			;; R1+R9							; 77-80		n 84
	vsubpd	zmm0, zmm0, zmm2			;; R1-R9							; 77-80		n 85

	zfmsubpd zmm2, zmm8, zmm25, zmm1		;; R3/sine * sine - R7						; 78-81		n 86
	zfmaddpd zmm8, zmm8, zmm25, zmm1		;; R3/sine * sine + R7						; 78-81		n 89

	zfmaddpd zmm1, zmm5, zmm25, zmm14		;; I3/sine * sine + I7						; 79-82		n 86
	zfmsubpd zmm5, zmm5, zmm25, zmm14		;; I3/sine * sine - I7						; 79-82		n 90

	zfmaddpd zmm14, zmm16, zmm24, zmm18		;; R2/sine * sine + R8						; 80-83		n 87
	zfmsubpd zmm16, zmm16, zmm24, zmm18		;; R2/sine * sine - R8						; 80-83		n 91

	zfmsubpd zmm18, zmm3, zmm24, zmm4		;; I2/sine * sine - I8						; 81-84		n 88
	zfmaddpd zmm3, zmm3, zmm24, zmm4		;; I2/sine * sine + I8						; 81-84		n 92

	zfmaddpd zmm4, zmm19, zmm9, zmm6		;; R4/sine * sine + R6						; 82-85		n 87
	zfmsubpd zmm19, zmm19, zmm9, zmm6		;; R4/sine * sine - R6						; 82-85		n 92

	zfmsubpd zmm6, zmm12, zmm9, zmm17		;; I4/sine * sine - I6						; 83-86		n 88
	zfmaddpd zmm12, zmm12, zmm9, zmm17		;; I4/sine * sine + I6						; 83-86		n 91

	zfmaddpd zmm17, zmm10, zmm7, zmm20		;; r1++ = (r1+r9) + R5/sine * sine				; 84-87		n 89
	zfnmaddpd zmm10, zmm10, zmm7, zmm20		;; r1+- = (r1+r9) - R5/sine * sine				; 84-87		n 90

	zfmaddpd zmm20, zmm15, zmm7, zmm0		;; r1-+ = (r1-r9) + I5/sine * sine				; 85-88		n 96
	zfnmaddpd zmm15, zmm15, zmm7, zmm0		;; r1-- = (r1-r9) - I5/sine * sine				; 85-88		n 98

	vaddpd	zmm0, zmm2, zmm1			;; r3-+ = (r3-r7) + (i3+i7)					; 86-89		n 96
	vsubpd	zmm2, zmm2, zmm1			;; r3-- = (r3-r7) - (i3+i7)					; 86-89		n 98

	vaddpd	zmm1, zmm14, zmm4			;; r2++ = (r2+r8) + (r4+r6)					; 87-90		n 93
	vsubpd	zmm14, zmm14, zmm4			;; r2+- = (r2+r8) - (r4+r6)					; 87-90		n 95

	vsubpd	zmm4, zmm18, zmm6			;; i2-- = (i2-i8) - (i4-i6)					; 88-91		n 94
	vaddpd	zmm18, zmm18, zmm6			;; i2-+ = (i2-i8) + (i4-i6)					; 88-91		n 95

	vaddpd	zmm6, zmm17, zmm8			;; r1+++ = (r1++) + (r3+r7)					; 89-92		n 93
	vsubpd	zmm17, zmm17, zmm8			;; r1++- = (r1++) - (r3+r7)					; 89-92		n 94

	vaddpd	zmm8, zmm10, zmm5			;; r1+-+ = (r1+-) + (i3-i7)					; 90-93		n 100
	vsubpd	zmm10, zmm10, zmm5			;; r1+-- = (r1+-) - (i3-i7)					; 90-93		n 101

	vaddpd	zmm5, zmm16, zmm12			;; r2-+ = (r2-r8) + (i4+i6)					; 91-94		n 97
	vsubpd	zmm16, zmm16, zmm12			;; r2-- = (r2-r8) - (i4+i6)					; 91-94		n 99

	vaddpd	zmm12, zmm3, zmm19			;; i2++ = (i2+i8) + (r4-r6)					; 92-95		n 97
	vsubpd	zmm3, zmm3, zmm19			;; i2+- = (i2+i8) - (r4-r6)					; 92-95		n 99

	vaddpd	zmm19, zmm6, zmm1			;; R1 = (r1+++) + (r2++)					; 93-96
	vsubpd	zmm6, zmm6, zmm1			;; R9 = (r1+++) - (r2++)					; 93-96

	vaddpd	zmm1, zmm17, zmm4			;; R5  = (r1++-) + (i2--)					; 94-97
	vsubpd	zmm17, zmm17, zmm4			;; R13 = (r1++-) - (i2--)					; 94-97

	vaddpd	zmm4, zmm14, zmm18			;; r2+-+ = (r2+-) + (i2-+)					; 95-98		n 100
	vsubpd	zmm14, zmm14, zmm18			;; r2+-- = (r2+-) - (i2-+)					; 95-98		n 101

	zfmaddpd zmm18, zmm0, zmm21, zmm20		;; r2_10o = (r1-+) + .707(r3-+)					; 96-99		n 102
	zfnmaddpd zmm0, zmm0, zmm21, zmm20		;; r6_14o = (r1-+) - .707(r3-+)					; 96-99		n 103

	zfmaddpd zmm20, zmm5, zmm11, zmm12		;; r2_10e/.383 = .924/.383(r2-+) + (i2++)			; 97-100	n 102
	zfmsubpd zmm12, zmm12, zmm11, zmm5		;; r6_14e/.383 = .924/.383(i2++) - (r2-+)			; 97-100	n 103
	zstore	[srcreg], zmm19				;; Save R1							; 97

	zfnmaddpd zmm5, zmm2, zmm21, zmm15		;; r4_12o = (r1--) - .707(r3--)					; 98-101	n 104
	zfmaddpd zmm2, zmm2, zmm21, zmm15		;; r8_16o = (r1--) + .707(r3--)					; 98-101	n 105
	zstore	[srcreg+64], zmm6			;; Save R9							; 97+1

	zfmaddpd zmm15, zmm3, zmm11, zmm16		;; r4_12e/.383 = .924/.383(i2+-) + (r2--)			; 99-102	n 104
	zfnmaddpd zmm16, zmm16, zmm11, zmm3		;; r8_16e/.383 = (i2+-) - .924/.383(r2--)			; 99-102	n 105
	zstore	[srcreg+d4], zmm1			;; Save R5							; 98+1

	zfmaddpd zmm3, zmm4, zmm21, zmm8		;; R3  = (r1+-+) + .707(r2+-+)					; 100-103
	zfnmaddpd zmm4, zmm4, zmm21, zmm8		;; R11 = (r1+-+) - .707(r2+-+)					; 100-103
	zstore	[srcreg+d4+64], zmm17			;; Save R13							; 98+2

	zfnmaddpd zmm8, zmm14, zmm21, zmm10		;; R7  = (r1+--) - .707(r2+--)					; 101-104
	zfmaddpd zmm14, zmm14, zmm21, zmm10		;; R15 = (r1+--) + .707(r2+--)					; 101-104

	zfmaddpd zmm10, zmm20, zmm23, zmm18		;; R2  = r2_10o + .383*r2_10e					; 102-105
	zfnmaddpd zmm20, zmm20, zmm23, zmm18		;; R10 = r2_10o - .383*r2_10e					; 102-105

	zfmaddpd zmm18, zmm12, zmm23, zmm0		;; R6  = r6_14o + .383*r6_14e					; 103-106
	zfnmaddpd zmm12, zmm12, zmm23, zmm0		;; R14 = r6_14o - .383*r6_14e					; 103-106

	zfmaddpd zmm0, zmm15, zmm23, zmm5		;; R4  = r4_12o + .383*r4_12e					; 104-107
	zfnmaddpd zmm15, zmm15, zmm23, zmm5		;; R12 = r4_12o - .383*r4_12e					; 104-107
	zstore	[srcreg+d2], zmm3			;; Save R3							; 104

	zfmaddpd zmm5, zmm16, zmm23, zmm2		;; R8  = r8_16o + .383*r8_16e					; 105-108
	zfnmaddpd zmm16, zmm16, zmm23, zmm2		;; R16 = r8_16o - .383*r8_16e					; 105-108

 	zstore	[srcreg+d2+64], zmm4			;; Save R11							; 104+1
	zstore	[srcreg+d4+d2], zmm8			;; Save R7							; 105+1
	zstore	[srcreg+d4+d2+64], zmm14		;; Save R15							; 105+2
	zstore	[srcreg+d1], zmm10			;; Save R2							; 106+2
	zstore	[srcreg+d1+64], zmm20			;; Save R10							; 106+3
	zstore	[srcreg+d4+d1], zmm18			;; Save R6							; 107+3
	zstore	[srcreg+d4+d1+64], zmm12		;; Save R14							; 107+4
	zstore	[srcreg+d2+d1], zmm0			;; Save R4							; 108+4
	zstore	[srcreg+d2+d1+64], zmm15		;; Save R12							; 108+5
	zstore	[srcreg+d4+d2+d1], zmm5			;; Save R8							; 109+5
	zstore	[srcreg+d4+d2+d1+64], zmm16		;; Save R16							; 109+6
	bump	srcreg, srcinc
	ENDM


zr8_eight_complex_fft_final_preload MACRO
	vbroadcastsd zmm31, ZMM_SQRTHALF
	ENDM
zr8_eight_complex_fft_final MACRO srcreg,srcinc,d1,d2,d4
	vmovapd	zmm15, [srcreg]			;; R1
	vmovapd	zmm0, [srcreg+d4]		;; R5
	vaddpd	zmm11, zmm15, zmm0		;; R1+R5 (new R1)				; 1-4		n 13
	vsubpd	zmm15, zmm15, zmm0		;; R1-R5 (new R5)				; 1-4		n 14

	vmovapd	zmm14, [srcreg+d2]		;; R3
	vmovapd	zmm0, [srcreg+d4+d2]		;; R7
	vaddpd	zmm5, zmm14, zmm0		;; R3+R7 (new R3)				; 2-5		n 13
	vsubpd	zmm14, zmm14, zmm0		;; R3-R7 (new R7)				; 2-5		n 17

	vmovapd	zmm22, [srcreg+d1]		;; R2
	vmovapd	zmm0, [srcreg+d4+d1]		;; R6
	vaddpd	zmm2, zmm22, zmm0		;; R2+R6 (new R2)				; 3-6		n 11
	vsubpd	zmm22, zmm22, zmm0		;; R2-R6 (new R6)				; 3-6		n 9

	vmovapd	zmm1, [srcreg+d2+d1]		;; R4
	vmovapd	zmm0, [srcreg+d4+d2+d1]		;; R8
	vaddpd	zmm8, zmm1, zmm0		;; R4+R8 (new R4)				; 4-7		n 11
	vsubpd	zmm1, zmm1, zmm0		;; R4-R8 (new R8)				; 4-7		n 9

	vmovapd	zmm20, [srcreg+64]		;; I1
	vmovapd	zmm0, [srcreg+d4+64]		;; I5
	vaddpd	zmm7, zmm20, zmm0		;; I1+I5 (new I1)				; 5-8		n 16
	vsubpd	zmm20, zmm20, zmm0		;; I1-I5 (new I5)				; 5-8		n 15

	vmovapd	zmm0, [srcreg+d2+64]		;; I3
	vmovapd	zmm19, [srcreg+d4+d2+64]	;; I7
	vaddpd	zmm10, zmm0, zmm19		;; I3+I7 (new I3)				; 6-9		n 16
	vsubpd	zmm0, zmm0, zmm19		;; I3-I7 (new I7)				; 6-9		n 18

	vmovapd	zmm21, [srcreg+d1+64]		;; I2
	vmovapd	zmm19, [srcreg+d4+d1+64]	;; I6
	vaddpd	zmm17, zmm21, zmm19		;; I2+I6 (new I2)				; 7-10		n 12
	vsubpd	zmm21, zmm21, zmm19		;; I2-I6 (new I6)				; 7-10		n 10

	vmovapd	zmm9, [srcreg+d2+d1+64]		;; I4
	vmovapd	zmm19, [srcreg+d4+d2+d1+64]	;; I8
	vaddpd	zmm3, zmm9, zmm19		;; I4+I8 (new I4)				; 8-11		n 12
	vsubpd	zmm9, zmm9, zmm19		;; I4-I8 (new I8)				; 8-11		n 10

	vaddpd	zmm19, zmm11, zmm5		;; R1 + R3 (newer R1)					; 63-66		n 73
	vsubpd	zmm11, zmm11, zmm5		;; R1 - R3 (newer R3)					; 64-67		n 84

	vaddpd	zmm5, zmm2, zmm8		;; R2 + R4 (newer R2)					; 65-68		n 73
	vsubpd	zmm2, zmm2, zmm8		;; R2 - R4 (newer R4)					; 66-69		n 86

	vaddpd	zmm8, zmm7, zmm10		;; I1 + I3 (newer I1)					; 67-70		n 73
	vsubpd	zmm7, zmm7, zmm10		;; I1 - I3 (newer I3)					; 68-71		n 86

	vaddpd	zmm16, zmm17, zmm3		;; I2 + I4 (newer I2)					; 69-72		n 73
	vsubpd	zmm17, zmm17, zmm3		;; I2 - I4 (newer I4)					; 70-73		n 84

	vaddpd	zmm13, zmm19, zmm5		;; R1 + R2 (final R1)					; 71-74
	vsubpd	zmm19, zmm19, zmm5		;; R1 - R2 (final R2)					; 72-75

	vaddpd	zmm5, zmm8, zmm16		;; I1 + I2 (final I1)					; 73-76
	vsubpd	zmm8, zmm8, zmm16		;; I1 - I2 (final I2)					; 74-77

	vsubpd	zmm12, zmm22, zmm9		;; R6 - I8 (new2 R6)					; 75-78		n 82
	vaddpd	zmm3, zmm22, zmm9		;; R6 + I8 (new2 R8)					; 76-79		n 83
	zstore	[srcreg], zmm13			;; Save R1						; 75
	zstore	[srcreg+d1], zmm19		;; Save R2						; 76

	vaddpd	zmm9, zmm21, zmm1		;; I6 + R8 (new2 I6)					; 77-80		n 82
	vsubpd	zmm16, zmm21, zmm1		;; I6 - R8 (new2 I8)					; 78-81		n 83
	zstore	[srcreg+64], zmm5		;; Save I1						; 77
	zstore	[srcreg+d1+64], zmm8		;; Save I2						; 78

	vaddpd	zmm6, zmm20, zmm14		;; I5 + R7 (newer I5)					; 79-82		n 86
	vsubpd	zmm20, zmm20, zmm14		;; I5 - R7 (newer I7)					; 80-83		n 87

	vsubpd	zmm18, zmm15, zmm0		;; R5 - I7 (newer R5)					; 81-83		n 88
	vaddpd	zmm15, zmm15, zmm0		;; R5 + I7 (newer R7)					; 81-84		n 89

	vaddpd	zmm4, zmm12, zmm9		;; I6 = R6 + I6 (newer I6/SQRTHALF)			; 82-85		n 86
	vsubpd	zmm12, zmm12, zmm9		;; R6 = R6 - I6 (newer R6/SQRTHALF)			; 82-85		n 88

	vsubpd	zmm10, zmm3, zmm16		;; R8 = R8 - I8 (newer R8/SQRTHALF)			; 83-86		n 87
	vaddpd	zmm3, zmm3, zmm16		;; I8 = R8 + I8 (newer I8/SQRTHALF)			; 83-86		n 89

	vsubpd	zmm14, zmm11, zmm17		;; R3 - I4 (final R3)					; 84-87
	vaddpd	zmm11, zmm11, zmm17		;; R3 + I4 (final R4)					; 84-87

	vaddpd	zmm0, zmm7, zmm2		;; I3 + R4 (final I3)					; 85-88
	vsubpd	zmm7, zmm7, zmm2		;; I3 - R4 (final I4)					; 85-88

	zfmaddpd zmm9, zmm4, zmm31, zmm6	;; I5 + I6 * SQRTHALF (final I5)			; 86-89
	zfnmaddpd zmm4, zmm4, zmm31, zmm6	;; I5 - I6 * SQRTHALF (final I6)			; 86-89

	zfmaddpd zmm2, zmm10, zmm31, zmm20	;; I7 + R8 * SQRTHALF (final I7)			; 87-90
	zfnmaddpd zmm10, zmm10, zmm31, zmm20	;; I7 - R8 * SQRTHALF (final I8)			; 87-90

	zfmaddpd zmm6, zmm12, zmm31, zmm18	;; R5 + R6 * SQRTHALF (final R5)			; 88-91
	zfnmaddpd zmm12, zmm12, zmm31, zmm18	;; R5 - R6 * SQRTHALF (final R6)			; 88-91
	zstore	[srcreg+d2], zmm14		;; Save R3						; 88

	zfnmaddpd zmm16, zmm3, zmm31, zmm15	;; R7 - I8 * SQRTHALF (final R7)			; 89-92
	zfmaddpd zmm3, zmm3, zmm31, zmm15	;; R7 + I8 * SQRTHALF (final R8)			; 89-92
	zstore	[srcreg+d2+d1], zmm11		;; Save R4						; 88+1

	zstore	[srcreg+d2+64], zmm0		;; Save I3						; 89+1
	zstore	[srcreg+d2+d1+64], zmm7		;; Save I4						; 89+2
	zstore	[srcreg+d4+64], zmm9		;; Save I5						; 90+2
	zstore	[srcreg+d4+d1+64], zmm4		;; Save I6						; 90+3
	zstore	[srcreg+d4+d2+64], zmm2		;; Save I7						; 91+3
	zstore	[srcreg+d4+d2+d1+64], zmm10	;; Save I8						; 91+4
	zstore	[srcreg+d4], zmm6		;; Save R5						; 92+4
	zstore	[srcreg+d4+d1], zmm12		;; Save R6						; 92+5
	zstore	[srcreg+d4+d2], zmm16		;; Save R7						; 93+5
	zstore	[srcreg+d4+d2+d1], zmm3		;; Save R8						; 93+6
	bump	srcreg, srcinc
	ENDM


zr8_eight_complex_with_square_preload MACRO
	vbroadcastsd zmm31, ZMM_SQRTHALF
	vbroadcastsd zmm30, ZMM_TWO
	ENDM
zr8_eight_complex_with_square MACRO srcreg,srcinc,d1,d2,d4,maxrpt,L1pt,L1pd
	LOCAL	orig, back_to_orig

	vmovapd	zmm15, [srcreg]			;; R1
	vmovapd	zmm0, [srcreg+d4]		;; R5
	vaddpd	zmm11, zmm15, zmm0		;; R1+R5 (new R1)				; 1-4		n 9
	vsubpd	zmm15, zmm15, zmm0		;; R1-R5 (new R5)				; 1-4		n 17

	vmovapd	zmm14, [srcreg+d2]		;; R3
	vmovapd	zmm0, [srcreg+d4+d2]		;; R7
	vaddpd	zmm5, zmm14, zmm0		;; R3+R7 (new R3)				; 2-5		n 9
	vsubpd	zmm14, zmm14, zmm0		;; R3-R7 (new R7)				; 2-5		n 16

	vmovapd	zmm3, [srcreg+d1]		;; R2
	vmovapd	zmm0, [srcreg+d4+d1]		;; R6
	vaddpd	zmm2, zmm3, zmm0		;; R2+R6 (new R2)				; 3-6		n 10
	vsubpd	zmm3, zmm3, zmm0		;; R2-R6 (new R6)				; 3-6		n 13

	vmovapd	zmm1, [srcreg+d2+d1]		;; R4
	vmovapd	zmm0, [srcreg+d4+d2+d1]		;; R8
	vaddpd	zmm8, zmm1, zmm0		;; R4+R8 (new R4)				; 4-7		n 10
	vsubpd	zmm1, zmm1, zmm0		;; R4-R8 (new R8)				; 4-7		n 12

	vmovapd	zmm20, [srcreg+64]		;; I1
	vmovapd	zmm0, [srcreg+d4+64]		;; I5
	vaddpd	zmm7, zmm20, zmm0		;; I1+I5 (new I1)				; 5-8		n 11
	vsubpd	zmm20, zmm20, zmm0		;; I1-I5 (new I5)				; 5-8		n 16

	vmovapd	zmm0, [srcreg+d2+64]		;; I3
	vmovapd	zmm19, [srcreg+d4+d2+64]	;; I7
	vaddpd	zmm10, zmm0, zmm19		;; I3+I7 (new I3)				; 6-9		n 11
	vsubpd	zmm0, zmm0, zmm19		;; I3-I7 (new I7)				; 6-9		n 17

	vmovapd	zmm16, [srcreg+d1+64]		;; I2
	vmovapd	zmm19, [srcreg+d4+d1+64]	;; I6
	vaddpd	zmm17, zmm16, zmm19		;; I2+I6 (new I2)				; 7-10		n 14
	vsubpd	zmm16, zmm16, zmm19		;; I2-I6 (new I6)				; 7-10		n 12

	vmovapd	zmm23, [srcreg+d2+d1+64]	;; I4
	vmovapd	zmm19, [srcreg+d4+d2+d1+64]	;; I8
	vaddpd	zmm22, zmm23, zmm19		;; I4+I8 (new I4)				; 8-11		n 14
	vsubpd	zmm23, zmm23, zmm19		;; I4-I8 (new I8)				; 8-11		n 13

	L1prefetchw srcreg+L1pd, L1pt
	vaddpd	zmm19, zmm11, zmm5		;; R1 + R3 (newer R1)				; 9-12		n 15
	vsubpd	zmm11, zmm11, zmm5		;; R1 - R3 (newer R3)				; 9-12		n 21

	L1prefetchw srcreg+d4+L1pd, L1pt
	vaddpd	zmm5, zmm2, zmm8		;; R2 + R4 (newer R2)				; 10-13		n 15
	vsubpd	zmm2, zmm2, zmm8		;; R2 - R4 (newer R4)				; 10-13		n 22

	L1prefetchw srcreg+d2+L1pd, L1pt
	vaddpd	zmm8, zmm7, zmm10		;; I1 + I3 (newer I1)				; 11-14		n 20
	vsubpd	zmm7, zmm7, zmm10		;; I1 - I3 (newer I3)				; 11-14		n 22

	L1prefetchw srcreg+d4+d2+L1pd, L1pt
	vaddpd	zmm9, zmm16, zmm1		;; I6 + R8 (new2 I6)				; 12-15		n 18
	vsubpd	zmm16, zmm16, zmm1		;; I6 - R8 (new2 I8)				; 12-15		n 19

	L1prefetchw srcreg+d1+L1pd, L1pt
	vsubpd	zmm12, zmm3, zmm23		;; R6 - I8 (new2 R6)				; 13-16		n 18
	vaddpd	zmm3, zmm3, zmm23		;; R6 + I8 (new2 R8)				; 13-16		n 19

	L1prefetchw srcreg+d4+d1+L1pd, L1pt
	vaddpd	zmm21, zmm17, zmm22		;; I2 + I4 (newer I2)				; 14-17		n 20
	vsubpd	zmm17, zmm17, zmm22		;; I2 - I4 (newer I4)				; 14-17		n 21

	L1prefetchw srcreg+d2+d1+L1pd, L1pt
	vaddpd	zmm13, zmm19, zmm5		;; R1 + R2 (final R1)				; 15-18		n 27
	vsubpd	zmm19, zmm19, zmm5		;; R1 - R2 (final R2)				; 15-18		n 33

	L1prefetchw srcreg+d4+d2+d1+L1pd, L1pt
	vaddpd	zmm6, zmm20, zmm14		;; I5 + R7 (newer I5)				; 16-19		n 23
	vsubpd	zmm20, zmm20, zmm14		;; I5 - R7 (newer I7)				; 16-19		n 24

	L1prefetchw srcreg+64+L1pd, L1pt
	vsubpd	zmm18, zmm15, zmm0		;; R5 - I7 (newer R5)				; 17-20		n 25
	vaddpd	zmm15, zmm15, zmm0		;; R5 + I7 (newer R7)				; 17-20		n 26

	L1prefetchw srcreg+d4+64+L1pd, L1pt
	vaddpd	zmm4, zmm12, zmm9		;; I6 = R6 + I6 (newer I6/SQRTHALF)		; 18-21		n 23
	vsubpd	zmm12, zmm12, zmm9		;; R6 = R6 - I6 (newer R6/SQRTHALF)		; 18-21		n 25

	L1prefetchw srcreg+d2+64+L1pd, L1pt
	vsubpd	zmm10, zmm3, zmm16		;; R8 = R8 - I8 (newer R8/SQRTHALF)		; 19-22		n 24
	vaddpd	zmm3, zmm3, zmm16		;; I8 = R8 + I8 (newer I8/SQRTHALF)		; 19-22		n 26

	L1prefetchw srcreg+d4+d2+64+L1pd, L1pt
	vaddpd	zmm5, zmm8, zmm21		;; I1 + I2 (final I1)				; 20-23		n 27
	vsubpd	zmm8, zmm8, zmm21		;; I1 - I2 (final I2)				; 20-23		n 37

	L1prefetchw srcreg+d1+64+L1pd, L1pt
	vsubpd	zmm14, zmm11, zmm17		;; R3 - I4 (final R3)				; 21-24		n 28
	vaddpd	zmm11, zmm11, zmm17		;; R3 + I4 (final R4)				; 21-24		n 34

	L1prefetchw srcreg+d4+d1+64+L1pd, L1pt
	vaddpd	zmm0, zmm7, zmm2		;; I3 + R4 (final I3)				; 22-25		n 28
	vsubpd	zmm7, zmm7, zmm2		;; I3 - R4 (final I4)				; 22-25		n 38

	L1prefetchw srcreg+d2+d1+64+L1pd, L1pt
	zfmaddpd zmm9, zmm4, zmm31, zmm6	;; I5 + I6 * SQRTHALF (final I5)		; 23-26		n 29
	zfnmaddpd zmm4, zmm4, zmm31, zmm6	;; I5 - I6 * SQRTHALF (final I6)		; 23-26		n 31

	L1prefetchw srcreg+d4+d2+d1+64+L1pd, L1pt
	zfmaddpd zmm2, zmm10, zmm31, zmm20	;; I7 + R8 * SQRTHALF (final I7)		; 24-27		n 30
	zfnmaddpd zmm10, zmm10, zmm31, zmm20	;; I7 - R8 * SQRTHALF (final I8)		; 24-27		n 32

	zfmaddpd zmm6, zmm12, zmm31, zmm18	;; R5 + R6 * SQRTHALF (final R5)		; 25-28		n 29
	zfnmaddpd zmm12, zmm12, zmm31, zmm18	;; R5 - R6 * SQRTHALF (final R6)		; 25-28		n 35

	zfnmaddpd zmm16, zmm3, zmm31, zmm15	;; R7 - I8 * SQRTHALF (final R7)		; 26-29		n 30
	zfmaddpd zmm3, zmm3, zmm31, zmm15	;; R7 + I8 * SQRTHALF (final R8)		; 26-29		n 36

	cmp	mul4_opcode, 0			;; See if we need to do more than the original type-2 FFT squaring
	je	short orig
	call	zcomplex_square_opcode		;; Handle more difficult cases --- exact same code as zr64.mac!
	jmp	back_to_orig
orig:
	;; Square the complex numbers
	vmulpd	zmm1, zmm13, zmm13		;; R1 * R1					; 27-30		n 31
	vmulpd	zmm13, zmm13, zmm5		;; R1 * I1 (I1/2)				; 27-30		n 37

	vmulpd	zmm15, zmm14, zmm14		;; R3 * R3					; 28-31		n 32
	vmulpd	zmm14, zmm14, zmm0		;; R3 * I3 (I3/2)				; 28-31		n 39

	vmulpd	zmm17, zmm9, zmm9		;; I5 * I5					; 29-32		n 33
	vmulpd	zmm9, zmm9, zmm6		;; I5 * R5 (I5/2)				; 29-32		n 35

 	vmulpd	zmm18, zmm2, zmm2		;; I7 * I7					; 30-33		n 34
	vmulpd	zmm2, zmm2, zmm16		;; I7 * R7 (I7/2)				; 30-33		n 40

	vmulpd	zmm20, zmm4, zmm4		;; I6 * I6					; 31-34		n 36
	zfnmaddpd zmm5, zmm5, zmm5, zmm1	;; R1^2 - I1 * I1 (R1)				; 31-34		n 44

	vmulpd	zmm1, zmm10, zmm10		;; I8 * I8					; 32-35		n 36
	zfnmaddpd zmm0, zmm0, zmm0, zmm15	;; R3^2 - I3 * I3 (R3)				; 32-35		n 45

	vmulpd	zmm15, zmm19, zmm19		;; R2 * R2					; 33-36		n 38
	zfmsubpd zmm6, zmm6, zmm6, zmm17	;; R5 * R5 - I5^2 (R5)				; 33-36		n 41

	vmulpd	zmm17, zmm11, zmm11		;; R4 * R4					; 34-37		n 38
	zfmsubpd zmm16, zmm16, zmm16, zmm18	;; R7 * R7 - I7^2 (R7)				; 34-37		n 42

	zfmaddpd zmm18, zmm4, zmm12, zmm9	;; I5/2 + I6 * R6 (new I5/2)			; 35-38		n 43
	zfnmaddpd zmm4, zmm4, zmm12, zmm9	;; I5/2 - I6 * R6 (new I6/2)			; 35-38		n 49

	zfmsubpd zmm12, zmm12, zmm12, zmm20	;; R6 * R6 - I6^2 (R6)				; 36-39		n 41
	zfmsubpd zmm1, zmm3, zmm3, zmm1		;; R8 * R8 - I8^2 (R8)				; 36-39		n 42

	zfmaddpd zmm9, zmm8, zmm19, zmm13	;; I1/2 + I2 * R2 (new I1/2)			; 37-40		n 46
	zfnmaddpd zmm13, zmm8, zmm19, zmm13	;; I1/2 - I2 * R2 (new I2/2)			; 37-40		n 52

	zfnmaddpd zmm15, zmm8, zmm8, zmm15	;; R2^2 - I2 * I2 (R2)				; 38-41		n 44
	zfnmaddpd zmm17, zmm7, zmm7, zmm17	;; R4^2 - I4 * I4 (R4)				; 38-41		n 45

	zfmaddpd zmm8, zmm7, zmm11, zmm14	;; I3/2 + I4 * R4 (new I3/2)			; 39-42		n 46
	zfnmaddpd zmm14, zmm7, zmm11, zmm14	;; I3/2 - I4 * R4 (new R4/2)			; 39-42		n 53

	zfmaddpd zmm7, zmm10, zmm3, zmm2	;; I7/2 + I8 * R8 (new I7/2)			; 40-43		n 48
	zfnmaddpd zmm2, zmm10, zmm3, zmm2	;; I7/2 - I8 * R8 (new R8/2)			; 40-43		n 50

back_to_orig:
	vaddpd	zmm3, zmm6, zmm12		;; R5 + R6 (new R5)				; 41-44		n 47
	vsubpd	zmm6, zmm6, zmm12		;; R5 - R6 (new R6)				; 41-44		n 49

	vaddpd	zmm10, zmm1, zmm16		;; R8 + R7 (new R7)				; 42-45		n 47
	vsubpd	zmm1, zmm1, zmm16		;; R8 - R7 (new I8)				; 42-45		n 50

	vmulpd	zmm18, zmm18, zmm30		;; I5/2 * 2 (new I5)				; 43-46		n 48
	;;; unused slot, well maybe it will get used						; 43-46	

	vaddpd	zmm11, zmm5, zmm15		;; R1 + R2 (new R1)				; 44-47		n 51
	vsubpd	zmm5, zmm5, zmm15		;; R1 - R2 (new R2)				; 44-47		n 53

 	vaddpd	zmm12, zmm17, zmm0		;; R4 + R3 (new R3)				; 45-48		n 51
	vsubpd	zmm17, zmm17, zmm0		;; R4 - R3 (new I4)				; 45-48		n 52

	vaddpd	zmm16, zmm9, zmm8		;; I1/2 + I3/2 (newer I1/2)			; 46-49		n 56
	vsubpd	zmm9, zmm9, zmm8		;; I1/2 - I3/2 (newer I3/2)			; 46-49		n 57

	vsubpd	zmm15, zmm10, zmm3		;; R7 - R5 (newer I7)				; 47-50		n 57
	vaddpd	zmm10, zmm10, zmm3		;; R7 + R5 (newer R5)				; 47-50		n 58

	zfmaddpd zmm0, zmm7, zmm30, zmm18	;; I5 + I7/2 * 2 (newer I5)			; 48-51		n 56
	zfnmaddpd zmm7, zmm7, zmm30, zmm18	;; I5 - I7/2 * 2 (newer R7)			; 48-51		n 59

	;; multiply R6/I6 by SQRTHALF - i*SQRTHALF
	;; multiply R8/I8 by SQRTHALF - i*SQRTHALF
	zfmsubpd zmm8, zmm4, zmm30, zmm6	;; I6 = I6/2 * 2 - R6				; 49-52		n 54
	zfnmaddpd zmm3, zmm2, zmm30, zmm1	;; I8 = I8 - R8/2 * 2				; 49-52		n 54

	zfmaddpd zmm4, zmm4, zmm30, zmm6	;; R6 = R6 + I6/2 * 2				; 50-53		n 55
	zfmaddpd zmm2, zmm2, zmm30, zmm1	;; R8 = R8/2 * 2 + I8				; 50-53		n 55

	vaddpd	zmm18, zmm11, zmm12		;; R1 + R3 (newer R1)				; 51-54		n 58
	vsubpd	zmm11, zmm11, zmm12		;; R1 - R3 (newer R3)				; 51-54		n 59

 	zfmaddpd zmm6, zmm13, zmm30, zmm17	;; I2/2 * 2 + I4 (newer I2)			; 52-55		n 60
	zfmsubpd zmm13, zmm13, zmm30, zmm17	;; I2/2 * 2 - I4 (newer I4)			; 52-55		n 62

	zfmaddpd zmm17, zmm14, zmm30, zmm5	;; R2 + R4/2 * 2 (newer R2)			; 53-56		n 61
	zfnmaddpd zmm14, zmm14, zmm30, zmm5	;; R2 - R4/2 * 2 (newer R4)			; 53-56		n 63

	vaddpd	zmm1, zmm8, zmm3		;; I6 + I8 (newer I6/SQRTHALF)			; 54-57		n 60
	vsubpd	zmm8, zmm8, zmm3		;; I6 - I8 (newer R8/SQRTHALF)			; 54-57		n 63

	vsubpd	zmm12, zmm2, zmm4		;; R8 - R6 (newer I8/SQRTHALF)			; 55-58		n 62
	vaddpd	zmm2, zmm2, zmm4		;; R8 + R6 (newer R6/SQRTHALF)			; 55-58		n 61

	zfmaddpd zmm3, zmm16, zmm30, zmm0	;; I1/2 * 2 + I5 (final I1)			; 56-59
	zfmsubpd zmm16, zmm16, zmm30, zmm0	;; I1/2 * 2 - I5 (final I5)			; 56-59

	zfmaddpd zmm4, zmm9, zmm30, zmm15	;; I3/2 * 2 + I7 (final I3)			; 57-60
	zfmsubpd zmm9, zmm9, zmm30, zmm15	;; I3/2 * 2 - I7 (final I7)			; 57-60

	vaddpd	zmm5, zmm18, zmm10		;; R1 + R5 (final R1)				; 58-61
	vsubpd	zmm18, zmm18, zmm10		;; R1 - R5 (final R5)				; 58-61

	vaddpd	zmm0, zmm11, zmm7		;; R3 + R7 (final R3)				; 59-62
	vsubpd	zmm11, zmm11, zmm7		;; R3 - R7 (final R7)				; 59-62

	zfmaddpd zmm15, zmm1, zmm31, zmm6	;; I2 + I6 * SQRTHALF (final I2)		; 60-63
	zfnmaddpd zmm1, zmm1, zmm31, zmm6	;; I2 - I6 * SQRTHALF (final I6)		; 60-63

	zfmaddpd zmm10, zmm2, zmm31, zmm17	;; R2 + R6 * SQRTHALF (final R2)		; 61-64
	zfnmaddpd zmm2, zmm2, zmm31, zmm17	;; R2 - R6 * SQRTHALF (final R6)		; 61-64

	zfmaddpd zmm7, zmm12, zmm31, zmm13	;; I4 + I8 * SQRTHALF (final I4)		; 62-65
	zfnmaddpd zmm12, zmm12, zmm31, zmm13	;; I4 - I8 * SQRTHALF (final I8)		; 62-65

	zfmaddpd zmm6, zmm8, zmm31, zmm14	;; R4 + R8 * SQRTHALF (final R4)		; 63-66
	zfnmaddpd zmm8, zmm8, zmm31, zmm14	;; R4 - R8 * SQRTHALF (final R8)		; 63-66

	zstore	[srcreg+r8], zmm5		;; Save R1						; 62
	zstore	[srcreg+r8+d4], zmm18		;; Save R5						; 62+1
	zstore	[srcreg+r8+d2+64], zmm4		;; Save I3						; 61+1
	zstore	[srcreg+r8+d4+d2+64], zmm9	;; Save I7						; 61+2
	zstore	[srcreg+r8+64], zmm3		;; Save I1						; 60
	zstore	[srcreg+r8+d4+64], zmm16	;; Save I5						; 60+1
	zstore	[srcreg+r8+d2], zmm0		;; Save R3						; 63+1
	zstore	[srcreg+r8+d4+d2], zmm11	;; Save R7						; 63+2
	zstore	[srcreg+r8+d1], zmm10		;; Save R2						; 65+2
	zstore	[srcreg+r8+d4+d1], zmm2		;; Save R6						; 65+3
	zstore	[srcreg+r8+d2+d1], zmm6		;; Save R4						; 67+3
	zstore	[srcreg+r8+d4+d2+d1], zmm8	;; Save R8						; 67+4
	zstore	[srcreg+r8+d1+64], zmm15	;; Save I2						; 64+5
	zstore	[srcreg+r8+d4+d1+64], zmm1	;; Save I6						; 64+5
	zstore	[srcreg+r8+d2+d1+64], zmm7	;; Save I4						; 66+5
	zstore	[srcreg+r8+d4+d2+d1+64], zmm12	;; Save I8						; 66+6
	bump	srcreg, srcinc
	ENDM


zr8_eight_complex_with_mult_preload MACRO
	vbroadcastsd zmm31, ZMM_SQRTHALF
	ENDM
zr8_eight_complex_with_mult MACRO srcreg,srcinc,d1,d2,d4
	LOCAL	orig, back_to_orig

	vmovapd	zmm15, [srcreg]			;; R1
	vmovapd	zmm0, [srcreg+d4]		;; R5
	vaddpd	zmm11, zmm15, zmm0		;; R1+R5 (new R1)				; 1-4		n 13
	vsubpd	zmm15, zmm15, zmm0		;; R1-R5 (new R5)				; 1-4		n 14

	vmovapd	zmm14, [srcreg+d2]		;; R3
	vmovapd	zmm0, [srcreg+d4+d2]		;; R7
	vaddpd	zmm5, zmm14, zmm0		;; R3+R7 (new R3)				; 2-5		n 13
	vsubpd	zmm14, zmm14, zmm0		;; R3-R7 (new R7)				; 2-5		n 17

	vmovapd	zmm22, [srcreg+d1]		;; R2
	vmovapd	zmm0, [srcreg+d4+d1]		;; R6
	vaddpd	zmm2, zmm22, zmm0		;; R2+R6 (new R2)				; 3-6		n 11
	vsubpd	zmm22, zmm22, zmm0		;; R2-R6 (new R6)				; 3-6		n 9

	vmovapd	zmm1, [srcreg+d2+d1]		;; R4
	vmovapd	zmm0, [srcreg+d4+d2+d1]		;; R8
	vaddpd	zmm8, zmm1, zmm0		;; R4+R8 (new R4)				; 4-7		n 11
	vsubpd	zmm1, zmm1, zmm0		;; R4-R8 (new R8)				; 4-7		n 9

	vmovapd	zmm20, [srcreg+64]		;; I1
	vmovapd	zmm0, [srcreg+d4+64]		;; I5
	vaddpd	zmm7, zmm20, zmm0		;; I1+I5 (new I1)				; 5-8		n 16
	vsubpd	zmm20, zmm20, zmm0		;; I1-I5 (new I5)				; 5-8		n 15

	vmovapd	zmm0, [srcreg+d2+64]		;; I3
	vmovapd	zmm19, [srcreg+d4+d2+64]	;; I7
	vaddpd	zmm10, zmm0, zmm19		;; I3+I7 (new I3)				; 6-9		n 16
	vsubpd	zmm0, zmm0, zmm19		;; I3-I7 (new I7)				; 6-9		n 18

	vmovapd	zmm21, [srcreg+d1+64]		;; I2
	vmovapd	zmm19, [srcreg+d4+d1+64]	;; I6
	vaddpd	zmm17, zmm21, zmm19		;; I2+I6 (new I2)				; 7-10		n 12
	vsubpd	zmm21, zmm21, zmm19		;; I2-I6 (new I6)				; 7-10		n 10

	vmovapd	zmm9, [srcreg+d2+d1+64]		;; I4
	vmovapd	zmm19, [srcreg+d4+d2+d1+64]	;; I8
	vaddpd	zmm3, zmm9, zmm19		;; I4+I8 (new I4)				; 8-11		n 12
	vsubpd	zmm9, zmm9, zmm19		;; I4-I8 (new I8)				; 8-11		n 10

	vaddpd	zmm19, zmm11, zmm5		;; R1 + R3 (newer R1)					; 63-66		n 73
	vsubpd	zmm11, zmm11, zmm5		;; R1 - R3 (newer R3)					; 64-67		n 84

	vaddpd	zmm5, zmm2, zmm8		;; R2 + R4 (newer R2)					; 65-68		n 71
	vsubpd	zmm2, zmm2, zmm8		;; R2 - R4 (newer R4)					; 66-69		n 86

	vaddpd	zmm8, zmm7, zmm10		;; I1 + I3 (newer I1)					; 67-70		n 73
	vsubpd	zmm7, zmm7, zmm10		;; I1 - I3 (newer I3)					; 68-71		n 85

	vaddpd	zmm16, zmm17, zmm3		;; I2 + I4 (newer I2)					; 69-72		n 73
	vsubpd	zmm17, zmm17, zmm3		;; I2 - I4 (newer I4)					; 70-73		n 84

	vaddpd	zmm13, zmm19, zmm5		;; R1 + R2 (final R1)					; 71-74		n 88
	vsubpd	zmm19, zmm19, zmm5		;; R1 - R2 (final R2)					; 72-75		n 89

	vaddpd	zmm5, zmm8, zmm16		;; I1 + I2 (final I1)					; 73-76		n 88
	vsubpd	zmm8, zmm8, zmm16		;; I1 - I2 (final I2)					; 74-77		n 89

	vsubpd	zmm12, zmm22, zmm9		;; R6 - I8 (new2 R6)					; 75-78		n 82
	vaddpd	zmm3, zmm22, zmm9		;; R6 + I8 (new2 R8)					; 76-79		n 83

	vaddpd	zmm9, zmm21, zmm1		;; I6 + R8 (new2 I6)					; 77-80		n 82
	vsubpd	zmm16, zmm21, zmm1		;; I6 - R8 (new2 I8)					; 78-81		n 83

	vaddpd	zmm6, zmm20, zmm14		;; I5 + R7 (newer I5)					; 79-82		n 86
	vsubpd	zmm20, zmm20, zmm14		;; I5 - R7 (newer I7)					; 80-83		n 87

	vsubpd	zmm18, zmm15, zmm0		;; R5 - I7 (newer R5)					; 81-83		n 91
	vaddpd	zmm15, zmm15, zmm0		;; R5 + I7 (newer R7)					; 81-84		n 98

	vaddpd	zmm4, zmm12, zmm9		;; I6 = R6 + I6 (newer I6/SQRTHALF)			; 82-85		n 86
	vsubpd	zmm12, zmm12, zmm9		;; R6 = R6 - I6 (newer R6/SQRTHALF)			; 82-85		n 91

	vsubpd	zmm10, zmm3, zmm16		;; R8 = R8 - I8 (newer R8/SQRTHALF)			; 83-86		n 87
	vaddpd	zmm3, zmm3, zmm16		;; I8 = R8 + I8 (newer I8/SQRTHALF)			; 83-86		n 98

	vsubpd	zmm14, zmm11, zmm17		;; R3 - I4 (final R3)					; 84-87		n 90
	vaddpd	zmm11, zmm11, zmm17		;; R3 + I4 (final R4)					; 84-87		n 95

	vaddpd	zmm0, zmm7, zmm2		;; I3 + R4 (final I3)					; 85-88		n 90
	vsubpd	zmm7, zmm7, zmm2		;; I3 - R4 (final I4)					; 85-88		n 95

	zfmaddpd zmm9, zmm4, zmm31, zmm6	;; I5 + I6 * SQRTHALF (final I5)			; 86-89		n 96
	zfnmaddpd zmm4, zmm4, zmm31, zmm6	;; I5 - I6 * SQRTHALF (final I6)			; 86-89		n 94

	zfmaddpd zmm2, zmm10, zmm31, zmm20	;; I7 + R8 * SQRTHALF (final I7)			; 87-90		n 102
	zfnmaddpd zmm10, zmm10, zmm31, zmm20	;; I7 - R8 * SQRTHALF (final I8)			; 87-90		n 103

	cmp	mul4_opcode, 0			;; See if we need to do more than the original type-3 FFT multiply
	je	short orig
	call	zcomplex_mult_opcode		;; Handle more difficult cases --- exact same code as zr64.mac!
	jmp	back_to_orig

orig:
	;; Multiply the complex numbers
	vmovapd	zmm30, [srcreg+rbp]		;; MemR1
	vmulpd	zmm20, zmm13, zmm30		;; A1 = R1 * MemR1					; 88-91		n 92
	vmulpd	zmm30, zmm5, zmm30		;; B1 = I1 * MemR1					; 88-91		n 92

	vmovapd	zmm21, [srcreg+rbp+d1]		;; MemR2
	vmulpd	zmm6, zmm19, zmm21		;; A2 = R2 * MemR2					; 89-92		n 93
	vmulpd	zmm21, zmm8, zmm21		;; B2 = I2 * MemR2					; 89-92		n 93

	vmovapd	zmm1, [srcreg+rbp+d2]		;; MemR3
	vmulpd	zmm17, zmm14, zmm1		;; A3 = R3 * MemR3					; 90-93		n 94
	vmulpd	zmm1, zmm0, zmm1		;; B3 = I3 * MemR3					; 90-93		n 94

	zfmaddpd zmm16, zmm12, zmm31, zmm18	;; R5 + R6 * SQRTHALF (final R5)			; 91-94		n 96
	zfnmaddpd zmm12, zmm12, zmm31, zmm18	;; R5 - R6 * SQRTHALF (final R6)			; 91-94		n 97

	vmovapd zmm18, [srcreg+rbp+64]		;; MemI1
	zfnmaddpd zmm20, zmm5, zmm18, zmm20	;; A1 - I1 * MemI1 (R1)					; 92-95		n 110
	zfmaddpd zmm18, zmm13, zmm18, zmm30	;; B1 + R1 * MemI1 (I1)					; 92-95		n 104

	vmovapd zmm30, [srcreg+rbp+d1+64]	;; MemI2
	zfnmaddpd zmm6, zmm8, zmm30, zmm6	;; A2 - I2 * MemI2 (R2)					; 93-96		n 110
	zfmaddpd zmm21, zmm19, zmm30, zmm21	;; B2 + R2 * MemI2 (I2)					; 93-96		n 104

	vmovapd zmm30, [srcreg+rbp+d2+64]	;; MemI3
	zfnmaddpd zmm17, zmm0, zmm30, zmm17	;; A3 - I3 * MemI3 (R3)					; 94-97		n 113
	zfmaddpd zmm1, zmm14, zmm30, zmm1	;; B3 + R3 * MemI3 (I3)					; 94-97		n 108

	vmovapd	zmm30, [srcreg+rbp+d2+d1]	;; MemR4
	vmulpd	zmm5, zmm11, zmm30		;; A4 = R4 * MemR4					; 95-98		n 99
	vmulpd	zmm13, zmm7, zmm30		;; B4 = I4 * MemR4					; 95-98		n 99

	vmovapd	zmm30, [srcreg+rbp+d4]		;; MemR5
	vmulpd	zmm8, zmm16, zmm30		;; A5 = R5 * MemR5					; 96-99		n 100
	vmulpd	zmm19, zmm9, zmm30		;; B5 = I5 * MemR5					; 96-99		n 100

	vmovapd	zmm30, [srcreg+rbp+d4+d1]	;; MemR6
	vmulpd	zmm0, zmm12, zmm30		;; A6 = R6 * MemR6					; 97-100	n 101
	vmulpd	zmm14, zmm4, zmm30		;; B6 = I6 * MemR6					; 97-100	n 101

	zfnmaddpd zmm30, zmm3, zmm31, zmm15	;; R7 - I8 * SQRTHALF (final R7)			; 98-101	n 102
	zfmaddpd zmm3, zmm3, zmm31, zmm15	;; R7 + I8 * SQRTHALF (final R8)			; 98-101	n 103

	vmovapd zmm15, [srcreg+rbp+d2+d1+64]	;; MemI4
	zfnmaddpd zmm5, zmm7, zmm15, zmm5	;; A4 - I4 * MemI4 (R4)					; 99-102	n 113
	zfmaddpd zmm13, zmm11, zmm15, zmm13	;; B4 + R4 * MemI4 (I4)					; 99-102	n 108

	vmovapd zmm15, [srcreg+rbp+d4+64]	;; MemI5
	zfnmaddpd zmm8, zmm9, zmm15, zmm8	;; A5 - I5 * MemI5 (R5)					; 100-103	n 109
	zfmaddpd zmm19, zmm16, zmm15, zmm19	;; B5 + R5 * MemI5 (I5)					; 100-103	n 105

	vmovapd zmm15, [srcreg+rbp+d4+d1+64]	;; MemI6
	zfnmaddpd zmm0, zmm4, zmm15, zmm0	;; A6 - I6 * MemI6 (R6)					; 101-104	n 109
	zfmaddpd zmm14, zmm12, zmm15, zmm14	;; B6 + R6 * MemI6 (I6)					; 101-104	n 105

	vmovapd	zmm15, [srcreg+rbp+d4+d2]	;; MemR7
	vmulpd	zmm7, zmm30, zmm15		;; A7 = R7 * MemR7					; 102-105	n 106
	vmulpd	zmm11, zmm2, zmm15		;; B7 = I7 * MemR7					; 102-105	n 106

	vmovapd	zmm15, [srcreg+rbp+d4+d2+d1]	;; MemR8
	vmulpd	zmm9, zmm3, zmm15		;; A8 = R8 * MemR8					; 103-106	n 107
	vmulpd	zmm16, zmm10, zmm15		;; B8 = I8 * MemR8					; 103-106	n 107

	vaddpd	zmm4, zmm18, zmm21		;; I1 + I2 (new I1)					; 104-107	n 114
	vsubpd	zmm18, zmm18, zmm21		;; I1 - I2 (new I2)					; 104-107	n 121

	vaddpd	zmm12, zmm19, zmm14		;; I5 + I6 (new I5)					; 105-108	n 116
	vsubpd	zmm14, zmm19, zmm14		;; I5 - I6 (new I6)					; 105-108	n 117

	vmovapd zmm15, [srcreg+rbp+d4+d2+64]	;; MemI7
	zfnmaddpd zmm7, zmm2, zmm15, zmm7	;; A7 - I7 * MemI7 (R7)					; 106-109	n 111
	zfmaddpd zmm11, zmm30, zmm15, zmm11	;; B7 + R7 * MemI7 (I7)					; 106-109	n 112

	vmovapd zmm15, [srcreg+rbp+d4+d2+d1+64]	;; MemI8
	zfnmaddpd zmm9, zmm10, zmm15, zmm9	;; A8 - I8 * MemI8 (R8)					; 107-110	n 111
	zfmaddpd zmm16, zmm3, zmm15, zmm16	;; B8 + R8 * MemI8 (I8)					; 107-110	n 112

back_to_orig:
	vaddpd	zmm21, zmm1, zmm13		;; I3 + I4 (new I3)					; 108-111	n 114
	vsubpd	zmm1, zmm1, zmm13		;; I3 - I4 (new R4)					; 108-111	n 127

	vaddpd	zmm13, zmm8, zmm0		;; R5 + R6 (new R5)					; 109-112	n 115
	vsubpd	zmm8, zmm8, zmm0		;; R5 - R6 (new R6)					; 109-112	n 117

	vaddpd	zmm10, zmm20, zmm6		;; R1 + R2 (new R1)					; 110-113	n 119
	vsubpd	zmm20, zmm20, zmm6		;; R1 - R2 (new R2)					; 110-113	n 127

	vaddpd	zmm3, zmm9, zmm7		;; R8 + R7 (new R7)					; 111-114	n 115
	vsubpd	zmm9, zmm9, zmm7		;; R8 - R7 (new I8)					; 111-114	n 118

	vaddpd	zmm15, zmm11, zmm16		;; I7 + I8 (new I7)					; 112-115	n 116
	vsubpd	zmm11, zmm11, zmm16		;; I7 - I8 (new R8)					; 112-115	n 118

 	vaddpd	zmm0, zmm5, zmm17		;; R4 + R3 (new R3)					; 113-116	n 119
	vsubpd	zmm5, zmm5, zmm17		;; R4 - R3 (new I4)					; 113-116	n 121

	vaddpd	zmm7, zmm4, zmm21		;; I1 + I3 (newer I1)					; 114-117	n 119
	vsubpd	zmm4, zmm4, zmm21		;; I1 - I3 (newer I3)					; 114-117	n 119

	vsubpd	zmm6, zmm3, zmm13		;; R7 - R5 (newer I7)					; 115-118	n 121
	vaddpd	zmm3, zmm3, zmm13		;; R7 + R5 (newer R5)					; 115-118	n 123

	vaddpd	zmm17, zmm12, zmm15		;; I5 + I7 (newer I5)					; 116-119	n 121
	vsubpd	zmm15, zmm12, zmm15		;; I5 - I7 (newer R7)					; 116-119	n 123

	;; multiply R6/I6 by SQRTHALF - i*SQRTHALF
	vsubpd	zmm21, zmm14, zmm8		;; I6 = I6 - R6						; 117-120	n 123
	vaddpd	zmm14, zmm8, zmm14		;; R6 = R6 + I6						; 117-120	n 125

	;; multiply R8/I8 by SQRTHALF - i*SQRTHALF
	vsubpd	zmm13, zmm9, zmm11		;; I8 = I8 - R8						; 118-121	n 123
	vaddpd	zmm11, zmm11, zmm9		;; R8 = R8 + I8						; 118-121	n 125

	vaddpd	zmm2, zmm10, zmm0		;; R1 + R3 (newer R1)					; 119-122	n 125
	vsubpd	zmm10, zmm10, zmm0		;; R1 - R3 (newer R3)					; 120-123	n 125

 	vaddpd	zmm12, zmm18, zmm5		;; I2 + I4 (newer I2)					; 121-124	n 127
	vsubpd	zmm18, zmm18, zmm5		;; I2 - I4 (newer I4)					; 122-125	n 127

	vaddpd	zmm8, zmm21, zmm13		;; I6 + I8 (newer I6/SQRTHALF)				; 123-126	n 129
	vsubpd	zmm21, zmm21, zmm13		;; I6 - I8 (newer R8/SQRTHALF)				; 124-127	n 131

	vsubpd	zmm9, zmm11, zmm14		;; R8 - R6 (newer I8/SQRTHALF)				; 125-128	n 129
	vaddpd	zmm11, zmm11, zmm14		;; R8 + R6 (newer R6/SQRTHALF)				; 126-129	n 131

	vaddpd	zmm0, zmm20, zmm1		;; R2 + R4 (newer R2)					; 127-130	n 132
	vsubpd	zmm1, zmm20, zmm1		;; R2 - R4 (newer R4)					; 128-131	n 132

	vaddpd	zmm5, zmm7, zmm17		;; I1 + I5 (final I1)					; 129-132
	vsubpd	zmm7, zmm7, zmm17		;; I1 - I5 (final I5)					; 130-133

	vaddpd	zmm13, zmm2, zmm3		;; R1 + R5 (final R1)					; 131-134
	vsubpd	zmm2, zmm2, zmm3		;; R1 - R5 (final R5)					; 132-135

	zfmaddpd zmm14, zmm8, zmm31, zmm12	;; I2 + I6 * SQRTHALF (final I2)			; 133-136
	zfnmaddpd zmm8, zmm8, zmm31, zmm12	;; I2 - I6 * SQRTHALF (final I6)			; 134-137

	zfmaddpd zmm3, zmm11, zmm31, zmm0	;; R2 + R6 * SQRTHALF (final R2)			; 135-138
	zfnmaddpd zmm11, zmm11, zmm31, zmm0	;; R2 - R6 * SQRTHALF (final R6)			; 136-139

	vaddpd	zmm12, zmm4, zmm6		;; I3 + I7 (final I3)					; 137-140
	vsubpd	zmm4, zmm4, zmm6		;; I3 - I7 (final I7)					; 138-141

	vaddpd	zmm0, zmm10, zmm15		;; R3 + R7 (final R3)					; 139-142
	vsubpd	zmm10, zmm10, zmm15		;; R3 - R7 (final R7)					; 140-143

	zfmaddpd zmm6, zmm9, zmm31, zmm18	;; I4 + I8 * SQRTHALF (final I4)			; 141-144
	zfnmaddpd zmm9, zmm9, zmm31, zmm18	;; I4 - I8 * SQRTHALF (final I8)			; 142-145

	zfmaddpd zmm15, zmm21, zmm31, zmm1	;; R4 + R8 * SQRTHALF (final R4)			; 143-146
	zfnmaddpd zmm21, zmm21, zmm31, zmm1	;; R4 - R8 * SQRTHALF (final R8)			; 144-147

;	L1prefetchw srcreg+L1pd, L1pt
;	L1prefetchw srcreg+64+L1pd, L1pt
;	L1prefetchw srcreg+d1+L1pd, L1pt
;	L1prefetchw srcreg+d1+64+L1pd, L1pt
;;	L1prefetchw srcreg+d2+L1pd, L1pt
;	L1prefetchw srcreg+d2+64+L1pd, L1pt
;	L1prefetchw srcreg+d2+d1+L1pd, L1pt
;	L1prefetchw srcreg+d2+d1+64+L1pd, L1pt
;	L1prefetchw srcreg+d4+L1pd, L1pt
;	L1prefetchw srcreg+d4+64+L1pd, L1pt
;	L1prefetchw srcreg+d4+d1+L1pd, L1pt
;	L1prefetchw srcreg+d4+d1+64+L1pd, L1pt
;	L1prefetchw srcreg+d4+d2+L1pd, L1pt
;	L1prefetchw srcreg+d4+d2+64+L1pd, L1pt
;	L1prefetchw srcreg+d4+d2+d1+L1pd, L1pt
;	L1prefetchw srcreg+d4+d2+d1+64+L1pd, L1pt

	zstore	[srcreg+r8], zmm13		;; Save R1						; 179
	zstore	[srcreg+r8+d4], zmm2		;; Save R5						; 179+1
	zstore	[srcreg+r8+d2+64], zmm12	;; Save I3						; 180+1
	zstore	[srcreg+r8+d4+d2+64], zmm4	;; Save I7						; 180+2
	zstore	[srcreg+r8+64], zmm5		;; Save I1						; 183
	zstore	[srcreg+r8+d4+64], zmm7		;; Save I5						; 183+1
	zstore	[srcreg+r8+d2], zmm0		;; Save R3						; 184+1
	zstore	[srcreg+r8+d4+d2], zmm10	;; Save R7						; 184+2
	zstore	[srcreg+r8+d1], zmm3		;; Save R2						; 185+2
	zstore	[srcreg+r8+d4+d1], zmm11	;; Save R6						; 185+3
	zstore	[srcreg+r8+d2+d1], zmm15	;; Save R4						; 186+3
	zstore	[srcreg+r8+d4+d2+d1], zmm21	;; Save R8						; 186+4
	zstore	[srcreg+r8+d1+64], zmm14	;; Save I2						; 187+5
	zstore	[srcreg+r8+d4+d1+64], zmm8	;; Save I6						; 187+5
	zstore	[srcreg+r8+d2+d1+64], zmm6	;; Save I4						; 188+5
	zstore	[srcreg+r8+d4+d2+d1+64], zmm9	;; Save I8						; 188+6
	bump	srcreg, srcinc
	ENDM

zr8_eight_complex_with_mulf_preload MACRO
	vbroadcastsd zmm31, ZMM_SQRTHALF
	ENDM
zr8_eight_complex_with_mulf MACRO srcreg,srcinc,d1,d2,d4
	LOCAL	orig, back_to_orig

	cmp	mul4_opcode, 0			;; See if we need to do more than the original type-4 FFT multiply
	je	short orig
	call	zcomplex_mulf_opcode		;; Handle more difficult cases --- exact same code as zr64.mac!
	jmp	back_to_orig
orig:

	;; Multiply the complex numbers
	vmovapd	zmm16, [srcreg][rbx]		;; R1
	vmovapd	zmm9, [srcreg+64][rbx]		;; I1
	vmovapd	zmm18, [srcreg+rbp]		;; MemR1
	vmulpd	zmm20, zmm16, zmm18		;; A1 = R1 * MemR1					; 90-93		n 94
	vmulpd	zmm18, zmm9, zmm18		;; B1 = I1 * MemR1					; 90-93		n 94

	vmovapd	zmm19, [srcreg+d1][rbx]		;; R2
	vmovapd	zmm8, [srcreg+d1+64][rbx]	;; I2
	vmovapd	zmm21, [srcreg+rbp+d1]		;; MemR2
	vmulpd	zmm6, zmm19, zmm21		;; A2 = R2 * MemR2					; 91-94		n 95
	vmulpd	zmm21, zmm8, zmm21		;; B2 = I2 * MemR2					; 91-94		n 95

	vmovapd	zmm14, [srcreg+d2][rbx]		;; R3
	vmovapd	zmm0, [srcreg+d2+64][rbx]	;; I3
	vmovapd	zmm1, [srcreg+rbp+d2]		;; MemR3
	vmulpd	zmm17, zmm14, zmm1		;; A3 = R3 * MemR3					; 92-95		n 96
	vmulpd	zmm1, zmm0, zmm1		;; B3 = I3 * MemR3					; 92-95		n 96

	vmovapd	zmm11, [srcreg+d2+d1][rbx]	;; R4
	vmovapd	zmm7, [srcreg+d2+d1+64][rbx]	;; I4
	vmovapd	zmm13, [srcreg+rbp+d2+d1]	;; MemR4
	vmulpd	zmm5, zmm11, zmm13		;; A4 = R4 * MemR4					; 93-96		n 97
	vmulpd	zmm13, zmm7, zmm13		;; B4 = I4 * MemR4					; 93-96		n 97

	vmovapd zmm30, [srcreg+rbp+64]		;; MemI1
	zfnmaddpd zmm20, zmm9, zmm30, zmm20	;; A1 - I1 * MemI1 (R1)					; 94-97		n 110
	zfmaddpd zmm18, zmm16, zmm30, zmm18	;; B1 + R1 * MemI1 (I1)					; 94-97		n 106

	vmovapd zmm30, [srcreg+rbp+d1+64]	;; MemI2
	zfnmaddpd zmm6, zmm8, zmm30, zmm6	;; A2 - I2 * MemI2 (R2)					; 95-98		n 110
	zfmaddpd zmm21, zmm19, zmm30, zmm21	;; B2 + R2 * MemI2 (I2)					; 95-98		n 106

	vmovapd zmm30, [srcreg+rbp+d2+64]	;; MemI3
	zfnmaddpd zmm17, zmm0, zmm30, zmm17	;; A3 - I3 * MemI3 (R3)					; 96-99		n 113
	zfmaddpd zmm1, zmm14, zmm30, zmm1	;; B3 + R3 * MemI3 (I3)					; 96-99		n 108

	vmovapd zmm30, [srcreg+rbp+d2+d1+64]	;; MemI4
	zfnmaddpd zmm5, zmm7, zmm30, zmm5	;; A4 - I4 * MemI4 (R4)					; 97-100	n 113
	zfmaddpd zmm13, zmm11, zmm30, zmm13	;; B4 + R4 * MemI4 (I4)					; 97-100	n 108

	vmovapd	zmm22, [srcreg+d4][rbx]		;; R5
	vmovapd	zmm23, [srcreg+d4+64][rbx]	;; I5
	vmovapd	zmm19, [srcreg+rbp+d4]		;; MemR5
	vmulpd	zmm8, zmm22, zmm19		;; A5 = R5 * MemR5					; 98-101	n 102
	vmulpd	zmm19, zmm23, zmm19		;; B5 = I5 * MemR5					; 98-101	n 102

	vmovapd	zmm12, [srcreg+d4+d1][rbx]	;; R6
	vmovapd	zmm4, [srcreg+d4+d1+64][rbx]	;; I6
	vmovapd	zmm14, [srcreg+rbp+d4+d1]	;; MemR6
	vmulpd	zmm0, zmm12, zmm14		;; A6 = R6 * MemR6					; 99-102	n 103
	vmulpd	zmm14, zmm4, zmm14		;; B6 = I6 * MemR6					; 99-102	n 103

	vmovapd	zmm30, [srcreg+d4+d2][rbx]	;; R7
	vmovapd	zmm2, [srcreg+d4+d2+64][rbx]	;; I7
	vmovapd	zmm11, [srcreg+rbp+d4+d2]	;; MemR7
	vmulpd	zmm7, zmm30, zmm11		;; A7 = R7 * MemR7					; 100-103	n 104
	vmulpd	zmm11, zmm2, zmm11		;; B7 = I7 * MemR7					; 100-103	n 104

	vmovapd	zmm3, [srcreg+d4+d2+d1][rbx]	;; R8
	vmovapd	zmm10, [srcreg+d4+d2+d1+64][rbx];; I8
	vmovapd	zmm16, [srcreg+rbp+d4+d2+d1]	;; MemR8
	vmulpd	zmm9, zmm3, zmm16		;; A8 = R8 * MemR8					; 101-104	n 105
	vmulpd	zmm16, zmm10, zmm16		;; B8 = I8 * MemR8					; 101-104	n 105

	vmovapd zmm15, [srcreg+rbp+d4+64]	;; MemI5
	zfnmaddpd zmm8, zmm23, zmm15, zmm8	;; A5 - I5 * MemI5 (R5)					; 102-105	n 109
	zfmaddpd zmm19, zmm22, zmm15, zmm19	;; B5 + R5 * MemI5 (I5)					; 102-105	n 107

	vmovapd zmm15, [srcreg+rbp+d4+d1+64]	;; MemI6
	zfnmaddpd zmm0, zmm4, zmm15, zmm0	;; A6 - I6 * MemI6 (R6)					; 103-106	n 109
	zfmaddpd zmm14, zmm12, zmm15, zmm14	;; B6 + R6 * MemI6 (I6)					; 103-106	n 107

	vmovapd zmm15, [srcreg+rbp+d4+d2+64]	;; MemI7
	zfnmaddpd zmm7, zmm2, zmm15, zmm7	;; A7 - I7 * MemI7 (R7)					; 104-107	n 111
	zfmaddpd zmm11, zmm30, zmm15, zmm11	;; B7 + R7 * MemI7 (I7)					; 104-107	n 112

	vmovapd zmm15, [srcreg+rbp+d4+d2+d1+64]	;; MemI8
	zfnmaddpd zmm9, zmm10, zmm15, zmm9	;; A8 - I8 * MemI8 (R8)					; 105-108	n 111
	zfmaddpd zmm16, zmm3, zmm15, zmm16	;; B8 + R8 * MemI8 (I8)					; 105-108	n 112

back_to_orig:
	vaddpd	zmm4, zmm18, zmm21		;; I1 + I2 (new I1)					; 106-109	n 114
	vsubpd	zmm18, zmm18, zmm21		;; I1 - I2 (new I2)					; 106-109	n 121

	vaddpd	zmm12, zmm19, zmm14		;; I5 + I6 (new I5)					; 107-110	n 116
	vsubpd	zmm14, zmm19, zmm14		;; I5 - I6 (new I6)					; 107-110	n 117

	vaddpd	zmm21, zmm1, zmm13		;; I3 + I4 (new I3)					; 108-111	n 114
	vsubpd	zmm1, zmm1, zmm13		;; I3 - I4 (new R4)					; 108-111	n 127

	vaddpd	zmm13, zmm8, zmm0		;; R5 + R6 (new R5)					; 109-112	n 115
	vsubpd	zmm8, zmm8, zmm0		;; R5 - R6 (new R6)					; 109-112	n 117

	vaddpd	zmm10, zmm20, zmm6		;; R1 + R2 (new R1)					; 110-113	n 119
	vsubpd	zmm20, zmm20, zmm6		;; R1 - R2 (new R2)					; 110-113	n 127

	vaddpd	zmm3, zmm9, zmm7		;; R8 + R7 (new R7)					; 111-114	n 115
	vsubpd	zmm9, zmm9, zmm7		;; R8 - R7 (new I8)					; 111-114	n 118

	vaddpd	zmm15, zmm11, zmm16		;; I7 + I8 (new I7)					; 112-115	n 116
	vsubpd	zmm11, zmm11, zmm16		;; I7 - I8 (new R8)					; 112-115	n 118

 	vaddpd	zmm0, zmm5, zmm17		;; R4 + R3 (new R3)					; 113-116	n 119
	vsubpd	zmm5, zmm5, zmm17		;; R4 - R3 (new I4)					; 113-116	n 121

	vaddpd	zmm7, zmm4, zmm21		;; I1 + I3 (newer I1)					; 114-117	n 119
	vsubpd	zmm4, zmm4, zmm21		;; I1 - I3 (newer I3)					; 114-117	n 119

	vsubpd	zmm6, zmm3, zmm13		;; R7 - R5 (newer I7)					; 115-118	n 121
	vaddpd	zmm3, zmm3, zmm13		;; R7 + R5 (newer R5)					; 115-118	n 123

	vaddpd	zmm17, zmm12, zmm15		;; I5 + I7 (newer I5)					; 116-119	n 121
	vsubpd	zmm15, zmm12, zmm15		;; I5 - I7 (newer R7)					; 116-119	n 123

	;; multiply R6/I6 by SQRTHALF - i*SQRTHALF
	vsubpd	zmm21, zmm14, zmm8		;; I6 = I6 - R6						; 117-120	n 123
	vaddpd	zmm14, zmm8, zmm14		;; R6 = R6 + I6						; 117-120	n 125

	;; multiply R8/I8 by SQRTHALF - i*SQRTHALF
	vsubpd	zmm13, zmm9, zmm11		;; I8 = I8 - R8						; 118-121	n 123
	vaddpd	zmm11, zmm11, zmm9		;; R8 = R8 + I8						; 118-121	n 125

	vaddpd	zmm2, zmm10, zmm0		;; R1 + R3 (newer R1)					; 119-122	n 125
	vsubpd	zmm10, zmm10, zmm0		;; R1 - R3 (newer R3)					; 120-123	n 125

 	vaddpd	zmm12, zmm18, zmm5		;; I2 + I4 (newer I2)					; 121-124	n 127
	vsubpd	zmm18, zmm18, zmm5		;; I2 - I4 (newer I4)					; 122-125	n 127

	vaddpd	zmm8, zmm21, zmm13		;; I6 + I8 (newer I6/SQRTHALF)				; 123-126	n 129
	vsubpd	zmm21, zmm21, zmm13		;; I6 - I8 (newer R8/SQRTHALF)				; 124-127	n 131

	vsubpd	zmm9, zmm11, zmm14		;; R8 - R6 (newer I8/SQRTHALF)				; 125-128	n 129
	vaddpd	zmm11, zmm11, zmm14		;; R8 + R6 (newer R6/SQRTHALF)				; 126-129	n 131

	vaddpd	zmm0, zmm20, zmm1		;; R2 + R4 (newer R2)					; 127-130	n 132
	vsubpd	zmm1, zmm20, zmm1		;; R2 - R4 (newer R4)					; 128-131	n 132

	vaddpd	zmm5, zmm7, zmm17		;; I1 + I5 (final I1)					; 129-132
	vsubpd	zmm7, zmm7, zmm17		;; I1 - I5 (final I5)					; 130-133

	vaddpd	zmm13, zmm2, zmm3		;; R1 + R5 (final R1)					; 131-134
	vsubpd	zmm2, zmm2, zmm3		;; R1 - R5 (final R5)					; 132-135

	zfmaddpd zmm14, zmm8, zmm31, zmm12	;; I2 + I6 * SQRTHALF (final I2)			; 133-136
	zfnmaddpd zmm8, zmm8, zmm31, zmm12	;; I2 - I6 * SQRTHALF (final I6)			; 134-137

	zfmaddpd zmm3, zmm11, zmm31, zmm0	;; R2 + R6 * SQRTHALF (final R2)			; 135-138
	zfnmaddpd zmm11, zmm11, zmm31, zmm0	;; R2 - R6 * SQRTHALF (final R6)			; 136-139

	vaddpd	zmm12, zmm4, zmm6		;; I3 + I7 (final I3)					; 137-140
	vsubpd	zmm4, zmm4, zmm6		;; I3 - I7 (final I7)					; 138-141

	vaddpd	zmm0, zmm10, zmm15		;; R3 + R7 (final R3)					; 139-142
	vsubpd	zmm10, zmm10, zmm15		;; R3 - R7 (final R7)					; 140-143

	zfmaddpd zmm6, zmm9, zmm31, zmm18	;; I4 + I8 * SQRTHALF (final I4)			; 141-144
	zfnmaddpd zmm9, zmm9, zmm31, zmm18	;; I4 - I8 * SQRTHALF (final I8)			; 142-145

	zfmaddpd zmm15, zmm21, zmm31, zmm1	;; R4 + R8 * SQRTHALF (final R4)			; 143-146
	zfnmaddpd zmm21, zmm21, zmm31, zmm1	;; R4 - R8 * SQRTHALF (final R8)			; 144-147

;	L1prefetchw srcreg+L1pd, L1pt
;	L1prefetchw srcreg+64+L1pd, L1pt
;	L1prefetchw srcreg+d1+L1pd, L1pt
;	L1prefetchw srcreg+d1+64+L1pd, L1pt
;;	L1prefetchw srcreg+d2+L1pd, L1pt
;	L1prefetchw srcreg+d2+64+L1pd, L1pt
;	L1prefetchw srcreg+d2+d1+L1pd, L1pt
;	L1prefetchw srcreg+d2+d1+64+L1pd, L1pt
;	L1prefetchw srcreg+d4+L1pd, L1pt
;	L1prefetchw srcreg+d4+64+L1pd, L1pt
;	L1prefetchw srcreg+d4+d1+L1pd, L1pt
;	L1prefetchw srcreg+d4+d1+64+L1pd, L1pt
;	L1prefetchw srcreg+d4+d2+L1pd, L1pt
;	L1prefetchw srcreg+d4+d2+64+L1pd, L1pt
;	L1prefetchw srcreg+d4+d2+d1+L1pd, L1pt
;	L1prefetchw srcreg+d4+d2+d1+64+L1pd, L1pt

	zstore	[srcreg], zmm13			;; Save R1						; 179
	zstore	[srcreg+d4], zmm2		;; Save R5						; 179+1
	zstore	[srcreg+d2+64], zmm12		;; Save I3						; 180+1
	zstore	[srcreg+d4+d2+64], zmm4		;; Save I7						; 180+2
	zstore	[srcreg+64], zmm5		;; Save I1						; 183
	zstore	[srcreg+d4+64], zmm7		;; Save I5						; 183+1
	zstore	[srcreg+d2], zmm0		;; Save R3						; 184+1
	zstore	[srcreg+d4+d2], zmm10		;; Save R7						; 184+2
	zstore	[srcreg+d1], zmm3		;; Save R2						; 185+2
	zstore	[srcreg+d4+d1], zmm11		;; Save R6						; 185+3
	zstore	[srcreg+d2+d1], zmm15		;; Save R4						; 186+3
	zstore	[srcreg+d4+d2+d1], zmm21	;; Save R8						; 186+4
	zstore	[srcreg+d1+64], zmm14		;; Save I2						; 187+5
	zstore	[srcreg+d4+d1+64], zmm8		;; Save I6						; 187+5
	zstore	[srcreg+d2+d1+64], zmm6		;; Save I4						; 188+5
	zstore	[srcreg+d4+d2+d1+64], zmm9	;; Save I8						; 188+6
	bump	srcreg, srcinc
	ENDM
