; Copyright 2011-2024 - Mersenne Research, Inc.  All rights reserved
; Author:  George Woltman
; Email: woltman@alum.mit.edu
;

;;
;; Macros for a radix-8 step in an FFT.  This is used in a radix-4 FFT
;; with an odd number of levels.
;;


;; Utility macro that lets us create and call common code to reduce executable size

yr8_dispatch MACRO call_type,routine,d1,d2,d4
IF	d1 EQ 64
	call	&routine&1
ELSEIF	d1 EQ 32*64 AND d2 EQ 64*64+128
	call	&routine&2
ELSEIF	d1 EQ 2*(64*64+64)			;;2*dist64
	call	&routine&3
ELSEIF	d1 EQ 8*(64*64+64)			;;8*dist64
	call	&routine&4
ELSE
	UNSUPPORTED8 d1
ENDIF
	ENDM

;;
;; ************************************* eight-complex-fft8 variants ******************************************
;;
;; In the negacyclic split premultiplier case, we apply part of the roots of -1 at the
;; end of the first pass.  Also, in the r4delay case we apply part of the first level
;; twiddles at the end of the first pass.  Thus we have 8 sin/cos multiplies instead
;; of the usual 7.
;;

yr8_sg8cl_eight_complex_fft8 MACRO srcreg,srcinc,d1,d2,d4,dstreg,dstinc,e1,e2,e4,screg,scinc,maxrpt,L1pt,L1pd

	;; On input the 64-byte cache lines hold these data values:
	;;	0	+1K	+1K	+1K	8K	+1K	+1K	+1K
	;;	1	...
	;;	2	...
	;;	3	...

	;; These are swizzled to:
	;;	0	+1	+1	+1	8K	+1	+1	+1
	;;	1K	...
	;;	2K	...
	;;	3K	...

	;; Swizzle using 8 shufpds and 8 perm2f128  (1 or 3 clocks each on port 5, throughput 1)

	vmovapd	ymm1, [srcreg]			;; R1
	vmovapd	ymm7, [srcreg+d1]		;; R2
	vshufpd	ymm0, ymm1, ymm7, 15		;; Shuffle R1 and R2 to create R1/R2 hi
	vshufpd	ymm1, ymm1, ymm7, 0		;; Shuffle R1 and R2 to create R1/R2 low

	vmovapd	ymm2, [srcreg+d2]		;; R3
	vmovapd	ymm7, [srcreg+d2+d1]		;; R4
	vshufpd	ymm3, ymm2, ymm7, 15		;; Shuffle R3 and R4 to create R3/R4 hi
	vshufpd	ymm2, ymm2, ymm7, 0		;; Shuffle R3 and R4 to create R3/R4 low

	ylow128s ymm4, ymm0, ymm3		;; Shuffle R1/R2 hi and R3/R4 hi (first R2)
	yhigh128s ymm0, ymm0, ymm3		;; Shuffle R1/R2 hi and R3/R4 hi (first R4)

	ylow128s ymm3, ymm1, ymm2		;; Shuffle R1/R2 low and R3/R4 low (first R1)
	yhigh128s ymm1, ymm1, ymm2		;; Shuffle R1/R2 low and R3/R4 low (first R3)

	vmovapd	ymm6, [srcreg+d4]		;; R5
	vmovapd	ymm2, [srcreg+d4+d1]		;; R6
	vshufpd	ymm5, ymm6, ymm2, 15		;; Shuffle R5 and R6 to create R5/R6 hi
	vshufpd	ymm6, ymm6, ymm2, 0		;; Shuffle R5 and R6 to create R5/R6 low

	vmovapd	ymm2, [srcreg+d4+d2]		;; R7
	vmovapd	ymm7, [srcreg+d4+d2+d1]		;; R8
	ystore	[dstreg+32], ymm0		;; Save first R4
	vshufpd	ymm0, ymm2, ymm7, 15		;; Shuffle R7 and R8 to create R7/R8 hi
	vshufpd	ymm2, ymm2, ymm7, 0		;; Shuffle R7 and R8 to create R7/R8 low

	ylow128s ymm7, ymm5, ymm0		;; Shuffle R5/R6 hi and R7/R8 hi (first R6)
	yhigh128s ymm5, ymm5, ymm0		;; Shuffle R5/R6 hi and R7/R8 hi (first R8)

	ylow128s ymm0, ymm6, ymm2		;; Shuffle R5/R6 low and R7/R8 low (first R5)
	yhigh128s ymm6, ymm6, ymm2		;; Shuffle R5/R6 low and R7/R8 low (first R7)

	L1prefetch srcreg+L1pd, L1pt
	L1prefetchw dstreg+L1pd, L1pt - L1PREFETCH_DEST_NONE

	vaddpd	ymm2, ymm3, ymm0		;; R1 + R5 (new R1)
	vsubpd	ymm3, ymm3, ymm0		;; R1 - R5 (new R5)

	vaddpd	ymm0, ymm1, ymm6		;; R3 + R7 (new R3)
	vsubpd	ymm1, ymm1, ymm6		;; R3 - R7 (new R7)

	L1prefetch srcreg+d1+L1pd, L1pt
	L1prefetchw dstreg+e1+L1pd, L1pt - L1PREFETCH_DEST_NONE

	vaddpd	ymm6, ymm4, ymm7		;; R2 + R6 (new R2)
	vsubpd	ymm4, ymm4, ymm7		;; R2 - R6 (new R6)

	vmovapd	ymm7, [dstreg+32]		;; Reload first R4
	ystore	[dstreg+e4], ymm3		;; Save new R5

	vaddpd	ymm3, ymm7, ymm5		;; R4 + R8 (new R4)
	vsubpd	ymm7, ymm7, ymm5		;; R4 - R8 (new R8)

	L1prefetch srcreg+d2+L1pd, L1pt
	L1prefetchw dstreg+e2+L1pd, L1pt - L1PREFETCH_DEST_NONE

	vaddpd	ymm5, ymm2, ymm0		;; R1 + R3 (newer R1)
	vsubpd	ymm2, ymm2, ymm0		;; R1 - R3 (newer R3)

	vaddpd	ymm0, ymm6, ymm3		;; R2 + R4 (newer R2)
	vsubpd	ymm6, ymm6, ymm3		;; R2 - R4 (newer R4)

	ystore	[dstreg+e4+e2], ymm1		;; Save new R7
	vmovapd	ymm1, [srcreg+32]		;; I1
	ystore	[dstreg+e4+e2+e1], ymm7		;; Save new R8
	vmovapd	ymm7, [srcreg+d1+32]		;; I2
	ystore	[dstreg+e1], ymm0		;; Save newer R2
	vshufpd	ymm0, ymm1, ymm7, 15		;; Shuffle I1 and I2 to create I1/I2 hi
	vshufpd	ymm1, ymm1, ymm7, 0		;; Shuffle I1 and I2 to create I1/I2 low

	vmovapd	ymm7, [srcreg+d2+d1+32]		;; I4
	ystore	[dstreg+e2], ymm2		;; Save newer R3
	vmovapd	ymm2, [srcreg+d2+32]		;; I3
	vshufpd	ymm3, ymm2, ymm7, 15		;; Shuffle I3 and I4 to create I3/I4 hi
	vshufpd	ymm2, ymm2, ymm7, 0		;; Shuffle I3 and I4 to create I3/I4 low

	ystore	[dstreg+e4+e1], ymm4		;; Save new R6
	ylow128s ymm4, ymm0, ymm3		;; Shuffle I1/I2 hi and I3/I4 hi (first I2)
	yhigh128s ymm0, ymm0, ymm3		;; Shuffle I1/I2 hi and I3/I4 hi (first I4)

	ylow128s ymm3, ymm1, ymm2		;; Shuffle I1/I2 low and I3/I4 low (first I1)
	yhigh128s ymm1, ymm1, ymm2		;; Shuffle I1/I2 low and I3/I4 low (first I3)

	vmovapd	ymm2, [srcreg+d4+d1+32]		;; I6
	ystore	[dstreg+e2+e1], ymm6		;; Save newer R4
	vmovapd	ymm6, [srcreg+d4+32]		;; I5
	ystore	[dstreg], ymm5			;; Save newer R1
	vshufpd	ymm5, ymm6, ymm2, 15		;; Shuffle I5 and I6 to create I5/I6 hi
	vshufpd	ymm6, ymm6, ymm2, 0		;; Shuffle I5 and I6 to create I5/I6 low

	vmovapd	ymm2, [srcreg+d4+d2+32]		;; I7
	vmovapd	ymm7, [srcreg+d4+d2+d1+32]	;; I8
	ystore	[dstreg+e2+e1+32], ymm0		;; Save first I4
	vshufpd	ymm0, ymm2, ymm7, 15		;; Shuffle I7 and I8 to create I7/I8 hi
	vshufpd	ymm2, ymm2, ymm7, 0		;; Shuffle I7 and I8 to create I7/I8 low

	ylow128s ymm7, ymm5, ymm0		;; Shuffle I5/I6 hi and I7/I8 hi (first I6)
	yhigh128s ymm5, ymm5, ymm0		;; Shuffle I5/I6 hi and I7/I8 hi (first I8)

	ylow128s ymm0, ymm6, ymm2		;; Shuffle I5/I6 low and I7/I8 low (first I5)
	yhigh128s ymm6, ymm6, ymm2		;; Shuffle I5/I6 low and I7/I8 low (first I7)

	L1prefetch srcreg+d2+d1+L1pd, L1pt
	L1prefetchw dstreg+e2+e1+L1pd, L1pt - L1PREFETCH_DEST_NONE

	vaddpd	ymm2, ymm3, ymm0		;; I1 + I5 (new I1)
	vsubpd	ymm3, ymm3, ymm0		;; I1 - I5 (new I5)

	vaddpd	ymm0, ymm1, ymm6		;; I3 + I7 (new I3)
	vsubpd	ymm1, ymm1, ymm6		;; I3 - I7 (new I7)

	L1prefetch srcreg+d4+L1pd, L1pt
	L1prefetchw dstreg+e4+L1pd, L1pt - L1PREFETCH_DEST_NONE

	vaddpd	ymm6, ymm2, ymm0		;; I1 + I3 (newer I1)
	vsubpd	ymm2, ymm2, ymm0		;; I1 - I3 (newer I3)

	vmovapd	ymm0, [dstreg+e2+e1+32]		;; Reload first I4
	ystore	[dstreg+32], ymm6		;; Save newer I1

	vaddpd	ymm6, ymm4, ymm7		;; I2 + I6 (new I2)
	vsubpd	ymm4, ymm4, ymm7		;; I2 - I6 (new I6)

	L1prefetch srcreg+d4+d1+L1pd, L1pt
	L1prefetchw dstreg+e4+e1+L1pd, L1pt - L1PREFETCH_DEST_NONE

	vaddpd	ymm7, ymm0, ymm5		;; I4 + I8 (new I4)
	vsubpd	ymm0, ymm0, ymm5		;; I4 - I8 (new I8)

	vaddpd	ymm5, ymm6, ymm7		;; I2 + I4 (newer I2)
	vsubpd	ymm6, ymm6, ymm7		;; I2 - I4 (newer I4)

	vmovapd	ymm7, [dstreg+e4+e2]		;; Reload new R7
	ystore	[dstreg+e2+32], ymm2		;; Save newer I3

 	vaddpd	ymm2, ymm3, ymm7		;; I5 + R7 (newer I5)
	vsubpd	ymm3, ymm3, ymm7		;; I5 - R7 (newer I7)

	vmovapd	ymm7, [dstreg+e4]		;; Reload new R5
	ystore	[dstreg+e1+32], ymm5		;; Save newer I2

	vaddpd	ymm5, ymm7, ymm1		;; R5 + I7 (newer R7)
	vsubpd	ymm7, ymm7, ymm1		;; R5 - I7 (newer R5)

	vmovapd	ymm1, [dstreg+e4+e1]		;; Reload new R6
	ystore	[dstreg+e2+e1+32], ymm6		;; Save newer I4

	vaddpd	ymm6, ymm1, ymm0		;; R6 + I8 (new R8)
	vsubpd	ymm1, ymm1, ymm0		;; R6 - I8 (new R6)

	vmovapd	ymm0, [dstreg+e4+e2+e1]		;; Reload new R8
	ystore	[dstreg+e4+32], ymm2		;; Save newer I5

	vaddpd	ymm2, ymm4, ymm0		;; I6 + R8 (new I6)
	vsubpd	ymm4, ymm4, ymm0		;; I6 - R8 (new I8)

	L1prefetch srcreg+d4+d2+L1pd, L1pt
	L1prefetchw dstreg+e4+e2+L1pd, L1pt - L1PREFETCH_DEST_NONE

	vsubpd	ymm0, ymm1, ymm2		;; R6 = R6 - I6
	vaddpd	ymm1, ymm1, ymm2		;; I6 = R6 + I6
	vsubpd	ymm2, ymm6, ymm4		;; R8 = R8 - I8
	vaddpd	ymm6, ymm6, ymm4		;; I8 = R8 + I8

	vmovapd	ymm4, YMM_SQRTHALF
	vmulpd	ymm0, ymm0, ymm4		;; R6 = R6 * SQRTHALF (newer R6)
	vmulpd	ymm1, ymm1, ymm4		;; I6 = I6 * SQRTHALF (newer I6)
	vmulpd	ymm2, ymm2, ymm4		;; R8 = R8 * SQRTHALF (newer R8)
	vmulpd	ymm6, ymm6, ymm4		;; I8 = I8 * SQRTHALF (newer I8)

	ystore	[dstreg+e4], ymm7		;; Save newer R5

;; the last level

	vsubpd	ymm4, ymm5, ymm6		;; R7 - I8 (last R7)
	vaddpd	ymm5, ymm5, ymm6		;; R7 + I8 (last R8)

	L1prefetch srcreg+d4+d2+d1+L1pd, L1pt
	L1prefetchw dstreg+e4+e2+e1+L1pd, L1pt - L1PREFETCH_DEST_NONE

	vsubpd	ymm6, ymm3, ymm2		;; I7 - R8 (last I8)
	vaddpd	ymm3, ymm3, ymm2		;; I7 + R8 (last I7)

	vmovapd	ymm2, [screg+192+32]		;; cosine/sine
	vmulpd	ymm7, ymm4, ymm2		;; A7 = R7 * cosine/sine
	vsubpd	ymm7, ymm7, ymm3		;; A7 = A7 - I7
	vmulpd	ymm3, ymm3, ymm2		;; B7 = I7 * cosine/sine
	vaddpd	ymm3, ymm3, ymm4		;; B7 = B7 + R7

	vmovapd	ymm2, [screg+448+32]		;; cosine/sine
	vmulpd	ymm4, ymm5, ymm2		;; A8 = R8 * cosine/sine
	vsubpd	ymm4, ymm4, ymm6		;; A8 = A8 - I8
	vmulpd	ymm6, ymm6, ymm2		;; B8 = I8 * cosine/sine
	vaddpd	ymm6, ymm6, ymm5		;; B8 = B8 + R8

	vmovapd	ymm2, [screg+192]		;; sine
	vmulpd	ymm7, ymm7, ymm2		;; A7 = A7 * sine (final R7)
	vmulpd	ymm3, ymm3, ymm2		;; B7 = B7 * sine (final I7)
	vmovapd	ymm2, [screg+448]		;; sine
	vmulpd	ymm4, ymm4, ymm2		;; A8 = A8 * sine (final R8)
	vmulpd	ymm6, ymm6, ymm2		;; B8 = B8 * sine (final I8)

	vmovapd	ymm2, [dstreg+e4]		;; Reload newer R5
	vsubpd	ymm5, ymm2, ymm0		;; R5 - R6 (last R6)
	vaddpd	ymm2, ymm2, ymm0		;; R5 + R6 (last R5)

	vmovapd	ymm0, [dstreg+e4+32]		;; Reload newer I5
	ystore	[dstreg+e4+e2], ymm7		;; Save R7

	vsubpd	ymm7, ymm0, ymm1		;; I5 - I6 (last I6)
	vaddpd	ymm0, ymm0, ymm1		;; I5 + I6 (last I5)

	vmovapd	ymm1, [screg+320+32]		;; cosine/sine
	ystore	[dstreg+e4+e2+32], ymm3		;; Save I7
	vmulpd	ymm3, ymm5, ymm1		;; A6 = R6 * cosine/sine
	vsubpd	ymm3, ymm3, ymm7		;; A6 = A6 - I6
	vmulpd	ymm7, ymm7, ymm1		;; B6 = I6 * cosine/sine
	vaddpd	ymm7, ymm7, ymm5		;; B6 = B6 + R6

	vmovapd	ymm1, [screg+64+32]		;; cosine/sine
	vmulpd	ymm5, ymm2, ymm1		;; A5 = R5 * cosine/sine
	vsubpd	ymm5, ymm5, ymm0		;; A5 = A5 - I5
	vmulpd	ymm0, ymm0, ymm1		;; B5 = I5 * cosine/sine
	vaddpd	ymm0, ymm0, ymm2		;; B5 = B5 + R5

	vmovapd	ymm1, [screg+320]		;; sine
	vmulpd	ymm3, ymm3, ymm1		;; A6 = A6 * sine (final R6)
	vmulpd	ymm7, ymm7, ymm1		;; B6 = B6 * sine (final I6)
	vmovapd	ymm1, [screg+64]		;; sine
	vmulpd	ymm5, ymm5, ymm1		;; A5 = A5 * sine (final R5)
	vmulpd	ymm0, ymm0, ymm1		;; B5 = B5 * sine (final I5)

	vmovapd	ymm2, [dstreg]			;; Reload newer R1
	vmovapd	ymm1, [dstreg+e1]		;; Reload newer R2
	ystore	[dstreg+e4+e2+e1], ymm4		;; Save R8

	vsubpd	ymm4, ymm2, ymm1		;; R1 - R2 (last R2)
	vaddpd	ymm2, ymm2, ymm1		;; R1 + R2 (last R1)

	vmovapd	ymm1, [dstreg+32]		;; Reload newer I1
	ystore	[dstreg+e4+e2+e1+32], ymm6	;; Save I8
	vmovapd	ymm6, [dstreg+e1+32]		;; Reload newer I2
	ystore	[dstreg+e4+e1], ymm3		;; Save R6

	vsubpd	ymm3, ymm1, ymm6		;; I1 - I2 (last I2)
	vaddpd	ymm1, ymm1, ymm6		;; I1 + I2 (last I1)

	vmovapd	ymm6, [screg+256+32]		;; cosine/sine
	ystore	[dstreg+e4+e1+32], ymm7		;; Save I6
	vmulpd	ymm7, ymm4, ymm6		;; A2 = R2 * cosine/sine
	vsubpd	ymm7, ymm7, ymm3		;; A2 = A2 - I2
	vmulpd	ymm3, ymm3, ymm6		;; B2 = I2 * cosine/sine
	vaddpd	ymm3, ymm3, ymm4		;; B2 = B2 + R2

	vmovapd	ymm6, [screg+0+32]		;; cosine/sine
	vmulpd	ymm4, ymm2, ymm6		;; A1 = R1 * cosine/sine
	vsubpd	ymm4, ymm4, ymm1		;; A1 = A1 - I1
	vmulpd	ymm1, ymm1, ymm6		;; B1 = I1 * cosine/sine
	vaddpd	ymm1, ymm1, ymm2		;; B1 = B1 + R1

	vmovapd	ymm6, [screg+256]		;; sine
	vmulpd	ymm7, ymm7, ymm6		;; A2 = A2 * sine (final R2)
	vmulpd	ymm3, ymm3, ymm6		;; B2 = B2 * sine (final I2)
	vmovapd	ymm6, [screg+0]			;; sine
	vmulpd	ymm4, ymm4, ymm6		;; A1 = A1 * sine (final R1)
	vmulpd	ymm1, ymm1, ymm6		;; B1 = B1 * sine (final I1)

	vmovapd	ymm2, [dstreg+e2]		;; Reload newer R3
	vmovapd	ymm6, [dstreg+e2+e1+32]		;; Reload newer I4
	ystore	[dstreg+e4], ymm5		;; Save R5

	vsubpd	ymm5, ymm2, ymm6		;; R3 - I4 (last R3)
	vaddpd	ymm2, ymm2, ymm6		;; R3 + I4 (last R4)

	vmovapd	ymm6, [dstreg+e2+32]		;; Reload newer I3
	ystore	[dstreg+e4+32], ymm0		;; Save I5
	vmovapd	ymm0, [dstreg+e2+e1]		;; Reload newer R4
	ystore	[dstreg+e1], ymm7		;; Save R2

	vsubpd	ymm7, ymm6, ymm0		;; I3 - R4 (last I4)
	vaddpd	ymm6, ymm6, ymm0		;; I3 + R4 (last I3)

	vmovapd	ymm0, [screg+128+32]		;; cosine/sine
	ystore	[dstreg+e1+32], ymm3		;; Save I2
	vmulpd	ymm3, ymm5, ymm0		;; A3 = R3 * cosine/sine
	vsubpd	ymm3, ymm3, ymm6		;; A3 = A3 - I3
	vmulpd	ymm6, ymm6, ymm0		;; B3 = I3 * cosine/sine
	vaddpd	ymm6, ymm6, ymm5		;; B3 = B3 + R3

	vmovapd	ymm0, [screg+384+32]		;; cosine/sine
	vmulpd	ymm5, ymm2, ymm0		;; A4 = R4 * cosine/sine
	vsubpd	ymm5, ymm5, ymm7		;; A4 = A4 - I4
	vmulpd	ymm7, ymm7, ymm0		;; B4 = I4 * cosine/sine
	vaddpd	ymm7, ymm7, ymm2		;; B4 = B4 + R4

	vmovapd	ymm0, [screg+128]
	vmulpd	ymm3, ymm3, ymm0		;; A3 = A3 * sine (final R3)
	vmulpd	ymm6, ymm6, ymm0		;; B3 = B3 * sine (final I3)
	vmovapd	ymm0, [screg+384]		;; sine
	vmulpd	ymm5, ymm5, ymm0		;; A4 = A4 * sine (final R4)
	vmulpd	ymm7, ymm7, ymm0		;; B4 = B4 * sine (final I4)

	ystore	[dstreg], ymm4			;; Save R1
	ystore	[dstreg+32], ymm1		;; Save I1
	ystore	[dstreg+e2], ymm3		;; Save R3
	ystore	[dstreg+e2+32], ymm6		;; Save I3
	ystore	[dstreg+e2+e1], ymm5		;; Save R4
	ystore	[dstreg+e2+e1+32], ymm7		;; Save I4

	bump	srcreg, srcinc
	bump	dstreg, dstinc
	bump	screg, scinc
	ENDM

;; 64-bit version

IFDEF X86_64

yr8_sg8cl_eight_complex_fft8 MACRO srcreg,srcinc,d1,d2,d4,dstreg,dstinc,e1,e2,e4,screg,scinc,maxrpt,L1pt,L1pd

	;; On input the 64-byte cache lines hold these data values:
	;;	0	+1K	+1K	+1K	8K	+1K	+1K	+1K
	;;	1	...
	;;	2	...
	;;	3	...

	;; These are swizzled to:
	;;	0	+1	+1	+1	8K	+1	+1	+1
	;;	1K	...
	;;	2K	...
	;;	3K	...

	;; Swizzle using 8 shufpds and 8 perm2f128  (1 or 3 clocks each on port 5, throughput 1)

	vmovapd	ymm1, [srcreg]			;; R1
	vmovapd	ymm7, [srcreg+d1]		;; R2
	vshufpd	ymm0, ymm1, ymm7, 15		;; Shuffle R1 and R2 to create R1/R2 hi
	vshufpd	ymm1, ymm1, ymm7, 0		;; Shuffle R1 and R2 to create R1/R2 low

	vmovapd	ymm2, [srcreg+d2]		;; R3
	vmovapd	ymm7, [srcreg+d2+d1]		;; R4
	vshufpd	ymm3, ymm2, ymm7, 15		;; Shuffle R3 and R4 to create R3/R4 hi
	vshufpd	ymm2, ymm2, ymm7, 0		;; Shuffle R3 and R4 to create R3/R4 low

	vmovapd	ymm6, [srcreg+d4]		;; R5
	vmovapd	ymm7, [srcreg+d4+d1]		;; R6
	vshufpd	ymm5, ymm6, ymm7, 15		;; Shuffle R5 and R6 to create R5/R6 hi
	vshufpd	ymm6, ymm6, ymm7, 0		;; Shuffle R5 and R6 to create R5/R6 low

	vmovapd	ymm4, [srcreg+d4+d2]		;; R7
	vmovapd	ymm8, [srcreg+d4+d2+d1]		;; R8
	vshufpd	ymm7, ymm4, ymm8, 15		;; Shuffle R7 and R8 to create R7/R8 hi
	vshufpd	ymm4, ymm4, ymm8, 0		;; Shuffle R7 and R8 to create R7/R8 low

	ylow128s ymm8, ymm1, ymm2		;; Shuffle R1/R2 low and R3/R4 low (first R1)
	yhigh128s ymm1, ymm1, ymm2		;; Shuffle R1/R2 low and R3/R4 low (first R3)
	vmovapd	ymm15, [srcreg+32]		;; I1

	ylow128s ymm2, ymm6, ymm4		;; Shuffle R5/R6 low and R7/R8 low (first R5)
	yhigh128s ymm6, ymm6, ymm4		;; Shuffle R5/R6 low and R7/R8 low (first R7)
	vmovapd	ymm14, [srcreg+d1+32]		;; I2

	ylow128s ymm4, ymm0, ymm3		;; Shuffle R1/R2 hi and R3/R4 hi (first R2)
	yhigh128s ymm0, ymm0, ymm3		;; Shuffle R1/R2 hi and R3/R4 hi (first R4)
	vmovapd	ymm12, [srcreg+d2+32]		;; I3

	ylow128s ymm3, ymm5, ymm7		;; Shuffle R5/R6 hi and R7/R8 hi (first R6)
	yhigh128s ymm5, ymm5, ymm7		;; Shuffle R5/R6 hi and R7/R8 hi (first R8)
	vmovapd	ymm13, [srcreg+d2+d1+32]	;; I4

	vaddpd	ymm7, ymm8, ymm2		;; R1 + R5 (new R1)
	vsubpd	ymm8, ymm8, ymm2		;; R1 - R5 (new R5)
	vmovapd	ymm11, [srcreg+d4+32]		;; I5

	vaddpd	ymm2, ymm1, ymm6		;; R3 + R7 (new R3)
	vsubpd	ymm1, ymm1, ymm6		;; R3 - R7 (new R7)
	vmovapd	ymm10, [srcreg+d4+d1+32]	;; I6

	vaddpd	ymm6, ymm4, ymm3		;; R2 + R6 (new R2)
	vsubpd	ymm4, ymm4, ymm3		;; R2 - R6 (new R6)
	vmovapd	ymm9, [srcreg+d4+d2+32]		;; I7

	vaddpd	ymm3, ymm0, ymm5		;; R4 + R8 (new R4)
	vsubpd	ymm0, ymm0, ymm5		;; R4 - R8 (new R8)
	L1prefetch srcreg+L1pd, L1pt
	L1prefetchw dstreg+L1pd, L1pt - L1PREFETCH_DEST_NONE

	vaddpd	ymm5, ymm7, ymm2		;; R1 + R3 (newer R1)
	vsubpd	ymm7, ymm7, ymm2		;; R1 - R3 (newer R3)

	vaddpd	ymm2, ymm6, ymm3		;; R2 + R4 (newer R2)
	vsubpd	ymm6, ymm6, ymm3		;; R2 - R4 (newer R4)
	L1prefetch srcreg+d1+L1pd, L1pt
	L1prefetchw dstreg+e1+L1pd, L1pt - L1PREFETCH_DEST_NONE

	ystore	[dstreg], ymm5			;; Temporarily save R1
	ystore	[dstreg+e1], ymm2		;; Temporarily save R2

	vshufpd	ymm2, ymm15, ymm14, 15		;; Shuffle I1 and I2 to create I1/I2 hi
	vshufpd	ymm15, ymm15, ymm14, 0		;; Shuffle I1 and I2 to create I1/I2 low
	vmovapd	ymm14, [srcreg+d4+d2+d1+32]	;; I8

	vshufpd	ymm3, ymm12, ymm13, 15		;; Shuffle I3 and I4 to create I3/I4 hi
	vshufpd	ymm12, ymm12, ymm13, 0		;; Shuffle I3 and I4 to create I3/I4 low
	L1prefetch srcreg+d2+L1pd, L1pt
	L1prefetchw dstreg+e2+L1pd, L1pt - L1PREFETCH_DEST_NONE

	vshufpd	ymm5, ymm11, ymm10, 15		;; Shuffle I5 and I6 to create I5/I6 hi
	vshufpd	ymm11, ymm11, ymm10, 0		;; Shuffle I5 and I6 to create I5/I6 low

	vshufpd	ymm10, ymm9, ymm14, 15		;; Shuffle I7 and I8 to create I7/I8 hi
	vshufpd	ymm9, ymm9, ymm14, 0		;; Shuffle I7 and I8 to create I7/I8 low
	L1prefetch srcreg+d2+d1+L1pd, L1pt
	L1prefetchw dstreg+e2+e1+L1pd, L1pt - L1PREFETCH_DEST_NONE

	ylow128s ymm14, ymm2, ymm3		;; Shuffle I1/I2 hi and I3/I4 hi (first I2)
	ylow128s ymm13, ymm5, ymm10		;; Shuffle I5/I6 hi and I7/I8 hi (first I6)
	yhigh128s ymm2, ymm2, ymm3		;; Shuffle I1/I2 hi and I3/I4 hi (first I4)
	yhigh128s ymm5, ymm5, ymm10		;; Shuffle I5/I6 hi and I7/I8 hi (first I8)
	vmovapd	ymm3, YMM_SQRTHALF

	vsubpd	ymm10, ymm14, ymm13		;; I2 - I6 (new I6)		; 1-3 - starting clock counts here for no particularly good reason
	vaddpd	ymm14, ymm14, ymm13		;; I2 + I6 (new I2)		; 2-4

	vsubpd	ymm13, ymm2, ymm5		;; I4 - I8 (new I8)		; 3-5
	vaddpd	ymm2, ymm2, ymm5		;; I4 + I8 (new I4)		; 4-6

	ylow128s ymm5, ymm15, ymm12		;; Shuffle I1/I2 low and I3/I4 low (first I1)
	yhigh128s ymm15, ymm15, ymm12		;; Shuffle I1/I2 low and I3/I4 low (first I3)

	ylow128s ymm12, ymm11, ymm9		;; Shuffle I5/I6 low and I7/I8 low (first I5)
	yhigh128s ymm11, ymm11, ymm9		;; Shuffle I5/I6 low and I7/I8 low (first I7)

	vaddpd	ymm9, ymm10, ymm0		;; I6 + R8 (new I6)		; 5-7
	vsubpd	ymm10, ymm10, ymm0		;; I6 - R8 (new I8)		; 6-8
	L1prefetch srcreg+d4+L1pd, L1pt
	L1prefetchw dstreg+e4+L1pd, L1pt - L1PREFETCH_DEST_NONE

	vaddpd	ymm0, ymm4, ymm13		;; R6 + I8 (new R8)		; 7-9
	vsubpd	ymm4, ymm4, ymm13		;; R6 - I8 (new R6)		; 8-10
	vmulpd	ymm9, ymm9, ymm3		;; I6 = I6 * SQRTHALF		;  8-12

	vaddpd	ymm13, ymm15, ymm11		;; I3 + I7 (new I3)		; 9-11
	vmulpd	ymm10, ymm10, ymm3		;; I8 = I8 * SQRTHALF		;  9-13

	vsubpd	ymm15, ymm15, ymm11		;; I3 - I7 (new I7)		; 10-12
	vmulpd	ymm0, ymm0, ymm3		;; R8 = R8 * SQRTHALF		;  10-14

	vaddpd	ymm11, ymm5, ymm12		;; I1 + I5 (new I1)		; 11-13
	vmulpd	ymm4, ymm4, ymm3		;; R6 = R6 * SQRTHALF		;  11-15
	L1prefetch srcreg+d4+d1+L1pd, L1pt
	L1prefetchw dstreg+e4+e1+L1pd, L1pt - L1PREFETCH_DEST_NONE

	vsubpd	ymm5, ymm5, ymm12		;; I1 - I5 (new I5)		; 12-14

	vsubpd	ymm12, ymm14, ymm2		;; I2 - I4 (newer I4)		; 13-15
	vaddpd	ymm14, ymm14, ymm2		;; I2 + I4 (newer I2)		; 14-16

	vsubpd	ymm2, ymm11, ymm13		;; I1 - I3 (newer I3)		; 15-17
	vaddpd	ymm11, ymm11, ymm13		;; I1 + I3 (newer I1)		; 16-18
	L1prefetch srcreg+d4+d2+L1pd, L1pt
	L1prefetchw dstreg+e4+e2+L1pd, L1pt - L1PREFETCH_DEST_NONE

	vsubpd	ymm13, ymm7, ymm12		;; R3 - I4 (last R3)		; 17-19
	vaddpd	ymm3, ymm2, ymm6		;; I3 + R4 (last I3)		; 18-20
	vaddpd	ymm7, ymm7, ymm12		;; R3 + I4 (last R4)		; 19-21
	vsubpd	ymm2, ymm2, ymm6		;; I3 - R4 (last I4)		; 20-22

	vmovapd	ymm12, [screg+128+32]		;; cosine/sine
	vmulpd	ymm6, ymm13, ymm12		;; A3 = R3 * cosine/sine	;  20-24
	vmulpd	ymm12, ymm3, ymm12		;; B3 = I3 * cosine/sine	;  21-25
	vsubpd	ymm6, ymm6, ymm3		;; A3 = A3 - I3			; 25-27
	vaddpd	ymm12, ymm12, ymm13		;; B3 = B3 + R3			; 26-28
	vmovapd	ymm3, [screg+384+32]		;; cosine/sine

	vsubpd	ymm13, ymm0, ymm10		;; R8 - I8 (newer R8)		; 21-23
	vaddpd	ymm0, ymm0, ymm10		;; R8 + I8 (newer I8)		; 22-24
	vsubpd	ymm10, ymm4, ymm9		;; R6 - I6 (newer R6)		; 23-25
	vaddpd	ymm4, ymm4, ymm9		;; R6 + I6 (newer I6)		; 24-26
	L1prefetch srcreg+d4+d2+d1+L1pd, L1pt
	L1prefetchw dstreg+e4+e2+e1+L1pd, L1pt - L1PREFETCH_DEST_NONE

	vmulpd	ymm9, ymm7, ymm3		;; A4 = R4 * cosine/sine	;  22-26
	vmulpd	ymm3, ymm2, ymm3		;; B4 = I4 * cosine/sine	;  23-27
	vsubpd	ymm9, ymm9, ymm2		;; A4 = A4 - I4			; 27-29
	vmovapd	ymm2, [screg+128]
	vaddpd	ymm3, ymm3, ymm7		;; B4 = B4 + R4			; 28-30

	vmulpd	ymm6, ymm6, ymm2		;; A3 = A3 * sine (final R3)	;  28-32
	vmulpd	ymm12, ymm12, ymm2		;; B3 = B3 * sine (final I3)	;  29-33
	vmovapd	ymm2, [screg+384]		;; sine

	vaddpd	ymm7, ymm8, ymm15		;; R5 + I7 (newer R7)		; 29-31
	vsubpd	ymm8, ymm8, ymm15		;; R5 - I7 (newer R5)		; 30-32

	vmulpd	ymm9, ymm9, ymm2		;; A4 = A4 * sine (final R4)	;  30-34
	vmulpd	ymm3, ymm3, ymm2		;; B4 = B4 * sine (final I4)	;  31-35
	vmovapd	ymm2, [screg+192+32]		;; cosine/sine

	vsubpd	ymm15, ymm5, ymm1		;; I5 - R7 (newer I7)		; 31-33
 	vaddpd	ymm5, ymm5, ymm1		;; I5 + R7 (newer I5)		; 32-34

	vsubpd	ymm1, ymm7, ymm0		;; R7 - I8 (last R7)		; 33-35
	ystore	[dstreg+e2], ymm6		;; Save R3			; 33
	vaddpd	ymm6, ymm15, ymm13		;; I7 + R8 (last I7)		; 34-36
	ystore	[dstreg+e2+32], ymm12		;; Save I3			; 34

	vaddpd	ymm7, ymm7, ymm0		;; R7 + I8 (last R8)		; 35-37
	vmovapd	ymm12, [screg+448+32]		;; cosine/sine

	vsubpd	ymm15, ymm15, ymm13		;; I7 - R8 (last I8)		; 36-38
	vmulpd	ymm13, ymm1, ymm2		;; A7 = R7 * cosine/sine	;  36-40
	ystore	[dstreg+e2+e1], ymm9		;; Save R4			; 35

	vsubpd	ymm9, ymm8, ymm10		;; R5 - R6 (last R6)		; 37-39
	vmulpd	ymm2, ymm6, ymm2		;; B7 = I7 * cosine/sine	;  37-41
	ystore	[dstreg+e2+e1+32], ymm3		;; Save I4			; 36

	vsubpd	ymm3, ymm5, ymm4		;; I5 - I6 (last I6)		; 38-40
	vmulpd	ymm0, ymm7, ymm12		;; A8 = R8 * cosine/sine	;  38-42

	vaddpd	ymm8, ymm8, ymm10		;; R5 + R6 (last R5)		; 39-41
	vmovapd	ymm10, [screg+320+32]		;; cosine/sine			
	vmulpd	ymm12, ymm15, ymm12		;; B8 = I8 * cosine/sine	;  39-43

	vaddpd	ymm5, ymm5, ymm4		;; I5 + I6 (last I5)		; 40-42
	vmulpd	ymm4, ymm9, ymm10		;; A6 = R6 * cosine/sine	;  40-44

	vsubpd	ymm13, ymm13, ymm6		;; A7 = A7 - I7			; 41-43
	vmovapd	ymm6, [screg+64+32]		;; cosine/sine
	vmulpd	ymm10, ymm3, ymm10		;; B6 = I6 * cosine/sine	;  41-45

	vaddpd	ymm2, ymm2, ymm1		;; B7 = B7 + R7			; 42-44
	vmulpd	ymm1, ymm8, ymm6		;; A5 = R5 * cosine/sine	;  42-46

	vsubpd	ymm0, ymm0, ymm15		;; A8 = A8 - I8			; 43-45
	vmovapd	ymm15, [screg+192]		;; sine
	vmulpd	ymm6, ymm5, ymm6		;; B5 = I5 * cosine/sine	;  43-47

	vaddpd	ymm12, ymm12, ymm7		;; B8 = B8 + R8			; 44-46
	vmovapd	ymm7, [dstreg]			;; Reload R1
	vmulpd	ymm13, ymm13, ymm15		;; A7 = A7 * sine (final R7)	;  44-48
	ystore	[dstreg+e4+e2], ymm13		;; Save R7			; 49

	vmovapd	ymm13, [dstreg+e1]		;; Reload R2
	vmulpd	ymm2, ymm2, ymm15		;; B7 = B7 * sine (final I7)	;  45-49
	vsubpd	ymm15, ymm7, ymm13		;; R1 - R2 (last R2)		; 45-47
	ystore	[dstreg+e4+e2+32], ymm2		;; Save I7			; 50

	vmovapd	ymm2, [screg+448]		;; sine
	vmulpd	ymm0, ymm0, ymm2		;; A8 = A8 * sine (final R8)	;  46-50
	ystore	[dstreg+e4+e2+e1], ymm0		;; Save R8			; 51
	vsubpd	ymm0, ymm11, ymm14		;; I1 - I2 (last I2)		; 46-48

	vaddpd	ymm7, ymm7, ymm13		;; R1 + R2 (last R1)		; 47-49
	vmovapd	ymm13, [screg+256+32]		;; cosine/sine
	vmulpd	ymm12, ymm12, ymm2		;; B8 = B8 * sine (final I8)	;  47-51

	vaddpd	ymm11, ymm11, ymm14		;; I1 + I2 (last I1)		; 48-50
	vmulpd	ymm14, ymm15, ymm13		;; A2 = R2 * cosine/sine	;  48-52
	vmovapd	ymm2, [screg+0+32]		;; cosine/sine

	vsubpd	ymm4, ymm4, ymm3		;; A6 = A6 - I6			; 49-51
	vmulpd	ymm13, ymm0, ymm13		;; B2 = I2 * cosine/sine	;  49-53
	vmovapd	ymm3, [screg+320]		;; sine

	vaddpd	ymm10, ymm10, ymm9		;; B6 = B6 + R6			; 50-52
	vmulpd	ymm9, ymm7, ymm2		;; A1 = R1 * cosine/sine	;  50-54

	vsubpd	ymm1, ymm1, ymm5		;; A5 = A5 - I5			; 51-53
	vmulpd	ymm2, ymm11, ymm2		;; B1 = I1 * cosine/sine	;  51-55
	vmovapd	ymm5, [screg+64]		;; sine

	vaddpd	ymm6, ymm6, ymm8		;; B5 = B5 + R5			; 52-54
	vmulpd	ymm4, ymm4, ymm3		;; A6 = A6 * sine (final R6)	;  52-56
	vmovapd	ymm8, [screg+256]		;; sine

	vsubpd	ymm14, ymm14, ymm0		;; A2 = A2 - I2			; 53-55
	vmulpd	ymm10, ymm10, ymm3		;; B6 = B6 * sine (final I6)	;  53-57
	vmovapd	ymm0, [screg+0]			;; sine

	vaddpd	ymm13, ymm13, ymm15		;; B2 = B2 + R2			; 54-56
	vmulpd	ymm1, ymm1, ymm5		;; A5 = A5 * sine (final R5)	;  54-58
	ystore	[dstreg+e4+e2+e1+32], ymm12	;; Save I8			; 52

	vsubpd	ymm9, ymm9, ymm11		;; A1 = A1 - I1			; 55-57
	vmulpd	ymm6, ymm6, ymm5		;; B5 = B5 * sine (final I5)	;  55-59

	vaddpd	ymm2, ymm2, ymm7		;; B1 = B1 + R1			; 56-58
	vmulpd	ymm14, ymm14, ymm8		;; A2 = A2 * sine (final R2)	;  56-60

	vmulpd	ymm13, ymm13, ymm8		;; B2 = B2 * sine (final I2)	;  57-61
	ystore	[dstreg+e4+e1], ymm4		;; Save R6			; 57

	vmulpd	ymm9, ymm9, ymm0		;; A1 = A1 * sine (final R1)	;  58-62
	ystore	[dstreg+e4+e1+32], ymm10	;; Save I6			; 58

	vmulpd	ymm2, ymm2, ymm0		;; B1 = B1 * sine (final I1)	;  59-63
	ystore	[dstreg+e4], ymm1		;; Save R5			; 59

	ystore	[dstreg+e4+32], ymm6		;; Save I5			; 60
	ystore	[dstreg+e1], ymm14		;; Save R2			; 61
	ystore	[dstreg+e1+32], ymm13		;; Save I2			; 62
	ystore	[dstreg], ymm9			;; Save R1			; 63
	ystore	[dstreg+32], ymm2		;; Save I1			; 64

	bump	srcreg, srcinc
	bump	dstreg, dstinc
	bump	screg, scinc
	ENDM

;; Haswell FMA3 version

;;IFDEF ORIGINAL_SWIZZLE_CODE
IF (@INSTR(,%yarch,<FMA3>) NE 0)

yr8_sg8cl_eight_complex_fft8 MACRO srcreg,srcinc,d1,d2,d4,dstreg,dstinc,e1,e2,e4,screg,scinc,maxrpt,L1pt,L1pd

	;; On input the 64-byte cache lines hold these data values:
	;;	0	+1K	+1K	+1K	8K	+1K	+1K	+1K
	;;	1	...
	;;	2	...
	;;	3	...

	;; These are swizzled to:
	;;	0	+1	+1	+1	8K	+1	+1	+1
	;;	1K	...
	;;	2K	...
	;;	3K	...

	;; Swizzle using 8 shufpds and 8 perm2f128  (1 or 3 clocks each on port 5, throughput 1)

	vmovapd	ymm1, [srcreg]			;; R1
	vmovapd	ymm2, [srcreg+d1]		;; R2
	vshufpd	ymm0, ymm1, ymm2, 15		;; Shuffle R1 and R2 to create R1/R2 hi		; 1
	vshufpd	ymm1, ymm1, ymm2, 0		;; Shuffle R1 and R2 to create R1/R2 low	; 2

	vmovapd	ymm3, [srcreg+d2]		;; R3
	vmovapd	ymm4, [srcreg+d2+d1]		;; R4
	vshufpd	ymm2, ymm3, ymm4, 15		;; Shuffle R3 and R4 to create R3/R4 hi		; 3
	vshufpd	ymm3, ymm3, ymm4, 0		;; Shuffle R3 and R4 to create R3/R4 low	; 4

	vmovapd	ymm5, [srcreg+d4]		;; R5
	vmovapd	ymm6, [srcreg+d4+d1]		;; R6
	vshufpd	ymm4, ymm5, ymm6, 15		;; Shuffle R5 and R6 to create R5/R6 hi		; 5
	vshufpd	ymm5, ymm5, ymm6, 0		;; Shuffle R5 and R6 to create R5/R6 low	; 6

	vmovapd	ymm7, [srcreg+d4+d2]		;; R7
	vmovapd	ymm8, [srcreg+d4+d2+d1]		;; R8
	vshufpd	ymm6, ymm7, ymm8, 15		;; Shuffle R7 and R8 to create R7/R8 hi		; 7
	vshufpd	ymm7, ymm7, ymm8, 0		;; Shuffle R7 and R8 to create R7/R8 low	; 8

	ylow128s ymm8, ymm1, ymm3		;; Shuffle R1/R2 low and R3/R4 low (first R1)	; 9-10		n 13
	yhigh128s ymm1, ymm1, ymm3		;; Shuffle R1/R2 low and R3/R4 low (first R3)	; 10-11		n 14
	vmovapd	ymm15, YMM_ONE

	ylow128s ymm3, ymm5, ymm7		;; Shuffle R5/R6 low and R7/R8 low (first R5)	; 11-12		n 13
	yhigh128s ymm5, ymm5, ymm7		;; Shuffle R5/R6 low and R7/R8 low (first R7)	; 12-13		n 14
	vmovapd	ymm9, [srcreg+32]		;; I1

	ylow128s ymm7, ymm0, ymm2		;; Shuffle R1/R2 hi and R3/R4 hi (first R2)	; 13-14		n 17
	yfmaddpd ymm11, ymm8, ymm15, ymm3	;; R1 + R5 (new R1)				; 13-17		n 19
	yfmsubpd ymm8, ymm8, ymm15, ymm3	;; R1 - R5 (new R5)				; 13-17		n 39
	vmovapd	ymm10, [srcreg+d1+32]		;; I2

	yhigh128s ymm0, ymm0, ymm2		;; Shuffle R1/R2 hi and R3/R4 hi (first R4)	; 14-15		n 18
	yfmaddpd ymm2, ymm1, ymm15, ymm5	;; R3 + R7 (new R3)				; 14-18		n 19
	yfmsubpd ymm1, ymm1, ymm15, ymm5	;; R3 - R7 (new R7)				; 14-18		n 41

	ylow128s ymm3, ymm4, ymm6		;; Shuffle R5/R6 hi and R7/R8 hi (first R6)	; 15-16		n 17
	vmovapd	ymm5, [srcreg+d2+32]		;; I3

	yhigh128s ymm4, ymm4, ymm6		;; Shuffle R5/R6 hi and R7/R8 hi (first R8)	; 16-17		n 18
	vmovapd	ymm12, [srcreg+d2+d1+32]	;; I4

	yfmaddpd ymm6, ymm7, ymm15, ymm3	;; R2 + R6 (new R2)				; 17-21		n 23
	yfmsubpd ymm7, ymm7, ymm15, ymm3	;; R2 - R6 (new R6)				; 17-21		n 37
	vmovapd	ymm13, [srcreg+d4+32]		;; I5

	vshufpd	ymm3, ymm9, ymm10, 15		;; Shuffle I1 and I2 to create I1/I2 hi		; 18
	yfmaddpd ymm14, ymm0, ymm15, ymm4	;; R4 + R8 (new R4)				; 18-22		n 23
	yfmsubpd ymm0, ymm0, ymm15, ymm4	;; R4 - R8 (new R8)				; 18-22		n 36

	vshufpd	ymm9, ymm9, ymm10, 0		;; Shuffle I1 and I2 to create I1/I2 low	; 19
	yfmaddpd ymm4, ymm11, ymm15, ymm2	;; R1 + R3 (newer R1)				; 19-23		n 28
	yfmsubpd ymm11, ymm11, ymm15, ymm2	;; R1 - R3 (newer R3)				; 19-23		n 44
	vmovapd	ymm10, [srcreg+d4+d1+32]	;; I6

	vshufpd	ymm2, ymm5, ymm12, 15		;; Shuffle I3 and I4 to create I3/I4 hi		; 20
	vshufpd	ymm5, ymm5, ymm12, 0		;; Shuffle I3 and I4 to create I3/I4 low	; 21
	L1prefetch srcreg+L1pd, L1pt
	L1prefetchw dstreg+L1pd, L1pt - L1PREFETCH_DEST_NONE

	vshufpd	ymm12, ymm13, ymm10, 15		;; Shuffle I5 and I6 to create I5/I6 hi		; 22

	vshufpd	ymm13, ymm13, ymm10, 0		;; Shuffle I5 and I6 to create I5/I6 low	; 23
	yfmaddpd ymm10, ymm6, ymm15, ymm14	;; R2 + R4 (newer R2)				; 23-27		n 28
	yfmsubpd ymm6, ymm6, ymm15, ymm14	;; R2 - R4 (newer R4)				; 23-27		n 45

	yfmsubpd ymm14, ymm4, ymm15, ymm10	;; R1 - R2 (last R2)				; 28-32
	yfmaddpd ymm4, ymm4, ymm15, ymm10	;; R1 + R2 (last R1)				; 28-32

	vmovapd	ymm10, [srcreg+d4+d2+32]	;; I7
	ystore	[dstreg+e1], ymm14		;; Temporarily save R2				; 33
	vmovapd	ymm14, [srcreg+d4+d2+d1+32]	;; I8
	ystore	[dstreg], ymm4			;; Temporarily save R1				; 33+1
	vshufpd	ymm4, ymm10, ymm14, 15		;; Shuffle I7 and I8 to create I7/I8 hi		; 24
	vshufpd	ymm10, ymm10, ymm14, 0		;; Shuffle I7 and I8 to create I7/I8 low	; 25

	ylow128s ymm14, ymm3, ymm2		;; Shuffle I1/I2 hi and I3/I4 hi (first I2)	; 26-27		n 30
	yhigh128s ymm3, ymm3, ymm2		;; Shuffle I1/I2 hi and I3/I4 hi (first I4)	; 27-28		n 31

	ylow128s ymm2, ymm12, ymm4		;; Shuffle I5/I6 hi and I7/I8 hi (first I6)	; 28-29		n 30
	yhigh128s ymm12, ymm12, ymm4		;; Shuffle I5/I6 hi and I7/I8 hi (first I8)	; 29-30		n 31
	L1prefetch srcreg+d1+L1pd, L1pt
	L1prefetchw dstreg+e1+L1pd, L1pt - L1PREFETCH_DEST_NONE

	yfmsubpd ymm4, ymm14, ymm15, ymm2	;; I2 - I6 (new I6)				; 30-34		n 36
	yfmaddpd ymm14, ymm14, ymm15, ymm2	;; I2 + I6 (new I2)				; 30-34		n 38
	yhigh128s ymm2, ymm9, ymm5		;; Shuffle I1/I2 low and I3/I4 low (first I3)	; 30-31		n 34

	ylow128s ymm9, ymm9, ymm5		;; Shuffle I1/I2 low and I3/I4 low (first I1)	; 31-32		n 35
	yfmsubpd ymm5, ymm3, ymm15, ymm12	;; I4 - I8 (new I8)				; 31-35		n 37
	yfmaddpd ymm3, ymm3, ymm15, ymm12	;; I4 + I8 (new I4)				; 31-35		n 38

	yhigh128s ymm12, ymm13, ymm10		;; Shuffle I5/I6 low and I7/I8 low (first I7)	; 32-33		n 34
	ylow128s ymm13, ymm13, ymm10		;; Shuffle I5/I6 low and I7/I8 low (first I5)	; 33-34		n 35
	L1prefetch srcreg+d2+L1pd, L1pt
	L1prefetchw dstreg+e2+L1pd, L1pt - L1PREFETCH_DEST_NONE

	yfmaddpd ymm10, ymm2, ymm15, ymm12	;; I3 + I7 (new I3)				; 34-38		n 40
	yfmsubpd ymm2, ymm2, ymm15, ymm12	;; I3 - I7 (new I7)				; 34-38		n 39

	yfmaddpd ymm12, ymm9, ymm15, ymm13	;; I1 + I5 (new I1)				; 35-39		n 40
	yfmsubpd ymm9, ymm9, ymm15, ymm13	;; I1 - I5 (new I5)				; 35-39		n 41
	L1prefetch srcreg+d2+d1+L1pd, L1pt
	L1prefetchw dstreg+e2+e1+L1pd, L1pt - L1PREFETCH_DEST_NONE

	yfmsubpd ymm13, ymm4, ymm15, ymm0	;; I6 - R8 (new2 I8)				; 36-40		n 42
	yfmaddpd ymm4, ymm4, ymm15, ymm0	;; I6 + R8 (new2 I6)				; 36-40		n 43

	yfmaddpd ymm0, ymm7, ymm15, ymm5	;; R6 + I8 (new2 R8)				; 37-41		n 42
	yfmsubpd ymm7, ymm7, ymm15, ymm5	;; R6 - I8 (new2 R6)				; 37-41		n 43

	yfmsubpd ymm5, ymm14, ymm15, ymm3	;; I2 - I4 (newer I4)				; 38-42		n 44
	yfmaddpd ymm14, ymm14, ymm15, ymm3	;; I2 + I4 (newer I2)				; 38-42		n 46
	L1prefetch srcreg+d4+L1pd, L1pt
	L1prefetchw dstreg+e4+L1pd, L1pt - L1PREFETCH_DEST_NONE

	yfmaddpd ymm3, ymm8, ymm15, ymm2	;; R5 + I7 (newer R7)				; 39-43		n 47
	yfmsubpd ymm8, ymm8, ymm15, ymm2	;; R5 - I7 (newer R5)				; 39-43		n 49

	yfmsubpd ymm2, ymm12, ymm15, ymm10	;; I1 - I3 (newer I3)				; 40-44		n 45
	yfmaddpd ymm12, ymm12, ymm15, ymm10	;; I1 + I3 (newer I1)				; 40-44		n 46

	yfmsubpd ymm10, ymm9, ymm15, ymm1	;; I5 - R7 (newer I7)				; 41-45		n 48
 	yfmaddpd ymm9, ymm9, ymm15, ymm1	;; I5 + R7 (newer I5)				; 41-45		n 50
	L1prefetch srcreg+d4+d1+L1pd, L1pt
	L1prefetchw dstreg+e4+e1+L1pd, L1pt - L1PREFETCH_DEST_NONE

	yfmaddpd ymm1, ymm0, ymm15, ymm13	;; R8 + I8 (newer I8/SQRTHALF)			; 42-46		n 47
	yfmsubpd ymm0, ymm0, ymm15, ymm13	;; R8 - I8 (newer R8/SQRTHALF)			; 42-46		n 48

	yfmsubpd ymm13, ymm7, ymm15, ymm4	;; R6 - I6 (newer R6/SQRTHALF)			; 43-47		n 49
	yfmaddpd ymm7, ymm7, ymm15, ymm4	;; R6 + I6 (newer I6/SQRTHALF)			; 43-47		n 50

	yfmsubpd ymm4, ymm11, ymm15, ymm5	;; R3 - I4 (last R3)				; 44-48		n 51
	yfmaddpd ymm11, ymm11, ymm15, ymm5	;; R3 + I4 (last R4)				; 44-48		n 52

	yfmaddpd ymm5, ymm2, ymm15, ymm6	;; I3 + R4 (last I3)				; 45-49		n 51
	yfmsubpd ymm2, ymm2, ymm15, ymm6	;; I3 - R4 (last I4)				; 45-49		n 52
	L1prefetch srcreg+d4+d2+L1pd, L1pt
	L1prefetchw dstreg+e4+e2+L1pd, L1pt - L1PREFETCH_DEST_NONE

	yfmsubpd ymm6, ymm12, ymm15, ymm14	;; I1 - I2 (last I2)				; 46-50		n 59
	yfmaddpd ymm12, ymm12, ymm15, ymm14	;; I1 + I2 (last I1)				; 46-50		n 60

	vmovapd	ymm15, YMM_SQRTHALF
	yfnmaddpd ymm14, ymm1, ymm15, ymm3	;; R7 - I8 * SQRTHALF (last R7)			; 47-51		n 53
	yfmaddpd ymm1, ymm1, ymm15, ymm3	;; R7 + I8 * SQRTHALF (last R8)			; 47-51		n 54

	yfmaddpd ymm3, ymm0, ymm15, ymm10	;; I7 + R8 * SQRTHALF (last I7)			; 48-52		n 53
	yfnmaddpd ymm0, ymm0, ymm15, ymm10	;; I7 - R8 * SQRTHALF (last I8)			; 48-52		n 54

	yfnmaddpd ymm10, ymm13, ymm15, ymm8	;; R5 - R6 * SQRTHALF (last R6)			; 49-53		n 55
	yfmaddpd ymm13, ymm13, ymm15, ymm8	;; R5 + R6 * SQRTHALF (last R5)			; 49-53		n 56
	L1prefetch srcreg+d4+d2+d1+L1pd, L1pt
	L1prefetchw dstreg+e4+e2+e1+L1pd, L1pt - L1PREFETCH_DEST_NONE

	yfnmaddpd ymm8, ymm7, ymm15, ymm9	;; I5 - I6 * SQRTHALF (last I6)			; 50-54		n 55
	yfmaddpd ymm7, ymm7, ymm15, ymm9	;; I5 + I6 * SQRTHALF (last I5)			; 50-54		n 56
	bump	srcreg, srcinc

	vmovapd	ymm15, [screg+128+32]		;; cosine/sine for R3/I3
	yfmsubpd ymm9, ymm4, ymm15, ymm5	;; A3 = R3 * cosine/sine - I3			; 51-55
	yfmaddpd ymm5, ymm5, ymm15, ymm4	;; B3 = I3 * cosine/sine + R3			; 51-55

	vmovapd	ymm15, [screg+384+32]		;; cosine/sine for R4/I4
	yfmsubpd ymm4, ymm11, ymm15, ymm2	;; A4 = R4 * cosine/sine - I4			; 52-56
	yfmaddpd ymm2, ymm2, ymm15, ymm11	;; B4 = I4 * cosine/sine + R4			; 52-56

	vmovapd	ymm15, [screg+192+32]		;; cosine/sine for R7/I7
	yfmsubpd ymm11, ymm14, ymm15, ymm3	;; A7 = R7 * cosine/sine - I7			; 53-57
	yfmaddpd ymm3, ymm3, ymm15, ymm14	;; B7 = I7 * cosine/sine + R7			; 53-57

	vmovapd	ymm15, [screg+448+32]		;; cosine/sine for R8/I8
	yfmsubpd ymm14, ymm1, ymm15, ymm0	;; A8 = R8 * cosine/sine - I8			; 54-58
	yfmaddpd ymm0, ymm0, ymm15, ymm1	;; B8 = I8 * cosine/sine + R8			; 54-58

	vmovapd	ymm15, [screg+320+32]		;; cosine/sine for R6/I6
	yfmsubpd ymm1, ymm10, ymm15, ymm8	;; A6 = R6 * cosine/sine - I6			; 55-59
	yfmaddpd ymm8, ymm8, ymm15, ymm10	;; B6 = I6 * cosine/sine + R6			; 55-59

	vmovapd	ymm15, [screg+64+32]		;; cosine/sine for R5/I5
	yfmsubpd ymm10, ymm13, ymm15, ymm7	;; A5 = R5 * cosine/sine - I5			; 56-60
	yfmaddpd ymm7, ymm7, ymm15, ymm13	;; B5 = I5 * cosine/sine + R5			; 56-60

	vmovapd	ymm15, [screg+128]		;; sine for R3/I3
	vmulpd	ymm9, ymm9, ymm15		;; A3 = A3 * sine (final R3)			; 57-61
	vmulpd	ymm5, ymm5, ymm15		;; B3 = B3 * sine (final I3)			; 57-61

	vmovapd	ymm15, [screg+384]		;; sine for R4/I4
	vmulpd	ymm4, ymm4, ymm15		;; A4 = A4 * sine (final R4)			; 58-62
	vmulpd	ymm2, ymm2, ymm15		;; B4 = B4 * sine (final I4)			; 58-62

	vmovapd	ymm15, [screg+256+32]		;; cosine/sine for R2/I2
	vmovapd	ymm13, [dstreg+e1]		;; Reload last R2
	ystore	[dstreg+e2], ymm9		;; Save R3					; 62
	yfmsubpd ymm9, ymm13, ymm15, ymm6	;; A2 = R2 * cosine/sine - I2			; 59-63
	yfmaddpd ymm6, ymm6, ymm15, ymm13	;; B2 = I2 * cosine/sine + R2			; 59-63

	vmovapd	ymm15, [screg+0+32]		;; cosine/sine for R1/I1
	vmovapd	ymm13, [dstreg]			;; Reload last R1
	ystore	[dstreg+e2+32], ymm5		;; Save I3					; 62+1
	yfmsubpd ymm5, ymm13, ymm15, ymm12	;; A1 = R1 * cosine/sine - I1			; 60-64
	yfmaddpd ymm12, ymm12, ymm15, ymm13	;; B1 = I1 * cosine/sine + R1			; 60-64

	vmovapd	ymm15, [screg+192]		;; sine for R7/I7
	vmulpd	ymm11, ymm11, ymm15		;; A7 = A7 * sine (final R7)			; 61-65
	vmulpd	ymm3, ymm3, ymm15		;; B7 = B7 * sine (final I7)			; 61-65

	vmovapd	ymm15, [screg+448]		;; sine for R8/I8
	vmulpd	ymm14, ymm14, ymm15		;; A8 = A8 * sine (final R8)			; 62-66
	vmulpd	ymm0, ymm0, ymm15		;; B8 = B8 * sine (final I8)			; 62-66

	vmovapd	ymm15, [screg+320]		;; sine for R6/I6
	vmulpd	ymm1, ymm1, ymm15		;; A6 = A6 * sine (final R6)			; 63-67
	vmulpd	ymm8, ymm8, ymm15		;; B6 = B6 * sine (final I6)			; 63-67

	vmovapd	ymm15, [screg+64]		;; sine for R5/I5
	vmulpd	ymm10, ymm10, ymm15		;; A5 = A5 * sine (final R5)			; 64-68
	vmulpd	ymm7, ymm7, ymm15		;; B5 = B5 * sine (final I5)			; 64-68
	ystore	[dstreg+e2+e1], ymm4		;; Save R4					; 63+1

	vmovapd	ymm15, [screg+256]		;; sine for R2/I2
	vmulpd	ymm9, ymm9, ymm15		;; A2 = A2 * sine (final R2)			; 65-69
	vmulpd	ymm6, ymm6, ymm15		;; B2 = B2 * sine (final I2)			; 65-69
	ystore	[dstreg+e2+e1+32], ymm2		;; Save I4					; 63+2

	vmovapd	ymm15, [screg+0]		;; sine for R1/I1
	vmulpd	ymm5, ymm5, ymm15		;; A1 = A1 * sine (final R1)			; 66-70
	vmulpd	ymm12, ymm12, ymm15		;; B1 = B1 * sine (final I1)			; 66-70
	ystore	[dstreg+e4+e2], ymm11		;; Save R7					; 66
	bump	screg, scinc

	ystore	[dstreg+e4+e2+32], ymm3		;; Save I7					; 66+1
	ystore	[dstreg+e4+e2+e1], ymm14	;; Save R8					; 67+1
	ystore	[dstreg+e4+e2+e1+32], ymm0	;; Save I8					; 67+2
	ystore	[dstreg+e4+e1], ymm1		;; Save R6					; 68+2
	ystore	[dstreg+e4+e1+32], ymm8		;; Save I6					; 68+3
	ystore	[dstreg+e4], ymm10		;; Save R5					; 69+3
	ystore	[dstreg+e4+32], ymm7		;; Save I5					; 69+4
	ystore	[dstreg+e1], ymm9		;; Save R2					; 70+4
	ystore	[dstreg+e1+32], ymm6		;; Save I2					; 70+5
	ystore	[dstreg], ymm5			;; Save R1					; 71+5
	ystore	[dstreg+32], ymm12		;; Save I1					; 71+6

	bump	dstreg, dstinc
	ENDM
ENDIF
;;ENDIF

;; Haswell FMA3 version --- On Skylake, new sizzle code is between zero and two clocks faster (within our benchmarking margin of error)
;; This code makes much less use of port 5, but increases use of the port 2&3 load units.

IFDEF NEW_SWIZZLE_CODE
IF (@INSTR(,%yarch,<FMA3>) NE 0)

yr8_sg8cl_eight_complex_fft8 MACRO srcreg,srcinc,d1,d2,d4,dstreg,dstinc,e1,e2,e4,screg,scinc,maxrpt,L1pt,L1pd

	;; On input the 64-byte cache lines hold these data values:
	;;	0	+1K	+1K	+1K	8K	+1K	+1K	+1K
	;;	1	...
	;;	2	...
	;;	3	...

	;; These are swizzled to:
	;;	0	+1	+1	+1	8K	+1	+1	+1
	;;	1K	...
	;;	2K	...
	;;	3K	...

	;; Swizzle using 8 shufpds and 8 perm2f128  (1 or 3 clocks each on port 5, throughput 1)

	vmovapd	xmm1, [srcreg]					;; R1 low
	vmovapd	xmm7, [srcreg+d1]				;; R2 low
	vinsertf128 ymm1, ymm1, [srcreg+d2], 1			;; R3/R1 low
	vinsertf128 ymm7, ymm7, [srcreg+d2+d1], 1		;; R4/R2 low
	vshufpd	ymm0, ymm1, ymm7, 0		;; Shuffle R3/R1 and R4/R2 lows (first R1)	; 1
	vshufpd	ymm1, ymm1, ymm7, 15		;; Shuffle R3/R1 and R4/R2 lows (first R2)	; 2

	vmovapd	xmm3, [srcreg+d4]				;; R5 low
	vmovapd	xmm7, [srcreg+d4+d1]				;; R6 low
	vinsertf128 ymm3, ymm3, [srcreg+d4+d2], 1		;; R7/R5 low
	vinsertf128 ymm7, ymm7, [srcreg+d4+d2+d1], 1		;; R8/R6 low
	vshufpd	ymm2, ymm3, ymm7, 0		;; Shuffle R7/R5 and R8/R6 lows (first R5)	; 3
	vshufpd	ymm3, ymm3, ymm7, 15		;; Shuffle R7/R5 and R8/R6 lows (first R6)	; 4

	vmovapd	xmm5, [srcreg][16]				;; R1 hi
	vmovapd	xmm7, [srcreg+d1][16]				;; R2 hi
	vinsertf128 ymm5, ymm5, [srcreg+d2][16], 1		;; R3/R1 hi
	vinsertf128 ymm7, ymm7, [srcreg+d2+d1][16], 1		;; R4/R2 hi
	vshufpd	ymm4, ymm5, ymm7, 0		;; Shuffle R3/R1 hi and R4/R2 hi (first R3)	; 5
	vshufpd	ymm5, ymm5, ymm7, 15		;; Shuffle R3/R1 hi and R4/R2 hi (first R4)	; 6

	vmovapd	xmm7, [srcreg+d4][16]				;; R5 hi
	vmovapd	xmm8, [srcreg+d4+d1][16]			;; R6 hi
	vinsertf128 ymm7, ymm7, [srcreg+d4+d2][16], 1		;; R7/R5 hi
	vinsertf128 ymm8, ymm8, [srcreg+d4+d2+d1][16], 1	;; R8/R6 hi
	vshufpd	ymm6, ymm7, ymm8, 0		;; Shuffle R7/R5 hi and R8/R6 hi (first R7)	; 7
	vshufpd	ymm7, ymm7, ymm8, 15		;; Shuffle R7/R5 hi and R8/R6 hi (first R8)	; 8

	vmovapd	ymm15, YMM_ONE
	yfmaddpd ymm8, ymm0, ymm15, ymm2	;; R1 + R5 (new R1)				; 4-8
	yfmsubpd ymm0, ymm0, ymm15, ymm2	;; R1 - R5 (new R5)				; 4-8
	vmovapd	xmm10, [srcreg+32]				;; I1 low

	yfmaddpd ymm2, ymm1, ymm15, ymm3	;; R2 + R6 (new R2)				; 5-9
	yfmsubpd ymm1, ymm1, ymm15, ymm3	;; R2 - R6 (new R6)				; 5-9
	vmovapd	xmm9, [srcreg+d1+32]				;; I2 low

	yfmaddpd ymm3, ymm4, ymm15, ymm6	;; R3 + R7 (new R3)				; 8-12
	yfmsubpd ymm4, ymm4, ymm15, ymm6	;; R3 - R7 (new R7)				; 8-12
	vinsertf128 ymm10, ymm10, [srcreg+d2+32], 1		;; I3/I1 low

	yfmaddpd ymm6, ymm5, ymm15, ymm7	;; R4 + R8 (new R4)				; 9-13
	yfmsubpd ymm5, ymm5, ymm15, ymm7	;; R4 - R8 (new R8)				; 9-13
	vinsertf128 ymm9, ymm9, [srcreg+d2+d1+32], 1		;; I4/I2 low

	vshufpd	ymm7, ymm10, ymm9, 15		;; Shuffle I3/I1 and I4/I2 lows (first I2)	; 9
	vshufpd	ymm10, ymm10, ymm9, 0		;; Shuffle I3/I1 and I4/I2 lows (first I1)	; 10

	vmovapd	xmm11, [srcreg+d4+32]				;; I5 low
	vmovapd	xmm12, [srcreg+d4+d1+32]			;; I6 low
	vinsertf128 ymm11, ymm11, [srcreg+d4+d2+32], 1		;; I7/I5 low
	vinsertf128 ymm12, ymm12, [srcreg+d4+d2+d1+32], 1	;; I8/I6 low
	vshufpd	ymm9, ymm11, ymm12, 15		;; Shuffle I7/I5 and I8/I6 lows (first I6)	; 11
	vshufpd	ymm11, ymm11, ymm12, 0		;; Shuffle I7/I5 and I8/I6 lows (first I5)	; 12

	yfmaddpd ymm12, ymm8, ymm15, ymm3	;; R1 + R3 (newer R1)				; 13-17
	yfmsubpd ymm8, ymm8, ymm15, ymm3	;; R1 - R3 (newer R3)				; 13-17
	vmovapd	xmm13, [srcreg+32][16]				;; I1 hi

	yfmaddpd ymm3, ymm2, ymm15, ymm6	;; R2 + R4 (newer R2)				; 14-18
	yfmsubpd ymm2, ymm2, ymm15, ymm6	;; R2 - R4 (newer R4)				; 14-18
	vmovapd	xmm14, [srcreg+d1+32][16]			;; I2 hi

	yfmsubpd ymm6, ymm7, ymm15, ymm9	;; I2 - I6 (new I6)				; 12-16
	yfmaddpd ymm7, ymm7, ymm15, ymm9	;; I2 + I6 (new I2)				; 12-16

	vinsertf128 ymm13, ymm13, [srcreg+d2+32][16], 1		;; I3/I1 hi
	vinsertf128 ymm14, ymm14, [srcreg+d2+d1+32][16], 1	;; I4/I2 hi
	vshufpd	ymm9, ymm13, ymm14, 15		;; Shuffle I3/I1 hi and I4/I2 hi (first I4)	; 13
	vshufpd	ymm13, ymm13, ymm14, 0		;; Shuffle I3/I1 hi and I4/I2 hi (first I3)	; 14

	yfmaddpd ymm14, ymm10, ymm15, ymm11	;; I1 + I5 (new I1)				; 15-19
	yfmsubpd ymm10, ymm10, ymm15, ymm11	;; I1 - I5 (new I5)				; 15-19

	yfmsubpd ymm11, ymm12, ymm15, ymm3	;; R1 - R2 (last R2)				; 19-23
	yfmaddpd ymm12, ymm12, ymm15, ymm3	;; R1 + R2 (last R1)				; 19-23
	vmovapd	xmm3, [srcreg+d4+32][16]			;; I5 hi

	ystore	[dstreg+e1], ymm11		;; Temporarily save R2				; 24
	vmovapd	xmm11, [srcreg+d4+d1+32][16]			;; I6 hi
	vinsertf128 ymm3, ymm3, [srcreg+d4+d2+32][16], 1	;; I7/I5 hi
	vinsertf128 ymm11, ymm11, [srcreg+d4+d2+d1+32][16], 1	;; I8/I6 hi
	ystore	[dstreg], ymm12			;; Temporarily save R1				; 24+1
	vshufpd	ymm12, ymm3, ymm11, 15		;; Shuffle I7/I5 hi and I8/I6 hi (first I8)	; 15
	vshufpd	ymm3, ymm3, ymm11, 0		;; Shuffle I7/I5 hi and I8/I6 hi (first I7)	; 16

	yfmsubpd ymm11, ymm9, ymm15, ymm12	;; I4 - I8 (new I8)				; 16-20
	yfmaddpd ymm9, ymm9, ymm15, ymm12	;; I4 + I8 (new I4)				; 16-20
	L1prefetch srcreg+L1pd, L1pt

	yfmsubpd ymm12, ymm13, ymm15, ymm3	;; I3 - I7 (new I7)				; 17-21
	yfmaddpd ymm13, ymm13, ymm15, ymm3	;; I3 + I7 (new I3)				; 17-21
	L1prefetchw dstreg+L1pd, L1pt - L1PREFETCH_DEST_NONE

	yfmsubpd ymm3, ymm6, ymm15, ymm5	;; I6 - R8 (new2 I8)				; 18-22
	yfmaddpd ymm6, ymm6, ymm15, ymm5	;; I6 + R8 (new2 I6)				; 18-22
	L1prefetch srcreg+d1+L1pd, L1pt

	yfmsubpd ymm5, ymm10, ymm15, ymm4	;; I5 - R7 (newer I7)				; 20-24
 	yfmaddpd ymm10, ymm10, ymm15, ymm4	;; I5 + R7 (newer I5)				; 20-24
	L1prefetchw dstreg+e1+L1pd, L1pt - L1PREFETCH_DEST_NONE

	yfmaddpd ymm4, ymm1, ymm15, ymm11	;; R6 + I8 (new2 R8)				; 21-25
	yfmsubpd ymm1, ymm1, ymm15, ymm11	;; R6 - I8 (new2 R6)				; 21-25
	L1prefetch srcreg+d2+L1pd, L1pt

	yfmsubpd ymm11, ymm7, ymm15, ymm9	;; I2 - I4 (newer I4)				; 22-26
	yfmaddpd ymm7, ymm7, ymm15, ymm9	;; I2 + I4 (newer I2)				; 22-26
	L1prefetchw dstreg+e2+L1pd, L1pt - L1PREFETCH_DEST_NONE

	yfmaddpd ymm9, ymm0, ymm15, ymm12	;; R5 + I7 (newer R7)				; 23-27
	yfmsubpd ymm0, ymm0, ymm15, ymm12	;; R5 - I7 (newer R5)				; 23-27
	L1prefetch srcreg+d2+d1+L1pd, L1pt

	yfmsubpd ymm12, ymm14, ymm15, ymm13	;; I1 - I3 (newer I3)				; 24-28
	yfmaddpd ymm14, ymm14, ymm15, ymm13	;; I1 + I3 (newer I1)				; 24-28
	L1prefetchw dstreg+e2+e1+L1pd, L1pt - L1PREFETCH_DEST_NONE

	yfmaddpd ymm13, ymm4, ymm15, ymm3	;; R8 + I8 (newer I8/SQRTHALF)			; 26-30
	yfmsubpd ymm4, ymm4, ymm15, ymm3	;; R8 - I8 (newer R8/SQRTHALF)			; 26-30
	L1prefetch srcreg+d4+d1+L1pd, L1pt

	yfmsubpd ymm3, ymm1, ymm15, ymm6	;; R6 - I6 (newer R6/SQRTHALF)			; 27-31
	yfmaddpd ymm1, ymm1, ymm15, ymm6	;; R6 + I6 (newer I6/SQRTHALF)			; 27-31
	L1prefetchw dstreg+e4+e1+L1pd, L1pt - L1PREFETCH_DEST_NONE

	yfmsubpd ymm6, ymm8, ymm15, ymm11	;; R3 - I4 (last R3)				; 28-32
	yfmaddpd ymm8, ymm8, ymm15, ymm11	;; R3 + I4 (last R4)				; 28-32
	L1prefetch srcreg+d4+L1pd, L1pt

	yfmaddpd ymm11, ymm12, ymm15, ymm2	;; I3 + R4 (last I3)				; 29-33
	yfmsubpd ymm12, ymm12, ymm15, ymm2	;; I3 - R4 (last I4)				; 29-33
	L1prefetchw dstreg+e4+L1pd, L1pt - L1PREFETCH_DEST_NONE

	yfmsubpd ymm2, ymm14, ymm15, ymm7	;; I1 - I2 (last I2)				; 30-34
	yfmaddpd ymm14, ymm14, ymm15, ymm7	;; I1 + I2 (last I1)				; 30-34

	vmovapd	ymm15, YMM_SQRTHALF
	yfnmaddpd ymm7, ymm13, ymm15, ymm9	;; R7 - I8 * SQRTHALF (last R7)			; 31-35
	yfmaddpd ymm13, ymm13, ymm15, ymm9	;; R7 + I8 * SQRTHALF (last R8)			; 31-35
	L1prefetch srcreg+d4+d2+L1pd, L1pt

	yfmaddpd ymm9, ymm4, ymm15, ymm5	;; I7 + R8 * SQRTHALF (last I7)			; 32-36
	yfnmaddpd ymm4, ymm4, ymm15, ymm5	;; I7 - R8 * SQRTHALF (last I8)			; 32-36
	L1prefetchw dstreg+e4+e2+L1pd, L1pt - L1PREFETCH_DEST_NONE

	yfnmaddpd ymm5, ymm3, ymm15, ymm0	;; R5 - R6 * SQRTHALF (last R6)			; 33-37
	yfmaddpd ymm3, ymm3, ymm15, ymm0	;; R5 + R6 * SQRTHALF (last R5)			; 33-37
	L1prefetch srcreg+d4+d2+d1+L1pd, L1pt

	yfnmaddpd ymm0, ymm1, ymm15, ymm10	;; I5 - I6 * SQRTHALF (last I6)			; 34-38
	yfmaddpd ymm1, ymm1, ymm15, ymm10	;; I5 + I6 * SQRTHALF (last I5)			; 34-38
	L1prefetchw dstreg+e4+e2+e1+L1pd, L1pt - L1PREFETCH_DEST_NONE
	bump	srcreg, srcinc

	vmovapd	ymm15, [screg+128+32]		;; cosine/sine for R3/I3
	yfmsubpd ymm10, ymm6, ymm15, ymm11	;; A3 = R3 * cosine/sine - I3			; 35-39
	yfmaddpd ymm11, ymm11, ymm15, ymm6	;; B3 = I3 * cosine/sine + R3			; 35-39

	vmovapd	ymm15, [screg+384+32]		;; cosine/sine for R4/I4
	yfmsubpd ymm6, ymm8, ymm15, ymm12	;; A4 = R4 * cosine/sine - I4			; 36-40
	yfmaddpd ymm12, ymm12, ymm15, ymm8	;; B4 = I4 * cosine/sine + R4			; 36-40

	vmovapd	ymm15, [screg+192+32]		;; cosine/sine for R7/I7
	yfmsubpd ymm8, ymm7, ymm15, ymm9	;; A7 = R7 * cosine/sine - I7			; 37-41
	yfmaddpd ymm9, ymm9, ymm15, ymm7	;; B7 = I7 * cosine/sine + R7			; 37-41

	vmovapd	ymm15, [screg+448+32]		;; cosine/sine for R8/I8
	yfmsubpd ymm7, ymm13, ymm15, ymm4	;; A8 = R8 * cosine/sine - I8			; 38-42
	yfmaddpd ymm4, ymm4, ymm15, ymm13	;; B8 = I8 * cosine/sine + R8			; 38-42

	vmovapd	ymm15, [screg+320+32]		;; cosine/sine for R6/I6
	yfmsubpd ymm13, ymm5, ymm15, ymm0	;; A6 = R6 * cosine/sine - I6			; 39-43
	yfmaddpd ymm0, ymm0, ymm15, ymm5	;; B6 = I6 * cosine/sine + R6			; 39-43

	vmovapd	ymm15, [screg+64+32]		;; cosine/sine for R5/I5
	yfmsubpd ymm5, ymm3, ymm15, ymm1	;; A5 = R5 * cosine/sine - I5			; 40-44
	yfmaddpd ymm1, ymm1, ymm15, ymm3	;; B5 = I5 * cosine/sine + R5			; 40-44

	vmovapd	ymm15, [screg+128]		;; sine for R3/I3
	vmulpd	ymm10, ymm10, ymm15		;; A3 = A3 * sine (final R3)			; 41-45
	vmulpd	ymm11, ymm11, ymm15		;; B3 = B3 * sine (final I3)			; 41-45

	vmovapd	ymm15, [screg+384]		;; sine for R4/I4
	vmulpd	ymm6, ymm6, ymm15		;; A4 = A4 * sine (final R4)			; 42-46
	vmulpd	ymm12, ymm12, ymm15		;; B4 = B4 * sine (final I4)			; 42-46

	vmovapd	ymm15, [screg+256+32]		;; cosine/sine for R2/I2
	vmovapd	ymm3, [dstreg+e1]		;; Reload last R2
	ystore	[dstreg+e2], ymm10		;; Save R3					; 46
	yfmsubpd ymm10, ymm3, ymm15, ymm2	;; A2 = R2 * cosine/sine - I2			; 43-47
	yfmaddpd ymm2, ymm2, ymm15, ymm3	;; B2 = I2 * cosine/sine + R2			; 43-47

	vmovapd	ymm15, [screg+0+32]		;; cosine/sine for R1/I1
	vmovapd	ymm3, [dstreg]			;; Reload last R1
	ystore	[dstreg+e2+32], ymm11		;; Save I3					; 46+1
	yfmsubpd ymm11, ymm3, ymm15, ymm14	;; A1 = R1 * cosine/sine - I1			; 44-48
	yfmaddpd ymm14, ymm14, ymm15, ymm3	;; B1 = I1 * cosine/sine + R1			; 44-48

	vmovapd	ymm15, [screg+192]		;; sine for R7/I7
	vmulpd	ymm8, ymm8, ymm15		;; A7 = A7 * sine (final R7)			; 45-49
	vmulpd	ymm9, ymm9, ymm15		;; B7 = B7 * sine (final I7)			; 45-49

	vmovapd	ymm15, [screg+448]		;; sine for R8/I8
	vmulpd	ymm7, ymm7, ymm15		;; A8 = A8 * sine (final R8)			; 46-50
	vmulpd	ymm4, ymm4, ymm15		;; B8 = B8 * sine (final I8)			; 46-50

	vmovapd	ymm15, [screg+320]		;; sine for R6/I6
	vmulpd	ymm13, ymm13, ymm15		;; A6 = A6 * sine (final R6)			; 47-51
	vmulpd	ymm0, ymm0, ymm15		;; B6 = B6 * sine (final I6)			; 47-51

	vmovapd	ymm15, [screg+64]		;; sine for R5/I5
	vmulpd	ymm5, ymm5, ymm15		;; A5 = A5 * sine (final R5)			; 48-52
	vmulpd	ymm1, ymm1, ymm15		;; B5 = B5 * sine (final I5)			; 48-52
	ystore	[dstreg+e2+e1], ymm6		;; Save R4					; 47+1

	vmovapd	ymm15, [screg+256]		;; sine for R2/I2
	vmulpd	ymm10, ymm10, ymm15		;; A2 = A2 * sine (final R2)			; 49-53
	vmulpd	ymm2, ymm2, ymm15		;; B2 = B2 * sine (final I2)			; 49-53
	ystore	[dstreg+e2+e1+32], ymm12	;; Save I4					; 47+2

	vmovapd	ymm15, [screg+0]		;; sine for R1/I1
	vmulpd	ymm11, ymm11, ymm15		;; A1 = A1 * sine (final R1)			; 50-54
	vmulpd	ymm14, ymm14, ymm15		;; B1 = B1 * sine (final I1)			; 50-54
	ystore	[dstreg+e4+e2], ymm8		;; Save R7					; 50
	bump	screg, scinc

	ystore	[dstreg+e4+e2+32], ymm9		;; Save I7					; 50+1
	ystore	[dstreg+e4+e2+e1], ymm7		;; Save R8					; 51+1
	ystore	[dstreg+e4+e2+e1+32], ymm4	;; Save I8					; 51+2
	ystore	[dstreg+e4+e1], ymm13		;; Save R6					; 52+2
	ystore	[dstreg+e4+e1+32], ymm0		;; Save I6					; 52+3
	ystore	[dstreg+e4], ymm5		;; Save R5					; 53+3
	ystore	[dstreg+e4+32], ymm1		;; Save I5					; 53+4
	ystore	[dstreg+e1], ymm10		;; Save R2					; 54+4
	ystore	[dstreg+e1+32], ymm2		;; Save I2					; 54+5
	ystore	[dstreg], ymm11			;; Save R1					; 55+5
	ystore	[dstreg+32], ymm14		;; Save I1					; 55+6

	bump	dstreg, dstinc
	ENDM
ENDIF
ENDIF

ENDIF

;;
;; ************************************* eight-complex-unfft8 variants ******************************************
;;

;
; Radix-8 inverse FFT building block for negacyclic delayed twiddle multipliers from
; the first few levels (r4delay).  Has 8 premult/sin/cos multipliers.
;

yr8_sg8cl_eight_complex_unfft8 MACRO srcreg,srcinc,d1,d2,d4,dstreg,dstinc,e1,e2,e4,screg,scinc,maxrpt,L1pt,L1pd

	vmovapd	ymm0, [screg+64+32]		;; cosine/sine
	vmovapd	ymm4, [srcreg+d4]		;; R5
	vmulpd	ymm5, ymm4, ymm0		;; A5 = R5 * cosine/sine
	vmovapd	ymm2, [srcreg+d4+32]		;; I5
	vaddpd	ymm5, ymm5, ymm2		;; A5 = A5 + I5
	vmulpd	ymm2, ymm2, ymm0		;; B5 = I5 * cosine/sine

	vmovapd	ymm0, [screg+320+32]		;; cosine/sine
	vmovapd	ymm6, [srcreg+d4+d1]		;; R6
	vmulpd	ymm7, ymm6, ymm0		;; A6 = R6 * cosine/sine
	vsubpd	ymm2, ymm2, ymm4		;; B5 = B5 - R5
	vmovapd ymm4, [srcreg+d4+d1+32]		;; I6
	vaddpd	ymm7, ymm7, ymm4		;; A6 = A6 + I6
	vmulpd	ymm4, ymm4, ymm0		;; B6 = I6 * cosine/sine
	vsubpd	ymm4, ymm4, ymm6		;; B6 = B6 - R6

	vmovapd	ymm6, [screg+64]		;; sine
	vmulpd	ymm5, ymm5, ymm6		;; A5 = A5 * sine (first R5)
	vmulpd	ymm2, ymm2, ymm6		;; B5 = B5 * sine (first I5)
	vmovapd	ymm6, [screg+320]		;; sine
	vmulpd	ymm7, ymm7, ymm6		;; A6 = A6 * sine (first R6)
	vmulpd	ymm4, ymm4, ymm6		;; B6 = B6 * sine (first I6)

	vsubpd	ymm6, ymm5, ymm7		;; R5 - R6 (new R6)
	vaddpd	ymm5, ymm5, ymm7		;; R5 + R6 (new R5)

	L1prefetch srcreg+L1pd, L1pt

	vsubpd	ymm7, ymm2, ymm4		;; I5 - I6 (new I6)
	vaddpd	ymm2, ymm2, ymm4		;; I5 + I6 (new I5)

	vmovapd	ymm0, [screg+192+32]		;; cosine/sine
	vmovapd	ymm4, [srcreg+d4+d2]		;; R7
	vmulpd	ymm3, ymm4, ymm0		;; A7 = R7 * cosine/sine
	vmovapd	ymm1, [srcreg+d4+d2+32]		;; I7
	vaddpd	ymm3, ymm3, ymm1		;; A7 = A7 + I7
	vmulpd	ymm1, ymm1, ymm0		;; B7 = I7 * cosine/sine

	vmovapd	ymm0, [screg+448+32]		;; cosine/sine
	ystore	[dstreg+e4+e1], ymm6		;; Save new R6
	vmovapd	ymm6, [srcreg+d4+d2+d1]		;; R8
	ystore	[dstreg+e4+e1+32], ymm7		;; Save new I6
	vmulpd	ymm7, ymm6, ymm0		;; A8 = R8 * cosine/sine
	vsubpd	ymm1, ymm1, ymm4		;; B7 = B7 - R7
	vmovapd ymm4, [srcreg+d4+d2+d1+32]	;; I8
	vaddpd	ymm7, ymm7, ymm4		;; A8 = A8 + I8
	vmulpd	ymm4, ymm4, ymm0		;; B8 = I8 * cosine/sine
	vsubpd	ymm4, ymm4, ymm6		;; B8 = B8 - R8

	vmovapd	ymm0, [screg+192]		;; sine
	vmulpd	ymm3, ymm3, ymm0		;; A7 = A7 * sine (first R7)
	vmulpd	ymm1, ymm1, ymm0		;; B7 = B7 * sine (first I7)
	vmovapd	ymm0, [screg+448]		;; sine
	vmulpd	ymm7, ymm7, ymm0		;; A8 = A8 * sine (first R8)
	vmulpd	ymm4, ymm4, ymm0		;; B8 = B8 * sine (first I8)

	vsubpd	ymm6, ymm7, ymm3		;; R8 - R7 (new I8)
	vaddpd	ymm7, ymm7, ymm3		;; R8 + R7 (new R7)

	L1prefetch srcreg+d1+L1pd, L1pt

	vsubpd	ymm3, ymm1, ymm4		;; I7 - I8 (new R8)
	vaddpd	ymm1, ymm1, ymm4		;; I7 + I8 (new I7)

	vmovapd	ymm0, [screg+0+32]		;; cosine/sine
	vmovapd	ymm4, [srcreg]			;; R1
	ystore	[dstreg+e4], ymm5		;; Save new R5
	vmulpd	ymm5, ymm4, ymm0		;; A1 = R1 * cosine/sine

	ystore	[dstreg+e4+32], ymm2		;; Save new I5
	vmovapd	ymm2, [screg+256+32]		;; cosine/sine
	ystore	[dstreg+e4+e2+e1+32], ymm6	;; Save new I8
	vmovapd	ymm6, [srcreg+d1]		;; R2
	ystore	[dstreg+e4+e2], ymm7		;; Save new R7
	vmulpd	ymm7, ymm6, ymm2		;; A2 = R2 * cosine/sine

	ystore	[dstreg+e4+e2+e1], ymm3		;; Save new R8
	vmovapd	ymm3, [srcreg+32]		;; I1
	vaddpd	ymm5, ymm5, ymm3		;; A1 = A1 + I1
	vmulpd	ymm3, ymm3, ymm0		;; B1 = I1 * cosine/sine
	vmovapd	ymm0, [srcreg+d1+32]		;; I2
	vaddpd	ymm7, ymm7, ymm0		;; A2 = A2 + I2
	vmulpd	ymm0, ymm0, ymm2		;; B2 = I2 * cosine/sine
	vsubpd	ymm3, ymm3, ymm4		;; B1 = B1 - R1
	vsubpd	ymm0, ymm0, ymm6		;; B2 = B2 - R2

	vmovapd	ymm2, [screg+0]			;; sine
	vmulpd	ymm5, ymm5, ymm2		;; A1 = A1 * sine (first R1)
	vmovapd	ymm4, [screg+256]		;; sine
	vmulpd	ymm7, ymm7, ymm4		;; A2 = A2 * sine (first R2)
	vmulpd	ymm3, ymm3, ymm2		;; B1 = B1 * sine (first I1)
	vmulpd	ymm0, ymm0, ymm4		;; B2 = B2 * sine (first I2)

	vsubpd	ymm2, ymm5, ymm7		;; R1 - R2 (new R2)
	vaddpd	ymm5, ymm5, ymm7		;; R1 + R2 (new R1)

	L1prefetch srcreg+d2+L1pd, L1pt

	vsubpd	ymm4, ymm3, ymm0		;; I1 - I2 (new I2)
	vaddpd	ymm3, ymm3, ymm0		;; I1 + I2 (new I1)

	vmovapd	ymm0, [screg+128+32]		;; cosine/sine
	vmovapd	ymm6, [srcreg+d2]		;; R3
	vmulpd	ymm7, ymm6, ymm0		;; A3 = R3 * cosine/sine
	ystore	[dstreg+e4+e2+32], ymm1		;; Save new I7
	vmovapd	ymm1, [srcreg+d2+32]		;; I3
	vaddpd	ymm7, ymm7, ymm1		;; A3 = A3 + I3
	vmulpd	ymm1, ymm1, ymm0		;; B3 = I3 * cosine/sine

	vmovapd	ymm0, [screg+384+32]		;; cosine/sine
	ystore	[dstreg+e1], ymm2		;; Save new R2
	vmovapd	ymm2, [srcreg+d2+d1]		;; R4
	ystore	[dstreg+e1+32], ymm4		;; Save new I2
	vmulpd	ymm4, ymm2, ymm0		;; A4 = R4 * cosine/sine
	vsubpd	ymm1, ymm1, ymm6		;; B3 = B3 - R3
	vmovapd ymm6, [srcreg+d2+d1+32]		;; I4
	vaddpd	ymm4, ymm4, ymm6		;; A4 = A4 + I4
	vmulpd	ymm6, ymm6, ymm0		;; B4 = I4 * cosine/sine
	vsubpd	ymm6, ymm6, ymm2		;; B4 = B4 - R4

	vmovapd	ymm0, [screg+128]		;; sine
	vmulpd	ymm7, ymm7, ymm0		;; A3 = A3 * sine (first R3)
	vmulpd	ymm1, ymm1, ymm0		;; B3 = B3 * sine (first I3)
	vmovapd	ymm0, [screg+384]		;; sine
	vmulpd	ymm4, ymm4, ymm0		;; A4 = A4 * sine (first R4)
	vmulpd	ymm6, ymm6, ymm0		;; B4 = B4 * sine (first I4)

	vsubpd	ymm0, ymm4, ymm7		;; R4 - R3 (new I4)
	vaddpd	ymm4, ymm4, ymm7		;; R4 + R3 (new R3)

	L1prefetch srcreg+d2+d1+L1pd, L1pt

	vsubpd	ymm7, ymm1, ymm6		;; I3 - I4 (new R4)
	vaddpd	ymm1, ymm1, ymm6		;; I3 + I4 (new I3)

	vmovapd	ymm2, [dstreg+e4+32]		;; Reload new I5
	vmovapd	ymm6, [dstreg+e4+e2+32]		;; Reload new I7
	ystore	[dstreg+e2+e1+32], ymm0		;; Save new I4
	vsubpd	ymm0, ymm2, ymm6		;; I5 - I7 (newer R7)
	vaddpd	ymm2, ymm2, ymm6		;; I5 + I7 (newer I5)

	vsubpd	ymm6, ymm5, ymm4		;; R1 - R3 (newer R3)
	vaddpd	ymm4, ymm5, ymm4		;; R1 + R3 (newer R1)

	L1prefetch srcreg+d4+L1pd, L1pt

	vsubpd	ymm5, ymm6, ymm0		;; R3 - R7 (final R7)
	vaddpd	ymm6, ymm6, ymm0		;; R3 + R7 (final R3)

	vsubpd	ymm0, ymm3, ymm1		;; I1 - I3 (newer I3)
	vaddpd	ymm3, ymm3, ymm1		;; I1 + I3 (newer I1)

	vmovapd	ymm1, [dstreg+e4+e2]		;; Reload new R7
	ystore	[dstreg+e4+e2], ymm5		;; Save final R7
	vmovapd	ymm5, [dstreg+e4]		;; Reload new R5
	ystore	[dstreg+e2], ymm6		;; Save final R3
	vaddpd	ymm6, ymm1, ymm5		;; R7 + R5 (newer R5)
	vsubpd	ymm1, ymm1, ymm5		;; R7 - R5 (newer I7)

	vsubpd	ymm5, ymm3, ymm2		;; I1 - I5 (final I5)
	vaddpd	ymm3, ymm3, ymm2		;; I1 + I5 (final I1)

	L1prefetch srcreg+d4+d1+L1pd, L1pt

	vsubpd	ymm2, ymm4, ymm6		;; R1 - R5 (final R5)
	vaddpd	ymm4, ymm4, ymm6		;; R1 + R5 (final R1)

	vsubpd	ymm6, ymm0, ymm1		;; I3 - I7 (final I7)
	vaddpd	ymm0, ymm0, ymm1		;; I3 + I7 (final I3)

	;; multiply R6/I6 by SQRTHALF - i*SQRTHALF
	vmovapd	ymm1, [dstreg+e4+e1+32]		;; Reload new I6
	ystore	[dstreg+e4+32], ymm5		;; Save final I5
	vmovapd	ymm5, [dstreg+e4+e1]		;; Reload new R6
	ystore	[dstreg+32], ymm3		;; Save final I1
	vsubpd	ymm3, ymm1, ymm5		;; I6 = I6 - R6
	vaddpd	ymm1, ymm1, ymm5		;; R6 = R6 + I6

	;; multiply R8/I8 by SQRTHALF - i*SQRTHALF
	vmovapd	ymm5, [dstreg+e4+e2+e1+32]	;; Reload new I8
	ystore	[dstreg+e4], ymm2		;; Save final R5
	vmovapd	ymm2, [dstreg+e4+e2+e1]		;; Reload new R8
	ystore	[dstreg], ymm4			;; Save final R1
	vsubpd	ymm4, ymm5, ymm2		;; I8 = I8 - R8
	vaddpd	ymm5, ymm5, ymm2		;; R8 = R8 + I8

	vmovapd	ymm2, YMM_SQRTHALF
	vmulpd	ymm3, ymm3, ymm2		;; I6 * SQRTHALF
	vmulpd	ymm1, ymm1, ymm2		;; R6 * SQRTHALF
	vmulpd	ymm4, ymm4, ymm2		;; I8 * SQRTHALF
	vmulpd	ymm5, ymm5, ymm2		;; R8 * SQRTHALF

	vmovapd	ymm2, [dstreg+e1]		;; Reload new R2
	ystore	[dstreg+e4+e2+32], ymm6		;; Save final I7
	vaddpd	ymm6, ymm2, ymm7		;; R2 + R4 (newer R2)
	vsubpd	ymm2, ymm2, ymm7		;; R2 - R4 (newer R4)

	vsubpd	ymm7, ymm5, ymm1		;; R8 - R6 (newer I8)
	vaddpd	ymm5, ymm5, ymm1		;; R8 + R6 (newer R6)

	L1prefetch srcreg+d4+d2+L1pd, L1pt

	vsubpd	ymm1, ymm3, ymm4		;; I6 - I8 (newer R8)
	vaddpd	ymm3, ymm3, ymm4		;; I6 + I8 (newer I6)

	vsubpd	ymm4, ymm6, ymm5		;; R2 - R6 (final R6)
	vaddpd	ymm6, ymm6, ymm5		;; R2 + R6 (final R2)

	vmovapd	ymm5, [dstreg+e1+32]		;; Reload new I2
	ystore	[dstreg+e4+e1], ymm4		;; Save final R6
	vmovapd	ymm4, [dstreg+e2+e1+32]		;; Reload new I4
	ystore	[dstreg+e1], ymm6		;; Save final R2
	vaddpd	ymm6, ymm5, ymm4		;; I2 + I4 (newer I2)
	vsubpd	ymm5, ymm5, ymm4		;; I2 - I4 (newer I4)

	vsubpd	ymm4, ymm2, ymm1		;; R4 - R8 (final R8)
	vaddpd	ymm2, ymm2, ymm1		;; R4 + R8 (final R4)

	vsubpd	ymm1, ymm6, ymm3		;; I2 - I6 (final I6)
	vaddpd	ymm6, ymm6, ymm3		;; I2 + I6 (final I2)

	L1prefetch srcreg+d4+d2+d1+L1pd, L1pt

	vsubpd	ymm3, ymm5, ymm7		;; I4 - I8 (final I8)
	vaddpd	ymm5, ymm5, ymm7		;; I4 + I8 (final I4)

	;; Swizzle using 4 shufpds and 4 perm2f128  (1 or 3 clocks each on port 5, throughput 1)

	vmovapd	ymm7, [dstreg+32]		;; Reload final I1
	ystore	[dstreg+e4+e2+e1], ymm4		;; Save final R8
	vshufpd	ymm4, ymm7, ymm6, 0		;; Shuffle I1 and I2 to create I1/I2 low
	vshufpd	ymm7, ymm7, ymm6, 15		;; Shuffle I1 and I2 to create I1/I2 hi

	vshufpd	ymm6, ymm0, ymm5, 0		;; Shuffle I3 and I4 to create I3/I4 low
	vshufpd	ymm0, ymm0, ymm5, 15		;; Shuffle I3 and I4 to create I3/I4 hi

	ylow128s ymm5, ymm4, ymm6		;; Shuffle I1/I2 low and I3/I4 low (final I1)
	yhigh128s ymm4, ymm4, ymm6		;; Shuffle I1/I2 low and I3/I4 low (final I3)

	ylow128s ymm6, ymm7, ymm0		;; Shuffle I1/I2 hi and I3/I4 hi (final I2)
	yhigh128s ymm7, ymm7, ymm0		;; Shuffle I1/I2 hi and I3/I4 hi (final I4)

	;; Swizzle using 4 shufpds and 4 perm2f128  (1 or 3 clocks each on port 5, throughput 1)

	vmovapd	ymm0, [dstreg]			;; Reload final R1
	ystore	[dstreg+32], ymm5		;; Save I1
	vmovapd	ymm5, [dstreg+e1]		;; Reload final R2
	ystore	[dstreg+e2+32], ymm4		;; Save I3
	vshufpd	ymm4, ymm0, ymm5, 0		;; Shuffle R1 and R2 to create R1/R2 low
	vshufpd	ymm0, ymm0, ymm5, 15		;; Shuffle R1 and R2 to create R1/R2 hi

	vmovapd	ymm5, [dstreg+e2]		;; Reload final R3
	ystore	[dstreg+e1+32], ymm6		;; Save I2
	vshufpd	ymm6, ymm5, ymm2, 0		;; Shuffle R3 and R4 to create R3/R4 low
	vshufpd	ymm5, ymm5, ymm2, 15		;; Shuffle R3 and R4 to create R3/R4 hi

	ylow128s ymm2, ymm4, ymm6		;; Shuffle R1/R2 low and R3/R4 low (final R1)
	yhigh128s ymm4, ymm4, ymm6		;; Shuffle R1/R2 low and R3/R4 low (final R3)

	ylow128s ymm6, ymm0, ymm5		;; Shuffle R1/R2 hi and R3/R4 hi (final R2)
	yhigh128s ymm0, ymm0, ymm5		;; Shuffle R1/R2 hi and R3/R4 hi (final R4)

	;; Swizzle using 4 shufpds and 4 perm2f128  (1 or 3 clocks each on port 5, throughput 1)

	vmovapd	ymm5, [dstreg+e4+32]		;; Reload final I5
	ystore	[dstreg+e2+e1+32], ymm7		;; Save I4
	vshufpd	ymm7, ymm5, ymm1, 0		;; Shuffle I5 and I6 to create I5/I6 low
	vshufpd	ymm5, ymm5, ymm1, 15		;; Shuffle I5 and I6 to create I5/I6 hi

	vmovapd	ymm1, [dstreg+e4+e2+32]		;; Reload final I7
	ystore	[dstreg], ymm2			;; Save R1
	vshufpd	ymm2, ymm1, ymm3, 0		;; Shuffle I7 and I8 to create I7/I8 low
	vshufpd	ymm1, ymm1, ymm3, 15		;; Shuffle I7 and I8 to create I7/I8 hi

	ylow128s ymm3, ymm7, ymm2		;; Shuffle I5/I6 low and I7/I8 low (final I5)
	yhigh128s ymm7, ymm7, ymm2		;; Shuffle I5/I6 low and I7/I8 low (final I7)

	ylow128s ymm2, ymm5, ymm1		;; Shuffle I5/I6 hi and I7/I8 hi (final I6)
	yhigh128s ymm5, ymm5, ymm1		;; Shuffle I5/I6 hi and I7/I8 hi (final I8)

	;; Swizzle using 4 shufpds and 4 perm2f128  (1 or 3 clocks each on port 5, throughput 1)

	vmovapd	ymm1, [dstreg+e4]		;; Reload final R5
	ystore	[dstreg+e2], ymm4		;; Save R3
	vmovapd	ymm4, [dstreg+e4+e1]		;; Reload final R6
	ystore	[dstreg+e1], ymm6		;; Save R2
	vshufpd	ymm6, ymm1, ymm4, 0		;; Shuffle R5 and R6 to create R5/R6 low
	vshufpd	ymm1, ymm1, ymm4, 15		;; Shuffle R5 and R6 to create R5/R6 hi

	vmovapd	ymm4, [dstreg+e4+e2]		;; Reload final R7
	ystore	[dstreg+e2+e1], ymm0		;; Save R4
	vmovapd	ymm0, [dstreg+e4+e2+e1]		;; Reload final R8
	ystore	[dstreg+e4+32], ymm3		;; Save I5
	vshufpd	ymm3, ymm4, ymm0, 0		;; Shuffle R7 and R8 to create R7/R8 low
	vshufpd	ymm4, ymm4, ymm0, 15		;; Shuffle R7 and R8 to create R7/R8 hi

	ylow128s ymm0, ymm6, ymm3		;; Shuffle R5/R6 low and R7/R8 low (final R5)
	yhigh128s ymm6, ymm6, ymm3		;; Shuffle R5/R6 low and R7/R8 low (final R7)

	ylow128s ymm3, ymm1, ymm4		;; Shuffle R5/R6 hi and R7/R8 hi (final R6)
	yhigh128s ymm1, ymm1, ymm4		;; Shuffle R5/R6 hi and R7/R8 hi (final R8)

	ystore	[dstreg+e4+e2+32], ymm7		;; Save I7
	ystore	[dstreg+e4+e1+32], ymm2		;; Save I6
	ystore	[dstreg+e4+e2+e1+32], ymm5	;; Save I8
	ystore	[dstreg+e4], ymm0		;; Save R5
	ystore	[dstreg+e4+e2], ymm6		;; Save R7
	ystore	[dstreg+e4+e1], ymm3		;; Save R6
	ystore	[dstreg+e4+e2+e1], ymm1		;; Save R8

	bump	srcreg, srcinc
	bump	dstreg, dstinc
	bump	screg, scinc
	ENDM

;; 64-bit version

IFDEF X86_64

yr8_sg8cl_eight_complex_unfft8 MACRO srcreg,srcinc,d1,d2,d4,dstreg,dstinc,e1,e2,e4,screg,scinc,maxrpt,L1pt,L1pd
	vmovapd	ymm0, [screg+0+32]		;; cosine/sine
	vmovapd	ymm3, [srcreg+32]		;; I1
	vmulpd	ymm8, ymm3, ymm0		;; B1 = I1 * cosine/sine	; 1-5

	vmovapd	ymm2, [screg+256+32]		;; cosine/sine
	vmovapd	ymm9, [srcreg+d1+32]		;; I2
	vmulpd	ymm10, ymm9, ymm2		;; B2 = I2 * cosine/sine	; 2-6

	vmovapd	ymm4, [srcreg]			;; R1
	vmulpd	ymm0, ymm4, ymm0		;; A1 = R1 * cosine/sine	; 3-7

	vmovapd	ymm6, [srcreg+d1]		;; R2
	vmulpd	ymm2, ymm6, ymm2		;; A2 = R2 * cosine/sine	; 4-8

	vmovapd	ymm5, [screg+128+32]		;; cosine/sine
	vmovapd	ymm1, [srcreg+d2+32]		;; I3
	vmulpd	ymm7, ymm1, ymm5		;; B3 = I3 * cosine/sine	; 5-9

	vsubpd	ymm8, ymm8, ymm4		;; B1 = B1 - R1			; 6-8
	vmovapd	ymm11, [screg+384+32]		;; cosine/sine
	vmovapd ymm12, [srcreg+d2+d1+32]	;; I4
	vmulpd	ymm4, ymm12, ymm11		;; B4 = I4 * cosine/sine	; 6-10

	vsubpd	ymm10, ymm10, ymm6		;; B2 = B2 - R2			; 7-9
	vmovapd	ymm13, [srcreg+d2]		;; R3
	vmulpd	ymm5, ymm13, ymm5		;; A3 = R3 * cosine/sine	; 7-11

	vaddpd	ymm0, ymm0, ymm3		;; A1 = A1 + I1			; 8-10
	vmovapd	ymm14, [srcreg+d2+d1]		;; R4
	vmulpd	ymm11, ymm14, ymm11		;; A4 = R4 * cosine/sine	; 8-12

	vaddpd	ymm2, ymm2, ymm9		;; A2 = A2 + I2			; 9-11
	vmovapd	ymm15, [screg+0]		;; sine
	vmulpd	ymm8, ymm8, ymm15		;; B1 = B1 * sine (first I1)	; 9-13

	vsubpd	ymm7, ymm7, ymm13		;; B3 = B3 - R3			; 10-12
	vmovapd	ymm6, [screg+256]		;; sine
	vmulpd	ymm10, ymm10, ymm6		;; B2 = B2 * sine (first I2)	; 10-14

	vsubpd	ymm4, ymm4, ymm14		;; B4 = B4 - R4			; 11-13
	vmulpd	ymm0, ymm0, ymm15		;; A1 = A1 * sine (first R1)	; 11-15
	vmovapd	ymm3, [screg+128]		;; sine

	vaddpd	ymm5, ymm5, ymm1		;; A3 = A3 + I3			; 12-14
	vmulpd	ymm2, ymm2, ymm6		;; A2 = A2 * sine (first R2)	; 12-16
	vmovapd	ymm9, [screg+384]		;; sine

	vaddpd	ymm11, ymm11, ymm12		;; A4 = A4 + I4			; 13-15
	vmulpd	ymm7, ymm7, ymm3		;; B3 = B3 * sine (first I3)	; 13-17
	vmovapd	ymm13, [screg+64+32]		;; cosine/sine

	vmulpd	ymm4, ymm4, ymm9		;; B4 = B4 * sine (first I4)	; 14-18
	vmovapd	ymm14, [srcreg+d4]		;; R5

	vaddpd	ymm12, ymm8, ymm10		;; I1 + I2 (new I1)		; 15-17
	vmulpd	ymm5, ymm5, ymm3		;; A3 = A3 * sine (first R3)	; 15-19
	vmovapd	ymm15, [srcreg+d4+32]		;; I5

	vsubpd	ymm8, ymm8, ymm10		;; I1 - I2 (new I2)		; 16-18
	vmulpd	ymm11, ymm11, ymm9		;; A4 = A4 * sine (first R4)	; 16-20
	vmovapd	ymm1, [screg+320+32]		;; cosine/sine

	vsubpd	ymm9, ymm0, ymm2		;; R1 - R2 (new R2)		; 17-19
	vmovapd	ymm6, [srcreg+d4+d1]		;; R6

	vaddpd	ymm0, ymm0, ymm2		;; R1 + R2 (new R1)		; 18-20
	vmulpd	ymm2, ymm14, ymm13		;; A5 = R5 * cosine/sine	; 18-22
	vmovapd ymm3, [srcreg+d4+d1+32]		;; I6
	ystore	[dstreg+32], ymm12		;; Save new I1			; 18

	vaddpd	ymm12, ymm7, ymm4		;; I3 + I4 (new I3)		; 19-21
	vmulpd	ymm13, ymm15, ymm13		;; B5 = I5 * cosine/sine	; 19-23
	vmovapd	ymm10, [screg+192+32]		;; cosine/sine
	ystore	[dstreg+e1+32], ymm8		;; Save new I2			; 19

	vsubpd	ymm7, ymm7, ymm4		;; I3 - I4 (new R4)		; 20-22
	vmulpd	ymm4, ymm6, ymm1		;; A6 = R6 * cosine/sine	; 20-24

	vsubpd	ymm8, ymm11, ymm5		;; R4 - R3 (new I4)		; 21-23
	vmulpd	ymm1, ymm3, ymm1		;; B6 = I6 * cosine/sine	; 21-25

	vaddpd	ymm11, ymm11, ymm5		;; R4 + R3 (new R3)		; 22-24
	vmovapd	ymm5, [srcreg+d4+d2]		;; R7
	ystore	[dstreg+e2+32], ymm12		;; Save new I3			; 22
	vmulpd	ymm12, ymm5, ymm10		;; A7 = R7 * cosine/sine	; 22-26

	vaddpd	ymm2, ymm2, ymm15		;; A5 = A5 + I5			; 23-25
	vmovapd	ymm15, [srcreg+d4+d2+32]	;; I7
	vmulpd	ymm10, ymm15, ymm10		;; B7 = I7 * cosine/sine	; 23-27

	vsubpd	ymm13, ymm13, ymm14		;; B5 = B5 - R5			; 24-26
	vmovapd	ymm14, [screg+448+32]		;; cosine/sine
	ystore	[dstreg+e2+e1+32], ymm8		;; Save new I4			; 24
	vmovapd	ymm8, [srcreg+d4+d2+d1]		;; R8

	vaddpd	ymm4, ymm4, ymm3		;; A6 = A6 + I6			; 25-27
	vmulpd	ymm3, ymm8, ymm14		;; A8 = R8 * cosine/sine	; 24-28

	vsubpd	ymm1, ymm1, ymm6		;; B6 = B6 - R6			; 26-28
	vmovapd ymm6, [srcreg+d4+d2+d1+32]	;; I8
	vmulpd	ymm14, ymm6, ymm14		;; B8 = I8 * cosine/sine	; 25-29

	vaddpd	ymm12, ymm12, ymm15		;; A7 = A7 + I7			; 27-29
	vmovapd	ymm15, [screg+64]		;; sine
	vmulpd	ymm2, ymm2, ymm15		;; A5 = A5 * sine (first R5)	; 26-30
	vmulpd	ymm13, ymm13, ymm15		;; B5 = B5 * sine (first I5)	; 27-31
	vmovapd	ymm15, [screg+320]		;; sine

	vsubpd	ymm10, ymm10, ymm5		;; B7 = B7 - R7			; 28-30
	vmulpd	ymm4, ymm4, ymm15		;; A6 = A6 * sine (first R6)	; 28-32
	vmovapd	ymm5, [screg+192]		;; sine

	vaddpd	ymm3, ymm3, ymm6		;; A8 = A8 + I8			; 29-31
	vmulpd	ymm1, ymm1, ymm15		;; B6 = B6 * sine (first I6)	; 29-33
	vmovapd	ymm6, [screg+448]		;; sine

	vsubpd	ymm14, ymm14, ymm8		;; B8 = B8 - R8			; 30-32
	vmulpd	ymm12, ymm12, ymm5		;; A7 = A7 * sine (first R7)	; 30-34
	vmovapd	ymm15, YMM_SQRTHALF

	vsubpd	ymm8, ymm0, ymm11		;; R1 - R3 (newer R3)		; 31-33
	vmulpd	ymm10, ymm10, ymm5		;; B7 = B7 * sine (first I7)	; 31-35

	vaddpd	ymm0, ymm0, ymm11		;; R1 + R3 (newer R1)		; 32-34
	vmulpd	ymm3, ymm3, ymm6		;; A8 = A8 * sine (first R8)	; 32-36
	L1prefetch srcreg+L1pd, L1pt

	vsubpd	ymm11, ymm2, ymm4		;; R5 - R6 (new R6)		; 33-35
	vmulpd	ymm14, ymm14, ymm6		;; B8 = B8 * sine (first I8)	; 33-37

	vaddpd	ymm2, ymm2, ymm4		;; R5 + R6 (new R5)		; 34-36

	vsubpd	ymm4, ymm13, ymm1		;; I5 - I6 (new I6)		; 35-37
	vaddpd	ymm13, ymm13, ymm1		;; I5 + I6 (new I5)		; 36-38
	L1prefetch srcreg+d1+L1pd, L1pt

	vsubpd	ymm1, ymm3, ymm12		;; R8 - R7 (new I8)		; 37-39
	vaddpd	ymm3, ymm3, ymm12		;; R8 + R7 (new R7)		; 38-40

	vsubpd	ymm12, ymm10, ymm14		;; I7 - I8 (new R8)		; 39-41
	vaddpd	ymm10, ymm10, ymm14		;; I7 + I8 (new I7)		; 40-42
	L1prefetch srcreg+d2+L1pd, L1pt

	vsubpd	ymm14, ymm4, ymm11		;; I6 = I6 - R6			; 41-43
	vaddpd	ymm11, ymm11, ymm4		;; R6 = R6 + I6			; 42-44

	vsubpd	ymm4, ymm1, ymm12		;; I8 = I8 - R8			; 43-45

	vaddpd	ymm12, ymm12, ymm1		;; R8 = R8 + I8			; 44-46
	vmulpd	ymm14, ymm14, ymm15		;; I6 * SQRTHALF		; 44-48
	L1prefetch srcreg+d2+d1+L1pd, L1pt

	vsubpd	ymm1, ymm13, ymm10		;; I5 - I7 (newer R7)		; 45-47
	vmulpd	ymm11, ymm11, ymm15		;; R6 * SQRTHALF		; 45-49

	vaddpd	ymm13, ymm13, ymm10		;; I5 + I7 (newer I5)		; 46-48
	vmulpd	ymm4, ymm4, ymm15		;; I8 * SQRTHALF		; 46-50

	vsubpd	ymm10, ymm9, ymm7		;; R2 - R4 (newer R4)		; 47-49
	vmulpd	ymm12, ymm12, ymm15		;; R8 * SQRTHALF		; 47-51

	vaddpd	ymm9, ymm9, ymm7		;; R2 + R4 (newer R2)		; 48-50
	L1prefetch srcreg+d4+L1pd, L1pt

	vaddpd	ymm7, ymm3, ymm2		;; R7 + R5 (newer R5)		; 49-51
	vsubpd	ymm3, ymm3, ymm2		;; R7 - R5 (newer I7)		; 50-52

	vaddpd	ymm2, ymm8, ymm1		;; R3 + R7 (final R3)		; 51-53
	vsubpd	ymm8, ymm8, ymm1		;; R3 - R7 (final R7)		; 52-54
	L1prefetch srcreg+d4+d1+L1pd, L1pt

	vsubpd	ymm1, ymm14, ymm4		;; I6 - I8 (newer R8)		; 53-55
	vaddpd	ymm14, ymm14, ymm4		;; I6 + I8 (newer I6)		; 54-56
	vaddpd	ymm4, ymm12, ymm11		;; R8 + R6 (newer R6)		; 55-57
	vsubpd	ymm12, ymm12, ymm11		;; R8 - R6 (newer I8)		; 56-58
	L1prefetch srcreg+d4+d2+L1pd, L1pt

	vaddpd	ymm11, ymm10, ymm1		;; R4 + R8 (final R4)		; 57-59
	vaddpd	ymm15, ymm0, ymm7		;; R1 + R5 (final R1)		; 58-60
	vaddpd	ymm6, ymm9, ymm4		;; R2 + R6 (final R2)		; 59-61

	vshufpd	ymm5, ymm2, ymm11, 0		;; Shuffle R3 and R4 to create R3/R4 low	; 60
	vsubpd	ymm10, ymm10, ymm1		;; R4 - R8 (final R8)				; 60-62
	vmovapd	ymm1, [dstreg+e1+32]		;; Reload new I2

	vshufpd	ymm2, ymm2, ymm11, 15		;; Shuffle R3 and R4 to create R3/R4 hi		; 61
	vsubpd	ymm0, ymm0, ymm7		;; R1 - R5 (final R5)				; 61-63
	vmovapd	ymm11, [dstreg+e2+e1+32]	;; Reload new I4

	vshufpd	ymm7, ymm15, ymm6, 0		;; Shuffle R1 and R2 to create R1/R2 low	; 62
	vsubpd	ymm9, ymm9, ymm4		;; R2 - R6 (final R6)				; 62-64
	L1prefetch srcreg+d4+d2+d1+L1pd, L1pt

	vshufpd	ymm15, ymm15, ymm6, 15		;; Shuffle R1 and R2 to create R1/R2 hi		; 63
	vaddpd	ymm6, ymm1, ymm11		;; I2 + I4 (newer I2)				; 63-65

	ylow128s ymm4, ymm7, ymm5		;; Shuffle R1/R2 low and R3/R4 low (final R1)	; 64-65
	vsubpd	ymm1, ymm1, ymm11		;; I2 - I4 (newer I4)				; 64-66
	vmovapd	ymm11, [dstreg+32]		;; Reload new I1

	yhigh128s ymm7, ymm7, ymm5		;; Shuffle R1/R2 low and R3/R4 low (final R3)	; 65-66
	vmovapd	ymm5, [dstreg+e2+32]		;; Reload new I3
	ystore	[dstreg], ymm4			;; Save R1					; 66
	vaddpd	ymm4, ymm11, ymm5		;; I1 + I3 (newer I1)				; 65-67

	vsubpd	ymm11, ymm11, ymm5		;; I1 - I3 (newer I3)				; 66-68
	ylow128s ymm5, ymm15, ymm2		;; Shuffle R1/R2 hi and R3/R4 hi (final R2)	; 66-67

	yhigh128s ymm15, ymm15, ymm2		;; Shuffle R1/R2 hi and R3/R4 hi (final R4)	; 67-68
	vaddpd	ymm2, ymm6, ymm14		;; I2 + I6 (final I2)				; 67-69
	ystore	[dstreg+e2], ymm7		;; Save R3					; 67

	vsubpd	ymm6, ymm6, ymm14		;; I2 - I6 (final I6)				; 68-70
	ystore	[dstreg+e1], ymm5		;; Save R2					; 68

	vshufpd	ymm5, ymm0, ymm9, 0		;; Shuffle R5 and R6 to create R5/R6 low	; 69
	vaddpd	ymm14, ymm4, ymm13		;; I1 + I5 (final I1)				; 69-71
	ystore	[dstreg+e2+e1], ymm15		;; Save R4					; 69

	vshufpd	ymm0, ymm0, ymm9, 15		;; Shuffle R5 and R6 to create R5/R6 hi		; 70
	vaddpd	ymm9, ymm1, ymm12		;; I4 + I8 (final I4)				; 70-72

	vshufpd	ymm15, ymm8, ymm10, 0		;; Shuffle R7 and R8 to create R7/R8 low	; 71
	vaddpd	ymm7, ymm11, ymm3		;; I3 + I7 (final I3)				; 71-73

	vshufpd	ymm8, ymm8, ymm10, 15		;; Shuffle R7 and R8 to create R7/R8 hi		; 72
	vsubpd	ymm4, ymm4, ymm13		;; I1 - I5 (final I5)				; 72-74

	vshufpd	ymm13, ymm14, ymm2, 0		;; Shuffle I1 and I2 to create I1/I2 low	; 73
	vsubpd	ymm1, ymm1, ymm12		;; I4 - I8 (final I8)				; 73-75

	vshufpd	ymm14, ymm14, ymm2, 15		;; Shuffle I1 and I2 to create I1/I2 hi		; 74
	vsubpd	ymm11, ymm11, ymm3		;; I3 - I7 (final I7)				; 74-76

	vshufpd	ymm3, ymm7, ymm9, 0		;; Shuffle I3 and I4 to create I3/I4 low	; 75
	vshufpd	ymm7, ymm7, ymm9, 15		;; Shuffle I3 and I4 to create I3/I4 hi		; 76

	vshufpd	ymm9, ymm4, ymm6, 0		;; Shuffle I5 and I6 to create I5/I6 low
	vshufpd	ymm4, ymm4, ymm6, 15		;; Shuffle I5 and I6 to create I5/I6 hi

	vshufpd	ymm6, ymm11, ymm1, 0		;; Shuffle I7 and I8 to create I7/I8 low
	vshufpd	ymm11, ymm11, ymm1, 15		;; Shuffle I7 and I8 to create I7/I8 hi

	ylow128s ymm1, ymm5, ymm15		;; Shuffle R5/R6 low and R7/R8 low (final R5)
	yhigh128s ymm5, ymm5, ymm15		;; Shuffle R5/R6 low and R7/R8 low (final R7)
	ylow128s ymm15, ymm0, ymm8		;; Shuffle R5/R6 hi and R7/R8 hi (final R6)
	yhigh128s ymm0, ymm0, ymm8		;; Shuffle R5/R6 hi and R7/R8 hi (final R8)

	ystore	[dstreg+e4], ymm1		;; Save R5
	ystore	[dstreg+e4+e2], ymm5		;; Save R7
	ystore	[dstreg+e4+e1], ymm15		;; Save R6
	ystore	[dstreg+e4+e2+e1], ymm0		;; Save R8

	ylow128s ymm8, ymm13, ymm3		;; Shuffle I1/I2 low and I3/I4 low (final I1)
	yhigh128s ymm13, ymm13, ymm3		;; Shuffle I1/I2 low and I3/I4 low (final I3)
	ylow128s ymm3, ymm14, ymm7		;; Shuffle I1/I2 hi and I3/I4 hi (final I2)
	yhigh128s ymm14, ymm14, ymm7		;; Shuffle I1/I2 hi and I3/I4 hi (final I4)

	ystore	[dstreg+32], ymm8		;; Save I1
	ystore	[dstreg+e2+32], ymm13		;; Save I3
	ystore	[dstreg+e1+32], ymm3		;; Save I2
	ystore	[dstreg+e2+e1+32], ymm14	;; Save I4

	ylow128s ymm7, ymm9, ymm6		;; Shuffle I5/I6 low and I7/I8 low (final I5)
	yhigh128s ymm9, ymm9, ymm6		;; Shuffle I5/I6 low and I7/I8 low (final I7)
	ylow128s ymm6, ymm4, ymm11		;; Shuffle I5/I6 hi and I7/I8 hi (final I6)
	yhigh128s ymm4, ymm4, ymm11		;; Shuffle I5/I6 hi and I7/I8 hi (final I8)

	ystore	[dstreg+e4+32], ymm7		;; Save I5
	ystore	[dstreg+e4+e2+32], ymm9		;; Save I7
	ystore	[dstreg+e4+e1+32], ymm6		;; Save I6
	ystore	[dstreg+e4+e2+e1+32], ymm4	;; Save I8

	bump	srcreg, srcinc
	bump	dstreg, dstinc
	bump	screg, scinc
	ENDM

;; Haswell FMA3 version

IF (@INSTR(,%yarch,<FMA3>) NE 0)

yr8_sg8cl_eight_complex_unfft8 MACRO srcreg,srcinc,d1,d2,d4,dstreg,dstinc,e1,e2,e4,screg,scinc,maxrpt,L1pt,L1pd
	vmovapd	ymm15, [screg+64+32]		;; cosine/sine for R5/I5
	vmovapd	ymm14, [srcreg+d4]		;; R5
	vmovapd	ymm1, [srcreg+d4+32]		;; I5
	yfmaddpd ymm0, ymm14, ymm15, ymm1	;; A5 = R5 * cosine/sine + I5			; 1-5		n 6
	yfmsubpd ymm1, ymm1, ymm15, ymm14	;; B5 = I5 * cosine/sine - R5			; 1-5		n 6

	vmovapd	ymm15, [screg+192+32]		;; cosine/sine for R7/I7
	vmovapd	ymm14, [srcreg+d4+d2]		;; R7
	vmovapd	ymm3, [srcreg+d4+d2+32]		;; I7
	yfmaddpd ymm2, ymm14, ymm15, ymm3	;; A7 = R7 * cosine/sine + I7			; 2-6		n 7
	yfmsubpd ymm3, ymm3, ymm15, ymm14	;; B7 = I7 * cosine/sine - R7			; 2-6		n 7

	vmovapd	ymm15, [screg+0+32]		;; cosine/sine for R1/I1
	vmovapd	ymm14, [srcreg]			;; R1
	vmovapd	ymm5, [srcreg+32]		;; I1
	yfmaddpd ymm4, ymm14, ymm15, ymm5	;; A1 = R1 * cosine/sine + I1			; 3-7		n 9
	yfmsubpd ymm5, ymm5, ymm15, ymm14	;; B1 = I1 * cosine/sine - R1			; 3-7		n 9

	vmovapd	ymm15, [screg+128+32]		;; cosine/sine for R3/I3
	vmovapd	ymm14, [srcreg+d2]		;; R3
	vmovapd	ymm7, [srcreg+d2+32]		;; I3
	yfmaddpd ymm6, ymm14, ymm15, ymm7	;; A3 = R3 * cosine/sine + I3			; 4-8		n 15
	yfmsubpd ymm7, ymm7, ymm15, ymm14	;; B3 = I3 * cosine/sine - R3			; 4-8		n 15

	vmovapd	ymm15, [screg+320+32]		;; cosine/sine for R6/I6
	vmovapd	ymm14, [srcreg+d4+d1]		;; R6
	vmovapd ymm9, [srcreg+d4+d1+32]		;; I6
	yfmaddpd ymm8, ymm14, ymm15, ymm9	;; A6 = R6 * cosine/sine + I6			; 5-9		n 11
	yfmsubpd ymm9, ymm9, ymm15, ymm14	;; B6 = I6 * cosine/sine - R6			; 5-9		n 12

	vmovapd	ymm15, [screg+64]		;; sine for R5/I5
	vmulpd	ymm0, ymm0, ymm15		;; A5 = A5 * sine (first R5)			; 6-10		n 11
	vmulpd	ymm1, ymm1, ymm15		;; B5 = B5 * sine (first I5)			; 6-10		n 12

	vmovapd	ymm15, [screg+192]		;; sine for R7/I7
	vmulpd	ymm2, ymm2, ymm15		;; A7 = A7 * sine (first R7)			; 7-11		n 13
	vmulpd	ymm3, ymm3, ymm15		;; B7 = B7 * sine (first I7)			; 7-11		n 14

	vmovapd	ymm15, [screg+448+32]		;; cosine/sine for R8/I8
	vmovapd	ymm14, [srcreg+d4+d2+d1]	;; R8
	vmovapd ymm11, [srcreg+d4+d2+d1+32]	;; I8
	yfmaddpd ymm10, ymm14, ymm15, ymm11	;; A8 = R8 * cosine/sine + I8			; 8-12		n 13
	yfmsubpd ymm11, ymm11, ymm15, ymm14	;; B8 = I8 * cosine/sine - R8			; 8-12		n 14

	vmovapd	ymm15, [screg+0]		;; sine for R1/I1
	vmulpd	ymm4, ymm4, ymm15		;; A1 = A1 * sine (first R1)			; 9-13		n 20
	vmulpd	ymm5, ymm5, ymm15		;; B1 = B1 * sine (first I1)			; 9-13		n 23

	vmovapd	ymm15, [screg+256+32]		;; cosine/sine for R2/I2
	vmovapd	ymm14, [srcreg+d1]		;; R2
	vmovapd	ymm13, [srcreg+d1+32]		;; I2
	yfmaddpd ymm12, ymm14, ymm15, ymm13	;; A2 = R2 * cosine/sine + I2			; 10-14		n 20
	yfmsubpd ymm13, ymm13, ymm15, ymm14	;; B2 = I2 * cosine/sine - R2			; 10-14		n 23

	vmovapd	ymm15, [screg+320]		;; sine for R6/I6
	yfnmaddpd ymm14, ymm8, ymm15, ymm0	;; R5 - (R6 = A6 * sine) (new R6)		; 11-15		n 17
	yfmaddpd ymm8, ymm8, ymm15, ymm0	;; R5 + (R6 = A6 * sine) (new R5)		; 11-15		n 18

	yfnmaddpd ymm0, ymm9, ymm15, ymm1	;; I5 - (I6 = B6 * sine) (new I6)		; 12-16		n 17
	yfmaddpd ymm9, ymm9, ymm15, ymm1	;; I5 + (I6 = B6 * sine) (new I5)		; 12-16		n 27

	vmovapd	ymm15, [screg+448]		;; sine for R8/I8
	yfmaddpd ymm1, ymm10, ymm15, ymm2	;; (R8 = A8 * sine) + R7 (new R7)		; 13-17		n 18
	yfmsubpd ymm10, ymm10, ymm15, ymm2	;; (R8 = A8 * sine) - R7 (new I8)		; 13-17		n 19

	yfnmaddpd ymm2, ymm11, ymm15, ymm3	;; I7 - (I8 = B8 * sine) (new R8)		; 14-18		n 19
	yfmaddpd ymm11, ymm11, ymm15, ymm3	;; I7 + (I8 = B8 * sine) (new I7)		; 14-18		n 27

	vmovapd	ymm15, [screg+128]		;; sine for R3/I3
	vmulpd	ymm6, ymm6, ymm15		;; A3 = A3 * sine (first R3)			; 15-19		n 22
	vmulpd	ymm7, ymm7, ymm15		;; B3 = B3 * sine (first I3)			; 15-19		n 21

	vmovapd ymm15, YMM_ONE
	yfmaddpd ymm3, ymm1, ymm15, ymm8	;; R7 + R5 (newer R5)				; 18-22
	yfmsubpd ymm1, ymm1, ymm15, ymm8	;; R7 - R5 (newer I7)				; 18-22

	vmovapd	ymm8, [srcreg+d2+d1]		;; R4
	ystore	[dstreg], ymm3			;; Save newer R5				; 23
	vmovapd ymm3, [srcreg+d2+d1+32]		;; I4
	ystore	[dstreg+32], ymm1		;; Save newer I7				; 23+1
	yfmsubpd ymm1, ymm3, [screg+384+32], ymm8 ;; B4 = I4 * cosine/sine - R4			; 16-20		n 21
	yfmaddpd ymm8, ymm8, [screg+384+32], ymm3 ;; A4 = R4 * cosine/sine + I4			; 16-20		n 22

	yfmsubpd ymm3, ymm0, ymm15, ymm14	;; I6 - R6 (new2 I6)				; 17-21		n 24
	yfmaddpd ymm14, ymm14, ymm15, ymm0	;; R6 + I6 (new2 R6)				; 17-21		n 25
	L1prefetch srcreg+L1pd, L1pt

	yfmsubpd ymm0, ymm10, ymm15, ymm2	;; I8 - R8 (new2 I8)				; 19-23		n 24
	yfmaddpd ymm2, ymm2, ymm15, ymm10	;; R8 + I8 (new2 R8)				; 19-23		n 25

	vmovapd	ymm15, [screg+256]		;; sine for R2/I2
	yfnmaddpd ymm10, ymm12, ymm15, ymm4	;; R1 - (R2 = A2 * sine) (new R2)		; 20-24		n 26
	yfmaddpd ymm12, ymm12, ymm15, ymm4	;; R1 + (R2 = A2 * sine) (new R1)		; 20-24		n 28

	vmovapd	ymm15, [screg+384]		;; sine for R4/I4
	yfnmaddpd ymm4, ymm1, ymm15, ymm7	;; I3 - (I4 = B4 * sine) (new R4)		; 21-25		n 26
	yfmaddpd ymm1, ymm1, ymm15, ymm7	;; I3 + (I4 = B4 * sine) (new I3)		; 21-25		n 30
	L1prefetch srcreg+d1+L1pd, L1pt

	yfmaddpd ymm7, ymm8, ymm15, ymm6	;; (R4 = A4 * sine) + R3 (new R3)		; 22-26		n 28
	yfmsubpd ymm8, ymm8, ymm15, ymm6	;; (R4 = A4 * sine) - R3 (new I4)		; 22-26		n 29

	vmovapd	ymm15, [screg+256]		;; sine for R2/I2
	yfnmaddpd ymm6, ymm13, ymm15, ymm5	;; I1 - (I2 = B2 * sine) (new I2)		; 23-27		n 29
	yfmaddpd ymm13, ymm13, ymm15, ymm5	;; I1 + (I2 = B2 * sine) (new I1)		; 23-27		n 30

	vmovapd ymm15, YMM_ONE
	yfmsubpd ymm5, ymm3, ymm15, ymm0	;; I6 - I8 (newer R8/SQRTHALF)			; 24-28		n 31
	yfmaddpd ymm3, ymm3, ymm15, ymm0	;; I6 + I8 (newer I6/SQRTHALF)			; 24-28		n 35

	yfmaddpd ymm0, ymm2, ymm15, ymm14	;; R8 + R6 (newer R6/SQRTHALF)			; 25-29		n 32
	yfmsubpd ymm2, ymm2, ymm15, ymm14	;; R8 - R6 (newer I8/SQRTHALF)			; 25-29		n 36
	L1prefetch srcreg+d2+L1pd, L1pt

	yfmsubpd ymm14, ymm10, ymm15, ymm4	;; R2 - R4 (newer R4)				; 26-30		n 31
	yfmaddpd ymm10, ymm10, ymm15, ymm4	;; R2 + R4 (newer R2)				; 26-30		n 32
	L1prefetch srcreg+d2+d1+L1pd, L1pt

	yfmsubpd ymm4, ymm9, ymm15, ymm11	;; I5 - I7 (newer R7)				; 27-31		n 33
	yfmaddpd ymm9, ymm9, ymm15, ymm11	;; I5 + I7 (newer I5)				; 27-31
	bump	screg, scinc

	yfmsubpd ymm11, ymm12, ymm15, ymm7	;; R1 - R3 (newer R3)				; 28-32		n 33
	yfmaddpd ymm12, ymm12, ymm15, ymm7	;; R1 + R3 (newer R1)				; 28-32		n 34
	L1prefetch srcreg+d4+L1pd, L1pt

	yfmaddpd ymm7, ymm6, ymm15, ymm8	;; I2 + I4 (newer I2)				; 29-33		n 35
	yfmsubpd ymm6, ymm6, ymm15, ymm8	;; I2 - I4 (newer I4)				; 29-33		n 36
	L1prefetch srcreg+d4+d1+L1pd, L1pt

	yfmaddpd ymm8, ymm13, ymm15, ymm1	;; I1 + I3 (newer I1)				; 30-34		n 37
	yfmsubpd ymm13, ymm13, ymm15, ymm1	;; I1 - I3 (newer I3)				; 30-34		n 38

	vmovapd	ymm1, YMM_SQRTHALF
	ystore	[dstreg+e1], ymm9		;; Save newer I5				; 32
	yfmaddpd ymm9, ymm5, ymm1, ymm14	;; R4 + R8 * SQRTHALF (final R4)		; 31-35		n 38
	yfnmaddpd ymm5, ymm5, ymm1, ymm14	;; R4 - R8 * SQRTHALF (final R8)		; 31-35		n 44

	yfmaddpd ymm14, ymm0, ymm1, ymm10	;; R2 + R6 * SQRTHALF (final R2)		; 32-36		n 40
	yfnmaddpd ymm0, ymm0, ymm1, ymm10	;; R2 - R6 * SQRTHALF (final R6)		; 32-36		n 42
	L1prefetch srcreg+d4+d2+L1pd, L1pt

	yfmaddpd ymm10, ymm11, ymm15, ymm4	;; R3 + R7 (final R3)				; 33-37		n 38
	yfmsubpd ymm11, ymm11, ymm15, ymm4	;; R3 - R7 (final R7)				; 33-37		n 44

	yfmaddpd ymm4, ymm12, ymm15, [dstreg]	;; R1 + R5 (final R1)				; 34-38		n 40
	yfmsubpd ymm12, ymm12, ymm15, [dstreg]	;; R1 - R5 (final R5)				; 34-38		n 42

	yfmaddpd ymm15, ymm3, ymm1, ymm7	;; I2 + I6 * SQRTHALF (final I2)		; 35-39		n 46
	yfnmaddpd ymm3, ymm3, ymm1, ymm7	;; I2 - I6 * SQRTHALF (final I6)		; 35-39		n 48
	L1prefetch srcreg+d4+d2+d1+L1pd, L1pt

	yfmaddpd ymm7, ymm2, ymm1, ymm6		;; I4 + I8 * SQRTHALF (final I4)		; 36-40		n 50
	yfnmaddpd ymm2, ymm2, ymm1, ymm6	;; I4 - I8 * SQRTHALF (final I8)		; 36-40		n 52
	vmovapd	ymm6, YMM_ONE
	bump	srcreg, srcinc

	vshufpd	ymm1, ymm10, ymm9, 0		;; Shuffle R3 and R4 to create R3/R4 low	; 38
	vshufpd	ymm10, ymm10, ymm9, 15		;; Shuffle R3 and R4 to create R3/R4 hi		; 39
	vmovapd	ymm9, [dstreg+e1]		;; Reload newer I5

	ystorelo [dstreg][16], ymm1		;; Save R1					; 39
	ystorehi [dstreg+e2][16], ymm1		;; Save R3					; 39+1
	yfmaddpd ymm1, ymm8, ymm6, ymm9		;; I1 + I5 (final I1)				; 37-41		n 46
	yfmsubpd ymm8, ymm8, ymm6, ymm9		;; I1 - I5 (final I5)				; 37-41		n 48

	vshufpd	ymm9, ymm4, ymm14, 0		;; Shuffle R1 and R2 to create R1/R2 low	; 40
	vshufpd	ymm4, ymm4, ymm14, 15		;; Shuffle R1 and R2 to create R1/R2 hi		; 41
	vmovapd	ymm14, [dstreg+32]		;; Reload newer I7

	ystorelo [dstreg+e1][16], ymm10		;; Save R2					; 40+1
	ystorehi [dstreg+e2+e1][16], ymm10	;; Save R4					; 40+2

	yfmaddpd ymm10, ymm13, ymm6, ymm14	;; I3 + I7 (final I3)				; 38-42		n 50
	yfmsubpd ymm13, ymm13, ymm6, ymm14	;; I3 - I7 (final I7)				; 38-42		n 52

	vshufpd	ymm14, ymm12, ymm0, 0		;; Shuffle R5 and R6 to create R5/R6 low	; 42
	vshufpd	ymm12, ymm12, ymm0, 15		;; Shuffle R5 and R6 to create R5/R6 hi		; 43

	ystorelo [dstreg], ymm9			;; Save R1					; 41+2
	ystorehi [dstreg+e2], ymm9		;; Save R3					; 41+3

	vshufpd	ymm0, ymm11, ymm5, 0		;; Shuffle R7 and R8 to create R7/R8 low	; 44
	vshufpd	ymm11, ymm11, ymm5, 15		;; Shuffle R7 and R8 to create R7/R8 hi		; 45

	ystorelo [dstreg+e1], ymm4		;; Save R2					; 42+3
	ystorehi [dstreg+e2+e1], ymm4		;; Save R4					; 42+4

	vshufpd	ymm5, ymm1, ymm15, 0		;; Shuffle I1 and I2 to create I1/I2 low	; 46		n 54
	vshufpd	ymm1, ymm1, ymm15, 15		;; Shuffle I1 and I2 to create I1/I2 hi		; 47		n 56

	ystorelo [dstreg+e4], ymm14		;; Save R5					; 43+4
	ystorehi [dstreg+e4+e2], ymm14		;; Save R7					; 43+5

	vshufpd	ymm15, ymm8, ymm3, 0		;; Shuffle I5 and I6 to create I5/I6 low	; 48		n 58
	vshufpd	ymm8, ymm8, ymm3, 15		;; Shuffle I5 and I6 to create I5/I6 hi		; 49		n 60

	ystorelo [dstreg+e4+e1], ymm12		;; Save R6					; 44+5
	ystorehi [dstreg+e4+e2+e1], ymm12	;; Save R8					; 44+6

	vshufpd	ymm3, ymm10, ymm7, 0		;; Shuffle I3 and I4 to create I3/I4 low	; 50		n 54
	vshufpd	ymm10, ymm10, ymm7, 15		;; Shuffle I3 and I4 to create I3/I4 hi		; 51		n 56

	ystorelo [dstreg+e4][16], ymm0		;; Save R5					; 45+6
	ystorehi [dstreg+e4+e2][16], ymm0	;; Save R7					; 45+7

	vshufpd	ymm7, ymm13, ymm2, 0		;; Shuffle I7 and I8 to create I7/I8 low	; 52		n 58
	vshufpd	ymm13, ymm13, ymm2, 15		;; Shuffle I7 and I8 to create I7/I8 hi		; 53		n 60

	ystorelo [dstreg+e4+e1][16], ymm11	;; Save R6					; 46+7
	ystorehi [dstreg+e4+e2+e1][16], ymm11	;; Save R8					; 46+8

	ylow128s ymm2, ymm5, ymm3		;; Shuffle I1/I2 low and I3/I4 low (final I1)	; 54-56
	yhigh128s ymm5, ymm5, ymm3		;; Shuffle I1/I2 low and I3/I4 low (final I3)	; 55-57
	ylow128s ymm3, ymm1, ymm10		;; Shuffle I1/I2 hi and I3/I4 hi (final I2)	; 56-58
	ystore	[dstreg+32], ymm2		;; Save I1					; 57
	yhigh128s ymm1, ymm1, ymm10		;; Shuffle I1/I2 hi and I3/I4 hi (final I4)	; 57-59
	ystore	[dstreg+e2+32], ymm5		;; Save I3					; 58

	ylow128s ymm10, ymm15, ymm7		;; Shuffle I5/I6 low and I7/I8 low (final I5)	; 58-60
	ystore	[dstreg+e1+32], ymm3		;; Save I2					; 59
	yhigh128s ymm15, ymm15, ymm7		;; Shuffle I5/I6 low and I7/I8 low (final I7)	; 59-61
	ystore	[dstreg+e2+e1+32], ymm1		;; Save I4					; 60
	ylow128s ymm7, ymm8, ymm13		;; Shuffle I5/I6 hi and I7/I8 hi (final I6)	; 60-62
	ystore	[dstreg+e4+32], ymm10		;; Save I5					; 61
	yhigh128s ymm8, ymm8, ymm13		;; Shuffle I5/I6 hi and I7/I8 hi (final I8)	; 61-63
	ystore	[dstreg+e4+e2+32], ymm15	;; Save I7					; 62
	ystore	[dstreg+e4+e1+32], ymm7		;; Save I6					; 63
	ystore	[dstreg+e4+e2+e1+32], ymm8	;; Save I8					; 64

	bump	dstreg, dstinc
	ENDM

ENDIF

ENDIF


;;
;; ************************************* eight-complex-with-square and variants ******************************************
;;

;;
;; The last three levels of the forward FFT are performed.
;; No sin/cos multipliers are needed.
;;

yr8_8cl_eight_complex_fft_final_preload MACRO
	ENDM
yr8_8cl_eight_complex_fft_final MACRO srcreg,srcinc,d1,d2,d4
	yr8_8c_simple_fft_part1 srcreg,d1,d2,d4
	yr8_8c_simple_fft_part2 srcreg,d1,d2,d4
	bump	srcreg, srcinc
	ENDM

yr8_8c_simple_fft_part1 MACRO srcreg,d1,d2,d4
	vmovapd	ymm0, [srcreg]			;; R1
	vmovapd	ymm1, [srcreg+d4]		;; R5
	vaddpd	ymm2, ymm0, ymm1		;; R1 + R5 (new R1)
	vsubpd	ymm0, ymm0, ymm1		;; R1 - R5 (new R5)

	vmovapd	ymm1, [srcreg+d2]		;; R3
	vmovapd	ymm4, [srcreg+d4+d2]		;; R7
	vaddpd	ymm3, ymm1, ymm4		;; R3 + R7 (new R3)
	vsubpd	ymm1, ymm1, ymm4		;; R3 - R7 (new R7)

	vmovapd	ymm4, [srcreg+32]		;; I1
	vmovapd	ymm5, [srcreg+d4+32]		;; I5
	vaddpd	ymm6, ymm4, ymm5		;; I1 + I5 (new I1)
	vsubpd	ymm4, ymm4, ymm5		;; I1 - I5 (new I5)

	vaddpd	ymm7, ymm2, ymm3		;; R1 + R3 (final R1)
	vsubpd	ymm2, ymm2, ymm3		;; R1 - R3 (final R3)

	ystore	[srcreg], ymm7			;; Save R1
	ystore	[srcreg+d2], ymm2		;; Save R3

	vmovapd	ymm5, [srcreg+d2+32]		;; I3
	vmovapd	ymm7, [srcreg+d4+d2+32]		;; I7
	vaddpd	ymm3, ymm5, ymm7		;; I3 + I7 (new I3)
	vsubpd	ymm5, ymm5, ymm7		;; I3 - I7 (new I7)

	vaddpd	ymm7, ymm6, ymm3		;; I1 + I3 (final I1)
	vsubpd	ymm6, ymm6, ymm3		;; I1 - I3 (final I3)

	ystore	[srcreg+32], ymm7		;; Save I1
	ystore	[srcreg+d2+32], ymm6		;; Save I3

	vaddpd	ymm7, ymm4, ymm1		;; I5 + R7 (final I5)
	vsubpd	ymm6, ymm0, ymm5		;; R5 - I7 (final R5)
	vsubpd	ymm4, ymm4, ymm1		;; I5 - R7 (final I7)
	vaddpd	ymm0, ymm0, ymm5		;; R5 + I7 (final R7)

	ystore	[srcreg+d4+32], ymm7		;; Save I5
	ystore	[srcreg+d4], ymm6		;; Save R5
	ystore	[srcreg+d4+d2+32], ymm4		;; Save I7
	ystore	[srcreg+d4+d2], ymm0		;; Save R7

	vmovapd	ymm0, [srcreg+d1]		;; R2
	vmovapd	ymm7, [srcreg+d4+d1]		;; R6
	vaddpd	ymm2, ymm0, ymm7		;; R2 + R6 (new R2)
	vsubpd	ymm0, ymm0, ymm7		;; R2 - R6 (new R6)

	vmovapd	ymm1, [srcreg+d2+d1]		;; R4
	vmovapd	ymm7, [srcreg+d4+d2+d1]		;; R8
	vaddpd	ymm3, ymm1, ymm7		;; R4 + R8 (new R4)
	vsubpd	ymm1, ymm1, ymm7		;; R4 - R8 (new R8)

	vmovapd	ymm4, [srcreg+d1+32]		;; I2
	vmovapd	ymm7, [srcreg+d4+d1+32]		;; I6
	vaddpd	ymm6, ymm4, ymm7		;; I2 + I6 (new I2)
	vsubpd	ymm4, ymm4, ymm7		;; I2 - I6 (new I6)

	vaddpd	ymm7, ymm2, ymm3		;; R2 + R4 (final R2)
	vsubpd	ymm2, ymm2, ymm3		;; R2 - R4 (final R4)

	ystore	[srcreg+d1], ymm7		;; Save R2
	ystore	[srcreg+d2+d1], ymm2		;; Save R4

	vmovapd	ymm5, [srcreg+d2+d1+32]		;; I4
	vmovapd	ymm7, [srcreg+d4+d2+d1+32]	;; I8
	vaddpd	ymm3, ymm5, ymm7		;; I4 + I8 (new I4)
	vsubpd	ymm5, ymm5, ymm7		;; I4 - I8 (new I8)

	vaddpd	ymm7, ymm6, ymm3		;; I2 + I4 (final I2)
	vsubpd	ymm6, ymm6, ymm3		;; I2 - I4 (final I4)

	ystore	[srcreg+d1+32], ymm7		;; Save I2
	ystore	[srcreg+d2+d1+32], ymm6		;; Save I4

	vaddpd	ymm7, ymm4, ymm1		;; I6 + R8 (new I6)
	vsubpd	ymm6, ymm0, ymm5		;; R6 - I8 (new R6)
	vsubpd	ymm4, ymm4, ymm1		;; I6 - R8 (new I8)
	vaddpd	ymm0, ymm0, ymm5		;; R6 + I8 (new R8)

	vsubpd	ymm1, ymm6, ymm7		;; R6 = R6 - I6
	vaddpd	ymm6, ymm6, ymm7		;; I6 = R6 + I6
	vmovapd	ymm5, YMM_SQRTHALF
	vmulpd	ymm1, ymm1, ymm5		;; R6 = R6 * SQRTHALF (final R6)
	vmulpd	ymm6, ymm6, ymm5		;; I6 = I6 * SQRTHALF (final I6)

	vsubpd	ymm2, ymm0, ymm4		;; R8 = R8 - I8
	vaddpd	ymm0, ymm0, ymm4		;; I8 = R8 + I8
	vmulpd	ymm2, ymm2, ymm5		;; R8 = R8 * SQRTHALF (final R8)
	vmulpd	ymm0, ymm0, ymm5		;; I8 = I8 * SQRTHALF (final I8)

	ystore	[srcreg+d4+d1], ymm1		;; Save R6
	ystore	[srcreg+d4+d1+32], ymm6		;; Save I6
	ystore	[srcreg+d4+d2+d1], ymm2		;; Save R8
	ystore	[srcreg+d4+d2+d1+32], ymm0	;; Save I8
	ENDM

yr8_8c_simple_fft_part2 MACRO srcreg,d1,d2,d4
	vmovapd	ymm0, [srcreg+d4]		;; R5
	vmovapd	ymm7, [srcreg+d4+d1] 		;; R6
	vsubpd	ymm2, ymm0, ymm7		;; R5 - R6 (new R6)
	vaddpd	ymm0, ymm0, ymm7		;; R5 + R6 (new R5)

	vmovapd	ymm1, [srcreg+d4+32]		;; I5
	vmovapd	ymm7, [srcreg+d4+d1+32]		;; I6
	vsubpd	ymm3, ymm1, ymm7		;; I5 - I6 (new I6)
	vaddpd	ymm1, ymm1, ymm7		;; I5 + I6 (new I5)

	ystore	[srcreg+d4+d1], ymm2		;; Save R6
	ystore	[srcreg+d4], ymm0		;; Save R5
	ystore	[srcreg+d4+d1+32], ymm3		;; Save I6
	ystore	[srcreg+d4+32], ymm1		;; Save I5

	vmovapd	ymm5, [srcreg+d4+d2]		;; R7
	vmovapd	ymm7, [srcreg+d4+d2+d1+32]	;; I8
	vsubpd	ymm4, ymm5, ymm7		;; R7 - I8 (new R7)
	vaddpd	ymm5, ymm5, ymm7		;; R7 + I8 (new R8)

	vmovapd	ymm1, [srcreg+d4+d2+32]		;; I7
	vmovapd	ymm7, [srcreg+d4+d2+d1]		;; R8
	vsubpd	ymm0, ymm1, ymm7		;; I7 - R8 (new I8)
	vaddpd	ymm1, ymm1, ymm7		;; I7 + R8 (new I7)

	ystore	[srcreg+d4+d2], ymm4		;; Save R7
	ystore	[srcreg+d4+d2+d1], ymm5		;; Save R8
	ystore	[srcreg+d4+d2+d1+32], ymm0	;; Save I8
	ystore	[srcreg+d4+d2+32], ymm1		;; Save I7

	vmovapd	ymm4, [srcreg]			;; R1
	vmovapd	ymm7, [srcreg+d1]		;; R2
	vsubpd	ymm3, ymm4, ymm7		;; R1 - R2 (new R2)
	vaddpd	ymm4, ymm4, ymm7		;; R1 + R2 (new R1)

	vmovapd	ymm6, [srcreg+32]		;; I1
	vmovapd	ymm7, [srcreg+d1+32]		;; I2
	vsubpd	ymm5, ymm6, ymm7		;; I1 - I2 (new I2)
	vaddpd	ymm6, ymm6, ymm7		;; I1 + I2 (new I1)

	ystore	[srcreg+d1], ymm3		;; Save R2
	ystore	[srcreg], ymm4			;; Save R1
	ystore	[srcreg+d1+32], ymm5		;; Save I2
	ystore	[srcreg+32], ymm6		;; Save I1

	vmovapd	ymm6, [srcreg+d2]		;; R3
	vmovapd	ymm7, [srcreg+d2+d1+32]		;; I4
	vsubpd	ymm5, ymm6, ymm7		;; R3 - I4 (new R3)
	vaddpd	ymm6, ymm6, ymm7		;; R3 + I4 (new R4)

	vmovapd	ymm4, [srcreg+d2+32]		;; I3
	vmovapd	ymm7, [srcreg+d2+d1]		;; R4
	vsubpd	ymm3, ymm4, ymm7		;; I3 - R4 (new I4)
	vaddpd	ymm4, ymm4, ymm7		;; I3 + R4 (new I3)

	ystore	[srcreg+d2], ymm5		;; Save R3
	ystore	[srcreg+d2+d1], ymm6		;; Save R4
	ystore	[srcreg+d2+d1+32], ymm3		;; Save I4
	ystore	[srcreg+d2+32], ymm4		;; Save I3
	ENDM

;;
;; The last three levels of the forward FFT are performed, point-wise
;; squaring, and first three levels of the inverse FFT are performed.
;; No sin/cos multipliers are needed.
;;

yr8_8cl_eight_complex_with_square_preload MACRO
	ENDM
yr8_8cl_eight_complex_with_square MACRO srcreg,srcinc,d1,d2,d4,maxrpt,L1pt,L1pd
	yr8_8c_simple_fft_part1 srcreg,d1,d2,d4
	yr8_8c_simple_fft_with_square srcreg,d1,d2,d4,L1pt,L1pd
	yr8_8c_simple_unfft srcreg,d1,d2,d4
	bump	srcreg, srcinc
	ENDM

yr8_8c_simple_fft_with_square MACRO srcreg,d1,d2,d4,L1pt,L1pd
	vmovapd	ymm0, [srcreg+d4]		;; R5
	vmovapd	ymm7, [srcreg+d4+d1] 		;; R6
	vsubpd	ymm2, ymm0, ymm7		;; R5 - R6 (new R6)
	vaddpd	ymm0, ymm0, ymm7		;; R5 + R6 (new R5)

	vmovapd	ymm1, [srcreg+d4+32]		;; I5
	vmovapd	ymm7, [srcreg+d4+d1+32]		;; I6
	vsubpd	ymm3, ymm1, ymm7		;; I5 - I6 (new I6)
	vaddpd	ymm1, ymm1, ymm7		;; I5 + I6 (new I5)

	L1prefetchw srcreg+L1pd, L1pt

	yp_complex_square ymm2, ymm3, ymm7	;; Square R6/I6
	yp_complex_square ymm0, ymm1, ymm7	;; Square R5/I5

	L1prefetchw srcreg+d1+L1pd, L1pt

	vsubpd	ymm4, ymm0, ymm2		;; R5 - R6 (new R6)
	vaddpd	ymm0, ymm0, ymm2		;; R5 + R6 (new R5)

	vsubpd	ymm5, ymm1, ymm3		;; I5 - I6 (new I6)
	vaddpd	ymm1, ymm1, ymm3		;; I5 + I6 (new I5)

	ystore	[srcreg+d4+d1], ymm4		;; Save R6
	ystore	[srcreg+d4], ymm0		;; Save R5
	ystore	[srcreg+d4+d1+32], ymm5		;; Save I6
	ystore	[srcreg+d4+32], ymm1		;; Save I5

	vmovapd	ymm5, [srcreg+d4+d2]		;; R7
	vmovapd	ymm7, [srcreg+d4+d2+d1+32]	;; I8
	vsubpd	ymm4, ymm5, ymm7		;; R7 - I8 (new R7)
	vaddpd	ymm5, ymm5, ymm7		;; R7 + I8 (new R8)

	vmovapd	ymm1, [srcreg+d4+d2+32]		;; I7
	vmovapd	ymm7, [srcreg+d4+d2+d1]		;; R8
	vsubpd	ymm0, ymm1, ymm7		;; I7 - R8 (new I8)
	vaddpd	ymm1, ymm1, ymm7		;; I7 + R8 (new I7)

	L1prefetchw srcreg+d2+L1pd, L1pt

	yp_complex_square ymm5, ymm0, ymm7	;; Square R8/I8
	yp_complex_square ymm4, ymm1, ymm7	;; Square R7/I7

	L1prefetchw srcreg+d2+d1+L1pd, L1pt

	vsubpd	ymm2, ymm5, ymm4		;; R8 - R7 (new I8)
	vaddpd	ymm5, ymm5, ymm4		;; R8 + R7 (new R7)

	vsubpd	ymm3, ymm1, ymm0		;; I7 - I8 (new R8)
	vaddpd	ymm1, ymm1, ymm0		;; I7 + I8 (new I7)

	ystore	[srcreg+d4+d2+d1+32], ymm2	;; Save I8
	ystore	[srcreg+d4+d2], ymm5		;; Save R7
	ystore	[srcreg+d4+d2+d1], ymm3		;; Save R8
	ystore	[srcreg+d4+d2+32], ymm1		;; Save I7

	vmovapd	ymm4, [srcreg]			;; R1
	vmovapd	ymm7, [srcreg+d1]		;; R2
	vsubpd	ymm3, ymm4, ymm7		;; R1 - R2 (new R2)
	vaddpd	ymm4, ymm4, ymm7		;; R1 + R2 (new R1)

	vmovapd	ymm6, [srcreg+32]		;; I1
	vmovapd	ymm7, [srcreg+d1+32]		;; I2
	vsubpd	ymm5, ymm6, ymm7		;; I1 - I2 (new I2)
	vaddpd	ymm6, ymm6, ymm7		;; I1 + I2 (new I1)

	L1prefetchw srcreg+d4+L1pd, L1pt

	yp_complex_square ymm3, ymm5, ymm7	;; Square R2/I2
	yp_complex_square ymm4, ymm6, ymm7	;; Square R1/I1

	L1prefetchw srcreg+d4+d1+L1pd, L1pt

	vsubpd	ymm0, ymm4, ymm3		;; R1 - R2 (new R2)
	vaddpd	ymm4, ymm4, ymm3		;; R1 + R2 (new R1)

	vsubpd	ymm1, ymm6, ymm5		;; I1 - I2 (new I2)
	vaddpd	ymm6, ymm6, ymm5		;; I1 + I2 (new I1)

	ystore	[srcreg+d1], ymm0		;; Save R2
	ystore	[srcreg], ymm4			;; Save R1
	ystore	[srcreg+d1+32], ymm1		;; Save I2
	ystore	[srcreg+32], ymm6		;; Save I1

	vmovapd	ymm6, [srcreg+d2]		;; R3
	vmovapd	ymm7, [srcreg+d2+d1+32]		;; I4
	vsubpd	ymm5, ymm6, ymm7		;; R3 - I4 (new R3)
	vaddpd	ymm6, ymm6, ymm7		;; R3 + I4 (new R4)

	vmovapd	ymm4, [srcreg+d2+32]		;; I3
	vmovapd	ymm7, [srcreg+d2+d1]		;; R4
	vsubpd	ymm3, ymm4, ymm7		;; I3 - R4 (new I4)
	vaddpd	ymm4, ymm4, ymm7		;; I3 + R4 (new I3)

	L1prefetchw srcreg+d4+d2+L1pd, L1pt

	yp_complex_square ymm6, ymm3, ymm7	;; Square R4/I4
	yp_complex_square ymm5, ymm4, ymm7	;; Square R3/I3

	L1prefetchw srcreg+d4+d2+d1+L1pd, L1pt

	vsubpd	ymm0, ymm6, ymm5		;; R4 - R3 (new I4)
	vaddpd	ymm6, ymm6, ymm5		;; R4 + R3 (new R3)

	vsubpd	ymm1, ymm4, ymm3		;; I3 - I4 (new R4)
	vaddpd	ymm4, ymm4, ymm3		;; I3 + I4 (new I3)

	ystore	[srcreg+d2+d1+32], ymm0		;; Save I4
	ystore	[srcreg+d2], ymm6		;; Save R3
	ystore	[srcreg+d2+d1], ymm1		;; Save R4
	ystore	[srcreg+d2+32], ymm4		;; Save I3
	ENDM

yr8_8c_simple_unfft MACRO srcreg,d1,d2,d4
	vmovapd	ymm1, [srcreg+d4+32]		;; I5
	vmovapd	ymm7, [srcreg+d4+d2+32]		;; I7
	vsubpd	ymm5, ymm1, ymm7		;; I5 - I7 (new R7)
	vaddpd	ymm1, ymm1, ymm7		;; I5 + I7 (new I5)

	vmovapd	ymm2, [srcreg]			;; R1
	vmovapd	ymm7, [srcreg+d2]		;; R3
	vsubpd	ymm4, ymm2, ymm7		;; R1 - R3 (new R3)
	vaddpd	ymm2, ymm2, ymm7		;; R1 + R3 (new R1)

	vsubpd	ymm0, ymm4, ymm5		;; R3 - R7 (final R7)
	vaddpd	ymm4, ymm4, ymm5		;; R3 + R7 (final R3)

	vmovapd	ymm3, [srcreg+d4+d2]		;; R7
	vmovapd	ymm7, [srcreg+d4]		;; R5
	vaddpd	ymm6, ymm3, ymm7		;; R7 + R5 (new R5)
	vsubpd	ymm3, ymm3, ymm7		;; R7 - R5 (new I7)

	ystore	[srcreg+d4+d2], ymm0		;; Save R7
	ystore	[srcreg+d2], ymm4		;; Save R3

	vmovapd	ymm4, [srcreg+32]		;; I1
	vmovapd	ymm7, [srcreg+d2+32]		;; I3
	vsubpd	ymm5, ymm4, ymm7		;; I1 - I3 (new I3)
	vaddpd	ymm4, ymm4, ymm7		;; I1 + I3 (new I1)

	vsubpd	ymm7, ymm5, ymm3		;; I3 - I7 (final I7)
	vaddpd	ymm5, ymm5, ymm3		;; I3 + I7 (final I3)

	ystore	[srcreg+d4+d2+32], ymm7		;; Save I7
	ystore	[srcreg+d2+32], ymm5		;; Save I3

	vsubpd	ymm0, ymm2, ymm6		;; R1 - R5 (final R5)
	vaddpd	ymm2, ymm2, ymm6		;; R1 + R5 (final R1)

	vsubpd	ymm6, ymm4, ymm1		;; I1 - I5 (final I5)
	vaddpd	ymm4, ymm4, ymm1		;; I1 + I5 (final I1)

	ystore	[srcreg+d4], ymm0		;; Save R5
	ystore	[srcreg], ymm2			;; Save R1
	ystore	[srcreg+d4+32], ymm6		;; Save I5
	ystore	[srcreg+32], ymm4		;; Save I1

	;; multiply R6/I6 by SQRTHALF - i*SQRTHALF
	vmovapd	ymm4, [srcreg+d4+d1+32]		;; I6
	vmovapd	ymm7, [srcreg+d4+d1]		;; R6
	vsubpd	ymm0, ymm4, ymm7		;; I6 = I6 - R6
	vaddpd	ymm4, ymm4, ymm7		;; R6 = R6 + I6

	;; multiply R8/I8 by SQRTHALF - i*SQRTHALF
	vmovapd	ymm2, [srcreg+d4+d2+d1+32]	;; I8
	vmovapd	ymm7, [srcreg+d4+d2+d1]		;; R8
	vsubpd	ymm1, ymm2, ymm7		;; I8 = I8 - R8
	vaddpd	ymm2, ymm2, ymm7		;; R8 = R8 + I8

	vmovapd	ymm5, YMM_SQRTHALF
	vmulpd	ymm0, ymm0, ymm5		;; I6 * SQRTHALF
	vmulpd	ymm4, ymm4, ymm5		;; R6 * SQRTHALF
	vmulpd	ymm1, ymm1, ymm5		;; I8 * SQRTHALF
	vmulpd	ymm2, ymm2, ymm5		;; R8 * SQRTHALF

	vmovapd	ymm6, [srcreg+d1]		;; R2
	vmovapd	ymm7, [srcreg+d2+d1]		;; R4
	vaddpd	ymm3, ymm6, ymm7		;; R2 + R4 (new R2)
	vsubpd	ymm6, ymm6, ymm7		;; R2 - R4 (new R4)

	vsubpd	ymm5, ymm2, ymm4		;; R8 - R6 (new I8)
	vaddpd	ymm2, ymm2, ymm4		;; R8 + R6 (new R6)

	vsubpd	ymm4, ymm0, ymm1		;; I6 - I8 (new R8)
	vaddpd	ymm0, ymm0, ymm1		;; I6 + I8 (new I6)

	vsubpd	ymm1, ymm3, ymm2		;; R2 - R6 (final R6)
	vaddpd	ymm3, ymm3, ymm2		;; R2 + R6 (final R2)

	ystore	[srcreg+d4+d1], ymm1		;; Save R6
	ystore	[srcreg+d1], ymm3		;; Save R2

	vmovapd	ymm1, [srcreg+d1+32]		;; I2
	vmovapd	ymm7, [srcreg+d2+d1+32]		;; I4
	vaddpd	ymm3, ymm1, ymm7		;; I2 + I4 (new I2)
	vsubpd	ymm1, ymm1, ymm7		;; I2 - I4 (new I4)

	vsubpd	ymm7, ymm6, ymm4		;; R4 - R8 (final R8)
	vaddpd	ymm6, ymm6, ymm4		;; R4 + R8 (final R4)

	vsubpd	ymm4, ymm3, ymm0		;; I2 - I6 (final I6)
	vaddpd	ymm3, ymm3, ymm0		;; I2 + I6 (final I2)

	vsubpd	ymm0, ymm1, ymm5		;; I4 - I8 (final I8)
	vaddpd	ymm1, ymm1, ymm5		;; I4 + I8 (final I4)

	ystore	[srcreg+d4+d2+d1], ymm7		;; Save R8
	ystore	[srcreg+d2+d1], ymm6		;; Save R4
	ystore	[srcreg+d4+d1+32], ymm4		;; Save I6
	ystore	[srcreg+d1+32], ymm3		;; Save I2
	ystore	[srcreg+d4+d2+d1+32], ymm0	;; Save I8
	ystore	[srcreg+d2+d1+32], ymm1		;; Save I4
	ENDM

;;
;; The last three levels of the forward FFT are performed, point-wise
;; multiplication, and first three levels of the inverse FFT are performed.
;; No sin/cos multipliers are needed.
;;

yr8_8cl_eight_complex_with_mult_preload MACRO
	ENDM
yr8_8cl_eight_complex_with_mult MACRO srcreg,srcinc,d1,d2,d4
	yr8_8c_simple_fft_part1 srcreg,d1,d2,d4
	yr8_8c_simple_fft_with_mult srcreg,srcreg+rbp,d1,d2,d4
	yr8_8c_simple_unfft srcreg,d1,d2,d4
	bump	srcreg, srcinc
	ENDM

yr8_8c_simple_fft_with_mult MACRO srcreg,altsrc,d1,d2,d4
	vmovapd	ymm0, [srcreg+d4]		;; R5
	vmovapd	ymm7, [srcreg+d4+d1] 		;; R6
	vsubpd	ymm2, ymm0, ymm7		;; R5 - R6 (new R6)
	vaddpd	ymm0, ymm0, ymm7		;; R5 + R6 (new R5)

	vmovapd	ymm1, [srcreg+d4+32]		;; I5
	vmovapd	ymm7, [srcreg+d4+d1+32]		;; I6
	vsubpd	ymm3, ymm1, ymm7		;; I5 - I6 (new I6)
	vaddpd	ymm1, ymm1, ymm7		;; I5 + I6 (new I5)

	yp_complex_mult ymm2, ymm3, [altsrc+d4+d1], [altsrc+d4+d1+32], ymm6, ymm7 ;; Mult R6/I6
	yp_complex_mult ymm0, ymm1, [altsrc+d4], [altsrc+d4+32], ymm6, ymm7 ;; Mult R5/I5

	vsubpd	ymm4, ymm0, ymm2		;; R5 - R6 (new R6)
	vaddpd	ymm0, ymm0, ymm2		;; R5 + R6 (new R5)

	vsubpd	ymm5, ymm1, ymm3		;; I5 - I6 (new I6)
	vaddpd	ymm1, ymm1, ymm3		;; I5 + I6 (new I5)

	ystore	[srcreg+d4+d1], ymm4		;; Save R6
	ystore	[srcreg+d4], ymm0		;; Save R5
	ystore	[srcreg+d4+d1+32], ymm5		;; Save I6
	ystore	[srcreg+d4+32], ymm1		;; Save I5

	vmovapd	ymm5, [srcreg+d4+d2]		;; R7
	vmovapd	ymm7, [srcreg+d4+d2+d1+32]	;; I8
	vsubpd	ymm4, ymm5, ymm7		;; R7 - I8 (new R7)
	vaddpd	ymm5, ymm5, ymm7		;; R7 + I8 (new R8)

	vmovapd	ymm1, [srcreg+d4+d2+32]		;; I7
	vmovapd	ymm7, [srcreg+d4+d2+d1]		;; R8
	vsubpd	ymm0, ymm1, ymm7		;; I7 - R8 (new I8)
	vaddpd	ymm1, ymm1, ymm7		;; I7 + R8 (new I7)

	yp_complex_mult ymm5, ymm0, [altsrc+d4+d2+d1], [altsrc+d4+d2+d1+32], ymm6, ymm7 ;; Mult R8/I8
	yp_complex_mult ymm4, ymm1, [altsrc+d4+d2], [altsrc+d4+d2+32], ymm6, ymm7 ;; Mult R7/I7

	vsubpd	ymm2, ymm5, ymm4		;; R8 - R7 (new I8)
	vaddpd	ymm5, ymm5, ymm4		;; R8 + R7 (new R7)

	vsubpd	ymm3, ymm1, ymm0		;; I7 - I8 (new R8)
	vaddpd	ymm1, ymm1, ymm0		;; I7 + I8 (new I7)

	ystore	[srcreg+d4+d2+d1+32], ymm2	;; Save I8
	ystore	[srcreg+d4+d2], ymm5		;; Save R7
	ystore	[srcreg+d4+d2+d1], ymm3		;; Save R8
	ystore	[srcreg+d4+d2+32], ymm1		;; Save I7

	vmovapd	ymm4, [srcreg]			;; R1
	vmovapd	ymm7, [srcreg+d1]		;; R2
	vsubpd	ymm3, ymm4, ymm7		;; R1 - R2 (new R2)
	vaddpd	ymm4, ymm4, ymm7		;; R1 + R2 (new R1)

	vmovapd	ymm6, [srcreg+32]		;; I1
	vmovapd	ymm7, [srcreg+d1+32]		;; I2
	vsubpd	ymm5, ymm6, ymm7		;; I1 - I2 (new I2)
	vaddpd	ymm6, ymm6, ymm7		;; I1 + I2 (new I1)

	yp_complex_mult ymm3, ymm5, [altsrc+d1], [altsrc+d1+32], ymm0, ymm7 ;; Mult R2/I2
	yp_complex_mult ymm4, ymm6, [altsrc], [altsrc+32], ymm0, ymm7 ;; Mult R1/I1

	vsubpd	ymm0, ymm4, ymm3		;; R1 - R2 (new R2)
	vaddpd	ymm4, ymm4, ymm3		;; R1 + R2 (new R1)

	vsubpd	ymm1, ymm6, ymm5		;; I1 - I2 (new I2)
	vaddpd	ymm6, ymm6, ymm5		;; I1 + I2 (new I1)

	ystore	[srcreg+d1], ymm0		;; Save R2
	ystore	[srcreg], ymm4			;; Save R1
	ystore	[srcreg+d1+32], ymm1		;; Save I2
	ystore	[srcreg+32], ymm6		;; Save I1

	vmovapd	ymm6, [srcreg+d2]		;; R3
	vmovapd	ymm7, [srcreg+d2+d1+32]		;; I4
	vsubpd	ymm5, ymm6, ymm7		;; R3 - I4 (new R3)
	vaddpd	ymm6, ymm6, ymm7		;; R3 + I4 (new R4)

	vmovapd	ymm4, [srcreg+d2+32]		;; I3
	vmovapd	ymm7, [srcreg+d2+d1]		;; R4
	vsubpd	ymm3, ymm4, ymm7		;; I3 - R4 (new I4)
	vaddpd	ymm4, ymm4, ymm7		;; I3 + R4 (new I3)

	yp_complex_mult ymm6, ymm3, [altsrc+d2+d1], [altsrc+d2+d1+32], ymm0, ymm7 ;; Mult R4/I4
	yp_complex_mult ymm5, ymm4, [altsrc+d2], [altsrc+d2+32], ymm0, ymm7 ;; Mult R3/I3

	vsubpd	ymm0, ymm6, ymm5		;; R4 - R3 (new I4)
	vaddpd	ymm6, ymm6, ymm5		;; R4 + R3 (new R3)

	vsubpd	ymm1, ymm4, ymm3		;; I3 - I4 (new R4)
	vaddpd	ymm4, ymm4, ymm3		;; I3 + I4 (new I3)

	ystore	[srcreg+d2+d1+32], ymm0		;; Save I4
	ystore	[srcreg+d2], ymm6		;; Save R3
	ystore	[srcreg+d2+d1], ymm1		;; Save R4
	ystore	[srcreg+d2+32], ymm4		;; Save I3
	ENDM

;;
;; Point-wise multiplication and first three levels of the inverse FFT are performed.
;; No sin/cos multipliers are needed.
;;

yr8_8cl_eight_complex_with_mulf_preload MACRO
	ENDM
yr8_8cl_eight_complex_with_mulf MACRO srcreg,srcinc,d1,d2,d4
	yr8_8c_simple_fft_with_mulf srcreg,d1,d2,d4
	yr8_8c_simple_unfft srcreg,d1,d2,d4
	bump	srcreg, srcinc
	ENDM

yr8_8c_simple_fft_with_mulf MACRO srcreg,d1,d2,d4
	vmovapd	ymm2, [srcreg+d4+d1][rbx]	;; R6
	vmovapd	ymm3, [srcreg+d4+d1+32][rbx]	;; I6
	vmovapd	ymm0, [srcreg+d4][rbx]		;; R5
	vmovapd	ymm1, [srcreg+d4+32][rbx]	;; I5

	yp_complex_mult ymm2, ymm3, [srcreg+d4+d1][rbp], [srcreg+d4+d1+32][rbp], ymm6, ymm7 ;; Mult R6/I6
	yp_complex_mult ymm0, ymm1, [srcreg+d4][rbp], [srcreg+d4+32][rbp], ymm6, ymm7 ;; Mult R5/I5

	vsubpd	ymm4, ymm0, ymm2		;; R5 - R6 (new R6)
	vaddpd	ymm0, ymm0, ymm2		;; R5 + R6 (new R5)

	vsubpd	ymm5, ymm1, ymm3		;; I5 - I6 (new I6)
	vaddpd	ymm1, ymm1, ymm3		;; I5 + I6 (new I5)

	ystore	[srcreg+d4+d1], ymm4		;; Save R6
	ystore	[srcreg+d4], ymm0		;; Save R5
	ystore	[srcreg+d4+d1+32], ymm5		;; Save I6
	ystore	[srcreg+d4+32], ymm1		;; Save I5

	vmovapd	ymm5, [srcreg+d4+d2+d1][rbx]	;; R8
	vmovapd	ymm0, [srcreg+d4+d2+d1+32][rbx]	;; I8
	vmovapd	ymm4, [srcreg+d4+d2][rbx]	;; R7
	vmovapd	ymm1, [srcreg+d4+d2+32][rbx]	;; I7

	yp_complex_mult ymm5, ymm0, [srcreg+d4+d2+d1][rbp], [srcreg+d4+d2+d1+32][rbp], ymm6, ymm7 ;; Mult R8/I8
	yp_complex_mult ymm4, ymm1, [srcreg+d4+d2][rbp], [srcreg+d4+d2+32][rbp], ymm6, ymm7 ;; Mult R7/I7

	vsubpd	ymm2, ymm5, ymm4		;; R8 - R7 (new I8)
	vaddpd	ymm5, ymm5, ymm4		;; R8 + R7 (new R7)

	vsubpd	ymm3, ymm1, ymm0		;; I7 - I8 (new R8)
	vaddpd	ymm1, ymm1, ymm0		;; I7 + I8 (new I7)

	ystore	[srcreg+d4+d2+d1+32], ymm2	;; Save I8
	ystore	[srcreg+d4+d2], ymm5		;; Save R7
	ystore	[srcreg+d4+d2+d1], ymm3		;; Save R8
	ystore	[srcreg+d4+d2+32], ymm1		;; Save I7

	vmovapd	ymm3, [srcreg+d1][rbx]		;; R2
	vmovapd	ymm5, [srcreg+d1+32][rbx]	;; I2
	vmovapd	ymm4, [srcreg][rbx]		;; R1
	vmovapd	ymm6, [srcreg+32][rbx]		;; I1

	yp_complex_mult ymm3, ymm5, [srcreg+d1][rbp], [srcreg+d1+32][rbp], ymm0, ymm7 ;; Mult R2/I2
	yp_complex_mult ymm4, ymm6, [srcreg][rbp], [srcreg+32][rbp], ymm0, ymm7 ;; Mult R1/I1

	vsubpd	ymm0, ymm4, ymm3		;; R1 - R2 (new R2)
	vaddpd	ymm4, ymm4, ymm3		;; R1 + R2 (new R1)

	vsubpd	ymm1, ymm6, ymm5		;; I1 - I2 (new I2)
	vaddpd	ymm6, ymm6, ymm5		;; I1 + I2 (new I1)

	ystore	[srcreg+d1], ymm0		;; Save R2
	ystore	[srcreg], ymm4			;; Save R1
	ystore	[srcreg+d1+32], ymm1		;; Save I2
	ystore	[srcreg+32], ymm6		;; Save I1

	vmovapd	ymm6, [srcreg+d2+d1][rbx]	;; R4
	vmovapd	ymm3, [srcreg+d2+d1+32][rbx]	;; I4
	vmovapd	ymm5, [srcreg+d2][rbx]		;; R3
	vmovapd	ymm4, [srcreg+d2+32][rbx]	;; I3

	yp_complex_mult ymm6, ymm3, [srcreg+d2+d1][rbp], [srcreg+d2+d1+32][rbp], ymm0, ymm7 ;; Mult R4/I4
	yp_complex_mult ymm5, ymm4, [srcreg+d2][rbp], [srcreg+d2+32][rbp], ymm0, ymm7 ;; Mult R3/I3

	vsubpd	ymm0, ymm6, ymm5		;; R4 - R3 (new I4)
	vaddpd	ymm6, ymm6, ymm5		;; R4 + R3 (new R3)

	vsubpd	ymm1, ymm4, ymm3		;; I3 - I4 (new R4)
	vaddpd	ymm4, ymm4, ymm3		;; I3 + I4 (new I3)

	ystore	[srcreg+d2+d1+32], ymm0		;; Save I4
	ystore	[srcreg+d2], ymm6		;; Save R3
	ystore	[srcreg+d2+d1], ymm1		;; Save R4
	ystore	[srcreg+d2+32], ymm4		;; Save I3
	ENDM


;; 64-bit version

IFDEF X86_64

yr8_8cl_eight_complex_with_square_preload MACRO
	ENDM
yr8_8cl_eight_complex_with_square MACRO srcreg,srcinc,d1,d2,d4,maxrpt,L1pt,L1pd

	vmovapd	ymm0, [srcreg]			;; R1
	vmovapd	ymm1, [srcreg+d4]		;; R5
	vaddpd	ymm2, ymm0, ymm1		;; R1 + R5 (new R1)		; 1-3
	vsubpd	ymm0, ymm0, ymm1		;; R1 - R5 (new R5)		; 2-4

	vmovapd	ymm1, [srcreg+d2]		;; R3
	vmovapd	ymm4, [srcreg+d4+d2]		;; R7
	vaddpd	ymm3, ymm1, ymm4		;; R3 + R7 (new R3)		; 3-5
	vsubpd	ymm1, ymm1, ymm4		;; R3 - R7 (new R7)		; 4-6

	vmovapd	ymm4, [srcreg+d1]		;; R2
	vmovapd	ymm6, [srcreg+d4+d1]		;; R6
	vaddpd	ymm5, ymm4, ymm6		;; R2 + R6 (new R2)		; 5-7
	vsubpd	ymm4, ymm4, ymm6		;; R2 - R6 (new R6)		; 6-8

	vmovapd	ymm6, [srcreg+d2+d1]		;; R4
	vmovapd	ymm8, [srcreg+d4+d2+d1]		;; R8
	vaddpd	ymm7, ymm6, ymm8		;; R4 + R8 (new R4)		; 7-9
	vsubpd	ymm6, ymm6, ymm8		;; R4 - R8 (new R8)		; 8-10
	vmovapd	ymm9, [srcreg+d1+32]		;; I2

	vaddpd	ymm8, ymm2, ymm3		;; R1 + R3 (newer R1)		; 9-11
	vsubpd	ymm2, ymm2, ymm3		;; R1 - R3 (newer R3)		; 10-12
	vmovapd	ymm10, [srcreg+d4+d1+32]	;; I6

	vaddpd	ymm3, ymm5, ymm7		;; R2 + R4 (newer R2)		; 11-13
	vsubpd	ymm5, ymm5, ymm7		;; R2 - R4 (newer R4)		; 12-14
	vmovapd	ymm11, [srcreg+d2+d1+32]	;; I4

	ystore	[srcreg], ymm8			;; Temporarily save R1		; 12
	vsubpd	ymm8, ymm9, ymm10		;; I2 - I6 (new I6)		; 13-15
	vaddpd	ymm9, ymm9, ymm10		;; I2 + I6 (new I2)		; 14-16
	vmovapd	ymm12, [srcreg+d4+d2+d1+32]	;; I8

	vsubpd	ymm10, ymm11, ymm12		;; I4 - I8 (new I8)		; 15-17
	vaddpd	ymm11, ymm11, ymm12		;; I4 + I8 (new I4)		; 16-18
	vmovapd	ymm13, YMM_SQRTHALF

	vaddpd	ymm12, ymm8, ymm6		;; I6 + R8 (new I6)		; 17-19
	vsubpd	ymm8, ymm8, ymm6		;; I6 - R8 (new I8)		; 18-20
	vmovapd	ymm6, [srcreg+32]		;; I1

	vsubpd	ymm14, ymm4, ymm10		;; R6 - I8 (new R6)		; 19-21
	vaddpd	ymm4, ymm4, ymm10		;; R6 + I8 (new R8)		; 20-22
	vmovapd	ymm10, [srcreg+d4+32]		;; I5

	vmulpd	ymm12, ymm12, ymm13		;; I6 = I6 * SQRTHALF (newer I6) ; 20-24
	vmulpd	ymm8, ymm8, ymm13		;; I8 = I8 * SQRTHALF (newer I8) ; 21-25
	vmulpd	ymm14, ymm14, ymm13		;; R6 = R6 * SQRTHALF (newer R6) ; 22-26
	vmulpd	ymm4, ymm4, ymm13		;; R8 = R8 * SQRTHALF (newer R8) ; 23-27
	vmovapd	ymm15, [srcreg+d2+32]		;; I3

	vaddpd	ymm13, ymm6, ymm10		;; I1 + I5 (new I1)		; 21-23
	vsubpd	ymm6, ymm6, ymm10		;; I1 - I5 (new I5)		; 22-24

	vmovapd	ymm10, [srcreg+d4+d2+32]	;; I7
	vaddpd	ymm7, ymm15, ymm10		;; I3 + I7 (new I3)		; 23-25
	vsubpd	ymm15, ymm15, ymm10		;; I3 - I7 (new I7)		; 24-26

	vsubpd	ymm10, ymm9, ymm11		;; I2 - I4 (newer I4)		; 25-27
	vaddpd	ymm9, ymm9, ymm11		;; I2 + I4 (newer I2)		; 26-28

	vsubpd	ymm11, ymm13, ymm7		;; I1 - I3 (newer I3)		; 27-29
	vaddpd	ymm13, ymm13, ymm7		;; I1 + I3 (newer I1)		; 28-30

	L1prefetchw srcreg+L1pd, L1pt

	vaddpd	ymm7, ymm2, ymm10		;; R3 + I4 (last R4)		; 29-31
	vsubpd	ymm2, ymm2, ymm10		;; R3 - I4 (last R3)		; 30-32

	vsubpd	ymm10, ymm11, ymm5		;; I3 - R4 (last I4)		; 31-33
	vaddpd	ymm11, ymm11, ymm5		;; I3 + R4 (last I3)		; 32-34

	vmulpd	ymm5, ymm7, ymm7		;; R4 * R4			; 32-36
	vmulpd	ymm7, ymm10, ymm7		;; I4 * R4 (I4/2)		; 34-38
	vmulpd	ymm10, ymm10, ymm10		;; I4 * I4			; 35-39
	vsubpd	ymm5, ymm5, ymm10		;; R4^2 - I4^2 (R4)		; 40-42

	vsubpd	ymm10, ymm14, ymm12		;; R6 = R6 - I6			; 33-35
	vaddpd	ymm14, ymm14, ymm12		;; I6 = R6 + I6			; 34-36

	L1prefetchw srcreg+d1+L1pd, L1pt

	vsubpd	ymm12, ymm0, ymm15		;; R5 - I7 (newer R5)		; 35-37
	vaddpd	ymm0, ymm0, ymm15		;; R5 + I7 (newer R7)		; 36-68

	vmulpd	ymm15, ymm2, ymm2		;; R3 * R3			; 36-40
	vmulpd	ymm2, ymm11, ymm2		;; I3 * R3 (I3/2)		; 37-41
	vmulpd	ymm11, ymm11, ymm11		;; I3 * I3			; 38-42
	vsubpd	ymm15, ymm15, ymm11		;; R3^2 - I3^2 (R3)		; 43-45

	vaddpd	ymm11, ymm6, ymm1		;; I5 + R7 (newer I5)		; 37-39
	vsubpd	ymm6, ymm6, ymm1		;; I5 - R7 (newer I7)		; 38-40

	vsubpd	ymm1, ymm12, ymm10		;; R5 - R6 (last R6)		; 39-41
	vaddpd	ymm12, ymm12, ymm10		;; R5 + R6 (last R5)		; 41-43

	L1prefetchw srcreg+d2+L1pd, L1pt

	vsubpd	ymm10, ymm11, ymm14		;; I5 - I6 (last I6)		; 42-44
	vaddpd	ymm11, ymm11, ymm14		;; I5 + I6 (last I5)		; 44-46

	vmulpd	ymm14, ymm1, ymm1		;; R6 * R6			; 42-45
	vmulpd	ymm1, ymm10, ymm1		;; I6 * R6 (I6/2)		; 45-49
	vmulpd	ymm10, ymm10, ymm10		;; I6 * I6			; 46-50
	vsubpd	ymm14, ymm14, ymm10		;; R6^2 - I6^2 (R6)		; 51-53

	vsubpd	ymm10, ymm4, ymm8		;; R8 = R8 - I8			; 45-47
	vaddpd	ymm4, ymm4, ymm8		;; I8 = R8 + I8			; 46-48

	L1prefetchw srcreg+d2+d1+L1pd, L1pt

	vaddpd	ymm8, ymm5, ymm15		;; R4 + R3 (new R3)		; 47-49
	vsubpd	ymm5, ymm5, ymm15		;; R4 - R3 (new I4)		; 48-50

	vmulpd	ymm15, ymm12, ymm12		;; R5 * R5			; 47-51
	vmulpd	ymm12, ymm11, ymm12		;; I5 * R5 (I5/2)		; 48-52
	vmulpd	ymm11, ymm11, ymm11		;; I5 * I5			; 49-53
	ystore	[srcreg+d2], ymm8		;; Temporarily save R3		; 50

	vsubpd	ymm8, ymm6, ymm10		;; I7 - R8 (last I8)		; 49-51
	vaddpd	ymm6, ymm6, ymm10		;; I7 + R8 (last I7)		; 50-52

	vaddpd	ymm10, ymm0, ymm4		;; R7 + I8 (last R8)		; 52-54
	vsubpd	ymm0, ymm0, ymm4		;; R7 - I8 (last R7)		; 53-55

	L1prefetchw srcreg+d4+L1pd, L1pt

	vaddpd	ymm4, ymm13, ymm9		;; I1 + I2 (last I1)		; 54-56
	vsubpd	ymm13, ymm13, ymm9		;; I1 - I2 (last I2)		; 55-57

	vsubpd	ymm15, ymm15, ymm11		;; R5^2 - I5^2 (R5)		; 56-58

	vmulpd	ymm9, ymm8, ymm8		;; I8 * I8			; 52-56
	vmulpd	ymm8, ymm8, ymm10		;; I8 * R8 (I8/2)		; 55-59
	vmulpd	ymm10, ymm10, ymm10		;; R8 * R8			; 56-60
	vsubpd	ymm10, ymm10, ymm9		;; R8^2 - I8^2 (R8)		; 61-63

	vmovapd	ymm9, [srcreg]			;; Reload R1
	vaddpd	ymm11, ymm9, ymm3		;; R1 + R2 (last R1)		; 57-59
	vsubpd	ymm9, ymm9, ymm3		;; R1 - R2 (last R2)		; 58-60

	vmulpd	ymm3, ymm6, ymm6		;; I7 * I7			; 57-61
	vmulpd	ymm6, ymm6, ymm0		;; I7 * R7 (I7/2)		; 58-62
	vmulpd	ymm0, ymm0, ymm0		;; R7 * R7			; 59-63
	vsubpd	ymm0, ymm0, ymm3		;; R7^2 - I7^2 (R7)		; 64-66

	vsubpd	ymm3, ymm2, ymm7		;; I3/2 - I4/2 (new R4/2)	; 59-61
	vaddpd	ymm2, ymm2, ymm7		;; I3/2 + I4/2 (new I3/2)	; 60-62

	vmulpd	ymm7, ymm4, ymm4		;; I1 * I1			; 60-64
	vmulpd	ymm4, ymm4, ymm11		;; I1 * R1 (I1/2)		; 61-65
	vmulpd	ymm11, ymm11, ymm11		;; R1 * R1			; 62-66
	vsubpd	ymm11, ymm11, ymm7		;; R1^2 - R1^2 (R1)		; 67-69

	vsubpd	ymm7, ymm12, ymm1		;; I5/2 - I6/2 (new I6/2)	; 62-64
	vaddpd	ymm12, ymm12, ymm1		;; I5/2 + I6/2 (new I5/2)	; 63-65

	vsubpd	ymm1, ymm6, ymm8		;; I7/2 - I8/2 (new R8/2)	; 65-67
	vaddpd	ymm6, ymm6, ymm8		;; I7/2 + I8/2 (new I7/2)	; 66-68

	vmulpd	ymm8, ymm13, ymm13		;; I2 * I2			; 63-67
	vmulpd	ymm13, ymm13, ymm9		;; I2 * R2 (I2/2)		; 64-68
	vmulpd	ymm9, ymm9, ymm9		;; R2 * R2			; 65-69
	vsubpd	ymm9, ymm9, ymm8		;; R2^2 - I2^2 (R2)		; 70-72

	vsubpd	ymm8, ymm15, ymm14		;; R5 - R6 (new R6)		; 68-70
	vaddpd	ymm15, ymm15, ymm14		;; R5 + R6 (new R5)		; 69-71

	vsubpd	ymm14, ymm4, ymm13		;; I1/2 - I2/2 (new I2/2)	; 71-73
	vaddpd	ymm4, ymm4, ymm13		;; I1/2 + I2/2 (new I1/2)	; 72-74

	vmovapd ymm13, YMM_TWO
	vmulpd	ymm3, ymm3, ymm13		;; R4/2 * 2			; 66-70
	vmulpd	ymm2, ymm2, ymm13		;; I3/2 * 2			; 67-71
	vmulpd	ymm7, ymm7, ymm13		;; I6/2 * 2			; 68-72
	vmulpd	ymm12, ymm12, ymm13		;; I5/2 * 2			; 69-73
	vmulpd	ymm1, ymm1, ymm13		;; R8/2 * 2			; 70-74
	vmulpd	ymm6, ymm6, ymm13		;; I7/2 * 2			; 71-75
	vmulpd	ymm14, ymm14, ymm13		;; I2/2 * 2			; 74-78
	vmulpd	ymm4, ymm4, ymm13		;; I1/2 * 2			; 75-79

	vsubpd	ymm13, ymm10, ymm0		;; R8 - R7 (new I8)		; 73-75
	vaddpd	ymm10, ymm10, ymm0		;; R8 + R7 (new R7)		; 74-76

	L1prefetchw srcreg+d4+d1+L1pd, L1pt

	vsubpd	ymm0, ymm11, ymm9		;; R1 - R2 (new R2)		; 75-77
	vaddpd	ymm11, ymm11, ymm9		;; R1 + R2 (new R1)		; 76-78

	;; multiply R6/I6 by SQRTHALF - i*SQRTHALF
	vsubpd	ymm9, ymm7, ymm8		;; I6 = I6 - R6			; 77-79
	vaddpd	ymm8, ymm8, ymm7		;; R6 = R6 + I6			; 78-80

	;; multiply R8/I8 by SQRTHALF - i*SQRTHALF
	vsubpd	ymm7, ymm13, ymm1		;; I8 = I8 - R8			; 79-81
	vaddpd	ymm1, ymm1, ymm13		;; R8 = R8 + I8			; 80-82

	vmovapd	ymm13, YMM_SQRTHALF
	vmulpd	ymm9, ymm9, ymm13		;; I6 * SQRTHALF		; 80-84
	vmulpd	ymm8, ymm8, ymm13		;; R6 * SQRTHALF		; 81-85
	vmulpd	ymm7, ymm7, ymm13		;; I8 * SQRTHALF		; 82-86
	vmulpd	ymm1, ymm1, ymm13		;; R8 * SQRTHALF		; 83-87

	L1prefetchw srcreg+d4+d2+L1pd, L1pt

	vaddpd	ymm13, ymm0, ymm3		;; R2 + R4 (newer R2)		; 81-83
	vsubpd	ymm0, ymm0, ymm3		;; R2 - R4 (newer R4)		; 82-84

	vaddpd	ymm3, ymm14, ymm5		;; I2 + I4 (newer I2)		; 83-85
	vsubpd	ymm14, ymm14, ymm5		;; I2 - I4 (newer I4)		; 84-86

	vsubpd	ymm5, ymm12, ymm6		;; I5 - I7 (newer R7)		; 85-87
	vaddpd	ymm12, ymm12, ymm6		;; I5 + I7 (newer I5)		; 86-88

	vsubpd	ymm6, ymm4, ymm2		;; I1 - I3 (newer I3)		; 87-89
	vaddpd	ymm4, ymm4, ymm2		;; I1 + I3 (newer I1)		; 88-90

	vsubpd	ymm2, ymm9, ymm7		;; I6 - I8 (newer R8)		; 89-91
	vaddpd	ymm9, ymm9, ymm7		;; I6 + I8 (newer I6)		; 90-92

	L1prefetchw srcreg+d4+d2+d1+L1pd, L1pt

	vsubpd	ymm7, ymm1, ymm8		;; R8 - R6 (newer I8)		; 91-93
	vaddpd	ymm1, ymm1, ymm8		;; R8 + R6 (newer R6)		; 92-94

	vsubpd	ymm8, ymm0, ymm2		;; R4 - R8 (final R8)		; 93-95
	vaddpd	ymm0, ymm0, ymm2		;; R4 + R8 (final R4)		; 94-96

	vsubpd	ymm2, ymm3, ymm9		;; I2 - I6 (final I6)		; 95-97
	vaddpd	ymm3, ymm3, ymm9		;; I2 + I6 (final I2)		; 96-98
	ystore	[srcreg+d4+d2+d1], ymm8		;; Save R8			; 96

	vsubpd	ymm9, ymm14, ymm7		;; I4 - I8 (final I8)		; 97-99
	vaddpd	ymm14, ymm14, ymm7		;; I4 + I8 (final I4)		; 98-100
	ystore	[srcreg+d2+d1], ymm0		;; Save R4			; 97
	ystore	[srcreg+d4+d1+32], ymm2		;; Save I6			; 98

	vsubpd	ymm7, ymm13, ymm1		;; R2 - R6 (final R6)		; 99-101
	vaddpd	ymm13, ymm13, ymm1		;; R2 + R6 (final R2)		; 100-102
	ystore	[srcreg+d1+32], ymm3		;; Save I2			; 99
	ystore	[srcreg+d4+d2+d1+32], ymm9	;; Save I8			; 100

	vmovapd	ymm1, [srcreg+d2]		;; Reload R3
	vsubpd	ymm8, ymm11, ymm1		;; R1 - R3 (newer R3)		; 101-103
	vaddpd	ymm11, ymm11, ymm1		;; R1 + R3 (newer R1)		; 102-104
	ystore	[srcreg+d2+d1+32], ymm14	;; Save I4			; 101
	ystore	[srcreg+d4+d1], ymm7		;; Save R6			; 102

	vaddpd	ymm1, ymm10, ymm15		;; R7 + R5 (newer R5)		; 103-105
	vsubpd	ymm10, ymm10, ymm15		;; R7 - R5 (newer I7)		; 104-106
	ystore	[srcreg+d1], ymm13		;; Save R2			; 103

	vsubpd	ymm15, ymm4, ymm12		;; I1 - I5 (final I5)		; 105-107
	vaddpd	ymm4, ymm4, ymm12		;; I1 + I5 (final I1)		; 106-108

	vsubpd	ymm12, ymm8, ymm5		;; R3 - R7 (final R7)		; 107-109
	vaddpd	ymm8, ymm8, ymm5		;; R3 + R7 (final R3)		; 108-110
	ystore	[srcreg+d4+32], ymm15		;; Save I5			; 108

	vsubpd	ymm5, ymm6, ymm10		;; I3 - I7 (final I7)		; 109-111
	vaddpd	ymm6, ymm6, ymm10		;; I3 + I7 (final I3)		; 110-112
	ystore	[srcreg+32], ymm4		;; Save I1			; 109
	ystore	[srcreg+d4+d2], ymm12		;; Save R7			; 110

	vsubpd	ymm10, ymm11, ymm1		;; R1 - R5 (final R5)		; 111-113
	vaddpd	ymm11, ymm11, ymm1		;; R1 + R5 (final R1)		; 112-114
	ystore	[srcreg+d2], ymm8		;; Save R3			; 111
	ystore	[srcreg+d4+d2+32], ymm5		;; Save I7			; 112

	ystore	[srcreg+d2+32], ymm6		;; Save I3			; 113
	ystore	[srcreg+d4], ymm10		;; Save R5			; 114
	ystore	[srcreg], ymm11			;; Save R1			; 115

	bump	srcreg, srcinc
	ENDM

;; Haswell FMA3 version

IF (@INSTR(,%yarch,<FMA3>) NE 0)

;; 16 loads, 16 stores, 9 temp stores, 9 temp loads, 13 muls, 112 FMA3, 53 reg copies = 228 uops.  228/4 = 57 clocks minimum -- less than the 65 clock optimal.
;; Timed at 69.3 clocks.
yr8_8cl_eight_complex_with_square MACRO srcreg,srcinc,d1,d2,d4,maxrpt,L1pt,L1pd

	vmovapd	ymm15, YMM_ONE			;; Load constant 1

	vmovapd	ymm1, [srcreg+d1+32]		;; I2
	vmovapd	ymm2, [srcreg+d4+d1+32]		;; I6
	yfmsubpd ymm0, ymm1, ymm15, ymm2	;; I2 - I6 (new I6)		; 1-5
	yfmaddpd ymm1, ymm1, ymm15, ymm2	;; I2 + I6 (new I2)		; 1-5

	vmovapd	ymm3, [srcreg+d2+d1]		;; R4
	vmovapd	ymm4, [srcreg+d4+d2+d1]		;; R8
	yfmsubpd ymm2, ymm3, ymm15, ymm4	;; R4 - R8 (new R8)		; 2-6
	yfmaddpd ymm3, ymm3, ymm15, ymm4	;; R4 + R8 (new R4)		; 2-6

	vmovapd	ymm5, [srcreg+d1]		;; R2
	vmovapd	ymm6, [srcreg+d4+d1]		;; R6
	yfmsubpd ymm4, ymm5, ymm15, ymm6	;; R2 - R6 (new R6)		; 3-7
	yfmaddpd ymm5, ymm5, ymm15, ymm6	;; R2 + R6 (new R2)		; 3-7

	vmovapd	ymm7, [srcreg+d2+d1+32]		;; I4
	vmovapd	ymm8, [srcreg+d4+d2+d1+32]	;; I8
	yfmsubpd ymm6, ymm7, ymm15, ymm8	;; I4 - I8 (new I8)		; 4-8
	yfmaddpd ymm7, ymm7, ymm15, ymm8	;; I4 + I8 (new I4)		; 4-8

	vmovapd	ymm9, [srcreg]			;; R1
	vmovapd	ymm10, [srcreg+d4]		;; R5
	yfmaddpd ymm8, ymm9, ymm15, ymm10	;; R1 + R5 (new R1)		; 5-9
	yfmsubpd ymm9, ymm9, ymm15, ymm10	;; R1 - R5 (new R5)		; 5-9

	vmovapd	ymm11, [srcreg+d2]		;; R3
	vmovapd	ymm12, [srcreg+d4+d2]		;; R7
	yfmaddpd ymm10, ymm11, ymm15, ymm12	;; R3 + R7 (new R3)		; 6-10
	yfmsubpd ymm11, ymm11, ymm15, ymm12	;; R3 - R7 (new R7)		; 6-10

	yfmaddpd ymm12, ymm0, ymm15, ymm2	;; I6 + R8 (newer I6)		; 7-11
	yfmsubpd ymm0, ymm0, ymm15, ymm2	;; I6 - R8 (newer I8)		; 7-11

	vmovapd	ymm13, [srcreg+d2+32]		;; I3
	vmovapd	ymm14, [srcreg+d4+d2+32]	;; I7
	yfmaddpd ymm2, ymm13, ymm15, ymm14	;; I3 + I7 (new I3)		; 8-12
	yfmsubpd ymm13, ymm13, ymm15, ymm14	;; I3 - I7 (new I7)		; 8-12

	yfmsubpd ymm14, ymm4, ymm15, ymm6	;; R6 - I8 (newer R6)		; 9-13
	yfmaddpd ymm4, ymm4, ymm15, ymm6	;; R6 + I8 (newer R8)		; 9-13

	yfmaddpd ymm6, ymm8, ymm15, ymm10	;; R1 + R3 (newer R1)		; 11-15
	yfmsubpd ymm8, ymm8, ymm15, ymm10	;; R1 - R3 (newer R3)		; 11-15
	ystore	[srcreg], ymm6			;; Temporarily save R1		; 16
	ystore	[srcreg+d2], ymm8		;; Temporarily save R3		; 16

	vmovapd	ymm8, [srcreg+32]		;; I1
	vmovapd	ymm10, [srcreg+d4+32]		;; I5
	yfmaddpd ymm6, ymm8, ymm15, ymm10	;; I1 + I5 (new I1)		; 10-14
	yfmsubpd ymm8, ymm8, ymm15, ymm10	;; I1 - I5 (new I5)		; 10-14

	yfmaddpd ymm10, ymm5, ymm15, ymm3	;; R2 + R4 (newer R2)		; 12-16
	yfmsubpd ymm5, ymm5, ymm15, ymm3	;; R2 - R4 (newer R4)		; 12-16

	yfmsubpd ymm3, ymm9, ymm15, ymm13	;; R5 - I7 (newer R5)		; 13-17
	yfmaddpd ymm9, ymm9, ymm15, ymm13	;; R5 + I7 (newer R7)		; 13-17
	L1prefetchw srcreg+L1pd, L1pt

	;; Mul R6 + I6i and R8 + I8i by SQRTHALF + SQRTHALFi
	yfmsubpd ymm13, ymm14, ymm15, ymm12	;; R6 - I6 (newest R6/SQRTHALF)	; 14-18
	yfmaddpd ymm14, ymm14, ymm15, ymm12	;; R6 + I6 (newest I6/SQRTHALF)	; 14-18

	yfmaddpd ymm12, ymm8, ymm15, ymm11	;; I5 + R7 (newer I5)		; 15-19
	yfmsubpd ymm8, ymm8, ymm15, ymm11	;; I5 - R7 (newer I7)		; 15-19

	yfmsubpd ymm11, ymm4, ymm15, ymm0	;; R8 - I8 (newest R8/SQRTHALF)	; 16-20
	yfmaddpd ymm4, ymm4, ymm15, ymm0	;; R8 + I8 (newest I8/SQRTHALF)	; 16-20
	L1prefetchw srcreg+d1+L1pd, L1pt

	yfmsubpd ymm0, ymm1, ymm15, ymm7	;; I2 - I4 (newer I4)		; 17-21
	yfmaddpd ymm1, ymm1, ymm15, ymm7	;; I2 + I4 (newer I2)		; 17-21
	ystore	[srcreg+d1], ymm10		;; Temporarily save R2		; 17
	vmovapd	ymm10, YMM_SQRTHALF

	yfmsubpd ymm7, ymm6, ymm15, ymm2	;; I1 - I3 (newer I3)		; 18-22
	yfmaddpd ymm6, ymm6, ymm15, ymm2	;; I1 + I3 (newer I1)		; 18-22

	yfmaddpd ymm2, ymm13, ymm10, ymm3	;; R5 + R6 * SQRTHALF (last R5)	; 19-23
	yfnmaddpd ymm13, ymm13, ymm10, ymm3	;; R5 - R6 * SQRTHALF (last R6)	; 19-23
	L1prefetchw srcreg+d2+L1pd, L1pt

	yfmaddpd ymm3, ymm14, ymm10, ymm12	;; I5 + I6 * SQRTHALF (last I5)	; 20-24
	yfnmaddpd ymm14, ymm14, ymm10, ymm12	;; I5 - I6 * SQRTHALF (last I6)	; 20-24

	yfmaddpd ymm12, ymm11, ymm10, ymm8	;; I7 + R8 * SQRTHALF (last I7)	; 21-25
	yfnmaddpd ymm11, ymm11, ymm10, ymm8	;; I7 - R8 * SQRTHALF (last I8)	; 21-25

	yfnmaddpd ymm8, ymm4, ymm10, ymm9	;; R7 - I8 * SQRTHALF (last R7)	; 22-26
	yfmaddpd ymm4, ymm4, ymm10, ymm9	;; R7 + I8 * SQRTHALF (last R8)	; 22-26
	vmovapd	ymm10, [srcreg+d2]		;; Reload R3
	ystore	[srcreg+d1+32], ymm1		;; Temporarily save I2		; 22

	yfmsubpd ymm9, ymm10, ymm15, ymm0	;; R3 - I4 (last R3)		; 23-27
	yfmaddpd ymm10, ymm10, ymm15, ymm0	;; R3 + I4 (last R4)		; 23-27
	ystore	[srcreg+32], ymm6		;; Temporarily save I1		; 23

	yfmaddpd ymm1, ymm7, ymm15, ymm5	;; I3 + R4 (last I3)		; 24-28
	yfmsubpd ymm7, ymm7, ymm15, ymm5	;; I3 - R4 (last I4)		; 24-28
	L1prefetchw srcreg+d2+d1+L1pd, L1pt

	vmulpd	ymm0, ymm2, ymm2		;; R5 * R5			; 25-29
	vmulpd	ymm2, ymm2, ymm3		;; R5 * I5 (I5/2)		; 25-29

	vmulpd	ymm6, ymm12, ymm12		;; I7 * I7			; 26-30
	vmulpd	ymm5, ymm13, ymm13		;; R6 * R6			; 26-30

	vmulpd	ymm12, ymm12, ymm8		;; I7 * R7 (I7/2)		; 27-31
	vmulpd	ymm15, ymm11, ymm11		;; I8 * I8			; 27-31

	yfnmaddpd ymm3, ymm3, ymm3, ymm0	;; R5^2 - I5 * I5 (R5)		; 30-34
	vmulpd	ymm0, ymm9, ymm9		;; R3 * R3			; 28-32

	yfmsubpd ymm8, ymm8, ymm8, ymm6		;; R7 * R7 - I7^2 (R7)		; 31-35
	vmulpd	ymm6, ymm10, ymm10		;; R4 * R4			; 28-32

	vmulpd	ymm9, ymm9, ymm1		;; R3 * I3 (I3/2)		; 29-33
	yfnmaddpd ymm1, ymm1, ymm1, ymm0	;; R3^2 - I3 * I3 (R3)		; 33-37

	yfnmaddpd ymm0, ymm14, ymm13, ymm2	;; I5/2 - I6 * R6 (new I6/2)	; 30-34
	yfmaddpd ymm13, ymm14, ymm13, ymm2	;; I5/2 + I6 * R6 (new I5/2)	; 31-35

	yfnmaddpd ymm14, ymm14, ymm14, ymm5	;; R6^2 - I6 * I6 (R6)		; 32-35
	yfmsubpd ymm2, ymm4, ymm4, ymm15	;; R8 * R8 - I8^2 (R8)		; 32-36

	yfnmaddpd ymm5, ymm11, ymm4, ymm12	;; I7/2 - I8 * R8 (new R8/2)	; 33-37
	L1prefetchw srcreg+d4+L1pd, L1pt

	yfmaddpd ymm11, ymm11, ymm4, ymm12	;; I7/2 + I8 * R8 (new I7/2)	; 34-38
	yfnmaddpd ymm6, ymm7, ymm7, ymm6	;; R4^2 - I4 * I4 (R4)		; 34-38
	vmovapd	ymm12, [srcreg+32]		;; Reload I1

	yfnmaddpd ymm4, ymm7, ymm10, ymm9	;; I3/2 - I4 * R4 (new R4/2)	; 35-39
	yfmaddpd ymm7, ymm7, ymm10, ymm9	;; I3/2 + I4 * R4 (new I3/2)	; 35-39

	vmovapd	ymm15, YMM_ONE			;; Load constant one
	vmovapd	ymm10, [srcreg+d1+32] 		;; Reload I2
	yfmaddpd ymm9, ymm12, ymm15, ymm10	;; I1 + I2 (last I1)		; 36-40
	yfmsubpd ymm12, ymm12, ymm15, ymm10	;; I1 - I2 (last I2)		; 36-40
 	vmovapd	ymm10, [srcreg]			;; Reload R1

 	ystore	[srcreg+d2+d1], ymm4		;; Temporarily save R4/2	; 40
	vmovapd	ymm4, [srcreg+d1] 		;; Reload R2
 	ystore	[srcreg+d2+32], ymm7		;; Temporarily save I3/2	; 40
	yfmaddpd ymm7, ymm10, ymm15, ymm4	;; R1 + R2 (last R1)		; 37-41
	yfmsubpd ymm10, ymm10, ymm15, ymm4	;; R1 - R2 (last R2)		; 37-41

	yfmsubpd ymm4, ymm3, ymm15, ymm14	;; R5 - R6 (new R6)		; 38-42
	yfmaddpd ymm3, ymm3, ymm15, ymm14	;; R5 + R6 (new R5)		; 38-42
	L1prefetchw srcreg+d4+d1+L1pd, L1pt

	yfmsubpd ymm14, ymm2, ymm15, ymm8	;; R8 - R7 (new I8)		; 39-43
	yfmaddpd ymm2, ymm2, ymm15, ymm8	;; R8 + R7 (new R7)		; 39-43

 	yfmaddpd ymm8, ymm6, ymm15, ymm1	;; R4 + R3 (new R3)		; 40-44
	yfmsubpd ymm6, ymm6, ymm15, ymm1	;; R4 - R3 (new I4)		; 40-44

	vmulpd	ymm1, ymm9, ymm9		;; I1 * I1			; 41-45
 	ystore	[srcreg+d2], ymm8		;; Temporarily save R3		; 45
	vmulpd	ymm8, ymm12, ymm12		;; I2 * I2			; 41-45

	vmulpd	ymm9, ymm9, ymm7		;; I1 * R1 (I1/2)		; 42-46
 	ystore	[srcreg+d2+d1+32], ymm6		;; Temporarily save I4		; 45
	vmovapd ymm6, YMM_TWO
	vmulpd	ymm13, ymm13, ymm6		;; I5/2 * 2			; 42-46

	yfmsubpd ymm7, ymm7, ymm7, ymm1		;; R1 * R1 - I1^2 (R1)		; 46-50
	yfmsubpd ymm8, ymm10, ymm10, ymm8	;; R2 * R2 - I2^2 (R2)		; 46-50

	;; multiply R6/I6 by SQRTHALF - i*SQRTHALF
	yfmsubpd ymm1, ymm0, ymm6, ymm4		;; I6 = I6/2 * 2 - R6		; 43-47
	yfmaddpd ymm0, ymm0, ymm6, ymm4		;; R6 = R6 + I6/2 * 2		; 43-47

	;; multiply R8/I8 by SQRTHALF - i*SQRTHALF
	yfnmaddpd ymm4, ymm5, ymm6, ymm14	;; I8 = I8 - R8/2 * 2		; 44-48
	yfmaddpd ymm5, ymm5, ymm6, ymm14	;; R8 = R8/2 * 2 + I8		; 44-48

	yfmaddpd ymm14, ymm2, ymm15, ymm3	;; R7 + R5 (newer R5)		; 45-49
	yfmsubpd ymm2, ymm2, ymm15, ymm3	;; R7 - R5 (newer I7)		; 45-49

	yfnmaddpd ymm3, ymm12, ymm10, ymm9	;; I1/2 - I2 * R2 (new I2/2)	; 47-51
	yfmaddpd ymm12, ymm12, ymm10, ymm9	;; I1/2 + I2 * R2 (new I1/2)	; 47-51
	L1prefetchw srcreg+d4+d2+L1pd, L1pt

	yfnmaddpd ymm10, ymm11, ymm6, ymm13	;; I5 - I7/2 * 2 (newer R7)	; 48-52
	yfmaddpd ymm11, ymm11, ymm6, ymm13	;; I5 + I7/2 * 2 (newer I5)	; 48-52

	yfmsubpd ymm9, ymm1, ymm15, ymm4	;; I6 - I8 (newer R8/SQRTHALF)	; 49-53
	yfmaddpd ymm1, ymm1, ymm15, ymm4	;; I6 + I8 (newer I6/SQRTHALF)	; 49-53

	yfmsubpd ymm13, ymm5, ymm15, ymm0	;; R8 - R6 (newer I8/SQRTHALF)	; 50-54
	yfmaddpd ymm5, ymm5, ymm15, ymm0	;; R8 + R6 (newer R6/SQRTHALF)	; 50-54
 	vmovapd	ymm0, [srcreg+d2+32]		;; Reload I3/2

	yfmsubpd ymm4, ymm7, ymm15, ymm8	;; R1 - R2 (new R2)		; 51-55
	yfmaddpd ymm7, ymm7, ymm15, ymm8	;; R1 + R2 (new R1)		; 51-55
	L1prefetchw srcreg+d4+d2+d1+L1pd, L1pt

	yfmaddpd ymm8, ymm12, ymm15, ymm0	;; I1/2 + I3/2 (newer I1/2)	; 52-56
	yfmsubpd ymm12, ymm12, ymm15, ymm0	;; I1/2 - I3/2 (newer I3/2)	; 52-56

	yfmsubpd ymm0, ymm8, ymm6, ymm11	;; I1/2 * 2 - I5 (final I5)	; 57-61
	yfmaddpd ymm8, ymm8, ymm6, ymm11	;; I1/2 * 2 + I5 (final I1)	; 57-61
 	vmovapd	ymm11, [srcreg+d2+d1+32]	;; Reload I4

	ystore	[srcreg+d4+32], ymm0		;; Save I5			; 62
 	yfmaddpd ymm0, ymm3, ymm6, ymm11	;; I2/2 * 2 + I4 (newer I2)	; 53-57
	yfmsubpd ymm3, ymm3, ymm6, ymm11	;; I2/2 * 2 - I4 (newer I4)	; 53-57
 	vmovapd	ymm11, [srcreg+d2+d1]		;; Reload R4/2

	ystore	[srcreg+32], ymm8		;; Save I1			; 62
	yfmaddpd ymm8, ymm11, ymm6, ymm4	;; R2 + R4/2 * 2 (newer R2)	; 56-60
	yfnmaddpd ymm11, ymm11, ymm6, ymm4	;; R2 - R4/2 * 2 (newer R4)	; 56-60

	yfmsubpd ymm4, ymm12, ymm6, ymm2	;; I3/2 * 2 - I7 (final I7)	; 58-62
	yfmaddpd ymm12, ymm12, ymm6, ymm2	;; I3/2 * 2 + I7 (final I3)	; 58-62
 	vmovapd	ymm6, [srcreg+d2]		;; Reload R3

	yfmsubpd ymm2, ymm7, ymm15, ymm6	;; R1 - R3 (newer R3)		; 59-63
	yfmaddpd ymm7, ymm7, ymm15, ymm6	;; R1 + R3 (newer R1)		; 59-63
	vmovapd	ymm6, YMM_SQRTHALF

	ystore	[srcreg+d4+d2+32], ymm4		;; Save I7			; 63
	yfnmaddpd ymm4, ymm1, ymm6, ymm0	;; I2 - I6 * SQRTHALF (final I6) ; 60-64
	yfmaddpd ymm1, ymm1, ymm6, ymm0		;; I2 + I6 * SQRTHALF (final I2) ; 60-64

	yfnmaddpd ymm0, ymm13, ymm6, ymm3	;; I4 - I8 * SQRTHALF (final I8) ; 61-65
	yfmaddpd ymm13, ymm13, ymm6, ymm3	;; I4 + I8 * SQRTHALF (final I4) ; 61-65

	yfnmaddpd ymm3, ymm9, ymm6, ymm11	;; R4 - R8 * SQRTHALF (final R8) ; 62-66
	yfmaddpd ymm9, ymm9, ymm6, ymm11	;; R4 + R8 * SQRTHALF (final R4) ; 62-66

	yfnmaddpd ymm11, ymm5, ymm6, ymm8	;; R2 - R6 * SQRTHALF (final R6) ; 63-67
	yfmaddpd ymm5, ymm5, ymm6, ymm8		;; R2 + R6 * SQRTHALF (final R2) ; 63-67
	ystore	[srcreg+d2+32], ymm12		;; Save I3			; 63

	yfmsubpd ymm8, ymm2, ymm15, ymm10	;; R3 - R7 (final R7)		; 64-68
	yfmaddpd ymm2, ymm2, ymm15, ymm10	;; R3 + R7 (final R3)		; 64-68

	yfmsubpd ymm10, ymm7, ymm15, ymm14	;; R1 - R5 (final R5)		; 65-69
	yfmaddpd ymm7, ymm7, ymm15, ymm14	;; R1 + R5 (final R1)		; 65-69
	ystore	[srcreg+d4+d1+32], ymm4		;; Save I6			; 65
	ystore	[srcreg+d1+32], ymm1		;; Save I2			; 65

	ystore	[srcreg+d4+d2+d1+32], ymm0	;; Save I8			; 66
	ystore	[srcreg+d2+d1+32], ymm13	;; Save I4			; 66
	ystore	[srcreg+d4+d2+d1], ymm3		;; Save R8			; 67
	ystore	[srcreg+d2+d1], ymm9		;; Save R4			; 67
	ystore	[srcreg+d4+d1], ymm11		;; Save R6			; 68
	ystore	[srcreg+d1], ymm5		;; Save R2			; 68
	ystore	[srcreg+d4+d2], ymm8		;; Save R7			; 69
	ystore	[srcreg+d2], ymm2		;; Save R3			; 69
	ystore	[srcreg+d4], ymm10		;; Save R5			; 70
	ystore	[srcreg], ymm7			;; Save R1			; 70

	bump	srcreg, srcinc
	ENDM

ENDIF

yr8_8cl_eight_complex_with_mult_preload MACRO
	ENDM
yr8_8cl_eight_complex_with_mult MACRO srcreg,srcinc,d1,d2,d4
	LOCAL	orig, back_to_orig

	vmovapd	ymm0, [srcreg]			;; R1
	vmovapd	ymm1, [srcreg+d4]		;; R5
	vaddpd	ymm2, ymm0, ymm1		;; R1 + R5 (new R1)		; 1-3
	vsubpd	ymm0, ymm0, ymm1		;; R1 - R5 (new R5)		; 2-4

	vmovapd	ymm1, [srcreg+d2]		;; R3
	vmovapd	ymm4, [srcreg+d4+d2]		;; R7
	vaddpd	ymm3, ymm1, ymm4		;; R3 + R7 (new R3)		; 3-5
	vsubpd	ymm1, ymm1, ymm4		;; R3 - R7 (new R7)		; 4-6

	vmovapd	ymm4, [srcreg+d1]		;; R2
	vmovapd	ymm6, [srcreg+d4+d1]		;; R6
	vaddpd	ymm5, ymm4, ymm6		;; R2 + R6 (new R2)		; 5-7
	vsubpd	ymm4, ymm4, ymm6		;; R2 - R6 (new R6)		; 6-8

	vmovapd	ymm6, [srcreg+d2+d1]		;; R4
	vmovapd	ymm8, [srcreg+d4+d2+d1]		;; R8
	vaddpd	ymm7, ymm6, ymm8		;; R4 + R8 (new R4)		; 7-9
	vsubpd	ymm6, ymm6, ymm8		;; R4 - R8 (new R8)		; 8-10
	vmovapd	ymm9, [srcreg+d1+32]		;; I2

	vaddpd	ymm8, ymm2, ymm3		;; R1 + R3 (newer R1)		; 9-11
	vsubpd	ymm2, ymm2, ymm3		;; R1 - R3 (newer R3)		; 10-12
	vmovapd	ymm10, [srcreg+d4+d1+32]	;; I6

	vaddpd	ymm3, ymm5, ymm7		;; R2 + R4 (newer R2)		; 11-13
	vsubpd	ymm5, ymm5, ymm7		;; R2 - R4 (newer R4)		; 12-14
	vmovapd	ymm11, [srcreg+d2+d1+32]	;; I4

	ystore	[srcreg], ymm8			;; Temporarily save newer R1	; 12
	vsubpd	ymm8, ymm9, ymm10		;; I2 - I6 (new I6)		; 13-15
	vaddpd	ymm9, ymm9, ymm10		;; I2 + I6 (new I2)		; 14-16
	vmovapd	ymm12, [srcreg+d4+d2+d1+32]	;; I8

	vsubpd	ymm10, ymm11, ymm12		;; I4 - I8 (new I8)		; 15-17
	vaddpd	ymm11, ymm11, ymm12		;; I4 + I8 (new I4)		; 16-18
	vmovapd	ymm13, YMM_SQRTHALF

	vaddpd	ymm12, ymm8, ymm6		;; I6 + R8 (new I6)		; 17-19
	vsubpd	ymm8, ymm8, ymm6		;; I6 - R8 (new I8)		; 18-20
	vmovapd	ymm6, [srcreg+32]		;; I1
	ystore	[srcreg+d1], ymm3		;; Temporarily save newer R2	; 14

	vsubpd	ymm14, ymm4, ymm10		;; R6 - I8 (new R6)		; 19-21
	vaddpd	ymm4, ymm4, ymm10		;; R6 + I8 (new R8)		; 20-22
	vmovapd	ymm10, [srcreg+d4+32]		;; I5

	vmulpd	ymm12, ymm12, ymm13		;; I6 = I6 * SQRTHALF (newer I6) ; 20-24
	vmulpd	ymm8, ymm8, ymm13		;; I8 = I8 * SQRTHALF (newer I8) ; 21-25
	vmulpd	ymm14, ymm14, ymm13		;; R6 = R6 * SQRTHALF (newer R6) ; 22-26
	vmulpd	ymm4, ymm4, ymm13		;; R8 = R8 * SQRTHALF (newer R8) ; 23-27
	vmovapd	ymm15, [srcreg+d2+32]		;; I3

	vaddpd	ymm13, ymm6, ymm10		;; I1 + I5 (new I1)		; 21-23
	vsubpd	ymm6, ymm6, ymm10		;; I1 - I5 (new I5)		; 22-24

	vmovapd	ymm10, [srcreg+d4+d2+32]	;; I7
	vaddpd	ymm7, ymm15, ymm10		;; I3 + I7 (new I3)		; 23-25
	vsubpd	ymm15, ymm15, ymm10		;; I3 - I7 (new I7)		; 24-26

	vsubpd	ymm10, ymm9, ymm11		;; I2 - I4 (newer I4)		; 25-27
	vaddpd	ymm9, ymm9, ymm11		;; I2 + I4 (newer I2)		; 26-28

	vsubpd	ymm11, ymm13, ymm7		;; I1 - I3 (newer I3)		; 27-29
	vaddpd	ymm13, ymm13, ymm7		;; I1 + I3 (newer I1)		; 28-30

	vaddpd	ymm7, ymm2, ymm10		;; R3 + I4 (last R4)		; 29-31
	vsubpd	ymm2, ymm2, ymm10		;; R3 - I4 (last R3)		; 30-32

	vsubpd	ymm10, ymm11, ymm5		;; I3 - R4 (last I4)		; 31-33
	vaddpd	ymm11, ymm11, ymm5		;; I3 + R4 (last I3)		; 32-34

	vsubpd	ymm5, ymm14, ymm12		;; R6 = R6 - I6			; 33-35
	vaddpd	ymm14, ymm14, ymm12		;; I6 = R6 + I6			; 34-36

	vsubpd	ymm12, ymm0, ymm15		;; R5 - I7 (newer R5)		; 35-37
	vaddpd	ymm0, ymm0, ymm15		;; R5 + I7 (newer R7)		; 36-68

	vaddpd	ymm15, ymm6, ymm1		;; I5 + R7 (newer I5)		; 37-39
	vsubpd	ymm6, ymm6, ymm1		;; I5 - R7 (newer I7)		; 38-40

	vsubpd	ymm1, ymm12, ymm5		;; R5 - R6 (last R6)		; 39-41
	vaddpd	ymm12, ymm12, ymm5		;; R5 + R6 (last R5)		; 40-42

	vsubpd	ymm5, ymm15, ymm14		;; I5 - I6 (last I6)		; 41-43
	vaddpd	ymm15, ymm15, ymm14		;; I5 + I6 (last I5)		; 42-44

	vsubpd	ymm14, ymm4, ymm8		;; R8 = R8 - I8			; 43-45
	vaddpd	ymm4, ymm4, ymm8		;; I8 = R8 + I8			; 44-46

	vsubpd	ymm8, ymm6, ymm14		;; I7 - R8 (last I8)		; 46-48
	vaddpd	ymm6, ymm6, ymm14		;; I7 + R8 (last I7)		; 47-49

	vaddpd	ymm14, ymm0, ymm4		;; R7 + I8 (last R8)		; 48-50
	vsubpd	ymm0, ymm0, ymm4		;; R7 - I8 (last R7)		; 49-51

	vaddpd	ymm4, ymm13, ymm9		;; I1 + I2 (last I1)		; 45-47
	vsubpd	ymm13, ymm13, ymm9		;; I1 - I2 (last I2)		; 50-52
	ystore	[srcreg+32], ymm4		;; Temporarily save I1		; 48

	cmp	mul4_opcode, 0			;; See if we need to do more than the original type-3 FFT multiply
	je	short orig
	yr8_dispatch call,y8complex_mult_opcode,d1,d2,d4 ;; Handle more difficult cases
	jmp	back_to_orig

orig:
	vmovapd	ymm3, [srcreg+d2+d1][rbp]	;; MemR4
	vmovapd	ymm9, [srcreg+d2+d1+32][rbp]	;; MemI4
	vmulpd	ymm4, ymm7, ymm3		;; R4 * MemR4			; 32-36
	vmulpd	ymm7, ymm7, ymm9		;; R4 * MemI4			; 33-37
	vmulpd	ymm9, ymm10, ymm9		;; I4 * MemI4			; 34-38
	vmulpd	ymm10, ymm10, ymm3		;; I4 * MemR4			; 35-39

	vmovapd	ymm3, [srcreg+d2][rbp]		;; MemR3
	vsubpd	ymm4, ymm4, ymm9		;; R4*MemR4-I4*MemI4 (new R4)	; 51-53
	vmovapd	ymm9, [srcreg+d2+32][rbp]	;; MemI3
	vaddpd	ymm7, ymm7, ymm10		;; R4*MemI4+I4*MemR4 (new I4)	; 52-54

	vmulpd	ymm10, ymm2, ymm3		;; R3 * MemR3			; 36-40
	vmulpd	ymm2, ymm2, ymm9		;; R3 * MemI3			; 37-41
	vmulpd	ymm9, ymm11, ymm9		;; I3 * MemI3			; 38-42
	vmulpd	ymm11, ymm11, ymm3		;; I3 * MemR3			; 39-43

	vmovapd	ymm3, [srcreg+d4+d1][rbp]	;; MemR6
	vsubpd	ymm10, ymm10, ymm9		;; R3*MemR3-I3*MemI3 (new R3)	; 53-55
	vmovapd	ymm9, [srcreg+d4+d1+32][rbp]	;; MemI6
	vaddpd	ymm2, ymm2, ymm11		;; R3*MemI3+I3*MemR3 (new I3)	; 54-56

	vmulpd	ymm11, ymm1, ymm3		;; R6 * MemR6			; 42-46
	vmulpd	ymm1, ymm1, ymm9		;; R6 * MemI6			; 43-47
	vmulpd	ymm9, ymm5, ymm9		;; I6 * MemI6			; 46-50
	vmulpd	ymm5, ymm5, ymm3		;; I6 * MemR6			; 47-51

	vmovapd	ymm3, [srcreg+d4][rbp]		;; MemR5
	vsubpd	ymm11, ymm11, ymm9		;; R6*MemR6-I6*MemI6 (new R6)	; 55-57
	vmovapd	ymm9, [srcreg+d4+32][rbp]	;; MemI5
	vaddpd	ymm1, ymm1, ymm5		;; R6*MemI6+I6*MemR6 (new I6)	; 56-58

	vmulpd	ymm5, ymm12, ymm3		;; R5 * MemR5			; 44-48
	vmulpd	ymm12, ymm12, ymm9		;; R5 * MemI5			; 45-49
	vmulpd	ymm9, ymm15, ymm9		;; I5 * MemI5			; 48-52
	vmulpd	ymm15, ymm15, ymm3		;; I5 * MemR5			; 49-53

	vmovapd	ymm3, [srcreg+d4+d2+d1][rbp]	;; MemR8
	vsubpd	ymm5, ymm5, ymm9		;; R5*MemR5-I5*MemI5 (new R5)	; 57-59
	vmovapd	ymm9, [srcreg+d4+d2+d1+32][rbp]	;; MemI8
	vaddpd	ymm12, ymm12, ymm15		;; R5*MemI5+I5*MemR5 (new I5)	; 58-60

	vmulpd	ymm15, ymm14, ymm3		;; R8 * MemR8			; 51-55
	vmulpd	ymm14, ymm14, ymm9		;; R8 * MemI8			; 52-56
	vmulpd	ymm9, ymm8, ymm9		;; I8 * MemI8			; 53-57
	vmulpd	ymm8, ymm8, ymm3		;; I8 * MemR8			; 54-58

	vmovapd	ymm3, [srcreg+d4+d2][rbp]	;; MemR7
	vsubpd	ymm15, ymm15, ymm9		;; R8*MemR8-I8*MemI8 (new R8)	; 59-61
	vmovapd	ymm9, [srcreg+d4+d2+32][rbp]	;; MemI7
	vaddpd	ymm14, ymm14, ymm8		;; R8*MemI8+I8*MemR8 (new I8)	; 60-62

	vmulpd	ymm8, ymm0, ymm3		;; R7 * MemR7			; 54-58
	vmulpd	ymm0, ymm0, ymm9		;; R7 * MemI7			; 55-59
	vmulpd	ymm9, ymm6, ymm9		;; I7 * MemI7			; 56-60
	vmulpd	ymm6, ymm6, ymm3		;; I7 * MemR7			; 57-61

	vmovapd	ymm3, [srcreg]			;; Reload newer R1
	vsubpd	ymm8, ymm8, ymm9		;; R7*MemR7-I7*MemI7 (new R7)	; 61-63
	vmovapd	ymm9, [srcreg+d1]		;; Reload newer R2
	vaddpd	ymm0, ymm0, ymm6		;; R7*MemI7+I7*MemR7 (new I7)	; 62-64

	vsubpd	ymm6, ymm3, ymm9		;; R1 - R2 (last R2)		; 63-65
	vaddpd	ymm3, ymm3, ymm9		;; R1 + R2 (last R1)		; 64-66

	vaddpd	ymm9, ymm4, ymm10		;; R4 + R3 (new R3)		; 65-67
	vsubpd	ymm4, ymm4, ymm10		;; R4 - R3 (new I4)		; 66-68

	vmovapd	ymm10, [srcreg+d1][rbp]		;; MemR2
	ystore	[srcreg+r8+d2], ymm9		;; Temporarily save new R3	; 68
	vmovapd	ymm9, [srcreg+d1+32][rbp]	;; MemI2
	ystore	[srcreg+r8+d2+32], ymm4		;; Temporarily save new I4	; 69

	vmulpd	ymm4, ymm6, ymm10		;; R2 * MemR2			; 66-70
	vmulpd	ymm6, ymm6, ymm9		;; R2 * MemI2			; 67-71
	vmulpd	ymm9, ymm13, ymm9		;; I2 * MemI2			; 68-72
	vmulpd	ymm13, ymm13, ymm10		;; I2 * MemR2			; 69-73

	vmovapd	ymm10, [srcreg][rbp]		;; MemR1
	vsubpd	ymm4, ymm4, ymm9		;; R2*MemR2-I2*MemI2 (new R2)	; 73-75
	vmovapd	ymm9, [srcreg+32][rbp]		;; MemI1
	vaddpd	ymm6, ymm6, ymm13		;; R2*MemI2+I2*MemR2 (new I2)	; 74-76

	vsubpd	ymm13, ymm2, ymm7		;; I3 - I4 (new R4)		; 67-69
	vaddpd	ymm2, ymm2, ymm7		;; I3 + I4 (new I3)		; 68-70

	vsubpd	ymm7, ymm12, ymm1		;; I5 - I6 (new I6)		; 69-71
	vaddpd	ymm12, ymm12, ymm1		;; I5 + I6 (new I5)		; 70-72

	vsubpd	ymm1, ymm5, ymm11		;; R5 - R6 (new R6)		; 71-73
	vaddpd	ymm5, ymm5, ymm11		;; R5 + R6 (new R5)		; 72-74

	vmulpd	ymm11, ymm3, ymm10		;; R1 * MemR1			; 70-74
	vmulpd	ymm3, ymm3, ymm9		;; R1 * MemI1			; 71-75
	vmulpd	ymm9, ymm9, [srcreg+32]		;; (reload I1) * MemI1		; 72-76
	vmulpd	ymm10, ymm10, [srcreg+32]	;; (reload I1) * MemR1		; 73-77

	vsubpd	ymm11, ymm11, ymm9		;; R1*MemR1-I1*MemI1 (new R1)	; 77-79
	vaddpd	ymm3, ymm3, ymm10		;; R1*MemI1+I1*MemR1 (new I1)	; 78-80

back_to_orig:
	vsubpd	ymm9, ymm0, ymm14		;; I7 - I8 (new R8)		; 75-77
	vaddpd	ymm0, ymm0, ymm14		;; I7 + I8 (new I7)		; 76-78

	vsubpd	ymm14, ymm15, ymm8		;; R8 - R7 (new I8)		; 79-81
	vaddpd	ymm15, ymm15, ymm8		;; R8 + R7 (new R7)		; 80-82

	vsubpd	ymm8, ymm3, ymm6		;; I1 - I2 (new I2)		; 81-83
	vaddpd	ymm3, ymm3, ymm6		;; I1 + I2 (new I1)		; 82-84

	vsubpd	ymm6, ymm11, ymm4		;; R1 - R2 (new R2)		; 83-85
	vaddpd	ymm11, ymm11, ymm4		;; R1 + R2 (new R1)		; 84-86

	;; multiply R6/I6 by SQRTHALF - i*SQRTHALF
	vsubpd	ymm4, ymm7, ymm1		;; I6 = I6 - R6			; 85-87
	vaddpd	ymm1, ymm1, ymm7		;; R6 = R6 + I6			; 86-88

	;; multiply R8/I8 by SQRTHALF - i*SQRTHALF
	vsubpd	ymm7, ymm14, ymm9		;; I8 = I8 - R8			; 87-89
	vaddpd	ymm9, ymm9, ymm14		;; R8 = R8 + I8			; 88-90

	vmovapd	ymm14, YMM_SQRTHALF
	vmulpd	ymm4, ymm4, ymm14		;; I6 * SQRTHALF		; 88-92
	vmulpd	ymm1, ymm1, ymm14		;; R6 * SQRTHALF		; 89-93
	vmulpd	ymm7, ymm7, ymm14		;; I8 * SQRTHALF		; 90-94
	vmulpd	ymm9, ymm9, ymm14		;; R8 * SQRTHALF		; 91-95

	vaddpd	ymm14, ymm6, ymm13		;; R2 + R4 (newer R2)		; 89-91
	vsubpd	ymm6, ymm6, ymm13		;; R2 - R4 (newer R4)		; 90-92

	vmovapd	ymm10, [srcreg+r8+d2+32]	;; Reload new I4
	vaddpd	ymm13, ymm8, ymm10		;; I2 + I4 (newer I2)		; 91-93
	vsubpd	ymm8, ymm8, ymm10		;; I2 - I4 (newer I4)		; 92-94

	vsubpd	ymm10, ymm12, ymm0		;; I5 - I7 (newer R7)		; 93-95
	vaddpd	ymm12, ymm12, ymm0		;; I5 + I7 (newer I5)		; 94-96

	vsubpd	ymm0, ymm3, ymm2		;; I1 - I3 (newer I3)		; 95-97
	vaddpd	ymm3, ymm3, ymm2		;; I1 + I3 (newer I1)		; 96-98

	vsubpd	ymm2, ymm4, ymm7		;; I6 - I8 (newer R8)		; 97-99
	vaddpd	ymm4, ymm4, ymm7		;; I6 + I8 (newer I6)		; 98-100

	vsubpd	ymm7, ymm9, ymm1		;; R8 - R6 (newer I8)		; 99-101
	vaddpd	ymm9, ymm9, ymm1		;; R8 + R6 (newer R6)		; 100-102

	vsubpd	ymm1, ymm6, ymm2		;; R4 - R8 (final R8)		; 101-103
	vaddpd	ymm6, ymm6, ymm2		;; R4 + R8 (final R4)		; 102-104

	vsubpd	ymm2, ymm13, ymm4		;; I2 - I6 (final I6)		; 103-105
	vaddpd	ymm13, ymm13, ymm4		;; I2 + I6 (final I2)		; 104-106
	ystore	[srcreg+r8+d4+d2+d1], ymm1	;; Save R8			; 104

	vsubpd	ymm4, ymm8, ymm7		;; I4 - I8 (final I8)		; 105-107
	vaddpd	ymm8, ymm8, ymm7		;; I4 + I8 (final I4)		; 106-108
	ystore	[srcreg+r8+d2+d1], ymm6		;; Save R4			; 105
	ystore	[srcreg+r8+d4+d1+32], ymm2	;; Save I6			; 106

	vsubpd	ymm7, ymm14, ymm9		;; R2 - R6 (final R6)		; 107-109
	vaddpd	ymm14, ymm14, ymm9		;; R2 + R6 (final R2)		; 108-110
	ystore	[srcreg+r8+d1+32], ymm13	;; Save I2			; 107
	ystore	[srcreg+r8+d4+d2+d1+32], ymm4	;; Save I8			; 108

	vmovapd	ymm1, [srcreg+r8+d2]		;; Reload new R3
	vsubpd	ymm9, ymm11, ymm1		;; R1 - R3 (newer R3)		; 109-111
	vaddpd	ymm11, ymm11, ymm1		;; R1 + R3 (newer R1)		; 110-112
	ystore	[srcreg+r8+d2+d1+32], ymm8	;; Save I4			; 109
	ystore	[srcreg+r8+d4+d1], ymm7		;; Save R6			; 110

	vaddpd	ymm1, ymm15, ymm5		;; R7 + R5 (newer R5)		; 111-113
	vsubpd	ymm15, ymm15, ymm5		;; R7 - R5 (newer I7)		; 112-114
	ystore	[srcreg+r8+d1], ymm14		;; Save R2			; 111

	vsubpd	ymm5, ymm3, ymm12		;; I1 - I5 (final I5)		; 113-115
	vaddpd	ymm3, ymm3, ymm12		;; I1 + I5 (final I1)		; 114-116

	vsubpd	ymm12, ymm9, ymm10		;; R3 - R7 (final R7)		; 115-117
	vaddpd	ymm9, ymm9, ymm10		;; R3 + R7 (final R3)		; 116-118
	ystore	[srcreg+r8+d4+32], ymm5		;; Save I5			; 116

	vsubpd	ymm10, ymm0, ymm15		;; I3 - I7 (final I7)		; 117-119
	vaddpd	ymm0, ymm0, ymm15		;; I3 + I7 (final I3)		; 118-120
	ystore	[srcreg+r8+32], ymm3		;; Save I1			; 117
	ystore	[srcreg+r8+d4+d2], ymm12	;; Save R7			; 118

	vsubpd	ymm15, ymm11, ymm1		;; R1 - R5 (final R5)		; 119-121
	vaddpd	ymm11, ymm11, ymm1		;; R1 + R5 (final R1)		; 120-122
	ystore	[srcreg+r8+d2], ymm9		;; Save R3			; 119
	ystore	[srcreg+r8+d4+d2+32], ymm10	;; Save I7			; 120

	ystore	[srcreg+r8+d2+32], ymm0		;; Save I3			; 121
	ystore	[srcreg+r8+d4], ymm15		;; Save R5			; 122
	ystore	[srcreg+r8], ymm11		;; Save R1			; 123

	bump	srcreg, srcinc
	ENDM

;; Same as above code between orig and back_to_orig, except we implement mul4_opcode options

yr8_8c_mult_opcode MACRO srcreg,d1,d2,d4
	LOCAL	mulsave, fftsave, nofftsave, dispatch, disptab, addmul, submul, muladd, muladdhard, mulsub, mulsubhard, unfft, done
	LOCAL	addmul312, addmul313, addmul132, addmul133, submul312, submul313, submul132, submul133

	movzx	r10, mul4_opcode		;; Load opcode
	cmp	r10, 80h			;; Test for FFT save
	ja	fftsave				;; opcode > 0x80, addmul,submul,muladd,mulsub,unfft with FFT save
	jb	nofftsave			;; opcode < 0x80, addmul,submul,muladd,mulsub,unfft without FFT save
						;; opcode == 0x80, plain mul with FFT save, fall through

	;; Squaring with FFT save is handled here! 
	test	rbp, rbp			;; If rbp is zero, this is a squaring
	jnz	mulsave

	ystore	[srcreg+d2+d1], ymm7		;; R4
	vmulpd	ymm4, ymm7, ymm7		;; R4 * R4			; 32-36
	vmulpd	ymm7, ymm7, ymm10		;; R4 * I4			; 33-37
	ystore	[srcreg+d2+d1+32], ymm10	;; I4
	vmulpd	ymm10, ymm10, ymm10		;; I4 * I4			; 34-38

	vsubpd	ymm4, ymm4, ymm10		;; R4*R4-I4*I4 (new R4)		; 51-53
	vaddpd	ymm7, ymm7, ymm7		;; 2*R4*I4 (new I4)		; 52-54

	ystore	[srcreg+d2], ymm2		;; R3
	vmulpd	ymm10, ymm2, ymm2		;; R3 * R3			; 36-40
	vmulpd	ymm2, ymm2, ymm11		;; R3 * I3			; 37-41
	ystore	[srcreg+d2+32], ymm11		;; I3
	vmulpd	ymm11, ymm11, ymm11		;; I3 * I3			; 38-42

	vsubpd	ymm10, ymm10, ymm11		;; R3*R3-I3*I3 (new R3)		; 53-55
	vaddpd	ymm2, ymm2, ymm2		;; 2*R3*I3 (new I3)		; 54-56

	ystore	[srcreg+d4+d1], ymm1		;; R6
	vmulpd	ymm11, ymm1, ymm1		;; R6 * R6			; 42-46
	vmulpd	ymm1, ymm1, ymm5		;; R6 * I6			; 43-47
	ystore	[srcreg+d4+d1+32], ymm5		;; I6
	vmulpd	ymm5, ymm5, ymm5		;; I6 * I6			; 46-50

	vsubpd	ymm11, ymm11, ymm5		;; R6*R6-I6*I6 (new R6)		; 55-57
	vaddpd	ymm1, ymm1, ymm1		;; 2*R6*I6 (new I6)		; 56-58

	ystore	[srcreg+d4], ymm12		;; R5
	vmulpd	ymm5, ymm12, ymm12		;; R5 * R5			; 44-48
	vmulpd	ymm12, ymm12, ymm15		;; R5 * I5			; 45-49
	ystore	[srcreg+d4+32], ymm15		;; I5
	vmulpd	ymm15, ymm15, ymm15		;; I5 * I5			; 48-52

	vsubpd	ymm5, ymm5, ymm15		;; R5*R5-I5*I5 (new R5)		; 57-59
	vaddpd	ymm12, ymm12, ymm12		;; 2*R5*I5 (new I5)		; 58-60

	ystore	[srcreg+d4+d2+d1], ymm14	;; R8
	vmulpd	ymm15, ymm14, ymm14		;; R8 * R8			; 51-55
	vmulpd	ymm14, ymm14, ymm8		;; R8 * I8			; 52-56
	ystore	[srcreg+d4+d2+d1+32], ymm8	;; I8
	vmulpd	ymm8, ymm8, ymm8		;; I8 * I8			; 53-57

	vsubpd	ymm15, ymm15, ymm8		;; R8*R8-I8*I8 (new R8)		; 59-61
	vaddpd	ymm14, ymm14, ymm14		;; 2*R8*I8 (new I8)		; 60-62

	ystore	[srcreg+d4+d2], ymm0		;; R7
	vmulpd	ymm8, ymm0, ymm0		;; R7 * R7			; 54-58
	vmulpd	ymm0, ymm0, ymm6		;; R7 * I7			; 55-59
	ystore	[srcreg+d4+d2+32], ymm6		;; I7
	vmulpd	ymm6, ymm6, ymm6		;; I7 * I7			; 56-60

	vmovapd	ymm3, [srcreg]			;; Reload newer R1
	vsubpd	ymm8, ymm8, ymm6		;; R7*R7-I7*I7 (new R7)		; 61-63
	vmovapd	ymm9, [srcreg+d1]		;; Reload newer R2
	vaddpd	ymm0, ymm0, ymm0		;; 2*R7*I7 (new I7)		; 62-64

	vsubpd	ymm6, ymm3, ymm9		;; R1 - R2 (last R2)		; 63-65
	vaddpd	ymm3, ymm3, ymm9		;; R1 + R2 (last R1)		; 64-66

	vaddpd	ymm9, ymm4, ymm10		;; R4 + R3 (new R3)		; 65-67
	vsubpd	ymm4, ymm4, ymm10		;; R4 - R3 (new I4)		; 66-68

	ystore	[srcreg+r8+d2], ymm9		;; Temporarily save new R3	; 68
	ystore	[srcreg+r8+d2+32], ymm4		;; Temporarily save new I4	; 69
	vmovapd	ymm9, [srcreg+32]		;; I1

	ystore	[srcreg+d1], ymm6		;; R2
	vmulpd	ymm4, ymm6, ymm6		;; R2 * R2			; 66-70
	vmulpd	ymm6, ymm6, ymm13		;; R2 * I2			; 67-71
	ystore	[srcreg+d1+32], ymm13		;; I2
	vmulpd	ymm13, ymm13, ymm13		;; I2 * I2			; 68-72

	vsubpd	ymm4, ymm4, ymm13		;; R2*R2-I2*I2 (new R2)		; 73-75
	vaddpd	ymm6, ymm6, ymm6		;; 2*R2*MemI2 (new I2)		; 74-76

	vsubpd	ymm13, ymm2, ymm7		;; I3 - I4 (new R4)		; 67-69
	vaddpd	ymm2, ymm2, ymm7		;; I3 + I4 (new I3)		; 68-70

	vsubpd	ymm7, ymm12, ymm1		;; I5 - I6 (new I6)		; 69-71
	vaddpd	ymm12, ymm12, ymm1		;; I5 + I6 (new I5)		; 70-72

	vsubpd	ymm1, ymm5, ymm11		;; R5 - R6 (new R6)		; 71-73
	vaddpd	ymm5, ymm5, ymm11		;; R5 + R6 (new R5)		; 72-74

	ystore	[srcreg], ymm3			;; R1
	vmulpd	ymm11, ymm3, ymm3		;; R1 * R1			; 70-74
	vmulpd	ymm3, ymm3, ymm9		;; R1 * I1			; 71-75
	vmulpd	ymm9, ymm9, ymm9		;; I1 * I1			; 72-76

	vsubpd	ymm11, ymm11, ymm9		;; R1*R1-I1*I1 (new R1)		; 77-79
	vaddpd	ymm3, ymm3, ymm3		;; 2*R1*I1 (new I1)		; 78-80
	jmp	done

mulsave:vmovapd	ymm3, [srcreg+d2+d1][rbp]	;; MemR4
	vmovapd	ymm9, [srcreg+d2+d1+32][rbp]	;; MemI4
	ystore	[srcreg+d2+d1], ymm7		;; R4
	vmulpd	ymm4, ymm7, ymm3		;; R4 * MemR4			; 32-36
	vmulpd	ymm7, ymm7, ymm9		;; R4 * MemI4			; 33-37
	ystore	[srcreg+d2+d1+32], ymm10	;; I4
	vmulpd	ymm9, ymm10, ymm9		;; I4 * MemI4			; 34-38
	vmulpd	ymm10, ymm10, ymm3		;; I4 * MemR4			; 35-39

	vmovapd	ymm3, [srcreg+d2][rbp]		;; MemR3
	vsubpd	ymm4, ymm4, ymm9		;; R4*MemR4-I4*MemI4 (new R4)	; 51-53
	vmovapd	ymm9, [srcreg+d2+32][rbp]	;; MemI3
	vaddpd	ymm7, ymm7, ymm10		;; R4*MemI4+I4*MemR4 (new I4)	; 52-54

	ystore	[srcreg+d2], ymm2		;; R3
	vmulpd	ymm10, ymm2, ymm3		;; R3 * MemR3			; 36-40
	vmulpd	ymm2, ymm2, ymm9		;; R3 * MemI3			; 37-41
	ystore	[srcreg+d2+32], ymm11		;; I3
	vmulpd	ymm9, ymm11, ymm9		;; I3 * MemI3			; 38-42
	vmulpd	ymm11, ymm11, ymm3		;; I3 * MemR3			; 39-43

	vmovapd	ymm3, [srcreg+d4+d1][rbp]	;; MemR6
	vsubpd	ymm10, ymm10, ymm9		;; R3*MemR3-I3*MemI3 (new R3)	; 53-55
	vmovapd	ymm9, [srcreg+d4+d1+32][rbp]	;; MemI6
	vaddpd	ymm2, ymm2, ymm11		;; R3*MemI3+I3*MemR3 (new I3)	; 54-56

	ystore	[srcreg+d4+d1], ymm1		;; R6
	vmulpd	ymm11, ymm1, ymm3		;; R6 * MemR6			; 42-46
	vmulpd	ymm1, ymm1, ymm9		;; R6 * MemI6			; 43-47
	ystore	[srcreg+d4+d1+32], ymm5		;; I6
	vmulpd	ymm9, ymm5, ymm9		;; I6 * MemI6			; 46-50
	vmulpd	ymm5, ymm5, ymm3		;; I6 * MemR6			; 47-51

	vmovapd	ymm3, [srcreg+d4][rbp]		;; MemR5
	vsubpd	ymm11, ymm11, ymm9		;; R6*MemR6-I6*MemI6 (new R6)	; 55-57
	vmovapd	ymm9, [srcreg+d4+32][rbp]	;; MemI5
	vaddpd	ymm1, ymm1, ymm5		;; R6*MemI6+I6*MemR6 (new I6)	; 56-58

	ystore	[srcreg+d4], ymm12		;; R5
	vmulpd	ymm5, ymm12, ymm3		;; R5 * MemR5			; 44-48
	vmulpd	ymm12, ymm12, ymm9		;; R5 * MemI5			; 45-49
	ystore	[srcreg+d4+32], ymm15		;; I5
	vmulpd	ymm9, ymm15, ymm9		;; I5 * MemI5			; 48-52
	vmulpd	ymm15, ymm15, ymm3		;; I5 * MemR5			; 49-53

	vmovapd	ymm3, [srcreg+d4+d2+d1][rbp]	;; MemR8
	vsubpd	ymm5, ymm5, ymm9		;; R5*MemR5-I5*MemI5 (new R5)	; 57-59
	vmovapd	ymm9, [srcreg+d4+d2+d1+32][rbp]	;; MemI8
	vaddpd	ymm12, ymm12, ymm15		;; R5*MemI5+I5*MemR5 (new I5)	; 58-60

	ystore	[srcreg+d4+d2+d1], ymm14	;; R8
	vmulpd	ymm15, ymm14, ymm3		;; R8 * MemR8			; 51-55
	vmulpd	ymm14, ymm14, ymm9		;; R8 * MemI8			; 52-56
	ystore	[srcreg+d4+d2+d1+32], ymm8	;; I8
	vmulpd	ymm9, ymm8, ymm9		;; I8 * MemI8			; 53-57
	vmulpd	ymm8, ymm8, ymm3		;; I8 * MemR8			; 54-58

	vmovapd	ymm3, [srcreg+d4+d2][rbp]	;; MemR7
	vsubpd	ymm15, ymm15, ymm9		;; R8*MemR8-I8*MemI8 (new R8)	; 59-61
	vmovapd	ymm9, [srcreg+d4+d2+32][rbp]	;; MemI7
	vaddpd	ymm14, ymm14, ymm8		;; R8*MemI8+I8*MemR8 (new I8)	; 60-62

	ystore	[srcreg+d4+d2], ymm0		;; R7
	vmulpd	ymm8, ymm0, ymm3		;; R7 * MemR7			; 54-58
	vmulpd	ymm0, ymm0, ymm9		;; R7 * MemI7			; 55-59
	ystore	[srcreg+d4+d2+32], ymm6		;; I7
	vmulpd	ymm9, ymm6, ymm9		;; I7 * MemI7			; 56-60
	vmulpd	ymm6, ymm6, ymm3		;; I7 * MemR7			; 57-61

	vmovapd	ymm3, [srcreg]			;; Reload newer R1
	vsubpd	ymm8, ymm8, ymm9		;; R7*MemR7-I7*MemI7 (new R7)	; 61-63
	vmovapd	ymm9, [srcreg+d1]		;; Reload newer R2
	vaddpd	ymm0, ymm0, ymm6		;; R7*MemI7+I7*MemR7 (new I7)	; 62-64

	vsubpd	ymm6, ymm3, ymm9		;; R1 - R2 (last R2)		; 63-65
	vaddpd	ymm3, ymm3, ymm9		;; R1 + R2 (last R1)		; 64-66

	vaddpd	ymm9, ymm4, ymm10		;; R4 + R3 (new R3)		; 65-67
	vsubpd	ymm4, ymm4, ymm10		;; R4 - R3 (new I4)		; 66-68

	vmovapd	ymm10, [srcreg+d1][rbp]		;; MemR2
	ystore	[srcreg+r8+d2], ymm9		;; Temporarily save new R3	; 68
	vmovapd	ymm9, [srcreg+d1+32][rbp]	;; MemI2
	ystore	[srcreg+r8+d2+32], ymm4		;; Temporarily save new I4	; 69

	ystore	[srcreg+d1], ymm6		;; R2
	vmulpd	ymm4, ymm6, ymm10		;; R2 * MemR2			; 66-70
	vmulpd	ymm6, ymm6, ymm9		;; R2 * MemI2			; 67-71
	ystore	[srcreg+d1+32], ymm13		;; I2
	vmulpd	ymm9, ymm13, ymm9		;; I2 * MemI2			; 68-72
	vmulpd	ymm13, ymm13, ymm10		;; I2 * MemR2			; 69-73

	vmovapd	ymm10, [srcreg][rbp]		;; MemR1
	vsubpd	ymm4, ymm4, ymm9		;; R2*MemR2-I2*MemI2 (new R2)	; 73-75
	vmovapd	ymm9, [srcreg+32][rbp]		;; MemI1
	vaddpd	ymm6, ymm6, ymm13		;; R2*MemI2+I2*MemR2 (new I2)	; 74-76

	vsubpd	ymm13, ymm2, ymm7		;; I3 - I4 (new R4)		; 67-69
	vaddpd	ymm2, ymm2, ymm7		;; I3 + I4 (new I3)		; 68-70

	vsubpd	ymm7, ymm12, ymm1		;; I5 - I6 (new I6)		; 69-71
	vaddpd	ymm12, ymm12, ymm1		;; I5 + I6 (new I5)		; 70-72

	vsubpd	ymm1, ymm5, ymm11		;; R5 - R6 (new R6)		; 71-73
	vaddpd	ymm5, ymm5, ymm11		;; R5 + R6 (new R5)		; 72-74

	ystore	[srcreg], ymm3			;; R1
	vmulpd	ymm11, ymm3, ymm10		;; R1 * MemR1			; 70-74
	vmulpd	ymm3, ymm3, ymm9		;; R1 * MemI1			; 71-75
	vmulpd	ymm9, ymm9, [srcreg+32]		;; (reload I1) * MemI1		; 72-76
	vmulpd	ymm10, ymm10, [srcreg+32]	;; (reload I1) * MemR1		; 73-77

	vsubpd	ymm11, ymm11, ymm9		;; R1*MemR1-I1*MemI1 (new R1)	; 77-79
	vaddpd	ymm3, ymm3, ymm10		;; R1*MemI1+I1*MemR1 (new I1)	; 78-80
	jmp	done

fftsave:ystore	[srcreg+d2+d1], ymm7		;; R4
	ystore	[srcreg+d2+d1+32], ymm10	;; I4
	ystore	[srcreg+d2], ymm2		;; R3
	ystore	[srcreg+d2+32], ymm11		;; I3
	ystore	[srcreg+d4+d1], ymm1		;; R6
	ystore	[srcreg+d4+d1+32], ymm5		;; I6
	ystore	[srcreg+d4], ymm12		;; R5
	ystore	[srcreg+d4+32], ymm15		;; I5
	ystore	[srcreg+d4+d2+d1], ymm14	;; R8
	ystore	[srcreg+d4+d2+d1+32], ymm8	;; I8
	ystore	[srcreg+d4+d2], ymm0		;; R7
	ystore	[srcreg+d4+d2+32], ymm6		;; I7
	ystore	[srcreg+d1+32], ymm13		;; I2
	and	r10, 7fh			;; Strip the fft save bit from mul4_opcode
						;; Fall through to nofftsave

nofftsave:
	vmovapd	ymm3, [srcreg]			;; Reload newer R1
	vmovapd	ymm9, [srcreg+d1]		;; Reload newer R2
	vsubpd	ymm4, ymm3, ymm9		;; R1 - R2 (last R2)		; 63-65
	vaddpd	ymm3, ymm3, ymm9		;; R1 + R2 (last R1)		; 64-66
	ystore	[srcreg+d1], ymm4		;; R2
	ystore	[srcreg], ymm3			;; R1

dispatch:
	mov	r9, OFFSET disptab		;; Jump table address
	mov	r10, [r9+r10*8-8]		;; Jump table offset
	mov	r9, SRC2ARG			;; Distance to s2 arg from gwaddmul4 or gwsubmul4 or s3 arg in gwmuladd4 or gwmulsub4
	jmp	r10
disptab:
	DQ	OFFSET addmul			;; 1 = addmul
	DQ	OFFSET submul			;; 2 = submul
	DQ	OFFSET muladd			;; 3 = muladd
	DQ	OFFSET mulsub			;; 4 = mulsub
	DQ	OFFSET unfft			;; 5 = unfft
	DQ	OFFSET addmul312		;; 6 = addmul FFTval mem1 mem2
	DQ	OFFSET addmul313		;; 7 = addmul FFTval mem1 FFTval
	DQ	OFFSET addmul132		;; 8 = addmul mem1 FFTval mem2
	DQ	OFFSET addmul133		;; 9 = addmul mem1 FFTval FFTval
	DQ	OFFSET submul312		;; 10 = submul FFTval mem1 mem2
	DQ	OFFSET submul313		;; 11 = submul FFTval mem1 FFTval
	DQ	OFFSET submul132		;; 12 = submul mem1 FFTval mem2
	DQ	OFFSET submul133		;; 13 = submul mem1 FFTval FFTval

addmul:	vmovapd	ymm3, [srcreg+d2+d1][rbp]	;; MemR4
	vaddpd	ymm3, ymm3, [srcreg+d2+d1][r9]	;; MemR4
	vmovapd	ymm9, [srcreg+d2+d1+32][rbp]	;; MemI4
	vaddpd	ymm9, ymm9, [srcreg+d2+d1+32][r9];; MemI4
	vmulpd	ymm4, ymm7, ymm3		;; R4 * MemR4			; 32-36
	vmulpd	ymm7, ymm7, ymm9		;; R4 * MemI4			; 33-37
	vmulpd	ymm9, ymm10, ymm9		;; I4 * MemI4			; 34-38
	vmulpd	ymm10, ymm10, ymm3		;; I4 * MemR4			; 35-39

	vmovapd	ymm3, [srcreg+d2][rbp]		;; MemR3
	vaddpd	ymm3, ymm3, [srcreg+d2][r9]	;; MemR3
	vsubpd	ymm4, ymm4, ymm9		;; R4*MemR4-I4*MemI4 (new R4)	; 51-53
	vmovapd	ymm9, [srcreg+d2+32][rbp]	;; MemI3
	vaddpd	ymm9, ymm9, [srcreg+d2+32][r9]	;; MemI3
	vaddpd	ymm7, ymm7, ymm10		;; R4*MemI4+I4*MemR4 (new I4)	; 52-54

	vmulpd	ymm10, ymm2, ymm3		;; R3 * MemR3			; 36-40
	vmulpd	ymm2, ymm2, ymm9		;; R3 * MemI3			; 37-41
	vmulpd	ymm9, ymm11, ymm9		;; I3 * MemI3			; 38-42
	vmulpd	ymm11, ymm11, ymm3		;; I3 * MemR3			; 39-43

	vmovapd	ymm3, [srcreg+d4+d1][rbp]	;; MemR6
	vaddpd	ymm3, ymm3, [srcreg+d4+d1][r9]	;; MemR6
	vsubpd	ymm10, ymm10, ymm9		;; R3*MemR3-I3*MemI3 (new R3)	; 53-55
	vmovapd	ymm9, [srcreg+d4+d1+32][rbp]	;; MemI6
	vaddpd	ymm9, ymm9, [srcreg+d4+d1+32][r9];; MemI6
	vaddpd	ymm2, ymm2, ymm11		;; R3*MemI3+I3*MemR3 (new I3)	; 54-56

	vmulpd	ymm11, ymm1, ymm3		;; R6 * MemR6			; 42-46
	vmulpd	ymm1, ymm1, ymm9		;; R6 * MemI6			; 43-47
	vmulpd	ymm9, ymm5, ymm9		;; I6 * MemI6			; 46-50
	vmulpd	ymm5, ymm5, ymm3		;; I6 * MemR6			; 47-51

	vmovapd	ymm3, [srcreg+d4][rbp]		;; MemR5
	vaddpd	ymm3, ymm3, [srcreg+d4][r9]	;; MemR5
	vsubpd	ymm11, ymm11, ymm9		;; R6*MemR6-I6*MemI6 (new R6)	; 55-57
	vmovapd	ymm9, [srcreg+d4+32][rbp]	;; MemI5
	vaddpd	ymm9, ymm9, [srcreg+d4+32][r9]	;; MemI5
	vaddpd	ymm1, ymm1, ymm5		;; R6*MemI6+I6*MemR6 (new I6)	; 56-58

	vmulpd	ymm5, ymm12, ymm3		;; R5 * MemR5			; 44-48
	vmulpd	ymm12, ymm12, ymm9		;; R5 * MemI5			; 45-49
	vmulpd	ymm9, ymm15, ymm9		;; I5 * MemI5			; 48-52
	vmulpd	ymm15, ymm15, ymm3		;; I5 * MemR5			; 49-53

	vmovapd	ymm3, [srcreg+d4+d2+d1][rbp]	;; MemR8
	vaddpd	ymm3, ymm3, [srcreg+d4+d2+d1][r9];; MemR8
	vsubpd	ymm5, ymm5, ymm9		;; R5*MemR5-I5*MemI5 (new R5)	; 57-59
	vmovapd	ymm9, [srcreg+d4+d2+d1+32][rbp]	;; MemI8
	vaddpd	ymm9, ymm9, [srcreg+d4+d2+d1+32][r9];; MemI8
	vaddpd	ymm12, ymm12, ymm15		;; R5*MemI5+I5*MemR5 (new I5)	; 58-60

	vmulpd	ymm15, ymm14, ymm3		;; R8 * MemR8			; 51-55
	vmulpd	ymm14, ymm14, ymm9		;; R8 * MemI8			; 52-56
	vmulpd	ymm9, ymm8, ymm9		;; I8 * MemI8			; 53-57
	vmulpd	ymm8, ymm8, ymm3		;; I8 * MemR8			; 54-58

	vmovapd	ymm3, [srcreg+d4+d2][rbp]	;; MemR7
	vaddpd	ymm3, ymm3, [srcreg+d4+d2][r9]	;; MemR7
	vsubpd	ymm15, ymm15, ymm9		;; R8*MemR8-I8*MemI8 (new R8)	; 59-61
	vmovapd	ymm9, [srcreg+d4+d2+32][rbp]	;; MemI7
	vaddpd	ymm9, ymm9, [srcreg+d4+d2+32][r9] ;; MemI7
	vaddpd	ymm14, ymm14, ymm8		;; R8*MemI8+I8*MemR8 (new I8)	; 60-62

	vmulpd	ymm8, ymm0, ymm3		;; R7 * MemR7			; 54-58
	vmulpd	ymm0, ymm0, ymm9		;; R7 * MemI7			; 55-59
	vmulpd	ymm9, ymm6, ymm9		;; I7 * MemI7			; 56-60
	vmulpd	ymm6, ymm6, ymm3		;; I7 * MemR7			; 57-61

	vsubpd	ymm8, ymm8, ymm9		;; R7*MemR7-I7*MemI7 (new R7)	; 61-63
	vaddpd	ymm0, ymm0, ymm6		;; R7*MemI7+I7*MemR7 (new I7)	; 62-64

	vmovapd	ymm6, [srcreg+d1]		;; last R2
	vmovapd	ymm3, [srcreg]			;; last R1

	vaddpd	ymm9, ymm4, ymm10		;; R4 + R3 (new R3)		; 65-67
	vsubpd	ymm4, ymm4, ymm10		;; R4 - R3 (new I4)		; 66-68

	vmovapd	ymm10, [srcreg+d1][rbp]		;; MemR2
	vaddpd	ymm10, ymm10, [srcreg+d1][r9]	;; MemR2
	ystore	[srcreg+r8+d2], ymm9		;; Temporarily save new R3	; 68
	vmovapd	ymm9, [srcreg+d1+32][rbp]	;; MemI2
	vaddpd	ymm9, ymm9, [srcreg+d1+32][r9]	;; MemI2
	ystore	[srcreg+r8+d2+32], ymm4		;; Temporarily save new I4	; 69

	vmulpd	ymm4, ymm6, ymm10		;; R2 * MemR2			; 66-70
	vmulpd	ymm6, ymm6, ymm9		;; R2 * MemI2			; 67-71
	vmulpd	ymm9, ymm13, ymm9		;; I2 * MemI2			; 68-72
	vmulpd	ymm13, ymm13, ymm10		;; I2 * MemR2			; 69-73

	vmovapd	ymm10, [srcreg][rbp]		;; MemR1
	vaddpd	ymm10, ymm10, [srcreg][r9]	;; MemR1
	vsubpd	ymm4, ymm4, ymm9		;; R2*MemR2-I2*MemI2 (new R2)	; 73-75
	vmovapd	ymm9, [srcreg+32][rbp]		;; MemI1
	vaddpd	ymm9, ymm9, [srcreg+32][r9]	;; MemI1
	vaddpd	ymm6, ymm6, ymm13		;; R2*MemI2+I2*MemR2 (new I2)	; 74-76

	vsubpd	ymm13, ymm2, ymm7		;; I3 - I4 (new R4)		; 67-69
	vaddpd	ymm2, ymm2, ymm7		;; I3 + I4 (new I3)		; 68-70

	vsubpd	ymm7, ymm12, ymm1		;; I5 - I6 (new I6)		; 69-71
	vaddpd	ymm12, ymm12, ymm1		;; I5 + I6 (new I5)		; 70-72

	vsubpd	ymm1, ymm5, ymm11		;; R5 - R6 (new R6)		; 71-73
	vaddpd	ymm5, ymm5, ymm11		;; R5 + R6 (new R5)		; 72-74

	vmulpd	ymm11, ymm3, ymm10		;; R1 * MemR1			; 70-74
	vmulpd	ymm3, ymm3, ymm9		;; R1 * MemI1			; 71-75
	vmulpd	ymm9, ymm9, [srcreg+32]		;; (reload I1) * MemI1		; 72-76
	vmulpd	ymm10, ymm10, [srcreg+32]	;; (reload I1) * MemR1		; 73-77

	vsubpd	ymm11, ymm11, ymm9		;; R1*MemR1-I1*MemI1 (new R1)	; 77-79
	vaddpd	ymm3, ymm3, ymm10		;; R1*MemI1+I1*MemR1 (new I1)	; 78-80
	jmp	done

addmul133:
addmul313:
	vaddpd	ymm3, ymm7, [srcreg+d2+d1][rbp]	;; R4+MemR4
	vaddpd	ymm9, ymm10, [srcreg+d2+d1+32][rbp];; I4+MemI4
	vmulpd	ymm4, ymm7, ymm3		;; R4 * MemR4			; 32-36
	vmulpd	ymm7, ymm7, ymm9		;; R4 * MemI4			; 33-37
	vmulpd	ymm9, ymm10, ymm9		;; I4 * MemI4			; 34-38
	vmulpd	ymm10, ymm10, ymm3		;; I4 * MemR4			; 35-39

	vaddpd	ymm3, ymm2, [srcreg+d2][rbp]	;; R3+MemR3
	vsubpd	ymm4, ymm4, ymm9		;; R4*MemR4-I4*MemI4 (new R4)	; 51-53
	vaddpd	ymm9,ymm11, [srcreg+d2+32][rbp]	;; I3+MemI3
	vaddpd	ymm7, ymm7, ymm10		;; R4*MemI4+I4*MemR4 (new I4)	; 52-54
	vmulpd	ymm10, ymm2, ymm3		;; R3 * MemR3			; 36-40
	vmulpd	ymm2, ymm2, ymm9		;; R3 * MemI3			; 37-41
	vmulpd	ymm9, ymm11, ymm9		;; I3 * MemI3			; 38-42
	vmulpd	ymm11, ymm11, ymm3		;; I3 * MemR3			; 39-43

	vaddpd	ymm3, ymm1, [srcreg+d4+d1][rbp]	;; R6+MemR6
	vsubpd	ymm10, ymm10, ymm9		;; R3*MemR3-I3*MemI3 (new R3)	; 53-55
	vaddpd	ymm9, ymm5, [srcreg+d4+d1+32][rbp];; I6+MemI6
	vaddpd	ymm2, ymm2, ymm11		;; R3*MemI3+I3*MemR3 (new I3)	; 54-56
	vmulpd	ymm11, ymm1, ymm3		;; R6 * MemR6			; 42-46
	vmulpd	ymm1, ymm1, ymm9		;; R6 * MemI6			; 43-47
	vmulpd	ymm9, ymm5, ymm9		;; I6 * MemI6			; 46-50
	vmulpd	ymm5, ymm5, ymm3		;; I6 * MemR6			; 47-51

	vaddpd	ymm3, ymm12, [srcreg+d4][rbp]	;; R5+MemR5
	vsubpd	ymm11, ymm11, ymm9		;; R6*MemR6-I6*MemI6 (new R6)	; 55-57
	vaddpd	ymm9, ymm15, [srcreg+d4+32][rbp];; I5+MemI5
	vaddpd	ymm1, ymm1, ymm5		;; R6*MemI6+I6*MemR6 (new I6)	; 56-58
	vmulpd	ymm5, ymm12, ymm3		;; R5 * MemR5			; 44-48
	vmulpd	ymm12, ymm12, ymm9		;; R5 * MemI5			; 45-49
	vmulpd	ymm9, ymm15, ymm9		;; I5 * MemI5			; 48-52
	vmulpd	ymm15, ymm15, ymm3		;; I5 * MemR5			; 49-53

	vaddpd	ymm3, ymm14, [srcreg+d4+d2+d1][rbp];; R8+MemR8
	vsubpd	ymm5, ymm5, ymm9		;; R5*MemR5-I5*MemI5 (new R5)	; 57-59
	vaddpd	ymm9, ymm8, [srcreg+d4+d2+d1+32][rbp];; I8+MemI8
	vaddpd	ymm12, ymm12, ymm15		;; R5*MemI5+I5*MemR5 (new I5)	; 58-60
	vmulpd	ymm15, ymm14, ymm3		;; R8 * MemR8			; 51-55
	vmulpd	ymm14, ymm14, ymm9		;; R8 * MemI8			; 52-56
	vmulpd	ymm9, ymm8, ymm9		;; I8 * MemI8			; 53-57
	vmulpd	ymm8, ymm8, ymm3		;; I8 * MemR8			; 54-58

	vaddpd	ymm3, ymm0, [srcreg+d4+d2][rbp]	;; R7+MemR7
	vsubpd	ymm15, ymm15, ymm9		;; R8*MemR8-I8*MemI8 (new R8)	; 59-61
	vaddpd	ymm9, ymm6, [srcreg+d4+d2+32][rbp];; I7+MemI7
	vaddpd	ymm14, ymm14, ymm8		;; R8*MemI8+I8*MemR8 (new I8)	; 60-62
	vmulpd	ymm8, ymm0, ymm3		;; R7 * MemR7			; 54-58
	vmulpd	ymm0, ymm0, ymm9		;; R7 * MemI7			; 55-59
	vmulpd	ymm9, ymm6, ymm9		;; I7 * MemI7			; 56-60
	vmulpd	ymm6, ymm6, ymm3		;; I7 * MemR7			; 57-61

	vsubpd	ymm8, ymm8, ymm9		;; R7*MemR7-I7*MemI7 (new R7)	; 61-63
	vaddpd	ymm0, ymm0, ymm6		;; R7*MemI7+I7*MemR7 (new I7)	; 62-64

	vmovapd	ymm6, [srcreg+d1]		;; last R2
	vmovapd	ymm3, [srcreg]			;; last R1

	vaddpd	ymm9, ymm4, ymm10		;; R4 + R3 (new R3)		; 65-67
	vsubpd	ymm4, ymm4, ymm10		;; R4 - R3 (new I4)		; 66-68

	vaddpd	ymm10, ymm6, [srcreg+d1][rbp]	;; R2+MemR2
	ystore	[srcreg+r8+d2], ymm9		;; Temporarily save new R3	; 68
	vaddpd	ymm9, ymm13, [srcreg+d1+32][rbp];; I2+MemI2
	ystore	[srcreg+r8+d2+32], ymm4		;; Temporarily save new I4	; 69
	vmulpd	ymm4, ymm6, ymm10		;; R2 * MemR2			; 66-70
	vmulpd	ymm6, ymm6, ymm9		;; R2 * MemI2			; 67-71
	vmulpd	ymm9, ymm13, ymm9		;; I2 * MemI2			; 68-72
	vmulpd	ymm13, ymm13, ymm10		;; I2 * MemR2			; 69-73

	vaddpd	ymm10, ymm3, [srcreg][rbp]	;; R1+MemR1
	vsubpd	ymm4, ymm4, ymm9		;; R2*MemR2-I2*MemI2 (new R2)	; 73-75
	vmovapd	ymm9, [srcreg+32]		;; Reload I1
	vaddpd	ymm9, ymm9, [srcreg+32][rbp]	;; I1+MemI1
	vaddpd	ymm6, ymm6, ymm13		;; R2*MemI2+I2*MemR2 (new I2)	; 74-76

	vsubpd	ymm13, ymm2, ymm7		;; I3 - I4 (new R4)		; 67-69
	vaddpd	ymm2, ymm2, ymm7		;; I3 + I4 (new I3)		; 68-70

	vsubpd	ymm7, ymm12, ymm1		;; I5 - I6 (new I6)		; 69-71
	vaddpd	ymm12, ymm12, ymm1		;; I5 + I6 (new I5)		; 70-72

	vsubpd	ymm1, ymm5, ymm11		;; R5 - R6 (new R6)		; 71-73
	vaddpd	ymm5, ymm5, ymm11		;; R5 + R6 (new R5)		; 72-74

	vmulpd	ymm11, ymm3, ymm10		;; R1 * MemR1			; 70-74
	vmulpd	ymm3, ymm3, ymm9		;; R1 * MemI1			; 71-75
	vmulpd	ymm9, ymm9, [srcreg+32]		;; (reload I1) * MemI1		; 72-76
	vmulpd	ymm10, ymm10, [srcreg+32]	;; (reload I1) * MemR1		; 73-77

	vsubpd	ymm11, ymm11, ymm9		;; R1*MemR1-I1*MemI1 (new R1)	; 77-79
	vaddpd	ymm3, ymm3, ymm10		;; R1*MemI1+I1*MemR1 (new I1)	; 78-80
	jmp	done

addmul132:
addmul312:
	vaddpd	ymm3, ymm7, [srcreg+d2+d1][rbp]	;; R4+MemR4
	vmovapd	ymm7, [srcreg+d2+d1][r9]	;; R4=MemR4
	vaddpd	ymm9, ymm10, [srcreg+d2+d1+32][rbp];; I4+MemI4
	vmovapd	ymm10, [srcreg+d2+d1+32][r9]	;; I4=MemI4
	vmulpd	ymm4, ymm7, ymm3		;; R4 * MemR4			; 32-36
	vmulpd	ymm7, ymm7, ymm9		;; R4 * MemI4			; 33-37
	vmulpd	ymm9, ymm10, ymm9		;; I4 * MemI4			; 34-38
	vmulpd	ymm10, ymm10, ymm3		;; I4 * MemR4			; 35-39

	vaddpd	ymm3, ymm2, [srcreg+d2][rbp]	;; R3+MemR3
	vmovapd	ymm2, [srcreg+d2][r9]		;; R3=MemR3
	vsubpd	ymm4, ymm4, ymm9		;; R4*MemR4-I4*MemI4 (new R4)	; 51-53
	vaddpd	ymm9,ymm11, [srcreg+d2+32][rbp]	;; I3+MemI3
	vmovapd	ymm11, [srcreg+d2+32][r9]	;; I3=MemI3
	vaddpd	ymm7, ymm7, ymm10		;; R4*MemI4+I4*MemR4 (new I4)	; 52-54
	vmulpd	ymm10, ymm2, ymm3		;; R3 * MemR3			; 36-40
	vmulpd	ymm2, ymm2, ymm9		;; R3 * MemI3			; 37-41
	vmulpd	ymm9, ymm11, ymm9		;; I3 * MemI3			; 38-42
	vmulpd	ymm11, ymm11, ymm3		;; I3 * MemR3			; 39-43

	vaddpd	ymm3, ymm1, [srcreg+d4+d1][rbp]	;; R6+MemR6
	vmovapd	ymm1, [srcreg+d4+d1][r9]	;; R6=MemR6
	vsubpd	ymm10, ymm10, ymm9		;; R3*MemR3-I3*MemI3 (new R3)	; 53-55
	vaddpd	ymm9, ymm5, [srcreg+d4+d1+32][rbp];; I6+MemI6
	vmovapd	ymm5, [srcreg+d4+d1+32][r9]	;; I6=MemI6
	vaddpd	ymm2, ymm2, ymm11		;; R3*MemI3+I3*MemR3 (new I3)	; 54-56
	vmulpd	ymm11, ymm1, ymm3		;; R6 * MemR6			; 42-46
	vmulpd	ymm1, ymm1, ymm9		;; R6 * MemI6			; 43-47
	vmulpd	ymm9, ymm5, ymm9		;; I6 * MemI6			; 46-50
	vmulpd	ymm5, ymm5, ymm3		;; I6 * MemR6			; 47-51

	vaddpd	ymm3, ymm12, [srcreg+d4][rbp]	;; R5+MemR5
	vmovapd	ymm12, [srcreg+d4][r9]		;; R5=MemR5
	vsubpd	ymm11, ymm11, ymm9		;; R6*MemR6-I6*MemI6 (new R6)	; 55-57
	vaddpd	ymm9, ymm15, [srcreg+d4+32][rbp];; I5+MemI5
	vmovapd	ymm15, [srcreg+d4+32][r9]	;; I5=MemI5
	vaddpd	ymm1, ymm1, ymm5		;; R6*MemI6+I6*MemR6 (new I6)	; 56-58
	vmulpd	ymm5, ymm12, ymm3		;; R5 * MemR5			; 44-48
	vmulpd	ymm12, ymm12, ymm9		;; R5 * MemI5			; 45-49
	vmulpd	ymm9, ymm15, ymm9		;; I5 * MemI5			; 48-52
	vmulpd	ymm15, ymm15, ymm3		;; I5 * MemR5			; 49-53

	vaddpd	ymm3, ymm14, [srcreg+d4+d2+d1][rbp];; R8+MemR8
	vmovapd	ymm14, [srcreg+d4+d2+d1][r9]	;; R8=MemR8
	vsubpd	ymm5, ymm5, ymm9		;; R5*MemR5-I5*MemI5 (new R5)	; 57-59
	vaddpd	ymm9, ymm8, [srcreg+d4+d2+d1+32][rbp];; I8+MemI8
	vmovapd	ymm8, [srcreg+d4+d2+d1+32][r9]	;; I8=MemI8
	vaddpd	ymm12, ymm12, ymm15		;; R5*MemI5+I5*MemR5 (new I5)	; 58-60
	vmulpd	ymm15, ymm14, ymm3		;; R8 * MemR8			; 51-55
	vmulpd	ymm14, ymm14, ymm9		;; R8 * MemI8			; 52-56
	vmulpd	ymm9, ymm8, ymm9		;; I8 * MemI8			; 53-57
	vmulpd	ymm8, ymm8, ymm3		;; I8 * MemR8			; 54-58

	vaddpd	ymm3, ymm0, [srcreg+d4+d2][rbp]	;; R7+MemR7
	vmovapd	ymm0, [srcreg+d4+d2][r9]	;; R7=MemR7
	vsubpd	ymm15, ymm15, ymm9		;; R8*MemR8-I8*MemI8 (new R8)	; 59-61
	vaddpd	ymm9, ymm6, [srcreg+d4+d2+32][rbp];; I7+MemI7
	vmovapd	ymm6, [srcreg+d4+d2+32][r9]	;; I7=MemI7
	vaddpd	ymm14, ymm14, ymm8		;; R8*MemI8+I8*MemR8 (new I8)	; 60-62
	vmulpd	ymm8, ymm0, ymm3		;; R7 * MemR7			; 54-58
	vmulpd	ymm0, ymm0, ymm9		;; R7 * MemI7			; 55-59
	vmulpd	ymm9, ymm6, ymm9		;; I7 * MemI7			; 56-60
	vmulpd	ymm6, ymm6, ymm3		;; I7 * MemR7			; 57-61

	vsubpd	ymm8, ymm8, ymm9		;; R7*MemR7-I7*MemI7 (new R7)	; 61-63
	vaddpd	ymm0, ymm0, ymm6		;; R7*MemI7+I7*MemR7 (new I7)	; 62-64

	vmovapd	ymm6, [srcreg+d1]		;; last R2
	vmovapd	ymm3, [srcreg]			;; last R1

	vaddpd	ymm9, ymm4, ymm10		;; R4 + R3 (new R3)		; 65-67
	vsubpd	ymm4, ymm4, ymm10		;; R4 - R3 (new I4)		; 66-68

	vaddpd	ymm10, ymm6, [srcreg+d1][rbp]	;; R2+MemR2
	vmovapd	ymm6, [srcreg+d1][r9]		;; R2=MemR2
	ystore	[srcreg+r8+d2], ymm9		;; Temporarily save new R3	; 68
	vaddpd	ymm9, ymm13, [srcreg+d1+32][rbp];; I2+MemI2
	vmovapd	ymm13, [srcreg+d1+32][r9]	;; I2=MemI2
	ystore	[srcreg+r8+d2+32], ymm4		;; Temporarily save new I4	; 69
	vmulpd	ymm4, ymm6, ymm10		;; R2 * MemR2			; 66-70
	vmulpd	ymm6, ymm6, ymm9		;; R2 * MemI2			; 67-71
	vmulpd	ymm9, ymm13, ymm9		;; I2 * MemI2			; 68-72
	vmulpd	ymm13, ymm13, ymm10		;; I2 * MemR2			; 69-73

	vaddpd	ymm10, ymm3, [srcreg][rbp]	;; R1+MemR1
	vmovapd	ymm3, [srcreg][r9]		;; R1=MemR1
	vsubpd	ymm4, ymm4, ymm9		;; R2*MemR2-I2*MemI2 (new R2)	; 73-75
	vmovapd	ymm9, [srcreg+32]		;; Reload I1
	vaddpd	ymm9, ymm9, [srcreg+32][rbp]	;; I1+MemI1
	vaddpd	ymm6, ymm6, ymm13		;; R2*MemI2+I2*MemR2 (new I2)	; 74-76

	vsubpd	ymm13, ymm2, ymm7		;; I3 - I4 (new R4)		; 67-69
	vaddpd	ymm2, ymm2, ymm7		;; I3 + I4 (new I3)		; 68-70

	vsubpd	ymm7, ymm12, ymm1		;; I5 - I6 (new I6)		; 69-71
	vaddpd	ymm12, ymm12, ymm1		;; I5 + I6 (new I5)		; 70-72

	vsubpd	ymm1, ymm5, ymm11		;; R5 - R6 (new R6)		; 71-73
	vaddpd	ymm5, ymm5, ymm11		;; R5 + R6 (new R5)		; 72-74

	vmulpd	ymm11, ymm3, ymm10		;; R1 * MemR1			; 70-74
	vmulpd	ymm3, ymm3, ymm9		;; R1 * MemI1			; 71-75
	vmulpd	ymm9, ymm9, [srcreg+32][r9]	;; (reload I1) * MemI1		; 72-76
	vmulpd	ymm10, ymm10, [srcreg+32][r9]	;; (reload I1) * MemR1		; 73-77

	vsubpd	ymm11, ymm11, ymm9		;; R1*MemR1-I1*MemI1 (new R1)	; 77-79
	vaddpd	ymm3, ymm3, ymm10		;; R1*MemI1+I1*MemR1 (new I1)	; 78-80
	jmp	done

submul:	vmovapd	ymm3, [srcreg+d2+d1][rbp]	;; MemR4
	vsubpd	ymm3, ymm3, [srcreg+d2+d1][r9]	;; MemR4
	vmovapd	ymm9, [srcreg+d2+d1+32][rbp]	;; MemI4
	vsubpd	ymm9, ymm9, [srcreg+d2+d1+32][r9];; MemI4
	vmulpd	ymm4, ymm7, ymm3		;; R4 * MemR4			; 32-36
	vmulpd	ymm7, ymm7, ymm9		;; R4 * MemI4			; 33-37
	vmulpd	ymm9, ymm10, ymm9		;; I4 * MemI4			; 34-38
	vmulpd	ymm10, ymm10, ymm3		;; I4 * MemR4			; 35-39

	vmovapd	ymm3, [srcreg+d2][rbp]		;; MemR3
	vsubpd	ymm3, ymm3, [srcreg+d2][r9]	;; MemR3
	vsubpd	ymm4, ymm4, ymm9		;; R4*MemR4-I4*MemI4 (new R4)	; 51-53
	vmovapd	ymm9, [srcreg+d2+32][rbp]	;; MemI3
	vsubpd	ymm9, ymm9, [srcreg+d2+32][r9]	;; MemI3
	vaddpd	ymm7, ymm7, ymm10		;; R4*MemI4+I4*MemR4 (new I4)	; 52-54

	vmulpd	ymm10, ymm2, ymm3		;; R3 * MemR3			; 36-40
	vmulpd	ymm2, ymm2, ymm9		;; R3 * MemI3			; 37-41
	vmulpd	ymm9, ymm11, ymm9		;; I3 * MemI3			; 38-42
	vmulpd	ymm11, ymm11, ymm3		;; I3 * MemR3			; 39-43

	vmovapd	ymm3, [srcreg+d4+d1][rbp]	;; MemR6
	vsubpd	ymm3, ymm3, [srcreg+d4+d1][r9]	;; MemR6
	vsubpd	ymm10, ymm10, ymm9		;; R3*MemR3-I3*MemI3 (new R3)	; 53-55
	vmovapd	ymm9, [srcreg+d4+d1+32][rbp]	;; MemI6
	vsubpd	ymm9, ymm9, [srcreg+d4+d1+32][r9];; MemI6
	vaddpd	ymm2, ymm2, ymm11		;; R3*MemI3+I3*MemR3 (new I3)	; 54-56

	vmulpd	ymm11, ymm1, ymm3		;; R6 * MemR6			; 42-46
	vmulpd	ymm1, ymm1, ymm9		;; R6 * MemI6			; 43-47
	vmulpd	ymm9, ymm5, ymm9		;; I6 * MemI6			; 46-50
	vmulpd	ymm5, ymm5, ymm3		;; I6 * MemR6			; 47-51

	vmovapd	ymm3, [srcreg+d4][rbp]		;; MemR5
	vsubpd	ymm3, ymm3, [srcreg+d4][r9]	;; MemR5
	vsubpd	ymm11, ymm11, ymm9		;; R6*MemR6-I6*MemI6 (new R6)	; 55-57
	vmovapd	ymm9, [srcreg+d4+32][rbp]	;; MemI5
	vsubpd	ymm9, ymm9, [srcreg+d4+32][r9]	;; MemI5
	vaddpd	ymm1, ymm1, ymm5		;; R6*MemI6+I6*MemR6 (new I6)	; 56-58

	vmulpd	ymm5, ymm12, ymm3		;; R5 * MemR5			; 44-48
	vmulpd	ymm12, ymm12, ymm9		;; R5 * MemI5			; 45-49
	vmulpd	ymm9, ymm15, ymm9		;; I5 * MemI5			; 48-52
	vmulpd	ymm15, ymm15, ymm3		;; I5 * MemR5			; 49-53

	vmovapd	ymm3, [srcreg+d4+d2+d1][rbp]	;; MemR8
	vsubpd	ymm3, ymm3, [srcreg+d4+d2+d1][r9];; MemR8
	vsubpd	ymm5, ymm5, ymm9		;; R5*MemR5-I5*MemI5 (new R5)	; 57-59
	vmovapd	ymm9, [srcreg+d4+d2+d1+32][rbp]	;; MemI8
	vsubpd	ymm9, ymm9, [srcreg+d4+d2+d1+32][r9];; MemI8
	vaddpd	ymm12, ymm12, ymm15		;; R5*MemI5+I5*MemR5 (new I5)	; 58-60

	vmulpd	ymm15, ymm14, ymm3		;; R8 * MemR8			; 51-55
	vmulpd	ymm14, ymm14, ymm9		;; R8 * MemI8			; 52-56
	vmulpd	ymm9, ymm8, ymm9		;; I8 * MemI8			; 53-57
	vmulpd	ymm8, ymm8, ymm3		;; I8 * MemR8			; 54-58

	vmovapd	ymm3, [srcreg+d4+d2][rbp]	;; MemR7
	vsubpd	ymm3, ymm3, [srcreg+d4+d2][r9]	;; MemR7
	vsubpd	ymm15, ymm15, ymm9		;; R8*MemR8-I8*MemI8 (new R8)	; 59-61
	vmovapd	ymm9, [srcreg+d4+d2+32][rbp]	;; MemI7
	vsubpd	ymm9, ymm9, [srcreg+d4+d2+32][r9] ;; MemI7
	vaddpd	ymm14, ymm14, ymm8		;; R8*MemI8+I8*MemR8 (new I8)	; 60-62

	vmulpd	ymm8, ymm0, ymm3		;; R7 * MemR7			; 54-58
	vmulpd	ymm0, ymm0, ymm9		;; R7 * MemI7			; 55-59
	vmulpd	ymm9, ymm6, ymm9		;; I7 * MemI7			; 56-60
	vmulpd	ymm6, ymm6, ymm3		;; I7 * MemR7			; 57-61

	vsubpd	ymm8, ymm8, ymm9		;; R7*MemR7-I7*MemI7 (new R7)	; 61-63
	vaddpd	ymm0, ymm0, ymm6		;; R7*MemI7+I7*MemR7 (new I7)	; 62-64

	vmovapd	ymm6, [srcreg+d1]		;; last R2
	vmovapd	ymm3, [srcreg]			;; last R1

	vaddpd	ymm9, ymm4, ymm10		;; R4 + R3 (new R3)		; 65-67
	vsubpd	ymm4, ymm4, ymm10		;; R4 - R3 (new I4)		; 66-68

	vmovapd	ymm10, [srcreg+d1][rbp]		;; MemR2
	vsubpd	ymm10, ymm10, [srcreg+d1][r9]	;; MemR2
	ystore	[srcreg+r8+d2], ymm9		;; Temporarily save new R3	; 68
	vmovapd	ymm9, [srcreg+d1+32][rbp]	;; MemI2
	vsubpd	ymm9, ymm9, [srcreg+d1+32][r9]	;; MemI2
	ystore	[srcreg+r8+d2+32], ymm4		;; Temporarily save new I4	; 69

	vmulpd	ymm4, ymm6, ymm10		;; R2 * MemR2			; 66-70
	vmulpd	ymm6, ymm6, ymm9		;; R2 * MemI2			; 67-71
	vmulpd	ymm9, ymm13, ymm9		;; I2 * MemI2			; 68-72
	vmulpd	ymm13, ymm13, ymm10		;; I2 * MemR2			; 69-73

	vmovapd	ymm10, [srcreg][rbp]		;; MemR1
	vsubpd	ymm10, ymm10, [srcreg][r9]	;; MemR1
	vsubpd	ymm4, ymm4, ymm9		;; R2*MemR2-I2*MemI2 (new R2)	; 73-75
	vmovapd	ymm9, [srcreg+32][rbp]		;; MemI1
	vsubpd	ymm9, ymm9, [srcreg+32][r9]	;; MemI1
	vaddpd	ymm6, ymm6, ymm13		;; R2*MemI2+I2*MemR2 (new I2)	; 74-76

	vsubpd	ymm13, ymm2, ymm7		;; I3 - I4 (new R4)		; 67-69
	vaddpd	ymm2, ymm2, ymm7		;; I3 + I4 (new I3)		; 68-70

	vsubpd	ymm7, ymm12, ymm1		;; I5 - I6 (new I6)		; 69-71
	vaddpd	ymm12, ymm12, ymm1		;; I5 + I6 (new I5)		; 70-72

	vsubpd	ymm1, ymm5, ymm11		;; R5 - R6 (new R6)		; 71-73
	vaddpd	ymm5, ymm5, ymm11		;; R5 + R6 (new R5)		; 72-74

	vmulpd	ymm11, ymm3, ymm10		;; R1 * MemR1			; 70-74
	vmulpd	ymm3, ymm3, ymm9		;; R1 * MemI1			; 71-75
	vmulpd	ymm9, ymm9, [srcreg+32]		;; (reload I1) * MemI1		; 72-76
	vmulpd	ymm10, ymm10, [srcreg+32]	;; (reload I1) * MemR1		; 73-77

	vsubpd	ymm11, ymm11, ymm9		;; R1*MemR1-I1*MemI1 (new R1)	; 77-79
	vaddpd	ymm3, ymm3, ymm10		;; R1*MemI1+I1*MemR1 (new I1)	; 78-80
	jmp	done

submul133:
	vmovapd	ymm3, [srcreg+d2+d1][rbp]	;; MemR4
	vsubpd	ymm3, ymm3, ymm7		;; MemR4-R4
	vmovapd	ymm9, [srcreg+d2+d1+32][rbp]	;; MemI4
	vsubpd	ymm9, ymm9, ymm10		;; MemI4-I4
	vmulpd	ymm4, ymm7, ymm3		;; R4 * MemR4			; 32-36
	vmulpd	ymm7, ymm7, ymm9		;; R4 * MemI4			; 33-37
	vmulpd	ymm9, ymm10, ymm9		;; I4 * MemI4			; 34-38
	vmulpd	ymm10, ymm10, ymm3		;; I4 * MemR4			; 35-39

	vmovapd	ymm3, [srcreg+d2][rbp]		;; MemR3
	vsubpd	ymm3, ymm3, ymm2		;; MemR3-R3
	vsubpd	ymm4, ymm4, ymm9		;; R4*MemR4-I4*MemI4 (new R4)	; 51-53
	vmovapd	ymm9, [srcreg+d2+32][rbp]	;; MemI3
	vsubpd	ymm9, ymm9, ymm11		;; MemI3-I3
	vaddpd	ymm7, ymm7, ymm10		;; R4*MemI4+I4*MemR4 (new I4)	; 52-54
	vmulpd	ymm10, ymm2, ymm3		;; R3 * MemR3			; 36-40
	vmulpd	ymm2, ymm2, ymm9		;; R3 * MemI3			; 37-41
	vmulpd	ymm9, ymm11, ymm9		;; I3 * MemI3			; 38-42
	vmulpd	ymm11, ymm11, ymm3		;; I3 * MemR3			; 39-43

	vmovapd	ymm3, [srcreg+d4+d1][rbp]	;; MemR6
	vsubpd	ymm3, ymm3, ymm1		;; MemR6-R6
	vsubpd	ymm10, ymm10, ymm9		;; R3*MemR3-I3*MemI3 (new R3)	; 53-55
	vmovapd	ymm9, [srcreg+d4+d1+32][rbp]	;; MemI6
	vsubpd	ymm9, ymm9, ymm5		;; MemI6-I6
	vaddpd	ymm2, ymm2, ymm11		;; R3*MemI3+I3*MemR3 (new I3)	; 54-56
	vmulpd	ymm11, ymm1, ymm3		;; R6 * MemR6			; 42-46
	vmulpd	ymm1, ymm1, ymm9		;; R6 * MemI6			; 43-47
	vmulpd	ymm9, ymm5, ymm9		;; I6 * MemI6			; 46-50
	vmulpd	ymm5, ymm5, ymm3		;; I6 * MemR6			; 47-51

	vmovapd	ymm3, [srcreg+d4][rbp]		;; MemR5
	vsubpd	ymm3, ymm3, ymm12		;; MemR5-R5
	vsubpd	ymm11, ymm11, ymm9		;; R6*MemR6-I6*MemI6 (new R6)	; 55-57
	vmovapd	ymm9, [srcreg+d4+32][rbp]	;; MemI5
	vsubpd	ymm9, ymm9, ymm15		;; MemI5-I5
	vaddpd	ymm1, ymm1, ymm5		;; R6*MemI6+I6*MemR6 (new I6)	; 56-58
	vmulpd	ymm5, ymm12, ymm3		;; R5 * MemR5			; 44-48
	vmulpd	ymm12, ymm12, ymm9		;; R5 * MemI5			; 45-49
	vmulpd	ymm9, ymm15, ymm9		;; I5 * MemI5			; 48-52
	vmulpd	ymm15, ymm15, ymm3		;; I5 * MemR5			; 49-53

	vmovapd	ymm3, [srcreg+d4+d2+d1][rbp]	;; MemR8
	vsubpd	ymm3, ymm3, ymm14		;; MemR8-R8
	vsubpd	ymm5, ymm5, ymm9		;; R5*MemR5-I5*MemI5 (new R5)	; 57-59
	vmovapd	ymm9, [srcreg+d4+d2+d1+32][rbp]	;; MemI8
	vsubpd	ymm9, ymm9, ymm8		;; MemI8-I8
	vaddpd	ymm12, ymm12, ymm15		;; R5*MemI5+I5*MemR5 (new I5)	; 58-60
	vmulpd	ymm15, ymm14, ymm3		;; R8 * MemR8			; 51-55
	vmulpd	ymm14, ymm14, ymm9		;; R8 * MemI8			; 52-56
	vmulpd	ymm9, ymm8, ymm9		;; I8 * MemI8			; 53-57
	vmulpd	ymm8, ymm8, ymm3		;; I8 * MemR8			; 54-58

	vmovapd	ymm3, [srcreg+d4+d2][rbp]	;; MemR7
	vsubpd	ymm3, ymm3, ymm0		;; MemR7-R7
	vsubpd	ymm15, ymm15, ymm9		;; R8*MemR8-I8*MemI8 (new R8)	; 59-61
	vmovapd	ymm9, [srcreg+d4+d2+32][rbp]	;; MemI7
	vsubpd	ymm9, ymm9, ymm6		;; MemI7-I7
	vaddpd	ymm14, ymm14, ymm8		;; R8*MemI8+I8*MemR8 (new I8)	; 60-62
	vmulpd	ymm8, ymm0, ymm3		;; R7 * MemR7			; 54-58
	vmulpd	ymm0, ymm0, ymm9		;; R7 * MemI7			; 55-59
	vmulpd	ymm9, ymm6, ymm9		;; I7 * MemI7			; 56-60
	vmulpd	ymm6, ymm6, ymm3		;; I7 * MemR7			; 57-61

	vsubpd	ymm8, ymm8, ymm9		;; R7*MemR7-I7*MemI7 (new R7)	; 61-63
	vaddpd	ymm0, ymm0, ymm6		;; R7*MemI7+I7*MemR7 (new I7)	; 62-64

	vmovapd	ymm6, [srcreg+d1]		;; last R2
	vmovapd	ymm3, [srcreg]			;; last R1

	vaddpd	ymm9, ymm4, ymm10		;; R4 + R3 (new R3)		; 65-67
	vsubpd	ymm4, ymm4, ymm10		;; R4 - R3 (new I4)		; 66-68

	vmovapd	ymm10, [srcreg+d1][rbp]		;; MemR2
	vsubpd	ymm10, ymm10, ymm6		;; MemR2-R2
	ystore	[srcreg+r8+d2], ymm9		;; Temporarily save new R3	; 68
	vmovapd	ymm9, [srcreg+d1+32][rbp]	;; MemI2
	vsubpd	ymm9, ymm9, ymm13		;; MemI2-I2
	ystore	[srcreg+r8+d2+32], ymm4		;; Temporarily save new I4	; 69
	vmulpd	ymm4, ymm6, ymm10		;; R2 * MemR2			; 66-70
	vmulpd	ymm6, ymm6, ymm9		;; R2 * MemI2			; 67-71
	vmulpd	ymm9, ymm13, ymm9		;; I2 * MemI2			; 68-72
	vmulpd	ymm13, ymm13, ymm10		;; I2 * MemR2			; 69-73

	vmovapd	ymm10, [srcreg][rbp]		;; MemR1
	vsubpd	ymm10, ymm10, ymm3		;; MemR1-R1
	vsubpd	ymm4, ymm4, ymm9		;; R2*MemR2-I2*MemI2 (new R2)	; 73-75
	vmovapd	ymm9, [srcreg+32][rbp]		;; MemI1
	vsubpd	ymm9, ymm9, [srcreg+32]		;; MemI1-I1
	vaddpd	ymm6, ymm6, ymm13		;; R2*MemI2+I2*MemR2 (new I2)	; 74-76

	vsubpd	ymm13, ymm2, ymm7		;; I3 - I4 (new R4)		; 67-69
	vaddpd	ymm2, ymm2, ymm7		;; I3 + I4 (new I3)		; 68-70

	vsubpd	ymm7, ymm12, ymm1		;; I5 - I6 (new I6)		; 69-71
	vaddpd	ymm12, ymm12, ymm1		;; I5 + I6 (new I5)		; 70-72

	vsubpd	ymm1, ymm5, ymm11		;; R5 - R6 (new R6)		; 71-73
	vaddpd	ymm5, ymm5, ymm11		;; R5 + R6 (new R5)		; 72-74

	vmulpd	ymm11, ymm3, ymm10		;; R1 * MemR1			; 70-74
	vmulpd	ymm3, ymm3, ymm9		;; R1 * MemI1			; 71-75
	vmulpd	ymm9, ymm9, [srcreg+32]		;; (reload I1) * MemI1		; 72-76
	vmulpd	ymm10, ymm10, [srcreg+32]	;; (reload I1) * MemR1		; 73-77

	vsubpd	ymm11, ymm11, ymm9		;; R1*MemR1-I1*MemI1 (new R1)	; 77-79
	vaddpd	ymm3, ymm3, ymm10		;; R1*MemI1+I1*MemR1 (new I1)	; 78-80
	jmp	done

submul313:
	vsubpd	ymm3, ymm7, [srcreg+d2+d1][rbp]	;; R4-MemR4
	vsubpd	ymm9, ymm10, [srcreg+d2+d1+32][rbp];; I4-MemI4
	vmulpd	ymm4, ymm7, ymm3		;; R4 * MemR4			; 32-36
	vmulpd	ymm7, ymm7, ymm9		;; R4 * MemI4			; 33-37
	vmulpd	ymm9, ymm10, ymm9		;; I4 * MemI4			; 34-38
	vmulpd	ymm10, ymm10, ymm3		;; I4 * MemR4			; 35-39

	vsubpd	ymm3, ymm2, [srcreg+d2][rbp]	;; R3-MemR3
	vsubpd	ymm4, ymm4, ymm9		;; R4*MemR4-I4*MemI4 (new R4)	; 51-53
	vsubpd	ymm9, ymm11, [srcreg+d2+32][rbp];; I3-MemI3
	vaddpd	ymm7, ymm7, ymm10		;; R4*MemI4+I4*MemR4 (new I4)	; 52-54
	vmulpd	ymm10, ymm2, ymm3		;; R3 * MemR3			; 36-40
	vmulpd	ymm2, ymm2, ymm9		;; R3 * MemI3			; 37-41
	vmulpd	ymm9, ymm11, ymm9		;; I3 * MemI3			; 38-42
	vmulpd	ymm11, ymm11, ymm3		;; I3 * MemR3			; 39-43

	vsubpd	ymm3, ymm1, [srcreg+d4+d1][rbp]	;; R6-MemR6
	vsubpd	ymm10, ymm10, ymm9		;; R3*MemR3-I3*MemI3 (new R3)	; 53-55
	vsubpd	ymm9, ymm5, [srcreg+d4+d1+32][rbp];; I6-MemI6
	vaddpd	ymm2, ymm2, ymm11		;; R3*MemI3+I3*MemR3 (new I3)	; 54-56
	vmulpd	ymm11, ymm1, ymm3		;; R6 * MemR6			; 42-46
	vmulpd	ymm1, ymm1, ymm9		;; R6 * MemI6			; 43-47
	vmulpd	ymm9, ymm5, ymm9		;; I6 * MemI6			; 46-50
	vmulpd	ymm5, ymm5, ymm3		;; I6 * MemR6			; 47-51

	vsubpd	ymm3, ymm12, [srcreg+d4][rbp]	;; R5-MemR5
	vsubpd	ymm11, ymm11, ymm9		;; R6*MemR6-I6*MemI6 (new R6)	; 55-57
	vsubpd	ymm9, ymm15, [srcreg+d4+32][rbp];; I5-MemI5
	vaddpd	ymm1, ymm1, ymm5		;; R6*MemI6+I6*MemR6 (new I6)	; 56-58
	vmulpd	ymm5, ymm12, ymm3		;; R5 * MemR5			; 44-48
	vmulpd	ymm12, ymm12, ymm9		;; R5 * MemI5			; 45-49
	vmulpd	ymm9, ymm15, ymm9		;; I5 * MemI5			; 48-52
	vmulpd	ymm15, ymm15, ymm3		;; I5 * MemR5			; 49-53

	vsubpd	ymm3, ymm14, [srcreg+d4+d2+d1][rbp];; R8-MemR8
	vsubpd	ymm5, ymm5, ymm9		;; R5*MemR5-I5*MemI5 (new R5)	; 57-59
	vsubpd	ymm9, ymm8, [srcreg+d4+d2+d1+32][rbp];; I8-MemI8
	vaddpd	ymm12, ymm12, ymm15		;; R5*MemI5+I5*MemR5 (new I5)	; 58-60
	vmulpd	ymm15, ymm14, ymm3		;; R8 * MemR8			; 51-55
	vmulpd	ymm14, ymm14, ymm9		;; R8 * MemI8			; 52-56
	vmulpd	ymm9, ymm8, ymm9		;; I8 * MemI8			; 53-57
	vmulpd	ymm8, ymm8, ymm3		;; I8 * MemR8			; 54-58

	vsubpd	ymm3, ymm0, [srcreg+d4+d2][rbp]	;; R7-MemR7
	vsubpd	ymm15, ymm15, ymm9		;; R8*MemR8-I8*MemI8 (new R8)	; 59-61
	vsubpd	ymm9, ymm6, [srcreg+d4+d2+32][rbp];; I7-MemI7
	vaddpd	ymm14, ymm14, ymm8		;; R8*MemI8+I8*MemR8 (new I8)	; 60-62
	vmulpd	ymm8, ymm0, ymm3		;; R7 * MemR7			; 54-58
	vmulpd	ymm0, ymm0, ymm9		;; R7 * MemI7			; 55-59
	vmulpd	ymm9, ymm6, ymm9		;; I7 * MemI7			; 56-60
	vmulpd	ymm6, ymm6, ymm3		;; I7 * MemR7			; 57-61

	vsubpd	ymm8, ymm8, ymm9		;; R7*MemR7-I7*MemI7 (new R7)	; 61-63
	vaddpd	ymm0, ymm0, ymm6		;; R7*MemI7+I7*MemR7 (new I7)	; 62-64

	vmovapd	ymm6, [srcreg+d1]		;; last R2
	vmovapd	ymm3, [srcreg]			;; last R1

	vaddpd	ymm9, ymm4, ymm10		;; R4 + R3 (new R3)		; 65-67
	vsubpd	ymm4, ymm4, ymm10		;; R4 - R3 (new I4)		; 66-68

	vsubpd	ymm10, ymm6, [srcreg+d1][rbp]	;; R2-MemR2
	ystore	[srcreg+r8+d2], ymm9		;; Temporarily save new R3	; 68
	vsubpd	ymm9, ymm13, [srcreg+d1+32][rbp];; I2-MemI2
	ystore	[srcreg+r8+d2+32], ymm4		;; Temporarily save new I4	; 69
	vmulpd	ymm4, ymm6, ymm10		;; R2 * MemR2			; 66-70
	vmulpd	ymm6, ymm6, ymm9		;; R2 * MemI2			; 67-71
	vmulpd	ymm9, ymm13, ymm9		;; I2 * MemI2			; 68-72
	vmulpd	ymm13, ymm13, ymm10		;; I2 * MemR2			; 69-73

	vsubpd	ymm10, ymm3, [srcreg][rbp]	;; R1-MemR1
	vsubpd	ymm4, ymm4, ymm9		;; R2*MemR2-I2*MemI2 (new R2)	; 73-75
	vmovapd	ymm9, [srcreg+32]		;; Reload I1
	vsubpd	ymm9, ymm9, [srcreg+32][rbp]	;; I1-MemI1
	vaddpd	ymm6, ymm6, ymm13		;; R2*MemI2+I2*MemR2 (new I2)	; 74-76

	vsubpd	ymm13, ymm2, ymm7		;; I3 - I4 (new R4)		; 67-69
	vaddpd	ymm2, ymm2, ymm7		;; I3 + I4 (new I3)		; 68-70

	vsubpd	ymm7, ymm12, ymm1		;; I5 - I6 (new I6)		; 69-71
	vaddpd	ymm12, ymm12, ymm1		;; I5 + I6 (new I5)		; 70-72

	vsubpd	ymm1, ymm5, ymm11		;; R5 - R6 (new R6)		; 71-73
	vaddpd	ymm5, ymm5, ymm11		;; R5 + R6 (new R5)		; 72-74

	vmulpd	ymm11, ymm3, ymm10		;; R1 * MemR1			; 70-74
	vmulpd	ymm3, ymm3, ymm9		;; R1 * MemI1			; 71-75
	vmulpd	ymm9, ymm9, [srcreg+32]		;; (reload I1) * MemI1		; 72-76
	vmulpd	ymm10, ymm10, [srcreg+32]	;; (reload I1) * MemR1		; 73-77

	vsubpd	ymm11, ymm11, ymm9		;; R1*MemR1-I1*MemI1 (new R1)	; 77-79
	vaddpd	ymm3, ymm3, ymm10		;; R1*MemI1+I1*MemR1 (new I1)	; 78-80
	jmp	done

submul132:
	vmovapd	ymm3, [srcreg+d2+d1][rbp]	;; MemR4
	vsubpd	ymm3, ymm3, ymm7		;; MemR4-R4
	vmovapd	ymm7, [srcreg+d2+d1][r9]	;; R4=MemR4
	vmovapd	ymm9, [srcreg+d2+d1+32][rbp]	;; MemI4
	vsubpd	ymm9, ymm9, ymm10		;; MemI4-I4
	vmovapd	ymm10, [srcreg+d2+d1+32][r9]	;; I4=MemI4
	vmulpd	ymm4, ymm7, ymm3		;; R4 * MemR4			; 32-36
	vmulpd	ymm7, ymm7, ymm9		;; R4 * MemI4			; 33-37
	vmulpd	ymm9, ymm10, ymm9		;; I4 * MemI4			; 34-38
	vmulpd	ymm10, ymm10, ymm3		;; I4 * MemR4			; 35-39

	vmovapd	ymm3, [srcreg+d2][rbp]		;; MemR3
	vsubpd	ymm3, ymm3, ymm2		;; MemR3-R3
	vmovapd	ymm2, [srcreg+d2][r9]		;; R3=MemR3
	vsubpd	ymm4, ymm4, ymm9		;; R4*MemR4-I4*MemI4 (new R4)	; 51-53
	vmovapd	ymm9, [srcreg+d2+32][rbp]	;; MemI3
	vsubpd	ymm9, ymm9, ymm11		;; MemI3-I3
	vmovapd	ymm11, [srcreg+d2+32][r9]	;; I3=MemI3
	vaddpd	ymm7, ymm7, ymm10		;; R4*MemI4+I4*MemR4 (new I4)	; 52-54
	vmulpd	ymm10, ymm2, ymm3		;; R3 * MemR3			; 36-40
	vmulpd	ymm2, ymm2, ymm9		;; R3 * MemI3			; 37-41
	vmulpd	ymm9, ymm11, ymm9		;; I3 * MemI3			; 38-42
	vmulpd	ymm11, ymm11, ymm3		;; I3 * MemR3			; 39-43

	vmovapd	ymm3, [srcreg+d4+d1][rbp]	;; MemR6
	vsubpd	ymm3, ymm3, ymm1		;; MemR6-R6
	vmovapd	ymm1, [srcreg+d4+d1][r9]	;; R6=MemR6
	vsubpd	ymm10, ymm10, ymm9		;; R3*MemR3-I3*MemI3 (new R3)	; 53-55
	vmovapd	ymm9, [srcreg+d4+d1+32][rbp]	;; MemI6
	vsubpd	ymm9, ymm9, ymm5		;; MemI6-I6
	vmovapd	ymm5, [srcreg+d4+d1+32][r9]	;; I6=MemI6
	vaddpd	ymm2, ymm2, ymm11		;; R3*MemI3+I3*MemR3 (new I3)	; 54-56
	vmulpd	ymm11, ymm1, ymm3		;; R6 * MemR6			; 42-46
	vmulpd	ymm1, ymm1, ymm9		;; R6 * MemI6			; 43-47
	vmulpd	ymm9, ymm5, ymm9		;; I6 * MemI6			; 46-50
	vmulpd	ymm5, ymm5, ymm3		;; I6 * MemR6			; 47-51

	vmovapd	ymm3, [srcreg+d4][rbp]		;; MemR5
	vsubpd	ymm3, ymm3, ymm12		;; MemR5-R5
	vmovapd	ymm12, [srcreg+d4][r9]		;; R5=MemR5
	vsubpd	ymm11, ymm11, ymm9		;; R6*MemR6-I6*MemI6 (new R6)	; 55-57
	vmovapd	ymm9, [srcreg+d4+32][rbp]	;; MemI5
	vsubpd	ymm9, ymm9, ymm15		;; MemI5-I5
	vmovapd	ymm15, [srcreg+d4+32][r9]	;; I5=MemI5
	vaddpd	ymm1, ymm1, ymm5		;; R6*MemI6+I6*MemR6 (new I6)	; 56-58
	vmulpd	ymm5, ymm12, ymm3		;; R5 * MemR5			; 44-48
	vmulpd	ymm12, ymm12, ymm9		;; R5 * MemI5			; 45-49
	vmulpd	ymm9, ymm15, ymm9		;; I5 * MemI5			; 48-52
	vmulpd	ymm15, ymm15, ymm3		;; I5 * MemR5			; 49-53

	vmovapd	ymm3, [srcreg+d4+d2+d1][rbp]	;; MemR8
	vsubpd	ymm3, ymm3, ymm14		;; MemR8-R8
	vmovapd	ymm14, [srcreg+d4+d2+d1][r9]	;; R8=MemR8
	vsubpd	ymm5, ymm5, ymm9		;; R5*MemR5-I5*MemI5 (new R5)	; 57-59
	vmovapd	ymm9, [srcreg+d4+d2+d1+32][rbp]	;; MemI8
	vsubpd	ymm9, ymm9, ymm8		;; MemI8-I8
	vmovapd	ymm8, [srcreg+d4+d2+d1+32][r9]	;; I8=MemI8
	vaddpd	ymm12, ymm12, ymm15		;; R5*MemI5+I5*MemR5 (new I5)	; 58-60
	vmulpd	ymm15, ymm14, ymm3		;; R8 * MemR8			; 51-55
	vmulpd	ymm14, ymm14, ymm9		;; R8 * MemI8			; 52-56
	vmulpd	ymm9, ymm8, ymm9		;; I8 * MemI8			; 53-57
	vmulpd	ymm8, ymm8, ymm3		;; I8 * MemR8			; 54-58

	vmovapd	ymm3, [srcreg+d4+d2][rbp]	;; MemR7
	vsubpd	ymm3, ymm3, ymm0		;; MemR7-R7
	vmovapd	ymm0, [srcreg+d4+d2][r9]	;; R7=MemR7
	vsubpd	ymm15, ymm15, ymm9		;; R8*MemR8-I8*MemI8 (new R8)	; 59-61
	vmovapd	ymm9, [srcreg+d4+d2+32][rbp]	;; MemI7
	vsubpd	ymm9, ymm9, ymm6		;; MemI7-I7
	vmovapd	ymm6, [srcreg+d4+d2+32][r9]	;; I7=MemI7
	vaddpd	ymm14, ymm14, ymm8		;; R8*MemI8+I8*MemR8 (new I8)	; 60-62
	vmulpd	ymm8, ymm0, ymm3		;; R7 * MemR7			; 54-58
	vmulpd	ymm0, ymm0, ymm9		;; R7 * MemI7			; 55-59
	vmulpd	ymm9, ymm6, ymm9		;; I7 * MemI7			; 56-60
	vmulpd	ymm6, ymm6, ymm3		;; I7 * MemR7			; 57-61

	vsubpd	ymm8, ymm8, ymm9		;; R7*MemR7-I7*MemI7 (new R7)	; 61-63
	vaddpd	ymm0, ymm0, ymm6		;; R7*MemI7+I7*MemR7 (new I7)	; 62-64

	vmovapd	ymm6, [srcreg+d1]		;; last R2
	vmovapd	ymm3, [srcreg]			;; last R1

	vaddpd	ymm9, ymm4, ymm10		;; R4 + R3 (new R3)		; 65-67
	vsubpd	ymm4, ymm4, ymm10		;; R4 - R3 (new I4)		; 66-68

	vmovapd	ymm10, [srcreg+d1][rbp]		;; MemR2
	vsubpd	ymm10, ymm10, ymm6		;; MemR2-R2
	vmovapd	ymm6, [srcreg+d1][r9]		;; R2=MemR2
	ystore	[srcreg+r8+d2], ymm9		;; Temporarily save new R3	; 68
	vmovapd	ymm9, [srcreg+d1+32][rbp]	;; MemI2
	vsubpd	ymm9, ymm9, ymm13		;; MemI2-I2
	vmovapd	ymm13, [srcreg+d1+32][r9]	;; I2=MemI2
	ystore	[srcreg+r8+d2+32], ymm4		;; Temporarily save new I4	; 69
	vmulpd	ymm4, ymm6, ymm10		;; R2 * MemR2			; 66-70
	vmulpd	ymm6, ymm6, ymm9		;; R2 * MemI2			; 67-71
	vmulpd	ymm9, ymm13, ymm9		;; I2 * MemI2			; 68-72
	vmulpd	ymm13, ymm13, ymm10		;; I2 * MemR2			; 69-73

	vmovapd	ymm10, [srcreg][rbp]		;; MemR1
	vsubpd	ymm10, ymm10, ymm3		;; MemR1-R1
	vmovapd	ymm3, [srcreg][r9]		;; R1=MemR1
	vsubpd	ymm4, ymm4, ymm9		;; R2*MemR2-I2*MemI2 (new R2)	; 73-75
	vmovapd	ymm9, [srcreg+32][rbp]		;; MemI1
	vsubpd	ymm9, ymm9, [srcreg+32]		;; MemI1-I1
	vaddpd	ymm6, ymm6, ymm13		;; R2*MemI2+I2*MemR2 (new I2)	; 74-76

	vsubpd	ymm13, ymm2, ymm7		;; I3 - I4 (new R4)		; 67-69
	vaddpd	ymm2, ymm2, ymm7		;; I3 + I4 (new I3)		; 68-70

	vsubpd	ymm7, ymm12, ymm1		;; I5 - I6 (new I6)		; 69-71
	vaddpd	ymm12, ymm12, ymm1		;; I5 + I6 (new I5)		; 70-72

	vsubpd	ymm1, ymm5, ymm11		;; R5 - R6 (new R6)		; 71-73
	vaddpd	ymm5, ymm5, ymm11		;; R5 + R6 (new R5)		; 72-74

	vmulpd	ymm11, ymm3, ymm10		;; R1 * MemR1			; 70-74
	vmulpd	ymm3, ymm3, ymm9		;; R1 * MemI1			; 71-75
	vmulpd	ymm9, ymm9, [srcreg+32][r9]	;; (reload I1) * MemI1		; 72-76
	vmulpd	ymm10, ymm10, [srcreg+32][r9]	;; (reload I1) * MemR1		; 73-77

	vsubpd	ymm11, ymm11, ymm9		;; R1*MemR1-I1*MemI1 (new R1)	; 77-79
	vaddpd	ymm3, ymm3, ymm10		;; R1*MemI1+I1*MemR1 (new I1)	; 78-80
	jmp	done

submul312:
	vsubpd	ymm3, ymm7, [srcreg+d2+d1][rbp]	;; R4-MemR4
	vmovapd	ymm7, [srcreg+d2+d1][r9]	;; R4=MemR4
	vsubpd	ymm9, ymm10, [srcreg+d2+d1+32][rbp];; I4-MemI4
	vmovapd	ymm10, [srcreg+d2+d1+32][r9]	;; I4=MemI4
	vmulpd	ymm4, ymm7, ymm3		;; R4 * MemR4			; 32-36
	vmulpd	ymm7, ymm7, ymm9		;; R4 * MemI4			; 33-37
	vmulpd	ymm9, ymm10, ymm9		;; I4 * MemI4			; 34-38
	vmulpd	ymm10, ymm10, ymm3		;; I4 * MemR4			; 35-39

	vsubpd	ymm3, ymm2, [srcreg+d2][rbp]	;; R3-MemR3
	vmovapd	ymm2, [srcreg+d2][r9]		;; R3=MemR3
	vsubpd	ymm4, ymm4, ymm9		;; R4*MemR4-I4*MemI4 (new R4)	; 51-53
	vsubpd	ymm9, ymm11, [srcreg+d2+32][rbp];; I3-MemI3
	vmovapd	ymm11, [srcreg+d2+32][r9]	;; I3=MemI3
	vaddpd	ymm7, ymm7, ymm10		;; R4*MemI4+I4*MemR4 (new I4)	; 52-54
	vmulpd	ymm10, ymm2, ymm3		;; R3 * MemR3			; 36-40
	vmulpd	ymm2, ymm2, ymm9		;; R3 * MemI3			; 37-41
	vmulpd	ymm9, ymm11, ymm9		;; I3 * MemI3			; 38-42
	vmulpd	ymm11, ymm11, ymm3		;; I3 * MemR3			; 39-43

	vsubpd	ymm3, ymm1, [srcreg+d4+d1][rbp]	;; R6-MemR6
	vmovapd	ymm1, [srcreg+d4+d1][r9]	;; R6=MemR6
	vsubpd	ymm10, ymm10, ymm9		;; R3*MemR3-I3*MemI3 (new R3)	; 53-55
	vsubpd	ymm9, ymm5, [srcreg+d4+d1+32][rbp];; I6-MemI6
	vmovapd	ymm5, [srcreg+d4+d1+32][r9]	;; I6=MemI6
	vaddpd	ymm2, ymm2, ymm11		;; R3*MemI3+I3*MemR3 (new I3)	; 54-56
	vmulpd	ymm11, ymm1, ymm3		;; R6 * MemR6			; 42-46
	vmulpd	ymm1, ymm1, ymm9		;; R6 * MemI6			; 43-47
	vmulpd	ymm9, ymm5, ymm9		;; I6 * MemI6			; 46-50
	vmulpd	ymm5, ymm5, ymm3		;; I6 * MemR6			; 47-51

	vsubpd	ymm3, ymm12, [srcreg+d4][rbp]	;; R5-MemR5
	vmovapd	ymm12, [srcreg+d4][r9]		;; R5=MemR5
	vsubpd	ymm11, ymm11, ymm9		;; R6*MemR6-I6*MemI6 (new R6)	; 55-57
	vsubpd	ymm9, ymm15, [srcreg+d4+32][rbp];; I5-MemI5
	vmovapd	ymm15, [srcreg+d4+32][r9]	;; I5=MemI5
	vaddpd	ymm1, ymm1, ymm5		;; R6*MemI6+I6*MemR6 (new I6)	; 56-58
	vmulpd	ymm5, ymm12, ymm3		;; R5 * MemR5			; 44-48
	vmulpd	ymm12, ymm12, ymm9		;; R5 * MemI5			; 45-49
	vmulpd	ymm9, ymm15, ymm9		;; I5 * MemI5			; 48-52
	vmulpd	ymm15, ymm15, ymm3		;; I5 * MemR5			; 49-53

	vsubpd	ymm3, ymm14, [srcreg+d4+d2+d1][rbp];; R8-MemR8
	vmovapd	ymm14, [srcreg+d4+d2+d1][r9]	;; R8=MemR8
	vsubpd	ymm5, ymm5, ymm9		;; R5*MemR5-I5*MemI5 (new R5)	; 57-59
	vsubpd	ymm9, ymm8, [srcreg+d4+d2+d1+32][rbp];; I8-MemI8
	vmovapd	ymm8, [srcreg+d4+d2+d1+32][r9]	;; I8=MemI8
	vaddpd	ymm12, ymm12, ymm15		;; R5*MemI5+I5*MemR5 (new I5)	; 58-60
	vmulpd	ymm15, ymm14, ymm3		;; R8 * MemR8			; 51-55
	vmulpd	ymm14, ymm14, ymm9		;; R8 * MemI8			; 52-56
	vmulpd	ymm9, ymm8, ymm9		;; I8 * MemI8			; 53-57
	vmulpd	ymm8, ymm8, ymm3		;; I8 * MemR8			; 54-58

	vsubpd	ymm3, ymm0, [srcreg+d4+d2][rbp]	;; R7-MemR7
	vmovapd	ymm0, [srcreg+d4+d2][r9]	;; R7=MemR7
	vsubpd	ymm15, ymm15, ymm9		;; R8*MemR8-I8*MemI8 (new R8)	; 59-61
	vsubpd	ymm9, ymm6, [srcreg+d4+d2+32][rbp];; I7-MemI7
	vmovapd	ymm6, [srcreg+d4+d2+32][r9]	;; I7=MemI7
	vaddpd	ymm14, ymm14, ymm8		;; R8*MemI8+I8*MemR8 (new I8)	; 60-62
	vmulpd	ymm8, ymm0, ymm3		;; R7 * MemR7			; 54-58
	vmulpd	ymm0, ymm0, ymm9		;; R7 * MemI7			; 55-59
	vmulpd	ymm9, ymm6, ymm9		;; I7 * MemI7			; 56-60
	vmulpd	ymm6, ymm6, ymm3		;; I7 * MemR7			; 57-61

	vsubpd	ymm8, ymm8, ymm9		;; R7*MemR7-I7*MemI7 (new R7)	; 61-63
	vaddpd	ymm0, ymm0, ymm6		;; R7*MemI7+I7*MemR7 (new I7)	; 62-64

	vmovapd	ymm6, [srcreg+d1]		;; last R2
	vmovapd	ymm3, [srcreg]			;; last R1

	vaddpd	ymm9, ymm4, ymm10		;; R4 + R3 (new R3)		; 65-67
	vsubpd	ymm4, ymm4, ymm10		;; R4 - R3 (new I4)		; 66-68

	vsubpd	ymm10, ymm6, [srcreg+d1][rbp]	;; R2-MemR2
	vmovapd	ymm6, [srcreg+d1][r9]		;; R2=MemR2
	ystore	[srcreg+r8+d2], ymm9		;; Temporarily save new R3	; 68
	vsubpd	ymm9, ymm13, [srcreg+d1+32][rbp];; I2-MemI2
	vmovapd	ymm13, [srcreg+d1+32][r9]	;; I2=MemI2
	ystore	[srcreg+r8+d2+32], ymm4		;; Temporarily save new I4	; 69
	vmulpd	ymm4, ymm6, ymm10		;; R2 * MemR2			; 66-70
	vmulpd	ymm6, ymm6, ymm9		;; R2 * MemI2			; 67-71
	vmulpd	ymm9, ymm13, ymm9		;; I2 * MemI2			; 68-72
	vmulpd	ymm13, ymm13, ymm10		;; I2 * MemR2			; 69-73

	vsubpd	ymm10, ymm3, [srcreg][rbp]	;; R1-MemR1
	vmovapd	ymm3, [srcreg][r9]		;; R1=MemR1
	vsubpd	ymm4, ymm4, ymm9		;; R2*MemR2-I2*MemI2 (new R2)	; 73-75
	vmovapd	ymm9, [srcreg+32]		;; Reload I1
	vsubpd	ymm9, ymm9, [srcreg+32][rbp]	;; I1-MemI1
	vaddpd	ymm6, ymm6, ymm13		;; R2*MemI2+I2*MemR2 (new I2)	; 74-76

	vsubpd	ymm13, ymm2, ymm7		;; I3 - I4 (new R4)		; 67-69
	vaddpd	ymm2, ymm2, ymm7		;; I3 + I4 (new I3)		; 68-70

	vsubpd	ymm7, ymm12, ymm1		;; I5 - I6 (new I6)		; 69-71
	vaddpd	ymm12, ymm12, ymm1		;; I5 + I6 (new I5)		; 70-72

	vsubpd	ymm1, ymm5, ymm11		;; R5 - R6 (new R6)		; 71-73
	vaddpd	ymm5, ymm5, ymm11		;; R5 + R6 (new R5)		; 72-74

	vmulpd	ymm11, ymm3, ymm10		;; R1 * MemR1			; 70-74
	vmulpd	ymm3, ymm3, ymm9		;; R1 * MemI1			; 71-75
	vmulpd	ymm9, ymm9, [srcreg+32][r9]	;; (reload I1) * MemI1		; 72-76
	vmulpd	ymm10, ymm10, [srcreg+32][r9]	;; (reload I1) * MemR1		; 73-77

	vsubpd	ymm11, ymm11, ymm9		;; R1*MemR1-I1*MemI1 (new R1)	; 77-79
	vaddpd	ymm3, ymm3, ymm10		;; R1*MemI1+I1*MemR1 (new I1)	; 78-80
	jmp	done

unfft:	vaddpd	ymm3, ymm7, ymm2		;; R4 + R3 (new R3)		; 65-67
	vsubpd	ymm4, ymm7, ymm2		;; R4 - R3 (new I4)		; 66-68
	ystore	[srcreg+r8+d2], ymm3		;; Temporarily save new R3	; 68
	ystore	[srcreg+r8+d2+32], ymm4		;; Temporarily save new I4	; 69

	vsubpd	ymm7, ymm15, ymm5		;; I5 - I6 (new I6)		; 69-71
	vaddpd	ymm3, ymm15, ymm5		;; I5 + I6 (new I5)		; 70-72

	vmovapd	ymm15, ymm14			;; R8
	vmovapd	ymm14, ymm8			;; I8
	vmovapd	ymm8, ymm0			;; R7
	vmovapd	ymm0, ymm6			;; I7
	vmovapd	ymm6, ymm13			;; I2

	vaddpd	ymm2, ymm11, ymm10		;; I3 + I4 (new I3)		; 68-70
	vsubpd	ymm13, ymm11, ymm10		;; I3 - I4 (new R4)		; 67-69

	vmovapd	ymm4, [srcreg+d1]		;; last R2
	vmovapd	ymm11, [srcreg]			;; last R1

	vaddpd	ymm5, ymm12, ymm1		;; R5 + R6 (new R5)		; 72-74
	vsubpd	ymm1, ymm12, ymm1		;; R5 - R6 (new R6)		; 71-73

	vmovapd	ymm12, ymm3			;; I5
	vmovapd	ymm3, [srcreg+32]		;; reload I1
	jmp	done

muladd:	vmovapd	ymm3, [srcreg+d2+d1][rbp]	;; MemR4
	vmovapd	ymm9, [srcreg+d2+d1+32][rbp]	;; MemI4
	vmulpd	ymm4, ymm7, ymm3		;; R4 * MemR4			; 32-36
	vmulpd	ymm7, ymm7, ymm9		;; R4 * MemI4			; 33-37
	vmulpd	ymm9, ymm10, ymm9		;; I4 * MemI4			; 34-38
	vmulpd	ymm10, ymm10, ymm3		;; I4 * MemR4			; 35-39

	vmovapd	ymm3, [srcreg+d2][rbp]		;; MemR3
	vsubpd	ymm4, ymm4, ymm9		;; R4*MemR4-I4*MemI4 (new R4)	; 51-53
	vmovapd	ymm9, [srcreg+d2+32][rbp]	;; MemI3
	vaddpd	ymm7, ymm7, ymm10		;; R4*MemI4+I4*MemR4 (new I4)	; 52-54

	vmulpd	ymm10, ymm2, ymm3		;; R3 * MemR3			; 36-40
	vmulpd	ymm2, ymm2, ymm9		;; R3 * MemI3			; 37-41
	vmulpd	ymm9, ymm11, ymm9		;; I3 * MemI3			; 38-42
	vmulpd	ymm11, ymm11, ymm3		;; I3 * MemR3			; 39-43

	vmovapd	ymm3, [srcreg+d4+d1][rbp]	;; MemR6
	vsubpd	ymm10, ymm10, ymm9		;; R3*MemR3-I3*MemI3 (new R3)	; 53-55
	vmovapd	ymm9, [srcreg+d4+d1+32][rbp]	;; MemI6
	vaddpd	ymm2, ymm2, ymm11		;; R3*MemI3+I3*MemR3 (new I3)	; 54-56

	vmulpd	ymm11, ymm1, ymm3		;; R6 * MemR6			; 42-46
	vmulpd	ymm1, ymm1, ymm9		;; R6 * MemI6			; 43-47
	vmulpd	ymm9, ymm5, ymm9		;; I6 * MemI6			; 46-50
	vmulpd	ymm5, ymm5, ymm3		;; I6 * MemR6			; 47-51

	vmovapd	ymm3, [srcreg+d4][rbp]		;; MemR5
	vsubpd	ymm11, ymm11, ymm9		;; R6*MemR6-I6*MemI6 (new R6)	; 55-57
	vmovapd	ymm9, [srcreg+d4+32][rbp]	;; MemI5
	vaddpd	ymm1, ymm1, ymm5		;; R6*MemI6+I6*MemR6 (new I6)	; 56-58

	vmulpd	ymm5, ymm12, ymm3		;; R5 * MemR5			; 44-48
	vmulpd	ymm12, ymm12, ymm9		;; R5 * MemI5			; 45-49
	vmulpd	ymm9, ymm15, ymm9		;; I5 * MemI5			; 48-52
	vmulpd	ymm15, ymm15, ymm3		;; I5 * MemR5			; 49-53

	vmovapd	ymm3, [srcreg+d4+d2+d1][rbp]	;; MemR8
	vsubpd	ymm5, ymm5, ymm9		;; R5*MemR5-I5*MemI5 (new R5)	; 57-59
	vmovapd	ymm9, [srcreg+d4+d2+d1+32][rbp]	;; MemI8
	vaddpd	ymm12, ymm12, ymm15		;; R5*MemI5+I5*MemR5 (new I5)	; 58-60

	vmulpd	ymm15, ymm14, ymm3		;; R8 * MemR8			; 51-55
	vmulpd	ymm14, ymm14, ymm9		;; R8 * MemI8			; 52-56
	vmulpd	ymm9, ymm8, ymm9		;; I8 * MemI8			; 53-57
	vmulpd	ymm8, ymm8, ymm3		;; I8 * MemR8			; 54-58

	vmovapd	ymm3, [srcreg+d4+d2][rbp]	;; MemR7
	vsubpd	ymm15, ymm15, ymm9		;; R8*MemR8-I8*MemI8 (new R8)	; 59-61
	vmovapd	ymm9, [srcreg+d4+d2+32][rbp]	;; MemI7
	vaddpd	ymm14, ymm14, ymm8		;; R8*MemI8+I8*MemR8 (new I8)	; 60-62

	vmulpd	ymm8, ymm0, ymm3		;; R7 * MemR7			; 54-58
	vmulpd	ymm0, ymm0, ymm9		;; R7 * MemI7			; 55-59
	vmulpd	ymm9, ymm6, ymm9		;; I7 * MemI7			; 56-60
	vmulpd	ymm6, ymm6, ymm3		;; I7 * MemR7			; 57-61

	vsubpd	ymm8, ymm8, ymm9		;; R7*MemR7-I7*MemI7 (new R7)	; 61-63
	vaddpd	ymm0, ymm0, ymm6		;; R7*MemI7+I7*MemR7 (new I7)	; 62-64

	vmovapd	ymm6, [srcreg+d1]		;; last R2
	vmovapd	ymm3, [srcreg]			;; last R1

	mov	r10, SRC3ARG			;; Load distance to FFT(1)
	test	r10, 3				;; Check if we need to multiply add-in by FFT(1)
	jz	muladdhard			;; Yes, do it the hard way

	vaddpd	ymm4, ymm4, [srcreg+r9+d2+d1]		;; R4 = R4 + MemR4
	vaddpd	ymm7, ymm7, [srcreg+r9+d2+d1+32]	;; I4 = I4 + MemI4
	vaddpd	ymm10, ymm10, [srcreg+r9+d2]		;; R3 = R3 + MemR3
	vaddpd	ymm2, ymm2, [srcreg+r9+d2+32]		;; I3 = I3 + MemI3
	vaddpd	ymm11, ymm11, [srcreg+r9+d4+d1]		;; R6 = R6 + MemR6
	vaddpd	ymm1, ymm1, [srcreg+r9+d4+d1+32]	;; I6 = I6 + MemI6
	vaddpd	ymm5, ymm5, [srcreg+r9+d4]		;; R5 = R5 + MemR5
	vaddpd	ymm12, ymm12, [srcreg+r9+d4+32]		;; I5 = I5 + MemI5
	vaddpd	ymm15, ymm15, [srcreg+r9+d4+d2+d1]	;; R8 = R8 + MemR8
	vaddpd	ymm14, ymm14, [srcreg+r9+d4+d2+d1+32]	;; I8 = I8 + MemI8
	vaddpd	ymm8, ymm8, [srcreg+r9+d4+d2]		;; R7 = R7 + MemR7
	vaddpd	ymm0, ymm0, [srcreg+r9+d4+d2+32]	;; I7 = I7 + MemI7

	vaddpd	ymm9, ymm4, ymm10		;; R4 + R3 (new R3)		; 65-67
	vsubpd	ymm4, ymm4, ymm10		;; R4 - R3 (new I4)		; 66-68

	vmovapd	ymm10, [srcreg+d1][rbp]		;; MemR2
	ystore	[srcreg+r8+d2], ymm9		;; Temporarily save new R3	; 68
	vmovapd	ymm9, [srcreg+d1+32][rbp]	;; MemI2
	ystore	[srcreg+r8+d2+32], ymm4		;; Temporarily save new I4	; 69
	vmulpd	ymm4, ymm6, ymm10		;; R2 * MemR2			; 66-70
	vmulpd	ymm6, ymm6, ymm9		;; R2 * MemI2			; 67-71
	vmulpd	ymm9, ymm13, ymm9		;; I2 * MemI2			; 68-72
	vmulpd	ymm13, ymm13, ymm10		;; I2 * MemR2			; 69-73

	vmovapd	ymm10, [srcreg][rbp]		;; MemR1
	vsubpd	ymm4, ymm4, ymm9		;; R2*MemR2-I2*MemI2 (new R2)	; 73-75
	vmovapd	ymm9, [srcreg+32][rbp]		;; MemI1
	vaddpd	ymm6, ymm6, ymm13		;; R2*MemI2+I2*MemR2 (new I2)	; 74-76

	vsubpd	ymm13, ymm2, ymm7		;; I3 - I4 (new R4)		; 67-69
	vaddpd	ymm2, ymm2, ymm7		;; I3 + I4 (new I3)		; 68-70

	vsubpd	ymm7, ymm12, ymm1		;; I5 - I6 (new I6)		; 69-71
	vaddpd	ymm12, ymm12, ymm1		;; I5 + I6 (new I5)		; 70-72

	vsubpd	ymm1, ymm5, ymm11		;; R5 - R6 (new R6)		; 71-73
	vaddpd	ymm5, ymm5, ymm11		;; R5 + R6 (new R5)		; 72-74

	vmulpd	ymm11, ymm3, ymm10		;; R1 * MemR1			; 70-74
	vmulpd	ymm3, ymm3, ymm9		;; R1 * MemI1			; 71-75
	vmulpd	ymm9, ymm9, [srcreg+32]		;; (reload I1) * MemI1		; 72-76
	vmulpd	ymm10, ymm10, [srcreg+32]	;; (reload I1) * MemR1		; 73-77

	vsubpd	ymm11, ymm11, ymm9		;; R1*MemR1-I1*MemI1 (new R1)	; 77-79
	vaddpd	ymm3, ymm3, ymm10		;; R1*MemI1+I1*MemR1 (new I1)	; 78-80

	vaddpd	ymm11, ymm11, [srcreg+r9]		;; R1 = R1 + MemR1
	vaddpd	ymm3, ymm3, [srcreg+r9+32]		;; I1 = I1 + MemI1
	vaddpd	ymm4, ymm4, [srcreg+r9+d1]		;; R2 = R2 + MemR2
	vaddpd	ymm6, ymm6, [srcreg+r9+d1+32]		;; I2 = I2 + MemI2
	jmp	done

muladdhard:
	vmovapd	YMM_TMP1, ymm3				;; Save 3 registers
	vmovapd	YMM_TMP2, ymm6
	vmovapd	YMM_TMP3, ymm13

	vmovapd	ymm3, [srcreg+r9+d2+d1]			;; MemR4
	vmovapd	ymm9, [srcreg+r10+d2+d1]		;; MemR4#2
	vmulpd	ymm13, ymm3, ymm9			;; MemR4*MemR4#2
	vaddpd	ymm4, ymm4, ymm13			;; R4 = R4 + MemR4*MemR4#2
	vmovapd	ymm6, [srcreg+r10+d2+d1+32]		;; MemI4#2
	vmulpd	ymm13, ymm3, ymm6			;; MemR4*MemI4#2
	vaddpd	ymm7, ymm7, ymm13			;; I4 = I4 + MemR4*MemI4#2
	vmovapd	ymm3, [srcreg+r9+d2+d1+32]		;; MemI4
	vmulpd	ymm13, ymm3, ymm6			;; MemI4*MemI4#2
	vsubpd	ymm4, ymm4, ymm13			;; R4 = R4 - MemI4*MemI4#2
	vmulpd	ymm13, ymm3, ymm9			;; MemI4*MemR4#2
	vaddpd	ymm7, ymm7, ymm13			;; I4 = I4 + MemI4*MemR4#2

	vmovapd	ymm3, [srcreg+r9+d2]			;; MemR3
	vmovapd	ymm9, [srcreg+r10+d2]			;; MemR3#2
	vmulpd	ymm13, ymm3, ymm9			;; MemR3*MemR3#2
	vaddpd	ymm10, ymm10, ymm13			;; R3 = R3 + MemR3*MemR3#2
	vmovapd	ymm6, [srcreg+r10+d2+32]		;; MemI3#2
	vmulpd	ymm13, ymm3, ymm6			;; MemR3*MemI3#2
	vaddpd	ymm2, ymm2, ymm13			;; I3 = I3 + MemR3*MemI3#2
	vmovapd	ymm3, [srcreg+r9+d2+32]			;; MemI3
	vmulpd	ymm13, ymm3, ymm6			;; MemI3*MemI3#2
	vsubpd	ymm10, ymm10, ymm13			;; R3 = R3 - MemI3*MemI3#2
	vmulpd	ymm13, ymm3, ymm9			;; MemI3*MemR3#2
	vaddpd	ymm2, ymm2, ymm13			;; I3 = I3 + MemI3*MemR3#2

	vmovapd	ymm3, [srcreg+r9+d4+d1]			;; MemR6
	vmovapd	ymm9, [srcreg+r10+d4+d1]		;; MemR6#2
	vmulpd	ymm13, ymm3, ymm9			;; MemR6*MemR6#2
	vaddpd	ymm11, ymm11, ymm13			;; R6 = R6 + MemR6*MemR6#2
	vmovapd	ymm6, [srcreg+r10+d4+d1+32]		;; MemI6#2
	vmulpd	ymm13, ymm3, ymm6			;; MemR6*MemI6#2
	vaddpd	ymm1, ymm1, ymm13			;; I6 = I6 + MemR6*MemI6#2
	vmovapd	ymm3, [srcreg+r9+d4+d1+32]		;; MemI6
	vmulpd	ymm13, ymm3, ymm6			;; MemI6*MemI6#2
	vsubpd	ymm11, ymm11, ymm13			;; R6 = R6 - MemI6*MemI6#2
	vmulpd	ymm13, ymm3, ymm9			;; MemI6*MemR6#2
	vaddpd	ymm1, ymm1, ymm13			;; I6 = I6 + MemI6*MemR6#2

	vmovapd	ymm3, [srcreg+r9+d4]			;; MemR5
	vmovapd	ymm9, [srcreg+r10+d4]			;; MemR5#2
	vmulpd	ymm13, ymm3, ymm9			;; MemR5*MemR5#2
	vaddpd	ymm5, ymm5, ymm13			;; R5 = R5 + MemR5*MemR5#2
	vmovapd	ymm6, [srcreg+r10+d4+32]		;; MemI5#2
	vmulpd	ymm13, ymm3, ymm6			;; MemR5*MemI5#2
	vaddpd	ymm12, ymm12, ymm13			;; I5 = I5 + MemR5*MemI5#2
	vmovapd	ymm3, [srcreg+r9+d4+32]			;; MemI5
	vmulpd	ymm13, ymm3, ymm6			;; MemI5*MemI5#2
	vsubpd	ymm5, ymm5, ymm13			;; R5 = R5 - MemI5*MemI5#2
	vmulpd	ymm13, ymm3, ymm9			;; MemI5*MemR5#2
	vaddpd	ymm12, ymm12, ymm13			;; I5 = I5 + MemI5*MemR5#2

	vmovapd	ymm3, [srcreg+r9+d4+d2+d1]		;; MemR8
	vmovapd	ymm9, [srcreg+r10+d4+d2+d1]		;; MemR8#2
	vmulpd	ymm13, ymm3, ymm9			;; MemR8*MemR8#2
	vaddpd	ymm15, ymm15, ymm13			;; R8 = R8 + MemR8*MemR8#2
	vmovapd	ymm6, [srcreg+r10+d4+d2+d1+32]		;; MemI8#2
	vmulpd	ymm13, ymm3, ymm6			;; MemR8*MemI8#2
	vaddpd	ymm14, ymm14, ymm13			;; I8 = I8 + MemR8*MemI8#2
	vmovapd	ymm3, [srcreg+r9+d4+d2+d1+32]		;; MemI8
	vmulpd	ymm13, ymm3, ymm6			;; MemI8*MemI8#2
	vsubpd	ymm15, ymm15, ymm13			;; R8 = R8 - MemI8*MemI8#2
	vmulpd	ymm13, ymm3, ymm9			;; MemI8*MemR8#2
	vaddpd	ymm14, ymm14, ymm13			;; I8 = I8 + MemI8*MemR8#2

	vmovapd	ymm3, [srcreg+r9+d4+d2]			;; MemR7
	vmovapd	ymm9, [srcreg+r10+d4+d2]		;; MemR7#2
	vmulpd	ymm13, ymm3, ymm9			;; MemR7*MemR7#2
	vaddpd	ymm8, ymm8, ymm13			;; R7 = R7 + MemR7*MemR7#2
	vmovapd	ymm6, [srcreg+r10+d4+d2+32]		;; MemI7#2
	vmulpd	ymm13, ymm3, ymm6			;; MemR7*MemI7#2
	vaddpd	ymm0, ymm0, ymm13			;; I7 = I7 + MemR7*MemI7#2
	vmovapd	ymm3, [srcreg+r9+d4+d2+32]		;; MemI7
	vmulpd	ymm13, ymm3, ymm6			;; MemI7*MemI7#2
	vsubpd	ymm8, ymm8, ymm13			;; R7 = R7 - MemI7*MemI7#2
	vmulpd	ymm13, ymm3, ymm9			;; MemI7*MemR7#2
	vaddpd	ymm0, ymm0, ymm13			;; I7 = I7 + MemI7*MemR7#2

	vaddpd	ymm9, ymm4, ymm10		;; R4 + R3 (new R3)		; 65-67
	vsubpd	ymm4, ymm4, ymm10		;; R4 - R3 (new I4)		; 66-68

	vmovapd	ymm10, [srcreg+d1][rbp]		;; MemR2
	ystore	[srcreg+r8+d2], ymm9		;; Temporarily save new R3	; 68
	vmovapd	ymm9, [srcreg+d1+32][rbp]	;; MemI2
	ystore	[srcreg+r8+d2+32], ymm4		;; Temporarily save new I4	; 69

	vmovapd	ymm6, YMM_TMP2
	vmulpd	ymm4, ymm6, ymm10		;; R2 * MemR2			; 66-70
	vmulpd	ymm6, ymm6, ymm9		;; R2 * MemI2			; 67-71
	vmovapd	ymm13, YMM_TMP3
	vmulpd	ymm9, ymm13, ymm9		;; I2 * MemI2			; 68-72
	vmulpd	ymm13, ymm13, ymm10		;; I2 * MemR2			; 69-73

	vmovapd	ymm10, [srcreg][rbp]		;; MemR1
	vsubpd	ymm4, ymm4, ymm9		;; R2*MemR2-I2*MemI2 (new R2)	; 73-75
	vmovapd	ymm9, [srcreg+32][rbp]		;; MemI1
	vaddpd	ymm6, ymm6, ymm13		;; R2*MemI2+I2*MemR2 (new I2)	; 74-76

	vsubpd	ymm13, ymm2, ymm7		;; I3 - I4 (new R4)		; 67-69
	vaddpd	ymm2, ymm2, ymm7		;; I3 + I4 (new I3)		; 68-70

	vsubpd	ymm7, ymm12, ymm1		;; I5 - I6 (new I6)		; 69-71
	vaddpd	ymm12, ymm12, ymm1		;; I5 + I6 (new I5)		; 70-72

	vsubpd	ymm1, ymm5, ymm11		;; R5 - R6 (new R6)		; 71-73
	vaddpd	ymm5, ymm5, ymm11		;; R5 + R6 (new R5)		; 72-74

	vmovapd	ymm3, YMM_TMP1
	vmulpd	ymm11, ymm3, ymm10		;; R1 * MemR1			; 70-74
	vmulpd	ymm3, ymm3, ymm9		;; R1 * MemI1			; 71-75
	vmulpd	ymm9, ymm9, [srcreg+32]		;; (reload I1) * MemI1		; 72-76
	vmulpd	ymm10, ymm10, [srcreg+32]	;; (reload I1) * MemR1		; 73-77

	vsubpd	ymm11, ymm11, ymm9		;; R1*MemR1-I1*MemI1 (new R1)	; 77-79
	vaddpd	ymm3, ymm3, ymm10		;; R1*MemI1+I1*MemR1 (new I1)	; 78-80

	vmovapd	YMM_TMP1, ymm0				;; Save 2 registers
	vmovapd	YMM_TMP2, ymm12

	vmovapd	ymm0, [srcreg+r9+d1]			;; MemR2
	vmovapd	ymm9, [srcreg+r10+d1]			;; MemR2#2
	vmulpd	ymm12, ymm0, ymm9			;; MemR2*MemR2#2
	vaddpd	ymm4, ymm4, ymm12			;; R2 = R2 + MemR2*MemR2#2
	vmovapd	ymm10, [srcreg+r10+d1+32]		;; MemI2#2
	vmulpd	ymm12, ymm0, ymm10			;; MemR2*MemI2#2
	vaddpd	ymm6, ymm6, ymm12			;; I2 = I2 + MemR2*MemI2#2
	vmovapd	ymm0, [srcreg+r9+d1+32]			;; MemI2
	vmulpd	ymm12, ymm0, ymm10			;; MemI2*MemI2#2
	vsubpd	ymm4, ymm4, ymm12			;; R2 = R2 - MemI2*MemI2#2
	vmulpd	ymm12, ymm0, ymm9			;; MemI2*MemR2#2
	vaddpd	ymm6, ymm6, ymm12			;; I2 = I2 + MemI2*MemR2#2

	vmovapd	ymm0, [srcreg+r9]			;; MemR1
	vmovapd	ymm9, [srcreg+r10]			;; MemR1#2
	vmulpd	ymm12, ymm0, ymm9			;; MemR1*MemR1#2
	vaddpd	ymm11, ymm11, ymm12			;; R1 = R1 + MemR1*MemR1#2
	vmovapd	ymm10, [srcreg+r10+32]			;; MemI1#2
	vmulpd	ymm12, ymm0, ymm10			;; MemR1*MemI1#2
	vaddpd	ymm3, ymm3, ymm12			;; I1 = I1 + MemR1*MemI1#2
	vmovapd	ymm0, [srcreg+r9+32]			;; MemI1
	vmulpd	ymm12, ymm0, ymm10			;; MemI1*MemI1#2
	vsubpd	ymm11, ymm11, ymm12			;; R1 = R1 - MemI1*MemI1#2
	vmulpd	ymm12, ymm0, ymm9			;; MemI1*MemR1#2
	vaddpd	ymm3, ymm3, ymm12			;; I1 = I1 + MemI1*MemR1#2

	vmovapd	ymm0, YMM_TMP1				;; Restore 2 registers
	vmovapd	ymm12, YMM_TMP2
	jmp	done

mulsub:	vmovapd	ymm3, [srcreg+d2+d1][rbp]	;; MemR4
	vmovapd	ymm9, [srcreg+d2+d1+32][rbp]	;; MemI4
	vmulpd	ymm4, ymm7, ymm3		;; R4 * MemR4			; 32-36
	vmulpd	ymm7, ymm7, ymm9		;; R4 * MemI4			; 33-37
	vmulpd	ymm9, ymm10, ymm9		;; I4 * MemI4			; 34-38
	vmulpd	ymm10, ymm10, ymm3		;; I4 * MemR4			; 35-39

	vmovapd	ymm3, [srcreg+d2][rbp]		;; MemR3
	vsubpd	ymm4, ymm4, ymm9		;; R4*MemR4-I4*MemI4 (new R4)	; 51-53
	vmovapd	ymm9, [srcreg+d2+32][rbp]	;; MemI3
	vaddpd	ymm7, ymm7, ymm10		;; R4*MemI4+I4*MemR4 (new I4)	; 52-54

	vmulpd	ymm10, ymm2, ymm3		;; R3 * MemR3			; 36-40
	vmulpd	ymm2, ymm2, ymm9		;; R3 * MemI3			; 37-41
	vmulpd	ymm9, ymm11, ymm9		;; I3 * MemI3			; 38-42
	vmulpd	ymm11, ymm11, ymm3		;; I3 * MemR3			; 39-43

	vmovapd	ymm3, [srcreg+d4+d1][rbp]	;; MemR6
	vsubpd	ymm10, ymm10, ymm9		;; R3*MemR3-I3*MemI3 (new R3)	; 53-55
	vmovapd	ymm9, [srcreg+d4+d1+32][rbp]	;; MemI6
	vaddpd	ymm2, ymm2, ymm11		;; R3*MemI3+I3*MemR3 (new I3)	; 54-56

	vmulpd	ymm11, ymm1, ymm3		;; R6 * MemR6			; 42-46
	vmulpd	ymm1, ymm1, ymm9		;; R6 * MemI6			; 43-47
	vmulpd	ymm9, ymm5, ymm9		;; I6 * MemI6			; 46-50
	vmulpd	ymm5, ymm5, ymm3		;; I6 * MemR6			; 47-51

	vmovapd	ymm3, [srcreg+d4][rbp]		;; MemR5
	vsubpd	ymm11, ymm11, ymm9		;; R6*MemR6-I6*MemI6 (new R6)	; 55-57
	vmovapd	ymm9, [srcreg+d4+32][rbp]	;; MemI5
	vaddpd	ymm1, ymm1, ymm5		;; R6*MemI6+I6*MemR6 (new I6)	; 56-58

	vmulpd	ymm5, ymm12, ymm3		;; R5 * MemR5			; 44-48
	vmulpd	ymm12, ymm12, ymm9		;; R5 * MemI5			; 45-49
	vmulpd	ymm9, ymm15, ymm9		;; I5 * MemI5			; 48-52
	vmulpd	ymm15, ymm15, ymm3		;; I5 * MemR5			; 49-53

	vmovapd	ymm3, [srcreg+d4+d2+d1][rbp]	;; MemR8
	vsubpd	ymm5, ymm5, ymm9		;; R5*MemR5-I5*MemI5 (new R5)	; 57-59
	vmovapd	ymm9, [srcreg+d4+d2+d1+32][rbp]	;; MemI8
	vaddpd	ymm12, ymm12, ymm15		;; R5*MemI5+I5*MemR5 (new I5)	; 58-60

	vmulpd	ymm15, ymm14, ymm3		;; R8 * MemR8			; 51-55
	vmulpd	ymm14, ymm14, ymm9		;; R8 * MemI8			; 52-56
	vmulpd	ymm9, ymm8, ymm9		;; I8 * MemI8			; 53-57
	vmulpd	ymm8, ymm8, ymm3		;; I8 * MemR8			; 54-58

	vmovapd	ymm3, [srcreg+d4+d2][rbp]	;; MemR7
	vsubpd	ymm15, ymm15, ymm9		;; R8*MemR8-I8*MemI8 (new R8)	; 59-61
	vmovapd	ymm9, [srcreg+d4+d2+32][rbp]	;; MemI7
	vaddpd	ymm14, ymm14, ymm8		;; R8*MemI8+I8*MemR8 (new I8)	; 60-62

	vmulpd	ymm8, ymm0, ymm3		;; R7 * MemR7			; 54-58
	vmulpd	ymm0, ymm0, ymm9		;; R7 * MemI7			; 55-59
	vmulpd	ymm9, ymm6, ymm9		;; I7 * MemI7			; 56-60
	vmulpd	ymm6, ymm6, ymm3		;; I7 * MemR7			; 57-61

	vsubpd	ymm8, ymm8, ymm9		;; R7*MemR7-I7*MemI7 (new R7)	; 61-63
	vaddpd	ymm0, ymm0, ymm6		;; R7*MemI7+I7*MemR7 (new I7)	; 62-64

	vmovapd	ymm6, [srcreg+d1]		;; last R2
	vmovapd	ymm3, [srcreg]			;; last R1

	mov	r10, SRC3ARG			;; Load distance to FFT(1)
	test	r10, 3				;; Check if we need to multiply add-in by FFT(1)
	jz	mulsubhard			;; Yes, do it the hard way

	vsubpd	ymm4, ymm4, [srcreg+r9+d2+d1]		;; R4 = R4 - MemR4
	vsubpd	ymm7, ymm7, [srcreg+r9+d2+d1+32]	;; I4 = I4 - MemI4
	vsubpd	ymm10, ymm10, [srcreg+r9+d2]		;; R3 = R3 - MemR3
	vsubpd	ymm2, ymm2, [srcreg+r9+d2+32]		;; I3 = I3 - MemI3
	vsubpd	ymm11, ymm11, [srcreg+r9+d4+d1]		;; R6 = R6 - MemR6
	vsubpd	ymm1, ymm1, [srcreg+r9+d4+d1+32]	;; I6 = I6 - MemI6
	vsubpd	ymm5, ymm5, [srcreg+r9+d4]		;; R5 = R5 - MemR5
	vsubpd	ymm12, ymm12, [srcreg+r9+d4+32]		;; I5 = I5 - MemI5
	vsubpd	ymm15, ymm15, [srcreg+r9+d4+d2+d1]	;; R8 = R8 - MemR8
	vsubpd	ymm14, ymm14, [srcreg+r9+d4+d2+d1+32]	;; I8 = I8 - MemI8
	vsubpd	ymm8, ymm8, [srcreg+r9+d4+d2]		;; R7 = R7 - MemR7
	vsubpd	ymm0, ymm0, [srcreg+r9+d4+d2+32]	;; I7 = I7 - MemI7

	vaddpd	ymm9, ymm4, ymm10		;; R4 + R3 (new R3)		; 65-67
	vsubpd	ymm4, ymm4, ymm10		;; R4 - R3 (new I4)		; 66-68

	vmovapd	ymm10, [srcreg+d1][rbp]		;; MemR2
	ystore	[srcreg+r8+d2], ymm9		;; Temporarily save new R3	; 68
	vmovapd	ymm9, [srcreg+d1+32][rbp]	;; MemI2
	ystore	[srcreg+r8+d2+32], ymm4		;; Temporarily save new I4	; 69
	vmulpd	ymm4, ymm6, ymm10		;; R2 * MemR2			; 66-70
	vmulpd	ymm6, ymm6, ymm9		;; R2 * MemI2			; 67-71
	vmulpd	ymm9, ymm13, ymm9		;; I2 * MemI2			; 68-72
	vmulpd	ymm13, ymm13, ymm10		;; I2 * MemR2			; 69-73

	vmovapd	ymm10, [srcreg][rbp]		;; MemR1
	vsubpd	ymm4, ymm4, ymm9		;; R2*MemR2-I2*MemI2 (new R2)	; 73-75
	vmovapd	ymm9, [srcreg+32][rbp]		;; MemI1
	vaddpd	ymm6, ymm6, ymm13		;; R2*MemI2+I2*MemR2 (new I2)	; 74-76

	vsubpd	ymm13, ymm2, ymm7		;; I3 - I4 (new R4)		; 67-69
	vaddpd	ymm2, ymm2, ymm7		;; I3 + I4 (new I3)		; 68-70

	vsubpd	ymm7, ymm12, ymm1		;; I5 - I6 (new I6)		; 69-71
	vaddpd	ymm12, ymm12, ymm1		;; I5 + I6 (new I5)		; 70-72

	vsubpd	ymm1, ymm5, ymm11		;; R5 - R6 (new R6)		; 71-73
	vaddpd	ymm5, ymm5, ymm11		;; R5 + R6 (new R5)		; 72-74

	vmulpd	ymm11, ymm3, ymm10		;; R1 * MemR1			; 70-74
	vmulpd	ymm3, ymm3, ymm9		;; R1 * MemI1			; 71-75
	vmulpd	ymm9, ymm9, [srcreg+32]		;; (reload I1) * MemI1		; 72-76
	vmulpd	ymm10, ymm10, [srcreg+32]	;; (reload I1) * MemR1		; 73-77

	vsubpd	ymm11, ymm11, ymm9		;; R1*MemR1-I1*MemI1 (new R1)	; 77-79
	vaddpd	ymm3, ymm3, ymm10		;; R1*MemI1+I1*MemR1 (new I1)	; 78-80

	vsubpd	ymm11, ymm11, [srcreg+r9]		;; R1 = R1 - MemR1
	vsubpd	ymm3, ymm3, [srcreg+r9+32]		;; I1 = I1 - MemI1
	vsubpd	ymm4, ymm4, [srcreg+r9+d1]		;; R2 = R2 - MemR2
	vsubpd	ymm6, ymm6, [srcreg+r9+d1+32]		;; I2 = I2 - MemI2
	jmp	done

mulsubhard:
	vmovapd	YMM_TMP1, ymm3				;; Save 3 registers
	vmovapd	YMM_TMP2, ymm6
	vmovapd	YMM_TMP3, ymm13

	vmovapd	ymm3, [srcreg+r9+d2+d1]			;; MemR4
	vmovapd	ymm9, [srcreg+r10+d2+d1]		;; MemR4#2
	vmulpd	ymm13, ymm3, ymm9			;; MemR4*MemR4#2
	vsubpd	ymm4, ymm4, ymm13			;; R4 = R4 - MemR4*MemR4#2
	vmovapd	ymm6, [srcreg+r10+d2+d1+32]		;; MemI4#2
	vmulpd	ymm13, ymm3, ymm6			;; MemR4*MemI4#2
	vsubpd	ymm7, ymm7, ymm13			;; I4 = I4 - MemR4*MemI4#2
	vmovapd	ymm3, [srcreg+r9+d2+d1+32]		;; MemI4
	vmulpd	ymm13, ymm3, ymm6			;; MemI4*MemI4#2
	vaddpd	ymm4, ymm4, ymm13			;; R4 = R4 + MemI4*MemI4#2
	vmulpd	ymm13, ymm3, ymm9			;; MemI4*MemR4#2
	vsubpd	ymm7, ymm7, ymm13			;; I4 = I4 - MemI4*MemR4#2

	vmovapd	ymm3, [srcreg+r9+d2]			;; MemR3
	vmovapd	ymm9, [srcreg+r10+d2]			;; MemR3#2
	vmulpd	ymm13, ymm3, ymm9			;; MemR3*MemR3#2
	vsubpd	ymm10, ymm10, ymm13			;; R3 = R3 - MemR3*MemR3#2
	vmovapd	ymm6, [srcreg+r10+d2+32]		;; MemI3#2
	vmulpd	ymm13, ymm3, ymm6			;; MemR3*MemI3#2
	vsubpd	ymm2, ymm2, ymm13			;; I3 = I3 - MemR3*MemI3#2
	vmovapd	ymm3, [srcreg+r9+d2+32]			;; MemI3
	vmulpd	ymm13, ymm3, ymm6			;; MemI3*MemI3#2
	vaddpd	ymm10, ymm10, ymm13			;; R3 = R3 + MemI3*MemI3#2
	vmulpd	ymm13, ymm3, ymm9			;; MemI3*MemR3#2
	vsubpd	ymm2, ymm2, ymm13			;; I3 = I3 - MemI3*MemR3#2

	vmovapd	ymm3, [srcreg+r9+d4+d1]			;; MemR6
	vmovapd	ymm9, [srcreg+r10+d4+d1]		;; MemR6#2
	vmulpd	ymm13, ymm3, ymm9			;; MemR6*MemR6#2
	vsubpd	ymm11, ymm11, ymm13			;; R6 = R6 - MemR6*MemR6#2
	vmovapd	ymm6, [srcreg+r10+d4+d1+32]		;; MemI6#2
	vmulpd	ymm13, ymm3, ymm6			;; MemR6*MemI6#2
	vsubpd	ymm1, ymm1, ymm13			;; I6 = I6 - MemR6*MemI6#2
	vmovapd	ymm3, [srcreg+r9+d4+d1+32]		;; MemI6
	vmulpd	ymm13, ymm3, ymm6			;; MemI6*MemI6#2
	vaddpd	ymm11, ymm11, ymm13			;; R6 = R6 + MemI6*MemI6#2
	vmulpd	ymm13, ymm3, ymm9			;; MemI6*MemR6#2
	vsubpd	ymm1, ymm1, ymm13			;; I6 = I6 - MemI6*MemR6#2

	vmovapd	ymm3, [srcreg+r9+d4]			;; MemR5
	vmovapd	ymm9, [srcreg+r10+d4]			;; MemR5#2
	vmulpd	ymm13, ymm3, ymm9			;; MemR5*MemR5#2
	vsubpd	ymm5, ymm5, ymm13			;; R5 = R5 - MemR5*MemR5#2
	vmovapd	ymm6, [srcreg+r10+d4+32]		;; MemI5#2
	vmulpd	ymm13, ymm3, ymm6			;; MemR5*MemI5#2
	vsubpd	ymm12, ymm12, ymm13			;; I5 = I5 - MemR5*MemI5#2
	vmovapd	ymm3, [srcreg+r9+d4+32]			;; MemI5
	vmulpd	ymm13, ymm3, ymm6			;; MemI5*MemI5#2
	vaddpd	ymm5, ymm5, ymm13			;; R5 = R5 + MemI5*MemI5#2
	vmulpd	ymm13, ymm3, ymm9			;; MemI5*MemR5#2
	vsubpd	ymm12, ymm12, ymm13			;; I5 = I5 - MemI5*MemR5#2

	vmovapd	ymm3, [srcreg+r9+d4+d2+d1]		;; MemR8
	vmovapd	ymm9, [srcreg+r10+d4+d2+d1]		;; MemR8#2
	vmulpd	ymm13, ymm3, ymm9			;; MemR8*MemR8#2
	vsubpd	ymm15, ymm15, ymm13			;; R8 = R8 - MemR8*MemR8#2
	vmovapd	ymm6, [srcreg+r10+d4+d2+d1+32]		;; MemI8#2
	vmulpd	ymm13, ymm3, ymm6			;; MemR8*MemI8#2
	vsubpd	ymm14, ymm14, ymm13			;; I8 = I8 - MemR8*MemI8#2
	vmovapd	ymm3, [srcreg+r9+d4+d2+d1+32]		;; MemI8
	vmulpd	ymm13, ymm3, ymm6			;; MemI8*MemI8#2
	vaddpd	ymm15, ymm15, ymm13			;; R8 = R8 + MemI8*MemI8#2
	vmulpd	ymm13, ymm3, ymm9			;; MemI8*MemR8#2
	vsubpd	ymm14, ymm14, ymm13			;; I8 = I8 - MemI8*MemR8#2

	vmovapd	ymm3, [srcreg+r9+d4+d2]			;; MemR7
	vmovapd	ymm9, [srcreg+r10+d4+d2]		;; MemR7#2
	vmulpd	ymm13, ymm3, ymm9			;; MemR7*MemR7#2
	vsubpd	ymm8, ymm8, ymm13			;; R7 = R7 - MemR7*MemR7#2
	vmovapd	ymm6, [srcreg+r10+d4+d2+32]		;; MemI7#2
	vmulpd	ymm13, ymm3, ymm6			;; MemR7*MemI7#2
	vsubpd	ymm0, ymm0, ymm13			;; I7 = I7 - MemR7*MemI7#2
	vmovapd	ymm3, [srcreg+r9+d4+d2+32]		;; MemI7
	vmulpd	ymm13, ymm3, ymm6			;; MemI7*MemI7#2
	vaddpd	ymm8, ymm8, ymm13			;; R7 = R7 + MemI7*MemI7#2
	vmulpd	ymm13, ymm3, ymm9			;; MemI7*MemR7#2
	vsubpd	ymm0, ymm0, ymm13			;; I7 = I7 - MemI7*MemR7#2

	vaddpd	ymm9, ymm4, ymm10		;; R4 + R3 (new R3)		; 65-67
	vsubpd	ymm4, ymm4, ymm10		;; R4 - R3 (new I4)		; 66-68

	vmovapd	ymm10, [srcreg+d1][rbp]		;; MemR2
	ystore	[srcreg+r8+d2], ymm9		;; Temporarily save new R3	; 68
	vmovapd	ymm9, [srcreg+d1+32][rbp]	;; MemI2
	ystore	[srcreg+r8+d2+32], ymm4		;; Temporarily save new I4	; 69

	vmovapd	ymm6, YMM_TMP2
	vmulpd	ymm4, ymm6, ymm10		;; R2 * MemR2			; 66-70
	vmulpd	ymm6, ymm6, ymm9		;; R2 * MemI2			; 67-71
	vmovapd	ymm13, YMM_TMP3
	vmulpd	ymm9, ymm13, ymm9		;; I2 * MemI2			; 68-72
	vmulpd	ymm13, ymm13, ymm10		;; I2 * MemR2			; 69-73

	vmovapd	ymm10, [srcreg][rbp]		;; MemR1
	vsubpd	ymm4, ymm4, ymm9		;; R2*MemR2-I2*MemI2 (new R2)	; 73-75
	vmovapd	ymm9, [srcreg+32][rbp]		;; MemI1
	vaddpd	ymm6, ymm6, ymm13		;; R2*MemI2+I2*MemR2 (new I2)	; 74-76

	vsubpd	ymm13, ymm2, ymm7		;; I3 - I4 (new R4)		; 67-69
	vaddpd	ymm2, ymm2, ymm7		;; I3 + I4 (new I3)		; 68-70

	vsubpd	ymm7, ymm12, ymm1		;; I5 - I6 (new I6)		; 69-71
	vaddpd	ymm12, ymm12, ymm1		;; I5 + I6 (new I5)		; 70-72

	vsubpd	ymm1, ymm5, ymm11		;; R5 - R6 (new R6)		; 71-73
	vaddpd	ymm5, ymm5, ymm11		;; R5 + R6 (new R5)		; 72-74

	vmovapd	ymm3, YMM_TMP1
	vmulpd	ymm11, ymm3, ymm10		;; R1 * MemR1			; 70-74
	vmulpd	ymm3, ymm3, ymm9		;; R1 * MemI1			; 71-75
	vmulpd	ymm9, ymm9, [srcreg+32]		;; (reload I1) * MemI1		; 72-76
	vmulpd	ymm10, ymm10, [srcreg+32]	;; (reload I1) * MemR1		; 73-77

	vsubpd	ymm11, ymm11, ymm9		;; R1*MemR1-I1*MemI1 (new R1)	; 77-79
	vaddpd	ymm3, ymm3, ymm10		;; R1*MemI1+I1*MemR1 (new I1)	; 78-80

	vmovapd	YMM_TMP1, ymm0				;; Save 2 registers
	vmovapd	YMM_TMP2, ymm12

	vmovapd	ymm0, [srcreg+r9+d1]			;; MemR2
	vmovapd	ymm9, [srcreg+r10+d1]			;; MemR2#2
	vmulpd	ymm12, ymm0, ymm9			;; MemR2*MemR2#2
	vsubpd	ymm4, ymm4, ymm12			;; R2 = R2 - MemR2*MemR2#2
	vmovapd	ymm10, [srcreg+r10+d1+32]		;; MemI2#2
	vmulpd	ymm12, ymm0, ymm10			;; MemR2*MemI2#2
	vsubpd	ymm6, ymm6, ymm12			;; I2 = I2 - MemR2*MemI2#2
	vmovapd	ymm0, [srcreg+r9+d1+32]			;; MemI2
	vmulpd	ymm12, ymm0, ymm10			;; MemI2*MemI2#2
	vaddpd	ymm4, ymm4, ymm12			;; R2 = R2 + MemI2*MemI2#2
	vmulpd	ymm12, ymm0, ymm9			;; MemI2*MemR2#2
	vsubpd	ymm6, ymm6, ymm12			;; I2 = I2 - MemI2*MemR2#2

	vmovapd	ymm0, [srcreg+r9]			;; MemR1
	vmovapd	ymm9, [srcreg+r10]			;; MemR1#2
	vmulpd	ymm12, ymm0, ymm9			;; MemR1*MemR1#2
	vsubpd	ymm11, ymm11, ymm12			;; R1 = R1 - MemR1*MemR1#2
	vmovapd	ymm10, [srcreg+r10+32]			;; MemI1#2
	vmulpd	ymm12, ymm0, ymm10			;; MemR1*MemI1#2
	vsubpd	ymm3, ymm3, ymm12			;; I1 = I1 - MemR1*MemI1#2
	vmovapd	ymm0, [srcreg+r9+32]			;; MemI1
	vmulpd	ymm12, ymm0, ymm10			;; MemI1*MemI1#2
	vaddpd	ymm11, ymm11, ymm12			;; R1 = R1 + MemI1*MemI1#2
	vmulpd	ymm12, ymm0, ymm9			;; MemI1*MemR1#2
	vsubpd	ymm3, ymm3, ymm12			;; I1 = I1 - MemI1*MemR1#2

	vmovapd	ymm0, YMM_TMP1				;; Restore 2 registers
	vmovapd	ymm12, YMM_TMP2

done:
	ENDM

yr8_8cl_eight_complex_with_mulf_preload MACRO
	ENDM
yr8_8cl_eight_complex_with_mulf MACRO srcreg,srcinc,d1,d2,d4
	LOCAL	orig, back_to_orig

	cmp	mul4_opcode, 0			;; See if we need to do more than the original type-4 FFT multiply
	je	short orig
	yr8_dispatch call,y8complex_mulf_opcode,d1,d2,d4 ;; Handle more difficult cases
	jmp	back_to_orig

orig:	vmovapd	ymm3, [srcreg+d2+d1][rbp]	;; MemR4
	vmovapd	ymm9, [srcreg+d2+d1+32][rbp]	;; MemI4
	vmovapd	ymm7, [srcreg+d2+d1][rbx]	;; R4
	vmovapd	ymm10, [srcreg+d2+d1+32][rbx]	;; I4
	vmulpd	ymm4, ymm7, ymm3		;; R4 * MemR4			; 32-36
	vmulpd	ymm7, ymm7, ymm9		;; R4 * MemI4			; 33-37
	vmulpd	ymm9, ymm10, ymm9		;; I4 * MemI4			; 34-38
	vmulpd	ymm10, ymm10, ymm3		;; I4 * MemR4			; 35-39

	vmovapd	ymm3, [srcreg+d2][rbp]		;; MemR3
	vsubpd	ymm4, ymm4, ymm9		;; R4*MemR4-I4*MemI4 (new R4)	; 51-53
	vmovapd	ymm9, [srcreg+d2+32][rbp]	;; MemI3
	vaddpd	ymm7, ymm7, ymm10		;; R4*MemI4+I4*MemR4 (new I4)	; 52-54

	vmovapd	ymm2, [srcreg+d2][rbx]		;; R3
	vmovapd	ymm11, [srcreg+d2+32][rbx]	;; I3
	vmulpd	ymm10, ymm2, ymm3		;; R3 * MemR3			; 36-40
	vmulpd	ymm2, ymm2, ymm9		;; R3 * MemI3			; 37-41
	vmulpd	ymm9, ymm11, ymm9		;; I3 * MemI3			; 38-42
	vmulpd	ymm11, ymm11, ymm3		;; I3 * MemR3			; 39-43

	vmovapd	ymm3, [srcreg+d4+d1][rbp]	;; MemR6
	vsubpd	ymm10, ymm10, ymm9		;; R3*MemR3-I3*MemI3 (new R3)	; 53-55
	vmovapd	ymm9, [srcreg+d4+d1+32][rbp]	;; MemI6
	vaddpd	ymm2, ymm2, ymm11		;; R3*MemI3+I3*MemR3 (new I3)	; 54-56

	vmovapd	ymm1, [srcreg+d4+d1][rbx]	;; R6
	vmovapd	ymm5, [srcreg+d4+d1+32][rbx]	;; I6
	vmulpd	ymm11, ymm1, ymm3		;; R6 * MemR6			; 42-46
	vmulpd	ymm1, ymm1, ymm9		;; R6 * MemI6			; 43-47
	vmulpd	ymm9, ymm5, ymm9		;; I6 * MemI6			; 46-50
	vmulpd	ymm5, ymm5, ymm3		;; I6 * MemR6			; 47-51

	vmovapd	ymm3, [srcreg+d4][rbp]		;; MemR5
	vsubpd	ymm11, ymm11, ymm9		;; R6*MemR6-I6*MemI6 (new R6)	; 55-57
	vmovapd	ymm9, [srcreg+d4+32][rbp]	;; MemI5
	vaddpd	ymm1, ymm1, ymm5		;; R6*MemI6+I6*MemR6 (new I6)	; 56-58

	vmovapd	ymm12, [srcreg+d4][rbx]		;; R5
	vmovapd	ymm15, [srcreg+d4+32][rbx]	;; I5
	vmulpd	ymm5, ymm12, ymm3		;; R5 * MemR5			; 44-48
	vmulpd	ymm12, ymm12, ymm9		;; R5 * MemI5			; 45-49
	vmulpd	ymm9, ymm15, ymm9		;; I5 * MemI5			; 48-52
	vmulpd	ymm15, ymm15, ymm3		;; I5 * MemR5			; 49-53

	vmovapd	ymm3, [srcreg+d4+d2+d1][rbp]	;; MemR8
	vsubpd	ymm5, ymm5, ymm9		;; R5*MemR5-I5*MemI5 (new R5)	; 57-59
	vmovapd	ymm9, [srcreg+d4+d2+d1+32][rbp]	;; MemI8
	vaddpd	ymm12, ymm12, ymm15		;; R5*MemI5+I5*MemR5 (new I5)	; 58-60

	vmovapd	ymm14, [srcreg+d4+d2+d1][rbx]	;; R8
	vmovapd	ymm8, [srcreg+d4+d2+d1+32][rbx]	;; I8
	vmulpd	ymm15, ymm14, ymm3		;; R8 * MemR8			; 50-54
	vmulpd	ymm14, ymm14, ymm9		;; R8 * MemI8			; 51-55
	vmulpd	ymm9, ymm8, ymm9		;; I8 * MemI8			; 52-56
	vmulpd	ymm8, ymm8, ymm3		;; I8 * MemR8			; 53-57

	vmovapd	ymm3, [srcreg+d4+d2][rbp]	;; MemR7
	vsubpd	ymm15, ymm15, ymm9		;; R8*MemR8-I8*MemI8 (new R8)	; 59-61
	vmovapd	ymm9, [srcreg+d4+d2+32][rbp]	;; MemI7
	vaddpd	ymm14, ymm14, ymm8		;; R8*MemI8+I8*MemR8 (new I8)	; 60-62

	vmovapd	ymm0, [srcreg+d4+d2][rbx]	;; R7
	vmovapd	ymm6, [srcreg+d4+d2+32][rbx]	;; I7
	vmulpd	ymm8, ymm0, ymm3		;; R7 * MemR7			; 54-58
	vmulpd	ymm0, ymm0, ymm9		;; R7 * MemI7			; 55-59
	vmulpd	ymm9, ymm6, ymm9		;; I7 * MemI7			; 56-60
	vmulpd	ymm6, ymm6, ymm3		;; I7 * MemR7			; 57-61

	vsubpd	ymm8, ymm8, ymm9		;; R7*MemR7-I7*MemI7 (new R7)	; 61-63
	vaddpd	ymm0, ymm0, ymm6		;; R7*MemI7+I7*MemR7 (new I7)	; 62-64

	vaddpd	ymm9, ymm4, ymm10		;; R4 + R3 (new R3)		; 65-67
	vsubpd	ymm4, ymm4, ymm10		;; R4 - R3 (new I4)		; 66-68

	vmovapd	ymm10, [srcreg+d1][rbp]		;; MemR2
	ystore	[srcreg+d2], ymm9		;; Temporarily save new R3	; 68
	vmovapd	ymm9, [srcreg+d1+32][rbp]	;; MemI2
	ystore	[srcreg+d2+32], ymm4		;; Temporarily save new I4	; 69

	vmovapd	ymm6, [srcreg+d1][rbx]		;; R2
	vmovapd	ymm13, [srcreg+d1+32][rbx]	;; I2
	vmulpd	ymm4, ymm6, ymm10		;; R2 * MemR2			; 66-70
	vmulpd	ymm6, ymm6, ymm9		;; R2 * MemI2			; 67-71
	vmulpd	ymm9, ymm13, ymm9		;; I2 * MemI2			; 68-72
	vmulpd	ymm13, ymm13, ymm10		;; I2 * MemR2			; 69-73

	vmovapd	ymm10, [srcreg][rbp]		;; MemR1
	vsubpd	ymm4, ymm4, ymm9		;; R2*MemR2-I2*MemI2 (new R2)	; 73-75
	vmovapd	ymm9, [srcreg+32][rbp]		;; MemI1
	vaddpd	ymm6, ymm6, ymm13		;; R2*MemI2+I2*MemR2 (new I2)	; 74-76

	vsubpd	ymm13, ymm2, ymm7		;; I3 - I4 (new R4)		; 67-69
	vaddpd	ymm2, ymm2, ymm7		;; I3 + I4 (new I3)		; 68-70

	vsubpd	ymm7, ymm12, ymm1		;; I5 - I6 (new I6)		; 69-71
	vaddpd	ymm12, ymm12, ymm1		;; I5 + I6 (new I5)		; 70-72

	vsubpd	ymm1, ymm5, ymm11		;; R5 - R6 (new R6)		; 71-73
	vaddpd	ymm5, ymm5, ymm11		;; R5 + R6 (new R5)		; 72-74

	vmovapd	ymm3, [srcreg][rbx]		;; R1
	vmulpd	ymm11, ymm3, ymm10		;; R1 * MemR1			; 70-74
	vmulpd	ymm3, ymm3, ymm9		;; R1 * MemI1			; 71-75
	vmulpd	ymm9, ymm9, [srcreg+32][rbx]	;; (load I1) * MemI1		; 72-76
	vmulpd	ymm10, ymm10, [srcreg+32][rbx]	;; (load I1) * MemR1		; 73-77

	vsubpd	ymm11, ymm11, ymm9		;; R1*MemR1-I1*MemI1 (new R1)	; 77-79
	vaddpd	ymm3, ymm3, ymm10		;; R1*MemI1+I1*MemR1 (new I1)	; 78-80

back_to_orig:
	vsubpd	ymm9, ymm0, ymm14		;; I7 - I8 (new R8)		; 75-77
	vaddpd	ymm0, ymm0, ymm14		;; I7 + I8 (new I7)		; 76-78

	vsubpd	ymm14, ymm15, ymm8		;; R8 - R7 (new I8)		; 79-81
	vaddpd	ymm15, ymm15, ymm8		;; R8 + R7 (new R7)		; 80-82

	vsubpd	ymm8, ymm3, ymm6		;; I1 - I2 (new I2)		; 81-83
	vaddpd	ymm3, ymm3, ymm6		;; I1 + I2 (new I1)		; 82-84

	vsubpd	ymm6, ymm11, ymm4		;; R1 - R2 (new R2)		; 83-85
	vaddpd	ymm11, ymm11, ymm4		;; R1 + R2 (new R1)		; 84-86

	;; multiply R6/I6 by SQRTHALF - i*SQRTHALF
	vsubpd	ymm4, ymm7, ymm1		;; I6 = I6 - R6			; 85-87
	vaddpd	ymm1, ymm1, ymm7		;; R6 = R6 + I6			; 86-88

	;; multiply R8/I8 by SQRTHALF - i*SQRTHALF
	vsubpd	ymm7, ymm14, ymm9		;; I8 = I8 - R8			; 87-89
	vaddpd	ymm9, ymm9, ymm14		;; R8 = R8 + I8			; 88-90

	vmovapd	ymm14, YMM_SQRTHALF
	vmulpd	ymm4, ymm4, ymm14		;; I6 * SQRTHALF		; 88-92
	vmulpd	ymm1, ymm1, ymm14		;; R6 * SQRTHALF		; 89-93
	vmulpd	ymm7, ymm7, ymm14		;; I8 * SQRTHALF		; 90-94
	vmulpd	ymm9, ymm9, ymm14		;; R8 * SQRTHALF		; 91-95

	vaddpd	ymm14, ymm6, ymm13		;; R2 + R4 (newer R2)		; 89-91
	vsubpd	ymm6, ymm6, ymm13		;; R2 - R4 (newer R4)		; 90-92

	vmovapd	ymm10, [srcreg+d2+32]		;; Reload new I4
	vaddpd	ymm13, ymm8, ymm10		;; I2 + I4 (newer I2)		; 91-93
	vsubpd	ymm8, ymm8, ymm10		;; I2 - I4 (newer I4)		; 92-94

	vsubpd	ymm10, ymm12, ymm0		;; I5 - I7 (newer R7)		; 93-95
	vaddpd	ymm12, ymm12, ymm0		;; I5 + I7 (newer I5)		; 94-96

	vsubpd	ymm0, ymm3, ymm2		;; I1 - I3 (newer I3)		; 95-97
	vaddpd	ymm3, ymm3, ymm2		;; I1 + I3 (newer I1)		; 96-98

	vsubpd	ymm2, ymm4, ymm7		;; I6 - I8 (newer R8)		; 97-99
	vaddpd	ymm4, ymm4, ymm7		;; I6 + I8 (newer I6)		; 98-100

	vsubpd	ymm7, ymm9, ymm1		;; R8 - R6 (newer I8)		; 99-101
	vaddpd	ymm9, ymm9, ymm1		;; R8 + R6 (newer R6)		; 100-102

	vsubpd	ymm1, ymm6, ymm2		;; R4 - R8 (final R8)		; 101-103
	vaddpd	ymm6, ymm6, ymm2		;; R4 + R8 (final R4)		; 102-104

	vsubpd	ymm2, ymm13, ymm4		;; I2 - I6 (final I6)		; 103-105
	vaddpd	ymm13, ymm13, ymm4		;; I2 + I6 (final I2)		; 104-106
	ystore	[srcreg+d4+d2+d1], ymm1		;; Save R8			; 104

	vsubpd	ymm4, ymm8, ymm7		;; I4 - I8 (final I8)		; 105-107
	vaddpd	ymm8, ymm8, ymm7		;; I4 + I8 (final I4)		; 106-108
	ystore	[srcreg+d2+d1], ymm6		;; Save R4			; 105
	ystore	[srcreg+d4+d1+32], ymm2		;; Save I6			; 106

	vsubpd	ymm7, ymm14, ymm9		;; R2 - R6 (final R6)		; 107-109
	vaddpd	ymm14, ymm14, ymm9		;; R2 + R6 (final R2)		; 108-110
	ystore	[srcreg+d1+32], ymm13		;; Save I2			; 107
	ystore	[srcreg+d4+d2+d1+32], ymm4	;; Save I8			; 108

	vmovapd	ymm1, [srcreg+d2]		;; Reload new R3
	vsubpd	ymm9, ymm11, ymm1		;; R1 - R3 (newer R3)		; 109-111
	vaddpd	ymm11, ymm11, ymm1		;; R1 + R3 (newer R1)		; 110-112
	ystore	[srcreg+d2+d1+32], ymm8		;; Save I4			; 109
	ystore	[srcreg+d4+d1], ymm7		;; Save R6			; 110

	vaddpd	ymm1, ymm15, ymm5		;; R7 + R5 (newer R5)		; 111-113
	vsubpd	ymm15, ymm15, ymm5		;; R7 - R5 (newer I7)		; 112-114
	ystore	[srcreg+d1], ymm14		;; Save R2			; 111

	vsubpd	ymm5, ymm3, ymm12		;; I1 - I5 (final I5)		; 113-115
	vaddpd	ymm3, ymm3, ymm12		;; I1 + I5 (final I1)		; 114-116

	vsubpd	ymm12, ymm9, ymm10		;; R3 - R7 (final R7)		; 115-117
	vaddpd	ymm9, ymm9, ymm10		;; R3 + R7 (final R3)		; 116-118
	ystore	[srcreg+d4+32], ymm5		;; Save I5			; 116

	vsubpd	ymm10, ymm0, ymm15		;; I3 - I7 (final I7)		; 117-119
	vaddpd	ymm0, ymm0, ymm15		;; I3 + I7 (final I3)		; 118-120
	ystore	[srcreg+32], ymm3		;; Save I1			; 117
	ystore	[srcreg+d4+d2], ymm12		;; Save R7			; 118

	vsubpd	ymm15, ymm11, ymm1		;; R1 - R5 (final R5)		; 119-121
	vaddpd	ymm11, ymm11, ymm1		;; R1 + R5 (final R1)		; 120-122
	ystore	[srcreg+d2], ymm9		;; Save R3			; 119
	ystore	[srcreg+d4+d2+32], ymm10	;; Save I7			; 120

	ystore	[srcreg+d2+32], ymm0		;; Save I3			; 121
	ystore	[srcreg+d4], ymm15		;; Save R5			; 122
	ystore	[srcreg], ymm11			;; Save R1			; 123

	bump	srcreg, srcinc
	ENDM

;; Same as above code between orig and back_to_orig, except we implement mul4_opcode options

yr8_8c_mulf_opcode MACRO srcreg,d1,d2,d4
	LOCAL	fma_or_unfft, fma, addmul, submul, muladd, muladdhard, mulsub, mulsubhard, unfft, done

	movzx	r10, mul4_opcode		;; Load the mul4_opcode
	mov	r9, SRC2ARG			;; Distance to s2 arg from gwaddmul4 or gwsubmul4 or s3 arg in gwmuladd4 or gwmulsub4
	cmp	r10, 2				;; Case off opcode
	jg	fma_or_unfft			;; 3,4,5=muladd,mulsub,unfft
	je	submul				;; 2=submul
	;;jl	addmul				;; 1=addmul

addmul:	vmovapd	ymm3, [srcreg+d2+d1][rbp]	;; MemR4
	vaddpd	ymm3, ymm3, [srcreg+d2+d1][r9]	;; MemR4
	vmovapd	ymm9, [srcreg+d2+d1+32][rbp]	;; MemI4
	vaddpd	ymm9, ymm9, [srcreg+d2+d1+32][r9];; MemI4
	vmovapd	ymm7, [srcreg+d2+d1][rbx]	;; R4
	vmovapd	ymm10, [srcreg+d2+d1+32][rbx]	;; I4
	vmulpd	ymm4, ymm7, ymm3		;; R4 * MemR4			; 32-36
	vmulpd	ymm7, ymm7, ymm9		;; R4 * MemI4			; 33-37
	vmulpd	ymm9, ymm10, ymm9		;; I4 * MemI4			; 34-38
	vmulpd	ymm10, ymm10, ymm3		;; I4 * MemR4			; 35-39

	vmovapd	ymm3, [srcreg+d2][rbp]		;; MemR3
	vaddpd	ymm3, ymm3, [srcreg+d2][r9]	;; MemR3
	vsubpd	ymm4, ymm4, ymm9		;; R4*MemR4-I4*MemI4 (new R4)	; 51-53
	vmovapd	ymm9, [srcreg+d2+32][rbp]	;; MemI3
	vaddpd	ymm9, ymm9, [srcreg+d2+32][r9]	;; MemI3
	vaddpd	ymm7, ymm7, ymm10		;; R4*MemI4+I4*MemR4 (new I4)	; 52-54

	vmovapd	ymm2, [srcreg+d2][rbx]		;; R3
	vmovapd	ymm11, [srcreg+d2+32][rbx]	;; I3
	vmulpd	ymm10, ymm2, ymm3		;; R3 * MemR3			; 36-40
	vmulpd	ymm2, ymm2, ymm9		;; R3 * MemI3			; 37-41
	vmulpd	ymm9, ymm11, ymm9		;; I3 * MemI3			; 38-42
	vmulpd	ymm11, ymm11, ymm3		;; I3 * MemR3			; 39-43

	vmovapd	ymm3, [srcreg+d4+d1][rbp]	;; MemR6
	vaddpd	ymm3, ymm3, [srcreg+d4+d1][r9]	;; MemR6
	vsubpd	ymm10, ymm10, ymm9		;; R3*MemR3-I3*MemI3 (new R3)	; 53-55
	vmovapd	ymm9, [srcreg+d4+d1+32][rbp]	;; MemI6
	vaddpd	ymm9, ymm9, [srcreg+d4+d1+32][r9];; MemI6
	vaddpd	ymm2, ymm2, ymm11		;; R3*MemI3+I3*MemR3 (new I3)	; 54-56

	vmovapd	ymm1, [srcreg+d4+d1][rbx]	;; R6
	vmovapd	ymm5, [srcreg+d4+d1+32][rbx]	;; I6
	vmulpd	ymm11, ymm1, ymm3		;; R6 * MemR6			; 42-46
	vmulpd	ymm1, ymm1, ymm9		;; R6 * MemI6			; 43-47
	vmulpd	ymm9, ymm5, ymm9		;; I6 * MemI6			; 46-50
	vmulpd	ymm5, ymm5, ymm3		;; I6 * MemR6			; 47-51

	vmovapd	ymm3, [srcreg+d4][rbp]		;; MemR5
	vaddpd	ymm3, ymm3, [srcreg+d4][r9]	;; MemR5
	vsubpd	ymm11, ymm11, ymm9		;; R6*MemR6-I6*MemI6 (new R6)	; 55-57
	vmovapd	ymm9, [srcreg+d4+32][rbp]	;; MemI5
	vaddpd	ymm9, ymm9, [srcreg+d4+32][r9]	;; MemI5
	vaddpd	ymm1, ymm1, ymm5		;; R6*MemI6+I6*MemR6 (new I6)	; 56-58

	vmovapd	ymm12, [srcreg+d4][rbx]		;; R5
	vmovapd	ymm15, [srcreg+d4+32][rbx]	;; I5
	vmulpd	ymm5, ymm12, ymm3		;; R5 * MemR5			; 44-48
	vmulpd	ymm12, ymm12, ymm9		;; R5 * MemI5			; 45-49
	vmulpd	ymm9, ymm15, ymm9		;; I5 * MemI5			; 48-52
	vmulpd	ymm15, ymm15, ymm3		;; I5 * MemR5			; 49-53

	vmovapd	ymm3, [srcreg+d4+d2+d1][rbp]	;; MemR8
	vaddpd	ymm3, ymm3, [srcreg+d4+d2+d1][r9];; MemR8
	vsubpd	ymm5, ymm5, ymm9		;; R5*MemR5-I5*MemI5 (new R5)	; 57-59
	vmovapd	ymm9, [srcreg+d4+d2+d1+32][rbp]	;; MemI8
	vaddpd	ymm9, ymm9, [srcreg+d4+d2+d1+32][r9];; MemI8
	vaddpd	ymm12, ymm12, ymm15		;; R5*MemI5+I5*MemR5 (new I5)	; 58-60

	vmovapd	ymm14, [srcreg+d4+d2+d1][rbx]	;; R8
	vmovapd	ymm8, [srcreg+d4+d2+d1+32][rbx]	;; I8
	vmulpd	ymm15, ymm14, ymm3		;; R8 * MemR8			; 50-54
	vmulpd	ymm14, ymm14, ymm9		;; R8 * MemI8			; 51-55
	vmulpd	ymm9, ymm8, ymm9		;; I8 * MemI8			; 52-56
	vmulpd	ymm8, ymm8, ymm3		;; I8 * MemR8			; 53-57

	vmovapd	ymm3, [srcreg+d4+d2][rbp]	;; MemR7
	vaddpd	ymm3, ymm3, [srcreg+d4+d2][r9]	;; MemR7
	vsubpd	ymm15, ymm15, ymm9		;; R8*MemR8-I8*MemI8 (new R8)	; 59-61
	vmovapd	ymm9, [srcreg+d4+d2+32][rbp]	;; MemI7
	vaddpd	ymm9, ymm9, [srcreg+d4+d2+32][r9];; MemI7
	vaddpd	ymm14, ymm14, ymm8		;; R8*MemI8+I8*MemR8 (new I8)	; 60-62

	vmovapd	ymm0, [srcreg+d4+d2][rbx]	;; R7
	vmovapd	ymm6, [srcreg+d4+d2+32][rbx]	;; I7
	vmulpd	ymm8, ymm0, ymm3		;; R7 * MemR7			; 54-58
	vmulpd	ymm0, ymm0, ymm9		;; R7 * MemI7			; 55-59
	vmulpd	ymm9, ymm6, ymm9		;; I7 * MemI7			; 56-60
	vmulpd	ymm6, ymm6, ymm3		;; I7 * MemR7			; 57-61

	vsubpd	ymm8, ymm8, ymm9		;; R7*MemR7-I7*MemI7 (new R7)	; 61-63
	vaddpd	ymm0, ymm0, ymm6		;; R7*MemI7+I7*MemR7 (new I7)	; 62-64

	vaddpd	ymm9, ymm4, ymm10		;; R4 + R3 (new R3)		; 65-67
	vsubpd	ymm4, ymm4, ymm10		;; R4 - R3 (new I4)		; 66-68

	vmovapd	ymm10, [srcreg+d1][rbp]		;; MemR2
	vaddpd	ymm10, ymm10, [srcreg+d1][r9]	;; MemR2
	ystore	[srcreg+d2], ymm9		;; Temporarily save new R3	; 68
	vmovapd	ymm9, [srcreg+d1+32][rbp]	;; MemI2
	vaddpd	ymm9, ymm9, [srcreg+d1+32][r9]	;; MemI2
	ystore	[srcreg+d2+32], ymm4		;; Temporarily save new I4	; 69

	vmovapd	ymm6, [srcreg+d1][rbx]		;; R2
	vmovapd	ymm13, [srcreg+d1+32][rbx]	;; I2
	vmulpd	ymm4, ymm6, ymm10		;; R2 * MemR2			; 66-70
	vmulpd	ymm6, ymm6, ymm9		;; R2 * MemI2			; 67-71
	vmulpd	ymm9, ymm13, ymm9		;; I2 * MemI2			; 68-72
	vmulpd	ymm13, ymm13, ymm10		;; I2 * MemR2			; 69-73

	vmovapd	ymm10, [srcreg][rbp]		;; MemR1
	vaddpd	ymm10, ymm10, [srcreg][r9]	;; MemR1
	vsubpd	ymm4, ymm4, ymm9		;; R2*MemR2-I2*MemI2 (new R2)	; 73-75
	vmovapd	ymm9, [srcreg+32][rbp]		;; MemI1
	vaddpd	ymm9, ymm9, [srcreg+32][r9]	;; MemI1
	vaddpd	ymm6, ymm6, ymm13		;; R2*MemI2+I2*MemR2 (new I2)	; 74-76

	vsubpd	ymm13, ymm2, ymm7		;; I3 - I4 (new R4)		; 67-69
	vaddpd	ymm2, ymm2, ymm7		;; I3 + I4 (new I3)		; 68-70

	vsubpd	ymm7, ymm12, ymm1		;; I5 - I6 (new I6)		; 69-71
	vaddpd	ymm12, ymm12, ymm1		;; I5 + I6 (new I5)		; 70-72

	vsubpd	ymm1, ymm5, ymm11		;; R5 - R6 (new R6)		; 71-73
	vaddpd	ymm5, ymm5, ymm11		;; R5 + R6 (new R5)		; 72-74

	vmovapd	ymm3, [srcreg][rbx]		;; R1
	vmulpd	ymm11, ymm3, ymm10		;; R1 * MemR1			; 70-74
	vmulpd	ymm3, ymm3, ymm9		;; R1 * MemI1			; 71-75
	vmulpd	ymm9, ymm9, [srcreg+32][rbx]	;; (load I1) * MemI1		; 72-76
	vmulpd	ymm10, ymm10, [srcreg+32][rbx]	;; (load I1) * MemR1		; 73-77

	vsubpd	ymm11, ymm11, ymm9		;; R1*MemR1-I1*MemI1 (new R1)	; 77-79
	vaddpd	ymm3, ymm3, ymm10		;; R1*MemI1+I1*MemR1 (new I1)	; 78-80
	jmp	done

submul:	vmovapd	ymm3, [srcreg+d2+d1][rbp]	;; MemR4
	vsubpd	ymm3, ymm3, [srcreg+d2+d1][r9]	;; MemR4
	vmovapd	ymm9, [srcreg+d2+d1+32][rbp]	;; MemI4
	vsubpd	ymm9, ymm9, [srcreg+d2+d1+32][r9];; MemI4
	vmovapd	ymm7, [srcreg+d2+d1][rbx]	;; R4
	vmovapd	ymm10, [srcreg+d2+d1+32][rbx]	;; I4
	vmulpd	ymm4, ymm7, ymm3		;; R4 * MemR4			; 32-36
	vmulpd	ymm7, ymm7, ymm9		;; R4 * MemI4			; 33-37
	vmulpd	ymm9, ymm10, ymm9		;; I4 * MemI4			; 34-38
	vmulpd	ymm10, ymm10, ymm3		;; I4 * MemR4			; 35-39

	vmovapd	ymm3, [srcreg+d2][rbp]		;; MemR3
	vsubpd	ymm3, ymm3, [srcreg+d2][r9]	;; MemR3
	vsubpd	ymm4, ymm4, ymm9		;; R4*MemR4-I4*MemI4 (new R4)	; 51-53
	vmovapd	ymm9, [srcreg+d2+32][rbp]	;; MemI3
	vsubpd	ymm9, ymm9, [srcreg+d2+32][r9]	;; MemI3
	vaddpd	ymm7, ymm7, ymm10		;; R4*MemI4+I4*MemR4 (new I4)	; 52-54

	vmovapd	ymm2, [srcreg+d2][rbx]		;; R3
	vmovapd	ymm11, [srcreg+d2+32][rbx]	;; I3
	vmulpd	ymm10, ymm2, ymm3		;; R3 * MemR3			; 36-40
	vmulpd	ymm2, ymm2, ymm9		;; R3 * MemI3			; 37-41
	vmulpd	ymm9, ymm11, ymm9		;; I3 * MemI3			; 38-42
	vmulpd	ymm11, ymm11, ymm3		;; I3 * MemR3			; 39-43

	vmovapd	ymm3, [srcreg+d4+d1][rbp]	;; MemR6
	vsubpd	ymm3, ymm3, [srcreg+d4+d1][r9]	;; MemR6
	vsubpd	ymm10, ymm10, ymm9		;; R3*MemR3-I3*MemI3 (new R3)	; 53-55
	vmovapd	ymm9, [srcreg+d4+d1+32][rbp]	;; MemI6
	vsubpd	ymm9, ymm9, [srcreg+d4+d1+32][r9];; MemI6
	vaddpd	ymm2, ymm2, ymm11		;; R3*MemI3+I3*MemR3 (new I3)	; 54-56

	vmovapd	ymm1, [srcreg+d4+d1][rbx]	;; R6
	vmovapd	ymm5, [srcreg+d4+d1+32][rbx]	;; I6
	vmulpd	ymm11, ymm1, ymm3		;; R6 * MemR6			; 42-46
	vmulpd	ymm1, ymm1, ymm9		;; R6 * MemI6			; 43-47
	vmulpd	ymm9, ymm5, ymm9		;; I6 * MemI6			; 46-50
	vmulpd	ymm5, ymm5, ymm3		;; I6 * MemR6			; 47-51

	vmovapd	ymm3, [srcreg+d4][rbp]		;; MemR5
	vsubpd	ymm3, ymm3, [srcreg+d4][r9]	;; MemR5
	vsubpd	ymm11, ymm11, ymm9		;; R6*MemR6-I6*MemI6 (new R6)	; 55-57
	vmovapd	ymm9, [srcreg+d4+32][rbp]	;; MemI5
	vsubpd	ymm9, ymm9, [srcreg+d4+32][r9]	;; MemI5
	vaddpd	ymm1, ymm1, ymm5		;; R6*MemI6+I6*MemR6 (new I6)	; 56-58

	vmovapd	ymm12, [srcreg+d4][rbx]		;; R5
	vmovapd	ymm15, [srcreg+d4+32][rbx]	;; I5
	vmulpd	ymm5, ymm12, ymm3		;; R5 * MemR5			; 44-48
	vmulpd	ymm12, ymm12, ymm9		;; R5 * MemI5			; 45-49
	vmulpd	ymm9, ymm15, ymm9		;; I5 * MemI5			; 48-52
	vmulpd	ymm15, ymm15, ymm3		;; I5 * MemR5			; 49-53

	vmovapd	ymm3, [srcreg+d4+d2+d1][rbp]	;; MemR8
	vsubpd	ymm3, ymm3, [srcreg+d4+d2+d1][r9];; MemR8
	vsubpd	ymm5, ymm5, ymm9		;; R5*MemR5-I5*MemI5 (new R5)	; 57-59
	vmovapd	ymm9, [srcreg+d4+d2+d1+32][rbp]	;; MemI8
	vsubpd	ymm9, ymm9, [srcreg+d4+d2+d1+32][r9];; MemI8
	vaddpd	ymm12, ymm12, ymm15		;; R5*MemI5+I5*MemR5 (new I5)	; 58-60

	vmovapd	ymm14, [srcreg+d4+d2+d1][rbx]	;; R8
	vmovapd	ymm8, [srcreg+d4+d2+d1+32][rbx]	;; I8
	vmulpd	ymm15, ymm14, ymm3		;; R8 * MemR8			; 50-54
	vmulpd	ymm14, ymm14, ymm9		;; R8 * MemI8			; 51-55
	vmulpd	ymm9, ymm8, ymm9		;; I8 * MemI8			; 52-56
	vmulpd	ymm8, ymm8, ymm3		;; I8 * MemR8			; 53-57

	vmovapd	ymm3, [srcreg+d4+d2][rbp]	;; MemR7
	vsubpd	ymm3, ymm3, [srcreg+d4+d2][r9]	;; MemR7
	vsubpd	ymm15, ymm15, ymm9		;; R8*MemR8-I8*MemI8 (new R8)	; 59-61
	vmovapd	ymm9, [srcreg+d4+d2+32][rbp]	;; MemI7
	vsubpd	ymm9, ymm9, [srcreg+d4+d2+32][r9];; MemI7
	vaddpd	ymm14, ymm14, ymm8		;; R8*MemI8+I8*MemR8 (new I8)	; 60-62

	vmovapd	ymm0, [srcreg+d4+d2][rbx]	;; R7
	vmovapd	ymm6, [srcreg+d4+d2+32][rbx]	;; I7
	vmulpd	ymm8, ymm0, ymm3		;; R7 * MemR7			; 54-58
	vmulpd	ymm0, ymm0, ymm9		;; R7 * MemI7			; 55-59
	vmulpd	ymm9, ymm6, ymm9		;; I7 * MemI7			; 56-60
	vmulpd	ymm6, ymm6, ymm3		;; I7 * MemR7			; 57-61

	vsubpd	ymm8, ymm8, ymm9		;; R7*MemR7-I7*MemI7 (new R7)	; 61-63
	vaddpd	ymm0, ymm0, ymm6		;; R7*MemI7+I7*MemR7 (new I7)	; 62-64

	vaddpd	ymm9, ymm4, ymm10		;; R4 + R3 (new R3)		; 65-67
	vsubpd	ymm4, ymm4, ymm10		;; R4 - R3 (new I4)		; 66-68

	vmovapd	ymm10, [srcreg+d1][rbp]		;; MemR2
	vsubpd	ymm10, ymm10, [srcreg+d1][r9]	;; MemR2
	ystore	[srcreg+d2], ymm9		;; Temporarily save new R3	; 68
	vmovapd	ymm9, [srcreg+d1+32][rbp]	;; MemI2
	vsubpd	ymm9, ymm9, [srcreg+d1+32][r9]	;; MemI2
	ystore	[srcreg+d2+32], ymm4		;; Temporarily save new I4	; 69

	vmovapd	ymm6, [srcreg+d1][rbx]		;; R2
	vmovapd	ymm13, [srcreg+d1+32][rbx]	;; I2
	vmulpd	ymm4, ymm6, ymm10		;; R2 * MemR2			; 66-70
	vmulpd	ymm6, ymm6, ymm9		;; R2 * MemI2			; 67-71
	vmulpd	ymm9, ymm13, ymm9		;; I2 * MemI2			; 68-72
	vmulpd	ymm13, ymm13, ymm10		;; I2 * MemR2			; 69-73

	vmovapd	ymm10, [srcreg][rbp]		;; MemR1
	vsubpd	ymm10, ymm10, [srcreg][r9]	;; MemR1
	vsubpd	ymm4, ymm4, ymm9		;; R2*MemR2-I2*MemI2 (new R2)	; 73-75
	vmovapd	ymm9, [srcreg+32][rbp]		;; MemI1
	vsubpd	ymm9, ymm9, [srcreg+32][r9]	;; MemI1
	vaddpd	ymm6, ymm6, ymm13		;; R2*MemI2+I2*MemR2 (new I2)	; 74-76

	vsubpd	ymm13, ymm2, ymm7		;; I3 - I4 (new R4)		; 67-69
	vaddpd	ymm2, ymm2, ymm7		;; I3 + I4 (new I3)		; 68-70

	vsubpd	ymm7, ymm12, ymm1		;; I5 - I6 (new I6)		; 69-71
	vaddpd	ymm12, ymm12, ymm1		;; I5 + I6 (new I5)		; 70-72

	vsubpd	ymm1, ymm5, ymm11		;; R5 - R6 (new R6)		; 71-73
	vaddpd	ymm5, ymm5, ymm11		;; R5 + R6 (new R5)		; 72-74

	vmovapd	ymm3, [srcreg][rbx]		;; R1
	vmulpd	ymm11, ymm3, ymm10		;; R1 * MemR1			; 70-74
	vmulpd	ymm3, ymm3, ymm9		;; R1 * MemI1			; 71-75
	vmulpd	ymm9, ymm9, [srcreg+32][rbx]	;; (load I1) * MemI1		; 72-76
	vmulpd	ymm10, ymm10, [srcreg+32][rbx]	;; (load I1) * MemR1		; 73-77

	vsubpd	ymm11, ymm11, ymm9		;; R1*MemR1-I1*MemI1 (new R1)	; 77-79
	vaddpd	ymm3, ymm3, ymm10		;; R1*MemI1+I1*MemR1 (new I1)	; 78-80
	jmp	done

fma_or_unfft:
	cmp	r10, 4				;; Case off opcode
	jbe	fma				;; 3=muladd, 4=mulsub, fma MUST NOT ALTER CONDITION CODES!
	;;ja	unfft				;; 5=unfft

unfft:	vmovapd	ymm10, [srcreg+d2][rbx]		;; R3
	vmovapd	ymm4, [srcreg+d2+d1][rbx]	;; R4

	vaddpd	ymm9, ymm4, ymm10		;; R4 + R3 (new R3)		; 65-67
	vsubpd	ymm4, ymm4, ymm10		;; R4 - R3 (new I4)		; 66-68

	vmovapd	ymm2, [srcreg+d2+32][rbx]	;; I3
	vmovapd	ymm7, [srcreg+d2+d1+32][rbx]	;; I4

	vsubpd	ymm13, ymm2, ymm7		;; I3 - I4 (new R4)		; 67-69
	vaddpd	ymm2, ymm2, ymm7		;; I3 + I4 (new I3)		; 68-70

	vmovapd	ymm12, [srcreg+d4+32][rbx]	;; I5
	vmovapd	ymm1, [srcreg+d4+d1+32][rbx]	;; I6

	vsubpd	ymm7, ymm12, ymm1		;; I5 - I6 (new I6)		; 69-71
	vaddpd	ymm12, ymm12, ymm1		;; I5 + I6 (new I5)		; 70-72

	vmovapd	ymm5, [srcreg+d4][rbx]		;; R5
	vmovapd	ymm11, [srcreg+d4+d1][rbx]	;; R6

	vsubpd	ymm1, ymm5, ymm11		;; R5 - R6 (new R6)		; 71-73
	vaddpd	ymm5, ymm5, ymm11		;; R5 + R6 (new R5)		; 72-74

	ystore	[srcreg+d2], ymm9		;; Temporarily save new R3	; 68
	ystore	[srcreg+d2+32], ymm4		;; Temporarily save new I4	; 69

	vmovapd	ymm11, [srcreg][rbx]		;; R1
	vmovapd	ymm3, [srcreg+32][rbx]		;; I1
	vmovapd	ymm4, [srcreg+d1][rbx]		;; R2
	vmovapd	ymm6, [srcreg+d1+32][rbx]	;; I2
	vmovapd	ymm8, [srcreg+d4+d2][rbx]	;; R7
	vmovapd	ymm0, [srcreg+d4+d2+32][rbx]	;; I7
	vmovapd	ymm15, [srcreg+d4+d2+d1][rbx]	;; R8
	vmovapd	ymm14, [srcreg+d4+d2+d1+32][rbx];; I8
	jmp	done

fma:	vmovapd	ymm3, [srcreg+d2+d1][rbp]	;; MemR4
	vmovapd	ymm9, [srcreg+d2+d1+32][rbp]	;; MemI4
	vmovapd	ymm7, [srcreg+d2+d1][rbx]	;; R4
	vmovapd	ymm10, [srcreg+d2+d1+32][rbx]	;; I4
	vmulpd	ymm4, ymm7, ymm3		;; R4 * MemR4			; 32-36
	vmulpd	ymm7, ymm7, ymm9		;; R4 * MemI4			; 33-37
	vmulpd	ymm9, ymm10, ymm9		;; I4 * MemI4			; 34-38
	vmulpd	ymm10, ymm10, ymm3		;; I4 * MemR4			; 35-39

	vmovapd	ymm3, [srcreg+d2][rbp]		;; MemR3
	vsubpd	ymm4, ymm4, ymm9		;; R4*MemR4-I4*MemI4 (new R4)	; 51-53
	vmovapd	ymm9, [srcreg+d2+32][rbp]	;; MemI3
	vaddpd	ymm7, ymm7, ymm10		;; R4*MemI4+I4*MemR4 (new I4)	; 52-54

	vmovapd	ymm2, [srcreg+d2][rbx]		;; R3
	vmovapd	ymm11, [srcreg+d2+32][rbx]	;; I3
	vmulpd	ymm10, ymm2, ymm3		;; R3 * MemR3			; 36-40
	vmulpd	ymm2, ymm2, ymm9		;; R3 * MemI3			; 37-41
	vmulpd	ymm9, ymm11, ymm9		;; I3 * MemI3			; 38-42
	vmulpd	ymm11, ymm11, ymm3		;; I3 * MemR3			; 39-43

	vmovapd	ymm3, [srcreg+d4+d1][rbp]	;; MemR6
	vsubpd	ymm10, ymm10, ymm9		;; R3*MemR3-I3*MemI3 (new R3)	; 53-55
	vmovapd	ymm9, [srcreg+d4+d1+32][rbp]	;; MemI6
	vaddpd	ymm2, ymm2, ymm11		;; R3*MemI3+I3*MemR3 (new I3)	; 54-56

	vmovapd	ymm1, [srcreg+d4+d1][rbx]	;; R6
	vmovapd	ymm5, [srcreg+d4+d1+32][rbx]	;; I6
	vmulpd	ymm11, ymm1, ymm3		;; R6 * MemR6			; 42-46
	vmulpd	ymm1, ymm1, ymm9		;; R6 * MemI6			; 43-47
	vmulpd	ymm9, ymm5, ymm9		;; I6 * MemI6			; 46-50
	vmulpd	ymm5, ymm5, ymm3		;; I6 * MemR6			; 47-51

	vmovapd	ymm3, [srcreg+d4][rbp]		;; MemR5
	vsubpd	ymm11, ymm11, ymm9		;; R6*MemR6-I6*MemI6 (new R6)	; 55-57
	vmovapd	ymm9, [srcreg+d4+32][rbp]	;; MemI5
	vaddpd	ymm1, ymm1, ymm5		;; R6*MemI6+I6*MemR6 (new I6)	; 56-58

	vmovapd	ymm12, [srcreg+d4][rbx]		;; R5
	vmovapd	ymm15, [srcreg+d4+32][rbx]	;; I5
	vmulpd	ymm5, ymm12, ymm3		;; R5 * MemR5			; 44-48
	vmulpd	ymm12, ymm12, ymm9		;; R5 * MemI5			; 45-49
	vmulpd	ymm9, ymm15, ymm9		;; I5 * MemI5			; 48-52
	vmulpd	ymm15, ymm15, ymm3		;; I5 * MemR5			; 49-53

	vmovapd	ymm3, [srcreg+d4+d2+d1][rbp]	;; MemR8
	vsubpd	ymm5, ymm5, ymm9		;; R5*MemR5-I5*MemI5 (new R5)	; 57-59
	vmovapd	ymm9, [srcreg+d4+d2+d1+32][rbp]	;; MemI8
	vaddpd	ymm12, ymm12, ymm15		;; R5*MemI5+I5*MemR5 (new I5)	; 58-60

	vmovapd	ymm14, [srcreg+d4+d2+d1][rbx]	;; R8
	vmovapd	ymm8, [srcreg+d4+d2+d1+32][rbx]	;; I8
	vmulpd	ymm15, ymm14, ymm3		;; R8 * MemR8			; 50-54
	vmulpd	ymm14, ymm14, ymm9		;; R8 * MemI8			; 51-55
	vmulpd	ymm9, ymm8, ymm9		;; I8 * MemI8			; 52-56
	vmulpd	ymm8, ymm8, ymm3		;; I8 * MemR8			; 53-57

	vmovapd	ymm3, [srcreg+d4+d2][rbp]	;; MemR7
	vsubpd	ymm15, ymm15, ymm9		;; R8*MemR8-I8*MemI8 (new R8)	; 59-61
	vmovapd	ymm9, [srcreg+d4+d2+32][rbp]	;; MemI7
	vaddpd	ymm14, ymm14, ymm8		;; R8*MemI8+I8*MemR8 (new I8)	; 60-62

	vmovapd	ymm0, [srcreg+d4+d2][rbx]	;; R7
	vmovapd	ymm6, [srcreg+d4+d2+32][rbx]	;; I7
	vmulpd	ymm8, ymm0, ymm3		;; R7 * MemR7			; 54-58
	vmulpd	ymm0, ymm0, ymm9		;; R7 * MemI7			; 55-59
	vmulpd	ymm9, ymm6, ymm9		;; I7 * MemI7			; 56-60
	vmulpd	ymm6, ymm6, ymm3		;; I7 * MemR7			; 57-61

	vsubpd	ymm8, ymm8, ymm9		;; R7*MemR7-I7*MemI7 (new R7)	; 61-63
	vaddpd	ymm0, ymm0, ymm6		;; R7*MemI7+I7*MemR7 (new I7)	; 62-64

	mov	r10, SRC3ARG			;; Load distance to FFT(1)
	je	mulsub				;; 4=mulsub
	;jb	muladd				;; 3=muladd, fall through

muladd:	test	r10, 3				;; Check if we need to multiply add-in by FFT(1)
	jz	muladdhard			;; Yes, do it the hard way

	vaddpd	ymm4, ymm4, [srcreg+r9+d2+d1]		;; R4 = R4 + MemR4
	vaddpd	ymm7, ymm7, [srcreg+r9+d2+d1+32]	;; I4 = I4 + MemI4
	vaddpd	ymm10, ymm10, [srcreg+r9+d2]		;; R3 = R3 + MemR3
	vaddpd	ymm2, ymm2, [srcreg+r9+d2+32]		;; I3 = I3 + MemI3
	vaddpd	ymm11, ymm11, [srcreg+r9+d4+d1]		;; R6 = R6 + MemR6
	vaddpd	ymm1, ymm1, [srcreg+r9+d4+d1+32]	;; I6 = I6 + MemI6
	vaddpd	ymm5, ymm5, [srcreg+r9+d4]		;; R5 = R5 + MemR5
	vaddpd	ymm12, ymm12, [srcreg+r9+d4+32]		;; I5 = I5 + MemI5
	vaddpd	ymm15, ymm15, [srcreg+r9+d4+d2+d1]	;; R8 = R8 + MemR8
	vaddpd	ymm14, ymm14, [srcreg+r9+d4+d2+d1+32]	;; I8 = I8 + MemI8
	vaddpd	ymm8, ymm8, [srcreg+r9+d4+d2]		;; R7 = R7 + MemR7
	vaddpd	ymm0, ymm0, [srcreg+r9+d4+d2+32]	;; I7 = I7 + MemI7

	vaddpd	ymm9, ymm4, ymm10		;; R4 + R3 (new R3)		; 65-67
	vsubpd	ymm4, ymm4, ymm10		;; R4 - R3 (new I4)		; 66-68

	vmovapd	ymm10, [srcreg+d1][rbp]		;; MemR2
	ystore	[srcreg+d2], ymm9		;; Temporarily save new R3	; 68
	vmovapd	ymm9, [srcreg+d1+32][rbp]	;; MemI2
	ystore	[srcreg+d2+32], ymm4		;; Temporarily save new I4	; 69
	vmovapd	ymm6, [srcreg+d1][rbx]		;; R2
	vmovapd	ymm13, [srcreg+d1+32][rbx]	;; I2
	vmulpd	ymm4, ymm6, ymm10		;; R2 * MemR2			; 66-70
	vmulpd	ymm6, ymm6, ymm9		;; R2 * MemI2			; 67-71
	vmulpd	ymm9, ymm13, ymm9		;; I2 * MemI2			; 68-72
	vmulpd	ymm13, ymm13, ymm10		;; I2 * MemR2			; 69-73

	vmovapd	ymm10, [srcreg][rbp]		;; MemR1
	vsubpd	ymm4, ymm4, ymm9		;; R2*MemR2-I2*MemI2 (new R2)	; 73-75
	vmovapd	ymm9, [srcreg+32][rbp]		;; MemI1
	vaddpd	ymm6, ymm6, ymm13		;; R2*MemI2+I2*MemR2 (new I2)	; 74-76

	vsubpd	ymm13, ymm2, ymm7		;; I3 - I4 (new R4)		; 67-69
	vaddpd	ymm2, ymm2, ymm7		;; I3 + I4 (new I3)		; 68-70

	vsubpd	ymm7, ymm12, ymm1		;; I5 - I6 (new I6)		; 69-71
	vaddpd	ymm12, ymm12, ymm1		;; I5 + I6 (new I5)		; 70-72

	vsubpd	ymm1, ymm5, ymm11		;; R5 - R6 (new R6)		; 71-73
	vaddpd	ymm5, ymm5, ymm11		;; R5 + R6 (new R5)		; 72-74

	vmovapd	ymm3, [srcreg][rbx]		;; R1
	vmulpd	ymm11, ymm3, ymm10		;; R1 * MemR1			; 70-74
	vmulpd	ymm3, ymm3, ymm9		;; R1 * MemI1			; 71-75
	vmulpd	ymm9, ymm9, [srcreg+32][rbx]	;; (load I1) * MemI1		; 72-76
	vmulpd	ymm10, ymm10, [srcreg+32][rbx]	;; (load I1) * MemR1		; 73-77

	vsubpd	ymm11, ymm11, ymm9		;; R1*MemR1-I1*MemI1 (new R1)	; 77-79
	vaddpd	ymm3, ymm3, ymm10		;; R1*MemI1+I1*MemR1 (new I1)	; 78-80

	vaddpd	ymm11, ymm11, [srcreg+r9]		;; R1 = R1 + MemR1
	vaddpd	ymm3, ymm3, [srcreg+r9+32]		;; I1 = I1 + MemI1
	vaddpd	ymm4, ymm4, [srcreg+r9+d1]		;; R2 = R2 + MemR2
	vaddpd	ymm6, ymm6, [srcreg+r9+d1+32]		;; I2 = I2 + MemI2
	jmp	done

muladdhard:
	vmovapd	ymm3, [srcreg+d2+d1][rbp]	;; MemR4
	vmovapd	ymm9, [srcreg+d2+d1+32][rbp]	;; MemI4
	vmovapd	ymm7, [srcreg+d2+d1][rbx]	;; R4
	vmovapd	ymm10, [srcreg+d2+d1+32][rbx]	;; I4
	vmulpd	ymm4, ymm7, ymm3		;; R4 * MemR4			; 32-36
	vmulpd	ymm7, ymm7, ymm9		;; R4 * MemI4			; 33-37
	vmulpd	ymm9, ymm10, ymm9		;; I4 * MemI4			; 34-38
	vmulpd	ymm10, ymm10, ymm3		;; I4 * MemR4			; 35-39

	vmovapd	ymm3, [srcreg+d2][rbp]		;; MemR3
	vsubpd	ymm4, ymm4, ymm9		;; R4*MemR4-I4*MemI4 (new R4)	; 51-53
	vmovapd	ymm9, [srcreg+d2+32][rbp]	;; MemI3
	vaddpd	ymm7, ymm7, ymm10		;; R4*MemI4+I4*MemR4 (new I4)	; 52-54

	vmovapd	ymm2, [srcreg+d2][rbx]		;; R3
	vmovapd	ymm11, [srcreg+d2+32][rbx]	;; I3
	vmulpd	ymm10, ymm2, ymm3		;; R3 * MemR3			; 36-40
	vmulpd	ymm2, ymm2, ymm9		;; R3 * MemI3			; 37-41
	vmulpd	ymm9, ymm11, ymm9		;; I3 * MemI3			; 38-42
	vmulpd	ymm11, ymm11, ymm3		;; I3 * MemR3			; 39-43

	vmovapd	ymm3, [srcreg+d4+d1][rbp]	;; MemR6
	vsubpd	ymm10, ymm10, ymm9		;; R3*MemR3-I3*MemI3 (new R3)	; 53-55
	vmovapd	ymm9, [srcreg+d4+d1+32][rbp]	;; MemI6
	vaddpd	ymm2, ymm2, ymm11		;; R3*MemI3+I3*MemR3 (new I3)	; 54-56

	vmovapd	ymm1, [srcreg+d4+d1][rbx]	;; R6
	vmovapd	ymm5, [srcreg+d4+d1+32][rbx]	;; I6
	vmulpd	ymm11, ymm1, ymm3		;; R6 * MemR6			; 42-46
	vmulpd	ymm1, ymm1, ymm9		;; R6 * MemI6			; 43-47
	vmulpd	ymm9, ymm5, ymm9		;; I6 * MemI6			; 46-50
	vmulpd	ymm5, ymm5, ymm3		;; I6 * MemR6			; 47-51

	vmovapd	ymm3, [srcreg+d4][rbp]		;; MemR5
	vsubpd	ymm11, ymm11, ymm9		;; R6*MemR6-I6*MemI6 (new R6)	; 55-57
	vmovapd	ymm9, [srcreg+d4+32][rbp]	;; MemI5
	vaddpd	ymm1, ymm1, ymm5		;; R6*MemI6+I6*MemR6 (new I6)	; 56-58

	vmovapd	ymm12, [srcreg+d4][rbx]		;; R5
	vmovapd	ymm15, [srcreg+d4+32][rbx]	;; I5
	vmulpd	ymm5, ymm12, ymm3		;; R5 * MemR5			; 44-48
	vmulpd	ymm12, ymm12, ymm9		;; R5 * MemI5			; 45-49
	vmulpd	ymm9, ymm15, ymm9		;; I5 * MemI5			; 48-52
	vmulpd	ymm15, ymm15, ymm3		;; I5 * MemR5			; 49-53

	vmovapd	ymm3, [srcreg+d4+d2+d1][rbp]	;; MemR8
	vsubpd	ymm5, ymm5, ymm9		;; R5*MemR5-I5*MemI5 (new R5)	; 57-59
	vmovapd	ymm9, [srcreg+d4+d2+d1+32][rbp]	;; MemI8
	vaddpd	ymm12, ymm12, ymm15		;; R5*MemI5+I5*MemR5 (new I5)	; 58-60

	vmovapd	ymm14, [srcreg+d4+d2+d1][rbx]	;; R8
	vmovapd	ymm8, [srcreg+d4+d2+d1+32][rbx]	;; I8
	vmulpd	ymm15, ymm14, ymm3		;; R8 * MemR8			; 50-54
	vmulpd	ymm14, ymm14, ymm9		;; R8 * MemI8			; 51-55
	vmulpd	ymm9, ymm8, ymm9		;; I8 * MemI8			; 52-56
	vmulpd	ymm8, ymm8, ymm3		;; I8 * MemR8			; 53-57

	vmovapd	ymm3, [srcreg+d4+d2][rbp]	;; MemR7
	vsubpd	ymm15, ymm15, ymm9		;; R8*MemR8-I8*MemI8 (new R8)	; 59-61
	vmovapd	ymm9, [srcreg+d4+d2+32][rbp]	;; MemI7
	vaddpd	ymm14, ymm14, ymm8		;; R8*MemI8+I8*MemR8 (new I8)	; 60-62

	vmovapd	ymm0, [srcreg+d4+d2][rbx]	;; R7
	vmovapd	ymm6, [srcreg+d4+d2+32][rbx]	;; I7
	vmulpd	ymm8, ymm0, ymm3		;; R7 * MemR7			; 54-58
	vmulpd	ymm0, ymm0, ymm9		;; R7 * MemI7			; 55-59
	vmulpd	ymm9, ymm6, ymm9		;; I7 * MemI7			; 56-60
	vmulpd	ymm6, ymm6, ymm3		;; I7 * MemR7			; 57-61

	vsubpd	ymm8, ymm8, ymm9		;; R7*MemR7-I7*MemI7 (new R7)	; 61-63
	vaddpd	ymm0, ymm0, ymm6		;; R7*MemI7+I7*MemR7 (new I7)	; 62-64

	vmovapd	ymm3, [srcreg+r9+d2+d1]			;; MemR4
	vmovapd	ymm9, [srcreg+r10+d2+d1]		;; MemR4#2
	vmulpd	ymm13, ymm3, ymm9			;; MemR4*MemR4#2
	vaddpd	ymm4, ymm4, ymm13			;; R4 = R4 + MemR4*MemR4#2
	vmovapd	ymm6, [srcreg+r10+d2+d1+32]		;; MemI4#2
	vmulpd	ymm13, ymm3, ymm6			;; MemR4*MemI4#2
	vaddpd	ymm7, ymm7, ymm13			;; I4 = I4 + MemR4*MemI4#2
	vmovapd	ymm3, [srcreg+r9+d2+d1+32]		;; MemI4
	vmulpd	ymm13, ymm3, ymm6			;; MemI4*MemI4#2
	vsubpd	ymm4, ymm4, ymm13			;; R4 = R4 - MemI4*MemI4#2
	vmulpd	ymm13, ymm3, ymm9			;; MemI4*MemR4#2
	vaddpd	ymm7, ymm7, ymm13			;; I4 = I4 + MemI4*MemR4#2

	vmovapd	ymm3, [srcreg+r9+d2]			;; MemR3
	vmovapd	ymm9, [srcreg+r10+d2]			;; MemR3#2
	vmulpd	ymm13, ymm3, ymm9			;; MemR3*MemR3#2
	vaddpd	ymm10, ymm10, ymm13			;; R3 = R3 + MemR3*MemR3#2
	vmovapd	ymm6, [srcreg+r10+d2+32]		;; MemI3#2
	vmulpd	ymm13, ymm3, ymm6			;; MemR3*MemI3#2
	vaddpd	ymm2, ymm2, ymm13			;; I3 = I3 + MemR3*MemI3#2
	vmovapd	ymm3, [srcreg+r9+d2+32]			;; MemI3
	vmulpd	ymm13, ymm3, ymm6			;; MemI3*MemI3#2
	vsubpd	ymm10, ymm10, ymm13			;; R3 = R3 - MemI3*MemI3#2
	vmulpd	ymm13, ymm3, ymm9			;; MemI3*MemR3#2
	vaddpd	ymm2, ymm2, ymm13			;; I3 = I3 + MemI3*MemR3#2

	vmovapd	ymm3, [srcreg+r9+d4+d1]			;; MemR6
	vmovapd	ymm9, [srcreg+r10+d4+d1]		;; MemR6#2
	vmulpd	ymm13, ymm3, ymm9			;; MemR6*MemR6#2
	vaddpd	ymm11, ymm11, ymm13			;; R6 = R6 + MemR6*MemR6#2
	vmovapd	ymm6, [srcreg+r10+d4+d1+32]		;; MemI6#2
	vmulpd	ymm13, ymm3, ymm6			;; MemR6*MemI6#2
	vaddpd	ymm1, ymm1, ymm13			;; I6 = I6 + MemR6*MemI6#2
	vmovapd	ymm3, [srcreg+r9+d4+d1+32]		;; MemI6
	vmulpd	ymm13, ymm3, ymm6			;; MemI6*MemI6#2
	vsubpd	ymm11, ymm11, ymm13			;; R6 = R6 - MemI6*MemI6#2
	vmulpd	ymm13, ymm3, ymm9			;; MemI6*MemR6#2
	vaddpd	ymm1, ymm1, ymm13			;; I6 = I6 + MemI6*MemR6#2

	vmovapd	ymm3, [srcreg+r9+d4]			;; MemR5
	vmovapd	ymm9, [srcreg+r10+d4]			;; MemR5#2
	vmulpd	ymm13, ymm3, ymm9			;; MemR5*MemR5#2
	vaddpd	ymm5, ymm5, ymm13			;; R5 = R5 + MemR5*MemR5#2
	vmovapd	ymm6, [srcreg+r10+d4+32]		;; MemI5#2
	vmulpd	ymm13, ymm3, ymm6			;; MemR5*MemI5#2
	vaddpd	ymm12, ymm12, ymm13			;; I5 = I5 + MemR5*MemI5#2
	vmovapd	ymm3, [srcreg+r9+d4+32]			;; MemI5
	vmulpd	ymm13, ymm3, ymm6			;; MemI5*MemI5#2
	vsubpd	ymm5, ymm5, ymm13			;; R5 = R5 - MemI5*MemI5#2
	vmulpd	ymm13, ymm3, ymm9			;; MemI5*MemR5#2
	vaddpd	ymm12, ymm12, ymm13			;; I5 = I5 + MemI5*MemR5#2

	vmovapd	ymm3, [srcreg+r9+d4+d2+d1]		;; MemR8
	vmovapd	ymm9, [srcreg+r10+d4+d2+d1]		;; MemR8#2
	vmulpd	ymm13, ymm3, ymm9			;; MemR8*MemR8#2
	vaddpd	ymm15, ymm15, ymm13			;; R8 = R8 + MemR8*MemR8#2
	vmovapd	ymm6, [srcreg+r10+d4+d2+d1+32]		;; MemI8#2
	vmulpd	ymm13, ymm3, ymm6			;; MemR8*MemI8#2
	vaddpd	ymm14, ymm14, ymm13			;; I8 = I8 + MemR8*MemI8#2
	vmovapd	ymm3, [srcreg+r9+d4+d2+d1+32]		;; MemI8
	vmulpd	ymm13, ymm3, ymm6			;; MemI8*MemI8#2
	vsubpd	ymm15, ymm15, ymm13			;; R8 = R8 - MemI8*MemI8#2
	vmulpd	ymm13, ymm3, ymm9			;; MemI8*MemR8#2
	vaddpd	ymm14, ymm14, ymm13			;; I8 = I8 + MemI8*MemR8#2

	vmovapd	ymm3, [srcreg+r9+d4+d2]			;; MemR7
	vmovapd	ymm9, [srcreg+r10+d4+d2]		;; MemR7#2
	vmulpd	ymm13, ymm3, ymm9			;; MemR7*MemR7#2
	vaddpd	ymm8, ymm8, ymm13			;; R7 = R7 + MemR7*MemR7#2
	vmovapd	ymm6, [srcreg+r10+d4+d2+32]		;; MemI7#2
	vmulpd	ymm13, ymm3, ymm6			;; MemR7*MemI7#2
	vaddpd	ymm0, ymm0, ymm13			;; I7 = I7 + MemR7*MemI7#2
	vmovapd	ymm3, [srcreg+r9+d4+d2+32]		;; MemI7
	vmulpd	ymm13, ymm3, ymm6			;; MemI7*MemI7#2
	vsubpd	ymm8, ymm8, ymm13			;; R7 = R7 - MemI7*MemI7#2
	vmulpd	ymm13, ymm3, ymm9			;; MemI7*MemR7#2
	vaddpd	ymm0, ymm0, ymm13			;; I7 = I7 + MemI7*MemR7#2

	vaddpd	ymm9, ymm4, ymm10		;; R4 + R3 (new R3)		; 65-67
	vsubpd	ymm4, ymm4, ymm10		;; R4 - R3 (new I4)		; 66-68

	vmovapd	ymm10, [srcreg+d1][rbp]		;; MemR2
	ystore	[srcreg+d2], ymm9		;; Temporarily save new R3	; 68
	vmovapd	ymm9, [srcreg+d1+32][rbp]	;; MemI2
	ystore	[srcreg+d2+32], ymm4		;; Temporarily save new I4	; 69
	vmovapd	ymm6, [srcreg+d1][rbx]		;; R2
	vmovapd	ymm13, [srcreg+d1+32][rbx]	;; I2
	vmulpd	ymm4, ymm6, ymm10		;; R2 * MemR2			; 66-70
	vmulpd	ymm6, ymm6, ymm9		;; R2 * MemI2			; 67-71
	vmulpd	ymm9, ymm13, ymm9		;; I2 * MemI2			; 68-72
	vmulpd	ymm13, ymm13, ymm10		;; I2 * MemR2			; 69-73

	vmovapd	ymm10, [srcreg][rbp]		;; MemR1
	vsubpd	ymm4, ymm4, ymm9		;; R2*MemR2-I2*MemI2 (new R2)	; 73-75
	vmovapd	ymm9, [srcreg+32][rbp]		;; MemI1
	vaddpd	ymm6, ymm6, ymm13		;; R2*MemI2+I2*MemR2 (new I2)	; 74-76

	vsubpd	ymm13, ymm2, ymm7		;; I3 - I4 (new R4)		; 67-69
	vaddpd	ymm2, ymm2, ymm7		;; I3 + I4 (new I3)		; 68-70

	vsubpd	ymm7, ymm12, ymm1		;; I5 - I6 (new I6)		; 69-71
	vaddpd	ymm12, ymm12, ymm1		;; I5 + I6 (new I5)		; 70-72

	vsubpd	ymm1, ymm5, ymm11		;; R5 - R6 (new R6)		; 71-73
	vaddpd	ymm5, ymm5, ymm11		;; R5 + R6 (new R5)		; 72-74

	vmovapd	ymm3, [srcreg][rbx]		;; R1
	vmulpd	ymm11, ymm3, ymm10		;; R1 * MemR1			; 70-74
	vmulpd	ymm3, ymm3, ymm9		;; R1 * MemI1			; 71-75
	vmulpd	ymm9, ymm9, [srcreg+32][rbx]	;; (load I1) * MemI1		; 72-76
	vmulpd	ymm10, ymm10, [srcreg+32][rbx]	;; (load I1) * MemR1		; 73-77

	vsubpd	ymm11, ymm11, ymm9		;; R1*MemR1-I1*MemI1 (new R1)	; 77-79
	vaddpd	ymm3, ymm3, ymm10		;; R1*MemI1+I1*MemR1 (new I1)	; 78-80

	vmovapd	YMM_TMP1, ymm0				;; Save 2 registers
	vmovapd	YMM_TMP2, ymm12

	vmovapd	ymm0, [srcreg+r9+d1]			;; MemR2
	vmovapd	ymm9, [srcreg+r10+d1]			;; MemR2#2
	vmulpd	ymm12, ymm0, ymm9			;; MemR2*MemR2#2
	vaddpd	ymm4, ymm4, ymm12			;; R2 = R2 + MemR2*MemR2#2
	vmovapd	ymm10, [srcreg+r10+d1+32]		;; MemI2#2
	vmulpd	ymm12, ymm0, ymm10			;; MemR2*MemI2#2
	vaddpd	ymm6, ymm6, ymm12			;; I2 = I2 + MemR2*MemI2#2
	vmovapd	ymm0, [srcreg+r9+d1+32]			;; MemI2
	vmulpd	ymm12, ymm0, ymm10			;; MemI2*MemI2#2
	vsubpd	ymm4, ymm4, ymm12			;; R2 = R2 - MemI2*MemI2#2
	vmulpd	ymm12, ymm0, ymm9			;; MemI2*MemR2#2
	vaddpd	ymm6, ymm6, ymm12			;; I2 = I2 + MemI2*MemR2#2

	vmovapd	ymm0, [srcreg+r9]			;; MemR1
	vmovapd	ymm9, [srcreg+r10]			;; MemR1#2
	vmulpd	ymm12, ymm0, ymm9			;; MemR1*MemR1#2
	vaddpd	ymm11, ymm11, ymm12			;; R1 = R1 + MemR1*MemR1#2
	vmovapd	ymm10, [srcreg+r10+32]			;; MemI1#2
	vmulpd	ymm12, ymm0, ymm10			;; MemR1*MemI1#2
	vaddpd	ymm3, ymm3, ymm12			;; I1 = I1 + MemR1*MemI1#2
	vmovapd	ymm0, [srcreg+r9+32]			;; MemI1
	vmulpd	ymm12, ymm0, ymm10			;; MemI1*MemI1#2
	vsubpd	ymm11, ymm11, ymm12			;; R1 = R1 - MemI1*MemI1#2
	vmulpd	ymm12, ymm0, ymm9			;; MemI1*MemR1#2
	vaddpd	ymm3, ymm3, ymm12			;; I1 = I1 + MemI1*MemR1#2

	vmovapd	ymm0, YMM_TMP1				;; Restore 2 registers
	vmovapd	ymm12, YMM_TMP2
	jmp	done

mulsub:	test	r10, 3				;; Check if we need to multiply add-in by FFT(1)
	jz	mulsubhard			;; Yes, do it the hard way

	vsubpd	ymm4, ymm4, [srcreg+r9+d2+d1]		;; R4 = R4 - MemR4
	vsubpd	ymm7, ymm7, [srcreg+r9+d2+d1+32]	;; I4 = I4 - MemI4
	vsubpd	ymm10, ymm10, [srcreg+r9+d2]		;; R3 = R3 - MemR3
	vsubpd	ymm2, ymm2, [srcreg+r9+d2+32]		;; I3 = I3 - MemI3
	vsubpd	ymm11, ymm11, [srcreg+r9+d4+d1]		;; R6 = R6 - MemR6
	vsubpd	ymm1, ymm1, [srcreg+r9+d4+d1+32]	;; I6 = I6 - MemI6
	vsubpd	ymm5, ymm5, [srcreg+r9+d4]		;; R5 = R5 - MemR5
	vsubpd	ymm12, ymm12, [srcreg+r9+d4+32]		;; I5 = I5 - MemI5
	vsubpd	ymm15, ymm15, [srcreg+r9+d4+d2+d1]	;; R8 = R8 - MemR8
	vsubpd	ymm14, ymm14, [srcreg+r9+d4+d2+d1+32]	;; I8 = I8 - MemI8
	vsubpd	ymm8, ymm8, [srcreg+r9+d4+d2]		;; R7 = R7 - MemR7
	vsubpd	ymm0, ymm0, [srcreg+r9+d4+d2+32]	;; I7 = I7 - MemI7

	vaddpd	ymm9, ymm4, ymm10		;; R4 + R3 (new R3)		; 65-67
	vsubpd	ymm4, ymm4, ymm10		;; R4 - R3 (new I4)		; 66-68

	vmovapd	ymm10, [srcreg+d1][rbp]		;; MemR2
	ystore	[srcreg+d2], ymm9		;; Temporarily save new R3	; 68
	vmovapd	ymm9, [srcreg+d1+32][rbp]	;; MemI2
	ystore	[srcreg+d2+32], ymm4		;; Temporarily save new I4	; 69
	vmovapd	ymm6, [srcreg+d1][rbx]		;; R2
	vmovapd	ymm13, [srcreg+d1+32][rbx]	;; I2
	vmulpd	ymm4, ymm6, ymm10		;; R2 * MemR2			; 66-70
	vmulpd	ymm6, ymm6, ymm9		;; R2 * MemI2			; 67-71
	vmulpd	ymm9, ymm13, ymm9		;; I2 * MemI2			; 68-72
	vmulpd	ymm13, ymm13, ymm10		;; I2 * MemR2			; 69-73

	vmovapd	ymm10, [srcreg][rbp]		;; MemR1
	vsubpd	ymm4, ymm4, ymm9		;; R2*MemR2-I2*MemI2 (new R2)	; 73-75
	vmovapd	ymm9, [srcreg+32][rbp]		;; MemI1
	vaddpd	ymm6, ymm6, ymm13		;; R2*MemI2+I2*MemR2 (new I2)	; 74-76

	vsubpd	ymm13, ymm2, ymm7		;; I3 - I4 (new R4)		; 67-69
	vaddpd	ymm2, ymm2, ymm7		;; I3 + I4 (new I3)		; 68-70

	vsubpd	ymm7, ymm12, ymm1		;; I5 - I6 (new I6)		; 69-71
	vaddpd	ymm12, ymm12, ymm1		;; I5 + I6 (new I5)		; 70-72

	vsubpd	ymm1, ymm5, ymm11		;; R5 - R6 (new R6)		; 71-73
	vaddpd	ymm5, ymm5, ymm11		;; R5 + R6 (new R5)		; 72-74

	vmovapd	ymm3, [srcreg][rbx]		;; R1
	vmulpd	ymm11, ymm3, ymm10		;; R1 * MemR1			; 70-74
	vmulpd	ymm3, ymm3, ymm9		;; R1 * MemI1			; 71-75
	vmulpd	ymm9, ymm9, [srcreg+32][rbx]	;; (load I1) * MemI1		; 72-76
	vmulpd	ymm10, ymm10, [srcreg+32][rbx]	;; (load I1) * MemR1		; 73-77

	vsubpd	ymm11, ymm11, ymm9		;; R1*MemR1-I1*MemI1 (new R1)	; 77-79
	vaddpd	ymm3, ymm3, ymm10		;; R1*MemI1+I1*MemR1 (new I1)	; 78-80

	vsubpd	ymm11, ymm11, [srcreg+r9]		;; R1 = R1 - MemR1
	vsubpd	ymm3, ymm3, [srcreg+r9+32]		;; I1 = I1 - MemI1
	vsubpd	ymm4, ymm4, [srcreg+r9+d1]		;; R2 = R2 - MemR2
	vsubpd	ymm6, ymm6, [srcreg+r9+d1+32]		;; I2 = I2 - MemI2
	jmp	done

mulsubhard:
	vmovapd	ymm3, [srcreg+d2+d1][rbp]	;; MemR4
	vmovapd	ymm9, [srcreg+d2+d1+32][rbp]	;; MemI4
	vmovapd	ymm7, [srcreg+d2+d1][rbx]	;; R4
	vmovapd	ymm10, [srcreg+d2+d1+32][rbx]	;; I4
	vmulpd	ymm4, ymm7, ymm3		;; R4 * MemR4			; 32-36
	vmulpd	ymm7, ymm7, ymm9		;; R4 * MemI4			; 33-37
	vmulpd	ymm9, ymm10, ymm9		;; I4 * MemI4			; 34-38
	vmulpd	ymm10, ymm10, ymm3		;; I4 * MemR4			; 35-39

	vmovapd	ymm3, [srcreg+d2][rbp]		;; MemR3
	vsubpd	ymm4, ymm4, ymm9		;; R4*MemR4-I4*MemI4 (new R4)	; 51-53
	vmovapd	ymm9, [srcreg+d2+32][rbp]	;; MemI3
	vaddpd	ymm7, ymm7, ymm10		;; R4*MemI4+I4*MemR4 (new I4)	; 52-54

	vmovapd	ymm2, [srcreg+d2][rbx]		;; R3
	vmovapd	ymm11, [srcreg+d2+32][rbx]	;; I3
	vmulpd	ymm10, ymm2, ymm3		;; R3 * MemR3			; 36-40
	vmulpd	ymm2, ymm2, ymm9		;; R3 * MemI3			; 37-41
	vmulpd	ymm9, ymm11, ymm9		;; I3 * MemI3			; 38-42
	vmulpd	ymm11, ymm11, ymm3		;; I3 * MemR3			; 39-43

	vmovapd	ymm3, [srcreg+d4+d1][rbp]	;; MemR6
	vsubpd	ymm10, ymm10, ymm9		;; R3*MemR3-I3*MemI3 (new R3)	; 53-55
	vmovapd	ymm9, [srcreg+d4+d1+32][rbp]	;; MemI6
	vaddpd	ymm2, ymm2, ymm11		;; R3*MemI3+I3*MemR3 (new I3)	; 54-56

	vmovapd	ymm1, [srcreg+d4+d1][rbx]	;; R6
	vmovapd	ymm5, [srcreg+d4+d1+32][rbx]	;; I6
	vmulpd	ymm11, ymm1, ymm3		;; R6 * MemR6			; 42-46
	vmulpd	ymm1, ymm1, ymm9		;; R6 * MemI6			; 43-47
	vmulpd	ymm9, ymm5, ymm9		;; I6 * MemI6			; 46-50
	vmulpd	ymm5, ymm5, ymm3		;; I6 * MemR6			; 47-51

	vmovapd	ymm3, [srcreg+d4][rbp]		;; MemR5
	vsubpd	ymm11, ymm11, ymm9		;; R6*MemR6-I6*MemI6 (new R6)	; 55-57
	vmovapd	ymm9, [srcreg+d4+32][rbp]	;; MemI5
	vaddpd	ymm1, ymm1, ymm5		;; R6*MemI6+I6*MemR6 (new I6)	; 56-58

	vmovapd	ymm12, [srcreg+d4][rbx]		;; R5
	vmovapd	ymm15, [srcreg+d4+32][rbx]	;; I5
	vmulpd	ymm5, ymm12, ymm3		;; R5 * MemR5			; 44-48
	vmulpd	ymm12, ymm12, ymm9		;; R5 * MemI5			; 45-49
	vmulpd	ymm9, ymm15, ymm9		;; I5 * MemI5			; 48-52
	vmulpd	ymm15, ymm15, ymm3		;; I5 * MemR5			; 49-53

	vmovapd	ymm3, [srcreg+d4+d2+d1][rbp]	;; MemR8
	vsubpd	ymm5, ymm5, ymm9		;; R5*MemR5-I5*MemI5 (new R5)	; 57-59
	vmovapd	ymm9, [srcreg+d4+d2+d1+32][rbp]	;; MemI8
	vaddpd	ymm12, ymm12, ymm15		;; R5*MemI5+I5*MemR5 (new I5)	; 58-60

	vmovapd	ymm14, [srcreg+d4+d2+d1][rbx]	;; R8
	vmovapd	ymm8, [srcreg+d4+d2+d1+32][rbx]	;; I8
	vmulpd	ymm15, ymm14, ymm3		;; R8 * MemR8			; 50-54
	vmulpd	ymm14, ymm14, ymm9		;; R8 * MemI8			; 51-55
	vmulpd	ymm9, ymm8, ymm9		;; I8 * MemI8			; 52-56
	vmulpd	ymm8, ymm8, ymm3		;; I8 * MemR8			; 53-57

	vmovapd	ymm3, [srcreg+d4+d2][rbp]	;; MemR7
	vsubpd	ymm15, ymm15, ymm9		;; R8*MemR8-I8*MemI8 (new R8)	; 59-61
	vmovapd	ymm9, [srcreg+d4+d2+32][rbp]	;; MemI7
	vaddpd	ymm14, ymm14, ymm8		;; R8*MemI8+I8*MemR8 (new I8)	; 60-62

	vmovapd	ymm0, [srcreg+d4+d2][rbx]	;; R7
	vmovapd	ymm6, [srcreg+d4+d2+32][rbx]	;; I7
	vmulpd	ymm8, ymm0, ymm3		;; R7 * MemR7			; 54-58
	vmulpd	ymm0, ymm0, ymm9		;; R7 * MemI7			; 55-59
	vmulpd	ymm9, ymm6, ymm9		;; I7 * MemI7			; 56-60
	vmulpd	ymm6, ymm6, ymm3		;; I7 * MemR7			; 57-61

	vsubpd	ymm8, ymm8, ymm9		;; R7*MemR7-I7*MemI7 (new R7)	; 61-63
	vaddpd	ymm0, ymm0, ymm6		;; R7*MemI7+I7*MemR7 (new I7)	; 62-64

	vmovapd	ymm3, [srcreg+r9+d2+d1]			;; MemR4
	vmovapd	ymm9, [srcreg+r10+d2+d1]		;; MemR4#2
	vmulpd	ymm13, ymm3, ymm9			;; MemR4*MemR4#2
	vsubpd	ymm4, ymm4, ymm13			;; R4 = R4 - MemR4*MemR4#2
	vmovapd	ymm6, [srcreg+r10+d2+d1+32]		;; MemI4#2
	vmulpd	ymm13, ymm3, ymm6			;; MemR4*MemI4#2
	vsubpd	ymm7, ymm7, ymm13			;; I4 = I4 - MemR4*MemI4#2
	vmovapd	ymm3, [srcreg+r9+d2+d1+32]		;; MemI4
	vmulpd	ymm13, ymm3, ymm6			;; MemI4*MemI4#2
	vaddpd	ymm4, ymm4, ymm13			;; R4 = R4 + MemI4*MemI4#2
	vmulpd	ymm13, ymm3, ymm9			;; MemI4*MemR4#2
	vsubpd	ymm7, ymm7, ymm13			;; I4 = I4 - MemI4*MemR4#2

	vmovapd	ymm3, [srcreg+r9+d2]			;; MemR3
	vmovapd	ymm9, [srcreg+r10+d2]			;; MemR3#2
	vmulpd	ymm13, ymm3, ymm9			;; MemR3*MemR3#2
	vsubpd	ymm10, ymm10, ymm13			;; R3 = R3 - MemR3*MemR3#2
	vmovapd	ymm6, [srcreg+r10+d2+32]		;; MemI3#2
	vmulpd	ymm13, ymm3, ymm6			;; MemR3*MemI3#2
	vsubpd	ymm2, ymm2, ymm13			;; I3 = I3 - MemR3*MemI3#2
	vmovapd	ymm3, [srcreg+r9+d2+32]			;; MemI3
	vmulpd	ymm13, ymm3, ymm6			;; MemI3*MemI3#2
	vaddpd	ymm10, ymm10, ymm13			;; R3 = R3 + MemI3*MemI3#2
	vmulpd	ymm13, ymm3, ymm9			;; MemI3*MemR3#2
	vsubpd	ymm2, ymm2, ymm13			;; I3 = I3 - MemI3*MemR3#2

	vmovapd	ymm3, [srcreg+r9+d4+d1]			;; MemR6
	vmovapd	ymm9, [srcreg+r10+d4+d1]		;; MemR6#2
	vmulpd	ymm13, ymm3, ymm9			;; MemR6*MemR6#2
	vsubpd	ymm11, ymm11, ymm13			;; R6 = R6 - MemR6*MemR6#2
	vmovapd	ymm6, [srcreg+r10+d4+d1+32]		;; MemI6#2
	vmulpd	ymm13, ymm3, ymm6			;; MemR6*MemI6#2
	vsubpd	ymm1, ymm1, ymm13			;; I6 = I6 - MemR6*MemI6#2
	vmovapd	ymm3, [srcreg+r9+d4+d1+32]		;; MemI6
	vmulpd	ymm13, ymm3, ymm6			;; MemI6*MemI6#2
	vaddpd	ymm11, ymm11, ymm13			;; R6 = R6 + MemI6*MemI6#2
	vmulpd	ymm13, ymm3, ymm9			;; MemI6*MemR6#2
	vsubpd	ymm1, ymm1, ymm13			;; I6 = I6 - MemI6*MemR6#2

	vmovapd	ymm3, [srcreg+r9+d4]			;; MemR5
	vmovapd	ymm9, [srcreg+r10+d4]			;; MemR5#2
	vmulpd	ymm13, ymm3, ymm9			;; MemR5*MemR5#2
	vsubpd	ymm5, ymm5, ymm13			;; R5 = R5 - MemR5*MemR5#2
	vmovapd	ymm6, [srcreg+r10+d4+32]		;; MemI5#2
	vmulpd	ymm13, ymm3, ymm6			;; MemR5*MemI5#2
	vsubpd	ymm12, ymm12, ymm13			;; I5 = I5 - MemR5*MemI5#2
	vmovapd	ymm3, [srcreg+r9+d4+32]			;; MemI5
	vmulpd	ymm13, ymm3, ymm6			;; MemI5*MemI5#2
	vaddpd	ymm5, ymm5, ymm13			;; R5 = R5 + MemI5*MemI5#2
	vmulpd	ymm13, ymm3, ymm9			;; MemI5*MemR5#2
	vsubpd	ymm12, ymm12, ymm13			;; I5 = I5 - MemI5*MemR5#2

	vmovapd	ymm3, [srcreg+r9+d4+d2+d1]		;; MemR8
	vmovapd	ymm9, [srcreg+r10+d4+d2+d1]		;; MemR8#2
	vmulpd	ymm13, ymm3, ymm9			;; MemR8*MemR8#2
	vsubpd	ymm15, ymm15, ymm13			;; R8 = R8 - MemR8*MemR8#2
	vmovapd	ymm6, [srcreg+r10+d4+d2+d1+32]		;; MemI8#2
	vmulpd	ymm13, ymm3, ymm6			;; MemR8*MemI8#2
	vsubpd	ymm14, ymm14, ymm13			;; I8 = I8 - MemR8*MemI8#2
	vmovapd	ymm3, [srcreg+r9+d4+d2+d1+32]		;; MemI8
	vmulpd	ymm13, ymm3, ymm6			;; MemI8*MemI8#2
	vaddpd	ymm15, ymm15, ymm13			;; R8 = R8 + MemI8*MemI8#2
	vmulpd	ymm13, ymm3, ymm9			;; MemI8*MemR8#2
	vsubpd	ymm14, ymm14, ymm13			;; I8 = I8 - MemI8*MemR8#2

	vmovapd	ymm3, [srcreg+r9+d4+d2]			;; MemR7
	vmovapd	ymm9, [srcreg+r10+d4+d2]		;; MemR7#2
	vmulpd	ymm13, ymm3, ymm9			;; MemR7*MemR7#2
	vsubpd	ymm8, ymm8, ymm13			;; R7 = R7 - MemR7*MemR7#2
	vmovapd	ymm6, [srcreg+r10+d4+d2+32]		;; MemI7#2
	vmulpd	ymm13, ymm3, ymm6			;; MemR7*MemI7#2
	vsubpd	ymm0, ymm0, ymm13			;; I7 = I7 - MemR7*MemI7#2
	vmovapd	ymm3, [srcreg+r9+d4+d2+32]		;; MemI7
	vmulpd	ymm13, ymm3, ymm6			;; MemI7*MemI7#2
	vaddpd	ymm8, ymm8, ymm13			;; R7 = R7 + MemI7*MemI7#2
	vmulpd	ymm13, ymm3, ymm9			;; MemI7*MemR7#2
	vsubpd	ymm0, ymm0, ymm13			;; I7 = I7 - MemI7*MemR7#2

	vaddpd	ymm9, ymm4, ymm10		;; R4 + R3 (new R3)		; 65-67
	vsubpd	ymm4, ymm4, ymm10		;; R4 - R3 (new I4)		; 66-68

	vmovapd	ymm10, [srcreg+d1][rbp]		;; MemR2
	ystore	[srcreg+d2], ymm9		;; Temporarily save new R3	; 68
	vmovapd	ymm9, [srcreg+d1+32][rbp]	;; MemI2
	ystore	[srcreg+d2+32], ymm4		;; Temporarily save new I4	; 69
	vmovapd	ymm6, [srcreg+d1][rbx]		;; R2
	vmovapd	ymm13, [srcreg+d1+32][rbx]	;; I2
	vmulpd	ymm4, ymm6, ymm10		;; R2 * MemR2			; 66-70
	vmulpd	ymm6, ymm6, ymm9		;; R2 * MemI2			; 67-71
	vmulpd	ymm9, ymm13, ymm9		;; I2 * MemI2			; 68-72
	vmulpd	ymm13, ymm13, ymm10		;; I2 * MemR2			; 69-73

	vmovapd	ymm10, [srcreg][rbp]		;; MemR1
	vsubpd	ymm4, ymm4, ymm9		;; R2*MemR2-I2*MemI2 (new R2)	; 73-75
	vmovapd	ymm9, [srcreg+32][rbp]		;; MemI1
	vaddpd	ymm6, ymm6, ymm13		;; R2*MemI2+I2*MemR2 (new I2)	; 74-76

	vsubpd	ymm13, ymm2, ymm7		;; I3 - I4 (new R4)		; 67-69
	vaddpd	ymm2, ymm2, ymm7		;; I3 + I4 (new I3)		; 68-70

	vsubpd	ymm7, ymm12, ymm1		;; I5 - I6 (new I6)		; 69-71
	vaddpd	ymm12, ymm12, ymm1		;; I5 + I6 (new I5)		; 70-72

	vsubpd	ymm1, ymm5, ymm11		;; R5 - R6 (new R6)		; 71-73
	vaddpd	ymm5, ymm5, ymm11		;; R5 + R6 (new R5)		; 72-74

	vmovapd	ymm3, [srcreg][rbx]		;; R1
	vmulpd	ymm11, ymm3, ymm10		;; R1 * MemR1			; 70-74
	vmulpd	ymm3, ymm3, ymm9		;; R1 * MemI1			; 71-75
	vmulpd	ymm9, ymm9, [srcreg+32][rbx]	;; (load I1) * MemI1		; 72-76
	vmulpd	ymm10, ymm10, [srcreg+32][rbx]	;; (load I1) * MemR1		; 73-77

	vsubpd	ymm11, ymm11, ymm9		;; R1*MemR1-I1*MemI1 (new R1)	; 77-79
	vaddpd	ymm3, ymm3, ymm10		;; R1*MemI1+I1*MemR1 (new I1)	; 78-80

	vmovapd	YMM_TMP1, ymm0				;; Save 2 registers
	vmovapd	YMM_TMP2, ymm12

	vmovapd	ymm0, [srcreg+r9+d1]			;; MemR2
	vmovapd	ymm9, [srcreg+r10+d1]			;; MemR2#2
	vmulpd	ymm12, ymm0, ymm9			;; MemR2*MemR2#2
	vsubpd	ymm4, ymm4, ymm12			;; R2 = R2 - MemR2*MemR2#2
	vmovapd	ymm10, [srcreg+r10+d1+32]		;; MemI2#2
	vmulpd	ymm12, ymm0, ymm10			;; MemR2*MemI2#2
	vsubpd	ymm6, ymm6, ymm12			;; I2 = I2 - MemR2*MemI2#2
	vmovapd	ymm0, [srcreg+r9+d1+32]			;; MemI2
	vmulpd	ymm12, ymm0, ymm10			;; MemI2*MemI2#2
	vaddpd	ymm4, ymm4, ymm12			;; R2 = R2 + MemI2*MemI2#2
	vmulpd	ymm12, ymm0, ymm9			;; MemI2*MemR2#2
	vsubpd	ymm6, ymm6, ymm12			;; I2 = I2 - MemI2*MemR2#2

	vmovapd	ymm0, [srcreg+r9]			;; MemR1
	vmovapd	ymm9, [srcreg+r10]			;; MemR1#2
	vmulpd	ymm12, ymm0, ymm9			;; MemR1*MemR1#2
	vsubpd	ymm11, ymm11, ymm12			;; R1 = R1 - MemR1*MemR1#2
	vmovapd	ymm10, [srcreg+r10+32]			;; MemI1#2
	vmulpd	ymm12, ymm0, ymm10			;; MemR1*MemI1#2
	vsubpd	ymm3, ymm3, ymm12			;; I1 = I1 - MemR1*MemI1#2
	vmovapd	ymm0, [srcreg+r9+32]			;; MemI1
	vmulpd	ymm12, ymm0, ymm10			;; MemI1*MemI1#2
	vaddpd	ymm11, ymm11, ymm12			;; R1 = R1 + MemI1*MemI1#2
	vmulpd	ymm12, ymm0, ymm9			;; MemI1*MemR1#2
	vsubpd	ymm3, ymm3, ymm12			;; I1 = I1 - MemI1*MemR1#2

	vmovapd	ymm0, YMM_TMP1				;; Restore 2 registers
	vmovapd	ymm12, YMM_TMP2

done:
	ENDM

ENDIF


;;
;; ************************************* sixteen-reals-fft8 variants ******************************************
;;

;;
;; Do three-and-7/8 levels of the forward FFT on 16 real data values with 7 sin/cos multipliers.
;; Output is 2 real values needing N+1 more levels and 7 complex values needing N more FFT levels.
;;

;; Used in last levels of pass 1 (split premultiplier and delay cases).  Swizzling.

yr8_sg8cl_2sc_sixteen_reals_fft8 MACRO srcreg,srcinc,d1,d2,d4,dstreg,dstinc,e1,e2,e4,screg1,scinc1,screg2,scinc2,maxrpt,L1pt,L1pd

	;; On input the 64-byte cache lines hold these data values:
	;;	0	+1K	+1K	+1K	8K	+1K	+1K	+1K
	;;	1	...
	;;	2	...
	;;	3	...

	;; These are swizzled to:
	;;	0	+1	+1	+1	8K	+1	+1	+1
	;;	1K	...
	;;	2K	...
	;;	3K	...

	;; Swizzle using 8 shufpds and 8 perm2f128  (1 or 3 clocks each on port 5, throughput 1)

	vmovapd	ymm1, [srcreg]			;; R1
	vmovapd	ymm7, [srcreg+d1]		;; R2
	vshufpd	ymm0, ymm1, ymm7, 15		;; Shuffle R1 and R2 to create R1/R2 hi
	vshufpd	ymm1, ymm1, ymm7, 0		;; Shuffle R1 and R2 to create R1/R2 low

	vmovapd	ymm2, [srcreg+d2]		;; R3
	vmovapd	ymm7, [srcreg+d2+d1]		;; R4
	vshufpd	ymm3, ymm2, ymm7, 15		;; Shuffle R3 and R4 to create R3/R4 hi
	vshufpd	ymm2, ymm2, ymm7, 0		;; Shuffle R3 and R4 to create R3/R4 low

	ylow128s ymm4, ymm0, ymm3		;; Shuffle R1/R2 hi and R3/R4 hi (first R2)
	yhigh128s ymm0, ymm0, ymm3		;; Shuffle R1/R2 hi and R3/R4 hi (first R4)

	ylow128s ymm3, ymm1, ymm2		;; Shuffle R1/R2 low and R3/R4 low (first R1)
	yhigh128s ymm1, ymm1, ymm2		;; Shuffle R1/R2 low and R3/R4 low (first R3)

	vmovapd	ymm2, [srcreg+32]		;; I1
	vmovapd	ymm7, [srcreg+d1+32]		;; I2
	vshufpd	ymm5, ymm2, ymm7, 15		;; Shuffle I1 and I2 to create I1/I2 hi
	vshufpd	ymm2, ymm2, ymm7, 0		;; Shuffle I1 and I2 to create I1/I2 low

	vmovapd	ymm6, [srcreg+d2+32]		;; I3
	vmovapd	ymm7, [srcreg+d2+d1+32]		;; I4
	ystore	[dstreg], ymm0			;; Save first R4
	vshufpd	ymm0, ymm6, ymm7, 15		;; Shuffle I3 and I4 to create I3/I4 hi
	vshufpd	ymm6, ymm6, ymm7, 0		;; Shuffle I3 and I4 to create I3/I4 low

	ylow128s ymm7, ymm5, ymm0		;; Shuffle I1/I2 hi and I3/I4 hi (first R10)
	yhigh128s ymm5, ymm5, ymm0		;; Shuffle I1/I2 hi and I3/I4 hi (first R12)

	ylow128s ymm0, ymm2, ymm6		;; Shuffle I1/I2 low and I3/I4 low (first R9)
	yhigh128s ymm2, ymm2, ymm6		;; Shuffle I1/I2 low and I3/I4 low (first R11)

	L1prefetch srcreg+L1pd, L1pt
	L1prefetchw dstreg+L1pd, L1pt - L1PREFETCH_DEST_NONE

	vaddpd	ymm6, ymm4, ymm7		;; R2 + R10 (new R2)
	vsubpd	ymm4, ymm4, ymm7		;; R2 - R10 (new R10)

	vaddpd	ymm7, ymm3, ymm0		;; R1 + R9 (new R1)
	vsubpd	ymm3, ymm3, ymm0		;; R1 - R9 (new R9)

	L1prefetch srcreg+d1+L1pd, L1pt
	L1prefetchw dstreg+e1+L1pd, L1pt - L1PREFETCH_DEST_NONE

	vaddpd	ymm0, ymm1, ymm2		;; R3 + R11 (new R3)
	vsubpd	ymm1, ymm1, ymm2		;; R3 - R11 (new R11)

	vmovapd	ymm2, [dstreg]			;; Reload first R4
	ystore	[dstreg+e1], ymm6		;; Save new R2
	vaddpd	ymm6, ymm2, ymm5		;; R4 + R12 (new R4)
	vsubpd	ymm2, ymm2, ymm5		;; R4 - R12 (new R12)

	vmovapd	ymm5, [srcreg+d4]		;; R5
	ystore	[dstreg+e4+e1], ymm4		;; Save new R10
	vmovapd	ymm4, [srcreg+d4+d1]		;; R6
	ystore	[dstreg], ymm7			;; Save new R1
	vshufpd	ymm7, ymm5, ymm4, 15		;; Shuffle R5 and R6 to create R5/R6 hi
	vshufpd	ymm5, ymm5, ymm4, 0		;; Shuffle R5 and R6 to create R5/R6 low

	vmovapd	ymm4, [srcreg+d4+d2]		;; R7
	ystore	[dstreg+e4], ymm3		;; Save new R9
	vmovapd	ymm3, [srcreg+d4+d2+d1]		;; R8
	ystore	[dstreg+e2], ymm0		;; Save new R3
	vshufpd	ymm0, ymm4, ymm3, 15		;; Shuffle R7 and R8 to create R7/R8 hi
	vshufpd	ymm4, ymm4, ymm3, 0		;; Shuffle R7 and R8 to create R7/R8 low

	ylow128s ymm3, ymm7, ymm0		;; Shuffle R5/R6 hi and R7/R8 hi (first R6)
	yhigh128s ymm7, ymm7, ymm0		;; Shuffle R5/R6 hi and R7/R8 hi (first R8)

	ylow128s ymm0, ymm5, ymm4		;; Shuffle R5/R6 low and R7/R8 low (first R5)
	yhigh128s ymm5, ymm5, ymm4		;; Shuffle R5/R6 low and R7/R8 low (first R7)

	vmovapd	ymm4, [srcreg+d4+32]		;; I5
	ystore	[dstreg+e4+e2], ymm1		;; Save new R11
	vmovapd	ymm1, [srcreg+d4+d1+32]		;; I6
	ystore	[dstreg+e2+e1], ymm6		;; Save new R4
	vshufpd	ymm6, ymm4, ymm1, 15		;; Shuffle I5 and I6 to create I5/I6 hi
	vshufpd	ymm4, ymm4, ymm1, 0		;; Shuffle I5 and I6 to create I5/I6 low

	vmovapd	ymm1, [srcreg+d4+d2+32]		;; I7
	ystore	[dstreg+e4+e2+e1], ymm2		;; Save new R12
	vmovapd	ymm2, [srcreg+d4+d2+d1+32]	;; I8
	ystore	[dstreg+32], ymm7		;; Save first R8
	vshufpd	ymm7, ymm1, ymm2, 15		;; Shuffle I7 and I8 to create I7/I8 hi
	vshufpd	ymm1, ymm1, ymm2, 0		;; Shuffle I7 and I8 to create I7/I8 low

	ylow128s ymm2, ymm6, ymm7		;; Shuffle I5/I6 hi and I7/I8 hi (first R14)
	yhigh128s ymm6, ymm6, ymm7		;; Shuffle I5/I6 hi and I7/I8 hi (first R16)

	ylow128s ymm7, ymm4, ymm1		;; Shuffle I5/I6 low and I7/I8 low (first R13)
	yhigh128s ymm4, ymm4, ymm1		;; Shuffle I5/I6 low and I7/I8 low (first R15)

	L1prefetch srcreg+d2+L1pd, L1pt
	L1prefetchw dstreg+e2+L1pd, L1pt - L1PREFETCH_DEST_NONE

	vaddpd	ymm1, ymm3, ymm2		;; R6 + R14 (new R6)
	vsubpd	ymm3, ymm3, ymm2		;; R6 - R14 (new R14)

	vaddpd	ymm2, ymm0, ymm7		;; R5 + R13 (new R5)
	vsubpd	ymm0, ymm0, ymm7		;; R5 - R13 (new R13)

	L1prefetch srcreg+d2+d1+L1pd, L1pt
	L1prefetchw dstreg+e2+e1+L1pd, L1pt - L1PREFETCH_DEST_NONE

	vaddpd	ymm7, ymm5, ymm4		;; R7 + R15 (new R7)
	vsubpd	ymm5, ymm5, ymm4		;; R7 - R15 (new R15)

	vmovapd	ymm4, [dstreg+32]		;; Reload first R8
	ystore	[dstreg+e4+e1+32], ymm3		;; Save new R14
	vaddpd	ymm3, ymm4, ymm6		;; R8 + R16 (new R8)
	vsubpd	ymm4, ymm4, ymm6		;; R8 - R16 (new R16)

	;; Even levels 2

	vmovapd	ymm6, [dstreg+e2+e1]		;; Reload new R4
	ystore	[dstreg+e4+32], ymm0		;; Save new R13
	vaddpd	ymm0, ymm6, ymm3		;; R4 + R8 (newer R4)
	vsubpd	ymm6, ymm6, ymm3		;; R4 - R8 (newer R8)

	vmovapd	ymm3, [dstreg+e1]		;; Reload new R2
	ystore	[dstreg+e4+e2+32], ymm5		;; Save new R15
	vaddpd	ymm5, ymm3, ymm1		;; R2 + R6 (newer R2)
	vsubpd	ymm3, ymm3, ymm1		;; R2 - R6 (newer R6)

	;; Even level 3

	L1prefetch srcreg+d4+L1pd, L1pt
	L1prefetchw dstreg+e4+L1pd, L1pt - L1PREFETCH_DEST_NONE

	vaddpd	ymm1, ymm5, ymm0		;; R2 + R4 (newest R2)
	vsubpd	ymm5, ymm5, ymm0		;; R2 - R4 (newest R4)

						;; R6/R8 morphs into newer R6/I6

	;; Premultipliers for even level 4

						;; mul R6/I6 by w^2 = .707 + .707i
	vsubpd	ymm0, ymm3, ymm6		;; R6 = R6 - I6
	vaddpd	ymm3, ymm3, ymm6		;; I6 = R6 + I6
	vmulpd	ymm0, ymm0, YMM_SQRTHALF	;; R6 = R6 * SQRTHALF (newest R6)
	vmulpd	ymm3, ymm3, YMM_SQRTHALF	;; I6 = I6 * SQRTHALF (newest I6)

	;; Odd levels 2

	vmovapd	ymm6, [dstreg]			;; Reload new R1
	ystore	[dstreg+e4+e2+e1+32], ymm4	;; Save new R16
	vsubpd	ymm4, ymm6, ymm2		;; R1 - R5 (newer R5)
	vaddpd	ymm6, ymm6, ymm2		;; R1 + R5 (newer R1)

	vmovapd	ymm2, [dstreg+e2]		;; Reload new R3
	ystore	[dstreg+e2+e1], ymm5		;; Save newest R4
	vaddpd	ymm5, ymm2, ymm7		;; R3 + R7 (newer R3)
	vsubpd	ymm2, ymm2, ymm7		;; R3 - R7 (newer R7)

	;; Odd level 3

	L1prefetch srcreg+d4+d1+L1pd, L1pt
	L1prefetchw dstreg+e4+e1+L1pd, L1pt - L1PREFETCH_DEST_NONE

 	vaddpd	ymm7, ymm6, ymm5		;; R1 + R3 (newest R1)
	vsubpd	ymm6, ymm6, ymm5		;; R1 - R3 (newest R3)

						;; R5/R7 morphs into newest R5/I5

	;; Last level

						;; R1/R2 becomes final R1 and final R2

						;; R3/R4 morphs into R3/I3

	ystore	[dstreg+32], ymm1		;; Save R2
	vmovapd	ymm1, [screg1+128+32]		;; cosine/sine for w^4 (8-complex w^2)
	vmulpd	ymm5, ymm6, ymm1		;; A3 = R3 * cosine/sine
	ystore	[dstreg], ymm7			;; Save R1
	vmovapd	ymm7, [dstreg+e2+e1]		;; Reload newest R4 which morped into I3
	vsubpd	ymm5, ymm5, ymm7		;; A3 = A3 - I3
	vmulpd	ymm7, ymm7, ymm1		;; B3 = I3 * cosine/sine
	vaddpd	ymm7, ymm7, ymm6		;; B3 = B3 + R3
	vmovapd	ymm1, [screg1+128]		;; sine for w^4 (8-complex w^2)
	vmulpd	ymm5, ymm5, ymm1		;; A3 = A3 * sine (final R3)
	vmulpd	ymm7, ymm7, ymm1		;; B3 = B3 * sine (final I3)

	vsubpd	ymm1, ymm4, ymm0		;; R5 - R6 (final R6)
	vaddpd	ymm4, ymm4, ymm0		;; R5 + R6 (final R5)

	L1prefetch srcreg+d4+d2+L1pd, L1pt
	L1prefetchw dstreg+e4+e2+L1pd, L1pt - L1PREFETCH_DEST_NONE

	vsubpd	ymm0, ymm2, ymm3		;; I5 - I6 (final I6)
	vaddpd	ymm2, ymm2, ymm3		;; I5 + I6 (final I5)

	vmovapd	ymm3, [screg1+64+32]		;; cosine/sine for w^2 (8-complex w^1)
	vmulpd	ymm6, ymm4, ymm3		;; A5 = R5 * cosine/sine
	vsubpd	ymm6, ymm6, ymm2		;; A5 = A5 - I5
	vmulpd	ymm2, ymm2, ymm3		;; B5 = I5 * cosine/sine
	vaddpd	ymm2, ymm2, ymm4		;; B5 = B5 + R5

	vmovapd	ymm3, [screg1+320+32]		;; cosine/sine for w^10 (8-complex w^5)
	vmulpd	ymm4, ymm1, ymm3		;; A6 = R6 * cosine/sine
	vsubpd	ymm4, ymm4, ymm0		;; A6 = A6 - I6
	vmulpd	ymm0, ymm0, ymm3		;; B6 = I6 * cosine/sine
	vaddpd	ymm0, ymm0, ymm1		;; B6 = B6 + R6

	vmovapd	ymm3, [screg1+64]		;; sine for w^2 (8-complex w^1)
	vmulpd	ymm6, ymm6, ymm3		;; A5 = A5 * sine (final R5)
	vmulpd	ymm2, ymm2, ymm3		;; B5 = B5 * sine (final I5)
	vmovapd	ymm3, [screg1+320]		;; sine for w^10 (8-complex w^5)
	vmulpd	ymm4, ymm4, ymm3		;; A6 = A6 * sine (final R6)
	vmulpd	ymm0, ymm0, ymm3		;; B6 = B6 * sine (final I6)

	;; Odd levels 2

						;; R9/R13 morphs into newer R9/I9
						;; R11/R15 morphs into newer R11/I11

	;; Even levels 2

						;; R10/R14 morphs into newer R10/I10
						;; R12/R16 morphs into newer R12/I12

	;; Premultipliers for even level 3

						;; mul R10/I10 by w^1 = .924 + .383i
	vmovapd ymm1, [dstreg+e4+e1]		;; Reload new R10
	vmovapd	ymm3, YMM_P924
	ystore	[dstreg+e1], ymm5		;; Save R3
	vmulpd	ymm5, ymm1, ymm3		;; R10 * .924
	ystore	[dstreg+e1+32], ymm7		;; Save I3
	vmovapd	ymm7, YMM_P383
	vmulpd	ymm1, ymm1, ymm7		;; R10 * .383
	ystore	[dstreg+e2+e1], ymm4		;; Save R6
	vmovapd ymm4, [dstreg+e4+e1+32]		;; Reload new R14 which morped into I10
	ystore	[dstreg+e2], ymm6		;; Save R5
	vmulpd	ymm6, ymm4, ymm7		;; I10 * .383
	vmulpd	ymm4, ymm4, ymm3		;; I10 * .924
	vsubpd	ymm5, ymm5, ymm6		;; Twiddled R10
	vaddpd	ymm1, ymm1, ymm4		;; Twiddled I10

						;; mul R12/I12 by w^3 = .383 + .924i
	vmovapd ymm6, [dstreg+e4+e2+e1]		;; Reload new R12
	vmulpd	ymm4, ymm6, ymm7		;; R12 * .383
	vmulpd	ymm6, ymm6, ymm3		;; R12 * .924
	ystore	[dstreg+e2+e1+32], ymm0		;; Save I6
	vmovapd ymm0, [dstreg+e4+e2+e1+32]	;; Reload new R16 which morped into I12
	vmulpd	ymm3, ymm0, ymm3		;; I12 * .924
	vmulpd	ymm0, ymm0, ymm7		;; I12 * .383
	vsubpd	ymm4, ymm4, ymm3		;; Twiddled R12
	vaddpd	ymm6, ymm6, ymm0		;; Twiddled I12

	;; Even level 3

	vaddpd	ymm7, ymm5, ymm4		;; R10 + R12 (newest R10)
	vsubpd	ymm5, ymm5, ymm4		;; R10 - R12 (newest R12)

	L1prefetch srcreg+d4+d2+d1+L1pd, L1pt
	L1prefetchw dstreg+e4+e2+e1+L1pd, L1pt - L1PREFETCH_DEST_NONE

	vaddpd	ymm3, ymm1, ymm6		;; I10 + I12 (newest I10)
	vsubpd	ymm1, ymm1, ymm6		;; I10 - I12 (newest I12)

	;; Premultipliers for odd level 3

						;; mul R11/I11 by SQRTHALF + i*SQRTHALF
	vmovapd	ymm0, [dstreg+e4+e2]		;; Reload new R11
	vmovapd ymm4, [dstreg+e4+e2+32]		;; Reload new R15 which morped into I11
	vsubpd	ymm6, ymm0, ymm4		;; R11 = R11 - I11
	vaddpd	ymm0, ymm0, ymm4		;; I11 = R11 + I11
	vmulpd	ymm6, ymm6, YMM_SQRTHALF	;; R11 = R11 * SQRTHALF
	vmulpd	ymm0, ymm0, YMM_SQRTHALF	;; I11 = I11 * SQRTHALF

	;; Odd level 3

	vmovapd	ymm4, [dstreg+e4]		;; Reload new R9
	ystore	[dstreg+e2+32], ymm2		;; Save I5
	vaddpd	ymm2, ymm4, ymm6		;; R9 + R11 (newest R9)
	vsubpd	ymm4, ymm4, ymm6		;; R9 - R11 (newest R11)

	vmovapd ymm6, [dstreg+e4+32]		;; Reload new R13 which morped into I9
	ystore	[dstreg+e4+e2+e1+32], ymm1	;; Save newest I12
	vaddpd	ymm1, ymm6, ymm0		;; I9 + I11 (newest I9)
	vsubpd	ymm6, ymm6, ymm0		;; I9 - I11 (newest I11)

	;; Last level

	vsubpd	ymm0, ymm2, ymm7		;; R9 - R10 (final R10)
	vaddpd	ymm2, ymm2, ymm7		;; R9 + R10 (final R9)

	vsubpd	ymm7, ymm1, ymm3		;; I9 - I10 (final I10)
	vaddpd	ymm1, ymm1, ymm3		;; I9 + I10 (final I9)

	vmovapd	ymm3, [screg2+0+32]		;; cosine/sine for w^1
	ystore	[dstreg+e4+e2+e1], ymm5		;; Save newest R12
	vmulpd	ymm5, ymm2, ymm3		;; A9 = R9 * cosine/sine
	vsubpd	ymm5, ymm5, ymm1		;; A9 = A9 - I9
	vmulpd	ymm1, ymm1, ymm3		;; B9 = I9 * cosine/sine
	vaddpd	ymm1, ymm1, ymm2		;; B9 = B9 + R9
	vmovapd	ymm3, [screg2+0]		;; sine for w^1
	vmulpd	ymm5, ymm5, ymm3		;; A9 = A9 * sine (final R9)
	vmulpd	ymm1, ymm1, ymm3		;; B9 = B9 * sine (final I9)

	vmovapd	ymm3, [screg2+128+32]		;; cosine/sine for w^9
	vmulpd	ymm2, ymm0, ymm3		;; A10 = R10 * cosine/sine
	vsubpd	ymm2, ymm2, ymm7		;; A10 = A10 - I10
	vmulpd	ymm7, ymm7, ymm3		;; B10 = I10 * cosine/sine
	vaddpd	ymm7, ymm7, ymm0		;; B10 = B10 + R10
	vmovapd	ymm3, [screg2+128]		;; sine for w^9
	vmulpd	ymm2, ymm2, ymm3		;; A10 = A10 * sine (final R10)
	vmulpd	ymm7, ymm7, ymm3		;; B10 = B10 * sine (final I10)

	vmovapd	ymm0, [dstreg+e4+e2+e1+32]	;; Reload newest I12
	vsubpd	ymm3, ymm4, ymm0		;; R11 - I12 (final R11)
	vaddpd	ymm4, ymm4, ymm0		;; R11 + I12 (final R12)

	vmovapd	ymm0, [dstreg+e4+e2+e1]		;; Reload newest R12
	ystore	[dstreg+e4], ymm5		;; Save R9
	vaddpd	ymm5, ymm6, ymm0		;; I11 + R12 (final I11)
	vsubpd	ymm6, ymm6, ymm0		;; I11 - R12 (final I12)

	vmovapd	ymm0, [screg2+64+32]		;; cosine/sine for w^5
	ystore	[dstreg+e4+32], ymm1		;; Save I9
	vmulpd	ymm1, ymm3, ymm0		;; A11 = R11 * cosine/sine
	vsubpd	ymm1, ymm1, ymm5		;; A11 = A11 - I11
	vmulpd	ymm5, ymm5, ymm0		;; B11 = I11 * cosine/sine
	vaddpd	ymm5, ymm5, ymm3		;; B11 = B11 + R11

	vmovapd	ymm0, [screg2+192+32]		;; cosine/sine for w^13
	vmulpd	ymm3, ymm4, ymm0		;; A12 = R12 * cosine/sine
	vsubpd	ymm3, ymm3, ymm6		;; A12 = A12 - I12
	vmulpd	ymm6, ymm6, ymm0		;; B12 = I12 * cosine/sine
	vaddpd	ymm6, ymm6, ymm4		;; B12 = B12 + R12

	vmovapd	ymm0, [screg2+64]		;; sine for w^5
	vmulpd	ymm1, ymm1, ymm0		;; A11 = A11 * sine (final R11)
	vmulpd	ymm5, ymm5, ymm0		;; B11 = B11 * sine (final I11)

	vmovapd	ymm0, [screg2+192]		;; sine for w^13
	vmulpd	ymm3, ymm3, ymm0		;; A12 = A12 * sine (final R12)
	vmulpd	ymm6, ymm6, ymm0		;; B12 = B12 * sine (final I12)

	ystore	[dstreg+e4+e1], ymm2		;; Save R10
	ystore	[dstreg+e4+e1+32], ymm7		;; Save I10
	ystore	[dstreg+e4+e2], ymm1		;; Save R11
	ystore	[dstreg+e4+e2+32], ymm5		;; Save I11
	ystore	[dstreg+e4+e2+e1], ymm3		;; Save R12
	ystore	[dstreg+e4+e2+e1+32], ymm6	;; Save I12

	bump	srcreg, srcinc
	bump	dstreg, dstinc
	bump	screg1, scinc1
	bump	screg2, scinc2
	ENDM


;;
;; ************************************* sixteen-reals-unfft8 variants ******************************************
;;

;; Used in last levels of pass 1 (split premultiplier and delay cases).  Swizzling.
;; In this particular case, we know that the sin/cos multipliers for R1/I1 are one.  Thus
;; we can get by with only 7 sin/cos multiplies.

yr8_sg8cl_2sc_sixteen_reals_unfft8 MACRO srcreg,srcinc,d1,d2,d4,dstreg,dstinc,e1,e2,e4,screg1,scinc1,screg2,scinc2,maxrpt,L1pt,L1pd

	vmovapd	ymm0, [screg1+64+32]		;; cosine/sine for w^2 (8-complex w^1)
	vmovapd	ymm1, [srcreg+d2]		;; R5
	vmulpd	ymm2, ymm1, ymm0		;; A5 = R5 * cosine/sine
	vmovapd	ymm3, [srcreg+d2+32]		;; I5
	vaddpd	ymm2, ymm2, ymm3		;; A5 = A5 + I5
	vmulpd	ymm3, ymm3, ymm0		;; B5 = I5 * cosine/sine
	vsubpd	ymm3, ymm3, ymm1		;; B5 = B5 - R5

	vmovapd	ymm4, [screg1+320+32]		;; cosine/sine for w^10 (8-complex w^5)
	vmovapd	ymm5, [srcreg+d2+d1]		;; R6
	vmulpd	ymm6, ymm5, ymm4		;; A6 = R6 * cosine/sine
	vmovapd	ymm7, [srcreg+d2+d1+32]		;; I6
	vaddpd	ymm6, ymm6, ymm7		;; A6 = A6 + I6
	vmulpd	ymm7, ymm7, ymm4		;; B6 = I6 * cosine/sine
	vsubpd	ymm7, ymm7, ymm5		;; B6 = B6 - R6

	vmovapd	ymm0, [screg1+128+32]		;; cosine/sine for w^4 (8-complex w^2)
	vmovapd	ymm1, [srcreg+d1]		;; R3
	vmulpd	ymm4, ymm1, ymm0		;; A3 = R3 * cosine/sine
	vmovapd	ymm5, [srcreg+d1+32]		;; I3
	vaddpd	ymm4, ymm4, ymm5		;; A3 = A3 + I3
	vmulpd	ymm5, ymm5, ymm0		;; B3 = I3 * cosine/sine
	vsubpd	ymm5, ymm5, ymm1		;; B3 = B3 - R3

	vmovapd ymm0, [screg1+64]		;; sine for w^2 (8-complex w^1)
	vmulpd	ymm2, ymm2, ymm0		;; A5 = A5 * sine (first R5)
	vmulpd	ymm3, ymm3, ymm0		;; B5 = B5 * sine (first I5)

	vmovapd ymm0, [screg1+320]		;; sine for w^10 (8-complex w^5)
	vmulpd	ymm6, ymm6, ymm0		;; A6 = A6 * sine (first R6)
	vmulpd	ymm7, ymm7, ymm0		;; B6 = B6 * sine (first I6)

	vmovapd ymm0, [screg1+128]		;; sine for w^4 (8-complex w^2)
	vmulpd	ymm4, ymm4, ymm0		;; A3 = A3 * sine (first R3)
	vmulpd	ymm5, ymm5, ymm0		;; B3 = B3 * sine (first I3)

						;; R3/I3 morphs into R3/R4

	vaddpd	ymm1, ymm2, ymm6		;; R5 + R6 (new R5)
	vsubpd	ymm2, ymm2, ymm6		;; R5 - R6 (new R6)

	L1prefetch srcreg+L1pd, L1pt

	vaddpd	ymm0, ymm3, ymm7		;; I5 + I6 (new I5)
	vsubpd	ymm3, ymm3, ymm7		;; I5 - I6 (new I6)

	vmovapd	ymm6, [srcreg]			;; R1
	vaddpd	ymm7, ymm6, ymm4		;; R1 + R3 (new R1)
	vsubpd	ymm6, ymm6, ymm4		;; R1 - R3 (new R3)

						;; mul R6/I6 by w^2 = .707 - .707i
	vaddpd	ymm4, ymm3, ymm2		;; R6 = I6 + R6
	vsubpd	ymm3, ymm3, ymm2		;; I6 = I6 - R6
	vmovapd	ymm2, YMM_SQRTHALF
	vmulpd	ymm4, ymm4, ymm2		;; R6 = R6 * SQRTHALF
	vmulpd	ymm3, ymm3, ymm2		;; I6 = I6 * SQRTHALF

						;; R5/I5 morphs into new R5/R7
						;; R6/I6 morph into new R6/R8

	vaddpd	ymm2, ymm7, ymm1		;; R1 + R5 (newer R1)
	vsubpd	ymm7, ymm7, ymm1		;; R1 - R5 (newer R5)

	vmovapd	ymm1, [srcreg+32]		;; R2
	ystore	[dstreg], ymm2			;; Save newer R1
	vaddpd	ymm2, ymm1, ymm5		;; R2 + R4 (new R2)
	vsubpd	ymm1, ymm1, ymm5		;; R2 - R4 (new R4)

	vaddpd	ymm5, ymm6, ymm0		;; R3 + R7 (newer R3)
	vsubpd	ymm6, ymm6, ymm0		;; R3 - R7 (newer R7)

	L1prefetch srcreg+d1+L1pd, L1pt

	vaddpd	ymm0, ymm2, ymm4		;; R2 + R6 (newer R2)
	vsubpd	ymm2, ymm2, ymm4		;; R2 - R6 (newer R6)

	vaddpd	ymm4, ymm1, ymm3		;; R4 + R8 (newer R4)
	vsubpd	ymm1, ymm1, ymm3		;; R4 - R8 (newer R8)

	vmovapd	ymm3, [screg2+0+32]		;; cosine/sine for w^1
	ystore	[dstreg+e4], ymm7		;; Save newer R5
	vmovapd	ymm7, [srcreg+d4]		;; R9
	ystore	[dstreg+e1], ymm5		;; Save newer R3
	vmulpd	ymm5, ymm7, ymm3		;; A9 = R9 * cosine/sine
	ystore	[dstreg+e4+e1], ymm6		;; Save newer R7
	vmovapd	ymm6, [srcreg+d4+32]		;; I9
	vaddpd	ymm5, ymm5, ymm6		;; A9 = A9 + I9
	vmulpd	ymm6, ymm6, ymm3		;; B9 = I9 * cosine/sine
	vsubpd	ymm6, ymm6, ymm7		;; B9 = B9 - R9

	vmovapd	ymm3, [screg2+128+32]		;; cosine/sine for w^9
	vmovapd	ymm7, [srcreg+d4+d1]		;; R10
	ystore	[dstreg+32], ymm0		;; Save newer R2
	vmulpd	ymm0, ymm7, ymm3		;; A10 = R10 * cosine/sine
	ystore	[dstreg+e4+32], ymm2		;; Save newer R6
	vmovapd	ymm2, [srcreg+d4+d1+32]		;; I10
	vaddpd	ymm0, ymm0, ymm2		;; A10 = A10 + I10
	vmulpd	ymm2, ymm2, ymm3		;; B10 = I10 * cosine/sine
	vsubpd	ymm2, ymm2, ymm7		;; B10 = B10 - R10

	vmovapd ymm3, [screg2+0]		;; sine for w^1
	vmulpd	ymm5, ymm5, ymm3		;; A9 = A9 * sine (first R9)
	vmulpd	ymm6, ymm6, ymm3		;; B9 = B9 * sine (first I9)

	vmovapd ymm7, [screg2+128]		;; sine for w^9
	vmulpd	ymm0, ymm0, ymm7		;; A10 = A10 * sine (first R10)
	vmulpd	ymm2, ymm2, ymm7		;; B10 = B10 * sine (first I10)

	vaddpd	ymm3, ymm5, ymm0		;; R9 + R10 (new R9)
	vsubpd	ymm5, ymm5, ymm0		;; R9 - R10 (new R10)

	L1prefetch srcreg+d2+L1pd, L1pt

	vaddpd	ymm7, ymm6, ymm2		;; I9 + I10 (new I9)
	vsubpd	ymm6, ymm6, ymm2		;; I9 - I10 (new I10)

	vmovapd	ymm0, [screg2+64+32]		;; cosine/sine for w^5
	vmovapd	ymm2, [srcreg+d4+d2]		;; R11
	ystore	[dstreg+e1+32], ymm4		;; Save newer R4
	vmulpd	ymm4, ymm2, ymm0		;; A11 = R11 * cosine/sine
	ystore	[dstreg+e4+e1+32], ymm1		;; Save newer R8
	vmovapd	ymm1, [srcreg+d4+d2+32]		;; I11
	vaddpd	ymm4, ymm4, ymm1		;; A11 = A11 + I11
	vmulpd	ymm1, ymm1, ymm0		;; B11 = I11 * cosine/sine
	vsubpd	ymm1, ymm1, ymm2		;; B11 = B11 - R11

	vmovapd	ymm0, [screg2+192+32]		;; cosine/sine for w^13
	vmovapd	ymm2, [srcreg+d4+d2+d1]		;; R12
	ystore	[dstreg+e2+e1], ymm5		;; Save new R10
	vmulpd	ymm5, ymm2, ymm0		;; A12 = R12 * cosine/sine
	ystore	[dstreg+e2+e1+32], ymm6		;; Save new I10
	vmovapd	ymm6, [srcreg+d4+d2+d1+32]	;; I12
	vaddpd	ymm5, ymm5, ymm6		;; A12 = A12 + I12
	vmulpd	ymm6, ymm6, ymm0		;; B12 = I12 * cosine/sine
	vsubpd	ymm6, ymm6, ymm2		;; B12 = B12 - R12

	vmovapd ymm0, [screg2+64]		;; sine for w^5
	vmulpd	ymm4, ymm4, ymm0		;; A11 = A11 * sine (first R11)
	vmulpd	ymm1, ymm1, ymm0		;; B11 = B11 * sine (first I11)

	vmovapd ymm2, [screg2+192]		;; sine for w^13
	vmulpd	ymm5, ymm5, ymm2		;; A12 = A12 * sine (first R12)
	vmulpd	ymm6, ymm6, ymm2		;; B12 = B12 * sine (first I12)

	vaddpd	ymm2, ymm5, ymm4		;; R12 + R11 (new R11)
	vsubpd	ymm5, ymm5, ymm4		;; R12 - R11 (new I12)

	L1prefetch srcreg+d2+d1+L1pd, L1pt

	vaddpd	ymm0, ymm1, ymm6		;; I11 + I12 (new I11)
	vsubpd	ymm1, ymm1, ymm6		;; I11 - I12 (new R12)

	vaddpd	ymm4, ymm3, ymm2		;; R9 + R11 (newer R9)
	vsubpd	ymm3, ymm3, ymm2		;; R9 - R11 (newer R11)

	vaddpd	ymm6, ymm7, ymm0		;; I9 + I11 (newer I9)
	vsubpd	ymm7, ymm7, ymm0		;; I9 - I11 (newer I11)

						;; mul R11/I11 by SQRTHALF - i*SQRTHALF
	vaddpd	ymm2, ymm7, ymm3		;; R11 = I11 + R11
	vsubpd	ymm7, ymm7, ymm3		;; I11 = I11 - R11
	vmovapd ymm0, YMM_SQRTHALF
	vmulpd	ymm2, ymm2, ymm0		;; R11 = R11 * SQRTHALF
	vmulpd	ymm7, ymm7, ymm0		;; I11 = I11 * SQRTHALF

						;; R9/I9 morphs into newer R9/R13
						;; R11/I11 morphs into newer R11/R15

	vmovapd	ymm0, [dstreg+e2+e1]		;; Reload new R10
	vaddpd	ymm3, ymm0, ymm1		;; R10 + R12 (newer R10)
	vsubpd	ymm0, ymm0, ymm1		;; R10 - R12 (newer R12)

	vmovapd	ymm1, [dstreg+e2+e1+32]		;; Reload new I10
	ystore	[dstreg+e2], ymm4		;; Save newer R9
	vaddpd	ymm4, ymm1, ymm5		;; I10 + I12 (newer I10)
	vsubpd	ymm1, ymm1, ymm5		;; I10 - I12 (newer I12)


						;; mul R10/I10 by w^1 = .924 - .383i
	vmovapd ymm5, YMM_P924
	ystore	[dstreg+e4+e2], ymm6		;; Save newer R13
	vmulpd	ymm6, ymm3, ymm5		;; R10 * .924
	ystore	[dstreg+e2+e1], ymm2		;; Save newer R11
	vmovapd ymm2, YMM_P383
	ystore	[dstreg+e4+e2+e1], ymm7		;; Save newer R15
	vmulpd	ymm7, ymm4, ymm2		;; I10 * .383
	vmulpd	ymm3, ymm3, ymm2		;; R10 * .383
	vmulpd	ymm4, ymm4, ymm5		;; I10 * .924
	vaddpd	ymm6, ymm6, ymm7		;; Twiddled R10
	vsubpd	ymm3, ymm4, ymm3		;; Twiddled I10

	L1prefetch srcreg+d4+L1pd, L1pt

						;; mul R12/I12 by w^3 = .383 - .924i
	vmulpd	ymm7, ymm0, ymm2		;; R12 * .383
	vmulpd	ymm4, ymm1, ymm5		;; I12 * .924
	vmulpd	ymm0, ymm0, ymm5		;; R12 * .924
	vmulpd	ymm1, ymm1, ymm2		;; I12 * .383
	vaddpd	ymm7, ymm7, ymm4		;; Twiddled R12
	vsubpd	ymm0, ymm1, ymm0		;; Twiddled I12

						;; R10/I10 morphs into newer R10/R14
						;; R12/I12 morphs into newer R12/R16

	;; Do last level, shuffle and store

	vmovapd	ymm5, [dstreg]			;; Reload newer R1
	vmovapd	ymm2, [dstreg+e2]		;; Reload newer R9
	vaddpd	ymm4, ymm5, ymm2		;; R1 + R9 (last R1)
	vsubpd	ymm5, ymm5, ymm2		;; R1 - R9 (last R9)

	vmovapd	ymm1, [dstreg+32]		;; Reload newer R2
	vaddpd	ymm2, ymm1, ymm6		;; R2 + R10 (last R2)
	vsubpd	ymm1, ymm1, ymm6		;; R2 - R10 (last R10)

	vmovapd	ymm6, [dstreg+e1]		;; Reload newer R3
	ystore	[dstreg+e4+e2+32], ymm3		;; Save newer R14
	vmovapd	ymm3, [dstreg+e2+e1]		;; Reload newer R11
	ystore	[dstreg+e4+e2+e1+32], ymm0	;; Save newer R16
	vaddpd	ymm0, ymm6, ymm3		;; R3 + R11 (last R3)
	vsubpd	ymm6, ymm6, ymm3		;; R3 - R11 (last R11)

	vmovapd	ymm3, [dstreg+e1+32]		;; Reload newer R4
	ystore	[dstreg+32], ymm5		;; Save last R9
	vaddpd	ymm5, ymm3, ymm7		;; R4 + R12 (last R4)
	vsubpd	ymm3, ymm3, ymm7		;; R4 - R12 (last R12)

	;; Swizzle using 4 shufpds and 4 perm2f128  (1 or 3 clocks each on port 5, throughput 1)

	L1prefetch srcreg+d4+d1+L1pd, L1pt

	vshufpd	ymm7, ymm4, ymm2, 0		;; Shuffle R1 and R2 to create R1/R2 low
	vshufpd	ymm4, ymm4, ymm2, 15		;; Shuffle R1 and R2 to create R1/R2 hi

	vshufpd	ymm2, ymm0, ymm5, 0		;; Shuffle R3 and R4 to create R3/R4 low
	vshufpd	ymm0, ymm0, ymm5, 15		;; Shuffle R3 and R4 to create R3/R4 hi

	ylow128s ymm5, ymm7, ymm2		;; Shuffle R1/R2 low and R3/R4 low (final R1)
	yhigh128s ymm7, ymm7, ymm2		;; Shuffle R1/R2 low and R3/R4 low (final R3)

	ylow128s ymm2, ymm4, ymm0		;; Shuffle R1/R2 hi and R3/R4 hi (final R2)
	yhigh128s ymm4, ymm4, ymm0		;; Shuffle R1/R2 hi and R3/R4 hi (final R4)

	;; Swizzle using 4 shufpds and 4 perm2f128  (1 or 3 clocks each on port 5, throughput 1)

	vmovapd	ymm0, [dstreg+32]		;; Reload last R9
	ystore	[dstreg], ymm5			;; Save R1

	vshufpd	ymm5, ymm0, ymm1, 0		;; Shuffle R9 and R10 to create R9/R10 low
	vshufpd	ymm0, ymm0, ymm1, 15		;; Shuffle R9 and R10 to create R9/R10 hi

	vshufpd	ymm1, ymm6, ymm3, 0		;; Shuffle R11 and R12 to create R11/R12 low
	vshufpd	ymm6, ymm6, ymm3, 15		;; Shuffle R11 and R12 to create R11/R12 hi

	L1prefetch srcreg+d4+d2+L1pd, L1pt

	ylow128s ymm3, ymm5, ymm1		;; Shuffle R9/R10 low and R11/R12 low (final R9)
	yhigh128s ymm5, ymm5, ymm1		;; Shuffle R9/R10 low and R11/R12 low (final R11)

	ylow128s ymm1, ymm0, ymm6		;; Shuffle R9/R10 hi and R11/R12 hi (final R10)
	yhigh128s ymm0, ymm0, ymm6		;; Shuffle R9/R10 hi and R11/R12 hi (final R12)

	vmovapd	ymm6, [dstreg+e4]		;; Reload newer R5
	ystore	[dstreg+e2], ymm7		;; Save R3
	vmovapd	ymm7, [dstreg+e4+e2]		;; Reload newer R13
	ystore	[dstreg+e1], ymm2		;; Save R2
	vaddpd	ymm2, ymm6, ymm7		;; R5 + R13 (last R5)
	vsubpd	ymm6, ymm6, ymm7		;; R5 - R13 (last R13)

	vmovapd	ymm7, [dstreg+e4+32]		;; Reload newer R6
	ystore	[dstreg+e2+e1], ymm4		;; Save R4
	vmovapd	ymm4, [dstreg+e4+e2+32]		;; Reload newer R14
	ystore	[dstreg+32], ymm3		;; Save R9
	vaddpd	ymm3, ymm7, ymm4		;; R6 + R14 (last R6)
	vsubpd	ymm7, ymm7, ymm4		;; R6 - R14 (last R14)

	vmovapd	ymm4, [dstreg+e4+e1]		;; Reload newer R7
	ystore	[dstreg+e2+32], ymm5		;; Save R11
	vmovapd	ymm5, [dstreg+e4+e2+e1]		;; Reload newer R15
	ystore	[dstreg+e1+32], ymm1		;; Save R10
	vaddpd	ymm1, ymm4, ymm5		;; R7 + R15 (last R7)
	vsubpd	ymm4, ymm4, ymm5		;; R7 - R15 (last R15)

	vmovapd	ymm5, [dstreg+e4+e1+32]		;; Reload newer R8
	ystore	[dstreg+e2+e1+32], ymm0		;; Save R12
	vmovapd	ymm0, [dstreg+e4+e2+e1+32]	;; Reload newer R16
	ystore	[dstreg+e4], ymm6		;; Save last R13
	vaddpd	ymm6, ymm5, ymm0		;; R8 + R16 (last R8)
	vsubpd	ymm5, ymm5, ymm0		;; R8 - R16 (last R16)

	;; Swizzle using 4 shufpds and 4 perm2f128  (1 or 3 clocks each on port 5, throughput 1)

	vshufpd	ymm0, ymm2, ymm3, 0		;; Shuffle R5 and R6 to create R5/R6 low
	vshufpd	ymm2, ymm2, ymm3, 15		;; Shuffle R5 and R6 to create R5/R6 hi

	vshufpd	ymm3, ymm1, ymm6, 0		;; Shuffle R7 and R8 to create R7/R8 low
	vshufpd	ymm1, ymm1, ymm6, 15		;; Shuffle R7 and R8 to create R7/R8 hi

	L1prefetch srcreg+d4+d2+d1+L1pd, L1pt

	ylow128s ymm6, ymm0, ymm3		;; Shuffle R5/R6 low and R7/R8 low (final R5)
	yhigh128s ymm0, ymm0, ymm3		;; Shuffle R5/R6 low and R7/R8 low (final R7)

	ylow128s ymm3, ymm2, ymm1		;; Shuffle R5/R6 hi and R7/R8 hi (final R6)
	yhigh128s ymm2, ymm2, ymm1		;; Shuffle R5/R6 hi and R7/R8 hi (final R8)

	;; Swizzle using 4 shufpds and 4 perm2f128  (1 or 3 clocks each on port 5, throughput 1)

	vmovapd	ymm1, [dstreg+e4]		;; Reload last R13
	ystore	[dstreg+e4], ymm6		;; Save R5
	vshufpd	ymm6, ymm1, ymm7, 0		;; Shuffle R13 and R14 to create R13/R14 low
	vshufpd	ymm1, ymm1, ymm7, 15		;; Shuffle R13 and R14 to create R13/R14 hi

	vshufpd	ymm7, ymm4, ymm5, 0		;; Shuffle R15 and R16 to create R15/R16 low
	vshufpd	ymm4, ymm4, ymm5, 15		;; Shuffle R15 and R16 to create R15/R16 hi

	ylow128s ymm5, ymm6, ymm7		;; Shuffle R13/R14 low and R15/R16 low (final R13)
	yhigh128s ymm6, ymm6, ymm7		;; Shuffle R13/R14 low and R15/R16 low (final R15)

	ylow128s ymm7, ymm1, ymm4		;; Shuffle R13/R14 hi and R15/R16 hi (final R14)
	yhigh128s ymm1, ymm1, ymm4		;; Shuffle R13/R14 hi and R15/R16 hi (final R16)

	ystore	[dstreg+e4+e2], ymm0		;; Save R7
	ystore	[dstreg+e4+e1], ymm3		;; Save R6
	ystore	[dstreg+e4+e2+e1], ymm2		;; Save R8
	ystore	[dstreg+e4+32], ymm5		;; Save R13
	ystore	[dstreg+e4+e2+32], ymm6		;; Save R15
	ystore	[dstreg+e4+e1+32], ymm7		;; Save R14
	ystore	[dstreg+e4+e2+e1+32], ymm1	;; Save R16

	bump	srcreg, srcinc
	bump	dstreg, dstinc
	bump	screg1, scinc1
	bump	screg2, scinc2
	ENDM


;;
;; ********************************* sixteen-reals-eight-complex-fft-with-square variants ***************************************
;;

;; Macro to do a sixteen_reals_fft and three eight_complex_fft in the final levels.
;; The sixteen-reals macro uses the lower double in a YMM register and the three eight-complex
;; use the upper 3 doubles in a YMM  register.  This isn't very efficient, but this macro is called only once.

yr8_8cl_sixteen_reals_eight_complex_fft_final MACRO srcreg,srcinc,d1,d2,d4
	;; Do the sixteen reals part 1
	yr8_16r_simple_fft_part1 srcreg,d1,d2,d4
	;; Do the three eight-complex part
	yr8_8c_simple_fft_part1 srcreg,d1,d2,d4
	yr8_8c_simple_fft_part2 srcreg,d1,d2,d4
	;; Do the sixteen reals part 2
	yr8_16r_simple_fft_part2 srcreg,d1,d2,d4
	bump	srcreg, srcinc
	ENDM

;; Do part 1 of the sixteen_reals_fft writing the results to YMM_TMP memory
;; R1,R2,R3,R4 are written to YMM_TMP1
;; R5,I5,R6,I6 are written to YMM_TMP2
;; R9,I9,R10,I10 are written to YMM_TMP3
;; R11,I11,R12,I12 are written to YMM_TMP4
yr8_16r_simple_fft_part1 MACRO srcreg,d1,d2,d4

	;; Odd levels 1 & 2

	vmovsd	xmm0, Q [srcreg]		;; R1
	vmovsd	xmm7, Q [srcreg+32]		;; R9
	vaddsd	xmm1, xmm0, xmm7		;; R1 + R9 (new R1)
	vsubsd	xmm0, xmm0, xmm7		;; R1 - R9 (new R9)

	vmovsd	xmm2, Q [srcreg+d4]		;; R5
	vmovsd	xmm7, Q [srcreg+d4+32]		;; R13
	vaddsd	xmm3, xmm2, xmm7		;; R5 + R13 (new R5)
	vsubsd	xmm2, xmm2, xmm7		;; R5 - R13 (new R13)

	vmovsd	xmm4, Q [srcreg+d2]		;; R3
	vmovsd	xmm7, Q [srcreg+d2+32]		;; R11
	vaddsd	xmm5, xmm4, xmm7		;; R3 + R11 (new R3)
	vsubsd	xmm4, xmm4, xmm7		;; R3 - R11 (new R11)

	vsubsd	xmm6, xmm1, xmm3		;; R1 - R5 (newer R5 & final R5)
	vaddsd	xmm1, xmm1, xmm3		;; R1 + R5 (newer R1)
	vmovsd	Q YMM_TMP2[0], xmm6		;; Save R5

	vmovsd	xmm7, Q [srcreg+d4+d2]		;; R7
	vmovsd	xmm6, Q [srcreg+d4+d2+32]	;; R15
	vaddsd	xmm3, xmm7, xmm6		;; R7 + R15 (new R7)
	vsubsd	xmm7, xmm7, xmm6		;; R7 - R15 (new R15)

	vsubsd	xmm6, xmm5, xmm3		;; R3 - R7 (newer R7 & final I5)
	vaddsd	xmm5, xmm5, xmm3		;; R3 + R7 (newer R3)
	vmovsd	Q YMM_TMP2[8], xmm6		;; Save I5

						;; R9/R13 morphs into newer R9/I9
						;; R11/R15 morphs into newer R11/I11

	;; Premultipliers for odd level 3

						;; mul R11/I11 by SQRTHALF + i*SQRTHALF
	vsubsd	xmm3, xmm4, xmm7		;; R11 = R11 - I11
	vaddsd	xmm4, xmm4, xmm7		;; I11 = R11 + I11
	vmulsd	xmm3, xmm3, Q YMM_SQRTHALF	;; R11 = R11 * SQRTHALF
	vmulsd	xmm4, xmm4, Q YMM_SQRTHALF	;; I11 = I11 * SQRTHALF

	;; Odd level 3

	vaddsd	xmm7, xmm1, xmm5		;; R1 + R3 (final R1)
	vsubsd	xmm1, xmm1, xmm5		;; R1 - R3 (final R3)
	vmovsd	Q YMM_TMP1[0], xmm7		;; Save R1
	vmovsd	Q YMM_TMP1[16], xmm1		;; Save R3

						;; R5/R7 morphed into final R5/I5

	vaddsd	xmm5, xmm0, xmm3		;; R9 + R11 (final R9)
	vsubsd	xmm0, xmm0, xmm3		;; R9 - R11 (final R11)
	vmovsd	Q YMM_TMP3[0], xmm5		;; Save R9
	vmovsd	Q YMM_TMP4[0], xmm0		;; Save R11

	vaddsd	xmm3, xmm2, xmm4		;; I9 + I11 (final I9)
	vsubsd	xmm2, xmm2, xmm4		;; I9 - I11 (final I11)
	vmovsd	Q YMM_TMP3[8], xmm3		;; Save I9
	vmovsd	Q YMM_TMP4[8], xmm2		;; Save I11

	;; Even levels 1 & 2

	vmovsd	xmm4, Q [srcreg+d2+d1]		;; R4
	vmovsd	xmm7, Q [srcreg+d2+d1+32]	;; R12
	vaddsd	xmm5, xmm4, xmm7		;; R4 + R12 (new R4)
	vsubsd	xmm4, xmm4, xmm7		;; R4 - R12 (new R12)

	vmovsd	xmm6, Q [srcreg+d4+d2+d1]	;; R8
	vmovsd	xmm3, Q [srcreg+d4+d2+d1+32]	;; R16
	vaddsd	xmm7, xmm6, xmm3		;; R8 + R16 (new R8)
	vsubsd	xmm6, xmm6, xmm3		;; R8 - R16 (new R16)

	vaddsd	xmm3, xmm5, xmm7		;; R4 + R8 (newer R4)
	vsubsd	xmm5, xmm5, xmm7		;; R4 - R8 (newer R8)
	vmovsd	Q YMM_TMP8, xmm3		;; Temporarily save newer R4

	vmovsd	xmm0, Q [srcreg+d1]		;; R2
	vmovsd	xmm7, Q [srcreg+d1+32]		;; R10
	vaddsd	xmm1, xmm0, xmm7		;; R2 + R10 (new R2)
	vsubsd	xmm0, xmm0, xmm7		;; R2 - R10 (new R10)

	vmovsd	xmm2, Q [srcreg+d4+d1]		;; R6
	vmovsd	xmm7, Q [srcreg+d4+d1+32]	;; R14
	vaddsd	xmm3, xmm2, xmm7		;; R6 + R14 (new R6)
	vsubsd	xmm2, xmm2, xmm7		;; R6 - R14 (new R14)

	vaddsd	xmm7, xmm1, xmm3		;; R2 + R6 (newer R2)
	vsubsd	xmm1, xmm1, xmm3		;; R2 - R6 (newer R6)

						;; R10/R14 morphs into newer R10/I10
						;; R12/R16 morphs into newer R12/I12

	;; Even level 3

	vaddsd	xmm3, xmm7, Q YMM_TMP8		;; R2 + R4 (final R2)
	vsubsd	xmm7, xmm7, Q YMM_TMP8		;; R2 - R4 (final R4)
	vmovsd	Q YMM_TMP1[8], xmm3		;; Save R2
	vmovsd	Q YMM_TMP1[24], xmm7		;; Save R4

	;; Premultipliers for even level 3

						;; mul R10/I10 by w^1 = .924 + .383i
	vmulsd	xmm3, xmm0, Q YMM_P924
	vmulsd	xmm7, xmm2, Q YMM_P383
	vsubsd	xmm3, xmm3, xmm7		;; Twiddled R10
	vmulsd	xmm0, xmm0, Q YMM_P383
	vmulsd	xmm2, xmm2, Q YMM_P924
	vaddsd	xmm0, xmm0, xmm2		;; Twiddled I10

						;; mul R12/I12 by w^3 = .383 + .924i
	vmulsd	xmm2, xmm4, Q YMM_P383
	vmulsd	xmm7, xmm6, Q YMM_P924
	vsubsd	xmm2, xmm2, xmm7		;; Twiddled R12
	vmulsd	xmm4, xmm4, Q YMM_P924
	vmulsd	xmm6, xmm6, Q YMM_P383
	vaddsd	xmm4, xmm4, xmm6		;; Twiddled I12

	;; More even level 3

						;; R6/R8 morph into newer R6/I6

	vaddsd	xmm6, xmm3, xmm2		;; R10 + R12 (final R10)
	vsubsd	xmm3, xmm3, xmm2		;; R10 - R12 (final R12)
	vmovsd	Q YMM_TMP3[16], xmm6		;; Save R10
	vmovsd	Q YMM_TMP4[16], xmm3		;; Save R12

	vaddsd	xmm2, xmm0, xmm4		;; I10 + I12 (final I10)
	vsubsd	xmm0, xmm0, xmm4		;; I10 - I12 (final I12)
	vmovsd	Q YMM_TMP3[24], xmm2		;; Save I10
	vmovsd	Q YMM_TMP4[24], xmm0		;; Save I12

	;; Premultipliers for even level 4

						;; mul R6/I6 by w^2 = .707 + .707i
	vsubsd	xmm0, xmm1, xmm5		;; R6 = R6 - I6
	vaddsd	xmm1, xmm1, xmm5		;; I6 = R6 + I6
	vmulsd	xmm0, xmm0, Q YMM_SQRTHALF	;; R6 = R6 * SQRTHALF (final R6)
	vmulsd	xmm1, xmm1, Q YMM_SQRTHALF	;; I6 = I6 * SQRTHALF (final I6)
	vmovsd	Q YMM_TMP2[16], xmm0		;; Save R6
	vmovsd	Q YMM_TMP2[24], xmm1		;; Save I6
	ENDM

yr8_16r_simple_fft_part2 MACRO srcreg,d1,d2,d4

	vmovsd	xmm0, Q YMM_TMP1[0]		;; R1
	vmovsd	xmm7, Q YMM_TMP1[8] 		;; R2
	vsubsd	xmm1, xmm0, xmm7		;; R1 - R2 (new R2)
	vaddsd	xmm0, xmm0, xmm7		;; R1 + R2 (new R1)

	vmovsd	xmm2, Q YMM_TMP1[16]		;; R3/R4 morphs into R3/I3
	vmovsd	xmm3, Q YMM_TMP1[24]

	vmovsd	Q [srcreg], xmm0		;; Save R1
	vmovsd	Q [srcreg+32], xmm1		;; Save R2
	vmovsd	Q [srcreg+d1], xmm2		;; Save R3
	vmovsd	Q [srcreg+d1+32], xmm3		;; Save I3

	vmovsd	xmm0, Q YMM_TMP2[0]		;; R5
	vmovsd	xmm7, Q YMM_TMP2[16] 		;; R6
	vsubsd	xmm1, xmm0, xmm7		;; R5 - R6 (new R6)
	vaddsd	xmm0, xmm0, xmm7		;; R5 + R6 (new R5)

	vmovsd	xmm2, Q YMM_TMP2[8]		;; I5
	vmovsd	xmm7, Q YMM_TMP2[24] 		;; I6
	vsubsd	xmm3, xmm2, xmm7		;; I5 - I6 (new I6)
	vaddsd	xmm2, xmm2, xmm7		;; I5 + I6 (new I5)

	vmovsd	Q [srcreg+d2+d1], xmm1		;; Save R6
	vmovsd	Q [srcreg+d2], xmm0		;; Save R5
	vmovsd	Q [srcreg+d2+d1+32], xmm3	;; Save I6
	vmovsd	Q [srcreg+d2+32], xmm2		;; Save I5

	vmovsd	xmm0, Q YMM_TMP3[0]		;; R9
	vmovsd	xmm7, Q YMM_TMP3[16] 		;; R10
	vsubsd	xmm1, xmm0, xmm7		;; R9 - R10 (new R10)
	vaddsd	xmm0, xmm0, xmm7		;; R9 + R10 (new R9)

	vmovsd	xmm2, Q YMM_TMP3[8]		;; I9
	vmovsd	xmm7, Q YMM_TMP3[24] 		;; I10
	vsubsd	xmm3, xmm2, xmm7		;; I9 - I10 (new I10)
	vaddsd	xmm2, xmm2, xmm7		;; I9 + I10 (new I9)

	vmovsd	Q [srcreg+d4+d1], xmm1		;; Save R10
	vmovsd	Q [srcreg+d4], xmm0		;; Save R9
	vmovsd	Q [srcreg+d4+d1+32], xmm3	;; Save I10
	vmovsd	Q [srcreg+d4+32], xmm2		;; Save I9

	vmovsd	xmm0, Q YMM_TMP4[0]		;; R11
	vmovsd	xmm7, Q YMM_TMP4[24] 		;; I12
	vsubsd	xmm1, xmm0, xmm7		;; R11 - I12 (new R11)
	vaddsd	xmm0, xmm0, xmm7		;; R11 + I12 (new R12)

	vmovsd	xmm2, Q YMM_TMP4[8]		;; I11
	vmovsd	xmm7, Q YMM_TMP4[16] 		;; R12
	vsubsd	xmm3, xmm2, xmm7		;; I11 - R12 (new I12)
	vaddsd	xmm2, xmm2, xmm7		;; I11 + R12 (new I11)

	vmovsd	Q [srcreg+d4+d2], xmm1		;; Save R11
	vmovsd	Q [srcreg+d4+d2+d1], xmm0	;; Save R12
	vmovsd	Q [srcreg+d4+d2+d1+32], xmm3	;; Save I12
	vmovsd	Q [srcreg+d4+d2+32], xmm2	;; Save I11
	ENDM


yr8_8cl_sixteen_reals_eight_complex_with_square MACRO srcreg,srcinc,d1,d2,d4
	;; Do the sixteen reals part 1
	yr8_16r_simple_fft_part1 srcreg,d1,d2,d4
	;; Do the three eight-complex part
	yr8_8c_simple_fft_part1 srcreg,d1,d2,d4
	yr8_8c_simple_fft_with_square srcreg,d1,d2,d4
	yr8_8c_simple_unfft srcreg,d1,d2,d4
	;; Do the remaining sixteen reals work
	yr8_16r_simple_fft_with_square srcreg,d1,d2,d4
	yr8_16r_simple_unfft srcreg,d1,d2,d4
	bump	srcreg, srcinc
	ENDM

yr8_16r_simple_fft_with_square MACRO srcreg,d1,d2,d4
	vmovsd	xmm0, Q YMM_TMP1[0]		;; R1
	vmovsd	xmm7, Q YMM_TMP1[8] 		;; R2
	vsubsd	xmm1, xmm0, xmm7		;; R1 - R2 (new R2)
	vaddsd	xmm0, xmm0, xmm7		;; R1 + R2 (new R1)

	vmulsd	xmm0, xmm0, xmm0		;; Square R1
	vmulsd	xmm1, xmm1, xmm1		;; Square R2
	vmovsd	Q [srcreg-16], xmm0		;; Save square of sum of FFT values
	vsubsd	xmm0, xmm0, xmm1		;; R1 - R2 (final R2)
	vmulsd	xmm0, xmm0, Q YMM_HALF		;; Mul R2 by HALF
	vaddsd	xmm1, xmm1, xmm0		;; R1 + R2 (final R1)

	vmovsd	xmm2, Q YMM_TMP1[16]		;; R3/R4 morphs into R3/I3
	vmovsd	xmm3, Q YMM_TMP1[24]

	ys_complex_square xmm2, xmm3, xmm7	;; Square R3/I3

						;; R3/I3 morphs into R3/R4

	vmovsd	Q YMM_TMP1[0], xmm1		;; Save R1
	vmovsd	Q YMM_TMP1[8], xmm0		;; Save R2
	vmovsd	Q YMM_TMP1[16], xmm2		;; Save R3
	vmovsd	Q YMM_TMP1[24], xmm3		;; Save R4

	vmovsd	xmm0, Q YMM_TMP2[0]		;; R5
	vmovsd	xmm7, Q YMM_TMP2[16] 		;; R6
	vsubsd	xmm1, xmm0, xmm7		;; R5 - R6 (new R6)
	vaddsd	xmm0, xmm0, xmm7		;; R5 + R6 (new R5)

	vmovsd	xmm2, Q YMM_TMP2[8]		;; I5
	vmovsd	xmm7, Q YMM_TMP2[24] 		;; I6
	vsubsd	xmm3, xmm2, xmm7		;; I5 - I6 (new I6)
	vaddsd	xmm2, xmm2, xmm7		;; I5 + I6 (new I5)

	ys_complex_square xmm0, xmm2, xmm7	;; Square R5/I5
	ys_complex_square xmm1, xmm3, xmm7	;; Square R6/I6

	vaddsd	xmm4, xmm0, xmm1		;; R5 + R6 (new R5)
	vsubsd	xmm0, xmm0, xmm1		;; R5 - R6 (new R6)

	vaddsd	xmm5, xmm2, xmm3		;; I5 + I6 (new I5)
	vsubsd	xmm2, xmm2, xmm3		;; I5 - I6 (new I6)

	vmovsd	Q YMM_TMP2[0], xmm4		;; Save R5
	vmovsd	Q YMM_TMP2[16], xmm0		;; Save R6
	vmovsd	Q YMM_TMP2[8], xmm5		;; Save I5
	vmovsd	Q YMM_TMP2[24], xmm2		;; Save I6

	vmovsd	xmm0, Q YMM_TMP3[0]		;; R9
	vmovsd	xmm7, Q YMM_TMP3[16] 		;; R10
	vsubsd	xmm1, xmm0, xmm7		;; R9 - R10 (new R10)
	vaddsd	xmm0, xmm0, xmm7		;; R9 + R10 (new R9)

	vmovsd	xmm2, Q YMM_TMP3[8]		;; I9
	vmovsd	xmm7, Q YMM_TMP3[24] 		;; I10
	vsubsd	xmm3, xmm2, xmm7		;; I9 - I10 (new I10)
	vaddsd	xmm2, xmm2, xmm7		;; I9 + I10 (new I9)

	ys_complex_square xmm0, xmm2, xmm7	;; Square R9/I9
	ys_complex_square xmm1, xmm3, xmm7	;; Square R10/I10

	vaddsd	xmm4, xmm0, xmm1		;; R9 + R10 (new R9)
	vsubsd	xmm0, xmm0, xmm1		;; R9 - R10 (new R10)

	vaddsd	xmm5, xmm2, xmm3		;; I9 + I10 (new I9)
	vsubsd	xmm2, xmm2, xmm3		;; I9 - I10 (new I10)

	vmovsd	Q YMM_TMP3[0], xmm4		;; Save R9
	vmovsd	Q YMM_TMP3[16], xmm0		;; Save R10
	vmovsd	Q YMM_TMP3[8], xmm5		;; Save I9
	vmovsd	Q YMM_TMP3[24], xmm2		;; Save I10

	vmovsd	xmm0, Q YMM_TMP4[0]		;; R11
	vmovsd	xmm7, Q YMM_TMP4[24] 		;; I12
	vsubsd	xmm1, xmm0, xmm7		;; R11 - I12 (new R11)
	vaddsd	xmm0, xmm0, xmm7		;; R11 + I12 (new R12)

	vmovsd	xmm2, Q YMM_TMP4[8]		;; I11
	vmovsd	xmm7, Q YMM_TMP4[16] 		;; R12
	vsubsd	xmm3, xmm2, xmm7		;; I11 - R12 (new I12)
	vaddsd	xmm2, xmm2, xmm7		;; I11 + R12 (new I11)

	ys_complex_square xmm1, xmm2, xmm7	;; Square R11/I11
	ys_complex_square xmm0, xmm3, xmm7	;; Square R12/I12

	vaddsd	xmm4, xmm0, xmm1		;; R12 + R11 (new R11)
	vsubsd	xmm0, xmm0, xmm1		;; R12 - R11 (new I12)

	vaddsd	xmm5, xmm2, xmm3		;; I11 + I12 (new I11)
	vsubsd	xmm2, xmm2, xmm3		;; I11 - I12 (new R12)

	vmovsd	Q YMM_TMP4[0], xmm4		;; Save R11
	vmovsd	Q YMM_TMP4[24], xmm0		;; Save I12
	vmovsd	Q YMM_TMP4[8], xmm5		;; Save I11
	vmovsd	Q YMM_TMP4[16], xmm2		;; Save R12
	ENDM

yr8_16r_simple_unfft MACRO srcreg,d1,d2,d4

	;; Even level 3

	vmovsd	xmm4, Q YMM_TMP3[16] 		;; R10
	vmovsd	xmm7, Q YMM_TMP4[16]		;; R12
	vaddsd	xmm0, xmm4, xmm7		;; R10 + R12 (new R10)
	vsubsd	xmm4, xmm4, xmm7		;; R10 - R12 (new R12)

	vmovsd	xmm6, Q YMM_TMP3[24]		;; I10
	vmovsd	xmm7, Q YMM_TMP4[24]		;; I12
	vaddsd	xmm2, xmm6, xmm7		;; I10 + I12 (new I10)
	vsubsd	xmm6, xmm6, xmm7		;; I10 - I12 (new I12)

	vmovsd	xmm7, Q YMM_TMP1[8]		;; R2
	vmovsd	xmm5, Q YMM_TMP1[24] 		;; R4
	vaddsd	xmm3, xmm7, xmm5		;; R2 + R4 (new R2)
	vsubsd	xmm7, xmm7, xmm5		;; R2 - R4 (new R4)

	;; Premultipliers for even level 3

						;; mul R10/I10 by w^1 = .924 - .383i
	vmulsd	xmm1, xmm0, Q YMM_P924
	vmulsd	xmm5, xmm2, Q YMM_P383
	vaddsd	xmm1, xmm1, xmm5		;; Twiddled R10
	vmulsd	xmm0, xmm0, Q YMM_P383
	vmulsd	xmm2, xmm2, Q YMM_P924
	vsubsd	xmm0, xmm2, xmm0		;; Twiddled I10

						;; mul R12/I12 by w^3 = .383 - .924i
	vmulsd	xmm5, xmm4, Q YMM_P383
	vmulsd	xmm2, xmm6, Q YMM_P924
	vaddsd	xmm5, xmm5, xmm2		;; Twiddled R12
	vmulsd	xmm4, xmm4, Q YMM_P924
	vmulsd	xmm6, xmm6, Q YMM_P383
	vsubsd	xmm4, xmm6, xmm4		;; Twiddled I12

	vmovsd	Q YMM_TMP8, xmm5		;; Temporarily save R12

	;; Premultipliers for even level 4

						;; mul R6/I6 by w^2 = .707 - .707i
	vmovsd	xmm5, Q YMM_TMP2[24]		;; I6
	vmovsd	xmm2, Q YMM_TMP2[16] 		;; R6
	vaddsd	xmm6, xmm5, xmm2		;; R6 = I6 + R6
	vsubsd	xmm5, xmm5, xmm2		;; I6 = I6 - R6
	vmulsd	xmm6, xmm6, Q YMM_SQRTHALF	;; R6 = R6 * SQRTHALF
	vmulsd	xmm5, xmm5, Q YMM_SQRTHALF	;; I6 = I6 * SQRTHALF

	;; Even level 3

						;; R6/I6 morph into new R6/R8

	;; Even level 2

	vaddsd	xmm2, xmm3, xmm6		;; R2 + R6 (newer R2)
	vsubsd	xmm3, xmm3, xmm6		;; R2 - R6 (newer R6)

	vaddsd	xmm6, xmm7, xmm5		;; R4 + R8 (newer R4)
	vsubsd	xmm7, xmm7, xmm5		;; R4 - R8 (newer R8)

						;; R10/I10 morphs into newer R10/R14
						;; R12/I12 morphs into newer R12/R16

	;; Even level 1

	vaddsd	xmm5, xmm2, xmm1		;; R2 + R10 (new R2)
	vsubsd	xmm2, xmm2, xmm1		;; R2 - R10 (new R10)

	vaddsd	xmm1, xmm3, xmm0		;; R6 + R14 (new R6)
	vsubsd	xmm3, xmm3, xmm0		;; R6 - R14 (new R14)

	vmovsd	Q [srcreg+d1], xmm5		;; R2
	vmovsd	Q [srcreg+d1+32], xmm2		;; R10
	vmovsd	Q [srcreg+d4+d1], xmm1		;; R6
	vmovsd	Q [srcreg+d4+d1+32], xmm3	;; R14

	vmovsd	xmm1, Q YMM_TMP8		;; Reload saved R12

	vaddsd	xmm2, xmm6, xmm1		;; R4 + R12 (new R4)
	vsubsd	xmm6, xmm6, xmm1		;; R4 - R12 (new R12)

	vaddsd	xmm3, xmm7, xmm4		;; R8 + R16 (new R8)
	vsubsd	xmm7, xmm7, xmm4		;; R8 - R16 (new R16)

	vmovsd	Q [srcreg+d2+d1], xmm2		;; R4
	vmovsd	Q [srcreg+d2+d1+32], xmm6	;; R12
	vmovsd	Q [srcreg+d4+d2+d1], xmm3	;; R8
	vmovsd	Q [srcreg+d4+d2+d1+32], xmm7	;; R16

	;; Odd level 3

	vmovsd	xmm4, Q YMM_TMP3[0]		;; R9
	vmovsd	xmm7, Q YMM_TMP4[0] 		;; R11
	vaddsd	xmm0, xmm4, xmm7		;; R9 + R11 (new R9)
	vsubsd	xmm4, xmm4, xmm7		;; R9 - R11 (new R11)

	vmovsd	xmm6, Q YMM_TMP3[8] 		;; I9
	vmovsd	xmm7, Q YMM_TMP4[8]		;; I11
	vaddsd	xmm2, xmm6, xmm7		;; I9 + I11 (new I9)
	vsubsd	xmm6, xmm6, xmm7		;; I9 - I11 (new I11)

	vmovsd	xmm7, Q YMM_TMP1[0]		;; R1
	vmovsd	xmm1, Q YMM_TMP1[16] 		;; R3
	vaddsd	xmm3, xmm7, xmm1		;; R1 + R3 (new R1)
	vsubsd	xmm7, xmm7, xmm1		;; R1 - R3 (new R3)

						;; R5/I5 morphs into new R5/R7

	;; Premultipliers for odd level 3

						;; mul R11/I11 by SQRTHALF - i*SQRTHALF
	vaddsd	xmm1, xmm6, xmm4		;; R11 = I11 + R11
	vsubsd	xmm6, xmm6, xmm4		;; I11 = I11 - R11
	vmulsd	xmm1, xmm1, Q YMM_SQRTHALF	;; R11 = R11 * SQRTHALF
	vmulsd	xmm6, xmm6, Q YMM_SQRTHALF	;; I11 = I11 * SQRTHALF

	;; Odd level 2

	vmovsd	xmm4, Q YMM_TMP2[0] 		;; R5 (a.k.a. new R5)
	vaddsd	xmm5, xmm3, xmm4		;; R1 + R5 (newer R1)
	vsubsd	xmm3, xmm3, xmm4		;; R1 - R5 (newer R5)

						;; R9/I9 morphs into newer R9/R13
						;; R11/I11 morphs into newer R11/R15

	;; Odd level 1

	vaddsd	xmm4, xmm5, xmm0		;; R1 + R9 (new R1)
	vsubsd	xmm5, xmm5, xmm0		;; R1 - R9 (new R9)

	vaddsd	xmm0, xmm3, xmm2		;; R5 + R13 (new R5)
	vsubsd	xmm3, xmm3, xmm2		;; R5 - R13 (new R13)

	vmovsd	Q [srcreg], xmm4		;; R1
	vmovsd	Q [srcreg+32], xmm5		;; R9
	vmovsd	Q [srcreg+d4], xmm0		;; R5
	vmovsd	Q [srcreg+d4+32], xmm3		;; R13

	;; Odd level 2

	vmovsd	xmm5, Q YMM_TMP2[8] 		;; I5 (a.k.a new R7)
	vaddsd	xmm0, xmm7, xmm5		;; R3 + R7 (newer R3)
	vsubsd	xmm7, xmm7, xmm5		;; R3 - R7 (newer R7)

	;; Odd level 1

	vaddsd	xmm5, xmm0, xmm1		;; R3 + R11 (new R3)
	vsubsd	xmm0, xmm0, xmm1		;; R3 - R11 (new R11)

	vaddsd	xmm1, xmm7, xmm6		;; R7 + R15 (new R7)
	vsubsd	xmm7, xmm7, xmm6		;; R7 - R15 (new R15)

	vmovsd	Q [srcreg+d2], xmm5		;; R3
	vmovsd	Q [srcreg+d2+32], xmm0		;; R11
	vmovsd	Q [srcreg+d4+d2], xmm1		;; R7
	vmovsd	Q [srcreg+d4+d2+32], xmm7	;; R15
	ENDM


yr8_8cl_sixteen_reals_eight_complex_with_mult MACRO srcreg,srcinc,d1,d2,d4
	;; Do the sixteen reals part 1
	yr8_16r_simple_fft_part1 srcreg,d1,d2,d4
	;; Do the three eight-complex part
	yr8_8c_simple_fft_part1 srcreg,d1,d2,d4
	yr8_8c_simple_fft_with_mult srcreg,srcreg+rbp,d1,d2,d4
	yr8_8c_simple_unfft srcreg,d1,d2,d4
	;; Do the remaining sixteen reals work
	yr8_16r_simple_fft_with_mult srcreg,srcreg+rbp,d1,d2,d4
	yr8_16r_simple_unfft srcreg,d1,d2,d4
	bump	srcreg, srcinc
	ENDM

yr8_16r_simple_fft_with_mult MACRO srcreg,altsrc,d1,d2,d4
	vmovsd	xmm0, Q YMM_TMP1[0]		;; R1
	vmovsd	xmm7, Q YMM_TMP1[8] 		;; R2
	vsubsd	xmm1, xmm0, xmm7		;; R1 - R2 (new R2)
	vaddsd	xmm0, xmm0, xmm7		;; R1 + R2 (new R1)

	vmulsd	xmm0, xmm0, Q [altsrc]		;; R1 * R1-from-mem
	vmulsd	xmm1, xmm1, Q [altsrc+32]	;; R2 * R2-from-mem
	vmovsd	Q [srcreg-16], xmm0		;; Save product of sum of FFT values
	vsubsd	xmm0, xmm0, xmm1		;; R1 - R2 (final R2)
	vmulsd	xmm0, xmm0, Q YMM_HALF		;; Mul R2 by HALF
	vaddsd	xmm1, xmm1, xmm0		;; R1 + R2 (final R1)

	vmovsd	xmm2, Q YMM_TMP1[16]		;; R3/R4 morphs into R3/I3
	vmovsd	xmm3, Q YMM_TMP1[24]

	ys_complex_mult xmm2, xmm3, Q [altsrc+d1], Q [altsrc+d1+32], xmm6, xmm7 ;; Mult R3/I3

						;; R3/I3 morphs into R3/R4

	vmovsd	Q YMM_TMP1[0], xmm1		;; Save R1
	vmovsd	Q YMM_TMP1[8], xmm0		;; Save R2
	vmovsd	Q YMM_TMP1[16], xmm2		;; Save R3
	vmovsd	Q YMM_TMP1[24], xmm3		;; Save R4

	vmovsd	xmm0, Q YMM_TMP2[0]		;; R5
	vmovsd	xmm7, Q YMM_TMP2[16] 		;; R6
	vsubsd	xmm1, xmm0, xmm7		;; R5 - R6 (new R6)
	vaddsd	xmm0, xmm0, xmm7		;; R5 + R6 (new R5)

	vmovsd	xmm2, Q YMM_TMP2[8]		;; I5
	vmovsd	xmm7, Q YMM_TMP2[24] 		;; I6
	vsubsd	xmm3, xmm2, xmm7		;; I5 - I6 (new I6)
	vaddsd	xmm2, xmm2, xmm7		;; I5 + I6 (new I5)

	ys_complex_mult xmm0, xmm2, Q [altsrc+d2], Q [altsrc+d2+32], xmm6, xmm7 ;; Mult R5/I5
	ys_complex_mult xmm1, xmm3, Q [altsrc+d2+d1], Q [altsrc+d2+d1+32], xmm6, xmm7 ;; Mult R6/I6

	vaddsd	xmm4, xmm0, xmm1		;; R5 + R6 (new R5)
	vsubsd	xmm0, xmm0, xmm1		;; R5 - R6 (new R6)

	vaddsd	xmm5, xmm2, xmm3		;; I5 + I6 (new I5)
	vsubsd	xmm2, xmm2, xmm3		;; I5 - I6 (new I6)

	vmovsd	Q YMM_TMP2[0], xmm4		;; Save R5
	vmovsd	Q YMM_TMP2[16], xmm0		;; Save R6
	vmovsd	Q YMM_TMP2[8], xmm5		;; Save I5
	vmovsd	Q YMM_TMP2[24], xmm2		;; Save I6

	vmovsd	xmm0, Q YMM_TMP3[0]		;; R9
	vmovsd	xmm7, Q YMM_TMP3[16] 		;; R10
	vsubsd	xmm1, xmm0, xmm7		;; R9 - R10 (new R10)
	vaddsd	xmm0, xmm0, xmm7		;; R9 + R10 (new R9)

	vmovsd	xmm2, Q YMM_TMP3[8]		;; I9
	vmovsd	xmm7, Q YMM_TMP3[24] 		;; I10
	vsubsd	xmm3, xmm2, xmm7		;; I9 - I10 (new I10)
	vaddsd	xmm2, xmm2, xmm7		;; I9 + I10 (new I9)

	ys_complex_mult xmm0, xmm2, Q [altsrc+d4], Q [altsrc+d4+32], xmm6, xmm7 ;; Mult R9/I9
	ys_complex_mult xmm1, xmm3, Q [altsrc+d4+d1], Q [altsrc+d4+d1+32], xmm6, xmm7 ;; Mult R10/I10

	vaddsd	xmm4, xmm0, xmm1		;; R9 + R10 (new R9)
	vsubsd	xmm0, xmm0, xmm1		;; R9 - R10 (new R10)

	vaddsd	xmm5, xmm2, xmm3		;; I9 + I10 (new I9)
	vsubsd	xmm2, xmm2, xmm3		;; I9 - I10 (new I10)

	vmovsd	Q YMM_TMP3[0], xmm4		;; Save R9
	vmovsd	Q YMM_TMP3[16], xmm0		;; Save R10
	vmovsd	Q YMM_TMP3[8], xmm5		;; Save I9
	vmovsd	Q YMM_TMP3[24], xmm2		;; Save I10

	vmovsd	xmm0, Q YMM_TMP4[0]		;; R11
	vmovsd	xmm7, Q YMM_TMP4[24] 		;; I12
	vsubsd	xmm1, xmm0, xmm7		;; R11 - I12 (new R11)
	vaddsd	xmm0, xmm0, xmm7		;; R11 + I12 (new R12)

	vmovsd	xmm2, Q YMM_TMP4[8]		;; I11
	vmovsd	xmm7, Q YMM_TMP4[16] 		;; R12
	vsubsd	xmm3, xmm2, xmm7		;; I11 - R12 (new I12)
	vaddsd	xmm2, xmm2, xmm7		;; I11 + R12 (new I11)

	ys_complex_mult xmm1, xmm2, Q [altsrc+d4+d2], Q [altsrc+d4+d2+32], xmm6, xmm7 ;; Mult R11/I11
	ys_complex_mult xmm0, xmm3, Q [altsrc+d4+d2+d1], Q [altsrc+d4+d2+d1+32], xmm6, xmm7 ;; Mult R12/I12

	vaddsd	xmm4, xmm0, xmm1		;; R12 + R11 (new R11)
	vsubsd	xmm0, xmm0, xmm1		;; R12 - R11 (new I12)

	vaddsd	xmm5, xmm2, xmm3		;; I11 + I12 (new I11)
	vsubsd	xmm2, xmm2, xmm3		;; I11 - I12 (new R12)

	vmovsd	Q YMM_TMP4[0], xmm4		;; Save R11
	vmovsd	Q YMM_TMP4[24], xmm0		;; Save I12
	vmovsd	Q YMM_TMP4[8], xmm5		;; Save I11
	vmovsd	Q YMM_TMP4[16], xmm2		;; Save R12
	ENDM


yr8_8cl_sixteen_reals_eight_complex_with_mulf MACRO srcreg,srcinc,d1,d2,d4
	;; Start the sixteen reals work
	yr8_16r_simple_fft_with_mulf srcreg,d1,d2,d4
	;; Do the three eight-complex part
	yr8_8c_simple_fft_with_mulf srcreg,d1,d2,d4
	yr8_8c_simple_unfft srcreg,d1,d2,d4
	;; Finish the sixteen reals work
	yr8_16r_simple_unfft srcreg,d1,d2,d4
	bump	srcreg, srcinc
	ENDM

yr8_16r_simple_fft_with_mulf MACRO srcreg,d1,d2,d4
	vmovsd	xmm0, Q [srcreg][rbx]		;; R1
	vmovsd	xmm1, Q [srcreg+32][rbx]	;; R2

	vmulsd	xmm0, xmm0, Q [srcreg][rbp]	;; R1 * R1-from-mem
	vmulsd	xmm1, xmm1, Q [srcreg+32][rbp]	;; R2 * R2-from-mem
	vmovsd	Q [srcreg-16], xmm0		;; Save product of sum of FFT values
	vsubsd	xmm0, xmm0, xmm1		;; R1 - R2 (final R2)
	vmulsd	xmm0, xmm0, Q YMM_HALF		;; Mul R2 by HALF
	vaddsd	xmm1, xmm1, xmm0		;; R1 + R2 (final R1)

	vmovsd	xmm2, Q [srcreg+d1][rbx]	;; R3/R4 morphs into R3/I3
	vmovsd	xmm3, Q [srcreg+d1+32][rbx]

	ys_complex_mult xmm2, xmm3, Q [srcreg+d1][rbp], Q [srcreg+d1+32][rbp], xmm6, xmm7 ;; Mult R3/I3

						;; R3/I3 morphs into R3/R4

	vmovsd	Q YMM_TMP1[0], xmm1		;; Save R1
	vmovsd	Q YMM_TMP1[8], xmm0		;; Save R2
	vmovsd	Q YMM_TMP1[16], xmm2		;; Save R3
	vmovsd	Q YMM_TMP1[24], xmm3		;; Save R4

	vmovsd	xmm0, Q [srcreg+d2][rbx]	;; R5
	vmovsd	xmm2, Q [srcreg+d2+32][rbx]	;; I5
	vmovsd	xmm1, Q [srcreg+d2+d1][rbx]	;; R6
	vmovsd	xmm3, Q [srcreg+d2+d1+32][rbx]	;; I6

	ys_complex_mult xmm0, xmm2, Q [srcreg+d2][rbp], Q [srcreg+d2+32][rbp], xmm6, xmm7 ;; Mult R5/I5
	ys_complex_mult xmm1, xmm3, Q [srcreg+d2+d1][rbp], Q [srcreg+d2+d1+32][rbp], xmm6, xmm7 ;; Mult R6/I6

	vaddsd	xmm4, xmm0, xmm1		;; R5 + R6 (new R5)
	vsubsd	xmm0, xmm0, xmm1		;; R5 - R6 (new R6)

	vaddsd	xmm5, xmm2, xmm3		;; I5 + I6 (new I5)
	vsubsd	xmm2, xmm2, xmm3		;; I5 - I6 (new I6)

	vmovsd	Q YMM_TMP2[0], xmm4		;; Save R5
	vmovsd	Q YMM_TMP2[16], xmm0		;; Save R6
	vmovsd	Q YMM_TMP2[8], xmm5		;; Save I5
	vmovsd	Q YMM_TMP2[24], xmm2		;; Save I6

	vmovsd	xmm0, Q [srcreg+d4][rbx]	;; R9
	vmovsd	xmm2, Q [srcreg+d4+32][rbx]	;; I9
	vmovsd	xmm1, Q [srcreg+d4+d1][rbx]	;; R10
	vmovsd	xmm3, Q [srcreg+d4+d1+32][rbx]	;; I10

	ys_complex_mult xmm0, xmm2, Q [srcreg+d4][rbp], Q [srcreg+d4+32][rbp], xmm6, xmm7 ;; Mult R9/I9
	ys_complex_mult xmm1, xmm3, Q [srcreg+d4+d1][rbp], Q [srcreg+d4+d1+32][rbp], xmm6, xmm7 ;; Mult R10/I10

	vaddsd	xmm4, xmm0, xmm1		;; R9 + R10 (new R9)
	vsubsd	xmm0, xmm0, xmm1		;; R9 - R10 (new R10)

	vaddsd	xmm5, xmm2, xmm3		;; I9 + I10 (new I9)
	vsubsd	xmm2, xmm2, xmm3		;; I9 - I10 (new I10)

	vmovsd	Q YMM_TMP3[0], xmm4		;; Save R9
	vmovsd	Q YMM_TMP3[16], xmm0		;; Save R10
	vmovsd	Q YMM_TMP3[8], xmm5		;; Save I9
	vmovsd	Q YMM_TMP3[24], xmm2		;; Save I10

	vmovsd	xmm1, Q [srcreg+d4+d2][rbx]	;; R11
	vmovsd	xmm2, Q [srcreg+d4+d2+32][rbx]	;; I11
	vmovsd	xmm0, Q [srcreg+d4+d2+d1][rbx] 	;; R12
	vmovsd	xmm3, Q [srcreg+d4+d2+d1+32][rbx] ;; I12

	ys_complex_mult xmm1, xmm2, Q [srcreg+d4+d2][rbp], Q [srcreg+d4+d2+32][rbp], xmm6, xmm7 ;; Mult R11/I11
	ys_complex_mult xmm0, xmm3, Q [srcreg+d4+d2+d1][rbp], Q [srcreg+d4+d2+d1+32][rbp], xmm6, xmm7 ;; Mult R12/I12

	vaddsd	xmm4, xmm0, xmm1		;; R12 + R11 (new R11)
	vsubsd	xmm0, xmm0, xmm1		;; R12 - R11 (new I12)

	vaddsd	xmm5, xmm2, xmm3		;; I11 + I12 (new I11)
	vsubsd	xmm2, xmm2, xmm3		;; I11 - I12 (new R12)

	vmovsd	Q YMM_TMP4[0], xmm4		;; Save R11
	vmovsd	Q YMM_TMP4[24], xmm0		;; Save I12
	vmovsd	Q YMM_TMP4[8], xmm5		;; Save I11
	vmovsd	Q YMM_TMP4[16], xmm2		;; Save R12
	ENDM

;; 64-bit implementation

IFDEF X86_64

yr8_8cl_sixteen_reals_eight_complex_with_mult MACRO srcreg,srcinc,d1,d2,d4
	LOCAL	orig, back_to_orig, nofftsave, disptab, addmul, submul, muladd, muladdhard, mulsub, mulsubhard, unfft, done
	LOCAL	addmul312, addmul313, addmul132, addmul133, submul312, submul313, submul132, submul133

	;; Do the sixteen reals

	;; Odd level 1

	vmovsd	xmm0, Q [srcreg]		;; R1
	vmovsd	xmm7, Q [srcreg+32]		;; R9
	vaddsd	xmm1, xmm0, xmm7		;; R1 + R9 (new R1)
	vsubsd	xmm0, xmm0, xmm7		;; R1 - R9 (new R9)

	vmovsd	xmm2, Q [srcreg+d4]		;; R5
	vmovsd	xmm7, Q [srcreg+d4+32]		;; R13
	vaddsd	xmm3, xmm2, xmm7		;; R5 + R13 (new R5)
	vsubsd	xmm2, xmm2, xmm7		;; R5 - R13 (new R13)

	vmovsd	xmm4, Q [srcreg+d2]		;; R3
	vmovsd	xmm7, Q [srcreg+d2+32]		;; R11
	vaddsd	xmm5, xmm4, xmm7		;; R3 + R11 (new R3)
	vsubsd	xmm4, xmm4, xmm7		;; R3 - R11 (new R11)

	vmovsd	xmm6, Q [srcreg+d4+d2]		;; R7
	vmovsd	xmm8, Q [srcreg+d4+d2+32]	;; R15
	vaddsd	xmm7, xmm6, xmm8		;; R7 + R15 (new R7)
	vsubsd	xmm6, xmm6, xmm8		;; R7 - R15 (new R15)

	;; Odd level 2

	vsubsd	xmm8, xmm1, xmm3		;; R1 - R5 (newer R5)
	vaddsd	xmm1, xmm1, xmm3		;; R1 + R5 (newer R1)

	vsubsd	xmm3, xmm5, xmm7		;; R3 - R7 (newer R7)
	vaddsd	xmm5, xmm5, xmm7		;; R3 + R7 (newer R3)

						;; R9/R13 morphs into newer R9/I9
						;; R11/R15 morphs into newer R11/I11

	;; Premultipliers for odd level 3

						;; mul R11/I11 by SQRTHALF + i*SQRTHALF
	vsubsd	xmm7, xmm4, xmm6		;; R11 = R11 - I11
	vaddsd	xmm4, xmm4, xmm6		;; I11 = R11 + I11
	vmulsd	xmm7, xmm7, Q YMM_SQRTHALF	;; R11 = R11 * SQRTHALF
	vmulsd	xmm4, xmm4, Q YMM_SQRTHALF	;; I11 = I11 * SQRTHALF

	;; Odd level 3

	vaddsd	xmm6, xmm1, xmm5		;; R1 + R3 (newest R1)
	vsubsd	xmm1, xmm1, xmm5		;; R1 - R3 (newest and final R3)
	vmovsd	Q YMM_TMP1[16], xmm1		;; Save final R3

						;; newer R5/R7 morphs into newest R5/I5

	vaddsd	xmm5, xmm0, xmm7		;; R9 + R11 (newest R9)
	vsubsd	xmm0, xmm0, xmm7		;; R9 - R11 (newest R11)

	vaddsd	xmm7, xmm2, xmm4		;; I9 + I11 (newest I9)
	vsubsd	xmm2, xmm2, xmm4		;; I9 - I11 (newest I11)

	;; Even level 1

	vmovsd	xmm12, Q [srcreg+d2+d1]		;; R4
	vmovsd	xmm1, Q [srcreg+d2+d1+32]	;; R12
	vaddsd	xmm13, xmm12, xmm1		;; R4 + R12 (new R4)
	vsubsd	xmm12, xmm12, xmm1		;; R4 - R12 (new R12)

	vmovsd	xmm14, Q [srcreg+d4+d2+d1]	;; R8
	vmovsd	xmm1, Q [srcreg+d4+d2+d1+32]	;; R16
	vaddsd	xmm15, xmm14, xmm1		;; R8 + R16 (new R8)
	vsubsd	xmm14, xmm14, xmm1		;; R8 - R16 (new R16)

	vmovsd	xmm4, Q [srcreg+d1]		;; R2
	vmovsd	xmm1, Q [srcreg+d1+32]		;; R10
	vaddsd	xmm9, xmm4, xmm1		;; R2 + R10 (new R2)
	vsubsd	xmm4, xmm4, xmm1		;; R2 - R10 (new R10)

	vmovsd	xmm10, Q [srcreg+d4+d1]		;; R6
	vmovsd	xmm1, Q [srcreg+d4+d1+32]	;; R14
	vaddsd	xmm11, xmm10, xmm1		;; R6 + R14 (new R6)
	vsubsd	xmm10, xmm10, xmm1		;; R6 - R14 (new R14)

	;; Even level 2

	vaddsd	xmm1, xmm13, xmm15		;; R4 + R8 (newer R4)
	vsubsd	xmm13, xmm13, xmm15		;; R4 - R8 (newer R8)

	vaddsd	xmm15, xmm9, xmm11		;; R2 + R6 (newer R2)
	vsubsd	xmm9, xmm9, xmm11		;; R2 - R6 (newer R6)

						;; R10/R14 morphs into newer R10/I10
						;; R12/R16 morphs into newer R12/I12

	;; Even level 3

	vaddsd	xmm11, xmm15, xmm1		;; R2 + R4 (newest R2)
	vsubsd	xmm15, xmm15, xmm1		;; R2 - R4 (newest R4 and final I3)
	vmovsd	Q YMM_TMP1[24], xmm15		;; Save final I3

	;; Premultipliers for even level 3

						;; mul R10/I10 by w^1 = .924 + .383i
	vmulsd	xmm1, xmm4, Q YMM_P924
	vmulsd	xmm15, xmm10, Q YMM_P383
	vsubsd	xmm1, xmm1, xmm15		;; Twiddled R10
	vmulsd	xmm4, xmm4, Q YMM_P383
	vmulsd	xmm10, xmm10, Q YMM_P924
	vaddsd	xmm4, xmm4, xmm10		;; Twiddled I10

						;; mul R12/I12 by w^3 = .383 + .924i
	vmulsd	xmm10, xmm12, Q YMM_P383
	vmulsd	xmm15, xmm14, Q YMM_P924
	vsubsd	xmm10, xmm10, xmm15		;; Twiddled R12
	vmulsd	xmm12, xmm12, Q YMM_P924
	vmulsd	xmm14, xmm14, Q YMM_P383
	vaddsd	xmm12, xmm12, xmm14		;; Twiddled I12

	;; More even level 3

						;; R6/R8 morph into newest R6/I6

	vaddsd	xmm14, xmm1, xmm10		;; R10 + R12 (newest R10)
	vsubsd	xmm1, xmm1, xmm10		;; R10 - R12 (newest R12)

	vaddsd	xmm10, xmm4, xmm12		;; I10 + I12 (newest I10)
	vsubsd	xmm4, xmm4, xmm12		;; I10 - I12 (newest I12)

	;; Premultipliers for level 4

						;; mul R6/I6 by w^2 = .707 + .707i
	vsubsd	xmm12, xmm13, xmm9		;; negR6 = I6 - R6
	vaddsd	xmm9, xmm9, xmm13		;; I6 = R6 + I6
	vmulsd	xmm12, xmm12, Q YMM_SQRTHALF	;; negR6 = negR6 * SQRTHALF
	vmulsd	xmm9, xmm9, Q YMM_SQRTHALF	;; I6 = I6 * SQRTHALF

	;; Level 4

	vaddsd	xmm13, xmm6, xmm11		;; R1 + R2 (final R1)
	vsubsd	xmm6, xmm6, xmm11		;; R1 - R2 (final R2)

						;; R3/R4 morphs into final R3/I3

	;; Done by 8-complex			;; R5 - negR6 (final R5)
						;; R5 + negR6 (final R6)

	;; Done by 8-complex			;; I5 + I6 (final I5)
						;; I5 - I6 (final I6)

	;; Done by 8-complex			;; R9 + R10 (final R9)
						;; R9 - R10 (final R10)

	;; Done by 8-complex			;; I9 + I10 (final I9)
						;; I9 - I10 (final I10)

	;; Done by 8-complex			;; R11 + I12 (final R12)
						;; R11 - I12 (final R11)

	;; Done by 8-complex			;; I11 + R12 (final I11)
						;; I11 - R12 (final I12)

	vmovsd	Q YMM_TMP2[0], xmm8		;; R5
	vmovsd	Q YMM_TMP2[8], xmm3		;; I5
	vmovsd	Q YMM_TMP2[16], xmm12		;; negR6
	vmovsd	Q YMM_TMP2[24], xmm9		;; I6
	vmovsd	Q YMM_TMP3[0], xmm5		;; R9
	vmovsd	Q YMM_TMP3[8], xmm7		;; I9
	vmovsd	Q YMM_TMP3[16], xmm14		;; R10
	vmovsd	Q YMM_TMP3[24], xmm10		;; I10
	vmovsd	Q YMM_TMP4[0], xmm0		;; R11
	vmovsd	Q YMM_TMP4[8], xmm2		;; I11
	vmovsd	Q YMM_TMP4[16], xmm1		;; R12
	vmovsd	Q YMM_TMP4[24], xmm4		;; I12
	vmovsd	Q YMM_TMP1[0], xmm13		;; R1
	vmovsd	Q YMM_TMP1[8], xmm6		;; R2

;; do 8-complex

	vmovapd	ymm0, [srcreg]			;; R1
	vmovapd	ymm1, [srcreg+d4]		;; R5
	vaddpd	ymm2, ymm0, ymm1		;; R1 + R5 (new R1)		; 1-3
	vsubpd	ymm0, ymm0, ymm1		;; R1 - R5 (new R5)		; 2-4

	vmovapd	ymm1, [srcreg+d2]		;; R3
	vmovapd	ymm4, [srcreg+d4+d2]		;; R7
	vaddpd	ymm3, ymm1, ymm4		;; R3 + R7 (new R3)		; 3-5
	vsubpd	ymm1, ymm1, ymm4		;; R3 - R7 (new R7)		; 4-6

	vmovapd	ymm4, [srcreg+d1]		;; R2
	vmovapd	ymm6, [srcreg+d4+d1]		;; R6
	vaddpd	ymm5, ymm4, ymm6		;; R2 + R6 (new R2)		; 5-7
	vsubpd	ymm4, ymm4, ymm6		;; R2 - R6 (new R6)		; 6-8

	vmovapd	ymm6, [srcreg+d2+d1]		;; R4
	vmovapd	ymm8, [srcreg+d4+d2+d1]		;; R8
	vaddpd	ymm7, ymm6, ymm8		;; R4 + R8 (new R4)		; 7-9
	vsubpd	ymm6, ymm6, ymm8		;; R4 - R8 (new R8)		; 8-10
	vmovapd	ymm9, [srcreg+d1+32]		;; I2

	vaddpd	ymm8, ymm2, ymm3		;; R1 + R3 (newer R1)		; 9-11
	vsubpd	ymm2, ymm2, ymm3		;; R1 - R3 (newer R3)		; 10-12
	vmovapd	ymm10, [srcreg+d4+d1+32]	;; I6

	vaddpd	ymm3, ymm5, ymm7		;; R2 + R4 (newer R2)		; 11-13
	vsubpd	ymm5, ymm5, ymm7		;; R2 - R4 (newer R4)		; 12-14
	vmovapd	ymm11, [srcreg+d2+d1+32]	;; I4

	ystore	[srcreg], ymm8			;; Temporarily save newer R1	; 12
	vsubpd	ymm8, ymm9, ymm10		;; I2 - I6 (new I6)		; 13-15
	vaddpd	ymm9, ymm9, ymm10		;; I2 + I6 (new I2)		; 14-16
	vmovapd	ymm12, [srcreg+d4+d2+d1+32]	;; I8

	vsubpd	ymm10, ymm11, ymm12		;; I4 - I8 (new I8)		; 15-17
	vaddpd	ymm11, ymm11, ymm12		;; I4 + I8 (new I4)		; 16-18
	vmovapd	ymm13, YMM_SQRTHALF

	vaddpd	ymm12, ymm8, ymm6		;; I6 + R8 (new I6)		; 17-19
	vsubpd	ymm8, ymm8, ymm6		;; I6 - R8 (new I8)		; 18-20
	vmovapd	ymm6, [srcreg+32]		;; I1
	ystore	[srcreg+d1], ymm3		;; Temporarily save newer R2	; 14

	vsubpd	ymm14, ymm4, ymm10		;; R6 - I8 (new R6)		; 19-21
	vaddpd	ymm4, ymm4, ymm10		;; R6 + I8 (new R8)		; 20-22
	vmovapd	ymm10, [srcreg+d4+32]		;; I5

	vmulpd	ymm12, ymm12, ymm13		;; I6 = I6 * SQRTHALF (newer I6) ; 20-24
	vmulpd	ymm8, ymm8, ymm13		;; I8 = I8 * SQRTHALF (newer I8) ; 21-25
	vmulpd	ymm14, ymm14, ymm13		;; R6 = R6 * SQRTHALF (newer R6) ; 22-26
	vmulpd	ymm4, ymm4, ymm13		;; R8 = R8 * SQRTHALF (newer R8) ; 23-27
	vmovapd	ymm15, [srcreg+d2+32]		;; I3

	vaddpd	ymm13, ymm6, ymm10		;; I1 + I5 (new I1)		; 21-23
	vsubpd	ymm6, ymm6, ymm10		;; I1 - I5 (new I5)		; 22-24

	vmovapd	ymm10, [srcreg+d4+d2+32]	;; I7
	vaddpd	ymm7, ymm15, ymm10		;; I3 + I7 (new I3)		; 23-25
	vsubpd	ymm15, ymm15, ymm10		;; I3 - I7 (new I7)		; 24-26

	vsubpd	ymm10, ymm9, ymm11		;; I2 - I4 (newer I4)		; 25-27
	vaddpd	ymm9, ymm9, ymm11		;; I2 + I4 (newer I2)		; 26-28

	vsubpd	ymm11, ymm13, ymm7		;; I1 - I3 (newer I3)		; 27-29
	vaddpd	ymm13, ymm13, ymm7		;; I1 + I3 (newer I1)		; 28-30

	vblendpd ymm2, ymm2, YMM_TMP2[0], 1	;; Blend in 16-real R5		; 
	vblendpd ymm10, ymm10, YMM_TMP2[16], 1	;; Blend in 16-real negR6	; 
	vaddpd	ymm7, ymm2, ymm10		;; R3 + I4 (last R4)		; 29-31
	vsubpd	ymm2, ymm2, ymm10		;; R3 - I4 (last R3)		; 30-32

	vblendpd ymm11, ymm11, YMM_TMP2[8], 1	;; Blend in 16-real I5		; 
	vblendpd ymm5, ymm5, YMM_TMP2[24], 1	;; Blend in 16-real I6		; 
	vsubpd	ymm10, ymm11, ymm5		;; I3 - R4 (last I4)		; 31-33
	vaddpd	ymm11, ymm11, ymm5		;; I3 + R4 (last I3)		; 32-34

	vsubpd	ymm5, ymm14, ymm12		;; R6 = R6 - I6			; 33-35
	vaddpd	ymm14, ymm14, ymm12		;; I6 = R6 + I6			; 34-36

	vsubpd	ymm12, ymm0, ymm15		;; R5 - I7 (newer R5)		; 35-37
	vaddpd	ymm0, ymm0, ymm15		;; R5 + I7 (newer R7)		; 36-68

	vaddpd	ymm15, ymm6, ymm1		;; I5 + R7 (newer I5)		; 37-39
	vsubpd	ymm6, ymm6, ymm1		;; I5 - R7 (newer I7)		; 38-40

	vblendpd ymm12, ymm12, YMM_TMP3[0], 1	;; Blend in 16-real R9		; 
	vblendpd ymm5, ymm5, YMM_TMP3[16], 1	;; Blend in 16-real R10		; 
	vsubpd	ymm1, ymm12, ymm5		;; R5 - R6 (last R6)		; 39-41
	vaddpd	ymm12, ymm12, ymm5		;; R5 + R6 (last R5)		; 40-42

	vblendpd ymm15, ymm15, YMM_TMP3[8], 1	;; Blend in 16-real I9		; 
	vblendpd ymm14, ymm14, YMM_TMP3[24], 1	;; Blend in 16-real I10		; 
	vsubpd	ymm5, ymm15, ymm14		;; I5 - I6 (last I6)		; 43-45
	vaddpd	ymm15, ymm15, ymm14		;; I5 + I6 (last I5)		; 44-46

	vsubpd	ymm14, ymm4, ymm8		;; R8 = R8 - I8			; 41-43
	vaddpd	ymm4, ymm4, ymm8		;; I8 = R8 + I8			; 42-44

	vblendpd ymm6, ymm6, YMM_TMP4[8], 1	;; Blend in 16-real I11		; 
	vblendpd ymm14, ymm14, YMM_TMP4[16], 1	;; Blend in 16-real R12		; 
	vsubpd	ymm8, ymm6, ymm14		;; I7 - R8 (last I8)		; 45-47
	vaddpd	ymm6, ymm6, ymm14		;; I7 + R8 (last I7)		; 46-48

	vblendpd ymm0, ymm0, YMM_TMP4[0], 1	;; Blend in 16-real R11		; 
	vblendpd ymm4, ymm4, YMM_TMP4[24], 1	;; Blend in 16-real I12		; 
	vaddpd	ymm14, ymm0, ymm4		;; R7 + I8 (last R8)		; 47-49
	vsubpd	ymm0, ymm0, ymm4		;; R7 - I8 (last R7)		; 48-50

	vaddpd	ymm4, ymm13, ymm9		;; I1 + I2 (last I1)		; 49-51
	vsubpd	ymm13, ymm13, ymm9		;; I1 - I2 (last I2)		; 50-52
	vblendpd ymm4, ymm4, YMM_TMP1[8], 1	;; Blend in 16-real I1		; 52
	vblendpd ymm13, ymm13, YMM_TMP1[24], 1	;; Blend in 16-real I2		; 53
	ystore	[srcreg+32], ymm4		;; Temporarily save I1		; 53

	movzx	r10, mul4_opcode		;; Load opcode
	cmp	r10, 80h			;; Test for FFT save
	jb	nofftsave			;; opcode < 0x80, plain,addmul,submul,muladd,mulsub,unfft without FFT save
	;;ja	fftsave				;; opcode >= 0x80, plain,addmul,submul,muladd,mulsub,unfft with FFT save

	;; Store FFT results
	ystore	[srcreg+d2+d1], ymm7		;; R4
	ystore	[srcreg+d2+d1+32], ymm10	;; I4
	ystore	[srcreg+d2], ymm2		;; R3
	ystore	[srcreg+d2+32], ymm11		;; I3
	ystore	[srcreg+d4+d1], ymm1		;; R6
	ystore	[srcreg+d4+d1+32], ymm5		;; I6
	ystore	[srcreg+d4], ymm12		;; R5
	ystore	[srcreg+d4+32], ymm15		;; I5
	ystore	[srcreg+d4+d2+d1], ymm14	;; R8
	ystore	[srcreg+d4+d2+d1+32], ymm8	;; I8
	ystore	[srcreg+d4+d2], ymm0		;; R7
	ystore	[srcreg+d4+d2+32], ymm6		;; I7
	ystore	[srcreg+d1+32], ymm13		;; I2
	and	r10, 7fh			;; Strip the fft save bit from mul4_opcode
						;; Fall through to compute and save R1 & R2

nofftsave:
	vmovapd	ymm3, [srcreg]			;; Reload newer R1
	vmovapd	ymm9, [srcreg+d1]		;; Reload newer R2
	vsubpd	ymm4, ymm3, ymm9		;; R1 - R2 (last R2)
	vaddpd	ymm3, ymm3, ymm9		;; R1 + R2 (last R1)
	vblendpd ymm4, ymm4, YMM_TMP1[16], 1	;; Blend in 16-real R2
	vblendpd ymm3, ymm3, YMM_TMP1[0], 1	;; Blend in 16-real R1
	ystore	[srcreg+d1], ymm4		;; R2
	ystore	[srcreg], ymm3			;; R1

	mov	r9, OFFSET disptab		;; Jump table address
	mov	r10, [r9+r10*8]			;; Jump table offset
	mov	r9, SRC2ARG			;; Distance to s2 arg from gwaddmul4 or gwsubmul4 or s3 arg in gwmuladd4 or gwmulsub4
	jmp	r10
disptab:
	DQ	OFFSET orig			;; 0 = plain
	DQ	OFFSET addmul			;; 1 = addmul
	DQ	OFFSET submul			;; 2 = submul
	DQ	OFFSET muladd			;; 3 = muladd
	DQ	OFFSET mulsub			;; 4 = mulsub
	DQ	OFFSET unfft			;; 5 = unfft
	DQ	OFFSET addmul312		;; 6 = addmul FFTval mem1 mem2
	DQ	OFFSET addmul313		;; 7 = addmul FFTval mem1 FFTval
	DQ	OFFSET addmul132		;; 8 = addmul mem1 FFTval mem2
	DQ	OFFSET addmul133		;; 9 = addmul mem1 FFTval FFTval
	DQ	OFFSET submul312		;; 10 = submul FFTval mem1 mem2
	DQ	OFFSET submul313		;; 11 = submul FFTval mem1 FFTval
	DQ	OFFSET submul132		;; 12 = submul mem1 FFTval mem2
	DQ	OFFSET submul133		;; 13 = submul mem1 FFTval FFTval

addmul:	vmovapd	ymm3, [srcreg+d2+d1][rbp]	;; MemR4
	vaddpd	ymm3, ymm3, [srcreg+d2+d1][r9]	;; MemR4
	vmovapd	ymm9, [srcreg+d2+d1+32][rbp]	;; MemI4
	vaddpd	ymm9, ymm9, [srcreg+d2+d1+32][r9];; MemI4
	vmulpd	ymm4, ymm7, ymm3		;; R4 * MemR4			; 32-36
	vmulpd	ymm7, ymm7, ymm9		;; R4 * MemI4			; 33-37
	vmulpd	ymm9, ymm10, ymm9		;; I4 * MemI4			; 34-38
	vmulpd	ymm10, ymm10, ymm3		;; I4 * MemR4			; 35-39

	vmovapd	ymm3, [srcreg+d2][rbp]		;; MemR3
	vaddpd	ymm3, ymm3, [srcreg+d2][r9]	;; MemR3
	vsubpd	ymm4, ymm4, ymm9		;; R4*MemR4-I4*MemI4 (new R4)	; 51-53
	vmovapd	ymm9, [srcreg+d2+32][rbp]	;; MemI3
	vaddpd	ymm9, ymm9, [srcreg+d2+32][r9]	;; MemI3
	vaddpd	ymm7, ymm7, ymm10		;; R4*MemI4+I4*MemR4 (new I4)	; 52-54
	vmulpd	ymm10, ymm2, ymm3		;; R3 * MemR3			; 36-40
	vmulpd	ymm2, ymm2, ymm9		;; R3 * MemI3			; 37-41
	vmulpd	ymm9, ymm11, ymm9		;; I3 * MemI3			; 38-42
	vmulpd	ymm11, ymm11, ymm3		;; I3 * MemR3			; 39-43

	vmovapd	ymm3, [srcreg+d4+d1][rbp]	;; MemR6
	vaddpd	ymm3, ymm3, [srcreg+d4+d1][r9]	;; MemR6
	vsubpd	ymm10, ymm10, ymm9		;; R3*MemR3-I3*MemI3 (new R3)	; 53-55
	vmovapd	ymm9, [srcreg+d4+d1+32][rbp]	;; MemI6
	vaddpd	ymm9, ymm9, [srcreg+d4+d1+32][r9];; MemI6
	vaddpd	ymm2, ymm2, ymm11		;; R3*MemI3+I3*MemR3 (new I3)	; 54-56
	vmulpd	ymm11, ymm1, ymm3		;; R6 * MemR6			; 42-46
	vmulpd	ymm1, ymm1, ymm9		;; R6 * MemI6			; 43-47
	vmulpd	ymm9, ymm5, ymm9		;; I6 * MemI6			; 46-50
	vmulpd	ymm5, ymm5, ymm3		;; I6 * MemR6			; 47-51

	vmovapd	ymm3, [srcreg+d4][rbp]		;; MemR5
	vaddpd	ymm3, ymm3, [srcreg+d4][r9]	;; MemR5
	vsubpd	ymm11, ymm11, ymm9		;; R6*MemR6-I6*MemI6 (new R6)	; 55-57
	vmovapd	ymm9, [srcreg+d4+32][rbp]	;; MemI5
	vaddpd	ymm9, ymm9, [srcreg+d4+32][r9]	;; MemI5
	vaddpd	ymm1, ymm1, ymm5		;; R6*MemI6+I6*MemR6 (new I6)	; 56-58
	vmulpd	ymm5, ymm12, ymm3		;; R5 * MemR5			; 44-48
	vmulpd	ymm12, ymm12, ymm9		;; R5 * MemI5			; 45-49
	vmulpd	ymm9, ymm15, ymm9		;; I5 * MemI5			; 48-52
	vmulpd	ymm15, ymm15, ymm3		;; I5 * MemR5			; 49-53

	vmovapd	ymm3, [srcreg+d4+d2+d1][rbp]	;; MemR8
	vaddpd	ymm3, ymm3, [srcreg+d4+d2+d1][r9];; MemR8
	vsubpd	ymm5, ymm5, ymm9		;; R5*MemR5-I5*MemI5 (new R5)	; 57-59
	vmovapd	ymm9, [srcreg+d4+d2+d1+32][rbp]	;; MemI8
	vaddpd	ymm9, ymm9, [srcreg+d4+d2+d1+32][r9];; MemI8
	vaddpd	ymm12, ymm12, ymm15		;; R5*MemI5+I5*MemR5 (new I5)	; 58-60
	vmulpd	ymm15, ymm14, ymm3		;; R8 * MemR8			; 50-54
	vmulpd	ymm14, ymm14, ymm9		;; R8 * MemI8			; 51-55
	vmulpd	ymm9, ymm8, ymm9		;; I8 * MemI8			; 52-56
	vmulpd	ymm8, ymm8, ymm3		;; I8 * MemR8			; 53-57

	vmovapd	ymm3, [srcreg+d4+d2][rbp]	;; MemR7
	vaddpd	ymm3, ymm3, [srcreg+d4+d2][r9]	;; MemR7
	vsubpd	ymm15, ymm15, ymm9		;; R8*MemR8-I8*MemI8 (new R8)	; 59-61
	vmovapd	ymm9, [srcreg+d4+d2+32][rbp]	;; MemI7
	vaddpd	ymm9, ymm9, [srcreg+d4+d2+32][r9];; MemI7
	vaddpd	ymm14, ymm14, ymm8		;; R8*MemI8+I8*MemR8 (new I8)	; 60-62
	vmulpd	ymm8, ymm0, ymm3		;; R7 * MemR7			; 54-58
	vmulpd	ymm0, ymm0, ymm9		;; R7 * MemI7			; 55-59
	vmulpd	ymm9, ymm6, ymm9		;; I7 * MemI7			; 56-60
	vmulpd	ymm6, ymm6, ymm3		;; I7 * MemR7			; 57-61

	vsubpd	ymm8, ymm8, ymm9		;; R7*MemR7-I7*MemI7 (new R7)	; 61-63
	vaddpd	ymm0, ymm0, ymm6		;; R7*MemI7+I7*MemR7 (new I7)	; 62-64

	vaddpd	ymm9, ymm4, ymm10		;; R4 + R3 (new R3)		; 65-67
	vsubpd	ymm4, ymm4, ymm10		;; R4 - R3 (new I4)		; 66-68

	vmovapd	ymm6, [srcreg+d1]		;; Reload R2
	vmovapd	ymm10, [srcreg+d1][rbp]		;; MemR2
	vaddpd	ymm10, ymm10, [srcreg+d1][r9]	;; MemR2
	ystore	YMM_TMP5, ymm9			;; Temporarily save new R3	; 68
	vmovapd	ymm9, [srcreg+d1+32][rbp]	;; MemI2
	vaddpd	ymm9, ymm9, [srcreg+d1+32][r9]	;; MemI2
	ystore	YMM_TMP6, ymm4			;; Temporarily save new I4	; 69
	vmulpd	ymm4, ymm6, ymm10		;; R2 * MemR2			; 66-70
	vmulpd	ymm6, ymm6, ymm9		;; R2 * MemI2			; 67-71
	vmulpd	ymm9, ymm13, ymm9		;; I2 * MemI2			; 68-72
	vmulpd	ymm13, ymm13, ymm10		;; I2 * MemR2			; 69-73

	vmovapd	ymm3, [srcreg]			;; Reload R1
	vmovapd	ymm10, [srcreg][rbp]		;; MemR1
	vaddpd	ymm10, ymm10, [srcreg][r9]	;; MemR1
	vsubpd	ymm4, ymm4, ymm9		;; R2*MemR2-I2*MemI2 (new R2)	; 73-75
	vmovapd	ymm9, [srcreg+32][rbp]		;; MemI1
	vaddpd	ymm9, ymm9, [srcreg+32][r9]	;; MemI1
	vaddpd	ymm6, ymm6, ymm13		;; R2*MemI2+I2*MemR2 (new I2)	; 74-76

	vsubpd	ymm13, ymm2, ymm7		;; I3 - I4 (new R4)		; 67-69
	vaddpd	ymm2, ymm2, ymm7		;; I3 + I4 (new I3)		; 68-70

	vsubpd	ymm7, ymm12, ymm1		;; I5 - I6 (new I6)		; 69-71
	vaddpd	ymm12, ymm12, ymm1		;; I5 + I6 (new I5)		; 70-72

	vsubpd	ymm1, ymm5, ymm11		;; R5 - R6 (new R6)		; 71-73
	vaddpd	ymm5, ymm5, ymm11		;; R5 + R6 (new R5)		; 72-74

	vmulpd	ymm11, ymm3, ymm10		;; R1 * MemR1			; 70-74
	vmulpd	ymm3, ymm3, ymm9		;; R1 * MemI1			; 71-75
	vmovsd	Q [srcreg+r8-16], xmm11		;; Save product of sum of FFT values
	vmulpd	ymm9, ymm9, [srcreg+32]		;; (reload I1) * MemI1		; 72-76
	vmulpd	ymm10, ymm10, [srcreg+32]	;; (reload I1) * MemR1		; 73-77
	vmovsd	Q YMM_TMP1[8], xmm9		;; I1

	vsubpd	ymm11, ymm11, ymm9		;; R1*MemR1-I1*MemI1 (new R1)	; 77-79
	vaddpd	ymm3, ymm3, ymm10		;; R1*MemI1+I1*MemR1 (new I1)	; 78-80
	jmp	back_to_orig

addmul133:
addmul313:
	vaddpd	ymm3, ymm7, [srcreg+d2+d1][rbp]	;; R4+MemR4
	vaddpd	ymm9, ymm10, [srcreg+d2+d1+32][rbp];; I4+MemI4
	vmulpd	ymm4, ymm7, ymm3		;; R4 * MemR4			; 32-36
	vmulpd	ymm7, ymm7, ymm9		;; R4 * MemI4			; 33-37
	vmulpd	ymm9, ymm10, ymm9		;; I4 * MemI4			; 34-38
	vmulpd	ymm10, ymm10, ymm3		;; I4 * MemR4			; 35-39

	vaddpd	ymm3, ymm2, [srcreg+d2][rbp]	;; R3+MemR3
	vsubpd	ymm4, ymm4, ymm9		;; R4*MemR4-I4*MemI4 (new R4)	; 51-53
	vaddpd	ymm9, ymm11, [srcreg+d2+32][rbp];; I3+MemI3
	vaddpd	ymm7, ymm7, ymm10		;; R4*MemI4+I4*MemR4 (new I4)	; 52-54
	vmulpd	ymm10, ymm2, ymm3		;; R3 * MemR3			; 36-40
	vmulpd	ymm2, ymm2, ymm9		;; R3 * MemI3			; 37-41
	vmulpd	ymm9, ymm11, ymm9		;; I3 * MemI3			; 38-42
	vmulpd	ymm11, ymm11, ymm3		;; I3 * MemR3			; 39-43

	vaddpd	ymm3, ymm1, [srcreg+d4+d1][rbp]	;; R6+MemR6
	vsubpd	ymm10, ymm10, ymm9		;; R3*MemR3-I3*MemI3 (new R3)	; 53-55
	vaddpd	ymm9, ymm5, [srcreg+d4+d1+32][rbp];; I6+MemI6
	vaddpd	ymm2, ymm2, ymm11		;; R3*MemI3+I3*MemR3 (new I3)	; 54-56
	vmulpd	ymm11, ymm1, ymm3		;; R6 * MemR6			; 42-46
	vmulpd	ymm1, ymm1, ymm9		;; R6 * MemI6			; 43-47
	vmulpd	ymm9, ymm5, ymm9		;; I6 * MemI6			; 46-50
	vmulpd	ymm5, ymm5, ymm3		;; I6 * MemR6			; 47-51

	vaddpd	ymm3, ymm12, [srcreg+d4][rbp]	;; R5+MemR5
	vsubpd	ymm11, ymm11, ymm9		;; R6*MemR6-I6*MemI6 (new R6)	; 55-57
	vaddpd	ymm9, ymm15, [srcreg+d4+32][rbp];; I5+MemI5
	vaddpd	ymm1, ymm1, ymm5		;; R6*MemI6+I6*MemR6 (new I6)	; 56-58
	vmulpd	ymm5, ymm12, ymm3		;; R5 * MemR5			; 44-48
	vmulpd	ymm12, ymm12, ymm9		;; R5 * MemI5			; 45-49
	vmulpd	ymm9, ymm15, ymm9		;; I5 * MemI5			; 48-52
	vmulpd	ymm15, ymm15, ymm3		;; I5 * MemR5			; 49-53

	vaddpd	ymm3, ymm14, [srcreg+d4+d2+d1][rbp];; R8+MemR8
	vsubpd	ymm5, ymm5, ymm9		;; R5*MemR5-I5*MemI5 (new R5)	; 57-59
	vaddpd	ymm9, ymm8, [srcreg+d4+d2+d1+32][rbp];; I8+MemI8
	vaddpd	ymm12, ymm12, ymm15		;; R5*MemI5+I5*MemR5 (new I5)	; 58-60
	vmulpd	ymm15, ymm14, ymm3		;; R8 * MemR8			; 50-54
	vmulpd	ymm14, ymm14, ymm9		;; R8 * MemI8			; 51-55
	vmulpd	ymm9, ymm8, ymm9		;; I8 * MemI8			; 52-56
	vmulpd	ymm8, ymm8, ymm3		;; I8 * MemR8			; 53-57

	vaddpd	ymm3, ymm0, [srcreg+d4+d2][rbp]	;; R7+MemR7
	vsubpd	ymm15, ymm15, ymm9		;; R8*MemR8-I8*MemI8 (new R8)	; 59-61
	vaddpd	ymm9, ymm6, [srcreg+d4+d2+32][rbp];; I7+MemI7
	vaddpd	ymm14, ymm14, ymm8		;; R8*MemI8+I8*MemR8 (new I8)	; 60-62
	vmulpd	ymm8, ymm0, ymm3		;; R7 * MemR7			; 54-58
	vmulpd	ymm0, ymm0, ymm9		;; R7 * MemI7			; 55-59
	vmulpd	ymm9, ymm6, ymm9		;; I7 * MemI7			; 56-60
	vmulpd	ymm6, ymm6, ymm3		;; I7 * MemR7			; 57-61

	vsubpd	ymm8, ymm8, ymm9		;; R7*MemR7-I7*MemI7 (new R7)	; 61-63
	vaddpd	ymm0, ymm0, ymm6		;; R7*MemI7+I7*MemR7 (new I7)	; 62-64

	vaddpd	ymm9, ymm4, ymm10		;; R4 + R3 (new R3)		; 65-67
	vsubpd	ymm4, ymm4, ymm10		;; R4 - R3 (new I4)		; 66-68

	vmovapd	ymm6, [srcreg+d1]		;; Reload R2
	vaddpd	ymm10, ymm6, [srcreg+d1][rbp]	;; R2+MemR2
	ystore	YMM_TMP5, ymm9			;; Temporarily save new R3	; 68
	vaddpd	ymm9, ymm13, [srcreg+d1+32][rbp];; I2+MemI2
	ystore	YMM_TMP6, ymm4			;; Temporarily save new I4	; 69
	vmulpd	ymm4, ymm6, ymm10		;; R2 * MemR2			; 66-70
	vmulpd	ymm6, ymm6, ymm9		;; R2 * MemI2			; 67-71
	vmulpd	ymm9, ymm13, ymm9		;; I2 * MemI2			; 68-72
	vmulpd	ymm13, ymm13, ymm10		;; I2 * MemR2			; 69-73

	vmovapd	ymm3, [srcreg]			;; Reload R1
	vaddpd	ymm10, ymm3, [srcreg][rbp]	;; R1+MemR1
	vsubpd	ymm4, ymm4, ymm9		;; R2*MemR2-I2*MemI2 (new R2)	; 73-75
	vmovapd	ymm9, [srcreg+32]		;; (reload I1)
	vaddpd	ymm9, ymm9, [srcreg+32][rbp]	;; I1+MemI1
	vaddpd	ymm6, ymm6, ymm13		;; R2*MemI2+I2*MemR2 (new I2)	; 74-76

	vsubpd	ymm13, ymm2, ymm7		;; I3 - I4 (new R4)		; 67-69
	vaddpd	ymm2, ymm2, ymm7		;; I3 + I4 (new I3)		; 68-70

	vsubpd	ymm7, ymm12, ymm1		;; I5 - I6 (new I6)		; 69-71
	vaddpd	ymm12, ymm12, ymm1		;; I5 + I6 (new I5)		; 70-72

	vsubpd	ymm1, ymm5, ymm11		;; R5 - R6 (new R6)		; 71-73
	vaddpd	ymm5, ymm5, ymm11		;; R5 + R6 (new R5)		; 72-74

	vmulpd	ymm11, ymm3, ymm10		;; R1 * MemR1			; 70-74
	vmulpd	ymm3, ymm3, ymm9		;; R1 * MemI1			; 71-75
	vmovsd	Q [srcreg+r8-16], xmm11		;; Save product of sum of FFT values
	vmulpd	ymm9, ymm9, [srcreg+32]		;; (reload I1) * MemI1		; 72-76
	vmulpd	ymm10, ymm10, [srcreg+32]	;; (reload I1) * MemR1		; 73-77
	vmovsd	Q YMM_TMP1[8], xmm9		;; I1

	vsubpd	ymm11, ymm11, ymm9		;; R1*MemR1-I1*MemI1 (new R1)	; 77-79
	vaddpd	ymm3, ymm3, ymm10		;; R1*MemI1+I1*MemR1 (new I1)	; 78-80
	jmp	back_to_orig

addmul132:
addmul312:
	vaddpd	ymm3, ymm7, [srcreg+d2+d1][rbp]	;; R4+MemR4
	vmovapd	ymm7, [srcreg+d2+d1][r9]	;; R4=MemR4
	vaddpd	ymm9, ymm10, [srcreg+d2+d1+32][rbp];; I4+MemI4
	vmovapd	ymm10, [srcreg+d2+d1+32][r9]	;; I4=MemI4
	vmulpd	ymm4, ymm7, ymm3		;; R4 * MemR4			; 32-36
	vmulpd	ymm7, ymm7, ymm9		;; R4 * MemI4			; 33-37
	vmulpd	ymm9, ymm10, ymm9		;; I4 * MemI4			; 34-38
	vmulpd	ymm10, ymm10, ymm3		;; I4 * MemR4			; 35-39

	vaddpd	ymm3, ymm2, [srcreg+d2][rbp]	;; R3+MemR3
	vmovapd	ymm2, [srcreg+d2][r9]		;; R3=MemR3
	vsubpd	ymm4, ymm4, ymm9		;; R4*MemR4-I4*MemI4 (new R4)	; 51-53
	vaddpd	ymm9, ymm11, [srcreg+d2+32][rbp];; I3+MemI3
	vmovapd	ymm11, [srcreg+d2+32][r9]	;; I3=MemI3
	vaddpd	ymm7, ymm7, ymm10		;; R4*MemI4+I4*MemR4 (new I4)	; 52-54
	vmulpd	ymm10, ymm2, ymm3		;; R3 * MemR3			; 36-40
	vmulpd	ymm2, ymm2, ymm9		;; R3 * MemI3			; 37-41
	vmulpd	ymm9, ymm11, ymm9		;; I3 * MemI3			; 38-42
	vmulpd	ymm11, ymm11, ymm3		;; I3 * MemR3			; 39-43

	vaddpd	ymm3, ymm1, [srcreg+d4+d1][rbp]	;; R6+MemR6
	vmovapd	ymm1, [srcreg+d4+d1][r9]	;; R6=MemR6
	vsubpd	ymm10, ymm10, ymm9		;; R3*MemR3-I3*MemI3 (new R3)	; 53-55
	vaddpd	ymm9, ymm5, [srcreg+d4+d1+32][rbp];; I6+MemI6
	vmovapd	ymm5, [srcreg+d4+d1+32][r9]	;; I6=MemI6
	vaddpd	ymm2, ymm2, ymm11		;; R3*MemI3+I3*MemR3 (new I3)	; 54-56
	vmulpd	ymm11, ymm1, ymm3		;; R6 * MemR6			; 42-46
	vmulpd	ymm1, ymm1, ymm9		;; R6 * MemI6			; 43-47
	vmulpd	ymm9, ymm5, ymm9		;; I6 * MemI6			; 46-50
	vmulpd	ymm5, ymm5, ymm3		;; I6 * MemR6			; 47-51

	vaddpd	ymm3, ymm12, [srcreg+d4][rbp]	;; R5+MemR5
	vmovapd	ymm12, [srcreg+d4][r9]		;; R5=MemR5
	vsubpd	ymm11, ymm11, ymm9		;; R6*MemR6-I6*MemI6 (new R6)	; 55-57
	vaddpd	ymm9, ymm15, [srcreg+d4+32][rbp];; I5+MemI5
	vmovapd	ymm15, [srcreg+d4+32][r9]	;; I5=MemI5
	vaddpd	ymm1, ymm1, ymm5		;; R6*MemI6+I6*MemR6 (new I6)	; 56-58
	vmulpd	ymm5, ymm12, ymm3		;; R5 * MemR5			; 44-48
	vmulpd	ymm12, ymm12, ymm9		;; R5 * MemI5			; 45-49
	vmulpd	ymm9, ymm15, ymm9		;; I5 * MemI5			; 48-52
	vmulpd	ymm15, ymm15, ymm3		;; I5 * MemR5			; 49-53

	vaddpd	ymm3, ymm14, [srcreg+d4+d2+d1][rbp];; R8+MemR8
	vmovapd	ymm14, [srcreg+d4+d2+d1][r9]	;; R8=MemR8
	vsubpd	ymm5, ymm5, ymm9		;; R5*MemR5-I5*MemI5 (new R5)	; 57-59
	vaddpd	ymm9, ymm8, [srcreg+d4+d2+d1+32][rbp];; I8+MemI8
	vmovapd	ymm8, [srcreg+d4+d2+d1+32][r9]	;; I8=MemI8
	vaddpd	ymm12, ymm12, ymm15		;; R5*MemI5+I5*MemR5 (new I5)	; 58-60
	vmulpd	ymm15, ymm14, ymm3		;; R8 * MemR8			; 50-54
	vmulpd	ymm14, ymm14, ymm9		;; R8 * MemI8			; 51-55
	vmulpd	ymm9, ymm8, ymm9		;; I8 * MemI8			; 52-56
	vmulpd	ymm8, ymm8, ymm3		;; I8 * MemR8			; 53-57

	vaddpd	ymm3, ymm0, [srcreg+d4+d2][rbp]	;; R7+MemR7
	vmovapd	ymm0, [srcreg+d4+d2][r9]	;; R7=MemR7
	vsubpd	ymm15, ymm15, ymm9		;; R8*MemR8-I8*MemI8 (new R8)	; 59-61
	vaddpd	ymm9, ymm6, [srcreg+d4+d2+32][rbp];; I7+MemI7
	vmovapd	ymm6, [srcreg+d4+d2+32][r9]	;; I7=MemI7
	vaddpd	ymm14, ymm14, ymm8		;; R8*MemI8+I8*MemR8 (new I8)	; 60-62
	vmulpd	ymm8, ymm0, ymm3		;; R7 * MemR7			; 54-58
	vmulpd	ymm0, ymm0, ymm9		;; R7 * MemI7			; 55-59
	vmulpd	ymm9, ymm6, ymm9		;; I7 * MemI7			; 56-60
	vmulpd	ymm6, ymm6, ymm3		;; I7 * MemR7			; 57-61

	vsubpd	ymm8, ymm8, ymm9		;; R7*MemR7-I7*MemI7 (new R7)	; 61-63
	vaddpd	ymm0, ymm0, ymm6		;; R7*MemI7+I7*MemR7 (new I7)	; 62-64

	vaddpd	ymm9, ymm4, ymm10		;; R4 + R3 (new R3)		; 65-67
	vsubpd	ymm4, ymm4, ymm10		;; R4 - R3 (new I4)		; 66-68

	vmovapd	ymm6, [srcreg+d1]		;; Reload R2
	vaddpd	ymm10, ymm6, [srcreg+d1][rbp]	;; R2+MemR2
	vmovapd	ymm6, [srcreg+d1][r9]		;; R2=MemR2
	ystore	YMM_TMP5, ymm9			;; Temporarily save new R3	; 68
	vaddpd	ymm9, ymm13, [srcreg+d1+32][rbp];; I2+MemI2
	vmovapd	ymm13, [srcreg+d1+32][r9]	;; I2=MemI2
	ystore	YMM_TMP6, ymm4			;; Temporarily save new I4	; 69
	vmulpd	ymm4, ymm6, ymm10		;; R2 * MemR2			; 66-70
	vmulpd	ymm6, ymm6, ymm9		;; R2 * MemI2			; 67-71
	vmulpd	ymm9, ymm13, ymm9		;; I2 * MemI2			; 68-72
	vmulpd	ymm13, ymm13, ymm10		;; I2 * MemR2			; 69-73

	vmovapd	ymm3, [srcreg]			;; Reload R1
	vaddpd	ymm10, ymm3, [srcreg][rbp]	;; R1+MemR1
	vmovapd	ymm3, [srcreg][r9]		;; R1=MemR1
	vsubpd	ymm4, ymm4, ymm9		;; R2*MemR2-I2*MemI2 (new R2)	; 73-75
	vmovapd	ymm9, [srcreg+32]		;; (reload I1)
	vaddpd	ymm9, ymm9, [srcreg+32][rbp]	;; I1+MemI1
	vaddpd	ymm6, ymm6, ymm13		;; R2*MemI2+I2*MemR2 (new I2)	; 74-76

	vsubpd	ymm13, ymm2, ymm7		;; I3 - I4 (new R4)		; 67-69
	vaddpd	ymm2, ymm2, ymm7		;; I3 + I4 (new I3)		; 68-70

	vsubpd	ymm7, ymm12, ymm1		;; I5 - I6 (new I6)		; 69-71
	vaddpd	ymm12, ymm12, ymm1		;; I5 + I6 (new I5)		; 70-72

	vsubpd	ymm1, ymm5, ymm11		;; R5 - R6 (new R6)		; 71-73
	vaddpd	ymm5, ymm5, ymm11		;; R5 + R6 (new R5)		; 72-74

	vmulpd	ymm11, ymm3, ymm10		;; R1 * MemR1			; 70-74
	vmulpd	ymm3, ymm3, ymm9		;; R1 * MemI1			; 71-75
	vmovsd	Q [srcreg+r8-16], xmm11		;; Save product of sum of FFT values
	vmulpd	ymm9, ymm9, [srcreg+32][r9]	;; (reload I1) * MemI1		; 72-76
	vmulpd	ymm10, ymm10, [srcreg+32][r9]	;; (reload I1) * MemR1		; 73-77
	vmovsd	Q YMM_TMP1[8], xmm9		;; I1

	vsubpd	ymm11, ymm11, ymm9		;; R1*MemR1-I1*MemI1 (new R1)	; 77-79
	vaddpd	ymm3, ymm3, ymm10		;; R1*MemI1+I1*MemR1 (new I1)	; 78-80
	jmp	back_to_orig

submul:	vmovapd	ymm3, [srcreg+d2+d1][rbp]	;; MemR4
	vsubpd	ymm3, ymm3, [srcreg+d2+d1][r9]	;; MemR4
	vmovapd	ymm9, [srcreg+d2+d1+32][rbp]	;; MemI4
	vsubpd	ymm9, ymm9, [srcreg+d2+d1+32][r9];; MemI4
	vmulpd	ymm4, ymm7, ymm3		;; R4 * MemR4			; 32-36
	vmulpd	ymm7, ymm7, ymm9		;; R4 * MemI4			; 33-37
	vmulpd	ymm9, ymm10, ymm9		;; I4 * MemI4			; 34-38
	vmulpd	ymm10, ymm10, ymm3		;; I4 * MemR4			; 35-39

	vmovapd	ymm3, [srcreg+d2][rbp]		;; MemR3
	vsubpd	ymm3, ymm3, [srcreg+d2][r9]	;; MemR3
	vsubpd	ymm4, ymm4, ymm9		;; R4*MemR4-I4*MemI4 (new R4)	; 51-53
	vmovapd	ymm9, [srcreg+d2+32][rbp]	;; MemI3
	vsubpd	ymm9, ymm9, [srcreg+d2+32][r9]	;; MemI3
	vaddpd	ymm7, ymm7, ymm10		;; R4*MemI4+I4*MemR4 (new I4)	; 52-54
	vmulpd	ymm10, ymm2, ymm3		;; R3 * MemR3			; 36-40
	vmulpd	ymm2, ymm2, ymm9		;; R3 * MemI3			; 37-41
	vmulpd	ymm9, ymm11, ymm9		;; I3 * MemI3			; 38-42
	vmulpd	ymm11, ymm11, ymm3		;; I3 * MemR3			; 39-43

	vmovapd	ymm3, [srcreg+d4+d1][rbp]	;; MemR6
	vsubpd	ymm3, ymm3, [srcreg+d4+d1][r9]	;; MemR6
	vsubpd	ymm10, ymm10, ymm9		;; R3*MemR3-I3*MemI3 (new R3)	; 53-55
	vmovapd	ymm9, [srcreg+d4+d1+32][rbp]	;; MemI6
	vsubpd	ymm9, ymm9, [srcreg+d4+d1+32][r9];; MemI6
	vaddpd	ymm2, ymm2, ymm11		;; R3*MemI3+I3*MemR3 (new I3)	; 54-56
	vmulpd	ymm11, ymm1, ymm3		;; R6 * MemR6			; 42-46
	vmulpd	ymm1, ymm1, ymm9		;; R6 * MemI6			; 43-47
	vmulpd	ymm9, ymm5, ymm9		;; I6 * MemI6			; 46-50
	vmulpd	ymm5, ymm5, ymm3		;; I6 * MemR6			; 47-51

	vmovapd	ymm3, [srcreg+d4][rbp]		;; MemR5
	vsubpd	ymm3, ymm3, [srcreg+d4][r9]	;; MemR5
	vsubpd	ymm11, ymm11, ymm9		;; R6*MemR6-I6*MemI6 (new R6)	; 55-57
	vmovapd	ymm9, [srcreg+d4+32][rbp]	;; MemI5
	vsubpd	ymm9, ymm9, [srcreg+d4+32][r9]	;; MemI5
	vaddpd	ymm1, ymm1, ymm5		;; R6*MemI6+I6*MemR6 (new I6)	; 56-58
	vmulpd	ymm5, ymm12, ymm3		;; R5 * MemR5			; 44-48
	vmulpd	ymm12, ymm12, ymm9		;; R5 * MemI5			; 45-49
	vmulpd	ymm9, ymm15, ymm9		;; I5 * MemI5			; 48-52
	vmulpd	ymm15, ymm15, ymm3		;; I5 * MemR5			; 49-53

	vmovapd	ymm3, [srcreg+d4+d2+d1][rbp]	;; MemR8
	vsubpd	ymm3, ymm3, [srcreg+d4+d2+d1][r9];; MemR8
	vsubpd	ymm5, ymm5, ymm9		;; R5*MemR5-I5*MemI5 (new R5)	; 57-59
	vmovapd	ymm9, [srcreg+d4+d2+d1+32][rbp]	;; MemI8
	vsubpd	ymm9, ymm9, [srcreg+d4+d2+d1+32][r9];; MemI8
	vaddpd	ymm12, ymm12, ymm15		;; R5*MemI5+I5*MemR5 (new I5)	; 58-60
	vmulpd	ymm15, ymm14, ymm3		;; R8 * MemR8			; 50-54
	vmulpd	ymm14, ymm14, ymm9		;; R8 * MemI8			; 51-55
	vmulpd	ymm9, ymm8, ymm9		;; I8 * MemI8			; 52-56
	vmulpd	ymm8, ymm8, ymm3		;; I8 * MemR8			; 53-57

	vmovapd	ymm3, [srcreg+d4+d2][rbp]	;; MemR7
	vsubpd	ymm3, ymm3, [srcreg+d4+d2][r9]	;; MemR7
	vsubpd	ymm15, ymm15, ymm9		;; R8*MemR8-I8*MemI8 (new R8)	; 59-61
	vmovapd	ymm9, [srcreg+d4+d2+32][rbp]	;; MemI7
	vsubpd	ymm9, ymm9, [srcreg+d4+d2+32][r9];; MemI7
	vaddpd	ymm14, ymm14, ymm8		;; R8*MemI8+I8*MemR8 (new I8)	; 60-62
	vmulpd	ymm8, ymm0, ymm3		;; R7 * MemR7			; 54-58
	vmulpd	ymm0, ymm0, ymm9		;; R7 * MemI7			; 55-59
	vmulpd	ymm9, ymm6, ymm9		;; I7 * MemI7			; 56-60
	vmulpd	ymm6, ymm6, ymm3		;; I7 * MemR7			; 57-61

	vsubpd	ymm8, ymm8, ymm9		;; R7*MemR7-I7*MemI7 (new R7)	; 61-63
	vaddpd	ymm0, ymm0, ymm6		;; R7*MemI7+I7*MemR7 (new I7)	; 62-64

	vaddpd	ymm9, ymm4, ymm10		;; R4 + R3 (new R3)		; 65-67
	vsubpd	ymm4, ymm4, ymm10		;; R4 - R3 (new I4)		; 66-68

	vmovapd	ymm6, [srcreg+d1]		;; Reload R2
	vmovapd	ymm10, [srcreg+d1][rbp]		;; MemR2
	vsubpd	ymm10, ymm10, [srcreg+d1][r9]	;; MemR2
	ystore	YMM_TMP5, ymm9			;; Temporarily save new R3	; 68
	vmovapd	ymm9, [srcreg+d1+32][rbp]	;; MemI2
	vsubpd	ymm9, ymm9, [srcreg+d1+32][r9]	;; MemI2
	ystore	YMM_TMP6, ymm4			;; Temporarily save new I4	; 69
	vmulpd	ymm4, ymm6, ymm10		;; R2 * MemR2			; 66-70
	vmulpd	ymm6, ymm6, ymm9		;; R2 * MemI2			; 67-71
	vmulpd	ymm9, ymm13, ymm9		;; I2 * MemI2			; 68-72
	vmulpd	ymm13, ymm13, ymm10		;; I2 * MemR2			; 69-73

	vmovapd	ymm3, [srcreg]			;; Reload R1
	vmovapd	ymm10, [srcreg][rbp]		;; MemR1
	vsubpd	ymm10, ymm10, [srcreg][r9]	;; MemR1
	vsubpd	ymm4, ymm4, ymm9		;; R2*MemR2-I2*MemI2 (new R2)	; 73-75
	vmovapd	ymm9, [srcreg+32][rbp]		;; MemI1
	vsubpd	ymm9, ymm9, [srcreg+32][r9]	;; MemI1
	vaddpd	ymm6, ymm6, ymm13		;; R2*MemI2+I2*MemR2 (new I2)	; 74-76

	vsubpd	ymm13, ymm2, ymm7		;; I3 - I4 (new R4)		; 67-69
	vaddpd	ymm2, ymm2, ymm7		;; I3 + I4 (new I3)		; 68-70

	vsubpd	ymm7, ymm12, ymm1		;; I5 - I6 (new I6)		; 69-71
	vaddpd	ymm12, ymm12, ymm1		;; I5 + I6 (new I5)		; 70-72

	vsubpd	ymm1, ymm5, ymm11		;; R5 - R6 (new R6)		; 71-73
	vaddpd	ymm5, ymm5, ymm11		;; R5 + R6 (new R5)		; 72-74

	vmulpd	ymm11, ymm3, ymm10		;; R1 * MemR1			; 70-74
	vmulpd	ymm3, ymm3, ymm9		;; R1 * MemI1			; 71-75
	vmovsd	Q [srcreg+r8-16], xmm11		;; Save product of sum of FFT values
	vmulpd	ymm9, ymm9, [srcreg+32]		;; (reload I1) * MemI1		; 72-76
	vmulpd	ymm10, ymm10, [srcreg+32]	;; (reload I1) * MemR1		; 73-77
	vmovsd	Q YMM_TMP1[8], xmm9		;; I1

	vsubpd	ymm11, ymm11, ymm9		;; R1*MemR1-I1*MemI1 (new R1)	; 77-79
	vaddpd	ymm3, ymm3, ymm10		;; R1*MemI1+I1*MemR1 (new I1)	; 78-80
	jmp	back_to_orig

submul133:
	vmovapd	ymm3, [srcreg+d2+d1][rbp]	;; MemR4
	vsubpd	ymm3, ymm3, ymm7		;; MemR4-R4
	vmovapd	ymm9, [srcreg+d2+d1+32][rbp]	;; MemI4
	vsubpd	ymm9, ymm9, ymm10		;; MemI4-I4
	vmulpd	ymm4, ymm7, ymm3		;; R4 * MemR4			; 32-36
	vmulpd	ymm7, ymm7, ymm9		;; R4 * MemI4			; 33-37
	vmulpd	ymm9, ymm10, ymm9		;; I4 * MemI4			; 34-38
	vmulpd	ymm10, ymm10, ymm3		;; I4 * MemR4			; 35-39

	vmovapd	ymm3, [srcreg+d2][rbp]		;; MemR3
	vsubpd	ymm3, ymm3, ymm2		;; MemR3-R3
	vsubpd	ymm4, ymm4, ymm9		;; R4*MemR4-I4*MemI4 (new R4)	; 51-53
	vmovapd	ymm9, [srcreg+d2+32][rbp]	;; MemI3
	vsubpd	ymm9, ymm9, ymm11		;; MemI3-I3
	vaddpd	ymm7, ymm7, ymm10		;; R4*MemI4+I4*MemR4 (new I4)	; 52-54
	vmulpd	ymm10, ymm2, ymm3		;; R3 * MemR3			; 36-40
	vmulpd	ymm2, ymm2, ymm9		;; R3 * MemI3			; 37-41
	vmulpd	ymm9, ymm11, ymm9		;; I3 * MemI3			; 38-42
	vmulpd	ymm11, ymm11, ymm3		;; I3 * MemR3			; 39-43

	vmovapd	ymm3, [srcreg+d4+d1][rbp]	;; MemR6
	vsubpd	ymm3, ymm3, ymm1		;; MemR6-R6
	vsubpd	ymm10, ymm10, ymm9		;; R3*MemR3-I3*MemI3 (new R3)	; 53-55
	vmovapd	ymm9, [srcreg+d4+d1+32][rbp]	;; MemI6
	vsubpd	ymm9, ymm9, ymm5		;; MemI6-I6
	vaddpd	ymm2, ymm2, ymm11		;; R3*MemI3+I3*MemR3 (new I3)	; 54-56
	vmulpd	ymm11, ymm1, ymm3		;; R6 * MemR6			; 42-46
	vmulpd	ymm1, ymm1, ymm9		;; R6 * MemI6			; 43-47
	vmulpd	ymm9, ymm5, ymm9		;; I6 * MemI6			; 46-50
	vmulpd	ymm5, ymm5, ymm3		;; I6 * MemR6			; 47-51

	vmovapd	ymm3, [srcreg+d4][rbp]		;; MemR5
	vsubpd	ymm3, ymm3, ymm12		;; MemR5-R5
	vsubpd	ymm11, ymm11, ymm9		;; R6*MemR6-I6*MemI6 (new R6)	; 55-57
	vmovapd	ymm9, [srcreg+d4+32][rbp]	;; MemI5
	vsubpd	ymm9, ymm9, ymm15		;; MemI5-I5
	vaddpd	ymm1, ymm1, ymm5		;; R6*MemI6+I6*MemR6 (new I6)	; 56-58
	vmulpd	ymm5, ymm12, ymm3		;; R5 * MemR5			; 44-48
	vmulpd	ymm12, ymm12, ymm9		;; R5 * MemI5			; 45-49
	vmulpd	ymm9, ymm15, ymm9		;; I5 * MemI5			; 48-52
	vmulpd	ymm15, ymm15, ymm3		;; I5 * MemR5			; 49-53

	vmovapd	ymm3, [srcreg+d4+d2+d1][rbp]	;; MemR8
	vsubpd	ymm3, ymm3, ymm14		;; MemR8-R8
	vsubpd	ymm5, ymm5, ymm9		;; R5*MemR5-I5*MemI5 (new R5)	; 57-59
	vmovapd	ymm9, [srcreg+d4+d2+d1+32][rbp]	;; MemI8
	vsubpd	ymm9, ymm9, ymm8		;; MemI8-I8
	vaddpd	ymm12, ymm12, ymm15		;; R5*MemI5+I5*MemR5 (new I5)	; 58-60
	vmulpd	ymm15, ymm14, ymm3		;; R8 * MemR8			; 50-54
	vmulpd	ymm14, ymm14, ymm9		;; R8 * MemI8			; 51-55
	vmulpd	ymm9, ymm8, ymm9		;; I8 * MemI8			; 52-56
	vmulpd	ymm8, ymm8, ymm3		;; I8 * MemR8			; 53-57

	vmovapd	ymm3, [srcreg+d4+d2][rbp]	;; MemR7
	vsubpd	ymm3, ymm3, ymm0		;; MemR7-R7
	vsubpd	ymm15, ymm15, ymm9		;; R8*MemR8-I8*MemI8 (new R8)	; 59-61
	vmovapd	ymm9, [srcreg+d4+d2+32][rbp]	;; MemI7
	vsubpd	ymm9, ymm9, ymm6		;; MemI7-I7
	vaddpd	ymm14, ymm14, ymm8		;; R8*MemI8+I8*MemR8 (new I8)	; 60-62
	vmulpd	ymm8, ymm0, ymm3		;; R7 * MemR7			; 54-58
	vmulpd	ymm0, ymm0, ymm9		;; R7 * MemI7			; 55-59
	vmulpd	ymm9, ymm6, ymm9		;; I7 * MemI7			; 56-60
	vmulpd	ymm6, ymm6, ymm3		;; I7 * MemR7			; 57-61

	vsubpd	ymm8, ymm8, ymm9		;; R7*MemR7-I7*MemI7 (new R7)	; 61-63
	vaddpd	ymm0, ymm0, ymm6		;; R7*MemI7+I7*MemR7 (new I7)	; 62-64

	vaddpd	ymm9, ymm4, ymm10		;; R4 + R3 (new R3)		; 65-67
	vsubpd	ymm4, ymm4, ymm10		;; R4 - R3 (new I4)		; 66-68

	vmovapd	ymm6, [srcreg+d1]		;; Reload R2
	vmovapd	ymm10, [srcreg+d1][rbp]		;; MemR2
	vsubpd	ymm10, ymm10, ymm6		;; MemR2-R2
	ystore	YMM_TMP5, ymm9			;; Temporarily save new R3	; 68
	vmovapd	ymm9, [srcreg+d1+32][rbp]	;; MemI2
	vsubpd	ymm9, ymm9, ymm13		;; MemI2-I2
	ystore	YMM_TMP6, ymm4			;; Temporarily save new I4	; 69
	vmulpd	ymm4, ymm6, ymm10		;; R2 * MemR2			; 66-70
	vmulpd	ymm6, ymm6, ymm9		;; R2 * MemI2			; 67-71
	vmulpd	ymm9, ymm13, ymm9		;; I2 * MemI2			; 68-72
	vmulpd	ymm13, ymm13, ymm10		;; I2 * MemR2			; 69-73

	vmovapd	ymm3, [srcreg]			;; Reload R1
	vmovapd	ymm10, [srcreg][rbp]		;; MemR1
	vsubpd	ymm10, ymm10, ymm3		;; MemR1-R1
	vsubpd	ymm4, ymm4, ymm9		;; R2*MemR2-I2*MemI2 (new R2)	; 73-75
	vmovapd	ymm9, [srcreg+32][rbp]		;; MemI1
	vsubpd	ymm9, ymm9, [srcreg+32]		;; MemI1-I1
	vaddpd	ymm6, ymm6, ymm13		;; R2*MemI2+I2*MemR2 (new I2)	; 74-76

	vsubpd	ymm13, ymm2, ymm7		;; I3 - I4 (new R4)		; 67-69
	vaddpd	ymm2, ymm2, ymm7		;; I3 + I4 (new I3)		; 68-70

	vsubpd	ymm7, ymm12, ymm1		;; I5 - I6 (new I6)		; 69-71
	vaddpd	ymm12, ymm12, ymm1		;; I5 + I6 (new I5)		; 70-72

	vsubpd	ymm1, ymm5, ymm11		;; R5 - R6 (new R6)		; 71-73
	vaddpd	ymm5, ymm5, ymm11		;; R5 + R6 (new R5)		; 72-74

	vmulpd	ymm11, ymm3, ymm10		;; R1 * MemR1			; 70-74
	vmulpd	ymm3, ymm3, ymm9		;; R1 * MemI1			; 71-75
	vmovsd	Q [srcreg+r8-16], xmm11		;; Save product of sum of FFT values
	vmulpd	ymm9, ymm9, [srcreg+32]		;; (reload I1) * MemI1		; 72-76
	vmulpd	ymm10, ymm10, [srcreg+32]	;; (reload I1) * MemR1		; 73-77
	vmovsd	Q YMM_TMP1[8], xmm9		;; I1

	vsubpd	ymm11, ymm11, ymm9		;; R1*MemR1-I1*MemI1 (new R1)	; 77-79
	vaddpd	ymm3, ymm3, ymm10		;; R1*MemI1+I1*MemR1 (new I1)	; 78-80
	jmp	back_to_orig

submul313:
	vsubpd	ymm3, ymm7, [srcreg+d2+d1][rbp]	;; R4-MemR4
	vsubpd	ymm9, ymm10, [srcreg+d2+d1+32][rbp];; I4-MemI4
	vmulpd	ymm4, ymm7, ymm3		;; R4 * MemR4			; 32-36
	vmulpd	ymm7, ymm7, ymm9		;; R4 * MemI4			; 33-37
	vmulpd	ymm9, ymm10, ymm9		;; I4 * MemI4			; 34-38
	vmulpd	ymm10, ymm10, ymm3		;; I4 * MemR4			; 35-39

	vsubpd	ymm3, ymm2, [srcreg+d2][rbp]	;; R3-MemR3
	vsubpd	ymm4, ymm4, ymm9		;; R4*MemR4-I4*MemI4 (new R4)	; 51-53
	vsubpd	ymm9, ymm11, [srcreg+d2+32][rbp];; I3-MemI3
	vaddpd	ymm7, ymm7, ymm10		;; R4*MemI4+I4*MemR4 (new I4)	; 52-54
	vmulpd	ymm10, ymm2, ymm3		;; R3 * MemR3			; 36-40
	vmulpd	ymm2, ymm2, ymm9		;; R3 * MemI3			; 37-41
	vmulpd	ymm9, ymm11, ymm9		;; I3 * MemI3			; 38-42
	vmulpd	ymm11, ymm11, ymm3		;; I3 * MemR3			; 39-43

	vsubpd	ymm3, ymm1, [srcreg+d4+d1][rbp]	;; R6-MemR6
	vsubpd	ymm10, ymm10, ymm9		;; R3*MemR3-I3*MemI3 (new R3)	; 53-55
	vsubpd	ymm9, ymm5, [srcreg+d4+d1+32][rbp];; I6-MemI6
	vaddpd	ymm2, ymm2, ymm11		;; R3*MemI3+I3*MemR3 (new I3)	; 54-56
	vmulpd	ymm11, ymm1, ymm3		;; R6 * MemR6			; 42-46
	vmulpd	ymm1, ymm1, ymm9		;; R6 * MemI6			; 43-47
	vmulpd	ymm9, ymm5, ymm9		;; I6 * MemI6			; 46-50
	vmulpd	ymm5, ymm5, ymm3		;; I6 * MemR6			; 47-51

	vsubpd	ymm3, ymm12, [srcreg+d4][rbp]	;; R5-MemR5
	vsubpd	ymm11, ymm11, ymm9		;; R6*MemR6-I6*MemI6 (new R6)	; 55-57
	vsubpd	ymm9, ymm15, [srcreg+d4+32][rbp];; I5-MemI5
	vaddpd	ymm1, ymm1, ymm5		;; R6*MemI6+I6*MemR6 (new I6)	; 56-58
	vmulpd	ymm5, ymm12, ymm3		;; R5 * MemR5			; 44-48
	vmulpd	ymm12, ymm12, ymm9		;; R5 * MemI5			; 45-49
	vmulpd	ymm9, ymm15, ymm9		;; I5 * MemI5			; 48-52
	vmulpd	ymm15, ymm15, ymm3		;; I5 * MemR5			; 49-53

	vsubpd	ymm3, ymm14, [srcreg+d4+d2+d1][rbp];; R8-MemR8
	vsubpd	ymm5, ymm5, ymm9		;; R5*MemR5-I5*MemI5 (new R5)	; 57-59
	vsubpd	ymm9, ymm8, [srcreg+d4+d2+d1+32][rbp];; I8-MemI8
	vaddpd	ymm12, ymm12, ymm15		;; R5*MemI5+I5*MemR5 (new I5)	; 58-60
	vmulpd	ymm15, ymm14, ymm3		;; R8 * MemR8			; 50-54
	vmulpd	ymm14, ymm14, ymm9		;; R8 * MemI8			; 51-55
	vmulpd	ymm9, ymm8, ymm9		;; I8 * MemI8			; 52-56
	vmulpd	ymm8, ymm8, ymm3		;; I8 * MemR8			; 53-57

	vsubpd	ymm3, ymm0, [srcreg+d4+d2][rbp]	;; R7-MemR7
	vsubpd	ymm15, ymm15, ymm9		;; R8*MemR8-I8*MemI8 (new R8)	; 59-61
	vsubpd	ymm9, ymm6, [srcreg+d4+d2+32][rbp];; I7-MemI7
	vaddpd	ymm14, ymm14, ymm8		;; R8*MemI8+I8*MemR8 (new I8)	; 60-62
	vmulpd	ymm8, ymm0, ymm3		;; R7 * MemR7			; 54-58
	vmulpd	ymm0, ymm0, ymm9		;; R7 * MemI7			; 55-59
	vmulpd	ymm9, ymm6, ymm9		;; I7 * MemI7			; 56-60
	vmulpd	ymm6, ymm6, ymm3		;; I7 * MemR7			; 57-61

	vsubpd	ymm8, ymm8, ymm9		;; R7*MemR7-I7*MemI7 (new R7)	; 61-63
	vaddpd	ymm0, ymm0, ymm6		;; R7*MemI7+I7*MemR7 (new I7)	; 62-64

	vaddpd	ymm9, ymm4, ymm10		;; R4 + R3 (new R3)		; 65-67
	vsubpd	ymm4, ymm4, ymm10		;; R4 - R3 (new I4)		; 66-68

	vmovapd	ymm6, [srcreg+d1]		;; Reload R2
	vsubpd	ymm10, ymm6, [srcreg+d1][rbp]	;; R2-MemR2
	ystore	YMM_TMP5, ymm9			;; Temporarily save new R3	; 68
	vsubpd	ymm9, ymm13, [srcreg+d1+32][rbp];; I2-MemI2
	ystore	YMM_TMP6, ymm4			;; Temporarily save new I4	; 69
	vmulpd	ymm4, ymm6, ymm10		;; R2 * MemR2			; 66-70
	vmulpd	ymm6, ymm6, ymm9		;; R2 * MemI2			; 67-71
	vmulpd	ymm9, ymm13, ymm9		;; I2 * MemI2			; 68-72
	vmulpd	ymm13, ymm13, ymm10		;; I2 * MemR2			; 69-73

	vmovapd	ymm3, [srcreg]			;; Reload R1
	vsubpd	ymm10, ymm3, [srcreg][rbp]	;; R1-MemR1
	vsubpd	ymm4, ymm4, ymm9		;; R2*MemR2-I2*MemI2 (new R2)	; 73-75
	vmovapd	ymm9, [srcreg+32]		;; Reload I1
	vsubpd	ymm9, ymm9, [srcreg+32][rbp]	;; I1-MemI1
	vaddpd	ymm6, ymm6, ymm13		;; R2*MemI2+I2*MemR2 (new I2)	; 74-76

	vsubpd	ymm13, ymm2, ymm7		;; I3 - I4 (new R4)		; 67-69
	vaddpd	ymm2, ymm2, ymm7		;; I3 + I4 (new I3)		; 68-70

	vsubpd	ymm7, ymm12, ymm1		;; I5 - I6 (new I6)		; 69-71
	vaddpd	ymm12, ymm12, ymm1		;; I5 + I6 (new I5)		; 70-72

	vsubpd	ymm1, ymm5, ymm11		;; R5 - R6 (new R6)		; 71-73
	vaddpd	ymm5, ymm5, ymm11		;; R5 + R6 (new R5)		; 72-74

	vmulpd	ymm11, ymm3, ymm10		;; R1 * MemR1			; 70-74
	vmulpd	ymm3, ymm3, ymm9		;; R1 * MemI1			; 71-75
	vmovsd	Q [srcreg+r8-16], xmm11		;; Save product of sum of FFT values
	vmulpd	ymm9, ymm9, [srcreg+32]		;; (reload I1) * MemI1		; 72-76
	vmulpd	ymm10, ymm10, [srcreg+32]	;; (reload I1) * MemR1		; 73-77
	vmovsd	Q YMM_TMP1[8], xmm9		;; I1

	vsubpd	ymm11, ymm11, ymm9		;; R1*MemR1-I1*MemI1 (new R1)	; 77-79
	vaddpd	ymm3, ymm3, ymm10		;; R1*MemI1+I1*MemR1 (new I1)	; 78-80
	jmp	back_to_orig

submul132:
	vmovapd	ymm3, [srcreg+d2+d1][rbp]	;; MemR4
	vsubpd	ymm3, ymm3, ymm7		;; MemR4-R4
	vmovapd	ymm7, [srcreg+d2+d1][r9]	;; R4=MemR4
	vmovapd	ymm9, [srcreg+d2+d1+32][rbp]	;; MemI4
	vsubpd	ymm9, ymm9, ymm10		;; MemI4-I4
	vmovapd	ymm10, [srcreg+d2+d1+32][r9]	;; I4=MemI4
	vmulpd	ymm4, ymm7, ymm3		;; R4 * MemR4			; 32-36
	vmulpd	ymm7, ymm7, ymm9		;; R4 * MemI4			; 33-37
	vmulpd	ymm9, ymm10, ymm9		;; I4 * MemI4			; 34-38
	vmulpd	ymm10, ymm10, ymm3		;; I4 * MemR4			; 35-39

	vmovapd	ymm3, [srcreg+d2][rbp]		;; MemR3
	vsubpd	ymm3, ymm3, ymm2		;; MemR3-R3
	vmovapd	ymm2, [srcreg+d2][r9]		;; R3=MemR3
	vsubpd	ymm4, ymm4, ymm9		;; R4*MemR4-I4*MemI4 (new R4)	; 51-53
	vmovapd	ymm9, [srcreg+d2+32][rbp]	;; MemI3
	vsubpd	ymm9, ymm9, ymm11		;; MemI3-I3
	vmovapd	ymm11, [srcreg+d2+32][r9]	;; I3=MemI3
	vaddpd	ymm7, ymm7, ymm10		;; R4*MemI4+I4*MemR4 (new I4)	; 52-54
	vmulpd	ymm10, ymm2, ymm3		;; R3 * MemR3			; 36-40
	vmulpd	ymm2, ymm2, ymm9		;; R3 * MemI3			; 37-41
	vmulpd	ymm9, ymm11, ymm9		;; I3 * MemI3			; 38-42
	vmulpd	ymm11, ymm11, ymm3		;; I3 * MemR3			; 39-43

	vmovapd	ymm3, [srcreg+d4+d1][rbp]	;; MemR6
	vsubpd	ymm3, ymm3, ymm1		;; MemR6-R6
	vmovapd	ymm1, [srcreg+d4+d1][r9]	;; R6=MemR6
	vsubpd	ymm10, ymm10, ymm9		;; R3*MemR3-I3*MemI3 (new R3)	; 53-55
	vmovapd	ymm9, [srcreg+d4+d1+32][rbp]	;; MemI6
	vsubpd	ymm9, ymm9, ymm5		;; MemI6-I6
	vmovapd	ymm5, [srcreg+d4+d1+32][r9]	;; I6=MemI6
	vaddpd	ymm2, ymm2, ymm11		;; R3*MemI3+I3*MemR3 (new I3)	; 54-56
	vmulpd	ymm11, ymm1, ymm3		;; R6 * MemR6			; 42-46
	vmulpd	ymm1, ymm1, ymm9		;; R6 * MemI6			; 43-47
	vmulpd	ymm9, ymm5, ymm9		;; I6 * MemI6			; 46-50
	vmulpd	ymm5, ymm5, ymm3		;; I6 * MemR6			; 47-51

	vmovapd	ymm3, [srcreg+d4][rbp]		;; MemR5
	vsubpd	ymm3, ymm3, ymm12		;; MemR5-R5
	vmovapd	ymm12, [srcreg+d4][r9]		;; R5=MemR5
	vsubpd	ymm11, ymm11, ymm9		;; R6*MemR6-I6*MemI6 (new R6)	; 55-57
	vmovapd	ymm9, [srcreg+d4+32][rbp]	;; MemI5
	vsubpd	ymm9, ymm9, ymm15		;; MemI5-I5
	vmovapd	ymm15, [srcreg+d4+32][r9]	;; I5=MemI5
	vaddpd	ymm1, ymm1, ymm5		;; R6*MemI6+I6*MemR6 (new I6)	; 56-58
	vmulpd	ymm5, ymm12, ymm3		;; R5 * MemR5			; 44-48
	vmulpd	ymm12, ymm12, ymm9		;; R5 * MemI5			; 45-49
	vmulpd	ymm9, ymm15, ymm9		;; I5 * MemI5			; 48-52
	vmulpd	ymm15, ymm15, ymm3		;; I5 * MemR5			; 49-53

	vmovapd	ymm3, [srcreg+d4+d2+d1][rbp]	;; MemR8
	vsubpd	ymm3, ymm3, ymm14		;; MemR8-R8
	vmovapd	ymm14, [srcreg+d4+d2+d1][r9]	;; R8=MemR8
	vsubpd	ymm5, ymm5, ymm9		;; R5*MemR5-I5*MemI5 (new R5)	; 57-59
	vmovapd	ymm9, [srcreg+d4+d2+d1+32][rbp]	;; MemI8
	vsubpd	ymm9, ymm9, ymm8		;; MemI8-I8
	vmovapd	ymm8, [srcreg+d4+d2+d1+32][r9]	;; I8=MemI8
	vaddpd	ymm12, ymm12, ymm15		;; R5*MemI5+I5*MemR5 (new I5)	; 58-60
	vmulpd	ymm15, ymm14, ymm3		;; R8 * MemR8			; 50-54
	vmulpd	ymm14, ymm14, ymm9		;; R8 * MemI8			; 51-55
	vmulpd	ymm9, ymm8, ymm9		;; I8 * MemI8			; 52-56
	vmulpd	ymm8, ymm8, ymm3		;; I8 * MemR8			; 53-57

	vmovapd	ymm3, [srcreg+d4+d2][rbp]	;; MemR7
	vsubpd	ymm3, ymm3, ymm0		;; MemR7-R7
	vmovapd	ymm0, [srcreg+d4+d2][r9]	;; R7=MemR7
	vsubpd	ymm15, ymm15, ymm9		;; R8*MemR8-I8*MemI8 (new R8)	; 59-61
	vmovapd	ymm9, [srcreg+d4+d2+32][rbp]	;; MemI7
	vsubpd	ymm9, ymm9, ymm6		;; MemI7-I7
	vmovapd	ymm6, [srcreg+d4+d2+32][r9]	;; I7=MemI7
	vaddpd	ymm14, ymm14, ymm8		;; R8*MemI8+I8*MemR8 (new I8)	; 60-62
	vmulpd	ymm8, ymm0, ymm3		;; R7 * MemR7			; 54-58
	vmulpd	ymm0, ymm0, ymm9		;; R7 * MemI7			; 55-59
	vmulpd	ymm9, ymm6, ymm9		;; I7 * MemI7			; 56-60
	vmulpd	ymm6, ymm6, ymm3		;; I7 * MemR7			; 57-61

	vsubpd	ymm8, ymm8, ymm9		;; R7*MemR7-I7*MemI7 (new R7)	; 61-63
	vaddpd	ymm0, ymm0, ymm6		;; R7*MemI7+I7*MemR7 (new I7)	; 62-64

	vaddpd	ymm9, ymm4, ymm10		;; R4 + R3 (new R3)		; 65-67
	vsubpd	ymm4, ymm4, ymm10		;; R4 - R3 (new I4)		; 66-68

	vmovapd	ymm10, [srcreg+d1][rbp]		;; MemR2
	vsubpd	ymm10, ymm10, [srcreg+d1]	;; MemR2-(reload R2)
	vmovapd	ymm6, [srcreg+d1][r9]		;; R2=MemR2
	ystore	YMM_TMP5, ymm9			;; Temporarily save new R3	; 68
	vmovapd	ymm9, [srcreg+d1+32][rbp]	;; MemI2
	vsubpd	ymm9, ymm9, ymm13		;; MemI2-I2
	vmovapd	ymm13, [srcreg+d1+32][r9]	;; I2=MemI2
	ystore	YMM_TMP6, ymm4			;; Temporarily save new I4	; 69
	vmulpd	ymm4, ymm6, ymm10		;; R2 * MemR2			; 66-70
	vmulpd	ymm6, ymm6, ymm9		;; R2 * MemI2			; 67-71
	vmulpd	ymm9, ymm13, ymm9		;; I2 * MemI2			; 68-72
	vmulpd	ymm13, ymm13, ymm10		;; I2 * MemR2			; 69-73

	vmovapd	ymm10, [srcreg][rbp]		;; MemR1
	vsubpd	ymm10, ymm10, [srcreg]		;; MemR1-R1
	vmovapd	ymm3, [srcreg][r9]		;; R1=MemR1
	vsubpd	ymm4, ymm4, ymm9		;; R2*MemR2-I2*MemI2 (new R2)	; 73-75
	vmovapd	ymm9, [srcreg+32][rbp]		;; MemI1
	vsubpd	ymm9, ymm9, [srcreg+32]		;; MemI1-I1
	vaddpd	ymm6, ymm6, ymm13		;; R2*MemI2+I2*MemR2 (new I2)	; 74-76

	vsubpd	ymm13, ymm2, ymm7		;; I3 - I4 (new R4)		; 67-69
	vaddpd	ymm2, ymm2, ymm7		;; I3 + I4 (new I3)		; 68-70

	vsubpd	ymm7, ymm12, ymm1		;; I5 - I6 (new I6)		; 69-71
	vaddpd	ymm12, ymm12, ymm1		;; I5 + I6 (new I5)		; 70-72

	vsubpd	ymm1, ymm5, ymm11		;; R5 - R6 (new R6)		; 71-73
	vaddpd	ymm5, ymm5, ymm11		;; R5 + R6 (new R5)		; 72-74

	vmulpd	ymm11, ymm3, ymm10		;; R1 * MemR1			; 70-74
	vmulpd	ymm3, ymm3, ymm9		;; R1 * MemI1			; 71-75
	vmovsd	Q [srcreg+r8-16], xmm11		;; Save product of sum of FFT values
	vmulpd	ymm9, ymm9, [srcreg+32][r9]	;; (reload I1) * MemI1		; 72-76
	vmulpd	ymm10, ymm10, [srcreg+32][r9]	;; (reload I1) * MemR1		; 73-77
	vmovsd	Q YMM_TMP1[8], xmm9		;; I1

	vsubpd	ymm11, ymm11, ymm9		;; R1*MemR1-I1*MemI1 (new R1)	; 77-79
	vaddpd	ymm3, ymm3, ymm10		;; R1*MemI1+I1*MemR1 (new I1)	; 78-80
	jmp	back_to_orig

submul312:
	vsubpd	ymm3, ymm7, [srcreg+d2+d1][rbp]	;; R4-MemR4
	vmovapd	ymm7, [srcreg+d2+d1][r9]	;; R4=MemR4
	vsubpd	ymm9, ymm10, [srcreg+d2+d1+32][rbp];; I4-MemI4
	vmovapd	ymm10, [srcreg+d2+d1+32][r9]	;; I4=MemI4
	vmulpd	ymm4, ymm7, ymm3		;; R4 * MemR4			; 32-36
	vmulpd	ymm7, ymm7, ymm9		;; R4 * MemI4			; 33-37
	vmulpd	ymm9, ymm10, ymm9		;; I4 * MemI4			; 34-38
	vmulpd	ymm10, ymm10, ymm3		;; I4 * MemR4			; 35-39

	vsubpd	ymm3, ymm2, [srcreg+d2][rbp]	;; R3-MemR3
	vmovapd	ymm2, [srcreg+d2][r9]		;; R3=MemR3
	vsubpd	ymm4, ymm4, ymm9		;; R4*MemR4-I4*MemI4 (new R4)	; 51-53
	vsubpd	ymm9, ymm11, [srcreg+d2+32][rbp];; I3-MemI3
	vmovapd	ymm11, [srcreg+d2+32][r9]	;; I3=MemI3
	vaddpd	ymm7, ymm7, ymm10		;; R4*MemI4+I4*MemR4 (new I4)	; 52-54
	vmulpd	ymm10, ymm2, ymm3		;; R3 * MemR3			; 36-40
	vmulpd	ymm2, ymm2, ymm9		;; R3 * MemI3			; 37-41
	vmulpd	ymm9, ymm11, ymm9		;; I3 * MemI3			; 38-42
	vmulpd	ymm11, ymm11, ymm3		;; I3 * MemR3			; 39-43

	vsubpd	ymm3, ymm1, [srcreg+d4+d1][rbp]	;; R6-MemR6
	vmovapd	ymm1, [srcreg+d4+d1][r9]	;; R6=MemR6
	vsubpd	ymm10, ymm10, ymm9		;; R3*MemR3-I3*MemI3 (new R3)	; 53-55
	vsubpd	ymm9, ymm5, [srcreg+d4+d1+32][rbp];; I6-MemI6
	vmovapd	ymm5, [srcreg+d4+d1+32][r9]	;; I6=MemI6
	vaddpd	ymm2, ymm2, ymm11		;; R3*MemI3+I3*MemR3 (new I3)	; 54-56
	vmulpd	ymm11, ymm1, ymm3		;; R6 * MemR6			; 42-46
	vmulpd	ymm1, ymm1, ymm9		;; R6 * MemI6			; 43-47
	vmulpd	ymm9, ymm5, ymm9		;; I6 * MemI6			; 46-50
	vmulpd	ymm5, ymm5, ymm3		;; I6 * MemR6			; 47-51

	vsubpd	ymm3, ymm12, [srcreg+d4][rbp]	;; R5-MemR5
	vmovapd	ymm12, [srcreg+d4][r9]		;; R5=MemR5
	vsubpd	ymm11, ymm11, ymm9		;; R6*MemR6-I6*MemI6 (new R6)	; 55-57
	vsubpd	ymm9, ymm15, [srcreg+d4+32][rbp];; I5-MemI5
	vmovapd	ymm15, [srcreg+d4+32][r9]	;; I5=MemI5
	vaddpd	ymm1, ymm1, ymm5		;; R6*MemI6+I6*MemR6 (new I6)	; 56-58
	vmulpd	ymm5, ymm12, ymm3		;; R5 * MemR5			; 44-48
	vmulpd	ymm12, ymm12, ymm9		;; R5 * MemI5			; 45-49
	vmulpd	ymm9, ymm15, ymm9		;; I5 * MemI5			; 48-52
	vmulpd	ymm15, ymm15, ymm3		;; I5 * MemR5			; 49-53

	vsubpd	ymm3, ymm14, [srcreg+d4+d2+d1][rbp];; R8-MemR8
	vmovapd	ymm14, [srcreg+d4+d2+d1][r9]	;; R8=MemR8
	vsubpd	ymm5, ymm5, ymm9		;; R5*MemR5-I5*MemI5 (new R5)	; 57-59
	vsubpd	ymm9, ymm8, [srcreg+d4+d2+d1+32][rbp];; I8-MemI8
	vmovapd	ymm8, [srcreg+d4+d2+d1+32][r9]	;; I8=MemI8
	vaddpd	ymm12, ymm12, ymm15		;; R5*MemI5+I5*MemR5 (new I5)	; 58-60
	vmulpd	ymm15, ymm14, ymm3		;; R8 * MemR8			; 50-54
	vmulpd	ymm14, ymm14, ymm9		;; R8 * MemI8			; 51-55
	vmulpd	ymm9, ymm8, ymm9		;; I8 * MemI8			; 52-56
	vmulpd	ymm8, ymm8, ymm3		;; I8 * MemR8			; 53-57

	vsubpd	ymm3, ymm0, [srcreg+d4+d2][rbp]	;; R7-MemR7
	vmovapd	ymm0, [srcreg+d4+d2][r9]	;; R7=MemR7
	vsubpd	ymm15, ymm15, ymm9		;; R8*MemR8-I8*MemI8 (new R8)	; 59-61
	vsubpd	ymm9, ymm6, [srcreg+d4+d2+32][rbp];; I7-MemI7
	vmovapd	ymm6, [srcreg+d4+d2+32][r9]	;; I7=MemI7
	vaddpd	ymm14, ymm14, ymm8		;; R8*MemI8+I8*MemR8 (new I8)	; 60-62
	vmulpd	ymm8, ymm0, ymm3		;; R7 * MemR7			; 54-58
	vmulpd	ymm0, ymm0, ymm9		;; R7 * MemI7			; 55-59
	vmulpd	ymm9, ymm6, ymm9		;; I7 * MemI7			; 56-60
	vmulpd	ymm6, ymm6, ymm3		;; I7 * MemR7			; 57-61

	vsubpd	ymm8, ymm8, ymm9		;; R7*MemR7-I7*MemI7 (new R7)	; 61-63
	vaddpd	ymm0, ymm0, ymm6		;; R7*MemI7+I7*MemR7 (new I7)	; 62-64

	vaddpd	ymm9, ymm4, ymm10		;; R4 + R3 (new R3)		; 65-67
	vsubpd	ymm4, ymm4, ymm10		;; R4 - R3 (new I4)		; 66-68

	vmovapd	ymm6, [srcreg+d1]		;; Reload R2
	vsubpd	ymm10, ymm6, [srcreg+d1][rbp]	;; R2-MemR2
	vmovapd	ymm6, [srcreg+d1][r9]		;; R2=MemR2
	ystore	YMM_TMP5, ymm9			;; Temporarily save new R3	; 68
	vsubpd	ymm9, ymm13, [srcreg+d1+32][rbp];; I2-MemI2
	vmovapd	ymm13, [srcreg+d1+32][r9]	;; I2=MemI2
	ystore	YMM_TMP6, ymm4			;; Temporarily save new I4	; 69
	vmulpd	ymm4, ymm6, ymm10		;; R2 * MemR2			; 66-70
	vmulpd	ymm6, ymm6, ymm9		;; R2 * MemI2			; 67-71
	vmulpd	ymm9, ymm13, ymm9		;; I2 * MemI2			; 68-72
	vmulpd	ymm13, ymm13, ymm10		;; I2 * MemR2			; 69-73

	vmovapd	ymm3, [srcreg]			;; Reload R1
	vsubpd	ymm10, ymm3, [srcreg][rbp]	;; R1-MemR1
	vmovapd	ymm3, [srcreg][r9]		;; R1=MemR1
	vsubpd	ymm4, ymm4, ymm9		;; R2*MemR2-I2*MemI2 (new R2)	; 73-75
	vmovapd	ymm9, [srcreg+32]		;; Reload I1
	vsubpd	ymm9, ymm9, [srcreg+32][rbp]	;; I1-MemI1
	vaddpd	ymm6, ymm6, ymm13		;; R2*MemI2+I2*MemR2 (new I2)	; 74-76

	vsubpd	ymm13, ymm2, ymm7		;; I3 - I4 (new R4)		; 67-69
	vaddpd	ymm2, ymm2, ymm7		;; I3 + I4 (new I3)		; 68-70

	vsubpd	ymm7, ymm12, ymm1		;; I5 - I6 (new I6)		; 69-71
	vaddpd	ymm12, ymm12, ymm1		;; I5 + I6 (new I5)		; 70-72

	vsubpd	ymm1, ymm5, ymm11		;; R5 - R6 (new R6)		; 71-73
	vaddpd	ymm5, ymm5, ymm11		;; R5 + R6 (new R5)		; 72-74

	vmulpd	ymm11, ymm3, ymm10		;; R1 * MemR1			; 70-74
	vmulpd	ymm3, ymm3, ymm9		;; R1 * MemI1			; 71-75
	vmovsd	Q [srcreg+r8-16], xmm11		;; Save product of sum of FFT values
	vmulpd	ymm9, ymm9, [srcreg+32][r9]	;; (reload I1) * MemI1		; 72-76
	vmulpd	ymm10, ymm10, [srcreg+32][r9]	;; (reload I1) * MemR1		; 73-77
	vmovsd	Q YMM_TMP1[8], xmm9		;; I1

	vsubpd	ymm11, ymm11, ymm9		;; R1*MemR1-I1*MemI1 (new R1)	; 77-79
	vaddpd	ymm3, ymm3, ymm10		;; R1*MemI1+I1*MemR1 (new I1)	; 78-80
	jmp	back_to_orig

unfft:	vaddpd	ymm9, ymm7, ymm2		;; R4 + R3 (new R3)		; 65-67
	vsubpd	ymm7, ymm7, ymm2		;; R4 - R3 (new I4)		; 66-68
	ystore	YMM_TMP5, ymm9			;; Temporarily save new R3	; 68
	ystore	YMM_TMP6, ymm7			;; Temporarily save new I4	; 69

	;inuse 0,1,5,6,8,10,11,12,13,14,15

	vsubpd	ymm7, ymm15, ymm5		;; I5 - I6 (new I6)		; 69-71
	vaddpd	ymm3, ymm15, ymm5		;; I5 + I6 (new I5)		; 70-72

	;inuse 0,1,3,6,7,8,10,11,12,13,14

	vmovapd	ymm15, ymm14			;; R8
	vmovapd	ymm14, ymm8			;; I8
	vmovapd	ymm8, ymm0			;; R7
	vmovapd	ymm0, ymm6			;; I7
	vmovapd	ymm6, ymm13			;; I2

	;inuse 0,1,3,6,7,8,10,11,12,14,15

	vaddpd	ymm2, ymm11, ymm10		;; I3 + I4 (new I3)		; 68-70
	vsubpd	ymm13, ymm11, ymm10		;; I3 - I4 (new R4)		; 67-69

	;inuse 0,1,2,3,6,7,8,12,13,14,15

	vmovapd	ymm11, [srcreg]			;; Reload newer R1
	vmovapd	ymm4, [srcreg+d1]		;; Reload newer R2
	vmovsd	Q [srcreg+r8-16], xmm11		;; Save product of sum of FFT values

	vaddpd	ymm5, ymm12, ymm1		;; R5 + R6 (new R5)		; 72-74
	vsubpd	ymm1, ymm12, ymm1		;; R5 - R6 (new R6)		; 71-73

	vmovapd ymm12, ymm3			;; I5
	vmovapd	ymm3, [srcreg+32]		;; I1
	vmovsd	Q YMM_TMP1[8], xmm3		;; I1
	jmp	back_to_orig

muladd:	vmovapd	ymm3, [srcreg+d2+d1][rbp]	;; MemR4
	vmovapd	ymm9, [srcreg+d2+d1+32][rbp]	;; MemI4
	vmulpd	ymm4, ymm7, ymm3		;; R4 * MemR4			; 32-36
	vmulpd	ymm7, ymm7, ymm9		;; R4 * MemI4			; 33-37
	vmulpd	ymm9, ymm10, ymm9		;; I4 * MemI4			; 34-38
	vmulpd	ymm10, ymm10, ymm3		;; I4 * MemR4			; 35-39

	vmovapd	ymm3, [srcreg+d2][rbp]		;; MemR3
	vsubpd	ymm4, ymm4, ymm9		;; R4*MemR4-I4*MemI4 (new R4)	; 51-53
	vmovapd	ymm9, [srcreg+d2+32][rbp]	;; MemI3
	vaddpd	ymm7, ymm7, ymm10		;; R4*MemI4+I4*MemR4 (new I4)	; 52-54
	vmulpd	ymm10, ymm2, ymm3		;; R3 * MemR3			; 36-40
	vmulpd	ymm2, ymm2, ymm9		;; R3 * MemI3			; 37-41
	vmulpd	ymm9, ymm11, ymm9		;; I3 * MemI3			; 38-42
	vmulpd	ymm11, ymm11, ymm3		;; I3 * MemR3			; 39-43

	vmovapd	ymm3, [srcreg+d4+d1][rbp]	;; MemR6
	vsubpd	ymm10, ymm10, ymm9		;; R3*MemR3-I3*MemI3 (new R3)	; 53-55
	vmovapd	ymm9, [srcreg+d4+d1+32][rbp]	;; MemI6
	vaddpd	ymm2, ymm2, ymm11		;; R3*MemI3+I3*MemR3 (new I3)	; 54-56
	vmulpd	ymm11, ymm1, ymm3		;; R6 * MemR6			; 42-46
	vmulpd	ymm1, ymm1, ymm9		;; R6 * MemI6			; 43-47
	vmulpd	ymm9, ymm5, ymm9		;; I6 * MemI6			; 46-50
	vmulpd	ymm5, ymm5, ymm3		;; I6 * MemR6			; 47-51

	vmovapd	ymm3, [srcreg+d4][rbp]		;; MemR5
	vsubpd	ymm11, ymm11, ymm9		;; R6*MemR6-I6*MemI6 (new R6)	; 55-57
	vmovapd	ymm9, [srcreg+d4+32][rbp]	;; MemI5
	vaddpd	ymm1, ymm1, ymm5		;; R6*MemI6+I6*MemR6 (new I6)	; 56-58
	vmulpd	ymm5, ymm12, ymm3		;; R5 * MemR5			; 44-48
	vmulpd	ymm12, ymm12, ymm9		;; R5 * MemI5			; 45-49
	vmulpd	ymm9, ymm15, ymm9		;; I5 * MemI5			; 48-52
	vmulpd	ymm15, ymm15, ymm3		;; I5 * MemR5			; 49-53

	vmovapd	ymm3, [srcreg+d4+d2+d1][rbp]	;; MemR8
	vsubpd	ymm5, ymm5, ymm9		;; R5*MemR5-I5*MemI5 (new R5)	; 57-59
	vmovapd	ymm9, [srcreg+d4+d2+d1+32][rbp]	;; MemI8
	vaddpd	ymm12, ymm12, ymm15		;; R5*MemI5+I5*MemR5 (new I5)	; 58-60
	vmulpd	ymm15, ymm14, ymm3		;; R8 * MemR8			; 50-54
	vmulpd	ymm14, ymm14, ymm9		;; R8 * MemI8			; 51-55
	vmulpd	ymm9, ymm8, ymm9		;; I8 * MemI8			; 52-56
	vmulpd	ymm8, ymm8, ymm3		;; I8 * MemR8			; 53-57

	vmovapd	ymm3, [srcreg+d4+d2][rbp]	;; MemR7
	vsubpd	ymm15, ymm15, ymm9		;; R8*MemR8-I8*MemI8 (new R8)	; 59-61
	vmovapd	ymm9, [srcreg+d4+d2+32][rbp]	;; MemI7
	vaddpd	ymm14, ymm14, ymm8		;; R8*MemI8+I8*MemR8 (new I8)	; 60-62
	vmulpd	ymm8, ymm0, ymm3		;; R7 * MemR7			; 54-58
	vmulpd	ymm0, ymm0, ymm9		;; R7 * MemI7			; 55-59
	vmulpd	ymm9, ymm6, ymm9		;; I7 * MemI7			; 56-60
	vmulpd	ymm6, ymm6, ymm3		;; I7 * MemR7			; 57-61

	vsubpd	ymm8, ymm8, ymm9		;; R7*MemR7-I7*MemI7 (new R7)	; 61-63
	vaddpd	ymm0, ymm0, ymm6		;; R7*MemI7+I7*MemR7 (new I7)	; 62-64

	vmovapd	ymm3, [srcreg]			;; Reload R1
	vmovapd	ymm6, [srcreg+d1]		;; Reload R2

	mov	r10, SRC3ARG			;; Load distance to FFT(1)
	test	r10, 3				;; Check if we need to multiply add-in by FFT(1)
	jz	muladdhard			;; Yes, do it the hard way

	vaddpd	ymm4, ymm4, [srcreg+r9+d2+d1]		;; R4 = R4 + MemR4
	vaddpd	ymm7, ymm7, [srcreg+r9+d2+d1+32]	;; I4 = I4 + MemI4
	vaddpd	ymm10, ymm10, [srcreg+r9+d2]		;; R3 = R3 + MemR3
	vaddpd	ymm2, ymm2, [srcreg+r9+d2+32]		;; I3 = I3 + MemI3
	vaddpd	ymm11, ymm11, [srcreg+r9+d4+d1]		;; R6 = R6 + MemR6
	vaddpd	ymm1, ymm1, [srcreg+r9+d4+d1+32]	;; I6 = I6 + MemI6
	vaddpd	ymm5, ymm5, [srcreg+r9+d4]		;; R5 = R5 + MemR5
	vaddpd	ymm12, ymm12, [srcreg+r9+d4+32]		;; I5 = I5 + MemI5
	vaddpd	ymm15, ymm15, [srcreg+r9+d4+d2+d1]	;; R8 = R8 + MemR8
	vaddpd	ymm14, ymm14, [srcreg+r9+d4+d2+d1+32]	;; I8 = I8 + MemI8
	vaddpd	ymm8, ymm8, [srcreg+r9+d4+d2]		;; R7 = R7 + MemR7
	vaddpd	ymm0, ymm0, [srcreg+r9+d4+d2+32]	;; I7 = I7 + MemI7

	vaddpd	ymm9, ymm4, ymm10		;; R4 + R3 (new R3)		; 65-67
	vsubpd	ymm4, ymm4, ymm10		;; R4 - R3 (new I4)		; 66-68

	vmovapd	ymm10, [srcreg+d1][rbp]		;; MemR2
	ystore	YMM_TMP5, ymm9			;; Temporarily save new R3	; 68
	vmovapd	ymm9, [srcreg+d1+32][rbp]	;; MemI2
	ystore	YMM_TMP6, ymm4			;; Temporarily save new I4	; 69
	vmulpd	ymm4, ymm6, ymm10		;; R2 * MemR2			; 66-70
	vmulpd	ymm6, ymm6, ymm9		;; R2 * MemI2			; 67-71
	vmulpd	ymm9, ymm13, ymm9		;; I2 * MemI2			; 68-72
	vmulpd	ymm13, ymm13, ymm10		;; I2 * MemR2			; 69-73

	vmovapd	ymm10, [srcreg][rbp]		;; MemR1
	vsubpd	ymm4, ymm4, ymm9		;; R2*MemR2-I2*MemI2 (new R2)	; 73-75
	vmovapd	ymm9, [srcreg+32][rbp]		;; MemI1
	vaddpd	ymm6, ymm6, ymm13		;; R2*MemI2+I2*MemR2 (new I2)	; 74-76

	vsubpd	ymm13, ymm2, ymm7		;; I3 - I4 (new R4)		; 67-69
	vaddpd	ymm2, ymm2, ymm7		;; I3 + I4 (new I3)		; 68-70

	vsubpd	ymm7, ymm12, ymm1		;; I5 - I6 (new I6)		; 69-71
	vaddpd	ymm12, ymm12, ymm1		;; I5 + I6 (new I5)		; 70-72

	vsubpd	ymm1, ymm5, ymm11		;; R5 - R6 (new R6)		; 71-73
	vaddpd	ymm5, ymm5, ymm11		;; R5 + R6 (new R5)		; 72-74

	vmulpd	ymm11, ymm3, ymm10		;; R1 * MemR1 (new R1a)		; 70-74
	vmulpd	ymm3, ymm3, ymm9		;; R1 * MemI1			; 71-75
	vmulpd	ymm9, ymm9, [srcreg+32]		;; (reload I1) * MemI1 (new R1b); 72-76
	vmulpd	ymm10, ymm10, [srcreg+32]	;; (reload I1) * MemR1		; 73-77

	vaddpd	ymm3, ymm3, ymm10		;; R1*MemI1+I1*MemR1 (new I1)	; 78-80
	vsubpd	ymm10, ymm11, ymm9		;; R1*MemR1-I1*MemI1 (new R1)	; 77-79

	vaddpd	ymm4, ymm4, [srcreg+r9+d1]		;; R2 = R2 + MemR2
	vaddpd	ymm6, ymm6, [srcreg+r9+d1+32]		;; I2 = I2 + MemI2
	vaddsd	xmm11, xmm11, [srcreg+r9]		;; R1a = R1a + MemR1a
	vmovsd	Q [srcreg+r8-16], xmm11		;; Save product of sum of FFT values
	vaddpd	ymm11, ymm10, [srcreg+r9]		;; R1 = R1 + MemR1 (lower word is irrelevant)
	vmovapd	ymm10, [srcreg+r9+32]			;; MemI1
	vaddsd	xmm9, xmm9, xmm10			;; R1b = R1b + MemR1b in MemI1
	vaddpd	ymm3, ymm3, ymm10			;; I1 = I1 + MemI1 (lower word is irrelevant)
	vmovsd	Q YMM_TMP1[8], xmm9			;; Save R1b
	jmp	back_to_orig

muladdhard:
	vmovapd	YMM_TMP5, ymm3				;; Save 3 registers
	vmovapd	YMM_TMP6, ymm6
	vmovapd	YMM_TMP7, ymm13

	vmovapd	ymm3, [srcreg+r9+d2+d1]			;; MemR4
	vmovapd	ymm9, [srcreg+r10+d2+d1]		;; MemR4#2
	vmulpd	ymm13, ymm3, ymm9			;; MemR4*MemR4#2
	vaddpd	ymm4, ymm4, ymm13			;; R4 = R4 + MemR4*MemR4#2
	vmovapd	ymm6, [srcreg+r10+d2+d1+32]		;; MemI4#2
	vmulpd	ymm13, ymm3, ymm6			;; MemR4*MemI4#2
	vaddpd	ymm7, ymm7, ymm13			;; I4 = I4 + MemR4*MemI4#2
	vmovapd	ymm3, [srcreg+r9+d2+d1+32]		;; MemI4
	vmulpd	ymm13, ymm3, ymm6			;; MemI4*MemI4#2
	vsubpd	ymm4, ymm4, ymm13			;; R4 = R4 - MemI4*MemI4#2
	vmulpd	ymm13, ymm3, ymm9			;; MemI4*MemR4#2
	vaddpd	ymm7, ymm7, ymm13			;; I4 = I4 + MemI4*MemR4#2

	vmovapd	ymm3, [srcreg+r9+d2]			;; MemR3
	vmovapd	ymm9, [srcreg+r10+d2]			;; MemR3#2
	vmulpd	ymm13, ymm3, ymm9			;; MemR3*MemR3#2
	vaddpd	ymm10, ymm10, ymm13			;; R3 = R3 + MemR3*MemR3#2
	vmovapd	ymm6, [srcreg+r10+d2+32]		;; MemI3#2
	vmulpd	ymm13, ymm3, ymm6			;; MemR3*MemI3#2
	vaddpd	ymm2, ymm2, ymm13			;; I3 = I3 + MemR3*MemI3#2
	vmovapd	ymm3, [srcreg+r9+d2+32]			;; MemI3
	vmulpd	ymm13, ymm3, ymm6			;; MemI3*MemI3#2
	vsubpd	ymm10, ymm10, ymm13			;; R3 = R3 - MemI3*MemI3#2
	vmulpd	ymm13, ymm3, ymm9			;; MemI3*MemR3#2
	vaddpd	ymm2, ymm2, ymm13			;; I3 = I3 + MemI3*MemR3#2

	vmovapd	ymm3, [srcreg+r9+d4+d1]			;; MemR6
	vmovapd	ymm9, [srcreg+r10+d4+d1]		;; MemR6#2
	vmulpd	ymm13, ymm3, ymm9			;; MemR6*MemR6#2
	vaddpd	ymm11, ymm11, ymm13			;; R6 = R6 + MemR6*MemR6#2
	vmovapd	ymm6, [srcreg+r10+d4+d1+32]		;; MemI6#2
	vmulpd	ymm13, ymm3, ymm6			;; MemR6*MemI6#2
	vaddpd	ymm1, ymm1, ymm13			;; I6 = I6 + MemR6*MemI6#2
	vmovapd	ymm3, [srcreg+r9+d4+d1+32]		;; MemI6
	vmulpd	ymm13, ymm3, ymm6			;; MemI6*MemI6#2
	vsubpd	ymm11, ymm11, ymm13			;; R6 = R6 - MemI6*MemI6#2
	vmulpd	ymm13, ymm3, ymm9			;; MemI6*MemR6#2
	vaddpd	ymm1, ymm1, ymm13			;; I6 = I6 + MemI6*MemR6#2

	vmovapd	ymm3, [srcreg+r9+d4]			;; MemR5
	vmovapd	ymm9, [srcreg+r10+d4]			;; MemR5#2
	vmulpd	ymm13, ymm3, ymm9			;; MemR5*MemR5#2
	vaddpd	ymm5, ymm5, ymm13			;; R5 = R5 + MemR5*MemR5#2
	vmovapd	ymm6, [srcreg+r10+d4+32]		;; MemI5#2
	vmulpd	ymm13, ymm3, ymm6			;; MemR5*MemI5#2
	vaddpd	ymm12, ymm12, ymm13			;; I5 = I5 + MemR5*MemI5#2
	vmovapd	ymm3, [srcreg+r9+d4+32]			;; MemI5
	vmulpd	ymm13, ymm3, ymm6			;; MemI5*MemI5#2
	vsubpd	ymm5, ymm5, ymm13			;; R5 = R5 - MemI5*MemI5#2
	vmulpd	ymm13, ymm3, ymm9			;; MemI5*MemR5#2
	vaddpd	ymm12, ymm12, ymm13			;; I5 = I5 + MemI5*MemR5#2

	vmovapd	ymm3, [srcreg+r9+d4+d2+d1]		;; MemR8
	vmovapd	ymm9, [srcreg+r10+d4+d2+d1]		;; MemR8#2
	vmulpd	ymm13, ymm3, ymm9			;; MemR8*MemR8#2
	vaddpd	ymm15, ymm15, ymm13			;; R8 = R8 + MemR8*MemR8#2
	vmovapd	ymm6, [srcreg+r10+d4+d2+d1+32]		;; MemI8#2
	vmulpd	ymm13, ymm3, ymm6			;; MemR8*MemI8#2
	vaddpd	ymm14, ymm14, ymm13			;; I8 = I8 + MemR8*MemI8#2
	vmovapd	ymm3, [srcreg+r9+d4+d2+d1+32]		;; MemI8
	vmulpd	ymm13, ymm3, ymm6			;; MemI8*MemI8#2
	vsubpd	ymm15, ymm15, ymm13			;; R8 = R8 - MemI8*MemI8#2
	vmulpd	ymm13, ymm3, ymm9			;; MemI8*MemR8#2
	vaddpd	ymm14, ymm14, ymm13			;; I8 = I8 + MemI8*MemR8#2

	vmovapd	ymm3, [srcreg+r9+d4+d2]			;; MemR7
	vmovapd	ymm9, [srcreg+r10+d4+d2]		;; MemR7#2
	vmulpd	ymm13, ymm3, ymm9			;; MemR7*MemR7#2
	vaddpd	ymm8, ymm8, ymm13			;; R7 = R7 + MemR7*MemR7#2
	vmovapd	ymm6, [srcreg+r10+d4+d2+32]		;; MemI7#2
	vmulpd	ymm13, ymm3, ymm6			;; MemR7*MemI7#2
	vaddpd	ymm0, ymm0, ymm13			;; I7 = I7 + MemR7*MemI7#2
	vmovapd	ymm3, [srcreg+r9+d4+d2+32]		;; MemI7
	vmulpd	ymm13, ymm3, ymm6			;; MemI7*MemI7#2
	vsubpd	ymm8, ymm8, ymm13			;; R7 = R7 - MemI7*MemI7#2
	vmulpd	ymm13, ymm3, ymm9			;; MemI7*MemR7#2
	vaddpd	ymm0, ymm0, ymm13			;; I7 = I7 + MemI7*MemR7#2

	vmovapd	ymm3, YMM_TMP5
	vmovapd	ymm6, YMM_TMP6
	vmovapd	ymm13, YMM_TMP7

	vaddpd	ymm9, ymm4, ymm10		;; R4 + R3 (new R3)		; 65-67
	vsubpd	ymm4, ymm4, ymm10		;; R4 - R3 (new I4)		; 66-68

	vmovapd	ymm10, [srcreg+d1][rbp]		;; MemR2
	ystore	YMM_TMP5, ymm9			;; Temporarily save new R3	; 68
	vmovapd	ymm9, [srcreg+d1+32][rbp]	;; MemI2
	ystore	YMM_TMP6, ymm4			;; Temporarily save new I4	; 69

	vmulpd	ymm4, ymm6, ymm10		;; R2 * MemR2			; 66-70
	vmulpd	ymm6, ymm6, ymm9		;; R2 * MemI2			; 67-71
	vmulpd	ymm9, ymm13, ymm9		;; I2 * MemI2			; 68-72
	vmulpd	ymm13, ymm13, ymm10		;; I2 * MemR2			; 69-73
	vmovapd	ymm10, [srcreg][rbp]		;; MemR1
	vsubpd	ymm4, ymm4, ymm9		;; R2*MemR2-I2*MemI2 (new R2)	; 73-75
	vmovapd	ymm9, [srcreg+32][rbp]		;; MemI1
	vaddpd	ymm6, ymm6, ymm13		;; R2*MemI2+I2*MemR2 (new I2)	; 74-76

	vsubpd	ymm13, ymm2, ymm7		;; I3 - I4 (new R4)		; 67-69
	vaddpd	ymm2, ymm2, ymm7		;; I3 + I4 (new I3)		; 68-70

	vsubpd	ymm7, ymm12, ymm1		;; I5 - I6 (new I6)		; 69-71
	vaddpd	ymm12, ymm12, ymm1		;; I5 + I6 (new I5)		; 70-72

	vsubpd	ymm1, ymm5, ymm11		;; R5 - R6 (new R6)		; 71-73
	vaddpd	ymm5, ymm5, ymm11		;; R5 + R6 (new R5)		; 72-74

	vmulpd	ymm11, ymm3, ymm10		;; R1 * MemR1			; 70-74
	vmulpd	ymm3, ymm3, ymm9		;; R1 * MemI1			; 71-75
	vmovsd	Q [srcreg+r8-16], xmm11		;; Save product of sum of FFT values (R1a)
	vmulpd	ymm9, ymm9, [srcreg+32]		;; (reload I1) * MemI1		; 72-76
	vmulpd	ymm10, ymm10, [srcreg+32]	;; (reload I1) * MemR1		; 73-77
	vmovsd	Q YMM_TMP1[8], xmm9		;; R1b a.k.a I1

	vsubpd	ymm11, ymm11, ymm9		;; R1*MemR1-I1*MemI1 (new R1)	; 77-79
	vaddpd	ymm3, ymm3, ymm10		;; R1*MemI1+I1*MemR1 (new I1)	; 78-80

	vmovapd	YMM_TMP7, ymm0				;; Save 2 registers
	vmovapd	YMM_TMP8, ymm12

	vmovapd	ymm0, [srcreg+r9+d1]			;; MemR2
	vmovapd	ymm9, [srcreg+r10+d1]			;; MemR2#2
	vmulpd	ymm12, ymm0, ymm9			;; MemR2*MemR2#2
	vaddpd	ymm4, ymm4, ymm12			;; R2 = R2 + MemR2*MemR2#2
	vmovapd	ymm10, [srcreg+r10+d1+32]		;; MemI2#2
	vmulpd	ymm12, ymm0, ymm10			;; MemR2*MemI2#2
	vaddpd	ymm6, ymm6, ymm12			;; I2 = I2 + MemR2*MemI2#2
	vmovapd	ymm0, [srcreg+r9+d1+32]			;; MemI2
	vmulpd	ymm12, ymm0, ymm10			;; MemI2*MemI2#2
	vsubpd	ymm4, ymm4, ymm12			;; R2 = R2 - MemI2*MemI2#2
	vmulpd	ymm12, ymm0, ymm9			;; MemI2*MemR2#2
	vaddpd	ymm6, ymm6, ymm12			;; I2 = I2 + MemI2*MemR2#2

	vmovapd	ymm0, [srcreg+r9]			;; MemR1
	vmovapd	ymm9, [srcreg+r10]			;; MemR1#2
	vmulpd	ymm12, ymm0, ymm9			;; MemR1*MemR1#2
	vaddpd	ymm11, ymm11, ymm12			;; R1 = R1 + MemR1*MemR1#2
	vmovsd	xmm10, [srcreg+r8-16] 		;; Reload R1a = product of sum of FFT values
	vaddsd	xmm10, xmm10, xmm12			;; R1a = R1a + MemR1*MemR1#2
	vmovsd	Q [srcreg+r8-16], xmm10		;; Save product of sum of FFT values
	vmovapd	ymm10, [srcreg+r10+32]			;; MemI1#2
	vmulpd	ymm12, ymm0, ymm10			;; MemR1*MemI1#2
	vaddpd	ymm3, ymm3, ymm12			;; I1 = I1 + MemR1*MemI1#2
	vmovapd	ymm0, [srcreg+r9+32]			;; MemI1
	vmulpd	ymm12, ymm0, ymm10			;; MemI1*MemI1#2
	vmovsd	xmm10, Q YMM_TMP1[8] 		;; R1b
	vaddpd	xmm10, xmm10, xmm12			;; R1b = R1b + MemR1b in MemI1*MemI1#2
	vmovsd	Q YMM_TMP1[8], xmm10		;; R1b
	vsubpd	ymm11, ymm11, ymm12			;; R1 = R1 - MemI1*MemI1#2
	vmulpd	ymm12, ymm0, ymm9			;; MemI1*MemR1#2
	vaddpd	ymm3, ymm3, ymm12			;; I1 = I1 + MemI1*MemR1#2

	vmovapd	ymm0, YMM_TMP7				;; Restore 2 registers
	vmovapd	ymm12, YMM_TMP8
	jmp	back_to_orig

mulsub:	vmovapd	ymm3, [srcreg+d2+d1][rbp]	;; MemR4
	vmovapd	ymm9, [srcreg+d2+d1+32][rbp]	;; MemI4
	vmulpd	ymm4, ymm7, ymm3		;; R4 * MemR4			; 32-36
	vmulpd	ymm7, ymm7, ymm9		;; R4 * MemI4			; 33-37
	vmulpd	ymm9, ymm10, ymm9		;; I4 * MemI4			; 34-38
	vmulpd	ymm10, ymm10, ymm3		;; I4 * MemR4			; 35-39

	vmovapd	ymm3, [srcreg+d2][rbp]		;; MemR3
	vsubpd	ymm4, ymm4, ymm9		;; R4*MemR4-I4*MemI4 (new R4)	; 51-53
	vmovapd	ymm9, [srcreg+d2+32][rbp]	;; MemI3
	vaddpd	ymm7, ymm7, ymm10		;; R4*MemI4+I4*MemR4 (new I4)	; 52-54
	vmulpd	ymm10, ymm2, ymm3		;; R3 * MemR3			; 36-40
	vmulpd	ymm2, ymm2, ymm9		;; R3 * MemI3			; 37-41
	vmulpd	ymm9, ymm11, ymm9		;; I3 * MemI3			; 38-42
	vmulpd	ymm11, ymm11, ymm3		;; I3 * MemR3			; 39-43

	vmovapd	ymm3, [srcreg+d4+d1][rbp]	;; MemR6
	vsubpd	ymm10, ymm10, ymm9		;; R3*MemR3-I3*MemI3 (new R3)	; 53-55
	vmovapd	ymm9, [srcreg+d4+d1+32][rbp]	;; MemI6
	vaddpd	ymm2, ymm2, ymm11		;; R3*MemI3+I3*MemR3 (new I3)	; 54-56
	vmulpd	ymm11, ymm1, ymm3		;; R6 * MemR6			; 42-46
	vmulpd	ymm1, ymm1, ymm9		;; R6 * MemI6			; 43-47
	vmulpd	ymm9, ymm5, ymm9		;; I6 * MemI6			; 46-50
	vmulpd	ymm5, ymm5, ymm3		;; I6 * MemR6			; 47-51

	vmovapd	ymm3, [srcreg+d4][rbp]		;; MemR5
	vsubpd	ymm11, ymm11, ymm9		;; R6*MemR6-I6*MemI6 (new R6)	; 55-57
	vmovapd	ymm9, [srcreg+d4+32][rbp]	;; MemI5
	vaddpd	ymm1, ymm1, ymm5		;; R6*MemI6+I6*MemR6 (new I6)	; 56-58
	vmulpd	ymm5, ymm12, ymm3		;; R5 * MemR5			; 44-48
	vmulpd	ymm12, ymm12, ymm9		;; R5 * MemI5			; 45-49
	vmulpd	ymm9, ymm15, ymm9		;; I5 * MemI5			; 48-52
	vmulpd	ymm15, ymm15, ymm3		;; I5 * MemR5			; 49-53

	vmovapd	ymm3, [srcreg+d4+d2+d1][rbp]	;; MemR8
	vsubpd	ymm5, ymm5, ymm9		;; R5*MemR5-I5*MemI5 (new R5)	; 57-59
	vmovapd	ymm9, [srcreg+d4+d2+d1+32][rbp]	;; MemI8
	vaddpd	ymm12, ymm12, ymm15		;; R5*MemI5+I5*MemR5 (new I5)	; 58-60
	vmulpd	ymm15, ymm14, ymm3		;; R8 * MemR8			; 50-54
	vmulpd	ymm14, ymm14, ymm9		;; R8 * MemI8			; 51-55
	vmulpd	ymm9, ymm8, ymm9		;; I8 * MemI8			; 52-56
	vmulpd	ymm8, ymm8, ymm3		;; I8 * MemR8			; 53-57

	vmovapd	ymm3, [srcreg+d4+d2][rbp]	;; MemR7
	vsubpd	ymm15, ymm15, ymm9		;; R8*MemR8-I8*MemI8 (new R8)	; 59-61
	vmovapd	ymm9, [srcreg+d4+d2+32][rbp]	;; MemI7
	vaddpd	ymm14, ymm14, ymm8		;; R8*MemI8+I8*MemR8 (new I8)	; 60-62
	vmulpd	ymm8, ymm0, ymm3		;; R7 * MemR7			; 54-58
	vmulpd	ymm0, ymm0, ymm9		;; R7 * MemI7			; 55-59
	vmulpd	ymm9, ymm6, ymm9		;; I7 * MemI7			; 56-60
	vmulpd	ymm6, ymm6, ymm3		;; I7 * MemR7			; 57-61

	vsubpd	ymm8, ymm8, ymm9		;; R7*MemR7-I7*MemI7 (new R7)	; 61-63
	vaddpd	ymm0, ymm0, ymm6		;; R7*MemI7+I7*MemR7 (new I7)	; 62-64

	vmovapd	ymm3, [srcreg]			;; Reload R1
	vmovapd	ymm6, [srcreg+d1]		;; Reload R2

	mov	r10, SRC3ARG			;; Load distance to FFT(1)
	test	r10, 3				;; Check if we need to multiply add-in by FFT(1)
	jz	mulsubhard			;; Yes, do it the hard way

	vsubpd	ymm4, ymm4, [srcreg+r9+d2+d1]		;; R4 = R4 - MemR4
	vsubpd	ymm7, ymm7, [srcreg+r9+d2+d1+32]	;; I4 = I4 - MemI4
	vsubpd	ymm10, ymm10, [srcreg+r9+d2]		;; R3 = R3 - MemR3
	vsubpd	ymm2, ymm2, [srcreg+r9+d2+32]		;; I3 = I3 - MemI3
	vsubpd	ymm11, ymm11, [srcreg+r9+d4+d1]		;; R6 = R6 - MemR6
	vsubpd	ymm1, ymm1, [srcreg+r9+d4+d1+32]	;; I6 = I6 - MemI6
	vsubpd	ymm5, ymm5, [srcreg+r9+d4]		;; R5 = R5 - MemR5
	vsubpd	ymm12, ymm12, [srcreg+r9+d4+32]		;; I5 = I5 - MemI5
	vsubpd	ymm15, ymm15, [srcreg+r9+d4+d2+d1]	;; R8 = R8 - MemR8
	vsubpd	ymm14, ymm14, [srcreg+r9+d4+d2+d1+32]	;; I8 = I8 - MemI8
	vsubpd	ymm8, ymm8, [srcreg+r9+d4+d2]		;; R7 = R7 - MemR7
	vsubpd	ymm0, ymm0, [srcreg+r9+d4+d2+32]	;; I7 = I7 - MemI7

	vaddpd	ymm9, ymm4, ymm10		;; R4 + R3 (new R3)		; 65-67
	vsubpd	ymm4, ymm4, ymm10		;; R4 - R3 (new I4)		; 66-68

	vmovapd	ymm10, [srcreg+d1][rbp]		;; MemR2
	ystore	YMM_TMP5, ymm9			;; Temporarily save new R3	; 68
	vmovapd	ymm9, [srcreg+d1+32][rbp]	;; MemI2
	ystore	YMM_TMP6, ymm4			;; Temporarily save new I4	; 69
	vmulpd	ymm4, ymm6, ymm10		;; R2 * MemR2			; 66-70
	vmulpd	ymm6, ymm6, ymm9		;; R2 * MemI2			; 67-71
	vmulpd	ymm9, ymm13, ymm9		;; I2 * MemI2			; 68-72
	vmulpd	ymm13, ymm13, ymm10		;; I2 * MemR2			; 69-73

	vmovapd	ymm10, [srcreg][rbp]		;; MemR1
	vsubpd	ymm4, ymm4, ymm9		;; R2*MemR2-I2*MemI2 (new R2)	; 73-75
	vmovapd	ymm9, [srcreg+32][rbp]		;; MemI1
	vaddpd	ymm6, ymm6, ymm13		;; R2*MemI2+I2*MemR2 (new I2)	; 74-76

	vsubpd	ymm13, ymm2, ymm7		;; I3 - I4 (new R4)		; 67-69
	vaddpd	ymm2, ymm2, ymm7		;; I3 + I4 (new I3)		; 68-70

	vsubpd	ymm7, ymm12, ymm1		;; I5 - I6 (new I6)		; 69-71
	vaddpd	ymm12, ymm12, ymm1		;; I5 + I6 (new I5)		; 70-72

	vsubpd	ymm1, ymm5, ymm11		;; R5 - R6 (new R6)		; 71-73
	vaddpd	ymm5, ymm5, ymm11		;; R5 + R6 (new R5)		; 72-74

	vmulpd	ymm11, ymm3, ymm10		;; R1 * MemR1 (new R1a)		; 70-74
	vmulpd	ymm3, ymm3, ymm9		;; R1 * MemI1			; 71-75
	vmulpd	ymm9, ymm9, [srcreg+32]		;; (reload I1) * MemI1 (new R1b); 72-76
	vmulpd	ymm10, ymm10, [srcreg+32]	;; (reload I1) * MemR1		; 73-77

	vaddpd	ymm3, ymm3, ymm10		;; R1*MemI1+I1*MemR1 (new I1)	; 78-80
	vsubpd	ymm10, ymm11, ymm9		;; R1*MemR1-I1*MemI1 (new R1)	; 77-79

	vsubpd	ymm4, ymm4, [srcreg+r9+d1]		;; R2 = R2 - MemR2
	vsubpd	ymm6, ymm6, [srcreg+r9+d1+32]		;; I2 = I2 - MemI2
	vsubsd	xmm11, xmm11, [srcreg+r9]		;; R1a = R1a - MemR1a
	vmovsd	Q [srcreg+r8-16], xmm11		;; Save product of sum of FFT values
	vsubpd	ymm11, ymm10, [srcreg+r9]		;; R1 = R1 - MemR1 (lower word is irrelevant)
	vmovapd	ymm10, [srcreg+r9+32]			;; MemI1
	vsubsd	xmm9, xmm9, xmm10			;; R1b = R1b - MemR1b in MemI1
	vsubpd	ymm3, ymm3, ymm10			;; I1 = I1 - MemI1 (lower word is irrelevant)
	vmovsd	Q YMM_TMP1[8], xmm9			;; Save R1b
	jmp	back_to_orig

mulsubhard:
	vmovapd	YMM_TMP5, ymm3				;; Save 3 registers
	vmovapd	YMM_TMP6, ymm6
	vmovapd	YMM_TMP7, ymm13

	vmovapd	ymm3, [srcreg+r9+d2+d1]			;; MemR4
	vmovapd	ymm9, [srcreg+r10+d2+d1]		;; MemR4#2
	vmulpd	ymm13, ymm3, ymm9			;; MemR4*MemR4#2
	vsubpd	ymm4, ymm4, ymm13			;; R4 = R4 - MemR4*MemR4#2
	vmovapd	ymm6, [srcreg+r10+d2+d1+32]		;; MemI4#2
	vmulpd	ymm13, ymm3, ymm6			;; MemR4*MemI4#2
	vsubpd	ymm7, ymm7, ymm13			;; I4 = I4 - MemR4*MemI4#2
	vmovapd	ymm3, [srcreg+r9+d2+d1+32]		;; MemI4
	vmulpd	ymm13, ymm3, ymm6			;; MemI4*MemI4#2
	vaddpd	ymm4, ymm4, ymm13			;; R4 = R4 + MemI4*MemI4#2
	vmulpd	ymm13, ymm3, ymm9			;; MemI4*MemR4#2
	vsubpd	ymm7, ymm7, ymm13			;; I4 = I4 - MemI4*MemR4#2

	vmovapd	ymm3, [srcreg+r9+d2]			;; MemR3
	vmovapd	ymm9, [srcreg+r10+d2]			;; MemR3#2
	vmulpd	ymm13, ymm3, ymm9			;; MemR3*MemR3#2
	vsubpd	ymm10, ymm10, ymm13			;; R3 = R3 - MemR3*MemR3#2
	vmovapd	ymm6, [srcreg+r10+d2+32]		;; MemI3#2
	vmulpd	ymm13, ymm3, ymm6			;; MemR3*MemI3#2
	vsubpd	ymm2, ymm2, ymm13			;; I3 = I3 - MemR3*MemI3#2
	vmovapd	ymm3, [srcreg+r9+d2+32]			;; MemI3
	vmulpd	ymm13, ymm3, ymm6			;; MemI3*MemI3#2
	vaddpd	ymm10, ymm10, ymm13			;; R3 = R3 + MemI3*MemI3#2
	vmulpd	ymm13, ymm3, ymm9			;; MemI3*MemR3#2
	vsubpd	ymm2, ymm2, ymm13			;; I3 = I3 - MemI3*MemR3#2

	vmovapd	ymm3, [srcreg+r9+d4+d1]			;; MemR6
	vmovapd	ymm9, [srcreg+r10+d4+d1]		;; MemR6#2
	vmulpd	ymm13, ymm3, ymm9			;; MemR6*MemR6#2
	vsubpd	ymm11, ymm11, ymm13			;; R6 = R6 - MemR6*MemR6#2
	vmovapd	ymm6, [srcreg+r10+d4+d1+32]		;; MemI6#2
	vmulpd	ymm13, ymm3, ymm6			;; MemR6*MemI6#2
	vsubpd	ymm1, ymm1, ymm13			;; I6 = I6 - MemR6*MemI6#2
	vmovapd	ymm3, [srcreg+r9+d4+d1+32]		;; MemI6
	vmulpd	ymm13, ymm3, ymm6			;; MemI6*MemI6#2
	vaddpd	ymm11, ymm11, ymm13			;; R6 = R6 + MemI6*MemI6#2
	vmulpd	ymm13, ymm3, ymm9			;; MemI6*MemR6#2
	vsubpd	ymm1, ymm1, ymm13			;; I6 = I6 - MemI6*MemR6#2

	vmovapd	ymm3, [srcreg+r9+d4]			;; MemR5
	vmovapd	ymm9, [srcreg+r10+d4]			;; MemR5#2
	vmulpd	ymm13, ymm3, ymm9			;; MemR5*MemR5#2
	vsubpd	ymm5, ymm5, ymm13			;; R5 = R5 - MemR5*MemR5#2
	vmovapd	ymm6, [srcreg+r10+d4+32]		;; MemI5#2
	vmulpd	ymm13, ymm3, ymm6			;; MemR5*MemI5#2
	vsubpd	ymm12, ymm12, ymm13			;; I5 = I5 - MemR5*MemI5#2
	vmovapd	ymm3, [srcreg+r9+d4+32]			;; MemI5
	vmulpd	ymm13, ymm3, ymm6			;; MemI5*MemI5#2
	vaddpd	ymm5, ymm5, ymm13			;; R5 = R5 + MemI5*MemI5#2
	vmulpd	ymm13, ymm3, ymm9			;; MemI5*MemR5#2
	vsubpd	ymm12, ymm12, ymm13			;; I5 = I5 - MemI5*MemR5#2

	vmovapd	ymm3, [srcreg+r9+d4+d2+d1]		;; MemR8
	vmovapd	ymm9, [srcreg+r10+d4+d2+d1]		;; MemR8#2
	vmulpd	ymm13, ymm3, ymm9			;; MemR8*MemR8#2
	vsubpd	ymm15, ymm15, ymm13			;; R8 = R8 - MemR8*MemR8#2
	vmovapd	ymm6, [srcreg+r10+d4+d2+d1+32]		;; MemI8#2
	vmulpd	ymm13, ymm3, ymm6			;; MemR8*MemI8#2
	vsubpd	ymm14, ymm14, ymm13			;; I8 = I8 - MemR8*MemI8#2
	vmovapd	ymm3, [srcreg+r9+d4+d2+d1+32]		;; MemI8
	vmulpd	ymm13, ymm3, ymm6			;; MemI8*MemI8#2
	vaddpd	ymm15, ymm15, ymm13			;; R8 = R8 + MemI8*MemI8#2
	vmulpd	ymm13, ymm3, ymm9			;; MemI8*MemR8#2
	vsubpd	ymm14, ymm14, ymm13			;; I8 = I8 - MemI8*MemR8#2

	vmovapd	ymm3, [srcreg+r9+d4+d2]			;; MemR7
	vmovapd	ymm9, [srcreg+r10+d4+d2]		;; MemR7#2
	vmulpd	ymm13, ymm3, ymm9			;; MemR7*MemR7#2
	vsubpd	ymm8, ymm8, ymm13			;; R7 = R7 - MemR7*MemR7#2
	vmovapd	ymm6, [srcreg+r10+d4+d2+32]		;; MemI7#2
	vmulpd	ymm13, ymm3, ymm6			;; MemR7*MemI7#2
	vsubpd	ymm0, ymm0, ymm13			;; I7 = I7 - MemR7*MemI7#2
	vmovapd	ymm3, [srcreg+r9+d4+d2+32]		;; MemI7
	vmulpd	ymm13, ymm3, ymm6			;; MemI7*MemI7#2
	vaddpd	ymm8, ymm8, ymm13			;; R7 = R7 + MemI7*MemI7#2
	vmulpd	ymm13, ymm3, ymm9			;; MemI7*MemR7#2
	vsubpd	ymm0, ymm0, ymm13			;; I7 = I7 - MemI7*MemR7#2

	vmovapd	ymm3, YMM_TMP5
	vmovapd	ymm6, YMM_TMP6
	vmovapd	ymm13, YMM_TMP7

	vaddpd	ymm9, ymm4, ymm10		;; R4 + R3 (new R3)		; 65-67
	vsubpd	ymm4, ymm4, ymm10		;; R4 - R3 (new I4)		; 66-68

	vmovapd	ymm10, [srcreg+d1][rbp]		;; MemR2
	ystore	YMM_TMP5, ymm9			;; Temporarily save new R3	; 68
	vmovapd	ymm9, [srcreg+d1+32][rbp]	;; MemI2
	ystore	YMM_TMP6, ymm4			;; Temporarily save new I4	; 69

	vmulpd	ymm4, ymm6, ymm10		;; R2 * MemR2			; 66-70
	vmulpd	ymm6, ymm6, ymm9		;; R2 * MemI2			; 67-71
	vmulpd	ymm9, ymm13, ymm9		;; I2 * MemI2			; 68-72
	vmulpd	ymm13, ymm13, ymm10		;; I2 * MemR2			; 69-73
	vmovapd	ymm10, [srcreg][rbp]		;; MemR1
	vsubpd	ymm4, ymm4, ymm9		;; R2*MemR2-I2*MemI2 (new R2)	; 73-75
	vmovapd	ymm9, [srcreg+32][rbp]		;; MemI1
	vaddpd	ymm6, ymm6, ymm13		;; R2*MemI2+I2*MemR2 (new I2)	; 74-76

	vsubpd	ymm13, ymm2, ymm7		;; I3 - I4 (new R4)		; 67-69
	vaddpd	ymm2, ymm2, ymm7		;; I3 + I4 (new I3)		; 68-70

	vsubpd	ymm7, ymm12, ymm1		;; I5 - I6 (new I6)		; 69-71
	vaddpd	ymm12, ymm12, ymm1		;; I5 + I6 (new I5)		; 70-72

	vsubpd	ymm1, ymm5, ymm11		;; R5 - R6 (new R6)		; 71-73
	vaddpd	ymm5, ymm5, ymm11		;; R5 + R6 (new R5)		; 72-74

	vmulpd	ymm11, ymm3, ymm10		;; R1 * MemR1			; 70-74
	vmulpd	ymm3, ymm3, ymm9		;; R1 * MemI1			; 71-75
	vmovsd	Q [srcreg+r8-16], xmm11		;; Save product of sum of FFT values
	vmulpd	ymm9, ymm9, [srcreg+32]		;; (reload I1) * MemI1		; 72-76
	vmulpd	ymm10, ymm10, [srcreg+32]	;; (reload I1) * MemR1		; 73-77
	vmovsd	Q YMM_TMP1[8], xmm9		;; I1

	vsubpd	ymm11, ymm11, ymm9		;; R1*MemR1-I1*MemI1 (new R1)	; 77-79
	vaddpd	ymm3, ymm3, ymm10		;; R1*MemI1+I1*MemR1 (new I1)	; 78-80

	vmovapd	YMM_TMP7, ymm0				;; Save 2 registers
	vmovapd	YMM_TMP8, ymm12

	vmovapd	ymm0, [srcreg+r9+d1]			;; MemR2
	vmovapd	ymm9, [srcreg+r10+d1]			;; MemR2#2
	vmulpd	ymm12, ymm0, ymm9			;; MemR2*MemR2#2
	vsubpd	ymm4, ymm4, ymm12			;; R2 = R2 - MemR2*MemR2#2
	vmovapd	ymm10, [srcreg+r10+d1+32]		;; MemI2#2
	vmulpd	ymm12, ymm0, ymm10			;; MemR2*MemI2#2
	vsubpd	ymm6, ymm6, ymm12			;; I2 = I2 - MemR2*MemI2#2
	vmovapd	ymm0, [srcreg+r9+d1+32]			;; MemI2
	vmulpd	ymm12, ymm0, ymm10			;; MemI2*MemI2#2
	vaddpd	ymm4, ymm4, ymm12			;; R2 = R2 + MemI2*MemI2#2
	vmulpd	ymm12, ymm0, ymm9			;; MemI2*MemR2#2
	vsubpd	ymm6, ymm6, ymm12			;; I2 = I2 - MemI2*MemR2#2

	vmovapd	ymm0, [srcreg+r9]			;; MemR1
	vmovapd	ymm9, [srcreg+r10]			;; MemR1#2
	vmulpd	ymm12, ymm0, ymm9			;; MemR1*MemR1#2
	vsubpd	ymm11, ymm11, ymm12			;; R1 = R1 - MemR1*MemR1#2
	vmovsd	xmm10, [srcreg+r8-16] 		;; Reload R1a = product of sum of FFT values
	vsubsd	xmm10, xmm10, xmm12			;; R1a = R1a - MemR1*MemR1#2
	vmovsd	Q [srcreg+r8-16], xmm10		;; Save product of sum of FFT values
	vmovapd	ymm10, [srcreg+r10+32]			;; MemI1#2
	vmulpd	ymm12, ymm0, ymm10			;; MemR1*MemI1#2
	vsubpd	ymm3, ymm3, ymm12			;; I1 = I1 - MemR1*MemI1#2
	vmovapd	ymm0, [srcreg+r9+32]			;; MemI1
	vmulpd	ymm12, ymm0, ymm10			;; MemI1*MemI1#2
	vmovsd	xmm10, Q YMM_TMP1[8] 		;; R1b
	vsubpd	xmm10, xmm10, xmm12			;; R1b = R1b - MemR1b in MemI1*MemI1#2
	vmovsd	Q YMM_TMP1[8], xmm10		;; R1b
	vaddpd	ymm11, ymm11, ymm12			;; R1 = R1 + MemI1*MemI1#2
	vmulpd	ymm12, ymm0, ymm9			;; MemI1*MemR1#2
	vsubpd	ymm3, ymm3, ymm12			;; I1 = I1 - MemI1*MemR1#2

	vmovapd	ymm0, YMM_TMP7				;; Restore 2 registers
	vmovapd	ymm12, YMM_TMP8
	jmp	back_to_orig

orig:	vmovapd	ymm3, [srcreg+d2+d1][rbp]	;; MemR4
	vmovapd	ymm9, [srcreg+d2+d1+32][rbp]	;; MemI4
	vmulpd	ymm4, ymm7, ymm3		;; R4 * MemR4			; 32-36
	vmulpd	ymm7, ymm7, ymm9		;; R4 * MemI4			; 33-37
	vmulpd	ymm9, ymm10, ymm9		;; I4 * MemI4			; 34-38
	vmulpd	ymm10, ymm10, ymm3		;; I4 * MemR4			; 35-39

	vmovapd	ymm3, [srcreg+d2][rbp]		;; MemR3
	vsubpd	ymm4, ymm4, ymm9		;; R4*MemR4-I4*MemI4 (new R4)	; 51-53
	vmovapd	ymm9, [srcreg+d2+32][rbp]	;; MemI3
	vaddpd	ymm7, ymm7, ymm10		;; R4*MemI4+I4*MemR4 (new I4)	; 52-54
	vmulpd	ymm10, ymm2, ymm3		;; R3 * MemR3			; 36-40
	vmulpd	ymm2, ymm2, ymm9		;; R3 * MemI3			; 37-41
	vmulpd	ymm9, ymm11, ymm9		;; I3 * MemI3			; 38-42
	vmulpd	ymm11, ymm11, ymm3		;; I3 * MemR3			; 39-43

	vmovapd	ymm3, [srcreg+d4+d1][rbp]	;; MemR6
	vsubpd	ymm10, ymm10, ymm9		;; R3*MemR3-I3*MemI3 (new R3)	; 53-55
	vmovapd	ymm9, [srcreg+d4+d1+32][rbp]	;; MemI6
	vaddpd	ymm2, ymm2, ymm11		;; R3*MemI3+I3*MemR3 (new I3)	; 54-56
	vmulpd	ymm11, ymm1, ymm3		;; R6 * MemR6			; 42-46
	vmulpd	ymm1, ymm1, ymm9		;; R6 * MemI6			; 43-47
	vmulpd	ymm9, ymm5, ymm9		;; I6 * MemI6			; 46-50
	vmulpd	ymm5, ymm5, ymm3		;; I6 * MemR6			; 47-51

	vmovapd	ymm3, [srcreg+d4][rbp]		;; MemR5
	vsubpd	ymm11, ymm11, ymm9		;; R6*MemR6-I6*MemI6 (new R6)	; 55-57
	vmovapd	ymm9, [srcreg+d4+32][rbp]	;; MemI5
	vaddpd	ymm1, ymm1, ymm5		;; R6*MemI6+I6*MemR6 (new I6)	; 56-58
	vmulpd	ymm5, ymm12, ymm3		;; R5 * MemR5			; 44-48
	vmulpd	ymm12, ymm12, ymm9		;; R5 * MemI5			; 45-49
	vmulpd	ymm9, ymm15, ymm9		;; I5 * MemI5			; 48-52
	vmulpd	ymm15, ymm15, ymm3		;; I5 * MemR5			; 49-53

	vmovapd	ymm3, [srcreg+d4+d2+d1][rbp]	;; MemR8
	vsubpd	ymm5, ymm5, ymm9		;; R5*MemR5-I5*MemI5 (new R5)	; 57-59
	vmovapd	ymm9, [srcreg+d4+d2+d1+32][rbp]	;; MemI8
	vaddpd	ymm12, ymm12, ymm15		;; R5*MemI5+I5*MemR5 (new I5)	; 58-60
	vmulpd	ymm15, ymm14, ymm3		;; R8 * MemR8			; 50-54
	vmulpd	ymm14, ymm14, ymm9		;; R8 * MemI8			; 51-55
	vmulpd	ymm9, ymm8, ymm9		;; I8 * MemI8			; 52-56
	vmulpd	ymm8, ymm8, ymm3		;; I8 * MemR8			; 53-57

	vmovapd	ymm3, [srcreg+d4+d2][rbp]	;; MemR7
	vsubpd	ymm15, ymm15, ymm9		;; R8*MemR8-I8*MemI8 (new R8)	; 59-61
	vmovapd	ymm9, [srcreg+d4+d2+32][rbp]	;; MemI7
	vaddpd	ymm14, ymm14, ymm8		;; R8*MemI8+I8*MemR8 (new I8)	; 60-62
	vmulpd	ymm8, ymm0, ymm3		;; R7 * MemR7			; 54-58
	vmulpd	ymm0, ymm0, ymm9		;; R7 * MemI7			; 55-59
	vmulpd	ymm9, ymm6, ymm9		;; I7 * MemI7			; 56-60
	vmulpd	ymm6, ymm6, ymm3		;; I7 * MemR7			; 57-61

	vsubpd	ymm8, ymm8, ymm9		;; R7*MemR7-I7*MemI7 (new R7)	; 61-63
	vaddpd	ymm0, ymm0, ymm6		;; R7*MemI7+I7*MemR7 (new I7)	; 62-64

	vaddpd	ymm9, ymm4, ymm10		;; R4 + R3 (new R3)		; 65-67
	vsubpd	ymm4, ymm4, ymm10		;; R4 - R3 (new I4)		; 66-68

	vmovapd	ymm6, [srcreg+d1]		;; Reload R2
	vmovapd	ymm10, [srcreg+d1][rbp]		;; MemR2
	ystore	YMM_TMP5, ymm9			;; Temporarily save new R3	; 68
	vmovapd	ymm9, [srcreg+d1+32][rbp]	;; MemI2
	ystore	YMM_TMP6, ymm4			;; Temporarily save new I4	; 69
	vmulpd	ymm4, ymm6, ymm10		;; R2 * MemR2			; 66-70
	vmulpd	ymm6, ymm6, ymm9		;; R2 * MemI2			; 67-71
	vmulpd	ymm9, ymm13, ymm9		;; I2 * MemI2			; 68-72
	vmulpd	ymm13, ymm13, ymm10		;; I2 * MemR2			; 69-73

	vmovapd	ymm3, [srcreg]			;; Reload R1
	vmovapd	ymm10, [srcreg][rbp]		;; MemR1
	vsubpd	ymm4, ymm4, ymm9		;; R2*MemR2-I2*MemI2 (new R2)	; 73-75
	vmovapd	ymm9, [srcreg+32][rbp]		;; MemI1
	vaddpd	ymm6, ymm6, ymm13		;; R2*MemI2+I2*MemR2 (new I2)	; 74-76

	vsubpd	ymm13, ymm2, ymm7		;; I3 - I4 (new R4)		; 67-69
	vaddpd	ymm2, ymm2, ymm7		;; I3 + I4 (new I3)		; 68-70

	vsubpd	ymm7, ymm12, ymm1		;; I5 - I6 (new I6)		; 69-71
	vaddpd	ymm12, ymm12, ymm1		;; I5 + I6 (new I5)		; 70-72

	vsubpd	ymm1, ymm5, ymm11		;; R5 - R6 (new R6)		; 71-73
	vaddpd	ymm5, ymm5, ymm11		;; R5 + R6 (new R5)		; 72-74

	vmulpd	ymm11, ymm3, ymm10		;; R1 * MemR1			; 70-74
	vmulpd	ymm3, ymm3, ymm9		;; R1 * MemI1			; 71-75
	vmovsd	Q [srcreg+r8-16], xmm11		;; Save product of sum of FFT values
	vmulpd	ymm9, ymm9, [srcreg+32]		;; (reload I1) * MemI1		; 72-76
	vmulpd	ymm10, ymm10, [srcreg+32]	;; (reload I1) * MemR1		; 73-77
	vmovsd	Q YMM_TMP1[8], xmm9		;; I1

	vsubpd	ymm11, ymm11, ymm9		;; R1*MemR1-I1*MemI1 (new R1)	; 77-79
	vaddpd	ymm3, ymm3, ymm10		;; R1*MemI1+I1*MemR1 (new I1)	; 78-80

back_to_orig:
	vsubpd	ymm9, ymm0, ymm14		;; I7 - I8 (new R8)		; 75-77
	vaddpd	ymm0, ymm0, ymm14		;; I7 + I8 (new I7)		; 76-78
	vmovsd	Q YMM_TMP4[8], xmm0		;; I7+I8
	vmovsd	Q YMM_TMP4[24], xmm9		;; I7-I8

	vsubpd	ymm14, ymm15, ymm8		;; R8 - R7 (new I8)		; 79-81
	vaddpd	ymm15, ymm15, ymm8		;; R8 + R7 (new R7)		; 80-82
	vmovsd	Q YMM_TMP4[0], xmm15		;; R8+R7
	vmovsd	Q YMM_TMP4[16], xmm14		;; R8-R7

	vmovsd	Q YMM_TMP1[24], xmm6		;; I2
	vsubpd	ymm8, ymm3, ymm6		;; I1 - I2 (new I2)		; 81-83
	vaddpd	ymm3, ymm3, ymm6		;; I1 + I2 (new I1)		; 82-84

	vmovsd	Q YMM_TMP1[16], xmm4		;; R2
	vsubpd	ymm6, ymm11, ymm4		;; R1 - R2 (new R2)		; 83-85
	vaddpd	ymm11, ymm11, ymm4		;; R1 + R2 (new R1)		; 84-86

	;; multiply R6/I6 by SQRTHALF - i*SQRTHALF
	vmovsd	Q YMM_TMP3[24], xmm7		;; I5-I6
	vmovsd	Q YMM_TMP3[16], xmm1		;; R5-R6
	vsubpd	ymm4, ymm7, ymm1		;; I6 = I6 - R6			; 85-87
	vaddpd	ymm1, ymm1, ymm7		;; R6 = R6 + I6			; 86-88

	;; multiply R8/I8 by SQRTHALF - i*SQRTHALF
	vsubpd	ymm7, ymm14, ymm9		;; I8 = I8 - R8			; 87-89
	vaddpd	ymm9, ymm9, ymm14		;; R8 = R8 + I8			; 88-90

	vmovapd	ymm14, YMM_SQRTHALF
	vmulpd	ymm4, ymm4, ymm14		;; I6 * SQRTHALF		; 88-92
	vmulpd	ymm1, ymm1, ymm14		;; R6 * SQRTHALF		; 89-93
	vmulpd	ymm7, ymm7, ymm14		;; I8 * SQRTHALF		; 90-94
	vmulpd	ymm9, ymm9, ymm14		;; R8 * SQRTHALF		; 91-95

	vmovsd	Q YMM_TMP2[24], xmm13		;; I3-I4
	vaddpd	ymm14, ymm6, ymm13		;; R2 + R4 (newer R2)		; 89-91
	vsubpd	ymm6, ymm6, ymm13		;; R2 - R4 (newer R4)		; 90-92

	vmovapd	ymm10, YMM_TMP6			;; Reload new I4
	vmovsd	Q YMM_TMP2[0], xmm9		;; R3+R4
	vmovsd	Q YMM_TMP2[16], xmm4		;; R4-R3

	vaddpd	ymm13, ymm8, ymm10		;; I2 + I4 (newer I2)		; 91-93
	vsubpd	ymm8, ymm8, ymm10		;; I2 - I4 (newer I4)		; 92-94

	vmovsd	Q YMM_TMP3[8], xmm12		;; I5+I6
	vsubpd	ymm10, ymm12, ymm0		;; I5 - I7 (newer R7)		; 93-95
	vaddpd	ymm12, ymm12, ymm0		;; I5 + I7 (newer I5)		; 94-96

	vmovsd	Q YMM_TMP2[8], xmm2		;; I3+I4
	vsubpd	ymm0, ymm3, ymm2		;; I1 - I3 (newer I3)		; 95-97
	vaddpd	ymm3, ymm3, ymm2		;; I1 + I3 (newer I1)		; 96-98

	vsubpd	ymm2, ymm4, ymm7		;; I6 - I8 (newer R8)		; 97-99
	vaddpd	ymm4, ymm4, ymm7		;; I6 + I8 (newer I6)		; 98-100

	vsubpd	ymm7, ymm9, ymm1		;; R8 - R6 (newer I8)		; 99-101
	vaddpd	ymm9, ymm9, ymm1		;; R8 + R6 (newer R6)		; 100-102

	vsubpd	ymm1, ymm6, ymm2		;; R4 - R8 (final R8)		; 101-103
	vaddpd	ymm6, ymm6, ymm2		;; R4 + R8 (final R4)		; 102-104

	vsubpd	ymm2, ymm13, ymm4		;; I2 - I6 (final I6)		; 103-105
	vaddpd	ymm13, ymm13, ymm4		;; I2 + I6 (final I2)		; 104-106
	ystore	[srcreg+r8+d4+d2+d1], ymm1	;; Save R8			; 104

	vsubpd	ymm4, ymm8, ymm7		;; I4 - I8 (final I8)		; 105-107
	vaddpd	ymm8, ymm8, ymm7		;; I4 + I8 (final I4)		; 106-108
	ystore	[srcreg+r8+d2+d1], ymm6		;; Save R4			; 105
	ystore	[srcreg+r8+d4+d1+32], ymm2	;; Save I6			; 106

	vsubpd	ymm7, ymm14, ymm9		;; R2 - R6 (final R6)		; 107-109
	vaddpd	ymm14, ymm14, ymm9		;; R2 + R6 (final R2)		; 108-110
	ystore	[srcreg+r8+d1+32], ymm13	;; Save I2			; 107
	ystore	[srcreg+r8+d4+d2+d1+32], ymm4	;; Save I8			; 108

	vmovapd	ymm1, YMM_TMP5			;; Reload new R3
	vsubpd	ymm9, ymm11, ymm1		;; R1 - R3 (newer R3)		; 109-111
	vaddpd	ymm11, ymm11, ymm1		;; R1 + R3 (newer R1)		; 110-112
	ystore	[srcreg+r8+d2+d1+32], ymm8	;; Save I4			; 109
	ystore	[srcreg+r8+d4+d1], ymm7		;; Save R6			; 110

	vmovsd	Q YMM_TMP3[0], xmm5		;; R5+R6
	vaddpd	ymm1, ymm15, ymm5		;; R7 + R5 (newer R5)		; 111-113
	vsubpd	ymm15, ymm15, ymm5		;; R7 - R5 (newer I7)		; 112-114
	ystore	[srcreg+r8+d1], ymm14		;; Save R2			; 111

	vsubpd	ymm5, ymm3, ymm12		;; I1 - I5 (final I5)		; 113-115
	vaddpd	ymm3, ymm3, ymm12		;; I1 + I5 (final I1)		; 114-116

	vsubpd	ymm12, ymm9, ymm10		;; R3 - R7 (final R7)		; 115-117
	vaddpd	ymm9, ymm9, ymm10		;; R3 + R7 (final R3)		; 116-118
	ystore	[srcreg+r8+d4+32], ymm5		;; Save I5			; 116

	vsubpd	ymm10, ymm0, ymm15		;; I3 - I7 (final I7)		; 117-119
	vaddpd	ymm0, ymm0, ymm15		;; I3 + I7 (final I3)		; 118-120
	ystore	[srcreg+r8+32], ymm3		;; Save I1			; 117
	ystore	[srcreg+r8+d4+d2], ymm12	;; Save R7			; 118

	vsubpd	ymm15, ymm11, ymm1		;; R1 - R5 (final R5)		; 119-121
	vaddpd	ymm11, ymm11, ymm1		;; R1 + R5 (final R1)		; 120-122
	ystore	[srcreg+r8+d2], ymm9		;; Save R3			; 119
	ystore	[srcreg+r8+d4+d2+32], ymm10	;; Save I7			; 120

	ystore	[srcreg+r8+d2+32], ymm0		;; Save I3			; 121
	ystore	[srcreg+r8+d4], ymm15		;; Save R5			; 122
	ystore	[srcreg+r8], ymm11		;; Save R1			; 123

	;; Do 16-reals from YMM_TMPs

	;; Level 4

	vmovsd	xmm0, [srcreg+r8-16]		;; R1
	vmovsd	xmm1, Q YMM_TMP1[8]		;; R2

	vsubsd	xmm0, xmm0, xmm1		;; R1 - R2 (new R2)
	vmulsd	xmm0, xmm0, Q YMM_HALF		;; Mul R2 by HALF
	vaddsd	xmm1, xmm1, xmm0		;; R1 + R2 (new R1)

						;; R3/I3 morphs into new R3/R4
	vmovsd	xmm9, Q YMM_TMP1[24]		;; I3 (new R4)

	vmovsd	xmm2, Q YMM_TMP6[0]		;; R6 - R5 (new negR6)
	vmovsd	xmm3, Q YMM_TMP2[24] 		;; I5 - I6 (new I6)

	vmovsd	xmm4, Q YMM_TMP3[16]		;; R9 - R10 (new R10)
	vmovsd	xmm5, Q YMM_TMP3[24]		;; I9 - I10 (new I10)

	vmovsd	xmm6, Q YMM_TMP4[24]		;; I11 - I12 (new R12)
	vmovsd	xmm7, Q YMM_TMP4[16]		;; R12 - R11 (new I12)

	;; Postmultipliers for level 4

						;; mul R6/I6 by w^2 = .707 - .707i
	vsubsd	xmm8, xmm3, xmm2		;; R6 = I6 - negR6
	vaddsd	xmm3, xmm3, xmm2		;; I6 = I6 + negR6
	vmovsd	xmm15, Q YMM_SQRTHALF
	vmulsd	xmm8, xmm8, xmm15		;; R6 = R6 * SQRTHALF
	vmulsd	xmm3, xmm3, xmm15		;; I6 = I6 * SQRTHALF

	;; Even level 3

	vaddsd	xmm2, xmm4, xmm6		;; R10 + R12 (newer R10)
	vsubsd	xmm4, xmm4, xmm6		;; R10 - R12 (newer R12)

	vaddsd	xmm6, xmm5, xmm7		;; I10 + I12 (newer I10)
	vsubsd	xmm5, xmm5, xmm7		;; I10 - I12 (newer I12)

	vaddsd	xmm7, xmm0, xmm9		;; R2 + R4 (newer R2)
	vsubsd	xmm0, xmm0, xmm9		;; R2 - R4 (newer R4)

	;; Postmultipliers for even level 3

						;; mul R10/I10 by w^1 = .924 - .383i
	vmovsd	xmm14, Q YMM_P924
	vmovsd	xmm13, Q YMM_P383
	vmulsd	xmm9, xmm2, xmm14
	vmulsd	xmm10, xmm6, xmm13
	vaddsd	xmm9, xmm9, xmm10		;; Twiddled R10
	vmulsd	xmm2, xmm2, xmm13
	vmulsd	xmm6, xmm6, xmm14
	vsubsd	xmm2, xmm6, xmm2		;; Twiddled I10

						;; mul R12/I12 by w^3 = .383 - .924i
	vmulsd	xmm6, xmm4, xmm13
	vmulsd	xmm10, xmm5, xmm14
	vaddsd	xmm6, xmm6, xmm10		;; Twiddled R12
	vmulsd	xmm4, xmm4, xmm14
	vmulsd	xmm5, xmm5, xmm13
	vsubsd	xmm4, xmm5, xmm4		;; Twiddled I12

						;; R6/I6 morph into newer R6/R8

	;; Even level 2

	vaddsd	xmm5, xmm7, xmm8		;; R2 + R6 (newest R2)
	vsubsd	xmm7, xmm7, xmm8		;; R2 - R6 (newest R6)

	vaddsd	xmm8, xmm0, xmm3		;; R4 + R8 (newest R4)
	vsubsd	xmm0, xmm0, xmm3		;; R4 - R8 (newest R8)

						;; R10/I10 morphs into newest R10/R14
						;; R12/I12 morphs into newest R12/R16

	;; Even level 1

	vaddsd	xmm3, xmm5, xmm9		;; R2 + R10 (final R2)
	vsubsd	xmm5, xmm5, xmm9		;; R2 - R10 (final R10)

	vaddsd	xmm9, xmm7, xmm2		;; R6 + R14 (final R6)
	vsubsd	xmm7, xmm7, xmm2		;; R6 - R14 (final R14)

	vaddsd	xmm2, xmm8, xmm6		;; R4 + R12 (final R4)
	vsubsd	xmm8, xmm8, xmm6		;; R4 - R12 (final R12)

	vaddsd	xmm6, xmm0, xmm4		;; R8 + R16 (final R8)
	vsubsd	xmm0, xmm0, xmm4		;; R8 - R16 (final R16)

	vmovsd	Q [srcreg+r8+d1], xmm3		;; R2
	vmovsd	Q [srcreg+r8+d1+32], xmm5	;; R10
	vmovsd	Q [srcreg+r8+d4+d1], xmm9	;; R6
	vmovsd	Q [srcreg+r8+d4+d1+32], xmm7	;; R14
	vmovsd	Q [srcreg+r8+d2+d1], xmm2	;; R4
	vmovsd	Q [srcreg+r8+d2+d1+32], xmm8	;; R12
	vmovsd	Q [srcreg+r8+d4+d2+d1], xmm6	;; R8
	vmovsd	Q [srcreg+r8+d4+d2+d1+32], xmm0	;; R16

	;; Odd level 3

	vmovsd	xmm4, Q YMM_TMP3[0] 		;; R9 + R10 (new R9)
	vmovsd	xmm5, Q YMM_TMP3[8] 		;; I9 + I10 (new I9)

	vmovsd	xmm6, Q YMM_TMP4[0]		;; R12 + R11 (new R11)
	vmovsd	xmm7, Q YMM_TMP4[8]		;; I11 + I12 (new I11)

	vmovsd	xmm0, Q YMM_TMP1[16]		;; R3 (new R3)

	vaddsd	xmm8, xmm4, xmm6		;; R9 + R11 (newer R9)
	vsubsd	xmm4, xmm4, xmm6		;; R9 - R11 (newer R11)

	vaddsd	xmm6, xmm5, xmm7		;; I9 + I11 (newer I9)
	vsubsd	xmm5, xmm5, xmm7		;; I9 - I11 (newer I11)

	vaddsd	xmm7, xmm1, xmm0		;; R1 + R3 (newer R1)
	vsubsd	xmm1, xmm1, xmm0		;; R1 - R3 (newer R3)

	vmovsd	xmm2, Q YMM_TMP5[0]		;; R5 + R6 (new R5)
	vmovsd	xmm3, Q YMM_TMP2[8]		;; I5 + I6 (new I5)

						;; R5/I5 morphs into newer R5/R7


	;; Postmultipliers for odd level 3

						;; mul R11/I11 by SQRTHALF - i*SQRTHALF
	vaddsd	xmm0, xmm5, xmm4		;; R11 = I11 + R11
	vsubsd	xmm5, xmm5, xmm4		;; I11 = I11 - R11
	vmulsd	xmm0, xmm0, xmm15		;; R11 = R11 * SQRTHALF
	vmulsd	xmm5, xmm5, xmm15		;; I11 = I11 * SQRTHALF

	;; Odd level 2

	vaddsd	xmm4, xmm7, xmm2		;; R1 + R5 (newest R1)
	vsubsd	xmm7, xmm7, xmm2		;; R1 - R5 (newest R5)

	vaddsd	xmm2, xmm1, xmm3		;; R3 + R7 (newest R3)
	vsubsd	xmm1, xmm1, xmm3		;; R3 - R7 (newest R7)

						;; R9/I9 morphs into newest R9/R13
						;; R11/I11 morphs into newest R11/R15

	;; Odd level 1

	vaddsd	xmm3, xmm4, xmm8		;; R1 + R9 (final R1)
	vsubsd	xmm4, xmm4, xmm8		;; R1 - R9 (final R9)

	vaddsd	xmm8, xmm7, xmm6		;; R5 + R13 (final R5)
	vsubsd	xmm7, xmm7, xmm6		;; R5 - R13 (final R13)

	vaddsd	xmm6, xmm2, xmm0		;; R3 + R11 (final R3)
	vsubsd	xmm2, xmm2, xmm0		;; R3 - R11 (final R11)

	vaddsd	xmm0, xmm1, xmm5		;; R7 + R15 (final R7)
	vsubsd	xmm1, xmm1, xmm5		;; R7 - R15 (final R15)

	vmovsd	Q [srcreg+r8], xmm3		;; R1
	vmovsd	Q [srcreg+r8+32], xmm4		;; R9
	vmovsd	Q [srcreg+r8+d4], xmm8		;; R5
	vmovsd	Q [srcreg+r8+d4+32], xmm7	;; R13
	vmovsd	Q [srcreg+r8+d2], xmm6		;; R3
	vmovsd	Q [srcreg+r8+d2+32], xmm2	;; R11
	vmovsd	Q [srcreg+r8+d4+d2], xmm0	;; R7
	vmovsd	Q [srcreg+r8+d4+d2+32], xmm1	;; R15

	bump	srcreg, srcinc
	ENDM

yr8_8cl_sixteen_reals_eight_complex_with_mulf MACRO srcreg,srcinc,d1,d2,d4
	LOCAL	orig, back_to_orig, fma_or_unfft, fma, addmul, submul, muladd, muladdhard, mulsub, mulsubhard, unfft

	mov	al, mul4_opcode			;; Load the mul4_opcode
	cmp	al, 0				;; See if we need to do more than the original type-4 FFT multiply
	je	orig				;; Jump if nothing special
	mov	r9, SRC2ARG			;; Distance to s2 arg from gwaddmul4 or gwsubmul4 or s3 arg from gwmuladd4 or gwmulsub4
	cmp	al, 2				;; Are we doing addmul or submul
	jg	fma_or_unfft			;; no, opcode == 3,4,5
	je	submul				;; opcode == 2
	;jb	addmul				;; opcode == 1

addmul:	vmovapd	ymm3, [srcreg+d2+d1][rbp]	;; MemR4
	vaddpd	ymm3, ymm3, [srcreg+d2+d1][r9]	;; MemR4
	vmovapd	ymm9, [srcreg+d2+d1+32][rbp]	;; MemI4
	vaddpd	ymm9, ymm9, [srcreg+d2+d1+32][r9];; MemI4
	vmovapd	ymm7, [srcreg+d2+d1][rbx]	;; R4
	vmovapd	ymm10, [srcreg+d2+d1+32][rbx]	;; I4
	vmulpd	ymm4, ymm7, ymm3		;; R4 * MemR4			; 32-36
	vmulpd	ymm7, ymm7, ymm9		;; R4 * MemI4			; 33-37
	vmulpd	ymm9, ymm10, ymm9		;; I4 * MemI4			; 34-38
	vmulpd	ymm10, ymm10, ymm3		;; I4 * MemR4			; 35-39

	vmovapd	ymm3, [srcreg+d2][rbp]		;; MemR3
	vaddpd	ymm3, ymm3, [srcreg+d2][r9]	;; MemR3
	vsubpd	ymm4, ymm4, ymm9		;; R4*MemR4-I4*MemI4 (new R4)	; 51-53
	vmovapd	ymm9, [srcreg+d2+32][rbp]	;; MemI3
	vaddpd	ymm9, ymm9, [srcreg+d2+32][r9]	;; MemI3
	vaddpd	ymm7, ymm7, ymm10		;; R4*MemI4+I4*MemR4 (new I4)	; 52-54

	vmovapd	ymm2, [srcreg+d2][rbx]		;; R3
	vmovapd	ymm11, [srcreg+d2+32][rbx]	;; I3
	vmulpd	ymm10, ymm2, ymm3		;; R3 * MemR3			; 36-40
	vmulpd	ymm2, ymm2, ymm9		;; R3 * MemI3			; 37-41
	vmulpd	ymm9, ymm11, ymm9		;; I3 * MemI3			; 38-42
	vmulpd	ymm11, ymm11, ymm3		;; I3 * MemR3			; 39-43

	vmovapd	ymm3, [srcreg+d4+d1][rbp]	;; MemR6
	vaddpd	ymm3, ymm3, [srcreg+d4+d1][r9]	;; MemR6
	vsubpd	ymm10, ymm10, ymm9		;; R3*MemR3-I3*MemI3 (new R3)	; 53-55
	vmovapd	ymm9, [srcreg+d4+d1+32][rbp]	;; MemI6
	vaddpd	ymm9, ymm9, [srcreg+d4+d1+32][r9];; MemI6
	vaddpd	ymm2, ymm2, ymm11		;; R3*MemI3+I3*MemR3 (new I3)	; 54-56

	vmovapd	ymm1, [srcreg+d4+d1][rbx]	;; R6
	vmovapd	ymm5, [srcreg+d4+d1+32][rbx]	;; I6
	vmulpd	ymm11, ymm1, ymm3		;; R6 * MemR6			; 42-46
	vmulpd	ymm1, ymm1, ymm9		;; R6 * MemI6			; 43-47
	vmulpd	ymm9, ymm5, ymm9		;; I6 * MemI6			; 46-50
	vmulpd	ymm5, ymm5, ymm3		;; I6 * MemR6			; 47-51

	vmovapd	ymm3, [srcreg+d4][rbp]		;; MemR5
	vaddpd	ymm3, ymm3, [srcreg+d4][r9]	;; MemR5
	vsubpd	ymm11, ymm11, ymm9		;; R6*MemR6-I6*MemI6 (new R6)	; 55-57
	vmovapd	ymm9, [srcreg+d4+32][rbp]	;; MemI5
	vaddpd	ymm9, ymm9, [srcreg+d4+32][r9]	;; MemI5
	vaddpd	ymm1, ymm1, ymm5		;; R6*MemI6+I6*MemR6 (new I6)	; 56-58

	vmovapd	ymm12, [srcreg+d4][rbx]		;; R5
	vmovapd	ymm15, [srcreg+d4+32][rbx]	;; I5
	vmulpd	ymm5, ymm12, ymm3		;; R5 * MemR5			; 44-48
	vmulpd	ymm12, ymm12, ymm9		;; R5 * MemI5			; 45-49
	vmulpd	ymm9, ymm15, ymm9		;; I5 * MemI5			; 48-52
	vmulpd	ymm15, ymm15, ymm3		;; I5 * MemR5			; 49-53

	vmovapd	ymm3, [srcreg+d4+d2+d1][rbp]	;; MemR8
	vaddpd	ymm3, ymm3, [srcreg+d4+d2+d1][r9];; MemR8
	vsubpd	ymm5, ymm5, ymm9		;; R5*MemR5-I5*MemI5 (new R5)	; 57-59
	vmovapd	ymm9, [srcreg+d4+d2+d1+32][rbp]	;; MemI8
	vaddpd	ymm9, ymm9, [srcreg+d4+d2+d1+32][r9];; MemI8
	vaddpd	ymm12, ymm12, ymm15		;; R5*MemI5+I5*MemR5 (new I5)	; 58-60

	vmovapd	ymm14, [srcreg+d4+d2+d1][rbx]	;; R8
	vmovapd	ymm8, [srcreg+d4+d2+d1+32][rbx]	;; I8
	vmulpd	ymm15, ymm14, ymm3		;; R8 * MemR8			; 50-54
	vmulpd	ymm14, ymm14, ymm9		;; R8 * MemI8			; 51-55
	vmulpd	ymm9, ymm8, ymm9		;; I8 * MemI8			; 52-56
	vmulpd	ymm8, ymm8, ymm3		;; I8 * MemR8			; 53-57

	vmovapd	ymm3, [srcreg+d4+d2][rbp]	;; MemR7
	vaddpd	ymm3, ymm3, [srcreg+d4+d2][r9]	;; MemR7
	vsubpd	ymm15, ymm15, ymm9		;; R8*MemR8-I8*MemI8 (new R8)	; 59-61
	vmovapd	ymm9, [srcreg+d4+d2+32][rbp]	;; MemI7
	vaddpd	ymm9, ymm9, [srcreg+d4+d2+32][r9];; MemI7
	vaddpd	ymm14, ymm14, ymm8		;; R8*MemI8+I8*MemR8 (new I8)	; 60-62

	vmovapd	ymm0, [srcreg+d4+d2][rbx]	;; R7
	vmovapd	ymm6, [srcreg+d4+d2+32][rbx]	;; I7
	vmulpd	ymm8, ymm0, ymm3		;; R7 * MemR7			; 54-58
	vmulpd	ymm0, ymm0, ymm9		;; R7 * MemI7			; 55-59
	vmulpd	ymm9, ymm6, ymm9		;; I7 * MemI7			; 56-60
	vmulpd	ymm6, ymm6, ymm3		;; I7 * MemR7			; 57-61

	vsubpd	ymm8, ymm8, ymm9		;; R7*MemR7-I7*MemI7 (new R7)	; 61-63
	vaddpd	ymm0, ymm0, ymm6		;; R7*MemI7+I7*MemR7 (new I7)	; 62-64

	vaddpd	ymm9, ymm4, ymm10		;; R4 + R3 (new R3)		; 65-67
	vsubpd	ymm4, ymm4, ymm10		;; R4 - R3 (new I4)		; 66-68

	vmovapd	ymm10, [srcreg+d1][rbp]		;; MemR2
	vaddpd	ymm10, ymm10, [srcreg+d1][r9]	;; MemR2
	ystore	YMM_TMP5, ymm9			;; Temporarily save new R3	; 68
	vmovapd	ymm9, [srcreg+d1+32][rbp]	;; MemI2
	vaddpd	ymm9, ymm9, [srcreg+d1+32][r9]	;; MemI2
	ystore	YMM_TMP6, ymm4			;; Temporarily save new I4	; 69

	vmovapd	ymm6, [srcreg+d1][rbx]		;; R2
	vmovapd	ymm13, [srcreg+d1+32][rbx]	;; I2
	vmulpd	ymm4, ymm6, ymm10		;; R2 * MemR2			; 66-70
	vmulpd	ymm6, ymm6, ymm9		;; R2 * MemI2			; 67-71
	vmulpd	ymm9, ymm13, ymm9		;; I2 * MemI2			; 68-72
	vmulpd	ymm13, ymm13, ymm10		;; I2 * MemR2			; 69-73

	vmovapd	ymm10, [srcreg][rbp]		;; MemR1
	vaddpd	ymm10, ymm10, [srcreg][r9]	;; MemR1
	vsubpd	ymm4, ymm4, ymm9		;; R2*MemR2-I2*MemI2 (new R2)	; 73-75
	vmovapd	ymm9, [srcreg+32][rbp]		;; MemI1
	vaddpd	ymm9, ymm9, [srcreg+32][r9]	;; MemI1
	vaddpd	ymm6, ymm6, ymm13		;; R2*MemI2+I2*MemR2 (new I2)	; 74-76

	vsubpd	ymm13, ymm2, ymm7		;; I3 - I4 (new R4)		; 67-69
	vaddpd	ymm2, ymm2, ymm7		;; I3 + I4 (new I3)		; 68-70

	vsubpd	ymm7, ymm12, ymm1		;; I5 - I6 (new I6)		; 69-71
	vaddpd	ymm12, ymm12, ymm1		;; I5 + I6 (new I5)		; 70-72

	vsubpd	ymm1, ymm5, ymm11		;; R5 - R6 (new R6)		; 71-73
	vaddpd	ymm5, ymm5, ymm11		;; R5 + R6 (new R5)		; 72-74

	vmovapd	ymm3, [srcreg][rbx]		;; R1
	vmulpd	ymm11, ymm3, ymm10		;; R1 * MemR1			; 70-74
	vmulpd	ymm3, ymm3, ymm9		;; R1 * MemI1			; 71-75
	vmovsd	Q [srcreg-16], xmm11		;; Save product of sum of FFT values
	vmulpd	ymm9, ymm9, [srcreg+32][rbx]	;; (load I1) * MemI1		; 72-76
	vmulpd	ymm10, ymm10, [srcreg+32][rbx]	;; (load I1) * MemR1		; 73-77
	vmovsd	Q YMM_TMP1[8], xmm9		;; I1

	vsubpd	ymm11, ymm11, ymm9		;; R1*MemR1-I1*MemI1 (new R1)	; 77-79
	vaddpd	ymm3, ymm3, ymm10		;; R1*MemI1+I1*MemR1 (new I1)	; 78-80
	jmp	back_to_orig

submul:	vmovapd	ymm3, [srcreg+d2+d1][rbp]	;; MemR4
	vsubpd	ymm3, ymm3, [srcreg+d2+d1][r9]	;; MemR4
	vmovapd	ymm9, [srcreg+d2+d1+32][rbp]	;; MemI4
	vsubpd	ymm9, ymm9, [srcreg+d2+d1+32][r9];; MemI4
	vmovapd	ymm7, [srcreg+d2+d1][rbx]	;; R4
	vmovapd	ymm10, [srcreg+d2+d1+32][rbx]	;; I4
	vmulpd	ymm4, ymm7, ymm3		;; R4 * MemR4			; 32-36
	vmulpd	ymm7, ymm7, ymm9		;; R4 * MemI4			; 33-37
	vmulpd	ymm9, ymm10, ymm9		;; I4 * MemI4			; 34-38
	vmulpd	ymm10, ymm10, ymm3		;; I4 * MemR4			; 35-39

	vmovapd	ymm3, [srcreg+d2][rbp]		;; MemR3
	vsubpd	ymm3, ymm3, [srcreg+d2][r9]	;; MemR3
	vsubpd	ymm4, ymm4, ymm9		;; R4*MemR4-I4*MemI4 (new R4)	; 51-53
	vmovapd	ymm9, [srcreg+d2+32][rbp]	;; MemI3
	vsubpd	ymm9, ymm9, [srcreg+d2+32][r9]	;; MemI3
	vaddpd	ymm7, ymm7, ymm10		;; R4*MemI4+I4*MemR4 (new I4)	; 52-54

	vmovapd	ymm2, [srcreg+d2][rbx]		;; R3
	vmovapd	ymm11, [srcreg+d2+32][rbx]	;; I3
	vmulpd	ymm10, ymm2, ymm3		;; R3 * MemR3			; 36-40
	vmulpd	ymm2, ymm2, ymm9		;; R3 * MemI3			; 37-41
	vmulpd	ymm9, ymm11, ymm9		;; I3 * MemI3			; 38-42
	vmulpd	ymm11, ymm11, ymm3		;; I3 * MemR3			; 39-43

	vmovapd	ymm3, [srcreg+d4+d1][rbp]	;; MemR6
	vsubpd	ymm3, ymm3, [srcreg+d4+d1][r9]	;; MemR6
	vsubpd	ymm10, ymm10, ymm9		;; R3*MemR3-I3*MemI3 (new R3)	; 53-55
	vmovapd	ymm9, [srcreg+d4+d1+32][rbp]	;; MemI6
	vsubpd	ymm9, ymm9, [srcreg+d4+d1+32][r9];; MemI6
	vaddpd	ymm2, ymm2, ymm11		;; R3*MemI3+I3*MemR3 (new I3)	; 54-56

	vmovapd	ymm1, [srcreg+d4+d1][rbx]	;; R6
	vmovapd	ymm5, [srcreg+d4+d1+32][rbx]	;; I6
	vmulpd	ymm11, ymm1, ymm3		;; R6 * MemR6			; 42-46
	vmulpd	ymm1, ymm1, ymm9		;; R6 * MemI6			; 43-47
	vmulpd	ymm9, ymm5, ymm9		;; I6 * MemI6			; 46-50
	vmulpd	ymm5, ymm5, ymm3		;; I6 * MemR6			; 47-51

	vmovapd	ymm3, [srcreg+d4][rbp]		;; MemR5
	vsubpd	ymm3, ymm3, [srcreg+d4][r9]	;; MemR5
	vsubpd	ymm11, ymm11, ymm9		;; R6*MemR6-I6*MemI6 (new R6)	; 55-57
	vmovapd	ymm9, [srcreg+d4+32][rbp]	;; MemI5
	vsubpd	ymm9, ymm9, [srcreg+d4+32][r9]	;; MemI5
	vaddpd	ymm1, ymm1, ymm5		;; R6*MemI6+I6*MemR6 (new I6)	; 56-58

	vmovapd	ymm12, [srcreg+d4][rbx]		;; R5
	vmovapd	ymm15, [srcreg+d4+32][rbx]	;; I5
	vmulpd	ymm5, ymm12, ymm3		;; R5 * MemR5			; 44-48
	vmulpd	ymm12, ymm12, ymm9		;; R5 * MemI5			; 45-49
	vmulpd	ymm9, ymm15, ymm9		;; I5 * MemI5			; 48-52
	vmulpd	ymm15, ymm15, ymm3		;; I5 * MemR5			; 49-53

	vmovapd	ymm3, [srcreg+d4+d2+d1][rbp]	;; MemR8
	vsubpd	ymm3, ymm3, [srcreg+d4+d2+d1][r9];; MemR8
	vsubpd	ymm5, ymm5, ymm9		;; R5*MemR5-I5*MemI5 (new R5)	; 57-59
	vmovapd	ymm9, [srcreg+d4+d2+d1+32][rbp]	;; MemI8
	vsubpd	ymm9, ymm9, [srcreg+d4+d2+d1+32][r9];; MemI8
	vaddpd	ymm12, ymm12, ymm15		;; R5*MemI5+I5*MemR5 (new I5)	; 58-60

	vmovapd	ymm14, [srcreg+d4+d2+d1][rbx]	;; R8
	vmovapd	ymm8, [srcreg+d4+d2+d1+32][rbx]	;; I8
	vmulpd	ymm15, ymm14, ymm3		;; R8 * MemR8			; 50-54
	vmulpd	ymm14, ymm14, ymm9		;; R8 * MemI8			; 51-55
	vmulpd	ymm9, ymm8, ymm9		;; I8 * MemI8			; 52-56
	vmulpd	ymm8, ymm8, ymm3		;; I8 * MemR8			; 53-57

	vmovapd	ymm3, [srcreg+d4+d2][rbp]	;; MemR7
	vsubpd	ymm3, ymm3, [srcreg+d4+d2][r9]	;; MemR7
	vsubpd	ymm15, ymm15, ymm9		;; R8*MemR8-I8*MemI8 (new R8)	; 59-61
	vmovapd	ymm9, [srcreg+d4+d2+32][rbp]	;; MemI7
	vsubpd	ymm9, ymm9, [srcreg+d4+d2+32][r9];; MemI7
	vaddpd	ymm14, ymm14, ymm8		;; R8*MemI8+I8*MemR8 (new I8)	; 60-62

	vmovapd	ymm0, [srcreg+d4+d2][rbx]	;; R7
	vmovapd	ymm6, [srcreg+d4+d2+32][rbx]	;; I7
	vmulpd	ymm8, ymm0, ymm3		;; R7 * MemR7			; 54-58
	vmulpd	ymm0, ymm0, ymm9		;; R7 * MemI7			; 55-59
	vmulpd	ymm9, ymm6, ymm9		;; I7 * MemI7			; 56-60
	vmulpd	ymm6, ymm6, ymm3		;; I7 * MemR7			; 57-61

	vsubpd	ymm8, ymm8, ymm9		;; R7*MemR7-I7*MemI7 (new R7)	; 61-63
	vaddpd	ymm0, ymm0, ymm6		;; R7*MemI7+I7*MemR7 (new I7)	; 62-64

	vaddpd	ymm9, ymm4, ymm10		;; R4 + R3 (new R3)		; 65-67
	vsubpd	ymm4, ymm4, ymm10		;; R4 - R3 (new I4)		; 66-68

	vmovapd	ymm10, [srcreg+d1][rbp]		;; MemR2
	vsubpd	ymm10, ymm10, [srcreg+d1][r9]	;; MemR2
	ystore	YMM_TMP5, ymm9			;; Temporarily save new R3	; 68
	vmovapd	ymm9, [srcreg+d1+32][rbp]	;; MemI2
	vsubpd	ymm9, ymm9, [srcreg+d1+32][r9]	;; MemI2
	ystore	YMM_TMP6, ymm4			;; Temporarily save new I4	; 69

	vmovapd	ymm6, [srcreg+d1][rbx]		;; R2
	vmovapd	ymm13, [srcreg+d1+32][rbx]	;; I2
	vmulpd	ymm4, ymm6, ymm10		;; R2 * MemR2			; 66-70
	vmulpd	ymm6, ymm6, ymm9		;; R2 * MemI2			; 67-71
	vmulpd	ymm9, ymm13, ymm9		;; I2 * MemI2			; 68-72
	vmulpd	ymm13, ymm13, ymm10		;; I2 * MemR2			; 69-73

	vmovapd	ymm10, [srcreg][rbp]		;; MemR1
	vsubpd	ymm10, ymm10, [srcreg][r9]	;; MemR1
	vsubpd	ymm4, ymm4, ymm9		;; R2*MemR2-I2*MemI2 (new R2)	; 73-75
	vmovapd	ymm9, [srcreg+32][rbp]		;; MemI1
	vsubpd	ymm9, ymm9, [srcreg+32][r9]	;; MemI1
	vaddpd	ymm6, ymm6, ymm13		;; R2*MemI2+I2*MemR2 (new I2)	; 74-76

	vsubpd	ymm13, ymm2, ymm7		;; I3 - I4 (new R4)		; 67-69
	vaddpd	ymm2, ymm2, ymm7		;; I3 + I4 (new I3)		; 68-70

	vsubpd	ymm7, ymm12, ymm1		;; I5 - I6 (new I6)		; 69-71
	vaddpd	ymm12, ymm12, ymm1		;; I5 + I6 (new I5)		; 70-72

	vsubpd	ymm1, ymm5, ymm11		;; R5 - R6 (new R6)		; 71-73
	vaddpd	ymm5, ymm5, ymm11		;; R5 + R6 (new R5)		; 72-74

	vmovapd	ymm3, [srcreg][rbx]		;; R1
	vmulpd	ymm11, ymm3, ymm10		;; R1 * MemR1			; 70-74
	vmulpd	ymm3, ymm3, ymm9		;; R1 * MemI1			; 71-75
	vmovsd	Q [srcreg-16], xmm11		;; Save product of sum of FFT values
	vmulpd	ymm9, ymm9, [srcreg+32][rbx]	;; (load I1) * MemI1		; 72-76
	vmulpd	ymm10, ymm10, [srcreg+32][rbx]	;; (load I1) * MemR1		; 73-77
	vmovsd	Q YMM_TMP1[8], xmm9		;; I1

	vsubpd	ymm11, ymm11, ymm9		;; R1*MemR1-I1*MemI1 (new R1)	; 77-79
	vaddpd	ymm3, ymm3, ymm10		;; R1*MemI1+I1*MemR1 (new I1)	; 78-80
	jmp	back_to_orig

fma_or_unfft:
	cmp	al, 4				;; Case off opcode
	jbe	fma				;; 3=muladd, 4=mulsub, fma MUST NOT ALTER CONDITION CODES!
	;;ja	unfft				;; 5=unfft

unfft:	vmovapd	ymm4, [srcreg+d2+d1][rbx]	;; R4
	vmovapd	ymm10, [srcreg+d2][rbx]		;; R3
	vaddpd	ymm9, ymm4, ymm10		;; R4 + R3 (new R3)		; 65-67
	vsubpd	ymm4, ymm4, ymm10		;; R4 - R3 (new I4)		; 66-68
	ystore	YMM_TMP5, ymm9			;; Temporarily save new R3	; 68
	ystore	YMM_TMP6, ymm4			;; Temporarily save new I4	; 69

	vmovapd	ymm2, [srcreg+d2+32][rbx]	;; I3
	vmovapd	ymm7, [srcreg+d2+d1+32][rbx]	;; I4
	vsubpd	ymm13, ymm2, ymm7		;; I3 - I4 (new R4)		; 67-69
	vaddpd	ymm2, ymm2, ymm7		;; I3 + I4 (new I3)		; 68-70

	vmovapd	ymm12, [srcreg+d4+32][rbx]	;; I5
	vmovapd	ymm1, [srcreg+d4+d1+32][rbx]	;; I6
	vsubpd	ymm7, ymm12, ymm1		;; I5 - I6 (new I6)		; 69-71
	vaddpd	ymm12, ymm12, ymm1		;; I5 + I6 (new I5)		; 70-72

	vmovapd	ymm5, [srcreg+d4][rbx]		;; R5
	vmovapd	ymm11, [srcreg+d4+d1][rbx]	;; R6
	vsubpd	ymm1, ymm5, ymm11		;; R5 - R6 (new R6)		; 71-73
	vaddpd	ymm5, ymm5, ymm11		;; R5 + R6 (new R5)		; 72-74

	vmovapd	ymm11, [srcreg][rbx]		;; R1
	vmovsd	Q [srcreg-16], xmm11		;; Save product of sum of FFT values
	vmovapd	ymm3, [srcreg+32][rbx]		;; I1
	vmovsd	Q YMM_TMP1[8], xmm3		;; I1

	vmovapd	ymm4, [srcreg+d1][rbx]		;; R2
	vmovapd	ymm6, [srcreg+d1+32][rbx]	;; I2
	vmovapd	ymm8, [srcreg+d4+d2][rbx]	;; R7
	vmovapd	ymm0, [srcreg+d4+d2+32][rbx]	;; I7
	vmovapd	ymm15, [srcreg+d4+d2+d1][rbx]	;; R8
	vmovapd	ymm14, [srcreg+d4+d2+d1+32][rbx];; I8
	jmp	back_to_orig

fma:	vmovapd	ymm3, [srcreg+d2+d1][rbp]	;; MemR4
	vmovapd	ymm9, [srcreg+d2+d1+32][rbp]	;; MemI4
	vmovapd	ymm7, [srcreg+d2+d1][rbx]	;; R4
	vmovapd	ymm10, [srcreg+d2+d1+32][rbx]	;; I4
	vmulpd	ymm4, ymm7, ymm3		;; R4 * MemR4			; 32-36
	vmulpd	ymm7, ymm7, ymm9		;; R4 * MemI4			; 33-37
	vmulpd	ymm9, ymm10, ymm9		;; I4 * MemI4			; 34-38
	vmulpd	ymm10, ymm10, ymm3		;; I4 * MemR4			; 35-39

	vmovapd	ymm3, [srcreg+d2][rbp]		;; MemR3
	vsubpd	ymm4, ymm4, ymm9		;; R4*MemR4-I4*MemI4 (new R4)	; 51-53
	vmovapd	ymm9, [srcreg+d2+32][rbp]	;; MemI3
	vaddpd	ymm7, ymm7, ymm10		;; R4*MemI4+I4*MemR4 (new I4)	; 52-54

	vmovapd	ymm2, [srcreg+d2][rbx]		;; R3
	vmovapd	ymm11, [srcreg+d2+32][rbx]	;; I3
	vmulpd	ymm10, ymm2, ymm3		;; R3 * MemR3			; 36-40
	vmulpd	ymm2, ymm2, ymm9		;; R3 * MemI3			; 37-41
	vmulpd	ymm9, ymm11, ymm9		;; I3 * MemI3			; 38-42
	vmulpd	ymm11, ymm11, ymm3		;; I3 * MemR3			; 39-43

	vmovapd	ymm3, [srcreg+d4+d1][rbp]	;; MemR6
	vsubpd	ymm10, ymm10, ymm9		;; R3*MemR3-I3*MemI3 (new R3)	; 53-55
	vmovapd	ymm9, [srcreg+d4+d1+32][rbp]	;; MemI6
	vaddpd	ymm2, ymm2, ymm11		;; R3*MemI3+I3*MemR3 (new I3)	; 54-56

	vmovapd	ymm1, [srcreg+d4+d1][rbx]	;; R6
	vmovapd	ymm5, [srcreg+d4+d1+32][rbx]	;; I6
	vmulpd	ymm11, ymm1, ymm3		;; R6 * MemR6			; 42-46
	vmulpd	ymm1, ymm1, ymm9		;; R6 * MemI6			; 43-47
	vmulpd	ymm9, ymm5, ymm9		;; I6 * MemI6			; 46-50
	vmulpd	ymm5, ymm5, ymm3		;; I6 * MemR6			; 47-51

	vmovapd	ymm3, [srcreg+d4][rbp]		;; MemR5
	vsubpd	ymm11, ymm11, ymm9		;; R6*MemR6-I6*MemI6 (new R6)	; 55-57
	vmovapd	ymm9, [srcreg+d4+32][rbp]	;; MemI5
	vaddpd	ymm1, ymm1, ymm5		;; R6*MemI6+I6*MemR6 (new I6)	; 56-58

	vmovapd	ymm12, [srcreg+d4][rbx]		;; R5
	vmovapd	ymm15, [srcreg+d4+32][rbx]	;; I5
	vmulpd	ymm5, ymm12, ymm3		;; R5 * MemR5			; 44-48
	vmulpd	ymm12, ymm12, ymm9		;; R5 * MemI5			; 45-49
	vmulpd	ymm9, ymm15, ymm9		;; I5 * MemI5			; 48-52
	vmulpd	ymm15, ymm15, ymm3		;; I5 * MemR5			; 49-53

	vmovapd	ymm3, [srcreg+d4+d2+d1][rbp]	;; MemR8
	vsubpd	ymm5, ymm5, ymm9		;; R5*MemR5-I5*MemI5 (new R5)	; 57-59
	vmovapd	ymm9, [srcreg+d4+d2+d1+32][rbp]	;; MemI8
	vaddpd	ymm12, ymm12, ymm15		;; R5*MemI5+I5*MemR5 (new I5)	; 58-60

	vmovapd	ymm14, [srcreg+d4+d2+d1][rbx]	;; R8
	vmovapd	ymm8, [srcreg+d4+d2+d1+32][rbx]	;; I8
	vmulpd	ymm15, ymm14, ymm3		;; R8 * MemR8			; 50-54
	vmulpd	ymm14, ymm14, ymm9		;; R8 * MemI8			; 51-55
	vmulpd	ymm9, ymm8, ymm9		;; I8 * MemI8			; 52-56
	vmulpd	ymm8, ymm8, ymm3		;; I8 * MemR8			; 53-57

	vmovapd	ymm3, [srcreg+d4+d2][rbp]	;; MemR7
	vsubpd	ymm15, ymm15, ymm9		;; R8*MemR8-I8*MemI8 (new R8)	; 59-61
	vmovapd	ymm9, [srcreg+d4+d2+32][rbp]	;; MemI7
	vaddpd	ymm14, ymm14, ymm8		;; R8*MemI8+I8*MemR8 (new I8)	; 60-62

	vmovapd	ymm0, [srcreg+d4+d2][rbx]	;; R7
	vmovapd	ymm6, [srcreg+d4+d2+32][rbx]	;; I7
	vmulpd	ymm8, ymm0, ymm3		;; R7 * MemR7			; 54-58
	vmulpd	ymm0, ymm0, ymm9		;; R7 * MemI7			; 55-59
	vmulpd	ymm9, ymm6, ymm9		;; I7 * MemI7			; 56-60
	vmulpd	ymm6, ymm6, ymm3		;; I7 * MemR7			; 57-61

	vsubpd	ymm8, ymm8, ymm9		;; R7*MemR7-I7*MemI7 (new R7)	; 61-63
	vaddpd	ymm0, ymm0, ymm6		;; R7*MemI7+I7*MemR7 (new I7)	; 62-64

	mov	r10, SRC3ARG			;; Load distance to FFT(1)
	je	mulsub				;; 4=mulsub
	;jb	muladd				;; 3=muladd, fall through

muladd:	test	r10, 3				;; Check if we need to multiply add-in by FFT(1)
	jz	muladdhard			;; Yes, do it the hard way

	vaddpd	ymm4, ymm4, [srcreg+r9+d2+d1]		;; R4 = R4 + MemR4
	vaddpd	ymm7, ymm7, [srcreg+r9+d2+d1+32]	;; I4 = I4 + MemI4
	vaddpd	ymm10, ymm10, [srcreg+r9+d2]		;; R3 = R3 + MemR3
	vaddpd	ymm2, ymm2, [srcreg+r9+d2+32]		;; I3 = I3 + MemI3
	vaddpd	ymm11, ymm11, [srcreg+r9+d4+d1]		;; R6 = R6 + MemR6
	vaddpd	ymm1, ymm1, [srcreg+r9+d4+d1+32]	;; I6 = I6 + MemI6
	vaddpd	ymm5, ymm5, [srcreg+r9+d4]		;; R5 = R5 + MemR5
	vaddpd	ymm12, ymm12, [srcreg+r9+d4+32]		;; I5 = I5 + MemI5
	vaddpd	ymm15, ymm15, [srcreg+r9+d4+d2+d1]	;; R8 = R8 + MemR8
	vaddpd	ymm14, ymm14, [srcreg+r9+d4+d2+d1+32]	;; I8 = I8 + MemI8
	vaddpd	ymm8, ymm8, [srcreg+r9+d4+d2]		;; R7 = R7 + MemR7
	vaddpd	ymm0, ymm0, [srcreg+r9+d4+d2+32]	;; I7 = I7 + MemI7

	vaddpd	ymm9, ymm4, ymm10		;; R4 + R3 (new R3)		; 65-67
	vsubpd	ymm4, ymm4, ymm10		;; R4 - R3 (new I4)		; 66-68

	vmovapd	ymm10, [srcreg+d1][rbp]		;; MemR2
	ystore	YMM_TMP5, ymm9			;; Temporarily save new R3	; 68
	vmovapd	ymm9, [srcreg+d1+32][rbp]	;; MemI2
	ystore	YMM_TMP6, ymm4			;; Temporarily save new I4	; 69
	vmovapd	ymm6, [srcreg+d1][rbx]		;; R2
	vmovapd	ymm13, [srcreg+d1+32][rbx]	;; I2
	vmulpd	ymm4, ymm6, ymm10		;; R2 * MemR2			; 66-70
	vmulpd	ymm6, ymm6, ymm9		;; R2 * MemI2			; 67-71
	vmulpd	ymm9, ymm13, ymm9		;; I2 * MemI2			; 68-72
	vmulpd	ymm13, ymm13, ymm10		;; I2 * MemR2			; 69-73

	vmovapd	ymm10, [srcreg][rbp]		;; MemR1
	vsubpd	ymm4, ymm4, ymm9		;; R2*MemR2-I2*MemI2 (new R2)	; 73-75
	vmovapd	ymm9, [srcreg+32][rbp]		;; MemI1
	vaddpd	ymm6, ymm6, ymm13		;; R2*MemI2+I2*MemR2 (new I2)	; 74-76

	vsubpd	ymm13, ymm2, ymm7		;; I3 - I4 (new R4)		; 67-69
	vaddpd	ymm2, ymm2, ymm7		;; I3 + I4 (new I3)		; 68-70

	vsubpd	ymm7, ymm12, ymm1		;; I5 - I6 (new I6)		; 69-71
	vaddpd	ymm12, ymm12, ymm1		;; I5 + I6 (new I5)		; 70-72

	vsubpd	ymm1, ymm5, ymm11		;; R5 - R6 (new R6)		; 71-73
	vaddpd	ymm5, ymm5, ymm11		;; R5 + R6 (new R5)		; 72-74

	vmovapd	ymm3, [srcreg][rbx]		;; R1
	vmulpd	ymm11, ymm3, ymm10		;; R1 * MemR1			; 70-74
	vmulpd	ymm3, ymm3, ymm9		;; R1 * MemI1			; 71-75
	vmovsd	Q [srcreg-16], xmm11		;; Save product of sum of FFT values
	vmulpd	ymm9, ymm9, [srcreg+32][rbx]	;; (load I1) * MemI1		; 72-76
	vmulpd	ymm10, ymm10, [srcreg+32][rbx]	;; (load I1) * MemR1		; 73-77
	vmovsd	Q YMM_TMP1[8], xmm9		;; I1

	vaddpd	ymm3, ymm3, ymm10		;; R1*MemI1+I1*MemR1 (new I1)	; 78-80
	vsubpd	ymm10, ymm11, ymm9		;; R1*MemR1-I1*MemI1 (new R1)	; 77-79

	vaddpd	ymm4, ymm4, [srcreg+r9+d1]		;; R2 = R2 + MemR2
	vaddpd	ymm6, ymm6, [srcreg+r9+d1+32]		;; I2 = I2 + MemI2
	vaddsd	xmm11, xmm11, [srcreg+r9]		;; R1a = R1a + MemR1a
	vmovsd	Q [srcreg-16], xmm11		;; Save product of sum of FFT values
	vaddpd	ymm11, ymm10, [srcreg+r9]		;; R1 = R1 + MemR1 (lower word is irrelevant)
	vmovapd	ymm10, [srcreg+r9+32]			;; MemI1
	vaddsd	xmm9, xmm9, xmm10			;; R1b = R1b + MemR1b in MemI1
	vaddpd	ymm3, ymm3, ymm10			;; I1 = I1 + MemI1 (lower word is irrelevant)
	vmovsd	Q YMM_TMP1[8], xmm9			;; Save R1b
	jmp	back_to_orig

muladdhard:
	vmovapd	ymm3, [srcreg+r9+d2+d1]			;; MemR4
	vmovapd	ymm9, [srcreg+r10+d2+d1]		;; MemR4#2
	vmulpd	ymm13, ymm3, ymm9			;; MemR4*MemR4#2
	vaddpd	ymm4, ymm4, ymm13			;; R4 = R4 + MemR4*MemR4#2
	vmovapd	ymm6, [srcreg+r10+d2+d1+32]		;; MemI4#2
	vmulpd	ymm13, ymm3, ymm6			;; MemR4*MemI4#2
	vaddpd	ymm7, ymm7, ymm13			;; I4 = I4 + MemR4*MemI4#2
	vmovapd	ymm3, [srcreg+r9+d2+d1+32]		;; MemI4
	vmulpd	ymm13, ymm3, ymm6			;; MemI4*MemI4#2
	vsubpd	ymm4, ymm4, ymm13			;; R4 = R4 - MemI4*MemI4#2
	vmulpd	ymm13, ymm3, ymm9			;; MemI4*MemR4#2
	vaddpd	ymm7, ymm7, ymm13			;; I4 = I4 + MemI4*MemR4#2

	vmovapd	ymm3, [srcreg+r9+d2]			;; MemR3
	vmovapd	ymm9, [srcreg+r10+d2]			;; MemR3#2
	vmulpd	ymm13, ymm3, ymm9			;; MemR3*MemR3#2
	vaddpd	ymm10, ymm10, ymm13			;; R3 = R3 + MemR3*MemR3#2
	vmovapd	ymm6, [srcreg+r10+d2+32]		;; MemI3#2
	vmulpd	ymm13, ymm3, ymm6			;; MemR3*MemI3#2
	vaddpd	ymm2, ymm2, ymm13			;; I3 = I3 + MemR3*MemI3#2
	vmovapd	ymm3, [srcreg+r9+d2+32]			;; MemI3
	vmulpd	ymm13, ymm3, ymm6			;; MemI3*MemI3#2
	vsubpd	ymm10, ymm10, ymm13			;; R3 = R3 - MemI3*MemI3#2
	vmulpd	ymm13, ymm3, ymm9			;; MemI3*MemR3#2
	vaddpd	ymm2, ymm2, ymm13			;; I3 = I3 + MemI3*MemR3#2

	vmovapd	ymm3, [srcreg+r9+d4+d1]			;; MemR6
	vmovapd	ymm9, [srcreg+r10+d4+d1]		;; MemR6#2
	vmulpd	ymm13, ymm3, ymm9			;; MemR6*MemR6#2
	vaddpd	ymm11, ymm11, ymm13			;; R6 = R6 + MemR6*MemR6#2
	vmovapd	ymm6, [srcreg+r10+d4+d1+32]		;; MemI6#2
	vmulpd	ymm13, ymm3, ymm6			;; MemR6*MemI6#2
	vaddpd	ymm1, ymm1, ymm13			;; I6 = I6 + MemR6*MemI6#2
	vmovapd	ymm3, [srcreg+r9+d4+d1+32]		;; MemI6
	vmulpd	ymm13, ymm3, ymm6			;; MemI6*MemI6#2
	vsubpd	ymm11, ymm11, ymm13			;; R6 = R6 - MemI6*MemI6#2
	vmulpd	ymm13, ymm3, ymm9			;; MemI6*MemR6#2
	vaddpd	ymm1, ymm1, ymm13			;; I6 = I6 + MemI6*MemR6#2

	vmovapd	ymm3, [srcreg+r9+d4]			;; MemR5
	vmovapd	ymm9, [srcreg+r10+d4]			;; MemR5#2
	vmulpd	ymm13, ymm3, ymm9			;; MemR5*MemR5#2
	vaddpd	ymm5, ymm5, ymm13			;; R5 = R5 + MemR5*MemR5#2
	vmovapd	ymm6, [srcreg+r10+d4+32]		;; MemI5#2
	vmulpd	ymm13, ymm3, ymm6			;; MemR5*MemI5#2
	vaddpd	ymm12, ymm12, ymm13			;; I5 = I5 + MemR5*MemI5#2
	vmovapd	ymm3, [srcreg+r9+d4+32]			;; MemI5
	vmulpd	ymm13, ymm3, ymm6			;; MemI5*MemI5#2
	vsubpd	ymm5, ymm5, ymm13			;; R5 = R5 - MemI5*MemI5#2
	vmulpd	ymm13, ymm3, ymm9			;; MemI5*MemR5#2
	vaddpd	ymm12, ymm12, ymm13			;; I5 = I5 + MemI5*MemR5#2

	vmovapd	ymm3, [srcreg+r9+d4+d2+d1]		;; MemR8
	vmovapd	ymm9, [srcreg+r10+d4+d2+d1]		;; MemR8#2
	vmulpd	ymm13, ymm3, ymm9			;; MemR8*MemR8#2
	vaddpd	ymm15, ymm15, ymm13			;; R8 = R8 + MemR8*MemR8#2
	vmovapd	ymm6, [srcreg+r10+d4+d2+d1+32]		;; MemI8#2
	vmulpd	ymm13, ymm3, ymm6			;; MemR8*MemI8#2
	vaddpd	ymm14, ymm14, ymm13			;; I8 = I8 + MemR8*MemI8#2
	vmovapd	ymm3, [srcreg+r9+d4+d2+d1+32]		;; MemI8
	vmulpd	ymm13, ymm3, ymm6			;; MemI8*MemI8#2
	vsubpd	ymm15, ymm15, ymm13			;; R8 = R8 - MemI8*MemI8#2
	vmulpd	ymm13, ymm3, ymm9			;; MemI8*MemR8#2
	vaddpd	ymm14, ymm14, ymm13			;; I8 = I8 + MemI8*MemR8#2

	vmovapd	ymm3, [srcreg+r9+d4+d2]			;; MemR7
	vmovapd	ymm9, [srcreg+r10+d4+d2]		;; MemR7#2
	vmulpd	ymm13, ymm3, ymm9			;; MemR7*MemR7#2
	vaddpd	ymm8, ymm8, ymm13			;; R7 = R7 + MemR7*MemR7#2
	vmovapd	ymm6, [srcreg+r10+d4+d2+32]		;; MemI7#2
	vmulpd	ymm13, ymm3, ymm6			;; MemR7*MemI7#2
	vaddpd	ymm0, ymm0, ymm13			;; I7 = I7 + MemR7*MemI7#2
	vmovapd	ymm3, [srcreg+r9+d4+d2+32]		;; MemI7
	vmulpd	ymm13, ymm3, ymm6			;; MemI7*MemI7#2
	vsubpd	ymm8, ymm8, ymm13			;; R7 = R7 - MemI7*MemI7#2
	vmulpd	ymm13, ymm3, ymm9			;; MemI7*MemR7#2
	vaddpd	ymm0, ymm0, ymm13			;; I7 = I7 + MemI7*MemR7#2

	vaddpd	ymm9, ymm4, ymm10		;; R4 + R3 (new R3)		; 65-67
	vsubpd	ymm4, ymm4, ymm10		;; R4 - R3 (new I4)		; 66-68

	vmovapd	ymm10, [srcreg+d1][rbp]		;; MemR2
	ystore	YMM_TMP5, ymm9			;; Temporarily save new R3	; 68
	vmovapd	ymm9, [srcreg+d1+32][rbp]	;; MemI2
	ystore	YMM_TMP6, ymm4			;; Temporarily save new I4	; 69

	vmovapd	ymm6, [srcreg+d1][rbx]		;; R2
	vmovapd	ymm13, [srcreg+d1+32][rbx]	;; I2
	vmulpd	ymm4, ymm6, ymm10		;; R2 * MemR2			; 66-70
	vmulpd	ymm6, ymm6, ymm9		;; R2 * MemI2			; 67-71
	vmulpd	ymm9, ymm13, ymm9		;; I2 * MemI2			; 68-72
	vmulpd	ymm13, ymm13, ymm10		;; I2 * MemR2			; 69-73

	vmovapd	ymm10, [srcreg][rbp]		;; MemR1
	vsubpd	ymm4, ymm4, ymm9		;; R2*MemR2-I2*MemI2 (new R2)	; 73-75
	vmovapd	ymm9, [srcreg+32][rbp]		;; MemI1
	vaddpd	ymm6, ymm6, ymm13		;; R2*MemI2+I2*MemR2 (new I2)	; 74-76

	vsubpd	ymm13, ymm2, ymm7		;; I3 - I4 (new R4)		; 67-69
	vaddpd	ymm2, ymm2, ymm7		;; I3 + I4 (new I3)		; 68-70

	vsubpd	ymm7, ymm12, ymm1		;; I5 - I6 (new I6)		; 69-71
	vaddpd	ymm12, ymm12, ymm1		;; I5 + I6 (new I5)		; 70-72

	vsubpd	ymm1, ymm5, ymm11		;; R5 - R6 (new R6)		; 71-73
	vaddpd	ymm5, ymm5, ymm11		;; R5 + R6 (new R5)		; 72-74

	vmovapd	ymm3, [srcreg][rbx]		;; R1
	vmulpd	ymm11, ymm3, ymm10		;; R1 * MemR1			; 70-74
	vmulpd	ymm3, ymm3, ymm9		;; R1 * MemI1			; 71-75
	vmovsd	Q [srcreg-16], xmm11		;; Save product of sum of FFT values
	vmulpd	ymm9, ymm9, [srcreg+32][rbx]	;; (load I1) * MemI1		; 72-76
	vmulpd	ymm10, ymm10, [srcreg+32][rbx]	;; (load I1) * MemR1		; 73-77
	vmovsd	Q YMM_TMP1[8], xmm9		;; I1

	vsubpd	ymm11, ymm11, ymm9		;; R1*MemR1-I1*MemI1 (new R1)	; 77-79
	vaddpd	ymm3, ymm3, ymm10		;; R1*MemI1+I1*MemR1 (new I1)	; 78-80

	vmovapd	YMM_TMP7, ymm0				;; Save 2 registers
	vmovapd	YMM_TMP8, ymm12

	vmovapd	ymm0, [srcreg+r9+d1]			;; MemR2
	vmovapd	ymm9, [srcreg+r10+d1]			;; MemR2#2
	vmulpd	ymm12, ymm0, ymm9			;; MemR2*MemR2#2
	vaddpd	ymm4, ymm4, ymm12			;; R2 = R2 + MemR2*MemR2#2
	vmovapd	ymm10, [srcreg+r10+d1+32]		;; MemI2#2
	vmulpd	ymm12, ymm0, ymm10			;; MemR2*MemI2#2
	vaddpd	ymm6, ymm6, ymm12			;; I2 = I2 + MemR2*MemI2#2
	vmovapd	ymm0, [srcreg+r9+d1+32]			;; MemI2
	vmulpd	ymm12, ymm0, ymm10			;; MemI2*MemI2#2
	vsubpd	ymm4, ymm4, ymm12			;; R2 = R2 - MemI2*MemI2#2
	vmulpd	ymm12, ymm0, ymm9			;; MemI2*MemR2#2
	vaddpd	ymm6, ymm6, ymm12			;; I2 = I2 + MemI2*MemR2#2

	vmovapd	ymm0, [srcreg+r9]			;; MemR1
	vmovapd	ymm9, [srcreg+r10]			;; MemR1#2
	vmulpd	ymm12, ymm0, ymm9			;; MemR1*MemR1#2
	vaddpd	ymm11, ymm11, ymm12			;; R1 = R1 + MemR1*MemR1#2
	vmovsd	xmm10, [srcreg-16] 		;; Reload R1a = product of sum of FFT values
	vaddsd	xmm10, xmm10, xmm12			;; R1a = R1a + MemR1*MemR1#2
	vmovsd	Q [srcreg-16], xmm10		;; Save product of sum of FFT values
	vmovapd	ymm10, [srcreg+r10+32]			;; MemI1#2
	vmulpd	ymm12, ymm0, ymm10			;; MemR1*MemI1#2
	vaddpd	ymm3, ymm3, ymm12			;; I1 = I1 + MemR1*MemI1#2
	vmovapd	ymm0, [srcreg+r9+32]			;; MemI1
	vmulpd	ymm12, ymm0, ymm10			;; MemI1*MemI1#2
	vmovsd	xmm10, Q YMM_TMP1[8] 		;; R1b
	vaddpd	xmm10, xmm10, xmm12			;; R1b = R1b + MemR1b in MemI1*MemI1#2
	vmovsd	Q YMM_TMP1[8], xmm10		;; R1b
	vsubpd	ymm11, ymm11, ymm12			;; R1 = R1 - MemI1*MemI1#2
	vmulpd	ymm12, ymm0, ymm9			;; MemI1*MemR1#2
	vaddpd	ymm3, ymm3, ymm12			;; I1 = I1 + MemI1*MemR1#2

	vmovapd	ymm0, YMM_TMP7				;; Restore 2 registers
	vmovapd	ymm12, YMM_TMP8
	jmp	back_to_orig

mulsub:	test	r10, 3				;; Check if we need to multiply add-in by FFT(1)
	jz	mulsubhard			;; Yes, do it the hard way

	vsubpd	ymm4, ymm4, [srcreg+r9+d2+d1]		;; R4 = R4 - MemR4
	vsubpd	ymm7, ymm7, [srcreg+r9+d2+d1+32]	;; I4 = I4 - MemI4
	vsubpd	ymm10, ymm10, [srcreg+r9+d2]		;; R3 = R3 - MemR3
	vsubpd	ymm2, ymm2, [srcreg+r9+d2+32]		;; I3 = I3 - MemI3
	vsubpd	ymm11, ymm11, [srcreg+r9+d4+d1]		;; R6 = R6 - MemR6
	vsubpd	ymm1, ymm1, [srcreg+r9+d4+d1+32]	;; I6 = I6 - MemI6
	vsubpd	ymm5, ymm5, [srcreg+r9+d4]		;; R5 = R5 - MemR5
	vsubpd	ymm12, ymm12, [srcreg+r9+d4+32]		;; I5 = I5 - MemI5
	vsubpd	ymm15, ymm15, [srcreg+r9+d4+d2+d1]	;; R8 = R8 - MemR8
	vsubpd	ymm14, ymm14, [srcreg+r9+d4+d2+d1+32]	;; I8 = I8 - MemI8
	vsubpd	ymm8, ymm8, [srcreg+r9+d4+d2]		;; R7 = R7 - MemR7
	vsubpd	ymm0, ymm0, [srcreg+r9+d4+d2+32]	;; I7 = I7 - MemI7

	vaddpd	ymm9, ymm4, ymm10		;; R4 + R3 (new R3)		; 65-67
	vsubpd	ymm4, ymm4, ymm10		;; R4 - R3 (new I4)		; 66-68

	vmovapd	ymm10, [srcreg+d1][rbp]		;; MemR2
	ystore	YMM_TMP5, ymm9			;; Temporarily save new R3	; 68
	vmovapd	ymm9, [srcreg+d1+32][rbp]	;; MemI2
	ystore	YMM_TMP6, ymm4			;; Temporarily save new I4	; 69

	vmovapd	ymm6, [srcreg+d1][rbx]		;; R2
	vmovapd	ymm13, [srcreg+d1+32][rbx]	;; I2
	vmulpd	ymm4, ymm6, ymm10		;; R2 * MemR2			; 66-70
	vmulpd	ymm6, ymm6, ymm9		;; R2 * MemI2			; 67-71
	vmulpd	ymm9, ymm13, ymm9		;; I2 * MemI2			; 68-72
	vmulpd	ymm13, ymm13, ymm10		;; I2 * MemR2			; 69-73

	vmovapd	ymm10, [srcreg][rbp]		;; MemR1
	vsubpd	ymm4, ymm4, ymm9		;; R2*MemR2-I2*MemI2 (new R2)	; 73-75
	vmovapd	ymm9, [srcreg+32][rbp]		;; MemI1
	vaddpd	ymm6, ymm6, ymm13		;; R2*MemI2+I2*MemR2 (new I2)	; 74-76

	vsubpd	ymm13, ymm2, ymm7		;; I3 - I4 (new R4)		; 67-69
	vaddpd	ymm2, ymm2, ymm7		;; I3 + I4 (new I3)		; 68-70

	vsubpd	ymm7, ymm12, ymm1		;; I5 - I6 (new I6)		; 69-71
	vaddpd	ymm12, ymm12, ymm1		;; I5 + I6 (new I5)		; 70-72

	vsubpd	ymm1, ymm5, ymm11		;; R5 - R6 (new R6)		; 71-73
	vaddpd	ymm5, ymm5, ymm11		;; R5 + R6 (new R5)		; 72-74

	vmovapd	ymm3, [srcreg][rbx]		;; R1
	vmulpd	ymm11, ymm3, ymm10		;; R1 * MemR1			; 70-74
	vmulpd	ymm3, ymm3, ymm9		;; R1 * MemI1			; 71-75
	vmovsd	Q [srcreg-16], xmm11		;; Save product of sum of FFT values
	vmulpd	ymm9, ymm9, [srcreg+32][rbx]	;; (load I1) * MemI1		; 72-76
	vmulpd	ymm10, ymm10, [srcreg+32][rbx]	;; (load I1) * MemR1		; 73-77
	vmovsd	Q YMM_TMP1[8], xmm9		;; I1

	vaddpd	ymm3, ymm3, ymm10		;; R1*MemI1+I1*MemR1 (new I1)	; 78-80
	vsubpd	ymm10, ymm11, ymm9		;; R1*MemR1-I1*MemI1 (new R1)	; 77-79

	vsubpd	ymm4, ymm4, [srcreg+r9+d1]		;; R2 = R2 - MemR2
	vsubpd	ymm6, ymm6, [srcreg+r9+d1+32]		;; I2 = I2 - MemI2
	vsubsd	xmm11, xmm11, [srcreg+r9]		;; R1a = R1a - MemR1a
	vmovsd	Q [srcreg-16], xmm11		;; Save product of sum of FFT values
	vsubpd	ymm11, ymm10, [srcreg+r9]		;; R1 = R1 - MemR1 (lower word is irrelevant)
	vmovapd	ymm10, [srcreg+r9+32]			;; MemI1
	vsubsd	xmm9, xmm9, xmm10			;; R1b = R1b - MemR1b in MemI1
	vsubpd	ymm3, ymm3, ymm10			;; I1 = I1 - MemI1 (lower word is irrelevant)
	vmovsd	Q YMM_TMP1[8], xmm9			;; Save R1b
	jmp	back_to_orig

mulsubhard:
	vmovapd	ymm3, [srcreg+r9+d2+d1]			;; MemR4
	vmovapd	ymm9, [srcreg+r10+d2+d1]		;; MemR4#2
	vmulpd	ymm13, ymm3, ymm9			;; MemR4*MemR4#2
	vsubpd	ymm4, ymm4, ymm13			;; R4 = R4 - MemR4*MemR4#2
	vmovapd	ymm6, [srcreg+r10+d2+d1+32]		;; MemI4#2
	vmulpd	ymm13, ymm3, ymm6			;; MemR4*MemI4#2
	vsubpd	ymm7, ymm7, ymm13			;; I4 = I4 - MemR4*MemI4#2
	vmovapd	ymm3, [srcreg+r9+d2+d1+32]		;; MemI4
	vmulpd	ymm13, ymm3, ymm6			;; MemI4*MemI4#2
	vaddpd	ymm4, ymm4, ymm13			;; R4 = R4 + MemI4*MemI4#2
	vmulpd	ymm13, ymm3, ymm9			;; MemI4*MemR4#2
	vsubpd	ymm7, ymm7, ymm13			;; I4 = I4 - MemI4*MemR4#2

	vmovapd	ymm3, [srcreg+r9+d2]			;; MemR3
	vmovapd	ymm9, [srcreg+r10+d2]			;; MemR3#2
	vmulpd	ymm13, ymm3, ymm9			;; MemR3*MemR3#2
	vsubpd	ymm10, ymm10, ymm13			;; R3 = R3 - MemR3*MemR3#2
	vmovapd	ymm6, [srcreg+r10+d2+32]		;; MemI3#2
	vmulpd	ymm13, ymm3, ymm6			;; MemR3*MemI3#2
	vsubpd	ymm2, ymm2, ymm13			;; I3 = I3 - MemR3*MemI3#2
	vmovapd	ymm3, [srcreg+r9+d2+32]			;; MemI3
	vmulpd	ymm13, ymm3, ymm6			;; MemI3*MemI3#2
	vaddpd	ymm10, ymm10, ymm13			;; R3 = R3 + MemI3*MemI3#2
	vmulpd	ymm13, ymm3, ymm9			;; MemI3*MemR3#2
	vsubpd	ymm2, ymm2, ymm13			;; I3 = I3 - MemI3*MemR3#2

	vmovapd	ymm3, [srcreg+r9+d4+d1]			;; MemR6
	vmovapd	ymm9, [srcreg+r10+d4+d1]		;; MemR6#2
	vmulpd	ymm13, ymm3, ymm9			;; MemR6*MemR6#2
	vsubpd	ymm11, ymm11, ymm13			;; R6 = R6 - MemR6*MemR6#2
	vmovapd	ymm6, [srcreg+r10+d4+d1+32]		;; MemI6#2
	vmulpd	ymm13, ymm3, ymm6			;; MemR6*MemI6#2
	vsubpd	ymm1, ymm1, ymm13			;; I6 = I6 - MemR6*MemI6#2
	vmovapd	ymm3, [srcreg+r9+d4+d1+32]		;; MemI6
	vmulpd	ymm13, ymm3, ymm6			;; MemI6*MemI6#2
	vaddpd	ymm11, ymm11, ymm13			;; R6 = R6 + MemI6*MemI6#2
	vmulpd	ymm13, ymm3, ymm9			;; MemI6*MemR6#2
	vsubpd	ymm1, ymm1, ymm13			;; I6 = I6 - MemI6*MemR6#2

	vmovapd	ymm3, [srcreg+r9+d4]			;; MemR5
	vmovapd	ymm9, [srcreg+r10+d4]			;; MemR5#2
	vmulpd	ymm13, ymm3, ymm9			;; MemR5*MemR5#2
	vsubpd	ymm5, ymm5, ymm13			;; R5 = R5 - MemR5*MemR5#2
	vmovapd	ymm6, [srcreg+r10+d4+32]		;; MemI5#2
	vmulpd	ymm13, ymm3, ymm6			;; MemR5*MemI5#2
	vsubpd	ymm12, ymm12, ymm13			;; I5 = I5 - MemR5*MemI5#2
	vmovapd	ymm3, [srcreg+r9+d4+32]			;; MemI5
	vmulpd	ymm13, ymm3, ymm6			;; MemI5*MemI5#2
	vaddpd	ymm5, ymm5, ymm13			;; R5 = R5 + MemI5*MemI5#2
	vmulpd	ymm13, ymm3, ymm9			;; MemI5*MemR5#2
	vsubpd	ymm12, ymm12, ymm13			;; I5 = I5 - MemI5*MemR5#2

	vmovapd	ymm3, [srcreg+r9+d4+d2+d1]		;; MemR8
	vmovapd	ymm9, [srcreg+r10+d4+d2+d1]		;; MemR8#2
	vmulpd	ymm13, ymm3, ymm9			;; MemR8*MemR8#2
	vsubpd	ymm15, ymm15, ymm13			;; R8 = R8 - MemR8*MemR8#2
	vmovapd	ymm6, [srcreg+r10+d4+d2+d1+32]		;; MemI8#2
	vmulpd	ymm13, ymm3, ymm6			;; MemR8*MemI8#2
	vsubpd	ymm14, ymm14, ymm13			;; I8 = I8 - MemR8*MemI8#2
	vmovapd	ymm3, [srcreg+r9+d4+d2+d1+32]		;; MemI8
	vmulpd	ymm13, ymm3, ymm6			;; MemI8*MemI8#2
	vaddpd	ymm15, ymm15, ymm13			;; R8 = R8 + MemI8*MemI8#2
	vmulpd	ymm13, ymm3, ymm9			;; MemI8*MemR8#2
	vsubpd	ymm14, ymm14, ymm13			;; I8 = I8 - MemI8*MemR8#2

	vmovapd	ymm3, [srcreg+r9+d4+d2]			;; MemR7
	vmovapd	ymm9, [srcreg+r10+d4+d2]		;; MemR7#2
	vmulpd	ymm13, ymm3, ymm9			;; MemR7*MemR7#2
	vsubpd	ymm8, ymm8, ymm13			;; R7 = R7 - MemR7*MemR7#2
	vmovapd	ymm6, [srcreg+r10+d4+d2+32]		;; MemI7#2
	vmulpd	ymm13, ymm3, ymm6			;; MemR7*MemI7#2
	vsubpd	ymm0, ymm0, ymm13			;; I7 = I7 - MemR7*MemI7#2
	vmovapd	ymm3, [srcreg+r9+d4+d2+32]		;; MemI7
	vmulpd	ymm13, ymm3, ymm6			;; MemI7*MemI7#2
	vaddpd	ymm8, ymm8, ymm13			;; R7 = R7 + MemI7*MemI7#2
	vmulpd	ymm13, ymm3, ymm9			;; MemI7*MemR7#2
	vsubpd	ymm0, ymm0, ymm13			;; I7 = I7 - MemI7*MemR7#2

	vaddpd	ymm9, ymm4, ymm10		;; R4 + R3 (new R3)		; 65-67
	vsubpd	ymm4, ymm4, ymm10		;; R4 - R3 (new I4)		; 66-68

	vmovapd	ymm10, [srcreg+d1][rbp]		;; MemR2
	ystore	YMM_TMP5, ymm9			;; Temporarily save new R3	; 68
	vmovapd	ymm9, [srcreg+d1+32][rbp]	;; MemI2
	ystore	YMM_TMP6, ymm4			;; Temporarily save new I4	; 69

	vmovapd	ymm6, [srcreg+d1][rbx]		;; R2
	vmovapd	ymm13, [srcreg+d1+32][rbx]	;; I2
	vmulpd	ymm4, ymm6, ymm10		;; R2 * MemR2			; 66-70
	vmulpd	ymm6, ymm6, ymm9		;; R2 * MemI2			; 67-71
	vmulpd	ymm9, ymm13, ymm9		;; I2 * MemI2			; 68-72
	vmulpd	ymm13, ymm13, ymm10		;; I2 * MemR2			; 69-73

	vmovapd	ymm10, [srcreg][rbp]		;; MemR1
	vsubpd	ymm4, ymm4, ymm9		;; R2*MemR2-I2*MemI2 (new R2)	; 73-75
	vmovapd	ymm9, [srcreg+32][rbp]		;; MemI1
	vaddpd	ymm6, ymm6, ymm13		;; R2*MemI2+I2*MemR2 (new I2)	; 74-76

	vsubpd	ymm13, ymm2, ymm7		;; I3 - I4 (new R4)		; 67-69
	vaddpd	ymm2, ymm2, ymm7		;; I3 + I4 (new I3)		; 68-70

	vsubpd	ymm7, ymm12, ymm1		;; I5 - I6 (new I6)		; 69-71
	vaddpd	ymm12, ymm12, ymm1		;; I5 + I6 (new I5)		; 70-72

	vsubpd	ymm1, ymm5, ymm11		;; R5 - R6 (new R6)		; 71-73
	vaddpd	ymm5, ymm5, ymm11		;; R5 + R6 (new R5)		; 72-74

	vmovapd	ymm3, [srcreg][rbx]		;; R1
	vmulpd	ymm11, ymm3, ymm10		;; R1 * MemR1			; 70-74
	vmulpd	ymm3, ymm3, ymm9		;; R1 * MemI1			; 71-75
	vmovsd	Q [srcreg-16], xmm11		;; Save product of sum of FFT values
	vmulpd	ymm9, ymm9, [srcreg+32][rbx]	;; (load I1) * MemI1		; 72-76
	vmulpd	ymm10, ymm10, [srcreg+32][rbx]	;; (load I1) * MemR1		; 73-77
	vmovsd	Q YMM_TMP1[8], xmm9		;; I1

	vsubpd	ymm11, ymm11, ymm9		;; R1*MemR1-I1*MemI1 (new R1)	; 77-79
	vaddpd	ymm3, ymm3, ymm10		;; R1*MemI1+I1*MemR1 (new I1)	; 78-80

	vmovapd	YMM_TMP7, ymm0				;; Save 2 registers
	vmovapd	YMM_TMP8, ymm12

	vmovapd	ymm0, [srcreg+r9+d1]			;; MemR2
	vmovapd	ymm9, [srcreg+r10+d1]			;; MemR2#2
	vmulpd	ymm12, ymm0, ymm9			;; MemR2*MemR2#2
	vsubpd	ymm4, ymm4, ymm12			;; R2 = R2 - MemR2*MemR2#2
	vmovapd	ymm10, [srcreg+r10+d1+32]		;; MemI2#2
	vmulpd	ymm12, ymm0, ymm10			;; MemR2*MemI2#2
	vsubpd	ymm6, ymm6, ymm12			;; I2 = I2 - MemR2*MemI2#2
	vmovapd	ymm0, [srcreg+r9+d1+32]			;; MemI2
	vmulpd	ymm12, ymm0, ymm10			;; MemI2*MemI2#2
	vaddpd	ymm4, ymm4, ymm12			;; R2 = R2 + MemI2*MemI2#2
	vmulpd	ymm12, ymm0, ymm9			;; MemI2*MemR2#2
	vsubpd	ymm6, ymm6, ymm12			;; I2 = I2 - MemI2*MemR2#2

	vmovapd	ymm0, [srcreg+r9]			;; MemR1
	vmovapd	ymm9, [srcreg+r10]			;; MemR1#2
	vmulpd	ymm12, ymm0, ymm9			;; MemR1*MemR1#2
	vsubpd	ymm11, ymm11, ymm12			;; R1 = R1 - MemR1*MemR1#2
	vmovsd	xmm10, [srcreg-16] 		;; Reload R1a = product of sum of FFT values
	vsubsd	xmm10, xmm10, xmm12			;; R1a = R1a - MemR1*MemR1#2
	vmovsd	Q [srcreg-16], xmm10		;; Save product of sum of FFT values
	vmovapd	ymm10, [srcreg+r10+32]			;; MemI1#2
	vmulpd	ymm12, ymm0, ymm10			;; MemR1*MemI1#2
	vsubpd	ymm3, ymm3, ymm12			;; I1 = I1 - MemR1*MemI1#2
	vmovapd	ymm0, [srcreg+r9+32]			;; MemI1
	vmulpd	ymm12, ymm0, ymm10			;; MemI1*MemI1#2
	vmovsd	xmm10, Q YMM_TMP1[8] 		;; R1b
	vsubpd	xmm10, xmm10, xmm12			;; R1b = R1b - MemR1b in MemI1*MemI1#2
	vmovsd	Q YMM_TMP1[8], xmm10		;; R1b
	vaddpd	ymm11, ymm11, ymm12			;; R1 = R1 + MemI1*MemI1#2
	vmulpd	ymm12, ymm0, ymm9			;; MemI1*MemR1#2
	vsubpd	ymm3, ymm3, ymm12			;; I1 = I1 - MemI1*MemR1#2

	vmovapd	ymm0, YMM_TMP7				;; Restore 2 registers
	vmovapd	ymm12, YMM_TMP8
	jmp	back_to_orig

orig:	vmovapd	ymm3, [srcreg+d2+d1][rbp]	;; MemR4
	vmovapd	ymm9, [srcreg+d2+d1+32][rbp]	;; MemI4
	vmovapd	ymm7, [srcreg+d2+d1][rbx]	;; R4
	vmovapd	ymm10, [srcreg+d2+d1+32][rbx]	;; I4
	vmulpd	ymm4, ymm7, ymm3		;; R4 * MemR4			; 32-36
	vmulpd	ymm7, ymm7, ymm9		;; R4 * MemI4			; 33-37
	vmulpd	ymm9, ymm10, ymm9		;; I4 * MemI4			; 34-38
	vmulpd	ymm10, ymm10, ymm3		;; I4 * MemR4			; 35-39

	vmovapd	ymm3, [srcreg+d2][rbp]		;; MemR3
	vsubpd	ymm4, ymm4, ymm9		;; R4*MemR4-I4*MemI4 (new R4)	; 51-53
	vmovapd	ymm9, [srcreg+d2+32][rbp]	;; MemI3
	vaddpd	ymm7, ymm7, ymm10		;; R4*MemI4+I4*MemR4 (new I4)	; 52-54

	vmovapd	ymm2, [srcreg+d2][rbx]		;; R3
	vmovapd	ymm11, [srcreg+d2+32][rbx]	;; I3
	vmulpd	ymm10, ymm2, ymm3		;; R3 * MemR3			; 36-40
	vmulpd	ymm2, ymm2, ymm9		;; R3 * MemI3			; 37-41
	vmulpd	ymm9, ymm11, ymm9		;; I3 * MemI3			; 38-42
	vmulpd	ymm11, ymm11, ymm3		;; I3 * MemR3			; 39-43

	vmovapd	ymm3, [srcreg+d4+d1][rbp]	;; MemR6
	vsubpd	ymm10, ymm10, ymm9		;; R3*MemR3-I3*MemI3 (new R3)	; 53-55
	vmovapd	ymm9, [srcreg+d4+d1+32][rbp]	;; MemI6
	vaddpd	ymm2, ymm2, ymm11		;; R3*MemI3+I3*MemR3 (new I3)	; 54-56

	vmovapd	ymm1, [srcreg+d4+d1][rbx]	;; R6
	vmovapd	ymm5, [srcreg+d4+d1+32][rbx]	;; I6
	vmulpd	ymm11, ymm1, ymm3		;; R6 * MemR6			; 42-46
	vmulpd	ymm1, ymm1, ymm9		;; R6 * MemI6			; 43-47
	vmulpd	ymm9, ymm5, ymm9		;; I6 * MemI6			; 46-50
	vmulpd	ymm5, ymm5, ymm3		;; I6 * MemR6			; 47-51

	vmovapd	ymm3, [srcreg+d4][rbp]		;; MemR5
	vsubpd	ymm11, ymm11, ymm9		;; R6*MemR6-I6*MemI6 (new R6)	; 55-57
	vmovapd	ymm9, [srcreg+d4+32][rbp]	;; MemI5
	vaddpd	ymm1, ymm1, ymm5		;; R6*MemI6+I6*MemR6 (new I6)	; 56-58

	vmovapd	ymm12, [srcreg+d4][rbx]		;; R5
	vmovapd	ymm15, [srcreg+d4+32][rbx]	;; I5
	vmulpd	ymm5, ymm12, ymm3		;; R5 * MemR5			; 44-48
	vmulpd	ymm12, ymm12, ymm9		;; R5 * MemI5			; 45-49
	vmulpd	ymm9, ymm15, ymm9		;; I5 * MemI5			; 48-52
	vmulpd	ymm15, ymm15, ymm3		;; I5 * MemR5			; 49-53

	vmovapd	ymm3, [srcreg+d4+d2+d1][rbp]	;; MemR8
	vsubpd	ymm5, ymm5, ymm9		;; R5*MemR5-I5*MemI5 (new R5)	; 57-59
	vmovapd	ymm9, [srcreg+d4+d2+d1+32][rbp]	;; MemI8
	vaddpd	ymm12, ymm12, ymm15		;; R5*MemI5+I5*MemR5 (new I5)	; 58-60

	vmovapd	ymm14, [srcreg+d4+d2+d1][rbx]	;; R8
	vmovapd	ymm8, [srcreg+d4+d2+d1+32][rbx]	;; I8
	vmulpd	ymm15, ymm14, ymm3		;; R8 * MemR8			; 50-54
	vmulpd	ymm14, ymm14, ymm9		;; R8 * MemI8			; 51-55
	vmulpd	ymm9, ymm8, ymm9		;; I8 * MemI8			; 52-56
	vmulpd	ymm8, ymm8, ymm3		;; I8 * MemR8			; 53-57

	vmovapd	ymm3, [srcreg+d4+d2][rbp]	;; MemR7
	vsubpd	ymm15, ymm15, ymm9		;; R8*MemR8-I8*MemI8 (new R8)	; 59-61
	vmovapd	ymm9, [srcreg+d4+d2+32][rbp]	;; MemI7
	vaddpd	ymm14, ymm14, ymm8		;; R8*MemI8+I8*MemR8 (new I8)	; 60-62

	vmovapd	ymm0, [srcreg+d4+d2][rbx]	;; R7
	vmovapd	ymm6, [srcreg+d4+d2+32][rbx]	;; I7
	vmulpd	ymm8, ymm0, ymm3		;; R7 * MemR7			; 54-58
	vmulpd	ymm0, ymm0, ymm9		;; R7 * MemI7			; 55-59
	vmulpd	ymm9, ymm6, ymm9		;; I7 * MemI7			; 56-60
	vmulpd	ymm6, ymm6, ymm3		;; I7 * MemR7			; 57-61

	vsubpd	ymm8, ymm8, ymm9		;; R7*MemR7-I7*MemI7 (new R7)	; 61-63
	vaddpd	ymm0, ymm0, ymm6		;; R7*MemI7+I7*MemR7 (new I7)	; 62-64

	vaddpd	ymm9, ymm4, ymm10		;; R4 + R3 (new R3)		; 65-67
	vsubpd	ymm4, ymm4, ymm10		;; R4 - R3 (new I4)		; 66-68

	vmovapd	ymm10, [srcreg+d1][rbp]		;; MemR2
	ystore	YMM_TMP5, ymm9			;; Temporarily save new R3	; 68
	vmovapd	ymm9, [srcreg+d1+32][rbp]	;; MemI2
	ystore	YMM_TMP6, ymm4			;; Temporarily save new I4	; 69

	vmovapd	ymm6, [srcreg+d1][rbx]		;; R2
	vmovapd	ymm13, [srcreg+d1+32][rbx]	;; I2
	vmulpd	ymm4, ymm6, ymm10		;; R2 * MemR2			; 66-70
	vmulpd	ymm6, ymm6, ymm9		;; R2 * MemI2			; 67-71
	vmulpd	ymm9, ymm13, ymm9		;; I2 * MemI2			; 68-72
	vmulpd	ymm13, ymm13, ymm10		;; I2 * MemR2			; 69-73

	vmovapd	ymm10, [srcreg][rbp]		;; MemR1
	vsubpd	ymm4, ymm4, ymm9		;; R2*MemR2-I2*MemI2 (new R2)	; 73-75
	vmovapd	ymm9, [srcreg+32][rbp]		;; MemI1
	vaddpd	ymm6, ymm6, ymm13		;; R2*MemI2+I2*MemR2 (new I2)	; 74-76

	vsubpd	ymm13, ymm2, ymm7		;; I3 - I4 (new R4)		; 67-69
	vaddpd	ymm2, ymm2, ymm7		;; I3 + I4 (new I3)		; 68-70

	vsubpd	ymm7, ymm12, ymm1		;; I5 - I6 (new I6)		; 69-71
	vaddpd	ymm12, ymm12, ymm1		;; I5 + I6 (new I5)		; 70-72

	vsubpd	ymm1, ymm5, ymm11		;; R5 - R6 (new R6)		; 71-73
	vaddpd	ymm5, ymm5, ymm11		;; R5 + R6 (new R5)		; 72-74

	vmovapd	ymm3, [srcreg][rbx]		;; R1
	vmulpd	ymm11, ymm3, ymm10		;; R1 * MemR1			; 70-74
	vmulpd	ymm3, ymm3, ymm9		;; R1 * MemI1			; 71-75
	vmovsd	Q [srcreg-16], xmm11		;; Save product of sum of FFT values
	vmulpd	ymm9, ymm9, [srcreg+32][rbx]	;; (load I1) * MemI1		; 72-76
	vmulpd	ymm10, ymm10, [srcreg+32][rbx]	;; (load I1) * MemR1		; 73-77
	vmovsd	Q YMM_TMP1[8], xmm9		;; I1

	vsubpd	ymm11, ymm11, ymm9		;; R1*MemR1-I1*MemI1 (new R1)	; 77-79
	vaddpd	ymm3, ymm3, ymm10		;; R1*MemI1+I1*MemR1 (new I1)	; 78-80

back_to_orig:
	vsubpd	ymm9, ymm0, ymm14		;; I7 - I8 (new R8)		; 75-77
	vaddpd	ymm0, ymm0, ymm14		;; I7 + I8 (new I7)		; 76-78
	vmovsd	Q YMM_TMP4[8], xmm0		;; I7+I8
	vmovsd	Q YMM_TMP4[24], xmm9		;; I7-I8

	vsubpd	ymm14, ymm15, ymm8		;; R8 - R7 (new I8)		; 79-81
	vaddpd	ymm15, ymm15, ymm8		;; R8 + R7 (new R7)		; 80-82
	vmovsd	Q YMM_TMP4[0], xmm15		;; R8+R7
	vmovsd	Q YMM_TMP4[16], xmm14		;; R8-R7

	vmovsd	Q YMM_TMP1[24], xmm6		;; I2
	vsubpd	ymm8, ymm3, ymm6		;; I1 - I2 (new I2)		; 81-83
	vaddpd	ymm3, ymm3, ymm6		;; I1 + I2 (new I1)		; 82-84

	vmovsd	Q YMM_TMP1[16], xmm4		;; R2
	vsubpd	ymm6, ymm11, ymm4		;; R1 - R2 (new R2)		; 83-85
	vaddpd	ymm11, ymm11, ymm4		;; R1 + R2 (new R1)		; 84-86

	;; multiply R6/I6 by SQRTHALF - i*SQRTHALF
	vmovsd	Q YMM_TMP3[24], xmm7		;; I5-I6
	vmovsd	Q YMM_TMP3[16], xmm1		;; R5-R6
	vsubpd	ymm4, ymm7, ymm1		;; I6 = I6 - R6			; 85-87
	vaddpd	ymm1, ymm1, ymm7		;; R6 = R6 + I6			; 86-88

	;; multiply R8/I8 by SQRTHALF - i*SQRTHALF
	vsubpd	ymm7, ymm14, ymm9		;; I8 = I8 - R8			; 87-89
	vaddpd	ymm9, ymm9, ymm14		;; R8 = R8 + I8			; 88-90

	vmovapd	ymm14, YMM_SQRTHALF
	vmulpd	ymm4, ymm4, ymm14		;; I6 * SQRTHALF		; 88-92
	vmulpd	ymm1, ymm1, ymm14		;; R6 * SQRTHALF		; 89-93
	vmulpd	ymm7, ymm7, ymm14		;; I8 * SQRTHALF		; 90-94
	vmulpd	ymm9, ymm9, ymm14		;; R8 * SQRTHALF		; 91-95

	vmovsd	Q YMM_TMP2[24], xmm13		;; I3-I4
	vaddpd	ymm14, ymm6, ymm13		;; R2 + R4 (newer R2)		; 89-91
	vsubpd	ymm6, ymm6, ymm13		;; R2 - R4 (newer R4)		; 90-92

	vmovapd	ymm10, YMM_TMP6			;; Reload new I4
	vmovsd	Q YMM_TMP2[0], xmm9		;; R3+R4
	vmovsd	Q YMM_TMP2[16], xmm4		;; R4-R3

	vaddpd	ymm13, ymm8, ymm10		;; I2 + I4 (newer I2)		; 91-93
	vsubpd	ymm8, ymm8, ymm10		;; I2 - I4 (newer I4)		; 92-94

	vmovsd	Q YMM_TMP3[8], xmm12		;; I5+I6
	vsubpd	ymm10, ymm12, ymm0		;; I5 - I7 (newer R7)		; 93-95
	vaddpd	ymm12, ymm12, ymm0		;; I5 + I7 (newer I5)		; 94-96

	vmovsd	Q YMM_TMP2[8], xmm2		;; I3+I4
	vsubpd	ymm0, ymm3, ymm2		;; I1 - I3 (newer I3)		; 95-97
	vaddpd	ymm3, ymm3, ymm2		;; I1 + I3 (newer I1)		; 96-98

	vsubpd	ymm2, ymm4, ymm7		;; I6 - I8 (newer R8)		; 97-99
	vaddpd	ymm4, ymm4, ymm7		;; I6 + I8 (newer I6)		; 98-100

	vsubpd	ymm7, ymm9, ymm1		;; R8 - R6 (newer I8)		; 99-101
	vaddpd	ymm9, ymm9, ymm1		;; R8 + R6 (newer R6)		; 100-102

	vsubpd	ymm1, ymm6, ymm2		;; R4 - R8 (final R8)		; 101-103
	vaddpd	ymm6, ymm6, ymm2		;; R4 + R8 (final R4)		; 102-104

	vsubpd	ymm2, ymm13, ymm4		;; I2 - I6 (final I6)		; 103-105
	vaddpd	ymm13, ymm13, ymm4		;; I2 + I6 (final I2)		; 104-106
	ystore	[srcreg+d4+d2+d1], ymm1		;; Save R8			; 104

	vsubpd	ymm4, ymm8, ymm7		;; I4 - I8 (final I8)		; 105-107
	vaddpd	ymm8, ymm8, ymm7		;; I4 + I8 (final I4)		; 106-108
	ystore	[srcreg+d2+d1], ymm6		;; Save R4			; 105
	ystore	[srcreg+d4+d1+32], ymm2		;; Save I6			; 106

	vsubpd	ymm7, ymm14, ymm9		;; R2 - R6 (final R6)		; 107-109
	vaddpd	ymm14, ymm14, ymm9		;; R2 + R6 (final R2)		; 108-110
	ystore	[srcreg+d1+32], ymm13		;; Save I2			; 107
	ystore	[srcreg+d4+d2+d1+32], ymm4	;; Save I8			; 108

	vmovapd	ymm1, YMM_TMP5			;; Reload new R3
	vsubpd	ymm9, ymm11, ymm1		;; R1 - R3 (newer R3)		; 109-111
	vaddpd	ymm11, ymm11, ymm1		;; R1 + R3 (newer R1)		; 110-112
	ystore	[srcreg+d2+d1+32], ymm8		;; Save I4			; 109
	ystore	[srcreg+d4+d1], ymm7		;; Save R6			; 110

	vmovsd	Q YMM_TMP3[0], xmm5		;; R5+R6
	vaddpd	ymm1, ymm15, ymm5		;; R7 + R5 (newer R5)		; 111-113
	vsubpd	ymm15, ymm15, ymm5		;; R7 - R5 (newer I7)		; 112-114
	ystore	[srcreg+d1], ymm14		;; Save R2			; 111

	vsubpd	ymm5, ymm3, ymm12		;; I1 - I5 (final I5)		; 113-115
	vaddpd	ymm3, ymm3, ymm12		;; I1 + I5 (final I1)		; 114-116

	vsubpd	ymm12, ymm9, ymm10		;; R3 - R7 (final R7)		; 115-117
	vaddpd	ymm9, ymm9, ymm10		;; R3 + R7 (final R3)		; 116-118
	ystore	[srcreg+d4+32], ymm5		;; Save I5			; 116

	vsubpd	ymm10, ymm0, ymm15		;; I3 - I7 (final I7)		; 117-119
	vaddpd	ymm0, ymm0, ymm15		;; I3 + I7 (final I3)		; 118-120
	ystore	[srcreg+32], ymm3		;; Save I1			; 117
	ystore	[srcreg+d4+d2], ymm12		;; Save R7			; 118

	vsubpd	ymm15, ymm11, ymm1		;; R1 - R5 (final R5)		; 119-121
	vaddpd	ymm11, ymm11, ymm1		;; R1 + R5 (final R1)		; 120-122
	ystore	[srcreg+d2], ymm9		;; Save R3			; 119
	ystore	[srcreg+d4+d2+32], ymm10	;; Save I7			; 120

	ystore	[srcreg+d2+32], ymm0		;; Save I3			; 121
	ystore	[srcreg+d4], ymm15		;; Save R5			; 122
	ystore	[srcreg], ymm11			;; Save R1			; 123

	;; Do 16-reals from YMM_TMPs

	;; Level 4

	vmovsd	xmm0, [srcreg-16]		;; R1
	vmovsd	xmm1, Q YMM_TMP1[8]		;; R2

	vsubsd	xmm0, xmm0, xmm1		;; R1 - R2 (new R2)
	vmulsd	xmm0, xmm0, Q YMM_HALF		;; Mul R2 by HALF
	vaddsd	xmm1, xmm1, xmm0		;; R1 + R2 (new R1)

						;; R3/I3 morphs into new R3/R4
	vmovsd	xmm9, Q YMM_TMP1[24]		;; I3 (new R4)

	vmovsd	xmm2, Q YMM_TMP6[0]		;; R6 - R5 (new negR6)
	vmovsd	xmm3, Q YMM_TMP2[24] 		;; I5 - I6 (new I6)

	vmovsd	xmm4, Q YMM_TMP3[16]		;; R9 - R10 (new R10)
	vmovsd	xmm5, Q YMM_TMP3[24]		;; I9 - I10 (new I10)

	vmovsd	xmm6, Q YMM_TMP4[24]		;; I11 - I12 (new R12)
	vmovsd	xmm7, Q YMM_TMP4[16]		;; R12 - R11 (new I12)

	;; Postmultipliers for level 4

						;; mul R6/I6 by w^2 = .707 - .707i
	vsubsd	xmm8, xmm3, xmm2		;; R6 = I6 - negR6
	vaddsd	xmm3, xmm3, xmm2		;; I6 = I6 + negR6
	vmovsd	xmm15, Q YMM_SQRTHALF
	vmulsd	xmm8, xmm8, xmm15		;; R6 = R6 * SQRTHALF
	vmulsd	xmm3, xmm3, xmm15		;; I6 = I6 * SQRTHALF

	;; Even level 3

	vaddsd	xmm2, xmm4, xmm6		;; R10 + R12 (newer R10)
	vsubsd	xmm4, xmm4, xmm6		;; R10 - R12 (newer R12)

	vaddsd	xmm6, xmm5, xmm7		;; I10 + I12 (newer I10)
	vsubsd	xmm5, xmm5, xmm7		;; I10 - I12 (newer I12)

	vaddsd	xmm7, xmm0, xmm9		;; R2 + R4 (newer R2)
	vsubsd	xmm0, xmm0, xmm9		;; R2 - R4 (newer R4)

	;; Postmultipliers for even level 3

						;; mul R10/I10 by w^1 = .924 - .383i
	vmovsd	xmm14, Q YMM_P924
	vmovsd	xmm13, Q YMM_P383
	vmulsd	xmm9, xmm2, xmm14
	vmulsd	xmm10, xmm6, xmm13
	vaddsd	xmm9, xmm9, xmm10		;; Twiddled R10
	vmulsd	xmm2, xmm2, xmm13
	vmulsd	xmm6, xmm6, xmm14
	vsubsd	xmm2, xmm6, xmm2		;; Twiddled I10

						;; mul R12/I12 by w^3 = .383 - .924i
	vmulsd	xmm6, xmm4, xmm13
	vmulsd	xmm10, xmm5, xmm14
	vaddsd	xmm6, xmm6, xmm10		;; Twiddled R12
	vmulsd	xmm4, xmm4, xmm14
	vmulsd	xmm5, xmm5, xmm13
	vsubsd	xmm4, xmm5, xmm4		;; Twiddled I12

						;; R6/I6 morph into newer R6/R8

	;; Even level 2

	vaddsd	xmm5, xmm7, xmm8		;; R2 + R6 (newest R2)
	vsubsd	xmm7, xmm7, xmm8		;; R2 - R6 (newest R6)

	vaddsd	xmm8, xmm0, xmm3		;; R4 + R8 (newest R4)
	vsubsd	xmm0, xmm0, xmm3		;; R4 - R8 (newest R8)

						;; R10/I10 morphs into newest R10/R14
						;; R12/I12 morphs into newest R12/R16

	;; Even level 1

	vaddsd	xmm3, xmm5, xmm9		;; R2 + R10 (final R2)
	vsubsd	xmm5, xmm5, xmm9		;; R2 - R10 (final R10)

	vaddsd	xmm9, xmm7, xmm2		;; R6 + R14 (final R6)
	vsubsd	xmm7, xmm7, xmm2		;; R6 - R14 (final R14)

	vaddsd	xmm2, xmm8, xmm6		;; R4 + R12 (final R4)
	vsubsd	xmm8, xmm8, xmm6		;; R4 - R12 (final R12)

	vaddsd	xmm6, xmm0, xmm4		;; R8 + R16 (final R8)
	vsubsd	xmm0, xmm0, xmm4		;; R8 - R16 (final R16)

	vmovsd	Q [srcreg+d1], xmm3		;; R2
	vmovsd	Q [srcreg+d1+32], xmm5		;; R10
	vmovsd	Q [srcreg+d4+d1], xmm9		;; R6
	vmovsd	Q [srcreg+d4+d1+32], xmm7	;; R14
	vmovsd	Q [srcreg+d2+d1], xmm2		;; R4
	vmovsd	Q [srcreg+d2+d1+32], xmm8	;; R12
	vmovsd	Q [srcreg+d4+d2+d1], xmm6	;; R8
	vmovsd	Q [srcreg+d4+d2+d1+32], xmm0	;; R16

	;; Odd level 3

	vmovsd	xmm4, Q YMM_TMP3[0] 		;; R9 + R10 (new R9)
	vmovsd	xmm5, Q YMM_TMP3[8] 		;; I9 + I10 (new I9)

	vmovsd	xmm6, Q YMM_TMP4[0]		;; R12 + R11 (new R11)
	vmovsd	xmm7, Q YMM_TMP4[8]		;; I11 + I12 (new I11)

	vmovsd	xmm0, Q YMM_TMP1[16]		;; R3 (new R3)

	vaddsd	xmm8, xmm4, xmm6		;; R9 + R11 (newer R9)
	vsubsd	xmm4, xmm4, xmm6		;; R9 - R11 (newer R11)

	vaddsd	xmm6, xmm5, xmm7		;; I9 + I11 (newer I9)
	vsubsd	xmm5, xmm5, xmm7		;; I9 - I11 (newer I11)

	vaddsd	xmm7, xmm1, xmm0		;; R1 + R3 (newer R1)
	vsubsd	xmm1, xmm1, xmm0		;; R1 - R3 (newer R3)

	vmovsd	xmm2, Q YMM_TMP5[0]		;; R5 + R6 (new R5)
	vmovsd	xmm3, Q YMM_TMP2[8]		;; I5 + I6 (new I5)

						;; R5/I5 morphs into newer R5/R7


	;; Postmultipliers for odd level 3

						;; mul R11/I11 by SQRTHALF - i*SQRTHALF
	vaddsd	xmm0, xmm5, xmm4		;; R11 = I11 + R11
	vsubsd	xmm5, xmm5, xmm4		;; I11 = I11 - R11
	vmulsd	xmm0, xmm0, xmm15		;; R11 = R11 * SQRTHALF
	vmulsd	xmm5, xmm5, xmm15		;; I11 = I11 * SQRTHALF

	;; Odd level 2

	vaddsd	xmm4, xmm7, xmm2		;; R1 + R5 (newest R1)
	vsubsd	xmm7, xmm7, xmm2		;; R1 - R5 (newest R5)

	vaddsd	xmm2, xmm1, xmm3		;; R3 + R7 (newest R3)
	vsubsd	xmm1, xmm1, xmm3		;; R3 - R7 (newest R7)

						;; R9/I9 morphs into newest R9/R13
						;; R11/I11 morphs into newest R11/R15

	;; Odd level 1

	vaddsd	xmm3, xmm4, xmm8		;; R1 + R9 (final R1)
	vsubsd	xmm4, xmm4, xmm8		;; R1 - R9 (final R9)

	vaddsd	xmm8, xmm7, xmm6		;; R5 + R13 (final R5)
	vsubsd	xmm7, xmm7, xmm6		;; R5 - R13 (final R13)

	vaddsd	xmm6, xmm2, xmm0		;; R3 + R11 (final R3)
	vsubsd	xmm2, xmm2, xmm0		;; R3 - R11 (final R11)

	vaddsd	xmm0, xmm1, xmm5		;; R7 + R15 (final R7)
	vsubsd	xmm1, xmm1, xmm5		;; R7 - R15 (final R15)

	vmovsd	Q [srcreg], xmm3		;; R1
	vmovsd	Q [srcreg+32], xmm4		;; R9
	vmovsd	Q [srcreg+d4], xmm8		;; R5
	vmovsd	Q [srcreg+d4+32], xmm7		;; R13
	vmovsd	Q [srcreg+d2], xmm6		;; R3
	vmovsd	Q [srcreg+d2+32], xmm2		;; R11
	vmovsd	Q [srcreg+d4+d2], xmm0		;; R7
	vmovsd	Q [srcreg+d4+d2+32], xmm1	;; R15

	bump	srcreg, srcinc
	ENDM

ENDIF

;;
;; ************************************* 16-reals-first-fft variants ******************************************
;;

;; These macros operate on 16 reals doing 4 levels of the FFT, applying
;; the sin/cos multipliers afterwards.  The output is 2 reals and 7 complex numbers.

;; To calculate a 16-reals FFT, we calculate 16 complex values in a brute force way (using a shorthand notation):
;; r1 + r2 + ... + r16	*  w^0000000000...
;; r1 + r2 + ... + r16	*  w^0123456789A...
;; r1 + r2 + ... + r16	*  w^02468ACE....
;;    ...
;; r1 + r2 + ... + r16	*  w^...A987654321
;; Note that Hermetian symmetry means we won't need to calculate the last 8 complex values.
;;
;; The sin/cos values (w = 16th root of unity) are:
;; w^1 = .924 + .383i
;; w^2 = .707 + .707i
;; w^3 = .383 + .924i
;; w^4 = 0 + 1i
;; w^5 = -.383 + .924i
;; w^6 = -.707 + .707i
;; w^7 = -.924 + .383i
;; w^8 = -1
;;
;; Applying the sin/cos values above (and noting that combining r2 and r16, r3 and r15, etc. will simplify calculations):
;; reals:
;; r1     +(r2+r16)     +(r3+r15)     +(r4+r14) + (r5+r13)     +(r6+r12)     +(r7+r11)     +(r8+r10) + r9
;; r1 +.924(r2+r16) +.707(r3+r15) +.383(r4+r14)            -.383(r6+r12) -.707(r7+r11) -.924(r8+r10) - r9
;; r1 +.707(r2+r16)               -.707(r4+r14) - (r5+r13) -.707(r6+r12)               +.707(r8+r10) + r9
;; r1 +.383(r2+r16) -.707(r3+r15) -.924(r4+r14)            +.924(r6+r12) +.707(r7+r11) -.383(r8+r10) - r9
;; r1                   -(r3+r15)               + (r5+r13)                   -(r7+r11)               + r9
;; r1 -.383(r2+r16) -.707(r3+r15) +.924(r4+r14)            -.924(r6+r12) +.707(r7+r11) +.383(r8+r10) - r9
;; r1 -.707(r2+r16)               +.707(r4+r14) - (r5+r13) +.707(r6+r12)               -.707(r8+r10) + r9
;; r1 -.924(r2+r16) +.707(r3+r15) -.383(r4+r14)            +.383(r6+r12) -.707(r7+r11) +.924(r8+r10) - r9
;; r1     -(r2+r16)     +(r3+r15)     -(r4+r14) + (r5+r13)     -(r6+r12)     +(r7+r11)     -(r8+r10) + r9
;;
;; imaginarys:
;; 0
;; +.383(r2-r16) +.707(r3-r15) +.924(r4-r14) + (r5-r13) +.924(r6-r12) +.707(r7-r11) +.383(r8-r10)
;; +.707(r2-r16)     +(r3-r15) +.707(r4-r14)            -.707(r6-r12)     -(r7-r11) -.707(r8-r10)
;; +.924(r2-r16) +.707(r3-r15) -.383(r4-r14) - (r5-r13) -.383(r6-r12) +.707(r7-r11) +.924(r8-r10)
;;      (r2-r16)                   -(r4-r14)                +(r6-r12)                   -(r8-r10)
;; +.924(r2-r16) -.707(r3-r15) -.383(r4-r14) + (r5-r13) -.383(r6-r12) -.707(r7-r11) +.924(r8-r10)
;; +.707(r2-r16)     -(r3-r15) +.707(r4-r14)            -.707(r6-r12)     +(r7-r11) -.707(r8-r10)
;; +.383(r2-r16) -.707(r3-r15) +.924(r4-r14) - (r5-r13) +.924(r6-r12) -.707(r7-r11) +.383(r8-r10)
;; 0
;;
;; There are many more symmetries we can take advantage of.   For example, the (r2+/-r16) column
;; always has the same multiplier as the (r8+/-r10) column.  This is true for all the "even" columns.
;; Also the computations for the 2nd row are very similar to the computations for the 8th row,
;; the 3rd row are similar to the 7th, etc.  Finally, note that for the odd columns, there are
;; only three multipliers to apply and can be combined with every fourth column.
;;
;; Lastly, output would normally be 7 complex and 2 reals.  but the users of this macro
;; expect us to "back up" the 2 reals by one level.  That is:
;;	real #1A:  r1 + r3+r15 + r5+r13 + ...
;;	real #1B:  r2+r16 + r4+r14 + ...

;; Simplifying, we get:
;; r2/r8 = r2o +/- r2e
;; r3/r7 = r3o +/- .707*r3e
;; r4/r6 = r4o +/- r4e
;;
;; r1A = r1+r9     +((r3+r15)+(r7+r11)) + (r5+r13)                
;; r2o = r1-r9 +.707((r3+r15)-(r7+r11))               
;; r3o = r1+r9                          - (r5+r13)                 
;; r4o = r1-r9 -.707((r3+r15)-(r7+r11))               
;; r5  = r1+r9     -((r3+r15)+(r7+r11)) + (r5+r13)                                  
;;
;; r1B =     +((r2+r16)+(r8+r10))     +((r4+r14)+(r6+r12))
;; r2e = +.924((r2+r16)-(r8+r10)) +.383((r4+r14)-(r6+r12))
;; r3e =     +((r2+r16)+(r8+r10))     -((r4+r14)+(r6+r12))
;; r4e = +.383((r2+r16)-(r8+r10)) -.924((r4+r14)-(r6+r12))
;;
;; i2/8 = i2e + i2o
;; i3/7 = .707*i3e + i3o
;; i4/6 = i4e + i4o
;;
;; i2e = +.383((r2-r16)+(r8-r10)) +.924((r4-r14)+(r6-r12))
;; i3e =     +((r2-r16)-(r8-r10))     +((r4-r14)-(r6-r12))
;; i4e = +.924((r2-r16)+(r8-r10)) -.383((r4-r14)+(r6-r12))
;; i5  =      ((r2-r16)-(r8-r10))     -((r4-r14)-(r6-r12))
;;
;; i2o = +.707((r3-r15)+(r7-r11)) + (r5-r13)   
;; i3o =     +((r3-r15)-(r7-r11))
;; i4o = +.707((r3-r15)+(r7-r11)) - (r5-r13)   

yr8_8cl_16_reals_fft_preload MACRO
	ENDM

yr8_8cl_16_reals_fft MACRO srcreg,srcinc,d1,d2,d4,screg,scinc,maxrpt,L1pt,L1pd

	vmovapd	ymm0, [srcreg+d2]		;; r3
	vaddpd	ymm0, ymm0, [srcreg+d4+d2+32]	;; r3+r15						; 1-3

	vmovapd	ymm1, [srcreg+d4+d2]		;; r7
	vaddpd	ymm1, ymm1, [srcreg+d2+32]	;; r7+r11						; 2-4

	vmovapd	ymm2, [srcreg+d4]		;; r5
	vaddpd	ymm2, ymm2, [srcreg+d4+32]	;; r5+r13						; 3-5

	vmovapd	ymm4, [srcreg]			;; r1
	vmovapd	ymm5, [srcreg+32]		;; r9
	vaddpd	ymm3, ymm4, ymm5		;; r1+r9						; 4-6

	vsubpd	ymm6, ymm0, ymm1		;; (r3+r15)-(r7+r11)					; 5-7
	vmovapd	ymm7, YMM_SQRTHALF

	vaddpd	ymm0, ymm0, ymm1		;; (r3+r15)+(r7+r11)					; 6-8

	vaddpd	ymm1, ymm3, ymm2		;; (r1+r9)+(r5+r13)					; 7-9
	L1prefetchw srcreg+L1pd, L1pt

	vsubpd	ymm4, ymm4, ymm5		;; r1-r9						; 8-10
	vmulpd	ymm6, ymm7, ymm6		;; .707((r3+r15)-(r7+r11))				;	8-12

	vsubpd	ymm3, ymm3, ymm2		;; (r1+r9)-(r5+r13) (r3o)				; 9-11
	vmovapd	ymm5, [srcreg+d1]		;; r2

	vaddpd	ymm2, ymm1, ymm0		;; (r1+r9)+(r5+r13) + ((r3+r15)+(r7+r11)) (r1A)		; 10-12

	vsubpd	ymm1, ymm1, ymm0		;; (r1+r9)+(r5+r13) - ((r3+r15)+(r7+r11)) (r5)		; 11-13

	vaddpd	ymm5, ymm5, [srcreg+d4+d2+d1+32] ;; r2+r16						; 12-14
	vmovapd	ymm0, [srcreg+d4+d2+d1]		;; r8
	ystore	YMM_TMPS[0*32], ymm3		;; Real odd-cols row #3					; 12

	vaddpd	ymm0, ymm0, [srcreg+d1+32]	;; r8+r10						; 13-15
	vmovapd	ymm3, [srcreg+d2+d1]		;; r4
	ystore	[srcreg], ymm2			;; Final real #1A					; 13

	vaddpd	ymm3, ymm3, [srcreg+d4+d1+32]	;; r4+r14						; 14-16
	vmovapd	ymm2, [srcreg+d4+d1]		;; r6
	ystore	YMM_TMPS[1*32], ymm1		;; Real row #5						; 14

	vaddpd	ymm2, ymm2, [srcreg+d2+d1+32]	;; r6+r12						; 15-17

	vsubpd	ymm1, ymm5, ymm0		;; (r2+r16)-(r8+r10)					; 16-18
	L1prefetchw srcreg+d1+L1pd, L1pt

	vaddpd	ymm5, ymm5, ymm0		;; (r2+r16)+(r8+r10)					; 17-19

	vsubpd	ymm0, ymm3, ymm2		;; (r4+r14)-(r6+r12)					; 18-20
	L1prefetchw srcreg+d2+L1pd, L1pt

	vaddpd	ymm3, ymm3, ymm2		;; (r4+r14)+(r6+r12)					; 19-21

	vaddpd	ymm2, ymm4, ymm6		;; (r1-r9) + .707((r3+r15)-(r7+r11)) (r2o)		; 20-22

	vsubpd	ymm4, ymm4, ymm6		;; (r1-r9) - .707((r3+r15)-(r7+r11)) (r4o)		; 21-23
	vmovapd ymm6, YMM_P924
	ystore	YMM_TMPS[2*32], ymm2		;; Real odd-cols row #2					; 23
	vmulpd	ymm2, ymm6, ymm1		;; .924((r2+r16)-(r8+r10))				;	19-23
	ystore	YMM_TMPS[3*32], ymm4		;; Real odd-cols row #4					; 24
	vmovapd ymm4, YMM_P383
	vmulpd	ymm1, ymm4, ymm1		;; .383((r2+r16)-(r8+r10))				;	20-24
	vmulpd	ymm4, ymm4, ymm0		;; .383((r4+r14)-(r6+r12))				;	21-25

	vmulpd	ymm6, ymm6, ymm0		;; .924((r4+r14)-(r6+r12))				;	22-26
	vsubpd	ymm0, ymm5, ymm3		;; ((r2+r16)+(r8+r10)) - ((r4+r14)+(r6+r12)) (r3e)	; 22-24

	vaddpd	ymm5, ymm5, ymm3		;; ((r2+r16)+(r8+r10)) + ((r4+r14)+(r6+r12)) (r1B)	; 23-25

	vmovapd	ymm3, [srcreg+d2]		;; r3
	vsubpd	ymm3, ymm3, [srcreg+d4+d2+32]	;; r3-r15						; 24-26

	ystore	[srcreg+32], ymm5		;; Save final real #1B					; 26
	vmovapd	ymm5, [srcreg+d4+d2]		;; r7
	vsubpd	ymm5, ymm5, [srcreg+d2+32]	;; r7-r11						; 25-27
	vmulpd	ymm0, ymm7, ymm0		;; .707*r3e						;	25-29

	vaddpd	ymm2, ymm2, ymm4		;; .924((r2+r16)-(r8+r10))+.383((r4+r14)-(r6+r12)) (r2e) ; 26-28
	vmovapd	ymm4, [srcreg+d4]		;; r5

	vsubpd	ymm1, ymm1, ymm6		;; .383((r2+r16)-(r8+r10))-.924((r4+r14)-(r6+r12)) (r4e) ; 27-29

	vsubpd	ymm4, ymm4, [srcreg+d4+32]	;; r5-r13						; 28-30

	vaddpd	ymm6, ymm3, ymm5		;; (r3-r15)+(r7-r11)					; 29-31
	ystore	YMM_TMPS[4*32], ymm2		;; Real even-cols row #2				; 29
	vmovapd	ymm2, [srcreg+d1]		;; r2

	vsubpd	ymm3, ymm3, ymm5		;; (r3-r15)-(r7-r11) (i3o)				; 30-32
	ystore	YMM_TMPS[5*32], ymm0		;; Real even-cols row #3				; 30

	vsubpd	ymm2, ymm2, [srcreg+d4+d2+d1+32] ;; r2-r16						; 31-33
	vmovapd	ymm5, [srcreg+d4+d2+d1]		;; r8
	ystore	YMM_TMPS[6*32], ymm1		;; Real even-cols row #4				; 30+1

	vsubpd	ymm5, ymm5, [srcreg+d1+32]	;; r8-r10						; 32-34
	vmulpd	ymm6, ymm7, ymm6		;; .707((r3-r15)+(r7-r11))				;	32-36

	vmovapd	ymm0, [srcreg+d2+d1]		;; r4
	vsubpd	ymm0, ymm0, [srcreg+d4+d1+32]	;; r4-r14						; 33-35
	vmovapd	ymm1, [srcreg+d4+d1]		;; r6
	ystore	YMM_TMPS[7*32], ymm3		;; Imag odd-cols row #3					; 33

	vsubpd	ymm1, ymm1, [srcreg+d2+d1+32]	;; r6-r12						; 34-36

	vaddpd	ymm3, ymm2, ymm5		;; (r2-r16)+(r8-r10)					; 35-37
	L1prefetchw srcreg+d2+d1+L1pd, L1pt

	vsubpd	ymm2, ymm2, ymm5		;; (r2-r16)-(r8-r10)					; 36-38

	vaddpd	ymm5, ymm0, ymm1		;; (r4-r14)+(r6-r12)					; 37-39
	L1prefetchw srcreg+d4+L1pd, L1pt

	vsubpd	ymm0, ymm0, ymm1		;; (r4-r14)-(r6-r12)					; 38-40

	vaddpd	ymm1, ymm6, ymm4		;; .707((r3-r15)+(r7-r11))+(r5-r13) (i2o)		; 39-41

	vsubpd	ymm6, ymm6, ymm4		;; .707((r3-r15)+(r7-r11))-(r5-r13) (i4o)		; 40-42
	vmovapd ymm4, YMM_P383
	ystore	YMM_TMPS[8*32], ymm1		;; Imag odd-cols row #2					; 42
	vmulpd	ymm1, ymm4, ymm3		;; .383((r2-r16)+(r8-r10))				;	38-42
	ystore	YMM_TMPS[9*32], ymm6		;; Imag odd-cols row #4					; 43
	vmovapd ymm6, YMM_P924
	vmulpd	ymm3, ymm6, ymm3		;; .924((r2-r16)+(r8-r10))				;	39-43
	vmulpd	ymm6, ymm6, ymm5		;; .924((r4-r14)+(r6-r12))				;	40-44

	vmulpd	ymm4, ymm4, ymm5		;; .383((r4-r14)+(r6-r12))				;	41-45
	vaddpd	ymm5, ymm2, ymm0		;; ((r2-r16)-(r8-r10))+((r4-r14)-(r6-r12)) (i3e)	; 41-43

	vsubpd	ymm2, ymm2, ymm0		;; ((r2-r16)-(r8-r10))-((r4-r14)-(r6-r12)) (i5)		; 42-44
	vmovapd	ymm0, YMM_TMPS[2*32]		;; Real odd-cols row #2
	vmulpd	ymm5, ymm7, ymm5		;; .707*i3e						;	44-48

	ystore	YMM_TMPS[2*32], ymm5		;; Imag even-cols row #3				; 49
	vmovapd	ymm7, YMM_TMPS[4*32]		;; Real even-cols row #2
	vsubpd	ymm5, ymm0, ymm7		;; Real #8						; 43-45

	vaddpd	ymm0, ymm0, ymm7		;; Real #2						; 44-46
	vmovapd	ymm7, YMM_TMPS[1*32]		;; Real #5

	vaddpd	ymm1, ymm1, ymm6		;; .383((r2-r16)+(r8-r10))+.924((r4-r14)+(r6-r12)) (i2e) ; 45-47
	vmovapd	ymm6, [screg+3*64+32]		;; cosine/sine for R5/I5

	vsubpd	ymm3, ymm3, ymm4		;; .924((r2-r16)+(r8-r10))-.383((r4-r14)+(r6-r12)) (i4e) ; 46-48

	;; Now combine the even and odd columns then do the post-multiply by twiddle factors.

	vmulpd	ymm4, ymm7, ymm6		;; A5 = R5 * cosine/sine				;	45-49
	vmulpd	ymm6, ymm2, ymm6		;; B5 = I5 * cosine/sine				;	46-50
	ystore	YMM_TMPS[1*32], ymm3		;; Imag even-cols row #4				; 49
	vmovapd	ymm3, [screg+3*64]		;; sine for R5/I5
	vsubpd	ymm4, ymm4, ymm2		;; A5 = A5 - I5						; 50-52
	vaddpd	ymm6, ymm6, ymm7		;; B5 = B5 + R5						; 51-53
	vmulpd	ymm4, ymm4, ymm3		;; A5 = A5 * sine (final R5)				;	53-57
	vmulpd	ymm6, ymm6, ymm3		;; B5 = B5 * sine (final I5)				;	54-58
	ystore	[srcreg+d4], ymm4		;; Save final R5
	ystore	[srcreg+d4+32], ymm6		;; Save final I5

	vmovapd	ymm7, YMM_TMPS[8*32]		;; Imag odd-cols row #2
	vsubpd	ymm2, ymm1, ymm7		;; Imag #8						; 48-50
	vaddpd	ymm1, ymm1, ymm7		;; Imag #2						; 49-51
	vmovapd	ymm3, [screg+6*64+32]		;; cosine/sine for R8/I8
	vmulpd	ymm6, ymm5, ymm3		;; A8 = R8 * cosine/sine
	vmovapd	ymm4, [screg+32]		;; cosine/sine for R2/I2
	vmulpd	ymm7, ymm0, ymm4		;; A2 = R2 * cosine/sine
	vsubpd	ymm6, ymm6, ymm2		;; A8 = A8 - I8
	vmulpd	ymm2, ymm2, ymm3		;; B8 = I8 * cosine/sine
	vsubpd	ymm7, ymm7, ymm1		;; A2 = A2 - I2
	L1prefetchw srcreg+d4+d1+L1pd, L1pt
	vmulpd	ymm1, ymm1, ymm4		;; B2 = I2 * cosine/sine
	vaddpd	ymm2, ymm2, ymm5		;; B8 = B8 + R8
	vmovapd	ymm3, [screg+6*64]		;; sine for R8/I8
	vmulpd	ymm6, ymm6, ymm3		;; A8 = A8 * sine (final R8)
	vaddpd	ymm1, ymm1, ymm0		;; B2 = B2 + R2
	vmovapd	ymm4, [screg]			;; sine for R2/I2
	vmulpd	ymm7, ymm7, ymm4		;; A2 = A2 * sine (final R2)
	vmulpd	ymm2, ymm2, ymm3		;; B8 = B8 * sine (final I8)
	vmulpd	ymm1, ymm1, ymm4		;; B2 = B2 * sine (final I2)
	ystore	[srcreg+d4+d2+d1], ymm6		;; Save final R8
	ystore	[srcreg+d4+d2+d1+32], ymm2	;; Save final I8
	ystore	[srcreg+d1], ymm7		;; Save final R2
	ystore	[srcreg+d1+32], ymm1		;; Save final I2

	vmovapd	ymm6, YMM_TMPS[0*32]		;; Real odd-cols row #3
	vmovapd	ymm7, YMM_TMPS[5*32]		;; Real even-cols row #3
	vsubpd	ymm0, ymm6, ymm7		;; Real #7
	vaddpd	ymm1, ymm6, ymm7		;; Real #3
	vmovapd	ymm6, YMM_TMPS[2*32]		;; Imag even-cols row #3
	vmovapd	ymm7, YMM_TMPS[7*32]		;; Imag odd-cols row #3
	vsubpd	ymm2, ymm6, ymm7		;; Imag #7
	vaddpd	ymm3, ymm6, ymm7		;; Imag #3

	vmovapd	ymm5, [screg+5*64+32]		;; cosine/sine for R7/I7
	vmulpd	ymm6, ymm0, ymm5		;; A7 = R7 * cosine/sine
	vmovapd	ymm4, [screg+64+32]		;; cosine/sine for R3/I3
	vmulpd	ymm7, ymm1, ymm4		;; A3 = R3 * cosine/sine
	vsubpd	ymm6, ymm6, ymm2		;; A7 = A7 - I7
	vmulpd	ymm2, ymm2, ymm5		;; B7 = I7 * cosine/sine
	vsubpd	ymm7, ymm7, ymm3		;; A3 = A3 - I3
	L1prefetchw srcreg+d4+d2+L1pd, L1pt
	vmulpd	ymm3, ymm3, ymm4		;; B3 = I3 * cosine/sine
	vaddpd	ymm2, ymm2, ymm0		;; B7 = B7 + R7
	vmovapd	ymm5, [screg+5*64]		;; sine for R7/I7
	vmulpd	ymm6, ymm6, ymm5		;; A7 = A7 * sine (final R7)
	vaddpd	ymm3, ymm3, ymm1		;; B3 = B3 + R3
	vmovapd	ymm4, [screg+64]		;; sine for R3/I3
	vmulpd	ymm7, ymm7, ymm4		;; A3 = A3 * sine (final R3)
	vmulpd	ymm2, ymm2, ymm5		;; B7 = B7 * sine (final I7)
	vmulpd	ymm3, ymm3, ymm4		;; B3 = B3 * sine (final I3)
	ystore	[srcreg+d4+d2], ymm6		;; Save final R7
	ystore	[srcreg+d4+d2+32], ymm2		;; Save final I7
	ystore	[srcreg+d2], ymm7		;; Save final R3
	ystore	[srcreg+d2+32], ymm3		;; Save final I3

	vmovapd	ymm6, YMM_TMPS[3*32]		;; Real odd-cols row #4
	vmovapd	ymm7, YMM_TMPS[6*32]		;; Real even-cols row #4
	vsubpd	ymm0, ymm6, ymm7		;; Real #6
	vaddpd	ymm1, ymm6, ymm7		;; Real #4
	vmovapd	ymm6, YMM_TMPS[1*32]		;; Imag even-cols row #4
	vmovapd	ymm7, YMM_TMPS[9*32]		;; Imag odd-cols row #4
	vsubpd	ymm2, ymm6, ymm7		;; Imag #6
	vaddpd	ymm3, ymm6, ymm7		;; Imag #4

	vmovapd	ymm5, [screg+4*64+32]		;; cosine/sine for R6/I6
	vmulpd	ymm6, ymm0, ymm5		;; A6 = R6 * cosine/sine
	vmovapd	ymm4, [screg+2*64+32]		;; cosine/sine for R4/I4
	vmulpd	ymm7, ymm1, ymm4		;; A4 = R4 * cosine/sine
	vsubpd	ymm6, ymm6, ymm2		;; A6 = A6 - I6
	vmulpd	ymm2, ymm2, ymm5		;; B6 = I6 * cosine/sine
	vsubpd	ymm7, ymm7, ymm3		;; A4 = A4 - I4
	L1prefetchw srcreg+d4+d2+d1+L1pd, L1pt
	vmulpd	ymm3, ymm3, ymm4		;; B4 = I4 * cosine/sine
	vaddpd	ymm2, ymm2, ymm0		;; B6 = B6 + R6
	vmovapd ymm5, [screg+4*64]		;; sine for R6/I6
	vmulpd	ymm6, ymm6, ymm5		;; A6 = A6 * sine (final R6)
	vaddpd	ymm3, ymm3, ymm1		;; B4 = B4 + R4
	vmovapd	ymm4, [screg+2*64]		;; sine for R4/I4
	vmulpd	ymm7, ymm7, ymm4		;; A4 = A4 * sine (final R4)
	vmulpd	ymm2, ymm2, ymm5		;; B6 = B6 * sine (final I6)
	vmulpd	ymm3, ymm3, ymm4		;; B4 = B4 * sine (final I4)
	ystore	[srcreg+d4+d1], ymm6		;; Save final R6
	ystore	[srcreg+d4+d1+32], ymm2		;; Save final I6
	ystore	[srcreg+d2+d1], ymm7		;; Save final R4
	ystore	[srcreg+d2+d1+32], ymm3		;; Save final I4

	bump	srcreg, srcinc
	bump	screg, scinc
	ENDM

;; 64-bit version

IFDEF X86_64

yr8_8cl_16_reals_fft_preload MACRO
	ENDM

yr8_8cl_16_reals_fft MACRO srcreg,srcinc,d1,d2,d4,screg,scinc,maxrpt,L1pt,L1pd

	vmovapd	ymm1, [srcreg+d1]		;; r2
	vmovapd	ymm2, [srcreg+d4+d2+d1+32]	;; r16
	vaddpd	ymm0, ymm1, ymm2		;; r2+r16						; 1-3			n 9

	vsubpd	ymm1, ymm1, ymm2		;; r2-r16						; 2-4			n 13

	vmovapd	ymm3, [srcreg+d4+d2+d1]		;; r8
	vmovapd	ymm4, [srcreg+d1+32]		;; r10
	vaddpd	ymm2, ymm3, ymm4		;; r8+r10						; 3-5			n 9

	vsubpd	ymm3, ymm3, ymm4		;; r8-r10						; 4-6			n 13

	vmovapd	ymm5, [srcreg+d2+d1]		;; r4
	vmovapd	ymm6, [srcreg+d4+d1+32]		;; r14
	vaddpd	ymm4, ymm5, ymm6		;; r4+r14						; 5-7			n 11

	vsubpd	ymm5, ymm5, ymm6		;; r4-r14						; 6-8			n 15

	vmovapd	ymm7, [srcreg+d4+d1]		;; r6
	vmovapd	ymm8, [srcreg+d2+d1+32]		;; r12
	vaddpd	ymm6, ymm7, ymm8		;; r6+r12						; 7-9			n 11

	vsubpd	ymm7, ymm7, ymm8		;; r6-r12						; 8-10			n 15
	vmovapd ymm9, YMM_P924

	vsubpd	ymm8, ymm0, ymm2		;; (r2+r16)-(r8+r10)					; 9-11			n 12
	vmovapd ymm10, YMM_P383

	vaddpd	ymm0, ymm0, ymm2		;; (r2+r16)+(r8+r10)					; 10-12			n 17
	vmovapd	ymm14, [srcreg+d2]		;; r3

	vsubpd	ymm2, ymm4, ymm6		;; (r4+r14)-(r6+r12)					; 11-13			n 14
	vmovapd	ymm15, [srcreg+d4+d2+32]	;; r15

	vaddpd	ymm4, ymm4, ymm6		;; (r4+r14)+(r6+r12)					; 12-14			n 17
	vmulpd	ymm6, ymm9, ymm8		;; .924((r2+r16)-(r8+r10))				;	12-16		n 19
	L1prefetchw srcreg+L1pd, L1pt

	vaddpd	ymm11, ymm1, ymm3		;; (r2-r16)+(r8-r10)					; 13-15			n 16
	vmulpd	ymm8, ymm10, ymm8		;; .383((r2+r16)-(r8+r10))				;	13-17		n 20

	vsubpd	ymm1, ymm1, ymm3		;; (r2-r16)-(r8-r10)					; 14-16			n 21
	vmulpd	ymm3, ymm10, ymm2		;; .383((r4+r14)-(r6+r12))				;	14-18		n 19
	L1prefetchw srcreg+d1+L1pd, L1pt

	vaddpd	ymm12, ymm5, ymm7		;; (r4-r14)+(r6-r12)					; 15-17			n 18
	vmulpd	ymm2, ymm9, ymm2		;; .924((r4+r14)-(r6+r12))				;	15-19		n 20

	vsubpd	ymm5, ymm5, ymm7		;; (r4-r14)-(r6-r12)					; 16-18			n 21
	vmulpd	ymm7, ymm10, ymm11		;; .383((r2-r16)+(r8-r10))				;	16-20		n 23
	L1prefetchw srcreg+d2+L1pd, L1pt

	vsubpd	ymm13, ymm0, ymm4		;; ((r2+r16)+(r8+r10)) - ((r4+r14)+(r6+r12)) (r3e)	; 17-19			n 36
	vmulpd	ymm11, ymm9, ymm11		;; .924((r2-r16)+(r8-r10))				;	17-21		n 24

	vaddpd	ymm0, ymm0, ymm4		;; ((r2+r16)+(r8+r10)) + ((r4+r14)+(r6+r12)) (r1B)	; 18-20
	vmulpd	ymm9, ymm9, ymm12		;; .924((r4-r14)+(r6-r12))				;	18-22		n 23
	L1prefetchw srcreg+d2+d1+L1pd, L1pt

	vaddpd	ymm6, ymm6, ymm3		;; .924((r2+r16)-(r8+r10))+.383((r4+r14)-(r6+r12)) (r2e) ; 19-21		n 53
	vmulpd	ymm10, ymm10, ymm12		;; .383((r4-r14)+(r6-r12))				;	19-23		n 24

	vsubpd	ymm8, ymm8, ymm2		;; .383((r2+r16)-(r8+r10))-.924((r4+r14)-(r6+r12)) (r4e) ; 20-22		n 57
	vmovapd	ymm4, [srcreg+d4+d2]		;; r7

	vaddpd	ymm2, ymm1, ymm5		;; ((r2-r16)-(r8-r10))+((r4-r14)-(r6-r12)) (i3e)	; 21-23			n 38
	vmovapd	ymm3, [srcreg+d2+32]		;; r11

	vsubpd	ymm1, ymm1, ymm5		;; ((r2-r16)-(r8-r10))-((r4-r14)-(r6-r12)) (i5)		; 22-24
	vmovapd	ymm12, [srcreg+d4]		;; r5

	vaddpd	ymm7, ymm7, ymm9		;; .383((r2-r16)+(r8-r10))+.924((r4-r14)+(r6-r12)) (i2e) ; 23-25		n 55
	vmovapd	ymm5, [srcreg+d4+32]		;; r13

	vsubpd	ymm11, ymm11, ymm10		;; .924((r2-r16)+(r8-r10))-.383((r4-r14)+(r6-r12)) (i4e) ; 24-26		n 59
	vmovapd	ymm9, [srcreg]			;; r1

	vaddpd	ymm10, ymm14, ymm15		;; r3+r15						; 25-27			n 33
	ystore	YMM_TMPS[0*32], ymm1		;; Save i5						; 25

	vsubpd	ymm14, ymm14, ymm15		;; r3-r15						; 26-28			n 37
	vmovapd	ymm1, [srcreg+32]		;; r9

	vaddpd	ymm15, ymm4, ymm3		;; r7+r11						; 27-29			n 33
	ystore	[srcreg+32], ymm0		;; Save final real #1B					; 21

	vsubpd	ymm4, ymm4, ymm3		;; r7-r11						; 28-30			n 37
	vmovapd	ymm0, YMM_SQRTHALF

	vaddpd	ymm3, ymm12, ymm5		;; r5+r13						; 29-31			n 35
	L1prefetchw srcreg+d4+L1pd, L1pt

	vsubpd	ymm12, ymm12, ymm5		;; r5-r13						; 30-32			n 47

	vaddpd	ymm5, ymm9, ymm1		;; r1+r9						; 31-33			n 35
	L1prefetchw srcreg+d4+d1+L1pd, L1pt

	vsubpd	ymm9, ymm9, ymm1		;; r1-r9						; 32-34			n 45

	vaddpd	ymm1, ymm10, ymm15		;; (r3+r15)+(r7+r11)					; 33-35			n 39
	L1prefetchw srcreg+d4+d2+L1pd, L1pt

	vsubpd	ymm10, ymm10, ymm15		;; (r3+r15)-(r7+r11)					; 34-36			n 40

	vaddpd	ymm15, ymm5, ymm3		;; (r1+r9)+(r5+r13)					; 35-37			n 39
	L1prefetchw srcreg+d4+d2+d1+L1pd, L1pt

	vsubpd	ymm5, ymm5, ymm3		;; (r1+r9)-(r5+r13) (r3o)				; 36-38			n 41
	vmulpd	ymm13, ymm0, ymm13		;; .707*r3e						;	36-40		n 41

	vsubpd	ymm3, ymm14, ymm4		;; (r3-r15)-(r7-r11) (i3o)				; 37-39			n 43

	vaddpd	ymm14, ymm14, ymm4		;; (r3-r15)+(r7-r11)					; 38-40			n 47
	vmulpd	ymm2, ymm0, ymm2		;; .707*i3e						;	38-42		n 43

	vsubpd	ymm4, ymm15, ymm1		;; (r1+r9)+(r5+r13) - ((r3+r15)+(r7+r11)) (r5)		; 39-41

	vaddpd	ymm15, ymm15, ymm1		;; (r1+r9)+(r5+r13) + ((r3+r15)+(r7+r11)) (r1A)		; 40-42
	vmulpd	ymm10, ymm0, ymm10		;; .707((r3+r15)-(r7+r11))				;	40-44		n 45

	vsubpd	ymm1, ymm5, ymm13		;; (ro3 - re3) Real #7					; 41-43			n 44

	ystore	YMM_TMPS[1*32], ymm4		;; Real row #5						; 42
	vsubpd	ymm4, ymm2, ymm3		;; (ie3 - io3) Imag #7					; 42-44			n 45
	vmulpd	ymm14, ymm0, ymm14		;; .707((r3-r15)+(r7-r11))				;	42-46		n 47

	vaddpd	ymm5, ymm5, ymm13		;; (ro3 + re3) Real #3					; 43-45			n 46
	vmovapd	ymm0, [screg+5*64+32]		;; cosine/sine for R7/I7
	ystore	[srcreg], ymm15			;; Final real #1A					; 43

	vaddpd	ymm2, ymm2, ymm3		;; (ie3 + io3) Imag #3					; 44-46			n 47
	vmulpd	ymm3, ymm1, ymm0		;; A7 = R7 * cosine/sine				;	44-48
	vmovapd	ymm13, [screg+64+32]		;; cosine/sine for R3/I3

	vaddpd	ymm15, ymm9, ymm10		;; (r1-r9) + .707((r3+r15)-(r7+r11)) (r2o)		; 45-47
	vmulpd	ymm0, ymm4, ymm0		;; B7 = I7 * cosine/sine				;	45-49

	vsubpd	ymm9, ymm9, ymm10		;; (r1-r9) - .707((r3+r15)-(r7+r11)) (r4o)		; 46-48
	vmulpd	ymm10, ymm5, ymm13		;; A3 = R3 * cosine/sine				;	46-50

	vmulpd	ymm13, ymm2, ymm13		;; B3 = I3 * cosine/sine				;	47-51
	vsubpd	ymm3, ymm3, ymm4		;; A7 = A7 - I7						; 49-51
	vaddpd	ymm4, ymm14, ymm12		;; .707((r3-r15)+(r7-r11))+(r5-r13) (i2o)		; 47-49
	vsubpd	ymm14, ymm14, ymm12		;; .707((r3-r15)+(r7-r11))-(r5-r13) (i4o)		; 48-50

	vaddpd	ymm0, ymm0, ymm1		;; B7 = B7 + R7						; 50-52
	vmovapd	ymm12, [screg+5*64]		;; sine for R7/I7

	vsubpd	ymm10, ymm10, ymm2		;; A3 = A3 - I3						; 51-53
	vmovapd	ymm1, [screg+64]		;; sine for R3/I3

	vaddpd	ymm13, ymm13, ymm5		;; B3 = B3 + R3						; 52-54
	vmulpd	ymm3, ymm3, ymm12		;; A7 = A7 * sine (final R7)				;	52-56

	vsubpd	ymm5, ymm15, ymm6		;; (ro2 - re2) Real #8					; 53-55
	vmulpd	ymm0, ymm0, ymm12		;; B7 = B7 * sine (final I7)				;	53-57

	vsubpd	ymm2, ymm7, ymm4		;; (ie2 - io2) Imag #8					; 54-56
	vmulpd	ymm10, ymm10, ymm1		;; A3 = A3 * sine (final R3)				;	54-58

	vaddpd	ymm15, ymm15, ymm6		;; (ro2 + re2) Real #2					; 55-57
	vmulpd	ymm13, ymm13, ymm1		;; B3 = B3 * sine (final I3)				;	55-59
	vmovapd	ymm6, [screg+6*64+32]		;; cosine/sine for R8/I8

	vaddpd	ymm7, ymm7, ymm4		;; (ie2 + io2) Imag #2					; 56-58
	vmulpd	ymm4, ymm5, ymm6		;; A8 = R8 * cosine/sine				;	56-60
	vmovapd	ymm12, [screg+32]		;; cosine/sine for R2/I2

	vsubpd	ymm1, ymm9, ymm8		;; (ro4 - re4) Real #6					; 57-59
	vmulpd	ymm6, ymm2, ymm6		;; B8 = I8 * cosine/sine				;	57-61
	ystore	[srcreg+d4+d2], ymm3		;; Save final R7					; 57

	vsubpd	ymm3, ymm11, ymm14		;; (ie4 - io4) Imag #6					; 58-60
	ystore	[srcreg+d4+d2+32], ymm0		;; Save final I7					; 58
	vmulpd	ymm0, ymm15, ymm12		;; A2 = R2 * cosine/sine				;	58-62

	vaddpd	ymm9, ymm9, ymm8		;; (ro4 + re4) Real #4					; 59-61
	vmulpd	ymm12, ymm7, ymm12		;; B2 = I2 * cosine/sine				;	59-63
	vmovapd	ymm8, [screg+4*64+32]		;; cosine/sine for R6/I6
	ystore	[srcreg+d2], ymm10		;; Save final R3					; 59

	vaddpd	ymm11, ymm11, ymm14		;; (ie4 + io4) Imag #4					; 60-62
	vmulpd	ymm14, ymm1, ymm8		;; A6 = R6 * cosine/sine				;	60-64
	ystore	[srcreg+d2+32], ymm13		;; Save final I3					; 60

	vsubpd	ymm4, ymm4, ymm2		;; A8 = A8 - I8						; 61-63
	vmulpd	ymm8, ymm3, ymm8		;; B6 = I6 * cosine/sine				;	61-65
	vmovapd	ymm13, [screg+2*64+32]		;; cosine/sine for R4/I4

	vaddpd	ymm6, ymm6, ymm5		;; B8 = B8 + R8						; 62-64
	vmulpd	ymm5, ymm9, ymm13		;; A4 = R4 * cosine/sine				;	62-66
	vmovapd	ymm2, [screg+3*64+32]		;; cosine/sine for R5/I5

	vsubpd	ymm0, ymm0, ymm7		;; A2 = A2 - I2						; 63-65
	vmulpd	ymm13, ymm11, ymm13		;; B4 = I4 * cosine/sine				;	63-67
	vmovapd	ymm7, YMM_TMPS[1*32]		;; Reload R5

	vaddpd	ymm12, ymm12, ymm15		;; B2 = B2 + R2						; 64-66
	vmulpd	ymm15, ymm7, ymm2		;; A5 = R5 * cosine/sine				;	64-68

	vsubpd	ymm14, ymm14, ymm3		;; A6 = A6 - I6						; 65-67
	vmovapd	ymm3, YMM_TMPS[0*32]		;; Reload I5
	vmulpd	ymm2, ymm3, ymm2		;; B5 = I5 * cosine/sine				;	65-69

	vaddpd	ymm8, ymm8, ymm1		;; B6 = B6 + R6						; 66-68
	vmovapd	ymm1, [screg+6*64]		;; sine for R8/I8
	vmulpd	ymm4, ymm4, ymm1		;; A8 = A8 * sine (final R8)				;	66-70

	vsubpd	ymm5, ymm5, ymm11		;; A4 = A4 - I4						; 67-69
	vmulpd	ymm6, ymm6, ymm1		;; B8 = B8 * sine (final I8)				;	67-71
	vmovapd	ymm11, [screg]			;; sine for R2/I2

	vaddpd	ymm13, ymm13, ymm9		;; B4 = B4 + R4						; 68-70
	vmulpd	ymm0, ymm0, ymm11		;; A2 = A2 * sine (final R2)				;	68-72
	vmovapd ymm1, [screg+4*64]		;; sine for R6/I6

	vsubpd	ymm15, ymm15, ymm3		;; A5 = A5 - I5						; 69-71
	vmulpd	ymm12, ymm12, ymm11		;; B2 = B2 * sine (final I2)				;	69-73
	vmovapd	ymm9, [screg+2*64]		;; sine for R4/I4

	vaddpd	ymm2, ymm2, ymm7		;; B5 = B5 + R5						; 70-72
	vmulpd	ymm14, ymm14, ymm1		;; A6 = A6 * sine (final R6)				;	70-74
	vmovapd	ymm3, [screg+3*64]		;; sine for R5/I5

	vmulpd	ymm8, ymm8, ymm1		;; B6 = B6 * sine (final I6)				;	71-75
	ystore	[srcreg+d4+d2+d1], ymm4		;; Save final R8					; 71

	vmulpd	ymm5, ymm5, ymm9		;; A4 = A4 * sine (final R4)				;	72-76
	ystore	[srcreg+d4+d2+d1+32], ymm6	;; Save final I8					; 72

	vmulpd	ymm13, ymm13, ymm9		;; B4 = B4 * sine (final I4)				;	73-77
	ystore	[srcreg+d1], ymm0		;; Save final R2					; 73

	vmulpd	ymm15, ymm15, ymm3		;; A5 = A5 * sine (final R5)				;	74-78
	ystore	[srcreg+d1+32], ymm12		;; Save final I2					; 74

	vmulpd	ymm2, ymm2, ymm3		;; B5 = B5 * sine (final I5)				;	75-79
	ystore	[srcreg+d4+d1], ymm14		;; Save final R6					; 75

	ystore	[srcreg+d4+d1+32], ymm8		;; Save final I6					; 76
	ystore	[srcreg+d2+d1], ymm5		;; Save final R4					; 77
	ystore	[srcreg+d2+d1+32], ymm13	;; Save final I4					; 78
	ystore	[srcreg+d4], ymm15		;; Save final R5					; 79
	ystore	[srcreg+d4+32], ymm2		;; Save final I5					; 80

	bump	srcreg, srcinc
	bump	screg, scinc
	ENDM

;; Haswell FMA3 version

IF (@INSTR(,%yarch,<FMA3>) NE 0)

yr8_8cl_16_reals_fft_preload MACRO
	ENDM

;; This is the Bulldozer version timed at 46.5 clocks.  Converting adds and subs to FMA3 did not seem to help.
yr8_8cl_16_reals_fft MACRO srcreg,srcinc,d1,d2,d4,screg,scinc,maxrpt,L1pt,L1pd

	vmovapd	ymm15, YMM_ONE
	vmovapd	ymm1, [srcreg+d1]		;; r2
	vmovapd	ymm2, [srcreg+d4+d2+d1+32]	;; r16
	vaddpd	ymm0, ymm1, ymm2		;; r2+r16						; 1-5			n 7
	yfmsubpd ymm1, ymm1, ymm15, ymm2	;; r2-r16						; 1-5			n 8

	vmovapd	ymm3, [srcreg+d4+d2+d1]		;; r8
	vmovapd	ymm4, [srcreg+d1+32]		;; r10
	vaddpd	ymm2, ymm3, ymm4		;; r8+r10						; 2-6			n 7
	yfmsubpd ymm3, ymm3, ymm15, ymm4	;; r8-r10						; 2-6			n 8

	vmovapd	ymm5, [srcreg+d2+d1]		;; r4
	vmovapd	ymm6, [srcreg+d4+d1+32]		;; r14
	vaddpd	ymm4, ymm5, ymm6		;; r4+r14						; 3-7			n 9
	yfmsubpd ymm5, ymm5, ymm15, ymm6	;; r4-r14						; 3-7			n 11

	vmovapd	ymm7, [srcreg+d4+d1]		;; r6
	vmovapd	ymm8, [srcreg+d2+d1+32]		;; r12
	vaddpd	ymm6, ymm7, ymm8		;; r6+r12						; 4-8			n 9
	yfmsubpd ymm7, ymm7, ymm15, ymm8	;; r6-r12						; 4-8			n 11

	vmovapd	ymm9, [srcreg+d2]		;; r3
	vmovapd	ymm10, [srcreg+d4+d2+32]	;; r15
	vaddpd	ymm8, ymm9, ymm10		;; r3+r15						; 5-9			n 12
	yfmsubpd ymm9, ymm9, ymm15, ymm10	;; r3-r15						; 5-9			n 13

	vmovapd	ymm11, [srcreg+d4+d2]		;; r7
	vmovapd	ymm12, [srcreg+d2+32]		;; r11
	vaddpd	ymm10, ymm11, ymm12		;; r7+r11						; 6-10			n 12
	yfmsubpd ymm11, ymm11, ymm15, ymm12	;; r7-r11						; 6-10			n 13

	vaddpd	ymm12, ymm0, ymm2		;; (r2+r16)+(r8+r10)					; 7-11			n 14
	yfmsubpd ymm0, ymm0, ymm15, ymm2	;; (r2+r16)-(r8+r10)					; 7-11			n 18
	vmovapd	ymm13, [srcreg]			;; r1

	vsubpd	ymm2, ymm1, ymm3		;; (r2-r16)-(r8-r10)					; 8-12			n 17
	yfmaddpd ymm1, ymm1, ymm15, ymm3	;; (r2-r16)+(r8-r10)					; 8-12			n 19
	vmovapd	ymm14, [srcreg+d4]		;; r5

	vaddpd	ymm3, ymm4, ymm6		;; (r4+r14)+(r6+r12)					; 9-13			n 14
	yfmsubpd ymm4, ymm4, ymm15, ymm6	;; (r4+r14)-(r6+r12)					; 9-13			n 18

	vaddpd	ymm13, ymm13, [srcreg+32]	;; r1+r9						; 10-14			n 16
	yfmaddpd ymm14, ymm14, ymm15, [srcreg+d4+32] ;; r5+r13						; 10-14			n 16

	vsubpd	ymm6, ymm5, ymm7		;; (r4-r14)-(r6-r12)					; 11-15			n 17
	yfmaddpd ymm5, ymm5, ymm15, ymm7	;; (r4-r14)+(r6-r12)					; 11-15			n 19
	L1prefetchw srcreg+L1pd, L1pt

	vsubpd	ymm7, ymm8, ymm10		;; (r3+r15)-(r7+r11)					; 12-16			n 20
	yfmaddpd ymm8, ymm8, ymm15, ymm10	;; (r3+r15)+(r7+r11)					; 12-16			n 22
	L1prefetchw srcreg+d1+L1pd, L1pt

	vaddpd	ymm10, ymm9, ymm11		;; (r3-r15)+(r7-r11)					; 13-17			n 21
	yfmsubpd ymm9, ymm9, ymm15, ymm11	;; (r3-r15)-(r7-r11) (i3o)				; 13-17			n 24

	vsubpd	ymm11, ymm12, ymm3		;; ((r2+r16)+(r8+r10)) - ((r4+r14)+(r6+r12)) (r3e)	; 14-18			n 23
	yfmaddpd ymm12, ymm12, ymm15, ymm3	;; ((r2+r16)+(r8+r10)) + ((r4+r14)+(r6+r12)) (r1B)	; 14-18

	vmovapd	ymm3, [srcreg]			;; r1
	vsubpd	ymm3, ymm3, [srcreg+32]		;; r1-r9						; 15-19			n 20
	ystore	[srcreg], ymm9			;; Temporarily save i3o					; 18
	vmovapd	ymm9, [srcreg+d4]		;; r5
	yfmsubpd ymm9, ymm9, ymm15, [srcreg+d4+32] ;; r5-r13						; 15-19			n 21

	ystore	[srcreg+32], ymm12		;; Save final real #1B					; 19
	vaddpd	ymm12, ymm13, ymm14		;; (r1+r9)+(r5+r13)					; 16-20			n 22
	yfmsubpd ymm13, ymm13, ymm15, ymm14	;; (r1+r9)-(r5+r13) (r3o)				; 16-20			n 23
	L1prefetchw srcreg+d2+L1pd, L1pt

	vaddpd	ymm14, ymm2, ymm6		;; ((r2-r16)-(r8-r10))+((r4-r14)-(r6-r12)) (i3e)	; 17-21			n 24
	yfmsubpd ymm2, ymm2, ymm15, ymm6	;; ((r2-r16)-(r8-r10))-((r4-r14)-(r6-r12)) (i5)		; 17-21			n 29

	vmovapd ymm15, YMM_P924_P383
	yfmaddpd ymm6, ymm15, ymm0, ymm4	;; .924/.383((r2+r16)-(r8+r10))+((r4+r14)-(r6+r12)) (r2e/.383) ; 18-22		n 25
	yfnmaddpd ymm4, ymm15, ymm4, ymm0	;; ((r2+r16)-(r8+r10))-.924/.383((r4+r14)-(r6+r12)) (r4e/.383) ; 18-22		n 27
	L1prefetchw srcreg+d2+d1+L1pd, L1pt

	yfmaddpd ymm0, ymm15, ymm5, ymm1	;; ((r2-r16)+(r8-r10))+.924/.383((r4-r14)+(r6-r12)) (i2e/.383) ; 19-23		n 26
	yfmsubpd ymm1, ymm15, ymm1, ymm5	;; .924/.383((r2-r16)+(r8-r10))-((r4-r14)+(r6-r12)) (i4e/.383) ; 19-23		n 28

	vmovapd	ymm15, YMM_SQRTHALF
	yfmaddpd ymm5, ymm15, ymm7, ymm3	;; (r1-r9) + .707((r3+r15)-(r7+r11)) (r2o)		; 20-24			n 25
	yfnmaddpd ymm7, ymm15, ymm7, ymm3	;; (r1-r9) - .707((r3+r15)-(r7+r11)) (r4o)		; 20-24			n 27
	L1prefetchw srcreg+d4+L1pd, L1pt

	yfmaddpd ymm3, ymm15, ymm10, ymm9	;; .707((r3-r15)+(r7-r11))+(r5-r13) (i2o)		; 21-25			n 26
	yfmsubpd ymm10, ymm15, ymm10, ymm9	;; .707((r3-r15)+(r7-r11))-(r5-r13) (i4o)		; 21-25			n 28

	vsubpd	ymm9, ymm12, ymm8		;; (r1+r9)+(r5+r13) - ((r3+r15)+(r7+r11)) (r5)		; 22-26			n 29
	yfmaddpd ymm12, ymm12, YMM_ONE, ymm8	;; (r1+r9)+(r5+r13) + ((r3+r15)+(r7+r11)) (r1A)		; 22-26
	L1prefetchw srcreg+d4+d1+L1pd, L1pt

	yfnmaddpd ymm8, ymm15, ymm11, ymm13	;; (ro3 - .707*re3) Real #7				; 23-27			n 30
	yfmaddpd ymm11, ymm15, ymm11, ymm13	;; (ro3 + .707*re3) Real #3				; 23-27			n 31

	vmovapd	ymm13, [srcreg]			;; Reload i3o
	ystore	[srcreg], ymm12			;; Final real #1A					; 27
	yfmsubpd ymm12, ymm15, ymm14, ymm13	;; (.707*ie3 - io3) Imag #7				; 24-28			n 30
	yfmaddpd ymm14, ymm15, ymm14, ymm13	;; (.707*ie3 + io3) Imag #3				; 24-28			n 31

	vmovapd ymm15, YMM_P383
	yfnmaddpd ymm13, ymm15, ymm6, ymm5	;; (ro2 - .383*re2) Real #8				; 25-29			n 32
	yfmaddpd ymm6, ymm15, ymm6, ymm5	;; (ro2 + .383*re2) Real #2				; 25-29			n 33

	yfmsubpd ymm5, ymm15, ymm0, ymm3	;; (.383*ie2 - io2) Imag #8				; 26-30			n 32
	yfmaddpd ymm0, ymm15, ymm0, ymm3	;; (.383*ie2 + io2) Imag #2				; 26-30			n 33
	L1prefetchw srcreg+d4+d2+L1pd, L1pt

	yfnmaddpd ymm3, ymm15, ymm4, ymm7	;; (ro4 - .383*re4) Real #6				; 27-31			n 34
	yfmaddpd ymm4, ymm15, ymm4, ymm7	;; (ro4 + .383*re4) Real #4				; 27-31			n 35
	L1prefetchw srcreg+d4+d2+d1+L1pd, L1pt

	yfmsubpd ymm7, ymm15, ymm1, ymm10	;; (.383*ie4 - io4) Imag #6				; 28-32			n 34 
	yfmaddpd ymm1, ymm15, ymm1, ymm10	;; (.383*ie4 + io4) Imag #4				; 28-32			n 35

	vmovapd	ymm15, [screg+3*64+32]		;; cosine/sine for R5/I5
	yfmsubpd ymm10, ymm9, ymm15, ymm2	;; A5 = R5 * cosine/sine - I5				; 29-33
	yfmaddpd ymm2, ymm2, ymm15, ymm9	;; B5 = I5 * cosine/sine + R5				; 29-33

	vmovapd	ymm15, [screg+5*64+32]		;; cosine/sine for R7/I7
	yfmsubpd ymm9, ymm8, ymm15, ymm12	;; A7 = R7 * cosine/sine - I7				; 30-34
	yfmaddpd ymm12, ymm12, ymm15, ymm8	;; B7 = I7 * cosine/sine + R7				; 30-34

	vmovapd	ymm15, [screg+64+32]		;; cosine/sine for R3/I3
	yfmsubpd ymm8, ymm11, ymm15, ymm14	;; A3 = R3 * cosine/sine - I3				; 31-35
	yfmaddpd ymm14, ymm14, ymm15, ymm11	;; B3 = I3 * cosine/sine + R3				; 31-35

	vmovapd	ymm15, [screg+6*64+32]		;; cosine/sine for R8/I8
	yfmsubpd ymm11, ymm13, ymm15, ymm5	;; A8 = R8 * cosine/sine - I8				; 32-26
	yfmaddpd ymm5, ymm5, ymm15, ymm13	;; B8 = I8 * cosine/sine + R8				; 32-36

	vmovapd	ymm15, [screg+32]		;; cosine/sine for R2/I2
	yfmsubpd ymm13, ymm6, ymm15, ymm0	;; A2 = R2 * cosine/sine - I2				; 33-37
	yfmaddpd ymm0, ymm0, ymm15, ymm6	;; B2 = I2 * cosine/sine + R2				; 33-37

	vmovapd	ymm15, [screg+3*64]		;; sine for R5/I5
	vmulpd	ymm10, ymm10, ymm15		;; A5 = A5 * sine (final R5)				; 34-38
	vmulpd	ymm2, ymm2, ymm15		;; B5 = B5 * sine (final I5)				; 34-38

	vmovapd	ymm15, [screg+4*64+32]		;; cosine/sine for R6/I6
	yfmsubpd ymm6, ymm3, ymm15, ymm7	;; A6 = R6 * cosine/sine - I6				; 35-39
	yfmaddpd ymm7, ymm7, ymm15, ymm3	;; B6 = I6 * cosine/sine + R6				; 35-39

	vmovapd	ymm15, [screg+5*64]		;; sine for R7/I7
	vmulpd	ymm9, ymm9, ymm15		;; A7 = A7 * sine (final R7)				; 36-40
	vmulpd	ymm12, ymm12, ymm15		;; B7 = B7 * sine (final I7)				; 36-40

	vmovapd	ymm15, [screg+2*64+32]		;; cosine/sine for R4/I4
	yfmsubpd ymm3, ymm4, ymm15, ymm1	;; A4 = R4 * cosine/sine - I4				; 37-41
	yfmaddpd ymm1, ymm1, ymm15, ymm4	;; B4 = I4 * cosine/sine + R4				; 37-41

	vmovapd	ymm15, [screg+64]		;; sine for R3/I3
	vmulpd	ymm8, ymm8, ymm15		;; A3 = A3 * sine (final R3)				; 38-42
	vmulpd	ymm14, ymm14, ymm15		;; B3 = B3 * sine (final I3)				; 38-42

	vmovapd	ymm15, [screg+6*64]		;; sine for R8/I8
	vmulpd	ymm11, ymm11, ymm15		;; A8 = A8 * sine (final R8)				; 39-43
	vmulpd	ymm5, ymm5, ymm15		;; B8 = B8 * sine (final I8)				; 39-43
	ystore	[srcreg+d4], ymm10		;; Save final R5					; 39

	vmovapd	ymm15, [screg]			;; sine for R2/I2
	vmulpd	ymm13, ymm13, ymm15		;; A2 = A2 * sine (final R2)				; 40-44
	vmulpd	ymm0, ymm0, ymm15		;; B2 = B2 * sine (final I2)				; 40-44
	ystore	[srcreg+d4+32], ymm2		;; Save final I5					; 39+1

	vmovapd ymm15, [screg+4*64]		;; sine for R6/I6
	vmulpd	ymm6, ymm6, ymm15		;; A6 = A6 * sine (final R6)				; 41-45
	vmulpd	ymm7, ymm7, ymm15		;; B6 = B6 * sine (final I6)				; 41-45
	ystore	[srcreg+d4+d2], ymm9		;; Save final R7					; 41

	vmovapd	ymm15, [screg+2*64]		;; sine for R4/I4
	vmulpd	ymm3, ymm3, ymm15		;; A4 = A4 * sine (final R4)				; 42-46
	vmulpd	ymm1, ymm1, ymm15		;; B4 = B4 * sine (final I4)				; 42-46
	ystore	[srcreg+d4+d2+32], ymm12	;; Save final I7					; 41+1

	ystore	[srcreg+d2], ymm8		;; Save final R3					; 43
	ystore	[srcreg+d2+32], ymm14		;; Save final I3					; 43+1
	ystore	[srcreg+d4+d2+d1], ymm11	;; Save final R8					; 44+1
	ystore	[srcreg+d4+d2+d1+32], ymm5	;; Save final I8					; 44+2
	ystore	[srcreg+d1], ymm13		;; Save final R2					; 45+2
	ystore	[srcreg+d1+32], ymm0		;; Save final I2					; 45+3
	ystore	[srcreg+d4+d1], ymm6		;; Save final R6					; 46+3
	ystore	[srcreg+d4+d1+32], ymm7		;; Save final I6					; 46+4
	ystore	[srcreg+d2+d1], ymm3		;; Save final R4					; 47+4
	ystore	[srcreg+d2+d1+32], ymm1		;; Save final I4					; 47+5

	bump	srcreg, srcinc
	bump	screg, scinc
	ENDM

ENDIF

ENDIF


;;
;; ************************************* 16-reals-last-unfft variants ******************************************
;;

;; These macros produce 16 reals after doing 4 levels of the inverse FFT applying
;; the sin/cos multipliers beforehand.  The input is 2 real and 7 complex numbers.

;; To calculate a 16-reals inverse FFT, we calculate 16 real values from 16 complex inputs in a brute force way.
;; First we note that the 16 complex values are computed from the 7 complex and 2 real inputs using Hermetian symmetry, thusly:
;; c1 = r1A + 0*i
;; c2 = r2 + i2*i
;; ...
;; c8 = r8 + i8*i
;; c9 = r1B + 0*i
;; c10 = r8 - i8*i
;; ...
;; c16 = r2 - i2*i 
;;
;; The brute force calculations are:
;;
;; c1 + c2 + ... + c16	*  w^-0000000000...
;; c1 + c2 + ... + c16	*  w^-0123456789A...
;; c1 + c2 + ... + c16	*  w^-02468ACE....
;;    ...
;; c1 + c2 + ... + c16	*  w^-...A987654321
;;
;; The sin/cos values (w = 16th root of unity) are:
;; w^-1 = .924 - .383i
;; w^-2 = .707 - .707i
;; w^-3 = .383 - .924i
;; w^-4 = 0 - 1i
;; w^-5 = -.383 - .924i
;; w^-6 = -.707 - .707i
;; w^-7 = -.924 - .383i
;; w^-8 = -1

;;
;; Applying the sin/cos values above, taking advantage of symmetry, and ignoring a lot of multiplies by 2:
;; r1     +(r2+r8)     +(r3+r7)     +(r4+r6) + r5 + r9
;; r1 +.924(r2-r8) +.707(r3-r7) +.383(r4-r6)      - r9 +.383(i2+i8) +.707(i3+i7) +.924(i4+i6) + i5
;; r1 +.707(r2+r8)              -.707(r4+r6) - r5 + r9 +.707(i2-i8)     +(i3-i7) +.707(i4-i6)
;; r1 +.383(r2-r8) -.707(r3-r7) -.924(r4-r6)      - r9 +.924(i2+i8) +.707(i3+i7) -.383(i4+i6) - i5
;; r1                  -(r3+r7)              + r5 + r9     +(i2-i8)                  -(i4-i6)
;; r1 -.383(r2-r8) -.707(r3-r7) +.924(r4-r6)      - r9 +.924(i2+i8) -.707(i3+i7) -.383(i4+i6) + i5
;; r1 -.707(r2+r8)              +.707(r4+r6) - r5 + r9 +.707(i2-i8)     -(i3-i7) +.707(i4-i6)
;; r1 -.924(r2-r8) +.707(r3-r7) -.383(r4-r6)      - r9 +.383(i2+i8) -.707(i3+i7) +.924(i4+i6) - i5
;; r1     -(r2+r8)     +(r3+r7)     -(r4+r6) + r5 + r9
;; ... r10 thru r16 are the same as r8 through r2 but with the sign of the imaginary component changed.
;;
;; Also remember that due to the funny way we do things reals input r1A = r1+r9 and r1B = r1-r9

;; Simplifying yields:
;;
;; r1/9 = r1o +/- r1e
;; r2/8 = r2o +/- r2e
;; r3/7 = r3o +/- .707*r3e
;; r4/6 = r4o +/- r4e
;;
;; r1o = r1+r9     +(r3+r7) + r5 
;; r2o = r1-r9 +.707(r3-r7)      
;; r3o = r1+r9              - r5 
;; r4o = r1-r9 -.707(r3-r7)      
;; r5  = r1+r9     -(r3+r7) + r5 
;;
;; r1e =     +(r2+r8)     +(r4+r6)
;; r2e = +.924(r2-r8) +.383(r4-r6)
;; r3e =     +(r2+r8)     -(r4+r6)
;; r4e = +.383(r2-r8) -.924(r4-r6)
;;
;; i2/8 = i2e +/- i2o
;; i3/7 = .707*i3e +/- i3o
;; i4/6 = i4e +/- i4o
;;
;; i2o =  +.707(i3+i7) + i5
;; i3o =      +(i3-i7)
;; i4o =  +.707(i3+i7) - i5
;;
;; i2e =  +.383(i2+i8) +.924(i4+i6)
;; i3e =      +(i2-i8)     +(i4-i6)
;; i4e =  +.924(i2+i8) -.383(i4+i6)
;; i5  =      +(i2-i8)     -(i4-i6)
;;
;; r2/16 = r2 +- i2
;; r3/15 = r3 +- i3
;; r4/14 = r4 +- i4
;; r5/13 = r5 +- i5
;; r6/12 = r6 +- i6
;; r7/11 = r7 +- i7
;; r8/10 = r8 +- i8

yr8_8cl_16_reals_unfft_preload MACRO
	ENDM

yr8_8cl_16_reals_unfft MACRO srcreg,srcinc,d1,d2,d4,screg,scinc,maxrpt,L1pt,L1pd

;; Apply the 7 twiddle factors.  Also do the first +/- on the reals so that
;; we have a perfect balance of adds and multiplies.

	vmovapd	ymm0, [screg+3*64+32]		;; cosine/sine for R5/I5
	vmovapd	ymm1, [srcreg+d4]		;; R5
	vmulpd	ymm2, ymm1, ymm0		;; A5 = R5 * cosine/sine
	vmovapd	ymm3, [srcreg+d4+32]		;; I5
	vmulpd	ymm0, ymm3, ymm0		;; B5 = I5 * cosine/sine
	vaddpd	ymm2, ymm2, ymm3		;; A5 = A5 + I5
	vsubpd	ymm0, ymm0, ymm1		;; B5 = B5 - R5
	vmovapd	ymm3, [screg+3*64]		;; sine for R5/I5
	vmulpd	ymm2, ymm2, ymm3		;; R5 = A5 * sine
	vmulpd	ymm0, ymm0, ymm3		;; I5 = B5 * sine
	ystore	YMM_TMPS[0*32], ymm2		;; Save R5
	ystore	YMM_TMPS[1*32], ymm0		;; Save I5

	vmovapd	ymm0, [screg+32]		;; cosine/sine for R2/I2
	vmovapd	ymm1, [srcreg+d1]		;; R2
	vmulpd	ymm2, ymm1, ymm0		;; A2 = R2 * cosine/sine
	vmovapd	ymm4, [screg+6*64+32]		;; cosine/sine for R8/I8
	vmovapd	ymm5, [srcreg+d4+d2+d1]		;; R8
	vmulpd	ymm6, ymm5, ymm4		;; A8 = R8 * cosine/sine
	vmovapd	ymm3, [srcreg+d1+32]		;; I2
	vaddpd	ymm2, ymm2, ymm3		;; A2 = A2 + I2
	vmulpd	ymm3, ymm3, ymm0		;; B2 = I2 * cosine/sine
	vmovapd	ymm7, [srcreg+d4+d2+d1+32]	;; I8
	vaddpd	ymm6, ymm6, ymm7		;; A8 = A8 + I8
	vmulpd	ymm7, ymm7, ymm4		;; B8 = I8 * cosine/sine
	vsubpd	ymm3, ymm3, ymm1		;; B2 = B2 - R2
	vmovapd	ymm0, [screg]			;; sine for R2/I2
	vmulpd	ymm2, ymm2, ymm0		;; R2 = A2 * sine
	vsubpd	ymm7, ymm7, ymm5		;; B8 = B8 - R8
	vmovapd	ymm4, [screg+6*64]		;; sine for R8/I8
	vmulpd	ymm6, ymm6, ymm4		;; R8 = A8 * sine
	vmulpd	ymm3, ymm3, ymm0		;; I2 = B2 * sine
	vmulpd	ymm7, ymm7, ymm4		;; I8 = B8 * sine
	vaddpd	ymm0, ymm2, ymm6		;; R2+R8
	vsubpd	ymm2, ymm2, ymm6		;; R2-R8
	vaddpd	ymm1, ymm3, ymm7		;; I2+I8
	vsubpd	ymm3, ymm3, ymm7		;; I2-I8
	ystore	YMM_TMPS[2*32], ymm0		;; Save R2+R8
	ystore	YMM_TMPS[3*32], ymm2		;; Save R2-R8
	ystore	YMM_TMPS[4*32], ymm1		;; Save I2+I8
	ystore	YMM_TMPS[5*32], ymm3		;; Save I2-I8

	vmovapd	ymm0, [screg+64+32]		;; cosine/sine for R3/I3
	vmovapd	ymm1, [srcreg+d2]		;; R3
	vmulpd	ymm2, ymm1, ymm0		;; A3 = R3 * cosine/sine
	vmovapd	ymm4, [screg+5*64+32]		;; cosine/sine for R7/I7
	vmovapd	ymm5, [srcreg+d4+d2]		;; R7
	vmulpd	ymm6, ymm5, ymm4		;; A7 = R7 * cosine/sine
	vmovapd	ymm3, [srcreg+d2+32]		;; I3
	vaddpd	ymm2, ymm2, ymm3		;; A3 = A3 + I3
	vmulpd	ymm3, ymm3, ymm0		;; B3 = I3 * cosine/sine
	vmovapd	ymm7, [srcreg+d4+d2+32]		;; I7
	vaddpd	ymm6, ymm6, ymm7		;; A7 = A7 + I7
	vmulpd	ymm7, ymm7, ymm4		;; B7 = I7 * cosine/sine
	vsubpd	ymm3, ymm3, ymm1		;; B3 = B3 - R3
	vmovapd	ymm0, [screg+64]		;; sine for R3/I3
	vmulpd	ymm2, ymm2, ymm0		;; R3 = A3 * sine
	vsubpd	ymm7, ymm7, ymm5		;; B7 = B7 - R7
	vmovapd	ymm4, [screg+5*64]		;; sine for R7/I7
	vmulpd	ymm6, ymm6, ymm4		;; R7 = A7 * sine
	vmulpd	ymm3, ymm3, ymm0		;; I3 = B3 * sine
	vmulpd	ymm7, ymm7, ymm4		;; I7 = B7 * sine
	vaddpd	ymm0, ymm2, ymm6		;; R3+R7
	vsubpd	ymm2, ymm2, ymm6		;; R3-R7
	vaddpd	ymm1, ymm3, ymm7		;; I3+I7
	vsubpd	ymm3, ymm3, ymm7		;; I3-I7
	ystore	YMM_TMPS[6*32], ymm0		;; Save R3+R7
	ystore	YMM_TMPS[7*32], ymm2		;; Save R3-R7
	ystore	YMM_TMPS[8*32], ymm1		;; Save I3+I7
	ystore	YMM_TMPS[9*32], ymm3		;; Save I3-I7 (i3o)

	vmovapd	ymm0, [screg+2*64+32]		;; cosine/sine for R4/I4
	vmovapd	ymm1, [srcreg+d2+d1]		;; R4
	vmulpd	ymm2, ymm1, ymm0		;; A4 = R4 * cosine/sine
	vmovapd	ymm4, [screg+4*64+32]		;; cosine/sine for R6/I6
	vmovapd	ymm5, [srcreg+d4+d1]		;; R6
	vmulpd	ymm6, ymm5, ymm4		;; A6 = R6 * cosine/sine
	vmovapd	ymm3, [srcreg+d2+d1+32]		;; I4
	vaddpd	ymm2, ymm2, ymm3		;; A4 = A4 + I4
	vmulpd	ymm3, ymm3, ymm0		;; B4 = I4 * cosine/sine
	vmovapd	ymm7, [srcreg+d4+d1+32]		;; I6
	vaddpd	ymm6, ymm6, ymm7		;; A6 = A6 + I6
	vmulpd	ymm7, ymm7, ymm4		;; B6 = I6 * cosine/sine
	vsubpd	ymm3, ymm3, ymm1		;; B4 = B4 - R4
	vmovapd	ymm0, [screg+2*64]		;; sine for R4/I4
	vmulpd	ymm2, ymm2, ymm0		;; R4 = A4 * sine
	vsubpd	ymm7, ymm7, ymm5		;; B6 = B6 - R6
	vmovapd	ymm4, [screg+4*64]		;; sine for R6/I6
	vmulpd	ymm6, ymm6, ymm4		;; R6 = A6 * sine
	vmulpd	ymm3, ymm3, ymm0		;; I4 = B4 * sine
	vmulpd	ymm7, ymm7, ymm4		;; I6 = B6 * sine
	vaddpd	ymm0, ymm2, ymm6		;; R4+R6
	vsubpd	ymm2, ymm2, ymm6		;; R4-R6
	vaddpd	ymm1, ymm3, ymm7		;; I4+I6
	vsubpd	ymm3, ymm3, ymm7		;; I4-I6
	ystore	YMM_TMPS[10*32], ymm0		;; Save R4+R6
	ystore	YMM_TMPS[11*32], ymm1		;; Save I4+I6
	ystore	YMM_TMPS[12*32], ymm3		;; Save I4-I6

	;; Do the 16 reals inverse FFT

	vmovapd	ymm0, YMM_TMPS[7*32]		;; r3-r7
	vmovapd ymm7, YMM_SQRTHALF
	vmulpd	ymm0, ymm7, ymm0		;; .707(r3-r7)						;	1-5

	vmovapd	ymm1, YMM_TMPS[3*32]		;; r2-r8
	vmovapd ymm6, YMM_P383
	vmulpd	ymm3, ymm6, ymm1		;; .383(r2-r8)						;	2-6

	vmovapd ymm5, YMM_P924
	vmulpd	ymm1, ymm5, ymm1		;; .924(r2-r8)						;	3-7

	vmovapd	ymm4, [srcreg+32]		;; r1-r9
	vmulpd	ymm5, ymm5, ymm2		;; .924(r4-r6)						;	4-8

	vmulpd	ymm2, ymm6, ymm2		;; .383(r4-r6)						;	5-9

	vsubpd	ymm6, ymm4, ymm0		;; (r1-r9) - .707(r3-r7) (r4o)				; 6-8
	vaddpd	ymm4, ymm4, ymm0		;; (r1-r9) + .707(r3-r7) (r2o)				; 7-9

	vmovapd	ymm0, YMM_TMPS[8*32]		;; i3+i7
	vmulpd	ymm7, ymm7, ymm0		;; .707(i3+i7)						;	6-10
	vmovapd	ymm0, YMM_TMPS[4*32]		;; i2+i8
	ystore	YMM_TMPS[3*32], ymm4		;; Save r2o						; 10
	vmulpd	ymm4, ymm0, YMM_P924		;; .924(i2+i8)						;	7-11

	vsubpd	ymm3, ymm3, ymm5		;; .383(r2-r8)-.924(r4-r6) (r4e)			; 9-11

	vmovapd ymm5, YMM_P383
	vmulpd	ymm0, ymm5, ymm0 		;; .383(i2+i8)						;	8-12

	vaddpd	ymm1, ymm1, ymm2		;; .924(r2-r8)+.383(r4-r6) (r2e)			; 10-12

	vmovapd	ymm2, YMM_TMPS[11*32]		;; i4+i6
	vmulpd	ymm5, ymm5, ymm2 		;; .383(i4+i6)						;	9-13
	vmulpd	ymm2, ymm2, YMM_P924		;; .924(i4+i6)						;	10-14

	vsubpd	ymm4, ymm4, ymm5		;; .924(i2+i8)-.383(i4+i6) (i4e)			; 14-16
	L1prefetchw srcreg+L1pd, L1pt

	vaddpd	ymm0, ymm0, ymm2		;; .383(i2+i8)+.924(i4+i6) (i2e)			; 15-17

	vmovapd	ymm5, YMM_TMPS[1*32]		;; i5
	vsubpd	ymm2, ymm7, ymm5		;; .707(i3+i7)-i5 (i4o)					; 11-13

	vaddpd	ymm7, ymm7, ymm5		;; .707(i3+i7)+i5 (i2o)					; 12-14
	L1prefetchw srcreg+d1+L1pd, L1pt

	vaddpd	ymm5, ymm6, ymm3		;; r4o + r4e (r4)					; 13-15

	vsubpd	ymm6, ymm6, ymm3		;; r4o - r4e (r6)					; 16-18
	L1prefetchw srcreg+d2+L1pd, L1pt

	vaddpd	ymm3, ymm4, ymm2		;; imag-cols row #4 (i4e + i4o)				; 17-19

	vsubpd	ymm4, ymm4, ymm2		;; imag-cols row #6 (i4e - i4o)				; 18-20
	L1prefetchw srcreg+d2+d1+L1pd, L1pt

	vaddpd	ymm2, ymm0, ymm7		;; imag-cols row #2 (i2e + i2o)				; 19-21

	vsubpd	ymm0, ymm0, ymm7		;; imag-cols row #8 (i2e - i2o)				; 20-22
	L1prefetchw srcreg+d4+L1pd, L1pt

	vsubpd	ymm7, ymm5, ymm3		;; (real#4 - imag#4) final R14				; 21-23

	vaddpd	ymm5, ymm5, ymm3		;; (real#4 + imag#4) final R4				; 22-24

	vmovapd	ymm3, YMM_TMPS[3*32]		;; Load r2o
	ystore	[srcreg+d4+d1+32], ymm7		;; Save R14						; 24
	vaddpd	ymm7, ymm3, ymm1		;; r2o + r2e (r2)					; 23-25

	vsubpd	ymm3, ymm3, ymm1		;; r2o - r2e (r8)					; 24-26
	L1prefetchw srcreg+d4+d1+L1pd, L1pt

	vsubpd	ymm1, ymm6, ymm4		;; (real#6 - imag#6) final R12				; 25-27
	ystore	[srcreg+d2+d1], ymm5		;; Save R4						; 25

	vaddpd	ymm6, ymm6, ymm4		;; (real#6 + imag#6) final R6				; 26-28
	vmovapd	ymm5, YMM_TMPS[2*32]		;; r2+r8

	vsubpd	ymm4, ymm7, ymm2		;; (real#2 - imag#2) final R16				; 27-29
	L1prefetchw srcreg+d4+d2+L1pd, L1pt

	vaddpd	ymm7, ymm7, ymm2		;; (real#2 + imag#2) final R2				; 28-30
	vmovapd	ymm2, YMM_TMPS[10*32]		;; r4+r6
	ystore	[srcreg+d2+d1+32], ymm1		;; Save R12						; 28

	vsubpd	ymm1, ymm3, ymm0		;; (real#8 - imag#8) final R10				; 29-31
	ystore	[srcreg+d4+d1], ymm6		;; Save R6						; 29
	vmovapd	ymm6, [srcreg]			;; r1+r9

	vaddpd	ymm3, ymm3, ymm0		;; (real#8 + imag#8) final R8				; 30-32
	vmovapd	ymm0, YMM_TMPS[0*32]		;; r5
	ystore	[srcreg+d4+d2+d1+32], ymm4	;; Save R16						; 30

	vsubpd	ymm4, ymm5, ymm2		;; (r2+r8)-(r4+r6) (r3e)				; 31-33			n 34
	ystore	[srcreg+d1], ymm7		;; Save R2						; 31
	vmovapd	ymm7, YMM_SQRTHALF

	vaddpd	ymm5, ymm5, ymm2		;; (r2+r8)+(r4+r6) (r1e)				; 32-34			n 41
	ystore	[srcreg+d1+32], ymm1		;; Save R10						; 32
	vmovapd	ymm1, YMM_TMPS[5*32]		;; i2-i8

	vaddpd	ymm2, ymm6, ymm0		;; (r1+r9) + r5						; 33-35			n 37
	ystore	[srcreg+d4+d2+d1], ymm3		;; Save R8						; 33
	vmovapd	ymm3, YMM_TMPS[12*32]		;; i4-i6

	vsubpd	ymm6, ymm6, ymm0		;; (r1+r9) - r5 (r3o)					; 34-36			n 39			
	vmulpd	ymm4, ymm7, ymm4		;; .707*r3e						;	34-38		n 39

	vaddpd	ymm0, ymm1, ymm3		;; (i2-i8)+(i4-i6) (i3e)				; 35-37			n 38
	L1prefetchw srcreg+d4+d2+d1+L1pd, L1pt

	vsubpd	ymm1, ymm1, ymm3		;; (i2-i8)-(i4-i6) (i5)					; 36-38			n 45

	vmovapd	ymm3, YMM_TMPS[6*32]		;; r3+r7
	vmulpd	ymm0, ymm7, ymm0		;; .707*i3e						;	38-42		n 43
	vaddpd	ymm7, ymm2, ymm3		;; (r1+r9)+r5 + (r3+r7)  (r1o)				; 37-39			n 41

	vsubpd	ymm2, ymm2, ymm3		;; (r1+r9)+r5 - (r3+r7)  (r5)				; 38-40			n 45

	vaddpd	ymm3, ymm6, ymm4		;; r3o + r3e (r3)					; 39-41

	vsubpd	ymm6, ymm6, ymm4		;; r3o - r3e (r7)					; 40-42

	vaddpd	ymm4, ymm7, ymm5		;; r1o + r1e (final R1)					; 41-43

	vsubpd	ymm7, ymm7, ymm5		;; r1o - r1e (final R9)					; 42-44

	vmovapd	ymm5, YMM_TMPS[9*32]		;; Load i3o
	ystore	[srcreg], ymm4			;; Save final R1					; 44
	vaddpd	ymm4, ymm0, ymm5		;; imag-cols row #3 (i3e + i3o)				; 43-45

	vsubpd	ymm0, ymm0, ymm5		;; imag-cols row #7 (i3e - i3o)				; 44-46

	vsubpd	ymm5, ymm2, ymm1		;; (real#5 - imag#5) final R13				; 45-47
	ystore	[srcreg+32], ymm7		;; Save final R9					; 45

	vaddpd	ymm2, ymm2, ymm1		;; (real#5 + imag#5) final R5				; 46-48

	vsubpd	ymm1, ymm3, ymm4		;; (real#3 - imag#3) final R15				; 47-49

	vaddpd	ymm3, ymm3, ymm4		;; (real#3 + imag#3) final R3				; 48-50
	ystore	[srcreg+d4+32], ymm5		;; Save R13						; 48

	vsubpd	ymm4, ymm6, ymm0		;; (real#7 - imag#7) final R11				; 49-51
	ystore	[srcreg+d4], ymm2		;; Save R5						; 49

	vaddpd	ymm6, ymm6, ymm0		;; (real#7 + imag#7) final R7				; 50-52
	ystore	[srcreg+d4+d2+32], ymm1		;; Save R15						; 50

	ystore	[srcreg+d2], ymm3		;; Save R3						; 51
	ystore	[srcreg+d2+32], ymm4		;; Save R11						; 52
	ystore	[srcreg+d4+d2], ymm6		;; Save R7						; 53

	bump	srcreg, srcinc
	bump	screg, scinc
	ENDM

;; 64-bit version

IFDEF X86_64

yr8_8cl_16_reals_unfft_preload MACRO
	ENDM

yr8_8cl_16_reals_unfft MACRO srcreg,srcinc,d1,d2,d4,screg,scinc,maxrpt,L1pt,L1pd

	vmovapd	ymm0, [screg+32]		;; cosine/sine for R2/I2
	vmovapd	ymm1, [srcreg+d1]		;; R2
	vmulpd	ymm2, ymm1, ymm0		;; A2 = R2 * cosine/sine				;	1-5

	vmovapd	ymm3, [srcreg+d1+32]		;; I2
	vmulpd	ymm0, ymm3, ymm0		;; B2 = I2 * cosine/sine				;	2-6

	vmovapd	ymm4, [screg+6*64+32]		;; cosine/sine for R8/I8
	vmovapd	ymm5, [srcreg+d4+d2+d1]		;; R8
	vmulpd	ymm6, ymm5, ymm4		;; A8 = R8 * cosine/sine				;	3-7

	vmovapd	ymm7, [srcreg+d4+d2+d1+32]	;; I8
	vmulpd	ymm4, ymm7, ymm4		;; B8 = I8 * cosine/sine				;	4-8

	vmovapd	ymm8, [screg+2*64+32]		;; cosine/sine for R4/I4
	vmovapd	ymm9, [srcreg+d2+d1]		;; R4
	vmulpd	ymm10, ymm9, ymm8		;; A4 = R4 * cosine/sine				;	5-9

	vmovapd	ymm11, [srcreg+d2+d1+32]	;; I4
	vaddpd	ymm2, ymm2, ymm3		;; A2 = A2 + I2						; 6-8
	vmulpd	ymm8, ymm11, ymm8		;; B4 = I4 * cosine/sine				;	6-10

	vmovapd	ymm12, [screg+4*64+32]		;; cosine/sine for R6/I6
	vmovapd	ymm13, [srcreg+d4+d1]		;; R6
	vsubpd	ymm0, ymm0, ymm1		;; B2 = B2 - R2						; 7-9
	vmulpd	ymm14, ymm13, ymm12		;; A6 = R6 * cosine/sine				;	7-11

	vmovapd	ymm15, [srcreg+d4+d1+32]	;; I6
	vaddpd	ymm6, ymm6, ymm7		;; A8 = A8 + I8						; 8-10
	vmulpd	ymm12, ymm15, ymm12		;; B6 = I6 * cosine/sine				;	8-12

	vmovapd	ymm3, [screg]			;; sine for R2/I2
	vsubpd	ymm4, ymm4, ymm5		;; B8 = B8 - R8						; 9-11
	vmulpd	ymm2, ymm2, ymm3		;; R2 = A2 * sine					;	9-13

	vmovapd	ymm1, [screg+6*64]		;; sine for R8/I8
	vaddpd	ymm10, ymm10, ymm11		;; A4 = A4 + I4						; 10-12
	vmulpd	ymm0, ymm0, ymm3		;; I2 = B2 * sine					;	10-14

	vmovapd	ymm7, [screg+2*64]		;; sine for R4/I4
	vsubpd	ymm8, ymm8, ymm9		;; B4 = B4 - R4						; 11-13
	vmulpd	ymm6, ymm6, ymm1		;; R8 = A8 * sine					;	11-15

	vmovapd	ymm5, [screg+4*64]		;; sine for R6/I6
	vaddpd	ymm14, ymm14, ymm15		;; A6 = A6 + I6						; 12-14
	vmulpd	ymm4, ymm4, ymm1		;; I8 = B8 * sine					;	12-16

	vmovapd	ymm11, [screg+64+32]		;; cosine/sine for R3/I3
	vsubpd	ymm12, ymm12, ymm13		;; B6 = B6 - R6						; 13-15
	vmulpd	ymm10, ymm10, ymm7		;; R4 = A4 * sine					;	13-17

	vmovapd	ymm3, [srcreg+d2]		;; R3
	vmulpd	ymm8, ymm8, ymm7		;; I4 = B4 * sine					;	14-18

	vmovapd	ymm9, [srcreg+d2+32]		;; I3
	vmulpd	ymm14, ymm14, ymm5		;; R6 = A6 * sine					;	15-19

	vmovapd	ymm15, [screg+5*64+32]		;; cosine/sine for R7/I7
	vsubpd	ymm7, ymm2, ymm6		;; R2-R8						; 16-18			n 21
	vmulpd	ymm12, ymm12, ymm5		;; I6 = B6 * sine					;	16-20

	vmovapd	ymm1, [srcreg+d4+d2]		;; R7
	vaddpd	ymm2, ymm2, ymm6		;; R2+R8						; 17-19			n 30
	vmulpd	ymm6, ymm3, ymm11		;; A3 = R3 * cosine/sine				;	17-21

	vaddpd	ymm5, ymm0, ymm4		;; I2+I8						; 18-20			n 31
	vmulpd	ymm11, ymm9, ymm11		;; B3 = I3 * cosine/sine				;	18-22

	vsubpd	ymm0, ymm0, ymm4		;; I2-I8						; 19-21			n 42
	vmulpd	ymm4, ymm1, ymm15		;; A7 = R7 * cosine/sine				;	19-23

	vsubpd	ymm13, ymm10, ymm14		;; R4-R6						; 20-22			n 23
	vaddpd	ymm10, ymm10, ymm14		;; R4+R6						; 21-23			n 30
	vmovapd	ymm14, [srcreg+d4+d2+32]	;; I7
	vmulpd	ymm15, ymm14, ymm15		;; B7 = I7 * cosine/sine				;	20-24
	vaddpd	ymm6, ymm6, ymm9		;; A3 = A3 + I3						; 22-24
	vmulpd	ymm9, ymm7, YMM_P383		;; .383(r2-r8)						;	21-25		n 28
	vmulpd	ymm7, ymm7, YMM_P924		;; .924(r2-r8)						;	22-26		n 29

	vsubpd	ymm11, ymm11, ymm3		;; B3 = B3 - R3						; 23-25
	vmulpd	ymm3, ymm13, YMM_P924		;; .924(r4-r6)						;	23-27		n 28

	vaddpd	ymm4, ymm4, ymm14		;; A7 = A7 + I7						; 24-26
	vmulpd	ymm13, ymm13, YMM_P383		;; .383(r4-r6)						;	24-28		n 29

	vmovapd	ymm14, [screg+64]		;; sine for R3/I3
	vsubpd	ymm15, ymm15, ymm1		;; B7 = B7 - R7						; 25-27
	vmulpd	ymm6, ymm6, ymm14		;; R3 = A3 * sine					;	25-29

	vaddpd	ymm1, ymm8, ymm12		;; I4+I6						; 26-28			n 33
	vmulpd	ymm11, ymm11, ymm14		;; I3 = B3 * sine					;	26-30

	vmovapd	ymm14, [screg+5*64]		;; sine for R7/I7
	vsubpd	ymm8, ymm8, ymm12		;; I4-I6						; 27-29			n 42
	vmulpd	ymm4, ymm4, ymm14		;; R7 = A7 * sine					;	27-31

	vmovapd	ymm12, [screg+3*64+32]		;; cosine/sine for R5/I5
	vsubpd	ymm9, ymm9, ymm3		;; .383(r2-r8)-.924(r4-r6) (r4e)			; 28-30			n 44
	vmulpd	ymm15, ymm15, ymm14		;; I7 = B7 * sine					;	28-32

	vmovapd	ymm3, [srcreg+d4+32]		;; I5
	vaddpd	ymm7, ymm7, ymm13		;; .924(r2-r8)+.383(r4-r6) (r2e)			; 29-31			n 48
	vmulpd	ymm14, ymm3, ymm12		;; B5 = I5 * cosine/sine				;	29-33

	vsubpd	ymm13, ymm2, ymm10		;; (r2+r8)-(r4+r6) (r3e)				; 30-32			n 36
	vaddpd	ymm2, ymm2, ymm10		;; (r2+r8)+(r4+r6) (r1e)				; 31-33			n 68
	vmovapd	ymm10, [srcreg+d4]		;; R5
	vmulpd	ymm12, ymm10, ymm12		;; A5 = R5 * cosine/sine				;	30-34
	ystore	YMM_TMPS[0*32], ymm2		;; Save r1e						; 34
	vsubpd	ymm2, ymm6, ymm4		;; R3-R7						; 32-34			n 35
	vaddpd	ymm6, ymm6, ymm4		;; R3+R7						; 33-35			n 64
	vmulpd	ymm4, ymm5, YMM_P924		;; .924(i2+i8)						;	31-35		n 38
	vmulpd	ymm5, ymm5, YMM_P383 		;; .383(i2+i8)						;	32-36		n 39
	ystore	YMM_TMPS[1*32], ymm6		;; Save R3+R7						; 36
	vaddpd	ymm6, ymm11, ymm15		;; I3+I7						; 34-36			n 37
	vsubpd	ymm11, ymm11, ymm15		;; I3-I7 (i3o)						; 35-37			n 70
	vmulpd	ymm15, ymm1, YMM_P383 		;; .383(i4+i6)						;	33-37		n 38
	vmulpd	ymm1, ymm1, YMM_P924		;; .924(i4+i6)						;	34-38		n 39
	vsubpd	ymm14, ymm14, ymm10		;; B5 = B5 - R5						; 36-38
	vmovapd	ymm10, YMM_SQRTHALF
	vmulpd	ymm2, ymm10, ymm2		;; .707(r3-r7)						;	35-39		n 40
	vmulpd	ymm13, ymm10, ymm13		;; .707*r3e						;	36-40		n 66

	vaddpd	ymm12, ymm12, ymm3		;; A5 = A5 + I5						; 37-39
	vmulpd	ymm6, ymm10, ymm6		;; .707(i3+i7)						;	37-41		n 46

	vmovapd	ymm3, [screg+3*64]		;; sine for R5/I5
	vsubpd	ymm4, ymm4, ymm15		;; .924(i2+i8)-.383(i4+i6) (i4e)			; 38-40			n 50

	vmovapd	ymm15, [srcreg+32]		;; r1-r9
	vaddpd	ymm5, ymm5, ymm1		;; .383(i2+i8)+.924(i4+i6) (i2e)			; 39-41			n 52
	vmulpd	ymm14, ymm14, ymm3		;; I5 = B5 * sine					;	39-43		n 46

	vsubpd	ymm1, ymm15, ymm2		;; (r1-r9) - .707(r3-r7) (r4o)				; 40-42			n 44
	vmulpd	ymm12, ymm12, ymm3		;; R5 = A5 * sine					;	40-44		n 60

	vmovapd	ymm3, [srcreg]			;; r1+r9
	vaddpd	ymm15, ymm15, ymm2		;; (r1-r9) + .707(r3-r7) (r2o)				; 41-43			n 48

	vaddpd	ymm2, ymm0, ymm8		;; (i2-i8)+(i4-i6) (i3e)				; 42-44			n 45
	L1prefetchw srcreg+L1pd, L1pt

	vsubpd	ymm0, ymm0, ymm8		;; (i2-i8)-(i4-i6) (new i5)				; 43-45			n 72

	vaddpd	ymm8, ymm1, ymm9		;; r4o + r4e (r4)					; 44-46			n 54
	L1prefetchw srcreg+d1+L1pd, L1pt

	vsubpd	ymm1, ymm1, ymm9		;; r4o - r4e (r6)					; 45-47			n 56
	vmulpd	ymm2, ymm10, ymm2		;; .707*i3e						;	45-49		n 70

	vmovapd	ymm10, YMM_TMPS[1*32]		;; Load r3+r7
	vsubpd	ymm9, ymm6, ymm14		;; .707(i3+i7)-i5 (i4o)					; 46-48			n 50

	vaddpd	ymm6, ymm6, ymm14		;; .707(i3+i7)+i5 (i2o)					; 47-49			n 52
	L1prefetchw srcreg+d2+L1pd, L1pt

	vaddpd	ymm14, ymm15, ymm7		;; r2o + r2e (r2)					; 48-50			n 58

	vsubpd	ymm15, ymm15, ymm7		;; r2o - r2e (r8)					; 49-51			n 62
	L1prefetchw srcreg+d2+d1+L1pd, L1pt

	vaddpd	ymm7, ymm4, ymm9		;; imag-cols row #4 (i4e + i4o)				; 50-52			n 54

	vsubpd	ymm4, ymm4, ymm9		;; imag-cols row #6 (i4e - i4o)				; 51-53			n 56
	L1prefetchw srcreg+d4+L1pd, L1pt

	vaddpd	ymm9, ymm5, ymm6		;; imag-cols row #2 (i2e + i2o)				; 52-54			n 58

	vsubpd	ymm5, ymm5, ymm6		;; imag-cols row #8 (i2e - i2o)				; 53-55			n 62
	L1prefetchw srcreg+d4+d1+L1pd, L1pt

	vsubpd	ymm6, ymm8, ymm7		;; (real#4 - imag#4) final R14				; 54-56

	vaddpd	ymm8, ymm8, ymm7		;; (real#4 + imag#4) final R4				; 55-57
	L1prefetchw srcreg+d4+d2+L1pd, L1pt

	vsubpd	ymm7, ymm1, ymm4		;; (real#6 - imag#6) final R12				; 56-58

	vaddpd	ymm1, ymm1, ymm4		;; (real#6 + imag#6) final R6				; 57-59
	ystore	[srcreg+d4+d1+32], ymm6		;; Save R14						; 57

	vmovapd	ymm6, YMM_TMPS[0*32]		;; Load r1e
	vsubpd	ymm4, ymm14, ymm9		;; (real#2 - imag#2) final R16				; 58-60
	ystore	[srcreg+d2+d1], ymm8		;; Save R4						; 58

	vaddpd	ymm14, ymm14, ymm9		;; (real#2 + imag#2) final R2				; 59-61
	ystore	[srcreg+d2+d1+32], ymm7		;; Save R12						; 59

	vaddpd	ymm9, ymm3, ymm12		;; (r1+r9) + r5						; 60-62			n 64
	ystore	[srcreg+d4+d1], ymm1		;; Save R6						; 60

	vsubpd	ymm3, ymm3, ymm12		;; (r1+r9) - r5 (r3o)					; 61-63			n 66
	ystore	[srcreg+d4+d2+d1+32], ymm4	;; Save R16						; 61

	vsubpd	ymm12, ymm15, ymm5		;; (real#8 - imag#8) final R10				; 62-64
	ystore	[srcreg+d1], ymm14		;; Save R2						; 62

	vaddpd	ymm15, ymm15, ymm5		;; (real#8 + imag#8) final R8				; 63-65

	vaddpd	ymm5, ymm9, ymm10		;; (r1+r9)+r5 + (r3+r7)  (r1o)				; 64-66			n 68

	vsubpd	ymm9, ymm9, ymm10		;; (r1+r9)+r5 - (r3+r7)  (new r5)			; 65-67			n 72
	ystore	[srcreg+d1+32], ymm12		;; Save R10						; 65

	vaddpd	ymm10, ymm3, ymm13		;; r3o + r3e (r3)					; 66-68			n 74
	ystore	[srcreg+d4+d2+d1], ymm15	;; Save R8						; 66

	vsubpd	ymm3, ymm3, ymm13		;; r3o - r3e (r7)					; 67-69			n 76
	L1prefetchw srcreg+d4+d2+d1+L1pd, L1pt

	vaddpd	ymm13, ymm5, ymm6		;; r1o + r1e (final R1)					; 68-70

	vsubpd	ymm5, ymm5, ymm6		;; r1o - r1e (final R9)					; 69-71

	vaddpd	ymm6, ymm2, ymm11		;; imag-cols row #3 (i3e + i3o)				; 70-72			n 74

	vsubpd	ymm2, ymm2, ymm11		;; imag-cols row #7 (i3e - i3o)				; 71-73			n 76
	ystore	[srcreg], ymm13			;; Save final R1					; 71

	vsubpd	ymm11, ymm9, ymm0		;; (real#5 - imag#5) final R13				; 72-74
	ystore	[srcreg+32], ymm5		;; Save final R9					; 72

	vaddpd	ymm9, ymm9, ymm0		;; (real#5 + imag#5) final R5				; 73-75

	vsubpd	ymm0, ymm10, ymm6		;; (real#3 - imag#3) final R15				; 74-76

	vaddpd	ymm10, ymm10, ymm6		;; (real#3 + imag#3) final R3				; 75-77
	ystore	[srcreg+d4+32], ymm11		;; Save R13						; 75

	vsubpd	ymm6, ymm3, ymm2		;; (real#7 - imag#7) final R11				; 76-78
	ystore	[srcreg+d4], ymm9		;; Save R5						; 76

	vaddpd	ymm3, ymm3, ymm2		;; (real#7 + imag#7) final R7				; 77-79
	ystore	[srcreg+d4+d2+32], ymm0		;; Save R15						; 77

	ystore	[srcreg+d2], ymm10		;; Save R3						; 78
	ystore	[srcreg+d2+32], ymm6		;; Save R11						; 79
	ystore	[srcreg+d4+d2], ymm3		;; Save R7						; 80

	bump	srcreg, srcinc
	bump	screg, scinc
	ENDM

;; Haswell FMA3 version

IF (@INSTR(,%yarch,<FMA3>) NE 0)

yr8_8cl_16_reals_unfft_preload MACRO
	ENDM

;; This is the Bulldozer version timed at 44.9 clocks.  Converting adds and subs to FMA3 did not seem to help.
yr8_8cl_16_reals_unfft MACRO srcreg,srcinc,d1,d2,d4,screg,scinc,maxrpt,L1pt,L1pd

	vmovapd	ymm3, [screg+32]		;; cosine/sine for R2/I2
	vmovapd	ymm2, [srcreg+d1]		;; R2
	vmovapd	ymm1, [srcreg+d1+32]		;; I2
	yfmaddpd ymm0, ymm2, ymm3, ymm1		;; A2 = R2 * cosine/sine + I2				; 1-5
	yfmsubpd ymm1, ymm1, ymm3, ymm2		;; B2 = I2 * cosine/sine - R2				; 1-5

	vmovapd	ymm5, [screg+2*64+32]		;; cosine/sine for R4/I4
	vmovapd	ymm4, [srcreg+d2+d1]		;; R4
	vmovapd	ymm3, [srcreg+d2+d1+32]		;; I4
	yfmaddpd ymm2, ymm4, ymm5, ymm3		;; A4 = R4 * cosine/sine + I4				; 2-6
	yfmsubpd ymm3, ymm3, ymm5, ymm4		;; B4 = I4 * cosine/sine - R4				; 2-6

	vmovapd	ymm7, [screg+64+32]		;; cosine/sine for R3/I3
	vmovapd	ymm6, [srcreg+d2]		;; R3
	vmovapd	ymm5, [srcreg+d2+32]		;; I3
	yfmaddpd ymm4, ymm6, ymm7, ymm5		;; A3 = R3 * cosine/sine + I3				; 3-7
	yfmsubpd ymm5, ymm5, ymm7, ymm6		;; B3 = I3 * cosine/sine - R3				; 3-7

	vmovapd	ymm9, [screg+3*64+32]		;; cosine/sine for R5/I5
	vmovapd	ymm8, [srcreg+d4]		;; R5
	vmovapd	ymm7, [srcreg+d4+32]		;; I5
	yfmaddpd ymm6, ymm8, ymm9, ymm7		;; A5 = R5 * cosine/sine + I5				; 4-8
	yfmsubpd ymm7, ymm7, ymm9, ymm8		;; B5 = I5 * cosine/sine - R5				; 4-8

	vmovapd	ymm11, [screg+6*64+32]		;; cosine/sine for R8/I8
	vmovapd	ymm10, [srcreg+d4+d2+d1]	;; R8
	vmovapd	ymm9, [srcreg+d4+d2+d1+32]	;; I8
	yfmaddpd ymm8, ymm10, ymm11, ymm9	;; A8 = R8 * cosine/sine + I8				; 5-9
	yfmsubpd ymm9, ymm9, ymm11, ymm10	;; B8 = I8 * cosine/sine - R8				; 5-9

	vmovapd	ymm10, [screg]			;; sine for R2/I2
	vmulpd	ymm0, ymm0, ymm10		;; R2 = A2 * sine					; 6-10
	vmulpd	ymm1, ymm1, ymm10		;; I2 = B2 * sine					; 6-10

	vmovapd	ymm13, [screg+4*64+32]		;; cosine/sine for R6/I6
	vmovapd	ymm12, [srcreg+d4+d1]		;; R6
	vmovapd	ymm11, [srcreg+d4+d1+32]	;; I6
	yfmaddpd ymm10, ymm12, ymm13, ymm11	;; A6 = R6 * cosine/sine + I6				; 7-11
	yfmsubpd ymm11, ymm11, ymm13, ymm12	;; B6 = I6 * cosine/sine - R6				; 7-11

	vmovapd	ymm12, [screg+2*64]		;; sine for R4/I4
	vmulpd	ymm2, ymm2, ymm12		;; R4 = A4 * sine					; 8-12
	vmulpd	ymm3, ymm3, ymm12		;; I4 = B4 * sine					; 8-12

	vmovapd	ymm15, [screg+5*64+32]		;; cosine/sine for R7/I7
	vmovapd	ymm14, [srcreg+d4+d2]		;; R7
	vmovapd	ymm13, [srcreg+d4+d2+32]	;; I7
	yfmaddpd ymm12, ymm14, ymm15, ymm13	;; A7 = R7 * cosine/sine + I7				; 9-13
	yfmsubpd ymm13, ymm13, ymm15, ymm14	;; B7 = I7 * cosine/sine - R7				; 9-13

	vmovapd	ymm14, [screg+64]		;; sine for R3/I3
	vmulpd	ymm4, ymm4, ymm14		;; R3 = A3 * sine					; 10-14
	vmulpd	ymm5, ymm5, ymm14		;; I3 = B3 * sine					; 10-14

	;;; if we had both sine and sine/.707 we could save a clock by using FMA on R5/I5
	vmovapd	ymm14, [screg+3*64]		;; sine for R5/I5
	vmulpd	ymm6, ymm6, ymm14		;; R5 = A5 * sine					; 11-15			n 18
	vmulpd	ymm7, ymm7, ymm14		;; I5 = B5 * sine					; 11-15			n 24

	vmovapd	ymm14, [screg+6*64]		;; sine for R8/I8
	yfmaddpd ymm15, ymm8, ymm14, ymm0	;; R2+(R8 = A8 * sine)					; 12-16			n 19
	yfnmaddpd ymm8, ymm8, ymm14, ymm0	;; R2-(R8 = A8 * sine)					; 12-16			n 21

	L1prefetchw srcreg+L1pd, L1pt
	yfnmaddpd ymm0, ymm9, ymm14, ymm1	;; I2-(I8 = B8 * sine)					; 13-17			n 20
	yfmaddpd ymm9, ymm9, ymm14, ymm1	;; I2+(I8 = B8 * sine)					; 13-17			n 22

	vmovapd	ymm14, [screg+4*64]		;; sine for R6/I6
	yfmaddpd ymm1, ymm10, ymm14, ymm2	;; R4+(R6 = A6 * sine)					; 14-18			n 19
	yfnmaddpd ymm10, ymm10, ymm14, ymm2	;; R4-(R6 = A6 * sine)					; 14-18			n 21

	L1prefetchw srcreg+d1+L1pd, L1pt
	yfnmaddpd ymm2, ymm11, ymm14, ymm3	;; I4-(I6 = B6 * sine)					; 15-19			n 20
	yfmaddpd ymm11, ymm11, ymm14, ymm3	;; I4+(I6 = B6 * sine)					; 15-19			n 22

	vmovapd	ymm14, [screg+5*64]		;; sine for R7/I7
	yfnmaddpd ymm3, ymm12, ymm14, ymm4	;; R3-(R7 = A7 * sine)					; 16-20			n 23
	yfmaddpd ymm12, ymm12, ymm14, ymm4	;; R3+(R7 = A7 * sine)					; 16-20			n 25

	yfmaddpd ymm4, ymm13, ymm14, ymm5	;; I3+(I7 = B7 * sine)					; 17-21			n 24
	yfnmaddpd ymm13, ymm13, ymm14, ymm5	;; I3-(I7 = B7 * sine) (i3o)				; 17-21			n 27

	vmovapd	ymm5, [srcreg]			;; r1+r9
	vmovapd	ymm14, YMM_ONE
	ystore	[srcreg], ymm13			;; Save i3o						; 22
	vaddpd	ymm13, ymm5, ymm6		;; (r1+r9) + r5						; 18-22			n 25
	yfmsubpd ymm5, ymm5, ymm14, ymm6	;; (r1+r9) - r5 (r3o)					; 18-22			n 26

	L1prefetchw srcreg+d2+L1pd, L1pt
	vsubpd	ymm6, ymm15, ymm1		;; (r2+r8)-(r4+r6) (r3e)				; 19-23			n 26
	yfmaddpd ymm15, ymm15, ymm14, ymm1	;; (r2+r8)+(r4+r6) (r1e)				; 19-23			n 30

	L1prefetchw srcreg+d2+d1+L1pd, L1pt
	vaddpd	ymm1, ymm0, ymm2		;; (i2-i8)+(i4-i6) (i3e)				; 20-24			n 27
	yfmsubpd ymm0, ymm0, ymm14, ymm2	;; (i2-i8)-(i4-i6) (new i5)				; 20-24			n 33

	vmovapd	ymm14, YMM_P924_P383
	yfnmaddpd ymm2, ymm14, ymm10, ymm8	;; (r2-r8)-.924/.383(r4-r6) (r4e/.383)			; 21-25			n 28
	yfmaddpd ymm8, ymm14, ymm8, ymm10	;; .924/.383(r2-r8)+(r4-r6) (r2e/.383)			; 21-25			n 30

	yfmsubpd ymm10, ymm14, ymm9, ymm11	;; .924/.383(i2+i8)-(i4+i6) (i4e/.383)			; 22-26			n 29
	yfmaddpd ymm11, ymm14, ymm11, ymm9	;; (i2+i8)+.924/.383(i4+i6) (i2e/.383)			; 22-26			n 33

	vmovapd	ymm9, [srcreg+32]		;; r1-r9
	vmovapd	ymm14, YMM_SQRTHALF
	ystore	YMM_TMPS[0*32], ymm0		;; Save i5						; 25
	yfnmaddpd ymm0, ymm14, ymm3, ymm9	;; (r1-r9) - .707(r3-r7) (r4o)				; 23-27			n 28
	yfmaddpd ymm3, ymm14, ymm3, ymm9	;; (r1-r9) + .707(r3-r7) (r2o)				; 23-27			n 31

	L1prefetchw srcreg+d4+L1pd, L1pt
	yfmsubpd ymm9, ymm14, ymm4, ymm7	;; .707(i3+i7)-i5 (i4o)					; 24-28			n 29
	yfmaddpd ymm4, ymm14, ymm4, ymm7	;; .707(i3+i7)+i5 (i2o)					; 24-28			n 33

	vaddpd ymm7, ymm13, ymm12		;; (r1+r9)+r5 + (r3+r7)  (r1o)				; 25-29			n 30
	yfmsubpd ymm13, ymm13, YMM_ONE, ymm12	;; (r1+r9)+r5 - (r3+r7)  (new r5)			; 25-29			n 33

	L1prefetchw srcreg+d4+d1+L1pd, L1pt
	yfmaddpd ymm12, ymm14, ymm6, ymm5	;; r3o + .707*r3e (r3)					; 26-30			n 32
	yfnmaddpd ymm6, ymm14, ymm6, ymm5	;; r3o - .707*r3e (r7)					; 26-30			n 34

	ystore	YMM_TMPS[1*32], ymm13		;; Save r5						; 30
	vmovapd	ymm13, [srcreg]			;; i3o
	yfmaddpd ymm5, ymm14, ymm1, ymm13	;; imag-cols row #3 (.707*i3e + i3o)			; 27-31			n 32
	yfmsubpd ymm1, ymm14, ymm1, ymm13	;; imag-cols row #7 (.707*i3e - i3o)			; 27-31			n 34

	vmovapd	ymm14, YMM_P383
	yfmaddpd ymm13, ymm14, ymm2, ymm0	;; r4o + .383*r4e (r4)					; 28-32			n 36
	yfnmaddpd ymm2, ymm14, ymm2, ymm0	;; r4o - .383*r4e (r6)					; 28-32			n 37

	L1prefetchw srcreg+d4+d2+L1pd, L1pt
	yfmaddpd ymm0, ymm14, ymm10, ymm9	;; imag-cols row #4 (.383*i4e + i4o)			; 29-33			n 36
	yfmsubpd ymm10, ymm14, ymm10, ymm9	;; imag-cols row #6 (.383*i4e - i4o)			; 29-33			n 37

	vaddpd	ymm9, ymm7, ymm15		;; r1o + r1e (final R1)					; 30-34
	ystore	[srcreg], ymm9			;; Save final R1					; 35
	vmovapd	ymm9, YMM_ONE
	yfmsubpd ymm7, ymm7, ymm9, ymm15	;; r1o - r1e (final R9)					; 30-34

	L1prefetchw srcreg+d4+d2+d1+L1pd, L1pt
	yfmaddpd ymm15, ymm14, ymm8, ymm3	;; r2o + .383*r2e (r2)					; 31-35			n 38
	yfnmaddpd ymm8, ymm14, ymm8, ymm3	;; r2o - .383*r2e (r8)					; 31-35			n 39

	vsubpd	ymm3, ymm12, ymm5		;; (real#3 - imag#3) final R15				; 32-36
	yfmaddpd ymm12, ymm12, ymm9, ymm5	;; (real#3 + imag#3) final R3				; 32-36

	yfmaddpd ymm5, ymm14, ymm11, ymm4	;; imag-cols row #2 (.383*i2e + i2o)			; 33-37			n 38
	yfmsubpd ymm11, ymm14, ymm11, ymm4	;; imag-cols row #8 (.383*i2e - i2o)			; 33-37			n 39
	vmovapd ymm14, YMM_TMPS[1*32]		;; r5

	vsubpd	ymm4, ymm6, ymm1		;; (real#7 - imag#7) final R11				; 34-38
	yfmaddpd ymm6, ymm6, ymm9, ymm1		;; (real#7 + imag#7) final R7				; 34-38

	vmovapd ymm1, YMM_TMPS[0*32]		;; i5
	ystore	[srcreg+32], ymm7		;; Save final R9					; 35+1
	vsubpd	ymm7, ymm14, ymm1		;; (real#5 - imag#5) final R13				; 35-39
	yfmaddpd ymm14, ymm14, ymm9, ymm1	;; (real#5 + imag#5) final R5				; 35-39

	vsubpd	ymm1, ymm13, ymm0		;; (real#4 - imag#4) final R14				; 36-40
	yfmaddpd ymm13, ymm13, ymm9, ymm0	;; (real#4 + imag#4) final R4				; 36-40

	vsubpd	ymm0, ymm2, ymm10		;; (real#6 - imag#6) final R12				; 37-41
	yfmaddpd ymm2, ymm2, ymm9, ymm10	;; (real#6 + imag#6) final R6				; 37-41
	ystore	[srcreg+d4+d2+32], ymm3		;; Save R15						; 37

	vsubpd	ymm10, ymm15, ymm5		;; (real#2 - imag#2) final R16				; 38-42
	yfmaddpd ymm15, ymm15, ymm9, ymm5	;; (real#2 + imag#2) final R2				; 38-42
	ystore	[srcreg+d2], ymm12		;; Save R3						; 37+1

	vsubpd	ymm5, ymm8, ymm11		;; (real#8 - imag#8) final R10				; 39-43
	yfmaddpd ymm8, ymm8, ymm9, ymm11	;; (real#8 + imag#8) final R8				; 39-43
	ystore	[srcreg+d2+32], ymm4		;; Save R11						; 39

	ystore	[srcreg+d4+d2], ymm6		;; Save R7						; 39+1
	ystore	[srcreg+d4+32], ymm7		;; Save R13						; 40+1
	ystore	[srcreg+d4], ymm14		;; Save R5						; 40+2 
	ystore	[srcreg+d4+d1+32], ymm1		;; Save R14						; 41+2
	ystore	[srcreg+d2+d1], ymm13		;; Save R4						; 41+3
	ystore	[srcreg+d2+d1+32], ymm0		;; Save R12						; 42+3
	ystore	[srcreg+d4+d1], ymm2		;; Save R6						; 42+4
	ystore	[srcreg+d4+d2+d1+32], ymm10	;; Save R16						; 43+4
	ystore	[srcreg+d1], ymm15		;; Save R2						; 43+5
	ystore	[srcreg+d1+32], ymm5		;; Save R10						; 44+5
	ystore	[srcreg+d4+d2+d1], ymm8		;; Save R8						; 44+6

	bump	srcreg, srcinc
	bump	screg, scinc
	ENDM

ENDIF

ENDIF





;;
;; ********************************* reduced sin/cos eight-complex-fft8 variants **************************************
;;
;; These macros are used in the last levels of pass 1.  These macros differ from the standard sg8cl macros
;; in that the sin/cos values are computed on the fly to reduce memory requirements.  To do this we cannot
;; store data as (cos/sin,sin).  Thus, these macros work on (cos,sin) values.

yr8_rsc_sg8cl_eight_complex_calc_sincos MACRO src1, src2, clm
	yr8_rsc_sg8cl_eight_complex_calc_sincos1 src1+0*YMM_SCD1, src2+0*YMM_SCD4, YMM_TMPS[0*YMM_SCD8]
	IF clm GE 2
	yr8_rsc_sg8cl_eight_complex_calc_sincos1 src1+1*YMM_SCD1, src2+1*YMM_SCD4, YMM_TMPS[1*YMM_SCD8]
	ENDIF
	IF clm GE 4
	yr8_rsc_sg8cl_eight_complex_calc_sincos1 src1+2*YMM_SCD1, src2+2*YMM_SCD4, YMM_TMPS[2*YMM_SCD8]
	yr8_rsc_sg8cl_eight_complex_calc_sincos1 src1+3*YMM_SCD1, src2+3*YMM_SCD4, YMM_TMPS[3*YMM_SCD8]
	ENDIF
	ENDM

yr8_rsc_sg8cl_eight_complex_calc_sincos1 MACRO src1, src2, dest
	vmovapd	ymm0, [src1+32]		;; cos
	vmovapd	ymm1, [src1]		;; sin
	ystore	dest[0*64+32], ymm0
	ystore	dest[0*64], ymm1

	;;a+bi	*	c+di	=	(ac-bd) + (ad+bc)i
	;;a+bi	*	c-di	=	(ac+bd) + (-ad+bc)i

	vmovapd	ymm2, [src2+32]		;; cos
	vmulpd	ymm4, ymm0, ymm2	;; cos * cos				;  1-5
	vmovapd	ymm3, [src2]		;; sin
	vmulpd	ymm5, ymm1, ymm3	;; sin * sin				;  2-6
	vmulpd	ymm3, ymm0, ymm3	;; ad = cos * sin			;  3-7
	vmulpd	ymm2, ymm1, ymm2	;; bc = sin * cos			;  4-8
	vsubpd	ymm6, ymm4, ymm5	;; new cos = cos * cos - sin * sin	; 7-9
	vaddpd	ymm4, ymm4, ymm5	;; new cos = cos * cos + sin * sin	; 8-10
	vaddpd	ymm5, ymm3, ymm2	;; new sin = ad + bc			; 9-11
	vsubpd	ymm7, ymm2, ymm3	;; new sin = bc - ad			; 10-12
	ystore	dest[1*64+32], ymm6						; 10
	ystore	dest[7*64+32], ymm4						; 11
	ystore	dest[1*64], ymm5						; 12
	ystore	dest[7*64], ymm7						; 13

	;;a+bi	*	c+di	=	(ac-bd) + (ad+bc)i
	;;a+bi	*	c-di	=	(ac+bd) + (-ad+bc)i

	vmovapd	ymm2, [src2+64+32]	;; cos
	vmulpd	ymm4, ymm0, ymm2	;; cos * cos				;  1-5
	vmovapd	ymm3, [src2+64]		;; sin
	vmulpd	ymm5, ymm1, ymm3	;; sin * sin				;  2-6
	vmulpd	ymm3, ymm0, ymm3	;; ad = cos * sin			;  3-7
	vmulpd	ymm2, ymm1, ymm2	;; bc = sin * cos			;  4-8
	vsubpd	ymm6, ymm4, ymm5	;; new cos = cos * cos - sin * sin	; 7-9
	vaddpd	ymm4, ymm4, ymm5	;; new cos = cos * cos + sin * sin	; 8-10
	vaddpd	ymm5, ymm3, ymm2	;; new sin = ad + bc			; 9-11
	vsubpd	ymm7, ymm2, ymm3	;; new sin = bc - ad			; 10-12
	ystore	dest[2*64+32], ymm6						; 10
	ystore	dest[6*64+32], ymm4						; 11
	ystore	dest[2*64], ymm5						; 12
	ystore	dest[6*64], ymm7						; 13

	;;a+bi	*	c+di	=	(ac-bd) + (ad+bc)i
	;;a+bi	*	c-di	=	(ac+bd) + (-ad+bc)i

	vmovapd	ymm2, [src2+128+32]	;; cos
	vmulpd	ymm4, ymm0, ymm2	;; cos * cos				;  1-5
	vmovapd	ymm3, [src2+128]	;; sin
	vmulpd	ymm5, ymm1, ymm3	;; sin * sin				;  2-6
	vmulpd	ymm3, ymm0, ymm3	;; ad = cos * sin			;  3-7
	vmulpd	ymm2, ymm1, ymm2	;; bc = sin * cos			;  4-8
	vsubpd	ymm6, ymm4, ymm5	;; new cos = cos * cos - sin * sin	; 7-9
	vaddpd	ymm4, ymm4, ymm5	;; new cos = cos * cos + sin * sin	; 8-10
	vaddpd	ymm5, ymm3, ymm2	;; new sin = ad + bc			; 9-11
	vsubpd	ymm7, ymm2, ymm3	;; new sin = bc - ad			; 10-12
	ystore	dest[3*64+32], ymm6						; 10
	ystore	dest[5*64+32], ymm4						; 11
	ystore	dest[3*64], ymm5						; 12
	ystore	dest[5*64], ymm7						; 13

	;;a+bi	*	c+di	=	(ac-bd) + (ad+bc)i

	vmovapd	ymm2, [src2+192+32]	;; cos
	vmulpd	ymm6, ymm0, ymm2	;; cos * cos				;  5-9
	vmovapd	ymm3, [src2+192]	;; sin
	vmulpd	ymm4, ymm1, ymm3	;; sin * sin				;  6-10
	vmulpd	ymm3, ymm0, ymm3	;; cos * sin				;  7-11
	vmulpd	ymm2, ymm1, ymm2	;; sin * cos				;  8-12
	vsubpd	ymm6, ymm6, ymm4	;; new cos = cos * cos - sin * sin	; 11-13
	vaddpd	ymm2, ymm2, ymm3	;; new sin = cos * sin + sin * cos	; 13-15
	ystore	dest[4*64+32], ymm6						; 14
	ystore	dest[4*64], ymm2						; 16
	ENDM

IFDEF X86_64

yr8_rsc_sg8cl_eight_complex_calc_sincos1 MACRO src1, src2, dest
	vmovapd	ymm0, [src1+32]		;; cos
	vmovapd	ymm1, [src1]		;; sin
	ystore	dest[0*64+32], ymm0
	ystore	dest[0*64], ymm1
	vmovapd	ymm2, [src2+32]		;; cos1
	vmulpd	ymm4, ymm0, ymm2	;; ac1 = cos * cos			;  1-5
	vmovapd	ymm3, [src2]		;; sin1
	vmulpd	ymm5, ymm1, ymm3	;; bd1 = sin * sin			;  2-6
	vmulpd	ymm3, ymm0, ymm3	;; ad1 = cos * sin			;  3-7
	vmulpd	ymm2, ymm1, ymm2	;; bc1 = sin * cos			;  4-8
	vmovapd	ymm8, [src2+64+32]	;; cos2
	vmulpd	ymm10, ymm0, ymm8	;; ac2 = cos * cos			;  5-9
	vmovapd	ymm9, [src2+64]		;; sin2
	vmulpd	ymm11, ymm1, ymm9	;; bd2 = sin * sin			;  6-10
	vsubpd	ymm6, ymm4, ymm5	;; new cos1 = ac - bd			; 7-9
	vmulpd	ymm9, ymm0, ymm9	;; ad2 = cos * sin			;  7-11
	vaddpd	ymm4, ymm4, ymm5	;; new cos1 = ac + bd			; 8-10
	vmulpd	ymm8, ymm1, ymm8	;; bc2 = sin * cos			;  8-12
	vaddpd	ymm5, ymm3, ymm2	;; new sin1 = ad + bc			; 9-11
	vmovapd	ymm12, [src2+128+32]	;; cos3
	vmulpd	ymm14, ymm0, ymm12	;; ac3 = cos * cos			;  9-13
	ystore	dest[1*64+32], ymm6						; 10
	vsubpd	ymm7, ymm2, ymm3	;; new sin1 = bc - ad			; 10-12
	vmovapd	ymm13, [src2+128]	;; sin3
	vmulpd	ymm15, ymm1, ymm13	;; bd3 = sin * sin			;  10-14
	ystore	dest[7*64+32], ymm4						; 11
	vsubpd	ymm2, ymm10, ymm11	;; new cos2 = ac - bd			; 11-13
	vmulpd	ymm13, ymm0, ymm13	;; ad3 = cos * sin			;  11-15
	ystore	dest[1*64], ymm5						; 12
	vaddpd	ymm10, ymm10, ymm11	;; new cos2 = ac + bd			; 12-14
	vmulpd	ymm12, ymm1, ymm12	;; bc3 = sin * cos			;  12-16
	ystore	dest[7*64], ymm7						; 13
	vaddpd	ymm11, ymm9, ymm8	;; new sin2 = ad + bc			; 13-15
	vmovapd	ymm7, [src2+192+32]	;; cos4
	vmulpd	ymm6, ymm0, ymm7	;; cos * cos4				;  13-17
	ystore	dest[2*64+32], ymm2						; 14
	vsubpd	ymm5, ymm8, ymm9	;; new sin2 = bc - ad			; 14-16
	vmovapd	ymm3, [src2+192]	;; sin4
	vmulpd	ymm4, ymm1, ymm3	;; sin * sin4				;  14-18
	ystore	dest[6*64+32], ymm10						; 15
	vsubpd	ymm8, ymm14, ymm15	;; new cos3 = ac - bd			; 15-17
	vmulpd	ymm3, ymm0, ymm3	;; cos * sin4				;  15-19
	ystore	dest[2*64], ymm11						; 16
	vaddpd	ymm14, ymm14, ymm15	;; new cos3 = ac + bd			; 16-18
	vmulpd	ymm7, ymm1, ymm7	;; sin * cos4				;  16-20
	ystore	dest[6*64], ymm5						; 17
	vaddpd	ymm15, ymm13, ymm12	;; new sin3 = ad + bc			; 17-19
	ystore	dest[3*64+32], ymm8						; 18
	vsubpd	ymm9, ymm12, ymm13	;; new sin3 = bc - ad			; 18-20
	ystore	dest[5*64+32], ymm14						; 19
	vsubpd	ymm6, ymm6, ymm4	;; new cos4 = cos * cos - sin * sin	; 19-21
	ystore	dest[3*64], ymm15						; 20
	ystore	dest[5*64], ymm9						; 21
	vaddpd	ymm3, ymm3, ymm7	;; new sin4 = cos * sin + sin * cos	; 21-23
	ystore	dest[4*64+32], ymm6						; 22
	ystore	dest[4*64], ymm3						; 24
	ENDM

IF (@INSTR(,%yarch,<FMA3>) NE 0)
yr8_rsc_sg8cl_eight_complex_calc_sincos1 MACRO src1, src2, dest
	vmovapd	ymm0, [src1+32]		;; cos
	vmovapd	ymm1, [src1]		;; sin
	ystore	dest[0*64+32], ymm0
	ystore	dest[0*64], ymm1
	vmovapd	ymm2, [src2+32]		;; cos1
	vmulpd	ymm3, ymm0, ymm2	;; ac1 = cos * cos1			; 1-5
	vmulpd	ymm2, ymm1, ymm2	;; bc1 = sin * cos1			; 1-5
	vmovapd	ymm4, [src2+64+32]	;; cos2
	vmulpd	ymm5, ymm0, ymm4	;; ac2 = cos * cos2			; 2-6
	vmulpd	ymm4, ymm1, ymm4	;; bc2 = sin * cos2			; 2-6
	vmovapd	ymm6, [src2+128+32]	;; cos3
	vmulpd	ymm7, ymm0, ymm6	;; ac3 = cos * cos3			; 3-7
	vmulpd	ymm6, ymm1, ymm6	;; bc3 = sin * cos3			; 3-7
	vmovapd	ymm8, [src2+192+32]	;; cos4
	vmulpd	ymm9, ymm0, ymm8	;; ac4 = cos * cos4			; 4-8
	vmulpd	ymm8, ymm1, ymm8	;; bc4 = sin * cos4			; 4-8
	vmovapd	ymm15, [src2]		;; sin1
	yfnmaddpd ymm10, ymm1, ymm15, ymm3 ;; new cos1 = ac1 - sin * sin1	; 6-10
	yfmaddpd ymm3, ymm1, ymm15, ymm3 ;; new cos7 = ac1 + sin * sin1		; 6-10
	yfmaddpd ymm11, ymm0, ymm15, ymm2 ;; new sin1 = cos * sin1 + bc1	; 7-11
	yfnmaddpd ymm2, ymm0, ymm15, ymm2 ;; new sin7 = bc1 - cos * sin1	; 7-11
	vmovapd	ymm15, [src2+64]	;; sin2
	yfnmaddpd ymm12, ymm1, ymm15, ymm5 ;; new cos2 = ac2 - sin * sin2	; 8-12
	yfmaddpd ymm5, ymm1, ymm15, ymm5 ;; new cos6 = ac2 + sin * sin2		; 8-12
	yfmaddpd ymm13, ymm0, ymm15, ymm4 ;; new sin2 = cos * sin2 + bc2	; 9-13
	yfnmaddpd ymm4, ymm0, ymm15, ymm4 ;; new sin6 = bc2 - cos * sin2	; 9-13
	vmovapd	ymm15, [src2+128]	;; sin3
	yfnmaddpd ymm14, ymm1, ymm15, ymm7 ;; new cos3 = ac3 - sin * sin3	; 10-14
	yfmaddpd ymm7, ymm1, ymm15, ymm7 ;; new cos5 = ac3 + sin * sin3		; 10-14
	ystore	dest[1*64+32], ymm10	;; cos1					; 11
	yfmaddpd ymm10, ymm0, ymm15, ymm6 ;; new sin3 = cos * sin3 + bc3	; 11-15
	yfnmaddpd ymm6, ymm0, ymm15, ymm6 ;; new sin5 = bc3 - cos * sin3	; 11-15
	vmovapd	ymm15, [src2+192]	;; sin4
	ystore	dest[7*64+32], ymm3	;; cos7					; 11+1
	yfnmaddpd ymm9, ymm1, ymm15, ymm9 ;; new cos4 = ac4 - sin * sin4	; 12-16
	yfmaddpd ymm8, ymm0, ymm15, ymm8 ;; new sin4 = cos * sin4 + bc4		; 12-16
	ystore	dest[1*64], ymm11	;; sin1					; 12+1
	ystore	dest[7*64], ymm2	;; sin7					; 12+2
	ystore	dest[2*64+32], ymm12	;; cos2					; 13+2
	ystore	dest[6*64+32], ymm5	;; cos6					; 13+3
	ystore	dest[2*64], ymm13	;; sin2					; 14+3
	ystore	dest[6*64], ymm4	;; sin6					; 14+4
	ystore	dest[3*64+32], ymm14	;; cos3					; 15+4
	ystore	dest[5*64+32], ymm7	;; cos5					; 15+5
	ystore	dest[3*64], ymm10	;; sin3					; 16+5
	ystore	dest[5*64], ymm6	;; sin5					; 16+6
	ystore	dest[4*64+32], ymm9	;; cos4					; 17+6
	ystore	dest[4*64], ymm8	;; sin4					; 17+7
	ENDM
ENDIF

ENDIF

;; Like the above, except we know that src1 would point to 1+0i
yr8_rsc_sg8cl_eight_complex_calc_sincos_simple MACRO src2, clm
	vmovapd	ymm0, YMM_ONE		;; cos
	vxorpd	ymm1, ymm1, ymm1	;; sin
	yr8_rsc_sg8cl_eight_complex_calc_sincos_simple1 src2+0*YMM_SCD4, YMM_TMPS[0*YMM_SCD8]
	IF clm GE 2
	yr8_rsc_sg8cl_eight_complex_calc_sincos_simple1 src2+1*YMM_SCD4, YMM_TMPS[1*YMM_SCD8]
	ENDIF
	IF clm GE 4
	yr8_rsc_sg8cl_eight_complex_calc_sincos_simple1 src2+2*YMM_SCD4, YMM_TMPS[2*YMM_SCD8]
	yr8_rsc_sg8cl_eight_complex_calc_sincos_simple1 src2+3*YMM_SCD4, YMM_TMPS[3*YMM_SCD8]
	ENDIF
	ENDM

yr8_rsc_sg8cl_eight_complex_calc_sincos_simple1 MACRO src2, dest
	ystore	dest[0*64+32], ymm0
	ystore	dest[0*64], ymm1

	vmovapd	ymm2, [src2+32]		;; cos
	vmovapd	ymm3, [src2]		;; sin
	ystore	dest[1*64+32], ymm2
	ystore	dest[1*64], ymm3
	ystore	dest[7*64+32], ymm2
	vsubpd	ymm3, ymm1, ymm3	;; new sin = - sin
	ystore	dest[7*64], ymm3

	vmovapd	ymm2, [src2+64+32]	;; cos
	vmovapd	ymm3, [src2+64]		;; sin
	ystore	dest[2*64+32], ymm2
	ystore	dest[2*64], ymm3
	ystore	dest[6*64+32], ymm2
	vsubpd	ymm3, ymm1, ymm3	;; new sin = - sin
	ystore	dest[6*64], ymm3

	vmovapd	ymm2, [src2+128+32]	;; cos
	vmovapd	ymm3, [src2+128]	;; sin
	ystore	dest[3*64+32], ymm2
	ystore	dest[3*64], ymm3
	ystore	dest[5*64+32], ymm2
	vsubpd	ymm3, ymm1, ymm3	;; new sin = - sin
	ystore	dest[5*64], ymm3

	vmovapd	ymm2, [src2+192+32]	;; cos
	vmovapd	ymm3, [src2+192]	;; sin
	ystore	dest[4*64+32], ymm2
	ystore	dest[4*64], ymm3
	ENDM


yr8_rsc_sg8cl_eight_complex_fft8_preload MACRO
	ENDM
yr8_rsc_sg8cl_eight_complex_fft8 MACRO srcreg,srcinc,d1,d2,d4,dstreg,dstinc,e1,e2,e4,screg,scinc,maxrpt,L1pt,L1pd

	vmovapd	ymm1, [srcreg]			;; R1
	vmovapd	ymm7, [srcreg+d1]		;; R2
	vshufpd	ymm0, ymm1, ymm7, 15		;; Shuffle R1 and R2 to create R1/R2 hi
	vshufpd	ymm1, ymm1, ymm7, 0		;; Shuffle R1 and R2 to create R1/R2 low

	vmovapd	ymm2, [srcreg+d2]		;; R3
	vmovapd	ymm7, [srcreg+d2+d1]		;; R4
	vshufpd	ymm3, ymm2, ymm7, 15		;; Shuffle R3 and R4 to create R3/R4 hi
	vshufpd	ymm2, ymm2, ymm7, 0		;; Shuffle R3 and R4 to create R3/R4 low

	ylow128s ymm4, ymm0, ymm3		;; Shuffle R1/R2 hi and R3/R4 hi (first R2)
	yhigh128s ymm0, ymm0, ymm3		;; Shuffle R1/R2 hi and R3/R4 hi (first R4)

	ylow128s ymm3, ymm1, ymm2		;; Shuffle R1/R2 low and R3/R4 low (first R1)
	yhigh128s ymm1, ymm1, ymm2		;; Shuffle R1/R2 low and R3/R4 low (first R3)

	vmovapd	ymm6, [srcreg+d4]		;; R5
	vmovapd	ymm2, [srcreg+d4+d1]		;; R6
	vshufpd	ymm5, ymm6, ymm2, 15		;; Shuffle R5 and R6 to create R5/R6 hi
	vshufpd	ymm6, ymm6, ymm2, 0		;; Shuffle R5 and R6 to create R5/R6 low

	vmovapd	ymm2, [srcreg+d4+d2]		;; R7
	vmovapd	ymm7, [srcreg+d4+d2+d1]		;; R8
	ystore	[dstreg+32], ymm0		;; Save first R4
	vshufpd	ymm0, ymm2, ymm7, 15		;; Shuffle R7 and R8 to create R7/R8 hi
	vshufpd	ymm2, ymm2, ymm7, 0		;; Shuffle R7 and R8 to create R7/R8 low

	ylow128s ymm7, ymm5, ymm0		;; Shuffle R5/R6 hi and R7/R8 hi (first R6)
	yhigh128s ymm5, ymm5, ymm0		;; Shuffle R5/R6 hi and R7/R8 hi (first R8)

	ylow128s ymm0, ymm6, ymm2		;; Shuffle R5/R6 low and R7/R8 low (first R5)
	yhigh128s ymm6, ymm6, ymm2		;; Shuffle R5/R6 low and R7/R8 low (first R7)

	L1prefetch srcreg+L1pd, L1pt
	L1prefetchw dstreg+L1pd, L1pt - L1PREFETCH_DEST_NONE

	vaddpd	ymm2, ymm3, ymm0		;; R1 + R5 (new R1)
	vsubpd	ymm3, ymm3, ymm0		;; R1 - R5 (new R5)

	vaddpd	ymm0, ymm1, ymm6		;; R3 + R7 (new R3)
	vsubpd	ymm1, ymm1, ymm6		;; R3 - R7 (new R7)

	L1prefetch srcreg+d1+L1pd, L1pt
	L1prefetchw dstreg+e1+L1pd, L1pt - L1PREFETCH_DEST_NONE

	vaddpd	ymm6, ymm4, ymm7		;; R2 + R6 (new R2)
	vsubpd	ymm4, ymm4, ymm7		;; R2 - R6 (new R6)

	vmovapd	ymm7, [dstreg+32]		;; Reload first R4
	ystore	[dstreg+e4], ymm3		;; Save new R5

	vaddpd	ymm3, ymm7, ymm5		;; R4 + R8 (new R4)
	vsubpd	ymm7, ymm7, ymm5		;; R4 - R8 (new R8)

	L1prefetch srcreg+d2+L1pd, L1pt
	L1prefetchw dstreg+e2+L1pd, L1pt - L1PREFETCH_DEST_NONE

	vaddpd	ymm5, ymm2, ymm0		;; R1 + R3 (newer R1)
	vsubpd	ymm2, ymm2, ymm0		;; R1 - R3 (newer R3)

	vaddpd	ymm0, ymm6, ymm3		;; R2 + R4 (newer R2)
	vsubpd	ymm6, ymm6, ymm3		;; R2 - R4 (newer R4)

	ystore	[dstreg+e4+e2], ymm1		;; Save new R7
	vmovapd	ymm1, [srcreg+32]		;; I1
	ystore	[dstreg+e4+e2+e1], ymm7		;; Save new R8
	vmovapd	ymm7, [srcreg+d1+32]		;; I2
	ystore	[dstreg+e1], ymm0		;; Save newer R2
	vshufpd	ymm0, ymm1, ymm7, 15		;; Shuffle I1 and I2 to create I1/I2 hi
	vshufpd	ymm1, ymm1, ymm7, 0		;; Shuffle I1 and I2 to create I1/I2 low

	vmovapd	ymm7, [srcreg+d2+d1+32]		;; I4
	ystore	[dstreg+e2], ymm2		;; Save newer R3
	vmovapd	ymm2, [srcreg+d2+32]		;; I3
	vshufpd	ymm3, ymm2, ymm7, 15		;; Shuffle I3 and I4 to create I3/I4 hi
	vshufpd	ymm2, ymm2, ymm7, 0		;; Shuffle I3 and I4 to create I3/I4 low

	ystore	[dstreg+e4+e1], ymm4		;; Save new R6
	ylow128s ymm4, ymm0, ymm3		;; Shuffle I1/I2 hi and I3/I4 hi (first I2)
	yhigh128s ymm0, ymm0, ymm3		;; Shuffle I1/I2 hi and I3/I4 hi (first I4)

	ylow128s ymm3, ymm1, ymm2		;; Shuffle I1/I2 low and I3/I4 low (first I1)
	yhigh128s ymm1, ymm1, ymm2		;; Shuffle I1/I2 low and I3/I4 low (first I3)

	vmovapd	ymm2, [srcreg+d4+d1+32]		;; I6
	ystore	[dstreg+e2+e1], ymm6		;; Save newer R4
	vmovapd	ymm6, [srcreg+d4+32]		;; I5
	ystore	[dstreg], ymm5			;; Save newer R1
	vshufpd	ymm5, ymm6, ymm2, 15		;; Shuffle I5 and I6 to create I5/I6 hi
	vshufpd	ymm6, ymm6, ymm2, 0		;; Shuffle I5 and I6 to create I5/I6 low

	vmovapd	ymm2, [srcreg+d4+d2+32]		;; I7
	vmovapd	ymm7, [srcreg+d4+d2+d1+32]	;; I8
	ystore	[dstreg+e2+e1+32], ymm0		;; Save first I4
	vshufpd	ymm0, ymm2, ymm7, 15		;; Shuffle I7 and I8 to create I7/I8 hi
	vshufpd	ymm2, ymm2, ymm7, 0		;; Shuffle I7 and I8 to create I7/I8 low

	ylow128s ymm7, ymm5, ymm0		;; Shuffle I5/I6 hi and I7/I8 hi (first I6)
	yhigh128s ymm5, ymm5, ymm0		;; Shuffle I5/I6 hi and I7/I8 hi (first I8)

	ylow128s ymm0, ymm6, ymm2		;; Shuffle I5/I6 low and I7/I8 low (first I5)
	yhigh128s ymm6, ymm6, ymm2		;; Shuffle I5/I6 low and I7/I8 low (first I7)

	L1prefetch srcreg+d2+d1+L1pd, L1pt
	L1prefetchw dstreg+e2+e1+L1pd, L1pt - L1PREFETCH_DEST_NONE

	vaddpd	ymm2, ymm3, ymm0		;; I1 + I5 (new I1)
	vsubpd	ymm3, ymm3, ymm0		;; I1 - I5 (new I5)

	vaddpd	ymm0, ymm1, ymm6		;; I3 + I7 (new I3)
	vsubpd	ymm1, ymm1, ymm6		;; I3 - I7 (new I7)

	L1prefetch srcreg+d4+L1pd, L1pt
	L1prefetchw dstreg+e4+L1pd, L1pt - L1PREFETCH_DEST_NONE

	vaddpd	ymm6, ymm2, ymm0		;; I1 + I3 (newer I1)
	vsubpd	ymm2, ymm2, ymm0		;; I1 - I3 (newer I3)

	vmovapd	ymm0, [dstreg+e2+e1+32]		;; Reload first I4
	ystore	[dstreg+32], ymm6		;; Save newer I1

	vaddpd	ymm6, ymm4, ymm7		;; I2 + I6 (new I2)
	vsubpd	ymm4, ymm4, ymm7		;; I2 - I6 (new I6)

	L1prefetch srcreg+d4+d1+L1pd, L1pt
	L1prefetchw dstreg+e4+e1+L1pd, L1pt - L1PREFETCH_DEST_NONE

	vaddpd	ymm7, ymm0, ymm5		;; I4 + I8 (new I4)
	vsubpd	ymm0, ymm0, ymm5		;; I4 - I8 (new I8)

	vaddpd	ymm5, ymm6, ymm7		;; I2 + I4 (newer I2)
	vsubpd	ymm6, ymm6, ymm7		;; I2 - I4 (newer I4)

	vmovapd	ymm7, [dstreg+e4+e2]		;; Reload new R7
	ystore	[dstreg+e2+32], ymm2		;; Save newer I3

 	vaddpd	ymm2, ymm3, ymm7		;; I5 + R7 (newer I5)
	vsubpd	ymm3, ymm3, ymm7		;; I5 - R7 (newer I7)

	vmovapd	ymm7, [dstreg+e4]		;; Reload new R5
	ystore	[dstreg+e1+32], ymm5		;; Save newer I2

	vaddpd	ymm5, ymm7, ymm1		;; R5 + I7 (newer R7)
	vsubpd	ymm7, ymm7, ymm1		;; R5 - I7 (newer R5)

	vmovapd	ymm1, [dstreg+e4+e1]		;; Reload new R6
	ystore	[dstreg+e2+e1+32], ymm6		;; Save newer I4

	vaddpd	ymm6, ymm1, ymm0		;; R6 + I8 (new R8)
	vsubpd	ymm1, ymm1, ymm0		;; R6 - I8 (new R6)

	vmovapd	ymm0, [dstreg+e4+e2+e1]		;; Reload new R8
	ystore	[dstreg+e4+32], ymm2		;; Save newer I5

	vaddpd	ymm2, ymm4, ymm0		;; I6 + R8 (new I6)
	vsubpd	ymm4, ymm4, ymm0		;; I6 - R8 (new I8)

	L1prefetch srcreg+d4+d2+L1pd, L1pt
	L1prefetchw dstreg+e4+e2+L1pd, L1pt - L1PREFETCH_DEST_NONE

	vsubpd	ymm0, ymm1, ymm2		;; R6 = R6 - I6
	vaddpd	ymm1, ymm1, ymm2		;; I6 = R6 + I6
	vsubpd	ymm2, ymm6, ymm4		;; R8 = R8 - I8
	vaddpd	ymm6, ymm6, ymm4		;; I8 = R8 + I8

	vmovapd	ymm4, YMM_SQRTHALF
	vmulpd	ymm0, ymm0, ymm4		;; R6 = R6 * SQRTHALF (newer R6)
	vmulpd	ymm1, ymm1, ymm4		;; I6 = I6 * SQRTHALF (newer I6)
	vmulpd	ymm2, ymm2, ymm4		;; R8 = R8 * SQRTHALF (newer R8)
	vmulpd	ymm6, ymm6, ymm4		;; I8 = I8 * SQRTHALF (newer I8)

	ystore	[dstreg+e4], ymm7		;; Save newer R5

;; the last level

	vsubpd	ymm4, ymm5, ymm6		;; R7 - I8 (last R7)
	vaddpd	ymm5, ymm5, ymm6		;; R7 + I8 (last R8)

	L1prefetch srcreg+d4+d2+d1+L1pd, L1pt
	L1prefetchw dstreg+e4+e2+e1+L1pd, L1pt - L1PREFETCH_DEST_NONE

	vsubpd	ymm6, ymm3, ymm2		;; I7 - R8 (last I8)
	vaddpd	ymm3, ymm3, ymm2		;; I7 + R8 (last I7)

	vmovapd	ymm2, [screg+192+32]		;; cosine for R7/I7
	vmulpd	ymm7, ymm4, ymm2		;; A7 = R7 * cosine
	vmulpd	ymm2, ymm3, ymm2		;; B7 = I7 * cosine
	vmulpd	ymm3, ymm3, [screg+192]		;; C7 = I7 * sine
	vmulpd	ymm4, ymm4, [screg+192]		;; D7 = R7 * sine
	vsubpd	ymm7, ymm7, ymm3		;; A7 = A7 - C7 (final R7)
	vaddpd	ymm2, ymm2, ymm4		;; B7 = B7 + D7 (final I7)

	vmovapd	ymm3, [screg+448+32]		;; cosine for R8/I8
	vmulpd	ymm4, ymm5, ymm3		;; A8 = R8 * cosine
	vmulpd	ymm3, ymm6, ymm3		;; B8 = I8 * cosine
	vmulpd	ymm6, ymm6, [screg+448]		;; C8 = I8 * sine
	vmulpd	ymm5, ymm5, [screg+448]		;; D8 = R8 * sine
	vsubpd	ymm4, ymm4, ymm6		;; A8 = A8 - C8 (final R8)
	vaddpd	ymm3, ymm3, ymm5		;; B8 = B8 + D8 (final I8)

	vmovapd	ymm6, [dstreg+e4]		;; Reload newer R5
	vsubpd	ymm5, ymm6, ymm0		;; R5 - R6 (last R6)
	vaddpd	ymm6, ymm6, ymm0		;; R5 + R6 (last R5)

	vmovapd	ymm0, [dstreg+e4+32]		;; Reload newer I5
	ystore	[dstreg+e4+e2], ymm7		;; Save R7

	vsubpd	ymm7, ymm0, ymm1		;; I5 - I6 (last I6)
	vaddpd	ymm0, ymm0, ymm1		;; I5 + I6 (last I5)

	vmovapd	ymm1, [screg+320+32]		;; cosine for R6/I6
	ystore	[dstreg+e4+e2+32], ymm2		;; Save I7
	vmulpd	ymm2, ymm5, ymm1		;; A6 = R6 * cosine
	vmulpd	ymm1, ymm7, ymm1		;; B6 = I6 * cosine
	vmulpd	ymm7, ymm7, [screg+320]		;; C6 = I6 * sine
	vmulpd	ymm5, ymm5, [screg+320]		;; D6 = R6 * sine
	vsubpd	ymm2, ymm2, ymm7		;; A6 = A6 - C6 (final R6)
	vaddpd	ymm1, ymm1, ymm5		;; B6 = B6 + D6 (final I6)

	vmovapd	ymm7, [screg+64+32]		;; cosine for R5/I5
	vmulpd	ymm5, ymm6, ymm7		;; A5 = R5 * cosine
	vmulpd	ymm7, ymm0, ymm7		;; B5 = I5 * cosine
	vmulpd	ymm0, ymm0, [screg+64]		;; C5 = I5 * sine
	vmulpd	ymm6, ymm6, [screg+64]		;; D5 = R5 * sine
	vsubpd	ymm5, ymm5, ymm0		;; A5 = A5 - C5 (final R5)
	vaddpd	ymm7, ymm7, ymm6		;; B5 = B5 + D5 (final I5)

	vmovapd	ymm6, [dstreg]			;; Reload newer R1
	vmovapd	ymm0, [dstreg+e1]		;; Reload newer R2
	ystore	[dstreg+e4+e2+e1], ymm4		;; Save R8

	vsubpd	ymm4, ymm6, ymm0		;; R1 - R2 (last R2)
	vaddpd	ymm6, ymm6, ymm0		;; R1 + R2 (last R1)

	vmovapd	ymm0, [dstreg+32]		;; Reload newer I1
	ystore	[dstreg+e4+e2+e1+32], ymm3	;; Save I8
	vmovapd	ymm3, [dstreg+e1+32]		;; Reload newer I2
	ystore	[dstreg+e4+e1], ymm2		;; Save R6

	vsubpd	ymm2, ymm0, ymm3		;; I1 - I2 (last I2)
	vaddpd	ymm0, ymm0, ymm3		;; I1 + I2 (last I1)

	vmovapd	ymm3, [screg+256+32]		;; cosine for R2/I2
	ystore	[dstreg+e4+e1+32], ymm1		;; Save I6
	vmulpd	ymm1, ymm4, ymm3		;; A2 = R2 * cosine
	vmulpd	ymm3, ymm2, ymm3		;; B2 = I2 * cosine
	vmulpd	ymm2, ymm2, [screg+256]		;; C2 = I2 * sine
	vmulpd	ymm4, ymm4, [screg+256]		;; D2 = R2 * sine
	vsubpd	ymm1, ymm1, ymm2		;; A2 = A2 - C2 (final R2)
	vaddpd	ymm3, ymm3, ymm4		;; B2 = B2 + D2 (final I2)

	vmovapd	ymm2, [screg+0+32]		;; cosine for R1/I1
	vmulpd	ymm4, ymm6, ymm2		;; A1 = R1 * cosine
	vmulpd	ymm2, ymm0, ymm2		;; B1 = I1 * cosine
	vmulpd	ymm0, ymm0, [screg+0]		;; C1 = I1 * sine
	vmulpd	ymm6, ymm6, [screg+0]		;; D1 = R1 * sine
	vsubpd	ymm4, ymm4, ymm0		;; A1 = A1 - C1 (final R1)
	vaddpd	ymm2, ymm2, ymm6		;; B1 = B1 + D1 (final I1)

	vmovapd	ymm0, [dstreg+e2]		;; Reload newer R3
	vmovapd	ymm6, [dstreg+e2+e1+32]		;; Reload newer I4
	ystore	[dstreg+e4], ymm5		;; Save R5

	vsubpd	ymm5, ymm0, ymm6		;; R3 - I4 (last R3)
	vaddpd	ymm0, ymm0, ymm6		;; R3 + I4 (last R4)

	vmovapd	ymm6, [dstreg+e2+32]		;; Reload newer I3
	ystore	[dstreg+e4+32], ymm7		;; Save I5
	vmovapd	ymm7, [dstreg+e2+e1]		;; Reload newer R4
	ystore	[dstreg+e1], ymm1		;; Save R2

	vsubpd	ymm1, ymm6, ymm7		;; I3 - R4 (last I4)
	vaddpd	ymm6, ymm6, ymm7		;; I3 + R4 (last I3)

	vmovapd	ymm7, [screg+128+32]		;; cosine for R3/I3
	ystore	[dstreg+e1+32], ymm3		;; Save I2
	vmulpd	ymm3, ymm5, ymm7		;; A3 = R3 * cosine
	vmulpd	ymm7, ymm6, ymm7		;; B3 = I3 * cosine
	vmulpd	ymm6, ymm6, [screg+128]		;; C3 = I3 * sine
	vmulpd	ymm5, ymm5, [screg+128]		;; D3 = R3 * sine
	vsubpd	ymm3, ymm3, ymm6		;; A3 = A3 - C3 (final R3)
	vaddpd	ymm7, ymm7, ymm5		;; B3 = B3 + D3 (final I3)

	vmovapd	ymm6, [screg+384+32]		;; cosine for R4/I4
	vmulpd	ymm5, ymm0, ymm6		;; A4 = R4 * cosine
	vmulpd	ymm6, ymm1, ymm6		;; B4 = I4 * cosine
	vmulpd	ymm1, ymm1, [screg+384]		;; C4 = I4 * sine
	vmulpd	ymm0, ymm0, [screg+384]		;; D4 = R4 * sine
	vsubpd	ymm5, ymm5, ymm1		;; A4 = A4 - C4 (final R4)
	vaddpd	ymm6, ymm6, ymm0		;; B4 = B4 + D4 (final I4)

	ystore	[dstreg], ymm4			;; Save R1
	ystore	[dstreg+32], ymm2		;; Save I1
	ystore	[dstreg+e2], ymm3		;; Save R3
	ystore	[dstreg+e2+32], ymm7		;; Save I3
	ystore	[dstreg+e2+e1], ymm5		;; Save R4
	ystore	[dstreg+e2+e1+32], ymm6		;; Save I4

	bump	srcreg, srcinc
	bump	dstreg, dstinc
	bump	screg, scinc
	ENDM

IFDEF X86_64

yr8_rsc_sg8cl_eight_complex_fft8 MACRO srcreg,srcinc,d1,d2,d4,dstreg,dstinc,e1,e2,e4,screg,scinc,maxrpt,L1pt,L1pd
	vmovapd	ymm1, [srcreg]			;; R1
	vmovapd	ymm7, [srcreg+d1]		;; R2
	vshufpd	ymm0, ymm1, ymm7, 15		;; Shuffle R1 and R2 to create R1/R2 hi		;  1
	vshufpd	ymm1, ymm1, ymm7, 0		;; Shuffle R1 and R2 to create R1/R2 low	;  2

	vmovapd	ymm2, [srcreg+d2]		;; R3
	vmovapd	ymm7, [srcreg+d2+d1]		;; R4
	vshufpd	ymm3, ymm2, ymm7, 15		;; Shuffle R3 and R4 to create R3/R4 hi		;  3
	vshufpd	ymm2, ymm2, ymm7, 0		;; Shuffle R3 and R4 to create R3/R4 low	;  4

	vmovapd	ymm6, [srcreg+d4]		;; R5
	vmovapd	ymm7, [srcreg+d4+d1]		;; R6
	vshufpd	ymm5, ymm6, ymm7, 15		;; Shuffle R5 and R6 to create R5/R6 hi		;  5
	vshufpd	ymm6, ymm6, ymm7, 0		;; Shuffle R5 and R6 to create R5/R6 low	;  6

	vmovapd	ymm4, [srcreg+d4+d2]		;; R7
	vmovapd	ymm8, [srcreg+d4+d2+d1]		;; R8
	vshufpd	ymm7, ymm4, ymm8, 15		;; Shuffle R7 and R8 to create R7/R8 hi		;  7
	vshufpd	ymm4, ymm4, ymm8, 0		;; Shuffle R7 and R8 to create R7/R8 low	;  8

	ylow128s ymm8, ymm1, ymm2		;; Shuffle R1/R2 low and R3/R4 low (first R1)	;  9-10
	L1prefetch srcreg+L1pd, L1pt
	L1prefetchw dstreg+L1pd, L1pt - L1PREFETCH_DEST_NONE

	ylow128s ymm9, ymm6, ymm4		;; Shuffle R5/R6 low and R7/R8 low (first R5)	;  10-11

	yhigh128s ymm1, ymm1, ymm2		;; Shuffle R1/R2 low and R3/R4 low (first R3)	;  11-12
	vmovapd	ymm15, [srcreg+32]		;; I1

	yhigh128s ymm6, ymm6, ymm4		;; Shuffle R5/R6 low and R7/R8 low (first R7)	;  12-13
	vaddpd	ymm4, ymm8, ymm9		;; R1 + R5 (new R1)				; 12-14
	vmovapd	ymm14, [srcreg+d1+32]		;; I2

	ylow128s ymm2, ymm0, ymm3		;; Shuffle R1/R2 hi and R3/R4 hi (first R2)	;  13-14
	vsubpd	ymm8, ymm8, ymm9		;; R1 - R5 (new R5)				; 13-15
	vmovapd	ymm13, [srcreg+d2+32]		;; I3

	ylow128s ymm9, ymm5, ymm7		;; Shuffle R5/R6 hi and R7/R8 hi (first R6)	;  14-15
	vaddpd	ymm10, ymm1, ymm6		;; R3 + R7 (new R3)				; 14-16
	vmovapd	ymm12, [srcreg+d2+d1+32]	;; I4

	yhigh128s ymm0, ymm0, ymm3		;; Shuffle R1/R2 hi and R3/R4 hi (first R4)	;  15-16
	vsubpd	ymm1, ymm1, ymm6		;; R3 - R7 (new R7)				; 15-17
	vmovapd	ymm11, [srcreg+d4+32]		;; I5

	yhigh128s ymm5, ymm5, ymm7		;; Shuffle R5/R6 hi and R7/R8 hi (first R8)	;  16-17
	vaddpd	ymm7, ymm2, ymm9		;; R2 + R6 (new R2)				; 16-18
	vmovapd	ymm3, [srcreg+d4+d1+32]		;; I6

	vsubpd	ymm2, ymm2, ymm9		;; R2 - R6 (new R6)				; 17-19
	L1prefetch srcreg+d1+L1pd, L1pt
	L1prefetchw dstreg+e1+L1pd, L1pt - L1PREFETCH_DEST_NONE

	vshufpd	ymm9, ymm15, ymm14, 15		;; Shuffle I1 and I2 to create I1/I2 hi		;  18
	vaddpd	ymm6, ymm0, ymm5		;; R4 + R8 (new R4)				; 18-20

	vshufpd	ymm15, ymm15, ymm14, 0		;; Shuffle I1 and I2 to create I1/I2 low	;  19
	vsubpd	ymm0, ymm0, ymm5		;; R4 - R8 (new R8)				; 19-21

	vshufpd	ymm5, ymm13, ymm12, 15		;; Shuffle I3 and I4 to create I3/I4 hi		;  20
	vaddpd	ymm14, ymm4, ymm10		;; R1 + R3 (newer R1)				; 20-22
	L1prefetch srcreg+d2+L1pd, L1pt
	L1prefetchw dstreg+e2+L1pd, L1pt - L1PREFETCH_DEST_NONE

	vshufpd	ymm13, ymm13, ymm12, 0		;; Shuffle I3 and I4 to create I3/I4 low	;  21
	vsubpd	ymm4, ymm4, ymm10		;; R1 - R3 (newer R3)				; 21-23

	vshufpd	ymm10, ymm11, ymm3, 15		;; Shuffle I5 and I6 to create I5/I6 hi		;  22
	vaddpd	ymm12, ymm7, ymm6		;; R2 + R4 (newer R2)				; 22-24

	ystore	[dstreg], ymm14			;; Temporarily save R1				; 23
	vshufpd	ymm11, ymm11, ymm3, 0		;; Shuffle I5 and I6 to create I5/I6 low	;  23
	vsubpd	ymm7, ymm7, ymm6		;; R2 - R4 (newer R4)				; 23-25

	vmovapd	ymm14, [srcreg+d4+d2+32]	;; I7
	vmovapd	ymm3, [srcreg+d4+d2+d1+32]	;; I8
	vshufpd	ymm6, ymm14, ymm3, 15		;; Shuffle I7 and I8 to create I7/I8 hi		;  24

	ystore	[dstreg+e1], ymm12		;; Temporarily save R2				; 25
	vshufpd	ymm14, ymm14, ymm3, 0		;; Shuffle I7 and I8 to create I7/I8 low	;  25

	ylow128s ymm3, ymm9, ymm5		;; Shuffle I1/I2 hi and I3/I4 hi (first I2)	;  26-27
	L1prefetch srcreg+d2+d1+L1pd, L1pt
	L1prefetchw dstreg+e2+e1+L1pd, L1pt - L1PREFETCH_DEST_NONE

	ylow128s ymm12, ymm10, ymm6		;; Shuffle I5/I6 hi and I7/I8 hi (first I6)	;  27-28

	yhigh128s ymm9, ymm9, ymm5		;; Shuffle I1/I2 hi and I3/I4 hi (first I4)	;  28-29

	yhigh128s ymm10, ymm10, ymm6		;; Shuffle I5/I6 hi and I7/I8 hi (first I8)	;  29-30
	vsubpd	ymm6, ymm3, ymm12		;; I2 - I6 (new I6)				; 29-31

	yhigh128s ymm5, ymm15, ymm13		;; Shuffle I1/I2 low and I3/I4 low (first I3)	;  30-31
	vaddpd	ymm3, ymm3, ymm12		;; I2 + I6 (new I2)				; 30-32

	yhigh128s ymm12, ymm11, ymm14		;; Shuffle I5/I6 low and I7/I8 low (first I7)	;  31-32
	L1prefetch srcreg+d4+L1pd, L1pt
	L1prefetchw dstreg+e4+L1pd, L1pt - L1PREFETCH_DEST_NONE

	ylow128s ymm15, ymm15, ymm13		;; Shuffle I1/I2 low and I3/I4 low (first I1)	;  32-33
	vsubpd	ymm13, ymm9, ymm10		;; I4 - I8 (new I8)				; 31-33
	vaddpd	ymm9, ymm9, ymm10		;; I4 + I8 (new I4)				; 32-34
	vmovapd	ymm10, YMM_SQRTHALF

	ylow128s ymm11, ymm11, ymm14		;; Shuffle I5/I6 low and I7/I8 low (first I5)	;  33-34
	vaddpd	ymm14, ymm5, ymm12		;; I3 + I7 (new I3)				; 33-35

	vsubpd	ymm5, ymm5, ymm12		;; I3 - I7 (new I7)				; 34-36
	L1prefetch srcreg+d4+d1+L1pd, L1pt
	L1prefetchw dstreg+e4+e1+L1pd, L1pt - L1PREFETCH_DEST_NONE

	vaddpd	ymm12, ymm6, ymm0		;; I6 + R8 (new I6)				; 35-37

	vsubpd	ymm6, ymm6, ymm0		;; I6 - R8 (new I8)				; 36-38

	vaddpd	ymm0, ymm2, ymm13		;; R6 + I8 (new R8)				; 37-39

	vsubpd	ymm2, ymm2, ymm13		;; R6 - I8 (new R6)				; 38-40
	vmulpd	ymm12, ymm12, ymm10		;; I6 = I6 * SQRTHALF				;  38-42

	vaddpd	ymm13, ymm15, ymm11		;; I1 + I5 (new I1)				; 39-41
	vmulpd	ymm6, ymm6, ymm10		;; I8 = I8 * SQRTHALF				;  39-43

	vsubpd	ymm15, ymm15, ymm11		;; I1 - I5 (new I5)				; 40-42
	vmulpd	ymm0, ymm0, ymm10		;; R8 = R8 * SQRTHALF				;  40-44

	vsubpd	ymm11, ymm3, ymm9		;; I2 - I4 (newer I4)				; 41-43
	vmulpd	ymm2, ymm2, ymm10		;; R6 = R6 * SQRTHALF				;  41-45
	vmovapd	ymm10, [screg+128+32]		;; cosine for R3/I3

	vaddpd	ymm3, ymm3, ymm9		;; I2 + I4 (newer I2)				; 42-44
	L1prefetch srcreg+d4+d2+L1pd, L1pt
	L1prefetchw dstreg+e4+e2+L1pd, L1pt - L1PREFETCH_DEST_NONE

	vsubpd	ymm9, ymm13, ymm14		;; I1 - I3 (newer I3)				; 43-45

	vaddpd	ymm13, ymm13, ymm14		;; I1 + I3 (newer I1)				; 44-46
	vmovapd	ymm14, [screg+128]		;; sine for R3/I3

	ystore	[dstreg+e1+32], ymm3		;; Temporarily save I2				; 45
	vsubpd	ymm3, ymm0, ymm6		;; R8 - I8 (newer R8)				; 45-47

	vaddpd	ymm0, ymm0, ymm6		;; R8 + I8 (newer I8)				; 46-48

	ystore	[dstreg+32], ymm13		;; Temporarily save I1				; 47
	vsubpd	ymm13, ymm4, ymm11		;; R3 - I4 (last R3)				; 47-49		n 50

	vaddpd	ymm6, ymm9, ymm7		;; I3 + R4 (last I3)				; 48-50		n 51
	L1prefetch srcreg+d4+d2+d1+L1pd, L1pt
	L1prefetchw dstreg+e4+e2+e1+L1pd, L1pt - L1PREFETCH_DEST_NONE

	vaddpd	ymm4, ymm4, ymm11		;; R3 + I4 (last R4)				; 49-51		n 54

	vsubpd	ymm9, ymm9, ymm7		;; I3 - R4 (last I4)				; 50-52		n 55
	vmulpd	ymm7, ymm13, ymm10		;; A3 = R3 * cosine				;  50-54

	vaddpd	ymm11, ymm8, ymm5		;; R5 + I7 (newer R7)				; 51-53
	vmulpd	ymm10, ymm6, ymm10		;; B3 = I3 * cosine				;  51-55

	vsubpd	ymm8, ymm8, ymm5		;; R5 - I7 (newer R5)				; 52-54
	vmulpd	ymm6, ymm6, ymm14		;; C3 = I3 * sine				;  52-56

	vsubpd	ymm5, ymm15, ymm1		;; I5 - R7 (newer I7)				; 53-55
	vmulpd	ymm13, ymm13, ymm14		;; D3 = R3 * sine				;  53-57
	vmovapd	ymm14, [screg+384+32]		;; cosine for R4/I4

	vaddpd	ymm15, ymm15, ymm1		;; I5 + R7 (newer I5)				; 54-56
	vmulpd	ymm1, ymm4, ymm14		;; A4 = R4 * cosine				;  54-58

	vmulpd	ymm14, ymm9, ymm14		;; B4 = I4 * cosine				;  55-59
	vsubpd	ymm7, ymm7, ymm6		;; A3 = A3 - C3 (final R3)			; 57-59
	vsubpd	ymm6, ymm11, ymm0		;; R7 - I8 (last R7)				; 55-57		n 58

	ystore	[dstreg+e2], ymm7		;; Save R3					; 60
	vmovapd	ymm7, [screg+384]		;; sine for R4/I4
	vmulpd	ymm9, ymm9, ymm7		;; C4 = I4 * sine				;  56-60
	vmulpd	ymm4, ymm4, ymm7		;; D4 = R4 * sine				;  57-61
	vaddpd	ymm7, ymm5, ymm3		;; I7 + R8 (last I7)				; 56-58		n 59

	vaddpd	ymm10, ymm10, ymm13		;; B3 = B3 + D3 (final I3)			; 58-60
	vmovapd	ymm13, [screg+192+32]		;; cosine for R7/I7
	ystore	[dstreg+e2+32], ymm10		;; Save I3					; 61
	vmulpd	ymm10, ymm6, ymm13		;; A7 = R7 * cosine				;  58-62

	vaddpd	ymm11, ymm11, ymm0		;; R7 + I8 (last R8)				; 59-61		n 62
	vmovapd	ymm0, [screg+192]		;; sine for R7/I7
	vmulpd	ymm13, ymm7, ymm13		;; B7 = I7 * cosine				;  59-63

	vsubpd	ymm5, ymm5, ymm3		;; I7 - R8 (last I8)				; 60-62		n 63
	vmulpd	ymm7, ymm7, ymm0		;; C7 = I7 * sine				;  60-64

	vsubpd	ymm3, ymm2, ymm12		;; R6 - I6 (newer R6)				; 61-63		n 64
	vmulpd	ymm6, ymm6, ymm0		;; D7 = R7 * sine				;  61-65
	vmovapd	ymm0, [screg+448+32]		;; cosine for R8/I8

	vaddpd	ymm2, ymm2, ymm12		;; R6 + I6 (newer I6)				; 62-64		n 65
	vmulpd	ymm12, ymm11, ymm0		;; A8 = R8 * cosine				;  62-66

	vsubpd	ymm1, ymm1, ymm9		;; A4 = A4 - C4 (final R4)			; 63-65
	vmulpd	ymm0, ymm5, ymm0		;; B8 = I8 * cosine				;  63-67

	vsubpd	ymm9, ymm8, ymm3		;; R5 - R6 (last R6)				; 64-66		n 67
	ystore	[dstreg+e2+e1], ymm1		;; Save R4					; 66
	vmovapd	ymm1, [screg+448]		;; sine for R8/I8
	vmulpd	ymm5, ymm5, ymm1		;; C8 = I8 * sine				;  64-68

	vmulpd	ymm11, ymm11, ymm1		;; D8 = R8 * sine				;  65-69
	vsubpd	ymm1, ymm15, ymm2		;; I5 - I6 (last I6)				; 65-67		n 68

	vaddpd	ymm14, ymm14, ymm4		;; B4 = B4 + D4 (final I4)			; 66-68
	vmovapd	ymm4, [screg+320+32]		;; cosine for R6/I6

	vaddpd	ymm8, ymm8, ymm3		;; R5 + R6 (last R5)				; 67-69		n 71
	vmulpd	ymm3, ymm9, ymm4		;; A6 = R6 * cosine				;  67-71

	vaddpd	ymm15, ymm15, ymm2		;; I5 + I6 (last I5)				; 68-70		n 72
	vmulpd	ymm4, ymm1, ymm4		;; B6 = I6 * cosine				;  68-72
	vmovapd	ymm2, [screg+320]		;; sine for R6/I6

	ystore	[dstreg+e2+e1+32], ymm14	;; Save I4					; 69
	vsubpd	ymm10, ymm10, ymm7		;; A7 = A7 - C7 (final R7)			; 69-71
	vmulpd	ymm1, ymm1, ymm2		;; C6 = I6 * sine				;  69-73
	vmovapd	ymm14, [screg+64+32]		;; cosine for R5/I5

	vaddpd	ymm13, ymm13, ymm6		;; B7 = B7 + D7 (final I7)			; 70-72
	vmulpd	ymm9, ymm9, ymm2		;; D6 = R6 * sine				;  70-74
	vmovapd	ymm7, [dstreg]			;; Reload R1

	vsubpd	ymm12, ymm12, ymm5		;; A8 = A8 - C8 (final R8)			; 71-73
	vmulpd	ymm5, ymm8, ymm14		;; A5 = R5 * cosine				;  71-75
	vmovapd	ymm6, [dstreg+e1]		;; Reload R2

	ystore	[dstreg+e4+e2], ymm10		;; Save R7					; 72
	vsubpd	ymm10, ymm7, ymm6		;; R1 - R2 (last R2)				; 72-74		n 75
	vmulpd	ymm14, ymm15, ymm14		;; B5 = I5 * cosine				;  72-76
	vmovapd	ymm2, [screg+64]		;; sine for R5/I5

	ystore	[dstreg+e4+e2+32], ymm13	;; Save I7					; 73
	vaddpd	ymm7, ymm7, ymm6		;; R1 + R2 (last R1)				; 73-75		n 77
	vmulpd	ymm15, ymm15, ymm2		;; C5 = I5 * sine				;  73-77
	vmovapd	ymm13, [dstreg+32]		;; Reload I1

	ystore	[dstreg+e4+e2+e1], ymm12	;; Save R8					; 74
	vaddpd	ymm0, ymm0, ymm11		;; B8 = B8 + D8 (final I8)			; 74-76
	vmulpd	ymm8, ymm8, ymm2		;; D5 = R5 * sine				;  74-78
	vmovapd	ymm6, [dstreg+e1+32]		;; Reload I2
	vmovapd	ymm12, [screg+256+32]		;; cosine for R2/I2

	vsubpd	ymm2, ymm13, ymm6		;; I1 - I2 (last I2)				; 75-77		n 78
	vmulpd	ymm11, ymm10, ymm12		;; A2 = R2 * cosine				;  75-79

	vaddpd	ymm13, ymm13, ymm6		;; I1 + I2 (last I1)				; 76-78		n 81
	vmovapd	ymm6, [screg+256]		;; sine for R2/I2
	vmulpd	ymm10, ymm10, ymm6		;; D2 = R2 * sine				;  76-80

	ystore	[dstreg+e4+e2+e1+32], ymm0	;; Save I8					; 77
	vmovapd	ymm0, [screg+0+32]		;; cosine for R1/I1
	vsubpd	ymm3, ymm3, ymm1		;; A6 = A6 - C6 (final R6)			; 77-79
	vmulpd	ymm1, ymm7, ymm0		;; A1 = R1 * cosine				;  77-81

	vaddpd	ymm4, ymm4, ymm9		;; B6 = B6 + D6 (final I6)			; 78-80
	vmulpd	ymm6, ymm2, ymm6		;; C2 = I2 * sine				;  78-82
	vmovapd	ymm9, [screg+0]			;; sine for R1/I1

	vsubpd	ymm5, ymm5, ymm15		;; A5 = A5 - C5 (final R5)			; 79-81
	vmulpd	ymm2, ymm2, ymm12		;; B2 = I2 * cosine				;  79-83

	ystore	[dstreg+e4+e1], ymm3		;; Save R6					; 80
	vaddpd	ymm14, ymm14, ymm8		;; B5 = B5 + D5 (final I5)			; 80-82
	vmulpd	ymm0, ymm13, ymm0		;; B1 = I1 * cosine				;  80-84

	ystore	[dstreg+e4+e1+32], ymm4		;; Save I6					; 81
	vmulpd	ymm13, ymm13, ymm9		;; C1 = I1 * sine				;  81-85

	ystore	[dstreg+e4], ymm5		;; Save R5					; 82
	vmulpd	ymm7, ymm7, ymm9		;; D1 = R1 * sine				;  82-86

	ystore	[dstreg+e4+32], ymm14		;; Save I5					; 83
	vsubpd	ymm11, ymm11, ymm6		;; A2 = A2 - C2 (final R2)			; 83-85

	vaddpd	ymm2, ymm2, ymm10		;; B2 = B2 + D2 (final I2)			; 84-86

	ystore	[dstreg+e1], ymm11		;; Save R2					; 86
	vsubpd	ymm1, ymm1, ymm13		;; A1 = A1 - C1 (final R1)			; 86-88

	ystore	[dstreg+e1+32], ymm2		;; Save I2					; 87
	vaddpd	ymm0, ymm0, ymm7		;; B1 = B1 + D1 (final I1)			; 87-89

	ystore	[dstreg], ymm1			;; Save R1					; 89
	ystore	[dstreg+32], ymm0		;; Save I1					; 90

	bump	srcreg, srcinc
	bump	dstreg, dstinc
	bump	screg, scinc
	ENDM

IF (@INSTR(,%yarch,<FMA3>) NE 0)

yr8_rsc_sg8cl_eight_complex_fft8 MACRO srcreg,srcinc,d1,d2,d4,dstreg,dstinc,e1,e2,e4,screg,scinc,maxrpt,L1pt,L1pd
	vmovapd	ymm1, [srcreg]			;; R1
	vmovapd	ymm7, [srcreg+d1]		;; R2
	vshufpd	ymm0, ymm1, ymm7, 15		;; Shuffle R1 and R2 to create R1/R2 hi		;  1
	vshufpd	ymm1, ymm1, ymm7, 0		;; Shuffle R1 and R2 to create R1/R2 low	;  2

	vmovapd	ymm2, [srcreg+d2]		;; R3
	vmovapd	ymm7, [srcreg+d2+d1]		;; R4
	vshufpd	ymm3, ymm2, ymm7, 15		;; Shuffle R3 and R4 to create R3/R4 hi		;  3
	vshufpd	ymm2, ymm2, ymm7, 0		;; Shuffle R3 and R4 to create R3/R4 low	;  4

	vmovapd	ymm6, [srcreg+d4]		;; R5
	vmovapd	ymm7, [srcreg+d4+d1]		;; R6
	vshufpd	ymm5, ymm6, ymm7, 15		;; Shuffle R5 and R6 to create R5/R6 hi		;  5
	vshufpd	ymm6, ymm6, ymm7, 0		;; Shuffle R5 and R6 to create R5/R6 low	;  6

	vmovapd	ymm4, [srcreg+d4+d2]		;; R7
	vmovapd	ymm8, [srcreg+d4+d2+d1]		;; R8
	vshufpd	ymm7, ymm4, ymm8, 15		;; Shuffle R7 and R8 to create R7/R8 hi		;  7
	vshufpd	ymm4, ymm4, ymm8, 0		;; Shuffle R7 and R8 to create R7/R8 low	;  8

	ylow128s ymm8, ymm1, ymm2		;; Shuffle R1/R2 low and R3/R4 low (first R1)	;  9-10

	ylow128s ymm9, ymm6, ymm4		;; Shuffle R5/R6 low and R7/R8 low (first R5)	;  10-11

	yhigh128s ymm1, ymm1, ymm2		;; Shuffle R1/R2 low and R3/R4 low (first R3)	;  11-12
	vmovapd	ymm15, [srcreg+32]		;; I1

	yhigh128s ymm6, ymm6, ymm4		;; Shuffle R5/R6 low and R7/R8 low (first R7)	;  12-13
	vaddpd	ymm4, ymm8, ymm9		;; R1 + R5 (new R1)				; 12-14
	vmovapd	ymm14, [srcreg+d1+32]		;; I2

	ylow128s ymm2, ymm0, ymm3		;; Shuffle R1/R2 hi and R3/R4 hi (first R2)	;  13-14
	vsubpd	ymm8, ymm8, ymm9		;; R1 - R5 (new R5)				; 13-15
	vmovapd	ymm13, [srcreg+d2+32]		;; I3

	ylow128s ymm9, ymm5, ymm7		;; Shuffle R5/R6 hi and R7/R8 hi (first R6)	;  14-15
	vaddpd	ymm10, ymm1, ymm6		;; R3 + R7 (new R3)				; 14-16
	vmovapd	ymm12, [srcreg+d2+d1+32]	;; I4

	yhigh128s ymm0, ymm0, ymm3		;; Shuffle R1/R2 hi and R3/R4 hi (first R4)	;  15-16
	vsubpd	ymm1, ymm1, ymm6		;; R3 - R7 (new R7)				; 15-17
	vmovapd	ymm11, [srcreg+d4+32]		;; I5

	yhigh128s ymm5, ymm5, ymm7		;; Shuffle R5/R6 hi and R7/R8 hi (first R8)	;  16-17
	vaddpd	ymm7, ymm2, ymm9		;; R2 + R6 (new R2)				; 16-18
	vmovapd	ymm3, [srcreg+d4+d1+32]		;; I6

	vsubpd	ymm2, ymm2, ymm9		;; R2 - R6 (new R6)				; 17-19
	L1prefetch srcreg+L1pd, L1pt
	L1prefetchw dstreg+L1pd, L1pt - L1PREFETCH_DEST_NONE

	vshufpd	ymm9, ymm15, ymm14, 15		;; Shuffle I1 and I2 to create I1/I2 hi		;  18
	vaddpd	ymm6, ymm0, ymm5		;; R4 + R8 (new R4)				; 18-20

	vshufpd	ymm15, ymm15, ymm14, 0		;; Shuffle I1 and I2 to create I1/I2 low	;  19
	vsubpd	ymm0, ymm0, ymm5		;; R4 - R8 (new R8)				; 19-21

	vshufpd	ymm5, ymm13, ymm12, 15		;; Shuffle I3 and I4 to create I3/I4 hi		;  20
	vaddpd	ymm14, ymm4, ymm10		;; R1 + R3 (newer R1)				; 20-22

	vshufpd	ymm13, ymm13, ymm12, 0		;; Shuffle I3 and I4 to create I3/I4 low	;  21
	vaddpd	ymm12, ymm7, ymm6		;; R2 + R4 (newer R2)				; 21-23

	vsubpd	ymm4, ymm4, ymm10		;; R1 - R3 (newer R3)				; 22-24
	vshufpd	ymm10, ymm11, ymm3, 15		;; Shuffle I5 and I6 to create I5/I6 hi		;  22

	vshufpd	ymm11, ymm11, ymm3, 0		;; Shuffle I5 and I6 to create I5/I6 low	;  23
	vsubpd	ymm7, ymm7, ymm6		;; R2 - R4 (newer R4)				; 23-25
	vmovapd	ymm3, [srcreg+d4+d2+32]		;; I7

	vaddpd	ymm6, ymm14, ymm12		;; R1 + R2 (last R1)				; 24-26

	vsubpd	ymm14, ymm14, ymm12		;; R1 - R2 (last R2)				; 25-27

	vmulpd	ymm12, ymm6, [screg+0+32]	;; R1*cos					; 27-31
	vmulpd	ymm6, ymm6, [screg+0]		;; R1*sin					; 27-31
	ystore	[dstreg+32], ymm12		;; Temporarily save R1*cos			; 32+1
	vmovapd	ymm12, [srcreg+d4+d2+d1+32]	;; I8
	ystore	[dstreg], ymm6			;; Temporarily save R1*sin			; 32

	vshufpd	ymm6, ymm3, ymm12, 15		;; Shuffle I7 and I8 to create I7/I8 hi		;  24
	L1prefetch srcreg+d1+L1pd, L1pt
	L1prefetchw dstreg+e1+L1pd, L1pt - L1PREFETCH_DEST_NONE

	vshufpd	ymm3, ymm3, ymm12, 0		;; Shuffle I7 and I8 to create I7/I8 low	;  25

	ylow128s ymm12, ymm9, ymm5		;; Shuffle I1/I2 hi and I3/I4 hi (first I2)	;  26-27	n 30

	yhigh128s ymm9, ymm9, ymm5		;; Shuffle I1/I2 hi and I3/I4 hi (first I4)	;  27-28	n 30

	ylow128s ymm5, ymm10, ymm6		;; Shuffle I5/I6 hi and I7/I8 hi (first I6)	;  28-29	n 31

	yhigh128s ymm10, ymm10, ymm6		;; Shuffle I5/I6 hi and I7/I8 hi (first I8)	;  29-30	n 31
	vmulpd	ymm6, ymm14, [screg+256+32]	;; R2*cos					; 28-32
	vmulpd	ymm14, ymm14, [screg+256]	;; R2*sin					; 28-32

	ystore	[dstreg+e1+32], ymm6		;; Temporarily save R2*cos			; 33+1
	vmovapd	ymm6, YMM_ONE
	ystore	[dstreg+e1], ymm14		;; Temporarily save R2*sin			; 33+2
	yfmsubpd ymm14, ymm12, ymm6, ymm5	;; I2 - I6 (new I6)				; 30-34		n 36
	yfmaddpd ymm12, ymm12, ymm6, ymm5	;; I2 + I6 (new I2)				; 30-34		n 38
	yhigh128s ymm5, ymm15, ymm13		;; Shuffle I1/I2 low and I3/I4 low (first I3)	;  30-31	n 34

	ylow128s ymm15, ymm15, ymm13		;; Shuffle I1/I2 low and I3/I4 low (first I1)	;  31-32	n 35
	yfmsubpd ymm13, ymm9, ymm6, ymm10	;; I4 - I8 (new I8)				; 31-35		n 37
	yfmaddpd ymm9, ymm9, ymm6, ymm10	;; I4 + I8 (new I4)				; 31-35		n 38

	yhigh128s ymm10, ymm11, ymm3		;; Shuffle I5/I6 low and I7/I8 low (first I7)	;  32-33	n 34
	L1prefetch srcreg+d2+L1pd, L1pt
	L1prefetchw dstreg+e2+L1pd, L1pt - L1PREFETCH_DEST_NONE

	ylow128s ymm11, ymm11, ymm3		;; Shuffle I5/I6 low and I7/I8 low (first I5)	;  33-34	n 35

	yfmsubpd ymm3, ymm5, ymm6, ymm10	;; I3 - I7 (new I7)				; 34-38		n 39
	yfmaddpd ymm5, ymm5, ymm6, ymm10	;; I3 + I7 (new I3)				; 34-38		n 40

	yfmaddpd ymm10, ymm15, ymm6, ymm11	;; I1 + I5 (new I1)				; 35-39		n 40
	yfmsubpd ymm15, ymm15, ymm6, ymm11	;; I1 - I5 (new I5)				; 35-39		n 41
	L1prefetch srcreg+d2+d1+L1pd, L1pt
	L1prefetchw dstreg+e2+e1+L1pd, L1pt - L1PREFETCH_DEST_NONE

	yfmsubpd ymm11, ymm14, ymm6, ymm0	;; I6 - R8 (new2 I8)				; 36-40		n 42
	yfmaddpd ymm14, ymm14, ymm6, ymm0	;; I6 + R8 (new2 I6)				; 36-40		n 43

	yfmaddpd ymm0, ymm2, ymm6, ymm13	;; R6 + I8 (new2 R8)				; 37-41		n 42
	yfmsubpd ymm2, ymm2, ymm6, ymm13	;; R6 - I8 (new2 R6)				; 37-41		n 43

	yfmsubpd ymm13, ymm12, ymm6, ymm9	;; I2 - I4 (newer I4)				; 38-42		n 44
	yfmaddpd ymm12, ymm12, ymm6, ymm9	;; I2 + I4 (newer I2)				; 38-42		n 46
	L1prefetch srcreg+d4+L1pd, L1pt
	L1prefetchw dstreg+e4+L1pd, L1pt - L1PREFETCH_DEST_NONE

	yfmaddpd ymm9, ymm8, ymm6, ymm3		;; R5 + I7 (newer R7)				; 39-43		n 47
	yfmsubpd ymm8, ymm8, ymm6, ymm3		;; R5 - I7 (newer R5)				; 39-43		n 49

	yfmsubpd ymm3, ymm10, ymm6, ymm5	;; I1 - I3 (newer I3)				; 40-44		n 45
	yfmaddpd ymm10, ymm10, ymm6, ymm5	;; I1 + I3 (newer I1)				; 40-44		n 46

	yfmsubpd ymm5, ymm15, ymm6, ymm1	;; I5 - R7 (newer I7)				; 41-45		n 48
 	yfmaddpd ymm15, ymm15, ymm6, ymm1	;; I5 + R7 (newer I5)				; 41-45		n 50
	L1prefetch srcreg+d4+d1+L1pd, L1pt
	L1prefetchw dstreg+e4+e1+L1pd, L1pt - L1PREFETCH_DEST_NONE

	yfmaddpd ymm1, ymm0, ymm6, ymm11	;; R8 + I8 (newer I8/SQRTHALF)			; 42-46		n 47
	yfmsubpd ymm0, ymm0, ymm6, ymm11	;; R8 - I8 (newer R8/SQRTHALF)			; 42-46		n 48

	yfmsubpd ymm11, ymm2, ymm6, ymm14	;; R6 - I6 (newer R6/SQRTHALF)			; 43-47		n 49
	yfmaddpd ymm2, ymm2, ymm6, ymm14	;; R6 + I6 (newer I6/SQRTHALF)			; 43-47		n 50

	yfmsubpd ymm14, ymm4, ymm6, ymm13	;; R3 - I4 (last R3)				; 44-48		n 51
	yfmaddpd ymm4, ymm4, ymm6, ymm13	;; R3 + I4 (last R4)				; 44-48		n 52

	yfmaddpd ymm13, ymm3, ymm6, ymm7	;; I3 + R4 (last I3)				; 45-49		n 51
	yfmsubpd ymm3, ymm3, ymm6, ymm7		;; I3 - R4 (last I4)				; 45-49		n 52
	L1prefetch srcreg+d4+d2+L1pd, L1pt
	L1prefetchw dstreg+e4+e2+L1pd, L1pt - L1PREFETCH_DEST_NONE

	yfmsubpd ymm7, ymm10, ymm6, ymm12	;; I1 - I2 (last I2)				; 46-50		n 59
	yfmaddpd ymm10, ymm10, ymm6, ymm12	;; I1 + I2 (last I1)				; 46-50		n 60

	vmovapd	ymm6, YMM_SQRTHALF
	yfnmaddpd ymm12, ymm1, ymm6, ymm9	;; R7 - I8 * SQRTHALF (last R7)			; 47-51		n 53
	yfmaddpd ymm1, ymm1, ymm6, ymm9		;; R7 + I8 * SQRTHALF (last R8)			; 47-51		n 54

	yfmaddpd ymm9, ymm0, ymm6, ymm5		;; I7 + R8 * SQRTHALF (last I7)			; 48-52		n 53
	yfnmaddpd ymm0, ymm0, ymm6, ymm5	;; I7 - R8 * SQRTHALF (last I8)			; 48-52		n 54

	yfnmaddpd ymm5, ymm11, ymm6, ymm8	;; R5 - R6 * SQRTHALF (last R6)			; 49-53		n 55
	yfmaddpd ymm11, ymm11, ymm6, ymm8	;; R5 + R6 * SQRTHALF (last R5)			; 49-53		n 56
	L1prefetch srcreg+d4+d2+d1+L1pd, L1pt
	L1prefetchw dstreg+e4+e2+e1+L1pd, L1pt - L1PREFETCH_DEST_NONE

	yfnmaddpd ymm8, ymm2, ymm6, ymm15	;; I5 - I6 * SQRTHALF (last I6)			; 50-54		n 55
	yfmaddpd ymm2, ymm2, ymm6, ymm15	;; I5 + I6 * SQRTHALF (last I5)			; 50-54		n 56
	bump	srcreg, srcinc

	vmovapd	ymm15, [screg+128+32]		;; cosine for R3/I3
	vmulpd	ymm6, ymm14, ymm15		;; A3 = R3 * cosine				; 51-55
	vmulpd	ymm15, ymm13, ymm15		;; B3 = I3 * cosine				; 51-55

	yfnmaddpd ymm13, ymm13, [screg+128], ymm6 ;; A3 = A3 - I3 * sine (final R3)		; 56-60
	yfmaddpd ymm14, ymm14, [screg+128], ymm15 ;; B3 = B3 + R3 * sine (final I3)		; 56-60

	vmovapd	ymm15, [screg+384+32]		;; cosine for R4/I4
	vmulpd	ymm6, ymm4, ymm15		;; A4 = R4 * cosine				; 52-56
	vmulpd	ymm15, ymm3, ymm15		;; B4 = I4 * cosine				; 52-56

	ystore	[dstreg+e2], ymm13		;; Save R3					; 61
	vmovapd	ymm13, [screg+384]		;; sine for R4/I4
	yfnmaddpd ymm3, ymm3, ymm13, ymm6	;; A4 = A4 - I4 * sine (final R4)		; 57-61
	yfmaddpd ymm4, ymm4, ymm13, ymm15	;; B4 = B4 + R4 * sine (final I4)		; 57-61

	vmovapd	ymm15, [screg+192+32]		;; cosine for R7/I7
	vmulpd	ymm6, ymm12, ymm15		;; A7 = R7 * cosine				; 53-57
	vmulpd	ymm15, ymm9, ymm15		;; B7 = I7 * cosine				; 53-57

	vmovapd	ymm13, [screg+448+32]		;; cosine for R8/I8
	ystore	[dstreg+e2+32], ymm14		;; Save I3					; 61+1
	vmulpd	ymm14, ymm1, ymm13		;; A8 = R8 * cosine				; 54-58
	vmulpd	ymm13, ymm0, ymm13		;; B8 = I8 * cosine				; 54-58

	ystore	[dstreg+e2+e1], ymm3		;; Save R4					; 62+1
	vmovapd	ymm3, [screg+192]		;; sine for R7/I7
	yfnmaddpd ymm9, ymm9, ymm3, ymm6	;; A7 = A7 - I7 * sine (final R7)		; 58-62
	yfmaddpd ymm12, ymm12, ymm3, ymm15	;; B7 = B7 + R7 * sine (final I7)		; 58-62

	vmovapd	ymm15, [screg+320+32]		;; cosine for R6/I6
	vmulpd	ymm6, ymm5, ymm15		;; A6 = R6 * cosine				; 55-59
	vmulpd	ymm15, ymm8, ymm15		;; B6 = I6 * cosine				; 55-59

	vmovapd	ymm3, [screg+64+32]		;; cosine for R5/I5
	ystore	[dstreg+e2+e1+32], ymm4		;; Save I4					; 62+2
	vmulpd	ymm4, ymm11, ymm3		;; A5 = R5 * cosine				; 59-63
	vmulpd	ymm3, ymm2, ymm3		;; B5 = I5 * cosine				; 59-63

	ystore	[dstreg+e4+e2], ymm9		;; Save R7					; 63+2
	vmovapd	ymm9, [screg+448]		;; sine for R8/I8
	yfnmaddpd ymm0, ymm0, ymm9, ymm14	;; A8 = A8 - I8 * sine (final R8)		; 60-64
	yfmaddpd ymm1, ymm1, ymm9, ymm13	;; B8 = B8 + R8 * sine (final I8)		; 60-64

	vmovapd	ymm14, [dstreg+32]		;; Reload last A1 = R1 * cosine
	vmovapd	ymm9, [dstreg]			;; Reload last B1 = R1 * sine
	yfnmaddpd ymm14, ymm10, [screg+0], ymm14 ;; A1 = A1 - I1 * sine (final R1)		; 61-65
	yfmaddpd ymm10, ymm10, [screg+0+32], ymm9 ;; B1 = I1 * cosine + B1 (final I1)		; 61-65

	vmovapd	ymm13, [dstreg+e1+32]		;; Reload last A2 = R2 * cosine
	vmovapd	ymm9, [dstreg+e1]		;; Reload last B2 = R2 * sine
	yfnmaddpd ymm13, ymm7, [screg+256], ymm13 ;; A2 = A2 - I2 * sine (final R2)		; 62-66
	yfmaddpd ymm7, ymm7, [screg+256+32], ymm9 ;; B2 = I2 * cosine + B2 (final I2)		; 62-66

	vmovapd	ymm9, [screg+320]		;; sine for R6/I6
	yfnmaddpd ymm8, ymm8, ymm9, ymm6	;; A6 = A6 - I6 * sine (final R6)		; 63-67
	yfmaddpd ymm5, ymm5, ymm9, ymm15	;; B6 = B6 + R6 * sine (final I6)		; 63-67

	vmovapd	ymm15, [screg+64]		;; sine for R5/I5
	yfnmaddpd ymm2, ymm2, ymm15, ymm4	;; A5 = A5 - I5 * sine (final R5)		; 64-68
	yfmaddpd ymm11, ymm11, ymm15, ymm3	;; B5 = B5 + R5 * sine (final I5)		; 64-68

	ystore	[dstreg+e4+e2+32], ymm12	;; Save I7					; 63+3
	ystore	[dstreg+e4+e2+e1], ymm0		;; Save R8					; 65+2
	ystore	[dstreg+e4+e2+e1+32], ymm1	;; Save I8					; 65+3
	ystore	[dstreg], ymm14			;; Save R1					; 66+3
	ystore	[dstreg+32], ymm10		;; Save I1					; 66+4
	ystore	[dstreg+e1], ymm13		;; Save R2					; 67+4
	ystore	[dstreg+e1+32], ymm7		;; Save I2					; 67+5
	ystore	[dstreg+e4+e1], ymm8		;; Save R6					; 68+5
	ystore	[dstreg+e4+e1+32], ymm5		;; Save I6					; 68+6
	ystore	[dstreg+e4], ymm2		;; Save R5					; 69+6
	ystore	[dstreg+e4+32], ymm11		;; Save I5					; 69+7

	bump	screg, scinc
	bump	dstreg, dstinc
	ENDM

ENDIF

ENDIF


; These versions use registers for distances between blocks.  This lets us share pass1 code.

yr8_rsc_sg8clreg_eight_complex_fft8_preload MACRO
	ENDM
yr8_rsc_sg8clreg_eight_complex_fft8 MACRO srcreg,srcinc,d1,d2,d4,dstreg,dstinc,e1reg,e3reg,dst4reg,screg,scinc,maxrpt,L1pt,L1preg,L1p4reg
	NOT IMPLMENTED IN 32-BIT
	ENDM

IFDEF X86_64

yr8_rsc_sg8clreg_eight_complex_fft8_preload MACRO
	ENDM
yr8_rsc_sg8clreg_eight_complex_fft8 MACRO srcreg,srcinc,d1,d2,d4,dstreg,dstinc,e1reg,e3reg,dst4reg,screg,scinc,maxrpt,L1pt,L1preg,L1p4reg
	vmovapd	ymm1, [srcreg]			;; R1
	vmovapd	ymm7, [srcreg+d1]		;; R2
	vshufpd	ymm0, ymm1, ymm7, 15		;; Shuffle R1 and R2 to create R1/R2 hi		;  1
	vshufpd	ymm1, ymm1, ymm7, 0		;; Shuffle R1 and R2 to create R1/R2 low	;  2

	vmovapd	ymm2, [srcreg+d2]		;; R3
	vmovapd	ymm7, [srcreg+d2+d1]		;; R4
	vshufpd	ymm3, ymm2, ymm7, 15		;; Shuffle R3 and R4 to create R3/R4 hi		;  3
	vshufpd	ymm2, ymm2, ymm7, 0		;; Shuffle R3 and R4 to create R3/R4 low	;  4

	vmovapd	ymm6, [srcreg+d4]		;; R5
	vmovapd	ymm7, [srcreg+d4+d1]		;; R6
	vshufpd	ymm5, ymm6, ymm7, 15		;; Shuffle R5 and R6 to create R5/R6 hi		;  5
	vshufpd	ymm6, ymm6, ymm7, 0		;; Shuffle R5 and R6 to create R5/R6 low	;  6

	vmovapd	ymm4, [srcreg+d4+d2]		;; R7
	vmovapd	ymm8, [srcreg+d4+d2+d1]		;; R8
	vshufpd	ymm7, ymm4, ymm8, 15		;; Shuffle R7 and R8 to create R7/R8 hi		;  7
	vshufpd	ymm4, ymm4, ymm8, 0		;; Shuffle R7 and R8 to create R7/R8 low	;  8

	ylow128s ymm8, ymm1, ymm2		;; Shuffle R1/R2 low and R3/R4 low (first R1)	;  9-10
	L1prefetchw L1preg, L1pt - L1PREFETCH_DEST_NONE

	ylow128s ymm9, ymm6, ymm4		;; Shuffle R5/R6 low and R7/R8 low (first R5)	;  10-11

	yhigh128s ymm1, ymm1, ymm2		;; Shuffle R1/R2 low and R3/R4 low (first R3)	;  11-12
	vmovapd	ymm15, [srcreg+32]		;; I1

	yhigh128s ymm6, ymm6, ymm4		;; Shuffle R5/R6 low and R7/R8 low (first R7)	;  12-13
	vaddpd	ymm4, ymm8, ymm9		;; R1 + R5 (new R1)				; 12-14
	vmovapd	ymm14, [srcreg+d1+32]		;; I2

	ylow128s ymm2, ymm0, ymm3		;; Shuffle R1/R2 hi and R3/R4 hi (first R2)	;  13-14
	vsubpd	ymm8, ymm8, ymm9		;; R1 - R5 (new R5)				; 13-15
	vmovapd	ymm13, [srcreg+d2+32]		;; I3

	ylow128s ymm9, ymm5, ymm7		;; Shuffle R5/R6 hi and R7/R8 hi (first R6)	;  14-15
	vaddpd	ymm10, ymm1, ymm6		;; R3 + R7 (new R3)				; 14-16
	vmovapd	ymm12, [srcreg+d2+d1+32]	;; I4

	yhigh128s ymm0, ymm0, ymm3		;; Shuffle R1/R2 hi and R3/R4 hi (first R4)	;  15-16
	vsubpd	ymm1, ymm1, ymm6		;; R3 - R7 (new R7)				; 15-17
	vmovapd	ymm11, [srcreg+d4+32]		;; I5

	yhigh128s ymm5, ymm5, ymm7		;; Shuffle R5/R6 hi and R7/R8 hi (first R8)	;  16-17
	vaddpd	ymm7, ymm2, ymm9		;; R2 + R6 (new R2)				; 16-18
	vmovapd	ymm3, [srcreg+d4+d1+32]		;; I6

	vsubpd	ymm2, ymm2, ymm9		;; R2 - R6 (new R6)				; 17-19
	L1prefetchw L1preg+e1reg, L1pt - L1PREFETCH_DEST_NONE

	vshufpd	ymm9, ymm15, ymm14, 15		;; Shuffle I1 and I2 to create I1/I2 hi		;  18
	vaddpd	ymm6, ymm0, ymm5		;; R4 + R8 (new R4)				; 18-20

	vshufpd	ymm15, ymm15, ymm14, 0		;; Shuffle I1 and I2 to create I1/I2 low	;  19
	vsubpd	ymm0, ymm0, ymm5		;; R4 - R8 (new R8)				; 19-21

	vshufpd	ymm5, ymm13, ymm12, 15		;; Shuffle I3 and I4 to create I3/I4 hi		;  20
	vaddpd	ymm14, ymm4, ymm10		;; R1 + R3 (newer R1)				; 20-22
	L1prefetchw L1preg+2*e1reg, L1pt - L1PREFETCH_DEST_NONE

	vshufpd	ymm13, ymm13, ymm12, 0		;; Shuffle I3 and I4 to create I3/I4 low	;  21
	vsubpd	ymm4, ymm4, ymm10		;; R1 - R3 (newer R3)				; 21-23

	vshufpd	ymm10, ymm11, ymm3, 15		;; Shuffle I5 and I6 to create I5/I6 hi		;  22
	vaddpd	ymm12, ymm7, ymm6		;; R2 + R4 (newer R2)				; 22-24

	ystore	[dstreg], ymm14			;; Temporarily save R1				; 23
	vshufpd	ymm11, ymm11, ymm3, 0		;; Shuffle I5 and I6 to create I5/I6 low	;  23
	vsubpd	ymm7, ymm7, ymm6		;; R2 - R4 (newer R4)				; 23-25

	vmovapd	ymm14, [srcreg+d4+d2+32]	;; I7
	vmovapd	ymm3, [srcreg+d4+d2+d1+32]	;; I8
	vshufpd	ymm6, ymm14, ymm3, 15		;; Shuffle I7 and I8 to create I7/I8 hi		;  24

	ystore	[dstreg+e1reg], ymm12		;; Temporarily save R2				; 25
	vshufpd	ymm14, ymm14, ymm3, 0		;; Shuffle I7 and I8 to create I7/I8 low	;  25

	ylow128s ymm3, ymm9, ymm5		;; Shuffle I1/I2 hi and I3/I4 hi (first I2)	;  26-27
	L1prefetchw L1preg+e3reg, L1pt - L1PREFETCH_DEST_NONE

	ylow128s ymm12, ymm10, ymm6		;; Shuffle I5/I6 hi and I7/I8 hi (first I6)	;  27-28

	yhigh128s ymm9, ymm9, ymm5		;; Shuffle I1/I2 hi and I3/I4 hi (first I4)	;  28-29

	yhigh128s ymm10, ymm10, ymm6		;; Shuffle I5/I6 hi and I7/I8 hi (first I8)	;  29-30
	vsubpd	ymm6, ymm3, ymm12		;; I2 - I6 (new I6)				; 29-31

	yhigh128s ymm5, ymm15, ymm13		;; Shuffle I1/I2 low and I3/I4 low (first I3)	;  30-31
	vaddpd	ymm3, ymm3, ymm12		;; I2 + I6 (new I2)				; 30-32

	yhigh128s ymm12, ymm11, ymm14		;; Shuffle I5/I6 low and I7/I8 low (first I7)	;  31-32
	L1prefetchw L1p4reg, L1pt - L1PREFETCH_DEST_NONE

	ylow128s ymm15, ymm15, ymm13		;; Shuffle I1/I2 low and I3/I4 low (first I1)	;  32-33
	vsubpd	ymm13, ymm9, ymm10		;; I4 - I8 (new I8)				; 31-33
	vaddpd	ymm9, ymm9, ymm10		;; I4 + I8 (new I4)				; 32-34
	vmovapd	ymm10, YMM_SQRTHALF

	ylow128s ymm11, ymm11, ymm14		;; Shuffle I5/I6 low and I7/I8 low (first I5)	;  33-34
	vaddpd	ymm14, ymm5, ymm12		;; I3 + I7 (new I3)				; 33-35

	vsubpd	ymm5, ymm5, ymm12		;; I3 - I7 (new I7)				; 34-36
	L1prefetchw L1p4reg+e1reg, L1pt - L1PREFETCH_DEST_NONE

	vaddpd	ymm12, ymm6, ymm0		;; I6 + R8 (new I6)				; 35-37

	vsubpd	ymm6, ymm6, ymm0		;; I6 - R8 (new I8)				; 36-38

	vaddpd	ymm0, ymm2, ymm13		;; R6 + I8 (new R8)				; 37-39

	vsubpd	ymm2, ymm2, ymm13		;; R6 - I8 (new R6)				; 38-40
	vmulpd	ymm12, ymm12, ymm10		;; I6 = I6 * SQRTHALF				;  38-42

	vaddpd	ymm13, ymm15, ymm11		;; I1 + I5 (new I1)				; 39-41
	vmulpd	ymm6, ymm6, ymm10		;; I8 = I8 * SQRTHALF				;  39-43

	vsubpd	ymm15, ymm15, ymm11		;; I1 - I5 (new I5)				; 40-42
	vmulpd	ymm0, ymm0, ymm10		;; R8 = R8 * SQRTHALF				;  40-44

	vsubpd	ymm11, ymm3, ymm9		;; I2 - I4 (newer I4)				; 41-43
	vmulpd	ymm2, ymm2, ymm10		;; R6 = R6 * SQRTHALF				;  41-45
	vmovapd	ymm10, [screg+128+32]		;; cosine for R3/I3

	vaddpd	ymm3, ymm3, ymm9		;; I2 + I4 (newer I2)				; 42-44
	L1prefetchw L1p4reg+2*e1reg, L1pt - L1PREFETCH_DEST_NONE

	vsubpd	ymm9, ymm13, ymm14		;; I1 - I3 (newer I3)				; 43-45

	vaddpd	ymm13, ymm13, ymm14		;; I1 + I3 (newer I1)				; 44-46
	vmovapd	ymm14, [screg+128]		;; sine for R3/I3

	ystore	[dstreg+e1reg+32], ymm3		;; Temporarily save I2				; 45
	vsubpd	ymm3, ymm0, ymm6		;; R8 - I8 (newer R8)				; 45-47

	vaddpd	ymm0, ymm0, ymm6		;; R8 + I8 (newer I8)				; 46-48

	ystore	[dstreg+32], ymm13		;; Temporarily save I1				; 47
	vsubpd	ymm13, ymm4, ymm11		;; R3 - I4 (last R3)				; 47-49		n 50

	vaddpd	ymm6, ymm9, ymm7		;; I3 + R4 (last I3)				; 48-50		n 51
	L1prefetchw L1p4reg+e3reg, L1pt - L1PREFETCH_DEST_NONE

	vaddpd	ymm4, ymm4, ymm11		;; R3 + I4 (last R4)				; 49-51		n 54

	vsubpd	ymm9, ymm9, ymm7		;; I3 - R4 (last I4)				; 50-52		n 55
	vmulpd	ymm7, ymm13, ymm10		;; A3 = R3 * cosine				;  50-54

	vaddpd	ymm11, ymm8, ymm5		;; R5 + I7 (newer R7)				; 51-53
	vmulpd	ymm10, ymm6, ymm10		;; B3 = I3 * cosine				;  51-55

	vsubpd	ymm8, ymm8, ymm5		;; R5 - I7 (newer R5)				; 52-54
	vmulpd	ymm6, ymm6, ymm14		;; C3 = I3 * sine				;  52-56

	vsubpd	ymm5, ymm15, ymm1		;; I5 - R7 (newer I7)				; 53-55
	vmulpd	ymm13, ymm13, ymm14		;; D3 = R3 * sine				;  53-57
	vmovapd	ymm14, [screg+384+32]		;; cosine for R4/I4

	vaddpd	ymm15, ymm15, ymm1		;; I5 + R7 (newer I5)				; 54-56
	vmulpd	ymm1, ymm4, ymm14		;; A4 = R4 * cosine				;  54-58

	vmulpd	ymm14, ymm9, ymm14		;; B4 = I4 * cosine				;  55-59
	vsubpd	ymm7, ymm7, ymm6		;; A3 = A3 - C3 (final R3)			; 57-59
	vsubpd	ymm6, ymm11, ymm0		;; R7 - I8 (last R7)				; 55-57		n 58

	ystore	[dstreg+2*e1reg], ymm7		;; Save R3					; 60
	vmovapd	ymm7, [screg+384]		;; sine for R4/I4
	vmulpd	ymm9, ymm9, ymm7		;; C4 = I4 * sine				;  56-60
	vmulpd	ymm4, ymm4, ymm7		;; D4 = R4 * sine				;  57-61
	vaddpd	ymm7, ymm5, ymm3		;; I7 + R8 (last I7)				; 56-58		n 59

	vaddpd	ymm10, ymm10, ymm13		;; B3 = B3 + D3 (final I3)			; 58-60
	vmovapd	ymm13, [screg+192+32]		;; cosine for R7/I7
	ystore	[dstreg+2*e1reg+32], ymm10	;; Save I3					; 61
	vmulpd	ymm10, ymm6, ymm13		;; A7 = R7 * cosine				;  58-62

	vaddpd	ymm11, ymm11, ymm0		;; R7 + I8 (last R8)				; 59-61		n 62
	vmovapd	ymm0, [screg+192]		;; sine for R7/I7
	vmulpd	ymm13, ymm7, ymm13		;; B7 = I7 * cosine				;  59-63

	vsubpd	ymm5, ymm5, ymm3		;; I7 - R8 (last I8)				; 60-62		n 63
	vmulpd	ymm7, ymm7, ymm0		;; C7 = I7 * sine				;  60-64

	vsubpd	ymm3, ymm2, ymm12		;; R6 - I6 (newer R6)				; 61-63		n 64
	vmulpd	ymm6, ymm6, ymm0		;; D7 = R7 * sine				;  61-65
	vmovapd	ymm0, [screg+448+32]		;; cosine for R8/I8

	vaddpd	ymm2, ymm2, ymm12		;; R6 + I6 (newer I6)				; 62-64		n 65
	vmulpd	ymm12, ymm11, ymm0		;; A8 = R8 * cosine				;  62-66

	vsubpd	ymm1, ymm1, ymm9		;; A4 = A4 - C4 (final R4)			; 63-65
	vmulpd	ymm0, ymm5, ymm0		;; B8 = I8 * cosine				;  63-67

	vsubpd	ymm9, ymm8, ymm3		;; R5 - R6 (last R6)				; 64-66		n 67
	ystore	[dstreg+e3reg], ymm1		;; Save R4					; 66
	vmovapd	ymm1, [screg+448]		;; sine for R8/I8
	vmulpd	ymm5, ymm5, ymm1		;; C8 = I8 * sine				;  64-68

	vmulpd	ymm11, ymm11, ymm1		;; D8 = R8 * sine				;  65-69
	vsubpd	ymm1, ymm15, ymm2		;; I5 - I6 (last I6)				; 65-67		n 68

	vaddpd	ymm14, ymm14, ymm4		;; B4 = B4 + D4 (final I4)			; 66-68
	vmovapd	ymm4, [screg+320+32]		;; cosine for R6/I6

	vaddpd	ymm8, ymm8, ymm3		;; R5 + R6 (last R5)				; 67-69		n 71
	vmulpd	ymm3, ymm9, ymm4		;; A6 = R6 * cosine				;  67-71

	vaddpd	ymm15, ymm15, ymm2		;; I5 + I6 (last I5)				; 68-70		n 72
	vmulpd	ymm4, ymm1, ymm4		;; B6 = I6 * cosine				;  68-72
	vmovapd	ymm2, [screg+320]		;; sine for R6/I6

	ystore	[dstreg+e3reg+32], ymm14	;; Save I4					; 69
	vsubpd	ymm10, ymm10, ymm7		;; A7 = A7 - C7 (final R7)			; 69-71
	vmulpd	ymm1, ymm1, ymm2		;; C6 = I6 * sine				;  69-73
	vmovapd	ymm14, [screg+64+32]		;; cosine for R5/I5

	vaddpd	ymm13, ymm13, ymm6		;; B7 = B7 + D7 (final I7)			; 70-72
	vmulpd	ymm9, ymm9, ymm2		;; D6 = R6 * sine				;  70-74
	vmovapd	ymm7, [dstreg]			;; Reload R1

	vsubpd	ymm12, ymm12, ymm5		;; A8 = A8 - C8 (final R8)			; 71-73
	vmulpd	ymm5, ymm8, ymm14		;; A5 = R5 * cosine				;  71-75
	vmovapd	ymm6, [dstreg+e1reg]		;; Reload R2

	ystore	[dst4reg+2*e1reg], ymm10	;; Save R7					; 72
	vsubpd	ymm10, ymm7, ymm6		;; R1 - R2 (last R2)				; 72-74		n 75
	vmulpd	ymm14, ymm15, ymm14		;; B5 = I5 * cosine				;  72-76
	vmovapd	ymm2, [screg+64]		;; sine for R5/I5

	ystore	[dst4reg+2*e1reg+32], ymm13	;; Save I7					; 73
	vaddpd	ymm7, ymm7, ymm6		;; R1 + R2 (last R1)				; 73-75		n 77
	vmulpd	ymm15, ymm15, ymm2		;; C5 = I5 * sine				;  73-77
	vmovapd	ymm13, [dstreg+32]		;; Reload I1

	ystore	[dst4reg+e3reg], ymm12		;; Save R8					; 74
	vaddpd	ymm0, ymm0, ymm11		;; B8 = B8 + D8 (final I8)			; 74-76
	vmulpd	ymm8, ymm8, ymm2		;; D5 = R5 * sine				;  74-78
	vmovapd	ymm6, [dstreg+e1reg+32]		;; Reload I2
	vmovapd	ymm12, [screg+256+32]		;; cosine for R2/I2

	vsubpd	ymm2, ymm13, ymm6		;; I1 - I2 (last I2)				; 75-77		n 78
	vmulpd	ymm11, ymm10, ymm12		;; A2 = R2 * cosine				;  75-79

	vaddpd	ymm13, ymm13, ymm6		;; I1 + I2 (last I1)				; 76-78		n 81
	vmovapd	ymm6, [screg+256]		;; sine for R2/I2
	vmulpd	ymm10, ymm10, ymm6		;; D2 = R2 * sine				;  76-80

	ystore	[dst4reg+e3reg+32], ymm0	;; Save I8					; 77
	vmovapd	ymm0, [screg+0+32]		;; cosine for R1/I1
	vsubpd	ymm3, ymm3, ymm1		;; A6 = A6 - C6 (final R6)			; 77-79
	vmulpd	ymm1, ymm7, ymm0		;; A1 = R1 * cosine				;  77-81

	vaddpd	ymm4, ymm4, ymm9		;; B6 = B6 + D6 (final I6)			; 78-80
	vmulpd	ymm6, ymm2, ymm6		;; C2 = I2 * sine				;  78-82
	vmovapd	ymm9, [screg+0]			;; sine for R1/I1

	vsubpd	ymm5, ymm5, ymm15		;; A5 = A5 - C5 (final R5)			; 79-81
	vmulpd	ymm2, ymm2, ymm12		;; B2 = I2 * cosine				;  79-83

	ystore	[dst4reg+e1reg], ymm3		;; Save R6					; 80
	vaddpd	ymm14, ymm14, ymm8		;; B5 = B5 + D5 (final I5)			; 80-82
	vmulpd	ymm0, ymm13, ymm0		;; B1 = I1 * cosine				;  80-84

	ystore	[dst4reg+e1reg+32], ymm4	;; Save I6					; 81
	vmulpd	ymm13, ymm13, ymm9		;; C1 = I1 * sine				;  81-85

	ystore	[dst4reg], ymm5			;; Save R5					; 82
	vmulpd	ymm7, ymm7, ymm9		;; D1 = R1 * sine				;  82-86

	ystore	[dst4reg+32], ymm14		;; Save I5					; 83
	vsubpd	ymm11, ymm11, ymm6		;; A2 = A2 - C2 (final R2)			; 83-85

	vaddpd	ymm2, ymm2, ymm10		;; B2 = B2 + D2 (final I2)			; 84-86

	ystore	[dstreg+e1reg], ymm11		;; Save R2					; 86
	vsubpd	ymm1, ymm1, ymm13		;; A1 = A1 - C1 (final R1)			; 86-88

	ystore	[dstreg+e1reg+32], ymm2		;; Save I2					; 87
	vaddpd	ymm0, ymm0, ymm7		;; B1 = B1 + D1 (final I1)			; 87-89

	ystore	[dstreg], ymm1			;; Save R1					; 89
	ystore	[dstreg+32], ymm0		;; Save I1					; 90

	bump	srcreg, srcinc
	bump	dstreg, dstinc
	bump	dst4reg, dstinc
	bump	screg, scinc
	bump	L1preg, dstinc
	bump	L1p4reg, dstinc
	ENDM

IF (@INSTR(,%yarch,<FMA3>) NE 0)

yr8_rsc_sg8clreg_eight_complex_fft8_preload MACRO
	ENDM
yr8_rsc_sg8clreg_eight_complex_fft8 MACRO srcreg,srcinc,d1,d2,d4,dstreg,dstinc,e1reg,e3reg,dst4reg,screg,scinc,maxrpt,L1pt,L1preg,L1p4reg
	vmovapd	ymm1, [srcreg]			;; R1
	vmovapd	ymm7, [srcreg+d1]		;; R2
	vshufpd	ymm0, ymm1, ymm7, 15		;; Shuffle R1 and R2 to create R1/R2 hi		;  1
	vshufpd	ymm1, ymm1, ymm7, 0		;; Shuffle R1 and R2 to create R1/R2 low	;  2

	vmovapd	ymm2, [srcreg+d2]		;; R3
	vmovapd	ymm7, [srcreg+d2+d1]		;; R4
	vshufpd	ymm3, ymm2, ymm7, 15		;; Shuffle R3 and R4 to create R3/R4 hi		;  3
	vshufpd	ymm2, ymm2, ymm7, 0		;; Shuffle R3 and R4 to create R3/R4 low	;  4

	vmovapd	ymm6, [srcreg+d4]		;; R5
	vmovapd	ymm7, [srcreg+d4+d1]		;; R6
	vshufpd	ymm5, ymm6, ymm7, 15		;; Shuffle R5 and R6 to create R5/R6 hi		;  5
	vshufpd	ymm6, ymm6, ymm7, 0		;; Shuffle R5 and R6 to create R5/R6 low	;  6

	vmovapd	ymm4, [srcreg+d4+d2]		;; R7
	vmovapd	ymm8, [srcreg+d4+d2+d1]		;; R8
	vshufpd	ymm7, ymm4, ymm8, 15		;; Shuffle R7 and R8 to create R7/R8 hi		;  7
	vshufpd	ymm4, ymm4, ymm8, 0		;; Shuffle R7 and R8 to create R7/R8 low	;  8

	ylow128s ymm8, ymm1, ymm2		;; Shuffle R1/R2 low and R3/R4 low (first R1)	;  9-10

	ylow128s ymm9, ymm6, ymm4		;; Shuffle R5/R6 low and R7/R8 low (first R5)	;  10-11

	yhigh128s ymm1, ymm1, ymm2		;; Shuffle R1/R2 low and R3/R4 low (first R3)	;  11-12
	vmovapd	ymm15, [srcreg+32]		;; I1

	yhigh128s ymm6, ymm6, ymm4		;; Shuffle R5/R6 low and R7/R8 low (first R7)	;  12-13
	vaddpd	ymm4, ymm8, ymm9		;; R1 + R5 (new R1)				; 12-14
	vmovapd	ymm14, [srcreg+d1+32]		;; I2

	ylow128s ymm2, ymm0, ymm3		;; Shuffle R1/R2 hi and R3/R4 hi (first R2)	;  13-14
	vsubpd	ymm8, ymm8, ymm9		;; R1 - R5 (new R5)				; 13-15
	vmovapd	ymm13, [srcreg+d2+32]		;; I3

	ylow128s ymm9, ymm5, ymm7		;; Shuffle R5/R6 hi and R7/R8 hi (first R6)	;  14-15
	vaddpd	ymm10, ymm1, ymm6		;; R3 + R7 (new R3)				; 14-16
	vmovapd	ymm12, [srcreg+d2+d1+32]	;; I4

	yhigh128s ymm0, ymm0, ymm3		;; Shuffle R1/R2 hi and R3/R4 hi (first R4)	;  15-16
	vsubpd	ymm1, ymm1, ymm6		;; R3 - R7 (new R7)				; 15-17
	vmovapd	ymm11, [srcreg+d4+32]		;; I5

	yhigh128s ymm5, ymm5, ymm7		;; Shuffle R5/R6 hi and R7/R8 hi (first R8)	;  16-17
	vaddpd	ymm7, ymm2, ymm9		;; R2 + R6 (new R2)				; 16-18
	vmovapd	ymm3, [srcreg+d4+d1+32]		;; I6

	vsubpd	ymm2, ymm2, ymm9		;; R2 - R6 (new R6)				; 17-19
	L1prefetchw L1preg, L1pt - L1PREFETCH_DEST_NONE

	vshufpd	ymm9, ymm15, ymm14, 15		;; Shuffle I1 and I2 to create I1/I2 hi		;  18
	vaddpd	ymm6, ymm0, ymm5		;; R4 + R8 (new R4)				; 18-20

	vshufpd	ymm15, ymm15, ymm14, 0		;; Shuffle I1 and I2 to create I1/I2 low	;  19
	vsubpd	ymm0, ymm0, ymm5		;; R4 - R8 (new R8)				; 19-21

	vshufpd	ymm5, ymm13, ymm12, 15		;; Shuffle I3 and I4 to create I3/I4 hi		;  20
	vaddpd	ymm14, ymm4, ymm10		;; R1 + R3 (newer R1)				; 20-22

	vshufpd	ymm13, ymm13, ymm12, 0		;; Shuffle I3 and I4 to create I3/I4 low	;  21
	vaddpd	ymm12, ymm7, ymm6		;; R2 + R4 (newer R2)				; 21-23

	vsubpd	ymm4, ymm4, ymm10		;; R1 - R3 (newer R3)				; 22-24
	vshufpd	ymm10, ymm11, ymm3, 15		;; Shuffle I5 and I6 to create I5/I6 hi		;  22

	vshufpd	ymm11, ymm11, ymm3, 0		;; Shuffle I5 and I6 to create I5/I6 low	;  23
	vsubpd	ymm7, ymm7, ymm6		;; R2 - R4 (newer R4)				; 23-25
	vmovapd	ymm3, [srcreg+d4+d2+32]		;; I7

	vaddpd	ymm6, ymm14, ymm12		;; R1 + R2 (last R1)				; 24-26

	vsubpd	ymm14, ymm14, ymm12		;; R1 - R2 (last R2)				; 25-27

	vmulpd	ymm12, ymm6, [screg+0+32]	;; R1*cos					; 27-31
	vmulpd	ymm6, ymm6, [screg+0]		;; R1*sin					; 27-31
	ystore	[dstreg+32], ymm12		;; Temporarily save R1*cos			; 32+1
	vmovapd	ymm12, [srcreg+d4+d2+d1+32]	;; I8
	ystore	[dstreg], ymm6			;; Temporarily save R1*sin			; 32

	vshufpd	ymm6, ymm3, ymm12, 15		;; Shuffle I7 and I8 to create I7/I8 hi		;  24
	L1prefetchw L1preg+e1reg, L1pt - L1PREFETCH_DEST_NONE

	vshufpd	ymm3, ymm3, ymm12, 0		;; Shuffle I7 and I8 to create I7/I8 low	;  25

	ylow128s ymm12, ymm9, ymm5		;; Shuffle I1/I2 hi and I3/I4 hi (first I2)	;  26-27	n 30

	yhigh128s ymm9, ymm9, ymm5		;; Shuffle I1/I2 hi and I3/I4 hi (first I4)	;  27-28	n 30

	ylow128s ymm5, ymm10, ymm6		;; Shuffle I5/I6 hi and I7/I8 hi (first I6)	;  28-29	n 31

	yhigh128s ymm10, ymm10, ymm6		;; Shuffle I5/I6 hi and I7/I8 hi (first I8)	;  29-30	n 31
	vmulpd	ymm6, ymm14, [screg+256+32]	;; R2*cos					; 28-32
	vmulpd	ymm14, ymm14, [screg+256]	;; R2*sin					; 28-32

	ystore	[dstreg+e1reg+32], ymm6		;; Temporarily save R2*cos			; 33+1
	vmovapd	ymm6, YMM_ONE
	ystore	[dstreg+e1reg], ymm14		;; Temporarily save R2*sin			; 33+2
	yfmsubpd ymm14, ymm12, ymm6, ymm5	;; I2 - I6 (new I6)				; 30-34		n 36
	yfmaddpd ymm12, ymm12, ymm6, ymm5	;; I2 + I6 (new I2)				; 30-34		n 38
	yhigh128s ymm5, ymm15, ymm13		;; Shuffle I1/I2 low and I3/I4 low (first I3)	;  30-31	n 34

	ylow128s ymm15, ymm15, ymm13		;; Shuffle I1/I2 low and I3/I4 low (first I1)	;  31-32	n 35
	yfmsubpd ymm13, ymm9, ymm6, ymm10	;; I4 - I8 (new I8)				; 31-35		n 37
	yfmaddpd ymm9, ymm9, ymm6, ymm10	;; I4 + I8 (new I4)				; 31-35		n 38

	yhigh128s ymm10, ymm11, ymm3		;; Shuffle I5/I6 low and I7/I8 low (first I7)	;  32-33	n 34
	L1prefetchw L1preg+2*e1reg, L1pt - L1PREFETCH_DEST_NONE

	ylow128s ymm11, ymm11, ymm3		;; Shuffle I5/I6 low and I7/I8 low (first I5)	;  33-34	n 35

	yfmsubpd ymm3, ymm5, ymm6, ymm10	;; I3 - I7 (new I7)				; 34-38		n 39
	yfmaddpd ymm5, ymm5, ymm6, ymm10	;; I3 + I7 (new I3)				; 34-38		n 40

	yfmaddpd ymm10, ymm15, ymm6, ymm11	;; I1 + I5 (new I1)				; 35-39		n 40
	yfmsubpd ymm15, ymm15, ymm6, ymm11	;; I1 - I5 (new I5)				; 35-39		n 41
	L1prefetchw L1preg+e3reg, L1pt - L1PREFETCH_DEST_NONE

	yfmsubpd ymm11, ymm14, ymm6, ymm0	;; I6 - R8 (new2 I8)				; 36-40		n 42
	yfmaddpd ymm14, ymm14, ymm6, ymm0	;; I6 + R8 (new2 I6)				; 36-40		n 43

	yfmaddpd ymm0, ymm2, ymm6, ymm13	;; R6 + I8 (new2 R8)				; 37-41		n 42
	yfmsubpd ymm2, ymm2, ymm6, ymm13	;; R6 - I8 (new2 R6)				; 37-41		n 43

	yfmsubpd ymm13, ymm12, ymm6, ymm9	;; I2 - I4 (newer I4)				; 38-42		n 44
	yfmaddpd ymm12, ymm12, ymm6, ymm9	;; I2 + I4 (newer I2)				; 38-42		n 46
	L1prefetchw L1p4reg, L1pt - L1PREFETCH_DEST_NONE

	yfmaddpd ymm9, ymm8, ymm6, ymm3		;; R5 + I7 (newer R7)				; 39-43		n 47
	yfmsubpd ymm8, ymm8, ymm6, ymm3		;; R5 - I7 (newer R5)				; 39-43		n 49

	yfmsubpd ymm3, ymm10, ymm6, ymm5	;; I1 - I3 (newer I3)				; 40-44		n 45
	yfmaddpd ymm10, ymm10, ymm6, ymm5	;; I1 + I3 (newer I1)				; 40-44		n 46

	yfmsubpd ymm5, ymm15, ymm6, ymm1	;; I5 - R7 (newer I7)				; 41-45		n 48
 	yfmaddpd ymm15, ymm15, ymm6, ymm1	;; I5 + R7 (newer I5)				; 41-45		n 50
	L1prefetchw L1p4reg+e1reg, L1pt - L1PREFETCH_DEST_NONE

	yfmaddpd ymm1, ymm0, ymm6, ymm11	;; R8 + I8 (newer I8/SQRTHALF)			; 42-46		n 47
	yfmsubpd ymm0, ymm0, ymm6, ymm11	;; R8 - I8 (newer R8/SQRTHALF)			; 42-46		n 48

	yfmsubpd ymm11, ymm2, ymm6, ymm14	;; R6 - I6 (newer R6/SQRTHALF)			; 43-47		n 49
	yfmaddpd ymm2, ymm2, ymm6, ymm14	;; R6 + I6 (newer I6/SQRTHALF)			; 43-47		n 50

	yfmsubpd ymm14, ymm4, ymm6, ymm13	;; R3 - I4 (last R3)				; 44-48		n 51
	yfmaddpd ymm4, ymm4, ymm6, ymm13	;; R3 + I4 (last R4)				; 44-48		n 52

	yfmaddpd ymm13, ymm3, ymm6, ymm7	;; I3 + R4 (last I3)				; 45-49		n 51
	yfmsubpd ymm3, ymm3, ymm6, ymm7		;; I3 - R4 (last I4)				; 45-49		n 52
	L1prefetchw L1p4reg+2*e1reg, L1pt - L1PREFETCH_DEST_NONE

	yfmsubpd ymm7, ymm10, ymm6, ymm12	;; I1 - I2 (last I2)				; 46-50		n 59
	yfmaddpd ymm10, ymm10, ymm6, ymm12	;; I1 + I2 (last I1)				; 46-50		n 60

	vmovapd	ymm6, YMM_SQRTHALF
	yfnmaddpd ymm12, ymm1, ymm6, ymm9	;; R7 - I8 * SQRTHALF (last R7)			; 47-51		n 53
	yfmaddpd ymm1, ymm1, ymm6, ymm9		;; R7 + I8 * SQRTHALF (last R8)			; 47-51		n 54

	yfmaddpd ymm9, ymm0, ymm6, ymm5		;; I7 + R8 * SQRTHALF (last I7)			; 48-52		n 53
	yfnmaddpd ymm0, ymm0, ymm6, ymm5	;; I7 - R8 * SQRTHALF (last I8)			; 48-52		n 54

	yfnmaddpd ymm5, ymm11, ymm6, ymm8	;; R5 - R6 * SQRTHALF (last R6)			; 49-53		n 55
	yfmaddpd ymm11, ymm11, ymm6, ymm8	;; R5 + R6 * SQRTHALF (last R5)			; 49-53		n 56
	L1prefetchw L1p4reg+e3reg, L1pt - L1PREFETCH_DEST_NONE

	yfnmaddpd ymm8, ymm2, ymm6, ymm15	;; I5 - I6 * SQRTHALF (last I6)			; 50-54		n 55
	yfmaddpd ymm2, ymm2, ymm6, ymm15	;; I5 + I6 * SQRTHALF (last I5)			; 50-54		n 56
	bump	srcreg, srcinc

	vmovapd	ymm15, [screg+128+32]		;; cosine for R3/I3
	vmulpd	ymm6, ymm14, ymm15		;; A3 = R3 * cosine				; 51-55
	vmulpd	ymm15, ymm13, ymm15		;; B3 = I3 * cosine				; 51-55

	yfnmaddpd ymm13, ymm13, [screg+128], ymm6 ;; A3 = A3 - I3 * sine (final R3)		; 56-60
	yfmaddpd ymm14, ymm14, [screg+128], ymm15 ;; B3 = B3 + R3 * sine (final I3)		; 56-60

	vmovapd	ymm15, [screg+384+32]		;; cosine for R4/I4
	vmulpd	ymm6, ymm4, ymm15		;; A4 = R4 * cosine				; 52-56
	vmulpd	ymm15, ymm3, ymm15		;; B4 = I4 * cosine				; 52-56

	ystore	[dstreg+2*e1reg], ymm13		;; Save R3					; 61
	vmovapd	ymm13, [screg+384]		;; sine for R4/I4
	yfnmaddpd ymm3, ymm3, ymm13, ymm6	;; A4 = A4 - I4 * sine (final R4)		; 57-61
	yfmaddpd ymm4, ymm4, ymm13, ymm15	;; B4 = B4 + R4 * sine (final I4)		; 57-61

	vmovapd	ymm15, [screg+192+32]		;; cosine for R7/I7
	vmulpd	ymm6, ymm12, ymm15		;; A7 = R7 * cosine				; 53-57
	vmulpd	ymm15, ymm9, ymm15		;; B7 = I7 * cosine				; 53-57

	vmovapd	ymm13, [screg+448+32]		;; cosine for R8/I8
	ystore	[dstreg+2*e1reg+32], ymm14	;; Save I3					; 61+1
	vmulpd	ymm14, ymm1, ymm13		;; A8 = R8 * cosine				; 54-58
	vmulpd	ymm13, ymm0, ymm13		;; B8 = I8 * cosine				; 54-58

	ystore	[dstreg+e3reg], ymm3		;; Save R4					; 62+1
	vmovapd	ymm3, [screg+192]		;; sine for R7/I7
	yfnmaddpd ymm9, ymm9, ymm3, ymm6	;; A7 = A7 - I7 * sine (final R7)		; 58-62
	yfmaddpd ymm12, ymm12, ymm3, ymm15	;; B7 = B7 + R7 * sine (final I7)		; 58-62

	vmovapd	ymm15, [screg+320+32]		;; cosine for R6/I6
	vmulpd	ymm6, ymm5, ymm15		;; A6 = R6 * cosine				; 55-59
	vmulpd	ymm15, ymm8, ymm15		;; B6 = I6 * cosine				; 55-59

	vmovapd	ymm3, [screg+64+32]		;; cosine for R5/I5
	ystore	[dstreg+e3reg+32], ymm4		;; Save I4					; 62+2
	vmulpd	ymm4, ymm11, ymm3		;; A5 = R5 * cosine				; 59-63
	vmulpd	ymm3, ymm2, ymm3		;; B5 = I5 * cosine				; 59-63

	ystore	[dst4reg+2*e1reg], ymm9		;; Save R7					; 63+2
	vmovapd	ymm9, [screg+448]		;; sine for R8/I8
	yfnmaddpd ymm0, ymm0, ymm9, ymm14	;; A8 = A8 - I8 * sine (final R8)		; 60-64
	yfmaddpd ymm1, ymm1, ymm9, ymm13	;; B8 = B8 + R8 * sine (final I8)		; 60-64

	vmovapd	ymm14, [dstreg+32]		;; Reload last A1 = R1 * cosine
	vmovapd	ymm9, [dstreg]			;; Reload last B1 = R1 * sine
	yfnmaddpd ymm14, ymm10, [screg+0], ymm14 ;; A1 = A1 - I1 * sine (final R1)		; 61-65
	yfmaddpd ymm10, ymm10, [screg+0+32], ymm9 ;; B1 = I1 * cosine + B1 (final I1)		; 61-65

	vmovapd	ymm13, [dstreg+e1reg+32]	;; Reload last A2 = R2 * cosine
	vmovapd	ymm9, [dstreg+e1reg]		;; Reload last B2 = R2 * sine
	yfnmaddpd ymm13, ymm7, [screg+256], ymm13 ;; A2 = A2 - I2 * sine (final R2)		; 62-66
	yfmaddpd ymm7, ymm7, [screg+256+32], ymm9 ;; B2 = I2 * cosine + B2 (final I2)		; 62-66

	vmovapd	ymm9, [screg+320]		;; sine for R6/I6
	yfnmaddpd ymm8, ymm8, ymm9, ymm6	;; A6 = A6 - I6 * sine (final R6)		; 63-67
	yfmaddpd ymm5, ymm5, ymm9, ymm15	;; B6 = B6 + R6 * sine (final I6)		; 63-67

	vmovapd	ymm15, [screg+64]		;; sine for R5/I5
	yfnmaddpd ymm2, ymm2, ymm15, ymm4	;; A5 = A5 - I5 * sine (final R5)		; 64-68
	yfmaddpd ymm11, ymm11, ymm15, ymm3	;; B5 = B5 + R5 * sine (final I5)		; 64-68

	ystore	[dst4reg+2*e1reg+32], ymm12	;; Save I7					; 63+3
	ystore	[dst4reg+e3reg], ymm0		;; Save R8					; 65+2
	ystore	[dst4reg+e3reg+32], ymm1	;; Save I8					; 65+3
	ystore	[dstreg], ymm14			;; Save R1					; 66+3
	ystore	[dstreg+32], ymm10		;; Save I1					; 66+4
	ystore	[dstreg+e1reg], ymm13		;; Save R2					; 67+4
	ystore	[dstreg+e1reg+32], ymm7		;; Save I2					; 67+5
	ystore	[dst4reg+e1reg], ymm8		;; Save R6					; 68+5
	ystore	[dst4reg+e1reg+32], ymm5	;; Save I6					; 68+6
	ystore	[dst4reg], ymm2			;; Save R5					; 69+6
	ystore	[dst4reg+32], ymm11		;; Save I5					; 69+7

	bump	screg, scinc
	bump	dstreg, dstinc
	bump	dst4reg, dstinc
	bump	L1preg, dstinc
	bump	L1p4reg, dstinc
	ENDM

ENDIF

ENDIF



yr8_rsc_sg8cl_eight_complex_unfft8_preload MACRO
	ENDM
yr8_rsc_sg8cl_eight_complex_unfft8 MACRO srcreg,srcinc,d1,d2,d4,dstreg,dstinc,e1,e2,e4,screg,scinc,maxrpt,L1pt,L1pd
	vmovapd	ymm0, [screg+64+32]		;; cosine for R5/I5
	vmovapd	ymm4, [srcreg+d4]		;; R5
	vmulpd	ymm5, ymm4, ymm0		;; A5 = R5 * cosine
	vmovapd	ymm2, [srcreg+d4+32]		;; I5
	vmulpd	ymm0, ymm2, ymm0		;; B5 = I5 * cosine
	vmulpd	ymm2, ymm2, [screg+64]		;; C5 = I5 * sine
	vmulpd	ymm4, ymm4, [screg+64]		;; D5 = R5 * sine
	vaddpd	ymm5, ymm5, ymm2		;; A5 = A5 + C5 (first R5)
	vsubpd	ymm0, ymm0, ymm4		;; B5 = B5 - D5 (first I5)

	vmovapd	ymm4, [screg+320+32]		;; cosine for R6/I6
	vmovapd	ymm6, [srcreg+d4+d1]		;; R6
	vmulpd	ymm7, ymm6, ymm4		;; A6 = R6 * cosine
	vmovapd ymm2, [srcreg+d4+d1+32]		;; I6
	vmulpd	ymm4, ymm2, ymm4		;; B6 = I6 * cosine
	vmulpd	ymm2, ymm2, [screg+320]		;; C6 = I6 * sine
	vmulpd	ymm6, ymm6, [screg+320]		;; D6 = R6 * sine
	vaddpd	ymm7, ymm7, ymm2		;; A6 = A6 + C6 (first R6)
	vsubpd	ymm4, ymm4, ymm6		;; B6 = B6 - D6 (first I6)

	vsubpd	ymm6, ymm5, ymm7		;; R5 - R6 (new R6)
	vaddpd	ymm5, ymm5, ymm7		;; R5 + R6 (new R5)

	L1prefetch srcreg+L1pd, L1pt

	vsubpd	ymm7, ymm0, ymm4		;; I5 - I6 (new I6)
	vaddpd	ymm0, ymm0, ymm4		;; I5 + I6 (new I5)

	vmovapd	ymm1, [screg+192+32]		;; cosine for R7/I7
	vmovapd	ymm4, [srcreg+d4+d2]		;; R7
	vmulpd	ymm3, ymm4, ymm1		;; A7 = R7 * cosine
	vmovapd	ymm2, [srcreg+d4+d2+32]		;; I7
	vmulpd	ymm1, ymm2, ymm1		;; B7 = I7 * cosine
	vmulpd	ymm2, ymm2, [screg+192]		;; C7 = I7 * sine
	vmulpd	ymm4, ymm4, [screg+192]		;; D7 = R7 * sine
	vaddpd	ymm3, ymm3, ymm2		;; A7 = A7 + C7 (first R7)
	vsubpd	ymm1, ymm1, ymm4		;; B7 = B7 - D7 (first I7)

	ystore	[dstreg+e4+e1], ymm6		;; Save new R6
	ystore	[dstreg+e4+e1+32], ymm7		;; Save new I6

	vmovapd	ymm4, [screg+448+32]		;; cosine for R8/I8
	vmovapd	ymm6, [srcreg+d4+d2+d1]		;; R8
	vmulpd	ymm7, ymm6, ymm4		;; A8 = R8 * cosine
	vmovapd ymm2, [srcreg+d4+d2+d1+32]	;; I8
	vmulpd	ymm4, ymm2, ymm4		;; B8 = I8 * cosine
	vmulpd	ymm2, ymm2, [screg+448]		;; C8 = I8 * sine
	vmulpd	ymm6, ymm6, [screg+448]		;; D8 = R8 * sine
	vaddpd	ymm7, ymm7, ymm2		;; A8 = A8 + C8 (first R8)
	vsubpd	ymm4, ymm4, ymm6		;; B8 = B8 - D8 (first I8)

	vsubpd	ymm6, ymm7, ymm3		;; R8 - R7 (new I8)
	vaddpd	ymm7, ymm7, ymm3		;; R8 + R7 (new R7)

	L1prefetch srcreg+d1+L1pd, L1pt

	vsubpd	ymm3, ymm1, ymm4		;; I7 - I8 (new R8)
	vaddpd	ymm1, ymm1, ymm4		;; I7 + I8 (new I7)

	ystore	[dstreg+e4], ymm5		;; Save new R5
	ystore	[dstreg+e4+e2+e1], ymm3		;; Save new R8

	vmovapd	ymm3, [screg+0+32]		;; cosine for R1/I1
	vmovapd	ymm4, [srcreg]			;; R1
	vmulpd	ymm5, ymm4, ymm3		;; A1 = R1 * cosine
	vmovapd	ymm2, [srcreg+32]		;; I1
	vmulpd	ymm3, ymm2, ymm3		;; B1 = I1 * cosine
	vmulpd	ymm2, ymm2, [screg+0]		;; C1 = I1 * sine
	vmulpd	ymm4, ymm4, [screg+0]		;; D1 = R1 * sine
	vaddpd	ymm5, ymm5, ymm2		;; A1 = A1 + C1 (first R1)
	vsubpd	ymm3, ymm3, ymm4		;; B1 = B1 - D1 (first I1)

	ystore	[dstreg+e4+32], ymm0		;; Save new I5
	ystore	[dstreg+e4+e2+e1+32], ymm6	;; Save new I8
	ystore	[dstreg+e4+e2], ymm7		;; Save new R7

	vmovapd	ymm2, [screg+256+32]		;; cosine for R2/I2
	vmovapd	ymm6, [srcreg+d1]		;; R2
	vmulpd	ymm7, ymm6, ymm2		;; A2 = R2 * cosine
	vmovapd	ymm0, [srcreg+d1+32]		;; I2
	vmulpd	ymm2, ymm0, ymm2		;; B2 = I2 * cosine
	vmulpd	ymm0, ymm0, [screg+256]		;; C2 = I2 * sine
	vmulpd	ymm6, ymm6, [screg+256]		;; D2 = R2 * sine
	vaddpd	ymm7, ymm7, ymm0		;; A2 = A2 + C2 (first R2)
	vsubpd	ymm2, ymm2, ymm6		;; B2 = B2 - D2 (first I2)

	vsubpd	ymm0, ymm5, ymm7		;; R1 - R2 (new R2)
	vaddpd	ymm5, ymm5, ymm7		;; R1 + R2 (new R1)

	L1prefetch srcreg+d2+L1pd, L1pt

	vsubpd	ymm4, ymm3, ymm2		;; I1 - I2 (new I2)
	vaddpd	ymm3, ymm3, ymm2		;; I1 + I2 (new I1)

	ystore	[dstreg+e4+e2+32], ymm1		;; Save new I7

	vmovapd	ymm1, [screg+128+32]		;; cosine for R3/I3
	vmovapd	ymm6, [srcreg+d2]		;; R3
	vmulpd	ymm7, ymm6, ymm1		;; A3 = R3 * cosine
	vmovapd	ymm2, [srcreg+d2+32]		;; I3
	vmulpd	ymm1, ymm2, ymm1		;; B3 = I3 * cosine
	vmulpd	ymm2, ymm2, [screg+128]		;; C3 = I3 * sine
	vmulpd	ymm6, ymm6, [screg+128]		;; D3 = R3 * sine
	vaddpd	ymm7, ymm7, ymm2		;; A3 = A3 + C3 (first R3)
	vsubpd	ymm1, ymm1, ymm6		;; B3 = B3 - D3 (first I3)

	ystore	[dstreg+e1], ymm0		;; Save new R2
	ystore	[dstreg+e1+32], ymm4		;; Save new I2

	vmovapd	ymm6, [screg+384+32]		;; cosine/sine for R4/I4
	vmovapd	ymm0, [srcreg+d2+d1]		;; R4
	vmulpd	ymm4, ymm0, ymm6		;; A4 = R4 * cosine/sine
	vmovapd ymm2, [srcreg+d2+d1+32]		;; I4
	vmulpd	ymm6, ymm2, ymm6		;; B4 = I4 * cosine/sine
	vmulpd	ymm2, ymm2, [screg+384]		;; C4 = I4 * sine
	vmulpd	ymm0, ymm0, [screg+384]		;; D4 = R4 * sine
	vaddpd	ymm4, ymm4, ymm2		;; A4 = A4 + C4 (first R4)
	vsubpd	ymm6, ymm6, ymm0		;; B4 = B4 - D4 (first I4)

	vsubpd	ymm2, ymm4, ymm7		;; R4 - R3 (new I4)
	vaddpd	ymm4, ymm4, ymm7		;; R4 + R3 (new R3)

	L1prefetch srcreg+d2+d1+L1pd, L1pt

	vsubpd	ymm7, ymm1, ymm6		;; I3 - I4 (new R4)
	vaddpd	ymm1, ymm1, ymm6		;; I3 + I4 (new I3)

	vmovapd	ymm0, [dstreg+e4+32]		;; Reload new I5
	vmovapd	ymm6, [dstreg+e4+e2+32]		;; Reload new I7
	ystore	[dstreg+e2+e1+32], ymm2		;; Save new I4
	vsubpd	ymm2, ymm0, ymm6		;; I5 - I7 (newer R7)
	vaddpd	ymm0, ymm0, ymm6		;; I5 + I7 (newer I5)

	vsubpd	ymm6, ymm5, ymm4		;; R1 - R3 (newer R3)
	vaddpd	ymm4, ymm5, ymm4		;; R1 + R3 (newer R1)

	L1prefetch srcreg+d4+L1pd, L1pt

	vsubpd	ymm5, ymm6, ymm2		;; R3 - R7 (final R7)
	vaddpd	ymm6, ymm6, ymm2		;; R3 + R7 (final R3)

	vsubpd	ymm2, ymm3, ymm1		;; I1 - I3 (newer I3)
	vaddpd	ymm3, ymm3, ymm1		;; I1 + I3 (newer I1)

	vmovapd	ymm1, [dstreg+e4+e2]		;; Reload new R7
	ystore	[dstreg+e4+e2], ymm5		;; Save final R7
	vmovapd	ymm5, [dstreg+e4]		;; Reload new R5
	ystore	[dstreg+e2], ymm6		;; Save final R3
	vaddpd	ymm6, ymm1, ymm5		;; R7 + R5 (newer R5)
	vsubpd	ymm1, ymm1, ymm5		;; R7 - R5 (newer I7)

	vsubpd	ymm5, ymm3, ymm0		;; I1 - I5 (final I5)
	vaddpd	ymm3, ymm3, ymm0		;; I1 + I5 (final I1)

	L1prefetch srcreg+d4+d1+L1pd, L1pt

	vsubpd	ymm0, ymm4, ymm6		;; R1 - R5 (final R5)
	vaddpd	ymm4, ymm4, ymm6		;; R1 + R5 (final R1)

	vsubpd	ymm6, ymm2, ymm1		;; I3 - I7 (final I7)
	vaddpd	ymm2, ymm2, ymm1		;; I3 + I7 (final I3)

	;; multiply R6/I6 by SQRTHALF - i*SQRTHALF
	vmovapd	ymm1, [dstreg+e4+e1+32]		;; Reload new I6
	ystore	[dstreg+e4+32], ymm5		;; Save final I5
	vmovapd	ymm5, [dstreg+e4+e1]		;; Reload new R6
	ystore	[dstreg+32], ymm3		;; Save final I1
	vsubpd	ymm3, ymm1, ymm5		;; I6 = I6 - R6
	vaddpd	ymm1, ymm1, ymm5		;; R6 = R6 + I6

	;; multiply R8/I8 by SQRTHALF - i*SQRTHALF
	vmovapd	ymm5, [dstreg+e4+e2+e1+32]	;; Reload new I8
	ystore	[dstreg+e4], ymm0		;; Save final R5
	vmovapd	ymm0, [dstreg+e4+e2+e1]		;; Reload new R8
	ystore	[dstreg], ymm4			;; Save final R1
	vsubpd	ymm4, ymm5, ymm0		;; I8 = I8 - R8
	vaddpd	ymm5, ymm5, ymm0		;; R8 = R8 + I8

	vmovapd	ymm0, YMM_SQRTHALF
	vmulpd	ymm3, ymm3, ymm0		;; I6 * SQRTHALF
	vmulpd	ymm1, ymm1, ymm0		;; R6 * SQRTHALF
	vmulpd	ymm4, ymm4, ymm0		;; I8 * SQRTHALF
	vmulpd	ymm5, ymm5, ymm0		;; R8 * SQRTHALF

	vmovapd	ymm0, [dstreg+e1]		;; Reload new R2
	ystore	[dstreg+e4+e2+32], ymm6		;; Save final I7
	vaddpd	ymm6, ymm0, ymm7		;; R2 + R4 (newer R2)
	vsubpd	ymm0, ymm0, ymm7		;; R2 - R4 (newer R4)

	vsubpd	ymm7, ymm5, ymm1		;; R8 - R6 (newer I8)
	vaddpd	ymm5, ymm5, ymm1		;; R8 + R6 (newer R6)

	L1prefetch srcreg+d4+d2+L1pd, L1pt

	vsubpd	ymm1, ymm3, ymm4		;; I6 - I8 (newer R8)
	vaddpd	ymm3, ymm3, ymm4		;; I6 + I8 (newer I6)

	vsubpd	ymm4, ymm6, ymm5		;; R2 - R6 (final R6)
	vaddpd	ymm6, ymm6, ymm5		;; R2 + R6 (final R2)

	vmovapd	ymm5, [dstreg+e1+32]		;; Reload new I2
	ystore	[dstreg+e4+e1], ymm4		;; Save final R6
	vmovapd	ymm4, [dstreg+e2+e1+32]		;; Reload new I4
	ystore	[dstreg+e1], ymm6		;; Save final R2
	vaddpd	ymm6, ymm5, ymm4		;; I2 + I4 (newer I2)
	vsubpd	ymm5, ymm5, ymm4		;; I2 - I4 (newer I4)

	vsubpd	ymm4, ymm0, ymm1		;; R4 - R8 (final R8)
	vaddpd	ymm0, ymm0, ymm1		;; R4 + R8 (final R4)

	vsubpd	ymm1, ymm6, ymm3		;; I2 - I6 (final I6)
	vaddpd	ymm6, ymm6, ymm3		;; I2 + I6 (final I2)

	L1prefetch srcreg+d4+d2+d1+L1pd, L1pt

	vsubpd	ymm3, ymm5, ymm7		;; I4 - I8 (final I8)
	vaddpd	ymm5, ymm5, ymm7		;; I4 + I8 (final I4)

	;; Swizzle using 4 shufpds and 4 perm2f128  (1 or 3 clocks each on port 5, throughput 1)

	vmovapd	ymm7, [dstreg+32]		;; Reload final I1
	ystore	[dstreg+e4+e2+e1], ymm4		;; Save final R8
	vshufpd	ymm4, ymm7, ymm6, 0		;; Shuffle I1 and I2 to create I1/I2 low
	vshufpd	ymm7, ymm7, ymm6, 15		;; Shuffle I1 and I2 to create I1/I2 hi

	vshufpd	ymm6, ymm2, ymm5, 0		;; Shuffle I3 and I4 to create I3/I4 low
	vshufpd	ymm2, ymm2, ymm5, 15		;; Shuffle I3 and I4 to create I3/I4 hi

	ylow128s ymm5, ymm4, ymm6		;; Shuffle I1/I2 low and I3/I4 low (final I1)
	yhigh128s ymm4, ymm4, ymm6		;; Shuffle I1/I2 low and I3/I4 low (final I3)

	ylow128s ymm6, ymm7, ymm2		;; Shuffle I1/I2 hi and I3/I4 hi (final I2)
	yhigh128s ymm7, ymm7, ymm2		;; Shuffle I1/I2 hi and I3/I4 hi (final I4)

	;; Swizzle using 4 shufpds and 4 perm2f128  (1 or 3 clocks each on port 5, throughput 1)

	vmovapd	ymm2, [dstreg]			;; Reload final R1
	ystore	[dstreg+32], ymm5		;; Save I1
	vmovapd	ymm5, [dstreg+e1]		;; Reload final R2
	ystore	[dstreg+e2+32], ymm4		;; Save I3
	vshufpd	ymm4, ymm2, ymm5, 0		;; Shuffle R1 and R2 to create R1/R2 low
	vshufpd	ymm2, ymm2, ymm5, 15		;; Shuffle R1 and R2 to create R1/R2 hi

	vmovapd	ymm5, [dstreg+e2]		;; Reload final R3
	ystore	[dstreg+e1+32], ymm6		;; Save I2
	vshufpd	ymm6, ymm5, ymm0, 0		;; Shuffle R3 and R4 to create R3/R4 low
	vshufpd	ymm5, ymm5, ymm0, 15		;; Shuffle R3 and R4 to create R3/R4 hi

	ylow128s ymm0, ymm4, ymm6		;; Shuffle R1/R2 low and R3/R4 low (final R1)
	yhigh128s ymm4, ymm4, ymm6		;; Shuffle R1/R2 low and R3/R4 low (final R3)

	ylow128s ymm6, ymm2, ymm5		;; Shuffle R1/R2 hi and R3/R4 hi (final R2)
	yhigh128s ymm2, ymm2, ymm5		;; Shuffle R1/R2 hi and R3/R4 hi (final R4)

	;; Swizzle using 4 shufpds and 4 perm2f128  (1 or 3 clocks each on port 5, throughput 1)

	vmovapd	ymm5, [dstreg+e4+32]		;; Reload final I5
	ystore	[dstreg+e2+e1+32], ymm7		;; Save I4
	vshufpd	ymm7, ymm5, ymm1, 0		;; Shuffle I5 and I6 to create I5/I6 low
	vshufpd	ymm5, ymm5, ymm1, 15		;; Shuffle I5 and I6 to create I5/I6 hi

	vmovapd	ymm1, [dstreg+e4+e2+32]		;; Reload final I7
	ystore	[dstreg], ymm0			;; Save R1
	vshufpd	ymm0, ymm1, ymm3, 0		;; Shuffle I7 and I8 to create I7/I8 low
	vshufpd	ymm1, ymm1, ymm3, 15		;; Shuffle I7 and I8 to create I7/I8 hi

	ylow128s ymm3, ymm7, ymm0		;; Shuffle I5/I6 low and I7/I8 low (final I5)
	yhigh128s ymm7, ymm7, ymm0		;; Shuffle I5/I6 low and I7/I8 low (final I7)

	ylow128s ymm0, ymm5, ymm1		;; Shuffle I5/I6 hi and I7/I8 hi (final I6)
	yhigh128s ymm5, ymm5, ymm1		;; Shuffle I5/I6 hi and I7/I8 hi (final I8)

	;; Swizzle using 4 shufpds and 4 perm2f128  (1 or 3 clocks each on port 5, throughput 1)

	vmovapd	ymm1, [dstreg+e4]		;; Reload final R5
	ystore	[dstreg+e2], ymm4		;; Save R3
	vmovapd	ymm4, [dstreg+e4+e1]		;; Reload final R6
	ystore	[dstreg+e1], ymm6		;; Save R2
	vshufpd	ymm6, ymm1, ymm4, 0		;; Shuffle R5 and R6 to create R5/R6 low
	vshufpd	ymm1, ymm1, ymm4, 15		;; Shuffle R5 and R6 to create R5/R6 hi

	vmovapd	ymm4, [dstreg+e4+e2]		;; Reload final R7
	ystore	[dstreg+e2+e1], ymm2		;; Save R4
	vmovapd	ymm2, [dstreg+e4+e2+e1]		;; Reload final R8
	ystore	[dstreg+e4+32], ymm3		;; Save I5
	vshufpd	ymm3, ymm4, ymm2, 0		;; Shuffle R7 and R8 to create R7/R8 low
	vshufpd	ymm4, ymm4, ymm2, 15		;; Shuffle R7 and R8 to create R7/R8 hi

	ylow128s ymm2, ymm6, ymm3		;; Shuffle R5/R6 low and R7/R8 low (final R5)
	yhigh128s ymm6, ymm6, ymm3		;; Shuffle R5/R6 low and R7/R8 low (final R7)

	ylow128s ymm3, ymm1, ymm4		;; Shuffle R5/R6 hi and R7/R8 hi (final R6)
	yhigh128s ymm1, ymm1, ymm4		;; Shuffle R5/R6 hi and R7/R8 hi (final R8)

	ystore	[dstreg+e4+e2+32], ymm7		;; Save I7
	ystore	[dstreg+e4+e1+32], ymm0		;; Save I6
	ystore	[dstreg+e4+e2+e1+32], ymm5	;; Save I8
	ystore	[dstreg+e4], ymm2		;; Save R5
	ystore	[dstreg+e4+e2], ymm6		;; Save R7
	ystore	[dstreg+e4+e1], ymm3		;; Save R6
	ystore	[dstreg+e4+e2+e1], ymm1		;; Save R8

	bump	srcreg, srcinc
	bump	dstreg, dstinc
	bump	screg, scinc
	ENDM

IFDEF X86_64

yr8_rsc_sg8cl_eight_complex_unfft8 MACRO srcreg,srcinc,d1,d2,d4,dstreg,dstinc,e1,e2,e4,screg,scinc,maxrpt,L1pt,L1pd
	vmovapd	ymm0, [screg+0+32]		;; cosine for R1/I1
	vmovapd	ymm1, [srcreg]			;; R1
	vmulpd	ymm2, ymm1, ymm0		;; A1 = R1 * cosine				;  1-5

	vmovapd	ymm3, [screg+0]			;; sine for R1/I1
	vmovapd	ymm4, [srcreg+32]		;; I1
	vmulpd	ymm5, ymm4, ymm3		;; C1 = I1 * sine				;  2-6

	vmovapd	ymm6, [screg+256+32]		;; cosine for R2/I2
	vmovapd	ymm7, [srcreg+d1]		;; R2
	vmulpd	ymm8, ymm7, ymm6		;; A2 = R2 * cosine				;  3-7

	vmovapd	ymm9, [screg+256]		;; sine for R2/I2
	vmovapd	ymm10, [srcreg+d1+32]		;; I2
	vmulpd	ymm11, ymm10, ymm9		;; C2 = I2 * sine				;  4-8

	vmulpd	ymm4, ymm4, ymm0		;; B1 = I1 * cosine				;  5-9
	vmovapd	ymm12, [screg+128+32]		;; cosine for R3/I3

	vmulpd	ymm1, ymm1, ymm3		;; D1 = R1 * sine				;  6-10
	vmovapd	ymm13, [srcreg+d2]		;; R3

	vaddpd	ymm2, ymm2, ymm5		;; A1 = A1 + C1 (first R1)			; 7-9
	vmulpd	ymm10, ymm10, ymm6		;; B2 = I2 * cosine				;  7-11
	vmovapd	ymm14, [screg+128]		;; sine for R3/I3

	vmulpd	ymm7, ymm7, ymm9		;; D2 = R2 * sine				;  8-12
	vmovapd	ymm15, [srcreg+d2+32]		;; I3

	vaddpd	ymm8, ymm8, ymm11		;; A2 = A2 + C2 (first R2)			; 9-11
	vmulpd	ymm11, ymm13, ymm12		;; A3 = R3 * cosine/sine			;  9-13
	vmovapd	ymm0, [screg+384+32]		;; cosine for R4/I4

	vmulpd	ymm9, ymm15, ymm14		;; C3 = I3 * sine				;  10-14
	vmovapd	ymm3, [srcreg+d2+d1]		;; R4

	vsubpd	ymm4, ymm4, ymm1		;; B1 = B1 - D1 (first I1)			; 11-13
	vmulpd	ymm1, ymm3, ymm0		;; A4 = R4 * cosine/sine			;  11-15
	vmovapd	ymm5, [screg+384]		;; sine for R4/I4

	vmovapd ymm6, [srcreg+d2+d1+32]		;; I4
	vsubpd	ymm10, ymm10, ymm7		;; B2 = B2 - D2 (first I2)			; 13-15
	vmulpd	ymm7, ymm6, ymm5		;; C4 = I4 * sine				;  12-16

	vmulpd	ymm15, ymm15, ymm12		;; B3 = I3 * cosine				;  13-17

	vsubpd	ymm12, ymm2, ymm8		;; R1 - R2 (new R2)				; 14-15 (or maybe 12-14)
	vmulpd	ymm13, ymm13, ymm14		;; D3 = R3 * sine				;  14-18
	vmovapd	ymm14, [screg+64+32]		;; cosine for R5/I5

	vaddpd	ymm2, ymm2, ymm8		;; R1 + R2 (new R1)				; 15-17 (or maybe 14-16)
	vmulpd	ymm6, ymm6, ymm0		;; B4 = I4 * cosine/sine			;  15-19
	vmovapd	ymm8, [srcreg+d4]		;; R5

	vaddpd	ymm11, ymm11, ymm9		;; A3 = A3 + C3 (first R3)			; 16-18 (or maybe 15-17)
	vmulpd	ymm3, ymm3, ymm5		;; D4 = R4 * sine				;  16-20
	vmovapd	ymm0, [srcreg+d4+32]		;; I5

	vaddpd	ymm1, ymm1, ymm7		;; A4 = A4 + C4 (first R4)			; 17-19
	vmulpd	ymm7, ymm8, ymm14		;; A5 = R5 * cosine				;  17-21
	vmovapd	ymm9, [screg+64]		;; sine for R5/I5

	vaddpd	ymm5, ymm4, ymm10		;; I1 + I2 (new I1)				; 18-20 (or maybe 16-18)
	vmulpd	ymm14, ymm0, ymm14		;; B5 = I5 * cosine				;  18-22

	vsubpd	ymm4, ymm4, ymm10		;; I1 - I2 (new I2)				; 19-21 (or maybe 18-20)
	vmulpd	ymm0, ymm0, ymm9		;; C5 = I5 * sine				;  19-23
	vmovapd	ymm10, [screg+320+32]		;; cosine for R6/I6

	vsubpd	ymm15, ymm15, ymm13		;; B3 = B3 - D3 (first I3)			; 20-22 (or maybe 19-21)
	vmulpd	ymm8, ymm8, ymm9		;; D5 = R5 * sine				;  20-24
	vmovapd	ymm13, [srcreg+d4+d1]		;; R6

	ystore	[dstreg+32], ymm5		;; Save new I1					; 21 (or maybe 19)
	vsubpd	ymm6, ymm6, ymm3		;; B4 = B4 - D4 (first I4)			; 21-23
	vmulpd	ymm3, ymm13, ymm10		;; A6 = R6 * cosine				;  21-25
	vmovapd ymm9, [srcreg+d4+d1+32]		;; I6

	ystore	[dstreg+e1+32], ymm4		;; Save new I2					; 22 (or maybe 21)
	vsubpd	ymm4, ymm1, ymm11		;; R4 - R3 (new I4)				; 22-24 (or maybe 20-22)
	vmulpd	ymm10, ymm9, ymm10		;; B6 = I6 * cosine				;  22-26
	vmovapd	ymm5, [screg+320]		;; sine for R6/I6

	vaddpd	ymm1, ymm1, ymm11		;; R4 + R3 (new R3)				; 23-25 (or maybe 22-24)
	vmulpd	ymm9, ymm9, ymm5		;; C6 = I6 * sine				;  23-27
	vmovapd	ymm11, [screg+192+32]		;; cosine for R7/I7

	vaddpd	ymm7, ymm7, ymm0		;; A5 = A5 + C5 (first R5)			; 24-26
	vmulpd	ymm13, ymm13, ymm5		;; D6 = R6 * sine				;  24-28
	vmovapd	ymm0, [srcreg+d4+d2]		;; R7

	ystore	[dstreg+e2+e1+32], ymm4		;; Save new I4					; 25 (or maybe 23)
	vsubpd	ymm14, ymm14, ymm8		;; B5 = B5 - D5 (first I5)			; 25-27
	vmulpd	ymm8, ymm0, ymm11		;; A7 = R7 * cosine				;  25-29
	vmovapd	ymm5, [srcreg+d4+d2+32]		;; I7

	vaddpd	ymm4, ymm15, ymm6		;; I3 + I4 (new I3)				; 26-28
	vmulpd	ymm11, ymm5, ymm11		;; B7 = I7 * cosine				;  26-30

	vsubpd	ymm15, ymm15, ymm6		;; I3 - I4 (new R4)				; 27-29
	vmovapd	ymm6, [screg+192]		;; sine for R7/I7
	vmulpd	ymm5, ymm5, ymm6		;; C7 = I7 * sine				;  27-31

	vaddpd	ymm3, ymm3, ymm9		;; A6 = A6 + C6 (first R6)			; 28-30
	vmulpd	ymm0, ymm0, ymm6		;; D7 = R7 * sine				;  28-32
	vmovapd	ymm9, [screg+448+32]		;; cosine for R8/I8

	vmovapd	ymm6, [srcreg+d4+d2+d1]		;; R8
	ystore	[dstreg+e2+32], ymm4		;; Save new I3					; 29
	vsubpd	ymm10, ymm10, ymm13		;; B6 = B6 - D6 (first I6)			; 29-31
	vmulpd	ymm13, ymm6, ymm9		;; A8 = R8 * cosine				;  29-33

	vsubpd	ymm4, ymm2, ymm1		;; R1 - R3 (newer R3)				; 30-32
	vaddpd	ymm2, ymm2, ymm1		;; R1 + R3 (newer R1)				; 31-33
	vmovapd ymm1, [srcreg+d4+d2+d1+32]	;; I8
	vmulpd	ymm9, ymm1, ymm9		;; B8 = I8 * cosine				;  30-34

	vaddpd	ymm8, ymm8, ymm5		;; A7 = A7 + C7 (first R7)			; 32-34
	vmovapd	ymm5, [screg+448]		;; sine for R8/I8
	vmulpd	ymm1, ymm1, ymm5		;; C8 = I8 * sine				;  31-35

	vmulpd	ymm6, ymm6, ymm5		;; D8 = R8 * sine				;  32-36
	vmovapd	ymm5, YMM_SQRTHALF

	vsubpd	ymm11, ymm11, ymm0		;; B7 = B7 - D7 (first I7)			; 33-35
	L1prefetch srcreg+L1pd, L1pt

	vsubpd	ymm0, ymm7, ymm3		;; R5 - R6 (new R6)				; 34-36

	vaddpd	ymm7, ymm7, ymm3		;; R5 + R6 (new R5)				; 35-37

	vaddpd	ymm13, ymm13, ymm1		;; A8 = A8 + C8 (first R8)			; 36-38
	L1prefetch srcreg+d1+L1pd, L1pt

	vsubpd	ymm9, ymm9, ymm6		;; B8 = B8 - D8 (first I8)			; 37-39

	vsubpd	ymm6, ymm14, ymm10		;; I5 - I6 (new I6)				; 38-40

	vaddpd	ymm14, ymm14, ymm10		;; I5 + I6 (new I5)				; 39-41
	L1prefetch srcreg+d2+L1pd, L1pt

	vsubpd	ymm10, ymm13, ymm8		;; R8 - R7 (new I8)				; 40-42

	vaddpd	ymm13, ymm13, ymm8		;; R8 + R7 (new R7)				; 41-43

	vsubpd	ymm8, ymm11, ymm9		;; I7 - I8 (new R8)				; 42-44
	L1prefetch srcreg+d2+d1+L1pd, L1pt

	vaddpd	ymm11, ymm11, ymm9		;; I7 + I8 (new I7)				; 43-45

	vsubpd	ymm9, ymm6, ymm0		;; I6 = I6 - R6					; 44-46

	vaddpd	ymm0, ymm0, ymm6		;; R6 = R6 + I6					; 45-47
	L1prefetch srcreg+d4+L1pd, L1pt

	vsubpd	ymm6, ymm10, ymm8		;; I8 = I8 - R8					; 46-48

	vaddpd	ymm8, ymm8, ymm10		;; R8 = R8 + I8					; 47-49
	vmulpd	ymm9, ymm9, ymm5		;; I6 * SQRTHALF				;  47-51

	vsubpd	ymm10, ymm14, ymm11		;; I5 - I7 (newer R7)				; 48-50
	vmulpd	ymm0, ymm0, ymm5		;; R6 * SQRTHALF				;  48-52
	L1prefetch srcreg+d4+d1+L1pd, L1pt

	vaddpd	ymm14, ymm14, ymm11		;; I5 + I7 (newer I5)				; 49-51
	vmulpd	ymm6, ymm6, ymm5		;; I8 * SQRTHALF				;  49-53

	vsubpd	ymm11, ymm12, ymm15		;; R2 - R4 (newer R4)				; 50-52
	vmulpd	ymm8, ymm8, ymm5		;; R8 * SQRTHALF				;  50-54

	vaddpd	ymm12, ymm12, ymm15		;; R2 + R4 (newer R2)				; 51-53
	L1prefetch srcreg+d4+d2+L1pd, L1pt

	vaddpd	ymm15, ymm13, ymm7		;; R7 + R5 (newer R5)				; 52-54

	vsubpd	ymm13, ymm13, ymm7		;; R7 - R5 (newer I7)				; 53-55

	vaddpd	ymm7, ymm4, ymm10		;; R3 + R7 (final R3)				; 54-56
	L1prefetch srcreg+d4+d2+d1+L1pd, L1pt

	vsubpd	ymm4, ymm4, ymm10		;; R3 - R7 (final R7)				; 55-57

	vsubpd	ymm10, ymm9, ymm6		;; I6 - I8 (newer R8)				; 56-58

	vaddpd	ymm9, ymm9, ymm6		;; I6 + I8 (newer I6)				; 57-59

	vaddpd	ymm6, ymm8, ymm0		;; R8 + R6 (newer R6)				; 58-60

	vsubpd	ymm8, ymm8, ymm0		;; R8 - R6 (newer I8)				; 59-61

	vaddpd	ymm0, ymm11, ymm10		;; R4 + R8 (final R4)				; 60-62

	vaddpd	ymm5, ymm2, ymm15		;; R1 + R5 (final R1)				; 61-63

	vaddpd	ymm1, ymm12, ymm6		;; R2 + R6 (final R2)				; 62-64

	vshufpd	ymm3, ymm7, ymm0, 0		;; Shuffle R3 and R4 to create R3/R4 low	; 63
	vsubpd	ymm11, ymm11, ymm10		;; R4 - R8 (final R8)				; 63-65
	vmovapd	ymm10, [dstreg+e1+32]		;; Reload new I2

	vshufpd	ymm7, ymm7, ymm0, 15		;; Shuffle R3 and R4 to create R3/R4 hi		; 64
	vsubpd	ymm2, ymm2, ymm15		;; R1 - R5 (final R5)				; 64-66
	vmovapd	ymm0, [dstreg+e2+e1+32]		;; Reload new I4

	vshufpd	ymm15, ymm5, ymm1, 0		;; Shuffle R1 and R2 to create R1/R2 low	; 65
	vsubpd	ymm12, ymm12, ymm6		;; R2 - R6 (final R6)				; 65-67

	vshufpd	ymm5, ymm5, ymm1, 15		;; Shuffle R1 and R2 to create R1/R2 hi		; 66
	vaddpd	ymm1, ymm10, ymm0		;; I2 + I4 (newer I2)				; 66-68

	ylow128s ymm6, ymm15, ymm3		;; Shuffle R1/R2 low and R3/R4 low (final R1)	; 67-68
	vsubpd	ymm10, ymm10, ymm0		;; I2 - I4 (newer I4)				; 67-69
	vmovapd	ymm0, [dstreg+32]		;; Reload new I1

	yhigh128s ymm15, ymm15, ymm3		;; Shuffle R1/R2 low and R3/R4 low (final R3)	; 68-69
	vmovapd	ymm3, [dstreg+e2+32]		;; Reload new I3
	ystore	[dstreg], ymm6			;; Save R1					; 69
	vaddpd	ymm6, ymm0, ymm3		;; I1 + I3 (newer I1)				; 68-70

	vsubpd	ymm0, ymm0, ymm3		;; I1 - I3 (newer I3)				; 69-71
	ylow128s ymm3, ymm5, ymm7		;; Shuffle R1/R2 hi and R3/R4 hi (final R2)	; 69-70

	yhigh128s ymm5, ymm5, ymm7		;; Shuffle R1/R2 hi and R3/R4 hi (final R4)	; 70-71
	vaddpd	ymm7, ymm1, ymm9		;; I2 + I6 (final I2)				; 70-72
	ystore	[dstreg+e2], ymm15		;; Save R3					; 70

	vsubpd	ymm1, ymm1, ymm9		;; I2 - I6 (final I6)				; 71-73
	ystore	[dstreg+e1], ymm3		;; Save R2					; 71

	vshufpd	ymm3, ymm2, ymm12, 0		;; Shuffle R5 and R6 to create R5/R6 low	; 72
	vaddpd	ymm9, ymm6, ymm14		;; I1 + I5 (final I1)				; 72-74
	ystore	[dstreg+e2+e1], ymm5		;; Save R4					; 72

	vshufpd	ymm2, ymm2, ymm12, 15		;; Shuffle R5 and R6 to create R5/R6 hi		; 73
	vaddpd	ymm12, ymm10, ymm8		;; I4 + I8 (final I4)				; 73-75

	vshufpd	ymm5, ymm4, ymm11, 0		;; Shuffle R7 and R8 to create R7/R8 low	; 74
	vaddpd	ymm15, ymm0, ymm13		;; I3 + I7 (final I3)				; 74-76

	vshufpd	ymm4, ymm4, ymm11, 15		;; Shuffle R7 and R8 to create R7/R8 hi		; 75
	vsubpd	ymm6, ymm6, ymm14		;; I1 - I5 (final I5)				; 75-77

	vshufpd	ymm14, ymm9, ymm7, 0		;; Shuffle I1 and I2 to create I1/I2 low	; 76
	vsubpd	ymm10, ymm10, ymm8		;; I4 - I8 (final I8)				; 76-78

	vshufpd	ymm9, ymm9, ymm7, 15		;; Shuffle I1 and I2 to create I1/I2 hi		; 77
	vsubpd	ymm0, ymm0, ymm13		;; I3 - I7 (final I7)				; 77-79

	vshufpd	ymm13, ymm15, ymm12, 0		;; Shuffle I3 and I4 to create I3/I4 low	; 78
	vshufpd	ymm15, ymm15, ymm12, 15		;; Shuffle I3 and I4 to create I3/I4 hi		; 79

	vshufpd	ymm12, ymm6, ymm1, 0		;; Shuffle I5 and I6 to create I5/I6 low
	vshufpd	ymm6, ymm6, ymm1, 15		;; Shuffle I5 and I6 to create I5/I6 hi

	vshufpd	ymm1, ymm0, ymm10, 0		;; Shuffle I7 and I8 to create I7/I8 low
	vshufpd	ymm0, ymm0, ymm10, 15		;; Shuffle I7 and I8 to create I7/I8 hi

	ylow128s ymm10, ymm3, ymm5		;; Shuffle R5/R6 low and R7/R8 low (final R5)
	yhigh128s ymm3, ymm3, ymm5		;; Shuffle R5/R6 low and R7/R8 low (final R7)
	ylow128s ymm5, ymm2, ymm4		;; Shuffle R5/R6 hi and R7/R8 hi (final R6)
	yhigh128s ymm2, ymm2, ymm4		;; Shuffle R5/R6 hi and R7/R8 hi (final R8)

	ystore	[dstreg+e4], ymm10		;; Save R5
	ystore	[dstreg+e4+e2], ymm3		;; Save R7
	ystore	[dstreg+e4+e1], ymm5		;; Save R6
	ystore	[dstreg+e4+e2+e1], ymm2		;; Save R8

	ylow128s ymm4, ymm14, ymm13		;; Shuffle I1/I2 low and I3/I4 low (final I1)
	yhigh128s ymm14, ymm14, ymm13		;; Shuffle I1/I2 low and I3/I4 low (final I3)
	ylow128s ymm13, ymm9, ymm15		;; Shuffle I1/I2 hi and I3/I4 hi (final I2)
	yhigh128s ymm9, ymm9, ymm15		;; Shuffle I1/I2 hi and I3/I4 hi (final I4)

	ystore	[dstreg+32], ymm4		;; Save I1
	ystore	[dstreg+e2+32], ymm14		;; Save I3
	ystore	[dstreg+e1+32], ymm13		;; Save I2
	ystore	[dstreg+e2+e1+32], ymm9		;; Save I4

	ylow128s ymm15, ymm12, ymm1		;; Shuffle I5/I6 low and I7/I8 low (final I5)
	yhigh128s ymm12, ymm12, ymm1		;; Shuffle I5/I6 low and I7/I8 low (final I7)
	ylow128s ymm1, ymm6, ymm0		;; Shuffle I5/I6 hi and I7/I8 hi (final I6)
	yhigh128s ymm6, ymm6, ymm0		;; Shuffle I5/I6 hi and I7/I8 hi (final I8)

	ystore	[dstreg+e4+32], ymm15		;; Save I5
	ystore	[dstreg+e4+e2+32], ymm12	;; Save I7
	ystore	[dstreg+e4+e1+32], ymm1		;; Save I6
	ystore	[dstreg+e4+e2+e1+32], ymm6	;; Save I8

	bump	srcreg, srcinc
	bump	dstreg, dstinc
	bump	screg, scinc
	ENDM

IF (@INSTR(,%yarch,<FMA3>) NE 0)

yr8_rsc_sg8cl_eight_complex_unfft8 MACRO srcreg,srcinc,d1,d2,d4,dstreg,dstinc,e1,e2,e4,screg,scinc,maxrpt,L1pt,L1pd
	vmovapd	ymm0, [screg+0+32]		;; cosine for R1/I1
	vmovapd	ymm1, [srcreg]			;; R1
	vmovapd	ymm2, [srcreg+32]		;; I1
	vmulpd	ymm3, ymm1, ymm0		;; A1 = R1 * cosine				; 1-5		n 6
	vmulpd	ymm0, ymm2, ymm0		;; B1 = I1 * cosine				; 1-5		n 6

	vmovapd	ymm4, [screg+256+32]		;; cosine for R2/I2
	vmovapd	ymm5, [srcreg+d1]		;; R2
	vmovapd	ymm6, [srcreg+d1+32]		;; I2
	vmulpd	ymm7, ymm5, ymm4		;; A2 = R2 * cosine				; 2-6		n 7
	vmulpd	ymm4, ymm6, ymm4		;; B2 = I2 * cosine				; 2-6		n 7

	vmovapd	ymm8, [screg+64+32]		;; cosine for R5/I5
	vmovapd	ymm9, [srcreg+d4]		;; R5
	vmovapd	ymm10, [srcreg+d4+32]		;; I5
	vmulpd	ymm11, ymm9, ymm8		;; A5 = R5 * cosine				; 3-7		n 8
	vmulpd	ymm8, ymm10, ymm8		;; B5 = I5 * cosine				; 3-7		n 8
	vmovapd	ymm15, [screg+320+32]		;; cosine for R6/I6
	vmovapd	ymm14, [srcreg+d4+d1]		;; R6
	vmovapd ymm13, [srcreg+d4+d1+32]	;; I6

	vmovapd	ymm12, [screg+0]		;; sine for R1/I1
	yfnmaddpd ymm1, ymm1, ymm12, ymm0	;; B1 = B1 - R1 * sine (first I1)		; 6-10		n 14
	yfmaddpd ymm2, ymm2, ymm12, ymm3	;; A1 = A1 + I1 * sine (first R1)		; 6-10		n 15
	vmovapd	ymm0, [screg+192+32]		;; cosine for R7/I7

	vmulpd	ymm3, ymm14, ymm15		;; A6 = R6 * cosine				; 4-8		n 9
	vmulpd	ymm15, ymm13, ymm15		;; B6 = I6 * cosine				; 4-8		n 9

	vmovapd	ymm12, [screg+256]		;; sine for R2/I2
	yfnmaddpd ymm5, ymm5, ymm12, ymm4	;; B2 = B2 - R2 * sine (first I2)		; 7-11		n 14
	yfmaddpd ymm6, ymm6, ymm12, ymm7	;; A2 = A2 + I2 * sine (first R2)		; 7-11		n 15
	vmovapd	ymm4, [srcreg+d4+d2]		;; R7
	vmovapd	ymm12, [srcreg+d4+d2+32]	;; I7

	vmovapd	ymm7, [screg+64]		;; sine for R5/I5
	yfmaddpd ymm10, ymm10, ymm7, ymm11	;; A5 = A5 + I5 * sine (first R5)		; 8-12		n 19
	yfnmaddpd ymm9, ymm9, ymm7, ymm8	;; B5 = B5 - R5 * sine (first I5)		; 8-12		n 20

	vmulpd	ymm11, ymm4, ymm0		;; A7 = R7 * cosine				; 5-9		n 10
	vmulpd	ymm0, ymm12, ymm0		;; B7 = I7 * cosine				; 5-9		n 10

	vmovapd	ymm7, [screg+320]		;; sine for R6/I6
	yfmaddpd ymm13, ymm13, ymm7, ymm3	;; A6 = A6 + I6 * sine (first R6)		; 9-13		n 19
	yfnmaddpd ymm14, ymm14, ymm7, ymm15	;; B6 = B6 - R6 * sine (first I6)		; 9-13		n 20

	vmovapd	ymm15, [screg+192]		;; sine for R7/I7
	yfmaddpd ymm12, ymm12, ymm15, ymm11	;; A7 = A7 + I7 * sine (first R7)		; 10-14		n 21
	yfnmaddpd ymm4, ymm4, ymm15, ymm0	;; B7 = B7 - R7 * sine (first I7)		; 10-14		n 22

	vmovapd	ymm8, [screg+448+32]		;; cosine for R8/I8
	vmovapd	ymm3, [srcreg+d4+d2+d1]		;; R8
	vmovapd ymm7, [srcreg+d4+d2+d1+32]	;; I8
	vmulpd	ymm0, ymm3, ymm8		;; A8 = R8 * cosine				; 11-15		n 16
	vmulpd	ymm8, ymm7, ymm8		;; B8 = I8 * cosine				; 11-15		n 16

	vmovapd	ymm11, [screg+128+32]		;; cosine for R3/I3

	vmovapd	ymm15, [screg+448]		;; sine for R8/I8
	yfmaddpd ymm7, ymm7, ymm15, ymm0	;; A8 = A8 + I8 * sine (first R8)		; 16-20		n 21
	yfnmaddpd ymm3, ymm3, ymm15, ymm8	;; B8 = B8 - R8 * sine (first I8)		; 16-20		n 22

	vmovapd	ymm0, [srcreg+d2]		;; R3
	vmovapd	ymm15, [srcreg+d2+32]		;; I3
	vmulpd	ymm8, ymm0, ymm11		;; A3 = R3 * cosine				; 12-16		n 17
	vmulpd	ymm11, ymm15, ymm11		;; B3 = I3 * cosine				; 12-16		n 17

	yfmaddpd ymm15, ymm15, [screg+128], ymm8 ;; A3 = A3 + I3 * sine (first R3)		; 17-21		n 23
	yfnmaddpd ymm0, ymm0, [screg+128], ymm11 ;; B3 = B3 - R3 * sine (first I3)		; 17-21		n 24

	vmovapd ymm8, YMM_ONE
	yfmsubpd ymm11, ymm1, ymm8, ymm5	;; I1 - I2 (new I2)				; 14-18		n 31
	yfmaddpd ymm1, ymm1, ymm8, ymm5		;; I1 + I2 (new I1)				; 14-18		n 35

	yfmaddpd ymm5, ymm2, ymm8, ymm6		;; R1 + R2 (new R1)				; 15-19		n 29
	yfmsubpd ymm2, ymm2, ymm8, ymm6		;; R1 - R2 (new R2)				; 15-19		n 30
	vmovapd	ymm8, [screg+384+32]		;; cosine for R4/I4
	vmovapd	ymm6, [srcreg+d2+d1]		;; R4

	ystore	[dstreg+32], ymm11		;; Temp save new I2				; 19
	vmovapd ymm11, [srcreg+d2+d1+32]	;; I4
	ystore	[dstreg], ymm1			;; Temp save new I1				; 19+1
	vmulpd	ymm1, ymm6, ymm8		;; A4 = R4 * cosine				; 13-17		n 18
	vmulpd	ymm8, ymm11, ymm8		;; B4 = I4 * cosine				; 13-17		n 18

	yfmaddpd ymm11, ymm11, [screg+384], ymm1 ;; A4 = A4 + I4 * sine (first R4)		; 18-22		n 23
	yfnmaddpd ymm6, ymm6, [screg+384], ymm8	;; B4 = B4 - R4 * sine (first I4)		; 18-22		n 24

	vmovapd ymm8, YMM_ONE
	yfmsubpd ymm1, ymm10, ymm8, ymm13	;; R5 - R6 (new R6)				; 19-23		n 25
	yfmaddpd ymm10, ymm10, ymm8, ymm13	;; R5 + R6 (new R5)				; 19-23		n 26
	L1prefetch srcreg+L1pd, L1pt

	yfmsubpd ymm13, ymm9, ymm8, ymm14	;; I5 - I6 (new I6)				; 20-24		n 25
	yfmaddpd ymm9, ymm9, ymm8, ymm14	;; I5 + I6 (new I5)				; 20-24		n 28

	yfmaddpd ymm14, ymm7, ymm8, ymm12	;; R8 + R7 (new R7)				; 21-25		n 26
	yfmsubpd ymm7, ymm7, ymm8, ymm12	;; R8 - R7 (new I8)				; 21-25		n 27
	L1prefetch srcreg+d1+L1pd, L1pt

	yfmsubpd ymm12, ymm4, ymm8, ymm3	;; I7 - I8 (new R8)				; 22-26		n 27
	yfmaddpd ymm4, ymm4, ymm8, ymm3		;; I7 + I8 (new I7)				; 22-26		n 28

	yfmaddpd ymm3, ymm11, ymm8, ymm15	;; R4 + R3 (new R3)				; 23-27		n 29
	yfmsubpd ymm11, ymm11, ymm8, ymm15	;; R4 - R3 (new I4)				; 23-27		n 31
	L1prefetch srcreg+d2+L1pd, L1pt

	yfmsubpd ymm15, ymm0, ymm8, ymm6	;; I3 - I4 (new R4)				; 24-28		n 30
	yfmaddpd ymm0, ymm0, ymm8, ymm6		;; I3 + I4 (new I3)				; 24-28		n 35

	yfmsubpd ymm6, ymm13, ymm8, ymm1	;; I6 - R6 (new2 I6)				; 25-29		n 32
	yfmaddpd ymm1, ymm1, ymm8, ymm13	;; R6 + I6 (new2 R6)				; 25-29		n 33
	L1prefetch srcreg+d2+d1+L1pd, L1pt

	yfmaddpd ymm13, ymm14, ymm8, ymm10	;; R7 + R5 (newer R5)				; 26-30		n 34
	yfmsubpd ymm14, ymm14, ymm8, ymm10	;; R7 - R5 (newer I7)				; 26-30		n 42

	yfmsubpd ymm10, ymm7, ymm8, ymm12	;; I8 - R8 (new2 I8)				; 27-31		n 32
	yfmaddpd ymm12, ymm12, ymm8, ymm7	;; R8 + I8 (new2 R8)				; 27-31		n 33
	L1prefetch srcreg+d4+L1pd, L1pt

	yfmsubpd ymm7, ymm9, ymm8, ymm4		;; I5 - I7 (newer R7)				; 28-32		n 36
	yfmaddpd ymm9, ymm9, ymm8, ymm4		;; I5 + I7 (newer I5)				; 28-32		n 41

	yfmsubpd ymm4, ymm5, ymm8, ymm3		;; R1 - R3 (newer R3)				; 29-33		n 36
	yfmaddpd ymm5, ymm5, ymm8, ymm3		;; R1 + R3 (newer R1)				; 29-33		n 34
	L1prefetch srcreg+d4+d1+L1pd, L1pt

	yfmsubpd ymm3, ymm2, ymm8, ymm15	;; R2 - R4 (newer R4)				; 30-34		n 37
	yfmaddpd ymm2, ymm2, ymm8, ymm15	;; R2 + R4 (newer R2)				; 30-34		n 38

	vmovapd	ymm15, [dstreg+32]		;; Reload new I2
	ystore	[dstreg+e1], ymm14		;; Temp save newer I7				; 31
	yfmaddpd ymm14, ymm15, ymm8, ymm11	;; I2 + I4 (newer I2)				; 31-35		n 39
	yfmsubpd ymm15, ymm15, ymm8, ymm11	;; I2 - I4 (newer I4)				; 31-35		n 40

	yfmsubpd ymm11, ymm6, ymm8, ymm10	;; I6 - I8 (newer R8/SQRTHALF)			; 32-36		n 37
	yfmaddpd ymm6, ymm6, ymm8, ymm10	;; I6 + I8 (newer I6/SQRTHALF)			; 32-36		n 39

	ystore	[dstreg+e1+32], ymm9		;; Temp save newer I5				; 33
	yfmaddpd ymm10, ymm12, ymm8, ymm1	;; R8 + R6 (newer R6/SQRTHALF)			; 33-37		n 38
	yfmsubpd ymm12, ymm12, ymm8, ymm1	;; R8 - R6 (newer I8/SQRTHALF)			; 33-37		n 40
	vmovapd	ymm9, [dstreg]			;; Reload new I1

	yfmaddpd ymm1, ymm5, ymm8, ymm13	;; R1 + R5 (final R1)				; 34-38		n 44
	yfmsubpd ymm5, ymm5, ymm8, ymm13	;; R1 - R5 (final R5)				; 34-38		n 46
	bump	screg, scinc

	yfmaddpd ymm13, ymm9, ymm8, ymm0	;; I1 + I3 (newer I1)				; 35-39		n 41
	yfmsubpd ymm9, ymm9, ymm8, ymm0		;; I1 - I3 (newer I3)				; 35-39		n 42
	L1prefetch srcreg+d4+d2+L1pd, L1pt

	yfmaddpd ymm0, ymm4, ymm8, ymm7		;; R3 + R7 (final R3)				; 36-40		n 42
	yfmsubpd ymm4, ymm4, ymm8, ymm7		;; R3 - R7 (final R7)				; 36-40		n 48

	vmovapd	ymm8, YMM_SQRTHALF
	yfmaddpd ymm7, ymm11, ymm8, ymm3	;; R4 + R8 * SQRTHALF (final R4)		; 37-41		n 42
	yfnmaddpd ymm11, ymm11, ymm8, ymm3	;; R4 - R8 * SQRTHALF (final R8)		; 37-41		n 48

	yfmaddpd ymm3, ymm10, ymm8, ymm2	;; R2 + R6 * SQRTHALF (final R2)		; 38-42		n 44
	yfnmaddpd ymm10, ymm10, ymm8, ymm2	;; R2 - R6 * SQRTHALF (final R6)		; 38-42		n 46
	L1prefetch srcreg+d4+d2+d1+L1pd, L1pt

	yfmaddpd ymm2, ymm6, ymm8, ymm14	;; I2 + I6 * SQRTHALF (final I2)		; 39-43		n 50
	yfnmaddpd ymm6, ymm6, ymm8, ymm14	;; I2 - I6 * SQRTHALF (final I6)		; 39-43		n 52

	yfmaddpd ymm14, ymm12, ymm8, ymm15	;; I4 + I8 * SQRTHALF (final I4)		; 40-44		n 54
	yfnmaddpd ymm12, ymm12, ymm8, ymm15	;; I4 - I8 * SQRTHALF (final I8)		; 40-44		n 56
	vmovapd	ymm8, YMM_ONE

	vshufpd	ymm15, ymm0, ymm7, 0		;; Shuffle R3 and R4 to create R3/R4 low	; 42
	vshufpd	ymm0, ymm0, ymm7, 15		;; Shuffle R3 and R4 to create R3/R4 hi		; 43
	vmovapd	ymm7, [dstreg+e1+32]		;; Reload newer I5

	ystorelo [dstreg][16], ymm15		;; Save R1					; 43
	ystorehi [dstreg+e2][16], ymm15		;; Save R3					; 43+1

	yfmaddpd ymm15, ymm13, ymm8, ymm7	;; I1 + I5 (final I1)				; 41-45		n 50
	yfmsubpd ymm13, ymm13, ymm8, ymm7	;; I1 - I5 (final I5)				; 41-45		n 52
	bump	srcreg, srcinc

	vshufpd	ymm7, ymm1, ymm3, 0		;; Shuffle R1 and R2 to create R1/R2 low	; 44
	vshufpd	ymm1, ymm1, ymm3, 15		;; Shuffle R1 and R2 to create R1/R2 hi		; 45
	vmovapd	ymm3, [dstreg+e1]		;; Reload newer I7

	ystorelo [dstreg+e1][16], ymm0		;; Save R2					; 44+1
	ystorehi [dstreg+e2+e1][16], ymm0	;; Save R4					; 44+2

	yfmaddpd ymm0, ymm9, ymm8, ymm3		;; I3 + I7 (final I3)				; 42-46		n 54
	yfmsubpd ymm9, ymm9, ymm8, ymm3		;; I3 - I7 (final I7)				; 42-46		n 56

	vshufpd	ymm3, ymm5, ymm10, 0		;; Shuffle R5 and R6 to create R5/R6 low	; 46
	vshufpd	ymm5, ymm5, ymm10, 15		;; Shuffle R5 and R6 to create R5/R6 hi		; 47

	ystorelo [dstreg], ymm7			;; Save R1					; 45+2
	ystorehi [dstreg+e2], ymm7		;; Save R3					; 45+3

	vshufpd	ymm10, ymm4, ymm11, 0		;; Shuffle R7 and R8 to create R7/R8 low	; 48
	vshufpd	ymm4, ymm4, ymm11, 15		;; Shuffle R7 and R8 to create R7/R8 hi		; 49

	ystorelo [dstreg+e1], ymm1		;; Save R2					; 46+3
	ystorehi [dstreg+e2+e1], ymm1		;; Save R4					; 46+4

	vshufpd	ymm11, ymm15, ymm2, 0		;; Shuffle I1 and I2 to create I1/I2 low	; 50
	vshufpd	ymm15, ymm15, ymm2, 15		;; Shuffle I1 and I2 to create I1/I2 hi		; 51

	ystorelo [dstreg+e4], ymm3		;; Save R5					; 47+4
	ystorehi [dstreg+e4+e2], ymm3		;; Save R7					; 47+5

	vshufpd	ymm2, ymm13, ymm6, 0		;; Shuffle I5 and I6 to create I5/I6 low	; 52
	vshufpd	ymm13, ymm13, ymm6, 15		;; Shuffle I5 and I6 to create I5/I6 hi		; 53

	ystorelo [dstreg+e4+e1], ymm5		;; Save R6					; 48+5
	ystorehi [dstreg+e4+e2+e1], ymm5	;; Save R8					; 48+6

	vshufpd	ymm6, ymm0, ymm14, 0		;; Shuffle I3 and I4 to create I3/I4 low	; 54
	vshufpd	ymm0, ymm0, ymm14, 15		;; Shuffle I3 and I4 to create I3/I4 hi		; 55

	ystorelo [dstreg+e4][16], ymm10		;; Save R5					; 49+6
	ystorehi [dstreg+e4+e2][16], ymm10	;; Save R7					; 49+7

	vshufpd	ymm14, ymm9, ymm12, 0		;; Shuffle I7 and I8 to create I7/I8 low	; 56
	vshufpd	ymm9, ymm9, ymm12, 15		;; Shuffle I7 and I8 to create I7/I8 hi		; 57

	ystorelo [dstreg+e4+e1][16], ymm4	;; Save R6					; 50+7
	ystorehi [dstreg+e4+e2+e1][16], ymm4	;; Save R8					; 50+8

	ylow128s ymm12, ymm11, ymm6		;; Shuffle I1/I2 low and I3/I4 low (final I1)	; 58-59
	yhigh128s ymm11, ymm11, ymm6		;; Shuffle I1/I2 low and I3/I4 low (final I3)	; 59-60
	ylow128s ymm6, ymm15, ymm0		;; Shuffle I1/I2 hi and I3/I4 hi (final I2)	; 60-61
	ystore	[dstreg+32], ymm12		;; Save I1					; 61
	yhigh128s ymm15, ymm15, ymm0		;; Shuffle I1/I2 hi and I3/I4 hi (final I4)	; 61-62
	ystore	[dstreg+e2+32], ymm11		;; Save I3					; 62

	ylow128s ymm0, ymm2, ymm14		;; Shuffle I5/I6 low and I7/I8 low (final I5)	; 62-63
	ystore	[dstreg+e1+32], ymm6		;; Save I2					; 63
	yhigh128s ymm2, ymm2, ymm14		;; Shuffle I5/I6 low and I7/I8 low (final I7)	; 63-64
	ystore	[dstreg+e2+e1+32], ymm15	;; Save I4					; 64
	ylow128s ymm14, ymm13, ymm9		;; Shuffle I5/I6 hi and I7/I8 hi (final I6)	; 64-65
	ystore	[dstreg+e4+32], ymm0		;; Save I5					; 65
	yhigh128s ymm13, ymm13, ymm9		;; Shuffle I5/I6 hi and I7/I8 hi (final I8)	; 65-66
	ystore	[dstreg+e4+e2+32], ymm2		;; Save I7					; 66
	ystore	[dstreg+e4+e1+32], ymm14	;; Save I6					; 67
	ystore	[dstreg+e4+e2+e1+32], ymm13	;; Save I8					; 68

	bump	dstreg, dstinc
	ENDM

ENDIF

ENDIF


; These versions use registers for distances between blocks.  This lets us share pass1 code.

yr8_rsc_sg8clreg_eight_complex_unfft8_preload MACRO
	ENDM
yr8_rsc_sg8clreg_eight_complex_unfft8 MACRO srcreg,srcinc,d1reg,d3reg,src4reg,dstreg,dstinc,e1,e2,e4,screg,scinc,maxrpt,L1pt,L1preg,L1p4reg
	NOT IMPLMENTED IN 32-BIT
	ENDM

IFDEF X86_64

yr8_rsc_sg8clreg_eight_complex_unfft8_preload MACRO
	ENDM
yr8_rsc_sg8clreg_eight_complex_unfft8 MACRO srcreg,srcinc,d1reg,d3reg,src4reg,dstreg,dstinc,e1,e2,e4,screg,scinc,maxrpt,L1pt,L1preg,L1p4reg
	vmovapd	ymm0, [screg+0+32]		;; cosine for R1/I1
	vmovapd	ymm1, [srcreg]			;; R1
	vmulpd	ymm2, ymm1, ymm0		;; A1 = R1 * cosine				;  1-5

	vmovapd	ymm3, [screg+0]			;; sine for R1/I1
	vmovapd	ymm4, [srcreg+32]		;; I1
	vmulpd	ymm5, ymm4, ymm3		;; C1 = I1 * sine				;  2-6

	vmovapd	ymm6, [screg+256+32]		;; cosine for R2/I2
	vmovapd	ymm7, [srcreg+d1reg]		;; R2
	vmulpd	ymm8, ymm7, ymm6		;; A2 = R2 * cosine				;  3-7

	vmovapd	ymm9, [screg+256]		;; sine for R2/I2
	vmovapd	ymm10, [srcreg+d1reg+32]	;; I2
	vmulpd	ymm11, ymm10, ymm9		;; C2 = I2 * sine				;  4-8

	vmulpd	ymm4, ymm4, ymm0		;; B1 = I1 * cosine				;  5-9
	vmovapd	ymm12, [screg+128+32]		;; cosine for R3/I3

	vmulpd	ymm1, ymm1, ymm3		;; D1 = R1 * sine				;  6-10
	vmovapd	ymm13, [srcreg+2*d1reg]		;; R3

	vaddpd	ymm2, ymm2, ymm5		;; A1 = A1 + C1 (first R1)			; 7-9
	vmulpd	ymm10, ymm10, ymm6		;; B2 = I2 * cosine				;  7-11
	vmovapd	ymm14, [screg+128]		;; sine for R3/I3

	vmulpd	ymm7, ymm7, ymm9		;; D2 = R2 * sine				;  8-12
	vmovapd	ymm15, [srcreg+2*d1reg+32]	;; I3

	vaddpd	ymm8, ymm8, ymm11		;; A2 = A2 + C2 (first R2)			; 9-11
	vmulpd	ymm11, ymm13, ymm12		;; A3 = R3 * cosine/sine			;  9-13
	vmovapd	ymm0, [screg+384+32]		;; cosine for R4/I4

	vmulpd	ymm9, ymm15, ymm14		;; C3 = I3 * sine				;  10-14
	vmovapd	ymm3, [srcreg+d3reg]		;; R4

	vsubpd	ymm4, ymm4, ymm1		;; B1 = B1 - D1 (first I1)			; 11-13
	vmulpd	ymm1, ymm3, ymm0		;; A4 = R4 * cosine/sine			;  11-15
	vmovapd	ymm5, [screg+384]		;; sine for R4/I4

	vmovapd ymm6, [srcreg+d3reg+32]		;; I4
	vsubpd	ymm10, ymm10, ymm7		;; B2 = B2 - D2 (first I2)			; 13-15
	vmulpd	ymm7, ymm6, ymm5		;; C4 = I4 * sine				;  12-16

	vmulpd	ymm15, ymm15, ymm12		;; B3 = I3 * cosine				;  13-17

	vsubpd	ymm12, ymm2, ymm8		;; R1 - R2 (new R2)				; 14-15 (or maybe 12-14)
	vmulpd	ymm13, ymm13, ymm14		;; D3 = R3 * sine				;  14-18
	vmovapd	ymm14, [screg+64+32]		;; cosine for R5/I5

	vaddpd	ymm2, ymm2, ymm8		;; R1 + R2 (new R1)				; 15-17 (or maybe 14-16)
	vmulpd	ymm6, ymm6, ymm0		;; B4 = I4 * cosine/sine			;  15-19
	vmovapd	ymm8, [src4reg]			;; R5

	vaddpd	ymm11, ymm11, ymm9		;; A3 = A3 + C3 (first R3)			; 16-18 (or maybe 15-17)
	vmulpd	ymm3, ymm3, ymm5		;; D4 = R4 * sine				;  16-20
	vmovapd	ymm0, [src4reg+32]		;; I5

	vaddpd	ymm1, ymm1, ymm7		;; A4 = A4 + C4 (first R4)			; 17-19
	vmulpd	ymm7, ymm8, ymm14		;; A5 = R5 * cosine				;  17-21
	vmovapd	ymm9, [screg+64]		;; sine for R5/I5

	vaddpd	ymm5, ymm4, ymm10		;; I1 + I2 (new I1)				; 18-20 (or maybe 16-18)
	vmulpd	ymm14, ymm0, ymm14		;; B5 = I5 * cosine				;  18-22

	vsubpd	ymm4, ymm4, ymm10		;; I1 - I2 (new I2)				; 19-21 (or maybe 18-20)
	vmulpd	ymm0, ymm0, ymm9		;; C5 = I5 * sine				;  19-23
	vmovapd	ymm10, [screg+320+32]		;; cosine for R6/I6

	vsubpd	ymm15, ymm15, ymm13		;; B3 = B3 - D3 (first I3)			; 20-22 (or maybe 19-21)
	vmulpd	ymm8, ymm8, ymm9		;; D5 = R5 * sine				;  20-24
	vmovapd	ymm13, [src4reg+d1reg]		;; R6

	ystore	[dstreg+32], ymm5		;; Save new I1					; 21 (or maybe 19)
	vsubpd	ymm6, ymm6, ymm3		;; B4 = B4 - D4 (first I4)			; 21-23
	vmulpd	ymm3, ymm13, ymm10		;; A6 = R6 * cosine				;  21-25
	vmovapd ymm9, [src4reg+d1reg+32]	;; I6

	ystore	[dstreg+e1+32], ymm4		;; Save new I2					; 22 (or maybe 21)
	vsubpd	ymm4, ymm1, ymm11		;; R4 - R3 (new I4)				; 22-24 (or maybe 20-22)
	vmulpd	ymm10, ymm9, ymm10		;; B6 = I6 * cosine				;  22-26
	vmovapd	ymm5, [screg+320]		;; sine for R6/I6

	vaddpd	ymm1, ymm1, ymm11		;; R4 + R3 (new R3)				; 23-25 (or maybe 22-24)
	vmulpd	ymm9, ymm9, ymm5		;; C6 = I6 * sine				;  23-27
	vmovapd	ymm11, [screg+192+32]		;; cosine for R7/I7

	vaddpd	ymm7, ymm7, ymm0		;; A5 = A5 + C5 (first R5)			; 24-26
	vmulpd	ymm13, ymm13, ymm5		;; D6 = R6 * sine				;  24-28
	vmovapd	ymm0, [src4reg+2*d1reg]		;; R7

	ystore	[dstreg+e2+e1+32], ymm4		;; Save new I4					; 25 (or maybe 23)
	vsubpd	ymm14, ymm14, ymm8		;; B5 = B5 - D5 (first I5)			; 25-27
	vmulpd	ymm8, ymm0, ymm11		;; A7 = R7 * cosine				;  25-29
	vmovapd	ymm5, [src4reg+2*d1reg+32]	;; I7

	vaddpd	ymm4, ymm15, ymm6		;; I3 + I4 (new I3)				; 26-28
	vmulpd	ymm11, ymm5, ymm11		;; B7 = I7 * cosine				;  26-30

	vsubpd	ymm15, ymm15, ymm6		;; I3 - I4 (new R4)				; 27-29
	vmovapd	ymm6, [screg+192]		;; sine for R7/I7
	vmulpd	ymm5, ymm5, ymm6		;; C7 = I7 * sine				;  27-31

	vaddpd	ymm3, ymm3, ymm9		;; A6 = A6 + C6 (first R6)			; 28-30
	vmulpd	ymm0, ymm0, ymm6		;; D7 = R7 * sine				;  28-32
	vmovapd	ymm9, [screg+448+32]		;; cosine for R8/I8

	vmovapd	ymm6, [src4reg+d3reg]		;; R8
	ystore	[dstreg+e2+32], ymm4		;; Save new I3					; 29
	vsubpd	ymm10, ymm10, ymm13		;; B6 = B6 - D6 (first I6)			; 29-31
	vmulpd	ymm13, ymm6, ymm9		;; A8 = R8 * cosine				;  29-33

	vsubpd	ymm4, ymm2, ymm1		;; R1 - R3 (newer R3)				; 30-32
	vaddpd	ymm2, ymm2, ymm1		;; R1 + R3 (newer R1)				; 31-33
	vmovapd ymm1, [src4reg+d3reg+32]	;; I8
	vmulpd	ymm9, ymm1, ymm9		;; B8 = I8 * cosine				;  30-34

	vaddpd	ymm8, ymm8, ymm5		;; A7 = A7 + C7 (first R7)			; 32-34
	vmovapd	ymm5, [screg+448]		;; sine for R8/I8
	vmulpd	ymm1, ymm1, ymm5		;; C8 = I8 * sine				;  31-35

	vmulpd	ymm6, ymm6, ymm5		;; D8 = R8 * sine				;  32-36
	vmovapd	ymm5, YMM_SQRTHALF

	vsubpd	ymm11, ymm11, ymm0		;; B7 = B7 - D7 (first I7)			; 33-35
	L1prefetch L1preg, L1pt

	vsubpd	ymm0, ymm7, ymm3		;; R5 - R6 (new R6)				; 34-36

	vaddpd	ymm7, ymm7, ymm3		;; R5 + R6 (new R5)				; 35-37

	vaddpd	ymm13, ymm13, ymm1		;; A8 = A8 + C8 (first R8)			; 36-38
	L1prefetch L1preg+d1reg, L1pt

	vsubpd	ymm9, ymm9, ymm6		;; B8 = B8 - D8 (first I8)			; 37-39

	vsubpd	ymm6, ymm14, ymm10		;; I5 - I6 (new I6)				; 38-40

	vaddpd	ymm14, ymm14, ymm10		;; I5 + I6 (new I5)				; 39-41
	L1prefetch L1preg+2*d1reg, L1pt

	vsubpd	ymm10, ymm13, ymm8		;; R8 - R7 (new I8)				; 40-42

	vaddpd	ymm13, ymm13, ymm8		;; R8 + R7 (new R7)				; 41-43

	vsubpd	ymm8, ymm11, ymm9		;; I7 - I8 (new R8)				; 42-44
	L1prefetch L1preg+d3reg, L1pt

	vaddpd	ymm11, ymm11, ymm9		;; I7 + I8 (new I7)				; 43-45

	vsubpd	ymm9, ymm6, ymm0		;; I6 = I6 - R6					; 44-46

	vaddpd	ymm0, ymm0, ymm6		;; R6 = R6 + I6					; 45-47
	L1prefetch L1p4reg, L1pt

	vsubpd	ymm6, ymm10, ymm8		;; I8 = I8 - R8					; 46-48

	vaddpd	ymm8, ymm8, ymm10		;; R8 = R8 + I8					; 47-49
	vmulpd	ymm9, ymm9, ymm5		;; I6 * SQRTHALF				;  47-51

	vsubpd	ymm10, ymm14, ymm11		;; I5 - I7 (newer R7)				; 48-50
	vmulpd	ymm0, ymm0, ymm5		;; R6 * SQRTHALF				;  48-52
	L1prefetch L1p4reg+d1reg, L1pt

	vaddpd	ymm14, ymm14, ymm11		;; I5 + I7 (newer I5)				; 49-51
	vmulpd	ymm6, ymm6, ymm5		;; I8 * SQRTHALF				;  49-53

	vsubpd	ymm11, ymm12, ymm15		;; R2 - R4 (newer R4)				; 50-52
	vmulpd	ymm8, ymm8, ymm5		;; R8 * SQRTHALF				;  50-54

	vaddpd	ymm12, ymm12, ymm15		;; R2 + R4 (newer R2)				; 51-53
	L1prefetch L1p4reg+2*d1reg, L1pt

	vaddpd	ymm15, ymm13, ymm7		;; R7 + R5 (newer R5)				; 52-54

	vsubpd	ymm13, ymm13, ymm7		;; R7 - R5 (newer I7)				; 53-55

	vaddpd	ymm7, ymm4, ymm10		;; R3 + R7 (final R3)				; 54-56
	L1prefetch L1p4reg+d3reg, L1pt

	vsubpd	ymm4, ymm4, ymm10		;; R3 - R7 (final R7)				; 55-57

	vsubpd	ymm10, ymm9, ymm6		;; I6 - I8 (newer R8)				; 56-58

	vaddpd	ymm9, ymm9, ymm6		;; I6 + I8 (newer I6)				; 57-59

	vaddpd	ymm6, ymm8, ymm0		;; R8 + R6 (newer R6)				; 58-60

	vsubpd	ymm8, ymm8, ymm0		;; R8 - R6 (newer I8)				; 59-61

	vaddpd	ymm0, ymm11, ymm10		;; R4 + R8 (final R4)				; 60-62

	vaddpd	ymm5, ymm2, ymm15		;; R1 + R5 (final R1)				; 61-63

	vaddpd	ymm1, ymm12, ymm6		;; R2 + R6 (final R2)				; 62-64

	vshufpd	ymm3, ymm7, ymm0, 0		;; Shuffle R3 and R4 to create R3/R4 low	; 63
	vsubpd	ymm11, ymm11, ymm10		;; R4 - R8 (final R8)				; 63-65
	vmovapd	ymm10, [dstreg+e1+32]		;; Reload new I2

	vshufpd	ymm7, ymm7, ymm0, 15		;; Shuffle R3 and R4 to create R3/R4 hi		; 64
	vsubpd	ymm2, ymm2, ymm15		;; R1 - R5 (final R5)				; 64-66
	vmovapd	ymm0, [dstreg+e2+e1+32]		;; Reload new I4

	vshufpd	ymm15, ymm5, ymm1, 0		;; Shuffle R1 and R2 to create R1/R2 low	; 65
	vsubpd	ymm12, ymm12, ymm6		;; R2 - R6 (final R6)				; 65-67

	vshufpd	ymm5, ymm5, ymm1, 15		;; Shuffle R1 and R2 to create R1/R2 hi		; 66
	vaddpd	ymm1, ymm10, ymm0		;; I2 + I4 (newer I2)				; 66-68

	ylow128s ymm6, ymm15, ymm3		;; Shuffle R1/R2 low and R3/R4 low (final R1)	; 67-68
	vsubpd	ymm10, ymm10, ymm0		;; I2 - I4 (newer I4)				; 67-69
	vmovapd	ymm0, [dstreg+32]		;; Reload new I1

	yhigh128s ymm15, ymm15, ymm3		;; Shuffle R1/R2 low and R3/R4 low (final R3)	; 68-69
	vmovapd	ymm3, [dstreg+e2+32]		;; Reload new I3
	ystore	[dstreg], ymm6			;; Save R1					; 69
	vaddpd	ymm6, ymm0, ymm3		;; I1 + I3 (newer I1)				; 68-70

	vsubpd	ymm0, ymm0, ymm3		;; I1 - I3 (newer I3)				; 69-71
	ylow128s ymm3, ymm5, ymm7		;; Shuffle R1/R2 hi and R3/R4 hi (final R2)	; 69-70

	yhigh128s ymm5, ymm5, ymm7		;; Shuffle R1/R2 hi and R3/R4 hi (final R4)	; 70-71
	vaddpd	ymm7, ymm1, ymm9		;; I2 + I6 (final I2)				; 70-72
	ystore	[dstreg+e2], ymm15		;; Save R3					; 70

	vsubpd	ymm1, ymm1, ymm9		;; I2 - I6 (final I6)				; 71-73
	ystore	[dstreg+e1], ymm3		;; Save R2					; 71

	vshufpd	ymm3, ymm2, ymm12, 0		;; Shuffle R5 and R6 to create R5/R6 low	; 72
	vaddpd	ymm9, ymm6, ymm14		;; I1 + I5 (final I1)				; 72-74
	ystore	[dstreg+e2+e1], ymm5		;; Save R4					; 72

	vshufpd	ymm2, ymm2, ymm12, 15		;; Shuffle R5 and R6 to create R5/R6 hi		; 73
	vaddpd	ymm12, ymm10, ymm8		;; I4 + I8 (final I4)				; 73-75

	vshufpd	ymm5, ymm4, ymm11, 0		;; Shuffle R7 and R8 to create R7/R8 low	; 74
	vaddpd	ymm15, ymm0, ymm13		;; I3 + I7 (final I3)				; 74-76

	vshufpd	ymm4, ymm4, ymm11, 15		;; Shuffle R7 and R8 to create R7/R8 hi		; 75
	vsubpd	ymm6, ymm6, ymm14		;; I1 - I5 (final I5)				; 75-77

	vshufpd	ymm14, ymm9, ymm7, 0		;; Shuffle I1 and I2 to create I1/I2 low	; 76
	vsubpd	ymm10, ymm10, ymm8		;; I4 - I8 (final I8)				; 76-78

	vshufpd	ymm9, ymm9, ymm7, 15		;; Shuffle I1 and I2 to create I1/I2 hi		; 77
	vsubpd	ymm0, ymm0, ymm13		;; I3 - I7 (final I7)				; 77-79

	vshufpd	ymm13, ymm15, ymm12, 0		;; Shuffle I3 and I4 to create I3/I4 low	; 78
	vshufpd	ymm15, ymm15, ymm12, 15		;; Shuffle I3 and I4 to create I3/I4 hi		; 79

	vshufpd	ymm12, ymm6, ymm1, 0		;; Shuffle I5 and I6 to create I5/I6 low
	vshufpd	ymm6, ymm6, ymm1, 15		;; Shuffle I5 and I6 to create I5/I6 hi

	vshufpd	ymm1, ymm0, ymm10, 0		;; Shuffle I7 and I8 to create I7/I8 low
	vshufpd	ymm0, ymm0, ymm10, 15		;; Shuffle I7 and I8 to create I7/I8 hi

	ylow128s ymm10, ymm3, ymm5		;; Shuffle R5/R6 low and R7/R8 low (final R5)
	yhigh128s ymm3, ymm3, ymm5		;; Shuffle R5/R6 low and R7/R8 low (final R7)
	ylow128s ymm5, ymm2, ymm4		;; Shuffle R5/R6 hi and R7/R8 hi (final R6)
	yhigh128s ymm2, ymm2, ymm4		;; Shuffle R5/R6 hi and R7/R8 hi (final R8)

	ystore	[dstreg+e4], ymm10		;; Save R5
	ystore	[dstreg+e4+e2], ymm3		;; Save R7
	ystore	[dstreg+e4+e1], ymm5		;; Save R6
	ystore	[dstreg+e4+e2+e1], ymm2		;; Save R8

	ylow128s ymm4, ymm14, ymm13		;; Shuffle I1/I2 low and I3/I4 low (final I1)
	yhigh128s ymm14, ymm14, ymm13		;; Shuffle I1/I2 low and I3/I4 low (final I3)
	ylow128s ymm13, ymm9, ymm15		;; Shuffle I1/I2 hi and I3/I4 hi (final I2)
	yhigh128s ymm9, ymm9, ymm15		;; Shuffle I1/I2 hi and I3/I4 hi (final I4)

	ystore	[dstreg+32], ymm4		;; Save I1
	ystore	[dstreg+e2+32], ymm14		;; Save I3
	ystore	[dstreg+e1+32], ymm13		;; Save I2
	ystore	[dstreg+e2+e1+32], ymm9		;; Save I4

	ylow128s ymm15, ymm12, ymm1		;; Shuffle I5/I6 low and I7/I8 low (final I5)
	yhigh128s ymm12, ymm12, ymm1		;; Shuffle I5/I6 low and I7/I8 low (final I7)
	ylow128s ymm1, ymm6, ymm0		;; Shuffle I5/I6 hi and I7/I8 hi (final I6)
	yhigh128s ymm6, ymm6, ymm0		;; Shuffle I5/I6 hi and I7/I8 hi (final I8)

	ystore	[dstreg+e4+32], ymm15		;; Save I5
	ystore	[dstreg+e4+e2+32], ymm12	;; Save I7
	ystore	[dstreg+e4+e1+32], ymm1		;; Save I6
	ystore	[dstreg+e4+e2+e1+32], ymm6	;; Save I8

	bump	srcreg, srcinc
	bump	src4reg, srcinc
	bump	dstreg, dstinc
	bump	screg, scinc
	bump	L1preg, srcinc
	bump	L1p4reg, srcinc
	ENDM

IF (@INSTR(,%yarch,<FMA3>) NE 0)

yr8_rsc_sg8clreg_eight_complex_unfft8_preload MACRO
	ENDM
yr8_rsc_sg8clreg_eight_complex_unfft8 MACRO srcreg,srcinc,d1reg,d3reg,src4reg,dstreg,dstinc,e1,e2,e4,screg,scinc,maxrpt,L1pt,L1preg,L1p4reg
	vmovapd	ymm0, [screg+0+32]		;; cosine for R1/I1
	vmovapd	ymm1, [srcreg]			;; R1
	vmovapd	ymm2, [srcreg+32]		;; I1
	vmulpd	ymm3, ymm1, ymm0		;; A1 = R1 * cosine				; 1-5		n 6
	vmulpd	ymm0, ymm2, ymm0		;; B1 = I1 * cosine				; 1-5		n 6

	vmovapd	ymm4, [screg+256+32]		;; cosine for R2/I2
	vmovapd	ymm5, [srcreg+d1reg]		;; R2
	vmovapd	ymm6, [srcreg+d1reg+32]		;; I2
	vmulpd	ymm7, ymm5, ymm4		;; A2 = R2 * cosine				; 2-6		n 7
	vmulpd	ymm4, ymm6, ymm4		;; B2 = I2 * cosine				; 2-6		n 7

	vmovapd	ymm8, [screg+64+32]		;; cosine for R5/I5
	vmovapd	ymm9, [src4reg]			;; R5
	vmovapd	ymm10, [src4reg+32]		;; I5
	vmulpd	ymm11, ymm9, ymm8		;; A5 = R5 * cosine				; 3-7		n 8
	vmulpd	ymm8, ymm10, ymm8		;; B5 = I5 * cosine				; 3-7		n 8
	vmovapd	ymm15, [screg+320+32]		;; cosine for R6/I6
	vmovapd	ymm14, [src4reg+d1reg]		;; R6
	vmovapd ymm13, [src4reg+d1reg+32]	;; I6

	vmovapd	ymm12, [screg+0]		;; sine for R1/I1
	yfnmaddpd ymm1, ymm1, ymm12, ymm0	;; B1 = B1 - R1 * sine (first I1)		; 6-10		n 14
	yfmaddpd ymm2, ymm2, ymm12, ymm3	;; A1 = A1 + I1 * sine (first R1)		; 6-10		n 15
	vmovapd	ymm0, [screg+192+32]		;; cosine for R7/I7

	vmulpd	ymm3, ymm14, ymm15		;; A6 = R6 * cosine				; 4-8		n 9
	vmulpd	ymm15, ymm13, ymm15		;; B6 = I6 * cosine				; 4-8		n 9

	vmovapd	ymm12, [screg+256]		;; sine for R2/I2
	yfnmaddpd ymm5, ymm5, ymm12, ymm4	;; B2 = B2 - R2 * sine (first I2)		; 7-11		n 14
	yfmaddpd ymm6, ymm6, ymm12, ymm7	;; A2 = A2 + I2 * sine (first R2)		; 7-11		n 15
	vmovapd	ymm4, [src4reg+2*d1reg]		;; R7
	vmovapd	ymm12, [src4reg+2*d1reg+32]	;; I7

	vmovapd	ymm7, [screg+64]		;; sine for R5/I5
	yfmaddpd ymm10, ymm10, ymm7, ymm11	;; A5 = A5 + I5 * sine (first R5)		; 8-12		n 19
	yfnmaddpd ymm9, ymm9, ymm7, ymm8	;; B5 = B5 - R5 * sine (first I5)		; 8-12		n 20

	vmulpd	ymm11, ymm4, ymm0		;; A7 = R7 * cosine				; 5-9		n 10
	vmulpd	ymm0, ymm12, ymm0		;; B7 = I7 * cosine				; 5-9		n 10

	vmovapd	ymm7, [screg+320]		;; sine for R6/I6
	yfmaddpd ymm13, ymm13, ymm7, ymm3	;; A6 = A6 + I6 * sine (first R6)		; 9-13		n 19
	yfnmaddpd ymm14, ymm14, ymm7, ymm15	;; B6 = B6 - R6 * sine (first I6)		; 9-13		n 20

	vmovapd	ymm15, [screg+192]		;; sine for R7/I7
	yfmaddpd ymm12, ymm12, ymm15, ymm11	;; A7 = A7 + I7 * sine (first R7)		; 10-14		n 21
	yfnmaddpd ymm4, ymm4, ymm15, ymm0	;; B7 = B7 - R7 * sine (first I7)		; 10-14		n 22

	vmovapd	ymm8, [screg+448+32]		;; cosine for R8/I8
	vmovapd	ymm3, [src4reg+d3reg]		;; R8
	vmovapd ymm7, [src4reg+d3reg+32]	;; I8
	vmulpd	ymm0, ymm3, ymm8		;; A8 = R8 * cosine				; 11-15		n 16
	vmulpd	ymm8, ymm7, ymm8		;; B8 = I8 * cosine				; 11-15		n 16

	vmovapd	ymm11, [screg+128+32]		;; cosine for R3/I3

	vmovapd	ymm15, [screg+448]		;; sine for R8/I8
	yfmaddpd ymm7, ymm7, ymm15, ymm0	;; A8 = A8 + I8 * sine (first R8)		; 16-20		n 21
	yfnmaddpd ymm3, ymm3, ymm15, ymm8	;; B8 = B8 - R8 * sine (first I8)		; 16-20		n 22

	vmovapd	ymm0, [srcreg+2*d1reg]		;; R3
	vmovapd	ymm15, [srcreg+2*d1reg+32]	;; I3
	vmulpd	ymm8, ymm0, ymm11		;; A3 = R3 * cosine				; 12-16		n 17
	vmulpd	ymm11, ymm15, ymm11		;; B3 = I3 * cosine				; 12-16		n 17

	yfmaddpd ymm15, ymm15, [screg+128], ymm8 ;; A3 = A3 + I3 * sine (first R3)		; 17-21		n 23
	yfnmaddpd ymm0, ymm0, [screg+128], ymm11 ;; B3 = B3 - R3 * sine (first I3)		; 17-21		n 24

	vmovapd ymm8, YMM_ONE
	yfmsubpd ymm11, ymm1, ymm8, ymm5	;; I1 - I2 (new I2)				; 14-18		n 31
	yfmaddpd ymm1, ymm1, ymm8, ymm5		;; I1 + I2 (new I1)				; 14-18		n 35

	yfmaddpd ymm5, ymm2, ymm8, ymm6		;; R1 + R2 (new R1)				; 15-19		n 29
	yfmsubpd ymm2, ymm2, ymm8, ymm6		;; R1 - R2 (new R2)				; 15-19		n 30
	vmovapd	ymm8, [screg+384+32]		;; cosine for R4/I4
	vmovapd	ymm6, [srcreg+d3reg]		;; R4

	ystore	[dstreg+32], ymm11		;; Temp save new I2				; 19
	vmovapd ymm11, [srcreg+d3reg+32]	;; I4
	ystore	[dstreg], ymm1			;; Temp save new I1				; 19+1
	vmulpd	ymm1, ymm6, ymm8		;; A4 = R4 * cosine				; 13-17		n 18
	vmulpd	ymm8, ymm11, ymm8		;; B4 = I4 * cosine				; 13-17		n 18

	yfmaddpd ymm11, ymm11, [screg+384], ymm1 ;; A4 = A4 + I4 * sine (first R4)		; 18-22		n 23
	yfnmaddpd ymm6, ymm6, [screg+384], ymm8	;; B4 = B4 - R4 * sine (first I4)		; 18-22		n 24

	vmovapd ymm8, YMM_ONE
	yfmsubpd ymm1, ymm10, ymm8, ymm13	;; R5 - R6 (new R6)				; 19-23		n 25
	yfmaddpd ymm10, ymm10, ymm8, ymm13	;; R5 + R6 (new R5)				; 19-23		n 26
	L1prefetch L1preg, L1pt

	yfmsubpd ymm13, ymm9, ymm8, ymm14	;; I5 - I6 (new I6)				; 20-24		n 25
	yfmaddpd ymm9, ymm9, ymm8, ymm14	;; I5 + I6 (new I5)				; 20-24		n 28

	yfmaddpd ymm14, ymm7, ymm8, ymm12	;; R8 + R7 (new R7)				; 21-25		n 26
	yfmsubpd ymm7, ymm7, ymm8, ymm12	;; R8 - R7 (new I8)				; 21-25		n 27
	L1prefetch L1preg+d1reg, L1pt

	yfmsubpd ymm12, ymm4, ymm8, ymm3	;; I7 - I8 (new R8)				; 22-26		n 27
	yfmaddpd ymm4, ymm4, ymm8, ymm3		;; I7 + I8 (new I7)				; 22-26		n 28

	yfmaddpd ymm3, ymm11, ymm8, ymm15	;; R4 + R3 (new R3)				; 23-27		n 29
	yfmsubpd ymm11, ymm11, ymm8, ymm15	;; R4 - R3 (new I4)				; 23-27		n 31
	L1prefetch L1preg+2*d1reg, L1pt

	yfmsubpd ymm15, ymm0, ymm8, ymm6	;; I3 - I4 (new R4)				; 24-28		n 30
	yfmaddpd ymm0, ymm0, ymm8, ymm6		;; I3 + I4 (new I3)				; 24-28		n 35

	yfmsubpd ymm6, ymm13, ymm8, ymm1	;; I6 - R6 (new2 I6)				; 25-29		n 32
	yfmaddpd ymm1, ymm1, ymm8, ymm13	;; R6 + I6 (new2 R6)				; 25-29		n 33
	L1prefetch L1preg+d3reg, L1pt

	yfmaddpd ymm13, ymm14, ymm8, ymm10	;; R7 + R5 (newer R5)				; 26-30		n 34
	yfmsubpd ymm14, ymm14, ymm8, ymm10	;; R7 - R5 (newer I7)				; 26-30		n 42

	yfmsubpd ymm10, ymm7, ymm8, ymm12	;; I8 - R8 (new2 I8)				; 27-31		n 32
	yfmaddpd ymm12, ymm12, ymm8, ymm7	;; R8 + I8 (new2 R8)				; 27-31		n 33
	L1prefetch L1p4reg, L1pt

	yfmsubpd ymm7, ymm9, ymm8, ymm4		;; I5 - I7 (newer R7)				; 28-32		n 36
	yfmaddpd ymm9, ymm9, ymm8, ymm4		;; I5 + I7 (newer I5)				; 28-32		n 41

	yfmsubpd ymm4, ymm5, ymm8, ymm3		;; R1 - R3 (newer R3)				; 29-33		n 36
	yfmaddpd ymm5, ymm5, ymm8, ymm3		;; R1 + R3 (newer R1)				; 29-33		n 34
	L1prefetch L1p4reg+d1reg, L1pt

	yfmsubpd ymm3, ymm2, ymm8, ymm15	;; R2 - R4 (newer R4)				; 30-34		n 37
	yfmaddpd ymm2, ymm2, ymm8, ymm15	;; R2 + R4 (newer R2)				; 30-34		n 38

	vmovapd	ymm15, [dstreg+32]		;; Reload new I2
	ystore	[dstreg+e1], ymm14		;; Temp save newer I7				; 31
	yfmaddpd ymm14, ymm15, ymm8, ymm11	;; I2 + I4 (newer I2)				; 31-35		n 39
	yfmsubpd ymm15, ymm15, ymm8, ymm11	;; I2 - I4 (newer I4)				; 31-35		n 40

	yfmsubpd ymm11, ymm6, ymm8, ymm10	;; I6 - I8 (newer R8/SQRTHALF)			; 32-36		n 37
	yfmaddpd ymm6, ymm6, ymm8, ymm10	;; I6 + I8 (newer I6/SQRTHALF)			; 32-36		n 39

	ystore	[dstreg+e1+32], ymm9		;; Temp save newer I5				; 33
	yfmaddpd ymm10, ymm12, ymm8, ymm1	;; R8 + R6 (newer R6/SQRTHALF)			; 33-37		n 38
	yfmsubpd ymm12, ymm12, ymm8, ymm1	;; R8 - R6 (newer I8/SQRTHALF)			; 33-37		n 40
	vmovapd	ymm9, [dstreg]			;; Reload new I1

	yfmaddpd ymm1, ymm5, ymm8, ymm13	;; R1 + R5 (final R1)				; 34-38		n 44
	yfmsubpd ymm5, ymm5, ymm8, ymm13	;; R1 - R5 (final R5)				; 34-38		n 46
	bump	screg, scinc

	yfmaddpd ymm13, ymm9, ymm8, ymm0	;; I1 + I3 (newer I1)				; 35-39		n 41
	yfmsubpd ymm9, ymm9, ymm8, ymm0		;; I1 - I3 (newer I3)				; 35-39		n 42
	L1prefetch L1p4reg+2*d1reg, L1pt

	yfmaddpd ymm0, ymm4, ymm8, ymm7		;; R3 + R7 (final R3)				; 36-40		n 42
	yfmsubpd ymm4, ymm4, ymm8, ymm7		;; R3 - R7 (final R7)				; 36-40		n 48

	vmovapd	ymm8, YMM_SQRTHALF
	yfmaddpd ymm7, ymm11, ymm8, ymm3	;; R4 + R8 * SQRTHALF (final R4)		; 37-41		n 42
	yfnmaddpd ymm11, ymm11, ymm8, ymm3	;; R4 - R8 * SQRTHALF (final R8)		; 37-41		n 48

	yfmaddpd ymm3, ymm10, ymm8, ymm2	;; R2 + R6 * SQRTHALF (final R2)		; 38-42		n 44
	yfnmaddpd ymm10, ymm10, ymm8, ymm2	;; R2 - R6 * SQRTHALF (final R6)		; 38-42		n 46
	L1prefetch L1p4reg+d3reg, L1pt

	yfmaddpd ymm2, ymm6, ymm8, ymm14	;; I2 + I6 * SQRTHALF (final I2)		; 39-43		n 50
	yfnmaddpd ymm6, ymm6, ymm8, ymm14	;; I2 - I6 * SQRTHALF (final I6)		; 39-43		n 52

	yfmaddpd ymm14, ymm12, ymm8, ymm15	;; I4 + I8 * SQRTHALF (final I4)		; 40-44		n 54
	yfnmaddpd ymm12, ymm12, ymm8, ymm15	;; I4 - I8 * SQRTHALF (final I8)		; 40-44		n 56
	vmovapd	ymm8, YMM_ONE

	vshufpd	ymm15, ymm0, ymm7, 0		;; Shuffle R3 and R4 to create R3/R4 low	; 42
	vshufpd	ymm0, ymm0, ymm7, 15		;; Shuffle R3 and R4 to create R3/R4 hi		; 43
	vmovapd	ymm7, [dstreg+e1+32]		;; Reload newer I5

	ystorelo [dstreg][16], ymm15		;; Save R1					; 43
	ystorehi [dstreg+e2][16], ymm15		;; Save R3					; 43+1

	yfmaddpd ymm15, ymm13, ymm8, ymm7	;; I1 + I5 (final I1)				; 41-45		n 50
	yfmsubpd ymm13, ymm13, ymm8, ymm7	;; I1 - I5 (final I5)				; 41-45		n 52
	bump	srcreg, srcinc

	vshufpd	ymm7, ymm1, ymm3, 0		;; Shuffle R1 and R2 to create R1/R2 low	; 44
	vshufpd	ymm1, ymm1, ymm3, 15		;; Shuffle R1 and R2 to create R1/R2 hi		; 45
	vmovapd	ymm3, [dstreg+e1]		;; Reload newer I7

	ystorelo [dstreg+e1][16], ymm0		;; Save R2					; 44+1
	ystorehi [dstreg+e2+e1][16], ymm0	;; Save R4					; 44+2

	yfmaddpd ymm0, ymm9, ymm8, ymm3		;; I3 + I7 (final I3)				; 42-46		n 54
	yfmsubpd ymm9, ymm9, ymm8, ymm3		;; I3 - I7 (final I7)				; 42-46		n 56
	bump	src4reg, srcinc

	vshufpd	ymm3, ymm5, ymm10, 0		;; Shuffle R5 and R6 to create R5/R6 low	; 46
	vshufpd	ymm5, ymm5, ymm10, 15		;; Shuffle R5 and R6 to create R5/R6 hi		; 47

	ystorelo [dstreg], ymm7			;; Save R1					; 45+2
	ystorehi [dstreg+e2], ymm7		;; Save R3					; 45+3

	vshufpd	ymm10, ymm4, ymm11, 0		;; Shuffle R7 and R8 to create R7/R8 low	; 48
	vshufpd	ymm4, ymm4, ymm11, 15		;; Shuffle R7 and R8 to create R7/R8 hi		; 49

	ystorelo [dstreg+e1], ymm1		;; Save R2					; 46+3
	ystorehi [dstreg+e2+e1], ymm1		;; Save R4					; 46+4

	vshufpd	ymm11, ymm15, ymm2, 0		;; Shuffle I1 and I2 to create I1/I2 low	; 50
	vshufpd	ymm15, ymm15, ymm2, 15		;; Shuffle I1 and I2 to create I1/I2 hi		; 51

	ystorelo [dstreg+e4], ymm3		;; Save R5					; 47+4
	ystorehi [dstreg+e4+e2], ymm3		;; Save R7					; 47+5

	vshufpd	ymm2, ymm13, ymm6, 0		;; Shuffle I5 and I6 to create I5/I6 low	; 52
	vshufpd	ymm13, ymm13, ymm6, 15		;; Shuffle I5 and I6 to create I5/I6 hi		; 53

	ystorelo [dstreg+e4+e1], ymm5		;; Save R6					; 48+5
	ystorehi [dstreg+e4+e2+e1], ymm5	;; Save R8					; 48+6

	vshufpd	ymm6, ymm0, ymm14, 0		;; Shuffle I3 and I4 to create I3/I4 low	; 54
	vshufpd	ymm0, ymm0, ymm14, 15		;; Shuffle I3 and I4 to create I3/I4 hi		; 55

	ystorelo [dstreg+e4][16], ymm10		;; Save R5					; 49+6
	ystorehi [dstreg+e4+e2][16], ymm10	;; Save R7					; 49+7

	vshufpd	ymm14, ymm9, ymm12, 0		;; Shuffle I7 and I8 to create I7/I8 low	; 56
	vshufpd	ymm9, ymm9, ymm12, 15		;; Shuffle I7 and I8 to create I7/I8 hi		; 57

	ystorelo [dstreg+e4+e1][16], ymm4	;; Save R6					; 50+7
	ystorehi [dstreg+e4+e2+e1][16], ymm4	;; Save R8					; 50+8

	ylow128s ymm12, ymm11, ymm6		;; Shuffle I1/I2 low and I3/I4 low (final I1)	; 58-59
	yhigh128s ymm11, ymm11, ymm6		;; Shuffle I1/I2 low and I3/I4 low (final I3)	; 59-60
	ylow128s ymm6, ymm15, ymm0		;; Shuffle I1/I2 hi and I3/I4 hi (final I2)	; 60-61
	ystore	[dstreg+32], ymm12		;; Save I1					; 61
	yhigh128s ymm15, ymm15, ymm0		;; Shuffle I1/I2 hi and I3/I4 hi (final I4)	; 61-62
	ystore	[dstreg+e2+32], ymm11		;; Save I3					; 62

	ylow128s ymm0, ymm2, ymm14		;; Shuffle I5/I6 low and I7/I8 low (final I5)	; 62-63
	ystore	[dstreg+e1+32], ymm6		;; Save I2					; 63
	yhigh128s ymm2, ymm2, ymm14		;; Shuffle I5/I6 low and I7/I8 low (final I7)	; 63-64
	ystore	[dstreg+e2+e1+32], ymm15	;; Save I4					; 64
	ylow128s ymm14, ymm13, ymm9		;; Shuffle I5/I6 hi and I7/I8 hi (final I6)	; 64-65
	ystore	[dstreg+e4+32], ymm0		;; Save I5					; 65
	yhigh128s ymm13, ymm13, ymm9		;; Shuffle I5/I6 hi and I7/I8 hi (final I8)	; 65-66
	ystore	[dstreg+e4+e2+32], ymm2		;; Save I7					; 66
	ystore	[dstreg+e4+e1+32], ymm14	;; Save I6					; 67
	ystore	[dstreg+e4+e2+e1+32], ymm13	;; Save I8					; 68

	bump	dstreg, dstinc
	bump	L1preg, srcinc
	bump	L1p4reg, srcinc
	ENDM

ENDIF

ENDIF



yr8_rsc_sg8cl_2sc_sixteen_reals_fft8_preload MACRO
	ENDM
yr8_rsc_sg8cl_2sc_sixteen_reals_fft8 MACRO srcreg,srcinc,d1,d2,d4,dstreg,dstinc,e1,e2,e4,screg1,scinc1,screg2,scinc2,maxrpt,L1pt,L1pd

	vmovapd	ymm1, [srcreg]			;; R1
	vmovapd	ymm7, [srcreg+d1]		;; R2
	vshufpd	ymm0, ymm1, ymm7, 15		;; Shuffle R1 and R2 to create R1/R2 hi
	vshufpd	ymm1, ymm1, ymm7, 0		;; Shuffle R1 and R2 to create R1/R2 low

	vmovapd	ymm2, [srcreg+d2]		;; R3
	vmovapd	ymm7, [srcreg+d2+d1]		;; R4
	vshufpd	ymm3, ymm2, ymm7, 15		;; Shuffle R3 and R4 to create R3/R4 hi
	vshufpd	ymm2, ymm2, ymm7, 0		;; Shuffle R3 and R4 to create R3/R4 low

	ylow128s ymm4, ymm0, ymm3		;; Shuffle R1/R2 hi and R3/R4 hi (first R2)
	yhigh128s ymm0, ymm0, ymm3		;; Shuffle R1/R2 hi and R3/R4 hi (first R4)

	ylow128s ymm3, ymm1, ymm2		;; Shuffle R1/R2 low and R3/R4 low (first R1)
	yhigh128s ymm1, ymm1, ymm2		;; Shuffle R1/R2 low and R3/R4 low (first R3)

	vmovapd	ymm2, [srcreg+32]		;; I1
	vmovapd	ymm7, [srcreg+d1+32]		;; I2
	vshufpd	ymm5, ymm2, ymm7, 15		;; Shuffle I1 and I2 to create I1/I2 hi
	vshufpd	ymm2, ymm2, ymm7, 0		;; Shuffle I1 and I2 to create I1/I2 low

	vmovapd	ymm6, [srcreg+d2+32]		;; I3
	vmovapd	ymm7, [srcreg+d2+d1+32]		;; I4
	ystore	[dstreg], ymm0			;; Save first R4
	vshufpd	ymm0, ymm6, ymm7, 15		;; Shuffle I3 and I4 to create I3/I4 hi
	vshufpd	ymm6, ymm6, ymm7, 0		;; Shuffle I3 and I4 to create I3/I4 low

	ylow128s ymm7, ymm5, ymm0		;; Shuffle I1/I2 hi and I3/I4 hi (first R10)
	yhigh128s ymm5, ymm5, ymm0		;; Shuffle I1/I2 hi and I3/I4 hi (first R12)

	ylow128s ymm0, ymm2, ymm6		;; Shuffle I1/I2 low and I3/I4 low (first R9)
	yhigh128s ymm2, ymm2, ymm6		;; Shuffle I1/I2 low and I3/I4 low (first R11)

	L1prefetch srcreg+L1pd, L1pt
	L1prefetchw dstreg+L1pd, L1pt - L1PREFETCH_DEST_NONE

	vaddpd	ymm6, ymm4, ymm7		;; R2 + R10 (new R2)
	vsubpd	ymm4, ymm4, ymm7		;; R2 - R10 (new R10)

	vaddpd	ymm7, ymm3, ymm0		;; R1 + R9 (new R1)
	vsubpd	ymm3, ymm3, ymm0		;; R1 - R9 (new R9)

	L1prefetch srcreg+d1+L1pd, L1pt
	L1prefetchw dstreg+e1+L1pd, L1pt - L1PREFETCH_DEST_NONE

	vaddpd	ymm0, ymm1, ymm2		;; R3 + R11 (new R3)
	vsubpd	ymm1, ymm1, ymm2		;; R3 - R11 (new R11)

	vmovapd	ymm2, [dstreg]			;; Reload first R4
	ystore	[dstreg+e1], ymm6		;; Save new R2
	vaddpd	ymm6, ymm2, ymm5		;; R4 + R12 (new R4)
	vsubpd	ymm2, ymm2, ymm5		;; R4 - R12 (new R12)

	vmovapd	ymm5, [srcreg+d4]		;; R5
	ystore	[dstreg+e4+e1], ymm4		;; Save new R10
	vmovapd	ymm4, [srcreg+d4+d1]		;; R6
	ystore	[dstreg], ymm7			;; Save new R1
	vshufpd	ymm7, ymm5, ymm4, 15		;; Shuffle R5 and R6 to create R5/R6 hi
	vshufpd	ymm5, ymm5, ymm4, 0		;; Shuffle R5 and R6 to create R5/R6 low

	vmovapd	ymm4, [srcreg+d4+d2]		;; R7
	ystore	[dstreg+e4], ymm3		;; Save new R9
	vmovapd	ymm3, [srcreg+d4+d2+d1]		;; R8
	ystore	[dstreg+e2], ymm0		;; Save new R3
	vshufpd	ymm0, ymm4, ymm3, 15		;; Shuffle R7 and R8 to create R7/R8 hi
	vshufpd	ymm4, ymm4, ymm3, 0		;; Shuffle R7 and R8 to create R7/R8 low

	ylow128s ymm3, ymm7, ymm0		;; Shuffle R5/R6 hi and R7/R8 hi (first R6)
	yhigh128s ymm7, ymm7, ymm0		;; Shuffle R5/R6 hi and R7/R8 hi (first R8)

	ylow128s ymm0, ymm5, ymm4		;; Shuffle R5/R6 low and R7/R8 low (first R5)
	yhigh128s ymm5, ymm5, ymm4		;; Shuffle R5/R6 low and R7/R8 low (first R7)

	vmovapd	ymm4, [srcreg+d4+32]		;; I5
	ystore	[dstreg+e4+e2], ymm1		;; Save new R11
	vmovapd	ymm1, [srcreg+d4+d1+32]		;; I6
	ystore	[dstreg+e2+e1], ymm6		;; Save new R4
	vshufpd	ymm6, ymm4, ymm1, 15		;; Shuffle I5 and I6 to create I5/I6 hi
	vshufpd	ymm4, ymm4, ymm1, 0		;; Shuffle I5 and I6 to create I5/I6 low

	vmovapd	ymm1, [srcreg+d4+d2+32]		;; I7
	ystore	[dstreg+e4+e2+e1], ymm2		;; Save new R12
	vmovapd	ymm2, [srcreg+d4+d2+d1+32]	;; I8
	ystore	[dstreg+32], ymm7		;; Save first R8
	vshufpd	ymm7, ymm1, ymm2, 15		;; Shuffle I7 and I8 to create I7/I8 hi
	vshufpd	ymm1, ymm1, ymm2, 0		;; Shuffle I7 and I8 to create I7/I8 low

	ylow128s ymm2, ymm6, ymm7		;; Shuffle I5/I6 hi and I7/I8 hi (first R14)
	yhigh128s ymm6, ymm6, ymm7		;; Shuffle I5/I6 hi and I7/I8 hi (first R16)

	ylow128s ymm7, ymm4, ymm1		;; Shuffle I5/I6 low and I7/I8 low (first R13)
	yhigh128s ymm4, ymm4, ymm1		;; Shuffle I5/I6 low and I7/I8 low (first R15)

	L1prefetch srcreg+d2+L1pd, L1pt
	L1prefetchw dstreg+e2+L1pd, L1pt - L1PREFETCH_DEST_NONE

	vaddpd	ymm1, ymm3, ymm2		;; R6 + R14 (new R6)
	vsubpd	ymm3, ymm3, ymm2		;; R6 - R14 (new R14)

	vaddpd	ymm2, ymm0, ymm7		;; R5 + R13 (new R5)
	vsubpd	ymm0, ymm0, ymm7		;; R5 - R13 (new R13)

	L1prefetch srcreg+d2+d1+L1pd, L1pt
	L1prefetchw dstreg+e2+e1+L1pd, L1pt - L1PREFETCH_DEST_NONE

	vaddpd	ymm7, ymm5, ymm4		;; R7 + R15 (new R7)
	vsubpd	ymm5, ymm5, ymm4		;; R7 - R15 (new R15)

	vmovapd	ymm4, [dstreg+32]		;; Reload first R8
	ystore	[dstreg+e4+e1+32], ymm3		;; Save new R14
	vaddpd	ymm3, ymm4, ymm6		;; R8 + R16 (new R8)
	vsubpd	ymm4, ymm4, ymm6		;; R8 - R16 (new R16)

	;; Even levels 2

	vmovapd	ymm6, [dstreg+e2+e1]		;; Reload new R4
	ystore	[dstreg+e4+32], ymm0		;; Save new R13
	vaddpd	ymm0, ymm6, ymm3		;; R4 + R8 (newer R4)
	vsubpd	ymm6, ymm6, ymm3		;; R4 - R8 (newer R8)

	vmovapd	ymm3, [dstreg+e1]		;; Reload new R2
	ystore	[dstreg+e4+e2+32], ymm5		;; Save new R15
	vaddpd	ymm5, ymm3, ymm1		;; R2 + R6 (newer R2)
	vsubpd	ymm3, ymm3, ymm1		;; R2 - R6 (newer R6)

	;; Even level 3

	L1prefetch srcreg+d4+L1pd, L1pt
	L1prefetchw dstreg+e4+L1pd, L1pt - L1PREFETCH_DEST_NONE

	vaddpd	ymm1, ymm5, ymm0		;; R2 + R4 (newest R2)
	vsubpd	ymm5, ymm5, ymm0		;; R2 - R4 (newest R4)

						;; R6/R8 morphs into newer R6/I6

	;; Premultipliers for even level 4

						;; mul R6/I6 by w^2 = .707 + .707i
	vsubpd	ymm0, ymm3, ymm6		;; R6 = R6 - I6
	vaddpd	ymm3, ymm3, ymm6		;; I6 = R6 + I6
	vmulpd	ymm0, ymm0, YMM_SQRTHALF	;; R6 = R6 * SQRTHALF (newest R6)
	vmulpd	ymm3, ymm3, YMM_SQRTHALF	;; I6 = I6 * SQRTHALF (newest I6)

	;; Odd levels 2

	vmovapd	ymm6, [dstreg]			;; Reload new R1
	ystore	[dstreg+e4+e2+e1+32], ymm4	;; Save new R16
	vsubpd	ymm4, ymm6, ymm2		;; R1 - R5 (newer R5)
	vaddpd	ymm6, ymm6, ymm2		;; R1 + R5 (newer R1)

	vmovapd	ymm2, [dstreg+e2]		;; Reload new R3
	ystore	[dstreg+e2+e1], ymm5		;; Save newest R4
	vaddpd	ymm5, ymm2, ymm7		;; R3 + R7 (newer R3)
	vsubpd	ymm2, ymm2, ymm7		;; R3 - R7 (newer R7)

	;; Odd level 3

	L1prefetch srcreg+d4+d1+L1pd, L1pt
	L1prefetchw dstreg+e4+e1+L1pd, L1pt - L1PREFETCH_DEST_NONE

 	vaddpd	ymm7, ymm6, ymm5		;; R1 + R3 (newest R1)
	vsubpd	ymm6, ymm6, ymm5		;; R1 - R3 (newest R3)

						;; R5/R7 morphs into newest R5/I5

	;; Last level

						;; R1/R2 becomes final R1 and final R2

						;; R3/R4 morphs into R3/I3

	ystore	[dstreg+32], ymm1		;; Save R2
	ystore	[dstreg], ymm7			;; Save R1

	vmovapd	ymm7, [screg1+128+32]		;; cosine for w^4 (8-complex w^2)
	vmulpd	ymm5, ymm6, ymm7		;; A3 = R3 * cosine
	vmovapd	ymm1, [dstreg+e2+e1]		;; Reload newest R4 which morped into I3
	vmulpd	ymm7, ymm1, ymm7		;; B3 = I3 * cosine
	vmulpd	ymm1, ymm1, [screg1+128]	;; C3 = I3 * sine
	vmulpd	ymm6, ymm6, [screg1+128]	;; D3 = R3 * sine
	vsubpd	ymm5, ymm5, ymm1		;; A3 = A3 - C3 (final R3)
	vaddpd	ymm7, ymm7, ymm6		;; B3 = B3 + D3 (final I3)

	vsubpd	ymm1, ymm4, ymm0		;; R5 - R6 (final R6)
	vaddpd	ymm4, ymm4, ymm0		;; R5 + R6 (final R5)

	L1prefetch srcreg+d4+d2+L1pd, L1pt
	L1prefetchw dstreg+e4+e2+L1pd, L1pt - L1PREFETCH_DEST_NONE

	vsubpd	ymm0, ymm2, ymm3		;; I5 - I6 (final I6)
	vaddpd	ymm2, ymm2, ymm3		;; I5 + I6 (final I5)

	vmovapd	ymm3, [screg1+64+32]		;; cosine for w^2 (8-complex w^1)
	vmulpd	ymm6, ymm4, ymm3		;; A5 = R5 * cosine
	vmulpd	ymm3, ymm2, ymm3		;; B5 = I5 * cosine
	vmulpd	ymm2, ymm2, [screg1+64]		;; C5 = I5 * sine
	vmulpd	ymm4, ymm4, [screg1+64]		;; D5 = R5 * sine
	vsubpd	ymm6, ymm6, ymm2		;; A5 = A5 - C5 (final R5)
	vaddpd	ymm2, ymm3, ymm4		;; B5 = B5 + D5 (final I5)

	vmovapd	ymm3, [screg1+320+32]		;; cosine for w^10 (8-complex w^5)
	vmulpd	ymm4, ymm1, ymm3		;; A6 = R6 * cosine
	vmulpd	ymm3, ymm0, ymm3		;; B6 = I6 * cosine
	vmulpd	ymm0, ymm0, [screg1+320]	;; C6 = I6 * sine
	vmulpd	ymm1, ymm1, [screg1+320]	;; D6 = R6 * sine
	vsubpd	ymm4, ymm4, ymm0		;; A6 = A6 - C6 (final R6)
	vaddpd	ymm0, ymm3, ymm1		;; B6 = B6 + D6 (final I6)

	;; Odd levels 2

						;; R9/R13 morphs into newer R9/I9
						;; R11/R15 morphs into newer R11/I11

	;; Even levels 2

						;; R10/R14 morphs into newer R10/I10
						;; R12/R16 morphs into newer R12/I12

	;; Premultipliers for even level 3

						;; mul R10/I10 by w^1 = .924 + .383i
	vmovapd ymm1, [dstreg+e4+e1]		;; Reload new R10
	vmovapd	ymm3, YMM_P924
	ystore	[dstreg+e1], ymm5		;; Save R3
	vmulpd	ymm5, ymm1, ymm3		;; R10 * .924
	ystore	[dstreg+e1+32], ymm7		;; Save I3
	vmovapd	ymm7, YMM_P383
	vmulpd	ymm1, ymm1, ymm7		;; R10 * .383
	ystore	[dstreg+e2+e1], ymm4		;; Save R6
	vmovapd ymm4, [dstreg+e4+e1+32]		;; Reload new R14 which morped into I10
	ystore	[dstreg+e2], ymm6		;; Save R5
	vmulpd	ymm6, ymm4, ymm7		;; I10 * .383
	vmulpd	ymm4, ymm4, ymm3		;; I10 * .924
	vsubpd	ymm5, ymm5, ymm6		;; Twiddled R10
	vaddpd	ymm1, ymm1, ymm4		;; Twiddled I10

						;; mul R12/I12 by w^3 = .383 + .924i
	vmovapd ymm6, [dstreg+e4+e2+e1]		;; Reload new R12
	vmulpd	ymm4, ymm6, ymm7		;; R12 * .383
	vmulpd	ymm6, ymm6, ymm3		;; R12 * .924
	ystore	[dstreg+e2+e1+32], ymm0		;; Save I6
	vmovapd ymm0, [dstreg+e4+e2+e1+32]	;; Reload new R16 which morped into I12
	vmulpd	ymm3, ymm0, ymm3		;; I12 * .924
	vmulpd	ymm0, ymm0, ymm7		;; I12 * .383
	vsubpd	ymm4, ymm4, ymm3		;; Twiddled R12
	vaddpd	ymm6, ymm6, ymm0		;; Twiddled I12

	;; Even level 3

	vaddpd	ymm7, ymm5, ymm4		;; R10 + R12 (newest R10)
	vsubpd	ymm5, ymm5, ymm4		;; R10 - R12 (newest R12)

	L1prefetch srcreg+d4+d2+d1+L1pd, L1pt
	L1prefetchw dstreg+e4+e2+e1+L1pd, L1pt - L1PREFETCH_DEST_NONE

	vaddpd	ymm3, ymm1, ymm6		;; I10 + I12 (newest I10)
	vsubpd	ymm1, ymm1, ymm6		;; I10 - I12 (newest I12)

	;; Premultipliers for odd level 3

						;; mul R11/I11 by SQRTHALF + i*SQRTHALF
	vmovapd	ymm0, [dstreg+e4+e2]		;; Reload new R11
	vmovapd ymm4, [dstreg+e4+e2+32]		;; Reload new R15 which morped into I11
	vsubpd	ymm6, ymm0, ymm4		;; R11 = R11 - I11
	vaddpd	ymm0, ymm0, ymm4		;; I11 = R11 + I11
	vmulpd	ymm6, ymm6, YMM_SQRTHALF	;; R11 = R11 * SQRTHALF
	vmulpd	ymm0, ymm0, YMM_SQRTHALF	;; I11 = I11 * SQRTHALF

	;; Odd level 3

	vmovapd	ymm4, [dstreg+e4]		;; Reload new R9
	ystore	[dstreg+e2+32], ymm2		;; Save I5
	vaddpd	ymm2, ymm4, ymm6		;; R9 + R11 (newest R9)
	vsubpd	ymm4, ymm4, ymm6		;; R9 - R11 (newest R11)

	vmovapd ymm6, [dstreg+e4+32]		;; Reload new R13 which morped into I9
	ystore	[dstreg+e4+e2+e1+32], ymm1	;; Save newest I12
	vaddpd	ymm1, ymm6, ymm0		;; I9 + I11 (newest I9)
	vsubpd	ymm6, ymm6, ymm0		;; I9 - I11 (newest I11)

	;; Last level

	vsubpd	ymm0, ymm2, ymm7		;; R9 - R10 (final R10)
	vaddpd	ymm2, ymm2, ymm7		;; R9 + R10 (final R9)

	vsubpd	ymm7, ymm1, ymm3		;; I9 - I10 (final I10)
	vaddpd	ymm1, ymm1, ymm3		;; I9 + I10 (final I9)

	vmovapd	ymm3, [screg2+0+32]		;; cosine/sine for w^1
	ystore	[dstreg+e4+e2+e1], ymm5		;; Save newest R12
	vmulpd	ymm5, ymm2, ymm3		;; A9 = R9 * cosine/sine
	vsubpd	ymm5, ymm5, ymm1		;; A9 = A9 - I9
	vmulpd	ymm1, ymm1, ymm3		;; B9 = I9 * cosine/sine
	vaddpd	ymm1, ymm1, ymm2		;; B9 = B9 + R9
	vmovapd	ymm3, [screg2+0]		;; sine for w^1
	vmulpd	ymm5, ymm5, ymm3		;; A9 = A9 * sine (final R9)
	vmulpd	ymm1, ymm1, ymm3		;; B9 = B9 * sine (final I9)

	vmovapd	ymm3, [screg2+128+32]		;; cosine/sine for w^9
	vmulpd	ymm2, ymm0, ymm3		;; A10 = R10 * cosine/sine
	vsubpd	ymm2, ymm2, ymm7		;; A10 = A10 - I10
	vmulpd	ymm7, ymm7, ymm3		;; B10 = I10 * cosine/sine
	vaddpd	ymm7, ymm7, ymm0		;; B10 = B10 + R10
	vmovapd	ymm3, [screg2+128]		;; sine for w^9
	vmulpd	ymm2, ymm2, ymm3		;; A10 = A10 * sine (final R10)
	vmulpd	ymm7, ymm7, ymm3		;; B10 = B10 * sine (final I10)

	vmovapd	ymm0, [dstreg+e4+e2+e1+32]	;; Reload newest I12
	vsubpd	ymm3, ymm4, ymm0		;; R11 - I12 (final R11)
	vaddpd	ymm4, ymm4, ymm0		;; R11 + I12 (final R12)

	vmovapd	ymm0, [dstreg+e4+e2+e1]		;; Reload newest R12
	ystore	[dstreg+e4], ymm5		;; Save R9
	vaddpd	ymm5, ymm6, ymm0		;; I11 + R12 (final I11)
	vsubpd	ymm6, ymm6, ymm0		;; I11 - R12 (final I12)

	vmovapd	ymm0, [screg2+64+32]		;; cosine/sine for w^5
	ystore	[dstreg+e4+32], ymm1		;; Save I9
	vmulpd	ymm1, ymm3, ymm0		;; A11 = R11 * cosine/sine
	vsubpd	ymm1, ymm1, ymm5		;; A11 = A11 - I11
	vmulpd	ymm5, ymm5, ymm0		;; B11 = I11 * cosine/sine
	vaddpd	ymm5, ymm5, ymm3		;; B11 = B11 + R11

	vmovapd	ymm0, [screg2+192+32]		;; cosine/sine for w^13
	vmulpd	ymm3, ymm4, ymm0		;; A12 = R12 * cosine/sine
	vsubpd	ymm3, ymm3, ymm6		;; A12 = A12 - I12
	vmulpd	ymm6, ymm6, ymm0		;; B12 = I12 * cosine/sine
	vaddpd	ymm6, ymm6, ymm4		;; B12 = B12 + R12

	vmovapd	ymm0, [screg2+64]		;; sine for w^5
	vmulpd	ymm1, ymm1, ymm0		;; A11 = A11 * sine (final R11)
	vmulpd	ymm5, ymm5, ymm0		;; B11 = B11 * sine (final I11)

	vmovapd	ymm0, [screg2+192]		;; sine for w^13
	vmulpd	ymm3, ymm3, ymm0		;; A12 = A12 * sine (final R12)
	vmulpd	ymm6, ymm6, ymm0		;; B12 = B12 * sine (final I12)

	ystore	[dstreg+e4+e1], ymm2		;; Save R10
	ystore	[dstreg+e4+e1+32], ymm7		;; Save I10
	ystore	[dstreg+e4+e2], ymm1		;; Save R11
	ystore	[dstreg+e4+e2+32], ymm5		;; Save I11
	ystore	[dstreg+e4+e2+e1], ymm3		;; Save R12
	ystore	[dstreg+e4+e2+e1+32], ymm6	;; Save I12

	bump	srcreg, srcinc
	bump	dstreg, dstinc
	bump	screg1, scinc1
	bump	screg2, scinc2
	ENDM

IFDEF X86_64

yr8_rsc_sg8cl_2sc_sixteen_reals_fft8_preload MACRO
	ENDM
yr8_rsc_sg8cl_2sc_sixteen_reals_fft8 MACRO srcreg,srcinc,d1,d2,d4,dstreg,dstinc,e1,e2,e4,screg1,scinc1,screg2,scinc2,maxrpt,L1pt,L1pd

	vmovapd	ymm0, [srcreg]			;; R1
	vmovapd	ymm2, [srcreg+d1]		;; R2
	vshufpd	ymm1, ymm0, ymm2, 15		;; Shuffle R1 and R2 to create R1/R2 hi		;  1
	vshufpd	ymm0, ymm0, ymm2, 0		;; Shuffle R1 and R2 to create R1/R2 low	;  2

	vmovapd	ymm2, [srcreg+d2]		;; R3
	vmovapd	ymm4, [srcreg+d2+d1]		;; R4
	vshufpd	ymm3, ymm2, ymm4, 15		;; Shuffle R3 and R4 to create R3/R4 hi		;  3
	vshufpd	ymm2, ymm2, ymm4, 0		;; Shuffle R3 and R4 to create R3/R4 low	;  4

	vmovapd	ymm4, [srcreg+32]		;; I1
	vmovapd	ymm6, [srcreg+d1+32]		;; I2
	vshufpd	ymm5, ymm4, ymm6, 15		;; Shuffle I1 and I2 to create I1/I2 hi		;  5
	vshufpd	ymm4, ymm4, ymm6, 0		;; Shuffle I1 and I2 to create I1/I2 low	;  6

	vmovapd	ymm6, [srcreg+d2+32]		;; I3
	vmovapd	ymm8, [srcreg+d2+d1+32]		;; I4
	vshufpd	ymm7, ymm6, ymm8, 15		;; Shuffle I3 and I4 to create I3/I4 hi		;  7
	vshufpd	ymm6, ymm6, ymm8, 0		;; Shuffle I3 and I4 to create I3/I4 low	;  8

	ylow128s ymm8, ymm1, ymm3		;; Shuffle R1/R2 hi and R3/R4 hi (first R2)	;  9-10
	vmovapd	ymm15, [srcreg+d4]		;; R5

	ylow128s ymm9, ymm5, ymm7		;; Shuffle I1/I2 hi and I3/I4 hi (first R10)	;  10-11
	vmovapd	ymm14, [srcreg+d4+d1]		;; R6

	yhigh128s ymm1, ymm1, ymm3		;; Shuffle R1/R2 hi and R3/R4 hi (first R4)	;  11-12
	vmovapd	ymm13, [srcreg+d4+d2]		;; R7

	yhigh128s ymm5, ymm5, ymm7		;; Shuffle I1/I2 hi and I3/I4 hi (first R12)	;  12-13
	vaddpd	ymm7, ymm8, ymm9		;; R2 + R10 (new R2)				; 12-14
	vmovapd	ymm12, [srcreg+d4+d2+d1]	;; R8

	ylow128s ymm3, ymm0, ymm2		;; Shuffle R1/R2 low and R3/R4 low (first R1)	;  13-14
	vsubpd	ymm8, ymm8, ymm9		;; R2 - R10 (new R10)				; 13-15
	vmovapd	ymm11, [srcreg+d4+32]		;; I5

	ylow128s ymm9, ymm4, ymm6		;; Shuffle I1/I2 low and I3/I4 low (first R9)	;  14-15
	vaddpd	ymm10, ymm1, ymm5		;; R4 + R12 (new R4)				; 14-16

	yhigh128s ymm0, ymm0, ymm2		;; Shuffle R1/R2 low and R3/R4 low (first R3)	;  15-16
	vsubpd	ymm1, ymm1, ymm5		;; R4 - R12 (new R12)				; 15-17
	vmovapd	ymm2, [srcreg+d4+d1+32]		;; I6

	yhigh128s ymm4, ymm4, ymm6		;; Shuffle I1/I2 low and I3/I4 low (first R11)	;  16-17
	vsubpd	ymm6, ymm3, ymm9		;; R1 - R9 (new R9)				; 16-18

	vaddpd	ymm3, ymm3, ymm9		;; R1 + R9 (new R1)				; 17-19
	L1prefetch srcreg+L1pd, L1pt
	L1prefetchw dstreg+L1pd, L1pt - L1PREFETCH_DEST_NONE

	vshufpd	ymm9, ymm15, ymm14, 15		;; Shuffle R5 and R6 to create R5/R6 hi		;  18
	vsubpd	ymm5, ymm0, ymm4		;; R3 - R11 (new R11)				; 18-20

	vshufpd	ymm15, ymm15, ymm14, 0		;; Shuffle R5 and R6 to create R5/R6 low	;  19
	ystore	[dstreg+e2], ymm6		;; Save new R9					; 19
	vaddpd	ymm0, ymm0, ymm4		;; R3 + R11 (new R3)				; 19-21
	vmovapd	ymm14, [srcreg+d4+d2+32]	;; I7

	vshufpd	ymm4, ymm13, ymm12, 15		;; Shuffle R7 and R8 to create R7/R8 hi		;  20
	vmovapd	ymm6, [srcreg+d4+d2+d1+32]	;; I8

	vshufpd	ymm13, ymm13, ymm12, 0		;; Shuffle R7 and R8 to create R7/R8 low	;  21
	ystore	[dstreg], ymm5			;; Save new R11					; 21

	vshufpd	ymm5, ymm11, ymm2, 15		;; Shuffle I5 and I6 to create I5/I6 hi		;  22
	L1prefetch srcreg+d1+L1pd, L1pt
	L1prefetchw dstreg+e1+L1pd, L1pt - L1PREFETCH_DEST_NONE

	vshufpd	ymm11, ymm11, ymm2, 0		;; Shuffle I5 and I6 to create I5/I6 low	;  23

	vshufpd	ymm2, ymm14, ymm6, 15		;; Shuffle I7 and I8 to create I7/I8 hi		;  24
	L1prefetch srcreg+d2+L1pd, L1pt
	L1prefetchw dstreg+e2+L1pd, L1pt - L1PREFETCH_DEST_NONE

	vshufpd	ymm14, ymm14, ymm6, 0		;; Shuffle I7 and I8 to create I7/I8 low	;  25

	ylow128s ymm6, ymm9, ymm4		;; Shuffle R5/R6 hi and R7/R8 hi (first R6)	;  26-27
	L1prefetch srcreg+d2+d1+L1pd, L1pt
	L1prefetchw dstreg+e2+e1+L1pd, L1pt - L1PREFETCH_DEST_NONE

	ylow128s ymm12, ymm5, ymm2		;; Shuffle I5/I6 hi and I7/I8 hi (first R14)	;  27-28

	yhigh128s ymm9, ymm9, ymm4		;; Shuffle R5/R6 hi and R7/R8 hi (first R8)	;  28-29

	yhigh128s ymm5, ymm5, ymm2		;; Shuffle I5/I6 hi and I7/I8 hi (first R16)	;  29-30
	vsubpd	ymm2, ymm6, ymm12		;; R6 - R14 (new R14)				; 29-31
	L1prefetch srcreg+d4+L1pd, L1pt
	L1prefetchw dstreg+e4+L1pd, L1pt - L1PREFETCH_DEST_NONE

	ylow128s ymm4, ymm15, ymm13		;; Shuffle R5/R6 low and R7/R8 low (first R5)	;  30-31
	vaddpd	ymm6, ymm6, ymm12		;; R6 + R14 (new R6)				; 30-32
	ylow128s ymm12, ymm11, ymm14		;; Shuffle I5/I6 low and I7/I8 low (first R13)	;  31-32
	yhigh128s ymm15, ymm15, ymm13		;; Shuffle R5/R6 low and R7/R8 low (first R7)	;  32-33
	vsubpd	ymm13, ymm9, ymm5		;; R8 - R16 (new R16)				; 31-33
	vaddpd	ymm9, ymm9, ymm5		;; R8 + R16 (new R8)				; 32-34
	vmovapd	ymm5, YMM_P924
	yhigh128s ymm11, ymm11, ymm14		;; Shuffle I5/I6 low and I7/I8 low (first R15)	;  33-34
	vaddpd	ymm14, ymm4, ymm12		;; R5 + R13 (new R5)				; 33-35
						;; R10/R14 morphs into newer R10/I10
						;; mul R10/I10 by w^1 = .924 + .383i
	vsubpd	ymm4, ymm4, ymm12		;; R5 - R13 (new R13)				; 34-36
	vmulpd	ymm12, ymm8, ymm5		;; R10 * .924					;  30-34
	ystore	[dstreg+e2+32], ymm4		;; Save new R13					; 37
	vaddpd	ymm4, ymm15, ymm11		;; R7 + R15 (new R7)				; 35-37
	vsubpd	ymm15, ymm15, ymm11		;; R7 - R15 (new R15)				; 36-38
	vmovapd	ymm11, YMM_P383
	vmulpd	ymm8, ymm8, ymm11		;; R10 * .383					;  31-35
	ystore	[dstreg+32], ymm15		;; Save new R15					; 39
	vmulpd	ymm15, ymm2, ymm11		;; I10 * .383					;  32-36
	vmulpd	ymm2, ymm2, ymm5		;; I10 * .924					;  33-37
						;; R12/R16 morphs into newer R12/I12
						;; mul R12/I12 by w^3 = .383 + .924i
	vsubpd	ymm12, ymm12, ymm15		;; Twiddled R10 = R10 * .924 - I10 * .383	; 37-39
	vmulpd	ymm15, ymm1, ymm11		;; R12 * .383					;  34-38
	vmulpd	ymm1, ymm1, ymm5		;; R12 * .924					;  35-39
	vmulpd	ymm5, ymm13, ymm5		;; I12 * .924					;  36-40
	vmulpd	ymm13, ymm13, ymm11		;; I12 * .383					;  37-41

	vaddpd	ymm8, ymm8, ymm2		;; Twiddled I10 = R10 * .383 + I10 * .924	; 38-40
	vmovapd	ymm11, [dstreg]			;; Reload new R11 which will morph into newer R11

	vsubpd	ymm2, ymm10, ymm9		;; R4 - R8 (newer R8)				; 39-41
	L1prefetch srcreg+d4+d1+L1pd, L1pt
	L1prefetchw dstreg+e4+e1+L1pd, L1pt - L1PREFETCH_DEST_NONE

	vaddpd	ymm10, ymm10, ymm9		;; R4 + R8 (newer R4)				; 40-42

	vsubpd	ymm9, ymm7, ymm6		;; R2 - R6 (newer R6)				; 41-43

	vaddpd	ymm7, ymm7, ymm6		;; R2 + R6 (newer R2)				; 42-44
	vmovapd	ymm6, [dstreg+32]		;; Reload new R15 which will morph into newer I11

	vsubpd	ymm15, ymm15, ymm5		;; Twiddled R12 = R12 * .383 - I12 * .924	; 43-45
	vmovapd	ymm5, YMM_SQRTHALF

	vaddpd	ymm1, ymm1, ymm13		;; Twiddled I12 = R12 * .924 + I12 * .383	; 44-46

	vsubpd	ymm13, ymm3, ymm14		;; R1 - R5 (newer R5)				; 45-47
	L1prefetch srcreg+d4+d2+L1pd, L1pt
	L1prefetchw dstreg+e4+e2+L1pd, L1pt - L1PREFETCH_DEST_NONE

	vaddpd	ymm3, ymm3, ymm14		;; R1 + R5 (newer R1)				; 46-48

						;; R6/R8 morphs into newer R6/I6
						;; mul R6/I6 by w^2 = .707 + .707i
	vsubpd	ymm14, ymm9, ymm2		;; R6 = R6 - I6					; 47-49

	vaddpd	ymm9, ymm9, ymm2		;; I6 = R6 + I6					; 48-50
	L1prefetch srcreg+d4+d2+d1+L1pd, L1pt
	L1prefetchw dstreg+e4+e2+e1+L1pd, L1pt - L1PREFETCH_DEST_NONE

						;; R11/R15 morphs into newer R11/I11
						;; mul R11/I11 by SQRTHALF + i*SQRTHALF
	vsubpd	ymm2, ymm11, ymm6		;; R11 = R11 - I11				; 49-51

	vaddpd	ymm11, ymm11, ymm6		;; I11 = R11 + I11				; 50-52
	vmulpd	ymm14, ymm14, ymm5		;; R6 = R6 * SQRTHALF (newest R6)		;  50-54

	vaddpd	ymm6, ymm0, ymm4		;; R3 + R7 (newer R3)				; 51-53
	vmulpd	ymm9, ymm9, ymm5		;; I6 = I6 * SQRTHALF (newest I6)		;  51-55

	vsubpd	ymm0, ymm0, ymm4		;; R3 - R7 (newer R7)				; 52-54
	vmulpd	ymm2, ymm2, ymm5		;; R11 = R11 * SQRTHALF				;  52-56


	vaddpd	ymm4, ymm7, ymm10		;; R2 + R4 (newest R2)				; 53-55
	vmulpd	ymm11, ymm11, ymm5		;; I11 = I11 * SQRTHALF				;  53-57
	vmovapd	ymm5, [screg1+128+32]		;; cosine for w^4 (8-complex w^2)

	vsubpd	ymm7, ymm7, ymm10		;; R2 - R4 (newest R4)				; 54-56

 	vaddpd	ymm10, ymm3, ymm6		;; R1 + R3 (newest R1)				; 55-57

						;; R1/R2 becomes final R1 and final R2
	ystore	[dstreg+32], ymm4		;; Save R2					; 56
	vsubpd	ymm3, ymm3, ymm6		;; R1 - R3 (newest R3)				; 56-58
	vmovapd	ymm4, [screg1+128]		;; sine for w^4 (8-complex w^2)

						;; R5/R7 morphs into newest R5/I5
						;; R3/R4 morphs into R3/I3
	vsubpd	ymm6, ymm13, ymm14		;; R5 - R6 (final R6)				; 57-59
	ystore	[dstreg], ymm10			;; Save R1					; 58
	vmulpd	ymm10, ymm7, ymm5		;; B3 = I3 * cosine				;  57-61

	vaddpd	ymm13, ymm13, ymm14		;; R5 + R6 (final R5)				; 58-60
	vmulpd	ymm7, ymm7, ymm4		;; C3 = I3 * sine				;  58-62

	vsubpd	ymm14, ymm0, ymm9		;; I5 - I6 (final I6)				; 59-61
	vmulpd	ymm5, ymm3, ymm5		;; A3 = R3 * cosine				;  59-63

	vaddpd	ymm0, ymm0, ymm9		;; I5 + I6 (final I5)				; 60-62
	vmulpd	ymm3, ymm3, ymm4		;; D3 = R3 * sine				;  60-64
	vmovapd	ymm9, [screg1+320+32]		;; cosine for w^10 (8-complex w^5)

	vaddpd	ymm4, ymm12, ymm15		;; R10 + R12 (newest R10)			; 61-63

	vsubpd	ymm12, ymm12, ymm15		;; R10 - R12 (newest R12)			; 62-64

	vaddpd	ymm15, ymm8, ymm1		;; I10 + I12 (newest I10)			; 63-65

	vsubpd	ymm8, ymm8, ymm1		;; I10 - I12 (newest I12)			; 64-66
	vmulpd	ymm1, ymm6, ymm9		;; A6 = R6 * cosine				;  61-65 (can safely be 64-68)

	vsubpd	ymm5, ymm5, ymm7		;; A3 = A3 - C3 (final R3)			; 65-67
	vmulpd	ymm9, ymm14, ymm9		;; B6 = I6 * cosine				;  62-66 (can safely be 65-69)

	vmovapd	ymm7, [screg1+320]		;; sine for w^10 (8-complex w^5)
	vaddpd	ymm10, ymm10, ymm3		;; B3 = B3 + D3 (final I3)			; 66-68
	vmulpd	ymm14, ymm14, ymm7		;; C6 = I6 * sine				;  63-67 (can safely be 66-70)

						;; R9/R13 morphs into newer R9/I9
	vmovapd	ymm3, [dstreg+e2]		;; Reload new R9 which morphed into newer R9
	vmulpd	ymm6, ymm6, ymm7		;; D6 = R6 * sine				;  64-68 (can safely be 67-71)
	vaddpd	ymm7, ymm3, ymm2		;; R9 + R11 (newest R9)				; 67-69

	ystore	[dstreg+e1], ymm5		;; Save R3					; 68
	vsubpd	ymm3, ymm3, ymm2		;; R9 - R11 (newest R11)			; 68-70
	vmovapd	ymm2, [dstreg+e2+32]		;; Reload new R13 which morphed into newer I9

	ystore	[dstreg+e1+32], ymm10		;; Save I3					; 69
	vaddpd	ymm10, ymm2, ymm11		;; I9 + I11 (newest I9)				; 69-71
	vmovapd	ymm5, [screg1+64+32]		;; cosine for w^2 (8-complex w^1)

	vsubpd	ymm2, ymm2, ymm11		;; I9 - I11 (newest I11)			; 70-72
	vmovapd	ymm11, [screg1+64]		;; sine for w^2 (8-complex w^1)

	vsubpd	ymm1, ymm1, ymm14		;; A6 = A6 - C6 (final R6)			; 71-73

	vaddpd	ymm9, ymm9, ymm6		;; B6 = B6 + D6 (final I6)			; 72-74
	vmulpd	ymm6, ymm13, ymm5		;; A5 = R5 * cosine				;  65-69 (can safely be 72-76)

	vaddpd	ymm14, ymm7, ymm4		;; R9 + R10 (final R9)				; 73-75
	vmulpd	ymm5, ymm0, ymm5		;; B5 = I5 * cosine				;  66-70 (can safely be 73-77)

	ystore	[dstreg+e2+e1], ymm1		;; Save R6					; 74
	vaddpd	ymm1, ymm10, ymm15		;; I9 + I10 (final I9)				; 74-76
	vmulpd	ymm0, ymm0, ymm11		;; C5 = I5 * sine				;  67-71 (can safely be 74-78)

	ystore	[dstreg+e2+e1+32], ymm9		;; Save I6					; 75
	vsubpd	ymm7, ymm7, ymm4		;; R9 - R10 (final R10)				; 75-77
	vmulpd	ymm13, ymm13, ymm11		;; D5 = R5 * sine				;  68-72 (can safely be 75-79)
	vmovapd	ymm9, [screg2+0+32]		;; cosine/sine for w^1

	vsubpd	ymm10, ymm10, ymm15		;; I9 - I10 (final I10)				; 76-78
	vmulpd	ymm15, ymm14, ymm9		;; A9 = R9 * cosine/sine			;  76-80

	vsubpd	ymm11, ymm3, ymm8		;; R11 - I12 (final R11)			; 77-79
	vmulpd	ymm9, ymm1, ymm9		;; B9 = I9 * cosine/sine			;  77-81

	vaddpd	ymm4, ymm2, ymm12		;; I11 + R12 (final I11)			; 78-80
	vaddpd	ymm3, ymm3, ymm8		;; R11 + I12 (final R12)			; 79-81
	vmovapd	ymm8, [screg2+128+32]		;; cosine/sine for w^9
	vsubpd	ymm2, ymm2, ymm12		;; I11 - R12 (final I12)			; 80-82
	vmulpd	ymm12, ymm7, ymm8		;; A10 = R10 * cosine/sine			;  78-82
	vmulpd	ymm8, ymm10, ymm8		;; B10 = I10 * cosine/sine			;  79-83
	vsubpd	ymm6, ymm6, ymm0		;; A5 = A5 - C5 (final R5)			; 81-83
	vmovapd	ymm0, [screg2+64+32]		;; cosine/sine for w^5
	vaddpd	ymm5, ymm5, ymm13		;; B5 = B5 + D5 (final I5)			; 82-84
	vmulpd	ymm13, ymm11, ymm0		;; A11 = R11 * cosine/sine			;  80-84
	vmulpd	ymm0, ymm4, ymm0		;; B11 = I11 * cosine/sine			;  81-85
	vsubpd	ymm15, ymm15, ymm1		;; A9 = A9 - I9					; 83-85
	vmovapd	ymm1, [screg2+192+32]		;; cosine/sine for w^13
	vaddpd	ymm9, ymm9, ymm14		;; B9 = B9 + R9					; 84-86
	vmulpd	ymm14, ymm3, ymm1		;; A12 = R12 * cosine/sine			;  82-86
	vmulpd	ymm1, ymm2, ymm1		;; B12 = I12 * cosine/sine			;  83-87
	ystore	[dstreg+e2], ymm6		;; Save R5					; 84
	vmovapd	ymm6, [screg2+0]		;; sine for w^1

	ystore	[dstreg+e2+32], ymm5		;; Save I5					; 85
	vsubpd	ymm12, ymm12, ymm10		;; A10 = A10 - I10				; 85-87
	vmovapd	ymm5, [screg2+128]		;; sine for w^9

	vaddpd	ymm8, ymm8, ymm7		;; B10 = B10 + R10				; 86-88
	vmulpd	ymm15, ymm15, ymm6		;; A9 = A9 * sine (final R9)			;  86-90
	vmovapd	ymm10, [screg2+64]		;; sine for w^5

	vsubpd	ymm13, ymm13, ymm4		;; A11 = A11 - I11				; 87-89
	vmulpd	ymm9, ymm9, ymm6		;; B9 = B9 * sine (final I9)			;  87-91
	vmovapd	ymm7, [screg2+192]		;; sine for w^13

	vaddpd	ymm0, ymm0, ymm11		;; B11 = B11 + R11				; 88-90
	vmulpd	ymm12, ymm12, ymm5		;; A10 = A10 * sine (final R10)			;  88-92

	vsubpd	ymm14, ymm14, ymm2		;; A12 = A12 - I12				; 89-91
	vmulpd	ymm8, ymm8, ymm5		;; B10 = B10 * sine (final I10)			;  89-93

	vaddpd	ymm1, ymm1, ymm3		;; B12 = B12 + R12				; 90-92
	vmulpd	ymm13, ymm13, ymm10		;; A11 = A11 * sine (final R11)			;  90-94

	ystore	[dstreg+e4], ymm15		;; Save R9					; 91
	vmulpd	ymm0, ymm0, ymm10		;; B11 = B11 * sine (final I11)			;  91-95

	ystore	[dstreg+e4+32], ymm9		;; Save I9					; 92
	vmulpd	ymm14, ymm14, ymm7		;; A12 = A12 * sine (final R12)			;  92-96

	ystore	[dstreg+e4+e1], ymm12		;; Save R10					; 93
	vmulpd	ymm1, ymm1, ymm7		;; B12 = B12 * sine (final I12)			;  93-97

	ystore	[dstreg+e4+e1+32], ymm8		;; Save I10					; 94
	ystore	[dstreg+e4+e2], ymm13		;; Save R11					; 95
	ystore	[dstreg+e4+e2+32], ymm0		;; Save I11					; 96
	ystore	[dstreg+e4+e2+e1], ymm14	;; Save R12					; 97
	ystore	[dstreg+e4+e2+e1+32], ymm1	;; Save I12					; 98

	bump	srcreg, srcinc
	bump	dstreg, dstinc
	bump	screg1, scinc1
	bump	screg2, scinc2
	ENDM

IF (@INSTR(,%yarch,<FMA3>) NE 0)

yr8_rsc_sg8cl_2sc_sixteen_reals_fft8_preload MACRO
	ENDM
yr8_rsc_sg8cl_2sc_sixteen_reals_fft8 MACRO srcreg,srcinc,d1,d2,d4,dstreg,dstinc,e1,e2,e4,screg1,scinc1,screg2,scinc2,maxrpt,L1pt,L1pd
	vmovapd	ymm0, [srcreg]			;; R1
	vmovapd	ymm2, [srcreg+d1]		;; R2
	vshufpd	ymm1, ymm0, ymm2, 15		;; Shuffle R1 and R2 to create R1/R2 hi		;  1
	vshufpd	ymm0, ymm0, ymm2, 0		;; Shuffle R1 and R2 to create R1/R2 low	;  2

	vmovapd	ymm2, [srcreg+d2]		;; R3
	vmovapd	ymm4, [srcreg+d2+d1]		;; R4
	vshufpd	ymm3, ymm2, ymm4, 15		;; Shuffle R3 and R4 to create R3/R4 hi		;  3
	vshufpd	ymm2, ymm2, ymm4, 0		;; Shuffle R3 and R4 to create R3/R4 low	;  4

	vmovapd	ymm4, [srcreg+32]		;; I1
	vmovapd	ymm6, [srcreg+d1+32]		;; I2
	vshufpd	ymm5, ymm4, ymm6, 15		;; Shuffle I1 and I2 to create I1/I2 hi		;  5
	vshufpd	ymm4, ymm4, ymm6, 0		;; Shuffle I1 and I2 to create I1/I2 low	;  6

	vmovapd	ymm6, [srcreg+d2+32]		;; I3
	vmovapd	ymm8, [srcreg+d2+d1+32]		;; I4
	vshufpd	ymm7, ymm6, ymm8, 15		;; Shuffle I3 and I4 to create I3/I4 hi		;  7
	vshufpd	ymm6, ymm6, ymm8, 0		;; Shuffle I3 and I4 to create I3/I4 low	;  8

	ylow128s ymm8, ymm1, ymm3		;; Shuffle R1/R2 hi and R3/R4 hi (first R2)	;  9-10
	vmovapd	ymm15, [srcreg+d4]		;; R5

	ylow128s ymm9, ymm5, ymm7		;; Shuffle I1/I2 hi and I3/I4 hi (first R10)	;  10-11
	vmovapd	ymm14, [srcreg+d4+d1]		;; R6

	yhigh128s ymm1, ymm1, ymm3		;; Shuffle R1/R2 hi and R3/R4 hi (first R4)	;  11-12
	vmovapd	ymm13, [srcreg+d4+d2]		;; R7

	yhigh128s ymm5, ymm5, ymm7		;; Shuffle I1/I2 hi and I3/I4 hi (first R12)	;  12-13
	vaddpd	ymm7, ymm8, ymm9		;; R2 + R10 (new R2)				; 12-14
	vmovapd	ymm12, [srcreg+d4+d2+d1]	;; R8

	ylow128s ymm3, ymm0, ymm2		;; Shuffle R1/R2 low and R3/R4 low (first R1)	;  13-14
	vsubpd	ymm8, ymm8, ymm9		;; R2 - R10 (new R10)				; 13-15
	vmovapd	ymm11, [srcreg+d4+32]		;; I5

	ylow128s ymm9, ymm4, ymm6		;; Shuffle I1/I2 low and I3/I4 low (first R9)	;  14-15
	vaddpd	ymm10, ymm1, ymm5		;; R4 + R12 (new R4)				; 14-16

	yhigh128s ymm0, ymm0, ymm2		;; Shuffle R1/R2 low and R3/R4 low (first R3)	;  15-16
	vsubpd	ymm1, ymm1, ymm5		;; R4 - R12 (new R12)				; 15-17
	vmovapd	ymm2, [srcreg+d4+d1+32]		;; I6

	yhigh128s ymm4, ymm4, ymm6		;; Shuffle I1/I2 low and I3/I4 low (first R11)	;  16-17
	vsubpd	ymm6, ymm3, ymm9		;; R1 - R9 (new R9)				; 16-18

	vaddpd	ymm3, ymm3, ymm9		;; R1 + R9 (new R1)				; 17-19
	L1prefetch srcreg+L1pd, L1pt
	L1prefetchw dstreg+L1pd, L1pt - L1PREFETCH_DEST_NONE

	vshufpd	ymm9, ymm15, ymm14, 15		;; Shuffle R5 and R6 to create R5/R6 hi		;  18
	vsubpd	ymm5, ymm0, ymm4		;; R3 - R11 (new R11)				; 18-20

	vshufpd	ymm15, ymm15, ymm14, 0		;; Shuffle R5 and R6 to create R5/R6 low	;  19
	ystore	[dstreg+e2], ymm6		;; Save new R9					; 19
	vaddpd	ymm0, ymm0, ymm4		;; R3 + R11 (new R3)				; 19-21
	vmovapd	ymm14, [srcreg+d4+d2+32]	;; I7

	vshufpd	ymm4, ymm13, ymm12, 15		;; Shuffle R7 and R8 to create R7/R8 hi		;  20
	vmovapd	ymm6, [srcreg+d4+d2+d1+32]	;; I8

	vshufpd	ymm13, ymm13, ymm12, 0		;; Shuffle R7 and R8 to create R7/R8 low	;  21
	ystore	[dstreg], ymm5			;; Save new R11					; 21

	vshufpd	ymm5, ymm11, ymm2, 15		;; Shuffle I5 and I6 to create I5/I6 hi		;  22
	L1prefetch srcreg+d1+L1pd, L1pt
	L1prefetchw dstreg+e1+L1pd, L1pt - L1PREFETCH_DEST_NONE

	vshufpd	ymm11, ymm11, ymm2, 0		;; Shuffle I5 and I6 to create I5/I6 low	;  23

	vshufpd	ymm2, ymm14, ymm6, 15		;; Shuffle I7 and I8 to create I7/I8 hi		;  24
	L1prefetch srcreg+d2+L1pd, L1pt
	L1prefetchw dstreg+e2+L1pd, L1pt - L1PREFETCH_DEST_NONE

	vshufpd	ymm14, ymm14, ymm6, 0		;; Shuffle I7 and I8 to create I7/I8 low	;  25

	ylow128s ymm6, ymm9, ymm4		;; Shuffle R5/R6 hi and R7/R8 hi (first R6)	;  26-27
	L1prefetch srcreg+d2+d1+L1pd, L1pt
	L1prefetchw dstreg+e2+e1+L1pd, L1pt - L1PREFETCH_DEST_NONE

	ylow128s ymm12, ymm5, ymm2		;; Shuffle I5/I6 hi and I7/I8 hi (first R14)	;  27-28

	yhigh128s ymm9, ymm9, ymm4		;; Shuffle R5/R6 hi and R7/R8 hi (first R8)	;  28-29

	yhigh128s ymm5, ymm5, ymm2		;; Shuffle I5/I6 hi and I7/I8 hi (first R16)	;  29-30
	vaddpd	ymm2, ymm6, ymm12		;; R6 + R14 (new R6)				; 29-31
	L1prefetch srcreg+d4+L1pd, L1pt
	L1prefetchw dstreg+e4+L1pd, L1pt - L1PREFETCH_DEST_NONE

	ylow128s ymm4, ymm15, ymm13		;; Shuffle R5/R6 low and R7/R8 low (first R5)	;  30-31
	vsubpd	ymm6, ymm6, ymm12		;; R6 - R14 (new R14)				; 30-32

	ylow128s ymm12, ymm11, ymm14		;; Shuffle I5/I6 low and I7/I8 low (first R13)	;  31-32
	yhigh128s ymm15, ymm15, ymm13		;; Shuffle R5/R6 low and R7/R8 low (first R7)	;  32-33
	vaddpd	ymm13, ymm9, ymm5		;; R8 + R16 (new R8)				; 31-33
	vsubpd	ymm9, ymm9, ymm5		;; R8 - R16 (new R16)				; 32-34
	vmovapd	ymm5, YMM_ONE

	yhigh128s ymm11, ymm11, ymm14		;; Shuffle I5/I6 low and I7/I8 low (first R15)	;  33-34
	yfmaddpd ymm14, ymm4, ymm5, ymm12	;; R5 + R13 (new R5)				; 33-37
	yfmsubpd ymm4, ymm4, ymm5, ymm12	;; R5 - R13 (new R13)				; 33-37
	L1prefetch srcreg+d4+d1+L1pd, L1pt
	L1prefetchw dstreg+e4+e1+L1pd, L1pt - L1PREFETCH_DEST_NONE

	yfmaddpd ymm12, ymm15, ymm5, ymm11	;; R7 + R15 (new R7)				; 35-39
	yfmsubpd ymm15, ymm15, ymm5, ymm11	;; R7 - R15 (new R15)				; 35-39

	yfmsubpd ymm11, ymm7, ymm5, ymm2	;; R2 - R6 (newer R6)				; 34-38		n 39
	ystore	[dstreg+e2+32], ymm4		;; Save new R13					; 38
	yfmsubpd ymm4, ymm10, ymm5, ymm13	;; R4 - R8 (newer R8)				; 34-38		n 39

	yfmaddpd ymm7, ymm7, ymm5, ymm2		;; R2 + R6 (newer R2)				; 36-40		n 44
	yfmaddpd ymm10, ymm10, ymm5, ymm13	;; R4 + R8 (newer R4)				; 36-40		n 44
	vmovapd	ymm2, YMM_P924_P383

	yfmaddpd ymm13, ymm3, ymm5, ymm14	;; R1 + R5 (newer R1)				; 38-42		n 45
	yfmsubpd ymm3, ymm3, ymm5, ymm14	;; R1 - R5 (newer R5)				; 38-42		n 46
	L1prefetch srcreg+d4+d2+L1pd, L1pt
	L1prefetchw dstreg+e4+e2+L1pd, L1pt - L1PREFETCH_DEST_NONE

						;; R6/R8 morphs into newer R6/I6
						;; mul R6/I6 by w^2 = .707 + .707i
	yfmsubpd ymm14, ymm11, ymm5, ymm4	;; R6 = R6 - I6					; 39-43		n 46
	yfmaddpd ymm11, ymm11, ymm5, ymm4	;; I6 = R6 + I6					; 39-43		n 47

	yfmaddpd ymm4, ymm0, ymm5, ymm12	;; R3 + R7 (newer R3)				; 40-44		n 45
	yfmsubpd ymm0, ymm0, ymm5, ymm12	;; R3 - R7 (newer R7)				; 40-44		n 47

						;; R10/R14 morphs into newer R10/I10
						;; mul R10/I10 by w^1 = .924 + .383i
	yfmsubpd ymm12, ymm8, ymm2, ymm6	;; R10*.924/.383 - I10 (newer2 R10/.383)	; 41-45		n 48
	yfmaddpd ymm6, ymm6, ymm2, ymm8		;; R10 + I10*.924/.383 (newer2 I10/.383)	; 41-45		n 50
	L1prefetch srcreg+d4+d2+d1+L1pd, L1pt
	L1prefetchw dstreg+e4+e2+e1+L1pd, L1pt - L1PREFETCH_DEST_NONE

						;; R12/R16 morphs into newer R12/I12
						;; mul R12/I12 by w^3 = .383 + .924i
	yfnmaddpd ymm8, ymm9, ymm2, ymm1	;; R12 - I12*.924/.383 (newer2 R12/.383)	; 42-46		n 48
	yfmaddpd ymm1, ymm1, ymm2, ymm9		;; R12*.924/.383 + I12 (newer2 I12/.383)	; 42-46		n 50

						;; R11/R15 morphs into newer R11/I11
						;; mul R11/I11 by SQRTHALF + i*SQRTHALF
	vmovapd	ymm2, [dstreg]			;; Reload new R11 which morphed into newer R11
	yfmsubpd ymm9, ymm2, ymm5, ymm15	;; R11 = R11 - I11				; 43-47		n 49
	yfmaddpd ymm2, ymm2, ymm5, ymm15	;; I11 = R11 + I11				; 43-47		n 50

	yfmsubpd ymm15, ymm7, ymm5, ymm10	;; R2 - R4 (newest R4)				; 44-48		n 52
	yfmaddpd ymm7, ymm7, ymm5, ymm10	;; R2 + R4 (newest R2)				; 44-48

	yfmsubpd ymm10, ymm13, ymm5, ymm4	;; R1 - R3 (newest R3)				; 45-49		n 52
 	yfmaddpd ymm13, ymm13, ymm5, ymm4	;; R1 + R3 (newest R1)				; 45-49
	vmovapd	ymm4, YMM_SQRTHALF

						;; R1/R2 becomes final R1 and final R2
						;; R5/R7 morphs into newest R5/I5
	ystore	[dstreg+32], ymm7		;; Save R2					; 49
	yfnmaddpd ymm7, ymm14, ymm4, ymm3	;; R5 - R6 * SQRTHALF (final R6)		; 46-50		n 53
	yfmaddpd ymm14, ymm14, ymm4, ymm3	;; R5 + R6 * SQRTHALF (final R5)		; 46-50		n 60

	yfnmaddpd ymm3, ymm11, ymm4, ymm0	;; I5 - I6 * SQRTHALF (final I6)		; 47-51		n 53
	yfmaddpd ymm11, ymm11, ymm4, ymm0	;; I5 + I6 * SQRTHALF (final I5)		; 47-51		n 60

	yfmaddpd ymm0, ymm12, ymm5, ymm8	;; R10 + R12 (newest R10/.383)			; 48-52		n 54
	yfmsubpd ymm12, ymm12, ymm5, ymm8	;; R10 - R12 (newest R12/.383)			; 48-52		n 56

						;; R9/R13 morphs into newer R9/I9
	vmovapd	ymm8, [dstreg+e2]		;; Reload new R9 which morphed into newer R9
	ystore	[dstreg], ymm13			;; Save R1					; 50
	yfmaddpd ymm13, ymm9, ymm4, ymm8	;; R9 + R11 * SQRTHALF (newest R9)		; 49-53		n 54
	yfnmaddpd ymm9, ymm9, ymm4, ymm8	;; R9 - R11 * SQRTHALF (newest R11)		; 49-53		n 56

	yfmaddpd ymm8, ymm6, ymm5, ymm1		;; I10 + I12 (newest I10/.383)			; 50-54		n 55
	yfmaddpd ymm5, ymm2, ymm4, [dstreg+e2+32] ;; I9 + I11 * SQRTHALF (newest I9)		; 50-54		n 55

	yfnmaddpd ymm2, ymm2, ymm4, [dstreg+e2+32] ;; I9 - I11 * SQRTHALF (newest I11)		; 51-55		n 56
	yfmsubpd ymm6, ymm6, YMM_ONE, ymm1	;; I10 - I12 (newest I12/.383)			; 51-55		n 56

						;; R3/R4 morphs into R3/I3
	vmovapd	ymm4, [screg1+128+32]		;; cosine for w^4 (8-complex w^2)
	vmulpd	ymm1, ymm10, ymm4		;; A3 = R3 * cosine				; 52-56		n 57
	vmulpd	ymm4, ymm15, ymm4		;; B3 = I3 * cosine				; 52-56		n 57

	yfnmaddpd ymm15, ymm15, [screg1+128], ymm1 ;; A3 = A3 - I3 * sine (final R3)		; 57-61
	yfmaddpd ymm10, ymm10, [screg1+128], ymm4 ;; B3 = B3 + R3 * sine (final I3)		; 57-61

	vmovapd	ymm4, [screg1+320+32]		;; cosine for w^10 (8-complex w^5)
	vmulpd	ymm1, ymm7, ymm4		;; A6 = R6 * cosine				; 53-57		n 58
	vmulpd	ymm4, ymm3, ymm4		;; B6 = I6 * cosine				; 53-57		n 58

	yfnmaddpd ymm3, ymm3, [screg1+320], ymm1 ;; A6 = A6 - I6 * sine (final R6)		; 58-62
	yfmaddpd ymm7, ymm7, [screg1+320], ymm4 ;; B6 = B6 + R6 * sine (final I6)		; 58-62

	vmovapd	ymm4, YMM_P383
	yfmaddpd ymm1, ymm0, ymm4, ymm13	;; R9 + R10 * .383 (final R9)			; 54-58		n 61
	yfnmaddpd ymm0, ymm0, ymm4, ymm13	;; R9 - R10 * .383 (final R10)			; 54-58		n 62

	yfmaddpd ymm13, ymm8, ymm4, ymm5	;; I9 + I10 * .383 (final I9)			; 55-59		n 61
	yfnmaddpd ymm8, ymm8, ymm4, ymm5	;; I9 - I10 * .383 (final I10)			; 55-59		n 62

	yfnmaddpd ymm5, ymm6, ymm4, ymm9	;; R11 - I12 * .383 (final R11)			; 56-60		n 63
	ystore	[dstreg+e1], ymm15		;; Save R3					; 62
	yfmaddpd ymm15, ymm12, ymm4, ymm2	;; I11 + R12 * .383 (final I11)			; 56-60

	yfmaddpd ymm6, ymm6, ymm4, ymm9		;; R11 + I12 * .383 (final R12)			; 59-63		n 64
	yfnmaddpd ymm12, ymm12, ymm4, ymm2	;; I11 - R12 * .383 (final I12)			; 59-63

	vmovapd	ymm9, [screg1+64+32]		;; cosine for w^2 (8-complex w^1)
	vmulpd	ymm2, ymm14, ymm9		;; A5 = R5 * cosine				; 60-64
	vmulpd	ymm9, ymm11, ymm9		;; B5 = I5 * cosine				; 60-64

	vmovapd	ymm4, [screg2+0+32]		;; cosine/sine for w^1
	ystore	[dstreg+e1+32], ymm10		;; Save I3					; 62+1
	yfmsubpd ymm10, ymm1, ymm4, ymm13	;; A9 = R9 * cosine/sine - I9			; 61-65
	yfmaddpd ymm13, ymm13, ymm4, ymm1	;; B9 = I9 * cosine/sine + R9			; 61-65

	vmovapd	ymm4, [screg2+128+32]		;; cosine/sine for w^9
	yfmsubpd ymm1, ymm0, ymm4, ymm8		;; A10 = R10 * cosine/sine - I10		; 62-66
	yfmaddpd ymm8, ymm8, ymm4, ymm0		;; B10 = I10 * cosine/sine + R10		; 62-66

	vmovapd	ymm4, [screg2+64+32]		;; cosine/sine for w^5
	yfmsubpd ymm0, ymm5, ymm4, ymm15	;; A11 = R11 * cosine/sine - I11		; 63-67
	yfmaddpd ymm15, ymm15, ymm4, ymm5	;; B11 = I11 * cosine/sine + R11		; 63-67

	vmovapd	ymm4, [screg2+192+32]		;; cosine/sine for w^13
	ystore	[dstreg+e2+e1], ymm3		;; Save R6					; 63+1
	yfmsubpd ymm5, ymm6, ymm4, ymm12	;; A12 = R12 * cosine/sine - I12		; 64-68
	yfmaddpd ymm12, ymm12, ymm4, ymm6	;; B12 = I12 * cosine/sine + R12		; 64-68

	vmovapd	ymm4, [screg1+64]		;; sine for w^2 (8-complex w^1)
	ystore	[dstreg+e2+e1+32], ymm7		;; Save I6					; 63+2
	yfnmaddpd ymm11, ymm11, ymm4, ymm2	;; A5 = A5 - I5 * sine (final R5)		; 65-69
	yfmaddpd ymm14, ymm14, ymm4, ymm9	;; B5 = B5 + R5 * sine (final I5)		; 65-69

	vmovapd	ymm4, [screg2+0]		;; sine for w^1
	vmulpd	ymm10, ymm10, ymm4		;; A9 = A9 * sine (final R9)			; 66-70
	vmulpd	ymm13, ymm13, ymm4		;; B9 = B9 * sine (final I9)			; 66-70

	vmovapd	ymm4, [screg2+128]		;; sine for w^9
	vmulpd	ymm1, ymm1, ymm4		;; A10 = A10 * sine (final R10)			; 67-71
	vmulpd	ymm8, ymm8, ymm4		;; B10 = B10 * sine (final I10)			; 67-71

	vmovapd	ymm4, [screg2+64]		;; sine for w^5
	vmulpd	ymm0, ymm0, ymm4		;; A11 = A11 * sine (final R11)			; 68-72
	vmulpd	ymm15, ymm15, ymm4		;; B11 = B11 * sine (final I11)			; 68-72

	vmovapd	ymm4, [screg2+192]		;; sine for w^13
	vmulpd	ymm5, ymm5, ymm4		;; A12 = A12 * sine (final R12)			; 69-73
	vmulpd	ymm12, ymm12, ymm4		;; B12 = B12 * sine (final I12)			; 69-73

	ystore	[dstreg+e2], ymm11		;; Save R5					; 70
	ystore	[dstreg+e2+32], ymm14		;; Save I5					; 70+1
	ystore	[dstreg+e4], ymm10		;; Save R9					; 71+1
	ystore	[dstreg+e4+32], ymm13		;; Save I9					; 71+2
	ystore	[dstreg+e4+e1], ymm1		;; Save R10					; 72+2
	ystore	[dstreg+e4+e1+32], ymm8		;; Save I10					; 72+3
	ystore	[dstreg+e4+e2], ymm0		;; Save R11					; 73+3
	ystore	[dstreg+e4+e2+32], ymm15	;; Save I11					; 73+4
	ystore	[dstreg+e4+e2+e1], ymm5		;; Save R12					; 74+4
	ystore	[dstreg+e4+e2+e1+32], ymm12	;; Save I12					; 74+5

	bump	srcreg, srcinc
	bump	dstreg, dstinc
	bump	screg1, scinc1
	bump	screg2, scinc2
	ENDM

ENDIF

ENDIF


; These versions use registers for distances between blocks.  This lets us share pass1 code.

yr8_rsc_sg8clreg_2sc_sixteen_reals_fft8_preload MACRO
	ENDM
yr8_rsc_sg8clreg_2sc_sixteen_reals_fft8 MACRO srcreg,srcinc,d1,d2,d4,dstreg,dstinc,e1reg,e3reg,dst4reg,screg1,scinc1,screg2,scinc2,maxrpt,L1pt,L1preg,L1p4reg
	NOT IMPLMENTED IN 32-BIT
	ENDM

IFDEF X86_64

yr8_rsc_sg8clreg_2sc_sixteen_reals_fft8_preload MACRO
	ENDM
yr8_rsc_sg8clreg_2sc_sixteen_reals_fft8 MACRO srcreg,srcinc,d1,d2,d4,dstreg,dstinc,e1reg,e3reg,dst4reg,screg1,scinc1,screg2,scinc2,maxrpt,L1pt,L1preg,L1p4reg

	vmovapd	ymm0, [srcreg]			;; R1
	vmovapd	ymm2, [srcreg+d1]		;; R2
	vshufpd	ymm1, ymm0, ymm2, 15		;; Shuffle R1 and R2 to create R1/R2 hi		;  1
	vshufpd	ymm0, ymm0, ymm2, 0		;; Shuffle R1 and R2 to create R1/R2 low	;  2

	vmovapd	ymm2, [srcreg+d2]		;; R3
	vmovapd	ymm4, [srcreg+d2+d1]		;; R4
	vshufpd	ymm3, ymm2, ymm4, 15		;; Shuffle R3 and R4 to create R3/R4 hi		;  3
	vshufpd	ymm2, ymm2, ymm4, 0		;; Shuffle R3 and R4 to create R3/R4 low	;  4

	vmovapd	ymm4, [srcreg+32]		;; I1
	vmovapd	ymm6, [srcreg+d1+32]		;; I2
	vshufpd	ymm5, ymm4, ymm6, 15		;; Shuffle I1 and I2 to create I1/I2 hi		;  5
	vshufpd	ymm4, ymm4, ymm6, 0		;; Shuffle I1 and I2 to create I1/I2 low	;  6

	vmovapd	ymm6, [srcreg+d2+32]		;; I3
	vmovapd	ymm8, [srcreg+d2+d1+32]		;; I4
	vshufpd	ymm7, ymm6, ymm8, 15		;; Shuffle I3 and I4 to create I3/I4 hi		;  7
	vshufpd	ymm6, ymm6, ymm8, 0		;; Shuffle I3 and I4 to create I3/I4 low	;  8

	ylow128s ymm8, ymm1, ymm3		;; Shuffle R1/R2 hi and R3/R4 hi (first R2)	;  9-10
	vmovapd	ymm15, [srcreg+d4]		;; R5

	ylow128s ymm9, ymm5, ymm7		;; Shuffle I1/I2 hi and I3/I4 hi (first R10)	;  10-11
	vmovapd	ymm14, [srcreg+d4+d1]		;; R6

	yhigh128s ymm1, ymm1, ymm3		;; Shuffle R1/R2 hi and R3/R4 hi (first R4)	;  11-12
	vmovapd	ymm13, [srcreg+d4+d2]		;; R7

	yhigh128s ymm5, ymm5, ymm7		;; Shuffle I1/I2 hi and I3/I4 hi (first R12)	;  12-13
	vaddpd	ymm7, ymm8, ymm9		;; R2 + R10 (new R2)				; 12-14
	vmovapd	ymm12, [srcreg+d4+d2+d1]	;; R8

	ylow128s ymm3, ymm0, ymm2		;; Shuffle R1/R2 low and R3/R4 low (first R1)	;  13-14
	vsubpd	ymm8, ymm8, ymm9		;; R2 - R10 (new R10)				; 13-15
	vmovapd	ymm11, [srcreg+d4+32]		;; I5

	ylow128s ymm9, ymm4, ymm6		;; Shuffle I1/I2 low and I3/I4 low (first R9)	;  14-15
	vaddpd	ymm10, ymm1, ymm5		;; R4 + R12 (new R4)				; 14-16

	yhigh128s ymm0, ymm0, ymm2		;; Shuffle R1/R2 low and R3/R4 low (first R3)	;  15-16
	vsubpd	ymm1, ymm1, ymm5		;; R4 - R12 (new R12)				; 15-17
	vmovapd	ymm2, [srcreg+d4+d1+32]		;; I6

	yhigh128s ymm4, ymm4, ymm6		;; Shuffle I1/I2 low and I3/I4 low (first R11)	;  16-17
	vsubpd	ymm6, ymm3, ymm9		;; R1 - R9 (new R9)				; 16-18

	vaddpd	ymm3, ymm3, ymm9		;; R1 + R9 (new R1)				; 17-19
	L1prefetchw L1preg, L1pt - L1PREFETCH_DEST_NONE

	vshufpd	ymm9, ymm15, ymm14, 15		;; Shuffle R5 and R6 to create R5/R6 hi		;  18
	vsubpd	ymm5, ymm0, ymm4		;; R3 - R11 (new R11)				; 18-20

	vshufpd	ymm15, ymm15, ymm14, 0		;; Shuffle R5 and R6 to create R5/R6 low	;  19
	ystore	[dstreg+2*e1reg], ymm6		;; Save new R9					; 19
	vaddpd	ymm0, ymm0, ymm4		;; R3 + R11 (new R3)				; 19-21
	vmovapd	ymm14, [srcreg+d4+d2+32]	;; I7

	vshufpd	ymm4, ymm13, ymm12, 15		;; Shuffle R7 and R8 to create R7/R8 hi		;  20
	vmovapd	ymm6, [srcreg+d4+d2+d1+32]	;; I8

	vshufpd	ymm13, ymm13, ymm12, 0		;; Shuffle R7 and R8 to create R7/R8 low	;  21
	ystore	[dstreg], ymm5			;; Save new R11					; 21

	vshufpd	ymm5, ymm11, ymm2, 15		;; Shuffle I5 and I6 to create I5/I6 hi		;  22
	L1prefetchw L1preg+e1reg, L1pt - L1PREFETCH_DEST_NONE

	vshufpd	ymm11, ymm11, ymm2, 0		;; Shuffle I5 and I6 to create I5/I6 low	;  23

	vshufpd	ymm2, ymm14, ymm6, 15		;; Shuffle I7 and I8 to create I7/I8 hi		;  24
	L1prefetchw L1preg+2*e1reg, L1pt - L1PREFETCH_DEST_NONE

	vshufpd	ymm14, ymm14, ymm6, 0		;; Shuffle I7 and I8 to create I7/I8 low	;  25

	ylow128s ymm6, ymm9, ymm4		;; Shuffle R5/R6 hi and R7/R8 hi (first R6)	;  26-27
	L1prefetchw L1preg+e3reg, L1pt - L1PREFETCH_DEST_NONE

	ylow128s ymm12, ymm5, ymm2		;; Shuffle I5/I6 hi and I7/I8 hi (first R14)	;  27-28

	yhigh128s ymm9, ymm9, ymm4		;; Shuffle R5/R6 hi and R7/R8 hi (first R8)	;  28-29

	yhigh128s ymm5, ymm5, ymm2		;; Shuffle I5/I6 hi and I7/I8 hi (first R16)	;  29-30
	vsubpd	ymm2, ymm6, ymm12		;; R6 - R14 (new R14)				; 29-31
	L1prefetchw L1p4reg, L1pt - L1PREFETCH_DEST_NONE

	ylow128s ymm4, ymm15, ymm13		;; Shuffle R5/R6 low and R7/R8 low (first R5)	;  30-31
	vaddpd	ymm6, ymm6, ymm12		;; R6 + R14 (new R6)				; 30-32
	ylow128s ymm12, ymm11, ymm14		;; Shuffle I5/I6 low and I7/I8 low (first R13)	;  31-32
	yhigh128s ymm15, ymm15, ymm13		;; Shuffle R5/R6 low and R7/R8 low (first R7)	;  32-33
	vsubpd	ymm13, ymm9, ymm5		;; R8 - R16 (new R16)				; 31-33
	vaddpd	ymm9, ymm9, ymm5		;; R8 + R16 (new R8)				; 32-34
	vmovapd	ymm5, YMM_P924
	yhigh128s ymm11, ymm11, ymm14		;; Shuffle I5/I6 low and I7/I8 low (first R15)	;  33-34
	vaddpd	ymm14, ymm4, ymm12		;; R5 + R13 (new R5)				; 33-35
						;; R10/R14 morphs into newer R10/I10
						;; mul R10/I10 by w^1 = .924 + .383i
	vsubpd	ymm4, ymm4, ymm12		;; R5 - R13 (new R13)				; 34-36
	vmulpd	ymm12, ymm8, ymm5		;; R10 * .924					;  30-34
	ystore	[dstreg+2*e1reg+32], ymm4	;; Save new R13					; 37
	vaddpd	ymm4, ymm15, ymm11		;; R7 + R15 (new R7)				; 35-37
	vsubpd	ymm15, ymm15, ymm11		;; R7 - R15 (new R15)				; 36-38
	vmovapd	ymm11, YMM_P383
	vmulpd	ymm8, ymm8, ymm11		;; R10 * .383					;  31-35
	ystore	[dstreg+32], ymm15		;; Save new R15					; 39
	vmulpd	ymm15, ymm2, ymm11		;; I10 * .383					;  32-36
	vmulpd	ymm2, ymm2, ymm5		;; I10 * .924					;  33-37
						;; R12/R16 morphs into newer R12/I12
						;; mul R12/I12 by w^3 = .383 + .924i
	vsubpd	ymm12, ymm12, ymm15		;; Twiddled R10 = R10 * .924 - I10 * .383	; 37-39
	vmulpd	ymm15, ymm1, ymm11		;; R12 * .383					;  34-38
	vmulpd	ymm1, ymm1, ymm5		;; R12 * .924					;  35-39
	vmulpd	ymm5, ymm13, ymm5		;; I12 * .924					;  36-40
	vmulpd	ymm13, ymm13, ymm11		;; I12 * .383					;  37-41

	vaddpd	ymm8, ymm8, ymm2		;; Twiddled I10 = R10 * .383 + I10 * .924	; 38-40
	vmovapd	ymm11, [dstreg]			;; Reload new R11 which will morph into newer R11

	vsubpd	ymm2, ymm10, ymm9		;; R4 - R8 (newer R8)				; 39-41
	L1prefetchw L1p4reg+e1reg, L1pt - L1PREFETCH_DEST_NONE

	vaddpd	ymm10, ymm10, ymm9		;; R4 + R8 (newer R4)				; 40-42

	vsubpd	ymm9, ymm7, ymm6		;; R2 - R6 (newer R6)				; 41-43

	vaddpd	ymm7, ymm7, ymm6		;; R2 + R6 (newer R2)				; 42-44
	vmovapd	ymm6, [dstreg+32]		;; Reload new R15 which will morph into newer I11

	vsubpd	ymm15, ymm15, ymm5		;; Twiddled R12 = R12 * .383 - I12 * .924	; 43-45
	vmovapd	ymm5, YMM_SQRTHALF

	vaddpd	ymm1, ymm1, ymm13		;; Twiddled I12 = R12 * .924 + I12 * .383	; 44-46

	vsubpd	ymm13, ymm3, ymm14		;; R1 - R5 (newer R5)				; 45-47
	L1prefetchw L1p4reg+2*e1reg, L1pt - L1PREFETCH_DEST_NONE

	vaddpd	ymm3, ymm3, ymm14		;; R1 + R5 (newer R1)				; 46-48

						;; R6/R8 morphs into newer R6/I6
						;; mul R6/I6 by w^2 = .707 + .707i
	vsubpd	ymm14, ymm9, ymm2		;; R6 = R6 - I6					; 47-49

	vaddpd	ymm9, ymm9, ymm2		;; I6 = R6 + I6					; 48-50
	L1prefetchw L1p4reg+e3reg, L1pt - L1PREFETCH_DEST_NONE

						;; R11/R15 morphs into newer R11/I11
						;; mul R11/I11 by SQRTHALF + i*SQRTHALF
	vsubpd	ymm2, ymm11, ymm6		;; R11 = R11 - I11				; 49-51

	vaddpd	ymm11, ymm11, ymm6		;; I11 = R11 + I11				; 50-52
	vmulpd	ymm14, ymm14, ymm5		;; R6 = R6 * SQRTHALF (newest R6)		;  50-54

	vaddpd	ymm6, ymm0, ymm4		;; R3 + R7 (newer R3)				; 51-53
	vmulpd	ymm9, ymm9, ymm5		;; I6 = I6 * SQRTHALF (newest I6)		;  51-55

	vsubpd	ymm0, ymm0, ymm4		;; R3 - R7 (newer R7)				; 52-54
	vmulpd	ymm2, ymm2, ymm5		;; R11 = R11 * SQRTHALF				;  52-56


	vaddpd	ymm4, ymm7, ymm10		;; R2 + R4 (newest R2)				; 53-55
	vmulpd	ymm11, ymm11, ymm5		;; I11 = I11 * SQRTHALF				;  53-57
	vmovapd	ymm5, [screg1+128+32]		;; cosine for w^4 (8-complex w^2)

	vsubpd	ymm7, ymm7, ymm10		;; R2 - R4 (newest R4)				; 54-56

 	vaddpd	ymm10, ymm3, ymm6		;; R1 + R3 (newest R1)				; 55-57

						;; R1/R2 becomes final R1 and final R2
	ystore	[dstreg+32], ymm4		;; Save R2					; 56
	vsubpd	ymm3, ymm3, ymm6		;; R1 - R3 (newest R3)				; 56-58
	vmovapd	ymm4, [screg1+128]		;; sine for w^4 (8-complex w^2)

						;; R5/R7 morphs into newest R5/I5
						;; R3/R4 morphs into R3/I3
	vsubpd	ymm6, ymm13, ymm14		;; R5 - R6 (final R6)				; 57-59
	ystore	[dstreg], ymm10			;; Save R1					; 58
	vmulpd	ymm10, ymm7, ymm5		;; B3 = I3 * cosine				;  57-61

	vaddpd	ymm13, ymm13, ymm14		;; R5 + R6 (final R5)				; 58-60
	vmulpd	ymm7, ymm7, ymm4		;; C3 = I3 * sine				;  58-62

	vsubpd	ymm14, ymm0, ymm9		;; I5 - I6 (final I6)				; 59-61
	vmulpd	ymm5, ymm3, ymm5		;; A3 = R3 * cosine				;  59-63

	vaddpd	ymm0, ymm0, ymm9		;; I5 + I6 (final I5)				; 60-62
	vmulpd	ymm3, ymm3, ymm4		;; D3 = R3 * sine				;  60-64
	vmovapd	ymm9, [screg1+320+32]		;; cosine for w^10 (8-complex w^5)

	vaddpd	ymm4, ymm12, ymm15		;; R10 + R12 (newest R10)			; 61-63

	vsubpd	ymm12, ymm12, ymm15		;; R10 - R12 (newest R12)			; 62-64

	vaddpd	ymm15, ymm8, ymm1		;; I10 + I12 (newest I10)			; 63-65

	vsubpd	ymm8, ymm8, ymm1		;; I10 - I12 (newest I12)			; 64-66
	vmulpd	ymm1, ymm6, ymm9		;; A6 = R6 * cosine				;  61-65 (can safely be 64-68)

	vsubpd	ymm5, ymm5, ymm7		;; A3 = A3 - C3 (final R3)			; 65-67
	vmulpd	ymm9, ymm14, ymm9		;; B6 = I6 * cosine				;  62-66 (can safely be 65-69)

	vmovapd	ymm7, [screg1+320]		;; sine for w^10 (8-complex w^5)
	vaddpd	ymm10, ymm10, ymm3		;; B3 = B3 + D3 (final I3)			; 66-68
	vmulpd	ymm14, ymm14, ymm7		;; C6 = I6 * sine				;  63-67 (can safely be 66-70)

						;; R9/R13 morphs into newer R9/I9
	vmovapd	ymm3, [dstreg+2*e1reg]		;; Reload new R9 which morphed into newer R9
	vmulpd	ymm6, ymm6, ymm7		;; D6 = R6 * sine				;  64-68 (can safely be 67-71)
	vaddpd	ymm7, ymm3, ymm2		;; R9 + R11 (newest R9)				; 67-69

	ystore	[dstreg+e1reg], ymm5		;; Save R3					; 68
	vsubpd	ymm3, ymm3, ymm2		;; R9 - R11 (newest R11)			; 68-70
	vmovapd	ymm2, [dstreg+2*e1reg+32]	;; Reload new R13 which morphed into newer I9

	ystore	[dstreg+e1reg+32], ymm10	;; Save I3					; 69
	vaddpd	ymm10, ymm2, ymm11		;; I9 + I11 (newest I9)				; 69-71
	vmovapd	ymm5, [screg1+64+32]		;; cosine for w^2 (8-complex w^1)

	vsubpd	ymm2, ymm2, ymm11		;; I9 - I11 (newest I11)			; 70-72
	vmovapd	ymm11, [screg1+64]		;; sine for w^2 (8-complex w^1)

	vsubpd	ymm1, ymm1, ymm14		;; A6 = A6 - C6 (final R6)			; 71-73

	vaddpd	ymm9, ymm9, ymm6		;; B6 = B6 + D6 (final I6)			; 72-74
	vmulpd	ymm6, ymm13, ymm5		;; A5 = R5 * cosine				;  65-69 (can safely be 72-76)

	vaddpd	ymm14, ymm7, ymm4		;; R9 + R10 (final R9)				; 73-75
	vmulpd	ymm5, ymm0, ymm5		;; B5 = I5 * cosine				;  66-70 (can safely be 73-77)

	ystore	[dstreg+e3reg], ymm1		;; Save R6					; 74
	vaddpd	ymm1, ymm10, ymm15		;; I9 + I10 (final I9)				; 74-76
	vmulpd	ymm0, ymm0, ymm11		;; C5 = I5 * sine				;  67-71 (can safely be 74-78)

	ystore	[dstreg+e3reg+32], ymm9		;; Save I6					; 75
	vsubpd	ymm7, ymm7, ymm4		;; R9 - R10 (final R10)				; 75-77
	vmulpd	ymm13, ymm13, ymm11		;; D5 = R5 * sine				;  68-72 (can safely be 75-79)
	vmovapd	ymm9, [screg2+0+32]		;; cosine/sine for w^1

	vsubpd	ymm10, ymm10, ymm15		;; I9 - I10 (final I10)				; 76-78
	vmulpd	ymm15, ymm14, ymm9		;; A9 = R9 * cosine/sine			;  76-80

	vsubpd	ymm11, ymm3, ymm8		;; R11 - I12 (final R11)			; 77-79
	vmulpd	ymm9, ymm1, ymm9		;; B9 = I9 * cosine/sine			;  77-81

	vaddpd	ymm4, ymm2, ymm12		;; I11 + R12 (final I11)			; 78-80
	vaddpd	ymm3, ymm3, ymm8		;; R11 + I12 (final R12)			; 79-81
	vmovapd	ymm8, [screg2+128+32]		;; cosine/sine for w^9
	vsubpd	ymm2, ymm2, ymm12		;; I11 - R12 (final I12)			; 80-82
	vmulpd	ymm12, ymm7, ymm8		;; A10 = R10 * cosine/sine			;  78-82
	vmulpd	ymm8, ymm10, ymm8		;; B10 = I10 * cosine/sine			;  79-83
	vsubpd	ymm6, ymm6, ymm0		;; A5 = A5 - C5 (final R5)			; 81-83
	vmovapd	ymm0, [screg2+64+32]		;; cosine/sine for w^5
	vaddpd	ymm5, ymm5, ymm13		;; B5 = B5 + D5 (final I5)			; 82-84
	vmulpd	ymm13, ymm11, ymm0		;; A11 = R11 * cosine/sine			;  80-84
	vmulpd	ymm0, ymm4, ymm0		;; B11 = I11 * cosine/sine			;  81-85
	vsubpd	ymm15, ymm15, ymm1		;; A9 = A9 - I9					; 83-85
	vmovapd	ymm1, [screg2+192+32]		;; cosine/sine for w^13
	vaddpd	ymm9, ymm9, ymm14		;; B9 = B9 + R9					; 84-86
	vmulpd	ymm14, ymm3, ymm1		;; A12 = R12 * cosine/sine			;  82-86
	vmulpd	ymm1, ymm2, ymm1		;; B12 = I12 * cosine/sine			;  83-87
	ystore	[dstreg+2*e1reg], ymm6		;; Save R5					; 84
	vmovapd	ymm6, [screg2+0]		;; sine for w^1

	ystore	[dstreg+2*e1reg+32], ymm5	;; Save I5					; 85
	vsubpd	ymm12, ymm12, ymm10		;; A10 = A10 - I10				; 85-87
	vmovapd	ymm5, [screg2+128]		;; sine for w^9

	vaddpd	ymm8, ymm8, ymm7		;; B10 = B10 + R10				; 86-88
	vmulpd	ymm15, ymm15, ymm6		;; A9 = A9 * sine (final R9)			;  86-90
	vmovapd	ymm10, [screg2+64]		;; sine for w^5

	vsubpd	ymm13, ymm13, ymm4		;; A11 = A11 - I11				; 87-89
	vmulpd	ymm9, ymm9, ymm6		;; B9 = B9 * sine (final I9)			;  87-91
	vmovapd	ymm7, [screg2+192]		;; sine for w^13

	vaddpd	ymm0, ymm0, ymm11		;; B11 = B11 + R11				; 88-90
	vmulpd	ymm12, ymm12, ymm5		;; A10 = A10 * sine (final R10)			;  88-92

	vsubpd	ymm14, ymm14, ymm2		;; A12 = A12 - I12				; 89-91
	vmulpd	ymm8, ymm8, ymm5		;; B10 = B10 * sine (final I10)			;  89-93

	vaddpd	ymm1, ymm1, ymm3		;; B12 = B12 + R12				; 90-92
	vmulpd	ymm13, ymm13, ymm10		;; A11 = A11 * sine (final R11)			;  90-94

	ystore	[dst4reg], ymm15		;; Save R9					; 91
	vmulpd	ymm0, ymm0, ymm10		;; B11 = B11 * sine (final I11)			;  91-95

	ystore	[dst4reg+32], ymm9		;; Save I9					; 92
	vmulpd	ymm14, ymm14, ymm7		;; A12 = A12 * sine (final R12)			;  92-96

	ystore	[dst4reg+e1reg], ymm12		;; Save R10					; 93
	vmulpd	ymm1, ymm1, ymm7		;; B12 = B12 * sine (final I12)			;  93-97

	ystore	[dst4reg+e1reg+32], ymm8	;; Save I10					; 94
	ystore	[dst4reg+2*e1reg], ymm13	;; Save R11					; 95
	ystore	[dst4reg+2*e1reg+32], ymm0	;; Save I11					; 96
	ystore	[dst4reg+e3reg], ymm14		;; Save R12					; 97
	ystore	[dst4reg+e3reg+32], ymm1	;; Save I12					; 98

	bump	srcreg, srcinc
	bump	dstreg, dstinc
	bump	dst4reg, dstinc
	bump	screg1, scinc1
	bump	screg2, scinc2
	bump	L1preg, dstinc
	bump	L1p4reg, dstinc
	ENDM

IF (@INSTR(,%yarch,<FMA3>) NE 0)

yr8_rsc_sg8clreg_2sc_sixteen_reals_fft8_preload MACRO
	ENDM
yr8_rsc_sg8clreg_2sc_sixteen_reals_fft8 MACRO srcreg,srcinc,d1,d2,d4,dstreg,dstinc,e1reg,e3reg,dst4reg,screg1,scinc1,screg2,scinc2,maxrpt,L1pt,L1preg,L1p4reg
	vmovapd	ymm0, [srcreg]			;; R1
	vmovapd	ymm2, [srcreg+d1]		;; R2
	vshufpd	ymm1, ymm0, ymm2, 15		;; Shuffle R1 and R2 to create R1/R2 hi		;  1
	vshufpd	ymm0, ymm0, ymm2, 0		;; Shuffle R1 and R2 to create R1/R2 low	;  2

	vmovapd	ymm2, [srcreg+d2]		;; R3
	vmovapd	ymm4, [srcreg+d2+d1]		;; R4
	vshufpd	ymm3, ymm2, ymm4, 15		;; Shuffle R3 and R4 to create R3/R4 hi		;  3
	vshufpd	ymm2, ymm2, ymm4, 0		;; Shuffle R3 and R4 to create R3/R4 low	;  4

	vmovapd	ymm4, [srcreg+32]		;; I1
	vmovapd	ymm6, [srcreg+d1+32]		;; I2
	vshufpd	ymm5, ymm4, ymm6, 15		;; Shuffle I1 and I2 to create I1/I2 hi		;  5
	vshufpd	ymm4, ymm4, ymm6, 0		;; Shuffle I1 and I2 to create I1/I2 low	;  6

	vmovapd	ymm6, [srcreg+d2+32]		;; I3
	vmovapd	ymm8, [srcreg+d2+d1+32]		;; I4
	vshufpd	ymm7, ymm6, ymm8, 15		;; Shuffle I3 and I4 to create I3/I4 hi		;  7
	vshufpd	ymm6, ymm6, ymm8, 0		;; Shuffle I3 and I4 to create I3/I4 low	;  8

	ylow128s ymm8, ymm1, ymm3		;; Shuffle R1/R2 hi and R3/R4 hi (first R2)	;  9-10
	vmovapd	ymm15, [srcreg+d4]		;; R5

	ylow128s ymm9, ymm5, ymm7		;; Shuffle I1/I2 hi and I3/I4 hi (first R10)	;  10-11
	vmovapd	ymm14, [srcreg+d4+d1]		;; R6

	yhigh128s ymm1, ymm1, ymm3		;; Shuffle R1/R2 hi and R3/R4 hi (first R4)	;  11-12
	vmovapd	ymm13, [srcreg+d4+d2]		;; R7

	yhigh128s ymm5, ymm5, ymm7		;; Shuffle I1/I2 hi and I3/I4 hi (first R12)	;  12-13
	vaddpd	ymm7, ymm8, ymm9		;; R2 + R10 (new R2)				; 12-14
	vmovapd	ymm12, [srcreg+d4+d2+d1]	;; R8

	ylow128s ymm3, ymm0, ymm2		;; Shuffle R1/R2 low and R3/R4 low (first R1)	;  13-14
	vsubpd	ymm8, ymm8, ymm9		;; R2 - R10 (new R10)				; 13-15
	vmovapd	ymm11, [srcreg+d4+32]		;; I5

	ylow128s ymm9, ymm4, ymm6		;; Shuffle I1/I2 low and I3/I4 low (first R9)	;  14-15
	vaddpd	ymm10, ymm1, ymm5		;; R4 + R12 (new R4)				; 14-16

	yhigh128s ymm0, ymm0, ymm2		;; Shuffle R1/R2 low and R3/R4 low (first R3)	;  15-16
	vsubpd	ymm1, ymm1, ymm5		;; R4 - R12 (new R12)				; 15-17
	vmovapd	ymm2, [srcreg+d4+d1+32]		;; I6

	yhigh128s ymm4, ymm4, ymm6		;; Shuffle I1/I2 low and I3/I4 low (first R11)	;  16-17
	vsubpd	ymm6, ymm3, ymm9		;; R1 - R9 (new R9)				; 16-18

	vaddpd	ymm3, ymm3, ymm9		;; R1 + R9 (new R1)				; 17-19
	L1prefetchw L1preg, L1pt - L1PREFETCH_DEST_NONE

	vshufpd	ymm9, ymm15, ymm14, 15		;; Shuffle R5 and R6 to create R5/R6 hi		;  18
	vsubpd	ymm5, ymm0, ymm4		;; R3 - R11 (new R11)				; 18-20

	vshufpd	ymm15, ymm15, ymm14, 0		;; Shuffle R5 and R6 to create R5/R6 low	;  19
	ystore	[dstreg+2*e1reg], ymm6		;; Save new R9					; 19
	vaddpd	ymm0, ymm0, ymm4		;; R3 + R11 (new R3)				; 19-21
	vmovapd	ymm14, [srcreg+d4+d2+32]	;; I7

	vshufpd	ymm4, ymm13, ymm12, 15		;; Shuffle R7 and R8 to create R7/R8 hi		;  20
	vmovapd	ymm6, [srcreg+d4+d2+d1+32]	;; I8

	vshufpd	ymm13, ymm13, ymm12, 0		;; Shuffle R7 and R8 to create R7/R8 low	;  21
	ystore	[dstreg], ymm5			;; Save new R11					; 21

	vshufpd	ymm5, ymm11, ymm2, 15		;; Shuffle I5 and I6 to create I5/I6 hi		;  22
	L1prefetchw L1preg+e1reg, L1pt - L1PREFETCH_DEST_NONE

	vshufpd	ymm11, ymm11, ymm2, 0		;; Shuffle I5 and I6 to create I5/I6 low	;  23

	vshufpd	ymm2, ymm14, ymm6, 15		;; Shuffle I7 and I8 to create I7/I8 hi		;  24
	L1prefetchw L1preg+2*e1reg, L1pt - L1PREFETCH_DEST_NONE

	vshufpd	ymm14, ymm14, ymm6, 0		;; Shuffle I7 and I8 to create I7/I8 low	;  25

	ylow128s ymm6, ymm9, ymm4		;; Shuffle R5/R6 hi and R7/R8 hi (first R6)	;  26-27
	L1prefetchw L1preg+e3reg, L1pt - L1PREFETCH_DEST_NONE

	ylow128s ymm12, ymm5, ymm2		;; Shuffle I5/I6 hi and I7/I8 hi (first R14)	;  27-28

	yhigh128s ymm9, ymm9, ymm4		;; Shuffle R5/R6 hi and R7/R8 hi (first R8)	;  28-29

	yhigh128s ymm5, ymm5, ymm2		;; Shuffle I5/I6 hi and I7/I8 hi (first R16)	;  29-30
	vaddpd	ymm2, ymm6, ymm12		;; R6 + R14 (new R6)				; 29-31
	L1prefetchw L1p4reg, L1pt - L1PREFETCH_DEST_NONE

	ylow128s ymm4, ymm15, ymm13		;; Shuffle R5/R6 low and R7/R8 low (first R5)	;  30-31
	vsubpd	ymm6, ymm6, ymm12		;; R6 - R14 (new R14)				; 30-32

	ylow128s ymm12, ymm11, ymm14		;; Shuffle I5/I6 low and I7/I8 low (first R13)	;  31-32
	yhigh128s ymm15, ymm15, ymm13		;; Shuffle R5/R6 low and R7/R8 low (first R7)	;  32-33
	vaddpd	ymm13, ymm9, ymm5		;; R8 + R16 (new R8)				; 31-33
	vsubpd	ymm9, ymm9, ymm5		;; R8 - R16 (new R16)				; 32-34
	vmovapd	ymm5, YMM_ONE

	yhigh128s ymm11, ymm11, ymm14		;; Shuffle I5/I6 low and I7/I8 low (first R15)	;  33-34
	yfmaddpd ymm14, ymm4, ymm5, ymm12	;; R5 + R13 (new R5)				; 33-37
	yfmsubpd ymm4, ymm4, ymm5, ymm12	;; R5 - R13 (new R13)				; 33-37
	L1prefetchw L1p4reg+e1reg, L1pt - L1PREFETCH_DEST_NONE

	yfmaddpd ymm12, ymm15, ymm5, ymm11	;; R7 + R15 (new R7)				; 35-39
	yfmsubpd ymm15, ymm15, ymm5, ymm11	;; R7 - R15 (new R15)				; 35-39

	yfmsubpd ymm11, ymm7, ymm5, ymm2	;; R2 - R6 (newer R6)				; 34-38		n 39
	ystore	[dstreg+2*e1reg+32], ymm4	;; Save new R13					; 38
	yfmsubpd ymm4, ymm10, ymm5, ymm13	;; R4 - R8 (newer R8)				; 34-38		n 39

	yfmaddpd ymm7, ymm7, ymm5, ymm2		;; R2 + R6 (newer R2)				; 36-40		n 44
	yfmaddpd ymm10, ymm10, ymm5, ymm13	;; R4 + R8 (newer R4)				; 36-40		n 44
	vmovapd	ymm2, YMM_P924_P383

	yfmaddpd ymm13, ymm3, ymm5, ymm14	;; R1 + R5 (newer R1)				; 38-42		n 45
	yfmsubpd ymm3, ymm3, ymm5, ymm14	;; R1 - R5 (newer R5)				; 38-42		n 46
	L1prefetchw L1p4reg+2*e1reg, L1pt - L1PREFETCH_DEST_NONE

						;; R6/R8 morphs into newer R6/I6
						;; mul R6/I6 by w^2 = .707 + .707i
	yfmsubpd ymm14, ymm11, ymm5, ymm4	;; R6 = R6 - I6					; 39-43		n 46
	yfmaddpd ymm11, ymm11, ymm5, ymm4	;; I6 = R6 + I6					; 39-43		n 47

	yfmaddpd ymm4, ymm0, ymm5, ymm12	;; R3 + R7 (newer R3)				; 40-44		n 45
	yfmsubpd ymm0, ymm0, ymm5, ymm12	;; R3 - R7 (newer R7)				; 40-44		n 47

						;; R10/R14 morphs into newer R10/I10
						;; mul R10/I10 by w^1 = .924 + .383i
	yfmsubpd ymm12, ymm8, ymm2, ymm6	;; R10*.924/.383 - I10 (newer2 R10/.383)	; 41-45		n 48
	yfmaddpd ymm6, ymm6, ymm2, ymm8		;; R10 + I10*.924/.383 (newer2 I10/.383)	; 41-45		n 50
	L1prefetchw L1p4reg+e3reg, L1pt - L1PREFETCH_DEST_NONE

						;; R12/R16 morphs into newer R12/I12
						;; mul R12/I12 by w^3 = .383 + .924i
	yfnmaddpd ymm8, ymm9, ymm2, ymm1	;; R12 - I12*.924/.383 (newer2 R12/.383)	; 42-46		n 48
	yfmaddpd ymm1, ymm1, ymm2, ymm9		;; R12*.924/.383 + I12 (newer2 I12/.383)	; 42-46		n 50

						;; R11/R15 morphs into newer R11/I11
						;; mul R11/I11 by SQRTHALF + i*SQRTHALF
	vmovapd	ymm2, [dstreg]			;; Reload new R11 which morphed into newer R11
	yfmsubpd ymm9, ymm2, ymm5, ymm15	;; R11 = R11 - I11				; 43-47		n 49
	yfmaddpd ymm2, ymm2, ymm5, ymm15	;; I11 = R11 + I11				; 43-47		n 50

	yfmsubpd ymm15, ymm7, ymm5, ymm10	;; R2 - R4 (newest R4)				; 44-48		n 52
	yfmaddpd ymm7, ymm7, ymm5, ymm10	;; R2 + R4 (newest R2)				; 44-48

	yfmsubpd ymm10, ymm13, ymm5, ymm4	;; R1 - R3 (newest R3)				; 45-49		n 52
 	yfmaddpd ymm13, ymm13, ymm5, ymm4	;; R1 + R3 (newest R1)				; 45-49
	vmovapd	ymm4, YMM_SQRTHALF

						;; R1/R2 becomes final R1 and final R2
						;; R5/R7 morphs into newest R5/I5
	ystore	[dstreg+32], ymm7		;; Save R2					; 49
	yfnmaddpd ymm7, ymm14, ymm4, ymm3	;; R5 - R6 * SQRTHALF (final R6)		; 46-50		n 53
	yfmaddpd ymm14, ymm14, ymm4, ymm3	;; R5 + R6 * SQRTHALF (final R5)		; 46-50		n 60

	yfnmaddpd ymm3, ymm11, ymm4, ymm0	;; I5 - I6 * SQRTHALF (final I6)		; 47-51		n 53
	yfmaddpd ymm11, ymm11, ymm4, ymm0	;; I5 + I6 * SQRTHALF (final I5)		; 47-51		n 60

	yfmaddpd ymm0, ymm12, ymm5, ymm8	;; R10 + R12 (newest R10/.383)			; 48-52		n 54
	yfmsubpd ymm12, ymm12, ymm5, ymm8	;; R10 - R12 (newest R12/.383)			; 48-52		n 56

						;; R9/R13 morphs into newer R9/I9
	vmovapd	ymm8, [dstreg+2*e1reg]		;; Reload new R9 which morphed into newer R9
	ystore	[dstreg], ymm13			;; Save R1					; 50
	yfmaddpd ymm13, ymm9, ymm4, ymm8	;; R9 + R11 * SQRTHALF (newest R9)		; 49-53		n 54
	yfnmaddpd ymm9, ymm9, ymm4, ymm8	;; R9 - R11 * SQRTHALF (newest R11)		; 49-53		n 56

	yfmaddpd ymm8, ymm6, ymm5, ymm1		;; I10 + I12 (newest I10/.383)			; 50-54		n 55
	yfmaddpd ymm5, ymm2, ymm4, [dstreg+2*e1reg+32] ;; I9 + I11 * SQRTHALF (newest I9)	; 50-54		n 55

	yfnmaddpd ymm2, ymm2, ymm4, [dstreg+2*e1reg+32] ;; I9 - I11 * SQRTHALF (newest I11)	; 51-55		n 56
	yfmsubpd ymm6, ymm6, YMM_ONE, ymm1	;; I10 - I12 (newest I12/.383)			; 51-55		n 56

						;; R3/R4 morphs into R3/I3
	vmovapd	ymm4, [screg1+128+32]		;; cosine for w^4 (8-complex w^2)
	vmulpd	ymm1, ymm10, ymm4		;; A3 = R3 * cosine				; 52-56		n 57
	vmulpd	ymm4, ymm15, ymm4		;; B3 = I3 * cosine				; 52-56		n 57

	yfnmaddpd ymm15, ymm15, [screg1+128], ymm1 ;; A3 = A3 - I3 * sine (final R3)		; 57-61
	yfmaddpd ymm10, ymm10, [screg1+128], ymm4 ;; B3 = B3 + R3 * sine (final I3)		; 57-61

	vmovapd	ymm4, [screg1+320+32]		;; cosine for w^10 (8-complex w^5)
	vmulpd	ymm1, ymm7, ymm4		;; A6 = R6 * cosine				; 53-57		n 58
	vmulpd	ymm4, ymm3, ymm4		;; B6 = I6 * cosine				; 53-57		n 58

	yfnmaddpd ymm3, ymm3, [screg1+320], ymm1 ;; A6 = A6 - I6 * sine (final R6)		; 58-62
	yfmaddpd ymm7, ymm7, [screg1+320], ymm4 ;; B6 = B6 + R6 * sine (final I6)		; 58-62

	vmovapd	ymm4, YMM_P383
	yfmaddpd ymm1, ymm0, ymm4, ymm13	;; R9 + R10 * .383 (final R9)			; 54-58		n 61
	yfnmaddpd ymm0, ymm0, ymm4, ymm13	;; R9 - R10 * .383 (final R10)			; 54-58		n 62

	yfmaddpd ymm13, ymm8, ymm4, ymm5	;; I9 + I10 * .383 (final I9)			; 55-59		n 61
	yfnmaddpd ymm8, ymm8, ymm4, ymm5	;; I9 - I10 * .383 (final I10)			; 55-59		n 62

	yfnmaddpd ymm5, ymm6, ymm4, ymm9	;; R11 - I12 * .383 (final R11)			; 56-60		n 63
	ystore	[dstreg+e1reg], ymm15		;; Save R3					; 62
	yfmaddpd ymm15, ymm12, ymm4, ymm2	;; I11 + R12 * .383 (final I11)			; 56-60

	yfmaddpd ymm6, ymm6, ymm4, ymm9		;; R11 + I12 * .383 (final R12)			; 59-63		n 64
	yfnmaddpd ymm12, ymm12, ymm4, ymm2	;; I11 - R12 * .383 (final I12)			; 59-63

	vmovapd	ymm9, [screg1+64+32]		;; cosine for w^2 (8-complex w^1)
	vmulpd	ymm2, ymm14, ymm9		;; A5 = R5 * cosine				; 60-64
	vmulpd	ymm9, ymm11, ymm9		;; B5 = I5 * cosine				; 60-64

	vmovapd	ymm4, [screg2+0+32]		;; cosine/sine for w^1
	ystore	[dstreg+e1reg+32], ymm10	;; Save I3					; 62+1
	yfmsubpd ymm10, ymm1, ymm4, ymm13	;; A9 = R9 * cosine/sine - I9			; 61-65
	yfmaddpd ymm13, ymm13, ymm4, ymm1	;; B9 = I9 * cosine/sine + R9			; 61-65

	vmovapd	ymm4, [screg2+128+32]		;; cosine/sine for w^9
	yfmsubpd ymm1, ymm0, ymm4, ymm8		;; A10 = R10 * cosine/sine - I10		; 62-66
	yfmaddpd ymm8, ymm8, ymm4, ymm0		;; B10 = I10 * cosine/sine + R10		; 62-66

	vmovapd	ymm4, [screg2+64+32]		;; cosine/sine for w^5
	yfmsubpd ymm0, ymm5, ymm4, ymm15	;; A11 = R11 * cosine/sine - I11		; 63-67
	yfmaddpd ymm15, ymm15, ymm4, ymm5	;; B11 = I11 * cosine/sine + R11		; 63-67

	vmovapd	ymm4, [screg2+192+32]		;; cosine/sine for w^13
	ystore	[dstreg+e3reg], ymm3		;; Save R6					; 63+1
	yfmsubpd ymm5, ymm6, ymm4, ymm12	;; A12 = R12 * cosine/sine - I12		; 64-68
	yfmaddpd ymm12, ymm12, ymm4, ymm6	;; B12 = I12 * cosine/sine + R12		; 64-68

	vmovapd	ymm4, [screg1+64]		;; sine for w^2 (8-complex w^1)
	ystore	[dstreg+e3reg+32], ymm7		;; Save I6					; 63+2
	yfnmaddpd ymm11, ymm11, ymm4, ymm2	;; A5 = A5 - I5 * sine (final R5)		; 65-69
	yfmaddpd ymm14, ymm14, ymm4, ymm9	;; B5 = B5 + R5 * sine (final I5)		; 65-69

	vmovapd	ymm4, [screg2+0]		;; sine for w^1
	vmulpd	ymm10, ymm10, ymm4		;; A9 = A9 * sine (final R9)			; 66-70
	vmulpd	ymm13, ymm13, ymm4		;; B9 = B9 * sine (final I9)			; 66-70

	vmovapd	ymm4, [screg2+128]		;; sine for w^9
	vmulpd	ymm1, ymm1, ymm4		;; A10 = A10 * sine (final R10)			; 67-71
	vmulpd	ymm8, ymm8, ymm4		;; B10 = B10 * sine (final I10)			; 67-71

	vmovapd	ymm4, [screg2+64]		;; sine for w^5
	vmulpd	ymm0, ymm0, ymm4		;; A11 = A11 * sine (final R11)			; 68-72
	vmulpd	ymm15, ymm15, ymm4		;; B11 = B11 * sine (final I11)			; 68-72

	vmovapd	ymm4, [screg2+192]		;; sine for w^13
	vmulpd	ymm5, ymm5, ymm4		;; A12 = A12 * sine (final R12)			; 69-73
	vmulpd	ymm12, ymm12, ymm4		;; B12 = B12 * sine (final I12)			; 69-73

	ystore	[dstreg+2*e1reg], ymm11		;; Save R5					; 70
	ystore	[dstreg+2*e1reg+32], ymm14	;; Save I5					; 70+1
	ystore	[dst4reg], ymm10		;; Save R9					; 71+1
	ystore	[dst4reg+32], ymm13		;; Save I9					; 71+2
	ystore	[dst4reg+e1reg], ymm1		;; Save R10					; 72+2
	ystore	[dst4reg+e1reg+32], ymm8	;; Save I10					; 72+3
	ystore	[dst4reg+2*e1reg], ymm0		;; Save R11					; 73+3
	ystore	[dst4reg+2*e1reg+32], ymm15	;; Save I11					; 73+4
	ystore	[dst4reg+e3reg], ymm5		;; Save R12					; 74+4
	ystore	[dst4reg+e3reg+32], ymm12	;; Save I12					; 74+5

	bump	srcreg, srcinc
	bump	dstreg, dstinc
	bump	dst4reg, dstinc
	bump	screg1, scinc1
	bump	screg2, scinc2
	bump	L1preg, dstinc
	bump	L1p4reg, dstinc
	ENDM

ENDIF

ENDIF




yr8_rsc_sg8cl_2sc_sixteen_reals_unfft8_preload MACRO
	ENDM
yr8_rsc_sg8cl_2sc_sixteen_reals_unfft8 MACRO srcreg,srcinc,d1,d2,d4,dstreg,dstinc,e1,e2,e4,screg1,scinc1,screg2,scinc2,maxrpt,L1pt,L1pd

	vmovapd	ymm3, [screg1+64+32]		;; cosine for w^2 (8-complex w^1)
	vmovapd	ymm1, [srcreg+d2]		;; R5
	vmulpd	ymm2, ymm1, ymm3		;; A5 = R5 * cosine
	vmovapd	ymm0, [srcreg+d2+32]		;; I5
	vmulpd	ymm3, ymm0, ymm3		;; B5 = I5 * cosine
	vmovapd ymm4, [screg1+64]		;; sine for w^2 (8-complex w^1)
	vmulpd	ymm0, ymm0, ymm4		;; C5 = I5 * sine
	vmulpd	ymm1, ymm1, ymm4		;; D5 = R5 * sine
	vaddpd	ymm2, ymm2, ymm0		;; A5 = A5 + C5 (first R5)
	vsubpd	ymm3, ymm3, ymm1		;; B5 = B5 - D5 (first I5)

	vmovapd	ymm7, [screg1+320+32]		;; cosine for w^10 (8-complex w^5)
	vmovapd	ymm5, [srcreg+d2+d1]		;; R6
	vmulpd	ymm6, ymm5, ymm7		;; A6 = R6 * cosine
	vmovapd	ymm4, [srcreg+d2+d1+32]		;; I6
	vmulpd	ymm7, ymm4, ymm7		;; B6 = I6 * cosine
	vmulpd	ymm4, ymm4, [screg1+320]	;; C6 = I6 * sine
	vmulpd	ymm5, ymm5, [screg1+320]	;; D6 = R6 * sine
	vaddpd	ymm6, ymm6, ymm4		;; A6 = A6 + C6 (first R6)
	vsubpd	ymm7, ymm7, ymm5		;; B6 = B6 - D6 (first I6)

	vmovapd	ymm5, [screg1+128+32]		;; cosine for w^4 (8-complex w^2)
	vmovapd	ymm1, [srcreg+d1]		;; R3
	vmulpd	ymm4, ymm1, ymm5		;; A3 = R3 * cosine
	vmovapd	ymm0, [srcreg+d1+32]		;; I3
	vmulpd	ymm5, ymm0, ymm5		;; B3 = I3 * cosine
	vmulpd	ymm0, ymm0, [screg1+128]	;; C3 = I3 * sine
	vmulpd	ymm1, ymm1, [screg1+128]	;; D3 = R3 * sine
	vaddpd	ymm4, ymm4, ymm0		;; A3 = A3 + C3 (first R3)
	vsubpd	ymm5, ymm5, ymm1		;; B3 = B3 - D3 (first I3)

						;; R3/I3 morphs into R3/R4

	vaddpd	ymm1, ymm2, ymm6		;; R5 + R6 (new R5)
	vsubpd	ymm2, ymm2, ymm6		;; R5 - R6 (new R6)

	L1prefetch srcreg+L1pd, L1pt

	vaddpd	ymm0, ymm3, ymm7		;; I5 + I6 (new I5)
	vsubpd	ymm3, ymm3, ymm7		;; I5 - I6 (new I6)

	vmovapd	ymm6, [srcreg]			;; R1
	vaddpd	ymm7, ymm6, ymm4		;; R1 + R3 (new R1)
	vsubpd	ymm6, ymm6, ymm4		;; R1 - R3 (new R3)

						;; mul R6/I6 by w^2 = .707 - .707i
	vaddpd	ymm4, ymm3, ymm2		;; R6 = I6 + R6
	vsubpd	ymm3, ymm3, ymm2		;; I6 = I6 - R6
	vmovapd	ymm2, YMM_SQRTHALF
	vmulpd	ymm4, ymm4, ymm2		;; R6 = R6 * SQRTHALF
	vmulpd	ymm3, ymm3, ymm2		;; I6 = I6 * SQRTHALF

						;; R5/I5 morphs into new R5/R7
						;; R6/I6 morph into new R6/R8

	vaddpd	ymm2, ymm7, ymm1		;; R1 + R5 (newer R1)
	vsubpd	ymm7, ymm7, ymm1		;; R1 - R5 (newer R5)

	vmovapd	ymm1, [srcreg+32]		;; R2
	ystore	[dstreg], ymm2			;; Save newer R1
	vaddpd	ymm2, ymm1, ymm5		;; R2 + R4 (new R2)
	vsubpd	ymm1, ymm1, ymm5		;; R2 - R4 (new R4)

	vaddpd	ymm5, ymm6, ymm0		;; R3 + R7 (newer R3)
	vsubpd	ymm6, ymm6, ymm0		;; R3 - R7 (newer R7)

	L1prefetch srcreg+d1+L1pd, L1pt

	vaddpd	ymm0, ymm2, ymm4		;; R2 + R6 (newer R2)
	vsubpd	ymm2, ymm2, ymm4		;; R2 - R6 (newer R6)

	vaddpd	ymm4, ymm1, ymm3		;; R4 + R8 (newer R4)
	vsubpd	ymm1, ymm1, ymm3		;; R4 - R8 (newer R8)

	vmovapd	ymm3, [screg2+0+32]		;; cosine/sine for w^1
	ystore	[dstreg+e4], ymm7		;; Save newer R5
	vmovapd	ymm7, [srcreg+d4]		;; R9
	ystore	[dstreg+e1], ymm5		;; Save newer R3
	vmulpd	ymm5, ymm7, ymm3		;; A9 = R9 * cosine/sine
	ystore	[dstreg+e4+e1], ymm6		;; Save newer R7
	vmovapd	ymm6, [srcreg+d4+32]		;; I9
	vaddpd	ymm5, ymm5, ymm6		;; A9 = A9 + I9
	vmulpd	ymm6, ymm6, ymm3		;; B9 = I9 * cosine/sine
	vsubpd	ymm6, ymm6, ymm7		;; B9 = B9 - R9

	vmovapd	ymm3, [screg2+128+32]		;; cosine/sine for w^9
	vmovapd	ymm7, [srcreg+d4+d1]		;; R10
	ystore	[dstreg+32], ymm0		;; Save newer R2
	vmulpd	ymm0, ymm7, ymm3		;; A10 = R10 * cosine/sine
	ystore	[dstreg+e4+32], ymm2		;; Save newer R6
	vmovapd	ymm2, [srcreg+d4+d1+32]		;; I10
	vaddpd	ymm0, ymm0, ymm2		;; A10 = A10 + I10
	vmulpd	ymm2, ymm2, ymm3		;; B10 = I10 * cosine/sine
	vsubpd	ymm2, ymm2, ymm7		;; B10 = B10 - R10

	vmovapd ymm3, [screg2+0]		;; sine for w^1
	vmulpd	ymm5, ymm5, ymm3		;; A9 = A9 * sine (first R9)
	vmulpd	ymm6, ymm6, ymm3		;; B9 = B9 * sine (first I9)

	vmovapd ymm7, [screg2+128]		;; sine for w^9
	vmulpd	ymm0, ymm0, ymm7		;; A10 = A10 * sine (first R10)
	vmulpd	ymm2, ymm2, ymm7		;; B10 = B10 * sine (first I10)

	vaddpd	ymm3, ymm5, ymm0		;; R9 + R10 (new R9)
	vsubpd	ymm5, ymm5, ymm0		;; R9 - R10 (new R10)

	L1prefetch srcreg+d2+L1pd, L1pt

	vaddpd	ymm7, ymm6, ymm2		;; I9 + I10 (new I9)
	vsubpd	ymm6, ymm6, ymm2		;; I9 - I10 (new I10)

	vmovapd	ymm0, [screg2+64+32]		;; cosine/sine for w^5
	vmovapd	ymm2, [srcreg+d4+d2]		;; R11
	ystore	[dstreg+e1+32], ymm4		;; Save newer R4
	vmulpd	ymm4, ymm2, ymm0		;; A11 = R11 * cosine/sine
	ystore	[dstreg+e4+e1+32], ymm1		;; Save newer R8
	vmovapd	ymm1, [srcreg+d4+d2+32]		;; I11
	vaddpd	ymm4, ymm4, ymm1		;; A11 = A11 + I11
	vmulpd	ymm1, ymm1, ymm0		;; B11 = I11 * cosine/sine
	vsubpd	ymm1, ymm1, ymm2		;; B11 = B11 - R11

	vmovapd	ymm0, [screg2+192+32]		;; cosine/sine for w^13
	vmovapd	ymm2, [srcreg+d4+d2+d1]		;; R12
	ystore	[dstreg+e2+e1], ymm5		;; Save new R10
	vmulpd	ymm5, ymm2, ymm0		;; A12 = R12 * cosine/sine
	ystore	[dstreg+e2+e1+32], ymm6		;; Save new I10
	vmovapd	ymm6, [srcreg+d4+d2+d1+32]	;; I12
	vaddpd	ymm5, ymm5, ymm6		;; A12 = A12 + I12
	vmulpd	ymm6, ymm6, ymm0		;; B12 = I12 * cosine/sine
	vsubpd	ymm6, ymm6, ymm2		;; B12 = B12 - R12

	vmovapd ymm0, [screg2+64]		;; sine for w^5
	vmulpd	ymm4, ymm4, ymm0		;; A11 = A11 * sine (first R11)
	vmulpd	ymm1, ymm1, ymm0		;; B11 = B11 * sine (first I11)

	vmovapd ymm2, [screg2+192]		;; sine for w^13
	vmulpd	ymm5, ymm5, ymm2		;; A12 = A12 * sine (first R12)
	vmulpd	ymm6, ymm6, ymm2		;; B12 = B12 * sine (first I12)

	vaddpd	ymm2, ymm5, ymm4		;; R12 + R11 (new R11)
	vsubpd	ymm5, ymm5, ymm4		;; R12 - R11 (new I12)

	L1prefetch srcreg+d2+d1+L1pd, L1pt

	vaddpd	ymm0, ymm1, ymm6		;; I11 + I12 (new I11)
	vsubpd	ymm1, ymm1, ymm6		;; I11 - I12 (new R12)

	vaddpd	ymm4, ymm3, ymm2		;; R9 + R11 (newer R9)
	vsubpd	ymm3, ymm3, ymm2		;; R9 - R11 (newer R11)

	vaddpd	ymm6, ymm7, ymm0		;; I9 + I11 (newer I9)
	vsubpd	ymm7, ymm7, ymm0		;; I9 - I11 (newer I11)

						;; mul R11/I11 by SQRTHALF - i*SQRTHALF
	vaddpd	ymm2, ymm7, ymm3		;; R11 = I11 + R11
	vsubpd	ymm7, ymm7, ymm3		;; I11 = I11 - R11
	vmovapd ymm0, YMM_SQRTHALF
	vmulpd	ymm2, ymm2, ymm0		;; R11 = R11 * SQRTHALF
	vmulpd	ymm7, ymm7, ymm0		;; I11 = I11 * SQRTHALF

						;; R9/I9 morphs into newer R9/R13
						;; R11/I11 morphs into newer R11/R15

	vmovapd	ymm0, [dstreg+e2+e1]		;; Reload new R10
	vaddpd	ymm3, ymm0, ymm1		;; R10 + R12 (newer R10)
	vsubpd	ymm0, ymm0, ymm1		;; R10 - R12 (newer R12)

	vmovapd	ymm1, [dstreg+e2+e1+32]		;; Reload new I10
	ystore	[dstreg+e2], ymm4		;; Save newer R9
	vaddpd	ymm4, ymm1, ymm5		;; I10 + I12 (newer I10)
	vsubpd	ymm1, ymm1, ymm5		;; I10 - I12 (newer I12)


						;; mul R10/I10 by w^1 = .924 - .383i
	vmovapd ymm5, YMM_P924
	ystore	[dstreg+e4+e2], ymm6		;; Save newer R13
	vmulpd	ymm6, ymm3, ymm5		;; R10 * .924
	ystore	[dstreg+e2+e1], ymm2		;; Save newer R11
	vmovapd ymm2, YMM_P383
	ystore	[dstreg+e4+e2+e1], ymm7		;; Save newer R15
	vmulpd	ymm7, ymm4, ymm2		;; I10 * .383
	vmulpd	ymm3, ymm3, ymm2		;; R10 * .383
	vmulpd	ymm4, ymm4, ymm5		;; I10 * .924
	vaddpd	ymm6, ymm6, ymm7		;; Twiddled R10 = R10 * .924 + I10 * .383
	vsubpd	ymm3, ymm4, ymm3		;; Twiddled I10 = I10 * .924 - R10 * .383

	L1prefetch srcreg+d4+L1pd, L1pt

						;; mul R12/I12 by w^3 = .383 - .924i
	vmulpd	ymm7, ymm0, ymm2		;; R12 * .383
	vmulpd	ymm4, ymm1, ymm5		;; I12 * .924
	vmulpd	ymm0, ymm0, ymm5		;; R12 * .924
	vmulpd	ymm1, ymm1, ymm2		;; I12 * .383
	vaddpd	ymm7, ymm7, ymm4		;; Twiddled R12 = R12 * .383 + I12 * .924
	vsubpd	ymm0, ymm1, ymm0		;; Twiddled I12 = I12 * .383 - R12 * .924

						;; R10/I10 morphs into newer R10/R14
						;; R12/I12 morphs into newer R12/R16

	;; Do last level, shuffle and store

	vmovapd	ymm5, [dstreg]			;; Reload newer R1
	vmovapd	ymm2, [dstreg+e2]		;; Reload newer R9
	vaddpd	ymm4, ymm5, ymm2		;; R1 + R9 (last R1)
	vsubpd	ymm5, ymm5, ymm2		;; R1 - R9 (last R9)

	vmovapd	ymm1, [dstreg+32]		;; Reload newer R2
	vaddpd	ymm2, ymm1, ymm6		;; R2 + R10 (last R2)
	vsubpd	ymm1, ymm1, ymm6		;; R2 - R10 (last R10)

	vmovapd	ymm6, [dstreg+e1]		;; Reload newer R3
	ystore	[dstreg+e4+e2+32], ymm3		;; Save newer R14
	vmovapd	ymm3, [dstreg+e2+e1]		;; Reload newer R11
	ystore	[dstreg+e4+e2+e1+32], ymm0	;; Save newer R16
	vaddpd	ymm0, ymm6, ymm3		;; R3 + R11 (last R3)
	vsubpd	ymm6, ymm6, ymm3		;; R3 - R11 (last R11)

	vmovapd	ymm3, [dstreg+e1+32]		;; Reload newer R4
	ystore	[dstreg+32], ymm5		;; Save last R9
	vaddpd	ymm5, ymm3, ymm7		;; R4 + R12 (last R4)
	vsubpd	ymm3, ymm3, ymm7		;; R4 - R12 (last R12)

	;; Swizzle using 4 shufpds and 4 perm2f128  (1 or 3 clocks each on port 5, throughput 1)

	L1prefetch srcreg+d4+d1+L1pd, L1pt

	vshufpd	ymm7, ymm4, ymm2, 0		;; Shuffle R1 and R2 to create R1/R2 low
	vshufpd	ymm4, ymm4, ymm2, 15		;; Shuffle R1 and R2 to create R1/R2 hi

	vshufpd	ymm2, ymm0, ymm5, 0		;; Shuffle R3 and R4 to create R3/R4 low
	vshufpd	ymm0, ymm0, ymm5, 15		;; Shuffle R3 and R4 to create R3/R4 hi

	ylow128s ymm5, ymm7, ymm2		;; Shuffle R1/R2 low and R3/R4 low (final R1)
	yhigh128s ymm7, ymm7, ymm2		;; Shuffle R1/R2 low and R3/R4 low (final R3)

	ylow128s ymm2, ymm4, ymm0		;; Shuffle R1/R2 hi and R3/R4 hi (final R2)
	yhigh128s ymm4, ymm4, ymm0		;; Shuffle R1/R2 hi and R3/R4 hi (final R4)

	;; Swizzle using 4 shufpds and 4 perm2f128  (1 or 3 clocks each on port 5, throughput 1)

	vmovapd	ymm0, [dstreg+32]		;; Reload last R9
	ystore	[dstreg], ymm5			;; Save R1

	vshufpd	ymm5, ymm0, ymm1, 0		;; Shuffle R9 and R10 to create R9/R10 low
	vshufpd	ymm0, ymm0, ymm1, 15		;; Shuffle R9 and R10 to create R9/R10 hi

	vshufpd	ymm1, ymm6, ymm3, 0		;; Shuffle R11 and R12 to create R11/R12 low
	vshufpd	ymm6, ymm6, ymm3, 15		;; Shuffle R11 and R12 to create R11/R12 hi

	L1prefetch srcreg+d4+d2+L1pd, L1pt

	ylow128s ymm3, ymm5, ymm1		;; Shuffle R9/R10 low and R11/R12 low (final R9)
	yhigh128s ymm5, ymm5, ymm1		;; Shuffle R9/R10 low and R11/R12 low (final R11)

	ylow128s ymm1, ymm0, ymm6		;; Shuffle R9/R10 hi and R11/R12 hi (final R10)
	yhigh128s ymm0, ymm0, ymm6		;; Shuffle R9/R10 hi and R11/R12 hi (final R12)

	vmovapd	ymm6, [dstreg+e4]		;; Reload newer R5
	ystore	[dstreg+e2], ymm7		;; Save R3
	vmovapd	ymm7, [dstreg+e4+e2]		;; Reload newer R13
	ystore	[dstreg+e1], ymm2		;; Save R2
	vaddpd	ymm2, ymm6, ymm7		;; R5 + R13 (last R5)
	vsubpd	ymm6, ymm6, ymm7		;; R5 - R13 (last R13)

	vmovapd	ymm7, [dstreg+e4+32]		;; Reload newer R6
	ystore	[dstreg+e2+e1], ymm4		;; Save R4
	vmovapd	ymm4, [dstreg+e4+e2+32]		;; Reload newer R14
	ystore	[dstreg+32], ymm3		;; Save R9
	vaddpd	ymm3, ymm7, ymm4		;; R6 + R14 (last R6)
	vsubpd	ymm7, ymm7, ymm4		;; R6 - R14 (last R14)

	vmovapd	ymm4, [dstreg+e4+e1]		;; Reload newer R7
	ystore	[dstreg+e2+32], ymm5		;; Save R11
	vmovapd	ymm5, [dstreg+e4+e2+e1]		;; Reload newer R15
	ystore	[dstreg+e1+32], ymm1		;; Save R10
	vaddpd	ymm1, ymm4, ymm5		;; R7 + R15 (last R7)
	vsubpd	ymm4, ymm4, ymm5		;; R7 - R15 (last R15)

	vmovapd	ymm5, [dstreg+e4+e1+32]		;; Reload newer R8
	ystore	[dstreg+e2+e1+32], ymm0		;; Save R12
	vmovapd	ymm0, [dstreg+e4+e2+e1+32]	;; Reload newer R16
	ystore	[dstreg+e4], ymm6		;; Save last R13
	vaddpd	ymm6, ymm5, ymm0		;; R8 + R16 (last R8)
	vsubpd	ymm5, ymm5, ymm0		;; R8 - R16 (last R16)

	;; Swizzle using 4 shufpds and 4 perm2f128  (1 or 3 clocks each on port 5, throughput 1)

	vshufpd	ymm0, ymm2, ymm3, 0		;; Shuffle R5 and R6 to create R5/R6 low
	vshufpd	ymm2, ymm2, ymm3, 15		;; Shuffle R5 and R6 to create R5/R6 hi

	vshufpd	ymm3, ymm1, ymm6, 0		;; Shuffle R7 and R8 to create R7/R8 low
	vshufpd	ymm1, ymm1, ymm6, 15		;; Shuffle R7 and R8 to create R7/R8 hi

	L1prefetch srcreg+d4+d2+d1+L1pd, L1pt

	ylow128s ymm6, ymm0, ymm3		;; Shuffle R5/R6 low and R7/R8 low (final R5)
	yhigh128s ymm0, ymm0, ymm3		;; Shuffle R5/R6 low and R7/R8 low (final R7)

	ylow128s ymm3, ymm2, ymm1		;; Shuffle R5/R6 hi and R7/R8 hi (final R6)
	yhigh128s ymm2, ymm2, ymm1		;; Shuffle R5/R6 hi and R7/R8 hi (final R8)

	;; Swizzle using 4 shufpds and 4 perm2f128  (1 or 3 clocks each on port 5, throughput 1)

	vmovapd	ymm1, [dstreg+e4]		;; Reload last R13
	ystore	[dstreg+e4], ymm6		;; Save R5
	vshufpd	ymm6, ymm1, ymm7, 0		;; Shuffle R13 and R14 to create R13/R14 low
	vshufpd	ymm1, ymm1, ymm7, 15		;; Shuffle R13 and R14 to create R13/R14 hi

	vshufpd	ymm7, ymm4, ymm5, 0		;; Shuffle R15 and R16 to create R15/R16 low
	vshufpd	ymm4, ymm4, ymm5, 15		;; Shuffle R15 and R16 to create R15/R16 hi

	ylow128s ymm5, ymm6, ymm7		;; Shuffle R13/R14 low and R15/R16 low (final R13)
	yhigh128s ymm6, ymm6, ymm7		;; Shuffle R13/R14 low and R15/R16 low (final R15)

	ylow128s ymm7, ymm1, ymm4		;; Shuffle R13/R14 hi and R15/R16 hi (final R14)
	yhigh128s ymm1, ymm1, ymm4		;; Shuffle R13/R14 hi and R15/R16 hi (final R16)

	ystore	[dstreg+e4+e2], ymm0		;; Save R7
	ystore	[dstreg+e4+e1], ymm3		;; Save R6
	ystore	[dstreg+e4+e2+e1], ymm2		;; Save R8
	ystore	[dstreg+e4+32], ymm5		;; Save R13
	ystore	[dstreg+e4+e2+32], ymm6		;; Save R15
	ystore	[dstreg+e4+e1+32], ymm7		;; Save R14
	ystore	[dstreg+e4+e2+e1+32], ymm1	;; Save R16

	bump	srcreg, srcinc
	bump	dstreg, dstinc
	bump	screg1, scinc1
	bump	screg2, scinc2
	ENDM

IFDEF X86_64

yr8_rsc_sg8cl_2sc_sixteen_reals_unfft8_preload MACRO
	ENDM
yr8_rsc_sg8cl_2sc_sixteen_reals_unfft8 MACRO srcreg,srcinc,d1,d2,d4,dstreg,dstinc,e1,e2,e4,screg1,scinc1,screg2,scinc2,maxrpt,L1pt,L1pd
	vmovapd	ymm0, [screg2+64+32]		;; cosine/sine for w^5
	vmovapd	ymm1, [srcreg+d4+d2]		;; R11
	vmulpd	ymm2, ymm1, ymm0		;; A11 = R11 * cosine/sine			;  1-5

	vmovapd	ymm3, [srcreg+d4+d2+32]		;; I11
	vmulpd	ymm0, ymm3, ymm0		;; B11 = I11 * cosine/sine			;  2-6

	vmovapd	ymm4, [screg2+192+32]		;; cosine/sine for w^13
	vmovapd	ymm5, [srcreg+d4+d2+d1]		;; R12
	vmulpd	ymm6, ymm5, ymm4		;; A12 = R12 * cosine/sine			;  3-7

	vmovapd	ymm7, [srcreg+d4+d2+d1+32]	;; I12
	vmulpd	ymm4, ymm7, ymm4		;; B12 = I12 * cosine/sine			;  4-8

	vmovapd	ymm8, [screg2+0+32]		;; cosine/sine for w^1
	vmovapd	ymm9, [srcreg+d4]		;; R9
	vmulpd	ymm10, ymm9, ymm8		;; A9 = R9 * cosine/sine			;  5-9

	vmovapd	ymm11, [srcreg+d4+32]		;; I9
	vaddpd	ymm2, ymm2, ymm3		;; A11 = A11 + I11				; 6-8
	vmulpd	ymm8, ymm11, ymm8		;; B9 = I9 * cosine/sine			;  6-10

	vmovapd	ymm12, [screg2+128+32]		;; cosine/sine for w^9
	vmovapd	ymm13, [srcreg+d4+d1]		;; R10
	vsubpd	ymm0, ymm0, ymm1		;; B11 = B11 - R11				; 7-9
	vmulpd	ymm14, ymm13, ymm12		;; A10 = R10 * cosine/sine			;  7-11

	vmovapd	ymm15, [srcreg+d4+d1+32]	;; I10
	vaddpd	ymm6, ymm6, ymm7		;; A12 = A12 + I12				; 8-10
	vmulpd	ymm12, ymm15, ymm12		;; B10 = I10 * cosine/sine			;  8-12

	vmovapd ymm3, [screg2+64]		;; sine for w^5
	vsubpd	ymm4, ymm4, ymm5		;; B12 = B12 - R12				; 9-11
	vmulpd	ymm2, ymm2, ymm3		;; A11 = A11 * sine (first R11)			;  9-13
	vmovapd ymm1, [screg2+192]		;; sine for w^13

	vaddpd	ymm10, ymm10, ymm11		;; A9 = A9 + I9					; 10-12
	vmulpd	ymm0, ymm0, ymm3		;; B11 = B11 * sine (first I11)			;  10-14
	vmovapd ymm7, [screg2+0]		;; sine for w^1

	vsubpd	ymm8, ymm8, ymm9		;; B9 = B9 - R9					; 11-13
	vmulpd	ymm6, ymm6, ymm1		;; A12 = A12 * sine (first R12)			;  11-15
	vmovapd ymm5, [screg2+128]		;; sine for w^9

	vaddpd	ymm14, ymm14, ymm15		;; A10 = A10 + I10				; 12-14
	vmulpd	ymm4, ymm4, ymm1		;; B12 = B12 * sine (first I12)			;  12-16
	vmovapd	ymm11, [screg1+64+32]		;; cosine for w^2 (8-complex w^1)

	vsubpd	ymm12, ymm12, ymm13		;; B10 = B10 - R10				; 13-15
	vmulpd	ymm10, ymm10, ymm7		;; A9 = A9 * sine (first R9)			;  13-17
	vmovapd	ymm3, [srcreg+d2]		;; R5

	vmulpd	ymm8, ymm8, ymm7		;; B9 = B9 * sine (first I9)			;  14-18
	vmovapd	ymm9, [srcreg+d2+32]		;; I5

	vmulpd	ymm14, ymm14, ymm5		;; A10 = A10 * sine (first R10)			;  15-19
	vmovapd ymm15, [screg1+64]		;; sine for w^2 (8-complex w^1)

	vaddpd	ymm7, ymm6, ymm2		;; R12 + R11 (new R11)				; 16-18
	vmulpd	ymm12, ymm12, ymm5		;; B10 = B10 * sine (first I10)			;  16-20
	vmovapd	ymm1, [screg1+320+32]		;; cosine for w^10 (8-complex w^5)

	vsubpd	ymm6, ymm6, ymm2		;; R12 - R11 (new I12)				; 17-19
	vmulpd	ymm2, ymm3, ymm11		;; A5 = R5 * cosine				;  17-21
	vmovapd	ymm13, [srcreg+d2+d1]		;; R6

	vaddpd	ymm5, ymm0, ymm4		;; I11 + I12 (new I11)				; 18-20
	vmulpd	ymm11, ymm9, ymm11		;; B5 = I5 * cosine				;  18-22

	vsubpd	ymm0, ymm0, ymm4		;; I11 - I12 (new R12)				; 19-21
	vmulpd	ymm9, ymm9, ymm15		;; C5 = I5 * sine				;  19-23

	vaddpd	ymm4, ymm10, ymm14		;; R9 + R10 (new R9)				; 20-22
	vmulpd	ymm3, ymm3, ymm15		;; D5 = R5 * sine				;  20-24

	vsubpd	ymm10, ymm10, ymm14		;; R9 - R10 (new R10)				; 21-23
	vmulpd	ymm14, ymm13, ymm1		;; A6 = R6 * cosine				;  21-25

	vaddpd	ymm15, ymm8, ymm12		;; I9 + I10 (new I9)				; 22-24
	vsubpd	ymm8, ymm8, ymm12		;; I9 - I10 (new I10)				; 23-25
	vmovapd	ymm12, [srcreg+d2+d1+32]	;; I6
	vmulpd	ymm1, ymm12, ymm1		;; B6 = I6 * cosine				;  22
	vaddpd	ymm2, ymm2, ymm9		;; A5 = A5 + C5 (first R5)			; 24-26
	vmovapd	ymm9, [screg1+320]		;; sine for w^10 (8-complex w^5)
	vmulpd	ymm12, ymm12, ymm9		;; C6 = I6 * sine				;  23-27
	vmulpd	ymm13, ymm13, ymm9		;; D6 = R6 * sine				;  24-28

	vsubpd	ymm11, ymm11, ymm3		;; B5 = B5 - D5 (first I5)			; 25-27
	vmovapd	ymm9, [screg1+128+32]		;; cosine for w^4 (8-complex w^2)

	vaddpd	ymm3, ymm10, ymm0		;; R10 + R12 (newer R10)			; 26-28

	vsubpd	ymm10, ymm10, ymm0		;; R10 - R12 (newer R12)			; 27-29
	vmovapd	ymm0, [srcreg+d1]		;; R3

	vaddpd	ymm14, ymm14, ymm12		;; A6 = A6 + C6 (first R6)			; 28-30

	vsubpd	ymm1, ymm1, ymm13		;; B6 = B6 - D6 (first I6)			; 29-31
	L1prefetch srcreg+L1pd, L1pt

	vaddpd	ymm13, ymm8, ymm6		;; I10 + I12 (newer I10)			; 30-32

	vsubpd	ymm8, ymm8, ymm6		;; I10 - I12 (newer I12)			; 31-33

	vsubpd	ymm6, ymm2, ymm14		;; R5 - R6 (new R6)				; 32-34
	L1prefetch srcreg+d1+L1pd, L1pt

	vaddpd	ymm2, ymm2, ymm14		;; R5 + R6 (new R5)				; 33-35
	vmulpd	ymm14, ymm0, ymm9		;; A3 = R3 * cosine				;  33-37

	vsubpd	ymm12, ymm11, ymm1		;; I5 - I6 (new I6)				; 34-36
	vaddpd	ymm11, ymm11, ymm1		;; I5 + I6 (new I5)				; 35-37
	vmovapd	ymm1, [srcreg+d1+32]		;; I3
	vmulpd	ymm9, ymm1, ymm9		;; B3 = I3 * cosine				;  34-38
	ystore	[dstreg+e1], ymm11		;; Temp save new I5 which morphs into newer R7	; 38
	vmovapd	ymm11, [screg1+128]		;; sine for w^4 (8-complex w^2)
	vmulpd	ymm0, ymm0, ymm11		;; D3 = R3 * sine				;  35-39

	vmulpd	ymm1, ymm1, ymm11		;; C3 = I3 * sine				;  36-40
	vsubpd	ymm11, ymm4, ymm7		;; R9 - R11 (newer R11)				; 36-38

	vaddpd	ymm4, ymm4, ymm7		;; R9 + R11 (newer R9)				; 37-39
	L1prefetch srcreg+d2+L1pd, L1pt

	vsubpd	ymm7, ymm15, ymm5		;; I9 - I11 (newer I11)				; 38-40

						;; mul R10/I10 by w^1 = .924 - .383i
	vaddpd	ymm15, ymm15, ymm5		;; I9 + I11 (newer I9)				; 39-41
	vmovapd ymm5, YMM_P924
	ystore	[dstreg], ymm4			;; Temp save newer R9				; 40
	vmulpd	ymm4, ymm3, ymm5		;; R10 * .924					;  39-43

	vaddpd	ymm14, ymm14, ymm1		;; A3 = A3 + C3 (first R3)			; 40-42
	vmovapd ymm1, YMM_P383
	vmulpd	ymm3, ymm3, ymm1		;; R10 * .383					;  40-44

	vsubpd	ymm9, ymm9, ymm0		;; B3 = B3 - D3 (first I3)			; 41-43
	vmulpd	ymm0, ymm13, ymm1		;; I10 * .383					;  41-45

						;; mul R6/I6 by w^2 = .707 - .707i
	ystore	[dstreg+32], ymm15		;; Temp save newer I9 which morphs into newer R13 ; 42
	vmulpd	ymm13, ymm13, ymm5		;; I10 * .924					;  42-46
	vaddpd	ymm15, ymm12, ymm6		;; R6 = I6 + R6					; 42-44

						;; mul R12/I12 by w^3 = .383 - .924i
	vsubpd	ymm12, ymm12, ymm6		;; I6 = I6 - R6					; 43-45
	vmulpd	ymm6, ymm10, ymm1		;; R12 * .383					;  43-47
	vmulpd	ymm10, ymm10, ymm5		;; R12 * .924					;  44-48
	vmulpd	ymm5, ymm8, ymm5		;; I12 * .924					;  45-49
	vmulpd	ymm8, ymm8, ymm1		;; I12 * .383					;  46-50
						;; mul R11/I11 by SQRTHALF - i*SQRTHALF
	vaddpd	ymm1, ymm7, ymm11		;; R11 = I11 + R11				; 44-46
	vsubpd	ymm7, ymm7, ymm11		;; I11 = I11 - R11				; 45-47
	vaddpd	ymm4, ymm4, ymm0		;; Twiddled R10 = R10 * .924 + I10 * .383	; 46-48
	vmovapd	ymm11, [srcreg]			;; R1

	vsubpd	ymm13, ymm13, ymm3		;; Twiddled I10 = I10 * .924 - R10 * .383	; 47-49
	vmovapd	ymm0, YMM_SQRTHALF

						;; R3/I3 morphs into R3/R4
	vaddpd	ymm3, ymm11, ymm14		;; R1 + R3 (new R1)				; 48-50

	vsubpd	ymm11, ymm11, ymm14		;; R1 - R3 (new R3)				; 49-51
	vmulpd	ymm15, ymm15, ymm0		;; R6 = R6 * SQRTHALF				;  49-53
	vmovapd	ymm14, [srcreg+32]		;; R2

	vaddpd	ymm6, ymm6, ymm5		;; Twiddled R12 = R12 * .383 + I12 * .924	; 50-52
	vmulpd	ymm12, ymm12, ymm0		;; I6 = I6 * SQRTHALF				;  50-54
	vmovapd	ymm5, [dstreg+e1]		;; Reload new I5 which morphs into newer R7

	vsubpd	ymm8, ymm8, ymm10		;; Twiddled I12 = I12 * .383 - R12 * .924	; 51-53
	vmulpd	ymm1, ymm1, ymm0		;; R11 = R11 * SQRTHALF				;  51

	vaddpd	ymm10, ymm14, ymm9		;; R2 + R4 (new R2)				; 52-54
	vmulpd	ymm7, ymm7, ymm0		;; I11 = I11 * SQRTHALF				;  52-56
	vmovapd	ymm0, [dstreg]			;; Reload newer R9

	vsubpd	ymm14, ymm14, ymm9		;; R2 - R4 (new R4)				; 53-55
	L1prefetch srcreg+d2+d1+L1pd, L1pt

						;; R5/I5 morphs into newer R5/R7
	vaddpd	ymm9, ymm3, ymm2		;; R1 + R5 (newer R1)				; 54-56

	vsubpd	ymm3, ymm3, ymm2		;; R1 - R5 (newer R5)				; 55-57

	vaddpd	ymm2, ymm11, ymm5		;; R3 + R7 (newer R3)				; 56-58
	L1prefetch srcreg+d4+L1pd, L1pt

	vsubpd	ymm11, ymm11, ymm5		;; R3 - R7 (newer R7)				; 57-59

						;; R6/I6 morphs into newer R6/R8
	vaddpd	ymm5, ymm10, ymm15		;; R2 + R6 (newer R2)				; 58-60

	vsubpd	ymm10, ymm10, ymm15		;; R2 - R6 (newer R6)				; 59-61
	L1prefetch srcreg+d4+d1+L1pd, L1pt

	vaddpd	ymm15, ymm14, ymm12		;; R4 + R8 (newer R4)				; 60-62

	vsubpd	ymm14, ymm14, ymm12		;; R4 - R8 (newer R8)				; 61-63

	;; Do last level, shuffle and store

						;; R9/I9 morphs into newer R9/R13
	vaddpd	ymm12, ymm9, ymm0		;; R1 + R9 (last R1)				; 62-64
	L1prefetch srcreg+d4+d2+L1pd, L1pt

	vsubpd	ymm9, ymm9, ymm0		;; R1 - R9 (last R9)				; 63-65

						;; R10/I10 morphs into newer R10/R14
	vaddpd	ymm0, ymm5, ymm4		;; R2 + R10 (last R2)				; 64-66

	vsubpd	ymm5, ymm5, ymm4		;; R2 - R10 (last R10)				; 65-67
	L1prefetch srcreg+d4+d2+d1+L1pd, L1pt

						;; R11/I11 morphs into newer R11/R15
	vaddpd	ymm4, ymm2, ymm1		;; R3 + R11 (last R3)				; 66-68

	vsubpd	ymm2, ymm2, ymm1		;; R3 - R11 (last R11)				; 67-69
	vshufpd	ymm1, ymm12, ymm0, 0		;; Shuffle R1 and R2 to create R1/R2 low	; 67

						;; R12/I12 morphs into newer R12/R16
	vshufpd	ymm12, ymm12, ymm0, 15		;; Shuffle R1 and R2 to create R1/R2 hi		; 68
	vaddpd	ymm0, ymm15, ymm6		;; R4 + R12 (last R4)				; 68-70

	vsubpd	ymm15, ymm15, ymm6		;; R4 - R12 (last R12)				; 69-71
	vshufpd	ymm6, ymm9, ymm5, 0		;; Shuffle R9 and R10 to create R9/R10 low	; 69

	vshufpd	ymm9, ymm9, ymm5, 15		;; Shuffle R9 and R10 to create R9/R10 hi	; 70

	vshufpd	ymm5, ymm4, ymm0, 0		;; Shuffle R3 and R4 to create R3/R4 low	; 71

	vshufpd	ymm4, ymm4, ymm0, 15		;; Shuffle R3 and R4 to create R3/R4 hi		; 72

	vshufpd	ymm0, ymm2, ymm15, 0		;; Shuffle R11 and R12 to create R11/R12 low	; 73

	vshufpd	ymm2, ymm2, ymm15, 15		;; Shuffle R11 and R12 to create R11/R12 hi	; 74

	ylow128s ymm15, ymm1, ymm5		;; Shuffle R1/R2 low and R3/R4 low (final R1)	; 75-76

	yhigh128s ymm1, ymm1, ymm5		;; Shuffle R1/R2 low and R3/R4 low (final R3)	; 76-77

	ystore	[dstreg], ymm15			;; Save R1					; 77
	ylow128s ymm5, ymm12, ymm4		;; Shuffle R1/R2 hi and R3/R4 hi (final R2)	; 77-78
	vmovapd	ymm15, [dstreg+32]		;; Reload newer R13

	ystore	[dstreg+e2], ymm1		;; Save R3					; 78
	yhigh128s ymm12, ymm12, ymm4		;; Shuffle R1/R2 hi and R3/R4 hi (final R4)	; 78-79

	;; Swizzle using 4 shufpds and 4 perm2f128  (1 or 3 clocks each on port 5, throughput 1)

	ystore	[dstreg+e1], ymm5		;; Save R2					; 79
	ylow128s ymm4, ymm6, ymm0		;; Shuffle R9/R10 low and R11/R12 low (final R9) ; 79-80
	vaddpd	ymm1, ymm3, ymm15		;; R5 + R13 (last R5)				; 79-81

	ystore	[dstreg+e2+e1], ymm12		;; Save R4					; 80
	yhigh128s ymm6, ymm6, ymm0		;; Shuffle R9/R10 low and R11/R12 low (final R11) ; 80-81
	vsubpd	ymm3, ymm3, ymm15		;; R5 - R13 (last R13)				; 80-82

	ystore	[dstreg+32], ymm4		;; Save R9					; 81
	ylow128s ymm0, ymm9, ymm2		;; Shuffle R9/R10 hi and R11/R12 hi (final R10)	; 81-82
	vaddpd	ymm15, ymm10, ymm13		;; R6 + R14 (last R6)				; 81-83

	ystore	[dstreg+e2+32], ymm6		;; Save R11					; 82
	yhigh128s ymm9, ymm9, ymm2		;; Shuffle R9/R10 hi and R11/R12 hi (final R12)	; 82-83
	vsubpd	ymm10, ymm10, ymm13		;; R6 - R14 (last R14)				; 82-84

	ystore	[dstreg+e1+32], ymm0		;; Save R10					; 83
	vaddpd	ymm13, ymm11, ymm7		;; R7 + R15 (last R7)				; 83-85

	ystore	[dstreg+e2+e1+32], ymm9		;; Save R12					; 84
	vshufpd	ymm2, ymm1, ymm15, 0		;; Shuffle R5 and R6 to create R5/R6 low	; 84
	vsubpd	ymm11, ymm11, ymm7		;; R7 - R15 (last R15)				; 84-86

	vshufpd	ymm1, ymm1, ymm15, 15		;; Shuffle R5 and R6 to create R5/R6 hi		; 85
	vaddpd	ymm7, ymm14, ymm8		;; R8 + R16 (last R8)				; 85-87

	vshufpd	ymm15, ymm3, ymm10, 0		;; Shuffle R13 and R14 to create R13/R14 low	; 86
	vsubpd	ymm14, ymm14, ymm8		;; R8 - R16 (last R16)				; 86-88

	vshufpd	ymm3, ymm3, ymm10, 15		;; Shuffle R13 and R14 to create R13/R14 hi	; 87

	vshufpd	ymm10, ymm13, ymm7, 0		;; Shuffle R7 and R8 to create R7/R8 low	; 88

	vshufpd	ymm13, ymm13, ymm7, 15		;; Shuffle R7 and R8 to create R7/R8 hi		; 89

	vshufpd	ymm7, ymm11, ymm14, 0		;; Shuffle R15 and R16 to create R15/R16 low	; 90

	vshufpd	ymm11, ymm11, ymm14, 15		;; Shuffle R15 and R16 to create R15/R16 hi	; 91

	ylow128s ymm14, ymm2, ymm10		;; Shuffle R5/R6 low and R7/R8 low (final R5)	; 92-93

	yhigh128s ymm2, ymm2, ymm10		;; Shuffle R5/R6 low and R7/R8 low (final R7)	; 93-94

	ystore	[dstreg+e4], ymm14		;; Save R5					; 94
	ylow128s ymm10, ymm1, ymm13		;; Shuffle R5/R6 hi and R7/R8 hi (final R6)	; 94-95

	ystore	[dstreg+e4+e2], ymm2		;; Save R7					; 95
	yhigh128s ymm1, ymm1, ymm13		;; Shuffle R5/R6 hi and R7/R8 hi (final R8)	; 95-96

	ystore	[dstreg+e4+e1], ymm10		;; Save R6					; 96
	ylow128s ymm13, ymm15, ymm7		;; Shuffle R13/R14 low and R15/R16 low (final R13) ; 96-97

	ystore	[dstreg+e4+e2+e1], ymm1		;; Save R8					; 97
	yhigh128s ymm15, ymm15, ymm7		;; Shuffle R13/R14 low and R15/R16 low (final R15) ; 97-98

	ystore	[dstreg+e4+32], ymm13		;; Save R13					; 98
	ylow128s ymm7, ymm3, ymm11		;; Shuffle R13/R14 hi and R15/R16 hi (final R14) ; 98-99

	ystore	[dstreg+e4+e2+32], ymm15	;; Save R15					; 99
	yhigh128s ymm3, ymm3, ymm11		;; Shuffle R13/R14 hi and R15/R16 hi (final R16) ; 99-100

	ystore	[dstreg+e4+e1+32], ymm7		;; Save R14					; 100
	ystore	[dstreg+e4+e2+e1+32], ymm3	;; Save R16					; 101

	bump	srcreg, srcinc
	bump	dstreg, dstinc
	bump	screg1, scinc1
	bump	screg2, scinc2
	ENDM

ENDIF



; These versions use registers for distances between blocks.  This lets us share pass1 code.

yr8_rsc_sg8clreg_2sc_sixteen_reals_unfft8_preload MACRO
	ENDM
yr8_rsc_sg8clreg_2sc_sixteen_reals_unfft8 MACRO srcreg,srcinc,d1reg,d3reg,src4reg,dstreg,dstinc,e1,e2,e4,screg1,scinc1,screg2,scinc2,maxrpt,L1pt,L1preg,L1p4reg
	NOT IMPLMENTED IN 32-BIT
	ENDM

IFDEF X86_64

yr8_rsc_sg8clreg_2sc_sixteen_reals_unfft8_preload MACRO
	ENDM
yr8_rsc_sg8clreg_2sc_sixteen_reals_unfft8 MACRO srcreg,srcinc,d1reg,d3reg,src4reg,dstreg,dstinc,e1,e2,e4,screg1,scinc1,screg2,scinc2,maxrpt,L1pt,L1preg,L1p4reg
	vmovapd	ymm0, [screg2+64+32]		;; cosine/sine for w^5
	vmovapd	ymm1, [src4reg+2*d1reg]		;; R11
	vmulpd	ymm2, ymm1, ymm0		;; A11 = R11 * cosine/sine			;  1-5

	vmovapd	ymm3, [src4reg+2*d1reg+32]	;; I11
	vmulpd	ymm0, ymm3, ymm0		;; B11 = I11 * cosine/sine			;  2-6

	vmovapd	ymm4, [screg2+192+32]		;; cosine/sine for w^13
	vmovapd	ymm5, [src4reg+d3reg]		;; R12
	vmulpd	ymm6, ymm5, ymm4		;; A12 = R12 * cosine/sine			;  3-7

	vmovapd	ymm7, [src4reg+d3reg+32]	;; I12
	vmulpd	ymm4, ymm7, ymm4		;; B12 = I12 * cosine/sine			;  4-8

	vmovapd	ymm8, [screg2+0+32]		;; cosine/sine for w^1
	vmovapd	ymm9, [src4reg]			;; R9
	vmulpd	ymm10, ymm9, ymm8		;; A9 = R9 * cosine/sine			;  5-9

	vmovapd	ymm11, [src4reg+32]		;; I9
	vaddpd	ymm2, ymm2, ymm3		;; A11 = A11 + I11				; 6-8
	vmulpd	ymm8, ymm11, ymm8		;; B9 = I9 * cosine/sine			;  6-10

	vmovapd	ymm12, [screg2+128+32]		;; cosine/sine for w^9
	vmovapd	ymm13, [src4reg+d1reg]		;; R10
	vsubpd	ymm0, ymm0, ymm1		;; B11 = B11 - R11				; 7-9
	vmulpd	ymm14, ymm13, ymm12		;; A10 = R10 * cosine/sine			;  7-11

	vmovapd	ymm15, [src4reg+d1reg+32]	;; I10
	vaddpd	ymm6, ymm6, ymm7		;; A12 = A12 + I12				; 8-10
	vmulpd	ymm12, ymm15, ymm12		;; B10 = I10 * cosine/sine			;  8-12

	vmovapd ymm3, [screg2+64]		;; sine for w^5
	vsubpd	ymm4, ymm4, ymm5		;; B12 = B12 - R12				; 9-11
	vmulpd	ymm2, ymm2, ymm3		;; A11 = A11 * sine (first R11)			;  9-13
	vmovapd ymm1, [screg2+192]		;; sine for w^13

	vaddpd	ymm10, ymm10, ymm11		;; A9 = A9 + I9					; 10-12
	vmulpd	ymm0, ymm0, ymm3		;; B11 = B11 * sine (first I11)			;  10-14
	vmovapd ymm7, [screg2+0]		;; sine for w^1

	vsubpd	ymm8, ymm8, ymm9		;; B9 = B9 - R9					; 11-13
	vmulpd	ymm6, ymm6, ymm1		;; A12 = A12 * sine (first R12)			;  11-15
	vmovapd ymm5, [screg2+128]		;; sine for w^9

	vaddpd	ymm14, ymm14, ymm15		;; A10 = A10 + I10				; 12-14
	vmulpd	ymm4, ymm4, ymm1		;; B12 = B12 * sine (first I12)			;  12-16
	vmovapd	ymm11, [screg1+64+32]		;; cosine for w^2 (8-complex w^1)

	vsubpd	ymm12, ymm12, ymm13		;; B10 = B10 - R10				; 13-15
	vmulpd	ymm10, ymm10, ymm7		;; A9 = A9 * sine (first R9)			;  13-17
	vmovapd	ymm3, [srcreg+2*d1reg]		;; R5

	vmulpd	ymm8, ymm8, ymm7		;; B9 = B9 * sine (first I9)			;  14-18
	vmovapd	ymm9, [srcreg+2*d1reg+32]	;; I5

	vmulpd	ymm14, ymm14, ymm5		;; A10 = A10 * sine (first R10)			;  15-19
	vmovapd ymm15, [screg1+64]		;; sine for w^2 (8-complex w^1)

	vaddpd	ymm7, ymm6, ymm2		;; R12 + R11 (new R11)				; 16-18
	vmulpd	ymm12, ymm12, ymm5		;; B10 = B10 * sine (first I10)			;  16-20
	vmovapd	ymm1, [screg1+320+32]		;; cosine for w^10 (8-complex w^5)

	vsubpd	ymm6, ymm6, ymm2		;; R12 - R11 (new I12)				; 17-19
	vmulpd	ymm2, ymm3, ymm11		;; A5 = R5 * cosine				;  17-21
	vmovapd	ymm13, [srcreg+d3reg]		;; R6

	vaddpd	ymm5, ymm0, ymm4		;; I11 + I12 (new I11)				; 18-20
	vmulpd	ymm11, ymm9, ymm11		;; B5 = I5 * cosine				;  18-22

	vsubpd	ymm0, ymm0, ymm4		;; I11 - I12 (new R12)				; 19-21
	vmulpd	ymm9, ymm9, ymm15		;; C5 = I5 * sine				;  19-23

	vaddpd	ymm4, ymm10, ymm14		;; R9 + R10 (new R9)				; 20-22
	vmulpd	ymm3, ymm3, ymm15		;; D5 = R5 * sine				;  20-24

	vsubpd	ymm10, ymm10, ymm14		;; R9 - R10 (new R10)				; 21-23
	vmulpd	ymm14, ymm13, ymm1		;; A6 = R6 * cosine				;  21-25

	vaddpd	ymm15, ymm8, ymm12		;; I9 + I10 (new I9)				; 22-24
	vsubpd	ymm8, ymm8, ymm12		;; I9 - I10 (new I10)				; 23-25
	vmovapd	ymm12, [srcreg+d3reg+32]	;; I6
	vmulpd	ymm1, ymm12, ymm1		;; B6 = I6 * cosine				;  22
	vaddpd	ymm2, ymm2, ymm9		;; A5 = A5 + C5 (first R5)			; 24-26
	vmovapd	ymm9, [screg1+320]		;; sine for w^10 (8-complex w^5)
	vmulpd	ymm12, ymm12, ymm9		;; C6 = I6 * sine				;  23-27
	vmulpd	ymm13, ymm13, ymm9		;; D6 = R6 * sine				;  24-28

	vsubpd	ymm11, ymm11, ymm3		;; B5 = B5 - D5 (first I5)			; 25-27
	vmovapd	ymm9, [screg1+128+32]		;; cosine for w^4 (8-complex w^2)

	vaddpd	ymm3, ymm10, ymm0		;; R10 + R12 (newer R10)			; 26-28

	vsubpd	ymm10, ymm10, ymm0		;; R10 - R12 (newer R12)			; 27-29
	vmovapd	ymm0, [srcreg+d1reg]		;; R3

	vaddpd	ymm14, ymm14, ymm12		;; A6 = A6 + C6 (first R6)			; 28-30

	vsubpd	ymm1, ymm1, ymm13		;; B6 = B6 - D6 (first I6)			; 29-31
	L1prefetch L1preg, L1pt

	vaddpd	ymm13, ymm8, ymm6		;; I10 + I12 (newer I10)			; 30-32

	vsubpd	ymm8, ymm8, ymm6		;; I10 - I12 (newer I12)			; 31-33

	vsubpd	ymm6, ymm2, ymm14		;; R5 - R6 (new R6)				; 32-34
	L1prefetch L1preg+d1reg, L1pt

	vaddpd	ymm2, ymm2, ymm14		;; R5 + R6 (new R5)				; 33-35
	vmulpd	ymm14, ymm0, ymm9		;; A3 = R3 * cosine				;  33-37

	vsubpd	ymm12, ymm11, ymm1		;; I5 - I6 (new I6)				; 34-36
	vaddpd	ymm11, ymm11, ymm1		;; I5 + I6 (new I5)				; 35-37
	vmovapd	ymm1, [srcreg+d1reg+32]		;; I3
	vmulpd	ymm9, ymm1, ymm9		;; B3 = I3 * cosine				;  34-38
	ystore	[dstreg+e1], ymm11		;; Temp save new I5 which morphs into newer R7	; 38
	vmovapd	ymm11, [screg1+128]		;; sine for w^4 (8-complex w^2)
	vmulpd	ymm0, ymm0, ymm11		;; D3 = R3 * sine				;  35-39

	vmulpd	ymm1, ymm1, ymm11		;; C3 = I3 * sine				;  36-40
	vsubpd	ymm11, ymm4, ymm7		;; R9 - R11 (newer R11)				; 36-38

	vaddpd	ymm4, ymm4, ymm7		;; R9 + R11 (newer R9)				; 37-39
	L1prefetch L1preg+2*d1reg, L1pt

	vsubpd	ymm7, ymm15, ymm5		;; I9 - I11 (newer I11)				; 38-40

						;; mul R10/I10 by w^1 = .924 - .383i
	vaddpd	ymm15, ymm15, ymm5		;; I9 + I11 (newer I9)				; 39-41
	vmovapd ymm5, YMM_P924
	ystore	[dstreg], ymm4			;; Temp save newer R9				; 40
	vmulpd	ymm4, ymm3, ymm5		;; R10 * .924					;  39-43

	vaddpd	ymm14, ymm14, ymm1		;; A3 = A3 + C3 (first R3)			; 40-42
	vmovapd ymm1, YMM_P383
	vmulpd	ymm3, ymm3, ymm1		;; R10 * .383					;  40-44

	vsubpd	ymm9, ymm9, ymm0		;; B3 = B3 - D3 (first I3)			; 41-43
	vmulpd	ymm0, ymm13, ymm1		;; I10 * .383					;  41-45

						;; mul R6/I6 by w^2 = .707 - .707i
	ystore	[dstreg+32], ymm15		;; Temp save newer I9 which morphs into newer R13 ; 42
	vmulpd	ymm13, ymm13, ymm5		;; I10 * .924					;  42-46
	vaddpd	ymm15, ymm12, ymm6		;; R6 = I6 + R6					; 42-44

						;; mul R12/I12 by w^3 = .383 - .924i
	vsubpd	ymm12, ymm12, ymm6		;; I6 = I6 - R6					; 43-45
	vmulpd	ymm6, ymm10, ymm1		;; R12 * .383					;  43-47
	vmulpd	ymm10, ymm10, ymm5		;; R12 * .924					;  44-48
	vmulpd	ymm5, ymm8, ymm5		;; I12 * .924					;  45-49
	vmulpd	ymm8, ymm8, ymm1		;; I12 * .383					;  46-50
						;; mul R11/I11 by SQRTHALF - i*SQRTHALF
	vaddpd	ymm1, ymm7, ymm11		;; R11 = I11 + R11				; 44-46
	vsubpd	ymm7, ymm7, ymm11		;; I11 = I11 - R11				; 45-47
	vaddpd	ymm4, ymm4, ymm0		;; Twiddled R10 = R10 * .924 + I10 * .383	; 46-48
	vmovapd	ymm11, [srcreg]			;; R1

	vsubpd	ymm13, ymm13, ymm3		;; Twiddled I10 = I10 * .924 - R10 * .383	; 47-49
	vmovapd	ymm0, YMM_SQRTHALF

						;; R3/I3 morphs into R3/R4
	vaddpd	ymm3, ymm11, ymm14		;; R1 + R3 (new R1)				; 48-50

	vsubpd	ymm11, ymm11, ymm14		;; R1 - R3 (new R3)				; 49-51
	vmulpd	ymm15, ymm15, ymm0		;; R6 = R6 * SQRTHALF				;  49-53
	vmovapd	ymm14, [srcreg+32]		;; R2

	vaddpd	ymm6, ymm6, ymm5		;; Twiddled R12 = R12 * .383 + I12 * .924	; 50-52
	vmulpd	ymm12, ymm12, ymm0		;; I6 = I6 * SQRTHALF				;  50-54
	vmovapd	ymm5, [dstreg+e1]		;; Reload new I5 which morphs into newer R7

	vsubpd	ymm8, ymm8, ymm10		;; Twiddled I12 = I12 * .383 - R12 * .924	; 51-53
	vmulpd	ymm1, ymm1, ymm0		;; R11 = R11 * SQRTHALF				;  51

	vaddpd	ymm10, ymm14, ymm9		;; R2 + R4 (new R2)				; 52-54
	vmulpd	ymm7, ymm7, ymm0		;; I11 = I11 * SQRTHALF				;  52-56
	vmovapd	ymm0, [dstreg]			;; Reload newer R9

	vsubpd	ymm14, ymm14, ymm9		;; R2 - R4 (new R4)				; 53-55
	L1prefetch L1preg+d3reg, L1pt

						;; R5/I5 morphs into newer R5/R7
	vaddpd	ymm9, ymm3, ymm2		;; R1 + R5 (newer R1)				; 54-56

	vsubpd	ymm3, ymm3, ymm2		;; R1 - R5 (newer R5)				; 55-57

	vaddpd	ymm2, ymm11, ymm5		;; R3 + R7 (newer R3)				; 56-58
	L1prefetch L1p4reg, L1pt

	vsubpd	ymm11, ymm11, ymm5		;; R3 - R7 (newer R7)				; 57-59

						;; R6/I6 morphs into newer R6/R8
	vaddpd	ymm5, ymm10, ymm15		;; R2 + R6 (newer R2)				; 58-60

	vsubpd	ymm10, ymm10, ymm15		;; R2 - R6 (newer R6)				; 59-61
	L1prefetch L1p4reg+d1reg, L1pt

	vaddpd	ymm15, ymm14, ymm12		;; R4 + R8 (newer R4)				; 60-62

	vsubpd	ymm14, ymm14, ymm12		;; R4 - R8 (newer R8)				; 61-63

	;; Do last level, shuffle and store

						;; R9/I9 morphs into newer R9/R13
	vaddpd	ymm12, ymm9, ymm0		;; R1 + R9 (last R1)				; 62-64
	L1prefetch L1p4reg+2*d1reg, L1pt

	vsubpd	ymm9, ymm9, ymm0		;; R1 - R9 (last R9)				; 63-65

						;; R10/I10 morphs into newer R10/R14
	vaddpd	ymm0, ymm5, ymm4		;; R2 + R10 (last R2)				; 64-66

	vsubpd	ymm5, ymm5, ymm4		;; R2 - R10 (last R10)				; 65-67
	L1prefetch L1p4reg+d3reg, L1pt

						;; R11/I11 morphs into newer R11/R15
	vaddpd	ymm4, ymm2, ymm1		;; R3 + R11 (last R3)				; 66-68

	vsubpd	ymm2, ymm2, ymm1		;; R3 - R11 (last R11)				; 67-69
	vshufpd	ymm1, ymm12, ymm0, 0		;; Shuffle R1 and R2 to create R1/R2 low	; 67

						;; R12/I12 morphs into newer R12/R16
	vshufpd	ymm12, ymm12, ymm0, 15		;; Shuffle R1 and R2 to create R1/R2 hi		; 68
	vaddpd	ymm0, ymm15, ymm6		;; R4 + R12 (last R4)				; 68-70

	vsubpd	ymm15, ymm15, ymm6		;; R4 - R12 (last R12)				; 69-71
	vshufpd	ymm6, ymm9, ymm5, 0		;; Shuffle R9 and R10 to create R9/R10 low	; 69

	vshufpd	ymm9, ymm9, ymm5, 15		;; Shuffle R9 and R10 to create R9/R10 hi	; 70

	vshufpd	ymm5, ymm4, ymm0, 0		;; Shuffle R3 and R4 to create R3/R4 low	; 71

	vshufpd	ymm4, ymm4, ymm0, 15		;; Shuffle R3 and R4 to create R3/R4 hi		; 72

	vshufpd	ymm0, ymm2, ymm15, 0		;; Shuffle R11 and R12 to create R11/R12 low	; 73

	vshufpd	ymm2, ymm2, ymm15, 15		;; Shuffle R11 and R12 to create R11/R12 hi	; 74

	ylow128s ymm15, ymm1, ymm5		;; Shuffle R1/R2 low and R3/R4 low (final R1)	; 75-76

	yhigh128s ymm1, ymm1, ymm5		;; Shuffle R1/R2 low and R3/R4 low (final R3)	; 76-77

	ystore	[dstreg], ymm15			;; Save R1					; 77
	ylow128s ymm5, ymm12, ymm4		;; Shuffle R1/R2 hi and R3/R4 hi (final R2)	; 77-78
	vmovapd	ymm15, [dstreg+32]		;; Reload newer R13

	ystore	[dstreg+e2], ymm1		;; Save R3					; 78
	yhigh128s ymm12, ymm12, ymm4		;; Shuffle R1/R2 hi and R3/R4 hi (final R4)	; 78-79

	;; Swizzle using 4 shufpds and 4 perm2f128  (1 or 3 clocks each on port 5, throughput 1)

	ystore	[dstreg+e1], ymm5		;; Save R2					; 79
	ylow128s ymm4, ymm6, ymm0		;; Shuffle R9/R10 low and R11/R12 low (final R9) ; 79-80
	vaddpd	ymm1, ymm3, ymm15		;; R5 + R13 (last R5)				; 79-81

	ystore	[dstreg+e2+e1], ymm12		;; Save R4					; 80
	yhigh128s ymm6, ymm6, ymm0		;; Shuffle R9/R10 low and R11/R12 low (final R11) ; 80-81
	vsubpd	ymm3, ymm3, ymm15		;; R5 - R13 (last R13)				; 80-82

	ystore	[dstreg+32], ymm4		;; Save R9					; 81
	ylow128s ymm0, ymm9, ymm2		;; Shuffle R9/R10 hi and R11/R12 hi (final R10)	; 81-82
	vaddpd	ymm15, ymm10, ymm13		;; R6 + R14 (last R6)				; 81-83

	ystore	[dstreg+e2+32], ymm6		;; Save R11					; 82
	yhigh128s ymm9, ymm9, ymm2		;; Shuffle R9/R10 hi and R11/R12 hi (final R12)	; 82-83
	vsubpd	ymm10, ymm10, ymm13		;; R6 - R14 (last R14)				; 82-84

	ystore	[dstreg+e1+32], ymm0		;; Save R10					; 83
	vaddpd	ymm13, ymm11, ymm7		;; R7 + R15 (last R7)				; 83-85

	ystore	[dstreg+e2+e1+32], ymm9		;; Save R12					; 84
	vshufpd	ymm2, ymm1, ymm15, 0		;; Shuffle R5 and R6 to create R5/R6 low	; 84
	vsubpd	ymm11, ymm11, ymm7		;; R7 - R15 (last R15)				; 84-86

	vshufpd	ymm1, ymm1, ymm15, 15		;; Shuffle R5 and R6 to create R5/R6 hi		; 85
	vaddpd	ymm7, ymm14, ymm8		;; R8 + R16 (last R8)				; 85-87

	vshufpd	ymm15, ymm3, ymm10, 0		;; Shuffle R13 and R14 to create R13/R14 low	; 86
	vsubpd	ymm14, ymm14, ymm8		;; R8 - R16 (last R16)				; 86-88

	vshufpd	ymm3, ymm3, ymm10, 15		;; Shuffle R13 and R14 to create R13/R14 hi	; 87

	vshufpd	ymm10, ymm13, ymm7, 0		;; Shuffle R7 and R8 to create R7/R8 low	; 88

	vshufpd	ymm13, ymm13, ymm7, 15		;; Shuffle R7 and R8 to create R7/R8 hi		; 89

	vshufpd	ymm7, ymm11, ymm14, 0		;; Shuffle R15 and R16 to create R15/R16 low	; 90

	vshufpd	ymm11, ymm11, ymm14, 15		;; Shuffle R15 and R16 to create R15/R16 hi	; 91

	ylow128s ymm14, ymm2, ymm10		;; Shuffle R5/R6 low and R7/R8 low (final R5)	; 92-93

	yhigh128s ymm2, ymm2, ymm10		;; Shuffle R5/R6 low and R7/R8 low (final R7)	; 93-94

	ystore	[dstreg+e4], ymm14		;; Save R5					; 94
	ylow128s ymm10, ymm1, ymm13		;; Shuffle R5/R6 hi and R7/R8 hi (final R6)	; 94-95

	ystore	[dstreg+e4+e2], ymm2		;; Save R7					; 95
	yhigh128s ymm1, ymm1, ymm13		;; Shuffle R5/R6 hi and R7/R8 hi (final R8)	; 95-96

	ystore	[dstreg+e4+e1], ymm10		;; Save R6					; 96
	ylow128s ymm13, ymm15, ymm7		;; Shuffle R13/R14 low and R15/R16 low (final R13) ; 96-97

	ystore	[dstreg+e4+e2+e1], ymm1		;; Save R8					; 97
	yhigh128s ymm15, ymm15, ymm7		;; Shuffle R13/R14 low and R15/R16 low (final R15) ; 97-98

	ystore	[dstreg+e4+32], ymm13		;; Save R13					; 98
	ylow128s ymm7, ymm3, ymm11		;; Shuffle R13/R14 hi and R15/R16 hi (final R14) ; 98-99

	ystore	[dstreg+e4+e2+32], ymm15	;; Save R15					; 99
	yhigh128s ymm3, ymm3, ymm11		;; Shuffle R13/R14 hi and R15/R16 hi (final R16) ; 99-100

	ystore	[dstreg+e4+e1+32], ymm7		;; Save R14					; 100
	ystore	[dstreg+e4+e2+e1+32], ymm3	;; Save R16					; 101

	bump	srcreg, srcinc
	bump	src4reg, srcinc
	bump	dstreg, dstinc
	bump	screg1, scinc1
	bump	screg2, scinc2
	bump	L1preg, srcinc
	bump	L1p4reg, srcinc
	ENDM

ENDIF
