; Copyright 2011-2013 - Mersenne Research, Inc.  All rights reserved
; Author:  George Woltman
; Email: woltman@alum.mit.edu
;

;;
;;
;; All new macros for version 27 of gwnum.  Do a radix-3 step in an FFT.
;; The forward FFT macros multiply by the sin/cos values at the end of the macro
;; and the inverse FFTs multiply by the sin/cos values at the start of the macro.
;; We also implement the Daniel J. Bernstein (DJB) "exponent-1" idea to save sin/cos
;; memory.
;;
;;

;;
;; ************************************* three-complex-djbfft variants ******************************************
;;

;; The standard version
yr3_3cl_three_complex_djbfft_preload MACRO
	yr3_3c_djbfft_cmn_preload
	ENDM
yr3_3cl_three_complex_djbfft MACRO srcreg,srcinc,d1,screg,scinc,maxrpt,L1pt,L1pd
	yr3_3c_djbfft_cmn srcreg,srcinc,0,d1,noexec,screg,screg+32,screg,scinc,maxrpt,L1pt,L1pd
	ENDM

;; Like standard but uses rbx to index into source data
yr3_f3cl_three_complex_djbfft_preload MACRO
	yr3_3c_djbfft_cmn_preload
	ENDM
yr3_f3cl_three_complex_djbfft MACRO srcreg,srcinc,d1,screg,scinc,maxrpt,L1pt,L1pd
	yr3_3c_djbfft_cmn srcreg,srcinc,rbx,d1,noexec,screg,screg+32,screg,scinc,maxrpt,L1pt,L1pd
	ENDM

;; Like standard, but uses a six_reals sin/cos table
yr3_r3cl_three_complex_djbfft_preload MACRO
	yr3_3c_djbfft_cmn_preload
	ENDM
yr3_r3cl_three_complex_djbfft MACRO srcreg,srcinc,d1,screg,scinc,maxrpt,L1pt,L1pd
	yr3_3c_djbfft_cmn srcreg,srcinc,0,d1,noexec,screg+64,screg+96,screg,scinc,maxrpt,L1pt,L1pd
	ENDM

; Like the standard version except vbroadcastsd is used to reduce sin/cos data
yr3_b3cl_three_complex_djbfft_preload MACRO
	yr3_3c_djbfft_cmn_preload
	ENDM
yr3_b3cl_three_complex_djbfft MACRO srcreg,srcinc,d1,screg,scinc,maxrpt,L1pt,L1pd
	yr3_3c_djbfft_cmn srcreg,srcinc,0,d1,exec,screg,screg+8,screg,scinc,maxrpt,L1pt,L1pd
	ENDM

; Like b3cl but extracts the sin/cos data to broadcasts from
; the six-real/three_complex sin/cos table
yr3_rb3cl_three_complex_djbfft_preload MACRO
	yr3_3c_djbfft_cmn_preload
	ENDM
yr3_rb3cl_three_complex_djbfft MACRO srcreg,srcinc,d1,screg,scinc,maxrpt,L1pt,L1pd
	yr3_3c_djbfft_cmn srcreg,srcinc,0,d1,exec,screg+8,screg+48,screg,scinc,maxrpt,L1pt,L1pd
	ENDM

;; Common code to do a 3-complex FFT.  The input values are R1+R4i, R2+R5i, R3+R6i
;; A 3-complex FFT is:
;; Res1:  (R1+R2+R3) + (I1+I2+I3)i
;; Res2:  (R1-.5R2-.866I2-.5R3+.866I3) + (I1-.5I2+.866R2-.5I3-.866R3)i
;; Res3:  (R1-.5R2+.866I2-.5R3-.866I3) + (I1-.5I2-.866R2-.5I3+.866R3)i
;; Then multiply 2 of the 3 results by twiddle factors.
yr3_3c_djbfft_cmn_preload MACRO
	ENDM
yr3_3c_djbfft_cmn MACRO srcreg,srcinc,srcoff,d1,bcast,screg1,screg2,screg,scinc,maxrpt,L1pt,L1pd
	vmovapd	ymm0, [srcreg+srcoff+d1]	;; R2
	vmovapd	ymm1, [srcreg+srcoff+2*d1]	;; R3
	vaddpd	ymm2, ymm0, ymm1		;; R2 + R3
	vmovapd	ymm3, [srcreg+srcoff+d1+32]	;; I2
	vmovapd	ymm4, [srcreg+srcoff+2*d1+32]	;; I3
	vaddpd	ymm5, ymm3, ymm4		;; I2 + I3
	vsubpd	ymm0, ymm0, ymm1		;; R2 - R3
	vmovapd	ymm1, YMM_HALF
	vmulpd	ymm6, ymm1, ymm2		;; 0.5 * (R2 + R3)
	vsubpd	ymm3, ymm3, ymm4		;; I2 - I3
	vmulpd	ymm1, ymm1, ymm5		;; 0.5 * (I2 + I3)
	vmovapd	ymm7, [srcreg+srcoff]		;; R1
	vaddpd	ymm2, ymm7, ymm2		;; R1 + R2 + R3 (final R1)
	vmovapd	ymm4, YMM_P866
	vmulpd	ymm0, ymm4, ymm0		;; 0.866 * (R2 - R3)
	vsubpd	ymm7, ymm7, ymm6		;; (R1-.5R2-.5R3)
	vmulpd	ymm3, ymm4, ymm3		;; 0.866 * (I2 - I3)
	vmovapd	ymm6, [srcreg+srcoff+32]	;; I1
	vaddpd	ymm5, ymm6, ymm5		;; I1 + I2 + I3 (final I1)
	vsubpd	ymm6, ymm6, ymm1		;; (I1-.5I2-.5I3)

	L1prefetchw srcreg+L1pd, L1pt

	vsubpd	ymm1, ymm7, ymm3		;; Final R2
	vaddpd	ymm7, ymm7, ymm3		;; Final R3
	vaddpd	ymm3, ymm6, ymm0		;; Final I2
	vsubpd	ymm6, ymm6, ymm0		;; Final I3

	L1prefetchw srcreg+d1+L1pd, L1pt

no bcast vmovapd ymm0, [screg2]			;; cosine/sine
bcast	vbroadcastsd ymm0, Q [screg2]		;; cosine/sine
	vmulpd	ymm4, ymm1, ymm0		;; A2 = R2 * cosine/sine
	vsubpd	ymm4, ymm4, ymm3		;; A2 = A2 - I2
	vmulpd	ymm3, ymm3, ymm0		;; B2 = I2 * cosine/sine
	vaddpd	ymm3, ymm3, ymm1		;; B2 = B2 + R2

	L1prefetchw srcreg+2*d1+L1pd, L1pt

	vmulpd	ymm1, ymm7, ymm0		;; A3 = R3 * cosine/sine
	vaddpd	ymm1, ymm1, ymm6		;; A3 = A3 + I3
	vmulpd	ymm6, ymm6, ymm0		;; B3 = I3 * cosine/sine
	vsubpd	ymm6, ymm6, ymm7		;; B3 = B3 - R3

no bcast vmovapd ymm0, [screg1]			;; sine
bcast	vbroadcastsd ymm0, Q [screg1]		;; sine
	vmulpd	ymm4, ymm4, ymm0		;; A2 = A2 * sine (new R2)
	vmulpd	ymm3, ymm3, ymm0		;; B2 = B2 * sine (new I2)
	vmulpd	ymm1, ymm1, ymm0		;; A3 = A3 * sine (new R3)
	vmulpd	ymm6, ymm6, ymm0		;; B3 = B3 * sine (new I3)

	ystore	[srcreg], ymm2			;; Save R1
	ystore	[srcreg+32], ymm5		;; Save I1
	ystore	[srcreg+d1], ymm4		;; Save R2
	ystore	[srcreg+d1+32], ymm3		;; Save I2
	ystore	[srcreg+2*d1], ymm1		;; Save R3
	ystore	[srcreg+2*d1+32], ymm6		;; Save I3
	bump	srcreg, srcinc
	bump	screg, scinc
	ENDM

;; 64-bit version

IFDEF X86_64

yr3_3c_djbfft_cmn_preload MACRO
	vmovapd	ymm15, YMM_HALF
	vmovapd	ymm14, YMM_P866
	ENDM

yr3_3c_djbfft_cmn MACRO srcreg,srcinc,srcoff,d1,bcast,screg1,screg2,screg,scinc,maxrpt,L1pt,L1pd
	IF maxrpt MOD 5 EQ 0
	yr3_3c_djbfft_unroll -1,noexec,noexec,exec,srcreg,srcinc,srcoff,d1,bcast,screg1,screg2,screg,scinc,L1pt,L1pd
	yr3_3c_djbfft_unroll 0,noexec,exec,exec,srcreg,srcinc,srcoff,d1,bcast,screg1,screg2,screg,scinc,L1pt,L1pd
	yr3_3c_djbfft_unroll 1,exec,exec,exec,srcreg,srcinc,srcoff,d1,bcast,screg1,screg2,screg,scinc,L1pt,L1pd
	yr3_3c_djbfft_unroll 2,exec,exec,exec,srcreg,srcinc,srcoff,d1,bcast,screg1,screg2,screg,scinc,L1pt,L1pd
	yr3_3c_djbfft_unroll 3,exec,exec,exec,srcreg,srcinc,srcoff,d1,bcast,screg1,screg2,screg,scinc,L1pt,L1pd
	yr3_3c_djbfft_unroll 4,exec,exec,noexec,srcreg,srcinc,srcoff,d1,bcast,screg1,screg2,screg,scinc,L1pt,L1pd
	yr3_3c_djbfft_unroll 5,exec,noexec,noexec,srcreg,srcinc,srcoff,d1,bcast,screg1,screg2,screg,scinc,L1pt,L1pd
	bump	srcreg, 5*srcinc
	bump	screg, 5*scinc
	ELSEIF maxrpt MOD 4 EQ 0
	yr3_3c_djbfft_unroll -1,noexec,noexec,exec,srcreg,srcinc,srcoff,d1,bcast,screg1,screg2,screg,scinc,L1pt,L1pd
	yr3_3c_djbfft_unroll 0,noexec,exec,exec,srcreg,srcinc,srcoff,d1,bcast,screg1,screg2,screg,scinc,L1pt,L1pd
	yr3_3c_djbfft_unroll 1,exec,exec,exec,srcreg,srcinc,srcoff,d1,bcast,screg1,screg2,screg,scinc,L1pt,L1pd
	yr3_3c_djbfft_unroll 2,exec,exec,exec,srcreg,srcinc,srcoff,d1,bcast,screg1,screg2,screg,scinc,L1pt,L1pd
	yr3_3c_djbfft_unroll 3,exec,exec,noexec,srcreg,srcinc,srcoff,d1,bcast,screg1,screg2,screg,scinc,L1pt,L1pd
	yr3_3c_djbfft_unroll 4,exec,noexec,noexec,srcreg,srcinc,srcoff,d1,bcast,screg1,screg2,screg,scinc,L1pt,L1pd
	bump	srcreg, 4*srcinc
	bump	screg, 4*scinc
	ELSEIF maxrpt MOD 3 EQ 0
	yr3_3c_djbfft_unroll -1,noexec,noexec,exec,srcreg,srcinc,srcoff,d1,bcast,screg1,screg2,screg,scinc,L1pt,L1pd
	yr3_3c_djbfft_unroll 0,noexec,exec,exec,srcreg,srcinc,srcoff,d1,bcast,screg1,screg2,screg,scinc,L1pt,L1pd
	yr3_3c_djbfft_unroll 1,exec,exec,exec,srcreg,srcinc,srcoff,d1,bcast,screg1,screg2,screg,scinc,L1pt,L1pd
	yr3_3c_djbfft_unroll 2,exec,exec,noexec,srcreg,srcinc,srcoff,d1,bcast,screg1,screg2,screg,scinc,L1pt,L1pd
	yr3_3c_djbfft_unroll 3,exec,noexec,noexec,srcreg,srcinc,srcoff,d1,bcast,screg1,screg2,screg,scinc,L1pt,L1pd
	bump	srcreg, 3*srcinc
	bump	screg, 3*scinc
	ELSEIF maxrpt MOD 2 EQ 0
	yr3_3c_djbfft_unroll -1,noexec,noexec,exec,srcreg,srcinc,srcoff,d1,bcast,screg1,screg2,screg,scinc,L1pt,L1pd
	yr3_3c_djbfft_unroll 0,noexec,exec,exec,srcreg,srcinc,srcoff,d1,bcast,screg1,screg2,screg,scinc,L1pt,L1pd
	yr3_3c_djbfft_unroll 1,exec,exec,noexec,srcreg,srcinc,srcoff,d1,bcast,screg1,screg2,screg,scinc,L1pt,L1pd
	yr3_3c_djbfft_unroll 2,exec,noexec,noexec,srcreg,srcinc,srcoff,d1,bcast,screg1,screg2,screg,scinc,L1pt,L1pd
	bump	srcreg, 2*srcinc
	bump	screg, 2*scinc
	ELSE
	yr3_3c_djbfft_unroll -1,noexec,noexec,exec,srcreg,srcinc,srcoff,d1,bcast,screg1,screg2,screg,scinc,L1pt,L1pd
	yr3_3c_djbfft_unroll 0,noexec,exec,noexec,srcreg,srcinc,srcoff,d1,bcast,screg1,screg2,screg,scinc,L1pt,L1pd
	yr3_3c_djbfft_unroll 1,exec,noexec,noexec,srcreg,srcinc,srcoff,d1,bcast,screg1,screg2,screg,scinc,L1pt,L1pd
	bump	srcreg, srcinc
	bump	screg, scinc
	ENDIF
	ENDM

yr3_3c_djbfft_unroll MACRO iter,prev,this,next,srcreg,srcinc,srcoff,d1,bcast,screg1,screg2,screg,scinc,L1pt,L1pd

;; On first call, init registers

	IF iter EQ -1
y0	TEXTEQU	<ymm0>
y1	TEXTEQU	<ymm1>
y2	TEXTEQU	<ymm2>
y3	TEXTEQU	<ymm3>
y4	TEXTEQU	<ymm4>
y5	TEXTEQU	<ymm5>
y6	TEXTEQU	<ymm6>
y7	TEXTEQU	<ymm7>
y8	TEXTEQU	<ymm8>
y9	TEXTEQU	<ymm9>
y10	TEXTEQU	<ymm10>
y11	TEXTEQU	<ymm11>
y12	TEXTEQU	<ymm12>
y13	TEXTEQU	<ymm13>
	ENDIF

;; On later calls, previous R2,A3,B2,B3,sine will be in y0-4.  This R1,I1,R2+R3,I2+I3,0.5*(R2+R3),0.5*(I2+I3),T2,T4 will be in y5-12.
;; The remaining register is free (ymm14 and ymm15 are preloaded constants 0.866 and 0.5).

this	vsubpd	y9, y5, y9				;; T1 = R1-.5R2-.5R3		; 1-3
prev	vmulpd	y1, y1, y4				;; A3 = A3 * sine (new R3)	; 1-5
this no bcast vmovapd y13, [screg2+iter*scinc]		;; cosine/sine
this bcast vbroadcastsd y13, Q [screg2+iter*scinc]	;; cosine/sine

this	vaddpd	y5, y5, y7				;; R1 + R2 + R3 (final R1)	; 2-4
prev	vmulpd	y2, y2, y4				;; B2 = B2 * sine (new I2)	; 2-6
next	vmovapd	y7, [srcreg+(iter+1)*srcinc+srcoff+d1]	;; R2

this	vsubpd	y10, y6, y10				;; T3 = I1-.5I2-.5I3		; 3-5
prev	vmulpd	y3, y3, y4				;; B3 = B3 * sine (new I3)	; 3-7
next	vmovapd	y4, [srcreg+(iter+1)*srcinc+srcoff+2*d1] ;; R3

this	vaddpd	y6, y6, y8				;; I1 + I2 + I3 (final I1)	; 4-6
this next yloop_unrolled_one

this	vsubpd	y8, y9, y11				;; T1-T2 (final R2)		; 5-7
prev	ystore	[srcreg+(iter-1)*srcinc+d1], y0		;; Save R2			; 5
next	vmovapd	y0, [srcreg+(iter+1)*srcinc+srcoff+d1+32] ;; I2

this	vaddpd	y9, y9, y11				;; T1+T2 (final R3)		; 6-8
this	ystore	[srcreg+iter*srcinc], y5		;; Save R1			; 6
next	vmovapd	y5, [srcreg+(iter+1)*srcinc+srcoff+2*d1+32] ;; I3

this	vaddpd	y11, y10, y12				;; T3+T4 (final I2)		; 7-9
prev	ystore	[srcreg+(iter-1)*srcinc+2*d1], y1	;; Save R3			; 7

this	vsubpd	y10, y10, y12				;; T3-T4 (final I3)		; 8-10
this	vmulpd	y12, y8, y13				;; A2 = R2 * cosine/sine	; 8-12
prev	ystore	[srcreg+(iter-1)*srcinc+d1+32], y2	;; Save I2			; 8
this	L1prefetchw srcreg+iter*srcinc+L1pd, L1pt

next	vaddpd	y2, y7, y4				;; R2 + R3			; 9-11
this	vmulpd	y1, y9, y13				;; A3 = R3 * cosine/sine	; 9-13
this	ystore	[srcreg+iter*srcinc+32], y6		;; Save I1			; 9
this no bcast vmovapd y6, [screg1+iter*scinc]		;; sine
this bcast vbroadcastsd y6, Q [screg1+iter*scinc]	;; sine

next	vsubpd	y7, y7, y4				;; R2 - R3			; 10-12
this	vmulpd	y4, y11, y13				;; B2 = I2 * cosine/sine	; 10-14
prev	ystore	[srcreg+(iter-1)*srcinc+2*d1+32], y3	;; Save I3			; 10

next	vaddpd	y3, y0, y5				;; I2 + I3			; 11-13
this	vmulpd	y13, y10, y13				;; B3 = I3 * cosine/sine	; 11-15

next	vsubpd	y0, y0, y5				;; I2 - I3			; 12-14
next	vmulpd	y5, ymm15, y2				;; 0.5 * (R2 + R3)		; 12-16
this	L1prefetchw srcreg+iter*srcinc+d1+L1pd, L1pt

this	vsubpd	y12, y12, y11				;; A2 = A2 - I2			; 13-15
next	vmulpd	y7, ymm14, y7				;; T4 = 0.866 * (R2 - R3)	; 13-17
next	vmovapd	y11, [srcreg+(iter+1)*srcinc+srcoff]	;; R1

this	vaddpd	y1, y1, y10				;; A3 = A3 + I3			; 14-16
next	vmulpd	y10, ymm15, y3				;; 0.5 * (I2 + I3)		; 14-18


this	vaddpd	y4, y4, y8				;; B2 = B2 + R2			; 15-17
next	vmulpd	y0, ymm14, y0				;; T2 = 0.866 * (I2 - I3)	; 15-19
next	vmovapd	y8, [srcreg+(iter+1)*srcinc+srcoff+32]	;; I1

this	vsubpd	y13, y13, y9				;; B3 = B3 - R3			; 16-18
this	vmulpd	y12, y12, y6				;; A2 = A2 * sine (new R2)	; 16-20
this	L1prefetchw srcreg+iter*srcinc+2*d1+L1pd, L1pt

;; Shuffle register assignments so that next call has R2,A3,B2,B3,sine in y0-4 and next R1,I1,R2+R3,I2+I3,0.5*(R2+R3),0.5*(I2+I3),T2,T4 in y5-12.

ytmp	TEXTEQU	y0
y0	TEXTEQU	y12
y12	TEXTEQU	y7
y7	TEXTEQU	y2
y2	TEXTEQU	y4
y4	TEXTEQU	y6
y6	TEXTEQU	y8
y8	TEXTEQU	y3
y3	TEXTEQU	y13
y13	TEXTEQU	y9
y9	TEXTEQU	y5
y5	TEXTEQU	y11
y11	TEXTEQU ytmp

	ENDM


;; Haswell FMA3 version

IF (@INSTR(,%yarch,<FMA3>) NE 0)

yr3_3c_djbfft_cmn_preload MACRO
	vmovapd	ymm15, YMM_HALF
	vmovapd	ymm14, YMM_P866
	vmovapd	ymm13, YMM_ONE
	ENDM

;; uops = 6 loads, 6 stores, 2 s/c loads, 4 muls, 16 fmas, 8 register copies = 42 uops / 4 = 10.5 clocks (above our 10 clock optimum)
;; Timed at just over 11 clocks.
yr3_3c_djbfft_unroll MACRO iter,prev,this,next,srcreg,srcinc,srcoff,d1,bcast,screg1,screg2,screg,scinc,L1pt,L1pd

;; On first call, init registers

	IF iter EQ -1
y0	TEXTEQU	<ymm0>
y1	TEXTEQU	<ymm1>
y2	TEXTEQU	<ymm2>
y3	TEXTEQU	<ymm3>
y4	TEXTEQU	<ymm4>
y5	TEXTEQU	<ymm5>
y6	TEXTEQU	<ymm6>
y7	TEXTEQU	<ymm7>
y8	TEXTEQU	<ymm8>
y9	TEXTEQU	<ymm9>
y10	TEXTEQU	<ymm10>
y11	TEXTEQU	<ymm11>
y12	TEXTEQU	<ymm12>
	ENDIF

;; On later calls, this R1,I1 are in y0-1.  Previous R2,I2,R3,I3,c/s will be in y2-6.  This T1,T3,R2-R3,I2-I3 will be in y7-10.
;; The remaining registers are free (ymm13, ymm14, and ymm15 are preloaded constants 1.0, 0.866, and 0.5 respectively). 

next	vmovapd	y11, [srcreg+(iter+1)*srcinc+srcoff+d1]	;; R2
prev	yfmsubpd y12, y2, y6, y3			;; A2 = R2 * cosine/sine - I2		; 1-5
prev	yfmaddpd y3, y3, y6, y2				;; B2 = I2 * cosine/sine + R2		; 1-5
this	L1prefetchw srcreg+iter*srcinc+L1pd, L1pt

next	vmovapd	y2, [srcreg+(iter+1)*srcinc+srcoff+2*d1] ;; R3
this	ystore	[srcreg+iter*srcinc], y0		;; Save R1				; 4
prev	yfmaddpd y0, y4, y6, y5				;; A3 = R3 * cosine/sine + I3		; 2-6
prev	yfmsubpd y5, y5, y6, y4				;; B3 = I3 * cosine/sine - R3		; 2-6

next	vmovapd	y6, [srcreg+(iter+1)*srcinc+srcoff+d1+32] ;; I2
next	yfmaddpd y4, y11, ymm13, y2			;; R2 + R3				; 3-7
next	yfmsubpd y11, y11, ymm13, y2			;; R2 - R3				; 3-7

next	vmovapd	y2, [srcreg+(iter+1)*srcinc+srcoff+2*d1+32] ;; I3
this	ystore	[srcreg+iter*srcinc+32], y1		;; Save I1				; 5
next	yfmaddpd y1, y6, ymm13, y2			;; I2 + I3				; 4-8
next	yfmsubpd y6, y6, ymm13, y2			;; I2 - I3				; 4-8

this	yfnmaddpd y2, ymm14, y10, y7, 1			;; T1 - 0.866 * (I2-I3) (newer R2)	; 5-9
this	yfmaddpd y10, ymm14, y10, y7			;; T1 + 0.866 * (I2-I3) (newer R3)	; 5-9
this	L1prefetchw srcreg+iter*srcinc+d1+L1pd, L1pt

this	yfmaddpd y7, ymm14, y9, y8, 1			;; T3 + 0.866 * (R2-R3) (newer I2)	; 6-10
this	yfnmaddpd y9, ymm14, y9, y8			;; T3 - 0.866 * (R2-R3) (newer I3)	; 6-10
this	L1prefetchw srcreg+iter*srcinc+2*d1+L1pd, L1pt

prev no bcast vmovapd y8, [screg1+(iter-1)*scinc]	;; sine
prev bcast vbroadcastsd y8, Q [screg1+(iter-1)*scinc]	;; sine
prev	vmulpd	y12, y12, y8				;; A2 = A2 * sine (final R2)		; 7-11
prev	vmulpd	y3, y3, y8				;; B2 = B2 * sine (final I2)		; 7-11
prev	ystore	[srcreg+(iter-1)*srcinc+d1], y12	;; Save R2				; 12

next	vmovapd	y12, [srcreg+(iter+1)*srcinc+srcoff]	;; R1
prev	vmulpd	y0, y0, y8				;; A3 = A3 * sine (final R3)		; 8-12
prev	vmulpd	y5, y5, y8				;; B3 = B3 * sine (final I3)		; 8-12
next	vmovapd	y8, [srcreg+(iter+1)*srcinc+srcoff+32]	;; I1
prev	ystore	[srcreg+(iter-1)*srcinc+d1+32], y3	;; Save I2				; 12

next	yfmaddpd y3, y12, ymm13, y4			;; R1 + (R2+R3) (newer & final R1)	; 9-13
next	yfnmaddpd y4, ymm15, y4, y12			;; T1 = R1 - 0.5 * (R2+R3)		; 9-13
this next yloop_unrolled_one

this no bcast vmovapd y12, [screg2+iter*scinc]		;; cosine/sine
this bcast vbroadcastsd y12, Q [screg2+iter*scinc]	;; cosine/sine
prev	ystore	[srcreg+(iter-1)*srcinc+2*d1], y0	;; Save R3				; 13
next	yfmaddpd y0, y8, ymm13, y1			;; I1 + (I2+I3) (newer & final I1)	; 10-14
next	yfnmaddpd y1, ymm15, y1, y8			;; T3 = I1 - 0.5 * (I2+I3)		; 10-14
prev	ystore	[srcreg+(iter-1)*srcinc+2*d1+32], y5	;; Save I3				; 13

;; Shuffle register assignments so that next R1,I1 are in y0-1, this R2,I2,R3,I3,c/s in y2-6, and next T1,T3,R2-R3,I2-I3 in y7-10.

ytmp	TEXTEQU	y0
y0	TEXTEQU	y3
y3	TEXTEQU	y7
y7	TEXTEQU	y4
y4	TEXTEQU	y10
y10	TEXTEQU	y6
y6	TEXTEQU	y12
y12	TEXTEQU	y5
y5	TEXTEQU	y9
y9	TEXTEQU	y11
y11	TEXTEQU	y8
y8	TEXTEQU	y1
y1	TEXTEQU ytmp

	ENDM

ENDIF

ENDIF

;;
;; ************************************* three-complex-djbunfft variants ******************************************
;;

;; The standard version
yr3_3cl_three_complex_djbunfft_preload MACRO
	yr3_3c_djbunfft_cmn_preload
	ENDM
yr3_3cl_three_complex_djbunfft MACRO srcreg,srcinc,d1,screg,scinc,maxrpt,L1pt,L1pd
	yr3_3c_djbunfft_cmn srcreg,srcinc,d1,noexec,screg,screg+32,screg,scinc,maxrpt,L1pt,L1pd
	ENDM

; Like standard but uses a six-reals sin/cos table
yr3_r3cl_three_complex_djbunfft_preload MACRO
	yr3_3c_djbunfft_cmn_preload
	ENDM
yr3_r3cl_three_complex_djbunfft MACRO srcreg,srcinc,d1,screg,scinc,maxrpt,L1pt,L1pd
	yr3_3c_djbunfft_cmn srcreg,srcinc,d1,noexec,screg+64,screg+96,screg,scinc,maxrpt,L1pt,L1pd
	ENDM

; Like the standard version except vbroadcastsd is used to reduce sin/cos data
yr3_b3cl_three_complex_djbunfft_preload MACRO
	yr3_3c_djbunfft_cmn_preload
	ENDM
yr3_b3cl_three_complex_djbunfft MACRO srcreg,srcinc,d1,screg,scinc,maxrpt,L1pt,L1pd
	yr3_3c_djbunfft_cmn srcreg,srcinc,d1,exec,screg,screg+8,screg,scinc,maxrpt,L1pt,L1pd
	ENDM

; Like b3cl but extracts the sin/cos data to broadcasts from
; the six-real/three_complex sin/cos table
yr3_rb3cl_three_complex_djbunfft_preload MACRO
	yr3_3c_djbunfft_cmn_preload
	ENDM
yr3_rb3cl_three_complex_djbunfft MACRO srcreg,srcinc,d1,screg,scinc,maxrpt,L1pt,L1pd
	yr3_3c_djbunfft_cmn srcreg,srcinc,d1,exec,screg+8,screg+48,screg,scinc,maxrpt,L1pt,L1pd
	ENDM

;; Do a 3-complex inverse FFT.  The input values are R1+R2i, R3+R4i, R5+R6i
;; First we apply twiddle factors to 2 of the 3 input numbers.
;; A 3-complex inverse FFT is:
;; Res1:  (R1+R2+R3) + (I1+I2+I3)i
;; Res2:  (R1-.5R2+.866I2-.5R3-.866I3) + (I1-.5I2-.866R2-.5I3+.866R3)i
;; Res3:  (R1-.5R2-.866I2-.5R3+.866I3) + (I1-.5I2+.866R2-.5I3-.866R3)i

yr3_3c_djbunfft_cmn_preload MACRO
	ENDM
yr3_3c_djbunfft_cmn MACRO srcreg,srcinc,d1,bcast,screg1,screg2,screg,scinc,maxrpt,L1pt,L1pd
no bcast vmovapd ymm0, [screg2]		;; cosine/sine
bcast	vbroadcastsd ymm0, Q [screg2]	;; cosine/sine
	vmovapd	ymm1, [srcreg+d1]	;; R2
	vmulpd	ymm2, ymm1, ymm0	;; A2 = R2 * cosine/sine
	vmovapd	ymm3, [srcreg+2*d1]	;; R3
	vmulpd	ymm4, ymm3, ymm0	;; A3 = R3 * cosine/sine
	vmovapd	ymm5, [srcreg+d1+32]	;; I2
	vaddpd	ymm2, ymm2, ymm5	;; A2 = A2 + I2
	vmovapd	ymm6, [srcreg+2*d1+32]	;; I3
	vsubpd	ymm4, ymm4, ymm6	;; A3 = A3 - I3
	vmulpd	ymm5, ymm5, ymm0	;; B2 = I2 * cosine/sine
	vmulpd	ymm6, ymm6, ymm0	;; B3 = I3 * cosine/sine
	vsubpd	ymm5, ymm5, ymm1	;; B2 = B2 - R2
	vaddpd	ymm6, ymm6, ymm3	;; B3 = B3 + R3
no bcast vmovapd ymm0, [screg1]		;; sine
bcast	vbroadcastsd ymm0, Q [screg1]	;; sine
	vmulpd	ymm2, ymm2, ymm0	;; A2 = A2 * sine (final R2)
	vmulpd	ymm4, ymm4, ymm0	;; A3 = A3 * sine (final R3)
	vmulpd	ymm5, ymm5, ymm0	;; B2 = B2 * sine (final I2)
	vmulpd	ymm6, ymm6, ymm0	;; B3 = B3 * sine (final I3)

	L1prefetchw srcreg+L1pd, L1pt

	vsubpd	ymm0, ymm2, ymm4	;; R2 - R3
	vaddpd	ymm2, ymm2, ymm4	;; R2 + R3
	vmovapd	ymm7, YMM_P866
	vmulpd	ymm0, ymm7, ymm0	;; 0.866 * (R2 - R3)
	vsubpd	ymm1, ymm5, ymm6	;; I2 - I3
	vaddpd	ymm5, ymm5, ymm6	;; I2 + I3

	L1prefetchw srcreg+d1+L1pd, L1pt

	vmulpd	ymm1, ymm7, ymm1	;; 0.866 * (I2 - I3)
	vmovapd	ymm7, YMM_HALF
	vmulpd	ymm4, ymm7, ymm2	;; 0.5 * (R2 + R3)
	vmovapd	ymm6, [srcreg]		;; R1
	vaddpd	ymm2, ymm6, ymm2	;; R1 + R2 + R3 (final R1)
	vsubpd	ymm6, ymm6, ymm4	;; (R1-.5R2-.5R3)

	L1prefetchw srcreg+2*d1+L1pd, L1pt

	vmulpd	ymm7, ymm7, ymm5	;; 0.5 * (I2 + I3)
	vmovapd	ymm3, [srcreg+32]	;; I1
	vaddpd	ymm5, ymm3, ymm5	;; I1 + I2 + I3 (final I1)
	vsubpd	ymm3, ymm3, ymm7	;; (I1-.5I2-.5I3)
	vsubpd	ymm7, ymm6, ymm1	;; Final R3
	vaddpd	ymm6, ymm6, ymm1	;; Final R2
	vsubpd	ymm1, ymm3, ymm0	;; Final I2
	vaddpd	ymm3, ymm3, ymm0	;; Final I3

	ystore	[srcreg], ymm2		;; Save R1
	ystore	[srcreg+32], ymm5	;; Save I1
	ystore	[srcreg+d1], ymm6	;; Save R2
	ystore	[srcreg+d1+32], ymm1	;; Save I2
	ystore	[srcreg+2*d1], ymm7	;; Save R3
	ystore	[srcreg+2*d1+32], ymm3	;; Save I3
	bump	srcreg, srcinc
	bump	screg, scinc
	ENDM

;; 64-bit version

IFDEF X86_64

yr3_3c_djbunfft_cmn_preload MACRO
	vmovapd	ymm15, YMM_HALF
	vmovapd	ymm14, YMM_P866
	ENDM

yr3_3c_djbunfft_cmn MACRO srcreg,srcinc,d1,bcast,screg1,screg2,screg,scinc,maxrpt,L1pt,L1pd
	IF maxrpt MOD 5 EQ 0
	yr3_3c_djbunfft_unroll -1,noexec,noexec,exec,srcreg,srcinc,d1,bcast,screg1,screg2,screg,scinc,L1pt,L1pd
	yr3_3c_djbunfft_unroll 0,noexec,exec,exec,srcreg,srcinc,d1,bcast,screg1,screg2,screg,scinc,L1pt,L1pd
	yr3_3c_djbunfft_unroll 1,exec,exec,exec,srcreg,srcinc,d1,bcast,screg1,screg2,screg,scinc,L1pt,L1pd
	yr3_3c_djbunfft_unroll 2,exec,exec,exec,srcreg,srcinc,d1,bcast,screg1,screg2,screg,scinc,L1pt,L1pd
	yr3_3c_djbunfft_unroll 3,exec,exec,exec,srcreg,srcinc,d1,bcast,screg1,screg2,screg,scinc,L1pt,L1pd
	yr3_3c_djbunfft_unroll 4,exec,exec,noexec,srcreg,srcinc,d1,bcast,screg1,screg2,screg,scinc,L1pt,L1pd
	yr3_3c_djbunfft_unroll 5,exec,noexec,noexec,srcreg,srcinc,d1,bcast,screg1,screg2,screg,scinc,L1pt,L1pd
	bump	srcreg, 5*srcinc
	bump	screg, 5*scinc
	ELSEIF maxrpt MOD 4 EQ 0
	yr3_3c_djbunfft_unroll -1,noexec,noexec,exec,srcreg,srcinc,d1,bcast,screg1,screg2,screg,scinc,L1pt,L1pd
	yr3_3c_djbunfft_unroll 0,noexec,exec,exec,srcreg,srcinc,d1,bcast,screg1,screg2,screg,scinc,L1pt,L1pd
	yr3_3c_djbunfft_unroll 1,exec,exec,exec,srcreg,srcinc,d1,bcast,screg1,screg2,screg,scinc,L1pt,L1pd
	yr3_3c_djbunfft_unroll 2,exec,exec,exec,srcreg,srcinc,d1,bcast,screg1,screg2,screg,scinc,L1pt,L1pd
	yr3_3c_djbunfft_unroll 3,exec,exec,noexec,srcreg,srcinc,d1,bcast,screg1,screg2,screg,scinc,L1pt,L1pd
	yr3_3c_djbunfft_unroll 4,exec,noexec,noexec,srcreg,srcinc,d1,bcast,screg1,screg2,screg,scinc,L1pt,L1pd
	bump	srcreg, 4*srcinc
	bump	screg, 4*scinc
	ELSEIF maxrpt MOD 3 EQ 0
	yr3_3c_djbunfft_unroll -1,noexec,noexec,exec,srcreg,srcinc,d1,bcast,screg1,screg2,screg,scinc,L1pt,L1pd
	yr3_3c_djbunfft_unroll 0,noexec,exec,exec,srcreg,srcinc,d1,bcast,screg1,screg2,screg,scinc,L1pt,L1pd
	yr3_3c_djbunfft_unroll 1,exec,exec,exec,srcreg,srcinc,d1,bcast,screg1,screg2,screg,scinc,L1pt,L1pd
	yr3_3c_djbunfft_unroll 2,exec,exec,noexec,srcreg,srcinc,d1,bcast,screg1,screg2,screg,scinc,L1pt,L1pd
	yr3_3c_djbunfft_unroll 3,exec,noexec,noexec,srcreg,srcinc,d1,bcast,screg1,screg2,screg,scinc,L1pt,L1pd
	bump	srcreg, 3*srcinc
	bump	screg, 3*scinc
	ELSEIF maxrpt MOD 2 EQ 0
	yr3_3c_djbunfft_unroll -1,noexec,noexec,exec,srcreg,srcinc,d1,bcast,screg1,screg2,screg,scinc,L1pt,L1pd
	yr3_3c_djbunfft_unroll 0,noexec,exec,exec,srcreg,srcinc,d1,bcast,screg1,screg2,screg,scinc,L1pt,L1pd
	yr3_3c_djbunfft_unroll 1,exec,exec,noexec,srcreg,srcinc,d1,bcast,screg1,screg2,screg,scinc,L1pt,L1pd
	yr3_3c_djbunfft_unroll 2,exec,noexec,noexec,srcreg,srcinc,d1,bcast,screg1,screg2,screg,scinc,L1pt,L1pd
	bump	srcreg, 2*srcinc
	bump	screg, 2*scinc
	ELSE
	yr3_3c_djbunfft_unroll -1,noexec,noexec,exec,srcreg,srcinc,d1,bcast,screg1,screg2,screg,scinc,L1pt,L1pd
	yr3_3c_djbunfft_unroll 0,noexec,exec,noexec,srcreg,srcinc,d1,bcast,screg1,screg2,screg,scinc,L1pt,L1pd
	yr3_3c_djbunfft_unroll 1,exec,noexec,noexec,srcreg,srcinc,d1,bcast,screg1,screg2,screg,scinc,L1pt,L1pd
	bump	srcreg, srcinc
	bump	screg, scinc
	ENDIF
	ENDM

yr3_3c_djbunfft_unroll MACRO iter,prev,this,next,srcreg,srcinc,d1,bcast,screg1,screg2,screg,scinc,L1pt,L1pd

;; On first call, init registers

	IF iter EQ -1
y0	TEXTEQU	<ymm0>
y1	TEXTEQU	<ymm1>
y2	TEXTEQU	<ymm2>
y3	TEXTEQU	<ymm3>
y4	TEXTEQU	<ymm4>
y5	TEXTEQU	<ymm5>
y6	TEXTEQU	<ymm6>
y7	TEXTEQU	<ymm7>
y8	TEXTEQU	<ymm8>
y9	TEXTEQU	<ymm9>
y10	TEXTEQU	<ymm10>
y11	TEXTEQU	<ymm11>
y12	TEXTEQU	<ymm12>
y13	TEXTEQU	<ymm13>
	ENDIF

;; On later calls, previous R1, I1, T1, T2, T3, T4 will be in y0-5.  This (R2,R3,I2,I3)/sine will be in y6-9.
;; The remaining 4 register are free (ymm14 and ymm15 are preloaded constants 0.866 and 0.5).

prev	vsubpd	y10, y2, y3				;; PREVIOUS T1-T2 (final R3)		; 1-3
this no bcast vmovapd y11, [screg1+iter*scinc]		;; sine
this bcast vbroadcastsd y11, Q [screg1+iter*scinc]	;; sine
this	vmulpd	y12, ymm15, y11				;; 0.5 * sine				; 1-5
prev	ystore	[srcreg+(iter-1)*srcinc], y0		;; Save R1				; 1

prev	vaddpd	y2, y2, y3				;; PREVIOUS T1+T2 (final R2)		; 2-4
next no bcast vmovapd y0, [screg2+(iter+1)*scinc]	;; NEXT cosine/sine
next bcast vbroadcastsd y0, Q [screg2+(iter+1)*scinc]	;; NEXT cosine/sine

prev	vsubpd	y13, y4, y5				;; PREVIOUS T3-T4 (final I2)		; 3-5
next	vmovapd	y3, [srcreg+(iter+1)*srcinc+d1]		;; NEXT R2
prev	ystore	[srcreg+(iter-1)*srcinc+32], y1		;; Save I1				; 3
next	vmulpd	y1, y3, y0				;; NEXT A2 = R2 * cosine/sine		; 3-7

prev	vaddpd	y4, y4, y5				;; PREVIOUS T3+T4 (final I3)		; 4-6
next	vmovapd	y5, [srcreg+(iter+1)*srcinc+2*d1]	;; NEXT R3
prev	ystore	[srcreg+(iter-1)*srcinc+2*d1], y10	;; PREVIOUS Save R3			; 4
next	vmulpd	y10, y5, y0				;; NEXT A3 = R3 * cosine/sine		; 4-8

prev	ystore	[srcreg+(iter-1)*srcinc+d1], y2		;; PREVIOUS Save R2			; 5
this	vaddpd	y2, y6, y7				;; R2/sine + R3/sine			; 5-7
prev	ystore	[srcreg+(iter-1)*srcinc+d1+32], y13	;; PREVIOUS Save I2			; 6
next	vmovapd	y13, [srcreg+(iter+1)*srcinc+d1+32]	;; NEXT I2
prev	ystore	[srcreg+(iter-1)*srcinc+2*d1+32], y4	;; PREVIOUS Save I3			; 7
next	vmulpd	y4, y13, y0				;; NEXT B2 = I2 * cosine/sine		; 5-9

this	vsubpd	y6, y6, y7				;; R2/sine - R3/sine			; 6-8
next	vmovapd	y7, [srcreg+(iter+1)*srcinc+2*d1+32]	;; NEXT I3
next	vmulpd	y0, y7, y0				;; NEXT B3 = I3 * cosine/sine		; 6-10

next	vaddpd	y1, y1, y13				;; NEXT A2 = A2 + I2 (new R2/sine)	; 9-11
this	vaddpd	y13, y8, y9				;; I2/sine + I3/sine			; 7-9
this	vsubpd	y8, y8, y9				;; I2/sine - I3/sine			; 8-10
this	vmulpd	y9, ymm14, y11				;; 0.866 * sine				; 7-11
this	L1prefetchw srcreg+iter*srcinc+L1pd, L1pt

next	vsubpd	y10, y10, y7				;; NEXT A3 = A3 - I3 (new R3/sine)	; 10-12
this	vmulpd	y7, y12, y2				;; 0.5 * (R2 + R3)			; 8-12

this	vmulpd	y2, y11, y2				;; R2 + R3				; 9-13
this next yloop_unrolled_one

this	vmulpd	y12, y12, y13				;; 0.5 * (I2 + I3)			; 10-14

next	vsubpd	y4, y4, y3				;; NEXT B2 = B2 - R2 (new I2/sine)	; 11-13
this	vmulpd	y13, y11, y13				;; I2 + I3				; 11-15
this	vmovapd	y3, [srcreg+iter*srcinc]		;; R1
this	L1prefetchw srcreg+iter*srcinc+d1+L1pd, L1pt

next	vaddpd	y0, y0, y5				;; NEXT B3 = B3 + R3 (new I3/sine)	; 12-14
this	vmulpd	y8, y9, y8				;; T2 = 0.866 * (I2 - I3)		; 12-16
this	vmovapd	y11, [srcreg+iter*srcinc+32]		;; I1

this	vsubpd	y7, y3, y7				;; T1 = R1-.5R2-.5R3			; 13-15
this	vmulpd	y6, y9, y6				;; T4 = 0.866 * (R2 - R3)		; 13-17

this	vaddpd	y3, y3, y2				;; R1 + R2 + R3 (final R1)		; 14-16

this	vsubpd	y12, y11, y12				;; T3 = I1-.5I2-.5I3			; 15-17
this	L1prefetchw srcreg+iter*srcinc+2*d1+L1pd, L1pt

this	vaddpd	y11, y11, y13				;; I1 + I2 + I3 (final I1)		; 16-18

;; Shuffle register assignments so that next call has R1, I1, T1, T2, T3, T4 in y0-5 and next (R2,R3,I2,I3)/sine in y6-9.

ytmp	TEXTEQU	y0
y0	TEXTEQU	y3
y3	TEXTEQU	y8
y8	TEXTEQU	y4
y4	TEXTEQU	y12
y12	TEXTEQU	y2
y2	TEXTEQU	y7
y7	TEXTEQU	y10
y10	TEXTEQU	y5
y5	TEXTEQU	y6
y6	TEXTEQU	y1
y1	TEXTEQU	y11
y11	TEXTEQU y9
y9	TEXTEQU	ytmp

	ENDM

;; Haswell FMA3 version

IF (@INSTR(,%yarch,<FMA3>) NE 0)

yr3_3c_djbunfft_cmn_preload MACRO
	vmovapd	ymm15, YMM_HALF
	vmovapd	ymm14, YMM_ONE
	ENDM

;; uops = 6 loads, 6 stores, 2 s/c loads, 1 constant load, 2 muls, 16 fmas, 8 register copies = 41 uops / 4 = 10.25 clocks (above our 9 clock optimum)
;; Timed at 10.5 clocks.
yr3_3c_djbunfft_unroll MACRO iter,prev,this,next,srcreg,srcinc,d1,bcast,screg1,screg2,screg,scinc,L1pt,L1pd

;; On first call, init registers

	IF iter EQ -1
y0	TEXTEQU	<ymm0>
y1	TEXTEQU	<ymm1>
y2	TEXTEQU	<ymm2>
y3	TEXTEQU	<ymm3>
y4	TEXTEQU	<ymm4>
y5	TEXTEQU	<ymm5>
y6	TEXTEQU	<ymm6>
y7	TEXTEQU	<ymm7>
y8	TEXTEQU	<ymm8>
y9	TEXTEQU	<ymm9>
y10	TEXTEQU	<ymm10>
y11	TEXTEQU	<ymm11>
y12	TEXTEQU	<ymm12>
y13	TEXTEQU	<ymm13>
	ENDIF

;; On later calls, previous R1,I1,T1,T3,(R2-R3)/sine,(I2-I3)/sine,0.866*sine will be in y0-6.  This R2/sine,R3/sine,I2/sine,I3/sine,sine will be in y7-11.
;; The remaining 4 register are free (ymm14 and ymm15 are preloaded constants 0.866 and 0.5).

this	vmulpd	y12, ymm15, y11				;; 0.5 * sine					; 1-5
this	vmulpd	y13, y11, YMM_P866			;; 0.866 * sine					; 1-5
prev	ystore	[srcreg+(iter-1)*srcinc], y0		;; Save R1					; 4
this	L1prefetchw srcreg+iter*srcinc+L1pd, L1pt

this	yfmaddpd y0, y7, ymm14, y8			;; R2/sine + R3/sine				; 2-6
this	yfmsubpd y7, y7, ymm14, y8			;; R2/sine - R3/sine				; 2-6
next no bcast vmovapd y8, [screg2+(iter+1)*scinc]	;; cosine/sine
next bcast vbroadcastsd y8, Q [screg2+(iter+1)*scinc]	;; cosine/sine
prev	ystore	[srcreg+(iter-1)*srcinc+32], y1		;; Save I1					; 5

this	yfmaddpd y1, y9, ymm14, y10			;; I2/sine + I3/sine				; 3-7
this	yfmsubpd y9, y9, ymm14, y10			;; I2/sine - I3/sine				; 3-7
this	L1prefetchw srcreg+iter*srcinc+d1+L1pd, L1pt

prev	yfmaddpd y10, y6, y5, y2			;; T1 + 0.866*sine * (I2-I3)/sine (final R2)	; 4-8
prev	yfnmaddpd y5, y6, y5, y2			;; T1 - 0.866*sine * (I2-I3)/sine (final R3)	; 4-8
next	vmovapd	y2, [srcreg+(iter+1)*srcinc+d1]		;; R2
prev	ystore	[srcreg+(iter-1)*srcinc+d1], y10	;; Save R2					; 9

prev	yfmaddpd y10, y6, y4, y3			;; T3 + 0.866*sine * (R2-R3)/sine (final I3)	; 5-9
prev	yfnmaddpd y6, y6, y4, y3			;; T3 - 0.866*sine * (R2-R3)/sine (final I2)	; 5-9
next	vmovapd	y4, [srcreg+(iter+1)*srcinc+d1+32]	;; I2

next	yfmaddpd y3, y2, y8, y4				;; R2 * cosine/sine + I2 (new R2/sine)		; 6-10
prev	ystore	[srcreg+(iter-1)*srcinc+2*d1], y5	;; Save R3					; 9
next	vmovapd	y5, [srcreg+(iter+1)*srcinc+2*d1]	;; R3
prev	ystore	[srcreg+(iter-1)*srcinc+2*d1+32], y10	;; Save I3					; 10
next	vmovapd	y10, [srcreg+(iter+1)*srcinc+2*d1+32]	;; I3
prev	ystore	[srcreg+(iter-1)*srcinc+d1+32], y6	;; Save I2					; 10
next	yfmsubpd y6, y5, y8, y10			;; R3 * cosine/sine - I3 (new R3/sine)		; 6-10

next	yfmsubpd y4, y4, y8, y2				;; I2 * cosine/sine - R2 (new I2/sine)		; 7-11
next	yfmaddpd y10, y10, y8, y5			;; I3 * cosine/sine + R3 (new I3/sine)		; 7-11
this	vmovapd	y2, [srcreg+iter*srcinc]		;; R1
this	L1prefetchw srcreg+iter*srcinc+2*d1+L1pd, L1pt

this	yfnmaddpd y8, y12, y0, y2			;; T1 = R1 - 0.5*sine * (R2+R3)/sine		; 8-12
this	yfmaddpd y0, y11, y0, y2			;; R1 + sine * (R2+R3)/sine (final R1)		; 8-12
this	vmovapd	y5, [srcreg+iter*srcinc+32]		;; I1
this next yloop_unrolled_one

this	yfnmaddpd y12, y12, y1, y5			;; T3 = I1 - 0.5*sine * (I2+I3)/sine		; 9-13
this	yfmaddpd y11, y11, y1, y5			;; I1 + sine * (I2+I3)/sine (final I1)		; 9-13
next no bcast vmovapd y2, [screg1+(iter+1)*scinc]	;; sine
next bcast vbroadcastsd y2, Q [screg1+(iter+1)*scinc]	;; sine

;; Shuffle register assignments so that next call has R1,I1,T1,T3,(R2-R3)/sine,(I2-I3)/sine,0.866*sine in y0-6,
;; and this R2/sine,R3/sine,I2/sine,I3/sine,sine in y7-11.

ytmp	TEXTEQU	y1
y1	TEXTEQU	y11
y11	TEXTEQU	y2
y2	TEXTEQU	y8
y8	TEXTEQU	y6
y6	TEXTEQU	y13
y13	TEXTEQU	y5
y5	TEXTEQU	y9
y9	TEXTEQU	y4
y4	TEXTEQU	y7
y7	TEXTEQU	y3
y3	TEXTEQU	y12
y12	TEXTEQU	ytmp

	ENDM

ENDIF

ENDIF


;;
;; ************************************* six-reals-fft variants ******************************************
;;

; R1 #1 = R1 + R3 + R5
; R1 #2 = R2 + R4 + R6
; R2 = R1 - R4 + 0.5 * (R2 - R3 - R5 + R6)
; I2 = 0.866 * (R2 + R3 - R5 - R6)
; R3 = R1 + R4 - 0.5 * (R2 + R3 + R5 + R6)
; I3 = 0.866 * (R2 - R3 + R5 - R6)

;; The standard version
yr3_3cl_six_reals_fft_preload MACRO
	yr3_6r_fft_cmn_preload
	ENDM
yr3_3cl_six_reals_fft MACRO srcreg,srcinc,d1,screg,scinc,maxrpt,L1pt,L1pd
	yr3_6r_fft_cmn srcreg,srcinc,0,d1,screg,screg+64,screg,scinc,,,maxrpt,L1pt,L1pd
	ENDM

; Like the standard version but uses two sin/cos ptrs
yr3_3cl_2sc_six_reals_fft_preload MACRO
	yr3_6r_fft_cmn_preload
	ENDM
yr3_3cl_2sc_six_reals_fft MACRO srcreg,srcinc,d1,screg1,scinc1,screg2,scinc2,maxrpt,L1pt,L1pd
	yr3_6r_fft_cmn srcreg,srcinc,0,d1,screg2,screg1,screg1,scinc1,screg2,scinc2,maxrpt,L1pt,L1pd
	ENDM

; Like the standard version but the first sin/cos value can be used in a three-complex macro at the same FFT level
yr3_3cl_csc_six_reals_fft_preload MACRO
	yr3_6r_fft_cmn_preload
	ENDM
yr3_3cl_csc_six_reals_fft MACRO srcreg,srcinc,d1,screg,scinc,maxrpt,L1pt,L1pd
	yr3_6r_fft_cmn srcreg,srcinc,0,d1,screg+64,screg,screg,scinc,,,maxrpt,L1pt,L1pd
	ENDM

; Like the standard version but indexes into source using rbx
yr3_f3cl_six_reals_fft_preload MACRO
	yr3_6r_fft_cmn_preload
	ENDM
yr3_f3cl_six_reals_fft MACRO srcreg,srcinc,d1,screg,scinc,maxrpt,L1pt,L1pd
	yr3_6r_fft_cmn srcreg,srcinc,rbx,d1,screg,screg+64,screg,scinc,,,maxrpt,L1pt,L1pd
	ENDM

; Like f3cl but uses 2 sin/cos ptrs
yr3_f3cl_2sc_six_reals_fft_preload MACRO
	yr3_6r_fft_cmn_preload
	ENDM
yr3_f3cl_2sc_six_reals_fft MACRO srcreg,srcinc,d1,screg1,scinc1,screg2,scinc2,maxrpt,L1pt,L1pd
	yr3_6r_fft_cmn srcreg,srcinc,rbx,d1,screg2,screg1,screg1,scinc1,screg2,scinc2,maxrpt,L1pt,L1pd
	ENDM

yr3_6r_fft_cmn_preload MACRO
	ENDM
yr3_6r_fft_cmn MACRO srcreg,srcinc,srcoff,d1,screg1,screg2,scregA,scincA,scregB,scincB,maxrpt,L1pt,L1pd
	vmovapd	ymm4, [srcreg+srcoff+d1]	;; R2
	vmovapd	ymm3, [srcreg+srcoff+2*d1+32]	;; R6
	vaddpd	ymm5, ymm4, ymm3		;; R2 + R6
	vsubpd	ymm4, ymm4, ymm3		;; R2 - R6
	vmovapd	ymm6, [srcreg+srcoff+2*d1]	;; R3
	vmovapd	ymm7, [srcreg+srcoff+d1+32]	;; R5
	vaddpd	ymm0, ymm6, ymm7		;; R3 + R5
	vsubpd	ymm6, ymm6, ymm7		;; R3 - R5

	L1prefetchw srcreg+L1pd, L1pt

	vmovapd	ymm7, YMM_HALF
	vmulpd	ymm1, ymm7, ymm5		;; 0.5 * (R2 + R6)
	vmovapd	ymm2, [srcreg+srcoff+32]	;; R4
	vaddpd	ymm5, ymm2, ymm5		;; final R1 #2 = R4 + R2 + R6
	vmulpd	ymm7, ymm7, ymm0		;; 0.5 * (R3 + R5)
	vmovapd	ymm3, YMM_P866
	vmulpd	ymm4, ymm3, ymm4		;; X = 0.866 * (R2 - R6)
	vmulpd	ymm6, ymm3, ymm6		;; Y = 0.866 * (R3 - R5)
	vmovapd	ymm3, [srcreg+srcoff]		;; R1
	vaddpd	ymm0, ymm3, ymm0		;; final R1 #1 = R1 + R3 + R5
	vsubpd	ymm2, ymm2, ymm1		;; B = R4 - 0.5 * (R2 + R6)
	vsubpd	ymm3, ymm3, ymm7		;; A = R1 - 0.5 * (R3 + R5)

	L1prefetchw srcreg+d1+L1pd, L1pt

	vsubpd	ymm7, ymm4, ymm6		;; X - Y (final I3)
	vaddpd	ymm1, ymm4, ymm6		;; X + Y (final I2)
	vaddpd	ymm6, ymm3, ymm2		;; A + B (final R3)
	vsubpd	ymm3, ymm3, ymm2		;; A - B (final R2)

	L1prefetchw srcreg+2*d1+L1pd, L1pt

	vmovapd	ymm2, [screg2+32]		;; cosine/sine
	vmulpd	ymm4, ymm7, ymm2		;; B3 = I3 * cosine/sine
	vaddpd	ymm4, ymm4, ymm6		;; B3 = B3 + R3
	vmulpd	ymm6, ymm6, ymm2		;; A3 = R3 * cosine/sine
	vsubpd	ymm6, ymm6, ymm7		;; A3 = A3 - I3

	vmovapd	ymm2, [screg1+32]		;; cosine/sine
	vmulpd	ymm7, ymm1, ymm2		;; B2 = I2 * cosine/sine
	vaddpd	ymm7, ymm7, ymm3		;; B2 = B2 + R2
	vmulpd	ymm3, ymm3, ymm2		;; A2 = R2 * cosine/sine
	vsubpd	ymm3, ymm3, ymm1		;; A2 = A2 - I2

	vmovapd	ymm1, [screg2]			;; sine
	vmulpd	ymm4, ymm4, ymm1		;; B3 = B3 * sine (new I3)
	vmulpd	ymm6, ymm6, ymm1		;; A3 = A3 * sine (new R3)
	vmovapd	ymm1, [screg1]			;; sine
	vmulpd	ymm7, ymm7, ymm1		;; B2 = B2 * sine (new I2)
	vmulpd	ymm3, ymm3, ymm1		;; A2 = A2 * sine (new R2)

	ystore	[srcreg], ymm0			;; Save R1
	ystore	[srcreg+32], ymm5		;; Save I1
	ystore	[srcreg+d1], ymm3		;; Save R2
	ystore	[srcreg+d1+32], ymm7		;; Save I2
	ystore	[srcreg+2*d1], ymm6		;; Save R3
	ystore	[srcreg+2*d1+32], ymm4		;; Save I3
	bump	srcreg, srcinc
	bump	scregA, scincA
	bump	scregB, scincB
	ENDM

;; 64-bit version

IFDEF X86_64

yr3_6r_fft_cmn_preload MACRO
	vmovapd	ymm14, YMM_HALF
	vmovapd	ymm15, YMM_P866
	ENDM
yr3_6r_fft_cmn MACRO srcreg,srcinc,srcoff,d1,screg1,screg2,scregA,scincA,scregB,scincB,maxrpt,L1pt,L1pd
	IF maxrpt MOD 4 EQ 0
	yr3_6r_fft_unroll -1,noexec,noexec,exec,srcreg,srcinc,srcoff,d1,screg1,screg2,scregA,scincA,L1pt,L1pd
	yr3_6r_fft_unroll 0,noexec,exec,exec,srcreg,srcinc,srcoff,d1,screg1,screg2,scregA,scincA,L1pt,L1pd
	yr3_6r_fft_unroll 1,exec,exec,exec,srcreg,srcinc,srcoff,d1,screg1,screg2,scregA,scincA,L1pt,L1pd
	yr3_6r_fft_unroll 2,exec,exec,exec,srcreg,srcinc,srcoff,d1,screg1,screg2,scregA,scincA,L1pt,L1pd
	yr3_6r_fft_unroll 3,exec,exec,noexec,srcreg,srcinc,srcoff,d1,screg1,screg2,scregA,scincA,L1pt,L1pd
	yr3_6r_fft_unroll 4,exec,noexec,noexec,srcreg,srcinc,srcoff,d1,screg1,screg2,scregA,scincA,L1pt,L1pd
	bump	srcreg, 4*srcinc
	bump	scregA, 4*scincA
	bump	scregB, 4*scincB
	ELSEIF maxrpt MOD 3 EQ 0
	yr3_6r_fft_unroll -1,noexec,noexec,exec,srcreg,srcinc,srcoff,d1,screg1,screg2,scregA,scincA,L1pt,L1pd
	yr3_6r_fft_unroll 0,noexec,exec,exec,srcreg,srcinc,srcoff,d1,screg1,screg2,scregA,scincA,L1pt,L1pd
	yr3_6r_fft_unroll 1,exec,exec,exec,srcreg,srcinc,srcoff,d1,screg1,screg2,scregA,scincA,L1pt,L1pd
	yr3_6r_fft_unroll 2,exec,exec,noexec,srcreg,srcinc,srcoff,d1,screg1,screg2,scregA,scincA,L1pt,L1pd
	yr3_6r_fft_unroll 3,exec,noexec,noexec,srcreg,srcinc,srcoff,d1,screg1,screg2,scregA,scincA,L1pt,L1pd
	bump	srcreg, 3*srcinc
	bump	scregA, 3*scincA
	bump	scregB, 3*scincB
	ELSEIF maxrpt MOD 2 EQ 0
	yr3_6r_fft_unroll -1,noexec,noexec,exec,srcreg,srcinc,srcoff,d1,screg1,screg2,scregA,scincB,L1pt,L1pd
	yr3_6r_fft_unroll 0,noexec,exec,exec,srcreg,srcinc,srcoff,d1,screg1,screg2,scregA,scincB,L1pt,L1pd
	yr3_6r_fft_unroll 1,exec,exec,noexec,srcreg,srcinc,srcoff,d1,screg1,screg2,scregA,scincB,L1pt,L1pd
	yr3_6r_fft_unroll 2,exec,noexec,noexec,srcreg,srcinc,srcoff,d1,screg1,screg2,scregA,scincB,L1pt,L1pd
	bump	srcreg, 2*srcinc
	bump	scregA, 2*scincA
	bump	scregB, 2*scincB
	ELSE
	yr3_6r_fft_unroll -1,noexec,noexec,exec,srcreg,srcinc,srcoff,d1,screg1,screg2,scregA,scincA,L1pt,L1pd
	yr3_6r_fft_unroll 0,noexec,exec,noexec,srcreg,srcinc,srcoff,d1,screg1,screg2,scregA,scincA,L1pt,L1pd
	yr3_6r_fft_unroll 1,exec,noexec,noexec,srcreg,srcinc,srcoff,d1,screg1,screg2,scregA,scincA,L1pt,L1pd
	bump	srcreg, srcinc
	bump	scregA, scincA
	bump	scregB, scincB
	ENDIF
	ENDM

;;; WARNING:  Unrolling only works in scincB is blank/zero or scincA equals scincB

yr3_6r_fft_unroll MACRO iter,prev,this,next,srcreg,srcinc,srcoff,d1,screg1,screg2,screg,scinc,L1pt,L1pd

;; On first call, init registers

	IF iter EQ -1
y0	TEXTEQU	<ymm0>
y1	TEXTEQU	<ymm1>
y2	TEXTEQU	<ymm2>
y3	TEXTEQU	<ymm3>
y4	TEXTEQU	<ymm4>
y5	TEXTEQU	<ymm5>
y6	TEXTEQU	<ymm6>
y7	TEXTEQU	<ymm7>
y8	TEXTEQU	<ymm8>
y9	TEXTEQU	<ymm9>
y10	TEXTEQU	<ymm10>
y11	TEXTEQU	<ymm11>
y12	TEXTEQU	<ymm12>
y13	TEXTEQU	<ymm13>
	ENDIF

;; On later calls, previous A2,B2,A3,B3,R2,I2,R3,I3 will be in y0-7.  This R2+R6,R2-R6,R3+R5,R3-R5,0.5*(R2+R6) will be in y8-12.
;; The remaining register is free (ymm14 and ymm15 are preloaded constants 0.866 and 0.5).

this	vmovapd	y13, [srcreg+iter*srcinc+srcoff+32] ;; R4
this	vaddpd	y8, y13, y8			;; final R1 #2 = R4 + R2 + R6	; 1-3
this	vmulpd	y9, ymm15, y9			;; X = 0.866 * (R2 - R6)	;	1-5

prev	vaddpd	y3, y3, y6			;; B3 = B3 + R3			; 2-4
this	vmulpd	y6, ymm14, y10			;; 0.5 * (R3 + R5)		;	2-6
this	L1prefetchw srcreg+iter*srcinc+L1pd, L1pt

this	ystore	[srcreg+iter*srcinc+32], y8	;; Save I1			; 4
this	vmovapd	y8, [srcreg+iter*srcinc+srcoff] ;; R1
this	vaddpd	y10, y8, y10			;; final R1 #1 = R1 + R3 + R5	; 3-5
this	vmulpd	y11, ymm15, y11			;; Y = 0.866 * (R3 - R5)	;	3-7

prev	vaddpd	y1, y1, y4			;; B2 = B2 + R2			; 4-6
prev	vmovapd	y4, [screg2+(iter-1)*scinc]	;; sine

prev	vsubpd	y2, y2, y7			;; A3 = A3 - I3			; 5-7
prev	vmulpd	y3, y3, y4			;; B3 = B3 * sine (new I3)	;	5-9
prev	vmovapd	y7, [screg1+(iter-1)*scinc]	;; sine

this	vsubpd	y13, y13, y12			;; B = R4 - 0.5 * (R2 + R6)	; 6-8
this	vmovapd	y12, [screg2+iter*scinc+32]	;; cosine/sine
this	ystore	[srcreg+iter*srcinc], y10	;; Save R1			; 6

prev	vsubpd	y0, y0, y5			;; A2 = A2 - I2			; 7-9
prev	vmulpd	y1, y1, y7			;; B2 = B2 * sine (new I2)	;	7-11
next	vmovapd	y10, [srcreg+(iter+1)*srcinc+srcoff+d1] ;; R2

this	vsubpd	y8, y8, y6			;; A = R1 - 0.5 * (R3 + R5)	; 8-10
prev	vmulpd	y2, y2, y4			;; A3 = A3 * sine (new R3)	;	8-12
next	vmovapd	y5, [srcreg+(iter+1)*srcinc+srcoff+2*d1+32] ;; R6

this	vsubpd	y4, y9, y11			;; X - Y (final I3)		; 9-11
this	vmovapd	y6, [screg1+iter*scinc+32]	;; cosine/sine

this	vaddpd	y9, y9, y11			;; X + Y (final I2)		; 10-12
prev	vmulpd	y0, y0, y7			;; A2 = A2 * sine (new R2)	;	10-13
next	vmovapd	y11, [srcreg+(iter+1)*srcinc+srcoff+2*d1] ;; R3
prev	ystore	[srcreg+(iter-1)*srcinc+2*d1+32], y3 ;; Save I3			; 10

this	vaddpd	y3, y8, y13			;; A + B (final R3)		; 11-13
next	vmovapd	y7, [srcreg+(iter+1)*srcinc+srcoff+d1+32] ;; R5

this	vsubpd	y8, y8, y13			;; A - B (final R2)		; 12-14
this	vmulpd	y13, y4, y12			;; B3 = I3 * cosine/sine	;	12-16
prev	ystore	[srcreg+(iter-1)*srcinc+d1+32], y1 ;; Save I2			; 12
this	L1prefetchw srcreg+iter*srcinc+d1+L1pd, L1pt

next	vaddpd	y1, y10, y5			;; R2 + R6			; 13-15
prev	ystore	[srcreg+(iter-1)*srcinc+2*d1], y2 ;; Save R3			; 13
this	vmulpd	y2, y9, y6			;; B2 = I2 * cosine/sine	;	13-17

next	vsubpd	y10, y10, y5			;; R2 - R6			; 14-16
this	vmulpd	y12, y3, y12			;; A3 = R3 * cosine/sine	;	14-18
this	L1prefetchw srcreg+iter*srcinc+2*d1+L1pd, L1pt

next	vaddpd	y5, y11, y7			;; R3 + R5			; 15-17
this	vmulpd	y6, y8, y6			;; A2 = R2 * cosine/sine	;	15-19
prev	ystore	[srcreg+(iter-1)*srcinc+d1], y0 ;; Save R2			; 15

next	vsubpd	y11, y11, y7			;; R3 - R5			; 16-18
next	vmulpd	y7, ymm14, y1			;; 0.5 * (R2 + R6)		;	16-20
this next yloop_unrolled_one

;; Shuffle register assignments so that next call has A2,B2,A3,B3,R2,I2,R3,I3 in y0-7 and next R2+R6,R2-R6,R3+R5,R3-R5,0.5*(R2+R6) in y8-12.

ytmp	TEXTEQU	y0
y0	TEXTEQU	y6
y6	TEXTEQU	y3
y3	TEXTEQU	y13
y13	TEXTEQU	ytmp
ytmp	TEXTEQU	y1
y1	TEXTEQU	y2
y2	TEXTEQU	y12
y12	TEXTEQU	y7
y7	TEXTEQU	y4
y4	TEXTEQU	y8
y8	TEXTEQU	ytmp
ytmp	TEXTEQU	y5
y5	TEXTEQU	y9
y9	TEXTEQU	y10
y10	TEXTEQU ytmp

	ENDM


;; Haswell FMA3 version

IF (@INSTR(,%yarch,<FMA3>) NE 0)

yr3_6r_fft_cmn_preload MACRO
	vmovapd	ymm13, YMM_ONE
	vmovapd	ymm14, YMM_HALF
	vmovapd	ymm15, YMM_P866
	ENDM

;; 6 loads, 6 stores, 4 sin/cos loads, 5 muls, 16 FMAs, 8 movs = 45 uops.  Best case is 45/4 = 11.25 clocks.
;; Timed at 12 clocks.
yr3_6r_fft_unroll MACRO iter,prev,this,next,srcreg,srcinc,srcoff,d1,screg1,screg2,screg,scinc,L1pt,L1pd

;; On first call, init registers

	IF iter EQ -1
y0	TEXTEQU	<ymm0>
y1	TEXTEQU	<ymm1>
y2	TEXTEQU	<ymm2>
y3	TEXTEQU	<ymm3>
y4	TEXTEQU	<ymm4>
y5	TEXTEQU	<ymm5>
y6	TEXTEQU	<ymm6>
y7	TEXTEQU	<ymm7>
y8	TEXTEQU	<ymm8>
y9	TEXTEQU	<ymm9>
y10	TEXTEQU	<ymm10>
y11	TEXTEQU	<ymm11>
y12	TEXTEQU	<ymm12>
	ENDIF

;; On later calls, previous R2,I2,R3,I3 will be in y0-3.  This R1,I1,A,B,R2-R6,R3-R5 will be in y4-9.
;; The remaining registers are free (ymm14 and ymm15 are preloaded constants 0.5 and 0.866, ymm13 reserved for Haswell).

prev	vmovapd	y10, [screg1+(iter-1)*scinc+32]	;; cosine/sine for R2/I2
prev	vmovapd	y11, [screg2+(iter-1)*scinc+32]	;; cosine/sine for R3/I3
this	vmulpd	y8, ymm15, y8			;; X = 0.866 * (R2-R6)		; 1-5

next	vmovapd	y12, [srcreg+(iter+1)*srcinc+srcoff+d1] ;; R2
this	ystore	[srcreg+iter*srcinc+32], y5	;; Save I1			; 4
prev	yfmsubpd y5, y0, y10, y1		;; A2 = R2 * cosine/sine - I2	; 2-6
prev	yfmaddpd y1, y1, y10, y0		;; B2 = I2 * cosine/sine + R2	; 2-6

next	vmovapd	y10, [srcreg+(iter+1)*srcinc+srcoff+2*d1+32] ;; R6
prev	yfmsubpd y0, y2, y11, y3		;; A3 = R3 * cosine/sine - I3	; 3-7
prev	yfmaddpd y3, y3, y11, y2		;; B3 = I3 * cosine/sine + R3	; 3-7
this	L1prefetchw srcreg+iter*srcinc+L1pd, L1pt

next	vmovapd	y11, [srcreg+(iter+1)*srcinc+srcoff+2*d1] ;; R3
next	yfmaddpd y2, y12, ymm13, y10		;; R2 + R6			; 4-8
next	yfmsubpd y12, y12, ymm13, y10		;; R2 - R6			; 4-8

next	vmovapd	y10, [srcreg+(iter+1)*srcinc+srcoff+d1+32] ;; R5
this	ystore	[srcreg+iter*srcinc], y4	;; Save R1			; 5
next	yfmaddpd y4, y11, ymm13, y10		;; R3 + R5			; 5-9
next	yfmsubpd y11, y11, ymm13, y10		;; R3 - R5			; 5-9

this	yfmsubpd y10, y6, ymm13, y7		;; A - B (new R2)		; 6-10
this	yfmaddpd y6, y6, ymm13, y7		;; A + B (new R3)		; 6-10
this	L1prefetchw srcreg+iter*srcinc+d1+L1pd, L1pt

this	yfmaddpd y7, ymm15, y9, y8		;; X + 0.866 * (R3-R5) (new I2)	; 7-11
this	yfnmaddpd y9, ymm15, y9, y8		;; X - 0.866 * (R3-R5) (new I3)	; 7-11
this	L1prefetchw srcreg+iter*srcinc+2*d1+L1pd, L1pt

prev	vmovapd	y8, [screg1+(iter-1)*scinc]	;; sine
prev	vmulpd	y5, y5, y8			;; A2 = A2 * sine (final R2)	; 8-12
prev	vmulpd	y1, y1, y8			;; B2 = B2 * sine (final I2)	; 8-12

prev	vmovapd	y8, [screg2+(iter-1)*scinc]	;; sine
prev	vmulpd	y0, y0, y8			;; A3 = A3 * sine (final R3)	; 9-13
prev	vmulpd	y3, y3, y8			;; B3 = B3 * sine (final I3)	; 9-13
this next yloop_unrolled_one

next	vmovapd	y8, [srcreg+(iter+1)*srcinc+srcoff+32] ;; R4
prev	ystore	[srcreg+(iter-1)*srcinc+d1], y5 ;; Save R2			; 13
next	yfmaddpd y5, y8, ymm13, y2		;; final R1 #2 = R4 + (R2+R6)	; 10-14
next	yfnmaddpd y2, ymm14, y2, y8		;; B = R4 - 0.5 * (R2+R6)	; 10-14

next	vmovapd	y8, [srcreg+(iter+1)*srcinc+srcoff] ;; R1
prev	ystore	[srcreg+(iter-1)*srcinc+d1+32], y1 ;; Save I2			; 13+1
next	yfmaddpd y1, y8, ymm13, y4		;; final R1 #1 = R1 + (R3+R5)	; 11-15
next	yfnmaddpd y4, ymm14, y4, y8		;; A = R1 - 0.5 * (R3+R5)	; 11-15
prev	ystore	[srcreg+(iter-1)*srcinc+2*d1], y0 ;; Save R3			; 14+1
prev	ystore	[srcreg+(iter-1)*srcinc+2*d1+32], y3 ;; Save I3			; 14+2

;; Shuffle register assignments so that next call has R2,I2,R3,I3 in y0-3 and R1,I1,A,B,R2-R6,R3-R5 in y4-9.

ytmp	TEXTEQU	y0
y0	TEXTEQU	y10
y10	TEXTEQU	y8
y8	TEXTEQU	y12
y12	TEXTEQU	y3
y3	TEXTEQU	y9
y9	TEXTEQU	y11
y11	TEXTEQU	ytmp
ytmp	TEXTEQU	y1
y1	TEXTEQU	y7
y7	TEXTEQU	y2
y2	TEXTEQU	y6
y6	TEXTEQU	y4
y4	TEXTEQU	ytmp

	ENDM

ENDIF

ENDIF


;;
;; ************************************* six-reals-unfft variants ******************************************
;;

; R1 = R1#1 + (R2 + R3)
; R2 = R1#2 + 0.5 * (R2 - R3) + 0.866 * (I2 + I3)
; R3 = R1#1 - 0.5 * (R2 + R3) + 0.866 * (I2 - I3)
; R4 = R1#2 - (R2 - R3)
; R5 = R1#1 - 0.5 * (R2 + R3) - 0.866 * (I2 - I3)
; R6 = R1#2 + 0.5 * (R2 - R3) - 0.866 * (I2 + I3)

; The standard version
yr3_3cl_six_reals_unfft_preload MACRO
	yr3_6r_unfft_cmn_preload
	ENDM
yr3_3cl_six_reals_unfft MACRO srcreg,srcinc,d1,screg,scinc,maxrpt,L1pt,L1pd
	yr3_6r_unfft_cmn srcreg,srcinc,d1,screg,screg+64,screg,scinc,,,maxrpt,L1pt,L1pd
	ENDM

; Like standard version but uses 2 sin/cos ptrs
yr3_3cl_2sc_six_reals_unfft_preload MACRO
	yr3_6r_unfft_cmn_preload
	ENDM
yr3_3cl_2sc_six_reals_unfft MACRO srcreg,srcinc,d1,screg1,scinc1,screg2,scinc2,maxrpt,L1pt,L1pd
	yr3_6r_unfft_cmn srcreg,srcinc,d1,screg2,screg1,screg1,scinc1,screg2,scinc2,maxrpt,L1pt,L1pd
	ENDM

; Like standard version but the first sin/cos value can be used by a three-complex macro at the same FFT level
yr3_3cl_csc_six_reals_unfft_preload MACRO
	yr3_6r_unfft_cmn_preload
	ENDM
yr3_3cl_csc_six_reals_unfft MACRO srcreg,srcinc,d1,screg,scinc,maxrpt,L1pt,L1pd
	yr3_6r_unfft_cmn srcreg,srcinc,d1,screg+64,screg,screg,scinc,,,maxrpt,L1pt,L1pd
	ENDM

yr3_6r_unfft_cmn_preload MACRO
	ENDM
yr3_6r_unfft_cmn MACRO srcreg,srcinc,d1,screg1,screg2,scregA,scincA,scregB,scincB,maxrpt,L1pt,L1pd
	vmovapd	ymm1, [srcreg+d1]		;; R2
	vmovapd	ymm2, [screg1+32]		;; cosine/sine
	vmulpd	ymm7, ymm1, ymm2		;; A2 = R2 * cosine/sine
	vmovapd	ymm3, [srcreg+2*d1]		;; R3
	vmovapd	ymm6, [screg2+32]		;; cosine/sine
	vmulpd	ymm4, ymm3, ymm6		;; A3 = R3 * cosine/sine
	vmovapd	ymm0, [srcreg+d1+32]		;; I2
	vaddpd	ymm7, ymm7, ymm0		;; A2 = A2 + I2
	vmovapd	ymm5, [srcreg+2*d1+32]		;; I3
	vaddpd	ymm4, ymm4, ymm5		;; A3 = A3 + I3
	vmulpd	ymm0, ymm0, ymm2		;; B2 = I2 * cosine/sine
	vmulpd	ymm5, ymm5, ymm6		;; B3 = I3 * cosine/sine
	vsubpd	ymm0, ymm0, ymm1		;; B2 = B2 - R2
	vsubpd	ymm5, ymm5, ymm3		;; B3 = B3 - R3
	vmovapd	ymm2, [screg1]			;; sine
	vmulpd	ymm7, ymm7, ymm2		;; A2 = A2 * sine (final R2)
	vmovapd	ymm6, [screg2]			;; sine
	vmulpd	ymm4, ymm4, ymm6		;; A3 = A3 * sine (final R3)
	vmulpd	ymm0, ymm0, ymm2		;; B2 = B2 * sine (final I2)
	vmulpd	ymm5, ymm5, ymm6		;; B3 = B3 * sine (final I3)

	L1prefetchw srcreg+L1pd, L1pt

	vsubpd	ymm2, ymm4, ymm7		;; R3 - R2
	vaddpd	ymm1, ymm7, ymm4		;; R2 + R3
	vsubpd	ymm3, ymm0, ymm5		;; I2 - I3
	vaddpd	ymm0, ymm0, ymm5		;; I2 + I3

	L1prefetchw srcreg+d1+L1pd, L1pt

	vmovapd	ymm7, YMM_P866
	vmulpd	ymm3, ymm7, ymm3		;; B = 0.866 * (I2 - I3)
	vmulpd	ymm4, ymm7, ymm0		;; Y = 0.866 * (I2 + I3)

	vmovapd	ymm7, YMM_HALF
	vmulpd	ymm5, ymm7, ymm1		;; 0.5 * (R2 + R3)
	vmulpd	ymm0, ymm7, ymm2		;; 0.5 * (R3 - R2)

	L1prefetchw srcreg+2*d1+L1pd, L1pt

	vmovapd	ymm7, [srcreg]			;; R1#1
	vaddpd	ymm1, ymm7, ymm1		;; R1#1 + R2 + R3 (final R1)
	vsubpd	ymm7, ymm7, ymm5		;; A = R1#1 - 0.5 * (R2 + R3)

	vmovapd	ymm5, [srcreg+32]		;; R1#2
	vaddpd	ymm2, ymm5, ymm2		;; R1#2 + (R3 - R2) (final R4)
	vsubpd	ymm5, ymm5, ymm0		;; X = R1#2 - 0.5 * (R3 - R2)
	vsubpd	ymm0, ymm7, ymm3		;; A - B (final R5)
	vaddpd	ymm7, ymm7, ymm3		;; A + B (final R3)
	vsubpd	ymm3, ymm5, ymm4		;; X - Y (final R6)
	vaddpd	ymm5, ymm5, ymm4		;; X + Y (final R2)

	ystore	[srcreg], ymm1			;; Save R1
	ystore	[srcreg+32], ymm2		;; Save I1
	ystore	[srcreg+d1], ymm5		;; Save R2
	ystore	[srcreg+d1+32], ymm0		;; Save I2
	ystore	[srcreg+2*d1], ymm7		;; Save R3
	ystore	[srcreg+2*d1+32], ymm3		;; Save I3
	bump	srcreg, srcinc
	bump	scregA, scincA
	bump	scregB, scincB
	ENDM

;; 64-bit version

IFDEF X86_64

yr3_6r_unfft_cmn_preload MACRO
	vmovapd	ymm14, YMM_HALF
	vmovapd	ymm15, YMM_P866
	ENDM
yr3_6r_unfft_cmn MACRO srcreg,srcinc,d1,screg1,screg2,scregA,scincA,scregB,scincB,maxrpt,L1pt,L1pd
	IF maxrpt MOD 4 EQ 0
	yr3_6r_unfft_unroll -1,noexec,noexec,exec,srcreg,srcinc,d1,screg1,screg2,scregA,scincA,L1pt,L1pd
	yr3_6r_unfft_unroll 0,noexec,exec,exec,srcreg,srcinc,d1,screg1,screg2,scregA,scincA,L1pt,L1pd
	yr3_6r_unfft_unroll 1,exec,exec,exec,srcreg,srcinc,d1,screg1,screg2,scregA,scincA,L1pt,L1pd
	yr3_6r_unfft_unroll 2,exec,exec,exec,srcreg,srcinc,d1,screg1,screg2,scregA,scincA,L1pt,L1pd
	yr3_6r_unfft_unroll 3,exec,exec,noexec,srcreg,srcinc,d1,screg1,screg2,scregA,scincA,L1pt,L1pd
	yr3_6r_unfft_unroll 4,exec,noexec,noexec,srcreg,srcinc,d1,screg1,screg2,scregA,scincA,L1pt,L1pd
	bump	srcreg, 4*srcinc
	bump	scregA, 4*scincA
	bump	scregB, 4*scincB
	ELSEIF maxrpt MOD 3 EQ 0
	yr3_6r_unfft_unroll -1,noexec,noexec,exec,srcreg,srcinc,d1,screg1,screg2,scregA,scincA,L1pt,L1pd
	yr3_6r_unfft_unroll 0,noexec,exec,exec,srcreg,srcinc,d1,screg1,screg2,scregA,scincA,L1pt,L1pd
	yr3_6r_unfft_unroll 1,exec,exec,exec,srcreg,srcinc,d1,screg1,screg2,scregA,scincA,L1pt,L1pd
	yr3_6r_unfft_unroll 2,exec,exec,noexec,srcreg,srcinc,d1,screg1,screg2,scregA,scincA,L1pt,L1pd
	yr3_6r_unfft_unroll 3,exec,noexec,noexec,srcreg,srcinc,d1,screg1,screg2,scregA,scincA,L1pt,L1pd
	bump	srcreg, 3*srcinc
	bump	scregA, 3*scincA
	bump	scregB, 3*scincB
	ELSEIF maxrpt MOD 2 EQ 0
	yr3_6r_unfft_unroll -1,noexec,noexec,exec,srcreg,srcinc,d1,screg1,screg2,scregA,scincA,L1pt,L1pd
	yr3_6r_unfft_unroll 0,noexec,exec,exec,srcreg,srcinc,d1,screg1,screg2,scregA,scincA,L1pt,L1pd
	yr3_6r_unfft_unroll 1,exec,exec,noexec,srcreg,srcinc,d1,screg1,screg2,scregA,scincA,L1pt,L1pd
	yr3_6r_unfft_unroll 2,exec,noexec,noexec,srcreg,srcinc,d1,screg1,screg2,scregA,scincA,L1pt,L1pd
	bump	srcreg, 2*srcinc
	bump	scregA, 2*scincA
	bump	scregB, 2*scincB
	ELSE
	yr3_6r_unfft_unroll -1,noexec,noexec,exec,srcreg,srcinc,d1,screg1,screg2,scregA,scincA,L1pt,L1pd
	yr3_6r_unfft_unroll 0,noexec,exec,noexec,srcreg,srcinc,d1,screg1,screg2,scregA,scincA,L1pt,L1pd
	yr3_6r_unfft_unroll 1,exec,noexec,noexec,srcreg,srcinc,d1,screg1,screg2,scregA,scincA,L1pt,L1pd
	bump	srcreg, srcinc
	bump	scregA, scincA
	bump	scregB, scincB
	ENDIF
	ENDM

;;; WARNING:  Unrolling only works in scincB is blank/zero or scincA equals scincB

yr3_6r_unfft_unroll MACRO iter,prev,this,next,srcreg,srcinc,d1,screg1,screg2,screg,scinc,L1pt,L1pd

;; On first call, init registers

	IF iter EQ -1
y0	TEXTEQU	<ymm0>
y1	TEXTEQU	<ymm1>
y2	TEXTEQU	<ymm2>
y3	TEXTEQU	<ymm3>
y4	TEXTEQU	<ymm4>
y5	TEXTEQU	<ymm5>
y6	TEXTEQU	<ymm6>
y7	TEXTEQU	<ymm7>
y8	TEXTEQU	<ymm8>
y9	TEXTEQU	<ymm9>
y10	TEXTEQU	<ymm10>
y11	TEXTEQU	<ymm11>
y12	TEXTEQU	<ymm12>
y13	TEXTEQU	<ymm13>
	ENDIF

;; On later calls, previous X,Y.A.B will be in y0-3.  This R1,I1,R2,R3,B2,B3,sine2,sine3 will be in y4-11.
;; The remaining registers are free (ymm14 and ymm15 are preloaded constants 0.5 and 0.866).

prev	vsubpd	y12, y0, y1			;; X - Y (final R6)			; 1-3
this	vmulpd	y8, y8, y10			;; B2 = B2 * sine (final I2)		;	1-5
next	vmovapd	y13, [srcreg+(iter+1)*srcinc+d1] ;; R2

prev	vaddpd	y0, y0, y1			;; X + Y (final R2)			; 2-4
this	vmulpd	y9, y9, y11			;; B3 = B3 * sine (final I3)		;	2-6
next	vmovapd	y10, [screg1+(iter+1)*scinc+32]	;; cosine/sine
prev	ystore	[srcreg+(iter-1)*srcinc+32], y5	;; Save I1				; 2

prev	vsubpd	y5, y2, y3			;; A - B (final R5)			; 3-5
next	vmovapd	y1, [srcreg+(iter+1)*srcinc+2*d1] ;; R3
prev	ystore	[srcreg+(iter-1)*srcinc], y4	;; Save R1				; 3

prev	vaddpd	y2, y2, y3			;; A + B (final R3)			; 4-6
next	vmulpd	y3, y13, y10			;; A2 = R2 * cosine/sine		;	4-8
next	vmovapd	y11, [screg2+(iter+1)*scinc+32]	;; cosine/sine
prev	ystore	[srcreg+(iter-1)*srcinc+2*d1+32], y12 ;; Save I3			; 4

this	vsubpd	y12, y7, y6			;; R3 - R2				; 5-7
next	vmulpd	y4, y1, y11			;; A3 = R3 * cosine/sine		;	5-9
prev	ystore	[srcreg+(iter-1)*srcinc+d1], y0 ;; Save R2				; 5
next	vmovapd	y0, [srcreg+(iter+1)*srcinc+d1+32] ;; I2

this	vaddpd	y6, y6, y7			;; R2 + R3				; 6-8
next	vmulpd	y10, y0, y10			;; B2 = I2 * cosine/sine		;	6-10
next	vmovapd	y7, [srcreg+(iter+1)*srcinc+2*d1+32] ;; I3
prev	ystore	[srcreg+(iter-1)*srcinc+d1+32], y5 ;; Save I2				; 6

this	vaddpd	y5, y8, y9			;; I2 + I3				; 7-9
next	vmulpd	y11, y7, y11			;; B3 = I3 * cosine/sine		;	7-11
prev	ystore	[srcreg+(iter-1)*srcinc+2*d1], y2 ;; Save R3				; 7

this	vsubpd	y8, y8, y9			;; I2 - I3				; 8-10
this	vmulpd	y9, ymm14, y12			;; 0.5 * (R3 - R2)			;	8-12
this	vmovapd	y2, [srcreg+iter*srcinc+32]	;; R1#2

next	vaddpd	y3, y3, y0			;; A2 = A2 + I2				; 9-11
this	vmulpd	y0, ymm14, y6			;; 0.5 * (R2 + R3)			;	9-13
this	L1prefetchw srcreg+iter*srcinc+L1pd, L1pt

next	vaddpd	y4, y4, y7			;; A3 = A3 + I3				; 10-12
this	vmulpd	y5, ymm15, y5			;; Y = 0.866 * (I2 + I3)		;	10-14
this	vmovapd	y7, [srcreg+iter*srcinc]	;; R1#1

next	vsubpd	y10, y10, y13			;; B2 = B2 - R2				; 11-13
this	vmulpd	y8, ymm15, y8			;; B = 0.866 * (I2 - I3)		;	11-15
next	vmovapd	y13, [screg1+(iter+1)*scinc]	;; sine

next	vsubpd	y11, y11, y1			;; B3 = B3 - R3				; 12-14
next	vmovapd	y1, [screg2+(iter+1)*scinc]	;; sine

this	vsubpd	y9, y2, y9			;; X = R1#2 - 0.5 * (R3 - R2)		; 13-15
this	L1prefetchw srcreg+iter*srcinc+d1+L1pd, L1pt

this	vsubpd	y0, y7, y0			;; A = R1#1 - 0.5 * (R2 + R3)		; 14-16

this	vaddpd	y2, y2, y12			;; R1#2 + (R3 - R2) (final R4)		; 15-17
next	vmulpd	y3, y3, y13			;; A2 = A2 * sine (final R2)		;	15-19
this	L1prefetchw srcreg+iter*srcinc+2*d1+L1pd, L1pt

this	vaddpd	y7, y7, y6			;; R1#1 + R2 + R3 (final R1)		; 16-18
next	vmulpd	y4, y4, y1			;; A3 = A3 * sine (final R3)		;	16-20
this next yloop_unrolled_one

;; Shuffle register assignments so that next call has X,Y.A.B in y0-3 and next R1,I1,R2,R3,B2,B3,sine2,sine3 in y4-11.

ytmp	TEXTEQU	y0
y0	TEXTEQU	y9
y9	TEXTEQU	y11
y11	TEXTEQU	y1
y1	TEXTEQU	y5
y5	TEXTEQU	y2
y2	TEXTEQU	ytmp
ytmp	TEXTEQU	y3
y3	TEXTEQU	y8
y8	TEXTEQU	y10
y10	TEXTEQU	y13
y13	TEXTEQU	y6
y6	TEXTEQU	ytmp
ytmp	TEXTEQU	y4
y4	TEXTEQU	y7
y7	TEXTEQU ytmp

	ENDM


;; Haswell FMA3 version

IF (@INSTR(,%yarch,<FMA3>) NE 0)

yr3_6r_unfft_cmn_preload MACRO
	vmovapd	ymm13, YMM_ONE
	vmovapd	ymm14, YMM_HALF
	vmovapd	ymm15, YMM_P866
	ENDM

;; 6 loads, 6 stores, 4 sin/cos loads, 2 muls, 16 FMAs, 8 movs = 42 uops.  Best case is 42/4 = 10.5 clocks.
;; Timed at 11.25 clocks.
yr3_6r_unfft_unroll MACRO iter,prev,this,next,srcreg,srcinc,d1,screg1,screg2,screg,scinc,L1pt,L1pd

;; On first call, init registers

	IF iter EQ -1
y0	TEXTEQU	<ymm0>
y1	TEXTEQU	<ymm1>
y2	TEXTEQU	<ymm2>
y3	TEXTEQU	<ymm3>
y4	TEXTEQU	<ymm4>
y5	TEXTEQU	<ymm5>
y6	TEXTEQU	<ymm6>
y7	TEXTEQU	<ymm7>
y8	TEXTEQU	<ymm8>
y9	TEXTEQU	<ymm9>
y10	TEXTEQU	<ymm10>
y11	TEXTEQU	<ymm11>
y12	TEXTEQU	<ymm12>
	ENDIF

;; On later iterations, previous R1#1,R1#2,R3-R2.R2+R3,I2+I3,I2-I3 are in y0-5.  This R2,I2,A3,B3 are in y6-9.
;; The remaining registers are free (ymm13, ymm14, and ymm15 are preloaded constants 1.0, 0.5, and 0.866).

prev	yfmaddpd y11, y1, ymm13, y2		;; R1#2 + (R3 - R2) (final R4)		; 1-5
prev	yfnmaddpd y2, ymm14, y2, y1		;; X = R1#2 - 0.5 * (R3 - R2)		; 1-5			n 6
next	vmovapd	y10, [srcreg+(iter+1)*srcinc+d1] ;; R2
this	L1prefetchw srcreg+iter*srcinc+d1+L1pd, L1pt

prev	yfmaddpd y12, y0, ymm13, y3		;; R1#1 + R2 + R3 (final R1)		; 2-6
prev	yfnmaddpd y3, ymm14, y3, y0		;; A = R1#1 - 0.5 * (R2 + R3)		; 2-6			n 9
next	vmovapd	y1, [srcreg+(iter+1)*srcinc+d1+32] ;; I2

next	vmovapd	y0, [screg1+(iter+1)*scinc+32]	;; cosine/sine for R2/I2
prev	ystore	[srcreg+(iter-1)*srcinc+32], y11 ;; Save I1				; 6
next	yfmaddpd y11, y10, y0, y1		;; A2 = R2 * cosine/sine + I2		; 3-7			n 8
next	yfmsubpd y1, y1, y0, y10		;; B2 = I2 * cosine/sine - R2		; 3-7
this	L1prefetchw srcreg+iter*srcinc+2*d1+L1pd, L1pt

this	vmovapd	y10, [screg2+iter*scinc]	;; sine
this	yfmsubpd y0, y8, y10, y6		;; A3 * sine - R2			; 4-8			n next 1
this	yfmaddpd y8, y8, y10, y6		;; R2 + A3 * sine			; 4-8			n next 2
next	vmovapd	y6, [srcreg+(iter+1)*srcinc+2*d1] ;; R3

prev	ystore	[srcreg+(iter-1)*srcinc], y12	;; Save R1				; 7
this	yfmaddpd y12, y9, y10, y7		;; I2 + B3 * sine			; 5-9			n next 6
this	yfnmaddpd y9, y9, y10, y7		;; I2 - B3 * sine			; 5-9			n next 9
next	vmovapd	y10, [srcreg+(iter+1)*srcinc+2*d1+32] ;; I3

prev	yfnmaddpd y7, ymm15, y4, y2		;; X - 0.866 * (I2 + I3) (final R6)	; 6-10
prev	yfmaddpd y4, ymm15, y4, y2		;; X + 0.866 * (I2 + I3) (final R2)	; 6-10
this	L1prefetchw srcreg+iter*srcinc+L1pd, L1pt

next	vmovapd	y2, [screg2+(iter+1)*scinc+32]	;; cosine/sine for R3/I3
prev	ystore	[srcreg+(iter-1)*srcinc+2*d1+32], y7 ;; Save I3				; 11
next	yfmaddpd y7, y6, y2, y10		;; A3 = R3 * cosine/sine + I3		; 7-11			n next 4
next	yfmsubpd y10, y10, y2, y6		;; B3 = I3 * cosine/sine - R3		; 7-11			n next 5

next	vmovapd	y2, [screg1+(iter+1)*scinc]	;; sine
next	vmulpd	y11, y11, y2			;; A2 = A2 * sine (new R2)		; 8-12			n next 4
next	vmulpd	y1, y1, y2			;; B2 = B2 * sine (new I2)		; 8-12			n next 5
this	vmovapd	y2, [srcreg+iter*srcinc+32]	;; R1#2
this next yloop_unrolled_one

prev	yfnmaddpd y6, ymm15, y5, y3		;; A - 0.866 * (I2 - I3) (final R5)	; 9-13
prev	yfmaddpd y5, ymm15, y5, y3		;; A + 0.866 * (I2 - I3) (final R3)	; 9-13
this	vmovapd	y3, [srcreg+iter*srcinc]	;; R1#1
prev	ystore	[srcreg+(iter-1)*srcinc+d1], y4 ;; Save R2				; 11+1
prev	ystore	[srcreg+(iter-1)*srcinc+d1+32], y6 ;; Save I2				; 14
prev	ystore	[srcreg+(iter-1)*srcinc+2*d1], y5 ;; Save R3				; 14+1

;; Shuffle register assignments so that this R1#1,R1#2,R3-R2.R2+R3,I2+I3,I2-I3 are in y0-5 and next R2,I2,A3,B3 are in y6-9.

ytmp	TEXTEQU	y0
y0	TEXTEQU	y3
y3	TEXTEQU	y8
y8	TEXTEQU	y7
y7	TEXTEQU	y1
y1	TEXTEQU	y2
y2	TEXTEQU	ytmp
ytmp	TEXTEQU	y4
y4	TEXTEQU	y12
y12	TEXTEQU	y5
y5	TEXTEQU	y9
y9	TEXTEQU	y10
y10	TEXTEQU	y6
y6	TEXTEQU	y11
y11	TEXTEQU	ytmp

	ENDM

ENDIF

ENDIF


;;
;; ************************************* six-reals-three-complex-fft variants ******************************************
;;

;; Macro to do one six_reals_fft and three three_complex_djbfft.
;; The six-reals operation is done in the lower double of the YMM
;; register.  The three-complex is done in the high doubles of the
;; YMM register.   This is REALLY funky, as we do both at the same
;; time within the full ymm register whenever possible.

yr3_3cl_six_reals_three_complex_djbfft_preload MACRO
	yr3_o6r_t3c_djbfft_mem_preload
	ENDM

yr3_3cl_six_reals_three_complex_djbfft MACRO srcreg,srcinc,d1,screg,scinc,maxrpt
	yr3_o6r_t3c_djbfft_mem [srcreg],[srcreg+32],[srcreg+d1],[srcreg+d1+32],[srcreg+2*d1],[srcreg+2*d1+32],screg
	ystore	[srcreg], ymm0			;; Save R1
	ystore	[srcreg+32], ymm5		;; Save I1
	ystore	[srcreg+d1], ymm3		;; Save R2
	ystore	[srcreg+d1+32], ymm7		;; Save I2
	ystore	[srcreg+2*d1], ymm6		;; Save R3
	ystore	[srcreg+2*d1+32], ymm4		;; Save I3
	bump	srcreg, srcinc
	bump	screg, scinc
	ENDM

yr3_o6r_t3c_djbfft_mem_preload MACRO
	ENDM

yr3_o6r_t3c_djbfft_mem MACRO mem1,mem2,mem3,mem4,mem5,mem6,screg
					;; Six-reals comments		;; Three complex comments
	vmovapd	ymm0, mem3		;; R2				;; R2
	vmovapd	ymm1, mem4		;; R5				;; I2
	vblendpd ymm4, ymm1, ymm0, 1	;; R2				;; I2
	vmovapd	ymm3, mem6		;; R6				;; I3
	vaddpd	ymm5, ymm4, ymm3	;; R2 + R6			;; I2 + I3
	vsubpd	ymm4, ymm4, ymm3	;; R2 - R6			;; I2 - I3
	vmovapd	ymm2, mem5		;; R3				;; R3
	vblendpd ymm6, ymm0, ymm2, 1	;; R3				;; R2
	vblendpd ymm7, ymm2, ymm1, 1	;; R5				;; R3
	vaddpd	ymm0, ymm6, ymm7	;; R3 + R5			;; R2 + R3
	vsubpd	ymm6, ymm6, ymm7	;; R3 - R5			;; R2 - R3

	vmovapd	ymm7, YMM_HALF
	vmulpd	ymm1, ymm7, ymm5	;; 0.5 * (R2 + R6)		;; 0.5 * (I2 + I3)
	vmovapd	ymm2, mem2		;; R4				;; I1
	vaddpd	ymm5, ymm2, ymm5	;; final R1 #2 = R4 + R2 + R6	;; I1 + I2 + I3 (final I1)
	vmulpd	ymm7, ymm7, ymm0	;; 0.5 * (R3 + R5)		;; 0.5 * (R2 + R3)
	vmovapd	ymm3, YMM_P866
	vmulpd	ymm4, ymm3, ymm4	;; X = 0.866 * (R2 - R6)	;; B = 0.866 * (I2 - I3)
	vmulpd	ymm6, ymm3, ymm6	;; Y = 0.866 * (R3 - R5)	;; Y = 0.866 * (R2 - R3)
	vmovapd	ymm3, mem1		;; R1				;; R1
	vaddpd	ymm0, ymm3, ymm0	;; final R1 #1 = R1 + R3 + R5	;; R1 + R2 + R3 (final R1)
	vsubpd	ymm2, ymm2, ymm1	;; B = R4 - 0.5 * (R2 + R6)	;; X = (I1-.5I2-.5I3)
	vsubpd	ymm3, ymm3, ymm7	;; A = R1 - 0.5 * (R3 + R5)	;; A = (R1-.5R2-.5R3)

	vblendpd ymm1, ymm2, ymm4, 1	;; X				;; X
	vsubpd	ymm7, ymm1, ymm6	;; X - Y (final I3)		;; X - Y (final I3)
	vaddpd	ymm1, ymm1, ymm6	;; X + Y (final I2)		;; X + Y (final I2)
	vblendpd ymm4, ymm4, ymm2, 1	;; B				;; B
	vaddpd	ymm6, ymm3, ymm4	;; A + B (final R3)		;; A + B (final R3)
	vsubpd	ymm3, ymm3, ymm4	;; A - B (final R2)		;; A - B (final R2)

	vmovapd	ymm2, [screg+64+32]	;; cosine/sine
	vmulpd	ymm4, ymm7, ymm2	;; B3 = I3 * cosine/sine
	vaddpd	ymm4, ymm4, ymm6	;; B3 = B3 + R3
	vmulpd	ymm6, ymm6, ymm2	;; A3 = R3 * cosine/sine
	vsubpd	ymm6, ymm6, ymm7	;; A3 = A3 - I3

	vmovapd	ymm2, [screg+32]	;; cosine/sine
	vmulpd	ymm7, ymm1, ymm2	;; B2 = I2 * cosine/sine
	vaddpd	ymm7, ymm7, ymm3	;; B2 = B2 + R2
	vmulpd	ymm3, ymm3, ymm2	;; A2 = R2 * cosine/sine
	vsubpd	ymm3, ymm3, ymm1	;; A2 = A2 - I2

	vmovapd	ymm1, [screg+64]	;; sine
	vmulpd	ymm4, ymm4, ymm1	;; B3 = B3 * sine (new I3)
	vmulpd	ymm6, ymm6, ymm1	;; A3 = A3 * sine (new R3)
	vmovapd	ymm1, [screg]		;; sine
	vmulpd	ymm7, ymm7, ymm1	;; B2 = B2 * sine (new I2)
	vmulpd	ymm3, ymm3, ymm1	;; A2 = A2 * sine (new R2)
	ENDM

;; 64-bit version

IFDEF X86_64

yr3_o6r_t3c_djbfft_mem_preload MACRO
	vmovapd	ymm14, YMM_HALF
	vmovapd	ymm13, YMM_P866
	ENDM

yr3_o6r_t3c_djbfft_mem MACRO mem1,mem2,mem3,mem4,mem5,mem6,screg
					;; Six-reals comments		;; Three complex comments
	vmovapd	ymm0, mem3		;; R2				;; R2
	vmovapd	ymm1, mem4		;; R5				;; I2
	vblendpd ymm4, ymm1, ymm0, 1	;; R2				;; I2
	vmovapd	ymm3, mem6		;; R6				;; I3
	vaddpd	ymm5, ymm4, ymm3	;; R2 + R6			;; I2 + I3
	vsubpd	ymm4, ymm4, ymm3	;; R2 - R6			;; I2 - I3
	vmovapd	ymm2, mem5		;; R3				;; R3
	vblendpd ymm6, ymm0, ymm2, 1	;; R3				;; R2
	vblendpd ymm7, ymm2, ymm1, 1	;; R5				;; R3
	vaddpd	ymm0, ymm6, ymm7	;; R3 + R5			;; R2 + R3
	vsubpd	ymm6, ymm6, ymm7	;; R3 - R5			;; R2 - R3

	vmulpd	ymm1, ymm14, ymm5	;; 0.5 * (R2 + R6)		;; 0.5 * (I2 + I3)
	vmovapd	ymm2, mem2		;; R4				;; I1
	vaddpd	ymm5, ymm2, ymm5	;; final R1 #2 = R4 + R2 + R6	;; I1 + I2 + I3 (final I1)
	vmulpd	ymm7, ymm14, ymm0	;; 0.5 * (R3 + R5)		;; 0.5 * (R2 + R3)
	vmulpd	ymm4, ymm13, ymm4	;; X = 0.866 * (R2 - R6)	;; B = 0.866 * (I2 - I3)
	vmulpd	ymm6, ymm13, ymm6	;; Y = 0.866 * (R3 - R5)	;; Y = 0.866 * (R2 - R3)
	vmovapd	ymm3, mem1		;; R1				;; R1
	vaddpd	ymm0, ymm3, ymm0	;; final R1 #1 = R1 + R3 + R5	;; R1 + R2 + R3 (final R1)
	vsubpd	ymm2, ymm2, ymm1	;; B = R4 - 0.5 * (R2 + R6)	;; X = (I1-.5I2-.5I3)
	vsubpd	ymm3, ymm3, ymm7	;; A = R1 - 0.5 * (R3 + R5)	;; A = (R1-.5R2-.5R3)

	vblendpd ymm1, ymm2, ymm4, 1	;; X				;; X
	vsubpd	ymm7, ymm1, ymm6	;; X - Y (final I3)		;; X - Y (final I3)
	vaddpd	ymm1, ymm1, ymm6	;; X + Y (final I2)		;; X + Y (final I2)
	vblendpd ymm4, ymm4, ymm2, 1	;; B				;; B
	vaddpd	ymm6, ymm3, ymm4	;; A + B (final R3)		;; A + B (final R3)
	vsubpd	ymm3, ymm3, ymm4	;; A - B (final R2)		;; A - B (final R2)

	vmovapd	ymm2, [screg+64+32]	;; cosine/sine
	vmulpd	ymm4, ymm7, ymm2	;; B3 = I3 * cosine/sine
	vaddpd	ymm4, ymm4, ymm6	;; B3 = B3 + R3
	vmulpd	ymm6, ymm6, ymm2	;; A3 = R3 * cosine/sine
	vsubpd	ymm6, ymm6, ymm7	;; A3 = A3 - I3

	vmovapd	ymm2, [screg+32]	;; cosine/sine
	vmulpd	ymm7, ymm1, ymm2	;; B2 = I2 * cosine/sine
	vaddpd	ymm7, ymm7, ymm3	;; B2 = B2 + R2
	vmulpd	ymm3, ymm3, ymm2	;; A2 = R2 * cosine/sine
	vsubpd	ymm3, ymm3, ymm1	;; A2 = A2 - I2

	vmovapd	ymm1, [screg+64]	;; sine
	vmulpd	ymm4, ymm4, ymm1	;; B3 = B3 * sine (new I3)
	vmulpd	ymm6, ymm6, ymm1	;; A3 = A3 * sine (new R3)
	vmovapd	ymm1, [screg]		;; sine
	vmulpd	ymm7, ymm7, ymm1	;; B2 = B2 * sine (new I2)
	vmulpd	ymm3, ymm3, ymm1	;; A2 = A2 * sine (new R2)
	ENDM

;; Haswell FMA3 version

IF (@INSTR(,%yarch,<FMA3>) NE 0)

yr3_o6r_t3c_djbfft_mem_preload MACRO
	vmovapd	ymm15, YMM_ONE
	vmovapd	ymm14, YMM_HALF
	vmovapd	ymm13, YMM_P866
	ENDM

yr3_o6r_t3c_djbfft_mem MACRO mem1,mem2,mem3,mem4,mem5,mem6,screg
					;; Six-reals comments		;; Three complex comments
	vmovapd	ymm0, mem3		;; R2				;; R2
	vmovapd	ymm1, mem4		;; R5				;; I2
	vblendpd ymm4, ymm1, ymm0, 1	;; R2				;; I2
	vmovapd	ymm3, mem6		;; R6				;; I3
	yfmaddpd ymm8, ymm4, ymm15, ymm3 ;; R2 + R6			;; I2 + I3
	yfmsubpd ymm4, ymm4, ymm15, ymm3 ;; R2 - R6			;; I2 - I3
	vmovapd	ymm2, mem5		;; R3				;; R3
	vblendpd ymm6, ymm0, ymm2, 1	;; R3				;; R2
	vblendpd ymm7, ymm2, ymm1, 1	;; R5				;; R3
	yfmaddpd ymm9, ymm6, ymm15, ymm7 ;; R3 + R5			;; R2 + R3
	yfmsubpd ymm6, ymm6, ymm15, ymm7 ;; R3 - R5			;; R2 - R3

	vmovapd	ymm2, mem2		;; R4				;; I1
	yfmaddpd ymm5, ymm2, ymm15, ymm8 ;; final R1 #2 = R4 + R2 + R6	;; I1 + I2 + I3 (final I1)
	vmulpd	ymm4, ymm13, ymm4	;; X = 0.866 * (R2 - R6)	;; B = 0.866 * (I2 - I3)
	vmulpd	ymm6, ymm13, ymm6	;; Y = 0.866 * (R3 - R5)	;; Y = 0.866 * (R2 - R3)
	vmovapd	ymm3, mem1		;; R1				;; R1
	yfmaddpd ymm0, ymm3, ymm15, ymm9 ;; final R1 #1 = R1 + R3 + R5	;; R1 + R2 + R3 (final R1)
	yfnmaddpd ymm2, ymm14, ymm8, ymm2 ;; B = R4 - 0.5 * (R2 + R6)	;; X = (I1-.5I2-.5I3)
	yfnmaddpd ymm3, ymm14, ymm9, ymm3 ;; A = R1 - 0.5 * (R3 + R5)	;; A = (R1-.5R2-.5R3)

	vblendpd ymm1, ymm2, ymm4, 1	;; X				;; X
	yfmsubpd ymm7, ymm1, ymm15, ymm6 ;; X - Y (final I3)		;; X - Y (final I3)
	yfmaddpd ymm1, ymm1, ymm15, ymm6 ;; X + Y (final I2)		;; X + Y (final I2)
	vblendpd ymm4, ymm4, ymm2, 1	;; B				;; B
	yfmaddpd ymm6, ymm3, ymm15, ymm4 ;; A + B (final R3)		;; A + B (final R3)
	yfmsubpd ymm3, ymm3, ymm15, ymm4 ;; A - B (final R2)		;; A - B (final R2)

	vmovapd	ymm2, [screg+64+32]	;; cosine/sine
	yfmaddpd ymm4, ymm7, ymm2, ymm6	;; B3 = I3 * cosine/sine + R3
	yfmsubpd ymm6, ymm6, ymm2, ymm7	;; A3 = R3 * cosine/sine - I3

	vmovapd	ymm2, [screg+32]	;; cosine/sine
	yfmaddpd ymm7, ymm1, ymm2, ymm3	;; B2 = I2 * cosine/sine + R2
	yfmsubpd ymm3, ymm3, ymm2, ymm1	;; A2 = R2 * cosine/sine - I2

	vmovapd	ymm1, [screg+64]	;; sine
	vmulpd	ymm4, ymm4, ymm1	;; B3 = B3 * sine (new I3)
	vmulpd	ymm6, ymm6, ymm1	;; A3 = A3 * sine (new R3)
	vmovapd	ymm1, [screg]		;; sine
	vmulpd	ymm7, ymm7, ymm1	;; B2 = B2 * sine (new I2)
	vmulpd	ymm3, ymm3, ymm1	;; A2 = A2 * sine (new R2)
	ENDM

ENDIF

ENDIF

;;
;; ************************************* six-reals-three-complex-unfft variants ******************************************
;;

;; Macro to do one six_reals_unfft and three three_complex_djbunfft.
;; The six-reals operation is done in the lower double of the YMM
;; register.  The three-complex is done in the high doubles of the
;; YMM register.   This is REALLY funky, as we do both at the same
;; time within the full ymm register whenever possible.

yr3_3cl_six_reals_three_complex_djbunfft_preload MACRO
	yr3_o6r_t3c_djbunfft_mem_preload
	ENDM

yr3_3cl_six_reals_three_complex_djbunfft MACRO srcreg,srcinc,d1,screg,scinc,maxrpt
	yr3_o6r_t3c_djbunfft_mem [srcreg],[srcreg+32],[srcreg+d1],[srcreg+d1+32],[srcreg+2*d1],[srcreg+2*d1+32],screg
	ystore	[srcreg], ymm1		;; Save R1
	ystore	[srcreg+32], ymm2	;; Save I1
	ystore	[srcreg+d1], ymm4	;; Save R2
	ystore	[srcreg+d1+32], ymm5	;; Save I2
	ystore	[srcreg+2*d1], ymm3	;; Save R3
	ystore	[srcreg+2*d1+32], ymm6	;; Save I3
	bump	srcreg, srcinc
	bump	screg, scinc
	ENDM

yr3_o6r_t3c_djbunfft_mem_preload MACRO
	ENDM

yr3_o6r_t3c_djbunfft_mem MACRO mem1,mem2,mem3,mem4,mem5,mem6,screg
					;; Six-reals comments		;; Three complex comments
	vmovapd	ymm1, mem3				;; R2
	vmovapd	ymm7, [screg+32]			;; cosine/sine
	vmulpd	ymm2, ymm1, ymm7			;; A2 = R2 * cosine/sine
	vmovapd	ymm3, mem5				;; R3
	vmovapd	ymm6, [screg+64+32]			;; cosine/sine
	vmulpd	ymm4, ymm3, ymm6			;; A3 = R3 * cosine/sine
	vmovapd	ymm0, mem4				;; I2
	vaddpd	ymm2, ymm2, ymm0			;; A2 = A2 + I2
	vmovapd	ymm5, mem6				;; I3
	vaddpd	ymm4, ymm4, ymm5			;; A3 = A3 + I3
	vmulpd	ymm0, ymm0, ymm7			;; B2 = I2 * cosine/sine
	vmulpd	ymm5, ymm5, ymm6			;; B3 = I3 * cosine/sine
	vsubpd	ymm0, ymm0, ymm1			;; B2 = B2 - R2
	vsubpd	ymm5, ymm5, ymm3			;; B3 = B3 - R3
	vmovapd	ymm7, [screg]				;; sine
	vmulpd	ymm2, ymm2, ymm7			;; A2 = A2 * sine (final R2)
	vmovapd	ymm6, [screg+64]			;; sine
	vmulpd	ymm4, ymm4, ymm6			;; A3 = A3 * sine (final R3)
	vmulpd	ymm0, ymm0, ymm7			;; B2 = B2 * sine (final I2)
	vmulpd	ymm5, ymm5, ymm6			;; B3 = B3 * sine (final I3)

	vblendpd ymm1, ymm2, ymm4, 1	;; R3				; R2
	vblendpd ymm4, ymm4, ymm2, 1	;; R2				; R3
	vsubpd	ymm2, ymm1, ymm4	;; R3 - R2			; R2 - R3
	vaddpd	ymm1, ymm1, ymm4	;; R2 + R3			; R2 + R3
	vsubpd	ymm3, ymm0, ymm5	;; I2 - I3			; I2 - I3
	vaddpd	ymm0, ymm0, ymm5	;; I2 + I3			; I2 + I3

	vmovapd	ymm7, YMM_P866
	vmulpd	ymm3, ymm7, ymm3	;; B = 0.866 * (I2 - I3)	; B = 0.866 * (I2 - I3)
	vblendpd ymm4, ymm2, ymm0, 1	;; I2 + I3			; R2 - R3
	vmulpd	ymm4, ymm7, ymm4	;; Y = 0.866 * (I2 + I3)	; Y = 0.866 * (R2 - R3)

	vmovapd	ymm7, YMM_HALF
	vmulpd	ymm5, ymm7, ymm1	;; 0.5 * (R2 + R3)		; 0.5 * (R2 + R3)
	vblendpd ymm6, ymm0, ymm2, 1	;; R3 - R2			; I2 + I3
	vmulpd	ymm0, ymm7, ymm6	;; 0.5 * (R3 - R2)		; 0.5 * (I2 + I3)

	vmovapd	ymm7, mem1		;; R1#1				; R1
	vaddpd	ymm1, ymm7, ymm1	;; R1#1 + R2 + R3 (final R1)	; R1 + R2 + R3 (final R1)
	vsubpd	ymm7, ymm7, ymm5	;; A = R1#1 - 0.5 * (R2 + R3)	; A = (R1-.5R2-.5R3)

	vmovapd	ymm5, mem2		;; R1#2				; I1
	vaddpd	ymm2, ymm5, ymm6	;; R1#2 + (R3 - R2) (final R4)	; I1 + I2 + I3 (final I1)
	vsubpd	ymm5, ymm5, ymm0	;; X = R1#2 - 0.5 * (R3 - R2)	; X = (I1-.5I2-.5I3)
	vsubpd	ymm0, ymm7, ymm3	;; A - B (final R5)		; A - B (final R3)
	vaddpd	ymm7, ymm7, ymm3	;; A + B (final R3)		; A + B (final R2)
	vsubpd	ymm3, ymm5, ymm4	;; X - Y (final R6)		; X - Y (final I2)
	vaddpd	ymm5, ymm5, ymm4	;; X + Y (final R2)		; X + Y (final I3)

	vblendpd ymm4, ymm7, ymm5, 1	;; final R2			; final R2
	vblendpd ymm6, ymm5, ymm3, 1	;; final R6			; final I3
	vblendpd ymm5, ymm3, ymm0, 1	;; final R5			; final I2
	vblendpd ymm3, ymm0, ymm7, 1	;; final R3			; final R3
	ENDM


;; 64-bit version

IFDEF X86_64

yr3_o6r_t3c_djbunfft_mem_preload MACRO
	vmovapd	ymm14, YMM_HALF
	vmovapd	ymm13, YMM_P866
	ENDM

yr3_o6r_t3c_djbunfft_mem MACRO mem1,mem2,mem3,mem4,mem5,mem6,screg
					;; Six-reals comments		;; Three complex comments
	vmovapd	ymm1, mem3				;; R2
	vmovapd	ymm7, [screg+32]			;; cosine/sine
	vmulpd	ymm2, ymm1, ymm7			;; A2 = R2 * cosine/sine
	vmovapd	ymm3, mem5				;; R3
	vmovapd	ymm6, [screg+64+32]			;; cosine/sine
	vmulpd	ymm4, ymm3, ymm6			;; A3 = R3 * cosine/sine
	vmovapd	ymm0, mem4				;; I2
	vaddpd	ymm2, ymm2, ymm0			;; A2 = A2 + I2
	vmovapd	ymm5, mem6				;; I3
	vaddpd	ymm4, ymm4, ymm5			;; A3 = A3 + I3
	vmulpd	ymm0, ymm0, ymm7			;; B2 = I2 * cosine/sine
	vmulpd	ymm5, ymm5, ymm6			;; B3 = I3 * cosine/sine
	vsubpd	ymm0, ymm0, ymm1			;; B2 = B2 - R2
	vsubpd	ymm5, ymm5, ymm3			;; B3 = B3 - R3
	vmovapd	ymm7, [screg]				;; sine
	vmulpd	ymm2, ymm2, ymm7			;; A2 = A2 * sine (final R2)
	vmovapd	ymm6, [screg+64]			;; sine
	vmulpd	ymm4, ymm4, ymm6			;; A3 = A3 * sine (final R3)
	vmulpd	ymm0, ymm0, ymm7			;; B2 = B2 * sine (final I2)
	vmulpd	ymm5, ymm5, ymm6			;; B3 = B3 * sine (final I3)

	vblendpd ymm1, ymm2, ymm4, 1	;; R3				; R2
	vblendpd ymm4, ymm4, ymm2, 1	;; R2				; R3
	vsubpd	ymm2, ymm1, ymm4	;; R3 - R2			; R2 - R3
	vaddpd	ymm1, ymm1, ymm4	;; R2 + R3			; R2 + R3
	vsubpd	ymm3, ymm0, ymm5	;; I2 - I3			; I2 - I3
	vaddpd	ymm0, ymm0, ymm5	;; I2 + I3			; I2 + I3

	vmulpd	ymm3, ymm13, ymm3	;; B = 0.866 * (I2 - I3)	; B = 0.866 * (I2 - I3)
	vblendpd ymm4, ymm2, ymm0, 1	;; I2 + I3			; R2 - R3
	vmulpd	ymm4, ymm13, ymm4	;; Y = 0.866 * (I2 + I3)	; Y = 0.866 * (R2 - R3)

	vmulpd	ymm5, ymm14, ymm1	;; 0.5 * (R2 + R3)		; 0.5 * (R2 + R3)
	vblendpd ymm6, ymm0, ymm2, 1	;; R3 - R2			; I2 + I3
	vmulpd	ymm0, ymm14, ymm6	;; 0.5 * (R3 - R2)		; 0.5 * (I2 + I3)

	vmovapd	ymm7, mem1		;; R1#1				; R1
	vaddpd	ymm1, ymm7, ymm1	;; R1#1 + R2 + R3 (final R1)	; R1 + R2 + R3 (final R1)
	vsubpd	ymm7, ymm7, ymm5	;; A = R1#1 - 0.5 * (R2 + R3)	; A = (R1-.5R2-.5R3)

	vmovapd	ymm5, mem2		;; R1#2				; I1
	vaddpd	ymm2, ymm5, ymm6	;; R1#2 + (R3 - R2) (final R4)	; I1 + I2 + I3 (final I1)
	vsubpd	ymm5, ymm5, ymm0	;; X = R1#2 - 0.5 * (R3 - R2)	; X = (I1-.5I2-.5I3)
	vsubpd	ymm0, ymm7, ymm3	;; A - B (final R5)		; A - B (final R3)
	vaddpd	ymm7, ymm7, ymm3	;; A + B (final R3)		; A + B (final R2)
	vsubpd	ymm3, ymm5, ymm4	;; X - Y (final R6)		; X - Y (final I2)
	vaddpd	ymm5, ymm5, ymm4	;; X + Y (final R2)		; X + Y (final I3)

	vblendpd ymm4, ymm7, ymm5, 1	;; final R2			; final R2
	vblendpd ymm6, ymm5, ymm3, 1	;; final R6			; final I3
	vblendpd ymm5, ymm3, ymm0, 1	;; final R5			; final I2
	vblendpd ymm3, ymm0, ymm7, 1	;; final R3			; final R3
	ENDM

;; Haswell FMA3 version

IF (@INSTR(,%yarch,<FMA3>) NE 0)

yr3_o6r_t3c_djbunfft_mem_preload MACRO
	vmovapd	ymm15, YMM_ONE
	vmovapd	ymm14, YMM_HALF
	vmovapd	ymm13, YMM_P866
	ENDM

yr3_o6r_t3c_djbunfft_mem MACRO mem1,mem2,mem3,mem4,mem5,mem6,screg
					;; Six-reals comments		;; Three complex comments
	vmovapd	ymm1, mem3				;; R2
	vmovapd	ymm0, mem4				;; I2
	vmovapd	ymm7, [screg+32]			;; cosine/sine
	yfmaddpd ymm2, ymm1, ymm7, ymm0			;; A2 = R2 * cosine/sine + I2
	yfmsubpd ymm0, ymm0, ymm7, ymm1			;; B2 = I2 * cosine/sine - R2
	vmovapd	ymm3, mem5				;; R3
	vmovapd	ymm5, mem6				;; I3
	vmovapd	ymm6, [screg+64+32]			;; cosine/sine
	yfmaddpd ymm4, ymm3, ymm6, ymm5			;; A3 = R3 * cosine/sine + I3
	yfmsubpd ymm5, ymm5, ymm6, ymm3			;; B3 = I3 * cosine/sine - R3
	vmovapd	ymm7, [screg]				;; sine
	vmulpd	ymm2, ymm2, ymm7			;; A2 = A2 * sine (final R2)
	vmovapd	ymm6, [screg+64]			;; sine
	vmulpd	ymm4, ymm4, ymm6			;; A3 = A3 * sine (final R3)
	vmulpd	ymm0, ymm0, ymm7			;; B2 = B2 * sine (final I2)
	vmulpd	ymm5, ymm5, ymm6			;; B3 = B3 * sine (final I3)

	vblendpd ymm1, ymm2, ymm4, 1	;; R3				; R2
	vblendpd ymm4, ymm4, ymm2, 1	;; R2				; R3
	yfmsubpd ymm2, ymm1, ymm15, ymm4 ;; R3 - R2			; R2 - R3
	yfmaddpd ymm1, ymm1, ymm15, ymm4 ;; R2 + R3			; R2 + R3
	yfmsubpd ymm3, ymm0, ymm15, ymm5 ;; I2 - I3			; I2 - I3
	yfmaddpd ymm0, ymm0, ymm15, ymm5 ;; I2 + I3			; I2 + I3

	vblendpd ymm4, ymm2, ymm0, 1	;; I2 + I3			; R2 - R3
	vblendpd ymm6, ymm0, ymm2, 1	;; R3 - R2			; I2 + I3

	vmovapd	ymm5, mem1		;; R1#1				; R1
	yfnmaddpd ymm7, ymm14, ymm1, ymm5 ;; A = R1#1 - 0.5 * (R2 + R3)	; A = (R1-.5R2-.5R3)
	yfmaddpd ymm1, ymm5, ymm15, ymm1 ;; R1#1 + R2 + R3 (final R1)	; R1 + R2 + R3 (final R1)

	vmovapd	ymm5, mem2		;; R1#2				; I1
	yfmaddpd ymm2, ymm5, ymm15, ymm6 ;; R1#2 + (R3 - R2) (final R4)	; I1 + I2 + I3 (final I1)
	yfnmaddpd ymm5, ymm14, ymm6, ymm5 ;; X = R1#2 - 0.5 * (R3 - R2)	; X = (I1-.5I2-.5I3)
	yfnmaddpd ymm0, ymm13, ymm3, ymm7 ;; A - 0.866 * (I2 - I3) (final R5) ; A - 0.866 * (I2 - I3) (final R3)
	yfmaddpd ymm7, ymm13, ymm3, ymm7 ;; A + 0.866 * (I2 - I3) (final R3) ; A + 0.866 * (I2 - I3) (final R2)
	yfnmaddpd ymm3, ymm13, ymm4, ymm5 ;; X - 0.866 * (I2 + I3) (final R6) ; X - 0.866 * (R2 - R3) (final I2)
	yfmaddpd ymm5, ymm13, ymm4, ymm5 ;; X + 0.866 * (I2 + I3) (final R2) ; X + 0.866 * (R2 - R3) (final I3)

	vblendpd ymm4, ymm7, ymm5, 1	;; final R2			; final R2
	vblendpd ymm6, ymm5, ymm3, 1	;; final R6			; final I3
	vblendpd ymm5, ymm3, ymm0, 1	;; final R5			; final I2
	vblendpd ymm3, ymm0, ymm7, 1	;; final R3			; final R3
	ENDM

ENDIF

ENDIF

;;
;; Macros for a radix-6 first step in an AVX real FFT.
;;

;;
;; ************************************* 24-reals-first-fft variants ******************************************
;;

;; These macros operate on 24 reals doing 4.585 levels of the FFT, applying
;; the sin/cos multipliers afterwards.  The output is 2 reals and 11 complex numbers.

;; To calculate a 24-reals FFT, we calculate 24 complex values in a brute force way (using a shorthand notation):
;; r1 + r2 + ... + r24	*  w^0000000000...
;; r1 + r2 + ... + r24	*  w^0123456789A...
;; r1 + r2 + ... + r24	*  w^02468ACE....
;;    ...
;; r1 + r2 + ... + r24	*  w^...A987654321
;; Note that Hermetian symmetry means we won't need to calculate the last 12 complex values.
;;
;; The sin/cos values (w = 24th root of unity) are:
;; w^1 = .966 + .259i
;; w^2 = .866 + .5i
;; w^3 = .707 + .707i
;; w^4 = .5   + .866i
;; w^5 = .259 + .966i
;; w^6 = 0 + 1i
;; w^7 = -.259 + .966i
;; w^8 = -.5   + .866i
;; w^9 = -.707 + .707i
;; w^10 = -.866 + .5i
;; w^11 = -.966 + .259i
;; w^12 = -1
;;
;; Applying the sin/cos values above (and noting that combining r2 and r24, r3 and r23, etc. will simplify calculations):
;; reals:
;; r1     +(r2+r24)     +(r3+r23)     +(r4+r22)     +(r5+r21)     +(r6+r20) + (r7+r19)     +(r8+r18)     +(r9+r17)     +(r10+r16)     +(r11+r15)     +(r12+r14) + r13
;; r1 +.966(r2+r24) +.866(r3+r23) +.707(r4+r22) +.500(r5+r21) +.259(r6+r20)            -.259(r8+r18) -.500(r9+r17) -.707(r10+r16) -.866(r11+r15) -.966(r12+r14) - r13
;; r1 +.866(r2+r24) +.500(r3+r23)               -.500(r5+r21) -.866(r6+r20) - (r7+r19) -.866(r8+r18) -.500(r9+r17)                +.500(r11+r15) +.866(r12+r14) + r13
;; r1 +.707(r2+r24)               -.707(r4+r22)     -(r5+r21) -.707(r6+r20)            +.707(r8+r18)     +(r9+r17) +.707(r10+r16)                -.707(r12+r14) - r13
;; r1 +.500(r2+r24) -.500(r3+r23)     -(r4+r22) -.500(r5+r21) +.500(r6+r20) + (r7+r19) +.500(r8+r18) -.500(r9+r17)     -(r10+r16) -.500(r11+r15) +.500(r12+r14) + r13
;; r1 +.259(r2+r24) -.866(r3+r23) -.707(r4+r22) +.500(r5+r21) +.966(r6+r20)            -.966(r8+r18) -.500(r9+r17) +.707(r10+r16) +.866(r11+r15) -.259(r12+r14) - r13
;; r1                   -(r3+r23)                   +(r5+r21)               - (r7+r19)                   +(r9+r17)                    -(r11+r15)                + r13
;; r1 -.259(r2+r24) -.866(r3+r23) +.707(r4+r22) +.500(r5+r21) -.966(r6+r20)            +.966(r8+r18) -.500(r9+r17) -.707(r10+r16) +.866(r11+r15) +.259(r12+r14) - r13
;; r1 -.500(r2+r24) -.500(r3+r23)     +(r4+r22) -.500(r5+r21) -.500(r6+r20) + (r7+r19) -.500(r8+r18) -.500(r9+r17)     +(r10+r16) -.500(r11+r15) -.500(r12+r14) + r13
;; r1 -.707(r2+r24)               +.707(r4+r22)     -(r5+r21) +.707(r6+r20)            -.707(r8+r18)     +(r9+r17) -.707(r10+r16)                +.707(r12+r14) - r13
;; r1 -.866(r2+r24) +.500(r3+r23)               -.500(r5+r21) +.866(r6+r20) - (r7+r19) +.866(r8+r18) -.500(r9+r17)                +.500(r11+r15) -.866(r12+r14) + r13
;; r1 -.966(r2+r24) +.866(r3+r23) -.707(r4+r22) +.500(r5+r21) -.259(r6+r20)            +.259(r8+r18) -.500(r9+r17) +.707(r10+r16) -.866(r11+r15) +.966(r12+r14) - r13
;; r1     -(r2+r24)     +(r3+r23)     -(r4+r22)     +(r5+r21)     -(r6+r20) + (r7+r19)     -(r8+r18)     +(r9+r17)     -(r10+r16)     +(r11+r15)     -(r12+r14) + r13
;;
;; imaginarys:
;; 0
;; +.259(r2-r24) +.500(r3-r23) +.707(r4-r22) +.866(r5-r21) +.966(r6-r20) + (r7-r19) +.966(r8-r18) +.866(r9-r17) +.707(r10-r16) +.500(r11-r15) +.259(r12-r14)
;; +.500(r2-r24) +.866(r3-r23)     +(r4-r22) +.866(r5-r21) +.500(r6-r20)            -.500(r8-r18) -.866(r9-r17)     -(r10-r16) -.866(r11-r15) -.500(r12-r14)
;; +.707(r2-r24)     +(r3-r23) +.707(r4-r22)               -.707(r6-r20) - (r7-r19) -.707(r8-r18)               +.707(r10-r16)     +(r11-r15) +.707(r12-r14)
;; +.866(r2-r24) +.866(r3-r23)               -.866(r5-r21) -.866(r6-r20)            +.866(r8-r18) +.866(r9-r17)                -.866(r11-r15) -.866(r12-r14)
;; +.966(r2-r24) +.500(r3-r23) -.707(r4-r22) -.866(r5-r21) +.259(r6-r20) + (r7-r19) +.259(r8-r18) -.866(r9-r17) -.707(r10-r16) +.500(r11-r15) +.966(r12-r14)
;;      (r2-r24)                   -(r4-r22)                   +(r6-r20)                -(r8-r18)                   +(r10-r16)                    -(r12-r14)
;; +.966(r2-r24) -.500(r3-r23) -.707(r4-r22) +.866(r5-r21) +.259(r6-r20) - (r7-r19) +.259(r8-r18) +.866(r9-r17) -.707(r10-r16) -.500(r11-r15) +.966(r12-r14)
;; +.866(r2-r24) -.866(r3-r23)               +.866(r5-r21) -.866(r6-r20)            +.866(r8-r18) -.866(r9-r17)                +.866(r11-r15) -.866(r12-r14)
;; +.707(r2-r24)     -(r3-r23) +.707(r4-r22)               -.707(r6-r20) + (r7-r19) -.707(r8-r18)               +.707(r10-r16)     -(r11-r15) +.707(r12-r14)
;; +.500(r2-r24) -.866(r3-r23)     +(r4-r22) -.866(r5-r21) +.500(r6-r20)            -.500(r8-r18) +.866(r9-r17)     -(r10-r16) +.866(r11-r15) -.500(r12-r14)
;; +.259(r2-r24) -.500(r3-r23) +.707(r4-r22) -.866(r5-r21) +.966(r6-r20) - (r7-r19) +.966(r8-r18) -.866(r9-r17) +.707(r10-r16) -.500(r11-r15) +.259(r12-r14)
;; 0
;;
;; There are many more symmetries we can take advantage of.   For example, the (r2+/-r24) column
;; always has the same multiplier as the (r12+/-r14) column.  This is true for all the "even" columns.
;; Also the computations for the 2nd row are very similar to the computations for the 12th row,
;; the 3rd row are similar to the 11th, etc.  Finally, note that for the odd columns, there are
;; only three multipliers to apply and can be combined with every fourth column.
;;
;; Lastly, output would normally be 11 complex and 2 reals.  but the users of this routine
;; expect us to "back up" the 2 reals by one level.  That is:
;;	real #1A:  r1 + r3+r23 + r5+r21 + ...
;;	real #1B:  r2+r24 + r4+r22 + ...

;; Simplifying, we get:
;; r2/12  = r2o +/- .707*r2e	r3/11 = r3o +/- .866 r3e	r4/10 = r4o +/- .707*r4e	r5/9 = r5o +/- r5e	r6/8 = r6o +/- .707*r6e
;; r1a = r17a + r17b,  r7 = r17a - r17b
;;
;; r15eA =         +((r2+r24) + (r12+r14))                                  +((r6+r20) + (r8+r18))
;; r1b =  r15eA                            +((r4+r22) + (r10+r16))
;; r2e =  +.966/.707((r2+r24) - (r12+r14)) +((r4+r22) - (r10+r16)) +.259/.707((r6+r20) - (r8+r18))
;; r3e =           +((r2+r24) + (r12+r14))                                  -((r6+r20) + (r8+r18))
;; r4e =           +((r2+r24) - (r12+r14)) -((r4+r22) - (r10+r16))          -((r6+r20) - (r8+r18))                                        
;; r5e =  +.500*r15eA                      -((r4+r22) + (r10+r16))
;; r6e =  +.259/.707((r2+r24) - (r12+r14)) -((r4+r22) - (r10+r16)) +.966/.707((r6+r20) - (r8+r18))
;; r17a=   r1+r13     +((r5+r21) + (r9+r17)) 
;; r17b=                                         +((r3+r23) + (r11+r15)) + (r7+r19)
;; r26oA = r1-r13 +.500((r5+r21) - (r9+r17))
;; r26oB =                                   +.866((r3+r23) - (r11+r15))
;; r35oA = r1+r13 -.500((r5+r21) + (r9+r17))
;; r35oB =                                   +.500((r3+r23) + (r11+r15)) - (r7+r19)
;; r2o =   r26oA                             + r26oB
;; r3o =   r35oA                             + r35oB
;; r4o =   r1-r13     -((r5+r21) - (r9+r17)) 
;; r5o =   r35oA                             - r35oB
;; r6o =   r26oA                             - r26oB

;; i2/12 = .707*i2e +/- i2o		i3/11 = i3e +/- .866 i3o	i4/10 = .707*i4e +/- i4o
;; i5/9  = .866*i5e +/- .866*i5o	i6/8  = .707*i6e +/- i6o
;;
;; i26oA = +.5((r3-r23) + (r11-r15))                        + (r7-r19)
;; i26oB =                           +.866((r5-r21) + (r9-r17))
;; i2o = i26oA + io26B
;; i3o =     +((r3-r23) - (r11-r15))     +((r5-r21) - (r9-r17))
;; i4o =     +((r3-r23) + (r11-r15))                        - (r7-r19)
;; i5o =     +((r3-r23) - (r11-r15))     -((r5-r21) - (r9-r17))
;; i6o = i26oA - io26B
;; i37eA =         ((r2-r24) - (r12-r14))                                  +((r6-r20) - (r8-r18))
;; i2e = +.259/.707((r2-r24) + (r12-r14)) +((r4-r22) + (r10-r16)) +.966/.707((r6-r20) + (r8-r18))
;; i3e = .500*i37eA                       +((r4-r22) - (r10-r16))
;; i4e =          +((r2-r24) + (r12-r14)) +((r4-r22) + (r10-r16))          -((r6-r20) + (r8-r18))
;; i5e =          +((r2-r24) - (r12-r14))                                  -((r6-r20) - (r8-r18))
;; i6e = +.966/.707((r2-r24) + (r12-r14)) -((r4-r22) + (r10-r16)) +.259/.707((r6-r20) + (r8-r18))
;; i7 =  i37eA                            -((r4-r22) - (r10-r16))

yr6_12cl_24_reals_fft_preload MACRO
	ENDM

yr6_12cl_24_reals_fft MACRO srcreg,srcinc,d1,d2,screg,scinc,maxrpt,L1pt,L1pd

	;; Do the odd columns for the real results

	vmovapd	ymm0, [srcreg+2*d2]		;; r5
	vaddpd	ymm0, ymm0, [srcreg+4*d2+32]	;; r5+r21						; 1-3
	vmovapd	ymm1, [srcreg+4*d2]		;; r9
	vaddpd	ymm1, ymm1, [srcreg+2*d2+32]	;; r9+r17						; 2-4
	vmovapd	ymm3, [srcreg]			;; r1
	vmovapd	ymm4, [srcreg+32]		;; r13

	vaddpd	ymm2, ymm3, ymm4		;; r1+r13						; 3-5
	vsubpd	ymm3, ymm3, ymm4		;; r1-r13						; 4-6

	vsubpd	ymm4, ymm0, ymm1		;; (r5+r21)-(r9+r17)					; 5-7
	vaddpd	ymm0, ymm0, ymm1		;; (r5+r21)+(r9+r17)					; 6-8

	vmovapd	ymm1, [srcreg+d2]		;; r3
	vaddpd	ymm1, ymm1, [srcreg+5*d2+32]	;; r3+r23						; 7-9

	vsubpd	ymm5, ymm3, ymm4		;; r1-r13 - ((r5+r21)-(r9+r17)) (r4o)			; 8-10
	vmovapd ymm7, YMM_HALF
	vmulpd	ymm4, ymm7, ymm4		;; .5*((r5+r21)-(r9+r17))				; 8-12

	vaddpd	ymm6, ymm2, ymm0		;; r1+r13 + ((r5+r21)+(r9+r17)) (r17a)			; 9-11
	vmulpd	ymm0, ymm7, ymm0		;; .5*((r5+r21)+(r9+r17))				; 9-13

	vmovapd	ymm7, [srcreg+5*d2]		;; r11
	vaddpd	ymm7, ymm7, [srcreg+d2+32]	;; r11+r15						; 10-12

	ystore	YMM_TMPS[0*32], ymm5		;; Real odd-cols row #4
	vmovapd	ymm5, [srcreg+3*d2]		;; r7
	vaddpd	ymm5, ymm5, [srcreg+3*d2+32]	;; r7+r19						; 11-13

	vaddpd	ymm3, ymm3, ymm4		;; r1-r13 + .5*((r5+r21)-(r9+r17)) (r26oA)		; 13-15
	L1prefetchw srcreg+L1pd, L1pt

	vsubpd	ymm2, ymm2, ymm0		;; r1+r13 - .5*((r5+r21)+(r9+r17)) (r35oA)		; 14-16

	vaddpd	ymm4, ymm1, ymm7		;; (r3+r23)+(r11+r15)					; 15-17
	L1prefetchw srcreg+d1+L1pd, L1pt

	vsubpd	ymm1, ymm1, ymm7		;; (r3+r23)-(r11+r15)					; 16-18

	vaddpd	ymm0, ymm4, ymm5		;; ((r3+r23)+(r11+r15)) + r7+r19 (r17b)			; 18-20
	vmulpd	ymm4, ymm4, YMM_HALF		;; .5*((r3+r23)+(r11+r15))				; 18-22

	vmulpd	ymm1, ymm1, YMM_P866		;; .866*((r3+r23)-(r11+r15)) (r26oB)			; 19-23
	L1prefetchw srcreg+d2+L1pd, L1pt

	vaddpd	ymm7, ymm6, ymm0		;; r17a + r17b (r1a)					; 21-23

	vsubpd	ymm6, ymm6, ymm0		;; r17a - r17b (r7)					; 22-24
	vmovapd	ymm0, [srcreg+d1]		;; r2

	vsubpd	ymm4, ymm4, ymm5		;; .5*((r3+r23)+(r11+r15)) - (r7+r19) (r35oB)		; 23-25
	L1prefetchw srcreg+d2+d1+L1pd, L1pt

	vaddpd	ymm5, ymm3, ymm1		;; r26oA + r26oB (r2o)					; 24-26

	vsubpd	ymm3, ymm3, ymm1		;; r26oA - r26oB (r6o)					; 25-27
	L1prefetchw srcreg+2*d2+L1pd, L1pt

	vaddpd	ymm1, ymm2, ymm4		;; r35oA + r35oB (r3o)					; 26-28

	vsubpd	ymm2, ymm2, ymm4		;; r35oA - r35oB (r5o)					; 27-29
	L1prefetchw srcreg+2*d2+d1+L1pd, L1pt

	;; Do the even columns for the real results

	vaddpd	ymm0, ymm0, [srcreg+5*d2+d1+32]	;; r2+r24						; 28-30

	vmovapd	ymm4, [srcreg+5*d2+d1]		;; r12
	vaddpd	ymm4, ymm4, [srcreg+d1+32]	;; r12+r14						; 29-31

	ystore	[srcreg], ymm7			;; Final real #1A					; 24
	vmovapd	ymm7, [srcreg+2*d2+d1]		;; r6
	vaddpd	ymm7, ymm7, [srcreg+3*d2+d1+32]	;; r6+r20						; 30-32

	ystore	YMM_TMPS[1*32], ymm6		;; Real row #7						; 25
	vmovapd	ymm6, [srcreg+3*d2+d1]		;; r8
	vaddpd	ymm6, ymm6, [srcreg+2*d2+d1+32]	;; r8+r18						; 31-33

	ystore	YMM_TMPS[2*32], ymm5		;; Real odd-cols row #2					; 27
	vaddpd	ymm5, ymm0, ymm4		;; (r2+r24)+(r12+r14)					; 32-34

	vsubpd	ymm0, ymm0, ymm4		;; (r2+r24)-(r12+r14)					; 33-35
	L1prefetchw srcreg+3*d2+L1pd, L1pt

	vaddpd	ymm4, ymm7, ymm6		;; (r6+r20)+(r8+r18)					; 34-36

	vsubpd	ymm7, ymm7, ymm6		;; (r6+r20)-(r8+r18)					; 35-37

	vmovapd	ymm6, [srcreg+d2+d1]		;; r4
	vaddpd	ymm6, ymm6, [srcreg+4*d2+d1+32]	;; r4+r22						; 36-38

	ystore	YMM_TMPS[3*32], ymm3		;; Real odd-cols row #6					; 28
	vmovapd	ymm3, [srcreg+4*d2+d1]		;; r10
	vaddpd	ymm3, ymm3, [srcreg+d2+d1+32]	;; r10+r16						; 37-39

	ystore	YMM_TMPS[4*32], ymm1		;; Real odd-cols row #3					; 29
	vaddpd	ymm1, ymm5, ymm4		;; ((r2+r24)+(r12+r14)) + ((r6+r20)+(r8+r18)) (r15eA)	; 38-40

	vsubpd	ymm5, ymm5, ymm4		;; ((r2+r24)+(r12+r14)) - ((r6+r20)+(r8+r18)) (r3e)	; 39-41
	L1prefetchw srcreg+3*d2+d1+L1pd, L1pt

	vaddpd	ymm4, ymm6, ymm3		;; (r4+r22)+(r10+r16)					; 40-42

	vsubpd	ymm6, ymm6, ymm3		;; (r4+r22)-(r10+r16)					; 41-43
	vmulpd	ymm3, ymm1, YMM_HALF		;; .5*r15eA						; 41-45

	ystore	YMM_TMPS[5*32], ymm2		;; Real odd-cols row #5					; 30
	vsubpd	ymm2, ymm0, ymm7		;; ((r2+r24)-(r12+r14))-((r6+r20)-(r8+r18))		; 42-44
	vmulpd	ymm5, ymm5, YMM_P866		;; .866*r3e						; 42-46

	vaddpd	ymm1, ymm1, ymm4		;; r15eA + ((r4+r22)+(r10+r16)) (r1b)			; 43-45
	ystore	[srcreg+32], ymm1		;; Save final real #1B (real even-cols row #1)		; 46
	vmulpd	ymm1, ymm0, YMM_P966		;; .966*((r2+r24)-(r12+r14))				; 43-47

	ystore	YMM_TMPS[6*32], ymm5		;; Real even-cols row #3				; 47
	vmulpd	ymm5, ymm6, YMM_SQRTHALF	;; .707*((r4+r22)-(r10+r16))				; 44-48

	vsubpd	ymm2, ymm2, ymm6		;; ((r2+r24)-(r12+r14))-((r6+r20)-(r8+r18))-((r4+r22)-(r10+r16)) (r4e) ; 45-47
	vmulpd	ymm0, ymm0, YMM_P259		;; .259*((r2+r24)-(r12+r14))				; 45-49

	vsubpd	ymm3, ymm3, ymm4		;; .5*r15eA - ((r4+r22)+(r10+r16)) (r5e)		; 46-48
	vmulpd	ymm6, ymm7, YMM_P259		;; .259*((r6+r20)-(r8+r18))				; 46-50

	vmovapd	ymm4, [srcreg+d2]		;; r3
	vsubpd	ymm4, ymm4, [srcreg+5*d2+32]	;; r3-r23						; 47-49
	vmulpd	ymm7, ymm7, YMM_P966		;; .966*((r6+r20)-(r8+r18))				; 47-51

	vmulpd	ymm2, ymm2, YMM_SQRTHALF	;; .707*r4e						; 48-52
	ystore	YMM_TMPS[7*32], ymm3		;; Real even-cols row #5				; 49
	vmovapd	ymm3, [srcreg+5*d2]		;; r11
	vsubpd	ymm3, ymm3, [srcreg+d2+32]	;; r11-r15						; 48-50

	vaddpd	ymm1, ymm1, ymm5		;; .966*((r2+r24)-(r12+r14))+.707*((r4+r22)-(r10+r16))	; 49-51

	vsubpd	ymm0, ymm0, ymm5		;; .259*((r2+r24)-(r12+r14))-.707*((r4+r22)-(r10+r16))	; 50-52

	vmovapd	ymm5, [srcreg+2*d2]		;; r5
	vsubpd	ymm5, ymm5, [srcreg+4*d2+32]	;; r5-r21						; 51-53

	vaddpd	ymm1, ymm1, ymm6		;; .966*((r2+r24)-(r12+r14))+.707*((r4+r22)-(r10+r16))+.259*((r6+r20)-(r8+r18)) (r2e) ; 52-54
	L1prefetchw srcreg+4*d2+L1pd, L1pt

	vaddpd	ymm0, ymm0, ymm7		;; .259*((r2+r24)-(r12+r14))-.707*((r4+r22)-(r10+r16))+.966*((r6+r20)-(r8+r18)) (r6e) ; 53-55

	;; Do the odd columns for the imaginary results

	vmovapd	ymm6, [srcreg+4*d2]		;; r9
	vsubpd	ymm6, ymm6, [srcreg+2*d2+32]	;; r9-r17						; 54-56

	vaddpd	ymm7, ymm4, ymm3		;; (r3-r23)+(r11-r15)					; 55-57
	L1prefetchw srcreg+4*d2+d1+L1pd, L1pt

	vsubpd	ymm4, ymm4, ymm3		;; (r3-r23)-(r11-r15)					; 56-58

	vsubpd	ymm3, ymm5, ymm6		;; (r5-r21)-(r9-r17)					; 57-59

	vaddpd	ymm5, ymm5, ymm6		;; (r5-r21)+(r9-r17)					; 58-60
	vmulpd	ymm6, ymm7, YMM_HALF		;; .5*((r3-r23)+(r11-r15))				; 58-62

	ystore	YMM_TMPS[8*32], ymm2		;; Real even-cols row #4				; 53
	vmovapd	ymm2, [srcreg+3*d2]		;; r7
	vsubpd	ymm2, ymm2, [srcreg+3*d2+32]	;; r7-r19						; 59-61

	ystore	YMM_TMPS[9*32], ymm1		;; Real even-cols row #2				; 55
	vaddpd	ymm1, ymm4, ymm3		;; ((r3-r23)-(r11-r15)) + ((r5-r21)-(r9-r17)) (i3o)	; 60-62

	vsubpd	ymm3, ymm4, ymm3		;; ((r3-r23)-(r11-r15)) - ((r5-r21)-(r9-r17)) (i5o)	; 61-63
	vmovapd	ymm4, YMM_P866
	vmulpd	ymm5, ymm4, ymm5		;; .866*((r5-r21)+(r9-r17)) (i26oB)			; 61-65

	vsubpd	ymm7, ymm7, ymm2		;; ((r3-r23)+(r11-r15)) - (r7-r19) (i4o)		; 62-64
	L1prefetchw srcreg+5*d2+L1pd, L1pt

	vaddpd	ymm6, ymm6, ymm2		;; .5*((r3-r23)+(r11-r15)) + (r7-r19) (i26oA)		; 63-65
	vmulpd	ymm1, ymm4, ymm1		;; .866*i3o						; 63-67

	vmovapd	ymm2, [srcreg+d1]		;; r2
	vsubpd	ymm2, ymm2, [srcreg+5*d2+d1+32]	;; r2-r24						; 64-66
	vmulpd	ymm3, ymm4, ymm3		;; .866*i5o						; 64-68

	ystore	YMM_TMPS[10*32], ymm0		;; Real even-cols row #6				; 56
	vmovapd	ymm0, [srcreg+5*d2+d1]		;; r12
	vsubpd	ymm0, ymm0, [srcreg+d1+32]	;; r12-r14						; 65-67

	ystore	YMM_TMPS[11*32], ymm7		;; Imag odd-cols row #4					; 65
	vaddpd	ymm7, ymm6, ymm5		;; i26oA + i26oB (i2o)					; 66-68

	vsubpd	ymm6, ymm6, ymm5		;; i26oA - i26oB (i6o)					; 67-69

	;; Do the even columns for the imaginary results

	vmovapd	ymm5, [srcreg+2*d2+d1]		;; r6
	vsubpd	ymm5, ymm5, [srcreg+3*d2+d1+32]	;; r6-r20						; 68-70

	ystore	YMM_TMPS[12*32], ymm1		;; Imag odd-cols row #3					; 68
	vmovapd	ymm1, [srcreg+3*d2+d1]		;; r8
	vsubpd	ymm1, ymm1, [srcreg+2*d2+d1+32]	;; r8-r18						; 69-71

	ystore	YMM_TMPS[13*32], ymm3		;; Imag odd-cols row #5					; 69
	vaddpd	ymm3, ymm2, ymm0		;; (r2-r24)+(r12-r14)					; 70-72

	vsubpd	ymm2, ymm2, ymm0		;; (r2-r24)-(r12-r14)					; 71-73
	L1prefetchw srcreg+5*d2+d1+L1pd, L1pt

	vaddpd	ymm0, ymm5, ymm1		;; (r6-r20)+(r8-r18)					; 72-74

	vsubpd	ymm5, ymm5, ymm1		;; (r6-r20)-(r8-r18)					; 73-75

	vmovapd	ymm1, [srcreg+d2+d1]		;; r4
	vsubpd	ymm1, ymm1, [srcreg+4*d2+d1+32]	;; r4-r22						; 74-76

	ystore	YMM_TMPS[14*32], ymm7		;; Imag odd-cols row #2					; 69+1
	vmovapd	ymm7, [srcreg+4*d2+d1]		;; r10
	vsubpd	ymm7, ymm7, [srcreg+d2+d1+32]	;; r10-r16						; 75-77

	ystore	YMM_TMPS[15*32], ymm6		;; Imag odd-cols row #6					; 70+1
	vaddpd	ymm6, ymm2, ymm5		;; ((r2-r24)-(r12-r14)) + ((r6-r20)-(r8-r18)) (i37eA)	; 76-78

	vsubpd	ymm2, ymm2, ymm5		;; ((r2-r24)-(r12-r14)) - ((r6-r20)-(r8-r18)) (i5e)	; 77-79

	vsubpd	ymm5, ymm1, ymm7		;; (r4-r22)-(r10-r16)					; 78-80

	vaddpd	ymm1, ymm1, ymm7		;; (r4-r22)+(r10-r16)					; 79-81
	vmulpd	ymm7, ymm6, YMM_HALF		;; .5*i37eA						; 79-83

	vmulpd	ymm2, ymm4, ymm2		;; .866*i5e						; 80-84
	vsubpd	ymm4, ymm3, ymm0		;; ((r2-r24)+(r12-r14))-((r6-r20)+(r8-r18))		; 80-82

	vsubpd	ymm6, ymm6, ymm5		;; i37eA - ((r4-r22)-(r10-r16)) (i7)			; 81-83
	ystore	YMM_TMPS[16*32], ymm6		;; Imag cols row #7					; 84
	vmulpd	ymm6, ymm3, YMM_P259		;; .259*((r2-r24)+(r12-r14))				; 73-77

	ystore	YMM_TMPS[17*32], ymm2		;; Imag even-cols row #5				; 85
	vmulpd	ymm2, ymm0, YMM_P966		;; .966*((r6-r20)+(r8-r18))				; 75-79

	vaddpd	ymm4, ymm4, ymm1		;; ((r2-r24)+(r12-r14))-((r6-r20)+(r8-r18))+((r4-r22)+(r10-r16)) (i4e) ; 83-85
	vmulpd	ymm3, ymm3, YMM_P966		;; .966*((r2-r24)+(r12-r14))				; 74-78

	vaddpd	ymm7, ymm7, ymm5		;; .5*i37eA + ((r4-r22)-(r10-r16)) (i3e)		; 84-86
	vmulpd	ymm0, ymm0, YMM_P259		;; .259*((r6-r20)+(r8-r18))				; 76-80

	vaddpd	ymm6, ymm6, ymm2		;; .259*((r2-r24)+(r12-r14))+.966*((r6-r20)+(r8-r18))	; 85-87  (80 at earliest)
	vmulpd	ymm1, ymm1, YMM_SQRTHALF	;; .707*((r4-r22)+(r10-r16))				; 81-85

	vaddpd	ymm3, ymm3, ymm0		;; .966*((r2-r24)+(r12-r14))+.259*((r6-r20)+(r8-r18))	; 86-88 (81 at earliest)
	vmulpd	ymm4, ymm4, YMM_SQRTHALF	;; .707*i4e						; 86-90

	ystore	YMM_TMPS[18*32], ymm7		;; Imag even-cols row #3				; 87

	vaddpd	ymm6, ymm6, ymm1		;; .259*((r2-r24)+(r12-r14))+.966*((r6-r20)+(r8-r18))+.707*((r4-r22)+(r10-r16)) (i2e) ; 88-90 (83 at earliest)

	vsubpd	ymm3, ymm3, ymm1		;; .966*((r2-r24)+(r12-r14))+.259*((r6-r20)+(r8-r18))-.707*((r4-r22)+(r10-r16)) (i6e) ; 89-91 (84 at earliest)

	ystore	YMM_TMPS[19*32], ymm4		;; Imag even-cols row #4				; 91
;;	ystore	YMM_TMPS[?*32], ymm6		;; Imag even-cols row #2				; 91+1
	ystore	YMM_TMPS[20*32], ymm3		;; Imag even-cols row #6				; 92

	;; Now combine the even and odd columns then do the post-multiply by twiddle factors.

	vmovapd	ymm3, YMM_TMPS[2*32]		;; Real odd-cols row #2
	vmovapd	ymm7, YMM_TMPS[9*32]		;; Real even-cols row #2
	vsubpd	ymm0, ymm3, ymm7		;; Real #12
	vaddpd	ymm1, ymm3, ymm7		;; Real #2
;;	vmovapd	ymm6, YMM_TMPS[?*32]		;; Imag even-cols row #2
	vmovapd	ymm5, YMM_TMPS[14*32]		;; Imag odd-cols row #2
	vsubpd	ymm2, ymm6, ymm5		;; Imag #12
	vaddpd	ymm5, ymm6, ymm5		;; Imag #2

	vmovapd	ymm3, [screg+10*64+32]		;; cosine/sine
	vmulpd	ymm6, ymm0, ymm3		;; A12 = R12 * cosine/sine
	vmovapd	ymm4, [screg+32]		;; cosine/sine
	vmulpd	ymm7, ymm1, ymm4		;; A2 = R2 * cosine/sine
	vsubpd	ymm6, ymm6, ymm2		;; A12 = A12 - I12
	vmulpd	ymm2, ymm2, ymm3		;; B12 = I12 * cosine/sine
	vsubpd	ymm7, ymm7, ymm5		;; A2 = A2 - I2
	vmulpd	ymm5, ymm5, ymm4		;; B2 = I2 * cosine/sine
	vaddpd	ymm2, ymm2, ymm0		;; B12 = B12 + R12
	vmovapd	ymm3, [screg+10*64]		;; sine
	vmulpd	ymm6, ymm6, ymm3		;; A12 = A12 * sine (final R12)
	vaddpd	ymm5, ymm5, ymm1		;; B2 = B2 + R2
	vmovapd	ymm4, [screg]			;; sine
	vmulpd	ymm7, ymm7, ymm4		;; A2 = A2 * sine (final R2)
	vmulpd	ymm2, ymm2, ymm3		;; B12 = B12 * sine (final I12)
	vmulpd	ymm5, ymm5, ymm4		;; B2 = B2 * sine (final I2)
	ystore	[srcreg+5*d2+d1], ymm6		;; Save final R12
	ystore	[srcreg+5*d2+d1+32], ymm2	;; Save final I12
	ystore	[srcreg+d1], ymm7		;; Save final R2
	ystore	[srcreg+d1+32], ymm5		;; Save final I2

	vmovapd	ymm6, YMM_TMPS[4*32]		;; Real odd-cols row #3
	vmovapd	ymm7, YMM_TMPS[6*32]		;; Real even-cols row #3
	vsubpd	ymm0, ymm6, ymm7		;; Real #11
	vaddpd	ymm1, ymm6, ymm7		;; Real #3
	vmovapd	ymm6, YMM_TMPS[18*32]		;; Imag even-cols row #3
	vmovapd	ymm7, YMM_TMPS[12*32]		;; Imag odd-cols row #3
	vsubpd	ymm2, ymm6, ymm7		;; Imag #11
	vaddpd	ymm3, ymm6, ymm7		;; Imag #3

	vmovapd	ymm5, [screg+9*64+32]		;; cosine/sine
	vmulpd	ymm6, ymm0, ymm5		;; A11 = R11 * cosine/sine
	vmovapd	ymm4, [screg+64+32]		;; cosine/sine
	vmulpd	ymm7, ymm1, ymm4		;; A3 = R3 * cosine/sine
	vsubpd	ymm6, ymm6, ymm2		;; A11 = A11 - I11
	vmulpd	ymm2, ymm2, ymm5		;; B11 = I11 * cosine/sine
	vsubpd	ymm7, ymm7, ymm3		;; A3 = A3 - I3
	vmulpd	ymm3, ymm3, ymm4		;; B3 = I3 * cosine/sine
	vaddpd	ymm2, ymm2, ymm0		;; B11 = B11 + R11
	vmovapd	ymm5, [screg+9*64]		;; sine
	vmulpd	ymm6, ymm6, ymm5		;; A11 = A11 * sine (final R11)
	vaddpd	ymm3, ymm3, ymm1		;; B3 = B3 + R3
	vmovapd	ymm4, [screg+64]		;; sine
	vmulpd	ymm7, ymm7, ymm4		;; A3 = A3 * sine (final R3)
	vmulpd	ymm2, ymm2, ymm5		;; B11 = B11 * sine (final I11)
	vmulpd	ymm3, ymm3, ymm4		;; B3 = B3 * sine (final I3)
	ystore	[srcreg+5*d2], ymm6		;; Save final R11
	ystore	[srcreg+5*d2+32], ymm2		;; Save final I11
	ystore	[srcreg+d2], ymm7		;; Save final R3
	ystore	[srcreg+d2+32], ymm3		;; Save final I3

	vmovapd	ymm6, YMM_TMPS[0*32]		;; Real odd-cols row #4
	vmovapd	ymm7, YMM_TMPS[8*32]		;; Real even-cols row #4
	vsubpd	ymm0, ymm6, ymm7		;; Real #10
	vaddpd	ymm1, ymm6, ymm7		;; Real #4
	vmovapd	ymm6, YMM_TMPS[19*32]		;; Imag even-cols row #4
	vmovapd	ymm7, YMM_TMPS[11*32]		;; Imag odd-cols row #4
	vsubpd	ymm2, ymm6, ymm7		;; Imag #10
	vaddpd	ymm3, ymm6, ymm7		;; Imag #4

	vmovapd	ymm5, [screg+8*64+32]		;; cosine/sine
	vmulpd	ymm6, ymm0, ymm5		;; A10 = R10 * cosine/sine
	vmovapd	ymm4, [screg+2*64+32]		;; cosine/sine
	vmulpd	ymm7, ymm1, ymm4		;; A4 = R4 * cosine/sine
	vsubpd	ymm6, ymm6, ymm2		;; A10 = A10 - I10
	vmulpd	ymm2, ymm2, ymm5		;; B10 = I10 * cosine/sine
	vsubpd	ymm7, ymm7, ymm3		;; A4 = A4 - I4
	vmulpd	ymm3, ymm3, ymm4		;; B4 = I4 * cosine/sine
	vaddpd	ymm2, ymm2, ymm0		;; B10 = B10 + R10
	vmovapd ymm5, [screg+8*64]		;; sine
	vmulpd	ymm6, ymm6, ymm5		;; A10 = A10 * sine (final R10)
	vaddpd	ymm3, ymm3, ymm1		;; B4 = B4 + R4
	vmovapd	ymm4, [screg+2*64]		;; sine
	vmulpd	ymm7, ymm7, ymm4		;; A4 = A4 * sine (final R4)
	vmulpd	ymm2, ymm2, ymm5		;; B10 = B10 * sine (final I10)
	vmulpd	ymm3, ymm3, ymm4		;; B4 = B4 * sine (final I4)
	ystore	[srcreg+4*d2+d1], ymm6		;; Save final R10
	ystore	[srcreg+4*d2+d1+32], ymm2	;; Save final I10
	ystore	[srcreg+d2+d1], ymm7		;; Save final R4
	ystore	[srcreg+d2+d1+32], ymm3		;; Save final I4

	vmovapd	ymm6, YMM_TMPS[5*32]		;; Real odd-cols row #5
	vmovapd	ymm7, YMM_TMPS[7*32]		;; Real even-cols row #5
	vsubpd	ymm0, ymm6, ymm7		;; Real #9
	vaddpd	ymm1, ymm6, ymm7		;; Real #5
	vmovapd	ymm6, YMM_TMPS[17*32]		;; Imag even-cols row #5
	vmovapd	ymm7, YMM_TMPS[13*32]		;; Imag odd-cols row #5
	vsubpd	ymm2, ymm6, ymm7		;; Imag #9
	vaddpd	ymm3, ymm6, ymm7		;; Imag #5

	vmovapd	ymm5, [screg+7*64+32]		;; cosine/sine
	vmulpd	ymm6, ymm0, ymm5		;; A9 = R9 * cosine/sine
	vmovapd	ymm4, [screg+3*64+32]		;; cosine/sine
	vmulpd	ymm7, ymm1, ymm4		;; A5 = R5 * cosine/sine
	vsubpd	ymm6, ymm6, ymm2		;; A9 = A9 - I9
	vmulpd	ymm2, ymm2, ymm5		;; B9 = I9 * cosine/sine
	vsubpd	ymm7, ymm7, ymm3		;; A5 = A5 - I5
	vmulpd	ymm3, ymm3, ymm4		;; B5 = I5 * cosine/sine
	vaddpd	ymm2, ymm2, ymm0		;; B9 = B9 + R9
	vmovapd	ymm5, [screg+7*64]		;; sine
	vmulpd	ymm6, ymm6, ymm5		;; A9 = A9 * sine (final R9)
	vaddpd	ymm3, ymm3, ymm1		;; B5 = B5 + R5
	vmovapd	ymm4, [screg+3*64]		;; sine
	vmulpd	ymm7, ymm7, ymm4		;; A5 = A5 * sine (final R5)
	vmulpd	ymm2, ymm2, ymm5		;; B9 = B9 * sine (final I9)
	vmulpd	ymm3, ymm3, ymm4		;; B5 = B5 * sine (final I5)
	ystore	[srcreg+4*d2], ymm6		;; Save final R9
	ystore	[srcreg+4*d2+32], ymm2		;; Save final I9
	ystore	[srcreg+2*d2], ymm7		;; Save final R5
	ystore	[srcreg+2*d2+32], ymm3		;; Save final I5

	vmovapd	ymm6, YMM_TMPS[3*32]		;; Real odd-cols row #6
	vmovapd	ymm7, YMM_TMPS[10*32]		;; Real even-cols row #6
	vsubpd	ymm0, ymm6, ymm7		;; Real #8
	vaddpd	ymm1, ymm6, ymm7		;; Real #6
	vmovapd	ymm6, YMM_TMPS[20*32]		;; Imag even-cols row #6
	vmovapd	ymm7, YMM_TMPS[15*32]		;; Imag odd-cols row #6
	vsubpd	ymm2, ymm6, ymm7		;; Imag #8
	vaddpd	ymm3, ymm6, ymm7		;; Imag #6

	vmovapd	ymm5, [screg+6*64+32]		;; cosine/sine
	vmulpd	ymm6, ymm0, ymm5		;; A8 = R8 * cosine/sine
	vmovapd	ymm4, [screg+4*64+32]		;; cosine/sine
	vmulpd	ymm7, ymm1, ymm4		;; A6 = R6 * cosine/sine
	vsubpd	ymm6, ymm6, ymm2		;; A8 = A8 - I8
	vmulpd	ymm2, ymm2, ymm5		;; B8 = I8 * cosine/sine
	vsubpd	ymm7, ymm7, ymm3		;; A6 = A6 - I6
	vmulpd	ymm3, ymm3, ymm4		;; B6 = I6 * cosine/sine
	vaddpd	ymm2, ymm2, ymm0		;; B8 = B8 + R8
	vmovapd	ymm5, [screg+6*64]		;; sine
	vmulpd	ymm6, ymm6, ymm5		;; A8 = A8 * sine (final R8)
	vaddpd	ymm3, ymm3, ymm1		;; B6 = B6 + R6
	vmovapd	ymm4, [screg+4*64]		;; sine
	vmulpd	ymm7, ymm7, ymm4		;; A6 = A6 * sine (final R6)
	vmulpd	ymm2, ymm2, ymm5		;; B8 = B8 * sine (final I8)
	vmulpd	ymm3, ymm3, ymm4		;; B6 = B6 * sine (final I6)
	ystore	[srcreg+3*d2+d1], ymm6		;; Save final R8
	ystore	[srcreg+3*d2+d1+32], ymm2	;; Save final I8
	ystore	[srcreg+2*d2+d1], ymm7		;; Save final R6
	ystore	[srcreg+2*d2+d1+32], ymm3	;; Save final I6

	vmovapd	ymm0, YMM_TMPS[1*32]		;; Real #7
	vmovapd	ymm1, YMM_TMPS[16*32]		;; Imag #7
	vmovapd	ymm5, [screg+5*64+32]		;; cosine/sine
	vmulpd	ymm4, ymm0, ymm5		;; A7 = R7 * cosine/sine
	vsubpd	ymm4, ymm4, ymm1		;; A7 = A7 - I7
	vmulpd	ymm1, ymm1, ymm5		;; B7 = I7 * cosine/sine
	vaddpd	ymm1, ymm1, ymm0		;; B7 = B7 + R7
	vmovapd	ymm5, [screg+5*64]		;; sine
	vmulpd	ymm4, ymm4, ymm5		;; A7 = A7 * sine (final R7)
	vmulpd	ymm1, ymm1, ymm5		;; B7 = B7 * sine (final I7)
	ystore	[srcreg+3*d2], ymm4		;; Save final R7
	ystore	[srcreg+3*d2+32], ymm1		;; Save final I7

	bump	srcreg, srcinc
	bump	screg, scinc
	ENDM

;; 64-bit version

IFDEF X86_64

yr6_12cl_24_reals_fft_preload MACRO
	ENDM

yr6_12cl_24_reals_fft MACRO srcreg,srcinc,d1,d2,screg,scinc,maxrpt,L1pt,L1pd

	;; Do the odd columns for the real results

	vmovapd	ymm0, [srcreg+2*d2]		;; r5
	vaddpd	ymm0, ymm0, [srcreg+4*d2+32]	;; r5+r21						; 1-3

	vmovapd	ymm1, [srcreg+4*d2]		;; r9
	vaddpd	ymm1, ymm1, [srcreg+2*d2+32]	;; r9+r17						; 2-4

	vmovapd	ymm3, [srcreg]			;; r1
	vmovapd	ymm4, [srcreg+32]		;; r13
	vaddpd	ymm2, ymm3, ymm4		;; r1+r13						; 3-5

	vsubpd	ymm3, ymm3, ymm4		;; r1-r13						; 4-6

	vsubpd	ymm4, ymm0, ymm1		;; (r5+r21)-(r9+r17)					; 5-7

	vaddpd	ymm0, ymm0, ymm1		;; (r5+r21)+(r9+r17)					; 6-8

	vmovapd	ymm1, [srcreg+d2]		;; r3
	vaddpd	ymm1, ymm1, [srcreg+5*d2+32]	;; r3+r23						; 7-9

	vsubpd	ymm5, ymm3, ymm4		;; r1-r13 - ((r5+r21)-(r9+r17)) (r4o)			; 8-10
	vmovapd ymm15, YMM_HALF
	vmulpd	ymm4, ymm15, ymm4		;; .5*((r5+r21)-(r9+r17))				; 8-12

	vaddpd	ymm6, ymm2, ymm0		;; r1+r13 + ((r5+r21)+(r9+r17)) (r17a)			; 9-11
	vmulpd	ymm0, ymm15, ymm0		;; .5*((r5+r21)+(r9+r17))				; 9-13

	vmovapd	ymm7, [srcreg+5*d2]		;; r11
	vaddpd	ymm7, ymm7, [srcreg+d2+32]	;; r11+r15						; 10-12

	vmovapd	ymm8, [srcreg+3*d2]		;; r7
	vaddpd	ymm8, ymm8, [srcreg+3*d2+32]	;; r7+r19						; 11-13

	vmovapd	ymm9, [srcreg+d1]		;; r2
	vaddpd	ymm9, ymm9, [srcreg+5*d2+d1+32]	;; r2+r24						; 12-14

	vaddpd	ymm3, ymm3, ymm4		;; r1-r13 + .5*((r5+r21)-(r9+r17)) (r26oA)		; 13-15
	vmovapd	ymm10, [srcreg+5*d2+d1]		;; r12

	vsubpd	ymm2, ymm2, ymm0		;; r1+r13 - .5*((r5+r21)+(r9+r17)) (r35oA)		; 14-16
	vmovapd	ymm0, [srcreg+2*d2+d1]		;; r6

	vaddpd	ymm4, ymm1, ymm7		;; (r3+r23)+(r11+r15)					; 15-17
	L1prefetchw srcreg+L1pd, L1pt

	vsubpd	ymm1, ymm1, ymm7		;; (r3+r23)-(r11+r15)					; 16-18
	ystore	YMM_TMPS[0*32], ymm5		;; Real odd-cols row #4					; 11

	vaddpd	ymm10, ymm10, [srcreg+d1+32]	;; r12+r14						; 17-19
	L1prefetchw srcreg+d1+L1pd, L1pt

	vaddpd	ymm7, ymm4, ymm8		;; ((r3+r23)+(r11+r15)) + r7+r19 (r17b)			; 18-20
	vmulpd	ymm4, ymm15, ymm4		;; .5*((r3+r23)+(r11+r15))				; 18-22

	vaddpd	ymm0, ymm0, [srcreg+3*d2+d1+32]	;; r6+r20						; 19-21
	vmovapd	ymm14, YMM_P866
	vmulpd	ymm1, ymm14, ymm1		;; .866*((r3+r23)-(r11+r15)) (r26oB)			; 19-23

	vmovapd	ymm5, [srcreg+3*d2+d1]		;; r8
	vaddpd	ymm5, ymm5, [srcreg+2*d2+d1+32]	;; r8+r18						; 20-22

	vaddpd	ymm11, ymm6, ymm7		;; r17a + r17b (r1a)					; 21-23
	L1prefetchw srcreg+d2+L1pd, L1pt

	vsubpd	ymm6, ymm6, ymm7		;; r17a - r17b (r7)					; 22-24

	vsubpd	ymm4, ymm4, ymm8		;; .5*((r3+r23)+(r11+r15)) - (r7+r19) (r35oB)		; 23-25
	L1prefetchw srcreg+d2+d1+L1pd, L1pt

	vaddpd	ymm8, ymm3, ymm1		;; r26oA + r26oB (r2o)					; 24-26
	ystore	[srcreg], ymm11			;; Final real #1A					; 24

	vsubpd	ymm3, ymm3, ymm1		;; r26oA - r26oB (r6o)					; 25-27
	ystore	YMM_TMPS[1*32], ymm6		;; Real row #7						; 25

	vaddpd	ymm1, ymm2, ymm4		;; r35oA + r35oB (r3o)					; 26-28
	L1prefetchw srcreg+2*d2+L1pd, L1pt

	vsubpd	ymm2, ymm2, ymm4		;; r35oA - r35oB (r5o)					; 27-29
	ystore	YMM_TMPS[2*32], ymm8		;; Real odd-cols row #2					; 27

	;; Do the even columns for the real results

	vaddpd	ymm4, ymm9, ymm10		;; (r2+r24)+(r12+r14)					; 28-30
	ystore	YMM_TMPS[3*32], ymm3		;; Real odd-cols row #6					; 28

	vsubpd	ymm9, ymm9, ymm10		;; (r2+r24)-(r12+r14)					; 29-31
	vmovapd	ymm6, [srcreg+d2+d1]		;; r4

	vaddpd	ymm10, ymm0, ymm5		;; (r6+r20)+(r8+r18)					; 30-32
	ystore	YMM_TMPS[4*32], ymm1		;; Real odd-cols row #3					; 29

	vsubpd	ymm0, ymm0, ymm5		;; (r6+r20)-(r8+r18)					; 31-33
	ystore	YMM_TMPS[5*32], ymm2		;; Real odd-cols row #5					; 30

	vaddpd	ymm6, ymm6, [srcreg+4*d2+d1+32]	;; r4+r22						; 32-34

	vmovapd	ymm3, [srcreg+4*d2+d1]		;; r10
	vaddpd	ymm3, ymm3, [srcreg+d2+d1+32]	;; r10+r16						; 33-35

	vaddpd	ymm1, ymm4, ymm10		;; ((r2+r24)+(r12+r14)) + ((r6+r20)+(r8+r18)) (r15eA)	; 34-36
	vmovapd	ymm13, YMM_P966

	vsubpd	ymm4, ymm4, ymm10		;; ((r2+r24)+(r12+r14)) - ((r6+r20)+(r8+r18)) (r3e)	; 35-37
	vmovapd	ymm7, [srcreg+5*d2]		;; r11

	vaddpd	ymm8, ymm6, ymm3		;; (r4+r22)+(r10+r16)					; 36-38
	L1prefetchw srcreg+2*d2+d1+L1pd, L1pt

	vsubpd	ymm6, ymm6, ymm3		;; (r4+r22)-(r10+r16)					; 37-39
	vmulpd	ymm3, ymm15, ymm1		;; .5*r15eA						; 37-41

	vsubpd	ymm2, ymm9, ymm0		;; ((r2+r24)-(r12+r14))-((r6+r20)-(r8+r18))		; 38-40
	vmulpd	ymm4, ymm14, ymm4		;; .866*r3e						; 38-42
	L1prefetchw srcreg+3*d2+L1pd, L1pt

	vaddpd	ymm1, ymm1, ymm8		;; r15eA + ((r4+r22)+(r10+r16)) (r1b)			; 39-41
	vmulpd	ymm10, ymm13, ymm9		;; .966*((r2+r24)-(r12+r14))				; 39-43

	vsubpd	ymm7, ymm7, [srcreg+d2+32]	;; r11-r15						; 40-42
	vmovapd	ymm12, YMM_SQRTHALF
	vmulpd	ymm5, ymm12, ymm6		;; .707*((r4+r22)-(r10+r16))				; 40-44

	vsubpd	ymm2, ymm2, ymm6		;; ((r2+r24)-(r12+r14))-((r6+r20)-(r8+r18))-((r4+r22)-(r10+r16)) (r4e) ; 41-43
	vmovapd	ymm11, YMM_P259
	vmulpd	ymm9, ymm11, ymm9		;; .259*((r2+r24)-(r12+r14))				; 41-45

	vsubpd	ymm3, ymm3, ymm8		;; .5*r15eA - ((r4+r22)+(r10+r16)) (r5e)		; 42-44
	vmulpd	ymm8, ymm11, ymm0		;; .259*((r6+r20)-(r8+r18))				; 42-46
	ystore	[srcreg+32], ymm1		;; Save final real #1B (real even-cols row #1)		; 42

	vmovapd	ymm1, [srcreg+d2]		;; r3
	vsubpd	ymm1, ymm1, [srcreg+5*d2+32]	;; r3-r23						; 43-45
	vmulpd	ymm0, ymm13, ymm0		;; .966*((r6+r20)-(r8+r18))				; 43-47
	ystore	YMM_TMPS[6*32], ymm4		;; Real even-cols row #3				; 43

	vmovapd	ymm4, [srcreg+2*d2]		;; r5
	vsubpd	ymm4, ymm4, [srcreg+4*d2+32]	;; r5-r21						; 44-46
	vmulpd	ymm2, ymm12, ymm2		;; .707*r4e						; 44-48

	vaddpd	ymm10, ymm10, ymm5		;; .966*((r2+r24)-(r12+r14))+.707*((r4+r22)-(r10+r16))	; 45-47
	L1prefetchw srcreg+3*d2+d1+L1pd, L1pt

	vsubpd	ymm9, ymm9, ymm5		;; .259*((r2+r24)-(r12+r14))-.707*((r4+r22)-(r10+r16))	; 46-48

	vmovapd	ymm5, [srcreg+4*d2]		;; r9
	vsubpd	ymm5, ymm5, [srcreg+2*d2+32]	;; r9-r17						; 47-49

	vaddpd	ymm10, ymm10, ymm8		;; .966*((r2+r24)-(r12+r14))+.707*((r4+r22)-(r10+r16))+.259*((r6+r20)-(r8+r18)) (r2e) ; 48-50
	L1prefetchw srcreg+4*d2+L1pd, L1pt

	vaddpd	ymm9, ymm9, ymm0		;; .259*((r2+r24)-(r12+r14))-.707*((r4+r22)-(r10+r16))+.966*((r6+r20)-(r8+r18)) (r6e) ; 49-51
	ystore	YMM_TMPS[7*32], ymm3		;; Real even-cols row #5				; 49

	;; Do the odd columns for the imaginary results

	vaddpd	ymm0, ymm1, ymm7		;; (r3-r23)+(r11-r15)					; 50-52
	L1prefetchw srcreg+4*d2+d1+L1pd, L1pt

	vsubpd	ymm1, ymm1, ymm7		;; (r3-r23)-(r11-r15)					; 51-53
	ystore	YMM_TMPS[8*32], ymm2		;; Real even-cols row #4				; 49+1

	vsubpd	ymm7, ymm4, ymm5		;; (r5-r21)-(r9-r17)					; 52-54
	vmovapd	ymm2, [srcreg+3*d2]		;; r7

	vaddpd	ymm4, ymm4, ymm5		;; (r5-r21)+(r9-r17)					; 53-55
	vmulpd	ymm5, ymm15, ymm0		;; .5*((r3-r23)+(r11-r15))				; 53-57

	vsubpd	ymm2, ymm2, [srcreg+3*d2+32]	;; r7-r19						; 54-56
	ystore	YMM_TMPS[9*32], ymm9		;; Real even-cols row #6				; 52

	vaddpd	ymm9, ymm1, ymm7		;; ((r3-r23)-(r11-r15)) + ((r5-r21)-(r9-r17)) (i3o)	; 55-57

	vsubpd	ymm1, ymm1, ymm7		;; ((r3-r23)-(r11-r15)) - ((r5-r21)-(r9-r17)) (i5o)	; 56-58
	vmulpd	ymm4, ymm14, ymm4		;; .866*((r5-r21)+(r9-r17)) (i26oB)			; 56-60
	vmovapd	ymm6, [srcreg+d1]		;; r2

	vsubpd	ymm0, ymm0, ymm2		;; ((r3-r23)+(r11-r15)) - (r7-r19) (i4o)		; 57-59
	L1prefetchw srcreg+5*d2+L1pd, L1pt

	vaddpd	ymm5, ymm5, ymm2		;; .5*((r3-r23)+(r11-r15)) + (r7-r19) (i26oA)		; 58-60
	vmulpd	ymm9, ymm14, ymm9		;; .866*i3o						; 58-62

	vsubpd	ymm6, ymm6, [srcreg+5*d2+d1+32]	;; r2-r24						; 59-61
	vmulpd	ymm1, ymm14, ymm1		;; .866*i5o						; 59-63

	vmovapd	ymm2, [srcreg+5*d2+d1]		;; r12
	vsubpd	ymm2, ymm2, [srcreg+d1+32]	;; r12-r14						; 60-62

	vaddpd	ymm7, ymm5, ymm4		;; i26oA + i26oB (i2o)					; 61-63
	vmovapd	ymm8, [srcreg+2*d2+d1]		;; r6

	vsubpd	ymm5, ymm5, ymm4		;; i26oA - i26oB (i6o)					; 62-64
	ystore	YMM_TMPS[10*32], ymm0		;; Imag odd-cols row #4					; 60

	;; Do the even columns for the imaginary results

	vsubpd	ymm8, ymm8, [srcreg+3*d2+d1+32]	;; r6-r20						; 63-65

	vmovapd	ymm3, [srcreg+3*d2+d1]		;; r8
	vsubpd	ymm3, ymm3, [srcreg+2*d2+d1+32]	;; r8-r18						; 64-66

	vaddpd	ymm4, ymm6, ymm2		;; (r2-r24)+(r12-r14)					; 65-67
	ystore	YMM_TMPS[11*32], ymm1		;; Imag odd-cols row #5					; 64

	vsubpd	ymm6, ymm6, ymm2		;; (r2-r24)-(r12-r14)					; 66-68
	vmovapd	ymm1, [srcreg+d2+d1]		;; r4

	vaddpd	ymm2, ymm8, ymm3		;; (r6-r20)+(r8-r18)					; 67-69
	L1prefetchw srcreg+5*d2+d1+L1pd, L1pt

	vsubpd	ymm8, ymm8, ymm3		;; (r6-r20)-(r8-r18)					; 68-70
	ystore	YMM_TMPS[12*32], ymm5		;; Imag odd-cols row #6					; 65

	vsubpd	ymm1, ymm1, [srcreg+4*d2+d1+32]	;; r4-r22						; 69-71

	vmovapd	ymm0, [srcreg+4*d2+d1]		;; r10
	vsubpd	ymm0, ymm0, [srcreg+d2+d1+32]	;; r10-r16						; 70-72
	vmulpd	ymm3, ymm11, ymm4		;; .259*((r2-r24)+(r12-r14))				; 70-74

	vaddpd	ymm5, ymm6, ymm8		;; ((r2-r24)-(r12-r14)) + ((r6-r20)-(r8-r18)) (i37eA)	; 71-73
	vmulpd	ymm14, ymm13, ymm2		;; .966*((r6-r20)+(r8-r18))				; 71-75

	vsubpd	ymm6, ymm6, ymm8		;; ((r2-r24)-(r12-r14)) - ((r6-r20)-(r8-r18)) (i5e)	; 72-74
	vmulpd	ymm13, ymm13, ymm4		;; .966*((r2-r24)+(r12-r14))				; 72-76

	vaddpd	ymm8, ymm1, ymm0		;; (r4-r22)+(r10-r16)					; 73-75
	vmulpd	ymm11, ymm11, ymm2		;; .259*((r6-r20)+(r8-r18))				; 73-77

	vsubpd	ymm1, ymm1, ymm0		;; (r4-r22)-(r10-r16)					; 74-76
	vmulpd	ymm0, ymm15, ymm5		;; .5*i37eA						; 74-78

	vsubpd	ymm4, ymm4, ymm2		;; ((r2-r24)+(r12-r14))-((r6-r20)+(r8-r18))		; 75-77
	vmulpd	ymm6, ymm6, YMM_P866		;; .866*i5e						; 75-79

	vaddpd	ymm3, ymm3, ymm14		;; .259*((r2-r24)+(r12-r14))+.966*((r6-r20)+(r8-r18))	; 76-78
	vmulpd	ymm2, ymm12, ymm8		;; .707*((r4-r22)+(r10-r16))				; 76-80
	vmovapd	ymm14, YMM_TMPS[1*32]		;; Real #7

	vsubpd	ymm5, ymm5, ymm1		;; i37eA - ((r4-r22)-(r10-r16)) (i7)			; 77-79

	vaddpd	ymm13, ymm13, ymm11		;; .966*((r2-r24)+(r12-r14))+.259*((r6-r20)+(r8-r18))	; 78-80
	vmovapd	ymm11, [screg+5*64+32]		;; cosine/sine for R7/I7

	vaddpd	ymm0, ymm0, ymm1		;; .5*i37eA + ((r4-r22)-(r10-r16)) (i3e)		; 79-81

	vaddpd	ymm4, ymm4, ymm8		;; ((r2-r24)+(r12-r14))-((r6-r20)+(r8-r18))+((r4-r22)+(r10-r16)) (i4e) ; 80-82
	vmovapd	ymm8, YMM_TMPS[2*32]		;; Real odd-cols row #2

	vaddpd	ymm3, ymm3, ymm2		;; .259*((r2-r24)+(r12-r14))+.966*((r6-r20)+(r8-r18))+.707*((r4-r22)+(r10-r16)) (i2e) ; 81-83
	vmulpd	ymm1, ymm14, ymm11		;; A7 = R7 * cosine/sine				;  81-85

	vsubpd	ymm13, ymm13, ymm2		;; .966*((r2-r24)+(r12-r14))+.259*((r6-r20)+(r8-r18))-.707*((r4-r22)+(r10-r16)) (i6e) ; 82-84
	vmulpd	ymm11, ymm5, ymm11		;; B7 = I7 * cosine/sine				;  82-86

	;; Now combine the even and odd columns then do the post-multiply by twiddle factors.

	vsubpd	ymm15, ymm8, ymm10		;; (ro2 - re2) Real #12					; 83-85
	vmulpd	ymm4, ymm12, ymm4		;; .707*i4e						; 83-87

	vsubpd	ymm2, ymm3, ymm7		;; (ie2 - io2) Imag #12					; 84-86
	vmovapd	ymm12, [screg+10*64+32]		;; cosine/sine for R12/I12

	vaddpd	ymm8, ymm8, ymm10		;; (ro2 + re2) Real #2					; 85-87
	ystore	YMM_TMPS[1*32], ymm6		;; Imag even-cols row #5				; 80

	vaddpd	ymm3, ymm3, ymm7		;; (ie2 + io2) Imag #2					; 86-88
	vmulpd	ymm10, ymm15, ymm12		;; A12 = R12 * cosine/sine				;  86-90
	vmovapd	ymm6, [screg+32]		;; cosine/sine for R2/I2

	vsubpd	ymm1, ymm1, ymm5		;; A7 = A7 - I7						; 87-89
	vmulpd	ymm12, ymm2, ymm12		;; B12 = I12 * cosine/sine				;  87-91
	vmovapd	ymm5, [screg+5*64]		;; sine for R7/I7

	vaddpd	ymm11, ymm11, ymm14		;; B7 = B7 + R7						; 88-90
	vmulpd	ymm7, ymm8, ymm6		;; A2 = R2 * cosine/sine				;  88-92

	vmulpd	ymm6, ymm3, ymm6		;; B2 = I2 * cosine/sine				;  89-93
	vmovapd	ymm14, [screg+10*64]		;; sine for R12/I12

	vmulpd	ymm1, ymm1, ymm5		;; A7 = A7 * sine (final R7)				;  90-94

	vsubpd	ymm10, ymm10, ymm2		;; A12 = A12 - I12					; 91-93
	vmulpd	ymm11, ymm11, ymm5		;; B7 = B7 * sine (final I7)				;  91-95

	vaddpd	ymm12, ymm12, ymm15		;; B12 = B12 + R12					; 92-94
	vmovapd	ymm2, YMM_TMPS[4*32]		;; Real odd-cols row #3

	vsubpd	ymm7, ymm7, ymm3		;; A2 = A2 - I2						; 93-95
	vmovapd	ymm5, YMM_TMPS[6*32]		;; Real even-cols row #3

	vaddpd	ymm6, ymm6, ymm8		;; B2 = B2 + R2						; 94-96
	vmulpd	ymm10, ymm10, ymm14		;; A12 = A12 * sine (final R12)				;  94-98
	vmovapd	ymm8, [screg]			;; sine for R2/I2

	vsubpd	ymm15, ymm2, ymm5		;; Real #11						; 95-97
	vmulpd	ymm12, ymm12, ymm14		;; B12 = B12 * sine (final I12)				;  95-99
	vmovapd	ymm14, [screg+9*64+32]		;; cosine/sine for R11/I11

	vsubpd	ymm3, ymm0, ymm9		;; Imag #11						; 96-98
	vmulpd	ymm7, ymm7, ymm8		;; A2 = A2 * sine (final R2)				;  96-100
	ystore	[srcreg+3*d2], ymm1		;; Save final R7					; 95

	vaddpd	ymm2, ymm2, ymm5		;; Real #3						; 97-99
	vmulpd	ymm6, ymm6, ymm8		;; B2 = B2 * sine (final I2)				;  97-101
	vmovapd	ymm1, YMM_TMPS[0*32]		;; Real odd-cols row #4

	vaddpd	ymm0, ymm0, ymm9		;; Imag #3						; 98-100
	vmulpd	ymm9, ymm15, ymm14		;; A11 = R11 * cosine/sine				;  98-102
	vmovapd	ymm5, YMM_TMPS[8*32]		;; Real even-cols row #4

	vsubpd	ymm8, ymm1, ymm5		;; Real #10						; 99-101
	vmulpd	ymm14, ymm3, ymm14		;; B11 = I11 * cosine/sine				;  99-103
	ystore	[srcreg+3*d2+32], ymm11		;; Save final I7					; 96

	vmovapd	ymm11, YMM_TMPS[10*32]		;; Imag odd-cols row #4
	ystore	[srcreg+5*d2+d1], ymm10		;; Save final R12					; 99
	vsubpd	ymm10, ymm4, ymm11		;; Imag #10						; 100-102
	ystore	[srcreg+5*d2+d1+32], ymm12	;; Save final I12					; 100
	vmovapd	ymm12, [screg+64+32]		;; cosine/sine for R3/I3
	ystore	[srcreg+d1], ymm7		;; Save final R2					; 101
	vmulpd	ymm7, ymm2, ymm12		;; A3 = R3 * cosine/sine				;  100-104

	vaddpd	ymm1, ymm1, ymm5		;; Real #4						; 101-103
	vmulpd	ymm12, ymm0, ymm12		;; B3 = I3 * cosine/sine				;  101-105
	vmovapd	ymm5, [screg+8*64+32]		;; cosine/sine for R10/I10

	vaddpd	ymm4, ymm4, ymm11		;; Imag #4						; 102-104
	vmulpd	ymm11, ymm8, ymm5		;; A10 = R10 * cosine/sine				;  102-106
	ystore	[srcreg+d1+32], ymm6		;; Save final I2					; 102

	vsubpd	ymm9, ymm9, ymm3		;; A11 = A11 - I11					; 103-105
	vmulpd	ymm5, ymm10, ymm5		;; B10 = I10 * cosine/sine				;  103-107
	vmovapd	ymm3, [screg+2*64+32]		;; cosine/sine for R4/I4

	vaddpd	ymm14, ymm14, ymm15		;; B11 = B11 + R11					; 104-106
	vmulpd	ymm15, ymm1, ymm3		;; A4 = R4 * cosine/sine				;  104-108
	vmovapd	ymm6, [screg+9*64]		;; sine for R11/I11

	vsubpd	ymm7, ymm7, ymm0		;; A3 = A3 - I3						; 105-107
	vmulpd	ymm3, ymm4, ymm3		;; B4 = I4 * cosine/sine				;  105-109
	vmovapd	ymm0, [screg+64]		;; sine for R3/I3

	vaddpd	ymm12, ymm12, ymm2		;; B3 = B3 + R3						; 106-108
	vmulpd	ymm9, ymm9, ymm6		;; A11 = A11 * sine (final R11)				;  106-110
	vmovapd ymm2, [screg+8*64]		;; sine for R10/I10

	vsubpd	ymm11, ymm11, ymm10		;; A10 = A10 - I10					; 107-109
	vmulpd	ymm14, ymm14, ymm6		;; B11 = B11 * sine (final I11)				;  107-111
	vmovapd	ymm10, [screg+2*64]		;; sine for R4/I4

	vaddpd	ymm5, ymm5, ymm8		;; B10 = B10 + R10					; 108-110
	vmulpd	ymm7, ymm7, ymm0		;; A3 = A3 * sine (final R3)				;  108-112
	vmovapd	ymm6, YMM_TMPS[5*32]		;; Real odd-cols row #5

	vsubpd	ymm15, ymm15, ymm4		;; A4 = A4 - I4						; 109-111
	vmulpd	ymm12, ymm12, ymm0		;; B3 = B3 * sine (final I3)				;  109-113
	vmovapd	ymm8, YMM_TMPS[7*32]		;; Real even-cols row #5

	vaddpd	ymm3, ymm3, ymm1		;; B4 = B4 + R4						; 110-112
	vmulpd	ymm11, ymm11, ymm2		;; A10 = A10 * sine (final R10)				;  110-114
	vmovapd	ymm0, YMM_TMPS[1*32]		;; Imag even-cols row #5

	vsubpd	ymm4, ymm6, ymm8		;; Real #9						; 111-113
	vmulpd	ymm5, ymm5, ymm2		;; B10 = B10 * sine (final I10)				;  111-115
	vmovapd	ymm1, YMM_TMPS[11*32]		;; Imag odd-cols row #5

	vsubpd	ymm2, ymm0, ymm1		;; Imag #9						; 112-114
	vmulpd	ymm15, ymm15, ymm10		;; A4 = A4 * sine (final R4)				;  112-116
	ystore	[srcreg+5*d2], ymm9		;; Save final R11					; 111
	vmovapd	ymm9, [screg+7*64+32]		;; cosine/sine for R9/I9

	vaddpd	ymm6, ymm6, ymm8		;; Real #5						; 113-115
	vmulpd	ymm3, ymm3, ymm10		;; B4 = B4 * sine (final I4)				;  113-117
	vmovapd	ymm10, YMM_TMPS[3*32]		;; Real odd-cols row #6

	vaddpd	ymm0, ymm0, ymm1		;; Imag #5						; 114-116
	vmulpd	ymm8, ymm4, ymm9		;; A9 = R9 * cosine/sine				;  114-118
	vmovapd	ymm1, YMM_TMPS[9*32]		;; Real even-cols row #6
	ystore	[srcreg+5*d2+32], ymm14		;; Save final I11					; 112

	vsubpd	ymm14, ymm10, ymm1		;; Real #8						; 115-117
	vmulpd	ymm9, ymm2, ymm9		;; B9 = I9 * cosine/sine				;  115-119
	ystore	[srcreg+d2], ymm7		;; Save final R3					; 113

	vmovapd	ymm7, YMM_TMPS[12*32]		;; Imag odd-cols row #6
	ystore	[srcreg+d2+32], ymm12		;; Save final I3					; 114
	vsubpd	ymm12, ymm13, ymm7		;; Imag #8						; 116-118
	ystore	[srcreg+4*d2+d1], ymm11		;; Save final R10					; 115
	vmovapd	ymm11, [screg+3*64+32]		;; cosine/sine for R5/I5
	ystore	[srcreg+4*d2+d1+32], ymm5	;; Save final I10					; 116
	vmulpd	ymm5, ymm6, ymm11		;; A5 = R5 * cosine/sine				;  116-120

	vaddpd	ymm10, ymm10, ymm1		;; Real #6						; 117-119
	vmulpd	ymm11, ymm0, ymm11		;; B5 = I5 * cosine/sine				;  117-121
	vmovapd	ymm1, [screg+6*64+32]		;; cosine/sine for R8/I8

	vaddpd	ymm13, ymm13, ymm7		;; Imag #6						; 118-120
	vmulpd	ymm7, ymm14, ymm1		;; A8 = R8 * cosine/sine				;  118-122
	ystore	[srcreg+d2+d1], ymm15		;; Save final R4					; 117

	vsubpd	ymm8, ymm8, ymm2		;; A9 = A9 - I9						; 119-121
	vmulpd	ymm1, ymm12, ymm1		;; B8 = I8 * cosine/sine				;  119-123
	vmovapd	ymm15, [screg+4*64+32]		;; cosine/sine for R6/I6

	vaddpd	ymm9, ymm9, ymm4		;; B9 = B9 + R9						; 120-122
	vmulpd	ymm2, ymm10, ymm15		;; A6 = R6 * cosine/sine				;  120-124
	vmovapd	ymm4, [screg+7*64]		;; sine for R9/I9

	vsubpd	ymm5, ymm5, ymm0		;; A5 = A5 - I5						; 121-123
	vmulpd	ymm15, ymm13, ymm15		;; B6 = I6 * cosine/sine				;  121-125
	vmovapd	ymm0, [screg+3*64]		;; sine for R5/I5

	vaddpd	ymm11, ymm11, ymm6		;; B5 = B5 + R5						; 122-124
	vmulpd	ymm8, ymm8, ymm4		;; A9 = A9 * sine (final R9)				;  122-126
	vmovapd	ymm6, [screg+6*64]		;; sine for R8/I8

	vsubpd	ymm7, ymm7, ymm12		;; A8 = A8 - I8						; 123-125
	vmulpd	ymm9, ymm9, ymm4		;; B9 = B9 * sine (final I9)				;  123-127
	vmovapd	ymm12, [screg+4*64]		;; sine for R6/I6

	vaddpd	ymm1, ymm1, ymm14		;; B8 = B8 + R8						; 124-126
	vmulpd	ymm5, ymm5, ymm0		;; A5 = A5 * sine (final R5)				;  124-128
	ystore	[srcreg+d2+d1+32], ymm3		;; Save final I4					; 118

	vsubpd	ymm2, ymm2, ymm13		;; A6 = A6 - I6						; 125-127
	vmulpd	ymm11, ymm11, ymm0		;; B5 = B5 * sine (final I5)				;  125-129

	vaddpd	ymm15, ymm15, ymm10		;; B6 = B6 + R6						; 126-128
	vmulpd	ymm7, ymm7, ymm6		;; A8 = A8 * sine (final R8)				;  126-130

	vmulpd	ymm1, ymm1, ymm6		;; B8 = B8 * sine (final I8)				;  127-131
	ystore	[srcreg+4*d2], ymm8		;; Save final R9					; 127

	vmulpd	ymm2, ymm2, ymm12		;; A6 = A6 * sine (final R6)				;  128-132
	ystore	[srcreg+4*d2+32], ymm9		;; Save final I9					; 128

	vmulpd	ymm15, ymm15, ymm12		;; B6 = B6 * sine (final I6)				;  129-133
	ystore	[srcreg+2*d2], ymm5		;; Save final R5					; 129

	ystore	[srcreg+2*d2+32], ymm11		;; Save final I5					; 130
	ystore	[srcreg+3*d2+d1], ymm7		;; Save final R8					; 131
	ystore	[srcreg+3*d2+d1+32], ymm1	;; Save final I8					; 132
	ystore	[srcreg+2*d2+d1], ymm2		;; Save final R6					; 133
	ystore	[srcreg+2*d2+d1+32], ymm15	;; Save final I6					; 134

	bump	srcreg, srcinc
	bump	screg, scinc
	ENDM

;; Haswell FMA3 version
;; This is the original Bulldozer version timed at 85.5 clocks.  Converting adds and subs to FMA3 did not improve the timings.

IF (@INSTR(,%yarch,<FMA3>) NE 0)

yr6_12cl_24_reals_fft_preload MACRO
	vmovapd	ymm15, YMM_ONE
	ENDM

yr6_12cl_24_reals_fft MACRO srcreg,srcinc,d1,d2,screg,scinc,maxrpt,L1pt,L1pd

	;; Do the odd columns for the real results

	vmovapd	ymm0, [srcreg+2*d2]		;; r5
	vaddpd	ymm0, ymm0, [srcreg+4*d2+32]	;; r5+r21						; 1-5		n 7
	vmovapd	ymm1, [srcreg+4*d2]		;; r9
	yfmaddpd ymm1, ymm1, ymm15, [srcreg+2*d2+32] ;; r9+r17						; 1-5		n 7

	vmovapd	ymm2, [srcreg+d2]		;; r3
	vaddpd	ymm2, ymm2, [srcreg+5*d2+32]	;; r3+r23						; 2-6		n 8
	vmovapd	ymm3, [srcreg+5*d2]		;; r11
	yfmaddpd ymm3, ymm3, ymm15, [srcreg+d2+32] ;; r11+r15						; 2-6		n 8

	vmovapd	ymm4, [srcreg+d1]		;; r2
	vaddpd	ymm4, ymm4, [srcreg+5*d2+d1+32]	;; r2+r24						; 3-7		n 9
	vmovapd	ymm5, [srcreg+5*d2+d1]		;; r12
	yfmaddpd ymm5, ymm5, ymm15, [srcreg+d1+32] ;; r12+r14						; 3-7		n 9

	vmovapd	ymm6, [srcreg+2*d2+d1]		;; r6
	vaddpd	ymm6, ymm6, [srcreg+3*d2+d1+32]	;; r6+r20						; 4-8		n 10
	vmovapd	ymm7, [srcreg+3*d2+d1]		;; r8
	yfmaddpd ymm7, ymm7, ymm15, [srcreg+2*d2+d1+32]	;; r8+r18					; 4-8		n 10

	vmovapd	ymm8, [srcreg+d2+d1]		;; r4
	vaddpd	ymm8, ymm8, [srcreg+4*d2+d1+32]	;; r4+r22						; 5-9		n 11
	vmovapd	ymm9, [srcreg+4*d2+d1]		;; r10
	yfmaddpd ymm9, ymm9, ymm15, [srcreg+d2+d1+32] ;; r10+r16					; 5-9		n 11

	vmovapd	ymm10, [srcreg]			;; r1
	vmovapd	ymm11, [srcreg+32]		;; r13
	vsubpd	ymm12, ymm10, ymm11		;; r1-r13						; 6-10		n 12
	yfmaddpd ymm10, ymm10, ymm15, ymm11	;; r1+r13						; 6-10		n 13

	vsubpd	ymm11, ymm0, ymm1		;; (r5+r21)-(r9+r17)					; 7-11		n 12
	yfmaddpd ymm0, ymm0, ymm15, ymm1	;; (r5+r21)+(r9+r17)					; 7-11		n 13

	vmovapd	ymm13, [srcreg+3*d2]		;; r7
	vaddpd	ymm13, ymm13, [srcreg+3*d2+32]	;; r7+r19						; 8-12		n 14
	yfmaddpd ymm1, ymm2, ymm15, ymm3	;; (r3+r23)+(r11+r15)					; 8-12		n 14

	vsubpd	ymm2, ymm2, ymm3		;; (r3+r23)-(r11+r15) (r26oB/.866)			; 9-13
	yfmaddpd ymm3, ymm4, ymm15, ymm5	;; (r2+r24)+(r12+r14)					; 9-13		n 15
	vmovapd ymm14, YMM_HALF

	vsubpd	ymm4, ymm4, ymm5		;; (r2+r24)-(r12+r14)					; 10-14		n 16
	yfmaddpd ymm5, ymm6, ymm15, ymm7	;; (r6+r20)+(r8+r18)					; 10-14		n 15
	L1prefetchw srcreg+L1pd, L1pt

	vsubpd	ymm6, ymm6, ymm7		;; (r6+r20)-(r8+r18)					; 11-15		n 21
	yfmsubpd ymm7, ymm8, ymm15, ymm9	;; (r4+r22)-(r10+r16)					; 11-15		n 16

	vaddpd	ymm8, ymm8, ymm9		;; (r4+r22)+(r10+r16)					; 12-16		n 32
	yfmaddpd ymm9, ymm14, ymm11, ymm12	;; r1-r13 + .5*((r5+r21)-(r9+r17)) (r26oA)		; 12-16
	L1prefetchw srcreg+d1+L1pd, L1pt

	vsubpd	ymm12, ymm12, ymm11		;; r1-r13 - ((r5+r21)-(r9+r17)) (r4o)			; 13-17
	yfmaddpd ymm11, ymm10, ymm15, ymm0	;; r1+r13 + ((r5+r21)+(r9+r17)) (r17a)			; 13-17		n 19

	yfnmaddpd ymm0, ymm14, ymm0, ymm10	;; r1+r13 - .5*((r5+r21)+(r9+r17)) (r35oA)		; 14-18		n 20
	vaddpd	ymm10, ymm1, ymm13		;; ((r3+r23)+(r11+r15)) + r7+r19 (r17b)			; 14-18		n 19
	ystore	YMM_TMPS[0*32], ymm2		;; r26oB / .866						; 14

	yfmsubpd ymm1, ymm14, ymm1, ymm13	;; .5*((r3+r23)+(r11+r15)) - (r7+r19) (r35oB)		; 15-19		n 20
	vsubpd	ymm13, ymm3, ymm5		;; ((r2+r24)+(r12+r14)) - ((r6+r20)+(r8+r18)) (r3e/.866) ; 15-19
	vmovapd	ymm2, YMM_P966_P707

	vaddpd	ymm3, ymm3, ymm5		;; ((r2+r24)+(r12+r14)) + ((r6+r20)+(r8+r18)) (r15eA)	; 16-20		n 32
	yfmsubpd ymm5, ymm4, ymm15, ymm7	;; ((r2+r24)-(r12+r14))-((r4+r22)-(r10+r16))		; 16-20		n 21

	yfmaddpd ymm14, ymm2, ymm4, ymm7	;; .966/.707*((r2+r24)-(r12+r14))+((r4+r22)-(r10+r16))	; 17-21		n 22
	ystore	YMM_TMPS[1*32], ymm9		;; r26oA						; 17
	vmovapd	ymm9, YMM_P259_P707
	yfmsubpd ymm4, ymm9, ymm4, ymm7		;; .259/.707*((r2+r24)-(r12+r14))-((r4+r22)-(r10+r16))	; 17-21		n 22

	vmovapd	ymm7, [srcreg+d1]		;; r2
	vsubpd	ymm7, ymm7, [srcreg+5*d2+d1+32]	;; r2-r24						; 18-22		n 27
	ystore	YMM_TMPS[2*32], ymm12		;; Real odd-cols row #4					; 18
	vmovapd	ymm12, [srcreg+5*d2+d1]		;; r12
	yfmsubpd ymm12, ymm12, ymm15, [srcreg+d1+32] ;; r12-r14						; 18-22		n 27

	ystore	YMM_TMPS[3*32], ymm13		;; Real even-cols row #3 / .866				; 20
	vaddpd	ymm13, ymm11, ymm10		;; r17a + r17b (r1a)					; 19-23
	yfmsubpd ymm11, ymm11, ymm15, ymm10	;; r17a - r17b (r7)					; 19-23

	vaddpd	ymm10, ymm0, ymm1		;; r35oA + r35oB (r3o)					; 20-24
	yfmsubpd ymm0, ymm0, ymm15, ymm1	;; r35oA - r35oB (r5o)					; 20-24

	vmovapd	ymm1, [srcreg+2*d2+d1]		;; r6
	vsubpd	ymm1, ymm1, [srcreg+3*d2+d1+32]	;; r6-r20						; 21-25		n 28
	yfmsubpd ymm5, ymm5, ymm15, ymm6	;; ((r2+r24)-(r12+r14))-((r4+r22)-(r10+r16))-((r6+r20)-(r8+r18)) (r4e/.707) ; 21-25

	yfmaddpd ymm14, ymm9, ymm6, ymm14	;; .966/.707*((r2+r24)-(r12+r14))+((r4+r22)-(r10+r16))+.259/.707*((r6+r20)-(r8+r18)) (r2e/.707) ; 22-26
	vmovapd	ymm9, [srcreg+3*d2+d1]		;; r8
	vsubpd	ymm9, ymm9, [srcreg+2*d2+d1+32] ;; r8-r18						; 22-26		n 28

	yfmaddpd ymm4, ymm2, ymm6, ymm4		;; .259/.707*((r2+r24)-(r12+r14))-((r4+r22)-(r10+r16))+.966/.707*((r6+r20)-(r8+r18)) (r6e/.707) ; 23-27
	;; There is room for a 1/2 clock stall here.  We'll probably use it as the next few clocks each require 4 loads.

	vmovapd	ymm2, [srcreg+d2+d1]		;; r4
	vsubpd	ymm2, ymm2, [srcreg+4*d2+d1+32]	;; r4-r22						; 24-28		n 29
	vmovapd	ymm6, [srcreg+4*d2+d1]		;; r10
	yfmsubpd ymm6, ymm6, ymm15, [srcreg+d2+d1+32] ;; r10-r16					; 24-28		n 29
	ystore	[srcreg], ymm13			;; Final real #1A					; 24

	vmovapd	ymm13, [srcreg+d2]		;; r3
	vsubpd	ymm13, ymm13, [srcreg+5*d2+32]	;; r3-r23						; 25-29		n 30
	ystore	YMM_TMPS[4*32], ymm11		;; Real row #7						; 24+1
	vmovapd	ymm11, [srcreg+5*d2]		;; r11
	yfmsubpd ymm11, ymm11, ymm15, [srcreg+d2+32] ;; r11-r15						; 25-29		n 30

	ystore	YMM_TMPS[5*32], ymm10		;; Real odd-cols row #3					; 25+1
	vmovapd	ymm10, [srcreg+2*d2]		;; r5
	vsubpd	ymm10, ymm10, [srcreg+4*d2+32]	;; r5-r21						; 26-30		n 31
	ystore	YMM_TMPS[6*32], ymm0		;; Real odd-cols row #5					; 25+2
	vmovapd	ymm0, [srcreg+4*d2]		;; r9
	yfmsubpd ymm0, ymm0, ymm15, [srcreg+2*d2+32] ;; r9-r17						; 26-30		n 31

	ystore	YMM_TMPS[7*32], ymm5		;; Real even-cols row #4 / .707				; 26+2
	vsubpd	ymm5, ymm7, ymm12		;; (r2-r24)-(r12-r14)					; 27-31		n 33
	yfmaddpd ymm7, ymm7, ymm15, ymm12	;; (r2-r24)+(r12-r14)					; 27-31		n 34
	L1prefetchw srcreg+d2+L1pd, L1pt

	vsubpd	ymm12, ymm1, ymm9		;; (r6-r20)-(r8-r18)					; 28-32		n 33
	yfmaddpd ymm1, ymm1, ymm15, ymm9	;; (r6-r20)+(r8-r18)					; 28-32		n 39

	vaddpd	ymm9, ymm2, ymm6		;; (r4-r22)+(r10-r16)					; 29-33		n 34
	yfmsubpd ymm2, ymm2, ymm15, ymm6	;; (r4-r22)-(r10-r16)					; 29-33		n 38
	vmovapd	ymm6, [srcreg+3*d2]		;; r7
	ystore	YMM_TMPS[8*32], ymm14		;; Real even-cols row #2 / .707				; 27+2

	vsubpd	ymm6, ymm6, [srcreg+3*d2+32]	;; r7-r19						; 30-34		n 35
	yfmaddpd ymm14, ymm13, ymm15, ymm11	;; (r3-r23)+(r11-r15)					; 30-34		n 35
	ystore	YMM_TMPS[9*32], ymm4		;; Real even-cols row #6 / .707				; 27+3

	vsubpd	ymm13, ymm13, ymm11		;; (r3-r23)-(r11-r15)					; 31-35		n 37
	yfmsubpd ymm11, ymm10, ymm15, ymm0	;; (r5-r21)-(r9-r17)					; 31-35		n 37
	vmovapd	ymm4, YMM_HALF

	vaddpd	ymm10, ymm10, ymm0		;; (r5-r21)+(r9-r17)					; 32-36		n 40
	yfmaddpd ymm0, ymm3, ymm15, ymm8	;; r15eA + ((r4+r22)+(r10+r16)) (r1b)			; 32-36
	L1prefetchw srcreg+d2+d1+L1pd, L1pt

	yfmsubpd ymm3, ymm4, ymm3, ymm8		;; .5*r15eA - ((r4+r22)+(r10+r16)) (r5e)		; 33-37		n 44
	vaddpd	ymm8, ymm5, ymm12		;; ((r2-r24)-(r12-r14)) + ((r6-r20)-(r8-r18)) (i37eA)	; 33-37		n 38

	vsubpd	ymm5, ymm5, ymm12		;; ((r2-r24)-(r12-r14)) - ((r6-r20)-(r8-r18)) (i5e)	; 34-38		n 39
	yfmaddpd ymm12, ymm7, ymm15, ymm9	;; ((r2-r24)+(r12-r14))+((r4-r22)+(r10-r16))		; 34-38		n 39
	L1prefetchw srcreg+2*d2+L1pd, L1pt

	ystore	[srcreg+32], ymm0		;; Save final real #1B (real even-cols row #1)		; 37
	yfmaddpd ymm0, ymm4, ymm14, ymm6	;; .5*((r3-r23)+(r11-r15)) + (r7-r19) (i26oA)		; 35-39		n 40
	vsubpd	ymm14, ymm14, ymm6		;; ((r3-r23)+(r11-r15)) - (r7-r19) (i4o)		; 35-39

	yfmaddpd ymm6, ymm7, YMM_P259_P707, ymm9 ;; .259/.707*((r2-r24)+(r12-r14))+((r4-r22)+(r10-r16)) ; 36-40		n 41
	ystore	YMM_TMPS[10*32], ymm14		;; Imag odd-cols row #4					; 40
	vmovapd	ymm14, YMM_P966_P707
	yfmsubpd ymm7, ymm14, ymm7, ymm9	;; .966/.707*((r2-r24)+(r12-r14))-((r4-r22)+(r10-r16))	; 36-40		n 41

	vaddpd	ymm9, ymm13, ymm11		;; ((r3-r23)-(r11-r15)) + ((r5-r21)-(r9-r17)) (i3o/.866) ; 37-41	n 43
	yfmsubpd ymm13, ymm13, ymm15, ymm11	;; ((r3-r23)-(r11-r15)) - ((r5-r21)-(r9-r17)) (i5o/.866) ; 37-41	n 45
	L1prefetchw srcreg+2*d2+d1+L1pd, L1pt

	yfmaddpd ymm11, ymm4, ymm8, ymm2	;; .5*i37eA + ((r4-r22)-(r10-r16)) (i3e)		; 38-42		n 43
	vsubpd	ymm8, ymm8, ymm2		;; i37eA - ((r4-r22)-(r10-r16)) (i7)			; 38-42		n 46

	vmovapd	ymm4, YMM_P866
	vmulpd	ymm5, ymm4, ymm5		;; .866*i5e						; 39-43		n 45
	vsubpd	ymm12, ymm12, ymm1		;; ((r2-r24)+(r12-r14))+((r4-r22)+(r10-r16))-((r6-r20)+(r8-r18)) (i4e) ; 39-43

	yfmaddpd ymm2, ymm4, ymm10, ymm0	;; i26oA + .866*((r5-r21)+(r9-r17)) (i2o)		; 40-44
	yfnmaddpd ymm0, ymm4, ymm10, ymm0	;; i26oA - .866*((r5-r21)+(r9-r17)) (i6o)		; 40-44

	yfmaddpd ymm6, ymm14, ymm1, ymm6	;; .259/.707*((r2-r24)+(r12-r14))+((r4-r22)+(r10-r16))+.966/.707*((r6-r20)+(r8-r18)) (i2e/.707) ; 41-45
	yfmaddpd ymm7, ymm1, YMM_P259_P707, ymm7 ;; .966/.707*((r2-r24)+(r12-r14))-((r4-r22)+(r10-r16))+.259/.707*((r6-r20)+(r8-r18)) (i6e/.707) ; 41-45

	vmovapd	ymm10, YMM_TMPS[5*32]		;; Real odd-cols row #3
	vmovapd	ymm14, YMM_TMPS[3*32]		;; Real even-cols row #3 / .866
	yfnmaddpd ymm1, ymm4, ymm14, ymm10	;; (ro3 - .866*re3) Real #11				; 42-46
	yfmaddpd ymm14, ymm4, ymm14, ymm10	;; (ro3 + .866*re3) Real #3				; 42-46

	yfnmaddpd ymm10, ymm4, ymm9, ymm11	;; (i3e - .866*i3o) Imag #11				; 43-47
	yfmaddpd ymm9, ymm4, ymm9, ymm11	;; (i3e + .866*i3o) Imag #3				; 43-47

	vmovapd	ymm11, YMM_TMPS[6*32]		;; Real odd-cols row #5
	ystore	YMM_TMPS[3*32], ymm12		;; Imag even-cols row #4				; 44
	vsubpd	ymm12, ymm11, ymm3		;; (ro5 - re5) Real #9					; 44-48
	yfmaddpd ymm11, ymm11, ymm15, ymm3	;; (ro5 + re5) Real #5					; 44-48

	yfnmaddpd ymm3, ymm4, ymm13, ymm5	;; (ie5 - .866*io5) Imag #9				; 45-49
	yfmaddpd ymm13, ymm4, ymm13, ymm5	;; (ie5 + .866*io5) Imag #5				; 45-49
	vmovapd	ymm5, YMM_TMPS[4*32]		;; Real #7
	ystore	YMM_TMPS[4*32], ymm2		;; Imag odd-cols row #2					; 45

	vmovapd	ymm2, [screg+5*64+32]		;; cosine/sine for R7/I7
	ystore	YMM_TMPS[5*32], ymm0		;; Imag odd-cols row #6					; 45+1
	yfmsubpd ymm0, ymm5, ymm2, ymm8		;; A7 = R7 * cosine/sine - I7				; 46-50
	yfmaddpd ymm8, ymm8, ymm2, ymm5		;; B7 = I7 * cosine/sine + R7				; 46-50

	vmovapd	ymm2, YMM_TMPS[1*32]		;; r26oA
	vmovapd	ymm5, YMM_TMPS[0*32]		;; r26oB / .866
	ystore	YMM_TMPS[0*32], ymm6		;; Imag even-cols row #2 / .707				; 46+1
	yfmaddpd ymm6, ymm4, ymm5, ymm2		;; r26oA + .866*r26oB (r2o)				; 47-51
	yfnmaddpd ymm4, ymm4, ymm5, ymm2	;; r26oA - .866*r26oB (r6o)				; 47-51

	vmovapd	ymm2, [screg+9*64+32]		;; cosine/sine for R11/I11
	yfmsubpd ymm5, ymm1, ymm2, ymm10	;; A11 = R11 * cosine/sine - I11			; 48-52
	yfmaddpd ymm10, ymm10, ymm2, ymm1	;; B11 = I11 * cosine/sine + R11			; 48-52
	ystore	YMM_TMPS[1*32], ymm7		;; Imag even-cols row #6 / .707				; 46+2

	vmovapd	ymm2, [screg+64+32]		;; cosine/sine for R3/I3
	yfmsubpd ymm1, ymm14, ymm2, ymm9	;; A3 = R3 * cosine/sine - I3				; 49-53
	yfmaddpd ymm9, ymm9, ymm2, ymm14	;; B3 = I3 * cosine/sine + R3				; 49-53

	vmovapd	ymm2, [screg+7*64+32]		;; cosine/sine for R9/I9
	yfmsubpd ymm14, ymm12, ymm2, ymm3	;; A9 = R9 * cosine/sine - I9				; 50-54
	yfmaddpd ymm3, ymm3, ymm2, ymm12	;; B9 = I9 * cosine/sine + R9				; 50-54
	L1prefetchw srcreg+3*d2+L1pd, L1pt

	vmovapd	ymm2, [screg+3*64+32]		;; cosine/sine for R5/I5
	yfmsubpd ymm12, ymm11, ymm2, ymm13	;; A5 = R5 * cosine/sine - I5				; 51-55
	yfmaddpd ymm13, ymm13, ymm2, ymm11	;; B5 = I5 * cosine/sine + R5				; 51-55

	vmovapd	ymm2, [screg+5*64]		;; sine for R7/I7
	vmulpd	ymm0, ymm0, ymm2		;; A7 = A7 * sine (final R7)				; 52-56
	vmulpd	ymm8, ymm8, ymm2		;; B7 = B7 * sine (final I7)				; 52-56

	vmovapd	ymm2, [screg+9*64]		;; sine for R11/I11
	vmulpd	ymm5, ymm5, ymm2		;; A11 = A11 * sine (final R11)				; 53-57
	vmulpd	ymm10, ymm10, ymm2		;; B11 = B11 * sine (final I11)				; 53-57
	L1prefetchw srcreg+3*d2+d1+L1pd, L1pt

	vmovapd	ymm2, [screg+64]		;; sine for R3/I3
	vmulpd	ymm1, ymm1, ymm2		;; A3 = A3 * sine (final R3)				; 54-58
	vmulpd	ymm9, ymm9, ymm2		;; B3 = B3 * sine (final I3)				; 54-58

	vmovapd	ymm2, [screg+7*64]		;; sine for R9/I9
	vmulpd	ymm14, ymm14, ymm2		;; A9 = A9 * sine (final R9)				; 55-59
	vmulpd	ymm3, ymm3, ymm2		;; B9 = B9 * sine (final I9)				; 55-59

	vmovapd	ymm2, [screg+3*64]		;; sine for R5/I5
	vmulpd	ymm12, ymm12, ymm2		;; A5 = A5 * sine (final R5)				; 56-60
	vmulpd	ymm13, ymm13, ymm2		;; B5 = B5 * sine (final I5)				; 56-60

	vmovapd	ymm11, YMM_SQRTHALF
	vmovapd	ymm7, YMM_TMPS[2*32]		;; Real odd-cols row #4
	vmovapd	ymm2, YMM_TMPS[7*32]		;; Real even-cols row #4 / .707
	ystore	[srcreg+3*d2], ymm0		;; Save final R7					; 57
	yfnmaddpd ymm0, ymm11, ymm2, ymm7	;; (ro4 - .707*re4) Real #10				; 57-61
	yfmaddpd ymm2, ymm11, ymm2, ymm7	;; (ro4 + .707*re4) Real #4				; 57-61

	vmovapd	ymm7, YMM_TMPS[3*32]		;; Imag even-cols row #4 / .707
	ystore	[srcreg+3*d2+32], ymm8		;; Save final I7					; 57+1
	vmovapd	ymm8, YMM_TMPS[10*32]		;; Imag odd-cols row #4
	ystore	[srcreg+5*d2], ymm5		;; Save final R11					; 58+1
	yfmsubpd ymm5, ymm11, ymm7, ymm8	;; (.707*i4e - i4o) Imag #10				; 58-62
	yfmaddpd ymm7, ymm11, ymm7, ymm8	;; (.707*i4e + i4o) Imag #4				; 58-62

	vmovapd	ymm8, YMM_TMPS[8*32]		;; Real even-cols row #2
	ystore	[srcreg+5*d2+32], ymm10		;; Save final I11					; 58+2
	yfnmaddpd ymm10, ymm11, ymm8, ymm6	;; (ro2 - .707*re2) Real #12				; 59-63
	yfmaddpd ymm8, ymm11, ymm8, ymm6	;; (ro2 + .707*re2) Real #2				; 59-63

	vmovapd	ymm6, YMM_TMPS[0*32]		;; Imag even-cols row #2 / .707
	ystore	[srcreg+d2], ymm1		;; Save final R3					; 59+2
	vmovapd	ymm1, YMM_TMPS[4*32]		;; Imag odd-cols row #2
	ystore	[srcreg+d2+32], ymm9		;; Save final I3					; 59+3
	yfmsubpd ymm9, ymm11, ymm6, ymm1	;; (.707*ie2 - io2) Imag #12				; 60-64
	yfmaddpd ymm6, ymm11, ymm6, ymm1	;; (.707*ie2 + io2) Imag #2				; 60-64

	vmovapd	ymm1, YMM_TMPS[9*32]		;; Real even-cols row #6 / .707
	ystore	[srcreg+4*d2], ymm14		;; Save final R9					; 60+3
	yfnmaddpd ymm14, ymm11, ymm1, ymm4	;; (ro6 - .707*re6) Real #8				; 61-65
	yfmaddpd ymm1, ymm11, ymm1, ymm4	;; (ro6 + .707*re6) Real #6				; 61-65

	vmovapd	ymm4, YMM_TMPS[1*32]		;; Imag even-cols row #6 / .707
	ystore	[srcreg+4*d2+32], ymm3		;; Save final I9					; 60+4
	vmovapd	ymm3, YMM_TMPS[5*32]		;; Imag odd-cols row #6
	ystore	[srcreg+2*d2], ymm12		;; Save final R5					; 61+4
	yfmsubpd ymm12, ymm11, ymm4, ymm3	;; (.707*ie6 - io6) Imag #8				; 62-66
	yfmaddpd ymm4, ymm11, ymm4, ymm3	;; (.707*ie6 + io6) Imag #6				; 62-66

	vmovapd	ymm3, [screg+8*64+32]		;; cosine/sine for R10/I10
	yfmsubpd ymm11, ymm0, ymm3, ymm5	;; A10 = R10 * cosine/sine - I10			; 63-67
	yfmaddpd ymm5, ymm5, ymm3, ymm0		;; B10 = I10 * cosine/sine + R10			; 63-67
	L1prefetchw srcreg+4*d2+L1pd, L1pt

	vmovapd	ymm3, [screg+2*64+32]		;; cosine/sine for R4/I4
	yfmsubpd ymm0, ymm2, ymm3, ymm7		;; A4 = R4 * cosine/sine - I4				; 64-68
	yfmaddpd ymm7, ymm7, ymm3, ymm2		;; B4 = I4 * cosine/sine + R4				; 64-68

	vmovapd	ymm3, [screg+10*64+32]		;; cosine/sine for R12/I12
	yfmsubpd ymm2, ymm10, ymm3, ymm9	;; A12 = R12 * cosine/sine - I12			; 65-69
	yfmaddpd ymm9, ymm9, ymm3, ymm10	;; B12 = I12 * cosine/sine + R12			; 65-69
	L1prefetchw srcreg+4*d2+d1+L1pd, L1pt

	vmovapd	ymm3, [screg+32]		;; cosine/sine for R2/I2
	yfmsubpd ymm10, ymm8, ymm3, ymm6	;; A2 = R2 * cosine/sine - I2				; 66-70
	yfmaddpd ymm6, ymm6, ymm3, ymm8		;; B2 = I2 * cosine/sine + R2				; 66-70
	ystore	[srcreg+2*d2+32], ymm13		;; Save final I5					; 61+5

	vmovapd	ymm3, [screg+6*64+32]		;; cosine/sine for R8/I8
	yfmsubpd ymm8, ymm14, ymm3, ymm12	;; A8 = R8 * cosine/sine - I8				; 67-71
	yfmaddpd ymm12, ymm12, ymm3, ymm14	;; B8 = I8 * cosine/sine + R8				; 67-71

	vmovapd	ymm3, [screg+4*64+32]		;; cosine/sine for R6/I6
	yfmsubpd ymm14, ymm1, ymm3, ymm4	;; A6 = R6 * cosine/sine - I6				; 68-72
	yfmaddpd ymm4, ymm4, ymm3, ymm1		;; B6 = I6 * cosine/sine + R6				; 68-72
	L1prefetchw srcreg+5*d2+L1pd, L1pt

	vmovapd ymm3, [screg+8*64]		;; sine for R10/I10
	vmulpd	ymm11, ymm11, ymm3		;; A10 = A10 * sine (final R10)				; 69-73
	vmulpd	ymm5, ymm5, ymm3		;; B10 = B10 * sine (final I10)				; 69-73

	vmovapd	ymm3, [screg+2*64]		;; sine for R4/I4
	vmulpd	ymm0, ymm0, ymm3		;; A4 = A4 * sine (final R4)				; 70-74
	vmulpd	ymm7, ymm7, ymm3		;; B4 = B4 * sine (final I4)				; 70-74
	L1prefetchw srcreg+5*d2+d1+L1pd, L1pt

	vmovapd	ymm3, [screg+10*64]		;; sine for R12/I12
	vmulpd	ymm2, ymm2, ymm3		;; A12 = A12 * sine (final R12)				; 71-75
	vmulpd	ymm9, ymm9, ymm3		;; B12 = B12 * sine (final I12)				; 71-75

	vmovapd	ymm3, [screg]			;; sine for R2/I2
	vmulpd	ymm10, ymm10, ymm3		;; A2 = A2 * sine (final R2)				; 72-76
	vmulpd	ymm6, ymm6, ymm3		;; B2 = B2 * sine (final I2)				; 72-76

	vmovapd	ymm3, [screg+6*64]		;; sine for R8/I8
	vmulpd	ymm8, ymm8, ymm3		;; A8 = A8 * sine (final R8)				; 73-77
	vmulpd	ymm12, ymm12, ymm3		;; B8 = B8 * sine (final I8)				; 73-77

	vmovapd	ymm3, [screg+4*64]		;; sine for R6/I6
	vmulpd	ymm14, ymm14, ymm3		;; A6 = A6 * sine (final R6)				; 74-78
	vmulpd	ymm4, ymm4, ymm3		;; B6 = B6 * sine (final I6)				; 74-78

	ystore	[srcreg+4*d2+d1], ymm11		;; Save final R10					; 74
	ystore	[srcreg+4*d2+d1+32], ymm5	;; Save final I10					; 74+1
	ystore	[srcreg+d2+d1], ymm0		;; Save final R4					; 75+1
	ystore	[srcreg+d2+d1+32], ymm7		;; Save final I4					; 75+2
	ystore	[srcreg+5*d2+d1], ymm2		;; Save final R12					; 76+2
	ystore	[srcreg+5*d2+d1+32], ymm9	;; Save final I12					; 76+3
	ystore	[srcreg+d1], ymm10		;; Save final R2					; 77+3
	ystore	[srcreg+d1+32], ymm6		;; Save final I2					; 77+4
	ystore	[srcreg+3*d2+d1], ymm8		;; Save final R8					; 78+4
	ystore	[srcreg+3*d2+d1+32], ymm12	;; Save final I8					; 78+5
	ystore	[srcreg+2*d2+d1], ymm14		;; Save final R6					; 79+5
	ystore	[srcreg+2*d2+d1+32], ymm4	;; Save final I6					; 79+6

	bump	srcreg, srcinc
	bump	screg, scinc
	ENDM

ENDIF

ENDIF

;;
;; ************************************* 24-reals-last-unfft variants ******************************************
;;

;; These macros produce 24 reals after doing 4.585 levels of the inverse FFT applying
;; the sin/cos multipliers beforehand.  The input is 2 real and 11 complex numbers.

;; To calculate a 24-reals inverse FFT, we calculate 24 real values from 24 complex inputs in a brute force way.
;; First we note that the 24 complex values are computed from the 11 complex and 2 real inputs using Hermetian symmetry, thusly:
;; c1 = r1A + 0*i
;; c2 = r2 + i2*i
;; ...
;; c12 = r12 + i12*i
;; c13 = r1B + 0*i
;; c14 = r12 - i12*i
;; ...
;; c24 = r2 - i2*i 
;;
;; The brute force calculations are:
;;
;; c1 + c2 + ... + c24	*  w^-0000000000...
;; c1 + c2 + ... + c24	*  w^-0123456789A...
;; c1 + c2 + ... + c24	*  w^-02468ACE....
;;    ...
;; c1 + c2 + ... + c24	*  w^-...A987654321
;;
;; The sin/cos values (w = 24th root of unity) are:
;; w^-1 = .966 - .259i
;; w^-2 = .866 - .5i
;; w^-3 = .707 - .707i
;; w^-4 = .5 - .866i
;; w^-5 = .259 - .966i
;; w^-6 = 0 - 1i
;; w^-7 = -.259 - .966i
;; w^-8 = -.5 - .866i
;; w^-9 = -.707 - .707i
;; w^-10 = -.866 - .5i
;; w^-11 = -.966 - .259i
;; w^-12 = -1

;;
;; Applying the sin/cos values above, taking advantage of symmetry, and ignoring a lot of multiplies by 2:
;; r1     +(r2+r12)     +(r3+r11)     +(r4+r10)     +(r5+r9)     +(r6+r8) + r7 + r13
;; r1 +.966(r2-r12) +.866(r3-r11) +.707(r4-r10) +.500(r5-r9) +.259(r6-r8)      - r13 +.259(i2+i12) +.500(i3+i11) +.707(i4+i10) +.866(i5+i9) +.966(i6+i8) + i7
;; r1 +.866(r2+r12) +.500(r3+r11)               -.500(r5+r9) -.866(r6+r8) - r7 + r13 +.500(i2-i12) +.866(i3-i11)     +(i4-i10) +.866(i5-i9) +.500(i6-i8)
;; r1 +.707(r2-r12)               -.707(r4-r10)     -(r5-r9) -.707(r6-r8)      - r13 +.707(i2+i12)     +(i3+i11) +.707(i4+i10)              -.707(i6+i8) - i7
;; r1 +.500(r2+r12) -.500(r3+r11)     -(r4+r10) -.500(r5+r9) +.500(r6+r8) + r7 + r13 +.866(i2-i12) +.866(i3-i11)               -.866(i5-i9) -.866(i6-i8)
;; r1 +.259(r2-r12) -.866(r3-r11) -.707(r4-r10) +.500(r5-r9) +.966(r6-r8)      - r13 +.966(i2+i12) +.500(i3+i11) -.707(i4+i10) -.866(i5+i9) +.259(i6+i8) + i7
;; r1                   -(r3+r11)                   +(r5+r9)              - r7 + r13     +(i2-i12)                   -(i4-i10)                  +(i6-i8)
;; r1 -.259(r2-r12) -.866(r3-r11) +.707(r4-r10) +.500(r5-r9) -.966(r6-r8)      - r13 +.966(i2+i12) -.500(i3+i11) -.707(i4+i10) +.866(i5+i9) +.259(i6+i8) - i7
;; r1 -.500(r2+r12) -.500(r3+r11)     +(r4+r10) -.500(r5+r9) -.500(r6+r8) + r7 + r13 +.866(i2-i12) -.866(i3-i11)               +.866(i5-i9) -.866(i6-i8)
;; r1 -.707(r2-r12)               +.707(r4-r10)     -(r5-r9) +.707(r6-r8)      - r13 +.707(i2+i12)     -(i3+i11) +.707(i4+i10)              -.707(i6+i8) + i7
;; r1 -.866(r2+r12) +.500(r3+r11)               -.500(r5+r9) +.866(r6+r8) - r7 + r13 +.500(i2-i12) -.866(i3-i11)     +(i4-i10) -.866(i5-i9) +.500(i6-i8)
;; r1 -.966(r2-r12) +.866(r3-r11) -.707(r4-r10) +.500(r5-r9) -.259(r6-r8)      - r13 +.259(i2+i12) -.500(i3+i11) +.707(i4+i10) -.866(i5+i9) +.966(i6+i8) - i7
;; r1     -(r2+r12)     +(r3+r11)     -(r4+r10)     +(r5+r9)     -(r6+r8) + r7 + r13
;; ... r14 thru r24 are the same as r12 through r2 but with the sign of the imaginary component changed.
;;
;; Also remember that due to the funny way we do things reals input r1A = r1+r13 and r1B = r1-13

;; Simplifying yields:
;;
;; r17oA = r1+r13     +(r5+r9)
;; r26oA = r1-r13 +.500(r5-r9)
;; r35oA = r1+r13 -.500(r5+r9)
;; r4o   = r1-r13     -(r5-r9)
;;
;; r17oB =     +(r3+r11) + r7
;; r26oB = +.866(r3-r11)
;; r35oB = +.500(r3+r11) - r7
;;
;; r1/7o = r17oA +/- r17oB
;; r2/6o = r26oA +/- r26oB
;; r3/5o = r35oA +/- r35oB
;;
;; r1e =     +(r2+r12)     +(r4+r10)     +(r6+r8)
;; r2e = +.966(r2-r12) +.707(r4-r10) +.259(r6-r8)
;; r3e =     +(r2+r12)                   -(r6+r8)
;; r4e =     +(r2-r12)     -(r4-r10)     -(r6-r8)
;; r5e = +.500(r2+r12)     -(r4+r10) +.500(r6+r8)
;; r6e = +.259(r2-r12) -.707(r4-r10) +.966(r6-r8)
;;
;; r1/13 = r1o +/- r1e
;; r2/12 = r2o +/- r2e
;; r3/11 = r3o +/- .866*r3e
;; r4/10 = r4o +/- .707*r4e
;; r5/9 = r5o +/- r5e
;; r6/8 = r6o +/- r6e
;;
;; i2e = +.259(i2+i12) +.707(i4+i10) +.966(i6+i8)
;; i3e = +.500(i2-i12)     +(i4-i10) +.500(i6-i8)
;; i4e =     +(i2+i12)     +(i4+i10)     -(i6+i8)
;; i5e =     +(i2-i12)                   -(i6-i8)
;; i6e = +.966(i2+i12) -.707(i4+i10) +.259(i6+i8)
;; i7  =     +(i2-i12)     -(i4-i10)     +(i6-i8)
;;
;; i2o = +.500(i3+i11) +.866(i5+i9) + i7
;; i3o =     +(i3-i11)     +(i5-i9)
;; i4o =     +(i3+i11)              - i7
;; i5o =     +(i3-i11)     -(i5-i9)
;; i6o = +.500(i3+i11) -.866(i5+i9) + i7
;;
;; i2/12 = i2e +- i2o
;; i3/11 = i3e +- .866*i3o
;; i4/10 = .707*i4e +- i4o
;; i5/9  = i5e +- i5o
;; i6/8  = i6e +- i6o
;;
;; r2/24 = r2 +- i2
;; r3/23 = r3 +- i3
;; r4/22 = r4 +- i4
;; r5/21 = r5 +- .866*i5
;; r6/20 = r6 +- i6
;; r7/19 = r7 +- i7
;; r8/18 = r8 +- i8
;; r9/17 = r9 +- .866*i9
;; r10/16 = r10 +- i10
;; r11/15 = r11 +- i11
;; r12/14 = r12 +- i12

yr6_12cl_24_reals_unfft_preload MACRO
	ENDM

yr6_12cl_24_reals_unfft MACRO srcreg,srcinc,d1,d2,screg,scinc,maxrpt,L1pt,L1pd

;; Apply the 11 twiddle factors.  Also do the first +/- on the reals so that
;; we have a perfect balance of adds and multiplies.

	vmovapd	ymm0, [screg+5*64+32]		;; cosine/sine for R7/I7
	vmovapd	ymm1, [srcreg+3*d2]		;; R7
	vmulpd	ymm2, ymm1, ymm0		;; A7 = R7 * cosine/sine
	vmovapd	ymm3, [srcreg+3*d2+32]		;; I7
	vaddpd	ymm2, ymm2, ymm3		;; A7 = A7 + I7
	vmulpd	ymm3, ymm3, ymm0		;; B7 = I7 * cosine/sine
	vsubpd	ymm3, ymm3, ymm1		;; B7 = B7 - R7
	vmovapd	ymm0, [screg+5*64]		;; sine for R7/I7
	vmulpd	ymm2, ymm2, ymm0		;; R7 = A7 * sine
	vmulpd	ymm3, ymm3, ymm0		;; I7 = B7 * sine
	ystore	YMM_TMPS[0*32], ymm2		;; Save R7
	ystore	YMM_TMPS[1*32], ymm3		;; Save I7

	vmovapd	ymm0, [screg+32]		;; cosine/sine for R2/I2
	vmovapd	ymm1, [srcreg+d1]		;; R2
	vmulpd	ymm2, ymm1, ymm0		;; A2 = R2 * cosine/sine
	vmovapd	ymm4, [screg+10*64+32]		;; cosine/sine for R12/I12
	vmovapd	ymm5, [srcreg+5*d2+d1]		;; R12
	vmulpd	ymm6, ymm5, ymm4		;; A12 = R12 * cosine/sine
	vmovapd	ymm3, [srcreg+d1+32]		;; I2
	vaddpd	ymm2, ymm2, ymm3		;; A2 = A2 + I2
	vmulpd	ymm3, ymm3, ymm0		;; B2 = I2 * cosine/sine
	vmovapd	ymm7, [srcreg+5*d2+d1+32]	;; I12
	vaddpd	ymm6, ymm6, ymm7		;; A12 = A12 + I12
	vmulpd	ymm7, ymm7, ymm4		;; B12 = I12 * cosine/sine
	vsubpd	ymm3, ymm3, ymm1		;; B2 = B2 - R2
	vmovapd	ymm0, [screg]			;; sine for R2/I2
	vmulpd	ymm2, ymm2, ymm0		;; R2 = A2 * sine
	vsubpd	ymm7, ymm7, ymm5		;; B12 = B12 - R12
	vmovapd	ymm4, [screg+10*64]		;; sine for R12/I12
	vmulpd	ymm6, ymm6, ymm4		;; R12 = A12 * sine
	vmulpd	ymm3, ymm3, ymm0		;; I2 = B2 * sine
	vmulpd	ymm7, ymm7, ymm4		;; I12 = B12 * sine
	vaddpd	ymm0, ymm2, ymm6		;; R2+R12
	vsubpd	ymm2, ymm2, ymm6		;; R2-R12
	L1prefetchw srcreg+L1pd, L1pt
	vaddpd	ymm1, ymm3, ymm7		;; I2+I12
	vsubpd	ymm3, ymm3, ymm7		;; I2-I12
	ystore	YMM_TMPS[2*32], ymm0		;; Save R2+R12
	ystore	YMM_TMPS[3*32], ymm2		;; Save R2-R12
	ystore	YMM_TMPS[4*32], ymm1		;; Save I2+I12
	ystore	YMM_TMPS[5*32], ymm3		;; Save I2-I12

	vmovapd	ymm0, [screg+64+32]		;; cosine/sine for R3/I3
	vmovapd	ymm1, [srcreg+d2]		;; R3
	vmulpd	ymm2, ymm1, ymm0		;; A3 = R3 * cosine/sine
	vmovapd	ymm4, [screg+9*64+32]		;; cosine/sine for R11/I11
	vmovapd	ymm5, [srcreg+5*d2]		;; R11
	vmulpd	ymm6, ymm5, ymm4		;; A11 = R11 * cosine/sine
	vmovapd	ymm3, [srcreg+d2+32]		;; I3
	vaddpd	ymm2, ymm2, ymm3		;; A3 = A3 + I3
	vmulpd	ymm3, ymm3, ymm0		;; B3 = I3 * cosine/sine
	vmovapd	ymm7, [srcreg+5*d2+32]		;; I11
	vaddpd	ymm6, ymm6, ymm7		;; A11 = A11 + I11
	vmulpd	ymm7, ymm7, ymm4		;; B11 = I11 * cosine/sine
	vsubpd	ymm3, ymm3, ymm1		;; B3 = B3 - R3
	vmovapd	ymm0, [screg+64]		;; sine for R3/I3
	vmulpd	ymm2, ymm2, ymm0		;; R3 = A3 * sine
	vsubpd	ymm7, ymm7, ymm5		;; B11 = B11 - R11
	vmovapd	ymm4, [screg+9*64]		;; sine for R11/I11
	vmulpd	ymm6, ymm6, ymm4		;; R11 = A11 * sine
	vmulpd	ymm3, ymm3, ymm0		;; I3 = B3 * sine
	vmulpd	ymm7, ymm7, ymm4		;; I11 = B11 * sine
	vaddpd	ymm0, ymm2, ymm6		;; R3+R11
	vsubpd	ymm2, ymm2, ymm6		;; R3-R11
	L1prefetchw srcreg+d1+L1pd, L1pt
	vaddpd	ymm1, ymm3, ymm7		;; I3+I11
	vsubpd	ymm3, ymm3, ymm7		;; I3-I11
	ystore	YMM_TMPS[6*32], ymm0		;; Save R3+R11
	ystore	YMM_TMPS[7*32], ymm2		;; Save R3-R11
	ystore	YMM_TMPS[8*32], ymm1		;; Save I3+I11
	ystore	YMM_TMPS[9*32], ymm3		;; Save I3-I11

	vmovapd	ymm0, [screg+2*64+32]		;; cosine/sine for R4/I4
	vmovapd	ymm1, [srcreg+d2+d1]		;; R4
	vmulpd	ymm2, ymm1, ymm0		;; A4 = R4 * cosine/sine
	vmovapd	ymm4, [screg+8*64+32]		;; cosine/sine for R10/I10
	vmovapd	ymm5, [srcreg+4*d2+d1]		;; R10
	vmulpd	ymm6, ymm5, ymm4		;; A10 = R10 * cosine/sine
	vmovapd	ymm3, [srcreg+d2+d1+32]		;; I4
	vaddpd	ymm2, ymm2, ymm3		;; A4 = A4 + I4
	vmulpd	ymm3, ymm3, ymm0		;; B4 = I4 * cosine/sine
	vmovapd	ymm7, [srcreg+4*d2+d1+32]	;; I10
	vaddpd	ymm6, ymm6, ymm7		;; A10 = A10 + I10
	vmulpd	ymm7, ymm7, ymm4		;; B10 = I10 * cosine/sine
	vsubpd	ymm3, ymm3, ymm1		;; B4 = B4 - R4
	vmovapd	ymm0, [screg+2*64]		;; sine for R4/I4
	vmulpd	ymm2, ymm2, ymm0		;; R4 = A4 * sine
	vsubpd	ymm7, ymm7, ymm5		;; B10 = B10 - R10
	vmovapd	ymm4, [screg+8*64]		;; sine for R10/I10
	vmulpd	ymm6, ymm6, ymm4		;; R10 = A10 * sine
	vmulpd	ymm3, ymm3, ymm0		;; I4 = B4 * sine
	vmulpd	ymm7, ymm7, ymm4		;; I10 = B10 * sine
	vaddpd	ymm0, ymm2, ymm6		;; R4+R10
	vsubpd	ymm2, ymm2, ymm6		;; R4-R10
	L1prefetchw srcreg+d2+L1pd, L1pt
	vaddpd	ymm1, ymm3, ymm7		;; I4+I10
	vsubpd	ymm3, ymm3, ymm7		;; I4-I10
	ystore	YMM_TMPS[10*32], ymm0		;; Save R4+R10
	ystore	YMM_TMPS[11*32], ymm2		;; Save R4-R10
	ystore	YMM_TMPS[12*32], ymm1		;; Save I4+I10
	ystore	YMM_TMPS[13*32], ymm3		;; Save I4-I10

	vmovapd	ymm0, [screg+3*64+32]		;; cosine/sine for R5/I5
	vmovapd	ymm1, [srcreg+2*d2]		;; R5
	vmulpd	ymm2, ymm1, ymm0		;; A5 = R5 * cosine/sine
	vmovapd	ymm4, [screg+7*64+32]		;; cosine/sine for R9/I9
	vmovapd	ymm5, [srcreg+4*d2]		;; R9
	vmulpd	ymm6, ymm5, ymm4		;; A9 = R9 * cosine/sine
	vmovapd	ymm3, [srcreg+2*d2+32]		;; I5
	vaddpd	ymm2, ymm2, ymm3		;; A5 = A5 + I5
	vmulpd	ymm3, ymm3, ymm0		;; B5 = I5 * cosine/sine
	vmovapd	ymm7, [srcreg+4*d2+32]		;; I9
	vaddpd	ymm6, ymm6, ymm7		;; A9 = A9 + I9
	vmulpd	ymm7, ymm7, ymm4		;; B9 = I9 * cosine/sine
	vsubpd	ymm3, ymm3, ymm1		;; B5 = B5 - R5
	vmovapd	ymm0, [screg+3*64]		;; sine for R5/I5
	vmulpd	ymm2, ymm2, ymm0		;; R5 = A5 * sine
	vsubpd	ymm7, ymm7, ymm5		;; B9 = B9 - R9
	vmovapd	ymm4, [screg+7*64]		;; sine for R9/I9
	vmulpd	ymm6, ymm6, ymm4		;; R9 = A9 * sine
	vmulpd	ymm3, ymm3, ymm0		;; I5 = B5 * sine
	vmulpd	ymm7, ymm7, ymm4		;; I9 = B9 * sine
	vaddpd	ymm0, ymm2, ymm6		;; R5+R9
	vsubpd	ymm2, ymm2, ymm6		;; R5-R9
	vaddpd	ymm1, ymm3, ymm7		;; I5+I9
	vsubpd	ymm3, ymm3, ymm7		;; I5-I9
	ystore	YMM_TMPS[14*32], ymm0		;; Save R5+R9
	ystore	YMM_TMPS[15*32], ymm2		;; Save R5-R9
	ystore	YMM_TMPS[16*32], ymm1		;; Save I5+I9
	ystore	YMM_TMPS[17*32], ymm3		;; Save I5-I9

	vmovapd	ymm0, [screg+4*64+32]		;; cosine/sine for R6/I6
	vmovapd	ymm1, [srcreg+2*d2+d1]		;; R6
	vmulpd	ymm2, ymm1, ymm0		;; A6 = R6 * cosine/sine
	vmovapd	ymm4, [screg+6*64+32]		;; cosine/sine for R8/I8
	vmovapd	ymm5, [srcreg+3*d2+d1]		;; R8
	vmulpd	ymm6, ymm5, ymm4		;; A8 = R8 * cosine/sine
	vmovapd	ymm3, [srcreg+2*d2+d1+32]	;; I6
	vaddpd	ymm2, ymm2, ymm3		;; A6 = A6 + I6
	vmulpd	ymm3, ymm3, ymm0		;; B6 = I6 * cosine/sine
	vmovapd	ymm7, [srcreg+3*d2+d1+32]	;; I8
	vaddpd	ymm6, ymm6, ymm7		;; A8 = A8 + I8
	vmulpd	ymm7, ymm7, ymm4		;; B8 = I8 * cosine/sine
	vsubpd	ymm3, ymm3, ymm1		;; B6 = B6 - R6
	vmovapd	ymm0, [screg+4*64]		;; sine for R6/I6
	vmulpd	ymm2, ymm2, ymm0		;; R6 = A6 * sine
	vsubpd	ymm7, ymm7, ymm5		;; B8 = B8 - R8
	vmovapd	ymm4, [screg+6*64]		;; sine for R8/I8
	vmulpd	ymm6, ymm6, ymm4		;; R8 = A8 * sine
	vmulpd	ymm3, ymm3, ymm0		;; I6 = B6 * sine
	vmulpd	ymm7, ymm7, ymm4		;; I8 = B8 * sine
	vaddpd	ymm0, ymm2, ymm6		;; R6+R8
	vsubpd	ymm2, ymm2, ymm6		;; R6-R8
	L1prefetchw srcreg+d2+d1+L1pd, L1pt
	vaddpd	ymm1, ymm3, ymm7		;; I6+I8
	vsubpd	ymm3, ymm3, ymm7		;; I6-I8
	ystore	YMM_TMPS[18*32], ymm0		;; Save R6+R8
;;	ystore	YMM_TMPS[?*32], ymm2		;; Save R6-R8
	ystore	YMM_TMPS[19*32], ymm1		;; Save I6+I8
	ystore	YMM_TMPS[20*32], ymm3		;; Save I6-I8

	;; Do the 24 reals inverse FFT

	;; Calculate even rows derived from real inputs

	vmovapd	ymm0, YMM_TMPS[11*32]		;; r4-r10
	vmovapd	ymm1, YMM_TMPS[3*32]		;; r2-r12
;;	vmovapd	ymm2, YMM_TMPS[?*32]		;; r6-r8

	vmulpd	ymm3, ymm0, YMM_SQRTHALF	;; .707(r4-r10)						; 1-5
	vsubpd	ymm4, ymm1, ymm0		;; (r2-r12) - (r4-r10)					; 1-3

	vmulpd	ymm0, ymm1, YMM_P966		;; .966(r2-r12)						; 2-6

	vmulpd	ymm1, ymm1, YMM_P259		;; .259(r2-r12)						; 3-7

	vmulpd	ymm5, ymm2, YMM_P259		;; .259(r6-r8)						; 4-8
	vsubpd	ymm4, ymm4, ymm2		;; (r2-r12) - (r4-r10) - (r6-r8) (r4e)			; 4-6

	vmulpd	ymm2, ymm2, YMM_P966		;; .966(r6-r8)						; 5-9

	vaddpd	ymm0, ymm0, ymm3		;; .966(r2-r12) + .707(r4-r10)				; 7-9

	vsubpd	ymm1, ymm1, ymm3		;; .259(r2-r12) - .707(r4-r10)				; 8-10

	vmovapd	ymm7, YMM_TMPS[15*32]		;; r5-r9
	vmulpd	ymm3, ymm7, YMM_HALF		;; .5(r5-r9)						; 6-10
	vmovapd	ymm6, [srcreg+32]		;; r1-r13
	vsubpd	ymm7, ymm6, ymm7		;; r1-r13 - (r5-r9)  (r4o)				; 6-8

	vaddpd	ymm0, ymm0, ymm5		;; .966(r2-r12) + .707(r4-r10) + .259(r6-r8) (r2e)	; 10-12

	vmovapd	ymm5, YMM_TMPS[7*32]		;; r3-r11
	vmulpd	ymm5, ymm5, YMM_P866		;; .866(r3-r11)  (r26oB)				; 7-11

	vmulpd	ymm4, ymm4, YMM_SQRTHALF	;; .707*r4e						; 9-13

	vaddpd	ymm1, ymm1, ymm2		;; .259(r2-r12) - .707(r4-r10) + .966(r6-r8) (r6e)	; 11-13
	L1prefetchw srcreg+2*d2+L1pd, L1pt

	vaddpd	ymm6, ymm6, ymm3		;; r1-r13 + .5(r5-r9)  (r26oA)				; 12-14

	vaddpd	ymm2, ymm7, ymm4		;; real-cols row #4 (ro4 + re4)				; 14-16
	L1prefetchw srcreg+2*d2+d1+L1pd, L1pt

	vaddpd	ymm3, ymm6, ymm5		;; r26oA + r26oB (r2o)					; 15-17

	vsubpd	ymm6, ymm6, ymm5		;; r26oA - r26oB (r6o)					; 16-18
	vmovapd	ymm5, YMM_TMPS[2*32]		;; r2+r12

	;; Combine even and odd columns (even rows)

	vsubpd	ymm7, ymm7, ymm4		;; real-cols row #10 (ro4 - re4)			; 17-19
	vmovapd	ymm4, YMM_TMPS[18*32]		;; r6+r8
	ystore	YMM_TMPS[2*32], ymm2		;; Save real-cols row #4				; 17

	vaddpd	ymm2, ymm3, ymm0		;; real-cols row #2 (ro2 + re2)				; 18-20
	L1prefetchw srcreg+3*d2+L1pd, L1pt

	vsubpd	ymm3, ymm3, ymm0		;; real-cols row #12 (ro2 - re2)			; 19-21

	vaddpd	ymm0, ymm6, ymm1		;; real-cols row #6 (ro6 + re6)				; 20-22
	ystore	YMM_TMPS[3*32], ymm7		;; Save real-cols row #10				; 20
	vmovapd	ymm7, YMM_TMPS[6*32]		;; r3+r11

	vsubpd	ymm6, ymm6, ymm1		;; real-cols row #8 (ro6 - re6)				; 21-23
	ystore	YMM_TMPS[6*32], ymm2		;; Save real-cols row #2				; 21

	;; Calculate odd rows derived from real inputs

	vaddpd	ymm1, ymm5, ymm4		;; (r2+r12)+(r6+r8)					; 22-24
	vmulpd	ymm2, ymm7, YMM_HALF		;; .5(r3+r11)						; 22-26
	ystore	YMM_TMPS[7*32], ymm3		;; Save real-cols row #12				; 22

	vsubpd	ymm5, ymm5, ymm4		;; (r2+r12)-(r6+r8) (r3e)				; 23-25
	vmovapd	ymm4, YMM_TMPS[14*32]		;; r5+r9
	vmulpd	ymm3, ymm4, YMM_HALF		;; .5(r5+r9)						; 23-27
	ystore	YMM_TMPS[11*32], ymm0		;; Save real-cols row #6				; 23

	vmovapd	ymm0, YMM_TMPS[0*32]		;; r7
	vaddpd	ymm7, ymm7, ymm0		;; (r3+r11)+r7 (r17oB)					; 24-26
	ystore	YMM_TMPS[0*32], ymm6		;; Save real-cols row #8				; 24

	vaddpd	ymm6, ymm1, YMM_TMPS[10*32]	;; (r2+r12)+(r6+r8)+(r4+r10) (r1e)			; 25-27
	vmulpd	ymm1, ymm1, YMM_HALF		;; .5((r2+r12)+(r6+r8))					; 25-29

	vaddpd	ymm4, ymm4, [srcreg]		;; (r1+r13)+(r5+r9) (r17oA)				; 26-28
	vmulpd	ymm5, ymm5, YMM_P866		;; .866*r3e						; 26-30

	vsubpd	ymm2, ymm2, ymm0		;; .5(r3+r11)-r7 (r35oB)				; 27-29

	vmovapd	ymm0, [srcreg]			;; r1+r13
	vsubpd	ymm0, ymm0, ymm3		;; (r1+r13)-.5(r5+r9) (r35oA)				; 28-30

	vaddpd	ymm3, ymm4, ymm7		;; r17oA + r17oB (r1o)					; 29-31

	vsubpd	ymm1, ymm1, YMM_TMPS[10*32]	;; .5((r2+r12)+(r6+r8))-(r4+r10) (r5e)			; 30-32

	vsubpd	ymm4, ymm4, ymm7		;; r17oA - r17oB (r7)					; 31-33
	L1prefetchw srcreg+3*d2+d1+L1pd, L1pt

	vaddpd	ymm7, ymm0, ymm2		;; r35oA + r35oB (r3o)					; 32-34

	vsubpd	ymm0, ymm0, ymm2		;; r35oA - r35oB (r5o)					; 33-35
	L1prefetchw srcreg+4*d2+L1pd, L1pt

	;; Combine even and odd columns (odd rows)

	vaddpd	ymm2, ymm3, ymm6		;; r1o + r1e (final R1)					; 34-36
	ystore	YMM_TMPS[10*32], ymm4		;; Save real-cols row #7				; 34
	vmovapd	ymm4, YMM_TMPS[4*32]		;; i2+i12

	vsubpd	ymm3, ymm3, ymm6		;; r1o - r1e (final R13)				; 35-37
	vmulpd	ymm6, ymm4, YMM_P259		;; .259(i2+i12)						; 35-39

	ystore	[srcreg], ymm2			;; Save final R1					; 37
	vaddpd	ymm2, ymm7, ymm5		;; r3o + r3e (real-cols row #3)				; 36-38
	ystore	[srcreg+32], ymm3		;; Save final R13					; 38
	vmovapd	ymm3, YMM_TMPS[12*32]		;; i4+i10
	ystore	YMM_TMPS[4*32], ymm2		;; Save real-cols row #3				; 39
	vmulpd	ymm2, ymm3, YMM_SQRTHALF	;; .707(i4+i10)						; 36-40

	vsubpd	ymm7, ymm7, ymm5		;; r3o - r3e (real-cols row #11)			; 37-39
	vmulpd	ymm5, ymm4, YMM_P966		;; .966(i2+i12)						; 37-41

	ystore	YMM_TMPS[12*32], ymm7		;; Save real-cols row #11				; 40
	vaddpd	ymm7, ymm0, ymm1		;; r5o + r5e (real-cols row #5)				; 38-40

	vsubpd	ymm0, ymm0, ymm1 		;; r5o - r5e (real-cols row #9)				; 39-41
	vmovapd	ymm1, YMM_TMPS[19*32]		;; i6+i8
	ystore	YMM_TMPS[14*32], ymm7		;; Save real-cols row #5				; 41
	vmulpd	ymm7, ymm1, YMM_P966		;; .966(i6+i8)						; 39-43

	;; Calculate even rows derived from imaginary inputs

	vaddpd	ymm4, ymm4, ymm3		;; (i2+i12)+(i4+i10)					; 40-42
	vmulpd	ymm3, ymm1, YMM_P259		;; .259(i6+i8)						; 40-44

	vaddpd	ymm6, ymm6, ymm2		;; .259(i2+i12)+.707(i4+i10)				; 41-43

	vsubpd	ymm5, ymm5, ymm2		;; .966(i2+i12)-.707(i4+i10)				; 42-44
	vmovapd	ymm2, YMM_TMPS[8*32]		;; i3+i11
	ystore	YMM_TMPS[8*32], ymm0		;; Save real-cols row #9				; 42
	vmulpd	ymm0, ymm2, YMM_HALF		;; .500(i3+i11)						; 41-45

	vsubpd	ymm4, ymm4, ymm1		;; (i2+i12)+(i4+i10)-(i6+i8) (i4e)			; 43-45
	vmovapd	ymm1, YMM_TMPS[16*32]		;; i5+i9

	vaddpd	ymm6, ymm6, ymm7		;; .259(i2+i12)+.707(i4+i10)+.966(i6+i8) (i2e)		; 44-46
	vmulpd	ymm1, ymm1, YMM_P866		;; .866(i5+i9)						; 44-48

	vaddpd	ymm5, ymm5, ymm3		;; .966(i2+i12)-.707(i4+i10)+.259(i6+i8) (i6e)		; 45-47

	vmovapd	ymm3, YMM_TMPS[1*32]		;; i7
	vaddpd	ymm0, ymm0, ymm3		;; .500(i3+i11)+i7					; 46-48
	vmulpd	ymm4, ymm4, YMM_SQRTHALF	;; .707*i4e						; 46-50

	vsubpd	ymm2, ymm2, ymm3		;; (i3+i11)-i7 (i4o)					; 47-49
	L1prefetchw srcreg+4*d2+d1+L1pd, L1pt

	vaddpd	ymm3, ymm0, ymm1		;; .500(i3+i11)+i7+.866(i5+i9) (i2o)			; 49-51

	vsubpd	ymm0, ymm0, ymm1		;; .500(i3+i11)+i7-.866(i5+i9) (i6o)			; 50-52

	vsubpd	ymm1, ymm4, ymm2		;; imag-cols row #10 (i4e - i4o)			; 51-53
	L1prefetchw srcreg+5*d2+L1pd, L1pt

	vaddpd	ymm4, ymm4, ymm2		;; imag-cols row #4 (i4e + i4o)				; 52-54

	vsubpd	ymm2, ymm6, ymm3		;; imag-cols row #12 (i2e - i2o)			; 53-55

	vaddpd	ymm6, ymm6, ymm3		;; imag-cols row #2 (i2e + i2o)				; 54-56
	L1prefetchw srcreg+5*d2+d1+L1pd, L1pt

	vsubpd	ymm3, ymm5, ymm0		;; imag-cols row #8 (i6e - i6o)				; 55-57

	vaddpd	ymm5, ymm5, ymm0		;; imag-cols row #6 (i6e + i6o)				; 56-58

	vmovapd	ymm0, YMM_TMPS[3*32]		;; Load real-cols row #10
	vsubpd	ymm7, ymm0, ymm1		;; (real#10 - imag#10) final R16
	vaddpd	ymm0, ymm0, ymm1		;; (real#10 + imag#10) final R10
	ystore	[srcreg+d2+d1+32], ymm7		;; Save R16
	ystore	[srcreg+4*d2+d1], ymm0		;; Save R10

	vmovapd	ymm1, YMM_TMPS[2*32]		;; Load real-cols row #4
	vsubpd	ymm7, ymm1, ymm4		;; (real#4 - imag#4) final R22
	vaddpd	ymm1, ymm1, ymm4		;; (real#4 + imag#4) final R4
	ystore	[srcreg+4*d2+d1+32], ymm7	;; Save R22
	ystore	[srcreg+d2+d1], ymm1		;; Save R4

	vmovapd	ymm4, YMM_TMPS[7*32]		;; Load real-cols row #12
	vsubpd	ymm7, ymm4, ymm2		;; (real#12 - imag#12) final R14
	vaddpd	ymm4, ymm4, ymm2		;; (real#12 + imag#12) final R12
	ystore	[srcreg+d1+32], ymm7		;; Save R14
	ystore	[srcreg+5*d2+d1], ymm4		;; Save R12

	vmovapd	ymm2, YMM_TMPS[6*32]		;; Load real-cols row #2
	vsubpd	ymm7, ymm2, ymm6		;; (real#2 - imag#2) final R24
	vaddpd	ymm2, ymm2, ymm6		;; (real#2 + imag#2) final R2
	ystore	[srcreg+5*d2+d1+32], ymm7	;; Save R24
	ystore	[srcreg+d1], ymm2		;; Save R2

	vmovapd	ymm6, YMM_TMPS[0*32]		;; Load real-cols row #8
	vsubpd	ymm7, ymm6, ymm3		;; (real#8 - imag#8) final R18
	vaddpd	ymm6, ymm6, ymm3		;; (real#8 + imag#8) final R8
	ystore	[srcreg+2*d2+d1+32], ymm7	;; Save R18
	ystore	[srcreg+3*d2+d1], ymm6		;; Save R8

	;; Calculate odd rows derived from imaginary inputs

	vmovapd	ymm0, YMM_TMPS[5*32]		;; i2-i12
	vmovapd	ymm1, YMM_TMPS[20*32]		;; i6-i8
	vaddpd	ymm4, ymm0, ymm1		;; (i2-i12)+(i6-i8)				; 67-69

	vsubpd	ymm0, ymm0, ymm1		;; (i2-i12)-(i6-i8) (i5e)			; 68-70

	vmovapd	ymm1, YMM_TMPS[9*32]		;; i3-i11
	vmovapd	ymm2, YMM_TMPS[17*32]		;; i5-i9
	vaddpd	ymm3, ymm1, ymm2		;; (i3-i11)+(i5-i9) (i3o)			; 69-71

	vsubpd	ymm1, ymm1, ymm2		;; (i3-i11)-(i5-i9) (i5o)			; 70-72
	vmulpd	ymm6, ymm4, YMM_HALF		;; .5((i2-i12)+(i6-i8))				; 70-74

	vmovapd	ymm2, YMM_TMPS[11*32]		;; Load real-cols row #6
	vsubpd	ymm7, ymm2, ymm5		;; (real#6 - imag#6) final R20			; 71-73
	vaddpd	ymm2, ymm2, ymm5		;; (real#6 + imag#6) final R6			; 72-74
	vmovapd ymm5, YMM_P866
	vmulpd	ymm3, ymm5, ymm3		;; .866*i3o					; 72-76

	ystore	[srcreg+3*d2+d1+32], ymm7	;; Save R20					; 74
	vaddpd	ymm7, ymm0, ymm1		;; i5e + i5o (i5)				; 73-75

	vsubpd	ymm0, ymm0, ymm1		;; i5e - i5o (i9)				; 74-76

	vmovapd	ymm1, YMM_TMPS[13*32]		;; i4-i10
	vaddpd	ymm6, ymm6, ymm1		;; .5((i2-i12)+(i6-i8))+(i4-i10) (i3e)		; 75-77
	ystore	[srcreg+2*d2+d1], ymm2		;; Save R6					; 75

	vsubpd	ymm4, ymm4, ymm1		;; (i2-i12)+(i6-i8)-(i4-i10) (i7)		; 76-78
	vmulpd	ymm7, ymm5, ymm7		;; .866*i5					; 76-80

	vmulpd	ymm0, ymm5, ymm0		;; .866*i9					; 77-81

	vaddpd	ymm1, ymm6, ymm3		;; i3e + i3o (i3)				; 78-80

	vsubpd	ymm6, ymm6, ymm3		;; i3e - i3o (i11)				; 79-81

	;; Combine even and odd columns, then real and imag data (odd rows)

	vmovapd	ymm3, YMM_TMPS[10*32]		;; Load real-cols row #7
	vsubpd	ymm5, ymm3, ymm4		;; (r7 - i7) final R19
	vaddpd	ymm3, ymm3, ymm4		;; (r7 + i7) final R7
	ystore	[srcreg+3*d2+32], ymm5		;; Save R19
	ystore	[srcreg+3*d2], ymm3		;; Save R7

	vmovapd	ymm3, YMM_TMPS[14*32]		;; Load real-cols row #5
	vsubpd	ymm5, ymm3, ymm7		;; (r5 - i5) final R21
	vaddpd	ymm3, ymm3, ymm7		;; (r5 + i5) final R5
	ystore	[srcreg+4*d2+32], ymm5		;; Save R21
	ystore	[srcreg+2*d2], ymm3		;; Save R5

	vmovapd	ymm3, YMM_TMPS[4*32]		;; Load real-cols row #3
	vsubpd	ymm5, ymm3, ymm1		;; (r3 - i3) final R23
	vaddpd	ymm3, ymm3, ymm1		;; (r3 + i3) final R3
	ystore	[srcreg+5*d2+32], ymm5		;; Save R23
	ystore	[srcreg+d2], ymm3		;; Save R3

	vmovapd	ymm3, YMM_TMPS[8*32]		;; Load real-cols row #9
	vsubpd	ymm5, ymm3, ymm0		;; (r9 - i9) final R17
	vaddpd	ymm3, ymm3, ymm0		;; (r9 + i9) final R9
	ystore	[srcreg+2*d2+32], ymm5		;; Save R17
	ystore	[srcreg+4*d2], ymm3		;; Save R9

	vmovapd	ymm3, YMM_TMPS[12*32]		;; Load real-cols row #11
	vsubpd	ymm5, ymm3, ymm6		;; (r11 - i11) final R15
	vaddpd	ymm3, ymm3, ymm6		;; (r11 + i11) final R11
	ystore	[srcreg+d2+32], ymm5		;; Save R15
	ystore	[srcreg+5*d2], ymm3		;; Save R11

	bump	srcreg, srcinc
	bump	screg, scinc
	ENDM

;; 64-bit version

IFDEF X86_64

yr6_12cl_24_reals_unfft_preload MACRO
	ENDM

yr6_12cl_24_reals_unfft MACRO srcreg,srcinc,d1,d2,screg,scinc,maxrpt,L1pt,L1pd

;; Apply the 11 twiddle factors.  Also do the first +/- on the reals so that
;; we have a perfect balance of adds and multiplies.

	vmovapd	ymm0, [screg+32]		;; cosine/sine for R2/I2
	vmovapd	ymm1, [srcreg+d1]		;; R2
	vmulpd	ymm2, ymm1, ymm0		;; A2 = R2 * cosine/sine				;	1-5

	vmovapd	ymm3, [srcreg+d1+32]		;; I2
	vmulpd	ymm0, ymm3, ymm0		;; B2 = I2 * cosine/sine				;	2-6

	vmovapd	ymm4, [screg+10*64+32]		;; cosine/sine for R12/I12
	vmovapd	ymm5, [srcreg+5*d2+d1]		;; R12
	vmulpd	ymm6, ymm5, ymm4		;; A12 = R12 * cosine/sine				;	3-7

	vmovapd	ymm7, [srcreg+5*d2+d1+32]	;; I12
	vmulpd	ymm4, ymm7, ymm4		;; B12 = I12 * cosine/sine				;	4-8

	vmovapd	ymm8, [screg+4*64+32]		;; cosine/sine for R6/I6
	vmovapd	ymm9, [srcreg+2*d2+d1]		;; R6
	vmulpd	ymm10, ymm9, ymm8		;; A6 = R6 * cosine/sine				;	5-9

	vmovapd	ymm11, [srcreg+2*d2+d1+32]	;; I6
	vaddpd	ymm2, ymm2, ymm3		;; A2 = A2 + I2						; 6-8
	vmulpd	ymm8, ymm11, ymm8		;; B6 = I6 * cosine/sine				;	6-10

	vmovapd	ymm12, [screg+6*64+32]		;; cosine/sine for R8/I8
	vmovapd	ymm13, [srcreg+3*d2+d1]		;; R8
	vsubpd	ymm0, ymm0, ymm1		;; B2 = B2 - R2						; 7-9
	vmulpd	ymm14, ymm13, ymm12		;; A8 = R8 * cosine/sine				;	7-11

	vaddpd	ymm6, ymm6, ymm7		;; A12 = A12 + I12					; 8-10
	vmovapd	ymm15, [srcreg+3*d2+d1+32]	;; I8
	vmulpd	ymm12, ymm15, ymm12		;; B8 = I8 * cosine/sine				;	8-12

	vmovapd	ymm3, [screg]			;; sine for R2/I2
	vsubpd	ymm4, ymm4, ymm5		;; B12 = B12 - R12					; 9-11
	vmulpd	ymm2, ymm2, ymm3		;; R2 = A2 * sine					;	9-13

	vaddpd	ymm10, ymm10, ymm11		;; A6 = A6 + I6						; 10-12
	vmulpd	ymm0, ymm0, ymm3		;; I2 = B2 * sine					;	10-14

	vmovapd	ymm1, [screg+10*64]		;; sine for R12/I12
	vsubpd	ymm8, ymm8, ymm9		;; B6 = B6 - R6						; 11-13
	vmulpd	ymm6, ymm6, ymm1		;; R12 = A12 * sine					;	11-15

	vaddpd	ymm14, ymm14, ymm15		;; A8 = A8 + I8						; 12-14
	vmulpd	ymm4, ymm4, ymm1		;; I12 = B12 * sine					;	12-16

	vmovapd	ymm7, [screg+4*64]		;; sine for R6/I6
	vsubpd	ymm12, ymm12, ymm13		;; B8 = B8 - R8						; 13-15
	vmulpd	ymm10, ymm10, ymm7		;; R6 = A6 * sine					;	13-17

	vmovapd	ymm5, [screg+6*64]		;; sine for R8/I8
	vmulpd	ymm8, ymm8, ymm7		;; I6 = B6 * sine					;	14-18

	vmovapd	ymm11, [screg+64+32]		;; cosine/sine for R3/I3
	vmulpd	ymm14, ymm14, ymm5		;; R8 = A8 * sine					;	15-19

	vmovapd	ymm3, [srcreg+d2]		;; R3
	vaddpd	ymm7, ymm2, ymm6		;; R2+R12						; 16-18
	vmulpd	ymm12, ymm12, ymm5		;; I8 = B8 * sine					;	16-20

	vmovapd	ymm9, [srcreg+d2+32]		;; I3
	vsubpd	ymm2, ymm2, ymm6		;; R2-R12						; 17-19
	vmulpd	ymm6, ymm3, ymm11		;; A3 = R3 * cosine/sine				;	17-21

	vmovapd	ymm15, [screg+9*64+32]		;; cosine/sine for R11/I11
	vaddpd	ymm5, ymm0, ymm4		;; I2+I12						; 18-20
	vmulpd	ymm11, ymm9, ymm11		;; B3 = I3 * cosine/sine				;	18-22

	vmovapd	ymm1, [srcreg+5*d2]		;; R11
	vsubpd	ymm0, ymm0, ymm4		;; I2-I12						; 19-21
	vmulpd	ymm4, ymm1, ymm15		;; A11 = R11 * cosine/sine				;	19-23
	ystore	YMM_TMPS[0*32], ymm7		;; Save R2+R12						; 19

	vmovapd	ymm13, [srcreg+5*d2+32]		;; I11
	vaddpd	ymm7, ymm10, ymm14		;; R6+R8						; 20-22
	vmulpd	ymm15, ymm13, ymm15		;; B11 = I11 * cosine/sine				;	20-24
	ystore	YMM_TMPS[1*32], ymm2		;; Save R2-R12						; 20

	vmovapd	ymm2, [screg+3*64+32]		;; cosine/sine for R5/I5
	vsubpd	ymm10, ymm10, ymm14		;; R6-R8						; 21-23
	vmovapd	ymm14, [srcreg+2*d2+32]		;; I5
	ystore	YMM_TMPS[2*32], ymm5		;; Save I2+I12						; 21
	vmulpd	ymm5, ymm14, ymm2		;; B5 = I5 * cosine/sine				;	21-25

	vaddpd	ymm6, ymm6, ymm9		;; A3 = A3 + I3						; 22-24
	vmovapd	ymm9, [srcreg+2*d2]		;; R5
	vmulpd	ymm2, ymm9, ymm2		;; A5 = R5 * cosine/sine				;	22-26
	ystore	YMM_TMPS[3*32], ymm0		;; Save I2-I12						; 22

	vmovapd	ymm0, [screg+7*64+32]		;; cosine/sine for R9/I9
	vsubpd	ymm11, ymm11, ymm3		;; B3 = B3 - R3						; 23-25
	vmovapd	ymm3, [srcreg+4*d2+32]		;; I9
	ystore	YMM_TMPS[4*32], ymm7		;; Save R6+R8						; 23
	vmulpd	ymm7, ymm3, ymm0		;; B9 = I9 * cosine/sine				;	23-27

	vaddpd	ymm4, ymm4, ymm13		;; A11 = A11 + I11					; 24-26
	vmovapd	ymm13, [srcreg+4*d2]		;; R9
	vmulpd	ymm0, ymm13, ymm0		;; A9 = R9 * cosine/sine				;	24-28
	ystore	YMM_TMPS[5*32], ymm10		;; Save R6-R8						; 24

	vmovapd	ymm10, [screg+64]		;; sine for R3/I3
	vsubpd	ymm15, ymm15, ymm1		;; B11 = B11 - R11					; 25-27
	vmulpd	ymm6, ymm6, ymm10		;; R3 = A3 * sine					;	25-29

	vmovapd	ymm1, [screg+9*64]		;; sine for R11/I11
	vsubpd	ymm5, ymm5, ymm9		;; B5 = B5 - R5						; 26-28
	vmulpd	ymm11, ymm11, ymm10		;; I3 = B3 * sine					;	26-30

	vmovapd	ymm9, [screg+3*64]		;; sine for R5/I5
	vaddpd	ymm2, ymm2, ymm14		;; A5 = A5 + I5						; 27-29
	vmulpd	ymm4, ymm4, ymm1		;; R11 = A11 * sine					;	27-31

	vmovapd	ymm10, [screg+7*64]		;; sine for R9/I9
	vsubpd	ymm7, ymm7, ymm13		;; B9 = B9 - R9						; 28-30
	vmulpd	ymm15, ymm15, ymm1		;; I11 = B11 * sine					;	28-32

	vmovapd	ymm14, [screg+2*64+32]		;; cosine/sine for R4/I4
	vaddpd	ymm0, ymm0, ymm3		;; A9 = A9 + I9						; 29-31
	vmulpd	ymm5, ymm5, ymm9		;; I5 = B5 * sine					;	29-33

	vmovapd	ymm13, [srcreg+d2+d1+32]	;; I4
	vaddpd	ymm1, ymm8, ymm12		;; I6+I8						; 30-32
	vmulpd	ymm2, ymm2, ymm9		;; R5 = A5 * sine					;	30-34

	vmovapd	ymm3, [srcreg+d2+d1]		;; R4
	vsubpd	ymm8, ymm8, ymm12		;; I6-I8						; 31-33
	vmulpd	ymm7, ymm7, ymm10		;; I9 = B9 * sine					;	31-35

	vmovapd	ymm9, [screg+8*64+32]		;; cosine/sine for R10/I10
	vaddpd	ymm12, ymm6, ymm4		;; R3+R11						; 32-34
	vmulpd	ymm0, ymm0, ymm10		;; R9 = A9 * sine					;	32-36

	vmovapd	ymm10, [srcreg+4*d2+d1+32]	;; I10
	vsubpd	ymm6, ymm6, ymm4		;; R3-R11						; 33-35
	vmulpd	ymm4, ymm13, ymm14		;; B4 = I4 * cosine/sine				;	33-37
	ystore	YMM_TMPS[6*32], ymm1		;; Save I6+I8						; 33

	vaddpd	ymm1, ymm11, ymm15		;; I3+I11						; 34-36
	vmulpd	ymm14, ymm3, ymm14		;; A4 = R4 * cosine/sine				;	34-38
	L1prefetchw srcreg+L1pd, L1pt

	vsubpd	ymm11, ymm11, ymm15		;; I3-I11						; 35-37
	vmulpd	ymm15, ymm10, ymm9		;; B10 = I10 * cosine/sine				;	35-39
	ystore	YMM_TMPS[7*32], ymm12		;; Save R3+R11						; 35

	vsubpd	ymm12, ymm5, ymm7		;; I5-I9						; 36-38
	ystore	YMM_TMPS[8*32], ymm6		;; Save R3-R11						; 36
	vmovapd	ymm6, [srcreg+4*d2+d1]		;; R10
	vmulpd	ymm9, ymm6, ymm9		;; A10 = R10 * cosine/sine				;	36-40

	vaddpd	ymm5, ymm5, ymm7		;; I5+I9						; 37-39
	vmovapd	ymm7, [screg+5*64+32]		;; cosine/sine for R7/I7
	ystore	YMM_TMPS[9*32], ymm1		;; Save I3+I11						; 37
	vmovapd	ymm1, [srcreg+3*d2+32]		;; I7

	vsubpd	ymm4, ymm4, ymm3		;; B4 = B4 - R4						; 38-40
	vmulpd	ymm3, ymm1, ymm7		;; B7 = I7 * cosine/sine				;	37-41

	vaddpd	ymm14, ymm14, ymm13		;; A4 = A4 + I4						; 39-41
	vmovapd	ymm13, [srcreg+3*d2]		;; R7
	vmulpd	ymm7, ymm13, ymm7		;; A7 = R7 * cosine/sine				;	38-42

	vsubpd	ymm15, ymm15, ymm6		;; B10 = B10 - R10					; 40-42
	ystore	YMM_TMPS[10*32], ymm5		;; Save I5+I9						; 40

	vmovapd	ymm6, [screg+2*64]		;; sine for R4/I4
	vaddpd	ymm9, ymm9, ymm10		;; A10 = A10 + I10					; 41-43
	vmulpd	ymm4, ymm4, ymm6		;; I4 = B4 * sine					;	41-45

	vmovapd	ymm10, [screg+8*64]		;; sine for R10/I10
	vsubpd	ymm3, ymm3, ymm13		;; B7 = B7 - R7						; 42-44
	vmulpd	ymm14, ymm14, ymm6		;; R4 = A4 * sine					;	42-46

	vmovapd	ymm13, [screg+5*64]		;; sine for R7/I7
	vaddpd	ymm7, ymm7, ymm1		;; A7 = A7 + I7						; 43-45
	vmulpd	ymm15, ymm15, ymm10		;; I10 = B10 * sine					;	43-47

	vmovapd	ymm5, YMM_TMPS[3*32]		;; i2-i12
	vaddpd	ymm1, ymm2, ymm0		;; R5+R9						; 44-46
	vmulpd	ymm9, ymm9, ymm10		;; R10 = A10 * sine					;	44-48

	vmovapd	ymm6, YMM_HALF
	vsubpd	ymm2, ymm2, ymm0		;; R5-R9						; 45-47
	vmulpd	ymm3, ymm3, ymm13		;; I7 = B7 * sine					;	45-49

	;; Do the 24 reals inverse FFT

	;; Calculate odd rows

	vmovapd ymm10, YMM_P866
	vaddpd	ymm0, ymm5, ymm8		;; (i2-i12)+(i6-i8)					; 46-48
	vmulpd	ymm7, ymm7, ymm13		;; R7 = A7 * sine					;	46-50

	vmovapd	ymm13, YMM_TMPS[7*32]		;; r3+r11
	vsubpd	ymm5, ymm5, ymm8		;; (i2-i12)-(i6-i8) (i5e)				; 47-49

	vaddpd	ymm8, ymm11, ymm12		;; (i3-i11)+(i5-i9) (i3o)				; 48-50
	ystore	YMM_TMPS[3*32], ymm2		;; Save R5-R9						; 48

	vsubpd	ymm11, ymm11, ymm12		;; (i3-i11)-(i5-i9) (i5o)				; 49-51
	vmulpd	ymm12, ymm6, ymm0		;; .5((i2-i12)+(i6-i8))					;	49-53
	L1prefetchw srcreg+d1+L1pd, L1pt

	vsubpd	ymm2, ymm4, ymm15		;; I4-I10						; 50-52
	vmulpd	ymm5, ymm10, ymm5		;; .866*i5e						;	50-54
	ystore	YMM_TMPS[7*32], ymm3		;; Save I7						; 50
	vmovapd	ymm3, YMM_TMPS[0*32]		;; r2+r12

	vaddpd	ymm4, ymm4, ymm15		;; I4+I10						; 51-53
	vmulpd	ymm8, ymm10, ymm8		;; .866*i3o						;	51-55
	L1prefetchw srcreg+d2+L1pd, L1pt

	vaddpd	ymm15, ymm14, ymm9		;; R4+R10						; 52-54
	vmulpd	ymm11, ymm10, ymm11		;; .866*i5o						;	52-56

	vsubpd	ymm14, ymm14, ymm9		;; R4-R10						; 53-55
	vmulpd	ymm9, ymm6, ymm13		;; .5(r3+r11)						;	53-57

	ystore	YMM_TMPS[0*32], ymm4		;; Save I4+I10						; 54
	vmovapd	ymm4, YMM_TMPS[4*32]		;; r6+r8
	ystore	YMM_TMPS[4*32], ymm14		;; Save R4-R10						; 56
	vaddpd	ymm14, ymm3, ymm4		;; (r2+r12)+(r6+r8)					; 54-56

	vsubpd	ymm3, ymm3, ymm4		;; (r2+r12)-(r6+r8) (r3e)				; 55-57
	L1prefetchw srcreg+d2+d1+L1pd, L1pt

	vaddpd	ymm12, ymm12, ymm2		;; .5((i2-i12)+(i6-i8))+(i4-i10) (i3e)			; 56-58

	vsubpd	ymm0, ymm0, ymm2		;; (i2-i12)+(i6-i8)-(i4-i10) (i7)			; 57-59
	vmulpd	ymm2, ymm6, ymm14		;; .5((r2+r12)+(r6+r8))					;	57-61
	L1prefetchw srcreg+2*d2+L1pd, L1pt

	vaddpd	ymm13, ymm13, ymm7		;; (r3+r11)+r7 (r17oB)					; 58-60
	vmulpd	ymm3, ymm10, ymm3		;; .866*r3e						;	58-62

	vsubpd	ymm9, ymm9, ymm7		;; .5(r3+r11)-r7 (r35oB)				; 59-61
	L1prefetchw srcreg+2*d2+d1+L1pd, L1pt

	vaddpd	ymm7, ymm5, ymm11		;; i5e + i5o (i5)					; 60-62
	vmulpd	ymm4, ymm6, ymm1		;; .5(r5+r9)						;	60-64

	vsubpd	ymm5, ymm5, ymm11		;; i5e - i5o (i9)					; 61-63

	vmovapd	ymm11, [srcreg]			;; r1+r13
	vaddpd	ymm14, ymm14, ymm15		;; (r2+r12)+(r6+r8)+(r4+r10) (r1e)			; 62-64

	vsubpd	ymm2, ymm2, ymm15		;; .5((r2+r12)+(r6+r8))-(r4+r10) (r5e)			; 63-65
	L1prefetchw srcreg+3*d2+L1pd, L1pt

	vaddpd	ymm1, ymm11, ymm1		;; (r1+r13)+(r5+r9) (r17oA)				; 64-66

	vsubpd	ymm11, ymm11, ymm4		;; (r1+r13)-.5(r5+r9) (r35oA)				; 65-67
	L1prefetchw srcreg+3*d2+d1+L1pd, L1pt

	vaddpd	ymm4, ymm12, ymm8		;; i3e + i3o (i3)					; 66-68
	vsubpd	ymm12, ymm12, ymm8		;; i3e - i3o (i11)					; 67-69

	vaddpd	ymm8, ymm1, ymm13		;; r17oA + r17oB (r1o)					; 68-70
	vsubpd	ymm1, ymm1, ymm13		;; r17oA - r17oB (r7)					; 69-71

	vaddpd	ymm13, ymm11, ymm9		;; r35oA + r35oB (r3o)					; 70-72
	vsubpd	ymm11, ymm11, ymm9		;; r35oA - r35oB (r5o)					; 71-73

	vaddpd	ymm9, ymm8, ymm14		;; r1o + r1e (final R1)					; 72-74
	L1prefetchw srcreg+4*d2+L1pd, L1pt

	vsubpd	ymm8, ymm8, ymm14		;; r1o - r1e (final R13)				; 73-75

	vaddpd	ymm14, ymm13, ymm3		;; r3o + r3e (real-cols row #3)				; 74-76
	L1prefetchw srcreg+4*d2+d1+L1pd, L1pt

	vsubpd	ymm13, ymm13, ymm3		;; r3o - r3e (real-cols row #11)			; 75-77
	ystore	[srcreg], ymm9			;; Save final R1					; 75

	vaddpd	ymm3, ymm11, ymm2		;; r5o + r5e (real-cols row #5)				; 76-78
	vmovapd	ymm15, [srcreg+32]		;; r1-r13
	ystore	[srcreg+32], ymm8		;; Save final R13					; 76

	vsubpd	ymm11, ymm11, ymm2 		;; r5o - r5e (real-cols row #9)				; 77-79
	vmovapd	ymm9, YMM_TMPS[4*32]		;; r4-r10

	vsubpd	ymm2, ymm1, ymm0		;; (r7 - i7) final R19					; 78-80
	vmovapd	ymm8, YMM_SQRTHALF

	vaddpd	ymm1, ymm1, ymm0		;; (r7 + i7) final R7					; 79-81

	vsubpd	ymm0, ymm14, ymm4		;; (r3 - i3) final R23					; 80-82

	vaddpd	ymm14, ymm14, ymm4		;; (r3 + i3) final R3					; 81-83
	ystore	[srcreg+3*d2+32], ymm2		;; Save R19						; 81
	vmovapd	ymm2, YMM_TMPS[1*32]		;; r2-r12

	vsubpd	ymm4, ymm13, ymm12		;; (r11 - i11) final R15				; 82-84
	ystore	[srcreg+3*d2], ymm1		;; Save R7						; 82
	vmovapd	ymm1, YMM_P966

	vaddpd	ymm13, ymm13, ymm12		;; (r11 + i11) final R11				; 83-85
	ystore	[srcreg+5*d2+32], ymm0		;; Save R23						; 83

	vsubpd	ymm12, ymm3, ymm7		;; (r5 - i5) final R21					; 84-86
	vmulpd	ymm0, ymm8, ymm9		;; .707(r4-r10)						;	84-88
	ystore	[srcreg+d2], ymm14		;; Save R3						; 84
	vmovapd	ymm14, YMM_P259

	vaddpd	ymm3, ymm3, ymm7		;; (r5 + i5) final R5					; 85-87
	vmulpd	ymm7, ymm1, ymm2		;; .966(r2-r12)						;	85-89
	ystore	[srcreg+d2+32], ymm4		;; Save R15						; 85
	vmovapd	ymm4, YMM_TMPS[3*32]		;; r5-r9

	;; Calculate even rows

	vsubpd	ymm9, ymm2, ymm9		;; (r2-r12) - (r4-r10)					; 86-88
	vmulpd	ymm2, ymm14, ymm2		;; .259(r2-r12)						;	86-90
	ystore	[srcreg+5*d2], ymm13		;; Save R11						; 86

	vsubpd	ymm13, ymm11, ymm5		;; (r9 - i9) final R17					; 87-89
	ystore	[srcreg+4*d2+32], ymm12		;; Save R21						; 87

	vaddpd	ymm11, ymm11, ymm5		;; (r9 + i9) final R9					; 88-90
	vmulpd	ymm12, ymm6, ymm4		;; .5(r5-r9)						;	88-92
	vmovapd	ymm5, YMM_TMPS[5*32]		;; r6-r8
	ystore	[srcreg+2*d2], ymm3		;; Save R5						; 88

	vsubpd	ymm9, ymm9, ymm5		;; (r2-r12) - (r4-r10) - (r6-r8) (r4e)			; 89-91
	vmulpd	ymm3, ymm14, ymm5		;; .259(r6-r8)						;	89-93

	vaddpd	ymm7, ymm7, ymm0		;; .966(r2-r12) + .707(r4-r10)				; 90-92
	vmulpd	ymm5, ymm1, ymm5		;; .966(r6-r8)						;	90-94
	ystore	[srcreg+2*d2+32], ymm13		;; Save R17						; 90
	vmovapd	ymm13, YMM_TMPS[8*32]		;; r3-r11

	vsubpd	ymm2, ymm2, ymm0		;; .259(r2-r12) - .707(r4-r10)				; 91-93
	vmulpd	ymm13, ymm10, ymm13		;; .866(r3-r11)  (r26oB)				;	91-95
	ystore	[srcreg+4*d2], ymm11		;; Save R9						; 91

	vsubpd	ymm4, ymm15, ymm4		;; r1-r13 - (r5-r9)  (r4o)				; 92-94
	vmulpd	ymm9, ymm8, ymm9		;; .707*r4e						;	92-96
	vmovapd	ymm0, YMM_TMPS[0*32]		;; i4+i10

	vaddpd	ymm15, ymm15, ymm12		;; r1-r13 + .5(r5-r9)  (r26oA)				; 93-95
	vmovapd	ymm11, YMM_TMPS[2*32]		;; i2+i12

	vaddpd	ymm7, ymm7, ymm3		;; .966(r2-r12) + .707(r4-r10) + .259(r6-r8) (r2e)	; 94-96
	L1prefetchw srcreg+5*d2+L1pd, L1pt

	vaddpd	ymm2, ymm2, ymm5		;; .259(r2-r12) - .707(r4-r10) + .966(r6-r8) (r6e)	; 95-97

	vaddpd	ymm5, ymm15, ymm13		;; r26oA + r26oB (r2o)					; 96-98

	vsubpd	ymm15, ymm15, ymm13		;; r26oA - r26oB (r6o)					; 97-99
	vmulpd	ymm13, ymm8, ymm0		;; .707(i4+i10)						;	97-101

	vaddpd	ymm3, ymm4, ymm9		;; real-cols row #4 (ro4 + re4)				; 98-100
	vmulpd	ymm12, ymm14, ymm11		;; .259(i2+i12)						;	98-102

	vsubpd	ymm4, ymm4, ymm9		;; real-cols row #10 (ro4 - re4)			; 99-101
	vmulpd	ymm9, ymm1, ymm11		;; .966(i2+i12)						;	99-103

	vaddpd	ymm11, ymm11, ymm0		;; (i2+i12)+(i4+i10)					; 100-102
	L1prefetchw srcreg+5*d2+d1+L1pd, L1pt

	vaddpd	ymm0, ymm5, ymm7		;; real-cols row #2 (ro2 + re2)				; 101-103

	vsubpd	ymm5, ymm5, ymm7		;; real-cols row #12 (ro2 - re2)			; 102-104

	vmovapd	ymm7, YMM_TMPS[9*32]		;; i3+i11
	vaddpd	ymm12, ymm12, ymm13		;; .259(i2+i12)+.707(i4+i10)				; 103-105
	vmulpd	ymm6, ymm6, ymm7		;; .500(i3+i11)						;	103-107

	vsubpd	ymm9, ymm9, ymm13		;; .966(i2+i12)-.707(i4+i10)				; 104-106
	vmovapd	ymm13, YMM_TMPS[6*32]		;; i6+i8
	vmulpd	ymm1, ymm1, ymm13		;; .966(i6+i8)						;	104-108

	vsubpd	ymm11, ymm11, ymm13		;; (i2+i12)+(i4+i10)-(i6+i8) (i4e)			; 105-107
	vmulpd	ymm13, ymm14, ymm13		;; .259(i6+i8)						;	105-109

	vaddpd	ymm14, ymm15, ymm2		;; real-cols row #6 (ro6 + re6)				; 106-108

	vsubpd	ymm15, ymm15, ymm2		;; real-cols row #8 (ro6 - re6)				; 107-109
	vmovapd	ymm2, YMM_TMPS[10*32]		;; i5+i9
	vmulpd	ymm2, ymm10, ymm2		;; .866(i5+i9)						;	107-111

	vmovapd	ymm10, YMM_TMPS[7*32]		;; i7
	vaddpd	ymm6, ymm6, ymm10		;; .500(i3+i11)+i7					; 108-110

	vsubpd	ymm7, ymm7, ymm10		;; (i3+i11)-i7 (i4o)					; 109-111
	vmulpd	ymm11, ymm8, ymm11		;; .707*i4e						;	109-113

	vaddpd	ymm12, ymm12, ymm1		;; .259(i2+i12)+.707(i4+i10)+.966(i6+i8) (i2e)		; 110-112

	vaddpd	ymm9, ymm9, ymm13		;; .966(i2+i12)-.707(i4+i10)+.259(i6+i8) (i6e)		; 111-113

	vaddpd	ymm13, ymm6, ymm2		;; .500(i3+i11)+i7+.866(i5+i9) (i2o)			; 112-114

	vsubpd	ymm6, ymm6, ymm2		;; .500(i3+i11)+i7-.866(i5+i9) (i6o)			; 113-115

	vsubpd	ymm2, ymm11, ymm7		;; imag-cols row #10 (i4e - i4o)			; 114-116

	vaddpd	ymm11, ymm11, ymm7		;; imag-cols row #4 (i4e + i4o)				; 115-117

	vsubpd	ymm7, ymm12, ymm13		;; imag-cols row #12 (i2e - i2o)			; 116-118

	vaddpd	ymm12, ymm12, ymm13		;; imag-cols row #2 (i2e + i2o)				; 117-119

	vsubpd	ymm13, ymm9, ymm6		;; imag-cols row #8 (i6e - i6o)				; 118-120

	vaddpd	ymm9, ymm9, ymm6		;; imag-cols row #6 (i6e + i6o)				; 119-121

	vsubpd	ymm6, ymm4, ymm2		;; (real#10 - imag#10) final R16			; 120-122

	vaddpd	ymm4, ymm4, ymm2		;; (real#10 + imag#10) final R10			; 121-123

	vsubpd	ymm2, ymm3, ymm11		;; (real#4 - imag#4) final R22				; 122-124

	vaddpd	ymm3, ymm3, ymm11		;; (real#4 + imag#4) final R4				; 123-125
	ystore	[srcreg+d2+d1+32], ymm6		;; Save R16						; 123

	vsubpd	ymm11, ymm5, ymm7		;; (real#12 - imag#12) final R14			; 124-126
	ystore	[srcreg+4*d2+d1], ymm4		;; Save R10						; 124

	vaddpd	ymm5, ymm5, ymm7		;; (real#12 + imag#12) final R12			; 125-127
	ystore	[srcreg+4*d2+d1+32], ymm2	;; Save R22						; 125

	vsubpd	ymm7, ymm0, ymm12		;; (real#2 - imag#2) final R24				; 126-128
	ystore	[srcreg+d2+d1], ymm3		;; Save R4						; 126

	vaddpd	ymm0, ymm0, ymm12		;; (real#2 + imag#2) final R2				; 127-129
	ystore	[srcreg+d1+32], ymm11		;; Save R14						; 127

	vsubpd	ymm12, ymm15, ymm13		;; (real#8 - imag#8) final R18				; 128-130
	ystore	[srcreg+5*d2+d1], ymm5		;; Save R12						; 128

	vaddpd	ymm15, ymm15, ymm13		;; (real#8 + imag#8) final R8				; 129-131
	ystore	[srcreg+5*d2+d1+32], ymm7	;; Save R24						; 129

	vsubpd	ymm13, ymm14, ymm9		;; (real#6 - imag#6) final R20				; 130-132
	ystore	[srcreg+d1], ymm0		;; Save R2						; 130

	vaddpd	ymm14, ymm14, ymm9		;; (real#6 + imag#6) final R6				; 131-133
	ystore	[srcreg+2*d2+d1+32], ymm12	;; Save R18						; 131

	ystore	[srcreg+3*d2+d1], ymm15		;; Save R8						; 132
	ystore	[srcreg+3*d2+d1+32], ymm13	;; Save R20						; 133
	ystore	[srcreg+2*d2+d1], ymm14		;; Save R6						; 134

	bump	srcreg, srcinc
	bump	screg, scinc
	ENDM

;; Haswell FMA3 version
;; This is the original Bulldozer version timed at 80.2 clocks.  Converting adds and subs to FMA3 did not improve the timings.

IF (@INSTR(,%yarch,<FMA3>) NE 0)

yr6_12cl_24_reals_unfft_preload MACRO
	ENDM

yr6_12cl_24_reals_unfft MACRO srcreg,srcinc,d1,d2,screg,scinc,maxrpt,L1pt,L1pd

;; Apply the 11 twiddle factors.  Also do the first +/- on the reals so that
;; we have a perfect balance of adds and multiplies.

	vmovapd	ymm3, [screg+64+32]		;; cosine/sine for R3/I3
	vmovapd	ymm2, [srcreg+d2]		;; R3
	vmovapd	ymm1, [srcreg+d2+32]		;; I3
	yfmaddpd ymm0, ymm2, ymm3, ymm1		;; A3 = R3 * cosine/sine + I3				; 1-5		n 6
	yfmsubpd ymm1, ymm1, ymm3, ymm2		;; B3 = I3 * cosine/sine - R3				; 1-5		n 6

	vmovapd	ymm5, [screg+3*64+32]		;; cosine/sine for R5/I5
	vmovapd	ymm4, [srcreg+2*d2]		;; R5
	vmovapd	ymm3, [srcreg+2*d2+32]		;; I5
	yfmaddpd ymm2, ymm4, ymm5, ymm3		;; A5 = R5 * cosine/sine + I5				; 2-6		n 7
	yfmsubpd ymm3, ymm3, ymm5, ymm4		;; B5 = I5 * cosine/sine - R5				; 2-6		n 7

	vmovapd	ymm7, [screg+5*64+32]		;; cosine/sine for R7/I7
	vmovapd	ymm6, [srcreg+3*d2]		;; R7
	vmovapd	ymm5, [srcreg+3*d2+32]		;; I7
	yfmaddpd ymm4, ymm6, ymm7, ymm5		;; A7 = R7 * cosine/sine + I7				; 3-7		n 8
	yfmsubpd ymm5, ymm5, ymm7, ymm6		;; B7 = I7 * cosine/sine - R7				; 3-7		n 8

	vmovapd	ymm9, [screg+9*64+32]		;; cosine/sine for R11/I11
	vmovapd	ymm8, [srcreg+5*d2]		;; R11
	vmovapd	ymm7, [srcreg+5*d2+32]		;; I11
	yfmaddpd ymm6, ymm8, ymm9, ymm7		;; A11 = R11 * cosine/sine + I11			; 4-8		n 11
	yfmsubpd ymm7, ymm7, ymm9, ymm8		;; B11 = I11 * cosine/sine - R11			; 4-8		n 12

	vmovapd	ymm11, [screg+7*64+32]		;; cosine/sine for R9/I9
	vmovapd	ymm10, [srcreg+4*d2]		;; R9
	vmovapd	ymm9, [srcreg+4*d2+32]		;; I9
	yfmaddpd ymm8, ymm10, ymm11, ymm9	;; A9 = R9 * cosine/sine + I9				; 5-9		n 13
	yfmsubpd ymm9, ymm9, ymm11, ymm10	;; B9 = I9 * cosine/sine - R9				; 5-9		n 14

	vmovapd	ymm10, [screg+64]		;; sine for R3/I3
	vmulpd	ymm0, ymm0, ymm10		;; R3 = A3 * sine					; 6-10		n 11
	vmulpd	ymm1, ymm1, ymm10		;; I3 = B3 * sine					; 6-10		n 12

	vmovapd	ymm10, [screg+3*64]		;; sine for R5/I5
	vmulpd	ymm2, ymm2, ymm10		;; R5 = A5 * sine					; 7-11		n 13
	vmulpd	ymm3, ymm3, ymm10		;; I5 = B5 * sine					; 7-11		n 14

	vmovapd	ymm10, [screg+5*64]		;; sine for R7/I7
	vmulpd	ymm4, ymm4, ymm10		;; R7 = A7 * sine					; 8-12		n 28
	vmulpd	ymm5, ymm5, ymm10		;; I7 = B7 * sine					; 8-12

	vmovapd	ymm13, [screg+32]		;; cosine/sine for R2/I2
	vmovapd	ymm12, [srcreg+d1]		;; R2
	vmovapd	ymm11, [srcreg+d1+32]		;; I2
	yfmaddpd ymm10, ymm12, ymm13, ymm11	;; A2 = R2 * cosine/sine + I2				; 9-13		n 17
	yfmsubpd ymm11, ymm11, ymm13, ymm12	;; B2 = I2 * cosine/sine - R2				; 9-13		n 17

	vmovapd	ymm15, [screg+4*64+32]		;; cosine/sine for R6/I6
	vmovapd	ymm14, [srcreg+2*d2+d1]		;; R6
	vmovapd	ymm13, [srcreg+2*d2+d1+32]	;; I6
	yfmaddpd ymm12, ymm14, ymm15, ymm13	;; A6 = R6 * cosine/sine + I6				; 10-14		n 19
	yfmsubpd ymm13, ymm13, ymm15, ymm14	;; B6 = I6 * cosine/sine - R6				; 10-14		n 19

	vmovapd	ymm15, [screg+9*64]		;; sine for R11/I11
	yfmaddpd ymm14, ymm6, ymm15, ymm0	;; R3+(R11 = A11 * sine)				; 11-15		n 28
	yfnmaddpd ymm6, ymm6, ymm15, ymm0	;; R3-(R11 = A11 * sine)				; 11-15

	yfmaddpd ymm0, ymm7, ymm15, ymm1	;; I3+(I11 = B11 * sine)				; 12-16
	yfnmaddpd ymm7, ymm7, ymm15, ymm1	;; I3-(I11 = B11 * sine)				; 12-16		n 32

	vmovapd	ymm15, [screg+7*64]		;; sine for R9/I9
	yfmaddpd ymm1, ymm8, ymm15, ymm2	;; R5+(R9 = A9 * sine)					; 13-17
	yfnmaddpd ymm8, ymm8, ymm15, ymm2	;; R5-(R9 = A9 * sine)					; 13-17
	ystore	YMM_TMPS[0*32], ymm4		;; Save R7						; 13
	vmovapd	ymm4, [screg+2*64+32]		;; cosine/sine for R4/I4

	yfmaddpd ymm2, ymm9, ymm15, ymm3	;; I5+(I9 = B9 * sine)					; 14-18
	yfnmaddpd ymm9, ymm9, ymm15, ymm3	;; I5-(I9 = B9 * sine)					; 14-18
	vmovapd	ymm15, [srcreg+d2+d1]		;; R4
	vmovapd	ymm3, [srcreg+d2+d1+32]		;; I4
	ystore	YMM_TMPS[1*32], ymm5		;; Save I7						; 13+1

	yfmaddpd ymm5, ymm15, ymm4, ymm3	;; A4 = R4 * cosine/sine + I4				; 15-19		n 21
	yfmsubpd ymm3, ymm3, ymm4, ymm15	;; B4 = I4 * cosine/sine - R4				; 15-19		n 21

	vmovapd	ymm4, [screg+10*64+32]		;; cosine/sine for R12/I12
	vmovapd	ymm15, [srcreg+5*d2+d1]		;; R12
	ystore	YMM_TMPS[2*32], ymm6		;; Save R3-R11						; 16
	vmovapd	ymm6, [srcreg+5*d2+d1+32]	;; I12
	ystore	YMM_TMPS[3*32], ymm0		;; Save I3+I11						; 17
	yfmaddpd ymm0, ymm15, ymm4, ymm6	;; A12 = R12 * cosine/sine + I12			; 16-20		n 22
	yfmsubpd ymm6, ymm6, ymm4, ymm15	;; B12 = I12 * cosine/sine - R12			; 16-20		n 23

	vmovapd	ymm4, [screg]			;; sine for R2/I2
	vmulpd	ymm10, ymm10, ymm4		;; R2 = A2 * sine					; 17-21		n 22
	vmulpd	ymm11, ymm11, ymm4		;; I2 = B2 * sine					; 17-21		n 23

	vmovapd	ymm4, [screg+6*64+32]		;; cosine/sine for R8/I8
	vmovapd	ymm15, [srcreg+3*d2+d1]		;; R8
	ystore	YMM_TMPS[4*32], ymm7		;; Save I3-I11						; 17+1
	vmovapd	ymm7, [srcreg+3*d2+d1+32]	;; I8
	ystore	YMM_TMPS[5*32], ymm8		;; Save R5-R9						; 18+1
	yfmaddpd ymm8, ymm15, ymm4, ymm7	;; A8 = R8 * cosine/sine + I8				; 18-22		n 24
	yfmsubpd ymm7, ymm7, ymm4, ymm15	;; B8 = I8 * cosine/sine - R8				; 18-22		n 25

	vmovapd	ymm4, [screg+4*64]		;; sine for R6/I6
	vmulpd	ymm12, ymm12, ymm4		;; R6 = A6 * sine					; 19-23		n 24
	vmulpd	ymm13, ymm13, ymm4		;; I6 = B6 * sine					; 19-23		n 25

	vmovapd	ymm4, [screg+8*64+32]		;; cosine/sine for R10/I10
	vmovapd	ymm15, [srcreg+4*d2+d1]		;; R10
	ystore	YMM_TMPS[6*32], ymm2		;; Save I5+I9						; 19+1
	vmovapd	ymm2, [srcreg+4*d2+d1+32]	;; I10
	ystore	YMM_TMPS[7*32], ymm9		;; Save I5-I9						; 19+2
	yfmaddpd ymm9, ymm15, ymm4, ymm2	;; A10 = R10 * cosine/sine + I10			; 20-24		n 26
	yfmsubpd ymm2, ymm2, ymm4, ymm15	;; B10 = I10 * cosine/sine - R10			; 20-24		n 27

	vmovapd	ymm4, [screg+2*64]		;; sine for R4/I4
	vmulpd	ymm5, ymm5, ymm4		;; R4 = A4 * sine					; 21-25		n 26
	vmulpd	ymm3, ymm3, ymm4		;; I4 = B4 * sine					; 21-25		n 27

	vmovapd	ymm4, [screg+10*64]		;; sine for R12/I12
	yfmaddpd ymm15, ymm0, ymm4, ymm10	;; R2+(R12 = A12 * sine)				; 22-26		n 29
	yfnmaddpd ymm0, ymm0, ymm4, ymm10	;; R2-(R12 = A12 * sine)				; 22-26

	yfnmaddpd ymm10, ymm6, ymm4, ymm11	;; I2-(I12 = B12 * sine)				; 23-27		n 31
	yfmaddpd ymm6, ymm6, ymm4, ymm11	;; I2+(I12 = B12 * sine)				; 23-27

	vmovapd	ymm4, [screg+6*64]		;; sine for R8/I8
	yfmaddpd ymm11, ymm8, ymm4, ymm12	;; R6+(R8 = A8 * sine)					; 24-28		n 29
	yfnmaddpd ymm8, ymm8, ymm4, ymm12	;; R6-(R8 = A8 * sine)					; 24-28

	yfnmaddpd ymm12, ymm7, ymm4, ymm13	;; I6-(I8 = B8 * sine)					; 25-29		n 31
	yfmaddpd ymm7, ymm7, ymm4, ymm13	;; I6+(I8 = B8 * sine)					; 25-29

	vmovapd	ymm4, [screg+8*64]		;; sine for R10/I10
	yfmaddpd ymm13, ymm9, ymm4, ymm5	;; R4+(R10 = A10 * sine)				; 26-30		n 34
	yfnmaddpd ymm9, ymm9, ymm4, ymm5	;; R4-(R10 = A10 * sine)				; 26-30
	vmovapd	ymm5, YMM_HALF

	ystore	YMM_TMPS[8*32], ymm0		;; Save R2-R12						; 27
	yfnmaddpd ymm0, ymm2, ymm4, ymm3	;; I4-(I10 = B10 * sine)				; 27-31		n 36
	yfmaddpd ymm2, ymm2, ymm4, ymm3		;; I4+(I10 = B10 * sine)				; 27-31

	;; Do the 24 reals inverse FFT

	vmovapd	ymm4, YMM_TMPS[0*32]		;; r7
	vmovapd	ymm3, [srcreg]			;; r1+r13
	ystore	YMM_TMPS[0*32], ymm6		;; Save I2+I12						; 28
	yfmsubpd ymm6, ymm5, ymm14, ymm4	;; .5(r3+r11)-r7 (r35oB)				; 28-32		n 33
	ystore	YMM_TMPS[9*32], ymm8		;; Save R6-R8						; 29
	yfnmaddpd ymm8, ymm5, ymm1, ymm3	;; (r1+r13)-.5(r5+r9) (r35oA)				; 28-32		n 33

	ystore	YMM_TMPS[10*32], ymm7		;; Save I6+I8						; 30
	vmovapd	ymm7, YMM_ONE
	ystore	YMM_TMPS[11*32], ymm2		;; Save I4+I10						; 32
	vaddpd	ymm2, ymm15, ymm11		;; (r2+r12)+(r6+r8)					; 29-33		n 34
	yfmsubpd ymm15, ymm15, ymm7, ymm11	;; (r2+r12)-(r6+r8) (r3e/.866)				; 29-33		n 38

	vaddpd	ymm14, ymm14, ymm4		;; (r3+r11)+r7 (r17oB)					; 30-34		n 35
	yfmaddpd ymm3, ymm3, ymm7, ymm1		;; (r1+r13)+(r5+r9) (r17oA)				; 30-34		n 35
	vmovapd	ymm11, YMM_TMPS[4*32]		;; i3-i11

	vaddpd	ymm1, ymm10, ymm12		;; (i2-i12)+(i6-i8)					; 31-35		n 36
	yfmsubpd ymm10, ymm10, ymm7, ymm12	;; (i2-i12)-(i6-i8) (i5e/.866)				; 31-35		n 37

	vmovapd	ymm4, YMM_TMPS[7*32]		;; i5-i9
	vsubpd	ymm12, ymm11, ymm4		;; (i3-i11)-(i5-i9) (i5o/.866)				; 32-36		n 37
	yfmaddpd ymm11, ymm11, ymm7, ymm4	;; (i3-i11)+(i5-i9) (i3o/.866)				; 32-36		n 41

	vaddpd	ymm4, ymm8, ymm6		;; r35oA + r35oB (r3o)					; 33-37		n 38
	yfmsubpd ymm8, ymm8, ymm7, ymm6		;; r35oA - r35oB (r5o)					; 33-37		n 39
	L1prefetchw srcreg+L1pd, L1pt

	yfmsubpd ymm6, ymm5, ymm2, ymm13	;; .5((r2+r12)+(r6+r8))-(r4+r10) (r5e)			; 34-38		n 39
	vaddpd	ymm2, ymm2, ymm13		;; (r2+r12)+(r6+r8)+(r4+r10) (r1e)			; 34-38		n 40
	L1prefetchw srcreg+d1+L1pd, L1pt

	vaddpd	ymm13, ymm3, ymm14		;; r17oA + r17oB (r1o)					; 35-39		n 40
	yfmsubpd ymm3, ymm3, ymm7, ymm14	;; r17oA - r17oB (r7)					; 35-39		n 42
	vmovapd ymm14, YMM_P866

	yfmaddpd ymm5, ymm5, ymm1, ymm0		;; .5((i2-i12)+(i6-i8))+(i4-i10) (i3e)			; 36-40		n 41
	vsubpd	ymm1, ymm1, ymm0		;; (i2-i12)+(i6-i8)-(i4-i10) (i7)			; 36-40		n 42
	L1prefetchw srcreg+d2+L1pd, L1pt

	vaddpd	ymm0, ymm10, ymm12		;; i5e + i5o (i5/.866)					; 37-41		n 44
	yfmsubpd ymm10, ymm10, ymm7, ymm12	;; i5e - i5o (i9/.866)					; 37-41		n 45

	yfmaddpd ymm12, ymm14, ymm15, ymm4	;; r3o + .866*r3e (real-cols row #3)			; 38-42		n 46
	yfnmaddpd ymm15, ymm14, ymm15, ymm4	;; r3o - .866*r3e (real-cols row #11)			; 38-42		n 56
	L1prefetchw srcreg+d2+d1+L1pd, L1pt

	vaddpd	ymm4, ymm8, ymm6		;; r5o + r5e (real-cols row #5)				; 39-43		n 44
	yfmsubpd ymm8, ymm8, ymm7, ymm6		;; r5o - r5e (real-cols row #9)				; 39-43		n 45

	vaddpd	ymm6, ymm13, ymm2		;; r1o + r1e (final R1)					; 40-44
	yfmsubpd ymm13, ymm13, ymm7, ymm2	;; r1o - r1e (final R13)				; 40-44
	L1prefetchw srcreg+2*d2+L1pd, L1pt

	yfmaddpd ymm2, ymm14, ymm11, ymm5	;; i3e + .866*i3o (i3)					; 41-45		n 46
	yfnmaddpd ymm11, ymm14, ymm11, ymm5	;; i3e - .866*i3o (i11)					; 41-45		n 56
	L1prefetchw srcreg+2*d2+d1+L1pd, L1pt

	vsubpd	ymm5, ymm3, ymm1		;; (r7 - i7) final R19					; 42-46
	yfmaddpd ymm3, ymm3, ymm7, ymm1		;; (r7 + i7) final R7					; 42-46

	vmovapd	ymm1, YMM_TMPS[8*32]		;; r2-r12
	vmovapd	ymm7, YMM_P966_P707
	ystore	[srcreg], ymm6			;; Save final R1					; 45
	vsubpd	ymm6, ymm1, ymm9		;; (r2-r12) - (r4-r10)					; 43-47		n 51
	ystore	[srcreg+3*d2+32], ymm5		;; Save R19						; 47
	yfmaddpd ymm5, ymm7, ymm1, ymm9		;; .966/.707(r2-r12) + (r4-r10)				; 43-47		n 51

	ystore	[srcreg+3*d2], ymm3		;; Save R7						; 47+1
	yfnmaddpd ymm3, ymm0, ymm14, ymm4	;; (r5 - i5*.866) final R21				; 44-48
	yfmaddpd ymm0, ymm0, ymm14, ymm4	;; (r5 + i5*.866) final R5				; 44-48

	yfnmaddpd ymm4, ymm10, ymm14, ymm8	;; (r9 - i9*.866) final R17				; 45-49
	yfmaddpd ymm10, ymm10, ymm14, ymm8	;; (r9 + i9*.866) final R9				; 45-49

	vsubpd	ymm14, ymm12, ymm2		;; (r3 - i3) final R23					; 46-50
	yfmaddpd ymm12, ymm12, YMM_ONE, ymm2	;; (r3 + i3) final R3					; 46-50

	vmovapd	ymm8, YMM_P259_P707
	yfmsubpd ymm1, ymm8, ymm1, ymm9		;; .259/.707(r2-r12) - (r4-r10)				; 47-51		n 52
	vmovapd	ymm2, YMM_TMPS[0*32]		;; i2+i12
	vmovapd	ymm9, YMM_TMPS[11*32]		;; i4+i10
	ystore	[srcreg+4*d2+32], ymm3		;; Save R21						; 49
	vaddpd	ymm3, ymm2, ymm9		;; (i2+i12)+(i4+i10)					; 47-51		n 52

	ystore	[srcreg+2*d2], ymm0		;; Save R5						; 49+1
	yfmaddpd ymm0, ymm8, ymm2, ymm9		;; .259/.707(i2+i12)+(i4+i10)				; 48-52		n 53
	yfmsubpd ymm2, ymm7, ymm2, ymm9		;; .966/.707(i2+i12)-(i4+i10)				; 48-52		n 53

	vmovapd	ymm9, [srcreg+32]		;; r1-r13
	ystore	[srcreg+32], ymm13		;; Save final R13					; 45+1
	vmovapd	ymm13, YMM_TMPS[5*32]		;; r5-r9
	ystore	[srcreg+2*d2+32], ymm4		;; Save R17						; 50+1
	vmovapd	ymm4, YMM_HALF
	ystore	[srcreg+4*d2], ymm10		;; Save R9						; 50+2
	yfmaddpd ymm10, ymm4, ymm13, ymm9	;; r1-r13 + .5(r5-r9)  (r26oA)				; 49-53		n 54
	vsubpd	ymm9, ymm9, ymm13		;; r1-r13 - (r5-r9)  (r4o)				; 49-53		n 57

	vmovapd	ymm13, YMM_TMPS[3*32]		;; i3+i11
	ystore	[srcreg+5*d2+32], ymm14		;; Save R23						; 51+2
	vmovapd	ymm14, YMM_TMPS[1*32]		;; i7
	yfmaddpd ymm4, ymm4, ymm13, ymm14	;; .5(i3+i11)+i7					; 50-54		n 55
	vsubpd	ymm13, ymm13, ymm14		;; (i3+i11)-i7 (i4o)					; 50-54		n 58

	vmovapd	ymm14, YMM_TMPS[9*32]		;; r6-r8
	vsubpd	ymm6, ymm6, ymm14		;; (r2-r12)-(r4-r10) - (r6-r8) (r4e/.707)		; 51-55		n 57
	yfmaddpd ymm5, ymm8, ymm14, ymm5	;; .966/.707(r2-r12)+(r4-r10)+.259/.707(r6-r8) (r2e/.707) ; 51-55	n 59

	yfmaddpd ymm1, ymm7, ymm14, ymm1	;; .259/.707(r2-r12)-(r4-r10)+.966/.707(r6-r8) (r6e/.707) ; 52-56	n 61
	vmovapd	ymm14, YMM_TMPS[10*32]		;; i6+i8
	vsubpd	ymm3, ymm3, ymm14		;; (i2+i12)+(i4+i10)-(i6+i8) (i4e/.707)			; 52-56		n 58

	yfmaddpd ymm0, ymm7, ymm14, ymm0	;; .259/.707(i2+i12)+(i4+i10)+.966/.707(i6+i8) (i2e/.707) ; 53-57	n 60
	yfmaddpd ymm2, ymm8, ymm14, ymm2	;; .966/.707(i2+i12)-(i4+i10)+.259/.707(i6+i8) (i6e/.707) ; 53-57	n 62

	vmovapd	ymm7, YMM_TMPS[2*32]		;; r3-r11
	vmovapd ymm8, YMM_P866
	yfmaddpd ymm14, ymm8, ymm7, ymm10	;; r26oA + .866(r3-r11) (r2o)				; 54-58		n 59
	yfnmaddpd ymm7, ymm8, ymm7, ymm10	;; r26oA - .866(r3-r11) (r6o)				; 54-58		n 61

	vmovapd	ymm10, YMM_TMPS[6*32]		;; i5+i9
	ystore	[srcreg+d2], ymm12		;; Save R3						; 51+3
	yfmaddpd ymm12, ymm8, ymm10, ymm4	;; .5(i3+i11)+i7+.866(i5+i9) (i2o)			; 55-59		n 60
	yfnmaddpd ymm8, ymm8, ymm10, ymm4	;; .5(i3+i11)+i7-.866(i5+i9) (i6o)			; 55-59		n 62

	vmovapd	ymm10, YMM_ONE
	vsubpd	ymm4, ymm15, ymm11		;; (r11 - i11) final R15				; 56-60
	yfmaddpd ymm15, ymm15, ymm10, ymm11	;; (r11 + i11) final R11				; 56-60

	vmovapd	ymm11, YMM_SQRTHALF
	ystore	[srcreg+d2+32], ymm4		;; Save R15						; 61
	yfnmaddpd ymm4, ymm11, ymm6, ymm9	;; real-cols row #10 (ro4 - .707*re4)			; 57-61		n 63
	yfmaddpd ymm6, ymm11, ymm6, ymm9	;; real-cols row #4 (ro4 + .707*re4)			; 57-61		n 64

	yfmsubpd ymm9, ymm3, ymm11, ymm13	;; imag-cols row #10 (i4e*.707 - i4o)			; 58-62		n 63
	yfmaddpd ymm3, ymm3, ymm11, ymm13	;; imag-cols row #4 (i4e*.707 + i4o)			; 58-62		n 64
	L1prefetchw srcreg+3*d2+L1pd, L1pt

	yfnmaddpd ymm13, ymm11, ymm5, ymm14	;; real-cols row #12 (ro2 - .707*re2)			; 59-63		n 65
	yfmaddpd ymm5, ymm11, ymm5, ymm14	;; real-cols row #2 (ro2 + .707*re2)			; 59-63		n 66
	L1prefetchw srcreg+3*d2+d1+L1pd, L1pt

	yfmsubpd ymm14, ymm0, ymm11, ymm12	;; imag-cols row #12 (i2e*.707 - i2o)			; 60-64		n 65
	yfmaddpd ymm0, ymm0, ymm11, ymm12	;; imag-cols row #2 (i2e*.707 + i2o)			; 60-64		n 66

	yfnmaddpd ymm12, ymm11, ymm1, ymm7	;; real-cols row #8 (ro6 - .707*re6)			; 61-65		n 67
	yfmaddpd ymm1, ymm11, ymm1, ymm7	;; real-cols row #6 (ro6 + .707*re6)			; 61-65		n 68
	L1prefetchw srcreg+4*d2+L1pd, L1pt

	yfmsubpd ymm7, ymm2, ymm11, ymm8	;; imag-cols row #8 (i6e*.707 - i6o)			; 62-66		n 67
	yfmaddpd ymm2, ymm2, ymm11, ymm8	;; imag-cols row #6 (i6e*.707 + i6o)			; 62-66		n 68
	ystore	[srcreg+5*d2], ymm15		;; Save R11						; 61+1

	vsubpd	ymm8, ymm4, ymm9		;; (real#10 - imag#10) final R16			; 63-67
	yfmaddpd ymm4, ymm4, ymm10, ymm9	;; (real#10 + imag#10) final R10			; 63-67
	L1prefetchw srcreg+4*d2+d1+L1pd, L1pt

	vsubpd	ymm9, ymm6, ymm3		;; (real#4 - imag#4) final R22				; 64-68
	yfmaddpd ymm6, ymm6, ymm10, ymm3	;; (real#4 + imag#4) final R4				; 64-68
	L1prefetchw srcreg+5*d2+L1pd, L1pt

	vsubpd	ymm3, ymm13, ymm14		;; (real#12 - imag#12) final R14			; 65-69
	yfmaddpd ymm13, ymm13, ymm10, ymm14	;; (real#12 + imag#12) final R12			; 65-69

	vsubpd	ymm14, ymm5, ymm0		;; (real#2 - imag#2) final R24				; 66-70
	yfmaddpd ymm5, ymm5, ymm10, ymm0	;; (real#2 + imag#2) final R2				; 66-70
	L1prefetchw srcreg+5*d2+d1+L1pd, L1pt

	vsubpd	ymm0, ymm12, ymm7		;; (real#8 - imag#8) final R18				; 67-71
	yfmaddpd ymm12, ymm12, ymm10, ymm7	;; (real#8 + imag#8) final R8				; 67-71

	vsubpd	ymm7, ymm1, ymm2		;; (real#6 - imag#6) final R20				; 68-72
	yfmaddpd ymm1, ymm1, ymm10, ymm2	;; (real#6 + imag#6) final R6				; 68-72

	ystore	[srcreg+d2+d1+32], ymm8		;; Save R16						; 68
	ystore	[srcreg+4*d2+d1], ymm4		;; Save R10						; 68+1
	ystore	[srcreg+4*d2+d1+32], ymm9	;; Save R22						; 69+1
	ystore	[srcreg+d2+d1], ymm6		;; Save R4						; 69+2
	ystore	[srcreg+d1+32], ymm3		;; Save R14						; 70+2
	ystore	[srcreg+5*d2+d1], ymm13		;; Save R12						; 70+3
	ystore	[srcreg+5*d2+d1+32], ymm14	;; Save R24						; 71+3
	ystore	[srcreg+d1], ymm5		;; Save R2						; 71+4
	ystore	[srcreg+2*d2+d1+32], ymm0	;; Save R18						; 72+4
	ystore	[srcreg+3*d2+d1], ymm12		;; Save R8						; 72+5
	ystore	[srcreg+3*d2+d1+32], ymm7	;; Save R20						; 73+5
	ystore	[srcreg+2*d2+d1], ymm1		;; Save R6						; 73+6

	bump	srcreg, srcinc
	bump	screg, scinc
	ENDM

ENDIF

ENDIF
