; Copyright 2011-2013 - Mersenne Research, Inc.  All rights reserved
; Author:  George Woltman
; Email: woltman@alum.mit.edu
;

;;
;;
;; All new macros for version 27 of gwnum.  Do a radix-5 step in an FFT.
;; The forward FFT macros multiply by the sin/cos values at the end of the macro
;; and the inverse FFTs multiply by the sin/cos values at the start of the macro.
;; We also implement the Daniel J. Bernstein (DJB) "exponent-1" idea to save sin/cos
;; memory.
;;
;;

;;
;; ************************************* five-complex-djbfft variants ******************************************
;;

;; The standard version
yr5_5cl_five_complex_djbfft_preload MACRO
	yr5_5c_djbfft_cmn_preload
	ENDM
yr5_5cl_five_complex_djbfft MACRO srcreg,srcinc,d1,screg,scinc,maxrpt,L1pt,L1pd
	yr5_5c_djbfft_cmn srcreg,srcinc,0,d1,noexec,screg,screg+64,32,screg,scinc,maxrpt,L1pt,L1pd
	ENDM

; Like 5cl but uses a ten-reals sin/cos data
yr5_r5cl_five_complex_djbfft_preload MACRO
	yr5_5c_djbfft_cmn_preload
	ENDM
yr5_r5cl_five_complex_djbfft MACRO srcreg,srcinc,d1,screg,scinc,maxrpt,L1pt,L1pd
	yr5_5c_djbfft_cmn srcreg,srcinc,0,d1,noexec,screg+64,screg+192,32,screg,scinc,maxrpt,L1pt,L1pd
	ENDM

;; Like 5cl but uses rbx to index into source
yr5_f5cl_five_complex_djbfft_preload MACRO
	yr5_5c_djbfft_cmn_preload
	ENDM
yr5_f5cl_five_complex_djbfft MACRO srcreg,srcinc,d1,screg,scinc,maxrpt,L1pt,L1pd
	yr5_5c_djbfft_cmn srcreg,srcinc,rbx,d1,noexec,screg,screg+64,32,screg,scinc,maxrpt,L1pt,L1pd
	ENDM

; Like the 5cl version except vbroadcastsd is used to reduce sin/cos data
yr5_b5cl_five_complex_djbfft_preload MACRO
	yr5_5c_djbfft_cmn_preload
	ENDM
yr5_b5cl_five_complex_djbfft MACRO srcreg,srcinc,d1,screg,scinc,maxrpt,L1pt,L1pd
	yr5_5c_djbfft_cmn srcreg,srcinc,0,d1,exec,screg,screg+16,8,screg,scinc,maxrpt,L1pt,L1pd
	ENDM

; Like b5cl but extracts the sin/cos data to broadcasts from
; the ten-real/five_complex sin/cos table
yr5_rb5cl_five_complex_djbfft_preload MACRO
	yr5_5c_djbfft_cmn_preload
	ENDM
yr5_rb5cl_five_complex_djbfft MACRO srcreg,srcinc,d1,screg,scinc,maxrpt,L1pt,L1pd
	yr5_5c_djbfft_cmn srcreg,srcinc,0,d1,exec,screg+8,screg+80,40,screg,scinc,maxrpt,L1pt,L1pd
	ENDM

;; Common code to do the 5-complex FFT.  A 5-complex FFT is:
;; r25a=r2+r5
;; r34a=r3+r4
;; i25s=i2-i5
;; i34s=i3-i4
;; outr(0) = r1 + r25a + r34a
;; t1=cos2*r25a + cos4*r34a + r1
;; t2=sin2*i25s + sin4*i34s
;; outr(1)=t1-t2
;; outr(4)=t1+t2
;; t3=cos4*r25a + cos2*r34a + r1
;; t4=sin4*i25s - sin2*i34s
;; outr(2)=t3-t4
;; outr(3)=t3+t4
;; r25s=r2-r5
;; r34s=r3-r4
;; i25a=i2+i5
;; i34a=i3+i4
;; outi(0)=i1+i25a+i34a
;; t5=cos2*i25a + cos4*i34a + i1
;; t6=sin2*r25s + sin4*r34s
;; outi(1)=t5+t6
;; outi(4)=t5-t6
;; t7=cos4*i25a + cos2*i34a + i1
;; t8=sin4*r25s - sin2*r34s
;; outi(2)=t7+t8
;; outi(3)=t7-t8
;; Where cos2 = cos 2*pi/5 = 0.309, sin2 = 0.951,
;; cos4 =-0.809, sin4 = 0.588
;; Finally, multiply 4 of the 5 results by twiddle factors.

yr5_5c_djbfft_cmn_preload MACRO
	ENDM
yr5_5c_djbfft_cmn MACRO srcreg,srcinc,srcoff,d1,bcast,screg1,screg2,cosoff,screg,scinc,maxrpt,L1pt,L1pd
	vmovapd	ymm0, [srcreg+srcoff+d1]	;; r2
	vaddpd	ymm0, ymm0, [srcreg+srcoff+4*d1] ;; r25a=r2+r5
	vmovapd	ymm1, [srcreg+srcoff+2*d1]	;; r3
	vaddpd	ymm1, ymm1, [srcreg+srcoff+3*d1] ;; r34a=r3+r4
	vmovapd	ymm2, [srcreg+srcoff+d1+32]	;; i2
	vsubpd	ymm2, ymm2, [srcreg+srcoff+4*d1+32] ;; i25s=i2-i5
	vmovapd	ymm3, [srcreg+srcoff+2*d1+32]	;; i3
	vsubpd	ymm3, ymm3, [srcreg+srcoff+3*d1+32] ;; i34s=i3-i4
	vmulpd	ymm4, ymm0, YMM_P309		;; cos2*r25a
	vmulpd	ymm5, ymm1, YMM_P809		;; -cos4*r34a
	vmulpd	ymm6, ymm2, YMM_P951		;; sin2*i25s
	vmulpd	ymm7, ymm3, YMM_P588		;; sin4*i34s
	vsubpd	ymm4, ymm4, ymm5		;; cos2*r25a + cos4*r34a
	vaddpd	ymm6, ymm6, ymm7		;; t2=sin2*i25s + sin4*i34s
	vmulpd	ymm5, ymm0, YMM_P809		;; -cos4*r25a
	vaddpd	ymm0, ymm0, ymm1		;; r25a + r34a
	vmovapd	ymm7, [srcreg+srcoff]		;; r1
	vaddpd	ymm4, ymm4, ymm7		;; t1=cos2*r25a + cos4*r34a + r1
	vaddpd	ymm0, ymm0, ymm7		;; outr(0) = r1 + r25a + r34a
	vmulpd	ymm1, ymm1, YMM_P309		;; cos2*r34a
	vmulpd	ymm2, ymm2, YMM_P588		;; sin4*i25s
	vmulpd	ymm3, ymm3, YMM_P951		;; sin2*i34s
	vsubpd	ymm5, ymm1, ymm5		;; cos4*r25a+cos2*r34a
	vsubpd	ymm2, ymm2, ymm3		;; t4=sin4*i25s-sin2*i34s
	vaddpd	ymm5, ymm5, ymm7		;; t3=cos4*r25a+cos2*r34a+r1
	vsubpd	ymm7, ymm4, ymm6		;; outr(1)=t1-t2
	vaddpd	ymm4, ymm4, ymm6		;; outr(4)=t1+t2
	vsubpd	ymm6, ymm5, ymm2		;; outr(2)=t3-t4
	vaddpd	ymm5, ymm5, ymm2		;; outr(3)=t3+t4

	L1prefetchw srcreg+L1pd, L1pt

	ystore	YMM_TMP1, ymm7			;; Save new r2
	ystore	YMM_TMP2, ymm6			;; Save new r3
	ystore	YMM_TMP3, ymm5			;; Save new r4
	ystore	YMM_TMP4, ymm4			;; Save new r5

	vmovapd	ymm2, [srcreg+srcoff+d1+32]	;; i2
	vaddpd	ymm2, ymm2, [srcreg+srcoff+4*d1+32] ;; i25a=i2+i5
	vmovapd	ymm3, [srcreg+srcoff+2*d1+32]	;; i3
	vaddpd	ymm3, ymm3, [srcreg+srcoff+3*d1+32] ;; i34a=i3+i4
	vmovapd	ymm4, [srcreg+srcoff+d1]	;; r2
	vsubpd	ymm4, ymm4, [srcreg+srcoff+4*d1] ;; r25s=r2-r5
	vmovapd	ymm1, [srcreg+srcoff+2*d1]	;; r3
	vsubpd	ymm1, ymm1, [srcreg+srcoff+3*d1] ;; r34s=r3-r4

	L1prefetchw srcreg+d1+L1pd, L1pt

	ystore	[srcreg], ymm0			;; Save R1

	vaddpd	ymm5, ymm2, ymm3		;; i25a+i34a
	vmovapd	ymm0, YMM_P309
	vmulpd	ymm6, ymm2, ymm0		;; cos2*i25a
	vmovapd	ymm7, YMM_P809
	vmulpd	ymm2, ymm2, ymm7		;; -cos4*i25a
	vmulpd	ymm7, ymm7, ymm3		;; -cos4*i34a
	vmulpd	ymm3, ymm3, ymm0		;; cos2*i34a

	vsubpd	ymm6, ymm6, ymm7		;; cos2*i25a + cos4*i34a
	vsubpd	ymm2, ymm3, ymm2		;; cos4*i25a + cos2*i34a

	L1prefetchw srcreg+2*d1+L1pd, L1pt

	vmovapd	ymm7, YMM_P951
	vmulpd	ymm0, ymm7, ymm4		;; sin2*r25s
	vmulpd	ymm3, ymm7, ymm1		;; sin2*r34s
	vmovapd	ymm7, YMM_P588
	vmulpd	ymm4, ymm7, ymm4		;; sin4*r25s
	vmulpd	ymm1, ymm7, ymm1		;; sin4*r34s
	vmovapd	ymm7, [srcreg+srcoff+32]	;; i1
	vaddpd	ymm6, ymm6, ymm7		;; t5=cos2*i25a + cos4*i34a + i1
	vaddpd	ymm2, ymm2, ymm7		;; t7=cos4*i25a + cos2*i34a + i1
	vsubpd	ymm4, ymm4, ymm3		;; t8=sin4*r25s - sin2*r34s
	vaddpd	ymm0, ymm0, ymm1		;; t6=sin2*r25s + sin4*r34s

	L1prefetchw srcreg+3*d1+L1pd, L1pt

	vsubpd	ymm1, ymm2, ymm4		;; outi(3)=t7-t8
	vaddpd	ymm2, ymm2, ymm4		;; outi(2)=t7+t8
	vsubpd	ymm4, ymm6, ymm0		;; outi(4)=t5-t6
	vaddpd	ymm6, ymm6, ymm0		;; outi(1)=t5+t6
	vaddpd	ymm5, ymm5, ymm7		;; outi(0)=i1+i25a+i34a

no bcast vmovapd ymm3, [screg1+cosoff]		;; cosine/sine
bcast	vbroadcastsd ymm3, Q [screg1+cosoff]	;; cosine/sine
	vmulpd	ymm7, ymm3, YMM_TMP4		;; A5 = R5 * cosine/sine
	vmulpd	ymm0, ymm3, YMM_TMP1		;; A2 = R2 * cosine/sine
	vaddpd	ymm7, ymm7, ymm4		;; A5 = A5 + I5
	vsubpd	ymm0, ymm0, ymm6		;; A2 = A2 - I2
	vmulpd	ymm4, ymm4, ymm3		;; B5 = I5 * cosine/sine
	vmulpd	ymm6, ymm6, ymm3		;; B2 = I2 * cosine/sine
	vsubpd	ymm4, ymm4, YMM_TMP4		;; B5 = B5 - R5
	vaddpd	ymm6, ymm6, YMM_TMP1		;; B2 = B2 + R2
	ystore	[srcreg+32], ymm5		;; Save I1
no bcast vmovapd ymm3, [screg1]			;; sine
bcast	vbroadcastsd ymm3, Q [screg1]		;; sine
	vmulpd	ymm7, ymm7, ymm3		;; A5 = A5 * sine (new R5)
	vmulpd	ymm0, ymm0, ymm3		;; A2 = A2 * sine (new R2)
	vmulpd	ymm4, ymm4, ymm3		;; B5 = B5 * sine (new I5)
	vmulpd	ymm6, ymm6, ymm3		;; B2 = B2 * sine (new I2)

	L1prefetchw srcreg+4*d1+L1pd, L1pt

no bcast vmovapd ymm3, [screg2+cosoff]		;; cosine/sine
bcast	vbroadcastsd ymm3, Q [screg2+cosoff]	;; cosine/sine
	vmulpd	ymm5, ymm3, YMM_TMP2 		;; A3 = R3 * cosine/sine
	vsubpd	ymm5, ymm5, ymm2		;; A3 = A3 - I3
	vmulpd	ymm2, ymm2, ymm3		;; B3 = I3 * cosine/sine
	ystore	[srcreg+d1], ymm0		;; Save R2
	vmulpd	ymm0, ymm3, YMM_TMP3		;; A4 = R4 * cosine/sine
	vaddpd	ymm0, ymm0, ymm1		;; A4 = A4 + I4
	vmulpd	ymm1, ymm1, ymm3		;; B4 = I4 * cosine/sine
	vaddpd	ymm2, ymm2, YMM_TMP2		;; B3 = B3 + R3
	vsubpd	ymm1, ymm1, YMM_TMP3		;; B4 = B4 - R4
no bcast vmovapd ymm3, [screg2]			;; sine
bcast	vbroadcastsd ymm3, Q [screg2]		;; sine
	vmulpd	ymm5, ymm5, ymm3		;; A3 = A3 * sine (new R3)
	vmulpd	ymm0, ymm0, ymm3		;; A4 = A4 * sine (new R4)
	vmulpd	ymm2, ymm2, ymm3		;; B3 = B3 * sine (new I3)
	vmulpd	ymm1, ymm1, ymm3		;; B4 = B4 * sine (new I4)

	ystore	[srcreg+d1+32], ymm6		;; Save I2
	ystore	[srcreg+2*d1], ymm5		;; Save R3
	ystore	[srcreg+2*d1+32], ymm2		;; Save I3
	ystore	[srcreg+3*d1], ymm0		;; Save R4
	ystore	[srcreg+3*d1+32], ymm1		;; Save I4
	ystore	[srcreg+4*d1], ymm7		;; Save R5
	ystore	[srcreg+4*d1+32], ymm4		;; Save I5

	bump	srcreg, srcinc
	bump	screg, scinc
	ENDM

;; 64-bit version

IFDEF X86_64

yr5_5c_djbfft_cmn_preload MACRO
	vmovapd	ymm13, YMM_P309			;; cos2
	vmovapd	ymm14, YMM_P951			;; sin2
	vmovapd	ymm15, YMM_P588			;; sin4
	ENDM

yr5_5c_djbfft_cmn MACRO srcreg,srcinc,srcoff,d1,bcast,screg1,screg2,cosoff,screg,scinc,maxrpt,L1pt,L1pd
	IF maxrpt MOD 5 EQ 0
	yr5_5c_djbfft_unroll -1,noexec,noexec,exec,srcreg,srcinc,srcoff,d1,bcast,screg1,screg2,cosoff,screg,scinc,L1pt,L1pd
	yr5_5c_djbfft_unroll 0,noexec,exec,exec,srcreg,srcinc,srcoff,d1,bcast,screg1,screg2,cosoff,screg,scinc,L1pt,L1pd
	yr5_5c_djbfft_unroll 1,exec,exec,exec,srcreg,srcinc,srcoff,d1,bcast,screg1,screg2,cosoff,screg,scinc,L1pt,L1pd
	yr5_5c_djbfft_unroll 2,exec,exec,exec,srcreg,srcinc,srcoff,d1,bcast,screg1,screg2,cosoff,screg,scinc,L1pt,L1pd
	yr5_5c_djbfft_unroll 3,exec,exec,exec,srcreg,srcinc,srcoff,d1,bcast,screg1,screg2,cosoff,screg,scinc,L1pt,L1pd
	yr5_5c_djbfft_unroll 4,exec,exec,noexec,srcreg,srcinc,srcoff,d1,bcast,screg1,screg2,cosoff,screg,scinc,L1pt,L1pd
	yr5_5c_djbfft_unroll 5,exec,noexec,noexec,srcreg,srcinc,srcoff,d1,bcast,screg1,screg2,cosoff,screg,scinc,L1pt,L1pd
	bump	srcreg, 5*srcinc
	bump	screg, 5*scinc
	ELSEIF maxrpt MOD 4 EQ 0
	yr5_5c_djbfft_unroll -1,noexec,noexec,exec,srcreg,srcinc,srcoff,d1,bcast,screg1,screg2,cosoff,screg,scinc,L1pt,L1pd
	yr5_5c_djbfft_unroll 0,noexec,exec,exec,srcreg,srcinc,srcoff,d1,bcast,screg1,screg2,cosoff,screg,scinc,L1pt,L1pd
	yr5_5c_djbfft_unroll 1,exec,exec,exec,srcreg,srcinc,srcoff,d1,bcast,screg1,screg2,cosoff,screg,scinc,L1pt,L1pd
	yr5_5c_djbfft_unroll 2,exec,exec,exec,srcreg,srcinc,srcoff,d1,bcast,screg1,screg2,cosoff,screg,scinc,L1pt,L1pd
	yr5_5c_djbfft_unroll 3,exec,exec,noexec,srcreg,srcinc,srcoff,d1,bcast,screg1,screg2,cosoff,screg,scinc,L1pt,L1pd
	yr5_5c_djbfft_unroll 4,exec,noexec,noexec,srcreg,srcinc,srcoff,d1,bcast,screg1,screg2,cosoff,screg,scinc,L1pt,L1pd
	bump	srcreg, 4*srcinc
	bump	screg, 4*scinc
	ELSEIF maxrpt MOD 3 EQ 0
	yr5_5c_djbfft_unroll -1,noexec,noexec,exec,srcreg,srcinc,srcoff,d1,bcast,screg1,screg2,cosoff,screg,scinc,L1pt,L1pd
	yr5_5c_djbfft_unroll 0,noexec,exec,exec,srcreg,srcinc,srcoff,d1,bcast,screg1,screg2,cosoff,screg,scinc,L1pt,L1pd
	yr5_5c_djbfft_unroll 1,exec,exec,exec,srcreg,srcinc,srcoff,d1,bcast,screg1,screg2,cosoff,screg,scinc,L1pt,L1pd
	yr5_5c_djbfft_unroll 2,exec,exec,noexec,srcreg,srcinc,srcoff,d1,bcast,screg1,screg2,cosoff,screg,scinc,L1pt,L1pd
	yr5_5c_djbfft_unroll 3,exec,noexec,noexec,srcreg,srcinc,srcoff,d1,bcast,screg1,screg2,cosoff,screg,scinc,L1pt,L1pd
	bump	srcreg, 3*srcinc
	bump	screg, 3*scinc
	ELSEIF maxrpt MOD 2 EQ 0
	yr5_5c_djbfft_unroll -1,noexec,noexec,exec,srcreg,srcinc,srcoff,d1,bcast,screg1,screg2,cosoff,screg,scinc,L1pt,L1pd
	yr5_5c_djbfft_unroll 0,noexec,exec,exec,srcreg,srcinc,srcoff,d1,bcast,screg1,screg2,cosoff,screg,scinc,L1pt,L1pd
	yr5_5c_djbfft_unroll 1,exec,exec,noexec,srcreg,srcinc,srcoff,d1,bcast,screg1,screg2,cosoff,screg,scinc,L1pt,L1pd
	yr5_5c_djbfft_unroll 2,exec,noexec,noexec,srcreg,srcinc,srcoff,d1,bcast,screg1,screg2,cosoff,screg,scinc,L1pt,L1pd
	bump	srcreg, 2*srcinc
	bump	screg, 2*scinc
	ELSE
	yr5_5c_djbfft_unroll -1,noexec,noexec,exec,srcreg,srcinc,srcoff,d1,bcast,screg1,screg2,cosoff,screg,scinc,L1pt,L1pd
	yr5_5c_djbfft_unroll 0,noexec,exec,noexec,srcreg,srcinc,srcoff,d1,bcast,screg1,screg2,cosoff,screg,scinc,L1pt,L1pd
	yr5_5c_djbfft_unroll 1,exec,noexec,noexec,srcreg,srcinc,srcoff,d1,bcast,screg1,screg2,cosoff,screg,scinc,L1pt,L1pd
	bump	srcreg, srcinc
	bump	screg, scinc
	ENDIF
	ENDM

yr5_5c_djbfft_unroll MACRO iter,prev,this,next,srcreg,srcinc,srcoff,d1,bcast,screg1,screg2,cosoff,screg,scinc,L1pt,L1pd

;; On first call, init registers

	IF iter EQ -1
y0	TEXTEQU	<ymm0>
y1	TEXTEQU	<ymm1>
y2	TEXTEQU	<ymm2>
y3	TEXTEQU	<ymm3>
y4	TEXTEQU	<ymm4>
y5	TEXTEQU	<ymm5>
y6	TEXTEQU	<ymm6>
y7	TEXTEQU	<ymm7>
y8	TEXTEQU	<ymm8>
y9	TEXTEQU	<ymm9>
y10	TEXTEQU	<ymm10>
y11	TEXTEQU	<ymm11>
y12	TEXTEQU	<ymm12>
	ENDIF

;; On later calls, y0-y12 contain this r2, this r3, prev R5, this r5, prev I5,
;; prev A3, prev B4, prev sine, prev I2, free, prev R2, prev R4, prev B3.
;; ymm13, ymm14 and ymm15 are preloaded constants cos2, sin2 and sin4.

this	vaddpd	y9, y0, y3				;; r25a=r2+r5			; 1-3
prev	vmulpd	y5, y5, y7				;; A3 = A3 * sine (final R3)	; 1-5
prev	ystore	[srcreg+(iter-1)*srcinc+4*d1], y2	;; Save R5			; 1
this	vmovapd	y2, [srcreg+iter*srcinc+srcoff+3*d1]	;; r4

prev	ystore	[srcreg+(iter-1)*srcinc+d1], y10	;; Save R2			; 2
this	vaddpd	y10, y1, y2				;; r34a=r3+r4			; 2-4
prev	vmulpd	y6, y6, y7				;; B4 = B4 * sine (final I4)	; 2-6
prev	ystore	[srcreg+(iter-1)*srcinc+4*d1+32], y4	;; Save I5			; 3
this	vmovapd	y4, [srcreg+iter*srcinc+srcoff+d1+32]	;; i2

this	vsubpd	y0, y0, y3				;; r25s=r2-r5			; 3-5
prev	vmulpd	y12, y12, y7				;; B3 = B3 * sine (final I3)	; 3-7
this	vmovapd	y7, [srcreg+iter*srcinc+srcoff+4*d1+32] ;; i5

this	vsubpd	y1, y1, y2				;; r34s=r3-r4			; 4-6
this	vmulpd	y2, ymm13, y9				;; cos2*r25a			; 4-8
prev	ystore	[srcreg+(iter-1)*srcinc+d1+32], y8	;; Save I2			; 4
this	vmovapd	y8, YMM_P809				;; -cos4

this	vsubpd	y3, y4, y7				;; i25s=i2-i5			; 5-7
prev	ystore	[srcreg+(iter-1)*srcinc+3*d1], y11	;; Save R4			; 5
this	vmulpd	y11, y8, y10				;; -cos4*r34a			; 5-9
prev	ystore	[srcreg+(iter-1)*srcinc+2*d1], y5	;; Save R3			; 6
this	vmovapd	y5, [srcreg+iter*srcinc+srcoff+2*d1+32] ;; i3

this	vaddpd	y4, y4, y7				;; i25a=i2+i5			; 6-8
this	vmulpd	y8, y8, y9				;; -cos4*r25a			; 6-10
prev	ystore	[srcreg+(iter-1)*srcinc+3*d1+32], y6	;; Save I4			; 7
this	vmovapd	y6, [srcreg+iter*srcinc+srcoff+3*d1+32] ;; i4

this	vsubpd	y7, y5, y6				;; i34s=i3-i4			; 7-9
prev	ystore	[srcreg+(iter-1)*srcinc+2*d1+32], y12	;; Save I3			; 8
this	vmulpd	y12, ymm13, y10				;; cos2*r34a			; 7-11

this	vaddpd	y5, y5, y6				;; i34a=i3+i4			; 8-10
this	vmulpd	y6, ymm14, y3				;; sin2*i25s			; 8-12
this	L1prefetchw srcreg+iter*srcinc+L1pd, L1pt

this	vaddpd	y9, y9, y10				;; r25a + r34a			; 9-11
this	vmulpd	y3, ymm15, y3				;; sin4*i25s			; 9-13

this	vsubpd	y2, y2, y11				;; cos2*r25a + cos4*r34a	; 10-12
this	vmulpd	y11, ymm15, y7				;; sin4*i34s			; 10-14

this	vmovapd	y10, [srcreg+iter*srcinc+srcoff] ;; r1
this	vsubpd	y8, y10, y8				;; t3 = r1 + cos4*r25a		; 11-13
this	vmulpd	y7, ymm14, y7				;; sin2*i34s			; 11-15

this	vaddpd	y9, y10, y9				;; outr(0) = r1 + r25a + r34a	; 12-14
this	ystore	[srcreg+iter*srcinc], y9		;; Save R1			; 15
this	vmulpd	y9, ymm13, y4				;; cos2*i25a			; 12-16

this	vaddpd	y2, y2, y10				;; t1=cos2*r25a + cos4*r34a + r1 ; 13-15
this	vmovapd	y10, YMM_P809				;; -cos4
this	L1prefetchw srcreg+iter*srcinc+d1+L1pd, L1pt

this	vaddpd	y8, y8, y12				;; t3 += cos2*r34a		; 14-16
this	vmulpd	y12, y10, y5				;; -cos4*i34a			; 13-17
this	vmulpd	y10, y10, y4				;; -cos4*i25a			; 14-18

this	vaddpd	y6, y6, y11				;; t2=sin2*i25s + sin4*i34s	; 15-17
this	vmulpd	y11, ymm13, y5				;; cos2*i34a			; 15-19

this	vsubpd	y3, y3, y7				;; t4=sin4*i25s - sin2*i34s	; 16-18
this	vmulpd	y7, ymm14, y0				;; sin2*r25s			; 16-20

this	vaddpd	y4, y4, y5				;; i25a+i34a			; 17-19
this	vmulpd	y5, ymm15, y1				;; sin4*r34s			; 17-21

this	vsubpd	y9, y9, y12				;; cos2*i25a + cos4*i34a	; 18-20
this	vmulpd	y1, ymm14, y1				;; sin2*r34s			; 18-22
this	L1prefetchw srcreg+iter*srcinc+2*d1+L1pd, L1pt

this	vmovapd	y12, [srcreg+iter*srcinc+srcoff+32]	;; I1
this	vsubpd	y10, y12, y10				;; t7=i1 + cos4*i25a		; 19-21
this	vmulpd	y0, ymm15, y0				;; sin4*r25s			; 19-23

this	vaddpd	y4, y12, y4				;; outi(0)=i1+i25a+i34a		; 20-22
this	ystore	[srcreg+iter*srcinc+32], y4		;; Save I1
this no bcast vmovapd y4, [screg1+iter*scinc+cosoff]	;; cosine/sine
this bcast vbroadcastsd y4, Q [screg1+iter*scinc+cosoff] ;; cosine/sine

this	vaddpd	y9, y9, y12				;; t5=cos2*i25a + cos4*i34a + i1 ; 21-23

this	vaddpd	y10, y10, y11				;; t7+=cos2*i34a		; 22-24

this	vaddpd	y7, y7, y5				;; t6=sin2*r25s + sin4*r34s	; 23-25
this	L1prefetchw srcreg+iter*srcinc+3*d1+L1pd, L1pt

this	vsubpd	y0, y0, y1				;; t8=sin4*r25s - sin2*r34s	; 24-26

this	vaddpd	y1, y2, y6				;; outr(4)=t1+t2		; 25-27

this	vsubpd	y2, y2, y6				;; outr(1)=t1-t2		; 26-28

this	vsubpd	y6, y9, y7				;; outi(4)=t5-t6		; 27-29

this	vaddpd	y9, y9, y7				;; outi(1)=t5+t6		; 28-30
this	vmulpd	y7, y1, y4				;; A5 = R5 * cosine/sine	; 28-32
this	L1prefetchw srcreg+iter*srcinc+4*d1+L1pd, L1pt

this	vaddpd	y5, y8, y3				;; outr(3)=t3+t4		; 29-31
this	vmulpd	y11, y2, y4				;; A2 = R2 * cosine/sine	; 29-33

this	vsubpd	y8, y8, y3				;; outr(2)=t3-t4		; 30-32
this	vmulpd	y3, y6, y4				;; B5 = I5 * cosine/sine	; 30-34

this	vsubpd	y12, y10, y0				;; outi(3)=t7-t8		; 31-33
this	vmulpd	y4, y9, y4				;; B2 = I2 * cosine/sine	; 31-35

this	vaddpd	y10, y10, y0				;; outi(2)=t7+t8		; 32-34
this no bcast vmovapd y0, [screg2+iter*scinc+cosoff]	;; cosine/sine
this bcast vbroadcastsd y0, Q [screg2+iter*scinc+cosoff] ;; cosine/sine

this	vaddpd	y7, y7, y6				;; A5 = A5 + I5			; 33-35
this	vmulpd	y6, y5, y0				;; A4 = R4 * cosine/sine	; 32-36
this next yloop_unrolled_one

this	vsubpd	y11, y11, y9				;; A2 = A2 - I2			; 34-36
this	vmulpd	y9, y8, y0				;; A3 = R3 * cosine/sine	; 33-37

this	vsubpd	y3, y3, y1				;; B5 = B5 - R5			; 35-37
this	vmulpd	y1, y12, y0				;; B4 = I4 * cosine/sine	; 34-38
this	vmulpd	y0, y10, y0				;; B3 = I3 * cosine/sine	; 35-39

this	vaddpd	y4, y4, y2				;; B2 = B2 + R2			; 36-38
this no bcast vmovapd y2, [screg1+iter*scinc]		;; sine
this bcast vbroadcastsd y2, Q [screg1+iter*scinc]	;; sine
this	vmulpd	y7, y7, y2				;; A5 = A5 * sine (final R5)	; 36-40

this	vaddpd	y6, y6, y12				;; A4 = A4 + I4			; 37-39
this	vmulpd	y11, y11, y2				;; A2 = A2 * sine (final R2)	; 37-41
this no bcast vmovapd y12, [screg2+iter*scinc]		;; sine
this bcast vbroadcastsd y12, Q [screg2+iter*scinc]	;; sine

this	vsubpd	y9, y9, y10				;; A3 = A3 - I3			; 38-40
this	vmulpd	y3, y3, y2				;; B5 = B5 * sine (final I5)	; 38-42
next	vmovapd	y10, [srcreg+(iter+1)*srcinc+srcoff+d1]	;; r2

this	vsubpd	y1, y1, y5				;; B4 = B4 - R4			; 39-41
this	vmulpd	y4, y4, y2				;; B2 = B2 * sine (final I2)	; 39-43
next	vmovapd	y5, [srcreg+(iter+1)*srcinc+srcoff+4*d1] ;; r5

this	vaddpd	y0, y0, y8				;; B3 = B3 + R3			; 40-42
this	vmulpd	y6, y6, y12				;; A4 = A4 * sine (final R4)	; 40-44
next	vmovapd	y2, [srcreg+(iter+1)*srcinc+srcoff+2*d1] ;; r3

;; Shuffle register assignments so that y0-y12 in the next call has this r2, this r3, prev R5,
;; this r5, prev I5, prev A3, prev B4, prev sine, prev I2, free, prev R2, prev R4, prev B3.

ytmp	TEXTEQU	y0
y0	TEXTEQU	y10
y10	TEXTEQU	y11
y11	TEXTEQU	y6
y6	TEXTEQU	y1
y1	TEXTEQU	y2
y2	TEXTEQU	y7
y7	TEXTEQU	y12
y12	TEXTEQU	ytmp
ytmp	TEXTEQU	y3
y3	TEXTEQU	y5
y5	TEXTEQU	y9
y9	TEXTEQU	y8
y8	TEXTEQU y4
y4	TEXTEQU ytmp

	ENDM

;; Haswell FMA3 version

IF (@INSTR(,%yarch,<FMA3>) NE 0)

yr5_5c_djbfft_cmn_preload MACRO
	ENDM

;; 10 loads, 10 stores, 4 sin/cos loads, 9 const loads, 40 FMAs, 6 muls, 18 reg copies  = 97 uops.  97/4 = 24.25 clock min.
;; Timed at 26 clocks
yr5_5c_djbfft_unroll MACRO iter,prev,this,next,srcreg,srcinc,srcoff,d1,bcast,screg1,screg2,cosoff,screg,scinc,L1pt,L1pd

;; On first call, init registers

	IF iter EQ -1
y0	TEXTEQU	<ymm0>
y1	TEXTEQU	<ymm1>
y2	TEXTEQU	<ymm2>
y3	TEXTEQU	<ymm3>
y4	TEXTEQU	<ymm4>
y5	TEXTEQU	<ymm5>
y6	TEXTEQU	<ymm6>
y7	TEXTEQU	<ymm7>
y8	TEXTEQU	<ymm8>
y9	TEXTEQU	<ymm9>
y10	TEXTEQU	<ymm10>
y11	TEXTEQU	<ymm11>
y12	TEXTEQU	<ymm12>
y13	TEXTEQU	<ymm13>
y14	TEXTEQU	<ymm14>
y15	TEXTEQU	<ymm15>
	ENDIF

;; On later calls, prev R2,I2,I5,R3S,I3S,R4S,I4S are in y0-6, and this r25a,r25s,i25a,i25s,r34a,r34s,cos2 are in y7-13.
;; The remaining registers are free.


this	vmovapd	y15, [srcreg+iter*srcinc+srcoff+2*d1+32]	;; i3
this	yfmaddpd y14, y13, y7, [srcreg+iter*srcinc+srcoff]	;; t1 = r1 + cos2*r25a			; 1-5 (used at 7)
this	yfmaddpd y13, y13, y9, [srcreg+iter*srcinc+srcoff+32]	;; t5 = i1 + cos2*i25a			; 1-5 (used at 7)		free cos2

prev	ystore	[srcreg+(iter-1)*srcinc+4*d1+32], y2		;; Save I5				; 27+1
this	vmovapd	y2, [srcreg+iter*srcinc+srcoff+3*d1+32]		;; i4
prev	ystore	[srcreg+(iter-1)*srcinc+d1], y0			;; Save R2				; 28+1
this	vmovapd	y0, YMM_ONE
prev	ystore	[srcreg+(iter-1)*srcinc+d1+32], y1		;; Save I2				; 28+2
this	yfmaddpd y1, y15, y0, y2				;; i34a=i3+i4				; 2-6 (used at 7)
this	yfmsubpd y15, y15, y0, y2				;; i34s=i3-i4				; 2-6 (used at 11)

prev no bcast vmovapd y2, [screg2+(iter-1)*scinc+cosoff]	;; cosine/sine for R3/I3/R4/I4
prev bcast vbroadcastsd y2, Q [screg2+(iter-1)*scinc+cosoff]	;; cosine/sine for R3/I3/R4/I4
prev	yfmaddpd y0, y5, y2, y6					;; R4S * cosine/sine + I4S (final R4)	; 3-7
prev	yfmsubpd y6, y6, y2, y5					;; I4S * cosine/sine - R4S (final I4)	; 3-7

this	vmovapd	y5, [srcreg+iter*srcinc+srcoff]			;; r1
prev	ystore	[srcreg+(iter-1)*srcinc+3*d1], y0		;; Save R4				; 8
this	vmovapd	y0, YMM_ONE
prev	ystore	[srcreg+(iter-1)*srcinc+3*d1+32], y6		;; Save I4				; 8+1
this	yfmaddpd y6, y5, y0, y7					;; tr0 = r1 + r25a			; 4-8 (used at 9)
this	yfnmaddpd y7, y7, YMM_P809, y5				;; t3 = r1 - -cos4*r25a			; 4-8 (used at 9)		free r25a, r1

this	vmovapd	y5, [srcreg+iter*srcinc+srcoff+32]		;; i1
this	yfmaddpd y0, y5, y0, y9					;; ti0 = i1 + i25a			; 5-9 (used at 10)
this	yfnmaddpd y9, y9, YMM_P809, y5				;; t7 = i1 - -cos4*i25a			; 5-9 (used at 10)		free i25a, i1

prev	yfmsubpd y5, y3, y2, y4					;; R3S * cosine/sine - I3S (final R3)	; 6-10
prev	yfmaddpd y4, y4, y2, y3					;; I3S * cosine/sine + R3S (final I3)	; 6-10				free cos/sin
this	L1prefetchw srcreg+iter*srcinc+L1pd, L1pt

this	vmovapd	y2, YMM_P809					;; -cos4
this	yfnmaddpd y14, y2, y11, y14				;; t1 = t1 - -cos4*r34a			; 7-11 (used at 12)
this	yfnmaddpd y13, y2, y1, y13				;; t5 = t5 - -cos4*i34a			; 7-11 (used at 12)		free -cos4

this	vmovapd	y3, YMM_P588_P951				;; sin4/sin2
this	yfmaddpd y2, y3, y12, y8				;; t6/sin2 = r25s + sin4/sin2*r34s	; 8-12 (used at 17)
this	yfmsubpd y8, y3, y8, y12				;; t8/sin2 = sin4/sin2*r25s - r34s	; 8-12 (used at 20)		free r25s, r34s
this	L1prefetchw srcreg+iter*srcinc+d1+L1pd, L1pt

this	vmovapd	y12, YMM_P309					;; cos2
prev	ystore	[srcreg+(iter-1)*srcinc+2*d1], y5		;; Save R3				; 11
this	vmovapd y5, YMM_ONE
no this next vmovapd y5, YMM_ONE
this	yfmaddpd y6, y6, y5, y11				;; tr0 = tr0 + r34a (final R1)		; 9-13
this	yfmaddpd y7, y12, y11, y7				;; t3 = t3 + cos2*r34a			; 9-13 (used at 14)		free r34a

this	yfmaddpd y0, y0, y5, y1					;; ti0 = ti0 + i34a (final I1)		; 10-14
this	yfmaddpd y9, y12, y1, y9				;; t7 = t7 + cos2*i34a			; 10-14 (used at 15)		free i34a, cos2

this	vmovapd y11, YMM_P951					;; sin2
this no bcast vmovapd y12, [screg1+iter*scinc]			;; sine for R2/I2/R5/I5
this bcast vbroadcastsd y12, Q [screg1+iter*scinc]		;; sine for R2/I2/R5/I5
this	yfmaddpd y1, y3, y15, y10				;; t2/sin2 = i25s + sin4/sin2*i34s	; 11-15 (used at 17)
prev	ystore	[srcreg+(iter-1)*srcinc+2*d1+32], y4		;; Save I3				; 11+1
this	vmulpd	y4, y11, y12					;; sin2sine25 = sin2 * sine25		; 11-15 (used at 17)

this	vmulpd	y14, y14, y12					;; t1s = t1 * sine25			; 12-16 (used at 17)
this	vmulpd	y13, y13, y12					;; t5s = t5 * sine25			; 12-16	(used at 17)		free sine25
this no bcast vmovapd y12, [screg2+iter*scinc]			;; sine for R3/I3/R4/I4
this bcast vbroadcastsd y12, Q [screg2+iter*scinc]		;; sine for R3/I3/R4/I4

this	yfmsubpd y3, y3, y10, y15				;; t4/sin2 = sin4/sin2*i25s - i34s	; 13-17 (used at 19)		free i25s, i34s, sin4/sin2
this	vmulpd	y11, y11, y12					;; sin2sine34 = sin2 * sine34		; 13-17 (used at 19)

next	vmovapd	y15, [srcreg+(iter+1)*srcinc+srcoff+d1]		;; r2
next	vmovapd	y10, [srcreg+(iter+1)*srcinc+srcoff+4*d1]	;; r5
this	ystore	[srcreg+iter*srcinc], y6			;; Save R1				; 14
next	yfmaddpd y6, y15, y5, y10				;; r25a=r2+r5				; 14-18 (used at next 1)
this	vmulpd	y7, y7, y12					;; t3s = t3 * sine34			; 14-18 (used at 19)

next	yfmsubpd y15, y15, y5, y10				;; r25s=r2-r5				; 15-19 (used at next 8)
this	vmulpd	y9, y9, y12					;; t7s = t7 * sine34			; 15-19 (used at 20)		free sine34

next	vmovapd	y12, [srcreg+(iter+1)*srcinc+srcoff+d1+32]	;; i2
next	vmovapd	y10, [srcreg+(iter+1)*srcinc+srcoff+4*d1+32]	;; i5
this	ystore	[srcreg+iter*srcinc+32], y0			;; Save I1				; 15
next	yfmaddpd y0, y12, y5, y10				;; i25a=i2+i5				; 16-20 (used at next 1)
next	yfmsubpd y12, y12, y5, y10				;; i25s=i2-i5				; 16-20	(used at next 11)
this	L1prefetchw srcreg+iter*srcinc+2*d1+L1pd, L1pt

this	yfmaddpd y10, y1, y4, y14				;; t1s + t2/sin2 * sin2sine25 (new R5S)	; 17-21 (used at 22)
this	yfnmaddpd y5, y2, y4, y13				;; t5s - t6/sin2 * sin2sine25 (new I5S)	; 17-21 (used at 22)
this	L1prefetchw srcreg+iter*srcinc+3*d1+L1pd, L1pt

this	yfnmaddpd y1, y1, y4, y14				;; t1s - t2/sin2 * sin2sine25 (new R2S)	; 18-22 (used at 23)
this	yfmaddpd y2, y2, y4, y13				;; t5s + t6/sin2 * sin2sine25 (new I2S)	; 18-22 (used at 23)
next	vmovapd	y14, [srcreg+(iter+1)*srcinc+srcoff+2*d1]	;; r3

this	yfmaddpd y4, y3, y11, y7				;; t3s + t4/sin2 * sin2sine34 (new R4S)	; 19-23 (used at next 3)
this	yfnmaddpd y3, y3, y11, y7				;; t3s - t4/sin2 * sin2sine34 (new R3S)	; 19-23 (used at next 6)
next	vmovapd	y13, [srcreg+(iter+1)*srcinc+srcoff+3*d1]	;; r4

this	yfnmaddpd y7, y8, y11, y9				;; t7s - t8/sin2 * sin2sine34 (new I4S)	; 20-24 (used at next 3)
this	yfmaddpd y8, y8, y11, y9				;; t7s + t8/sin2 * sin2sine34 (new I3S)	; 20-24 (used at next 6)

next	vmovapd	y11, YMM_ONE
next	yfmaddpd y9, y14, y11, y13				;; r34a=r3+r4				; 21-25 (used at next 7)
next	yfmsubpd y14, y14, y11, y13				;; r34s=r3-r4				; 21-25 (used at next 8)
this	L1prefetchw srcreg+iter*srcinc+4*d1+L1pd, L1pt

this no bcast vmovapd y11, [screg1+iter*scinc+cosoff]		;; cosine/sine for R2/I2/R5/I5
this bcast vbroadcastsd y11, Q [screg1+iter*scinc+cosoff]	;; cosine/sine for R2/I2/R5/I5
this	yfmaddpd y13, y10, y11, y5				;; R5S * cosine/sine + I5S (final R5)	; 22-26
this	yfmsubpd y5, y5, y11, y10				;; I5S * cosine/sine - R5S (final I5)	; 22-26

this	yfmsubpd y10, y1, y11, y2				;; R2S * cosine/sine - I2S (final R2)	; 23-27
this	yfmaddpd y2, y2, y11, y1				;; I2S * cosine/sine + R2S (final I2)	; 23-27
next	vmovapd	y11, YMM_P309					;; cos2
this next yloop_unrolled_one
this	ystore	[srcreg+iter*srcinc+4*d1], y13			;; Save R5				; 27

;; Shuffle register assignments so that this R2,I2,I5,R3S,I3S,R4S,I4S are in y0-6, and next r25a,r25s,i25a,i25s,r34a,r34s,cos2 are in y7-13.

ytmp	TEXTEQU	y0
y0	TEXTEQU	y10
y10	TEXTEQU	y12
y12	TEXTEQU	y14
y14	TEXTEQU	y1
y1	TEXTEQU	y2
y2	TEXTEQU	y5
y5	TEXTEQU	y4
y4	TEXTEQU	y8
y8	TEXTEQU	y15
y15	TEXTEQU	y13
y13	TEXTEQU	y11
y11	TEXTEQU	y9
y9	TEXTEQU ytmp
ytmp	TEXTEQU y6
y6	TEXTEQU y7
y7	TEXTEQU ytmp

	ENDM

ENDIF

ENDIF

;;
;; ************************************* five-complex-djbunfft variants ******************************************
;;

;; The standard version
yr5_5cl_five_complex_djbunfft_preload MACRO
	yr5_5c_djbunfft_cmn_preload
	ENDM
yr5_5cl_five_complex_djbunfft MACRO srcreg,srcinc,d1,screg,scinc,maxrpt,L1pt,L1pd
	yr5_5c_djbunfft_cmn srcreg,srcinc,d1,noexec,screg,screg+64,32,screg,scinc,maxrpt,L1pt,L1pd
	ENDM

; Like 5cl but uses a ten-reals sin/cos table
yr5_r5cl_five_complex_djbunfft_preload MACRO
	yr5_5c_djbunfft_cmn_preload
	ENDM
yr5_r5cl_five_complex_djbunfft MACRO srcreg,srcinc,d1,screg,scinc,maxrpt,L1pt,L1pd
	yr5_5c_djbunfft_cmn srcreg,srcinc,d1,noexec,screg+64,screg+192,32,screg,scinc,maxrpt,L1pt,L1pd
	ENDM

; Like the 5cl version except vbroadcastsd is used to reduce sin/cos data
yr5_b5cl_five_complex_djbunfft_preload MACRO
	yr5_5c_djbunfft_cmn_preload
	ENDM
yr5_b5cl_five_complex_djbunfft MACRO srcreg,srcinc,d1,screg,scinc,maxrpt,L1pt,L1pd
	yr5_5c_djbunfft_cmn srcreg,srcinc,d1,exec,screg,screg+16,8,screg,scinc,maxrpt,L1pt,L1pd
	ENDM

; Like b5cl but extracts the sin/cos data to broadcast from
; the ten-real/five_complex sin/cos table
yr5_rb5cl_five_complex_djbunfft_preload MACRO
	yr5_5c_djbunfft_cmn_preload
	ENDM
yr5_rb5cl_five_complex_djbunfft MACRO srcreg,srcinc,d1,screg,scinc,maxrpt,L1pt,L1pd
	yr5_5c_djbunfft_cmn srcreg,srcinc,d1,exec,screg+8,screg+80,40,screg,scinc,maxrpt,L1pt,L1pd
	ENDM


;; Common code to do the 5-complex inverse FFT.
;; First we apply twiddle factors to 4 of the 5 input numbers.
;; A 5-complex inverse FFT is like the forward FFT except all the sin values are negated.

yr5_5c_djbunfft_cmn_preload MACRO
	ENDM
yr5_5c_djbunfft_cmn MACRO srcreg,srcinc,d1,bcast,screg1,screg2,cosoff,screg,scinc,maxrpt,L1pt,L1pd
	vmovapd	ymm2, [srcreg+d1]		;; Load R2
no bcast vmovapd ymm0, [screg1+cosoff]		;; cosine/sine
bcast	vbroadcastsd ymm0, Q [screg1+cosoff]	;; cosine/sine
	vmulpd	ymm1, ymm2, ymm0		;; A2 = R2 * cosine/sine
	vmovapd	ymm3, [srcreg+4*d1]		;; Load R5
	vmulpd	ymm7, ymm3, ymm0		;; A5 = R5 * cosine/sine
	vmovapd	ymm4, [srcreg+d1+32]		;; I2
	vaddpd	ymm1, ymm1, ymm4		;; A2 = A2 + I2
	vmovapd	ymm6, [srcreg+4*d1+32]		;; I5
	vsubpd	ymm7, ymm7, ymm6		;; A5 = A5 - I5
	vmulpd	ymm4, ymm4, ymm0		;; B2 = I2 * cosine/sine
	vmulpd	ymm6, ymm6, ymm0		;; B5 = I5 * cosine/sine
	vsubpd	ymm4, ymm4, ymm2		;; B2 = B2 - R2
	vaddpd	ymm6, ymm6, ymm3		;; B5 = B5 + R5
no bcast vmovapd ymm0, [screg1]			;; sine
bcast	vbroadcastsd ymm0, Q [screg1]		;; sine
	vmulpd	ymm1, ymm1, ymm0		;; A2 = A2 * sine (new R2)
	vmulpd	ymm4, ymm4, ymm0		;; B2 = B2 * sine (new I2)
	vmulpd	ymm7, ymm7, ymm0		;; A5 = A5 * sine (new R5)
	vmulpd	ymm6, ymm6, ymm0		;; B5 = B5 * sine (new I5)

	vsubpd	ymm2, ymm1, ymm7		;; r25s=r2-r5
	vaddpd	ymm1, ymm1, ymm7		;; r25a=r2+r5
	vsubpd	ymm7, ymm4, ymm6		;; i25s=i2-i5
	vaddpd	ymm4, ymm4, ymm6		;; i25a=i2+i5

	ystore	YMM_TMP1, ymm2			;; Save r2-r5
	ystore	YMM_TMP2, ymm4			;; Save i2+i5

	vmovapd	ymm4, [srcreg+3*d1]		;; Load R4
no bcast vmovapd ymm0, [screg2+cosoff]		;; cosine/sine
bcast	vbroadcastsd ymm0, Q [screg2+cosoff]	;; cosine/sine
	vmulpd	ymm5, ymm4, ymm0		;; A4 = R4 * cosine/sine
	vmovapd	ymm6, [srcreg+2*d1]		;; Load R3
	vmulpd	ymm3, ymm6, ymm0		;; A3 = R3 * cosine/sine
	vmovapd	ymm2, [srcreg+3*d1+32]		;; I4
	vsubpd	ymm5, ymm5, ymm2		;; A4 = A4 - I4
	vmulpd	ymm2, ymm2, ymm0		;; B4 = I4 * cosine/sine
	vaddpd	ymm2, ymm2, ymm4		;; B4 = B4 + R4
	vmovapd	ymm4, [srcreg+2*d1+32]		;; I3
	vaddpd	ymm3, ymm3, ymm4		;; A3 = A3 + I3
	vmulpd	ymm4, ymm4, ymm0		;; B3 = I3 * cosine/sine
	vsubpd	ymm4, ymm4, ymm6		;; B3 = B3 - R3
no bcast vmovapd ymm0, [screg2]			;; sine
bcast	vbroadcastsd ymm0, Q [screg2]		;; sine
	vmulpd	ymm5, ymm5, ymm0		;; A4 = A4 * sine (new R4)
	vmulpd	ymm2, ymm2, ymm0		;; B4 = B4 * sine (new I4)
	vmulpd	ymm3, ymm3, ymm0		;; A3 = A3 * sine (new R3)
	vmulpd	ymm4, ymm4, ymm0		;; B3 = B3 * sine (new I3)

	L1prefetchw srcreg+L1pd, L1pt

	vsubpd	ymm6, ymm3, ymm5		;; r34s=r3-r4
	vaddpd	ymm3, ymm3, ymm5		;; r34a=r3+r4
	ystore	YMM_TMP3, ymm6			;; Save r3-r4
	vsubpd	ymm5, ymm4, ymm2		;; i34s=i3-i4
	vaddpd	ymm4, ymm4, ymm2		;; i34a=i3+i4
	ystore	YMM_TMP4, ymm4			;; Save i3+i4

	L1prefetchw srcreg+d1+L1pd, L1pt

	vmulpd	ymm0, ymm1, YMM_P309		;; cos2*r25a
	vmulpd	ymm2, ymm3, YMM_P809		;; -cos4*r34a
	vmulpd	ymm4, ymm7, YMM_P951		;; sin2*i25s
	vmulpd	ymm6, ymm5, YMM_P588		;; sin4*i34s
	vsubpd	ymm0, ymm0, ymm2		;; cos2*r25a + cos4*r34a
	vaddpd	ymm4, ymm4, ymm6		;; t2=sin2*i25s + sin4*i34s
	vmulpd	ymm2, ymm1, YMM_P809		;; -cos4*r25a
	vaddpd	ymm1, ymm1, ymm3		;; r25a + r34a
	vmovapd	ymm6, [srcreg]			;; r1
	vaddpd	ymm0, ymm0, ymm6		;; t1=cos2*r25a + cos4*r34a + r1
	vaddpd	ymm1, ymm1, ymm6		;; outr(0) = r1 + r25a + r34a
	vmulpd	ymm3, ymm3, YMM_P309		;; cos2*r34a
	vmulpd	ymm7, ymm7, YMM_P588		;; sin4*i25s
	vmulpd	ymm5, ymm5, YMM_P951		;; sin2*i34s
	vsubpd	ymm2, ymm3, ymm2		;; cos4*r25a+cos2*r34a
	vsubpd	ymm7, ymm7, ymm5		;; t4=sin4*i25s-sin2*i34s
	vaddpd	ymm2, ymm2, ymm6		;; t3=cos4*r25a+cos2*r34a+r1

	L1prefetchw srcreg+2*d1+L1pd, L1pt

	vsubpd	ymm6, ymm0, ymm4		;; outr(4)=t1-t2
	vaddpd	ymm0, ymm0, ymm4		;; outr(1)=t1+t2
	vsubpd	ymm4, ymm2, ymm7		;; outr(3)=t3-t4
	vaddpd	ymm2, ymm2, ymm7		;; outr(2)=t3+t4

	ystore	[srcreg], ymm1			;; Save R1
	ystore	[srcreg+d1], ymm0		;; Save R2
	ystore	[srcreg+2*d1], ymm2		;; Save R3
	ystore	[srcreg+3*d1], ymm4		;; Save R4
	ystore	[srcreg+4*d1], ymm6		;; Save R5

	vmovapd	ymm0, YMM_TMP1			;; r25s=r2-r5
	vmovapd	ymm2, YMM_TMP2			;; i25a=i2+i5
	vmovapd	ymm1, YMM_TMP3			;; r34s=r3-r4
	vmovapd	ymm3, YMM_TMP4			;; i34a=i3+i4

	L1prefetchw srcreg+3*d1+L1pd, L1pt

	vaddpd	ymm5, ymm2, ymm3		;; i25a+i34a
	vmovapd	ymm4, YMM_P309
	vmulpd	ymm6, ymm2, ymm4		;; cos2*i25a
	vmovapd	ymm7, YMM_P809
	vmulpd	ymm2, ymm2, ymm7		;; -cos4*i25a
	vmulpd	ymm7, ymm3, ymm7		;; -cos4*i34a
	vmulpd	ymm3, ymm3, ymm4		;; cos2*i34a

	L1prefetchw srcreg+4*d1+L1pd, L1pt

	vmulpd	ymm4, ymm0, YMM_P951		;; sin2*r25s
	vsubpd	ymm6, ymm6, ymm7		;; cos2*i25a + cos4*i34a
	vmovapd	ymm7, YMM_P588
	vmulpd	ymm0, ymm0, ymm7		;; sin4*r25s
	vmulpd	ymm7, ymm1, ymm7		;; sin4*r34s
	vmulpd	ymm1, ymm1, YMM_P951		;; sin2*r34s
	vsubpd	ymm2, ymm3, ymm2		;; cos4*i25a + cos2*i34a
	vmovapd	ymm3, [srcreg+32]		;; I1
	vaddpd	ymm6, ymm6, ymm3		;; t5=cos2*i25a + cos4*i34a + i1
	vaddpd	ymm4, ymm4, ymm7		;; t6=sin2*r25s + sin4*r34s
	vaddpd	ymm2, ymm2, ymm3		;; t7=cos4*i25a + cos2*i34a + i1
	vsubpd	ymm0, ymm0, ymm1		;; t8=sin4*r25s - sin2*r34s

	vsubpd	ymm7, ymm6, ymm4		;; outi(1)=t5-t6
	vaddpd	ymm6, ymm6, ymm4		;; outi(4)=t5+t6
	vsubpd	ymm1, ymm2, ymm0		;; outi(2)=t7-t8
	vaddpd	ymm2, ymm2, ymm0		;; outi(3)=t7+t8
	vaddpd	ymm5, ymm5, ymm3		;; outi(0)=i1+i25a+i34a

;;	ystore	[srcreg], ymm4		;; Save R1
	ystore	[srcreg+32], ymm5	;; Save I1
;;	ystore	[srcreg+d1], ymm3	;; Save R2
	ystore	[srcreg+d1+32], ymm7	;; Save I2
;;	ystore	[srcreg+2*d1], ymm3	;; Save R3
	ystore	[srcreg+2*d1+32], ymm1	;; Save I3
;;	ystore	[srcreg+3*d1], ymm1	;; Save R4
	ystore	[srcreg+3*d1+32], ymm2	;; Save I4
;;	ystore	[srcreg+4*d1], ymm1	;; Save R5
	ystore	[srcreg+4*d1+32], ymm6	;; Save I5
	bump	srcreg, srcinc
	bump	screg, scinc
	ENDM

;; 64-bit version

IFDEF X86_64

yr5_5c_djbunfft_cmn_preload MACRO
	vmovapd	ymm13, YMM_P309			;; cos2
	vmovapd	ymm14, YMM_P951			;; sin2
	vmovapd	ymm15, YMM_P588			;; sin4
	ENDM

yr5_5c_djbunfft_cmn MACRO srcreg,srcinc,d1,bcast,screg1,screg2,cosoff,screg,scinc,maxrpt,L1pt,L1pd
	IF maxrpt MOD 5 EQ 0
	yr5_5c_djbunfft_unroll -1,noexec,noexec,exec,srcreg,srcinc,d1,bcast,screg1,screg2,cosoff,screg,scinc,L1pt,L1pd
	yr5_5c_djbunfft_unroll 0,noexec,exec,exec,srcreg,srcinc,d1,bcast,screg1,screg2,cosoff,screg,scinc,L1pt,L1pd
	yr5_5c_djbunfft_unroll 1,exec,exec,exec,srcreg,srcinc,d1,bcast,screg1,screg2,cosoff,screg,scinc,L1pt,L1pd
	yr5_5c_djbunfft_unroll 2,exec,exec,exec,srcreg,srcinc,d1,bcast,screg1,screg2,cosoff,screg,scinc,L1pt,L1pd
	yr5_5c_djbunfft_unroll 3,exec,exec,exec,srcreg,srcinc,d1,bcast,screg1,screg2,cosoff,screg,scinc,L1pt,L1pd
	yr5_5c_djbunfft_unroll 4,exec,exec,noexec,srcreg,srcinc,d1,bcast,screg1,screg2,cosoff,screg,scinc,L1pt,L1pd
	yr5_5c_djbunfft_unroll 5,exec,noexec,noexec,srcreg,srcinc,d1,bcast,screg1,screg2,cosoff,screg,scinc,L1pt,L1pd
	bump	srcreg, 5*srcinc
	bump	screg, 5*scinc
	ELSEIF maxrpt MOD 4 EQ 0
	yr5_5c_djbunfft_unroll -1,noexec,noexec,exec,srcreg,srcinc,d1,bcast,screg1,screg2,cosoff,screg,scinc,L1pt,L1pd
	yr5_5c_djbunfft_unroll 0,noexec,exec,exec,srcreg,srcinc,d1,bcast,screg1,screg2,cosoff,screg,scinc,L1pt,L1pd
	yr5_5c_djbunfft_unroll 1,exec,exec,exec,srcreg,srcinc,d1,bcast,screg1,screg2,cosoff,screg,scinc,L1pt,L1pd
	yr5_5c_djbunfft_unroll 2,exec,exec,exec,srcreg,srcinc,d1,bcast,screg1,screg2,cosoff,screg,scinc,L1pt,L1pd
	yr5_5c_djbunfft_unroll 3,exec,exec,noexec,srcreg,srcinc,d1,bcast,screg1,screg2,cosoff,screg,scinc,L1pt,L1pd
	yr5_5c_djbunfft_unroll 4,exec,noexec,noexec,srcreg,srcinc,d1,bcast,screg1,screg2,cosoff,screg,scinc,L1pt,L1pd
	bump	srcreg, 4*srcinc
	bump	screg, 4*scinc
	ELSEIF maxrpt MOD 3 EQ 0
	yr5_5c_djbunfft_unroll -1,noexec,noexec,exec,srcreg,srcinc,d1,bcast,screg1,screg2,cosoff,screg,scinc,L1pt,L1pd
	yr5_5c_djbunfft_unroll 0,noexec,exec,exec,srcreg,srcinc,d1,bcast,screg1,screg2,cosoff,screg,scinc,L1pt,L1pd
	yr5_5c_djbunfft_unroll 1,exec,exec,exec,srcreg,srcinc,d1,bcast,screg1,screg2,cosoff,screg,scinc,L1pt,L1pd
	yr5_5c_djbunfft_unroll 2,exec,exec,noexec,srcreg,srcinc,d1,bcast,screg1,screg2,cosoff,screg,scinc,L1pt,L1pd
	yr5_5c_djbunfft_unroll 3,exec,noexec,noexec,srcreg,srcinc,d1,bcast,screg1,screg2,cosoff,screg,scinc,L1pt,L1pd
	bump	srcreg, 3*srcinc
	bump	screg, 3*scinc
	ELSEIF maxrpt MOD 2 EQ 0
	yr5_5c_djbunfft_unroll -1,noexec,noexec,exec,srcreg,srcinc,d1,bcast,screg1,screg2,cosoff,screg,scinc,L1pt,L1pd
	yr5_5c_djbunfft_unroll 0,noexec,exec,exec,srcreg,srcinc,d1,bcast,screg1,screg2,cosoff,screg,scinc,L1pt,L1pd
	yr5_5c_djbunfft_unroll 1,exec,exec,noexec,srcreg,srcinc,d1,bcast,screg1,screg2,cosoff,screg,scinc,L1pt,L1pd
	yr5_5c_djbunfft_unroll 2,exec,noexec,noexec,srcreg,srcinc,d1,bcast,screg1,screg2,cosoff,screg,scinc,L1pt,L1pd
	bump	srcreg, 2*srcinc
	bump	screg, 2*scinc
	ELSE
	yr5_5c_djbunfft_unroll -1,noexec,noexec,exec,srcreg,srcinc,d1,bcast,screg1,screg2,cosoff,screg,scinc,L1pt,L1pd
	yr5_5c_djbunfft_unroll 0,noexec,exec,noexec,srcreg,srcinc,d1,bcast,screg1,screg2,cosoff,screg,scinc,L1pt,L1pd
	yr5_5c_djbunfft_unroll 1,exec,noexec,noexec,srcreg,srcinc,d1,bcast,screg1,screg2,cosoff,screg,scinc,L1pt,L1pd
	bump	srcreg, srcinc
	bump	screg, scinc
	ENDIF
	ENDM

yr5_5c_djbunfft_unroll MACRO iter,prev,this,next,srcreg,srcinc,d1,bcast,screg1,screg2,cosoff,screg,scinc,L1pt,L1pd

;; On first call, init registers

	IF iter EQ -1
y0	TEXTEQU	<ymm0>
y1	TEXTEQU	<ymm1>
y2	TEXTEQU	<ymm2>
y3	TEXTEQU	<ymm3>
y4	TEXTEQU	<ymm4>
y5	TEXTEQU	<ymm5>
y6	TEXTEQU	<ymm6>
y7	TEXTEQU	<ymm7>
y8	TEXTEQU	<ymm8>
y9	TEXTEQU	<ymm9>
y10	TEXTEQU	<ymm10>
y11	TEXTEQU	<ymm11>
y12	TEXTEQU	<ymm12>
	ENDIF

;; On later calls, y0-y11 contain this A2,A3,A4,A5,B2,R2,R3,R4,R5,I2,cos/sin1,cos/sin2.
;; y12 contains prev R3.  ymm13, ymm14 and ymm15 are preloaded constants cos2, sin2 and sin4.

this	vaddpd	y0, y0, y9				;; A2 = A2 + I2			; 1-3
this	vmovapd	y9, [srcreg+iter*srcinc+4*d1+32]	;; I5
this	vmulpd	y10, y9, y10				;; B5 = I5 * cosine/sine	; 1-5

this	vsubpd	y3, y3, y9				;; A5 = A5 - I5			; 2-4
this	vmovapd	y9, [srcreg+iter*srcinc+3*d1+32]	;; I4
prev	ystore	[srcreg+(iter-1)*srcinc+2*d1], y12	;; Save R3			; 3
this	vmulpd	y12, y9, y11				;; B4 = I4 * cosine/sine	; 2-6

this	vsubpd	y2, y2, y9				;; A4 = A4 - I4			; 3-5
this	vmovapd	y9, [srcreg+iter*srcinc+2*d1+32]	;; I3
this	vmulpd	y11, y9, y11				;; B3 = I3 * cosine/sine	; 3-7

this	vaddpd	y1, y1, y9				;; A3 = A3 + I3			; 4-6
this no bcast vmovapd y9, [screg1+iter*scinc]		;; sine
this bcast vbroadcastsd y9, Q [screg1+iter*scinc]	;; sine
this	vmulpd	y0, y0, y9				;; A2 = A2 * sine (new R2)	; 4-8

this	vsubpd	y4, y4, y5				;; B2 = B2 - R2			; 5-7
this	vmulpd	y3, y3, y9				;; A5 = A5 * sine (new R5)	; 5-9
this no bcast vmovapd y5, [screg2+iter*scinc]		;; sine
this bcast vbroadcastsd y5, Q [screg2+iter*scinc]	;; sine

this	vaddpd	y10, y10, y8				;; B5 = B5 + R5			; 6-8
this	vmulpd	y2, y2, y5				;; A4 = A4 * sine (new R4)	; 6-10
this	vmovapd	y8, YMM_P809				;; -cos4

this	vaddpd	y12, y12, y7				;; B4 = B4 + R4			; 7-9
this	vmulpd	y1, y1, y5				;; A3 = A3 * sine (new R3)	; 7-11
this	L1prefetchw srcreg+iter*srcinc+L1pd, L1pt

this	vsubpd	y11, y11, y6				;; B3 = B3 - R3			; 8-10
this	vmulpd	y4, y4, y9				;; B2 = B2 * sine (new I2)	; 8-12

													;; Empty ADD slot :(
this	vmulpd	y10, y10, y9				;; B5 = B5 * sine (new I5)	; 9-13

this	vaddpd	y9, y0, y3				;; r25a=r2+r5			; 10-12
this	vmulpd	y12, y12, y5				;; B4 = B4 * sine (new I4)	; 10-14
this	L1prefetchw srcreg+iter*srcinc+d1+L1pd, L1pt

this	vsubpd	y0, y0, y3				;; r25s=r2-r5			; 11-13
this	vmulpd	y11, y11, y5				;; B3 = B3 * sine (new I3)	; 11-15

this	vaddpd	y5, y1, y2				;; r34a=r3+r4			; 12-14

this	vsubpd	y1, y1, y2				;; r34s=r3-r4			; 13-15
this	vmulpd	y2, ymm13, y9				;; cos2*r25a			; 13-17
this	L1prefetchw srcreg+iter*srcinc+2*d1+L1pd, L1pt

this	vaddpd	y3, y4, y10				;; i25a=i2+i5			; 14-16
this	vmulpd	y6, y8, y9				;; -cos4*r25a			; 14-18

this	vaddpd	y9, y9, y5				;; r25a + r34a			; 15-17

this	vaddpd	y7, y11, y12				;; i34a=i3+i4			; 16-18
this	L1prefetchw srcreg+iter*srcinc+3*d1+L1pd, L1pt

this	vsubpd	y4, y4, y10				;; i25s=i2-i5			; 17-19
this	vmulpd	y10, y8, y5				;; -cos4*r34a			; 15-19
this	vmulpd	y5, ymm13, y5				;; cos2*r34a			; 16-20

this	vsubpd	y11, y11, y12				;; i34s=i3-i4			; 18-20
this	vmovapd	y12, [srcreg+iter*srcinc]		;; r1

this	vaddpd	y9, y12, y9				;; outr(0) = r1 + r25a + r34a	; 19-21
this	ystore	[srcreg+iter*srcinc], y9		;; Save R1
this	vmulpd	y9, ymm13, y3				;; cos2*i25a			; 17-21

this	vsubpd	y2, y2, y10				;; cos2*r25a + cos4*r34a	; 20-22
this	vmulpd	y10, y8, y3				;; -cos4*i25a			; 18-22
this	vmulpd	y8, y8, y7				;; -cos4*i34a			; 19-23

this	vsubpd	y5, y5, y6				;; cos4*r25a + cos2*r34a	; 21-23
this	vmulpd	y6, ymm13, y7				;; cos2*i34a			; 20-24
this	L1prefetchw srcreg+iter*srcinc+4*d1+L1pd, L1pt

this	vaddpd	y3, y3, y7				;; i25a+i34a			; 22-24
this	vmulpd	y7, ymm14, y0				;; sin2*r25s			; 21-25
this	vmulpd	y0, ymm15, y0				;; sin4*r25s			; 22-26

this	vaddpd	y2, y2, y12				;; t1=cos2*r25a + cos4*r34a + r1 ; 23-25

this	vaddpd	y5, y5, y12				;; t3=cos4*r25a + cos2*r34a + r1 ; 24-26
this	vmulpd	y12, ymm15, y1				;; sin4*r34s			; 23-27
this	vmulpd	y1, ymm14, y1				;; sin2*r34s			; 24-28

this	vsubpd	y9, y9, y8				;; cos2*i25a + cos4*i34a	; 25-27
this	vmulpd	y8, ymm14, y4				;; sin2*i25s			; 25-29

this	vsubpd	y6, y6, y10				;; cos4*i25a + cos2*i34a	; 26-28
this	vmovapd	y10, [srcreg+iter*srcinc+32]		;; I1

this	vaddpd	y3, y10, y3				;; outi(0) = i1 + i25a + i34a	; 27-29
this	ystore	[srcreg+iter*srcinc+32], y3		;; Save I1
this	vmulpd	y3, ymm15, y11				;; sin4*i34s			; 26-30
this	vmulpd	y4, ymm15, y4				;; sin4*i25s			; 27-31

this	vaddpd	y9, y9, y10				;; t5=cos2*i25a + cos4*i34a + i1 ; 28-30
this	vmulpd	y11, ymm14, y11				;; sin2*i34s			; 28-32

this	vaddpd	y7, y7, y12				;; t6=sin2*r25s + sin4*r34s	; 29-31
next no bcast vmovapd y12, [screg1+(iter+1)*scinc+cosoff] ;; cosine/sine
next bcast vbroadcastsd y12, Q [screg1+(iter+1)*scinc+cosoff] ;; cosine/sine

this	vaddpd	y6, y6, y10				;; t7=cos4*i25a + cos2*i34a + i1 ; 30-32
next	vmovapd	y10, [srcreg+(iter+1)*srcinc+d1]	;; R2

this	vsubpd	y0, y0, y1				;; t8=sin4*r25s - sin2*r34s	; 31-33
next	vmovapd	y1, [srcreg+(iter+1)*srcinc+4*d1]	;; R5

this	vaddpd	y8, y8, y3				;; t2=sin2*i25s + sin4*i34s	; 32-34
next	vmovapd	y3, [srcreg+(iter+1)*srcinc+3*d1]	;; R4

this	vsubpd	y4, y4, y11				;; t4=sin4*i25s - sin2*i34s	; 33-35
this next yloop_unrolled_one

this	vsubpd	y11, y9, y7				;; outi(1)=t5-t6		; 34-36
this	ystore	[srcreg+iter*srcinc+d1+32], y11		;; Save I2			; 37
next no bcast vmovapd y11, [screg2+(iter+1)*scinc+cosoff] ;; cosine/sine
next bcast vbroadcastsd y11, Q [screg2+(iter+1)*scinc+cosoff] ;; cosine/sine

this	vaddpd	y9, y9, y7				;; outi(4)=t5+t6		; 35-37
this	ystore	[srcreg+iter*srcinc+4*d1+32], y9	;; Save I5			; 38
next	vmovapd	y9, [srcreg+(iter+1)*srcinc+2*d1]	;; R3

this	vsubpd	y7, y2, y8				;; outr(4)=t1-t2		; 36-38
this	ystore	[srcreg+iter*srcinc+4*d1], y7		;; Save R5			; 39
next	vmovapd	y7, [srcreg+(iter+1)*srcinc+d1+32]	;; I2

this	vaddpd	y2, y2, y8				;; outr(1)=t1+t2		; 37-39
this	ystore	[srcreg+iter*srcinc+d1], y2		;; Save R2			; 40
next	vmulpd	y2, y10, y12				;; A2 = R2 * cosine/sine	; 37-41

this	vsubpd	y8, y6, y0				;; outi(2)=t7-t8		; 38-40
this	ystore	[srcreg+iter*srcinc+2*d1+32], y8	;; Save I3			; 41
next	vmulpd	y8, y1, y12				;; A5 = R5 * cosine/sine	; 38-42

this	vaddpd	y6, y6, y0				;; outi(3)=t7+t8		; 39-41
this	ystore	[srcreg+iter*srcinc+3*d1+32], y6	;; Save I4			; 42
next	vmulpd	y6, y3, y11				;; A4 = R4 * cosine/sine	; 39-43

this	vsubpd	y0, y5, y4				;; outr(3)=t3-t4		; 40-42
this	ystore	[srcreg+iter*srcinc+3*d1], y0		;; Save R4			; 43
next	vmulpd	y0, y9, y11				;; A3 = R3 * cosine/sine	; 40-44

this	vaddpd	y5, y5, y4				;; outr(2)=t3+t4		; 41-43
next	vmulpd	y4, y7, y12				;; B2 = I2 * cosine/sine	; 41-45

;; Shuffle register assignments so that y0-y11 in the next call has this
;; A2,A3,A4,A5,B2,R2,R3,R4,R5,I2,cos/sin1,cos/sin2. and y12 contains prev R3.

ytmp	TEXTEQU	y0
y0	TEXTEQU	y2
y2	TEXTEQU	y6
y6	TEXTEQU	y9
y9	TEXTEQU	y7
y7	TEXTEQU	y3
y3	TEXTEQU	y8
y8	TEXTEQU	y1
y1	TEXTEQU	ytmp
ytmp	TEXTEQU	y5
y5	TEXTEQU	y10
y10	TEXTEQU	y12
y12	TEXTEQU ytmp

	ENDM

;; Haswell FMA3 version

IF (@INSTR(,%yarch,<FMA3>) NE 0)

yr5_5c_djbunfft_cmn_preload MACRO
	ENDM

;; 10 loads, 10 stores, 4 sin/cos loads, 7 const loads, 40 FMAs, 4 muls, 18 reg copies  = 93 uops.  93/4 = 23.25 clock min.
yr5_5c_djbunfft_unroll MACRO iter,prev,this,next,srcreg,srcinc,d1,bcast,screg1,screg2,cosoff,screg,scinc,L1pt,L1pd

;; On first call, init registers

	IF iter EQ -1
y0	TEXTEQU	<ymm0>
y1	TEXTEQU	<ymm1>
y2	TEXTEQU	<ymm2>
y3	TEXTEQU	<ymm3>
y4	TEXTEQU	<ymm4>
y5	TEXTEQU	<ymm5>
y6	TEXTEQU	<ymm6>
y7	TEXTEQU	<ymm7>
y8	TEXTEQU	<ymm8>
y9	TEXTEQU	<ymm9>
y10	TEXTEQU	<ymm10>
y11	TEXTEQU	<ymm11>
y12	TEXTEQU	<ymm12>
y13	TEXTEQU	<ymm13>
y14	TEXTEQU	<ymm14>
y15	TEXTEQU	<ymm15>
	ENDIF

;; On later calls, prev R2,R3,R4,t5,t6,t7,t8 is in y0-6, and this A2,B2,A3,B3,R5,I5,c/s25,c/s34,sine25 is in y7-y15.

prev	ystore	[srcreg+(iter-1)*srcinc+d1], y0		;; Save R2			;	26+1
this	yfmsubpd y0, y11, y13, y12			;; A5 = R5 * cosine/sine - I5	; 1-5
this	vmulpd	y7, y7, y15				;; A2 = A2 * sine (new R2)	; 1-5

this	yfmaddpd y12, y12, y13, y11			;; B5 = I5 * cosine/sine + R5	; 2-6			free c/s25
this	vmulpd	y8, y8, y15				;; B2 = B2 * sine (new I2)	; 2-6

this	vmovapd	y11, [srcreg+iter*srcinc+3*d1]		;; R4
this	vmovapd	y13, [srcreg+iter*srcinc+3*d1+32]	;; I4
prev	ystore	[srcreg+(iter-1)*srcinc+3*d1], y2	;; Save R4			;	27+1
this	yfmsubpd y2, y11, y14, y13			;; A4 = R4 * cosine/sine - I4	; 3-7
prev	ystore	[srcreg+(iter-1)*srcinc+2*d1], y1	;; Save R3			;	27+2
this no bcast vmovapd y1, [screg2+iter*scinc]		;; sine for R3/I3/R4/I4
this bcast vbroadcastsd y1, Q [screg2+iter*scinc]	;; sine for R3/I3/R4/I4
this	vmulpd	y9, y9, y1				;; A3 = A3 * sine (new R3)	; 3-7
this	L1prefetchw srcreg+iter*srcinc+L1pd, L1pt

this	yfmaddpd y13, y13, y14, y11			;; B4 = I4 * cosine/sine + R4	; 4-8			free c/s34
this	vmulpd	y10, y10, y1				;; B3 = B3 * sine (new I3)	; 4-8

prev	vmovapd y14, YMM_P951				;; sin2
prev	yfnmaddpd y11, y14, y4, y3			;; t5 - sin2 * t6/sin2 (final I2) ;	5-9
prev	yfmaddpd y4, y14, y4, y3			;; t5 + sin2 * t6/sin2 (final I5) ;	5-9
this	L1prefetchw srcreg+iter*srcinc+d1+L1pd, L1pt

this	yfmaddpd y3, y0, y15, y7			;; r25a = r2 + A5 * sine	; 6-10
this	yfnmaddpd y0, y0, y15, y7			;; r25s = r2 - A5 * sine	; 6-10

this	yfmaddpd y7, y12, y15, y8			;; i25a = i2 + B5 * sine	; 7-11
this	yfnmaddpd y12, y12, y15, y8			;; i25s = i2 - B5 * sine	; 7-11			free sine25
this	vmovapd	y15, [srcreg+iter*srcinc]		;; r1

this	yfmaddpd y8, y2, y1, y9				;; r34a = r3 + A4 * sine	; 8-12
this	yfnmaddpd y2, y2, y1, y9			;; r34s = r3 - A4 * sine	; 8-12
this	L1prefetchw srcreg+iter*srcinc+3*d1+L1pd, L1pt

this	yfmaddpd y9, y13, y1, y10			;; i34a = i3 + B4 * sine	; 9-13
this	yfnmaddpd y13, y13, y1, y10			;; i34s = i3 - B4 * sine	; 9-13			free sine34
this	vmovapd y1, YMM_ONE

prev	yfnmaddpd y10, y14, y6, y5			;; t7 - sin2 * t8/sin2 (final I3) ; 10-14
prev	yfmaddpd y14, y14, y6, y5			;; t7 + sin2 * t8/sin2 (final I4) ; 10-14		free sin2
this	vmovapd	y6, YMM_P309				;; cos2
prev	ystore	[srcreg+(iter-1)*srcinc+d1+32], y11	;; Save I2			;	10

this	yfmaddpd y5, y15, y1, y3			;; tr0 = r1 + r25a		; 11-15
this	yfmaddpd y11, y6, y3, y15			;; t1 = r1 + cos2*r25a		; 11-15
prev	ystore	[srcreg+(iter-1)*srcinc+4*d1+32], y4	;; Save I5			;	10+1

this	vmovapd	y4, YMM_P809				;; -cos4
this	yfnmaddpd y3, y4, y3, y15			;; t3 = r1 - -cos4*r25a		; 12-16			free r1, r25a
this	vmovapd	y15, [srcreg+iter*srcinc+32]		;; i1
this	yfmaddpd y1, y15, y1, y7			;; ti0 = i1 + i25a		; 12-16
this	L1prefetchw srcreg+iter*srcinc+2*d1+L1pd, L1pt

prev	ystore	[srcreg+(iter-1)*srcinc+2*d1+32], y10	;; Save I3			;	15
this	yfmaddpd y10, y6, y7, y15			;; t5 = i1 + cos2*i25a		; 13-17
this	yfnmaddpd y7, y4, y7, y15			;; t7 = i1 - -cos4*i25a		; 13-17			free i1, i25a

this	vmovapd	y15, YMM_P588_P951			;; sin4/sin2
prev	ystore	[srcreg+(iter-1)*srcinc+3*d1+32], y14	;; Save I4			;	15+1
this	yfmaddpd y14, y15, y13, y12			;; t2/sin2 = i25s + sin4/sin2*i34s ; 14-18
this	yfmsubpd y12, y15, y12, y13			;; t4/sin2 = sin4/sin2*i25s - i34s ; 14-18		free i25s, i34s
this	L1prefetchw srcreg+iter*srcinc+4*d1+L1pd, L1pt

this	yfmaddpd y13, y15, y2, y0			;; t6/sin2 = r25s + sin4/sin2*r34s ; 15-19
this	yfmsubpd y15, y15, y0, y2			;; t8/sin2 = sin4/sin2*r25s - r34s ; 15-19		free r25s, r34s, sin4/sin2

this	vmovapd y0, YMM_ONE
this	yfmaddpd y5, y5, y0, y8				;; tr0 = tr0 + r34a (final R1)	; 16-20
this	yfnmaddpd y11, y4, y8, y11			;; t1 = t1 - -cos4*r34a		; 16-20
next	vmovapd	y2, [srcreg+(iter+1)*srcinc+d1]		;; R2

this	yfmaddpd y3, y6, y8, y3				;; t3 = t3 + cos2*r34a		; 17-21			free r34a
this	yfmaddpd y1, y1, y0, y9				;; ti0 = ti0 + i34a (final I1)	; 17-21
next	vmovapd	y8, [srcreg+(iter+1)*srcinc+d1+32]	;; I2

next no bcast vmovapd y0, [screg1+(iter+1)*scinc+cosoff] ;; cosine/sine for R2/I2/R5/I5
next bcast vbroadcastsd y0, Q [screg1+(iter+1)*scinc+cosoff] ;; cosine/sine for R2/I2/R5/I5
this	ystore	[srcreg+iter*srcinc], y5		;; Save R1			; 21
next	yfmaddpd y5, y2, y0, y8				;; A2 = R2 * cosine/sine + I2	; 18-22
next	yfmsubpd y8, y8, y0, y2				;; B2 = I2 * cosine/sine - R2	; 18-22
next	vmovapd	y2, [srcreg+(iter+1)*srcinc+2*d1]	;; R3

this	yfnmaddpd y10, y4, y9, y10			;; t5 = t5 - -cos4*i34a		; 19-23			free -cos4
next	vmovapd	y4, [srcreg+(iter+1)*srcinc+2*d1+32]	;; I3
this	yfmaddpd y7, y6, y9, y7				;; t7 = t7 + cos2*i34a		; 19-23			free i34a, cos2
this next yloop_unrolled_one

next no bcast vmovapd y6, [screg2+(iter+1)*scinc+cosoff] ;; cosine/sine for R3/I3/R4/I4
next bcast vbroadcastsd y6, Q [screg2+(iter+1)*scinc+cosoff] ;; cosine/sine for R3/I3/R4/I4
next	yfmaddpd y9, y2, y6, y4				;; A3 = R3 * cosine/sine + I3	; 20-24
next	yfmsubpd y4, y4, y6, y2				;; B3 = I3 * cosine/sine - R3	; 20-24
this	vmovapd y2, YMM_P951				;; sin2

this	ystore	[srcreg+iter*srcinc+32], y1		;; Save I1			; 22
this	yfnmaddpd y1, y2, y14, y11			;; t1 - sin2 * t2/sin2 (final R5) ; 21-25
this	yfmaddpd y14, y2, y14, y11			;; t1 + sin2 * t2/sin2 (final R2) ; 21-25

this	yfnmaddpd y11, y2, y12, y3			;; t3 - sin2 * t4/sin2 (final R4) ; 22-26
this	yfmaddpd y2, y2, y12, y3			;; t3 + sin2 * t4/sin2 (final R3) ; 22-26		free sin2
next	vmovapd	y12, [srcreg+(iter+1)*srcinc+4*d1]	;; R5
next	vmovapd	y3, [srcreg+(iter+1)*srcinc+4*d1+32]	;; I5
this	ystore	[srcreg+iter*srcinc+4*d1], y1		;; Save R5			; 26
next no bcast vmovapd y1, [screg1+(iter+1)*scinc]	;; sine for R2/I2/R5/I5
next bcast vbroadcastsd y1, Q [screg1+(iter+1)*scinc]	;; sine for R2/I2/R5/I5

;; Shuffle register assignments so that this R2,R3,R4,t5,t6,t7,t8 is in y0-6, and next A2,B2,A3,B3,R5,I5,c/s25,c/s34,sine25 is in y7-y15.

ytmp	TEXTEQU	y0
y0	TEXTEQU	y14
y14	TEXTEQU	y6
y6	TEXTEQU	y15
y15	TEXTEQU	y1
y1	TEXTEQU	y2
y2	TEXTEQU	y11
y11	TEXTEQU	y12
y12	TEXTEQU	y3
y3	TEXTEQU	y10
y10	TEXTEQU	y4
y4	TEXTEQU	y13
y13	TEXTEQU	ytmp
ytmp	TEXTEQU	y5
y5	TEXTEQU	y7
y7	TEXTEQU	ytmp
	ENDM

ENDIF

ENDIF


;;
;; ************************************* ten-reals-fft variants ******************************************
;;

yr5_5cl_ten_reals_fft_preload MACRO
	yr5_10r_fft_cmn_preload
	ENDM
yr5_5cl_ten_reals_fft MACRO srcreg,srcinc,d1,screg,scinc,maxrpt,L1pt,L1pd
	yr5_10r_fft_cmn srcreg,srcinc,0,d1,screg,screg+64,screg+128,screg+192,screg,scinc,,,maxrpt,L1pt,L1pd
	ENDM

; Like the 5cl version but uses two sin/cos pointers
yr5_5cl_2sc_ten_reals_fft_preload MACRO
	yr5_10r_fft_cmn_preload
	ENDM
yr5_5cl_2sc_ten_reals_fft MACRO srcreg,srcinc,d1,screg1,scinc1,screg2,scinc2,maxrpt,L1pt,L1pd
	yr5_10r_fft_cmn srcreg,srcinc,0,d1,screg2,screg1,screg2+64,screg1+64,screg1,scinc1,screg2,scinc2,maxrpt,L1pt,L1pd
	ENDM

; Like the 5cl version but the sin/cos data can be used by a five_complex macro at the same FFT level
yr5_5cl_csc_ten_reals_fft_preload MACRO
	yr5_10r_fft_cmn_preload
	ENDM
yr5_5cl_csc_ten_reals_fft MACRO srcreg,srcinc,d1,screg,scinc,maxrpt,L1pt,L1pd
	yr5_10r_fft_cmn srcreg,srcinc,0,d1,screg+128,screg,screg+192,screg+64,screg,scinc,,,maxrpt,L1pt,L1pd
	ENDM

; Like the 5cl version but offsets source by rbx
yr5_f5cl_ten_reals_fft_preload MACRO
	yr5_10r_fft_cmn_preload
	ENDM
yr5_f5cl_ten_reals_fft MACRO srcreg,srcinc,d1,screg,scinc,maxrpt,L1pt,L1pd
	yr5_10r_fft_cmn srcreg,srcinc,rbx,d1,screg,screg+64,screg+128,screg+192,screg,scinc,,,maxrpt,L1pt,L1pd
	ENDM

; Like the f5cl version but uses two sin/cos ptrs
yr5_f5cl_2sc_ten_reals_fft_preload MACRO
	yr5_10r_fft_cmn_preload
	ENDM
yr5_f5cl_2sc_ten_reals_fft MACRO srcreg,srcinc,d1,screg1,scinc1,screg2,scinc2,maxrpt,L1pt,L1pd
	yr5_10r_fft_cmn srcreg,srcinc,rbx,d1,screg2,screg1,screg2+64,screg1+64,screg1,scinc1,screg2,scinc2,maxrpt,L1pt,L1pd
	ENDM

;; To calculate a 10-reals FFT (in a shorthand notation):
;; r1 + r2 + r3 + r4 + r5 + r6 + r7 + r8 + r9 + r10	*  w^0000000000
;; r1 + r2 + r3 + r4 + r5 + r6 + r7 + r8 + r9 + r10	*  w^0123456789
;; r1 + r2 + r3 + r4 + r5 + r6 + r7 + r8 + r9 + r10	*  w^0246802468
;; r1 + r2 + r3 + r4 + r5 + r6 + r7 + r8 + r9 + r10	*  w^0369258147
;; r1 + r2 + r3 + r4 + r5 + r6 + r7 + r8 + r9 + r10	*  w^0482604826
;; r1 + r2 + r3 + r4 + r5 + r6 + r7 + r8 + r9 + r10	*  w^0505050505
;; r1 + r2 + r3 + r4 + r5 + r6 + r7 + r8 + r9 + r10	*  w^0628406284
;; r1 + r2 + r3 + r4 + r5 + r6 + r7 + r8 + r9 + r10	*  w^0741852963
;; r1 + r2 + r3 + r4 + r5 + r6 + r7 + r8 + r9 + r10	*  w^0864208642
;; r1 + r2 + r3 + r4 + r5 + r6 + r7 + r8 + r9 + r10	*  w^0987654321
;; Noting that w^5 = -1 and that Hermetian symmetry means we won't need
;; to calculate the last 5 rows:
;; r1 + r2 + r3 + r4 + r5 + r6 + r7 + r8 + r9 + r10	*  w^0000000000
;; r1 + r2 + r3 + r4 + r5 - r6 - r7 - r8 - r9 - r10	*  w^0123401234
;; r1 + r2 + r3 - r4 - r5 + r6 + r7 + r8 - r9 - r10	*  w^0241302413
;; r1 + r2 - r3 - r4 + r5 - r6 - r7 + r8 + r9 - r10	*  w^0314203142
;; r1 + r2 - r3 + r4 - r5 + r6 + r7 - r8 + r9 - r10	*  w^0432104321
;; Reorganize into odds and evens, that is
;; r1 + r3 + r5 + r7 + r9  * powers + (r2 + r4 + r6 + r8 + r10) * powers
;; Thus:
;; r1 + r3 + r5 + r7 + r9	*  w^00000	+ r2 + r4 + r6 + r8 + r10	*  w^00000
;; r1 + r3 + r5 - r7 - r9	*  w^02413	+ r2 + r4 - r6 - r8 - r10	*  w^13024
;; r1 + r3 - r5 + r7 - r9	*  w^04321	+ r2 - r4 + r6 + r8 - r10	*  w^21043
;; r1 - r3 + r5 - r7 + r9	*  w^01234	+ r2 - r4 - r6 + r8 - r10	*  w^34012
;; r1 - r3 - r5 + r7 + r9	*  w^03142	+ r2 + r4 + r6 - r8 - r10	*  w^42031
;; Apply the sin/cos values:
;; w^1/10 = .809 + .588i
;; w^2/10 = .309 + .951i
;; w^3/10 = -.309 + .951i
;; w^4/10 = -.809 + .588i
;; reals:
;; r1 + r3 + r5 + r7 + r9			+ r2 + r4 + r6 + r8 + r10
;; r1 + .309r3 - .809r5 - .809r7 + .309r9	+ .809r2 - .309r4 - r6 - .309r8 + .809r10
;; r1 - .809r3 + .309r5 + .309r7 - .809r9	+ .309r2 - .809r4 + r6 - .809r8 + .309r10
;; r1 - .809r3 + .309r5 + .309r7 - .809r9	+ -.309r2 + .809r4 - r6 + .809r8 - .309r10
;; r1 + .309r3 - .809r5 - .809r7 + .309r9	+ -.809r2 + .309r4 + r6 + .309r8 - .809r10
;; imaginarys:
;; 0						+ 0
;;  + .951r3 + .588r5 - .588r7 - .951r9		+ .588r2 + .951r4 - .951r8 - .588r10
;;  + .588r3 - .951r5 + .951r7 - .588r9		+ .951r2 - .588r4 + .588r8 - .951r10
;;  - .588r3 + .951r5 - .951r7 + .588r9		+ .951r2 - .588r4 + .588r8 - .951r10
;;  - .951r3 - .588r5 + .588r7 + .951r9		+ .588r2 + .951r4 - .951r8 - .588r10
;; Further simplifying:
;; reals:
;; r1 + r3 + r5 + r7 + r9		+ r2 + r4 + r6 + r8 + r10
;; r1 + .309(r3+r9) - .809(r5+r7)	+ .809(r2+r10) - .309(r4+r8) - r6
;; r1 - .809(r3+r9) + .309(r5+r7)	+ .309(r2+r10) - .809(r4+r8) + r6
;; r1 - .809(r3+r9) + .309(r5+r7)	+ -.309(r2+r10) + .809(r4+r8) - r6
;; r1 + .309(r3+r9) - .809(r5+r7)	+ -.809(r2+r10) + .309(r4+r8) + r6
;; imaginarys:
;; 0					+ 0
;;  + .951(r3-r9) + .588(r5-r7)		+ .588(r2-r10) + .951(r4-r8)
;;  + .588(r3-r9) - .951(r5-r7)		+ .951(r2-r10) - .588(r4-r8)
;;  - .588(r3-r9) + .951(r5-r7)		+ .951(r2-r10) - .588(r4-r8)
;;  - .951(r3-r9) - .588(r5-r7)		+ .588(r2-r10) + .951(r4-r8)

yr5_10r_fft_cmn_preload MACRO
	ENDM
yr5_10r_fft_cmn MACRO srcreg,srcinc,srcoff,d1,screg1,screg2,screg3,screg4,scregA,scincA,scregB,scincB,maxrpt,L1pt,L1pd
	vmovapd	ymm0, [srcreg+srcoff+2*d1]	;; R3
	vmovapd ymm1, [srcreg+srcoff+3*d1+32]	;; R9
	vsubpd	ymm7, ymm0, ymm1		;; R3-R9
	vaddpd	ymm0, ymm0, ymm1		;; R3+R9
	vmovapd	ymm3, [srcreg+srcoff+4*d1]	;; R5
	vmovapd ymm4, [srcreg+srcoff+d1+32]	;; R7
	vsubpd	ymm6, ymm3, ymm4		;; R5-R7
	vaddpd	ymm3, ymm3, ymm4		;; R5+R7
	vmovapd	ymm1, YMM_P951
	vmulpd	ymm2, ymm1, ymm7		;; new oddI2 = .951*(R3-R9)
	vmovapd	ymm5, YMM_P588
	vmulpd	ymm7, ymm5, ymm7		;; new oddI3 = .588*(R3-R9)
	vmulpd	ymm5, ymm5, ymm6		;; .588*(R5-R7)
	vmulpd	ymm1, ymm1, ymm6		;; .951*(R5-R7)
	vaddpd	ymm2, ymm2, ymm5		;; new oddI2 += .588*(R5-R7)
	vsubpd	ymm7, ymm7, ymm1		;; new oddI3 -= .951*(R5-R7)
	vmovapd	ymm1, YMM_P309
	vmulpd	ymm5, ymm1, ymm0		;; new oddR2 = .309*(R3+R9)
	vmovapd	ymm4, [srcreg+srcoff]		;; R1
	vaddpd	ymm5, ymm5, ymm4		;; new oddR2 += R1
	vmovapd	ymm6, YMM_P809
	ystore	YMM_TMP6, ymm2			;; Save oddI2
	vmulpd	ymm2, ymm6, ymm0		;; new -oddR3 = -.809*(R3+R9)
	vaddpd	ymm0, ymm4, ymm0		;; new oddR1 = R1+R3+R9
	vsubpd	ymm2, ymm4, ymm2		;; new oddR3 += R1
	vmulpd	ymm4, ymm6, ymm3		;; --.809*(R5+R7)
	vaddpd	ymm0, ymm0, ymm3		;; new oddR1 = R1+R3+R5+R7+R9
	vmulpd	ymm3, ymm1, ymm3		;; .309*(R5+R7)
	vsubpd	ymm5, ymm5, ymm4		;; new oddR2 += -.809*(R5+R7)
	vaddpd	ymm2, ymm2, ymm3		;; new oddR3 += .309*(R5+R7)
	ystore	YMM_TMP8, ymm7			;; Save oddI3
	ystore	[srcreg], ymm0			;; Save oddR1
	ystore	YMM_TMP1, ymm5			;; Save oddR2
	ystore	YMM_TMP3, ymm2			;; Save oddR3

	vmovapd ymm2, [srcreg+srcoff+d1]	;; R2
	vmovapd	ymm5, [srcreg+srcoff+4*d1+32]	;; R10
	vsubpd	ymm7, ymm2, ymm5		;; R2-R10
	vaddpd	ymm2, ymm2, ymm5		;; R2+R10
	vmovapd ymm4, [srcreg+srcoff+3*d1]	;; R4
	vmovapd	ymm3, [srcreg+srcoff+2*d1+32]	;; R8
	vsubpd	ymm6, ymm4, ymm3		;; R4-R8
	vaddpd	ymm4, ymm4, ymm3		;; R4+R8

	L1prefetchw srcreg+L1pd, L1pt

	vmovapd	ymm0, YMM_P951
	vmulpd	ymm1, ymm0, ymm7		;; new evenI3 = .951*(R2-R10)
	vmovapd	ymm3, YMM_P588
	vmulpd	ymm7, ymm3, ymm7		;; new evenI2 = .588*(R2-R10)
	vmulpd	ymm3, ymm3, ymm6		;; .588*(R4-R8)
	vmulpd	ymm0, ymm0, ymm6		;; .951*(R4-R8)
	vsubpd	ymm1, ymm1, ymm3		;; new evenI3 -= .588*(R4-R8)
	vaddpd	ymm7, ymm7, ymm0		;; new evenI2 += .951*(R4-R8)

	vmulpd	ymm3, ymm2, YMM_P309		;; new evenR3 = .309*(R2+R10)
	vmovapd	ymm5, [srcreg+srcoff+32]	;; R6
	vaddpd	ymm3, ymm3, ymm5		;; new evenR3 += R6
	vmovapd	ymm6, YMM_P809
	vmulpd	ymm0, ymm6, ymm2		;; new -evenR5 = -.809*(R2+R10)
	vaddpd	ymm2, ymm5, ymm2		;; new evenR1 = R6+R2+R10
	vsubpd	ymm0, ymm5, ymm0		;; new evenR5 += R6
	vmulpd	ymm6, ymm6, ymm4		;; --.809*(R4+R8)
	vaddpd	ymm2, ymm2, ymm4		;; new evenR1 = R6+R2+R4+R8+R10
	vmulpd	ymm4, ymm4, YMM_P309		;; .309*(R4+R8)
	vsubpd	ymm3, ymm3, ymm6		;; new evenR3 += -.809*(R4+R8)
	vaddpd	ymm0, ymm0, ymm4		;; new evenR5 += .309*(R4+R8)
	ystore	[srcreg+32], ymm2		;; Save evenR1

	L1prefetchw srcreg+d1+L1pd, L1pt

	vmovapd	ymm2, YMM_TMP1			;; oddR2
	vaddpd	ymm4, ymm2, ymm0		;; New R5 = oddR2 + evenR5
	vmovapd	ymm6, YMM_TMP6			;; oddI2
	vsubpd	ymm5, ymm7, ymm6		;; New I5 = evenI2 - oddI2
	vsubpd	ymm2, ymm2, ymm0		;; New R2 = oddR2 - evenR5
	vaddpd	ymm7, ymm7, ymm6		;; New I2 = evenI2 + oddI2

	vmovapd	ymm0, [screg4+32]		;; cosine/sine for w^4
	vmulpd	ymm6, ymm4, ymm0		;; A5 = R5 * cosine/sine
	vsubpd	ymm6, ymm6, ymm5		;; A5 = A5 - I5
	vmulpd	ymm5, ymm5, ymm0		;; B5 = I5 * cosine/sine
	vaddpd	ymm5, ymm5, ymm4		;; B5 = B5 + R5

	vmovapd	ymm0, [screg1+32]		;; cosine/sine for w^1
	vmulpd	ymm4, ymm2, ymm0		;; A2 = R2 * cosine/sine
	vsubpd	ymm4, ymm4, ymm7		;; A2 = A2 - I2
	vmulpd	ymm7, ymm7, ymm0		;; B2 = I2 * cosine/sine
	vaddpd	ymm7, ymm7, ymm2		;; B2 = B2 + R2

	L1prefetchw srcreg+2*d1+L1pd, L1pt

	vmovapd	ymm2, [screg4]
	vmulpd	ymm6, ymm6, ymm2		;; A5 = A5 * sine (final R5)
	vmulpd	ymm5, ymm5, ymm2		;; B5 = B5 * sine (final I5)
	vmovapd	ymm2, [screg1]
	vmulpd	ymm4, ymm4, ymm2		;; A2 = A2 * sine (final R2)
	vmulpd	ymm7, ymm7, ymm2		;; B2 = B2 * sine (final I2)

	ystore	[srcreg+d1], ymm4		;; Save final R2
	ystore	[srcreg+d1+32], ymm7		;; Save final I2

	vmovapd	ymm4, YMM_TMP3			;; oddR3
	vsubpd	ymm0, ymm4, ymm3		;; New R4 = oddR3 - evenR3
	vaddpd	ymm4, ymm4, ymm3		;; New R3 = oddR3 + evenR3
	vmovapd	ymm7, YMM_TMP8			;; oddI3
	vsubpd	ymm2, ymm1, ymm7		;; New I4 = evenI3 - oddI3
	vaddpd	ymm1, ymm1, ymm7		;; New I3 = evenI3 + oddI3

	L1prefetchw srcreg+3*d1+L1pd, L1pt

	vmovapd	ymm3, [screg3+32]		;; cosine/sine for w^3
	vmulpd	ymm7, ymm0, ymm3		;; A4 = R4 * cosine/sine
	vsubpd	ymm7, ymm7, ymm2		;; A4 = A4 - I4
	vmulpd	ymm2, ymm2, ymm3		;; B4 = I4 * cosine/sine
	vaddpd	ymm2, ymm2, ymm0		;; B4 = B4 + R4

	vmovapd	ymm3, [screg2+32]		;; cosine/sine for w^2
	vmulpd	ymm0, ymm4, ymm3		;; A3 = R3 * cosine/sine
	vsubpd	ymm0, ymm0, ymm1		;; A3 = A3 - I3
	vmulpd	ymm1, ymm1, ymm3		;; B3 = I3 * cosine/sine
	vaddpd	ymm1, ymm1, ymm4		;; B3 = B3 + R3

	L1prefetchw srcreg+4*d1+L1pd, L1pt

	vmovapd	ymm4, [screg3]
	vmulpd	ymm7, ymm7, ymm4		;; A4 = A4 * sine (final R4)
	vmulpd	ymm2, ymm2, ymm4		;; B4 = B4 * sine (final I4)
	vmovapd	ymm4, [screg2]
	vmulpd	ymm0, ymm0, ymm4		;; A3 = A3 * sine (final R3)
	vmulpd	ymm1, ymm1, ymm4		;; B3 = B3 * sine (final I3)

	ystore	[srcreg+2*d1], ymm0		;; Save R3
	ystore	[srcreg+2*d1+32], ymm1		;; Save I3
	ystore	[srcreg+3*d1], ymm7		;; Save R4
	ystore	[srcreg+3*d1+32], ymm2		;; Save I4
	ystore	[srcreg+4*d1], ymm6		;; Save R5
	ystore	[srcreg+4*d1+32], ymm5		;; Save I5

	bump	srcreg, srcinc
	bump	scregA, scincA
	bump	scregB, scincB
	ENDM


;;
;; ************************************* ten-reals-unfft variants ******************************************
;;

;; The standard version
yr5_5cl_ten_reals_unfft_preload MACRO
	yr5_10r_unfft_cmn_preload
	ENDM
yr5_5cl_ten_reals_unfft MACRO srcreg,srcinc,d1,screg,scinc,maxrpt,L1pt,L1pd
	yr5_10r_unfft_cmn srcreg,srcinc,d1,screg,screg+64,screg+128,screg+192,screg,scinc,,,maxrpt,L1pt,L1pd
	ENDM

;; Like 5cl except that two sin/cos ptrs are used
yr5_5cl_2sc_ten_reals_unfft_preload MACRO
	yr5_10r_unfft_cmn_preload
	ENDM
yr5_5cl_2sc_ten_reals_unfft MACRO srcreg,srcinc,d1,screg1,scinc1,screg2,scinc2,maxrpt,L1pt,L1pd
	yr5_10r_unfft_cmn srcreg,srcinc,d1,screg2,screg1,screg2+64,screg1+64,screg1,scinc1,screg2,scinc2,maxrpt,L1pt,L1pd
	ENDM

;; Like 5cl except that five_complex macros at the same FFT level can use the sin/cos data
yr5_5cl_csc_ten_reals_unfft_preload MACRO
	yr5_10r_unfft_cmn_preload
	ENDM
yr5_5cl_csc_ten_reals_unfft MACRO srcreg,srcinc,d1,screg,scinc,maxrpt,L1pt,L1pd
	yr5_10r_unfft_cmn srcreg,srcinc,d1,screg+128,screg,screg+192,screg+64,screg,scinc,,,maxrpt,L1pt,L1pd
	ENDM

;; To calculate a 10-reals unFFT (in a shorthand notation):
;; c1 + c2 + c3 + c4 + c5 + c6 + c7 + c8 + c9 + c10	*  w^0000000000
;; c1 + c2 + c3 + c4 + c5 + c6 + c7 + c8 + c9 + c10	*  w^0123456789
;; c1 + c2 + c3 + c4 + c5 + c6 + c7 + c8 + c9 + c10	*  w^0246802468
;; c1 + c2 + c3 + c4 + c5 + c6 + c7 + c8 + c9 + c10	*  w^0369258147
;; c1 + c2 + c3 + c4 + c5 + c6 + c7 + c8 + c9 + c10	*  w^0482604826
;; c1 + c2 + c3 + c4 + c5 + c6 + c7 + c8 + c9 + c10	*  w^0505050505
;; c1 + c2 + c3 + c4 + c5 + c6 + c7 + c8 + c9 + c10	*  w^0628406284
;; c1 + c2 + c3 + c4 + c5 + c6 + c7 + c8 + c9 + c10	*  w^0741852963
;; c1 + c2 + c3 + c4 + c5 + c6 + c7 + c8 + c9 + c10	*  w^0864208642
;; c1 + c2 + c3 + c4 + c5 + c6 + c7 + c8 + c9 + c10	*  w^0987654321
;; Noting that w^5 = -1
;; c1 + c2 + c3 + c4 + c5 + c6 + c7 + c8 + c9 + c10	*  w^0000000000
;; c1 + c2 + c3 + c4 + c5 - c6 - c7 - c8 - c9 - c10	*  w^0123401234
;; c1 + c2 + c3 - c4 - c5 + c6 + c7 + c8 - c9 - c10	*  w^0241302413
;; c1 + c2 - c3 - c4 + c5 - c6 - c7 + c8 + c9 - c10	*  w^0314203142
;; c1 + c2 - c3 + c4 - c5 + c6 + c7 - c8 + c9 - c10	*  w^0432104321
;; c1 - c2 + c3 - c4 + c5 - c6 + c7 - c8 + c9 - c10	*  w^0000000000
;; c1 - c2 + c3 - c4 + c5 + c6 - c7 + c8 - c9 + c10	*  w^0123401234
;; c1 - c2 + c3 + c4 - c5 - c6 + c7 - c8 - c9 + c10	*  w^0241302413
;; c1 - c2 - c3 + c4 + c5 + c6 - c7 - c8 + c9 + c10	*  w^0314203142
;; c1 - c2 - c3 - c4 - c5 - c6 + c7 + c8 + c9 + c10	*  w^0432104321
;; incoming is:	r1_1 = c1r + c6r
;;		c2 = c2r + c2i
;;		c3 = c3r + c3i
;;		c4 = c4r + c4i
;;		c5 = c5r + c5i
;;		r1_2 = c1r - c6r
;;		c7 = c5r - c5i	(implied)
;;		c8 = c4r - c4i	(implied)
;;		c9 = c3r - c3i	(implied)
;;		c10 = c2r - c2i	(implied)
;; And noticing the signs of the real and imaginary parts of the sin/cos values:
;; w^1/10 = .809 - .588i
;; w^2/10 = .309 - .951i
;; w^3/10 = -.309 - .951i
;; w^4/10 = -.809 - .588i
;; We get reals:
;; r1_1 + 2c2 + 2c3 + 2c4 + 2c5	*  w^00000
;; r1_2 + 2c2 + 2c3 + 2c4 + 2c5	*  w^01234
;; r1_1 + 2c2 + 2c3 - 2c4 - 2c5	*  w^02413
;; r1_2 + 2c2 - 2c3 - 2c4 + 2c5	*  w^03142
;; r1_1 + 2c2 - 2c3 + 2c4 - 2c5	*  w^04321
;; r1_2 - 2c2 + 2c3 - 2c4 + 2c5	*  w^00000
;; r1_1 - 2c2 + 2c3 - 2c4 + 2c5	*  w^01234
;; r1_2 - 2c2 + 2c3 + 2c4 - 2c5	*  w^02413
;; r1_1 - 2c2 - 2c3 + 2c4 + 2c5	*  w^03142
;; r1_2 - 2c2 - 2c3 - 2c4 - 2c5	*  w^04321
;; Now drop the multiplication by 2 (the actual r1_1 and r1_2 inputs are already doubled)
;; and expand the sin/cos multipliers:
;; r1_1 + c2r + c3r + c4r + c5r
;; r1_2 + .809c2r + .588c2i + .309c3r + .951c3i - .309c4r + .951c4i - .809c5r + .588c5i
;; r1_1 + .309c2r + .951c2i - .809c3r + .588c3i - .809c4r - .588c4i + .309c5r - .951c5i
;; r1_2 - .309c2r + .951c2i - .809c3r - .588c3i + .809c4r - .588c4i + .309c5r + .951c5i
;; r1_1 - .809c2r + .588c2i + .309c3r - .951c3i + .309c4r + .951c4i - .809c5r - .588c5i
;; r1_2 - c2r + c3r - c4r + c5r
;; r1_1 - .809c2r - .588c2i + .309c3r + .951c3i + .309c4r - .951c4i - .809c5r + .588c5i
;; r1_2 - .309c2r - .951c2i - .809c3r + .588c3i + .809c4r + .588c4i + .309c5r - .951c5i
;; r1_1 + .309c2r - .951c2i - .809c3r - .588c3i - .809c4r + .588c4i + .309c5r + .951c5i
;; r1_2 + .809c2r - .588c2i + .309c3r - .951c3i - .309c4r - .951c4i - .809c5r - .588c5i
;; Simplify:
;; r1_1 + c2r + c3r + c4r + c5r
;; r1_2 + .809(c2r-c5r) + .588(c2i+c5i) + .309(c3r-c4r) + .951(c3i+c4i)
;; r1_1 + .309(c2r+c5r) + .951(c2i-c5i) - .809(c3r+c4r) + .588(c3i-c4i)
;; r1_2 - .309(c2r-c5r) + .951(c2i+c5i) - .809(c3r-c4r) - .588(c3i+c4i)
;; r1_1 - .809(c2r+c5r) + .588(c2i-c5i) + .309(c3r+c4r) - .951(c3i-c4i)
;; r1_2 - c2r + c3r - c4r + c5r
;; r1_1 - .809(c2r+c5r) - .588(c2i-c5i) + .309(c3r+c4r) + .951(c3i-c4i)
;; r1_2 - .309(c2r-c5r) - .951(c2i+c5i) - .809(c3r-c4r) + .588(c3i+c4i)
;; r1_1 + .309(c2r+c5r) - .951(c2i-c5i) - .809(c3r+c4r) - .588(c3i-c4i)
;; r1_2 + .809(c2r-c5r) - .588(c2i+c5i) + .309(c3r-c4r) - .951(c3i+c4i)

yr5_10r_unfft_cmn_preload MACRO
	ENDM
yr5_10r_unfft_cmn MACRO srcreg,srcinc,d1,screg1,screg2,screg3,screg4,scregA,scincA,scregB,scincB,maxrpt,L1pt,L1pd
	vmovapd	ymm2, [srcreg+d1]		;; R2
	vmovapd	ymm4, [screg1+32]		;; cosine/sine
	vmulpd	ymm1, ymm2, ymm4		;; A2 = R2 * cosine/sine
	vmovapd	ymm5, [srcreg+4*d1]		;; R5
	vmovapd	ymm6, [screg4+32]		;; cosine/sine
	vmulpd	ymm0, ymm5, ymm6		;; A5 = R5 * cosine/sine
	vmovapd	ymm3, [srcreg+d1+32]		;; I2
	vaddpd	ymm1, ymm1, ymm3		;; A2 = A2 + I2
	vmovapd	ymm7, [srcreg+4*d1+32]		;; I5
	vaddpd	ymm0, ymm0, ymm7		;; A5 = A5 + I5
	vmulpd	ymm3, ymm3, ymm4		;; B2 = I2 * cosine/sine
	vmulpd	ymm7, ymm7, ymm6		;; B5 = I5 * cosine/sine
	vsubpd	ymm3, ymm3, ymm2		;; B2 = B2 - R2
	vsubpd	ymm7, ymm7, ymm5		;; B5 = B5 - R5
	vmovapd	ymm4, [screg1]			;; sine
	vmulpd	ymm1, ymm1, ymm4		;; A2 = A2 * sine (new R2)
	vmovapd	ymm6, [screg4]			;; sine
	vmulpd	ymm0, ymm0, ymm6		;; A5 = A5 * sine (new R5)
	vmulpd	ymm3, ymm3, ymm4		;; B2 = B2 * sine (new I2)
	vmulpd	ymm7, ymm7, ymm6		;; B5 = B5 * sine (new I5)
	vaddpd	ymm2, ymm0, ymm1		;; R2+R5
	vsubpd	ymm0, ymm0, ymm1		;; NEG(R2-R5) = R5-R2
	vaddpd	ymm1, ymm3, ymm7		;; I2+I5
	vsubpd	ymm3, ymm3, ymm7		;; I2-I5
	ystore	YMM_TMP1, ymm2			;; Temp save R2+R5
	ystore	YMM_TMP2, ymm3			;; Temp save I2-I5
	ystore	YMM_TMP3, ymm0			;; Temp save NEG(R2-R5)
	ystore	YMM_TMP4, ymm1			;; Temp save I2+I5

	vmovapd	ymm2, [srcreg+3*d1]		;; R4
	vmovapd	ymm4, [screg3+32]		;; cosine/sine
	vmulpd	ymm1, ymm2, ymm4		;; A4 = R4 * cosine/sine
	vmovapd	ymm5, [srcreg+2*d1]		;; R3
	vmovapd	ymm6, [screg2+32]		;; cosine/sine
	vmulpd	ymm0, ymm5, ymm6		;; A3 = R3 * cosine/sine
	vmovapd	ymm3, [srcreg+3*d1+32]		;; I4
	vaddpd	ymm1, ymm1, ymm3		;; A4 = A4 + I4
	vmovapd	ymm7, [srcreg+2*d1+32]		;; I3
	vaddpd	ymm0, ymm0, ymm7		;; A3 = A3 + I3
	vmulpd	ymm3, ymm3, ymm4		;; B4 = I4 * cosine/sine
	vmulpd	ymm7, ymm7, ymm6		;; B3 = I3 * cosine/sine
	vsubpd	ymm3, ymm3, ymm2		;; B4 = B4 - R4
	vsubpd	ymm7, ymm7, ymm5		;; B3 = B3 - R3
	vmovapd	ymm4, [screg3]			;; sine
	vmulpd	ymm1, ymm1, ymm4		;; A4 = A4 * sine (new R4)
	vmovapd	ymm6, [screg2]			;; sine
	vmulpd	ymm0, ymm0, ymm6		;; A3 = A3 * sine (new R3)
	vmulpd	ymm3, ymm3, ymm4		;; B4 = B4 * sine (new I4)
	vmulpd	ymm7, ymm7, ymm6		;; B3 = B3 * sine (new I3)
	vaddpd	ymm2, ymm0, ymm1		;; R3+R4
	vsubpd	ymm0, ymm0, ymm1		;; R3-R4
	vaddpd	ymm1, ymm7, ymm3		;; I3+I4
	vsubpd	ymm7, ymm7, ymm3		;; I3-I4
	ystore	YMM_TMP5, ymm0			;; Temp save R3-R4
	ystore	YMM_TMP6, ymm1			;; Temp save I3+I4

	L1prefetchw srcreg+L1pd, L1pt

	vmovapd	ymm1, YMM_TMP1			;; Reload R2+R5
	vmovapd	ymm4, YMM_P309
	vmulpd	ymm5, ymm4, ymm1		;; new R3 = .309*(R2+R5)
	vmovapd	ymm6, [srcreg]			;; R1_1
	vaddpd	ymm5, ymm5, ymm6		;; new R3 += R1_1
	vmovapd	ymm0, YMM_P809
	vmulpd	ymm3, ymm0, ymm1		;; new -R5 = -.809*(R2+R5)
	vaddpd	ymm1, ymm6, ymm1		;; new R1 = R1_1+R2+R5
	vsubpd	ymm3, ymm6, ymm3		;; new R5 += R1_1
	vmulpd	ymm0, ymm0, ymm2		;; --.809*(R3+R4)
	vaddpd	ymm1, ymm1, ymm2		;; final R1 = R1_1+R2+R5+R3+R4
	vmulpd	ymm2, ymm4, ymm2		;; .309*(R3+R4)
	vsubpd	ymm5, ymm5, ymm0		;; new R3 += -.809*(R3+R4)
	vaddpd	ymm3, ymm3, ymm2		;; new R5 += .309*(R3+R4)

	L1prefetchw srcreg+d1+L1pd, L1pt

	vmovapd	ymm0, YMM_TMP2			;; Reload I2-I5
	vmovapd	ymm6, YMM_P951
	vmulpd	ymm2, ymm6, ymm0		;; tmp1 = .951*(I2-I5)
	vmovapd	ymm4, YMM_P588
	vmulpd	ymm0, ymm4, ymm0		;; tmp2 = .588*(I2-I5)
	vmulpd	ymm4, ymm4, ymm7		;; .588*(I3-I4)
	vmulpd	ymm7, ymm6, ymm7		;; .951*(I3-I4)
	vaddpd	ymm2, ymm2, ymm4		;; tmp1 += .588*(I3-I4)
	vsubpd	ymm0, ymm0, ymm7		;; tmp2 -= .951*(I3-I4)

	vsubpd	ymm4, ymm5, ymm2		;; final R9 = new R3 - tmp1
	vaddpd	ymm5, ymm5, ymm2		;; final R3 = new R3 + tmp1
	vsubpd	ymm2, ymm3, ymm0		;; final R7 = new R5 - tmp2
	vaddpd	ymm3, ymm3, ymm0		;; final R5 = new R5 + tmp2

	L1prefetchw srcreg+2*d1+L1pd, L1pt

	ystore	[srcreg], ymm1			;; Save final R1
	ystore	[srcreg+2*d1], ymm5		;; Save final R3
	ystore	[srcreg+4*d1], ymm3		;; Save final R5

	vmovapd	ymm3, YMM_TMP5			;; Reload R3-R4
	vmovapd	ymm0, YMM_P309
	vmulpd	ymm1, ymm0, ymm3		;; new R2 = .309*(R3-R4)
	vmovapd	ymm7, [srcreg+32]		;; R1_2
	vaddpd	ymm1, ymm1, ymm7		;; new R2 += R1_2
	vmovapd	ymm5, YMM_P809
	vmulpd	ymm6, ymm5, ymm3		;; new -R4 = -.809*(R3-R4)
	vaddpd	ymm3, ymm7, ymm3		;; new R6 = R1_2+(R3-R4)
	vsubpd	ymm6, ymm7, ymm6		;; new R4 += R1_2
	vmovapd	ymm7, YMM_TMP3			;; Reload NEG(R2-R5)
	vmulpd	ymm5, ymm5, ymm7		;; --.809*NEG(R2-R5)
	vaddpd	ymm3, ymm3, ymm7		;; final R6 = R6+(R3-R4)+NEG(R2-R5)
	vmulpd	ymm0, ymm0, ymm7		;; .309*NEG(R2-R5)
	vsubpd	ymm1, ymm1, ymm5		;; new R2 += -.809*NEG(R2-R5)
	vaddpd	ymm6, ymm6, ymm0		;; new R4 += .309*NEG(R2-R5)

	L1prefetchw srcreg+3*d1+L1pd, L1pt

	vmovapd	ymm0, YMM_TMP6			;; Reload I3+I4
	vmovapd	ymm5, YMM_P588
	vmulpd	ymm7, ymm5, ymm0		;; tmp2 = .588*(I3+I4)
	vmulpd	ymm0, ymm0, YMM_P951		;; tmp1 = .951*(I3+I4)
	ystore	[srcreg+32], ymm3		;; Save final I1
	vmovapd	ymm3, YMM_TMP4			;; Reload I2+I5
	vmulpd	ymm5, ymm5, ymm3		;; .588*(I2+I5)
	vmulpd	ymm3, ymm3, YMM_P951		;; .951*(I2+I5)
	vaddpd	ymm0, ymm0, ymm5		;; tmp1 += .588*(I2+I5)
	vsubpd	ymm7, ymm7, ymm3		;; tmp2 -= .951*(I2+I5)

	L1prefetchw srcreg+4*d1+L1pd, L1pt

	vsubpd	ymm3, ymm1, ymm0		;; Final R10 = new R2 - tmp1
	vaddpd	ymm1, ymm1, ymm0		;; Final R2 = new R2 + tmp1
	vsubpd	ymm0, ymm6, ymm7		;; Final R4 = new R4 - tmp2
	vaddpd	ymm6, ymm6, ymm7		;; Final R8 = new R4 + tmp2

	ystore	[srcreg+d1], ymm1		;; Save R2
	ystore	[srcreg+3*d1], ymm0		;; Save R4
	ystore	[srcreg+d1+32], ymm2		;; Save I2
	ystore	[srcreg+2*d1+32], ymm6		;; Save I3
	ystore	[srcreg+3*d1+32], ymm4		;; Save I4
	ystore	[srcreg+4*d1+32], ymm3		;; Save I5
	bump	srcreg, srcinc
	bump	scregA, scincA
	bump	scregB, scincB
	ENDM


;;
;; ************************************* ten-reals-five-complex-fft variants ******************************************
;;

yr5_5cl_ten_reals_five_complex_djbfft_preload MACRO
	yr5_o10r_t5c_djbfft_mem_preload
	ENDM
yr5_5cl_ten_reals_five_complex_djbfft MACRO srcreg,srcinc,d1,screg,scinc,maxrpt
	d2=2*d1
	d3=3*d1
	d4=4*d1
	yr5_o10r_t5c_djbfft_mem [srcreg],[srcreg+d1],[srcreg+d2],[srcreg+d3],[srcreg+d4],screg,[srcreg],[srcreg+32],[srcreg+d1],[srcreg+d1+32]
;;	ystore	[srcreg+d1], ymm2		;; Save R2
;;	ystore	[srcreg+d1+32], ymm4		;; Save I2
	ystore	[srcreg+d2], ymm4		;; Save R3
	ystore	[srcreg+d2+32], ymm7		;; Save I3
	ystore	[srcreg+d3], ymm5		;; Save R4
	ystore	[srcreg+d3+32], ymm1		;; Save I4
	ystore	[srcreg+d4], ymm6		;; Save R5
	ystore	[srcreg+d4+32], ymm3		;; Save I5
	bump	srcreg, srcinc
	bump	screg, scinc
	ENDM

;; To calculate a 10-reals FFT (in a shorthand notation):
;; r1 + r2 + r3 + r4 + r5 + r6 + r7 + r8 + r9 + r10	*  w^0000000000
;; r1 + r2 + r3 + r4 + r5 + r6 + r7 + r8 + r9 + r10	*  w^0123456789
;; r1 + r2 + r3 + r4 + r5 + r6 + r7 + r8 + r9 + r10	*  w^0246802468
;; r1 + r2 + r3 + r4 + r5 + r6 + r7 + r8 + r9 + r10	*  w^0369258147
;; r1 + r2 + r3 + r4 + r5 + r6 + r7 + r8 + r9 + r10	*  w^0482604826
;; r1 + r2 + r3 + r4 + r5 + r6 + r7 + r8 + r9 + r10	*  w^0505050505
;; r1 + r2 + r3 + r4 + r5 + r6 + r7 + r8 + r9 + r10	*  w^0628406284
;; r1 + r2 + r3 + r4 + r5 + r6 + r7 + r8 + r9 + r10	*  w^0741852963
;; r1 + r2 + r3 + r4 + r5 + r6 + r7 + r8 + r9 + r10	*  w^0864208642
;; r1 + r2 + r3 + r4 + r5 + r6 + r7 + r8 + r9 + r10	*  w^0987654321
;; Noting that w^5 = -1 and that Hermetian symmetry means we won't need
;; to calculate the last 5 rows:
;; r1 + r2 + r3 + r4 + r5 + r6 + r7 + r8 + r9 + r10	*  w^0000000000
;; r1 + r2 + r3 + r4 + r5 - r6 - r7 - r8 - r9 - r10	*  w^0123401234
;; r1 + r2 + r3 - r4 - r5 + r6 + r7 + r8 - r9 - r10	*  w^0241302413
;; r1 + r2 - r3 - r4 + r5 - r6 - r7 + r8 + r9 - r10	*  w^0314203142
;; r1 + r2 - r3 + r4 - r5 + r6 + r7 - r8 + r9 - r10	*  w^0432104321
;; Reorganize into odds and evens, that is
;; r1 + r3 + r5 + r7 + r9  * powers + (r2 + r4 + r6 + r8 + r10) * powers
;; Thus:
;; r1 + r3 + r5 + r7 + r9	*  w^00000	+ r2 + r4 + r6 + r8 + r10	*  w^00000
;; r1 + r3 + r5 - r7 - r9	*  w^02413	+ r2 + r4 - r6 - r8 - r10	*  w^13024
;; r1 + r3 - r5 + r7 - r9	*  w^04321	+ r2 - r4 + r6 + r8 - r10	*  w^21043
;; r1 - r3 + r5 - r7 + r9	*  w^01234	+ r2 - r4 - r6 + r8 - r10	*  w^34012
;; r1 - r3 - r5 + r7 + r9	*  w^03142	+ r2 + r4 + r6 - r8 - r10	*  w^42031
;; Apply the sin/cos values:
;; w^1/10 = .809 + .588i
;; w^2/10 = .309 + .951i
;; w^3/10 = -.309 + .951i
;; w^4/10 = -.809 + .588i
;; reals:
;; r1 + r3 + r5 + r7 + r9			+ r2 + r4 + r6 + r8 + r10
;; r1 + .309r3 - .809r5 - .809r7 + .309r9	+ .809r2 - .309r4 - r6 - .309r8 + .809r10
;; r1 - .809r3 + .309r5 + .309r7 - .809r9	+ .309r2 - .809r4 + r6 - .809r8 + .309r10
;; r1 - .809r3 + .309r5 + .309r7 - .809r9	+ -.309r2 + .809r4 - r6 + .809r8 - .309r10
;; r1 + .309r3 - .809r5 - .809r7 + .309r9	+ -.809r2 + .309r4 + r6 + .309r8 - .809r10
;; imaginarys:
;; 0						+ 0
;;  + .951r3 + .588r5 - .588r7 - .951r9		+ .588r2 + .951r4 - .951r8 - .588r10
;;  + .588r3 - .951r5 + .951r7 - .588r9		+ .951r2 - .588r4 + .588r8 - .951r10
;;  - .588r3 + .951r5 - .951r7 + .588r9		+ .951r2 - .588r4 + .588r8 - .951r10
;;  - .951r3 - .588r5 + .588r7 + .951r9		+ .588r2 + .951r4 - .951r8 - .588r10
;; Further simplifying:
;; reals:
;; r1 + r3 + r5 + r7 + r9		+ r2 + r4 + r6 + r8 + r10
;; r1 + .309(r3+r9) - .809(r5+r7)	+ .809(r2+r10) - .309(r4+r8) - r6
;; r1 - .809(r3+r9) + .309(r5+r7)	+ .309(r2+r10) - .809(r4+r8) + r6
;; r1 - .809(r3+r9) + .309(r5+r7)	+ -.309(r2+r10) + .809(r4+r8) - r6
;; r1 + .309(r3+r9) - .809(r5+r7)	+ -.809(r2+r10) + .309(r4+r8) + r6
;; imaginarys:
;; 0					+ 0
;;  + .951(r3-r9) + .588(r5-r7)		+ .588(r2-r10) + .951(r4-r8)
;;  + .588(r3-r9) - .951(r5-r7)		+ .951(r2-r10) - .588(r4-r8)
;;  - .588(r3-r9) + .951(r5-r7)		+ .951(r2-r10) - .588(r4-r8)
;;  - .951(r3-r9) - .588(r5-r7)		+ .588(r2-r10) + .951(r4-r8)

yr5_o10r_t5c_djbfft_mem_preload MACRO
	ENDM

yr5_o10r_t5c_djbfft_mem MACRO memr1,memr2,memr3,memr4,memr5,screg,dst1,dst2,dst3,dst4
						;; Ten reals comments		; Five complex comments
	vmovapd	ymm2, memr2		;; R2					; r2
	vmovapd	ymm3, memr3		;; R3					; r3
	vblendpd ymm0, ymm2, ymm3, 1	;; R3					; r2
	vmovapd	ymm5, memr5		;; R5					; r5
	vblendpd ymm1, ymm5, memr4[32], 1 ;; R9					; r5
	vsubpd	ymm7, ymm0, ymm1	;; R3-R9				; r25s=r2-r5
	vaddpd	ymm0, ymm0, ymm1	;; R3+R9				; r25a=r2+r5
	vblendpd ymm3, ymm3, ymm5, 1	;; R5					; r3
	vmovapd	ymm4, memr4		;; R4					; r4
	vblendpd ymm4, ymm4, memr2[32], 1 ;; R7					; r4
	vsubpd	ymm6, ymm3, ymm4	;; R5-R7				; r34s=r3-r4
	vaddpd	ymm3, ymm3, ymm4	;; R5+R7				; r34a=r3+r4
	vmovapd	ymm1, YMM_P951
	vmulpd	ymm2, ymm1, ymm7	;; new oddI2 = .951*(R3-R9)		; sin2*r25s
	vmovapd	ymm5, YMM_P588
	vmulpd	ymm7, ymm5, ymm7	;; new oddI3 = .588*(R3-R9)		; sin4*r25s
	vmulpd	ymm5, ymm5, ymm6	;; .588*(R5-R7)				; sin4*r34s
	vmulpd	ymm1, ymm1, ymm6	;; .951*(R5-R7)				; sin2*r34s
	vaddpd	ymm2, ymm2, ymm5	;; new oddI2 += .588*(R5-R7)		; t6=sin2*r25s + sin4*r34s
	vsubpd	ymm7, ymm7, ymm1	;; new oddI3 -= .951*(R5-R7)		; t8=sin4*r25s - sin2*r34s
	vmovapd	ymm1, YMM_P309
	vmulpd	ymm5, ymm1, ymm0	;; new oddR2 = .309*(R3+R9)		; cos2*r25a
	vmovapd	ymm4, memr1		;; R1					; r1
	vaddpd	ymm5, ymm5, ymm4	;; new oddR2 += R1			; cos2*r25a + r1
	vmovapd	ymm6, YMM_P809
	ystore	YMM_TMP6, ymm2		;; Save oddI2				; Save t6
	vmulpd	ymm2, ymm6, ymm0	;; new -oddR3 = -.809*(R3+R9)		; cos4*r25a
	vaddpd	ymm0, ymm4, ymm0	;; new oddR1 = R1+R3+R9			; r1 + r25a
	vsubpd	ymm2, ymm4, ymm2	;; new oddR3 += R1			; cos4*r25a + r1
	vmulpd	ymm4, ymm6, ymm3	;; --.809*(R5+R7)			; cos4*r34a
	vaddpd	ymm0, ymm0, ymm3	;; new oddR1 = R1+R3+R5+R7+R9		; outr(0) = r1+r25a+r34a
	vmulpd	ymm3, ymm1, ymm3	;; .309*(R5+R7)				; cos2*r34a
	vsubpd	ymm5, ymm5, ymm4	;; new oddR2 += -.809*(R5+R7)		; t1=cos2*r25a + cos4*r34a + r1
	vaddpd	ymm2, ymm2, ymm3	;; new oddR3 += .309*(R5+R7)		; t3=cos4*r25a + cos2*r34a + r1
	ystore	YMM_TMP8, ymm7		;; Save oddI3				; Save t8
	ystore	dst1, ymm0		;; Save oddR1				; Save R1
	ystore	YMM_TMP1, ymm5		;; Save oddR2				; Save t1
	ystore	YMM_TMP3, ymm2		;; Save oddR3				; Save t3

	vmovapd	ymm2, memr2[32]		;; R7					; i2
	vblendpd ymm2, ymm2, memr2, 1	;; R2					; i2
	vmovapd	ymm5, memr5[32]		;; R10					; i5
	vsubpd	ymm7, ymm2, ymm5	;; R2-R10				; i25s=i2-i5
	vaddpd	ymm2, ymm2, ymm5	;; R2+R10				; i25a=i2+i5
	vmovapd	ymm4, memr4[32]		;; R9					; i4
	vblendpd ymm4, ymm4, memr4, 1	;; R4					; i4
	vmovapd	ymm3, memr3[32]		;; R8					; i3
	vsubpd	ymm6, ymm4, ymm3	;; R4-R8				; NEGi34s=i4-i3
	vaddpd	ymm4, ymm4, ymm3	;; R4+R8				; i34a=i4+i3

	vmovapd	ymm0, YMM_P951
	vmulpd	ymm1, ymm0, ymm7	;; new evenI3 = .951*(R2-R10)		; sin2*i25s
	vmovapd	ymm3, YMM_P588
	vmulpd	ymm7, ymm3, ymm7	;; new evenI2 = .588*(R2-R10)		; sin4*i25s
	vmulpd	ymm3, ymm3, ymm6	;; .588*(R4-R8)				; sin4*NEGi34s
	vmulpd	ymm0, ymm0, ymm6	;; .951*(R4-R8)				; sin2*NEGi34s
	vsubpd	ymm1, ymm1, ymm3	;; new evenI3 -= .588*(R4-R8)		; t2=sin2*i25s - sin4*NEGi34s
	vaddpd	ymm7, ymm7, ymm0	;; new evenI2 += .951*(R4-R8)		; t4=sin4*i25s + sin2*NEGi34s

	vmulpd	ymm3, ymm2, YMM_P309	;; new evenR3 = .309*(R2+R10)		; cos2*i25a
	vmovapd	ymm5, memr1[32]		;; R6					; i1
	vaddpd	ymm3, ymm3, ymm5	;; new evenR3 += R6			; cos2*i25a + i1
	vmovapd	ymm6, YMM_P809
	vmulpd	ymm0, ymm6, ymm2	;; new -evenR5 = -.809*(R2+R10)		; cos4*i25a
	vaddpd	ymm2, ymm5, ymm2	;; new evenR1 = R6+R2+R10		; i1 + i25a
	vsubpd	ymm0, ymm5, ymm0	;; new evenR5 += R6			; cos4*i25a+ i1
	vmulpd	ymm6, ymm6, ymm4	;; --.809*(R4+R8)			; cos4*i34a
	vaddpd	ymm2, ymm2, ymm4	;; new evenR1 = R6+R2+R4+R8+R10		; outi(0)=i1+i25a+i34a
	vmulpd	ymm4, ymm4, YMM_P309	;; .309*(R4+R8)				; cos2*i34a
	vsubpd	ymm3, ymm3, ymm6	;; new evenR3 += -.809*(R4+R8)		; t5=cos2*i25a + cos4*i34a + i1
	vaddpd	ymm0, ymm0, ymm4	;; new evenR5 += .309*(R4+R8)		; t7=cos4*i25a + cos2*i34a + i1
	ystore	dst2, ymm2		;; Save evenR1				; Save I1

	vblendpd ymm5, ymm3, ymm7, 1	;; evenI2				; t5
	vblendpd ymm4, ymm7, ymm3, 1	;; evenR3				; t4
	vblendpd ymm2, ymm1, ymm0, 1	;; evenR5				; t2
	vblendpd ymm7, ymm0, ymm1, 1	;; evenI3				; t7

	vmovapd	ymm1, YMM_TMP1		;; oddR2				; t1
	vaddpd	ymm0, ymm1, ymm2	;; New R5 = oddR2 + evenR5		; outr(4)=t1+t2
	vmovapd	ymm6, YMM_TMP6		;; oddI2				; t6
	vsubpd	ymm3, ymm5, ymm6	;; New I5 = evenI2 - oddI2		; outi(4)=t5-t6
	vsubpd	ymm1, ymm1, ymm2	;; New R2 = oddR2 - evenR5		; outr(1)=t1-t2
	vaddpd	ymm5, ymm5, ymm6	;; New I2 = evenI2 + oddI2		; outi(1)=t5+t6

	vmovapd	ymm2, [screg+192+32]			;; cosine/sine for w^4
	vmulpd	ymm6, ymm0, ymm2			;; A5 = R5 * cosine/sine
	vsubpd	ymm6, ymm6, ymm3			;; A5 = A5 - I5
	vmulpd	ymm3, ymm3, ymm2			;; B5 = I5 * cosine/sine
	vaddpd	ymm3, ymm3, ymm0			;; B5 = B5 + R5

	vmovapd	ymm2, [screg+32]			;; cosine/sine for w^1
	vmulpd	ymm0, ymm1, ymm2			;; A2 = R2 * cosine/sine
	vsubpd	ymm0, ymm0, ymm5			;; A2 = A2 - I2
	vmulpd	ymm5, ymm5, ymm2			;; B2 = I2 * cosine/sine
	vaddpd	ymm5, ymm5, ymm1			;; B2 = B2 + R2

	vmovapd	ymm1, [screg+192]
	vmulpd	ymm6, ymm6, ymm1			;; A5 = A5 * sine (final R5)
	vmulpd	ymm3, ymm3, ymm1			;; B5 = B5 * sine (final I5)
	vmovapd	ymm1, [screg]
	vmulpd	ymm0, ymm0, ymm1			;; A2 = A2 * sine (final R2)
	vmulpd	ymm5, ymm5, ymm1			;; B2 = B2 * sine (final I2)

	ystore	dst3, ymm0				;; Save final R2
	ystore	dst4, ymm5				;; Save final I2

	vmovapd	ymm0, YMM_TMP3		;; oddR3				; t3
	vsubpd	ymm2, ymm0, ymm4	;; New R4 = oddR3 - evenR3		; outr(2)=t3-t4
	vaddpd	ymm0, ymm0, ymm4	;; New R3 = oddR3 + evenR3		; outr(3)=t3+t4
	vmovapd	ymm5, YMM_TMP8		;; oddI3				; t8
	vsubpd	ymm1, ymm7, ymm5	;; New I4 = evenI3 - oddI3		; outi(3)=t7-t8
	vaddpd	ymm7, ymm7, ymm5	;; New I3 = evenI3 + oddI3		; outi(2)=t7+t8

	vblendpd ymm4, ymm0, ymm2, 1			;; R4
	vblendpd ymm2, ymm2, ymm0, 1			;; R3

	vmovapd	ymm0, [screg+128+32]			;; cosine/sine for w^3
	vmulpd	ymm5, ymm4, ymm0			;; A4 = R4 * cosine/sine
	vsubpd	ymm5, ymm5, ymm1			;; A4 = A4 - I4
	vmulpd	ymm1, ymm1, ymm0			;; B4 = I4 * cosine/sine
	vaddpd	ymm1, ymm1, ymm4			;; B4 = B4 + R4

	vmovapd	ymm0, [screg+64+32]			;; cosine/sine for w^2
	vmulpd	ymm4, ymm2, ymm0			;; A3 = R3 * cosine/sine
	vsubpd	ymm4, ymm4, ymm7			;; A3 = A3 - I3
	vmulpd	ymm7, ymm7, ymm0			;; B3 = I3 * cosine/sine
	vaddpd	ymm7, ymm7, ymm2			;; B3 = B3 + R3

	vmovapd	ymm2, [screg+128]
	vmulpd	ymm5, ymm5, ymm2			;; A4 = A4 * sine (final R4)
	vmulpd	ymm1, ymm1, ymm2			;; B4 = B4 * sine (final I4)
	vmovapd	ymm2, [screg+64]
	vmulpd	ymm4, ymm4, ymm2			;; A3 = A3 * sine (final R3)
	vmulpd	ymm7, ymm7, ymm2			;; B3 = B3 * sine (final I3)
	ENDM

;; 64-bit version

IFDEF X86_64

yr5_o10r_t5c_djbfft_mem_preload MACRO
	vmovapd	ymm15, YMM_P809
	vmovapd	ymm14, YMM_P309
	vmovapd	ymm13, YMM_P588
	vmovapd	ymm12, YMM_P951
	ENDM

yr5_o10r_t5c_djbfft_mem MACRO memr1,memr2,memr3,memr4,memr5,screg,dst1,dst2,dst3,dst4
						;; Ten reals comments		; Five complex comments
	vmovapd	ymm7, memr2		;; R2					; r2
	vmovapd	ymm2, memr2[32]		;; R7					; i2
	vblendpd ymm1, ymm2, ymm7, 1	;; R2					; i2

	vmovapd	ymm3, memr5[32]		;; R10					; i5
	vsubpd	ymm10, ymm1, ymm3	;; R2-R10				; i25s=i2-i5			; 1-3
	vaddpd	ymm1, ymm1, ymm3	;; R2+R10				; i25a=i2+i5			; 2-4

	vmovapd	ymm3, memr4		;; R4					; r4
	vmovapd	ymm9, memr4[32]		;; R9					; i4
	vblendpd ymm8, ymm9, ymm3, 1	;; R4					; i4

	vmovapd	ymm4, memr3[32]		;; R8					; i3
	vsubpd	ymm11, ymm8, ymm4	;; R4-R8				; NEGi34s=i4-i3			; 3-5
	vaddpd	ymm8, ymm8, ymm4	;; R4+R8				; i34a=i4+i3			; 4-6

	vmulpd	ymm5, ymm12, ymm10	;; new evenI3 = .951*(R2-R10)		; sin2*i25s			;	4-8
	vmulpd	ymm10, ymm13, ymm10	;; new evenI2 = .588*(R2-R10)		; sin4*i25s			;	5-9

	vmovapd	ymm4, memr3		;; R3					; r3
	vblendpd ymm0, ymm7, ymm4, 1	;; R3					; r2

	vmovapd	ymm6, memr5		;; R5					; r5
	vblendpd ymm7, ymm6, ymm9, 1	;; R9					; r5

	vsubpd	ymm9, ymm0, ymm7	;; R3-R9				; r25s=r2-r5			; 5-7
	vaddpd	ymm0, ymm0, ymm7	;; R3+R9				; r25a=r2+r5			; 6-8

	vblendpd ymm4, ymm4, ymm6, 1	;; R5					; r3
	vblendpd ymm3, ymm3, ymm2, 1	;; R7					; r4

	vmulpd	ymm7, ymm13, ymm11	;; .588*(R4-R8)				; sin4*NEGi34s			;	6-10
	vmulpd	ymm11, ymm12, ymm11	;; .951*(R4-R8)				; sin2*NEGi34s			;	7-11

	vsubpd	ymm6, ymm4, ymm3	;; R5-R7				; r34s=r3-r4			; 7-9
	vaddpd	ymm4, ymm4, ymm3	;; R5+R7				; r34a=r3+r4			; 8-10

	vmulpd	ymm3, ymm14, ymm1	;; new evenR3 = .309*(R2+R10)		; cos2*i25a			;	8-12
	vmulpd	ymm2, ymm15, ymm1	;; new -evenR5 = -.809*(R2+R10)		; cos4*i25a			;	9-13

	vsubpd	ymm5, ymm5, ymm7	;; new evenI3 -= .588*(R4-R8)		; t2=sin2*i25s - sin4*NEGi34s	; 11-13

	vmovapd	ymm7, memr1[32]		;; R6					; i1
	vaddpd	ymm1, ymm7, ymm1	;; new evenR1 = R6+R2+R10		; i1 + i25a			; 10-12

	vaddpd	ymm10, ymm10, ymm11	;; new evenI2 += .951*(R4-R8)		; t4=sin4*i25s + sin2*NEGi34s	; 12-14
	vmulpd	ymm11, ymm15, ymm8	;; --.809*(R4+R8)			; cos4*i34a			;	10-14

	vaddpd	ymm3, ymm3, ymm7	;; new evenR3 += R6			; cos2*i25a + i1		; 13-15
	vsubpd	ymm2, ymm7, ymm2	;; new evenR5 += R6			; cos4*i25a + i1		; 14-16

	vmulpd	ymm7, ymm14, ymm8	;; .309*(R4+R8)				; cos2*i34a			;	11-15

	vaddpd	ymm1, ymm1, ymm8	;; new evenR1 = R6+R2+R4+R8+R10		; outi(0)=i1+i25a+i34a		; 15-17

	vmulpd	ymm8, ymm12, ymm9	;; new oddI2 = .951*(R3-R9)		; sin2*r25s			;	12-16
	vmulpd	ymm9, ymm13, ymm9	;; new oddI3 = .588*(R3-R9)		; sin4*r25s			;	13-17

	vsubpd	ymm3, ymm3, ymm11	;; new evenR3 += -.809*(R4+R8)		; t5=cos2*i25a + cos4*i34a + i1	; 16-18

	vmulpd	ymm11, ymm13, ymm6	;; .588*(R5-R7)				; sin4*r34s			;	14-18
	vmulpd	ymm6, ymm12, ymm6	;; .951*(R5-R7)				; sin2*r34s			;	15-19

	vaddpd	ymm2, ymm2, ymm7	;; new evenR5 += .309*(R4+R8)		; t7=cos4*i25a + cos2*i34a + i1	; 17-19

	vmulpd	ymm7, ymm14, ymm0	;; new oddR2 = .309*(R3+R9)		; cos2*r25a			;	16-20
	ystore	dst2, ymm1		;; Save evenR1				; Save I1			;		18
	vmulpd	ymm1, ymm15, ymm0	;; new -oddR3 = -.809*(R3+R9)		; cos4*r25a			;	17-21

	vaddpd	ymm8, ymm8, ymm11	;; new oddI2 += .588*(R5-R7)		; t6=sin2*r25s + sin4*r34s	; 19-21

	vmovapd	ymm11, memr1		;; R1					; r1
	vaddpd	ymm0, ymm11, ymm0	;; new oddR1 = R1+R3+R9			; r1 + r25a			; 18-20

	vsubpd	ymm9, ymm9, ymm6	;; new oddI3 -= .951*(R5-R7)		; t8=sin4*r25s - sin2*r34s	; 20-22

	vmulpd	ymm6, ymm15, ymm4	;; --.809*(R5+R7)			; cos4*r34a			;	18-22

	vaddpd	ymm7, ymm7, ymm11	;; new oddR2 += R1			; cos2*r25a + r1		; 21-23
	vsubpd	ymm1, ymm11, ymm1	;; new oddR3 += R1			; cos4*r25a + r1		; 22-24

	vmulpd	ymm11, ymm14, ymm4	;; .309*(R5+R7)				; cos2*r34a			;	19-23

	vaddpd	ymm0, ymm0, ymm4	;; new oddR1 = R1+R3+R5+R7+R9		; outr(0) = r1+r25a+r34a	; 23-25

	vblendpd ymm4, ymm3, ymm10, 1	;; evenI2				; t5				;		19
	vblendpd ymm10, ymm10, ymm3, 1	;; evenR3				; t4				;		20

	vsubpd	ymm7, ymm7, ymm6	;; new oddR2 += -.809*(R5+R7)		; t1=cos2*r25a + cos4*r34a + r1	; 24-26
	vaddpd	ymm1, ymm1, ymm11	;; new oddR3 += .309*(R5+R7)		; t3=cos4*r25a + cos2*r34a + r1	; 25-27

	vmovapd	ymm6, [screg+192+32]			;; cosine/sine for w^4

	vblendpd ymm3, ymm5, ymm2, 1	;; evenR5				; t2				;		21
	vblendpd ymm2, ymm2, ymm5, 1	;; evenI3				; t7				;		22

	vsubpd	ymm5, ymm4, ymm8	;; New I5 = evenI2 - oddI2		; outi(4)=t5-t6			; 26-28
	vaddpd	ymm4, ymm4, ymm8	;; New I2 = evenI2 + oddI2		; outi(1)=t5+t6			; 27-29

	vmovapd	ymm11, [screg+32]			;; cosine/sine for w^1
	ystore	dst1, ymm0		;; Save oddR1				; Save R1			;		26

	vaddpd	ymm0, ymm7, ymm3	;; New R5 = oddR2 + evenR5		; outr(4)=t1+t2			; 28-30
	vsubpd	ymm7, ymm7, ymm3	;; New R2 = oddR2 - evenR5		; outr(1)=t1-t2			; 29-31

	vmulpd	ymm3, ymm5, ymm6			;; B5 = I5 * cosine/sine				;	30-34

	vsubpd	ymm8, ymm1, ymm10	;; New R4 = oddR3 - evenR3		; outr(2)=t3-t4			; 30-32
	vaddpd	ymm1, ymm1, ymm10	;; New R3 = oddR3 + evenR3		; outr(3)=t3+t4			; 31-33

	vmulpd	ymm6, ymm0, ymm6			;; A5 = R5 * cosine/sine				;	31-35

	vsubpd	ymm10, ymm2, ymm9	;; New I4 = evenI3 - oddI3		; outi(3)=t7-t8			; 32-34
	vaddpd	ymm2, ymm2, ymm9	;; New I3 = evenI3 + oddI3		; outi(2)=t7+t8			; 33-35

	vmulpd	ymm9, ymm4, ymm11			;; B2 = I2 * cosine/sine				;	32-36
	vmulpd	ymm11, ymm7, ymm11			;; A2 = R2 * cosine/sine				;	33-37

	vaddpd	ymm3, ymm3, ymm0			;; B5 = B5 + R5						; 35-37

	vblendpd ymm0, ymm1, ymm8, 1			;; R4							;		34
	vblendpd ymm8, ymm8, ymm1, 1			;; R3							;		35
	vmovapd	ymm1, [screg+128+32]			;; cosine/sine for w^3

	vsubpd	ymm6, ymm6, ymm5			;; A5 = A5 - I5						; 36-38

	vmulpd	ymm5, ymm0, ymm1			;; A4 = R4 * cosine/sine				;	34-38
	vmulpd	ymm1, ymm10, ymm1			;; B4 = I4 * cosine/sine				;	35-39

	vaddpd	ymm9, ymm9, ymm7			;; B2 = B2 + R2						; 37-39
	vmovapd	ymm7, [screg+64+32]			;; cosine/sine for w^2
	vsubpd	ymm11, ymm11, ymm4			;; A2 = A2 - I2						; 38-40

	vmulpd	ymm4, ymm8, ymm7			;; A3 = R3 * cosine/sine				;	36-40
	vmulpd	ymm7, ymm2, ymm7			;; B3 = I3 * cosine/sine				;	37-41

	vsubpd	ymm5, ymm5, ymm10			;; A4 = A4 - I4						; 39-41
	vmovapd	ymm10, [screg+192]

	vmulpd	ymm3, ymm3, ymm10			;; B5 = B5 * sine (final I5)				;	38-42
	vmulpd	ymm6, ymm6, ymm10			;; A5 = A5 * sine (final R5)				;	39-43
	vmovapd	ymm10, [screg]

	vaddpd	ymm1, ymm1, ymm0			;; B4 = B4 + R4						; 40-42
	vmovapd	ymm0, [screg+128]

	vmulpd	ymm9, ymm9, ymm10			;; B2 = B2 * sine (final I2)				;	40-44
	vmulpd	ymm11, ymm11, ymm10			;; A2 = A2 * sine (final R2)				;	41-45
	vmovapd	ymm10, [screg+64]

	vsubpd	ymm4, ymm4, ymm2			;; A3 = A3 - I3						; 41-43
	vaddpd	ymm7, ymm7, ymm8			;; B3 = B3 + R3						; 42-44

	vmulpd	ymm5, ymm5, ymm0			;; A4 = A4 * sine (final R4)				;	42-46
	vmulpd	ymm1, ymm1, ymm0			;; B4 = B4 * sine (final I4)				;	43-47

	vmulpd	ymm4, ymm4, ymm10			;; A3 = A3 * sine (final R3)				;	44-48
	vmulpd	ymm7, ymm7, ymm10			;; B3 = B3 * sine (final I3)				;	45-49

	ystore	dst4, ymm9				;; Save final I2					;		45
	ystore	dst3, ymm11				;; Save final R2					;		46

	ENDM

;; Haswell FMA3 version

IF (@INSTR(,%yarch,<FMA3>) NE 0)

yr5_o10r_t5c_djbfft_mem_preload MACRO
	vmovapd	ymm15, YMM_P809
	vmovapd	ymm14, YMM_P309
	vmovapd	ymm13, YMM_P588
	vmovapd	ymm12, YMM_P951
	vmovapd	ymm11, YMM_ONE
	ENDM

yr5_o10r_t5c_djbfft_mem MACRO memr1,memr2,memr3,memr4,memr5,screg,dst1,dst2,dst3,dst4
						;; Ten reals comments		; Five complex comments
	vmovapd	ymm4, memr2		;; R2					; r2
	vmovapd	ymm7, memr2[32]		;; R7					; i2
	vblendpd ymm10, ymm7, ymm4, 1	;; R2					; i2

	vmovapd	ymm3, memr5[32]		;; R10					; i5

	yfmsubpd ymm1, ymm10, ymm11, ymm3 ;; R2-R10				; i25s=i2-i5			; 1-5
	yfmaddpd ymm10, ymm10, ymm11, ymm3 ;; R2+R10				; i25a=i2+i5			;	1-5

	vmovapd	ymm8, memr3		;; R3					; r3
	vblendpd ymm0, ymm4, ymm8, 1	;; R3					; r2

	vmovapd	ymm6, memr5		;; R5					; r5
	vmovapd	ymm9, memr4[32]		;; R9					; i4
	vblendpd ymm5, ymm6, ymm9, 1	;; R9					; r5

	yfmsubpd ymm4, ymm0, ymm11, ymm5 ;; R3-R9				; r25s=r2-r5			; 2-6
	yfmaddpd ymm0, ymm0, ymm11, ymm5 ;; R3+R9				; r25a=r2+r5			;	2-6

	vmovapd	ymm3, memr4		;; R4					; r4
	vblendpd ymm2, ymm9, ymm3, 1	;; R4					; i4
	vblendpd ymm3, ymm3, ymm7, 1	;; R7					; r4

	vmovapd	ymm9, memr3[32]		;; R8					; i3
	yfmsubpd ymm7, ymm2, ymm11, ymm9 ;; R4-R8				; NEGi34s=i4-i3			; 3-7
	yfmaddpd ymm2, ymm2, ymm11, ymm9 ;; R4+R8				; i34a=i4+i3			;	3-7

	vblendpd ymm8, ymm8, ymm6, 1	;; R5					; r3
	vmovapd	ymm5, memr1[32]		;; R6					; i1

	yfmsubpd ymm6, ymm8, ymm11, ymm3 ;; R5-R7				; r34s=r3-r4			; 4-8
	yfmaddpd ymm8, ymm8, ymm11, ymm3 ;; R5+R7				; r34a=r3+r4			;	4-8

	vmulpd	ymm3, ymm12, ymm1	;; new evenI3 = .951*(R2-R10)		; sin2*i25s			;	6-10
	vmulpd	ymm1, ymm13, ymm1	;; new evenI2 = .588*(R2-R10)		; sin4*i25s			; 6-10

	yfmaddpd ymm9, ymm14, ymm10, ymm5 ;; new evenR3 = R6+.309*(R2+R10)	; cos2*i25a + i1		;	7-11

	yfnmaddpd ymm3, ymm13, ymm7, ymm3 ;; new evenI3 -= .588*(R4-R8)		; t2=sin2*i25s - sin4*NEGi34s	;	11-15
	yfmaddpd ymm1, ymm12, ymm7, ymm1 ;; new evenI2 += .951*(R4-R8)		; t4=sin4*i25s + sin2*NEGi34s	; 11-15

	yfnmaddpd ymm7, ymm15, ymm10, ymm5 ;; new evenR5 = R6-.809*(R2+R10)	; cos4*i25a + i1		; 7-11
	yfmaddpd ymm5, ymm5, ymm11, ymm10 ;; new evenR1 = R6+R2+R10		; i1 + i25a			;	8-12

	vmulpd	ymm10, ymm12, ymm4	;; new oddI2 = .951*(R3-R9)		; sin2*r25s			;	8-12
	vmulpd	ymm4, ymm13, ymm4	;; new oddI3 = .588*(R3-R9)		; sin4*r25s			; 9-13

	yfnmaddpd ymm9, ymm15, ymm2, ymm9 ;; new evenR3 += -.809*(R4+R8)	; t5=cos2*i25a + cos4*i34a + i1	;	12-16
	yfmaddpd ymm7, ymm14, ymm2, ymm7 ;; new evenR5 += .309*(R4+R8)		; t7=cos4*i25a + cos2*i34a + i1	; 12-16
	yfmaddpd ymm5, ymm5, ymm11, ymm2 ;; new evenR1 = R6+R2+R4+R8+R10	; outi(0)=i1+i25a+i34a		;	13-17
	vmovapd	ymm2, memr1		;; R1					; r1
	ystore	dst2, ymm5		;; Save evenR1				; Save I1			;		18

	yfmaddpd ymm5, ymm14, ymm0, ymm2 ;; new oddR2 = R1+.309*(R3+R9)		; cos2*r25a + r1		;	9-13

	yfmaddpd ymm10, ymm13, ymm6, ymm10 ;; new oddI2 += .588*(R5-R7)		; t6=sin2*r25s + sin4*r34s	;	13-17
	yfnmaddpd ymm4, ymm12, ymm6, ymm4 ;; new oddI3 -= .951*(R5-R7)		; t8=sin4*r25s - sin2*r34s	; 14-18

	yfnmaddpd ymm6, ymm15, ymm0, ymm2 ;; new oddR3 = R1-.809*(R3+R9)	; cos4*r25a + r1		; 10-14
	yfmaddpd ymm2, ymm2, ymm11, ymm0 ;; new oddR1 = R1+R3+R9		; r1 + r25a			;	10-14

	yfnmaddpd ymm5, ymm15, ymm8, ymm5 ;; new oddR2 += -.809*(R5+R7)		; t1=cos2*r25a + cos4*r34a + r1	;	14-18
	yfmaddpd ymm6, ymm14, ymm8, ymm6 ;; new oddR3 += .309*(R5+R7)		; t3=cos4*r25a + cos2*r34a + r1	; 15-19
	yfmaddpd ymm2, ymm2, ymm11, ymm8 ;; new oddR1 = R1+R3+R5+R7+R9		; outr(0) = r1+r25a+r34a	;	15-19

	vblendpd ymm8, ymm9, ymm1, 1	;; evenI2				; t5				;		17
	vblendpd ymm0, ymm3, ymm7, 1	;; evenR5				; t2				;		18
	vblendpd ymm1, ymm1, ymm9, 1	;; evenR3				; t4				;		19
	vblendpd ymm7, ymm7, ymm3, 1	;; evenI3				; t7				;		20

	yfmsubpd ymm3, ymm8, ymm11, ymm10 ;; New I5 = evenI2 - oddI2		; outi(4)=t5-t6			;	18-22
	yfmaddpd ymm8, ymm8, ymm11, ymm10 ;; New I2 = evenI2 + oddI2		; outi(1)=t5+t6			; 18-22
	vmovapd	ymm9, [screg+192+32]			;; cosine/sine for w^4

	yfmaddpd ymm10, ymm5, ymm11, ymm0 ;; New R5 = oddR2 + evenR5		; outr(4)=t1+t2			;	19-23
	yfmsubpd ymm5, ymm5, ymm11, ymm0 ;; New R2 = oddR2 - evenR5		; outr(1)=t1-t2			; 19-23

	yfmsubpd ymm0, ymm6, ymm11, ymm1 ;; New R4 = oddR3 - evenR3		; outr(2)=t3-t4			;	20-24
	yfmaddpd ymm6, ymm6, ymm11, ymm1 ;; New R3 = oddR3 + evenR3		; outr(3)=t3+t4			; 20-24
	ystore	dst1, ymm2		;; Save oddR1				; Save R1			;		20

	yfmsubpd ymm1, ymm7, ymm11, ymm4 ;; New I4 = evenI3 - oddI3		; outi(3)=t7-t8			;	21-25
	yfmaddpd ymm7, ymm7, ymm11, ymm4 ;; New I3 = evenI3 + oddI3		; outi(2)=t7+t8			; 21-25
	vmovapd	ymm2, [screg+32]			;; cosine/sine for w^1

	vblendpd ymm4, ymm6, ymm0, 1			;; R4							;		25
	vblendpd ymm0, ymm0, ymm6, 1			;; R3							;		26

	yfmsubpd ymm6, ymm10, ymm9, ymm3		;; A5 = R5 * cosine/sine - I5				;	24-28
	yfmaddpd ymm3, ymm3, ymm9, ymm10		;; B5 = I5 * cosine/sine + R5				; 24-28
	vmovapd	ymm10, [screg+128+32]			;; cosine/sine for w^3

	yfmsubpd ymm9, ymm5, ymm2, ymm8			;; A2 = R2 * cosine/sine - I2				;	25-29
	yfmaddpd ymm8, ymm8, ymm2, ymm5			;; B2 = I2 * cosine/sine + R2				; 25-29
	vmovapd	ymm2, [screg+64+32]			;; cosine/sine for w^2

	yfmsubpd ymm5, ymm4, ymm10, ymm1		;; A4 = R4 * cosine/sine - I4				;	26-30
	yfmaddpd ymm1, ymm1, ymm10, ymm4		;; B4 = I4 * cosine/sine + R4				; 26-30
	vmovapd	ymm10, [screg+192]

	yfmsubpd ymm4, ymm0, ymm2, ymm7			;; A3 = R3 * cosine/sine - I3				;	27-31
	yfmaddpd ymm7, ymm7, ymm2, ymm0			;; B3 = I3 * cosine/sine + R3				; 27-31
	vmovapd	ymm2, [screg]

	vmulpd	ymm6, ymm6, ymm10			;; A5 = A5 * sine (final R5)				;	29-33
	vmulpd	ymm3, ymm3, ymm10			;; B5 = B5 * sine (final I5)				; 29-33
	vmovapd	ymm0, [screg+128]

	vmulpd	ymm9, ymm9, ymm2			;; A2 = A2 * sine (final R2)				;	30-34
	vmulpd	ymm8, ymm8, ymm2			;; B2 = B2 * sine (final I2)				; 30-34
	vmovapd	ymm10, [screg+64]

	vmulpd	ymm5, ymm5, ymm0			;; A4 = A4 * sine (final R4)				;	31-35
	vmulpd	ymm1, ymm1, ymm0			;; B4 = B4 * sine (final I4)				; 31-35

	vmulpd	ymm4, ymm4, ymm10			;; A3 = A3 * sine (final R3)				;	32-36
	vmulpd	ymm7, ymm7, ymm10			;; B3 = B3 * sine (final I3)				; 32-36

	ystore	dst3, ymm9				;; Save final R2					;		35
	ystore	dst4, ymm8				;; Save final I2					;		35+1

	ENDM

ENDIF

ENDIF

;;
;;
;; ************************************* ten-reals-five-complex-unfft variants ******************************************
;;

;; Macro to do a ten_reals_unfft and three five_complex_djbunfft in pass 2.
;; The ten-reals operation is done in the lower double of the YMM
;; register.  The five complex operation is done in the upper values
;; of a YMM register.

yr5_5cl_ten_reals_five_complex_djbunfft_preload MACRO
	yr5_o10r_t5c_djbunfft_mem_preload
	ENDM
yr5_5cl_ten_reals_five_complex_djbunfft MACRO srcreg,srcinc,d1,screg,scinc,maxrpt
	d2 = 2*d1
	d3 = 3*d1
	d4 = 4*d1
	yr5_o10r_t5c_djbunfft_mem [srcreg],[srcreg+d1],[srcreg+d2],[srcreg+d3],[srcreg+d4],screg,[srcreg],[srcreg+d2],[srcreg+d4],[srcreg+32]
;	ystore	[srcreg], ymm4		;; Save R1
	ystore	[srcreg+d1], ymm2	;; Save R2
;	ystore	[srcreg+d2], ymm3	;; Save R3
	ystore	[srcreg+d3], ymm7	;; Save R4
;	ystore	[srcreg+d4], ymm0	;; Save R5
;	ystore	[srcreg+32], ymm2	;; Save I1
	ystore	[srcreg+d1+32], ymm0	;; Save I2
	ystore	[srcreg+d2+32], ymm1	;; Save I3
	ystore	[srcreg+d3+32], ymm3	;; Save I4
	ystore	[srcreg+d4+32], ymm6	;; Save I5
	bump	srcreg, srcinc
	bump	screg, scinc
	ENDM

;; To calculate a 10-reals unFFT (in a shorthand notation):
;; c1 + c2 + c3 + c4 + c5 + c6 + c7 + c8 + c9 + c10	*  w^0000000000
;; c1 + c2 + c3 + c4 + c5 + c6 + c7 + c8 + c9 + c10	*  w^0123456789
;; c1 + c2 + c3 + c4 + c5 + c6 + c7 + c8 + c9 + c10	*  w^0246802468
;; c1 + c2 + c3 + c4 + c5 + c6 + c7 + c8 + c9 + c10	*  w^0369258147
;; c1 + c2 + c3 + c4 + c5 + c6 + c7 + c8 + c9 + c10	*  w^0482604826
;; c1 + c2 + c3 + c4 + c5 + c6 + c7 + c8 + c9 + c10	*  w^0505050505
;; c1 + c2 + c3 + c4 + c5 + c6 + c7 + c8 + c9 + c10	*  w^0628406284
;; c1 + c2 + c3 + c4 + c5 + c6 + c7 + c8 + c9 + c10	*  w^0741852963
;; c1 + c2 + c3 + c4 + c5 + c6 + c7 + c8 + c9 + c10	*  w^0864208642
;; c1 + c2 + c3 + c4 + c5 + c6 + c7 + c8 + c9 + c10	*  w^0987654321
;; Noting that w^5 = -1
;; c1 + c2 + c3 + c4 + c5 + c6 + c7 + c8 + c9 + c10	*  w^0000000000
;; c1 + c2 + c3 + c4 + c5 - c6 - c7 - c8 - c9 - c10	*  w^0123401234
;; c1 + c2 + c3 - c4 - c5 + c6 + c7 + c8 - c9 - c10	*  w^0241302413
;; c1 + c2 - c3 - c4 + c5 - c6 - c7 + c8 + c9 - c10	*  w^0314203142
;; c1 + c2 - c3 + c4 - c5 + c6 + c7 - c8 + c9 - c10	*  w^0432104321
;; c1 - c2 + c3 - c4 + c5 - c6 + c7 - c8 + c9 - c10	*  w^0000000000
;; c1 - c2 + c3 - c4 + c5 + c6 - c7 + c8 - c9 + c10	*  w^0123401234
;; c1 - c2 + c3 + c4 - c5 - c6 + c7 - c8 - c9 + c10	*  w^0241302413
;; c1 - c2 - c3 + c4 + c5 + c6 - c7 - c8 + c9 + c10	*  w^0314203142
;; c1 - c2 - c3 - c4 - c5 - c6 + c7 + c8 + c9 + c10	*  w^0432104321
;; incoming is:	r1_1 = c1r + c6r
;;		c2 = c2r + c2i
;;		c3 = c3r + c3i
;;		c4 = c4r + c4i
;;		c5 = c5r + c5i
;;		r1_2 = c1r - c6r
;;		c7 = c5r - c5i	(implied)
;;		c8 = c4r - c4i	(implied)
;;		c9 = c3r - c3i	(implied)
;;		c10 = c2r - c2i	(implied)
;; And noticing the signs of the real and imaginary parts of the sin/cos values:
;; w^1/10 = .809 - .588i
;; w^2/10 = .309 - .951i
;; w^3/10 = -.309 - .951i
;; w^4/10 = -.809 - .588i
;; We get reals:
;; r1_1 + 2c2 + 2c3 + 2c4 + 2c5	*  w^00000
;; r1_2 + 2c2 + 2c3 + 2c4 + 2c5	*  w^01234
;; r1_1 + 2c2 + 2c3 - 2c4 - 2c5	*  w^02413
;; r1_2 + 2c2 - 2c3 - 2c4 + 2c5	*  w^03142
;; r1_1 + 2c2 - 2c3 + 2c4 - 2c5	*  w^04321
;; r1_2 - 2c2 + 2c3 - 2c4 + 2c5	*  w^00000
;; r1_1 - 2c2 + 2c3 - 2c4 + 2c5	*  w^01234
;; r1_2 - 2c2 + 2c3 + 2c4 - 2c5	*  w^02413
;; r1_1 - 2c2 - 2c3 + 2c4 + 2c5	*  w^03142
;; r1_2 - 2c2 - 2c3 - 2c4 - 2c5	*  w^04321
;; Now drop the multiplication by 2 (the actual r1_1 and r1_2 inputs are already doubled)
;; and expand the sin/cos multipliers:
;; r1_1 + c2r + c3r + c4r + c5r
;; r1_2 + .809c2r + .588c2i + .309c3r + .951c3i - .309c4r + .951c4i - .809c5r + .588c5i
;; r1_1 + .309c2r + .951c2i - .809c3r + .588c3i - .809c4r - .588c4i + .309c5r - .951c5i
;; r1_2 - .309c2r + .951c2i - .809c3r - .588c3i + .809c4r - .588c4i + .309c5r + .951c5i
;; r1_1 - .809c2r + .588c2i + .309c3r - .951c3i + .309c4r + .951c4i - .809c5r - .588c5i
;; r1_2 - c2r + c3r - c4r + c5r
;; r1_1 - .809c2r - .588c2i + .309c3r + .951c3i + .309c4r - .951c4i - .809c5r + .588c5i
;; r1_2 - .309c2r - .951c2i - .809c3r + .588c3i + .809c4r + .588c4i + .309c5r - .951c5i
;; r1_1 + .309c2r - .951c2i - .809c3r - .588c3i - .809c4r + .588c4i + .309c5r + .951c5i
;; r1_2 + .809c2r - .588c2i + .309c3r - .951c3i - .309c4r - .951c4i - .809c5r - .588c5i
;; Simplify:
;; r1_1 + c2r + c3r + c4r + c5r
;; r1_2 + .809(c2r-c5r) + .588(c2i+c5i) + .309(c3r-c4r) + .951(c3i+c4i)
;; r1_1 + .309(c2r+c5r) + .951(c2i-c5i) - .809(c3r+c4r) + .588(c3i-c4i)
;; r1_2 - .309(c2r-c5r) + .951(c2i+c5i) - .809(c3r-c4r) - .588(c3i+c4i)
;; r1_1 - .809(c2r+c5r) + .588(c2i-c5i) + .309(c3r+c4r) - .951(c3i-c4i)
;; r1_2 - c2r + c3r - c4r + c5r
;; r1_1 - .809(c2r+c5r) - .588(c2i-c5i) + .309(c3r+c4r) + .951(c3i-c4i)
;; r1_2 - .309(c2r-c5r) - .951(c2i+c5i) - .809(c3r-c4r) + .588(c3i+c4i)
;; r1_1 + .309(c2r+c5r) - .951(c2i-c5i) - .809(c3r+c4r) - .588(c3i-c4i)
;; r1_2 + .809(c2r-c5r) - .588(c2i+c5i) + .309(c3r-c4r) - .951(c3i+c4i)

yr5_o10r_t5c_djbunfft_mem_preload MACRO
	ENDM

yr5_o10r_t5c_djbunfft_mem MACRO memr1,memr2,memr3,memr4,memr5,screg,dstr1,dstr3,dstr5,dsti1
					;; Ten reals comments		;; Five complex comments
	vmovapd	ymm2, memr2				;; R2
	vmovapd	ymm4, [screg+32]			;; cosine/sine
	vmulpd	ymm1, ymm2, ymm4			;; A2 = R2 * cosine/sine
	vmovapd	ymm5, memr5				;; R5
	vmovapd	ymm6, [screg+192+32]			;; cosine/sine
	vmulpd	ymm0, ymm5, ymm6			;; A5 = R5 * cosine/sine
	vmovapd	ymm3, memr2[32]				;; I2
	vaddpd	ymm1, ymm1, ymm3			;; A2 = A2 + I2
	vmovapd	ymm7, memr5[32]				;; I5
	vaddpd	ymm0, ymm0, ymm7			;; A5 = A5 + I5
	vmulpd	ymm3, ymm3, ymm4			;; B2 = I2 * cosine/sine
	vmulpd	ymm7, ymm7, ymm6			;; B5 = I5 * cosine/sine
	vsubpd	ymm3, ymm3, ymm2			;; B2 = B2 - R2
	vsubpd	ymm7, ymm7, ymm5			;; B5 = B5 - R5
	vmovapd	ymm4, [screg]				;; sine
	vmulpd	ymm1, ymm1, ymm4			;; A2 = A2 * sine (new R2)
	vmovapd	ymm6, [screg+192]			;; sine
	vmulpd	ymm0, ymm0, ymm6			;; A5 = A5 * sine (new R5)
	vmulpd	ymm3, ymm3, ymm4			;; B2 = B2 * sine (new I2)
	vmulpd	ymm7, ymm7, ymm6			;; B5 = B5 * sine (new I5)
	vaddpd	ymm2, ymm0, ymm1	;; R2+R5			; r25a=r2+r5
	vsubpd	ymm0, ymm0, ymm1	;; NEG(R2-R5) = R5-R2		; NEGr25s=r5-r2
	vaddpd	ymm1, ymm3, ymm7	;; I2+I5			; i25a=i2+i5
	vsubpd	ymm3, ymm3, ymm7	;; I2-I5			; i25s=i2-i5
	vblendpd ymm4, ymm1, ymm0, 1	;; NEG(R2-R5)			; i25a
	vblendpd ymm0, ymm0, ymm1, 1	;; I2+I5			; NEGr25s
	ystore	YMM_TMP1, ymm2		;; Temp save R2+R5		; Temp save r25a
	ystore	YMM_TMP2, ymm3		;; Temp save I2-I5		; Temp save i25s
	ystore	YMM_TMP3, ymm4		;; Temp save NEG(R2-R5)		; Temp save i25a
	ystore	YMM_TMP4, ymm0		;; Temp save I2+I5		; Temp save NEGr25s

	vmovapd	ymm2, memr4				;; R4
	vmovapd	ymm4, [screg+128+32]			;; cosine/sine
	vmulpd	ymm1, ymm2, ymm4			;; A4 = R4 * cosine/sine
	vmovapd	ymm5, memr3				;; R3
	vmovapd	ymm6, [screg+64+32]			;; cosine/sine
	vmulpd	ymm0, ymm5, ymm6			;; A3 = R3 * cosine/sine
	vmovapd	ymm3, memr4[32]				;; I4
	vaddpd	ymm1, ymm1, ymm3			;; A4 = A4 + I4
	vmovapd	ymm7, memr3[32]				;; I3
	vaddpd	ymm0, ymm0, ymm7			;; A3 = A3 + I3
	vmulpd	ymm3, ymm3, ymm4			;; B4 = I4 * cosine/sine
	vmulpd	ymm7, ymm7, ymm6			;; B3 = I3 * cosine/sine
	vsubpd	ymm3, ymm3, ymm2			;; B4 = B4 - R4
	vsubpd	ymm7, ymm7, ymm5			;; B3 = B3 - R3
	vmovapd	ymm4, [screg+128]			;; sine
	vmulpd	ymm1, ymm1, ymm4			;; A4 = A4 * sine (new R4)
	vmovapd	ymm6, [screg+64]			;; sine
	vmulpd	ymm0, ymm0, ymm6			;; A3 = A3 * sine (new R3)
	vmulpd	ymm3, ymm3, ymm4			;; B4 = B4 * sine (new I4)
	vmulpd	ymm7, ymm7, ymm6			;; B3 = B3 * sine (new I3)
	vaddpd	ymm2, ymm0, ymm1	;; R3+R4			; r34a=r3+r4
	vsubpd	ymm0, ymm0, ymm1	;; R3-R4			; r34s=r3-r4
	vaddpd	ymm1, ymm7, ymm3	;; I3+I4			; i34a=i3+i4
	vsubpd	ymm7, ymm7, ymm3	;; I3-I4			; i34s=i3-i4
	vblendpd ymm3, ymm1, ymm0, 1	;; R3-R4			; i34a
	vblendpd ymm0, ymm0, ymm1, 1	;; I3+I4			; r34s
	ystore	YMM_TMP5, ymm3		;; Temp save R3-R4		; Temp save i34a
	ystore	YMM_TMP6, ymm0		;; Temp save I3+I4		; Temp save r34s

	vmovapd	ymm1, YMM_TMP1		;; Reload R2+R5			; Reload save r25a
	vmovapd	ymm4, YMM_P309
	vmulpd	ymm5, ymm4, ymm1	;; new R3 = .309*(R2+R5)	; cos2*r25a
	vmovapd	ymm6, memr1		;; R1_1				; r1
	vaddpd	ymm5, ymm5, ymm6	;; new R3 += R1_1		; cos2*r25a + r1
	vmovapd	ymm0, YMM_P809
	vmulpd	ymm3, ymm0, ymm1	;; new -R5 = -.809*(R2+R5)	; cos4*r25a
	vaddpd	ymm1, ymm6, ymm1	;; new R1 = R1_1+R2+R5		; r1 + r25a
	vsubpd	ymm3, ymm6, ymm3	;; new R5 += R1_1		; cos4*r25a + r1
	vmulpd	ymm0, ymm0, ymm2	;; --.809*(R3+R4)		; cos4*r34a
	vaddpd	ymm1, ymm1, ymm2	;; final R1 = R1_1+R2+R5+R3+R4	; outr(0) = r1 + r25a + r34a
	vmulpd	ymm2, ymm4, ymm2	;; .309*(R3+R4)			; cos2*r34a
	vsubpd	ymm5, ymm5, ymm0	;; new R3 += -.809*(R3+R4)	; t1=cos2*r25a + cos4*r34a + r1
	vaddpd	ymm3, ymm3, ymm2	;; new R5 += .309*(R3+R4)	; t3=cos4*r25a + cos2*r34a + r1

	vmovapd	ymm0, YMM_TMP2		;; Reload I2-I5			; Reload i25s
	vmovapd	ymm6, YMM_P951
	vmulpd	ymm2, ymm6, ymm0	;; tmp1 = .951*(I2-I5)		; sin2*i25s
	vmovapd	ymm4, YMM_P588
	vmulpd	ymm0, ymm4, ymm0	;; tmp2 = .588*(I2-I5)		; sin4*i25s
	vmulpd	ymm4, ymm4, ymm7	;; .588*(I3-I4)			; sin4*i34s
	vmulpd	ymm7, ymm6, ymm7	;; .951*(I3-I4)			; sin2*i34s
	vaddpd	ymm2, ymm2, ymm4	;; tmp1 += .588*(I3-I4)		; t2=sin2*i25s + sin4*i34s
	vsubpd	ymm0, ymm0, ymm7	;; tmp2 -= .951*(I3-I4)		; t4=sin4*i25s - sin2*i34s

	vsubpd	ymm4, ymm5, ymm2	;; final R9 = new R3 - tmp1	; outr(4)=t1-t2
	vaddpd	ymm5, ymm5, ymm2	;; final R3 = new R3 + tmp1	; outr(1)=t1+t2
	vsubpd	ymm2, ymm3, ymm0	;; final R7 = new R5 - tmp2	; outr(3)=t3-t4
	vaddpd	ymm3, ymm3, ymm0	;; final R5 = new R5 + tmp2	; outr(2)=t3+t4
	ystore	dstr1, ymm1				;; Save final R1
	vblendpd ymm1, ymm3, ymm5, 1			;; final R3
	ystore	dstr3, ymm1				;; Save final R3
	vblendpd ymm1, ymm4, ymm3, 1			;; final R5
	ystore	dstr5, ymm1				;; Save final R5
	vblendpd ymm5, ymm5, ymm4, 1	;; final R9 (I4)		; final R2

	vmovapd	ymm3, YMM_TMP5		;; Reload R3-R4			; Reload i34a
	vmovapd	ymm0, YMM_P309
	vmulpd	ymm1, ymm0, ymm3	;; new R2 = .309*(R3-R4)	; cos2*i34a
	vmovapd	ymm7, memr1[32]		;; R1_2				; i1
	vaddpd	ymm1, ymm1, ymm7	;; new R2 += R1_2		; cos2*i34a + i1
	vmovapd	ymm4, YMM_P809
	vmulpd	ymm6, ymm4, ymm3	;; new -R4 = -.809*(R3-R4)	; cos4*i34a
	vaddpd	ymm3, ymm7, ymm3	;; new R6 = R1_2+(R3-R4)	; i1 + i34a
	vsubpd	ymm6, ymm7, ymm6	;; new R4 += R1_2		; cos4*i34a + i1
	vmovapd	ymm7, YMM_TMP3		;; Reload NEG(R2-R5)		; Reload i25a
	vmulpd	ymm4, ymm4, ymm7	;; --.809*NEG(R2-R5)		; cos4*i25a
	vaddpd	ymm3, ymm3, ymm7	;; final R6 = R6+(R3-R4)+NEG(R2-R5) ; outi(0)=i1 + i34a + i25a
	vmulpd	ymm0, ymm0, ymm7	;; .309*NEG(R2-R5)		; cos2*i25a
	vsubpd	ymm1, ymm1, ymm4	;; new R2 += -.809*NEG(R2-R5)	; t7=cos2*i34a + i1 + cos4*i25a
	vaddpd	ymm6, ymm6, ymm0	;; new R4 += .309*NEG(R2-R5)	; t5=cos4*i34a + i1 + cos2*i25a

	vmovapd	ymm0, YMM_TMP6		;; Reload I3+I4			; Reload r34s
	vmovapd	ymm4, YMM_P588
	vmulpd	ymm7, ymm4, ymm0	;; tmp2 = .588*(I3+I4)		; sin4*r34s
	vmulpd	ymm0, ymm0, YMM_P951	;; tmp1 = .951*(I3+I4)		; sin2*r34s
	ystore	dsti1, ymm3				;; Save final I1
	vmovapd	ymm3, YMM_TMP4		;; Reload I2+I5			; Reload NEGr25s
	vmulpd	ymm4, ymm4, ymm3	;; .588*(I2+I5)			; sin4*NEGr25s
	vmulpd	ymm3, ymm3, YMM_P951	;; .951*(I2+I5)			; sin2*NEGr25s
	vaddpd	ymm0, ymm0, ymm4	;; tmp1 += .588*(I2+I5)		; t8=sin2*r34s + sin4*NEGr25s
	vsubpd	ymm7, ymm7, ymm3	;; tmp2 -= .951*(I2+I5)		; t6=sin4*r34s - sin2*NEGr25s

	vsubpd	ymm3, ymm1, ymm0	;; Final R10 = new R2 - tmp1	; outi(3)=t7-t8
	vaddpd	ymm1, ymm1, ymm0	;; Final R2 = new R2 + tmp1	; outi(2)=t7+t8
	vsubpd	ymm0, ymm6, ymm7	;; Final R4 = new R4 - tmp2	; outi(1)=t5-t6
	vaddpd	ymm6, ymm6, ymm7	;; Final R8 = new R4 + tmp2	; outi(4)=t5+t6

	vblendpd ymm7, ymm2, ymm0, 1	;; final R4			; final R4
	vblendpd ymm0, ymm0, ymm2, 1	;; final R7 (I2)		; final I2
	vblendpd ymm2, ymm5, ymm1, 1	;; final R2			; final R2
	vblendpd ymm1, ymm1, ymm6, 1	;; final R8 (I3)		; final I3
	vblendpd ymm6, ymm6, ymm3, 1	;; final R10 (I5)		; final I5
	vblendpd ymm3, ymm3, ymm5, 1	;; final R9 (I4)		; final I4
	ENDM

;; 64-bit version

IFDEF X86_64

yr5_o10r_t5c_djbunfft_mem_preload MACRO
	vmovapd	ymm15, YMM_P809
	vmovapd	ymm14, YMM_P309
	vmovapd	ymm13, YMM_P588
	vmovapd	ymm12, YMM_P951
	ENDM

yr5_o10r_t5c_djbunfft_mem MACRO memr1,memr2,memr3,memr4,memr5,screg,dstr1,dstr3,dstr5,dsti1
					;; Ten reals comments		;; Five complex comments
	vmovapd	ymm2, memr2				;; R2
	vmovapd	ymm4, [screg+32]			;; cosine/sine
	vmulpd	ymm1, ymm2, ymm4			;; A2 = R2 * cosine/sine				;	1-5

	vmovapd	ymm5, memr5				;; R5
	vmovapd	ymm6, [screg+192+32]			;; cosine/sine
	vmulpd	ymm0, ymm5, ymm6			;; A5 = R5 * cosine/sine				;	2-6

	vmovapd	ymm3, memr2[32]				;; I2
	vmulpd	ymm4, ymm3, ymm4			;; B2 = I2 * cosine/sine				;	3-7

	vmovapd	ymm7, memr5[32]				;; I5
	vmulpd	ymm6, ymm7, ymm6			;; B5 = I5 * cosine/sine				;	4-8

	vmovapd	ymm8, memr4[32]				;; I4
	vmovapd	ymm9, [screg+128+32]			;; cosine/sine
	vmulpd	ymm10, ymm8, ymm9			;; B4 = I4 * cosine/sine				;	5-9

	vaddpd	ymm1, ymm1, ymm3			;; A2 = A2 + I2						; 6-8

	vmovapd	ymm3, memr3[32]				;; I3
	vmovapd	ymm11, [screg+64+32]			;; cosine/sine

	vaddpd	ymm0, ymm0, ymm7			;; A5 = A5 + I5						; 7-9

	vmulpd	ymm7, ymm3, ymm11			;; B3 = I3 * cosine/sine				;	6-10

	vsubpd	ymm4, ymm4, ymm2			;; B2 = B2 - R2						; 8-10

	vmovapd	ymm2, memr4				;; R4
	vmulpd	ymm9, ymm2, ymm9			;; A4 = R4 * cosine/sine				;	7-11

	vsubpd	ymm6, ymm6, ymm5			;; B5 = B5 - R5						; 9-11

	vmovapd	ymm5, memr3				;; R3
	vmulpd	ymm11, ymm5, ymm11			;; A3 = R3 * cosine/sine				;	8-12

	vsubpd	ymm10, ymm10, ymm2			;; B4 = B4 - R4						; 10-12

	vmovapd	ymm2, [screg]				;; sine
	vmulpd	ymm1, ymm1, ymm2			;; A2 = A2 * sine (new R2)				;	9-13

	vsubpd	ymm7, ymm7, ymm5			;; B3 = B3 - R3						; 11-13

	vmovapd	ymm5, [screg+192]			;; sine
	vmulpd	ymm0, ymm0, ymm5			;; A5 = A5 * sine (new R5)				;	10-14

	vmulpd	ymm4, ymm4, ymm2			;; B2 = B2 * sine (new I2)				;	11-15
	vmovapd	ymm2, [screg+128]			;; sine

	vaddpd	ymm9, ymm9, ymm8			;; A4 = A4 + I4						; 12-14
	vmovapd	ymm8, [screg+64]			;; sine

	vmulpd	ymm6, ymm6, ymm5			;; B5 = B5 * sine (new I5)				;	12-16

	vaddpd	ymm11, ymm11, ymm3			;; A3 = A3 + I3						; 13-15

	vmulpd	ymm10, ymm10, ymm2			;; B4 = B4 * sine (new I4)				;	13-17

	vmulpd	ymm7, ymm7, ymm8			;; B3 = B3 * sine (new I3)				;	14-18

	vmulpd	ymm9, ymm9, ymm2			;; A4 = A4 * sine (new R4)				;	15-19

	vaddpd	ymm2, ymm1, ymm0	;; R2+R5			; r25a=r2+r5				; 15-17
	vsubpd	ymm0, ymm0, ymm1	;; NEG(R2-R5) = R5-R2		; NEGr25s=r5-r2				; 16-18

	vmulpd	ymm11, ymm11, ymm8			;; A3 = A3 * sine (new R3)				;	16-20

	vsubpd	ymm8, ymm4, ymm6	;; I2-I5			; i25s=i2-i5				; 17-19
	vaddpd	ymm4, ymm4, ymm6	;; I2+I5			; i25a=i2+i5				; 18-20

	vmulpd	ymm6, ymm14, ymm2	;; new R3 = .309*(R2+R5)	; cos2*r25a				;	18-22
	vmulpd	ymm1, ymm15, ymm2	;; new -R5 = -.809*(R2+R5)	; cos4*r25a				;	19-23

	vsubpd	ymm3, ymm7, ymm10	;; I3-I4			; i34s=i3-i4				; 19-21
	vaddpd	ymm7, ymm7, ymm10	;; I3+I4			; i34a=i3+i4				; 20-22

	vmulpd	ymm10, ymm12, ymm8	;; tmp1 = .951*(I2-I5)		; sin2*i25s				;	20-24
	vmulpd	ymm8, ymm13, ymm8	;; tmp2 = .588*(I2-I5)		; sin4*i25s				;	21-25

	vaddpd	ymm5, ymm11, ymm9	;; R3+R4			; r34a=r3+r4				; 21-23
	vsubpd	ymm11, ymm11, ymm9	;; R3-R4			; r34s=r3-r4				; 22-24

	vmovapd	ymm9, memr1		;; R1_1				; r1
	vaddpd	ymm2, ymm9, ymm2	;; new R1 = R1_1+R2+R5		; r1 + r25a				; 23-25
	vaddpd	ymm6, ymm6, ymm9	;; new R3 += R1_1		; cos2*r25a + r1			; 24-26
	vsubpd	ymm1, ymm9, ymm1	;; new R5 += R1_1		; cos4*r25a + r1			; 25-27

	vmulpd	ymm9, ymm13, ymm3	;; .588*(I3-I4)			; sin4*i34s				;	22-26
	vmulpd	ymm3, ymm12, ymm3	;; .951*(I3-I4)			; sin2*i34s				;	23-27

	vaddpd	ymm2, ymm2, ymm5	;; final R1 = R1_1+R2+R5+R3+R4	; outr(0) = r1 + r25a + r34a		; 26-28

	vaddpd	ymm10, ymm10, ymm9	;; tmp1 += .588*(I3-I4)		; t2=sin2*i25s + sin4*i34s		; 27-29

	vmulpd	ymm9, ymm15, ymm5	;; --.809*(R3+R4)		; cos4*r34a				;	24-28
	vmulpd	ymm5, ymm14, ymm5	;; .309*(R3+R4)			; cos2*r34a				;	25-29

	vsubpd	ymm8, ymm8, ymm3	;; tmp2 -= .951*(I3-I4)		; t4=sin4*i25s - sin2*i34s		; 28-30

	vblendpd ymm3, ymm0, ymm4, 1	;; I2+I5			; NEGr25s				;		21
	vblendpd ymm4, ymm4, ymm0, 1	;; NEG(R2-R5)			; i25a					;		22

	vblendpd ymm0, ymm11, ymm7, 1	;; I3+I4			; r34s					;		25
	vblendpd ymm7, ymm7, ymm11, 1	;; R3-R4			; i34a					;		26

	vsubpd	ymm6, ymm6, ymm9	;; new R3 += -.809*(R3+R4)	; t1=cos2*r25a + cos4*r34a + r1		; 29-31
	vaddpd	ymm1, ymm1, ymm5	;; new R5 += .309*(R3+R4)	; t3=cos4*r25a + cos2*r34a + r1		; 30-32

	vmulpd	ymm5, ymm12, ymm0	;; tmp1 = .951*(I3+I4)		; sin2*r34s				;	29-33
	vmulpd	ymm0, ymm13, ymm0	;; tmp2 = .588*(I3+I4)		; sin4*r34s				;	30-34

	ystore	dstr1, ymm2				;; Save final R1					;		29

	vmulpd	ymm2, ymm13, ymm3	;; .588*(I2+I5)			; sin4*NEGr25s				;	31-41
	vmulpd	ymm3, ymm12, ymm3	;; .951*(I2+I5)			; sin2*NEGr25s				;	32-36

	vsubpd	ymm9, ymm6, ymm10	;; final R9 = new R3 - tmp1	; outr(4)=t1-t2				; 32-34
	vaddpd	ymm6, ymm6, ymm10	;; final R3 = new R3 + tmp1	; outr(1)=t1+t2				; 33-35

	vmulpd	ymm10, ymm14, ymm7	;; new R2 = .309*(R3-R4)	; cos2*i34a				;	33-37

	vaddpd	ymm11, ymm1, ymm8	;; final R5 = new R5 + tmp2	; outr(2)=t3+t4				; 34-36
	vsubpd	ymm1, ymm1, ymm8	;; final R7 = new R5 - tmp2	; outr(3)=t3-t4				; 35-37

	vmulpd	ymm8, ymm15, ymm7	;; new -R4 = -.809*(R3-R4)	; cos4*i34a				;	34-38

	vaddpd	ymm5, ymm5, ymm2	;; tmp1 += .588*(I2+I5)		; t8=sin2*r34s + sin4*NEGr25s		; 36-40

	vmulpd	ymm2, ymm15, ymm4	;; --.809*NEG(R2-R5)		; cos4*i25a				;	36-40

	vsubpd	ymm0, ymm0, ymm3	;; tmp2 -= .951*(I2+I5)		; t6=sin4*r34s - sin2*NEGr25s		; 37-41

	vblendpd ymm3, ymm11, ymm6, 1			;; final R3						;		37
	ystore	dstr3, ymm3				;; Save final R3					;		38

	vmulpd	ymm3, ymm14, ymm4	;; .309*NEG(R2-R5)		; cos2*i25a				;	37-41

	vblendpd ymm11, ymm9, ymm11, 1			;; final R5						;		38
	ystore	dstr5, ymm11				;; Save final R5					;		39

	vmovapd	ymm11, memr1[32]	;; R1_2				; i1
	vaddpd	ymm10, ymm10, ymm11	;; new R2 += R1_2		; cos2*i34a + i1			; 38-40
	vsubpd	ymm8, ymm11, ymm8	;; new R4 += R1_2		; cos4*i34a + i1			; 39-41
	vaddpd	ymm11, ymm11, ymm7	;; new R6 = R1_2+(R3-R4)	; i1 + i34a				; 40-42

	vsubpd	ymm10, ymm10, ymm2	;; new R2 += -.809*NEG(R2-R5)	; t7=cos2*i34a + i1 + cos4*i25a		; 41-43
	vaddpd	ymm8, ymm8, ymm3	;; new R4 += .309*NEG(R2-R5)	; t5=cos4*i34a + i1 + cos2*i25a		; 42-44
	vaddpd	ymm11, ymm11, ymm4	;; final R6 = R6+(R3-R4)+NEG(R2-R5) ; outi(0)=i1 + i34a + i25a		; 43-45

	vsubpd	ymm4, ymm10, ymm5	;; Final R10 = new R2 - tmp1	; outi(3)=t7-t8				; 44-46
	vaddpd	ymm10, ymm10, ymm5	;; Final R2 = new R2 + tmp1	; outi(2)=t7+t8				; 45-47
	vsubpd	ymm5, ymm8, ymm0	;; Final R4 = new R4 - tmp2	; outi(1)=t5-t6				; 46-48
	vaddpd	ymm8, ymm8, ymm0	;; Final R8 = new R4 + tmp2	; outi(4)=t5+t6				; 47-49

	ystore	dsti1, ymm11				;; Save final I1					;		46

	vblendpd ymm3, ymm4, ymm9, 1	;; final R9 (I4)		; final I4				;		47
	vblendpd ymm2, ymm6, ymm10, 1	;; final R2			; final R2				;		48
	vblendpd ymm7, ymm1, ymm5, 1	;; final R4			; final R4				;		49
	vblendpd ymm0, ymm5, ymm1, 1	;; final R7 (I2)		; final I2				;		50
	vblendpd ymm1, ymm10, ymm8, 1	;; final R8 (I3)		; final I3				;		51
	vblendpd ymm6, ymm8, ymm4, 1	;; final R10 (I5)		; final I5				;		52
	ENDM


;; Haswell FMA3 version

IF (@INSTR(,%yarch,<FMA3>) NE 0)

yr5_o10r_t5c_djbunfft_mem_preload MACRO
	vmovapd	ymm15, YMM_P809
	vmovapd	ymm14, YMM_P309
	vmovapd	ymm13, YMM_P588
	vmovapd	ymm12, YMM_P951
	vmovapd	ymm11, YMM_ONE
	ENDM

yr5_o10r_t5c_djbunfft_mem MACRO memr1,memr2,memr3,memr4,memr5,screg,dstr1,dstr3,dstr5,dsti1
					;; Ten reals comments		;; Five complex comments
	vmovapd	ymm8, memr2				;; R2
	vmovapd	ymm9, memr2[32]				;; I2
	vmovapd	ymm4, [screg+32]			;; cosine/sine
	yfmaddpd ymm1, ymm8, ymm4, ymm9			;; A2 = R2 * cosine/sine + I2				;	1-5
	yfmsubpd ymm9, ymm9, ymm4, ymm8			;; B2 = I2 * cosine/sine - R2				; 1-5

	vmovapd	ymm8, memr4				;; R4
	vmovapd	ymm2, memr4[32]				;; I4
	vmovapd	ymm3, [screg+128+32]			;; cosine/sine
	yfmaddpd ymm0, ymm8, ymm3, ymm2			;; A4 = R4 * cosine/sine + I4				;	2-6
	yfmsubpd ymm2, ymm2, ymm3, ymm8			;; B4 = I4 * cosine/sine - R4				; 2-6

	vmovapd	ymm5, memr5				;; R5
	vmovapd	ymm7, memr5[32]				;; I5
	vmovapd	ymm6, [screg+192+32]			;; cosine/sine
	yfmaddpd ymm10, ymm5, ymm6, ymm7		;; A5 = R5 * cosine/sine + I5				;	3-7
	yfmsubpd ymm7, ymm7, ymm6, ymm5			;; B5 = I5 * cosine/sine - R5				; 3-7

	vmovapd	ymm5, memr3				;; R3
	vmovapd	ymm4, memr3[32]				;; I3
	vmovapd	ymm3, [screg+64+32]			;; cosine/sine
	yfmaddpd ymm8, ymm5, ymm3, ymm4			;; A3 = R3 * cosine/sine + I3				;	4-8
	yfmsubpd ymm4, ymm4, ymm3, ymm5			;; B3 = I3 * cosine/sine - R3				; 4-8

	vmovapd	ymm3, [screg]				;; sine
	vmulpd	ymm1, ymm1, ymm3			;; A2 = A2 * sine (new R2)				;	6-10
	vmulpd	ymm9, ymm9, ymm3			;; B2 = B2 * sine (new I2)				; 6-10

	vmovapd	ymm3, [screg+128]			;; sine
	vmulpd	ymm0, ymm0, ymm3			;; A4 = A4 * sine (new R4)				;	7-11
	vmulpd	ymm2, ymm2, ymm3			;; B4 = B4 * sine (new I4)				; 7-11

	vmovapd	ymm5, [screg+192]			;; sine
	yfnmaddpd ymm6, ymm7, ymm5, ymm9 ;; I2-I5			; i25s=i2-i5				;	11-15
	yfmaddpd ymm7, ymm7, ymm5, ymm9	;; I2+I5			; i25a=i2+i5				; 11-15

	vmovapd	ymm3, [screg+64]			;; sine
	yfmaddpd ymm9, ymm10, ymm5, ymm1 ;; R2+R5			; r25a=r2+r5				;	12-16
	yfmsubpd ymm10, ymm10, ymm5, ymm1 ;; NEG(R2-R5) = R5-R2		; NEGr25s=r5-r2				; 12-16

	yfmsubpd ymm5, ymm4, ymm3, ymm2 ;; I3-I4			; i34s=i3-i4				;	13-17
	yfmaddpd ymm4, ymm4, ymm3, ymm2 ;; I3+I4			; i34a=i3+i4				; 13-17

	vmovapd	ymm1, memr1		;; R1_1				; r1
	yfmaddpd ymm2, ymm8, ymm3, ymm0 ;; R3+R4			; r34a=r3+r4				;	14-18
	yfmsubpd ymm8, ymm8, ymm3, ymm0 ;; R3-R4			; r34s=r3-r4				; 14-18

	vmulpd	ymm0, ymm12, ymm6	;; tmp1 = .951*(I2-I5)		; sin2*i25s				;	16-20
	vmulpd	ymm6, ymm13, ymm6	;; tmp2 = .588*(I2-I5)		; sin4*i25s				; 16-20

	vblendpd ymm3, ymm10, ymm7, 1	;; I2+I5			; NEGr25s				;		17
	vblendpd ymm7, ymm7, ymm10, 1	;; NEG(R2-R5)			; i25a					;		18

	yfmaddpd ymm10, ymm14, ymm9, ymm1 ;; new R3 = R1_1+.309*(R2+R5)	; cos2*r25a + r1			;	17-21

	yfmaddpd ymm0, ymm13, ymm5, ymm0 ;; tmp1 += .588*(I3-I4)	; t2=sin2*i25s + sin4*i34s		;	21-25
	yfnmaddpd ymm6, ymm12, ymm5, ymm6 ;; tmp2 -= .951*(I3-I4)	; t4=sin4*i25s - sin2*i34s		; 21-25

	yfnmaddpd ymm5, ymm15, ymm9, ymm1 ;; new R5 = R1_1-.809*(R2+R5)	; cos4*r25a + r1			; 17-21
	yfmaddpd ymm1, ymm1, ymm11, ymm9 ;; new R1 = R1_1+R2+R5		; r1 + r25a				;	18-22

	vblendpd ymm9, ymm8, ymm4, 1	;; I3+I4			; r34s					;		19
	vblendpd ymm4, ymm4, ymm8, 1	;; R3-R4			; i34a					;		20

	vmulpd	ymm8, ymm13, ymm3	;; tmp3 = .588*(I2+I5)		; sin4*NEGr25s				; 18-22
	vmulpd	ymm3, ymm12, ymm3	;; tmp4 = .951*(I2+I5)		; sin2*NEGr25s				;	19-23

	yfnmaddpd ymm10, ymm15, ymm2, ymm10 ;; new R3 += -.809*(R3+R4)	; t1=cos2*r25a + cos4*r34a + r1		;	22-26
	yfmaddpd ymm5, ymm14, ymm2, ymm5 ;; new R5 += .309*(R3+R4)	; t3=cos4*r25a + cos2*r34a + r1		; 22-26
	yfmaddpd ymm1, ymm1, ymm11, ymm2 ;; final R1 = R1_1+R2+R5+R3+R4	; outr(0) = r1 + r25a + r34a		;	23-27

	vmovapd	ymm2, memr1[32]		;; R1_2				; i1
	ystore	dstr1, ymm1				;; Save final R1					;		28
	yfnmaddpd ymm1, ymm15, ymm7, ymm2 ;; new R2 = R1_2-.809*NEG(R2-R5); cos4*i25a + i1			; 19-23

	yfmaddpd ymm8, ymm12, ymm9, ymm8 ;; tmp3 += .951*(I3+I4)	; t8=sin2*r34s + sin4*NEGr25s		; 23-27
	yfmsubpd ymm3, ymm13, ymm9, ymm3 ;; tmp4 = .588*(I3+I4)-tmp4	; t6=sin4*r34s - sin2*NEGr25s		;	24-28

	yfmaddpd ymm9, ymm14, ymm7, ymm2 ;; new R4 = R1_2+.309*NEG(R2-R5); cos2*i25a + i1			;	20-24
	yfmaddpd ymm2, ymm2, ymm11, ymm7 ;; new R6 = R1_2+NEG(R2-R5)	; i1 + i25a				; 20-24

	yfmaddpd ymm1, ymm14, ymm4, ymm1 ;; new R2 += .309*(R3-R4)	; t7=cos2*i34a + i1 + cos4*i25a		; 24-28
	yfnmaddpd ymm9, ymm15, ymm4, ymm9 ;; new R4 += -.809*(R3-R4)	; t5=cos4*i34a + i1 + cos2*i25a		;	25-29
	yfmaddpd ymm2, ymm2, ymm11, ymm4 ;; final R6 = R6+(R3-R4)+(R3-R4); outi(0)=i1 + i34a + i34a		; 25-29

	yfmsubpd ymm4, ymm10, ymm11, ymm0 ;; final R9 = new R3 - tmp1	; outr(4)=t1-t2				;	27-31
	yfmaddpd ymm10, ymm10, ymm11, ymm0 ;; final R3 = new R3 + tmp1	; outr(1)=t1+t2				; 27-31

	yfmaddpd ymm0, ymm5, ymm11, ymm6 ;; final R5 = new R5 + tmp2	; outr(2)=t3+t4				;	28-32
	yfmsubpd ymm5, ymm5, ymm11, ymm6 ;; final R7 = new R5 - tmp2	; outr(3)=t3-t4				; 28-32

	yfmsubpd ymm6, ymm1, ymm11, ymm8 ;; Final R10 = new R2 - tmp3	; outi(3)=t7-t8				;	29-33
	yfmaddpd ymm1, ymm1, ymm11, ymm8 ;; Final R2 = new R2 + tmp3	; outi(2)=t7+t8				; 29-33

	ystore	dsti1, ymm2				;; Save final I1					;		30
	yfmsubpd ymm8, ymm9, ymm11, ymm3 ;; Final R4 = new R4 - tmp4	; outi(1)=t5-t6				;	30-34
	yfmaddpd ymm9, ymm9, ymm11, ymm3 ;; Final R8 = new R4 + tmp4	; outi(4)=t5+t6				; 30-34

	vblendpd ymm3, ymm0, ymm10, 1			;; final R3						;		33
	ystore	dstr3, ymm3				;; Save final R3					;		34
	vblendpd ymm3, ymm4, ymm0, 1			;; final R5						;		34
	ystore	dstr5, ymm3				;; Save final R5					;		35

	vblendpd ymm3, ymm6, ymm4, 1	;; final R9 (I4)		; final I4				;		35
	vblendpd ymm2, ymm10, ymm1, 1	;; final R2			; final R2				;		36
	vblendpd ymm7, ymm5, ymm8, 1	;; final R4			; final R4				;		37
	vblendpd ymm0, ymm8, ymm5, 1	;; final R7 (I2)		; final I2				;		38
	vblendpd ymm1, ymm1, ymm9, 1	;; final R8 (I3)		; final I3				;		39
	vblendpd ymm6, ymm9, ymm6, 1	;; final R10 (I5)		; final I5				;		40

	ENDM

ENDIF

ENDIF


;;
;; ************************************* 20-reals-fft variants ******************************************
;;

;; This should in theory be faster than an 8-real step 1 followed by a 5-complex (or 10-real) step 2.
;; To see this, count the adds and muls to process 80 reals as either
;;    1)  10 * eight-reals, 2 ten-reals and 6 five-complex, or
;;    2)  4 * twenty-reals, 1/2 * (one eight-reals and 9 four-complex)

;; These macros operate on twenty reals doing 4.32 levels of the FFT.  The output is
;; 2 reals and 9 complex numbers.

;; To calculate a 20-reals FFT, we calculate 20 complex values in a brute force way (using a shorthand notation):
;; r1 + r2 + ... + r20	*  w^0000000000...
;; r1 + r2 + ... + r20	*  w^0123456789A...
;; r1 + r2 + ... + r20	*  w^02468ACE....
;;    ...
;; r1 + r2 + ... + r20	*  w^...A987654321
;; Note that Hermetian symmetry means we won't need to calculate the last 10 complex values.
;;
;; The sin/cos values (w = 20th root of unity) are:
;; w^1 = .951 + .309i
;; w^2 = .809 + .588i
;; w^3 = .588 + .809i
;; w^4 = .309 + .951i
;; w^5 = 0 + 1i
;; w^6 = -.309 + .951i
;; w^7 = -.588 + .809i
;; w^8 = -.809 + .588i
;; w^9 = -.951 + .309i
;; w^10 = -1
;;
;; Applying the sin/cos values above (and noting that combining r2 and r20, r3 and r19, etc. will simplify calculations):
;; reals:
;; r1     +(r2+r20)     +(r3+r19)     +(r4+r18)     +(r5+r17) + (r6+r16)     +(r7+r15)     +(r8+r14)     +(r9+r13)     +(r10+r12) + r11
;; r1 +.951(r2+r20) +.809(r3+r19) +.588(r4+r18) +.309(r5+r17)            -.309(r7+r15) -.588(r8+r14) -.809(r9+r13) -.951(r10+r12) - r11
;; r1 +.809(r2+r20) +.309(r3+r19) -.309(r4+r18) -.809(r5+r17) - (r6+r16) -.809(r7+r15) -.309(r8+r14) +.309(r9+r13) +.809(r10+r12) + r11
;; r1 +.588(r2+r20) -.309(r3+r19) -.951(r4+r18) -.809(r5+r17)            +.809(r7+r15) +.951(r8+r14) +.309(r9+r13) -.588(r10+r12) - r11
;; r1 +.309(r2+r20) -.809(r3+r19) -.809(r4+r18) +.309(r5+r17) + (r6+r16) +.309(r7+r15) -.809(r8+r14) -.809(r9+r13) +.309(r10+r12) + r11
;; r1                   -(r3+r19)                   +(r5+r17)                -(r7+r15)                   +(r9+r13)                - r11
;; r1 -.309(r2+r20) -.809(r3+r19) +.809(r4+r18) +.309(r5+r17) - (r6+r16) +.309(r7+r15) +.809(r8+r14) -.809(r9+r13) -.309(r10+r12) + r11
;; r1 -.588(r2+r20) -.309(r3+r19) +.951(r4+r18) -.809(r5+r17)            +.809(r7+r15) -.951(r8+r14) +.309(r9+r13) +.588(r10+r12) - r11
;; r1 -.809(r2+r20) +.309(r3+r19) +.309(r4+r18) -.809(r5+r17) + (r6+r16) -.809(r7+r15) +.309(r8+r14) +.309(r9+r13) -.809(r10+r12) + r11
;; r1 -.951(r2+r20) +.809(r3+r19) -.588(r4+r18) +.309(r5+r17)            -.309(r7+r15) +.588(r8+r14) -.809(r9+r13) +.951(r10+r12) - r11
;; r1     -(r2+r20)     +(r3+r19)     -(r4+r18)     +(r5+r17) - (r6+r16)     +(r7+r15)     -(r8+r14)     +(r9+r13)     -(r10+r12) + r11
;;
;; imaginarys:
;; 0
;; +.309(r2-r20) +.588(r3-r19) +.809(r4-r18) +.951(r5-r17) + (r6-r16) +.951(r7-r15) +.809(r8-r14) +.588(r9-r13) +.309(r10-r12)
;; +.588(r2-r20) +.951(r3-r19) +.951(r4-r18) +.588(r5-r17)            -.588(r7-r15) -.951(r8-r14) -.951(r9-r13) -.588(r10-r12)
;; +.809(r2-r20) +.951(r3-r19) +.309(r4-r18) -.588(r5-r17) - (r6-r16) -.588(r7-r15) +.309(r8-r14) +.951(r9-r13) +.809(r10-r12)
;; +.951(r2-r20) +.588(r3-r19) -.588(r4-r18) -.951(r5-r17)            +.951(r7-r15) +.588(r8-r14) -.588(r9-r13) -.951(r10-r12)
;;      (r2-r20)                   -(r4-r18)               + (r6-r16)                   -(r8-r14)                   +(r10-r12)
;; +.951(r2-r20) -.588(r3-r19) -.588(r4-r18) +.951(r5-r17)            -.951(r7-r15) +.588(r8-r14) +.588(r9-r13) -.951(r10-r12)
;; +.809(r2-r20) -.951(r3-r19) +.309(r4-r18) +.588(r5-r17) - (r6-r16) +.588(r7-r15) +.309(r8-r14) -.951(r9-r13) +.809(r10-r12)
;; +.588(r2-r20) -.951(r3-r19) +.951(r4-r18) -.588(r5-r17)            +.588(r7-r15) -.951(r8-r14) +.951(r9-r13) -.588(r10-r12)
;; +.309(r2-r20) -.588(r3-r19) +.809(r4-r18) -.951(r5-r17) + (r6-r16) -.951(r7-r15) +.809(r8-r14) -.588(r9-r13) +.309(r10-r12)
;; 0
;;
;; There are many more symmetries we can take advantage of.   For example, the (r2+/-r20) column
;; always has the same multiplier as the (r10+/-r12) column.  This is true for all the "even" columns.
;; Also the computations for the 2nd row are very similar to the computations for the 10th row,
;; the 3rd row are similar to the 9th, etc.  Finally, note that for the odd columns, there are
;; only two multipliers to apply and can be combined with every fourth column.
;;
;; Lastly, output would normally be 9 complex and 2 reals but the users of this routine
;; expect us to "back up" the 2 reals by one level.  That is:
;;	real #1A:  r1 + r3+r19 + r5+r17 + ...
;;	real #1B:  r2+r20 + r4+r18 + ...

yr5_10cl_20_reals_fft_preload MACRO
	ENDM

yr5_10cl_20_reals_fft MACRO srcreg,srcinc,d1,d2,screg,scinc,maxrpt,L1pt,L1pd

	;; Do the odd columns for the real results

	vmovapd	ymm0, [srcreg+2*d2]		;; r5
	vaddpd	ymm0, ymm0, [srcreg+3*d2+32]	;; r5+r17
	vmovapd	ymm3, YMM_P309
	vmulpd	ymm1, ymm3, ymm0		;; .309(r5+r17)
	vmovapd	ymm7, YMM_P809
	vmulpd	ymm4, ymm7, ymm0		;; .809(r5+r17)
	vmovapd	ymm2, [srcreg]			;; r1
	vaddpd	ymm0, ymm2, ymm0		;; r1+(r5+r17)
	vaddpd	ymm1, ymm2, ymm1		;; r1+.309(r5+r17)
	vsubpd	ymm2, ymm2, ymm4		;; r1-.809(r5+r17)

	vmovapd	ymm4, [srcreg+4*d2]		;; r9
	vaddpd	ymm4, ymm4, [srcreg+d2+32]	;; r9+r13
	vmulpd	ymm5, ymm7, ymm4		;; .809(r9+r13)
	vaddpd	ymm0, ymm0, ymm4		;; r1+(r5+r17)+(r9+r13)
	vmulpd	ymm4, ymm3, ymm4		;; .309(r9+r13)
	vsubpd	ymm1, ymm1, ymm5		;; r1+.309(r5+r17)-.809(r9+r13)
	vaddpd	ymm2, ymm2, ymm4		;; r1-.809(r5+r17)+.309(r9+r13)

	vmovapd	ymm5, [srcreg+d2]		;; r3
	vaddpd	ymm5, ymm5, [srcreg+4*d2+32]	;; r3+r19
	vmulpd	ymm6, ymm7, ymm5		;; .809(r3+r19)
	vmulpd	ymm7, ymm3, ymm5		;; .309(r3+r19)
	vmovapd	ymm4, [srcreg+32]		;; r11
	vaddpd	ymm5, ymm5, ymm4		;; (r3+r19)+r11
	vsubpd	ymm6, ymm6, ymm4		;; .809(r3+r19)-r11
	vaddpd	ymm7, ymm7, ymm4		;; .309(r3+r19)+r11

	vmovapd	ymm4, [srcreg+3*d2]		;; r7
	vaddpd	ymm4, ymm4, [srcreg+2*d2+32]	;; r7+r15
	vmulpd	ymm3, ymm3, ymm4		;; .309(r7+r15)
	vaddpd	ymm5, ymm5, ymm4		;; (r3+r19)+(r7+r15)+r11
	vmulpd	ymm4, ymm4, YMM_P809		;; .809(r7+r15)
	vsubpd	ymm6, ymm6, ymm3		;; .809(r3+r19)-.309(r7+r15)-r11
	vsubpd	ymm7, ymm7, ymm4		;; .309(r3+r19)-.809(r7+r15)+r11

	vsubpd	ymm3, ymm0, ymm5		;; Real odd-cols row #6 (final real #6)
	vaddpd	ymm0, ymm0, ymm5		;; Real odd-cols row #1 (final real #1A)

	yloop_optional_early_prefetch

	vsubpd	ymm4, ymm1, ymm6		;; Real odd-cols row #5
	vaddpd	ymm1, ymm1, ymm6		;; Real odd-cols row #2

	L1prefetchw srcreg+L1pd, L1pt

	vsubpd	ymm5, ymm2, ymm7		;; Real odd-cols row #4
	vaddpd	ymm2, ymm2, ymm7		;; Real odd-cols row #3

	ystore	YMM_TMPS[4*32],ymm3		;; Real #6
	ystore	[srcreg], ymm0			;; Final real #1A
	ystore	YMM_TMPS[3*32], ymm4		;; Real odd-cols row #5
	ystore	YMM_TMPS[0*32], ymm1		;; Real odd-cols row #2
	ystore	YMM_TMPS[2*32], ymm5		;; Real odd-cols row #4
	ystore	YMM_TMPS[1*32], ymm2		;; Real odd-cols row #3

	;; Do the even columns for the real results

	vmovapd	ymm7, [srcreg+d1]		;; r2
	vaddpd	ymm7, ymm7, [srcreg+4*d2+d1+32]	;; r2+r20
	vmovapd	ymm0, [srcreg+4*d2+d1]		;; r10
	vaddpd	ymm0, ymm0, [srcreg+d1+32]	;; r10+r12
	vsubpd	ymm1, ymm7, ymm0		;; (r2+r20)-(r10+r12)
	vaddpd	ymm7, ymm7, ymm0		;; (r2+r20)+(r10+r12)

	vmovapd	ymm2, YMM_P951
	vmulpd	ymm6, ymm2, ymm1		;; .951((r2+r20)-(r10+r12))
	vmovapd	ymm5, YMM_P588
	vmulpd	ymm1, ymm5, ymm1		;; .588((r2+r20)-(r10+r12))

	vmovapd	ymm4, [srcreg+d2+d1]		;; r4
	vaddpd	ymm4, ymm4, [srcreg+3*d2+d1+32]	;; r4+r18
	vmovapd	ymm3, [srcreg+3*d2+d1]		;; r8
	vaddpd	ymm3, ymm3, [srcreg+d2+d1+32]	;; r8+r14
	vsubpd	ymm0, ymm4, ymm3		;; (r4+r18)-(r8+r14)
	vaddpd	ymm4, ymm4, ymm3		;; (r4+r18)+(r8+r14)

	L1prefetchw srcreg+d1+L1pd, L1pt

	vmulpd	ymm5, ymm5, ymm0		;; .588((r4+r18)-(r8+r14))
	vaddpd	ymm6, ymm6, ymm5		;; .951((r2+r20)-(r10+r12))+.588((r4+r18)-(r8+r14))
	vmulpd	ymm2, ymm2, ymm0		;; .951((r4+r18)-(r8+r14))
	vsubpd	ymm1, ymm1, ymm2		;; .588((r2+r20)-(r10+r12))-.951((r4+r18)-(r8+r14))

	vmovapd	ymm2, YMM_P809
	vmulpd	ymm0, ymm2, ymm7		;; .809((r2+r20)+(r10+r12))
	vmovapd	ymm3, YMM_P309
	vmulpd	ymm5, ymm3, ymm7		;; .309((r2+r20)+(r10+r12))

	ystore	YMM_TMPS[5*32], ymm6		;; Save real even-cols row #2
	ystore	YMM_TMPS[6*32], ymm1		;; Save real even-cols row #4

	yloop_optional_early_prefetch

	vaddpd	ymm7, ymm7, ymm4		;; ((r2+r20)+(r10+r12))+((r4+r18)+(r8+r14))
	vmulpd	ymm1, ymm3, ymm4		;; .309((r4+r18)+(r8+r14))
	vsubpd	ymm0, ymm0, ymm1		;; .809((r2+r20)+(r10+r12))-.309((r4+r18)+(r8+r14))
	vmulpd	ymm1, ymm2, ymm4		;; .809((r4+r18)+(r8+r14))
	vsubpd	ymm5, ymm5, ymm1		;; .309((r2+r20)+(r10+r12))-.809((r4+r18)+(r8+r14))

	vmovapd	ymm1, [srcreg+2*d2+d1]		;; r6
	vmovapd	ymm6, [srcreg+2*d2+d1+32]	;; r16
	vsubpd	ymm4, ymm1, ymm6		;; r6-r16
	vaddpd	ymm1, ymm1, ymm6		;; r6+r16
	vaddpd	ymm7, ymm7, ymm1		;; ((r2+r20)+(r10+r12))+((r4+r18)+(r8+r14))+(r6+r16)
	vsubpd	ymm0, ymm0, ymm1		;; .809((r2+r20)+(r10+r12))-.309((r4+r18)+(r8+r14))-(r6+r16)
	vaddpd	ymm5, ymm5, ymm1		;; .309((r2+r20)+(r10+r12))-.809((r4+r18)+(r8+r14))+(r6+r16)

	ystore	[srcreg+32], ymm7		;; Save final real #1B (real even-cols row #1)
	ystore	YMM_TMPS[7*32], ymm0		;; Save real even-cols row #3
	ystore	YMM_TMPS[8*32], ymm5		;; Save real even-cols row #5

	;; Do the even columns for the imaginary results

	vmovapd	ymm1, [srcreg+d1]		;; r2
	vsubpd	ymm1, ymm1, [srcreg+4*d2+d1+32]	;; r2-r20
	vmovapd	ymm6, [srcreg+4*d2+d1]		;; r10
	vsubpd	ymm6, ymm6, [srcreg+d1+32]	;; r10-r12
	vaddpd	ymm7, ymm1, ymm6		;; (r2-r20)+(r10-r12)
	vsubpd	ymm1, ymm1, ymm6		;; (r2-r20)-(r10-r12)

	vmulpd	ymm0, ymm3, ymm7		;; .309((r2-r20)+(r10-r12))
	vmulpd	ymm5, ymm2, ymm7		;; .809((r2-r20)+(r10-r12))

	L1prefetchw srcreg+d2+L1pd, L1pt

	vaddpd	ymm7, ymm7, ymm4		;; ((r2-r20)+(r10-r12))+(r6-r16)
	vaddpd	ymm0, ymm0, ymm4		;; .309((r2-r20)+(r10-r12))+(r6-r16)
	vsubpd	ymm5, ymm5, ymm4		;; .809((r2-r20)+(r10-r12))-(r6-r16)

	vmovapd	ymm4, [srcreg+d2+d1]		;; r4
	vsubpd	ymm4, ymm4, [srcreg+3*d2+d1+32]	;; r4-r18
	vmovapd	ymm6, [srcreg+3*d2+d1]		;; r8
	vsubpd	ymm6, ymm6, [srcreg+d2+d1+32]	;; r8-r14

	vaddpd	ymm3, ymm4, ymm6		;; (r4-r18)+(r8-r14)
	vsubpd	ymm4, ymm4, ymm6		;; (r4-r18)-(r8-r14)

	L1prefetchw srcreg+d2+d1+L1pd, L1pt

	vsubpd	ymm7, ymm7, ymm3		;; ((r2-r20)+(r10-r12))-((r4-r18)+(r8-r14))+(r6-r16)
	vmulpd	ymm2, ymm2, ymm3		;; .809((r4-r18)+(r8-r14))
	vaddpd	ymm0, ymm0, ymm2		;; .309((r2-r20)+(r10-r12))+.809((r4-r18)+(r8-r14))+(r6-r16)
	vmulpd	ymm3, ymm3, YMM_P309		;; .309((r4-r18)+(r8-r14))
	vaddpd	ymm5, ymm5, ymm3		;; .809((r2-r20)+(r10-r12))+.309((r4-r18)+(r8-r14))-(r6-r16)

	ystore	YMM_TMPS[9*32], ymm7		;; Save imag row #6
	ystore	YMM_TMPS[10*32], ymm0		;; Save imag even-cols row #2
	ystore	YMM_TMPS[11*32], ymm5		;; Save imag even-cols row #4

	vmovapd	ymm2, YMM_P588
	vmulpd	ymm0, ymm2, ymm1		;; .588((r2-r20)-(r10-r12))
	vmovapd	ymm3, YMM_P951
	vmulpd	ymm5, ymm3, ymm1		;; .951((r2-r20)-(r10-r12))

	L1prefetchw srcreg+2*d2+L1pd, L1pt

	vmulpd	ymm6, ymm3, ymm4		;; .951((r4-r18)-(r8-r14))
	vaddpd	ymm0, ymm0, ymm6		;; .588((r2-r20)-(r10-r12))+.951((r4-r18)-(r8-r14))
	vmulpd	ymm4, ymm2, ymm4		;; .588((r4-r18)-(r8-r14))
	vsubpd	ymm5, ymm5, ymm4		;; .951((r2-r20)-(r10-r12))-.588((r4-r18)-(r8-r14))

	ystore	YMM_TMPS[12*32], ymm0		;; Save imag even-cols row #3
	ystore	YMM_TMPS[13*32], ymm5		;; Save imag even-cols row #5

	;; Do the odd columns for the imag results

	vmovapd	ymm1, [srcreg+2*d2]		;; r5
	vsubpd	ymm1, ymm1, [srcreg+3*d2+32]	;; r5-r17
	vmulpd	ymm6, ymm3, ymm1		;; .951(r5-r17)
	vmulpd	ymm7, ymm2, ymm1		;; .588(r5-r17)

	vmovapd	ymm4, [srcreg+4*d2]		;; r9
	vsubpd	ymm4, ymm4, [srcreg+d2+32]	;; r9-r13
	vmulpd	ymm0, ymm2, ymm4		;; .588(r9-r13)
	vmulpd	ymm4, ymm3, ymm4		;; .951(r9-r13)
	vaddpd	ymm6, ymm6, ymm0		;; .951(r5-r17)+.588(r9-r13)
	vsubpd	ymm7, ymm7, ymm4		;; .588(r5-r17)-.951(r9-r13)

	vmovapd	ymm5, [srcreg+d2]		;; r3
	vsubpd	ymm5, ymm5, [srcreg+4*d2+32]	;; r3-r19
	vmulpd	ymm1, ymm2, ymm5		;; .588(r3-r19)
	vmulpd	ymm5, ymm3, ymm5		;; .951(r3-r19)

	vmovapd	ymm0, [srcreg+3*d2]		;; r7
	vsubpd	ymm0, ymm0, [srcreg+2*d2+32]	;; r7-r15
	vmulpd	ymm4, ymm3, ymm0		;; .951(r7-r15)
	vmulpd	ymm0, ymm2, ymm0		;; .588(r7-r15)
	vaddpd	ymm1, ymm1, ymm4		;; .588(r3-r19)+.951(r7-r15)
	vsubpd	ymm5, ymm5, ymm0		;; .951(r3-r19)-.588(r7-r15)

	yloop_optional_early_prefetch

	vaddpd	ymm2, ymm1, ymm6		;; Imag odd-cols row #2
	vsubpd	ymm1, ymm1, ymm6		;; Imag odd-cols row #5

	L1prefetchw srcreg+2*d2+d1+L1pd, L1pt

	vaddpd	ymm3, ymm5, ymm7		;; Imag odd-cols row #3
	vsubpd	ymm5, ymm5, ymm7		;; Imag odd-cols row #4

	ystore	YMM_TMPS[17*32], ymm1		;; Imag odd-cols row #5
	ystore	YMM_TMPS[15*32], ymm3		;; Imag odd-cols row #3
	ystore	YMM_TMPS[16*32], ymm5		;; Imag odd-cols row #4

	;; Now combine the even and odd columns then do the post-multiply by twiddle factors.

	vmovapd	ymm1, YMM_TMPS[0*32]		;; Real odd-cols row #2
	vmovapd	ymm4, YMM_TMPS[5*32]		;; Real even-cols row #2
	vsubpd	ymm0, ymm1, ymm4		;; Real #10
	vaddpd	ymm1, ymm1, ymm4		;; Real #2
	vmovapd	ymm4, YMM_TMPS[10*32]		;; Imag even-cols row #2
	vsubpd	ymm3, ymm4, ymm2		;; Imag #10
	vaddpd	ymm2, ymm4, ymm2		;; Imag #2

	vmovapd	ymm5, [screg+8*64+32]		;; cosine/sine
	vmulpd	ymm6, ymm0, ymm5		;; A10 = R10 * cosine/sine
	vsubpd	ymm6, ymm6, ymm3		;; A10 = A10 - I10
	vmulpd	ymm3, ymm3, ymm5		;; B10 = I10 * cosine/sine
	vmovapd	ymm4, [screg+32]		;; cosine/sine
	vmulpd	ymm7, ymm1, ymm4		;; A2 = R2 * cosine/sine
	vsubpd	ymm7, ymm7, ymm2		;; A2 = A2 - I2
	vmulpd	ymm2, ymm2, ymm4		;; B2 = I2 * cosine/sine
	vaddpd	ymm3, ymm3, ymm0		;; B10 = B10 + R10
	vaddpd	ymm2, ymm2, ymm1		;; B2 = B2 + R2
	vmulpd	ymm6, ymm6, [screg+8*64]	;; A10 = A10 * sine (final R10)
	vmulpd	ymm3, ymm3, [screg+8*64]	;; B10 = B10 * sine (final I10)
	vmulpd	ymm7, ymm7, [screg]		;; A2 = A2 * sine (final R2)
	vmulpd	ymm2, ymm2, [screg]		;; B2 = B2 * sine (final I2)
	ystore	[srcreg+4*d2+d1], ymm6		;; Save final R10
	ystore	[srcreg+4*d2+d1+32], ymm3	;; Save final I10
	ystore	[srcreg+d1], ymm7		;; Save final R2
	ystore	[srcreg+d1+32], ymm2		;; Save final I2

	vmovapd	ymm1, YMM_TMPS[1*32]		;; Real odd-cols row #3
	vmovapd	ymm2, YMM_TMPS[7*32]		;; Real even-cols row #3
	vsubpd	ymm0, ymm1, ymm2		;; Real #9
	vaddpd	ymm1, ymm1, ymm2		;; Real #3
	vmovapd	ymm3, YMM_TMPS[12*32]		;; Imag even-cols row #3
	vmovapd	ymm4, YMM_TMPS[15*32]		;; Imag odd-cols row #3
	vsubpd	ymm2, ymm3, ymm4		;; Imag #9
	vaddpd	ymm3, ymm3, ymm4		;; Imag #3

	vmovapd	ymm5, [screg+7*64+32]		;; cosine/sine
	vmulpd	ymm6, ymm0, ymm5		;; A9 = R9 * cosine/sine
	vsubpd	ymm6, ymm6, ymm2		;; A9 = A9 - I9
	vmulpd	ymm2, ymm2, ymm5		;; B9 = I9 * cosine/sine
	vmovapd	ymm4, [screg+64+32]		;; cosine/sine
	vmulpd	ymm7, ymm1, ymm4		;; A3 = R3 * cosine/sine
	vsubpd	ymm7, ymm7, ymm3		;; A3 = A3 - I3
	vmulpd	ymm3, ymm3, ymm4		;; B3 = I3 * cosine/sine
	vaddpd	ymm2, ymm2, ymm0		;; B9 = B9 + R9
	vaddpd	ymm3, ymm3, ymm1		;; B3 = B3 + R3
	L1prefetchw srcreg+3*d2+L1pd, L1pt
	vmulpd	ymm6, ymm6, [screg+7*64]	;; A9 = A9 * sine (final R9)
	vmulpd	ymm2, ymm2, [screg+7*64]	;; B9 = B9 * sine (final I9)
	vmulpd	ymm7, ymm7, [screg+64]		;; A3 = A3 * sine (final R3)
	vmulpd	ymm3, ymm3, [screg+64]		;; B3 = B3 * sine (final I3)
	ystore	[srcreg+4*d2], ymm6		;; Save final R9
	ystore	[srcreg+4*d2+32], ymm2		;; Save final I9
	ystore	[srcreg+d2], ymm7		;; Save final R3
	ystore	[srcreg+d2+32], ymm3		;; Save final I3

	vmovapd	ymm1, YMM_TMPS[2*32]		;; Real odd-cols row #4
	vmovapd	ymm2, YMM_TMPS[6*32]		;; Real even-cols row #4
	vsubpd	ymm0, ymm1, ymm2		;; Real #8
	vaddpd	ymm1, ymm1, ymm2		;; Real #4
	vmovapd	ymm3, YMM_TMPS[11*32]		;; Imag even-cols row #4
	vmovapd	ymm4, YMM_TMPS[16*32]		;; Imag odd-cols row #4
	vsubpd	ymm2, ymm3, ymm4		;; Imag #8
	vaddpd	ymm3, ymm3, ymm4		;; Imag #4

	yloop_optional_early_prefetch

	vmovapd	ymm5, [screg+6*64+32]		;; cosine/sine
	vmulpd	ymm6, ymm0, ymm5		;; A8 = R8 * cosine/sine
	vsubpd	ymm6, ymm6, ymm2		;; A8 = A8 - I8
	vmulpd	ymm2, ymm2, ymm5		;; B8 = I8 * cosine/sine
	vmovapd	ymm4, [screg+2*64+32]		;; cosine/sine
	vmulpd	ymm7, ymm1, ymm4		;; A4 = R4 * cosine/sine
	vsubpd	ymm7, ymm7, ymm3		;; A4 = A4 - I4
	vmulpd	ymm3, ymm3, ymm4		;; B4 = I4 * cosine/sine
	vaddpd	ymm2, ymm2, ymm0		;; B8 = B8 + R8
	vaddpd	ymm3, ymm3, ymm1		;; B4 = B4 + R4
	L1prefetchw srcreg+3*d2+d1+L1pd, L1pt
	vmulpd	ymm6, ymm6, [screg+6*64]	;; A8 = A8 * sine (final R8)
	vmulpd	ymm2, ymm2, [screg+6*64]	;; B8 = B8 * sine (final I8)
	vmulpd	ymm7, ymm7, [screg+2*64]	;; A4 = A4 * sine (final R4)
	vmulpd	ymm3, ymm3, [screg+2*64]	;; B4 = B4 * sine (final I4)
	ystore	[srcreg+3*d2+d1], ymm6		;; Save final R8
	ystore	[srcreg+3*d2+d1+32], ymm2	;; Save final I8
	ystore	[srcreg+d2+d1], ymm7		;; Save final R4
	ystore	[srcreg+d2+d1+32], ymm3		;; Save final I4

	vmovapd	ymm1, YMM_TMPS[3*32]		;; Real odd-cols row #5
	vmovapd	ymm2, YMM_TMPS[8*32]		;; Real even-cols row #5
	vsubpd	ymm0, ymm1, ymm2		;; Real #7
	vaddpd	ymm1, ymm1, ymm2		;; Real #5
	vmovapd	ymm3, YMM_TMPS[13*32]		;; Imag even-cols row #5
	vmovapd	ymm4, YMM_TMPS[17*32]		;; Imag odd-cols row #5
	vsubpd	ymm2, ymm3, ymm4		;; Imag #7
	vaddpd	ymm3, ymm3, ymm4		;; Imag #5

	vmovapd	ymm5, [screg+5*64+32]		;; cosine/sine
	vmulpd	ymm6, ymm0, ymm5		;; A7 = R7 * cosine/sine
	vsubpd	ymm6, ymm6, ymm2		;; A7 = A7 - I7
	vmulpd	ymm2, ymm2, ymm5		;; B7 = I7 * cosine/sine
	vmovapd	ymm4, [screg+3*64+32]		;; cosine/sine
	vmulpd	ymm7, ymm1, ymm4		;; A5 = R5 * cosine/sine
	vsubpd	ymm7, ymm7, ymm3		;; A5 = A5 - I5
	vmulpd	ymm3, ymm3, ymm4		;; B5 = I5 * cosine/sine
	vaddpd	ymm2, ymm2, ymm0		;; B7 = B7 + R7
	vaddpd	ymm3, ymm3, ymm1		;; B5 = B5 + R5
	L1prefetchw srcreg+4*d2+L1pd, L1pt
	vmulpd	ymm6, ymm6, [screg+5*64]	;; A7 = A7 * sine (final R7)
	vmulpd	ymm2, ymm2, [screg+5*64]	;; B7 = B7 * sine (final I7)
	vmulpd	ymm7, ymm7, [screg+3*64]	;; A5 = A5 * sine (final R5)
	vmulpd	ymm3, ymm3, [screg+3*64]	;; B5 = B5 * sine (final I5)
	ystore	[srcreg+3*d2], ymm6		;; Save final R7
	ystore	[srcreg+3*d2+32], ymm2		;; Save final I7
	ystore	[srcreg+2*d2], ymm7		;; Save final R5
	ystore	[srcreg+2*d2+32], ymm3		;; Save final I5

	vmovapd	ymm1, YMM_TMPS[4*32]		;; Real #6
	vmovapd	ymm2, YMM_TMPS[9*32]		;; Imag #6
	vmovapd	ymm5, [screg+4*64+32]		;; cosine/sine
	vmulpd	ymm0, ymm1, ymm5		;; A6 = R6 * cosine/sine
	vsubpd	ymm0, ymm0, ymm2		;; A6 = A6 - I6
	L1prefetchw srcreg+4*d2+d1+L1pd, L1pt
	vmulpd	ymm2, ymm2, ymm5		;; B6 = I6 * cosine/sine
	vaddpd	ymm2, ymm2, ymm1		;; B6 = B6 + R6
	vmulpd	ymm0, ymm0, [screg+4*64]	;; A6 = A6 * sine (final R6)
	vmulpd	ymm2, ymm2, [screg+4*64]	;; B6 = B6 * sine (final I6)
	ystore	[srcreg+2*d2+d1], ymm0		;; Save final R6
	ystore	[srcreg+2*d2+d1+32], ymm2	;; Save final I6

	bump	srcreg, srcinc
	bump	screg, scinc
	ENDM

;; 64-bit version

IFDEF X86_64

yr5_10cl_20_reals_fft_preload MACRO
	vmovapd	ymm12, YMM_P309
	vmovapd	ymm13, YMM_P809
	vmovapd	ymm14, YMM_P588
	vmovapd	ymm15, YMM_P951
	ENDM

yr5_10cl_20_reals_fft MACRO srcreg,srcinc,d1,d2,screg,scinc,maxrpt,L1pt,L1pd
	vmovapd	ymm0, [srcreg+2*d2]		;; r5
	vaddpd	ymm0, ymm0, [srcreg+3*d2+32]	;; r5+r17						; 1-3

	vmovapd	ymm1, [srcreg+d2]		;; r3
	vaddpd	ymm1, ymm1, [srcreg+4*d2+32]	;; r3+r19						; 2-4

	vmovapd	ymm2, [srcreg+4*d2]		;; r9
	vaddpd	ymm2, ymm2, [srcreg+d2+32]	;; r9+r13						; 3-5

	vmovapd	ymm3, [srcreg+3*d2]		;; r7
	vaddpd	ymm3, ymm3, [srcreg+2*d2+32]	;; r7+r15						; 4-6
	vmulpd	ymm4, ymm12, ymm0		;; .309(r5+r17)						;	4-8

	vmovapd	ymm5, [srcreg]			;; r1
	vaddpd	ymm6, ymm5, ymm0		;; r1+(r5+r17)						; 5-7
	vmulpd	ymm0, ymm13, ymm0		;; .809(r5+r17)						;	5-9

	vmovapd	ymm7, [srcreg+32]		;; r11
	vaddpd	ymm8, ymm1, ymm7		;; (r3+r19)+r11						; 6-8
	vmulpd	ymm9, ymm13, ymm1		;; .809(r3+r19)						;	6-10

	yloop_optional_early_prefetch

	vaddpd	ymm6, ymm6, ymm2		;; r1+(r5+r17)+(r9+r13)					; 7-9
	vmulpd	ymm1, ymm12, ymm1		;; .309(r3+r19)						;	7-11

	vaddpd	ymm8, ymm8, ymm3		;; (r3+r19)+(r7+r15)+r11				; 8-10
	vmulpd	ymm10, ymm13, ymm2		;; .809(r9+r13)						;	8-12

	L1prefetchw srcreg+L1pd, L1pt

	vaddpd	ymm4, ymm5, ymm4		;; r1+.309(r5+r17)					; 9-11
	vmulpd	ymm2, ymm12, ymm2		;; .309(r9+r13)						;	9-13

	vsubpd	ymm5, ymm5, ymm0		;; r1-.809(r5+r17)					; 10-12
	vmulpd	ymm0, ymm12, ymm3		;; .309(r7+r15)						;	10-14
	vmovapd	ymm11, [srcreg+d1]		;; r2

	vsubpd	ymm9, ymm9, ymm7		;; .809(r3+r19)-r11					; 11-13
	vmulpd	ymm3, ymm13, ymm3		;; .809(r7+r15)						;	11-15

	vaddpd	ymm1, ymm1, ymm7		;; .309(r3+r19)+r11					; 12-14
	vmovapd	ymm7, [srcreg+4*d2+d1]		;; r10

	vsubpd	ymm4, ymm4, ymm10		;; r1+.309(r5+r17)-.809(r9+r13)				; 13-15
	vmovapd	ymm10, [srcreg+d2+d1]		;; r4

	vaddpd	ymm5, ymm5, ymm2		;; r1-.809(r5+r17)+.309(r9+r13)				; 14-16
	vmovapd	ymm2, [srcreg+3*d2+d1]		;; r8

	vsubpd	ymm9, ymm9, ymm0		;; .809(r3+r19)-.309(r7+r15)-r11			; 15-17
	vmovapd	ymm0, [srcreg+2*d2+d1]		;; r6

	vsubpd	ymm1, ymm1, ymm3		;; .309(r3+r19)-.809(r7+r15)+r11			; 16-18

	L1prefetchw srcreg+d1+L1pd, L1pt

	vsubpd	ymm3, ymm6, ymm8		;; Real odd-cols row #6 (final real #6)			; 17-19
	vaddpd	ymm6, ymm6, ymm8		;; Real odd-cols row #1 (final real #1A) 		; 18-20

	L1prefetchw srcreg+d2+L1pd, L1pt

	vsubpd	ymm8, ymm4, ymm9		;; Real odd-cols row #5					; 19-21
	vaddpd	ymm4, ymm4, ymm9		;; Real odd-cols row #2					; 20-22

	vsubpd	ymm11, ymm11, [srcreg+4*d2+d1+32] ;; r2-r20						; 21-23
	ystore	[srcreg], ymm6			;; Final real #1A					; 21

	vsubpd	ymm7, ymm7, [srcreg+d1+32]	;; r10-r12						; 22-24
	ystore	YMM_TMPS[3*32], ymm8		;; Real odd-cols row #5					; 22

	vsubpd	ymm10, ymm10, [srcreg+3*d2+d1+32] ;; r4-r18						; 23-25
	ystore	YMM_TMPS[0*32], ymm4		;; Real odd-cols row #2					; 23

	vsubpd	ymm2, ymm2, [srcreg+d2+d1+32]	;; r8-r14						; 24-26

	L1prefetchw srcreg+d2+d1+L1pd, L1pt

	vaddpd	ymm4, ymm11, ymm7		;; (r2-r20)+(r10-r12)					; 25-27
	vsubpd	ymm11, ymm11, ymm7		;; (r2-r20)-(r10-r12)					; 26-28

	yloop_optional_early_prefetch

	vaddpd	ymm7, ymm10, ymm2		;; (r4-r18)+(r8-r14)					; 27-29
	vsubpd	ymm10, ymm10, ymm2		;; (r4-r18)-(r8-r14)					; 28-30

	vsubpd	ymm0, ymm0, [srcreg+2*d2+d1+32]	;; r6-r16						; 29-31

	vmulpd	ymm2, ymm12, ymm4		;; .309((r2-r20)+(r10-r12))				;	28-32
	vmulpd	ymm8, ymm13, ymm4		;; .809((r2-r20)+(r10-r12))				;	29-33

	vsubpd	ymm4, ymm4, ymm7		;; ((r2-r20)+(r10-r12))-((r4-r18)+(r8-r14)) 		; 30-32

	vmulpd	ymm6, ymm13, ymm7		;; .809((r4-r18)+(r8-r14))				;	30-34
	vmulpd	ymm7, ymm12, ymm7		;; .309((r4-r18)+(r8-r14))				;	31-35

	L1prefetchw srcreg+2*d2+L1pd, L1pt

	vsubpd	ymm9, ymm5, ymm1		;; Real odd-cols row #4					; 31-33
	vaddpd	ymm5, ymm5, ymm1		;; Real odd-cols row #3					; 32-34

	vmulpd	ymm1, ymm14, ymm11		;; .588((r2-r20)-(r10-r12))				;	32-36
	vmulpd	ymm11, ymm15, ymm11		;; .951((r2-r20)-(r10-r12))				;	33-37

	vaddpd	ymm4, ymm4, ymm0		;; ((r2-r20)+(r10-r12))-((r4-r18)+(r8-r14))+(r6-r16) 	; 33-35			(I6)
	ystore	YMM_TMPS[2*32], ymm9		;; Real odd-cols row #4					; 34

	vmovapd	ymm9, [srcreg+2*d2]		;; r5
	vsubpd	ymm9, ymm9, [srcreg+3*d2+32]	;; r5-r17						; 34-36
	ystore	YMM_TMPS[1*32], ymm5		;; Real odd-cols row #3					; 35

	vmulpd	ymm5, ymm15, ymm10		;; .951((r4-r18)-(r8-r14))				;	34-38
	vmulpd	ymm10, ymm14, ymm10		;; .588((r4-r18)-(r8-r14))				;	35-39

	vaddpd	ymm2, ymm2, ymm6		;; .309((r2-r20)+(r10-r12))+.809((r4-r18)+(r8-r14)) 	; 35-37
	vmovapd	ymm6, [srcreg+4*d2]		;; r9

	vaddpd	ymm8, ymm8, ymm7		;; .809((r2-r20)+(r10-r12))+.309((r4-r18)+(r8-r14)) 	; 36-38

	vsubpd	ymm6, ymm6, [srcreg+d2+32]	;; r9-r13						; 37-39
	vmovapd	ymm7, [screg+4*64+32]		;; cosine/sine

	vaddpd	ymm2, ymm2, ymm0		;; .309((r2-r20)+(r10-r12))+.809((r4-r18)+(r8-r14))+(r6-r16) ; 38-40
	vsubpd	ymm8, ymm8, ymm0		;; .809((r2-r20)+(r10-r12))+.309((r4-r18)+(r8-r14))-(r6-r16) ; 39-41

	vmulpd	ymm0, ymm3, ymm7		;; A6 = R6 * cosine/sine				;	36-40
	vmulpd	ymm7, ymm4, ymm7		;; B6 = I6 * cosine/sine				;	37-41

	ystore	YMM_TMPS[10*32], ymm2		;; Save imag even-cols row #2				; 41
	vmovapd	ymm2, [srcreg+d2]		;; r3
	vsubpd	ymm2, ymm2, [srcreg+4*d2+32]	;; r3-r19						; 40-42

	L1prefetchw srcreg+2*d2+d1+L1pd, L1pt

	vsubpd	ymm0, ymm0, ymm4		;; A6 = A6 - I6						; 41-43
	vmovapd	ymm4, [srcreg+3*d2]		;; r7

	vaddpd	ymm7, ymm7, ymm3		;; B6 = B6 + R6						; 42-44
	ystore	YMM_TMPS[11*32], ymm8		;; Save imag even-cols row #4				; 42

	vmulpd	ymm3, ymm15, ymm9		;; .951(r5-r17)						;	41-45
	vmulpd	ymm9, ymm14, ymm9		;; .588(r5-r17)						;	42-46

	vsubpd	ymm4, ymm4, [srcreg+2*d2+32]	;; r7-r15						; 43-45

	vmulpd	ymm8, ymm14, ymm6		;; .588(r9-r13)						;	43-47
	vmulpd	ymm6, ymm15, ymm6		;; .951(r9-r13)						;	44-48

	yloop_optional_early_prefetch

	vaddpd	ymm1, ymm1, ymm5		;; .588((r2-r20)-(r10-r12))+.951((r4-r18)-(r8-r14))	; 44-46
	vmovapd ymm5, [screg+4*64]		;; sine

	vsubpd	ymm11, ymm11, ymm10		;; .951((r2-r20)-(r10-r12))-.588((r4-r18)-(r8-r14))	; 45-47
	vmovapd	ymm10, [srcreg+d1]		;; r2

	vmulpd	ymm0, ymm0, ymm5		;; A6 = A6 * sine (final R6)				;	45-49
	vmulpd	ymm7, ymm7, ymm5		;; B6 = B6 * sine (final I6)				;	46-50

	vaddpd	ymm10, ymm10, [srcreg+4*d2+d1+32] ;; r2+r20						; 46-48

	vmovapd	ymm5, [srcreg+4*d2+d1]		;; r10
	vaddpd	ymm5, ymm5, [srcreg+d1+32]	;; r10+r12						; 47-49

	ystore	YMM_TMPS[4*32], ymm1		;; Save imag even-cols row #3				; 47
	vmulpd	ymm1, ymm14, ymm2		;; .588(r3-r19)						;	47-51

	ystore	YMM_TMPS[5*32], ymm11		;; Save imag even-cols row #5				; 48
	vmovapd	ymm11, [srcreg+2*d2+d1]		;; r6
	vaddpd	ymm11, ymm11, [srcreg+2*d2+d1+32] ;; r6+r16						; 48-50
	vmulpd	ymm2, ymm15, ymm2		;; .951(r3-r19)						;	48-52

	ystore	[srcreg+2*d2+d1], ymm0		;; Save final R6					; 50
	vmovapd	ymm0, [srcreg+d2+d1]		;; r4
	vaddpd	ymm0, ymm0, [srcreg+3*d2+d1+32] ;; r4+r18						; 49-51

	ystore	[srcreg+2*d2+d1+32], ymm7	;; Save final I6					; 51
	vmovapd	ymm7, [srcreg+3*d2+d1]		;; r8
	vaddpd	ymm7, ymm7, [srcreg+d2+d1+32]	;; r8+r14						; 50-52

	vaddpd	ymm3, ymm3, ymm8		;; .951(r5-r17)+.588(r9-r13)				; 51-53
	vmulpd	ymm8, ymm15, ymm4		;; .951(r7-r15)						;	49-53
	vmulpd	ymm4, ymm14, ymm4		;; .588(r7-r15)						;	50-54

	vsubpd	ymm9, ymm9, ymm6		;; .588(r5-r17)-.951(r9-r13)				; 52-54

	L1prefetchw srcreg+3*d2+L1pd, L1pt

	vaddpd	ymm6, ymm10, ymm5		;; (r2+r20)+(r10+r12)					; 53-55
	vsubpd	ymm10, ymm10, ymm5		;; (r2+r20)-(r10+r12)					; 54-56
	vaddpd	ymm5, ymm0, ymm7		;; (r4+r18)+(r8+r14)					; 55-57
	vsubpd	ymm0, ymm0, ymm7		;; (r4+r18)-(r8+r14)					; 56-58

	vmulpd	ymm7, ymm13, ymm6		;; .809((r2+r20)+(r10+r12))				;	56-60

	yloop_optional_early_prefetch

	vaddpd	ymm1, ymm1, ymm8		;; .588(r3-r19)+.951(r7-r15)				; 57-59
	vmulpd	ymm8, ymm12, ymm6		;; .309((r2+r20)+(r10+r12))				;	57-61

	L1prefetchw srcreg+3*d2+d1+L1pd, L1pt

	vsubpd	ymm2, ymm2, ymm4		;; .951(r3-r19)-.588(r7-r15)				; 58-60

	vaddpd	ymm6, ymm6, ymm5		;; ((r2+r20)+(r10+r12))+((r4+r18)+(r8+r14)) 		; 59-61

	vaddpd	ymm4, ymm1, ymm3		;; Imag odd-cols row #2					; 60-62
	vsubpd	ymm1, ymm1, ymm3		;; Imag odd-cols row #5					; 61-63
	vmulpd	ymm3, ymm12, ymm5		;; .309((r4+r18)+(r8+r14))				;	58-62
	vmulpd	ymm5, ymm13, ymm5		;; .809((r4+r18)+(r8+r14))				;	59-63

	ystore	YMM_TMPS[7*32], ymm1		;; Imag odd-cols row #5					; 64
	vaddpd	ymm1, ymm2, ymm9		;; Imag odd-cols row #3					; 62-64
	vsubpd	ymm2, ymm2, ymm9		;; Imag odd-cols row #4					; 63-65

	vmulpd	ymm9, ymm15, ymm10		;; .951((r2+r20)-(r10+r12))				;	60-64
	vmulpd	ymm10, ymm14, ymm10		;; .588((r2+r20)-(r10+r12))				;	61-65

	ystore	YMM_TMPS[9*32], ymm2		;; Imag odd-cols row #4					; 66
	vmulpd	ymm2, ymm14, ymm0		;; .588((r4+r18)-(r8+r14))				;	62-66
	vmulpd	ymm0, ymm15, ymm0		;; .951((r4+r18)-(r8+r14))				;	63-67

	vsubpd	ymm7, ymm7, ymm3		;; .809((r2+r20)+(r10+r12))-.309((r4+r18)+(r8+r14)) 	; 64-66
	vmovapd	ymm3, YMM_TMPS[1*32]		;; Real odd-cols row #3	

	vsubpd	ymm8, ymm8, ymm5		;; .309((r2+r20)+(r10+r12))-.809((r4+r18)+(r8+r14)) 	; 65-67
	vmovapd	ymm5, YMM_TMPS[4*32]		;; Imag even-cols row #3

	vaddpd	ymm6, ymm6, ymm11		;; ((r2+r20)+(r10+r12))+((r4+r18)+(r8+r14))+(r6+r16) 	; 66-68

	vsubpd	ymm7, ymm7, ymm11		;; .809((r2+r20)+(r10+r12))-.309((r4+r18)+(r8+r14))-(r6+r16) ; 67-69

	vaddpd	ymm8, ymm8, ymm11		;; .309((r2+r20)+(r10+r12))-.809((r4+r18)+(r8+r14))+(r6+r16) ; 68-70
	vmovapd	ymm11, [screg+7*64+32]		;; cosine/sine

	vaddpd	ymm9, ymm9, ymm2		;; .951((r2+r20)-(r10+r12))+.588((r4+r18)-(r8+r14)) 	; 69-71
	vmovapd	ymm2, [screg+64+32]		;; cosine/sine

	vsubpd	ymm10, ymm10, ymm0		;; .588((r2+r20)-(r10+r12))-.951((r4+r18)-(r8+r14)) 	; 70-72
	ystore	[srcreg+32], ymm6		;; Save final real #1B (real even-cols row #1)		; 69

	;; Now combine the even and odd columns then do the post-multiply by twiddle factors.

	vsubpd	ymm6, ymm3, ymm7		;; Real #9						; 71-73
	ystore	YMM_TMPS[8*32], ymm8		;; Save real even-cols row #5				; 71

	vsubpd	ymm8, ymm5, ymm1		;; Imag #9						; 72-74
	vmovapd	ymm0, YMM_TMPS[10*32]		;; Imag even-cols row #2

	vaddpd	ymm3, ymm3, ymm7		;; Real #3						; 73-75
	vmovapd	ymm7, YMM_TMPS[0*32]		;; Real odd-cols row #2
	ystore	YMM_TMPS[6*32], ymm10		;; Save real even-cols row #4				; 73

	vaddpd	ymm5, ymm5, ymm1		;; Imag #3						; 74-76

	vmulpd	ymm1, ymm6, ymm11		;; A9 = R9 * cosine/sine				;	74-78
	vmulpd	ymm11, ymm8, ymm11		;; B9 = I9 * cosine/sine				;	75-79

	vsubpd	ymm10, ymm7, ymm9		;; Real #10						; 75-77
	vaddpd	ymm7, ymm7, ymm9		;; Real #2						; 76-78

	L1prefetchw srcreg+4*d2+L1pd, L1pt

	vsubpd	ymm9, ymm0, ymm4		;; Imag #10						; 77-79
	vaddpd	ymm0, ymm0, ymm4		;; Imag #2						; 78-80

	vmulpd	ymm4, ymm3, ymm2		;; A3 = R3 * cosine/sine				;	76-80
	vmulpd	ymm2, ymm5, ymm2		;; B3 = I3 * cosine/sine				;	77-81

	vsubpd	ymm1, ymm1, ymm8		;; A9 = A9 - I9						; 79-81
	vmovapd	ymm8, [screg+8*64+32]		;; cosine/sine
	vaddpd	ymm11, ymm11, ymm6		;; B9 = B9 + R9						; 80-82
	vmulpd	ymm6, ymm10, ymm8		;; A10 = R10 * cosine/sine				;	78-82

	vsubpd	ymm4, ymm4, ymm5		;; A3 = A3 - I3						; 81-83
	vmovapd	ymm5, [screg+32]		;; cosine/sine
	vaddpd	ymm2, ymm2, ymm3		;; B3 = B3 + R3						; 82-84
	vmulpd	ymm3, ymm7, ymm5		;; A2 = R2 * cosine/sine				;	79-83

	L1prefetchw srcreg+4*d2+d1+L1pd, L1pt

	vmulpd	ymm8, ymm9, ymm8		;; B10 = I10 * cosine/sine				;	80-84
	vmulpd	ymm5, ymm0, ymm5		;; B2 = I2 * cosine/sine				;	81-85

	vsubpd	ymm6, ymm6, ymm9		;; A10 = A10 - I10					; 83-85
	vmovapd	ymm9, [screg+7*64]		;; sine
	vmulpd	ymm1, ymm1, ymm9		;; A9 = A9 * sine (final R9)				;	82-86
	vmulpd	ymm11, ymm11, ymm9		;; B9 = B9 * sine (final I9)				;	83-87
	vmovapd	ymm9, [screg+64]		;; sine

	vsubpd	ymm3, ymm3, ymm0		;; A2 = A2 - I2						; 84-86
	vmovapd ymm0, [screg+8*64]		;; sine

	vaddpd	ymm8, ymm8, ymm10		;; B10 = B10 + R10					; 85-87
	vmovapd ymm10, [screg]			;; sine

	vmulpd	ymm4, ymm4, ymm9		;; A3 = A3 * sine (final R3)				;	84-88
	vmulpd	ymm2, ymm2, ymm9		;; B3 = B3 * sine (final I3)				;	85-89
	vmovapd	ymm9, YMM_TMPS[2*32]		;; Real odd-cols row #4

	vaddpd	ymm5, ymm5, ymm7		;; B2 = B2 + R2						; 86-88
	vmulpd	ymm6, ymm6, ymm0		;; A10 = A10 * sine (final R10)				;	86-90
	vmovapd	ymm7, YMM_TMPS[6*32]		;; Real even-cols row #4

	vmulpd	ymm3, ymm3, ymm10		;; A2 = A2 * sine (final R2)				;	87-91
	ystore	[srcreg+4*d2], ymm1		;; Save final R9					; 87
	vsubpd	ymm1, ymm9, ymm7		;; Real #8						; 87-89

	vaddpd	ymm9, ymm9, ymm7		;; Real #4						; 88-90
	vmovapd	ymm7, YMM_TMPS[11*32]		;; Imag even-cols row #4
	vmulpd	ymm8, ymm8, ymm0		;; B10 = B10 * sine (final I10)				;	88-92
	vmovapd	ymm0, YMM_TMPS[9*32]		;; Imag odd-cols row #4
	ystore	[srcreg+4*d2+32], ymm11		;; Save final I9					; 88

	vsubpd	ymm11, ymm7, ymm0		;; Imag #8						; 89-91
	vmulpd	ymm5, ymm5, ymm10		;; B2 = B2 * sine (final I2)				;	89-93
	vmovapd	ymm10, [screg+6*64+32]		;; cosine/sine
	ystore	[srcreg+d2], ymm4		;; Save final R3					; 89
	vmovapd	ymm4, YMM_TMPS[3*32]		;; Real odd-cols row #5

	vaddpd	ymm7, ymm7, ymm0		;; Imag #4						; 90-92
	vmulpd	ymm0, ymm1, ymm10		;; A8 = R8 * cosine/sine				;	90-94
	ystore	[srcreg+d2+32], ymm2		;; Save final I3					; 90
	vmovapd	ymm2, YMM_TMPS[8*32]		;; Real even-cols row #5

	ystore	[srcreg+4*d2+d1], ymm6		;; Save final R10					; 91
	vsubpd	ymm6, ymm4, ymm2		;; Real #7						; 91-93

	vaddpd	ymm4, ymm4, ymm2		;; Real #5						; 92-94
	vmovapd	ymm2, YMM_TMPS[5*32]		;; Imag even-cols row #5
	ystore	[srcreg+d1], ymm3		;; Save final R2					; 92
	vmovapd	ymm3, YMM_TMPS[7*32]		;; Imag odd-cols row #5

	ystore	[srcreg+4*d2+d1+32], ymm8	;; Save final I10					; 93
	vsubpd	ymm8, ymm2, ymm3		;; Imag #7						; 93-95

	vaddpd	ymm2, ymm2, ymm3		;; Imag #5						; 94-96
	vmovapd	ymm3, [screg+2*64+32]		;; cosine/sine
	ystore	[srcreg+d1+32], ymm5		;; Save final I2					; 94
	vmulpd	ymm5, ymm9, ymm3		;; A4 = R4 * cosine/sine				;	91-95
	vmulpd	ymm10, ymm11, ymm10		;; B8 = I8 * cosine/sine				;	92-96

	vmulpd	ymm3, ymm7, ymm3		;; B4 = I4 * cosine/sine				;	93-97

	vsubpd	ymm0, ymm0, ymm11		;; A8 = A8 - I8						; 95-97
	vmovapd	ymm11, [screg+5*64+32]		;; cosine/sine
	vsubpd	ymm5, ymm5, ymm7		;; A4 = A4 - I4						; 96-98
	vmulpd	ymm7, ymm6, ymm11		;; A7 = R7 * cosine/sine				;	94-98

	vaddpd	ymm10, ymm10, ymm1		;; B8 = B8 + R8						; 97-99
	vmovapd	ymm1, [screg+3*64+32]		;; cosine/sine
	vaddpd	ymm3, ymm3, ymm9		;; B4 = B4 + R4						; 98-100
	vmulpd	ymm9, ymm4, ymm1		;; A5 = R5 * cosine/sine				;	95-99
	vmulpd	ymm11, ymm8, ymm11		;; B7 = I7 * cosine/sine				;	96-100
	vmulpd	ymm1, ymm2, ymm1		;; B5 = I5 * cosine/sine				;	97-101

	vsubpd	ymm7, ymm7, ymm8		;; A7 = A7 - I7						; 99-101
	vmovapd	ymm8, [screg+6*64]		;; sine
	vmulpd	ymm0, ymm0, ymm8		;; A8 = A8 * sine (final R8)				;	98-102

	vsubpd	ymm9, ymm9, ymm2		;; A5 = A5 - I5						; 100-102
	vmovapd	ymm2, [screg+2*64]		;; sine
	vmulpd	ymm5, ymm5, ymm2		;; A4 = A4 * sine (final R4)				;	99-103
	vmulpd	ymm10, ymm10, ymm8		;; B8 = B8 * sine (final I8)				;	100-104
	vmovapd	ymm8, [screg+5*64]		;; sine
	vmulpd	ymm3, ymm3, ymm2		;; B4 = B4 * sine (final I4)				;	101-105
	vmovapd	ymm2, [screg+3*64]		;; sine

	vaddpd	ymm11, ymm11, ymm6		;; B7 = B7 + R7						; 101-103
	vaddpd	ymm1, ymm1, ymm4		;; B5 = B5 + R5						; 102-104

	vmulpd	ymm7, ymm7, ymm8		;; A7 = A7 * sine (final R7)				;	102-106
	vmulpd	ymm9, ymm9, ymm2		;; A5 = A5 * sine (final R5)				;	103-107
	vmulpd	ymm11, ymm11, ymm8		;; B7 = B7 * sine (final I7)				;	104-108
	vmulpd	ymm1, ymm1, ymm2		;; B5 = B5 * sine (final I5)				;	105-109

	ystore	[srcreg+3*d2+d1], ymm0		;; Save final R8					; 103
	ystore	[srcreg+d2+d1], ymm5		;; Save final R4					; 104
	ystore	[srcreg+3*d2+d1+32], ymm10	;; Save final I8					; 105
	ystore	[srcreg+d2+d1+32], ymm3		;; Save final I4					; 106

	ystore	[srcreg+3*d2], ymm7		;; Save final R7					; 107
	ystore	[srcreg+2*d2], ymm9		;; Save final R5					; 108
	ystore	[srcreg+3*d2+32], ymm11		;; Save final I7					; 109
	ystore	[srcreg+2*d2+32], ymm1		;; Save final I5					; 110

	bump	srcreg, srcinc
	bump	screg, scinc
	ENDM

;; Haswell FMA3 version

IF (@INSTR(,%yarch,<FMA3>) NE 0)

yr5_10cl_20_reals_fft_preload MACRO
	vmovapd ymm15, YMM_ONE
	ENDM

;; Timed at 71 clocks.  That's 9 clocks worse than theoretically possible.  Converting vaddpd and vsubpd to FMA3 did not seem to help.
yr5_10cl_20_reals_fft MACRO srcreg,srcinc,d1,d2,screg,scinc,maxrpt,L1pt,L1pd
	vmovapd	ymm1, [srcreg+2*d2]		;; r5
	vmovapd	ymm2, [srcreg+3*d2+32]		;; r17
	vaddpd	ymm0, ymm1, ymm2		;; r5+r17						; 1-5
	yfmsubpd ymm1, ymm1, ymm15, ymm2	;; r5-r17						; 1-5

	vmovapd	ymm3, [srcreg+d2]		;; r3
	vmovapd	ymm4, [srcreg+4*d2+32]		;; r19
	vaddpd	ymm2, ymm3, ymm4		;; r3+r19						; 2-6
	yfmsubpd ymm3, ymm3, ymm15, ymm4	;; r3-r19						; 2-6

	vmovapd	ymm5, [srcreg+4*d2]		;; r9
	vmovapd	ymm6, [srcreg+d2+32]		;; r13
	vsubpd	ymm4, ymm5, ymm6		;; r9-r13						; 3-7
	yfmaddpd ymm5, ymm5, ymm15, ymm6	;; r9+r13						; 3-7

	vmovapd	ymm7, [srcreg+3*d2]		;; r7
	vmovapd	ymm8, [srcreg+2*d2+32]		;; r15
	vsubpd	ymm6, ymm7, ymm8		;; r7-r15						; 4-8
	yfmaddpd ymm7, ymm7, ymm15, ymm8	;; r7+r15						; 4-8

	L1prefetchw srcreg+L1pd, L1pt									; stall 5

	vmovapd	ymm8, [srcreg]			;; r1
	vaddpd	ymm9, ymm8, ymm0		;; r1+(r5+r17)						; 6-10
	vmovapd	ymm15, YMM_P809
	yfnmaddpd ymm10, ymm15, ymm0, ymm8	;; r1-.809(r5+r17)					; 6-10
	vmovapd	ymm14, YMM_P309
	yfmaddpd ymm0, ymm14, ymm0, ymm8	;; r1+.309(r5+r17)					; 7-11

	vmovapd	ymm8, [srcreg+32]		;; r11
	yfmaddpd ymm11, ymm14, ymm2, ymm8	;; .309(r3+r19)+r11					; 7-11
	vaddpd	ymm12, ymm2, ymm8		;; (r3+r19)+r11						; 8-12
	yfmsubpd ymm2, ymm15, ymm2, ymm8	;; .809(r3+r19)-r11					; 8-12

	vmovapd	ymm13, YMM_P588_P951
	yfmaddpd ymm8, ymm13, ymm4, ymm1	;; (r5-r17)+.588/.951(r9-r13) (io25B/.951)		; 9-13
	yfmsubpd ymm4, ymm13, ymm1, ymm4	;; .588/.951(r5-r17)-(r9-r13) (io34B/.951)		; 9-13

	yfmaddpd ymm1, ymm13, ymm3, ymm6	;; .588/.951(r3-r19)+(r7-r15) (io25A/.951)		; 10-14
	yfnmaddpd ymm3, ymm13, ymm6, ymm3	;; (r3-r19)-.588/.951(r7-r15) (io34A/.951)		; 10-14
	vmovapd	ymm6, [srcreg+d1]		;; r2

	vaddpd	ymm9, ymm9, ymm5		;; r1+(r5+r17)+(r9+r13)  (ro16A)			; 11-15
	yfmaddpd ymm10, ymm14, ymm5, ymm10	;; r1-.809(r5+r17)+.309(r9+r13)	(ro34A)			; 11-15
	L1prefetchw srcreg+d1+L1pd, L1pt
	yfnmaddpd ymm0, ymm15, ymm5, ymm0	;; r1+.309(r5+r17)-.809(r9+r13) (ro25A)			; 12-16
	vmovapd	ymm13, [srcreg+4*d2+d1+32]	;; r20

	yloop_optional_early_prefetch

	yfnmaddpd ymm11, ymm15, ymm7, ymm11	;; .309(r3+r19)-.809(r7+r15)+r11 (ro34B)		; 12-16
 	vaddpd	ymm12, ymm12, ymm7		;; (r3+r19)+(r7+r15)+r11  (ro16B)			; 13-17
	yfnmaddpd ymm2, ymm14, ymm7, ymm2	;; .809(r3+r19)-.309(r7+r15)-r11 (ro25B)		; 13-17
	vmovapd ymm5, YMM_ONE

	vsubpd	ymm7, ymm6, ymm13		;; r2-r20						; 14-18
	yfmaddpd ymm6, ymm6, ymm5, ymm13	;; r2+r20						; 14-18
	vmovapd	ymm13, [srcreg+4*d2+d1]		;; r10
	ystore	YMM_TMPS[0*32], ymm4		;; Save io34b/.951					; 14

	vaddpd	ymm4, ymm1, ymm8		;; Imag odd-cols row #2/.951				; 15-19
	yfmsubpd ymm1, ymm1, ymm5, ymm8		;; Imag odd-cols row #5/.951				; 15-19
	vmovapd	ymm8, [srcreg+d1+32]		;; r12
	ystore	YMM_TMPS[1*32], ymm3		;; Save io34a/.951					; 15

	vsubpd	ymm3, ymm13, ymm8		;; r10-r12						; 16-20
	yfmaddpd ymm13, ymm13, ymm5, ymm8	;; r10+r12						; 16-20
	vmovapd	ymm8, [srcreg+d2+d1]		;; r4
	ystore	YMM_TMPS[2*32], ymm10		;; Save ro34a						; 16

	vmovapd	ymm10, [srcreg+3*d2+d1+32]	;; r18
	ystore	YMM_TMPS[3*32], ymm11		;; Save ro34b						; 17
	vsubpd	ymm11, ymm8, ymm10		;; r4-r18						; 17-21
	yfmaddpd ymm8, ymm8, ymm5, ymm10	;; r4+r18						; 17-21

	vmovapd	ymm10, [srcreg+3*d2+d1]		;; r8
	ystore	YMM_TMPS[4*32], ymm4		;; Save imag odd #2/.951				; 20
	vmovapd	ymm4, [srcreg+d2+d1+32]		;; r14
	ystore	YMM_TMPS[5*32], ymm1		;; Save imag odd #5/.951				; 20+1
	vsubpd	ymm1, ymm10, ymm4		;; r8-r14						; 18-22
	yfmaddpd ymm10, ymm10, ymm5, ymm4	;; r8+r14						; 18-22

	yloop_optional_early_prefetch

	vsubpd	ymm4, ymm9, ymm12		;; Real odd-cols row #6 (final real #6)			; 19-23
	yfmaddpd ymm9, ymm9, ymm5, ymm12	;; Real odd-cols row #1 (final real #1A) 		; 19-23

	vmovapd	ymm12, [srcreg+2*d2+d1]		;; r6
	ystore	YMM_TMPS[6*32], ymm4		;; Real #6						; 24
	vmovapd	ymm4, [srcreg+2*d2+d1+32]	;; r16
	ystore	[srcreg], ymm9			;; Final real #1A					; 24+1
	vaddpd	ymm9, ymm12, ymm4		;; r6+r16						; 20-24
	yfmsubpd ymm12, ymm12, ymm5, ymm4	;; r6-r16						; 20-24

	vaddpd	ymm4, ymm7, ymm3		;; (r2-r20)+(r10-r12)					; 21-25
	yfmsubpd ymm7, ymm7, ymm5, ymm3		;; (r2-r20)-(r10-r12)					; 21-25
	L1prefetchw srcreg+d2+L1pd, L1pt

	vaddpd	ymm3, ymm6, ymm13		;; (r2+r20)+(r10+r12)					; 22-26
	yfmsubpd ymm6, ymm6, ymm5, ymm13	;; (r2+r20)-(r10+r12)					; 22-26
	L1prefetchw srcreg+d2+d1+L1pd, L1pt

	vaddpd	ymm13, ymm11, ymm1		;; (r4-r18)+(r8-r14)					; 23-27
	yfmsubpd ymm11, ymm11, ymm5, ymm1	;; (r4-r18)-(r8-r14)					; 23-27
	L1prefetchw srcreg+2*d2+L1pd, L1pt

	vaddpd	ymm1, ymm8, ymm10		;; (r4+r18)+(r8+r14)					; 24-28
	yfmsubpd ymm8, ymm8, ymm5, ymm10	;; (r4+r18)-(r8+r14)					; 24-28
	L1prefetchw srcreg+2*d2+d1+L1pd, L1pt

	vsubpd	ymm10, ymm0, ymm2		;; Real odd-cols row #5					; 25-29
	yfmaddpd ymm0, ymm0, ymm5, ymm2		;; Real odd-cols row #2					; 25-29
	L1prefetchw srcreg+3*d2+L1pd, L1pt

	vaddpd	ymm2, ymm4, ymm12		;; ((r2-r20)+(r10-r12))+(r6-r16))	 		; 26-30
	yfmaddpd ymm5, ymm14, ymm4, ymm12	;; .309((r2-r20)+(r10-r12)+(r6-r16))			; 26-30
	L1prefetchw srcreg+3*d2+d1+L1pd, L1pt

	yloop_optional_early_prefetch

	yfmsubpd ymm4, ymm15, ymm4, ymm12	;; .809((r2-r20)+(r10-r12))-(r6-r16)			; 27-31

	yfmaddpd ymm12, ymm14, ymm3, ymm9	;; .309((r2+r20)+(r10+r12))+(r6+r16)			; 27-31
	L1prefetchw srcreg+4*d2+L1pd, L1pt
	ystore	YMM_TMPS[7*32], ymm10		;; Real odd #5						; 30
	vaddpd	ymm10, ymm3, ymm9		;; ((r2+r20)+(r10+r12))+(r6+r16)		 	; 28-32
	yfmsubpd ymm3, ymm15, ymm3, ymm9	;; .809((r2+r20)+(r10+r12))-(r6+r16)			; 28-32

	vmovapd	ymm9, YMM_P951
	vmulpd	ymm7, ymm7, ymm9		;; ((r2-r20)-(r10-r12))*.951				; 29-33
	vmulpd	ymm11, ymm11, ymm9		;; ((r4-r18)-(r8-r14))*.951				; 29-33

	vmovapd	ymm9, YMM_P588_P951
	ystore	YMM_TMPS[8*32], ymm0		;; Real odd #2						; 30+1
	yfmaddpd ymm0, ymm9, ymm8, ymm6		;; ((r2+r20)-(r10+r12))+.588/.951((r4+r18)-(r8+r14)) (re2/.951)		; 30-34
	yfmsubpd ymm8, ymm9, ymm6, ymm8		;; .588/.951((r2+r20)-(r10+r12))-((r4+r18)-(r8+r14)) (re4/.951)		; 30-34
	vmovapd	ymm6, YMM_TMPS[8*32]		;; Real odd-cols row #2

	vsubpd	ymm2, ymm2, ymm13		;; ((r2-r20)+(r10-r12))-((r4-r18)+(r8-r14))+(r6-r16) (I6)		; 31-35
	yfmaddpd ymm5, ymm15, ymm13, ymm5	;; .309((r2-r20)+(r10-r12))+.809((r4-r18)+(r8-r14))+(r6-r16) (ie2)	; 31-35
	L1prefetchw srcreg+4*d2+d1+L1pd, L1pt
	yfmaddpd ymm4, ymm14, ymm13, ymm4	;; .809((r2-r20)+(r10-r12))+.309((r4-r18)+(r8-r14))-(r6-r16) (ie4)	; 32-36
	vmovapd	ymm13, YMM_P951

	yfnmaddpd ymm12, ymm15, ymm1, ymm12	;; .309((r2+r20)+(r10+r12))-.809((r4+r18)+(r8+r14))+(r6+r16) (re5)	; 32-36
	vaddpd	ymm10, ymm10, ymm1		;; ((r2+r20)+(r10+r12))+((r4+r18)+(r8+r14))+(r6+r16) (re1)		; 33-37
	yfnmaddpd ymm3, ymm14, ymm1, ymm3	;; .809((r2+r20)+(r10+r12))-.309((r4+r18)+(r8+r14))-(r6+r16) (re3)	; 33-37
	vmovapd	ymm1, YMM_TMPS[4*32]		;; Imag odd-cols row #2/.951

	yfnmaddpd ymm14, ymm9, ymm11, ymm7	;; ((r2-r20)-(r10-r12))*.951-.588/.951((r4-r18)-(r8-r14))*.951 (ie5)	; 34-38
	yfmaddpd ymm11, ymm9, ymm7, ymm11	;; .588/.951((r2-r20)-(r10-r12))*.951+((r4-r18)-(r8-r14))*.951 (ie3)	; 34-38
	vmovapd	ymm15, YMM_ONE

	;; Now combine the even and odd columns then do the post-multiply by twiddle factors.

	yfnmaddpd ymm7, ymm13, ymm0, ymm6	;; Real #10 (odd - even * .951)				; 35-39
	yfmaddpd ymm0, ymm13, ymm0, ymm6	;; Real #2 (odd + even * .951)				; 35-39
	vmovapd	ymm9, YMM_TMPS[7*32]		;; Real odd-cols row #5

	yfnmaddpd ymm6, ymm13, ymm1, ymm5	;; Imag #10 (even - odd * .951)				; 36-40
	yfmaddpd ymm1, ymm13, ymm1, ymm5	;; Imag #2 (even + odd * .951)				; 36-40
	vmovapd	ymm5, YMM_TMPS[5*32]		;; Imag odd-cols row #5/.951
	ystore	YMM_TMPS[11*32], ymm8		;; Real even #4/.951					; 35

	vsubpd	ymm8, ymm9, ymm12		;; Real #7 (odd - even)					; 37-41
	yfmaddpd ymm9, ymm9, ymm15, ymm12	;; Real #5 (odd + even)					; 37-41
	vmovapd	ymm12, YMM_TMPS[1*32]		;; Imag odd 34A/.951
	ystore	YMM_TMPS[9*32], ymm2		;; Imag #6						; 36

	yfnmaddpd ymm2, ymm13, ymm5, ymm14	;; Imag #7 (even - odd * .951)				; 38-42
	yfmaddpd ymm5, ymm13, ymm5, ymm14	;; Imag #5 (even + odd * .951)				; 38-42
	vmovapd	ymm14, YMM_TMPS[0*32]		;; Imag odd 34B/.951

	vaddpd	ymm13, ymm12, ymm14		;; Imag odd-cols row #3/.951				; 39-43
	yfmsubpd ymm12, ymm12, ymm15, ymm14	;; Imag odd-cols row #4/.951				; 39-43
	vmovapd	ymm14, YMM_TMPS[2*32]		;; Real odd 34A
	ystore	YMM_TMPS[10*32], ymm4		;; Imag even #4						; 37

	vmovapd	ymm4, YMM_TMPS[3*32]		;; Real odd 34B
	ystore	[srcreg+32], ymm10		;; Save final real #1B (real even-cols row #1)		; 38
	vsubpd	ymm10, ymm14, ymm4		;; Real odd-cols row #4					; 40-44
	yfmaddpd ymm14, ymm14, ymm15, ymm4	;; Real odd-cols row #3					; 40-44

	yloop_optional_early_prefetch

	vmovapd	ymm15, [screg+8*64+32]		;; cosine/sine for R10/I10
	yfmsubpd ymm4, ymm7, ymm15, ymm6	;; A10 = R10 * cosine/sine - I10			; 41-45
	yfmaddpd ymm6, ymm6, ymm15, ymm7	;; B10 = I10 * cosine/sine + R10			; 41-45

	vmovapd	ymm15, [screg+32]		;; cosine/sine for R2/I2
	yfmsubpd ymm7, ymm0, ymm15, ymm1	;; A2 = R2 * cosine/sine - I2				; 42-46
	yfmaddpd ymm1, ymm1, ymm15, ymm0	;; B2 = I2 * cosine/sine + R2				; 42-46

	vmovapd	ymm15, [screg+5*64+32]		;; cosine/sine for R7/I7
	yfmsubpd ymm0, ymm8, ymm15, ymm2	;; A7 = R7 * cosine/sine - I7				; 43-47
	yfmaddpd ymm2, ymm2, ymm15, ymm8	;; B7 = I7 * cosine/sine + R7				; 43-47

	vmovapd	ymm15, [screg+3*64+32]		;; cosine/sine for R5/I5
	yfmsubpd ymm8, ymm9, ymm15, ymm5	;; A5 = R5 * cosine/sine - I5				; 44-48
	yfmaddpd ymm5, ymm5, ymm15, ymm9	;; B5 = I5 * cosine/sine + R5				; 44-48

	vmovapd	ymm15, YMM_P951
	yfnmaddpd ymm9, ymm15, ymm13, ymm11	;; Imag #9 (even - odd * .951)				; 45-49
	yfmaddpd ymm13, ymm15, ymm13, ymm11	;; Imag #3 (even + odd * .951)				; 45-49

	vmovapd ymm11, [screg+8*64]		;; sine for R10/I10
	vmulpd	ymm4, ymm4, ymm11		;; A10 = A10 * sine (final R10)				; 46-50
	vmulpd	ymm6, ymm6, ymm11		;; B10 = B10 * sine (final I10)				; 46-50

	vmovapd	ymm15, YMM_ONE
	vsubpd	ymm11, ymm14, ymm3		;; Real #9 (odd - even)					; 47-51
	yfmaddpd ymm14, ymm14, ymm15, ymm3	;; Real #3 (odd + even)					; 47-51

	vmovapd ymm3, [screg]			;; sine for R2/I2
	vmulpd	ymm7, ymm7, ymm3		;; A2 = A2 * sine (final R2)				; 48-52
	vmulpd	ymm1, ymm1, ymm3		;; B2 = B2 * sine (final I2)				; 48-52

	vmovapd	ymm3, YMM_TMPS[11*32]		;; Real even-cols row #4/.951
	ystore	[srcreg+4*d2+d1], ymm4		;; Save final R10					; 51
	vmovapd	ymm4, YMM_P951
	ystore	[srcreg+4*d2+d1+32], ymm6	;; Save final I10					; 51+1
	yfnmaddpd ymm6, ymm4, ymm3, ymm10	;; Real #8 (odd - even * .951)				; 49-53
	yfmaddpd ymm3, ymm4, ymm3, ymm10	;; Real #4 (odd + even * .951)				; 49-53

	vmovapd	ymm10, [screg+5*64]		;; sine for R7/I7
	vmulpd	ymm0, ymm0, ymm10		;; A7 = A7 * sine (final R7)				; 50-54
	vmulpd	ymm2, ymm2, ymm10		;; B7 = B7 * sine (final I7)				; 50-54

	vmovapd	ymm10, YMM_TMPS[10*32]		;; Imag even-cols row #4
	ystore	[srcreg+d1], ymm7		;; Save final R2					; 53
	yfnmaddpd ymm7, ymm4, ymm12, ymm10	;; Imag #8 (even - odd * .951)				; 51-55
	yfmaddpd ymm12, ymm4, ymm12, ymm10	;; Imag #4 (even + odd * .951)				; 51-55

	vmovapd	ymm10, [screg+3*64]		;; sine for R5/I5
	vmulpd	ymm8, ymm8, ymm10		;; A5 = A5 * sine (final R5)				; 52-56
	vmulpd	ymm5, ymm5, ymm10		;; B5 = B5 * sine (final I5)				; 52-56

	vmovapd	ymm10, [screg+7*64+32]		;; cosine/sine for R9/I9
	yfmsubpd ymm4, ymm11, ymm10, ymm9	;; A9 = R9 * cosine/sine - I9				; 53-57
	yfmaddpd ymm9, ymm9, ymm10, ymm11	;; B9 = I9 * cosine/sine + R9				; 53-57

	vmovapd	ymm10, [screg+64+32]		;; cosine/sine for R3/I3
	yfmsubpd ymm11, ymm14, ymm10, ymm13	;; A3 = R3 * cosine/sine - I3				; 54-58
	yfmaddpd ymm13, ymm13, ymm10, ymm14	;; B3 = I3 * cosine/sine + R3				; 54-58

	vmovapd	ymm10, [screg+4*64+32]		;; cosine/sine for R6/I6
	vmovapd	ymm14, YMM_TMPS[6*32]		;; R6
	ystore	[srcreg+d1+32], ymm1		;; Save final I2					; 53+1
	vmovapd	ymm1, YMM_TMPS[9*32]		;; I6
	ystore	[srcreg+3*d2], ymm0		;; Save final R7					; 55
	yfmsubpd ymm0, ymm14, ymm10, ymm1	;; A6 = R6 * cosine/sine - I6				; 55-59
	yfmaddpd ymm1, ymm1, ymm10, ymm14	;; B6 = I6 * cosine/sine + R6				; 55-59

	vmovapd	ymm10, [screg+6*64+32]		;; cosine/sine for R8/I8
	yfmsubpd ymm14, ymm6, ymm10, ymm7	;; A8 = R8 * cosine/sine - I8				; 56-60
	yfmaddpd ymm7, ymm7, ymm10, ymm6	;; B8 = I8 * cosine/sine + R8				; 56-60
	vmovapd	ymm10, [screg+2*64+32]		;; cosine/sine for R4/I4
	ystore	[srcreg+3*d2+32], ymm2		;; Save final I7					; 55+1

	yfmsubpd ymm6, ymm3, ymm10, ymm12	;; A4 = R4 * cosine/sine - I4				; 57-61
	yfmaddpd ymm12, ymm12, ymm10, ymm3	;; B4 = I4 * cosine/sine + R4				; 57-61
	vmovapd	ymm10, [screg+7*64]		;; sine for R9/I9
	ystore	[srcreg+2*d2], ymm8		;; Save final R5					; 57

	vmulpd	ymm4, ymm4, ymm10		;; A9 = A9 * sine (final R9)				; 58-62
	vmulpd	ymm9, ymm9, ymm10		;; B9 = B9 * sine (final I9)				; 58-62
	vmovapd	ymm10, [screg+64]		;; sine for R3/I3
	ystore	[srcreg+2*d2+32], ymm5		;; Save final I5					; 57+1

	vmulpd	ymm11, ymm11, ymm10		;; A3 = A3 * sine (final R3)				; 59-63
	vmulpd	ymm13, ymm13, ymm10		;; B3 = B3 * sine (final I3)				; 59-63
	vmovapd ymm10, [screg+4*64]		;; sine for R6/I6

	vmulpd	ymm0, ymm0, ymm10		;; A6 = A6 * sine (final R6)				; 60-64
	vmulpd	ymm1, ymm1, ymm10		;; B6 = B6 * sine (final I6)				; 60-64
	vmovapd	ymm10, [screg+6*64]		;; sine for R8/I8

	vmulpd	ymm14, ymm14, ymm10		;; A8 = A8 * sine (final R8)				; 61-65
	vmulpd	ymm7, ymm7, ymm10		;; B8 = B8 * sine (final I8)				; 61-65
	vmovapd	ymm10, [screg+2*64]		;; sine for R4/I4

	vmulpd	ymm6, ymm6, ymm10		;; A4 = A4 * sine (final R4)				; 62-66
	vmulpd	ymm12, ymm12, ymm10		;; B4 = B4 * sine (final I4)				; 62-66

	ystore	[srcreg+4*d2], ymm4		;; Save final R9					; 63
	ystore	[srcreg+4*d2+32], ymm9		;; Save final I9					; 63+1
	ystore	[srcreg+d2], ymm11		;; Save final R3					; 64+1
	ystore	[srcreg+d2+32], ymm13		;; Save final I3					; 64+2
	ystore	[srcreg+2*d2+d1], ymm0		;; Save final R6					; 65+2
	ystore	[srcreg+2*d2+d1+32], ymm1	;; Save final I6					; 65+3
	ystore	[srcreg+3*d2+d1], ymm14		;; Save final R8					; 66+3
	ystore	[srcreg+3*d2+d1+32], ymm7	;; Save final I8					; 66+4
	ystore	[srcreg+d2+d1], ymm6		;; Save final R4					; 67+4
	ystore	[srcreg+d2+d1+32], ymm12	;; Save final I4					; 67+5

	bump	srcreg, srcinc
	bump	screg, scinc
	ENDM

ENDIF

ENDIF

;;
;; ************************************* 20-reals-last-unfft variants ******************************************
;;

;; These macros produce 20 reals after doing 4.32 levels of the inverse FFT applying
;; the sin/cos multipliers beforehand.  The input is 2 real and 9 complex numbers.

;; To calculate a 20-reals inverse FFT, we calculate 20 real values from 20 complex inputs in a brute force way.
;; First we note that the 20 complex values are computed from the 9 complex and 2 real inputs using Hermetian symmetry, thusly:
;; c1 = r1A + 0*i
;; c2 = r2 + i2*i
;; ...
;; c10 = r10 + i10*i
;; c11 = r1B + 0*i
;; c12 = r10 - i10*i
;; ...
;; c20 = r2 - i2*i 
;;
;; The brute force calculations are:
;;
;; c1 + c2 + ... + c20	*  w^-0000000000...
;; c1 + c2 + ... + c20	*  w^-0123456789A...
;; c1 + c2 + ... + c20	*  w^-02468ACE....
;;    ...
;; c1 + c2 + ... + c20	*  w^-...A987654321
;;
;; The sin/cos values (w = 20th root of unity) are:
;; w^-1 = .951 - .309i
;; w^-2 = .809 - .588i
;; w^-3 = .588 - .809i
;; w^-4 = .309 - .951i
;; w^-5 = 0 - 1i
;; w^-6 = -.309 - .951i
;; w^-7 = -.588 - .809i
;; w^-8 = -.809 - .588i
;; w^-9 = -.951 - .309i
;; w^-10 = -1
;;
;; Applying the sin/cos values above, taking advantage of symmetry, and ignoring a lot of multiplies by 2:
;; r1     +(r2+r10)     +(r3+r9)     +(r4+r8)     +(r5+r7) + r6 + r11
;; r1 +.951(r2-r10) +.809(r3-r9) +.588(r4-r8) +.309(r5-r7)      - r11 +.309(i2+i10) +.588*(i3+i9) +.809(i4+i8) +.951(i5+i7) + i6
;; r1 +.809(r2+r10) +.309(r3+r9) -.309(r4+r8) -.809(r5+r7) - r6 + r11 +.588(i2-i10) +.951*(i3-i9) +.951(i4-i8) +.588(i5-i7)
;; r1 +.588(r2-r10) -.309(r3-r9) -.951(r4-r8) -.809(r5-r7)      - r11 +.809(i2+i10) +.951*(i3+i9) +.309(i4+i8) -.588(i5+i7) - i6
;; r1 +.309(r2+r10) -.809(r3+r9) -.809(r4+r8) +.309(r5+r7) + r6 + r11 +.951(i2-i10) +.588*(i3-i9) -.588(i4-i8) -.951(i5-i7)
;; r1                   -(r3-r9)                  +(r5-r7)      - r11     +(i2+i10)                   -(i4+i8)              + i6
;; r1 -.309(r2+r10) -.809(r3+r9) +.809(r4+r8) +.309(r5+r7) - r6 + r11 +.951(i2-i10) -.588*(i3-i9) -.588(i4-i8) +.951(i5-i7)
;; r1 -.588(r2-r10) -.309(r3-r9) +.951(r4-r8) -.809(r5-r7)      - r11 +.809(i2+i10) -.951*(i3+i9) +.309(i4+i8) +.588(i5+i7) - i6
;; r1 -.809(r2+r10) +.309(r3+r9) +.309(r4+r8) -.809(r5+r7) + r6 + r11 +.588(i2-i10) -.951*(i3-i9) +.951(i4-i8) -.588(i5-i7)
;; r1 -.951(r2-r10) +.809(r3-r9) -.588(r4-r8) +.309(r5-r7)      - r11 +.309(i2+i10) -.588*(i3+i9) +.809(i4+i8) -.951(i5+i7) + i6
;; r1     -(r2+r10)     +(r3+r9)     -(r4+r8)     +(r5+r7) - r6 + r11
;; r1 -.951(r2-r10) +.809(r3-r9) -.588(r4-r8) +.309(r5-r7)      - r11 -.309(i2+i10) +.588*(i3+i9) -.809(i4+i8) +.951(i5+i7)
;; ... r13 thru r20 are the same as r8 through r1 but with the sign of the imaginary component changed.
;;
;; Also remember that due to the funny way we do things reals input r1A = r1+r11 and r1B = r1-11

yr5_10cl_20_reals_unfft_preload MACRO
	ENDM

yr5_10cl_20_reals_unfft MACRO srcreg,srcinc,d1,d2,screg,scinc,maxrpt,L1pt,L1pd

;; Apply the 9 twiddle factors.  Also do the first +/- on the reals so that
;; we have a perfect balance of adds and multiplies.

	vmovapd	ymm0, [screg+32]		;; cosine/sine
	vmovapd	ymm1, [srcreg+d1]		;; R2
	vmulpd	ymm2, ymm1, ymm0		;; A2 = R2 * cosine/sine
	vmovapd	ymm4, [screg+8*64+32]		;; cosine/sine
	vmovapd	ymm5, [srcreg+4*d2+d1]		;; R10
	vmulpd	ymm6, ymm5, ymm4		;; A10 = R10 * cosine/sine
	vmovapd	ymm3, [srcreg+d1+32]		;; I2
	vaddpd	ymm2, ymm2, ymm3		;; A2 = A2 + I2
	vmulpd	ymm3, ymm3, ymm0		;; B2 = I2 * cosine/sine
	vmovapd	ymm7, [srcreg+4*d2+d1+32]	;; I10
	vaddpd	ymm6, ymm6, ymm7		;; A10 = A10 + I10
	vmulpd	ymm7, ymm7, ymm4		;; B10 = I10 * cosine/sine
	vsubpd	ymm3, ymm3, ymm1		;; B2 = B2 - R2
	vmovapd	ymm1, [screg]			;; sine
	vmulpd	ymm2, ymm2, ymm1		;; R2 = A2 * sine
	vsubpd	ymm7, ymm7, ymm5		;; B10 = B10 - R10
	vmovapd	ymm5, [screg+8*64]		;; sine
	vmulpd	ymm6, ymm6, ymm5		;; R10 = A10 * sine
	vmulpd	ymm3, ymm3, ymm1		;; I2 = B2 * sine
	vmulpd	ymm7, ymm7, ymm5		;; I10 = B10 * sine
	L1prefetchw srcreg+L1pd, L1pt
	vaddpd	ymm1, ymm2, ymm6		;; R2+R10
	vsubpd	ymm2, ymm2, ymm6		;; R2-R10
	vaddpd	ymm5, ymm3, ymm7		;; I2+I10
	vsubpd	ymm3, ymm3, ymm7		;; I2-I10
	ystore	YMM_TMPS[16*32], ymm1		;; Save R2+R10
	ystore	YMM_TMPS[0*32], ymm2		;; Save R2-R10
	ystore	YMM_TMPS[17*32], ymm5		;; Save I2+I10
	ystore	YMM_TMPS[1*32], ymm3		;; Save I2-I10

	vmovapd	ymm0, [screg+64+32]		;; cosine/sine
	vmovapd	ymm1, [srcreg+d2]		;; R3
	vmulpd	ymm2, ymm1, ymm0		;; A3 = R3 * cosine/sine
	vmovapd	ymm4, [screg+7*64+32]		;; cosine/sine
	vmovapd	ymm5, [srcreg+4*d2]		;; R9
	vmulpd	ymm6, ymm5, ymm4		;; A9 = R9 * cosine/sine
	vmovapd	ymm3, [srcreg+d2+32]		;; I3
	vaddpd	ymm2, ymm2, ymm3		;; A3 = A3 + I3
	vmulpd	ymm3, ymm3, ymm0		;; B3 = I3 * cosine/sine
	vmovapd	ymm7, [srcreg+4*d2+32]		;; I9
	vaddpd	ymm6, ymm6, ymm7		;; A9 = A9 + I9
	vmulpd	ymm7, ymm7, ymm4		;; B9 = I9 * cosine/sine
	vsubpd	ymm3, ymm3, ymm1		;; B3 = B3 - R3
	vmovapd	ymm1, [screg+64]		;; sine
	vmulpd	ymm2, ymm2, ymm1		;; R3 = A3 * sine
	vsubpd	ymm7, ymm7, ymm5		;; B9 = B9 - R9
	vmovapd	ymm5, [screg+7*64]		;; sine
	vmulpd	ymm6, ymm6, ymm5		;; R9 = A9 * sine
	vmulpd	ymm3, ymm3, ymm1		;; I3 = B3 * sine
	vmulpd	ymm7, ymm7, ymm5		;; I9 = B9 * sine
	L1prefetchw srcreg+d1+L1pd, L1pt
	vaddpd	ymm1, ymm2, ymm6		;; R3+R9
	vsubpd	ymm2, ymm2, ymm6		;; R3-R9
	vaddpd	ymm5, ymm3, ymm7		;; I3+I9
	vsubpd	ymm3, ymm3, ymm7		;; I3-I9
	ystore	YMM_TMPS[14*32], ymm1		;; Save R3+R9
	ystore	YMM_TMPS[2*32], ymm2		;; Save R3-R9
	ystore	YMM_TMPS[15*32], ymm5		;; Save I3+I9
	ystore	YMM_TMPS[3*32], ymm3		;; Save I3-I9

	vmovapd	ymm0, [screg+2*64+32]		;; cosine/sine
	vmovapd	ymm1, [srcreg+d2+d1]		;; R4
	vmulpd	ymm2, ymm1, ymm0		;; A4 = R4 * cosine/sine
	vmovapd	ymm4, [screg+6*64+32]		;; cosine/sine
	vmovapd	ymm5, [srcreg+3*d2+d1]		;; R8
	vmulpd	ymm6, ymm5, ymm4		;; A8 = R8 * cosine/sine
	vmovapd	ymm3, [srcreg+d2+d1+32]		;; I4
	vaddpd	ymm2, ymm2, ymm3		;; A4 = A4 + I4
	vmulpd	ymm3, ymm3, ymm0		;; B4 = I4 * cosine/sine
	vmovapd	ymm7, [srcreg+3*d2+d1+32]	;; I8
	vaddpd	ymm6, ymm6, ymm7		;; A8 = A8 + I8
	vmulpd	ymm7, ymm7, ymm4		;; B8 = I8 * cosine/sine
	vsubpd	ymm3, ymm3, ymm1		;; B4 = B4 - R4
	vmovapd	ymm1, [screg+2*64]		;; sine
	vmulpd	ymm2, ymm2, ymm1		;; R4 = A4 * sine
	vsubpd	ymm7, ymm7, ymm5		;; B8 = B8 - R8
	vmovapd	ymm5, [screg+6*64]		;; sine
	vmulpd	ymm6, ymm6, ymm5		;; R8 = A8 * sine
	vmulpd	ymm3, ymm3, ymm1		;; I4 = B4 * sine
	vmulpd	ymm7, ymm7, ymm5		;; I8 = B8 * sine
	L1prefetchw srcreg+d2+L1pd, L1pt
	vaddpd	ymm1, ymm2, ymm6		;; R4+R8
	vsubpd	ymm2, ymm2, ymm6		;; R4-R8
	vaddpd	ymm5, ymm3, ymm7		;; I4+I8
	vsubpd	ymm3, ymm3, ymm7		;; I4-I8
	ystore	YMM_TMPS[12*32], ymm1		;; Save R4+R8
	ystore	YMM_TMPS[4*32], ymm2		;; Save R4-R8
	ystore	YMM_TMPS[13*32], ymm5		;; Save I4+I8
	ystore	YMM_TMPS[5*32], ymm3		;; Save I4-I8

	vmovapd	ymm0, [screg+3*64+32]		;; cosine/sine
	vmovapd	ymm1, [srcreg+2*d2]		;; R5
	vmulpd	ymm2, ymm1, ymm0		;; A5 = R5 * cosine/sine
	vmovapd	ymm4, [screg+5*64+32]		;; cosine/sine
	vmovapd	ymm5, [srcreg+3*d2]		;; R7
	vmulpd	ymm6, ymm5, ymm4		;; A7 = R7 * cosine/sine
	vmovapd	ymm3, [srcreg+2*d2+32]		;; I5
	vaddpd	ymm2, ymm2, ymm3		;; A5 = A5 + I5
	vmulpd	ymm3, ymm3, ymm0		;; B5 = I5 * cosine/sine
	vmovapd	ymm7, [srcreg+3*d2+32]		;; I7
	vaddpd	ymm6, ymm6, ymm7		;; A7 = A7 + I7
	vmulpd	ymm7, ymm7, ymm4		;; B7 = I7 * cosine/sine
	vsubpd	ymm3, ymm3, ymm1		;; B5 = B5 - R5
	vmovapd	ymm1, [screg+3*64]		;; sine
	vmulpd	ymm2, ymm2, ymm1		;; R5 = A5 * sine
	vsubpd	ymm7, ymm7, ymm5		;; B7 = B7 - R7
	vmovapd	ymm5, [screg+5*64]		;; sine
	vmulpd	ymm6, ymm6, ymm5		;; R7 = A7 * sine
	vmulpd	ymm3, ymm3, ymm1		;; I5 = B5 * sine
	vmulpd	ymm7, ymm7, ymm5		;; I7 = B7 * sine
	L1prefetchw srcreg+d2+d1+L1pd, L1pt
	vaddpd	ymm1, ymm2, ymm6		;; R5+R7
	vsubpd	ymm2, ymm2, ymm6		;; R5-R7
	vaddpd	ymm5, ymm3, ymm7		;; I5+I7
	vsubpd	ymm3, ymm3, ymm7		;; I5-I7
	ystore	YMM_TMPS[10*32], ymm1		;; Save R5+R7
	ystore	YMM_TMPS[6*32], ymm2		;; Save R5-R7
	ystore	YMM_TMPS[11*32], ymm5		;; Save I5+I7
	ystore	YMM_TMPS[7*32], ymm3		;; Save I5-I7

	vmovapd	ymm0, [screg+4*64+32]		;; cosine/sine
	vmovapd	ymm1, [srcreg+2*d2+d1]		;; R6
	vmulpd	ymm2, ymm1, ymm0		;; A6 = R6 * cosine/sine
	vmovapd	ymm3, [srcreg+2*d2+d1+32]	;; I6
	vaddpd	ymm2, ymm2, ymm3		;; A6 = A6 + I6
	vmulpd	ymm3, ymm3, ymm0		;; B6 = I6 * cosine/sine
	vsubpd	ymm3, ymm3, ymm1		;; B6 = B6 - R6
	vmovapd	ymm1, [screg+4*64]		;; sine
	vmulpd	ymm2, ymm2, ymm1		;; R6 = A6 * sine
	vmulpd	ymm3, ymm3, ymm1		;; I6 = B6 * sine
	ystore	YMM_TMPS[8*32], ymm2		;; Save R6
	ystore	YMM_TMPS[9*32], ymm3		;; Save I6

	;; Calculate even columns derived from real inputs (even rows)

	vmovapd	ymm0, YMM_TMPS[0*32]		;; r2-r10
	vmovapd	ymm6, YMM_P951
	vmulpd	ymm4, ymm6, ymm0		;; .951(r2-r10)
	vmovapd	ymm7, YMM_P588
	vmulpd	ymm5, ymm7, ymm0		;; .588(r2-r10)

	yloop_optional_early_prefetch

	vmovapd	ymm0, YMM_TMPS[4*32]		;; r4-r8
	vmulpd	ymm7, ymm7, ymm0		;; .588(r4-r8)
	vaddpd	ymm4, ymm4, ymm7		;; .951(r2-r10)+.588(r4-r8)
	vmulpd	ymm6, ymm6, ymm0		;; .951(r4-r8)
	vsubpd	ymm5, ymm5, ymm6		;; .588(r2-r10)-.951(r4-r8)

	;; Calculate odd columns derived from real inputs (even rows)

	vmovapd	ymm0, YMM_TMPS[2*32]		;; r3-r9
	vmovapd	ymm2, YMM_P809
	vmulpd	ymm6, ymm2, ymm0		;; .809(r3-r9)
	vmovapd	ymm7, [srcreg+32]		;; r1-r11
	vaddpd	ymm6, ymm7, ymm6		;; r1+.809(r3-r9)-r11
	vmovapd	ymm3, YMM_P309
	vmulpd	ymm1, ymm3, ymm0		;; .309(r3-r9)
	vsubpd	ymm0, ymm0, ymm7		;; -(r1-(r3-r9)-r11)
	vsubpd	ymm7, ymm7, ymm1		;; r1-.309(r3-r9)-r11

	yloop_optional_early_prefetch

	vmovapd	ymm1, YMM_TMPS[6*32]		;; r5-r7
	vsubpd	ymm0, ymm1, ymm0		;; r1-(r3-r9)+(r5-r7)-r11
	ystore	[srcreg+2*d2+d1], ymm0		;; Save odd-real-cols row #6 (also is real-cols row #6)
	vmulpd	ymm0, ymm3, ymm1		;; .309(r5-r7)
	vaddpd	ymm6, ymm6, ymm0		;; r1+.809(r3-r9)+.309(r5-r7)-r11
	vmulpd	ymm1, ymm2, ymm1		;; .809(r5-r7)
	vsubpd	ymm7, ymm7, ymm1		;; r1-.309(r3-r9)-.809(r5-r7)-r11

	L1prefetchw srcreg+2*d2+L1pd, L1pt

	;; Combine even and odd columns (even rows)

	vaddpd	ymm0, ymm6, ymm4		;; real-cols row #2 (odd#2 + even#2)
	vsubpd	ymm6, ymm6, ymm4		;; real-cols row #10 (odd#2 - even#2)
	vaddpd	ymm1, ymm7, ymm5		;; real-cols row #4 (odd#4 + even#4)
	vsubpd	ymm7, ymm7, ymm5		;; real-cols row #8 (odd#4 - even#4)

	ystore	YMM_TMPS[0*32], ymm0		;; Save real-cols row #2
	ystore	YMM_TMPS[2*32], ymm6		;; Save real-cols row #10
	ystore	YMM_TMPS[4*32], ymm1		;; Save real-cols row #4
	ystore	YMM_TMPS[6*32], ymm7		;; Save real-cols row #8

	;; Calculate even columns derived from real inputs (odd rows)

	vmovapd	ymm0, YMM_TMPS[16*32]		;; r2+r10
	vmulpd	ymm6, ymm2, ymm0		;; .809(r2+r10)
	vmulpd	ymm7, ymm3, ymm0		;; .309(r2+r10)

	vmovapd	ymm1, YMM_TMPS[12*32]		;; r4+r8
	vaddpd	ymm0, ymm0, ymm1		;; (r2+r10)+(r4+r8)
	vmulpd	ymm4, ymm3, ymm1		;; .309(r4+r8)
	vsubpd	ymm6, ymm6, ymm4		;; .809(r2+r10)-.309(r4+r8)
	vmulpd	ymm1, ymm2, ymm1		;; .809(r4+r8)
	vsubpd	ymm7, ymm7, ymm1		;; .309(r2+r10)-.809(r4+r8)

	L1prefetchw srcreg+2*d2+d1+L1pd, L1pt

	vmovapd	ymm4, YMM_TMPS[8*32]		;; r6
	vaddpd	ymm0, ymm0, ymm4		;; (r2+r10)+(r4+r8)+r6
	vsubpd	ymm6, ymm6, ymm4		;; .809(r2+r10)-.309(r4+r8)-r6
	vaddpd	ymm7, ymm7, ymm4		;; .309(r2+r10)-.809(r4+r8)+r6

	;; Calculate odd columns derived from real inputs (odd rows)
	;; From above, even-real-cols row #1,3,5 are in ymm0,ymm6,ymm7

	yloop_optional_early_prefetch

	vmovapd	ymm1, YMM_TMPS[14*32]		;; r3+r9
	vmulpd	ymm4, ymm3, ymm1		;; .309(r3+r9)
	vmulpd	ymm5, ymm2, ymm1		;; .809(r3+r9)
	vmovapd	ymm3, [srcreg]			;; r1+r11
	vaddpd	ymm1, ymm3, ymm1		;; r1+(r3+r9)+r11
	vaddpd	ymm4, ymm3, ymm4		;; r1+.309(r3+r9)+r11
	vsubpd	ymm5, ymm3, ymm5		;; r1-.809(r3+r9)+r11

	L1prefetchw srcreg+3*d2+L1pd, L1pt

	vmovapd	ymm3, YMM_TMPS[10*32]		;; r5+r7
	vaddpd	ymm1, ymm1, ymm3		;; r1+(r3+r9)+(r5+r7)+r11
	vmulpd	ymm2, ymm2, ymm3		;; .809(r5+r7)
	vsubpd	ymm4, ymm4, ymm2		;; r1+.309(r3+r9)-.809(r5+r7)+r11
	vmulpd	ymm3, ymm3, YMM_P309		;; .309(r5+r7)
	vaddpd	ymm5, ymm5, ymm3		;; r1-.809(r3+r9)+.309(r5+r7)+r11

	;; Combine even and odd columns (odd rows)

	vaddpd	ymm3, ymm1, ymm0		;; real-cols row #1 (and final R1)
	vsubpd	ymm1, ymm1, ymm0		;; real-cols row #11 (and final R11)
	ystore	[srcreg], ymm3			;; Save final R1
	ystore	[srcreg+32], ymm1		;; Save final R11

	vaddpd	ymm0, ymm4, ymm6		;; real-cols row #3
	vsubpd	ymm4, ymm4, ymm6		;; real-cols row #9
	ystore	YMM_TMPS[8*32], ymm0		;; Save real-cols row #3
	ystore	YMM_TMPS[10*32], ymm4		;; Save real-cols row #9

	vaddpd	ymm0, ymm5, ymm7		;; real-cols row #5
	vsubpd	ymm5, ymm5, ymm7 		;; real-cols row #7
	ystore	YMM_TMPS[12*32], ymm0		;; Save real-cols row #5
	ystore	YMM_TMPS[14*32], ymm5		;; Save real-cols row #7

	;; Calculate even columns derived from imaginary inputs (even rows)

	vmovapd	ymm0, YMM_TMPS[17*32]		;; i2+i10
	vmovapd	ymm2, YMM_P309
	vmulpd	ymm6, ymm2, ymm0		;; .309(i2+i10)
	vmovapd	ymm3, YMM_P809
	vmulpd	ymm7, ymm3, ymm0		;; .809(i2+i10)

	vmovapd	ymm1, YMM_TMPS[13*32]		;; i4+i8
	vsubpd	ymm0, ymm0, ymm1		;; (i2+i10)-(i4+i8)
	vmulpd	ymm4, ymm3, ymm1		;; .809(i4+i8)
	vaddpd	ymm6, ymm6, ymm4		;; .309(i2+i10)+.809(i4+i8)
	vmulpd	ymm1, ymm2, ymm1		;; .309(i4+i8)
	vaddpd	ymm7, ymm7, ymm1		;; .809(i2+i10)+.309(i4+i8)

	L1prefetchw srcreg+3*d2+d1+L1pd, L1pt

	vmovapd	ymm4, YMM_TMPS[9*32]		;; i6
	vaddpd	ymm0, ymm0, ymm4		;; (i2+i10)-(i4+i8)+i6
	vaddpd	ymm6, ymm6, ymm4		;; .309(i2+i10)+.809(i4+i8)+i6
	vsubpd	ymm7, ymm7, ymm4		;; .809(i2+i10)+.309(i4+i8)-i6

	;; Combine real and imaginary data for row #6

	vmovapd	ymm4, [srcreg+2*d2+d1]		;; Load real-cols row #6
	vaddpd	ymm1, ymm4, ymm0		;; final R6
	vsubpd	ymm4, ymm4, ymm0		;; final R16
	ystore	[srcreg+2*d2+d1], ymm1		;; Save R6
	ystore	[srcreg+2*d2+d1+32], ymm4	;; Save R16

	;; Calculate odd columns derived from imaginary inputs (even rows)
	;; From above, even-imag-cols row #2,4 are in ymm6, ymm7

	vmovapd	ymm4, YMM_TMPS[15*32]		;; i3+i9
	vmovapd	ymm1, YMM_P588
	vmulpd	ymm3, ymm1, ymm4		;; .588(i3+i9)
	vmovapd	ymm2, YMM_P951
	vmulpd	ymm4, ymm2, ymm4		;; .951(i3+i9)

	vmovapd	ymm0, YMM_TMPS[11*32]		;; i5+i7
	vmulpd	ymm5, ymm2, ymm0		;; .951(i5+i7)
	vaddpd	ymm3, ymm3, ymm5		;; .588(i3+i9)+.951(i5+i7)
	vmulpd	ymm0, ymm1, ymm0		;; .588(i5+i7)
	vsubpd	ymm4, ymm4, ymm0		;; .951(i3+i9)-.588(i5+i7)

	;; Combine even and odd columns, then real and imag data (even rows)

	L1prefetchw srcreg+4*d2+L1pd, L1pt

	vsubpd	ymm0, ymm6, ymm3		;; imag-cols row #10 (even#2 - odd#2)
	vaddpd	ymm6, ymm6, ymm3		;; imag-cols row #2 (even#2 + odd#2)
	vsubpd	ymm5, ymm7, ymm4		;; imag-cols row #8 (even#4 - odd#4)
	vaddpd	ymm7, ymm7, ymm4		;; imag-cols row #4 (even#4 + odd#4)

	vmovapd	ymm3, YMM_TMPS[2*32]		;; Load real-cols row #10
	vaddpd	ymm4, ymm3, ymm0		;; final R10
	vsubpd	ymm3, ymm3, ymm0		;; final R12
	ystore	[srcreg+4*d2+d1], ymm4		;; Save R10
	ystore	[srcreg+d1+32], ymm3		;; Save R12

	vmovapd	ymm0, YMM_TMPS[0*32]		;; Load real-cols row #2
	vaddpd	ymm4, ymm0, ymm6		;; final R2
	vsubpd	ymm0, ymm0, ymm6		;; final R20
	ystore	[srcreg+d1], ymm4		;; Save R2
	ystore	[srcreg+4*d2+d1+32], ymm0	;; Save R20

	vmovapd	ymm0, YMM_TMPS[6*32]		;; Load real-cols row #8
	vaddpd	ymm4, ymm0, ymm5		;; final R8
	vsubpd	ymm0, ymm0, ymm5		;; final R14
	ystore	[srcreg+3*d2+d1], ymm4		;; Save R8
	ystore	[srcreg+d2+d1+32], ymm0		;; Save R14

	vmovapd	ymm0, YMM_TMPS[4*32]		;; Load real-cols row #4
	vaddpd	ymm4, ymm0, ymm7		;; final R4
	vsubpd	ymm0, ymm0, ymm7		;; final R18
	ystore	[srcreg+d2+d1], ymm4		;; Save R4
	ystore	[srcreg+3*d2+d1+32], ymm0	;; Save R18

	;; Calculate even columns derived from imaginary inputs (odd rows)

	vmovapd	ymm7, YMM_TMPS[1*32]		;; i2-i10
	vmulpd	ymm6, ymm1, ymm7		;; .588(i2-i10)
	vmulpd	ymm7, ymm2, ymm7		;; .951(i2-i10)

	vmovapd	ymm4, YMM_TMPS[5*32]		;; i4-i8
	vmulpd	ymm3, ymm2, ymm4		;; .951(i4-i8)
	vaddpd	ymm6, ymm6, ymm3		;; .588(i2-i10)+.951(i4-i8)
	vmulpd	ymm4, ymm1, ymm4		;; .588(i4-i8)
	vsubpd	ymm7, ymm7, ymm4		;; .951(i2-i10)-.588(i4-i8)

	;; Calculate odd columns derived from imaginary inputs (odd rows)
	;; From above, even-imag-cols row #3,5 are in ymm6,ymm7

	vmovapd	ymm4, YMM_TMPS[3*32]		;; i3-i9
	vmulpd	ymm3, ymm2, ymm4		;; .951(i3-i9)
	vmulpd	ymm4, ymm1, ymm4		;; .588(i3-i9)

	yloop_optional_early_prefetch

	vmovapd	ymm0, YMM_TMPS[7*32]		;; i5-i7
	vmulpd	ymm1, ymm1, ymm0		;; .588(i5-i7)
	vaddpd	ymm3, ymm3, ymm1		;; .951(i3-i9)+.588(i5-i7)
	vmulpd	ymm0, ymm2, ymm0		;; .951(i5-i7)
	vsubpd	ymm4, ymm4, ymm0		;; .588(i3-i9)-.951(i5-i7)

	;; Combine even and odd columns, then real and imag data (odd rows)

	L1prefetchw srcreg+4*d2+d1+L1pd, L1pt

	vaddpd	ymm0, ymm6, ymm3		;; imag-cols row #3 (even#3 + odd#3)
	vsubpd	ymm6, ymm6, ymm3		;; imag-cols row #9 (even#3 - odd#3)
	vaddpd	ymm1, ymm7, ymm4		;; imag-cols row #5 (even#5 + odd#5)
	vsubpd	ymm7, ymm7, ymm4		;; imag-cols row #7 (even#5 - odd#5)

	vmovapd	ymm2, YMM_TMPS[8*32]		;; Load real-cols row #3
	vaddpd	ymm3, ymm2, ymm0		;; final R3
	vsubpd	ymm2, ymm2, ymm0		;; final R19
	vmovapd	ymm0, YMM_TMPS[10*32]		;; Load real-cols row #9
	vaddpd	ymm4, ymm0, ymm6		;; final R9
	vsubpd	ymm0, ymm0, ymm6		;; final R13
	ystore	[srcreg+d2], ymm3		;; Save R3
	ystore	[srcreg+4*d2+32], ymm2		;; Save R19
	ystore	[srcreg+4*d2], ymm4		;; Save R9
	ystore	[srcreg+d2+32], ymm0		;; Save R13

	vmovapd	ymm5, YMM_TMPS[12*32]		;; Load real-cols row #5
	vaddpd	ymm3, ymm5, ymm1		;; final R5
	vsubpd	ymm5, ymm5, ymm1		;; final R17
	vmovapd	ymm6, YMM_TMPS[14*32]		;; Load real-cols row #7
	vaddpd	ymm1, ymm6, ymm7		;; final R7
	vsubpd	ymm6, ymm6, ymm7		;; final R15
	ystore	[srcreg+2*d2], ymm3		;; Save R5
	ystore	[srcreg+3*d2+32], ymm5		;; Save R17
	ystore	[srcreg+3*d2], ymm1		;; Save R7
	ystore	[srcreg+2*d2+32], ymm6		;; Save R15

	bump	srcreg, srcinc
	bump	screg, scinc
	ENDM

;; 64-bit version

IFDEF X86_64

yr5_10cl_20_reals_unfft_preload MACRO
	vmovapd	ymm12, YMM_P309
	vmovapd	ymm13, YMM_P809
	vmovapd	ymm14, YMM_P588
	vmovapd	ymm15, YMM_P951
	ENDM

yr5_10cl_20_reals_unfft MACRO srcreg,srcinc,d1,d2,screg,scinc,maxrpt,L1pt,L1pd

	vmovapd	ymm0, [screg+32]		;; cosine/sine
	vmovapd	ymm1, [srcreg+d1]		;; R2
	vmulpd	ymm2, ymm1, ymm0		;; A2 = R2 * cosine/sine				;	1-5

	vmovapd	ymm4, [screg+8*64+32]		;; cosine/sine
	vmovapd	ymm5, [srcreg+4*d2+d1]		;; R10
	vmulpd	ymm6, ymm5, ymm4		;; A10 = R10 * cosine/sine				;	2-6

	vmovapd	ymm3, [srcreg+d1+32]		;; I2
	vmulpd	ymm0, ymm3, ymm0		;; B2 = I2 * cosine/sine				;	3-7

	vmovapd	ymm7, [srcreg+4*d2+d1+32]	;; I10
	vmulpd	ymm4, ymm7, ymm4		;; B10 = I10 * cosine/sine				;	4-8

	vmovapd	ymm8, [screg+64+32]		;; cosine/sine
	vmovapd	ymm9, [srcreg+d2]		;; R3
	vmulpd	ymm10, ymm9, ymm8		;; A3 = R3 * cosine/sine				;	5-9
	vmovapd	ymm11, [screg+7*64+32]		;; cosine/sine

	vaddpd	ymm2, ymm2, ymm3		;; A2 = A2 + I2						; 6-8
	vmovapd	ymm3, [srcreg+4*d2]		;; R9
	vaddpd	ymm6, ymm6, ymm7		;; A10 = A10 + I10					; 7-9
	vmulpd	ymm7, ymm3, ymm11		;; A9 = R9 * cosine/sine				;	6-10

	vsubpd	ymm0, ymm0, ymm1		;; B2 = B2 - R2						; 8-10
	vmovapd	ymm1, [srcreg+d2+32]		;; I3
	vmulpd	ymm8, ymm1, ymm8		;; B3 = I3 * cosine/sine				;	7-11

	vsubpd	ymm4, ymm4, ymm5		;; B10 = B10 - R10					; 9-11
	vmovapd	ymm5, [srcreg+4*d2+32]		;; I9
	vmulpd	ymm11, ymm5, ymm11		;; B9 = I9 * cosine/sine				;	8-12

	vaddpd	ymm10, ymm10, ymm1		;; A3 = A3 + I3						; 10-12
	vmovapd	ymm1, [screg]			;; sine
	vmulpd	ymm2, ymm2, ymm1		;; R2 = A2 * sine					;	9-13

	vaddpd	ymm7, ymm7, ymm5		;; A9 = A9 + I9						; 11-13
	vmovapd	ymm5, [screg+8*64]		;; sine
	vmulpd	ymm6, ymm6, ymm5		;; R10 = A10 * sine					;	10-14

	vmulpd	ymm0, ymm0, ymm1		;; I2 = B2 * sine					;	11-15
	vmovapd	ymm1, [screg+64]		;; sine

	vsubpd	ymm8, ymm8, ymm9		;; B3 = B3 - R3						; 12-14
	vmulpd	ymm4, ymm4, ymm5		;; I10 = B10 * sine					;	12-16
	vmovapd	ymm9, [screg+7*64]		;; sine

	vsubpd	ymm11, ymm11, ymm3		;; B9 = B9 - R9						; 13-15
	vmulpd	ymm10, ymm10, ymm1		;; R3 = A3 * sine					;	13-17
	vmovapd	ymm5, [screg+2*64+32]		;; cosine/sine

	vmulpd	ymm7, ymm7, ymm9		;; R9 = A9 * sine					;	14-18

	vaddpd	ymm3, ymm2, ymm6		;; R2+R10						; 15-17
	vmulpd	ymm8, ymm8, ymm1		;; I3 = B3 * sine					;	15-19
	vmovapd	ymm1, [srcreg+d2+d1]		;; R4

	vsubpd	ymm2, ymm2, ymm6		;; R2-R10						; 16-18
	vmulpd	ymm11, ymm11, ymm9		;; I9 = B9 * sine					;	16-20
	vmovapd	ymm6, [screg+6*64+32]		;; cosine/sine

	vaddpd	ymm9, ymm0, ymm4		;; I2+I10						; 17-19

	vsubpd	ymm0, ymm0, ymm4		;; I2-I10						; 18-20
	vmulpd	ymm4, ymm1, ymm5		;; A4 = R4 * cosine/sine				;	18-22
	ystore	YMM_TMPS[8*32], ymm3		;; Save R2+R10						; 18
	vmovapd	ymm3, [srcreg+3*d2+d1]		;; R8

	ystore	YMM_TMPS[0*32], ymm2		;; Save R2-R10						; 19
	vaddpd	ymm2, ymm10, ymm7		;; R3+R9						; 19-21
	ystore	YMM_TMPS[9*32], ymm9		;; Save I2+I10						; 20
	vmulpd	ymm9, ymm3, ymm6		;; A8 = R8 * cosine/sine				;	19-23

	vsubpd	ymm10, ymm10, ymm7		;; R3-R9						; 20-22
	vmovapd	ymm7, [srcreg+d2+d1+32]		;; I4
	vmulpd	ymm5, ymm7, ymm5		;; B4 = I4 * cosine/sine				;	20-24

	ystore	YMM_TMPS[1*32], ymm0		;; Save I2-I10						; 21
	vaddpd	ymm0, ymm8, ymm11		;; I3+I9						; 21-23
	ystore	YMM_TMPS[4*32], ymm2		;; Save R3+R9						; 22
	vmovapd	ymm2, [srcreg+3*d2+d1+32]	;; I8
	vmulpd	ymm6, ymm2, ymm6		;; B8 = I8 * cosine/sine				;	21-25

	L1prefetchw srcreg+L1pd, L1pt

	vsubpd	ymm8, ymm8, ymm11		;; I3-I9						; 22-24
	vmovapd	ymm11, [screg+3*64+32]		;; cosine/sine
	ystore	YMM_TMPS[2*32], ymm10		;; Save R3-R9						; 23
	vmovapd	ymm10, [srcreg+2*d2]		;; R5
	ystore	YMM_TMPS[6*32], ymm0		;; Save I3+I9						; 24
	vmulpd	ymm0, ymm10, ymm11		;; A5 = R5 * cosine/sine				;	22-26

	vaddpd	ymm4, ymm4, ymm7		;; A4 = A4 + I4						; 23-25
	vmovapd	ymm7, [screg+5*64+32]		;; cosine/sine
	vaddpd	ymm9, ymm9, ymm2		;; A8 = A8 + I8						; 24-26
	vmovapd	ymm2, [srcreg+3*d2]		;; R7
	ystore	YMM_TMPS[3*32], ymm8		;; Save I3-I9						; 25
	vmulpd	ymm8, ymm2, ymm7		;; A7 = R7 * cosine/sine				;	23-27

	vsubpd	ymm5, ymm5, ymm1		;; B4 = B4 - R4						; 25-27
	vmovapd	ymm1, [srcreg+2*d2+32]		;; I5
	vmulpd	ymm11, ymm1, ymm11		;; B5 = I5 * cosine/sine				;	24-28

	yloop_optional_early_prefetch

	vsubpd	ymm6, ymm6, ymm3		;; B8 = B8 - R8						; 26-28
	vmovapd	ymm3, [srcreg+3*d2+32]		;; I7
	vmulpd	ymm7, ymm3, ymm7		;; B7 = I7 * cosine/sine				;	25-29

	vaddpd	ymm0, ymm0, ymm1		;; A5 = A5 + I5						; 27-29
	vmovapd	ymm1, [screg+2*64]		;; sine
	vmulpd	ymm4, ymm4, ymm1		;; R4 = A4 * sine					;	26-30

	vaddpd	ymm8, ymm8, ymm3		;; A7 = A7 + I7						; 28-30
	vmovapd	ymm3, [screg+6*64]		;; sine
	vmulpd	ymm9, ymm9, ymm3		;; R8 = A8 * sine					;	27-31

	vmulpd	ymm5, ymm5, ymm1		;; I4 = B4 * sine					;	28-32
	vmovapd	ymm1, [screg+3*64]		;; sine

	vsubpd	ymm11, ymm11, ymm10		;; B5 = B5 - R5						; 29-31
	vmulpd	ymm6, ymm6, ymm3		;; I8 = B8 * sine					;	29-33
	vmovapd	ymm10, [screg+5*64]		;; sine

	vsubpd	ymm7, ymm7, ymm2		;; B7 = B7 - R7						; 30-32
	vmulpd	ymm0, ymm0, ymm1		;; R5 = A5 * sine					;	30-34
	vmovapd	ymm3, [screg+4*64+32]		;; cosine/sine

	vmulpd	ymm8, ymm8, ymm10		;; R7 = A7 * sine					;	31-35

	yloop_optional_early_prefetch

	vaddpd	ymm2, ymm4, ymm9		;; R4+R8						; 32-34
	vmulpd	ymm11, ymm11, ymm1		;; I5 = B5 * sine					;	32-36
	vmovapd	ymm1, [srcreg+2*d2+d1]		;; R6

	vsubpd	ymm4, ymm4, ymm9		;; R4-R8						; 33-35
	vmulpd	ymm7, ymm7, ymm10		;; I7 = B7 * sine					;	33-37

	L1prefetchw srcreg+d1+L1pd, L1pt

	vaddpd	ymm10, ymm5, ymm6		;; I4+I8						; 34-36
	vmulpd	ymm9, ymm1, ymm3		;; A6 = R6 * cosine/sine				;	34-38

	vsubpd	ymm5, ymm5, ymm6		;; I4-I8						; 35-37
	vmovapd	ymm6, [srcreg+2*d2+d1+32]	;; I6
	vmulpd	ymm3, ymm6, ymm3		;; B6 = I6 * cosine/sine				;	35-39
	ystore	YMM_TMPS[12*32], ymm2		;; Save R4+R8						; 35

	vaddpd	ymm2, ymm0, ymm8		;; R5+R7						; 36-38
	ystore	YMM_TMPS[13*32], ymm10		;; Save I4+I8						; 37
	vmovapd	ymm10, YMM_TMPS[0*32]		;; r2-r10
	ystore	YMM_TMPS[5*32], ymm5		;; Save I4-I8						; 38
	vmulpd	ymm5, ymm15, ymm10		;; .951(r2-r10)						;	36-40

	vsubpd	ymm0, ymm0, ymm8		;; R5-R7						; 37-39
	vmulpd	ymm10, ymm14, ymm10		;; .588(r2-r10)						;	37-41

	vaddpd	ymm8, ymm11, ymm7		;; I5+I7						; 38-40
	ystore	YMM_TMPS[10*32], ymm2		;; Save R5+R7						; 39
	vmulpd	ymm2, ymm14, ymm4		;; .588(r4-r8)						;	38-42

	L1prefetchw srcreg+d2+L1pd, L1pt

	vsubpd	ymm11, ymm11, ymm7		;; I5-I7						; 39-41
	vmulpd	ymm4, ymm15, ymm4		;; .951(r4-r8)						;	39-43
	vmovapd	ymm7, YMM_TMPS[2*32]		;; r3-r9

	yloop_optional_early_prefetch

	vaddpd	ymm9, ymm9, ymm6		;; A6 = A6 + I6						; 40-42
	vmulpd	ymm6, ymm13, ymm7		;; .809(r3-r9)						;	40-44

	vsubpd	ymm3, ymm3, ymm1		;; B6 = B6 - R6						; 41-43
	vmulpd	ymm1, ymm12, ymm7		;; .309(r3-r9)						;	41-45
	ystore	YMM_TMPS[11*32], ymm8		;; Save I5+I7						; 41
	vmovapd	ymm8, [srcreg+32]		;; r1-r11

	vsubpd	ymm7, ymm8, ymm7		;; r1-(r3-r9)-r11					; 42-44
	ystore	YMM_TMPS[7*32], ymm11		;; Save I5-I7						; 42
	vmulpd	ymm11, ymm12, ymm0		;; .309(r5-r7)						;	42-46

	vaddpd	ymm5, ymm5, ymm2		;; .951(r2-r10)+.588(r4-r8) (even#2)			; 43-45
	vmulpd	ymm2, ymm13, ymm0		;; .809(r5-r7)						;	43-47

	vsubpd	ymm10, ymm10, ymm4		;; .588(r2-r10)-.951(r4-r8) (even#4)			; 44-46
	vmovapd	ymm4, YMM_TMPS[9*32]		;; i2+i10

	vaddpd	ymm6, ymm8, ymm6		;; r1+.809(r3-r9)-r11					; 45-47

	L1prefetchw srcreg+d2+d1+L1pd, L1pt

	vsubpd	ymm8, ymm8, ymm1		;; r1-.309(r3-r9)-r11					; 46-48
	vmulpd	ymm1, ymm12, ymm4		;; .309(i2+i10)						;	44-48

	vaddpd	ymm7, ymm7, ymm0		;; r1-(r3-r9)+(r5-r7)-r11				; 47-49
	vmulpd	ymm0, ymm13, ymm4		;; .809(i2+i10)						;	45-49

	vaddpd	ymm6, ymm6, ymm11		;; r1+.809(r3-r9)+.309(r5-r7)-r11 (odd#2)		; 48-50
	vmovapd	ymm11, YMM_TMPS[13*32]		;; i4+i8

	vsubpd	ymm8, ymm8, ymm2		;; r1-.309(r3-r9)-.809(r5-r7)-r11 (odd#4)		; 49-51
	vmulpd	ymm2, ymm13, ymm11		;; .809(i4+i8)						;	46-50

	L1prefetchw srcreg+2*d2+L1pd, L1pt

	vsubpd	ymm4, ymm4, ymm11		;; (i2+i10)-(i4+i8)					; 50-52
	vmulpd	ymm11, ymm12, ymm11		;; .309(i4+i8)						;	47-51

	vaddpd	ymm1, ymm1, ymm2		;; .309(i2+i10)+.809(i4+i8)				; 51-53
	vmovapd	ymm2, [screg+4*64]		;; sine
	vmulpd	ymm3, ymm3, ymm2		;; I6 = B6 * sine					;	48-52
	vmulpd	ymm9, ymm9, ymm2		;; R6 = A6 * sine					;	49-53
	vmovapd	ymm2, YMM_TMPS[6*32]		;; i3+i9

	vaddpd	ymm0, ymm0, ymm11		;; .809(i2+i10)+.309(i4+i8)				; 52-54
	vmulpd	ymm11, ymm14, ymm2		;; .588(i3+i9)						;	50-54
	vmulpd	ymm2, ymm15, ymm2		;; .951(i3+i9)						;	51-55

	vaddpd	ymm4, ymm4, ymm3		;; (i2+i10)-(i4+i8)+i6					; 53-55

	yloop_optional_early_prefetch

	vaddpd	ymm1, ymm1, ymm3		;; .309(i2+i10)+.809(i4+i8)+i6 (even#2)			; 54-56

	vsubpd	ymm0, ymm0, ymm3		;; .809(i2+i10)+.309(i4+i8)-i6 (even#4)			; 55-57

	vaddpd	ymm3, ymm7, ymm4		;; final R6						; 56-58

	vsubpd	ymm7, ymm7, ymm4		;; final R16						; 57-59
	vmovapd	ymm4, YMM_TMPS[11*32]		;; i5+i7
	ystore	[srcreg+2*d2+d1], ymm3		;; Save R6						; 59
	vmulpd	ymm3, ymm15, ymm4		;; .951(i5+i7)						;	52-56
	vmulpd	ymm4, ymm14, ymm4		;; .588(i5+i7)						;	53-57

	L1prefetchw srcreg+2*d2+d1+L1pd, L1pt

	vaddpd	ymm11, ymm11, ymm3		;; .588(i3+i9)+.951(i5+i7) (odd#2)			; 58-60
	vmovapd	ymm3, YMM_TMPS[8*32]		;; r2+r10

	vsubpd	ymm2, ymm2, ymm4		;; .951(i3+i9)-.588(i5+i7) (odd#4)			; 59-61

	vaddpd	ymm4, ymm6, ymm5		;; real-cols row #2 (odd#2 + even#2)			; 60-62
	vsubpd	ymm6, ymm6, ymm5		;; real-cols row #10 (odd#2 - even#2)			; 61-63

	L1prefetchw srcreg+3*d2+L1pd, L1pt

	vaddpd	ymm5, ymm8, ymm10		;; real-cols row #4 (odd#4 + even#4)			; 62-64
	ystore	[srcreg+2*d2+d1+32], ymm7	;; Save R16						; 62
	vsubpd	ymm8, ymm8, ymm10		;; real-cols row #8 (odd#4 - even#4)			; 63-65

	vsubpd	ymm10, ymm1, ymm11		;; imag-cols row #10 (even#2 - odd#2)			; 64-66
	vaddpd	ymm1, ymm1, ymm11		;; imag-cols row #2 (even#2 + odd#2)			; 65-67
	vsubpd	ymm11, ymm0, ymm2		;; imag-cols row #8 (even#4 - odd#4)			; 66-68
	vaddpd	ymm0, ymm0, ymm2		;; imag-cols row #4 (even#4 + odd#4)			; 67-69

	L1prefetchw srcreg+3*d2+d1+L1pd, L1pt

	vaddpd	ymm2, ymm6, ymm10		;; final R10						; 68-70
	vsubpd	ymm6, ymm6, ymm10		;; final R12						; 69-71

	vaddpd	ymm10, ymm4, ymm1		;; final R2						; 70-72
	vmulpd	ymm7, ymm13, ymm3		;; .809(r2+r10)						;	70-74

	vsubpd	ymm4, ymm4, ymm1		;; final R20						; 71-73
	vmulpd	ymm1, ymm12, ymm3		;; .309(r2+r10)						;	71-75
	ystore	[srcreg+4*d2+d1], ymm2		;; Save R10						; 71

	L1prefetchw srcreg+4*d2+L1pd, L1pt

	vaddpd	ymm2, ymm8, ymm11		;; final R8						; 72-74
	ystore	[srcreg+d1+32], ymm6		;; Save R12						; 72
	vmovapd	ymm6, YMM_TMPS[12*32]		;; r4+r8
	ystore	[srcreg+d1], ymm10		;; Save R2						; 73
	vmulpd	ymm10, ymm12, ymm6		;; .309(r4+r8)						;	72-76

	vsubpd	ymm8, ymm8, ymm11		;; final R14						; 73-75
	vmulpd	ymm11, ymm13, ymm6		;; .809(r4+r8)						;	73-77

	ystore	[srcreg+4*d2+d1+32], ymm4	;; Save R20						; 74
	vaddpd	ymm4, ymm5, ymm0		;; final R4						; 74-76
	ystore	[srcreg+3*d2+d1], ymm2		;; Save R8						; 75
	vmovapd	ymm2, YMM_TMPS[4*32]		;; r3+r9
	ystore	[srcreg+d2+d1+32], ymm8		;; Save R14						; 76
	vmulpd	ymm8, ymm12, ymm2		;; .309(r3+r9)						;	74-78

	vsubpd	ymm5, ymm5, ymm0		;; final R18						; 75-77
	vmulpd	ymm0, ymm13, ymm2		;; .809(r3+r9)						;	75-79

	vaddpd	ymm3, ymm3, ymm6		;; (r2+r10)+(r4+r8)					; 76-78
	vmovapd	ymm6, YMM_TMPS[10*32]		;; r5+r7
	ystore	[srcreg+d2+d1], ymm4		;; Save R4						; 77
	vmulpd	ymm4, ymm13, ymm6		;; .809(r5+r7)						;	76-80

	vsubpd	ymm7, ymm7, ymm10		;; .809(r2+r10)-.309(r4+r8)				; 77-79
	vmulpd	ymm10, ymm12, ymm6		;; .309(r5+r7)						;	77-81

	vsubpd	ymm1, ymm1, ymm11		;; .309(r2+r10)-.809(r4+r8)				; 78-80
	vmovapd	ymm11, [srcreg]			;; r1+r11
	ystore	[srcreg+3*d2+d1+32], ymm5	;; Save R18						; 78
	vmovapd	ymm5, YMM_TMPS[1*32]		;; i2-i10

	vaddpd	ymm3, ymm3, ymm9		;; (r2+r10)+(r4+r8)+r6					; 79-81
	vsubpd	ymm7, ymm7, ymm9		;; .809(r2+r10)-.309(r4+r8)-r6				; 80-82
	vaddpd	ymm1, ymm1, ymm9		;; .309(r2+r10)-.809(r4+r8)+r6				; 81-83

	vaddpd	ymm2, ymm11, ymm2		;; r1+(r3+r9)+r11					; 82-84

	vaddpd	ymm8, ymm11, ymm8		;; r1+.309(r3+r9)+r11					; 83-85
	vmulpd	ymm9, ymm14, ymm5		;; .588(i2-i10)						;	83-87

	vsubpd	ymm11, ymm11, ymm0		;; r1-.809(r3+r9)+r11					; 84-86
	vmulpd	ymm5, ymm15, ymm5		;; .951(i2-i10)						;	84-88
	vmovapd	ymm0, YMM_TMPS[5*32]		;; i4-i8

	vaddpd	ymm2, ymm2, ymm6		;; r1+(r3+r9)+(r5+r7)+r11				; 85-87
	vmulpd	ymm6, ymm15, ymm0		;; .951(i4-i8)						;	85-89

	vsubpd	ymm8, ymm8, ymm4		;; r1+.309(r3+r9)-.809(r5+r7)+r11			; 86-88
	vmulpd	ymm0, ymm14, ymm0		;; .588(i4-i8)						;	86-90
	vmovapd	ymm4, YMM_TMPS[3*32]		;; i3-i9

	vaddpd	ymm11, ymm11, ymm10		;; r1-.809(r3+r9)+.309(r5+r7)+r11			; 87-89

	vaddpd	ymm10, ymm2, ymm3		;; real-cols row #1 (and final R1)			; 88-90
	ystore	[srcreg], ymm10			;; Save final R1					; 91
	vmulpd	ymm10, ymm15, ymm4		;; .951(i3-i9)						;	87-91
	vmulpd	ymm4, ymm14, ymm4		;; .588(i3-i9)						;	88-92

	vsubpd	ymm2, ymm2, ymm3		;; real-cols row #11 (and final R11)			; 89-91
	vmovapd	ymm3, YMM_TMPS[7*32]		;; i5-i7
	ystore	[srcreg+32], ymm2		;; Save final R11					; 92
	vmulpd	ymm2, ymm14, ymm3		;; .588(i5-i7)						;	89-93

	vaddpd	ymm9, ymm9, ymm6		;; .588(i2-i10)+.951(i4-i8)				; 90-92
	vmulpd	ymm3, ymm15, ymm3		;; .951(i5-i7)						;	90-94

	vsubpd	ymm5, ymm5, ymm0		;; .951(i2-i10)-.588(i4-i8)				; 91-93

	vaddpd	ymm0, ymm8, ymm7		;; real-cols row #3					; 92-94
	vsubpd	ymm8, ymm8, ymm7		;; real-cols row #9					; 93-95

	vaddpd	ymm10, ymm10, ymm2		;; .951(i3-i9)+.588(i5-i7)				; 94-96
	vsubpd	ymm4, ymm4, ymm3		;; .588(i3-i9)-.951(i5-i7)				; 95-97

	L1prefetchw srcreg+4*d2+d1+L1pd, L1pt

	vaddpd	ymm3, ymm11, ymm1		;; real-cols row #5					; 96-98
	vsubpd	ymm11, ymm11, ymm1 		;; real-cols row #7					; 97-99

	vaddpd	ymm1, ymm9, ymm10		;; imag-cols row #3 (even#3 + odd#3)			; 98-100
	vsubpd	ymm9, ymm9, ymm10		;; imag-cols row #9 (even#3 - odd#3)			; 99-101
	vaddpd	ymm10, ymm5, ymm4		;; imag-cols row #5 (even#5 + odd#5)			; 100-102
	vsubpd	ymm5, ymm5, ymm4		;; imag-cols row #7 (even#5 - odd#5)			; 101-103

	vaddpd	ymm4, ymm0, ymm1		;; final R3						; 102-104
	vsubpd	ymm0, ymm0, ymm1		;; final R19						; 103-105
	vaddpd	ymm1, ymm8, ymm9		;; final R9						; 104-106
	vsubpd	ymm8, ymm8, ymm9		;; final R13						; 105-107
	ystore	[srcreg+d2], ymm4		;; Save R3						; 105

	vaddpd	ymm2, ymm3, ymm10		;; final R5						; 106-108
	ystore	[srcreg+4*d2+32], ymm0		;; Save R19						; 106

	vsubpd	ymm3, ymm3, ymm10		;; final R17						; 107-109
	ystore	[srcreg+4*d2], ymm1		;; Save R9						; 107

	vaddpd	ymm7, ymm11, ymm5		;; final R7						; 108-110
	ystore	[srcreg+d2+32], ymm8		;; Save R13						; 108

	vsubpd	ymm11, ymm11, ymm5		;; final R15						; 109-111
	ystore	[srcreg+2*d2], ymm2		;; Save R5						; 109
	ystore	[srcreg+3*d2+32], ymm3		;; Save R17						; 110
	ystore	[srcreg+3*d2], ymm7		;; Save R7						; 111
	ystore	[srcreg+2*d2+32], ymm11		;; Save R15						; 112

	bump	srcreg, srcinc
	bump	screg, scinc
	ENDM

;; Haswell FMA3 version

IF (@INSTR(,%yarch,<FMA3>) NE 0)

yr5_10cl_20_reals_unfft_preload MACRO
	ENDM

;; Timed at 65.5 clocks.  That's 8.5 clocks worse than theoretically possible.  Converting vaddpd and vsubpd to FMA3 did not seem to help.
yr5_10cl_20_reals_unfft MACRO srcreg,srcinc,d1,d2,screg,scinc,maxrpt,L1pt,L1pd
	vmovapd	ymm3, [screg+32]		;; cosine/sine for R2/I2
	vmovapd	ymm2, [srcreg+d1]		;; R2
	vmovapd	ymm1, [srcreg+d1+32]		;; I2
	yfmaddpd ymm0, ymm2, ymm3, ymm1		;; A2 = R2 * cosine/sine + I2				; 1-5
	yfmsubpd ymm1, ymm1, ymm3, ymm2		;; B2 = I2 * cosine/sine - R2				; 1-5

	vmovapd	ymm5, [screg+64+32]		;; cosine/sine for R3/I3
	vmovapd	ymm4, [srcreg+d2]		;; R3
	vmovapd	ymm3, [srcreg+d2+32]		;; I3
	yfmaddpd ymm2, ymm4, ymm5, ymm3		;; A3 = R3 * cosine/sine + I3				; 2-6
	yfmsubpd ymm3, ymm3, ymm5, ymm4		;; B3 = I3 * cosine/sine - R3				; 2-6

	vmovapd	ymm7, [screg+2*64+32]		;; cosine/sine for R4/I4
	vmovapd	ymm6, [srcreg+d2+d1]		;; R4
	vmovapd	ymm5, [srcreg+d2+d1+32]		;; I4
	yfmaddpd ymm4, ymm6, ymm7, ymm5		;; A4 = R4 * cosine/sine + I4				; 3-7
	yfmsubpd ymm5, ymm5, ymm7, ymm6		;; B4 = I4 * cosine/sine - R4				; 3-7

	vmovapd	ymm9, [screg+3*64+32]		;; cosine/sine for R5/I5
	vmovapd	ymm8, [srcreg+2*d2]		;; R5
	vmovapd	ymm7, [srcreg+2*d2+32]		;; I5
	yfmaddpd ymm6, ymm8, ymm9, ymm7		;; A5 = R5 * cosine/sine + I5				; 4-8
	yfmsubpd ymm7, ymm7, ymm9, ymm8		;; B5 = I5 * cosine/sine - R5				; 4-8

	vmovapd	ymm11, [screg+8*64+32]		;; cosine/sine for R10/I10
	vmovapd	ymm10, [srcreg+4*d2+d1]		;; R10
	vmovapd	ymm9, [srcreg+4*d2+d1+32]	;; I10
	yfmaddpd ymm8, ymm10, ymm11, ymm9	;; A10 = R10 * cosine/sine + I10			; 5-9
	yfmsubpd ymm9, ymm9, ymm11, ymm10	;; B10 = I10 * cosine/sine - R10			; 5-9

	vmovapd	ymm10, [screg]			;; sine for R2/I2
	vmulpd	ymm0, ymm0, ymm10		;; R2 = A2 * sine					; 6-10
	vmulpd	ymm1, ymm1, ymm10		;; I2 = B2 * sine					; 6-10

	vmovapd	ymm13, [screg+7*64+32]		;; cosine/sine for R9/I9
	vmovapd	ymm12, [srcreg+4*d2]		;; R9
	vmovapd	ymm11, [srcreg+4*d2+32]		;; I9
	yfmaddpd ymm10, ymm12, ymm13, ymm11	;; A9 = R9 * cosine/sine + I9				; 7-11		n 17
	yfmsubpd ymm11, ymm11, ymm13, ymm12	;; B9 = I9 * cosine/sine - R9				; 7-11

	vmovapd	ymm12, [screg+64]		;; sine for R3/I3
	vmulpd	ymm2, ymm2, ymm12		;; R3 = A3 * sine					; 8-12		n 17
	vmulpd	ymm3, ymm3, ymm12		;; I3 = B3 * sine					; 8-12

	vmovapd	ymm12, [screg+2*64]		;; sine for R4/I4
	vmulpd	ymm4, ymm4, ymm12		;; R4 = A4 * sine					; 9-13		n 19
	vmulpd	ymm5, ymm5, ymm12		;; I4 = B4 * sine					; 9-13

	vmovapd	ymm15, [screg+4*64+32]		;; cosine/sine for R6/I6
	vmovapd	ymm14, [srcreg+2*d2+d1]		;; R6
	vmovapd	ymm13, [srcreg+2*d2+d1+32]	;; I6
	yfmaddpd ymm12, ymm14, ymm15, ymm13	;; A6 = R6 * cosine/sine + I6				; 10-14		n 15
	yfmsubpd ymm13, ymm13, ymm15, ymm14	;; B6 = I6 * cosine/sine - R6				; 10-14

	vmovapd	ymm14, [screg+8*64]		;; sine for R10/I10
	yfmaddpd ymm15, ymm8, ymm14, ymm0	;; R2+(R10 = A10 * sine)				; 11-15		n 22
	yfnmaddpd ymm8, ymm8, ymm14, ymm0	;; R2-(R10 = A10 * sine)				; 11-15

	yfnmaddpd ymm0, ymm9, ymm14, ymm1	;; I2-(I10 = B10 * sine)				; 12-16		n 25
	yfmaddpd ymm9, ymm9, ymm14, ymm1	;; I2+(I10 = B10 * sine)				; 12-16

	vmovapd	ymm14, [screg+6*64+32]		;; cosine/sine for R8/I8
	vmovapd	ymm1, [srcreg+3*d2+d1]		;; R8
	ystore	YMM_TMPS[0*32], ymm15		;; Save R2+R10						; 16
	vmovapd	ymm15, [srcreg+3*d2+d1+32]	;; I8
	ystore	YMM_TMPS[1*32], ymm8		;; Save R2-R10						; 16+1
	yfmsubpd ymm8, ymm15, ymm14, ymm1	;; B8 = I8 * cosine/sine - R8				; 13-17		n 19
	yfmaddpd ymm1, ymm1, ymm14, ymm15	;; A8 = R8 * cosine/sine + I8				; 13-17

	vmovapd	ymm14, [screg+3*64]		;; sine for R5/I5
	vmulpd	ymm6, ymm6, ymm14		;; R5 = A5 * sine					; 14-18		n 21		
	vmulpd	ymm7, ymm7, ymm14		;; I5 = B5 * sine					; 14-18

	vmovapd	ymm15, [screg+4*64]		;; sine for R6/I6
	vmulpd	ymm12, ymm12, ymm15		;; R6 = A6 * sine					; 15-19		n 22
	vmulpd	ymm13, ymm13, ymm15		;; I6 = B6 * sine					; 15-19

	vmovapd	ymm14, [screg+5*64+32]		;; cosine/sine for R7/I7
	vmovapd	ymm15, [srcreg+3*d2]		;; R7
	ystore	YMM_TMPS[2*32], ymm9		;; Save I2+I10						; 17+1
	vmovapd	ymm9, [srcreg+3*d2+32]		;; I7
	ystore	YMM_TMPS[3*32], ymm13		;; Save I6						; 20
	yfmsubpd ymm13, ymm9, ymm14, ymm15	;; B7 = I7 * cosine/sine - R7				; 16-20		n 21
	yfmaddpd ymm15, ymm15, ymm14, ymm9	;; A7 = R7 * cosine/sine + I7				; 16-20

	vmovapd	ymm14, [screg+7*64]		;; sine for R9/I9
	yfmaddpd ymm9, ymm10, ymm14, ymm2	;; R3+(R9 = A9 * sine)					; 17-21		n 23
	yfnmaddpd ymm10, ymm10, ymm14, ymm2	;; R3-(R9 = A9 * sine)					; 17-21

	yloop_optional_early_prefetch

	yfnmaddpd ymm2, ymm11, ymm14, ymm3	;; I3-(I9 = B9 * sine)					; 18-22
	yfmaddpd ymm11, ymm11, ymm14, ymm3	;; I3+(I9 = B9 * sine)					; 18-22

	vmovapd	ymm14, [screg+6*64]		;; sine for R8/I8
	yfnmaddpd ymm3, ymm8, ymm14, ymm5	;; I4-(I8 = B8 * sine)					; 19-23		n 24
	yfmaddpd ymm8, ymm8, ymm14, ymm5	;; I4+(I8 = B8 * sine)					; 19-23

	yfmaddpd ymm5, ymm1, ymm14, ymm4	;; R4+(R8 = A8 * sine)					; 20-24		n 26
	yfnmaddpd ymm1, ymm1, ymm14, ymm4	;; R4-(R8 = A8 * sine)					; 20-24

	vmovapd	ymm14, [screg+5*64]		;; sine for R7/I7
	yfnmaddpd ymm4, ymm13, ymm14, ymm7	;; I5-(I7 = B7 * sine)					; 21-25		n 26
	ystore	YMM_TMPS[4*32], ymm10		;; Save R3-R9						; 22
	yfmaddpd ymm10, ymm15, ymm14, ymm6	;; R5+(R7 = A7 * sine)					; 21-25

	yfnmaddpd ymm15, ymm15, ymm14, ymm6	;; R5-(R7 = A7 * sine)					; 22-26
	vmovapd	ymm14, YMM_P809
	vmovapd	ymm6, YMM_TMPS[0*32]		;; r2+r10
	ystore	YMM_TMPS[5*32], ymm11		;; Save I3+I9						; 23
	yfmsubpd ymm11, ymm14, ymm6, ymm12	;; .809(r2+r10)-r6					; 22-26

	ystore	YMM_TMPS[6*32], ymm8		;; Save I4+I8						; 24
	yfmaddpd ymm8, ymm6, YMM_P309, ymm12	;; .309(r2+r10)+r6					; 23-27
	ystore	YMM_TMPS[7*32], ymm1		;; Save R4-R8						; 25
	vmovapd	ymm1, [srcreg]			;; r1+r11
	ystore	YMM_TMPS[8*32], ymm15		;; Save R5-R7						; 27
	yfmaddpd ymm15, ymm9, YMM_P309, ymm1	;; r1+.309(r3+r9)+r11					; 23-27

	yloop_optional_early_prefetch

	vaddpd	ymm6, ymm6, ymm12		;; (r2+r10)+r6						; 24-28
	yfnmaddpd ymm12, ymm14, ymm9, ymm1	;; r1-.809(r3+r9)+r11					; 24-28

	vaddpd	ymm1, ymm1, ymm9		;; r1+(r3+r9)+r11					; 25-29
	vmovapd	ymm14, YMM_P588_P951
	yfmaddpd ymm9, ymm14, ymm0, ymm3	;; .588/.951(i2-i10)+(i4-i8) (imag even#3 / .951)	; 25-29		n 31

	yfnmaddpd ymm3, ymm14, ymm3, ymm0	;; (i2-i10)-.588/.951(i4-i8) (imag even#5 / .951)	; 26-30		n 32
	yfmaddpd ymm0, ymm14, ymm4, ymm2	;; (i3-i9)+.588/.951(i5-i7) (imag odd#3 / .951)		; 26-30		n 31

	yfmsubpd ymm2, ymm14, ymm2, ymm4	;; .588/.951(i3-i9)-(i5-i7) (imag odd#5 / .951)		; 27-31		n 32
	vmovapd	ymm14, YMM_P309
	yfnmaddpd ymm11, ymm14, ymm5, ymm11	;; .809(r2+r10)-.309(r4+r8)-r6 (r39b)			; 27-31		n 33

	vmovapd	ymm4, YMM_P809
	yfnmaddpd ymm8, ymm4, ymm5, ymm8	;; .309(r2+r10)-.809(r4+r8)+r6 (r57b)			; 28-32		n 34
	yfnmaddpd ymm15, ymm4, ymm10, ymm15	;; r1+.309(r3+r9)-.809(r5+r7)+r11 (r39a)		; 28-32		n 33

	vaddpd	ymm6, ymm6, ymm5		;; (r2+r10)+(r4+r8)+r6 (r111b)				; 29-33		n 35
	yfmaddpd ymm12, ymm14, ymm10, ymm12	;; r1-.809(r3+r9)+.309(r5+r7)+r11 (r57a)		; 29-33		n 34

	vaddpd	ymm1, ymm1, ymm10		;; r1+(r3+r9)+(r5+r7)+r11 (r111a)			; 30-34		n 35
	yfmaddpd ymm13, ymm13, [screg+5*64], ymm7 ;; I5+(I7 = B7 * sine)				; 30-34

	vmovapd	ymm5, YMM_ONE
	vaddpd	ymm10, ymm9, ymm0		;; imag-cols row #3 / .951 (even#3 + odd#3)		; 31-35
	yfmsubpd ymm9, ymm9, ymm5, ymm0		;; imag-cols row #9 / .951 (even#3 - odd#3)		; 31-35

	yloop_optional_early_prefetch

	vaddpd	ymm0, ymm3, ymm2		;; imag-cols row #5 / .951 (even#5 + odd#5)		; 32-36
	yfmsubpd ymm3, ymm3, ymm5, ymm2		;; imag-cols row #7 / .951 (even#5 - odd#5)		; 32-36
	vmovapd	ymm7, YMM_P951

	vaddpd	ymm2, ymm15, ymm11		;; real-cols row #3					; 33-37
	yfmsubpd ymm15, ymm15, ymm5, ymm11	;; real-cols row #9					; 33-37
	L1prefetchw srcreg+L1pd, L1pt

	vaddpd	ymm11, ymm12, ymm8		;; real-cols row #5					; 34-38
	yfmsubpd ymm12, ymm12, ymm5, ymm8	;; real-cols row #7					; 34-38
	L1prefetchw srcreg+d1+L1pd, L1pt

	vaddpd	ymm8, ymm1, ymm6		;; real-cols row #1 (and final R1)			; 35-39
	yfmsubpd ymm1, ymm1, ymm5, ymm6		;; real-cols row #11 (and final R11)			; 35-39
	vmovapd	ymm6, [srcreg+32]		;; r1-r11

	yfmaddpd ymm5, ymm10, ymm7, ymm2	;; final R3 (real#3 + imag#3 * .951)			; 38-42
	yfnmaddpd ymm10, ymm10, ymm7, ymm2	;; final R19 (real#3 - imag#3 * .951)			; 38-42

	vmovapd	ymm2, YMM_TMPS[4*32]		;; r3-r9
	ystore	[srcreg], ymm8			;; Save final R1					; 40
	yfmaddpd ymm8, ymm4, ymm2, ymm6		;; r1+.809(r3-r9)-r11					; 36-40		n 44
	ystore	[srcreg+32], ymm1		;; Save final R11					; 40+1
	vsubpd	ymm1, ymm6, ymm2		;; r1-(r3-r9)-r11					; 36-40		n 45

	yfnmaddpd ymm2, ymm14, ymm2, ymm6	;; r1-.309(r3-r9)-r11					; 37-41		n 45

	yloop_optional_early_prefetch

	yfmaddpd ymm6, ymm9, ymm7, ymm15	;; final R9 (real#9 + imag#9 * .951)			; 39-43
	yfnmaddpd ymm9, ymm9, ymm7, ymm15	;; final R13 (real#9 - imag#9 * .951)			; 39-43

	vmovapd	ymm15, YMM_TMPS[2*32]		;; i2+i10
	ystore	[srcreg+d2], ymm5		;; Save R3						; 43
	vmovapd	ymm5, YMM_TMPS[3*32]		;; i6
	ystore	[srcreg+4*d2+32], ymm10		;; Save R19						; 43+1
	yfmaddpd ymm10, ymm14, ymm15, ymm5	;; .309(i2+i10)+i6					; 37-41		n 45

	ystore	[srcreg+4*d2], ymm6		;; Save R9						; 44+1
	vaddpd	ymm6, ymm15, ymm5		;; (i2+i10)+i6						; 40-44		n 46
	yfmsubpd ymm15, ymm4, ymm15, ymm5	;; .809(i2+i10)-i6					; 40-44		n 46

	yfmaddpd ymm5, ymm0, ymm7, ymm11	;; final R5 (real#5 + imag#5 * .951)			; 41-45
	yfnmaddpd ymm0, ymm0, ymm7, ymm11	;; final R17 (real#5 - imag#5 * .951)			; 41-45

	vmovapd	ymm7, YMM_TMPS[1*32]		;; r2-r10
	vmovapd	ymm11, YMM_TMPS[7*32]		;; r4-r8
	ystore	[srcreg+d2+32], ymm9		;; Save R13						; 44+2
	vmovapd	ymm9, YMM_P588_P951
	ystore	[srcreg+2*d2], ymm5		;; Save R5						; 46+1
	yfmaddpd ymm5, ymm9, ymm11, ymm7	;; (r2-r10)+.588/.951(r4-r8) (real even#2 / .951)	; 42-46		n 49
	yfmsubpd ymm7, ymm9, ymm7, ymm11	;; .588/.951(r2-r10)-(r4-r8) (real even#4 / .951)	; 42-46		n 51

	vmovapd	ymm11, YMM_TMPS[5*32]		;; i3+i9
	ystore	[srcreg+3*d2+32], ymm0		;; Save R17						; 46+2
	yfmaddpd ymm0, ymm9, ymm11, ymm13	;; .588/.951(i3+i9)+(i5+i7) (imag odd#2 / .951)		; 43-47		n 49
	yfnmaddpd ymm13, ymm9, ymm13, ymm11	;; (i3+i9)-.588/.951(i5+i7) (imag odd#4 / .951)		; 43-47		n 51

	vmovapd	ymm9, YMM_TMPS[8*32]		;; r5-r7
	vmovapd	ymm11, YMM_TMPS[6*32]		;; i4+i8
	yfmaddpd ymm8, ymm14, ymm9, ymm8	;; r1+.809(r3-r9)+.309(r5-r7)-r11 (real odd#2)		; 44-48		n 49
	yfmaddpd ymm10, ymm4, ymm11, ymm10	;; .309(i2+i10)+.809(i4+i8)+i6 (imag even#2)		; 44-48		n 49

	vaddpd	ymm1, ymm1, ymm9		;; r1-(r3-r9)+(r5-r7)-r11 (real odd#6)			; 45-49	 	n 53
	yfnmaddpd ymm2, ymm4, ymm9, ymm2	;; r1-.309(r3-r9)-.809(r5-r7)-r11 (real odd#4)		; 45-49		n 51
	vmovapd	ymm9, YMM_P951

	vsubpd	ymm6, ymm6, ymm11		;; (i2+i10)-(i4+i8)+i6 (imag even#6)			; 46-50		n 53
	yfmaddpd ymm15, ymm14, ymm11, ymm15	;; .809(i2+i10)+.309(i4+i8)-i6 (imag even#4)		; 46-50		n 51
	L1prefetchw srcreg+d2+L1pd, L1pt

	yfmaddpd ymm11, ymm3, ymm9, ymm12	;; final R7 (real#7 + imag#7 * .951)			; 47-51
	yfnmaddpd ymm3, ymm3, ymm9, ymm12	;; final R15 (real#7 - imag#7 * .951)			; 47-51
	vmovapd	ymm12, YMM_ONE

	L1prefetchw srcreg+d2+d1+L1pd, L1pt								; 48 -- stall

	yfmaddpd ymm4, ymm5, ymm9, ymm8		;; real-cols row #2 (odd#2 + even#2 * .951)		; 49-53
	yfmaddpd ymm14, ymm0, ymm9, ymm10	;; imag-cols row #2 (even#2 + odd#2 * .951)		; 49-53
	L1prefetchw srcreg+2*d2+L1pd, L1pt

	yfnmaddpd ymm5, ymm5, ymm9, ymm8	;; real-cols row #10 (odd#2 - even#2 * .951)		; 50-54
	yfnmaddpd ymm0, ymm0, ymm9, ymm10	;; imag-cols row #10 (even#2 - odd#2 * .951)		; 50-54
	L1prefetchw srcreg+2*d2+d1+L1pd, L1pt

	yfmaddpd ymm8, ymm7, ymm9, ymm2		;; real-cols row #4 (odd#4 + even#4 * .951)		; 51-55
	yfmaddpd ymm10, ymm13, ymm9, ymm15	;; imag-cols row #4 (even#4 + odd#4 * .951)		; 51-55
	L1prefetchw srcreg+3*d2+L1pd, L1pt

	yfnmaddpd ymm7, ymm7, ymm9, ymm2	;; real-cols row #8 (odd#4 - even#4 * .951)		; 52-56
	yfnmaddpd ymm13, ymm13, ymm9, ymm15	;; imag-cols row #8 (even#4 - odd#4 * .951)		; 52-56
	ystore	[srcreg+3*d2], ymm11		;; Save R7						; 52

	vaddpd	ymm2, ymm1, ymm6		;; final R6 (odd#6 + even#6)				; 53-57
	yfmsubpd ymm1, ymm1, ymm12, ymm6	;; final R16 (odd#6 - even#6)				; 53-57
	ystore	[srcreg+2*d2+32], ymm3		;; Save R15						; 52+1

	vaddpd	ymm6, ymm4, ymm14		;; final R2 (real#2 + imag#2)				; 54-58
	yfmsubpd ymm4, ymm4, ymm12, ymm14	;; final R20 (real#2 - imag#2)				; 54-58
	L1prefetchw srcreg+3*d2+d1+L1pd, L1pt

	vaddpd	ymm14, ymm5, ymm0		;; final R10 (real#10 + imag#10)			; 55-59
	yfmsubpd ymm5, ymm5, ymm12, ymm0	;; final R12 (real#10 - imag#10)			; 55-59
	L1prefetchw srcreg+4*d2+L1pd, L1pt

	vaddpd	ymm0, ymm8, ymm10		;; final R4 (real#4 + imag#4)				; 56-60
	yfmsubpd ymm8, ymm8, ymm12, ymm10	;; final R18 (real#4 - imag#4)				; 56-60
	L1prefetchw srcreg+4*d2+d1+L1pd, L1pt

	vaddpd	ymm10, ymm7, ymm13		;; final R8 (real#8 + imag#8)				; 57-61
	yfmsubpd ymm7, ymm7, ymm12, ymm13	;; final R14 (real#8 - imag#8)				; 57-61

	ystore	[srcreg+2*d2+d1], ymm2		;; Save R6						; 58
	ystore	[srcreg+2*d2+d1+32], ymm1	;; Save R16						; 58+1
	ystore	[srcreg+d1], ymm6		;; Save R2						; 59+1
	ystore	[srcreg+4*d2+d1+32], ymm4	;; Save R20						; 59+2
	ystore	[srcreg+4*d2+d1], ymm14		;; Save R10						; 60+2
	ystore	[srcreg+d1+32], ymm5		;; Save R12						; 60+3
	ystore	[srcreg+d2+d1], ymm0		;; Save R4						; 61+3
	ystore	[srcreg+3*d2+d1+32], ymm8	;; Save R18						; 61+4
	ystore	[srcreg+3*d2+d1], ymm10		;; Save R8						; 62+4
	ystore	[srcreg+d2+d1+32], ymm7		;; Save R14						; 62+5

	bump	srcreg, srcinc
	bump	screg, scinc
	ENDM

ENDIF

ENDIF

