; Copyright 2011-2024 - Mersenne Research, Inc.  All rights reserved
; Author:  George Woltman
; Email: woltman@alum.mit.edu
;

;; Include other macro files needed by the AVX versions of our primarily radix-4 traditional FFTs
;; Only the 64-bit macros are heavily optimized.  We figure that anyone running AVX CPUs is likely
;; to run a 64-bit operating system.

INCLUDE yr3.mac
INCLUDE yr5.mac
INCLUDE yr7.mac
INCLUDE yr8.mac

;;
;;
;; All new macros for version 27 of gwnum where we do a very traditional, primarily
;; radix-4, FFT.  The forward FFT macros multiply by the sin/cos values at the end
;; of the macro and the inverse FFTs multiply by the sin/cos values at the start of
;; the macro.  We also implement the Daniel J. Bernstein (DJB) "exponent-1" idea to
;; save sin/cos memory.
;;
;;

;; Utility macro that lets us create and call common code to reduce executable size

yr4_dispatch MACRO call_type,routine,d1,d2
IF	d1 EQ 64
IFDEF X86_64
	call_type &routine&1
ELSEIF push_amt GT 0
	call_type &routine&1
ELSE
	call_type &routine&1a
ENDIF
ELSEIF	d1 EQ 4*64
	call_type &routine&2
ELSEIF	d1 EQ 16*64
	call_type &routine&3
ELSEIF	d1 EQ 64*64+64		;;dist64
	call_type &routine&4
ELSEIF	d1 EQ 64*64+128		;;dist64
	call_type &routine&4a
ELSEIF	d1 EQ 4*(64*64+64)	;;4*dist64
	call_type &routine&5
ELSEIF	d1 EQ 16*(64*64+64)	;;16*dist64
	call_type &routine&6
ELSE
	UNSUPPORTED d1
ENDIF
	ENDM

;; Note that macros have funny abbreviations at the front.  Here is
;; how to decipher most of it (ex: yr4_rb4cl_four_complex_djbfft):
;;
;; yr4_	- all macros in this file start with this.  y=AVX, r4=radix-4
;; f - first levels (uses rbx as an offset to source)
;; e - load sin/cos data from an earlier bigger sin/cos table (only needed when using broadcast)
;; r - load sin/cos data from a slightly larger real sin/cos table
;; b - load compressed sin/cos data using broadcast instructions
;; s - swizzle the outputs (fft) or swizzle the inputs (unfft)
;; g - takes separate destination register
;; 4cl_ - operates on 4 cache lines
;; 2sc_ - uses two register pointers to retrieve the sin/cos data
;; csc_ - uses one combined sin/cos register pointer (ptr usable by four complex and eight reals)
;; rsc_ - reduced sin/cos, generates sin/cos data from two smaller sets of sin/cos data
;;

;; In forward FFTs,
;;    input values are R1+R5i, R2+R6i, R3+R7i, R4+R8i
;;    output values are R1+R2i, R3+R4i, R5+R6i, R7+R8i

;; In inverse FFTs,
;;    input values are R1+R2i, R3+R4i, R5+R6i, R7+R8i
;;    output values are R1+R5i, R2+R6i, R3+R7i, R4+R8i

;;
;; ************************************* four-complex-djbfft variants ******************************************
;;
;; Macros to do Daniel J. Bernstein's exponent-1 butterflies.  The
;; difference with a standard four-complex-fft is in the postmultiply step.
;; Instead of multiplying by w^2x, w^x, w^3x, we multiply by w^2x, w^x, w^-x.
;;

; Basic four-complex DJB FFT building block
yr4_4cl_four_complex_djbfft_preload MACRO
	yr4_4c_djbfft_cmn_preload noexec
	ENDM
yr4_4cl_four_complex_djbfft MACRO srcreg,srcinc,d1,d2,screg,scinc,maxrpt,L1pt,L1pd
	yr4_4c_djbfft_cmn srcreg,srcinc,0,d1,d2,noexec,screg,screg+64,32,noexec,screg,scinc,maxrpt,L1pt,L1pd
	ENDM

; Like the 4cl version except vbroadcastsd is used to reduce sin/cos data
yr4_b4cl_four_complex_djbfft_preload MACRO
	yr4_4c_djbfft_cmn_preload noexec
	ENDM
yr4_b4cl_four_complex_djbfft MACRO srcreg,srcinc,d1,d2,screg,scinc,maxrpt,L1pt,L1pd
	yr4_4c_djbfft_cmn srcreg,srcinc,0,d1,d2,exec,screg,screg+16,8,noexec,screg,scinc,maxrpt,L1pt,L1pd
	ENDM

; Like b4cl but extracts the sin/cos data to broadcast from an earlier level's uncompressed sin/cos data
yr4_eb4cl_four_complex_djbfft_preload MACRO
	yr4_4c_djbfft_cmn_preload noexec
	ENDM
yr4_eb4cl_four_complex_djbfft MACRO srcreg,srcinc,d1,d2,screg,scinc,maxrpt,L1pt,L1pd
	yr4_4c_djbfft_cmn srcreg,srcinc,0,d1,d2,exec,screg,screg+64,32,noexec,screg,scinc,maxrpt,L1pt,L1pd
	ENDM

; Like b4cl but extracts the sin/cos data to broadcast from the eight-real/four_complex sin/cos table
yr4_rb4cl_four_complex_djbfft_preload MACRO
	yr4_4c_djbfft_cmn_preload noexec
	ENDM
yr4_rb4cl_four_complex_djbfft MACRO srcreg,srcinc,d1,d2,screg,scinc,maxrpt,L1pt,L1pd
	yr4_4c_djbfft_cmn srcreg,srcinc,0,d1,d2,exec,screg+8,screg+80,40,noexec,screg,scinc,maxrpt,L1pt,L1pd
	ENDM

; Like 4cl but swizzles the outputs.  Used in the middle levels of pass 2.
yr4_s4cl_four_complex_djbfft_preload MACRO
	yr4_4c_djbfft_cmn_preload exec
	ENDM
yr4_s4cl_four_complex_djbfft MACRO srcreg,srcinc,d1,d2,screg,scinc,maxrpt,L1pt,L1pd
	yr4_4c_djbfft_cmn srcreg,srcinc,0,d1,d2,noexec,screg,screg+64,32,exec,screg,scinc,maxrpt,L1pt,L1pd
	ENDM

; Like s4cl but uses rbx to get input data.  Used in the first levels of pass 2.
yr4_fs4cl_four_complex_djbfft_preload MACRO
	yr4_4c_djbfft_cmn_preload exec
	ENDM
yr4_fs4cl_four_complex_djbfft MACRO srcreg,srcinc,d1,d2,screg,scinc,maxrpt,L1pt,L1pd
	yr4_4c_djbfft_cmn srcreg,srcinc,rbx,d1,d2,noexec,screg,screg+64,32,exec,screg,scinc,maxrpt,L1pt,L1pd
	ENDM

; Common macro to get the four complex djbfft job done

yr4_4c_djbfft_cmn_preload MACRO swiz
	ENDM

yr4_4c_djbfft_cmn MACRO srcreg,srcinc,srcoff,d1,d2,bcast,screg1,screg2,scoff,swiz,screg,scinc,maxrpt,L1pt,L1pd
	vmovapd	ymm0, [srcreg+srcoff]		;; R1
	vmovapd	ymm6, [srcreg+srcoff+d2]	;; R3
	vaddpd	ymm2, ymm0, ymm6		;; R1 + R3 (new R1)		; 1-3

	vmovapd	ymm1, [srcreg+srcoff+d1]	;; R2
	vmovapd	ymm7, [srcreg+srcoff+d2+d1]	;; R4
	vaddpd	ymm3, ymm1, ymm7		;; R2 + R4 (new R2)		; 2-4

	vsubpd	ymm0, ymm0, ymm6		;; R1 - R3 (new R3)		; 3-5
	vsubpd	ymm1, ymm1, ymm7		;; R2 - R4 (new R4)		; 4-6

	vmovapd	ymm4, [srcreg+srcoff+32]	;; I1
	vmovapd	ymm7, [srcreg+srcoff+d2+32]	;; I3
	vaddpd	ymm6, ymm4, ymm7		;; I1 + I3 (new I1)		; 5-7

	L1prefetchw srcreg+L1pd, L1pt

	vsubpd	ymm5, ymm2, ymm3		;; R1 - R2 (final R2)		; 6-8
	vaddpd	ymm2, ymm2, ymm3		;; R1 + R2 (final R1)		; 7-9

	ystore	[srcreg], ymm2			;; Save R1

	vmovapd	ymm2, [srcreg+srcoff+d1+32]	;; I2
	vaddpd	ymm3, ymm2, [srcreg+srcoff+d2+d1+32] ;; I2 + I4 (new I2)	; 8-10
	vsubpd	ymm2, ymm2, [srcreg+srcoff+d2+d1+32] ;; I2 - I4 (new I4)	; 9-11

	L1prefetchw srcreg+d1+L1pd, L1pt

	vsubpd	ymm4, ymm4, ymm7		;; I1 - I3 (new I3)		; 10-12

	vsubpd	ymm7, ymm6, ymm3		;; I1 - I2 (final I2)		; 11-13
	vaddpd	ymm6, ymm6, ymm3		;; I1 + I2 (final I1)		; 12-14

	ystore	[srcreg+32], ymm6		;; Save I1

	vsubpd	ymm3, ymm0, ymm2		;; R3 - I4 (final R3)		; 13-15
	vaddpd	ymm6, ymm4, ymm1		;; I3 + R4 (final I3)		; 14-16

	L1prefetchw srcreg+d2+L1pd, L1pt

	vaddpd	ymm0, ymm0, ymm2		;; R3 + I4 (final R4)		; 15-17
	vsubpd	ymm4, ymm4, ymm1		;; I3 - R4 (final I4)		; 16-18

no bcast vmovapd ymm1, [screg2+32]		;; cosine/sine
bcast	vbroadcastsd ymm1, Q [screg2+scoff]	;; cosine/sine
	vmulpd	ymm2, ymm5, ymm1		;; A2 = R2 * cosine/sine	;  9-13
	vsubpd	ymm2, ymm2, ymm7		;; A2 = A2 - I2			; 17-19

	vmulpd	ymm7, ymm7, ymm1		;; B2 = I2 * cosine/sine	; 14-18
	vaddpd	ymm7, ymm7, ymm5		;; B2 = B2 + R2			; 19-21

no bcast vmovapd ymm1, [screg1+32]		;; cosine/sine
bcast	vbroadcastsd ymm1, Q [screg1+scoff]	;; cosine/sine
	vmulpd	ymm5, ymm3, ymm1		;; A3 = R3 * cosine/sine	; 16-20
	vsubpd	ymm5, ymm5, ymm6		;; A3 = A3 - I3			; 21-23

	vmulpd	ymm6, ymm6, ymm1		;; B3 = I3 * cosine/sine	; 17-21
	vaddpd	ymm6, ymm6, ymm3		;; B3 = B3 + R3			; 22-24

	L1prefetchw srcreg+d2+d1+L1pd, L1pt

	vmulpd	ymm3, ymm0, ymm1		;; A4 = R4 * cosine/sine	; 18-22
	vaddpd	ymm3, ymm3, ymm4		;; A4 = A4 + I4			; 23-25

	vmulpd	ymm4, ymm4, ymm1		;; B4 = I4 * cosine/sine	; 19-23
	vsubpd	ymm4, ymm4, ymm0		;; B4 = B4 - R4			; 24-26

no bcast vmovapd ymm1, [screg2]			;; Sine
bcast	vbroadcastsd ymm1, Q [screg2]		;; Sine
	vmulpd	ymm2, ymm2, ymm1		;; A2 = A2 * sine (final R2)	; 20-24
	vmulpd	ymm7, ymm7, ymm1		;; B2 = B2 * sine (final I2)	; 22-26

no bcast vmovapd ymm1, [screg1]			;; Sine
bcast	vbroadcastsd ymm1, Q [screg1]		;; Sine
	vmulpd	ymm5, ymm5, ymm1		;; A3 = A3 * sine (final R3)	; 24-28
	vmulpd	ymm6, ymm6, ymm1		;; B3 = B3 * sine (final I3)	; 25-29

	vmulpd	ymm3, ymm3, ymm1		;; A4 = A4 * sine (final R4)	; 26-30
	vmulpd	ymm4, ymm4, ymm1		;; B4 = B4 * sine (final I4)	; 27-31

	;; Non-swizzling save

;;	ystore	[srcreg], ymm0			;; Save R1
;;	ystore	[srcreg+32], ymm0		;; Save I1
no swiz	ystore	[srcreg+d1], ymm2		;; Save R2
no swiz	ystore	[srcreg+d1+32], ymm7		;; Save I2
no swiz	ystore	[srcreg+d2], ymm5		;; Save R3
no swiz	ystore	[srcreg+d2+32], ymm6		;; Save I3
no swiz	ystore	[srcreg+d2+d1], ymm3		;; Save R4
no swiz	ystore	[srcreg+d2+d1+32], ymm4		;; Save I4

	;; Swizzle using 8 shufpds and 8 perm2f128  (1 or 3 clocks each on port 5, throughput 1)

swiz	vmovapd	ymm0, [srcreg]			;; Reload saved R1
swiz	vshufpd	ymm1, ymm0, ymm2, 0		;; Shuffle R1 and R2 to create R1/R2 low
swiz	vshufpd	ymm0, ymm0, ymm2, 15		;; Shuffle R1 and R2 to create R1/R2 hi

swiz	vshufpd	ymm2, ymm5, ymm3, 0		;; Shuffle R3 and R4 to create R3/R4 low
swiz	vshufpd	ymm5, ymm5, ymm3, 15		;; Shuffle R3 and R4 to create R3/R4 hi

swiz	ylow128s ymm3, ymm1, ymm2		;; Shuffle R1/R2 low and R3/R4 low (new R1)
swiz	yhigh128s ymm1, ymm1, ymm2		;; Shuffle R1/R2 low and R3/R4 low (new R3)

swiz	ylow128s ymm2, ymm0, ymm5		;; Shuffle R1/R2 hi and R3/R4 hi (new R2)
swiz	yhigh128s ymm0, ymm0, ymm5		;; Shuffle R1/R2 hi and R3/R4 hi (new R4)

swiz	vmovapd	ymm5, [srcreg+32]		;; Reload saved I1

swiz	ystore	[srcreg], ymm3			;; Save R1
swiz	ystore	[srcreg+d2], ymm1		;; Save R3
swiz	ystore	[srcreg+d1], ymm2		;; Save R2
swiz	ystore	[srcreg+d2+d1], ymm0		;; Save R4

swiz	vshufpd	ymm0, ymm5, ymm7, 0		;; Shuffle I1 and I2 to create I1/I2 low
swiz	vshufpd	ymm5, ymm5, ymm7, 15		;; Shuffle I1 and I2 to create I1/I2 hi

swiz	vshufpd	ymm7, ymm6, ymm4, 0		;; Shuffle I3 and I4 to create I3/I4 low
swiz	vshufpd	ymm6, ymm6, ymm4, 15		;; Shuffle I3 and I4 to create I3/I4 hi

swiz	ylow128s ymm4, ymm0, ymm7		;; Shuffle I1/I2 low and I3/I4 low (new I1)
swiz	yhigh128s ymm0, ymm0, ymm7		;; Shuffle I1/I2 low and I3/I4 low (new I3)

swiz	ylow128s ymm7, ymm5, ymm6		;; Shuffle I1/I2 hi and I3/I4 hi (new I2)
swiz	yhigh128s ymm5, ymm5, ymm6		;; Shuffle I1/I2 hi and I3/I4 hi (new I4)

swiz	ystore	[srcreg+32], ymm4		;; Save I1
swiz	ystore	[srcreg+d2+32], ymm0		;; Save I3
swiz	ystore	[srcreg+d1+32], ymm7		;; Save I2
swiz	ystore	[srcreg+d2+d1+32], ymm5		;; Save I4

	bump	srcreg, srcinc
	bump	screg, scinc
	ENDM

;; 64-bit version

IFDEF X86_64

yr4_4c_djbfft_cmn_preload MACRO swiz
no swiz yr4_4c_djbfft_noswiz_unroll_preload
swiz	yr4_4c_djbfft_swiz_unroll_preload
	ENDM

yr4_4c_djbfft_cmn MACRO srcreg,srcinc,srcoff,d1,d2,bcast,screg1,screg2,scoff,swiz,screg,scinc,maxrpt,L1pt,L1pd
no swiz yr4_4c_djbfft_noswiz_cmn srcreg,srcinc,srcoff,d1,d2,bcast,screg1,screg2,scoff,screg,scinc,maxrpt,L1pt,L1pd
swiz	yr4_4c_djbfft_swiz_cmn srcreg,srcinc,srcoff,d1,d2,bcast,screg1,screg2,scoff,screg,scinc,maxrpt,L1pt,L1pd
	ENDM

yr4_4c_djbfft_noswiz_cmn MACRO srcreg,srcinc,srcoff,d1,d2,bcast,screg1,screg2,scoff,screg,scinc,maxrpt,L1pt,L1pd
	IF maxrpt MOD 5 EQ 0
	yr4_4c_djbfft_noswiz_unroll -1,noexec,noexec,exec,srcreg,srcinc,srcoff,d1,d2,bcast,screg1,screg2,scoff,screg,scinc,L1pt,L1pd
	yr4_4c_djbfft_noswiz_unroll 0,noexec,exec,exec,srcreg,srcinc,srcoff,d1,d2,bcast,screg1,screg2,scoff,screg,scinc,L1pt,L1pd
	yr4_4c_djbfft_noswiz_unroll 1,exec,exec,exec,srcreg,srcinc,srcoff,d1,d2,bcast,screg1,screg2,scoff,screg,scinc,L1pt,L1pd
	yr4_4c_djbfft_noswiz_unroll 2,exec,exec,exec,srcreg,srcinc,srcoff,d1,d2,bcast,screg1,screg2,scoff,screg,scinc,L1pt,L1pd
	yr4_4c_djbfft_noswiz_unroll 3,exec,exec,exec,srcreg,srcinc,srcoff,d1,d2,bcast,screg1,screg2,scoff,screg,scinc,L1pt,L1pd
	yr4_4c_djbfft_noswiz_unroll 4,exec,exec,noexec,srcreg,srcinc,srcoff,d1,d2,bcast,screg1,screg2,scoff,screg,scinc,L1pt,L1pd
	yr4_4c_djbfft_noswiz_unroll 5,exec,noexec,noexec,srcreg,srcinc,srcoff,d1,d2,bcast,screg1,screg2,scoff,screg,scinc,L1pt,L1pd
	bump	srcreg, 5*srcinc
	bump	screg, 5*scinc
	ELSEIF maxrpt MOD 4 EQ 0
	yr4_4c_djbfft_noswiz_unroll -1,noexec,noexec,exec,srcreg,srcinc,srcoff,d1,d2,bcast,screg1,screg2,scoff,screg,scinc,L1pt,L1pd
	yr4_4c_djbfft_noswiz_unroll 0,noexec,exec,exec,srcreg,srcinc,srcoff,d1,d2,bcast,screg1,screg2,scoff,screg,scinc,L1pt,L1pd
	yr4_4c_djbfft_noswiz_unroll 1,exec,exec,exec,srcreg,srcinc,srcoff,d1,d2,bcast,screg1,screg2,scoff,screg,scinc,L1pt,L1pd
	yr4_4c_djbfft_noswiz_unroll 2,exec,exec,exec,srcreg,srcinc,srcoff,d1,d2,bcast,screg1,screg2,scoff,screg,scinc,L1pt,L1pd
	yr4_4c_djbfft_noswiz_unroll 3,exec,exec,noexec,srcreg,srcinc,srcoff,d1,d2,bcast,screg1,screg2,scoff,screg,scinc,L1pt,L1pd
	yr4_4c_djbfft_noswiz_unroll 4,exec,noexec,noexec,srcreg,srcinc,srcoff,d1,d2,bcast,screg1,screg2,scoff,screg,scinc,L1pt,L1pd
	bump	srcreg, 4*srcinc
	bump	screg, 4*scinc
	ELSEIF maxrpt MOD 3 EQ 0
	yr4_4c_djbfft_noswiz_unroll -1,noexec,noexec,exec,srcreg,srcinc,srcoff,d1,d2,bcast,screg1,screg2,scoff,screg,scinc,L1pt,L1pd
	yr4_4c_djbfft_noswiz_unroll 0,noexec,exec,exec,srcreg,srcinc,srcoff,d1,d2,bcast,screg1,screg2,scoff,screg,scinc,L1pt,L1pd
	yr4_4c_djbfft_noswiz_unroll 1,exec,exec,exec,srcreg,srcinc,srcoff,d1,d2,bcast,screg1,screg2,scoff,screg,scinc,L1pt,L1pd
	yr4_4c_djbfft_noswiz_unroll 2,exec,exec,noexec,srcreg,srcinc,srcoff,d1,d2,bcast,screg1,screg2,scoff,screg,scinc,L1pt,L1pd
	yr4_4c_djbfft_noswiz_unroll 3,exec,noexec,noexec,srcreg,srcinc,srcoff,d1,d2,bcast,screg1,screg2,scoff,screg,scinc,L1pt,L1pd
	bump	srcreg, 3*srcinc
	bump	screg, 3*scinc
	ELSEIF maxrpt MOD 2 EQ 0
	yr4_4c_djbfft_noswiz_unroll -1,noexec,noexec,exec,srcreg,srcinc,srcoff,d1,d2,bcast,screg1,screg2,scoff,screg,scinc,L1pt,L1pd
	yr4_4c_djbfft_noswiz_unroll 0,noexec,exec,exec,srcreg,srcinc,srcoff,d1,d2,bcast,screg1,screg2,scoff,screg,scinc,L1pt,L1pd
	yr4_4c_djbfft_noswiz_unroll 1,exec,exec,noexec,srcreg,srcinc,srcoff,d1,d2,bcast,screg1,screg2,scoff,screg,scinc,L1pt,L1pd
	yr4_4c_djbfft_noswiz_unroll 2,exec,noexec,noexec,srcreg,srcinc,srcoff,d1,d2,bcast,screg1,screg2,scoff,screg,scinc,L1pt,L1pd
	bump	srcreg, 2*srcinc
	bump	screg, 2*scinc
	ELSE
	yr4_4c_djbfft_noswiz_unroll -1,noexec,noexec,exec,srcreg,srcinc,srcoff,d1,d2,bcast,screg1,screg2,scoff,screg,scinc,L1pt,L1pd
	yr4_4c_djbfft_noswiz_unroll 0,noexec,exec,noexec,srcreg,srcinc,srcoff,d1,d2,bcast,screg1,screg2,scoff,screg,scinc,L1pt,L1pd
	yr4_4c_djbfft_noswiz_unroll 1,exec,noexec,noexec,srcreg,srcinc,srcoff,d1,d2,bcast,screg1,screg2,scoff,screg,scinc,L1pt,L1pd
	bump	srcreg, srcinc
	bump	screg, scinc
	ENDIF
	ENDM

yr4_4c_djbfft_noswiz_unroll_preload MACRO
	ENDM

yr4_4c_djbfft_noswiz_unroll MACRO iter,prev,this,next,srcreg,srcinc,srcoff,d1,d2,bcast,screg1,screg2,scoff,screg,scinc,L1pt,L1pd

;; On first call, init registers

	IF iter EQ -1
y0	TEXTEQU	<ymm0>
y1	TEXTEQU	<ymm1>
y2	TEXTEQU	<ymm2>
y3	TEXTEQU	<ymm3>
y4	TEXTEQU	<ymm4>
y5	TEXTEQU	<ymm5>
y6	TEXTEQU	<ymm6>
y7	TEXTEQU	<ymm7>
y8	TEXTEQU	<ymm8>
y9	TEXTEQU	<ymm9>
y10	TEXTEQU	<ymm10>
y11	TEXTEQU	<ymm11>
y12	TEXTEQU	<ymm12>
y13	TEXTEQU	<ymm13>
y14	TEXTEQU	<ymm14>
y15	TEXTEQU	<ymm15>
	ENDIF

;; On later calls, previous R3,R4,I3,B4,A2,B2,sine1,sine2 will be in y0-7.  This R1,R3,I2,I4,I1,I3 will be in y8-13.
;; The remaining registers are free.

this	vsubpd	y14, y8, y9				;; R1 - R3 (new R3)		; 1-3
prev	vmulpd	y3, y3, y6				;; B4 = B4 * sine (final I4)	; 1-5
this	vmovapd	y6, [srcreg+iter*srcinc+srcoff+d1]	;; R2

this	vaddpd	y8, y8, y9				;; R1 + R3 (new R1)		; 2-4
prev	vmulpd	y4, y4, y7				;; A2 = A2 * sine (final R2)	; 2-6
this	vmovapd	y9, [srcreg+iter*srcinc+srcoff+d2+d1]	;; R4

this	vsubpd	y15, y10, y11				;; I2 - I4 (new I4)		; 3-5
prev	vmulpd	y5, y5, y7				;; B2 = B2 * sine (final I2)	; 3-7
this no bcast vmovapd y7, [screg1+iter*scinc+32]	;; cosine/sine
this bcast vbroadcastsd y7, Q [screg1+iter*scinc+scoff]	;; cosine/sine
prev	ystore	[srcreg+(iter-1)*srcinc+d2], y0		;; Save R3			; 3

this	vaddpd	y10, y10, y11				;; I2 + I4 (new I2)		; 4-6
this no bcast vmovapd y11, [screg2+iter*scinc+32]	;; cosine/sine
this bcast vbroadcastsd y11, Q [screg2+iter*scinc+scoff] ;; cosine/sine
prev	ystore	[srcreg+(iter-1)*srcinc+d2+d1], y1	;; Save R4			; 4

this	vsubpd	y1, y12, y13				;; I1 - I3 (new I3)		; 5-7
this no bcast vmovapd y0, [screg1+iter*scinc]		;; Sine
this bcast vbroadcastsd y0, Q [screg1+iter*scinc]	;; Sine
prev	ystore	[srcreg+(iter-1)*srcinc+d2+32], y2	;; Save I3			; 5

this	vaddpd	y12, y12, y13				;; I1 + I3 (new I1)		; 6-8
prev	ystore	[srcreg+(iter-1)*srcinc+d2+d1+32], y3	;; Save I4			; 6
this	L1prefetchw srcreg+iter*srcinc+L1pd, L1pt

this	vsubpd	y3, y6, y9				;; R2 - R4 (new R4)		; 7-9
prev	ystore	[srcreg+(iter-1)*srcinc+d1], y4		;; Save R2			; 7

this	vaddpd	y6, y6, y9				;; R2 + R4 (new R2)		; 8-10
prev	ystore	[srcreg+(iter-1)*srcinc+d1+32], y5	;; Save I2			; 8

this	vsubpd	y5, y14, y15				;; R3 - I4 (final R3)		; 9-11
this	L1prefetchw srcreg+iter*srcinc+d1+L1pd, L1pt

this	vaddpd	y14, y14, y15				;; R3 + I4 (final R4)		; 10-12

this	vaddpd	y15, y1, y3				;; I3 + R4 (final I3)		; 11-13

this	vsubpd	y1, y1, y3				;; I3 - R4 (final I4)		; 12-14
this	vmulpd	y3, y5, y7				;; A3 = R3 * cosine/sine	; 12-16
this	L1prefetchw srcreg+iter*srcinc+d2+L1pd, L1pt

this	vsubpd	y9, y8, y6				;; R1 - R2 (final R2)		; 13-15
this	vmulpd	y4, y14, y7				;; A4 = R4 * cosine/sine	; 13-17

this	vsubpd	y13, y12, y10				;; I1 - I2 (final I2)		; 14-16
this	vmulpd	y2, y15, y7				;; B3 = I3 * cosine/sine	; 14-18
this next yloop_unrolled_one

this	vaddpd	y8, y8, y6				;; R1 + R2 (final R1)		; 15-17
this	vmulpd	y7, y1, y7				;; B4 = I4 * cosine/sine	; 15-19
next	vmovapd	y6, [srcreg+(iter+1)*srcinc+srcoff]	;; R1

this	vaddpd	y12, y12, y10				;; I1 + I2 (final I1)		; 16-18
this	vmulpd	y10, y9, y11				;; A2 = R2 * cosine/sine	; 16-20
this	L1prefetchw srcreg+iter*srcinc+d2+d1+L1pd, L1pt

this	vsubpd	y3, y3, y15				;; A3 = A3 - I3			; 17-19
this	vmulpd	y11, y13, y11				;; B2 = I2 * cosine/sine	; 17-21
next	vmovapd	y15, [srcreg+(iter+1)*srcinc+srcoff+d2]	;; R3

this	vaddpd	y4, y4, y1				;; A4 = A4 + I4			; 18-20
this no bcast vmovapd y1, [screg2+iter*scinc]		;; Sine
this bcast vbroadcastsd y1, Q [screg2+iter*scinc]	;; Sine
this	ystore	[srcreg+iter*srcinc], y8		;; Save R1			; 18

this	vaddpd	y2, y2, y5				;; B3 = B3 + R3			; 19-21
next	vmovapd	y5, [srcreg+(iter+1)*srcinc+srcoff+d1+32] ;; I2
this	ystore	[srcreg+iter*srcinc+32], y12		;; Save I1			; 19

this	vsubpd	y7, y7, y14				;; B4 = B4 - R4			; 20-22
this	vmulpd	y3, y3, y0				;; A3 = A3 * sine (final R3)	; 20-24
next	vmovapd	y14, [srcreg+(iter+1)*srcinc+srcoff+d2+d1+32] ;; I4

this	vsubpd	y10, y10, y13				;; A2 = A2 - I2			; 21-23
this	vmulpd	y4, y4, y0				;; A4 = A4 * sine (final R4)	; 21-25
next	vmovapd	y13, [srcreg+(iter+1)*srcinc+srcoff+32]	;; I1

this	vaddpd	y11, y11, y9				;; B2 = B2 + R2			; 22-24
this	vmulpd	y2, y2, y0				;; B3 = B3 * sine (final I3)	; 22-26
next	vmovapd	y9, [srcreg+(iter+1)*srcinc+srcoff+d2+32] ;; I3

;; Shuffle register assignments so that next call has R3,R4,I3,B4,A2,B2,sine1,sine2 in y0-7 and next R1,R3,I2,I4,I1,I3 in y8-13.

ytmp	TEXTEQU	y0
y0	TEXTEQU	y3
y3	TEXTEQU	y7
y7	TEXTEQU	y1
y1	TEXTEQU	y4
y4	TEXTEQU	y10
y10	TEXTEQU	y5
y5	TEXTEQU	y11
y11	TEXTEQU	y14
y14	TEXTEQU	y8
y8	TEXTEQU	y6
y6	TEXTEQU	ytmp
ytmp	TEXTEQU y9
y9	TEXTEQU	y15
y15	TEXTEQU	y12
y12	TEXTEQU	y13
y13	TEXTEQU ytmp
	ENDM

;; Haswell FMA3 version
;; Basic implementation originally written for Bulldozer. 16 clocks.

IF (@INSTR(,%yarch,<NOT_BEST_FMA3>) NE 0)

yr4_4c_djbfft_noswiz_unroll_preload MACRO
	vmovapd	ymm15, YMM_ONE
	ENDM

yr4_4c_djbfft_noswiz_unroll MACRO iter,prev,this,next,srcreg,srcinc,srcoff,d1,d2,bcast,screg1,screg2,scoff,screg,scinc,L1pt,L1pd

;; On first call, init registers

	IF iter EQ -1
y0	TEXTEQU	<ymm0>
y1	TEXTEQU	<ymm1>
y2	TEXTEQU	<ymm2>
y3	TEXTEQU	<ymm3>
y4	TEXTEQU	<ymm4>
y5	TEXTEQU	<ymm5>
y6	TEXTEQU	<ymm6>
y7	TEXTEQU	<ymm7>
y8	TEXTEQU	<ymm8>
y9	TEXTEQU	<ymm9>
y10	TEXTEQU	<ymm10>
y11	TEXTEQU	<ymm11>
y12	TEXTEQU	<ymm12>
y13	TEXTEQU	<ymm13>
y14	TEXTEQU	<ymm14>
	ENDIF

;; On later calls, previous A2,B2,A3,B3,A4,B4,c/s1 will be in y0-6.  This new R1,new I1,new R3,new I3,I2,I4 will be in y7-12.
;; The remaining registers are free.  Register y15 is reserved for the constant YMM_ONE.

this	vaddpd	y13, y11, y12				;; I2 + I4 (new I2)			; 1-5
this	vsubpd y11, y11, y12			;; I2 - I4 (new I4)			; 1-5
this	vmovapd	y14, [srcreg+iter*srcinc+srcoff+d1]	;; R2
this	L1prefetchw srcreg+iter*srcinc+L1pd, L1pt

prev	yfmsubpd y12, y2, y6, y3			;; final R3 = A3 * cosine/sine - B3	; 2-6
prev	yfmaddpd y3, y3, y6, y2				;; final I3 = B3 * cosine/sine + A3	; 2-6
this	vmovapd	y2, [srcreg+iter*srcinc+srcoff+d2+d1]	;; R4

prev	ystore	[srcreg+(iter-1)*srcinc+d2], y12	;; Save R3				; 7
this	vaddpd	y12, y14, y2				;; R2 + R4 (new R2)			; 3-7
this	vsubpd y14, y14, y2			;; R2 - R4 (new R4)			; 3-7
this no bcast vmovapd y2, [screg1+iter*scinc]		;; Sine
this bcast vbroadcastsd y2, Q [screg1+iter*scinc]	;; Sine

this	vmulpd	y9, y9, y2				;; R3S = R3 * sine			; 4-8
this	vmulpd	y10, y10, y2				;; I3S = I3 * sine			; 4-8
this	L1prefetchw srcreg+iter*srcinc+d1+L1pd, L1pt
this next yloop_unrolled_one

prev	ystore	[srcreg+(iter-1)*srcinc+d2+32], y3	;; Save I3				; 7
prev	yfmaddpd y3, y4, y6, y5				;; final R4 = A4 * cosine/sine + B4	; 5-9
prev	yfmsubpd y5, y5, y6, y4				;; final I4 = B4 * cosine/sine - A4	; 5-9
prev no bcast vmovapd y4, [screg2+(iter-1)*scinc]	;; Sine
prev bcast vbroadcastsd y4, Q [screg2+(iter-1)*scinc]	;; Sine

this	vsubpd	y6, y8, y13				;; I1 - I2 (newer I2)			; 6-10
this	vaddpd y8, y8, y13			;; I1 + I2 (newer & final I1)		; 6-10
next	vmovapd	y13, [srcreg+(iter+1)*srcinc+srcoff]	;; R1
this	L1prefetchw srcreg+iter*srcinc+d2+L1pd, L1pt

prev	vmulpd	y0, y0, y4				;; final R2 = A2 * sine			; 7-11
prev	vmulpd	y1, y1, y4				;; final I2 = B2 * sine			; 7-11
next	vmovapd	y4, [srcreg+(iter+1)*srcinc+srcoff+d2]	;; R3
this	L1prefetchw srcreg+iter*srcinc+d2+d1+L1pd, L1pt

prev	ystore	[srcreg+(iter-1)*srcinc+d2+d1], y3	;; Save R4				; 10
this	vsubpd	y3, y7, y12				;; R1 - R2 (newer R2)			; 8-12
this	vaddpd y7, y7, y12			;; R1 + R2 (newer & final R1)		; 8-12
next	vmovapd	y12, [srcreg+(iter+1)*srcinc+srcoff+32]	;; I1

prev	ystore	[srcreg+(iter-1)*srcinc+d2+d1+32], y5	;; Save I4				; 10
this	yfnmaddpd y5, y11, y2, y9			;; A3 = R3S - I4 * sine (newer R3*sine)	; 9-13
this	yfmaddpd y11, y11, y2, y9			;; A4 = R3S + I4 * sine (newer R4*sine)	; 9-13
next	vmovapd	y9, [srcreg+(iter+1)*srcinc+srcoff+d2+32] ;; I3

this	ystore	[srcreg+iter*srcinc+32], y8		;; Save I1				; 11
this	yfmaddpd y8, y14, y2, y10			;; B3 = I3S + R4 * sine (newer I3*sine)	; 10-14
this	yfnmaddpd y14, y14, y2, y10			;; B4 = I3S - R4 * sine (newer I4*sine)	; 10-14
this no bcast vmovapd y2, [screg2+iter*scinc+32]	;; cosine/sine
this bcast vbroadcastsd y2, Q [screg2+iter*scinc+scoff]	;; cosine/sine

next	vsubpd	y10, y13, y4				;; R1 - R3 (new R3)			; 11-15
next	vaddpd y13, y13, y4			;; R1 + R3 (new R1)			; 11-15
next	vmovapd	y4, [srcreg+(iter+1)*srcinc+srcoff+d1+32] ;; I2
prev	ystore	[srcreg+(iter-1)*srcinc+d1], y0		;; Save R2				; 12

next	vsubpd	y0, y12, y9				;; I1 - I3 (new I3)			; 12-16
next	vaddpd y12, y12, y9			;; I1 + I3 (new I1)			; 12-16
next	vmovapd	y9, [srcreg+(iter+1)*srcinc+srcoff+d2+d1+32] ;; I4
prev	ystore	[srcreg+(iter-1)*srcinc+d1+32], y1	;; Save I2				; 12

this	yfmsubpd y1, y3, y2, y6				;; A2 = R2 * cosine/sine - I2		; 13-17
this	yfmaddpd y6, y6, y2, y3				;; B2 = I2 * cosine/sine + R2		; 13-17
this no bcast vmovapd y2, [screg1+iter*scinc+32]	;; cosine/sine
this bcast vbroadcastsd y2, Q [screg1+iter*scinc+scoff] ;; cosine/sine
this	ystore	[srcreg+iter*srcinc], y7		;; Save R1				; 13

;; Shuffle register assignments so that next call has A2,B2,A3,B3,A4,B4,c/s1 in y0-6, and next new R1,new I1,new R3,new I3,I2,I4 in y7-12.

ytmp	TEXTEQU	y0
y0	TEXTEQU	y1
y1	TEXTEQU	y6
y6	TEXTEQU	y2
y2	TEXTEQU	y5
y5	TEXTEQU	y14
y14	TEXTEQU	y3
y3	TEXTEQU	y8
y8	TEXTEQU	y12
y12	TEXTEQU	y9
y9	TEXTEQU	y10
y10	TEXTEQU	ytmp
ytmp	TEXTEQU y4
y4	TEXTEQU	y11
y11	TEXTEQU	ytmp
ytmp	TEXTEQU y7
y7	TEXTEQU	y13
y13	TEXTEQU ytmp
	ENDM

ENDIF


;; Haswell FMA3 version
;; One-half the adds converted to FMA3.  Alas, still 16 clocks.

IF (@INSTR(,%yarch,<NOT_BEST_FMA3>) NE 0)

yr4_4c_djbfft_noswiz_unroll_preload MACRO
	vmovapd	ymm15, YMM_ONE
	ENDM

yr4_4c_djbfft_noswiz_unroll MACRO iter,prev,this,next,srcreg,srcinc,srcoff,d1,d2,bcast,screg1,screg2,scoff,screg,scinc,L1pt,L1pd

;; On first call, init registers

	IF iter EQ -1
y0	TEXTEQU	<ymm0>
y1	TEXTEQU	<ymm1>
y2	TEXTEQU	<ymm2>
y3	TEXTEQU	<ymm3>
y4	TEXTEQU	<ymm4>
y5	TEXTEQU	<ymm5>
y6	TEXTEQU	<ymm6>
y7	TEXTEQU	<ymm7>
y8	TEXTEQU	<ymm8>
y9	TEXTEQU	<ymm9>
y10	TEXTEQU	<ymm10>
y11	TEXTEQU	<ymm11>
y12	TEXTEQU	<ymm12>
y13	TEXTEQU	<ymm13>
y14	TEXTEQU	<ymm14>
	ENDIF

;; On later calls, previous A2,B2,A3,B3,A4,B4,c/s1 will be in y0-6.  This new R1,new I1,new R3,new I3,I2,I4 will be in y7-12.
;; The remaining registers are free.  Register y15 is used for the constant YMM_ONE.

this	vaddpd	y13, y11, y12				;; I2 + I4 (new I2)			; 1-5
this	yfmsubpd y11, y11, ymm15, y12			;; I2 - I4 (new I4)			; 1-5
this	vmovapd	y14, [srcreg+iter*srcinc+srcoff+d1]	;; R2
this	L1prefetchw srcreg+iter*srcinc+L1pd, L1pt

prev	yfmsubpd y12, y2, y6, y3			;; final R3 = A3 * cosine/sine - B3	; 2-6
prev	yfmaddpd y3, y3, y6, y2				;; final I3 = B3 * cosine/sine + A3	; 2-6
this	vmovapd	y2, [srcreg+iter*srcinc+srcoff+d2+d1]	;; R4

prev	ystore	[srcreg+(iter-1)*srcinc+d2], y12	;; Save R3				; 7
this	vaddpd	y12, y14, y2				;; R2 + R4 (new R2)			; 3-7
this	yfmsubpd y14, y14, ymm15, y2			;; R2 - R4 (new R4)			; 3-7
this no bcast vmovapd y2, [screg1+iter*scinc]		;; Sine
this bcast vbroadcastsd y2, Q [screg1+iter*scinc]	;; Sine

this	vmulpd	y9, y9, y2				;; R3S = R3 * sine			; 4-8
this	vmulpd	y10, y10, y2				;; I3S = I3 * sine			; 4-8
this	L1prefetchw srcreg+iter*srcinc+d1+L1pd, L1pt
this next yloop_unrolled_one

prev	ystore	[srcreg+(iter-1)*srcinc+d2+32], y3	;; Save I3				; 7
prev	yfmaddpd y3, y4, y6, y5				;; final R4 = A4 * cosine/sine + B4	; 5-9
prev	yfmsubpd y5, y5, y6, y4				;; final I4 = B4 * cosine/sine - A4	; 5-9
prev no bcast vmovapd y4, [screg2+(iter-1)*scinc]	;; Sine
prev bcast vbroadcastsd y4, Q [screg2+(iter-1)*scinc]	;; Sine

this	vsubpd	y6, y8, y13				;; I1 - I2 (newer I2)			; 6-10
this	yfmaddpd y8, y8, ymm15, y13			;; I1 + I2 (newer & final I1)		; 6-10
next	vmovapd	y13, [srcreg+(iter+1)*srcinc+srcoff]	;; R1
this	L1prefetchw srcreg+iter*srcinc+d2+L1pd, L1pt

prev	vmulpd	y0, y0, y4				;; final R2 = A2 * sine			; 7-11
prev	vmulpd	y1, y1, y4				;; final I2 = B2 * sine			; 7-11
next	vmovapd	y4, [srcreg+(iter+1)*srcinc+srcoff+d2]	;; R3
this	L1prefetchw srcreg+iter*srcinc+d2+d1+L1pd, L1pt

prev	ystore	[srcreg+(iter-1)*srcinc+d2+d1], y3	;; Save R4				; 10
this	vsubpd	y3, y7, y12				;; R1 - R2 (newer R2)			; 8-12
this	yfmaddpd y7, y7, ymm15, y12			;; R1 + R2 (newer & final R1)		; 8-12
next	vmovapd	y12, [srcreg+(iter+1)*srcinc+srcoff+32]	;; I1

prev	ystore	[srcreg+(iter-1)*srcinc+d2+d1+32], y5	;; Save I4				; 10
this	yfnmaddpd y5, y11, y2, y9			;; A3 = R3S - I4 * sine (newer R3*sine)	; 9-13
this	yfmaddpd y11, y11, y2, y9			;; A4 = R3S + I4 * sine (newer R4*sine)	; 9-13
next	vmovapd	y9, [srcreg+(iter+1)*srcinc+srcoff+d2+32] ;; I3

this	ystore	[srcreg+iter*srcinc+32], y8		;; Save I1				; 11
this	yfmaddpd y8, y14, y2, y10			;; B3 = I3S + R4 * sine (newer I3*sine)	; 10-14
this	yfnmaddpd y14, y14, y2, y10			;; B4 = I3S - R4 * sine (newer I4*sine)	; 10-14
this no bcast vmovapd y2, [screg2+iter*scinc+32]	;; cosine/sine
this bcast vbroadcastsd y2, Q [screg2+iter*scinc+scoff]	;; cosine/sine

next	vsubpd	y10, y13, y4				;; R1 - R3 (new R3)			; 11-15
next	yfmaddpd y13, y13, ymm15, y4			;; R1 + R3 (new R1)			; 11-15
next	vmovapd	y4, [srcreg+(iter+1)*srcinc+srcoff+d1+32] ;; I2
prev	ystore	[srcreg+(iter-1)*srcinc+d1], y0		;; Save R2				; 12

next	vsubpd	y0, y12, y9				;; I1 - I3 (new I3)			; 12-16
next	yfmaddpd y12, y12, ymm15, y9			;; I1 + I3 (new I1)			; 12-16
next	vmovapd	y9, [srcreg+(iter+1)*srcinc+srcoff+d2+d1+32] ;; I4
prev	ystore	[srcreg+(iter-1)*srcinc+d1+32], y1	;; Save I2				; 12

this	yfmsubpd y1, y3, y2, y6				;; A2 = R2 * cosine/sine - I2		; 13-17
this	yfmaddpd y6, y6, y2, y3				;; B2 = I2 * cosine/sine + R2		; 13-17
this no bcast vmovapd y2, [screg1+iter*scinc+32]	;; cosine/sine
this bcast vbroadcastsd y2, Q [screg1+iter*scinc+scoff] ;; cosine/sine
this	ystore	[srcreg+iter*srcinc], y7		;; Save R1				; 13

;; Shuffle register assignments so that next call has A2,B2,A3,B3,A4,B4,c/s1 in y0-6, and next new R1,new I1,new R3,new I3,I2,I4 in y7-12.

ytmp	TEXTEQU	y0
y0	TEXTEQU	y1
y1	TEXTEQU	y6
y6	TEXTEQU	y2
y2	TEXTEQU	y5
y5	TEXTEQU	y14
y14	TEXTEQU	y3
y3	TEXTEQU	y8
y8	TEXTEQU	y12
y12	TEXTEQU	y9
y9	TEXTEQU	y10
y10	TEXTEQU	ytmp
ytmp	TEXTEQU y4
y4	TEXTEQU	y11
y11	TEXTEQU	ytmp
ytmp	TEXTEQU y7
y7	TEXTEQU	y13
y13	TEXTEQU ytmp
	ENDM

ENDIF


;; Haswell FMA3 version
;; Attemptimg to optimally place adds so that they are executed every other clock cycle.  This reduces
;; the number of register copies.  To make exactly 25% of the float instructions adds and subtracts, we must
;; unroll 2 iterations.  Alas, still 16 clocks.

IF (@INSTR(,%yarch,<NOT_BEST_FMA3>) NE 0)

yr4_4c_djbfft_noswiz_unroll_preload MACRO
	vmovapd	ymm15, YMM_ONE
	ENDM

;; uops = 8 s/c loads, 16 loads, 16 stores, 8 muls, 44 fma/add/sub, 9 mov = 101 uops / 4 = 25.25 clocks (barely under 13*2!!)
;; Untested - should be optimal, but timed at 32=16*2 clocks
yr4_4c_djbfft_noswiz_unroll2 MACRO iter,prev,this,next,srcreg,srcinc,srcoff,d1,d2,bcast,screg1,screg2,scoff,screg,scinc,L1pt,L1pd

;; On first call, init registers

	IF iter LE -1
y0	TEXTEQU	<ymm0>
y1	TEXTEQU	<ymm1>
y2	TEXTEQU	<ymm2>
y3	TEXTEQU	<ymm3>
y4	TEXTEQU	<ymm4>
y5	TEXTEQU	<ymm5>
y6	TEXTEQU	<ymm6>
y7	TEXTEQU	<ymm7>
y8	TEXTEQU	<ymm8>
y9	TEXTEQU	<ymm9>
y10	TEXTEQU	<ymm10>
y11	TEXTEQU	<ymm11>
y12	TEXTEQU	<ymm12>
y13	TEXTEQU	<ymm13>
y14	TEXTEQU	<ymm14>
	ENDIF

;; On later calls, previous I3S_2,R4_2,sine_2,R2_2,I2_2,c/s_2,A3_2,B3_2,A4_2,I1_2,I2,R1_2 will be in y0-11.  This R1,R3 will be in y12-13.
;; The remaining register is free.  Register y15 is reserved for the constant YMM_ONE.

prev	yfnmaddpd y1, y1, y2, y0			;; B4_2 = I3S_2 - R4_2 * sine_2		; 1-5			n 8
this	vsubpd	y2, y12, y13				;; R1 - R3 (new R3)			;	1-3		n 4
this	vmovapd	y14, [srcreg+iter*srcinc+srcoff+32]	;; I1

this	yfmaddpd y12, y12, ymm15, y13			;; R1 + R3 (new R1)			; 2-6			n 11
prev	yfmsubpd y13, y3, y5, y4, 2			;; A2_2 = R2_2 * cosine/sine_2 - I2_2	;	2-6		n 12		r 2**
this	vmovapd	y0, [srcreg+iter*srcinc+srcoff+d2+32]	;; I3
prev	ystore	[srcreg+(iter-1)*srcinc+32], y9		;; Save I1_2				;		28

this	vsubpd	y9, y14, y0				;; I1 - I3 (new I3)			;	3-5		n 8
this	yfmaddpd y14, y14, ymm15, y0			;; I1 + I3 (new I1)			; 3-7			n 9
this no bcast vmovapd y0, [screg1+iter*scinc]		;; Sine
this bcast vbroadcastsd y0, Q [screg1+iter*scinc]	;; Sine
prev	ystore	[srcreg+(iter-2)*srcinc+d1+32], y10	;; Save I2				;		29

prev	yfmaddpd y4, y4, y5, y3				;; B2_2 = I2_2 * cosine/sine_2 + R2_2	; 4-8			n 17
this	vmulpd	y2, y2, y0				;; R3S = R3 * sine			;	4-8		n 10		r 4**
this	vmovapd	y10, [srcreg+iter*srcinc+srcoff+d1+32]	;; I2

this	vmovapd	y5, [srcreg+iter*srcinc+srcoff+d2+d1+32] ;; I4
this	vaddpd	y3, y10, y5				;; I2 + I4 (new I2)			;	5-7		n 9
this	yfmsubpd y10, y10, ymm15, y5			;; I2 - I4 (new I4)			; 5-9			n 10		r 5*
prev no bcast vmovapd y5, [screg1+(iter-1)*scinc+32]	;; cosine/sine_2
prev bcast vbroadcastsd y5, Q [screg1+(iter-1)*scinc+scoff] ;; cosine/sine_2
prev	ystore	[srcreg+(iter-1)*srcinc], y11		;; Save R1_2				;		31

prev	yfmsubpd y11, y6, y5, y7, 2			;; final R3_2 = A3_2 * cos/sin_2 - B3_2	; 6-10					r 5
prev	yfmaddpd y7, y7, y5, y6				;; final I3_2 = B3_2 * cos/sin_2 + A3_2	;	6-10
this	vmovapd	y6, [srcreg+iter*srcinc+srcoff+d1]	;; R2

prev	ystore	[srcreg+(iter-1)*srcinc+d2], y11	;; Save R3_2				;		11
this	vmovapd	y11, [srcreg+iter*srcinc+srcoff+d2+d1]	;; R4
prev	ystore	[srcreg+(iter-1)*srcinc+d2+32], y7	;; Save I3_2				;		11
this	vaddpd	y7, y6, y11				;; R2 + R4 (new R2)			;	7-9		n 11
this	yfmsubpd y6, y6, ymm15, y11			;; R2 - R4 (new R4)			; 7-11			n 13

this	vmulpd	y9, y9, y0				;; I3S = I3 * sine			; 8-12			n 13
prev	yfmaddpd y11, y8, y5, y1, 2			;; final R4_2 = A4_2 * cos/sin_2 + B4_2	;	8-12				r 6

prev	yfmsubpd y1, y1, y5, y8				;; final I4_2 = B4_2 * cos/sin_2 - A4_2	; 9-13
this	vsubpd	y5, y14, y3				;; I1 - I2 (newer I2)			;	9-11		n 14		r 9*		***

this	yfnmaddpd y8, y10, y0, y2, 2			;; A3 = R3S - I4 * sine (newer R3*sine)	; 10-14			n 19
this	yfmaddpd y14, y14, ymm15, y3			;; I1 + I2 (newer & final I1)		;	10-14
prev no bcast vmovapd y3, [screg2+(iter-1)*scinc]	;; Sine_2
prev bcast vbroadcastsd y3, Q [screg2+(iter-1)*scinc]	;; Sine_2

this	yfmaddpd y10, y10, y0, y2			;; A4 = R3S + I4 * sine (newer R4*sine)	; 11-15			n 20
this	vsubpd	y2, y12, y7				;; R1 - R2 (newer R2)			;	11-13		n 14
prev	ystore	[srcreg+(iter-1)*srcinc+d2+d1], y11	;; Save R4_2				;		13
this	vmovapd	y11, [srcreg+(iter+1)*srcinc+srcoff]	;; R1_2

this	yfmaddpd y12, y12, ymm15, y7			;; R1 + R2 (newer & final R1)		; 12-16
prev	vmulpd	y13, y13, y3				;; final R2_2 = A2_2 * sine_2		;	12-16
this	vmovapd	y7, [srcreg+(iter+1)*srcinc+srcoff+d2]	;; R3_2

prev	ystore	[srcreg+(iter-1)*srcinc+d2+d1+32], y1	;; Save I4_2				;		14
this	vsubpd	y1, y11, y7				;; R1_2 - R3_2 (new R3_2)		;	13-15		n 16
this	ystore	[srcreg+iter*srcinc+32], y14		;; Save I1				;		15
this	yfmaddpd y14, y6, y0, y9, 2			;; B3 = I3S + R4 * sine (newer I3*sine)	; 13-17			n 19

this	yfnmaddpd y6, y6, y0, y9			;; B4 = I3S - R4 * sine (newer I4*sine)	; 14-18			n 22
this no bcast vmovapd y0, [screg2+iter*scinc+32]	;; cosine/sine
this bcast vbroadcastsd y0, Q [screg2+iter*scinc+scoff]	;; cosine/sine
this	yfmsubpd y9, y2, y0, y5, 2			;; A2 = R2 * cosine/sine - I2		;	14-18		n 22

this	yfmaddpd y5, y5, y0, y2				;; B2 = I2 * cosine/sine + R2		; 15-19			n 24
this	vmovapd	y0, [srcreg+(iter+1)*srcinc+srcoff+32]	;; I1_2
this	vmovapd	y2, [srcreg+(iter+1)*srcinc+srcoff+d2+32] ;; I3_2
this	ystore	[srcreg+iter*srcinc], y12		;; Save R1				;		17
this	vsubpd	y12, y0, y2				;; I1_2 - I3_2 (new I3_2)		;	15-17		n 18

this	yfmaddpd y0, y0, ymm15, y2			;; I1_2 + I3_2 (new I1_2)		; 16-20			n 23
this no bcast vmovapd y2, [screg1+(iter+1)*scinc]	;; Sine_2
this bcast vbroadcastsd y2, Q [screg1+(iter+1)*scinc]	;; Sine_2
this	vmulpd	y1, y1, y2				;; R3S_2 = R3_2 * sine_2		;	16-20		n 24

prev	vmulpd	y4, y4, y3				;; final I2_2 = B2_2 * sine_2		; 17-21
this	vmovapd	y3, [srcreg+(iter+1)*srcinc+srcoff+d1+32] ;; I2_2
prev	ystore	[srcreg+(iter-1)*srcinc+d1], y13	;; Save R2_2				;		17
this	vmovapd	y13, [srcreg+(iter+1)*srcinc+srcoff+d2+d1+32] ;; I4_2
prev	ystore	[srcreg+(iter-1)*srcinc+d1+32], y4	;; Save I2_2				;		22
this	vaddpd	y4, y3, y13				;; I2_2 + I4_2 (new I2_2)		;	17-19		n 23

this	yfmaddpd y11, y11, ymm15, y7			;; R1_2 + R3_2 (new R1_2)		; 18-22			n 25
this	vmulpd	y12, y12, y2				;; I3S_2 = I3_2 * sine_2		;	18-22		n 26
this no bcast vmovapd y7, [screg1+iter*scinc+32]	;; cosine/sine
this bcast vbroadcastsd y7, Q [screg1+iter*scinc+scoff] ;; cosine/sine

this	vsubpd	y3, y3, y13				;; I2_2 - I4_2 (new I4_2)		;	19-21		n 24
this	yfmsubpd y13, y8, y7, y14, 2			;; final R3 = A3 * cosine/sine - B3	; 19-23

this	yfmaddpd y14, y14, y7, y8			;; final I3 = B3 * cosine/sine + A3	;	20-24
this	yfmaddpd y8, y10, y7, y6, 2			;; final R4 = A4 * cosine/sine + B4	; 20-24

this	ystore	[srcreg+iter*srcinc+d2], y13		;; Save R3				;		24
this	vmovapd	y13, [srcreg+(iter+1)*srcinc+srcoff+d1]	;; R2_2
this	ystore	[srcreg+iter*srcinc+d2+32], y14		;; Save I3				;		25
this	vmovapd	y14, [srcreg+(iter+1)*srcinc+srcoff+d2+d1] ;; R4_2
this	ystore	[srcreg+iter*srcinc+d2+d1], y8		;; Save R4				;		25
this	vaddpd	y8, y13, y14				;; R2_2 + R4_2 (new R2_2)		;	21-23		n 25
this	yfmsubpd y13, y13, ymm15, y14			;; R2_2 - R4_2 (new R4_2)		; 21-25		n 26
this no bcast vmovapd y14, [screg2+iter*scinc]		;; Sine
this bcast vbroadcastsd y14, Q [screg2+iter*scinc]	;; Sine

this	yfmsubpd y6, y6, y7, y10			;; final I4 = B4 * cosine/sine - A4	; 22-26
this	vmulpd	y9, y9, y14				;; final R2 = A2 * sine			;	22-26
next	vmovapd	y7, [srcreg+(iter+2)*srcinc+srcoff]	;; R1

this	vsubpd	y10, y0, y4				;; I1_2 - I2_2 (newer I2_2)		;	23-25		n 28
this	yfmaddpd y0, y0, ymm15, y4			;; I1_2 + I2_2 (newer & final I1_2)	; 23-27
next	vmovapd	y4, [srcreg+(iter+2)*srcinc+srcoff+d2]	;; R3

this	vmulpd	y5, y5, y14				;; final I2 = B2 * sine			; 24-28
this	yfnmaddpd y14, y3, y2, y1, 2			;; A3_2 = R3S_2 - I4_2 * sine_2		;	24-28		n 32

this	ystore	[srcreg+iter*srcinc+d2+d1+32], y6	;; Save I4				;		27
this	vsubpd	y6, y11, y8				;; R1_2 - R2_2 (newer R2_2)		;	25-27		n 28
this	yfmaddpd y11, y11, ymm15, y8			;; R1_2 + R2_2 (newer & final R1_2)	; 25-29
this no bcast vmovapd y8, [screg2+(iter+1)*scinc+32]	;; cosine/sine_2
this bcast vbroadcastsd y8, Q [screg2+(iter+1)*scinc+scoff] ;; cosine/sine_2

this	ystore	[srcreg+iter*srcinc+d1], y9		;; Save R2				;		27
this	yfmaddpd y9, y13, y2, y12, 2			;; B3_2 = I3S_2 + R4_2 * sine_2		; 26-30			n 32
this	yfmaddpd y3, y3, y2, y1				;; A4_2 = R3S_2 + I4_2 * sine_2		;	26-30		n 34

this	L1prefetchw srcreg+iter*srcinc+L1pd, L1pt
this	L1prefetchw srcreg+iter*srcinc+d1+L1pd, L1pt
this	L1prefetchw srcreg+iter*srcinc+d2+L1pd, L1pt
this	L1prefetchw srcreg+iter*srcinc+d2+d1+L1pd, L1pt
this next yloop_unrolled_one

this	L1prefetchw srcreg+(iter+1)*srcinc+L1pd, L1pt
this	L1prefetchw srcreg+(iter+1)*srcinc+d1+L1pd, L1pt
this	L1prefetchw srcreg+(iter+1)*srcinc+d2+L1pd, L1pt
this	L1prefetchw srcreg+(iter+1)*srcinc+d2+d1+L1pd, L1pt
this next yloop_unrolled_one

;; Shuffle register assignments so that next call has this I3S_2,R4_2,sine_2,R2_2,I2_2,c/s_2,A3_2,B3_2,A4_2,I1_2,I2,R1_2 in y0-11, and next R1,R3 in y12-13.

ytmp	TEXTEQU	y0
y0	TEXTEQU	y12
y12	TEXTEQU	y7
y7	TEXTEQU	y9
y9	TEXTEQU	ytmp
ytmp	TEXTEQU	y1
y1	TEXTEQU	y13
y13	TEXTEQU	y4
y4	TEXTEQU	y10
y10	TEXTEQU	y5
y5	TEXTEQU	y8
y8	TEXTEQU	y3
y3	TEXTEQU y6
y6	TEXTEQU	y14
y14	TEXTEQU	ytmp
	ENDM

ENDIF


;; Haswell FMA3 version
;; Abandoning all attempts at using add and subtract instructions.  We use FMA3 which requires a lot of register copies.

IF (@INSTR(,%yarch,<FMA3>) NE 0)

yr4_4c_djbfft_noswiz_unroll_preload MACRO
	vmovapd	ymm15, YMM_ONE
	ENDM

;; Timed at 15 clocks.
yr4_4c_djbfft_noswiz_unroll MACRO iter,prev,this,next,srcreg,srcinc,srcoff,d1,d2,bcast,screg1,screg2,scoff,screg,scinc,L1pt,L1pd

;; On first call, init registers

	IF iter EQ -1
y0	TEXTEQU	<ymm0>
y1	TEXTEQU	<ymm1>
y2	TEXTEQU	<ymm2>
y3	TEXTEQU	<ymm3>
y4	TEXTEQU	<ymm4>
y5	TEXTEQU	<ymm5>
y6	TEXTEQU	<ymm6>
y7	TEXTEQU	<ymm7>
y8	TEXTEQU	<ymm8>
y9	TEXTEQU	<ymm9>
y10	TEXTEQU	<ymm10>
y11	TEXTEQU	<ymm11>
y12	TEXTEQU	<ymm12>
y13	TEXTEQU	<ymm13>
y14	TEXTEQU	<ymm14>
	ENDIF

;; On later calls, previous A2,B2,A3,B3,A4,B4,c/s1 will be in y0-6.  This new R1,new I1,new R3,new I3,I2,I4 will be in y7-12.
;; The remaining registers are free.  Register ymm15 is reserved for the constant YMM_ONE.

this	yfmaddpd y13, y11, ymm15, y12			;; I2 + I4 (new I2)			; 1-5
this	yfmsubpd y11, y11, ymm15, y12			;; I2 - I4 (new I4)			; 1-5
this	vmovapd	y14, [srcreg+iter*srcinc+srcoff+d1]	;; R2
this	L1prefetchw srcreg+iter*srcinc+L1pd, L1pt

prev	yfmsubpd y12, y2, y6, y3			;; final R3 = A3 * cosine/sine - B3	; 2-6
prev	yfmaddpd y3, y3, y6, y2				;; final I3 = B3 * cosine/sine + A3	; 2-6
this	vmovapd	y2, [srcreg+iter*srcinc+srcoff+d2+d1]	;; R4

prev	ystore	[srcreg+(iter-1)*srcinc+d2], y12	;; Save R3				; 7
this	yfmaddpd y12, y14, ymm15, y2			;; R2 + R4 (new R2)			; 3-7
this	yfmsubpd y14, y14, ymm15, y2			;; R2 - R4 (new R4)			; 3-7
this no bcast vmovapd y2, [screg1+iter*scinc]		;; Sine
this bcast vbroadcastsd y2, Q [screg1+iter*scinc]	;; Sine

this	vmulpd	y9, y9, y2				;; R3S = R3 * sine			; 4-8
this	vmulpd	y10, y10, y2				;; I3S = I3 * sine			; 4-8
this	L1prefetchw srcreg+iter*srcinc+d1+L1pd, L1pt
this next yloop_unrolled_one

prev	ystore	[srcreg+(iter-1)*srcinc+d2+32], y3	;; Save I3				; 7
prev	yfmaddpd y3, y4, y6, y5				;; final R4 = A4 * cosine/sine + B4	; 5-9
prev	yfmsubpd y5, y5, y6, y4				;; final I4 = B4 * cosine/sine - A4	; 5-9
prev no bcast vmovapd y4, [screg2+(iter-1)*scinc]	;; Sine
prev bcast vbroadcastsd y4, Q [screg2+(iter-1)*scinc]	;; Sine

this	yfmsubpd y6, y8, ymm15, y13			;; I1 - I2 (newer I2)			; 6-10
this	yfmaddpd y8, y8, ymm15, y13			;; I1 + I2 (newer & final I1)		; 6-10
next	vmovapd	y13, [srcreg+(iter+1)*srcinc+srcoff]	;; R1
this	L1prefetchw srcreg+iter*srcinc+d2+L1pd, L1pt

prev	vmulpd	y0, y0, y4				;; final R2 = A2 * sine			; 7-11
prev	vmulpd	y1, y1, y4				;; final I2 = B2 * sine			; 7-11
next	vmovapd	y4, [srcreg+(iter+1)*srcinc+srcoff+d2]	;; R3
this	L1prefetchw srcreg+iter*srcinc+d2+d1+L1pd, L1pt

prev	ystore	[srcreg+(iter-1)*srcinc+d2+d1], y3	;; Save R4				; 10
this	yfmsubpd y3, y7, ymm15, y12			;; R1 - R2 (newer R2)			; 8-12
this	yfmaddpd y7, y7, ymm15, y12			;; R1 + R2 (newer & final R1)		; 8-12
next	vmovapd	y12, [srcreg+(iter+1)*srcinc+srcoff+32]	;; I1

prev	ystore	[srcreg+(iter-1)*srcinc+d2+d1+32], y5	;; Save I4				; 10
this	yfnmaddpd y5, y11, y2, y9			;; A3 = R3S - I4 * sine (newer R3*sine)	; 9-13
this	yfmaddpd y11, y11, y2, y9			;; A4 = R3S + I4 * sine (newer R4*sine)	; 9-13
next	vmovapd	y9, [srcreg+(iter+1)*srcinc+srcoff+d2+32] ;; I3

this	ystore	[srcreg+iter*srcinc+32], y8		;; Save I1				; 11
this	yfmaddpd y8, y14, y2, y10			;; B3 = I3S + R4 * sine (newer I3*sine)	; 10-14
this	yfnmaddpd y14, y14, y2, y10			;; B4 = I3S - R4 * sine (newer I4*sine)	; 10-14
this no bcast vmovapd y2, [screg2+iter*scinc+32]	;; cosine/sine
this bcast vbroadcastsd y2, Q [screg2+iter*scinc+scoff]	;; cosine/sine

next	yfmsubpd y10, y13, ymm15, y4			;; R1 - R3 (new R3)			; 11-15
next	yfmaddpd y13, y13, ymm15, y4			;; R1 + R3 (new R1)			; 11-15
next	vmovapd	y4, [srcreg+(iter+1)*srcinc+srcoff+d1+32] ;; I2
prev	ystore	[srcreg+(iter-1)*srcinc+d1], y0		;; Save R2				; 12

next	yfmsubpd y0, y12, ymm15, y9			;; I1 - I3 (new I3)			; 12-16
next	yfmaddpd y12, y12, ymm15, y9			;; I1 + I3 (new I1)			; 12-16
next	vmovapd	y9, [srcreg+(iter+1)*srcinc+srcoff+d2+d1+32] ;; I4
prev	ystore	[srcreg+(iter-1)*srcinc+d1+32], y1	;; Save I2				; 12

this	yfmsubpd y1, y3, y2, y6				;; A2 = R2 * cosine/sine - I2		; 13-17
this	yfmaddpd y6, y6, y2, y3				;; B2 = I2 * cosine/sine + R2		; 13-17
this no bcast vmovapd y2, [screg1+iter*scinc+32]	;; cosine/sine
this bcast vbroadcastsd y2, Q [screg1+iter*scinc+scoff] ;; cosine/sine
this	ystore	[srcreg+iter*srcinc], y7		;; Save R1				; 13

;; Shuffle register assignments so that next call has A2,B2,A3,B3,A4,B4,c/s1 in y0-6, and next new R1,new I1,new R3,new I3,I2,I4 in y7-12.

ytmp	TEXTEQU	y0
y0	TEXTEQU	y1
y1	TEXTEQU	y6
y6	TEXTEQU	y2
y2	TEXTEQU	y5
y5	TEXTEQU	y14
y14	TEXTEQU	y3
y3	TEXTEQU	y8
y8	TEXTEQU	y12
y12	TEXTEQU	y9
y9	TEXTEQU	y10
y10	TEXTEQU	ytmp
ytmp	TEXTEQU y4
y4	TEXTEQU	y11
y11	TEXTEQU	ytmp
ytmp	TEXTEQU y7
y7	TEXTEQU	y13
y13	TEXTEQU ytmp
	ENDM

ENDIF

;; Haswell FMA3 version
;; This is an untested all FMA3 version that should be optimal.  Alas, it is no faster.  Timed at ~15 clocks.
;; This was written when I thought there was a 1 clock delay to send a result to the other FPU port.

IF (@INSTR(,%yarch,<NOT_BEST_FMA3>) NE 0)

yr4_4c_djbfft_noswiz_unroll_preload MACRO
	vmovapd	ymm15, YMM_ONE
	ENDM

;; uops = 4 s/c loads, 8 loads, 8 stores, 4 muls, 22 fma, 11 mov = 57 uops / 4 = 14.25 clocks
yr4_4c_djbfft_noswiz_unroll MACRO iter,prev,this,next,srcreg,srcinc,srcoff,d1,d2,bcast,screg1,screg2,scoff,screg,scinc,L1pt,L1pd

;; On first call, init registers

	IF iter EQ -1
y0	TEXTEQU	<ymm0>
y1	TEXTEQU	<ymm1>
y2	TEXTEQU	<ymm2>
y3	TEXTEQU	<ymm3>
y4	TEXTEQU	<ymm4>
y5	TEXTEQU	<ymm5>
y6	TEXTEQU	<ymm6>
y7	TEXTEQU	<ymm7>
y8	TEXTEQU	<ymm8>
y9	TEXTEQU	<ymm9>
y10	TEXTEQU	<ymm10>
y11	TEXTEQU	<ymm11>
y12	TEXTEQU	<ymm12>
y13	TEXTEQU	<ymm13>
y14	TEXTEQU	<ymm14>
	ENDIF

;; On later calls, previous R2,I2,A3,B3,A4,B4,R1,c/s2 will be in y0-7.  This new R1,new R3,new I3,I1,I3,I2 will be in y8-13.
;; The remaining register is free.  Register ymm15 is reserved for the constant YMM_ONE.

this	yfmaddpd y11, y11, ymm15, y12			;; I1 + I3 (new I1)			; 1-5					n 7
this	vmovapd	y14, [srcreg+iter*srcinc+srcoff+d2+d1+32] ;; I4
this	L1prefetchw srcreg+iter*srcinc+L1pd, L1pt

prev	yfmsubpd y12, y0, y7, y1, 2			;; A2 = R2 * cosine/sine - I2		;	1-5				n 8
prev	yfmaddpd y1, y1, y7, y0				;; B2 = I2 * cosine/sine + R2		; 2-6					n 9
prev	ystore	[srcreg+(iter-1)*srcinc], y6		;; Save R1				; 15
this no bcast vmovapd y6, [screg1+iter*scinc+32]	;; cosine/sine
this bcast vbroadcastsd y6, Q [screg1+iter*scinc+scoff] ;; cosine/sine

this	yfmaddpd y7, y13, ymm15, y14, 2			;; I2 + I4 (new I2)			;	2-6				n 7
prev	yfmsubpd y0, y2, y6, y3				;; final R3 = A3 * cosine/sine - B3	; 3-7
this	L1prefetchw srcreg+iter*srcinc+d1+L1pd, L1pt

this	yfmsubpd y13, y13, ymm15, y14, 2		;; I2 - I4 (new I4)			;	3-7				n 10
prev	yfmaddpd y3, y3, y6, y2				;; final I3 = B3 * cosine/sine + A3	; 4-8

this	vmovapd	y14, [srcreg+iter*srcinc+srcoff+d1]	;; R2
this	vmovapd	y2, [srcreg+iter*srcinc+srcoff+d2+d1]	;; R4
prev	ystore	[srcreg+(iter-1)*srcinc+d2], y0		;; Save R3				; 8
this	yfmaddpd y0, y14, ymm15, y2, 2			;; R2 + R4 (new R2)			;	4-8				n 9
this	yfmsubpd y14, y14, ymm15, y2			;; R2 - R4 (new R4)			; 5-9					n 11

this no bcast vmovapd y2, [screg1+iter*scinc]		;; Sine
this bcast vbroadcastsd y2, Q [screg1+iter*scinc]	;; Sine
this	vmulpd	y9, y9, y2				;; R3S = R3 * sine			;	5-9				n 10
this	vmulpd	y10, y10, y2				;; I3S = I3 * sine			; 6-10					n 11
this	L1prefetchw srcreg+iter*srcinc+d2+L1pd, L1pt

prev	ystore	[srcreg+(iter-1)*srcinc+d2+32], y3	;; Save I3				; 9
prev	yfmaddpd y3, y4, y6, y5, 2			;; final R4 = A4 * cosine/sine + B4	;	6-10
prev	yfmsubpd y5, y5, y6, y4				;; final I4 = B4 * cosine/sine - A4	; 7-11

prev no bcast vmovapd y6, [screg2+(iter-1)*scinc]	;; Sine
prev bcast vbroadcastsd y6, Q [screg2+(iter-1)*scinc]	;; Sine
this	yfmsubpd y4, y11, ymm15, y7, 2			;; I1 - I2 (newer I2)			;	7-11				n 14
this	yfmaddpd y11, y11, ymm15, y7			;; I1 + I2 (newer & final I1)		; 8-12
this	L1prefetchw srcreg+iter*srcinc+d2+d1+L1pd, L1pt

prev	vmulpd	y12, y12, y6				;; final R2 = A2 * sine			;	8-12
prev	vmulpd	y1, y1, y6				;; final I2 = B2 * sine			; 9-13
next	vmovapd	y7, [srcreg+(iter+1)*srcinc+srcoff]	;; R1

this	yfmsubpd y6, y8, ymm15, y0, 2			;; R1 - R2 (newer R2)			;	9-13				n 14
this	yfmaddpd y8, y8, ymm15, y0			;; R1 + R2 (newer & final R1)		; 10-14
next	vmovapd	y0, [srcreg+(iter+1)*srcinc+srcoff+d2]	;; R3

prev	ystore	[srcreg+(iter-1)*srcinc+d2+d1], y3	;; Save R4				; 11
this	yfnmaddpd y3, y13, y2, y9, 2			;; A3 = R3S - I4 * sine (newer R3*sine)	;	10-14				n 16
prev	ystore	[srcreg+(iter-1)*srcinc+d2+d1+32], y5	;; Save I4				; 12
this	yfmaddpd y5, y14, y2, y10, 2			;; B3 = I3S + R4 * sine (newer I3*sine)	; 11-15					n 16
this next yloop_unrolled_one

this	yfmaddpd y13, y13, y2, y9			;; A4 = R3S + I4 * sine (newer R4*sine)	;	11-15				n 19
next	vmovapd	y9, [srcreg+(iter+1)*srcinc+srcoff+32]	;; I1
this	yfnmaddpd y14, y14, y2, y10			;; B4 = I3S - R4 * sine (newer I4*sine)	; 12-16					n 19
next	vmovapd	y2, [srcreg+(iter+1)*srcinc+srcoff+d2+32] ;; I3

next	yfmsubpd y10, y7, ymm15, y0, 2			;; R1 - R3 (new R3)			;	12-16				n 18
next	yfmaddpd y7, y7, ymm15, y0			;; R1 + R3 (new R1)			; 13-17					n 22
this no bcast vmovapd y0, [screg2+iter*scinc+32]	;; cosine/sine
this bcast vbroadcastsd y0, Q [screg2+iter*scinc+scoff]	;; cosine/sine

this	ystore	[srcreg+iter*srcinc+32], y11		;; Save I1				; 13
next	yfmsubpd y11, y9, ymm15, y2, 2			;; I1 - I3 (new I3)			;	13-17				n 19
prev	ystore	[srcreg+(iter-1)*srcinc+d1], y12	;; Save R2				; 13
next	vmovapd	y12, [srcreg+(iter+1)*srcinc+srcoff+d1+32] ;; I2
prev	ystore	[srcreg+(iter-1)*srcinc+d1+32], y1	;; Save I2				; 14

;; Shuffle register assignments so that this R2,I2,A3,B3,A4,B4,R1,c/s2 are in y0-7, and next new R1,new R3,new I3,I1,I3,I2 are in q8-13.

ytmp	TEXTEQU	y0
y0	TEXTEQU	y6
y6	TEXTEQU	y8
y8	TEXTEQU	y7
y7	TEXTEQU	ytmp
ytmp	TEXTEQU	y1
y1	TEXTEQU	y4
y4	TEXTEQU	y13
y13	TEXTEQU	y12
y12	TEXTEQU	y2
y2	TEXTEQU	y3
y3	TEXTEQU	y5
y5	TEXTEQU y14
y14	TEXTEQU	ytmp
ytmp	TEXTEQU	y9
y9	TEXTEQU y10
y10	TEXTEQU	y11
y11	TEXTEQU ytmp
	ENDM

ENDIF


yr4_4c_djbfft_swiz_cmn MACRO srcreg,srcinc,srcoff,d1,d2,bcast,screg1,screg2,scoff,screg,scinc,maxrpt,L1pt,L1pd
	IF maxrpt MOD 5 EQ 0
	yr4_4c_djbfft_swiz_unroll -1,noexec,noexec,exec,srcreg,srcinc,srcoff,d1,d2,bcast,screg1,screg2,scoff,screg,scinc,L1pt,L1pd
	yr4_4c_djbfft_swiz_unroll 0,noexec,exec,exec,srcreg,srcinc,srcoff,d1,d2,bcast,screg1,screg2,scoff,screg,scinc,L1pt,L1pd
	yr4_4c_djbfft_swiz_unroll 1,exec,exec,exec,srcreg,srcinc,srcoff,d1,d2,bcast,screg1,screg2,scoff,screg,scinc,L1pt,L1pd
	yr4_4c_djbfft_swiz_unroll 2,exec,exec,exec,srcreg,srcinc,srcoff,d1,d2,bcast,screg1,screg2,scoff,screg,scinc,L1pt,L1pd
	yr4_4c_djbfft_swiz_unroll 3,exec,exec,exec,srcreg,srcinc,srcoff,d1,d2,bcast,screg1,screg2,scoff,screg,scinc,L1pt,L1pd
	yr4_4c_djbfft_swiz_unroll 4,exec,exec,noexec,srcreg,srcinc,srcoff,d1,d2,bcast,screg1,screg2,scoff,screg,scinc,L1pt,L1pd
	yr4_4c_djbfft_swiz_unroll 5,exec,noexec,noexec,srcreg,srcinc,srcoff,d1,d2,bcast,screg1,screg2,scoff,screg,scinc,L1pt,L1pd
	bump	srcreg, 5*srcinc
	bump	screg, 5*scinc
	ELSEIF maxrpt MOD 4 EQ 0
	yr4_4c_djbfft_swiz_unroll -1,noexec,noexec,exec,srcreg,srcinc,srcoff,d1,d2,bcast,screg1,screg2,scoff,screg,scinc,L1pt,L1pd
	yr4_4c_djbfft_swiz_unroll 0,noexec,exec,exec,srcreg,srcinc,srcoff,d1,d2,bcast,screg1,screg2,scoff,screg,scinc,L1pt,L1pd
	yr4_4c_djbfft_swiz_unroll 1,exec,exec,exec,srcreg,srcinc,srcoff,d1,d2,bcast,screg1,screg2,scoff,screg,scinc,L1pt,L1pd
	yr4_4c_djbfft_swiz_unroll 2,exec,exec,exec,srcreg,srcinc,srcoff,d1,d2,bcast,screg1,screg2,scoff,screg,scinc,L1pt,L1pd
	yr4_4c_djbfft_swiz_unroll 3,exec,exec,noexec,srcreg,srcinc,srcoff,d1,d2,bcast,screg1,screg2,scoff,screg,scinc,L1pt,L1pd
	yr4_4c_djbfft_swiz_unroll 4,exec,noexec,noexec,srcreg,srcinc,srcoff,d1,d2,bcast,screg1,screg2,scoff,screg,scinc,L1pt,L1pd
	bump	srcreg, 4*srcinc
	bump	screg, 4*scinc
	ELSEIF maxrpt MOD 3 EQ 0
	yr4_4c_djbfft_swiz_unroll -1,noexec,noexec,exec,srcreg,srcinc,srcoff,d1,d2,bcast,screg1,screg2,scoff,screg,scinc,L1pt,L1pd
	yr4_4c_djbfft_swiz_unroll 0,noexec,exec,exec,srcreg,srcinc,srcoff,d1,d2,bcast,screg1,screg2,scoff,screg,scinc,L1pt,L1pd
	yr4_4c_djbfft_swiz_unroll 1,exec,exec,exec,srcreg,srcinc,srcoff,d1,d2,bcast,screg1,screg2,scoff,screg,scinc,L1pt,L1pd
	yr4_4c_djbfft_swiz_unroll 2,exec,exec,noexec,srcreg,srcinc,srcoff,d1,d2,bcast,screg1,screg2,scoff,screg,scinc,L1pt,L1pd
	yr4_4c_djbfft_swiz_unroll 3,exec,noexec,noexec,srcreg,srcinc,srcoff,d1,d2,bcast,screg1,screg2,scoff,screg,scinc,L1pt,L1pd
	bump	srcreg, 3*srcinc
	bump	screg, 3*scinc
	ELSEIF maxrpt MOD 2 EQ 0
	yr4_4c_djbfft_swiz_unroll -1,noexec,noexec,exec,srcreg,srcinc,srcoff,d1,d2,bcast,screg1,screg2,scoff,screg,scinc,L1pt,L1pd
	yr4_4c_djbfft_swiz_unroll 0,noexec,exec,exec,srcreg,srcinc,srcoff,d1,d2,bcast,screg1,screg2,scoff,screg,scinc,L1pt,L1pd
	yr4_4c_djbfft_swiz_unroll 1,exec,exec,noexec,srcreg,srcinc,srcoff,d1,d2,bcast,screg1,screg2,scoff,screg,scinc,L1pt,L1pd
	yr4_4c_djbfft_swiz_unroll 2,exec,noexec,noexec,srcreg,srcinc,srcoff,d1,d2,bcast,screg1,screg2,scoff,screg,scinc,L1pt,L1pd
	bump	srcreg, 2*srcinc
	bump	screg, 2*scinc
	ELSE
	yr4_4c_djbfft_swiz_unroll -1,noexec,noexec,exec,srcreg,srcinc,srcoff,d1,d2,bcast,screg1,screg2,scoff,screg,scinc,L1pt,L1pd
	yr4_4c_djbfft_swiz_unroll 0,noexec,exec,noexec,srcreg,srcinc,srcoff,d1,d2,bcast,screg1,screg2,scoff,screg,scinc,L1pt,L1pd
	yr4_4c_djbfft_swiz_unroll 1,exec,noexec,noexec,srcreg,srcinc,srcoff,d1,d2,bcast,screg1,screg2,scoff,screg,scinc,L1pt,L1pd
	bump	srcreg, srcinc
	bump	screg, scinc
	ENDIF
	ENDM

yr4_4c_djbfft_swiz_unroll_preload MACRO
	ENDM

yr4_4c_djbfft_swiz_unroll MACRO iter,prev,this,next,srcreg,srcinc,srcoff,d1,d2,bcast,screg1,screg2,scoff,screg,scinc,L1pt,L1pd

;; On first call, init registers

	IF iter EQ -1
y0	TEXTEQU	<ymm0>
y1	TEXTEQU	<ymm1>
y2	TEXTEQU	<ymm2>
y3	TEXTEQU	<ymm3>
y4	TEXTEQU	<ymm4>
y5	TEXTEQU	<ymm5>
y6	TEXTEQU	<ymm6>
y7	TEXTEQU	<ymm7>
y8	TEXTEQU	<ymm8>
y9	TEXTEQU	<ymm9>
y10	TEXTEQU	<ymm10>
y11	TEXTEQU	<ymm11>
y12	TEXTEQU	<ymm12>
y13	TEXTEQU	<ymm13>
y14	TEXTEQU	<ymm14>
y15	TEXTEQU	<ymm15>
	ENDIF

;; On later calls, previous R1,I1,R3,R4,R2,B3,B4,B2,sine1,sine2 will be in y0-9.  This R1,R3,I2,I4 will be in y10-13.
;; The remaining registers are free.

this	vsubpd	y14, y10, y11			;; R1 - R3 (new R3)			; 1-3
prev	vmulpd	y5, y5, y8			;; B3 = B3 * sine (final I3)		; 23-27
this	vmovapd	y15, [srcreg+iter*srcinc+srcoff+d1] ;; R2

this	vaddpd	y10, y10, y11			;; R1 + R3 (new R1)			; 2-4
prev	vshufpd	y11, y2, y3, 0			;; Shuffle R3 and R4 to create R3/R4 low ; 24
prev	vmulpd	y6, y6, y8			;; B4 = B4 * sine (final I4)		; 24-28
this	L1prefetchw srcreg+iter*srcinc+L1pd, L1pt

this	vsubpd	y8, y12, y13			;; I2 - I4 (new I4)			; 3-5
prev	vshufpd	y2, y2, y3, 15			;; Shuffle R3 and R4 to create R3/R4 hi	; 25
prev	vmulpd	y7, y7, y9			;; B2 = B2 * sine (final I2)		; 25-29
this	vmovapd	y3, [srcreg+iter*srcinc+srcoff+d2+d1] ;; R4

this	vaddpd	y9, y15, y3			;; R2 + R4 (new R2)			; 4-6
this	vsubpd	y15, y15, y3			;; R2 - R4 (new R4)			; 5-7
prev	vshufpd	y3, y0, y4, 0			;; Shuffle R1 and R2 to create R1/R2 low ; 26
prev	vshufpd	y0, y0, y4, 15			;; Shuffle R1 and R2 to create R1/R2 hi	; 27
this	L1prefetchw srcreg+iter*srcinc+d1+L1pd, L1pt

prev	ylow128s y4, y3, y11			;; Shuffle R1/R2 low and R3/R4 low (new R1) ; 28-29
prev	ystore	[srcreg+(iter-1)*srcinc], y4	;; Save R1				; 30
this	vsubpd	y4, y14, y8			;; R3 - I4 (final R3)			; 6-8

this	vaddpd	y14, y14, y8			;; R3 + I4 (final R4)			; 7-9
prev	yhigh128s y3, y3, y11			;; Shuffle R1/R2 low and R3/R4 low (new R3) ; 29-30

this	vsubpd	y11, y10, y9			;; R1 - R2 (final R2)			; 8-10
prev	ylow128s y8, y0, y2			;; Shuffle R1/R2 hi and R3/R4 hi (new R2) ; 30-31

prev	yhigh128s y0, y0, y2			;; Shuffle R1/R2 hi and R3/R4 hi (new R4) ; 31-32
this	vmovapd	y2, [srcreg+iter*srcinc+srcoff+32] ;; I1
prev	ystore	[srcreg+(iter-1)*srcinc+d2], y3	;; Save R3				; 31
this	vmovapd	y3, [srcreg+iter*srcinc+srcoff+d2+32] ;; I3
prev	ystore	[srcreg+(iter-1)*srcinc+d1], y8	;; Save R2				; 32
this	vsubpd	y8, y2, y3			;; I1 - I3 (new I3)			; 9-11

this	vaddpd	y12, y12, y13			;; I2 + I4 (new I2)			; 10-12
this no bcast vmovapd y13, [screg1+iter*scinc+32] ;; cosine/sine
this bcast vbroadcastsd y13, Q [screg1+iter*scinc+scoff] ;; cosine/sine
prev	ystore	[srcreg+(iter-1)*srcinc+d2+d1], y0 ;; Save R4				; 33
this	vmulpd	y0, y4, y13			;; A3 = R3 * cosine/sine		; 10-14

this	vaddpd	y2, y2, y3			;; I1 + I3 (new I1)			; 11-13
prev	vshufpd	y3, y5, y6, 0			;; Shuffle I3 and I4 to create I3/I4 low ; 33
prev	vshufpd	y5, y5, y6, 15			;; Shuffle I3 and I4 to create I3/I4 hi	; 34
prev	vshufpd	y6, y1, y7, 0			;; Shuffle I1 and I2 to create I1/I2 low ; 35
prev	vshufpd	y1, y1, y7, 15			;; Shuffle I1 and I2 to create I1/I2 hi	; 36
prev	ylow128s y7, y6, y3			;; Shuffle I1/I2 low and I3/I4 low (new I1) ; 37-38
prev	ystore	[srcreg+(iter-1)*srcinc+32], y7	;; Save I1				; 39
this	vmulpd	y7, y14, y13			;; A4 = R4 * cosine/sine		; 11-15

prev	yhigh128s y6, y6, y3			;; Shuffle I1/I2 low and I3/I4 low (new I3) ; 38-39
this	vaddpd	y3, y8, y15			;; I3 + R4 (final I3)			; 12-14
prev	ystore	[srcreg+(iter-1)*srcinc+d2+32], y6 ;; Save I3				; 40
this no bcast vmovapd y6, [screg2+iter*scinc+32] ;; cosine/sine
this bcast vbroadcastsd y6, Q [screg2+iter*scinc+scoff]	;; cosine/sine
this	vsubpd	y8, y8, y15			;; I3 - R4 (final I4)			; 13-15
prev	ylow128s y15, y1, y5			;; Shuffle I1/I2 hi and I3/I4 hi (new I2) ; 39-40
prev	yhigh128s y1, y1, y5			;; Shuffle I1/I2 hi and I3/I4 hi (new I4) ; 40-41
this	vmulpd	y5, y11, y6			;; A2 = R2 * cosine/sine		; 12-16

prev	ystore	[srcreg+(iter-1)*srcinc+d1+32], y15 ;; Save I2				; 41
this	vsubpd	y15, y2, y12			;; I1 - I2 (final I2)			; 14-16

this	vsubpd	y0, y0, y3			;; A3 = A3 - I3				; 15-17
this	vmulpd	y3, y3, y13			;; B3 = I3 * cosine/sine		; 15-19
this	L1prefetchw srcreg+iter*srcinc+d2+L1pd, L1pt

this	vaddpd	y7, y7, y8			;; A4 = A4 + I4				; 16-18
this	vmulpd	y8, y8, y13			;; B4 = I4 * cosine/sine		; 16-20
this no bcast vmovapd y13, [screg1+iter*scinc]	;; Sine
this bcast vbroadcastsd y13, Q [screg1+iter*scinc] ;; Sine

this	vsubpd	y5, y5, y15			;; A2 = A2 - I2				; 17-19
this	vmulpd	y15, y15, y6			;; B2 = I2 * cosine/sine		; 17-21
this no bcast vmovapd y6, [screg2+iter*scinc]	;; Sine
this bcast vbroadcastsd y6, Q [screg2+iter*scinc] ;; Sine

this	vaddpd	y10, y10, y9			;; R1 + R2 (final R1)			; 18-20
this	vmulpd	y0, y0, y13			;; A3 = A3 * sine (final R3)		; 18-22
prev	ystore	[srcreg+(iter-1)*srcinc+d2+d1+32], y1 ;; Save I4			; 42
this	L1prefetchw srcreg+iter*srcinc+d2+d1+L1pd, L1pt

this	vaddpd	y2, y2, y12			;; I1 + I2 (final I1)			; 19-21
this	vmulpd	y7, y7, y13			;; A4 = A4 * sine (final R4)		; 19-23
next	vmovapd	y12, [srcreg+(iter+1)*srcinc+srcoff] ;; R1

this	vaddpd	y3, y3, y4			;; B3 = B3 + R3				; 20-22
this	vmulpd	y5, y5, y6			;; A2 = A2 * sine (final R2)		; 20-24
next	vmovapd	y4, [srcreg+(iter+1)*srcinc+srcoff+d2] ;; R3

this	vsubpd	y8, y8, y14			;; B4 = B4 - R4				; 21-23
next	vmovapd	y14, [srcreg+(iter+1)*srcinc+srcoff+d1+32] ;; I2
this next yloop_unrolled_one

this	vaddpd	y15, y15, y11			;; B2 = B2 + R2				; 22-24
next	vmovapd	y11, [srcreg+(iter+1)*srcinc+srcoff+d2+d1+32] ;; I4

;; Shuffle register assignments so that next call has R1,I1,R3,R4,R2,B3,B4,B2,sine1,sine2 in y0-9 and next R1,R3,I2,I4 will be in y10-13.

ytmp	TEXTEQU	y0
y0	TEXTEQU	y10
y10	TEXTEQU	y12
y12	TEXTEQU	y14
y14	TEXTEQU	y9
y9	TEXTEQU	y6
y6	TEXTEQU	y8
y8	TEXTEQU	y13
y13	TEXTEQU	y11
y11	TEXTEQU	y4
y4	TEXTEQU	y5
y5	TEXTEQU y3
y3	TEXTEQU	y7
y7	TEXTEQU	y15
y15	TEXTEQU	y1
y1	TEXTEQU	y2
y2	TEXTEQU ytmp
	ENDM


;; Haswell FMA3 version
;; This is the basic Bulldozer version

IF (@INSTR(,%yarch,<NOT_BEST_FMA3>) NE 0)

yr4_4c_djbfft_swiz_unroll MACRO iter,prev,this,next,srcreg,srcinc,srcoff,d1,d2,bcast,screg1,screg2,scoff,screg,scinc,L1pt,L1pd

;; On first call, init registers

	IF iter EQ -1
y0	TEXTEQU	<ymm0>
y1	TEXTEQU	<ymm1>
y2	TEXTEQU	<ymm2>
y3	TEXTEQU	<ymm3>
y4	TEXTEQU	<ymm4>
y5	TEXTEQU	<ymm5>
y6	TEXTEQU	<ymm6>
y7	TEXTEQU	<ymm7>
y8	TEXTEQU	<ymm8>
y9	TEXTEQU	<ymm9>
y10	TEXTEQU	<ymm10>
y11	TEXTEQU	<ymm11>
y12	TEXTEQU	<ymm12>
y13	TEXTEQU	<ymm13>
y14	TEXTEQU	<ymm14>
y15	TEXTEQU	<ymm15>
	ENDIF

;; On later calls, previous R1,I1,R2,I2,R3,I3,R4,I4 will be in y0-7.  This R1,I1,R2,I2,R3,I3,I4 will be in y8-14.
;; The remaining registers are free.

prev	vshufpd	y15, y4, y6, 0				;; Shuffle R3 and R4 to create R3/R4 lo	; 1-2,2-3 port 1
prev	ystorelo [srcreg+(iter-1)*srcinc][16], y15	;; Save R1 hi				; 3
prev	vshufpd	y4, y4, y6, 15				;; Shuffle R3 and R4 to create R3/R4 hi	; 3-4,4-5 port 1
prev	ystorehi [srcreg+(iter-1)*srcinc+d2][16], y15	;; Save R3 hi				; 4
this	vaddpd	y6, y11, y14				;; I2 + I4 (new I2)			; 1-5,2-6 port 0	needed 13
this	vmovapd	y15, [srcreg+iter*srcinc+srcoff+d2+d1]	;; R4
this	vsubpd	y11, y11, y14				;; I2 - I4 (new I4)			; 3-7,4-8 port 0
prev	ystorelo [srcreg+(iter-1)*srcinc+d1][16], y4	;; Save R2 hi				; 5

prev	vshufpd	y14, y0, y2, 0				;; Shuffle R1 and R2 to create R1/R2 lo	; 5-6,6-7 port 1
prev	ystorehi [srcreg+(iter-1)*srcinc+d2+d1][16], y4 ;; Save R4 hi				; 6
this	vaddpd	y4, y10, y15				;; R2 + R4 (new R2)			; 5-9,6-10 port 0	needed 17
prev	ystorelo [srcreg+(iter-1)*srcinc], y14		;; Save R1 lo				; 7
prev	vshufpd	y0, y0, y2, 15				;; Shuffle R1 and R2 to create R1/R2 hi	; 7-8,8-9 port 1
this no bcast vmovapd y2, [screg1+iter*scinc]		;; Sine for R3/I3/R4/I4
this bcast vbroadcastsd y2, Q [screg1+iter*scinc]	;; Sine
this	vsubpd	y10, y10, y15				;; R2 - R4 (new R4)			; 7-11,8-12 port 0
prev	ystorehi [srcreg+(iter-1)*srcinc+d2], y14	;; Save R3 lo				; 8

prev	vshufpd	y15, y5, y7, 0				;; Shuffle I3 and I4 to create I3/I4 lo	; 9-10,10-11 port 1
prev	ystorelo [srcreg+(iter-1)*srcinc+d1], y0	;; Save R2 lo				; 9
this	vmulpd	y12, y12, y2				;; R3S = R3 * sine			; 9-13,10-14 port 0	needed 19
prev	ystorehi [srcreg+(iter-1)*srcinc+d2+d1], y0	;; Save R4 lo				; 10
prev	vshufpd	y5, y5, y7, 15				;; Shuffle I3 and I4 to create I3/I4 hi	; 11-12,12-13 port 1
prev	ystorelo [srcreg+(iter-1)*srcinc+32][16], y15	;; Save I1 hi				; 11
this	vmulpd	y13, y13, y2				;; I3S = I3 * sine			; 11-15,12-16 port 0
prev	ystorehi [srcreg+(iter-1)*srcinc+d2+32][16], y15 ;; Save I3 hi				; 12

prev	vshufpd	y14, y1, y3, 0				;; Shuffle I1 and I2 to create I1/I2 lo	; 13-14,14-15 port 1
prev	ystorelo [srcreg+(iter-1)*srcinc+d1+32][16], y5 ;; Save I2 hi				; 13
this	vsubpd	y0, y9, y6				;; I1 - I2 (newer I2)			; 13-17,14-18 port 0	needed 24
prev	ystorehi [srcreg+(iter-1)*srcinc+d2+d1+32][16], y5 ;; Save I4 hi			; 14
prev	vshufpd	y1, y1, y3, 15				;; Shuffle I1 and I2 to create I1/I2 hi	; 15-16,16-17 port 1
prev	ystorelo [srcreg+(iter-1)*srcinc+32], y14	;; Save I1 lo				; 15
this	vaddpd	y9, y9, y6				;; I1 + I2 (newer & final I1)		; 15-21,16-22 port 0
prev	ystorehi [srcreg+(iter-1)*srcinc+d2+32], y14	;; Save I3 lo				; 16

this	vsubpd	y7, y8, y4				;; R1 - R2 (newer R2)			; 17-21			needed 23
prev	ystorelo [srcreg+(iter-1)*srcinc+d1+32], y1	;; Save I2 lo				; 17
this	vaddpd	y8, y8, y4				;; R1 + R2 (newer & final R1)		; 18-24
prev	ystorehi [srcreg+(iter-1)*srcinc+d2+d1+32], y1	;; Save I4 lo				; 18

this	yfnmaddpd y15, y11, y2, y12			;; R3S - I4 * sine (newer R3S)		; 19-23			needed 26
this no bcast vmovapd y5, [screg2+iter*scinc+32]	;; cosine/sine for R2/I2
this bcast vbroadcastsd y5, Q [screg2+iter*scinc+scoff]	;; cosine/sine
this	yfmaddpd y11, y11, y2, y12			;; R3S + I4 * sine (newer R4S)		; 20-24
next	vmovapd	y3, [srcreg+(iter+1)*srcinc+srcoff]	;; R1

this	yfmaddpd y6, y10, y2, y13			;; I3S + R4 * sine (newer I3S)		; 21-25			needed 26
next	vmovapd	y14, [srcreg+(iter+1)*srcinc+srcoff+d2]	;; R3
this	yfnmaddpd y10, y10, y2, y13			;; I3S - R4 * sine (newer I4S)		; 22-26
this no bcast vmovapd y4, [screg1+iter*scinc+32]	;; cosine/sine for R3/I3/R4/I4
this bcast vbroadcastsd y4, Q [screg1+iter*scinc+scoff] ;; cosine/sine

this	yfmsubpd y1, y7, y5, y0				;; A2 = R2 * cosine/sine - I2		; 23-27  5 clks		needed 32
this	L1prefetchw srcreg+iter*srcinc+L1pd, L1pt
this	yfmaddpd y0, y0, y5, y7				;; B2 = I2 * cosine/sine + R2		; 24-28
this no bcast vmovapd y12, [screg2+iter*scinc]		;; Sine for R2/I2
this bcast vbroadcastsd y12, Q [screg2+iter*scinc]	;; Sine

next	vsubpd	y2, y3, y14				;; R1 - R3 (new R3)			; 25-29		12
this	L1prefetchw srcreg+iter*srcinc+d1+L1pd, L1pt

this	yfmsubpd y13, y15, y4, y6			;; R3S * cosine/sine - I3S (final R3)	; 26-32	 7 clocks!	needed next 1
next	vmovapd	y5, [srcreg+(iter+1)*srcinc+srcoff+32]	;; I1
this	yfmaddpd y6, y6, y4, y15			;; I3S * cosine/sine + R3S (final I3)	; 27-33
next	vmovapd	y7, [srcreg+(iter+1)*srcinc+srcoff+d2+32] ;; I3

this	yfmaddpd y15, y11, y4, y10			;; R4S * cosine/sine + I4S (final R4)	; 28-34	 7 clocks!	needed next 1		
this	L1prefetchw srcreg+iter*srcinc+d2+L1pd, L1pt
this	yfmsubpd y10, y10, y4, y11			;; I4S * cosine/sine - R4S (final I4)	; 29-35
next	vmovapd	y4, [srcreg+(iter+1)*srcinc+srcoff+d1+32] ;; I2

this	vmulpd	y1, y1, y12				;; A2 * sine (final R2)			; 30-36	 7 clocks!	needed next 5		
next	vmovapd	y11, [srcreg+(iter+1)*srcinc+srcoff+d2+d1+32] ;; I4
this	vmulpd	y0, y0, y12				;; B2 * sine (final I2)			; 31-37
this	L1prefetchw srcreg+iter*srcinc+d2+d1+L1pd, L1pt

next	vaddpd	y3, y3, y14				;; R1 + R3 (new R1)			; 32-36
next	vmovapd	y12, [srcreg+(iter+1)*srcinc+srcoff+d1]	;; R2

next	vsubpd	y14, y5, y7				;; I1 - I3 (new I3)			; 33-37			needed 11
this next yloop_unrolled_one
next	vaddpd	y5, y5, y7				;; I1 + I3 (new I1)			; 34-38

;; Shuffle register assignments so that this R1,I1,R2,I2,R3,I3,R4,I4 are in y0-7 and next R1,I1,R2,I2,R3,I3,I4 are in y8-14.

ytmp	TEXTEQU	y0
y0	TEXTEQU	y8
y8	TEXTEQU	y3
y3	TEXTEQU	ytmp
ytmp	TEXTEQU	y1
y1	TEXTEQU	y9
y9	TEXTEQU	y5
y5	TEXTEQU	y6
y6	TEXTEQU	y15
y15	TEXTEQU	y7
y7	TEXTEQU	y10
y10	TEXTEQU y12
y12	TEXTEQU	y2
y2	TEXTEQU	ytmp
ytmp	TEXTEQU	y4
y4	TEXTEQU	y13
y13	TEXTEQU	y14
y14	TEXTEQU	y11
y11	TEXTEQU ytmp
	ENDM

ENDIF

;; Haswell FMA3 version

IF (@INSTR(,%yarch,<FMA3>) NE 0)

;; Studying various Haswell options for this macro proved very interesting.  We have to worry about clogging port 5.
;; First we tested how to best swizzle and store the data:
;;	16 clocks for 8 shufs, then 16 ystorehi/lo
;;	18 clocks for 8 shufs, 8 permf128s, 8 stores  (the shuf latency 1 vs. permf128 latency 3 causes 2 clock bubble on port 5)
;;	15 clocks(!) for 8 shufs, 8 storehilo, 4 permf128s, 4 stores
;;	14 clocks(!!) for  8 shufs, 12 storehilo, 2 permf128s, 2 stores
;; Now we add in 26 muls to simulate the 26 FMA instructions
;;	16 clocks, 26 muls, 8 shufs, 16 storehilo
;;	15+ clocks, 26 muls, 8 shufs, 12 storehilo, 2 permf128s, 2 stores
;; Now we add in the 11 register copies that will be required because we don't have an FMA4 instruction
;;	17.7 clocks, 26 muls, 11 reg copies, 8 shufs, 12 storehilo, 2 permf128s, 2 stores
;; Now we tried 6 add instructions to save 6 reg copies (in hopes that we can work around the scheduling bubbles the adds will introduce)
;;	17 clocks, 6 adds, 20 muls, 5 reg copies, 8 shufs, 16 storehilo
;; Finally, we added 8 load instructions to fully simulate this macro
;;	19 clocks, 6 adds, 20 muls, 5 reg copies, 8 shufs, 16 storehilo, 8 loads
;;	18.4 clocks, 6 adds, 20 muls, 5 reg copies, 8 shufs, 12 storehilo, 2 permf128s, 2 stores, 8 loads
;;	17.4 clocks, 6 adds, 20 muls, 5 reg copies, 8 shufs, 8 storehilo, 4 permf128s, 4 stores, 8 loads
;;	18.4 clocks, 6 adds, 20 muls, 5 reg copies, 8 shufs, 4 storehilo, 6 permf128s, 6 stores, 8 loads
;; Conclusions:  We will shoot for a 17 clock version that uses 6 add instructions.

;; Sadly, this macro has been timed at 20.3 clocks

yr4_4c_djbfft_swiz_unroll_preload MACRO
	vmovapd	ymm15, YMM_ONE
	ENDM

yr4_4c_djbfft_swiz_unroll MACRO iter,prev,this,next,srcreg,srcinc,srcoff,d1,d2,bcast,screg1,screg2,scoff,screg,scinc,L1pt,L1pd

;; On first call, init registers

	IF iter EQ -1
y0	TEXTEQU	<ymm0>
y1	TEXTEQU	<ymm1>
y2	TEXTEQU	<ymm2>
y3	TEXTEQU	<ymm3>
y4	TEXTEQU	<ymm4>
y5	TEXTEQU	<ymm5>
y6	TEXTEQU	<ymm6>
y7	TEXTEQU	<ymm7>
y8	TEXTEQU	<ymm8>
y9	TEXTEQU	<ymm9>
y10	TEXTEQU	<ymm10>
y11	TEXTEQU	<ymm11>
y12	TEXTEQU	<ymm12>
y13	TEXTEQU	<ymm13>
y14	TEXTEQU	<ymm14>
	ENDIF

;; On later calls, previous R1,I1,R2,I2,R3,I3,R4,I4 will be in y0-7.  This R1,R3,I1,I2,I3,I4 will be in y8-13.
;; The remaining registers are free.  Register y15 is used for the constant YMM_ONE.

this	vaddpd	y14, y11, y13				;; I2 + I4 (new I2)				; 1-3				needed 4
this	yfmsubpd y11, y11, ymm15, y13			;; I2 - I4 (new I4)				;	1-5			needed 9
prev	vshufpd	y13, y0, y2, 0				;; Shuffle R1 and R2 to create R1/R2 low	;		1

prev	vshufpd	y0, y0, y2, 15				;; Shuffle R1 and R2 to create R1/R2 hi		;		2
this	vmovapd	y2, [srcreg+iter*srcinc+srcoff+d1]	;; R2
prev	ystorelo [srcreg+(iter-1)*srcinc], y13		;; Save R1 lo					;			2
prev	ystorehi [srcreg+(iter-1)*srcinc+d2], y13	;; Save R3 lo					;			3
this	vmovapd	y13, [srcreg+iter*srcinc+srcoff+d2+d1]	;; R4
prev	ystorelo [srcreg+(iter-1)*srcinc+d1], y0	;; Save R2 lo					;			4
prev	ystorehi [srcreg+(iter-1)*srcinc+d2+d1], y0	;; Save R4 lo					;			5
prev	vshufpd	y0, y4, y6, 0				;; Shuffle R3 and R4 to create R3/R4 low	;		3
prev	vshufpd	y4, y4, y6, 15				;; Shuffle R3 and R4 to create R3/R4 hi		;		4
this	vaddpd	y6, y2, y13				;; R2 + R4 (new R2)				; 2-4				needed 5
prev	ystorelo [srcreg+(iter-1)*srcinc][16], y0	;; Save R1 hi					;			6
prev	ystorehi [srcreg+(iter-1)*srcinc+d2][16], y0	;; Save R3 hi					;			7
this no bcast vmovapd y0, [screg1+iter*scinc]		;; Sine for R3/I3/R4/I4
this bcast vbroadcastsd y0, Q [screg1+iter*scinc]	;; Sine
this	vmulpd	y12, y12, y0				;; I3S = I3 * sine				;	2-6			needed 7

this	yfmsubpd y2, y2, ymm15, y13			;; R2 - R4 (new R4)				; 3-5				needed 7
this	vmulpd	y9, y9, y0				;; R3S = R3 * sine				;	3-7			needed 9
this	L1prefetchw srcreg+iter*srcinc+L1pd, L1pt

this	vsubpd	y13, y10, y14				;; I1 - I2 (newer I2)				; 4-6				needed 8
this	yfmaddpd y10, y10, ymm15, y14			;; I1 + I2 (newer & final I1)			;	4-8			needed next 7

this	vsubpd	y14, y8, y6				;; R1 - R2 (newer R2)				; 5-7				needed 8
this	yfmaddpd y8, y8, ymm15, y6			;; R1 + R2 (newer & final R1)			;	5-9			needed next 1
this	L1prefetchw srcreg+iter*srcinc+d1+L1pd, L1pt

prev	vshufpd	y6, y5, y7, 0				;; Shuffle I3 and I4 to create I3/I4 low	;		5
prev	vshufpd	y5, y5, y7, 15				;; Shuffle I3 and I4 to create I3/I4 hi		;		6
prev	vshufpd	y7, y1, y3, 0				;; Shuffle I1 and I2 to create I1/I2 low	;		7
prev	vshufpd	y1, y1, y3, 15				;; Shuffle I1 and I2 to create I1/I2 hi		;		8
next	vmovapd	y3, [srcreg+(iter+1)*srcinc+srcoff+32]	;; I1
prev	ystorelo [srcreg+(iter-1)*srcinc+d1][16], y4	;; Save R2 hi					;			8
prev	ystorehi [srcreg+(iter-1)*srcinc+d2+d1][16], y4 ;; Save R4 hi					;			9
prev	ylow128s y4, y7, y6				;; Shuffle I1/I2 low and I3/I4 low (new I1)	;		9-10
prev	ystore	[srcreg+(iter-1)*srcinc+32], y4		;; Save I1					;			11
next	vmovapd	y4, [srcreg+(iter+1)*srcinc+srcoff+d2+32] ;; I3
prev	yhigh128s y7, y7, y6				;; Shuffle I1/I2 low and I3/I4 low (new I3)	;		10-11
next	vsubpd	y6, y3, y4				;; I1 - I3 (new I3)				; 6-8				needed next 2
next	yfmaddpd y3, y3, ymm15, y4			;; I1 + I3 (new I1)				;	6-10			needed next 4

this	yfmaddpd y4, y2, y0, y12			;; I3S + R4 * sine (newer I3S)			; 7-11				needed 14
this	yfnmaddpd y2, y2, y0, y12			;; I3S - R4 * sine (newer I4S)			;	7-11			needed 15

this no bcast vmovapd y12, [screg2+iter*scinc+32]	;; cosine/sine for R2/I2
this bcast vbroadcastsd y12, Q [screg2+iter*scinc+scoff] ;; cosine/sine
prev	ystore	[srcreg+(iter-1)*srcinc+d2+32], y7	;; Save I3					;			12
this	yfmsubpd y7, y14, y12, y13			;; A2 = R2 * cosine/sine - I2			; 8-12				needed 13
this	yfmaddpd y13, y13, y12, y14			;; B2 = I2 * cosine/sine + R2			;	8-12			needed 13
next	vmovapd	y12, [srcreg+(iter+1)*srcinc+srcoff]	;; R1

this	yfnmaddpd y14, y11, y0, y9			;; R3S - I4 * sine (newer R3S)			; 9-13				needed 14
this	yfmaddpd y11, y11, y0, y9			;; R3S + I4 * sine (newer R4S)			;	9-13			needed 15
this	L1prefetchw srcreg+iter*srcinc+d2+L1pd, L1pt

	;; FPU PORTS IDLE	(2 clock penalty to switch to adds)					; 10-14

this	L1prefetchw srcreg+iter*srcinc+d2+d1+L1pd, L1pt

	;; FPU PORTS IDLE	(2 clock penalty to switch to adds)					; 11-15

prev	ylow128s y0, y1, y5				;; Shuffle I1/I2 hi and I3/I4 hi (new I2)	;		11-12
prev	yhigh128s y1, y1, y5				;; Shuffle I1/I2 hi and I3/I4 hi (new I4)	;		12-13
next	vmovapd	y5, [srcreg+(iter+1)*srcinc+srcoff+d2]	;; R3
next	vsubpd	y9, y12, y5				;; R1 - R3 (new R3)				; 12-14				needed next 3
next	yfmaddpd y12, y12, ymm15, y5			;; R1 + R3 (new R1)				;	12-16			needed next 5

this no bcast vmovapd y5, [screg2+iter*scinc]		;; Sine for R2/I2
this bcast vbroadcastsd y5, Q [screg2+iter*scinc]	;; Sine
this	vmulpd	y7, y7, y5				;; A2 * sine (final R2)				; 13-17				needed next 1
this	vmulpd	y13, y13, y5				;; B2 * sine (final I2)				;	13-17			needed next 7
this no bcast vmovapd y5, [screg1+iter*scinc+32]	;; cosine/sine for R3/I3/R4/I4
this bcast vbroadcastsd y5, Q [screg1+iter*scinc+scoff] ;; cosine/sine
prev	ystore	[srcreg+(iter-1)*srcinc+d1+32], y0	;; Save I2					;			13

this	yfmsubpd y0, y14, y5, y4			;; R3S * cosine/sine - I3S (final R3)		; 14-18				needed next 3
this	yfmaddpd y4, y4, y5, y14			;; I3S * cosine/sine + R3S (final I3)		;	14-18			needed next 5
next	vmovapd	y14, [srcreg+(iter+1)*srcinc+srcoff+d1+32] ;; I2
prev	ystore	[srcreg+(iter-1)*srcinc+d2+d1+32], y1	;; Save I4					;			14

this	yfmaddpd y1, y11, y5, y2			;; R4S * cosine/sine + I4S (final R4)		; 15-19				needed next 3
this	yfmsubpd y2, y2, y5, y11			;; I4S * cosine/sine - R4S (final I4)		;	15-19			needed next 5
next	vmovapd	y5, [srcreg+(iter+1)*srcinc+srcoff+d2+d1+32] ;; I4

	;; FPU PORTS IDLE	(2 clock penalty to switch to adds)					; 16-20

this next yloop_unrolled_one

	;; FPU PORTS IDLE	(2 clock penalty to switch to adds)					; 17-21

;; Shuffle register assignments so that this R1,I1,R2,I2,R3,I3,R4,I4 are in y0-7 and next R1,R3,I1,I2,I3,I4 are in y8-13.

ytmp	TEXTEQU	y0
y0	TEXTEQU	y8
y8	TEXTEQU	y12
y12	TEXTEQU	y6
y6	TEXTEQU	y1
y1	TEXTEQU	y10
y10	TEXTEQU	y3
y3	TEXTEQU	y13
y13	TEXTEQU	y5
y5	TEXTEQU	y4
y4	TEXTEQU ytmp
ytmp	TEXTEQU	y2
y2	TEXTEQU	y7
y7	TEXTEQU	ytmp
ytmp	TEXTEQU	y11
y11	TEXTEQU	y14
y14	TEXTEQU	ytmp
	ENDM
ENDIF

ENDIF


;;
;; ************************************* four-complex-djbunfft variants ******************************************
;;

; Basic four-complex DJB inverse FFT building block
yr4_4cl_four_complex_djbunfft_preload MACRO
	yr4_4c_djbunfft_cmn_preload noexec
	ENDM
yr4_4cl_four_complex_djbunfft MACRO srcreg,srcinc,d1,d2,screg,scinc,maxrpt,L1pt,L1pd
	yr4_4c_djbunfft_cmn srcreg,srcinc,d1,d2,noexec,screg,screg+64,32,noexec,screg,scinc,maxrpt,L1pt,L1pd
	ENDM

; Like the 4cl version except vbroadcastsd is used to reduce sin/cos data
yr4_b4cl_four_complex_djbunfft_preload MACRO
	yr4_4c_djbunfft_cmn_preload noexec
	ENDM
yr4_b4cl_four_complex_djbunfft MACRO srcreg,srcinc,d1,d2,screg,scinc,maxrpt,L1pt,L1pd
	yr4_4c_djbunfft_cmn srcreg,srcinc,d1,d2,exec,screg,screg+16,8,noexec,screg,scinc,maxrpt,L1pt,L1pd
	ENDM

; Like b4cl but extracts the sin/cos data to broadcast from a earlier level's uncompressed sin/cos data
yr4_eb4cl_four_complex_djbunfft_preload MACRO
	yr4_4c_djbunfft_cmn_preload noexec
	ENDM
yr4_eb4cl_four_complex_djbunfft MACRO srcreg,srcinc,d1,d2,screg,scinc,maxrpt,L1pt,L1pd
	yr4_4c_djbunfft_cmn srcreg,srcinc,d1,d2,exec,screg,screg+64,32,noexec,screg,scinc,maxrpt,L1pt,L1pd
	ENDM

; Like b4cl but extracts the sin/cos data to broadcast from the eight-real/four_complex sin/cos table
yr4_rb4cl_four_complex_djbunfft_preload MACRO
	yr4_4c_djbunfft_cmn_preload noexec
	ENDM
yr4_rb4cl_four_complex_djbunfft MACRO srcreg,srcinc,d1,d2,screg,scinc,maxrpt,L1pt,L1pd
	yr4_4c_djbunfft_cmn srcreg,srcinc,d1,d2,exec,screg+8,screg+80,40,noexec,screg,scinc,maxrpt,L1pt,L1pd
	ENDM

; Like the 4cl version except the inputs are swizzled.
yr4_s4cl_four_complex_djbunfft_preload MACRO
	yr4_4c_djbunfft_cmn_preload exec
	ENDM
yr4_s4cl_four_complex_djbunfft MACRO srcreg,srcinc,d1,d2,screg,scinc,maxrpt,L1pt,L1pd
	yr4_4c_djbunfft_cmn srcreg,srcinc,d1,d2,noexec,screg,screg+64,32,exec,screg,scinc,maxrpt,L1pt,L1pd
	ENDM

;; Common macros to get the four complex djbunfft job done

yr4_4c_djbunfft_cmn_preload MACRO swiz
	ENDM

yr4_4c_djbunfft_cmn MACRO srcreg,srcinc,d1,d2,bcast,screg1,screg2,scoff,swiz,screg,scinc,maxrpt,L1pt,L1pd
no swiz	yr4_4c_djbunfft_noswiz srcreg,srcinc,d1,d2,bcast,screg1,screg2,scoff,screg,scinc,maxrpt,L1pt,L1pd
swiz	yr4_4c_djbunfft_swiz srcreg,srcinc,d1,d2,bcast,screg1,screg2,scoff,screg,scinc,maxrpt,L1pt,L1pd
	ENDM

yr4_4c_djbunfft_noswiz MACRO srcreg,srcinc,d1,d2,bcast,screg1,screg2,scoff,screg,scinc,maxrpt,L1pt,L1pd
no bcast vmovapd ymm3, [screg2+32]		;; cosine/sine
bcast	vbroadcastsd ymm3, Q [screg2+scoff]	;; cosine/sine
	vmovapd	ymm2, [srcreg+d1]		;; R2
	vmulpd	ymm6, ymm2, ymm3		;; A2 = R2 * cosine/sine		; 1-5
	vmovapd	ymm0, [srcreg+d1+32]		;; I2
	vmulpd	ymm3, ymm3, ymm0		;; B2 = I2 * cosine/sine		; 2-6

no bcast vmovapd ymm5, [screg1+32]		;; cosine/sine
bcast	vbroadcastsd ymm5, Q [screg1+scoff]	;; cosine/sine
	vmovapd	ymm4, [srcreg+d2]		;; R3
	vmulpd	ymm7, ymm4, ymm5		;; A3 = R3 * cosine/sine		; 3-7
	vmovapd	ymm1, [srcreg+d2+32]		;; I3

	vaddpd	ymm6, ymm6, ymm0		;; A2 = A2 + I2				; 6-8

	vmulpd	ymm0, ymm1, ymm5		;; B3 = I3 * cosine/sine		; 4-8
	L1prefetchw srcreg+L1pd, L1pt

	vsubpd	ymm3, ymm3, ymm2		;; B2 = B2 - R2				; 7-9
	vaddpd	ymm7, ymm7, ymm1		;; A3 = A3 + I3				; 8-10

	vmovapd	ymm2, [srcreg+d2+d1]		;; R4
	vmulpd	ymm1, ymm2, ymm5		;; A4 = R4 * cosine/sine		; 5-9

	vsubpd	ymm0, ymm0, ymm4		;; B3 = B3 - R3				; 9-11

	vmovapd	ymm4, [srcreg+d2+d1+32]		;; I4
	vmulpd	ymm5, ymm4, ymm5		;; B4 = I4 * cosine/sine		; 6-10

	vsubpd	ymm1, ymm1, ymm4		;; A4 = A4 - I4				; 10-12

no bcast vmovapd ymm4, [screg2]			;; Sine
bcast	vbroadcastsd ymm4, Q [screg2]		;; Sine
	vmulpd	ymm6, ymm6, ymm4		;; A2 = A2 * sine (new R2)		; 9-13
	vmulpd	ymm3, ymm3, ymm4		;; B2 = B2 * sine (new I2)		; 10-14
	L1prefetchw srcreg+d1+L1pd, L1pt

	vaddpd	ymm5, ymm5, ymm2		;; B4 = B4 + R4				; 11-13

no bcast vmovapd ymm4, [screg1]			;; Sine
bcast	vbroadcastsd ymm4, Q [screg1]		;; Sine
	vmulpd	ymm7, ymm7, ymm4		;; A3 = A3 * sine (new R3)		; 11-15
	vmulpd	ymm0, ymm0, ymm4		;; B3 = B3 * sine (new I3)		; 12-16
	vmulpd	ymm1, ymm1, ymm4		;; A4 = A4 * sine (new R4)		; 13-17
	vmulpd	ymm5, ymm5, ymm4		;; B4 = B4 * sine (new I4)		; 14-18

	vmovapd	ymm2, [srcreg]			;; R1
	vaddpd	ymm4, ymm2, ymm6		;; R1 + R2 (new R1)			; 14-16
	vsubpd	ymm2, ymm2, ymm6		;; R1 - R2 (new R2)			; 15-17
	L1prefetchw srcreg+d2+L1pd, L1pt

	vaddpd	ymm6, ymm1, ymm7		;; R4 + R3 (new R3)			; 16-18
	vsubpd	ymm1, ymm1, ymm7		;; R4 - R3 (new I4)			; 17-19

	vsubpd	ymm7, ymm0, ymm5		;; I3 - I4 (new R4)			; 18-20
	vaddpd	ymm0, ymm0, ymm5		;; I3 + I4 (new I3)			; 19-21

	vsubpd	ymm5, ymm4, ymm6		;; R1 - R3 (final R3)			; 20-22
	vaddpd	ymm4, ymm4, ymm6		;; R1 + R3 (final R1)			; 21-23

	vmovapd	ymm6, [srcreg+32]		;; I1
	ystore	[srcreg], ymm4			;; Store R1

	vsubpd	ymm4, ymm6, ymm3		;; I1 - I2 (new I2)			; 22-24
	vaddpd	ymm6, ymm6, ymm3		;; I1 + I2 (new I1)			; 23-25
	L1prefetchw srcreg+d2+d1+L1pd, L1pt

	vsubpd	ymm3, ymm2, ymm7		;; R2 - R4 (final R4)			; 24-26
	vaddpd	ymm2, ymm2, ymm7		;; R2 + R4 (final R2)			; 25-27

	vsubpd	ymm7, ymm4, ymm1		;; I2 - I4 (final I4)			; 26-28
	vaddpd	ymm4, ymm4, ymm1		;; I2 + I4 (final I2)			; 27-29

	vsubpd	ymm1, ymm6, ymm0		;; I1 - I3 (final I3)			; 28-30
	vaddpd	ymm6, ymm6, ymm0		;; I1 + I3 (final I1)			; 29-31

;;	ystore	[srcreg], ymm			;; Save R1
	ystore	[srcreg+32], ymm6		;; Save I1
	ystore	[srcreg+d1], ymm2		;; Save R2
	ystore	[srcreg+d1+32], ymm4		;; Save I2
	ystore	[srcreg+d2], ymm5		;; Save R3
	ystore	[srcreg+d2+32], ymm1		;; Save I3
	ystore	[srcreg+d2+d1], ymm3		;; Save R4
	ystore	[srcreg+d2+d1+32], ymm7		;; Save I4
	bump	srcreg, srcinc
	bump	screg, scinc
	ENDM

yr4_4c_djbunfft_swiz MACRO srcreg,srcinc,d1,d2,bcast,screg1,screg2,scoff,screg,scinc,maxrpt,L1pt,L1pd

	;; Swizzle using 8 shufpds and 8 perm2f128  (1 or 3 clocks each on port 5, throughput 1)

	vmovapd	ymm1, [srcreg]			;; R1
	vmovapd	ymm7, [srcreg+d1]		;; R2
	vshufpd	ymm0, ymm1, ymm7, 15		;; Shuffle R1 and R2 to create R1/R2 hi			; 1
	vshufpd	ymm1, ymm1, ymm7, 0		;; Shuffle R1 and R2 to create R1/R2 low		; 2

	vmovapd	ymm2, [srcreg+d2]		;; R3
	vmovapd	ymm7, [srcreg+d2+d1]		;; R4
	vshufpd	ymm3, ymm2, ymm7, 15		;; Shuffle R3 and R4 to create R3/R4 hi			; 3
	vshufpd	ymm2, ymm2, ymm7, 0		;; Shuffle R3 and R4 to create R3/R4 low		; 4

	ylow128s ymm4, ymm0, ymm3		;; Shuffle R1/R2 hi and R3/R4 hi (new R2)		; 5-6
	yhigh128s ymm0, ymm0, ymm3		;; Shuffle R1/R2 hi and R3/R4 hi (new R4)		; 6-7
	L1prefetchw srcreg+L1pd, L1pt

	ylow128s ymm3, ymm1, ymm2		;; Shuffle R1/R2 low and R3/R4 low (new R1)		; 7-8
	yhigh128s ymm1, ymm1, ymm2		;; Shuffle R1/R2 low and R3/R4 low (new R3)		; 8-9

	vmovapd	ymm6, [srcreg+32]		;; I1
	vmovapd	ymm2, [srcreg+d1+32]		;; I2
	vshufpd	ymm5, ymm6, ymm2, 15		;; Shuffle I1 and I2 to create I1/I2 hi
	vshufpd	ymm6, ymm6, ymm2, 0		;; Shuffle I1 and I2 to create I1/I2 low

	ystore	[srcreg], ymm3			;; Temporarily save new R1

	vmovapd	ymm2, [srcreg+d2+32]		;; I3
	vmovapd	ymm7, [srcreg+d2+d1+32]		;; I4
	vshufpd	ymm3, ymm2, ymm7, 15		;; Shuffle I3 and I4 to create I3/I4 hi
	vshufpd	ymm2, ymm2, ymm7, 0		;; Shuffle I3 and I4 to create I3/I4 low

	ylow128s ymm7, ymm5, ymm3		;; Shuffle I1/I2 hi and I3/I4 hi (new I2)
	yhigh128s ymm5, ymm5, ymm3		;; Shuffle I1/I2 hi and I3/I4 hi (new I4)
	L1prefetchw srcreg+d1+L1pd, L1pt

	ylow128s ymm3, ymm6, ymm2		;; Shuffle I1/I2 low and I3/I4 low (new I1)
	yhigh128s ymm6, ymm6, ymm2		;; Shuffle I1/I2 low and I3/I4 low (new I3)

	ystore	[srcreg+32], ymm3		;; Temporarily save new I1

	vmovapd ymm3, [screg2+scoff]		;; cosine/sine
	vmulpd	ymm2, ymm4, ymm3		;; A2 = R2 * cosine/sine		; 1-5
	vmulpd	ymm3, ymm7, ymm3		;; B2 = I2 * cosine/sine		; 2-6
	vaddpd	ymm2, ymm2, ymm7		;; A2 = A2 + I2				; 6-8
	vsubpd	ymm3, ymm3, ymm4		;; B2 = B2 - R2				; 7-9

	vmovapd ymm7, [screg1+scoff]		;; cosine/sine
	vmulpd	ymm4, ymm1, ymm7		;; A3 = R3 * cosine/sine		; 3-7
	vaddpd	ymm4, ymm4, ymm6		;; A3 = A3 + I3				; 8-10
	vmulpd	ymm6, ymm6, ymm7		;; B3 = I3 * cosine/sine		; 4-8
	vsubpd	ymm6, ymm6, ymm1		;; B3 = B3 - R3				; 9-11
	L1prefetchw srcreg+d2+L1pd, L1pt

	vmulpd	ymm1, ymm0, ymm7		;; A4 = R4 * cosine/sine		; 5-9
	vmulpd	ymm7, ymm5, ymm7		;; B4 = I4 * cosine/sine		; 6-10
	vsubpd	ymm1, ymm1, ymm5		;; A4 = A4 - I4				; 10-12

	vmovapd ymm5, [screg2]			;; Sine
	vmulpd	ymm2, ymm2, ymm5		;; A2 = A2 * sine (new R2)		; 9-13
	vmulpd	ymm3, ymm3, ymm5		;; B2 = B2 * sine (new I2)		; 10-14

	vaddpd	ymm7, ymm7, ymm0		;; B4 = B4 + R4				; 11-13
	L1prefetchw srcreg+d2+d1+L1pd, L1pt

	vmovapd ymm5, [screg1]			;; Sine
	vmulpd	ymm4, ymm4, ymm5		;; A3 = A3 * sine (new R3)		; 11-15
	vmulpd	ymm6, ymm6, ymm5		;; B3 = B3 * sine (new I3)		; 12-16
	vmulpd	ymm1, ymm1, ymm5		;; A4 = A4 * sine (new R4)		; 13-17
	vmulpd	ymm7, ymm7, ymm5		;; B4 = B4 * sine (new I4)		; 14-18

	vmovapd	ymm0, [srcreg]			;; Reload R1
	vaddpd	ymm5, ymm0, ymm2		;; R1 + R2 (newer R1)			; 14-16
	vsubpd	ymm0, ymm0, ymm2		;; R1 - R2 (newer R2)			; 15-17

	vaddpd	ymm2, ymm1, ymm4		;; R4 + R3 (newer R3)			; 16-18
	vsubpd	ymm1, ymm1, ymm4		;; R4 - R3 (newer I4)			; 17-19

	vsubpd	ymm4, ymm6, ymm7		;; I3 - I4 (newer R4)			; 18-20
	vaddpd	ymm6, ymm6, ymm7		;; I3 + I4 (newer I3)			; 19-21

	vsubpd	ymm7, ymm5, ymm2		;; R1 - R3 (final R3)			; 20-22
	vaddpd	ymm5, ymm5, ymm2		;; R1 + R3 (final R1)			; 21-23

	vmovapd	ymm2, [srcreg+32]		;; Reload I1
	ystore	[srcreg], ymm5			;; Store R1

	vsubpd	ymm5, ymm2, ymm3		;; I1 - I2 (newer I2)			; 22-24
	vaddpd	ymm2, ymm2, ymm3		;; I1 + I2 (newer I1)			; 23-25

	vsubpd	ymm3, ymm0, ymm4		;; R2 - R4 (final R4)			; 24-26
	vaddpd	ymm0, ymm0, ymm4		;; R2 + R4 (final R2)			; 25-27

	vsubpd	ymm4, ymm5, ymm1		;; I2 - I4 (final I4)			; 26-28
	vaddpd	ymm5, ymm5, ymm1		;; I2 + I4 (final I2)			; 27-29

	vsubpd	ymm1, ymm2, ymm6		;; I1 - I3 (final I3)			; 28-30
	vaddpd	ymm2, ymm2, ymm6		;; I1 + I3 (final I1)			; 29-31

	ystore	[srcreg+32], ymm2		;; Save I1
	ystore	[srcreg+d1], ymm0		;; Save R2
	ystore	[srcreg+d1+32], ymm5		;; Save I2
	ystore	[srcreg+d2], ymm7		;; Save R3
	ystore	[srcreg+d2+32], ymm1		;; Save I3
	ystore	[srcreg+d2+d1], ymm3		;; Save R4
	ystore	[srcreg+d2+d1+32], ymm4		;; Save I4
	bump	srcreg, srcinc
	bump	screg, scinc
	ENDM

IFDEF X86_64

yr4_4c_djbunfft_cmn_preload MACRO swiz
no swiz yr4_4c_djbunfft_noswiz_unroll_preload
swiz	yr4_4c_djbunfft_swiz_unroll_preload
	ENDM

yr4_4c_djbunfft_cmn MACRO srcreg,srcinc,d1,d2,bcast,screg1,screg2,scoff,swiz,screg,scinc,maxrpt,L1pt,L1pd
no swiz yr4_4c_djbunfft_noswiz_cmn srcreg,srcinc,d1,d2,bcast,screg1,screg2,scoff,screg,scinc,maxrpt,L1pt,L1pd
swiz	yr4_4c_djbunfft_swiz_cmn srcreg,srcinc,d1,d2,bcast,screg1,screg2,scoff,screg,scinc,maxrpt,L1pt,L1pd
	ENDM

yr4_4c_djbunfft_noswiz_cmn MACRO srcreg,srcinc,d1,d2,bcast,screg1,screg2,scoff,screg,scinc,maxrpt,L1pt,L1pd
	IF maxrpt MOD 5 EQ 0
	yr4_4c_djbunfft_noswiz_unroll -1,noexec,noexec,exec,srcreg,srcinc,d1,d2,bcast,screg1,screg2,scoff,screg,scinc,L1pt,L1pd
	yr4_4c_djbunfft_noswiz_unroll 0,noexec,exec,exec,srcreg,srcinc,d1,d2,bcast,screg1,screg2,scoff,screg,scinc,L1pt,L1pd
	yr4_4c_djbunfft_noswiz_unroll 1,exec,exec,exec,srcreg,srcinc,d1,d2,bcast,screg1,screg2,scoff,screg,scinc,L1pt,L1pd
	yr4_4c_djbunfft_noswiz_unroll 2,exec,exec,exec,srcreg,srcinc,d1,d2,bcast,screg1,screg2,scoff,screg,scinc,L1pt,L1pd
	yr4_4c_djbunfft_noswiz_unroll 3,exec,exec,exec,srcreg,srcinc,d1,d2,bcast,screg1,screg2,scoff,screg,scinc,L1pt,L1pd
	yr4_4c_djbunfft_noswiz_unroll 4,exec,exec,noexec,srcreg,srcinc,d1,d2,bcast,screg1,screg2,scoff,screg,scinc,L1pt,L1pd
	yr4_4c_djbunfft_noswiz_unroll 5,exec,noexec,noexec,srcreg,srcinc,d1,d2,bcast,screg1,screg2,scoff,screg,scinc,L1pt,L1pd
	bump	srcreg, 5*srcinc
	bump	screg, 5*scinc
	ELSEIF maxrpt MOD 4 EQ 0
	yr4_4c_djbunfft_noswiz_unroll -1,noexec,noexec,exec,srcreg,srcinc,d1,d2,bcast,screg1,screg2,scoff,screg,scinc,L1pt,L1pd
	yr4_4c_djbunfft_noswiz_unroll 0,noexec,exec,exec,srcreg,srcinc,d1,d2,bcast,screg1,screg2,scoff,screg,scinc,L1pt,L1pd
	yr4_4c_djbunfft_noswiz_unroll 1,exec,exec,exec,srcreg,srcinc,d1,d2,bcast,screg1,screg2,scoff,screg,scinc,L1pt,L1pd
	yr4_4c_djbunfft_noswiz_unroll 2,exec,exec,exec,srcreg,srcinc,d1,d2,bcast,screg1,screg2,scoff,screg,scinc,L1pt,L1pd
	yr4_4c_djbunfft_noswiz_unroll 3,exec,exec,noexec,srcreg,srcinc,d1,d2,bcast,screg1,screg2,scoff,screg,scinc,L1pt,L1pd
	yr4_4c_djbunfft_noswiz_unroll 4,exec,noexec,noexec,srcreg,srcinc,d1,d2,bcast,screg1,screg2,scoff,screg,scinc,L1pt,L1pd
	bump	srcreg, 4*srcinc
	bump	screg, 4*scinc
	ELSEIF maxrpt MOD 3 EQ 0
	yr4_4c_djbunfft_noswiz_unroll -1,noexec,noexec,exec,srcreg,srcinc,d1,d2,bcast,screg1,screg2,scoff,screg,scinc,L1pt,L1pd
	yr4_4c_djbunfft_noswiz_unroll 0,noexec,exec,exec,srcreg,srcinc,d1,d2,bcast,screg1,screg2,scoff,screg,scinc,L1pt,L1pd
	yr4_4c_djbunfft_noswiz_unroll 1,exec,exec,exec,srcreg,srcinc,d1,d2,bcast,screg1,screg2,scoff,screg,scinc,L1pt,L1pd
	yr4_4c_djbunfft_noswiz_unroll 2,exec,exec,noexec,srcreg,srcinc,d1,d2,bcast,screg1,screg2,scoff,screg,scinc,L1pt,L1pd
	yr4_4c_djbunfft_noswiz_unroll 3,exec,noexec,noexec,srcreg,srcinc,d1,d2,bcast,screg1,screg2,scoff,screg,scinc,L1pt,L1pd
	bump	srcreg, 3*srcinc
	bump	screg, 3*scinc
	ELSEIF maxrpt MOD 2 EQ 0
	yr4_4c_djbunfft_noswiz_unroll -1,noexec,noexec,exec,srcreg,srcinc,d1,d2,bcast,screg1,screg2,scoff,screg,scinc,L1pt,L1pd
	yr4_4c_djbunfft_noswiz_unroll 0,noexec,exec,exec,srcreg,srcinc,d1,d2,bcast,screg1,screg2,scoff,screg,scinc,L1pt,L1pd
	yr4_4c_djbunfft_noswiz_unroll 1,exec,exec,noexec,srcreg,srcinc,d1,d2,bcast,screg1,screg2,scoff,screg,scinc,L1pt,L1pd
	yr4_4c_djbunfft_noswiz_unroll 2,exec,noexec,noexec,srcreg,srcinc,d1,d2,bcast,screg1,screg2,scoff,screg,scinc,L1pt,L1pd
	bump	srcreg, 2*srcinc
	bump	screg, 2*scinc
	ELSE
	yr4_4c_djbunfft_noswiz_unroll -1,noexec,noexec,exec,srcreg,srcinc,d1,d2,bcast,screg1,screg2,scoff,screg,scinc,L1pt,L1pd
	yr4_4c_djbunfft_noswiz_unroll 0,noexec,exec,noexec,srcreg,srcinc,d1,d2,bcast,screg1,screg2,scoff,screg,scinc,L1pt,L1pd
	yr4_4c_djbunfft_noswiz_unroll 1,exec,noexec,noexec,srcreg,srcinc,d1,d2,bcast,screg1,screg2,scoff,screg,scinc,L1pt,L1pd
	bump	srcreg, srcinc
	bump	screg, scinc
	ENDIF
	ENDM

yr4_4c_djbunfft_noswiz_unroll_preload MACRO
	ENDM

yr4_4c_djbunfft_noswiz_unroll MACRO iter,prev,this,next,srcreg,srcinc,d1,d2,bcast,screg1,screg2,scoff,screg,scinc,L1pt,L1pd

;; On first call, init registers

	IF iter EQ -1
y0	TEXTEQU	<ymm0>
y1	TEXTEQU	<ymm1>
y2	TEXTEQU	<ymm2>
y3	TEXTEQU	<ymm3>
y4	TEXTEQU	<ymm4>
y5	TEXTEQU	<ymm5>
y6	TEXTEQU	<ymm6>
y7	TEXTEQU	<ymm7>
y8	TEXTEQU	<ymm8>
y9	TEXTEQU	<ymm9>
y10	TEXTEQU	<ymm10>
y11	TEXTEQU	<ymm11>
y12	TEXTEQU	<ymm12>
y13	TEXTEQU	<ymm13>
y14	TEXTEQU	<ymm14>
y15	TEXTEQU	<ymm15>
	ENDIF

;; On later calls, previous R2,I3,I1 will be in y0-2.  This R4,R3,R2,I4,I3,I2,A4,A3,A2,B4,B3,c/s2 will be in y3-14.
;; The remaining register is free.

this	vsubpd	y9, y9, y6			;; A4 = A4 - I4 (new R4/sine)		; 1-3
this	vmulpd	y14, y8, y14			;; B2 = I2 * cosine/sine		; 1-5
this no bcast vmovapd y6, [screg2+iter*scinc]	;; Sine
this bcast vbroadcastsd y6, Q [screg2+iter*scinc] ;; Sine

this	vaddpd	y10, y10, y7			;; A3 = A3 + I3 (new R3/sine)		; 2-4
this no bcast vmovapd y7, [screg1+iter*scinc]	;; Sine
this bcast vbroadcastsd y7, Q [screg1+iter*scinc] ;; Sine

this	vaddpd	y11, y11, y8			;; A2 = A2 + I2				; 3-5
this	vmovapd	y8, [srcreg+iter*srcinc]	;; R1

this	vaddpd	y12, y12, y3			;; B4 = B4 + R4 (new I4/sine)		; 4-6
this	vmovapd	y3, [srcreg+iter*srcinc+32]	;; I1

this	vsubpd	y13, y13, y4			;; B3 = B3 - R3 (new I3/sine)		; 5-7
prev	ystore	[srcreg+(iter-1)*srcinc+d1], y0	;; Save R2				; 1
this	L1prefetchw srcreg+iter*srcinc+L1pd, L1pt

this	vsubpd	y14, y14, y5			;; B2 = B2 - R2				; 6-8
this	vmulpd	y11, y11, y6			;; A2 = A2 * sine (new R2)		; 6-10
prev	ystore	[srcreg+(iter-1)*srcinc+d2+32], y1 ;; Save I3				; 2

this	vaddpd	y1, y9, y10			;; C3 = R4/sine + R3/sine (newer R3/sine) ; 7-9
prev	ystore	[srcreg+(iter-1)*srcinc+32], y2	;; Save I1				; 3

this	vsubpd	y9, y9, y10			;; D4 = R4/sine - R3/sine (newer I4/sine) ; 8-10
this	L1prefetchw srcreg+iter*srcinc+d1+L1pd, L1pt

this	vsubpd	y10, y13, y12			;; C4 = I3/sine - I4/sine (newer R4/sine) ; 9-11
this	vmulpd	y14, y14, y6			;; B2 = B2 * sine (new I2)		; 9-13
next no bcast vmovapd y15, [screg1+(iter+1)*scinc+32] ;; cosine/sine
next bcast	vbroadcastsd y15, Q [screg1+(iter+1)*scinc+scoff] ;; cosine/sine

this	vaddpd	y13, y13, y12			;; D3 = I3/sine + I4/sine (newer I3/sine) ; 10-12
this	vmulpd	y1, y1, y7			;; C3 * sine (newer R3)			; 10-14
next	vmovapd	y4, [srcreg+(iter+1)*srcinc+d2+d1] ;; R4

this	vaddpd	y12, y8, y11			;; R1 + R2 (newer R1)			; 11-13
this	vmulpd	y9, y9, y7			;; D4 * sine (newer I4)			; 11-15
next	vmovapd	y0, [srcreg+(iter+1)*srcinc+d2]	;; R3

this	vsubpd	y8, y8, y11			;; R1 - R2 (newer R2)			; 12-14
this	vmulpd	y10, y10, y7			;; C4 * sine (newer R4)			; 12-16
next no bcast vmovapd y5, [screg2+(iter+1)*scinc+32] ;; cosine/sine
next bcast vbroadcastsd y5, Q [screg2+(iter+1)*scinc+scoff] ;; cosine/sine

this	vsubpd	y11, y3, y14			;; I1 - I2 (newer I2)			; 13-15
this	vmulpd	y13, y13, y7			;; D3 * sine (newer I3)			; 13-17
next	vmovapd	y2, [srcreg+(iter+1)*srcinc+d1]	;; R2

this	vaddpd	y3, y3, y14			;; I1 + I2 (newer I1)			; 14-16
this	L1prefetchw srcreg+iter*srcinc+d2+L1pd, L1pt

this	vsubpd	y14, y12, y1			;; R1 - R3 (final R3)			; 15-17
next	vmovapd	y6, [srcreg+(iter+1)*srcinc+d2+d1+32] ;; I4

this	vaddpd	y12, y12, y1			;; R1 + R3 (final R1)			; 16-18
next	vmovapd	y7, [srcreg+(iter+1)*srcinc+d2+32] ;; I3

this	ystore	[srcreg+iter*srcinc+d2], y14	;; Save R3				; 18
this	vsubpd	y14, y11, y9			;; I2 - I4 (final I4)			; 17-19
next	vmovapd	y1, [srcreg+(iter+1)*srcinc+d1+32] ;; I2

this	vaddpd	y11, y11, y9			;; I2 + I4 (final I2)			; 18-20
next	vmulpd	y9, y4, y15			;; A4 = R4 * cosine/sine		; 18-22
this	L1prefetchw srcreg+iter*srcinc+d2+d1+L1pd, L1pt

this	ystore	[srcreg+iter*srcinc], y12	;; Save R1				; 19
this	vsubpd	y12, y8, y10			;; R2 - R4 (final R4)			; 19-21
this	ystore	[srcreg+iter*srcinc+d2+d1+32], y14 ;; Save I4				; 20
next	vmulpd	y14, y0, y15			;; A3 = R3 * cosine/sine		; 19-23

this	vaddpd	y8, y8, y10			;; R2 + R4 (final R2)			; 20-22
next	vmulpd	y10, y2, y5			;; A2 = R2 * cosine/sine		; 20-24

this	ystore	[srcreg+iter*srcinc+d1+32], y11	;; Save I2				; 21
this	vsubpd	y11, y3, y13			;; I1 - I3 (final I3)			; 21-23
this	ystore	[srcreg+iter*srcinc+d2+d1], y12	;; Save R4				; 22
next	vmulpd	y12, y6, y15			;; B4 = I4 * cosine/sine		; 21-25

this	vaddpd	y3, y3, y13			;; I1 + I3 (final I1)			; 22-24
next	vmulpd	y13, y7, y15			;; B3 = I3 * cosine/sine		; 22-26
this next yloop_unrolled_one

;; Shuffle register assignments so that next call has R2,I3,I1 in y0-2 and next R4,R3,R2,I4,I3,I2,A4,A3,A2,B4,B3,c/s2 in y3-14.

ytmp	TEXTEQU	y0
y0	TEXTEQU	y8
y8	TEXTEQU	y1
y1	TEXTEQU	y11
y11	TEXTEQU	y10
y10	TEXTEQU	y14
y14	TEXTEQU	y5
y5	TEXTEQU	y2
y2	TEXTEQU	y3
y3	TEXTEQU	y4
y4	TEXTEQU	ytmp
	ENDM

;; Haswell FMA3 version
;; Basic implementation originally written for Bulldozer.  13.6 clocks.

IF (@INSTR(,%yarch,<FMA3>) NE 0)

yr4_4c_djbunfft_noswiz_unroll_preload MACRO
	vmovapd	ymm15, YMM_ONE
	ENDM

;; uops = 4 s/c loads, 8 loads, 8 stores, 22 fma/add/sub, 11 mov = 53 uops / 4 = 12.25 clocks
yr4_4c_djbunfft_noswiz_unroll MACRO iter,prev,this,next,srcreg,srcinc,d1,d2,bcast,screg1,screg2,scoff,screg,scinc,L1pt,L1pd

;; On first call, init registers

	IF iter EQ -1
y0	TEXTEQU	<ymm0>
y1	TEXTEQU	<ymm1>
y2	TEXTEQU	<ymm2>
y3	TEXTEQU	<ymm3>
y4	TEXTEQU	<ymm4>
y5	TEXTEQU	<ymm5>
y6	TEXTEQU	<ymm6>
y7	TEXTEQU	<ymm7>
y8	TEXTEQU	<ymm8>
y9	TEXTEQU	<ymm9>
y10	TEXTEQU	<ymm10>
y11	TEXTEQU	<ymm11>
y12	TEXTEQU	<ymm12>
y13	TEXTEQU	<ymm13>
y14	TEXTEQU	<ymm14>
	ENDIF

;; On later calls, previous R1,I1,R2,I2,R3/sine,I3/sine,R4/sine,I4/sine will be in y0-7.  This R2,I2,R3,I3,c/s1,c/s2 will be in y8-13.
;; The remaining registers are free.  Register y15 is reserved for the constant YMM_ONE.

this	yfmaddpd y14, y8, y13, y9			;; R2/sine = R2 * cosine/sine + I2	; 1-5
this	yfmsubpd y9, y9, y13, y8			;; I2/sine = I2 * cosine/sine - R2	; 1-5
prev no bcast vmovapd y13, [screg1+(iter-1)*scinc]	;; Sine
prev bcast vbroadcastsd y13, Q [screg1+(iter-1)*scinc]	;; Sine
this	L1prefetchw srcreg+iter*srcinc+L1pd, L1pt

this	yfmaddpd y8, y10, y12, y11			;; R3/sine = R3 * cosine/sine + I3	; 2-6
this	yfmsubpd y11, y11, y12, y10			;; I3/sine = I3 * cosine/sine - R3	; 2-6
this	L1prefetchw srcreg+iter*srcinc+d1+L1pd, L1pt

prev	yfnmaddpd y10, y4, y13, y0			;; R1 - R3/sine * sine (final R3)	; 3-7
prev	yfmaddpd y4, y4, y13, y0			;; R1 + R3/sine * sine (final R1)	; 3-7
this	vmovapd	y0, [srcreg+iter*srcinc+d2+d1]		;; R4
prev	ystore	[srcreg+(iter-1)*srcinc+d2], y10	;; Save R3				; 8
this	vmovapd	y10, [srcreg+iter*srcinc+d2+d1+32]	;; I4

prev	ystore	[srcreg+(iter-1)*srcinc], y4		;; Save R1				; 8
this	yfmsubpd y4, y0, y12, y10			;; R4/sine = R4 * cosine/sine - I4	; 4-8
this	yfmaddpd y10, y10, y12, y0			;; I4/sine = I4 * cosine/sine + R4	; 4-8
this	vmovapd	y12, [srcreg+iter*srcinc]		;; R1

prev	yfnmaddpd y0, y6, y13, y2			;; R2 - R4/sine * sine (final R4)	; 5-9
prev	yfmaddpd y6, y6, y13, y2			;; R2 + R4/sine * sine (final R2)	; 5-9
this no bcast vmovapd y2, [screg2+iter*scinc]		;; Sine
this bcast vbroadcastsd y2, Q [screg2+iter*scinc]	;; Sine
this	L1prefetchw srcreg+iter*srcinc+d2+L1pd, L1pt

prev	ystore	[srcreg+(iter-1)*srcinc+d2+d1], y0	;; Save R4				; 10
this	yfmaddpd y0, y14, y2, y12			;; R1 + R2/sine * sine (new R1)		; 6-10
this	yfnmaddpd y14, y14, y2, y12			;; R1 - R2/sine * sine (new R2)		; 6-10
this	vmovapd	y12, [srcreg+iter*srcinc+32]		;; I1

prev	ystore	[srcreg+(iter-1)*srcinc+d1], y6		;; Save R2				; 10
prev	yfnmaddpd y6, y7, y13, y3			;; I2 - I4/sine * sine (final I4)	; 7-11
prev	yfmaddpd y7, y7, y13, y3			;; I2 + I4/sine * sine (final I2)	; 7-11
next no bcast vmovapd y3, [screg2+(iter+1)*scinc+32]	;; cosine/sine
next bcast vbroadcastsd y3, Q [screg2+(iter+1)*scinc+scoff] ;; cosine/sine

prev	ystore	[srcreg+(iter-1)*srcinc+d2+d1+32], y6	;; Save I4				; 12
prev	yfnmaddpd y6, y5, y13, y1			;; I1 - I3/sine * sine (final I3)	; 8-12
prev	yfmaddpd y5, y5, y13, y1			;; I1 + I3/sine * sine (final I1)	; 8-12
next	vmovapd	y13, [srcreg+(iter+1)*srcinc+d1]	;; R2
this	L1prefetchw srcreg+iter*srcinc+d2+d1+L1pd, L1pt

this	yfmaddpd y1, y4, ymm15, y8			;; R4/sine + R3/sine (new R3/sine)	; 9-13
this	yfmsubpd y4, y4, ymm15, y8			;; R4/sine - R3/sine (new I4/sine)	; 9-13
next	vmovapd	y8, [srcreg+(iter+1)*srcinc+d1+32]	;; I2
this next yloop_unrolled_one

prev	ystore	[srcreg+(iter-1)*srcinc+d1+32], y7	;; Save I2				; 12
this	yfmsubpd y7, y11, ymm15, y10			;; I3/sine - I4/sine (new R4/sine)	; 10-14
this	yfmaddpd y11, y11, ymm15, y10			;; I3/sine + I4/sine (new I3/sine)	; 10-14
next no bcast vmovapd y10, [screg1+(iter+1)*scinc+32]	;; cosine/sine
next bcast vbroadcastsd y10, Q [screg1+(iter+1)*scinc+scoff] ;; cosine/sine

prev	ystore	[srcreg+(iter-1)*srcinc+d2+32], y6	;; Save I3				; 13
this	yfnmaddpd y6, y9, y2, y12			;; I1 - I2/sine * sine (new I2)		; 11-15
this	yfmaddpd y9, y9, y2, y12			;; I1 + I2/sine * sine (new I1)		; 11-15
next	vmovapd	y2, [srcreg+(iter+1)*srcinc+d2]		;; R3
next	vmovapd	y12, [srcreg+(iter+1)*srcinc+d2+32]	;; I3
prev	ystore	[srcreg+(iter-1)*srcinc+32], y5		;; Save I1				; 13

;; Shuffle register assignments so that next call has R1,I1,R2,I2,R3/sine,I3/sine,R4/sine,I4/sine in y0-7 and next R2,I2,R3,I3,c/s1,c/s2 in y8-13.

ytmp	TEXTEQU	y1
y1	TEXTEQU	y9
y9	TEXTEQU	y8
y8	TEXTEQU	y13
y13	TEXTEQU	y3
y3	TEXTEQU	y6
y6	TEXTEQU	y7
y7	TEXTEQU	y4
y4	TEXTEQU	ytmp
ytmp	TEXTEQU	y2
y2	TEXTEQU	y14
y14	TEXTEQU	y5
y5	TEXTEQU	y11
y11	TEXTEQU	y12
y12	TEXTEQU	y10
y10	TEXTEQU	ytmp
	ENDM

ENDIF

;; Haswell FMA3 version
;; This is an untested FMA3 version that should be optimal.  Alas, it is no faster.  Timed at ~14 clocks.
;; This was written when I thought there was a 1 clock delay to send a result to the other FPU port.

IF (@INSTR(,%yarch,<NOT_BEST_FMA3>) NE 0)

yr4_4c_djbunfft_noswiz_unroll_preload MACRO
	vmovapd	ymm15, YMM_ONE
	ENDM

;; uops = 4 s/c loads, 8 loads, 8 stores, 22 fma/add/sub, 11 mov = 53 uops / 4 = 12.25 clocks
yr4_4c_djbunfft_noswiz_unroll MACRO iter,prev,this,next,srcreg,srcinc,d1,d2,bcast,screg1,screg2,scoff,screg,scinc,L1pt,L1pd

;; On first call, init registers

	IF iter EQ -1
y0	TEXTEQU	<ymm0>
y1	TEXTEQU	<ymm1>
y2	TEXTEQU	<ymm2>
y3	TEXTEQU	<ymm3>
y4	TEXTEQU	<ymm4>
y5	TEXTEQU	<ymm5>
y6	TEXTEQU	<ymm6>
y7	TEXTEQU	<ymm7>
y8	TEXTEQU	<ymm8>
y9	TEXTEQU	<ymm9>
y10	TEXTEQU	<ymm10>
y11	TEXTEQU	<ymm11>
y12	TEXTEQU	<ymm12>
y13	TEXTEQU	<ymm13>
y14	TEXTEQU	<ymm14>
	ENDIF

;; On later calls, previous I1,I2/sine,sine,R1,R3/sine,R2,R4/sine,I2,I4/sine,I3/sine will be in y0-9.  This R3,I3,c/s,R4,I4 will be in y10-14.
;; Register y15 is reserved for the constant YMM_ONE.

prev	yfmaddpd y1, y1, y2, y0				;; I1 + I2/sine * sine (new I1)		;	1-5		n 7
prev no bcast vmovapd y2, [screg1+(iter-1)*scinc]	;; Sine
prev bcast vbroadcastsd y2, Q [screg1+(iter-1)*scinc]	;; Sine

this	yfmaddpd y0, y10, y12, y11			;; R3/sine = R3 * cosine/sine + I3	; 1-5			n 8
this	yfmsubpd y11, y11, y12, y10			;; I3/sine = I3 * cosine/sine - R3	;	2-6		n 10

this	yfmsubpd y10, y13, y12, y14			;; R4/sine = R4 * cosine/sine - I4	; 2-6			n 8
this	yfmaddpd y14, y14, y12, y13			;; I4/sine = I4 * cosine/sine + R4	;	3-7		n 10
this no bcast vmovapd y12, [screg2+iter*scinc+32]	;; cosine/sine
this bcast vbroadcastsd y12, Q [screg2+iter*scinc+scoff] ;; cosine/sine

prev	yfnmaddpd y13, y4, y2, y3			;; R1 - R3/sine * sine (final R3)	; 3-7
prev	yfmaddpd y4, y4, y2, y3				;; R1 + R3/sine * sine (final R1)	;	4-8

this	vmovapd	y3, [srcreg+iter*srcinc+d1]		;; R2
prev	ystore	[srcreg+(iter-1)*srcinc+d2], y13	;; Save R3				; 8
this	vmovapd	y13, [srcreg+iter*srcinc+d1+32]		;; I2
prev	ystore	[srcreg+(iter-1)*srcinc], y4		;; Save R1				; 9
this	yfmaddpd y4, y3, y12, y13			;; R2/sine = R2 * cosine/sine + I2	; 4-8			n 9
this	yfmsubpd y13, y13, y12, y3			;; I2/sine = I2 * cosine/sine - R2	;	5-9		n 11

prev	yfnmaddpd y12, y6, y2, y5			;; R2 - R4/sine * sine (final R4)	; 5-9
prev	yfmaddpd y6, y6, y2, y5				;; R2 + R4/sine * sine (final R2)	;	6-10
this no bcast vmovapd y3, [screg2+iter*scinc]		;; Sine
this bcast vbroadcastsd y3, Q [screg2+iter*scinc]	;; Sine

prev	yfnmaddpd y5, y8, y2, y7			;; I2 - I4/sine * sine (final I4)	; 6-10
prev	yfmaddpd y8, y8, y2, y7				;; I2 + I4/sine * sine (final I2)	;	7-11
this	vmovapd	y7, [srcreg+iter*srcinc]		;; R1

prev	ystore	[srcreg+(iter-1)*srcinc+d2+d1], y12	;; Save R4				; 10
prev	yfnmaddpd y12, y9, y2, y1			;; I1 - I3/sine * sine (final I3)	; 7-11
prev	yfmaddpd y9, y9, y2, y1				;; I1 + I3/sine * sine (final I1)	;	8-12
this	vmovapd	y2, [srcreg+iter*srcinc+32]		;; I1

prev	ystore	[srcreg+(iter-1)*srcinc+d1], y6		;; Save R2				; 11
this	yfmaddpd y1, y10, ymm15, y0			;; R4/sine + R3/sine (new R3/sine)	; 8-12			n 14
this	yfmsubpd y10, y10, ymm15, y0			;; R4/sine - R3/sine (new I4/sine)	;	9-13		n 17
next	vmovapd	y0, [srcreg+(iter+1)*srcinc+d2]		;; R3

prev	ystore	[srcreg+(iter-1)*srcinc+d2+d1+32], y5	;; Save I4				; 11+1
this	yfmaddpd y6, y4, y3, y7				;; R1 + R2/sine * sine (new R1)		; 9-13			n 14
this	yfnmaddpd y4, y4, y3, y7			;; R1 - R2/sine * sine (new R2)		;	10-14		n 16
next	vmovapd	y7, [srcreg+(iter+1)*srcinc+d2+32]	;; I3

prev	ystore	[srcreg+(iter-1)*srcinc+d1+32], y8	;; Save I2				; 12+1
this	yfmsubpd y5, y11, ymm15, y14			;; I3/sine - I4/sine (new R4/sine)	; 10-14			n 16
this	yfmaddpd y11, y11, ymm15, y14			;; I3/sine + I4/sine (new I3/sine)	;	11-15		n 18
next no bcast vmovapd y14, [screg1+(iter+1)*scinc+32]	;; cosine/sine
next bcast vbroadcastsd y14, Q [screg1+(iter+1)*scinc+scoff] ;; cosine/sine

prev	ystore	[srcreg+(iter-1)*srcinc+d2+32], y12	;; Save I3				; 12+2
this	yfnmaddpd y8, y13, y3, y2			;; I1 - I2/sine * sine (new I2)		; 11-15			n 17
next	vmovapd	y12, [srcreg+(iter+1)*srcinc+d2+d1]	;; R4
prev	ystore	[srcreg+(iter-1)*srcinc+32], y9		;; Save I1				; 13+2
next	vmovapd	y9, [srcreg+(iter+1)*srcinc+d2+d1+32]	;; I4

this	L1prefetchw srcreg+iter*srcinc+L1pd, L1pt
this	L1prefetchw srcreg+iter*srcinc+d1+L1pd, L1pt
this	L1prefetchw srcreg+iter*srcinc+d2+L1pd, L1pt
this	L1prefetchw srcreg+iter*srcinc+d2+d1+L1pd, L1pt
this next yloop_unrolled_one

;; Shuffle register assignments so that this I1,I2/sine,sine,R1,R3/sine,R2,R4/sine,I2,I4/sine,I3/sine in y0-9 and next R3,I3,c/s,R4,I4 in y10-14.

ytmp	TEXTEQU	y0
y0	TEXTEQU	y2
y2	TEXTEQU	y3
y3	TEXTEQU	y6
y6	TEXTEQU	y5
y5	TEXTEQU	y4
y4	TEXTEQU	y1
y1	TEXTEQU	y13
y13	TEXTEQU	y12
y12	TEXTEQU	y14
y14	TEXTEQU	y9
y9	TEXTEQU	y11
y11	TEXTEQU	y7
y7	TEXTEQU	y8
y8	TEXTEQU	y10
y10	TEXTEQU	ytmp

	ENDM

ENDIF


yr4_4c_djbunfft_swiz_cmn MACRO srcreg,srcinc,d1,d2,bcast,screg1,screg2,scoff,screg,scinc,maxrpt,L1pt,L1pd
	IF maxrpt MOD 5 EQ 0
	yr4_4c_djbunfft_swiz_unroll -1,noexec,noexec,exec,srcreg,srcinc,d1,d2,bcast,screg1,screg2,scoff,screg,scinc,L1pt,L1pd
	yr4_4c_djbunfft_swiz_unroll 0,noexec,exec,exec,srcreg,srcinc,d1,d2,bcast,screg1,screg2,scoff,screg,scinc,L1pt,L1pd
	yr4_4c_djbunfft_swiz_unroll 1,exec,exec,exec,srcreg,srcinc,d1,d2,bcast,screg1,screg2,scoff,screg,scinc,L1pt,L1pd
	yr4_4c_djbunfft_swiz_unroll 2,exec,exec,exec,srcreg,srcinc,d1,d2,bcast,screg1,screg2,scoff,screg,scinc,L1pt,L1pd
	yr4_4c_djbunfft_swiz_unroll 3,exec,exec,exec,srcreg,srcinc,d1,d2,bcast,screg1,screg2,scoff,screg,scinc,L1pt,L1pd
	yr4_4c_djbunfft_swiz_unroll 4,exec,exec,noexec,srcreg,srcinc,d1,d2,bcast,screg1,screg2,scoff,screg,scinc,L1pt,L1pd
	yr4_4c_djbunfft_swiz_unroll 5,exec,noexec,noexec,srcreg,srcinc,d1,d2,bcast,screg1,screg2,scoff,screg,scinc,L1pt,L1pd
	bump	srcreg, 5*srcinc
	bump	screg, 5*scinc
	ELSEIF maxrpt MOD 4 EQ 0
	yr4_4c_djbunfft_swiz_unroll -1,noexec,noexec,exec,srcreg,srcinc,d1,d2,bcast,screg1,screg2,scoff,screg,scinc,L1pt,L1pd
	yr4_4c_djbunfft_swiz_unroll 0,noexec,exec,exec,srcreg,srcinc,d1,d2,bcast,screg1,screg2,scoff,screg,scinc,L1pt,L1pd
	yr4_4c_djbunfft_swiz_unroll 1,exec,exec,exec,srcreg,srcinc,d1,d2,bcast,screg1,screg2,scoff,screg,scinc,L1pt,L1pd
	yr4_4c_djbunfft_swiz_unroll 2,exec,exec,exec,srcreg,srcinc,d1,d2,bcast,screg1,screg2,scoff,screg,scinc,L1pt,L1pd
	yr4_4c_djbunfft_swiz_unroll 3,exec,exec,noexec,srcreg,srcinc,d1,d2,bcast,screg1,screg2,scoff,screg,scinc,L1pt,L1pd
	yr4_4c_djbunfft_swiz_unroll 4,exec,noexec,noexec,srcreg,srcinc,d1,d2,bcast,screg1,screg2,scoff,screg,scinc,L1pt,L1pd
	bump	srcreg, 4*srcinc
	bump	screg, 4*scinc
	ELSEIF maxrpt MOD 3 EQ 0
	yr4_4c_djbunfft_swiz_unroll -1,noexec,noexec,exec,srcreg,srcinc,d1,d2,bcast,screg1,screg2,scoff,screg,scinc,L1pt,L1pd
	yr4_4c_djbunfft_swiz_unroll 0,noexec,exec,exec,srcreg,srcinc,d1,d2,bcast,screg1,screg2,scoff,screg,scinc,L1pt,L1pd
	yr4_4c_djbunfft_swiz_unroll 1,exec,exec,exec,srcreg,srcinc,d1,d2,bcast,screg1,screg2,scoff,screg,scinc,L1pt,L1pd
	yr4_4c_djbunfft_swiz_unroll 2,exec,exec,noexec,srcreg,srcinc,d1,d2,bcast,screg1,screg2,scoff,screg,scinc,L1pt,L1pd
	yr4_4c_djbunfft_swiz_unroll 3,exec,noexec,noexec,srcreg,srcinc,d1,d2,bcast,screg1,screg2,scoff,screg,scinc,L1pt,L1pd
	bump	srcreg, 3*srcinc
	bump	screg, 3*scinc
	ELSEIF maxrpt MOD 2 EQ 0
	yr4_4c_djbunfft_swiz_unroll -1,noexec,noexec,exec,srcreg,srcinc,d1,d2,bcast,screg1,screg2,scoff,screg,scinc,L1pt,L1pd
	yr4_4c_djbunfft_swiz_unroll 0,noexec,exec,exec,srcreg,srcinc,d1,d2,bcast,screg1,screg2,scoff,screg,scinc,L1pt,L1pd
	yr4_4c_djbunfft_swiz_unroll 1,exec,exec,noexec,srcreg,srcinc,d1,d2,bcast,screg1,screg2,scoff,screg,scinc,L1pt,L1pd
	yr4_4c_djbunfft_swiz_unroll 2,exec,noexec,noexec,srcreg,srcinc,d1,d2,bcast,screg1,screg2,scoff,screg,scinc,L1pt,L1pd
	bump	srcreg, 2*srcinc
	bump	screg, 2*scinc
	ELSE
	yr4_4c_djbunfft_swiz_unroll -1,noexec,noexec,exec,srcreg,srcinc,d1,d2,bcast,screg1,screg2,scoff,screg,scinc,L1pt,L1pd
	yr4_4c_djbunfft_swiz_unroll 0,noexec,exec,noexec,srcreg,srcinc,d1,d2,bcast,screg1,screg2,scoff,screg,scinc,L1pt,L1pd
	yr4_4c_djbunfft_swiz_unroll 1,exec,noexec,noexec,srcreg,srcinc,d1,d2,bcast,screg1,screg2,scoff,screg,scinc,L1pt,L1pd
	bump	srcreg, srcinc
	bump	screg, scinc
	ENDIF
	ENDM

yr4_4c_djbunfft_swiz_unroll_preload MACRO
	ENDM

yr4_4c_djbunfft_swiz_unroll MACRO iter,prev,this,next,srcreg,srcinc,d1,d2,bcast,screg1,screg2,scoff,screg,scinc,L1pt,L1pd

;; On first call, init registers

	IF iter EQ -1
y0	TEXTEQU	<ymm0>
y1	TEXTEQU	<ymm1>
y2	TEXTEQU	<ymm2>
y3	TEXTEQU	<ymm3>
y4	TEXTEQU	<ymm4>
y5	TEXTEQU	<ymm5>
y6	TEXTEQU	<ymm6>
y7	TEXTEQU	<ymm7>
y8	TEXTEQU	<ymm8>
y9	TEXTEQU	<ymm9>
y10	TEXTEQU	<ymm10>
y11	TEXTEQU	<ymm11>
y12	TEXTEQU	<ymm12>
y13	TEXTEQU	<ymm13>
y14	TEXTEQU	<ymm14>
y15	TEXTEQU	<ymm15>
	ENDIF

;; On later calls,  This R4,R3,R2,R1,I4,I3,I2,I1,A4,A3,A2,B4,B3,c/s2 will be in y0-13.
;; The remaining registers are free.

this	vsubpd	y8, y8, y4			;; A4 = A4 - I4 (new R4/sine)		; 1-3
this	vmulpd	y13, y6, y13			;; B2 = I2 * cosine/sine		; 1-5
this no bcast vmovapd y14, [screg2+iter*scinc]	;; Sine
this bcast vbroadcastsd y14, Q [screg2+iter*scinc] ;; Sine

this	vaddpd	y9, y9, y5			;; A3 = A3 + I3 (new R3/sine)		; 2-4
next	vmovapd	y15, [srcreg+(iter+1)*srcinc]	;; R1

this	vaddpd	y10, y10, y6			;; A2 = A2 + I2				; 3-5
next	vmovapd	y4, [srcreg+(iter+1)*srcinc+d1] ;; R2

this	vaddpd	y11, y11, y0			;; B4 = B4 + R4 (new I4/sine)		; 4-6
next	vmovapd	y5, [srcreg+(iter+1)*srcinc+d2]	;; R3

this	vsubpd	y12, y12, y1			;; B3 = B3 - R3 (new I3/sine)		; 5-7
next	vmovapd	y6, [srcreg+(iter+1)*srcinc+d2+d1] ;; R4

this	vsubpd	y13, y13, y2			;; B2 = B2 - R2				; 6-8
this	vmulpd	y10, y10, y14			;; A2 = A2 * sine (new R2)		; 6-10
this no bcast vmovapd y0, [screg1+iter*scinc]	;; Sine
this bcast vbroadcastsd y0, Q [screg1+iter*scinc] ;; Sine
next	vshufpd	y2, y15, y4, 15			;; Shuffle R1 and R2 to create R1/R2 hi	; 6

this	vaddpd	y1, y8, y9			;; C3 = R4/sine + R3/sine (newer R3/sine) ; 7-9
next	vshufpd	y15, y15, y4, 0			;; Shuffle R1 and R2 to create R1/R2 low ; 7
this	L1prefetchw srcreg+iter*srcinc+L1pd, L1pt

this	vsubpd	y8, y8, y9			;; D4 = R4/sine - R3/sine (newer I4/sine) ; 8-10
next	vshufpd	y9, y5, y6, 15			;; Shuffle R3 and R4 to create R3/R4 hi	; 8

this	vsubpd	y4, y12, y11			;; C4 = I3/sine - I4/sine (newer R4/sine) ; 9-11
this	vmulpd	y13, y13, y14			;; B2 = B2 * sine (new I2)		; 9-13
next	vshufpd	y5, y5, y6, 0			;; Shuffle R3 and R4 to create R3/R4 low ; 9

this	vaddpd	y12, y12, y11			;; D3 = I3/sine + I4/sine (newer I3/sine) ; 10-12
this	vmulpd	y1, y1, y0			;; C3 * sine (newer R3)			; 10-14
next	yhigh128s y11, y2, y9			;; Shuffle R1/R2 hi and R3/R4 hi (new R4) ; 10-11
this	L1prefetchw srcreg+iter*srcinc+d1+L1pd, L1pt

this	vaddpd	y6, y3, y10			;; R1 + R2 (newer R1)			; 11-13
this	vmulpd	y8, y8, y0			;; D4 * sine (newer I4)			; 11-15
next	yhigh128s y14, y15, y5			;; Shuffle R1/R2 low and R3/R4 low (new R3) ; 11-12

this	vsubpd	y3, y3, y10			;; R1 - R2 (newer R2)			; 12-14
this	vmulpd	y4, y4, y0			;; C4 * sine (newer R4)			; 12-16
next	ylow128s y2, y2, y9			;; Shuffle R1/R2 hi and R3/R4 hi (new R2) ; 12-13
next	vmovapd	y10, [srcreg+(iter+1)*srcinc+d2+32] ;; I3

this	vsubpd	y9, y7, y13			;; I1 - I2 (newer I2)			; 13-15
this	vmulpd	y12, y12, y0			;; D3 * sine (newer I3)			; 13-17
next	ylow128s y15, y15, y5			;; Shuffle R1/R2 low and R3/R4 low (new R1) ; 13-14
next	vmovapd	y5, [srcreg+(iter+1)*srcinc+d2+d1+32] ;; I4

this	vaddpd	y7, y7, y13			;; I1 + I2 (newer I1)			; 14-16
next	vmovapd	y0, [srcreg+(iter+1)*srcinc+32]	;; I1

this	vsubpd	y13, y6, y1			;; R1 - R3 (final R3)			; 15-17
this	ystore	[srcreg+iter*srcinc+d2], y13	;; Save R3				; 18
next	vshufpd	y13, y10, y5, 15		;; Shuffle I3 and I4 to create I3/I4 hi	; 15
this	L1prefetchw srcreg+iter*srcinc+d2+L1pd, L1pt

this	vaddpd	y6, y6, y1			;; R1 + R3 (final R1)			; 16-18
next	vmovapd	y1, [srcreg+(iter+1)*srcinc+d1+32] ;; I2
next	vshufpd	y10, y10, y5, 0			;; Shuffle I3 and I4 to create I3/I4 low ; 16

this	vsubpd	y5, y9, y8			;; I2 - I4 (final I4)			; 17-19
this	ystore	[srcreg+iter*srcinc], y6	;; Save R1				; 19
next no bcast vmovapd y6, [screg1+(iter+1)*scinc+32] ;; cosine/sine
next bcast vbroadcastsd y6, Q [screg1+(iter+1)*scinc+scoff] ;; cosine/sine
this	ystore	[srcreg+iter*srcinc+d2+d1+32], y5 ;; Save I4				; 20
next	vshufpd	y5, y0, y1, 15			;; Shuffle I1 and I2 to create I1/I2 hi	; 17

this	vaddpd	y9, y9, y8			;; I2 + I4 (final I2)			; 18-20
next	vmulpd	y8, y11, y6			;; A4 = R4 * cosine/sine		; 18-22
next	vshufpd	y0, y0, y1, 0			;; Shuffle I1 and I2 to create I1/I2 low ; 18
this	L1prefetchw srcreg+iter*srcinc+d2+d1+L1pd, L1pt

this	vsubpd	y1, y3, y4			;; R2 - R4 (final R4)			; 19-21
this	ystore	[srcreg+iter*srcinc+d1+32], y9	;; Save I2				; 21
next	vmulpd	y9, y14, y6			;; A3 = R3 * cosine/sine		; 19-23
this	ystore	[srcreg+iter*srcinc+d2+d1], y1	;; Save R4				; 22
next	yhigh128s y1, y5, y13			;; Shuffle I1/I2 hi and I3/I4 hi (new I4) ; 19-20

this	vaddpd	y3, y3, y4			;; R2 + R4 (final R2)			; 20-22
next	yhigh128s y4, y0, y10			;; Shuffle I1/I2 low and I3/I4 low (new I3) ; 20-21
this	ystore	[srcreg+iter*srcinc+d1], y3	;; Save R2				; 23
next no bcast vmovapd y3, [screg2+(iter+1)*scinc+32] ;; cosine/sine
next bcast vbroadcastsd y3, Q [screg2+(iter+1)*scinc+scoff] ;; cosine/sine
next	ylow128s y5, y5, y13			;; Shuffle I1/I2 hi and I3/I4 hi (new I2) ; 21-22
next	vmulpd	y13, y2, y3			;; A2 = R2 * cosine/sine		; 20-24

next	ylow128s y0, y0, y10			;; Shuffle I1/I2 low and I3/I4 low (new I1) ; 22-23
this	vsubpd	y10, y7, y12			;; I1 - I3 (final I3)			; 21-23
this	ystore	[srcreg+iter*srcinc+d2+32], y10	;; Save I3				; 2
next	vmulpd	y10, y1, y6			;; B4 = I4 * cosine/sine		; 21-25

this	vaddpd	y7, y7, y12			;; I1 + I3 (final I1)			; 22-24
next	vmulpd	y6, y4, y6			;; B3 = I3 * cosine/sine		; 22-26
this	ystore	[srcreg+iter*srcinc+32], y7	;; Save I1				; 3
this next yloop_unrolled_one

;; Shuffle register assignments so that next call has next R4,R3,R2,R1,I4,I3,I2,I1,A4,A3,A2,B4,B3,c/s2 in y0-13.

ytmp	TEXTEQU	y0
y0	TEXTEQU	y11
y11	TEXTEQU	y10
y10	TEXTEQU	y13
y13	TEXTEQU	y3
y3	TEXTEQU	y15
y15	TEXTEQU	y7
y7	TEXTEQU	ytmp
ytmp	TEXTEQU	y1
y1	TEXTEQU	y14
y14	TEXTEQU	y12
y12	TEXTEQU	y6
y6	TEXTEQU	y5
y5	TEXTEQU	y4
y4	TEXTEQU	ytmp
	ENDM


;; Haswell FMA3 version

IF (@INSTR(,%yarch,<FMA3>) NE 0)

yr4_4c_djbunfft_swiz_unroll_preload MACRO
	ENDM

;; Shuffles on port 5 limit us to 18 clocks.  Actually, timed at 21.25 clocks.
yr4_4c_djbunfft_swiz_unroll MACRO iter,prev,this,next,srcreg,srcinc,d1,d2,bcast,screg1,screg2,scoff,screg,scinc,L1pt,L1pd

;; On first call, init registers

	IF iter EQ -1
y0	TEXTEQU	<ymm0>
y1	TEXTEQU	<ymm1>
y2	TEXTEQU	<ymm2>
y3	TEXTEQU	<ymm3>
y4	TEXTEQU	<ymm4>
y5	TEXTEQU	<ymm5>
y6	TEXTEQU	<ymm6>
y7	TEXTEQU	<ymm7>
y8	TEXTEQU	<ymm8>
y9	TEXTEQU	<ymm9>
y10	TEXTEQU	<ymm10>
y11	TEXTEQU	<ymm11>
y12	TEXTEQU	<ymm12>
y13	TEXTEQU	<ymm13>
y14	TEXTEQU	<ymm14>
y15	TEXTEQU	<ymm15>
	ENDIF

;; On later iterations prev I1,I2,I3,I4 are in y0-3, and this R1,R2/sine,R3/sine,R4/sine,I1,I2/sine,I3/sine,I4/sine are in y4-y11.
;; The remaining registers are free.

next	vmovapd	y12, [srcreg+(iter+1)*srcinc]		;; R1
next	vmovapd	y13, [srcreg+(iter+1)*srcinc+d1]	;; R2
next	vshufpd	y14, y12, y13, 15			;; Shuffle R1 and R2 to create R1/R2 hi		;		1
next	vmovapd	y15, [srcreg+(iter+1)*srcinc+d2]	;; R3
prev	ystore	[srcreg+(iter-1)*srcinc+d2+d1+32], y3	;; Save I4					;			1

next	vshufpd	y12, y12, y13, 0			;; Shuffle R1 and R2 to create R1/R2 low	;		2
next	vmovapd	y3, [srcreg+(iter+1)*srcinc+d2+d1]	;; R4
prev	ystore	[srcreg+(iter-1)*srcinc+d1+32], y1	;; Save I2					;			2

next	vshufpd	y1, y15, y3, 15				;; Shuffle R3 and R4 to create R3/R4 hi		;		3
this no bcast vmovapd y13, [screg2+iter*scinc]		;; Sine
this bcast vbroadcastsd y13, Q [screg2+iter*scinc]	;; Sine
prev	ystore	[srcreg+(iter-1)*srcinc+d2+32], y2	;; Save I3					;			3

next	vshufpd	y15, y15, y3, 0				;; Shuffle R3 and R4 to create R3/R4 low	;		4
this	yfmaddpd y3, y5, y13, y4			;; R1 + R2/sine * sine (new R1)			; 4-8
this	yfnmaddpd y5, y5, y13, y4			;; R1 - R2/sine * sine (new R2)			;	4-8
this	vmovapd	y2, YMM_ONE
prev	ystore	[srcreg+(iter-1)*srcinc+32], y0		;; Save I1					;			4

this	yfmaddpd y0, y7, y2, y6				;; R4/sine + R3/sine (new R3/sine)		; 5-9
this	yfmsubpd y7, y7, y2, y6				;; R4/sine - R3/sine (new I4/sine)		;	5-9
next	vmovapd	y4, [srcreg+(iter+1)*srcinc+32]		;; I1

this	yfmsubpd y6, y10, y2, y11			;; I3/sine - I4/sine (new R4/sine)		; 6-10
this	yfmaddpd y10, y10, y2, y11			;; I3/sine + I4/sine (new I3/sine)		;	6-10
next	vmovapd	y2, [srcreg+(iter+1)*srcinc+d1+32]	;; I2
next	vshufpd	y11, y4, y2, 15				;; Shuffle I1 and I2 to create I1/I2 hi		;		5
next	vshufpd	y4, y4, y2, 0				;; Shuffle I1 and I2 to create I1/I2 low	;		6

this	yfnmaddpd y2, y9, y13, y8			;; I1 - I2/sine * sine (new I2)			; 7-11
this	yfmaddpd y9, y9, y13, y8			;; I1 + I2/sine * sine (new I1)			;	7-11

this no bcast vmovapd y13, [screg1+iter*scinc]		;; Sine
this bcast vbroadcastsd y13, Q [screg1+iter*scinc]	;; Sine
this	yfnmaddpd y8, y0, y13, y3			;; R1 - R3/sine * sine (final R3)		; 10-14
this	yfmaddpd y0, y0, y13, y3			;; R1 + R3/sine * sine (final R1)		;	10-14

next	vmovapd	y3, [srcreg+(iter+1)*srcinc+d2+32]	;; I3
this	ystore	[srcreg+iter*srcinc+d2], y8		;; Save R3					;			15
next	vmovapd	y8, [srcreg+(iter+1)*srcinc+d2+d1+32]	;; I4
this	ystore	[srcreg+iter*srcinc], y0		;; Save R1					;			16
next	vshufpd	y0, y3, y8, 15				;; Shuffle I3 and I4 to create I3/I4 hi		;		7
next	vshufpd	y3, y3, y8, 0				;; Shuffle I3 and I4 to create I3/I4 low	;		8

next	yhigh128s y8, y14, y1				;; Shuffle R1/R2 hi and R3/R4 hi (new R4)	;		9-11

next	ylow128s y14, y14, y1				;; Shuffle R1/R2 hi and R3/R4 hi (new R2)	;		10-12
this	L1prefetchw srcreg+iter*srcinc+L1pd, L1pt

next	yhigh128s y1, y12, y15				;; Shuffle R1/R2 low and R3/R4 low (new R3)	;		11-13
next	ylow128s y12, y12, y15				;; Shuffle R1/R2 low and R3/R4 low (new R1)	;		12-14
this	yfnmaddpd y15, y6, y13, y5			;; R2 - R4/sine * sine (final R4)		; 11-15
this	yfmaddpd y6, y6, y13, y5			;; R2 + R4/sine * sine (final R2)		;	11-15

this	yfnmaddpd y5, y7, y13, y2			;; I2 - I4/sine * sine (final I4)		; 12-16
this	yfmaddpd y7, y7, y13, y2			;; I2 + I4/sine * sine (final I2)		;	12-16
this	L1prefetchw srcreg+iter*srcinc+d1+L1pd, L1pt

next	yhigh128s y2, y11, y0				;; Shuffle I1/I2 hi and I3/I4 hi (new I4)	;		13-15
next	ylow128s y11, y11, y0				;; Shuffle I1/I2 hi and I3/I4 hi (new I2)	;		14-16

this	yfnmaddpd y0, y10, y13, y9			;; I1 - I3/sine * sine (final I3)		; 13-17
this	yfmaddpd y10, y10, y13, y9			;; I1 + I3/sine * sine (final I1)		;	13-17
next no bcast vmovapd y9, [screg1+(iter+1)*scinc+32]	;; cosine/sine
next bcast vbroadcastsd y9, Q [screg1+(iter+1)*scinc+scoff] ;; cosine/sine

next	yhigh128s y13, y4, y3				;; Shuffle I1/I2 low and I3/I4 low (new I3)	;		15-17
this	L1prefetchw srcreg+iter*srcinc+d2+L1pd, L1pt

next	ylow128s y4, y4, y3				;; Shuffle I1/I2 low and I3/I4 low (new I1)	;		16-18

next	yfmsubpd y3, y8, y9, y2				;; R4/sine = R4 * cosine/sine - I4		; 16-20
next	yfmaddpd y2, y2, y9, y8				;; I4/sine = I4 * cosine/sine + R4		;	16-20
next no bcast vmovapd y8, [screg2+(iter+1)*scinc+32]	;; cosine/sine
next bcast vbroadcastsd y8, Q [screg2+(iter+1)*scinc+scoff] ;; cosine/sine

	;; Port 5 stalls 2 clocks here (bubble introduced by shufpd latency 1 vs. vpermf128 latency 3)

this	ystore	[srcreg+iter*srcinc+d2+d1], y15		;; Save R4					;			17
next	yfmaddpd y15, y14, y8, y11			;; R2/sine = R2 * cosine/sine + I2		; 17-21
next	yfmsubpd y11, y11, y8, y14			;; I2/sine = I2 * cosine/sine - R2		;	17-21
this next yloop_unrolled_one

next	yfmaddpd y8, y1, y9, y13			;; R3/sine = R3 * cosine/sine + I3		; 18-22
next	yfmsubpd y13, y13, y9, y1			;; I3/sine = I3 * cosine/sine - R3		;	18-22
this	ystore	[srcreg+iter*srcinc+d1], y6		;; Save R2					;			18
this	L1prefetchw srcreg+iter*srcinc+d2+d1+L1pd, L1pt


;; Shuffle register assignments so that next iteration has this I1,I2,I3,I4 in y0-3 and next R1,R2/sine,R3/sine,R4/sine,I1,I2/sine,I3/sine,I4/sine in y4-y11.

ytmp	TEXTEQU	y0
y0	TEXTEQU	y10
y10	TEXTEQU	y13
y13	TEXTEQU	y1
y1	TEXTEQU	y7
y7	TEXTEQU	y3
y3	TEXTEQU	y5
y5	TEXTEQU	y15
y15	TEXTEQU	y6
y6	TEXTEQU	y8
y8	TEXTEQU	y4
y4	TEXTEQU	y12
y12	TEXTEQU	y9
y9	TEXTEQU	y11
y11	TEXTEQU	y2
y2	TEXTEQU	ytmp
	ENDM

ENDIF

ENDIF


;;
;; ************************************* four-complex-first-fft variants ******************************************
;;

;; This code applies the roots-of-minus-1 premultipliers in a negacyclic one-pass FFT.
;; It also applies the three sin/cos multipliers after a radix-4 butterfly.  We save memory
;; by splitting the roots-of-minus-1 premultipliers such that every macro uses the 
;; same premultiplier data and we have 4 sin/cos postmultipliers.  Every sin/cos postmultiplier,
;; a very big table anyway, is multiplied by the other part of the split roots-of-minus-1.
;; The common premultiplier data is 1, .924+.383i, SQRTHALF+SQRTHALFi, .383+.924i.
;; This scheme is used by the simple radix-4 DJB "r4" FFTs.

yr4_fs4cl_four_complex_first_djbfft_preload MACRO
	ENDM
yr4_fs4cl_four_complex_first_djbfft MACRO srcreg,srcinc,d1,d2,screg,scinc,maxrpt,L1pt,L1pd
	vmovapd	ymm7, YMM_P924
	vmovapd	ymm6, [srcreg+d1][rbx]		;; R2
	vmulpd	ymm4, ymm6, ymm7		;; A2 = R2 * .924		;  1-5
	vmovapd	ymm1, YMM_P383
	vmovapd	ymm2, [srcreg+d1+32][rbx]	;; R6 (I2)
	vmulpd	ymm0, ymm2, ymm1		;; C2 = I2 * .383		;  2-6
	vmulpd	ymm6, ymm6, ymm1		;; B2 = R2 * .383		;  3-7
	vmulpd	ymm2, ymm2, ymm7		;; D2 = I2 * .924		;  4-8
	vmovapd	ymm3, [srcreg+d2+d1][rbx]	;; R4
	vmulpd	ymm5, ymm3, ymm1		;; A4 = R4 * .383		;  5-9
	vsubpd	ymm4, ymm4, ymm0		;; A2 = A2 - C2 (new R2)	; 7-9
	vaddpd	ymm6, ymm6, ymm2		;; B2 = B2 + D2 (new I2)	; 9-11
	vmovapd	ymm2, [srcreg+d2+d1+32][rbx]	;; R8 (I4)
	vmulpd	ymm0, ymm2, ymm7		;; C4 = I4 * .924		;  6-10
	vmulpd	ymm7, ymm3, ymm7		;; B4 = R4 * .924		;  7-11
	vmulpd	ymm1, ymm2, ymm1		;; D4 = I4 * .383		;  8-12
	vsubpd	ymm5, ymm5, ymm0		;; A4 = A4 - C4 (new R4)	; 11-13

	vmovapd	ymm2, [srcreg+d2][rbx]		;; R3
	vmovapd	ymm3, [srcreg+d2+32][rbx]	;; R7 (I3)
	vsubpd	ymm0, ymm2, ymm3		;; A3 = R3 - I3			;  1-3  (move upward in 64-bit version)
	vaddpd	ymm2, ymm2, ymm3		;; B3 = R3 + I3			;  2-4

	L1prefetchw srcreg+L1pd, L1pt

	vaddpd	ymm7, ymm7, ymm1		;; B4 = B4 + D4 (new I4)	; 13-15

	vmovapd	ymm1, YMM_SQRTHALF
	vmulpd	ymm0, ymm0, ymm1		;; A3 = A3 * SQRTHALF (new R3)	;  9-13
	vmulpd	ymm2, ymm2, ymm1		;; B3 = B3 * SQRTHALF (new I3)	;  10-14

	L1prefetchw srcreg+d1+L1pd, L1pt

	vsubpd	ymm3, ymm4, ymm5		;; R2 - R4 (new R4)		; 14-16
	vaddpd	ymm4, ymm4, ymm5		;; R2 + R4 (new R2)		; 15-17

	vsubpd	ymm1, ymm6, ymm7		;; I2 - I4 (new I4)		; 16-18
	vaddpd	ymm6, ymm6, ymm7		;; I2 + I4 (new I2)		; 17-19

	vmovapd	ymm5, [srcreg][rbx]		;; R1
	vsubpd	ymm7, ymm5, ymm0		;; R1 - R3 (new R3)		; 18-20
	vaddpd	ymm5, ymm5, ymm0		;; R1 + R3 (new R1)		; 19-21

	L1prefetchw srcreg+d2+L1pd, L1pt

	vsubpd	ymm0, ymm7, ymm1		;; R3 - I4 (new R3)		; 21-23
	vaddpd	ymm7, ymm7, ymm1		;; R3 + I4 (new R4)		; 22-24
	vsubpd	ymm1, ymm5, ymm4		;; R1 - R2 (new R2)		; 23-25
	vaddpd	ymm5, ymm5, ymm4		;; R1 + R2 (new R1)		; 24-ystore

	ystore	[srcreg], ymm5			;; Save R1			; eliminate in 64-bit version

	vmovapd	ymm4, [srcreg+32][rbx]		;; R5 (I1)
	vsubpd	ymm5, ymm4, ymm2		;; I1 - I3 (new I3)		; 20-22	move forward in 64-bit
	vaddpd	ymm4, ymm4, ymm2		;; I1 + I3 (new I1)		; 25-27

	L1prefetchw srcreg+d2+d1+L1pd, L1pt

	vsubpd	ymm2, ymm5, ymm3		;; I3 - R4 (new I4)		; 26-28
	vaddpd	ymm5, ymm5, ymm3		;; I3 + R4 (new I3)		; 27-29
	vsubpd	ymm3, ymm4, ymm6		;; I1 - I2 (new I2)		; 28-30
	vaddpd	ymm4, ymm4, ymm6		;; I1 + I2 (new I1)		; 29-31

	ystore	[srcreg+32], ymm4		;; Save I1			; eliminate in 64-bit version

	vmovapd	ymm4, [screg+64+32]		;; cosine/sine
	vmulpd	ymm6, ymm0, ymm4		;; A3 = R3 * cosine/sine
	vsubpd	ymm6, ymm6, ymm5		;; A3 = A3 - I3
	vmulpd	ymm5, ymm5, ymm4		;; B3 = I3 * cosine/sine
	vaddpd	ymm5, ymm5, ymm0		;; B3 = B3 + R3

	vmovapd	ymm4, [screg+128+32]		;; cosine/sine
	vmulpd	ymm0, ymm1, ymm4		;; A2 = R2 * cosine/sine
	vsubpd	ymm0, ymm0, ymm3		;; A2 = A2 - I2
	vmulpd	ymm3, ymm3, ymm4		;; B2 = I2 * cosine/sine
	vaddpd	ymm3, ymm3, ymm1		;; B2 = B2 + R2

	vmovapd	ymm4, [screg+192+32]		;; cosine/sine
	vmulpd	ymm1, ymm7, ymm4		;; A4 = R4 * cosine/sine
	vsubpd	ymm1, ymm1, ymm2		;; A4 = A4 - I4
	vmulpd	ymm2, ymm2, ymm4		;; B4 = I4 * cosine/sine
	vaddpd	ymm2, ymm2, ymm7		;; B4 = B4 + R4

	vmovapd	ymm4, [srcreg]			;; Restore R1
	vmulpd	ymm4, ymm4, [screg+0+32]	;; A1 = R1 * cosine/sine
	vmovapd	ymm7, [srcreg+32]		;; Restore I1
	vsubpd	ymm4, ymm4, ymm7		;; A1 = A1 - I1
	vmulpd	ymm7, ymm7, [screg+0+32]	;; B1 = I1 * cosine/sine
	vaddpd	ymm7, ymm7, [srcreg]		;; B1 = B1 + R1

	vmulpd	ymm6, ymm6, [screg+64]		;; A3 = A3 * sine (final R3)
	vmulpd	ymm5, ymm5, [screg+64]		;; B3 = B3 * sine (final I3)
	vmulpd	ymm0, ymm0, [screg+128]		;; A2 = A2 * sine (final R2)
	vmulpd	ymm3, ymm3, [screg+128]		;; B2 = B2 * sine (final I2)
	vmulpd	ymm1, ymm1, [screg+192]		;; A4 = A4 * sine (final R4)
	vmulpd	ymm2, ymm2, [screg+192]		;; B4 = B4 * sine (final I4)
	vmulpd	ymm4, ymm4, [screg+0]		;; A1 = A1 * sine (final R1)
	vmulpd	ymm7, ymm7, [screg+0]		;; B1 = B1 * sine (final I1)

	ystore	[srcreg+32], ymm7		;; Save I1			; eliminate in 64-bit version

	;; R1 (0 1 2 3) is ymm4, R2 (8 9 10 11) is ymm0, R3 is ymm6, R4 is ymm1, I1 is in srcreg+32, I2 is ymm3, I3 is ymm5, I4 is ymm2

	;; Swizzle the 64-byte cache lines to hold these data values:
	;;	0	8	16	24	4	12	20	28
	;;	1	9	...
	;;	2	10	...
	;;	3	11	...

	;; Swizzle using 8 shufpds and 8 perm2f128  (1 or 3 clocks each on port 5, throughput 1)

	vshufpd	ymm7, ymm4, ymm0, 0		;; Shuffle R1 and R2 to create 0 8 2 10
	vshufpd	ymm4, ymm4, ymm0, 15		;; Shuffle R1 and R2 to create 1 9 3 11

	vshufpd	ymm0, ymm6, ymm1, 0		;; Shuffle R3 and R4 to create 16 24 18 26
	vshufpd	ymm6, ymm6, ymm1, 15		;; Shuffle R3 and R4 to create 17 25 19 27

	ylow128s ymm1, ymm7, ymm0		;; Shuffle R1/R2 low and R3/R4 low (0 8 16 24)
	yhigh128s ymm7, ymm7, ymm0		;; Shuffle R1/R2 low and R3/R4 low (2 10 18 26)

	ylow128s ymm0, ymm4, ymm6		;; Shuffle R1/R2 hi and R3/R4 hi (1 9 17 25)
	yhigh128s ymm4, ymm4, ymm6		;; Shuffle R1/R2 hi and R3/R4 hi (3 11 19 27)

	ystore	[srcreg], ymm1			;; Save (0 8 16 24)
	ystore	[srcreg+d2], ymm7		;; Save (2 10 18 26)
	ystore	[srcreg+d1], ymm0		;; Save (1 9 17 25)
	ystore	[srcreg+d2+d1], ymm4		;; Save (3 11 19 27)

	vmovapd	ymm1, [srcreg+32]		;; Reload saved I1

	vshufpd	ymm7, ymm1, ymm3, 0		;; Shuffle I1 and I2 to create 4 12 6 14
	vshufpd	ymm1, ymm1, ymm3, 15		;; Shuffle I1 and I2 to create 5 13 7 15

	vshufpd	ymm3, ymm5, ymm2, 0		;; Shuffle I3 and I4 to create 16 28 22 30
	vshufpd	ymm5, ymm5, ymm2, 15		;; Shuffle I3 and I4 to create 21 29 23 31

	ylow128s ymm2, ymm7, ymm3		;; Shuffle I1/I2 low and I3/I4 low (4 12 20 28)
	yhigh128s ymm7, ymm7, ymm3		;; Shuffle I1/I2 low and I3/I4 low (6 14 22 30)

	ylow128s ymm3, ymm1, ymm5		;; Shuffle I1/I2 hi and I3/I4 hi (5 13 21 29)
	yhigh128s ymm1, ymm1, ymm5		;; Shuffle I1/I2 hi and I3/I4 hi (7 15 23 31)

	ystore	[srcreg+32], ymm2		;; Save (4 12 20 28)
	ystore	[srcreg+d2+32], ymm7		;; Save (6 14 22 30)
	ystore	[srcreg+d1+32], ymm3		;; Save (5 13 21 29)
	ystore	[srcreg+d2+d1+32], ymm1		;; Save (7 15 23 31)

	bump	srcreg, srcinc
	bump	screg, scinc
	ENDM

;; This code applies the roots-of-minus-1 premultipliers in a negacyclic FFT.
;; It also applies the three sin/cos multipliers after the first radix-4 butterfly.
;; We save memory by splitting the roots-of-minus-1 premultipliers such that every
;; macro uses the same premultiplier data.  The rest of the premultiplier is applied
;; at the end of pass 1.  This scheme is used by the r4delay and r4dwpn FFTs.

; The sin/cos and premultiplier data is combined in one table.  The premultiplier data
; is at screg the sin/cos data is at screg+256.
yr4_4cl_csc_four_complex_first_djbfft MACRO srcreg,srcinc,d1,d2,screg,scinc,maxrpt,L1pt,L1pd
	vmovapd ymm6, [screg+0+32]		;; cosine/sine
	vmovapd	ymm1, [srcreg]			;; R1
	vmulpd	ymm7, ymm1, ymm6		;; A1 = R1 * cosine/sine
	vmovapd	ymm2, [srcreg+32]		;; R5 (I1)
	vsubpd	ymm7, ymm7, ymm2		;; A1 = A1 - I1
	vmulpd	ymm2, ymm2, ymm6		;; B1 = I1 * cosine/sine
	vaddpd	ymm2, ymm2, ymm1		;; B1 = B1 + R1

	vmovapd ymm3, [screg+128+32]		;; cosine/sine
	vmovapd	ymm0, [srcreg+d2]		;; R3
	vmulpd	ymm5, ymm0, ymm3		;; A3 = R3 * cosine/sine
	vmovapd	ymm1, [srcreg+d2+32]		;; R7 (I3)
	vsubpd	ymm5, ymm5, ymm1		;; A3 = A3 - I3
	vmulpd	ymm1, ymm1, ymm3		;; B3 = I3 * cosine/sine
	vaddpd	ymm1, ymm1, ymm0		;; B3 = B3 + R3

	vmovapd ymm0, [screg+0]			;; sine
	vmulpd	ymm7, ymm7, ymm0		;; A1 = A1 * sine
	vmulpd	ymm2, ymm2, ymm0		;; B1 = B1 * sine
	vmovapd ymm0, [screg+128]		;; sine
	vmulpd	ymm5, ymm5, ymm0		;; A3 = A3 * sine
	vmulpd	ymm1, ymm1, ymm0		;; B3 = B3 * sine

	vsubpd	ymm0, ymm7, ymm5		;; R1 - R3 (new R3)
	vaddpd	ymm7, ymm7, ymm5		;; R1 + R3 (new R1)

	L1prefetchw srcreg+L1pd, L1pt

	vsubpd	ymm5, ymm2, ymm1		;; I1 - I3 (new I3)
	vaddpd	ymm2, ymm2, ymm1		;; I1 + I3 (new I1)

	ystore	[srcreg], ymm7			;; Temporarily save new R1
	ystore	[srcreg+32], ymm2		;; Temporarily save new I1

	vmovapd ymm7, [screg+64+32]		;; cosine/sine
	vmovapd	ymm3, [srcreg+d1]		;; R2
	vmulpd	ymm6, ymm3, ymm7		;; A2 = R2 * cosine/sine
	vmovapd	ymm4, [srcreg+d1+32]		;; R6 (I2)
	vsubpd	ymm6, ymm6, ymm4		;; A2 = A2 - I2
	vmulpd	ymm4, ymm4, ymm7		;; B2 = I2 * cosine/sine
	vaddpd	ymm4, ymm4, ymm3		;; B2 = B2 + R2

	vmovapd ymm7, [screg+192+32]		;; cosine/sine
	vmovapd	ymm1, [srcreg+d2+d1]		;; R4
	vmulpd	ymm3, ymm1, ymm7		;; A4 = R4 * cosine/sine
	vmovapd	ymm2, [srcreg+d2+d1+32]		;; R8 (I4)
	vsubpd	ymm3, ymm3, ymm2		;; A4 = A4 - I4
	vmulpd	ymm2, ymm2, ymm7		;; B4 = I4 * cosine/sine
	vaddpd	ymm2, ymm2, ymm1		;; B4 = B4 + R4

	L1prefetchw srcreg+d1+L1pd, L1pt

	vmovapd ymm1, [screg+64]		;; sine
	vmulpd	ymm6, ymm6, ymm1		;; A2 = A2 * sine
	vmulpd	ymm4, ymm4, ymm1		;; B2 = B2 * sine
	vmovapd ymm1, [screg+192]		;; sine
	vmulpd	ymm3, ymm3, ymm1		;; A4 = A4 * sine
	vmulpd	ymm2, ymm2, ymm1		;; B4 = B4 * sine

	vsubpd	ymm1, ymm6, ymm3		;; R2 - R4 (new R4)
	vaddpd	ymm6, ymm6, ymm3		;; R2 + R4 (new R2)

	vsubpd	ymm3, ymm4, ymm2		;; I2 - I4 (new I4)
	vaddpd	ymm4, ymm4, ymm2		;; I2 + I4 (new I2)

	L1prefetchw srcreg+d2+L1pd, L1pt

	vsubpd	ymm2, ymm0, ymm3		;; R3 - I4 (newer R3)
	vaddpd	ymm0, ymm0, ymm3		;; R3 + I4 (newer R4)

	vsubpd	ymm3, ymm5, ymm1		;; I3 - R4 (newer I4)
	vaddpd	ymm5, ymm5, ymm1		;; I3 + R4 (newer I3)

	vmovapd	ymm1, [screg+256+0+32]		;; cosine/sine
	vmulpd	ymm7, ymm2, ymm1		;; A3 = R3 * cosine/sine
	vsubpd	ymm7, ymm7, ymm5		;; A3 = A3 - I3
	vmulpd	ymm5, ymm5, ymm1		;; B3 = I3 * cosine/sine
	vaddpd	ymm5, ymm5, ymm2		;; B3 = B3 + R3

	vmulpd	ymm2, ymm0, ymm1		;; A4 = R4 * cosine/sine
	vaddpd	ymm2, ymm2, ymm3		;; A4 = A4 + I4
	vmulpd	ymm3, ymm3, ymm1		;; B4 = I4 * cosine/sine
	vsubpd	ymm3, ymm3, ymm0		;; B4 = B4 - R4

	L1prefetchw srcreg+d2+d1+L1pd, L1pt

	vmovapd	ymm0, [screg+256+0]
	vmulpd	ymm7, ymm7, ymm0		;; A3 = A3 * sine (final R3)
	vmulpd	ymm5, ymm5, ymm0		;; B3 = B3 * sine (final I3)
	vmulpd	ymm2, ymm2, ymm0		;; A4 = A4 * sine (final R4)
	vmulpd	ymm3, ymm3, ymm0		;; B4 = B4 * sine (final I4)

	ystore	[srcreg+d2], ymm7		;; Save R3
	ystore	[srcreg+d2+32], ymm5		;; Save I3
	ystore	[srcreg+d2+d1], ymm2		;; Save R4
	ystore	[srcreg+d2+d1+32], ymm3		;; Save I4

	vmovapd	ymm0, [srcreg] 			;; Reload new R1
	vmovapd	ymm1, [srcreg+32] 		;; Reload new I1

	vsubpd	ymm2, ymm0, ymm6		;; R1 - R2 (newer R2)
	vaddpd	ymm0, ymm0, ymm6		;; R1 + R2 (newer R1)

	vsubpd	ymm3, ymm1, ymm4		;; I1 - I2 (newer I2)
	vaddpd	ymm1, ymm1, ymm4		;; I1 + I2 (newer I1)

	ystore	[srcreg], ymm0			;; Save R1
	ystore	[srcreg+32], ymm1		;; Save I1

	vmovapd	ymm7, [screg+256+64+32]		;; cosine/sine
	vmulpd	ymm4, ymm2, ymm7		;; A2 = R2 * cosine/sine
	vsubpd	ymm4, ymm4, ymm3		;; A2 = A2 - I2
	vmulpd	ymm3, ymm3, ymm7		;; B2 = I2 * cosine/sine
	vaddpd	ymm3, ymm3, ymm2		;; B2 = B2 + R2

	vmulpd	ymm4, ymm4, [screg+256+64]	;; A2 = A2 * sine (final R2)
	vmulpd	ymm3, ymm3, [screg+256+64]	;; B2 = B2 * sine (final I2)

	ystore	[srcreg+d1], ymm4		;; Save R2
	ystore	[srcreg+d1+32], ymm3		;; Save I2
	bump	srcreg, srcinc
	bump	screg, scinc
	ENDM

;; 64-bit version

IFDEF X86_64

yr4_fs4cl_four_complex_first_djbfft_preload MACRO
	vbroadcastsd ymm15, YMM_P924
	vbroadcastsd ymm14, YMM_P383
	vbroadcastsd ymm13, YMM_SQRTHALF
	ENDM
yr4_fs4cl_four_complex_first_djbfft MACRO srcreg,srcinc,d1,d2,screg,scinc,maxrpt,L1pt,L1pd
	vmovapd	ymm6, [srcreg+d1][rbx]		;; R2
	vmovapd	ymm2, [srcreg+d1+32][rbx]	;; R6 (I2)
	vmulpd	ymm4, ymm6, ymm15		;; A2 = R2 * .924		; 1-5
	vmulpd	ymm0, ymm2, ymm14		;; C2 = I2 * .383		; 2-6
	vmulpd	ymm6, ymm6, ymm14		;; B2 = R2 * .383		; 3-7
	vmulpd	ymm2, ymm2, ymm15		;; D2 = I2 * .924		; 4-8

	vmovapd	ymm8, [srcreg+d2][rbx]		;; R3
	vmovapd	ymm10, [srcreg+d2+32][rbx]	;; R7 (I3)
	vsubpd	ymm9, ymm8, ymm10		;; A3 = R3 - I3			; 1-3
	vaddpd	ymm8, ymm8, ymm10		;; B3 = R3 + I3			; 2-4

	vmulpd	ymm9, ymm9, ymm13		;; A3 = A3 * SQRTHALF (new R3)	; 5-9
	vmulpd	ymm8, ymm8, ymm13		;; B3 = B3 * SQRTHALF (new I3)	; 6-10

	vmovapd	ymm7, [srcreg+d2+d1][rbx]	;; R4
	vmovapd	ymm1, [srcreg+d2+d1+32][rbx]	;; R8 (I4)
	vmulpd	ymm5, ymm7, ymm14		;; A4 = R4 * .383		; 7-11
	vmulpd	ymm3, ymm1, ymm15		;; C4 = I4 * .924		; 8-12
	vmulpd	ymm7, ymm7, ymm15		;; B4 = R4 * .924		; 9-13
	vmulpd	ymm1, ymm1, ymm14		;; D4 = I4 * .383		; 10-14

	vsubpd	ymm4, ymm4, ymm0		;; A2 = A2 - C2 (new R2)	; 7-9
	vaddpd	ymm6, ymm6, ymm2		;; B2 = B2 + D2 (new I2)	; 9-11

	vmovapd	ymm0, [srcreg][rbx]		;; R1
	vsubpd	ymm2, ymm0, ymm9		;; R1 - R3 (new R3)		; 10-12		r10
	vaddpd	ymm0, ymm0, ymm9		;; R1 + R3 (new R1)		; 11-13

	vmovapd	ymm9, [srcreg+32][rbx]		;; R5 (I1)
	vsubpd	ymm10, ymm9, ymm8		;; I1 - I3 (new I3)		; 12-14		r11

	vsubpd	ymm5, ymm5, ymm3		;; A4 = A4 - C4 (new R4)	; 13-15

	vaddpd	ymm9, ymm9, ymm8		;; I1 + I3 (new I1)		; 14-16

	vaddpd	ymm7, ymm7, ymm1		;; B4 = B4 + D4 (new I4)	; 15-17

; two aparts

	vsubpd	ymm3, ymm4, ymm5		;; R2 - R4 (new R4)		; 16-18		r16
	vaddpd	ymm4, ymm4, ymm5		;; R2 + R4 (new R2)		; 17-19

	vsubpd	ymm5, ymm6, ymm7		;; I2 - I4 (new I4)		; 18-20		r18
	vaddpd	ymm6, ymm6, ymm7		;; I2 + I4 (new I2)		; 19-21

; one aparts

	vsubpd	ymm7, ymm10, ymm3		;; I3 - R4 (new I4)		; 20-22		r19
	vaddpd	ymm10, ymm10, ymm3		;; I3 + R4 (new I3)		; 21-23		r19
	vsubpd	ymm3, ymm0, ymm4		;; R1 - R2 (new R2)		; 22-24		r20
	vaddpd	ymm0, ymm0, ymm4		;; R1 + R2 (new R1)		; 23-25		r20
	vsubpd	ymm4, ymm2, ymm5		;; R3 - I4 (new R3)		; 24-26		r21
	vaddpd	ymm2, ymm2, ymm5		;; R3 + I4 (new R4)		; 25-27		r21
	vsubpd	ymm5, ymm9, ymm6		;; I1 - I2 (new I2)		; 26-28		r22
	vaddpd	ymm9, ymm9, ymm6		;; I1 + I2 (new I1)		; 27-29		r22

	vmovapd	ymm12, [screg+192+32]		;; cosine/sine
	vmulpd	ymm6, ymm7, ymm12		;; B4 = I4 * cosine/sine	; 23-27		r23
	vaddpd	ymm6, ymm6, ymm2		;; B4 = B4 + R4			; 28-30
	vmulpd	ymm2, ymm2, ymm12		;; A4 = R4 * cosine/sine	; 28-32		r28
	vsubpd	ymm2, ymm2, ymm7		;; A4 = A4 - I4			; 33-35

	vmovapd	ymm12, [screg+64+32]		;; cosine/sine
	vmulpd	ymm7, ymm10, ymm12		;; B3 = I3 * cosine/sine	; 24-28		r24
	vaddpd	ymm7, ymm7, ymm4		;; B3 = B3 + R3			; 29-31
	vmulpd	ymm4, ymm4, ymm12		;; A3 = R3 * cosine/sine	; 27-31		r27
	vsubpd	ymm4, ymm4, ymm10		;; A3 = A3 - I3			; 32-34

	vmovapd	ymm12, [screg+128+32]		;; cosine/sine
	vmulpd	ymm10, ymm3, ymm12		;; A2 = R2 * cosine/sine	; 25-29		r25
	vsubpd	ymm10, ymm10, ymm5		;; A2 = A2 - I2			; 30-32
	vmulpd	ymm5, ymm5, ymm12		;; B2 = I2 * cosine/sine	; 29-33		r29
	vaddpd	ymm5, ymm5, ymm3		;; B2 = B2 + R2			; 34-36

	vmovapd	ymm12, [screg+0+32]		;; cosine/sine
	vmulpd	ymm3, ymm0, ymm12		;; A1 = R1 * cosine/sine	; 26-30		r26
	vsubpd	ymm3, ymm3, ymm9		;; A1 = A1 - I1			; 31-33
	vmulpd	ymm9, ymm9, ymm12		;; B1 = I1 * cosine/sine	; 30-34		r30
	vaddpd	ymm9, ymm9, ymm0		;; B1 = B1 + R1			; 35-37

;preloads
	vmovapd	ymm12, [screg+192]		;; sine
	vmulpd	ymm6, ymm6, ymm12		;; B4 = B4 * sine (final I4)
	vmulpd	ymm7, ymm7, [screg+64]		;; B3 = B3 * sine (final I3)
	vmulpd	ymm10, ymm10, [screg+128]	;; A2 = A2 * sine (final R2)
	vmulpd	ymm3, ymm3, [screg+0]		;; A1 = A1 * sine (final R1)
	vmulpd	ymm4, ymm4, [screg+64]		;; A3 = A3 * sine (final R3)
	vmulpd	ymm2, ymm2, ymm12		;; A4 = A4 * sine (final R4)
	vmulpd	ymm5, ymm5, [screg+128]		;; B2 = B2 * sine (final I2)
	vmulpd	ymm9, ymm9, [screg+0]		;; B1 = B1 * sine (final I1)
	bump	screg, scinc

	L1prefetchw srcreg+d1+L1pd, L1pt
	vshufpd	ymm0, ymm3, ymm10, 0		;; Shuffle R1 and R2 to create 0 8 2 10
	vshufpd	ymm3, ymm3, ymm10, 15		;; Shuffle R1 and R2 to create 1 9 3 11

	L1prefetchw srcreg+d2+L1pd, L1pt
	vshufpd	ymm10, ymm4, ymm2, 0		;; Shuffle R3 and R4 to create 16 24 18 26
	vshufpd	ymm4, ymm4, ymm2, 15		;; Shuffle R3 and R4 to create 17 25 19 27

	L1prefetchw srcreg+d2+d1+L1pd, L1pt
	ylow128s ymm2, ymm0, ymm10		;; Shuffle R1/R2 low and R3/R4 low (0 8 16 24)
	yhigh128s ymm0, ymm0, ymm10		;; Shuffle R1/R2 low and R3/R4 low (2 10 18 26)

	L1prefetchw srcreg+L1pd, L1pt
	ylow128s ymm10, ymm3, ymm4		;; Shuffle R1/R2 hi and R3/R4 hi (1 9 17 25)
	yhigh128s ymm3, ymm3, ymm4		;; Shuffle R1/R2 hi and R3/R4 hi (3 11 19 27)

	ystore	[srcreg], ymm2			;; Save (0 8 16 24)
	ystore	[srcreg+d2], ymm0		;; Save (2 10 18 26)
	ystore	[srcreg+d1], ymm10		;; Save (1 9 17 25)
	ystore	[srcreg+d2+d1], ymm3		;; Save (3 11 19 27)

	vshufpd	ymm0, ymm9, ymm5, 0		;; Shuffle I1 and I2 to create 4 12 6 14
	vshufpd	ymm9, ymm9, ymm5, 15		;; Shuffle I1 and I2 to create 5 13 7 15

	vshufpd	ymm5, ymm7, ymm6, 0		;; Shuffle I3 and I4 to create 16 28 22 30
	vshufpd	ymm7, ymm7, ymm6, 15		;; Shuffle I3 and I4 to create 21 29 23 31

	ylow128s ymm6, ymm0, ymm5		;; Shuffle I1/I2 low and I3/I4 low (4 12 20 28)
	yhigh128s ymm0, ymm0, ymm5		;; Shuffle I1/I2 low and I3/I4 low (6 14 22 30)

	ylow128s ymm5, ymm9, ymm7		;; Shuffle I1/I2 hi and I3/I4 hi (5 13 21 29)
	yhigh128s ymm9, ymm9, ymm7		;; Shuffle I1/I2 hi and I3/I4 hi (7 15 23 31)

	ystore	[srcreg+32], ymm6		;; Save (4 12 20 28)
	ystore	[srcreg+d2+32], ymm0		;; Save (6 14 22 30)
	ystore	[srcreg+d1+32], ymm5		;; Save (5 13 21 29)
	ystore	[srcreg+d2+d1+32], ymm9		;; Save (7 15 23 31)
	bump	srcreg, srcinc
	ENDM

yr4_4cl_csc_four_complex_first_djbfft MACRO srcreg,srcinc,d1,d2,screg,scinc,maxrpt,L1pt,L1pd
	IF maxrpt MOD 5 EQ 0
	yr4_4c_first_djbfft_unroll -1,noexec,noexec,exec,srcreg,srcinc,d1,d2,screg,scinc,L1pt,L1pd
	yr4_4c_first_djbfft_unroll 0,noexec,exec,exec,srcreg,srcinc,d1,d2,screg,scinc,L1pt,L1pd
	yr4_4c_first_djbfft_unroll 1,exec,exec,exec,srcreg,srcinc,d1,d2,screg,scinc,L1pt,L1pd
	yr4_4c_first_djbfft_unroll 2,exec,exec,exec,srcreg,srcinc,d1,d2,screg,scinc,L1pt,L1pd
	yr4_4c_first_djbfft_unroll 3,exec,exec,exec,srcreg,srcinc,d1,d2,screg,scinc,L1pt,L1pd
	yr4_4c_first_djbfft_unroll 4,exec,exec,noexec,srcreg,srcinc,d1,d2,screg,scinc,L1pt,L1pd
	yr4_4c_first_djbfft_unroll 5,exec,noexec,noexec,srcreg,srcinc,d1,d2,screg,scinc,L1pt,L1pd
	bump	srcreg, 5*srcinc
	bump	screg, 5*scinc
	ELSEIF maxrpt MOD 4 EQ 0
	yr4_4c_first_djbfft_unroll -1,noexec,noexec,exec,srcreg,srcinc,d1,d2,screg,scinc,L1pt,L1pd
	yr4_4c_first_djbfft_unroll 0,noexec,exec,exec,srcreg,srcinc,d1,d2,screg,scinc,L1pt,L1pd
	yr4_4c_first_djbfft_unroll 1,exec,exec,exec,srcreg,srcinc,d1,d2,screg,scinc,L1pt,L1pd
	yr4_4c_first_djbfft_unroll 2,exec,exec,exec,srcreg,srcinc,d1,d2,screg,scinc,L1pt,L1pd
	yr4_4c_first_djbfft_unroll 3,exec,exec,noexec,srcreg,srcinc,d1,d2,screg,scinc,L1pt,L1pd
	yr4_4c_first_djbfft_unroll 4,exec,noexec,noexec,srcreg,srcinc,d1,d2,screg,scinc,L1pt,L1pd
	bump	srcreg, 4*srcinc
	bump	screg, 4*scinc
	ELSEIF maxrpt MOD 3 EQ 0
	yr4_4c_first_djbfft_unroll -1,noexec,noexec,exec,srcreg,srcinc,d1,d2,screg,scinc,L1pt,L1pd
	yr4_4c_first_djbfft_unroll 0,noexec,exec,exec,srcreg,srcinc,d1,d2,screg,scinc,L1pt,L1pd
	yr4_4c_first_djbfft_unroll 1,exec,exec,exec,srcreg,srcinc,d1,d2,screg,scinc,L1pt,L1pd
	yr4_4c_first_djbfft_unroll 2,exec,exec,noexec,srcreg,srcinc,d1,d2,screg,scinc,L1pt,L1pd
	yr4_4c_first_djbfft_unroll 3,exec,noexec,noexec,srcreg,srcinc,d1,d2,screg,scinc,L1pt,L1pd
	bump	srcreg, 3*srcinc
	bump	screg, 3*scinc
	ELSEIF maxrpt MOD 2 EQ 0
	yr4_4c_first_djbfft_unroll -1,noexec,noexec,exec,srcreg,srcinc,d1,d2,screg,scinc,L1pt,L1pd
	yr4_4c_first_djbfft_unroll 0,noexec,exec,exec,srcreg,srcinc,d1,d2,screg,scinc,L1pt,L1pd
	yr4_4c_first_djbfft_unroll 1,exec,exec,noexec,srcreg,srcinc,d1,d2,screg,scinc,L1pt,L1pd
	yr4_4c_first_djbfft_unroll 2,exec,noexec,noexec,srcreg,srcinc,d1,d2,screg,scinc,L1pt,L1pd
	bump	srcreg, 2*srcinc
	bump	screg, 2*scinc
	ELSE
	yr4_4c_first_djbfft_unroll -1,noexec,noexec,exec,srcreg,srcinc,d1,d2,screg,scinc,L1pt,L1pd
	yr4_4c_first_djbfft_unroll 0,noexec,exec,noexec,srcreg,srcinc,d1,d2,screg,scinc,L1pt,L1pd
	yr4_4c_first_djbfft_unroll 1,exec,noexec,noexec,srcreg,srcinc,d1,d2,screg,scinc,L1pt,L1pd
	bump	srcreg, srcinc
	bump	screg, scinc
	ENDIF
	ENDM

yr4_4c_first_djbfft_unroll MACRO iter,prev,this,next,srcreg,srcinc,d1,d2,screg,scinc,L1pt,L1pd

;; On first call, init registers

	IF iter EQ -1
y0	TEXTEQU	<ymm0>
y1	TEXTEQU	<ymm1>
y2	TEXTEQU	<ymm2>
y3	TEXTEQU	<ymm3>
y4	TEXTEQU	<ymm4>
y5	TEXTEQU	<ymm5>
y6	TEXTEQU	<ymm6>
y7	TEXTEQU	<ymm7>
y8	TEXTEQU	<ymm8>
y9	TEXTEQU	<ymm9>
y10	TEXTEQU	<ymm10>
y11	TEXTEQU	<ymm11>
y12	TEXTEQU	<ymm12>
y13	TEXTEQU	<ymm13>
y14	TEXTEQU	<ymm14>
y15	TEXTEQU	<ymm15>
	ENDIF

;; On later calls, previous A2,B2,A3,B3,A4,B4 will be in y0-5.  This R1,R3,I1,I2,I3,A1,A3,B1,B3,c/s2 will be in y6-15.

this	vsubpd	y11, y11, y8			;; A1 = A1 - I1					; 1-3
this	vmulpd	y8, y9, y15			;; B2 = I2 * cosine/sine			;	0-4

this	vsubpd	y12, y12, y10			;; A3 = A3 - I3					; 2-4
this	vmovapd	y10, [srcreg+iter*srcinc+d2+d1+32] ;; R8 (I4)

this	vaddpd	y13, y13, y6			;; B1 = B1 + R1					; 3-5
this	vmulpd	y6, y10, [screg+iter*scinc+192+32] ;; B4 = I4 * cosine/sine			;	1-5

this	vaddpd	y14, y14, y7			;; B3 = B3 + R3					; 4-6
this	vmovapd	y7, [srcreg+iter*srcinc+d1]	;; R2
this	vmulpd	y15, y7, y15			;; A2 = R2 * cosine/sine			;	2-6

this	vaddpd	y8, y8, y7			;; B2 = B2 + R2					; 5-7
this	vmovapd	y7, [srcreg+iter*srcinc+d2+d1]	;; R4

this	vaddpd	y6, y6, y7			;; B4 = B4 + R4					; 6-8
this	vmulpd	y7, y7, [screg+iter*scinc+192+32] ;; A4 = R4 * cosine/sine			;	3-7

this	vsubpd	y15, y15, y9			;; A2 = A2 - I2					; 7-9
this	vmovapd y9, [screg+iter*scinc+0]	;; sine
this	vmulpd	y11, y11, y9			;; A1 = A1 * sine				;	4-8

this	vsubpd	y7, y7, y10			;; A4 = A4 - I4					; 8-10
this	vmovapd y10, [screg+iter*scinc+128]	;; sine
this	vmulpd	y12, y12, y10			;; A3 = A3 * sine				;	5-9
this	vmulpd	y13, y13, y9			;; B1 = B1 * sine				;	6-10
this	vmulpd	y14, y14, y10			;; B3 = B3 * sine				;	7-11
this	vmovapd y9, [screg+iter*scinc+64]	;; sine
this	vmulpd	y8, y8, y9			;; B2 = B2 * sine				;	8-12

this	vmovapd y10, [screg+iter*scinc+192]	;; sine
this	vmulpd	y6, y6, y10			;; B4 = B4 * sine				;	9-13

this	vmulpd	y15, y15, y9			;; A2 = A2 * sine				;	10-14
this	vsubpd	y9, y11, y12			;; R1 - R3 (new R3)				; 10-12

this	vaddpd	y11, y11, y12			;; R1 + R3 (new R1)				; 11-13
this	vmulpd	y7, y7, y10			;; A4 = A4 * sine				;	11-15
prev	vmovapd	y12, [screg+(iter-1)*scinc+256+0]

this	vsubpd	y10, y13, y14			;; I1 - I3 (new I3)				; 12-14
prev	vmulpd	y2, y2, y12			;; A3 = A3 * sine (final R3)			;	12-16
this next yloop_unrolled_one

this	vaddpd	y13, y13, y14			;; I1 + I3 (new I1)				; 13-15
prev	vmulpd	y3, y3, y12			;; B3 = B3 * sine (final I3)			;	13-17

this	vsubpd	y14, y8, y6			;; I2 - I4 (new I4)				; 14-16
prev	vmulpd	y4, y4, y12			;; A4 = A4 * sine (final R4)			;	14-18
this	L1prefetchw srcreg+iter*srcinc+L1pd, L1pt

this	vaddpd	y8, y8, y6			;; I2 + I4 (new I2)				; 15-17
prev	vmulpd	y5, y5, y12			;; B4 = B4 * sine (final I4)			;	15-19
prev	vmovapd	y6, [screg+(iter-1)*scinc+256+64] ;; sine

this	vsubpd	y12, y15, y7			;; R2 - R4 (new R4)				; 16-18
prev	vmulpd	y0, y0, y6			;; A2 = A2 * sine (final R2)			;	16-20
this	L1prefetchw srcreg+iter*srcinc+d1+L1pd, L1pt

this	vaddpd	y15, y15, y7			;; R2 + R4 (new R2)				; 17-19
prev	vmulpd	y1, y1, y6			;; B2 = B2 * sine (final I2)			;	17-21
this	vmovapd	y7, [screg+iter*scinc+256+0+32]	;; cosine/sine
prev	ystore	[srcreg+(iter-1)*srcinc+d2], y2	;; Save R3					; 17

this	vsubpd	y6, y9, y14			;; R3 - I4 (newer R3)				; 18-20
this	vmovapd	y2, [screg+iter*scinc+256+64+32] ;; cosine/sine
prev	ystore	[srcreg+(iter-1)*srcinc+d2+32], y3 ;; Save I3					; 18

this	vaddpd	y3, y10, y12			;; I3 + R4 (newer I3)				; 19-21
prev	ystore	[srcreg+(iter-1)*srcinc+d2+d1], y4 ;; Save R4					; 19

this	vaddpd	y9, y9, y14			;; R3 + I4 (newer R4)				; 20-22
next	vmovapd y4, [screg+(iter+1)*scinc+0+32]	;; cosine/sine
prev	ystore	[srcreg+(iter-1)*srcinc+d2+d1+32], y5 ;; Save I4				; 20

this	vsubpd	y10, y10, y12			;; I3 - R4 (newer I4)				; 21-23
this	vmulpd	y12, y6, y7			;; A3 = R3 * cosine/sine			;	21-25
prev	ystore	[srcreg+(iter-1)*srcinc+d1], y0 ;; Save R2					; 21

this	vsubpd	y0, y11, y15			;; R1 - R2 (newer R2)				; 22-24
this	vmulpd	y5, y3, y7			;; B3 = I3 * cosine/sine			;	22-26
prev	ystore	[srcreg+(iter-1)*srcinc+d1+32], y1 ;; Save I2					; 22

this	vsubpd	y1, y13, y8			;; I1 - I2 (newer I2)				; 23-25		
this	vmulpd	y14, y9, y7			;; A4 = R4 * cosine/sine			;	23-27
this	L1prefetchw srcreg+iter*srcinc+d2+L1pd, L1pt

this	vaddpd	y11, y11, y15			;; R1 + R2 (newer R1)				; 24-26
this	vmulpd	y7, y10, y7			;; B4 = I4 * cosine/sine			;	24-28
next	vmovapd	y15, [srcreg+(iter+1)*srcinc]	;; R1

this	vaddpd	y13, y13, y8			;; I1 + I2 (newer I1)				; 25-27
this	vmulpd	y8, y0, y2			;; A2 = R2 * cosine/sine			;	25-29
this	L1prefetchw srcreg+iter*srcinc+d2+d1+L1pd, L1pt

this	vsubpd	y12, y12, y3			;; A3 = A3 - I3					; 26-28
this	vmulpd	y2, y1, y2			;; B2 = I2 * cosine/sine			;	26-30
next	vmovapd y3, [screg+(iter+1)*scinc+128+32] ;; cosine/sine

this	vaddpd	y5, y5, y6			;; B3 = B3 + R3					; 27-29
next	vmulpd	y6, y15, y4			;; A1 = R1 * cosine/sine			;	27-31
this	ystore	[srcreg+iter*srcinc], y11	;; Save R1					; 27
next	vmovapd	y11, [srcreg+(iter+1)*srcinc+d2] ;; R3

this	vaddpd	y14, y14, y10			;; A4 = A4 + I4					; 28-30
next	vmulpd	y10, y11, y3			;; A3 = R3 * cosine/sine			;	28-32
this	ystore	[srcreg+iter*srcinc+32], y13	;; Save I1					; 28
next	vmovapd	y13, [srcreg+(iter+1)*srcinc+32] ;; R5 (I1)

this	vsubpd	y7, y7, y9			;; B4 = B4 - R4					; 29-31
next	vmulpd	y4, y13, y4			;; B1 = I1 * cosine/sine			;	29-33
next	vmovapd	y9, [srcreg+(iter+1)*srcinc+d2+32] ;; R7 (I3)

this	vsubpd	y8, y8, y1			;; A2 = A2 - I2					; 30-32
next	vmulpd	y3, y9, y3			;; B3 = I3 * cosine/sine			;	30-34
next	vmovapd y1, [screg+(iter+1)*scinc+64+32] ;; cosine/sine

this	vaddpd	y2, y2, y0			;; B2 = B2 + R2					; 31-33
next	vmovapd	y0, [srcreg+(iter+1)*srcinc+d1+32] ;; R6 (I2)

;; Shuffle register assignments so that next call has A2,B2,A3,B3,A4,B4 in y0-5 and next R1,R3,I1,I2,I3,A1,A3,B1,B3,c/s2 will be in y6-15.

ytmp	TEXTEQU	y0
y0	TEXTEQU	y8
y8	TEXTEQU	y13
y13	TEXTEQU	y4
y4	TEXTEQU	y14
y14	TEXTEQU	y3
y3	TEXTEQU	y5
y5	TEXTEQU	y7
y7	TEXTEQU	y11
y11	TEXTEQU	y6
y6	TEXTEQU	y15
y15	TEXTEQU	y1
y1	TEXTEQU y2
y2	TEXTEQU	y12
y12	TEXTEQU	y10
y10	TEXTEQU	y9
y9	TEXTEQU	ytmp
	ENDM

;; Haswell FMA3 version

IF (@INSTR(,%yarch,<FMA3>) NE 0)

;; 8 loads, 8 stores, 11 sin/cos loads, 1 const load, 8 muls, 30 FMAs, 15 reg copies = 81 uops = 20.25 clocks best case.
;; Timed at 22.5 clocks
yr4_4c_first_djbfft_unroll MACRO iter,prev,this,next,srcreg,srcinc,d1,d2,screg,scinc,L1pt,L1pd

;; On first call, init registers

	IF iter EQ -1
y0	TEXTEQU	<ymm0>
y1	TEXTEQU	<ymm1>
y2	TEXTEQU	<ymm2>
y3	TEXTEQU	<ymm3>
y4	TEXTEQU	<ymm4>
y5	TEXTEQU	<ymm5>
y6	TEXTEQU	<ymm6>
y7	TEXTEQU	<ymm7>
y8	TEXTEQU	<ymm8>
y9	TEXTEQU	<ymm9>
y10	TEXTEQU	<ymm10>
y11	TEXTEQU	<ymm11>
y12	TEXTEQU	<ymm12>
y13	TEXTEQU	<ymm13>
y14	TEXTEQU	<ymm14>
y15	TEXTEQU	<ymm15>
	ENDIF

;; On later calls, previous R3,I3,R4,I4 are in y0-3.  This R1,I1,R2,I2,R3,I3,R4,I4,sine34,one are in y4-13.
;; The remaining two registers are free.

this	vmulpd	y8, y8, y12			;; R3S = R3 * sine				; 1-5		n 7
this	vmulpd	y9, y9, y12			;; I3S = I3 * sine				; 1-5		n 12
next	vmovapd y14, [screg+(iter+1)*scinc+0+32] ;; cosine/sine for R1/I1

this	yfmsubpd y15, y4, y13, y6		;; R1 - R2 (newer R2)				; 2-6		n 8
this	yfmaddpd y4, y4, y13, y6		;; R1 + R2 (newer and final R1)			; 2-6
next	vmovapd	y6, [srcreg+(iter+1)*srcinc]	;; R1

prev	ystore	[srcreg+(iter-1)*srcinc+d2], y0	;; Save R3					; 23
this	yfmsubpd y0, y5, y13, y7		;; I1 - I2 (newer I2)				; 3-7		n 8
this	yfmaddpd y5, y5, y13, y7		;; I1 + I2 (newer and final I1)			; 3-7

next	vmovapd	y13, [srcreg+(iter+1)*srcinc+32] ;; I1
next	yfmsubpd y7, y6, y14, y13		;; A1 = R1 * cosine/sine - I1			; 4-8		n 9
next	yfmaddpd y13, y13, y14, y6		;; B1 = I1 * cosine/sine + R1			; 4-8		n 9

next	vmovapd y14, [screg+(iter+1)*scinc+64+32] ;; cosine/sine for R2/I2
next	vmovapd	y6, [srcreg+(iter+1)*srcinc+d1]	;; R2
prev	ystore	[srcreg+(iter-1)*srcinc+d2+32], y1 ;; Save I3					; 23+1
next	vmovapd	y1, [srcreg+(iter+1)*srcinc+d1+32] ;; I2
prev	ystore	[srcreg+(iter-1)*srcinc+d2+d1], y2 ;; Save R4					; 24+1
next	yfmsubpd y2, y6, y14, y1		;; A2 = R2 * cosine/sine - I2			; 5-9		n 11
next	yfmaddpd y1, y1, y14, y6		;; B2 = I2 * cosine/sine + R2			; 5-9		n 11

next	vmovapd y14, [screg+(iter+1)*scinc+128+32] ;; cosine/sine for R3/I3
next	vmovapd	y6, [srcreg+(iter+1)*srcinc+d2] ;; R3
prev	ystore	[srcreg+(iter-1)*srcinc+d2+d1+32], y3 ;; Save I4				; 24+2
next	vmovapd	y3, [srcreg+(iter+1)*srcinc+d2+32] ;; I3
this	ystore	[srcreg+iter*srcinc], y4	;; Save R1					; 7+1
next	yfmsubpd y4, y6, y14, y3		;; A3 = R3 * cosine/sine - I3			; 6-10		n 14
next	yfmaddpd y3, y3, y14, y6		;; B3 = I3 * cosine/sine + R3			; 6-10		n 15

this next yloop_unrolled_one
this	yfnmaddpd y14, y11, y12, y8		;; R3S - I4 * sine (newer R3S)			; 7-11 		n 18
this	yfmaddpd y11, y11, y12, y8		;; R3S + I4 * sine (newer R4S)			; 7-11		n 19
this	L1prefetchw srcreg+iter*srcinc+L1pd, L1pt

this	vmovapd	y6, [screg+iter*scinc+256+64+32] ;; cosine/sine for R2/I2
this	yfmsubpd y8, y15, y6, y0		;; A2 = R2 * cosine/sine - I2			; 8-12		n 13
this	yfmaddpd y0, y0, y6, y15		;; B2 = I2 * cosine/sine + R2			; 8-12		n 13

next	vmovapd y6, [screg+(iter+1)*scinc+0]	;; sine for R1/I1
next	vmulpd	y7, y7, y6			;; R1 = A1 * sine				; 9-13		n 14
next	vmulpd	y13, y13, y6			;; I1 = B1 * sine				; 9-13		n 15
this	ystore	[srcreg+iter*srcinc+32], y5	;; Save I1					; 8+1

next	vmovapd	y6, [srcreg+(iter+1)*srcinc+d2+d1]	;; R4
next	vmovapd	y15, [srcreg+(iter+1)*srcinc+d2+d1+32] ;; I4
next	yfmsubpd y5, y6, [screg+(iter+1)*scinc+192+32], y15 ;; A4 = R4 * cosine/sine - I4	; 10-14		n 16
next	yfmaddpd y15, y15, [screg+(iter+1)*scinc+192+32], y6 ;; B4 = I4 * cosine/sine + R4	; 10-14		n 17

next	vmovapd y6, [screg+(iter+1)*scinc+64]	;; sine for R2/I2
next	vmulpd	y2, y2, y6			;; R2 = A2 * sine				; 11-15		n 16
next	vmulpd	y1, y1, y6			;; I2 = B2 * sine				; 11-15		n 17

this	L1prefetchw srcreg+iter*srcinc+d1+L1pd, L1pt
this	yfmaddpd y6, y10, y12, y9		;; I3S + R4 * sine (newer I3S)			; 12-16		n 18
this	yfnmaddpd y10, y10, y12, y9		;; I3S - R4 * sine (newer I4S)			; 12-16		n 19

this	vmovapd	y12, [screg+iter*scinc+256+64]	;; sine for R2/I2
this	vmulpd	y8, y8, y12			;; A2 = A2 * sine (final R2)			; 13-17
this	vmulpd	y0, y0, y12			;; B2 = B2 * sine (final I2)			; 13-17

next	vmovapd y12, [screg+(iter+1)*scinc+128]	;; sine for R3/I3
next	yfnmaddpd y9, y4, y12, y7		;; R1 - (R3 = A3 * sine) (new R3)		; 14-18		n 20
next	yfmaddpd y4, y4, y12, y7		;; R1 + (R3 = A3 * sine) (new R1)		; 14-18		n 21

this	L1prefetchw srcreg+iter*srcinc+d2+L1pd, L1pt
next	yfnmaddpd y7, y3, y12, y13		;; I1 - (I3 = B3 * sine) (new I3)		; 15-19		n 20
next	yfmaddpd y3, y3, y12, y13		;; I1 + (I3 = B3 * sine) (new I1)		; 15-19		n 22

next	vmovapd y12, [screg+(iter+1)*scinc+192]	;; sine for R4/I4
next	yfmaddpd y13, y5, y12, y2		;; R2 + (R4 = A4 * sine) (new R2)		; 16-20		n 21
next	yfnmaddpd y5, y5, y12, y2		;; R2 - (R4 = A4 * sine) (new R4)		; 16-20		n 31

this	L1prefetchw srcreg+iter*srcinc+d2+d1+L1pd, L1pt
next	yfmaddpd y2, y15, y12, y1		;; I2 + (I4 = B4 * sine) (new I2)		; 17-21		n 22
next	yfnmaddpd y15, y15, y12, y1		;; I2 - (I4 = B4 * sine) (new I4)		; 17-21		n 26

this	vmovapd	y12, [screg+iter*scinc+256+0+32];; cosine/sine for R3/I3/R4/I4
this	yfmsubpd y1, y14, y12, y6		;; R3S * cosine/sine - I3S (final R3)		; 18-22
this	yfmaddpd y6, y6, y12, y14		;; I3S * cosine/sine + R3S (final I3)		; 18-22
next	vmovapd	y14, [screg+(iter+1)*scinc+256+0] ;; sine for R3/I3/R4/I4
this	ystore	[srcreg+iter*srcinc+d1], y8	;; Save R2					; 18

this	yfmaddpd y8, y11, y12, y10		;; R4S * cosine/sine + I4S (final R4)		; 19-23
this	yfmsubpd y10, y10, y12, y11		;; I4S * cosine/sine - R4S (final I4)		; 19-23
next	vmovapd y12, YMM_ONE
this	ystore	[srcreg+iter*srcinc+d1+32], y0	;; Save I2					; 18+1

;; Shuffle register assignments so that this R3,I3,R4,I4 are in y0-3 and next R1,I1,R2,I2,R3,I3,R4,I4,sine34,one are in y4-13.

ytmp	TEXTEQU	y0
y0	TEXTEQU	y1
y1	TEXTEQU	y6
y6	TEXTEQU	y13
y13	TEXTEQU	y12
y12	TEXTEQU	y14
y14	TEXTEQU	y11
y11	TEXTEQU	y15
y15	TEXTEQU	ytmp
ytmp	TEXTEQU	y2
y2	TEXTEQU	y8
y8	TEXTEQU	y9
y9	TEXTEQU y7
y7	TEXTEQU	ytmp
ytmp	TEXTEQU	y3
y3	TEXTEQU	y10
y10	TEXTEQU	y5
y5	TEXTEQU	ytmp
	ENDM

ENDIF

ENDIF


;;
;; ************************************* four-complex-last-unfft variants ******************************************
;;

;; This code applies the sin/cos postmultipliers before a radix-4 butterfly.
;; Then it applies the premultipliers since the negacyclic inverse FFT is complete.
;; We save memory by splitting the roots-of-minus-1 premultipliers such that every
;; macro uses the same premultiplier data.  Every sin/cos postmultiplier, a very big
;; table anyway, is multiplied by the other part of the split roots-of-minus-1.
;; The common premultiplier data is 1, .924-.383i, SQRTHALF-SQRTHALFi, .383-.924i.

yr4_s4cl_four_complex_last_unfft_preload MACRO
	ENDM
yr4_s4cl_four_complex_last_unfft MACRO srcreg,srcinc,d1,d2,screg,scinc,maxrpt,L1pt,L1pd

	;; On input the 64-byte cache lines hold these data values:
	;;	0	8	16	24	4	12	20	28
	;;	1	...
	;;	2	...
	;;	3	...

	;; These are swizzled to:
	;;	0	1	2	3	4	5	6	7
	;;	8	...
	;;	16	...
	;;	24	...

	;; Swizzle using 8 shufpds and 8 perm2f128  (1 or 3 clocks each on port 5, throughput 1)

	vmovapd	ymm1, [srcreg]			;; R1
	vmovapd	ymm7, [srcreg+d1]		;; R2
	vshufpd	ymm0, ymm1, ymm7, 15		;; Shuffle R1 and R2 to create 8 9 24 25		; 1
	vshufpd	ymm1, ymm1, ymm7, 0		;; Shuffle R1 and R2 to create 0 1 16 17		; 2

	vmovapd	ymm2, [srcreg+d2]		;; R3
	vmovapd	ymm7, [srcreg+d2+d1]		;; R4
	vshufpd	ymm3, ymm2, ymm7, 15		;; Shuffle R3 and R4 to create 10 11 26 27		; 3
	vshufpd	ymm2, ymm2, ymm7, 0		;; Shuffle R3 and R4 to create 2 3 18 19		; 4

	ylow128s ymm4, ymm0, ymm3		;; Shuffle R1/R2 hi and R3/R4 hi (8 9 10 11)		; 5-6
	yhigh128s ymm0, ymm0, ymm3		;; Shuffle R1/R2 hi and R3/R4 hi (24 25 26 27)		; 6-7

	ylow128s ymm3, ymm1, ymm2		;; Shuffle R1/R2 low and R3/R4 low (0 1 2 3)		; 7-8
	yhigh128s ymm1, ymm1, ymm2		;; Shuffle R1/R2 low and R3/R4 low (16 17 18 19)	; 8-9

	vmovapd	ymm6, [srcreg+32]		;; I1
	vmovapd	ymm2, [srcreg+d1+32]		;; I2
	vshufpd	ymm5, ymm6, ymm2, 15		;; Shuffle I1 and I2 to create 12 13 28 29
	vshufpd	ymm6, ymm6, ymm2, 0		;; Shuffle I1 and I2 to create 4 5 20 21

	ystore	[srcreg], ymm3			;; Temporarily save R1 (0 1 2 3)

	vmovapd	ymm2, [srcreg+d2+32]		;; I3
	vmovapd	ymm7, [srcreg+d2+d1+32]		;; I4
	vshufpd	ymm3, ymm2, ymm7, 15		;; Shuffle I3 and I4 to create 14 15 30 31
	vshufpd	ymm2, ymm2, ymm7, 0		;; Shuffle I3 and I4 to create 6 7 22 23

	ylow128s ymm7, ymm5, ymm3		;; Shuffle I1/I2 hi and I3/I4 hi (12 13 14 15)
	yhigh128s ymm5, ymm5, ymm3		;; Shuffle I1/I2 hi and I3/I4 hi (28 29 30 31)

	ylow128s ymm3, ymm6, ymm2		;; Shuffle I1/I2 low and I3/I4 low (4 5 6 7)
	yhigh128s ymm6, ymm6, ymm2		;; Shuffle I1/I2 low and I3/I4 low (20 21 22 23)

	ystore	[srcreg+32], ymm3		;; Temporarily save I1 (4 5 6 7)

	vmovapd	ymm3, [screg+128+32]		;; cosine/sine
	vmulpd	ymm2, ymm4, ymm3		;; A2 = R2 * cosine/sine
	vaddpd	ymm2, ymm2, ymm7		;; A2 = A2 + I2
	vmulpd	ymm7, ymm7, ymm3		;; B2 = I2 * cosine/sine
	vsubpd	ymm7, ymm7, ymm4		;; B2 = B2 - R2

	vmovapd	ymm3, [screg+192+32]		;; cosine/sine
	vmulpd	ymm4, ymm0, ymm3		;; A4 = R4 * cosine/sine
	vaddpd	ymm4, ymm4, ymm5		;; A4 = A4 + I4
	vmulpd	ymm5, ymm5, ymm3		;; B4 = I4 * cosine/sine
	vsubpd	ymm5, ymm5, ymm0		;; B4 = B4 - R4

	vmovapd	ymm3, [screg+64+32]		;; cosine/sine
	vmulpd	ymm0, ymm1, ymm3		;; A3 = R3 * cosine/sine
	vaddpd	ymm0, ymm0, ymm6		;; A3 = A3 + I3
	vmulpd	ymm6, ymm6, ymm3		;; B3 = I3 * cosine/sine
	vsubpd	ymm6, ymm6, ymm1		;; B3 = B3 - R3

	L1prefetchw srcreg+L1pd, L1pt

	vmovapd	ymm3, [screg+128]		;; sine
	vmulpd	ymm2, ymm2, ymm3		;; A2 = A2 * sine (new R2)
	vmulpd	ymm7, ymm7, ymm3		;; B2 = B2 * sine (new I2)

	vmovapd	ymm3, [screg+192]		;; sine
	vmulpd	ymm4, ymm4, ymm3		;; A4 = A4 * sine (new R4)
	vmulpd	ymm5, ymm5, ymm3		;; B4 = B4 * sine (new I4)

	vmovapd	ymm3, [screg+64]		;; sine
	vmulpd	ymm0, ymm0, ymm3		;; A3 = A3 * sine (new R3)
	vmulpd	ymm6, ymm6, ymm3		;; B3 = B3 * sine (new I3)

	L1prefetchw srcreg+d1+L1pd, L1pt

	vsubpd	ymm1, ymm4, ymm0		;; R4 = R4 - R3 (new I4)
	vaddpd	ymm4, ymm4, ymm0		;; R3 = R4 + R3 (new R3)

	vsubpd	ymm0, ymm6, ymm5		;; I3 = I3 - I4 (new R4)
	vaddpd	ymm6, ymm6, ymm5		;; I4 = I3 + I4 (new I3)

	ystore	[srcreg+d1], ymm0		;; Temporarily save R4
	ystore	[srcreg+d1+32], ymm1		;; Temporarily save I4

	vmovapd	ymm5, [srcreg]			;; Reload R1
	vmovapd	ymm3, [screg+0+32]		;; cosine/sine
	vmulpd	ymm0, ymm5, ymm3		;; A1 = R1 * cosine/sine
	vmovapd	ymm1, [srcreg+32]		;; Reload I1
	vaddpd	ymm0, ymm0, ymm1		;; A1 = A1 + I1
	vmulpd	ymm1, ymm1, ymm3		;; B1 = I1 * cosine/sine
	vsubpd	ymm1, ymm1, ymm5		;; B1 = B1 - R1
	vmovapd	ymm3, [screg+0]			;; Load sine
	vmulpd	ymm0, ymm0, ymm3		;; A1 = A1 * sine (new R1)
	vmulpd	ymm1, ymm1, ymm3		;; B1 = B1 * sine (new I1)

	L1prefetchw srcreg+d2+L1pd, L1pt

	vsubpd	ymm5, ymm0, ymm2		;; R1 - R2 (new R2)
	vaddpd	ymm0, ymm0, ymm2		;; R1 + R2 (new R1)

	vsubpd	ymm2, ymm1, ymm7		;; I1 - I2 (new I2)
	vaddpd	ymm1, ymm1, ymm7		;; I1 + I2 (new I1)

	L1prefetchw srcreg+d2+d1+L1pd, L1pt

	vsubpd	ymm7, ymm0, ymm4		;; R1 - R3 (new R3)
	vaddpd	ymm0, ymm0, ymm4		;; R1 + R3 (final R1)

	vsubpd	ymm4, ymm1, ymm6		;; I1 - I3 (new I3)
	vaddpd	ymm1, ymm1, ymm6		;; I1 + I3 (final I1)

	ystore	[srcreg], ymm0			;; Save R1
	ystore	[srcreg+32], ymm1		;; Save I1

	vmovapd	ymm3, [srcreg+d1]		;; Reload R4
	vsubpd	ymm0, ymm5, ymm3		;; R2 - R4 (new R4)
	vaddpd	ymm5, ymm5, ymm3		;; R2 + R4 (new R2)

	vmovapd	ymm3, [srcreg+d1+32]		;; Reload I4
	vsubpd	ymm1, ymm2, ymm3		;; I2 - I4 (new I4)
	vaddpd	ymm2, ymm2, ymm3		;; I2 + I4 (new I2)

	vmulpd	ymm7, ymm7, YMM_SQRTHALF	;; A3 = R3 * SQRTHALF
	vmulpd	ymm4, ymm4, YMM_SQRTHALF	;; B3 = I3 * SQRTHALF
	vsubpd	ymm6, ymm4, ymm7		;; B3 = B3 - A3 (final I3)
	vaddpd	ymm4, ymm4, ymm7		;; A3 = A3 + B3	(final R3)

	ystore	[srcreg+d2], ymm4		;; Save R3
	ystore	[srcreg+d2+32], ymm6		;; Save I3

	vmovapd	ymm3, YMM_P924
	vmulpd	ymm7, ymm5, ymm3		;; A2 = R2 * .924
	vmovapd	ymm4, YMM_P383
	vmulpd	ymm6, ymm2, ymm4		;; C2 = I2 * .383
	vaddpd	ymm7, ymm7, ymm6		;; A2 = A2 + C2 (final R2)
	vmulpd	ymm5, ymm5, ymm4		;; B2 = R2 * .383
	vmulpd	ymm2, ymm2, ymm3		;; D2 = I2 * .924
	vsubpd	ymm2, ymm2, ymm5		;; D2 = D2 - B2 (final I2)

	vmulpd	ymm6, ymm0, ymm4		;; A4 = R4 * .383
	vmulpd	ymm5, ymm1, ymm3		;; C4 = I4 * .924
	vaddpd	ymm6, ymm6, ymm5		;; A4 = A4 + C4	(final R4)
	vmulpd	ymm0, ymm0, ymm3		;; B4 = R4 * .924
	vmulpd	ymm1, ymm1, ymm4		;; D4 = I4 * .383
	vsubpd	ymm1, ymm1, ymm0		;; D4 = D4 - B4	(final I4)

	ystore	[srcreg+d1], ymm7		;; Save R2
	ystore	[srcreg+d1+32], ymm2		;; Save I2
	ystore	[srcreg+d2+d1], ymm6		;; Save R4
	ystore	[srcreg+d2+d1+32], ymm1		;; Save I4
	bump	srcreg, srcinc
	bump	screg, scinc
	ENDM

;; This code applies the sin/cos multipliers before a radix-4 butterfly.
;; After the butterfly, it applies the negacyclic premultipliers since the inverse FFT is complete.
;; The screg and pmreg data is combined
yr4_4cl_csc_four_complex_last_djbunfft MACRO srcreg,srcinc,d1,d2,screg,scinc,maxrpt,L1pt,L1pd
	vmovapd	ymm0, [screg+256+64+32]		;; cosine/sine
	vmovapd	ymm1, [srcreg+d1]		;; R2
	vmulpd	ymm5, ymm1, ymm0		;; A2 = R2 * cosine/sine
	vmovapd	ymm2, [srcreg+d1+32]		;; I2
	vaddpd	ymm5, ymm5, ymm2		;; A2 = A2 + I2
	vmulpd	ymm2, ymm2, ymm0		;; B2 = I2 * cosine/sine
	vsubpd	ymm2, ymm2, ymm1		;; B2 = B2 - R2

	vmovapd	ymm0, [screg+256+0+32]		;; cosine/sine
	vmovapd	ymm1, [srcreg+d2+d1]		;; R4
	vmulpd	ymm7, ymm1, ymm0		;; A4 = R4 * cosine/sine
	vmovapd	ymm3, [srcreg+d2+d1+32]		;; I4
	vsubpd	ymm7, ymm7, ymm3		;; A4 = A4 - I4
	vmulpd	ymm3, ymm3, ymm0		;; B4 = I4 * cosine/sine
	vaddpd	ymm3, ymm3, ymm1		;; B4 = B4 + R4

	vmovapd	ymm1, [srcreg+d2]		;; R3
	vmulpd	ymm6, ymm1, ymm0		;; A3 = R3 * cosine/sine
	vmovapd	ymm4, [srcreg+d2+32]		;; I3
	vaddpd	ymm6, ymm6, ymm4		;; A3 = A3 + I3
	vmulpd	ymm4, ymm4, ymm0		;; B3 = I3 * cosine/sine
	vsubpd	ymm4, ymm4, ymm1		;; B3 = B3 - R3

	vmulpd	ymm5, ymm5, [screg+256+64]	;; A2 = A2 * sine (new R2)
	vmulpd	ymm2, ymm2, [screg+256+64]	;; B2 = B2 * sine (new I2)

	vmovapd	ymm0, [screg+256+0]		;; sine
	vmulpd	ymm7, ymm7, ymm0		;; A4 = A4 * sine (new R4)
	vmulpd	ymm3, ymm3, ymm0		;; B4 = B4 * sine (new I4)
	vmulpd	ymm6, ymm6, ymm0		;; A3 = A3 * sine (new R3)
	vmulpd	ymm4, ymm4, ymm0		;; B3 = B3 * sine (new I3)

	L1prefetchw srcreg+L1pd, L1pt

	vmovapd	ymm0, [srcreg]			;; R1
	vsubpd	ymm1, ymm0, ymm5		;; R1 - R2 (new R2)
	vaddpd	ymm0, ymm0, ymm5		;; R1 + R2 (new R1)
	vsubpd	ymm5, ymm7, ymm6		;; R4 - R3 (new I4)
	vaddpd	ymm7, ymm7, ymm6		;; R4 + R3 (new R3)

	L1prefetchw srcreg+d1+L1pd, L1pt

	vsubpd	ymm6, ymm4, ymm3		;; I3 - I4 (new R4)
	vaddpd	ymm4, ymm4, ymm3		;; I3 + I4 (new I3)

	vsubpd	ymm3, ymm1, ymm6		;; R2 - R4 (newer R4)
	vaddpd	ymm1, ymm1, ymm6		;; R2 + R4 (newer R2)

	L1prefetchw srcreg+d2+L1pd, L1pt

	ystore	[srcreg], ymm3			;; Temporarily save R4

	vmovapd	ymm6, [srcreg+32]		;; I1
	vsubpd	ymm3, ymm6, ymm2		;; I1 - I2 (new I2)
	vaddpd	ymm6, ymm6, ymm2		;; I1 + I2 (new I1)

	vsubpd	ymm2, ymm0, ymm7		;; R1 - R3 (newer R3)
	vaddpd	ymm0, ymm0, ymm7		;; R1 + R3 (newer R1)

	L1prefetchw srcreg+d2+d1+L1pd, L1pt

	vaddpd	ymm7, ymm3, ymm5		;; I2 + I4 (newer I2)
	vsubpd	ymm3, ymm3, ymm5		;; I2 - I4 (newer I4)

	vsubpd	ymm5, ymm6, ymm4		;; I1 - I3 (newer I3)
	vaddpd	ymm6, ymm6, ymm4		;; I1 + I3 (newer I1)

	ystore	[srcreg+32], ymm3		;; Temporarily save I4

	vmovapd ymm3, [screg+0+32]		;; cosine/sine
	vmulpd	ymm4, ymm0, ymm3		;; A1 = R1 * cosine/sine
	vaddpd	ymm4, ymm4, ymm6		;; A1 = A1 + I1
	vmulpd	ymm6, ymm6, ymm3		;; B1 = I1 * cosine/sine
	vsubpd	ymm6, ymm6, ymm0		;; B1 = B1 - R1

	vmovapd ymm3, [screg+128+32]		;; cosine/sine
	vmulpd	ymm0, ymm2, ymm3		;; A3 = R3 * cosine/sine
	vaddpd	ymm0, ymm0, ymm5		;; A3 = A3 + I3
	vmulpd	ymm5, ymm5, ymm3		;; B3 = I3 * cosine/sine
	vsubpd	ymm5, ymm5, ymm2		;; B3 = B3 - R3

	vmovapd ymm3, [screg+64+32]		;; cosine/sine
	vmulpd	ymm2, ymm1, ymm3		;; A2 = R2 * cosine/sine
	vaddpd	ymm2, ymm2, ymm7		;; A2 = A2 + I2
	vmulpd	ymm7, ymm7, ymm3		;; B2 = I2 * cosine/sine
	vsubpd	ymm7, ymm7, ymm1		;; B2 = B2 - R2

	vmovapd ymm1, [screg+0]			;; sine
	vmulpd	ymm4, ymm4, ymm1		;; A1 = A1 * sine
	vmulpd	ymm6, ymm6, ymm1		;; B1 = B1 * sine

	vmovapd ymm3, [screg+192+32]		;; cosine/sine
	vmulpd	ymm1, ymm3, [srcreg]		;; A4 = R4 * cosine/sine
	vmulpd	ymm3, ymm3, [srcreg+32]		;; B4 = I4 * cosine/sine
	vaddpd	ymm1, ymm1, [srcreg+32]		;; A4 = A4 + I4
	vsubpd	ymm3, ymm3, [srcreg]		;; B4 = B4 - R4

	ystore	[srcreg], ymm4			;; Save R1
	ystore	[srcreg+32], ymm6		;; Save I1

	vmovapd ymm4, [screg+128]		;; sine
	vmulpd	ymm0, ymm0, ymm4		;; A3 = A3 * sine
	vmulpd	ymm5, ymm5, ymm4		;; B3 = B3 * sine
	vmovapd ymm4, [screg+64]		;; sine
	vmulpd	ymm2, ymm2, ymm4		;; A2 = A2 * sine
	vmulpd	ymm7, ymm7, ymm4		;; B2 = B2 * sine
	vmovapd ymm4, [screg+192]		;; sine
	vmulpd	ymm1, ymm1, ymm4		;; A4 = A4 * sine
	vmulpd	ymm3, ymm3, ymm4		;; B4 = B4 * sine

	ystore	[srcreg+d1], ymm2		;; Save R2
	ystore	[srcreg+d1+32], ymm7		;; Save I2
	ystore	[srcreg+d2], ymm0		;; Save R3
	ystore	[srcreg+d2+32], ymm5		;; Save I3
	ystore	[srcreg+d2+d1], ymm1		;; Save R4
	ystore	[srcreg+d2+d1+32], ymm3		;; Save I4
	bump	srcreg, srcinc
	bump	screg, scinc
	ENDM

;; 64-bit version

IFDEF X86_64

yr4_s4cl_four_complex_last_unfft_preload MACRO
	vbroadcastsd ymm15, YMM_P924
	vbroadcastsd ymm14, YMM_P383
	vbroadcastsd ymm13, YMM_SQRTHALF
	ENDM
yr4_s4cl_four_complex_last_unfft MACRO srcreg,srcinc,d1,d2,screg,scinc,maxrpt,L1pt,L1pd

	;; Swizzle using 8 shufpds and 8 perm2f128  (1 or 3 clocks each on port 5, throughput 1)

	vmovapd	ymm1, [srcreg]			;; R1
	vmovapd	ymm7, [srcreg+d1]		;; R2
	vshufpd	ymm0, ymm1, ymm7, 15		;; Shuffle R1 and R2 to create 8 9 24 25		; 1
	vshufpd	ymm1, ymm1, ymm7, 0		;; Shuffle R1 and R2 to create 0 1 16 17		; 2

	vmovapd	ymm2, [srcreg+d2]		;; R3
	vmovapd	ymm7, [srcreg+d2+d1]		;; R4
	vshufpd	ymm3, ymm2, ymm7, 15		;; Shuffle R3 and R4 to create 10 11 26 27		; 3
	vshufpd	ymm2, ymm2, ymm7, 0		;; Shuffle R3 and R4 to create 2 3 18 19		; 4

	ylow128s ymm4, ymm0, ymm3		;; Shuffle R1/R2 hi and R3/R4 hi (8 9 10 11)		; 5-7
	yhigh128s ymm0, ymm0, ymm3		;; Shuffle R1/R2 hi and R3/R4 hi (24 25 26 27)		; 6-8

	ylow128s ymm3, ymm1, ymm2		;; Shuffle R1/R2 low and R3/R4 low (0 1 2 3)		; 7-9
	yhigh128s ymm1, ymm1, ymm2		;; Shuffle R1/R2 low and R3/R4 low (16 17 18 19)	; 8-10

	vmovapd	ymm6, [srcreg+32]		;; I1
	vmovapd	ymm2, [srcreg+d1+32]		;; I2
	vshufpd	ymm5, ymm6, ymm2, 15		;; Shuffle I1 and I2 to create 12 13 28 29		; 9
	vshufpd	ymm6, ymm6, ymm2, 0		;; Shuffle I1 and I2 to create 4 5 20 21		; 10

	vmovapd	ymm2, [srcreg+d2+32]		;; I3
	vmovapd	ymm7, [srcreg+d2+d1+32]		;; I4
	vshufpd	ymm8, ymm2, ymm7, 15		;; Shuffle I3 and I4 to create 14 15 30 31		; 11
	vshufpd	ymm2, ymm2, ymm7, 0		;; Shuffle I3 and I4 to create 6 7 22 23		; 12

	ylow128s ymm7, ymm5, ymm8		;; Shuffle I1/I2 hi and I3/I4 hi (12 13 14 15)		; 13-15
	yhigh128s ymm5, ymm5, ymm8		;; Shuffle I1/I2 hi and I3/I4 hi (28 29 30 31)		; 14-16

	ylow128s ymm8, ymm6, ymm2		;; Shuffle I1/I2 low and I3/I4 low (4 5 6 7)		; 15-17
	yhigh128s ymm6, ymm6, ymm2		;; Shuffle I1/I2 low and I3/I4 low (20 21 22 23)	; 16-18

	vmovapd	ymm12, [screg+128+32]		;; cosine/sine
	vmulpd	ymm2, ymm4, ymm12		;; A2 = R2 * cosine/sine				; 8-12
	vaddpd	ymm2, ymm2, ymm7		;; A2 = A2 + I2
	vmulpd	ymm7, ymm7, ymm12		;; B2 = I2 * cosine/sine				; 16-20
	vsubpd	ymm7, ymm7, ymm4		;; B2 = B2 - R2

	vmovapd	ymm12, [screg+192+32]		;; cosine/sine
	vmulpd	ymm4, ymm0, ymm12		;; A4 = R4 * cosine/sine				; 9-13
	vaddpd	ymm4, ymm4, ymm5		;; A4 = A4 + I4
	vmulpd	ymm5, ymm5, ymm12		;; B4 = I4 * cosine/sine				; 17-21
	vsubpd	ymm5, ymm5, ymm0		;; B4 = B4 - R4

	vmovapd	ymm12, [screg+64+32]		;; cosine/sine
	vmulpd	ymm0, ymm1, ymm12		;; A3 = R3 * cosine/sine
	vaddpd	ymm0, ymm0, ymm6		;; A3 = A3 + I3
	vmulpd	ymm6, ymm6, ymm12		;; B3 = I3 * cosine/sine
	vsubpd	ymm6, ymm6, ymm1		;; B3 = B3 - R3

	L1prefetchw srcreg+L1pd, L1pt

	vmovapd	ymm12, [screg+128]		;; sine
	vmulpd	ymm2, ymm2, ymm12		;; A2 = A2 * sine (new R2)
	vmulpd	ymm7, ymm7, ymm12		;; B2 = B2 * sine (new I2)

	vmovapd	ymm12, [screg+192]		;; sine
	vmulpd	ymm4, ymm4, ymm12		;; A4 = A4 * sine (new R4)
	vmulpd	ymm5, ymm5, ymm12		;; B4 = B4 * sine (new I4)

	vmovapd	ymm12, [screg+64]		;; sine
	vmulpd	ymm0, ymm0, ymm12		;; A3 = A3 * sine (new R3)
	vmulpd	ymm6, ymm6, ymm12		;; B3 = B3 * sine (new I3)

	L1prefetchw srcreg+d1+L1pd, L1pt

	vsubpd	ymm1, ymm4, ymm0		;; R4 = R4 - R3 (new I4)
	vaddpd	ymm4, ymm4, ymm0		;; R3 = R4 + R3 (new R3)

	vsubpd	ymm0, ymm6, ymm5		;; I3 = I3 - I4 (new R4)
	vaddpd	ymm6, ymm6, ymm5		;; I4 = I3 + I4 (new I3)

	vmovapd	ymm12, [screg+0+32]		;; cosine/sine
	vmulpd	ymm5, ymm3, ymm12		;; A1 = R1 * cosine/sine
	vaddpd	ymm5, ymm5, ymm8		;; A1 = A1 + I1
	vmulpd	ymm8, ymm8, ymm12		;; B1 = I1 * cosine/sine
	vsubpd	ymm8, ymm8, ymm3		;; B1 = B1 - R1
	vmovapd	ymm12, [screg+0]		;; Load sine
	vmulpd	ymm5, ymm5, ymm12		;; A1 = A1 * sine (new R1)
	vmulpd	ymm8, ymm8, ymm12		;; B1 = B1 * sine (new I1)
	bump	screg, scinc

	L1prefetchw srcreg+d2+L1pd, L1pt

	vsubpd	ymm3, ymm5, ymm2		;; R1 - R2 (new R2)
	vaddpd	ymm5, ymm5, ymm2		;; R1 + R2 (new R1)

	vsubpd	ymm2, ymm8, ymm7		;; I1 - I2 (new I2)
	vaddpd	ymm8, ymm8, ymm7		;; I1 + I2 (new I1)

	L1prefetchw srcreg+d2+d1+L1pd, L1pt

	vsubpd	ymm7, ymm5, ymm4		;; R1 - R3 (new R3)
	vaddpd	ymm5, ymm5, ymm4		;; R1 + R3 (final R1)

	vsubpd	ymm4, ymm8, ymm6		;; I1 - I3 (new I3)
	vaddpd	ymm8, ymm8, ymm6		;; I1 + I3 (final I1)

	ystore	[srcreg], ymm5			;; Save R1
	ystore	[srcreg+32], ymm8		;; Save I1

	vsubpd	ymm5, ymm3, ymm0		;; R2 - R4 (new R4)
	vaddpd	ymm3, ymm3, ymm0		;; R2 + R4 (new R2)

	vsubpd	ymm8, ymm2, ymm1		;; I2 - I4 (new I4)
	vaddpd	ymm2, ymm2, ymm1		;; I2 + I4 (new I2)

	vmulpd	ymm7, ymm7, ymm13		;; A3 = R3 * SQRTHALF
	vmulpd	ymm4, ymm4, ymm13		;; B3 = I3 * SQRTHALF
	vsubpd	ymm6, ymm4, ymm7		;; B3 = B3 - A3 (final I3)
	vaddpd	ymm4, ymm4, ymm7		;; A3 = A3 + B3	(final R3)

	ystore	[srcreg+d2], ymm4		;; Save R3
	ystore	[srcreg+d2+32], ymm6		;; Save I3

	vmulpd	ymm7, ymm3, ymm15		;; A2 = R2 * .924
	vmulpd	ymm6, ymm2, ymm14		;; C2 = I2 * .383
	vaddpd	ymm7, ymm7, ymm6		;; A2 = A2 + C2 (final R2)
	vmulpd	ymm3, ymm3, ymm14		;; B2 = R2 * .383
	vmulpd	ymm2, ymm2, ymm15		;; D2 = I2 * .924
	vsubpd	ymm2, ymm2, ymm3		;; D2 = D2 - B2 (final I2)

	vmulpd	ymm6, ymm5, ymm14		;; A4 = R4 * .383
	vmulpd	ymm3, ymm8, ymm15		;; C4 = I4 * .924
	vaddpd	ymm6, ymm6, ymm3		;; A4 = A4 + C4	(final R4)
	vmulpd	ymm5, ymm5, ymm15		;; B4 = R4 * .924
	vmulpd	ymm8, ymm8, ymm14		;; D4 = I4 * .383
	vsubpd	ymm8, ymm8, ymm5		;; D4 = D4 - B4	(final I4)

	ystore	[srcreg+d1], ymm7		;; Save R2
	ystore	[srcreg+d1+32], ymm2		;; Save I2
	ystore	[srcreg+d2+d1], ymm6		;; Save R4
	ystore	[srcreg+d2+d1+32], ymm8		;; Save I4
	bump	srcreg, srcinc
	ENDM

yr4_4cl_csc_four_complex_last_djbunfft MACRO srcreg,srcinc,d1,d2,screg,scinc,maxrpt,L1pt,L1pd
	IF maxrpt MOD 5 EQ 0
	yr4_4c_last_djbunfft_unroll -1,noexec,noexec,exec,srcreg,srcinc,d1,d2,screg,scinc,L1pt,L1pd
	yr4_4c_last_djbunfft_unroll 0,noexec,exec,exec,srcreg,srcinc,d1,d2,screg,scinc,L1pt,L1pd
	yr4_4c_last_djbunfft_unroll 1,exec,exec,exec,srcreg,srcinc,d1,d2,screg,scinc,L1pt,L1pd
	yr4_4c_last_djbunfft_unroll 2,exec,exec,exec,srcreg,srcinc,d1,d2,screg,scinc,L1pt,L1pd
	yr4_4c_last_djbunfft_unroll 3,exec,exec,exec,srcreg,srcinc,d1,d2,screg,scinc,L1pt,L1pd
	yr4_4c_last_djbunfft_unroll 4,exec,exec,noexec,srcreg,srcinc,d1,d2,screg,scinc,L1pt,L1pd
	yr4_4c_last_djbunfft_unroll 5,exec,noexec,noexec,srcreg,srcinc,d1,d2,screg,scinc,L1pt,L1pd
	bump	srcreg, 5*srcinc
	bump	screg, 5*scinc
	ELSEIF maxrpt MOD 4 EQ 0
	yr4_4c_last_djbunfft_unroll -1,noexec,noexec,exec,srcreg,srcinc,d1,d2,screg,scinc,L1pt,L1pd
	yr4_4c_last_djbunfft_unroll 0,noexec,exec,exec,srcreg,srcinc,d1,d2,screg,scinc,L1pt,L1pd
	yr4_4c_last_djbunfft_unroll 1,exec,exec,exec,srcreg,srcinc,d1,d2,screg,scinc,L1pt,L1pd
	yr4_4c_last_djbunfft_unroll 2,exec,exec,exec,srcreg,srcinc,d1,d2,screg,scinc,L1pt,L1pd
	yr4_4c_last_djbunfft_unroll 3,exec,exec,noexec,srcreg,srcinc,d1,d2,screg,scinc,L1pt,L1pd
	yr4_4c_last_djbunfft_unroll 4,exec,noexec,noexec,srcreg,srcinc,d1,d2,screg,scinc,L1pt,L1pd
	bump	srcreg, 4*srcinc
	bump	screg, 4*scinc
	ELSEIF maxrpt MOD 3 EQ 0
	yr4_4c_last_djbunfft_unroll -1,noexec,noexec,exec,srcreg,srcinc,d1,d2,screg,scinc,L1pt,L1pd
	yr4_4c_last_djbunfft_unroll 0,noexec,exec,exec,srcreg,srcinc,d1,d2,screg,scinc,L1pt,L1pd
	yr4_4c_last_djbunfft_unroll 1,exec,exec,exec,srcreg,srcinc,d1,d2,screg,scinc,L1pt,L1pd
	yr4_4c_last_djbunfft_unroll 2,exec,exec,noexec,srcreg,srcinc,d1,d2,screg,scinc,L1pt,L1pd
	yr4_4c_last_djbunfft_unroll 3,exec,noexec,noexec,srcreg,srcinc,d1,d2,screg,scinc,L1pt,L1pd
	bump	srcreg, 3*srcinc
	bump	screg, 3*scinc
	ELSEIF maxrpt MOD 2 EQ 0
	yr4_4c_last_djbunfft_unroll -1,noexec,noexec,exec,srcreg,srcinc,d1,d2,screg,scinc,L1pt,L1pd
	yr4_4c_last_djbunfft_unroll 0,noexec,exec,exec,srcreg,srcinc,d1,d2,screg,scinc,L1pt,L1pd
	yr4_4c_last_djbunfft_unroll 1,exec,exec,noexec,srcreg,srcinc,d1,d2,screg,scinc,L1pt,L1pd
	yr4_4c_last_djbunfft_unroll 2,exec,noexec,noexec,srcreg,srcinc,d1,d2,screg,scinc,L1pt,L1pd
	bump	srcreg, 2*srcinc
	bump	screg, 2*scinc
	ELSE
	yr4_4c_last_djbunfft_unroll -1,noexec,noexec,exec,srcreg,srcinc,d1,d2,screg,scinc,L1pt,L1pd
	yr4_4c_last_djbunfft_unroll 0,noexec,exec,noexec,srcreg,srcinc,d1,d2,screg,scinc,L1pt,L1pd
	yr4_4c_last_djbunfft_unroll 1,exec,noexec,noexec,srcreg,srcinc,d1,d2,screg,scinc,L1pt,L1pd
	bump	srcreg, srcinc
	bump	screg, scinc
	ENDIF
	ENDM

yr4_4c_last_djbunfft_unroll MACRO iter,prev,this,next,srcreg,srcinc,d1,d2,screg,scinc,L1pt,L1pd

;; On first call, init registers

	IF iter EQ -1
y0	TEXTEQU	<ymm0>
y1	TEXTEQU	<ymm1>
y2	TEXTEQU	<ymm2>
y3	TEXTEQU	<ymm3>
y4	TEXTEQU	<ymm4>
y5	TEXTEQU	<ymm5>
y6	TEXTEQU	<ymm6>
y7	TEXTEQU	<ymm7>
y8	TEXTEQU	<ymm8>
y9	TEXTEQU	<ymm9>
y10	TEXTEQU	<ymm10>
y11	TEXTEQU	<ymm11>
y12	TEXTEQU	<ymm12>
y13	TEXTEQU	<ymm13>
y14	TEXTEQU	<ymm14>
y15	TEXTEQU	<ymm15>
	ENDIF

;; On later calls, previous A2,B2,A3,B3,A1,B1 will be in y0-5.  This R3,I3,R4,I4,A2,B2,A3,B3,A4,B4 will be in y6-15.

this	vsubpd	y14, y14, y9		;; A4 = A4 - I4 (R4/sine)		; 1-3
prev	vmovapd y9, [screg+(iter-1)*scinc+64] ;; sine
prev	vmulpd	y1, y1, y9		;; B2 = B2 * sine			;	0-4
prev	vmulpd	y0, y0, y9		;; A2 = A2 * sine			;	1-5

this	vaddpd	y12, y12, y7		;; A3 = A3 + I3 (R3/sine)		; 2-4
this	vmovapd	y9, [screg+iter*scinc+256+64] ;; sine
this	vmulpd	y11, y11, y9		;; B2 = B2 * sine (new I2)		;	2-6

this	vaddpd	y15, y15, y8		;; B4 = B4 + R4 (I4/sine)		; 3-5
this	vmulpd	y10, y10, y9		;; A2 = A2 * sine (new R2)		;	3-7
prev	vmovapd y7, [screg+(iter-1)*scinc+128] ;; sine

this	vsubpd	y13, y13, y6		;; B3 = B3 - R3 (I3/sine)		; 4-6
prev	vmulpd	y2, y2, y7		;; A3 = A3 * sine			;	4-8
prev	vmovapd y8, [screg+(iter-1)*scinc+0] ;; sine

this	vsubpd	y6, y14, y12		;; R4/sine - R3/sine (new I4/sine)	; 5-7
prev	vmulpd	y3, y3, y7		;; B3 = B3 * sine			;	5-9
this	vmovapd	y9, [screg+iter*scinc+256+0] ;; sine
prev	ystore	[srcreg+(iter-1)*srcinc+d1+32], y1 ;; Save I2			; 5

this	vaddpd	y14, y14, y12		;; R4/sine + R3/sine (new R3/sine)	; 6-8
prev	vmulpd	y4, y4, y8		;; A1 = A1 * sine			;	6-10
this	vmovapd	y7, [srcreg+iter*srcinc+32] ;; I1
prev	ystore	[srcreg+(iter-1)*srcinc+d1], y0 ;; Save R2			; 6

this	vsubpd	y0, y13, y15		;; I3/sine - I4/sine (new R4/sine)	; 7-9
prev	vmulpd	y5, y5, y8		;; B1 = B1 * sine			;	7-11
this	vmovapd	y1, [srcreg+iter*srcinc] ;; R1

this	vaddpd	y13, y13, y15		;; I3/sine + I4/sine (new I3/sine)	; 8-10
this	vmulpd	y6, y6, y9		;; new I4/sine * sine			;	8-12
next	vmovapd	y12, [screg+(iter+1)*scinc+256+64+32] ;; cosine/sine

this	vsubpd	y15, y7, y11		;; I1 - I2 (new I2)			; 9-11
this	vmulpd	y14, y14, y9		;; new R3/sine * sine			;	9-13
next	vmovapd	y8, [srcreg+(iter+1)*srcinc+d1+32] ;; I2
prev	ystore	[srcreg+(iter-1)*srcinc+d2], y2 ;; Save R3			; 9

this	vaddpd	y7, y7, y11		;; I1 + I2 (new I1)			; 10-12
this	vmulpd	y0, y0, y9		;; new R4/sine * sine			;	10-14
next	vmovapd	y2, [srcreg+(iter+1)*srcinc+d1] ;; R2
prev	ystore	[srcreg+(iter-1)*srcinc+d2+32], y3 ;; Save I3			; 10

this	vsubpd	y3, y1, y10		;; R1 - R2 (new R2)			; 11-13
this	vmulpd	y13, y13, y9		;; new I3/sine * sine			;	11-15
this	vmovapd y11, [screg+iter*scinc+192+32]	;; cosine/sine
prev	ystore	[srcreg+(iter-1)*srcinc], y4 ;; Save R1				; 11

this	vaddpd	y1, y1, y10		;; R1 + R2 (new R1)			; 12-14
prev	ystore	[srcreg+(iter-1)*srcinc+32], y5 ;; Save I1			; 12

this	vsubpd	y5, y15, y6		;; I2 - I4 (newer I4)			; 13-15
this next yloop_unrolled_one

this	vaddpd	y15, y15, y6		;; I2 + I4 (newer I2)			; 14-16
next	vmulpd	y6, y8, y12		;; B2 = I2 * cosine/sine		;	14-18
this	L1prefetchw srcreg+iter*srcinc+L1pd, L1pt

this	vsubpd	y10, y3, y0		;; R2 - R4 (newer R4)			; 15-17
next	vmulpd	y12, y2, y12		;; A2 = R2 * cosine/sine		;	15-19

this	vaddpd	y3, y3, y0		;; R2 + R4 (newer R2)			; 16-18
this	vmulpd	y0, y5, y11		;; B4 = I4 * cosine/sine		;	16-20
this	L1prefetchw srcreg+iter*srcinc+d1+L1pd, L1pt

this	vsubpd	y4, y1, y14		;; R1 - R3 (newer R3)			; 17-19

this	vsubpd	y9, y7, y13		;; I1 - I3 (newer I3)			; 18-20
this	L1prefetchw srcreg+iter*srcinc+d2+L1pd, L1pt

this	vaddpd	y1, y1, y14		;; R1 + R3 (newer R1)			; 19-21
this	vmovapd y14, [screg+iter*scinc+64+32] ;; cosine/sine

this	vaddpd	y7, y7, y13		;; I1 + I3 (newer I1)			; 20-22
this	vmulpd	y13, y15, y14		;; B2 = I2 * cosine/sine		;	17-21
this	vmulpd	y11, y10, y11		;; A4 = R4 * cosine/sine		;	18-22
this	vmulpd	y14, y3, y14		;; A2 = R2 * cosine/sine		;	19-23

this	vsubpd	y0, y0, y10		;; B4 = B4 - R4				; 21-23
this	vmovapd y10, [screg+iter*scinc+128+32] ;; cosine/sine

this	vsubpd	y13, y13, y3		;; B2 = B2 - R2				; 22-24
this	vmulpd	y3, y4, y10		;; A3 = R3 * cosine/sine		;	20-24
this	vmulpd	y10, y9, y10		;; B3 = I3 * cosine/sine		;	21-25
this	L1prefetchw srcreg+iter*srcinc+d2+d1+L1pd, L1pt

this	vaddpd	y11, y11, y5		;; A4 = A4 + I4				; 23-25
this	vmovapd y5, [screg+iter*scinc+0+32] ;; cosine/sine

this	vaddpd	y14, y14, y15		;; A2 = A2 + I2				; 24-26
this	vmulpd	y15, y1, y5		;; A1 = R1 * cosine/sine		;	22-26
this	vmulpd	y5, y7, y5		;; B1 = I1 * cosine/sine		;	23-27

next	vsubpd	y6, y6, y2		;; B2 = B2 - R2				; 25-27
this	vmovapd y2, [screg+iter*scinc+192] ;; sine
this	vmulpd	y0, y0, y2		;; B4 = B4 * sine			;	24-28
this	ystore	[srcreg+iter*srcinc+d2+d1+32], y0 ;; Save I4			; 29
next	vmovapd	y0, [screg+(iter+1)*scinc+256+0+32] ;; cosine/sine

next	vaddpd	y12, y12, y8		;; A2 = A2 + I2				; 26-28
next	vmovapd	y8, [srcreg+(iter+1)*srcinc+d2+d1] ;; R4

this	vaddpd	y3, y3, y9		;; A3 = A3 + I3				; 27-29
next	vmulpd	y9, y8, y0		;; A4 = R4 * cosine/sine		;	25-29
this	vmulpd	y11, y11, y2		;; A4 = A4 * sine			;	26-30
next	vmovapd	y2, [srcreg+(iter+1)*srcinc+d2] ;; R3
this	ystore	[srcreg+iter*srcinc+d2+d1], y11 ;; Save R4			; 31
next	vmulpd	y11, y2, y0		;; A3 = R3 * cosine/sine		;	27-31

this	vsubpd	y10, y10, y4		;; B3 = B3 - R3				; 28-30
next	vmovapd	y4, [srcreg+(iter+1)*srcinc+d2+d1+32] ;; I4

this	vaddpd	y15, y15, y7		;; A1 = A1 + I1				; 29-31
next	vmulpd	y7, y4, y0		;; B4 = I4 * cosine/sine		;	28-32

this	vsubpd	y5, y5, y1		;; B1 = B1 - R1				; 30-32
next	vmovapd	y1, [srcreg+(iter+1)*srcinc+d2+32] ;; I3
next	vmulpd	y0, y1, y0		;; B3 = I3 * cosine/sine		;	29-33

;; Shuffle register assignments so that next call has A2,B2,A3,B3,A1,B1 in y0-5 and next R3,I3,R4,I4,A2,B2,A3,B3,A4,B4 will be in y6-15.

ytmp	TEXTEQU	y0
y0	TEXTEQU	y14
y14	TEXTEQU	y9
y9	TEXTEQU	y4
y4	TEXTEQU	y15
y15	TEXTEQU	y7
y7	TEXTEQU	y1
y1	TEXTEQU	y13
y13	TEXTEQU	ytmp
ytmp	TEXTEQU	y2
y2	TEXTEQU	y3
y3	TEXTEQU	y10
y10	TEXTEQU	y12
y12	TEXTEQU y11
y11	TEXTEQU	y6
y6	TEXTEQU	ytmp
	ENDM

;; Haswell FMA3 version

IF (@INSTR(,%yarch,<FMA3>) NE 0)

;; 8 loads, 8 stores, 12 sin/cos loads, 1 const load, 8 muls, 30 FMAs, 15 reg copies = 82 uops = 20.5 clocks best case.
;; Timed at 23.2 clocks.
yr4_4c_last_djbunfft_unroll MACRO iter,prev,this,next,srcreg,srcinc,d1,d2,screg,scinc,L1pt,L1pd

;; On first call, init registers

	IF iter EQ -1
y0	TEXTEQU	<ymm0>
y1	TEXTEQU	<ymm1>
y2	TEXTEQU	<ymm2>
y3	TEXTEQU	<ymm3>
y4	TEXTEQU	<ymm4>
y5	TEXTEQU	<ymm5>
y6	TEXTEQU	<ymm6>
y7	TEXTEQU	<ymm7>
y8	TEXTEQU	<ymm8>
y9	TEXTEQU	<ymm9>
y10	TEXTEQU	<ymm10>
y11	TEXTEQU	<ymm11>
y12	TEXTEQU	<ymm12>
y13	TEXTEQU	<ymm13>
y14	TEXTEQU	<ymm14>
y15	TEXTEQU	<ymm15>
	ENDIF

;; On later calls, previous R1,I1,R2,I2,R3,I3 will be in y0-5.  This R1,I1,I2,R3/sine,I3/sine,R4/sine,I4/sine,A2,sine2 are in y6-14.
;; y15 is free.

next	vmovapd	y15, [screg+(iter+1)*scinc+256+0+32]	;; cosine/sine for R3/I3/R4/I4
prev	ystore	[srcreg+(iter-1)*srcinc+d1+32], y3	;; Save I2				; 22+1
this	yfnmaddpd y3, y13, y14, y6			;; R1 - (R2 = A2 * sine) (new R2)	; 1-5		n 6
this	yfmaddpd y13, y13, y14, y6			;; R1 + (R2 = A2 * sine) (new R1)	; 1-5		n 7

next	vmovapd	y14, [srcreg+(iter+1)*srcinc+d2+d1]	;; R4
next	vmovapd	y6, [srcreg+(iter+1)*srcinc+d2+d1+32]	;; I4
prev	ystore	[srcreg+(iter-1)*srcinc+d1], y2		;; Save R2				; 22+2
next	yfmsubpd y2, y14, y15, y6			;; R4 * cosine/sine - I4 (R4/sine)	; 2-6		n 9
next	yfmaddpd y6, y6, y15, y14			;; I4 * cosine/sine + R4 (I4/sine)	; 2-6		n 10

next	vmovapd	y14, [srcreg+(iter+1)*srcinc+d2]	;; R3
prev	ystore	[srcreg+(iter-1)*srcinc+d2], y4		;; Save R3				; 23+2
next	vmovapd	y4, [srcreg+(iter+1)*srcinc+d2+32]	;; I3
prev	ystore	[srcreg+(iter-1)*srcinc+d2+32], y5	;; Save I3				; 23+3
next	yfmaddpd y5, y14, y15, y4			;; R3 * cosine/sine + I3 (R3/sine)	; 3-7		n 9
next	yfmsubpd y4, y4, y15, y14			;; I3 * cosine/sine - R3 (I3/sine)	; 3-7		n 10

next	vmovapd	y15, [screg+(iter+1)*scinc+256+64+32]	;; cosine/sine for R2/I2
next	vmovapd	y14, [srcreg+(iter+1)*srcinc+d1]	;; R2
prev	ystore	[srcreg+(iter-1)*srcinc], y0		;; Save R1				; 24+3
next	vmovapd	y0, [srcreg+(iter+1)*srcinc+d1+32]	;; I2
prev	ystore	[srcreg+(iter-1)*srcinc+32], y1		;; Save I1				; 24+4
next	yfmaddpd y1, y14, y15, y0			;; A2 = R2 * cosine/sine + I2		; 4-8		n 20
next	yfmsubpd y0, y0, y15, y14			;; B2 = I2 * cosine/sine - R2		; 4-8		n 15

this	vmovapd	y15, [screg+iter*scinc+256+0]		;; sine for R3/I3/R4/I4
this	yfnmaddpd y14, y12, y15, y8			;; I2 - I4/sine * sine (newer I4)	; 5-9		n 11
this	yfmaddpd y12, y12, y15, y8			;; I2 + I4/sine * sine (newer I2)	; 5-9		n 12
this next yloop_unrolled_one

this	yfnmaddpd y8, y11, y15, y3			;; R2 - R4/sine * sine (newer R4)	; 6-10		n 11
this	yfmaddpd y11, y11, y15, y3			;; R2 + R4/sine * sine (newer R2)	; 6-10		n 12
this	L1prefetchw srcreg+iter*srcinc+L1pd, L1pt

this	yfnmaddpd y3, y9, y15, y13			;; R1 - R3/sine * sine (newer R3)	; 7-11		n 13
this	yfmaddpd y9, y9, y15, y13			;; R1 + R3/sine * sine (newer R1)	; 7-11		n 14
this	L1prefetchw srcreg+iter*srcinc+d1+L1pd, L1pt

this	yfnmaddpd y13, y10, y15, y7			;; I1 - I3/sine * sine (newer I3)	; 8-12		n 13
this	yfmaddpd y10, y10, y15, y7			;; I1 + I3/sine * sine (newer I1)	; 8-12		n 14

next	vmovapd y15, YMM_ONE
next	yfmsubpd y7, y2, y15, y5			;; R4/sine - R3/sine (new I4/sine)	; 9-13		n 24
next	yfmaddpd y2, y2, y15, y5			;; R4/sine + R3/sine (new R3/sine)	; 9-13		n 26
this	L1prefetchw srcreg+iter*srcinc+d2+L1pd, L1pt

next	yfmsubpd y5, y4, y15, y6			;; I3/sine - I4/sine (new R4/sine)	; 10-14		n 25
next	yfmaddpd y4, y4, y15, y6			;; I3/sine + I4/sine (new I3/sine)	; 10-14		n 27
this	L1prefetchw srcreg+iter*srcinc+d2+d1+L1pd, L1pt

this	vmovapd y15, [screg+iter*scinc+192+32]		;; cosine/sine for R4/I4
this	yfmsubpd y6, y14, y15, y8			;; B4 = I4 * cosine/sine - R4		; 11-15		n 16
this	yfmaddpd y8, y8, y15, y14			;; A4 = R4 * cosine/sine + I4		; 11-15

this	vmovapd y15, [screg+iter*scinc+64+32]		;; cosine/sine for R2/I2
this	yfmsubpd y14, y12, y15, y11			;; B2 = I2 * cosine/sine - R2		; 12-16		n 17
this	yfmaddpd y11, y11, y15, y12			;; A2 = R2 * cosine/sine + I2		; 12-16

this	vmovapd y15, [screg+iter*scinc+128+32]		;; cosine/sine for R3/I3
this	yfmaddpd y12, y3, y15, y13			;; A3 = R3 * cosine/sine + I3		; 13-17		n 18
this	yfmsubpd y13, y13, y15, y3			;; B3 = I3 * cosine/sine - R3		; 13-17

this	vmovapd y15, [screg+iter*scinc+0+32]		;; cosine/sine for R1/I1
this	yfmaddpd y3, y9, y15, y10			;; A1 = R1 * cosine/sine + I1		; 14-18		n 19
this	yfmsubpd y10, y10, y15, y9			;; B1 = I1 * cosine/sine - R1		; 14-18
next	vmovapd	y9, [srcreg+(iter+1)*srcinc+32]		;; I1

this	vmovapd y15, [screg+iter*scinc+192]		;; sine for R4/I4
this	vmulpd	y6, y6, y15				;; B4 = B4 * sine (final I4)		; 16-20  (out-of-order to free a reg)
this	vmulpd	y8, y8, y15				;; A4 = A4 * sine (final R4)		; 16-20

next	vmovapd	y15, [screg+(iter+1)*scinc+256+64]	;; sine for R2/I2
this	ystore	[srcreg+iter*srcinc+d2+d1+32], y6	;; Save I4				; 21
next	yfnmaddpd y6, y0, y15, y9			;; I1 - (I2 = B2 * sine) (new I2)	; 15-19		n 24
next	yfmaddpd y0, y0, y15, y9			;; I1 + (I2 = B2 * sine) (new I1)	; 15-19		n 27

this	vmovapd y9, [screg+iter*scinc+64]		;; sine for R2/I2
this	vmulpd	y14, y14, y9				;; B2 = B2 * sine (final I2)		; 17-21
this	vmulpd	y11, y11, y9				;; A2 = A2 * sine (final R2)		; 17-21
next	vmovapd	y9, [srcreg+(iter+1)*srcinc]		;; R1

this	ystore	[srcreg+iter*srcinc+d2+d1], y8		;; Save R4				; 21+1
this	vmovapd y8, [screg+iter*scinc+128]		;; sine for R3/I3
this	vmulpd	y12, y12, y8				;; A3 = A3 * sine (final R3)		; 18-22
this	vmulpd	y13, y13, y8				;; B3 = B3 * sine (final I3)		; 18-22

this	vmovapd y8, [screg+iter*scinc+0]		;; sine for R1/I1
this	vmulpd	y3, y3, y8				;; A1 = A1 * sine (final R1)		; 19-23
this	vmulpd	y10, y10, y8				;; B1 = B1 * sine (final I1)		; 19-23

;; Shuffle register assignments so that this R1,I1,R2,I2,R3,I3 are in y0-5 and next R1,I1,I2,R3/sine,I3/sine,R4/sine,I4/sine,A2,sine2 are in y6-14.

ytmp	TEXTEQU	y0
y0	TEXTEQU	y3
y3	TEXTEQU	y14
y14	TEXTEQU	y15
y15	TEXTEQU	y8
y8	TEXTEQU	y6
y6	TEXTEQU	y9
y9	TEXTEQU	y2
y2	TEXTEQU	y11
y11	TEXTEQU	y5
y5	TEXTEQU	y13
y13	TEXTEQU	y1
y1	TEXTEQU	y10
y10	TEXTEQU y4
y4	TEXTEQU	y12
y12	TEXTEQU	y7
y7	TEXTEQU	ytmp
	ENDM

ENDIF

ENDIF


;;
;; ******************************* four-complex-with-partial-normalization variants *************************************
;;
;; These macros are used in pass 1 of r4dwpn two pass FFTs.  They are like the standard four-complex
;; DJBFFT macros except that a normalization multiplier has been pre-applied to the sine multiplier.
;; Consequently, the forward FFT and inverse FFT use different sine multipliers.
;; Also, a normalization multiplier must be applied to the final R1/I1 value.
;;

;; In this version of the _wpn macros, we use 2 s/c ptrs to save 16 bytes of memory.
;; We also group the data used in the forward and inverse FFT together on their own cache lines.

;; screg1 is normalization weights for R1/I1
;; screg2+0, screg2+64+0, screg2+128+0 is weighted sin/cos values for R2/I2 (w^n)
;; screg2+0+32, screg2+64+32, screg2+128+32 is weighted sin/cos values for R3/I3 (w^2n)

yr4_b4cl_wpn_four_complex_djbfft_preload MACRO
	ENDM
yr4_b4cl_wpn_four_complex_djbfft MACRO srcreg,srcinc,d1,d2,screg1,scinc1,screg2,scinc2,maxrpt,L1pt,L1pd
	vmovapd	ymm0, [srcreg]		;; R1
	vmovapd	ymm6, [srcreg+d2]	;; R3
	vaddpd	ymm2, ymm0, ymm6	;; R1 + R3 (new R1)		; 1-3

	vmovapd	ymm1, [srcreg+d1]	;; R2
	vmovapd	ymm7, [srcreg+d2+d1]	;; R4
	vaddpd	ymm3, ymm1, ymm7	;; R2 + R4 (new R2)		; 2-4

	vsubpd	ymm0, ymm0, ymm6	;; R1 - R3 (new R3)		; 3-5
	vsubpd	ymm1, ymm1, ymm7	;; R2 - R4 (new R4)		; 4-6

	vmovapd	ymm4, [srcreg+32]	;; I1
	vmovapd	ymm7, [srcreg+d2+32]	;; I3
	vaddpd	ymm6, ymm4, ymm7	;; I1 + I3 (new I1)		; 5-7

	L1prefetchw srcreg+L1pd, L1pt

	vsubpd	ymm5, ymm2, ymm3	;; R1 - R2 (final R2)		; 6-8
	vaddpd	ymm2, ymm2, ymm3	;; R1 + R2 (final R1)		; 7-9

	vbroadcastsd ymm3, Q [screg1]	;; Load normalization multiplier for R1
	vmulpd	ymm2, ymm2, ymm3	;; Apply normalization multiplier to R1
	ystore	[srcreg], ymm2		;; Save R1

	vmovapd	ymm2, [srcreg+d1+32]	;; I2
	vaddpd	ymm3, ymm2, [srcreg+d2+d1+32] ;; I2 + I4 (new I2)	; 8-10
	vsubpd	ymm2, ymm2, [srcreg+d2+d1+32] ;; I2 - I4 (new I4)	; 9-11

	L1prefetchw srcreg+d1+L1pd, L1pt

	vsubpd	ymm4, ymm4, ymm7	;; I1 - I3 (new I3)		; 10-12

	vsubpd	ymm7, ymm6, ymm3	;; I1 - I2 (final I2)		; 11-13
	vaddpd	ymm6, ymm6, ymm3	;; I1 + I2 (final I1)		; 12-14

	vbroadcastsd ymm3, Q [screg1]	;; Load normalization multiplier for I1
	vmulpd	ymm6, ymm6, ymm3	;; Apply normalization multiplier to I1
	ystore	[srcreg+32], ymm6	;; Save I1

	vsubpd	ymm3, ymm0, ymm2	;; R3 - I4 (final R3)		; 13-15
	vaddpd	ymm6, ymm4, ymm1	;; I3 + R4 (final I3)		; 14-16

	L1prefetchw srcreg+d2+L1pd, L1pt

	vaddpd	ymm0, ymm0, ymm2	;; R3 + I4 (final R4)		; 15-17
	vsubpd	ymm4, ymm4, ymm1	;; I3 - R4 (final I4)		; 16-18

	vmovapd ymm1, [screg2+32]	;; cosine/sine
	vmulpd	ymm2, ymm5, ymm1	;; A2 = R2 * cosine/sine	;  9-13
	vsubpd	ymm2, ymm2, ymm7	;; A2 = A2 - I2			; 17-19

	vmulpd	ymm7, ymm7, ymm1	;; B2 = I2 * cosine/sine	; 14-18
	vaddpd	ymm7, ymm7, ymm5	;; B2 = B2 + R2			; 19-21

	vmovapd ymm1, [screg2+0]	;; cosine/sine
	vmulpd	ymm5, ymm3, ymm1	;; A3 = R3 * cosine/sine	; 16-20
	vsubpd	ymm5, ymm5, ymm6	;; A3 = A3 - I3			; 21-23

	vmulpd	ymm6, ymm6, ymm1	;; B3 = I3 * cosine/sine	; 17-21
	vaddpd	ymm6, ymm6, ymm3	;; B3 = B3 + R3			; 22-24

	L1prefetchw srcreg+d2+d1+L1pd, L1pt

	vmulpd	ymm3, ymm0, ymm1	;; A4 = R4 * cosine/sine	; 18-22
	vaddpd	ymm3, ymm3, ymm4	;; A4 = A4 + I4			; 23-25

	vmulpd	ymm4, ymm4, ymm1	;; B4 = I4 * cosine/sine	; 19-23
	vsubpd	ymm4, ymm4, ymm0	;; B4 = B4 - R4			; 24-26

	vmovapd ymm1, [screg2+64+32]	;; Sine
	vmulpd	ymm2, ymm2, ymm1	;; A2 = A2 * sine (final R2)	; 20-24
	vmulpd	ymm7, ymm7, ymm1	;; B2 = B2 * sine (final I2)	; 22-26

	vmovapd ymm1, [screg2+64+0]	;; Sine
	vmulpd	ymm5, ymm5, ymm1	;; A3 = A3 * sine (final R3)	; 24-28
	vmulpd	ymm6, ymm6, ymm1	;; B3 = B3 * sine (final I3)	; 25-29

	vmulpd	ymm3, ymm3, ymm1	;; A4 = A4 * sine (final R4)	; 26-30
	vmulpd	ymm4, ymm4, ymm1	;; B4 = B4 * sine (final I4)	; 27-31

;;	ystore	[srcreg], ymm0		;; Save R1
;;	ystore	[srcreg+32], ymm0	;; Save I1
	ystore	[srcreg+d1], ymm2	;; Save R2
	ystore	[srcreg+d1+32], ymm7	;; Save I2
	ystore	[srcreg+d2], ymm5	;; Save R3
	ystore	[srcreg+d2+32], ymm6	;; Save I3
	ystore	[srcreg+d2+d1], ymm3	;; Save R4
	ystore	[srcreg+d2+d1+32], ymm4	;; Save I4
	bump	srcreg, srcinc
	bump	screg1, scinc1
	bump	screg2, scinc2
	ENDM


yr4_b4cl_wpn_four_complex_djbunfft_preload MACRO
	ENDM
yr4_b4cl_wpn_four_complex_djbunfft MACRO srcreg,srcinc,d1,d2,screg1,scinc1,screg2,scinc2,maxrpt,L1pt,L1pd
	vmovapd ymm3, [screg2+32]	;; cosine/sine
	vmovapd	ymm2, [srcreg+d1]	;; R2
	vmulpd	ymm6, ymm2, ymm3	;; A2 = R2 * cosine/sine		; 1-5
	vmovapd	ymm0, [srcreg+d1+32]	;; I2
	vmulpd	ymm3, ymm3, ymm0	;; B2 = I2 * cosine/sine		; 2-6

	vmovapd ymm5, [screg2+0]	;; cosine/sine
	vmovapd	ymm4, [srcreg+d2]	;; R3
	vmulpd	ymm7, ymm4, ymm5	;; A3 = R3 * cosine/sine		; 3-7
	vmovapd	ymm1, [srcreg+d2+32]	;; I3

	vaddpd	ymm6, ymm6, ymm0	;; A2 = A2 + I2				; 6-8

	vmulpd	ymm0, ymm1, ymm5	;; B3 = I3 * cosine/sine		; 4-8

	vsubpd	ymm3, ymm3, ymm2	;; B2 = B2 - R2				; 7-9
	vaddpd	ymm7, ymm7, ymm1	;; A3 = A3 + I3				; 8-10

	vmovapd	ymm2, [srcreg+d2+d1]	;; R4
	vmulpd	ymm1, ymm2, ymm5	;; A4 = R4 * cosine/sine		; 5-9

	vsubpd	ymm0, ymm0, ymm4	;; B3 = B3 - R3				; 9-11

	vmovapd	ymm4, [srcreg+d2+d1+32]	;; I4
	vmulpd	ymm5, ymm4, ymm5	;; B4 = I4 * cosine/sine		; 6-10

	vsubpd	ymm1, ymm1, ymm4	;; A4 = A4 - I4				; 10-12
	vaddpd	ymm5, ymm5, ymm2	;; B4 = B4 + R4				; 11-13

	vbroadcastsd ymm2, Q [screg1]	;; normalization_inverse
	vmulpd	ymm2, ymm2, [srcreg]	;; R1 * normalization_inverse		; 8-12

	vmovapd ymm4, [screg2+128+32]	;; Sine * normalization_inverse
	vmulpd	ymm6, ymm6, ymm4	;; A2 = A2 * sine (new R2)		; 9-13
	vmulpd	ymm3, ymm3, ymm4	;; B2 = B2 * sine (new I2)		; 10-14

	L1prefetchw srcreg+L1pd, L1pt

	vmovapd ymm4, [screg2+128+0]	;; Sine * normalization_inverse
	vmulpd	ymm7, ymm7, ymm4	;; A3 = A3 * sine (new R3)		; 11-15
	vmulpd	ymm0, ymm0, ymm4	;; B3 = B3 * sine (new I3)		; 12-16
	vmulpd	ymm1, ymm1, ymm4	;; A4 = A4 * sine (new R4)		; 13-17
	vmulpd	ymm5, ymm5, ymm4	;; B4 = B4 * sine (new I4)		; 14-18

	vaddpd	ymm4, ymm2, ymm6	;; R1 + R2 (new R1)			; 14-16
	vsubpd	ymm2, ymm2, ymm6	;; R1 - R2 (new R2)			; 15-17

	L1prefetchw srcreg+d1+L1pd, L1pt

	vaddpd	ymm6, ymm1, ymm7	;; R4 + R3 (new R3)			; 16-18
	vsubpd	ymm1, ymm1, ymm7	;; R4 - R3 (new I4)			; 17-19

	vsubpd	ymm7, ymm0, ymm5	;; I3 - I4 (new R4)			; 18-20
	vaddpd	ymm0, ymm0, ymm5	;; I3 + I4 (new I3)			; 19-21

	L1prefetchw srcreg+d2+L1pd, L1pt

	vsubpd	ymm5, ymm4, ymm6	;; R1 - R3 (final R3)			; 20-22
	vaddpd	ymm4, ymm4, ymm6	;; R1 + R3 (final R1)			; 21-23

	vbroadcastsd ymm6, Q [screg1]	;; normalization_inverse
	vmulpd	ymm6, ymm6, [srcreg+32]	;; I1 * normalization_inverse		; 15-19
	ystore	[srcreg], ymm4		;; Store R1

	L1prefetchw srcreg+d2+d1+L1pd, L1pt

	vsubpd	ymm4, ymm6, ymm3	;; I1 - I2 (new I2)			; 22-24
	vaddpd	ymm6, ymm6, ymm3	;; I1 + I2 (new I1)			; 23-25

	vsubpd	ymm3, ymm2, ymm7	;; R2 - R4 (final R4)			; 24-26
	vaddpd	ymm2, ymm2, ymm7	;; R2 + R4 (final R2)			; 25-27

	vsubpd	ymm7, ymm4, ymm1	;; I2 - I4 (final I4)			; 26-28
	vaddpd	ymm4, ymm4, ymm1	;; I2 + I4 (final I2)			; 27-29

	vsubpd	ymm1, ymm6, ymm0	;; I1 - I3 (final I3)			; 28-30
	vaddpd	ymm6, ymm6, ymm0	;; I1 + I3 (final I1)			; 29-31

;;	ystore	[srcreg], ymm0		;; Save R1
	ystore	[srcreg+32], ymm6	;; Save I1
	ystore	[srcreg+d1], ymm2	;; Save R2
	ystore	[srcreg+d1+32], ymm4	;; Save I2
	ystore	[srcreg+d2], ymm5	;; Save R3
	ystore	[srcreg+d2+32], ymm1	;; Save I3
	ystore	[srcreg+d2+d1], ymm3	;; Save R4
	ystore	[srcreg+d2+d1+32], ymm7	;; Save I4
	bump	srcreg, srcinc
	bump	screg1, scinc1
	bump	screg2, scinc2
	ENDM


;; 64-bit version

IFDEF X86_64

yr4_b4cl_wpn_four_complex_djbfft_preload MACRO
	yr4_4c_wpn_djbfft_unroll_preload
	ENDM
yr4_b4cl_wpn_four_complex_djbfft MACRO srcreg,srcinc,d1,d2,screg1,scinc1,screg2,scinc2,maxrpt,L1pt,L1pd
	IF maxrpt MOD 5 EQ 0
	yr4_4c_wpn_djbfft_unroll -1,noexec,noexec,exec,srcreg,srcinc,d1,d2,screg1,scinc1,screg2,scinc2,L1pt,L1pd
	yr4_4c_wpn_djbfft_unroll 0,noexec,exec,exec,srcreg,srcinc,d1,d2,screg1,scinc1,screg2,scinc2,L1pt,L1pd
	yr4_4c_wpn_djbfft_unroll 1,exec,exec,exec,srcreg,srcinc,d1,d2,screg1,scinc1,screg2,scinc2,L1pt,L1pd
	yr4_4c_wpn_djbfft_unroll 2,exec,exec,exec,srcreg,srcinc,d1,d2,screg1,scinc1,screg2,scinc2,L1pt,L1pd
	yr4_4c_wpn_djbfft_unroll 3,exec,exec,exec,srcreg,srcinc,d1,d2,screg1,scinc1,screg2,scinc2,L1pt,L1pd
	yr4_4c_wpn_djbfft_unroll 4,exec,exec,noexec,srcreg,srcinc,d1,d2,screg1,scinc1,screg2,scinc2,L1pt,L1pd
	yr4_4c_wpn_djbfft_unroll 5,exec,noexec,noexec,srcreg,srcinc,d1,d2,screg1,scinc1,screg2,scinc2,L1pt,L1pd
	bump	srcreg, 5*srcinc
	bump	screg1, 5*scinc1
	bump	screg2, 5*scinc2
	ELSEIF maxrpt MOD 4 EQ 0
	yr4_4c_wpn_djbfft_unroll -1,noexec,noexec,exec,srcreg,srcinc,d1,d2,screg1,scinc1,screg2,scinc2,L1pt,L1pd
	yr4_4c_wpn_djbfft_unroll 0,noexec,exec,exec,srcreg,srcinc,d1,d2,screg1,scinc1,screg2,scinc2,L1pt,L1pd
	yr4_4c_wpn_djbfft_unroll 1,exec,exec,exec,srcreg,srcinc,d1,d2,screg1,scinc1,screg2,scinc2,L1pt,L1pd
	yr4_4c_wpn_djbfft_unroll 2,exec,exec,exec,srcreg,srcinc,d1,d2,screg1,scinc1,screg2,scinc2,L1pt,L1pd
	yr4_4c_wpn_djbfft_unroll 3,exec,exec,noexec,srcreg,srcinc,d1,d2,screg1,scinc1,screg2,scinc2,L1pt,L1pd
	yr4_4c_wpn_djbfft_unroll 4,exec,noexec,noexec,srcreg,srcinc,d1,d2,screg1,scinc1,screg2,scinc2,L1pt,L1pd
	bump	srcreg, 4*srcinc
	bump	screg1, 4*scinc1
	bump	screg2, 4*scinc2
	ELSEIF maxrpt MOD 3 EQ 0
	yr4_4c_wpn_djbfft_unroll -1,noexec,noexec,exec,srcreg,srcinc,d1,d2,screg1,scinc1,screg2,scinc2,L1pt,L1pd
	yr4_4c_wpn_djbfft_unroll 0,noexec,exec,exec,srcreg,srcinc,d1,d2,screg1,scinc1,screg2,scinc2,L1pt,L1pd
	yr4_4c_wpn_djbfft_unroll 1,exec,exec,exec,srcreg,srcinc,d1,d2,screg1,scinc1,screg2,scinc2,L1pt,L1pd
	yr4_4c_wpn_djbfft_unroll 2,exec,exec,noexec,srcreg,srcinc,d1,d2,screg1,scinc1,screg2,scinc2,L1pt,L1pd
	yr4_4c_wpn_djbfft_unroll 3,exec,noexec,noexec,srcreg,srcinc,d1,d2,screg1,scinc1,screg2,scinc2,L1pt,L1pd
	bump	srcreg, 3*srcinc
	bump	screg1, 3*scinc1
	bump	screg2, 3*scinc2
	ELSEIF maxrpt MOD 2 EQ 0
	yr4_4c_wpn_djbfft_unroll -1,noexec,noexec,exec,srcreg,srcinc,d1,d2,screg1,scinc1,screg2,scinc2,L1pt,L1pd
	yr4_4c_wpn_djbfft_unroll 0,noexec,exec,exec,srcreg,srcinc,d1,d2,screg1,scinc1,screg2,scinc2,L1pt,L1pd
	yr4_4c_wpn_djbfft_unroll 1,exec,exec,noexec,srcreg,srcinc,d1,d2,screg1,scinc1,screg2,scinc2,L1pt,L1pd
	yr4_4c_wpn_djbfft_unroll 2,exec,noexec,noexec,srcreg,srcinc,d1,d2,screg1,scinc1,screg2,scinc2,L1pt,L1pd
	bump	srcreg, 2*srcinc
	bump	screg1, 2*scinc1
	bump	screg2, 2*scinc2
	ELSE
	yr4_4c_wpn_djbfft_unroll -1,noexec,noexec,exec,srcreg,srcinc,d1,d2,screg1,scinc1,screg2,scinc2,L1pt,L1pd
	yr4_4c_wpn_djbfft_unroll 0,noexec,exec,noexec,srcreg,srcinc,d1,d2,screg1,scinc1,screg2,scinc2,L1pt,L1pd
	yr4_4c_wpn_djbfft_unroll 1,exec,noexec,noexec,srcreg,srcinc,d1,d2,screg1,scinc1,screg2,scinc2,L1pt,L1pd
	bump	srcreg, srcinc
	bump	screg1, scinc1
	bump	screg2, scinc2
	ENDIF
	ENDM

yr4_4c_wpn_djbfft_unroll_preload MACRO
	ENDM
yr4_4c_wpn_djbfft_unroll MACRO iter,prev,this,next,srcreg,srcinc,d1,d2,screg1,scinc1,screg2,scinc2,L1pt,L1pd

;; On first call, init registers

	IF iter EQ -1
y0	TEXTEQU	<ymm0>
y1	TEXTEQU	<ymm1>
y2	TEXTEQU	<ymm2>
y3	TEXTEQU	<ymm3>
y4	TEXTEQU	<ymm4>
y5	TEXTEQU	<ymm5>
y6	TEXTEQU	<ymm6>
y7	TEXTEQU	<ymm7>
y8	TEXTEQU	<ymm8>
y9	TEXTEQU	<ymm9>
y10	TEXTEQU	<ymm10>
y11	TEXTEQU	<ymm11>
y12	TEXTEQU	<ymm12>
y13	TEXTEQU	<ymm13>
y14	TEXTEQU	<ymm14>
y15	TEXTEQU	<ymm15>
	ENDIF

;; On later calls, previous R3,R4,I3,B4,A2,B2,sine1,sine2 will be in y0-7.  This R1,R3,I2,I4,I1,I3 will be in y8-13.
;; The remaining registers are free.

this	vsubpd	y14, y8, y9				;; R1 - R3 (new R3)		; 1-3
prev	vmulpd	y3, y3, y6				;; B4 = B4 * sine (final I4)	; 1-5
this	vmovapd	y6, [srcreg+iter*srcinc+d1]		;; R2

this	vaddpd	y8, y8, y9				;; R1 + R3 (new R1)		; 2-4
prev	vmulpd	y4, y4, y7				;; A2 = A2 * sine (final R2)	; 2-6
this	vmovapd	y9, [srcreg+iter*srcinc+d2+d1]		;; R4

this	vsubpd	y15, y10, y11				;; I2 - I4 (new I4)		; 3-5
prev	vmulpd	y5, y5, y7				;; B2 = B2 * sine (final I2)	; 3-7
this	vmovapd y7, [screg2+iter*scinc2+0]		;; cosine/sine
prev	ystore	[srcreg+(iter-1)*srcinc+d2], y0		;; Save R3			; 3

this	vaddpd	y10, y10, y11				;; I2 + I4 (new I2)		; 4-6
this	vmovapd y11, [screg2+iter*scinc2+32]		;; cosine/sine
prev	ystore	[srcreg+(iter-1)*srcinc+d2+d1], y1	;; Save R4			; 4

this	vsubpd	y1, y12, y13				;; I1 - I3 (new I3)		; 5-7
this	vmovapd y0, [screg2+iter*scinc2+64+0]		;; Sine
prev	ystore	[srcreg+(iter-1)*srcinc+d2+32], y2	;; Save I3			; 5

this	vaddpd	y12, y12, y13				;; I1 + I3 (new I1)		; 6-8
prev	ystore	[srcreg+(iter-1)*srcinc+d2+d1+32], y3	;; Save I4			; 6
this	L1prefetchw srcreg+iter*srcinc+L1pd, L1pt

this	vsubpd	y3, y6, y9				;; R2 - R4 (new R4)		; 7-9
prev	ystore	[srcreg+(iter-1)*srcinc+d1], y4		;; Save R2			; 7

this	vaddpd	y6, y6, y9				;; R2 + R4 (new R2)		; 8-10
prev	ystore	[srcreg+(iter-1)*srcinc+d1+32], y5	;; Save I2			; 8

this	vsubpd	y5, y14, y15				;; R3 - I4 (final R3)		; 9-11
this	L1prefetchw srcreg+iter*srcinc+d1+L1pd, L1pt

this	vaddpd	y14, y14, y15				;; R3 + I4 (final R4)		; 10-12

this	vaddpd	y15, y1, y3				;; I3 + R4 (final I3)		; 11-13

this	vsubpd	y1, y1, y3				;; I3 - R4 (final I4)		; 12-14
this	vmulpd	y3, y5, y7				;; A3 = R3 * cosine/sine	; 12-16
this	L1prefetchw srcreg+iter*srcinc+d2+L1pd, L1pt

this	vsubpd	y9, y8, y6				;; R1 - R2 (final R2)		; 13-15
this	vmulpd	y4, y14, y7				;; A4 = R4 * cosine/sine	; 13-17

this	vsubpd	y13, y12, y10				;; I1 - I2 (final I2)		; 14-16
this	vmulpd	y2, y15, y7				;; B3 = I3 * cosine/sine	; 14-18
this next yloop_unrolled_one

this	vaddpd	y8, y8, y6				;; R1 + R2 (final R1)		; 15-17
this	vmulpd	y7, y1, y7				;; B4 = I4 * cosine/sine	; 15-19
this	vbroadcastsd y6, Q [screg1+iter*scinc1]		;; Normalization multiplier for R1 & I1

this	vaddpd	y12, y12, y10				;; I1 + I2 (final I1)		; 16-18
this	vmulpd	y10, y9, y11				;; A2 = R2 * cosine/sine	; 16-20
this	L1prefetchw srcreg+iter*srcinc+d2+d1+L1pd, L1pt

this	vsubpd	y3, y3, y15				;; A3 = A3 - I3			; 17-19
this	vmulpd	y11, y13, y11				;; B2 = I2 * cosine/sine	; 17-21
next	vmovapd	y15, [srcreg+(iter+1)*srcinc]		;; R1

this	vaddpd	y4, y4, y1				;; A4 = A4 + I4			; 18-20
this	vmulpd	y8, y8, y6				;; Apply normalization multiplier to R1	; 18-22
next	vmovapd	y1, [srcreg+(iter+1)*srcinc+d2]		;; R3

this	vaddpd	y2, y2, y5				;; B3 = B3 + R3			; 19-21
this	vmulpd	y12, y12, y6				;; Apply normalization multiplier to I1 ; 19-23
this	vmovapd y5, [screg2+iter*scinc2+64+32]		;; Sine
next	vmovapd	y6, [srcreg+(iter+1)*srcinc+d1+32]	;; I2

this	vsubpd	y7, y7, y14				;; B4 = B4 - R4			; 20-22
this	vmulpd	y3, y3, y0				;; A3 = A3 * sine (final R3)	; 20-24
next	vmovapd	y14, [srcreg+(iter+1)*srcinc+d2+d1+32]	;; I4

this	vsubpd	y10, y10, y13				;; A2 = A2 - I2			; 21-23
this	vmulpd	y4, y4, y0				;; A4 = A4 * sine (final R4)	; 21-25
next	vmovapd	y13, [srcreg+(iter+1)*srcinc+32]	;; I1
this	ystore	[srcreg+iter*srcinc], y8		;; Save R1			; 23

this	vaddpd	y11, y11, y9				;; B2 = B2 + R2			; 22-24
this	vmulpd	y2, y2, y0				;; B3 = B3 * sine (final I3)	; 22-26
next	vmovapd	y9, [srcreg+(iter+1)*srcinc+d2+32]	;; I3
this	ystore	[srcreg+iter*srcinc+32], y12		;; Save I1			; 24

;; Shuffle register assignments so that next call has R3,R4,I3,B4,A2,B2,sine1,sine2 in y0-7 and next R1,R3,I2,I4,I1,I3 in y8-13.

ytmp	TEXTEQU	y0
y0	TEXTEQU	y3
y3	TEXTEQU	y7
y7	TEXTEQU	y5
y5	TEXTEQU	y11
y11	TEXTEQU	y14
y14	TEXTEQU	y8
y8	TEXTEQU	y15
y15	TEXTEQU	y12
y12	TEXTEQU	y13
y13	TEXTEQU y9
y9	TEXTEQU	y1
y1	TEXTEQU	y4
y4	TEXTEQU	y10
y10	TEXTEQU	y6
y6	TEXTEQU	ytmp
	ENDM


;; Haswell FMA3 version

IF (@INSTR(,%yarch,<FMA3>) NE 0)

yr4_4c_wpn_djbfft_unroll_preload MACRO
	vmovapd	ymm15, YMM_ONE
	ENDM

;; uops = 5 s/c loads, 8 loads, 8 stores, 6 muls, 22 fma, 11 mov = 60 uops / 4 = 15 clocks
;; Timed at 16.1 clocks
yr4_4c_wpn_djbfft_unroll MACRO iter,prev,this,next,srcreg,srcinc,d1,d2,screg1,scinc1,screg2,scinc2,L1pt,L1pd

;; On first call, init registers

	IF iter EQ -1
y0	TEXTEQU	<ymm0>
y1	TEXTEQU	<ymm1>
y2	TEXTEQU	<ymm2>
y3	TEXTEQU	<ymm3>
y4	TEXTEQU	<ymm4>
y5	TEXTEQU	<ymm5>
y6	TEXTEQU	<ymm6>
y7	TEXTEQU	<ymm7>
y8	TEXTEQU	<ymm8>
y9	TEXTEQU	<ymm9>
y10	TEXTEQU	<ymm10>
y11	TEXTEQU	<ymm11>
y12	TEXTEQU	<ymm12>
y13	TEXTEQU	<ymm13>
y14	TEXTEQU	<ymm14>
	ENDIF

;; On later calls, previous R3,I3,A2,B2,R4S,I4S,c/s34 will be in y0-6.  This R1,I1,R2,I2,R3,I3,R4,I4 will be in y7-14.
;; The remaining register is reserved for constant YMM_ONE.

prev	ystore	[srcreg+(iter-1)*srcinc+d2], y0		;; Save R3				; 18
this	yfmaddpd y0, y8, ymm15, y12			;; I1 + I3 (new I1)			; 1-5	n 6
this	yfmsubpd y8, y8, ymm15, y12			;; I1 - I3 (new I3)			; 1-5	n 10
this	vmovapd y12, [screg2+iter*scinc2+64+0]		;; Sine for R3/I3/R4/I4

this	vmulpd	y11, y11, y12				;; R3S = R3 * sine			; 2-6	n 7
this	vmulpd	y13, y13, y12				;; R4S = R4 * sine			; 2-6
this	L1prefetchw srcreg+iter*srcinc+L1pd, L1pt

prev	ystore	[srcreg+(iter-1)*srcinc+d2+32], y1	;; Save I3				; 18
prev	yfmaddpd y1, y4, y6, y5				;; R4S * cosine/sine + I4S (final R4)	; 3-7
prev	yfmsubpd y5, y5, y6, y4				;; I4S * cosine/sine - R4S (final I4)	; 3-7
prev	vmovapd y6, [screg2+(iter-1)*scinc2+64+32]	;; Sine for R2/I2

prev	vmulpd	y2, y2, y6				;; A2 = A2 * sine (final R2)		; 4-8
prev	vmulpd	y3, y3, y6				;; B2 = B2 * sine (final I2)		; 4-8
next	vmovapd	y4, [srcreg+(iter+1)*srcinc]		;; R1

this	yfmsubpd y6, y7, ymm15, y9			;; R1 - R2 (newer R2)			; 5-9	n 11
this	yfmaddpd y7, y7, ymm15, y9			;; R1 + R2 (newer R1)			; 5-9	n 12
next	vmovapd	y9, [srcreg+(iter+1)*srcinc+d2]		;; R3
this	L1prefetchw srcreg+iter*srcinc+d1+L1pd, L1pt

prev	ystore	[srcreg+(iter-1)*srcinc+d2+d1], y1	;; Save R4				; 8
this	yfmsubpd y1, y0, ymm15, y10			;; I1 - I2 (newer I2)			; 6-10	n 11
this	yfmaddpd y0, y0, ymm15, y10			;; I1 + I2 (newer I1)			; 6-10	n 12
next	vmovapd	y10, [srcreg+(iter+1)*srcinc+d1]	;; R2

prev	ystore	[srcreg+(iter-1)*srcinc+d2+d1+32], y5	;; Save I4				; 8
this	yfnmaddpd y5, y14, y12, y11			;; R3S - I4 * sine (newer R3S)		; 7-11	n 13
this	yfmaddpd y14, y14, y12, y11			;; R3S + I4 * sine (newer R4S)		; 7-11	n 17
next	vmovapd	y11, [srcreg+(iter+1)*srcinc+d2+d1]	;; R4

prev	ystore	[srcreg+(iter-1)*srcinc+d1], y2		;; Save R2				; 9
this	yfmaddpd y2, y8, y12, y13			;; I3 * sine + R4S (newer I3S)		; 8-12	n 13
this	yfmsubpd y8, y8, y12, y13			;; I3 * sine - R4S (newer I4S)		; 8-12	n 17
this	vmovapd y12, [screg2+iter*scinc2+32]		;; cosine/sine for R2/I2

next	yfmsubpd y13, y4, ymm15, y9			;; R1 - R3 (new R3)			; 9-13	n 16
next	yfmaddpd y4, y4, ymm15, y9			;; R1 + R3 (new R1)			; 9-13	n 19
this	vbroadcastsd y9, Q [screg1+iter*scinc1]		;; Normalization multiplier for R1 & I1
this	L1prefetchw srcreg+iter*srcinc+d2+L1pd, L1pt

prev	ystore	[srcreg+(iter-1)*srcinc+d1+32], y3	;; Save I2				; 9
next	yfmsubpd y3, y10, ymm15, y11			;; R2 - R4 (new R4)			; 10-14	n 16
next	yfmaddpd y10, y10, ymm15, y11			;; R2 + R4 (new R2)			; 10-14	n 19

this	yfmsubpd y11, y6, y12, y1			;; A2 = R2 * cosine/sine - I2		; 11-15	n 18
this	yfmaddpd y1, y1, y12, y6			;; B2 = I2 * cosine/sine + R2		; 11-15
this	vmovapd y12, [screg2+iter*scinc2+0]		;; cosine/sine for R3/I3/R4/I4

this	vmulpd	y7, y7, y9				;; Apply normalization multiplier to R1	; 12-16
this	vmulpd	y0, y0, y9				;; Apply normalization multiplier to I1 ; 12-16
next	vmovapd	y6, [srcreg+(iter+1)*srcinc+d1+32]	;; I2
this	L1prefetchw srcreg+iter*srcinc+d2+d1+L1pd, L1pt

this	yfmsubpd y9, y5, y12, y2			;; R3S * cosine/sine - I3S (final R3)	; 13-17
this	yfmaddpd y2, y2, y12, y5			;; I3S * cosine/sine + R3S (final I3)	; 13-17
next	vmovapd	y5, [srcreg+(iter+1)*srcinc+d2+d1+32]	;; I4
this next yloop_unrolled_one

this	ystore	[srcreg+iter*srcinc], y7		;; Save R1				; 17
next	yfmaddpd y7, y6, ymm15, y5			;; I2 + I4 (new I2)			; 14-18	n 20
next	yfmsubpd y6, y6, ymm15, y5			;; I2 - I4 (new I4)			; 14-18	n 21
next	vmovapd	y5, [srcreg+(iter+1)*srcinc+32]		;; I1
this	ystore	[srcreg+iter*srcinc+32], y0		;; Save I1				; 17
next	vmovapd	y0, [srcreg+(iter+1)*srcinc+d2+32]	;; I3

;; Shuffle register assignments so that this R3,I3,A2,B2,R4S,I4S,c/s34 are in y0-6 and next R1,I1,R2,I2,R3,I3,R4,I4 are in y7-14.

ytmp	TEXTEQU	y0
y0	TEXTEQU	y9
y9	TEXTEQU	y10
y10	TEXTEQU	y7
y7	TEXTEQU	y4
y4	TEXTEQU	y14
y14	TEXTEQU	y6
y6	TEXTEQU	y12
y12	TEXTEQU	ytmp
ytmp	TEXTEQU	y1
y1	TEXTEQU y2
y2	TEXTEQU	y11
y11	TEXTEQU	y13
y13	TEXTEQU	y3
y3	TEXTEQU	ytmp
ytmp	TEXTEQU	y5
y5	TEXTEQU	y8
y8	TEXTEQU	ytmp

	ENDM

ENDIF

yr4_b4cl_wpn_four_complex_djbunfft_preload MACRO
	yr4_4c_wpn_djbunfft_unroll_preload
	ENDM
yr4_b4cl_wpn_four_complex_djbunfft MACRO srcreg,srcinc,d1,d2,screg1,scinc1,screg2,scinc2,maxrpt,L1pt,L1pd
	IF maxrpt MOD 5 EQ 0
	yr4_4c_wpn_djbunfft_unroll -1,noexec,noexec,exec,srcreg,srcinc,d1,d2,screg1,scinc1,screg2,scinc2,L1pt,L1pd
	yr4_4c_wpn_djbunfft_unroll 0,noexec,exec,exec,srcreg,srcinc,d1,d2,screg1,scinc1,screg2,scinc2,L1pt,L1pd
	yr4_4c_wpn_djbunfft_unroll 1,exec,exec,exec,srcreg,srcinc,d1,d2,screg1,scinc1,screg2,scinc2,L1pt,L1pd
	yr4_4c_wpn_djbunfft_unroll 2,exec,exec,exec,srcreg,srcinc,d1,d2,screg1,scinc1,screg2,scinc2,L1pt,L1pd
	yr4_4c_wpn_djbunfft_unroll 3,exec,exec,exec,srcreg,srcinc,d1,d2,screg1,scinc1,screg2,scinc2,L1pt,L1pd
	yr4_4c_wpn_djbunfft_unroll 4,exec,exec,noexec,srcreg,srcinc,d1,d2,screg1,scinc1,screg2,scinc2,L1pt,L1pd
	yr4_4c_wpn_djbunfft_unroll 5,exec,noexec,noexec,srcreg,srcinc,d1,d2,screg1,scinc1,screg2,scinc2,L1pt,L1pd
	bump	srcreg, 5*srcinc
	bump	screg1, 5*scinc1
	bump	screg2, 5*scinc2
	ELSEIF maxrpt MOD 4 EQ 0
	yr4_4c_wpn_djbunfft_unroll -1,noexec,noexec,exec,srcreg,srcinc,d1,d2,screg1,scinc1,screg2,scinc2,L1pt,L1pd
	yr4_4c_wpn_djbunfft_unroll 0,noexec,exec,exec,srcreg,srcinc,d1,d2,screg1,scinc1,screg2,scinc2,L1pt,L1pd
	yr4_4c_wpn_djbunfft_unroll 1,exec,exec,exec,srcreg,srcinc,d1,d2,screg1,scinc1,screg2,scinc2,L1pt,L1pd
	yr4_4c_wpn_djbunfft_unroll 2,exec,exec,exec,srcreg,srcinc,d1,d2,screg1,scinc1,screg2,scinc2,L1pt,L1pd
	yr4_4c_wpn_djbunfft_unroll 3,exec,exec,noexec,srcreg,srcinc,d1,d2,screg1,scinc1,screg2,scinc2,L1pt,L1pd
	yr4_4c_wpn_djbunfft_unroll 4,exec,noexec,noexec,srcreg,srcinc,d1,d2,screg1,scinc1,screg2,scinc2,L1pt,L1pd
	bump	srcreg, 4*srcinc
	bump	screg1, 4*scinc1
	bump	screg2, 4*scinc2
	ELSEIF maxrpt MOD 3 EQ 0
	yr4_4c_wpn_djbunfft_unroll -1,noexec,noexec,exec,srcreg,srcinc,d1,d2,screg1,scinc1,screg2,scinc2,L1pt,L1pd
	yr4_4c_wpn_djbunfft_unroll 0,noexec,exec,exec,srcreg,srcinc,d1,d2,screg1,scinc1,screg2,scinc2,L1pt,L1pd
	yr4_4c_wpn_djbunfft_unroll 1,exec,exec,exec,srcreg,srcinc,d1,d2,screg1,scinc1,screg2,scinc2,L1pt,L1pd
	yr4_4c_wpn_djbunfft_unroll 2,exec,exec,noexec,srcreg,srcinc,d1,d2,screg1,scinc1,screg2,scinc2,L1pt,L1pd
	yr4_4c_wpn_djbunfft_unroll 3,exec,noexec,noexec,srcreg,srcinc,d1,d2,screg1,scinc1,screg2,scinc2,L1pt,L1pd
	bump	srcreg, 3*srcinc
	bump	screg1, 3*scinc1
	bump	screg2, 3*scinc2
	ELSEIF maxrpt MOD 2 EQ 0
	yr4_4c_wpn_djbunfft_unroll -1,noexec,noexec,exec,srcreg,srcinc,d1,d2,screg1,scinc1,screg2,scinc2,L1pt,L1pd
	yr4_4c_wpn_djbunfft_unroll 0,noexec,exec,exec,srcreg,srcinc,d1,d2,screg1,scinc1,screg2,scinc2,L1pt,L1pd
	yr4_4c_wpn_djbunfft_unroll 1,exec,exec,noexec,srcreg,srcinc,d1,d2,screg1,scinc1,screg2,scinc2,L1pt,L1pd
	yr4_4c_wpn_djbunfft_unroll 2,exec,noexec,noexec,srcreg,srcinc,d1,d2,screg1,scinc1,screg2,scinc2,L1pt,L1pd
	bump	srcreg, 2*srcinc
	bump	screg1, 2*scinc1
	bump	screg2, 2*scinc2
	ELSE
	yr4_4c_wpn_djbunfft_unroll -1,noexec,noexec,exec,srcreg,srcinc,d1,d2,screg1,scinc1,screg2,scinc2,L1pt,L1pd
	yr4_4c_wpn_djbunfft_unroll 0,noexec,exec,noexec,srcreg,srcinc,d1,d2,screg1,scinc1,screg2,scinc2,L1pt,L1pd
	yr4_4c_wpn_djbunfft_unroll 1,exec,noexec,noexec,srcreg,srcinc,d1,d2,screg1,scinc1,screg2,scinc2,L1pt,L1pd
	bump	srcreg, srcinc
	bump	screg1, scinc1
	bump	screg2, scinc2
	ENDIF
	ENDM

yr4_4c_wpn_djbunfft_unroll_preload MACRO
	ENDM
yr4_4c_wpn_djbunfft_unroll MACRO iter,prev,this,next,srcreg,srcinc,d1,d2,screg1,scinc1,screg2,scinc2,L1pt,L1pd

;; On first call, init registers

	IF iter EQ -1
y0	TEXTEQU	<ymm0>
y1	TEXTEQU	<ymm1>
y2	TEXTEQU	<ymm2>
y3	TEXTEQU	<ymm3>
y4	TEXTEQU	<ymm4>
y5	TEXTEQU	<ymm5>
y6	TEXTEQU	<ymm6>
y7	TEXTEQU	<ymm7>
y8	TEXTEQU	<ymm8>
y9	TEXTEQU	<ymm9>
y10	TEXTEQU	<ymm10>
y11	TEXTEQU	<ymm11>
y12	TEXTEQU	<ymm12>
y13	TEXTEQU	<ymm13>
y14	TEXTEQU	<ymm14>
y15	TEXTEQU	<ymm15>
	ENDIF

;; On later calls, previous R2,I3,I1 will be in y0-2.  This R4,R3,R2,I4,I3,I2,A4,A3,A2,B4,B3,c/s2 will be in y3-14.
;; The remaining register is free.

this	vsubpd	y9, y9, y6			;; A4 = A4 - I4 (new R4/sine)		; 1-3
this	vmulpd	y14, y8, y14			;; B2 = I2 * cosine/sine		; 1-5
this	vbroadcastsd y15, Q [screg1+iter*scinc1] ;; normalization_inverse for R1 & I1

this	vaddpd	y10, y10, y7			;; A3 = A3 + I3 (new R3/sine)		; 2-4
this	vmovapd	y6, [srcreg+iter*srcinc]	;; R1

this	vaddpd	y11, y11, y8			;; A2 = A2 + I2				; 3-5
this	vmulpd	y6, y6, y15			;; R1 * normalization_inverse		; 3-7
this	vmovapd	y7, [srcreg+iter*srcinc+32]	;; I1

this	vaddpd	y12, y12, y3			;; B4 = B4 + R4 (new I4/sine)		; 4-6
this	vmulpd	y7, y7, y15			;; I1 * normalization_inverse		; 4-8
this	vmovapd y8, [screg2+iter*scinc2+128+32]	;; Sine

this	vsubpd	y13, y13, y4			;; B3 = B3 - R3 (new I3/sine)		; 5-7
this	vmovapd y3, [screg2+iter*scinc2+128+0]	;; Sine
prev	ystore	[srcreg+(iter-1)*srcinc+d1], y0	;; Save R2				; 1

this	vsubpd	y14, y14, y5			;; B2 = B2 - R2				; 6-8
this	vmulpd	y11, y11, y8			;; A2 = A2 * sine (new R2)		; 6-10
prev	ystore	[srcreg+(iter-1)*srcinc+d2+32], y1 ;; Save I3				; 2
this	L1prefetchw srcreg+iter*srcinc+L1pd, L1pt

this	vaddpd	y1, y9, y10			;; C3 = R4/sine + R3/sine (newer R3/sine) ; 7-9
prev	ystore	[srcreg+(iter-1)*srcinc+32], y2	;; Save I1				; 3

this	vsubpd	y9, y9, y10			;; D4 = R4/sine - R3/sine (newer I4/sine) ; 8-10
this	L1prefetchw srcreg+iter*srcinc+d1+L1pd, L1pt

this	vsubpd	y10, y13, y12			;; C4 = I3/sine - I4/sine (newer R4/sine) ; 9-11
this	vmulpd	y14, y14, y8			;; B2 = B2 * sine (new I2)		; 9-13
next	vmovapd y15, [screg2+(iter+1)*scinc2+0]	;; cosine/sine

this	vaddpd	y13, y13, y12			;; D3 = I3/sine + I4/sine (newer I3/sine) ; 10-12
this	vmulpd	y1, y1, y3			;; C3 * sine (newer R3)			; 10-14
next	vmovapd	y4, [srcreg+(iter+1)*srcinc+d2+d1] ;; R4

this	vaddpd	y12, y6, y11			;; R1 + R2 (newer R1)			; 11-13
this	vmulpd	y9, y9, y3			;; D4 * sine (newer I4)			; 11-15
next	vmovapd	y0, [srcreg+(iter+1)*srcinc+d2]	;; R3

this	vsubpd	y6, y6, y11			;; R1 - R2 (newer R2)			; 12-14
this	vmulpd	y10, y10, y3			;; C4 * sine (newer R4)			; 12-16
next	vmovapd y5, [screg2+(iter+1)*scinc2+32]	;; cosine/sine

this	vsubpd	y11, y7, y14			;; I1 - I2 (newer I2)			; 13-15
this	vmulpd	y13, y13, y3			;; D3 * sine (newer I3)			; 13-17
next	vmovapd	y2, [srcreg+(iter+1)*srcinc+d1]	;; R2

this	vaddpd	y7, y7, y14			;; I1 + I2 (newer I1)			; 14-16
this	L1prefetchw srcreg+iter*srcinc+d2+L1pd, L1pt

this	vsubpd	y14, y12, y1			;; R1 - R3 (final R3)			; 15-17
next	vmovapd	y8, [srcreg+(iter+1)*srcinc+d2+d1+32] ;; I4

this	vaddpd	y12, y12, y1			;; R1 + R3 (final R1)			; 16-18
next	vmovapd	y3, [srcreg+(iter+1)*srcinc+d2+32] ;; I3

this	ystore	[srcreg+iter*srcinc+d2], y14	;; Save R3				; 18
this	vsubpd	y14, y11, y9			;; I2 - I4 (final I4)			; 17-19
next	vmovapd	y1, [srcreg+(iter+1)*srcinc+d1+32] ;; I2

this	vaddpd	y11, y11, y9			;; I2 + I4 (final I2)			; 18-20
next	vmulpd	y9, y4, y15			;; A4 = R4 * cosine/sine		; 18-22
this	L1prefetchw srcreg+iter*srcinc+d2+d1+L1pd, L1pt

this	ystore	[srcreg+iter*srcinc], y12	;; Save R1				; 19
this	vsubpd	y12, y6, y10			;; R2 - R4 (final R4)			; 19-21
this	ystore	[srcreg+iter*srcinc+d2+d1+32], y14 ;; Save I4				; 20
next	vmulpd	y14, y0, y15			;; A3 = R3 * cosine/sine		; 19-23

this	vaddpd	y6, y6, y10			;; R2 + R4 (final R2)			; 20-22
next	vmulpd	y10, y2, y5			;; A2 = R2 * cosine/sine		; 20-24

this	ystore	[srcreg+iter*srcinc+d1+32], y11	;; Save I2				; 21
this	vsubpd	y11, y7, y13			;; I1 - I3 (final I3)			; 21-23
this	ystore	[srcreg+iter*srcinc+d2+d1], y12	;; Save R4				; 22
next	vmulpd	y12, y8, y15			;; B4 = I4 * cosine/sine		; 21-25

this	vaddpd	y7, y7, y13			;; I1 + I3 (final I1)			; 22-24
next	vmulpd	y13, y3, y15			;; B3 = I3 * cosine/sine		; 22-26
this next yloop_unrolled_one

;; Shuffle register assignments so that next call has R2,I3,I1 in y0-2 and next R4,R3,R2,I4,I3,I2,A4,A3,A2,B4,B3,c/s2 in y3-14.

ytmp	TEXTEQU	y0
y0	TEXTEQU	y6
y6	TEXTEQU y8
y8	TEXTEQU	y1
y1	TEXTEQU	y11
y11	TEXTEQU	y10
y10	TEXTEQU	y14
y14	TEXTEQU	y5
y5	TEXTEQU	y2
y2	TEXTEQU	y7
y7	TEXTEQU	y3
y3	TEXTEQU	y4
y4	TEXTEQU	ytmp
	ENDM


;; Haswell FMA3 version

IF (@INSTR(,%yarch,<FMA3>) NE 0)

yr4_4c_wpn_djbunfft_unroll_preload MACRO
	vmovapd ymm15, YMM_ONE
	ENDM

;; uops = 5 s/c loads, 8 loads, 8 stores, 2 muls, 22 fma, 11 mov = 56 uops / 4 = 14 clocks
;; Timed at 14.7 clocks
yr4_4c_wpn_djbunfft_unroll MACRO iter,prev,this,next,srcreg,srcinc,d1,d2,screg1,scinc1,screg2,scinc2,L1pt,L1pd

;; On first call, init registers

	IF iter EQ -1
y0	TEXTEQU	<ymm0>
y1	TEXTEQU	<ymm1>
y2	TEXTEQU	<ymm2>
y3	TEXTEQU	<ymm3>
y4	TEXTEQU	<ymm4>
y5	TEXTEQU	<ymm5>
y6	TEXTEQU	<ymm6>
y7	TEXTEQU	<ymm7>
y8	TEXTEQU	<ymm8>
y9	TEXTEQU	<ymm9>
y10	TEXTEQU	<ymm10>
y11	TEXTEQU	<ymm11>
y12	TEXTEQU	<ymm12>
y13	TEXTEQU	<ymm13>
y14	TEXTEQU	<ymm14>
	ENDIF

;; On later calls, previous I1,I2,I3/sine,I4/sine,sine34 will be in y0-4.
;; This R1,I1,R2/sine,I2/sine,R3/sine,I3/sine,R4/sine,I4/sine will be in y5-12.
;; The remaining registers are free.  Register y15 is reserved for the constant YMM_ONE.

prev	yfnmaddpd y13, y3, y4, y1			;; I2 - I4/sine * sine (final I4)	; 1-5
prev	yfmaddpd y3, y3, y4, y1				;; I2 + I4/sine * sine (final I2)	; 1-5
this	vmovapd y14, [screg2+iter*scinc2+128+32]	;; Sine for R2/I2

prev	yfnmaddpd y1, y2, y4, y0			;; I1 - I3/sine * sine (final I3)	; 2-6
prev	yfmaddpd y2, y2, y4, y0				;; I1 + I3/sine * sine (final I1)	; 2-6
next	vmovapd y4, [screg2+(iter+1)*scinc2+0]		;; cosine/sine for R3/I3/R4/I4
this	L1prefetchw srcreg+iter*srcinc+L1pd, L1pt

this	yfmaddpd y0, y11, ymm15, y9			;; R4/sine + R3/sine (newer R3/sine)	; 3-7
this	yfmsubpd y11, y11, ymm15, y9			;; R4/sine - R3/sine (newer I4/sine)	; 3-7
next	vmovapd	y9, [srcreg+(iter+1)*srcinc+d2+d1]	;; R4
this	L1prefetchw srcreg+iter*srcinc+d1+L1pd, L1pt

prev	ystore	[srcreg+(iter-1)*srcinc+d2+d1+32], y13	;; Save I4				; 6
this	yfmaddpd y13, y7, y14, y5			;; R1 + R2/sine * sine (newer R1)	; 4-8
this	yfnmaddpd y7, y7, y14, y5			;; R1 - R2/sine * sine (newer R2)	; 4-8
next	vmovapd	y5, [srcreg+(iter+1)*srcinc+d2+d1+32]	;; I4

prev	ystore	[srcreg+(iter-1)*srcinc+d1+32], y3	;; Save I2				; 6
this	yfmsubpd y3, y10, ymm15, y12			;; I3/sine - I4/sine (newer R4/sine)	; 5-9
this	yfmaddpd y10, y10, ymm15, y12			;; I3/sine + I4/sine (newer I3/sine)	; 5-9
next	vmovapd	y12, [srcreg+(iter+1)*srcinc+d2]	;; R3

prev	ystore	[srcreg+(iter-1)*srcinc+d2+32], y1	;; Save I3				; 7
next	yfmsubpd y1, y9, y4, y5				;; R4 * cosine/sine - I4 (new R4/sine)	; 6-10
next	yfmaddpd y5, y5, y4, y9				;; I4 * cosine/sine + R4 (new I4/sine)	; 6-10
next	vmovapd	y9, [srcreg+(iter+1)*srcinc+d2+32]	;; I3

prev	ystore	[srcreg+(iter-1)*srcinc+32], y2		;; Save I1				; 7
next	yfmaddpd y2, y12, y4, y9			;; R3 * cosine/sine + I3 (new R3/sine)	; 7-11
next	yfmsubpd y9, y9, y4, y12			;; I3 * cosine/sine - R3 (new I3/sine)	; 7-11
this	vmovapd y4, [screg2+iter*scinc2+128+0]		;; Sine for R3/I3/R4/I4

this	yfnmaddpd y12, y8, y14, y6			;; I1 - I2/sine * sine (newer I2)	; 8-12
this	yfmaddpd y8, y8, y14, y6			;; I1 + I2/sine * sine (newer I1)	; 8-12
next	vmovapd y14, [screg2+(iter+1)*scinc2+32]	;; cosine/sine for R2/I2
this	L1prefetchw srcreg+iter*srcinc+d2+L1pd, L1pt

this	yfnmaddpd y6, y0, y4, y13			;; R1 - R3/sine * sine (final R3)	; 9-13
this	yfmaddpd y0, y0, y4, y13			;; R1 + R3/sine * sine (final R1)	; 9-13
next	vmovapd	y13, [srcreg+(iter+1)*srcinc+d1]	;; R2
this	L1prefetchw srcreg+iter*srcinc+d2+d1+L1pd, L1pt

this	ystore	[srcreg+iter*srcinc+d2], y6		;; Save R3				; 14
this	yfnmaddpd y6, y3, y4, y7			;; R2 - R4/sine * sine (final R4)	; 10-14
this	yfmaddpd y3, y3, y4, y7				;; R2 + R4/sine * sine (final R2)	; 10-14
next	vmovapd	y7, [srcreg+(iter+1)*srcinc+d1+32]	;; I2

this	ystore	[srcreg+iter*srcinc], y0		;; Save R1				; 14
next	yfmaddpd y0, y13, y14, y7			;; R2 * cosine/sine + I2 (new R2/sine)	; 11-15
this	ystore	[srcreg+iter*srcinc+d2+d1], y6		;; Save R4				; 15
next	vbroadcastsd y6, Q [screg1+(iter+1)*scinc1]	;; normalization_inverse for R1 & I1
this	ystore	[srcreg+iter*srcinc+d1], y3		;; Save R2				; 15
next	vmovapd	y3, [srcreg+(iter+1)*srcinc]		;; R1
next	vmulpd	y3, y3, y6				;; R1 * normalization_inverse		; 11-15

next	yfmsubpd y7, y7, y14, y13			;; I2 * cosine/sine - R2 (new I2/sine)	; 12-16
next	vmovapd	y14, [srcreg+(iter+1)*srcinc+32]	;; I1
next	vmulpd	y14, y14, y6				;; I1 * normalization_inverse		; 12-16
this next yloop_unrolled_one

;; Shuffle register assignments so that this I1,I2,I3/sine,I4/sine,sine34 are in y0-4, and
;; next R1,I1,R2/sine,I2/sine,R3/sine,I3/sine,R4/sine,I4/sine are in y5-12.

ytmp	TEXTEQU	y0
y0	TEXTEQU	y8
y8	TEXTEQU y7
y7	TEXTEQU	ytmp
ytmp	TEXTEQU	y1
y1	TEXTEQU	y12
y12	TEXTEQU	y5
y5	TEXTEQU	y3
y3	TEXTEQU	y11
y11	TEXTEQU	ytmp
ytmp	TEXTEQU	y2
y2	TEXTEQU	y10
y10	TEXTEQU	y9
y9	TEXTEQU	ytmp
ytmp	TEXTEQU	y6
y6	TEXTEQU	y14
y14	TEXTEQU	ytmp

	ENDM

ENDIF

ENDIF


;;
;; ************************************* four-complex-with-square and variants ******************************************
;;
;; These macros are used in the last levels of pass 2 in two pass FFTs.
;;

;; Macros to do four four_complex_fft in the final levels of an FFT.

yr4_4cl_four_complex_fft_final_preload MACRO
	ENDM
yr4_4cl_four_complex_fft_final MACRO srcreg,srcinc,d1,d2
	yr4_4c_simple_fft_mem [srcreg],[srcreg+d1],[srcreg+d2],[srcreg+d2+d1],[srcreg+32],[srcreg+d1+32],[srcreg+d2+32],[srcreg+d2+d1+32],[srcreg]
;;	ystore	[srcreg], ymm0			;; Save R1
	ystore	[srcreg+32], ymm4		;; Save I1
	ystore	[srcreg+d1], ymm2		;; Save R2
	ystore	[srcreg+d1+32], ymm1		;; Save I2
	ystore	[srcreg+d2], ymm7		;; Save R3
	ystore	[srcreg+d2+32], ymm3		;; Save I3
	ystore	[srcreg+d2+d1], ymm0		;; Save R4
	ystore	[srcreg+d2+d1+32], ymm6		;; Save I4
	bump	srcreg, srcinc
	ENDM

yr4_4cl_four_complex_with_square_preload MACRO
	ENDM
yr4_4cl_four_complex_with_square MACRO srcreg,srcinc,d1,d2,maxrpt,L1pt,L1pd

	vmovapd	ymm0, [srcreg]			;; R1
	vmovapd	ymm7, [srcreg+d2]		;; R3
	vaddpd	ymm2, ymm0, ymm7		;; R1 + R3 (new R1)
	vsubpd	ymm0, ymm0, ymm7		;; R1 - R3 (new R3)

	vmovapd	ymm1, [srcreg+d1]		;; R2
	vmovapd	ymm7, [srcreg+d2+d1]		;; R4
	vaddpd	ymm3, ymm1, ymm7		;; R2 + R4 (new R2)
	vsubpd	ymm1, ymm1, ymm7		;; R2 - R4 (new R4)

	vaddpd	ymm6, ymm2, ymm3		;; R1 + R2 (final R1)
	vsubpd	ymm2, ymm2, ymm3		;; R1 - R2 (final R2)

	vmovapd	ymm5, [srcreg+d1+32]		;; I2
	vmovapd	ymm7, [srcreg+d2+d1+32]		;; I4
	vsubpd	ymm3, ymm5, ymm7		;; I2 - I4 (new I4)
	vaddpd	ymm5, ymm5, ymm7		;; I2 + I4 (new I2)

	vmovapd	ymm4, [srcreg+32]		;; I1
	vmovapd	ymm7, [srcreg+d2+32]		;; I3

	ystore	[srcreg], ymm6			;; Save R1

	vsubpd	ymm6, ymm4, ymm7		;; I1 - I3 (new I3)
	vaddpd	ymm4, ymm4, ymm7		;; I1 + I3 (new I1)

	L1prefetchw srcreg+L1pd, L1pt

	vsubpd	ymm7, ymm0, ymm3		;; R3 - I4 (final R3)
	vaddpd	ymm0, ymm0, ymm3		;; R3 + I4 (final R4)

	vaddpd	ymm3, ymm6, ymm1		;; I3 + R4 (final I3)
	vsubpd	ymm6, ymm6, ymm1		;; I3 - R4 (final I4)

	L1prefetchw srcreg+d1+L1pd, L1pt

	vsubpd	ymm1, ymm4, ymm5		;; I1 - I2 (final I2)
	vaddpd	ymm4, ymm4, ymm5		;; I1 + I2 (final I1)

	yp_complex_square ymm2, ymm1, ymm5	;; Square R2, I2
	yp_complex_square ymm7, ymm3, ymm5	;; Square R3, I3
	yp_complex_square ymm0, ymm6, ymm5	;; Square R4, I4
	ystore	[srcreg+d1+32], ymm1		;; Save I2
	vmovapd	ymm1, [srcreg]			;; Reload R1
	yp_complex_square ymm1, ymm4, ymm5	;; Square R1, I1

	vsubpd	ymm5, ymm1, ymm2		;; R1 - R2 (new R2)
	vaddpd	ymm1, ymm1, ymm2		;; R1 + R2 (new R1)

	L1prefetchw srcreg+d2+L1pd, L1pt

	vsubpd	ymm2, ymm0, ymm7		;; R4 - R3 (new I4)
	vaddpd	ymm0, ymm0, ymm7		;; R4 + R3 (new R3)

	vsubpd	ymm7, ymm3, ymm6		;; I3 - I4 (new R4)
	vaddpd	ymm3, ymm3, ymm6		;; I3 + I4 (new I3)

	vsubpd	ymm6, ymm1, ymm0		;; R1 - R3 (final R3)
	vaddpd	ymm1, ymm1, ymm0		;; R1 + R3 (final R1)

	L1prefetchw srcreg+d2+d1+L1pd, L1pt

	vsubpd	ymm0, ymm5, ymm7		;; R2 - R4 (final R4)
	vaddpd	ymm5, ymm5, ymm7		;; R2 + R4 (final R2)

	vmovapd	ymm7, [srcreg+d1+32]		;; Reload I2

	ystore	[srcreg], ymm1			;; Save R1

	vsubpd	ymm1, ymm4, ymm7		;; I1 - I2 (new I2)
	vaddpd	ymm4, ymm4, ymm7		;; I1 + I2 (new I1)

	vsubpd	ymm7, ymm1, ymm2		;; I2 - I4 (final I4)
	vaddpd	ymm1, ymm1, ymm2		;; I2 + I4 (final I2)

	vsubpd	ymm2, ymm4, ymm3		;; I1 - I3 (final I3)
	vaddpd	ymm4, ymm4, ymm3		;; I1 + I3 (final I1)

	ystore	[srcreg+32], ymm4		;; Save I1
	ystore	[srcreg+d1], ymm5		;; Save R2
	ystore	[srcreg+d1+32], ymm1		;; Save I2
	ystore	[srcreg+d2], ymm6		;; Save R3
	ystore	[srcreg+d2+32], ymm2		;; Save I3
	ystore	[srcreg+d2+d1], ymm0		;; Save R4
	ystore	[srcreg+d2+d1+32], ymm7		;; Save I4
	bump	srcreg, srcinc
	ENDM

yr4_4cl_four_complex_with_mult_preload MACRO
	ENDM
yr4_4cl_four_complex_with_mult MACRO srcreg,srcinc,d1,d2
	yr4_4c_simple_fft_mem [srcreg],[srcreg+d1],[srcreg+d2],[srcreg+d2+d1],[srcreg+32],[srcreg+d1+32],[srcreg+d2+32],[srcreg+d2+d1+32],[srcreg]
	ystore	[srcreg+32], ymm4		;; Save I1
	yp_complex_mult ymm2, ymm1, [srcreg+d1][rbp], [srcreg+d1+32][rbp], ymm4, ymm5 ;; Mult R2, I2
	yp_complex_mult ymm7, ymm3, [srcreg+d2][rbp], [srcreg+d2+32][rbp], ymm4, ymm5 ;; Mult R3, I3
	yp_complex_mult ymm0, ymm6, [srcreg+d2+d1][rbp], [srcreg+d2+d1+32][rbp], ymm4, ymm5 ;; Mult R4, I4
	ystore	[srcreg+d1], ymm2		;; Save R2
	ystore	[srcreg+d1+32], ymm1		;; Save I2
	ystore	[srcreg+d2], ymm7		;; Save R3
	ystore	[srcreg+d2+32], ymm3		;; Save I3
	ystore	[srcreg+d2+d1], ymm0		;; Save R4
	ystore	[srcreg+d2+d1+32], ymm6		;; Save I4
	vmovapd	ymm2, [srcreg]			;; Reload R1
	vmovapd	ymm4, [srcreg+32]		;; Reload I1
	yp_complex_mult ymm2, ymm4, [srcreg][rbp], [srcreg+32][rbp], ymm3, ymm5 ;; Mult R1, I1
	ystore	[srcreg], ymm2			;; Save R1
	ystore	[srcreg+32], ymm4		;; Save I1
	yr4_4c_simple_unfft_mem [srcreg],[srcreg+32],[srcreg+d1],[srcreg+d1+32],[srcreg+d2],[srcreg+d2+32],[srcreg+d2+d1],[srcreg+d2+d1+32],[srcreg]
;;	ystore	[srcreg], ymm0			;; Save R1
	ystore	[srcreg+32], ymm4		;; Save I1
	ystore	[srcreg+d1], ymm5		;; Save R2
	ystore	[srcreg+d1+32], ymm7		;; Save I2
	ystore	[srcreg+d2], ymm2		;; Save R3
	ystore	[srcreg+d2+32], ymm3		;; Save I3
	ystore	[srcreg+d2+d1], ymm1		;; Save R4
	ystore	[srcreg+d2+d1+32], ymm6		;; Save I4
	bump	srcreg, srcinc
	ENDM

;; 64-bit version

IFDEF X86_64

yr4_4cl_four_complex_with_mult_preload MACRO
	ENDM
yr4_4cl_four_complex_with_mult MACRO srcreg,srcinc,d1,d2
	LOCAL	orig, back_to_orig

	vmovapd	ymm0, [srcreg]			;; R1
	vmovapd	ymm7, [srcreg+d2]		;; R3
	vaddpd	ymm2, ymm0, ymm7		;; R1 + R3 (new R1)		; 1-3
	vsubpd	ymm0, ymm0, ymm7		;; R1 - R3 (new R3)		; 2-4

	vmovapd	ymm1, [srcreg+d1]		;; R2
	vmovapd	ymm7, [srcreg+d2+d1]		;; R4
	vaddpd	ymm3, ymm1, ymm7		;; R2 + R4 (new R2)		; 3-5
	vsubpd	ymm1, ymm1, ymm7		;; R2 - R4 (new R4)		; 4-6

	vmovapd	ymm4, [srcreg+32]		;; I1
	vmovapd	ymm7, [srcreg+d2+32]		;; I3
	vaddpd	ymm6, ymm4, ymm7		;; I1 + I3 (new I1)		; 5-7
	vsubpd	ymm4, ymm4, ymm7		;; I1 - I3 (new I3)		; 6-8

	vmovapd	ymm5, [srcreg+d1+32]		;; I2
	vmovapd	ymm8, [srcreg+d2+d1+32]		;; I4
	vaddpd	ymm7, ymm5, ymm8		;; I2 + I4 (new I2)		; 7-9
	vsubpd	ymm5, ymm5, ymm8		;; I2 - I4 (new I4)		; 8-10

	vaddpd	ymm8, ymm2, ymm3		;; R1 + R2 (final R1)		; 9-11
	vsubpd	ymm2, ymm2, ymm3		;; R1 - R2 (final R2)		; 10-12

	vaddpd	ymm3, ymm6, ymm7		;; I1 + I2 (final I1)		; 11-13
	vsubpd	ymm6, ymm6, ymm7		;; I1 - I2 (final I2)		; 12-14

	vsubpd	ymm7, ymm0, ymm5		;; R3 - I4 (final R3)		; 13-15
	vaddpd	ymm0, ymm0, ymm5		;; R3 + I4 (final R4)		; 14-16

	cmp	mul4_opcode, 0			;; See if we need to do more than the original type-3 FFT multiply
	je	short orig
	yr4_dispatch CALLP,ycomplex_mult_opcode,d1,d2 ;; Handle more difficult cases
	jmp	back_to_orig

orig:	vmovapd	ymm9, [srcreg][rbp]		;; MemR1
	vmovapd	ymm10, [srcreg+32][rbp]		;; MemI1
	vmulpd	ymm5, ymm8, ymm9		;; R1 * MemR1			; 12-16
	vmulpd	ymm8, ymm8, ymm10		;; R1 * MemI1			; 13-17
	vmulpd	ymm11, ymm3, ymm10		;; I1 * MemI1			; 14-18
	vmulpd	ymm3, ymm3, ymm9		;; I1 * MemR1			; 15-19

	vmovapd	ymm10, [srcreg+d1][rbp]		;; MemR2
	vmovapd	ymm12, [srcreg+d1+32][rbp]	;; MemI2

	vaddpd	ymm9, ymm4, ymm1		;; I3 + R4 (final I3)		; 15-17
	vsubpd	ymm4, ymm4, ymm1		;; I3 - R4 (final I4)		; 16-18

	vmulpd	ymm1, ymm2, ymm10		;; R2 * MemR2			; 16-20
	vmulpd	ymm2, ymm2, ymm12		;; R2 * MemI2			; 17-21
	vmulpd	ymm13, ymm6, ymm12		;; I2 * MemI2			; 18-22
	vmulpd	ymm6, ymm6, ymm10		;; I2 * MemR2			; 19-23

	vmovapd	ymm10, [srcreg+d2][rbp]		;; MemR3
	vmovapd	ymm12, [srcreg+d2+32][rbp]	;; MemI3

	vsubpd	ymm5, ymm5, ymm11		;; R1*MemR1-I1*MemI1 (new R1)	; 19-21
	vaddpd	ymm8, ymm8, ymm3		;; R1*MemI1+I1*MemR1 (new I1)	; 20-22

	vmulpd	ymm3, ymm7, ymm10		;; R3 * MemR3			; 20-24
	vmulpd	ymm7, ymm7, ymm12		;; R3 * MemI3			; 21-25
	vmulpd	ymm11, ymm9, ymm12		;; I3 * MemI3			; 22-26
	vmulpd	ymm9, ymm9, ymm10		;; I3 * MemR3			; 23-27

	vmovapd	ymm10, [srcreg+d2+d1][rbp]	;; MemR4
	vmovapd	ymm12, [srcreg+d2+d1+32][rbp]	;; MemI4

	vsubpd	ymm1, ymm1, ymm13		;; R2*MemR2-I2*MemI2 (new R2)	; 23-25
	vaddpd	ymm2, ymm2, ymm6		;; R2*MemI2+I2*MemR2 (new I2)	; 24-26

	vmulpd	ymm6, ymm0, ymm10		;; R4 * MemR4			; 24-28
	vmulpd	ymm0, ymm0, ymm12		;; R4 * MemI4			; 25-29
	vmulpd	ymm13, ymm4, ymm12		;; I4 * MemI4			; 26-30
	vmulpd	ymm4, ymm4, ymm10		;; I4 * MemR4			; 27-31

	vaddpd	ymm10, ymm5, ymm1		;; R1 + R2 (new R1)		; 26-28
	vsubpd	ymm5, ymm5, ymm1		;; R1 - R2 (new R2)		; 27-29

	vsubpd	ymm3, ymm3, ymm11		;; R3*MemR3-I3*MemI3 (new R3)	; 28-30
	vaddpd	ymm7, ymm7, ymm9		;; R3*MemI3+I3*MemR3 (new I3)	; 29-31

	vsubpd	ymm6, ymm6, ymm13		;; R4*MemR4-I4*MemI4 (new R4)	; 31-33
	vaddpd	ymm0, ymm0, ymm4		;; R4*MemI4+I4*MemR4 (new I4)	; 32-34

back_to_orig:
	vaddpd	ymm1, ymm8, ymm2		;; I1 + I2 (new I1)		; 30-32
	vsubpd	ymm8, ymm8, ymm2		;; I1 - I2 (new I2)		; 33-35

	vaddpd	ymm2, ymm6, ymm3		;; R4 + R3 (new R3)		; 34-36
	vsubpd	ymm6, ymm6, ymm3		;; R4 - R3 (new I4)		; 35-37

	vaddpd	ymm3, ymm7, ymm0		;; I3 + I4 (new I3)		; 36-38
	vsubpd	ymm7, ymm7, ymm0		;; I3 - I4 (new R4)		; 37-39

	vaddpd	ymm0, ymm10, ymm2		;; R1 + R3 (final R1)		; 38-40
	vsubpd	ymm10, ymm10, ymm2		;; R1 - R3 (final R3)		; 39-41

	vaddpd	ymm2, ymm8, ymm6		;; I2 + I4 (final I2)		; 40-42
	vsubpd	ymm8, ymm8, ymm6		;; I2 - I4 (final I4)		; 41-43

	vaddpd	ymm4, ymm1, ymm3		;; I1 + I3 (final I1)		; 42-44
	vsubpd	ymm1, ymm1, ymm3		;; I1 - I3 (final I3)		; 43-45

	vaddpd	ymm3, ymm5, ymm7		;; R2 + R4 (final R2)		; 44-46
	vsubpd	ymm5, ymm5, ymm7		;; R2 - R4 (final R4)		; 45-47

	ystore	[srcreg+r8], ymm0		;; Save R1
	ystore	[srcreg+r8+32], ymm4		;; Save I1
	ystore	[srcreg+r8+d1], ymm3		;; Save R2
	ystore	[srcreg+r8+d1+32], ymm2		;; Save I2
	ystore	[srcreg+r8+d2], ymm10		;; Save R3
	ystore	[srcreg+r8+d2+32], ymm1		;; Save I3
	ystore	[srcreg+r8+d2+d1], ymm5		;; Save R4
	ystore	[srcreg+r8+d2+d1+32], ymm8	;; Save I4
	bump	srcreg, srcinc
	ENDM

;; Same as above code between orig and back_to_orig, except we implement mul4_opcode options

yr4_4c_mult_opcode MACRO srcreg,d1,d2
	LOCAL	mulsave, fftsave, nofftsave, dispatch, disptab, addmul, submul, muladd, muladdhard, mulsub, mulsubhard, unfft, done
	LOCAL	addmul312, addmul313, addmul132, addmul133, submul312, submul313, submul132, submul133

	movzx	r10, mul4_opcode		;; Load opcode
	cmp	r10, 80h			;; Test for FFT save
	ja	fftsave				;; opcode > 0x80, addmul,submul,muladd,mulsub,unfft with FFT save
	jb	nofftsave			;; opcode < 0x80, addmul,submul,muladd,mulsub,unfft without FFT save
						;; opcode == 0x80, plain mul with FFT save, fall through

	;; Squaring with FFT save is handled here! 
	test	rbp, rbp			;; If rbp is zero, this is a squaring
	jnz	mulsave

	ystore	[srcreg], ymm8			;; R1
	vmulpd	ymm5, ymm8, ymm8		;; R1 * R1			; 12-16
	vmulpd	ymm8, ymm8, ymm3		;; R1 * I1			; 13-17
	ystore	[srcreg+32], ymm3		;; I1
	vmulpd	ymm3, ymm3, ymm3		;; I1 * I1			; 14-18

	vaddpd	ymm9, ymm4, ymm1		;; I3 + R4 (final I3)		; 15-17
	vsubpd	ymm4, ymm4, ymm1		;; I3 - R4 (final I4)		; 16-18

	ystore	[srcreg+d1], ymm2		;; R2
	vmulpd	ymm1, ymm2, ymm2		;; R2 * R2			; 16-20
	vmulpd	ymm2, ymm2, ymm6		;; R2 * I2			; 17-21
	ystore	[srcreg+d1+32], ymm6		;; I2
	vmulpd	ymm6, ymm6, ymm6		;; I2 * I2			; 18-22

	vsubpd	ymm5, ymm5, ymm3		;; R1*R1-I1*I1 (new R1)		; 19-21
	vaddpd	ymm8, ymm8, ymm8		;; 2*R1*I1 (new I1)		; 20-22

	ystore	[srcreg+d2], ymm7		;; R3
	vmulpd	ymm3, ymm7, ymm7		;; R3 * R3			; 20-24
	vmulpd	ymm7, ymm7, ymm9		;; R3 * I3			; 21-25
	ystore	[srcreg+d2+32], ymm9		;; I3
	vmulpd	ymm9, ymm9, ymm9		;; I3 * I3			; 22-26

	ystore	[srcreg+d2+d1], ymm0		;; R4
	ystore	[srcreg+d2+d1+32], ymm4		;; I4

	vsubpd	ymm1, ymm1, ymm6		;; R2*R2-I2*I2 (new R2)		; 23-25
	vaddpd	ymm2, ymm2, ymm2		;; 2*R2*I2 (new I2)		; 24-26

	vmulpd	ymm6, ymm0, ymm0		;; R4 * R4			; 24-28
	vmulpd	ymm0, ymm0, ymm4		;; R4 * I4			; 25-29
	vmulpd	ymm4, ymm4, ymm4		;; I4 * I4			; 26-30

	vaddpd	ymm10, ymm5, ymm1		;; R1 + R2 (new R1)		; 26-28
	vsubpd	ymm5, ymm5, ymm1		;; R1 - R2 (new R2)		; 27-29

	vsubpd	ymm3, ymm3, ymm9		;; R3*R3-I3*I3 (new R3)		; 28-30
	vaddpd	ymm7, ymm7, ymm7		;; 2*R3*I3 (new I3)		; 29-31

	vsubpd	ymm6, ymm6, ymm4		;; R4*R4-I4*I4 (new R4)		; 31-33
	vaddpd	ymm0, ymm0, ymm0		;; 2*R4*I4 (new I4)		; 32-34
	jmp	done

mulsave:vmovapd	ymm9, [srcreg][rbp]		;; MemR1
	vmovapd	ymm10, [srcreg+32][rbp]		;; MemI1
	ystore	[srcreg], ymm8			;; R1
	vmulpd	ymm5, ymm8, ymm9		;; R1 * MemR1			; 12-16
	vmulpd	ymm8, ymm8, ymm10		;; R1 * MemI1			; 13-17
	ystore	[srcreg+32], ymm3		;; I1
	vmulpd	ymm11, ymm3, ymm10		;; I1 * MemI1			; 14-18
	vmulpd	ymm3, ymm3, ymm9		;; I1 * MemR1			; 15-19

	vmovapd	ymm10, [srcreg+d1][rbp]		;; MemR2
	vmovapd	ymm12, [srcreg+d1+32][rbp]	;; MemI2

	vaddpd	ymm9, ymm4, ymm1		;; I3 + R4 (final I3)		; 15-17
	vsubpd	ymm4, ymm4, ymm1		;; I3 - R4 (final I4)		; 16-18

	ystore	[srcreg+d1], ymm2		;; R2
	vmulpd	ymm1, ymm2, ymm10		;; R2 * MemR2			; 16-20
	vmulpd	ymm2, ymm2, ymm12		;; R2 * MemI2			; 17-21
	ystore	[srcreg+d1+32], ymm6		;; I2
	vmulpd	ymm13, ymm6, ymm12		;; I2 * MemI2			; 18-22
	vmulpd	ymm6, ymm6, ymm10		;; I2 * MemR2			; 19-23

	vmovapd	ymm10, [srcreg+d2][rbp]		;; MemR3
	vmovapd	ymm12, [srcreg+d2+32][rbp]	;; MemI3

	vsubpd	ymm5, ymm5, ymm11		;; R1*MemR1-I1*MemI1 (new R1)	; 19-21
	vaddpd	ymm8, ymm8, ymm3		;; R1*MemI1+I1*MemR1 (new I1)	; 20-22

	ystore	[srcreg+d2], ymm7		;; R3
	vmulpd	ymm3, ymm7, ymm10		;; R3 * MemR3			; 20-24
	vmulpd	ymm7, ymm7, ymm12		;; R3 * MemI3			; 21-25
	ystore	[srcreg+d2+32], ymm9		;; I3
	vmulpd	ymm11, ymm9, ymm12		;; I3 * MemI3			; 22-26
	vmulpd	ymm9, ymm9, ymm10		;; I3 * MemR3			; 23-27

	vmovapd	ymm10, [srcreg+d2+d1][rbp]	;; MemR4
	vmovapd	ymm12, [srcreg+d2+d1+32][rbp]	;; MemI4

	ystore	[srcreg+d2+d1], ymm0		;; R4
	ystore	[srcreg+d2+d1+32], ymm4		;; I4

	vsubpd	ymm1, ymm1, ymm13		;; R2*MemR2-I2*MemI2 (new R2)	; 23-25
	vaddpd	ymm2, ymm2, ymm6		;; R2*MemI2+I2*MemR2 (new I2)	; 24-26

	vmulpd	ymm6, ymm0, ymm10		;; R4 * MemR4			; 24-28
	vmulpd	ymm0, ymm0, ymm12		;; R4 * MemI4			; 25-29
	vmulpd	ymm13, ymm4, ymm12		;; I4 * MemI4			; 26-30
	vmulpd	ymm4, ymm4, ymm10		;; I4 * MemR4			; 27-31

	vaddpd	ymm10, ymm5, ymm1		;; R1 + R2 (new R1)		; 26-28
	vsubpd	ymm5, ymm5, ymm1		;; R1 - R2 (new R2)		; 27-29

	vsubpd	ymm3, ymm3, ymm11		;; R3*MemR3-I3*MemI3 (new R3)	; 28-30
	vaddpd	ymm7, ymm7, ymm9		;; R3*MemI3+I3*MemR3 (new I3)	; 29-31

	vsubpd	ymm6, ymm6, ymm13		;; R4*MemR4-I4*MemI4 (new R4)	; 31-33
	vaddpd	ymm0, ymm0, ymm4		;; R4*MemI4+I4*MemR4 (new I4)	; 32-34
	jmp	done

fftsave:
	ystore	[srcreg], ymm8			;; R1
	ystore	[srcreg+32], ymm3		;; I1
	vaddpd	ymm9, ymm4, ymm1		;; I3 + R4 (final I3)		; 15-17
	vsubpd	ymm4, ymm4, ymm1		;; I3 - R4 (final I4)		; 16-18
	ystore	[srcreg+d1], ymm2		;; R2
	ystore	[srcreg+d1+32], ymm6		;; I2
	ystore	[srcreg+d2], ymm7		;; R3
	ystore	[srcreg+d2+32], ymm9		;; I3
	ystore	[srcreg+d2+d1], ymm0		;; R4
	ystore	[srcreg+d2+d1+32], ymm4		;; I4
	and	r10, 7fh			;; Strip the fft save bit from mul4_opcode
	jmp	short dispatch			

nofftsave:
	vaddpd	ymm9, ymm4, ymm1		;; I3 + R4 (final I3)		; 15-17
	vsubpd	ymm4, ymm4, ymm1		;; I3 - R4 (final I4)		; 16-18

dispatch:
	mov	r9, OFFSET disptab		;; Jump table address
	mov	r10, [r9+r10*8-8]		;; Jump table offset
	mov	r9, SRC2ARG			;; Distance to s2 arg from gwaddmul4 or gwsubmul4 or s3 arg in gwmuladd4 or gwmulsub4
	jmp	r10
disptab:
	DQ	OFFSET addmul			;; 1 = addmul
	DQ	OFFSET submul			;; 2 = submul
	DQ	OFFSET muladd			;; 3 = muladd
	DQ	OFFSET mulsub			;; 4 = mulsub
	DQ	OFFSET unfft			;; 5 = unfft
	DQ	OFFSET addmul312		;; 6 = addmul FFTval mem1 mem2
	DQ	OFFSET addmul313		;; 7 = addmul FFTval mem1 FFTval
	DQ	OFFSET addmul132		;; 8 = addmul mem1 FFTval mem2
	DQ	OFFSET addmul133		;; 9 = addmul mem1 FFTval FFTval
	DQ	OFFSET submul312		;; 10 = submul FFTval mem1 mem2
	DQ	OFFSET submul313		;; 11 = submul FFTval mem1 FFTval
	DQ	OFFSET submul132		;; 12 = submul mem1 FFTval mem2
	DQ	OFFSET submul133		;; 13 = submul mem1 FFTval FFTval

addmul:	vmovapd	ymm1, [srcreg][rbp]		;; MemR1
	vaddpd	ymm1, ymm1, [srcreg][r9]	;; MemR1
	vmovapd	ymm10, [srcreg+32][rbp]		;; MemI1
	vaddpd	ymm10, ymm10, [srcreg+32][r9]	;; MemI1
	vmulpd	ymm5, ymm8, ymm1		;; R1 * MemR1			; 12-16
	vmulpd	ymm8, ymm8, ymm10		;; R1 * MemI1			; 13-17
	vmulpd	ymm11, ymm3, ymm10		;; I1 * MemI1			; 14-18
	vmulpd	ymm3, ymm3, ymm1		;; I1 * MemR1			; 15-19

	vmovapd	ymm10, [srcreg+d1][rbp]		;; MemR2
	vaddpd	ymm10, ymm10, [srcreg+d1][r9]	;; MemR2
	vmovapd	ymm12, [srcreg+d1+32][rbp]	;; MemI2
	vaddpd	ymm12, ymm12, [srcreg+d1+32][r9];; MemI2

	vmulpd	ymm1, ymm2, ymm10		;; R2 * MemR2			; 16-20
	vmulpd	ymm2, ymm2, ymm12		;; R2 * MemI2			; 17-21
	vmulpd	ymm13, ymm6, ymm12		;; I2 * MemI2			; 18-22
	vmulpd	ymm6, ymm6, ymm10		;; I2 * MemR2			; 19-23

	vmovapd	ymm10, [srcreg+d2][rbp]		;; MemR3
	vaddpd	ymm10, ymm10, [srcreg+d2][r9]	;; MemR3
	vmovapd	ymm12, [srcreg+d2+32][rbp]	;; MemI3
	vaddpd	ymm12, ymm12, [srcreg+d2+32][r9];; MemI3

	vsubpd	ymm5, ymm5, ymm11		;; R1*MemR1-I1*MemI1 (new R1)	; 19-21
	vaddpd	ymm8, ymm8, ymm3		;; R1*MemI1+I1*MemR1 (new I1)	; 20-22

	vmulpd	ymm3, ymm7, ymm10		;; R3 * MemR3			; 20-24
	vmulpd	ymm7, ymm7, ymm12		;; R3 * MemI3			; 21-25
	vmulpd	ymm11, ymm9, ymm12		;; I3 * MemI3			; 22-26
	vmulpd	ymm9, ymm9, ymm10		;; I3 * MemR3			; 23-27

	vmovapd	ymm10, [srcreg+d2+d1][rbp]	;; MemR4
	vaddpd	ymm10, ymm10, [srcreg+d2+d1][r9];; MemR4
	vmovapd	ymm12, [srcreg+d2+d1+32][rbp]	;; MemI4
	vaddpd	ymm12, ymm12, [srcreg+d2+d1+32][r9];; MemI4

	vsubpd	ymm1, ymm1, ymm13		;; R2*MemR2-I2*MemI2 (new R2)	; 23-25
	vaddpd	ymm2, ymm2, ymm6		;; R2*MemI2+I2*MemR2 (new I2)	; 24-26

	vmulpd	ymm6, ymm0, ymm10		;; R4 * MemR4			; 24-28
	vmulpd	ymm0, ymm0, ymm12		;; R4 * MemI4			; 25-29
	vmulpd	ymm13, ymm4, ymm12		;; I4 * MemI4			; 26-30
	vmulpd	ymm4, ymm4, ymm10		;; I4 * MemR4			; 27-31

	vaddpd	ymm10, ymm5, ymm1		;; R1 + R2 (new R1)		; 26-28
	vsubpd	ymm5, ymm5, ymm1		;; R1 - R2 (new R2)		; 27-29

	vsubpd	ymm3, ymm3, ymm11		;; R3*MemR3-I3*MemI3 (new R3)	; 28-30
	vaddpd	ymm7, ymm7, ymm9		;; R3*MemI3+I3*MemR3 (new I3)	; 29-31

	vsubpd	ymm6, ymm6, ymm13		;; R4*MemR4-I4*MemI4 (new R4)	; 31-33
	vaddpd	ymm0, ymm0, ymm4		;; R4*MemI4+I4*MemR4 (new I4)	; 32-34
	jmp	done

addmul133:
addmul313:
	vaddpd	ymm1, ymm8, [srcreg][rbp]	;; R1+MemR1
	vaddpd	ymm10, ymm3, [srcreg+32][rbp]	;; I1+MemI1
	vmulpd	ymm5, ymm8, ymm1		;; R1 * MemR1			; 12-16
	vmulpd	ymm8, ymm8, ymm10		;; R1 * MemI1			; 13-17
	vmulpd	ymm11, ymm3, ymm10		;; I1 * MemI1			; 14-18
	vmulpd	ymm3, ymm3, ymm1		;; I1 * MemR1			; 15-19

	vaddpd	ymm10, ymm2, [srcreg+d1][rbp]	;; R2+MemR2
	vaddpd	ymm12, ymm6, [srcreg+d1+32][rbp];; I2+MemI2
	vmulpd	ymm1, ymm2, ymm10		;; R2 * MemR2			; 16-20
	vmulpd	ymm2, ymm2, ymm12		;; R2 * MemI2			; 17-21
	vmulpd	ymm13, ymm6, ymm12		;; I2 * MemI2			; 18-22
	vmulpd	ymm6, ymm6, ymm10		;; I2 * MemR2			; 19-23

	vaddpd	ymm10, ymm7, [srcreg+d2][rbp]	;; R3+MemR3
	vaddpd	ymm12, ymm9, [srcreg+d2+32][rbp];; I3+MemI3

	vsubpd	ymm5, ymm5, ymm11		;; R1*MemR1-I1*MemI1 (new R1)	; 19-21
	vaddpd	ymm8, ymm8, ymm3		;; R1*MemI1+I1*MemR1 (new I1)	; 20-22

	vmulpd	ymm3, ymm7, ymm10		;; R3 * MemR3			; 20-24
	vmulpd	ymm7, ymm7, ymm12		;; R3 * MemI3			; 21-25
	vmulpd	ymm11, ymm9, ymm12		;; I3 * MemI3			; 22-26
	vmulpd	ymm9, ymm9, ymm10		;; I3 * MemR3			; 23-27

	vaddpd	ymm10, ymm0, [srcreg+d2+d1][rbp];; R4+MemR4
	vaddpd	ymm12, ymm4, [srcreg+d2+d1+32][rbp];; I4+MemI4

	vsubpd	ymm1, ymm1, ymm13		;; R2*MemR2-I2*MemI2 (new R2)	; 23-25
	vaddpd	ymm2, ymm2, ymm6		;; R2*MemI2+I2*MemR2 (new I2)	; 24-26

	vmulpd	ymm6, ymm0, ymm10		;; R4 * MemR4			; 24-28
	vmulpd	ymm0, ymm0, ymm12		;; R4 * MemI4			; 25-29
	vmulpd	ymm13, ymm4, ymm12		;; I4 * MemI4			; 26-30
	vmulpd	ymm4, ymm4, ymm10		;; I4 * MemR4			; 27-31

	vaddpd	ymm10, ymm5, ymm1		;; R1 + R2 (new R1)		; 26-28
	vsubpd	ymm5, ymm5, ymm1		;; R1 - R2 (new R2)		; 27-29

	vsubpd	ymm3, ymm3, ymm11		;; R3*MemR3-I3*MemI3 (new R3)	; 28-30
	vaddpd	ymm7, ymm7, ymm9		;; R3*MemI3+I3*MemR3 (new I3)	; 29-31

	vsubpd	ymm6, ymm6, ymm13		;; R4*MemR4-I4*MemI4 (new R4)	; 31-33
	vaddpd	ymm0, ymm0, ymm4		;; R4*MemI4+I4*MemR4 (new I4)	; 32-34
	jmp	done

addmul132:
addmul312:
	vaddpd	ymm1, ymm8, [srcreg][rbp]	;; R1+MemR1
	vmovapd	ymm8, [srcreg][r9]		;; R1=MemR1
	vaddpd	ymm10, ymm3, [srcreg+32][rbp]	;; I1+MemI1
	vmovapd	ymm3, [srcreg+32][r9]		;; I1=MemI1
	vmulpd	ymm5, ymm8, ymm1		;; R1 * MemR1			; 12-16
	vmulpd	ymm8, ymm8, ymm10		;; R1 * MemI1			; 13-17
	vmulpd	ymm11, ymm3, ymm10		;; I1 * MemI1			; 14-18
	vmulpd	ymm3, ymm3, ymm1		;; I1 * MemR1			; 15-19

	vaddpd	ymm10, ymm2, [srcreg+d1][rbp]	;; R2+MemR2
	vmovapd	ymm2, [srcreg+d1][r9]		;; R2=MemR2
	vaddpd	ymm12, ymm6, [srcreg+d1+32][rbp];; I2+MemI2
	vmovapd	ymm6, [srcreg+d1+32][r9]	;; I2=MemI2
	vmulpd	ymm1, ymm2, ymm10		;; R2 * MemR2			; 16-20
	vmulpd	ymm2, ymm2, ymm12		;; R2 * MemI2			; 17-21
	vmulpd	ymm13, ymm6, ymm12		;; I2 * MemI2			; 18-22
	vmulpd	ymm6, ymm6, ymm10		;; I2 * MemR2			; 19-23

	vaddpd	ymm10, ymm7, [srcreg+d2][rbp]	;; R3+MemR3
	vmovapd	ymm7, [srcreg+d2][r9]		;; R3=MemR3
	vaddpd	ymm12, ymm9, [srcreg+d2+32][rbp];; I3+MemI3
	vmovapd	ymm9, [srcreg+d2+32][r9]	;; I3=MemI3

	vsubpd	ymm5, ymm5, ymm11		;; R1*MemR1-I1*MemI1 (new R1)	; 19-21
	vaddpd	ymm8, ymm8, ymm3		;; R1*MemI1+I1*MemR1 (new I1)	; 20-22

	vmulpd	ymm3, ymm7, ymm10		;; R3 * MemR3			; 20-24
	vmulpd	ymm7, ymm7, ymm12		;; R3 * MemI3			; 21-25
	vmulpd	ymm11, ymm9, ymm12		;; I3 * MemI3			; 22-26
	vmulpd	ymm9, ymm9, ymm10		;; I3 * MemR3			; 23-27

	vaddpd	ymm10, ymm0, [srcreg+d2+d1][rbp];; R4+MemR4
	vmovapd	ymm0, [srcreg+d2+d1][r9]	;; R4=MemR4
	vaddpd	ymm12, ymm4, [srcreg+d2+d1+32][rbp];; I4+MemI4
	vmovapd	ymm4, [srcreg+d2+d1+32][r9]	;; I4=MemI4

	vsubpd	ymm1, ymm1, ymm13		;; R2*MemR2-I2*MemI2 (new R2)	; 23-25
	vaddpd	ymm2, ymm2, ymm6		;; R2*MemI2+I2*MemR2 (new I2)	; 24-26

	vmulpd	ymm6, ymm0, ymm10		;; R4 * MemR4			; 24-28
	vmulpd	ymm0, ymm0, ymm12		;; R4 * MemI4			; 25-29
	vmulpd	ymm13, ymm4, ymm12		;; I4 * MemI4			; 26-30
	vmulpd	ymm4, ymm4, ymm10		;; I4 * MemR4			; 27-31

	vaddpd	ymm10, ymm5, ymm1		;; R1 + R2 (new R1)		; 26-28
	vsubpd	ymm5, ymm5, ymm1		;; R1 - R2 (new R2)		; 27-29

	vsubpd	ymm3, ymm3, ymm11		;; R3*MemR3-I3*MemI3 (new R3)	; 28-30
	vaddpd	ymm7, ymm7, ymm9		;; R3*MemI3+I3*MemR3 (new I3)	; 29-31

	vsubpd	ymm6, ymm6, ymm13		;; R4*MemR4-I4*MemI4 (new R4)	; 31-33
	vaddpd	ymm0, ymm0, ymm4		;; R4*MemI4+I4*MemR4 (new I4)	; 32-34
	jmp	done

submul:	vmovapd	ymm1, [srcreg][rbp]		;; MemR1
	vsubpd	ymm1, ymm1, [srcreg][r9]	;; MemR1
	vmovapd	ymm10, [srcreg+32][rbp]		;; MemI1
	vsubpd	ymm10, ymm10, [srcreg+32][r9]	;; MemR1
	vmulpd	ymm5, ymm8, ymm1		;; R1 * MemR1			; 12-16
	vmulpd	ymm8, ymm8, ymm10		;; R1 * MemI1			; 13-17
	vmulpd	ymm11, ymm3, ymm10		;; I1 * MemI1			; 14-18
	vmulpd	ymm3, ymm3, ymm1		;; I1 * MemR1			; 15-19

	vmovapd	ymm10, [srcreg+d1][rbp]		;; MemR2
	vsubpd	ymm10, ymm10, [srcreg+d1][r9]	;; MemR2
	vmovapd	ymm12, [srcreg+d1+32][rbp]	;; MemI2
	vsubpd	ymm12, ymm12, [srcreg+d1+32][r9];; MemI2

	vmulpd	ymm1, ymm2, ymm10		;; R2 * MemR2			; 16-20
	vmulpd	ymm2, ymm2, ymm12		;; R2 * MemI2			; 17-21
	vmulpd	ymm13, ymm6, ymm12		;; I2 * MemI2			; 18-22
	vmulpd	ymm6, ymm6, ymm10		;; I2 * MemR2			; 19-23

	vmovapd	ymm10, [srcreg+d2][rbp]		;; MemR3
	vsubpd	ymm10, ymm10, [srcreg+d2][r9]	;; MemR3
	vmovapd	ymm12, [srcreg+d2+32][rbp]	;; MemI3
	vsubpd	ymm12, ymm12, [srcreg+d2+32][r9];; MemI3

	vsubpd	ymm5, ymm5, ymm11		;; R1*MemR1-I1*MemI1 (new R1)	; 19-21
	vaddpd	ymm8, ymm8, ymm3		;; R1*MemI1+I1*MemR1 (new I1)	; 20-22

	vmulpd	ymm3, ymm7, ymm10		;; R3 * MemR3			; 20-24
	vmulpd	ymm7, ymm7, ymm12		;; R3 * MemI3			; 21-25
	vmulpd	ymm11, ymm9, ymm12		;; I3 * MemI3			; 22-26
	vmulpd	ymm9, ymm9, ymm10		;; I3 * MemR3			; 23-27

	vmovapd	ymm10, [srcreg+d2+d1][rbp]	;; MemR4
	vsubpd	ymm10, ymm10, [srcreg+d2+d1][r9];; MemR4
	vmovapd	ymm12, [srcreg+d2+d1+32][rbp]	;; MemI4
	vsubpd	ymm12, ymm12, [srcreg+d2+d1+32][r9];; MemI4

	vsubpd	ymm1, ymm1, ymm13		;; R2*MemR2-I2*MemI2 (new R2)	; 23-25
	vaddpd	ymm2, ymm2, ymm6		;; R2*MemI2+I2*MemR2 (new I2)	; 24-26

	vmulpd	ymm6, ymm0, ymm10		;; R4 * MemR4			; 24-28
	vmulpd	ymm0, ymm0, ymm12		;; R4 * MemI4			; 25-29
	vmulpd	ymm13, ymm4, ymm12		;; I4 * MemI4			; 26-30
	vmulpd	ymm4, ymm4, ymm10		;; I4 * MemR4			; 27-31

	vaddpd	ymm10, ymm5, ymm1		;; R1 + R2 (new R1)		; 26-28
	vsubpd	ymm5, ymm5, ymm1		;; R1 - R2 (new R2)		; 27-29

	vsubpd	ymm3, ymm3, ymm11		;; R3*MemR3-I3*MemI3 (new R3)	; 28-30
	vaddpd	ymm7, ymm7, ymm9		;; R3*MemI3+I3*MemR3 (new I3)	; 29-31

	vsubpd	ymm6, ymm6, ymm13		;; R4*MemR4-I4*MemI4 (new R4)	; 31-33
	vaddpd	ymm0, ymm0, ymm4		;; R4*MemI4+I4*MemR4 (new I4)	; 32-34
	jmp	done

submul133:
	vmovapd	ymm1, [srcreg][rbp]		;; MemR1
	vsubpd	ymm1, ymm1, ymm8		;; MemR1-R1
	vmovapd	ymm10, [srcreg+32][rbp]		;; MemI1
	vsubpd	ymm10, ymm10, ymm3		;; MemI1-I1
	vmulpd	ymm5, ymm8, ymm1		;; R1 * MemR1			; 12-16
	vmulpd	ymm8, ymm8, ymm10		;; R1 * MemI1			; 13-17
	vmulpd	ymm11, ymm3, ymm10		;; I1 * MemI1			; 14-18
	vmulpd	ymm3, ymm3, ymm1		;; I1 * MemR1			; 15-19

	vmovapd	ymm10, [srcreg+d1][rbp]		;; MemR2
	vsubpd	ymm10, ymm10, ymm2		;; MemR2-R2
	vmovapd	ymm12, [srcreg+d1+32][rbp]	;; MemI2
	vsubpd	ymm12, ymm12, ymm6		;; MemI2-I2
	vmulpd	ymm1, ymm2, ymm10		;; R2 * MemR2			; 16-20
	vmulpd	ymm2, ymm2, ymm12		;; R2 * MemI2			; 17-21
	vmulpd	ymm13, ymm6, ymm12		;; I2 * MemI2			; 18-22
	vmulpd	ymm6, ymm6, ymm10		;; I2 * MemR2			; 19-23

	vmovapd	ymm10, [srcreg+d2][rbp]		;; MemR3
	vsubpd	ymm10, ymm10, ymm7		;; MemR3-R3
	vmovapd	ymm12, [srcreg+d2+32][rbp]	;; MemI3
	vsubpd	ymm12, ymm12, ymm9		;; MemI3-I3

	vsubpd	ymm5, ymm5, ymm11		;; R1*MemR1-I1*MemI1 (new R1)	; 19-21
	vaddpd	ymm8, ymm8, ymm3		;; R1*MemI1+I1*MemR1 (new I1)	; 20-22

	vmulpd	ymm3, ymm7, ymm10		;; R3 * MemR3			; 20-24
	vmulpd	ymm7, ymm7, ymm12		;; R3 * MemI3			; 21-25
	vmulpd	ymm11, ymm9, ymm12		;; I3 * MemI3			; 22-26
	vmulpd	ymm9, ymm9, ymm10		;; I3 * MemR3			; 23-27

	vmovapd	ymm10, [srcreg+d2+d1][rbp]	;; MemR4
	vsubpd	ymm10, ymm10, ymm0		;; MemR4-R4
	vmovapd	ymm12, [srcreg+d2+d1+32][rbp]	;; MemI4
	vsubpd	ymm12, ymm12, ymm4		;; MemI4-I4

	vsubpd	ymm1, ymm1, ymm13		;; R2*MemR2-I2*MemI2 (new R2)	; 23-25
	vaddpd	ymm2, ymm2, ymm6		;; R2*MemI2+I2*MemR2 (new I2)	; 24-26

	vmulpd	ymm6, ymm0, ymm10		;; R4 * MemR4			; 24-28
	vmulpd	ymm0, ymm0, ymm12		;; R4 * MemI4			; 25-29
	vmulpd	ymm13, ymm4, ymm12		;; I4 * MemI4			; 26-30
	vmulpd	ymm4, ymm4, ymm10		;; I4 * MemR4			; 27-31

	vaddpd	ymm10, ymm5, ymm1		;; R1 + R2 (new R1)		; 26-28
	vsubpd	ymm5, ymm5, ymm1		;; R1 - R2 (new R2)		; 27-29

	vsubpd	ymm3, ymm3, ymm11		;; R3*MemR3-I3*MemI3 (new R3)	; 28-30
	vaddpd	ymm7, ymm7, ymm9		;; R3*MemI3+I3*MemR3 (new I3)	; 29-31

	vsubpd	ymm6, ymm6, ymm13		;; R4*MemR4-I4*MemI4 (new R4)	; 31-33
	vaddpd	ymm0, ymm0, ymm4		;; R4*MemI4+I4*MemR4 (new I4)	; 32-34
	jmp	done

submul313:
	vsubpd	ymm1, ymm8, [srcreg][rbp]	;; R1-MemR1
	vsubpd	ymm10, ymm3, [srcreg+32][rbp]	;; I1-MemI1
	vmulpd	ymm5, ymm8, ymm1		;; R1 * MemR1			; 12-16
	vmulpd	ymm8, ymm8, ymm10		;; R1 * MemI1			; 13-17
	vmulpd	ymm11, ymm3, ymm10		;; I1 * MemI1			; 14-18
	vmulpd	ymm3, ymm3, ymm1		;; I1 * MemR1			; 15-19

	vsubpd	ymm10, ymm2, [srcreg+d1][rbp]	;; R2-MemR2
	vsubpd	ymm12, ymm6, [srcreg+d1+32][rbp];; I2-MemI2
	vmulpd	ymm1, ymm2, ymm10		;; R2 * MemR2			; 16-20
	vmulpd	ymm2, ymm2, ymm12		;; R2 * MemI2			; 17-21
	vmulpd	ymm13, ymm6, ymm12		;; I2 * MemI2			; 18-22
	vmulpd	ymm6, ymm6, ymm10		;; I2 * MemR2			; 19-23

	vsubpd	ymm10, ymm7, [srcreg+d2][rbp]	;; R3-MemR3
	vsubpd	ymm12, ymm9, [srcreg+d2+32][rbp];; I3-MemI3

	vsubpd	ymm5, ymm5, ymm11		;; R1*MemR1-I1*MemI1 (new R1)	; 19-21
	vaddpd	ymm8, ymm8, ymm3		;; R1*MemI1+I1*MemR1 (new I1)	; 20-22

	vmulpd	ymm3, ymm7, ymm10		;; R3 * MemR3			; 20-24
	vmulpd	ymm7, ymm7, ymm12		;; R3 * MemI3			; 21-25
	vmulpd	ymm11, ymm9, ymm12		;; I3 * MemI3			; 22-26
	vmulpd	ymm9, ymm9, ymm10		;; I3 * MemR3			; 23-27

	vsubpd	ymm10, ymm0, [srcreg+d2+d1][rbp];; R4-MemR4
	vsubpd	ymm12, ymm4, [srcreg+d2+d1+32][rbp];; I4-MemI4

	vsubpd	ymm1, ymm1, ymm13		;; R2*MemR2-I2*MemI2 (new R2)	; 23-25
	vaddpd	ymm2, ymm2, ymm6		;; R2*MemI2+I2*MemR2 (new I2)	; 24-26

	vmulpd	ymm6, ymm0, ymm10		;; R4 * MemR4			; 24-28
	vmulpd	ymm0, ymm0, ymm12		;; R4 * MemI4			; 25-29
	vmulpd	ymm13, ymm4, ymm12		;; I4 * MemI4			; 26-30
	vmulpd	ymm4, ymm4, ymm10		;; I4 * MemR4			; 27-31

	vaddpd	ymm10, ymm5, ymm1		;; R1 + R2 (new R1)		; 26-28
	vsubpd	ymm5, ymm5, ymm1		;; R1 - R2 (new R2)		; 27-29

	vsubpd	ymm3, ymm3, ymm11		;; R3*MemR3-I3*MemI3 (new R3)	; 28-30
	vaddpd	ymm7, ymm7, ymm9		;; R3*MemI3+I3*MemR3 (new I3)	; 29-31

	vsubpd	ymm6, ymm6, ymm13		;; R4*MemR4-I4*MemI4 (new R4)	; 31-33
	vaddpd	ymm0, ymm0, ymm4		;; R4*MemI4+I4*MemR4 (new I4)	; 32-34
	jmp	done

submul132:
	vmovapd	ymm1, [srcreg][rbp]		;; MemR1
	vsubpd	ymm1, ymm1, ymm8		;; MemR1-R1
	vmovapd	ymm8, [srcreg][r9]		;; R1=MemR1
	vmovapd	ymm10, [srcreg+32][rbp]		;; MemI1
	vsubpd	ymm10, ymm10, ymm3		;; MemI1-I1
	vmovapd	ymm3, [srcreg+32][r9]		;; I1=MemI1
	vmulpd	ymm5, ymm8, ymm1		;; R1 * MemR1			; 12-16
	vmulpd	ymm8, ymm8, ymm10		;; R1 * MemI1			; 13-17
	vmulpd	ymm11, ymm3, ymm10		;; I1 * MemI1			; 14-18
	vmulpd	ymm3, ymm3, ymm1		;; I1 * MemR1			; 15-19

	vmovapd	ymm10, [srcreg+d1][rbp]		;; MemR2
	vsubpd	ymm10, ymm10, ymm2		;; MemR2-R2
	vmovapd	ymm2, [srcreg+d1][r9]		;; R2=MemR2
	vmovapd	ymm12, [srcreg+d1+32][rbp]	;; MemI2
	vsubpd	ymm12, ymm12, ymm6		;; MemI2-I2
	vmovapd	ymm6, [srcreg+d1+32][r9]	;; I2=MemI2
	vmulpd	ymm1, ymm2, ymm10		;; R2 * MemR2			; 16-20
	vmulpd	ymm2, ymm2, ymm12		;; R2 * MemI2			; 17-21
	vmulpd	ymm13, ymm6, ymm12		;; I2 * MemI2			; 18-22
	vmulpd	ymm6, ymm6, ymm10		;; I2 * MemR2			; 19-23

	vmovapd	ymm10, [srcreg+d2][rbp]		;; MemR3
	vsubpd	ymm10, ymm10, ymm7		;; MemR3-R3
	vmovapd	ymm7, [srcreg+d2][r9]		;; R3=MemR3
	vmovapd	ymm12, [srcreg+d2+32][rbp]	;; MemI3
	vsubpd	ymm12, ymm12, ymm9		;; MemI3-I3
	vmovapd	ymm9, [srcreg+d2+32][r9]	;; I3=MemI3

	vsubpd	ymm5, ymm5, ymm11		;; R1*MemR1-I1*MemI1 (new R1)	; 19-21
	vaddpd	ymm8, ymm8, ymm3		;; R1*MemI1+I1*MemR1 (new I1)	; 20-22

	vmulpd	ymm3, ymm7, ymm10		;; R3 * MemR3			; 20-24
	vmulpd	ymm7, ymm7, ymm12		;; R3 * MemI3			; 21-25
	vmulpd	ymm11, ymm9, ymm12		;; I3 * MemI3			; 22-26
	vmulpd	ymm9, ymm9, ymm10		;; I3 * MemR3			; 23-27

	vmovapd	ymm10, [srcreg+d2+d1][rbp]	;; MemR4
	vsubpd	ymm10, ymm10, ymm0		;; MemR4-R4
	vmovapd	ymm0, [srcreg+d2+d1][r9]	;; R4=MemR4
	vmovapd	ymm12, [srcreg+d2+d1+32][rbp]	;; MemI4
	vsubpd	ymm12, ymm12, ymm4		;; MemI4-I4
	vmovapd	ymm4, [srcreg+d2+d1+32][r9]	;; I4=MemI4

	vsubpd	ymm1, ymm1, ymm13		;; R2*MemR2-I2*MemI2 (new R2)	; 23-25
	vaddpd	ymm2, ymm2, ymm6		;; R2*MemI2+I2*MemR2 (new I2)	; 24-26

	vmulpd	ymm6, ymm0, ymm10		;; R4 * MemR4			; 24-28
	vmulpd	ymm0, ymm0, ymm12		;; R4 * MemI4			; 25-29
	vmulpd	ymm13, ymm4, ymm12		;; I4 * MemI4			; 26-30
	vmulpd	ymm4, ymm4, ymm10		;; I4 * MemR4			; 27-31

	vaddpd	ymm10, ymm5, ymm1		;; R1 + R2 (new R1)		; 26-28
	vsubpd	ymm5, ymm5, ymm1		;; R1 - R2 (new R2)		; 27-29

	vsubpd	ymm3, ymm3, ymm11		;; R3*MemR3-I3*MemI3 (new R3)	; 28-30
	vaddpd	ymm7, ymm7, ymm9		;; R3*MemI3+I3*MemR3 (new I3)	; 29-31

	vsubpd	ymm6, ymm6, ymm13		;; R4*MemR4-I4*MemI4 (new R4)	; 31-33
	vaddpd	ymm0, ymm0, ymm4		;; R4*MemI4+I4*MemR4 (new I4)	; 32-34
	jmp	done

submul312:
	vsubpd	ymm1, ymm8, [srcreg][rbp]	;; R1-MemR1
	vmovapd	ymm8, [srcreg][r9]		;; R1=MemR1
	vsubpd	ymm10, ymm3, [srcreg+32][rbp]	;; I1-MemI1
	vmovapd	ymm3, [srcreg+32][r9]		;; I1=MemI1
	vmulpd	ymm5, ymm8, ymm1		;; R1 * MemR1			; 12-16
	vmulpd	ymm8, ymm8, ymm10		;; R1 * MemI1			; 13-17
	vmulpd	ymm11, ymm3, ymm10		;; I1 * MemI1			; 14-18
	vmulpd	ymm3, ymm3, ymm1		;; I1 * MemR1			; 15-19

	vsubpd	ymm10, ymm2, [srcreg+d1][rbp]	;; R2-MemR2
	vmovapd	ymm2, [srcreg+d1][r9]		;; R2=MemR2
	vsubpd	ymm12, ymm6, [srcreg+d1+32][rbp];; I2-MemI2
	vmovapd	ymm6, [srcreg+d1+32][r9]	;; I2=MemI2
	vmulpd	ymm1, ymm2, ymm10		;; R2 * MemR2			; 16-20
	vmulpd	ymm2, ymm2, ymm12		;; R2 * MemI2			; 17-21
	vmulpd	ymm13, ymm6, ymm12		;; I2 * MemI2			; 18-22
	vmulpd	ymm6, ymm6, ymm10		;; I2 * MemR2			; 19-23

	vsubpd	ymm10, ymm7, [srcreg+d2][rbp]	;; R3-MemR3
	vmovapd	ymm7, [srcreg+d2][r9]		;; R3=MemR3
	vsubpd	ymm12, ymm9, [srcreg+d2+32][rbp];; I3-MemI3
	vmovapd	ymm9, [srcreg+d2+32][r9]	;; I3=MemI3

	vsubpd	ymm5, ymm5, ymm11		;; R1*MemR1-I1*MemI1 (new R1)	; 19-21
	vaddpd	ymm8, ymm8, ymm3		;; R1*MemI1+I1*MemR1 (new I1)	; 20-22

	vmulpd	ymm3, ymm7, ymm10		;; R3 * MemR3			; 20-24
	vmulpd	ymm7, ymm7, ymm12		;; R3 * MemI3			; 21-25
	vmulpd	ymm11, ymm9, ymm12		;; I3 * MemI3			; 22-26
	vmulpd	ymm9, ymm9, ymm10		;; I3 * MemR3			; 23-27

	vsubpd	ymm10, ymm0, [srcreg+d2+d1][rbp];; R4-MemR4
	vmovapd	ymm0, [srcreg+d2+d1][r9]	;; R4=MemR4
	vsubpd	ymm12, ymm4, [srcreg+d2+d1+32][rbp];; I4-MemI4
	vmovapd	ymm4, [srcreg+d2+d1+32][r9]	;; I4=MemI4

	vsubpd	ymm1, ymm1, ymm13		;; R2*MemR2-I2*MemI2 (new R2)	; 23-25
	vaddpd	ymm2, ymm2, ymm6		;; R2*MemI2+I2*MemR2 (new I2)	; 24-26

	vmulpd	ymm6, ymm0, ymm10		;; R4 * MemR4			; 24-28
	vmulpd	ymm0, ymm0, ymm12		;; R4 * MemI4			; 25-29
	vmulpd	ymm13, ymm4, ymm12		;; I4 * MemI4			; 26-30
	vmulpd	ymm4, ymm4, ymm10		;; I4 * MemR4			; 27-31

	vaddpd	ymm10, ymm5, ymm1		;; R1 + R2 (new R1)		; 26-28
	vsubpd	ymm5, ymm5, ymm1		;; R1 - R2 (new R2)		; 27-29

	vsubpd	ymm3, ymm3, ymm11		;; R3*MemR3-I3*MemI3 (new R3)	; 28-30
	vaddpd	ymm7, ymm7, ymm9		;; R3*MemI3+I3*MemR3 (new I3)	; 29-31

	vsubpd	ymm6, ymm6, ymm13		;; R4*MemR4-I4*MemI4 (new R4)	; 31-33
	vaddpd	ymm0, ymm0, ymm4		;; R4*MemI4+I4*MemR4 (new I4)	; 32-34
	jmp	done

unfft:	vaddpd	ymm10, ymm8, ymm2		;; R1 + R2 (new R1)		; 26-28
	vsubpd	ymm5, ymm8, ymm2		;; R1 - R2 (new R2)		; 27-29
	vmovapd ymm8, ymm3			;; I1
	vmovapd ymm2, ymm6			;; I2
	vmovapd ymm3, ymm7			;; R3
	vmovapd ymm6, ymm0			;; R4
	vmovapd	ymm7, ymm9			;; I3
	vmovapd	ymm0, ymm4			;; I4
	jmp	done

muladd:	vmovapd	ymm1, [srcreg][rbp]		;; MemR1
	vmovapd	ymm10, [srcreg+32][rbp]		;; MemI1
	vmulpd	ymm5, ymm8, ymm1		;; R1 * MemR1			; 12-16
	vmulpd	ymm8, ymm8, ymm10		;; R1 * MemI1			; 13-17
	vmulpd	ymm11, ymm3, ymm10		;; I1 * MemI1			; 14-18
	vmulpd	ymm3, ymm3, ymm1		;; I1 * MemR1			; 15-19

	vmovapd	ymm10, [srcreg+d1][rbp]		;; MemR2
	vmovapd	ymm12, [srcreg+d1+32][rbp]	;; MemI2

	vmulpd	ymm1, ymm2, ymm10		;; R2 * MemR2			; 16-20
	vmulpd	ymm2, ymm2, ymm12		;; R2 * MemI2			; 17-21
	vmulpd	ymm13, ymm6, ymm12		;; I2 * MemI2			; 18-22
	vmulpd	ymm6, ymm6, ymm10		;; I2 * MemR2			; 19-23

	vmovapd	ymm10, [srcreg+d2][rbp]		;; MemR3
	vmovapd	ymm12, [srcreg+d2+32][rbp]	;; MemI3

	vsubpd	ymm5, ymm5, ymm11		;; R1*MemR1-I1*MemI1 (new R1)	; 19-21
	vaddpd	ymm8, ymm8, ymm3		;; R1*MemI1+I1*MemR1 (new I1)	; 20-22

	vmulpd	ymm3, ymm7, ymm10		;; R3 * MemR3			; 20-24
	vmulpd	ymm7, ymm7, ymm12		;; R3 * MemI3			; 21-25
	vmulpd	ymm11, ymm9, ymm12		;; I3 * MemI3			; 22-26
	vmulpd	ymm9, ymm9, ymm10		;; I3 * MemR3			; 23-27

	vmovapd	ymm10, [srcreg+d2+d1][rbp]	;; MemR4
	vmovapd	ymm12, [srcreg+d2+d1+32][rbp]	;; MemI4

	vsubpd	ymm1, ymm1, ymm13		;; R2*MemR2-I2*MemI2 (new R2)	; 23-25
	vaddpd	ymm2, ymm2, ymm6		;; R2*MemI2+I2*MemR2 (new I2)	; 24-26

	vmulpd	ymm6, ymm0, ymm10		;; R4 * MemR4			; 24-28
	vmulpd	ymm0, ymm0, ymm12		;; R4 * MemI4			; 25-29
	vmulpd	ymm13, ymm4, ymm12		;; I4 * MemI4			; 26-30
	vmulpd	ymm4, ymm4, ymm10		;; I4 * MemR4			; 27-31

	vsubpd	ymm3, ymm3, ymm11		;; R3*MemR3-I3*MemI3 (new R3)	; 28-30
	vaddpd	ymm7, ymm7, ymm9		;; R3*MemI3+I3*MemR3 (new I3)	; 29-31

	vsubpd	ymm6, ymm6, ymm13		;; R4*MemR4-I4*MemI4 (new R4)	; 31-33
	vaddpd	ymm0, ymm0, ymm4		;; R4*MemI4+I4*MemR4 (new I4)	; 32-34

	mov	r10, SRC3ARG			;; Load distance to FFT(1)
	test	r10, 3				;; Check if we need to multiply add-in by FFT(1)
	jz	muladdhard			;; Yes, do it the hard way

	vaddpd	ymm5, ymm5, [srcreg+r9]			;; R1 = R1 + MemR1
	vaddpd	ymm8, ymm8, [srcreg+r9+32]		;; I1 = I1 + MemI1
	vaddpd	ymm1, ymm1, [srcreg+r9+d1]		;; R2 = R2 + MemR2
	vaddpd	ymm2, ymm2, [srcreg+r9+d1+32]		;; I2 = I2 + MemI2
	vaddpd	ymm3, ymm3, [srcreg+r9+d2]		;; R3 = R3 + MemR3
	vaddpd	ymm7, ymm7, [srcreg+r9+d2+32]		;; I3 = I3 + MemI3
	vaddpd	ymm6, ymm6, [srcreg+r9+d2+d1]		;; R4 = R4 + MemR4
	vaddpd	ymm0, ymm0, [srcreg+r9+d2+d1+32]	;; I4 = I4 + MemI4

	vaddpd	ymm10, ymm5, ymm1		;; R1 + R2 (new R1)		; 26-28
	vsubpd	ymm5, ymm5, ymm1		;; R1 - R2 (new R2)		; 27-29
	jmp	done

muladdhard:
	vmovapd	ymm4, [srcreg+r9]			;; MemR1
	vmovapd	ymm9, [srcreg+r10]			;; MemR1#2
	vmulpd	ymm11, ymm4, ymm9			;; MemR1*MemR1#2
	vaddpd	ymm5, ymm5, ymm11			;; R1 = R1 + MemR1*MemR1#2
	vmovapd	ymm10, [srcreg+r10+32]			;; MemI1#2
	vmulpd	ymm11, ymm4, ymm10			;; MemR1*MemI1#2
	vaddpd	ymm8, ymm8, ymm11			;; I1 = I1 + MemR1*MemI1#2
	vmovapd	ymm4, [srcreg+r9+32]			;; MemI1
	vmulpd	ymm11, ymm4, ymm10			;; MemI1*MemI1#2
	vsubpd	ymm5, ymm5, ymm11			;; R1 = R1 - MemI1*MemI1#2
	vmulpd	ymm11, ymm4, ymm9			;; MemI1*MemR1#2
	vaddpd	ymm8, ymm8, ymm11			;; I1 = I1 + MemI1*MemR1#2

	vmovapd	ymm4, [srcreg+r9+d1]			;; MemR2
	vmovapd	ymm9, [srcreg+r10+d1]			;; MemR2#2
	vmulpd	ymm11, ymm4, ymm9			;; MemR2*MemR2#2
	vaddpd	ymm1, ymm1, ymm11			;; R2 = R2 + MemR2*MemR2#2
	vmovapd	ymm10, [srcreg+r10+d1+32]		;; MemI2#2
	vmulpd	ymm11, ymm4, ymm10			;; MemR2*MemI2#2
	vaddpd	ymm2, ymm2, ymm11			;; I2 = I2 + MemR2*MemI2#2
	vmovapd	ymm4, [srcreg+r9+d1+32]			;; MemI2
	vmulpd	ymm11, ymm4, ymm10			;; MemI2*MemI2#2
	vsubpd	ymm1, ymm1, ymm11			;; R2 = R2 - MemI2*MemI2#2
	vmulpd	ymm11, ymm4, ymm9			;; MemI2*MemR2#2
	vaddpd	ymm2, ymm2, ymm11			;; I2 = I2 + MemI2*MemR2#2

	vmovapd	ymm4, [srcreg+r9+d2]			;; MemR3
	vmovapd	ymm9, [srcreg+r10+d2]			;; MemR3#2
	vmulpd	ymm11, ymm4, ymm9			;; MemR3*MemR3#2
	vaddpd	ymm3, ymm3, ymm11			;; R3 = R3 + MemR3*MemR3#2
	vmovapd	ymm10, [srcreg+r10+d2+32]		;; MemI3#2
	vmulpd	ymm11, ymm4, ymm10			;; MemR3*MemI3#2
	vaddpd	ymm7, ymm7, ymm11			;; I3 = I3 + MemR3*MemI3#2
	vmovapd	ymm4, [srcreg+r9+d2+32]			;; MemI3
	vmulpd	ymm11, ymm4, ymm10			;; MemI3*MemI3#2
	vsubpd	ymm3, ymm3, ymm11			;; R3 = R3 - MemI3*MemI3#2
	vmulpd	ymm11, ymm4, ymm9			;; MemI3*MemR3#2
	vaddpd	ymm7, ymm7, ymm11			;; I3 = I3 + MemI3*MemR3#2

	vmovapd	ymm4, [srcreg+r9+d2+d1]			;; MemR4
	vmovapd	ymm9, [srcreg+r10+d2+d1]		;; MemR4#2
	vmulpd	ymm11, ymm4, ymm9			;; MemR4*MemR4#2
	vaddpd	ymm6, ymm6, ymm11			;; R4 = R4 + MemR4*MemR4#2
	vmovapd	ymm10, [srcreg+r10+d2+d1+32]		;; MemI4#2
	vmulpd	ymm11, ymm4, ymm10			;; MemR4*MemI4#2
	vaddpd	ymm0, ymm0, ymm11			;; I4 = I4 + MemR4*MemI4#2
	vmovapd	ymm4, [srcreg+r9+d2+d1+32]		;; MemI4
	vmulpd	ymm11, ymm4, ymm10			;; MemI4*MemI4#2
	vsubpd	ymm6, ymm6, ymm11			;; R4 = R4 - MemI4*MemI4#2
	vmulpd	ymm11, ymm4, ymm9			;; MemI4*MemR4#2
	vaddpd	ymm0, ymm0, ymm11			;; I4 = I4 + MemI4*MemR4#2

	vaddpd	ymm10, ymm5, ymm1		;; R1 + R2 (new R1)		; 26-28
	vsubpd	ymm5, ymm5, ymm1		;; R1 - R2 (new R2)		; 27-29
	jmp	done

mulsub:	vmovapd	ymm1, [srcreg][rbp]		;; MemR1
	vmovapd	ymm10, [srcreg+32][rbp]		;; MemI1
	vmulpd	ymm5, ymm8, ymm1		;; R1 * MemR1			; 12-16
	vmulpd	ymm8, ymm8, ymm10		;; R1 * MemI1			; 13-17
	vmulpd	ymm11, ymm3, ymm10		;; I1 * MemI1			; 14-18
	vmulpd	ymm3, ymm3, ymm1		;; I1 * MemR1			; 15-19

	vmovapd	ymm10, [srcreg+d1][rbp]		;; MemR2
	vmovapd	ymm12, [srcreg+d1+32][rbp]	;; MemI2

	vmulpd	ymm1, ymm2, ymm10		;; R2 * MemR2			; 16-20
	vmulpd	ymm2, ymm2, ymm12		;; R2 * MemI2			; 17-21
	vmulpd	ymm13, ymm6, ymm12		;; I2 * MemI2			; 18-22
	vmulpd	ymm6, ymm6, ymm10		;; I2 * MemR2			; 19-23

	vmovapd	ymm10, [srcreg+d2][rbp]		;; MemR3
	vmovapd	ymm12, [srcreg+d2+32][rbp]	;; MemI3

	vsubpd	ymm5, ymm5, ymm11		;; R1*MemR1-I1*MemI1 (new R1)	; 19-21
	vaddpd	ymm8, ymm8, ymm3		;; R1*MemI1+I1*MemR1 (new I1)	; 20-22

	vmulpd	ymm3, ymm7, ymm10		;; R3 * MemR3			; 20-24
	vmulpd	ymm7, ymm7, ymm12		;; R3 * MemI3			; 21-25
	vmulpd	ymm11, ymm9, ymm12		;; I3 * MemI3			; 22-26
	vmulpd	ymm9, ymm9, ymm10		;; I3 * MemR3			; 23-27

	vmovapd	ymm10, [srcreg+d2+d1][rbp]	;; MemR4
	vmovapd	ymm12, [srcreg+d2+d1+32][rbp]	;; MemI4

	vsubpd	ymm1, ymm1, ymm13		;; R2*MemR2-I2*MemI2 (new R2)	; 23-25
	vaddpd	ymm2, ymm2, ymm6		;; R2*MemI2+I2*MemR2 (new I2)	; 24-26

	vmulpd	ymm6, ymm0, ymm10		;; R4 * MemR4			; 24-28
	vmulpd	ymm0, ymm0, ymm12		;; R4 * MemI4			; 25-29
	vmulpd	ymm13, ymm4, ymm12		;; I4 * MemI4			; 26-30
	vmulpd	ymm4, ymm4, ymm10		;; I4 * MemR4			; 27-31

	vsubpd	ymm3, ymm3, ymm11		;; R3*MemR3-I3*MemI3 (new R3)	; 28-30
	vaddpd	ymm7, ymm7, ymm9		;; R3*MemI3+I3*MemR3 (new I3)	; 29-31

	vsubpd	ymm6, ymm6, ymm13		;; R4*MemR4-I4*MemI4 (new R4)	; 31-33
	vaddpd	ymm0, ymm0, ymm4		;; R4*MemI4+I4*MemR4 (new I4)	; 32-34

	mov	r10, SRC3ARG			;; Load distance to FFT(1)
	test	r10, 3				;; Check if we need to multiply add-in by FFT(1)
	jz	mulsubhard			;; Yes, do it the hard way

	vsubpd	ymm5, ymm5, [srcreg+r9]			;; R1 = R1 - MemR1
	vsubpd	ymm8, ymm8, [srcreg+r9+32]		;; I1 = I1 - MemI1
	vsubpd	ymm1, ymm1, [srcreg+r9+d1]		;; R2 = R2 - MemR2
	vsubpd	ymm2, ymm2, [srcreg+r9+d1+32]		;; I2 = I2 - MemI2
	vsubpd	ymm3, ymm3, [srcreg+r9+d2]		;; R3 = R3 - MemR3
	vsubpd	ymm7, ymm7, [srcreg+r9+d2+32]		;; I3 = I3 - MemI3
	vsubpd	ymm6, ymm6, [srcreg+r9+d2+d1]		;; R4 = R4 - MemR4
	vsubpd	ymm0, ymm0, [srcreg+r9+d2+d1+32]	;; I4 = I4 - MemI4

	vaddpd	ymm10, ymm5, ymm1		;; R1 + R2 (new R1)		; 26-28
	vsubpd	ymm5, ymm5, ymm1		;; R1 - R2 (new R2)		; 27-29
	jmp	done

mulsubhard:
	vmovapd	ymm4, [srcreg+r9]			;; MemR1
	vmovapd	ymm9, [srcreg+r10]			;; MemR1#2
	vmulpd	ymm11, ymm4, ymm9			;; MemR1*MemR1#2
	vsubpd	ymm5, ymm5, ymm11			;; R1 = R1 - MemR1*MemR1#2
	vmovapd	ymm10, [srcreg+r10+32]			;; MemI1#2
	vmulpd	ymm11, ymm4, ymm10			;; MemR1*MemI1#2
	vsubpd	ymm8, ymm8, ymm11			;; I1 = I1 - MemR1*MemI1#2
	vmovapd	ymm4, [srcreg+r9+32]			;; MemI1
	vmulpd	ymm11, ymm4, ymm10			;; MemI1*MemI1#2
	vaddpd	ymm5, ymm5, ymm11			;; R1 = R1 + MemI1*MemI1#2
	vmulpd	ymm11, ymm4, ymm9			;; MemI1*MemR1#2
	vsubpd	ymm8, ymm8, ymm11			;; I1 = I1 - MemI1*MemR1#2

	vmovapd	ymm4, [srcreg+r9+d1]			;; MemR2
	vmovapd	ymm9, [srcreg+r10+d1]			;; MemR2#2
	vmulpd	ymm11, ymm4, ymm9			;; MemR2*MemR2#2
	vsubpd	ymm1, ymm1, ymm11			;; R2 = R2 - MemR2*MemR2#2
	vmovapd	ymm10, [srcreg+r10+d1+32]		;; MemI2#2
	vmulpd	ymm11, ymm4, ymm10			;; MemR2*MemI2#2
	vsubpd	ymm2, ymm2, ymm11			;; I2 = I2 - MemR2*MemI2#2
	vmovapd	ymm4, [srcreg+r9+d1+32]			;; MemI2
	vmulpd	ymm11, ymm4, ymm10			;; MemI2*MemI2#2
	vaddpd	ymm1, ymm1, ymm11			;; R2 = R2 + MemI2*MemI2#2
	vmulpd	ymm11, ymm4, ymm9			;; MemI2*MemR2#2
	vsubpd	ymm2, ymm2, ymm11			;; I2 = I2 - MemI2*MemR2#2

	vmovapd	ymm4, [srcreg+r9+d2]			;; MemR3
	vmovapd	ymm9, [srcreg+r10+d2]			;; MemR3#2
	vmulpd	ymm11, ymm4, ymm9			;; MemR3*MemR3#2
	vsubpd	ymm3, ymm3, ymm11			;; R3 = R3 - MemR3*MemR3#2
	vmovapd	ymm10, [srcreg+r10+d2+32]		;; MemI3#2
	vmulpd	ymm11, ymm4, ymm10			;; MemR3*MemI3#2
	vsubpd	ymm7, ymm7, ymm11			;; I3 = I3 - MemR3*MemI3#2
	vmovapd	ymm4, [srcreg+r9+d2+32]			;; MemI3
	vmulpd	ymm11, ymm4, ymm10			;; MemI3*MemI3#2
	vaddpd	ymm3, ymm3, ymm11			;; R3 = R3 + MemI3*MemI3#2
	vmulpd	ymm11, ymm4, ymm9			;; MemI3*MemR3#2
	vsubpd	ymm7, ymm7, ymm11			;; I3 = I3 - MemI3*MemR3#2

	vmovapd	ymm4, [srcreg+r9+d2+d1]			;; MemR4
	vmovapd	ymm9, [srcreg+r10+d2+d1]		;; MemR4#2
	vmulpd	ymm11, ymm4, ymm9			;; MemR4*MemR4#2
	vsubpd	ymm6, ymm6, ymm11			;; R4 = R4 - MemR4*MemR4#2
	vmovapd	ymm10, [srcreg+r10+d2+d1+32]		;; MemI4#2
	vmulpd	ymm11, ymm4, ymm10			;; MemR4*MemI4#2
	vsubpd	ymm0, ymm0, ymm11			;; I4 = I4 - MemR4*MemI4#2
	vmovapd	ymm4, [srcreg+r9+d2+d1+32]		;; MemI4
	vmulpd	ymm11, ymm4, ymm10			;; MemI4*MemI4#2
	vaddpd	ymm6, ymm6, ymm11			;; R4 = R4 + MemI4*MemI4#2
	vmulpd	ymm11, ymm4, ymm9			;; MemI4*MemR4#2
	vsubpd	ymm0, ymm0, ymm11			;; I4 = I4 - MemI4*MemR4#2

	vaddpd	ymm10, ymm5, ymm1		;; R1 + R2 (new R1)		; 26-28
	vsubpd	ymm5, ymm5, ymm1		;; R1 - R2 (new R2)		; 27-29

done:
	ENDM

;; Haswell FMA3 version

IF (@INSTR(,%yarch,<FMA3>) NE 0)

yr4_4cl_four_complex_with_mult_preload MACRO
	ENDM
yr4_4cl_four_complex_with_mult MACRO srcreg,srcinc,d1,d2
	LOCAL	orig, back_to_orig

	vmovapd	ymm0, [srcreg]			;; R1
	vmovapd	ymm7, [srcreg+d2]		;; R3
	vaddpd	ymm2, ymm0, ymm7		;; R1 + R3 (new R1)		; 1-3
	vsubpd	ymm0, ymm0, ymm7		;; R1 - R3 (new R3)		; 2-4

	vmovapd	ymm1, [srcreg+d1]		;; R2
	vmovapd	ymm7, [srcreg+d2+d1]		;; R4
	vaddpd	ymm3, ymm1, ymm7		;; R2 + R4 (new R2)		; 3-5
	vsubpd	ymm1, ymm1, ymm7		;; R2 - R4 (new R4)		; 4-6

	vmovapd	ymm4, [srcreg+32]		;; I1
	vmovapd	ymm7, [srcreg+d2+32]		;; I3
	vaddpd	ymm6, ymm4, ymm7		;; I1 + I3 (new I1)		; 5-7
	vsubpd	ymm4, ymm4, ymm7		;; I1 - I3 (new I3)		; 6-8

	vmovapd	ymm5, [srcreg+d1+32]		;; I2
	vmovapd	ymm8, [srcreg+d2+d1+32]		;; I4
	vaddpd	ymm7, ymm5, ymm8		;; I2 + I4 (new I2)		; 7-9
	vsubpd	ymm5, ymm5, ymm8		;; I2 - I4 (new I4)		; 8-10

	vaddpd	ymm8, ymm2, ymm3		;; R1 + R2 (final R1)		; 9-11
	vsubpd	ymm2, ymm2, ymm3		;; R1 - R2 (final R2)		; 10-12

	vaddpd	ymm3, ymm6, ymm7		;; I1 + I2 (final I1)		; 11-13
	vsubpd	ymm6, ymm6, ymm7		;; I1 - I2 (final I2)		; 12-14

	vsubpd	ymm7, ymm0, ymm5		;; R3 - I4 (final R3)		; 13-15
	vaddpd	ymm0, ymm0, ymm5		;; R3 + I4 (final R4)		; 14-16

	cmp	mul4_opcode, 0			;; See if we need to do more than the original type-3 FFT multiply
	je	short orig
	yr4_dispatch CALLP,ycomplex_mult_opcode,d1,d2 ;; Handle more difficult cases
	jmp	back_to_orig

orig:	vmovapd	ymm10, [srcreg][rbp]		;; MemR1
	vmovapd	ymm12, [srcreg+32][rbp]		;; MemI1
	vmulpd	ymm5, ymm8, ymm10		;; R1 * MemR1			; 12-16
	vmulpd	ymm8, ymm8, ymm12		;; R1 * MemI1			; 13-17

	vmovapd	ymm11, [srcreg+d1][rbp]		;; MemR2
	vmovapd	ymm13, [srcreg+d1+32][rbp]	;; MemI2
	vmulpd	ymm14, ymm2, ymm11		;; R2 * MemR2			; 14-18
	vmulpd	ymm2, ymm2, ymm13		;; R2 * MemI2			; 15-19

	vaddpd	ymm9, ymm4, ymm1		;; I3 + R4 (final I3)		; 15-17
	vsubpd	ymm4, ymm4, ymm1		;; I3 - R4 (final I4)		; 16-18

	yfnmaddpd ymm5, ymm3, ymm12, ymm5	;; R1*MemR1-I1*MemI1 (new R1)	; 17-21
	yfmaddpd ymm8, ymm3, ymm10, ymm8	;; R1*MemI1+I1*MemR1 (new I1)	; 18-22

	vmovapd	ymm10, [srcreg+d2][rbp]		;; MemR3
	vmovapd	ymm12, [srcreg+d2+32][rbp]	;; MemI3
	vmulpd	ymm1, ymm7, ymm10		;; R3 * MemR3			; 16-20
	vmulpd	ymm7, ymm7, ymm12		;; R3 * MemI3			; 17-21

	yfnmaddpd ymm14, ymm6, ymm13, ymm14	;; R2*MemR2-I2*MemI2 (new R2)	; 19-23
	yfmaddpd ymm2, ymm6, ymm11, ymm2	;; R2*MemI2+I2*MemR2 (new I2)	; 20-24

	vmovapd	ymm11, [srcreg+d2+d1][rbp]	;; MemR4
	vmovapd	ymm13, [srcreg+d2+d1+32][rbp]	;; MemI4

	vmulpd	ymm3, ymm0, ymm11		;; R4 * MemR4			; 18-22
	vmulpd	ymm0, ymm0, ymm13		;; R4 * MemI4			; 19-23

	yfnmaddpd ymm1, ymm9, ymm12, ymm1	;; R3*MemR3-I3*MemI3 (new R3)	; 21-25
	yfmaddpd ymm7, ymm9, ymm10, ymm7	;; R3*MemI3+I3*MemR3 (new I3)	; 22-26

	yfnmaddpd ymm3, ymm4, ymm13, ymm3	;; R4*MemR4-I4*MemI4 (new R4)	; 23-27
	yfmaddpd ymm0, ymm4, ymm11, ymm0	;; R4*MemI4+I4*MemR4 (new I4)	; 24-28

back_to_orig:
	vaddpd	ymm4, ymm5, ymm14		;; R1 + R2 (new R1)		; 24-26
	vsubpd	ymm5, ymm5, ymm14		;; R1 - R2 (new R2)		; 25-27

	vaddpd	ymm6, ymm8, ymm2		;; I1 + I2 (new I1)		; 26-28
	vsubpd	ymm8, ymm8, ymm2		;; I1 - I2 (new I2)		; 27-29

	vaddpd	ymm2, ymm3, ymm1		;; R4 + R3 (new R3)		; 28-30
	vsubpd	ymm3, ymm3, ymm1		;; R4 - R3 (new I4)		; 29-31

	vaddpd	ymm1, ymm7, ymm0		;; I3 + I4 (new I3)		; 30-32
	vsubpd	ymm7, ymm7, ymm0		;; I3 - I4 (new R4)		; 31-33

	vaddpd	ymm0, ymm4, ymm2		;; R1 + R3 (final R1)		; 32-34
	vsubpd	ymm4, ymm4, ymm2		;; R1 - R3 (final R3)		; 33-35

	vaddpd	ymm2, ymm8, ymm3		;; I2 + I4 (final I2)		; 34-36
	vsubpd	ymm8, ymm8, ymm3		;; I2 - I4 (final I4)		; 35-37

	vaddpd	ymm3, ymm6, ymm1		;; I1 + I3 (final I1)		; 36-38
	vsubpd	ymm6, ymm6, ymm1		;; I1 - I3 (final I3)		; 37-39

	vaddpd	ymm1, ymm5, ymm7		;; R2 + R4 (final R2)		; 38-40
	vsubpd	ymm5, ymm5, ymm7		;; R2 - R4 (final R4)		; 39-41

	ystore	[srcreg+r8], ymm0		;; Save R1
	ystore	[srcreg+r8+32], ymm3		;; Save I1
	ystore	[srcreg+r8+d1], ymm1		;; Save R2
	ystore	[srcreg+r8+d1+32], ymm2		;; Save I2
	ystore	[srcreg+r8+d2], ymm4		;; Save R3
	ystore	[srcreg+r8+d2+32], ymm6		;; Save I3
	ystore	[srcreg+r8+d2+d1], ymm5		;; Save R4
	ystore	[srcreg+r8+d2+d1+32], ymm8	;; Save I4
	bump	srcreg, srcinc
	ENDM

;; Same as above code between orig and back_to_orig, except we implement mul4_opcode options

yr4_4c_mult_opcode MACRO srcreg,d1,d2
	LOCAL	mulsave, fftsave, nofftsave, dispatch, disptab, addmul, submul, muladd, muladdhard, mulsub, mulsubhard, unfft, done
	LOCAL	addmul312, addmul313, addmul132, addmul133, submul312, submul313, submul132, submul133

	movzx	r10, mul4_opcode		;; Load opcode
	cmp	r10, 80h			;; Test for FFT save
	ja	fftsave				;; opcode > 0x80, addmul,submul,muladd,mulsub,unfft with FFT save
	jb	nofftsave			;; opcode < 0x80, addmul,submul,muladd,mulsub,unfft without FFT save
						;; opcode == 0x80, plain mul with FFT save, fall through

	;; Squaring with FFT save is handled here! 
	test	rbp, rbp			;; If rbp is zero, this is a squaring
	jnz	mulsave

	ystore	[srcreg], ymm8			;; R1
	vmulpd	ymm5, ymm8, ymm8		;; R1 * R1			; 12-16
	vmulpd	ymm8, ymm8, ymm3		;; R1 * I1			; 13-17

	ystore	[srcreg+d1], ymm2		;; R2
	vmulpd	ymm14, ymm2, ymm2		;; R2 * R2			; 14-18
	vmulpd	ymm2, ymm2, ymm6		;; R2 * I2			; 15-19

	vaddpd	ymm9, ymm4, ymm1		;; I3 + R4 (final I3)		; 15-17
	vsubpd	ymm4, ymm4, ymm1		;; I3 - R4 (final I4)		; 16-18

	ystore	[srcreg+32], ymm3		;; I1
	yfnmaddpd ymm5, ymm3, ymm3, ymm5	;; R1*R1-I1*I1 (new R1)		; 17-21
	vaddpd	ymm8, ymm8, ymm8		;; 2*R1*I1 (new I1)		; 18-22

	ystore	[srcreg+d2], ymm7		;; R3
	vmulpd	ymm1, ymm7, ymm7		;; R3 * R3			; 16-20
	vmulpd	ymm7, ymm7, ymm9		;; R3 * I3			; 17-21

	ystore	[srcreg+d1+32], ymm6		;; I2
	yfnmaddpd ymm14, ymm6, ymm6, ymm14	;; R2*R2-I2*I2 (new R2)		; 19-23
	vaddpd	ymm2, ymm2, ymm2		;; 2*R2*I2 (new I2)		; 20-24

	ystore	[srcreg+d2+32], ymm9		;; I3
	ystore	[srcreg+d2+d1], ymm0		;; R4
	ystore	[srcreg+d2+d1+32], ymm4		;; I4

	vmulpd	ymm3, ymm0, ymm0		;; R4 * R4			; 18-22
	vmulpd	ymm0, ymm0, ymm4		;; R4 * I4			; 19-23

	yfnmaddpd ymm1, ymm9, ymm9, ymm1	;; R3*R3-I3*I3 (new R3)		; 21-25
	vaddpd	ymm7, ymm7, ymm7		;; 2*R3*I3 (new I3)		; 22-26

	yfnmaddpd ymm3, ymm4, ymm4, ymm3	;; R4*R4-I4*I4 (new R4)		; 23-27
	vaddpd	ymm0, ymm0, ymm0		;; 2*R4*I4 (new I4)		; 24-28
	jmp	done

mulsave:vmovapd	ymm10, [srcreg][rbp]		;; MemR1
	vmovapd	ymm12, [srcreg+32][rbp]		;; MemI1
	ystore	[srcreg], ymm8			;; R1
	vmulpd	ymm5, ymm8, ymm10		;; R1 * MemR1			; 12-16
	vmulpd	ymm8, ymm8, ymm12		;; R1 * MemI1			; 13-17

	vmovapd	ymm11, [srcreg+d1][rbp]		;; MemR2
	vmovapd	ymm13, [srcreg+d1+32][rbp]	;; MemI2
	ystore	[srcreg+d1], ymm2		;; R2
	vmulpd	ymm14, ymm2, ymm11		;; R2 * MemR2			; 14-18
	vmulpd	ymm2, ymm2, ymm13		;; R2 * MemI2			; 15-19

	vaddpd	ymm9, ymm4, ymm1		;; I3 + R4 (final I3)		; 15-17
	vsubpd	ymm4, ymm4, ymm1		;; I3 - R4 (final I4)		; 16-18

	ystore	[srcreg+32], ymm3		;; I1
	yfnmaddpd ymm5, ymm3, ymm12, ymm5	;; R1*MemR1-I1*MemI1 (new R1)	; 17-21
	yfmaddpd ymm8, ymm3, ymm10, ymm8	;; R1*MemI1+I1*MemR1 (new I1)	; 18-22

	vmovapd	ymm10, [srcreg+d2][rbp]		;; MemR3
	vmovapd	ymm12, [srcreg+d2+32][rbp]	;; MemI3
	ystore	[srcreg+d2], ymm7		;; R3
	vmulpd	ymm1, ymm7, ymm10		;; R3 * MemR3			; 16-20
	vmulpd	ymm7, ymm7, ymm12		;; R3 * MemI3			; 17-21

	ystore	[srcreg+d1+32], ymm6		;; I2
	yfnmaddpd ymm14, ymm6, ymm13, ymm14	;; R2*MemR2-I2*MemI2 (new R2)	; 19-23
	yfmaddpd ymm2, ymm6, ymm11, ymm2	;; R2*MemI2+I2*MemR2 (new I2)	; 20-24

	vmovapd	ymm11, [srcreg+d2+d1][rbp]	;; MemR4
	vmovapd	ymm13, [srcreg+d2+d1+32][rbp]	;; MemI4

	ystore	[srcreg+d2+32], ymm9		;; I3
	ystore	[srcreg+d2+d1], ymm0		;; R4
	ystore	[srcreg+d2+d1+32], ymm4		;; I4

	vmulpd	ymm3, ymm0, ymm11		;; R4 * MemR4			; 18-22
	vmulpd	ymm0, ymm0, ymm13		;; R4 * MemI4			; 19-23

	yfnmaddpd ymm1, ymm9, ymm12, ymm1	;; R3*MemR3-I3*MemI3 (new R3)	; 21-25
	yfmaddpd ymm7, ymm9, ymm10, ymm7	;; R3*MemI3+I3*MemR3 (new I3)	; 22-26

	yfnmaddpd ymm3, ymm4, ymm13, ymm3	;; R4*MemR4-I4*MemI4 (new R4)	; 23-27
	yfmaddpd ymm0, ymm4, ymm11, ymm0	;; R4*MemI4+I4*MemR4 (new I4)	; 24-28
	jmp	done

fftsave:ystore	[srcreg], ymm8			;; R1
	ystore	[srcreg+d1], ymm2		;; R2
	vaddpd	ymm9, ymm4, ymm1		;; I3 + R4 (final I3)		; 15-17
	vsubpd	ymm4, ymm4, ymm1		;; I3 - R4 (final I4)		; 16-18
	ystore	[srcreg+32], ymm3		;; I1
	ystore	[srcreg+d2], ymm7		;; R3
	ystore	[srcreg+d1+32], ymm6		;; I2
	ystore	[srcreg+d2+32], ymm9		;; I3
	ystore	[srcreg+d2+d1], ymm0		;; R4
	ystore	[srcreg+d2+d1+32], ymm4		;; I4
	and	r10, 7fh			;; Strip the fft save bit from mul4_opcode
	jmp	short dispatch			

nofftsave:
	vaddpd	ymm9, ymm4, ymm1		;; I3 + R4 (final I3)		; 15-17
	vsubpd	ymm4, ymm4, ymm1		;; I3 - R4 (final I4)		; 16-18

dispatch:
	mov	r9, OFFSET disptab		;; Jump table address
	mov	r10, [r9+r10*8-8]		;; Jump table offset
	mov	r9, SRC2ARG			;; Distance to s2 arg from gwaddmul4 or gwsubmul4 or s3 arg in gwmuladd4 or gwmulsub4
	jmp	r10
disptab:
	DQ	OFFSET addmul			;; 1 = addmul
	DQ	OFFSET submul			;; 2 = submul
	DQ	OFFSET muladd			;; 3 = muladd
	DQ	OFFSET mulsub			;; 4 = mulsub
	DQ	OFFSET unfft			;; 5 = unfft
	DQ	OFFSET addmul312		;; 6 = addmul FFTval mem1 mem2
	DQ	OFFSET addmul313		;; 7 = addmul FFTval mem1 FFTval
	DQ	OFFSET addmul132		;; 8 = addmul mem1 FFTval mem2
	DQ	OFFSET addmul133		;; 9 = addmul mem1 FFTval FFTval
	DQ	OFFSET submul312		;; 10 = submul FFTval mem1 mem2
	DQ	OFFSET submul313		;; 11 = submul FFTval mem1 FFTval
	DQ	OFFSET submul132		;; 12 = submul mem1 FFTval mem2
	DQ	OFFSET submul133		;; 13 = submul mem1 FFTval FFTval

addmul:	vmovapd	ymm10, [srcreg][rbp]		;; MemR1
	vaddpd	ymm10, ymm10, [srcreg][r9]	;; MemR1
	vmovapd	ymm12, [srcreg+32][rbp]		;; MemI1
	vaddpd	ymm12, ymm12, [srcreg+32][r9]	;; MemI1
	vmulpd	ymm5, ymm8, ymm10		;; R1 * MemR1			; 12-16
	vmulpd	ymm8, ymm8, ymm12		;; R1 * MemI1			; 13-17

	vmovapd	ymm11, [srcreg+d1][rbp]		;; MemR2
	vaddpd	ymm11, ymm11, [srcreg+d1][r9]	;; MemR2
	vmovapd	ymm13, [srcreg+d1+32][rbp]	;; MemI2
	vaddpd	ymm13, ymm13, [srcreg+d1+32][r9];; MemI2
	vmulpd	ymm14, ymm2, ymm11		;; R2 * MemR2			; 14-18
	vmulpd	ymm2, ymm2, ymm13		;; R2 * MemI2			; 15-19

	yfnmaddpd ymm5, ymm3, ymm12, ymm5	;; R1*MemR1-I1*MemI1 (new R1)	; 17-21
	yfmaddpd ymm8, ymm3, ymm10, ymm8	;; R1*MemI1+I1*MemR1 (new I1)	; 18-22

	vmovapd	ymm10, [srcreg+d2][rbp]		;; MemR3
	vaddpd	ymm10, ymm10, [srcreg+d2][r9]	;; MemR3
	vmovapd	ymm12, [srcreg+d2+32][rbp]	;; MemI3
	vaddpd	ymm12, ymm12, [srcreg+d2+32][r9];; MemI3
	vmulpd	ymm1, ymm7, ymm10		;; R3 * MemR3			; 16-20
	vmulpd	ymm7, ymm7, ymm12		;; R3 * MemI3			; 17-21

	yfnmaddpd ymm14, ymm6, ymm13, ymm14	;; R2*MemR2-I2*MemI2 (new R2)	; 19-23
	yfmaddpd ymm2, ymm6, ymm11, ymm2	;; R2*MemI2+I2*MemR2 (new I2)	; 20-24

	vmovapd	ymm11, [srcreg+d2+d1][rbp]	;; MemR4
	vaddpd	ymm11, ymm11, [srcreg+d2+d1][r9];; MemR4
	vmovapd	ymm13, [srcreg+d2+d1+32][rbp]	;; MemI4
	vaddpd	ymm13, ymm13, [srcreg+d2+d1+32][r9];; MemI4

	vmulpd	ymm3, ymm0, ymm11		;; R4 * MemR4			; 18-22
	vmulpd	ymm0, ymm0, ymm13		;; R4 * MemI4			; 19-23

	yfnmaddpd ymm1, ymm9, ymm12, ymm1	;; R3*MemR3-I3*MemI3 (new R3)	; 21-25
	yfmaddpd ymm7, ymm9, ymm10, ymm7	;; R3*MemI3+I3*MemR3 (new I3)	; 22-26

	yfnmaddpd ymm3, ymm4, ymm13, ymm3	;; R4*MemR4-I4*MemI4 (new R4)	; 23-27
	yfmaddpd ymm0, ymm4, ymm11, ymm0	;; R4*MemI4+I4*MemR4 (new I4)	; 24-28
	jmp	done

addmul133:
addmul313:
	vaddpd	ymm10, ymm8, [srcreg][rbp]	;; R1+MemR1
	vaddpd	ymm12, ymm3, [srcreg+32][rbp]	;; I1+MemI1
	vmulpd	ymm5, ymm8, ymm10		;; R1 * MemR1			; 12-16
	vmulpd	ymm8, ymm8, ymm12		;; R1 * MemI1			; 13-17

	vaddpd	ymm11, ymm2, [srcreg+d1][rbp]	;; R2+MemR2
	vaddpd	ymm13, ymm6, [srcreg+d1+32][rbp];; I2+MemI2
	vmulpd	ymm14, ymm2, ymm11		;; R2 * MemR2			; 14-18
	vmulpd	ymm2, ymm2, ymm13		;; R2 * MemI2			; 15-19

	yfnmaddpd ymm5, ymm3, ymm12, ymm5	;; R1*MemR1-I1*MemI1 (new R1)	; 17-21
	yfmaddpd ymm8, ymm3, ymm10, ymm8	;; R1*MemI1+I1*MemR1 (new I1)	; 18-22

	vaddpd	ymm10, ymm7, [srcreg+d2][rbp]	;; R3+MemR3
	vaddpd	ymm12, ymm9, [srcreg+d2+32][rbp];; I3+MemI3
	vmulpd	ymm1, ymm7, ymm10		;; R3 * MemR3			; 16-20
	vmulpd	ymm7, ymm7, ymm12		;; R3 * MemI3			; 17-21

	yfnmaddpd ymm14, ymm6, ymm13, ymm14	;; R2*MemR2-I2*MemI2 (new R2)	; 19-23
	yfmaddpd ymm2, ymm6, ymm11, ymm2	;; R2*MemI2+I2*MemR2 (new I2)	; 20-24

	vaddpd	ymm11, ymm0, [srcreg+d2+d1][rbp];; R4+MemR4
	vaddpd	ymm13, ymm4, [srcreg+d2+d1+32][rbp];; I4+MemI4
	vmulpd	ymm3, ymm0, ymm11		;; R4 * MemR4			; 18-22
	vmulpd	ymm0, ymm0, ymm13		;; R4 * MemI4			; 19-23

	yfnmaddpd ymm1, ymm9, ymm12, ymm1	;; R3*MemR3-I3*MemI3 (new R3)	; 21-25
	yfmaddpd ymm7, ymm9, ymm10, ymm7	;; R3*MemI3+I3*MemR3 (new I3)	; 22-26

	yfnmaddpd ymm3, ymm4, ymm13, ymm3	;; R4*MemR4-I4*MemI4 (new R4)	; 23-27
	yfmaddpd ymm0, ymm4, ymm11, ymm0	;; R4*MemI4+I4*MemR4 (new I4)	; 24-28
	jmp	done

addmul132:
addmul312:
	vaddpd	ymm10, ymm8, [srcreg][rbp]	;; R1+MemR1
	vmovapd	ymm8, [srcreg][r9]		;; R1=MemR1
	vaddpd	ymm12, ymm3, [srcreg+32][rbp]	;; I1+MemI1
	vmovapd	ymm3, [srcreg+32][r9]		;; I1=MemI1
	vmulpd	ymm5, ymm8, ymm10		;; R1 * MemR1			; 12-16
	vmulpd	ymm8, ymm8, ymm12		;; R1 * MemI1			; 13-17

	vaddpd	ymm11, ymm2, [srcreg+d1][rbp]	;; R2+MemR2
	vmovapd	ymm2, [srcreg+d1][r9]		;; R2=MemR2
	vaddpd	ymm13, ymm6, [srcreg+d1+32][rbp];; I2+MemI2
	vmovapd	ymm6, [srcreg+d1+32][r9]	;; I2=MemI2
	vmulpd	ymm14, ymm2, ymm11		;; R2 * MemR2			; 14-18
	vmulpd	ymm2, ymm2, ymm13		;; R2 * MemI2			; 15-19

	yfnmaddpd ymm5, ymm3, ymm12, ymm5	;; R1*MemR1-I1*MemI1 (new R1)	; 17-21
	yfmaddpd ymm8, ymm3, ymm10, ymm8	;; R1*MemI1+I1*MemR1 (new I1)	; 18-22

	vaddpd	ymm10, ymm7, [srcreg+d2][rbp]	;; R3+MemR3
	vmovapd	ymm7, [srcreg+d2][r9]		;; R3=MemR3
	vaddpd	ymm12, ymm9, [srcreg+d2+32][rbp];; I3+MemI3
	vmovapd	ymm9, [srcreg+d2+32][r9]	;; I3=MemI3
	vmulpd	ymm1, ymm7, ymm10		;; R3 * MemR3			; 16-20
	vmulpd	ymm7, ymm7, ymm12		;; R3 * MemI3			; 17-21

	yfnmaddpd ymm14, ymm6, ymm13, ymm14	;; R2*MemR2-I2*MemI2 (new R2)	; 19-23
	yfmaddpd ymm2, ymm6, ymm11, ymm2	;; R2*MemI2+I2*MemR2 (new I2)	; 20-24

	vaddpd	ymm11, ymm0, [srcreg+d2+d1][rbp];; R4+MemR4
	vmovapd	ymm0, [srcreg+d2+d1][r9]	;; R4=MemR4
	vaddpd	ymm13, ymm4, [srcreg+d2+d1+32][rbp];; I4+MemI4
	vmovapd	ymm4, [srcreg+d2+d1+32][r9]	;; I4=MemI4
	vmulpd	ymm3, ymm0, ymm11		;; R4 * MemR4			; 18-22
	vmulpd	ymm0, ymm0, ymm13		;; R4 * MemI4			; 19-23

	yfnmaddpd ymm1, ymm9, ymm12, ymm1	;; R3*MemR3-I3*MemI3 (new R3)	; 21-25
	yfmaddpd ymm7, ymm9, ymm10, ymm7	;; R3*MemI3+I3*MemR3 (new I3)	; 22-26

	yfnmaddpd ymm3, ymm4, ymm13, ymm3	;; R4*MemR4-I4*MemI4 (new R4)	; 23-27
	yfmaddpd ymm0, ymm4, ymm11, ymm0	;; R4*MemI4+I4*MemR4 (new I4)	; 24-28
	jmp	done

submul:	vmovapd	ymm10, [srcreg][rbp]		;; MemR1
	vsubpd	ymm10, ymm10, [srcreg][r9]	;; MemR1
	vmovapd	ymm12, [srcreg+32][rbp]		;; MemI1
	vsubpd	ymm12, ymm12, [srcreg+32][r9]	;; MemI1
	vmulpd	ymm5, ymm8, ymm10		;; R1 * MemR1			; 12-16
	vmulpd	ymm8, ymm8, ymm12		;; R1 * MemI1			; 13-17

	vmovapd	ymm11, [srcreg+d1][rbp]		;; MemR2
	vsubpd	ymm11, ymm11, [srcreg+d1][r9]	;; MemR2
	vmovapd	ymm13, [srcreg+d1+32][rbp]	;; MemI2
	vsubpd	ymm13, ymm13, [srcreg+d1+32][r9];; MemI2
	vmulpd	ymm14, ymm2, ymm11		;; R2 * MemR2			; 14-18
	vmulpd	ymm2, ymm2, ymm13		;; R2 * MemI2			; 15-19

	yfnmaddpd ymm5, ymm3, ymm12, ymm5	;; R1*MemR1-I1*MemI1 (new R1)	; 17-21
	yfmaddpd ymm8, ymm3, ymm10, ymm8	;; R1*MemI1+I1*MemR1 (new I1)	; 18-22

	vmovapd	ymm10, [srcreg+d2][rbp]		;; MemR3
	vsubpd	ymm10, ymm10, [srcreg+d2][r9]	;; MemR3
	vmovapd	ymm12, [srcreg+d2+32][rbp]	;; MemI3
	vsubpd	ymm12, ymm12, [srcreg+d2+32][r9];; MemI3
	vmulpd	ymm1, ymm7, ymm10		;; R3 * MemR3			; 16-20
	vmulpd	ymm7, ymm7, ymm12		;; R3 * MemI3			; 17-21

	yfnmaddpd ymm14, ymm6, ymm13, ymm14	;; R2*MemR2-I2*MemI2 (new R2)	; 19-23
	yfmaddpd ymm2, ymm6, ymm11, ymm2	;; R2*MemI2+I2*MemR2 (new I2)	; 20-24

	vmovapd	ymm11, [srcreg+d2+d1][rbp]	;; MemR4
	vsubpd	ymm11, ymm11, [srcreg+d2+d1][r9];; MemR4
	vmovapd	ymm13, [srcreg+d2+d1+32][rbp]	;; MemI4
	vsubpd	ymm13, ymm13, [srcreg+d2+d1+32][r9];; MemI4

	vmulpd	ymm3, ymm0, ymm11		;; R4 * MemR4			; 18-22
	vmulpd	ymm0, ymm0, ymm13		;; R4 * MemI4			; 19-23

	yfnmaddpd ymm1, ymm9, ymm12, ymm1	;; R3*MemR3-I3*MemI3 (new R3)	; 21-25
	yfmaddpd ymm7, ymm9, ymm10, ymm7	;; R3*MemI3+I3*MemR3 (new I3)	; 22-26

	yfnmaddpd ymm3, ymm4, ymm13, ymm3	;; R4*MemR4-I4*MemI4 (new R4)	; 23-27
	yfmaddpd ymm0, ymm4, ymm11, ymm0	;; R4*MemI4+I4*MemR4 (new I4)	; 24-28
	jmp	done

submul133:
	vmovapd	ymm10, [srcreg][rbp]		;; MemR1
	vsubpd	ymm10, ymm10, ymm8		;; MemR1-R1
	vmovapd	ymm12, [srcreg+32][rbp]		;; MemI1
	vsubpd	ymm12, ymm12, ymm3		;; MemI1-I1
	vmulpd	ymm5, ymm8, ymm10		;; R1 * MemR1			; 12-16
	vmulpd	ymm8, ymm8, ymm12		;; R1 * MemI1			; 13-17

	vmovapd	ymm11, [srcreg+d1][rbp]		;; MemR2
	vsubpd	ymm11, ymm11, ymm2		;; MemR2-R2
	vmovapd	ymm13, [srcreg+d1+32][rbp]	;; MemI2
	vsubpd	ymm13, ymm13, ymm6		;; MemI2-I2
	vmulpd	ymm14, ymm2, ymm11		;; R2 * MemR2			; 14-18
	vmulpd	ymm2, ymm2, ymm13		;; R2 * MemI2			; 15-19

	yfnmaddpd ymm5, ymm3, ymm12, ymm5	;; R1*MemR1-I1*MemI1 (new R1)	; 17-21
	yfmaddpd ymm8, ymm3, ymm10, ymm8	;; R1*MemI1+I1*MemR1 (new I1)	; 18-22

	vmovapd	ymm10, [srcreg+d2][rbp]		;; MemR3
	vsubpd	ymm10, ymm10, ymm7		;; MemR3-R3
	vmovapd	ymm12, [srcreg+d2+32][rbp]	;; MemI3
	vsubpd	ymm12, ymm12, ymm9		;; MemI3-I3
	vmulpd	ymm1, ymm7, ymm10		;; R3 * MemR3			; 16-20
	vmulpd	ymm7, ymm7, ymm12		;; R3 * MemI3			; 17-21

	yfnmaddpd ymm14, ymm6, ymm13, ymm14	;; R2*MemR2-I2*MemI2 (new R2)	; 19-23
	yfmaddpd ymm2, ymm6, ymm11, ymm2	;; R2*MemI2+I2*MemR2 (new I2)	; 20-24

	vmovapd	ymm11, [srcreg+d2+d1][rbp]	;; MemR4
	vsubpd	ymm11, ymm11, ymm0		;; MemR4-R4
	vmovapd	ymm13, [srcreg+d2+d1+32][rbp]	;; MemI4
	vsubpd	ymm13, ymm13, ymm4		;; MemI4-I4
	vmulpd	ymm3, ymm0, ymm11		;; R4 * MemR4			; 18-22
	vmulpd	ymm0, ymm0, ymm13		;; R4 * MemI4			; 19-23

	yfnmaddpd ymm1, ymm9, ymm12, ymm1	;; R3*MemR3-I3*MemI3 (new R3)	; 21-25
	yfmaddpd ymm7, ymm9, ymm10, ymm7	;; R3*MemI3+I3*MemR3 (new I3)	; 22-26

	yfnmaddpd ymm3, ymm4, ymm13, ymm3	;; R4*MemR4-I4*MemI4 (new R4)	; 23-27
	yfmaddpd ymm0, ymm4, ymm11, ymm0	;; R4*MemI4+I4*MemR4 (new I4)	; 24-28
	jmp	done

submul313:
	vsubpd	ymm10, ymm8, [srcreg][rbp]	;; R1-MemR1
	vsubpd	ymm12, ymm3, [srcreg+32][rbp]	;; I1-MemI1
	vmulpd	ymm5, ymm8, ymm10		;; R1 * MemR1			; 12-16
	vmulpd	ymm8, ymm8, ymm12		;; R1 * MemI1			; 13-17

	vsubpd	ymm11, ymm2, [srcreg+d1][rbp]	;; R2-MemR2
	vsubpd	ymm13, ymm6, [srcreg+d1+32][rbp];; I2-MemI2
	vmulpd	ymm14, ymm2, ymm11		;; R2 * MemR2			; 14-18
	vmulpd	ymm2, ymm2, ymm13		;; R2 * MemI2			; 15-19

	yfnmaddpd ymm5, ymm3, ymm12, ymm5	;; R1*MemR1-I1*MemI1 (new R1)	; 17-21
	yfmaddpd ymm8, ymm3, ymm10, ymm8	;; R1*MemI1+I1*MemR1 (new I1)	; 18-22

	vsubpd	ymm10, ymm7, [srcreg+d2][rbp]	;; R3-MemR3
	vsubpd	ymm12, ymm9, [srcreg+d2+32][rbp];; I3-MemI3
	vmulpd	ymm1, ymm7, ymm10		;; R3 * MemR3			; 16-20
	vmulpd	ymm7, ymm7, ymm12		;; R3 * MemI3			; 17-21

	yfnmaddpd ymm14, ymm6, ymm13, ymm14	;; R2*MemR2-I2*MemI2 (new R2)	; 19-23
	yfmaddpd ymm2, ymm6, ymm11, ymm2	;; R2*MemI2+I2*MemR2 (new I2)	; 20-24

	vsubpd	ymm11, ymm0, [srcreg+d2+d1][rbp];; R4-MemR4
	vsubpd	ymm13, ymm4, [srcreg+d2+d1+32][rbp];; I4-MemI4
	vmulpd	ymm3, ymm0, ymm11		;; R4 * MemR4			; 18-22
	vmulpd	ymm0, ymm0, ymm13		;; R4 * MemI4			; 19-23

	yfnmaddpd ymm1, ymm9, ymm12, ymm1	;; R3*MemR3-I3*MemI3 (new R3)	; 21-25
	yfmaddpd ymm7, ymm9, ymm10, ymm7	;; R3*MemI3+I3*MemR3 (new I3)	; 22-26

	yfnmaddpd ymm3, ymm4, ymm13, ymm3	;; R4*MemR4-I4*MemI4 (new R4)	; 23-27
	yfmaddpd ymm0, ymm4, ymm11, ymm0	;; R4*MemI4+I4*MemR4 (new I4)	; 24-28
	jmp	done

submul132:
	vmovapd	ymm10, [srcreg][rbp]		;; MemR1
	vsubpd	ymm10, ymm10, ymm8		;; MemR1-R1
	vmovapd	ymm8, [srcreg][r9]		;; R1=MemR1
	vmovapd	ymm12, [srcreg+32][rbp]		;; MemI1
	vsubpd	ymm12, ymm12, ymm3		;; MemI1-I1
	vmovapd	ymm3, [srcreg+32][r9]		;; I1=MemI1
	vmulpd	ymm5, ymm8, ymm10		;; R1 * MemR1			; 12-16
	vmulpd	ymm8, ymm8, ymm12		;; R1 * MemI1			; 13-17

	vmovapd	ymm11, [srcreg+d1][rbp]		;; MemR2
	vsubpd	ymm11, ymm11, ymm2		;; MemR2-R2
	vmovapd	ymm2, [srcreg+d1][r9]		;; R2=MemR2
	vmovapd	ymm13, [srcreg+d1+32][rbp]	;; MemI2
	vsubpd	ymm13, ymm13, ymm6		;; MemI2-I2
	vmovapd	ymm6, [srcreg+d1+32][r9]	;; I2=MemI2
	vmulpd	ymm14, ymm2, ymm11		;; R2 * MemR2			; 14-18
	vmulpd	ymm2, ymm2, ymm13		;; R2 * MemI2			; 15-19

	yfnmaddpd ymm5, ymm3, ymm12, ymm5	;; R1*MemR1-I1*MemI1 (new R1)	; 17-21
	yfmaddpd ymm8, ymm3, ymm10, ymm8	;; R1*MemI1+I1*MemR1 (new I1)	; 18-22

	vmovapd	ymm10, [srcreg+d2][rbp]		;; MemR3
	vsubpd	ymm10, ymm10, ymm7		;; MemR3-R3
	vmovapd	ymm7, [srcreg+d2][r9]		;; R3=MemR3
	vmovapd	ymm12, [srcreg+d2+32][rbp]	;; MemI3
	vsubpd	ymm12, ymm12, ymm9		;; MemI3-I3
	vmovapd	ymm9, [srcreg+d2+32][r9]	;; I3=MemI3
	vmulpd	ymm1, ymm7, ymm10		;; R3 * MemR3			; 16-20
	vmulpd	ymm7, ymm7, ymm12		;; R3 * MemI3			; 17-21

	yfnmaddpd ymm14, ymm6, ymm13, ymm14	;; R2*MemR2-I2*MemI2 (new R2)	; 19-23
	yfmaddpd ymm2, ymm6, ymm11, ymm2	;; R2*MemI2+I2*MemR2 (new I2)	; 20-24

	vmovapd	ymm11, [srcreg+d2+d1][rbp]	;; MemR4
	vsubpd	ymm11, ymm11, ymm0		;; MemR4-R4
	vmovapd	ymm0, [srcreg+d2+d1][r9]	;; R4=MemR4
	vmovapd	ymm13, [srcreg+d2+d1+32][rbp]	;; MemI4
	vsubpd	ymm13, ymm13, ymm4		;; MemI4-I4
	vmovapd	ymm4, [srcreg+d2+d1+32][r9]	;; I4=MemI4
	vmulpd	ymm3, ymm0, ymm11		;; R4 * MemR4			; 18-22
	vmulpd	ymm0, ymm0, ymm13		;; R4 * MemI4			; 19-23

	yfnmaddpd ymm1, ymm9, ymm12, ymm1	;; R3*MemR3-I3*MemI3 (new R3)	; 21-25
	yfmaddpd ymm7, ymm9, ymm10, ymm7	;; R3*MemI3+I3*MemR3 (new I3)	; 22-26

	yfnmaddpd ymm3, ymm4, ymm13, ymm3	;; R4*MemR4-I4*MemI4 (new R4)	; 23-27
	yfmaddpd ymm0, ymm4, ymm11, ymm0	;; R4*MemI4+I4*MemR4 (new I4)	; 24-28
	jmp	done

submul312:
	vsubpd	ymm10, ymm8, [srcreg][rbp]	;; R1-MemR1
	vmovapd	ymm8, [srcreg][r9]		;; R1=MemR1
	vsubpd	ymm12, ymm3, [srcreg+32][rbp]	;; I1-MemI1
	vmovapd	ymm3, [srcreg+32][r9]		;; I1=MemI1
	vmulpd	ymm5, ymm8, ymm10		;; R1 * MemR1			; 12-16
	vmulpd	ymm8, ymm8, ymm12		;; R1 * MemI1			; 13-17

	vsubpd	ymm11, ymm2, [srcreg+d1][rbp]	;; R2-MemR2
	vmovapd	ymm2, [srcreg+d1][r9]		;; R2=MemR2
	vsubpd	ymm13, ymm6, [srcreg+d1+32][rbp];; I2-MemI2
	vmovapd	ymm6, [srcreg+d1+32][r9]	;; I2=MemI2
	vmulpd	ymm14, ymm2, ymm11		;; R2 * MemR2			; 14-18
	vmulpd	ymm2, ymm2, ymm13		;; R2 * MemI2			; 15-19

	yfnmaddpd ymm5, ymm3, ymm12, ymm5	;; R1*MemR1-I1*MemI1 (new R1)	; 17-21
	yfmaddpd ymm8, ymm3, ymm10, ymm8	;; R1*MemI1+I1*MemR1 (new I1)	; 18-22

	vsubpd	ymm10, ymm7, [srcreg+d2][rbp]	;; R3-MemR3
	vmovapd	ymm7, [srcreg+d2][r9]		;; R3=MemR3
	vsubpd	ymm12, ymm9, [srcreg+d2+32][rbp];; I3-MemI3
	vmovapd	ymm9, [srcreg+d2+32][r9]	;; I3=MemI3
	vmulpd	ymm1, ymm7, ymm10		;; R3 * MemR3			; 16-20
	vmulpd	ymm7, ymm7, ymm12		;; R3 * MemI3			; 17-21

	yfnmaddpd ymm14, ymm6, ymm13, ymm14	;; R2*MemR2-I2*MemI2 (new R2)	; 19-23
	yfmaddpd ymm2, ymm6, ymm11, ymm2	;; R2*MemI2+I2*MemR2 (new I2)	; 20-24

	vsubpd	ymm11, ymm0, [srcreg+d2+d1][rbp];; R4-MemR4
	vmovapd	ymm0, [srcreg+d2+d1][r9]	;; R4=MemR4
	vsubpd	ymm13, ymm4, [srcreg+d2+d1+32][rbp];; I4-MemI4
	vmovapd	ymm4, [srcreg+d2+d1+32][r9]	;; I4=MemI4
	vmulpd	ymm3, ymm0, ymm11		;; R4 * MemR4			; 18-22
	vmulpd	ymm0, ymm0, ymm13		;; R4 * MemI4			; 19-23

	yfnmaddpd ymm1, ymm9, ymm12, ymm1	;; R3*MemR3-I3*MemI3 (new R3)	; 21-25
	yfmaddpd ymm7, ymm9, ymm10, ymm7	;; R3*MemI3+I3*MemR3 (new I3)	; 22-26

	yfnmaddpd ymm3, ymm4, ymm13, ymm3	;; R4*MemR4-I4*MemI4 (new R4)	; 23-27
	yfmaddpd ymm0, ymm4, ymm11, ymm0	;; R4*MemI4+I4*MemR4 (new I4)	; 24-28
	jmp	done

unfft:	vmovapd ymm5, ymm8			;; R1
	vmovapd ymm8, ymm3			;; I1
	vmovapd ymm14, ymm2			;; R2
	vmovapd ymm2, ymm6			;; I2
	vmovapd ymm3, ymm0			;; R4
	vmovapd	ymm0, ymm4			;; I4
	vmovapd ymm1, ymm7			;; R3
	vmovapd ymm7, ymm9			;; I3
	jmp	done

muladd:	vmovapd	ymm10, [srcreg][rbp]		;; MemR1
	vmovapd	ymm12, [srcreg+32][rbp]		;; MemI1
	vmulpd	ymm5, ymm8, ymm10		;; R1 * MemR1			; 12-16
	vmulpd	ymm8, ymm8, ymm12		;; R1 * MemI1			; 13-17

	vmovapd	ymm11, [srcreg+d1][rbp]		;; MemR2
	vmovapd	ymm13, [srcreg+d1+32][rbp]	;; MemI2
	vmulpd	ymm14, ymm2, ymm11		;; R2 * MemR2			; 14-18
	vmulpd	ymm2, ymm2, ymm13		;; R2 * MemI2			; 15-19

	yfnmaddpd ymm5, ymm3, ymm12, ymm5	;; R1*MemR1-I1*MemI1 (new R1)	; 17-21
	yfmaddpd ymm8, ymm3, ymm10, ymm8	;; R1*MemI1+I1*MemR1 (new I1)	; 18-22

	vmovapd	ymm10, [srcreg+d2][rbp]		;; MemR3
	vmovapd	ymm12, [srcreg+d2+32][rbp]	;; MemI3
	vmulpd	ymm1, ymm7, ymm10		;; R3 * MemR3			; 16-20
	vmulpd	ymm7, ymm7, ymm12		;; R3 * MemI3			; 17-21

	yfnmaddpd ymm14, ymm6, ymm13, ymm14	;; R2*MemR2-I2*MemI2 (new R2)	; 19-23
	yfmaddpd ymm2, ymm6, ymm11, ymm2	;; R2*MemI2+I2*MemR2 (new I2)	; 20-24

	vmovapd	ymm11, [srcreg+d2+d1][rbp]	;; MemR4
	vmovapd	ymm13, [srcreg+d2+d1+32][rbp]	;; MemI4

	vmulpd	ymm3, ymm0, ymm11		;; R4 * MemR4			; 18-22
	vmulpd	ymm0, ymm0, ymm13		;; R4 * MemI4			; 19-23

	yfnmaddpd ymm1, ymm9, ymm12, ymm1	;; R3*MemR3-I3*MemI3 (new R3)	; 21-25
	yfmaddpd ymm7, ymm9, ymm10, ymm7	;; R3*MemI3+I3*MemR3 (new I3)	; 22-26

	yfnmaddpd ymm3, ymm4, ymm13, ymm3	;; R4*MemR4-I4*MemI4 (new R4)	; 23-27
	yfmaddpd ymm0, ymm4, ymm11, ymm0	;; R4*MemI4+I4*MemR4 (new I4)	; 24-28

	mov	r10, SRC3ARG			;; Load distance to FFT(1)
	test	r10, 3				;; Check if we need to multiply add-in by FFT(1)
	jz	muladdhard			;; Yes, do it the hard way

	vaddpd	ymm5, ymm5, [srcreg+r9]			;; R1 = R1 + MemR1
	vaddpd	ymm8, ymm8, [srcreg+r9+32]		;; I1 = I1 + MemI1
	vaddpd	ymm14, ymm14, [srcreg+r9+d1]		;; R2 = R2 + MemR2
	vaddpd	ymm2, ymm2, [srcreg+r9+d1+32]		;; I2 = I2 + MemI2
	vaddpd	ymm1, ymm1, [srcreg+r9+d2]		;; R3 = R3 + MemR3
	vaddpd	ymm7, ymm7, [srcreg+r9+d2+32]		;; I3 = I3 + MemI3
	vaddpd	ymm3, ymm3, [srcreg+r9+d2+d1]		;; R4 = R4 + MemR4
	vaddpd	ymm0, ymm0, [srcreg+r9+d2+d1+32]	;; I4 = I4 + MemI4
	jmp	done

muladdhard:
	vmovapd	ymm4, [srcreg+r9]			;; MemR1
	vmovapd	ymm9, [srcreg+r10]			;; MemR1#2
	yfmaddpd ymm5, ymm4, ymm9, ymm5			;; R1 = R1 + MemR1*MemR1#2
	vmovapd	ymm10, [srcreg+r10+32]			;; MemI1#2
	yfmaddpd ymm8, ymm4, ymm10, ymm8		;; I1 = I1 + MemR1*MemI1#2
	vmovapd	ymm4, [srcreg+r9+32]			;; MemI1
	yfnmaddpd ymm5, ymm4, ymm10, ymm5		;; R1 = R1 - MemI1*MemI1#2
	yfmaddpd ymm8, ymm4, ymm9, ymm8			;; I1 = I1 + MemI1*MemR1#2

	vmovapd	ymm4, [srcreg+r9+d1]			;; MemR2
	vmovapd	ymm9, [srcreg+r10+d1]			;; MemR2#2
	yfmaddpd ymm14, ymm4, ymm9, ymm14		;; R2 = R2 + MemR2*MemR2#2
	vmovapd	ymm10, [srcreg+r10+d1+32]		;; MemI2#2
	yfmaddpd ymm2, ymm4, ymm10, ymm2		;; I2 = I2 + MemR2*MemI2#2
	vmovapd	ymm4, [srcreg+r9+d1+32]			;; MemI2
	yfnmaddpd ymm14, ymm4, ymm10, ymm14		;; R2 = R2 - MemI2*MemI2#2
	yfmaddpd ymm2, ymm4, ymm9, ymm2			;; I2 = I2 + MemI2*MemR2#2

	vmovapd	ymm4, [srcreg+r9+d2]			;; MemR3
	vmovapd	ymm9, [srcreg+r10+d2]			;; MemR3#2
	yfmaddpd ymm1, ymm4, ymm9, ymm1			;; R3 = R3 + MemR3*MemR3#2
	vmovapd	ymm10, [srcreg+r10+d2+32]		;; MemI3#2
	yfmaddpd ymm7, ymm4, ymm10, ymm7		;; I3 = I3 + MemR3*MemI3#2
	vmovapd	ymm4, [srcreg+r9+d2+32]			;; MemI3
	yfnmaddpd ymm1, ymm4, ymm10, ymm1		;; R3 = R3 - MemI3*MemI3#2
	yfmaddpd ymm7, ymm4, ymm9, ymm7			;; I3 = I3 + MemI3*MemR3#2

	vmovapd	ymm4, [srcreg+r9+d2+d1]			;; MemR4
	vmovapd	ymm9, [srcreg+r10+d2+d1]		;; MemR4#2
	yfmaddpd ymm3, ymm4, ymm9, ymm3			;; R4 = R4 + MemR4*MemR4#2
	vmovapd	ymm10, [srcreg+r10+d2+d1+32]		;; MemI4#2
	yfmaddpd ymm0, ymm4, ymm10, ymm0		;; I4 = I4 + MemR4*MemI4#2
	vmovapd	ymm4, [srcreg+r9+d2+d1+32]		;; MemI4
	yfnmaddpd ymm3, ymm4, ymm10, ymm3		;; R4 = R4 - MemI4*MemI4#2
	yfmaddpd ymm0, ymm4, ymm9, ymm0			;; I4 = I4 + MemI4*MemR4#2
	jmp	done

mulsub:	vmovapd	ymm10, [srcreg][rbp]		;; MemR1
	vmovapd	ymm12, [srcreg+32][rbp]		;; MemI1
	vmulpd	ymm5, ymm8, ymm10		;; R1 * MemR1			; 12-16
	vmulpd	ymm8, ymm8, ymm12		;; R1 * MemI1			; 13-17

	vmovapd	ymm11, [srcreg+d1][rbp]		;; MemR2
	vmovapd	ymm13, [srcreg+d1+32][rbp]	;; MemI2
	vmulpd	ymm14, ymm2, ymm11		;; R2 * MemR2			; 14-18
	vmulpd	ymm2, ymm2, ymm13		;; R2 * MemI2			; 15-19

	yfnmaddpd ymm5, ymm3, ymm12, ymm5	;; R1*MemR1-I1*MemI1 (new R1)	; 17-21
	yfmaddpd ymm8, ymm3, ymm10, ymm8	;; R1*MemI1+I1*MemR1 (new I1)	; 18-22

	vmovapd	ymm10, [srcreg+d2][rbp]		;; MemR3
	vmovapd	ymm12, [srcreg+d2+32][rbp]	;; MemI3
	vmulpd	ymm1, ymm7, ymm10		;; R3 * MemR3			; 16-20
	vmulpd	ymm7, ymm7, ymm12		;; R3 * MemI3			; 17-21

	yfnmaddpd ymm14, ymm6, ymm13, ymm14	;; R2*MemR2-I2*MemI2 (new R2)	; 19-23
	yfmaddpd ymm2, ymm6, ymm11, ymm2	;; R2*MemI2+I2*MemR2 (new I2)	; 20-24

	vmovapd	ymm11, [srcreg+d2+d1][rbp]	;; MemR4
	vmovapd	ymm13, [srcreg+d2+d1+32][rbp]	;; MemI4

	vmulpd	ymm3, ymm0, ymm11		;; R4 * MemR4			; 18-22
	vmulpd	ymm0, ymm0, ymm13		;; R4 * MemI4			; 19-23

	yfnmaddpd ymm1, ymm9, ymm12, ymm1	;; R3*MemR3-I3*MemI3 (new R3)	; 21-25
	yfmaddpd ymm7, ymm9, ymm10, ymm7	;; R3*MemI3+I3*MemR3 (new I3)	; 22-26

	yfnmaddpd ymm3, ymm4, ymm13, ymm3	;; R4*MemR4-I4*MemI4 (new R4)	; 23-27
	yfmaddpd ymm0, ymm4, ymm11, ymm0	;; R4*MemI4+I4*MemR4 (new I4)	; 24-28

	mov	r10, SRC3ARG			;; Load distance to FFT(1)
	test	r10, 3				;; Check if we need to multiply add-in by FFT(1)
	jz	mulsubhard			;; Yes, do it the hard way

	vsubpd	ymm5, ymm5, [srcreg+r9]			;; R1 = R1 + MemR1
	vsubpd	ymm8, ymm8, [srcreg+r9+32]		;; I1 = I1 + MemI1
	vsubpd	ymm14, ymm14, [srcreg+r9+d1]		;; R2 = R2 + MemR2
	vsubpd	ymm2, ymm2, [srcreg+r9+d1+32]		;; I2 = I2 + MemI2
	vsubpd	ymm1, ymm1, [srcreg+r9+d2]		;; R3 = R3 + MemR3
	vsubpd	ymm7, ymm7, [srcreg+r9+d2+32]		;; I3 = I3 + MemI3
	vsubpd	ymm3, ymm3, [srcreg+r9+d2+d1]		;; R4 = R4 + MemR4
	vsubpd	ymm0, ymm0, [srcreg+r9+d2+d1+32]	;; I4 = I4 + MemI4
	jmp	done

mulsubhard:
	vmovapd	ymm4, [srcreg+r9]			;; MemR1
	vmovapd	ymm9, [srcreg+r10]			;; MemR1#2
	yfnmaddpd ymm5, ymm4, ymm9, ymm5		;; R1 = R1 - MemR1*MemR1#2
	vmovapd	ymm10, [srcreg+r10+32]			;; MemI1#2
	yfnmaddpd ymm8, ymm4, ymm10, ymm8		;; I1 = I1 - MemR1*MemI1#2
	vmovapd	ymm4, [srcreg+r9+32]			;; MemI1
	yfmaddpd ymm5, ymm4, ymm10, ymm5		;; R1 = R1 + MemI1*MemI1#2
	yfnmaddpd ymm8, ymm4, ymm9, ymm8		;; I1 = I1 - MemI1*MemR1#2

	vmovapd	ymm4, [srcreg+r9+d1]			;; MemR2
	vmovapd	ymm9, [srcreg+r10+d1]			;; MemR2#2
	yfnmaddpd ymm14, ymm4, ymm9, ymm14		;; R2 = R2 - MemR2*MemR2#2
	vmovapd	ymm10, [srcreg+r10+d1+32]		;; MemI2#2
	yfnmaddpd ymm2, ymm4, ymm10, ymm2		;; I2 = I2 - MemR2*MemI2#2
	vmovapd	ymm4, [srcreg+r9+d1+32]			;; MemI2
	yfmaddpd ymm14, ymm4, ymm10, ymm14		;; R2 = R2 + MemI2*MemI2#2
	yfnmaddpd ymm2, ymm4, ymm9, ymm2		;; I2 = I2 - MemI2*MemR2#2

	vmovapd	ymm4, [srcreg+r9+d2]			;; MemR3
	vmovapd	ymm9, [srcreg+r10+d2]			;; MemR3#2
	yfnmaddpd ymm1, ymm4, ymm9, ymm1		;; R3 = R3 - MemR3*MemR3#2
	vmovapd	ymm10, [srcreg+r10+d2+32]		;; MemI3#2
	yfnmaddpd ymm7, ymm4, ymm10, ymm7		;; I3 = I3 - MemR3*MemI3#2
	vmovapd	ymm4, [srcreg+r9+d2+32]			;; MemI3
	yfmaddpd ymm1, ymm4, ymm10, ymm1		;; R3 = R3 + MemI3*MemI3#2
	yfnmaddpd ymm7, ymm4, ymm9, ymm7		;; I3 = I3 - MemI3*MemR3#2

	vmovapd	ymm4, [srcreg+r9+d2+d1]			;; MemR4
	vmovapd	ymm9, [srcreg+r10+d2+d1]		;; MemR4#2
	yfnmaddpd ymm3, ymm4, ymm9, ymm3		;; R4 = R4 - MemR4*MemR4#2
	vmovapd	ymm10, [srcreg+r10+d2+d1+32]		;; MemI4#2
	yfnmaddpd ymm0, ymm4, ymm10, ymm0		;; I4 = I4 - MemR4*MemI4#2
	vmovapd	ymm4, [srcreg+r9+d2+d1+32]		;; MemI4
	yfmaddpd ymm3, ymm4, ymm10, ymm3		;; R4 = R4 + MemI4*MemI4#2
	yfnmaddpd ymm0, ymm4, ymm9, ymm0		;; I4 = I4 - MemI4*MemR4#2

done:
	ENDM

ENDIF

ENDIF

yr4_4cl_four_complex_with_mulf_preload MACRO
	ENDM
yr4_4cl_four_complex_with_mulf MACRO srcreg,srcinc,d1,d2
	vmovapd	ymm2, [srcreg+d1][rbx]		;; R2
	vmovapd	ymm1, [srcreg+d1+32][rbx]	;; I2
	yp_complex_mult ymm2, ymm1, [srcreg+d1][rbp], [srcreg+d1+32][rbp], ymm4, ymm5 ;; Mult R2, I2
	vmovapd	ymm7, [srcreg+d2][rbx]		;; R3
	vmovapd	ymm3, [srcreg+d2+32][rbx]	;; I3
	yp_complex_mult ymm7, ymm3, [srcreg+d2][rbp], [srcreg+d2+32][rbp], ymm4, ymm5 ;; Mult R3, I3
	vmovapd	ymm0, [srcreg+d2+d1][rbx]	;; R4
	vmovapd	ymm6, [srcreg+d2+d1+32][rbx]	;; I4
	yp_complex_mult ymm0, ymm6, [srcreg+d2+d1][rbp], [srcreg+d2+d1+32][rbp], ymm4, ymm5 ;; Mult R4, I4
	ystore	[srcreg+d1], ymm2		;; Save R2
	ystore	[srcreg+d1+32], ymm1		;; Save I2
	ystore	[srcreg+d2], ymm7		;; Save R3
	ystore	[srcreg+d2+32], ymm3		;; Save I3
	ystore	[srcreg+d2+d1], ymm0		;; Save R4
	ystore	[srcreg+d2+d1+32], ymm6		;; Save I4
	vmovapd	ymm2, [srcreg][rbx]		;; R1
	vmovapd	ymm4, [srcreg+32][rbx]		;; I1
	yp_complex_mult ymm2, ymm4, [srcreg][rbp], [srcreg+32][rbp], ymm3, ymm5 ;; Mult R1, I1
	ystore	[srcreg], ymm2			;; Save R1
	ystore	[srcreg+32], ymm4		;; Save I1
	yr4_4c_simple_unfft_mem [srcreg],[srcreg+32],[srcreg+d1],[srcreg+d1+32],[srcreg+d2],[srcreg+d2+32],[srcreg+d2+d1],[srcreg+d2+d1+32],[srcreg]
;;	ystore	[srcreg], ymm0			;; Save R1
	ystore	[srcreg+32], ymm4		;; Save I1
	ystore	[srcreg+d1], ymm5		;; Save R2
	ystore	[srcreg+d1+32], ymm7		;; Save I2
	ystore	[srcreg+d2], ymm2		;; Save R3
	ystore	[srcreg+d2+32], ymm3		;; Save I3
	ystore	[srcreg+d2+d1], ymm1		;; Save R4
	ystore	[srcreg+d2+d1+32], ymm6		;; Save I4
	bump	srcreg, srcinc
	ENDM

;; 64-bit version

IFDEF X86_64

yr4_4cl_four_complex_with_mulf_preload MACRO
	ENDM
yr4_4cl_four_complex_with_mulf MACRO srcreg,srcinc,d1,d2
	LOCAL	orig, back_to_orig

	cmp	mul4_opcode, 0			;; See if we need to do more than the original type-4 FFT multiply
	je	short orig
	yr4_dispatch CALLP,ycomplex_mulf_opcode,d1,d2 ;; Handle more difficult cases
	jmp	back_to_orig

orig:	vmovapd	ymm10, [srcreg][rbp]		;; MemR1
	vmovapd	ymm0, [srcreg][rbx]		;; R1
	vmovapd	ymm11, [srcreg+32][rbp]		;; MemI1
	vmovapd	ymm1, [srcreg+32][rbx]		;; I1
	vmulpd	ymm2, ymm0, ymm10		;; R1 * MemR1			; 1-5
	vmulpd	ymm0, ymm0, ymm11		;; R1 * MemI1			; 2-6
	vmulpd	ymm3, ymm1, ymm11		;; I1 * MemI1			; 3-7
	vmulpd	ymm1, ymm1, ymm10		;; I1 * MemR1			; 4-8

	vmovapd	ymm14, [srcreg+d1][rbp]		;; MemR2
	vmovapd	ymm4, [srcreg+d1][rbx]		;; R2
	vmovapd	ymm15, [srcreg+d1+32][rbp]	;; MemI2
	vmovapd	ymm5, [srcreg+d1+32][rbx]	;; I2
	vmulpd	ymm6, ymm4, ymm14		;; R2 * MemR2			; 5-9
	vmulpd	ymm4, ymm4, ymm15		;; R2 * MemI2			; 6-10
	vmulpd	ymm7, ymm5, ymm15		;; I2 * MemI2			; 7-11
	vmulpd	ymm5, ymm5, ymm14		;; I2 * MemR2			; 8-12

	vsubpd	ymm2, ymm2, ymm3		;; R1*MemR1 - I1*MemI1 (new R1)	; 8-10
	vaddpd	ymm0, ymm0, ymm1		;; R1*MemI1 + I1*MemR1 (new I1)	; 9-11

	vmovapd	ymm14, [srcreg+d2][rbp]		;; MemR3
	vmovapd	ymm1, [srcreg+d2][rbx]		;; R3
	vmovapd	ymm15, [srcreg+d2+32][rbp]	;; MemI3
	vmovapd	ymm3, [srcreg+d2+32][rbx]	;; I3
	vmulpd	ymm8, ymm1, ymm14		;; R3 * MemR3			; 9-13
	vmulpd	ymm1, ymm1, ymm15		;; R3 * MemI3			; 10-14
	vmulpd	ymm9, ymm3, ymm15		;; I3 * MemI3			; 11-15
	vmulpd	ymm3, ymm3, ymm14		;; I3 * MemR3			; 12-16

	vsubpd	ymm6, ymm6, ymm7		;; R2*MemR2 - I2*MemI2 (new R2)	; 12-14
	vaddpd	ymm4, ymm4, ymm5		;; R2*MemI2 + I2*MemR2 (new I2)	; 13-15

	vmovapd	ymm14, [srcreg+d2+d1][rbp]	;; MemR4
	vmovapd	ymm5, [srcreg+d2+d1][rbx]	;; R4
	vmovapd	ymm15, [srcreg+d2+d1+32][rbp]	;; MemI4
	vmovapd	ymm7, [srcreg+d2+d1+32][rbx]	;; I4

	vmulpd	ymm10, ymm5, ymm14		;; R4 * MemR4			; 13-17
	vmulpd	ymm5, ymm5, ymm15		;; R4 * MemI4			; 14-18
	vmulpd	ymm11, ymm7, ymm15		;; I4 * MemI4			; 15-19
	vmulpd	ymm7, ymm7, ymm14		;; I4 * MemR4			; 16-20

	vaddpd	ymm12, ymm2, ymm6		;; R1 + R2 (new R1)		; 15-17
	vsubpd	ymm2, ymm2, ymm6		;; R1 - R2 (new R2)		; 16-17

	vsubpd	ymm8, ymm8, ymm9		;; R3*MemR3 - I3*MemI3 (new R3)	; 17-19
	vaddpd	ymm1, ymm1, ymm3		;; R3*MemI3 + I3*MemR3 (new I3)	; 18-20

	vsubpd	ymm10, ymm10, ymm11		;; R4*MemR4 - I4*MemI4 (new R4)	; 20-22
	vaddpd	ymm5, ymm5, ymm7		;; R4*MemI4 + I4*MemR4 (new I4)	; 21-23

back_to_orig:
	vaddpd	ymm3, ymm0, ymm4		;; I1 + I2 (new I1)		; 19-21
	vsubpd	ymm0, ymm0, ymm4		;; I1 - I2 (new I2)		; 22-24

	vaddpd	ymm6, ymm10, ymm8		;; R4 + R3 (new R3)		; 23-25
	vsubpd	ymm10, ymm10, ymm8		;; R4 - R3 (new I4)		; 24-26

	vaddpd	ymm4, ymm1, ymm5		;; I3 + I4 (new I3)		; 25-27
	vsubpd	ymm1, ymm1, ymm5		;; I3 - I4 (new R4)		; 26-28

	vaddpd	ymm5, ymm12, ymm6		;; R1 + R3 (final R1)		; 27-29
	vsubpd	ymm12, ymm12, ymm6		;; R1 - R3 (final R3)		; 28-30

	vaddpd	ymm6, ymm0, ymm10		;; I2 + I4 (final I2)		; 29-31
	vsubpd	ymm0, ymm0, ymm10		;; I2 - I4 (final I4)		; 30-32

	vaddpd	ymm7, ymm2, ymm1		;; R2 + R4 (final R2)		; 31-33
	vsubpd	ymm2, ymm2, ymm1		;; R2 - R4 (final R4)		; 32-34

	vaddpd	ymm1, ymm3, ymm4		;; I1 + I3 (final I1)		; 33-35
	vsubpd	ymm3, ymm3, ymm4		;; I1 - I3 (final I3)		; 34-36

	ystore	[srcreg], ymm5			;; Save R1
	ystore	[srcreg+32], ymm1		;; Save I1
	ystore	[srcreg+d1], ymm7		;; Save R2
	ystore	[srcreg+d1+32], ymm6		;; Save I2
	ystore	[srcreg+d2], ymm12		;; Save R3
	ystore	[srcreg+d2+32], ymm3		;; Save I3
	ystore	[srcreg+d2+d1], ymm2		;; Save R4
	ystore	[srcreg+d2+d1+32], ymm0		;; Save I4
	bump	srcreg, srcinc
	ENDM

;; Same as above code between orig and back_to_orig, except we implement mul4_opcode options

yr4_4c_mulf_opcode MACRO srcreg,d1,d2
	LOCAL	fma_or_unfft, fma, addmul, submul, muladd, muladdhard, mulsub, mulsubhard, unfft, done

	movzx	r10, mul4_opcode		;; Load the mul4_opcode
	mov	r9, SRC2ARG			;; Distance to s2 arg from gwaddmul4 or gwsubmul4 or s3 arg in gwmuladd4 or gwmulsub4
	cmp	r10, 2				;; Case off opcode
	jg	fma_or_unfft			;; 3,4,5=muladd,mulsub,unfft
	je	submul				;; 2=submul
	;;jl	addmul				;; 1=addmul

addmul:	vmovapd	ymm10, [srcreg][rbp]		;; MemR1
	vaddpd	ymm10, ymm10, [srcreg][r9]	;; MemR1
	vmovapd	ymm0, [srcreg][rbx]		;; R1
	vmovapd	ymm11, [srcreg+32][rbp]		;; MemI1
	vaddpd	ymm11, ymm11, [srcreg+32][r9]	;; MemI1
	vmovapd	ymm1, [srcreg+32][rbx]		;; I1
	vmulpd	ymm2, ymm0, ymm10		;; R1 * MemR1			; 1-5
	vmulpd	ymm0, ymm0, ymm11		;; R1 * MemI1			; 2-6
	vmulpd	ymm3, ymm1, ymm11		;; I1 * MemI1			; 3-7
	vmulpd	ymm1, ymm1, ymm10		;; I1 * MemR1			; 4-8

	vmovapd	ymm14, [srcreg+d1][rbp]		;; MemR2
	vaddpd	ymm14, ymm14, [srcreg+d1][r9]	;; MemR2
	vmovapd	ymm4, [srcreg+d1][rbx]		;; R2
	vmovapd	ymm15, [srcreg+d1+32][rbp]	;; MemI2
	vaddpd	ymm15, ymm15, [srcreg+d1+32][r9];; MemI2
	vmovapd	ymm5, [srcreg+d1+32][rbx]	;; I2
	vmulpd	ymm6, ymm4, ymm14		;; R2 * MemR2			; 5-9
	vmulpd	ymm4, ymm4, ymm15		;; R2 * MemI2			; 6-10
	vmulpd	ymm7, ymm5, ymm15		;; I2 * MemI2			; 7-11
	vmulpd	ymm5, ymm5, ymm14		;; I2 * MemR2			; 8-12

	vsubpd	ymm2, ymm2, ymm3		;; R1*MemR1 - I1*MemI1 (new R1)	; 8-10
	vaddpd	ymm0, ymm0, ymm1		;; R1*MemI1 + I1*MemR1 (new I1)	; 9-11

	vmovapd	ymm14, [srcreg+d2][rbp]		;; MemR3
	vaddpd	ymm14, ymm14, [srcreg+d2][r9]	;; MemR3
	vmovapd	ymm1, [srcreg+d2][rbx]		;; R3
	vmovapd	ymm15, [srcreg+d2+32][rbp]	;; MemI3
	vaddpd	ymm15, ymm15, [srcreg+d2+32][r9];; MemI3
	vmovapd	ymm3, [srcreg+d2+32][rbx]	;; I3
	vmulpd	ymm8, ymm1, ymm14		;; R3 * MemR3			; 9-13
	vmulpd	ymm1, ymm1, ymm15		;; R3 * MemI3			; 10-14
	vmulpd	ymm9, ymm3, ymm15		;; I3 * MemI3			; 11-15
	vmulpd	ymm3, ymm3, ymm14		;; I3 * MemR3			; 12-16

	vsubpd	ymm6, ymm6, ymm7		;; R2*MemR2 - I2*MemI2 (new R2)	; 12-14
	vaddpd	ymm4, ymm4, ymm5		;; R2*MemI2 + I2*MemR2 (new I2)	; 13-15

	vmovapd	ymm14, [srcreg+d2+d1][rbp]	;; MemR4
	vaddpd	ymm14, ymm14, [srcreg+d2+d1][r9];; MemR4
	vmovapd	ymm5, [srcreg+d2+d1][rbx]	;; R4
	vmovapd	ymm15, [srcreg+d2+d1+32][rbp]	;; MemI4
	vaddpd	ymm15, ymm15, [srcreg+d2+d1+32][r9];; MemI4
	vmovapd	ymm7, [srcreg+d2+d1+32][rbx]	;; I4

	vmulpd	ymm10, ymm5, ymm14		;; R4 * MemR4			; 13-17
	vmulpd	ymm5, ymm5, ymm15		;; R4 * MemI4			; 14-18
	vmulpd	ymm11, ymm7, ymm15		;; I4 * MemI4			; 15-19
	vmulpd	ymm7, ymm7, ymm14		;; I4 * MemR4			; 16-20

	vaddpd	ymm12, ymm2, ymm6		;; R1 + R2 (new R1)		; 15-17
	vsubpd	ymm2, ymm2, ymm6		;; R1 - R2 (new R2)		; 16-17

	vsubpd	ymm8, ymm8, ymm9		;; R3*MemR3 - I3*MemI3 (new R3)	; 17-19
	vaddpd	ymm1, ymm1, ymm3		;; R3*MemI3 + I3*MemR3 (new I3)	; 18-20

	vsubpd	ymm10, ymm10, ymm11		;; R4*MemR4 - I4*MemI4 (new R4)	; 20-22
	vaddpd	ymm5, ymm5, ymm7		;; R4*MemI4 + I4*MemR4 (new I4)	; 21-23
	jmp	done

submul:	vmovapd	ymm10, [srcreg][rbp]		;; MemR1
	vsubpd	ymm10, ymm10, [srcreg][r9]	;; MemR1
	vmovapd	ymm0, [srcreg][rbx]		;; R1
	vmovapd	ymm11, [srcreg+32][rbp]		;; MemI1
	vsubpd	ymm11, ymm11, [srcreg+32][r9]	;; MemI1
	vmovapd	ymm1, [srcreg+32][rbx]		;; I1
	vmulpd	ymm2, ymm0, ymm10		;; R1 * MemR1			; 1-5
	vmulpd	ymm0, ymm0, ymm11		;; R1 * MemI1			; 2-6
	vmulpd	ymm3, ymm1, ymm11		;; I1 * MemI1			; 3-7
	vmulpd	ymm1, ymm1, ymm10		;; I1 * MemR1			; 4-8

	vmovapd	ymm14, [srcreg+d1][rbp]		;; MemR2
	vsubpd	ymm14, ymm14, [srcreg+d1][r9]	;; MemR2
	vmovapd	ymm4, [srcreg+d1][rbx]		;; R2
	vmovapd	ymm15, [srcreg+d1+32][rbp]	;; MemI2
	vsubpd	ymm15, ymm15, [srcreg+d1+32][r9];; MemI2
	vmovapd	ymm5, [srcreg+d1+32][rbx]	;; I2
	vmulpd	ymm6, ymm4, ymm14		;; R2 * MemR2			; 5-9
	vmulpd	ymm4, ymm4, ymm15		;; R2 * MemI2			; 6-10
	vmulpd	ymm7, ymm5, ymm15		;; I2 * MemI2			; 7-11
	vmulpd	ymm5, ymm5, ymm14		;; I2 * MemR2			; 8-12

	vsubpd	ymm2, ymm2, ymm3		;; R1*MemR1 - I1*MemI1 (new R1)	; 8-10
	vaddpd	ymm0, ymm0, ymm1		;; R1*MemI1 + I1*MemR1 (new I1)	; 9-11

	vmovapd	ymm14, [srcreg+d2][rbp]		;; MemR3
	vsubpd	ymm14, ymm14, [srcreg+d2][r9]	;; MemR3
	vmovapd	ymm1, [srcreg+d2][rbx]		;; R3
	vmovapd	ymm15, [srcreg+d2+32][rbp]	;; MemI3
	vsubpd	ymm15, ymm15, [srcreg+d2+32][r9];; MemI3
	vmovapd	ymm3, [srcreg+d2+32][rbx]	;; I3
	vmulpd	ymm8, ymm1, ymm14		;; R3 * MemR3			; 9-13
	vmulpd	ymm1, ymm1, ymm15		;; R3 * MemI3			; 10-14
	vmulpd	ymm9, ymm3, ymm15		;; I3 * MemI3			; 11-15
	vmulpd	ymm3, ymm3, ymm14		;; I3 * MemR3			; 12-16

	vsubpd	ymm6, ymm6, ymm7		;; R2*MemR2 - I2*MemI2 (new R2)	; 12-14
	vaddpd	ymm4, ymm4, ymm5		;; R2*MemI2 + I2*MemR2 (new I2)	; 13-15

	vmovapd	ymm14, [srcreg+d2+d1][rbp]	;; MemR4
	vsubpd	ymm14, ymm14, [srcreg+d2+d1][r9];; MemR4
	vmovapd	ymm5, [srcreg+d2+d1][rbx]	;; R4
	vmovapd	ymm15, [srcreg+d2+d1+32][rbp]	;; MemI4
	vsubpd	ymm15, ymm15, [srcreg+d2+d1+32][r9];; MemI4
	vmovapd	ymm7, [srcreg+d2+d1+32][rbx]	;; I4

	vmulpd	ymm10, ymm5, ymm14		;; R4 * MemR4			; 13-17
	vmulpd	ymm5, ymm5, ymm15		;; R4 * MemI4			; 14-18
	vmulpd	ymm11, ymm7, ymm15		;; I4 * MemI4			; 15-19
	vmulpd	ymm7, ymm7, ymm14		;; I4 * MemR4			; 16-20

	vaddpd	ymm12, ymm2, ymm6		;; R1 + R2 (new R1)		; 15-17
	vsubpd	ymm2, ymm2, ymm6		;; R1 - R2 (new R2)		; 16-17

	vsubpd	ymm8, ymm8, ymm9		;; R3*MemR3 - I3*MemI3 (new R3)	; 17-19
	vaddpd	ymm1, ymm1, ymm3		;; R3*MemI3 + I3*MemR3 (new I3)	; 18-20

	vsubpd	ymm10, ymm10, ymm11		;; R4*MemR4 - I4*MemI4 (new R4)	; 20-22
	vaddpd	ymm5, ymm5, ymm7		;; R4*MemI4 + I4*MemR4 (new I4)	; 21-23
	jmp	done

fma_or_unfft:
	cmp	r10, 4				;; Case off opcode
	jbe	fma				;; 3=muladd, 4=mulsub, fma MUST NOT ALTER CONDITION CODES!
	;;ja	unfft				;; 5=unfft

unfft:	vmovapd	ymm2, [srcreg][rbx]		;; R1
	vmovapd	ymm6, [srcreg+d1][rbx]		;; R2
	vaddpd	ymm12, ymm2, ymm6		;; R1 + R2 (new R1)		; 15-17
	vsubpd	ymm2, ymm2, ymm6		;; R1 - R2 (new R2)		; 16-17
	vmovapd	ymm0, [srcreg+32][rbx]		;; I1
	vmovapd	ymm4, [srcreg+d1+32][rbx]	;; I2
	vmovapd	ymm8, [srcreg+d2][rbx]		;; R3
	vmovapd	ymm1, [srcreg+d2+32][rbx]	;; I3
	vmovapd	ymm10, [srcreg+d2+d1][rbx]	;; R4
	vmovapd	ymm5, [srcreg+d2+d1+32][rbx]	;; I4
	jmp	done

fma:	vmovapd	ymm10, [srcreg][rbp]		;; MemR1
	vmovapd	ymm0, [srcreg][rbx]		;; R1
	vmovapd	ymm11, [srcreg+32][rbp]		;; MemI1
	vmovapd	ymm1, [srcreg+32][rbx]		;; I1
	vmulpd	ymm2, ymm0, ymm10		;; R1 * MemR1			; 1-5
	vmulpd	ymm0, ymm0, ymm11		;; R1 * MemI1			; 2-6
	vmulpd	ymm3, ymm1, ymm11		;; I1 * MemI1			; 3-7
	vmulpd	ymm1, ymm1, ymm10		;; I1 * MemR1			; 4-8

	vmovapd	ymm14, [srcreg+d1][rbp]		;; MemR2
	vmovapd	ymm4, [srcreg+d1][rbx]		;; R2
	vmovapd	ymm15, [srcreg+d1+32][rbp]	;; MemI2
	vmovapd	ymm5, [srcreg+d1+32][rbx]	;; I2
	vmulpd	ymm6, ymm4, ymm14		;; R2 * MemR2			; 5-9
	vmulpd	ymm4, ymm4, ymm15		;; R2 * MemI2			; 6-10
	vmulpd	ymm7, ymm5, ymm15		;; I2 * MemI2			; 7-11
	vmulpd	ymm5, ymm5, ymm14		;; I2 * MemR2			; 8-12

	vsubpd	ymm2, ymm2, ymm3		;; R1*MemR1 - I1*MemI1 (new R1)	; 8-10
	vaddpd	ymm0, ymm0, ymm1		;; R1*MemI1 + I1*MemR1 (new I1)	; 9-11

	vmovapd	ymm14, [srcreg+d2][rbp]		;; MemR3
	vmovapd	ymm1, [srcreg+d2][rbx]		;; R3
	vmovapd	ymm15, [srcreg+d2+32][rbp]	;; MemI3
	vmovapd	ymm3, [srcreg+d2+32][rbx]	;; I3
	vmulpd	ymm8, ymm1, ymm14		;; R3 * MemR3			; 9-13
	vmulpd	ymm1, ymm1, ymm15		;; R3 * MemI3			; 10-14
	vmulpd	ymm9, ymm3, ymm15		;; I3 * MemI3			; 11-15
	vmulpd	ymm3, ymm3, ymm14		;; I3 * MemR3			; 12-16

	vsubpd	ymm6, ymm6, ymm7		;; R2*MemR2 - I2*MemI2 (new R2)	; 12-14
	vaddpd	ymm4, ymm4, ymm5		;; R2*MemI2 + I2*MemR2 (new I2)	; 13-15

	vmovapd	ymm14, [srcreg+d2+d1][rbp]	;; MemR4
	vmovapd	ymm5, [srcreg+d2+d1][rbx]	;; R4
	vmovapd	ymm15, [srcreg+d2+d1+32][rbp]	;; MemI4
	vmovapd	ymm7, [srcreg+d2+d1+32][rbx]	;; I4

	vmulpd	ymm10, ymm5, ymm14		;; R4 * MemR4			; 13-17
	vmulpd	ymm5, ymm5, ymm15		;; R4 * MemI4			; 14-18
	vmulpd	ymm11, ymm7, ymm15		;; I4 * MemI4			; 15-19
	vmulpd	ymm7, ymm7, ymm14		;; I4 * MemR4			; 16-20

	vsubpd	ymm8, ymm8, ymm9		;; R3*MemR3 - I3*MemI3 (new R3)	; 17-19
	vaddpd	ymm1, ymm1, ymm3		;; R3*MemI3 + I3*MemR3 (new I3)	; 18-20

	vsubpd	ymm10, ymm10, ymm11		;; R4*MemR4 - I4*MemI4 (new R4)	; 20-22
	vaddpd	ymm5, ymm5, ymm7		;; R4*MemI4 + I4*MemR4 (new I4)	; 21-23

	mov	r10, SRC3ARG			;; Load distance to FFT(1)
	je	mulsub				;; 4=mulsub
	;jb	muladd				;; 3=muladd, fall through

muladd:	test	r10, 3				;; Check if we need to multiply add-in by FFT(1)
	jz	muladdhard			;; Yes, do it the hard way

	vaddpd	ymm2, ymm2, [srcreg+r9]			;; R1 = R1 + MemR1
	vaddpd	ymm0, ymm0, [srcreg+r9+32]		;; I1 = I1 + MemI1
	vaddpd	ymm6, ymm6, [srcreg+r9+d1]		;; R2 = R2 + MemR2
	vaddpd	ymm4, ymm4, [srcreg+r9+d1+32]		;; I2 = I2 + MemI2
	vaddpd	ymm8, ymm8, [srcreg+r9+d2]		;; R3 = R3 + MemR3
	vaddpd	ymm1, ymm1, [srcreg+r9+d2+32]		;; I3 = I3 + MemI3
	vaddpd	ymm10, ymm10, [srcreg+r9+d2+d1]		;; R4 = R4 + MemR4
	vaddpd	ymm5, ymm5, [srcreg+r9+d2+d1+32]	;; I4 = I4 + MemI4

	vaddpd	ymm12, ymm2, ymm6		;; R1 + R2 (new R1)		; 15-17
	vsubpd	ymm2, ymm2, ymm6		;; R1 - R2 (new R2)		; 16-17
	jmp	done

muladdhard:
	vmovapd	ymm3, [srcreg+r9]			;; MemR1
	vmovapd	ymm9, [srcreg+r10]			;; MemR1#2
	vmulpd	ymm11, ymm3, ymm9			;; MemR1*MemR1#2
	vaddpd	ymm2, ymm2, ymm11			;; R1 = R1 + MemR1*MemR1#2
	vmovapd	ymm7, [srcreg+r10+32]			;; MemI1#2
	vmulpd	ymm11, ymm3, ymm7			;; MemR1*MemI1#2
	vaddpd	ymm0, ymm0, ymm11			;; I1 = I1 + MemR1*MemI1#2
	vmovapd	ymm3, [srcreg+r9+32]			;; MemI1
	vmulpd	ymm11, ymm3, ymm7			;; MemI1*MemI1#2
	vsubpd	ymm2, ymm2, ymm11			;; R1 = R1 - MemI1*MemI1#2
	vmulpd	ymm11, ymm3, ymm9			;; MemI1*MemR1#2
	vaddpd	ymm0, ymm0, ymm11			;; I1 = I1 + MemI1*MemR1#2

	vmovapd	ymm3, [srcreg+r9+d1]			;; MemR2
	vmovapd	ymm9, [srcreg+r10+d1]			;; MemR2#2
	vmulpd	ymm11, ymm3, ymm9			;; MemR2*MemR2#2
	vaddpd	ymm6, ymm6, ymm11			;; R2 = R2 + MemR2*MemR2#2
	vmovapd	ymm7, [srcreg+r10+d1+32]		;; MemI2#2
	vmulpd	ymm11, ymm3, ymm7			;; MemR2*MemI2#2
	vaddpd	ymm4, ymm4, ymm11			;; I2 = I2 + MemR2*MemI2#2
	vmovapd	ymm3, [srcreg+r9+d1+32]			;; MemI2
	vmulpd	ymm11, ymm3, ymm7			;; MemI2*MemI2#2
	vsubpd	ymm6, ymm6, ymm11			;; R2 = R2 - MemI2*MemI2#2
	vmulpd	ymm11, ymm3, ymm9			;; MemI2*MemR2#2
	vaddpd	ymm4, ymm4, ymm11			;; I2 = I2 + MemI2*MemR2#2

	vmovapd	ymm3, [srcreg+r9+d2]			;; MemR3
	vmovapd	ymm9, [srcreg+r10+d2]			;; MemR3#2
	vmulpd	ymm11, ymm3, ymm9			;; MemR3*MemR3#2
	vaddpd	ymm8, ymm8, ymm11			;; R3 = R3 + MemR3*MemR3#2
	vmovapd	ymm7, [srcreg+r10+d2+32]		;; MemI3#2
	vmulpd	ymm11, ymm3, ymm7			;; MemR3*MemI3#2
	vaddpd	ymm1, ymm1, ymm11			;; I3 = I3 + MemR3*MemI3#2
	vmovapd	ymm3, [srcreg+r9+d2+32]			;; MemI3
	vmulpd	ymm11, ymm3, ymm7			;; MemI3*MemI3#2
	vsubpd	ymm8, ymm8, ymm11			;; R3 = R3 - MemI3*MemI3#2
	vmulpd	ymm11, ymm3, ymm9			;; MemI3*MemR3#2
	vaddpd	ymm1, ymm1, ymm11			;; I3 = I3 + MemI3*MemR3#2

	vmovapd	ymm3, [srcreg+r9+d2+d1]			;; MemR4
	vmovapd	ymm9, [srcreg+r10+d2+d1]		;; MemR4#2
	vmulpd	ymm11, ymm3, ymm9			;; MemR4*MemR4#2
	vaddpd	ymm10, ymm10, ymm11			;; R4 = R4 + MemR4*MemR4#2
	vmovapd	ymm7, [srcreg+r10+d2+d1+32]		;; MemI4#2
	vmulpd	ymm11, ymm3, ymm7			;; MemR4*MemI4#2
	vaddpd	ymm5, ymm5, ymm11			;; I4 = I4 + MemR4*MemI4#2
	vmovapd	ymm3, [srcreg+r9+d2+d1+32]		;; MemI4
	vmulpd	ymm11, ymm3, ymm7			;; MemI4*MemI4#2
	vsubpd	ymm10, ymm10, ymm11			;; R4 = R4 - MemI4*MemI4#2
	vmulpd	ymm11, ymm3, ymm9			;; MemI4*MemR4#2
	vaddpd	ymm5, ymm5, ymm11			;; I4 = I4 + MemI4*MemR4#2

	vaddpd	ymm12, ymm2, ymm6		;; R1 + R2 (new R1)		; 15-17
	vsubpd	ymm2, ymm2, ymm6		;; R1 - R2 (new R2)		; 16-17
	jmp	done

mulsub:	test	r10, 3				;; Check if we need to multiply add-in by FFT(1)
	jz	mulsubhard			;; Yes, do it the hard way

	vsubpd	ymm2, ymm2, [srcreg+r9]			;; R1 = R1 - MemR1
	vsubpd	ymm0, ymm0, [srcreg+r9+32]		;; I1 = I1 - MemI1
	vsubpd	ymm6, ymm6, [srcreg+r9+d1]		;; R2 = R2 - MemR2
	vsubpd	ymm4, ymm4, [srcreg+r9+d1+32]		;; I2 = I2 - MemI2
	vsubpd	ymm8, ymm8, [srcreg+r9+d2]		;; R3 = R3 - MemR3
	vsubpd	ymm1, ymm1, [srcreg+r9+d2+32]		;; I3 = I3 - MemI3
	vsubpd	ymm10, ymm10, [srcreg+r9+d2+d1]		;; R4 = R4 - MemR4
	vsubpd	ymm5, ymm5, [srcreg+r9+d2+d1+32]	;; I4 = I4 - MemI4

	vaddpd	ymm12, ymm2, ymm6		;; R1 + R2 (new R1)		; 15-17
	vsubpd	ymm2, ymm2, ymm6		;; R1 - R2 (new R2)		; 16-17
	jmp	done

mulsubhard:
	vmovapd	ymm3, [srcreg+r9]			;; MemR1
	vmovapd	ymm9, [srcreg+r10]			;; MemR1#2
	vmulpd	ymm11, ymm3, ymm9			;; MemR1*MemR1#2
	vsubpd	ymm2, ymm2, ymm11			;; R1 = R1 - MemR1*MemR1#2
	vmovapd	ymm7, [srcreg+r10+32]			;; MemI1#2
	vmulpd	ymm11, ymm3, ymm7			;; MemR1*MemI1#2
	vsubpd	ymm0, ymm0, ymm11			;; I1 = I1 - MemR1*MemI1#2
	vmovapd	ymm3, [srcreg+r9+32]			;; MemI1
	vmulpd	ymm11, ymm3, ymm7			;; MemI1*MemI1#2
	vaddpd	ymm2, ymm2, ymm11			;; R1 = R1 + MemI1*MemI1#2
	vmulpd	ymm11, ymm3, ymm9			;; MemI1*MemR1#2
	vsubpd	ymm0, ymm0, ymm11			;; I1 = I1 - MemI1*MemR1#2

	vmovapd	ymm3, [srcreg+r9+d1]			;; MemR2
	vmovapd	ymm9, [srcreg+r10+d1]			;; MemR2#2
	vmulpd	ymm11, ymm3, ymm9			;; MemR2*MemR2#2
	vsubpd	ymm6, ymm6, ymm11			;; R2 = R2 - MemR2*MemR2#2
	vmovapd	ymm7, [srcreg+r10+d1+32]		;; MemI2#2
	vmulpd	ymm11, ymm3, ymm7			;; MemR2*MemI2#2
	vsubpd	ymm4, ymm4, ymm11			;; I2 = I2 - MemR2*MemI2#2
	vmovapd	ymm3, [srcreg+r9+d1+32]			;; MemI2
	vmulpd	ymm11, ymm3, ymm7			;; MemI2*MemI2#2
	vaddpd	ymm6, ymm6, ymm11			;; R2 = R2 + MemI2*MemI2#2
	vmulpd	ymm11, ymm3, ymm9			;; MemI2*MemR2#2
	vsubpd	ymm4, ymm4, ymm11			;; I2 = I2 - MemI2*MemR2#2

	vmovapd	ymm3, [srcreg+r9+d2]			;; MemR3
	vmovapd	ymm9, [srcreg+r10+d2]			;; MemR3#2
	vmulpd	ymm11, ymm3, ymm9			;; MemR3*MemR3#2
	vsubpd	ymm8, ymm8, ymm11			;; R3 = R3 - MemR3*MemR3#2
	vmovapd	ymm7, [srcreg+r10+d2+32]		;; MemI3#2
	vmulpd	ymm11, ymm3, ymm7			;; MemR3*MemI3#2
	vsubpd	ymm1, ymm1, ymm11			;; I3 = I3 - MemR3*MemI3#2
	vmovapd	ymm3, [srcreg+r9+d2+32]			;; MemI3
	vmulpd	ymm11, ymm3, ymm7			;; MemI3*MemI3#2
	vaddpd	ymm8, ymm8, ymm11			;; R3 = R3 + MemI3*MemI3#2
	vmulpd	ymm11, ymm3, ymm9			;; MemI3*MemR3#2
	vsubpd	ymm1, ymm1, ymm11			;; I3 = I3 - MemI3*MemR3#2

	vmovapd	ymm3, [srcreg+r9+d2+d1]			;; MemR4
	vmovapd	ymm9, [srcreg+r10+d2+d1]		;; MemR4#2
	vmulpd	ymm11, ymm3, ymm9			;; MemR4*MemR4#2
	vsubpd	ymm10, ymm10, ymm11			;; R4 = R4 - MemR4*MemR4#2
	vmovapd	ymm7, [srcreg+r10+d2+d1+32]		;; MemI4#2
	vmulpd	ymm11, ymm3, ymm7			;; MemR4*MemI4#2
	vsubpd	ymm5, ymm5, ymm11			;; I4 = I4 - MemR4*MemI4#2
	vmovapd	ymm3, [srcreg+r9+d2+d1+32]		;; MemI4
	vmulpd	ymm11, ymm3, ymm7			;; MemI4*MemI4#2
	vaddpd	ymm10, ymm10, ymm11			;; R4 = R4 + MemI4*MemI4#2
	vmulpd	ymm11, ymm3, ymm9			;; MemI4*MemR4#2
	vsubpd	ymm5, ymm5, ymm11			;; I4 = I4 - MemI4*MemR4#2

	vaddpd	ymm12, ymm2, ymm6		;; R1 + R2 (new R1)		; 15-17
	vsubpd	ymm2, ymm2, ymm6		;; R1 - R2 (new R2)		; 16-17

done:
	ENDM

;; Haswell FMA3 version

IF (@INSTR(,%yarch,<FMA3>) NE 0)

yr4_4cl_four_complex_with_mulf_preload MACRO
	ENDM
yr4_4cl_four_complex_with_mulf MACRO srcreg,srcinc,d1,d2
	LOCAL	orig, back_to_orig

	cmp	mul4_opcode, 0			;; See if we need to do more than the original type-4 FFT multiply
	je	short orig
	yr4_dispatch CALLP,ycomplex_mulf_opcode,d1,d2 ;; Handle more difficult cases
	jmp	back_to_orig

orig:	vmovapd	ymm14, [srcreg][rbp]		;; MemR1
	vmovapd	ymm0, [srcreg][rbx]		;; R1
	vmovapd	ymm15, [srcreg+32][rbp]		;; MemI1
	vmulpd	ymm2, ymm0, ymm14		;; R1 * MemR1			; 1-5
	vmulpd	ymm0, ymm0, ymm15		;; R1 * MemI1			; 2-6

	vmovapd	ymm12, [srcreg+d1][rbp]		;; MemR2
	vmovapd	ymm3, [srcreg+d1][rbx]		;; R2
	vmovapd	ymm13, [srcreg+d1+32][rbp]	;; MemI2
	vmulpd	ymm5, ymm3, ymm12		;; R2 * MemR2			; 3-7
	vmulpd	ymm3, ymm3, ymm13		;; R2 * MemI2			; 4-8

	vmovapd	ymm10, [srcreg+d2][rbp]		;; MemR3
	vmovapd	ymm6, [srcreg+d2][rbx]		;; R3
	vmovapd	ymm11, [srcreg+d2+32][rbp]	;; MemI3
	vmulpd	ymm7, ymm6, ymm10		;; R3 * MemR3			; 5-9
	vmulpd	ymm6, ymm6, ymm11		;; R3 * MemI3			; 6-10

	vmovapd	ymm1, [srcreg+32][rbx]		;; I1
	yfnmaddpd ymm2, ymm1, ymm15, ymm2	;; R1*MemR1-I1*MemI1 (new R1)	; 6-10
	yfmaddpd ymm0, ymm1, ymm14, ymm0	;; R1*MemI1+I1*MemR1 (new I1)	; 7-11

	vmovapd	ymm14, [srcreg+d2+d1][rbp]	;; MemR4
	vmovapd	ymm1, [srcreg+d2+d1][rbx]	;; R4
	vmovapd	ymm15, [srcreg+d2+d1+32][rbp]	;; MemI4

	vmulpd	ymm4, ymm1, ymm14		;; R4 * MemR4			; 7-11
	vmulpd	ymm1, ymm1, ymm15		;; R4 * MemI4			; 8-12

	vmovapd	ymm9, [srcreg+d1+32][rbx]	;; I2
	yfnmaddpd ymm5, ymm9, ymm13, ymm5	;; R2*MemR2-I2*MemI2 (new R2)	; 8-12
	yfmaddpd ymm3, ymm9, ymm12, ymm3	;; R2*MemI2+I2*MemR2 (new I2)	; 9-13

	vmovapd	ymm9, [srcreg+d2+32][rbx]	;; I3
	yfnmaddpd ymm7, ymm9, ymm11, ymm7	;; R3*MemR3-I3*MemI3 (new R3)	; 10-14
	yfmaddpd ymm6, ymm9, ymm10, ymm6	;; R3*MemI3+I3*MemR3 (new I3)	; 11-15

	vmovapd	ymm9, [srcreg+d2+d1+32][rbx]	;; I4
	yfnmaddpd ymm4, ymm9, ymm15, ymm4	;; R4*MemR4-I4*MemI4 (new R4)	; 12-16
	yfmaddpd ymm1, ymm9, ymm14, ymm1	;; R4*MemI4+I4*MemR4 (new I4)	; 13-17

back_to_orig:
	vaddpd	ymm8, ymm2, ymm5		;; R1 + R2 (new R1)		; 13-15
	vsubpd	ymm2, ymm2, ymm5		;; R1 - R2 (new R2)		; 14-16

	vaddpd	ymm5, ymm0, ymm3		;; I1 + I2 (new I1)		; 15-17
	vsubpd	ymm0, ymm0, ymm3		;; I1 - I2 (new I2)		; 16-18

	vaddpd	ymm3, ymm4, ymm7		;; R4 + R3 (new R3)		; 17-19
	vsubpd	ymm4, ymm4, ymm7		;; R4 - R3 (new I4)		; 18-20

	vaddpd	ymm7, ymm6, ymm1		;; I3 + I4 (new I3)		; 19-21
	vsubpd	ymm6, ymm6, ymm1		;; I3 - I4 (new R4)		; 20-22

	vaddpd	ymm1, ymm8, ymm3		;; R1 + R3 (final R1)		; 21-23
	vsubpd	ymm8, ymm8, ymm3		;; R1 - R3 (final R3)		; 22-24

	vaddpd	ymm3, ymm0, ymm4		;; I2 + I4 (final I2)		; 23-25
	vsubpd	ymm0, ymm0, ymm4		;; I2 - I4 (final I4)		; 24-26

	vaddpd	ymm4, ymm5, ymm7		;; I1 + I3 (final I1)		; 25-27
	vsubpd	ymm5, ymm5, ymm7		;; I1 - I3 (final I3)		; 26-28

	vaddpd	ymm7, ymm2, ymm6		;; R2 + R4 (final R2)		; 27-29
	vsubpd	ymm2, ymm2, ymm6		;; R2 - R4 (final R4)		; 28-30

	ystore	[srcreg], ymm1			;; Save R1
	ystore	[srcreg+32], ymm4		;; Save I1
	ystore	[srcreg+d1], ymm7		;; Save R2
	ystore	[srcreg+d1+32], ymm3		;; Save I2
	ystore	[srcreg+d2], ymm8		;; Save R3
	ystore	[srcreg+d2+32], ymm5		;; Save I3
	ystore	[srcreg+d2+d1], ymm2		;; Save R4
	ystore	[srcreg+d2+d1+32], ymm0		;; Save I4
	bump	srcreg, srcinc
	ENDM

;; Same as above code between orig and back_to_orig, except we implement mul4_opcode options

yr4_4c_mulf_opcode MACRO srcreg,d1,d2
	LOCAL	fma_or_unfft, fma, addmul, submul, muladd, muladdhard, mulsub, mulsubhard, unfft, done

	movzx	r10, mul4_opcode		;; Load the mul4_opcode
	mov	r9, SRC2ARG			;; Distance to s2 arg from gwaddmul4 or gwsubmul4 or s3 arg in gwmuladd4 or gwmulsub4
	cmp	r10, 2				;; Case off opcode
	jg	fma_or_unfft			;; 3,4,5=muladd,mulsub,unfft
	je	submul				;; 2=submul
	;;jl	addmul				;; 1=addmul

addmul:	vmovapd	ymm14, [srcreg][rbp]		;; MemR1
	vaddpd	ymm14, ymm14, [srcreg][r9]	;; MemR1
	vmovapd	ymm0, [srcreg][rbx]		;; R1
	vmovapd	ymm15, [srcreg+32][rbp]		;; MemI1
	vaddpd	ymm15, ymm15, [srcreg+32][r9]	;; MemI1
	vmulpd	ymm2, ymm0, ymm14		;; R1 * MemR1			; 1-5
	vmulpd	ymm0, ymm0, ymm15		;; R1 * MemI1			; 2-6

	vmovapd	ymm12, [srcreg+d1][rbp]		;; MemR2
	vaddpd	ymm12, ymm12, [srcreg+d1][r9]	;; MemR2
	vmovapd	ymm3, [srcreg+d1][rbx]		;; R2
	vmovapd	ymm13, [srcreg+d1+32][rbp]	;; MemI2
	vaddpd	ymm13, ymm13, [srcreg+d1+32][r9];; MemI2
	vmulpd	ymm5, ymm3, ymm12		;; R2 * MemR2			; 3-7
	vmulpd	ymm3, ymm3, ymm13		;; R2 * MemI2			; 4-8

	vmovapd	ymm10, [srcreg+d2][rbp]		;; MemR3
	vaddpd	ymm10, ymm10, [srcreg+d2][r9]	;; MemR3
	vmovapd	ymm6, [srcreg+d2][rbx]		;; R3
	vmovapd	ymm11, [srcreg+d2+32][rbp]	;; MemI3
	vaddpd	ymm11, ymm11, [srcreg+d2+32][r9];; MemI3
	vmulpd	ymm7, ymm6, ymm10		;; R3 * MemR3			; 5-9
	vmulpd	ymm6, ymm6, ymm11		;; R3 * MemI3			; 6-10

	vmovapd	ymm1, [srcreg+32][rbx]		;; I1
	yfnmaddpd ymm2, ymm1, ymm15, ymm2	;; R1*MemR1-I1*MemI1 (new R1)	; 6-10
	yfmaddpd ymm0, ymm1, ymm14, ymm0	;; R1*MemI1+I1*MemR1 (new I1)	; 7-11

	vmovapd	ymm14, [srcreg+d2+d1][rbp]	;; MemR4
	vaddpd	ymm14, ymm14, [srcreg+d2+d1][r9];; MemR4
	vmovapd	ymm1, [srcreg+d2+d1][rbx]	;; R4
	vmovapd	ymm15, [srcreg+d2+d1+32][rbp]	;; MemI4
	vaddpd	ymm15, ymm15, [srcreg+d2+d1+32][r9];; MemI4

	vmulpd	ymm4, ymm1, ymm14		;; R4 * MemR4			; 7-11
	vmulpd	ymm1, ymm1, ymm15		;; R4 * MemI4			; 8-12

	vmovapd	ymm9, [srcreg+d1+32][rbx]	;; I2
	yfnmaddpd ymm5, ymm9, ymm13, ymm5	;; R2*MemR2-I2*MemI2 (new R2)	; 8-12
	yfmaddpd ymm3, ymm9, ymm12, ymm3	;; R2*MemI2+I2*MemR2 (new I2)	; 9-13

	vmovapd	ymm9, [srcreg+d2+32][rbx]	;; I3
	yfnmaddpd ymm7, ymm9, ymm11, ymm7	;; R3*MemR3-I3*MemI3 (new R3)	; 10-14
	yfmaddpd ymm6, ymm9, ymm10, ymm6	;; R3*MemI3+I3*MemR3 (new I3)	; 11-15

	vmovapd	ymm9, [srcreg+d2+d1+32][rbx]	;; I4
	yfnmaddpd ymm4, ymm9, ymm15, ymm4	;; R4*MemR4-I4*MemI4 (new R4)	; 12-16
	yfmaddpd ymm1, ymm9, ymm14, ymm1	;; R4*MemI4+I4*MemR4 (new I4)	; 13-17
	jmp	done

submul:	vmovapd	ymm14, [srcreg][rbp]		;; MemR1
	vsubpd	ymm14, ymm14, [srcreg][r9]	;; MemR1
	vmovapd	ymm0, [srcreg][rbx]		;; R1
	vmovapd	ymm15, [srcreg+32][rbp]		;; MemI1
	vsubpd	ymm15, ymm15, [srcreg+32][r9]	;; MemI1
	vmulpd	ymm2, ymm0, ymm14		;; R1 * MemR1			; 1-5
	vmulpd	ymm0, ymm0, ymm15		;; R1 * MemI1			; 2-6

	vmovapd	ymm12, [srcreg+d1][rbp]		;; MemR2
	vsubpd	ymm12, ymm12, [srcreg+d1][r9]	;; MemR2
	vmovapd	ymm3, [srcreg+d1][rbx]		;; R2
	vmovapd	ymm13, [srcreg+d1+32][rbp]	;; MemI2
	vsubpd	ymm13, ymm13, [srcreg+d1+32][r9];; MemI2
	vmulpd	ymm5, ymm3, ymm12		;; R2 * MemR2			; 3-7
	vmulpd	ymm3, ymm3, ymm13		;; R2 * MemI2			; 4-8

	vmovapd	ymm10, [srcreg+d2][rbp]		;; MemR3
	vsubpd	ymm10, ymm10, [srcreg+d2][r9]	;; MemR3
	vmovapd	ymm6, [srcreg+d2][rbx]		;; R3
	vmovapd	ymm11, [srcreg+d2+32][rbp]	;; MemI3
	vsubpd	ymm11, ymm11, [srcreg+d2+32][r9];; MemI3
	vmulpd	ymm7, ymm6, ymm10		;; R3 * MemR3			; 5-9
	vmulpd	ymm6, ymm6, ymm11		;; R3 * MemI3			; 6-10

	vmovapd	ymm1, [srcreg+32][rbx]		;; I1
	yfnmaddpd ymm2, ymm1, ymm15, ymm2	;; R1*MemR1-I1*MemI1 (new R1)	; 6-10
	yfmaddpd ymm0, ymm1, ymm14, ymm0	;; R1*MemI1+I1*MemR1 (new I1)	; 7-11

	vmovapd	ymm14, [srcreg+d2+d1][rbp]	;; MemR4
	vsubpd	ymm14, ymm14, [srcreg+d2+d1][r9];; MemR4
	vmovapd	ymm1, [srcreg+d2+d1][rbx]	;; R4
	vmovapd	ymm15, [srcreg+d2+d1+32][rbp]	;; MemI4
	vsubpd	ymm15, ymm15, [srcreg+d2+d1+32][r9];; MemI4

	vmulpd	ymm4, ymm1, ymm14		;; R4 * MemR4			; 7-11
	vmulpd	ymm1, ymm1, ymm15		;; R4 * MemI4			; 8-12

	vmovapd	ymm9, [srcreg+d1+32][rbx]	;; I2
	yfnmaddpd ymm5, ymm9, ymm13, ymm5	;; R2*MemR2-I2*MemI2 (new R2)	; 8-12
	yfmaddpd ymm3, ymm9, ymm12, ymm3	;; R2*MemI2+I2*MemR2 (new I2)	; 9-13

	vmovapd	ymm9, [srcreg+d2+32][rbx]	;; I3
	yfnmaddpd ymm7, ymm9, ymm11, ymm7	;; R3*MemR3-I3*MemI3 (new R3)	; 10-14
	yfmaddpd ymm6, ymm9, ymm10, ymm6	;; R3*MemI3+I3*MemR3 (new I3)	; 11-15

	vmovapd	ymm9, [srcreg+d2+d1+32][rbx]	;; I4
	yfnmaddpd ymm4, ymm9, ymm15, ymm4	;; R4*MemR4-I4*MemI4 (new R4)	; 12-16
	yfmaddpd ymm1, ymm9, ymm14, ymm1	;; R4*MemI4+I4*MemR4 (new I4)	; 13-17
	jmp	done

fma_or_unfft:
	cmp	r10, 4				;; Case off opcode
	jbe	fma				;; 3=muladd, 4=mulsub, fma MUST NOT ALTER CONDITION CODES!
	;;ja	unfft				;; 5=unfft

unfft:	vmovapd	ymm2, [srcreg][rbx]		;; R1
	vmovapd	ymm0, [srcreg+32][rbx]		;; I1
	vmovapd	ymm5, [srcreg+d1][rbx]		;; R2
	vmovapd	ymm3, [srcreg+d1+32][rbx]	;; I2
	vmovapd	ymm7, [srcreg+d2][rbx]		;; R3
	vmovapd	ymm6, [srcreg+d2+32][rbx]	;; I3
	vmovapd	ymm4, [srcreg+d2+d1][rbx]	;; R4
	vmovapd	ymm1, [srcreg+d2+d1+32][rbx]	;; I4
	jmp	done

fma:	vmovapd	ymm14, [srcreg][rbp]		;; MemR1
	vmovapd	ymm0, [srcreg][rbx]		;; R1
	vmovapd	ymm15, [srcreg+32][rbp]		;; MemI1
	vmulpd	ymm2, ymm0, ymm14		;; R1 * MemR1			; 1-5
	vmulpd	ymm0, ymm0, ymm15		;; R1 * MemI1			; 2-6

	vmovapd	ymm12, [srcreg+d1][rbp]		;; MemR2
	vmovapd	ymm3, [srcreg+d1][rbx]		;; R2
	vmovapd	ymm13, [srcreg+d1+32][rbp]	;; MemI2
	vmulpd	ymm5, ymm3, ymm12		;; R2 * MemR2			; 3-7
	vmulpd	ymm3, ymm3, ymm13		;; R2 * MemI2			; 4-8

	vmovapd	ymm10, [srcreg+d2][rbp]		;; MemR3
	vmovapd	ymm6, [srcreg+d2][rbx]		;; R3
	vmovapd	ymm11, [srcreg+d2+32][rbp]	;; MemI3
	vmulpd	ymm7, ymm6, ymm10		;; R3 * MemR3			; 5-9
	vmulpd	ymm6, ymm6, ymm11		;; R3 * MemI3			; 6-10

	vmovapd	ymm1, [srcreg+32][rbx]		;; I1
	yfnmaddpd ymm2, ymm1, ymm15, ymm2	;; R1*MemR1-I1*MemI1 (new R1)	; 6-10
	yfmaddpd ymm0, ymm1, ymm14, ymm0	;; R1*MemI1+I1*MemR1 (new I1)	; 7-11

	vmovapd	ymm14, [srcreg+d2+d1][rbp]	;; MemR4
	vmovapd	ymm1, [srcreg+d2+d1][rbx]	;; R4
	vmovapd	ymm15, [srcreg+d2+d1+32][rbp]	;; MemI4

	vmulpd	ymm4, ymm1, ymm14		;; R4 * MemR4			; 7-11
	vmulpd	ymm1, ymm1, ymm15		;; R4 * MemI4			; 8-12

	vmovapd	ymm9, [srcreg+d1+32][rbx]	;; I2
	yfnmaddpd ymm5, ymm9, ymm13, ymm5	;; R2*MemR2-I2*MemI2 (new R2)	; 8-12
	yfmaddpd ymm3, ymm9, ymm12, ymm3	;; R2*MemI2+I2*MemR2 (new I2)	; 9-13

	vmovapd	ymm9, [srcreg+d2+32][rbx]	;; I3
	yfnmaddpd ymm7, ymm9, ymm11, ymm7	;; R3*MemR3-I3*MemI3 (new R3)	; 10-14
	yfmaddpd ymm6, ymm9, ymm10, ymm6	;; R3*MemI3+I3*MemR3 (new I3)	; 11-15

	vmovapd	ymm9, [srcreg+d2+d1+32][rbx]	;; I4
	yfnmaddpd ymm4, ymm9, ymm15, ymm4	;; R4*MemR4-I4*MemI4 (new R4)	; 12-16
	yfmaddpd ymm1, ymm9, ymm14, ymm1	;; R4*MemI4+I4*MemR4 (new I4)	; 13-17

	mov	r10, SRC3ARG			;; Load distance to FFT(1)
	je	mulsub				;; 4=mulsub
	;jb	muladd				;; 3=muladd, fall through

muladd:	test	r10, 3				;; Check if we need to multiply add-in by FFT(1)
	jz	muladdhard			;; Yes, do it the hard way

	vaddpd	ymm2, ymm2, [srcreg+r9]			;; R1 = R1 + MemR1
	vaddpd	ymm0, ymm0, [srcreg+r9+32]		;; I1 = I1 + MemI1
	vaddpd	ymm5, ymm5, [srcreg+r9+d1]		;; R2 = R2 + MemR2
	vaddpd	ymm3, ymm3, [srcreg+r9+d1+32]		;; I2 = I2 + MemI2
	vaddpd	ymm7, ymm7, [srcreg+r9+d2]		;; R3 = R3 + MemR3
	vaddpd	ymm6, ymm6, [srcreg+r9+d2+32]		;; I3 = I3 + MemI3
	vaddpd	ymm4, ymm4, [srcreg+r9+d2+d1]		;; R4 = R4 + MemR4
	vaddpd	ymm1, ymm1, [srcreg+r9+d2+d1+32]	;; I4 = I4 + MemI4
	jmp	done

muladdhard:
	vmovapd	ymm8, [srcreg+r9]			;; MemR1
	vmovapd	ymm9, [srcreg+r10]			;; MemR1#2
	yfmaddpd ymm2, ymm8, ymm9, ymm2			;; R1 = R1 + MemR1*MemR1#2
	vmovapd	ymm10, [srcreg+r10+32]			;; MemI1#2
	yfmaddpd ymm0, ymm8, ymm10, ymm0		;; I1 = I1 + MemR1*MemI1#2
	vmovapd	ymm8, [srcreg+r9+32]			;; MemI1
	yfnmaddpd ymm2, ymm8, ymm10, ymm2		;; R1 = R1 - MemI1*MemI1#2
	yfmaddpd ymm0, ymm8, ymm9, ymm0			;; I1 = I1 + MemI1*MemR1#2

	vmovapd	ymm8, [srcreg+r9+d1]			;; MemR2
	vmovapd	ymm9, [srcreg+r10+d1]			;; MemR2#2
	yfmaddpd ymm5, ymm8, ymm9, ymm5			;; R2 = R2 + MemR2*MemR2#2
	vmovapd	ymm10, [srcreg+r10+d1+32]		;; MemI2#2
	yfmaddpd ymm3, ymm8, ymm10, ymm3		;; I2 = I2 + MemR2*MemI2#2
	vmovapd	ymm8, [srcreg+r9+d1+32]			;; MemI2
	yfnmaddpd ymm5, ymm8, ymm10, ymm5		;; R2 = R2 - MemI2*MemI2#2
	yfmaddpd ymm3, ymm8, ymm9, ymm3			;; I2 = I2 + MemI2*MemR2#2

	vmovapd	ymm8, [srcreg+r9+d2]			;; MemR3
	vmovapd	ymm9, [srcreg+r10+d2]			;; MemR3#2
	yfmaddpd ymm7, ymm8, ymm9, ymm7			;; R3 = R3 + MemR3*MemR3#2
	vmovapd	ymm10, [srcreg+r10+d2+32]		;; MemI3#2
	yfmaddpd ymm6, ymm8, ymm10, ymm6		;; I3 = I3 + MemR3*MemI3#2
	vmovapd	ymm8, [srcreg+r9+d2+32]			;; MemI3
	yfnmaddpd ymm7, ymm8, ymm10, ymm7		;; R3 = R3 - MemI3*MemI3#2
	yfmaddpd ymm6, ymm8, ymm9, ymm6			;; I3 = I3 + MemI3*MemR3#2

	vmovapd	ymm8, [srcreg+r9+d2+d1]			;; MemR4
	vmovapd	ymm9, [srcreg+r10+d2+d1]		;; MemR4#2
	yfmaddpd ymm4, ymm8, ymm9, ymm4			;; R4 = R4 + MemR4*MemR4#2
	vmovapd	ymm10, [srcreg+r10+d2+d1+32]		;; MemI4#2
	yfmaddpd ymm1, ymm8, ymm10, ymm1		;; I4 = I4 + MemR4*MemI4#2
	vmovapd	ymm8, [srcreg+r9+d2+d1+32]		;; MemI4
	yfnmaddpd ymm4, ymm8, ymm10, ymm4		;; R4 = R4 - MemI4*MemI4#2
	yfmaddpd ymm1, ymm8, ymm9, ymm1			;; I4 = I4 + MemI4*MemR4#2
	jmp	done

mulsub:	test	r10, 3				;; Check if we need to multiply add-in by FFT(1)
	jz	mulsubhard			;; Yes, do it the hard way

	vsubpd	ymm2, ymm2, [srcreg+r9]			;; R1 = R1 + MemR1
	vsubpd	ymm0, ymm0, [srcreg+r9+32]		;; I1 = I1 + MemI1
	vsubpd	ymm5, ymm5, [srcreg+r9+d1]		;; R2 = R2 + MemR2
	vsubpd	ymm3, ymm3, [srcreg+r9+d1+32]		;; I2 = I2 + MemI2
	vsubpd	ymm7, ymm7, [srcreg+r9+d2]		;; R3 = R3 + MemR3
	vsubpd	ymm6, ymm6, [srcreg+r9+d2+32]		;; I3 = I3 + MemI3
	vsubpd	ymm4, ymm4, [srcreg+r9+d2+d1]		;; R4 = R4 + MemR4
	vsubpd	ymm1, ymm1, [srcreg+r9+d2+d1+32]	;; I4 = I4 + MemI4
	jmp	done

mulsubhard:
	vmovapd	ymm8, [srcreg+r9]			;; MemR1
	vmovapd	ymm9, [srcreg+r10]			;; MemR1#2
	yfnmaddpd ymm2, ymm8, ymm9, ymm2		;; R1 = R1 - MemR1*MemR1#2
	vmovapd	ymm10, [srcreg+r10+32]			;; MemI1#2
	yfnmaddpd ymm0, ymm8, ymm10, ymm0		;; I1 = I1 - MemR1*MemI1#2
	vmovapd	ymm8, [srcreg+r9+32]			;; MemI1
	yfmaddpd ymm2, ymm8, ymm10, ymm2		;; R1 = R1 + MemI1*MemI1#2
	yfnmaddpd ymm0, ymm8, ymm9, ymm0		;; I1 = I1 - MemI1*MemR1#2

	vmovapd	ymm8, [srcreg+r9+d1]			;; MemR2
	vmovapd	ymm9, [srcreg+r10+d1]			;; MemR2#2
	yfnmaddpd ymm5, ymm8, ymm9, ymm5			;; R2 = R2 - MemR2*MemR2#2
	vmovapd	ymm10, [srcreg+r10+d1+32]		;; MemI2#2
	yfnmaddpd ymm3, ymm8, ymm10, ymm3		;; I2 = I2 - MemR2*MemI2#2
	vmovapd	ymm8, [srcreg+r9+d1+32]			;; MemI2
	yfmaddpd ymm5, ymm8, ymm10, ymm5		;; R2 = R2 + MemI2*MemI2#2
	yfnmaddpd ymm3, ymm8, ymm9, ymm3		;; I2 = I2 - MemI2*MemR2#2

	vmovapd	ymm8, [srcreg+r9+d2]			;; MemR3
	vmovapd	ymm9, [srcreg+r10+d2]			;; MemR3#2
	yfnmaddpd ymm7, ymm8, ymm9, ymm7		;; R3 = R3 - MemR3*MemR3#2
	vmovapd	ymm10, [srcreg+r10+d2+32]		;; MemI3#2
	yfnmaddpd ymm6, ymm8, ymm10, ymm6		;; I3 = I3 - MemR3*MemI3#2
	vmovapd	ymm8, [srcreg+r9+d2+32]			;; MemI3
	yfmaddpd ymm7, ymm8, ymm10, ymm7		;; R3 = R3 + MemI3*MemI3#2
	yfnmaddpd ymm6, ymm8, ymm9, ymm6		;; I3 = I3 - MemI3*MemR3#2

	vmovapd	ymm8, [srcreg+r9+d2+d1]			;; MemR4
	vmovapd	ymm9, [srcreg+r10+d2+d1]		;; MemR4#2
	yfnmaddpd ymm4, ymm8, ymm9, ymm4		;; R4 = R4 - MemR4*MemR4#2
	vmovapd	ymm10, [srcreg+r10+d2+d1+32]		;; MemI4#2
	yfnmaddpd ymm1, ymm8, ymm10, ymm1		;; I4 = I4 - MemR4*MemI4#2
	vmovapd	ymm8, [srcreg+r9+d2+d1+32]		;; MemI4
	yfmaddpd ymm4, ymm8, ymm10, ymm4		;; R4 = R4 + MemI4*MemI4#2
	yfnmaddpd ymm1, ymm8, ymm9, ymm1		;; I4 = I4 - MemI4*MemR4#2

done:
	ENDM

ENDIF

ENDIF


;; Does a four-complex FFT with no sin/cos data -- appropriate only in the last FFT levels.
yr4_4c_simple_fft_mem MACRO mem1,mem2,mem3,mem4,mem5,mem6,mem7,mem8,dst1
	vmovapd	ymm0, mem1		;; R1
	vmovapd	ymm7, mem3		;; R3
	vaddpd	ymm2, ymm0, ymm7	;; R1 + R3 (new R1)
	vsubpd	ymm0, ymm0, ymm7	;; R1 - R3 (new R3)

	vmovapd	ymm1, mem2		;; R2
	vmovapd	ymm7, mem4		;; R4
	vaddpd	ymm3, ymm1, ymm7	;; R2 + R4 (new R2)
	vsubpd	ymm1, ymm1, ymm7	;; R2 - R4 (new R4)

	vaddpd	ymm6, ymm2, ymm3	;; R1 + R2 (final R1)
	vsubpd	ymm2, ymm2, ymm3	;; R1 - R2 (final R2)

	vmovapd	ymm5, mem6		;; I2
	vmovapd	ymm7, mem8		;; I4
	vsubpd	ymm3, ymm5, ymm7	;; I2 - I4 (new I4)
	vaddpd	ymm5, ymm5, ymm7	;; I2 + I4 (new I2)

	vmovapd	ymm4, mem5		;; I1
	vmovapd	ymm7, mem7		;; I3

	ystore	dst1, ymm6		;; Save R1

	vsubpd	ymm6, ymm4, ymm7	;; I1 - I3 (new I3)
	vaddpd	ymm4, ymm4, ymm7	;; I1 + I3 (new I1)

	vsubpd	ymm7, ymm0, ymm3	;; R3 - I4 (final R3)
	vaddpd	ymm0, ymm0, ymm3	;; R3 + I4 (final R4)

	vaddpd	ymm3, ymm6, ymm1	;; I3 + R4 (final I3)
	vsubpd	ymm6, ymm6, ymm1	;; I3 - R4 (final I4)

	vsubpd	ymm1, ymm4, ymm5	;; I1 - I2 (final I2)
	vaddpd	ymm4, ymm4, ymm5	;; I1 + I2 (final I1)
	ENDM

;; Does a four-complex inverse FFT with no sin/cos data -- appropriate only in the last FFT levels.
yr4_4c_simple_unfft_mem MACRO mem1,mem2,mem3,mem4,mem5,mem6,mem7,mem8,dst1
	vmovapd	ymm7, mem1		;; R1
	vmovapd	ymm0, mem3		;; R2
	vsubpd	ymm5, ymm7, ymm0	;; R1 - R2 (new R2)
	vaddpd	ymm7, ymm7, ymm0	;; R1 + R2 (new R1)

	vmovapd	ymm1, mem7		;; R4
	vmovapd	ymm0, mem5		;; R3
	vsubpd	ymm3, ymm1, ymm0	;; R4 - R3 (new I4)
	vaddpd	ymm1, ymm1, ymm0	;; R4 + R3 (new R3)

	vmovapd	ymm0, mem6		;; I3
	vmovapd	ymm2, mem8		;; I4
	vsubpd	ymm4, ymm0, ymm2	;; I3 - I4 (new R4)
	vaddpd	ymm0, ymm0, ymm2	;; I3 + I4 (new I3)

	vsubpd	ymm2, ymm7, ymm1	;; R1 - R3 (final R3)
	vaddpd	ymm7, ymm7, ymm1	;; R1 + R3 (final R1)

	vsubpd	ymm1, ymm5, ymm4	;; R2 - R4 (final R4)
	vaddpd	ymm5, ymm5, ymm4	;; R2 + R4 (final R2)

	ystore	dst1, ymm7

	vmovapd	ymm4, mem2		;; I1
	vmovapd	ymm6, mem4		;; I2
	vsubpd	ymm7, ymm4, ymm6	;; I1 - I2 (new I2)
	vaddpd	ymm4, ymm4, ymm6	;; I1 + I2 (new I1)

	vsubpd	ymm6, ymm7, ymm3	;; I2 - I4 (final I4)
	vaddpd	ymm7, ymm7, ymm3	;; I2 + I4 (final I2)

	vsubpd	ymm3, ymm4, ymm0	;; I1 - I3 (final I3)
	vaddpd	ymm4, ymm4, ymm0	;; I1 + I3 (final I1)
	ENDM

;; 64-bit version

IFDEF X86_64

yr4_4cl_four_complex_with_square_preload MACRO
	vmovapd	ymm15, YMM_TWO
	ENDM

yr4_4cl_four_complex_with_square MACRO srcreg,srcinc,d1,d2,maxrpt,L1pt,L1pd
	IF maxrpt MOD 5 EQ 0
	yr4_4c_with_square_unroll -1,noexec,noexec,exec,srcreg,srcinc,d1,d2,L1pt,L1pd
	yr4_4c_with_square_unroll 0,noexec,exec,exec,srcreg,srcinc,d1,d2,L1pt,L1pd
	yr4_4c_with_square_unroll 1,exec,exec,exec,srcreg,srcinc,d1,d2,L1pt,L1pd
	yr4_4c_with_square_unroll 2,exec,exec,exec,srcreg,srcinc,d1,d2,L1pt,L1pd
	yr4_4c_with_square_unroll 3,exec,exec,exec,srcreg,srcinc,d1,d2,L1pt,L1pd
	yr4_4c_with_square_unroll 4,exec,exec,noexec,srcreg,srcinc,d1,d2,L1pt,L1pd
	yr4_4c_with_square_unroll 5,exec,noexec,noexec,srcreg,srcinc,d1,d2,L1pt,L1pd
	bump	srcreg, 5*srcinc
	ELSEIF maxrpt MOD 4 EQ 0
	yr4_4c_with_square_unroll -1,noexec,noexec,exec,srcreg,srcinc,d1,d2,L1pt,L1pd
	yr4_4c_with_square_unroll 0,noexec,exec,exec,srcreg,srcinc,d1,d2,L1pt,L1pd
	yr4_4c_with_square_unroll 1,exec,exec,exec,srcreg,srcinc,d1,d2,L1pt,L1pd
	yr4_4c_with_square_unroll 2,exec,exec,exec,srcreg,srcinc,d1,d2,L1pt,L1pd
	yr4_4c_with_square_unroll 3,exec,exec,noexec,srcreg,srcinc,d1,d2,L1pt,L1pd
	yr4_4c_with_square_unroll 4,exec,noexec,noexec,srcreg,srcinc,d1,d2,L1pt,L1pd
	bump	srcreg, 4*srcinc
	ELSEIF maxrpt MOD 3 EQ 0
	yr4_4c_with_square_unroll -1,noexec,noexec,exec,srcreg,srcinc,d1,d2,L1pt,L1pd
	yr4_4c_with_square_unroll 0,noexec,exec,exec,srcreg,srcinc,d1,d2,L1pt,L1pd
	yr4_4c_with_square_unroll 1,exec,exec,exec,srcreg,srcinc,d1,d2,L1pt,L1pd
	yr4_4c_with_square_unroll 2,exec,exec,noexec,srcreg,srcinc,d1,d2,L1pt,L1pd
	yr4_4c_with_square_unroll 3,exec,noexec,noexec,srcreg,srcinc,d1,d2,L1pt,L1pd
	bump	srcreg, 3*srcinc
	ELSEIF maxrpt MOD 2 EQ 0
	yr4_4c_with_square_unroll -1,noexec,noexec,exec,srcreg,srcinc,d1,d2,L1pt,L1pd
	yr4_4c_with_square_unroll 0,noexec,exec,exec,srcreg,srcinc,d1,d2,L1pt,L1pd
	yr4_4c_with_square_unroll 1,exec,exec,noexec,srcreg,srcinc,d1,d2,L1pt,L1pd
	yr4_4c_with_square_unroll 2,exec,noexec,noexec,srcreg,srcinc,d1,d2,L1pt,L1pd
	bump	srcreg, 2*srcinc
	ELSE
	yr4_4c_with_square_unroll -1,noexec,noexec,exec,srcreg,srcinc,d1,d2,L1pt,L1pd
	yr4_4c_with_square_unroll 0,noexec,exec,noexec,srcreg,srcinc,d1,d2,L1pt,L1pd
	yr4_4c_with_square_unroll 1,exec,noexec,noexec,srcreg,srcinc,d1,d2,L1pt,L1pd
	bump	srcreg, srcinc
	ENDIF
	ENDM

yr4_4c_with_square_unroll MACRO iter,prev,this,next,srcreg,srcinc,d1,d2,L1pt,L1pd

;; On first call, init registers

	IF iter EQ -1
y0	TEXTEQU	<ymm0>
y1	TEXTEQU	<ymm1>
y2	TEXTEQU	<ymm2>
y3	TEXTEQU	<ymm3>
y4	TEXTEQU	<ymm4>
y5	TEXTEQU	<ymm5>
y6	TEXTEQU	<ymm6>
y7	TEXTEQU	<ymm7>
y8	TEXTEQU	<ymm8>
y9	TEXTEQU	<ymm9>
y10	TEXTEQU	<ymm10>
y11	TEXTEQU	<ymm11>
y12	TEXTEQU	<ymm12>
y13	TEXTEQU	<ymm13>
y14	TEXTEQU	<ymm14>
	ENDIF

;; On later calls, previous R2,I3,I1 will be in y0-2.  This R1,R2,R3,R4,new R1,I1,I3 will be in y3-9.
;; The remaining registers are free.  ymm15 is preloaded with the constant 2.

this	vaddpd	y14, y4, y6		;; R2 + R4 (new R2)			; 1-3
this	vmovapd	y10, [srcreg+iter*srcinc+d1+32]	;; I2
prev	ystore	[srcreg+(iter-1)*srcinc+d1], y0 ;; Save R2			; 1

this	vaddpd	y13, y8, y9		;; I1 + I3 (new I1)			; 2-4
this	vmovapd	y11, [srcreg+iter*srcinc+d2+d1+32] ;; I4
prev	ystore	[srcreg+(iter-1)*srcinc+d2+32], y1 ;; Save I3			; 2

this	vaddpd	y1, y10, y11		;; I2 + I4 (new I2)			; 3-5
prev	ystore	[srcreg+(iter-1)*srcinc+32], y2 ;; Save I1			; 3

this	vaddpd	y2, y7, y14		;; R1 + R2 (final R1)			; 4-6
next	vmovapd	y12, [srcreg+(iter+1)*srcinc] ;; R1

this	vsubpd	y7, y7, y14		;; R1 - R2 (final R2)			; 5-7
next	vmovapd	y0, [srcreg+(iter+1)*srcinc+d2] ;; R3

this	vaddpd	y14, y13, y1		;; I1 + I2 (final I1)			; 6-8

this	vsubpd	y13, y13, y1		;; I1 - I2 (final I2)			; 7-9
this	vmulpd	y1, y2, y2		;; R1 * R1				; 7-11

this	vsubpd	y3, y3, y5		;; R1 - R3 (new R3)			; 8-10
this	vmulpd	y5, y7, y7		;; R2 * R2				; 8-12
this next yloop_unrolled_one

this	vsubpd	y10, y10, y11		;; I2 - I4 (new I4)			; 9-11
this	vmulpd	y11, y14, y14		;; I1 * I1				; 9-13

this	vsubpd	y4, y4, y6		;; R2 - R4 (new R4)			; 10-12
this	vmulpd	y6, y13, y13		;; I2 * I2				; 10-14

this	vsubpd	y8, y8, y9		;; I1 - I3 (new I3)			; 11-13
this	vmulpd	y14, y14, y2		;; I1 * R1 (I1/2)			; 11-15

this	vsubpd	y2, y3, y10		;; R3 - I4 (final R3)			; 12-14
this	vmulpd	y13, y13, y7		;; I2 * R2 (I2/2)			; 12-16
this	L1prefetchw srcreg+iter*srcinc+L1pd, L1pt

this	vaddpd	y3, y3, y10		;; R3 + I4 (final R4)			; 13-15

this	vaddpd	y10, y8, y4		;; I3 + R4 (final I3)			; 14-16

this	vsubpd	y8, y8, y4		;; I3 - R4 (final I4)			; 15-17
this	vmulpd	y4, y2, y2		;; R3 * R3				; 15-19

this	vsubpd	y1, y1, y11		;; R1^2 - I1^2 (R1)			; 16-18
this	vmulpd	y11, y3, y3		;; R4 * R4				; 16-20
this	L1prefetchw srcreg+iter*srcinc+d1+L1pd, L1pt

this	vsubpd	y5, y5, y6		;; R2^2 - I2^2 (R2)			; 17-19
this	vmulpd	y6, y10, y10		;; I3 * I3				; 17-21

this	vsubpd	y7, y14, y13		;; I1/2 - I2/2 (new I2/2)		; 18-20
this	vmulpd	y9, y8, y8		;; I4 * I4				; 18-22

this	vaddpd	y14, y14, y13		;; I1/2 + I2/2 (new I1/2)		; 19-21
this	vmulpd	y10, y10, y2		;; I3 * R3 (I3/2)			; 19-23
next	vmovapd	y13, [srcreg+(iter+1)*srcinc+d1] ;; R2

this	vsubpd	y2, y1, y5		;; R1 - R2 (new R2)			; 20-22
this	vmulpd	y8, y8, y3		;; I4 * R4 (I4/2)			; 20-24
next	vmovapd	y3, [srcreg+(iter+1)*srcinc+d2+d1] ;; R4

this	vaddpd	y1, y1, y5		;; R1 + R2 (new R1)			; 21-23
this	vmulpd	y14, y14, ymm15		;; I1/2 * 2				; 21-25

this	vsubpd	y4, y4, y6		;; R3^2 - I3^2 (R3)			; 22-24
this	vmulpd	y7, y7, ymm15		;; I2/2 * 2				; 22-26

this	vsubpd	y11, y11, y9		;; R4^2 - I4^2 (R4)			; 23-25
this	L1prefetchw srcreg+iter*srcinc+d2+L1pd, L1pt

next	vaddpd	y9, y12, y0		;; R1 + R3 (new R1)			; 24-26

this	vsubpd	y6, y10, y8		;; I3/2 - I4/2 (new R4/2)		; 25-27

this	vaddpd	y5, y11, y4		;; R4 + R3 (new R3)			; 26-28

this	vaddpd	y10, y10, y8		;; I3/2 + I4/2 (new I3/2)		; 27-29
next	vmovapd	y8, [srcreg+(iter+1)*srcinc+32] ;; I1

this	vsubpd	y11, y11, y4		;; R4 - R3 (new I4)			; 28-30
this	vmulpd	y6, y6, ymm15		;; R4/2 * 2				; 28-32

this	vsubpd	y4, y1, y5		;; R1 - R3 (final R3)			; 29-31
this	L1prefetchw srcreg+iter*srcinc+d2+d1+L1pd, L1pt

this	vaddpd	y1, y1, y5		;; R1 + R3 (final R1)			; 30-32
this	vmulpd	y10, y10, ymm15		;; I3/2 * 2				; 30-34

this	vsubpd	y5, y7, y11		;; I2 - I4 (final I4)			; 31-33

this	vaddpd	y7, y7, y11		;; I2 + I4 (final I2)			; 32-34
next	vmovapd	y11, [srcreg+(iter+1)*srcinc+d2+32] ;; I3
this	ystore	[srcreg+iter*srcinc+d2], y4 ;; Save R3				; 32

this	vsubpd	y4, y2, y6		;; R2 - R4 (final R4)			; 33-35
this	ystore	[srcreg+iter*srcinc], y1 ;; Save R1				; 33

this	vaddpd	y2, y2, y6		;; R2 + R4 (final R2)			; 34-36
this	ystore	[srcreg+iter*srcinc+d2+d1+32], y5 ;; Save I4			; 34

this	vsubpd	y5, y14, y10		;; I1 - I3 (final I3)			; 35-37
this	ystore	[srcreg+iter*srcinc+d1+32], y7 ;; Save I2			; 35

this	vaddpd	y14, y14, y10		;; I1 + I3 (final I1)			; 36-38
this	ystore	[srcreg+iter*srcinc+d2+d1], y4 ;; Save R4			; 36

;; Shuffle register assignments so that next call has R2,I3,I1 in y0-2 and next R1,R2,R3,R4,new R1,I1,I3 in y3-9.

ytmp	TEXTEQU	y0
y0	TEXTEQU	y2
y2	TEXTEQU	y14
y14	TEXTEQU	y4
y4	TEXTEQU	y13
y13	TEXTEQU	y6
y6	TEXTEQU	y3
y3	TEXTEQU	y12
y12	TEXTEQU	y7
y7	TEXTEQU	y9
y9	TEXTEQU	y11
y11	TEXTEQU	y1
y1	TEXTEQU y5
y5	TEXTEQU	ytmp
	ENDM

ENDIF


;; Haswell FMA3 version

IF (@INSTR(,%yarch,<FMA3>) NE 0)

yr4_4cl_four_complex_with_square_preload MACRO
	vmovapd	ymm15, YMM_ONE
	ENDM

;; 8 loads, 8 stores, 6 muls, 37 FMAs, 16 reg copies, 2 constant loads  = 87 uops = 87/4 = 21.75 clocks
;; Timed at 24.5 clocks.
yr4_4c_with_square_unroll MACRO iter,prev,this,next,srcreg,srcinc,d1,d2,L1pt,L1pd

;; On first call, init registers

	IF iter EQ -1
y0	TEXTEQU	<ymm0>
y1	TEXTEQU	<ymm1>
y2	TEXTEQU	<ymm2>
y3	TEXTEQU	<ymm3>
y4	TEXTEQU	<ymm4>
y5	TEXTEQU	<ymm5>
y6	TEXTEQU	<ymm6>
y7	TEXTEQU	<ymm7>
y8	TEXTEQU	<ymm8>
y9	TEXTEQU	<ymm9>
y10	TEXTEQU	<ymm10>
y11	TEXTEQU	<ymm11>
y12	TEXTEQU	<ymm12>
y13	TEXTEQU	<ymm13>
y14	TEXTEQU	<ymm14>
	ENDIF

;; On later calls, previous I1,R2,I2,I3.I4 will be in y0-4.  This R1,I1,R2,I2,R3,I3,R4,I4 will be in y5-12.
;; The remaining registers are free.  ymm15 is reserved for the constants 1 and 2.

this	vmulpd	y13, y5, y5			;; R1 * R1				; 1-5 (used 6)
this	vmulpd	y14, y7, y7			;; R2 * R2				; 1-5
prev	ystore	[srcreg+(iter-1)*srcinc+d1], y1	;; Save R2				; 23

this	vmulpd	y1, y10, y10			;; I3 * I3				; 2-6 (used 7)
prev	ystore	[srcreg+(iter-1)*srcinc+d2+d1+32], y4 ;; Save I4			; 24
this	vmulpd	y4, y12, y12			;; I4 * I4				; 2-6
prev	ystore	[srcreg+(iter-1)*srcinc+d1+32], y2 ;; Save I2				; 24

next	vmovapd	y2, [srcreg+(iter+1)*srcinc]	;; R1
prev	ystore	[srcreg+(iter-1)*srcinc+d2+32], y3 ;; Save I3				; 25
next	vmovapd	y3, [srcreg+(iter+1)*srcinc+d2] ;; R3
prev	ystore	[srcreg+(iter-1)*srcinc+32], y0	;; Save I1				; 25

next	yfmaddpd y0, y2, ymm15, y3		;; R1 + R3 (fwd new R1)			;	3-7  (used 13)
next	yfmsubpd y2, y2, ymm15, y3		;; R1 - R3 (fwd new R3)			;	3-7  (used 21/22)
next	vmovapd	y3, [srcreg+(iter+1)*srcinc+d1] ;; R2

this	vmulpd	y10, y10, y9			;; I3 * R3 (I3/2)			; 4-8  (used 9)
this	vmulpd	y5, y6, y5			;; I1 * R1 (I1/2)			; 4-8  (used 10)
this	L1prefetchw srcreg+iter*srcinc+L1pd, L1pt

this	yfnmaddpd y6, y6, y6, y13		;; R1^2 - I1 * I1 (R1)			; 6-10  (used 11)
this	yfnmaddpd y14, y8, y8, y14		;; R2^2 - I2 * I2 (R2)			; 6-10
next	vmovapd	y13, [srcreg+(iter+1)*srcinc+d2+d1] ;; R4

this	yfmsubpd y9, y9, y9, y1			;; R3 * R3 - I3^2 (R3)			; 7-11  (used 12)
this	yfmsubpd y4, y11, y11, y4		;; R4 * R4 - I4^2 (R4)			; 7-11
this	L1prefetchw srcreg+iter*srcinc+d1+L1pd, L1pt

next	yfmaddpd y1, y3, ymm15, y13		;; R2 + R4 (fwd new R2)			;	5-9  (used 13)
next	yfmsubpd y3, y3, ymm15, y13		;; R2 - R4 (fwd new R4)			;	5-9  (used 15)

this	yfmaddpd y13, y12, y11, y10		;; I3/2 + I4 * R4 (new I3/2)		; 9-13  (used 14)
this	yfnmaddpd y12, y12, y11, y10		;; I3/2 - I4 * R4 (new R4/2)		; 9-13  (used 18)
next	vmovapd	y11, [srcreg+(iter+1)*srcinc+32] ;; I1

this	yfnmaddpd y10, y8, y7, y5		;; I1/2 - I2 * R2 (new I2/2)		; 10-14  (used 19)
this	yfmaddpd y8, y8, y7, y5			;; I1/2 + I2 * R2 (new I1/2)		; 10-14  (used 20)
next	vmovapd	y7, [srcreg+(iter+1)*srcinc+d2+32] ;; I3

next	yfmsubpd y5, y11, ymm15, y7		;; I1 - I3 (fwd new I3)			;	8-12  (used 15)
next	yfmaddpd y11, y11, ymm15, y7		;; I1 + I3 (fwd new I1)			;	8-12  (used 21/22)
this	L1prefetchw srcreg+iter*srcinc+d2+L1pd, L1pt

this	yfmaddpd y7, y6, ymm15, y14		;; R1 + R2 (new R1)			; 11-15  (used 17)
this	yfmsubpd y6, y6, ymm15, y14		;; R1 - R2 (new R2)			; 11-15  (used 18)

this	yfmaddpd y14, y4, ymm15, y9		;; R4 + R3 (new R3)			; 12-16  (used 17)
this	yfmsubpd y4, y4, ymm15, y9		;; R4 - R3 (new I4)			; 12-16  (used 19)
this	L1prefetchw srcreg+iter*srcinc+d2+d1+L1pd, L1pt

next	yfmaddpd y9, y0, ymm15, y1		;; R1 + R2 (fwd final R1)		;	13-17  (used next 1)
next	yfmsubpd y0, y0, ymm15, y1		;; R1 - R2 (fwd final R2)		;	13-17

this	yfmaddpd y13, y13, ymm15, y13		;; I3/2 * 2				; 14-18  (used 20)
this next yloop_unrolled_one

next	yfmaddpd y1, y5, ymm15, y3		;; I3 + R4 (fwd final I3)		;	15-19  (used next 2)
next	yfmsubpd y5, y5, ymm15, y3		;; I3 - R4 (fwd final I4)		;	15-19

this	yfmaddpd y3, y7, ymm15, y14		;; R1 + R3 (final R1)			; 17-21
this	yfmsubpd y7, y7, ymm15, y14		;; R1 - R3 (final R3)			; 17-21
next	vmovapd	y14, [srcreg+(iter+1)*srcinc+d1+32] ;; I2
this	ystore	[srcreg+iter*srcinc], y3	;; Save R1				; 22
next	vmovapd	y3, [srcreg+(iter+1)*srcinc+d2+d1+32] ;; I4
this	ystore	[srcreg+iter*srcinc+d2], y7	;; Save R3				; 22

next	yfmaddpd y7, y14, ymm15, y3		;; I2 + I4 (fwd new I2)			;	16-20  (used 21/22)
next	yfmsubpd y14, y14, ymm15, y3		;; I2 - I4 (fwd new I4)			;	16-20  (used 21/22)

	vmovapd	ymm15, YMM_TWO			;; Load constant 2
this	yfnmaddpd y3, y12, ymm15, y6		;; R2 - 2 * R4/2 (final R4)		; 18-22
this	yfmaddpd y12, y12, ymm15, y6		;; R2 + 2 * R4/2 (final R2)		; 18-22

this	yfmsubpd y6, y10, ymm15, y4		;; 2 * I2/2 - I4 (final I4)		; 19-23
this	yfmaddpd y10, y10, ymm15, y4		;; 2 * I2/2 + I4 (final I2)		; 19-23

this	yfmsubpd y4, y8, ymm15, y13		;; 2 * I1/2 - I3 (final I3)		; 20-24
this	yfmaddpd y8, y8, ymm15, y13		;; 2 * I1/2 + I3 (final I1)		; 20-24
	vmovapd	ymm15, YMM_ONE			;; Load constant 1

next	yfmaddpd y13, y11, ymm15, y7		;; I1 + I2 (fwd final I1)		;	21-25 (used next 4)
this	ystore	[srcreg+iter*srcinc+d2+d1], y3	;; Save R4				; 23
next	yfmsubpd y3, y2, ymm15, y14		;; R3 - I4 (fwd final R3)		;	21-25 (used next 4)

next	yfmsubpd y11, y11, ymm15, y7		;; I1 - I2 (fwd final I2)		;	22-26 (used next 6)
next	yfmaddpd y2, y2, ymm15, y14		;; R3 + I4 (fwd final R4)		;	22-26 (used next 7)

;; Shuffle register assignments so that next call has I1,R2,I2,I3.I4 in y0-4 and next R1,I1,R2,I2,R3,I3,R4,I4 in y5-12.

ytmp	TEXTEQU	y0
y0	TEXTEQU	y8
y8	TEXTEQU	y11
y11	TEXTEQU	y2
y2	TEXTEQU	y10
y10	TEXTEQU	y1
y1	TEXTEQU	y12
y12	TEXTEQU	y5
y5	TEXTEQU	y9
y9	TEXTEQU	y3
y3	TEXTEQU	y4
y4	TEXTEQU	y6
y6	TEXTEQU y13
y13	TEXTEQU y7
y7	TEXTEQU	ytmp
	ENDM

ENDIF

;; Haswell FMA3 version
;; Attemptimg a version that uses add and sub instructions to reduce the number of register copies.  Untested, but alas timed at a worse 27.5 clocks.

IF (@INSTR(,%yarch,<NOT_BEST_FMA3>) NE 0)

yr4_4cl_four_complex_with_square_preload MACRO
	ENDM

;; 10 loads, 8 stores, 22 adds, 21 FMA/muls, 5 reg copies, 1 constant loads  = 67 uops = 67/4 = 16.75 clocks.
;; This is well below our 22 clock best possible speed.
yr4_4c_with_square_unroll MACRO iter,prev,this,next,srcreg,srcinc,d1,d2,L1pt,L1pd

;; On first call, init registers

	IF iter EQ -1
y0	TEXTEQU	<ymm0>
y1	TEXTEQU	<ymm1>
y2	TEXTEQU	<ymm2>
y3	TEXTEQU	<ymm3>
y4	TEXTEQU	<ymm4>
y5	TEXTEQU	<ymm5>
y6	TEXTEQU	<ymm6>
y7	TEXTEQU	<ymm7>
y8	TEXTEQU	<ymm8>
y9	TEXTEQU	<ymm9>
y10	TEXTEQU	<ymm10>
y11	TEXTEQU	<ymm11>
y12	TEXTEQU	<ymm12>
y13	TEXTEQU	<ymm13>
y14	TEXTEQU	<ymm14>
y15	TEXTEQU	<ymm15>
	ENDIF

;; On later calls, previous R4,R2,I4,I2,I3,I1 will be in y0-5.  This newI3,newR4,newR3,newI4,newI1,newI2,R1,R2,I3 will be in y6-14.
;; The remaining register is free.

this	vsubpd	y6, y6, y7			;; I3 - R4 (fwd final I4)		; 1-3 (used 4)
this	vmulpd	y7, y12, y12			;; R1 * R1				;	1-5 (used 6)
prev	ystore	[srcreg+(iter-1)*srcinc+d2+d1], y0 ;; Save R4				;		22+1
this	L1prefetchw srcreg+iter*srcinc+L1pd, L1pt

this	vsubpd	y0, y8, y9			;; R3 - I4 (fwd final R3)		; 2-4 (used 5)
this	vmulpd	y15, y13, y13			;; R2 * R2				;	2-6 (used 7)
prev	ystore	[srcreg+(iter-1)*srcinc+d1], y1	;; Save R2				;		23+1

this	vaddpd	y1, y10, y11			;; I1 + I2 (fwd final I1)		; 3-5 (used 6)
prev	ystore	[srcreg+(iter-1)*srcinc+d2+d1+32], y2 ;; Save I4			;		24+1
this	vmulpd	y2, y14, y14			;; I3 * I3				;	3-7 (used 9)
this	L1prefetchw srcreg+iter*srcinc+d1+L1pd, L1pt

this	vsubpd	y10, y10, y11			;; I1 - I2 (fwd final I2)		; 4-6 (used 7)
this	vmulpd	y11, y6, y6			;; I4 * I4				;	4-8 (used 10)
prev	ystore	[srcreg+(iter-1)*srcinc+d1+32], y3 ;; Save I2				;		25+1
next	vmovapd	y3, [srcreg+(iter+1)*srcinc]	;; R1

this	vaddpd	y8, y8, y9			;; R3 + I4 (fwd final R4)		; 5-7 (used 10)
this	vmulpd	y14, y14, y0			;; I3 * R3 (I3/2)			;	5-9 (used 11)
prev	ystore	[srcreg+(iter-1)*srcinc+d2+32], y4 ;; Save I3				;		26+1

next	vaddpd	y9, y3, [srcreg+(iter+1)*srcinc+d2] ;; R1 + R3 (fwd new R1)		; 6-8 (used 10)
this	yfnmaddpd y7, y1, y1, y7		;; R1^2 - I1 * I1 (R1)			;	6-10 (used 12)
next	vmovapd	y4, [srcreg+(iter+1)*srcinc+d1] ;; R2
prev	ystore	[srcreg+(iter-1)*srcinc+32], y5	;; Save I1				;		27+1

next	vaddpd	y5, y4, [srcreg+(iter+1)*srcinc+d2+d1] ;; R2 + R4 (fwd new R2)		; 7-9 (used 10)
this	yfnmaddpd y15, y10, y10, y15		;; R2^2 - I2 * I2 (R2)			;	7-11 (used 12)

next	vsubpd	y3, y3, [srcreg+(iter+1)*srcinc+d2] ;; R1 - R3 (fwd new R3)		; 8-10 (used next 2)
this	vmulpd	y1, y1, y12			;; I1 * R1 (I1/2)			;	8-12 (used 13)

next	vsubpd	y4, y4, [srcreg+(iter+1)*srcinc+d2+d1] ;; R2 - R4 (fwd new R4)		; 9-11 (used 22)
this	yfmsubpd y0, y0, y0, y2			;; R3 * R3 - I3^2 (R3)			;	9-13 (used 15)
this	L1prefetchw srcreg+iter*srcinc+d2+L1pd, L1pt

next	vaddpd	y2, y9, y5			;; R1 + R2 (fwd final R1)		; 10-12 (used next 1)
this	yfmsubpd y11, y8, y8, y11		;; R4 * R4 - I4^2 (R4)			;	10-14 (used 15)

next	vsubpd	y9, y9, y5			;; R1 - R2 (fwd final R2)		; 11-13 (used next 2)
this	yfmaddpd y5, y6, y8, y14		;; I3/2 + I4 * R4 (new I3/2)		;	11-15 (used 16)
this	L1prefetchw srcreg+iter*srcinc+d2+d1+L1pd, L1pt

this	vsubpd	y12, y7, y15			;; R1 - R2 (new R2)			; 12-14 (used 17)
this	yfnmaddpd y6, y6, y8, y14		;; I3/2 - I4 * R4 (new R4/2)		;	12-16 (used 17)
next	vmovapd	y8, [srcreg+(iter+1)*srcinc+32] ;; I1

this	vaddpd	y7, y7, y15			;; R1 + R2 (new R1)			; 13-15 (used 18)
this	yfnmaddpd y15, y10, y13, y1		;; I1/2 - I2 * R2 (new I2/2)		;	13-17 (used 19)
next	vmovapd	y14, [srcreg+(iter+1)*srcinc+d2+32] ;; I3

this	yfmaddpd y10, y10, y13, y1		;; I1/2 + I2 * R2 (new I1/2)		;	14-18 (used 21)
next	vsubpd	y13, y8, y14			;; I1 - I3 (fwd new I3)			; 14-16 (used 22)

this	vaddpd	y1, y11, y0			;; R4 + R3 (new R3)			; 15-17 (used 18)
this next yloop_unrolled_one

this	vsubpd	y11, y11, y0			;; R4 - R3 (new I4)			; 16-18 (used 19)
this	vmovapd	y0, YMM_TWO			;; Load constant 2
this	vmulpd	y5, y5, y0			;; I3/2 * 2				;	16-20 (used 21)

next	vaddpd	y8, y8, y14			;; I1 + I3 (fwd new I1)			; 17-19 (used next 3)
this	yfnmaddpd y14, y6, y0, y12		;; R2 - R4/2 * 2 (final R4)		;	17-21

this	yfmaddpd y6, y6, y0, y12		;; R2 + R4/2 * 2 (final R2)		;	18-22
this	vaddpd	y12, y7, y1			;; R1 + R3 (final R1)			; 18-20

this	vsubpd	y7, y7, y1			;; R1 - R3 (final R3)			; 19-21
this	yfmsubpd y1, y15, y0, y11		;; I2/2 * 2 - I4 (final I4)		;	19-23

this	ystore	[srcreg+iter*srcinc], y12	;; Save R1				;		21
next	vmovapd	y12, [srcreg+(iter+1)*srcinc+d1+32] ;; I2
this	ystore	[srcreg+iter*srcinc+d2], y7	;; Save R3				;		22
next	vmovapd	y7, [srcreg+(iter+1)*srcinc+d2+d1+32] ;; I4
this	yfmaddpd y15, y15, y0, y11		;; I2/2 * 2 + I4 (final I2)		;	20-24
next	vsubpd	y11, y12, y7			;; I2 - I4 (fwd new I4)			; 20-22 (used next 2)

next	vaddpd	y12, y12, y7			;; I2 + I4 (fwd new I2)			; 21-23 (used next 3)
this	yfmsubpd y7, y10, y0, y5		;; I1/2 * 2 - I3 (final I3)		;	21-25

this	yfmaddpd y10, y10, y0, y5		;; I1/2 * 2 + I3 (final I1)		;	22-26
next	vaddpd	y5, y13, y4			;; I3 + R4 (fwd final I3)		; 22-24 (used next 3)

;; Shuffle register assignments so that this R4,R2,I4,I2,I3,I1 in y0-5 and next newI3,newR4,newR3,newI4,newI1,newI2,R1,R2,I3 in y6-14.

ytmp	TEXTEQU	y0
y0	TEXTEQU	y14
y14	TEXTEQU	y5
y5	TEXTEQU	y10
y10	TEXTEQU	y8
y8	TEXTEQU	y3
y3	TEXTEQU	y15
y15	TEXTEQU	ytmp
ytmp	TEXTEQU	y1
y1	TEXTEQU	y6
y6	TEXTEQU	y13
y13	TEXTEQU	y9
y9	TEXTEQU y11
y11	TEXTEQU y12
y12	TEXTEQU	y2
y2	TEXTEQU	ytmp
ytmp	TEXTEQU	y4
y4	TEXTEQU	y7
y7	TEXTEQU	ytmp
	ENDM

ENDIF


;;
;; ************************************* four-complex-fft4 variants ******************************************
;;
;; These macros are used in the last levels of pass 1.  Four sin/cos multipliers are needed to
;; finish off the partial sin/cos multiplies that were done in the first levels of pass 1.
;; FFTs of type r4delay and r4dwpn do this to reduce memory usage at the cost of some extra
;; complex multiplies.


;;
;; In the split premultiplier case, we apply part of the roots of -1 at the
;; end of the first pass.  Thus we have 4 sin/cos/premultipliers instead
;; of the usual 3.
;;

;; Used in last levels of pass 1 (split premultiplier, delay, and dwpn cases).  Swizzling.
yr4_sg4cl_four_complex_fft4_preload MACRO
	ENDM
yr4_sg4cl_four_complex_fft4 MACRO srcreg,srcinc,d1,d2,dstreg,dstinc,e1,e2,screg,scinc,maxrpt,L1pt,L1pd

	;; On input the 64-byte cache lines hold these data values:
	;;	0	+1K	+1K	+1K	4K	+1K	+1K	+1K
	;;	1	...
	;;	2	...
	;;	3	...

	;; These are swizzled to:
	;;	0	+1	+1	+1	4K	+1	+1	+1
	;;	1K	...
	;;	2K	...
	;;	3K	...

	;; Swizzle using 8 shufpds and 8 perm2f128  (1 or 3 clocks each on port 5, throughput 1)

	vmovapd	ymm1, [srcreg]			;; R1
	vmovapd	ymm7, [srcreg+d1]		;; R2
	vshufpd	ymm0, ymm1, ymm7, 15		;; Shuffle R1 and R2 to create R1/R2 hi			; 1
	vshufpd	ymm1, ymm1, ymm7, 0		;; Shuffle R1 and R2 to create R1/R2 low		; 2

	vmovapd	ymm2, [srcreg+d2]		;; R3
	vmovapd	ymm7, [srcreg+d2+d1]		;; R4
	vshufpd	ymm3, ymm2, ymm7, 15		;; Shuffle R3 and R4 to create R3/R4 hi			; 3
	vshufpd	ymm2, ymm2, ymm7, 0		;; Shuffle R3 and R4 to create R3/R4 low		; 4

	ylow128s ymm4, ymm0, ymm3		;; Shuffle R1/R2 hi and R3/R4 hi (new R2)		; 5-6
	yhigh128s ymm0, ymm0, ymm3		;; Shuffle R1/R2 hi and R3/R4 hi (new R4)		; 6-7

	ylow128s ymm3, ymm1, ymm2		;; Shuffle R1/R2 low and R3/R4 low (new R1)		; 7-8
	yhigh128s ymm1, ymm1, ymm2		;; Shuffle R1/R2 low and R3/R4 low (new R3)		; 8-9

	vaddpd	ymm5, ymm4, ymm0		;; R2 + R4 (newer R2)
	vsubpd	ymm4, ymm4, ymm0		;; R2 - R4 (newer R4)

	L1prefetch srcreg+L1pd, L1pt
	L1prefetchw dstreg+L1pd, L1pt - L1PREFETCH_DEST_NONE

	vaddpd	ymm0, ymm3, ymm1		;; R1 + R3 (newer R1)
	vsubpd	ymm3, ymm3, ymm1		;; R1 - R3 (newer R3)

	vsubpd	ymm1, ymm0, ymm5		;; R1 - R2 (final R2)
	vaddpd	ymm0, ymm0, ymm5		;; R1 + R2 (final R1)

	vmovapd	ymm6, [srcreg+32]		;; I1
	vmovapd	ymm2, [srcreg+d1+32]		;; I2
	vshufpd	ymm5, ymm6, ymm2, 15		;; Shuffle I1 and I2 to create I1/I2 hi
	vshufpd	ymm6, ymm6, ymm2, 0		;; Shuffle I1 and I2 to create I1/I2 low

	ystore	[dstreg], ymm0			;; Temporarily save final R1

	vmovapd	ymm2, [srcreg+d2+32]		;; I3
	vmovapd	ymm7, [srcreg+d2+d1+32]		;; I4
	vshufpd	ymm0, ymm2, ymm7, 15		;; Shuffle I3 and I4 to create I3/I4 hi
	vshufpd	ymm2, ymm2, ymm7, 0		;; Shuffle I3 and I4 to create I3/I4 low

	ylow128s ymm7, ymm5, ymm0		;; Shuffle I1/I2 hi and I3/I4 hi (new I2)
	yhigh128s ymm5, ymm5, ymm0		;; Shuffle I1/I2 hi and I3/I4 hi (new I4)

	ylow128s ymm0, ymm6, ymm2		;; Shuffle I1/I2 low and I3/I4 low (new I1)
	yhigh128s ymm6, ymm6, ymm2		;; Shuffle I1/I2 low and I3/I4 low (new I3)

	vaddpd	ymm2, ymm7, ymm5		;; I2 + I4 (newer I2)
	vsubpd	ymm7, ymm7, ymm5		;; I2 - I4 (newer I4)

	L1prefetch srcreg+d1+L1pd, L1pt
	L1prefetchw dstreg+e1+L1pd, L1pt - L1PREFETCH_DEST_NONE

	vaddpd	ymm5, ymm0, ymm6		;; I1 + I3 (newer I1)
	vsubpd	ymm0, ymm0, ymm6		;; I1 - I3 (newer I3)

	vsubpd	ymm6, ymm3, ymm7		;; R3 - I4 (final R3)
	vaddpd	ymm3, ymm3, ymm7		;; R3 + I4 (final R4)

	vsubpd	ymm7, ymm0, ymm4		;; I3 - R4 (final I4)
	vaddpd	ymm0, ymm0, ymm4		;; I3 + R4 (final I3)

	L1prefetch srcreg+d2+L1pd, L1pt
	L1prefetchw dstreg+e2+L1pd, L1pt - L1PREFETCH_DEST_NONE

	vsubpd	ymm4, ymm5, ymm2		;; I1 - I2 (final I2)
	vaddpd	ymm5, ymm5, ymm2		;; I1 + I2 (final I1)

	ystore	[dstreg+32], ymm5		;; Temporarily save final I1

	vmovapd	ymm2, [screg+64+32]		;; cosine/sine
	vmulpd	ymm5, ymm6, ymm2		;; A3 = R3 * cosine/sine
	vsubpd	ymm5, ymm5, ymm0		;; A3 = A3 - I3
	vmulpd	ymm0, ymm0, ymm2		;; B3 = I3 * cosine/sine
	vaddpd	ymm0, ymm0, ymm6		;; B3 = B3 + R3

	L1prefetch srcreg+d2+d1+L1pd, L1pt
	L1prefetchw dstreg+e2+e1+L1pd, L1pt - L1PREFETCH_DEST_NONE

	vmovapd	ymm2, [screg+128+32]		;; cosine/sine
	vmulpd	ymm6, ymm1, ymm2		;; A2 = R2 * cosine/sine
	vsubpd	ymm6, ymm6, ymm4		;; A2 = A2 - I2
	vmulpd	ymm4, ymm4, ymm2		;; B2 = I2 * cosine/sine
	vaddpd	ymm4, ymm4, ymm1		;; B2 = B2 + R2

	vmovapd	ymm2, [screg+192+32]		;; cosine/sine
	vmulpd	ymm1, ymm3, ymm2		;; A4 = R4 * cosine/sine
	vsubpd	ymm1, ymm1, ymm7		;; A4 = A4 - I4
	vmulpd	ymm7, ymm7, ymm2		;; B4 = I4 * cosine/sine
	vaddpd	ymm7, ymm7, ymm3		;; B4 = B4 + R4

	vmovapd	ymm2, [screg+64]
	vmulpd	ymm5, ymm5, ymm2		;; A3 = A3 * sine (final R3)
	vmulpd	ymm0, ymm0, ymm2		;; B3 = B3 * sine (final I3)
	vmovapd	ymm2, [screg+128]
	vmulpd	ymm6, ymm6, ymm2		;; A2 = A2 * sine (final R2)
	vmulpd	ymm4, ymm4, ymm2		;; B2 = B2 * sine (final I2)

	ystore	[dstreg+e2], ymm5		;; Save R3
	ystore	[dstreg+e2+32], ymm0		;; Save I3

	vmovapd	ymm2, [screg+0+32]		;; cosine/sine
	vmovapd	ymm3, [dstreg]			;; Reload R1
	vmulpd	ymm5, ymm3, ymm2		;; A1 = R1 * cosine/sine
	vmovapd	ymm0, [dstreg+32]		;; Restore I1
	vsubpd	ymm5, ymm5, ymm0		;; A1 = A1 - I1
	vmulpd	ymm0, ymm0, ymm2		;; B1 = I1 * cosine/sine
	vaddpd	ymm0, ymm0, ymm3		;; B1 = B1 + R1

	vmovapd	ymm3, [screg+192]		;; sine
	vmulpd	ymm1, ymm1, ymm3		;; A4 = A4 * sine (final R4)
	vmulpd	ymm7, ymm7, ymm3		;; B4 = B4 * sine (final I4)
	vmovapd	ymm3, [screg+0]			;; sine
	vmulpd	ymm5, ymm5, ymm3		;; A1 = A1 * sine (final R1)
	vmulpd	ymm0, ymm0, ymm3		;; B1 = B1 * sine (final I1)

	ystore	[dstreg], ymm5			;; Save R1
	ystore	[dstreg+32], ymm0		;; Save I1
	ystore	[dstreg+e1], ymm6		;; Save R2
	ystore	[dstreg+e1+32], ymm4		;; Save I2
	ystore	[dstreg+e2+e1], ymm1		;; Save R4
	ystore	[dstreg+e2+e1+32], ymm7		;; Save I4

	bump	srcreg, srcinc
	bump	dstreg, dstinc
	bump	screg, scinc
	ENDM


;; 64-bit version

IFDEF X86_64

yr4_sg4cl_four_complex_fft4_preload MACRO
	ENDM
yr4_sg4cl_four_complex_fft4 MACRO srcreg,srcinc,d1,d2,dstreg,dstinc,e1,e2,screg,scinc,maxrpt,L1pt,L1pd

	;; On input the 64-byte cache lines hold these data values:
	;;	0	+1K	+1K	+1K	4K	+1K	+1K	+1K
	;;	1	...
	;;	2	...
	;;	3	...

	;; These are swizzled to:
	;;	0	+1	+1	+1	4K	+1	+1	+1
	;;	1K	...
	;;	2K	...
	;;	3K	...

	;; Swizzle using 8 shufpds and 8 perm2f128  (1 or 3 clocks each on port 5, throughput 1)

	vmovapd	ymm1, [srcreg]			;; R1
	vmovapd	ymm7, [srcreg+d1]		;; R2
	vshufpd	ymm0, ymm1, ymm7, 15		;; Shuffle R1 and R2 to create R1/R2 hi			; 1
	vshufpd	ymm1, ymm1, ymm7, 0		;; Shuffle R1 and R2 to create R1/R2 low		; 2

	vmovapd	ymm2, [srcreg+d2]		;; R3
	vmovapd	ymm7, [srcreg+d2+d1]		;; R4
	vshufpd	ymm3, ymm2, ymm7, 15		;; Shuffle R3 and R4 to create R3/R4 hi			; 3
	vshufpd	ymm2, ymm2, ymm7, 0		;; Shuffle R3 and R4 to create R3/R4 low		; 4

	ylow128s ymm4, ymm0, ymm3		;; Shuffle R1/R2 hi and R3/R4 hi (new R2)		; 5-6
	yhigh128s ymm0, ymm0, ymm3		;; Shuffle R1/R2 hi and R3/R4 hi (new R4)		; 6-7

	ylow128s ymm3, ymm1, ymm2		;; Shuffle R1/R2 low and R3/R4 low (new R1)		; 7-8
	yhigh128s ymm1, ymm1, ymm2		;; Shuffle R1/R2 low and R3/R4 low (new R3)		; 8-9

	vaddpd	ymm5, ymm4, ymm0		;; R2 + R4 (newer R2)
	vsubpd	ymm4, ymm4, ymm0		;; R2 - R4 (newer R4)

	L1prefetch srcreg+L1pd, L1pt
	L1prefetchw dstreg+L1pd, L1pt - L1PREFETCH_DEST_NONE

	vaddpd	ymm0, ymm3, ymm1		;; R1 + R3 (newer R1)
	vsubpd	ymm3, ymm3, ymm1		;; R1 - R3 (newer R3)

	vsubpd	ymm1, ymm0, ymm5		;; R1 - R2 (final R2)
	vaddpd	ymm8, ymm0, ymm5		;; R1 + R2 (final R1)

	vmovapd	ymm6, [srcreg+32]		;; I1
	vmovapd	ymm2, [srcreg+d1+32]		;; I2
	vshufpd	ymm5, ymm6, ymm2, 15		;; Shuffle I1 and I2 to create I1/I2 hi
	vshufpd	ymm6, ymm6, ymm2, 0		;; Shuffle I1 and I2 to create I1/I2 low

	vmovapd	ymm2, [srcreg+d2+32]		;; I3
	vmovapd	ymm7, [srcreg+d2+d1+32]		;; I4
	vshufpd	ymm0, ymm2, ymm7, 15		;; Shuffle I3 and I4 to create I3/I4 hi
	vshufpd	ymm2, ymm2, ymm7, 0		;; Shuffle I3 and I4 to create I3/I4 low

	ylow128s ymm7, ymm5, ymm0		;; Shuffle I1/I2 hi and I3/I4 hi (new I2)
	yhigh128s ymm5, ymm5, ymm0		;; Shuffle I1/I2 hi and I3/I4 hi (new I4)

	ylow128s ymm0, ymm6, ymm2		;; Shuffle I1/I2 low and I3/I4 low (new I1)
	yhigh128s ymm6, ymm6, ymm2		;; Shuffle I1/I2 low and I3/I4 low (new I3)

	vaddpd	ymm2, ymm7, ymm5		;; I2 + I4 (newer I2)
	vsubpd	ymm7, ymm7, ymm5		;; I2 - I4 (newer I4)

	L1prefetch srcreg+d1+L1pd, L1pt
	L1prefetchw dstreg+e1+L1pd, L1pt - L1PREFETCH_DEST_NONE

	vaddpd	ymm5, ymm0, ymm6		;; I1 + I3 (newer I1)
	vsubpd	ymm0, ymm0, ymm6		;; I1 - I3 (newer I3)

	vsubpd	ymm6, ymm3, ymm7		;; R3 - I4 (final R3)
	vaddpd	ymm3, ymm3, ymm7		;; R3 + I4 (final R4)

	vsubpd	ymm7, ymm0, ymm4		;; I3 - R4 (final I4)
	vaddpd	ymm0, ymm0, ymm4		;; I3 + R4 (final I3)

	L1prefetch srcreg+d2+L1pd, L1pt
	L1prefetchw dstreg+e2+L1pd, L1pt - L1PREFETCH_DEST_NONE

	vsubpd	ymm4, ymm5, ymm2		;; I1 - I2 (final I2)
	vaddpd	ymm9, ymm5, ymm2		;; I1 + I2 (final I1)

	vmovapd	ymm2, [screg+64+32]		;; cosine/sine
	vmulpd	ymm5, ymm6, ymm2		;; A3 = R3 * cosine/sine
	vsubpd	ymm5, ymm5, ymm0		;; A3 = A3 - I3
	vmulpd	ymm0, ymm0, ymm2		;; B3 = I3 * cosine/sine
	vaddpd	ymm0, ymm0, ymm6		;; B3 = B3 + R3

	L1prefetch srcreg+d2+d1+L1pd, L1pt
	L1prefetchw dstreg+e2+e1+L1pd, L1pt - L1PREFETCH_DEST_NONE

	vmovapd	ymm2, [screg+128+32]		;; cosine/sine
	vmulpd	ymm6, ymm1, ymm2		;; A2 = R2 * cosine/sine
	vsubpd	ymm6, ymm6, ymm4		;; A2 = A2 - I2
	vmulpd	ymm4, ymm4, ymm2		;; B2 = I2 * cosine/sine
	vaddpd	ymm4, ymm4, ymm1		;; B2 = B2 + R2

	vmovapd	ymm2, [screg+192+32]		;; cosine/sine
	vmulpd	ymm1, ymm3, ymm2		;; A4 = R4 * cosine/sine
	vsubpd	ymm1, ymm1, ymm7		;; A4 = A4 - I4
	vmulpd	ymm7, ymm7, ymm2		;; B4 = I4 * cosine/sine
	vaddpd	ymm7, ymm7, ymm3		;; B4 = B4 + R4

	vmovapd	ymm2, [screg+64]
	vmulpd	ymm5, ymm5, ymm2		;; A3 = A3 * sine (final R3)
	vmulpd	ymm0, ymm0, ymm2		;; B3 = B3 * sine (final I3)
	vmovapd	ymm2, [screg+128]
	vmulpd	ymm6, ymm6, ymm2		;; A2 = A2 * sine (final R2)
	vmulpd	ymm4, ymm4, ymm2		;; B2 = B2 * sine (final I2)

	ystore	[dstreg+e2], ymm5		;; Save R3
	ystore	[dstreg+e2+32], ymm0		;; Save I3

	vmovapd	ymm2, [screg+0+32]		;; cosine/sine
	vmulpd	ymm5, ymm8, ymm2		;; A1 = R1 * cosine/sine
	vsubpd	ymm5, ymm5, ymm9		;; A1 = A1 - I1
	vmulpd	ymm0, ymm9, ymm2		;; B1 = I1 * cosine/sine
	vaddpd	ymm0, ymm0, ymm8		;; B1 = B1 + R1

	vmovapd	ymm3, [screg+192]		;; sine
	vmulpd	ymm1, ymm1, ymm3		;; A4 = A4 * sine (final R4)
	vmulpd	ymm7, ymm7, ymm3		;; B4 = B4 * sine (final I4)
	vmovapd	ymm3, [screg+0]			;; sine
	vmulpd	ymm5, ymm5, ymm3		;; A1 = A1 * sine (final R1)
	vmulpd	ymm0, ymm0, ymm3		;; B1 = B1 * sine (final I1)

	ystore	[dstreg], ymm5			;; Save R1
	ystore	[dstreg+32], ymm0		;; Save I1
	ystore	[dstreg+e1], ymm6		;; Save R2
	ystore	[dstreg+e1+32], ymm4		;; Save I2
	ystore	[dstreg+e2+e1], ymm1		;; Save R4
	ystore	[dstreg+e2+e1+32], ymm7		;; Save I4

	bump	srcreg, srcinc
	bump	dstreg, dstinc
	bump	screg, scinc
	ENDM

;; Haswell FMA3 version
;; This is the original Bulldozer version.  Timed at 28.2 clocks.

IF (@INSTR(,%yarch,<FMA3>) NE 0)

yr4_sg4cl_four_complex_fft4_preload MACRO
	vmovapd	ymm15, YMM_ONE
	ENDM

yr4_sg4cl_four_complex_fft4 MACRO srcreg,srcinc,d1,d2,dstreg,dstinc,e1,e2,screg,scinc,maxrpt,L1pt,L1pd

	;; On input the 64-byte cache lines hold these data values:
	;;	0	+1K	+1K	+1K	4K	+1K	+1K	+1K
	;;	1	...
	;;	2	...
	;;	3	...

	;; These are swizzled to:
	;;	0	+1	+1	+1	4K	+1	+1	+1
	;;	1K	...
	;;	2K	...
	;;	3K	...

	;; Swizzle using 8 shufpds and 8 perm2f128  (1 or 3 clocks each on port 5, throughput 1)

	vmovapd	ymm1, [srcreg]			;; R1
	vmovapd	ymm2, [srcreg+d1]		;; R2
	vshufpd	ymm0, ymm1, ymm2, 15		;; Shuffle R1 and R2 to create R1/R2 hi			; 1
	vshufpd	ymm1, ymm1, ymm2, 0		;; Shuffle R1 and R2 to create R1/R2 low		; 2

	vmovapd	ymm3, [srcreg+d2]		;; R3
	vmovapd	ymm4, [srcreg+d2+d1]		;; R4
	vshufpd	ymm2, ymm3, ymm4, 15		;; Shuffle R3 and R4 to create R3/R4 hi			; 3
	vshufpd	ymm3, ymm3, ymm4, 0		;; Shuffle R3 and R4 to create R3/R4 low		; 4

	vmovapd	ymm5, [srcreg+32]		;; I1
	vmovapd	ymm6, [srcreg+d1+32]		;; I2
	vshufpd	ymm4, ymm5, ymm6, 15		;; Shuffle I1 and I2 to create I1/I2 hi			; 5
	vshufpd	ymm5, ymm5, ymm6, 0		;; Shuffle I1 and I2 to create I1/I2 low		; 6

	vmovapd	ymm7, [srcreg+d2+32]		;; I3
	vmovapd	ymm8, [srcreg+d2+d1+32]		;; I4
	vshufpd	ymm6, ymm7, ymm8, 15		;; Shuffle I3 and I4 to create I3/I4 hi			; 7
	vshufpd	ymm7, ymm7, ymm8, 0		;; Shuffle I3 and I4 to create I3/I4 low		; 8

	ylow128s ymm8, ymm0, ymm2		;; Shuffle R1/R2 hi and R3/R4 hi (new R2)		; 9-11
	yhigh128s ymm0, ymm0, ymm2		;; Shuffle R1/R2 hi and R3/R4 hi (new R4)		; 10-12
	L1prefetch srcreg+L1pd, L1pt
	L1prefetchw dstreg+L1pd, L1pt - L1PREFETCH_DEST_NONE

	ylow128s ymm2, ymm1, ymm3		;; Shuffle R1/R2 low and R3/R4 low (new R1)		; 11-13
	yhigh128s ymm1, ymm1, ymm3		;; Shuffle R1/R2 low and R3/R4 low (new R3)		; 12-14

	yfmaddpd ymm3, ymm8, ymm15, ymm0	;; R2 + R4 (newer R2)					; 12-14
	yfmsubpd ymm8, ymm8, ymm15, ymm0	;; R2 - R4 (newer R4)					; 12-16
	L1prefetch srcreg+d1+L1pd, L1pt
	L1prefetchw dstreg+e1+L1pd, L1pt - L1PREFETCH_DEST_NONE

	ylow128s ymm0, ymm4, ymm6		;; Shuffle I1/I2 hi and I3/I4 hi (new I2)		; 13-15
	yhigh128s ymm4, ymm4, ymm6		;; Shuffle I1/I2 hi and I3/I4 hi (new I4)		; 14-16

	yfmaddpd ymm6, ymm2, ymm15, ymm1	;; R1 + R3 (newer R1)					; 14-16
	yfmsubpd ymm2, ymm2, ymm15, ymm1	;; R1 - R3 (newer R3)					; 14-18

	ylow128s ymm1, ymm5, ymm7		;; Shuffle I1/I2 low and I3/I4 low (new I1)		; 15-17
	yhigh128s ymm5, ymm5, ymm7		;; Shuffle I1/I2 low and I3/I4 low (new I3)		; 16-18

	yfmsubpd ymm7, ymm0, ymm15, ymm4	;; I2 - I4 (newer I4)					; 16-18
	yfmaddpd ymm0, ymm0, ymm15, ymm4	;; I2 + I4 (newer I2)					; 16-20
	vmovapd	ymm9, [screg+128+32]		;; cosine/sine for R2/I2

	yfmsubpd ymm4, ymm6, ymm15, ymm3	;; R1 - R2 (final R2)					; 17-19
	yfmaddpd ymm6, ymm6, ymm15, ymm3	;; R1 + R2 (final R1)					; 17-21
	vmovapd	ymm10, [screg+0+32]		;; cosine/sine for R1/I1

	yfmaddpd ymm3, ymm1, ymm15, ymm5	;; I1 + I3 (newer I1)					; 18-20
	yfmsubpd ymm1, ymm1, ymm15, ymm5	;; I1 - I3 (newer I3)					; 18-22
	vmovapd	ymm11, [screg+64+32]		;; cosine/sine for R3/I3

	yfmsubpd ymm5, ymm2, ymm15, ymm7	;; R3 - I4 (final R3)					; 19-21
	yfmaddpd ymm2, ymm2, ymm15, ymm7	;; R3 + I4 (final R4)					; 19-23
	vmovapd	ymm12, [screg+192+32]		;; cosine/sine for R4/I4

	yfmsubpd ymm7, ymm3, ymm15, ymm0	;; I1 - I2 (final I2)					; 21-23
	yfmaddpd ymm3, ymm3, ymm15, ymm0	;; I1 + I2 (final I1)					; 21-25
	vmovapd	ymm13, [screg+128]		;; sine for R2/I2

	yfmaddpd ymm0, ymm1, ymm15, ymm8	;; I3 + R4 (final I3)					; 23-25
	yfmsubpd ymm1, ymm1, ymm15, ymm8	;; I3 - R4 (final I4)					; 23-27
	vmovapd	ymm14, [screg+0]		;; sine for R1/I1

	yfmsubpd ymm8, ymm4, ymm9, ymm7		;; A2 = R2 * cosine/sine - I2				; 24-28
	yfmaddpd ymm7, ymm7, ymm9, ymm4		;; B2 = I2 * cosine/sine + R2				; 24-28
	vmovapd	ymm9, [screg+64]		;; sine for R3/I3

	yfmsubpd ymm4, ymm6, ymm10, ymm3	;; A1 = R1 * cosine/sine - I1				; 26-30
	yfmaddpd ymm3, ymm3, ymm10, ymm6	;; B1 = I1 * cosine/sine + R1				; 26-30
	vmovapd	ymm10, [screg+192]		;; sine for R4/I4

	yfmsubpd ymm6, ymm5, ymm11, ymm0	;; A3 = R3 * cosine/sine - I3				; 27-31
	yfmaddpd ymm0, ymm0, ymm11, ymm5	;; B3 = I3 * cosine/sine + R3				; 27-31
	L1prefetch srcreg+d2+L1pd, L1pt
	L1prefetchw dstreg+e2+L1pd, L1pt - L1PREFETCH_DEST_NONE

	yfmsubpd ymm5, ymm2, ymm12, ymm1	;; A4 = R4 * cosine/sine - I4				; 28-32
	yfmaddpd ymm1, ymm1, ymm12, ymm2	;; B4 = I4 * cosine/sine + R4				; 28-32

	vmulpd	ymm8, ymm8, ymm13		;; A2 = A2 * sine (final R2)				; 29-33
	vmulpd	ymm7, ymm7, ymm13		;; B2 = B2 * sine (final I2)				; 29-33
	L1prefetch srcreg+d2+d1+L1pd, L1pt
	L1prefetchw dstreg+e2+e1+L1pd, L1pt - L1PREFETCH_DEST_NONE

	vmulpd	ymm4, ymm4, ymm14		;; A1 = A1 * sine (final R1)				; 31-35
	vmulpd	ymm3, ymm3, ymm14		;; B1 = B1 * sine (final I1)				; 31-35

	vmulpd	ymm6, ymm6, ymm9		;; A3 = A3 * sine (final R3)				; 32-36
	vmulpd	ymm0, ymm0, ymm9		;; B3 = B3 * sine (final I3)				; 32-36

	vmulpd	ymm5, ymm5, ymm10		;; A4 = A4 * sine (final R4)				; 33-37
	vmulpd	ymm1, ymm1, ymm10		;; B4 = B4 * sine (final I4)				; 33-37

	ystore	[dstreg+e1], ymm8		;; Save R2						; 34
	ystore	[dstreg+e1+32], ymm7		;; Save I2						; 34+1
	ystore	[dstreg], ymm4			;; Save R1						; 36
	ystore	[dstreg+32], ymm3		;; Save I1						; 36+1
	ystore	[dstreg+e2], ymm6		;; Save R3						; 37+1
	ystore	[dstreg+e2+32], ymm0		;; Save I3						; 37+2
	ystore	[dstreg+e2+e1], ymm5		;; Save R4						; 38+2
	ystore	[dstreg+e2+e1+32], ymm1		;; Save I4						; 38+3

	bump	srcreg, srcinc
	bump	dstreg, dstinc
	bump	screg, scinc
	ENDM

ENDIF

;; Haswell FMA3 version
;; Trying to create an unrolled version.  Alas, it timed at a much worse 40 clocks.

IF (@INSTR(,%yarch,<NOT_BEST_FMA3>) NE 0)

yr4_sg4cl_four_complex_fft4_preload MACRO
	vmovapd	ymm15, YMM_ONE
	ENDM

yr4_sg4cl_four_complex_fft4 MACRO srcreg,srcinc,d1,d2,dstreg,dstinc,e1,e2,screg,scinc,maxrpt,L1pt,L1pd
	IF maxrpt MOD 5 EQ 0
	yr4_sg4c_djbfft_unroll -1,noexec,noexec,exec,srcreg,srcinc,d1,d2,dstreg,dstinc,e1,e2,screg,scinc,L1pt,L1pd
	yr4_sg4c_djbfft_unroll 0,noexec,exec,exec,srcreg,srcinc,d1,d2,dstreg,dstinc,e1,e2,screg,scinc,L1pt,L1pd
	yr4_sg4c_djbfft_unroll 1,exec,exec,exec,srcreg,srcinc,d1,d2,dstreg,dstinc,e1,e2,screg,scinc,L1pt,L1pd
	yr4_sg4c_djbfft_unroll 2,exec,exec,exec,srcreg,srcinc,d1,d2,dstreg,dstinc,e1,e2,screg,scinc,L1pt,L1pd
	yr4_sg4c_djbfft_unroll 3,exec,exec,exec,srcreg,srcinc,d1,d2,dstreg,dstinc,e1,e2,screg,scinc,L1pt,L1pd
	yr4_sg4c_djbfft_unroll 4,exec,exec,noexec,srcreg,srcinc,d1,d2,dstreg,dstinc,e1,e2,screg,scinc,L1pt,L1pd
	yr4_sg4c_djbfft_unroll 5,exec,noexec,noexec,srcreg,srcinc,d1,d2,dstreg,dstinc,e1,e2,screg,scinc,L1pt,L1pd
	bump	srcreg, 5*srcinc
	bump	dstreg, 5*dstinc
	bump	screg, 5*scinc
	ELSEIF maxrpt MOD 4 EQ 0
	yr4_sg4c_djbfft_unroll -1,noexec,noexec,exec,srcreg,srcinc,d1,d2,dstreg,dstinc,e1,e2,screg,scinc,L1pt,L1pd
	yr4_sg4c_djbfft_unroll 0,noexec,exec,exec,srcreg,srcinc,d1,d2,dstreg,dstinc,e1,e2,screg,scinc,L1pt,L1pd
	yr4_sg4c_djbfft_unroll 1,exec,exec,exec,srcreg,srcinc,d1,d2,dstreg,dstinc,e1,e2,screg,scinc,L1pt,L1pd
	yr4_sg4c_djbfft_unroll 2,exec,exec,exec,srcreg,srcinc,d1,d2,dstreg,dstinc,e1,e2,screg,scinc,L1pt,L1pd
	yr4_sg4c_djbfft_unroll 3,exec,exec,noexec,srcreg,srcinc,d1,d2,dstreg,dstinc,e1,e2,screg,scinc,L1pt,L1pd
	yr4_sg4c_djbfft_unroll 4,exec,noexec,noexec,srcreg,srcinc,d1,d2,dstreg,dstinc,e1,e2,screg,scinc,L1pt,L1pd
	bump	srcreg, 4*srcinc
	bump	dstreg, 4*dstinc
	bump	screg, 4*scinc
	ELSEIF maxrpt MOD 3 EQ 0
	yr4_sg4c_djbfft_unroll -1,noexec,noexec,exec,srcreg,srcinc,d1,d2,dstreg,dstinc,e1,e2,screg,scinc,L1pt,L1pd
	yr4_sg4c_djbfft_unroll 0,noexec,exec,exec,srcreg,srcinc,d1,d2,dstreg,dstinc,e1,e2,screg,scinc,L1pt,L1pd
	yr4_sg4c_djbfft_unroll 1,exec,exec,exec,srcreg,srcinc,d1,d2,dstreg,dstinc,e1,e2,screg,scinc,L1pt,L1pd
	yr4_sg4c_djbfft_unroll 2,exec,exec,noexec,srcreg,srcinc,d1,d2,dstreg,dstinc,e1,e2,screg,scinc,L1pt,L1pd
	yr4_sg4c_djbfft_unroll 3,exec,noexec,noexec,srcreg,srcinc,d1,d2,dstreg,dstinc,e1,e2,screg,scinc,L1pt,L1pd
	bump	srcreg, 3*srcinc
	bump	dstreg, 3*dstinc
	bump	screg, 3*scinc
	ELSEIF maxrpt MOD 2 EQ 0
	yr4_sg4c_djbfft_unroll -1,noexec,noexec,exec,srcreg,srcinc,d1,d2,dstreg,dstinc,e1,e2,screg,scinc,L1pt,L1pd
	yr4_sg4c_djbfft_unroll 0,noexec,exec,exec,srcreg,srcinc,d1,d2,dstreg,dstinc,e1,e2,screg,scinc,L1pt,L1pd
	yr4_sg4c_djbfft_unroll 1,exec,exec,noexec,srcreg,srcinc,d1,d2,dstreg,dstinc,e1,e2,screg,scinc,L1pt,L1pd
	yr4_sg4c_djbfft_unroll 2,exec,noexec,noexec,srcreg,srcinc,d1,d2,dstreg,dstinc,e1,e2,screg,scinc,L1pt,L1pd
	bump	srcreg, 2*srcinc
	bump	dstreg, 2*dstinc
	bump	screg, 2*scinc
	ELSE
	yr4_sg4c_djbfft_unroll -1,noexec,noexec,exec,srcreg,srcinc,d1,d2,dstreg,dstinc,e1,e2,screg,scinc,L1pt,L1pd
	yr4_sg4c_djbfft_unroll 0,noexec,exec,noexec,srcreg,srcinc,d1,d2,dstreg,dstinc,e1,e2,screg,scinc,L1pt,L1pd
	yr4_sg4c_djbfft_unroll 1,exec,noexec,noexec,srcreg,srcinc,d1,d2,dstreg,dstinc,e1,e2,screg,scinc,L1pt,L1pd
	bump	srcreg, srcinc
	bump	dstreg, dstinc
	bump	screg, scinc
	ENDIF
	ENDM

;; Port 5 will limits our best case to 20 clocks.  Alas, timed at 40 clocks.
yr4_sg4c_djbfft_unroll MACRO iter,prev,this,next,srcreg,srcinc,d1,d2,dstreg,dstinc,e1,e2,screg,scinc,L1pt,L1pd

;; On first iteration, init registers

	IF iter EQ -1
y0	TEXTEQU	<ymm0>
y1	TEXTEQU	<ymm1>
y2	TEXTEQU	<ymm2>
y3	TEXTEQU	<ymm3>
y4	TEXTEQU	<ymm4>
y5	TEXTEQU	<ymm5>
y6	TEXTEQU	<ymm6>
y7	TEXTEQU	<ymm7>
y8	TEXTEQU	<ymm8>
y9	TEXTEQU	<ymm9>
y10	TEXTEQU	<ymm10>
y11	TEXTEQU	<ymm11>
y12	TEXTEQU	<ymm12>
y13	TEXTEQU	<ymm13>
y14	TEXTEQU	<ymm14>
	ENDIF

;; On later iterations, previous R1,I1,R2,I2,I4 will be in y0-4, and this R1,I1,R2,I2,R3,I3,R4,I4 are in y5-12.
;; The remaining registers are free.  Register ymm15 is reserved for YMM_ONE or a sin/cos value.

next	vmovapd	y14, [srcreg+(iter+1)*srcinc]		;; R1
next	vmovapd	y13, [srcreg+(iter+1)*srcinc+d1]	;; R2
prev	ystore	[dstreg+(iter-1)*dstinc+e2+e1+32], y4	;; Save I4				;			42+2
next	vshufpd	y4, y14, y13, 15			;; Shuffle R1 and R2 to create R1/R2 hi	;		1
next	vshufpd	y14, y14, y13, 0			;; Shuffle R1 and R2 to create R1/R2 low ;		2
this	yfmaddpd y13, y6, ymm15, y10			;; I1 + I3 (newer I1)			; 21-25
this	yfmsubpd y6, y6, ymm15, y10			;; I1 - I3 (newer I3)			;	21-25

next	vmovapd	y10, [srcreg+(iter+1)*srcinc+d2]	;; R3
prev	ystore	[dstreg+(iter-1)*dstinc+e1], y2		;; Save R2				;			43+2
next	vmovapd	y2, [srcreg+(iter+1)*srcinc+d2+d1]	;; R4
prev	ystore	[dstreg+(iter-1)*dstinc+e1+32], y3	;; Save I2				;			43+3
next	vshufpd	y3, y10, y2, 15				;; Shuffle R3 and R4 to create R3/R4 hi	;		3
next	vshufpd	y10, y10, y2, 0				;; Shuffle R3 and R4 to create R3/R4 low ;		4
this	yfmsubpd y2, y9, ymm15, y12			;; R3 - I4 (final R3)			; 24-28
this	yfmaddpd y9, y9, ymm15, y12			;; R3 + I4 (final R4)			;	24-28

next	ylow128s y12, y4, y3				;; Shuffle R1/R2 hi and R3/R4 hi (new R2) ;		5-7
next	yhigh128s y4, y4, y3				;; Shuffle R1/R2 hi and R3/R4 hi (new R4) ;		6-8
this	yfmaddpd y3, y6, ymm15, y11			;; I3 + R4 (final I3)			; 26-30
this	yfmsubpd y6, y6, ymm15, y11			;; I3 - R4 (final I4)			;	26-30
this	L1prefetch srcreg+iter*srcinc+L1pd, L1pt
this	L1prefetchw dstreg+iter*dstinc+L1pd, L1pt - L1PREFETCH_DEST_NONE

next	ylow128s y11, y14, y10				;; Shuffle R1/R2 low and R3/R4 low (new R1) ;		7-9
next	yhigh128s y14, y14, y10				;; Shuffle R1/R2 low and R3/R4 low (new R3) ;		8-10
this	yfmsubpd y10, y13, ymm15, y8			;; I1 - I2 (final I2)			; 27-31
this	yfmaddpd y13, y13, ymm15, y8			;; I1 + I2 (final I1)			;	27-31
next	vmovapd	y8, [srcreg+(iter+1)*srcinc+32]		;; I1
prev	ystore	[dstreg+(iter-1)*dstinc], y0		;; Save R1				;			44+3

next	yfmaddpd y0, y12, ymm15, y4			;; R2 + R4 (newer R2)			; 9-13
next	yfmsubpd y12, y12, ymm15, y4			;; R2 - R4 (newer R4)			;	9-13
next	vmovapd	y4, [srcreg+(iter+1)*srcinc+d1+32]	;; I2
prev	ystore	[dstreg+(iter-1)*dstinc+32], y1		;; Save I1				;			44+4
	;; PORT 5 IDLE	(2 clock penalty switching from vpermf128 to vshufpd)			;		9
	;; PORT 5 IDLE	(2 clock penalty switching from vpermf128 to vshufpd)			;		10

next	vshufpd	y1, y8, y4, 15				;; Shuffle I1 and I2 to create I1/I2 hi	;		11
next	vshufpd	y8, y8, y4, 0				;; Shuffle I1 and I2 to create I1/I2 low ;		12
next	yfmaddpd y4, y11, ymm15, y14			;; R1 + R3 (newer R1)			; 11-15
next	yfmsubpd y11, y11, ymm15, y14			;; R1 - R3 (newer R3)			;	11-15

this	vmovapd	ymm15, [screg+iter*scinc+64+32]		;; cosine/sine for R3/I3
this	yfmsubpd y14, y2, ymm15, y3			;; A3 = R3 * cosine/sine - I3		; 31-35
this	yfmaddpd y3, y3, ymm15, y2			;; B3 = I3 * cosine/sine + R3		;	31-35

this	vmovapd	ymm15, [screg+iter*scinc+192+32]	;; cosine/sine for R4/I4
this	yfmsubpd y2, y9, ymm15, y6			;; A4 = R4 * cosine/sine - I4		; 32-36
this	yfmaddpd y6, y6, ymm15, y9			;; B4 = I4 * cosine/sine + R4		;	32-36
next	vmovapd	y9, [srcreg+(iter+1)*srcinc+d2+32]	;; I3

this	vmovapd	ymm15, [screg+iter*scinc+64]		;; sine for R3/I3
this	vmulpd	y14, y14, ymm15				;; A3 = A3 * sine (final R3)		; 36-40
this	vmulpd	y3, y3, ymm15				;; B3 = B3 * sine (final I3)		;	36-40

this	ystore	[dstreg+iter*dstinc+e2], y14		;; Save R3				;			41
next	vmovapd	y14, [srcreg+(iter+1)*srcinc+d2+d1+32]	;; I4
this	ystore	[dstreg+iter*dstinc+e2+32], y3		;; Save I3				;			41+1
next	vshufpd	y3, y9, y14, 15				;; Shuffle I3 and I4 to create I3/I4 hi	;		13
next	vshufpd	y9, y9, y14, 0				;; Shuffle I3 and I4 to create I3/I4 low ;		14
this	L1prefetch srcreg+iter*srcinc+d1+L1pd, L1pt
this	L1prefetchw dstreg+iter*dstinc+e1+L1pd, L1pt - L1PREFETCH_DEST_NONE

this	vmovapd	ymm15, [screg+iter*scinc+128+32]	;; cosine/sine for R2/I2
this	yfmsubpd y14, y7, ymm15, y10			;; A2 = R2 * cosine/sine - I2		; 33-37
this	yfmaddpd y10, y10, ymm15, y7			;; B2 = I2 * cosine/sine + R2		;	33-37

this	vmovapd	ymm15, [screg+iter*scinc+0+32]		;; cosine/sine for R1/I1
this	yfmsubpd y7, y5, ymm15, y13			;; A1 = R1 * cosine/sine - I1		; 34-38
this	yfmaddpd y13, y13, ymm15, y5			;; B1 = I1 * cosine/sine + R1		;	34-38

next	ylow128s y5, y1, y3				;; Shuffle I1/I2 hi and I3/I4 hi (new I2) ;		15-17
next	yhigh128s y1, y1, y3				;; Shuffle I1/I2 hi and I3/I4 hi (new I4) ;		16-18
this	L1prefetch srcreg+iter*srcinc+d2+L1pd, L1pt
this	L1prefetchw dstreg+iter*dstinc+e2+L1pd, L1pt - L1PREFETCH_DEST_NONE

next	ylow128s y3, y8, y9				;; Shuffle I1/I2 low and I3/I4 low (new I1) ;		17-19
next	yhigh128s y8, y8, y9				;; Shuffle I1/I2 low and I3/I4 low (new I3) ;		18-20
this	L1prefetch srcreg+iter*srcinc+d2+d1+L1pd, L1pt
this	L1prefetchw dstreg+iter*dstinc+e2+e1+L1pd, L1pt - L1PREFETCH_DEST_NONE

this	vmovapd	ymm15, [screg+iter*scinc+192]		;; sine for R4/I4
this	vmulpd	y2, y2, ymm15				;; A4 = A4 * sine (final R4)		; 37-41
this	vmulpd	y6, y6, ymm15				;; B4 = B4 * sine (final I4)		;	37-41

this	vmovapd ymm15, YMM_ONE
next	yfmsubpd y9, y4, ymm15, y0			;; R1 - R2 (final R2)			; 16-20
next	yfmaddpd y4, y4, ymm15, y0			;; R1 + R2 (final R1)			;	16-20

next	yfmsubpd y0, y5, ymm15, y1			;; I2 - I4 (newer I4)			; 19-23
next	yfmaddpd y5, y5, ymm15, y1			;; I2 + I4 (newer I2)			;	19-23

this	vmovapd	y1, [screg+iter*scinc+128]		;; sine for R2/I2
this	vmulpd	y14, y14, y1				;; A2 = A2 * sine (final R2)		; 38-42
this	vmulpd	y10, y10, y1				;; B2 = B2 * sine (final I2)		;	38-42

this	vmovapd	y1, [screg+iter*scinc+0]		;; sine for R1/I1
this	vmulpd	y7, y7, y1				;; A1 = A1 * sine (final R1)		; 39-43
this	vmulpd	y13, y13, y1				;; B1 = B1 * sine (final I1)		;	39-43
this	ystore	[dstreg+iter*dstinc+e2+e1], y2		;; Save R4				;			42+1
this next yloop_unrolled_one

	;; PORT 5 IDLE	(2 clock penalty switching from vpermf128 to vshufpd)			;		19
	;; PORT 5 IDLE	(2 clock penalty switching from vpermf128 to vshufpd)			;		20

;; Shuffle register assignments so that this R1,I1,R2,I2,I4 are in y0-4 and next R1,I1,R2,I2,R3,I3,R4,I4 in y5-12.

ytmp	TEXTEQU	y0
y0	TEXTEQU	y7
y7	TEXTEQU	y9
y9	TEXTEQU	y11
y11	TEXTEQU	y12
y12	TEXTEQU	ytmp
ytmp	TEXTEQU	y1
y1	TEXTEQU	y13
y13	TEXTEQU	y2
y2	TEXTEQU	y14
y14	TEXTEQU	ytmp
ytmp	TEXTEQU	y3
y3	TEXTEQU y10
y10	TEXTEQU y8
y8	TEXTEQU	y5
y5	TEXTEQU	y4
y4	TEXTEQU	y6
y6	TEXTEQU	ytmp
	ENDM

ENDIF

ENDIF


;;
;; ************************************* four-complex-unfft4 variants ******************************************
;;

;; Used in last levels of pass 1 (r4delay and r4dwpn cases).  Swizzling.
yr4_sg4cl_four_complex_unfft4_preload MACRO
	ENDM
yr4_sg4cl_four_complex_unfft4 MACRO srcreg,srcinc,d1,d2,dstreg,dstinc,e1,e2,screg,scinc,maxrpt,L1pt,L1pd
	vmovapd	ymm7, [screg+0+32]		;; cosine/sine
	vmovapd	ymm0, [srcreg]			;; R1
	vmulpd	ymm6, ymm0, ymm7		;; A1 = R1 * cosine/sine

	vmovapd	ymm5, [screg+128+32]		;; cosine/sine
	vmovapd	ymm4, [srcreg+d1]		;; R2
	vmulpd	ymm2, ymm4, ymm5		;; A2 = R2 * cosine/sine

	vmovapd	ymm1, [srcreg+32]		;; I1
	vaddpd	ymm6, ymm6, ymm1		;; A1 = A1 + I1
	vmulpd	ymm1, ymm1, ymm7		;; B1 = I1 * cosine/sine
	vmovapd	ymm3, [srcreg+d1+32]		;; I2
	vaddpd	ymm2, ymm2, ymm3		;; A2 = A2 + I2
	vmulpd	ymm3, ymm3, ymm5		;; B2 = I2 * cosine/sine
	vsubpd	ymm1, ymm1, ymm0		;; B1 = B1 - R1
	vsubpd	ymm3, ymm3, ymm4		;; B2 = B2 - R2

	vmulpd	ymm6, ymm6, [screg+0]		;; A1 = A1 * sine (new R1)
	vmulpd	ymm2, ymm2, [screg+128]		;; A2 = A2 * sine (new R2)

	vmovapd	ymm0, [screg+64+32]		;; cosine/sine
	vmovapd	ymm4, [srcreg+d2]		;; R3
	vmulpd	ymm5, ymm4, ymm0		;; A3 = R3 * cosine/sine

	vsubpd	ymm7, ymm6, ymm2		;; R1 - R2 (newer R2)
	vaddpd	ymm6, ymm6, ymm2		;; R1 + R2 (newer R1)

	ystore	[dstreg], ymm6			;; Save newer R1 temporarily
	ystore	[dstreg+32], ymm7		;; Save newer R2 temporarily

	vmovapd	ymm2, [srcreg+d2+32]		;; I3
	vaddpd	ymm5, ymm5, ymm2		;; A3 = A3 + I3
	vmulpd	ymm2, ymm2, ymm0		;; B3 = I3 * cosine/sine

	vmovapd	ymm0, [screg+192+32]		;; cosine/sine
	vmovapd	ymm6, [srcreg+d2+d1]		;; R4
	vmulpd	ymm7, ymm6, ymm0		;; A4 = R4 * cosine/sine
	vsubpd	ymm2, ymm2, ymm4		;; B3 = B3 - R3
	vmovapd ymm4, [srcreg+d2+d1+32]		;; I4
	vaddpd	ymm7, ymm7, ymm4		;; A4 = A4 + I4
	vmulpd	ymm4, ymm4, ymm0		;; B4 = I4 * cosine/sine
	vsubpd	ymm4, ymm4, ymm6		;; B4 = B4 - R4

	vmulpd	ymm1, ymm1, [screg+0]		;; B1 = B1 * sine (new I1)
	vmulpd	ymm3, ymm3, [screg+128]		;; B2 = B2 * sine (new I2)
	vmovapd	ymm6, [screg+64]		;; sine
	vmulpd	ymm5, ymm5, ymm6		;; A3 = A3 * sine (new R3)
	vmulpd	ymm2, ymm2, ymm6		;; B3 = B3 * sine (new I3)
	vmovapd	ymm6, [screg+192]		;; sine
	vmulpd	ymm7, ymm7, ymm6		;; A4 = A4 * sine (new R4)
	vmulpd	ymm4, ymm4, ymm6		;; B4 = B4 * sine (new I4)

	vsubpd	ymm0, ymm1, ymm3		;; I1 - I2 (newer I2)
	vaddpd	ymm1, ymm1, ymm3		;; I1 + I2 (newer I1)

	L1prefetch srcreg+L1pd, L1pt

	vsubpd	ymm3, ymm7, ymm5		;; R4 - R3 (newer I4)
	vaddpd	ymm7, ymm7, ymm5		;; R4 + R3 (newer R3)

	vsubpd	ymm5, ymm2, ymm4		;; I3 - I4 (newer R4)
	vaddpd	ymm2, ymm2, ymm4		;; I3 + I4 (newer I3)

	L1prefetch srcreg+d1+L1pd, L1pt

	vsubpd	ymm4, ymm0, ymm3		;; I2 - I4 (final I4)
	vaddpd	ymm0, ymm0, ymm3		;; I2 + I4 (final I2)

	vsubpd	ymm3, ymm1, ymm2		;; I1 - I3 (final I3)
	vaddpd	ymm1, ymm1, ymm2		;; I1 + I3 (final I1)

	L1prefetch srcreg+d2+L1pd, L1pt

	;; Swizzle using 4 shufpds and 4 perm2f128  (1 or 3 clocks each on port 5, throughput 1)

	vshufpd	ymm2, ymm1, ymm0, 0		;; Shuffle I1 and I2 to create I1/I2 low
	vshufpd	ymm1, ymm1, ymm0, 15		;; Shuffle I1 and I2 to create I1/I2 hi

	vshufpd	ymm0, ymm3, ymm4, 0		;; Shuffle I3 and I4 to create I3/I4 low
	vshufpd	ymm3, ymm3, ymm4, 15		;; Shuffle I3 and I4 to create I3/I4 hi

	ylow128s ymm4, ymm2, ymm0		;; Shuffle I1/I2 low and I3/I4 low (final I1)
	yhigh128s ymm2, ymm2, ymm0		;; Shuffle I1/I2 low and I3/I4 low (final I3)

	ylow128s ymm0, ymm1, ymm3		;; Shuffle I1/I2 hi and I3/I4 hi (final I2)
	yhigh128s ymm1, ymm1, ymm3		;; Shuffle I1/I2 hi and I3/I4 hi (final I4)

	vmovapd	ymm3, [dstreg]			;; Reload newer R1
	vmovapd	ymm6, [dstreg+32]		;; Reload newer R2

	ystore	[dstreg+32], ymm4		;; Save I1
	ystore	[dstreg+e2+32], ymm2		;; Save I3

	vsubpd	ymm4, ymm6, ymm5		;; R2 - R4 (final R4)
	vaddpd	ymm6, ymm6, ymm5		;; R2 + R4 (final R2)

	L1prefetch srcreg+d2+d1+L1pd, L1pt

	vsubpd	ymm5, ymm3, ymm7		;; R1 - R3 (final R3)
	vaddpd	ymm3, ymm3, ymm7		;; R1 + R3 (final R1)

	ystore	[dstreg+e1+32], ymm0		;; Save I2
	ystore	[dstreg+e2+e1+32], ymm1		;; Save I4

	;; Swizzle using 4 shufpds and 4 perm2f128  (1 or 3 clocks each on port 5, throughput 1)

	vshufpd	ymm0, ymm3, ymm6, 0		;; Shuffle R1 and R2 to create R1/R2 low
	vshufpd	ymm3, ymm3, ymm6, 15		;; Shuffle R1 and R2 to create R1/R2 hi

	vshufpd	ymm6, ymm5, ymm4, 0		;; Shuffle R3 and R4 to create R3/R4 low
	vshufpd	ymm5, ymm5, ymm4, 15		;; Shuffle R3 and R4 to create R3/R4 hi

	ylow128s ymm4, ymm0, ymm6		;; Shuffle R1/R2 low and R3/R4 low (final R1)
	yhigh128s ymm0, ymm0, ymm6		;; Shuffle R1/R2 low and R3/R4 low (final R3)

	ylow128s ymm6, ymm3, ymm5		;; Shuffle R1/R2 hi and R3/R4 hi (final R2)
	yhigh128s ymm3, ymm3, ymm5		;; Shuffle R1/R2 hi and R3/R4 hi (final R4)

	ystore	[dstreg], ymm4			;; Save R1
	ystore	[dstreg+e2], ymm0		;; Save R3
	ystore	[dstreg+e1], ymm6		;; Save R2
	ystore	[dstreg+e2+e1], ymm3		;; Save R4

	bump	srcreg, srcinc
	bump	dstreg, dstinc
	bump	screg, scinc
	ENDM

IFDEF X86_64

yr4_sg4cl_four_complex_unfft4_preload MACRO
	ENDM
yr4_sg4cl_four_complex_unfft4 MACRO srcreg,srcinc,d1,d2,dstreg,dstinc,e1,e2,screg,scinc,maxrpt,L1pt,L1pd
	vmovapd	ymm0, [screg+0+32]		;; cosine/sine
	vmovapd	ymm1, [srcreg+32]		;; I1
	vmulpd	ymm2, ymm1, ymm0		;; B1 = I1 * cosine/sine			; 1-5

	vmovapd	ymm3, [screg+128+32]		;; cosine/sine
	vmovapd	ymm4, [srcreg+d1+32]		;; I2
	vmulpd	ymm5, ymm4, ymm3		;; B2 = I2 * cosine/sine			; 2-6

	vmovapd	ymm6, [screg+64+32]		;; cosine/sine
	vmovapd	ymm7, [srcreg+d2]		;; R3
	vmulpd	ymm8, ymm7, ymm6		;; A3 = R3 * cosine/sine			; 3-7

	vmovapd	ymm9, [screg+192+32]		;; cosine/sine
	vmovapd	ymm10, [srcreg+d2+d1]		;; R4
	vmulpd	ymm11, ymm10, ymm9		;; A4 = R4 * cosine/sine			; 4-8

	vmovapd	ymm12, [srcreg+d2+32]		;; I3
	vmulpd	ymm6, ymm12, ymm6		;; B3 = I3 * cosine/sine			; 5-9

	vmovapd	ymm13, [srcreg]			;; R1
	vsubpd	ymm2, ymm2, ymm13		;; B1 = B1 - R1					; 6-8
	vmovapd ymm14, [srcreg+d2+d1+32]	;; I4
	vmulpd	ymm9, ymm14, ymm9		;; B4 = I4 * cosine/sine			; 6-10

	vmovapd	ymm15, [srcreg+d1]		;; R2
	vsubpd	ymm5, ymm5, ymm15		;; B2 = B2 - R2					; 7-9
	vmulpd	ymm0, ymm13, ymm0		;; A1 = R1 * cosine/sine			; 7-11
	vmovapd	ymm13, [screg+0]		;; sine

	vaddpd	ymm8, ymm8, ymm12		;; A3 = A3 + I3					; 8-10
	vmulpd	ymm3, ymm15, ymm3		;; A2 = R2 * cosine/sine			; 8-12
	vmovapd	ymm12, [screg+128]		;; sine

	vaddpd	ymm11, ymm11, ymm14		;; A4 = A4 + I4					; 9-11
	vmulpd	ymm2, ymm2, ymm13		;; B1 = B1 * sine (new I1)			; 9-13
	vmovapd	ymm14, [screg+64]		;; sine

	vsubpd	ymm6, ymm6, ymm7		;; B3 = B3 - R3					; 10-12
	vmulpd	ymm5, ymm5, ymm12		;; B2 = B2 * sine (new I2)			; 10-14
	vmovapd	ymm7, [screg+192]		;; sine

	vsubpd	ymm9, ymm9, ymm10		;; B4 = B4 - R4					; 11-13
	vmulpd	ymm8, ymm8, ymm14		;; A3 = A3 * sine (new R3)			; 11-15

	vaddpd	ymm0, ymm0, ymm1		;; A1 = A1 + I1					; 12-14
	vmulpd	ymm11, ymm11, ymm7		;; A4 = A4 * sine (new R4)			; 12-16

	vaddpd	ymm3, ymm3, ymm4		;; A2 = A2 + I2					; 13-15
	vmulpd	ymm6, ymm6, ymm14		;; B3 = B3 * sine (new I3)			; 13-17

	vmulpd	ymm9, ymm9, ymm7		;; B4 = B4 * sine (new I4)			; 14-18
	L1prefetch srcreg+L1pd, L1pt

	vsubpd	ymm7, ymm2, ymm5		;; I1 - I2 (newer I2)				; 15-17
	vmulpd	ymm0, ymm0, ymm13		;; A1 = A1 * sine (new R1)			; 15-19

	vaddpd	ymm2, ymm2, ymm5		;; I1 + I2 (newer I1)				; 16-18
	vmulpd	ymm3, ymm3, ymm12		;; A2 = A2 * sine (new R2)			; 16-20

	vsubpd	ymm12, ymm11, ymm8		;; R4 - R3 (newer I4)				; 17-19
	vaddpd	ymm11, ymm11, ymm8		;; R4 + R3 (newer R3)				; 18-20
	vaddpd	ymm8, ymm6, ymm9		;; I3 + I4 (newer I3)				; 19-21
	L1prefetch srcreg+d1+L1pd, L1pt

	vsubpd	ymm5, ymm7, ymm12		;; I2 - I4 (final I4)				; 20-22
	vaddpd	ymm7, ymm7, ymm12		;; I2 + I4 (final I2)				; 21-23
	vsubpd	ymm12, ymm2, ymm8		;; I1 - I3 (final I3)				; 22-24
	vaddpd	ymm2, ymm2, ymm8		;; I1 + I3 (final I1)				; 23-25
	L1prefetch srcreg+d2+L1pd, L1pt

	vaddpd	ymm8, ymm0, ymm3		;; R1 + R2 (newer R1)				; 24-26

	vsubpd	ymm6, ymm6, ymm9		;; I3 - I4 (newer R4)				; 25-27
	vshufpd	ymm9, ymm12, ymm5, 0		;; Shuffle I3 and I4 to create I3/I4 low	; 25

	vsubpd	ymm0, ymm0, ymm3		;; R1 - R2 (newer R2)				; 26-28
	vshufpd	ymm12, ymm12, ymm5, 15		;; Shuffle I3 and I4 to create I3/I4 hi		; 26

	vsubpd	ymm5, ymm8, ymm11		;; R1 - R3 (final R3)				; 27-29
	vshufpd	ymm3, ymm2, ymm7, 0		;; Shuffle I1 and I2 to create I1/I2 low	; 27
	L1prefetch srcreg+d2+d1+L1pd, L1pt

	vaddpd	ymm8, ymm8, ymm11		;; R1 + R3 (final R1)				; 28-30
	vshufpd	ymm2, ymm2, ymm7, 15		;; Shuffle I1 and I2 to create I1/I2 hi		; 28

	vsubpd	ymm7, ymm0, ymm6		;; R2 - R4 (final R4)				; 29-31
	ylow128s ymm11, ymm3, ymm9		;; Shuffle I1/I2 low and I3/I4 low (final I1)	; 29-30

	vaddpd	ymm0, ymm0, ymm6		;; R2 + R4 (final R2)				; 30-32
	yhigh128s ymm3, ymm3, ymm9		;; Shuffle I1/I2 low and I3/I4 low (final I3)	; 30-31

	ylow128s ymm9, ymm2, ymm12		;; Shuffle I1/I2 hi and I3/I4 hi (final I2)	; 31-32
	ystore	[dstreg+32], ymm11		;; Save I1					; 31

	yhigh128s ymm2, ymm2, ymm12		;; Shuffle I1/I2 hi and I3/I4 hi (final I4)	; 32-33
	ystore	[dstreg+e2+32], ymm3		;; Save I3					; 32

	ystore	[dstreg+e1+32], ymm9		;; Save I2					; 33

	vshufpd	ymm9, ymm5, ymm7, 0		;; Shuffle R3 and R4 to create R3/R4 low	; 34
	ystore	[dstreg+e2+e1+32], ymm2		;; Save I4					; 34

	vshufpd	ymm5, ymm5, ymm7, 15		;; Shuffle R3 and R4 to create R3/R4 hi		; 35

	vshufpd	ymm7, ymm8, ymm0, 0		;; Shuffle R1 and R2 to create R1/R2 low	; 36

	vshufpd	ymm8, ymm8, ymm0, 15		;; Shuffle R1 and R2 to create R1/R2 hi		; 37

	ylow128s ymm0, ymm7, ymm9		;; Shuffle R1/R2 low and R3/R4 low (final R1)	; 38-39

	yhigh128s ymm7, ymm7, ymm9		;; Shuffle R1/R2 low and R3/R4 low (final R3)	; 39-40

	ylow128s ymm9, ymm8, ymm5		;; Shuffle R1/R2 hi and R3/R4 hi (final R2)	; 40-41
	ystore	[dstreg], ymm0			;; Save R1					; 40

	yhigh128s ymm8, ymm8, ymm5		;; Shuffle R1/R2 hi and R3/R4 hi (final R4)	; 41-42
	ystore	[dstreg+e2], ymm7		;; Save R3					; 41

	ystore	[dstreg+e1], ymm9		;; Save R2					; 42
	ystore	[dstreg+e2+e1], ymm8		;; Save R4					; 43

	bump	srcreg, srcinc
	bump	dstreg, dstinc
	bump	screg, scinc
	ENDM

;; Haswell FMA3 version

IF (@INSTR(,%yarch,<FMA3>) NE 0)

yr4_sg4cl_four_complex_unfft4_preload MACRO
	vmovapd	ymm15, YMM_ONE
	ENDM

yr4_sg4cl_four_complex_unfft4 MACRO srcreg,srcinc,d1,d2,dstreg,dstinc,e1,e2,screg,scinc,maxrpt,L1pt,L1pd
	vmovapd	ymm3, [screg+0+32]		;; cosine/sine for R1/I1
	vmovapd	ymm2, [srcreg]			;; R1
	vmovapd	ymm1, [srcreg+32]		;; I1
	yfmaddpd ymm0, ymm2, ymm3, ymm1		;; A1 = R1 * cosine/sine + I1			; 1-5
	yfmsubpd ymm1, ymm1, ymm3, ymm2		;; B1 = I1 * cosine/sine - R1			; 1-5

	vmovapd	ymm5, [screg+64+32]		;; cosine/sine for R3/I3
	vmovapd	ymm4, [srcreg+d2]		;; R3
	vmovapd	ymm3, [srcreg+d2+32]		;; I3
	yfmaddpd ymm2, ymm4, ymm5, ymm3		;; A3 = R3 * cosine/sine + I3			; 2-6
	yfmsubpd ymm3, ymm3, ymm5, ymm4		;; B3 = I3 * cosine/sine - R3			; 2-6

	vmovapd	ymm7, [screg+128+32]		;; cosine/sine for R2/I2
	vmovapd	ymm6, [srcreg+d1]		;; R2
	vmovapd	ymm5, [srcreg+d1+32]		;; I2
	yfmaddpd ymm4, ymm6, ymm7, ymm5		;; A2 = R2 * cosine/sine + I2			; 3-7
	yfmsubpd ymm5, ymm5, ymm7, ymm6		;; B2 = I2 * cosine/sine - R2			; 3-7

	vmovapd	ymm9, [screg+192+32]		;; cosine/sine for R4/I4
	vmovapd	ymm8, [srcreg+d2+d1]		;; R4
	vmovapd ymm7, [srcreg+d2+d1+32]		;; I4
	yfmaddpd ymm6, ymm8, ymm9, ymm7		;; A4 = R4 * cosine/sine + I4			; 4-8
	yfmsubpd ymm7, ymm7, ymm9, ymm8		;; B4 = I4 * cosine/sine - R4			; 4-8

	vmovapd	ymm8, [screg+0]			;; sine for R1/I1
	vmulpd	ymm0, ymm0, ymm8		;; A1 = A1 * sine (new R1)			; 6-10
	vmulpd	ymm1, ymm1, ymm8		;; B1 = B1 * sine (new I1)			; 6-10

	vmovapd	ymm9, [screg+64]		;; sine for R3/I3
	vmulpd	ymm2, ymm2, ymm9		;; A3 = A3 * sine (new R3)			; 7-11
	vmulpd	ymm3, ymm3, ymm9		;; B3 = B3 * sine (new I3)			; 7-11

	vmovapd	ymm10, [screg+128]		;; sine for R2/I2
	yfnmaddpd ymm8, ymm5, ymm10, ymm1	;; I1 - (I2 = B2 * sine) (newer I2)		; 11-15
	yfmaddpd ymm5, ymm5, ymm10, ymm1	;; I1 + (I2 = B2 * sine) (newer I1)		; 11-15

	vmovapd	ymm11, [screg+192]		;; sine for R4/I4
	yfmsubpd ymm1, ymm6, ymm11, ymm2	;; (R4 = A4 * sine) - R3 (newer I4)		; 12-16
	yfmaddpd ymm6, ymm6, ymm11, ymm2	;; (R4 = A4 * sine) + R3 (newer R3)		; 12-16

	yfmaddpd ymm2, ymm7, ymm11, ymm3	;; I3 + (I4 = B4 * sine) (newer I3)		; 13-17
	yfnmaddpd ymm7, ymm7, ymm11, ymm3	;; I3 - (I4 = B4 * sine) (newer R4)		; 13-17
	L1prefetch srcreg+L1pd, L1pt

	yfmaddpd ymm3, ymm4, ymm10, ymm0	;; R1 + (R2 = A2 * sine) (newer R1)		; 14-18
	yfnmaddpd ymm4, ymm4, ymm10, ymm0	;; R1 - (R2 = A2 * sine) (newer R2)		; 14-18

	vsubpd	ymm0, ymm8, ymm1		;; I2 - I4 (final I4)				; 17-19
	yfmaddpd ymm8, ymm8, ymm15, ymm1	;; I2 + I4 (final I2)				; 17-21

	vsubpd	ymm1, ymm5, ymm2		;; I1 - I3 (final I3)				; 18-20
	yfmaddpd ymm5, ymm5, ymm15, ymm2	;; I1 + I3 (final I1)				; 18-22
	L1prefetch srcreg+d1+L1pd, L1pt

	vsubpd	ymm2, ymm3, ymm6		;; R1 - R3 (final R3)				; 19-21
	yfmaddpd ymm3, ymm3, ymm15, ymm6	;; R1 + R3 (final R1)				; 19-23

	vsubpd	ymm6, ymm4, ymm7		;; R2 - R4 (final R4)				; 20-22
	yfmaddpd ymm4, ymm4, ymm15, ymm7	;; R2 + R4 (final R2)				; 20-24

	vshufpd	ymm7, ymm1, ymm0, 0		;; Shuffle I3 and I4 to create I3/I4 low	; 21
	vshufpd	ymm1, ymm1, ymm0, 15		;; Shuffle I3 and I4 to create I3/I4 hi		; 22
	L1prefetch srcreg+d2+L1pd, L1pt

	ystorelo [dstreg+32][16], ymm7		;; Save I1					;	22
	ystorehi [dstreg+e2+32][16], ymm7	;; Save I3					;	23

	ystorelo [dstreg+e1+32][16], ymm1	;; Save I2					;	24
	ystorehi [dstreg+e2+e1+32][16], ymm1	;; Save I4					;	25

	vshufpd	ymm0, ymm5, ymm8, 0		;; Shuffle I1 and I2 to create I1/I2 low	; 23
	vshufpd	ymm5, ymm5, ymm8, 15		;; Shuffle I1 and I2 to create I1/I2 hi		; 24

	ystorelo [dstreg+32], ymm0		;; Save I1					;	26
	ystorehi [dstreg+e2+32], ymm0		;; Save I3					;	27

	ystorelo [dstreg+e1+32], ymm5		;; Save I2					;	28
	ystorehi [dstreg+e2+e1+32], ymm5	;; Save I4					;	29

	vshufpd	ymm8, ymm2, ymm6, 0		;; Shuffle R3 and R4 to create R3/R4 low	; 25
	vshufpd	ymm2, ymm2, ymm6, 15		;; Shuffle R3 and R4 to create R3/R4 hi		; 26

	vshufpd	ymm6, ymm3, ymm4, 0		;; Shuffle R1 and R2 to create R1/R2 low	; 27
	vshufpd	ymm3, ymm3, ymm4, 15		;; Shuffle R1 and R2 to create R1/R2 hi		; 28
	L1prefetch srcreg+d2+d1+L1pd, L1pt

	ylow128s ymm1, ymm6, ymm8		;; Shuffle R1/R2 low and R3/R4 low (final R1)	; 29-31
	yhigh128s ymm6, ymm6, ymm8		;; Shuffle R1/R2 low and R3/R4 low (final R3)	; 30-32

	ylow128s ymm8, ymm3, ymm2		;; Shuffle R1/R2 hi and R3/R4 hi (final R2)	; 31-33
	yhigh128s ymm3, ymm3, ymm2		;; Shuffle R1/R2 hi and R3/R4 hi (final R4)	; 32-34
	ystore	[dstreg], ymm1			;; Save R1					;	32
	ystore	[dstreg+e2], ymm6		;; Save R3					;	33

	ystore	[dstreg+e1], ymm8		;; Save R2					;	34
	ystore	[dstreg+e2+e1], ymm3		;; Save R4					;	35

	bump	srcreg, srcinc
	bump	dstreg, dstinc
	bump	screg, scinc
	ENDM

ENDIF

ENDIF


;;
;; ************************************* eight-reals-fft variants ******************************************
;;

;; These macros operate on eight reals doing 2 and 3/4 levels of the FFT and applying
;; the sin/cos multipliers afterwards.  The output is 2 reals (only 2 levels of FFT done)
;; and 3 complex numbers (3 levels of FFT performed).  These macros take a screg
;; that points to twiddles w^n, w^2n, and w^5n.

;; Standard eight-reals FFT macro.
yr4_4cl_eight_reals_fft_preload MACRO
	yr4_8r_fft_cmn_preload
	ENDM
yr4_4cl_eight_reals_fft MACRO srcreg,srcinc,d1,d2,screg,scinc,maxrpt,L1pt,L1pd
	yr4_8r_fft_cmn srcreg,srcinc,0,d1,d2,screg,screg+64,screg+128,screg,scinc,maxrpt,L1pt,L1pd
	ENDM

;; Like 4cl but uses one combined sin/cos data ptr.  The first two sin/cos entries can be used by
;; four complex macros at the same FFT level.  The third and fourth sin/cos entries are used by
;; this eight reals macro.
yr4_4cl_csc_eight_reals_fft_preload MACRO
	yr4_8r_fft_cmn_preload
	ENDM
yr4_4cl_csc_eight_reals_fft MACRO srcreg,srcinc,d1,d2,screg,scinc,maxrpt,L1pt,L1pd
	yr4_8r_fft_cmn srcreg,srcinc,0,d1,d2,screg+128,screg,screg+192,screg,scinc,maxrpt,L1pt,L1pd
	ENDM

;; Like 4cl but swizzles the outputs before storing the results.
;; This macro is used in the middle levels of the second pass of two pass AVX FFTs.
yr4_s4cl_eight_reals_fft_preload MACRO
	yr4_s8r_fft_cmn_preload
	ENDM
yr4_s4cl_eight_reals_fft MACRO srcreg,srcinc,d1,d2,screg,scinc,maxrpt,L1pt,L1pd
	yr4_s8r_fft_cmn srcreg,srcinc,0,d1,d2,screg,screg+64,screg+128,screg,scinc,,,maxrpt,L1pt,L1pd
	ENDM

;; Like s4cl but uses two sin/cos data ptrs.
yr4_s4cl_2sc_eight_reals_fft_preload MACRO
	yr4_s8r_fft_cmn_preload
	ENDM
yr4_s4cl_2sc_eight_reals_fft MACRO srcreg,srcinc,d1,d2,screg1,scinc1,screg2,scinc2,maxrpt,L1pt,L1pd
	yr4_s8r_fft_cmn srcreg,srcinc,0,d1,d2,screg2,screg1,screg2+64,screg1,scinc1,screg2,scinc2,maxrpt,L1pt,L1pd
	ENDM

;; Like s4cl but uses one combined sin/cos data ptr.  The first two sin/cos entries can be used by
;; four complex macros at the same FFT level.  The third and fourth sin/cos entries are used by
;; this eight reals macro.
yr4_s4cl_csc_eight_reals_fft_preload MACRO
	yr4_s8r_fft_cmn_preload
	ENDM
yr4_s4cl_csc_eight_reals_fft MACRO srcreg,srcinc,d1,d2,screg,scinc,maxrpt,L1pt,L1pd
	yr4_s8r_fft_cmn srcreg,srcinc,0,d1,d2,screg+128,screg,screg+192,screg,scinc,,,maxrpt,L1pt,L1pd
	ENDM

;; Like s4cl but offsets the input by rbx.
;; This macro is used in the first levels of one pass and two pass AVX FFTs.
yr4_fs4cl_eight_reals_fft_preload MACRO
	yr4_s8r_fft_cmn_preload
	ENDM
yr4_fs4cl_eight_reals_fft MACRO srcreg,srcinc,d1,d2,screg,scinc,maxrpt,L1pt,L1pd
	yr4_s8r_fft_cmn srcreg,srcinc,rbx,d1,d2,screg,screg+64,screg+128,screg,scinc,,,maxrpt,L1pt,L1pd
	ENDM

;; Like fs4cl but uses 2 sin/cos ptrs.
yr4_fs4cl_2sc_eight_reals_fft_preload MACRO
	yr4_s8r_fft_cmn_preload
	ENDM
yr4_fs4cl_2sc_eight_reals_fft MACRO srcreg,srcinc,d1,d2,screg1,scinc1,screg2,scinc2,maxrpt,L1pt,L1pd
	yr4_s8r_fft_cmn srcreg,srcinc,rbx,d1,d2,screg2,screg1,screg2+64,screg1,scinc1,screg2,scinc2,maxrpt,L1pt,L1pd
	ENDM

;; Common macro for eight-reals-FFT doing 2 and 3/4 levels.

yr4_8r_fft_cmn_preload MACRO
	ENDM
yr4_8r_fft_cmn MACRO srcreg,srcinc,srcoff,d1,d2,screg1,screg2,screg5,screg,scinc,maxrpt,L1pt,L1pd
	vmovapd	ymm0, [srcreg+srcoff+d2+d1]	;; R4
	vmovapd	ymm7, [srcreg+srcoff+d2+d1+32]	;; R8
	vsubpd	ymm3, ymm0, ymm7	;; new R8 = R4 - R8			; 1-3
	vaddpd	ymm7, ymm0, ymm7	;; new R4 = R4 + R8			; 2-4

	vmovapd	ymm4, [srcreg+srcoff+d1]	;; R2
	vmovapd	ymm5, [srcreg+srcoff+d1+32]	;; R6
	vsubpd	ymm1, ymm4, ymm5	;; new R6 = R2 - R6			; 3-5
	vaddpd	ymm5, ymm4, ymm5	;; new R2 = R2 + R6			; 4-6

	vbroadcastsd ymm2, Q YMM_SQRTHALF ;; sqrt(1/2)
	vmulpd	ymm3, ymm3, ymm2	;; R8 = R8 * square root		;  4-8
	vmulpd	ymm1, ymm1, ymm2	;; R6 = R6 * square root		;  6-10

	vmovapd	ymm2, [srcreg+srcoff]		;; R1
	vmovapd	ymm4, [srcreg+srcoff+32]	;; R5
	vsubpd	ymm0, ymm2, ymm4	;; new R5 = R1 - R5			; 5-7
	vaddpd	ymm4, ymm2, ymm4	;; new R1 = R1 + R5			; 6-8

	vaddpd	ymm2, ymm5, ymm7	;; R2 + R4 (final I1, a.k.a 2nd real result) ; 7-9
	vsubpd	ymm5, ymm5, ymm7	;; R2 - R4 (final I2)			; 8-10

	vmovapd	ymm6, [srcreg+srcoff+d2]	;; R3
	vmovapd	ymm7, [srcreg+srcoff+d2+32]	;; R7

	ystore	[srcreg+32], ymm2	;; Save final I1			; 10

	vaddpd	ymm2, ymm6, ymm7	;; new R3 = R3 + R7			; 9-11
	vsubpd	ymm6, ymm6, ymm7	;; new R7 = R3 - R7			; 10-12

	L1prefetchw srcreg+L1pd, L1pt

	vsubpd	ymm7, ymm1, ymm3	;; R6 = R6 - R8 (Real part)		; 11-13
	vaddpd	ymm3, ymm1, ymm3	;; R8 = R6 + R8 (Imaginary part)	; 12-14

	vaddpd	ymm1, ymm4, ymm2	;; R1 + R3 (final R1)			; 13-15
	vsubpd	ymm4, ymm4, ymm2	;; R1 - R3 (final R2)			; 14-16

	L1prefetchw srcreg+d1+L1pd, L1pt

	vsubpd	ymm2, ymm0, ymm7	;; R5 - R6 (final R4)			; 15-17
	vaddpd	ymm0, ymm0, ymm7	;; R5 + R6 (final R3)			; 16-18

	vsubpd	ymm7, ymm6, ymm3	;; R7 - R8 (final I4)			; 17-19
	vaddpd	ymm6, ymm6, ymm3	;; R7 + R8 (final I3)			; 18-20

	ystore	[srcreg], ymm1		;; Save final R1			; 16
	vmovapd	ymm1, [screg2+32]	;; cosine/sine for w^2n
	vmulpd	ymm3, ymm4, ymm1	;; A2 = R2 * cosine/sine for w^2n	;  17-21
	vsubpd	ymm3, ymm3, ymm5	;; A2 = A2 - I2				; 22-24
	vmulpd	ymm5, ymm5, ymm1	;; B2 = I2 * cosine/sine for w^2n	;  18-22
	vaddpd	ymm5, ymm5, ymm4	;; B2 = B2 + R2				; 23-25

	L1prefetchw srcreg+d2+L1pd, L1pt

	vmovapd	ymm1, [screg5+32]	;; cosine/sine for w^5n
	vmulpd	ymm4, ymm2, ymm1	;; A4 = R4 * cosine/sine for w^5n	;  19-23
	vsubpd	ymm4, ymm4, ymm7	;; A4 = A4 - I4				; 24-26
	vmulpd	ymm7, ymm7, ymm1	;; B4 = I4 * cosine/sine for w^5n	;  20-24
	vaddpd	ymm7, ymm7, ymm2	;; B4 = B4 + R4				; 25-27

	vmovapd	ymm1, [screg1+32]	;; cosine/sine for w^n
	vmulpd	ymm2, ymm0, ymm1	;; A3 = R3 * cosine/sine for w^n	;  21-25
	vsubpd	ymm2, ymm2, ymm6	;; A3 = A3 - I3				; 26-28
	vmulpd	ymm6, ymm6, ymm1	;; B3 = I3 * cosine/sine for w^n	;  22-26
	vaddpd	ymm6, ymm6, ymm0	;; B3 = B3 + R3				; 27-29

	L1prefetchw srcreg+d2+d1+L1pd, L1pt

	vmovapd	ymm0, [screg2]		;; sine for w^2n
	vmulpd	ymm3, ymm3, ymm0	;; A2 = A2 * sine (final R2)		;  25-29
	vmulpd	ymm5, ymm5, ymm0	;; B2 = B2 * sine (final I2)		;  26-30
	vmovapd	ymm0, [screg5]		;; sine for w^5n
	vmulpd	ymm4, ymm4, ymm0	;; A4 = A4 * sine (final R4)		;  27-31
	vmulpd	ymm7, ymm7, ymm0	;; B4 = B4 * sine (final I4)		;  28-32
	vmovapd	ymm0, [screg1]		;; sine for w^n
	vmulpd	ymm2, ymm2, ymm0	;; A3 = A3 * sine (final R3)		;  29-33
	vmulpd	ymm6, ymm6, ymm0	;; B3 = B3 * sine (final I3)		;  30-34

	ystore	[srcreg+d1], ymm3	;; Save R2
	ystore	[srcreg+d1+32], ymm5	;; Save I2
	ystore	[srcreg+d2], ymm2	;; Save R3
	ystore	[srcreg+d2+32], ymm6	;; Save I3
	ystore	[srcreg+d2+d1], ymm4	;; Save R4
	ystore	[srcreg+d2+d1+32], ymm7	;; Save I4

	bump	srcreg, srcinc
	bump	screg, scinc
	ENDM

yr4_s8r_fft_cmn_preload MACRO
	ENDM
yr4_s8r_fft_cmn MACRO srcreg,srcinc,srcoff,d1,d2,screg1,screg2,screg5,scregA,scincA,scregB,scincB,maxrpt,L1pt,L1pd
	vmovapd	ymm0, [srcreg+srcoff+d2+d1]	;; R4
	vmovapd	ymm7, [srcreg+srcoff+d2+d1+32]	;; R8
	vsubpd	ymm3, ymm0, ymm7	;; new R8 = R4 - R8			; 1-3
	vaddpd	ymm7, ymm0, ymm7	;; new R4 = R4 + R8			; 2-4

	vmovapd	ymm4, [srcreg+srcoff+d1]	;; R2
	vmovapd	ymm5, [srcreg+srcoff+d1+32]	;; R6
	vsubpd	ymm1, ymm4, ymm5	;; new R6 = R2 - R6			; 3-5
	vaddpd	ymm5, ymm4, ymm5	;; new R2 = R2 + R6			; 4-6

	vbroadcastsd ymm2, Q YMM_SQRTHALF ;; sqrt(1/2)
	vmulpd	ymm3, ymm3, ymm2	;; R8 = R8 * square root		;  4-8
	vmulpd	ymm1, ymm1, ymm2	;; R6 = R6 * square root		;  6-10

	vmovapd	ymm2, [srcreg+srcoff]		;; R1
	vmovapd	ymm4, [srcreg+srcoff+32]	;; R5
	vsubpd	ymm0, ymm2, ymm4	;; new R5 = R1 - R5			; 5-7
	vaddpd	ymm4, ymm2, ymm4	;; new R1 = R1 + R5			; 6-8

	vaddpd	ymm2, ymm5, ymm7	;; R2 + R4 (final I1, a.k.a 2nd real result) ; 7-9
	vsubpd	ymm5, ymm5, ymm7	;; R2 - R4 (final I2)			; 8-10

	vmovapd	ymm6, [srcreg+srcoff+d2]	;; R3
	vmovapd	ymm7, [srcreg+srcoff+d2+32]	;; R7

	ystore	[srcreg+32], ymm2	;; Save final I1			; 10

	vaddpd	ymm2, ymm6, ymm7	;; new R3 = R3 + R7			; 9-11
	vsubpd	ymm6, ymm6, ymm7	;; new R7 = R3 - R7			; 10-12

	L1prefetchw srcreg+L1pd, L1pt

	vsubpd	ymm7, ymm1, ymm3	;; R6 = R6 - R8 (Real part)		; 11-13
	vaddpd	ymm3, ymm1, ymm3	;; R8 = R6 + R8 (Imaginary part)	; 12-14

	vaddpd	ymm1, ymm4, ymm2	;; R1 + R3 (final R1)			; 13-15
	vsubpd	ymm4, ymm4, ymm2	;; R1 - R3 (final R2)			; 14-16

	L1prefetchw srcreg+d1+L1pd, L1pt

	vsubpd	ymm2, ymm0, ymm7	;; R5 - R6 (final R4)			; 15-17
	vaddpd	ymm0, ymm0, ymm7	;; R5 + R6 (final R3)			; 16-18

	vsubpd	ymm7, ymm6, ymm3	;; R7 - R8 (final I4)			; 17-19
	vaddpd	ymm6, ymm6, ymm3	;; R7 + R8 (final I3)			; 18-20

	ystore	[srcreg], ymm1		;; Save final R1			; 16
	vmovapd	ymm1, [screg2+32]	;; cosine/sine for w^2n
	vmulpd	ymm3, ymm4, ymm1	;; A2 = R2 * cosine/sine for w^2n	;  17-21
	vsubpd	ymm3, ymm3, ymm5	;; A2 = A2 - I2				; 22-24
	vmulpd	ymm5, ymm5, ymm1	;; B2 = I2 * cosine/sine for w^2n	;  18-22
	vaddpd	ymm5, ymm5, ymm4	;; B2 = B2 + R2				; 23-25

	L1prefetchw srcreg+d2+L1pd, L1pt

	vmovapd	ymm1, [screg5+32]	;; cosine/sine for w^5n
	vmulpd	ymm4, ymm2, ymm1	;; A4 = R4 * cosine/sine for w^5n	;  19-23
	vsubpd	ymm4, ymm4, ymm7	;; A4 = A4 - I4				; 24-26
	vmulpd	ymm7, ymm7, ymm1	;; B4 = I4 * cosine/sine for w^5n	;  20-24
	vaddpd	ymm7, ymm7, ymm2	;; B4 = B4 + R4				; 25-27

	vmovapd	ymm1, [screg1+32]	;; cosine/sine for w^n
	vmulpd	ymm2, ymm0, ymm1	;; A3 = R3 * cosine/sine for w^n	;  21-25
	vsubpd	ymm2, ymm2, ymm6	;; A3 = A3 - I3				; 26-28
	vmulpd	ymm6, ymm6, ymm1	;; B3 = I3 * cosine/sine for w^n	;  22-26
	vaddpd	ymm6, ymm6, ymm0	;; B3 = B3 + R3				; 27-29

	L1prefetchw srcreg+d2+d1+L1pd, L1pt

	vmovapd	ymm0, [screg2]		;; sine for w^2n
	vmulpd	ymm3, ymm3, ymm0	;; A2 = A2 * sine (final R2)		;  25-29
	vmulpd	ymm5, ymm5, ymm0	;; B2 = B2 * sine (final I2)		;  26-30
	vmovapd	ymm0, [screg5]		;; sine for w^5n
	vmulpd	ymm4, ymm4, ymm0	;; A4 = A4 * sine (final R4)		;  27-31
	vmulpd	ymm7, ymm7, ymm0	;; B4 = B4 * sine (final I4)		;  28-32
	vmovapd	ymm0, [screg1]		;; sine for w^n
	vmulpd	ymm2, ymm2, ymm0	;; A3 = A3 * sine (final R3)		;  29-33
	vmulpd	ymm6, ymm6, ymm0	;; B3 = B3 * sine (final I3)		;  30-34

	;; Swizzle using 8 shufpds and 8 perm2f128  (1 or 3 clocks each on port 5, throughput 1)

	vmovapd	ymm1, [srcreg]		;; Reload saved R1

	vshufpd	ymm0, ymm1, ymm3, 0	;; Shuffle R1 and R2 to create 0 8 2 10
	vshufpd	ymm1, ymm1, ymm3, 15	;; Shuffle R1 and R2 to create 1 9 3 11

	vshufpd	ymm3, ymm2, ymm4, 0	;; Shuffle R3 and R4 to create 16 24 18 26
	vshufpd	ymm2, ymm2, ymm4, 15	;; Shuffle R3 and R4 to create 17 25 19 27

	ylow128s ymm4, ymm0, ymm3	;; Shuffle R1/R2 low and R3/R4 low (0 8 16 24)
	yhigh128s ymm0, ymm0, ymm3	;; Shuffle R1/R2 low and R3/R4 low (2 10 18 26)

	ylow128s ymm3, ymm1, ymm2	;; Shuffle R1/R2 hi and R3/R4 hi (1 9 17 25)
	yhigh128s ymm1, ymm1, ymm2	;; Shuffle R1/R2 hi and R3/R4 hi (3 11 19 27)

	ystore	[srcreg], ymm4		;; Save (0 8 16 24)
	ystore	[srcreg+d2], ymm0	;; Save (2 10 18 26)
	ystore	[srcreg+d1], ymm3	;; Save (1 9 17 25)
	ystore	[srcreg+d2+d1], ymm1	;; Save (3 11 19 27)

	vmovapd	ymm4, [srcreg+32]	;; Reload saved I1

	vshufpd	ymm0, ymm4, ymm5, 0	;; Shuffle I1 and I2 to create 4 12 6 14
	vshufpd	ymm4, ymm4, ymm5, 15	;; Shuffle I1 and I2 to create 5 13 7 15

	vshufpd	ymm5, ymm6, ymm7, 0	;; Shuffle I3 and I4 to create 16 28 22 30
	vshufpd	ymm6, ymm6, ymm7, 15	;; Shuffle I3 and I4 to create 21 29 23 31

	ylow128s ymm7, ymm0, ymm5	;; Shuffle I1/I2 low and I3/I4 low (4 12 20 28)
	yhigh128s ymm0, ymm0, ymm5	;; Shuffle I1/I2 low and I3/I4 low (6 14 22 30)

	ylow128s ymm5, ymm4, ymm6	;; Shuffle I1/I2 hi and I3/I4 hi (5 13 21 29)
	yhigh128s ymm4, ymm4, ymm6	;; Shuffle I1/I2 hi and I3/I4 hi (7 15 23 31)

	ystore	[srcreg+32], ymm7	;; Save (4 12 20 28)
	ystore	[srcreg+d2+32], ymm0	;; Save (6 14 22 30)
	ystore	[srcreg+d1+32], ymm5	;; Save (5 13 21 29)
	ystore	[srcreg+d2+d1+32], ymm4	;; Save (7 15 23 31)

	bump	srcreg, srcinc
	bump	scregA, scincA
	bump	scregB, scincB
	ENDM

;; 64-bit version

IFDEF X86_64

yr4_8r_fft_cmn_preload MACRO
	vmovapd	ymm15, YMM_SQRTHALF
	ENDM

yr4_8r_fft_cmn MACRO srcreg,srcinc,srcoff,d1,d2,screg1,screg2,screg5,screg,scinc,maxrpt,L1pt,L1pd
	IF maxrpt MOD 5 EQ 0
	yr4_8r_fft_unroll -1,noexec,noexec,exec,srcreg,srcinc,srcoff,d1,d2,screg1,screg2,screg5,screg,scinc,L1pt,L1pd
	yr4_8r_fft_unroll 0,noexec,exec,exec,srcreg,srcinc,srcoff,d1,d2,screg1,screg2,screg5,screg,scinc,L1pt,L1pd
	yr4_8r_fft_unroll 1,exec,exec,exec,srcreg,srcinc,srcoff,d1,d2,screg1,screg2,screg5,screg,scinc,L1pt,L1pd
	yr4_8r_fft_unroll 2,exec,exec,exec,srcreg,srcinc,srcoff,d1,d2,screg1,screg2,screg5,screg,scinc,L1pt,L1pd
	yr4_8r_fft_unroll 3,exec,exec,exec,srcreg,srcinc,srcoff,d1,d2,screg1,screg2,screg5,screg,scinc,L1pt,L1pd
	yr4_8r_fft_unroll 4,exec,exec,noexec,srcreg,srcinc,srcoff,d1,d2,screg1,screg2,screg5,screg,scinc,L1pt,L1pd
	yr4_8r_fft_unroll 5,exec,noexec,noexec,srcreg,srcinc,srcoff,d1,d2,screg1,screg2,screg5,screg,scinc,L1pt,L1pd
	bump	srcreg, 5*srcinc
	bump	screg, 5*scinc
	ELSEIF maxrpt MOD 4 EQ 0
	yr4_8r_fft_unroll -1,noexec,noexec,exec,srcreg,srcinc,srcoff,d1,d2,screg1,screg2,screg5,screg,scinc,L1pt,L1pd
	yr4_8r_fft_unroll 0,noexec,exec,exec,srcreg,srcinc,srcoff,d1,d2,screg1,screg2,screg5,screg,scinc,L1pt,L1pd
	yr4_8r_fft_unroll 1,exec,exec,exec,srcreg,srcinc,srcoff,d1,d2,screg1,screg2,screg5,screg,scinc,L1pt,L1pd
	yr4_8r_fft_unroll 2,exec,exec,exec,srcreg,srcinc,srcoff,d1,d2,screg1,screg2,screg5,screg,scinc,L1pt,L1pd
	yr4_8r_fft_unroll 3,exec,exec,noexec,srcreg,srcinc,srcoff,d1,d2,screg1,screg2,screg5,screg,scinc,L1pt,L1pd
	yr4_8r_fft_unroll 4,exec,noexec,noexec,srcreg,srcinc,srcoff,d1,d2,screg1,screg2,screg5,screg,scinc,L1pt,L1pd
	bump	srcreg, 4*srcinc
	bump	screg, 4*scinc
	ELSEIF maxrpt MOD 3 EQ 0
	yr4_8r_fft_unroll -1,noexec,noexec,exec,srcreg,srcinc,srcoff,d1,d2,screg1,screg2,screg5,screg,scinc,L1pt,L1pd
	yr4_8r_fft_unroll 0,noexec,exec,exec,srcreg,srcinc,srcoff,d1,d2,screg1,screg2,screg5,screg,scinc,L1pt,L1pd
	yr4_8r_fft_unroll 1,exec,exec,exec,srcreg,srcinc,srcoff,d1,d2,screg1,screg2,screg5,screg,scinc,L1pt,L1pd
	yr4_8r_fft_unroll 2,exec,exec,noexec,srcreg,srcinc,srcoff,d1,d2,screg1,screg2,screg5,screg,scinc,L1pt,L1pd
	yr4_8r_fft_unroll 3,exec,noexec,noexec,srcreg,srcinc,srcoff,d1,d2,screg1,screg2,screg5,screg,scinc,L1pt,L1pd
	bump	srcreg, 3*srcinc
	bump	screg, 3*scinc
	ELSEIF maxrpt MOD 2 EQ 0
	yr4_8r_fft_unroll -1,noexec,noexec,exec,srcreg,srcinc,srcoff,d1,d2,screg1,screg2,screg5,screg,scinc,L1pt,L1pd
	yr4_8r_fft_unroll 0,noexec,exec,exec,srcreg,srcinc,srcoff,d1,d2,screg1,screg2,screg5,screg,scinc,L1pt,L1pd
	yr4_8r_fft_unroll 1,exec,exec,noexec,srcreg,srcinc,srcoff,d1,d2,screg1,screg2,screg5,screg,scinc,L1pt,L1pd
	yr4_8r_fft_unroll 2,exec,noexec,noexec,srcreg,srcinc,srcoff,d1,d2,screg1,screg2,screg5,screg,scinc,L1pt,L1pd
	bump	srcreg, 2*srcinc
	bump	screg, 2*scinc
	ELSE
	yr4_8r_fft_unroll -1,noexec,noexec,exec,srcreg,srcinc,srcoff,d1,d2,screg1,screg2,screg5,screg,scinc,L1pt,L1pd
	yr4_8r_fft_unroll 0,noexec,exec,noexec,srcreg,srcinc,srcoff,d1,d2,screg1,screg2,screg5,screg,scinc,L1pt,L1pd
	yr4_8r_fft_unroll 1,exec,noexec,noexec,srcreg,srcinc,srcoff,d1,d2,screg1,screg2,screg5,screg,scinc,L1pt,L1pd
	bump	srcreg, srcinc
	bump	screg, scinc
	ENDIF
	ENDM

yr4_8r_fft_unroll MACRO iter,prev,this,next,srcreg,srcinc,srcoff,d1,d2,screg1,screg2,screg5,screg,scinc,L1pt,L1pd

;; On first call, init registers

	IF iter EQ -1
y0	TEXTEQU	<ymm0>
y1	TEXTEQU	<ymm1>
y2	TEXTEQU	<ymm2>
y3	TEXTEQU	<ymm3>
y4	TEXTEQU	<ymm4>
y5	TEXTEQU	<ymm5>
y6	TEXTEQU	<ymm6>
y7	TEXTEQU	<ymm7>
y8	TEXTEQU	<ymm8>
y9	TEXTEQU	<ymm9>
y10	TEXTEQU	<ymm10>
y11	TEXTEQU	<ymm11>
y12	TEXTEQU	<ymm12>
y13	TEXTEQU	<ymm13>
y14	TEXTEQU	<ymm14>
	ENDIF

;; On later calls, previous R2,I2,R4,B4,A3,B3,sine5,sine1 will be in y0-7.  This R2,R4,R6,R8,R1 will be in y8-12.
;; The remaining registers are free,  ymm15 preloaded with YMM_SQRTHALF.

this	vsubpd	y14, y9, y11	;; new R8 = R4 - R8			; 1-3
prev	vmulpd	y3, y3, y6	;; B4 = B4 * sine (final I4)		;  1-5
this	vmovapd	y6, [srcreg+iter*srcinc+srcoff+32] ;; R5

this	vsubpd	y13, y8, y10	;; new R6 = R2 - R6			; 2-4
prev	vmulpd	y4, y4, y7	;; A3 = A3 * sine (final R3)		;  2-6
prev	ystore	[srcreg+(iter-1)*srcinc+d1], y0	;; Save R2		; 3
this	vmovapd	y0, [srcreg+iter*srcinc+srcoff+d2] ;; R3

this	vaddpd	y9, y9, y11	;; new R4 = R4 + R8			; 3-5
prev	vmulpd	y5, y5, y7	;; B3 = B3 * sine (final I3)		;  3-7
this	vmovapd	y7, [srcreg+iter*srcinc+srcoff+d2+32] ;; R7

this	vaddpd	y8, y8, y10	;; new R2 = R2 + R6			; 4-6
this	vmulpd	y14, y14, ymm15	;; R8 = R8 * square root 1/2		;  4-8
this	vmovapd	y10, [screg2+iter*scinc+32] ;; cosine/sine for w^2n

this	vaddpd	y11, y12, y6	;; new R1 = R1 + R5			; 5-7
this	vmulpd	y13, y13, ymm15	;; R6 = R6 * square root 1/2		;  5-9
prev	ystore	[srcreg+(iter-1)*srcinc+d1+32], y1 ;; Save I2		; 4

this	vaddpd	y1, y0, y7	;; new R3 = R3 + R7			; 6-8
prev	ystore	[srcreg+(iter-1)*srcinc+d2+d1], y2 ;; Save R4		; 5
this	vmovapd	y2, [screg5+iter*scinc+32] ;; cosine/sine for w^5n

this	vsubpd	y12, y12, y6	;; new R5 = R1 - R5			; 7-9
prev	ystore	[srcreg+(iter-1)*srcinc+d2+d1+32], y3 ;; Save I4	; 6

this	vsubpd	y0, y0, y7	;; new R7 = R3 - R7			; 8-10
this	L1prefetchw srcreg+iter*srcinc+L1pd, L1pt
prev	ystore	[srcreg+(iter-1)*srcinc+d2], y4	;; Save R3		; 7

this	vsubpd	y4, y11, y1	;; R1 - R3 (final R2)			; 9-11
prev	ystore	[srcreg+(iter-1)*srcinc+d2+32], y5 ;; Save I3		; 8

this	vsubpd	y5, y13, y14	;; R6 = R6 - R8 (Real part)		; 10-12

this	vaddpd	y13, y13, y14	;; R8 = R6 + R8 (Imaginary part)	; 11-13
this	L1prefetchw srcreg+iter*srcinc+d1+L1pd, L1pt

this	vsubpd	y14, y8, y9	;; R2 - R4 (final I2)			; 12-14

this	vsubpd	y7, y12, y5	;; R5 - R6 (final R4)			; 13-15
this next yloop_unrolled_one

this	vsubpd	y3, y0, y13	;; R7 - R8 (final I4)			; 14-16
this	vmulpd	y6, y4, y10	;; A2 = R2 * cosine/sine for w^2n	;  14-18
this	L1prefetchw srcreg+iter*srcinc+d2+L1pd, L1pt

this	vaddpd	y12, y12, y5	;; R5 + R6 (final R3)			; 15-17
this	vmulpd	y10, y14, y10	;; B2 = I2 * cosine/sine for w^2n	;  15-19
this	vmovapd	y5, [screg1+iter*scinc+32] ;; cosine/sine for w^n

this	vaddpd	y0, y0, y13	;; R7 + R8 (final I3)			; 16-18
this	vmulpd	y13, y7, y2	;; A4 = R4 * cosine/sine for w^5n	;  16-20
this	L1prefetchw srcreg+iter*srcinc+d2+d1+L1pd, L1pt

this	vaddpd	y11, y11, y1	;; R1 + R3 (final R1)			; 17-19
this	vmulpd	y2, y3, y2	;; B4 = I4 * cosine/sine for w^5n	;  17-21
this	vmovapd	y1, [screg2+iter*scinc] ;; sine for w^2n

this	vaddpd	y8, y8, y9	;; R2 + R4 (final I1, a.k.a 2nd real result) ; 18-20
this	vmulpd	y9, y12, y5	;; A3 = R3 * cosine/sine for w^n	;  18-22
this	ystore	[srcreg+iter*srcinc], y11 ;; Save R1			; 20
this	vmovapd	y11, [screg5+iter*scinc] ;; sine for w^5n

this	vsubpd	y6, y6, y14	;; A2 = A2 - I2				; 19-21
this	vmulpd	y5, y0, y5	;; B3 = I3 * cosine/sine for w^n	;  19-23
next	vmovapd	y14, [srcreg+(iter+1)*srcinc+srcoff+d2+d1] ;; R4

this	vaddpd	y10, y10, y4	;; B2 = B2 + R2				; 20-22
next	vmovapd	y4, [srcreg+(iter+1)*srcinc+srcoff+d2+d1+32] ;; R8

this	vsubpd	y13, y13, y3	;; A4 = A4 - I4				; 21-23
next	vmovapd	y3, [srcreg+(iter+1)*srcinc+srcoff+d1] ;; R2
this	ystore	[srcreg+iter*srcinc+32], y8 ;; Save I1			; 21

this	vaddpd	y2, y2, y7	;; B4 = B4 + R4				; 22-24
this	vmulpd	y6, y6, y1	;; A2 = A2 * sine (final R2)		;  22-26
next	vmovapd	y7, [srcreg+(iter+1)*srcinc+srcoff+d1+32] ;; R6

this	vsubpd	y9, y9, y0	;; A3 = A3 - I3				; 23-25
this	vmulpd	y10, y10, y1	;; B2 = B2 * sine (final I2)		;  23-27
this	vmovapd	y1, [screg1+iter*scinc] ;; sine for w^n

this	vaddpd	y5, y5, y12	;; B3 = B3 + R3				; 24-26
this	vmulpd	y13, y13, y11	;; A4 = A4 * sine (final R4)		;  24-28
next	vmovapd	y12, [srcreg+(iter+1)*srcinc+srcoff] ;; R1

;; Shuffle register assignments so that next call has R2,I2,R4,B4,A3,B3,sine5,sine1 in y0-7 and next R2,R4,R6,R8,R1 in y8-12.

ytmp	TEXTEQU	y0
y0	TEXTEQU	y6
y6	TEXTEQU	y11
y11	TEXTEQU	y4
y4	TEXTEQU	y9
y9	TEXTEQU	y14
y14	TEXTEQU	y8
y8	TEXTEQU	y3
y3	TEXTEQU	y2
y2	TEXTEQU	y13
y13	TEXTEQU	ytmp
ytmp	TEXTEQU y1
y1	TEXTEQU	y10
y10	TEXTEQU	y7
y7	TEXTEQU ytmp
	ENDM


;; Haswell FMA3 version

IF (@INSTR(,%yarch,<FMA3>) NE 0)

yr4_8r_fft_cmn_preload MACRO
	vmovapd ymm15, YMM_ONE
	ENDM

;; uops = 6 s/c loads, 8 loads, 8 stores, 1 constant load, 6 muls, 24 fma, 12 mov = 65 uops / 4 = 16.25 clocks
;; Timed at 17.1 clocks
yr4_8r_fft_unroll MACRO iter,prev,this,next,srcreg,srcinc,srcoff,d1,d2,screg1,screg2,screg5,screg,scinc,L1pt,L1pd

;; On first call, init registers

	IF iter EQ -1
y0	TEXTEQU	<ymm0>
y1	TEXTEQU	<ymm1>
y2	TEXTEQU	<ymm2>
y3	TEXTEQU	<ymm3>
y4	TEXTEQU	<ymm4>
y5	TEXTEQU	<ymm5>
y6	TEXTEQU	<ymm6>
y7	TEXTEQU	<ymm7>
y8	TEXTEQU	<ymm8>
y9	TEXTEQU	<ymm9>
y10	TEXTEQU	<ymm10>
y11	TEXTEQU	<ymm11>
y12	TEXTEQU	<ymm12>
y13	TEXTEQU	<ymm13>
y14	TEXTEQU	<ymm14>
	ENDIF

;; On later calls, previous R2,I2,R3,I3,R4,I4 will be in y0-5.  This R1,R2,R3,R4,R5,R6,R7,R8 will be in y6-13.
;; The remaining registers are free,  Register ymm15 preloaded with YMM_ONE.

this	vmovapd	y14, YMM_SQRTHALF		;; Load constant
prev	ystore	[srcreg+(iter-1)*srcinc+d2+d1], y4 ;; Save R4				; 18
this	yfnmaddpd y4, y11, y14, y10		;; R5 - R6 * SQRTHALF (newer R4)	; 1-5		(used 7)
this	yfmaddpd y11, y11, y14, y10		;; R5 + R6 * SQRTHALF (newer R3)	; 1-5		(used 8)
next	vmovapd	y10, [srcreg+(iter+1)*srcinc+srcoff+d2+d1] ;; R4

prev	ystore	[srcreg+(iter-1)*srcinc+d2+d1+32], y5 ;; Save I4			; 18
this	yfnmaddpd y5, y13, y14, y12		;; R7 - R8 * SQRTHALF (newer I4)	; 2-6		(used 7)
this	yfmaddpd y13, y13, y14, y12		;; R7 + R8 * SQRTHALF (newer I3)	; 2-6		(used 8)
next	vmovapd	y12, [srcreg+(iter+1)*srcinc+srcoff+d2+d1+32] ;; R8

this	yfmsubpd y14, y7, ymm15, y9		;; R2 - R4 (newer I2)			; 3-7		(used 9)
this	yfmaddpd y7, y7, ymm15, y9		;; R2 + R4 (newer & final I1, a.k.a 2nd real result) ; 3-7
next	vmovapd	y9, [srcreg+(iter+1)*srcinc+srcoff+d1] ;; R2
prev	ystore	[srcreg+(iter-1)*srcinc+d2], y2 ;; Save R3				; 19

this	yfmsubpd y2, y6, ymm15, y8		;; R1 - R3 (newer R2)			; 4-8		(used 9)
this	yfmaddpd y6, y6, ymm15, y8		;; R1 + R3 (newer & final R1)		; 4-8
next	vmovapd	y8, [srcreg+(iter+1)*srcinc+srcoff+d1+32] ;; R6
prev	ystore	[srcreg+(iter-1)*srcinc+d2+32], y3 ;; Save I3				; 19

next	yfmsubpd y3, y10, ymm15, y12		;; new R8 = R4 - R8			;	5-9	(used 11)
next	yfmaddpd y10, y10, ymm15, y12		;; new R4 = R4 + R8			;	5-9	(used next 3)
this	vmovapd	y12, [screg5+iter*scinc+32]	;; cosine/sine for w^5n
prev	ystore	[srcreg+(iter-1)*srcinc+d1], y0 ;; Save R2				; 20

next	yfmsubpd y0, y9, ymm15, y8		;; new R6 = R2 - R6			;	6-10	(used 11)
next	yfmaddpd y9, y9, ymm15, y8		;; new R2 = R2 + R6			;	6-10	(used next 3)
this	vmovapd	y8, [screg1+iter*scinc+32]	;; cosine/sine for w^n
prev	ystore	[srcreg+(iter-1)*srcinc+d1+32], y1 ;; Save I2				; 20

this	yfmsubpd y1, y4, y12, y5		;; A4 = R4 * cosine/sine for w^5n - I4	; 7-11		(used 13)
this	yfmaddpd y5, y5, y12, y4		;; B4 = I4 * cosine/sine for w^5n + R4	; 7-11
this	vmovapd	y12, [screg2+iter*scinc+32]	;; cosine/sine for w^2n
this	L1prefetchw srcreg+iter*srcinc+L1pd, L1pt

this	yfmsubpd y4, y11, y8, y13		;; A3 = R3 * cosine/sine for w^n - I3	; 8-12		(used 14)
this	yfmaddpd y11, y13, y8, y11		;; B3 = I3 * cosine/sine for w^n + R3	; 8-12
next	vmovapd	y8, [srcreg+(iter+1)*srcinc+srcoff] ;; R1

this	yfmsubpd y13, y2, y12, y14		;; A2 = R2 * cosine/sine for w^2n - I2	; 9-13		(used 15)
this	yfmaddpd y2, y14, y12, y2		;; B2 = I2 * cosine/sine for w^2n + R2	; 9-13
next	vmovapd	y12, [srcreg+(iter+1)*srcinc+srcoff+32] ;; R5
this	L1prefetchw srcreg+iter*srcinc+d1+L1pd, L1pt

next	yfmsubpd y14, y8, ymm15, y12		;; new R5 = R1 - R5			;	10-14	(used next 1)
next	yfmaddpd y8, y8, ymm15, y12		;; new R1 = R1 + R5			;	10-14	(used next 4)
next	vmovapd	y12, [srcreg+(iter+1)*srcinc+srcoff+d2] ;; R3
this	ystore	[srcreg+iter*srcinc+32], y7	;; Save I1				; 8

next	yfmsubpd y7, y0, ymm15, y3		;; R6 = R6 - R8 (Real part)		;	11-15	(used next 1)
next	yfmaddpd y0, y0, ymm15, y3		;; R8 = R6 + R8 (Imaginary part)	;	11-15	(used next 2)
next	vmovapd	y3, [srcreg+(iter+1)*srcinc+srcoff+d2+32] ;; R7
this	ystore	[srcreg+iter*srcinc], y6	;; Save R1				; 9

next	yfmsubpd y6, y12, ymm15, y3		;; new R7 = R3 - R7			;	12-16	(used next 2)
next	yfmaddpd y12, y12, ymm15, y3		;; new R3 = R3 + R7			;	12-16	(used next 4)
this	L1prefetchw srcreg+iter*srcinc+d2+L1pd, L1pt

this	vmovapd	y3, [screg5+iter*scinc]		;; sine for w^5n
this	vmulpd	y1, y1, y3			;; A4 = A4 * sine for w^5n (final R4)	; 13-17
this	vmulpd	y5, y5, y3			;; B4 = B4 * sine for w^5n (final I4)	; 13-17
this	L1prefetchw srcreg+iter*srcinc+d2+d1+L1pd, L1pt

this	vmovapd	y3, [screg1+iter*scinc]		;; sine for w^n
this	vmulpd	y4, y4, y3			;; A3 = A3 * sine for w^n (final R3)	; 14-18
this	vmulpd	y11, y11, y3			;; B3 = B3 * sine for w^n (final I3)	; 14-18
this next yloop_unrolled_one

this	vmovapd	y3, [screg2+iter*scinc]		;; sine for w^2n
this	vmulpd	y13, y13, y3			;; A2 = A2 * sine for w^2n (final R2)	; 15-19
this	vmulpd	y2, y2, y3			;; B2 = B2 * sine for w^2n (final I2)	; 15-19

;; Shuffle register assignments so that next call has R2,I2,R3,I3,R4,I4 in y0-5 and next R1,R2,R3,R4,R5,R6,R7,R8 in y6-13.

ytmp	TEXTEQU	y0
y0	TEXTEQU	y13
y13	TEXTEQU	ytmp
ytmp	TEXTEQU	y1
y1	TEXTEQU	y2
y2	TEXTEQU	y4
y4	TEXTEQU	ytmp
ytmp	TEXTEQU	y3
y3	TEXTEQU	y11
y11	TEXTEQU	y7
y7	TEXTEQU	y9
y9	TEXTEQU	y10
y10	TEXTEQU	y14
y14	TEXTEQU	ytmp
ytmp	TEXTEQU	y6
y6	TEXTEQU y8
y8	TEXTEQU	y12
y12	TEXTEQU	ytmp
	ENDM

ENDIF

;; 64-bit swizzling version

yr4_s8r_fft_cmn_preload MACRO
	vmovapd ymm14, YMM_SQRTHALF
	ENDM

yr4_s8r_fft_cmn MACRO srcreg,srcinc,srcoff,d1,d2,screg1,screg2,screg5,scregA,scincA,scregB,scincB,maxrpt,L1pt,L1pd
	vmovapd	ymm0, [srcreg+srcoff+d2+d1]	;; R4
	vmovapd	ymm7, [srcreg+srcoff+d2+d1+32]	;; R8
	vsubpd	ymm3, ymm0, ymm7		;; new R8 = R4 - R8			; 1-3
	vaddpd	ymm7, ymm0, ymm7		;; new R4 = R4 + R8			; 2-4

	vmovapd	ymm4, [srcreg+srcoff+d1]	;; R2
	vmovapd	ymm5, [srcreg+srcoff+d1+32]	;; R6
	vsubpd	ymm1, ymm4, ymm5		;; new R6 = R2 - R6			; 3-5
	vaddpd	ymm5, ymm4, ymm5		;; new R2 = R2 + R6			; 4-6

	vmulpd	ymm3, ymm3, ymm14		;; R8 = R8 * square root		;  4-8
	vmulpd	ymm1, ymm1, ymm14		;; R6 = R6 * square root		;  6-10

	vmovapd	ymm2, [srcreg+srcoff]		;; R1
	vmovapd	ymm4, [srcreg+srcoff+32]	;; R5
	vsubpd	ymm0, ymm2, ymm4		;; new R5 = R1 - R5			; 5-7
	vaddpd	ymm4, ymm2, ymm4		;; new R1 = R1 + R5			; 6-8

	vaddpd	ymm9, ymm5, ymm7		;; R2 + R4 (final I1, a.k.a 2nd real result) ; 7-9
	vsubpd	ymm5, ymm5, ymm7		;; R2 - R4 (final I2)			; 8-10

	vmovapd	ymm6, [srcreg+srcoff+d2]	;; R3
	vmovapd	ymm7, [srcreg+srcoff+d2+32]	;; R7

	vaddpd	ymm2, ymm6, ymm7		;; new R3 = R3 + R7			; 9-11
	vsubpd	ymm6, ymm6, ymm7		;; new R7 = R3 - R7			; 10-12

	L1prefetchw srcreg+L1pd, L1pt

	vsubpd	ymm7, ymm1, ymm3		;; R6 = R6 - R8 (Real part)		; 11-13
	vaddpd	ymm3, ymm1, ymm3		;; R8 = R6 + R8 (Imaginary part)	; 12-14

	vaddpd	ymm10, ymm4, ymm2		;; R1 + R3 (final R1)			; 13-15
	vsubpd	ymm4, ymm4, ymm2		;; R1 - R3 (final R2)			; 14-16

	L1prefetchw srcreg+d1+L1pd, L1pt

	vsubpd	ymm2, ymm0, ymm7		;; R5 - R6 (final R4)			; 15-17
	vaddpd	ymm0, ymm0, ymm7		;; R5 + R6 (final R3)			; 16-18

	vsubpd	ymm7, ymm6, ymm3		;; R7 - R8 (final I4)			; 17-19
	vaddpd	ymm6, ymm6, ymm3		;; R7 + R8 (final I3)			; 18-20

	vmovapd	ymm1, [screg2+32]		;; cosine/sine for w^2n
	vmulpd	ymm3, ymm4, ymm1		;; A2 = R2 * cosine/sine for w^2n	;  17-21
	vsubpd	ymm3, ymm3, ymm5		;; A2 = A2 - I2				; 22-24
	vmulpd	ymm5, ymm5, ymm1		;; B2 = I2 * cosine/sine for w^2n	;  18-22
	vaddpd	ymm5, ymm5, ymm4		;; B2 = B2 + R2				; 23-25

	L1prefetchw srcreg+d2+L1pd, L1pt

	vmovapd	ymm1, [screg5+32]		;; cosine/sine for w^5n
	vmulpd	ymm4, ymm2, ymm1		;; A4 = R4 * cosine/sine for w^5n	;  19-23
	vsubpd	ymm4, ymm4, ymm7		;; A4 = A4 - I4				; 24-26
	vmulpd	ymm7, ymm7, ymm1		;; B4 = I4 * cosine/sine for w^5n	;  20-24
	vaddpd	ymm7, ymm7, ymm2		;; B4 = B4 + R4				; 25-27

	vmovapd	ymm1, [screg1+32]		;; cosine/sine for w^n
	vmulpd	ymm2, ymm0, ymm1		;; A3 = R3 * cosine/sine for w^n	;  21-25
	vsubpd	ymm2, ymm2, ymm6		;; A3 = A3 - I3				; 26-28
	vmulpd	ymm6, ymm6, ymm1		;; B3 = I3 * cosine/sine for w^n	;  22-26
	vaddpd	ymm6, ymm6, ymm0		;; B3 = B3 + R3				; 27-29

	L1prefetchw srcreg+d2+d1+L1pd, L1pt

	vmovapd	ymm0, [screg2]			;; sine for w^2n
	vmulpd	ymm3, ymm3, ymm0		;; A2 = A2 * sine (final R2)		;  25-29
	vmulpd	ymm5, ymm5, ymm0		;; B2 = B2 * sine (final I2)		;  26-30
	vmovapd	ymm0, [screg5]			;; sine for w^5n
	vmulpd	ymm4, ymm4, ymm0		;; A4 = A4 * sine (final R4)		;  27-31
	vmulpd	ymm7, ymm7, ymm0		;; B4 = B4 * sine (final I4)		;  28-32
	vmovapd	ymm0, [screg1]			;; sine for w^n
	vmulpd	ymm2, ymm2, ymm0		;; A3 = A3 * sine (final R3)		;  29-33
	vmulpd	ymm6, ymm6, ymm0		;; B3 = B3 * sine (final I3)		;  30-34

	;; Swizzle using 8 shufpds and 8 perm2f128  (1 or 3 clocks each on port 5, throughput 1)

	vshufpd	ymm0, ymm10, ymm3, 0		;; Shuffle R1 and R2 to create 0 8 2 10
	vshufpd	ymm1, ymm10, ymm3, 15		;; Shuffle R1 and R2 to create 1 9 3 11

	vshufpd	ymm3, ymm2, ymm4, 0		;; Shuffle R3 and R4 to create 16 24 18 26
	vshufpd	ymm2, ymm2, ymm4, 15		;; Shuffle R3 and R4 to create 17 25 19 27

	ylow128s ymm4, ymm0, ymm3		;; Shuffle R1/R2 low and R3/R4 low (0 8 16 24)
	yhigh128s ymm0, ymm0, ymm3		;; Shuffle R1/R2 low and R3/R4 low (2 10 18 26)

	ylow128s ymm3, ymm1, ymm2		;; Shuffle R1/R2 hi and R3/R4 hi (1 9 17 25)
	yhigh128s ymm1, ymm1, ymm2		;; Shuffle R1/R2 hi and R3/R4 hi (3 11 19 27)

	ystore	[srcreg], ymm4			;; Save (0 8 16 24)
	ystore	[srcreg+d2], ymm0		;; Save (2 10 18 26)
	ystore	[srcreg+d1], ymm3		;; Save (1 9 17 25)
	ystore	[srcreg+d2+d1], ymm1		;; Save (3 11 19 27)

	vshufpd	ymm0, ymm9, ymm5, 0		;; Shuffle I1 and I2 to create 4 12 6 14
	vshufpd	ymm4, ymm9, ymm5, 15		;; Shuffle I1 and I2 to create 5 13 7 15

	vshufpd	ymm5, ymm6, ymm7, 0		;; Shuffle I3 and I4 to create 16 28 22 30
	vshufpd	ymm6, ymm6, ymm7, 15		;; Shuffle I3 and I4 to create 21 29 23 31

	ylow128s ymm7, ymm0, ymm5		;; Shuffle I1/I2 low and I3/I4 low (4 12 20 28)
	yhigh128s ymm0, ymm0, ymm5		;; Shuffle I1/I2 low and I3/I4 low (6 14 22 30)

	ylow128s ymm5, ymm4, ymm6		;; Shuffle I1/I2 hi and I3/I4 hi (5 13 21 29)
	yhigh128s ymm4, ymm4, ymm6		;; Shuffle I1/I2 hi and I3/I4 hi (7 15 23 31)

	ystore	[srcreg+32], ymm7		;; Save (4 12 20 28)
	ystore	[srcreg+d2+32], ymm0		;; Save (6 14 22 30)
	ystore	[srcreg+d1+32], ymm5		;; Save (5 13 21 29)
	ystore	[srcreg+d2+d1+32], ymm4		;; Save (7 15 23 31)

	bump	srcreg, srcinc
	bump	scregA, scincA
	bump	scregB, scincB
	ENDM

;; Haswell FMA3 version

IF (@INSTR(,%yarch,<FMA3>) NE 0)

yr4_s8r_fft_cmn_preload MACRO
	vmovapd	ymm15, YMM_ONE
	vmovapd ymm14, YMM_SQRTHALF
	ENDM

yr4_s8r_fft_cmn MACRO srcreg,srcinc,srcoff,d1,d2,screg1,screg2,screg5,scregA,scincA,scregB,scincB,maxrpt,L1pt,L1pd
	vmovapd	ymm0, [srcreg+srcoff+d2+d1]	;; R4
	vmovapd	ymm7, [srcreg+srcoff+d2+d1+32]	;; R8
	vsubpd	ymm3, ymm0, ymm7		;; new R8 = R4 - R8				; 1-3
	yfmaddpd ymm7, ymm0, ymm15, ymm7	;; new R4 = R4 + R8				;	1-5

	vmovapd	ymm5, [srcreg+srcoff+d1]	;; R2
	vmovapd	ymm4, [srcreg+srcoff+d1+32]	;; R6
	vsubpd	ymm1, ymm5, ymm4		;; new R6 = R2 - R6				; 2-4
	yfmaddpd ymm5, ymm5, ymm15, ymm4	;; new R2 = R2 + R6				;	2-6

	vmovapd	ymm0, [srcreg+srcoff]		;; R1
	vmovapd	ymm2, [srcreg+srcoff+32]	;; R5
	vaddpd	ymm4, ymm0, ymm2		;; new R1 = R1 + R5				; 3-5
	yfmsubpd ymm0, ymm0, ymm15, ymm2	;; new R5 = R1 - R5				;	3-7

	vmovapd	ymm6, [srcreg+srcoff+d2]	;; R3
	vmovapd	ymm8, [srcreg+srcoff+d2+32]	;; R7
	vaddpd	ymm2, ymm6, ymm8		;; new R3 = R3 + R7				; 4-6
	yfmsubpd ymm6, ymm6, ymm15, ymm8	;; new R7 = R3 - R7				;	4-8

	vsubpd	ymm8, ymm1, ymm3		;; R6 = R6 - R8					; 5-7
	yfmaddpd ymm1, ymm1, ymm15, ymm3	;; R8 = R6 + R8					;	5-9
	L1prefetchw srcreg+L1pd, L1pt

	vsubpd	ymm3, ymm5, ymm7		;; R2 - R4 (final I2)				; 7-9
	yfmaddpd ymm5, ymm5, ymm15, ymm7	;; R2 + R4 (final I1, a.k.a 2nd real result)	;	7-11
	vmovapd	ymm9, [screg2+32]		;; cosine/sine for w^2n

	vsubpd	ymm7, ymm4, ymm2		;; R1 - R3 (final R2)				; 8-10
	yfmaddpd ymm4, ymm4, ymm15, ymm2	;; R1 + R3 (final R1)				;	8-12
	L1prefetchw srcreg+d1+L1pd, L1pt

	yfnmaddpd ymm2, ymm8, ymm14, ymm0	;; R5 - R6 * square root half (final R4)	; 9-13
	yfmaddpd ymm8, ymm8, ymm14, ymm0	;; R5 + R6 * square root half (final R3)	;	9-13
	vmovapd	ymm10, [screg5+32]		;; cosine/sine for w^5n

	yfnmaddpd ymm0, ymm1, ymm14, ymm6	;; R7 - R8 * square root half (final I4)	; 10-14
	yfmaddpd ymm1, ymm1, ymm14, ymm6	;; R7 + R8 * square root half (final I3)	;	10-14
	vmovapd	ymm11, [screg1+32]		;; cosine/sine for w^n

	yfmsubpd ymm6, ymm7, ymm9, ymm3		;; A2 = R2 * cosine/sine for w^2n - I2		; 11-15
	yfmaddpd ymm3, ymm3, ymm9, ymm7		;; B2 = I2 * cosine/sine for w^2n + R2		;	11-15
	vmovapd	ymm12, [screg2]			;; sine for w^2n

	yfmsubpd ymm7, ymm2, ymm10, ymm0	;; A4 = R4 * cosine/sine for w^5n - I4		; 15-19
	yfmaddpd ymm0, ymm0, ymm10, ymm2	;; B4 = I4 * cosine/sine for w^5n + R4		;	15-19
	vmovapd	ymm13, [screg5]			;; sine for w^5n

	yfmsubpd ymm2, ymm8, ymm11, ymm1	;; A3 = R3 * cosine/sine for w^n - I3		; 16-20
	yfmaddpd ymm1, ymm1, ymm11, ymm8	;; B3 = I3 * cosine/sine for w^n + R3		;	16-20
	vmovapd	ymm10, [screg1]			;; sine for w^n

	vmulpd	ymm6, ymm6, ymm12		;; A2 = A2 * sine (final R2)			; 17-21
	vmulpd	ymm3, ymm3, ymm12		;; B2 = B2 * sine (final I2)			;	17-21

	vmulpd	ymm7, ymm7, ymm13		;; A4 = A4 * sine (final R4)			; 20-24
	vmulpd	ymm0, ymm0, ymm13		;; B4 = B4 * sine (final I4)			;	20-24

	vmulpd	ymm2, ymm2, ymm10		;; A3 = A3 * sine (final R3)			; 21-25
	vmulpd	ymm1, ymm1, ymm10		;; B3 = B3 * sine (final I3)			;	21-25

	bump	scregA, scincA
	bump	scregB, scincB

	;; Swizzle using 8 shufpds and 8 perm2f128  (1 or 3 clocks each on port 5, throughput 1)

	vshufpd	ymm8, ymm4, ymm6, 0		;; Shuffle R1 and R2 to create 0 8 2 10
	vshufpd	ymm4, ymm4, ymm6, 15		;; Shuffle R1 and R2 to create 1 9 3 11
	L1prefetchw srcreg+d2+L1pd, L1pt

	vshufpd	ymm6, ymm2, ymm7, 0		;; Shuffle R3 and R4 to create 16 24 18 26
	vshufpd	ymm2, ymm2, ymm7, 15		;; Shuffle R3 and R4 to create 17 25 19 27

	vshufpd	ymm7, ymm5, ymm3, 0		;; Shuffle I1 and I2 to create 4 12 6 14
	vshufpd	ymm5, ymm5, ymm3, 15		;; Shuffle I1 and I2 to create 5 13 7 15
	L1prefetchw srcreg+d2+d1+L1pd, L1pt

	vshufpd	ymm3, ymm1, ymm0, 0		;; Shuffle I3 and I4 to create 16 28 22 30
	vshufpd	ymm1, ymm1, ymm0, 15		;; Shuffle I3 and I4 to create 21 29 23 31

	ylow128s ymm0, ymm8, ymm6		;; Shuffle R1/R2 low and R3/R4 low (0 8 16 24)
	yhigh128s ymm8, ymm8, ymm6		;; Shuffle R1/R2 low and R3/R4 low (2 10 18 26)

	ylow128s ymm6, ymm4, ymm2		;; Shuffle R1/R2 hi and R3/R4 hi (1 9 17 25)
	yhigh128s ymm4, ymm4, ymm2		;; Shuffle R1/R2 hi and R3/R4 hi (3 11 19 27)

	ystore	[srcreg], ymm0			;; Save (0 8 16 24)
	ystore	[srcreg+d2], ymm8		;; Save (2 10 18 26)
	ystore	[srcreg+d1], ymm6		;; Save (1 9 17 25)
	ystore	[srcreg+d2+d1], ymm4		;; Save (3 11 19 27)

	ylow128s ymm2, ymm7, ymm3		;; Shuffle I1/I2 low and I3/I4 low (4 12 20 28)
	yhigh128s ymm7, ymm7, ymm3		;; Shuffle I1/I2 low and I3/I4 low (6 14 22 30)

	ylow128s ymm3, ymm5, ymm1		;; Shuffle I1/I2 hi and I3/I4 hi (5 13 21 29)
	yhigh128s ymm5, ymm5, ymm1		;; Shuffle I1/I2 hi and I3/I4 hi (7 15 23 31)

	ystore	[srcreg+32], ymm2		;; Save (4 12 20 28)
	ystore	[srcreg+d2+32], ymm7		;; Save (6 14 22 30)
	ystore	[srcreg+d1+32], ymm3		;; Save (5 13 21 29)
	ystore	[srcreg+d2+d1+32], ymm5		;; Save (7 15 23 31)

	bump	srcreg, srcinc
	ENDM

ENDIF

ENDIF


;;
;; ************************************* eight-reals-unfft variants ******************************************
;;

;; These macros produce eight reals after doing 2 and 3/4 levels of the inverse FFT applying
;; the sin/cos multipliers beforehand.  The input is 2 reals (only 2 levels of inverse FFT done)
;; and 3 complex numbers (3 levels of inverse FFT performed).  These macros take a screg
;; that points to twiddles w^n, w^2n, and w^5n.

;; Standard eight-reals unfft.
yr4_4cl_eight_reals_unfft_preload MACRO
	yr4_8r_unfft_cmn_preload
	ENDM
yr4_4cl_eight_reals_unfft MACRO srcreg,srcinc,d1,d2,screg,scinc,maxrpt,L1pt,L1pd
	yr4_8r_unfft_cmn srcreg,srcinc,d1,d2,screg,screg+64,screg+128,screg,scinc,maxrpt,L1pt,L1pd
	ENDM

;; Like 4cl but uses one combined sin/cos data ptr.  The first two sin/cos entries can be used by
;; four complex macros at the same FFT level.  The third and fourth sin/cos entries are used by
;; this eight reals macro.
yr4_4cl_csc_eight_reals_unfft_preload MACRO
	yr4_8r_unfft_cmn_preload
	ENDM
yr4_4cl_csc_eight_reals_unfft MACRO srcreg,srcinc,d1,d2,screg,scinc,maxrpt,L1pt,L1pd
	yr4_8r_unfft_cmn srcreg,srcinc,d1,d2,screg+128,screg,screg+192,screg,scinc,maxrpt,L1pt,L1pd
	ENDM

;; Like 4cl, but swizzles the inputs.
;; This version is used in the last levels of a one pass AVX FFT.
yr4_s4cl_eight_reals_unfft_preload MACRO
	yr4_s8r_unfft_cmn_preload
	ENDM
yr4_s4cl_eight_reals_unfft MACRO srcreg,srcinc,d1,d2,screg,scinc,maxrpt,L1pt,L1pd
	yr4_s8r_unfft_cmn srcreg,srcinc,d1,d2,screg,screg+64,screg+128,screg,scinc,,,maxrpt,L1pt,L1pd
	ENDM

;; Like s4cl but uses two sin/cos data ptrs.
yr4_s4cl_2sc_eight_reals_unfft_preload MACRO
	yr4_s8r_unfft_cmn_preload
	ENDM
yr4_s4cl_2sc_eight_reals_unfft MACRO srcreg,srcinc,d1,d2,screg1,scinc1,screg2,scinc2,maxrpt,L1pt,L1pd
	yr4_s8r_unfft_cmn srcreg,srcinc,d1,d2,screg2,screg1,screg2+64,screg1,scinc1,screg2,scinc2,maxrpt,L1pt,L1pd
	ENDM

;; Like s4cl but uses one combined sin/cos data ptr.  The first two sin/cos entries can be used by
;; four complex macros at the same FFT level.  The third and fourth sin/cos entries are used by
;; this eight reals macro.
yr4_s4cl_csc_eight_reals_unfft_preload MACRO
	yr4_s8r_unfft_cmn_preload
	ENDM
yr4_s4cl_csc_eight_reals_unfft MACRO srcreg,srcinc,d1,d2,screg,scinc,maxrpt,L1pt,L1pd
	yr4_s8r_unfft_cmn srcreg,srcinc,d1,d2,screg+128,screg,screg+192,screg,scinc,,,maxrpt,L1pt,L1pd
	ENDM

;; Common code for eight reals unfft
yr4_8r_unfft_cmn_preload MACRO
	ENDM
yr4_8r_unfft_cmn MACRO srcreg,srcinc,d1,d2,screg1,screg2,screg5,screg,scinc,maxrpt,L1pt,L1pd
	vmovapd	ymm5, [screg5+32]	;; cosine/sine for w^5n
	vmovapd	ymm2, [srcreg+d2+d1]	;; R4
	vmulpd	ymm4, ymm2, ymm5	;; A4 = R4 * cosine/sine		; 1-5

	vmovapd	ymm6, [screg1+32] 	;; cosine/sine for w^n
	vmovapd	ymm3, [srcreg+d2]	;; R3
	vmulpd	ymm0, ymm3, ymm6	;; A3 = R3 * cosine/sine for w^n	; 2-6

	vmovapd	ymm7, [srcreg+d2+d1+32]	;; I4
	vmulpd	ymm5, ymm7, ymm5	;; B4 = I4 * cosine/sine		; 3-7

	vmovapd	ymm1, [srcreg+d2+32]	;; I3
	vmulpd	ymm6, ymm1, ymm6	;; B3 = I3 * cosine/sine for w^n	; 4-8

	vaddpd	ymm4, ymm4, ymm7	;; A4 = A4 + I4				; 6-8
	vaddpd	ymm0, ymm0, ymm1	;; A3 = A3 + I3				; 7-9
	vsubpd	ymm5, ymm5, ymm2	;; B4 = B4 - R4				; 8-10

	vmovapd	ymm7, [screg2+32]	;; cosine/sine for w^2n
	vmovapd	ymm1, [srcreg+d1]	;; R2
	vmulpd	ymm2, ymm1, ymm7	;; A2 = R2 * cosine/sine		; 5-9

	vsubpd	ymm6, ymm6, ymm3	;; B3 = B3 - R3				; 9-11

	vmovapd	ymm3, [srcreg+d1+32]	;; I2
	vmulpd	ymm7, ymm3, ymm7	;; B2 = I2 * cosine/sine		; 6-10

	vaddpd	ymm2, ymm2, ymm3	;; A2 = A2 + I2				; 10-12
	vsubpd	ymm7, ymm7, ymm1	;; B2 = B2 - R2				; 11-13

	L1prefetchw srcreg+L1pd, L1pt

	vmovapd	ymm1, [screg5]		;; sine for w^5n			
	vmulpd	ymm4, ymm4, ymm1	;; new R7 = A4 * sine			; 9-13
	vmovapd	ymm3, [screg1]		;; sine for w^n
	vmulpd	ymm0, ymm0, ymm3	;; new R5 = A3 * sine			; 10-14
	vmulpd	ymm5, ymm5, ymm1	;; new R8 = B4 * sine			; 11-15
	vmulpd	ymm6, ymm6, ymm3	;; new R6 = B3 * sine			; 12-16
	vmovapd	ymm1, [screg2]		;; sine for w^2n
	vmulpd	ymm2, ymm2, ymm1	;; new R3 = A2 * sine			; 13-17
	vmulpd	ymm7, ymm7, ymm1	;; new R4 = B2 * sine			; 14-18

	L1prefetchw srcreg+d1+L1pd, L1pt

	vsubpd	ymm1, ymm0, ymm4	;; new R6 = R5 - R7			; 15-17
	vaddpd	ymm0, ymm0, ymm4	;; new R5 = R5 + R7			; 16-18

	vsubpd	ymm4, ymm6, ymm5	;; new R8 = R6 - R8			; 17-19
	vaddpd	ymm6, ymm6, ymm5	;; new R7 = R6 + R8			; 18-20

	vaddpd	ymm5, ymm1, ymm4	;; R6 = R6 + R8				; 20-22
	vsubpd	ymm4, ymm4, ymm1	;; R8 = R8 - R6				; 21-23

	L1prefetchw srcreg+d2+L1pd, L1pt

	vmovapd	ymm1, [srcreg]		;; R1
	vsubpd	ymm3, ymm1, ymm2	;; R1 - R3 (new R3)			; 19-21
	vaddpd	ymm1, ymm1, ymm2	;; R1 + R3 (new R1)			; 22-24

	vsubpd	ymm2, ymm3, ymm6	;; R3 - R7 (final R7)			; 23-25
	vmulpd	ymm5, ymm5, YMM_SQRTHALF ;; R6 = R6 * square root of 1/2	; 23-27

	vaddpd	ymm3, ymm3, ymm6	;; R3 + R7 (final R3)			; 24-26
	vmulpd	ymm4, ymm4, YMM_SQRTHALF ;; R8 = R8 * square root of 1/2	; 24-28

	vmovapd	ymm6, [srcreg+32]	;; I1 (a.k.a R2)
	ystore	[srcreg+d2+32], ymm2	;; Save R7				; 26
	vaddpd	ymm2, ymm6, ymm7	;; R2 + R4 (new R2)			; 25-27

	vsubpd	ymm6, ymm6, ymm7	;; R2 - R4 (new R4)			; 26-28

	vsubpd	ymm7, ymm1, ymm0	;; R1 - R5 (final R5)			; 27-29
	ystore	[srcreg+d2], ymm3	;; Save R3				; 27

	vaddpd	ymm1, ymm1, ymm0	;; R1 + R5 (final R1)			; 28-30

	L1prefetchw srcreg+d2+d1+L1pd, L1pt

	vsubpd	ymm0, ymm2, ymm5	;; R2 - R6 (final R6)			; 29-31

	vaddpd	ymm2, ymm2, ymm5	;; R2 + R6 (final R2)			; 30-32
	ystore	[srcreg+32], ymm7	;; Save R5				; 30

	vsubpd	ymm7, ymm6, ymm4	;; R4 - R8 (final R8)			; 31-33
	ystore	[srcreg], ymm1		;; Save R1				; 31

	vaddpd	ymm6, ymm6, ymm4	;; R4 + R8 (final R4)			; 32-34
	ystore	[srcreg+d1+32], ymm0	;; Save R6				; 32

	ystore	[srcreg+d1], ymm2	;; Save R2				; 33
	ystore	[srcreg+d2+d1+32], ymm7	;; Save R8				; 34
	ystore	[srcreg+d2+d1], ymm6	;; Save R4				; 35

	bump	srcreg, srcinc
	bump	screg, scinc
	ENDM

;; Common swizzling eight reals unfft
yr4_s8r_unfft_cmn_preload MACRO
	ENDM
yr4_s8r_unfft_cmn MACRO srcreg,srcinc,d1,d2,screg1,screg2,screg5,scregA,scincA,scregB,scincB,maxrpt,L1pt,L1pd

	;; On input the 64-byte cache lines hold these data values:
	;;	0	8	16	24	4	12	20	28
	;;	1	...
	;;	2	...
	;;	3	...

	;; These are swizzled to:
	;;	0	1	2	3	4	5	6	7
	;;	8	...
	;;	16	...
	;;	24	...

	;; Swizzle using 8 shufpds and 8 perm2f128  (1 or 3 clocks each on port 5, throughput 1)

	vmovapd	ymm1, [srcreg]		;; R1
	vmovapd	ymm7, [srcreg+d1]	;; R2
	vshufpd	ymm0, ymm1, ymm7, 15	;; Shuffle R1 and R2 to create 8 9 24 25		; 1
	vshufpd	ymm1, ymm1, ymm7, 0	;; Shuffle R1 and R2 to create 0 1 16 17		; 2

	vmovapd	ymm2, [srcreg+d2]	;; R3
	vmovapd	ymm7, [srcreg+d2+d1]	;; R4
	vshufpd	ymm3, ymm2, ymm7, 15	;; Shuffle R3 and R4 to create 10 11 26 27		; 3
	vshufpd	ymm2, ymm2, ymm7, 0	;; Shuffle R3 and R4 to create 2 3 18 19		; 4

	ylow128s ymm4, ymm0, ymm3	;; Shuffle R1/R2 hi and R3/R4 hi (8 9 10 11)		; 5-6
	yhigh128s ymm0, ymm0, ymm3	;; Shuffle R1/R2 hi and R3/R4 hi (24 25 26 27)		; 6-7

	ylow128s ymm3, ymm1, ymm2	;; Shuffle R1/R2 low and R3/R4 low (0 1 2 3)		; 7-8
	yhigh128s ymm1, ymm1, ymm2	;; Shuffle R1/R2 low and R3/R4 low (16 17 18 19)	; 8-9

	vmovapd	ymm6, [srcreg+32]	;; I1
	vmovapd	ymm2, [srcreg+d1+32]	;; I2
	vshufpd	ymm5, ymm6, ymm2, 15	;; Shuffle I1 and I2 to create 12 13 28 29
	vshufpd	ymm6, ymm6, ymm2, 0	;; Shuffle I1 and I2 to create 4 5 20 21

	ystore	[srcreg], ymm3		;; Temporarily save R1 (0 1 2 3)

	vmovapd	ymm2, [srcreg+d2+32]	;; I3
	vmovapd	ymm7, [srcreg+d2+d1+32]	;; I4
	vshufpd	ymm3, ymm2, ymm7, 15	;; Shuffle I3 and I4 to create 14 15 30 31
	vshufpd	ymm2, ymm2, ymm7, 0	;; Shuffle I3 and I4 to create 6 7 22 23

	ylow128s ymm7, ymm5, ymm3	;; Shuffle I1/I2 hi and I3/I4 hi (12 13 14 15)
	yhigh128s ymm5, ymm5, ymm3	;; Shuffle I1/I2 hi and I3/I4 hi (28 29 30 31)

	ylow128s ymm3, ymm6, ymm2	;; Shuffle I1/I2 low and I3/I4 low (4 5 6 7)
	yhigh128s ymm6, ymm6, ymm2	;; Shuffle I1/I2 low and I3/I4 low (20 21 22 23)

	ystore	[srcreg+32], ymm1	;; Temporarily save R3 (16 17 18 19)

	vmovapd	ymm1, [screg2+32]	;; cosine/sine for w^2n
	vmulpd	ymm2, ymm4, ymm1	;; A2 = R2 * cosine/sine
	vaddpd	ymm2, ymm2, ymm7	;; A2 = A2 + I2
	vmulpd	ymm7, ymm7, ymm1	;; B2 = I2 * cosine/sine
	vsubpd	ymm7, ymm7, ymm4	;; B2 = B2 - R2

	vmovapd	ymm1, [screg5+32]	;; cosine/sine for w^5n
	vmulpd	ymm4, ymm0, ymm1	;; A4 = R4 * cosine/sine
	vaddpd	ymm4, ymm4, ymm5	;; A4 = A4 + I4
	vmulpd	ymm5, ymm5, ymm1	;; B4 = I4 * cosine/sine
	vsubpd	ymm5, ymm5, ymm0	;; B4 = B4 - R4

	vmovapd	ymm1, [screg1+32] 	;; cosine/sine for w^n
	vmulpd	ymm0, ymm1, [srcreg+32]	;; A3 = R3 * cosine/sine for w^n
	vaddpd	ymm0, ymm0, ymm6	;; A3 = A3 + I3
	vmulpd	ymm6, ymm6, ymm1	;; B3 = I3 * cosine/sine for w^n
	vsubpd	ymm6, ymm6, [srcreg+32]	;; B3 = B3 - R3

	L1prefetchw srcreg+L1pd, L1pt

	vmovapd	ymm1, [screg2]		;; sine for w^2n
	vmulpd	ymm2, ymm2, ymm1	;; new R3 = A2 * sine
	vmulpd	ymm7, ymm7, ymm1	;; new R4 = B2 * sine
	vmovapd	ymm1, [screg5]		;; sine for w^5n
	vmulpd	ymm4, ymm4, ymm1	;; new R7 = A4 * sine
	vmulpd	ymm5, ymm5, ymm1	;; new R8 = B4 * sine
	vmovapd	ymm1, [screg1]		;; sine for w^n
	vmulpd	ymm0, ymm0, ymm1	;; new R5 = A3 * sine
	vmulpd	ymm6, ymm6, ymm1	;; new R6 = B3 * sine

	L1prefetchw srcreg+d1+L1pd, L1pt

	vsubpd	ymm1, ymm0, ymm4	;; new R6 = R5 - R7
	vaddpd	ymm0, ymm0, ymm4	;; new R5 = R5 + R7

	vsubpd	ymm4, ymm6, ymm5	;; new R8 = R6 - R8
	vaddpd	ymm6, ymm6, ymm5	;; new R7 = R6 + R8

	L1prefetchw srcreg+d2+L1pd, L1pt

	vaddpd	ymm5, ymm1, ymm4	;; R6 = R6 + R8
	vsubpd	ymm4, ymm4, ymm1	;; R8 = R8 - R6

	vmulpd	ymm5, ymm5, YMM_SQRTHALF ;; R6 = R6 * square root of 1/2
	vmulpd	ymm4, ymm4, YMM_SQRTHALF ;; R8 = R8 * square root of 1/2

	vsubpd	ymm1, ymm3, ymm7	;; R2 - R4 (new R4)
	vaddpd	ymm3, ymm3, ymm7	;; R2 + R4 (new R2)

	L1prefetchw srcreg+d2+d1+L1pd, L1pt

	vsubpd	ymm7, ymm1, ymm4	;; R4 - R8 (final R8)
	vaddpd	ymm1, ymm1, ymm4	;; R4 + R8 (final R4)

	vsubpd	ymm4, ymm3, ymm5	;; R2 - R6 (final R6)
	vaddpd	ymm3, ymm3, ymm5	;; R2 + R6 (final R2)

	vmovapd ymm5, [srcreg]		;; Reload R1

	ystore	[srcreg+d2+d1+32], ymm7	;; Save R8
	ystore	[srcreg+d2+d1], ymm1	;; Save R4

	vsubpd	ymm7, ymm5, ymm2	;; R1 - R3 (new R3)
	vaddpd	ymm5, ymm5, ymm2	;; R1 + R3 (new R1)

	vsubpd	ymm2, ymm7, ymm6	;; R3 - R7 (final R7)
	vaddpd	ymm7, ymm7, ymm6	;; R3 + R7 (final R3)

	vsubpd	ymm6, ymm5, ymm0	;; R1 - R5 (final R5)
	vaddpd	ymm5, ymm5, ymm0	;; R1 + R5 (final R1)

	ystore	[srcreg+d1+32], ymm4	;; Save R6
	ystore	[srcreg+d1], ymm3	;; Save R2

	ystore	[srcreg+d2+32], ymm2	;; Save R7
	ystore	[srcreg+d2], ymm7	;; Save R3

	ystore	[srcreg+32], ymm6	;; Save R5
	ystore	[srcreg], ymm5		;; Save R1

	bump	srcreg, srcinc
	bump	scregA, scincA
	bump	scregB, scincB
	ENDM

;; 64-bit version

IFDEF X86_64

yr4_8r_unfft_cmn_preload MACRO
	vmovapd	ymm15, YMM_SQRTHALF
	ENDM
yr4_8r_unfft_cmn MACRO srcreg,srcinc,d1,d2,screg1,screg2,screg5,screg,scinc,maxrpt,L1pt,L1pd
	IF maxrpt MOD 5 EQ 0
	yr4_8r_unfft_unroll -1,noexec,noexec,exec,srcreg,srcinc,d1,d2,screg1,screg2,screg5,screg,scinc,L1pt,L1pd
	yr4_8r_unfft_unroll 0,noexec,exec,exec,srcreg,srcinc,d1,d2,screg1,screg2,screg5,screg,scinc,L1pt,L1pd
	yr4_8r_unfft_unroll 1,exec,exec,exec,srcreg,srcinc,d1,d2,screg1,screg2,screg5,screg,scinc,L1pt,L1pd
	yr4_8r_unfft_unroll 2,exec,exec,exec,srcreg,srcinc,d1,d2,screg1,screg2,screg5,screg,scinc,L1pt,L1pd
	yr4_8r_unfft_unroll 3,exec,exec,exec,srcreg,srcinc,d1,d2,screg1,screg2,screg5,screg,scinc,L1pt,L1pd
	yr4_8r_unfft_unroll 4,exec,exec,noexec,srcreg,srcinc,d1,d2,screg1,screg2,screg5,screg,scinc,L1pt,L1pd
	yr4_8r_unfft_unroll 5,exec,noexec,noexec,srcreg,srcinc,d1,d2,screg1,screg2,screg5,screg,scinc,L1pt,L1pd
	bump	srcreg, 5*srcinc
	bump	screg, 5*scinc
	ELSEIF maxrpt MOD 4 EQ 0
	yr4_8r_unfft_unroll -1,noexec,noexec,exec,srcreg,srcinc,d1,d2,screg1,screg2,screg5,screg,scinc,L1pt,L1pd
	yr4_8r_unfft_unroll 0,noexec,exec,exec,srcreg,srcinc,d1,d2,screg1,screg2,screg5,screg,scinc,L1pt,L1pd
	yr4_8r_unfft_unroll 1,exec,exec,exec,srcreg,srcinc,d1,d2,screg1,screg2,screg5,screg,scinc,L1pt,L1pd
	yr4_8r_unfft_unroll 2,exec,exec,exec,srcreg,srcinc,d1,d2,screg1,screg2,screg5,screg,scinc,L1pt,L1pd
	yr4_8r_unfft_unroll 3,exec,exec,noexec,srcreg,srcinc,d1,d2,screg1,screg2,screg5,screg,scinc,L1pt,L1pd
	yr4_8r_unfft_unroll 4,exec,noexec,noexec,srcreg,srcinc,d1,d2,screg1,screg2,screg5,screg,scinc,L1pt,L1pd
	bump	srcreg, 4*srcinc
	bump	screg, 4*scinc
	ELSEIF maxrpt MOD 3 EQ 0
	yr4_8r_unfft_unroll -1,noexec,noexec,exec,srcreg,srcinc,d1,d2,screg1,screg2,screg5,screg,scinc,L1pt,L1pd
	yr4_8r_unfft_unroll 0,noexec,exec,exec,srcreg,srcinc,d1,d2,screg1,screg2,screg5,screg,scinc,L1pt,L1pd
	yr4_8r_unfft_unroll 1,exec,exec,exec,srcreg,srcinc,d1,d2,screg1,screg2,screg5,screg,scinc,L1pt,L1pd
	yr4_8r_unfft_unroll 2,exec,exec,noexec,srcreg,srcinc,d1,d2,screg1,screg2,screg5,screg,scinc,L1pt,L1pd
	yr4_8r_unfft_unroll 3,exec,noexec,noexec,srcreg,srcinc,d1,d2,screg1,screg2,screg5,screg,scinc,L1pt,L1pd
	bump	srcreg, 3*srcinc
	bump	screg, 3*scinc
	ELSEIF maxrpt MOD 2 EQ 0
	yr4_8r_unfft_unroll -1,noexec,noexec,exec,srcreg,srcinc,d1,d2,screg1,screg2,screg5,screg,scinc,L1pt,L1pd
	yr4_8r_unfft_unroll 0,noexec,exec,exec,srcreg,srcinc,d1,d2,screg1,screg2,screg5,screg,scinc,L1pt,L1pd
	yr4_8r_unfft_unroll 1,exec,exec,noexec,srcreg,srcinc,d1,d2,screg1,screg2,screg5,screg,scinc,L1pt,L1pd
	yr4_8r_unfft_unroll 2,exec,noexec,noexec,srcreg,srcinc,d1,d2,screg1,screg2,screg5,screg,scinc,L1pt,L1pd
	bump	srcreg, 2*srcinc
	bump	screg, 2*scinc
	ELSE
	yr4_8r_unfft_unroll -1,noexec,noexec,exec,srcreg,srcinc,d1,d2,screg1,screg2,screg5,screg,scinc,L1pt,L1pd
	yr4_8r_unfft_unroll 0,noexec,exec,noexec,srcreg,srcinc,d1,d2,screg1,screg2,screg5,screg,scinc,L1pt,L1pd
	yr4_8r_unfft_unroll 1,exec,noexec,noexec,srcreg,srcinc,d1,d2,screg1,screg2,screg5,screg,scinc,L1pt,L1pd
	bump	srcreg, srcinc
	bump	screg, scinc
	ENDIF
	ENDM

yr4_8r_unfft_unroll MACRO iter,prev,this,next,srcreg,srcinc,d1,d2,screg1,screg2,screg5,screg,scinc,L1pt,L1pd

;; On first call, init registers

	IF iter EQ -1
y0	TEXTEQU	<ymm0>
y1	TEXTEQU	<ymm1>
y2	TEXTEQU	<ymm2>
y3	TEXTEQU	<ymm3>
y4	TEXTEQU	<ymm4>
y5	TEXTEQU	<ymm5>
y6	TEXTEQU	<ymm6>
y7	TEXTEQU	<ymm7>
y8	TEXTEQU	<ymm8>
y9	TEXTEQU	<ymm9>
y10	TEXTEQU	<ymm10>
y11	TEXTEQU	<ymm11>
y12	TEXTEQU	<ymm12>
y13	TEXTEQU	<ymm13>
y14	TEXTEQU	<ymm14>
	ENDIF

;; On later calls, previous R2,R4,R6,R8 will be in y0-3.  This A4,A3,A2,B4,B3,I4,I3,R4,R3,R2,c/s will be in y4-14.
;; ymm15 has YMM_SQRTHALF.

this	vaddpd	y4, y4, y9			;; A4 = A4 + I4				; 1-3
this	vmovapd	y9, [srcreg+iter*srcinc+d1+32]	;; I2

this	vaddpd	y5, y5, y10			;; A3 = A3 + I3				; 2-4
this	vmulpd	y14, y9, y14			;; B2 = I2 * cosine/sine w^2		; 1-5
this	vmovapd	y10, [screg5+iter*scinc]	;; sine for w^5			

this	vsubpd	y7, y7, y11			;; B4 = B4 - R4				; 3-5
this	vmovapd	y11, [screg1+iter*scinc]	;; sine for w^1

this	vsubpd	y8, y8, y12			;; B3 = B3 - R3				; 4-6
this	vmulpd	y4, y4, y10			;; new R7 = A4 * sine w^5		; 4-8
this	vmovapd	y12, [screg2+iter*scinc]	;; sine for w^2

this	vaddpd	y6, y6, y9			;; A2 = A2 + I2				; 5-7
this	vmulpd	y5, y5, y11			;; new R5 = A3 * sine w^1		; 5-9
this	vmovapd	y9, [srcreg+iter*srcinc]	;; R1

this	vsubpd	y14, y14, y13			;; B2 = B2 - R2				; 6-8
this	vmulpd	y7, y7, y10			;; new R8 = B4 * sine w^5		; 6-10
this	vmovapd	y10, [srcreg+iter*srcinc+32]	;; I1 (a.k.a R2)

prev	vaddpd	y0, y0, y2			;; R2 + R6 (final R2)			; 7-9
this	vmulpd	y8, y8, y11			;; new R6 = B3 * sine w^1		; 7-11
next	vmovapd	y11, [screg5+(iter+1)*scinc+32]	;; cosine/sine for w^5

prev	vsubpd	y2, y1, y3			;; R4 - R8 (final R8)			; 8-10
this	vmulpd	y6, y6, y12			;; new R3 = A2 * sine w^2		; 8-12
next	vmovapd	y13, [srcreg+(iter+1)*srcinc+d2+d1] ;; R4

prev	vaddpd	y1, y1, y3			;; R4 + R8 (final R4)			; 9-11
this	vmulpd	y14, y14, y12			;; new R4 = B2 * sine w^2		; 9-13
next	vmovapd	y12, [screg1+(iter+1)*scinc+32]	;; cosine/sine for w^1

this	vsubpd	y3, y5, y4			;; new R6 = R5 - R7			; 10-12
prev	ystore	[srcreg+(iter-1)*srcinc+d1], y0 ;; Save R2				; 10
this	L1prefetchw srcreg+iter*srcinc+L1pd, L1pt

this	vaddpd	y5, y5, y4			;; new R5 = R5 + R7			; 11-13
prev	ystore	[srcreg+(iter-1)*srcinc+d2+d1+32], y2 ;; Save R8			; 11

this	vsubpd	y2, y8, y7			;; new R8 = R6 - R8			; 12-14
prev	ystore	[srcreg+(iter-1)*srcinc+d2+d1], y1 ;; Save R4				; 12

this	vaddpd	y8, y8, y7			;; new R7 = R6 + R8			; 13-15
next	vmovapd	y4, [srcreg+(iter+1)*srcinc+d2+d1+32] ;; I4
this	L1prefetchw srcreg+iter*srcinc+d1+L1pd, L1pt

this	vsubpd	y1, y9, y6			;; R1 - R3 (new R3)			; 14-16
next	vmovapd	y7, [srcreg+(iter+1)*srcinc+d2]	;; R3

this	vaddpd	y0, y3, y2			;; R6 = R6 + R8				; 15-17
this next yloop_unrolled_one

this	vsubpd	y2, y2, y3			;; R8 = R8 - R6				; 16-18
next	vmovapd	y3, [screg2+(iter+1)*scinc+32]	;; cosine/sine for w^2

this	vaddpd	y9, y9, y6			;; R1 + R3 (new R1)			; 17-19
this	L1prefetchw srcreg+iter*srcinc+d2+L1pd, L1pt

this	vsubpd	y6, y1, y8			;; R3 - R7 (final R7)			; 18-20
this	vmulpd	y0, y0, ymm15			;; R6 = R6 * square root of 1/2		; 18-22

this	vaddpd	y1, y1, y8			;; R3 + R7 (final R3)			; 19-21
this	vmulpd	y2, y2, ymm15			;; R8 = R8 * square root of 1/2		; 19-23
this	L1prefetchw srcreg+iter*srcinc+d2+d1+L1pd, L1pt

this	vaddpd	y8, y10, y14			;; R2 + R4 (new R2)			; 20-22
this	ystore	[srcreg+iter*srcinc+d2+32], y6	;; Save R7				; 21
next	vmulpd	y6, y13, y11			;; A4 = R4 * cosine/sine w^5		; 20-24

this	vsubpd	y10, y10, y14			;; R2 - R4 (new R4)			; 21-23
next	vmulpd	y14, y7, y12			;; A3 = R3 * cosine/sine w^1		; 21-25
this	ystore	[srcreg+iter*srcinc+d2], y1	;; Save R3				; 22

this	vsubpd	y1, y9, y5			;; R1 - R5 (final R5)			; 22-24
next	vmulpd	y11, y4, y11			;; B4 = I4 * cosine/sine w^5		; 22-26
this	ystore	[srcreg+iter*srcinc+32], y1	;; Save R5				; 25
next	vmovapd	y1, [srcreg+(iter+1)*srcinc+d2+32] ;; I3

this	vaddpd	y9, y9, y5			;; R1 + R5 (final R1)			; 23-25
next	vmulpd	y12, y1, y12			;; B3 = I3 * cosine/sine w^1		; 23-27
next	vmovapd	y5, [srcreg+(iter+1)*srcinc+d1] ;; R2
this	ystore	[srcreg+iter*srcinc], y9	;; Save R1				; 26

this	vsubpd	y9, y8, y0			;; R2 - R6 (final R6)			; 24-26
this	ystore	[srcreg+iter*srcinc+d1+32], y9	;; Save R6				; 27
next	vmulpd	y9, y5, y3			;; A2 = R2 * cosine/sine w^2		; 24-28

;; Shuffle register assignments so that next call has R2,R4,R6,R8 in y0-3 and next A4,A3,A2,B4,B3,I4,I3,R4,R3,R2,c/s in y4-14.

ytmp	TEXTEQU	y0
y0	TEXTEQU	y8
y8	TEXTEQU	y12
y12	TEXTEQU	y7
y7	TEXTEQU	y11
y11	TEXTEQU	y13
y13	TEXTEQU	y5
y5	TEXTEQU	y14
y14	TEXTEQU	y3
y3	TEXTEQU	y2
y2	TEXTEQU	ytmp
ytmp	TEXTEQU y1
y1	TEXTEQU	y10
y10	TEXTEQU	ytmp
ytmp	TEXTEQU y4
y4	TEXTEQU	y6
y6	TEXTEQU	y9
y9	TEXTEQU ytmp
	ENDM


;; Haswell FMA3 version

IF (@INSTR(,%yarch,<FMA3>) NE 0)

yr4_8r_unfft_cmn_preload MACRO
	vmovapd	ymm14, YMM_ONE
	vmovapd	ymm15, YMM_SQRTHALF
	ENDM

;; uops = 6 s/c loads, 8 loads, 8 stores, 2 muls, 24 fma, 12 mov = 60 uops / 4 = 15 clocks
;; Timed at 16.1 clocks
yr4_8r_unfft_unroll MACRO iter,prev,this,next,srcreg,srcinc,d1,d2,screg1,screg2,screg5,screg,scinc,L1pt,L1pd

;; On first call, init registers

	IF iter EQ -1
y0	TEXTEQU	<ymm0>
y1	TEXTEQU	<ymm1>
y2	TEXTEQU	<ymm2>
y3	TEXTEQU	<ymm3>
y4	TEXTEQU	<ymm4>
y5	TEXTEQU	<ymm5>
y6	TEXTEQU	<ymm6>
y7	TEXTEQU	<ymm7>
y8	TEXTEQU	<ymm8>
y9	TEXTEQU	<ymm9>
y10	TEXTEQU	<ymm10>
y11	TEXTEQU	<ymm11>
y12	TEXTEQU	<ymm12>
y13	TEXTEQU	<ymm13>
	ENDIF

;; On later calls, previous R2,R4,R6,R8 will be in y0-3.  This sine w^2,sine w^5,R1,A3,A4,R5,R6,A7,A8 will be in y4-12.
;; The last two registers are reserved for YMM_ONE and YMM_SQRTHALF.

this	yfnmaddpd y13, y11, y5, y9		;; newer R6 = R5 - A7 * sine w^5	; 1-5	(used 7)
this	yfmaddpd y11, y11, y5, y9		;; newer R5 = R5 + A7 * sine w^5	; 1-5	(used 10)
next	vmovapd	y9, [srcreg+(iter+1)*srcinc+d2]	;; R3
prev	ystore	[srcreg+(iter-1)*srcinc+d1], y0 ;; Save R2				; 17

this	yfnmaddpd y0, y12, y5, y10		;; newer R8 = R6 - A8 * sine w^5	; 2-6	(used 7)
this	yfmaddpd y12, y12, y5, y10		;; newer R7 = R6 + A8 * sine w^5	; 2-6	(used 8)
next	vmovapd	y5, [srcreg+(iter+1)*srcinc+d2+32] ;; I3
this	L1prefetchw srcreg+iter*srcinc+L1pd, L1pt

this	yfnmaddpd y10, y7, y4, y6		;; R1 - A3 * sine w^2 (newer R3)	; 3-7	(used 8)
this	yfmaddpd y7, y7, y4, y6			;; R1 + A3 * sine w^2 (newer R1)	; 3-7	(used 10)
next	vmovapd	y6, [screg1+(iter+1)*scinc+32]	;; cosine/sine for w^1
prev	ystore	[srcreg+(iter-1)*srcinc+d1+32], y2 ;; Save R6				; 17

next	yfmaddpd y2, y9, y6, y5			;; A5 = R3 * cosine/sine w^1 + I3	;	4-8  (used 9)
next	yfmsubpd y5, y5, y6, y9			;; A6 = I3 * cosine/sine w^1 - R3	;	4-8
this	vmovapd	y6, [srcreg+iter*srcinc+32]	;; I1 (a.k.a R2)
next	vmovapd	y9, [srcreg+(iter+1)*srcinc+d2+d1] ;; R4
prev	ystore	[srcreg+(iter-1)*srcinc+d2+d1], y1 ;; Save R4				; 18

this	yfmaddpd y1, y8, y4, y6			;; R2 + A4 * sine w^2 (newer R2)	; 5-9	(used 12)
this	yfnmaddpd y8, y8, y4, y6		;; R2 - A4 * sine w^2 (newer R4)	; 5-9	(used 13)
next	vmovapd	y4, [srcreg+(iter+1)*srcinc+d2+d1+32] ;; I4
next	vmovapd	y6, [screg5+(iter+1)*scinc+32]	;; cosine/sine for w^5
prev	ystore	[srcreg+(iter-1)*srcinc+d2+d1+32], y3 ;; Save R8			; 18

next	yfmaddpd y3, y9, y6, y4			;; A7 = R4 * cosine/sine w^5 + I4	;	6-10  (used next 1)
next	yfmsubpd y4, y4, y6, y9			;; A8 = I4 * cosine/sine w^5 - R4	;	6-10  (used next 2)
next	vmovapd	y6, [screg1+(iter+1)*scinc]	;; sine for w^1
this	L1prefetchw srcreg+iter*srcinc+d1+L1pd, L1pt

this	yfmaddpd y9, y13, ymm14, y0		;; newest R6 = R6 + R8			; 7-11  (used 12)
this	yfmsubpd y13, y0, ymm14, y13		;; newest R8 = R8 - R6			; 7-11	(used 13)
this next yloop_unrolled_one

this	yfmaddpd y0, y10, ymm14, y12		;; R3 + R7 (final R3)			; 8-12
this	yfmsubpd y10, y10, ymm14, y12		;; R3 - R7 (final R7)			; 8-12
next	vmovapd	y12, [srcreg+(iter+1)*srcinc+d1] ;; R2
this	L1prefetchw srcreg+iter*srcinc+d2+L1pd, L1pt

next	vmulpd	y2, y2, y6			;; new R5 = A5 * sine w^1		;	9-13  (used next 1)
next	vmulpd	y5, y5, y6			;; new R6 = A6 * sine w^1		;	9-13  (used next 2)
next	vmovapd	y6, [srcreg+(iter+1)*srcinc+d1+32] ;; I2
this	ystore	[srcreg+iter*srcinc+d2], y0	;; Save R3				; 13

this	yfmaddpd y0, y7, ymm14, y11		;; R1 + R5 (final R1)			; 10-14
this	yfmsubpd y7, y7, ymm14, y11		;; R1 - R5 (final R5)			; 10-14
next	vmovapd	y11, [screg2+(iter+1)*scinc+32]	;; cosine/sine for w^2
this	ystore	[srcreg+iter*srcinc+d2+32], y10	;; Save R7				; 13

next	yfmaddpd y10, y12, y11, y6		;; A3 = R2 * cosine/sine w^2 + I2	;	11-15  (used next 3)
next	yfmsubpd y6, y6, y11, y12		;; A4 = I2 * cosine/sine w^2 - R2	;	11-15  (used next 5)
next	vmovapd	y11, [screg5+(iter+1)*scinc]	;; sine for w^5
this	L1prefetchw srcreg+iter*srcinc+d2+d1+L1pd, L1pt

this	yfmaddpd y12, y9, ymm15, y1		;; R2 + R6 * SQRTHALF (final R2)	; 12-16
this	yfnmaddpd y9, y9, ymm15, y1		;; R2 - R6 * SQRTHALF (final R6)	; 12-16
next	vmovapd	y1, [screg2+(iter+1)*scinc]	;; sine for w^2
this	ystore	[srcreg+iter*srcinc], y0	;; Save R1				; 15

this	yfmaddpd y0, y13, ymm15, y8		;; R4 + R8 * SQRTHALF (final R4)	; 13-17
this	yfnmaddpd y13, y13, ymm15, y8		;; R4 - R8 * SQRTHALF (final R8)	; 13-17
next	vmovapd	y8, [srcreg+(iter+1)*srcinc]	;; R1
this	ystore	[srcreg+iter*srcinc+32], y7	;; Save R5				; 15

;; Shuffle register assignments so that next call has R2,R4,R6,R8 in y0-3 and next sine w^2,sine w^5,R1,A3,A4,R5,R6,A7,A8 in y4-12.

ytmp	TEXTEQU	y0
y0	TEXTEQU	y12
y12	TEXTEQU	y4
y4	TEXTEQU	y1
y1	TEXTEQU	ytmp
ytmp	TEXTEQU	y2
y2	TEXTEQU	y9
y9	TEXTEQU	ytmp
ytmp	TEXTEQU	y3
y3	TEXTEQU	y13
y13	TEXTEQU	y7
y7	TEXTEQU y10
y10	TEXTEQU	y5
y5	TEXTEQU	y11
y11	TEXTEQU ytmp
ytmp	TEXTEQU	y6
y6	TEXTEQU	y8
y8	TEXTEQU ytmp
	ENDM

ENDIF

;; 64-bit swizzling version

yr4_s8r_unfft_cmn_preload MACRO
	vmovapd ymm14, YMM_SQRTHALF
	ENDM

yr4_s8r_unfft_cmn MACRO srcreg,srcinc,d1,d2,screg1,screg2,screg5,scregA,scincA,scregB,scincB,maxrpt,L1pt,L1pd

	;; On input the 64-byte cache lines hold these data values:
	;;	0	8	16	24	4	12	20	28
	;;	1	...
	;;	2	...
	;;	3	...

	;; These are swizzled to:
	;;	0	1	2	3	4	5	6	7
	;;	8	...
	;;	16	...
	;;	24	...

	;; Swizzle using 8 shufpds and 8 perm2f128  (1 or 3 clocks each on port 5, throughput 1)

	vmovapd	ymm1, [srcreg]		;; R1
	vmovapd	ymm7, [srcreg+d1]	;; R2
	vshufpd	ymm0, ymm1, ymm7, 15	;; Shuffle R1 and R2 to create 8 9 24 25		; 1
	vshufpd	ymm1, ymm1, ymm7, 0	;; Shuffle R1 and R2 to create 0 1 16 17		; 2

	vmovapd	ymm2, [srcreg+d2]	;; R3
	vmovapd	ymm7, [srcreg+d2+d1]	;; R4
	vshufpd	ymm3, ymm2, ymm7, 15	;; Shuffle R3 and R4 to create 10 11 26 27		; 3
	vshufpd	ymm2, ymm2, ymm7, 0	;; Shuffle R3 and R4 to create 2 3 18 19		; 4

	ylow128s ymm4, ymm0, ymm3	;; Shuffle R1/R2 hi and R3/R4 hi (8 9 10 11)		; 5-6
	yhigh128s ymm0, ymm0, ymm3	;; Shuffle R1/R2 hi and R3/R4 hi (24 25 26 27)		; 6-7

	ylow128s ymm9, ymm1, ymm2	;; Shuffle R1/R2 low and R3/R4 low (0 1 2 3)		; 7-8
	yhigh128s ymm10, ymm1, ymm2	;; Shuffle R1/R2 low and R3/R4 low (16 17 18 19)	; 8-9

	vmovapd	ymm6, [srcreg+32]	;; I1
	vmovapd	ymm2, [srcreg+d1+32]	;; I2
	vshufpd	ymm5, ymm6, ymm2, 15	;; Shuffle I1 and I2 to create 12 13 28 29
	vshufpd	ymm6, ymm6, ymm2, 0	;; Shuffle I1 and I2 to create 4 5 20 21

	vmovapd	ymm2, [srcreg+d2+32]	;; I3
	vmovapd	ymm7, [srcreg+d2+d1+32]	;; I4
	vshufpd	ymm3, ymm2, ymm7, 15	;; Shuffle I3 and I4 to create 14 15 30 31
	vshufpd	ymm2, ymm2, ymm7, 0	;; Shuffle I3 and I4 to create 6 7 22 23

	ylow128s ymm7, ymm5, ymm3	;; Shuffle I1/I2 hi and I3/I4 hi (12 13 14 15)
	yhigh128s ymm5, ymm5, ymm3	;; Shuffle I1/I2 hi and I3/I4 hi (28 29 30 31)

	ylow128s ymm3, ymm6, ymm2	;; Shuffle I1/I2 low and I3/I4 low (4 5 6 7)
	yhigh128s ymm6, ymm6, ymm2	;; Shuffle I1/I2 low and I3/I4 low (20 21 22 23)

	vmovapd	ymm1, [screg2+32]	;; cosine/sine for w^2n
	vmulpd	ymm2, ymm4, ymm1	;; A2 = R2 * cosine/sine
	vaddpd	ymm2, ymm2, ymm7	;; A2 = A2 + I2
	vmulpd	ymm7, ymm7, ymm1	;; B2 = I2 * cosine/sine
	vsubpd	ymm7, ymm7, ymm4	;; B2 = B2 - R2

	vmovapd	ymm1, [screg5+32]	;; cosine/sine for w^5n
	vmulpd	ymm4, ymm0, ymm1	;; A4 = R4 * cosine/sine
	vaddpd	ymm4, ymm4, ymm5	;; A4 = A4 + I4
	vmulpd	ymm5, ymm5, ymm1	;; B4 = I4 * cosine/sine
	vsubpd	ymm5, ymm5, ymm0	;; B4 = B4 - R4

	vmovapd	ymm1, [screg1+32] 	;; cosine/sine for w^n
	vmulpd	ymm0, ymm1, ymm10	;; A3 = R3 * cosine/sine for w^n
	vaddpd	ymm0, ymm0, ymm6	;; A3 = A3 + I3
	vmulpd	ymm6, ymm6, ymm1	;; B3 = I3 * cosine/sine for w^n
	vsubpd	ymm6, ymm6, ymm10	;; B3 = B3 - R3

	L1prefetchw srcreg+L1pd, L1pt

	vmovapd	ymm1, [screg2]		;; sine for w^2n
	vmulpd	ymm2, ymm2, ymm1	;; new R3 = A2 * sine
	vmulpd	ymm7, ymm7, ymm1	;; new R4 = B2 * sine
	vmovapd	ymm1, [screg5]		;; sine for w^5n
	vmulpd	ymm4, ymm4, ymm1	;; new R7 = A4 * sine
	vmulpd	ymm5, ymm5, ymm1	;; new R8 = B4 * sine
	vmovapd	ymm1, [screg1]		;; sine for w^n
	vmulpd	ymm0, ymm0, ymm1	;; new R5 = A3 * sine
	vmulpd	ymm6, ymm6, ymm1	;; new R6 = B3 * sine

	L1prefetchw srcreg+d1+L1pd, L1pt

	vsubpd	ymm1, ymm0, ymm4	;; new R6 = R5 - R7
	vaddpd	ymm0, ymm0, ymm4	;; new R5 = R5 + R7

	vsubpd	ymm4, ymm6, ymm5	;; new R8 = R6 - R8
	vaddpd	ymm6, ymm6, ymm5	;; new R7 = R6 + R8

	L1prefetchw srcreg+d2+L1pd, L1pt

	vaddpd	ymm5, ymm1, ymm4	;; R6 = R6 + R8
	vsubpd	ymm4, ymm4, ymm1	;; R8 = R8 - R6

	vmulpd	ymm5, ymm5, ymm14	;; R6 = R6 * square root of 1/2
	vmulpd	ymm4, ymm4, ymm14	;; R8 = R8 * square root of 1/2

	vsubpd	ymm1, ymm3, ymm7	;; R2 - R4 (new R4)
	vaddpd	ymm3, ymm3, ymm7	;; R2 + R4 (new R2)

	L1prefetchw srcreg+d2+d1+L1pd, L1pt

	vsubpd	ymm7, ymm1, ymm4	;; R4 - R8 (final R8)
	vaddpd	ymm1, ymm1, ymm4	;; R4 + R8 (final R4)

	vsubpd	ymm4, ymm3, ymm5	;; R2 - R6 (final R6)
	vaddpd	ymm3, ymm3, ymm5	;; R2 + R6 (final R2)

	ystore	[srcreg+d2+d1+32], ymm7	;; Save R8
	ystore	[srcreg+d2+d1], ymm1	;; Save R4

	vsubpd	ymm7, ymm9, ymm2	;; R1 - R3 (new R3)
	vaddpd	ymm5, ymm9, ymm2	;; R1 + R3 (new R1)

	vsubpd	ymm2, ymm7, ymm6	;; R3 - R7 (final R7)
	vaddpd	ymm7, ymm7, ymm6	;; R3 + R7 (final R3)

	vsubpd	ymm6, ymm5, ymm0	;; R1 - R5 (final R5)
	vaddpd	ymm5, ymm5, ymm0	;; R1 + R5 (final R1)

	ystore	[srcreg+d1+32], ymm4	;; Save R6
	ystore	[srcreg+d1], ymm3	;; Save R2

	ystore	[srcreg+d2+32], ymm2	;; Save R7
	ystore	[srcreg+d2], ymm7	;; Save R3

	ystore	[srcreg+32], ymm6	;; Save R5
	ystore	[srcreg], ymm5		;; Save R1

	bump	srcreg, srcinc
	bump	scregA, scincA
	bump	scregB, scincB
	ENDM

;; Haswell FMA3 version

IF (@INSTR(,%yarch,<FMA3>) NE 0)

yr4_s8r_unfft_cmn_preload MACRO
	vmovapd ymm15, YMM_ONE
	vmovapd ymm14, YMM_SQRTHALF
	ENDM

yr4_s8r_unfft_cmn MACRO srcreg,srcinc,d1,d2,screg1,screg2,screg5,scregA,scincA,scregB,scincB,maxrpt,L1pt,L1pd

	;; On input the 64-byte cache lines hold these data values:
	;;	0	8	16	24	4	12	20	28
	;;	1	...
	;;	2	...
	;;	3	...

	;; These are swizzled to:
	;;	0	1	2	3	4	5	6	7
	;;	8	...
	;;	16	...
	;;	24	...

	;; Swizzle using 8 shufpds and 8 perm2f128  (1 or 3 clocks each on port 5, throughput 1)

	vmovapd	ymm1, [srcreg]			;; R1
	vmovapd	ymm7, [srcreg+d1]		;; R2
	vshufpd	ymm0, ymm1, ymm7, 15		;; Shuffle R1 and R2 to create 8 9 24 25		; 1
	vshufpd	ymm1, ymm1, ymm7, 0		;; Shuffle R1 and R2 to create 0 1 16 17		; 2

	vmovapd	ymm2, [srcreg+d2]		;; R3
	vmovapd	ymm7, [srcreg+d2+d1]		;; R4
	vshufpd	ymm3, ymm2, ymm7, 15		;; Shuffle R3 and R4 to create 10 11 26 27		; 3
	vshufpd	ymm2, ymm2, ymm7, 0		;; Shuffle R3 and R4 to create 2 3 18 19		; 4

	vmovapd	ymm6, [srcreg+32]		;; I1
	vmovapd	ymm7, [srcreg+d1+32]		;; I2
	vshufpd	ymm5, ymm6, ymm7, 15		;; Shuffle I1 and I2 to create 12 13 28 29		; 5
	vshufpd	ymm6, ymm6, ymm7, 0		;; Shuffle I1 and I2 to create 4 5 20 21		; 6

	vmovapd	ymm4, [srcreg+d2+32]		;; I3
	vmovapd	ymm8, [srcreg+d2+d1+32]		;; I4
	vshufpd	ymm7, ymm4, ymm8, 15		;; Shuffle I3 and I4 to create 14 15 30 31		; 7
	vshufpd	ymm4, ymm4, ymm8, 0		;; Shuffle I3 and I4 to create 6 7 22 23		; 8

	yhigh128s ymm8, ymm0, ymm3		;; Shuffle R1/R2 hi and R3/R4 hi (24 25 26 27) (R4)	; 9-11
	yhigh128s ymm9, ymm5, ymm7		;; Shuffle I1/I2 hi and I3/I4 hi (28 29 30 31) (I4)	; 10-12
	vmovapd	ymm10, [screg5+32]		;; cosine/sine for w^5n

	ylow128s ymm0, ymm0, ymm3		;; Shuffle R1/R2 hi and R3/R4 hi (8 9 10 11) (R2)	; 11-13
	ylow128s ymm5, ymm5, ymm7		;; Shuffle I1/I2 hi and I3/I4 hi (12 13 14 15) (I2)	; 12-14
	L1prefetchw srcreg+L1pd, L1pt

	yhigh128s ymm3, ymm1, ymm2		;; Shuffle R1/R2 low and R3/R4 low (16 17 18 19) (R3)	; 13-15
	yhigh128s ymm7, ymm6, ymm4		;; Shuffle I1/I2 low and I3/I4 low (20 21 22 23) (I3)	; 14-16
	vmovapd	ymm11, [screg2+32]		;; cosine/sine for w^2n

	ylow128s ymm1, ymm1, ymm2		;; Shuffle R1/R2 low and R3/R4 low (0 1 2 3) (R1)	; 15-17
	ylow128s ymm6, ymm6, ymm4		;; Shuffle I1/I2 low and I3/I4 low (4 5 6 7) (I1 aka R2); 16-18

	yfmaddpd ymm2, ymm8, ymm10, ymm9	;; A4 = R4 * cosine/sine + I4				; 13-17
	yfmsubpd ymm9, ymm9, ymm10, ymm8	;; B4 = I4 * cosine/sine - R4				;	13-17
	vmovapd	ymm12, [screg1+32]		;; cosine/sine for w^n

	yfmaddpd ymm4, ymm0, ymm11, ymm5	;; A2 = R2 * cosine/sine + I2				; 15-19
	yfmsubpd ymm5, ymm5, ymm11, ymm0	;; B2 = I2 * cosine/sine - R2				;	15-19
	vmovapd	ymm13, [screg5]			;; sine for w^5n

	yfmaddpd ymm0, ymm3, ymm12, ymm7	;; A3 = R3 * cosine/sine for w^n + I3			; 17-21
	yfmsubpd ymm7, ymm7, ymm12, ymm3	;; B3 = I3 * cosine/sine for w^n - R3			;	17-21
	vmovapd	ymm10, [screg2]			;; sine for w^2n

	vmulpd	ymm2, ymm2, ymm13		;; new R7 = A4 * sine					; 18-22
	vmulpd	ymm9, ymm9, ymm13		;; new R8 = B4 * sine					;	18-22
	L1prefetchw srcreg+d1+L1pd, L1pt

	yfnmaddpd ymm3, ymm4, ymm10, ymm1	;; R1 - (R3 = A2 * sine) (new R3)			; 20-24
	yfmaddpd ymm4, ymm4, ymm10, ymm1	;; R1 + (R3 = A2 * sine) (new R1)			;	20-24
	vmovapd	ymm11, [screg1]			;; sine for w^n

	yfnmaddpd ymm8, ymm5, ymm10, ymm6	;; R2 - (R4 = B2 * sine) (new R4)			; 21-25
	yfmaddpd ymm5, ymm5, ymm10, ymm6	;; R2 + (R4 = B2 * sine) (new R2)			;	21-25
	L1prefetchw srcreg+d2+L1pd, L1pt

	yfmsubpd ymm1, ymm0, ymm11, ymm2	;; new R6 = (R5 = A3 * sine) - R7			; 23-27
	yfmsubpd ymm6, ymm7, ymm11, ymm9	;; new R8 = (R6 = B3 * sine) - R8			;	23-27

	yfmaddpd ymm0, ymm0, ymm11, ymm2	;; new R5 = (R5 = A3 * sine) + R7			; 24-28
	yfmaddpd ymm7, ymm7, ymm11, ymm9	;; new R7 = (R6 = B3 * sine) + R8			;	24-28
	L1prefetchw srcreg+d2+d1+L1pd, L1pt

	bump	scregA, scincA
	bump	scregB, scincB

	vaddpd	ymm2, ymm1, ymm6		;; R6 = R6 + R8						; 28-30
	vsubpd	ymm1, ymm6, ymm1		;; R8 = R8 - R6						; 29-31

	vsubpd	ymm6, ymm3, ymm7		;; R3 - R7 (final R7)					; 30-32
	yfmaddpd ymm3, ymm3, ymm15, ymm7	;; R3 + R7 (final R3)					;	29-33

	vsubpd	ymm7, ymm4, ymm0		;; R1 - R5 (final R5)					; 31-33
	yfmaddpd ymm4, ymm4, ymm15, ymm0	;; R1 + R5 (final R1)					;	30-34

	ystore	[srcreg+d2+32], ymm6		;; Save R7						;		33
	ystore	[srcreg+d2], ymm3		;; Save R3						;		34

	yfnmaddpd ymm0, ymm2, ymm14, ymm5	;; R2 - R6 * square root of 1/2 (final R6)		;	31-35
	yfmaddpd ymm2, ymm2, ymm14, ymm5	;; R2 + R6 * square root of 1/2 (final R2)		; 32-36

	ystore	[srcreg+32], ymm7		;; Save R5						;		34+1
	ystore	[srcreg], ymm4			;; Save R1						;		35+1

	yfnmaddpd ymm5, ymm1, ymm14, ymm8	;; R4 - R8 * square root of 1/2 (final R8)		;	32-36
	yfmaddpd ymm1, ymm1, ymm14, ymm8	;; R4 + R8 * square root of 1/2 (final R4)		; 33-37

	ystore	[srcreg+d1+32], ymm0		;; Save R6						;		36+1
	ystore	[srcreg+d1], ymm2		;; Save R2						;		37+1
	ystore	[srcreg+d2+d1+32], ymm5		;; Save R8						;		37+2
	ystore	[srcreg+d2+d1], ymm1		;; Save R4						;		38+2

	bump	srcreg, srcinc
	ENDM

ENDIF

ENDIF


;;
;; ******************************* eight-real-with-partial-normalization variants *************************************
;;
;; These macros are used in pass 1 of yr4dwpn two pass FFTs.  They are like the standard eight-real
;; FFT macros except that a normalization multiplier has been pre-applied to the sine multiplier.
;; Consequently, the forward FFT and inverse FFT use different sine multipliers.
;; Also, a normalization multiplier must be applied to the final R1/I1 value.
;;

;; Uses one combined sin/cos data ptr.  The first two sin/cos entries can be used by four complex
;; macros at the same FFT level.  The third and fourth sin/cos entries are used by this eight reals macro.
yr4_b4cl_csc_wpn_eight_reals_fft_preload MACRO
	ENDM
yr4_b4cl_csc_wpn_eight_reals_fft MACRO srcreg,srcinc,d1,d2,screg1,scinc1,screg2,scinc2,maxrpt,L1pt,L1pd
	;; screg1 is normalization weights for R1/I1
	;; screg2+0, screg2+64+0, screg2+128+0 is weighted sin/cos values for R2/I2 (w^2n)
	;; screg2+192+0, screg2+256+0, screg2+320+0 is weighted sin/cos values for R3/I3 (w^n)
	;; screg2+192+32, screg2+256+32, screg2+320+32 is weighted sin/cos values for R4/I4 (w^5n)

	vmovapd	ymm0, [srcreg+d2+d1]	;; R4
	vmovapd	ymm7, [srcreg+d2+d1+32]	;; R8
	vsubpd	ymm3, ymm0, ymm7	;; new R8 = R4 - R8			; 1-3
	vaddpd	ymm7, ymm0, ymm7	;; new R4 = R4 + R8			; 2-4

	vmovapd	ymm4, [srcreg+d1]	;; R2
	vmovapd	ymm5, [srcreg+d1+32]	;; R6
	vsubpd	ymm1, ymm4, ymm5	;; new R6 = R2 - R6			; 3-5
	vaddpd	ymm5, ymm4, ymm5	;; new R2 = R2 + R6			; 4-6

	vmulpd	ymm3, ymm3, YMM_SQRTHALF ;; R8 = R8 * square root		;  4-8
	vmulpd	ymm1, ymm1, YMM_SQRTHALF ;; R6 = R6 * square root		;  6-10

	vmovapd	ymm2, [srcreg]		;; R1
	vmovapd	ymm4, [srcreg+32]	;; R5
	vsubpd	ymm0, ymm2, ymm4	;; new R5 = R1 - R5			; 5-7
	vaddpd	ymm4, ymm2, ymm4	;; new R1 = R1 + R5			; 6-8

	vaddpd	ymm2, ymm5, ymm7	;; R2 + R4 (final I1, a.k.a 2nd real result) ; 7-9
	vsubpd	ymm5, ymm5, ymm7	;; R2 - R4 (final I2)			; 8-10

	vmovapd	ymm6, [srcreg+d2]	;; R3
	vbroadcastsd ymm7, Q [screg1]	;; normalization
	vmulpd	ymm2, ymm2, ymm7	;; I1 * normalization			;  10-14
	vmovapd	ymm7, [srcreg+d2+32]	;; R7
	ystore	[srcreg+32], ymm2	;; Save final I1			;  15

	vaddpd	ymm2, ymm6, ymm7	;; new R3 = R3 + R7			; 9-11
	vsubpd	ymm6, ymm6, ymm7	;; new R7 = R3 - R7			; 10-12

	L1prefetchw srcreg+L1pd, L1pt

	vsubpd	ymm7, ymm1, ymm3	;; R6 = R6 - R8 (Real part)		; 11-13
	vaddpd	ymm3, ymm1, ymm3	;; R8 = R6 + R8 (Imaginary part)	; 12-14

	vaddpd	ymm1, ymm4, ymm2	;; R1 + R3 (final R1)			; 13-15
	vsubpd	ymm4, ymm4, ymm2	;; R1 - R3 (final R2)			; 14-16

	L1prefetchw srcreg+d1+L1pd, L1pt

	vsubpd	ymm2, ymm0, ymm7	;; R5 - R6 (final R4)			; 15-17
	vaddpd	ymm0, ymm0, ymm7	;; R5 + R6 (final R3)			; 16-18

	vsubpd	ymm7, ymm6, ymm3	;; R7 - R8 (final I4)			; 17-19
	vaddpd	ymm6, ymm6, ymm3	;; R7 + R8 (final I3)			; 18-20

	vmulpd	ymm3, ymm4, [screg2+0]	;; A2 = R2 * cosine/sine for w^2n	;  17-21
	vsubpd	ymm3, ymm3, ymm5	;; A2 = A2 - I2				; 22-24
	vmulpd	ymm5, ymm5, [screg2+0]	;; B2 = I2 * cosine/sine for w^2n	;  18-22
	vaddpd	ymm5, ymm5, ymm4	;; B2 = B2 + R2				; 23-25

	L1prefetchw srcreg+d2+L1pd, L1pt

	vmulpd	ymm4, ymm2, [screg2+192+32] ;; A4 = R4 * cosine/sine for w^5n	;  19-23
	vsubpd	ymm4, ymm4, ymm7	;; A4 = A4 - I4				; 24-26
	vmulpd	ymm7, ymm7, [screg2+192+32] ;; B4 = I4 * cosine/sine for w^5n	;  20-24
	vaddpd	ymm7, ymm7, ymm2	;; B4 = B4 + R4				; 25-27

	vmulpd	ymm2, ymm0, [screg2+192+0] ;; A3 = R3 * cosine/sine for w^n	;  21-25
	vsubpd	ymm2, ymm2, ymm6	;; A3 = A3 - I3				; 26-28
	vmulpd	ymm6, ymm6, [screg2+192+0] ;; B3 = I3 * cosine/sine for w^n	;  22-26
	vaddpd	ymm6, ymm6, ymm0	;; B3 = B3 + R3				; 27-29

	vbroadcastsd ymm0, Q [screg1]	;; normalization
	vmulpd	ymm1, ymm1, ymm0	;; R1 * normalization			;  23-27

	L1prefetchw srcreg+d2+d1+L1pd, L1pt

	vmovapd	ymm0, [screg2+64+0]	;; sine * normalization for w^2n
	vmulpd	ymm3, ymm3, ymm0	;; A2 = A2 * sine (final R2)		;  25-29
	vmulpd	ymm5, ymm5, ymm0	;; B2 = B2 * sine (final I2)		;  26-30
	vmovapd	ymm0, [screg2+256+32]	;; sine * normalization for w^5n
	vmulpd	ymm4, ymm4, ymm0	;; A4 = A4 * sine (final R4)		;  27-31
	vmulpd	ymm7, ymm7, ymm0	;; B4 = B4 * sine (final I4)		;  28-32
	vmovapd	ymm0, [screg2+256+0]	;; sine * normalization for w^n
	vmulpd	ymm2, ymm2, ymm0	;; A3 = A3 * sine (final R3)		;  29-33
	vmulpd	ymm6, ymm6, ymm0	;; B3 = B3 * sine (final I3)		;  30-34

	ystore	[srcreg], ymm1		;; Save R1
	;;ystore [srcreg+32], ymm0	;; Save I1
	ystore	[srcreg+d1], ymm3	;; Save R2
	ystore	[srcreg+d1+32], ymm5	;; Save I2
	ystore	[srcreg+d2], ymm2	;; Save R3
	ystore	[srcreg+d2+32], ymm6	;; Save I3
	ystore	[srcreg+d2+d1], ymm4	;; Save R4
	ystore	[srcreg+d2+d1+32], ymm7	;; Save I4
	bump	srcreg, srcinc
	bump	screg1, scinc1
	bump	screg2, scinc2
	ENDM

;; 64-bit version

IFDEF X86_64

;; Uses one combined sin/cos data ptr.  The first two sin/cos entries can be used by four complex
;; macros at the same FFT level.  The third and fourth sin/cos entries are used by this eight reals macro.
yr4_b4cl_csc_wpn_eight_reals_fft_preload MACRO
	vmovapd	ymm15, YMM_SQRTHALF
	ENDM
yr4_b4cl_csc_wpn_eight_reals_fft MACRO srcreg,srcinc,d1,d2,screg1,scinc1,screg2,scinc2,maxrpt,L1pt,L1pd
	;; screg1 is normalization weights for R1/I1
	;; screg2+0, screg2+64+0, screg2+128+0 is weighted sin/cos values for R2/I2 (w^2n)
	;; screg2+192+0, screg2+256+0, screg2+320+0 is weighted sin/cos values for R3/I3 (w^n)
	;; screg2+192+32, screg2+256+32, screg2+320+32 is weighted sin/cos values for R4/I4 (w^5n)

	vmovapd	ymm1, [srcreg+d2+d1]	;; R4
	vmovapd	ymm2, [srcreg+d2+d1+32]	;; R8
	vsubpd	ymm0, ymm1, ymm2	;; new R8 = R4 - R8			; 1-3

	vmovapd	ymm5, [srcreg+d1]	;; R2
	vmovapd	ymm6, [srcreg+d1+32]	;; R6
	vsubpd	ymm4, ymm5, ymm6	;; new R6 = R2 - R6			; 2-4

	vaddpd	ymm1, ymm1, ymm2	;; new R4 = R4 + R8			; 3-5

	vaddpd	ymm5, ymm5, ymm6	;; new R2 = R2 + R6			; 4-6
	vmulpd	ymm0, ymm0, ymm15	;; R8 = R8 * square root		;  4-8

	vmovapd	ymm3, [srcreg]		;; R1
	vmovapd	ymm6, [srcreg+32]	;; R5
	vsubpd	ymm2, ymm3, ymm6	;; new R5 = R1 - R5			; 5-7
	vmulpd	ymm4, ymm4, ymm15	;; R6 = R6 * square root		;  5-9

	vaddpd	ymm3, ymm3, ymm6	;; new R1 = R1 + R5			; 6-8

	vmovapd	ymm6, [srcreg+d2]	;; R3
	vmovapd	ymm8, [srcreg+d2+32]	;; R7
	vaddpd	ymm7, ymm6, ymm8	;; new R3 = R3 + R7			; 7-9

	vsubpd	ymm6, ymm6, ymm8	;; new R7 = R3 - R7			; 8-10
	vmovapd	ymm8, [screg2+0]	;; cosine/sine for w^2n

	vsubpd	ymm9, ymm5, ymm1	;; R2 - R4 (final I2)			; 9-11
	vmovapd	ymm10, [screg2+192+32]	;; cosine/sine for w^5n

	vsubpd	ymm11, ymm4, ymm0	;; R6 = R6 - R8 (Real part)		; 10-12

	vaddpd	ymm4, ymm4, ymm0	;; R8 = R6 + R8 (Imaginary part)	; 11-13
	vmovapd	ymm12, [screg2+192+0]	;; cosine/sine for w^n

	vsubpd	ymm0, ymm3, ymm7	;; R1 - R3 (final R2)			; 12-14
	vmulpd	ymm13, ymm9, ymm8	;; B2 = I2 * cosine/sine for w^2n	;  12-16

	vaddpd	ymm5, ymm5, ymm1	;; R2 + R4 (final I1, a.k.a 2nd real result) ; 13-15
	vbroadcastsd ymm14, Q [screg1]	;; normalization

	vsubpd	ymm1, ymm2, ymm11	;; R5 - R6 (final R4)			; 14-16
	L1prefetchw srcreg+L1pd, L1pt

	vaddpd	ymm2, ymm2, ymm11	;; R5 + R6 (final R3)			; 15-17
	vmulpd	ymm8, ymm0, ymm8	;; A2 = R2 * cosine/sine for w^2n	;  15-19

	vsubpd	ymm11, ymm6, ymm4	;; R7 - R8 (final I4)			; 16-18
	vmulpd	ymm5, ymm5, ymm14	;; I1 * normalization			;  16-20
	L1prefetchw srcreg+d1+L1pd, L1pt

	vaddpd	ymm6, ymm6, ymm4	;; R7 + R8 (final I3)			; 17-19
	vmulpd	ymm4, ymm1, ymm10	;; A4 = R4 * cosine/sine for w^5n	;  17-21

	vaddpd	ymm3, ymm3, ymm7	;; R1 + R3 (final R1)			; 18-20
	vmulpd	ymm7, ymm2, ymm12	;; A3 = R3 * cosine/sine for w^n	;  18-22
	L1prefetchw srcreg+d2+L1pd, L1pt

	vaddpd	ymm13, ymm13, ymm0	;; B2 = B2 + R2				; 19-21
	vmulpd	ymm10, ymm11, ymm10	;; B4 = I4 * cosine/sine for w^5n	;  19-23
	vmovapd	ymm0, [screg2+64+0]	;; sine * normalization for w^2n

	vsubpd	ymm8, ymm8, ymm9	;; A2 = A2 - I2				; 20-22
	vmulpd	ymm12, ymm6, ymm12	;; B3 = I3 * cosine/sine for w^n	;  20-24
	vmovapd	ymm9, [screg2+256+32]	;; sine * normalization for w^5n

	vmulpd	ymm3, ymm3, ymm14	;; R1 * normalization			;  21-25
	ystore	[srcreg+32], ymm5	;; Save final I1			;  21

	vsubpd	ymm4, ymm4, ymm11	;; A4 = A4 - I4				; 22-24
	vmulpd	ymm13, ymm13, ymm0	;; B2 = B2 * sine (final I2)		;  22-26
	vmovapd	ymm14, [screg2+256+0]	;; sine * normalization for w^n

	vsubpd	ymm7, ymm7, ymm6	;; A3 = A3 - I3				; 23-25
	vmulpd	ymm8, ymm8, ymm0	;; A2 = A2 * sine (final R2)		;  23-27

	vaddpd	ymm10, ymm10, ymm1	;; B4 = B4 + R4				; 24-26
	L1prefetchw srcreg+d2+d1+L1pd, L1pt

	vaddpd	ymm12, ymm12, ymm2	;; B3 = B3 + R3				; 25-27
	vmulpd	ymm4, ymm4, ymm9	;; A4 = A4 * sine (final R4)		;  25-29

	vmulpd	ymm7, ymm7, ymm14	;; A3 = A3 * sine (final R3)		;  26-30
	ystore	[srcreg], ymm3		;; Save R1				; 26

	vmulpd	ymm10, ymm10, ymm9	;; B4 = B4 * sine (final I4)		;  27-31
	ystore	[srcreg+d1+32], ymm13	;; Save I2				; 27

	vmulpd	ymm12, ymm12, ymm14	;; B3 = B3 * sine (final I3)		;  28-32
	ystore	[srcreg+d1], ymm8	;; Save R2				; 28

	ystore	[srcreg+d2+d1], ymm4	;; Save R4				; 30
	ystore	[srcreg+d2], ymm7	;; Save R3				; 31
	ystore	[srcreg+d2+d1+32], ymm10 ;; Save I4				; 32
	ystore	[srcreg+d2+32], ymm12	;; Save I3				; 33

	bump	srcreg, srcinc
	bump	screg1, scinc1
	bump	screg2, scinc2
	ENDM

;; Haswell FMA3 version -- could be faster with unrolling

IF (@INSTR(,%yarch,<FMA3>) NE 0)

;; Uses one combined sin/cos data ptr.  The first two sin/cos entries can be used by four complex
;; macros at the same FFT level.  The third and fourth sin/cos entries are used by this eight reals macro.
yr4_b4cl_csc_wpn_eight_reals_fft_preload MACRO
	vmovapd	ymm14, YMM_ONE
	vmovapd	ymm15, YMM_SQRTHALF
	ENDM

;; Timed at 20.35 clocks
yr4_b4cl_csc_wpn_eight_reals_fft MACRO srcreg,srcinc,d1,d2,screg1,scinc1,screg2,scinc2,maxrpt,L1pt,L1pd
	;; screg1 is normalization weights for R1/I1
	;; screg2+0, screg2+64+0, screg2+128+0 is weighted sin/cos values for R2/I2 (w^2n)
	;; screg2+192+0, screg2+256+0, screg2+320+0 is weighted sin/cos values for R3/I3 (w^n)
	;; screg2+192+32, screg2+256+32, screg2+320+32 is weighted sin/cos values for R4/I4 (w^5n)

	vmovapd	ymm1, [srcreg+d2+d1]		;; R4
	vmovapd	ymm2, [srcreg+d2+d1+32]		;; R8
	yfmsubpd ymm0, ymm1, ymm14, ymm2	;; new R8 = R4 - R8			; 1-5
	vmovapd	ymm5, [srcreg+d1]		;; R2
	vmovapd	ymm6, [srcreg+d1+32]		;; R6
	yfmsubpd ymm4, ymm5, ymm14, ymm6	;; new R6 = R2 - R6			; 1-5

	yfmaddpd ymm1, ymm1, ymm14, ymm2	;; new R4 = R4 + R8			; 2-6
	yfmaddpd ymm5, ymm5, ymm14, ymm6	;; new R2 = R2 + R6			; 2-6

	vmovapd	ymm3, [srcreg]			;; R1
	vmovapd	ymm6, [srcreg+32]		;; R5
	yfmaddpd ymm2, ymm3, ymm14, ymm6	;; new R1 = R1 + R5			; 3-7
	yfmsubpd ymm3, ymm3, ymm14, ymm6	;; new R5 = R1 - R5			; 3-7

	vmovapd	ymm6, [srcreg+d2]		;; R3
	vmovapd	ymm8, [srcreg+d2+32]		;; R7
	yfmaddpd ymm7, ymm6, ymm14, ymm8	;; new R3 = R3 + R7			; 4-8
	yfmsubpd ymm6, ymm6, ymm14, ymm8	;; new R7 = R3 - R7			; 4-8

	yfmsubpd ymm8, ymm4, ymm14, ymm0	;; R6 = R6 - R8 (Real part)		; 6-10
	yfmaddpd ymm4, ymm4, ymm14, ymm0	;; R8 = R6 + R8 (Imaginary part)	; 6-10
	vmovapd	ymm0, [screg2+0]		;; cosine/sine for w^2n

	yfmaddpd ymm9, ymm5, ymm14, ymm1	;; R2 + R4 (final I1, a.k.a 2nd real result) ; 7-11
	yfmsubpd ymm5, ymm5, ymm14, ymm1	;; R2 - R4 (final I2)			; 7-11
	vbroadcastsd ymm1, Q [screg1]		;; normalization

	L1prefetchw srcreg+L1pd, L1pt

	yfmaddpd ymm10, ymm2, ymm14, ymm7	;; R1 + R3 (final R1)			; 9-13
	yfmsubpd ymm2, ymm2, ymm14, ymm7	;; R1 - R3 (final R2)			; 9-13
	vmovapd	ymm11, [screg2+192+32]		;; cosine/sine for w^5n

	yfnmaddpd ymm7, ymm8, ymm15, ymm3	;; R5 - R6 * square root (final R4)	; 11-15
	yfnmaddpd ymm12, ymm4, ymm15, ymm6	;; R7 - R8 * square root (final I4)	; 11-15
	vmovapd	ymm13, [screg2+192+0]		;; cosine/sine for w^n

	yfmaddpd ymm8, ymm8, ymm15, ymm3	;; R5 + R6 * square root (final R3)	; 12-16
	yfmaddpd ymm4, ymm4, ymm15, ymm6	;; R7 + R8 * square root (final I3)	; 12-16
	vmovapd	ymm6, [screg2+64+0]		;; sine * normalization for w^2n

	L1prefetchw srcreg+d1+L1pd, L1pt

	yfmsubpd ymm3, ymm2, ymm0, ymm5		;; A2 = R2 * cosine/sine for w^2n - I2	; 14-18
	yfmaddpd ymm5, ymm5, ymm0, ymm2		;; B2 = I2 * cosine/sine for w^2n + R2	; 14-18
	vmovapd	ymm0, [screg2+256+32]		;; sine * normalization for w^5n

	vmulpd	ymm10, ymm10, ymm1		;; R1 * normalization			; 15-19
	vmulpd	ymm9, ymm9, ymm1		;; I1 * normalization			; 15-19
	vmovapd	ymm2, [screg2+256+0]		;; sine * normalization for w^n

	yfmsubpd ymm1, ymm7, ymm11, ymm12	;; A4 = R4 * cosine/sine for w^5n - I4	; 16-20
	yfmaddpd ymm12, ymm12, ymm11, ymm7	;; B4 = I4 * cosine/sine for w^5n + R4	; 16-20

	yfmsubpd ymm11, ymm8, ymm13, ymm4	;; A3 = R3 * cosine/sine for w^n - I3	; 17-21
	yfmaddpd ymm4, ymm4, ymm13, ymm8	;; B3 = I3 * cosine/sine for w^n + R3	; 17-21

	L1prefetchw srcreg+d2+L1pd, L1pt

	vmulpd	ymm3, ymm3, ymm6		;; A2 = A2 * sine (final R2)		; 19-23
	vmulpd	ymm5, ymm5, ymm6		;; B2 = B2 * sine (final I2)		; 19-23

	ystore	[srcreg], ymm10			;; Save R1				; 20
	L1prefetchw srcreg+d2+d1+L1pd, L1pt

	vmulpd	ymm1, ymm1, ymm0		;; A4 = A4 * sine (final R4)		; 21-25
	vmulpd	ymm12, ymm12, ymm0		;; B4 = B4 * sine (final I4)		; 21-25
	ystore	[srcreg+32], ymm9		;; Save I1				; 20+1

	vmulpd	ymm11, ymm11, ymm2		;; A3 = A3 * sine (final R3)		; 22-26
	vmulpd	ymm4, ymm4, ymm2		;; B3 = B3 * sine (final I3)		; 22-26

	ystore	[srcreg+d1], ymm3		;; Save R2				; 24
	ystore	[srcreg+d1+32], ymm5		;; Save I2				; 24+1
	ystore	[srcreg+d2+d1], ymm1		;; Save R4				; 26
	ystore	[srcreg+d2+d1+32], ymm12	;; Save I4				; 26+1
	ystore	[srcreg+d2], ymm11		;; Save R3				; 27+1
	ystore	[srcreg+d2+32], ymm4		;; Save I3				; 27+2

	bump	srcreg, srcinc
	bump	screg1, scinc1
	bump	screg2, scinc2
	ENDM

ENDIF

ENDIF

;; Uses one combined sin/cos data ptr.  The first two sin/cos entries can be used by four complex
;; macros at the same FFT level.  The third and fourth sin/cos entries are used by this eight reals macro.
yr4_b4cl_csc_wpn_eight_reals_unfft_preload MACRO
	ENDM
yr4_b4cl_csc_wpn_eight_reals_unfft MACRO srcreg,srcinc,d1,d2,screg1,scinc1,screg2,scinc2,maxrpt,L1pt,L1pd
	;; screg1 is normalization weights for R1/I1
	;; screg2+0, screg2+64+0, screg2+128+0 is weighted sin/cos values for R2/I2 (w^2n)
	;; screg2+192+0, screg2+256+0, screg2+320+0 is weighted sin/cos values for R3/I3 (w^n)
	;; screg2+192+32, screg2+256+32, screg2+320+32 is weighted sin/cos values for R4/I4 (w^5n)

	vmovapd	ymm1, [screg2+0]	;; cosine/sine for w^2n
	vmovapd	ymm4, [srcreg+d1]	;; R2
	vmulpd	ymm2, ymm4, ymm1	;; A2 = R2 * cosine/sine
	vmovapd	ymm7, [srcreg+d1+32]	;; I2
	vaddpd	ymm2, ymm2, ymm7	;; A2 = A2 + I2
	vmulpd	ymm7, ymm7, ymm1	;; B2 = I2 * cosine/sine
	vsubpd	ymm7, ymm7, ymm4	;; B2 = B2 - R2

	vmovapd	ymm1, [screg2+192+32]	;; cosine/sine for w^5n
	vmovapd	ymm0, [srcreg+d2+d1]	;; R4
	vmulpd	ymm4, ymm0, ymm1	;; A4 = R4 * cosine/sine
	vmovapd	ymm5, [srcreg+d2+d1+32]	;; I4
	vaddpd	ymm4, ymm4, ymm5	;; A4 = A4 + I4
	vmulpd	ymm5, ymm5, ymm1	;; B4 = I4 * cosine/sine
	vsubpd	ymm5, ymm5, ymm0	;; B4 = B4 - R4

	vmovapd	ymm1, [screg2+192+0] 	;; cosine/sine for w^n
	vmovapd	ymm3, [srcreg+d2]	;; R3
	vmulpd	ymm0, ymm1, ymm3	;; A3 = R3 * cosine/sine for w^n
	vmovapd	ymm6, [srcreg+d2+32]	;; I3
	vaddpd	ymm0, ymm0, ymm6	;; A3 = A3 + I3
	vmulpd	ymm6, ymm6, ymm1	;; B3 = I3 * cosine/sine for w^n
	vsubpd	ymm6, ymm6, ymm3	;; B3 = B3 - R3

	L1prefetchw srcreg+L1pd, L1pt

	vmovapd	ymm1, [screg2+128+0]	;; sine * normalization_inverse for w^2n
	vmulpd	ymm2, ymm2, ymm1	;; new R3 = A2 * sine
	vmulpd	ymm7, ymm7, ymm1	;; new R4 = B2 * sine
	vmovapd	ymm1, [screg2+320+32]	;; sine * normalization_inverse for w^5n
	vmulpd	ymm4, ymm4, ymm1	;; new R7 = A4 * sine
	vmulpd	ymm5, ymm5, ymm1	;; new R8 = B4 * sine
	vmovapd	ymm1, [screg2+320+0]	;; sine * normalization_inverse for w^n
	vmulpd	ymm0, ymm0, ymm1	;; new R5 = A3 * sine
	vmulpd	ymm6, ymm6, ymm1	;; new R6 = B3 * sine

	L1prefetchw srcreg+d1+L1pd, L1pt

	vsubpd	ymm1, ymm0, ymm4	;; new R6 = R5 - R7
	vaddpd	ymm0, ymm0, ymm4	;; new R5 = R5 + R7

	vsubpd	ymm4, ymm6, ymm5	;; new R8 = R6 - R8
	vaddpd	ymm6, ymm6, ymm5	;; new R7 = R6 + R8

	vbroadcastsd ymm3, Q [screg1]	;; normalization_inverse
	vmulpd	ymm3, ymm3, [srcreg+32]	;; R2 * normalization_inverse

	vaddpd	ymm5, ymm1, ymm4	;; R6 = R6 + R8
	vsubpd	ymm4, ymm4, ymm1	;; R8 = R8 - R6

	L1prefetchw srcreg+d2+L1pd, L1pt

	vmulpd	ymm5, ymm5, YMM_SQRTHALF ;; R6 = R6 * square root of 1/2
	vmulpd	ymm4, ymm4, YMM_SQRTHALF ;; R8 = R8 * square root of 1/2

	vsubpd	ymm1, ymm3, ymm7	;; R2 - R4 (new R4)
	vaddpd	ymm3, ymm3, ymm7	;; R2 + R4 (new R2)

	L1prefetchw srcreg+d2+d1+L1pd, L1pt

	vsubpd	ymm7, ymm1, ymm4	;; R4 - R8 (final R8)
	vaddpd	ymm1, ymm1, ymm4	;; R4 + R8 (final R4)

	vsubpd	ymm4, ymm3, ymm5	;; R2 - R6 (final R6)
	vaddpd	ymm3, ymm3, ymm5	;; R2 + R6 (final R2)

	vbroadcastsd ymm5, Q [screg1]	;; normalization_inverse
	vmulpd	ymm5, ymm5, [srcreg]	;; R1 * normalization_inverse

	ystore	[srcreg+d2+d1+32], ymm7	;; Save R8
	ystore	[srcreg+d2+d1], ymm1	;; Save R4

	vsubpd	ymm7, ymm5, ymm2	;; R1 - R3 (new R3)
	vaddpd	ymm5, ymm5, ymm2	;; R1 + R3 (new R1)

	vsubpd	ymm2, ymm7, ymm6	;; R3 - R7 (final R7)
	vaddpd	ymm7, ymm7, ymm6	;; R3 + R7 (final R3)

	vsubpd	ymm6, ymm5, ymm0	;; R1 - R5 (final R5)
	vaddpd	ymm5, ymm5, ymm0	;; R1 + R5 (final R1)

	ystore	[srcreg+d1+32], ymm4	;; Save R6
	ystore	[srcreg+d1], ymm3	;; Save R2

	ystore	[srcreg+d2+32], ymm2	;; Save R7
	ystore	[srcreg+d2], ymm7	;; Save R3

	ystore	[srcreg+32], ymm6	;; Save R5
	ystore	[srcreg], ymm5		;; Save R1

	bump	srcreg, srcinc
	bump	screg1, scinc1
	bump	screg2, scinc2
	ENDM

;; 64-bit version

IFDEF X86_64

;; Uses one combined sin/cos data ptr.  The first two sin/cos entries can be used by four complex
;; macros at the same FFT level.  The third and fourth sin/cos entries are used by this eight reals macro.
yr4_b4cl_csc_wpn_eight_reals_unfft_preload MACRO
	vmovapd	ymm15, YMM_SQRTHALF
	ENDM
yr4_b4cl_csc_wpn_eight_reals_unfft MACRO srcreg,srcinc,d1,d2,screg1,scinc1,screg2,scinc2,maxrpt,L1pt,L1pd
	;; screg1 is normalization weights for R1/I1
	;; screg2+0, screg2+64+0, screg2+128+0 is weighted sin/cos values for R2/I2 (w^2n)
	;; screg2+192+0, screg2+256+0, screg2+320+0 is weighted sin/cos values for R3/I3 (w^n)
	;; screg2+192+32, screg2+256+32, screg2+320+32 is weighted sin/cos values for R4/I4 (w^5n)

	vmovapd	ymm10, [screg2+192+0] 	;; cosine/sine for w^n
	vmovapd	ymm1, [srcreg+d2]	;; R3
	vmulpd	ymm0, ymm1, ymm10	;; A3 = R3 * cosine/sine for w^n	; 1-5

	vmovapd	ymm11, [screg2+192+32]	;; cosine/sine for w^5n
	vmovapd	ymm3, [srcreg+d2+d1]	;; R4
	vmulpd	ymm2, ymm3, ymm11	;; A4 = R4 * cosine/sine		; 2-6

	vmovapd	ymm5, [srcreg+d2+32]	;; I3
	vmulpd	ymm4, ymm5, ymm10	;; B3 = I3 * cosine/sine for w^n	; 3-7

	vmovapd	ymm7, [srcreg+d2+d1+32]	;; I4
	vmulpd	ymm6, ymm7, ymm11	;; B4 = I4 * cosine/sine		; 4-8

	vmovapd	ymm12, [screg2+0]	;; cosine/sine for w^2n
	vmovapd	ymm9, [srcreg+d1]	;; R2
	vmulpd	ymm8, ymm9, ymm12	;; A2 = R2 * cosine/sine		; 5-9

	vaddpd	ymm0, ymm0, ymm5	;; A3 = A3 + I3				; 6-8
	vmovapd	ymm11, [srcreg+d1+32]	;; I2
	vmulpd	ymm10, ymm11, ymm12	;; B2 = I2 * cosine/sine		; 6-10

	vaddpd	ymm2, ymm2, ymm7	;; A4 = A4 + I4				; 7-9
	vbroadcastsd ymm13, Q [screg1]	;; normalization_inverse
	vmulpd	ymm5, ymm13, [srcreg]	;; R1 * normalization_inverse		; 7-11

	vsubpd	ymm4, ymm4, ymm1	;; B3 = B3 - R3				; 8-10
	vmulpd	ymm7, ymm13, [srcreg+32];; R2 * normalization_inverse		; 8-12

	vsubpd	ymm6, ymm6, ymm3	;; B4 = B4 - R4				; 9-11
	vmovapd	ymm1, [screg2+320+0]	;; sine * normalization_inverse for w^n
	vmulpd	ymm0, ymm0, ymm1	;; new R5 = A3 * sine			; 9-13

	vaddpd	ymm8, ymm8, ymm11	;; A2 = A2 + I2				; 10-12
	vmovapd	ymm3, [screg2+320+32]	;; sine * normalization_inverse for w^5n
	vmulpd	ymm2, ymm2, ymm3	;; new R7 = A4 * sine			; 10-14

	vsubpd	ymm10, ymm10, ymm9	;; B2 = B2 - R2				; 11-13
	vmulpd	ymm4, ymm4, ymm1	;; new R6 = B3 * sine			; 11-15
	vmovapd	ymm9, [screg2+128+0]	;; sine * normalization_inverse for w^2n

	vmulpd	ymm6, ymm6, ymm3	;; new R8 = B4 * sine			; 12-16

	vmulpd	ymm8, ymm8, ymm9	;; new R3 = A2 * sine			; 13-17
	L1prefetchw srcreg+L1pd, L1pt

	vmulpd	ymm10, ymm10, ymm9	;; new R4 = B2 * sine			; 14-18

	vsubpd	ymm1, ymm0, ymm2	;; new R6 = R5 - R7			; 15-17

	vaddpd	ymm0, ymm0, ymm2	;; new R5 = R5 + R7			; 16-18
	L1prefetchw srcreg+d1+L1pd, L1pt

	vsubpd	ymm3, ymm4, ymm6	;; new R8 = R6 - R8			; 17-19

	vaddpd	ymm4, ymm4, ymm6	;; new R7 = R6 + R8			; 18-20

	vaddpd	ymm9, ymm5, ymm8	;; R1 + R3 (new R1)			; 19-21
	L1prefetchw srcreg+d2+L1pd, L1pt

	vaddpd	ymm2, ymm1, ymm3	;; R6 = R6 + R8				; 20-22

	vsubpd	ymm1, ymm3, ymm1	;; R8 = R8 - R6				; 21-23

	vsubpd	ymm5, ymm5, ymm8	;; R1 - R3 (new R3)			; 22-24
	L1prefetchw srcreg+d2+d1+L1pd, L1pt

	vaddpd	ymm6, ymm7, ymm10	;; R2 + R4 (new R2)			; 23-25
	vmulpd	ymm2, ymm2, ymm15	;; R6 = R6 * square root of 1/2		; 23-27

	vsubpd	ymm7, ymm7, ymm10	;; R2 - R4 (new R4)			; 24-26
	vmulpd	ymm1, ymm1, ymm15	;; R8 = R8 * square root of 1/2		; 24-28

	vsubpd	ymm3, ymm9, ymm0	;; R1 - R5 (final R5)			; 25-27

	vaddpd	ymm9, ymm9, ymm0	;; R1 + R5 (final R1)			; 26-28

	vsubpd	ymm8, ymm5, ymm4	;; R3 - R7 (final R7)			; 27-29

	vaddpd	ymm5, ymm5, ymm4	;; R3 + R7 (final R3)			; 28-30
	ystore	[srcreg+32], ymm3	;; Save R5				; 28

	vsubpd	ymm0, ymm6, ymm2	;; R2 - R6 (final R6)			; 29-31
	ystore	[srcreg], ymm9		;; Save R1				; 29

	vaddpd	ymm6, ymm6, ymm2	;; R2 + R6 (final R2)			; 30-32
	ystore	[srcreg+d2+32], ymm8	;; Save R7				; 30

	vsubpd	ymm4, ymm7, ymm1	;; R4 - R8 (final R8)			; 31-33
	ystore	[srcreg+d2], ymm5	;; Save R3				; 31

	vaddpd	ymm7, ymm7, ymm1	;; R4 + R8 (final R4)			; 32-34
	ystore	[srcreg+d1+32], ymm0	;; Save R6				; 32

	ystore	[srcreg+d1], ymm6	;; Save R2				; 33
	ystore	[srcreg+d2+d1+32], ymm4	;; Save R8				; 34
	ystore	[srcreg+d2+d1], ymm7	;; Save R4				; 35

	bump	srcreg, srcinc
	bump	screg1, scinc1
	bump	screg2, scinc2
	ENDM

;; Haswell FMA3 version -- could be faster with unrolling

IF (@INSTR(,%yarch,<FMA3>) NE 0)

;; Uses one combined sin/cos data ptr.  The first two sin/cos entries can be used by four complex
;; macros at the same FFT level.  The third and fourth sin/cos entries are used by this eight reals macro.
yr4_b4cl_csc_wpn_eight_reals_unfft_preload MACRO
	vmovapd	ymm14, YMM_ONE
	vmovapd	ymm15, YMM_SQRTHALF
	ENDM

;; Timed at 19.35 clocks
yr4_b4cl_csc_wpn_eight_reals_unfft MACRO srcreg,srcinc,d1,d2,screg1,scinc1,screg2,scinc2,maxrpt,L1pt,L1pd
	;; screg1 is normalization weights for R1/I1
	;; screg2+0, screg2+64+0, screg2+128+0 is weighted sin/cos values for R2/I2 (w^2n)
	;; screg2+192+0, screg2+256+0, screg2+320+0 is weighted sin/cos values for R3/I3 (w^n)
	;; screg2+192+32, screg2+256+32, screg2+320+32 is weighted sin/cos values for R4/I4 (w^5n)

	vmovapd	ymm3, [screg2+192+0]		;; cosine/sine for w^n
	vmovapd	ymm2, [srcreg+d2]		;; R3
	vmovapd	ymm1, [srcreg+d2+32]		;; I3
	yfmaddpd ymm0, ymm2, ymm3, ymm1		;; A3 = R3 * cosine/sine for w^n + I3	; 1-5
	yfmsubpd ymm1, ymm1, ymm3, ymm2		;; B3 = I3 * cosine/sine for w^n - R3	; 1-5

	vmovapd	ymm5, [screg2+0]		;; cosine/sine for w^2n
	vmovapd	ymm4, [srcreg+d1]		;; R2
	vmovapd	ymm3, [srcreg+d1+32]		;; I2
	yfmaddpd ymm2, ymm4, ymm5, ymm3		;; A2 = R2 * cosine/sine + I2		; 2-6
	yfmsubpd ymm3, ymm3, ymm5, ymm4		;; B2 = I2 * cosine/sine - R2		; 2-6

	vbroadcastsd ymm6, Q [screg1]		;; normalization_inverse
	vmulpd	ymm4, ymm6, [srcreg]		;; R1 * normalization_inverse		; 3-7
	vmulpd	ymm5, ymm6, [srcreg+32]		;; R2 * normalization_inverse		; 3-7

	vmovapd	ymm9, [screg2+192+32]		;; cosine/sine for w^5n
	vmovapd	ymm8, [srcreg+d2+d1]		;; R4
	vmovapd	ymm7, [srcreg+d2+d1+32]		;; I4
	yfmaddpd ymm6, ymm8, ymm9, ymm7		;; A4 = R4 * cosine/sine + I4		; 4-8
	yfmsubpd ymm7, ymm7, ymm9, ymm8		;; B4 = I4 * cosine/sine - R4		; 4-8

	vmovapd	ymm9, [screg2+320+0]		;; sine * normalization_inverse for w^n
	vmulpd	ymm0, ymm0, ymm9		;; new R5 = A3 * sine			; 6-10
	vmulpd	ymm1, ymm1, ymm9		;; new R6 = B3 * sine			; 6-10

	vmovapd	ymm9, [screg2+128+0]		;; sine * normalization_inverse for w^2n
	yfmaddpd ymm10, ymm2, ymm9, ymm4	;; R1 + (R3 = A2 * sine) (new R1)	; 8-12
	yfnmaddpd ymm2, ymm2, ymm9, ymm4	;; R1 - (R3 = A2 * sine) (new R3)	; 8-12

	yfmaddpd ymm4, ymm3, ymm9, ymm5		;; R2 + (R4 = B2 * sine) (new R2)	; 9-13
	yfnmaddpd ymm3, ymm3, ymm9, ymm5	;; R2 - (R4 = B2 * sine) (new R4)	; 9-13

	vmovapd	ymm9, [screg2+320+32]		;; sine * normalization_inverse for w^5n
	yfnmaddpd ymm5, ymm6, ymm9, ymm0	;; new R6 = R5 - (R7 = A4 * sine)	; 11-15
	yfnmaddpd ymm11, ymm7, ymm9, ymm1	;; new R8 = R6 - (R8 = B4 * sine)	; 11-15
	L1prefetchw srcreg+L1pd, L1pt

	yfmaddpd ymm6, ymm6, ymm9, ymm0		;; new R5 = R5 + (R7 = A4 * sine)	; 12-16
	yfmaddpd ymm7, ymm7, ymm9, ymm1		;; new R7 = R6 + (R8 = B4 * sine)	; 12-16
	L1prefetchw srcreg+d1+L1pd, L1pt

	yfmaddpd ymm0, ymm5, ymm14, ymm11	;; R6 = R6 + R8				; 16-20
	yfmsubpd ymm11, ymm11, ymm14, ymm5	;; R8 = R8 - R6				; 16-20
	L1prefetchw srcreg+d2+L1pd, L1pt

	yfmsubpd ymm9, ymm10, ymm14, ymm6	;; R1 - R5 (final R5)			; 17-21
	yfmaddpd ymm10, ymm10, ymm14, ymm6	;; R1 + R5 (final R1)			; 17-21

	yfmsubpd ymm1, ymm2, ymm14, ymm7	;; R3 - R7 (final R7)			; 18-22
	yfmaddpd ymm2, ymm2, ymm14, ymm7	;; R3 + R7 (final R3)			; 18-22
	L1prefetchw srcreg+d2+d1+L1pd, L1pt

	yfnmaddpd ymm6, ymm0, ymm15, ymm4	;; R2 - R6 * square root of 1/2 (final R6) ; 21-25
	yfmaddpd ymm0, ymm0, ymm15, ymm4	;; R2 + R6 * square root of 1/2 (final R2) ; 21-25

	yfnmaddpd ymm7, ymm11, ymm15, ymm3	;; R4 - R8 * square root of 1/2 (final R8) ; 22-26
	yfmaddpd ymm3, ymm11, ymm15, ymm3	;; R4 + R8 * square root of 1/2 (final R4) ; 22-26

	ystore	[srcreg+32], ymm9	;; Save R5				; 22
	ystore	[srcreg], ymm10		;; Save R1				; 22+1
	ystore	[srcreg+d2+32], ymm1	;; Save R7				; 23+1
	ystore	[srcreg+d2], ymm2	;; Save R3				; 23+2
	ystore	[srcreg+d1+32], ymm6	;; Save R6				; 26
	ystore	[srcreg+d1], ymm0	;; Save R2				; 26+1
	ystore	[srcreg+d2+d1+32], ymm7	;; Save R8				; 27+1
	ystore	[srcreg+d2+d1], ymm3	;; Save R4				; 27+2

	bump	srcreg, srcinc
	bump	screg1, scinc1
	bump	screg2, scinc2
	ENDM

ENDIF

ENDIF


;;
;; ************************************* eight-reals-fft4 variants ******************************************
;;
;; These macros are used in the last levels of pass 1.  Four sin/cos multipliers are needed to
;; finish off the partial sin/cos multiplies that were done in the first levels of pass 1.
;; FFTs of type r4delay and r4dwpn do this to reduce memory usage at the cost of some extra
;; complex multiplies.

;; Used in last levels of pass 1 (split premultiplier, delay, and dwpn cases).  Swizzling.
;; In this particular case, we know that the sin/cos multipliers for R1/I1 are one.  Thus
;; we can get by with only 3 sin/cos multiplies.
yr4_sg4cl_2sc_eight_reals_fft4_preload MACRO
	ENDM
yr4_sg4cl_2sc_eight_reals_fft4 MACRO srcreg,srcinc,d1,d2,dstreg,dstinc,e1,e2,screg1,scinc1,screg2,scinc2,maxrpt,L1pt,L1pd

	;; On input the 64-byte cache lines hold these data values:
	;;	0	+1K	+1K	+1K	4	+1K	+1K	+1K
	;;	1	...
	;;	2	...
	;;	3	...

	;; These are swizzled to:
	;;	0	+1	+1	+1	4K	+1	+1	+1
	;;	1K	...
	;;	2K	...
	;;	3K	...

	;; Swizzle using 8 shufpds and 8 perm2f128  (1 or 3 clocks each on port 5, throughput 1)

	vmovapd	ymm1, [srcreg]			;; R1
	vmovapd	ymm7, [srcreg+d1]		;; R2
	vshufpd	ymm0, ymm1, ymm7, 15		;; Shuffle R1 and R2 to create R1/R2 hi			; 1
	vshufpd	ymm1, ymm1, ymm7, 0		;; Shuffle R1 and R2 to create R1/R2 low		; 2

	vmovapd	ymm2, [srcreg+d2]		;; R3
	vmovapd	ymm7, [srcreg+d2+d1]		;; R4
	vshufpd	ymm3, ymm2, ymm7, 15		;; Shuffle R3 and R4 to create R3/R4 hi			; 3
	vshufpd	ymm2, ymm2, ymm7, 0		;; Shuffle R3 and R4 to create R3/R4 low		; 4

	ylow128s ymm4, ymm0, ymm3		;; Shuffle R1/R2 hi and R3/R4 hi (new R2)		; 5-6
	yhigh128s ymm0, ymm0, ymm3		;; Shuffle R1/R2 hi and R3/R4 hi (new R4)		; 6-7

	ylow128s ymm3, ymm1, ymm2		;; Shuffle R1/R2 low and R3/R4 low (new R1)		; 7-8
	yhigh128s ymm1, ymm1, ymm2		;; Shuffle R1/R2 low and R3/R4 low (new R3)		; 8-9

	ystore	[dstreg], ymm1			;; Temporarily save new R3

	vmovapd	ymm6, [srcreg+32]		;; R5
	vmovapd	ymm2, [srcreg+d1+32]		;; R6
	vshufpd	ymm5, ymm6, ymm2, 15		;; Shuffle R5 and R6 to create R5/R6 hi
	vshufpd	ymm6, ymm6, ymm2, 0		;; Shuffle R5 and R6 to create R5/R6 low

	vmovapd	ymm2, [srcreg+d2+32]		;; R7
	vmovapd	ymm7, [srcreg+d2+d1+32]		;; R8
	vshufpd	ymm1, ymm2, ymm7, 15		;; Shuffle R7 and R8 to create R7/R8 hi
	vshufpd	ymm2, ymm2, ymm7, 0		;; Shuffle R7 and R8 to create R7/R8 low

	ylow128s ymm7, ymm5, ymm1		;; Shuffle R5/R6 hi and R7/R8 hi (new R6)
	yhigh128s ymm5, ymm5, ymm1		;; Shuffle R5/R6 hi and R7/R8 hi (new R8)

	L1prefetch srcreg+L1pd, L1pt
	L1prefetchw dstreg+L1pd, L1pt - L1PREFETCH_DEST_NONE

	ylow128s ymm1, ymm6, ymm2		;; Shuffle R5/R6 low and R7/R8 low (new R5)
	yhigh128s ymm6, ymm6, ymm2		;; Shuffle R5/R6 low and R7/R8 low (new R7)

	vsubpd	ymm2, ymm0, ymm5	;; new R8 = R4 - R8			; 1-3
	vaddpd	ymm0, ymm0, ymm5	;; new R4 = R4 + R8			; 2-4

	vsubpd	ymm5, ymm4, ymm7	;; new R6 = R2 - R6			; 3-5
	vaddpd	ymm4, ymm4, ymm7	;; new R2 = R2 + R6			; 4-6

	vmulpd	ymm2, ymm2, YMM_SQRTHALF ;; R8 = R8 * square root		;  4-8
	vmulpd	ymm5, ymm5, YMM_SQRTHALF ;; R6 = R6 * square root		;  6-10

	vsubpd	ymm7, ymm3, ymm1	;; new R5 = R1 - R5			; 5-7
	vaddpd	ymm3, ymm3, ymm1	;; new R1 = R1 + R5			; 6-8

	L1prefetch srcreg+d1+L1pd, L1pt
	L1prefetchw dstreg+e1+L1pd, L1pt - L1PREFETCH_DEST_NONE

	vaddpd	ymm1, ymm4, ymm0	;; R2 + R4 (final I1, a.k.a 2nd real result) ; 7-9
	vsubpd	ymm4, ymm4, ymm0	;; R2 - R4 (final I2)			; 8-10

	ystore	[dstreg+32], ymm1	;; Save final I1			; 10

	vmovapd	ymm0, [dstreg]		;; Reload saved new R3
	vaddpd	ymm1, ymm0, ymm6	;; new R3 = R3 + R7			; 9-11
	vsubpd	ymm0, ymm0, ymm6	;; new R7 = R3 - R7			; 10-12

	vsubpd	ymm6, ymm5, ymm2	;; R6 = R6 - R8 (Real part)		; 11-13
	vaddpd	ymm5, ymm5, ymm2	;; R8 = R6 + R8 (Imaginary part)	; 12-14

	L1prefetch srcreg+d2+L1pd, L1pt
	L1prefetchw dstreg+e2+L1pd, L1pt - L1PREFETCH_DEST_NONE

	vaddpd	ymm2, ymm3, ymm1	;; R1 + R3 (final R1)			; 13-15
	vsubpd	ymm3, ymm3, ymm1	;; R1 - R3 (final R2)			; 14-16

	vsubpd	ymm1, ymm7, ymm6	;; R5 - R6 (final R4)			; 15-17
	vaddpd	ymm7, ymm7, ymm6	;; R5 + R6 (final R3)			; 16-18

	L1prefetch srcreg+d2+d1+L1pd, L1pt
	L1prefetchw dstreg+e2+e1+L1pd, L1pt - L1PREFETCH_DEST_NONE

	vsubpd	ymm6, ymm0, ymm5	;; R7 - R8 (final I4)			; 17-19
	vaddpd	ymm0, ymm0, ymm5	;; R7 + R8 (final I3)			; 18-20

	ystore	[dstreg], ymm2		;; Save final R1			; 16

	;; screg1 is sin/cos values for R1/I1
	;; screg1+64 is sin/cos values for R2/I2 (w^2n)
	;; screg2 is sin/cos values for R3/I3 (w^n)
	;; screg2+64 is sin/cos values for R4/I4 (w^5n)

	vmovapd	ymm2, [screg2+32]		;; cosine/sine
	vmulpd	ymm5, ymm7, ymm2		;; A3 = R3 * cosine/sine
	vsubpd	ymm5, ymm5, ymm0		;; A3 = A3 - I3
	vmulpd	ymm0, ymm0, ymm2		;; B3 = I3 * cosine/sine
	vaddpd	ymm0, ymm0, ymm7		;; B3 = B3 + R3

	vmovapd	ymm2, [(screg1+64)+32]		;; cosine/sine
	vmulpd	ymm7, ymm3, ymm2		;; A2 = R2 * cosine/sine
	vsubpd	ymm7, ymm7, ymm4		;; A2 = A2 - I2
	vmulpd	ymm4, ymm4, ymm2		;; B2 = I2 * cosine/sine
	vaddpd	ymm4, ymm4, ymm3		;; B2 = B2 + R2

	vmovapd	ymm2, [(screg2+64)+32]		;; cosine/sine
	vmulpd	ymm3, ymm1, ymm2		;; A4 = R4 * cosine/sine
	vsubpd	ymm3, ymm3, ymm6		;; A4 = A4 - I4
	vmulpd	ymm6, ymm6, ymm2		;; B4 = I4 * cosine/sine
	vaddpd	ymm6, ymm6, ymm1		;; B4 = B4 + R4

	vmovapd	ymm2, [screg2]
	vmulpd	ymm5, ymm5, ymm2		;; A3 = A3 * sine (final R3)
	vmulpd	ymm0, ymm0, ymm2		;; B3 = B3 * sine (final I3)
	vmovapd	ymm2, [(screg1+64)]
	vmulpd	ymm7, ymm7, ymm2		;; A2 = A2 * sine (final R2)
	vmulpd	ymm4, ymm4, ymm2		;; B2 = B2 * sine (final I2)

	ystore	[dstreg+e2], ymm5		;; Save R3
	ystore	[dstreg+e2+32], ymm0		;; Save I3

	vmulpd	ymm3, ymm3, [(screg2+64)]	;; A4 = A4 * sine (final R4)
	vmulpd	ymm6, ymm6, [(screg2+64)]	;; B4 = B4 * sine (final I4)

	ystore	[dstreg+e1], ymm7		;; Save R2
	ystore	[dstreg+e1+32], ymm4		;; Save I2
	ystore	[dstreg+e2+e1], ymm3		;; Save R4
	ystore	[dstreg+e2+e1+32], ymm6		;; Save I4

	bump	srcreg, srcinc
	bump	dstreg, dstinc
	bump	screg1, scinc1
	bump	screg2, scinc2
	ENDM

;;
;; ************************************* eight-real-unfft4 variants ******************************************
;;

;; Used in last levels of pass 1 (r4delay and r4dwpn cases).  Swizzling.
;; In this particular case, we know that the sin/cos multipliers for R1/I1 are one.  Thus
;; we can get by with only 3 sin/cos multiplies.
yr4_sg4cl_2sc_eight_reals_unfft4 MACRO srcreg,srcinc,d1,d2,dstreg,dstinc,e1,e2,screg1,scinc1,screg2,scinc2,maxrpt,L1pt,L1pd

	;; screg1 is sin/cos values for R1/I1
	;; screg1+64 is sin/cos values for R2/I2 (w^2n)
	;; screg2 is sin/cos values for R3/I3 (w^n)
	;; screg2+64 is sin/cos values for R4/I4 (w^5n)

	vmovapd	ymm5, [(screg1+64)+32]		;; cosine/sine
	vmovapd	ymm4, [srcreg+d1]		;; R2
	vmulpd	ymm2, ymm4, ymm5		;; A2 = R2 * cosine/sine
	vmovapd	ymm3, [srcreg+d1+32]		;; I2
	vaddpd	ymm2, ymm2, ymm3		;; A2 = A2 + I2
	vmulpd	ymm3, ymm3, ymm5		;; B2 = I2 * cosine/sine
	vsubpd	ymm3, ymm3, ymm4		;; B2 = B2 - R2

	vmovapd	ymm0, [screg2+32]		;; cosine/sine
	vmovapd	ymm4, [srcreg+d2]		;; R3
	vmulpd	ymm5, ymm4, ymm0		;; A3 = R3 * cosine/sine
	vmovapd	ymm1, [srcreg+d2+32]		;; I3
	vaddpd	ymm5, ymm5, ymm1		;; A3 = A3 + I3
	vmulpd	ymm1, ymm1, ymm0		;; B3 = I3 * cosine/sine
	vsubpd	ymm1, ymm1, ymm4		;; B3 = B3 - R3

	vmovapd	ymm0, [(screg2+64)+32]		;; cosine/sine
	vmovapd	ymm6, [srcreg+d2+d1]		;; R4
	vmulpd	ymm7, ymm6, ymm0		;; A4 = R4 * cosine/sine
	vmovapd ymm4, [srcreg+d2+d1+32]		;; I4
	vaddpd	ymm7, ymm7, ymm4		;; A4 = A4 + I4
	vmulpd	ymm4, ymm4, ymm0		;; B4 = I4 * cosine/sine
	vsubpd	ymm4, ymm4, ymm6		;; B4 = B4 - R4

	vmulpd	ymm2, ymm2, [screg1+64]		;; A2 = A2 * sine (new R3)
	vmulpd	ymm3, ymm3, [screg1+64]		;; B2 = B2 * sine (new R4)
	vmulpd	ymm5, ymm5, [screg2]		;; A3 = A3 * sine (new R5)
	vmulpd	ymm1, ymm1, [screg2]		;; B3 = B3 * sine (new R6)
	vmulpd	ymm7, ymm7, [screg2+64]		;; A4 = A4 * sine (new R7)
	vmulpd	ymm4, ymm4, [screg2+64]		;; B4 = B4 * sine (new R8)

	vsubpd	ymm0, ymm5, ymm7	;; R5 - R7 (newer R6)
	vaddpd	ymm5, ymm5, ymm7	;; R5 + R7 (newer R5)

	L1prefetch srcreg+L1pd, L1pt

	vsubpd	ymm7, ymm1, ymm4	;; R6 - R8 (newer R8)
	vaddpd	ymm1, ymm1, ymm4	;; R6 + R8 (newer R7)

	vaddpd	ymm4, ymm0, ymm7	;; R6 = R6 + R8
	vsubpd	ymm7, ymm7, ymm0	;; R8 = R8 - R6

	vmulpd	ymm4, ymm4, YMM_SQRTHALF ;; R6 = R6 * square root of 1/2
	vmulpd	ymm7, ymm7, YMM_SQRTHALF ;; R8 = R8 * square root of 1/2

	vmovapd	ymm6, [srcreg+32]	;; I1 (a.k.a new R2)

	vsubpd	ymm0, ymm6, ymm3	;; R2 - R4 (newer R4)
	vaddpd	ymm6, ymm6, ymm3	;; R2 + R4 (newer R2)

	L1prefetch srcreg+d1+L1pd, L1pt

	vsubpd	ymm3, ymm0, ymm7	;; R4 - R8 (final R8)
	vaddpd	ymm0, ymm0, ymm7	;; R4 + R8 (final R4)

	vmovapd	ymm7, [srcreg]		;; R1
	ystore	[dstreg], ymm3		;; Save final R8 temporarily
	vaddpd	ymm3, ymm7, ymm2	;; R1 + R3 (newer R1)
	vsubpd	ymm7, ymm7, ymm2	;; R1 - R3 (newer R3)

	vsubpd	ymm2, ymm6, ymm4	;; R2 - R6 (final R6)
	vaddpd	ymm6, ymm6, ymm4	;; R2 + R6 (final R2)

	L1prefetch srcreg+d2+L1pd, L1pt

	vsubpd	ymm4, ymm3, ymm5	;; R1 - R5 (final R5)
	vaddpd	ymm3, ymm3, ymm5	;; R1 + R5 (final R1)

	vsubpd	ymm5, ymm7, ymm1	;; R3 - R7 (final R7)
	vaddpd	ymm7, ymm7, ymm1	;; R3 + R7 (final R3)

	;; Swizzle using 4 shufpds and 4 perm2f128  (1 or 3 clocks each on port 5, throughput 1)

	vshufpd	ymm1, ymm3, ymm6, 0		;; Shuffle R1 and R2 to create R1/R2 low
	vshufpd	ymm3, ymm3, ymm6, 15		;; Shuffle R1 and R2 to create R1/R2 hi

	vshufpd	ymm6, ymm7, ymm0, 0		;; Shuffle R3 and R4 to create R3/R4 low
	vshufpd	ymm7, ymm7, ymm0, 15		;; Shuffle R3 and R4 to create R3/R4 hi

	L1prefetch srcreg+d2+d1+L1pd, L1pt

	ylow128s ymm0, ymm1, ymm6		;; Shuffle R1/R2 low and R3/R4 low (final R1)
	yhigh128s ymm1, ymm1, ymm6		;; Shuffle R1/R2 low and R3/R4 low (final R3)

	ylow128s ymm6, ymm3, ymm7		;; Shuffle R1/R2 hi and R3/R4 hi (final R2)
	yhigh128s ymm3, ymm3, ymm7		;; Shuffle R1/R2 hi and R3/R4 hi (final R4)

	vmovapd	ymm7, [dstreg]			;; Reload final R8

	ystore	[dstreg], ymm0			;; Save R1
	ystore	[dstreg+e2], ymm1		;; Save R3
	ystore	[dstreg+e1], ymm6		;; Save R2
	ystore	[dstreg+e2+e1], ymm3		;; Save R4

	;; Swizzle using 4 shufpds and 4 perm2f128  (1 or 3 clocks each on port 5, throughput 1)

	vshufpd	ymm0, ymm4, ymm2, 0		;; Shuffle R5 and R6 to create R5/R6 low
	vshufpd	ymm4, ymm4, ymm2, 15		;; Shuffle R5 and R6 to create R5/R6 hi

	vshufpd	ymm2, ymm5, ymm7, 0		;; Shuffle R7 and R8 to create R7/R8 low
	vshufpd	ymm5, ymm5, ymm7, 15		;; Shuffle R7 and R8 to create R7/R8 hi

	ylow128s ymm7, ymm0, ymm2		;; Shuffle R5/R6 low and R7/R8 low (final R5)
	yhigh128s ymm0, ymm0, ymm2		;; Shuffle R5/R6 low and R7/R8 low (final R7)

	ylow128s ymm2, ymm4, ymm5		;; Shuffle R5/R6 hi and R7/R8 hi (final R6)
	yhigh128s ymm4, ymm4, ymm5		;; Shuffle R5/R6 hi and R7/R8 hi (final R8)

	ystore	[dstreg+32], ymm7		;; Save R5
	ystore	[dstreg+e2+32], ymm0		;; Save R7
	ystore	[dstreg+e1+32], ymm2		;; Save R6
	ystore	[dstreg+e2+e1+32], ymm4		;; Save R8

	bump	srcreg, srcinc
	bump	dstreg, dstinc
	bump	screg1, scinc1
	bump	screg2, scinc2
	ENDM

;;
;; ********************************* one-eight-reals-three-four-complex-fft variants ***************************************
;;

;; Macro to do one eight_reals_fft and three four_complex_djbfft.
;; Having eight-real data and four-complex data in the same YMM register
;; is not an ideal situation.  However, sometimes one cannot get the
;; perfect memory layout.

yr4_4cl_eight_reals_four_complex_djbfft_preload MACRO
	yr4_o8r_t4c_djbfft_mem_preload
	ENDM
yr4_4cl_eight_reals_four_complex_djbfft MACRO srcreg,srcinc,d1,d2,screg,scinc,maxrpt
	yr4_o8r_t4c_djbfft_mem [srcreg],[srcreg+d1],[srcreg+d2],[srcreg+d2+d1],[srcreg+32],[srcreg+d1+32],[srcreg+d2+32],[srcreg+d2+d1+32],screg,[srcreg]
;;	ystore	[srcreg], ymm7		;; Save R1
	ystore	[srcreg+32], ymm0	;; Save I1
	ystore	[srcreg+d1], ymm3	;; Save R2
	ystore	[srcreg+d1+32], ymm6	;; Save I2
	ystore	[srcreg+d2], ymm2	;; Save R3
	ystore	[srcreg+d2+32], ymm1	;; Save I3
	ystore	[srcreg+d2+d1], ymm5	;; Save R4
	ystore	[srcreg+d2+d1+32], ymm4	;; Save I4
	bump	srcreg, srcinc
	bump	screg, scinc
	ENDM

;; Does a yr4_x8r_fft_mem on the low word of the ymm register
;; Does a yr4_x4c_djbfft_mem on the high words of the ymm register
;; This is REALLY funky, as we do both at the same time within
;; the full ymm register whenever possible.
yr4_o8r_t4c_djbfft_mem_preload MACRO
	ENDM
yr4_o8r_t4c_djbfft_mem MACRO mem1,mem2,mem3,mem4,mem5,mem6,mem7,mem8,screg,dst
					;; Eight-reals comments		; Three four-complex comments
	vmovapd	ymm2, mem6		;; R6				; I2
	vmovapd	ymm6, mem4		;; R4				; R4
	vblendpd ymm3, ymm2, ymm6, 1	;; R4				; I2
	vmovapd	ymm5, mem8		;; R8				; I4
	vsubpd	ymm4, ymm3, ymm5	;; interim R8 = R4 - R8		; I2 - I4 (new I4)

	vblendpd ymm6, ymm6, ymm2, 1	;; R6				; R4
	vmovapd	ymm1, mem2		;; R2				; R2
	vsubpd	ymm7, ymm1, ymm6	;; interim R6 = R2 - R6		; R2 - R4 (new R4)

	vaddpd	ymm3, ymm3, ymm5	;; new R4 = R4 + R8		; I2 + I4 (new I2)
	vaddpd	ymm1, ymm1, ymm6	;; new R2 = R2 + R6		; R2 + R4 (new R2)

	vmulsd	xmm0, xmm4, Q YMM_SQRTHALF;; interim R8 * square root	; meaningless
	vmulsd	xmm2, xmm7, Q YMM_SQRTHALF;; interim R6 * square root	; meaningless

	vaddsd	xmm5, xmm2, xmm0	;; new R8 = interim R6 + R8	; meaningless
	vsubsd	xmm2, xmm0, xmm2	;; new negR6 = interim R8 - R6	; meaningless

	vblendpd ymm7, ymm7, ymm5, 1	;; new R8			; new R4
	vblendpd ymm4, ymm4, ymm2, 1	;; new negR6			; new I4

	vmovapd	ymm0, mem3		;; R3				; R3
	vmovapd	ymm2, mem5		;; R5				; I1
	vblendpd ymm6, ymm0, ymm2, 1	;; R5				; R3
	vblendpd ymm2, ymm2, ymm0, 1	;; R3				; I1

	vaddpd	ymm0, ymm2, mem7	;; new R3 = R3 + R7		; I1 + I3 (new I1)
	vblendpd ymm5, ymm1, ymm0, 1	;; new R3			; new R2
	vblendpd ymm0, ymm0, ymm1, 1	;; new R2			; new I1
	vsubpd	ymm2, ymm2, mem7	;; new R7 = R3 - R7		; I1 - I3 (new I3)

	vaddpd	ymm1, ymm2, ymm7	;; R7 + R8 (final I3)		; I3 + R4 (final I3)
	vsubpd	ymm2, ymm2, ymm7	;; R7 - R8 (final I4)		; I3 - R4 (final I4)

	vmovapd	ymm7, mem1		;; R1				; R1
	ystore	YMM_TMP1, ymm2		;; Temporarily save final I4
	vaddpd	ymm2, ymm7, ymm6	;; new R1 = R1 + R5		; R1 + R3 (new R1)
	vsubpd	ymm7, ymm7, ymm6	;; new R5 = R1 - R5		; R1 - R3 (new R3)

	vsubpd	ymm6, ymm0, ymm3	;; R2 - R4 (final I2)		; I1 - I2 (final I2)
	vaddpd	ymm0, ymm0, ymm3	;; R2 + R4 (final I1, a.k.a 2nd real result) ; I1 + I2 (final I1)

	vaddpd	ymm3, ymm2, ymm5	;; R1 + R3 (final R1)		; R1 + R2 (final R1)
	vsubpd	ymm2, ymm2, ymm5	;; R1 - R3 (final R2)		; R1 - R2 (final R2)

	vsubpd	ymm5, ymm7, ymm4	;; R5 - negR6 (final R3)	; R3 - I4 (final R3)
	vaddpd	ymm7, ymm7, ymm4	;; R5 + negR6 (final R4)	; R3 + I4 (final R4)

	ystore	dst, ymm3		;; Save final R1

	vmovapd	ymm4, [screg+64+32]	;; cosine/sine
	vmulpd	ymm3, ymm2, ymm4	;; A2 = R2 * cosine/sine
	vsubpd	ymm3, ymm3, ymm6	;; A2 = A2 - I2

	vmulpd	ymm6, ymm6, ymm4	;; B2 = I2 * cosine/sine
	vaddpd	ymm6, ymm6, ymm2	;; B2 = B2 + R2

	vmovapd	ymm4, [screg+32]	;; cosine/sine
	vmulpd	ymm2, ymm5, ymm4	;; A3 = R3 * cosine/sine
	vsubpd	ymm2, ymm2, ymm1	;; A3 = A3 - I3

	vmulpd	ymm1, ymm1, ymm4	;; B3 = I3 * cosine/sine
	vaddpd	ymm1, ymm1, ymm5	;; B3 = B3 + R3

	vmovapd	ymm4, [screg+128+32]	;; cosine/sine
	vmulpd	ymm5, ymm7, ymm4	;; A4 = R4 * cosine/sine
	vsubpd	ymm5, ymm5, YMM_TMP1	;; A4 = A4 - I4

	vmulpd	ymm4, ymm4, YMM_TMP1	;; B4 = I4 * cosine/sine
	vaddpd	ymm4, ymm4, ymm7	;; B4 = B4 + R4

	vmovapd	ymm7, [screg+64]	;; Sine
	vmulpd	ymm3, ymm3, ymm7	;; A2 = A2 * sine (final R2)
	vmulpd	ymm6, ymm6, ymm7	;; B2 = B2 * sine (final I2)

	vmovapd	ymm7, [screg]		;; Sine
	vmulpd	ymm2, ymm2, ymm7	;; A3 = A3 * sine (final R3)
	vmulpd	ymm1, ymm1, ymm7	;; B3 = B3 * sine (final I3)

	vmovapd	ymm7, [screg+128]	;; Sine
	vmulpd	ymm5, ymm5, ymm7	;; A4 = A4 * sine (final R4)
	vmulpd	ymm4, ymm4, ymm7	;; B4 = B4 * sine (final I4)
	ENDM

;; 64-bit version

IFDEF X86_64

yr4_o8r_t4c_djbfft_mem_preload MACRO
	vmovapd	ymm15, YMM_ONE		;; One
	vsubpd	ymm14, ymm15, ymm15	;; Zero
	vblendpd ymm15, ymm15, YMM_SQRTHALF, 1 ;; Three ones and one sqrthalf
	vblendpd ymm14, ymm14, ymm15, 1 ;; Three zeroes and one sqrthalf
	ENDM

yr4_o8r_t4c_djbfft_mem MACRO mem1,mem2,mem3,mem4,mem5,mem6,mem7,mem8,screg,dst
					;; Eight-reals comments		; Three four-complex comments
	vmovapd	ymm13, mem6		;; R6				; I2
	vmovapd	ymm7, mem4		;; R4				; R4
	vblendpd ymm5, ymm13, ymm7, 1	;; R4				; I2
	vmovapd	ymm4, mem8		;; R8				; I4
	vsubpd	ymm1, ymm5, ymm4	;; interim R8 = R4 - R8		; I2 - I4 (new I4)		; 1-3

	vblendpd ymm7, ymm7, ymm13, 1	;; R6				; R4
	vmovapd	ymm11, mem2		;; R2				; R2
	vsubpd	ymm6, ymm11, ymm7	;; interim R6 = R2 - R6		; R2 - R4 (new R4)		; 2-4

	vaddpd	ymm11, ymm11, ymm7	;; new R2 = R2 + R6		; R2 + R4 (new R2)		; 3-5
	vaddpd	ymm5, ymm5, ymm4	;; new R4 = R4 + R8		; I2 + I4 (new I2)		; 4-6

	vmovapd	ymm7, mem3		;; R3				; R3
	vmovapd	ymm9, mem5		;; R5				; I1
	vblendpd ymm8, ymm9, ymm7, 1	;; R3				; I1

	vmulpd	ymm2, ymm1, ymm14	;; interim R8 * square root	; zero				;	4-8
	vmulpd	ymm6, ymm6, ymm15	;; interim R6 * square root	; new R4			;	5-9

	vmovapd	ymm3, mem7		;; R7				; I3
	vaddpd	ymm10, ymm8, ymm3	;; new R3 = R3 + R7		; I1 + I3 (new I1)		; 5-7
	vsubpd	ymm8, ymm8, ymm3	;; new R7 = R3 - R7		; I1 - I3 (new I3)		; 6-8

	vblendpd ymm7, ymm7, ymm9, 1	;; R5				; R3
	vmovapd	ymm12, mem1		;; R1				; R1
	vaddpd	ymm13, ymm12, ymm7	;; new R1 = R1 + R5		; R1 + R3 (new R1)		; 7-9
	vsubpd	ymm12, ymm12, ymm7	;; new R5 = R1 - R5		; R1 - R3 (new R3)		; 8-10

	vblendpd ymm0, ymm10, ymm11, 1	;; new R2			; new I1			;		8
	vblendpd ymm4, ymm11, ymm10, 1	;; new R3			; new R2			;		9

	vsubpd	ymm7, ymm0, ymm5	;; R2 - R4 (final I2)		; I1 - I2 (final I2)		; 9-11

	vaddpd	ymm9, ymm6, ymm2	;; new R8 = interim R6 + R8	; new R4			; 10-12
	vsubsd	xmm6, xmm2, xmm6	;; new negR6 = interim R8 - R6	; meaningless			; 11-13
	vmovapd	ymm3, [screg+64+32]	;; cosine/sine

	vsubpd	ymm10, ymm13, ymm4	;; R1 - R3 (final R2)		; R1 - R2 (final R2)		; 12-14
	vmovapd	ymm2, [screg+32]	;; cosine/sine

	vaddpd	ymm11, ymm8, ymm9	;; R7 + R8 (final I3)		; I3 + R4 (final I3)		; 13-15
	vsubpd	ymm8, ymm8, ymm9	;; R7 - R8 (final I4)		; I3 - R4 (final I4)		; 14-16

	vblendpd ymm1, ymm1, ymm6, 1	;; new negR6			; new I4			;		14

	vmulpd	ymm6, ymm7, ymm3	;; B2 = I2 * cosine/sine					;	14-18
	vmulpd	ymm3, ymm10, ymm3	;; A2 = R2 * cosine/sine					;	15-19

	vsubpd	ymm9, ymm12, ymm1	;; R5 - negR6 (final R3)	; R3 - I4 (final R3)		; 15-17
	vaddpd	ymm12, ymm12, ymm1	;; R5 + negR6 (final R4)	; R3 + I4 (final R4)		; 16-18

	vmulpd	ymm1, ymm11, ymm2	;; B3 = I3 * cosine/sine					;	16-20

	vaddpd	ymm0, ymm0, ymm5	;; R2 + R4 (final I1, a.k.a 2nd real result) ; I1 + I2 (final I1); 17-19
	vmovapd	ymm5, [screg+128+32]	;; cosine/sine

	vaddpd	ymm13, ymm13, ymm4	;; R1 + R3 (final R1)		; R1 + R2 (final R1)		; 18-20

	vmulpd	ymm4, ymm8, ymm5	;; B4 = I4 * cosine/sine					;	17-21

	vmulpd	ymm2, ymm9, ymm2	;; A3 = R3 * cosine/sine					;	18-22

	vaddpd	ymm6, ymm6, ymm10	;; B2 = B2 + R2							; 19-21
	vmulpd	ymm5, ymm12, ymm5	;; A4 = R4 * cosine/sine					;	19-23
	vmovapd	ymm10, [screg+64]	;; Sine

	vsubpd	ymm3, ymm3, ymm7	;; A2 = A2 - I2							; 20-22
	vmovapd	ymm7, [screg]		;; Sine

	vaddpd	ymm1, ymm1, ymm9	;; B3 = B3 + R3							; 21-23
	vmovapd	ymm9, [screg+128]	;; Sine
	ystore	dst, ymm13		;; Save final R1						;		21

	vaddpd	ymm4, ymm4, ymm12	;; B4 = B4 + R4							; 22-24
	vmulpd	ymm6, ymm6, ymm10	;; B2 = B2 * sine (final I2)					;	22-26

	vsubpd	ymm2, ymm2, ymm11	;; A3 = A3 - I3							; 23-25
	vmulpd	ymm3, ymm3, ymm10	;; A2 = A2 * sine (final R2)					;	23-27

	vsubpd	ymm5, ymm5, ymm8	;; A4 = A4 - I4							; 24-26
	vmulpd	ymm1, ymm1, ymm7	;; B3 = B3 * sine (final I3)					;	24-28

	vmulpd	ymm4, ymm4, ymm9	;; B4 = B4 * sine (final I4)					;	25-29

	vmulpd	ymm2, ymm2, ymm7	;; A3 = A3 * sine (final R3)					;	26-30

	vmulpd	ymm5, ymm5, ymm9	;; A4 = A4 * sine (final R4)					;	27-31

	ENDM

;; Haswell FMA3 version -- could be faster with unrolling

IF (@INSTR(,%yarch,<FMA3>) NE 0)

yr4_o8r_t4c_djbfft_mem_preload MACRO
	vmovapd	ymm15, YMM_ONE		;; One
	vblendpd ymm14, ymm15, YMM_SQRTHALF, 1 ;; Three ones and one sqrthalf
	ENDM

yr4_o8r_t4c_djbfft_mem MACRO mem1,mem2,mem3,mem4,mem5,mem6,mem7,mem8,screg,dst
					;; Eight-reals comments		; Three four-complex comments
	vmovapd	ymm13, mem6		;; R6				; I2
	vmovapd	ymm7, mem4		;; R4				; R4
	vblendpd ymm5, ymm13, ymm7, 1	;; R4				; I2
	vmovapd	ymm4, mem8		;; R8				; I4
	yfmsubpd ymm1, ymm5, ymm15, ymm4 ;; interim R8 = R4 - R8	; I2 - I4 (new I4)		;	1-5

	vblendpd ymm7, ymm7, ymm13, 1	;; R6				; R4
	vmovapd	ymm11, mem2		;; R2				; R2
	yfmsubpd ymm6, ymm11, ymm15, ymm7 ;; interim R6 = R2 - R6	; R2 - R4 (new R4)		; 1-5

	yfmaddpd ymm11, ymm11, ymm15, ymm7 ;; new R2 = R2 + R6		; R2 + R4 (new R2)		;	2-6
	yfmaddpd ymm5, ymm5, ymm15, ymm4 ;; new R4 = R4 + R8		; I2 + I4 (new I2)		; 2-6

	vmovapd	ymm7, mem3		;; R3				; R3
	vmovapd	ymm9, mem5		;; R5				; I1
	vblendpd ymm8, ymm9, ymm7, 1	;; R3				; I1

	vmovapd	ymm3, mem7		;; R7				; I3
	yfmaddpd ymm10, ymm8, ymm15, ymm3 ;; new R3 = R3 + R7		; I1 + I3 (new I1)		;	3-7
	yfmsubpd ymm8, ymm8, ymm15, ymm3 ;; new R7 = R3 - R7		; I1 - I3 (new I3)		; 3-7

	vblendpd ymm7, ymm7, ymm9, 1	;; R5				; R3
	vmovapd	ymm12, mem1		;; R1				; R1
	yfmaddpd ymm13, ymm12, ymm15, ymm7 ;; new R1 = R1 + R5		; R1 + R3 (new R1)		;	4-8
	yfmsubpd ymm12, ymm12, ymm15, ymm7 ;; new R5 = R1 - R5		; R1 - R3 (new R3)		; 4-8

	vblendpd ymm0, ymm10, ymm11, 1	;; new R2			; new I1			;		8
	vblendpd ymm4, ymm11, ymm10, 1	;; new R3			; new R2			;		9

	yfmaddpd ymm9, ymm6, ymm15, ymm1 ;; new R8 = interim R6 + R8	; meaningless			;	6-10
	yfmsubpd ymm3, ymm1, ymm15, ymm6 ;; new negR6 = interim R8 - R6	; meaningless			; 6-10

	vblendpd ymm9, ymm6, ymm9, 1	;; new R8			; new R4			;		11
	vblendpd ymm1, ymm1, ymm3, 1	;; new negR6			; new I4			;		12
	vmovapd	ymm3, [screg+64+32]			;; cosine/sine

	yfmsubpd ymm7, ymm0, ymm15, ymm5 ;; R2 - R4 (final I2)		; I1 - I2 (final I2)		;	9-13
	yfmaddpd ymm0, ymm0, ymm15, ymm5 ;; R2 + R4 (final I1, a.k.a 2nd real result) ; I1 + I2 (final I1) ; 9-13
	vmovapd	ymm2, [screg+32]			;; cosine/sine

	yfmsubpd ymm10, ymm13, ymm15, ymm4 ;; R1 - R3 (final R2)	; R1 - R2 (final R2)		;	10-14
	yfmaddpd ymm13, ymm13, ymm15, ymm4 ;; R1 + R3 (final R1)	; R1 + R2 (final R1)		; 10-14
	vmovapd	ymm5, [screg+128+32]			;; cosine/sine

	yfmaddpd ymm11, ymm9, ymm14, ymm8 ;; R7 + R8*SQRTHALF (final I3) ; I3 + R4*1 (final I3)		;	12-16
	yfnmaddpd ymm8, ymm9, ymm14, ymm8 ;; R7 - R8*SQRTHALF (final I4) ; I3 - R4*1 (final I4)		; 12-16

	yfnmaddpd ymm9, ymm1, ymm14, ymm12 ;; R5 - negR6*SQRTHALF (final R3) ; R3 - I4*1 (final R3)	;	13-17
	yfmaddpd ymm12, ymm1, ymm14, ymm12 ;; R5 + negR6*SQRTHALF (final R4) ; R3 + I4*1 (final R4)	; 13-17

	yfmaddpd ymm6, ymm7, ymm3, ymm10		;; B2 = I2 * cosine/sine + R2			;	14-18
	yfmsubpd ymm3, ymm10, ymm3, ymm7		;; A2 = R2 * cosine/sine - I2			; 14-18
	vmovapd	ymm10, [screg+64]			;; Sine
	vmovapd	ymm7, [screg]				;; Sine
	ystore	dst, ymm13				;; Save final R1				;		15

	yfmaddpd ymm1, ymm11, ymm2, ymm9		;; B3 = I3 * cosine/sine + R3			;	18-22
	yfmsubpd ymm2, ymm9, ymm2, ymm11		;; A3 = R3 * cosine/sine - I3			; 18-22
	vmovapd	ymm9, [screg+128]			;; Sine

	yfmaddpd ymm4, ymm8, ymm5, ymm12		;; B4 = I4 * cosine/sine + R4			;	19-23
	yfmsubpd ymm5, ymm12, ymm5, ymm8		;; A4 = R4 * cosine/sine - I4			; 19-23

	vmulpd	ymm6, ymm6, ymm10			;; B2 = B2 * sine (final I2)			;	20-24
	vmulpd	ymm3, ymm3, ymm10			;; A2 = A2 * sine (final R2)			; 20-24
	vmulpd	ymm1, ymm1, ymm7			;; B3 = B3 * sine (final I3)			;	23-27
	vmulpd	ymm2, ymm2, ymm7			;; A3 = A3 * sine (final R3)			; 23-27
	vmulpd	ymm4, ymm4, ymm9			;; B4 = B4 * sine (final I4)			;	24-28
	vmulpd	ymm5, ymm5, ymm9			;; A4 = A4 * sine (final R4)			; 24-28
	ENDM

ENDIF

ENDIF

;;
;; ********************************* one-eight-reals-three-four-complex-unfft variants ***************************************
;;

;; Macro to do one eight_reals_unfft and three four_complex_djbunfft.
;; The eight-reals operation is done in the lower word of the YMM
;; register.  This isn't very efficient, but this macro isn't called a whole lot.

yr4_4cl_eight_reals_four_complex_djbunfft_preload MACRO
	yr4_o8r_t4c_djbunfft_mem_preload
	ENDM
yr4_4cl_eight_reals_four_complex_djbunfft MACRO srcreg,srcinc,d1,d2,screg,scinc,maxrpt
	yr4_o8r_t4c_djbunfft_mem [srcreg],[srcreg+32],[srcreg+d1],[srcreg+d1+32],[srcreg+d2],[srcreg+d2+32],[srcreg+d2+d1],[srcreg+d2+d1+32],screg,[srcreg]
;;	ystore	[srcreg], ymm1		;; Save R1
	ystore	[srcreg+32], ymm6	;; Save R5/I1
	ystore	[srcreg+d1], ymm0	;; Save R2
	ystore	[srcreg+d1+32], ymm2	;; Save R6/I2
	ystore	[srcreg+d2], ymm5	;; Save R3
	ystore	[srcreg+d2+32], ymm4	;; Save R7/I3
	ystore	[srcreg+d2+d1], ymm7	;; Save R4
	ystore	[srcreg+d2+d1+32], ymm3	;; Save R8/I4
	bump	srcreg, srcinc
	bump	screg, scinc
	ENDM

yr4_o8r_t4c_djbunfft_mem_preload MACRO
	ENDM
yr4_o8r_t4c_djbunfft_mem MACRO mem1,mem2,mem3,mem4,mem5,mem6,mem7,mem8,screg,dst
	vmovapd	ymm3, [screg+64+32]	;; cosine/sine
	vmovapd	ymm2, mem3		;; R2
	vmulpd	ymm6, ymm2, ymm3	;; A2 = R2 * cosine/sine		; 1-5
	vmovapd	ymm0, mem4		;; I2
	vmulpd	ymm3, ymm3, ymm0	;; B2 = I2 * cosine/sine		; 2-6

	vmovapd	ymm5, [screg+32]	;; cosine/sine
	vmovapd	ymm4, mem5		;; R3
	vmulpd	ymm7, ymm4, ymm5	;; A3 = R3 * cosine/sine		; 3-7
	vmovapd	ymm1, mem6		;; I3

	vaddpd	ymm6, ymm6, ymm0	;; A2 = A2 + I2				; 6-8

	vmulpd	ymm0, ymm1, ymm5	;; B3 = I3 * cosine/sine		; 4-8

	vsubpd	ymm3, ymm3, ymm2	;; B2 = B2 - R2				; 7-9
	vaddpd	ymm7, ymm7, ymm1	;; A3 = A3 + I3				; 8-10

	vmovapd	ymm5, [screg+128+32]	;; cosine/sine
	vmovapd	ymm2, mem7		;; R4
	vmulpd	ymm1, ymm2, ymm5	;; A4 = R4 * cosine/sine		; 5-9

	vsubpd	ymm0, ymm0, ymm4	;; B3 = B3 - R3				; 9-11

	vmovapd	ymm4, mem8		;; I4
	vmulpd	ymm5, ymm4, ymm5	;; B4 = I4 * cosine/sine		; 6-10

	vmulpd	ymm6, ymm6, [screg+64]	;; A2 = A2 * sine (new R2)		; 9-13
	vmulpd	ymm3, ymm3, [screg+64]	;; B2 = B2 * sine (new I2)		; 10-14

	vaddpd	ymm1, ymm1, ymm4	;; A4 = A4 + I4				; 10-12
	vsubpd	ymm5, ymm5, ymm2	;; B4 = B4 - R4				; 11-13

	vmovapd	ymm4, [screg]		;; Sine
	vmulpd	ymm7, ymm7, ymm4	;; A3 = A3 * sine (new R3)		; 11-15
	vmulpd	ymm0, ymm0, ymm4	;; B3 = B3 * sine (new I3)		; 12-16
	vmovapd	ymm4, [screg+128]	;; Sine
	vmulpd	ymm1, ymm1, ymm4	;; A4 = A4 * sine (new R4)		; 13-17
	vmulpd	ymm5, ymm5, ymm4	;; B4 = B4 * sine (new I4)		; 14-18

					;; Eight reals comments		; Four complex comments
					;; new R3 = A2
					;; new R4 = B2
					;; new R5 = A3
					;; new R6 = B3
					;; new R7 = A4
					;; new R8 = B4

	vsubpd	ymm2, ymm0, ymm5	;; R6 - R8 (new R8)		; I3 - I4 (new R4)
	vaddpd	ymm0, ymm0, ymm5	;; R6 + R8 (new R7)		; I3 + I4 (new I3)

	vsubpd	ymm4, ymm7, ymm1	;; R5 - R7 (new R6)		; R3 - R4 (new negI4)
	vaddpd	ymm7, ymm7, ymm1	;; R5 + R7 (new R5)		; R3 + R4 (new R3)

	vmovapd	ymm1, mem2		;; R2				; I1
	vsubpd	ymm5, ymm1, ymm3	;; R2 - R4 (new R4)		; I1 - I2 (new I2)
	vaddpd	ymm1, ymm1, ymm3	;; R2 + R4 (new R2)		; I1 + I2 (new I1)

	ystore	YMM_TMP1, ymm0		;; Temporarily save new R7/new I3

	vaddsd	xmm3, xmm4, xmm2	;; R6 = R6 + R8			; meaningless
	vsubsd	xmm0, xmm4, xmm2	;; negR8 = R6 - R8		; meaningless

	vmulsd	xmm3, xmm3, Q YMM_SQRTHALF ;; R6 *= square root of 1/2	; meaningless
	vmulsd	xmm0, xmm0, Q YMM_SQRTHALF ;; negR8 *= square root of 1/2 ; meaningless

	vblendpd ymm2, ymm2, ymm3, 1	;; new R6			; new R4
	vblendpd ymm4, ymm4, ymm0, 1	;; new negR8			; new negI4

	vaddpd	ymm3, ymm5, ymm4	;; R4 + negR8 (final R8)	; I2 + negI4 (final I4)
	vsubpd	ymm5, ymm5, ymm4	;; R4 - negR8 (final R4)	; I2 - negI4 (final I2)

	vmovapd	ymm0, mem1		;; R1				; R1
	vaddpd	ymm4, ymm0, ymm6	;; R1 + R3 (new R1)		; R1 + R2 (new R1)
	vsubpd	ymm0, ymm0, ymm6	;; R1 - R3 (new R3)		; R1 - R2 (new R2)

	vblendpd ymm6, ymm1, ymm0, 1	;; new R3			; new I1
	vblendpd ymm0, ymm0, ymm1, 1	;; new R2			; new R2

	vsubpd	ymm1, ymm4, ymm7	;; R1 - R5 (final R5)		; R1 - R3 (final R3)
	vaddpd	ymm4, ymm4, ymm7	;; R1 + R5 (final R1)		; R1 + R3 (final R1)

	vsubpd	ymm7, ymm0, ymm2	;; R2 - R6 (final R6)		; R2 - R4 (final R4)
	vaddpd	ymm0, ymm0, ymm2	;; R2 + R6 (final R2)		; R2 + R4 (final R2)

	ystore	dst, ymm4		;; Store R1

	vmovapd	ymm2, YMM_TMP1		;; Reload saved new R7/new I3
	vsubpd	ymm4, ymm6, ymm2	;; R3 - R7 (final R7)		; I1 - I3 (final I3)
	vaddpd	ymm6, ymm6, ymm2	;; R3 + R7 (final R3)		; I1 + I3 (final I1)

	vblendpd ymm2, ymm5, ymm7, 1	;; final R6			; final I2
	vblendpd ymm7, ymm7, ymm5, 1	;; final R4			; final R4

	vblendpd ymm5, ymm1, ymm6, 1	;; final R3			; final R3
	vblendpd ymm6, ymm6, ymm1, 1	;; final R5			; final I1
	ENDM

;; 64-bit version

IFDEF X86_64

yr4_o8r_t4c_djbunfft_mem_preload MACRO
	vmovapd	ymm15, YMM_SQRTHALF
	ENDM

yr4_o8r_t4c_djbunfft_mem MACRO mem1,mem2,mem3,mem4,mem5,mem6,mem7,mem8,screg,dst
	vmovapd	ymm3, [screg+64+32]	;; cosine/sine
	vmovapd	ymm2, mem3		;; R2
	vmulpd	ymm6, ymm2, ymm3	;; A2 = R2 * cosine/sine		; 1-5
	vmovapd	ymm8, mem4		;; I2
	vmulpd	ymm3, ymm3, ymm8	;; B2 = I2 * cosine/sine		; 2-6

	vmovapd	ymm5, [screg+32]	;; cosine/sine
	vmovapd	ymm4, mem5		;; R3
	vmulpd	ymm7, ymm4, ymm5	;; A3 = R3 * cosine/sine		; 3-7
	vmovapd	ymm1, mem6		;; I3

	vaddpd	ymm6, ymm6, ymm8	;; A2 = A2 + I2				; 6-8

	vmulpd	ymm8, ymm1, ymm5	;; B3 = I3 * cosine/sine		; 4-8

	vsubpd	ymm3, ymm3, ymm2	;; B2 = B2 - R2				; 7-9
	vaddpd	ymm7, ymm7, ymm1	;; A3 = A3 + I3				; 8-10

	vmovapd	ymm5, [screg+128+32]	;; cosine/sine
	vmovapd	ymm2, mem7		;; R4
	vmulpd	ymm1, ymm2, ymm5	;; A4 = R4 * cosine/sine		; 5-9

	vsubpd	ymm8, ymm8, ymm4	;; B3 = B3 - R3				; 9-11

	vmovapd	ymm4, mem8		;; I4
	vmulpd	ymm5, ymm4, ymm5	;; B4 = I4 * cosine/sine		; 6-10

	vmulpd	ymm6, ymm6, [screg+64]	;; A2 = A2 * sine (new R2)		; 9-13
	vmulpd	ymm3, ymm3, [screg+64]	;; B2 = B2 * sine (new I2)		; 10-14

	vaddpd	ymm1, ymm1, ymm4	;; A4 = A4 + I4				; 10-12
	vsubpd	ymm5, ymm5, ymm2	;; B4 = B4 - R4				; 11-13

	vmovapd	ymm4, [screg]		;; Sine
	vmulpd	ymm7, ymm7, ymm4	;; A3 = A3 * sine (new R3)		; 11-15
	vmulpd	ymm8, ymm8, ymm4	;; B3 = B3 * sine (new I3)		; 12-16
	vmovapd	ymm4, [screg+128]	;; Sine
	vmulpd	ymm1, ymm1, ymm4	;; A4 = A4 * sine (new R4)		; 13-17
	vmulpd	ymm5, ymm5, ymm4	;; B4 = B4 * sine (new I4)		; 14-18

					;; Eight reals comments		; Four complex comments
					;; new R3 = A2
					;; new R4 = B2
					;; new R5 = A3
					;; new R6 = B3
					;; new R7 = A4
					;; new R8 = B4

	vsubpd	ymm2, ymm8, ymm5	;; R6 - R8 (new R8)		; I3 - I4 (new R4)
	vaddpd	ymm8, ymm8, ymm5	;; R6 + R8 (new R7)		; I3 + I4 (new I3)

	vsubpd	ymm4, ymm7, ymm1	;; R5 - R7 (new R6)		; R3 - R4 (new negI4)
	vaddpd	ymm7, ymm7, ymm1	;; R5 + R7 (new R5)		; R3 + R4 (new R3)

	vmovapd	ymm1, mem2		;; R2				; I1
	vsubpd	ymm5, ymm1, ymm3	;; R2 - R4 (new R4)		; I1 - I2 (new I2)
	vaddpd	ymm1, ymm1, ymm3	;; R2 + R4 (new R2)		; I1 + I2 (new I1)

	vaddsd	xmm3, xmm4, xmm2	;; R6 = R6 + R8			; meaningless
	vsubsd	xmm0, xmm4, xmm2	;; negR8 = R6 - R8		; meaningless

	vmulsd	xmm3, xmm3, xmm15	 ;; R6 *= square root of 1/2	; meaningless
	vmulsd	xmm0, xmm0, xmm15	 ;; negR8 *= square root of 1/2	; meaningless

	vblendpd ymm2, ymm2, ymm3, 1	;; new R6			; new R4
	vblendpd ymm4, ymm4, ymm0, 1	;; new negR8			; new negI4

	vaddpd	ymm3, ymm5, ymm4	;; R4 + negR8 (final R8)	; I2 + negI4 (final I4)
	vsubpd	ymm5, ymm5, ymm4	;; R4 - negR8 (final R4)	; I2 - negI4 (final I2)

	vmovapd	ymm0, mem1		;; R1				; R1
	vaddpd	ymm4, ymm0, ymm6	;; R1 + R3 (new R1)		; R1 + R2 (new R1)
	vsubpd	ymm0, ymm0, ymm6	;; R1 - R3 (new R3)		; R1 - R2 (new R2)

	vblendpd ymm6, ymm1, ymm0, 1	;; new R3			; new I1
	vblendpd ymm0, ymm0, ymm1, 1	;; new R2			; new R2

	vsubpd	ymm1, ymm4, ymm7	;; R1 - R5 (final R5)		; R1 - R3 (final R3)
	vaddpd	ymm4, ymm4, ymm7	;; R1 + R5 (final R1)		; R1 + R3 (final R1)

	vsubpd	ymm7, ymm0, ymm2	;; R2 - R6 (final R6)		; R2 - R4 (final R4)
	vaddpd	ymm0, ymm0, ymm2	;; R2 + R6 (final R2)		; R2 + R4 (final R2)

	ystore	dst, ymm4		;; Store R1

	vsubpd	ymm4, ymm6, ymm8	;; R3 - R7 (final R7)		; I1 - I3 (final I3)
	vaddpd	ymm6, ymm6, ymm8	;; R3 + R7 (final R3)		; I1 + I3 (final I1)

	vblendpd ymm2, ymm5, ymm7, 1	;; final R6			; final I2
	vblendpd ymm7, ymm7, ymm5, 1	;; final R4			; final R4

	vblendpd ymm5, ymm1, ymm6, 1	;; final R3			; final R3
	vblendpd ymm6, ymm6, ymm1, 1	;; final R5			; final I1
	ENDM

;; Haswell FMA3 version -- could be faster with unrolling

IF (@INSTR(,%yarch,<FMA3>) NE 0)

yr4_o8r_t4c_djbunfft_mem_preload MACRO
	vmovapd	ymm15, YMM_ONE
	vblendpd ymm14, ymm15, YMM_SQRTHALF, 1
	ENDM

yr4_o8r_t4c_djbunfft_mem MACRO mem1,mem2,mem3,mem4,mem5,mem6,mem7,mem8,screg,dst
	vmovapd	ymm8, [screg+32]			;; cosine/sine
	vmovapd	ymm2, mem5				;; R3
	vmovapd	ymm4, mem6				;; I3
	yfmaddpd ymm7, ymm2, ymm8, ymm4			;; A3 = R3 * cosine/sine + I3			;	1-5
	yfmsubpd ymm4, ymm4, ymm8, ymm2			;; B3 = I3 * cosine/sine - R3			; 1-5

	vmovapd	ymm12, [screg+64+32]			;; cosine/sine
	vmovapd	ymm11, mem3				;; R2
	vmovapd	ymm10, mem4				;; I2
	yfmaddpd ymm6, ymm11, ymm12, ymm10		;; A2 = R2 * cosine/sine + I2			;	2-6
	yfmsubpd ymm10, ymm10, ymm12, ymm11		;; B2 = I2 * cosine/sine - R2			; 2-6

	vmovapd	ymm5, [screg+128+32]			;; cosine/sine
	vmovapd	ymm0, mem7				;; R4
	vmovapd	ymm8, mem8				;; I4
	yfmaddpd ymm1, ymm0, ymm5, ymm8			;; A4 = R4 * cosine/sine + I4			;	3-7
	yfmsubpd ymm8, ymm8, ymm5, ymm0			;; B4 = I4 * cosine/sine - R4			; 3-7

	vmovapd	ymm13, [screg]				;; Sine
	vmulpd	ymm7, ymm7, ymm13			;; A3 = A3 * sine				;	6-10
	vmulpd	ymm4, ymm4, ymm13			;; B3 = B3 * sine				; 6-10


					;; Eight reals comments		; Four complex comments
					;; R5 = A3			R3 = A3
					;; R6 = B3			I3 = B3
					;; R7/sine = A4			R4/sine = A4
					;; R8/sine = B4			I4/sine = B4
					;; R3/sine = A2			R2/sine = A2
					;; R4/sine = B2			I2/sine = B2

	vmovapd	ymm0, mem2		;; R2				; I1
	vmovapd	ymm5, [screg+64]			;; Sine
	yfmaddpd ymm9, ymm10, ymm5, ymm0 ;; R2 + R4 * sine (new R2)	; I1 + I2 * sine (new I1)	;	7-11
	yfnmaddpd ymm10, ymm10, ymm5, ymm0 ;; R2 - R4 * sine (new R4)	; I1 - I2 * sine (new I2)	; 7-11

	vmovapd	ymm3, mem1		;; R1				; R1
	yfmaddpd ymm12, ymm6, ymm5, ymm3 ;; R1 + R3 * sine (new R1)	; R1 + R2 * sine (new R1)	;	8-12
	yfnmaddpd ymm6, ymm6, ymm5, ymm3 ;; R1 - R3 * sine (new R3)	; R1 - R2 * sine (new R2)	; 8-12

	vmovapd	ymm2, [screg+128]			;; Sine
	yfnmaddpd ymm3, ymm1, ymm2, ymm7 ;; R5 - R7 * sine (new R6)	; R3 - R4 * sine (new negI4)	;	11-15
	yfnmaddpd ymm0, ymm8, ymm2, ymm4 ;; R6 - R8 * sine (new R8)	; I3 - I4 * sine (new R4)	; 11-15

	yfmaddpd ymm1, ymm1, ymm2, ymm7 ;; R5 + R7 * sine (new R5)	; R3 + R4 * sine (new R3)	;	12-16
	yfmaddpd ymm8, ymm8, ymm2, ymm4 ;; R6 + R8 * sine (new R7)	; I3 + I4 * sine (new I3)	; 12-16

	vblendpd ymm4, ymm9, ymm6, 1	;; new R3			; new I1			;		13		
	vblendpd ymm6, ymm6, ymm9, 1	;; new R2			; new R2			;		14

	yfmaddpd ymm13, ymm3, ymm15, ymm0 ;; R6 = R6 + R8		; meaningless			;	16-20
	yfmsubpd ymm9, ymm3, ymm15, ymm0 ;; negR8 = R6 - R8		; meaningless			; 16-20

	yfmsubpd ymm11, ymm12, ymm15, ymm1 ;; R1 - R5 (final R5)	; R1 - R3 (final R3)		;	17-21
	yfmaddpd ymm12, ymm12, ymm15, ymm1 ;; R1 + R5 (final R1)	; R1 + R3 (final R1)		; 17-21

	yfmaddpd ymm1, ymm4, ymm15, ymm8 ;; R3 + R7 (final R3)		; I1 + I3 (final I1)		;	18-22
	yfmsubpd ymm4, ymm4, ymm15, ymm8 ;; R3 - R7 (final R7)		; I1 - I3 (final I3)		; 18-22

	vblendpd ymm0, ymm0, ymm13, 1	;; new R6/SQRTHALF		; new R4			;		21

	yfnmaddpd ymm8, ymm0, ymm14, ymm6 ;; R2 - R6*SQRTHALF (final R6) ; R2 - R4*1 (final R4)		;	22-26
	yfmaddpd ymm0, ymm0, ymm14, ymm6 ;; R2 + R6*SQRTHALF (final R2)	; R2 + R4*1 (final R2)		; 22-26
	vblendpd ymm3, ymm3, ymm9, 1	;; new negR8/SQRTHALF		; new negI4			;		22
	ystore	dst, ymm12		;; Store R1							;		22

	yfnmaddpd ymm9, ymm3, ymm14, ymm10 ;; R4 - negR8*SQRTHALF (final R4) ; I2 - negI4*1 (final I2)	;	23-27
	yfmaddpd ymm3, ymm3, ymm14, ymm10 ;; R4 + negR8*SQRTHALF (final R8) ; I2 + negI4*1 (final I4)	; 23-27
	vblendpd ymm5, ymm11, ymm1, 1	;; final R3			; final R3			;		23

	vblendpd ymm6, ymm1, ymm11, 1	;; final R5			; final I1			;		24
	vblendpd ymm2, ymm9, ymm8, 1	;; final R6			; final I2			;		28
	vblendpd ymm7, ymm8, ymm9, 1	;; final R4			; final R4			;		29
	ENDM

ENDIF

ENDIF

;;
;; ********************************* one-eight-reals-three-four-complex-fft-with-square variants ***************************************
;;

;; Macro to do one eight_reals_fft and three four_complex_fft in the final levels of an FFT.
;; The eight-reals data uses only one quarter of a YMM register.  The rest of the YMM register
;; contains four-complex data.  We handle this problem somewhat inefficiently, but this macro
;; is called only once during an FFT.

yr4_4cl_eight_reals_four_complex_fft_final MACRO srcreg,srcinc,d1,d2
	yr4_o8r_t4c_simple_fft_mem [srcreg],[srcreg+d1],[srcreg+d2],[srcreg+d2+d1],[srcreg+32],[srcreg+d1+32],[srcreg+d2+32],[srcreg+d2+d1+32],[srcreg+d2+d1],[srcreg+d2+d1+32]
	ystore	[srcreg], ymm3			;; Save R1
	ystore	[srcreg+32], ymm0		;; Save I1
	ystore	[srcreg+d1], ymm2		;; Save R2
	ystore	[srcreg+d1+32], ymm6		;; Save I2
	ystore	[srcreg+d2], ymm1		;; Save R3
	ystore	[srcreg+d2+32], ymm5		;; Save I3
;;	ystore	[srcreg+d2+d1], ymm4		;; Save R4
;;	ystore	[srcreg+d2+d1+32], ymm7		;; Save I4
	bump	srcreg, srcinc
	ENDM

yr4_4cl_eight_reals_four_complex_with_square MACRO srcreg,srcinc,d1,d2
	yr4_o8r_t4c_simple_fft_mem [srcreg],[srcreg+d1],[srcreg+d2],[srcreg+d2+d1],[srcreg+32],[srcreg+d1+32],[srcreg+d2+32],[srcreg+d2+d1+32],[srcreg+d2+d1],[srcreg+d2+d1+32]
	yp_complex_square ymm2, ymm6, ymm4	;; Square R2, I2
	yp_complex_square ymm1, ymm5, ymm4	;; Square R3, I3
	ystore	[srcreg+d1], ymm2		;; Save R2
	ystore	[srcreg+d1+32], ymm6		;; Save I2
	ystore	[srcreg+d2], ymm1		;; Save R3
	ystore	[srcreg+d2+32], ymm5		;; Save I3
	vmulsd	xmm4, xmm3, xmm3		;; Square R1 low value
	vmovsd	Q [srcreg-16], xmm4		;; Save square of sum of FFT values
	vmulsd	xmm7, xmm0, xmm0		;; Square I1 low value (it is actually a real value)
	yp_complex_square ymm3, ymm0, ymm5	;; Square R1, I1 (3 high values)
	vblendpd ymm3, ymm3, ymm4, 1		;; Blend R1 real and complex results
	vblendpd ymm0, ymm0, ymm7, 1		;; Blend I1 real and complex results
	ystore	[srcreg], ymm3			;; Save R1
	ystore	[srcreg+32], ymm0		;; Save I1
	vmovapd	ymm0, [srcreg+d2+d1]		;; Reload R4
	vmovapd	ymm6, [srcreg+d2+d1+32]		;; Reload I4
	yp_complex_square ymm0, ymm6, ymm5	;; Square R4, I4
	ystore	[srcreg+d2+d1], ymm0		;; Save R4
	ystore	[srcreg+d2+d1+32], ymm6		;; Save I4
	yr4_o8r_t4c_simple_unfft_mem [srcreg],[srcreg+32],[srcreg+d1],[srcreg+d1+32],[srcreg+d2],[srcreg+d2+32],[srcreg+d2+d1],[srcreg+d2+d1+32],[srcreg]
;;	ystore	[srcreg], ymm0			;; Save R1
	ystore	[srcreg+32], ymm4		;; Save I1
	ystore	[srcreg+d1], ymm5		;; Save R2
	ystore	[srcreg+d1+32], ymm7		;; Save I2
	ystore	[srcreg+d2], ymm2		;; Save R3
	ystore	[srcreg+d2+32], ymm3		;; Save I3
	ystore	[srcreg+d2+d1], ymm1		;; Save R4
	ystore	[srcreg+d2+d1+32], ymm6		;; Save I4
	bump	srcreg, srcinc
	ENDM

yr4_4cl_eight_reals_four_complex_with_mult MACRO srcreg,srcinc,d1,d2
	yr4_o8r_t4c_simple_fft_mem [srcreg],[srcreg+d1],[srcreg+d2],[srcreg+d2+d1],[srcreg+32],[srcreg+d1+32],[srcreg+d2+32],[srcreg+d2+d1+32],[srcreg+d2+d1],[srcreg+d2+d1+32]
	yp_complex_mult ymm2, ymm6, [srcreg+d1][rbp], [srcreg+d1+32][rbp], ymm4, ymm7 ;; Mult R2, I2
	yp_complex_mult ymm1, ymm5, [srcreg+d2][rbp], [srcreg+d2+32][rbp], ymm4, ymm7 ;; Mult R3, I3
	ystore	[srcreg+d1], ymm2		;; Save R2
	ystore	[srcreg+d1+32], ymm6		;; Save I2
	ystore	[srcreg+d2], ymm1		;; Save R3
	ystore	[srcreg+d2+32], ymm5		;; Save I3
	vmovapd	ymm6, [srcreg][rbp]		;; Load other R1
	vmulsd	xmm2, xmm3, xmm6		;; Mult R1 low values
	vmovsd	Q [srcreg-16], xmm2		;; Save product of sums of FFT values
	vmovapd	ymm7, [srcreg+32][rbp]		;; Load other I1
	vmulsd	xmm1, xmm0, xmm7		;; Mult I1 low values (they are actually real values)
	yp_complex_mult ymm3, ymm0, ymm6, ymm7, ymm4, ymm5 ;; Mult R1, I1 (3 high values)
	vblendpd ymm3, ymm3, ymm2, 1		;; Blend R1 real and complex results
	vblendpd ymm0, ymm0, ymm1, 1		;; Blend I1 real and complex results
	ystore	[srcreg], ymm3			;; Save R1
	ystore	[srcreg+32], ymm0		;; Save I1
	vmovapd	ymm0, [srcreg+d2+d1]		;; Reload R4
	vmovapd	ymm6, [srcreg+d2+d1+32]		;; Reload I4
	yp_complex_mult ymm0, ymm6, [srcreg+d2+d1][rbp], [srcreg+d2+d1+32][rbp], ymm4, ymm5 ;; Mult R4, I4
	ystore	[srcreg+d2+d1], ymm0		;; Save R4
	ystore	[srcreg+d2+d1+32], ymm6		;; Save I4
	yr4_o8r_t4c_simple_unfft_mem [srcreg],[srcreg+32],[srcreg+d1],[srcreg+d1+32],[srcreg+d2],[srcreg+d2+32],[srcreg+d2+d1],[srcreg+d2+d1+32],[srcreg]
;;	ystore	[srcreg], ymm0			;; Save R1
	ystore	[srcreg+32], ymm4		;; Save I1
	ystore	[srcreg+d1], ymm5		;; Save R2
	ystore	[srcreg+d1+32], ymm7		;; Save I2
	ystore	[srcreg+d2], ymm2		;; Save R3
	ystore	[srcreg+d2+32], ymm3		;; Save I3
	ystore	[srcreg+d2+d1], ymm1		;; Save R4
	ystore	[srcreg+d2+d1+32], ymm6		;; Save I4
	bump	srcreg, srcinc
	ENDM

;; 64-bit version

IFDEF X86_64

yr4_4cl_eight_reals_four_complex_with_mult MACRO srcreg,srcinc,d1,d2
	LOCAL	orig, back_to_orig, nofftsave, disptab, addmul, submul, muladd, muladdhard, mulsub, mulsubhard, unfft, done
	LOCAL	addmul312, addmul313, addmul132, addmul133, submul312, submul313, submul132, submul133

						;; Eight-reals comments		; Three four-complex comments
	vmovapd	ymm6, [srcreg+d2+d1]		;; R4				; R4
	vmovapd	ymm7, [srcreg+d2+d1+32]		;; R8				; I4
	vsubsd	xmm0, xmm6, xmm7		;; interim R8 = R4 - R8		; meaningless

	vmovapd	ymm1, [srcreg+d1]		;; R2				; R2
	vmovapd	ymm5, [srcreg+d1+32]		;; R6				; I2
	vsubsd	xmm2, xmm1, xmm5		;; interim R6 = R2 - R6		; meaningless

	vmulsd	xmm0, xmm0, Q YMM_SQRTHALF	;; interim R8 * square root	; meaningless
	vmulsd	xmm2, xmm2, Q YMM_SQRTHALF	;; interim R6 * square root	; meaningless

	vblendpd ymm3, ymm5, ymm6, 1		;; R4				; I2
	vaddpd	ymm3, ymm3, ymm7		;; new R4 = R4 + R8		; I2 + I4 (new I2)

	vblendpd ymm4, ymm5, ymm0, 1		;; interim R8			; I2
	vblendpd ymm7, ymm7, ymm2, 1		;; interim R6			; I4
	vsubpd	ymm4, ymm4, ymm7		;; new negR6 = interim R8 - R6	; I2 - I4 (new I4)

	vaddsd	xmm2, xmm2, xmm0		;; new R8 = interim R6 + R8	; meaningless

	vsubpd	ymm7, ymm1, ymm6		;; meaningless			; R2 - R4 (new R4)
	vblendpd ymm7, ymm7, ymm2, 1		;; new R8			; new R4

	vaddsd	xmm5, xmm1, xmm5		;; new R2 = R2 + R6		; meaningless

	vaddpd	ymm1, ymm1, ymm6		;; meaningless			; R2 + R4 (new R2)

	vmovapd	ymm0, [srcreg+d2]		;; R3				; R3
	vmovapd	ymm2, [srcreg+32]		;; R5				; I1
	vblendpd ymm6, ymm0, ymm2, 1		;; R5				; R3
	vblendpd ymm2, ymm2, ymm0, 1		;; R3				; I1

	vmovapd	ymm9, [srcreg+d2+32]		;; R7				; R7
	vaddpd	ymm0, ymm2, ymm9		;; new R3 = R3 + R7		; I1 + I3 (new I1)
	vsubpd	ymm2, ymm2, ymm9		;; new R7 = R3 - R7		; I1 - I3 (new I3)

	vblendpd ymm1, ymm1, ymm0, 1		;; new R3			; new R2
	vblendpd ymm0, ymm0, ymm5, 1		;; new R2			; new I1

	vaddpd	ymm5, ymm2, ymm7		;; R7 + R8 (final I3)		; I3 + R4 (final I3)
	vsubpd	ymm2, ymm2, ymm7		;; R7 - R8 (final I4)		; I3 - R4 (final I4)

	vmovapd	ymm7, [srcreg]			;; R1				; R1
	vaddpd	ymm8, ymm7, ymm6		;; new R1 = R1 + R5		; R1 + R3 (new R1)
	vsubpd	ymm7, ymm7, ymm6		;; new R5 = R1 - R5		; R1 - R3 (new R3)

	vsubpd	ymm6, ymm0, ymm3		;; R2 - R4 (new R4, final I2)	; I1 - I2 (final I2)
	vaddpd	ymm0, ymm0, ymm3		;; R2 + R4 (new R2)		; I1 + I2 (final I1)

	vaddpd	ymm3, ymm8, ymm1		;; R1 + R3 (new R1)		; R1 + R2 (final R1)
	vsubpd	ymm8, ymm8, ymm1		;; R1 - R3 (new R3, final R2)	; R1 - R2 (final R2)

	vsubpd	ymm1, ymm7, ymm4		;; R5 - negR6 (final R3)	; R3 - I4 (final R3)
	vaddpd	ymm7, ymm7, ymm4		;; R5 + negR6 (final R4)	; R3 + I4 (final R4)

	vsubsd	xmm4, xmm3, xmm0		;; R1 - R2 (final I1, a.k.a 2nd real result)
	vaddsd	xmm9, xmm3, xmm0		;; R1 + R2 (final R1)		; meaningless
	vblendpd ymm0, ymm0, ymm4, 1		;; final I1			; final I1
	vblendpd ymm3, ymm3, ymm9, 1		;; final R1			; final R1

	movzx	r10, mul4_opcode		;; Load opcode
	cmp	r10, 80h			;; Test for FFT save
	jb	nofftsave			;; opcode < 0x80, plain,addmul,submul,muladd,mulsub,unfft without FFT save
	;;ja	fftsave				;; opcode >= 0x80, plain,addmul,submul,muladd,mulsub,unfft with FFT save

	;; Store FFT results
	ystore	[srcreg], ymm3			;; R1			R1a
	ystore	[srcreg+32], ymm0		;; I1			R1b
	ystore	[srcreg+d1], ymm8		;; R2
	ystore	[srcreg+d1+32], ymm6		;; I2
	ystore	[srcreg+d2], ymm1		;; R3
	ystore	[srcreg+d2+32], ymm5		;; I3
	ystore	[srcreg+d2+d1], ymm7		;; R4
	ystore	[srcreg+d2+d1+32], ymm2		;; I4
	and	r10, 7fh			;; Strip the fft save bit from mul4_opcode

nofftsave:
	mov	r9, OFFSET disptab		;; Jump table address
	mov	r10, [r9+r10*8]			;; Jump table offset
	mov	r9, SRC2ARG			;; Distance to s2 arg from gwaddmul4 or gwsubmul4 or s3 arg in gwmuladd4 or gwmulsub4
	jmp	r10
disptab:
	DQ	OFFSET orig			;; 0 = plain
	DQ	OFFSET addmul			;; 1 = addmul
	DQ	OFFSET submul			;; 2 = submul
	DQ	OFFSET muladd			;; 3 = muladd
	DQ	OFFSET mulsub			;; 4 = mulsub
	DQ	OFFSET unfft			;; 5 = unfft
	DQ	OFFSET addmul312		;; 6 = addmul FFTval mem1 mem2
	DQ	OFFSET addmul313		;; 7 = addmul FFTval mem1 FFTval
	DQ	OFFSET addmul132		;; 8 = addmul mem1 FFTval mem2
	DQ	OFFSET addmul133		;; 9 = addmul mem1 FFTval FFTval
	DQ	OFFSET submul312		;; 10 = submul FFTval mem1 mem2
	DQ	OFFSET submul313		;; 11 = submul FFTval mem1 FFTval
	DQ	OFFSET submul132		;; 12 = submul mem1 FFTval mem2
	DQ	OFFSET submul133		;; 13 = submul mem1 FFTval FFTval

addmul:	vmovapd	ymm14, [srcreg+d1][rbp]		;; MemR2
	vaddpd	ymm14, ymm14, [srcreg+d1][r9]	;; MemR2
	vmovapd	ymm15, [srcreg+d1+32][rbp]	;; MemI2
	vaddpd	ymm15, ymm15, [srcreg+d1+32][r9];; MemI2
	vmulpd	ymm4, ymm8, ymm14		;; R2 * MemR2
	vmulpd	ymm8, ymm8, ymm15		;; R2 * MemI2
	vmulpd	ymm9, ymm6, ymm15		;; I2 * MemI2
	vmulpd	ymm6, ymm6, ymm14		;; I2 * MemR2
	vsubpd	ymm4, ymm4, ymm9		;; R2*MemR2 - I2*MemI2 (new R2)
	vaddpd	ymm8, ymm8, ymm6		;; R2*MemI2 + I2*MemR2 (new I2)

	vmovapd	ymm14, [srcreg+d2][rbp]		;; MemR3
	vaddpd	ymm14, ymm14, [srcreg+d2][r9]	;; MemR3
	vmovapd	ymm15, [srcreg+d2+32][rbp]	;; MemI3
	vaddpd	ymm15, ymm15, [srcreg+d2+32][r9];; MemI3
	vmulpd	ymm6, ymm1, ymm14		;; R3 * MemR3
	vmulpd	ymm1, ymm1, ymm15		;; R3 * MemI3
	vmulpd	ymm9, ymm5, ymm15		;; I3 * MemI3
	vmulpd	ymm5, ymm5, ymm14		;; I3 * MemR3
	vsubpd	ymm6, ymm6, ymm9		;; R3*MemR3 - I3*MemI3 (new R3)
	vaddpd	ymm1, ymm1, ymm5		;; R3*MemI3 + I3*MemR3 (new I3)

	vmovapd	ymm14, [srcreg+d2+d1][rbp]	;; MemR4
	vaddpd	ymm14, ymm14, [srcreg+d2+d1][r9];; MemR4
	vmovapd	ymm15, [srcreg+d2+d1+32][rbp]	;; MemI4
	vaddpd	ymm15, ymm15, [srcreg+d2+d1+32][r9];; MemI4
	vmulpd	ymm5, ymm7, ymm14		;; R4 * MemR4
	vmulpd	ymm7, ymm7, ymm15		;; R4 * MemI4
	vmulpd	ymm9, ymm2, ymm15		;; I4 * MemI4
	vmulpd	ymm2, ymm2, ymm14		;; I4 * MemR4
	vsubpd	ymm5, ymm5, ymm9		;; R4*MemR4 - I4*MemI4 (new R4)
	vaddpd	ymm7, ymm7, ymm2		;; R4*MemI4 + I4*MemR4 (new I4)

	vmovapd	ymm14, [srcreg][rbp]		;; MemR1
	vaddpd	ymm14, ymm14, [srcreg][r9]	;; MemR1
	vmovapd	ymm15, [srcreg+32][rbp]		;; MemI1
	vaddpd	ymm15, ymm15, [srcreg+32][r9]	;; MemI1
	vmulpd	ymm2, ymm3, ymm14		;; R1 * MemR1		(new real R1)
	vmulpd	ymm3, ymm3, ymm15		;; R1 * MemI1
	vmulpd	ymm9, ymm0, ymm15		;; I1 * MemI1		(new real I1)
	vmulpd	ymm0, ymm0, ymm14		;; I1 * MemR1
	vsubpd	ymm10, ymm2, ymm9		;; R1*MemR1 - I1*MemI1 (new complex R1)
	vaddpd	ymm3, ymm3, ymm0		;; R1*MemI1 + I1*MemR1 (new complex I1)
	jmp	back_to_orig

addmul133:
addmul313:
	vaddpd	ymm14, ymm8, [srcreg+d1][rbp]	;; R2+MemR2
	vaddpd	ymm15, ymm6, [srcreg+d1+32][rbp];; I2+MemI2
	vmulpd	ymm4, ymm8, ymm14		;; R2 * MemR2
	vmulpd	ymm8, ymm8, ymm15		;; R2 * MemI2
	vmulpd	ymm9, ymm6, ymm15		;; I2 * MemI2
	vmulpd	ymm6, ymm6, ymm14		;; I2 * MemR2
	vsubpd	ymm4, ymm4, ymm9		;; R2*MemR2 - I2*MemI2 (new R2)
	vaddpd	ymm8, ymm8, ymm6		;; R2*MemI2 + I2*MemR2 (new I2)

	vaddpd	ymm14, ymm1, [srcreg+d2][rbp]	;; R3+MemR3
	vaddpd	ymm15, ymm5, [srcreg+d2+32][rbp];; I3+MemI3
	vmulpd	ymm6, ymm1, ymm14		;; R3 * MemR3
	vmulpd	ymm1, ymm1, ymm15		;; R3 * MemI3
	vmulpd	ymm9, ymm5, ymm15		;; I3 * MemI3
	vmulpd	ymm5, ymm5, ymm14		;; I3 * MemR3
	vsubpd	ymm6, ymm6, ymm9		;; R3*MemR3 - I3*MemI3 (new R3)
	vaddpd	ymm1, ymm1, ymm5		;; R3*MemI3 + I3*MemR3 (new I3)

	vaddpd	ymm14, ymm7, [srcreg+d2+d1][rbp];; R4+MemR4
	vaddpd	ymm15, ymm2, [srcreg+d2+d1+32][rbp];; I4+MemI4
	vmulpd	ymm5, ymm7, ymm14		;; R4 * MemR4
	vmulpd	ymm7, ymm7, ymm15		;; R4 * MemI4
	vmulpd	ymm9, ymm2, ymm15		;; I4 * MemI4
	vmulpd	ymm2, ymm2, ymm14		;; I4 * MemR4
	vsubpd	ymm5, ymm5, ymm9		;; R4*MemR4 - I4*MemI4 (new R4)
	vaddpd	ymm7, ymm7, ymm2		;; R4*MemI4 + I4*MemR4 (new I4)

	vaddpd	ymm14, ymm3, [srcreg][rbp]	;; R1+MemR1
	vaddpd	ymm15, ymm0, [srcreg+32][rbp]	;; I1+MemI1
	vmulpd	ymm2, ymm3, ymm14		;; R1 * MemR1		(new real R1)
	vmulpd	ymm3, ymm3, ymm15		;; R1 * MemI1
	vmulpd	ymm9, ymm0, ymm15		;; I1 * MemI1		(new real I1)
	vmulpd	ymm0, ymm0, ymm14		;; I1 * MemR1
	vsubpd	ymm10, ymm2, ymm9		;; R1*MemR1 - I1*MemI1 (new complex R1)
	vaddpd	ymm3, ymm3, ymm0		;; R1*MemI1 + I1*MemR1 (new complex I1)
	jmp	back_to_orig

addmul132:
addmul312:
	vaddpd	ymm14, ymm8, [srcreg+d1][rbp]	;; R2+MemR2
	vmovapd	ymm8, [srcreg+d1][r9]		;; R2=MemR2
	vaddpd	ymm15, ymm6, [srcreg+d1+32][rbp];; I2+MemI2
	vmovapd	ymm6, [srcreg+d1+32][r9]	;; I2=MemI2
	vmulpd	ymm4, ymm8, ymm14		;; R2 * MemR2
	vmulpd	ymm8, ymm8, ymm15		;; R2 * MemI2
	vmulpd	ymm9, ymm6, ymm15		;; I2 * MemI2
	vmulpd	ymm6, ymm6, ymm14		;; I2 * MemR2
	vsubpd	ymm4, ymm4, ymm9		;; R2*MemR2 - I2*MemI2 (new R2)
	vaddpd	ymm8, ymm8, ymm6		;; R2*MemI2 + I2*MemR2 (new I2)

	vaddpd	ymm14, ymm1, [srcreg+d2][rbp]	;; R3+MemR3
	vmovapd	ymm1, [srcreg+d2][r9]		;; R3=MemR3
	vaddpd	ymm15, ymm5, [srcreg+d2+32][rbp];; I3+MemI3
	vmovapd	ymm5, [srcreg+d2+32][r9]	;; I3=MemI3
	vmulpd	ymm6, ymm1, ymm14		;; R3 * MemR3
	vmulpd	ymm1, ymm1, ymm15		;; R3 * MemI3
	vmulpd	ymm9, ymm5, ymm15		;; I3 * MemI3
	vmulpd	ymm5, ymm5, ymm14		;; I3 * MemR3
	vsubpd	ymm6, ymm6, ymm9		;; R3*MemR3 - I3*MemI3 (new R3)
	vaddpd	ymm1, ymm1, ymm5		;; R3*MemI3 + I3*MemR3 (new I3)

	vaddpd	ymm14, ymm7, [srcreg+d2+d1][rbp];; R4+MemR4
	vmovapd	ymm7, [srcreg+d2+d1][r9]	;; R4=MemR4
	vaddpd	ymm15, ymm2, [srcreg+d2+d1+32][rbp];; I4+MemI4
	vmovapd	ymm2, [srcreg+d2+d1+32][r9]	;; I4=MemI4
	vmulpd	ymm5, ymm7, ymm14		;; R4 * MemR4
	vmulpd	ymm7, ymm7, ymm15		;; R4 * MemI4
	vmulpd	ymm9, ymm2, ymm15		;; I4 * MemI4
	vmulpd	ymm2, ymm2, ymm14		;; I4 * MemR4
	vsubpd	ymm5, ymm5, ymm9		;; R4*MemR4 - I4*MemI4 (new R4)
	vaddpd	ymm7, ymm7, ymm2		;; R4*MemI4 + I4*MemR4 (new I4)

	vaddpd	ymm14, ymm3, [srcreg][rbp]	;; R1+MemR1
	vmovapd	ymm3, [srcreg][r9]		;; R1=MemR1
	vaddpd	ymm15, ymm0, [srcreg+32][rbp]	;; I1+MemI1
	vmovapd	ymm0, [srcreg+32][r9]		;; I1=MemI1
	vmulpd	ymm2, ymm3, ymm14		;; R1 * MemR1		(new real R1)
	vmulpd	ymm3, ymm3, ymm15		;; R1 * MemI1
	vmulpd	ymm9, ymm0, ymm15		;; I1 * MemI1		(new real I1)
	vmulpd	ymm0, ymm0, ymm14		;; I1 * MemR1
	vsubpd	ymm10, ymm2, ymm9		;; R1*MemR1 - I1*MemI1 (new complex R1)
	vaddpd	ymm3, ymm3, ymm0		;; R1*MemI1 + I1*MemR1 (new complex I1)
	jmp	back_to_orig

submul:	vmovapd	ymm14, [srcreg+d1][rbp]		;; MemR2
	vsubpd	ymm14, ymm14, [srcreg+d1][r9]	;; MemR2
	vmovapd	ymm15, [srcreg+d1+32][rbp]	;; MemI2
	vsubpd	ymm15, ymm15, [srcreg+d1+32][r9];; MemI2
	vmulpd	ymm4, ymm8, ymm14		;; R2 * MemR2
	vmulpd	ymm8, ymm8, ymm15		;; R2 * MemI2
	vmulpd	ymm9, ymm6, ymm15		;; I2 * MemI2
	vmulpd	ymm6, ymm6, ymm14		;; I2 * MemR2
	vsubpd	ymm4, ymm4, ymm9		;; R2*MemR2 - I2*MemI2 (new R2)
	vaddpd	ymm8, ymm8, ymm6		;; R2*MemI2 + I2*MemR2 (new I2)

	vmovapd	ymm14, [srcreg+d2][rbp]		;; MemR3
	vsubpd	ymm14, ymm14, [srcreg+d2][r9]	;; MemR3
	vmovapd	ymm15, [srcreg+d2+32][rbp]	;; MemI3
	vsubpd	ymm15, ymm15, [srcreg+d2+32][r9];; MemI3
	vmulpd	ymm6, ymm1, ymm14		;; R3 * MemR3
	vmulpd	ymm1, ymm1, ymm15		;; R3 * MemI3
	vmulpd	ymm9, ymm5, ymm15		;; I3 * MemI3
	vmulpd	ymm5, ymm5, ymm14		;; I3 * MemR3
	vsubpd	ymm6, ymm6, ymm9		;; R3*MemR3 - I3*MemI3 (new R3)
	vaddpd	ymm1, ymm1, ymm5		;; R3*MemI3 + I3*MemR3 (new I3)

	vmovapd	ymm14, [srcreg+d2+d1][rbp]	;; MemR4
	vsubpd	ymm14, ymm14, [srcreg+d2+d1][r9];; MemR4
	vmovapd	ymm15, [srcreg+d2+d1+32][rbp]	;; MemI4
	vsubpd	ymm15, ymm15, [srcreg+d2+d1+32][r9];; MemI4
	vmulpd	ymm5, ymm7, ymm14		;; R4 * MemR4
	vmulpd	ymm7, ymm7, ymm15		;; R4 * MemI4
	vmulpd	ymm9, ymm2, ymm15		;; I4 * MemI4
	vmulpd	ymm2, ymm2, ymm14		;; I4 * MemR4
	vsubpd	ymm5, ymm5, ymm9		;; R4*MemR4 - I4*MemI4 (new R4)
	vaddpd	ymm7, ymm7, ymm2		;; R4*MemI4 + I4*MemR4 (new I4)

	vmovapd	ymm14, [srcreg][rbp]		;; MemR1
	vsubpd	ymm14, ymm14, [srcreg][r9]	;; MemR1
	vmovapd	ymm15, [srcreg+32][rbp]		;; MemI1
	vsubpd	ymm15, ymm15, [srcreg+32][r9]	;; MemI1
	vmulpd	ymm2, ymm3, ymm14		;; R1 * MemR1		(new real R1)
	vmulpd	ymm3, ymm3, ymm15		;; R1 * MemI1
	vmulpd	ymm9, ymm0, ymm15		;; I1 * MemI1		(new real I1)
	vmulpd	ymm0, ymm0, ymm14		;; I1 * MemR1
	vsubpd	ymm10, ymm2, ymm9		;; R1*MemR1 - I1*MemI1 (new complex R1)
	vaddpd	ymm3, ymm3, ymm0		;; R1*MemI1 + I1*MemR1 (new complex I1)
	jmp	back_to_orig

submul133:
	vmovapd	ymm14, [srcreg+d1][rbp]		;; MemR2
	vsubpd	ymm14, ymm14, ymm8		;; MemR2-R2
	vmovapd	ymm15, [srcreg+d1+32][rbp]	;; MemI2
	vsubpd	ymm15, ymm15, ymm6		;; MemI2-I2
	vmulpd	ymm4, ymm8, ymm14		;; R2 * MemR2
	vmulpd	ymm8, ymm8, ymm15		;; R2 * MemI2
	vmulpd	ymm9, ymm6, ymm15		;; I2 * MemI2
	vmulpd	ymm6, ymm6, ymm14		;; I2 * MemR2
	vsubpd	ymm4, ymm4, ymm9		;; R2*MemR2 - I2*MemI2 (new R2)
	vaddpd	ymm8, ymm8, ymm6		;; R2*MemI2 + I2*MemR2 (new I2)

	vmovapd	ymm14, [srcreg+d2][rbp]		;; MemR3
	vsubpd	ymm14, ymm14, ymm1		;; MemR3-R3
	vmovapd	ymm15, [srcreg+d2+32][rbp]	;; MemI3
	vsubpd	ymm15, ymm15, ymm5		;; MemI3-I3
	vmulpd	ymm6, ymm1, ymm14		;; R3 * MemR3
	vmulpd	ymm1, ymm1, ymm15		;; R3 * MemI3
	vmulpd	ymm9, ymm5, ymm15		;; I3 * MemI3
	vmulpd	ymm5, ymm5, ymm14		;; I3 * MemR3
	vsubpd	ymm6, ymm6, ymm9		;; R3*MemR3 - I3*MemI3 (new R3)
	vaddpd	ymm1, ymm1, ymm5		;; R3*MemI3 + I3*MemR3 (new I3)

	vmovapd	ymm14, [srcreg+d2+d1][rbp]	;; MemR4
	vsubpd	ymm14, ymm14, ymm7		;; MemR4-R4
	vmovapd	ymm15, [srcreg+d2+d1+32][rbp]	;; MemI4
	vsubpd	ymm15, ymm15, ymm2		;; MemI4-I4
	vmulpd	ymm5, ymm7, ymm14		;; R4 * MemR4
	vmulpd	ymm7, ymm7, ymm15		;; R4 * MemI4
	vmulpd	ymm9, ymm2, ymm15		;; I4 * MemI4
	vmulpd	ymm2, ymm2, ymm14		;; I4 * MemR4
	vsubpd	ymm5, ymm5, ymm9		;; R4*MemR4 - I4*MemI4 (new R4)
	vaddpd	ymm7, ymm7, ymm2		;; R4*MemI4 + I4*MemR4 (new I4)

	vmovapd	ymm14, [srcreg][rbp]		;; MemR1
	vsubpd	ymm14, ymm14, ymm3		;; MemR1-R1
	vmovapd	ymm15, [srcreg+32][rbp]		;; MemI1
	vsubpd	ymm15, ymm15, ymm0		;; MemI1-I1
	vmulpd	ymm2, ymm3, ymm14		;; R1 * MemR1		(new real R1)
	vmulpd	ymm3, ymm3, ymm15		;; R1 * MemI1
	vmulpd	ymm9, ymm0, ymm15		;; I1 * MemI1		(new real I1)
	vmulpd	ymm0, ymm0, ymm14		;; I1 * MemR1
	vsubpd	ymm10, ymm2, ymm9		;; R1*MemR1 - I1*MemI1 (new complex R1)
	vaddpd	ymm3, ymm3, ymm0		;; R1*MemI1 + I1*MemR1 (new complex I1)
	jmp	back_to_orig

submul313:
	vsubpd	ymm14, ymm8, [srcreg+d1][rbp]	;; R2-MemR2
	vsubpd	ymm15, ymm6, [srcreg+d1+32][rbp];; I2-MemI2
	vmulpd	ymm4, ymm8, ymm14		;; R2 * MemR2
	vmulpd	ymm8, ymm8, ymm15		;; R2 * MemI2
	vmulpd	ymm9, ymm6, ymm15		;; I2 * MemI2
	vmulpd	ymm6, ymm6, ymm14		;; I2 * MemR2
	vsubpd	ymm4, ymm4, ymm9		;; R2*MemR2 - I2*MemI2 (new R2)
	vaddpd	ymm8, ymm8, ymm6		;; R2*MemI2 + I2*MemR2 (new I2)

	vsubpd	ymm14, ymm1, [srcreg+d2][rbp]	;; R3-MemR3
	vsubpd	ymm15, ymm5, [srcreg+d2+32][rbp];; I3-MemI3
	vmulpd	ymm6, ymm1, ymm14		;; R3 * MemR3
	vmulpd	ymm1, ymm1, ymm15		;; R3 * MemI3
	vmulpd	ymm9, ymm5, ymm15		;; I3 * MemI3
	vmulpd	ymm5, ymm5, ymm14		;; I3 * MemR3
	vsubpd	ymm6, ymm6, ymm9		;; R3*MemR3 - I3*MemI3 (new R3)
	vaddpd	ymm1, ymm1, ymm5		;; R3*MemI3 + I3*MemR3 (new I3)

	vsubpd	ymm14, ymm7, [srcreg+d2+d1][rbp];; R4-MemR4
	vsubpd	ymm15, ymm2, [srcreg+d2+d1+32][rbp];; I4-MemI4
	vmulpd	ymm5, ymm7, ymm14		;; R4 * MemR4
	vmulpd	ymm7, ymm7, ymm15		;; R4 * MemI4
	vmulpd	ymm9, ymm2, ymm15		;; I4 * MemI4
	vmulpd	ymm2, ymm2, ymm14		;; I4 * MemR4
	vsubpd	ymm5, ymm5, ymm9		;; R4*MemR4 - I4*MemI4 (new R4)
	vaddpd	ymm7, ymm7, ymm2		;; R4*MemI4 + I4*MemR4 (new I4)

	vsubpd	ymm14, ymm3, [srcreg][rbp]	;; R1-MemR1
	vsubpd	ymm15, ymm0, [srcreg+32][rbp]	;; I1-MemI1
	vmulpd	ymm2, ymm3, ymm14		;; R1 * MemR1		(new real R1)
	vmulpd	ymm3, ymm3, ymm15		;; R1 * MemI1
	vmulpd	ymm9, ymm0, ymm15		;; I1 * MemI1		(new real I1)
	vmulpd	ymm0, ymm0, ymm14		;; I1 * MemR1
	vsubpd	ymm10, ymm2, ymm9		;; R1*MemR1 - I1*MemI1 (new complex R1)
	vaddpd	ymm3, ymm3, ymm0		;; R1*MemI1 + I1*MemR1 (new complex I1)
	jmp	back_to_orig

submul132:
	vmovapd	ymm14, [srcreg+d1][rbp]		;; MemR2
	vsubpd	ymm14, ymm14, ymm8		;; MemR2-R2
	vmovapd	ymm8, [srcreg+d1][r9]		;; R2=MemR2
	vmovapd	ymm15, [srcreg+d1+32][rbp]	;; MemI2
	vsubpd	ymm15, ymm15, ymm6		;; MemI2-I2
	vmovapd	ymm6, [srcreg+d1+32][r9]	;; I2=MemI2
	vmulpd	ymm4, ymm8, ymm14		;; R2 * MemR2
	vmulpd	ymm8, ymm8, ymm15		;; R2 * MemI2
	vmulpd	ymm9, ymm6, ymm15		;; I2 * MemI2
	vmulpd	ymm6, ymm6, ymm14		;; I2 * MemR2
	vsubpd	ymm4, ymm4, ymm9		;; R2*MemR2 - I2*MemI2 (new R2)
	vaddpd	ymm8, ymm8, ymm6		;; R2*MemI2 + I2*MemR2 (new I2)

	vmovapd	ymm14, [srcreg+d2][rbp]		;; MemR3
	vsubpd	ymm14, ymm14, ymm1		;; MemR3-R3
	vmovapd	ymm1, [srcreg+d2][r9]		;; R3=MemR3
	vmovapd	ymm15, [srcreg+d2+32][rbp]	;; MemI3
	vsubpd	ymm15, ymm15, ymm5		;; MemI3-I3
	vmovapd	ymm5, [srcreg+d2+32][r9]	;; I3=MemI3
	vmulpd	ymm6, ymm1, ymm14		;; R3 * MemR3
	vmulpd	ymm1, ymm1, ymm15		;; R3 * MemI3
	vmulpd	ymm9, ymm5, ymm15		;; I3 * MemI3
	vmulpd	ymm5, ymm5, ymm14		;; I3 * MemR3
	vsubpd	ymm6, ymm6, ymm9		;; R3*MemR3 - I3*MemI3 (new R3)
	vaddpd	ymm1, ymm1, ymm5		;; R3*MemI3 + I3*MemR3 (new I3)

	vmovapd	ymm14, [srcreg+d2+d1][rbp]	;; MemR4
	vsubpd	ymm14, ymm14, ymm7		;; MemR4-R4
	vmovapd	ymm7, [srcreg+d2+d1][r9]	;; R4=MemR4
	vmovapd	ymm15, [srcreg+d2+d1+32][rbp]	;; MemI4
	vsubpd	ymm15, ymm15, ymm2		;; MemI4-I4
	vmovapd	ymm2, [srcreg+d2+d1+32][r9]	;; I4=MemI4
	vmulpd	ymm5, ymm7, ymm14		;; R4 * MemR4
	vmulpd	ymm7, ymm7, ymm15		;; R4 * MemI4
	vmulpd	ymm9, ymm2, ymm15		;; I4 * MemI4
	vmulpd	ymm2, ymm2, ymm14		;; I4 * MemR4
	vsubpd	ymm5, ymm5, ymm9		;; R4*MemR4 - I4*MemI4 (new R4)
	vaddpd	ymm7, ymm7, ymm2		;; R4*MemI4 + I4*MemR4 (new I4)

	vmovapd	ymm14, [srcreg][rbp]		;; MemR1
	vsubpd	ymm14, ymm14, ymm3		;; MemR1-R1
	vmovapd	ymm3, [srcreg][r9]		;; R1=MemR1
	vmovapd	ymm15, [srcreg+32][rbp]		;; MemI1
	vsubpd	ymm15, ymm15, ymm0		;; MemI1-I1
	vmovapd	ymm0, [srcreg+32][r9]		;; I1=MemI1
	vmulpd	ymm2, ymm3, ymm14		;; R1 * MemR1		(new real R1)
	vmulpd	ymm3, ymm3, ymm15		;; R1 * MemI1
	vmulpd	ymm9, ymm0, ymm15		;; I1 * MemI1		(new real I1)
	vmulpd	ymm0, ymm0, ymm14		;; I1 * MemR1
	vsubpd	ymm10, ymm2, ymm9		;; R1*MemR1 - I1*MemI1 (new complex R1)
	vaddpd	ymm3, ymm3, ymm0		;; R1*MemI1 + I1*MemR1 (new complex I1)
	jmp	back_to_orig

submul312:
	vsubpd	ymm14, ymm8, [srcreg+d1][rbp]	;; R2-MemR2
	vmovapd	ymm8, [srcreg+d1][r9]		;; R2=MemR2
	vsubpd	ymm15, ymm6, [srcreg+d1+32][rbp];; I2-MemI2
	vmovapd	ymm6, [srcreg+d1+32][r9]	;; I2=MemI2
	vmulpd	ymm4, ymm8, ymm14		;; R2 * MemR2
	vmulpd	ymm8, ymm8, ymm15		;; R2 * MemI2
	vmulpd	ymm9, ymm6, ymm15		;; I2 * MemI2
	vmulpd	ymm6, ymm6, ymm14		;; I2 * MemR2
	vsubpd	ymm4, ymm4, ymm9		;; R2*MemR2 - I2*MemI2 (new R2)
	vaddpd	ymm8, ymm8, ymm6		;; R2*MemI2 + I2*MemR2 (new I2)

	vsubpd	ymm14, ymm1, [srcreg+d2][rbp]	;; R3-MemR3
	vmovapd	ymm1, [srcreg+d2][r9]		;; R3=MemR3
	vsubpd	ymm15, ymm5, [srcreg+d2+32][rbp];; I3-MemI3
	vmovapd	ymm5, [srcreg+d2+32][r9]	;; I3=MemI3
	vmulpd	ymm6, ymm1, ymm14		;; R3 * MemR3
	vmulpd	ymm1, ymm1, ymm15		;; R3 * MemI3
	vmulpd	ymm9, ymm5, ymm15		;; I3 * MemI3
	vmulpd	ymm5, ymm5, ymm14		;; I3 * MemR3
	vsubpd	ymm6, ymm6, ymm9		;; R3*MemR3 - I3*MemI3 (new R3)
	vaddpd	ymm1, ymm1, ymm5		;; R3*MemI3 + I3*MemR3 (new I3)

	vsubpd	ymm14, ymm7, [srcreg+d2+d1][rbp];; R4-MemR4
	vmovapd	ymm7, [srcreg+d2+d1][r9]	;; R4=MemR4
	vsubpd	ymm15, ymm2, [srcreg+d2+d1+32][rbp];; I4-MemI4
	vmovapd	ymm2, [srcreg+d2+d1+32][r9]	;; I4=MemI4
	vmulpd	ymm5, ymm7, ymm14		;; R4 * MemR4
	vmulpd	ymm7, ymm7, ymm15		;; R4 * MemI4
	vmulpd	ymm9, ymm2, ymm15		;; I4 * MemI4
	vmulpd	ymm2, ymm2, ymm14		;; I4 * MemR4
	vsubpd	ymm5, ymm5, ymm9		;; R4*MemR4 - I4*MemI4 (new R4)
	vaddpd	ymm7, ymm7, ymm2		;; R4*MemI4 + I4*MemR4 (new I4)

	vsubpd	ymm14, ymm3, [srcreg][rbp]	;; R1-MemR1
	vmovapd	ymm3, [srcreg][r9]		;; R1=MemR1
	vsubpd	ymm15, ymm0, [srcreg+32][rbp]	;; I1-MemI1
	vmovapd	ymm0, [srcreg+32][r9]		;; I1=MemI1
	vmulpd	ymm2, ymm3, ymm14		;; R1 * MemR1		(new real R1)
	vmulpd	ymm3, ymm3, ymm15		;; R1 * MemI1
	vmulpd	ymm9, ymm0, ymm15		;; I1 * MemI1		(new real I1)
	vmulpd	ymm0, ymm0, ymm14		;; I1 * MemR1
	vsubpd	ymm10, ymm2, ymm9		;; R1*MemR1 - I1*MemI1 (new complex R1)
	vaddpd	ymm3, ymm3, ymm0		;; R1*MemI1 + I1*MemR1 (new complex I1)
	jmp	back_to_orig

unfft:	vmovapd ymm10, ymm3			;; R1
	vmovapd ymm3, ymm0			;; I1
	vmovapd ymm4, ymm8			;; R2
	vmovapd ymm8, ymm6			;; I2
	vmovapd ymm6, ymm1			;; R3
	vmovapd ymm1, ymm5			;; I3
	vmovapd ymm5, ymm7			;; R4
	vmovapd ymm7, ymm2			;; I4
	vmovapd	ymm2, ymm10			;; R1 (back_to_orig uses low double)
	vmovapd	ymm9, ymm3			;; I1 (back_to_orig uses low double)
	jmp	back_to_orig

muladd:	vmovapd	ymm14, [srcreg+d1][rbp]		;; MemR2
	vmovapd	ymm15, [srcreg+d1+32][rbp]	;; MemI2
	vmulpd	ymm4, ymm8, ymm14		;; R2 * MemR2
	vmulpd	ymm8, ymm8, ymm15		;; R2 * MemI2
	vmulpd	ymm9, ymm6, ymm15		;; I2 * MemI2
	vmulpd	ymm6, ymm6, ymm14		;; I2 * MemR2
	vsubpd	ymm4, ymm4, ymm9		;; R2*MemR2 - I2*MemI2 (new R2)
	vaddpd	ymm8, ymm8, ymm6		;; R2*MemI2 + I2*MemR2 (new I2)

	vmovapd	ymm14, [srcreg+d2][rbp]		;; MemR3
	vmovapd	ymm15, [srcreg+d2+32][rbp]	;; MemI3
	vmulpd	ymm6, ymm1, ymm14		;; R3 * MemR3
	vmulpd	ymm1, ymm1, ymm15		;; R3 * MemI3
	vmulpd	ymm9, ymm5, ymm15		;; I3 * MemI3
	vmulpd	ymm5, ymm5, ymm14		;; I3 * MemR3
	vsubpd	ymm6, ymm6, ymm9		;; R3*MemR3 - I3*MemI3 (new R3)
	vaddpd	ymm1, ymm1, ymm5		;; R3*MemI3 + I3*MemR3 (new I3)

	vmovapd	ymm14, [srcreg+d2+d1][rbp]	;; MemR4
	vmovapd	ymm15, [srcreg+d2+d1+32][rbp]	;; MemI4
	vmulpd	ymm5, ymm7, ymm14		;; R4 * MemR4
	vmulpd	ymm7, ymm7, ymm15		;; R4 * MemI4
	vmulpd	ymm9, ymm2, ymm15		;; I4 * MemI4
	vmulpd	ymm2, ymm2, ymm14		;; I4 * MemR4
	vsubpd	ymm5, ymm5, ymm9		;; R4*MemR4 - I4*MemI4 (new R4)
	vaddpd	ymm7, ymm7, ymm2		;; R4*MemI4 + I4*MemR4 (new I4)

	vmovapd	ymm14, [srcreg][rbp]		;; MemR1
	vmovapd	ymm15, [srcreg+32][rbp]		;; MemI1
	vmulpd	ymm2, ymm3, ymm14		;; R1 * MemR1		(new real R1)
	vmulpd	ymm3, ymm3, ymm15		;; R1 * MemI1
	vmulpd	ymm9, ymm0, ymm15		;; I1 * MemI1		(new real I1)
	vmulpd	ymm0, ymm0, ymm14		;; I1 * MemR1
	vsubpd	ymm10, ymm2, ymm9		;; R1*MemR1 - I1*MemI1 (new complex R1)
	vaddpd	ymm3, ymm3, ymm0		;; R1*MemI1 + I1*MemR1 (new complex I1)

	mov	r10, SRC3ARG			;; Load distance to FFT(1)
	test	r10, 3				;; Check if we need to multiply add-in by FFT(1)
	jz	muladdhard			;; Yes, do it the hard way

	vaddpd	ymm4, ymm4, [srcreg+r9+d1]		;; R2 = R2 + MemR2
	vaddpd	ymm8, ymm8, [srcreg+r9+d1+32]		;; I2 = I2 + MemI2
	vaddpd	ymm6, ymm6, [srcreg+r9+d2]		;; R3 = R3 + MemR3
	vaddpd	ymm1, ymm1, [srcreg+r9+d2+32]		;; I3 = I3 + MemI3
	vaddpd	ymm5, ymm5, [srcreg+r9+d2+d1]		;; R4 = R4 + MemR4
	vaddpd	ymm7, ymm7, [srcreg+r9+d2+d1+32]	;; I4 = I4 + MemI4
	vmovapd	ymm0, [srcreg+r9]			;; MemR1
	vaddpd	ymm10, ymm10, ymm0			;; R1 = R1 + MemR1 (lower word is irrelevant)
	vaddpd	ymm2, ymm2, ymm0			;; R1a = R1a + MemR1a (upper 3 words are irrelevant)
	vmovapd	ymm0, [srcreg+r9+32]			;; MemI1
	vaddpd	ymm3, ymm3, ymm0			;; I1 = I1 + MemI1 (lower word is irrelevant)
	vaddpd	ymm9, ymm9, ymm0			;; R1b = R1b + MemR1b in MemI1 (upper 3 words are irrelevant)
	jmp	back_to_orig

muladdhard:
	vmovapd	ymm0, [srcreg+r9+d1]			;; MemR2
	vmovapd	ymm12, [srcreg+r10+d1]			;; MemR2#2
	vmulpd	ymm11, ymm0, ymm12			;; MemR2*MemR2#2
	vaddpd	ymm4, ymm4, ymm11			;; R2 = R2 + MemR2*MemR2#2
	vmovapd	ymm13, [srcreg+r10+d1+32]		;; MemI2#2
	vmulpd	ymm11, ymm0, ymm13			;; MemR2*MemI2#2
	vaddpd	ymm8, ymm8, ymm11			;; I2 = I2 + MemR2*MemI2#2
	vmovapd	ymm0, [srcreg+r9+d1+32]			;; MemI2
	vmulpd	ymm11, ymm0, ymm13			;; MemI2*MemI2#2
	vsubpd	ymm4, ymm4, ymm11			;; R2 = R2 - MemI2*MemI2#2
	vmulpd	ymm11, ymm0, ymm12			;; MemI2*MemR2#2
	vaddpd	ymm8, ymm8, ymm11			;; I2 = I2 + MemI2*MemR2#2

	vmovapd	ymm0, [srcreg+r9+d2]			;; MemR3
	vmovapd	ymm12, [srcreg+r10+d2]			;; MemR3#2
	vmulpd	ymm11, ymm0, ymm12			;; MemR3*MemR3#2
	vaddpd	ymm6, ymm6, ymm11			;; R3 = R3 + MemR3*MemR3#2
	vmovapd	ymm13, [srcreg+r10+d2+32]		;; MemI3#2
	vmulpd	ymm11, ymm0, ymm13			;; MemR3*MemI3#2
	vaddpd	ymm1, ymm1, ymm11			;; I3 = I3 + MemR3*MemI3#2
	vmovapd	ymm0, [srcreg+r9+d2+32]			;; MemI3
	vmulpd	ymm11, ymm0, ymm13			;; MemI3*MemI3#2
	vsubpd	ymm6, ymm6, ymm11			;; R3 = R3 - MemI3*MemI3#2
	vmulpd	ymm11, ymm0, ymm12			;; MemI3*MemR3#2
	vaddpd	ymm1, ymm1, ymm11			;; I3 = I3 + MemI3*MemR3#2

	vmovapd	ymm0, [srcreg+r9+d2+d1]			;; MemR4
	vmovapd	ymm12, [srcreg+r10+d2+d1]		;; MemR4#2
	vmulpd	ymm11, ymm0, ymm12			;; MemR4*MemR4#2
	vaddpd	ymm5, ymm5, ymm11			;; R4 = R4 + MemR4*MemR4#2
	vmovapd	ymm13, [srcreg+r10+d2+d1+32]		;; MemI4#2
	vmulpd	ymm11, ymm0, ymm13			;; MemR4*MemI4#2
	vaddpd	ymm7, ymm7, ymm11			;; I4 = I4 + MemR4*MemI4#2
	vmovapd	ymm0, [srcreg+r9+d2+d1+32]		;; MemI4
	vmulpd	ymm11, ymm0, ymm13			;; MemI4*MemI4#2
	vsubpd	ymm5, ymm5, ymm11			;; R4 = R4 - MemI4*MemI4#2
	vmulpd	ymm11, ymm0, ymm12			;; MemI4*MemR4#2
	vaddpd	ymm7, ymm7, ymm11			;; I4 = I4 + MemI4*MemR4#2

	vmovapd	ymm0, [srcreg+r9]			;; MemR1
	vmovapd	ymm12, [srcreg+r10]			;; MemR1#2
	vmulpd	ymm11, ymm0, ymm12			;; MemR1*MemR1#2
	vaddpd	ymm10, ymm10, ymm11			;; R1 = R1 + MemR1*MemR1#2 (lower word is irrelevant)
	vaddpd	ymm2, ymm2, ymm11			;; R1a = R1a + MemR1*MemR1#2 (upper 3 words are irrelevant)
	vmovapd	ymm13, [srcreg+r10+32]			;; MemI1#2
	vmulpd	ymm11, ymm0, ymm13			;; MemR1*MemI1#2
	vaddpd	ymm3, ymm3, ymm11			;; I1 = I1 + MemR1*MemI1#2 (lower word is irrelevant)
	vmovapd	ymm0, [srcreg+r9+32]			;; MemI1
	vmulpd	ymm11, ymm0, ymm13			;; MemI1*MemI1#2
	vsubpd	ymm10, ymm10, ymm11			;; R1 = R1 - MemI1*MemI1#2
	vaddpd	ymm9, ymm9, ymm11			;; R1b = R1b + MemR1b in MemI1*MemI1#2 (upper 3 words are irrelevant)
	vmulpd	ymm11, ymm0, ymm12			;; MemI1*MemR1#2
	vaddpd	ymm3, ymm3, ymm11			;; I1 = I1 + MemI1*MemR1#2 (lower word is irrelevant)
	jmp	back_to_orig

mulsub:	vmovapd	ymm14, [srcreg+d1][rbp]		;; MemR2
	vmovapd	ymm15, [srcreg+d1+32][rbp]	;; MemI2
	vmulpd	ymm4, ymm8, ymm14		;; R2 * MemR2
	vmulpd	ymm8, ymm8, ymm15		;; R2 * MemI2
	vmulpd	ymm9, ymm6, ymm15		;; I2 * MemI2
	vmulpd	ymm6, ymm6, ymm14		;; I2 * MemR2
	vsubpd	ymm4, ymm4, ymm9		;; R2*MemR2 - I2*MemI2 (new R2)
	vaddpd	ymm8, ymm8, ymm6		;; R2*MemI2 + I2*MemR2 (new I2)

	vmovapd	ymm14, [srcreg+d2][rbp]		;; MemR3
	vmovapd	ymm15, [srcreg+d2+32][rbp]	;; MemI3
	vmulpd	ymm6, ymm1, ymm14		;; R3 * MemR3
	vmulpd	ymm1, ymm1, ymm15		;; R3 * MemI3
	vmulpd	ymm9, ymm5, ymm15		;; I3 * MemI3
	vmulpd	ymm5, ymm5, ymm14		;; I3 * MemR3
	vsubpd	ymm6, ymm6, ymm9		;; R3*MemR3 - I3*MemI3 (new R3)
	vaddpd	ymm1, ymm1, ymm5		;; R3*MemI3 + I3*MemR3 (new I3)

	vmovapd	ymm14, [srcreg+d2+d1][rbp]	;; MemR4
	vmovapd	ymm15, [srcreg+d2+d1+32][rbp]	;; MemI4
	vmulpd	ymm5, ymm7, ymm14		;; R4 * MemR4
	vmulpd	ymm7, ymm7, ymm15		;; R4 * MemI4
	vmulpd	ymm9, ymm2, ymm15		;; I4 * MemI4
	vmulpd	ymm2, ymm2, ymm14		;; I4 * MemR4
	vsubpd	ymm5, ymm5, ymm9		;; R4*MemR4 - I4*MemI4 (new R4)
	vaddpd	ymm7, ymm7, ymm2		;; R4*MemI4 + I4*MemR4 (new I4)

	vmovapd	ymm14, [srcreg][rbp]		;; MemR1
	vmovapd	ymm15, [srcreg+32][rbp]		;; MemI1
	vmulpd	ymm2, ymm3, ymm14		;; R1 * MemR1		(new real R1)
	vmulpd	ymm3, ymm3, ymm15		;; R1 * MemI1
	vmulpd	ymm9, ymm0, ymm15		;; I1 * MemI1		(new real I1)
	vmulpd	ymm0, ymm0, ymm14		;; I1 * MemR1
	vsubpd	ymm10, ymm2, ymm9		;; R1*MemR1 - I1*MemI1 (new complex R1)
	vaddpd	ymm3, ymm3, ymm0		;; R1*MemI1 + I1*MemR1 (new complex I1)

	mov	r10, SRC3ARG			;; Load distance to FFT(1)
	test	r10, 3				;; Check if we need to multiply add-in by FFT(1)
	jz	mulsubhard			;; Yes, do it the hard way

	vsubpd	ymm4, ymm4, [srcreg+r9+d1]		;; R2 = R2 - MemR2
	vsubpd	ymm8, ymm8, [srcreg+r9+d1+32]		;; I2 = I2 - MemI2
	vsubpd	ymm6, ymm6, [srcreg+r9+d2]		;; R3 = R3 - MemR3
	vsubpd	ymm1, ymm1, [srcreg+r9+d2+32]		;; I3 = I3 - MemI3
	vsubpd	ymm5, ymm5, [srcreg+r9+d2+d1]		;; R4 = R4 - MemR4
	vsubpd	ymm7, ymm7, [srcreg+r9+d2+d1+32]	;; I4 = I4 - MemI4
	vmovapd	ymm0, [srcreg+r9]			;; MemR1
	vsubpd	ymm10, ymm10, ymm0			;; R1 = R1 - MemR1 (lower word is irrelevant)
	vsubpd	ymm2, ymm2, ymm0			;; R1a = R1a - MemR1a (upper 3 words are irrelevant)
	vmovapd	ymm0, [srcreg+r9+32]			;; MemI1
	vsubpd	ymm3, ymm3, ymm0			;; I1 = I1 - MemI1 (lower word is irrelevant)
	vsubpd	ymm9, ymm9, ymm0			;; R1b = R1b - MemR1b in MemI1 (upper 3 words are irrelevant)
	jmp	back_to_orig

mulsubhard:
	vmovapd	ymm0, [srcreg+r9+d1]			;; MemR2
	vmovapd	ymm12, [srcreg+r10+d1]			;; MemR2#2
	vmulpd	ymm11, ymm0, ymm12			;; MemR2*MemR2#2
	vsubpd	ymm4, ymm4, ymm11			;; R2 = R2 - MemR2*MemR2#2
	vmovapd	ymm13, [srcreg+r10+d1+32]		;; MemI2#2
	vmulpd	ymm11, ymm0, ymm13			;; MemR2*MemI2#2
	vsubpd	ymm8, ymm8, ymm11			;; I2 = I2 - MemR2*MemI2#2
	vmovapd	ymm0, [srcreg+r9+d1+32]			;; MemI2
	vmulpd	ymm11, ymm0, ymm13			;; MemI2*MemI2#2
	vaddpd	ymm4, ymm4, ymm11			;; R2 = R2 + MemI2*MemI2#2
	vmulpd	ymm11, ymm0, ymm12			;; MemI2*MemR2#2
	vsubpd	ymm8, ymm8, ymm11			;; I2 = I2 - MemI2*MemR2#2

	vmovapd	ymm0, [srcreg+r9+d2]			;; MemR3
	vmovapd	ymm12, [srcreg+r10+d2]			;; MemR3#2
	vmulpd	ymm11, ymm0, ymm12			;; MemR3*MemR3#2
	vsubpd	ymm6, ymm6, ymm11			;; R3 = R3 - MemR3*MemR3#2
	vmovapd	ymm13, [srcreg+r10+d2+32]		;; MemI3#2
	vmulpd	ymm11, ymm0, ymm13			;; MemR3*MemI3#2
	vsubpd	ymm1, ymm1, ymm11			;; I3 = I3 - MemR3*MemI3#2
	vmovapd	ymm0, [srcreg+r9+d2+32]			;; MemI3
	vmulpd	ymm11, ymm0, ymm13			;; MemI3*MemI3#2
	vaddpd	ymm6, ymm6, ymm11			;; R3 = R3 + MemI3*MemI3#2
	vmulpd	ymm11, ymm0, ymm12			;; MemI3*MemR3#2
	vsubpd	ymm1, ymm1, ymm11			;; I3 = I3 - MemI3*MemR3#2

	vmovapd	ymm0, [srcreg+r9+d2+d1]			;; MemR4
	vmovapd	ymm12, [srcreg+r10+d2+d1]		;; MemR4#2
	vmulpd	ymm11, ymm0, ymm12			;; MemR4*MemR4#2
	vsubpd	ymm5, ymm5, ymm11			;; R4 = R4 - MemR4*MemR4#2
	vmovapd	ymm13, [srcreg+r10+d2+d1+32]		;; MemI4#2
	vmulpd	ymm11, ymm0, ymm13			;; MemR4*MemI4#2
	vsubpd	ymm7, ymm7, ymm11			;; I4 = I4 - MemR4*MemI4#2
	vmovapd	ymm0, [srcreg+r9+d2+d1+32]		;; MemI4
	vmulpd	ymm11, ymm0, ymm13			;; MemI4*MemI4#2
	vaddpd	ymm5, ymm5, ymm11			;; R4 = R4 + MemI4*MemI4#2
	vmulpd	ymm11, ymm0, ymm12			;; MemI4*MemR4#2
	vsubpd	ymm7, ymm7, ymm11			;; I4 = I4 - MemI4*MemR4#2

	vmovapd	ymm0, [srcreg+r9]			;; MemR1
	vmovapd	ymm12, [srcreg+r10]			;; MemR1#2
	vmulpd	ymm11, ymm0, ymm12			;; MemR1*MemR1#2
	vsubpd	ymm10, ymm10, ymm11			;; R1 = R1 - MemR1*MemR1#2 (lower word is irrelevant)
	vsubpd	ymm2, ymm2, ymm11			;; R1a = R1a - MemR1*MemR1#2 (upper 3 words are irrelevant)
	vmovapd	ymm13, [srcreg+r10+32]			;; MemI1#2
	vmulpd	ymm11, ymm0, ymm13			;; MemR1*MemI1#2
	vsubpd	ymm3, ymm3, ymm11			;; I1 = I1 - MemR1*MemI1#2 (lower word is irrelevant)
	vmovapd	ymm0, [srcreg+r9+32]			;; MemI1
	vmulpd	ymm11, ymm0, ymm13			;; MemI1*MemI1#2
	vaddpd	ymm10, ymm10, ymm11			;; R1 = R1 + MemI1*MemI1#2
	vsubpd	ymm9, ymm9, ymm11			;; R1b = R1b - MemR1b in MemI1*MemI1#2 (upper 3 words are irrelevant)
	vmulpd	ymm11, ymm0, ymm12			;; MemI1*MemR1#2
	vsubpd	ymm3, ymm3, ymm11			;; I1 = I1 - MemI1*MemR1#2 (lower word is irrelevant)
	jmp	back_to_orig

orig:	vmovapd	ymm14, [srcreg+d1][rbp]		;; MemR2
	vmovapd	ymm15, [srcreg+d1+32][rbp]	;; MemI2
	vmulpd	ymm4, ymm8, ymm14		;; R2 * MemR2
	vmulpd	ymm8, ymm8, ymm15		;; R2 * MemI2
	vmulpd	ymm9, ymm6, ymm15		;; I2 * MemI2
	vmulpd	ymm6, ymm6, ymm14		;; I2 * MemR2
	vsubpd	ymm4, ymm4, ymm9		;; R2*MemR2 - I2*MemI2 (new R2)
	vaddpd	ymm8, ymm8, ymm6		;; R2*MemI2 + I2*MemR2 (new I2)

	vmovapd	ymm14, [srcreg+d2][rbp]		;; MemR3
	vmovapd	ymm15, [srcreg+d2+32][rbp]	;; MemI3
	vmulpd	ymm6, ymm1, ymm14		;; R3 * MemR3
	vmulpd	ymm1, ymm1, ymm15		;; R3 * MemI3
	vmulpd	ymm9, ymm5, ymm15		;; I3 * MemI3
	vmulpd	ymm5, ymm5, ymm14		;; I3 * MemR3
	vsubpd	ymm6, ymm6, ymm9		;; R3*MemR3 - I3*MemI3 (new R3)
	vaddpd	ymm1, ymm1, ymm5		;; R3*MemI3 + I3*MemR3 (new I3)

	vmovapd	ymm14, [srcreg+d2+d1][rbp]	;; MemR4
	vmovapd	ymm15, [srcreg+d2+d1+32][rbp]	;; MemI4
	vmulpd	ymm5, ymm7, ymm14		;; R4 * MemR4
	vmulpd	ymm7, ymm7, ymm15		;; R4 * MemI4
	vmulpd	ymm9, ymm2, ymm15		;; I4 * MemI4
	vmulpd	ymm2, ymm2, ymm14		;; I4 * MemR4
	vsubpd	ymm5, ymm5, ymm9		;; R4*MemR4 - I4*MemI4 (new R4)
	vaddpd	ymm7, ymm7, ymm2		;; R4*MemI4 + I4*MemR4 (new I4)

	vmovapd	ymm14, [srcreg][rbp]		;; MemR1
	vmovapd	ymm15, [srcreg+32][rbp]		;; MemI1
	vmulpd	ymm2, ymm3, ymm14		;; R1 * MemR1		(new real R1)
	vmulpd	ymm3, ymm3, ymm15		;; R1 * MemI1
	vmulpd	ymm9, ymm0, ymm15		;; I1 * MemI1		(new real I1)
	vmulpd	ymm0, ymm0, ymm14		;; I1 * MemR1
	vsubpd	ymm10, ymm2, ymm9		;; R1*MemR1 - I1*MemI1 (new complex R1)
	vaddpd	ymm3, ymm3, ymm0		;; R1*MemI1 + I1*MemR1 (new complex I1)

back_to_orig:
	vmovsd	Q [srcreg+r8-16], xmm2		;; Save product of sums of FFT values
	vblendpd ymm2, ymm10, ymm2, 1		;; Blend R1 real and complex results
	vblendpd ymm3, ymm3, ymm9, 1		;; Blend I1 real and complex results

	;; Do the eight-reals part with R1,R2,R3,R4,R5,R6,R7,R8 terminology instead of R1,I1,R2,I2,R3,I3,R4,I4

						;; Eight-reals comments		; Three four-complex comments
	vaddpd	ymm0, ymm5, ymm6		;; new R5 = R7 + R5		; R4 + R3 (new R3)
	vsubpd	ymm5, ymm5, ymm6		;; new negR6 = R7 - R5		; R4 - R3 (new I4)

	vaddpd	ymm6, ymm1, ymm7		;; new R7 = R6 + R8		; I3 + I4 (new I3)
	vsubpd	ymm1, ymm1, ymm7		;; new R8 = R6 - R8		; I3 - I4 (new R4)

	vsubsd	xmm9, xmm2, xmm3		;; new R2 = R1 - R2
	vmulsd	xmm9, xmm9, Q YMM_HALF		;; Mul new R2 by HALF
	vaddsd	xmm10, xmm9, xmm3		;; new R1 = R1 + R2

	vsubsd	xmm11, xmm1, xmm5		;; R6 = R8 - negR6
	vaddsd	xmm12, xmm1, xmm5		;; R8 = R8 + negR6

	vblendpd ymm2, ymm2, ymm10, 1		;;	new R1				R1
	vaddpd	ymm7, ymm2, ymm4		;; newR1 + R3 (new R1)		; R1 + R2 (new R1)
	vsubpd	ymm2, ymm2, ymm4		;; newR1 - R3 (new R3)		; R1 - R2 (new R2)

	vblendpd ymm3, ymm3, ymm9, 1		;;	new R2				I1
	vaddpd	ymm4, ymm3, ymm8		;; newR2 + R4 (new R2)		; I1 + I2 (new I1)
	vsubpd	ymm3, ymm3, ymm8		;; newR2 - R4 (new R4)		; I1 - I2 (new I2)

	vmulsd	xmm11, xmm11, Q YMM_SQRTHALF	;; R6 = R6 * square root of 1/2
	vmulsd	xmm12, xmm12, Q YMM_SQRTHALF	;; R8 = R8 * square root of 1/2

	vaddpd	ymm8, ymm7, ymm0		;; R1 + R5 (final R1)		; R1 + R3 (final R1)
	vsubpd	ymm7, ymm7, ymm0		;; R1 - R5 (final R5)		; R1 - R3 (final R3)

	vblendpd ymm1, ymm1, ymm6, 1		;;	R7				R4
	vaddpd	ymm0, ymm2, ymm1		;; R3 + R7 (final R3)		; R2 + R4 (final R2)
	vsubpd	ymm2, ymm2, ymm1		;; R3 - R7 (final R7)		; R2 - R4 (final R4)

	vblendpd ymm6, ymm6, ymm11, 1		;;	R6				I3
	vaddpd	ymm1, ymm4, ymm6		;; R2 + R6 (final R2)		; I1 + I3 (final I1)
	vsubpd	ymm4, ymm4, ymm6		;; R2 - R6 (final R6)		; I1 - I3 (final I3)

	vblendpd ymm5, ymm5, ymm12, 1		;;	R8				I4
	vaddpd	ymm6, ymm3, ymm5		;; R4 + R8 (final R4)		; I2 + I4 (final I2)
	vsubpd	ymm3, ymm3, ymm5		;; R4 - R8 (final R8)		; I2 - I4 (final I4)

	vblendpd ymm5, ymm0, ymm1, 1		;;	R2				R2
	vblendpd ymm9, ymm7, ymm0, 1		;;	R3				R3
	vblendpd ymm0, ymm2, ymm6, 1		;;	R4				R4
	vblendpd ymm10, ymm1, ymm7, 1		;;	R5				I1
	vblendpd ymm7, ymm6, ymm4, 1		;;	R6				I2
	vblendpd ymm6, ymm4, ymm2, 1		;;	R7				I3

	ystore	[srcreg+r8], ymm8		;; Save R1
	ystore	[srcreg+r8+32], ymm10		;; Save I1
	ystore	[srcreg+r8+d1], ymm5		;; Save R2
	ystore	[srcreg+r8+d1+32], ymm7		;; Save I2
	ystore	[srcreg+r8+d2], ymm9		;; Save R3
	ystore	[srcreg+r8+d2+32], ymm6		;; Save I3
	ystore	[srcreg+r8+d2+d1], ymm0		;; Save R4
	ystore	[srcreg+r8+d2+d1+32], ymm3	;; Save I4
	bump	srcreg, srcinc
	ENDM

ENDIF

yr4_4cl_eight_reals_four_complex_with_mulf MACRO srcreg,srcinc,d1,d2
	vmovapd	ymm2, [srcreg+d1][rbx]		;; R2
	vmovapd	ymm1, [srcreg+d1+32][rbx]	;; I2
	yp_complex_mult ymm2, ymm1, [srcreg+d1][rbp], [srcreg+d1+32][rbp], ymm4, ymm5 ;; Mult R2, I2
	vmovapd	ymm7, [srcreg+d2][rbx]		;; R3
	vmovapd	ymm3, [srcreg+d2+32][rbx]	;; I3
	yp_complex_mult ymm7, ymm3, [srcreg+d2][rbp], [srcreg+d2+32][rbp], ymm4, ymm5 ;; Mult R3, I3
	vmovapd	ymm0, [srcreg+d2+d1][rbx]	;; R4
	vmovapd	ymm6, [srcreg+d2+d1+32][rbx]	;; I4
	yp_complex_mult ymm0, ymm6, [srcreg+d2+d1][rbp], [srcreg+d2+d1+32][rbp], ymm4, ymm5 ;; Mult R4, I4
	ystore	[srcreg+d1], ymm2		;; Save R2
	ystore	[srcreg+d1+32], ymm1		;; Save I2
	ystore	[srcreg+d2], ymm7		;; Save R3
	ystore	[srcreg+d2+32], ymm3		;; Save I3
	ystore	[srcreg+d2+d1], ymm0		;; Save R4
	ystore	[srcreg+d2+d1+32], ymm6		;; Save I4
	vmovapd	ymm2, [srcreg][rbx]		;; R1
	vmovapd	ymm6, [srcreg][rbp]		;; Other R1
	vmulsd	xmm0, xmm2, xmm6		;; Mult R1 low values
	vmovsd	Q [srcreg-16], xmm0		;; Save product of sums of FFT values
	vmovapd	ymm4, [srcreg+32][rbx]		;; I1
	vmovapd	ymm7, [srcreg+32][rbp]		;; Other I1
	vmulsd	xmm1, xmm4, xmm7		;; Mult I1 low values (they are actually real values)
	yp_complex_mult ymm2, ymm4, ymm6, ymm7, ymm3, ymm5 ;; Mult R1, I1 (3 high values)
	vblendpd ymm2, ymm2, ymm0, 1		;; Blend R1 real and complex results
	vblendpd ymm4, ymm4, ymm1, 1		;; Blend I1 real and complex results
	ystore	[srcreg], ymm2			;; Save R1
	ystore	[srcreg+32], ymm4		;; Save I1
	yr4_o8r_t4c_simple_unfft_mem [srcreg],[srcreg+32],[srcreg+d1],[srcreg+d1+32],[srcreg+d2],[srcreg+d2+32],[srcreg+d2+d1],[srcreg+d2+d1+32],[srcreg]
;;	ystore	[srcreg], ymm0			;; Save R1
	ystore	[srcreg+32], ymm4		;; Save I1
	ystore	[srcreg+d1], ymm5		;; Save R2
	ystore	[srcreg+d1+32], ymm7		;; Save I2
	ystore	[srcreg+d2], ymm2		;; Save R3
	ystore	[srcreg+d2+32], ymm3		;; Save I3
	ystore	[srcreg+d2+d1], ymm1		;; Save R4
	ystore	[srcreg+d2+d1+32], ymm6		;; Save I4
	bump	srcreg, srcinc
	ENDM

;; 64-bit version

IFDEF X86_64

yr4_4cl_eight_reals_four_complex_with_mulf MACRO srcreg,srcinc,d1,d2
	LOCAL	orig, back_to_orig, no_save_fft, fma_or_unfft, fma, addmul, submul, muladd, muladdhard, mulsub, mulsubhard, unfft

	mov	al, mul4_opcode			;; Load the mul4_opcode
	cmp	al, 0				;; See if we need to do more than the original type-4 FFT multiply
	je	orig				;; Jump if nothing special
	mov	r9, SRC2ARG			;; Distance to s2 arg from gwaddmul4 or gwsubmul4
	cmp	al, 2				;; Case off opcode
	jg	fma_or_unfft			;; 3,4,5=muladd,mulsub,unfft
	je	submul				;; 2=submul
	;jl	addmul				;; 1=addmul, fall through

addmul:	vmovapd	ymm14, [srcreg+d1][rbp]		;; MemR2
	vaddpd	ymm14, ymm14, [srcreg+d1][r9]	;; MemR2
	vmovapd	ymm15, [srcreg+d1+32][rbp]	;; MemI2
	vaddpd	ymm15, ymm15, [srcreg+d1+32][r9];; MemI2
	vmovapd	ymm8, [srcreg+d1][rbx]		;; R2
	vmovapd	ymm6, [srcreg+d1+32][rbx]	;; I2
	vmulpd	ymm4, ymm8, ymm14		;; R2 * MemR2
	vmulpd	ymm8, ymm8, ymm15		;; R2 * MemI2
	vmulpd	ymm9, ymm6, ymm15		;; I2 * MemI2
	vmulpd	ymm6, ymm6, ymm14		;; I2 * MemR2
	vsubpd	ymm4, ymm4, ymm9		;; R2*MemR2 - I2*MemI2 (new R2)
	vaddpd	ymm8, ymm8, ymm6		;; R2*MemI2 + I2*MemR2 (new I2)

	vmovapd	ymm14, [srcreg+d2][rbp]		;; MemR3
	vaddpd	ymm14, ymm14, [srcreg+d2][r9]	;; MemR3
	vmovapd	ymm15, [srcreg+d2+32][rbp]	;; MemI3
	vaddpd	ymm15, ymm15, [srcreg+d2+32][r9];; MemI3
	vmovapd	ymm1, [srcreg+d2][rbx]		;; R3
	vmovapd	ymm5, [srcreg+d2+32][rbx]	;; I3
	vmulpd	ymm6, ymm1, ymm14		;; R3 * MemR3
	vmulpd	ymm1, ymm1, ymm15		;; R3 * MemI3
	vmulpd	ymm9, ymm5, ymm15		;; I3 * MemI3
	vmulpd	ymm5, ymm5, ymm14		;; I3 * MemR3
	vsubpd	ymm6, ymm6, ymm9		;; R3*MemR3 - I3*MemI3 (new R3)
	vaddpd	ymm1, ymm1, ymm5		;; R3*MemI3 + I3*MemR3 (new I3)

	vmovapd	ymm14, [srcreg+d2+d1][rbp]	;; MemR4
	vaddpd	ymm14, ymm14, [srcreg+d2+d1][r9];; MemR4
	vmovapd	ymm15, [srcreg+d2+d1+32][rbp]	;; MemI4
	vaddpd	ymm15, ymm15, [srcreg+d2+d1+32][r9];; MemI4
	vmovapd	ymm7, [srcreg+d2+d1][rbx]	;; R4
	vmovapd	ymm2, [srcreg+d2+d1+32][rbx]	;; I4
	vmulpd	ymm5, ymm7, ymm14		;; R4 * MemR4
	vmulpd	ymm7, ymm7, ymm15		;; R4 * MemI4
	vmulpd	ymm9, ymm2, ymm15		;; I4 * MemI4
	vmulpd	ymm2, ymm2, ymm14		;; I4 * MemR4
	vsubpd	ymm5, ymm5, ymm9		;; R4*MemR4 - I4*MemI4 (new R4)
	vaddpd	ymm7, ymm7, ymm2		;; R4*MemI4 + I4*MemR4 (new I4)

	vmovapd	ymm14, [srcreg][rbp]		;; MemR1
	vaddpd	ymm14, ymm14, [srcreg][r9]	;; MemR1
	vmovapd	ymm15, [srcreg+32][rbp]		;; MemI1
	vaddpd	ymm15, ymm15, [srcreg+32][r9]	;; MemI1
	vmovapd	ymm3, [srcreg][rbx]		;; R1
	vmovapd	ymm0, [srcreg+32][rbx]		;; I1
	vmulpd	ymm2, ymm3, ymm14		;; R1 * MemR1		(new real R1)
	vmulpd	ymm3, ymm3, ymm15		;; R1 * MemI1
	vmulpd	ymm9, ymm0, ymm15		;; I1 * MemI1		(new real I1)
	vmulpd	ymm0, ymm0, ymm14		;; I1 * MemR1
	vsubpd	ymm10, ymm2, ymm9		;; R1*MemR1 - I1*MemI1 (new complex R1)
	vaddpd	ymm3, ymm3, ymm0		;; R1*MemI1 + I1*MemR1 (new complex I1)
	jmp	back_to_orig

submul:	vmovapd	ymm14, [srcreg+d1][rbp]		;; MemR2
	vsubpd	ymm14, ymm14, [srcreg+d1][r9]	;; MemR2
	vmovapd	ymm15, [srcreg+d1+32][rbp]	;; MemI2
	vsubpd	ymm15, ymm15, [srcreg+d1+32][r9];; MemI2
	vmovapd	ymm8, [srcreg+d1][rbx]		;; R2
	vmovapd	ymm6, [srcreg+d1+32][rbx]	;; I2
	vmulpd	ymm4, ymm8, ymm14		;; R2 * MemR2
	vmulpd	ymm8, ymm8, ymm15		;; R2 * MemI2
	vmulpd	ymm9, ymm6, ymm15		;; I2 * MemI2
	vmulpd	ymm6, ymm6, ymm14		;; I2 * MemR2
	vsubpd	ymm4, ymm4, ymm9		;; R2*MemR2 - I2*MemI2 (new R2)
	vaddpd	ymm8, ymm8, ymm6		;; R2*MemI2 + I2*MemR2 (new I2)

	vmovapd	ymm14, [srcreg+d2][rbp]		;; MemR3
	vsubpd	ymm14, ymm14, [srcreg+d2][r9]	;; MemR3
	vmovapd	ymm15, [srcreg+d2+32][rbp]	;; MemI3
	vsubpd	ymm15, ymm15, [srcreg+d2+32][r9];; MemI3
	vmovapd	ymm1, [srcreg+d2][rbx]		;; R3
	vmovapd	ymm5, [srcreg+d2+32][rbx]	;; I3
	vmulpd	ymm6, ymm1, ymm14		;; R3 * MemR3
	vmulpd	ymm1, ymm1, ymm15		;; R3 * MemI3
	vmulpd	ymm9, ymm5, ymm15		;; I3 * MemI3
	vmulpd	ymm5, ymm5, ymm14		;; I3 * MemR3
	vsubpd	ymm6, ymm6, ymm9		;; R3*MemR3 - I3*MemI3 (new R3)
	vaddpd	ymm1, ymm1, ymm5		;; R3*MemI3 + I3*MemR3 (new I3)

	vmovapd	ymm14, [srcreg+d2+d1][rbp]	;; MemR4
	vsubpd	ymm14, ymm14, [srcreg+d2+d1][r9];; MemR4
	vmovapd	ymm15, [srcreg+d2+d1+32][rbp]	;; MemI4
	vsubpd	ymm15, ymm15, [srcreg+d2+d1+32][r9];; MemI4
	vmovapd	ymm7, [srcreg+d2+d1][rbx]	;; R4
	vmovapd	ymm2, [srcreg+d2+d1+32][rbx]	;; I4
	vmulpd	ymm5, ymm7, ymm14		;; R4 * MemR4
	vmulpd	ymm7, ymm7, ymm15		;; R4 * MemI4
	vmulpd	ymm9, ymm2, ymm15		;; I4 * MemI4
	vmulpd	ymm2, ymm2, ymm14		;; I4 * MemR4
	vsubpd	ymm5, ymm5, ymm9		;; R4*MemR4 - I4*MemI4 (new R4)
	vaddpd	ymm7, ymm7, ymm2		;; R4*MemI4 + I4*MemR4 (new I4)

	vmovapd	ymm14, [srcreg][rbp]		;; MemR1
	vsubpd	ymm14, ymm14, [srcreg][r9]	;; MemR1
	vmovapd	ymm15, [srcreg+32][rbp]		;; MemI1
	vsubpd	ymm15, ymm15, [srcreg+32][r9]	;; MemI1
	vmovapd	ymm3, [srcreg][rbx]		;; R1
	vmovapd	ymm0, [srcreg+32][rbx]		;; I1
	vmulpd	ymm2, ymm3, ymm14		;; R1 * MemR1		(new real R1)
	vmulpd	ymm3, ymm3, ymm15		;; R1 * MemI1
	vmulpd	ymm9, ymm0, ymm15		;; I1 * MemI1		(new real I1)
	vmulpd	ymm0, ymm0, ymm14		;; I1 * MemR1
	vsubpd	ymm10, ymm2, ymm9		;; R1*MemR1 - I1*MemI1 (new complex R1)
	vaddpd	ymm3, ymm3, ymm0		;; R1*MemI1 + I1*MemR1 (new complex I1)
	jmp	back_to_orig

fma_or_unfft:
	cmp	al, 4				;; Case off opcode
	jbe	fma				;; 3=muladd, 4=mulsub, fma MUST NOT ALTER CONDITION CODES!
	;;ja	unfft				;; 5=unfft

unfft:	vmovapd	ymm10, [srcreg][rbx]		;; R1
	vmovapd	ymm3, [srcreg+32][rbx]		;; I1
	vmovapd	ymm4, [srcreg+d1][rbx]		;; R2
	vmovapd	ymm8, [srcreg+d1+32][rbx]	;; I2
	vmovapd	ymm6, [srcreg+d2][rbx]		;; R3
	vmovapd	ymm1, [srcreg+d2+32][rbx]	;; I3
	vmovapd	ymm5, [srcreg+d2+d1][rbx]	;; R4
	vmovapd	ymm7, [srcreg+d2+d1+32][rbx]	;; I4
	vmovapd	ymm2, ymm10			;; R1 (back_to_orig uses low double)
	vmovapd	ymm9, ymm3			;; I1 (back_to_orig uses low double)
	jmp	back_to_orig

fma:	vmovapd	ymm14, [srcreg+d1][rbp]		;; MemR2
	vmovapd	ymm15, [srcreg+d1+32][rbp]	;; MemI2
	vmovapd	ymm8, [srcreg+d1][rbx]		;; R2
	vmovapd	ymm6, [srcreg+d1+32][rbx]	;; I2
	vmulpd	ymm4, ymm8, ymm14		;; R2 * MemR2
	vmulpd	ymm8, ymm8, ymm15		;; R2 * MemI2
	vmulpd	ymm9, ymm6, ymm15		;; I2 * MemI2
	vmulpd	ymm6, ymm6, ymm14		;; I2 * MemR2
	vsubpd	ymm4, ymm4, ymm9		;; R2*MemR2 - I2*MemI2 (new R2)
	vaddpd	ymm8, ymm8, ymm6		;; R2*MemI2 + I2*MemR2 (new I2)

	vmovapd	ymm14, [srcreg+d2][rbp]		;; MemR3
	vmovapd	ymm15, [srcreg+d2+32][rbp]	;; MemI3
	vmovapd	ymm1, [srcreg+d2][rbx]		;; R3
	vmovapd	ymm5, [srcreg+d2+32][rbx]	;; I3
	vmulpd	ymm6, ymm1, ymm14		;; R3 * MemR3
	vmulpd	ymm1, ymm1, ymm15		;; R3 * MemI3
	vmulpd	ymm9, ymm5, ymm15		;; I3 * MemI3
	vmulpd	ymm5, ymm5, ymm14		;; I3 * MemR3
	vsubpd	ymm6, ymm6, ymm9		;; R3*MemR3 - I3*MemI3 (new R3)
	vaddpd	ymm1, ymm1, ymm5		;; R3*MemI3 + I3*MemR3 (new I3)

	vmovapd	ymm14, [srcreg+d2+d1][rbp]	;; MemR4
	vmovapd	ymm15, [srcreg+d2+d1+32][rbp]	;; MemI4
	vmovapd	ymm7, [srcreg+d2+d1][rbx]	;; R4
	vmovapd	ymm2, [srcreg+d2+d1+32][rbx]	;; I4
	vmulpd	ymm5, ymm7, ymm14		;; R4 * MemR4
	vmulpd	ymm7, ymm7, ymm15		;; R4 * MemI4
	vmulpd	ymm9, ymm2, ymm15		;; I4 * MemI4
	vmulpd	ymm2, ymm2, ymm14		;; I4 * MemR4
	vsubpd	ymm5, ymm5, ymm9		;; R4*MemR4 - I4*MemI4 (new R4)
	vaddpd	ymm7, ymm7, ymm2		;; R4*MemI4 + I4*MemR4 (new I4)

	vmovapd	ymm14, [srcreg][rbp]		;; MemR1
	vmovapd	ymm15, [srcreg+32][rbp]		;; MemI1
	vmovapd	ymm3, [srcreg][rbx]		;; R1
	vmovapd	ymm0, [srcreg+32][rbx]		;; I1
	vmulpd	ymm2, ymm3, ymm14		;; R1 * MemR1		(new real R1)
	vmulpd	ymm3, ymm3, ymm15		;; R1 * MemI1
	vmulpd	ymm9, ymm0, ymm15		;; I1 * MemI1		(new real I1)
	vmulpd	ymm0, ymm0, ymm14		;; I1 * MemR1
	vsubpd	ymm10, ymm2, ymm9		;; R1*MemR1 - I1*MemI1 (new complex R1)
	vaddpd	ymm3, ymm3, ymm0		;; R1*MemI1 + I1*MemR1 (new complex I1)

	mov	r10, SRC3ARG			;; Load distance to FFT(1)
	je	mulsub				;; 4=mulsub
	;jb	muladd				;; 3=muladd, fall through

muladd:	test	r10, 3				;; Check if we need to multiply add-in by FFT(1)
	jz	muladdhard			;; Yes, do it the hard way

	vaddpd	ymm4, ymm4, [srcreg+r9+d1]		;; R2 = R2 + MemR2
	vaddpd	ymm8, ymm8, [srcreg+r9+d1+32]		;; I2 = I2 + MemI2
	vaddpd	ymm6, ymm6, [srcreg+r9+d2]		;; R3 = R3 + MemR3
	vaddpd	ymm1, ymm1, [srcreg+r9+d2+32]		;; I3 = I3 + MemI3
	vaddpd	ymm5, ymm5, [srcreg+r9+d2+d1]		;; R4 = R4 + MemR4
	vaddpd	ymm7, ymm7, [srcreg+r9+d2+d1+32]	;; I4 = I4 + MemI4
	vmovapd	ymm0, [srcreg+r9]			;; MemR1
	vaddpd	ymm10, ymm10, ymm0			;; R1 = R1 + MemR1 (lower word is irrelevant)
	vaddpd	ymm2, ymm2, ymm0			;; R1a = R1a + MemR1a (upper 3 words are irrelevant)
	vmovapd	ymm0, [srcreg+r9+32]			;; MemI1
	vaddpd	ymm3, ymm3, ymm0			;; I1 = I1 + MemI1 (lower word is irrelevant)
	vaddpd	ymm9, ymm9, ymm0			;; R1b = R1b + MemR1b in MemI1 (upper 3 words are irrelevant)
	jmp	back_to_orig

muladdhard:
	vmovapd	ymm0, [srcreg+r9+d1]			;; MemR2
	vmovapd	ymm12, [srcreg+r10+d1]			;; MemR2#2
	vmulpd	ymm11, ymm0, ymm12			;; MemR2*MemR2#2
	vaddpd	ymm4, ymm4, ymm11			;; R2 = R2 + MemR2*MemR2#2
	vmovapd	ymm13, [srcreg+r10+d1+32]		;; MemI2#2
	vmulpd	ymm11, ymm0, ymm13			;; MemR2*MemI2#2
	vaddpd	ymm8, ymm8, ymm11			;; I2 = I2 + MemR2*MemI2#2
	vmovapd	ymm0, [srcreg+r9+d1+32]			;; MemI2
	vmulpd	ymm11, ymm0, ymm13			;; MemI2*MemI2#2
	vsubpd	ymm4, ymm4, ymm11			;; R2 = R2 - MemI2*MemI2#2
	vmulpd	ymm11, ymm0, ymm12			;; MemI2*MemR2#2
	vaddpd	ymm8, ymm8, ymm11			;; I2 = I2 + MemI2*MemR2#2

	vmovapd	ymm0, [srcreg+r9+d2]			;; MemR3
	vmovapd	ymm12, [srcreg+r10+d2]			;; MemR3#2
	vmulpd	ymm11, ymm0, ymm12			;; MemR3*MemR3#2
	vaddpd	ymm6, ymm6, ymm11			;; R3 = R3 + MemR3*MemR3#2
	vmovapd	ymm13, [srcreg+r10+d2+32]		;; MemI3#2
	vmulpd	ymm11, ymm0, ymm13			;; MemR3*MemI3#2
	vaddpd	ymm1, ymm1, ymm11			;; I3 = I3 + MemR3*MemI3#2
	vmovapd	ymm0, [srcreg+r9+d2+32]			;; MemI3
	vmulpd	ymm11, ymm0, ymm13			;; MemI3*MemI3#2
	vsubpd	ymm6, ymm6, ymm11			;; R3 = R3 - MemI3*MemI3#2
	vmulpd	ymm11, ymm0, ymm12			;; MemI3*MemR3#2
	vaddpd	ymm1, ymm1, ymm11			;; I3 = I3 + MemI3*MemR3#2

	vmovapd	ymm0, [srcreg+r9+d2+d1]			;; MemR4
	vmovapd	ymm12, [srcreg+r10+d2+d1]		;; MemR4#2
	vmulpd	ymm11, ymm0, ymm12			;; MemR4*MemR4#2
	vaddpd	ymm5, ymm5, ymm11			;; R4 = R4 + MemR4*MemR4#2
	vmovapd	ymm13, [srcreg+r10+d2+d1+32]		;; MemI4#2
	vmulpd	ymm11, ymm0, ymm13			;; MemR4*MemI4#2
	vaddpd	ymm7, ymm7, ymm11			;; I4 = I4 + MemR4*MemI4#2
	vmovapd	ymm0, [srcreg+r9+d2+d1+32]		;; MemI4
	vmulpd	ymm11, ymm0, ymm13			;; MemI4*MemI4#2
	vsubpd	ymm5, ymm5, ymm11			;; R4 = R4 - MemI4*MemI4#2
	vmulpd	ymm11, ymm0, ymm12			;; MemI4*MemR4#2
	vaddpd	ymm7, ymm7, ymm11			;; I4 = I4 + MemI4*MemR4#2

	vmovapd	ymm0, [srcreg+r9]			;; MemR1
	vmovapd	ymm12, [srcreg+r10]			;; MemR1#2
	vmulpd	ymm11, ymm0, ymm12			;; MemR1*MemR1#2
	vaddpd	ymm10, ymm10, ymm11			;; R1 = R1 + MemR1*MemR1#2 (lower word is irrelevant)
	vaddpd	ymm2, ymm2, ymm11			;; R1a = R1a + MemR1*MemR1#2 (upper 3 words are irrelevant)
	vmovapd	ymm13, [srcreg+r10+32]			;; MemI1#2
	vmulpd	ymm11, ymm0, ymm13			;; MemR1*MemI1#2
	vaddpd	ymm3, ymm3, ymm11			;; I1 = I1 + MemR1*MemI1#2 (lower word is irrelevant)
	vmovapd	ymm0, [srcreg+r9+32]			;; MemI1
	vmulpd	ymm11, ymm0, ymm13			;; MemI1*MemI1#2
	vsubpd	ymm10, ymm10, ymm11			;; R1 = R1 - MemI1*MemI1#2
	vaddpd	ymm9, ymm9, ymm11			;; R1b = R1b + MemR1b in MemI1*MemI1#2 (upper 3 words are irrelevant)
	vmulpd	ymm11, ymm0, ymm12			;; MemI1*MemR1#2
	vaddpd	ymm3, ymm3, ymm11			;; I1 = I1 + MemI1*MemR1#2 (lower word is irrelevant)
	jmp	back_to_orig

mulsub:	test	r10, 3				;; Check if we need to multiply add-in by FFT(1)
	jz	mulsubhard			;; Yes, do it the hard way

	vsubpd	ymm4, ymm4, [srcreg+r9+d1]		;; R2 = R2 - MemR2
	vsubpd	ymm8, ymm8, [srcreg+r9+d1+32]		;; I2 = I2 - MemI2
	vsubpd	ymm6, ymm6, [srcreg+r9+d2]		;; R3 = R3 - MemR3
	vsubpd	ymm1, ymm1, [srcreg+r9+d2+32]		;; I3 = I3 - MemI3
	vsubpd	ymm5, ymm5, [srcreg+r9+d2+d1]		;; R4 = R4 - MemR4
	vsubpd	ymm7, ymm7, [srcreg+r9+d2+d1+32]	;; I4 = I4 - MemI4
	vmovapd	ymm0, [srcreg+r9]			;; MemR1
	vsubpd	ymm10, ymm10, ymm0			;; R1 = R1 - MemR1 (lower word is irrelevant)
	vsubpd	ymm2, ymm2, ymm0			;; R1a = R1a - MemR1a (upper 3 words are irrelevant)
	vmovapd	ymm0, [srcreg+r9+32]			;; MemI1
	vsubpd	ymm3, ymm3, ymm0			;; I1 = I1 - MemI1 (lower word is irrelevant)
	vsubpd	ymm9, ymm9, ymm0			;; R1b = R1b - MemR1b in MemI1 (upper 3 words are irrelevant)
	jmp	back_to_orig

mulsubhard:
	vmovapd	ymm0, [srcreg+r9+d1]			;; MemR2
	vmovapd	ymm12, [srcreg+r10+d1]			;; MemR2#2
	vmulpd	ymm11, ymm0, ymm12			;; MemR2*MemR2#2
	vsubpd	ymm4, ymm4, ymm11			;; R2 = R2 - MemR2*MemR2#2
	vmovapd	ymm13, [srcreg+r10+d1+32]		;; MemI2#2
	vmulpd	ymm11, ymm0, ymm13			;; MemR2*MemI2#2
	vsubpd	ymm8, ymm8, ymm11			;; I2 = I2 - MemR2*MemI2#2
	vmovapd	ymm0, [srcreg+r9+d1+32]			;; MemI2
	vmulpd	ymm11, ymm0, ymm13			;; MemI2*MemI2#2
	vaddpd	ymm4, ymm4, ymm11			;; R2 = R2 + MemI2*MemI2#2
	vmulpd	ymm11, ymm0, ymm12			;; MemI2*MemR2#2
	vsubpd	ymm8, ymm8, ymm11			;; I2 = I2 - MemI2*MemR2#2

	vmovapd	ymm0, [srcreg+r9+d2]			;; MemR3
	vmovapd	ymm12, [srcreg+r10+d2]			;; MemR3#2
	vmulpd	ymm11, ymm0, ymm12			;; MemR3*MemR3#2
	vsubpd	ymm6, ymm6, ymm11			;; R3 = R3 - MemR3*MemR3#2
	vmovapd	ymm13, [srcreg+r10+d2+32]		;; MemI3#2
	vmulpd	ymm11, ymm0, ymm13			;; MemR3*MemI3#2
	vsubpd	ymm1, ymm1, ymm11			;; I3 = I3 - MemR3*MemI3#2
	vmovapd	ymm0, [srcreg+r9+d2+32]			;; MemI3
	vmulpd	ymm11, ymm0, ymm13			;; MemI3*MemI3#2
	vaddpd	ymm6, ymm6, ymm11			;; R3 = R3 + MemI3*MemI3#2
	vmulpd	ymm11, ymm0, ymm12			;; MemI3*MemR3#2
	vsubpd	ymm1, ymm1, ymm11			;; I3 = I3 - MemI3*MemR3#2

	vmovapd	ymm0, [srcreg+r9+d2+d1]			;; MemR4
	vmovapd	ymm12, [srcreg+r10+d2+d1]		;; MemR4#2
	vmulpd	ymm11, ymm0, ymm12			;; MemR4*MemR4#2
	vsubpd	ymm5, ymm5, ymm11			;; R4 = R4 - MemR4*MemR4#2
	vmovapd	ymm13, [srcreg+r10+d2+d1+32]		;; MemI4#2
	vmulpd	ymm11, ymm0, ymm13			;; MemR4*MemI4#2
	vsubpd	ymm7, ymm7, ymm11			;; I4 = I4 - MemR4*MemI4#2
	vmovapd	ymm0, [srcreg+r9+d2+d1+32]		;; MemI4
	vmulpd	ymm11, ymm0, ymm13			;; MemI4*MemI4#2
	vaddpd	ymm5, ymm5, ymm11			;; R4 = R4 + MemI4*MemI4#2
	vmulpd	ymm11, ymm0, ymm12			;; MemI4*MemR4#2
	vsubpd	ymm7, ymm7, ymm11			;; I4 = I4 - MemI4*MemR4#2

	vmovapd	ymm0, [srcreg+r9]			;; MemR1
	vmovapd	ymm12, [srcreg+r10]			;; MemR1#2
	vmulpd	ymm11, ymm0, ymm12			;; MemR1*MemR1#2
	vsubpd	ymm10, ymm10, ymm11			;; R1 = R1 - MemR1*MemR1#2 (lower word is irrelevant)
	vsubpd	ymm2, ymm2, ymm11			;; R1a = R1a - MemR1*MemR1#2 (upper 3 words are irrelevant)
	vmovapd	ymm13, [srcreg+r10+32]			;; MemI1#2
	vmulpd	ymm11, ymm0, ymm13			;; MemR1*MemI1#2
	vsubpd	ymm3, ymm3, ymm11			;; I1 = I1 - MemR1*MemI1#2 (lower word is irrelevant)
	vmovapd	ymm0, [srcreg+r9+32]			;; MemI1
	vmulpd	ymm11, ymm0, ymm13			;; MemI1*MemI1#2
	vaddpd	ymm10, ymm10, ymm11			;; R1 = R1 + MemI1*MemI1#2
	vsubpd	ymm9, ymm9, ymm11			;; R1b = R1b - MemR1b in MemI1*MemI1#2 (upper 3 words are irrelevant)
	vmulpd	ymm11, ymm0, ymm12			;; MemI1*MemR1#2
	vsubpd	ymm3, ymm3, ymm11			;; I1 = I1 - MemI1*MemR1#2 (lower word is irrelevant)
	jmp	back_to_orig

orig:	vmovapd	ymm14, [srcreg+d1][rbp]		;; MemR2
	vmovapd	ymm15, [srcreg+d1+32][rbp]	;; MemI2
	vmovapd	ymm8, [srcreg+d1][rbx]		;; R2
	vmovapd	ymm6, [srcreg+d1+32][rbx]	;; I2
	vmulpd	ymm4, ymm8, ymm14		;; R2 * MemR2
	vmulpd	ymm8, ymm8, ymm15		;; R2 * MemI2
	vmulpd	ymm9, ymm6, ymm15		;; I2 * MemI2
	vmulpd	ymm6, ymm6, ymm14		;; I2 * MemR2
	vsubpd	ymm4, ymm4, ymm9		;; R2*MemR2 - I2*MemI2 (new R2)
	vaddpd	ymm8, ymm8, ymm6		;; R2*MemI2 + I2*MemR2 (new I2)

	vmovapd	ymm14, [srcreg+d2][rbp]		;; MemR3
	vmovapd	ymm15, [srcreg+d2+32][rbp]	;; MemI3
	vmovapd	ymm1, [srcreg+d2][rbx]		;; R3
	vmovapd	ymm5, [srcreg+d2+32][rbx]	;; I3
	vmulpd	ymm6, ymm1, ymm14		;; R3 * MemR3
	vmulpd	ymm1, ymm1, ymm15		;; R3 * MemI3
	vmulpd	ymm9, ymm5, ymm15		;; I3 * MemI3
	vmulpd	ymm5, ymm5, ymm14		;; I3 * MemR3
	vsubpd	ymm6, ymm6, ymm9		;; R3*MemR3 - I3*MemI3 (new R3)
	vaddpd	ymm1, ymm1, ymm5		;; R3*MemI3 + I3*MemR3 (new I3)

	vmovapd	ymm14, [srcreg+d2+d1][rbp]	;; MemR4
	vmovapd	ymm15, [srcreg+d2+d1+32][rbp]	;; MemI4
	vmovapd	ymm7, [srcreg+d2+d1][rbx]	;; R4
	vmovapd	ymm2, [srcreg+d2+d1+32][rbx]	;; I4
	vmulpd	ymm5, ymm7, ymm14		;; R4 * MemR4
	vmulpd	ymm7, ymm7, ymm15		;; R4 * MemI4
	vmulpd	ymm9, ymm2, ymm15		;; I4 * MemI4
	vmulpd	ymm2, ymm2, ymm14		;; I4 * MemR4
	vsubpd	ymm5, ymm5, ymm9		;; R4*MemR4 - I4*MemI4 (new R4)
	vaddpd	ymm7, ymm7, ymm2		;; R4*MemI4 + I4*MemR4 (new I4)

	vmovapd	ymm14, [srcreg][rbp]		;; MemR1
	vmovapd	ymm15, [srcreg+32][rbp]		;; MemI1
	vmovapd	ymm3, [srcreg][rbx]		;; R1
	vmovapd	ymm0, [srcreg+32][rbx]		;; I1
	vmulpd	ymm2, ymm3, ymm14		;; R1 * MemR1		(new real R1)
	vmulpd	ymm3, ymm3, ymm15		;; R1 * MemI1
	vmulpd	ymm9, ymm0, ymm15		;; I1 * MemI1		(new real I1)
	vmulpd	ymm0, ymm0, ymm14		;; I1 * MemR1
	vsubpd	ymm10, ymm2, ymm9		;; R1*MemR1 - I1*MemI1 (new complex R1)
	vaddpd	ymm3, ymm3, ymm0		;; R1*MemI1 + I1*MemR1 (new complex I1)

back_to_orig:
	vmovsd	Q [srcreg-16], xmm2		;; Save product of sums of FFT values
	vblendpd ymm2, ymm10, ymm2, 1		;; Blend R1 real and complex results
	vblendpd ymm3, ymm3, ymm9, 1		;; Blend I1 real and complex results

	;; Do the eight-reals part with R1,R2,R3,R4,R5,R6,R7,R8 terminology instead of R1,I1,R2,I2,R3,I3,R4,I4

						;; Eight-reals comments		; Three four-complex comments
	vaddpd	ymm0, ymm5, ymm6		;; new R5 = R7 + R5		; R4 + R3 (new R3)
	vsubpd	ymm5, ymm5, ymm6		;; new negR6 = R7 - R5		; R4 - R3 (new I4)

	vaddpd	ymm6, ymm1, ymm7		;; new R7 = R6 + R8		; I3 + I4 (new I3)
	vsubpd	ymm1, ymm1, ymm7		;; new R8 = R6 - R8		; I3 - I4 (new R4)

	vsubsd	xmm9, xmm2, xmm3		;; new R2 = R1 - R2
	vmulsd	xmm9, xmm9, Q YMM_HALF		;; Mul new R2 by HALF
	vaddsd	xmm10, xmm9, xmm3		;; new R1 = R1 + R2

	vsubsd	xmm11, xmm1, xmm5		;; R6 = R8 - negR6
	vaddsd	xmm12, xmm1, xmm5		;; R8 = R8 + negR6

	vblendpd ymm2, ymm2, ymm10, 1		;;	new R1				R1
	vaddpd	ymm7, ymm2, ymm4		;; newR1 + R3 (new R1)		; R1 + R2 (new R1)
	vsubpd	ymm2, ymm2, ymm4		;; newR1 - R3 (new R3)		; R1 - R2 (new R2)

	vblendpd ymm3, ymm3, ymm9, 1		;;	new R2				I1
	vaddpd	ymm4, ymm3, ymm8		;; newR2 + R4 (new R2)		; I1 + I2 (new I1)
	vsubpd	ymm3, ymm3, ymm8		;; newR2 - R4 (new R4)		; I1 - I2 (new I2)

	vmulsd	xmm11, xmm11, Q YMM_SQRTHALF	;; R6 = R6 * square root of 1/2
	vmulsd	xmm12, xmm12, Q YMM_SQRTHALF	;; R8 = R8 * square root of 1/2

	vaddpd	ymm8, ymm7, ymm0		;; R1 + R5 (final R1)		; R1 + R3 (final R1)
	vsubpd	ymm7, ymm7, ymm0		;; R1 - R5 (final R5)		; R1 - R3 (final R3)

	vblendpd ymm1, ymm1, ymm6, 1		;;	R7				R4
	vaddpd	ymm0, ymm2, ymm1		;; R3 + R7 (final R3)		; R2 + R4 (final R2)
	vsubpd	ymm2, ymm2, ymm1		;; R3 - R7 (final R7)		; R2 - R4 (final R4)

	vblendpd ymm6, ymm6, ymm11, 1		;;	R6				I3
	vaddpd	ymm1, ymm4, ymm6		;; R2 + R6 (final R2)		; I1 + I3 (final I1)
	vsubpd	ymm4, ymm4, ymm6		;; R2 - R6 (final R6)		; I1 - I3 (final I3)

	vblendpd ymm5, ymm5, ymm12, 1		;;	R8				I4
	vaddpd	ymm6, ymm3, ymm5		;; R4 + R8 (final R4)		; I2 + I4 (final I2)
	vsubpd	ymm3, ymm3, ymm5		;; R4 - R8 (final R8)		; I2 - I4 (final I4)

	vblendpd ymm5, ymm0, ymm1, 1		;;	R2				R2
	vblendpd ymm9, ymm7, ymm0, 1		;;	R3				R3
	vblendpd ymm0, ymm2, ymm6, 1		;;	R4				R4
	vblendpd ymm10, ymm1, ymm7, 1		;;	R5				I1
	vblendpd ymm7, ymm6, ymm4, 1		;;	R6				I2
	vblendpd ymm6, ymm4, ymm2, 1		;;	R7				I3

	ystore	[srcreg], ymm8			;; Save R1
	ystore	[srcreg+32], ymm10		;; Save I1
	ystore	[srcreg+d1], ymm5		;; Save R2
	ystore	[srcreg+d1+32], ymm7		;; Save I2
	ystore	[srcreg+d2], ymm9		;; Save R3
	ystore	[srcreg+d2+32], ymm6		;; Save I3
	ystore	[srcreg+d2+d1], ymm0		;; Save R4
	ystore	[srcreg+d2+d1+32], ymm3		;; Save I4
	bump	srcreg, srcinc
	ENDM

ENDIF

;; Does a yr4_8r_fft_mem on the one low value of the ymm register
;; Does a yr4_4c_fft_mem on the three high values of the ymm register
;; This is REALLY funky, as we do both at the same time within
;; the full ymm register whenever possible.  We could improve this code
;; a few clocks by implementing the improvements in yr4_o8r_t4c_djbfft_mem.
yr4_o8r_t4c_simple_fft_mem MACRO mem1,mem2,mem3,mem4,mem5,mem6,mem7,mem8,destR4,destI4
					;; Eight-reals comments		; Three four-complex comments
	vmovapd	ymm6, mem4		;; R4				; R4
	vmovapd	ymm7, mem8		;; R8				; I4
	vsubsd	xmm0, xmm6, xmm7	;; interim R8 = R4 - R8		; meaningless

	vmovapd	ymm1, mem2		;; R2				; R2
	vmovapd	ymm5, mem6		;; R6				; I2
	vsubsd	xmm2, xmm1, xmm5	;; interim R6 = R2 - R6		; meaningless

	vmulsd	xmm0, xmm0, Q YMM_SQRTHALF;; interim R8 * square root	; meaningless
	vmulsd	xmm2, xmm2, Q YMM_SQRTHALF;; interim R6 * square root	; meaningless

	vblendpd ymm3, ymm5, ymm6, 1	;; R4				; I2
	vaddpd	ymm3, ymm3, ymm7	;; new R4 = R4 + R8		; I2 + I4 (new I2)

	vblendpd ymm4, ymm5, ymm0, 1	;; interim R8			; I2
	vblendpd ymm7, ymm7, ymm2, 1	;; interim R6			; I4
	vsubpd	ymm4, ymm4, ymm7	;; new negR6 = interim R8 - R6	; I2 - I4 (new I4)

	vaddsd	xmm2, xmm2, xmm0	;; new R8 = interim R6 + R8	; meaningless

	vsubpd	ymm7, ymm1, ymm6	;; meaningless			; R2 - R4 (new R4)
	vblendpd ymm7, ymm7, ymm2, 1	;; new R8			; new R4

	vaddsd	xmm5, xmm1, xmm5	;; new R2 = R2 + R6		; meaningless

	vaddpd	ymm1, ymm1, ymm6	;; meaningless			; R2 + R4 (new R2)

	vmovapd	ymm0, mem3		;; R3				; R3
	vmovapd	ymm2, mem5		;; R5				; I1
	vblendpd ymm6, ymm0, ymm2, 1	;; R5				; R3
	vblendpd ymm2, ymm2, ymm0, 1	;; R3				; I1

	vaddpd	ymm0, ymm2, mem7	;; new R3 = R3 + R7		; I1 + I3 (new I1)
	vblendpd ymm1, ymm1, ymm0, 1	;; new R3			; new R2
	vblendpd ymm0, ymm0, ymm5, 1	;; new R2			; new I1

	vsubpd	ymm2, ymm2, mem7	;; new R7 = R3 - R7		; I1 - I3 (new I3)

	vaddpd	ymm5, ymm2, ymm7	;; R7 + R8 (final I3)		; I3 + R4 (final I3)
	vsubpd	ymm2, ymm2, ymm7	;; R7 - R8 (final I4)		; I3 - R4 (final I4)

	vmovapd	ymm7, mem1		;; R1				; R1
	ystore	destI4, ymm2		;; Save final I4
	vaddpd	ymm2, ymm7, ymm6	;; new R1 = R1 + R5		; R1 + R3 (new R1)
	vsubpd	ymm7, ymm7, ymm6	;; new R5 = R1 - R5		; R1 - R3 (new R3)

	vsubpd	ymm6, ymm0, ymm3	;; R2 - R4 (new R4, final I2)	; I1 - I2 (final I2)
	vaddpd	ymm0, ymm0, ymm3	;; R2 + R4 (new R2)		; I1 + I2 (final I1)

	vaddpd	ymm3, ymm2, ymm1	;; R1 + R3 (new R1)		; R1 + R2 (final R1)
	vsubpd	ymm2, ymm2, ymm1	;; R1 - R3 (new R3, final R2)	; R1 - R2 (final R2)

	vsubpd	ymm1, ymm7, ymm4	;; R5 - negR6 (final R3)	; R3 - I4 (final R3)
	vaddpd	ymm7, ymm7, ymm4	;; R5 + negR6 (final R4)	; R3 + I4 (final R4)

	vsubsd	xmm4, xmm3, xmm0	;; R1 - R2 (final I1, a.k.a 2nd real result)
	ystore	destR4, ymm7		;; Save final R4
	vaddsd	xmm7, xmm3, xmm0	;; R1 + R2 (final R1)		; meaningless
	vblendpd ymm0, ymm0, ymm4, 1	;; final I1			; final I1
	vblendpd ymm3, ymm3, ymm7, 1	;; final R1			; final R1
	ENDM

yr4_o8r_t4c_simple_unfft_mem MACRO mem1,mem2,mem3,mem4,mem5,mem6,mem7,mem8,dst1

	;; Do the eight-reals part

	vmovsd	xmm1, Q mem5		;; R5
	vmovsd	xmm0, Q mem7		;; R7
	vsubsd	xmm4, xmm1, xmm0	;; new R6 = R5 - R7
	vaddsd	xmm1, xmm1, xmm0	;; new R5 = R5 + R7

	vmovsd	xmm7, Q mem6		;; R6
	vmovsd	xmm0, Q mem8		;; R8
	vsubsd	xmm5, xmm7, xmm0	;; new R8 = R6 - R8
	vaddsd	xmm7, xmm7, xmm0	;; new R7 = R6 + R8

	vaddsd	xmm3, xmm4, xmm5	;; R6 = R6 + R8
	vsubsd	xmm4, xmm5, xmm4	;; R8 = R8 - R6

	vmovsd	xmm2, Q mem1		;; R1
	vmovsd	xmm6, Q mem2		;; R2
	vsubsd	xmm2, xmm2, xmm6	;; new R2 = R1 - R2
	vmulsd	xmm2, xmm2, Q YMM_HALF	;; Mul new R2 by HALF
	vaddsd	xmm6, xmm2, xmm6	;; new R1 = R1 + R2

	vmulsd	xmm3, xmm3, Q YMM_SQRTHALF;; R6 = R6 * square root of 1/2
	vmulsd	xmm4, xmm4, Q YMM_SQRTHALF;; R8 = R8 * square root of 1/2

	vmovsd	xmm0, Q mem4		;; R4
	vaddsd	xmm5, xmm2, xmm0	;; R2 + R4 (new R2)
	vsubsd	xmm2, xmm2, xmm0	;; R2 - R4 (new R4)

	vsubsd	xmm0, xmm5, xmm3	;; R2 - R6 (final R6)
	vaddsd	xmm5, xmm5, xmm3	;; R2 + R6 (final R2)

	vsubsd	xmm3, xmm2, xmm4	;; R4 - R8 (final R8)
	vaddsd	xmm2, xmm2, xmm4	;; R4 + R8 (final R4)

	vmovsd	Q YMM_TMP6, xmm0	;; Save R6
	vmovsd	Q YMM_TMP2, xmm5	;; Save R2

	vmovsd	xmm4, Q mem3		;; R3
	vaddsd	xmm0, xmm6, xmm4	;; R1 + R3 (new R1)
	vsubsd	xmm6, xmm6, xmm4	;; R1 - R3 (new R3)

	vmovsd	Q YMM_TMP8, xmm3	;; Save R8
	vmovsd	Q YMM_TMP4, xmm2	;; Save R4

	vsubsd	xmm2, xmm0, xmm1	;; R1 - R5 (final R5)
	vaddsd	xmm0, xmm0, xmm1	;; R1 + R5 (final R1)

	vsubsd	xmm3, xmm6, xmm7	;; R3 - R7 (final R7)
	vaddsd	xmm6, xmm6, xmm7	;; R3 + R7 (final R3)

	vmovsd	Q YMM_TMP5, xmm2	;; Save R5
	vmovsd	Q YMM_TMP1, xmm0	;; Save R1

	vmovsd	Q YMM_TMP7, xmm3	;; Save R7
	vmovsd	Q YMM_TMP3, xmm6	;; Save R3

	;; Do the four complex part

	vmovapd	ymm7, mem1		;; R1
	vmovapd	ymm0, mem3		;; R2
	vsubpd	ymm5, ymm7, ymm0	;; R1 - R2 (new R2)
	vaddpd	ymm7, ymm7, ymm0	;; R1 + R2 (new R1)

	vmovapd	ymm1, mem7		;; R4
	vmovapd	ymm0, mem5		;; R3
	vsubpd	ymm3, ymm1, ymm0	;; R4 - R3 (new I4)
	vaddpd	ymm1, ymm1, ymm0	;; R4 + R3 (new R3)

	vmovapd	ymm0, mem6		;; I3
	vmovapd	ymm2, mem8		;; I4
	vsubpd	ymm4, ymm0, ymm2	;; I3 - I4 (new R4)
	vaddpd	ymm0, ymm0, ymm2	;; I3 + I4 (new I3)

	vsubpd	ymm2, ymm7, ymm1	;; R1 - R3 (final R3)
	vaddpd	ymm7, ymm7, ymm1	;; R1 + R3 (final R1)

	vsubpd	ymm1, ymm5, ymm4	;; R2 - R4 (final R4)
	vaddpd	ymm5, ymm5, ymm4	;; R2 + R4 (final R2)

	vblendpd ymm2, ymm2, YMM_TMP3, 1 ;; Blend in R3 real result
	vblendpd ymm7, ymm7, YMM_TMP1, 1 ;; Blend in R1 real result

	vblendpd ymm1, ymm1, YMM_TMP4, 1 ;; Blend in R4 real result
	vblendpd ymm5, ymm5, YMM_TMP2, 1 ;; Blend in R2 real result

	ystore	dst1, ymm7

	vmovapd	ymm4, mem2		;; I1
	vmovapd	ymm6, mem4		;; I2
	vsubpd	ymm7, ymm4, ymm6	;; I1 - I2 (new I2)
	vaddpd	ymm4, ymm4, ymm6	;; I1 + I2 (new I1)

	vsubpd	ymm6, ymm7, ymm3	;; I2 - I4 (final I4)
	vaddpd	ymm7, ymm7, ymm3	;; I2 + I4 (final I2)

	vsubpd	ymm3, ymm4, ymm0	;; I1 - I3 (final I3)
	vaddpd	ymm4, ymm4, ymm0	;; I1 + I3 (final I1)

	vblendpd ymm6, ymm6, YMM_TMP8, 1 ;; Blend in R8 real result
	vblendpd ymm7, ymm7, YMM_TMP6, 1 ;; Blend in R6 real result

	vblendpd ymm3, ymm3, YMM_TMP7, 1 ;; Blend in R7 real result
	vblendpd ymm4, ymm4, YMM_TMP5, 1 ;; Blend in R5 real result
	ENDM


;;
;; ******************************* four-complex-with-partial-normalization4 variants *************************************
;;
;; These macros are used in pass 1 of yr4dwpn two pass FFTs.  They are like the original four-complex-wpn
;; macros except that 4 normalization multipliers are pre-applied rather than post-applying one normalization
;; multiplier.  This results in more multiplies (slower macros), but reduces group multiplier data and also
;; reduces variable sine/cosine data used in pass 1.  On a Haswell machine, this is a net positive.
;;

;; screg1 is normalization weights
;; screg2+0+0, screg2+0+32 is weighted sin/cos values for R3/I3/R4/I4 (w^2n)
;; screg2+64+0, screg2+64+32 is weighted sin/cos values for R2/I2 (w^n)

yr4_b4cl_wpn4_four_complex_djbfft_preload MACRO
	ENDM
yr4_b4cl_wpn4_four_complex_djbfft MACRO srcreg,srcinc,d1,d2,screg1,scinc1,screg2,scinc2,maxrpt,L1pt,L1pd
	vbroadcastsd ymm4, Q [screg1+0]	;; Load normalization multiplier for R1/I1
	vmovapd	ymm0, [srcreg]		;; R1
	vmulpd	ymm0, ymm0, ymm4	;; Apply weight1 to R1				;  1-5
	vbroadcastsd ymm7, Q [screg1+16];; Load normalization multiplier for R3/I3
	vmovapd	ymm6, [srcreg+d2]	;; R3
	vmulpd	ymm6, ymm6, ymm7	;; Apply weight3 to R3				;  2-6
	vmulpd	ymm4, ymm4, [srcreg+32]	;; Apply weight1 to I1				;  3-7
	vmulpd	ymm7, ymm7, [srcreg+d2+32];; Apply weight3 to I3			;  4-8
	vbroadcastsd ymm5, Q [screg1+8]	;; Load normalization multiplier for R2/I2
	vmovapd	ymm1, [srcreg+d1]	;; R2
	vmulpd	ymm1, ymm1, ymm5	;; Apply weight2 to R2				;  5-9
	vbroadcastsd ymm3, Q [screg1+24];; Load normalization multiplier for R4/I4
	vmulpd	ymm3, ymm3, [srcreg+d2+d1] ;; Apply weight4 to R4			;  6-10
	vmulpd	ymm5, ymm5, [srcreg+d1+32] ;; Apply weight2 to I2			;  7-11
	vaddpd	ymm2, ymm0, ymm6	;; R1 + R3 (new R1)				; 7-9
	vsubpd	ymm0, ymm0, ymm6	;; R1 - R3 (new R3)				; 8-10
	L1prefetchw srcreg+L1pd, L1pt
	vaddpd	ymm6, ymm4, ymm7	;; I1 + I3 (new I1)				; 9-11
	vsubpd	ymm4, ymm4, ymm7	;; I1 - I3 (new I3)				; 10-12
	vaddpd	ymm7, ymm1, ymm3	;; R2 + R4 (new R2)				; 11-13
	vsubpd	ymm1, ymm1, ymm3	;; R2 - R4 (new R4)				; 12-14
	vsubpd	ymm3, ymm2, ymm7	;; R1 - R2 (final R2)				; 14-16
	vaddpd	ymm2, ymm2, ymm7	;; R1 + R2 (final R1)				; 15-17
	vbroadcastsd ymm7, Q [screg1+24];; Reload normalization multiplier for R4/I4
	vmulpd	ymm7, ymm7, [srcreg+d2+d1+32] ;; Apply weight4 to I4			;  8-12
	ystore	[srcreg], ymm2		;; Save R1
	vaddpd	ymm2, ymm5, ymm7	;; I2 + I4 (new I2)				; 13-15
	vsubpd	ymm5, ymm5, ymm7	;; I2 - I4 (new I4)				; 16-18
	L1prefetchw srcreg+d1+L1pd, L1pt
	vsubpd	ymm7, ymm6, ymm2	;; I1 - I2 (final I2)				; 17-19
	vaddpd	ymm6, ymm6, ymm2	;; I1 + I2 (final I1)				; 18-20
	vsubpd	ymm2, ymm0, ymm5	;; R3 - I4 (final R3)				; 19-21
	ystore	[srcreg+32], ymm6	;; Save I1
	vaddpd	ymm6, ymm4, ymm1	;; I3 + R4 (final I3)				; 20-22
	L1prefetchw srcreg+d2+L1pd, L1pt
	vaddpd	ymm0, ymm0, ymm5	;; R3 + I4 (final R4)				; 21-23
	vsubpd	ymm4, ymm4, ymm1	;; I3 - R4 (final I4)				; 22-24
	vmovapd ymm5, [screg2+64+32]	;; cosine/sine2
	vmulpd	ymm1, ymm3, ymm5	;; A2 = R2 * cosine/sine			;  17-21
	vsubpd	ymm1, ymm1, ymm7	;; A2 = A2 - I2					; 23-25
	vmulpd	ymm7, ymm7, ymm5	;; B2 = I2 * cosine/sine			;  20-24
	vaddpd	ymm7, ymm7, ymm3	;; B2 = B2 + R2					; 25-27
	vmovapd ymm5, [screg2+0+32]	;; cosine/sine34
	vmulpd	ymm3, ymm2, ymm5	;; A3 = R3 * cosine/sine			;  22-26
	vsubpd	ymm3, ymm3, ymm6	;; A3 = A3 - I3					; 27-29
	vmulpd	ymm6, ymm6, ymm5	;; B3 = I3 * cosine/sine			;  23-27
	vaddpd	ymm6, ymm6, ymm2	;; B3 = B3 + R3					; 28-30
	L1prefetchw srcreg+d2+d1+L1pd, L1pt
	vmulpd	ymm2, ymm0, ymm5	;; A4 = R4 * cosine/sine			;  24-28
	vaddpd	ymm2, ymm2, ymm4	;; A4 = A4 + I4					; 29-31
	vmulpd	ymm4, ymm4, ymm5	;; B4 = I4 * cosine/sine			;  25-29
	vsubpd	ymm4, ymm4, ymm0	;; B4 = B4 - R4					; 30-32
	vmovapd ymm5, [screg2+64+0]	;; Sine2
	vmulpd	ymm1, ymm1, ymm5	;; A2 = A2 * sine (final R2)			;  26-30
	vmulpd	ymm7, ymm7, ymm5	;; B2 = B2 * sine (final I2)			;  28-32
	vmovapd ymm5, [screg2+0+0]	;; Sine34
	vmulpd	ymm3, ymm3, ymm5	;; A3 = A3 * sine (final R3)			;  30-34
	vmulpd	ymm6, ymm6, ymm5	;; B3 = B3 * sine (final I3)			;  31-35
	vmulpd	ymm2, ymm2, ymm5	;; A4 = A4 * sine (final R4)			;  32-36
	vmulpd	ymm4, ymm4, ymm5	;; B4 = B4 * sine (final I4)			;  33-37
	ystore	[srcreg+d1], ymm1	;; Save R2
	ystore	[srcreg+d1+32], ymm7	;; Save I2
	ystore	[srcreg+d2], ymm3	;; Save R3
	ystore	[srcreg+d2+32], ymm6	;; Save I3
	ystore	[srcreg+d2+d1], ymm2	;; Save R4
	ystore	[srcreg+d2+d1+32], ymm4	;; Save I4
	bump	srcreg, srcinc
	bump	screg1, scinc1
	bump	screg2, scinc2
	ENDM

;; 64-bit version

IFDEF X86_64

yr4_b4cl_wpn4_four_complex_djbfft_preload MACRO
	yr4_4c_wpn4_djbfft_unroll_preload
	ENDM
yr4_b4cl_wpn4_four_complex_djbfft MACRO srcreg,srcinc,d1,d2,screg1,scinc1,screg2,scinc2,maxrpt,L1pt,L1pd
	IF maxrpt MOD 5 EQ 0
	yr4_4c_wpn4_djbfft_unroll -1,noexec,noexec,exec,srcreg,srcinc,d1,d2,screg1,scinc1,screg2,scinc2,L1pt,L1pd
	yr4_4c_wpn4_djbfft_unroll 0,noexec,exec,exec,srcreg,srcinc,d1,d2,screg1,scinc1,screg2,scinc2,L1pt,L1pd
	yr4_4c_wpn4_djbfft_unroll 1,exec,exec,exec,srcreg,srcinc,d1,d2,screg1,scinc1,screg2,scinc2,L1pt,L1pd
	yr4_4c_wpn4_djbfft_unroll 2,exec,exec,exec,srcreg,srcinc,d1,d2,screg1,scinc1,screg2,scinc2,L1pt,L1pd
	yr4_4c_wpn4_djbfft_unroll 3,exec,exec,exec,srcreg,srcinc,d1,d2,screg1,scinc1,screg2,scinc2,L1pt,L1pd
	yr4_4c_wpn4_djbfft_unroll 4,exec,exec,noexec,srcreg,srcinc,d1,d2,screg1,scinc1,screg2,scinc2,L1pt,L1pd
	yr4_4c_wpn4_djbfft_unroll 5,exec,noexec,noexec,srcreg,srcinc,d1,d2,screg1,scinc1,screg2,scinc2,L1pt,L1pd
	bump	srcreg, 5*srcinc
	bump	screg1, 5*scinc1
	bump	screg2, 5*scinc2
	ELSEIF maxrpt MOD 4 EQ 0
	yr4_4c_wpn4_djbfft_unroll -1,noexec,noexec,exec,srcreg,srcinc,d1,d2,screg1,scinc1,screg2,scinc2,L1pt,L1pd
	yr4_4c_wpn4_djbfft_unroll 0,noexec,exec,exec,srcreg,srcinc,d1,d2,screg1,scinc1,screg2,scinc2,L1pt,L1pd
	yr4_4c_wpn4_djbfft_unroll 1,exec,exec,exec,srcreg,srcinc,d1,d2,screg1,scinc1,screg2,scinc2,L1pt,L1pd
	yr4_4c_wpn4_djbfft_unroll 2,exec,exec,exec,srcreg,srcinc,d1,d2,screg1,scinc1,screg2,scinc2,L1pt,L1pd
	yr4_4c_wpn4_djbfft_unroll 3,exec,exec,noexec,srcreg,srcinc,d1,d2,screg1,scinc1,screg2,scinc2,L1pt,L1pd
	yr4_4c_wpn4_djbfft_unroll 4,exec,noexec,noexec,srcreg,srcinc,d1,d2,screg1,scinc1,screg2,scinc2,L1pt,L1pd
	bump	srcreg, 4*srcinc
	bump	screg1, 4*scinc1
	bump	screg2, 4*scinc2
	ELSEIF maxrpt MOD 3 EQ 0
	yr4_4c_wpn4_djbfft_unroll -1,noexec,noexec,exec,srcreg,srcinc,d1,d2,screg1,scinc1,screg2,scinc2,L1pt,L1pd
	yr4_4c_wpn4_djbfft_unroll 0,noexec,exec,exec,srcreg,srcinc,d1,d2,screg1,scinc1,screg2,scinc2,L1pt,L1pd
	yr4_4c_wpn4_djbfft_unroll 1,exec,exec,exec,srcreg,srcinc,d1,d2,screg1,scinc1,screg2,scinc2,L1pt,L1pd
	yr4_4c_wpn4_djbfft_unroll 2,exec,exec,noexec,srcreg,srcinc,d1,d2,screg1,scinc1,screg2,scinc2,L1pt,L1pd
	yr4_4c_wpn4_djbfft_unroll 3,exec,noexec,noexec,srcreg,srcinc,d1,d2,screg1,scinc1,screg2,scinc2,L1pt,L1pd
	bump	srcreg, 3*srcinc
	bump	screg1, 3*scinc1
	bump	screg2, 3*scinc2
	ELSEIF maxrpt MOD 2 EQ 0
	yr4_4c_wpn4_djbfft_unroll -1,noexec,noexec,exec,srcreg,srcinc,d1,d2,screg1,scinc1,screg2,scinc2,L1pt,L1pd
	yr4_4c_wpn4_djbfft_unroll 0,noexec,exec,exec,srcreg,srcinc,d1,d2,screg1,scinc1,screg2,scinc2,L1pt,L1pd
	yr4_4c_wpn4_djbfft_unroll 1,exec,exec,noexec,srcreg,srcinc,d1,d2,screg1,scinc1,screg2,scinc2,L1pt,L1pd
	yr4_4c_wpn4_djbfft_unroll 2,exec,noexec,noexec,srcreg,srcinc,d1,d2,screg1,scinc1,screg2,scinc2,L1pt,L1pd
	bump	srcreg, 2*srcinc
	bump	screg1, 2*scinc1
	bump	screg2, 2*scinc2
	ELSE
	yr4_4c_wpn4_djbfft_unroll -1,noexec,noexec,exec,srcreg,srcinc,d1,d2,screg1,scinc1,screg2,scinc2,L1pt,L1pd
	yr4_4c_wpn4_djbfft_unroll 0,noexec,exec,noexec,srcreg,srcinc,d1,d2,screg1,scinc1,screg2,scinc2,L1pt,L1pd
	yr4_4c_wpn4_djbfft_unroll 1,exec,noexec,noexec,srcreg,srcinc,d1,d2,screg1,scinc1,screg2,scinc2,L1pt,L1pd
	bump	srcreg, srcinc
	bump	screg1, scinc1
	bump	screg2, scinc2
	ENDIF
	ENDM

yr4_4c_wpn4_djbfft_unroll_preload MACRO
	ENDM
yr4_4c_wpn4_djbfft_unroll MACRO iter,prev,this,next,srcreg,srcinc,d1,d2,screg1,scinc1,screg2,scinc2,L1pt,L1pd

;; On first call, init registers

	IF iter EQ -1
y0	TEXTEQU	<ymm0>
y1	TEXTEQU	<ymm1>
y2	TEXTEQU	<ymm2>
y3	TEXTEQU	<ymm3>
y4	TEXTEQU	<ymm4>
y5	TEXTEQU	<ymm5>
y6	TEXTEQU	<ymm6>
y7	TEXTEQU	<ymm7>
y8	TEXTEQU	<ymm8>
y9	TEXTEQU	<ymm9>
y10	TEXTEQU	<ymm10>
y11	TEXTEQU	<ymm11>
y12	TEXTEQU	<ymm12>
y13	TEXTEQU	<ymm13>
y14	TEXTEQU	<ymm14>
y15	TEXTEQU	<ymm15>
	ENDIF

;; On later calls, previous A2,B2,A3,B3,A4,B4,sine will be in y0-6.  This R1,R2,R3,R4,I1,I2,I3,I4 will be in y7-14.
;; The remaining register is free.

this	vsubpd	y15, y7, y9				;; R1 - R3 (new R3)			; 1-3		n 9
prev	vmulpd	y4, y4, y6				;; A4 = A4 * sine (final R4)		;  1-5
this	L1prefetchw srcreg+iter*srcinc+L1pd, L1pt

this	vaddpd	y7, y7, y9				;; R1 + R3 (new R1)			; 2-4		n 13
prev	vmulpd	y3, y3, y6				;; B3 = B3 * sine (final I3)		;  2-6

this	vsubpd	y9, y11, y13				;; I1 - I3 (new I3)			; 3-5		n 11
prev	vmulpd	y5, y5, y6				;; B4 = B4 * sine (final I4)		;  3-7
prev	vmovapd y6, [screg2+(iter-1)*scinc2+64+0]	;; Sine2

this	vaddpd	y11, y11, y13				;; I1 + I3 (new I1)			; 4-6		n 14
prev	vmulpd	y0, y0, y6				;; A2 = A2 * sine (final R2)		;  4-8
next	vbroadcastsd y13, Q [screg1+(iter+1)*scinc1+0]	;; Normalization multiplier for R1 & I1

prev	ystore	[srcreg+(iter-1)*srcinc+d2], y2		;; Save R3				; 5
this	vsubpd	y2, y12, y14				;; I2 - I4 (new I4)			; 5-7		n 9
prev	vmulpd	y1, y1, y6				;; B2 = B2 * sine (final I2)		;  5-9
next	vmovapd	y6, [srcreg+(iter+1)*srcinc]		;; R1

prev	ystore	[srcreg+(iter-1)*srcinc+d2+d1], y4	;; Save R4				; 6
this	vaddpd	y12, y12, y14				;; I2 + I4 (new I2)			; 6-8		n 14
this	L1prefetchw srcreg+iter*srcinc+d1+L1pd, L1pt

prev	ystore	[srcreg+(iter-1)*srcinc+d2+32], y3	;; Save I3				; 7
this	vsubpd	y3, y8, y10				;; R2 - R4 (new R4)			; 7-9		n 11

prev	ystore	[srcreg+(iter-1)*srcinc+d2+d1+32], y5	;; Save I4				; 8
this	vaddpd	y8, y8, y10				;; R2 + R4 (new R2)			; 8-10		n 13
next	vmulpd	y6, y6, y13				;; Apply normalization multiplier to R1	;  8-12			n 1
this	L1prefetchw srcreg+iter*srcinc+d2+L1pd, L1pt

prev	ystore	[srcreg+(iter-1)*srcinc+d1], y0		;; Save R2				; 9
this	vsubpd	y0, y15, y2				;; R3 - I4 (final R3)			; 9-11		n 12
next	vmulpd	y13, y13, [srcreg+(iter+1)*srcinc+32]	;; Apply normalization multiplier to I1 ;  9-13			n 3
next	vbroadcastsd y10, Q [screg1+(iter+1)*scinc1+16]	;; Normalization multiplier for R3 & I3

prev	ystore	[srcreg+(iter-1)*srcinc+d1+32], y1	;; Save I2				; 10
this	vaddpd	y15, y15, y2				;; R3 + I4 (final R4)			; 10-12		n 13
next	vmovapd	y2, [srcreg+(iter+1)*srcinc+d2]		;; R3
next	vmulpd	y2, y2, y10				;; Apply normalization multiplier to R3	;  10-14		n 1

this	vaddpd	y1, y9, y3				;; I3 + R4 (final I3)			; 11-13		n 14
next	vmulpd	y10, y10, [srcreg+(iter+1)*srcinc+d2+32];; Apply normalization multiplier to I3 ;  11-15		n 3

this	vsubpd	y9, y9, y3				;; I3 - R4 (final I4)			; 12-14		n 15
this	vmovapd y3, [screg2+iter*scinc2+0+32]		;; cosine/sine34
this	vmulpd	y5, y0, y3				;; A3 = R3 * cosine/sine		;  12-16	n 17

this	vsubpd	y14, y7, y8				;; R1 - R2 (final R2)			; 13-15		n 16
this	vsubpd	y4, y11, y12				;; I1 - I2 (final I2)			; 14-16		n 17
this	vaddpd	y7, y7, y8				;; R1 + R2 (final R1)			; 15-17
this	vmulpd	y8, y15, y3				;; A4 = R4 * cosine/sine		;  13-17	n 18
this	L1prefetchw srcreg+iter*srcinc+d2+d1+L1pd, L1pt

this	vaddpd	y11, y11, y12				;; I1 + I2 (final I1)			; 16-18
this	vmulpd	y12, y1, y3				;; B3 = I3 * cosine/sine		;  14-18	n 19
this	vmulpd	y3, y9, y3				;; B4 = I4 * cosine/sine		;  15-19	n 20

this	ystore	[srcreg+iter*srcinc], y7		;; Save R1				; 18
this	vmovapd y7, [screg2+iter*scinc2+64+32]		;; cosine/sine2
this	ystore	[srcreg+iter*srcinc+32], y11		;; Save I1				; 19
this	vmulpd	y11, y14, y7				;; A2 = R2 * cosine/sine		;  16-20	n 21

this	vsubpd	y5, y5, y1				;; A3 = A3 - I3				; 17-19		n 20
this	vmulpd	y7, y4, y7				;; B2 = I2 * cosine/sine		;  17-21	n 22
next	vbroadcastsd y1, Q [screg1+(iter+1)*scinc1+8]	;; Normalization multiplier for R2 & I2

this	vaddpd	y8, y8, y9				;; A4 = A4 + I4				; 18-20		n 21
next	vmovapd	y9, [srcreg+(iter+1)*srcinc+d1+32]	;; I2
next	vmulpd	y9, y9, y1				;; Apply normalization multiplier to I2 ;  18-22		n 5
this next yloop_unrolled_one

this	vaddpd	y12, y12, y0				;; B3 = B3 + R3				; 19-21		n 22
next	vmulpd	y1, y1, [srcreg+(iter+1)*srcinc+d1]	;; Apply normalization multiplier to R2	;  19-23		n 7

next	vbroadcastsd y0, Q [screg1+(iter+1)*scinc1+24]	;; Normalization multiplier for R4 & I4
this	vsubpd	y3, y3, y15				;; B4 = B4 - R4				; 20-22		n 1
next	vmovapd	y15, [srcreg+(iter+1)*srcinc+d2+d1+32]	;; I4
next	vmulpd	y15, y15, y0				;; Apply normalization multiplier to I4 ;  20-24		n 5

this	vsubpd	y11, y11, y4				;; A2 = A2 - I2				; 21-23
next	vmulpd	y0, y0, [srcreg+(iter+1)*srcinc+d2+d1]	;; Apply normalization multiplier to R4	;  21-25		n 7
this	vmovapd y4, [screg2+iter*scinc2+0+0]		;; Sine34

this	vaddpd	y7, y7, y14				;; B2 = B2 + R2				; 22-24
this	vmulpd	y5, y5, y4				;; A3 = A3 * sine (final R3)		;  22-26

;; Shuffle register assignments so that this A2,B2,A3,B3,A4,B4,sine34 are in y0-6 and next R1,R2,R3,R4,I1,I2,I3,I4 in y7-14.

ytmp	TEXTEQU	y0
y0	TEXTEQU	y11
y11	TEXTEQU	y13
y13	TEXTEQU	y10
y10	TEXTEQU	ytmp
ytmp	TEXTEQU	y1
y1	TEXTEQU	y7
y7	TEXTEQU	y6
y6	TEXTEQU	y4
y4	TEXTEQU	y8
y8	TEXTEQU ytmp
ytmp	TEXTEQU	y2
y2	TEXTEQU	y5
y5	TEXTEQU	y3
y3	TEXTEQU	y12
y12	TEXTEQU	y9
y9	TEXTEQU	ytmp
ytmp	TEXTEQU	y14
y14	TEXTEQU	y15
y15	TEXTEQU	ytmp
	ENDM

;; Haswell FMA3 version

IF (@INSTR(,%yarch,<FMA3>) NE 0)

yr4_4c_wpn4_djbfft_unroll_preload MACRO
	vmovapd	ymm15, YMM_ONE
	ENDM

;; uops = 8 s/c loads, 8 loads, 8 stores, 8 muls, 22 fma, 11 mov = 65 uops / 4 = 16.25 clocks
;; Timed at 19.2 clocks
yr4_4c_wpn4_djbfft_unroll MACRO iter,prev,this,next,srcreg,srcinc,d1,d2,screg1,scinc1,screg2,scinc2,L1pt,L1pd

;; On first call, init registers

	IF iter EQ -1
y0	TEXTEQU	<ymm0>
y1	TEXTEQU	<ymm1>
y2	TEXTEQU	<ymm2>
y3	TEXTEQU	<ymm3>
y4	TEXTEQU	<ymm4>
y5	TEXTEQU	<ymm5>
y6	TEXTEQU	<ymm6>
y7	TEXTEQU	<ymm7>
y8	TEXTEQU	<ymm8>
y9	TEXTEQU	<ymm9>
y10	TEXTEQU	<ymm10>
y11	TEXTEQU	<ymm11>
y12	TEXTEQU	<ymm12>
y13	TEXTEQU	<ymm13>
y14	TEXTEQU	<ymm14>
	ENDIF

;; On later calls, previous A2,B2,R3S,I3S,R4S,I4S will be in y0-5.  This weight1,weight2,R1,R3,R4,I1,I3,I4 will be in y6-13.
;; The remaining register is reserved for constant YMM_ONE.

this	yfmsubpd y14, y8, y6, y9			;; R1 * weight1 - R3 (new R3)		; 1-5		n 8
this	yfmaddpd y8, y8, y6, y9				;; R1 * weight1 + R3 (new R1)		; 1-5		n 9
this	L1prefetchw srcreg+iter*srcinc+L1pd, L1pt

this	yfmaddpd y9, y11, y6, y12			;; I1 * weight1 + I3 (new I1)		; 2-6		n 10
this	yfmsubpd y11, y11, y6, y12			;; I1 * weight1 - I3 (new I3)		; 2-6		n 14

prev	vmovapd y12, [screg2+(iter-1)*scinc2+0+32]	;; cosine/sine for R3/I3/R4/I4
prev	yfmsubpd y6, y2, y12, y3			;; R3S * cosine/sine - I3S (final R3)	; 4-8
prev	yfmaddpd y3, y3, y12, y2			;; I3S * cosine/sine + R3S (final I3)	; 4-8

this	vmovapd	y2, [srcreg+iter*srcinc+d1]		;; R2
prev	ystore	[srcreg+(iter-1)*srcinc+d2], y6		;; Save R3				;	9
this	yfmsubpd y6, y2, y7, y10			;; R2 * weight2 - R4 (new R4)		; 3-7		n 8
this	yfmaddpd y2, y2, y7, y10			;; R2 * weight2 + R4 (new R2)		; 3-7		n 9

this	vmovapd	y10, [srcreg+iter*srcinc+d1+32]		;; I2
prev	ystore	[srcreg+(iter-1)*srcinc+d2+32], y3	;; Save I3				;	10
this	yfmaddpd y3, y10, y7, y13			;; I2 * weight2 + I4 (new I2)		; 5-9		n 10
this	yfmsubpd y10, y10, y7, y13			;; I2 * weight2 - I4 (new I4)		; 5-9		n 13
prev	vmovapd y7, [screg2+(iter-1)*scinc2+64+0]	;; Sine for R2/I2

prev	yfmaddpd y13, y4, y12, y5			;; R4S * cosine/sine + I4S (final R4)	; 6-10
prev	yfmsubpd y5, y5, y12, y4			;; I4S * cosine/sine - R4S (final I4)	; 6-10
this	vmovapd y12, [screg2+iter*scinc2+0+0]		;; Sine for R3/I3/R4/I4

prev	vmulpd	y0, y0, y7				;; A2 = A2 * sine (final R2)		; 7-11
prev	vmulpd	y1, y1, y7				;; B2 = B2 * sine (final I2)		; 7-11
next	vbroadcastsd y4, Q [screg1+(iter+1)*scinc1+16]	;; Normalization multiplier for R3 & I3

this	vmulpd	y14, y14, y12				;; R3S = R3 * sine			; 8-12		n 13
this	vmulpd	y6, y6, y12				;; R4S = R4 * sine			; 8-12		n 14
this	L1prefetchw srcreg+iter*srcinc+d1+L1pd, L1pt

this	yfmsubpd y7, y8, ymm15, y2			;; R1 - R2 (newer R2)			; 9-13		n 15
this	yfmaddpd y8, y8, ymm15, y2			;; R1 + R2 (final R1)			; 9-13
this next yloop_unrolled_one

this	yfmsubpd y2, y9, ymm15, y3			;; I1 - I2 (newer I2)			; 10-14		n 15
this	yfmaddpd y9, y9, ymm15, y3			;; I1 + I2 (final I1)			; 10-14
this	L1prefetchw srcreg+iter*srcinc+d2+L1pd, L1pt

next	vmulpd	y3, y4, [srcreg+(iter+1)*srcinc+d2]	;; R3 = R3 * weight3			; 11-15		n 16
next	vmulpd	y4, y4, [srcreg+(iter+1)*srcinc+d2+32]	;; I3 = I3 * weight3			; 11-15		n 17
prev	ystore	[srcreg+(iter-1)*srcinc+d2+d1], y13	;; Save R4				;	11

next	vbroadcastsd y13, Q [screg1+(iter+1)*scinc1+24]	;; Normalization multiplier for R4 & I4
prev	ystore	[srcreg+(iter-1)*srcinc+d2+d1+32], y5	;; Save I4				;	12
next	vmulpd	y5, y13, [srcreg+(iter+1)*srcinc+d2+d1] ;; R4 = R4 * weight4			; 12-16		n 18
next	vmulpd	y13, y13, [srcreg+(iter+1)*srcinc+d2+d1+32] ;; I4 = I4 * weight4		; 12-16		n 20

prev	ystore	[srcreg+(iter-1)*srcinc+d1], y0		;; Save R2				;	13
this	yfnmaddpd y0, y10, y12, y14			;; R3S - I4 * sine (newer R3S)		; 13-17		n 19
this	yfmaddpd y10, y10, y12, y14			;; R3S + I4 * sine (newer R4S)		; 13-17		n 21
this	L1prefetchw srcreg+iter*srcinc+d2+d1+L1pd, L1pt

this	yfmaddpd y14, y11, y12, y6			;; I3 * sine + R4S (newer I3S)		; 14-18		n 19
this	yfmsubpd y11, y11, y12, y6			;; I3 * sine - R4S (newer I4S)		; 14-18		n 21
this	vmovapd y12, [screg2+iter*scinc2+64+32]		;; cosine/sine for R2/I2
next	vbroadcastsd y6, Q [screg1+(iter+1)*scinc1+0]	;; Normalization multiplier for R1 & I1
prev	ystore	[srcreg+(iter-1)*srcinc+d1+32], y1	;; Save I2				;	14

this	yfmsubpd y1, y7, y12, y2			;; A2 = R2 * cosine/sine - I2		; 15-19		n 22
this	yfmaddpd y2, y2, y12, y7			;; B2 = I2 * cosine/sine + R2		; 15-19		n 22
next	vmovapd	y12, [srcreg+(iter+1)*srcinc]		;; R1
next	vmovapd	y7, [srcreg+(iter+1)*srcinc+32]		;; I1
this	ystore	[srcreg+iter*srcinc], y8		;; Save R1				;	15
next	vbroadcastsd y8, Q [screg1+(iter+1)*scinc1+8]	;; Normalization multiplier for R2 & I2
this	ystore	[srcreg+iter*srcinc+32], y9		;; Save I1				;	16

;; Shuffle register assignments so that this A2,B2,R3S,I3S,R4S,I4S are in y0-5 and next weight1,weight2,R1,R3,R4,I1,I3,I4 are in y6-13.

ytmp	TEXTEQU	y0
y0	TEXTEQU	y1
y1	TEXTEQU	y2
y2	TEXTEQU	ytmp
ytmp	TEXTEQU	y3
y3	TEXTEQU	y14
y14	TEXTEQU	y9
y9	TEXTEQU	ytmp
ytmp	TEXTEQU	y4
y4	TEXTEQU	y10
y10	TEXTEQU	y5
y5	TEXTEQU	y11
y11	TEXTEQU y7
y7	TEXTEQU	y8
y8	TEXTEQU	y12
y12	TEXTEQU	ytmp
	ENDM
ENDIF

ENDIF







yr4_b4cl_wpn4_four_complex_djbunfft_preload MACRO
	ENDM
yr4_b4cl_wpn4_four_complex_djbunfft MACRO srcreg,srcinc,d1,d2,screg1,scinc1,screg2,scinc2,maxrpt,L1pt,L1pd
	vmovapd ymm3, [screg2+64+32]	;; cosine/sine2
	vmovapd	ymm2, [srcreg+d1]	;; R2
	vmulpd	ymm6, ymm2, ymm3	;; A2 = R2 * cosine/sine		;  1-5
	vmovapd	ymm0, [srcreg+d1+32]	;; I2
	vmulpd	ymm3, ymm3, ymm0	;; B2 = I2 * cosine/sine		;  2-6

	vmovapd ymm5, [screg2+0+32]	;; cosine/sine34
	vmovapd	ymm4, [srcreg+d2]	;; R3
	vmulpd	ymm7, ymm4, ymm5	;; A3 = R3 * cosine/sine		;  3-7
	vmovapd	ymm1, [srcreg+d2+32]	;; I3

	vaddpd	ymm6, ymm6, ymm0	;; A2 = A2 + I2				; 6-8

	vmulpd	ymm0, ymm1, ymm5	;; B3 = I3 * cosine/sine		;  4-8

	vsubpd	ymm3, ymm3, ymm2	;; B2 = B2 - R2				; 7-9
	vaddpd	ymm7, ymm7, ymm1	;; A3 = A3 + I3				; 8-10

	vmovapd	ymm2, [srcreg+d2+d1]	;; R4
	vmulpd	ymm1, ymm2, ymm5	;; A4 = R4 * cosine/sine		;  5-9

	vsubpd	ymm0, ymm0, ymm4	;; B3 = B3 - R3				; 9-11

	vmovapd	ymm4, [srcreg+d2+d1+32]	;; I4
	vmulpd	ymm5, ymm4, ymm5	;; B4 = I4 * cosine/sine		;  6-10

	vsubpd	ymm1, ymm1, ymm4	;; A4 = A4 - I4				; 10-12
	vaddpd	ymm5, ymm5, ymm2	;; B4 = B4 + R4				; 11-13

	vmovapd ymm4, [screg2+64+0]	;; Sine2
	vmulpd	ymm6, ymm6, ymm4	;; A2 = A2 * sine (new R2)		;  9-13
	vmulpd	ymm3, ymm3, ymm4	;; B2 = B2 * sine (new I2)		;  10-14

	vmovapd ymm4, [screg2+0+0]	;; Sine34
	vmulpd	ymm7, ymm7, ymm4	;; A3 = A3 * sine (new R3)		;  11-15
	vmulpd	ymm0, ymm0, ymm4	;; B3 = B3 * sine (new I3)		;  12-16
	vmulpd	ymm1, ymm1, ymm4	;; A4 = A4 * sine (new R4)		;  13-17
	vmulpd	ymm5, ymm5, ymm4	;; B4 = B4 * sine (new I4)		;  14-18
	L1prefetchw srcreg+L1pd, L1pt

	vmovapd	ymm2, [srcreg]		;; R1
	vaddpd	ymm4, ymm2, ymm6	;; R1 + R2 (new R1)			; 14-16
	vsubpd	ymm2, ymm2, ymm6	;; R1 - R2 (new R2)			; 15-17
	L1prefetchw srcreg+d1+L1pd, L1pt

	vaddpd	ymm6, ymm1, ymm7	;; R4 + R3 (new R3)			; 16-18
	vsubpd	ymm1, ymm1, ymm7	;; R4 - R3 (new I4)			; 17-19
	L1prefetchw srcreg+d2+L1pd, L1pt

	vsubpd	ymm7, ymm0, ymm5	;; I3 - I4 (new R4)			; 18-20
	vaddpd	ymm0, ymm0, ymm5	;; I3 + I4 (new I3)			; 19-21
	L1prefetchw srcreg+d2+d1+L1pd, L1pt

	vsubpd	ymm5, ymm4, ymm6	;; R1 - R3 (final R3)			; 20-22
	vaddpd	ymm4, ymm4, ymm6	;; R1 + R3 (final R1)			; 21-23

	vbroadcastsd ymm6, Q [screg1+16] ;; normalization_inverse for R3/I3
	vmulpd	ymm5, ymm5, ymm6	;; R3 * weight3				;  23-27
	ystore	[srcreg+d2], ymm5	;; Save R3				; 28

	vmovapd	ymm6, [srcreg+32]	;; I1
	vsubpd	ymm5, ymm6, ymm3	;; I1 - I2 (new I2)			; 22-24
	vaddpd	ymm6, ymm6, ymm3	;; I1 + I2 (new I1)			; 23-25

	vsubpd	ymm3, ymm2, ymm7	;; R2 - R4 (final R4)			; 24-26
	vaddpd	ymm2, ymm2, ymm7	;; R2 + R4 (final R2)			; 25-27

	vbroadcastsd ymm7, Q [screg1+0]	;; normalization_inverse for R1/I1
	vmulpd	ymm4, ymm4, ymm7	;; R1 * weight1				;  24-28
	ystore	[srcreg], ymm4		;; Save R1				; 29

	vsubpd	ymm4, ymm5, ymm1	;; I2 - I4 (final I4)			; 26-28
	vaddpd	ymm5, ymm5, ymm1	;; I2 + I4 (final I2)			; 27-29

	vbroadcastsd ymm1, Q [screg1+24] ;; normalization_inverse for R4/I4
	vmulpd	ymm3, ymm3, ymm1	;; R4 * weight4				;  27-31
	ystore	[srcreg+d2+d1], ymm3	;; Save R4				; 32

	vbroadcastsd ymm3, Q [screg1+8]	;; normalization_inverse for R2/I2
	vmulpd	ymm2, ymm2, ymm3	;; R2 * weight2				;  28-32
	ystore	[srcreg+d1], ymm2	;; Save R2				; 33

	vsubpd	ymm2, ymm6, ymm0	;; I1 - I3 (final I3)			; 28-30
	vaddpd	ymm6, ymm6, ymm0	;; I1 + I3 (final I1)			; 29-31
	vbroadcastsd ymm0, Q [screg1+16] ;; normalization_inverse for R3/I3

	vmulpd	ymm4, ymm4, ymm1	;; I4 * weight4				;  29-33
	vmulpd	ymm5, ymm5, ymm3	;; I2 * weight2				;  30-34
	vmulpd	ymm2, ymm2, ymm0	;; I3 * weight3				;  31-35
	vmulpd	ymm6, ymm6, ymm7	;; I1 * weight1				;  32-36
	ystore	[srcreg+d2+d1+32], ymm4	;; Save I4				; 34
	ystore	[srcreg+d1+32], ymm5	;; Save I2				; 35
	ystore	[srcreg+d2+32], ymm2	;; Save I3				; 36
	ystore	[srcreg+32], ymm6	;; Save I1				; 37

	bump	srcreg, srcinc
	bump	screg1, scinc1
	bump	screg2, scinc2
	ENDM


;; 64-bit version

IFDEF X86_64

yr4_b4cl_wpn4_four_complex_djbunfft_preload MACRO
	yr4_4c_wpn4_djbunfft_unroll_preload
	ENDM
yr4_b4cl_wpn4_four_complex_djbunfft MACRO srcreg,srcinc,d1,d2,screg1,scinc1,screg2,scinc2,maxrpt,L1pt,L1pd
	IF maxrpt MOD 5 EQ 0
	yr4_4c_wpn4_djbunfft_unroll -1,noexec,noexec,exec,srcreg,srcinc,d1,d2,screg1,scinc1,screg2,scinc2,L1pt,L1pd
	yr4_4c_wpn4_djbunfft_unroll 0,noexec,exec,exec,srcreg,srcinc,d1,d2,screg1,scinc1,screg2,scinc2,L1pt,L1pd
	yr4_4c_wpn4_djbunfft_unroll 1,exec,exec,exec,srcreg,srcinc,d1,d2,screg1,scinc1,screg2,scinc2,L1pt,L1pd
	yr4_4c_wpn4_djbunfft_unroll 2,exec,exec,exec,srcreg,srcinc,d1,d2,screg1,scinc1,screg2,scinc2,L1pt,L1pd
	yr4_4c_wpn4_djbunfft_unroll 3,exec,exec,exec,srcreg,srcinc,d1,d2,screg1,scinc1,screg2,scinc2,L1pt,L1pd
	yr4_4c_wpn4_djbunfft_unroll 4,exec,exec,noexec,srcreg,srcinc,d1,d2,screg1,scinc1,screg2,scinc2,L1pt,L1pd
	yr4_4c_wpn4_djbunfft_unroll 5,exec,noexec,noexec,srcreg,srcinc,d1,d2,screg1,scinc1,screg2,scinc2,L1pt,L1pd
	bump	srcreg, 5*srcinc
	bump	screg1, 5*scinc1
	bump	screg2, 5*scinc2
	ELSEIF maxrpt MOD 4 EQ 0
	yr4_4c_wpn4_djbunfft_unroll -1,noexec,noexec,exec,srcreg,srcinc,d1,d2,screg1,scinc1,screg2,scinc2,L1pt,L1pd
	yr4_4c_wpn4_djbunfft_unroll 0,noexec,exec,exec,srcreg,srcinc,d1,d2,screg1,scinc1,screg2,scinc2,L1pt,L1pd
	yr4_4c_wpn4_djbunfft_unroll 1,exec,exec,exec,srcreg,srcinc,d1,d2,screg1,scinc1,screg2,scinc2,L1pt,L1pd
	yr4_4c_wpn4_djbunfft_unroll 2,exec,exec,exec,srcreg,srcinc,d1,d2,screg1,scinc1,screg2,scinc2,L1pt,L1pd
	yr4_4c_wpn4_djbunfft_unroll 3,exec,exec,noexec,srcreg,srcinc,d1,d2,screg1,scinc1,screg2,scinc2,L1pt,L1pd
	yr4_4c_wpn4_djbunfft_unroll 4,exec,noexec,noexec,srcreg,srcinc,d1,d2,screg1,scinc1,screg2,scinc2,L1pt,L1pd
	bump	srcreg, 4*srcinc
	bump	screg1, 4*scinc1
	bump	screg2, 4*scinc2
	ELSEIF maxrpt MOD 3 EQ 0
	yr4_4c_wpn4_djbunfft_unroll -1,noexec,noexec,exec,srcreg,srcinc,d1,d2,screg1,scinc1,screg2,scinc2,L1pt,L1pd
	yr4_4c_wpn4_djbunfft_unroll 0,noexec,exec,exec,srcreg,srcinc,d1,d2,screg1,scinc1,screg2,scinc2,L1pt,L1pd
	yr4_4c_wpn4_djbunfft_unroll 1,exec,exec,exec,srcreg,srcinc,d1,d2,screg1,scinc1,screg2,scinc2,L1pt,L1pd
	yr4_4c_wpn4_djbunfft_unroll 2,exec,exec,noexec,srcreg,srcinc,d1,d2,screg1,scinc1,screg2,scinc2,L1pt,L1pd
	yr4_4c_wpn4_djbunfft_unroll 3,exec,noexec,noexec,srcreg,srcinc,d1,d2,screg1,scinc1,screg2,scinc2,L1pt,L1pd
	bump	srcreg, 3*srcinc
	bump	screg1, 3*scinc1
	bump	screg2, 3*scinc2
	ELSEIF maxrpt MOD 2 EQ 0
	yr4_4c_wpn4_djbunfft_unroll -1,noexec,noexec,exec,srcreg,srcinc,d1,d2,screg1,scinc1,screg2,scinc2,L1pt,L1pd
	yr4_4c_wpn4_djbunfft_unroll 0,noexec,exec,exec,srcreg,srcinc,d1,d2,screg1,scinc1,screg2,scinc2,L1pt,L1pd
	yr4_4c_wpn4_djbunfft_unroll 1,exec,exec,noexec,srcreg,srcinc,d1,d2,screg1,scinc1,screg2,scinc2,L1pt,L1pd
	yr4_4c_wpn4_djbunfft_unroll 2,exec,noexec,noexec,srcreg,srcinc,d1,d2,screg1,scinc1,screg2,scinc2,L1pt,L1pd
	bump	srcreg, 2*srcinc
	bump	screg1, 2*scinc1
	bump	screg2, 2*scinc2
	ELSE
	yr4_4c_wpn4_djbunfft_unroll -1,noexec,noexec,exec,srcreg,srcinc,d1,d2,screg1,scinc1,screg2,scinc2,L1pt,L1pd
	yr4_4c_wpn4_djbunfft_unroll 0,noexec,exec,noexec,srcreg,srcinc,d1,d2,screg1,scinc1,screg2,scinc2,L1pt,L1pd
	yr4_4c_wpn4_djbunfft_unroll 1,exec,noexec,noexec,srcreg,srcinc,d1,d2,screg1,scinc1,screg2,scinc2,L1pt,L1pd
	bump	srcreg, srcinc
	bump	screg1, scinc1
	bump	screg2, scinc2
	ENDIF
	ENDM

yr4_4c_wpn4_djbunfft_unroll_preload MACRO
	ENDM
yr4_4c_wpn4_djbunfft_unroll MACRO iter,prev,this,next,srcreg,srcinc,d1,d2,screg1,scinc1,screg2,scinc2,L1pt,L1pd

;; On first call, init registers

	IF iter EQ -1
y0	TEXTEQU	<ymm0>
y1	TEXTEQU	<ymm1>
y2	TEXTEQU	<ymm2>
y3	TEXTEQU	<ymm3>
y4	TEXTEQU	<ymm4>
y5	TEXTEQU	<ymm5>
y6	TEXTEQU	<ymm6>
y7	TEXTEQU	<ymm7>
y8	TEXTEQU	<ymm8>
y9	TEXTEQU	<ymm9>
y10	TEXTEQU	<ymm10>
y11	TEXTEQU	<ymm11>
y12	TEXTEQU	<ymm12>
y13	TEXTEQU	<ymm13>
y14	TEXTEQU	<ymm14>
y15	TEXTEQU	<ymm15>
	ENDIF

;; On later calls, previous R1,R2,R3,R4,I1,I2,I3,I4 are in y0-7.  This R2,R3/s,R4/s,B2,I3/s,I4/s,sine2 are in y8-14.
;; The remaining register is free.

prev	vbroadcastsd y15, Q [screg1+(iter-1)*scinc1+16] ;; normalization_inverse for R3 & I3
prev	ystore	[srcreg+(iter-1)*srcinc+d2+d1+32], y7 ;; Save I4			; 1
this	vaddpd	y7, y10, y9			;; C3 = R4/s + R3/s (newer R3/sine)	; 1-3		n 4
this	vmulpd	y11, y11, y14			;; B2 = B2 * sine (new I2)		; 1-5		n 7

prev	ystore	[srcreg+(iter-1)*srcinc+d2+d1], y3 ;; Save R4				; 2
this	vsubpd	y10, y10, y9			;; D4 = R4/s - R3/s (newer I4/sine)	; 2-4		n 5
prev	vmulpd	y2, y2, y15			;; R3 * normalization_inverse		;  2-6
this	vmovapd y14, [screg2+iter*scinc2+0+0]	;; Sine34

prev	ystore	[srcreg+(iter-1)*srcinc+d1+32], y5 ;; Save I2				; 3
this	vsubpd	y5, y12, y13			;; C4 = I3/s - I4/s (newer R4/sine)	; 3-5		n 6
prev	vmulpd	y6, y6, y15			;; I3 * normalization_inverse		;  3-7
this	vmovapd	y3, [srcreg+iter*srcinc]	;; R1

prev	ystore	[srcreg+(iter-1)*srcinc+d1], y1	;; Save R2				; 4
this	vaddpd	y12, y12, y13			;; D3 = I3/s + I4/s (newer I3/sine)	; 4-6		n 7
this	vmulpd	y7, y7, y14			;; C3 * sine (newer R3)			;  4-8		n 9
this	vmovapd	y9, [srcreg+iter*srcinc+32]	;; I1

this	vaddpd	y13, y3, y8			;; R1 + R2 (newer R1)			; 5-7		n 9
this	vmulpd	y10, y10, y14			;; D4 * sine (newer I4)			;  5-9		n 11
prev	vbroadcastsd y15, Q [screg1+(iter-1)*scinc1+0] ;; normalization_inverse for R1 & I1

this	vsubpd	y3, y3, y8			;; R1 - R2 (newer R2)			; 6-8		n 13
this	vmulpd	y5, y5, y14			;; C4 * sine (newer R4)			;  6-10		n 13
next	vmovapd y1, [screg2+(iter+1)*scinc2+64+32] ;; cosine/sine2

prev	ystore	[srcreg+(iter-1)*srcinc+d2], y2	;; Save R3				; 7
this	vsubpd	y2, y9, y11			;; I1 - I2 (newer I2)			; 7-9		n 11
this	vmulpd	y12, y12, y14			;; D3 * sine (newer I3)			;  7-11		n 15
next	vmovapd	y8, [srcreg+(iter+1)*srcinc+d1]	;; R2

prev	ystore	[srcreg+(iter-1)*srcinc+d2+32], y6 ;; Save I3				; 8
this	vaddpd	y9, y9, y11			;; I1 + I2 (newer I1)			; 8-10		n 15
prev	vmulpd	y0, y0, y15			;; R1 * normalization_inverse		;  8-12
next	vmovapd	y14, [srcreg+(iter+1)*srcinc+d1+32] ;; I2

this	vsubpd	y11, y13, y7			;; R1 - R3 (final R3)			; 9-11		n 12
prev	vmulpd	y4, y4, y15			;; I1 * normalization_inverse		;  9-13
next	vmovapd y6, [screg2+(iter+1)*scinc2+0+32] ;; cosine/sine34

this	vaddpd	y13, y13, y7			;; R1 + R3 (final R1)			; 10-12		n 13
next	vmovapd	y15, [srcreg+(iter+1)*srcinc+d2+d1] ;; R4

this	vsubpd	y7, y2, y10			;; I2 - I4 (final I4)			; 11-13		n 14
this	L1prefetchw srcreg+iter*srcinc+L1pd, L1pt

this	vaddpd	y2, y2, y10			;; I2 + I4 (final I2)			; 12-14		n 15
next	vmulpd	y10, y8, y1			;; A2 = R2 * cosine/sine		;  12-16	n 17
this next yloop_unrolled_one

prev	ystore	[srcreg+(iter-1)*srcinc], y0	;; Save R1				; 13
this	vsubpd	y0, y3, y5			;; R2 - R4 (final R4)			; 13-15		n 16
next	vmulpd	y1, y14, y1			;; B2 = I2 * cosine/sine		;  13-17	n 18
this	L1prefetchw srcreg+iter*srcinc+d1+L1pd, L1pt

prev	ystore	[srcreg+(iter-1)*srcinc+32], y4	;; Save I1				; 14
this	vaddpd	y3, y3, y5			;; R2 + R4 (final R2)			; 14-16		n 17
next	vmulpd	y5, y15, y6			;; A4 = R4 * cosine/sine		;  14-18	n 19
this	L1prefetchw srcreg+iter*srcinc+d2+L1pd, L1pt

this	vsubpd	y4, y9, y12			;; I1 - I3 (final I3)			; 15-17		n 18
this	vaddpd	y9, y9, y12			;; I1 + I3 (final I1)			; 16-18		n 19
next	vmovapd	y12, [srcreg+(iter+1)*srcinc+d2] ;; R3
next	vaddpd	y10, y10, y14			;; A2 = A2 + I2				; 17-19		n 21
next	vmulpd	y14, y12, y6			;; A3 = R3 * cosine/sine		;  15-19	n 20
this	L1prefetchw srcreg+iter*srcinc+d2+d1+L1pd, L1pt

next	vsubpd	y1, y1, y8			;; B2 = B2 - R2				; 18-20		n 3
next	vmovapd	y8, [srcreg+(iter+1)*srcinc+d2+d1+32] ;; I4
next	vsubpd	y5, y5, y8			;; A4 = A4 - I4 (new R4/sine)		; 19-21		n 1
next	vmulpd	y8, y8, y6			;; B4 = I4 * cosine/sine		;  16-20	n 21
next	vmulpd	y6, y6, [srcreg+(iter+1)*srcinc+d2+32] ;; B3 = I3 * cosine/sine		;  17-21	n 22

next	vaddpd	y14, y14, [srcreg+(iter+1)*srcinc+d2+32] ;; A3 = A3 + I3 (new R3/sine)	; 20-22		n 1

next	vaddpd	y8, y8, y15			;; B4 = B4 + R4 (new I4/sine)		; 21-23		n 3
this	vbroadcastsd y15, Q [screg1+iter*scinc1+24] ;; normalization_inverse for R4 & I4
this	vmulpd	y7, y7, y15			;; I4 * normalization_inverse		;  18-22
this	vmulpd	y0, y0, y15			;; R4 * normalization_inverse		;  19-23
this	vbroadcastsd y15, Q [screg1+iter*scinc1+8] ;; normalization_inverse for R2 & I2
this	vmulpd	y2, y2, y15			;; I2 * normalization_inverse		;  20-24
this	vmulpd	y3, y3, y15			;; R2 * normalization_inverse		;  21-25

next	vmovapd y15, [screg2+(iter+1)*scinc2+64+0] ;; Sine2
next	vsubpd	y6, y6, y12			;; B3 = B3 - R3 (new I3/sine)		; 22-24		n 3
next	vmulpd	y10, y10, y15			;; A2 = A2 * sine (new R2)		;  22-26	n 5

;; Shuffle register assignments so that this R1,R2,R3,R4,I1,I2,I3,I4 are in y0-7 and next R2,R3/s,R4/s,B2,I3/s,I4/s,sine2 are in y8-14.

ytmp	TEXTEQU	y0
y0	TEXTEQU	y13
y13	TEXTEQU	y8
y8	TEXTEQU y10
y10	TEXTEQU	y5
y5	TEXTEQU	y2
y2	TEXTEQU	y11
y11	TEXTEQU	y1
y1	TEXTEQU	y3
y3	TEXTEQU	ytmp
ytmp	TEXTEQU	y4
y4	TEXTEQU	y9
y9	TEXTEQU	y14
y14	TEXTEQU	y15
y15	TEXTEQU	y12
y12	TEXTEQU	y6
y6	TEXTEQU	ytmp
	ENDM


;; Haswell FMA3 version

IF (@INSTR(,%yarch,<FMA3>) NE 0)

yr4_4c_wpn4_djbunfft_unroll_preload MACRO
	ENDM

;; uops = 8 s/c loads, 8 loads, 8 stores, 8 muls, 22 fma, 11 mov = 65 uops / 4 = 16.25 clocks
;; Timed at 19.4 clocks
yr4_4c_wpn4_djbunfft_unroll MACRO iter,prev,this,next,srcreg,srcinc,d1,d2,screg1,scinc1,screg2,scinc2,L1pt,L1pd

;; On first call, init registers

	IF iter EQ -1
y0	TEXTEQU	<ymm0>
y1	TEXTEQU	<ymm1>
y2	TEXTEQU	<ymm2>
y3	TEXTEQU	<ymm3>
y4	TEXTEQU	<ymm4>
y5	TEXTEQU	<ymm5>
y6	TEXTEQU	<ymm6>
y7	TEXTEQU	<ymm7>
y8	TEXTEQU	<ymm8>
y9	TEXTEQU	<ymm9>
y10	TEXTEQU	<ymm10>
y11	TEXTEQU	<ymm11>
y12	TEXTEQU	<ymm12>
y13	TEXTEQU	<ymm13>
y14	TEXTEQU	<ymm14>
y15	TEXTEQU	<ymm15>
	ENDIF

;; On later calls, previous R1,R2,R3,R4,I1,I2,I3,I4 will be in y0-7.  This R2,I2,R3/s,I3/s,R4/s,I4/s,c/s2 will be in y8-14.

this	yfmaddpd y15, y8, y14, y9			;; R2 * cosine/sine + I2 (new R2/sine)	; 1-5		n 6
this	yfmsubpd y9, y9, y14, y8			;; I2 * cosine/sine - R2 (new I2/sine)	; 1-5		n 8

prev	vbroadcastsd y14, Q [screg1+(iter-1)*scinc1+16]	;; normalization_inverse for R3 & I3
prev	vbroadcastsd y8, Q [screg1+(iter-1)*scinc1+0]	;; normalization_inverse for R1 & I1
prev	vmulpd	y2, y2, y14				;; R3 * normalization_inverse		; 2-6
prev	vmulpd	y0, y0, y8				;; R1 * normalization_inverse		; 2-6
this	L1prefetchw srcreg+iter*srcinc+L1pd, L1pt

prev	vmulpd	y6, y6, y14				;; I3 * normalization_inverse		; 3-7
prev	vmulpd	y4, y4, y8				;; I1 * normalization_inverse		; 3-7

prev	vbroadcastsd y14, Q [screg1+(iter-1)*scinc1+24]	;; normalization_inverse for R4 & I4
prev	vbroadcastsd y8, Q [screg1+(iter-1)*scinc1+8]	;; normalization_inverse for R2 & I2
prev	vmulpd	y3, y3, y14				;; R4 * normalization_inverse		; 4-8
prev	vmulpd	y1, y1, y8				;; R2 * normalization_inverse		; 4-8

prev	ystore	[srcreg+(iter-1)*srcinc+d2], y2		;; Save R3				;	7
this	vmovapd	y2, YMM_ONE
prev	ystore	[srcreg+(iter-1)*srcinc], y0		;; Save R1				;	8
this	yfmaddpd y0, y12, y2, y10			;; R4/sine + R3/sine (newer R3/sine)	; 5-9		n 12
this	yfmsubpd y12, y12, y2, y10			;; R4/sine - R3/sine (newer I4/sine)	; 5-9		n 15

this	vmovapd y10, [screg2+iter*scinc2+64+0]		;; Sine for R2/I2
prev	ystore	[srcreg+(iter-1)*srcinc+d2+32], y6	;; Save I3				;	9
this	vmovapd	y6, [srcreg+iter*srcinc]		;; R1
prev	ystore	[srcreg+(iter-1)*srcinc+32], y4		;; Save I1				;	10
this	yfmaddpd y4, y15, y10, y6			;; R1 + R2/sine * sine (newer R1)	; 6-10		n 12
this	yfnmaddpd y15, y15, y10, y6			;; R1 - R2/sine * sine (newer R2)	; 6-10		n 14
this	L1prefetchw srcreg+iter*srcinc+d1+L1pd, L1pt

this	yfmaddpd y6, y11, y2, y13			;; I3/sine + I4/sine (newer I3/sine)	; 7-11		n 13
this	yfmsubpd y11, y11, y2, y13			;; I3/sine - I4/sine (newer R4/sine)	; 7-11		n 14

this	vmovapd	y2, [srcreg+iter*srcinc+32]		;; I1
this	yfmaddpd y13, y9, y10, y2			;; I1 + I2/sine * sine (newer I1)	; 8-12		n 13
this	yfnmaddpd y9, y9, y10, y2			;; I1 - I2/sine * sine (newer I2)	; 8-12		n 15
next	vmovapd y10, [screg2+(iter+1)*scinc2+0+32]	;; cosine/sine for R3/I3/R4/I4

this	L1prefetchw srcreg+iter*srcinc+d2+L1pd, L1pt
prev	vmulpd	y7, y7, y14				;; I4 * normalization_inverse		; 9-13
prev	vmulpd	y5, y5, y8				;; I2 * normalization_inverse		; 9-13

next	vmovapd	y2, [srcreg+(iter+1)*srcinc+d2]		;; R3
next	vmovapd	y14, [srcreg+(iter+1)*srcinc+d2+32]	;; I3
next	yfmaddpd y8, y2, y10, y14			;; R3 * cosine/sine + I3 (new R3/sine)	; 10-14		n 20
next	yfmsubpd y14, y14, y10, y2			;; I3 * cosine/sine - R3 (new I3/sine)	; 10-14		n 22

next	vmovapd	y2, [srcreg+(iter+1)*srcinc+d2+d1]	;; R4
prev	ystore	[srcreg+(iter-1)*srcinc+d2+d1], y3	;; Save R4				;	11
next	vmovapd	y3, [srcreg+(iter+1)*srcinc+d2+d1+32]	;; I4
prev	ystore	[srcreg+(iter-1)*srcinc+d1], y1		;; Save R2				;	12
next	yfmsubpd y1, y2, y10, y3			;; R4 * cosine/sine - I4 (new R4/sine)	; 11-15		n 20
next	yfmaddpd y3, y3, y10, y2			;; I4 * cosine/sine + R4 (new I4/sine)	; 11-15		n 22

this	vmovapd y10, [screg2+iter*scinc2+0+0]		;; Sine for R3/I3/R4/I4
this	yfnmaddpd y2, y0, y10, y4			;; R1 - R3/sine * sine (final R3)	; 12-16		n 2
this	yfmaddpd y0, y0, y10, y4			;; R1 + R3/sine * sine (final R1)	; 12-16		n 2
this	L1prefetchw srcreg+iter*srcinc+d2+d1+L1pd, L1pt

this	yfnmaddpd y4, y6, y10, y13			;; I1 - I3/sine * sine (final I3)	; 13-17		n 3
this	yfmaddpd y6, y6, y10, y13			;; I1 + I3/sine * sine (final I1)	; 13-17		n 3
next	vmovapd y13, [screg2+(iter+1)*scinc2+64+32]	;; cosine/sine for R2/I2
this next yloop_unrolled_one

prev	ystore	[srcreg+(iter-1)*srcinc+d2+d1+32], y7	;; Save I4				;	14
this	yfnmaddpd y7, y11, y10, y15			;; R2 - R4/sine * sine (final R4)	; 14-18		n 4
this	yfmaddpd y11, y11, y10, y15			;; R2 + R4/sine * sine (final R2)	; 14-18		n 4
next	vmovapd	y15, [srcreg+(iter+1)*srcinc+d1]	;; R2

prev	ystore	[srcreg+(iter-1)*srcinc+d1+32], y5	;; Save I2				;	15
this	yfnmaddpd y5, y12, y10, y9			;; I2 - I4/sine * sine (final I4)	; 15-19		n 5
this	yfmaddpd y12, y12, y10, y9			;; I2 + I4/sine * sine (final I2)	; 15-19		n 5
next	vmovapd	y10, [srcreg+(iter+1)*srcinc+d1+32]	;; I2

;; Shuffle register assignments so that this R1,R2,R3,R4,I1,I2,I3,I4 are in y0-7, and
;; next R2,I2,R3/s,I3/s,R4/s,I4/s,c/s2 are in y8-14.

ytmp	TEXTEQU	y1
y1	TEXTEQU	y11
y11	TEXTEQU y14
y14	TEXTEQU	y13
y13	TEXTEQU	y3
y3	TEXTEQU	y7
y7	TEXTEQU	y5
y5	TEXTEQU	y12
y12	TEXTEQU	ytmp
ytmp	TEXTEQU	y4
y4	TEXTEQU	y6
y6	TEXTEQU	ytmp
ytmp	TEXTEQU	y8
y8	TEXTEQU	y15
y15	TEXTEQU	y9
y9	TEXTEQU	y10
y10	TEXTEQU	ytmp
	ENDM
ENDIF

ENDIF










;;
;; ******************************* eight-real-with-partial-normalization4 variants *************************************
;;
;; These macros are used in pass 1 of yr4dwpn two pass FFTs.  They are like the original eight-real-wpn
;; macros except that 4 normalization multipliers are pre-applied rather than post-applying one normalization
;; multiplier.  This results in more multiplies (slower macros), but reduces group multiplier data and also
;; reduces variable sine/cosine data used in pass 1.  On a Haswell machine, this is a net positive.
;;

;; Uses one combined sin/cos data ptr.  The first two sin/cos entries can be used by four complex
;; macros at the same FFT level.  The third and fourth sin/cos entries are used by this eight reals macro.

;; screg1 is normalization weights
;; screg2+0, screg2+32 is sin/cos values for R2/I2 (w^2n)
;; screg2+128+0, screg2+128+32 is sin/cos values for R3/I3 (w^n)
;; screg2+192+0, screg2+192+32 is sin/cos values for R4/I4 (w^5n)

yr4_b4cl_csc_wpn4_eight_reals_fft_preload MACRO
	ENDM
yr4_b4cl_csc_wpn4_eight_reals_fft MACRO srcreg,srcinc,d1,d2,screg1,scinc1,screg2,scinc2,maxrpt,L1pt,L1pd
	vmovapd	ymm0, [srcreg+d2+d1]	;; R4
	vmovapd	ymm7, [srcreg+d2+d1+32]	;; R8
	vsubpd	ymm3, ymm0, ymm7	;; new R8 = R4 - R8			; 1-3
	vaddpd	ymm7, ymm0, ymm7	;; new R4 = R4 + R8			; 2-4

	vmovapd	ymm4, [srcreg+d1]	;; R2
	vmovapd	ymm5, [srcreg+d1+32]	;; R6
	vsubpd	ymm1, ymm4, ymm5	;; new R6 = R2 - R6			; 3-5
	vaddpd	ymm5, ymm4, ymm5	;; new R2 = R2 + R6			; 4-6

	vbroadcastsd ymm0, Q [screg1+24];; normalization for R4/I4
	vmulpd	ymm3, ymm3, ymm0	;; I4 * normalization			;  4-8
	vmulpd	ymm7, ymm7, ymm0	;; R4 * normalization			;  5-9

	vmovapd	ymm2, [srcreg]		;; R1
	vmovapd	ymm4, [srcreg+32]	;; R5
	vsubpd	ymm0, ymm2, ymm4	;; new R5 = R1 - R5			; 5-7
	vaddpd	ymm4, ymm2, ymm4	;; new R1 = R1 + R5			; 6-8

	vbroadcastsd ymm6, Q [screg1+8]	;; normalization for R2/I2
	vmulpd	ymm1, ymm1, ymm6	;; I2 * normalization			;  6-10
	vmulpd	ymm5, ymm5, ymm6	;; R2 * normalization			;  7-11

	vbroadcastsd ymm2, Q [screg1]	;; normalization for R1/I1
	vmulpd	ymm0, ymm0, ymm2	;; I1 * normalization			;  9-13
	vmulpd	ymm4, ymm4, ymm2	;; R1 * normalization			;  8-12

	vmulpd	ymm3, ymm3, YMM_SQRTHALF ;; R8 = R8 * square root		;  10-14
	vmulpd	ymm1, ymm1, YMM_SQRTHALF ;; R6 = R6 * square root		;  11-15

	vaddpd	ymm2, ymm5, ymm7	;; R2 + R4 (final I1 / 2nd real result)	; 12-14
	vsubpd	ymm5, ymm5, ymm7	;; R2 - R4 (final I2)			; 13-15

	vmovapd	ymm6, [srcreg+d2]	;; R3
	vmovapd	ymm7, [srcreg+d2+32]	;; R7
	ystore	[srcreg+32], ymm2	;; Save I1				;  15
	vaddpd	ymm2, ymm6, ymm7	;; new R3 = R3 + R7			; 7-9
	vsubpd	ymm6, ymm6, ymm7	;; new R7 = R3 - R7			; 8-10

	vbroadcastsd ymm7, Q [screg1+16];; normalization for R3/I3
	vmulpd	ymm2, ymm2, ymm7	;; R3 * normalization			;  12-16
	vmulpd	ymm6, ymm6, ymm7	;; I3 * normalization			;  13-17

	L1prefetchw srcreg+L1pd, L1pt

	vsubpd	ymm7, ymm1, ymm3	;; R6 = R6 - R8 (Real part)		; 16-18
	vaddpd	ymm1, ymm1, ymm3	;; R8 = R6 + R8 (Imaginary part)	; 17-19

	vsubpd	ymm3, ymm4, ymm2	;; R1 - R3 (final R2)			; 18-20
	vaddpd	ymm4, ymm4, ymm2	;; R1 + R3 (final R1)			; 19-21

	L1prefetchw srcreg+d1+L1pd, L1pt

	vsubpd	ymm2, ymm0, ymm7	;; R5 - R6 (final R4)			; 20-22
	vaddpd	ymm0, ymm0, ymm7	;; R5 + R6 (final R3)			; 21-23

	ystore	[srcreg], ymm4		;; Save R1				; 22
	vsubpd	ymm7, ymm6, ymm1	;; R7 - R8 (final I4)			; 22-24
	vaddpd	ymm6, ymm6, ymm1	;; R7 + R8 (final I3)			; 23-25

	vmovapd	ymm4, [screg2+0+32]	;; cosine/sine for w^2n
	vmulpd	ymm1, ymm5, ymm4	;; B2 = I2 * cosine/sine for w^2n	;  16-20
	vaddpd	ymm1, ymm1, ymm3	;; B2 = B2 + R2				; 24-26
	vmulpd	ymm3, ymm3, ymm4	;; A2 = R2 * cosine/sine for w^2n	;  21-25
	vsubpd	ymm3, ymm3, ymm5	;; A2 = A2 - I2				; 25-27

	L1prefetchw srcreg+d2+L1pd, L1pt

	vmovapd	ymm4, [screg2+192+32]	;; cosine/sine for w^5n
	vmulpd	ymm5, ymm2, ymm4	;; A4 = R4 * cosine/sine for w^5n	;  23-27
	vsubpd	ymm5, ymm5, ymm7	;; A4 = A4 - I4				; 28-30
	vmulpd	ymm7, ymm7, ymm4	;; B4 = I4 * cosine/sine for w^5n	;  25-29
	vaddpd	ymm7, ymm7, ymm2	;; B4 = B4 + R4				; 30-32

	vmovapd	ymm4, [screg2+128+32]	;; cosine/sine for w^n
	vmulpd	ymm2, ymm0, ymm4	;; A3 = R3 * cosine/sine for w^n	;  24-28
	vsubpd	ymm2, ymm2, ymm6	;; A3 = A3 - I3				; 29-31
	vmulpd	ymm6, ymm6, ymm4	;; B3 = I3 * cosine/sine for w^n	;  26-30
	vaddpd	ymm6, ymm6, ymm0	;; B3 = B3 + R3				; 31-33

	L1prefetchw srcreg+d2+d1+L1pd, L1pt

	vmovapd	ymm0, [screg2+0+0]	;; sine for w^2n
	vmulpd	ymm1, ymm1, ymm0	;; B2 = B2 * sine (final I2)		;  27-31
	vmulpd	ymm3, ymm3, ymm0	;; A2 = A2 * sine (final R2)		;  28-32
	vmovapd	ymm0, [screg2+192+0]	;; sine for w^5n
	vmulpd	ymm5, ymm5, ymm0	;; A4 = A4 * sine (final R4)		;  31-35
	vmulpd	ymm7, ymm7, ymm0	;; B4 = B4 * sine (final I4)		;  33-37
	vmovapd	ymm0, [screg2+128+0]	;; sine for w^n
	vmulpd	ymm2, ymm2, ymm0	;; A3 = A3 * sine (final R3)		;  32-36
	vmulpd	ymm6, ymm6, ymm0	;; B3 = B3 * sine (final I3)		;  34-38

	ystore	[srcreg+d1+32], ymm1	;; Save I2
	ystore	[srcreg+d1], ymm3	;; Save R2
	ystore	[srcreg+d2+d1], ymm5	;; Save R4
	ystore	[srcreg+d2+d1+32], ymm7	;; Save I4
	ystore	[srcreg+d2], ymm2	;; Save R3
	ystore	[srcreg+d2+32], ymm6	;; Save I3
	bump	srcreg, srcinc
	bump	screg1, scinc1
	bump	screg2, scinc2
	ENDM

;; 64-bit version

IFDEF X86_64

;; Uses one combined sin/cos data ptr.  The first two sin/cos entries can be used by four complex
;; macros at the same FFT level.  The third and fourth sin/cos entries are used by this eight reals macro.
yr4_b4cl_csc_wpn4_eight_reals_fft_preload MACRO
	vmovapd	ymm15, YMM_SQRTHALF
	ENDM
yr4_b4cl_csc_wpn4_eight_reals_fft MACRO srcreg,srcinc,d1,d2,screg1,scinc1,screg2,scinc2,maxrpt,L1pt,L1pd
	vmovapd	ymm0, [srcreg+d2+d1]	;; R4
	vmovapd	ymm7, [srcreg+d2+d1+32]	;; R8
	vsubpd	ymm3, ymm0, ymm7	;; new R8 = R4 - R8			; 1-3
	vaddpd	ymm7, ymm0, ymm7	;; new R4 = R4 + R8			; 2-4

	vmovapd	ymm4, [srcreg+d1]	;; R2
	vmovapd	ymm5, [srcreg+d1+32]	;; R6
	vsubpd	ymm1, ymm4, ymm5	;; new R6 = R2 - R6			; 3-5
	vaddpd	ymm5, ymm4, ymm5	;; new R2 = R2 + R6			; 4-6

	vbroadcastsd ymm0, Q [screg1+24];; normalization for R4/I4
	vmulpd	ymm3, ymm3, ymm0	;; I4 * weight4				;  4-8
	vmulpd	ymm7, ymm7, ymm0	;; R4 * weight4				;  5-9

	vmovapd	ymm2, [srcreg]		;; R1
	vmovapd	ymm4, [srcreg+32]	;; R5
	vsubpd	ymm0, ymm2, ymm4	;; new R5 = R1 - R5			; 5-7
	vaddpd	ymm4, ymm2, ymm4	;; new R1 = R1 + R5			; 6-8

	vbroadcastsd ymm6, Q [screg1+8]	;; normalization for R2/I2
	vmulpd	ymm1, ymm1, ymm6	;; I2 * weight2				;  6-10
	vmulpd	ymm5, ymm5, ymm6	;; R2 * weight2				;  7-11

	vmovapd	ymm6, [srcreg+d2]	;; R3
	vmovapd	ymm9, [srcreg+d2+32]	;; R7
	vaddpd	ymm2, ymm6, ymm9	;; new R3 = R3 + R7			; 7-9
	vsubpd	ymm6, ymm6, ymm9	;; new R7 = R3 - R7			; 8-10

	vbroadcastsd ymm10, Q [screg1]	;; normalization for R1/I1
	vmulpd	ymm0, ymm0, ymm10	;; I1 * weight1				;  8-12
	vmulpd	ymm4, ymm4, ymm10	;; R1 * weight1				;  9-13

	vbroadcastsd ymm11, Q [screg1+16];; normalization for R3/I3
	vmulpd	ymm3, ymm3, ymm15	;; R8 = R8 * square root		;  10-14
	vmulpd	ymm1, ymm1, ymm15	;; R6 = R6 * square root		;  11-15

	vaddpd	ymm12, ymm5, ymm7	;; R2 + R4 (final I1 / 2nd real result)	; 12-14
	vmulpd	ymm2, ymm2, ymm11	;; R3 * weight3				;  12-16
	L1prefetchw srcreg+L1pd, L1pt

	vsubpd	ymm5, ymm5, ymm7	;; R2 - R4 (final I2)			; 13-15
	vmulpd	ymm6, ymm6, ymm11	;; I3 * weight3				;  13-17
	vmovapd	ymm13, [screg2+0+32]	;; cosine/sine for w^2n

	L1prefetchw srcreg+d1+L1pd, L1pt

	ystore	[srcreg+32], ymm12	;; Save I1				;  15

	vsubpd	ymm7, ymm1, ymm3	;; R6 = R6 - R8 (Real part)		; 16-18
	vmulpd	ymm14, ymm5, ymm13	;; B2 = I2 * cosine/sine for w^2n	;  16-20
	vmovapd	ymm9, [screg2+192+32]	;; cosine/sine for w^5n

	vaddpd	ymm1, ymm1, ymm3	;; R8 = R6 + R8 (Imaginary part)	; 17-19
	L1prefetchw srcreg+d2+L1pd, L1pt

	vsubpd	ymm3, ymm4, ymm2	;; R1 - R3 (final R2)			; 18-20
	vmovapd	ymm10, [screg2+128+32]	;; cosine/sine for w^n

	vaddpd	ymm4, ymm4, ymm2	;; R1 + R3 (final R1)			; 19-21
	vmovapd	ymm11, [screg2+0+0]	;; sine for w^2n

	vsubpd	ymm2, ymm0, ymm7	;; R5 - R6 (final R4)			; 20-22
	L1prefetchw srcreg+d2+d1+L1pd, L1pt

	vaddpd	ymm0, ymm0, ymm7	;; R5 + R6 (final R3)			; 21-23
	vmulpd	ymm12, ymm3, ymm13	;; A2 = R2 * cosine/sine for w^2n	;  21-25
	vmovapd	ymm13, [screg2+192+0]	;; sine for w^5n

	ystore	[srcreg], ymm4		;; Save R1				; 22
	vsubpd	ymm7, ymm6, ymm1	;; R7 - R8 (final I4)			; 22-24

	vaddpd	ymm6, ymm6, ymm1	;; R7 + R8 (final I3)			; 23-25
	vmulpd	ymm1, ymm2, ymm9	;; A4 = R4 * cosine/sine for w^5n	;  23-27

	vaddpd	ymm14, ymm14, ymm3	;; B2 = B2 + R2				; 24-26
	vmulpd	ymm3, ymm0, ymm10	;; A3 = R3 * cosine/sine for w^n	;  24-28

	vsubpd	ymm12, ymm12, ymm5	;; A2 = A2 - I2				; 25-27
	vmulpd	ymm5, ymm7, ymm9	;; B4 = I4 * cosine/sine for w^5n	;  25-29
	vmovapd	ymm9, [screg2+128+0]	;; sine for w^n

	vmulpd	ymm4, ymm6, ymm10	;; B3 = I3 * cosine/sine for w^n	;  26-30

	vmulpd	ymm14, ymm14, ymm11	;; B2 = B2 * sine (final I2)		;  27-31

	vsubpd	ymm1, ymm1, ymm7	;; A4 = A4 - I4				; 28-30
	vmulpd	ymm12, ymm12, ymm11	;; A2 = A2 * sine (final R2)		;  28-32

	vsubpd	ymm3, ymm3, ymm6	;; A3 = A3 - I3				; 29-31

	vaddpd	ymm5, ymm5, ymm2	;; B4 = B4 + R4				; 30-32

	vaddpd	ymm4, ymm4, ymm0	;; B3 = B3 + R3				; 31-33
	vmulpd	ymm1, ymm1, ymm13	;; A4 = A4 * sine (final R4)		;  31-35

	ystore	[srcreg+d1+32], ymm14	;; Save I2				; 32
	vmulpd	ymm3, ymm3, ymm9	;; A3 = A3 * sine (final R3)		;  32-36

	ystore	[srcreg+d1], ymm12	;; Save R2				; 33
	vmulpd	ymm5, ymm5, ymm13	;; B4 = B4 * sine (final I4)		;  33-37

	vmulpd	ymm4, ymm4, ymm9	;; B3 = B3 * sine (final I3)		;  34-38

	ystore	[srcreg+d2+d1], ymm1	;; Save R4				; 36
	ystore	[srcreg+d2], ymm3	;; Save R3				; 37
	ystore	[srcreg+d2+d1+32], ymm5	;; Save I4				; 38
	ystore	[srcreg+d2+32], ymm4	;; Save I3				; 39
	bump	srcreg, srcinc
	bump	screg1, scinc1
	bump	screg2, scinc2
	ENDM

;; Haswell FMA3 version -- could be faster with unrolling

IF (@INSTR(,%yarch,<FMA3>) NE 0)

;; Uses one combined sin/cos data ptr.  The first two sin/cos entries can be used by four complex
;; macros at the same FFT level.  The third and fourth sin/cos entries are used by this eight reals macro.
yr4_b4cl_csc_wpn4_eight_reals_fft_preload MACRO
	vmovapd	ymm14, YMM_ONE
	vmovapd	ymm15, YMM_SQRTHALF
	ENDM

;; Timed at ?? clocks
yr4_b4cl_csc_wpn4_eight_reals_fft MACRO srcreg,srcinc,d1,d2,screg1,scinc1,screg2,scinc2,maxrpt,L1pt,L1pd

	vbroadcastsd ymm9, Q [screg1+8]		;; normalization for R2/I2
	vmovapd	ymm5, [srcreg+d1]		;; R2
	vmulpd	ymm5, ymm5, ymm9		;; R2 * weight2				; 1-5

	vbroadcastsd ymm10, Q [screg1+0]	;; normalization for R1/I1
	vmovapd	ymm3, [srcreg]			;; R1
	vmulpd	ymm3, ymm3, ymm10		;; R1 * weight1				; 1-5

	vbroadcastsd ymm11, Q [screg1+16]	;; normalization for R3/I3
	vmovapd	ymm6, [srcreg+d2]		;; R3
	vmulpd	ymm6, ymm6, ymm11		;; R3 * weight3				; 2-6

	vmovapd	ymm1, [srcreg+d2+d1]		;; R4
	vmovapd	ymm2, [srcreg+d2+d1+32]		;; R8
	yfmsubpd ymm0, ymm1, ymm14, ymm2	;; new R8 = R4 - R8			; 2-6
	yfmaddpd ymm1, ymm1, ymm14, ymm2	;; new R4 = R4 + R8			; 3-7

	vmovapd	ymm12, [srcreg+d1+32]		;; R6
	yfnmaddpd ymm4, ymm12, ymm9, ymm5	;; new R6 = R2 - R6 * weight2			; 6-10
	yfmaddpd ymm5, ymm12, ymm9, ymm5	;; new R2 = R2 + R6 * weight2			; 6-10

	vmovapd	ymm12, [srcreg+32]		;; R5
	yfmaddpd ymm2, ymm12, ymm10, ymm3	;; new R1 = R1 + R5 * weight1			; 7-11
	yfnmaddpd ymm3, ymm12, ymm10, ymm3	;; new R5 = R1 - R5 * weight1			; 7-11

	vmovapd	ymm8, [srcreg+d2+32]		;; R7
	yfmaddpd ymm7, ymm8, ymm11, ymm6	;; new R3 = R3 + R7 * weight3			; 8-12
	yfnmaddpd ymm6, ymm8, ymm11, ymm6	;; new R7 = R3 - R7 * weight3			; 8-12

	vbroadcastsd ymm10, Q [screg1+24]	;; normalization for R4/I4
	yfnmaddpd ymm8, ymm0, ymm10, ymm4	;; R6 = R6 - R8 * weight4 (Real part)		; 11-15
	yfmaddpd ymm4, ymm0, ymm10, ymm4	;; R8 = R6 + R8 * weight4 (Imaginary part)	; 11-15
	vmovapd	ymm0, [screg2+0+32]		;; cosine/sine for w^2n

	yfmaddpd ymm9, ymm1, ymm10, ymm5	;; R2 + R4 * weight4 (final I1, a.k.a 2nd real result) ; 12-16
	yfnmaddpd ymm5, ymm1, ymm10, ymm5	;; R2 - R4 * weight4 (final I2)			; 12-16
	ystore	[srcreg+32], ymm9		;; Save I1				; 17
	L1prefetchw srcreg+L1pd, L1pt

	yfmaddpd ymm10, ymm2, ymm14, ymm7	;; R1 + R3 (final R1)			; 13-17
	yfmsubpd ymm2, ymm2, ymm14, ymm7	;; R1 - R3 (final R2)			; 13-17
	ystore	[srcreg], ymm10			;; Save R1				; 18
	L1prefetchw srcreg+d2+d1+L1pd, L1pt

	vmovapd	ymm11, [screg2+192+32]		;; cosine/sine for w^5n
	yfnmaddpd ymm7, ymm8, ymm15, ymm3	;; R5 - R6 * square root (final R4)	; 16-20
	yfnmaddpd ymm12, ymm4, ymm15, ymm6	;; R7 - R8 * square root (final I4)	; 16-20

	vmovapd	ymm13, [screg2+128+32]		;; cosine/sine for w^n
	yfmaddpd ymm8, ymm8, ymm15, ymm3	;; R5 + R6 * square root (final R3)	; 17-21
	yfmaddpd ymm4, ymm4, ymm15, ymm6	;; R7 + R8 * square root (final I3)	; 17-21
	vmovapd	ymm6, [screg2+0+0]		;; sine for w^2n

	L1prefetchw srcreg+d1+L1pd, L1pt

	yfmsubpd ymm3, ymm2, ymm0, ymm5		;; A2 = R2 * cosine/sine for w^2n - I2	; 18-22
	yfmaddpd ymm5, ymm5, ymm0, ymm2		;; B2 = I2 * cosine/sine for w^2n + R2	; 18-22
	vmovapd	ymm0, [screg2+192+0]		;; sine for w^5n
	vmovapd	ymm2, [screg2+128+0]		;; sine for w^n

	yfmsubpd ymm1, ymm7, ymm11, ymm12	;; A4 = R4 * cosine/sine for w^5n - I4	; 21-25
	yfmaddpd ymm12, ymm12, ymm11, ymm7	;; B4 = I4 * cosine/sine for w^5n + R4	; 21-25

	yfmsubpd ymm11, ymm8, ymm13, ymm4	;; A3 = R3 * cosine/sine for w^n - I3	; 22-26
	yfmaddpd ymm4, ymm4, ymm13, ymm8	;; B3 = I3 * cosine/sine for w^n + R3	; 22-26

	L1prefetchw srcreg+d2+L1pd, L1pt

	vmulpd	ymm3, ymm3, ymm6		;; A2 = A2 * sine (final R2)		; 23-27
	vmulpd	ymm5, ymm5, ymm6		;; B2 = B2 * sine (final I2)		; 23-27

	vmulpd	ymm1, ymm1, ymm0		;; A4 = A4 * sine (final R4)		; 26-30
	vmulpd	ymm12, ymm12, ymm0		;; B4 = B4 * sine (final I4)		; 26-30

	vmulpd	ymm11, ymm11, ymm2		;; A3 = A3 * sine (final R3)		; 27-31
	vmulpd	ymm4, ymm4, ymm2		;; B3 = B3 * sine (final I3)		; 27-31

	ystore	[srcreg+d1], ymm3		;; Save R2				; 24
	ystore	[srcreg+d1+32], ymm5		;; Save I2				; 24+1
	ystore	[srcreg+d2+d1], ymm1		;; Save R4				; 26
	ystore	[srcreg+d2+d1+32], ymm12	;; Save I4				; 26+1
	ystore	[srcreg+d2], ymm11		;; Save R3				; 27+1
	ystore	[srcreg+d2+32], ymm4		;; Save I3				; 27+2

	bump	srcreg, srcinc
	bump	screg1, scinc1
	bump	screg2, scinc2
	ENDM

ENDIF

ENDIF




;; Uses one combined sin/cos data ptr.  The first two sin/cos entries can be used by four complex
;; macros at the same FFT level.  The third and fourth sin/cos entries are used by this eight reals macro.

;; screg1 is normalization weights for R1/I1
;; screg2+0, screg2+0+32 is weighted sin/cos values for R2/I2 (w^2n)
;; screg2+128+0, screg2+128+32 is weighted sin/cos values for R3/I3 (w^n)
;; screg2+192+0, screg2+192+32 is weighted sin/cos values for R4/I4 (w^5n)

yr4_b4cl_csc_wpn4_eight_reals_unfft_preload MACRO
	ENDM
yr4_b4cl_csc_wpn4_eight_reals_unfft MACRO srcreg,srcinc,d1,d2,screg1,scinc1,screg2,scinc2,maxrpt,L1pt,L1pd

	vmovapd	ymm1, [screg2+0+32]	;; cosine/sine for w^2n
	vmovapd	ymm4, [srcreg+d1]	;; R2
	vmulpd	ymm2, ymm4, ymm1	;; A2 = R2 * cosine/sine
	vmovapd	ymm7, [srcreg+d1+32]	;; I2
	vaddpd	ymm2, ymm2, ymm7	;; A2 = A2 + I2
	vmulpd	ymm7, ymm7, ymm1	;; B2 = I2 * cosine/sine
	vsubpd	ymm7, ymm7, ymm4	;; B2 = B2 - R2

	vmovapd	ymm1, [screg2+192+32]	;; cosine/sine for w^5n
	vmovapd	ymm0, [srcreg+d2+d1]	;; R4
	vmulpd	ymm4, ymm0, ymm1	;; A4 = R4 * cosine/sine
	vmovapd	ymm5, [srcreg+d2+d1+32]	;; I4
	vaddpd	ymm4, ymm4, ymm5	;; A4 = A4 + I4
	vmulpd	ymm5, ymm5, ymm1	;; B4 = I4 * cosine/sine
	vsubpd	ymm5, ymm5, ymm0	;; B4 = B4 - R4

	vmovapd	ymm1, [screg2+128+32] 	;; cosine/sine for w^n
	vmovapd	ymm3, [srcreg+d2]	;; R3
	vmulpd	ymm0, ymm1, ymm3	;; A3 = R3 * cosine/sine for w^n
	vmovapd	ymm6, [srcreg+d2+32]	;; I3
	vaddpd	ymm0, ymm0, ymm6	;; A3 = A3 + I3
	vmulpd	ymm6, ymm6, ymm1	;; B3 = I3 * cosine/sine for w^n
	vsubpd	ymm6, ymm6, ymm3	;; B3 = B3 - R3

	L1prefetchw srcreg+L1pd, L1pt

	vmovapd	ymm1, [screg2+0+0]	;; sine for w^2n
	vmulpd	ymm2, ymm2, ymm1	;; new R3 = A2 * sine
	vmulpd	ymm7, ymm7, ymm1	;; new R4 = B2 * sine
	vmovapd	ymm1, [screg2+192+0]	;; sine for w^5n
	vmulpd	ymm4, ymm4, ymm1	;; new R7 = A4 * sine
	vmulpd	ymm5, ymm5, ymm1	;; new R8 = B4 * sine
	vmovapd	ymm1, [screg2+128+0]	;; sine for w^n
	vmulpd	ymm0, ymm0, ymm1	;; new R5 = A3 * sine
	vmulpd	ymm6, ymm6, ymm1	;; new R6 = B3 * sine

	L1prefetchw srcreg+d1+L1pd, L1pt

	vsubpd	ymm1, ymm0, ymm4	;; new R6 = R5 - R7
	vaddpd	ymm0, ymm0, ymm4	;; new R5 = R5 + R7

	vsubpd	ymm4, ymm6, ymm5	;; new R8 = R6 - R8
	vaddpd	ymm6, ymm6, ymm5	;; new R7 = R6 + R8

	vaddpd	ymm5, ymm1, ymm4	;; R6 = R6 + R8
	vsubpd	ymm4, ymm4, ymm1	;; R8 = R8 - R6

	L1prefetchw srcreg+d2+L1pd, L1pt

	vmulpd	ymm5, ymm5, YMM_SQRTHALF ;; R6 = R6 * square root of 1/2
	vmulpd	ymm4, ymm4, YMM_SQRTHALF ;; R8 = R8 * square root of 1/2

	vmovapd	ymm3, [srcreg+32]	;; R2

	vsubpd	ymm1, ymm3, ymm7	;; R2 - R4 (new R4)
	vaddpd	ymm3, ymm3, ymm7	;; R2 + R4 (new R2)

	L1prefetchw srcreg+d2+d1+L1pd, L1pt

	vbroadcastsd ymm7, Q [screg1+24] ;; normalization_inverse for R4/I4
	vmulpd	ymm1, ymm1, ymm7	;; R4 * weight4
	vmulpd	ymm4, ymm4, ymm7	;; I4 * weight4

	vbroadcastsd ymm7, Q [screg1+8]	;; normalization_inverse for R2/I2
	vmulpd	ymm3, ymm3, ymm7	;; R2 * weight2
	vmulpd	ymm5, ymm5, ymm7	;; I2 * weight2

	vsubpd	ymm7, ymm1, ymm4	;; R4 - R8 (final R8)
	vaddpd	ymm1, ymm1, ymm4	;; R4 + R8 (final R4)

	vsubpd	ymm4, ymm3, ymm5	;; R2 - R6 (final R6)
	vaddpd	ymm3, ymm3, ymm5	;; R2 + R6 (final R2)

	vmovapd	ymm5, [srcreg]		;; R1

	ystore	[srcreg+d2+d1+32], ymm7	;; Save R8
	ystore	[srcreg+d2+d1], ymm1	;; Save R4

	vsubpd	ymm7, ymm5, ymm2	;; R1 - R3 (new R3)
	vaddpd	ymm5, ymm5, ymm2	;; R1 + R3 (new R1)

	vbroadcastsd ymm2, Q [screg1+16] ;; normalization_inverse for R3/I3
	vmulpd	ymm7, ymm7, ymm2	;; R3 * weight3
	vmulpd	ymm6, ymm6, ymm2	;; I3 * weight3

	vbroadcastsd ymm2, Q [screg1]	;; normalization_inverse for R1/I1
	vmulpd	ymm5, ymm5, ymm2	;; R1 * weight1
	vmulpd	ymm0, ymm0, ymm2	;; I1 * weight1

	vsubpd	ymm2, ymm7, ymm6	;; R3 - R7 (final R7)
	vaddpd	ymm7, ymm7, ymm6	;; R3 + R7 (final R3)

	vsubpd	ymm6, ymm5, ymm0	;; R1 - R5 (final R5)
	vaddpd	ymm5, ymm5, ymm0	;; R1 + R5 (final R1)

	ystore	[srcreg+d1+32], ymm4	;; Save R6
	ystore	[srcreg+d1], ymm3	;; Save R2

	ystore	[srcreg+d2+32], ymm2	;; Save R7
	ystore	[srcreg+d2], ymm7	;; Save R3

	ystore	[srcreg+32], ymm6	;; Save R5
	ystore	[srcreg], ymm5		;; Save R1

	bump	srcreg, srcinc
	bump	screg1, scinc1
	bump	screg2, scinc2
	ENDM

;; 64-bit version

IFDEF X86_64

;; Uses one combined sin/cos data ptr.  The first two sin/cos entries can be used by four complex
;; macros at the same FFT level.  The third and fourth sin/cos entries are used by this eight reals macro.
yr4_b4cl_csc_wpn4_eight_reals_unfft_preload MACRO
	vmovapd	ymm15, YMM_SQRTHALF
	ENDM
yr4_b4cl_csc_wpn4_eight_reals_unfft MACRO srcreg,srcinc,d1,d2,screg1,scinc1,screg2,scinc2,maxrpt,L1pt,L1pd

	vmovapd	ymm10, [screg2+128+32] 	;; cosine/sine for w^n
	vmovapd	ymm1, [srcreg+d2]	;; R3
	vmulpd	ymm0, ymm1, ymm10	;; A3 = R3 * cosine/sine for w^n	; 1-5

	vmovapd	ymm11, [screg2+192+32]	;; cosine/sine for w^5n
	vmovapd	ymm3, [srcreg+d2+d1]	;; R4
	vmulpd	ymm2, ymm3, ymm11	;; A4 = R4 * cosine/sine		; 2-6

	vmovapd	ymm5, [srcreg+d2+32]	;; I3
	vmulpd	ymm4, ymm5, ymm10	;; B3 = I3 * cosine/sine for w^n	; 3-7

	vmovapd	ymm7, [srcreg+d2+d1+32]	;; I4
	vmulpd	ymm6, ymm7, ymm11	;; B4 = I4 * cosine/sine		; 4-8

	vmovapd	ymm12, [screg2+0+32]	;; cosine/sine for w^2n
	vmovapd	ymm9, [srcreg+d1]	;; R2
	vmulpd	ymm8, ymm9, ymm12	;; A2 = R2 * cosine/sine		; 5-9

	vaddpd	ymm0, ymm0, ymm5	;; A3 = A3 + I3				; 6-8
	vmovapd	ymm11, [srcreg+d1+32]	;; I2
	vmulpd	ymm10, ymm11, ymm12	;; B2 = I2 * cosine/sine		; 6-10

	vaddpd	ymm2, ymm2, ymm7	;; A4 = A4 + I4				; 7-9

	vsubpd	ymm4, ymm4, ymm1	;; B3 = B3 - R3				; 8-10

	vsubpd	ymm6, ymm6, ymm3	;; B4 = B4 - R4				; 9-11
	vmovapd	ymm1, [screg2+128+0]	;; sine for w^n
	vmulpd	ymm0, ymm0, ymm1	;; new R5 = A3 * sine			; 9-13

	vaddpd	ymm8, ymm8, ymm11	;; A2 = A2 + I2				; 10-12
	vmovapd	ymm3, [screg2+192+0]	;; sine for w^5n
	vmulpd	ymm2, ymm2, ymm3	;; new R7 = A4 * sine			; 10-14

	vsubpd	ymm10, ymm10, ymm9	;; B2 = B2 - R2				; 11-13
	vmulpd	ymm4, ymm4, ymm1	;; new R6 = B3 * sine			; 11-15
	vmovapd	ymm9, [screg2+0+0]	;; sine for w^2n

	vmulpd	ymm6, ymm6, ymm3	;; new R8 = B4 * sine			; 12-16
	vmovapd	ymm5, [srcreg]		;; R1

	vmulpd	ymm8, ymm8, ymm9	;; new R3 = A2 * sine			; 13-17
	L1prefetchw srcreg+L1pd, L1pt

	vmulpd	ymm10, ymm10, ymm9	;; new R4 = B2 * sine			; 14-18
	vmovapd	ymm7, [srcreg+32]	;; R2

	vsubpd	ymm1, ymm0, ymm2	;; new R6 = R5 - R7			; 15-17
	vbroadcastsd ymm13, Q [screg1]	;; normalization_inverse for R1/I1

	vaddpd	ymm0, ymm0, ymm2	;; new R5 = R5 + R7			; 16-18
	vbroadcastsd ymm14, Q [screg1+16] ;; normalization_inverse for R3/I3

	vsubpd	ymm3, ymm4, ymm6	;; new R8 = R6 - R8			; 17-19
	vbroadcastsd ymm12, Q [screg1+8] ;; normalization_inverse for R2/I2

	vaddpd	ymm4, ymm4, ymm6	;; new R7 = R6 + R8			; 18-20
	vbroadcastsd ymm11, Q [screg1+24] ;; normalization_inverse for R4/I4

	vaddpd	ymm9, ymm5, ymm8	;; R1 + R3 (new R1)			; 19-21
	vmulpd	ymm0, ymm0, ymm13	;; I1 * weight1				; 19-23
	L1prefetchw srcreg+d1+L1pd, L1pt

	vaddpd	ymm2, ymm1, ymm3	;; R6 = R6 + R8				; 20-22

	vsubpd	ymm1, ymm3, ymm1	;; R8 = R8 - R6				; 21-23
	vmulpd	ymm4, ymm4, ymm14	;; I3 * weight3				; 21-25
	L1prefetchw srcreg+d2+L1pd, L1pt

	vsubpd	ymm5, ymm5, ymm8	;; R1 - R3 (new R3)			; 22-24
	vmulpd	ymm9, ymm9, ymm13	;; R1 * weight1				; 22-26

;; IDEA: precompute weight2/4 times sqrthalf to reduce latency in calculating R6/R8

	vaddpd	ymm6, ymm7, ymm10	;; R2 + R4 (new R2)			; 23-25
	vmulpd	ymm2, ymm2, ymm15	;; R6 = R6 * square root of 1/2		; 23-27
	L1prefetchw srcreg+d2+d1+L1pd, L1pt

	vsubpd	ymm7, ymm7, ymm10	;; R2 - R4 (new R4)			; 24-26
	vmulpd	ymm1, ymm1, ymm15	;; R8 = R8 * square root of 1/2		; 24-28

	vmulpd	ymm5, ymm5, ymm14	;; R3 * weight3				; 25-29

	vmulpd	ymm6, ymm6, ymm12	;; R2 * weight2				; 26-30

	vsubpd	ymm3, ymm9, ymm0	;; R1 - R5 (final R5)			; 27-29
	vmulpd	ymm7, ymm7, ymm11	;; R4 * weight4				; 27-31

	vaddpd	ymm9, ymm9, ymm0	;; R1 + R5 (final R1)			; 28-30
	vmulpd	ymm2, ymm2, ymm12	;; I2 * weight2				; 28-32

	vmulpd	ymm1, ymm1, ymm11	;; I4 * weight4				; 29-33

	ystore	[srcreg+32], ymm3	;; Save R5				; 30
	vsubpd	ymm8, ymm5, ymm4	;; R3 - R7 (final R7)			; 30-32

	ystore	[srcreg], ymm9		;; Save R1				; 31
	vaddpd	ymm5, ymm5, ymm4	;; R3 + R7 (final R3)			; 31-33

	ystore	[srcreg+d2+32], ymm8	;; Save R7				; 33
	vsubpd	ymm0, ymm6, ymm2	;; R2 - R6 (final R6)			; 33-35

	ystore	[srcreg+d2], ymm5	;; Save R3				; 34
	vaddpd	ymm6, ymm6, ymm2	;; R2 + R6 (final R2)			; 34-36

	vsubpd	ymm4, ymm7, ymm1	;; R4 - R8 (final R8)			; 35-37

	ystore	[srcreg+d1+32], ymm0	;; Save R6				; 36
	vaddpd	ymm7, ymm7, ymm1	;; R4 + R8 (final R4)			; 36-38

	ystore	[srcreg+d1], ymm6	;; Save R2				; 37
	ystore	[srcreg+d2+d1+32], ymm4	;; Save R8				; 38
	ystore	[srcreg+d2+d1], ymm7	;; Save R4				; 39

	bump	srcreg, srcinc
	bump	screg1, scinc1
	bump	screg2, scinc2
	ENDM

;; Haswell FMA3 version -- could be faster with unrolling

IF (@INSTR(,%yarch,<FMA3>) NE 0)

;; Uses one combined sin/cos data ptr.  The first two sin/cos entries can be used by four complex
;; macros at the same FFT level.  The third and fourth sin/cos entries are used by this eight reals macro.
yr4_b4cl_csc_wpn4_eight_reals_unfft_preload MACRO
	vmovapd	ymm14, YMM_ONE
	vmovapd	ymm15, YMM_SQRTHALF
	ENDM

;; Timed at ? clocks
yr4_b4cl_csc_wpn4_eight_reals_unfft MACRO srcreg,srcinc,d1,d2,screg1,scinc1,screg2,scinc2,maxrpt,L1pt,L1pd
	vmovapd	ymm3, [screg2+128+32]		;; cosine/sine for w^n
	vmovapd	ymm2, [srcreg+d2]		;; R3
	vmovapd	ymm1, [srcreg+d2+32]		;; I3
	yfmaddpd ymm0, ymm2, ymm3, ymm1		;; A3 = R3 * cosine/sine for w^n + I3	; 1-5
	yfmsubpd ymm1, ymm1, ymm3, ymm2		;; B3 = I3 * cosine/sine for w^n - R3	; 1-5

	vmovapd	ymm5, [screg2+0+32]		;; cosine/sine for w^2n
	vmovapd	ymm4, [srcreg+d1]		;; R2
	vmovapd	ymm3, [srcreg+d1+32]		;; I2
	yfmaddpd ymm2, ymm4, ymm5, ymm3		;; A2 = R2 * cosine/sine + I2		; 2-6
	yfmsubpd ymm3, ymm3, ymm5, ymm4		;; B2 = I2 * cosine/sine - R2		; 2-6

	vmovapd	ymm9, [screg2+192+32]		;; cosine/sine for w^5n
	vmovapd	ymm8, [srcreg+d2+d1]		;; R4
	vmovapd	ymm7, [srcreg+d2+d1+32]		;; I4
	yfmaddpd ymm6, ymm8, ymm9, ymm7		;; A4 = R4 * cosine/sine + I4		; 3-7
	yfmsubpd ymm7, ymm7, ymm9, ymm8		;; B4 = I4 * cosine/sine - R4		; 3-7

	vmovapd	ymm9, [screg2+128+0]		;; sine for w^n
	vmulpd	ymm0, ymm0, ymm9		;; new R5 = A3 * sine			; 6-10
	vmulpd	ymm1, ymm1, ymm9		;; new R6 = B3 * sine			; 6-10

	vmovapd	ymm4, [srcreg]			;; R1
	vmovapd	ymm9, [screg2+0+0]		;; sine for w^2n
	yfmaddpd ymm10, ymm2, ymm9, ymm4	;; R1 + (R3 = A2 * sine) (new R1)	; 7-11
	yfnmaddpd ymm2, ymm2, ymm9, ymm4	;; R1 - (R3 = A2 * sine) (new R3)	; 7-11

	vmovapd	ymm5, [srcreg+32]		;; R2
	yfmaddpd ymm4, ymm3, ymm9, ymm5		;; R2 + (R4 = B2 * sine) (new R2)	; 8-12
	yfnmaddpd ymm3, ymm3, ymm9, ymm5	;; R2 - (R4 = B2 * sine) (new R4)	; 8-12

	vmovapd	ymm9, [screg2+192+0]		;; sine for w^5n
	yfnmaddpd ymm5, ymm6, ymm9, ymm0	;; new R6 = R5 - (R7 = A4 * sine)	; 11-15
	yfnmaddpd ymm11, ymm7, ymm9, ymm1	;; new R8 = R6 - (R8 = B4 * sine)	; 11-15
	L1prefetchw srcreg+L1pd, L1pt

	yfmaddpd ymm6, ymm6, ymm9, ymm0		;; new R5 = R5 + (R7 = A4 * sine)	; 12-16
	yfmaddpd ymm7, ymm7, ymm9, ymm1		;; new R7 = R6 + (R8 = B4 * sine)	; 12-16
	L1prefetchw srcreg+d1+L1pd, L1pt

	yfmaddpd ymm0, ymm5, ymm14, ymm11	;; R6 = R6 + R8				; 16-20
	yfmsubpd ymm11, ymm11, ymm14, ymm5	;; R8 = R8 - R6				; 16-20
	L1prefetchw srcreg+d2+L1pd, L1pt

	vbroadcastsd ymm5, Q [screg1+0]		;; normalization_inverse for R1/I1
	vmulpd	ymm10, ymm10, ymm5		;; R1 * weight1				; 13-17
	yfnmaddpd ymm9, ymm6, ymm5, ymm10	;; R1 - R5 * weight1 (final R5)		; 18-22
	yfmaddpd ymm10, ymm6, ymm5, ymm10	;; R1 + R5 * weight1 (final R1)		; 18-22

	vbroadcastsd ymm5, Q [screg1+16]	;; normalization_inverse for R3/I3
	vmulpd	ymm2, ymm2, ymm5		;; R3 * weight3				; 13-17
	yfnmaddpd ymm1, ymm7, ymm5, ymm2	;; R3 - R7 * weight3 (final R7)		; 19-23
	yfmaddpd ymm2, ymm7, ymm5, ymm2		;; R3 + R7 * weight3 (final R3)		; 19-23
	L1prefetchw srcreg+d2+d1+L1pd, L1pt

	yfnmaddpd ymm6, ymm0, ymm15, ymm4	;; R2 - R6 * square root of 1/2 (final R6) ; 21-25
	yfmaddpd ymm0, ymm0, ymm15, ymm4	;; R2 + R6 * square root of 1/2 (final R2) ; 21-25

	yfnmaddpd ymm7, ymm11, ymm15, ymm3	;; R4 - R8 * square root of 1/2 (final R8) ; 22-26
	yfmaddpd ymm3, ymm11, ymm15, ymm3	;; R4 + R8 * square root of 1/2 (final R4) ; 22-26

	vbroadcastsd ymm5, Q [screg1+8]		;; normalization_inverse for R2/I2
	vmulpd	ymm0, ymm0, ymm5		;; R2 * weight2				; 26-30
	vmulpd	ymm6, ymm6, ymm5		;; R6 * weight2				; 26-30
	vbroadcastsd ymm5, Q [screg1+24]	;; normalization_inverse for R4/I4
	vmulpd	ymm3, ymm3, ymm5		;; R4 * weight4				; 27-31
	vmulpd	ymm7, ymm7, ymm5		;; R8 * weight4				; 27-31

	ystore	[srcreg+32], ymm9	;; Save R5				; 23
	ystore	[srcreg], ymm10		;; Save R1				; 23+1
	ystore	[srcreg+d2+32], ymm1	;; Save R7				; 24+1
	ystore	[srcreg+d2], ymm2	;; Save R3				; 24+2
	ystore	[srcreg+d1], ymm0	;; Save R2				; 31
	ystore	[srcreg+d1+32], ymm6	;; Save R6				; 31+1
	ystore	[srcreg+d2+d1], ymm3	;; Save R4				; 32+1
	ystore	[srcreg+d2+d1+32], ymm7	;; Save R8				; 32+2

	bump	srcreg, srcinc
	bump	screg1, scinc1
	bump	screg2, scinc2
	ENDM

ENDIF

ENDIF








;;
;; ********************************* reduced sin/cos four-complex-fft4 variants **************************************
;;
;; These macros are used in the last levels of pass 1.  These macros differ from the standard sg4cl macros
;; in that the sin/cos values are computed on the fly to reduce memory requirements.  To do this we cannot
;; store data as (cos/sin,sin).  Thus, these macros work on (cos,sin) values.

yr4_rsc_sg4cl_four_complex_calc_sincos MACRO src1, src2, clm
	yr4_rsc_sg4cl_four_complex_calc_sincos1 src1+0*YMM_SCD1, src2+0*YMM_SCD2, YMM_TMPS[0*YMM_SCD4]
	IF clm GE 2
	yr4_rsc_sg4cl_four_complex_calc_sincos1 src1+1*YMM_SCD1, src2+1*YMM_SCD2, YMM_TMPS[1*YMM_SCD4]
	ENDIF
	IF clm GE 4
	yr4_rsc_sg4cl_four_complex_calc_sincos1 src1+2*YMM_SCD1, src2+2*YMM_SCD2, YMM_TMPS[2*YMM_SCD4]
	yr4_rsc_sg4cl_four_complex_calc_sincos1 src1+3*YMM_SCD1, src2+3*YMM_SCD2, YMM_TMPS[3*YMM_SCD4]
	ENDIF
	ENDM

yr4_rsc_sg4cl_four_complex_calc_sincos1 MACRO src1, src2, dest
	vmovapd	ymm0, [src1+32]		;; cos
	vmovapd	ymm1, [src1]		;; sin
	ystore	dest[0*64+32], ymm0
	ystore	dest[0*64], ymm1

	;;a+bi	*	c+di	=	(ac-bd) + (ad+bc)i
	;;a+bi	*	c-di	=	(ac+bd) + (-ad+bc)i

	vmovapd	ymm2, [src2+32]		;; cos
	vmulpd	ymm4, ymm0, ymm2	;; cos * cos				;  1-5
	vmovapd	ymm3, [src2]		;; sin
	vmulpd	ymm5, ymm1, ymm3	;; sin * sin				;  2-6
	vmulpd	ymm3, ymm0, ymm3	;; ad = cos * sin			;  3-7
	vmulpd	ymm2, ymm1, ymm2	;; bc = sin * cos			;  4-8
	vsubpd	ymm6, ymm4, ymm5	;; new cos = cos * cos - sin * sin	; 7-9
	vaddpd	ymm4, ymm4, ymm5	;; new cos = cos * cos + sin * sin	; 8-10
	vaddpd	ymm5, ymm3, ymm2	;; new sin = ad + bc			; 9-11
	vsubpd	ymm7, ymm2, ymm3	;; new sin = bc - ad			; 10-12

	;;a+bi	*	c+di	=	(ac-bd) + (ad+bc)i

	vmovapd	ymm2, [src2+64+32]	;; cos
	ystore	dest[1*64+32], ymm6						; 10
	vmulpd	ymm6, ymm0, ymm2	;; cos * cos				;  5-9
	vmovapd	ymm3, [src2+64]		;; sin
	ystore	dest[3*64+32], ymm4						; 11
	vmulpd	ymm4, ymm1, ymm3	;; sin * sin				;  6-10
	vmulpd	ymm3, ymm0, ymm3	;; cos * sin				;  7-11
	vmulpd	ymm2, ymm1, ymm2	;; sin * cos				;  8-12
	vsubpd	ymm6, ymm6, ymm4	;; new cos = cos * cos - sin * sin	; 11-13
	ystore	dest[1*64], ymm5						; 12
	vaddpd	ymm2, ymm2, ymm3	;; new sin = cos * sin + sin * cos	; 13-15
	ystore	dest[3*64], ymm7						; 13
	ystore	dest[2*64+32], ymm6						; 14
	ystore	dest[2*64], ymm2						; 16
	ENDM

IFDEF X86_64

yr4_rsc_sg4cl_four_complex_calc_sincos1 MACRO src1, src2, dest
	vmovapd	ymm0, [src1+32]		;; cos
	vmovapd	ymm1, [src1]		;; sin
	ystore	dest[0*64+32], ymm0
	ystore	dest[0*64], ymm1
	vmovapd	ymm2, [src2+32]		;; cos
	vmulpd	ymm4, ymm0, ymm2	;; cos * cos				;  1-5
	vmovapd	ymm3, [src2]		;; sin
	vmulpd	ymm5, ymm1, ymm3	;; sin * sin				;  2-6
	vmulpd	ymm3, ymm0, ymm3	;; ad = cos * sin			;  3-7
	vmulpd	ymm2, ymm1, ymm2	;; bc = sin * cos			;  4-8
	vmovapd	ymm12, [src2+64+32]	;; cos
	vmulpd	ymm14, ymm0, ymm12	;; cos * cos				;  5-9
	vmovapd	ymm13, [src2+64]	;; sin
	vmulpd	ymm15, ymm1, ymm13	;; sin * sin				;  6-10
	vsubpd	ymm6, ymm4, ymm5	;; new cos = cos * cos - sin * sin	; 7-9
	vmulpd	ymm13, ymm0, ymm13	;; cos * sin				;  7-11
	vaddpd	ymm4, ymm4, ymm5	;; new cos = cos * cos + sin * sin	; 8-10
	vmulpd	ymm12, ymm1, ymm12	;; sin * cos				;  8-12
	vaddpd	ymm5, ymm3, ymm2	;; new sin = ad + bc			; 9-11
	vsubpd	ymm7, ymm2, ymm3	;; new sin = bc - ad			; 10-12
	ystore	dest[1*64+32], ymm6						; 10
	ystore	dest[3*64+32], ymm4						; 11
	vsubpd	ymm14, ymm14, ymm15	;; new cos = cos * cos - sin * sin	; 11-13
	ystore	dest[1*64], ymm5						; 12
	vaddpd	ymm12, ymm12, ymm13	;; new sin = cos * sin + sin * cos	; 13-15
	ystore	dest[3*64], ymm7						; 13
	ystore	dest[2*64+32], ymm14						; 14
	ystore	dest[2*64], ymm12						; 16
	ENDM

IF (@INSTR(,%yarch,<FMA3>) NE 0)
yr4_rsc_sg4cl_four_complex_calc_sincos1 MACRO src1, src2, dest
	vmovapd	ymm0, [src1+32]		;; cos
	vmovapd	ymm2, [src2+32]		;; cos
	vmulpd	ymm4, ymm0, ymm2	;; ac = cos * cos			; 1-5
	vmovapd	ymm3, [src2]		;; sin
	vmulpd	ymm5, ymm0, ymm3	;; ad = cos * sin			; 1-5
	vmovapd	ymm12, [src2+64+32]	;; cos
	vmulpd	ymm14, ymm0, ymm12	;; ac = cos * cos			; 2-6
	vmovapd	ymm13, [src2+64]	;; sin
	vmulpd	ymm15, ymm0, ymm13	;; ad = cos * sin			; 2-6
	vmovapd	ymm1, [src1]		;; sin
	ystore	dest[0*64+32], ymm0
	ystore	dest[0*64], ymm1
	yfnmaddpd ymm7, ymm1, ymm3, ymm4 ;; new cos = ac - sin * sin		; 6-10
	yfmaddpd ymm4, ymm1, ymm3, ymm4	;; new cos = ac + sin * sin		; 6-10
	yfmaddpd ymm6, ymm1, ymm2, ymm5	;; new sin = ad + sin * cos		; 7-11
	yfmsubpd ymm5, ymm1, ymm2, ymm5	;; new sin = sin * cos - ad		; 7-11
	yfnmaddpd ymm14, ymm1, ymm13, ymm14 ;; new cos = ac - sin * sin		; 8-12
	yfmaddpd ymm12, ymm1, ymm12, ymm15 ;; new sin = ad + sin * cos		; 8-12
	ystore	dest[1*64+32], ymm7						; 11
	ystore	dest[3*64+32], ymm4						; 11+1
	ystore	dest[1*64], ymm6						; 12+1
	ystore	dest[3*64], ymm5						; 12+2
	ystore	dest[2*64+32], ymm14						; 13+2
	ystore	dest[2*64], ymm12						; 13+3
	ENDM
ENDIF

ENDIF

;; Like the above, except we know that src1 would point to 1+0i
yr4_rsc_sg4cl_four_complex_calc_sincos_simple MACRO src2, clm
	vmovapd	ymm0, YMM_ONE		;; cos
	vxorpd	ymm1, ymm1, ymm1	;; sin
	yr4_rsc_sg4cl_four_complex_calc_sincos_simple1 src2+0*YMM_SCD2, YMM_TMPS[0*YMM_SCD4]
	IF clm GE 2
	yr4_rsc_sg4cl_four_complex_calc_sincos_simple1 src2+1*YMM_SCD2, YMM_TMPS[1*YMM_SCD4]
	ENDIF
	IF clm GE 4
	yr4_rsc_sg4cl_four_complex_calc_sincos_simple1 src2+2*YMM_SCD2, YMM_TMPS[2*YMM_SCD4]
	yr4_rsc_sg4cl_four_complex_calc_sincos_simple1 src2+3*YMM_SCD2, YMM_TMPS[3*YMM_SCD4]
	ENDIF
	ENDM

yr4_rsc_sg4cl_four_complex_calc_sincos_simple1 MACRO src2, dest
	ystore	dest[0*64+32], ymm0
	ystore	dest[0*64], ymm1
	vmovapd	ymm2, [src2+32]		;; cos
	vmovapd	ymm3, [src2]		;; sin
	ystore	dest[1*64+32], ymm2
	ystore	dest[1*64], ymm3
	ystore	dest[3*64+32], ymm2
	vsubpd	ymm3, ymm1, ymm3	;; new sin = - sin
	ystore	dest[3*64], ymm3
	vmovapd	ymm2, [src2+64+32]	;; cos
	vmovapd	ymm3, [src2+64]		;; sin
	ystore	dest[2*64+32], ymm2
	ystore	dest[2*64], ymm3
	ENDM


yr4_rsc_sg4cl_four_complex_fft4_preload MACRO
	ENDM
yr4_rsc_sg4cl_four_complex_fft4 MACRO srcreg,srcinc,d1,d2,dstreg,dstinc,e1,e2,screg,scinc,maxrpt,L1pt,L1pd

	vmovapd	ymm1, [srcreg]			;; R1
	vmovapd	ymm7, [srcreg+d1]		;; R2
	vshufpd	ymm0, ymm1, ymm7, 15		;; Shuffle R1 and R2 to create R1/R2 hi			; 1
	vshufpd	ymm1, ymm1, ymm7, 0		;; Shuffle R1 and R2 to create R1/R2 low		; 2

	vmovapd	ymm2, [srcreg+d2]		;; R3
	vmovapd	ymm7, [srcreg+d2+d1]		;; R4
	vshufpd	ymm3, ymm2, ymm7, 15		;; Shuffle R3 and R4 to create R3/R4 hi			; 3
	vshufpd	ymm2, ymm2, ymm7, 0		;; Shuffle R3 and R4 to create R3/R4 low		; 4

	ylow128s ymm4, ymm0, ymm3		;; Shuffle R1/R2 hi and R3/R4 hi (new R2)		; 5-6
	yhigh128s ymm0, ymm0, ymm3		;; Shuffle R1/R2 hi and R3/R4 hi (new R4)		; 6-7

	ylow128s ymm3, ymm1, ymm2		;; Shuffle R1/R2 low and R3/R4 low (new R1)		; 7-8
	yhigh128s ymm1, ymm1, ymm2		;; Shuffle R1/R2 low and R3/R4 low (new R3)		; 8-9

	vaddpd	ymm5, ymm4, ymm0		;; R2 + R4 (newer R2)
	vsubpd	ymm4, ymm4, ymm0		;; R2 - R4 (newer R4)

	L1prefetch srcreg+L1pd, L1pt
	L1prefetchw dstreg+L1pd, L1pt - L1PREFETCH_DEST_NONE

	vaddpd	ymm0, ymm3, ymm1		;; R1 + R3 (newer R1)
	vsubpd	ymm3, ymm3, ymm1		;; R1 - R3 (newer R3)

	vsubpd	ymm1, ymm0, ymm5		;; R1 - R2 (final R2)
	vaddpd	ymm0, ymm0, ymm5		;; R1 + R2 (final R1)

	vmovapd	ymm6, [srcreg+32]		;; I1
	vmovapd	ymm2, [srcreg+d1+32]		;; I2
	vshufpd	ymm5, ymm6, ymm2, 15		;; Shuffle I1 and I2 to create I1/I2 hi
	vshufpd	ymm6, ymm6, ymm2, 0		;; Shuffle I1 and I2 to create I1/I2 low

	ystore	[dstreg], ymm0			;; Temporarily save final R1

	vmovapd	ymm2, [srcreg+d2+32]		;; I3
	vmovapd	ymm7, [srcreg+d2+d1+32]		;; I4
	vshufpd	ymm0, ymm2, ymm7, 15		;; Shuffle I3 and I4 to create I3/I4 hi
	vshufpd	ymm2, ymm2, ymm7, 0		;; Shuffle I3 and I4 to create I3/I4 low

	ylow128s ymm7, ymm5, ymm0		;; Shuffle I1/I2 hi and I3/I4 hi (new I2)
	yhigh128s ymm5, ymm5, ymm0		;; Shuffle I1/I2 hi and I3/I4 hi (new I4)

	ylow128s ymm0, ymm6, ymm2		;; Shuffle I1/I2 low and I3/I4 low (new I1)
	yhigh128s ymm6, ymm6, ymm2		;; Shuffle I1/I2 low and I3/I4 low (new I3)

	vaddpd	ymm2, ymm7, ymm5		;; I2 + I4 (newer I2)
	vsubpd	ymm7, ymm7, ymm5		;; I2 - I4 (newer I4)

	L1prefetch srcreg+d1+L1pd, L1pt
	L1prefetchw dstreg+e1+L1pd, L1pt - L1PREFETCH_DEST_NONE

	vaddpd	ymm5, ymm0, ymm6		;; I1 + I3 (newer I1)
	vsubpd	ymm0, ymm0, ymm6		;; I1 - I3 (newer I3)

	vsubpd	ymm6, ymm3, ymm7		;; R3 - I4 (final R3)
	vaddpd	ymm3, ymm3, ymm7		;; R3 + I4 (final R4)

	vsubpd	ymm7, ymm0, ymm4		;; I3 - R4 (final I4)
	vaddpd	ymm0, ymm0, ymm4		;; I3 + R4 (final I3)

	L1prefetch srcreg+d2+L1pd, L1pt
	L1prefetchw dstreg+e2+L1pd, L1pt - L1PREFETCH_DEST_NONE

	vsubpd	ymm4, ymm5, ymm2		;; I1 - I2 (final I2)
	vaddpd	ymm5, ymm5, ymm2		;; I1 + I2 (final I1)

	ystore	[dstreg+32], ymm5		;; Temporarily save final I1

	vmovapd	ymm2, [screg+64+32]		;; cosine
	vmulpd	ymm5, ymm6, ymm2		;; A3 = R3 * cosine
	vmulpd	ymm2, ymm0, ymm2		;; B3 = I3 * cosine
	vmulpd	ymm0, ymm0, [screg+64]		;; C3 = I3 * sine
	vmulpd	ymm6, ymm6, [screg+64]		;; D3 = R3 * sine
	vsubpd	ymm5, ymm5, ymm0		;; A3 = A3 - C3 (final R3)
	vaddpd	ymm2, ymm2, ymm6		;; B3 = B3 + D3 (final I3)
	ystore	[dstreg+e2], ymm5		;; Save R3
	ystore	[dstreg+e2+32], ymm2		;; Save I3

	L1prefetch srcreg+d2+d1+L1pd, L1pt
	L1prefetchw dstreg+e2+e1+L1pd, L1pt - L1PREFETCH_DEST_NONE

	vmovapd	ymm0, [screg+128+32]		;; cosine
	vmulpd	ymm6, ymm1, ymm0		;; A2 = R2 * cosine
	vmulpd	ymm0, ymm4, ymm0		;; B2 = I2 * cosine

	vmovapd	ymm5, [screg+192+32]		;; cosine
	vmulpd	ymm2, ymm3, ymm5		;; A4 = R4 * cosine
	vmulpd	ymm5, ymm7, ymm5		;; B4 = I4 * cosine

	vmulpd	ymm4, ymm4, [screg+128]		;; C2 = I2 * sine
	vmulpd	ymm1, ymm1, [screg+128]		;; D2 = R2 * sine

	vsubpd	ymm6, ymm6, ymm4		;; A2 = A2 - C2 (final R2)
	vaddpd	ymm0, ymm0, ymm1		;; B2 = B2 + D2 (final I2)
	ystore	[dstreg+e1], ymm6		;; Save R2
	ystore	[dstreg+e1+32], ymm0		;; Save I2

	vmovapd	ymm4, [screg+192]		;; sine
	vmulpd	ymm7, ymm7, ymm4		;; C4 = I4 * sine
	vmulpd	ymm3, ymm3, ymm4		;; D4 = R4 * sine

	vsubpd	ymm2, ymm2, ymm7		;; A4 = A4 - C4 (final R4)
	vaddpd	ymm5, ymm5, ymm3		;; B4 = B4 + D4 (final I4)
	ystore	[dstreg+e2+e1], ymm2		;; Save R4
	ystore	[dstreg+e2+e1+32], ymm5		;; Save I4

	vmovapd	ymm2, [screg+0+32]		;; cosine
	vmovapd	ymm3, [dstreg]			;; Reload R1
	vmulpd	ymm5, ymm3, ymm2		;; A1 = R1 * cosine
	vmovapd	ymm0, [dstreg+32]		;; Reload I1
	vmulpd	ymm2, ymm0, ymm2		;; B1 = I1 * cosine

	vmovapd	ymm7, [screg+0]			;; sine
	vmulpd	ymm0, ymm0, ymm7		;; C1 = I1 * sine
	vmulpd	ymm3, ymm3, ymm7		;; D1 = R1 * sine

	vsubpd	ymm5, ymm5, ymm0		;; A1 = A1 - C1 (final R1)
	vaddpd	ymm2, ymm2, ymm3		;; B1 = B1 + D1 (final I1)

	ystore	[dstreg], ymm5			;; Save R1
	ystore	[dstreg+32], ymm2		;; Save I1

	bump	srcreg, srcinc
	bump	dstreg, dstinc
	bump	screg, scinc
	ENDM


;; 64-bit version

IFDEF X86_64

yr4_rsc_sg4cl_four_complex_fft4_preload MACRO
	ENDM
yr4_rsc_sg4cl_four_complex_fft4 MACRO srcreg,srcinc,d1,d2,dstreg,dstinc,e1,e2,screg,scinc,maxrpt,L1pt,L1pd

	vmovapd	ymm1, [srcreg]			;; R1
	vmovapd	ymm7, [srcreg+d1]		;; R2
	vshufpd	ymm0, ymm1, ymm7, 15		;; Shuffle R1 and R2 to create R1/R2 hi			; 1
	vshufpd	ymm1, ymm1, ymm7, 0		;; Shuffle R1 and R2 to create R1/R2 low		; 2

	vmovapd	ymm2, [srcreg+d2]		;; R3
	vmovapd	ymm7, [srcreg+d2+d1]		;; R4
	vshufpd	ymm3, ymm2, ymm7, 15		;; Shuffle R3 and R4 to create R3/R4 hi			; 3
	vshufpd	ymm2, ymm2, ymm7, 0		;; Shuffle R3 and R4 to create R3/R4 low		; 4

	ylow128s ymm4, ymm0, ymm3		;; Shuffle R1/R2 hi and R3/R4 hi (new R2)		; 5-6
	yhigh128s ymm0, ymm0, ymm3		;; Shuffle R1/R2 hi and R3/R4 hi (new R4)		; 6-7

	ylow128s ymm3, ymm1, ymm2		;; Shuffle R1/R2 low and R3/R4 low (new R1)		; 7-8
	yhigh128s ymm1, ymm1, ymm2		;; Shuffle R1/R2 low and R3/R4 low (new R3)		; 8-9

	vaddpd	ymm5, ymm4, ymm0		;; R2 + R4 (newer R2)
	vsubpd	ymm4, ymm4, ymm0		;; R2 - R4 (newer R4)

	L1prefetch srcreg+L1pd, L1pt
	L1prefetchw dstreg+L1pd, L1pt - L1PREFETCH_DEST_NONE

	vaddpd	ymm0, ymm3, ymm1		;; R1 + R3 (newer R1)
	vsubpd	ymm3, ymm3, ymm1		;; R1 - R3 (newer R3)

	vsubpd	ymm1, ymm0, ymm5		;; R1 - R2 (final R2)
	vaddpd	ymm8, ymm0, ymm5		;; R1 + R2 (final R1)

	vmovapd	ymm6, [srcreg+32]		;; I1
	vmovapd	ymm2, [srcreg+d1+32]		;; I2
	vshufpd	ymm5, ymm6, ymm2, 15		;; Shuffle I1 and I2 to create I1/I2 hi
	vshufpd	ymm6, ymm6, ymm2, 0		;; Shuffle I1 and I2 to create I1/I2 low

	vmovapd	ymm2, [srcreg+d2+32]		;; I3
	vmovapd	ymm7, [srcreg+d2+d1+32]		;; I4
	vshufpd	ymm0, ymm2, ymm7, 15		;; Shuffle I3 and I4 to create I3/I4 hi
	vshufpd	ymm2, ymm2, ymm7, 0		;; Shuffle I3 and I4 to create I3/I4 low

	ylow128s ymm7, ymm5, ymm0		;; Shuffle I1/I2 hi and I3/I4 hi (new I2)
	yhigh128s ymm5, ymm5, ymm0		;; Shuffle I1/I2 hi and I3/I4 hi (new I4)

	ylow128s ymm0, ymm6, ymm2		;; Shuffle I1/I2 low and I3/I4 low (new I1)
	yhigh128s ymm6, ymm6, ymm2		;; Shuffle I1/I2 low and I3/I4 low (new I3)

	vaddpd	ymm2, ymm7, ymm5		;; I2 + I4 (newer I2)
	vsubpd	ymm7, ymm7, ymm5		;; I2 - I4 (newer I4)

	L1prefetch srcreg+d1+L1pd, L1pt
	L1prefetchw dstreg+e1+L1pd, L1pt - L1PREFETCH_DEST_NONE

	vaddpd	ymm5, ymm0, ymm6		;; I1 + I3 (newer I1)
	vsubpd	ymm0, ymm0, ymm6		;; I1 - I3 (newer I3)

	vsubpd	ymm6, ymm3, ymm7		;; R3 - I4 (final R3)
	vaddpd	ymm3, ymm3, ymm7		;; R3 + I4 (final R4)

	vaddpd	ymm7, ymm0, ymm4		;; I3 + R4 (final I3)
	vsubpd	ymm0, ymm0, ymm4		;; I3 - R4 (final I4)

	L1prefetch srcreg+d2+L1pd, L1pt
	L1prefetchw dstreg+e2+L1pd, L1pt - L1PREFETCH_DEST_NONE

	vsubpd	ymm4, ymm5, ymm2		;; I1 - I2 (final I2)
	vaddpd	ymm9, ymm5, ymm2		;; I1 + I2 (final I1)

	vmovapd	ymm2, [screg+64+32]		;; cosine
	vmulpd	ymm5, ymm6, ymm2		;; A3 = R3 * cosine		;  1-5
	vmulpd	ymm2, ymm7, ymm2		;; B3 = I3 * cosine		;  2-6
	vmovapd	ymm15, [screg+64]
	vmulpd	ymm7, ymm7, ymm15		;; C3 = I3 * sine		;  3-7
	vmulpd	ymm6, ymm6, ymm15		;; D3 = R3 * sine		;  4-8

	L1prefetch srcreg+d2+d1+L1pd, L1pt
	L1prefetchw dstreg+e2+e1+L1pd, L1pt - L1PREFETCH_DEST_NONE

	vmovapd	ymm10, [screg+192+32]		;; cosine
	vmulpd	ymm11, ymm3, ymm10		;; A4 = R4 * cosine		;  5-9
	vmulpd	ymm10, ymm0, ymm10		;; B4 = I4 * cosine		;  6-10
	vmovapd	ymm15, [screg+192]		;; sine
	vmulpd	ymm0, ymm0, ymm15		;; C4 = I4 * sine		;  7-11
	vmulpd	ymm3, ymm3, ymm15		;; D4 = R4 * sine		;  8-12

	vsubpd	ymm5, ymm5, ymm7		;; A3 = A3 - C3 (final R3)	; 8-10
	vaddpd	ymm2, ymm2, ymm6		;; B3 = B3 + D3 (final I3)	; 9-11

	vmovapd	ymm12, [screg+128+32]		;; cosine
	vmulpd	ymm13, ymm1, ymm12		;; A2 = R2 * cosine		;  9-13
	vmulpd	ymm12, ymm4, ymm12		;; B2 = I2 * cosine		;  10-14
	vmovapd	ymm15, [screg+128]
	ystore	[dstreg+e2], ymm5		;; Save R3			; 11
	vmulpd	ymm4, ymm4, ymm15		;; C2 = I2 * sine		;  11-15
	ystore	[dstreg+e2+32], ymm2		;; Save I3			; 12
	vmulpd	ymm1, ymm1, ymm15		;; D2 = R2 * sine		;  12-16

	vsubpd	ymm11, ymm11, ymm0		;; A4 = A4 - C4 (final R4)	; 12-14
	vaddpd	ymm10, ymm10, ymm3		;; B4 = B4 + D4 (final I4)	; 13-15

	vmovapd	ymm2, [screg+0+32]		;; cosine
	vmulpd	ymm5, ymm8, ymm2		;; A1 = R1 * cosine		;  13-17
	vmulpd	ymm2, ymm9, ymm2		;; B1 = I1 * cosine		;  14-18
	vmovapd	ymm15, [screg+0]		;; sine
	ystore	[dstreg+e2+e1], ymm11		;; Save R4			; 15
	vmulpd	ymm9, ymm9, ymm15		;; C1 = I1 * sine		;  15-19
	ystore	[dstreg+e2+e1+32], ymm10	;; Save I4			; 16
	vmulpd	ymm8, ymm8, ymm15		;; D1 = R1 * sine		;  16-20

	vsubpd	ymm13, ymm13, ymm4		;; A2 = A2 - C2 (final R2)	; 16-18
	vaddpd	ymm12, ymm12, ymm1		;; B2 = B2 + D2 (final I2)	; 17-19

	ystore	[dstreg+e1], ymm13		;; Save R2			; 19
	ystore	[dstreg+e1+32], ymm12		;; Save I2			; 20

	vsubpd	ymm5, ymm5, ymm9		;; A1 = A1 - C1 (final R1)	; 20-22
	vaddpd	ymm2, ymm2, ymm8		;; B1 = B1 + D1 (final I1)	; 21-23
	ystore	[dstreg], ymm5			;; Save R1			; 23
	ystore	[dstreg+32], ymm2		;; Save I1			; 24

	bump	srcreg, srcinc
	bump	dstreg, dstinc
	bump	screg, scinc
	ENDM

;; Haswell FMA3 version
;; Timed at 36.8 clocks.

;;IFDEF ORIGINAL_SWIZZLE_CODE
IF (@INSTR(,%yarch,<FMA3>) NE 0)

yr4_rsc_sg4cl_four_complex_fft4_preload MACRO
	vmovapd	ymm15, YMM_ONE
	ENDM

yr4_rsc_sg4cl_four_complex_fft4 MACRO srcreg,srcinc,d1,d2,dstreg,dstinc,e1,e2,screg,scinc,maxrpt,L1pt,L1pd
	vmovapd	ymm1, [srcreg]			;; R1
	vmovapd	ymm2, [srcreg+d1]		;; R2
	vshufpd	ymm0, ymm1, ymm2, 15		;; Shuffle R1 and R2 to create R1/R2 hi			; 1
	vshufpd	ymm1, ymm1, ymm2, 0		;; Shuffle R1 and R2 to create R1/R2 low		; 2

	vmovapd	ymm3, [srcreg+d2]		;; R3
	vmovapd	ymm4, [srcreg+d2+d1]		;; R4
	vshufpd	ymm2, ymm3, ymm4, 15		;; Shuffle R3 and R4 to create R3/R4 hi			; 3
	vshufpd	ymm3, ymm3, ymm4, 0		;; Shuffle R3 and R4 to create R3/R4 low		; 4

	vmovapd	ymm5, [srcreg+32]		;; I1
	vmovapd	ymm6, [srcreg+d1+32]		;; I2
	vshufpd	ymm4, ymm5, ymm6, 15		;; Shuffle I1 and I2 to create I1/I2 hi			; 5
	vshufpd	ymm5, ymm5, ymm6, 0		;; Shuffle I1 and I2 to create I1/I2 low		; 6

	vmovapd	ymm7, [srcreg+d2+32]		;; I3
	vmovapd	ymm8, [srcreg+d2+d1+32]		;; I4
	vshufpd	ymm6, ymm7, ymm8, 15		;; Shuffle I3 and I4 to create I3/I4 hi			; 7
	vshufpd	ymm7, ymm7, ymm8, 0		;; Shuffle I3 and I4 to create I3/I4 low		; 8

	ylow128s ymm8, ymm0, ymm2		;; Shuffle R1/R2 hi and R3/R4 hi (new R2)		; 9-10
	yhigh128s ymm0, ymm0, ymm2		;; Shuffle R1/R2 hi and R3/R4 hi (new R4)		; 10-11
	L1prefetch srcreg+L1pd, L1pt
	L1prefetchw dstreg+L1pd, L1pt - L1PREFETCH_DEST_NONE

	ylow128s ymm2, ymm1, ymm3		;; Shuffle R1/R2 low and R3/R4 low (new R1)		; 11-12
	yhigh128s ymm1, ymm1, ymm3		;; Shuffle R1/R2 low and R3/R4 low (new R3)		; 12-13

	yfmaddpd ymm3, ymm8, ymm15, ymm0	;; R2 + R4 (newer R2)					; 12-16
	yfmsubpd ymm8, ymm8, ymm15, ymm0	;; R2 - R4 (newer R4)					; 12-16
	L1prefetch srcreg+d1+L1pd, L1pt
	L1prefetchw dstreg+e1+L1pd, L1pt - L1PREFETCH_DEST_NONE

	ylow128s ymm0, ymm4, ymm6		;; Shuffle I1/I2 hi and I3/I4 hi (new I2)		; 13-14
	yhigh128s ymm4, ymm4, ymm6		;; Shuffle I1/I2 hi and I3/I4 hi (new I4)		; 14-15

	yfmaddpd ymm6, ymm2, ymm15, ymm1	;; R1 + R3 (newer R1)					; 14-18
	yfmsubpd ymm2, ymm2, ymm15, ymm1	;; R1 - R3 (newer R3)					; 14-18
	vmovapd	ymm9, [screg+128+32]		;; cosine for R2/I2

	ylow128s ymm1, ymm5, ymm7		;; Shuffle I1/I2 low and I3/I4 low (new I1)		; 15-16
	yhigh128s ymm5, ymm5, ymm7		;; Shuffle I1/I2 low and I3/I4 low (new I3)		; 16-17

	yfmsubpd ymm7, ymm0, ymm15, ymm4	;; I2 - I4 (newer I4)					; 16-20
	yfmaddpd ymm0, ymm0, ymm15, ymm4	;; I2 + I4 (newer I2)					; 16-20
	vmovapd	ymm10, [screg+128]		;; sine for R2/I2

	L1prefetch srcreg+d2+L1pd, L1pt
	L1prefetchw dstreg+e2+L1pd, L1pt - L1PREFETCH_DEST_NONE

	yfmaddpd ymm4, ymm1, ymm15, ymm5	;; I1 + I3 (newer I1)					; 18-22
	yfmsubpd ymm1, ymm1, ymm15, ymm5	;; I1 - I3 (newer I3)					; 18-22
	vmovapd	ymm11, [screg+0+32]		;; cosine for R1/I1

	yfmsubpd ymm5, ymm6, ymm15, ymm3	;; R1 - R2 (final R2)					; 19-23
	yfmaddpd ymm6, ymm6, ymm15, ymm3	;; R1 + R2 (final R1)					; 19-23
	vmovapd	ymm12, [screg+0]		;; sine for R1/I1

	L1prefetch srcreg+d2+d1+L1pd, L1pt
	L1prefetchw dstreg+e2+e1+L1pd, L1pt - L1PREFETCH_DEST_NONE
	bump	srcreg, srcinc

	yfmsubpd ymm3, ymm2, ymm15, ymm7	;; R3 - I4 (final R3)					; 21-25
	yfmaddpd ymm2, ymm2, ymm15, ymm7	;; R3 + I4 (final R4)					; 21-25
	vmovapd	ymm14, [screg+64+32]		;; cosine for R3/I3

	yfmsubpd ymm7, ymm4, ymm15, ymm0	;; I1 - I2 (final I2)					; 23-27
	yfmaddpd ymm4, ymm4, ymm15, ymm0	;; I1 + I2 (final I1)					; 23-27

	yfmaddpd ymm0, ymm1, ymm15, ymm8	;; I3 + R4 (final I3)					; 24-28
	yfmsubpd ymm1, ymm1, ymm15, ymm8	;; I3 - R4 (final I4)					; 24-28

	vmulpd	ymm8, ymm5, ymm9		;; A2 = R2 * cosine					; 25-29
	vmulpd	ymm5, ymm5, ymm10		;; B2 = R2 * sine					; 25-29

	vmulpd	ymm13, ymm6, ymm11		;; A1 = R1 * cosine					; 26-30
	vmulpd	ymm6, ymm6, ymm12		;; B1 = R1 * sine					; 26-30

	yfnmaddpd ymm8, ymm7, ymm10, ymm8	;; A2 = A2 - I2 * sine (final R2)			; 30-34
	vmulpd	ymm10, ymm3, ymm14		;; A3 = R3 * cosine					; 27-31
	yfmaddpd ymm7, ymm7, ymm9, ymm5		;; B2 = I2 * cosine + B2 (final I2)			; 30-34
	vmovapd	ymm9, [screg+64]		;; sine for R3/I3
	vmulpd	ymm3, ymm3, ymm9		;; B3 = R3 * sine					; 27-31

	vmovapd	ymm5, [screg+192+32]		;; cosine for R4/I4
	yfnmaddpd ymm13, ymm4, ymm12, ymm13	;; A1 = A1 - I1 * sine (final R1)			; 31-35
	vmulpd	ymm12, ymm2, ymm5		;; A4 = R4 * cosine					; 28-32
	yfmaddpd ymm4, ymm4, ymm11, ymm6	;; B1 = I1 * cosine + B1 (final I1)			; 31-35
	vmovapd	ymm11, [screg+192]		;; sine for R4/I4
	vmulpd	ymm2, ymm2, ymm11		;; B4 = R4 * sine					; 28-32
	bump	screg, scinc

	yfnmaddpd ymm10, ymm0, ymm9, ymm10	;; A3 = A3 - I3 * sine (final R3)			; 32-36
	yfmaddpd ymm0, ymm0, ymm14, ymm3	;; B3 = I3 * cosine + B3 (final I3)			; 32-36

	yfnmaddpd ymm12, ymm1, ymm11, ymm12	;; A4 = A4 - I4 * sine (final R4)			; 33-37
	yfmaddpd ymm1, ymm1, ymm5, ymm2		;; B4 = I4 * cosine + B4 (final I4)			; 33-37

	ystore	[dstreg+e1], ymm8		;; Save R2						; 35
	ystore	[dstreg+e1+32], ymm7		;; Save I2						; 35+1
	ystore	[dstreg], ymm13			;; Save R1						; 36+1
	ystore	[dstreg+32], ymm4		;; Save I1						; 36+2
	ystore	[dstreg+e2], ymm10		;; Save R3						; 37+2
	ystore	[dstreg+e2+32], ymm0		;; Save I3						; 37+3
	ystore	[dstreg+e2+e1], ymm12		;; Save R4						; 38+3
	ystore	[dstreg+e2+e1+32], ymm1		;; Save I4						; 38+4
	bump	dstreg, dstinc
	ENDM
ENDIF
;;ENDIF

;; Haswell FMA3 version --- On Skylake, new sizzle code is between minus one and one clocks faster (within our benchmarking margin of error)
;; This code makes much less use of port 5, but increases use of the port 2&3 load units.

IFDEF NEW_SWIZZLE_CODE
IF (@INSTR(,%yarch,<FMA3>) NE 0)

yr4_rsc_sg4cl_four_complex_fft4_preload MACRO
	vmovapd	ymm15, YMM_ONE
	ENDM

yr4_rsc_sg4cl_four_complex_fft4 MACRO srcreg,srcinc,d1,d2,dstreg,dstinc,e1,e2,screg,scinc,maxrpt,L1pt,L1pd

;; 16 arithmetic ops on ports 01
;; 24 load ops on ports 23
;; 4 prefetch ops on 23
;; 8 shuffle ops on port 5

	vmovapd	xmm1, [srcreg]					;; R1 low			;   1
	vmovapd	xmm7, [srcreg+d1]				;; R2 low			;   1
	vinsertf128 ymm1, ymm1, [srcreg+d2], 1			;; R3/R1 low			;   2
	vinsertf128 ymm7, ymm7, [srcreg+d2+d1], 1		;; R4/R2 low			;   2
	vshufpd	ymm0, ymm1, ymm7, 15		;; Shuffle R3/R1 and R4/R2 lows (first R2)	;  3
	vshufpd	ymm1, ymm1, ymm7, 0		;; Shuffle R3/R1 and R4/R2 lows (first R1)	;  4

	vmovapd	xmm5, [srcreg][16]				;; R1 hi			;   3
	vmovapd	xmm7, [srcreg+d1][16]				;; R2 hi			;   3
	vinsertf128 ymm5, ymm5, [srcreg+d2][16], 1		;; R3/R1 hi			;   4
	vinsertf128 ymm7, ymm7, [srcreg+d2+d1][16], 1		;; R4/R2 hi			;   4
	vshufpd	ymm4, ymm5, ymm7, 15		;; Shuffle R3/R1 hi and R4/R2 hi (first R4)	;  5
	vshufpd	ymm5, ymm5, ymm7, 0		;; Shuffle R3/R1 hi and R4/R2 hi (first R3)	;  6

	vmovapd	xmm10, [srcreg+32]				;; I1 low			;   5
	vmovapd	xmm9, [srcreg+d1+32]				;; I2 low			;   5
	vinsertf128 ymm10, ymm10, [srcreg+d2+32], 1		;; I3/I1 low			;   6
	vinsertf128 ymm9, ymm9, [srcreg+d2+d1+32], 1		;; I4/I2 low			;   6

	yfmaddpd ymm3, ymm0, ymm15, ymm4	;; R2 + R4 (newer R2)				; 6-10		n 12
	yfmsubpd ymm0, ymm0, ymm15, ymm4	;; R2 - R4 (newer R4)				; 6-10		n 17
	vmovapd	xmm13, [srcreg+32][16]				;; I1 hi			;   7

	yfmaddpd ymm6, ymm1, ymm15, ymm5	;; R1 + R3 (newer R1)				; 7-11		n 12
	yfmsubpd ymm1, ymm1, ymm15, ymm5	;; R1 - R3 (newer R3)				; 7-11		n 15
	vmovapd	xmm14, [srcreg+d1+32][16]			;; I2 hi			;   7

	vshufpd	ymm7, ymm10, ymm9, 15		;; Shuffle I3/I1 and I4/I2 lows (first I2)	;  7
	vshufpd	ymm10, ymm10, ymm9, 0		;; Shuffle I3/I1 and I4/I2 lows (first I1)	;  8

	vinsertf128 ymm13, ymm13, [srcreg+d2+32][16], 1		;; I3/I1 hi			;   8
	vinsertf128 ymm14, ymm14, [srcreg+d2+d1+32][16], 1	;; I4/I2 hi			;   8
	vshufpd	ymm9, ymm13, ymm14, 15		;; Shuffle I3/I1 hi and I4/I2 hi (first I4)	;  9
	vshufpd	ymm13, ymm13, ymm14, 0		;; Shuffle I3/I1 hi and I4/I2 hi (first I3)	;  10

	yfmsubpd ymm14, ymm7, ymm15, ymm9	;; I2 - I4 (newer I4)				; 10-14		n 15
	yfmaddpd ymm7, ymm7, ymm15, ymm9	;; I2 + I4 (newer I2)				; 10-14		n 16
	L1prefetch srcreg+L1pd, L1pt
	L1prefetchw dstreg+L1pd, L1pt - L1PREFETCH_DEST_NONE

	yfmaddpd ymm9, ymm10, ymm15, ymm13	;; I1 + I3 (newer I1)				; 11-15		n 16
	yfmsubpd ymm10, ymm10, ymm15, ymm13	;; I1 - I3 (newer I3)				; 11-15		n 17
	L1prefetch srcreg+d1+L1pd, L1pt
	L1prefetchw dstreg+e1+L1pd, L1pt - L1PREFETCH_DEST_NONE

	yfmsubpd ymm13, ymm6, ymm15, ymm3	;; R1 - R2 (final R2)				; 12-16		n 18
	yfmaddpd ymm6, ymm6, ymm15, ymm3	;; R1 + R2 (final R1)				; 12-16		n 19
	vmovapd	ymm2, [screg+128+32]		;; cosine for R2/I2
	L1prefetch srcreg+d2+L1pd, L1pt
	L1prefetchw dstreg+e2+L1pd, L1pt - L1PREFETCH_DEST_NONE

	yfmsubpd ymm3, ymm1, ymm15, ymm14	;; R3 - I4 (final R3)				; 15-19		n 20
	yfmaddpd ymm1, ymm1, ymm15, ymm14	;; R3 + I4 (final R4)				; 15-19		n 21
	vmovapd	ymm8, [screg+128]		;; sine for R2/I2
	L1prefetch srcreg+d2+d1+L1pd, L1pt
	L1prefetchw dstreg+e2+e1+L1pd, L1pt - L1PREFETCH_DEST_NONE

	yfmsubpd ymm14, ymm9, ymm15, ymm7	;; I1 - I2 (final I2)				; 16-20
	yfmaddpd ymm9, ymm9, ymm15, ymm7	;; I1 + I2 (final I1)				; 16-20
	vmovapd	ymm11, [screg+0+32]		;; cosine for R1/I1

	yfmaddpd ymm7, ymm10, ymm15, ymm0	;; I3 + R4 (final I3)				; 17-21
	yfmsubpd ymm10, ymm10, ymm15, ymm0	;; I3 - R4 (final I4)				; 17-21
	vmovapd	ymm12, [screg+0]		;; sine for R1/I1

	vmulpd	ymm0, ymm13, ymm2		;; A2 = R2 * cosine				; 18-22
	vmulpd	ymm13, ymm13, ymm8		;; B2 = R2 * sine				; 18-22
	vmovapd	ymm4, [screg+64+32]		;; cosine for R3/I3

	vmulpd	ymm5, ymm6, ymm11		;; A1 = R1 * cosine				; 19-23
	vmulpd	ymm6, ymm6, ymm12		;; B1 = R1 * sine				; 19-23
	bump	srcreg, srcinc

	yfnmaddpd ymm0, ymm14, ymm8, ymm0	;; A2 = A2 - I2 * sine (final R2)		; 23-27
	yfmaddpd ymm13, ymm14, ymm2, ymm13	;; B2 = I2 * cosine + B2 (final I2)		; 23-27

	vmovapd	ymm8, [screg+64]		;; sine for R3/I3
	vmulpd	ymm2, ymm3, ymm4		;; A3 = R3 * cosine				; 20-24
	vmulpd	ymm3, ymm3, ymm8		;; B3 = R3 * sine				; 20-24
	vmovapd	ymm14, [screg+192+32]		;; cosine for R4/I4

	yfnmaddpd ymm5, ymm9, ymm12, ymm5	;; A1 = A1 - I1 * sine (final R1)		; 24-28
	yfmaddpd ymm6, ymm9, ymm11, ymm6	;; B1 = I1 * cosine + B1 (final I1)		; 24-28

	vmovapd	ymm12, [screg+192]		;; sine for R4/I4
	vmulpd	ymm11, ymm1, ymm14		;; A4 = R4 * cosine				; 21-25
	vmulpd	ymm1, ymm1, ymm12		;; B4 = R4 * sine				; 21-25
	bump	screg, scinc

	yfnmaddpd ymm2, ymm7, ymm8, ymm2	;; A3 = A3 - I3 * sine (final R3)		; 25-29
	yfmaddpd ymm3, ymm7, ymm4, ymm3		;; B3 = I3 * cosine + B3 (final I3)		; 25-29

	yfnmaddpd ymm11, ymm10, ymm12, ymm11	;; A4 = A4 - I4 * sine (final R4)		; 26-30
	yfmaddpd ymm1, ymm10, ymm14, ymm1	;; B4 = I4 * cosine + B4 (final I4)		; 26-30

	ystore	[dstreg+e1], ymm0		;; Save R2					; 28
	ystore	[dstreg+e1+32], ymm13		;; Save I2					; 28+1
	ystore	[dstreg], ymm5			;; Save R1					; 29+1
	ystore	[dstreg+32], ymm6		;; Save I1					; 29+2
	ystore	[dstreg+e2], ymm2		;; Save R3					; 30+2
	ystore	[dstreg+e2+32], ymm3		;; Save I3					; 30+3
	ystore	[dstreg+e2+e1], ymm11		;; Save R4					; 31+3
	ystore	[dstreg+e2+e1+32], ymm1		;; Save I4					; 31+4
	bump	dstreg, dstinc

	ENDM
ENDIF
ENDIF

ENDIF

; These versions use registers for distances between blocks.  This lets us share pass1 code.

yr4_rsc_sg4clreg_four_complex_fft4_preload MACRO
	ENDM
yr4_rsc_sg4clreg_four_complex_fft4 MACRO srcreg,srcinc,d1,d2,dstreg,dstinc,e1reg,e3reg,screg,scinc,maxrpt,L1pt,L1preg
	NOT IMPLMENTED IN 32-BIT
	ENDM

;; 64-bit version

IFDEF X86_64

yr4_rsc_sg4clreg_four_complex_fft4_preload MACRO
	ENDM
yr4_rsc_sg4clreg_four_complex_fft4 MACRO srcreg,srcinc,d1,d2,dstreg,dstinc,e1reg,e3reg,screg,scinc,maxrpt,L1pt,L1preg

	vmovapd	ymm1, [srcreg]			;; R1
	vmovapd	ymm7, [srcreg+d1]		;; R2
	vshufpd	ymm0, ymm1, ymm7, 15		;; Shuffle R1 and R2 to create R1/R2 hi			; 1
	vshufpd	ymm1, ymm1, ymm7, 0		;; Shuffle R1 and R2 to create R1/R2 low		; 2

	vmovapd	ymm2, [srcreg+d2]		;; R3
	vmovapd	ymm7, [srcreg+d2+d1]		;; R4
	vshufpd	ymm3, ymm2, ymm7, 15		;; Shuffle R3 and R4 to create R3/R4 hi			; 3
	vshufpd	ymm2, ymm2, ymm7, 0		;; Shuffle R3 and R4 to create R3/R4 low		; 4

	ylow128s ymm4, ymm0, ymm3		;; Shuffle R1/R2 hi and R3/R4 hi (new R2)		; 5-6
	yhigh128s ymm0, ymm0, ymm3		;; Shuffle R1/R2 hi and R3/R4 hi (new R4)		; 6-7

	ylow128s ymm3, ymm1, ymm2		;; Shuffle R1/R2 low and R3/R4 low (new R1)		; 7-8
	yhigh128s ymm1, ymm1, ymm2		;; Shuffle R1/R2 low and R3/R4 low (new R3)		; 8-9

	vaddpd	ymm5, ymm4, ymm0		;; R2 + R4 (newer R2)
	vsubpd	ymm4, ymm4, ymm0		;; R2 - R4 (newer R4)

	L1prefetchw L1preg, L1pt - L1PREFETCH_DEST_NONE

	vaddpd	ymm0, ymm3, ymm1		;; R1 + R3 (newer R1)
	vsubpd	ymm3, ymm3, ymm1		;; R1 - R3 (newer R3)

	vsubpd	ymm1, ymm0, ymm5		;; R1 - R2 (final R2)
	vaddpd	ymm8, ymm0, ymm5		;; R1 + R2 (final R1)

	vmovapd	ymm6, [srcreg+32]		;; I1
	vmovapd	ymm2, [srcreg+d1+32]		;; I2
	vshufpd	ymm5, ymm6, ymm2, 15		;; Shuffle I1 and I2 to create I1/I2 hi
	vshufpd	ymm6, ymm6, ymm2, 0		;; Shuffle I1 and I2 to create I1/I2 low

	vmovapd	ymm2, [srcreg+d2+32]		;; I3
	vmovapd	ymm7, [srcreg+d2+d1+32]		;; I4
	vshufpd	ymm0, ymm2, ymm7, 15		;; Shuffle I3 and I4 to create I3/I4 hi
	vshufpd	ymm2, ymm2, ymm7, 0		;; Shuffle I3 and I4 to create I3/I4 low

	ylow128s ymm7, ymm5, ymm0		;; Shuffle I1/I2 hi and I3/I4 hi (new I2)
	yhigh128s ymm5, ymm5, ymm0		;; Shuffle I1/I2 hi and I3/I4 hi (new I4)

	ylow128s ymm0, ymm6, ymm2		;; Shuffle I1/I2 low and I3/I4 low (new I1)
	yhigh128s ymm6, ymm6, ymm2		;; Shuffle I1/I2 low and I3/I4 low (new I3)

	vaddpd	ymm2, ymm7, ymm5		;; I2 + I4 (newer I2)
	vsubpd	ymm7, ymm7, ymm5		;; I2 - I4 (newer I4)

	L1prefetchw L1preg+e1reg, L1pt - L1PREFETCH_DEST_NONE

	vaddpd	ymm5, ymm0, ymm6		;; I1 + I3 (newer I1)
	vsubpd	ymm0, ymm0, ymm6		;; I1 - I3 (newer I3)

	vsubpd	ymm6, ymm3, ymm7		;; R3 - I4 (final R3)
	vaddpd	ymm3, ymm3, ymm7		;; R3 + I4 (final R4)

	vaddpd	ymm7, ymm0, ymm4		;; I3 + R4 (final I3)
	vsubpd	ymm0, ymm0, ymm4		;; I3 - R4 (final I4)

	L1prefetchw L1preg+2*e1reg, L1pt - L1PREFETCH_DEST_NONE

	vsubpd	ymm4, ymm5, ymm2		;; I1 - I2 (final I2)
	vaddpd	ymm9, ymm5, ymm2		;; I1 + I2 (final I1)

	vmovapd	ymm2, [screg+64+32]		;; cosine
	vmulpd	ymm5, ymm6, ymm2		;; A3 = R3 * cosine		;  1-5
	vmulpd	ymm2, ymm7, ymm2		;; B3 = I3 * cosine		;  2-6
	vmovapd	ymm15, [screg+64]
	vmulpd	ymm7, ymm7, ymm15		;; C3 = I3 * sine		;  3-7
	vmulpd	ymm6, ymm6, ymm15		;; D3 = R3 * sine		;  4-8

	L1prefetchw L1preg+e3reg, L1pt - L1PREFETCH_DEST_NONE

	vmovapd	ymm10, [screg+192+32]		;; cosine
	vmulpd	ymm11, ymm3, ymm10		;; A4 = R4 * cosine		;  5-9
	vmulpd	ymm10, ymm0, ymm10		;; B4 = I4 * cosine		;  6-10
	vmovapd	ymm15, [screg+192]		;; sine
	vmulpd	ymm0, ymm0, ymm15		;; C4 = I4 * sine		;  7-11
	vmulpd	ymm3, ymm3, ymm15		;; D4 = R4 * sine		;  8-12

	vsubpd	ymm5, ymm5, ymm7		;; A3 = A3 - C3 (final R3)	; 8-10
	vaddpd	ymm2, ymm2, ymm6		;; B3 = B3 + D3 (final I3)	; 9-11

	vmovapd	ymm12, [screg+128+32]		;; cosine
	vmulpd	ymm13, ymm1, ymm12		;; A2 = R2 * cosine		;  9-13
	vmulpd	ymm12, ymm4, ymm12		;; B2 = I2 * cosine		;  10-14
	vmovapd	ymm15, [screg+128]
	ystore	[dstreg+2*e1reg], ymm5		;; Save R3			; 11
	vmulpd	ymm4, ymm4, ymm15		;; C2 = I2 * sine		;  11-15
	ystore	[dstreg+2*e1reg+32], ymm2	;; Save I3			; 12
	vmulpd	ymm1, ymm1, ymm15		;; D2 = R2 * sine		;  12-16

	vsubpd	ymm11, ymm11, ymm0		;; A4 = A4 - C4 (final R4)	; 12-14
	vaddpd	ymm10, ymm10, ymm3		;; B4 = B4 + D4 (final I4)	; 13-15

	vmovapd	ymm2, [screg+0+32]		;; cosine
	vmulpd	ymm5, ymm8, ymm2		;; A1 = R1 * cosine		;  13-17
	vmulpd	ymm2, ymm9, ymm2		;; B1 = I1 * cosine		;  14-18
	vmovapd	ymm15, [screg+0]		;; sine
	ystore	[dstreg+e3reg], ymm11		;; Save R4			; 15
	vmulpd	ymm9, ymm9, ymm15		;; C1 = I1 * sine		;  15-19
	ystore	[dstreg+e3reg+32], ymm10	;; Save I4			; 16
	vmulpd	ymm8, ymm8, ymm15		;; D1 = R1 * sine		;  16-20

	vsubpd	ymm13, ymm13, ymm4		;; A2 = A2 - C2 (final R2)	; 16-18
	vaddpd	ymm12, ymm12, ymm1		;; B2 = B2 + D2 (final I2)	; 17-19

	ystore	[dstreg+e1reg], ymm13		;; Save R2			; 19
	ystore	[dstreg+e1reg+32], ymm12	;; Save I2			; 20

	vsubpd	ymm5, ymm5, ymm9		;; A1 = A1 - C1 (final R1)	; 20-22
	vaddpd	ymm2, ymm2, ymm8		;; B1 = B1 + D1 (final I1)	; 21-23
	ystore	[dstreg], ymm5			;; Save R1			; 23
	ystore	[dstreg+32], ymm2		;; Save I1			; 24

	bump	srcreg, srcinc
	bump	dstreg, dstinc
	bump	screg, scinc
	bump	L1preg, dstinc
	ENDM

;; Haswell FMA3 version
;; Timed at 36.8 clocks.

;;IFDEF ORIGINAL_SWIZZLE_CODE
IF (@INSTR(,%yarch,<FMA3>) NE 0)

yr4_rsc_sg4clreg_four_complex_fft4_preload MACRO
	vmovapd	ymm15, YMM_ONE
	ENDM
yr4_rsc_sg4clreg_four_complex_fft4 MACRO srcreg,srcinc,d1,d2,dstreg,dstinc,e1reg,e3reg,screg,scinc,maxrpt,L1pt,L1preg
	vmovapd	ymm1, [srcreg]			;; R1
	vmovapd	ymm2, [srcreg+d1]		;; R2
	vshufpd	ymm0, ymm1, ymm2, 15		;; Shuffle R1 and R2 to create R1/R2 hi			; 1
	vshufpd	ymm1, ymm1, ymm2, 0		;; Shuffle R1 and R2 to create R1/R2 low		; 2

	vmovapd	ymm3, [srcreg+d2]		;; R3
	vmovapd	ymm4, [srcreg+d2+d1]		;; R4
	vshufpd	ymm2, ymm3, ymm4, 15		;; Shuffle R3 and R4 to create R3/R4 hi			; 3
	vshufpd	ymm3, ymm3, ymm4, 0		;; Shuffle R3 and R4 to create R3/R4 low		; 4

	vmovapd	ymm5, [srcreg+32]		;; I1
	vmovapd	ymm6, [srcreg+d1+32]		;; I2
	vshufpd	ymm4, ymm5, ymm6, 15		;; Shuffle I1 and I2 to create I1/I2 hi			; 5
	vshufpd	ymm5, ymm5, ymm6, 0		;; Shuffle I1 and I2 to create I1/I2 low		; 6

	vmovapd	ymm7, [srcreg+d2+32]		;; I3
	vmovapd	ymm8, [srcreg+d2+d1+32]		;; I4
	vshufpd	ymm6, ymm7, ymm8, 15		;; Shuffle I3 and I4 to create I3/I4 hi			; 7
	vshufpd	ymm7, ymm7, ymm8, 0		;; Shuffle I3 and I4 to create I3/I4 low		; 8

	ylow128s ymm8, ymm0, ymm2		;; Shuffle R1/R2 hi and R3/R4 hi (new R2)		; 9-10
	yhigh128s ymm0, ymm0, ymm2		;; Shuffle R1/R2 hi and R3/R4 hi (new R4)		; 10-11
	L1prefetchw L1preg, L1pt - L1PREFETCH_DEST_NONE

	ylow128s ymm2, ymm1, ymm3		;; Shuffle R1/R2 low and R3/R4 low (new R1)		; 11-12
	yhigh128s ymm1, ymm1, ymm3		;; Shuffle R1/R2 low and R3/R4 low (new R3)		; 12-13

	yfmaddpd ymm3, ymm8, ymm15, ymm0	;; R2 + R4 (newer R2)					; 12-16
	yfmsubpd ymm8, ymm8, ymm15, ymm0	;; R2 - R4 (newer R4)					; 12-16
	L1prefetchw L1preg+e1reg, L1pt - L1PREFETCH_DEST_NONE

	ylow128s ymm0, ymm4, ymm6		;; Shuffle I1/I2 hi and I3/I4 hi (new I2)		; 13-14
	yhigh128s ymm4, ymm4, ymm6		;; Shuffle I1/I2 hi and I3/I4 hi (new I4)		; 14-15

	yfmaddpd ymm6, ymm2, ymm15, ymm1	;; R1 + R3 (newer R1)					; 14-18
	yfmsubpd ymm2, ymm2, ymm15, ymm1	;; R1 - R3 (newer R3)					; 14-18
	vmovapd	ymm9, [screg+128+32]		;; cosine for R2/I2

	ylow128s ymm1, ymm5, ymm7		;; Shuffle I1/I2 low and I3/I4 low (new I1)		; 15-16
	yhigh128s ymm5, ymm5, ymm7		;; Shuffle I1/I2 low and I3/I4 low (new I3)		; 16-17

	yfmsubpd ymm7, ymm0, ymm15, ymm4	;; I2 - I4 (newer I4)					; 16-20
	yfmaddpd ymm0, ymm0, ymm15, ymm4	;; I2 + I4 (newer I2)					; 16-20
	vmovapd	ymm10, [screg+128]		;; sine for R2/I2

	L1prefetchw L1preg+2*e1reg, L1pt - L1PREFETCH_DEST_NONE

	yfmaddpd ymm4, ymm1, ymm15, ymm5	;; I1 + I3 (newer I1)					; 18-22
	yfmsubpd ymm1, ymm1, ymm15, ymm5	;; I1 - I3 (newer I3)					; 18-22
	vmovapd	ymm11, [screg+0+32]		;; cosine for R1/I1

	yfmsubpd ymm5, ymm6, ymm15, ymm3	;; R1 - R2 (final R2)					; 19-23
	yfmaddpd ymm6, ymm6, ymm15, ymm3	;; R1 + R2 (final R1)					; 19-23
	vmovapd	ymm12, [screg+0]		;; sine for R1/I1

	L1prefetchw L1preg+e3reg, L1pt - L1PREFETCH_DEST_NONE
	bump	srcreg, srcinc

	yfmsubpd ymm3, ymm2, ymm15, ymm7	;; R3 - I4 (final R3)					; 21-25
	yfmaddpd ymm2, ymm2, ymm15, ymm7	;; R3 + I4 (final R4)					; 21-25
	vmovapd	ymm14, [screg+64+32]		;; cosine for R3/I3

	yfmsubpd ymm7, ymm4, ymm15, ymm0	;; I1 - I2 (final I2)					; 23-27
	yfmaddpd ymm4, ymm4, ymm15, ymm0	;; I1 + I2 (final I1)					; 23-27

	yfmaddpd ymm0, ymm1, ymm15, ymm8	;; I3 + R4 (final I3)					; 24-28
	yfmsubpd ymm1, ymm1, ymm15, ymm8	;; I3 - R4 (final I4)					; 24-28

	vmulpd	ymm8, ymm5, ymm9		;; A2 = R2 * cosine					; 25-29
	vmulpd	ymm5, ymm5, ymm10		;; B2 = R2 * sine					; 25-29

	vmulpd	ymm13, ymm6, ymm11		;; A1 = R1 * cosine					; 26-30
	vmulpd	ymm6, ymm6, ymm12		;; B1 = R1 * sine					; 26-30

	yfnmaddpd ymm8, ymm7, ymm10, ymm8	;; A2 = A2 - I2 * sine (final R2)			; 30-34
	vmulpd	ymm10, ymm3, ymm14		;; A3 = R3 * cosine					; 27-31
	yfmaddpd ymm7, ymm7, ymm9, ymm5		;; B2 = I2 * cosine + B2 (final I2)			; 30-34
	vmovapd	ymm9, [screg+64]		;; sine for R3/I3
	vmulpd	ymm3, ymm3, ymm9		;; B3 = R3 * sine					; 27-31

	vmovapd	ymm5, [screg+192+32]		;; cosine for R4/I4
	yfnmaddpd ymm13, ymm4, ymm12, ymm13	;; A1 = A1 - I1 * sine (final R1)			; 31-35
	vmulpd	ymm12, ymm2, ymm5		;; A4 = R4 * cosine					; 28-32
	yfmaddpd ymm4, ymm4, ymm11, ymm6	;; B1 = I1 * cosine + B1 (final I1)			; 31-35
	vmovapd	ymm11, [screg+192]		;; sine for R4/I4
	vmulpd	ymm2, ymm2, ymm11		;; B4 = R4 * sine					; 28-32
	bump	screg, scinc

	yfnmaddpd ymm10, ymm0, ymm9, ymm10	;; A3 = A3 - I3 * sine (final R3)			; 32-36
	yfmaddpd ymm0, ymm0, ymm14, ymm3	;; B3 = I3 * cosine + B3 (final I3)			; 32-36

	yfnmaddpd ymm12, ymm1, ymm11, ymm12	;; A4 = A4 - I4 * sine (final R4)			; 33-37
	yfmaddpd ymm1, ymm1, ymm5, ymm2		;; B4 = I4 * cosine + B4 (final I4)			; 33-37

	ystore	[dstreg+e1reg], ymm8		;; Save R2						; 35
	ystore	[dstreg+e1reg+32], ymm7		;; Save I2						; 35+1
	ystore	[dstreg], ymm13			;; Save R1						; 36+1
	ystore	[dstreg+32], ymm4		;; Save I1						; 36+2
	ystore	[dstreg+2*e1reg], ymm10		;; Save R3						; 37+2
	ystore	[dstreg+2*e1reg+32], ymm0	;; Save I3						; 37+3
	ystore	[dstreg+e3reg], ymm12		;; Save R4						; 38+3
	ystore	[dstreg+e3reg+32], ymm1		;; Save I4						; 38+4
	bump	dstreg, dstinc
	bump	L1preg, dstinc
	ENDM
ENDIF
;;ENDIF

;; Haswell FMA3 version --- On Skylake, new sizzle code is between minus one and one clocks faster (within our benchmarking margin of error)
;; This code makes much less use of port 5, but increases use of the port 2&3 load units.

IFDEF NEW_SWIZZLE_CODE
IF (@INSTR(,%yarch,<FMA3>) NE 0)

yr4_rsc_sg4clreg_four_complex_fft4_preload MACRO
	vmovapd	ymm15, YMM_ONE
	ENDM
yr4_rsc_sg4clreg_four_complex_fft4 MACRO srcreg,srcinc,d1,d2,dstreg,dstinc,e1reg,e3reg,screg,scinc,maxrpt,L1pt,L1preg

;; 16 arithmetic ops on ports 01
;; 24 load ops on ports 23
;; 4 prefetch ops on 23
;; 8 shuffle ops on port 5

	vmovapd	xmm1, [srcreg]					;; R1 low			;   1
	vmovapd	xmm7, [srcreg+d1]				;; R2 low			;   1
	vinsertf128 ymm1, ymm1, [srcreg+d2], 1			;; R3/R1 low			;   2
	vinsertf128 ymm7, ymm7, [srcreg+d2+d1], 1		;; R4/R2 low			;   2
	vshufpd	ymm0, ymm1, ymm7, 15		;; Shuffle R3/R1 and R4/R2 lows (first R2)	;  3
	vshufpd	ymm1, ymm1, ymm7, 0		;; Shuffle R3/R1 and R4/R2 lows (first R1)	;  4

	vmovapd	xmm5, [srcreg][16]				;; R1 hi			;   3
	vmovapd	xmm7, [srcreg+d1][16]				;; R2 hi			;   3
	vinsertf128 ymm5, ymm5, [srcreg+d2][16], 1		;; R3/R1 hi			;   4
	vinsertf128 ymm7, ymm7, [srcreg+d2+d1][16], 1		;; R4/R2 hi			;   4
	vshufpd	ymm4, ymm5, ymm7, 15		;; Shuffle R3/R1 hi and R4/R2 hi (first R4)	;  5
	vshufpd	ymm5, ymm5, ymm7, 0		;; Shuffle R3/R1 hi and R4/R2 hi (first R3)	;  6

	vmovapd	xmm10, [srcreg+32]				;; I1 low			;   5
	vmovapd	xmm9, [srcreg+d1+32]				;; I2 low			;   5
	vinsertf128 ymm10, ymm10, [srcreg+d2+32], 1		;; I3/I1 low			;   6
	vinsertf128 ymm9, ymm9, [srcreg+d2+d1+32], 1		;; I4/I2 low			;   6

	yfmaddpd ymm3, ymm0, ymm15, ymm4	;; R2 + R4 (newer R2)				; 6-10		n 12
	yfmsubpd ymm0, ymm0, ymm15, ymm4	;; R2 - R4 (newer R4)				; 6-10		n 17
	vmovapd	xmm13, [srcreg+32][16]				;; I1 hi			;   7

	yfmaddpd ymm6, ymm1, ymm15, ymm5	;; R1 + R3 (newer R1)				; 7-11		n 12
	yfmsubpd ymm1, ymm1, ymm15, ymm5	;; R1 - R3 (newer R3)				; 7-11		n 15
	vmovapd	xmm14, [srcreg+d1+32][16]			;; I2 hi			;   7

	vshufpd	ymm7, ymm10, ymm9, 15		;; Shuffle I3/I1 and I4/I2 lows (first I2)	;  7
	vshufpd	ymm10, ymm10, ymm9, 0		;; Shuffle I3/I1 and I4/I2 lows (first I1)	;  8

	vinsertf128 ymm13, ymm13, [srcreg+d2+32][16], 1		;; I3/I1 hi			;   8
	vinsertf128 ymm14, ymm14, [srcreg+d2+d1+32][16], 1	;; I4/I2 hi			;   8
	vshufpd	ymm9, ymm13, ymm14, 15		;; Shuffle I3/I1 hi and I4/I2 hi (first I4)	;  9
	vshufpd	ymm13, ymm13, ymm14, 0		;; Shuffle I3/I1 hi and I4/I2 hi (first I3)	;  10

	yfmsubpd ymm14, ymm7, ymm15, ymm9	;; I2 - I4 (newer I4)				; 10-14		n 15
	yfmaddpd ymm7, ymm7, ymm15, ymm9	;; I2 + I4 (newer I2)				; 10-14		n 16
	L1prefetchw L1preg, L1pt - L1PREFETCH_DEST_NONE

	yfmaddpd ymm9, ymm10, ymm15, ymm13	;; I1 + I3 (newer I1)				; 11-15		n 16
	yfmsubpd ymm10, ymm10, ymm15, ymm13	;; I1 - I3 (newer I3)				; 11-15		n 17
	L1prefetchw L1preg+e1reg, L1pt - L1PREFETCH_DEST_NONE

	yfmsubpd ymm13, ymm6, ymm15, ymm3	;; R1 - R2 (final R2)				; 12-16		n 18
	yfmaddpd ymm6, ymm6, ymm15, ymm3	;; R1 + R2 (final R1)				; 12-16		n 19
	vmovapd	ymm2, [screg+128+32]		;; cosine for R2/I2
	L1prefetchw L1preg+2*e1reg, L1pt - L1PREFETCH_DEST_NONE

	yfmsubpd ymm3, ymm1, ymm15, ymm14	;; R3 - I4 (final R3)				; 15-19		n 20
	yfmaddpd ymm1, ymm1, ymm15, ymm14	;; R3 + I4 (final R4)				; 15-19		n 21
	vmovapd	ymm8, [screg+128]		;; sine for R2/I2
	L1prefetchw L1preg+e3reg, L1pt - L1PREFETCH_DEST_NONE

	yfmsubpd ymm14, ymm9, ymm15, ymm7	;; I1 - I2 (final I2)				; 16-20
	yfmaddpd ymm9, ymm9, ymm15, ymm7	;; I1 + I2 (final I1)				; 16-20
	vmovapd	ymm11, [screg+0+32]		;; cosine for R1/I1

	yfmaddpd ymm7, ymm10, ymm15, ymm0	;; I3 + R4 (final I3)				; 17-21
	yfmsubpd ymm10, ymm10, ymm15, ymm0	;; I3 - R4 (final I4)				; 17-21
	vmovapd	ymm12, [screg+0]		;; sine for R1/I1

	vmulpd	ymm0, ymm13, ymm2		;; A2 = R2 * cosine				; 18-22
	vmulpd	ymm13, ymm13, ymm8		;; B2 = R2 * sine				; 18-22
	vmovapd	ymm4, [screg+64+32]		;; cosine for R3/I3

	vmulpd	ymm5, ymm6, ymm11		;; A1 = R1 * cosine				; 19-23
	vmulpd	ymm6, ymm6, ymm12		;; B1 = R1 * sine				; 19-23
	bump	srcreg, srcinc

	yfnmaddpd ymm0, ymm14, ymm8, ymm0	;; A2 = A2 - I2 * sine (final R2)		; 23-27
	yfmaddpd ymm13, ymm14, ymm2, ymm13	;; B2 = I2 * cosine + B2 (final I2)		; 23-27

	vmovapd	ymm8, [screg+64]		;; sine for R3/I3
	vmulpd	ymm2, ymm3, ymm4		;; A3 = R3 * cosine				; 20-24
	vmulpd	ymm3, ymm3, ymm8		;; B3 = R3 * sine				; 20-24
	vmovapd	ymm14, [screg+192+32]		;; cosine for R4/I4

	yfnmaddpd ymm5, ymm9, ymm12, ymm5	;; A1 = A1 - I1 * sine (final R1)		; 24-28
	yfmaddpd ymm6, ymm9, ymm11, ymm6	;; B1 = I1 * cosine + B1 (final I1)		; 24-28

	vmovapd	ymm12, [screg+192]		;; sine for R4/I4
	vmulpd	ymm11, ymm1, ymm14		;; A4 = R4 * cosine				; 21-25
	vmulpd	ymm1, ymm1, ymm12		;; B4 = R4 * sine				; 21-25
	bump	screg, scinc

	yfnmaddpd ymm2, ymm7, ymm8, ymm2	;; A3 = A3 - I3 * sine (final R3)		; 25-29
	yfmaddpd ymm3, ymm7, ymm4, ymm3		;; B3 = I3 * cosine + B3 (final I3)		; 25-29

	yfnmaddpd ymm11, ymm10, ymm12, ymm11	;; A4 = A4 - I4 * sine (final R4)		; 26-30
	yfmaddpd ymm1, ymm10, ymm14, ymm1	;; B4 = I4 * cosine + B4 (final I4)		; 26-30

	ystore	[dstreg+e1reg], ymm0		;; Save R2					; 28
	ystore	[dstreg+e1reg+32], ymm13	;; Save I2					; 28+1
	ystore	[dstreg], ymm5			;; Save R1					; 29+1
	ystore	[dstreg+32], ymm6		;; Save I1					; 29+2
	ystore	[dstreg+2*e1reg], ymm2		;; Save R3					; 30+2
	ystore	[dstreg+2*e1reg+32], ymm3	;; Save I3					; 30+3
	ystore	[dstreg+e3reg], ymm11		;; Save R4					; 31+3
	ystore	[dstreg+e3reg+32], ymm1		;; Save I4					; 31+4
	bump	dstreg, dstinc
	bump	L1preg, dstinc

	ENDM
ENDIF
ENDIF

ENDIF

;;
;; ********************************* reduced sin/cos four-complex-unfft4 variants **************************************
;;

;; Used in last levels of pass 1.  Swizzling.
yr4_rsc_sg4cl_four_complex_unfft4_preload MACRO
	ENDM
yr4_rsc_sg4cl_four_complex_unfft4 MACRO srcreg,srcinc,d1,d2,dstreg,dstinc,e1,e2,screg,scinc,maxrpt,L1pt,L1pd
	vmovapd	ymm7, [screg+0+32]		;; cosine
	vmovapd	ymm0, [srcreg]			;; R1
	vmulpd	ymm6, ymm0, ymm7		;; A1 = R1 * cosine
	vmovapd	ymm5, [screg+128+32]		;; cosine
	vmovapd	ymm4, [srcreg+d1]		;; R2
	vmulpd	ymm2, ymm4, ymm5		;; A2 = R2 * cosine

	vmovapd	ymm1, [srcreg+32]		;; I1
	vmulpd	ymm7, ymm1, ymm7		;; B1 = I1 * cosine
	vmovapd	ymm3, [srcreg+d1+32]		;; I2
	vmulpd	ymm5, ymm3, ymm5		;; B2 = I2 * cosine

	vmulpd	ymm1, ymm1, [screg+0]		;; C1 = I1 * sine
	vmulpd	ymm3, ymm3, [screg+128]		;; C2 = I2 * sine
	vmulpd	ymm0, ymm0, [screg+0]		;; D1 = R1 * sine
	vmulpd	ymm4, ymm4, [screg+128]		;; D2 = R2 * sine

	vaddpd	ymm6, ymm6, ymm1		;; A1 = A1 + C1 (new R1)
	vaddpd	ymm2, ymm2, ymm3		;; A2 = A2 + C2 (new R2)
	vsubpd	ymm7, ymm7, ymm0		;; B1 = B1 - D1 (new I1)
	vsubpd	ymm5, ymm5, ymm4		;; B2 = B2 - D2 (new I2)

	vsubpd	ymm4, ymm6, ymm2		;; R1 - R2 (newer R2)
	vaddpd	ymm6, ymm6, ymm2		;; R1 + R2 (newer R1)

	ystore	[dstreg+32], ymm4		;; Save newer R2 temporarily
	ystore	[dstreg], ymm6			;; Save newer R1 temporarily

	vmovapd	ymm0, [screg+64+32]		;; cosine
	vmovapd	ymm4, [srcreg+d2]		;; R3
	vmulpd	ymm1, ymm4, ymm0		;; A3 = R3 * cosine
	vmovapd	ymm2, [srcreg+d2+32]		;; I3
	vmulpd	ymm0, ymm2, ymm0		;; B3 = I3 * cosine

	vmovapd	ymm6, [screg+64]		;; sine
	vmulpd	ymm2, ymm2, ymm6		;; C3 = I3 * sine
	vmulpd	ymm4, ymm4, ymm6		;; D3 = R3 * sine

	vaddpd	ymm1, ymm1, ymm2		;; A3 = A3 + C3 (new R3)
	vsubpd	ymm0, ymm0, ymm4		;; B3 = B3 - D3 (new I3)

	vmovapd	ymm2, [screg+192+32]		;; cosine
	vmovapd	ymm6, [srcreg+d2+d1]		;; R4
	vmulpd	ymm4, ymm6, ymm2		;; A4 = R4 * cosine
	vmovapd ymm3, [srcreg+d2+d1+32]		;; I4
	vmulpd	ymm2, ymm3, ymm2		;; B4 = I4 * cosine

	vmulpd	ymm3, ymm3, [screg+192]		;; C4 = I4 * sine
	vmulpd	ymm6, ymm6, [screg+192]		;; D4 = R4 * sine

	vaddpd	ymm4, ymm4, ymm3		;; A4 = A4 + C4 (new R4)
	vsubpd	ymm2, ymm2, ymm6		;; B4 = B4 - D4 (new I4)

	vsubpd	ymm6, ymm7, ymm5		;; I1 - I2 (newer I2)
	vaddpd	ymm7, ymm7, ymm5		;; I1 + I2 (newer I1)

	L1prefetch srcreg+L1pd, L1pt

	vsubpd	ymm3, ymm4, ymm1		;; R4 - R3 (newer I4)
	vaddpd	ymm4, ymm4, ymm1		;; R4 + R3 (newer R3)

	vsubpd	ymm5, ymm0, ymm2		;; I3 - I4 (newer R4)
	vaddpd	ymm0, ymm0, ymm2		;; I3 + I4 (newer I3)

	L1prefetch srcreg+d1+L1pd, L1pt

	vsubpd	ymm2, ymm6, ymm3		;; I2 - I4 (final I4)
	vaddpd	ymm6, ymm6, ymm3		;; I2 + I4 (final I2)

	vsubpd	ymm3, ymm7, ymm0		;; I1 - I3 (final I3)
	vaddpd	ymm7, ymm7, ymm0		;; I1 + I3 (final I1)

	L1prefetch srcreg+d2+L1pd, L1pt

	;; Swizzle using 4 shufpds and 4 perm2f128  (1 or 3 clocks each on port 5, throughput 1)

	vshufpd	ymm1, ymm7, ymm6, 0		;; Shuffle I1 and I2 to create I1/I2 low
	vshufpd	ymm7, ymm7, ymm6, 15		;; Shuffle I1 and I2 to create I1/I2 hi

	vshufpd	ymm0, ymm3, ymm2, 0		;; Shuffle I3 and I4 to create I3/I4 low
	vshufpd	ymm3, ymm3, ymm2, 15		;; Shuffle I3 and I4 to create I3/I4 hi

	ylow128s ymm2, ymm1, ymm0		;; Shuffle I1/I2 low and I3/I4 low (final I1)
	yhigh128s ymm1, ymm1, ymm0		;; Shuffle I1/I2 low and I3/I4 low (final I3)

	ylow128s ymm0, ymm7, ymm3		;; Shuffle I1/I2 hi and I3/I4 hi (final I2)
	yhigh128s ymm7, ymm7, ymm3		;; Shuffle I1/I2 hi and I3/I4 hi (final I4)

	vmovapd	ymm3, [dstreg]			;; Reload newer R1
	vmovapd	ymm6, [dstreg+32]		;; Reload newer R2

	ystore	[dstreg+32], ymm2		;; Save I1
	ystore	[dstreg+e2+32], ymm1		;; Save I3

	vsubpd	ymm2, ymm6, ymm5		;; R2 - R4 (final R4)
	vaddpd	ymm6, ymm6, ymm5		;; R2 + R4 (final R2)

	L1prefetch srcreg+d2+d1+L1pd, L1pt

	vsubpd	ymm5, ymm3, ymm4		;; R1 - R3 (final R3)
	vaddpd	ymm3, ymm3, ymm4		;; R1 + R3 (final R1)

	ystore	[dstreg+e1+32], ymm0		;; Save I2
	ystore	[dstreg+e2+e1+32], ymm7		;; Save I4

	;; Swizzle using 4 shufpds and 4 perm2f128  (1 or 3 clocks each on port 5, throughput 1)

	vshufpd	ymm0, ymm3, ymm6, 0		;; Shuffle R1 and R2 to create R1/R2 low
	vshufpd	ymm3, ymm3, ymm6, 15		;; Shuffle R1 and R2 to create R1/R2 hi

	vshufpd	ymm6, ymm5, ymm2, 0		;; Shuffle R3 and R4 to create R3/R4 low
	vshufpd	ymm5, ymm5, ymm2, 15		;; Shuffle R3 and R4 to create R3/R4 hi

	ylow128s ymm4, ymm0, ymm6		;; Shuffle R1/R2 low and R3/R4 low (final R1)
	yhigh128s ymm0, ymm0, ymm6		;; Shuffle R1/R2 low and R3/R4 low (final R3)

	ylow128s ymm6, ymm3, ymm5		;; Shuffle R1/R2 hi and R3/R4 hi (final R2)
	yhigh128s ymm3, ymm3, ymm5		;; Shuffle R1/R2 hi and R3/R4 hi (final R4)

	ystore	[dstreg], ymm4			;; Save R1
	ystore	[dstreg+e2], ymm0		;; Save R3
	ystore	[dstreg+e1], ymm6		;; Save R2
	ystore	[dstreg+e2+e1], ymm3		;; Save R4

	bump	srcreg, srcinc
	bump	dstreg, dstinc
	bump	screg, scinc
	ENDM

IFDEF X86_64

yr4_rsc_sg4cl_four_complex_unfft4_preload MACRO
	ENDM
yr4_rsc_sg4cl_four_complex_unfft4 MACRO srcreg,srcinc,d1,d2,dstreg,dstinc,e1,e2,screg,scinc,maxrpt,L1pt,L1pd
	vmovapd	ymm7, [screg+0+32]		;; cosine for R1/I1
	vmovapd	ymm0, [srcreg]			;; R1
	vmulpd	ymm6, ymm0, ymm7		;; A1 = R1 * cosine				;  1-5

	vmovapd	ymm5, [screg+128+32]		;; cosine for R2/I2
	vmovapd	ymm4, [srcreg+d1]		;; R2
	vmulpd	ymm2, ymm4, ymm5		;; A2 = R2 * cosine				;  2-6

	vmovapd	ymm1, [srcreg+32]		;; I1
	vmulpd	ymm7, ymm1, ymm7		;; B1 = I1 * cosine				;  3-7

	vmovapd	ymm3, [srcreg+d1+32]		;; I2
	vmulpd	ymm5, ymm3, ymm5		;; B2 = I2 * cosine				;  4-8

	vmovapd	ymm8, [screg+0]			;; sine for R1/I1
	vmulpd	ymm1, ymm1, ymm8		;; C1 = I1 * sine				;  5-9

	vmovapd	ymm9, [screg+128]		;; sine for R2/I2
	vmulpd	ymm3, ymm3, ymm9		;; C2 = I2 * sine				;  6-10
	vmovapd	ymm10, [screg+64+32]		;; cosine for R3/I3

	vmulpd	ymm0, ymm0, ymm8		;; D1 = R1 * sine				;  7-11
	vmovapd	ymm11, [srcreg+d2]		;; R3

	vmulpd	ymm4, ymm4, ymm9		;; D2 = R2 * sine				;  8-12
	vmovapd	ymm12, [srcreg+d2+32]		;; I3

	vmulpd	ymm13, ymm11, ymm10		;; A3 = R3 * cosine				;  9-13
	vmovapd	ymm14, [screg+192+32]		;; cosine for R4/I4

	vaddpd	ymm6, ymm6, ymm1		;; A1 = A1 + C1 (new R1)			; 10-12
	vmulpd	ymm10, ymm12, ymm10		;; B3 = I3 * cosine				;  10-14
	vmovapd	ymm15, [srcreg+d2+d1]		;; R4

	vaddpd	ymm2, ymm2, ymm3		;; A2 = A2 + C2 (new R2)			; 11-13
	vmulpd	ymm3, ymm15, ymm14		;; A4 = R4 * cosine				;  11-15
	vmovapd ymm8, [srcreg+d2+d1+32]		;; I4

	vsubpd	ymm7, ymm7, ymm0		;; B1 = B1 - D1 (new I1)			; 12-14
	vmulpd	ymm14, ymm8, ymm14		;; B4 = I4 * cosine				;  12-16
	vmovapd	ymm9, [screg+64]		;; sine for R3/I3

	vsubpd	ymm5, ymm5, ymm4		;; B2 = B2 - D2 (new I2)			; 13-15
	vmulpd	ymm12, ymm12, ymm9		;; C3 = I3 * sine				;  13-17
	vmovapd	ymm1, [screg+192]		;; sine for R4/I4

	vsubpd	ymm4, ymm6, ymm2		;; R1 - R2 (newer R2)				; 14-16
	vmulpd	ymm8, ymm8, ymm1		;; C4 = I4 * sine				;  14-18

	vaddpd	ymm6, ymm6, ymm2		;; R1 + R2 (newer R1)				; 15-17
	vmulpd	ymm11, ymm11, ymm9		;; D3 = R3 * sine				;  15-19

	vsubpd	ymm9, ymm7, ymm5		;; I1 - I2 (newer I2)				; 16-18
	vmulpd	ymm15, ymm15, ymm1		;; D4 = R4 * sine				;  16-20

	vaddpd	ymm7, ymm7, ymm5		;; I1 + I2 (newer I1)				; 17-19
	L1prefetch srcreg+L1pd, L1pt

	vaddpd	ymm13, ymm13, ymm12		;; A3 = A3 + C3 (new R3)			; 18-20

	vaddpd	ymm3, ymm3, ymm8		;; A4 = A4 + C4 (new R4)			; 19-21

	vsubpd	ymm10, ymm10, ymm11		;; B3 = B3 - D3 (new I3)			; 20-22

	vsubpd	ymm14, ymm14, ymm15		;; B4 = B4 - D4 (new I4)			; 21-23
	L1prefetch srcreg+d1+L1pd, L1pt

	vsubpd	ymm15, ymm3, ymm13		;; R4 - R3 (newer I4)				; 22-24

	vaddpd	ymm3, ymm3, ymm13		;; R4 + R3 (newer R3)				; 23-25

	vaddpd	ymm13, ymm10, ymm14		;; I3 + I4 (newer I3)				; 24-26

	vsubpd	ymm11, ymm9, ymm15		;; I2 - I4 (final I4)				; 25-27
	L1prefetch srcreg+d2+L1pd, L1pt

	vaddpd	ymm9, ymm9, ymm15		;; I2 + I4 (final I2)				; 26-28

	vsubpd	ymm15, ymm7, ymm13		;; I1 - I3 (final I3)				; 27-29

	vsubpd	ymm10, ymm10, ymm14		;; I3 - I4 (newer R4)				; 28-30

	vaddpd	ymm7, ymm7, ymm13		;; I1 + I3 (final I1)				; 29-31
	L1prefetch srcreg+d2+d1+L1pd, L1pt

	vshufpd	ymm13, ymm15, ymm11, 0		;; Shuffle I3 and I4 to create I3/I4 low	; 30
	vsubpd	ymm14, ymm6, ymm3		;; R1 - R3 (final R3)				; 30-32

	vshufpd	ymm15, ymm15, ymm11, 15		;; Shuffle I3 and I4 to create I3/I4 hi		; 31
	vsubpd	ymm11, ymm4, ymm10		;; R2 - R4 (final R4)				; 31-33

	vshufpd	ymm8, ymm7, ymm9, 0		;; Shuffle I1 and I2 to create I1/I2 low	; 32
	vaddpd	ymm6, ymm6, ymm3		;; R1 + R3 (final R1)				; 32-34

	vshufpd	ymm7, ymm7, ymm9, 15		;; Shuffle I1 and I2 to create I1/I2 hi		; 33
	vaddpd	ymm4, ymm4, ymm10		;; R2 + R4 (final R2)				; 33-35

	vshufpd	ymm10, ymm14, ymm11, 0		;; Shuffle R3 and R4 to create R3/R4 low	; 34
	vshufpd	ymm14, ymm14, ymm11, 15		;; Shuffle R3 and R4 to create R3/R4 hi		; 35
	vshufpd	ymm11, ymm6, ymm4, 0		;; Shuffle R1 and R2 to create R1/R2 low	; 36
	vshufpd	ymm6, ymm6, ymm4, 15		;; Shuffle R1 and R2 to create R1/R2 hi		; 37

	ylow128s ymm4, ymm8, ymm13		;; Shuffle I1/I2 low and I3/I4 low (final I1)	; 38-39
	yhigh128s ymm8, ymm8, ymm13		;; Shuffle I1/I2 low and I3/I4 low (final I3)	; 39-40
	ystore	[dstreg+32], ymm4		;; Save I1					; 40
	ylow128s ymm13, ymm7, ymm15		;; Shuffle I1/I2 hi and I3/I4 hi (final I2)	; 40-41
	ystore	[dstreg+e2+32], ymm8		;; Save I3					; 41
	yhigh128s ymm7, ymm7, ymm15		;; Shuffle I1/I2 hi and I3/I4 hi (final I4)	; 41-42
	ystore	[dstreg+e1+32], ymm13		;; Save I2					; 42
	ylow128s ymm15, ymm11, ymm10		;; Shuffle R1/R2 low and R3/R4 low (final R1)	; 42-43
	ystore	[dstreg+e2+e1+32], ymm7		;; Save I4					; 43
	yhigh128s ymm11, ymm11, ymm10		;; Shuffle R1/R2 low and R3/R4 low (final R3)	; 43-44
	ystore	[dstreg], ymm15			;; Save R1					; 44
	ylow128s ymm10, ymm6, ymm14		;; Shuffle R1/R2 hi and R3/R4 hi (final R2)	; 44-45
	ystore	[dstreg+e2], ymm11		;; Save R3					; 45
	yhigh128s ymm6, ymm6, ymm14		;; Shuffle R1/R2 hi and R3/R4 hi (final R4)	; 45-46
	ystore	[dstreg+e1], ymm10		;; Save R2					; 46
	ystore	[dstreg+e2+e1], ymm6		;; Save R4					; 47

	bump	srcreg, srcinc
	bump	dstreg, dstinc
	bump	screg, scinc
	ENDM

;; Haswell FMA3 version

IF (@INSTR(,%yarch,<FMA3>) NE 0)

yr4_rsc_sg4cl_four_complex_unfft4_preload MACRO
	ENDM
yr4_rsc_sg4cl_four_complex_unfft4 MACRO srcreg,srcinc,d1,d2,dstreg,dstinc,e1,e2,screg,scinc,maxrpt,L1pt,L1pd
	vmovapd	ymm0, [screg+0+32]		;; cosine for R1/I1
	vmovapd	ymm1, [srcreg+32]		;; I1
	vmulpd	ymm2, ymm1, ymm0		;; B1 = I1 * cosine				; 1-5
	vmovapd	ymm3, [screg+128+32]		;; cosine for R2/I2
	vmovapd	ymm4, [srcreg+d1+32]		;; I2
	vmulpd	ymm5, ymm4, ymm3		;; B2 = I2 * cosine				; 1-5

	vmovapd	ymm6, [screg+64+32]		;; cosine for R3/I3
	vmovapd	ymm7, [srcreg+d2]		;; R3
	vmulpd	ymm8, ymm7, ymm6		;; A3 = R3 * cosine				; 2-6
	vmovapd	ymm9, [screg+192+32]		;; cosine for R4/I4
	vmovapd	ymm10, [srcreg+d2+d1]		;; R4
	vmulpd	ymm11, ymm10, ymm9		;; A4 = R4 * cosine				; 2-6

	vmovapd	ymm12, [srcreg+d2+32]		;; I3
	vmulpd	ymm6, ymm12, ymm6		;; B3 = I3 * cosine				; 3-7
	vmovapd ymm13, [srcreg+d2+d1+32]	;; I4
	vmulpd	ymm9, ymm13, ymm9		;; B4 = I4 * cosine				; 3-7

	vmovapd	ymm14, [srcreg]			;; R1
	vmulpd	ymm0, ymm14, ymm0		;; A1 = R1 * cosine				; 4-8
	vmovapd	ymm15, [srcreg+d1]		;; R2
	vmulpd	ymm3, ymm15, ymm3		;; A2 = R2 * cosine				; 4-8

	yfnmaddpd ymm14, ymm14, [screg+0], ymm2	;; B1 = B1 - R1 * sine (new I1)			; 6-10
	vmovapd	ymm2, [screg+128]		;; sine for R2/I2
	yfnmaddpd ymm15, ymm15, ymm2, ymm5	;; B2 = B2 - R2 * sine (new I2)			; 6-10

	vmovapd	ymm5, [screg+64]		;; sine for R3/I3
	yfmaddpd ymm12, ymm12, ymm5, ymm8	;; A3 = A3 + I3 * sine (new R3)			; 7-11
	vmovapd	ymm8, [screg+192]		;; sine for R4/I4
	yfmaddpd ymm13, ymm13, ymm8, ymm11	;; A4 = A4 + I4 * sine (new R4)			; 7-11

	yfnmaddpd ymm7, ymm7, ymm5, ymm6	;; B3 = B3 - R3 * sine (new I3)			; 8-12
	yfnmaddpd ymm10, ymm10, ymm8, ymm9	;; B4 = B4 - R4 * sine (new I4)			; 8-12
	vmovapd	ymm11, YMM_ONE

	yfmaddpd ymm1, ymm1, [screg+0], ymm0	;; A1 = A1 + I1 * sine (new R1)			; 9-13
	yfmaddpd ymm4, ymm4, ymm2, ymm3		;; A2 = A2 + I2 * sine (new R2)			; 9-13
	bump	screg, scinc

	L1prefetch srcreg+L1pd, L1pt
	L1prefetch srcreg+d1+L1pd, L1pt

	vsubpd	ymm3, ymm14, ymm15		;; I1 - I2 (newer I2)				; 11-13		n 15
	yfmaddpd ymm14, ymm14, ymm11, ymm15	;; I1 + I2 (newer I1)				; 11-15		n 16

	vsubpd	ymm15, ymm13, ymm12		;; R4 - R3 (newer I4)				; 12-14		n 15
	yfmaddpd ymm13, ymm13, ymm11, ymm12	;; R4 + R3 (newer R3)				; 12-16		n 19

	vaddpd	ymm12, ymm7, ymm10		;; I3 + I4 (newer I3)				; 13-15		n 16
	yfmsubpd ymm7, ymm7, ymm11, ymm10	;; I3 - I4 (newer R4)				; 13-17		n 20

	vaddpd	ymm10, ymm1, ymm4		;; R1 + R2 (newer R1)				; 14-16		n 19
	yfmsubpd ymm1, ymm1, ymm11, ymm4	;; R1 - R2 (newer R2)				; 14-18		n 20

	vsubpd	ymm4, ymm3, ymm15		;; I2 - I4 (final I4)				; 15-17		n 19
	L1prefetch srcreg+d2+L1pd, L1pt
	vsubpd	ymm2, ymm14, ymm12		;; I1 - I3 (final I3)				; 16-18		n 19

	vaddpd	ymm3, ymm3, ymm15		;; I2 + I4 (final I2)				; 17-19		n 21
	vaddpd	ymm14, ymm14, ymm12		;; I1 + I3 (final I1)				; 18-20		n 21
	L1prefetch srcreg+d2+d1+L1pd, L1pt
	bump	srcreg, srcinc

	vshufpd	ymm12, ymm2, ymm4, 0		;; Shuffle I3 and I4 to create I3/I4 low	; 19
	vsubpd	ymm15, ymm10, ymm13		;; R1 - R3 (final R3)				; 19-21		n 23

	vshufpd	ymm2, ymm2, ymm4, 15		;; Shuffle I3 and I4 to create I3/I4 hi		; 20
	vsubpd	ymm0, ymm1, ymm7		;; R2 - R4 (final R4)				; 20-22		n 23

	vshufpd	ymm4, ymm14, ymm3, 0		;; Shuffle I1 and I2 to create I1/I2 low	; 21
	vaddpd	ymm10, ymm10, ymm13		;; R1 + R3 (final R1)				; 21-23		n 25

	vshufpd	ymm14, ymm14, ymm3, 15		;; Shuffle I1 and I2 to create I1/I2 hi		; 22
	vaddpd	ymm1, ymm1, ymm7		;; R2 + R4 (final R2)				; 22-24		n 25

	vshufpd	ymm7, ymm15, ymm0, 0		;; Shuffle R3 and R4 to create R3/R4 low	; 23
	vshufpd	ymm15, ymm15, ymm0, 15		;; Shuffle R3 and R4 to create R3/R4 hi		; 24
	vshufpd	ymm0, ymm10, ymm1, 0		;; Shuffle R1 and R2 to create R1/R2 low	; 25
	vshufpd	ymm10, ymm10, ymm1, 15		;; Shuffle R1 and R2 to create R1/R2 hi		; 26

	ylow128s ymm1, ymm4, ymm12		;; Shuffle I1/I2 low and I3/I4 low (final I1)	; 27-28
	yhigh128s ymm4, ymm4, ymm12		;; Shuffle I1/I2 low and I3/I4 low (final I3)	; 28-29
	ystore	[dstreg+32], ymm1		;; Save I1					; 29
	ylow128s ymm12, ymm14, ymm2		;; Shuffle I1/I2 hi and I3/I4 hi (final I2)	; 29-30
	ystore	[dstreg+e2+32], ymm4		;; Save I3					; 30
	yhigh128s ymm14, ymm14, ymm2		;; Shuffle I1/I2 hi and I3/I4 hi (final I4)	; 30-31
	ystore	[dstreg+e1+32], ymm12		;; Save I2					; 31
	ylow128s ymm2, ymm0, ymm7		;; Shuffle R1/R2 low and R3/R4 low (final R1)	; 31-32
	ystore	[dstreg+e2+e1+32], ymm14	;; Save I4					; 32
	yhigh128s ymm0, ymm0, ymm7		;; Shuffle R1/R2 low and R3/R4 low (final R3)	; 32-33
	ystore	[dstreg], ymm2			;; Save R1					; 33
	ylow128s ymm7, ymm10, ymm15		;; Shuffle R1/R2 hi and R3/R4 hi (final R2)	; 33-34
	ystore	[dstreg+e2], ymm0		;; Save R3					; 34
	yhigh128s ymm10, ymm10, ymm15		;; Shuffle R1/R2 hi and R3/R4 hi (final R4)	; 34-35
	ystore	[dstreg+e1], ymm7		;; Save R2					; 35
	ystore	[dstreg+e2+e1], ymm10		;; Save R4					; 36
	bump	dstreg, dstinc
	ENDM

ENDIF

ENDIF

; These versions use registers for distances between blocks.  This lets us share pass1 code.

;; Used in last levels of pass 1.  Swizzling.
yr4_rsc_sg4clreg_four_complex_unfft4_preload MACRO
	ENDM
yr4_rsc_sg4clreg_four_complex_unfft4 MACRO srcreg,srcinc,d1reg,d3reg,dstreg,dstinc,e1,e2,screg,scinc,maxrpt,L1pt,L1preg
	NOT IMPLMENTED IN 32-BIT
	ENDM

IFDEF X86_64

yr4_rsc_sg4clreg_four_complex_unfft4_preload MACRO
	ENDM
yr4_rsc_sg4clreg_four_complex_unfft4 MACRO srcreg,srcinc,d1reg,d3reg,dstreg,dstinc,e1,e2,screg,scinc,maxrpt,L1pt,L1preg
	vmovapd	ymm7, [screg+0+32]		;; cosine for R1/I1
	vmovapd	ymm0, [srcreg]			;; R1
	vmulpd	ymm6, ymm0, ymm7		;; A1 = R1 * cosine				;  1-5

	vmovapd	ymm5, [screg+128+32]		;; cosine for R2/I2
	vmovapd	ymm4, [srcreg+d1reg]		;; R2
	vmulpd	ymm2, ymm4, ymm5		;; A2 = R2 * cosine				;  2-6

	vmovapd	ymm1, [srcreg+32]		;; I1
	vmulpd	ymm7, ymm1, ymm7		;; B1 = I1 * cosine				;  3-7

	vmovapd	ymm3, [srcreg+d1reg+32]		;; I2
	vmulpd	ymm5, ymm3, ymm5		;; B2 = I2 * cosine				;  4-8

	vmovapd	ymm8, [screg+0]			;; sine for R1/I1
	vmulpd	ymm1, ymm1, ymm8		;; C1 = I1 * sine				;  5-9

	vmovapd	ymm9, [screg+128]		;; sine for R2/I2
	vmulpd	ymm3, ymm3, ymm9		;; C2 = I2 * sine				;  6-10
	vmovapd	ymm10, [screg+64+32]		;; cosine for R3/I3

	vmulpd	ymm0, ymm0, ymm8		;; D1 = R1 * sine				;  7-11
	vmovapd	ymm11, [srcreg+2*d1reg]		;; R3

	vmulpd	ymm4, ymm4, ymm9		;; D2 = R2 * sine				;  8-12
	vmovapd	ymm12, [srcreg+2*d1reg+32]	;; I3

	vmulpd	ymm13, ymm11, ymm10		;; A3 = R3 * cosine				;  9-13
	vmovapd	ymm14, [screg+192+32]		;; cosine for R4/I4

	vaddpd	ymm6, ymm6, ymm1		;; A1 = A1 + C1 (new R1)			; 10-12
	vmulpd	ymm10, ymm12, ymm10		;; B3 = I3 * cosine				;  10-14
	vmovapd	ymm15, [srcreg+d3reg]		;; R4

	vaddpd	ymm2, ymm2, ymm3		;; A2 = A2 + C2 (new R2)			; 11-13
	vmulpd	ymm3, ymm15, ymm14		;; A4 = R4 * cosine				;  11-15
	vmovapd ymm8, [srcreg+d3reg+32]		;; I4

	vsubpd	ymm7, ymm7, ymm0		;; B1 = B1 - D1 (new I1)			; 12-14
	vmulpd	ymm14, ymm8, ymm14		;; B4 = I4 * cosine				;  12-16
	vmovapd	ymm9, [screg+64]		;; sine for R3/I3

	vsubpd	ymm5, ymm5, ymm4		;; B2 = B2 - D2 (new I2)			; 13-15
	vmulpd	ymm12, ymm12, ymm9		;; C3 = I3 * sine				;  13-17
	vmovapd	ymm1, [screg+192]		;; sine for R4/I4

	vsubpd	ymm4, ymm6, ymm2		;; R1 - R2 (newer R2)				; 14-16
	vmulpd	ymm8, ymm8, ymm1		;; C4 = I4 * sine				;  14-18

	vaddpd	ymm6, ymm6, ymm2		;; R1 + R2 (newer R1)				; 15-17
	vmulpd	ymm11, ymm11, ymm9		;; D3 = R3 * sine				;  15-19

	vsubpd	ymm9, ymm7, ymm5		;; I1 - I2 (newer I2)				; 16-18
	vmulpd	ymm15, ymm15, ymm1		;; D4 = R4 * sine				;  16-20

	vaddpd	ymm7, ymm7, ymm5		;; I1 + I2 (newer I1)				; 17-19
	L1prefetch L1preg, L1pt

	vaddpd	ymm13, ymm13, ymm12		;; A3 = A3 + C3 (new R3)			; 18-20

	vaddpd	ymm3, ymm3, ymm8		;; A4 = A4 + C4 (new R4)			; 19-21

	vsubpd	ymm10, ymm10, ymm11		;; B3 = B3 - D3 (new I3)			; 20-22

	vsubpd	ymm14, ymm14, ymm15		;; B4 = B4 - D4 (new I4)			; 21-23
	L1prefetch L1preg+d1reg, L1pt

	vsubpd	ymm15, ymm3, ymm13		;; R4 - R3 (newer I4)				; 22-24

	vaddpd	ymm3, ymm3, ymm13		;; R4 + R3 (newer R3)				; 23-25

	vaddpd	ymm13, ymm10, ymm14		;; I3 + I4 (newer I3)				; 24-26

	vsubpd	ymm11, ymm9, ymm15		;; I2 - I4 (final I4)				; 25-27
	L1prefetch L1preg+2*d1reg, L1pt

	vaddpd	ymm9, ymm9, ymm15		;; I2 + I4 (final I2)				; 26-28

	vsubpd	ymm15, ymm7, ymm13		;; I1 - I3 (final I3)				; 27-29

	vsubpd	ymm10, ymm10, ymm14		;; I3 - I4 (newer R4)				; 28-30

	vaddpd	ymm7, ymm7, ymm13		;; I1 + I3 (final I1)				; 29-31
	L1prefetch L1preg+d3reg, L1pt

	vshufpd	ymm13, ymm15, ymm11, 0		;; Shuffle I3 and I4 to create I3/I4 low	; 30
	vsubpd	ymm14, ymm6, ymm3		;; R1 - R3 (final R3)				; 30-32

	vshufpd	ymm15, ymm15, ymm11, 15		;; Shuffle I3 and I4 to create I3/I4 hi		; 31
	vsubpd	ymm11, ymm4, ymm10		;; R2 - R4 (final R4)				; 31-33

	vshufpd	ymm8, ymm7, ymm9, 0		;; Shuffle I1 and I2 to create I1/I2 low	; 32
	vaddpd	ymm6, ymm6, ymm3		;; R1 + R3 (final R1)				; 32-34

	vshufpd	ymm7, ymm7, ymm9, 15		;; Shuffle I1 and I2 to create I1/I2 hi		; 33
	vaddpd	ymm4, ymm4, ymm10		;; R2 + R4 (final R2)				; 33-35

	vshufpd	ymm10, ymm14, ymm11, 0		;; Shuffle R3 and R4 to create R3/R4 low	; 34
	vshufpd	ymm14, ymm14, ymm11, 15		;; Shuffle R3 and R4 to create R3/R4 hi		; 35
	vshufpd	ymm11, ymm6, ymm4, 0		;; Shuffle R1 and R2 to create R1/R2 low	; 36
	vshufpd	ymm6, ymm6, ymm4, 15		;; Shuffle R1 and R2 to create R1/R2 hi		; 37

	ylow128s ymm4, ymm8, ymm13		;; Shuffle I1/I2 low and I3/I4 low (final I1)	; 38-39
	yhigh128s ymm8, ymm8, ymm13		;; Shuffle I1/I2 low and I3/I4 low (final I3)	; 39-40
	ystore	[dstreg+32], ymm4		;; Save I1					; 40
	ylow128s ymm13, ymm7, ymm15		;; Shuffle I1/I2 hi and I3/I4 hi (final I2)	; 40-41
	ystore	[dstreg+e2+32], ymm8		;; Save I3					; 41
	yhigh128s ymm7, ymm7, ymm15		;; Shuffle I1/I2 hi and I3/I4 hi (final I4)	; 41-42
	ystore	[dstreg+e1+32], ymm13		;; Save I2					; 42
	ylow128s ymm15, ymm11, ymm10		;; Shuffle R1/R2 low and R3/R4 low (final R1)	; 42-43
	ystore	[dstreg+e2+e1+32], ymm7		;; Save I4					; 43
	yhigh128s ymm11, ymm11, ymm10		;; Shuffle R1/R2 low and R3/R4 low (final R3)	; 43-44
	ystore	[dstreg], ymm15			;; Save R1					; 44
	ylow128s ymm10, ymm6, ymm14		;; Shuffle R1/R2 hi and R3/R4 hi (final R2)	; 44-45
	ystore	[dstreg+e2], ymm11		;; Save R3					; 45
	yhigh128s ymm6, ymm6, ymm14		;; Shuffle R1/R2 hi and R3/R4 hi (final R4)	; 45-46
	ystore	[dstreg+e1], ymm10		;; Save R2					; 46
	ystore	[dstreg+e2+e1], ymm6		;; Save R4					; 47

	bump	srcreg, srcinc
	bump	dstreg, dstinc
	bump	screg, scinc
	bump	L1preg, srcinc
	ENDM

;; Haswell FMA3 version

IF (@INSTR(,%yarch,<FMA3>) NE 0)

yr4_rsc_sg4clreg_four_complex_unfft4_preload MACRO
	ENDM
yr4_rsc_sg4clreg_four_complex_unfft4 MACRO srcreg,srcinc,d1reg,d3reg,dstreg,dstinc,e1,e2,screg,scinc,maxrpt,L1pt,L1preg
	vmovapd	ymm0, [screg+0+32]		;; cosine for R1/I1
	vmovapd	ymm1, [srcreg+32]		;; I1
	vmulpd	ymm2, ymm1, ymm0		;; B1 = I1 * cosine				; 1-5
	vmovapd	ymm3, [screg+128+32]		;; cosine for R2/I2
	vmovapd	ymm4, [srcreg+d1reg+32]		;; I2
	vmulpd	ymm5, ymm4, ymm3		;; B2 = I2 * cosine				; 1-5

	vmovapd	ymm6, [screg+64+32]		;; cosine for R3/I3
	vmovapd	ymm7, [srcreg+2*d1reg]		;; R3
	vmulpd	ymm8, ymm7, ymm6		;; A3 = R3 * cosine				; 2-6
	vmovapd	ymm9, [screg+192+32]		;; cosine for R4/I4
	vmovapd	ymm10, [srcreg+d3reg]		;; R4
	vmulpd	ymm11, ymm10, ymm9		;; A4 = R4 * cosine				; 2-6

	vmovapd	ymm12, [srcreg+2*d1reg+32]	;; I3
	vmulpd	ymm6, ymm12, ymm6		;; B3 = I3 * cosine				; 3-7
	vmovapd ymm13, [srcreg+d3reg+32]	;; I4
	vmulpd	ymm9, ymm13, ymm9		;; B4 = I4 * cosine				; 3-7

	vmovapd	ymm14, [srcreg]			;; R1
	vmulpd	ymm0, ymm14, ymm0		;; A1 = R1 * cosine				; 4-8
	vmovapd	ymm15, [srcreg+d1reg]		;; R2
	vmulpd	ymm3, ymm15, ymm3		;; A2 = R2 * cosine				; 4-8

	yfnmaddpd ymm14, ymm14, [screg+0], ymm2	;; B1 = B1 - R1 * sine (new I1)			; 6-10
	vmovapd	ymm2, [screg+128]		;; sine for R2/I2
	yfnmaddpd ymm15, ymm15, ymm2, ymm5	;; B2 = B2 - R2 * sine (new I2)			; 6-10

	vmovapd	ymm5, [screg+64]		;; sine for R3/I3
	yfmaddpd ymm12, ymm12, ymm5, ymm8	;; A3 = A3 + I3 * sine (new R3)			; 7-11
	vmovapd	ymm8, [screg+192]		;; sine for R4/I4
	yfmaddpd ymm13, ymm13, ymm8, ymm11	;; A4 = A4 + I4 * sine (new R4)			; 7-11

	yfnmaddpd ymm7, ymm7, ymm5, ymm6	;; B3 = B3 - R3 * sine (new I3)			; 8-12
	yfnmaddpd ymm10, ymm10, ymm8, ymm9	;; B4 = B4 - R4 * sine (new I4)			; 8-12
	vmovapd	ymm11, YMM_ONE

	yfmaddpd ymm1, ymm1, [screg+0], ymm0	;; A1 = A1 + I1 * sine (new R1)			; 9-13
	yfmaddpd ymm4, ymm4, ymm2, ymm3		;; A2 = A2 + I2 * sine (new R2)			; 9-13
	bump	screg, scinc

	L1prefetch L1preg, L1pt
	L1prefetch L1preg+d1reg, L1pt

	vsubpd	ymm3, ymm14, ymm15		;; I1 - I2 (newer I2)				; 11-13		n 15
	yfmaddpd ymm14, ymm14, ymm11, ymm15	;; I1 + I2 (newer I1)				; 11-15		n 16

	vsubpd	ymm15, ymm13, ymm12		;; R4 - R3 (newer I4)				; 12-14		n 15
	yfmaddpd ymm13, ymm13, ymm11, ymm12	;; R4 + R3 (newer R3)				; 12-16		n 19

	vaddpd	ymm12, ymm7, ymm10		;; I3 + I4 (newer I3)				; 13-15		n 16
	yfmsubpd ymm7, ymm7, ymm11, ymm10	;; I3 - I4 (newer R4)				; 13-17		n 20

	vaddpd	ymm10, ymm1, ymm4		;; R1 + R2 (newer R1)				; 14-16		n 19
	yfmsubpd ymm1, ymm1, ymm11, ymm4	;; R1 - R2 (newer R2)				; 14-18		n 20

	vsubpd	ymm4, ymm3, ymm15		;; I2 - I4 (final I4)				; 15-17		n 19
	L1prefetch L1preg+2*d1reg, L1pt
	vsubpd	ymm2, ymm14, ymm12		;; I1 - I3 (final I3)				; 16-18		n 19

	vaddpd	ymm3, ymm3, ymm15		;; I2 + I4 (final I2)				; 17-19		n 21
	vaddpd	ymm14, ymm14, ymm12		;; I1 + I3 (final I1)				; 18-20		n 21
	L1prefetch L1preg+d3reg, L1pt
	bump	srcreg, srcinc

	vshufpd	ymm12, ymm2, ymm4, 0		;; Shuffle I3 and I4 to create I3/I4 low	; 19
	vsubpd	ymm15, ymm10, ymm13		;; R1 - R3 (final R3)				; 19-21		n 23

	vshufpd	ymm2, ymm2, ymm4, 15		;; Shuffle I3 and I4 to create I3/I4 hi		; 20
	vsubpd	ymm0, ymm1, ymm7		;; R2 - R4 (final R4)				; 20-22		n 23

	vshufpd	ymm4, ymm14, ymm3, 0		;; Shuffle I1 and I2 to create I1/I2 low	; 21
	vaddpd	ymm10, ymm10, ymm13		;; R1 + R3 (final R1)				; 21-23		n 25

	vshufpd	ymm14, ymm14, ymm3, 15		;; Shuffle I1 and I2 to create I1/I2 hi		; 22
	vaddpd	ymm1, ymm1, ymm7		;; R2 + R4 (final R2)				; 22-24		n 25
	bump	L1preg, srcinc

	vshufpd	ymm7, ymm15, ymm0, 0		;; Shuffle R3 and R4 to create R3/R4 low	; 23
	vshufpd	ymm15, ymm15, ymm0, 15		;; Shuffle R3 and R4 to create R3/R4 hi		; 24
	vshufpd	ymm0, ymm10, ymm1, 0		;; Shuffle R1 and R2 to create R1/R2 low	; 25
	vshufpd	ymm10, ymm10, ymm1, 15		;; Shuffle R1 and R2 to create R1/R2 hi		; 26

	ylow128s ymm1, ymm4, ymm12		;; Shuffle I1/I2 low and I3/I4 low (final I1)	; 27-28
	yhigh128s ymm4, ymm4, ymm12		;; Shuffle I1/I2 low and I3/I4 low (final I3)	; 28-29
	ystore	[dstreg+32], ymm1		;; Save I1					; 29
	ylow128s ymm12, ymm14, ymm2		;; Shuffle I1/I2 hi and I3/I4 hi (final I2)	; 29-30
	ystore	[dstreg+e2+32], ymm4		;; Save I3					; 30
	yhigh128s ymm14, ymm14, ymm2		;; Shuffle I1/I2 hi and I3/I4 hi (final I4)	; 30-31
	ystore	[dstreg+e1+32], ymm12		;; Save I2					; 31
	ylow128s ymm2, ymm0, ymm7		;; Shuffle R1/R2 low and R3/R4 low (final R1)	; 31-32
	ystore	[dstreg+e2+e1+32], ymm14	;; Save I4					; 32
	yhigh128s ymm0, ymm0, ymm7		;; Shuffle R1/R2 low and R3/R4 low (final R3)	; 32-33
	ystore	[dstreg], ymm2			;; Save R1					; 33
	ylow128s ymm7, ymm10, ymm15		;; Shuffle R1/R2 hi and R3/R4 hi (final R2)	; 33-34
	ystore	[dstreg+e2], ymm0		;; Save R3					; 34
	yhigh128s ymm10, ymm10, ymm15		;; Shuffle R1/R2 hi and R3/R4 hi (final R4)	; 34-35
	ystore	[dstreg+e1], ymm7		;; Save R2					; 35
	ystore	[dstreg+e2+e1], ymm10		;; Save R4					; 36
	bump	dstreg, dstinc
	ENDM

ENDIF

ENDIF


;;
;; ********************************** reduced sin/cos eight-reals-fft4 variants ***************************************
;;
;; These macros are used in the last levels of pass 1.  Four sin/cos multipliers are needed to
;; finish off the partial sin/cos multiplies that were done in the first levels of pass 1.

;; Used in last levels of pass 1.  Swizzling.
;; In this particular case, we know that the sin/cos multipliers for R1/I1 are one.  Thus
;; we can get by with only 3 sin/cos multiplies.
yr4_rsc_sg4cl_2sc_eight_reals_fft4_preload MACRO
	ENDM
yr4_rsc_sg4cl_2sc_eight_reals_fft4 MACRO srcreg,srcinc,d1,d2,dstreg,dstinc,e1,e2,screg1,scinc1,screg2,scinc2,maxrpt,L1pt,L1pd
	vmovapd	ymm1, [srcreg]			;; R1
	vmovapd	ymm7, [srcreg+d1]		;; R2
	vshufpd	ymm0, ymm1, ymm7, 15		;; Shuffle R1 and R2 to create R1/R2 hi			; 1
	vshufpd	ymm1, ymm1, ymm7, 0		;; Shuffle R1 and R2 to create R1/R2 low		; 2

	vmovapd	ymm2, [srcreg+d2]		;; R3
	vmovapd	ymm7, [srcreg+d2+d1]		;; R4
	vshufpd	ymm3, ymm2, ymm7, 15		;; Shuffle R3 and R4 to create R3/R4 hi			; 3
	vshufpd	ymm2, ymm2, ymm7, 0		;; Shuffle R3 and R4 to create R3/R4 low		; 4

	ylow128s ymm4, ymm0, ymm3		;; Shuffle R1/R2 hi and R3/R4 hi (new R2)		; 5-6
	yhigh128s ymm0, ymm0, ymm3		;; Shuffle R1/R2 hi and R3/R4 hi (new R4)		; 6-7

	ylow128s ymm3, ymm1, ymm2		;; Shuffle R1/R2 low and R3/R4 low (new R1)		; 7-8
	yhigh128s ymm1, ymm1, ymm2		;; Shuffle R1/R2 low and R3/R4 low (new R3)		; 8-9

	ystore	[dstreg], ymm1			;; Temporarily save new R3

	vmovapd	ymm6, [srcreg+32]		;; R5
	vmovapd	ymm2, [srcreg+d1+32]		;; R6
	vshufpd	ymm5, ymm6, ymm2, 15		;; Shuffle R5 and R6 to create R5/R6 hi
	vshufpd	ymm6, ymm6, ymm2, 0		;; Shuffle R5 and R6 to create R5/R6 low

	vmovapd	ymm2, [srcreg+d2+32]		;; R7
	vmovapd	ymm7, [srcreg+d2+d1+32]		;; R8
	vshufpd	ymm1, ymm2, ymm7, 15		;; Shuffle R7 and R8 to create R7/R8 hi
	vshufpd	ymm2, ymm2, ymm7, 0		;; Shuffle R7 and R8 to create R7/R8 low

	ylow128s ymm7, ymm5, ymm1		;; Shuffle R5/R6 hi and R7/R8 hi (new R6)
	yhigh128s ymm5, ymm5, ymm1		;; Shuffle R5/R6 hi and R7/R8 hi (new R8)

	L1prefetch srcreg+L1pd, L1pt
	L1prefetchw dstreg+L1pd, L1pt - L1PREFETCH_DEST_NONE

	ylow128s ymm1, ymm6, ymm2		;; Shuffle R5/R6 low and R7/R8 low (new R5)
	yhigh128s ymm6, ymm6, ymm2		;; Shuffle R5/R6 low and R7/R8 low (new R7)

	vsubpd	ymm2, ymm0, ymm5		;; new R8 = R4 - R8			; 1-3
	vaddpd	ymm0, ymm0, ymm5		;; new R4 = R4 + R8			; 2-4

	vsubpd	ymm5, ymm4, ymm7		;; new R6 = R2 - R6			; 3-5
	vaddpd	ymm4, ymm4, ymm7		;; new R2 = R2 + R6			; 4-6

	vmulpd	ymm2, ymm2, YMM_SQRTHALF	;; R8 = R8 * square root		;  4-8
	vmulpd	ymm5, ymm5, YMM_SQRTHALF	;; R6 = R6 * square root		;  6-10

	vsubpd	ymm7, ymm3, ymm1		;; new R5 = R1 - R5			; 5-7
	vaddpd	ymm3, ymm3, ymm1		;; new R1 = R1 + R5			; 6-8

	L1prefetch srcreg+d1+L1pd, L1pt
	L1prefetchw dstreg+e1+L1pd, L1pt - L1PREFETCH_DEST_NONE

	vaddpd	ymm1, ymm4, ymm0		;; R2 + R4 (final I1, a.k.a 2nd real result) ; 7-9
	vsubpd	ymm4, ymm4, ymm0		;; R2 - R4 (final I2)			; 8-10

	ystore	[dstreg+32], ymm1		;; Save final I1			; 10

	vmovapd	ymm0, [dstreg]			;; Reload saved new R3
	vaddpd	ymm1, ymm0, ymm6		;; new R3 = R3 + R7			; 9-11
	vsubpd	ymm0, ymm0, ymm6		;; new R7 = R3 - R7			; 10-12

	vsubpd	ymm6, ymm5, ymm2		;; R6 = R6 - R8 (Real part)		; 11-13
	vaddpd	ymm5, ymm5, ymm2		;; R8 = R6 + R8 (Imaginary part)	; 12-14

	L1prefetch srcreg+d2+L1pd, L1pt
	L1prefetchw dstreg+e2+L1pd, L1pt - L1PREFETCH_DEST_NONE

	vaddpd	ymm2, ymm3, ymm1		;; R1 + R3 (final R1)			; 13-15
	vsubpd	ymm3, ymm3, ymm1		;; R1 - R3 (final R2)			; 14-16

	vsubpd	ymm1, ymm7, ymm6		;; R5 - R6 (final R4)			; 15-17
	vaddpd	ymm7, ymm7, ymm6		;; R5 + R6 (final R3)			; 16-18

	L1prefetch srcreg+d2+d1+L1pd, L1pt
	L1prefetchw dstreg+e2+e1+L1pd, L1pt - L1PREFETCH_DEST_NONE

	vsubpd	ymm6, ymm0, ymm5		;; R7 - R8 (final I4)			; 17-19
	vaddpd	ymm0, ymm0, ymm5		;; R7 + R8 (final I3)			; 18-20

	ystore	[dstreg], ymm2			;; Save final R1			; 16

	;; screg1+64 is sin/cos values for R2/I2 (w^2n)
	;; screg2 is sin/cos values for R3/I3 (w^n)
	;; screg2+64 is sin/cos values for R4/I4 (w^5n)

	vmovapd	ymm2, [screg2+32]		;; cosine/sine
	vmulpd	ymm5, ymm7, ymm2		;; A3 = R3 * cosine/sine
	vsubpd	ymm5, ymm5, ymm0		;; A3 = A3 - I3
	vmulpd	ymm0, ymm0, ymm2		;; B3 = I3 * cosine/sine
	vaddpd	ymm0, ymm0, ymm7		;; B3 = B3 + R3

	vmovapd	ymm2, [screg1+64+32]		;; cosine
	vmulpd	ymm7, ymm3, ymm2		;; A2 = R2 * cosine
	vmulpd	ymm2, ymm4, ymm2		;; B2 = I2 * cosine
	vmulpd	ymm4, ymm4, [screg1+64]		;; C2 = I2 * sine
	vmulpd	ymm3, ymm3, [screg1+64]		;; D2 = R2 * sine
	vsubpd	ymm7, ymm7, ymm4		;; A2 = A2 - C2 (final R2)
	vaddpd	ymm2, ymm2, ymm3		;; B2 = B2 + D2 (final I2)

	vmovapd	ymm4, [screg2+64+32]		;; cosine/sine
	vmulpd	ymm3, ymm1, ymm4		;; A4 = R4 * cosine/sine
	vsubpd	ymm3, ymm3, ymm6		;; A4 = A4 - I4
	vmulpd	ymm6, ymm6, ymm4		;; B4 = I4 * cosine/sine
	vaddpd	ymm6, ymm6, ymm1		;; B4 = B4 + R4

	vmovapd	ymm4, [screg2]
	vmulpd	ymm5, ymm5, ymm4		;; A3 = A3 * sine (final R3)
	vmulpd	ymm0, ymm0, ymm4		;; B3 = B3 * sine (final I3)
	ystore	[dstreg+e2], ymm5		;; Save R3
	ystore	[dstreg+e2+32], ymm0		;; Save I3

	vmulpd	ymm3, ymm3, [screg2+64]		;; A4 = A4 * sine (final R4)
	vmulpd	ymm6, ymm6, [screg2+64]		;; B4 = B4 * sine (final I4)

	ystore	[dstreg+e1], ymm7		;; Save R2
	ystore	[dstreg+e1+32], ymm2		;; Save I2
	ystore	[dstreg+e2+e1], ymm3		;; Save R4
	ystore	[dstreg+e2+e1+32], ymm6		;; Save I4

	bump	srcreg, srcinc
	bump	dstreg, dstinc
	bump	screg1, scinc1
	bump	screg2, scinc2
	ENDM

IFDEF X86_64

yr4_rsc_sg4cl_2sc_eight_reals_fft4_preload MACRO
	vmovapd	ymm15, YMM_SQRTHALF
	ENDM
yr4_rsc_sg4cl_2sc_eight_reals_fft4 MACRO srcreg,srcinc,d1,d2,dstreg,dstinc,e1,e2,screg1,scinc1,screg2,scinc2,maxrpt,L1pt,L1pd
	vmovapd	ymm1, [srcreg]			;; R1
	vmovapd	ymm7, [srcreg+d1]		;; R2
	vshufpd	ymm0, ymm1, ymm7, 15		;; Shuffle R1 and R2 to create R1/R2 hi			; 1
	vshufpd	ymm1, ymm1, ymm7, 0		;; Shuffle R1 and R2 to create R1/R2 low		; 2

	vmovapd	ymm3, [srcreg+d2]		;; R3
	vmovapd	ymm7, [srcreg+d2+d1]		;; R4
	vshufpd	ymm2, ymm3, ymm7, 15		;; Shuffle R3 and R4 to create R3/R4 hi			; 3
	vshufpd	ymm3, ymm3, ymm7, 0		;; Shuffle R3 and R4 to create R3/R4 low		; 4

	vmovapd	ymm5, [srcreg+32]		;; R5
	vmovapd	ymm7, [srcreg+d1+32]		;; R6
	vshufpd	ymm4, ymm5, ymm7, 15		;; Shuffle R5 and R6 to create R5/R6 hi			; 5
	vshufpd	ymm5, ymm5, ymm7, 0		;; Shuffle R5 and R6 to create R5/R6 low		; 6

	vmovapd	ymm7, [srcreg+d2+32]		;; R7
	vmovapd	ymm8, [srcreg+d2+d1+32]		;; R8
	vshufpd	ymm6, ymm7, ymm8, 15		;; Shuffle R7 and R8 to create R7/R8 hi			; 7
	vshufpd	ymm7, ymm7, ymm8, 0		;; Shuffle R7 and R8 to create R7/R8 low		; 8

	yhigh128s ymm8, ymm0, ymm2		;; Shuffle R1/R2 hi and R3/R4 hi (new R4)		; 9-10
	L1prefetch srcreg+L1pd, L1pt
	L1prefetchw dstreg+L1pd, L1pt - L1PREFETCH_DEST_NONE

	yhigh128s ymm9, ymm4, ymm6		;; Shuffle R5/R6 hi and R7/R8 hi (new R8)		; 10-11

	ylow128s ymm0, ymm0, ymm2		;; Shuffle R1/R2 hi and R3/R4 hi (new R2)		; 11-12

	ylow128s ymm4, ymm4, ymm6		;; Shuffle R5/R6 hi and R7/R8 hi (new R6)		; 12-13
	vsubpd	ymm2, ymm8, ymm9		;; new R8 = R4 - R8					; 12-14

	ylow128s ymm6, ymm1, ymm3		;; Shuffle R1/R2 low and R3/R4 low (new R1)		; 13-14
	vaddpd	ymm8, ymm8, ymm9		;; new R4 = R4 + R8					; 13-15
	L1prefetch srcreg+d1+L1pd, L1pt
	L1prefetchw dstreg+e1+L1pd, L1pt - L1PREFETCH_DEST_NONE

	ylow128s ymm9, ymm5, ymm7		;; Shuffle R5/R6 low and R7/R8 low (new R5)		; 14-15
	vsubpd	ymm10, ymm0, ymm4		;; new R6 = R2 - R6					; 14-16

	yhigh128s ymm1, ymm1, ymm3		;; Shuffle R1/R2 low and R3/R4 low (new R3)		; 15-16
	vaddpd	ymm0, ymm0, ymm4		;; new R2 = R2 + R6					; 15-17
	vmulpd	ymm2, ymm2, ymm15		;; R8 = R8 * square root				;  15-19

	yhigh128s ymm5, ymm5, ymm7		;; Shuffle R5/R6 low and R7/R8 low (new R7)		; 16-17
	vaddpd	ymm7, ymm6, ymm9		;; new R1 = R1 + R5					; 16-18

	vsubpd	ymm6, ymm6, ymm9		;; new R5 = R1 - R5					; 17-19
	vmulpd	ymm10, ymm10, ymm15		;; R6 = R6 * square root				;  17-21

	vaddpd	ymm9, ymm1, ymm5		;; new R3 = R3 + R7					; 18-20
	vmovapd	ymm11, [screg1+64+32]		;; cosine for R2/I2

	vsubpd	ymm1, ymm1, ymm5		;; new R7 = R3 - R7					; 19-21
	vmovapd	ymm12, [screg1+64]		;; sine for R2/I2

	vsubpd	ymm5, ymm0, ymm8		;; R2 - R4 (final I2)					; 20-22
	vmovapd	ymm13, [screg2+64+32]		;; cosine/sine for R4/I4

	vsubpd	ymm4, ymm7, ymm9		;; R1 - R3 (final R2)					; 21-23
	L1prefetch srcreg+d2+L1pd, L1pt
	L1prefetchw dstreg+e2+L1pd, L1pt - L1PREFETCH_DEST_NONE

	vsubpd	ymm3, ymm10, ymm2		;; R6 = R6 - R8 (Real part)				; 22-24

	vaddpd	ymm10, ymm10, ymm2		;; R8 = R6 + R8 (Imaginary part)			; 23-25
	vmulpd	ymm2, ymm5, ymm11		;; B2 = I2 * cosine					;  23-27

	vaddpd	ymm0, ymm0, ymm8		;; R2 + R4 (final I1, a.k.a 2nd real result)		; 24-26
	vmulpd	ymm11, ymm4, ymm11		;; A2 = R2 * cosine					;  24-28
	L1prefetch srcreg+d2+d1+L1pd, L1pt
	L1prefetchw dstreg+e2+e1+L1pd, L1pt - L1PREFETCH_DEST_NONE

	vsubpd	ymm8, ymm6, ymm3		;; R5 - R6 (final R4)					; 25-27
	vmulpd	ymm5, ymm5, ymm12		;; C2 = I2 * sine					;  25-29

	vsubpd	ymm14, ymm1, ymm10		;; R7 - R8 (final I4)					; 26-28
	vmulpd	ymm4, ymm4, ymm12		;; D2 = R2 * sine					;  26-30
	vmovapd	ymm12, [screg2+32]		;; cosine/sine for R3/I3

	vaddpd	ymm6, ymm6, ymm3		;; R5 + R6 (final R3)					; 27-29
	ystore	[dstreg+32], ymm0		;; Save final I1					; 27
	vmovapd	ymm3, [screg2+64]		;; sine for R4/I4

	vaddpd	ymm1, ymm1, ymm10		;; R7 + R8 (final I3)					; 28-30
	vmulpd	ymm10, ymm8, ymm13		;; A4 = R4 * cosine/sine				;  28-32
	vmovapd	ymm0, [screg2]			;; sine for R3/I3

	vaddpd	ymm7, ymm7, ymm9		;; R1 + R3 (final R1)					; 29-31
	vmulpd	ymm13, ymm14, ymm13		;; B4 = I4 * cosine/sine				;  29-33

	vsubpd	ymm11, ymm11, ymm5		;; A2 = A2 - C2 (final R2)				; 30-32
	vmulpd	ymm5, ymm6, ymm12		;; A3 = R3 * cosine/sine				;  30-34

	vaddpd	ymm2, ymm2, ymm4		;; B2 = B2 + D2 (final I2)				; 31-33
	vmulpd	ymm12, ymm1, ymm12		;; B3 = I3 * cosine/sine				;  31-35

	ystore	[dstreg], ymm7			;; Save final R1					; 32

	vsubpd	ymm10, ymm10, ymm14		;; A4 = A4 - I4						; 33-35
	ystore	[dstreg+e1], ymm11		;; Save R2						; 33

	vaddpd	ymm13, ymm13, ymm8		;; B4 = B4 + R4						; 34-36
	ystore	[dstreg+e1+32], ymm2		;; Save I2						; 34

	vsubpd	ymm5, ymm5, ymm1		;; A3 = A3 - I3						; 35-37

	vaddpd	ymm12, ymm12, ymm6		;; B3 = B3 + R3						; 36-38
	vmulpd	ymm10, ymm10, ymm3		;; A4 = A4 * sine (final R4)				;  36-40

	vmulpd	ymm13, ymm13, ymm3		;; B4 = B4 * sine (final I4)				;  37-41

	vmulpd	ymm5, ymm5, ymm0		;; A3 = A3 * sine (final R3)				;  38-41

	vmulpd	ymm12, ymm12, ymm0		;; B3 = B3 * sine (final I3)				;  39-43

	ystore	[dstreg+e2+e1], ymm10		;; Save R4						; 41
	ystore	[dstreg+e2+e1+32], ymm13	;; Save I4						; 42
	ystore	[dstreg+e2], ymm5		;; Save R3						; 43
	ystore	[dstreg+e2+32], ymm12		;; Save I3						; 44

	bump	srcreg, srcinc
	bump	dstreg, dstinc
	bump	screg1, scinc1
	bump	screg2, scinc2
	ENDM

;; Haswell FMA3 version

IF (@INSTR(,%yarch,<FMA3>) NE 0)

yr4_rsc_sg4cl_2sc_eight_reals_fft4_preload MACRO
	vmovapd	ymm14, YMM_ONE
	vmovapd	ymm15, YMM_SQRTHALF
	ENDM

yr4_rsc_sg4cl_2sc_eight_reals_fft4 MACRO srcreg,srcinc,d1,d2,dstreg,dstinc,e1,e2,screg1,scinc1,screg2,scinc2,maxrpt,L1pt,L1pd
	vmovapd	ymm1, [srcreg]			;; R1
	vmovapd	ymm7, [srcreg+d1]		;; R2
	vshufpd	ymm0, ymm1, ymm7, 15		;; Shuffle R1 and R2 to create R1/R2 hi			; 1  (Skylake & Haswell timings)
	vshufpd	ymm1, ymm1, ymm7, 0		;; Shuffle R1 and R2 to create R1/R2 low		; 2

	vmovapd	ymm3, [srcreg+d2]		;; R3
	vmovapd	ymm7, [srcreg+d2+d1]		;; R4
	vshufpd	ymm2, ymm3, ymm7, 15		;; Shuffle R3 and R4 to create R3/R4 hi			; 3
	vshufpd	ymm3, ymm3, ymm7, 0		;; Shuffle R3 and R4 to create R3/R4 low		; 4

	vmovapd	ymm5, [srcreg+32]		;; R5
	vmovapd	ymm7, [srcreg+d1+32]		;; R6
	vshufpd	ymm4, ymm5, ymm7, 15		;; Shuffle R5 and R6 to create R5/R6 hi			; 5
	vshufpd	ymm5, ymm5, ymm7, 0		;; Shuffle R5 and R6 to create R5/R6 low		; 6

	vmovapd	ymm7, [srcreg+d2+32]		;; R7
	vmovapd	ymm8, [srcreg+d2+d1+32]		;; R8
	vshufpd	ymm6, ymm7, ymm8, 15		;; Shuffle R7 and R8 to create R7/R8 hi			; 7
	vshufpd	ymm7, ymm7, ymm8, 0		;; Shuffle R7 and R8 to create R7/R8 low		; 8
	L1prefetch srcreg+L1pd, L1pt

	yhigh128s ymm8, ymm0, ymm2		;; Shuffle R1/R2 hi and R3/R4 hi (new R4)		; 9-11
	yhigh128s ymm9, ymm4, ymm6		;; Shuffle R5/R6 hi and R7/R8 hi (new R8)		; 10-12
	L1prefetch srcreg+d1+L1pd, L1pt

	ylow128s ymm0, ymm0, ymm2		;; Shuffle R1/R2 hi and R3/R4 hi (new R2)		; 11-13
	ylow128s ymm4, ymm4, ymm6		;; Shuffle R5/R6 hi and R7/R8 hi (new R6)		; 12-14
	L1prefetch srcreg+d2+L1pd, L1pt

	yfmsubpd ymm2, ymm8, ymm14, ymm9	;; new R8 = R4 - R8					; 13-16	13-17
	yfmaddpd ymm8, ymm8, ymm14, ymm9	;; new R4 = R4 + R8					; 13-16	13-17
	L1prefetch srcreg+d2+d1+L1pd, L1pt

	ylow128s ymm6, ymm1, ymm3		;; Shuffle R1/R2 low and R3/R4 low (new R1)		; 13-15
	ylow128s ymm9, ymm5, ymm7		;; Shuffle R5/R6 low and R7/R8 low (new R5)		; 14-16
	L1prefetchw dstreg+L1pd, L1pt - L1PREFETCH_DEST_NONE

	yhigh128s ymm1, ymm1, ymm3		;; Shuffle R1/R2 low and R3/R4 low (new R3)		; 15-17
	yhigh128s ymm5, ymm5, ymm7		;; Shuffle R5/R6 low and R7/R8 low (new R7)		; 16-18
	L1prefetchw dstreg+e1+L1pd, L1pt - L1PREFETCH_DEST_NONE

	yfmsubpd ymm10, ymm0, ymm14, ymm4	;; new R6 = R2 - R6					; 15-18	15-19
	yfmaddpd ymm0, ymm0, ymm14, ymm4	;; new R2 = R2 + R6					; 15-18	15-19

	yfmaddpd ymm7, ymm6, ymm14, ymm9	;; new R1 = R1 + R5					; 17-20	17-21
	yfmsubpd ymm6, ymm6, ymm14, ymm9	;; new R5 = R1 - R5					; 17-20	17-21
	L1prefetchw dstreg+e2+L1pd, L1pt - L1PREFETCH_DEST_NONE

	yfmaddpd ymm9, ymm1, ymm14, ymm5	;; new R3 = R3 + R7					; 19-22	19-23
	yfmsubpd ymm1, ymm1, ymm14, ymm5	;; new R7 = R3 - R7					; 19-22	19-23
	L1prefetchw dstreg+e2+e1+L1pd, L1pt - L1PREFETCH_DEST_NONE

	yfmsubpd ymm3, ymm10, ymm14, ymm2	;; R6 = R6 - R8 (Real part)				; 20-23 20-24
	yfmaddpd ymm10, ymm10, ymm14, ymm2	;; R8 = R6 + R8 (Imaginary part)			; 20-23	20-24

	yfmsubpd ymm5, ymm0, ymm14, ymm8	;; R2 - R4 (final I2)					; 21-24	21-25
	yfmaddpd ymm0, ymm0, ymm14, ymm8	;; R2 + R4 (final I1, a.k.a 2nd real result)		; 21-24	21-25
	vmovapd	ymm12, [screg1+64]		;; sine for R2/I2

	yfmsubpd ymm4, ymm7, ymm14, ymm9	;; R1 - R3 (final R2)					; 23-26	24-28
	yfmaddpd ymm7, ymm7, ymm14, ymm9	;; R1 + R3 (final R1)					; 23-26	24-28
	vmovapd	ymm13, [screg2+64+32]		;; cosine/sine for R4/I4

	yfnmaddpd ymm8, ymm3, ymm15, ymm6	;; R5 - R6 * SQRTHALF (final R4)			; 24-27	25-29
	yfmaddpd ymm6, ymm3, ymm15, ymm6	;; R5 + R6 * SQRTHALF (final R3)			; 24-27	25-29
	vmovapd	ymm9, [screg2+32]		;; cosine/sine for R3/I3

	yfnmaddpd ymm2, ymm10, ymm15, ymm1	;; R7 - R8 * SQRTHALF (final I4)			; 25-28	26-30
	yfmaddpd ymm1, ymm10, ymm15, ymm1	;; R7 + R8 * SQRTHALF (final I3)			; 25-28	26-30

	ystore	[dstreg+32], ymm0		;; Save final I1					; 25	26
	ystore	[dstreg], ymm7			;; Save final R1					; 27	29
	bump	srcreg, srcinc

	vmulpd	ymm0, ymm5, ymm12		;; C2 = I2 * sine					; 27-30 29-33
	vmulpd	ymm7, ymm4, ymm12		;; D2 = R2 * sine					; 27-30	29-33
	vmovapd	ymm11, [screg1+64+32]		;; cosine for R2/I2

	yfmsubpd ymm10, ymm8, ymm13, ymm2	;; A4 = R4 * cosine/sine - I4				; 29-32	31-35
	yfmaddpd ymm13, ymm2, ymm13, ymm8	;; B4 = I4 * cosine/sine + R4				; 29-32	31-35
	vmovapd	ymm3, [screg2+64]		;; sine for R4/I4

	yfmsubpd ymm2, ymm6, ymm9, ymm1		;; A3 = R3 * cosine/sine - I3				; 30-33	32-36
	yfmaddpd ymm12, ymm1, ymm9, ymm6	;; B3 = I3 * cosine/sine + R3				; 30-33	32-36
	vmovapd	ymm1, [screg2]			;; sine for R3/I3

	yfmsubpd ymm4, ymm4, ymm11, ymm0	;; A2 = R2 * cosine - C2 (final R2)			; 31-34	34-38
	yfmaddpd ymm5, ymm5, ymm11, ymm7	;; B2 = I2 * cosine + D2 (final I2)			; 31-34	34-38
	bump	screg1, scinc1

	vmulpd	ymm10, ymm10, ymm3		;; A4 = A4 * sine (final R4)				; 33-36	36-40
	vmulpd	ymm13, ymm13, ymm3		;; B4 = B4 * sine (final I4)				; 33-36	36-40

	vmulpd	ymm2, ymm2, ymm1		;; A3 = A3 * sine (final R3)				; 34-37	37-41
	vmulpd	ymm12, ymm12, ymm1		;; B3 = B3 * sine (final I3)				; 34-37	37-41
	bump	screg2, scinc2

	ystore	[dstreg+e1], ymm4		;; Save R2						; 35	39
	ystore	[dstreg+e1+32], ymm5		;; Save I2						; 36	40
	ystore	[dstreg+e2+e1], ymm10		;; Save R4						; 37	41
	ystore	[dstreg+e2+e1+32], ymm13	;; Save I4						; 38	42
	ystore	[dstreg+e2], ymm2		;; Save R3						; 39	43
	ystore	[dstreg+e2+32], ymm12		;; Save I3						; 40	44

	bump	dstreg, dstinc
	ENDM

ENDIF

ENDIF


; These versions use registers for distances between blocks.  This lets us share pass1 code.

;; Used in last levels of pass 1.  Swizzling.
;; In this particular case, we know that the sin/cos multipliers for R1/I1 are one.  Thus
;; we can get by with only 3 sin/cos multiplies.
yr4_rsc_sg4clreg_2sc_eight_reals_fft4_preload MACRO
	ENDM
yr4_rsc_sg4clreg_2sc_eight_reals_fft4 MACRO srcreg,srcinc,d1,d2,dstreg,dstinc,e1reg,e3reg,screg1,scinc1,screg2,scinc2,maxrpt,L1pt,L1preg
	NOT IMPLMENTED IN 32-BIT
	ENDM

IFDEF X86_64

yr4_rsc_sg4clreg_2sc_eight_reals_fft4_preload MACRO
	vmovapd	ymm15, YMM_SQRTHALF
	ENDM
yr4_rsc_sg4clreg_2sc_eight_reals_fft4 MACRO srcreg,srcinc,d1,d2,dstreg,dstinc,e1reg,e3reg,screg1,scinc1,screg2,scinc2,maxrpt,L1pt,L1preg
	vmovapd	ymm1, [srcreg]			;; R1
	vmovapd	ymm7, [srcreg+d1]		;; R2
	vshufpd	ymm0, ymm1, ymm7, 15		;; Shuffle R1 and R2 to create R1/R2 hi			; 1
	vshufpd	ymm1, ymm1, ymm7, 0		;; Shuffle R1 and R2 to create R1/R2 low		; 2

	vmovapd	ymm3, [srcreg+d2]		;; R3
	vmovapd	ymm7, [srcreg+d2+d1]		;; R4
	vshufpd	ymm2, ymm3, ymm7, 15		;; Shuffle R3 and R4 to create R3/R4 hi			; 3
	vshufpd	ymm3, ymm3, ymm7, 0		;; Shuffle R3 and R4 to create R3/R4 low		; 4

	vmovapd	ymm5, [srcreg+32]		;; R5
	vmovapd	ymm7, [srcreg+d1+32]		;; R6
	vshufpd	ymm4, ymm5, ymm7, 15		;; Shuffle R5 and R6 to create R5/R6 hi			; 5
	vshufpd	ymm5, ymm5, ymm7, 0		;; Shuffle R5 and R6 to create R5/R6 low		; 6

	vmovapd	ymm7, [srcreg+d2+32]		;; R7
	vmovapd	ymm8, [srcreg+d2+d1+32]		;; R8
	vshufpd	ymm6, ymm7, ymm8, 15		;; Shuffle R7 and R8 to create R7/R8 hi			; 7
	vshufpd	ymm7, ymm7, ymm8, 0		;; Shuffle R7 and R8 to create R7/R8 low		; 8

	yhigh128s ymm8, ymm0, ymm2		;; Shuffle R1/R2 hi and R3/R4 hi (new R4)		; 9-10
	L1prefetchw L1preg, L1pt - L1PREFETCH_DEST_NONE

	yhigh128s ymm9, ymm4, ymm6		;; Shuffle R5/R6 hi and R7/R8 hi (new R8)		; 10-11

	ylow128s ymm0, ymm0, ymm2		;; Shuffle R1/R2 hi and R3/R4 hi (new R2)		; 11-12

	ylow128s ymm4, ymm4, ymm6		;; Shuffle R5/R6 hi and R7/R8 hi (new R6)		; 12-13
	vsubpd	ymm2, ymm8, ymm9		;; new R8 = R4 - R8					; 12-14

	ylow128s ymm6, ymm1, ymm3		;; Shuffle R1/R2 low and R3/R4 low (new R1)		; 13-14
	vaddpd	ymm8, ymm8, ymm9		;; new R4 = R4 + R8					; 13-15
	L1prefetchw L1preg+e1reg, L1pt - L1PREFETCH_DEST_NONE

	ylow128s ymm9, ymm5, ymm7		;; Shuffle R5/R6 low and R7/R8 low (new R5)		; 14-15
	vsubpd	ymm10, ymm0, ymm4		;; new R6 = R2 - R6					; 14-16

	yhigh128s ymm1, ymm1, ymm3		;; Shuffle R1/R2 low and R3/R4 low (new R3)		; 15-16
	vaddpd	ymm0, ymm0, ymm4		;; new R2 = R2 + R6					; 15-17
	vmulpd	ymm2, ymm2, ymm15		;; R8 = R8 * square root				;  15-19

	yhigh128s ymm5, ymm5, ymm7		;; Shuffle R5/R6 low and R7/R8 low (new R7)		; 16-17
	vaddpd	ymm7, ymm6, ymm9		;; new R1 = R1 + R5					; 16-18

	vsubpd	ymm6, ymm6, ymm9		;; new R5 = R1 - R5					; 17-19
	vmulpd	ymm10, ymm10, ymm15		;; R6 = R6 * square root				;  17-21

	vaddpd	ymm9, ymm1, ymm5		;; new R3 = R3 + R7					; 18-20
	vmovapd	ymm11, [screg1+64+32]		;; cosine for R2/I2

	vsubpd	ymm1, ymm1, ymm5		;; new R7 = R3 - R7					; 19-21
	vmovapd	ymm12, [screg1+64]		;; sine for R2/I2

	vsubpd	ymm5, ymm0, ymm8		;; R2 - R4 (final I2)					; 20-22
	vmovapd	ymm13, [screg2+64+32]		;; cosine/sine for R4/I4

	vsubpd	ymm4, ymm7, ymm9		;; R1 - R3 (final R2)					; 21-23
	L1prefetchw L1preg+2*e1reg, L1pt - L1PREFETCH_DEST_NONE

	vsubpd	ymm3, ymm10, ymm2		;; R6 = R6 - R8 (Real part)				; 22-24

	vaddpd	ymm10, ymm10, ymm2		;; R8 = R6 + R8 (Imaginary part)			; 23-25
	vmulpd	ymm2, ymm5, ymm11		;; B2 = I2 * cosine					;  23-27

	vaddpd	ymm0, ymm0, ymm8		;; R2 + R4 (final I1, a.k.a 2nd real result)		; 24-26
	vmulpd	ymm11, ymm4, ymm11		;; A2 = R2 * cosine					;  24-28
	L1prefetchw L1preg+e3reg, L1pt - L1PREFETCH_DEST_NONE

	vsubpd	ymm8, ymm6, ymm3		;; R5 - R6 (final R4)					; 25-27
	vmulpd	ymm5, ymm5, ymm12		;; C2 = I2 * sine					;  25-29

	vsubpd	ymm14, ymm1, ymm10		;; R7 - R8 (final I4)					; 26-28
	vmulpd	ymm4, ymm4, ymm12		;; D2 = R2 * sine					;  26-30
	vmovapd	ymm12, [screg2+32]		;; cosine/sine for R3/I3

	vaddpd	ymm6, ymm6, ymm3		;; R5 + R6 (final R3)					; 27-29
	ystore	[dstreg+32], ymm0		;; Save final I1					; 27
	vmovapd	ymm3, [screg2+64]		;; sine for R4/I4

	vaddpd	ymm1, ymm1, ymm10		;; R7 + R8 (final I3)					; 28-30
	vmulpd	ymm10, ymm8, ymm13		;; A4 = R4 * cosine/sine				;  28-32
	vmovapd	ymm0, [screg2]			;; sine for R3/I3

	vaddpd	ymm7, ymm7, ymm9		;; R1 + R3 (final R1)					; 29-31
	vmulpd	ymm13, ymm14, ymm13		;; B4 = I4 * cosine/sine				;  29-33

	vsubpd	ymm11, ymm11, ymm5		;; A2 = A2 - C2 (final R2)				; 30-32
	vmulpd	ymm5, ymm6, ymm12		;; A3 = R3 * cosine/sine				;  30-34

	vaddpd	ymm2, ymm2, ymm4		;; B2 = B2 + D2 (final I2)				; 31-33
	vmulpd	ymm12, ymm1, ymm12		;; B3 = I3 * cosine/sine				;  31-35

	ystore	[dstreg], ymm7			;; Save final R1					; 32

	vsubpd	ymm10, ymm10, ymm14		;; A4 = A4 - I4						; 33-35
	ystore	[dstreg+e1reg], ymm11		;; Save R2						; 33

	vaddpd	ymm13, ymm13, ymm8		;; B4 = B4 + R4						; 34-36
	ystore	[dstreg+e1reg+32], ymm2		;; Save I2						; 34

	vsubpd	ymm5, ymm5, ymm1		;; A3 = A3 - I3						; 35-37

	vaddpd	ymm12, ymm12, ymm6		;; B3 = B3 + R3						; 36-38
	vmulpd	ymm10, ymm10, ymm3		;; A4 = A4 * sine (final R4)				;  36-40

	vmulpd	ymm13, ymm13, ymm3		;; B4 = B4 * sine (final I4)				;  37-41

	vmulpd	ymm5, ymm5, ymm0		;; A3 = A3 * sine (final R3)				;  38-41

	vmulpd	ymm12, ymm12, ymm0		;; B3 = B3 * sine (final I3)				;  39-43

	ystore	[dstreg+e3reg], ymm10		;; Save R4						; 41
	ystore	[dstreg+e3reg+32], ymm13	;; Save I4						; 42
	ystore	[dstreg+2*e1reg], ymm5		;; Save R3						; 43
	ystore	[dstreg+2*e1reg+32], ymm12	;; Save I3						; 44

	bump	srcreg, srcinc
	bump	dstreg, dstinc
	bump	screg1, scinc1
	bump	screg2, scinc2
	bump	L1preg, dstinc
	ENDM

;; Haswell FMA3 version

IF (@INSTR(,%yarch,<FMA3>) NE 0)

yr4_rsc_sg4clreg_2sc_eight_reals_fft4_preload MACRO
	vmovapd	ymm14, YMM_ONE
	vmovapd	ymm15, YMM_SQRTHALF
	ENDM

yr4_rsc_sg4clreg_2sc_eight_reals_fft4 MACRO srcreg,srcinc,d1,d2,dstreg,dstinc,e1reg,e3reg,screg1,scinc1,screg2,scinc2,maxrpt,L1pt,L1preg
	vmovapd	ymm1, [srcreg]			;; R1
	vmovapd	ymm7, [srcreg+d1]		;; R2
	vshufpd	ymm0, ymm1, ymm7, 15		;; Shuffle R1 and R2 to create R1/R2 hi			; 1  (Skylake & Haswell timings)
	vshufpd	ymm1, ymm1, ymm7, 0		;; Shuffle R1 and R2 to create R1/R2 low		; 2

	vmovapd	ymm3, [srcreg+d2]		;; R3
	vmovapd	ymm7, [srcreg+d2+d1]		;; R4
	vshufpd	ymm2, ymm3, ymm7, 15		;; Shuffle R3 and R4 to create R3/R4 hi			; 3
	vshufpd	ymm3, ymm3, ymm7, 0		;; Shuffle R3 and R4 to create R3/R4 low		; 4

	vmovapd	ymm5, [srcreg+32]		;; R5
	vmovapd	ymm7, [srcreg+d1+32]		;; R6
	vshufpd	ymm4, ymm5, ymm7, 15		;; Shuffle R5 and R6 to create R5/R6 hi			; 5
	vshufpd	ymm5, ymm5, ymm7, 0		;; Shuffle R5 and R6 to create R5/R6 low		; 6

	vmovapd	ymm7, [srcreg+d2+32]		;; R7
	vmovapd	ymm8, [srcreg+d2+d1+32]		;; R8
	vshufpd	ymm6, ymm7, ymm8, 15		;; Shuffle R7 and R8 to create R7/R8 hi			; 7
	vshufpd	ymm7, ymm7, ymm8, 0		;; Shuffle R7 and R8 to create R7/R8 low		; 8

	yhigh128s ymm8, ymm0, ymm2		;; Shuffle R1/R2 hi and R3/R4 hi (new R4)		; 9-11
	yhigh128s ymm9, ymm4, ymm6		;; Shuffle R5/R6 hi and R7/R8 hi (new R8)		; 10-12

	ylow128s ymm0, ymm0, ymm2		;; Shuffle R1/R2 hi and R3/R4 hi (new R2)		; 11-13
	ylow128s ymm4, ymm4, ymm6		;; Shuffle R5/R6 hi and R7/R8 hi (new R6)		; 12-14

	yfmsubpd ymm2, ymm8, ymm14, ymm9	;; new R8 = R4 - R8					; 13-16	13-17
	yfmaddpd ymm8, ymm8, ymm14, ymm9	;; new R4 = R4 + R8					; 13-16	13-17

	ylow128s ymm6, ymm1, ymm3		;; Shuffle R1/R2 low and R3/R4 low (new R1)		; 13-15
	ylow128s ymm9, ymm5, ymm7		;; Shuffle R5/R6 low and R7/R8 low (new R5)		; 14-16
	L1prefetchw L1preg, L1pt - L1PREFETCH_DEST_NONE

	yhigh128s ymm1, ymm1, ymm3		;; Shuffle R1/R2 low and R3/R4 low (new R3)		; 15-17
	yhigh128s ymm5, ymm5, ymm7		;; Shuffle R5/R6 low and R7/R8 low (new R7)		; 16-18
	L1prefetchw L1preg+e1reg, L1pt - L1PREFETCH_DEST_NONE

	yfmsubpd ymm10, ymm0, ymm14, ymm4	;; new R6 = R2 - R6					; 15-18	15-19
	yfmaddpd ymm0, ymm0, ymm14, ymm4	;; new R2 = R2 + R6					; 15-18	15-19

	yfmaddpd ymm7, ymm6, ymm14, ymm9	;; new R1 = R1 + R5					; 17-20	17-21
	yfmsubpd ymm6, ymm6, ymm14, ymm9	;; new R5 = R1 - R5					; 17-20	17-21
	L1prefetchw L1preg+2*e1reg, L1pt - L1PREFETCH_DEST_NONE

	yfmaddpd ymm9, ymm1, ymm14, ymm5	;; new R3 = R3 + R7					; 19-22	19-23
	yfmsubpd ymm1, ymm1, ymm14, ymm5	;; new R7 = R3 - R7					; 19-22	19-23
	L1prefetchw L1preg+e3reg, L1pt - L1PREFETCH_DEST_NONE

	yfmsubpd ymm3, ymm10, ymm14, ymm2	;; R6 = R6 - R8 (Real part)				; 20-23 20-24
	yfmaddpd ymm10, ymm10, ymm14, ymm2	;; R8 = R6 + R8 (Imaginary part)			; 20-23	20-24

	yfmsubpd ymm5, ymm0, ymm14, ymm8	;; R2 - R4 (final I2)					; 21-24	21-25
	yfmaddpd ymm0, ymm0, ymm14, ymm8	;; R2 + R4 (final I1, a.k.a 2nd real result)		; 21-24	21-25
	vmovapd	ymm12, [screg1+64]		;; sine for R2/I2

	yfmsubpd ymm4, ymm7, ymm14, ymm9	;; R1 - R3 (final R2)					; 23-26	24-28
	yfmaddpd ymm7, ymm7, ymm14, ymm9	;; R1 + R3 (final R1)					; 23-26	24-28
	vmovapd	ymm13, [screg2+64+32]		;; cosine/sine for R4/I4

	yfnmaddpd ymm8, ymm3, ymm15, ymm6	;; R5 - R6 * SQRTHALF (final R4)			; 24-27	25-29
	yfmaddpd ymm6, ymm3, ymm15, ymm6	;; R5 + R6 * SQRTHALF (final R3)			; 24-27	25-29
	vmovapd	ymm9, [screg2+32]		;; cosine/sine for R3/I3

	yfnmaddpd ymm2, ymm10, ymm15, ymm1	;; R7 - R8 * SQRTHALF (final I4)			; 25-28	26-30
	yfmaddpd ymm1, ymm10, ymm15, ymm1	;; R7 + R8 * SQRTHALF (final I3)			; 25-28	26-30

	ystore	[dstreg+32], ymm0		;; Save final I1					; 25	26
	ystore	[dstreg], ymm7			;; Save final R1					; 27	29
	bump	srcreg, srcinc

	vmulpd	ymm0, ymm5, ymm12		;; C2 = I2 * sine					; 27-30 29-33
	vmulpd	ymm7, ymm4, ymm12		;; D2 = R2 * sine					; 27-30	29-33
	vmovapd	ymm11, [screg1+64+32]		;; cosine for R2/I2

	yfmsubpd ymm10, ymm8, ymm13, ymm2	;; A4 = R4 * cosine/sine - I4				; 29-32	31-35
	yfmaddpd ymm13, ymm2, ymm13, ymm8	;; B4 = I4 * cosine/sine + R4				; 29-32	31-35
	vmovapd	ymm3, [screg2+64]		;; sine for R4/I4

	yfmsubpd ymm2, ymm6, ymm9, ymm1		;; A3 = R3 * cosine/sine - I3				; 30-33	32-36
	yfmaddpd ymm12, ymm1, ymm9, ymm6	;; B3 = I3 * cosine/sine + R3				; 30-33	32-36
	vmovapd	ymm1, [screg2]			;; sine for R3/I3

	yfmsubpd ymm4, ymm4, ymm11, ymm0	;; A2 = R2 * cosine - C2 (final R2)			; 31-34	34-38
	yfmaddpd ymm5, ymm5, ymm11, ymm7	;; B2 = I2 * cosine + D2 (final I2)			; 31-34	34-38
	bump	screg1, scinc1

	vmulpd	ymm10, ymm10, ymm3		;; A4 = A4 * sine (final R4)				; 33-36	36-40
	vmulpd	ymm13, ymm13, ymm3		;; B4 = B4 * sine (final I4)				; 33-36	36-40

	vmulpd	ymm2, ymm2, ymm1		;; A3 = A3 * sine (final R3)				; 34-37	37-41
	vmulpd	ymm12, ymm12, ymm1		;; B3 = B3 * sine (final I3)				; 34-37	37-41
	bump	screg2, scinc2

	ystore	[dstreg+e1reg], ymm4		;; Save R2						; 35	39
	ystore	[dstreg+e1reg+32], ymm5		;; Save I2						; 36	40
	ystore	[dstreg+e3reg], ymm10		;; Save R4						; 37	41
	ystore	[dstreg+e3reg+32], ymm13	;; Save I4						; 38	42
	ystore	[dstreg+2*e1reg], ymm2		;; Save R3						; 39	43
	ystore	[dstreg+2*e1reg+32], ymm12	;; Save I3						; 40	44

	bump	dstreg, dstinc
	bump	L1preg, dstinc
	ENDM

ENDIF

ENDIF

;;
;; ********************************** reduced sin/cos eight-real-unfft4 variants **************************************
;;

;; Used in last levels of pass 1.  Swizzling.
;; In this particular case, we know that the sin/cos multipliers for R1/I1 are one.  Thus
;; we can get by with only 3 sin/cos multiplies.
yr4_rsc_sg4cl_2sc_eight_reals_unfft4_preload MACRO
	ENDM
yr4_rsc_sg4cl_2sc_eight_reals_unfft4 MACRO srcreg,srcinc,d1,d2,dstreg,dstinc,e1,e2,screg1,scinc1,screg2,scinc2,maxrpt,L1pt,L1pd

	;; screg1+64 is sin/cos values for R2/I2 (w^2n)
	;; screg2 is sin/cos values for R3/I3 (w^n)
	;; screg2+64 is sin/cos values for R4/I4 (w^5n)

	vmovapd	ymm3, [screg1+64+32]		;; cosine
	vmovapd	ymm4, [srcreg+d1]		;; R2
	vmulpd	ymm2, ymm4, ymm3		;; A2 = R2 * cosine
	vmovapd	ymm5, [srcreg+d1+32]		;; I2
	vmulpd	ymm3, ymm5, ymm3		;; B2 = I2 * cosine
	vmulpd	ymm5, ymm5, [screg1+64]		;; C2 = I2 * sine
	vmulpd	ymm4, ymm4, [screg1+64]		;; D2 = R2 * sine
	vaddpd	ymm2, ymm2, ymm5		;; A2 = A2 + C2 (new R3)
	vsubpd	ymm3, ymm3, ymm4		;; B2 = B2 - D2 (new R4)

	vmovapd	ymm0, [screg2+32]		;; cosine/sine
	vmovapd	ymm4, [srcreg+d2]		;; R3
	vmulpd	ymm5, ymm4, ymm0		;; A3 = R3 * cosine/sine
	vmovapd	ymm1, [srcreg+d2+32]		;; I3
	vaddpd	ymm5, ymm5, ymm1		;; A3 = A3 + I3
	vmulpd	ymm1, ymm1, ymm0		;; B3 = I3 * cosine/sine
	vsubpd	ymm1, ymm1, ymm4		;; B3 = B3 - R3

	vmovapd	ymm0, [screg2+64+32]		;; cosine/sine
	vmovapd	ymm6, [srcreg+d2+d1]		;; R4
	vmulpd	ymm7, ymm6, ymm0		;; A4 = R4 * cosine/sine
	vmovapd ymm4, [srcreg+d2+d1+32]		;; I4
	vaddpd	ymm7, ymm7, ymm4		;; A4 = A4 + I4
	vmulpd	ymm4, ymm4, ymm0		;; B4 = I4 * cosine/sine
	vsubpd	ymm4, ymm4, ymm6		;; B4 = B4 - R4

	vmulpd	ymm5, ymm5, [screg2]		;; A3 = A3 * sine (new R5)
	vmulpd	ymm1, ymm1, [screg2]		;; B3 = B3 * sine (new R6)
	vmulpd	ymm7, ymm7, [screg2+64]		;; A4 = A4 * sine (new R7)
	vmulpd	ymm4, ymm4, [screg2+64]		;; B4 = B4 * sine (new R8)

	vsubpd	ymm0, ymm5, ymm7		;; R5 - R7 (newer R6)
	vaddpd	ymm5, ymm5, ymm7		;; R5 + R7 (newer R5)

	L1prefetch srcreg+L1pd, L1pt

	vsubpd	ymm7, ymm1, ymm4		;; R6 - R8 (newer R8)
	vaddpd	ymm1, ymm1, ymm4		;; R6 + R8 (newer R7)

	vaddpd	ymm4, ymm0, ymm7		;; R6 = R6 + R8
	vsubpd	ymm7, ymm7, ymm0		;; R8 = R8 - R6

	vmulpd	ymm4, ymm4, YMM_SQRTHALF	;; R6 = R6 * square root of 1/2
	vmulpd	ymm7, ymm7, YMM_SQRTHALF	;; R8 = R8 * square root of 1/2

	vmovapd	ymm6, [srcreg+32]		;; I1 (a.k.a new R2)

	vsubpd	ymm0, ymm6, ymm3		;; R2 - R4 (newer R4)
	vaddpd	ymm6, ymm6, ymm3		;; R2 + R4 (newer R2)

	L1prefetch srcreg+d1+L1pd, L1pt

	vsubpd	ymm3, ymm0, ymm7		;; R4 - R8 (final R8)
	vaddpd	ymm0, ymm0, ymm7		;; R4 + R8 (final R4)

	vmovapd	ymm7, [srcreg]			;; R1
	ystore	[dstreg], ymm3			;; Save final R8 temporarily
	vaddpd	ymm3, ymm7, ymm2		;; R1 + R3 (newer R1)
	vsubpd	ymm7, ymm7, ymm2		;; R1 - R3 (newer R3)

	vsubpd	ymm2, ymm6, ymm4		;; R2 - R6 (final R6)
	vaddpd	ymm6, ymm6, ymm4		;; R2 + R6 (final R2)

	L1prefetch srcreg+d2+L1pd, L1pt

	vsubpd	ymm4, ymm3, ymm5		;; R1 - R5 (final R5)
	vaddpd	ymm3, ymm3, ymm5		;; R1 + R5 (final R1)

	vsubpd	ymm5, ymm7, ymm1		;; R3 - R7 (final R7)
	vaddpd	ymm7, ymm7, ymm1		;; R3 + R7 (final R3)

	;; Swizzle using 4 shufpds and 4 perm2f128  (1 or 3 clocks each on port 5, throughput 1)

	vshufpd	ymm1, ymm3, ymm6, 0		;; Shuffle R1 and R2 to create R1/R2 low
	vshufpd	ymm3, ymm3, ymm6, 15		;; Shuffle R1 and R2 to create R1/R2 hi

	vshufpd	ymm6, ymm7, ymm0, 0		;; Shuffle R3 and R4 to create R3/R4 low
	vshufpd	ymm7, ymm7, ymm0, 15		;; Shuffle R3 and R4 to create R3/R4 hi

	L1prefetch srcreg+d2+d1+L1pd, L1pt

	ylow128s ymm0, ymm1, ymm6		;; Shuffle R1/R2 low and R3/R4 low (final R1)
	yhigh128s ymm1, ymm1, ymm6		;; Shuffle R1/R2 low and R3/R4 low (final R3)

	ylow128s ymm6, ymm3, ymm7		;; Shuffle R1/R2 hi and R3/R4 hi (final R2)
	yhigh128s ymm3, ymm3, ymm7		;; Shuffle R1/R2 hi and R3/R4 hi (final R4)

	vmovapd	ymm7, [dstreg]			;; Reload final R8

	ystore	[dstreg], ymm0			;; Save R1
	ystore	[dstreg+e2], ymm1		;; Save R3
	ystore	[dstreg+e1], ymm6		;; Save R2
	ystore	[dstreg+e2+e1], ymm3		;; Save R4

	;; Swizzle using 4 shufpds and 4 perm2f128  (1 or 3 clocks each on port 5, throughput 1)

	vshufpd	ymm0, ymm4, ymm2, 0		;; Shuffle R5 and R6 to create R5/R6 low
	vshufpd	ymm4, ymm4, ymm2, 15		;; Shuffle R5 and R6 to create R5/R6 hi

	vshufpd	ymm2, ymm5, ymm7, 0		;; Shuffle R7 and R8 to create R7/R8 low
	vshufpd	ymm5, ymm5, ymm7, 15		;; Shuffle R7 and R8 to create R7/R8 hi

	ylow128s ymm7, ymm0, ymm2		;; Shuffle R5/R6 low and R7/R8 low (final R5)
	yhigh128s ymm0, ymm0, ymm2		;; Shuffle R5/R6 low and R7/R8 low (final R7)

	ylow128s ymm2, ymm4, ymm5		;; Shuffle R5/R6 hi and R7/R8 hi (final R6)
	yhigh128s ymm4, ymm4, ymm5		;; Shuffle R5/R6 hi and R7/R8 hi (final R8)

	ystore	[dstreg+32], ymm7		;; Save R5
	ystore	[dstreg+e2+32], ymm0		;; Save R7
	ystore	[dstreg+e1+32], ymm2		;; Save R6
	ystore	[dstreg+e2+e1+32], ymm4		;; Save R8

	bump	srcreg, srcinc
	bump	dstreg, dstinc
	bump	screg1, scinc1
	bump	screg2, scinc2
	ENDM

IFDEF X86_64

yr4_rsc_sg4cl_2sc_eight_reals_unfft4_preload MACRO
	vmovapd	ymm15, YMM_SQRTHALF
	ENDM
yr4_rsc_sg4cl_2sc_eight_reals_unfft4 MACRO srcreg,srcinc,d1,d2,dstreg,dstinc,e1,e2,screg1,scinc1,screg2,scinc2,maxrpt,L1pt,L1pd

	vmovapd	ymm0, [screg2+32]		;; cosine/sine for R3/I3
	vmovapd	ymm1, [srcreg+d2]		;; R3
	vmulpd	ymm2, ymm1, ymm0		;; A3 = R3 * cosine/sine			;  1-5

	vmovapd	ymm3, [screg2+64+32]		;; cosine/sine for R4/I4
	vmovapd	ymm4, [srcreg+d2+d1]		;; R4
	vmulpd	ymm5, ymm4, ymm3		;; A4 = R4 * cosine/sine			;  2-6

	vmovapd	ymm6, [srcreg+d2+32]		;; I3
	vmulpd	ymm0, ymm6, ymm0		;; B3 = I3 * cosine/sine			;  3-7

	vmovapd ymm7, [srcreg+d2+d1+32]		;; I4
	vmulpd	ymm3, ymm7, ymm3		;; B4 = I4 * cosine/sine			;  4-8

	vmovapd	ymm8, [screg1+64+32]		;; cosine for R2/I2
	vmovapd	ymm9, [srcreg+d1]		;; R2
	vmulpd	ymm10, ymm9, ymm8		;; A2 = R2 * cosine				;  5-9

	vmovapd	ymm11, [srcreg+d1+32]		;; I2
	vaddpd	ymm2, ymm2, ymm6		;; A3 = A3 + I3					; 6-8
	vmulpd	ymm8, ymm11, ymm8		;; B2 = I2 * cosine				;  6-10

	vmovapd	ymm12, [screg1+64]		;; sine for R2/I2
	vaddpd	ymm5, ymm5, ymm7		;; A4 = A4 + I4					; 7-9
	vmulpd	ymm7, ymm11, ymm12		;; C2 = I2 * sine				;  7-11

	vsubpd	ymm0, ymm0, ymm1		;; B3 = B3 - R3					; 8-10
	vmulpd	ymm12, ymm9, ymm12		;; D2 = R2 * sine				;  8-12

	vmovapd	ymm13, [screg2]			;; sine for R3/I3
	vsubpd	ymm3, ymm3, ymm4		;; B4 = B4 - R4					; 9-11
	vmulpd	ymm2, ymm2, ymm13		;; A3 = A3 * sine (new R5)			;  9-13

	vmovapd	ymm14, [screg2+64]		;; sine for R4/I4
	vmulpd	ymm5, ymm5, ymm14		;; A4 = A4 * sine (new R7)			;  10-14

	vmulpd	ymm0, ymm0, ymm13		;; B3 = B3 * sine (new R6)			;  11-15
	vmovapd	ymm6, [srcreg]			;; R1

	vaddpd	ymm10, ymm10, ymm7		;; A2 = A2 + C2 (new R3)			; 12-14
	vmulpd	ymm3, ymm3, ymm14		;; B4 = B4 * sine (new R8)			;  12-16

	vsubpd	ymm8, ymm8, ymm12		;; B2 = B2 - D2 (new R4)			; 13-15
	vmovapd	ymm1, [srcreg+32]		;; I1 (a.k.a new R2)

	L1prefetch srcreg+L1pd, L1pt

	vsubpd	ymm12, ymm2, ymm5		;; R5 - R7 (newer R6)				; 15-17

	vaddpd	ymm2, ymm2, ymm5		;; R5 + R7 (newer R5)				; 16-18

	vsubpd	ymm5, ymm0, ymm3		;; R6 - R8 (newer R8)				; 17-19
	L1prefetch srcreg+d1+L1pd, L1pt

	vaddpd	ymm0, ymm0, ymm3		;; R6 + R8 (newer R7)				; 18-20

	vaddpd	ymm3, ymm6, ymm10		;; R1 + R3 (newer R1)				; 19-21

	vaddpd	ymm14, ymm12, ymm5		;; R6 = R6 + R8					; 20-22

	vsubpd	ymm5, ymm5, ymm12		;; R8 = R8 - R6					; 21-23
	L1prefetch srcreg+d2+L1pd, L1pt

	vsubpd	ymm6, ymm6, ymm10		;; R1 - R3 (newer R3)				; 22-24

	vaddpd	ymm10, ymm1, ymm8		;; R2 + R4 (newer R2)				; 23-25
	vmulpd	ymm14, ymm14, ymm15		;; R6 = R6 * square root of 1/2			;  23-27

	vsubpd	ymm1, ymm1, ymm8		;; R2 - R4 (newer R4)				; 24-26
	vmulpd	ymm5, ymm5, ymm15		;; R8 = R8 * square root of 1/2			;  24-28

	vaddpd	ymm8, ymm3, ymm2		;; R1 + R5 (final R1)				; 25-27
	L1prefetch srcreg+d2+d1+L1pd, L1pt

	vsubpd	ymm3, ymm3, ymm2		;; R1 - R5 (final R5)				; 26-28

	vaddpd	ymm2, ymm6, ymm0		;; R3 + R7 (final R3)				; 27-29

	vaddpd	ymm12, ymm10, ymm14		;; R2 + R6 (final R2)				; 28-30

	vaddpd	ymm7, ymm1, ymm5		;; R4 + R8 (final R4)				; 29-31

	vsubpd	ymm10, ymm10, ymm14		;; R2 - R6 (final R6)				; 30-32

	vshufpd	ymm14, ymm8, ymm12, 0		;; Shuffle R1 and R2 to create R1/R2 low	; 31
	vsubpd	ymm1, ymm1, ymm5		;; R4 - R8 (final R8)				; 31-33

	vshufpd	ymm8, ymm8, ymm12, 15		;; Shuffle R1 and R2 to create R1/R2 hi		; 32
	vsubpd	ymm6, ymm6, ymm0		;; R3 - R7 (final R7)				; 32-34

	vshufpd	ymm0, ymm2, ymm7, 0		;; Shuffle R3 and R4 to create R3/R4 low	; 33
	vshufpd	ymm2, ymm2, ymm7, 15		;; Shuffle R3 and R4 to create R3/R4 hi		; 34
	vshufpd	ymm7, ymm3, ymm10, 0		;; Shuffle R5 and R6 to create R5/R6 low	; 35
	vshufpd	ymm3, ymm3, ymm10, 15		;; Shuffle R5 and R6 to create R5/R6 hi		; 36
	vshufpd	ymm10, ymm6, ymm1, 0		;; Shuffle R7 and R8 to create R7/R8 low	; 37
	vshufpd	ymm6, ymm6, ymm1, 15		;; Shuffle R7 and R8 to create R7/R8 hi		; 38

	ylow128s ymm1, ymm14, ymm0		;; Shuffle R1/R2 low and R3/R4 low (final R1)	; 39-40
	yhigh128s ymm14, ymm14, ymm0		;; Shuffle R1/R2 low and R3/R4 low (final R3)	; 40-41
	ystore	[dstreg], ymm1			;; Save R1					; 41
	ylow128s ymm0, ymm8, ymm2		;; Shuffle R1/R2 hi and R3/R4 hi (final R2)	; 41-42
	ystore	[dstreg+e2], ymm14		;; Save R3					; 42
	yhigh128s ymm8, ymm8, ymm2		;; Shuffle R1/R2 hi and R3/R4 hi (final R4)	; 42-43
	ystore	[dstreg+e1], ymm0		;; Save R2					; 43
	ylow128s ymm2, ymm7, ymm10		;; Shuffle R5/R6 low and R7/R8 low (final R5)	; 43-44
	ystore	[dstreg+e2+e1], ymm8		;; Save R4					; 44
	yhigh128s ymm7, ymm7, ymm10		;; Shuffle R5/R6 low and R7/R8 low (final R7)	; 44-45
	ystore	[dstreg+32], ymm2		;; Save R5					; 45
	ylow128s ymm10, ymm3, ymm6		;; Shuffle R5/R6 hi and R7/R8 hi (final R6)	; 45-46
	ystore	[dstreg+e2+32], ymm7		;; Save R7					; 46
	yhigh128s ymm3, ymm3, ymm6		;; Shuffle R5/R6 hi and R7/R8 hi (final R8)	; 46-47
	ystore	[dstreg+e1+32], ymm10		;; Save R6					; 47
	ystore	[dstreg+e2+e1+32], ymm3		;; Save R8					; 48

	bump	srcreg, srcinc
	bump	dstreg, dstinc
	bump	screg1, scinc1
	bump	screg2, scinc2
	ENDM

;; Haswell FMA3 version

IF (@INSTR(,%yarch,<FMA3>) NE 0)

yr4_rsc_sg4cl_2sc_eight_reals_unfft4_preload MACRO
	vmovapd	ymm14, YMM_ONE
	vmovapd	ymm15, YMM_SQRTHALF
	ENDM
yr4_rsc_sg4cl_2sc_eight_reals_unfft4 MACRO srcreg,srcinc,d1,d2,dstreg,dstinc,e1,e2,screg1,scinc1,screg2,scinc2,maxrpt,L1pt,L1pd

	vmovapd	ymm0, [screg2+64+32]		;; cosine/sine for R4/I4
	vmovapd	ymm1, [srcreg+d2+d1]		;; R4
	vmovapd ymm2, [srcreg+d2+d1+32]		;; I4
	yfmaddpd ymm3, ymm1, ymm0, ymm2		;; A4 = R4 * cosine/sine + I4			; 1-4	1-5 (Skylake & Haswell timings)
	yfmsubpd ymm2, ymm2, ymm0, ymm1		;; B4 = I4 * cosine/sine - R4			; 1-4	1-5

	vmovapd	ymm0, [screg1+64]		;; sine for R2/I2
	vmovapd	ymm1, [srcreg+d1+32]		;; I2
	vmulpd	ymm5, ymm1, ymm0		;; C2 = I2 * sine				; 2-5	2-6
	vmovapd	ymm4, [srcreg+d1]		;; R2
	vmulpd	ymm6, ymm4, ymm0		;; D2 = R2 * sine				; 2-5	2-6

	vmovapd	ymm0, [screg2+32]		;; cosine/sine for R3/I3
	vmovapd	ymm7, [srcreg+d2]		;; R3
	vmovapd	ymm8, [srcreg+d2+32]		;; I3
	yfmaddpd ymm9, ymm7, ymm0, ymm8		;; A3 = R3 * cosine/sine + I3 (new R5/sine)	; 3-6	3-7
	yfmsubpd ymm8, ymm8, ymm0, ymm7		;; B3 = I3 * cosine/sine - R3 (new R6/sine)	; 3-6	3-7

	vmovapd	ymm0, [screg2+64]		;; sine for R4/I4
	vmulpd	ymm3, ymm3, ymm0		;; A4 = A4 * sine (new R7)			; 5-8	6-10
	vmulpd	ymm2, ymm2, ymm0		;; B4 = B4 * sine (new R8)			; 5-8	6-10

	vmovapd	ymm0, [screg1+64+32]		;; cosine for R2/I2
	yfmaddpd ymm4, ymm4, ymm0, ymm5		;; A2 = R2 * cosine + C2 (new R3)		; 6-9	7-11
	yfmsubpd ymm1, ymm1, ymm0, ymm6		;; B2 = I2 * cosine - D2 (new R4)		; 6-9	7-11

	vmovapd	ymm0, [screg2]			;; sine for original R3/I3 (new R5/R6)
	yfmsubpd ymm5, ymm9, ymm0, ymm3		;; R5/sine * sine - R7 (newer R6)		; 9-12	11-15
	yfmaddpd ymm9, ymm9, ymm0, ymm3		;; R5/sine * sine + R7 (newer R5)		; 9-12	11-15

	yfmsubpd ymm3, ymm8, ymm0, ymm2		;; R6/sine * sine - R8 (newer R8)		; 10-13	12-16
	yfmaddpd ymm8, ymm8, ymm0, ymm2		;; R6/sine * sine + R8 (newer R7)		; 10-13	12-16

	vmovapd	ymm2, [srcreg]			;; R1
	yfmaddpd ymm0, ymm2, ymm14, ymm4	;; R1 + R3 (newer R1)				; 11-14	13-17
	yfmsubpd ymm2, ymm2, ymm14, ymm4	;; R1 - R3 (newer R3)				; 11-14	13-17

	vmovapd	ymm4, [srcreg+32]		;; I1 (a.k.a new R2)
	yfmaddpd ymm6, ymm4, ymm14, ymm1	;; R2 + R4 (newer R2)				; 12-15	14-18
	yfmsubpd ymm4, ymm4, ymm14, ymm1	;; R2 - R4 (newer R4)				; 12-15	14-18
	L1prefetch srcreg+L1pd, L1pt

	yfmaddpd ymm1, ymm5, ymm14, ymm3	;; R6 = R6 + R8					; 14-17	17-21
	yfmsubpd ymm3, ymm3, ymm14, ymm5	;; R8 = R8 - R6					; 14-17	17-21
	L1prefetch srcreg+d1+L1pd, L1pt

	yfmaddpd ymm5, ymm0, ymm14, ymm9	;; R1 + R5 (final R1)				; 15-18	18-22
	yfmsubpd ymm0, ymm0, ymm14, ymm9	;; R1 - R5 (final R5)				; 15-18	18-22
	L1prefetch srcreg+d2+L1pd, L1pt

	yfmaddpd ymm9, ymm2, ymm14, ymm8	;; R3 + R7 (final R3)				; 16-19	19-23
	yfmsubpd ymm2, ymm2, ymm14, ymm8	;; R3 - R7 (final R7)				; 16-19	19-23
	L1prefetch srcreg+d2+d1+L1pd, L1pt

	yfmaddpd ymm8, ymm1, ymm15, ymm6	;; R2 + R6 * SQRTHALF (final R2)		; 18-21	22-26
	yfnmaddpd ymm1, ymm1, ymm15, ymm6	;; R2 - R6 * SQRTHALF (final R6)		; 18-21	22-26
	yfmaddpd ymm6, ymm3, ymm15, ymm4	;; R4 + R8 * SQRTHALF (final R4)		; 19-22	23-27
	yfnmaddpd ymm3, ymm3, ymm15, ymm4	;; R4 - R8 * SQRTHALF (final R8)		; 19-22	23-27

	vshufpd	ymm4, ymm5, ymm8, 0		;; Shuffle R1 and R2 to create R1/R2 low	; 22	27
	vshufpd	ymm5, ymm5, ymm8, 15		;; Shuffle R1 and R2 to create R1/R2 hi		; 23	28
	vshufpd	ymm8, ymm9, ymm6, 0		;; Shuffle R3 and R4 to create R3/R4 low	; 24	29
	vshufpd	ymm9, ymm9, ymm6, 15		;; Shuffle R3 and R4 to create R3/R4 hi		; 25	30
	vshufpd	ymm6, ymm0, ymm1, 0		;; Shuffle R5 and R6 to create R5/R6 low	; 26	31
	vshufpd	ymm0, ymm0, ymm1, 15		;; Shuffle R5 and R6 to create R5/R6 hi		; 27	32
	vshufpd	ymm1, ymm2, ymm3, 0		;; Shuffle R7 and R8 to create R7/R8 low	; 28	33
	vshufpd	ymm2, ymm2, ymm3, 15		;; Shuffle R7 and R8 to create R7/R8 hi		; 29	34

	ylow128s ymm3, ymm4, ymm8		;; Shuffle R1/R2 low and R3/R4 low (final R1)	; 30-32	35-37
	yhigh128s ymm4, ymm4, ymm8		;; Shuffle R1/R2 low and R3/R4 low (final R3)	; 31-33	36-38
	ystore	[dstreg], ymm3			;; Save R1					; 33	38
	ylow128s ymm8, ymm5, ymm9		;; Shuffle R1/R2 hi and R3/R4 hi (final R2)	; 32-34	37-39
	ystore	[dstreg+e2], ymm4		;; Save R3					; 34	39
	yhigh128s ymm5, ymm5, ymm9		;; Shuffle R1/R2 hi and R3/R4 hi (final R4)	; 33-35	38-40
	ystore	[dstreg+e1], ymm8		;; Save R2					; 35	40
	ylow128s ymm9, ymm6, ymm1		;; Shuffle R5/R6 low and R7/R8 low (final R5)	; 34-36	39-41
	ystore	[dstreg+e2+e1], ymm5		;; Save R4					; 36	41
	yhigh128s ymm6, ymm6, ymm1		;; Shuffle R5/R6 low and R7/R8 low (final R7)	; 35-37	40-42
	ystore	[dstreg+32], ymm9		;; Save R5					; 37	42
	ylow128s ymm1, ymm0, ymm2		;; Shuffle R5/R6 hi and R7/R8 hi (final R6)	; 36-38	41-43
	ystore	[dstreg+e2+32], ymm6		;; Save R7					; 38	43
	yhigh128s ymm0, ymm0, ymm2		;; Shuffle R5/R6 hi and R7/R8 hi (final R8)	; 37-39	42-44
	ystore	[dstreg+e1+32], ymm1		;; Save R6					; 39	44
	ystore	[dstreg+e2+e1+32], ymm0		;; Save R8					; 40	45

	bump	srcreg, srcinc
	bump	dstreg, dstinc
	bump	screg1, scinc1
	bump	screg2, scinc2
	ENDM

ENDIF

ENDIF


; These versions use registers for distances between blocks.  This lets us share pass1 code.

;; Used in last levels of pass 1.  Swizzling.
;; In this particular case, we know that the sin/cos multipliers for R1/I1 are one.  Thus
;; we can get by with only 3 sin/cos multiplies.
yr4_rsc_sg4clreg_2sc_eight_reals_unfft4_preload MACRO
	ENDM
yr4_rsc_sg4clreg_2sc_eight_reals_unfft4 MACRO srcreg,srcinc,d1reg,d3reg,dstreg,dstinc,e1,e2,screg1,scinc1,screg2,scinc2,maxrpt,L1pt,L1preg
	NOT IMPLMENTED IN 32-BIT
	ENDM

IFDEF X86_64

yr4_rsc_sg4clreg_2sc_eight_reals_unfft4_preload MACRO
	vmovapd	ymm15, YMM_SQRTHALF
	ENDM
yr4_rsc_sg4clreg_2sc_eight_reals_unfft4 MACRO srcreg,srcinc,d1reg,d3reg,dstreg,dstinc,e1,e2,screg1,scinc1,screg2,scinc2,maxrpt,L1pt,L1preg

	vmovapd	ymm0, [screg2+32]		;; cosine/sine for R3/I3
	vmovapd	ymm1, [srcreg+2*d1reg]		;; R3
	vmulpd	ymm2, ymm1, ymm0		;; A3 = R3 * cosine/sine			;  1-5

	vmovapd	ymm3, [screg2+64+32]		;; cosine/sine for R4/I4
	vmovapd	ymm4, [srcreg+d3reg]		;; R4
	vmulpd	ymm5, ymm4, ymm3		;; A4 = R4 * cosine/sine			;  2-6

	vmovapd	ymm6, [srcreg+2*d1reg+32]	;; I3
	vmulpd	ymm0, ymm6, ymm0		;; B3 = I3 * cosine/sine			;  3-7

	vmovapd ymm7, [srcreg+d3reg+32]		;; I4
	vmulpd	ymm3, ymm7, ymm3		;; B4 = I4 * cosine/sine			;  4-8

	vmovapd	ymm8, [screg1+64+32]		;; cosine for R2/I2
	vmovapd	ymm9, [srcreg+d1reg]		;; R2
	vmulpd	ymm10, ymm9, ymm8		;; A2 = R2 * cosine				;  5-9

	vmovapd	ymm11, [srcreg+d1reg+32]	;; I2
	vaddpd	ymm2, ymm2, ymm6		;; A3 = A3 + I3					; 6-8
	vmulpd	ymm8, ymm11, ymm8		;; B2 = I2 * cosine				;  6-10

	vmovapd	ymm12, [screg1+64]		;; sine for R2/I2
	vaddpd	ymm5, ymm5, ymm7		;; A4 = A4 + I4					; 7-9
	vmulpd	ymm7, ymm11, ymm12		;; C2 = I2 * sine				;  7-11

	vsubpd	ymm0, ymm0, ymm1		;; B3 = B3 - R3					; 8-10
	vmulpd	ymm12, ymm9, ymm12		;; D2 = R2 * sine				;  8-12

	vmovapd	ymm13, [screg2]			;; sine for R3/I3
	vsubpd	ymm3, ymm3, ymm4		;; B4 = B4 - R4					; 9-11
	vmulpd	ymm2, ymm2, ymm13		;; A3 = A3 * sine (new R5)			;  9-13

	vmovapd	ymm14, [screg2+64]		;; sine for R4/I4
	vmulpd	ymm5, ymm5, ymm14		;; A4 = A4 * sine (new R7)			;  10-14

	vmulpd	ymm0, ymm0, ymm13		;; B3 = B3 * sine (new R6)			;  11-15
	vmovapd	ymm6, [srcreg]			;; R1

	vaddpd	ymm10, ymm10, ymm7		;; A2 = A2 + C2 (new R3)			; 12-14
	vmulpd	ymm3, ymm3, ymm14		;; B4 = B4 * sine (new R8)			;  12-16

	vsubpd	ymm8, ymm8, ymm12		;; B2 = B2 - D2 (new R4)			; 13-15
	vmovapd	ymm1, [srcreg+32]		;; I1 (a.k.a new R2)

	L1prefetch L1preg, L1pt

	vsubpd	ymm12, ymm2, ymm5		;; R5 - R7 (newer R6)				; 15-17

	vaddpd	ymm2, ymm2, ymm5		;; R5 + R7 (newer R5)				; 16-18

	vsubpd	ymm5, ymm0, ymm3		;; R6 - R8 (newer R8)				; 17-19
	L1prefetch L1preg+d1reg, L1pt

	vaddpd	ymm0, ymm0, ymm3		;; R6 + R8 (newer R7)				; 18-20

	vaddpd	ymm3, ymm6, ymm10		;; R1 + R3 (newer R1)				; 19-21

	vaddpd	ymm14, ymm12, ymm5		;; R6 = R6 + R8					; 20-22

	vsubpd	ymm5, ymm5, ymm12		;; R8 = R8 - R6					; 21-23
	L1prefetch L1preg+2*d1reg, L1pt

	vsubpd	ymm6, ymm6, ymm10		;; R1 - R3 (newer R3)				; 22-24

	vaddpd	ymm10, ymm1, ymm8		;; R2 + R4 (newer R2)				; 23-25
	vmulpd	ymm14, ymm14, ymm15		;; R6 = R6 * square root of 1/2			;  23-27

	vsubpd	ymm1, ymm1, ymm8		;; R2 - R4 (newer R4)				; 24-26
	vmulpd	ymm5, ymm5, ymm15		;; R8 = R8 * square root of 1/2			;  24-28

	vaddpd	ymm8, ymm3, ymm2		;; R1 + R5 (final R1)				; 25-27
	L1prefetch L1preg+d3reg, L1pt

	vsubpd	ymm3, ymm3, ymm2		;; R1 - R5 (final R5)				; 26-28

	vaddpd	ymm2, ymm6, ymm0		;; R3 + R7 (final R3)				; 27-29

	vaddpd	ymm12, ymm10, ymm14		;; R2 + R6 (final R2)				; 28-30

	vaddpd	ymm7, ymm1, ymm5		;; R4 + R8 (final R4)				; 29-31

	vsubpd	ymm10, ymm10, ymm14		;; R2 - R6 (final R6)				; 30-32

	vshufpd	ymm14, ymm8, ymm12, 0		;; Shuffle R1 and R2 to create R1/R2 low	; 31
	vsubpd	ymm1, ymm1, ymm5		;; R4 - R8 (final R8)				; 31-33

	vshufpd	ymm8, ymm8, ymm12, 15		;; Shuffle R1 and R2 to create R1/R2 hi		; 32
	vsubpd	ymm6, ymm6, ymm0		;; R3 - R7 (final R7)				; 32-34

	vshufpd	ymm0, ymm2, ymm7, 0		;; Shuffle R3 and R4 to create R3/R4 low	; 33
	vshufpd	ymm2, ymm2, ymm7, 15		;; Shuffle R3 and R4 to create R3/R4 hi		; 34
	vshufpd	ymm7, ymm3, ymm10, 0		;; Shuffle R5 and R6 to create R5/R6 low	; 35
	vshufpd	ymm3, ymm3, ymm10, 15		;; Shuffle R5 and R6 to create R5/R6 hi		; 36
	vshufpd	ymm10, ymm6, ymm1, 0		;; Shuffle R7 and R8 to create R7/R8 low	; 37
	vshufpd	ymm6, ymm6, ymm1, 15		;; Shuffle R7 and R8 to create R7/R8 hi		; 38

	ylow128s ymm1, ymm14, ymm0		;; Shuffle R1/R2 low and R3/R4 low (final R1)	; 39-40
	yhigh128s ymm14, ymm14, ymm0		;; Shuffle R1/R2 low and R3/R4 low (final R3)	; 40-41
	ystore	[dstreg], ymm1			;; Save R1					; 41
	ylow128s ymm0, ymm8, ymm2		;; Shuffle R1/R2 hi and R3/R4 hi (final R2)	; 41-42
	ystore	[dstreg+e2], ymm14		;; Save R3					; 42
	yhigh128s ymm8, ymm8, ymm2		;; Shuffle R1/R2 hi and R3/R4 hi (final R4)	; 42-43
	ystore	[dstreg+e1], ymm0		;; Save R2					; 43
	ylow128s ymm2, ymm7, ymm10		;; Shuffle R5/R6 low and R7/R8 low (final R5)	; 43-44
	ystore	[dstreg+e2+e1], ymm8		;; Save R4					; 44
	yhigh128s ymm7, ymm7, ymm10		;; Shuffle R5/R6 low and R7/R8 low (final R7)	; 44-45
	ystore	[dstreg+32], ymm2		;; Save R5					; 45
	ylow128s ymm10, ymm3, ymm6		;; Shuffle R5/R6 hi and R7/R8 hi (final R6)	; 45-46
	ystore	[dstreg+e2+32], ymm7		;; Save R7					; 46
	yhigh128s ymm3, ymm3, ymm6		;; Shuffle R5/R6 hi and R7/R8 hi (final R8)	; 46-47
	ystore	[dstreg+e1+32], ymm10		;; Save R6					; 47
	ystore	[dstreg+e2+e1+32], ymm3		;; Save R8					; 48

	bump	srcreg, srcinc
	bump	dstreg, dstinc
	bump	screg1, scinc1
	bump	screg2, scinc2
	bump	L1preg, srcinc
	ENDM

;; Haswell FMA3 version

IF (@INSTR(,%yarch,<FMA3>) NE 0)

yr4_rsc_sg4clreg_2sc_eight_reals_unfft4_preload MACRO
	vmovapd	ymm14, YMM_ONE
	vmovapd	ymm15, YMM_SQRTHALF
	ENDM
yr4_rsc_sg4clreg_2sc_eight_reals_unfft4 MACRO srcreg,srcinc,d1reg,d3reg,dstreg,dstinc,e1,e2,screg1,scinc1,screg2,scinc2,maxrpt,L1pt,L1preg

	vmovapd	ymm0, [screg2+64+32]		;; cosine/sine for R4/I4
	vmovapd	ymm1, [srcreg+d3reg]		;; R4
	vmovapd ymm2, [srcreg+d3reg+32]		;; I4
	yfmaddpd ymm3, ymm1, ymm0, ymm2		;; A4 = R4 * cosine/sine + I4			; 1-4	1-5 (Skylake & Haswell timings)
	yfmsubpd ymm2, ymm2, ymm0, ymm1		;; B4 = I4 * cosine/sine - R4			; 1-4	1-5

	vmovapd	ymm0, [screg1+64]		;; sine for R2/I2
	vmovapd	ymm1, [srcreg+d1reg+32]		;; I2
	vmulpd	ymm5, ymm1, ymm0		;; C2 = I2 * sine				; 2-5	2-6
	vmovapd	ymm4, [srcreg+d1reg]		;; R2
	vmulpd	ymm6, ymm4, ymm0		;; D2 = R2 * sine				; 2-5	2-6

	vmovapd	ymm0, [screg2+32]		;; cosine/sine for R3/I3
	vmovapd	ymm7, [srcreg+2*d1reg]		;; R3
	vmovapd	ymm8, [srcreg+2*d1reg+32]	;; I3
	yfmaddpd ymm9, ymm7, ymm0, ymm8		;; A3 = R3 * cosine/sine + I3 (new R5/sine)	; 3-6	3-7
	yfmsubpd ymm8, ymm8, ymm0, ymm7		;; B3 = I3 * cosine/sine - R3 (new R6/sine)	; 3-6	3-7

	vmovapd	ymm0, [screg2+64]		;; sine for R4/I4
	vmulpd	ymm3, ymm3, ymm0		;; A4 = A4 * sine (new R7)			; 5-8	6-10
	vmulpd	ymm2, ymm2, ymm0		;; B4 = B4 * sine (new R8)			; 5-8	6-10

	vmovapd	ymm0, [screg1+64+32]		;; cosine for R2/I2
	yfmaddpd ymm4, ymm4, ymm0, ymm5		;; A2 = R2 * cosine + C2 (new R3)		; 6-9	7-11
	yfmsubpd ymm1, ymm1, ymm0, ymm6		;; B2 = I2 * cosine - D2 (new R4)		; 6-9	7-11

	vmovapd	ymm0, [screg2]			;; sine for original R3/I3 (new R5/R6)
	yfmsubpd ymm5, ymm9, ymm0, ymm3		;; R5/sine * sine - R7 (newer R6)		; 9-12	11-15
	yfmaddpd ymm9, ymm9, ymm0, ymm3		;; R5/sine * sine + R7 (newer R5)		; 9-12	11-15

	yfmsubpd ymm3, ymm8, ymm0, ymm2		;; R6/sine * sine - R8 (newer R8)		; 10-13	12-16
	yfmaddpd ymm8, ymm8, ymm0, ymm2		;; R6/sine * sine + R8 (newer R7)		; 10-13	12-16

	vmovapd	ymm2, [srcreg]			;; R1
	yfmaddpd ymm0, ymm2, ymm14, ymm4	;; R1 + R3 (newer R1)				; 11-14	13-17
	yfmsubpd ymm2, ymm2, ymm14, ymm4	;; R1 - R3 (newer R3)				; 11-14	13-17

	vmovapd	ymm4, [srcreg+32]		;; I1 (a.k.a new R2)
	yfmaddpd ymm6, ymm4, ymm14, ymm1	;; R2 + R4 (newer R2)				; 12-15	14-18
	yfmsubpd ymm4, ymm4, ymm14, ymm1	;; R2 - R4 (newer R4)				; 12-15	14-18
	L1prefetch L1preg, L1pt

	yfmaddpd ymm1, ymm5, ymm14, ymm3	;; R6 = R6 + R8					; 14-17	17-21
	yfmsubpd ymm3, ymm3, ymm14, ymm5	;; R8 = R8 - R6					; 14-17	17-21
	L1prefetch L1preg+d1reg, L1pt

	yfmaddpd ymm5, ymm0, ymm14, ymm9	;; R1 + R5 (final R1)				; 15-18	18-22
	yfmsubpd ymm0, ymm0, ymm14, ymm9	;; R1 - R5 (final R5)				; 15-18	18-22
	L1prefetch L1preg+2*d1reg, L1pt

	yfmaddpd ymm9, ymm2, ymm14, ymm8	;; R3 + R7 (final R3)				; 16-19	19-23
	yfmsubpd ymm2, ymm2, ymm14, ymm8	;; R3 - R7 (final R7)				; 16-19	19-23
	L1prefetch L1preg+d3reg, L1pt

	yfmaddpd ymm8, ymm1, ymm15, ymm6	;; R2 + R6 * SQRTHALF (final R2)		; 18-21	22-26
	yfnmaddpd ymm1, ymm1, ymm15, ymm6	;; R2 - R6 * SQRTHALF (final R6)		; 18-21	22-26
	yfmaddpd ymm6, ymm3, ymm15, ymm4	;; R4 + R8 * SQRTHALF (final R4)		; 19-22	23-27
	yfnmaddpd ymm3, ymm3, ymm15, ymm4	;; R4 - R8 * SQRTHALF (final R8)		; 19-22	23-27

	vshufpd	ymm4, ymm5, ymm8, 0		;; Shuffle R1 and R2 to create R1/R2 low	; 22	27
	vshufpd	ymm5, ymm5, ymm8, 15		;; Shuffle R1 and R2 to create R1/R2 hi		; 23	28
	vshufpd	ymm8, ymm9, ymm6, 0		;; Shuffle R3 and R4 to create R3/R4 low	; 24	29
	vshufpd	ymm9, ymm9, ymm6, 15		;; Shuffle R3 and R4 to create R3/R4 hi		; 25	30
	vshufpd	ymm6, ymm0, ymm1, 0		;; Shuffle R5 and R6 to create R5/R6 low	; 26	31
	vshufpd	ymm0, ymm0, ymm1, 15		;; Shuffle R5 and R6 to create R5/R6 hi		; 27	32
	vshufpd	ymm1, ymm2, ymm3, 0		;; Shuffle R7 and R8 to create R7/R8 low	; 28	33
	vshufpd	ymm2, ymm2, ymm3, 15		;; Shuffle R7 and R8 to create R7/R8 hi		; 29	34

	ylow128s ymm3, ymm4, ymm8		;; Shuffle R1/R2 low and R3/R4 low (final R1)	; 30-32	35-37
	yhigh128s ymm4, ymm4, ymm8		;; Shuffle R1/R2 low and R3/R4 low (final R3)	; 31-33	36-38
	ystore	[dstreg], ymm3			;; Save R1					; 33	38
	ylow128s ymm8, ymm5, ymm9		;; Shuffle R1/R2 hi and R3/R4 hi (final R2)	; 32-34	37-39
	ystore	[dstreg+e2], ymm4		;; Save R3					; 34	39
	yhigh128s ymm5, ymm5, ymm9		;; Shuffle R1/R2 hi and R3/R4 hi (final R4)	; 33-35	38-40
	ystore	[dstreg+e1], ymm8		;; Save R2					; 35	40
	ylow128s ymm9, ymm6, ymm1		;; Shuffle R5/R6 low and R7/R8 low (final R5)	; 34-36	39-41
	ystore	[dstreg+e2+e1], ymm5		;; Save R4					; 36	41
	yhigh128s ymm6, ymm6, ymm1		;; Shuffle R5/R6 low and R7/R8 low (final R7)	; 35-37	40-42
	ystore	[dstreg+32], ymm9		;; Save R5					; 37	42
	ylow128s ymm1, ymm0, ymm2		;; Shuffle R5/R6 hi and R7/R8 hi (final R6)	; 36-38	41-43
	ystore	[dstreg+e2+32], ymm6		;; Save R7					; 38	43
	yhigh128s ymm0, ymm0, ymm2		;; Shuffle R5/R6 hi and R7/R8 hi (final R8)	; 37-39	42-44
	ystore	[dstreg+e1+32], ymm1		;; Save R6					; 39	44
	ystore	[dstreg+e2+e1+32], ymm0		;; Save R8					; 40	45

	bump	srcreg, srcinc
	bump	dstreg, dstinc
	bump	screg1, scinc1
	bump	screg2, scinc2
	bump	L1preg, srcinc
	ENDM

ENDIF

ENDIF



;; New optimizations in 2024.

IF (@INSTR(,%yarch,<FMA3>) NE 0)

; Use all 16 registers in first and last one-pass fft macros

yr4_fs4cl_four_complex_first_djbfft_preload MACRO
	vbroadcastsd ymm15, YMM_P924
	vbroadcastsd ymm14, YMM_P383
	vbroadcastsd ymm13, YMM_SQRTHALF
	vbroadcastsd ymm12, YMM_ONE
	ENDM

yr4_fs4cl_four_complex_first_djbfft MACRO srcreg,srcinc,d1,d2,screg,scinc,maxrpt,L1pt,L1pd
	vmovapd	ymm8, [srcreg+d2][rbx]		;; R3				; Haswell	KabyLake
	vmulpd	ymm8, ymm8, ymm13		;; A3 = R3 * SQRTHALF (new R3)	; 1-5		1-4

	vmovapd	ymm6, [srcreg+d1][rbx]		;; R2
	vmulpd	ymm4, ymm6, ymm15		;; A2 = R2 * .924		; 2-6		2-5
	vmulpd	ymm6, ymm6, ymm14		;; B2 = R2 * .383		; 2-6		2-5

	vmovapd	ymm7, [srcreg+d2+d1][rbx]	;; R4
	vmulpd	ymm5, ymm7, ymm14		;; A4 = R4 * .383		; 3-7		3-6
	vmulpd	ymm7, ymm7, ymm15		;; B4 = R4 * .924		; 3-7		3-6

	vmovapd	ymm10, [srcreg+d2+32][rbx]	;; R7 (I3)
	yfnmaddpd ymm9, ymm10, ymm13, ymm8	;; new R3 = A3 - I3*SQRTHALF	; 6-10		5-8
	yfmaddpd ymm8, ymm10, ymm13, ymm8	;; new I3 = A3 + I3*SQRTHALF	; 6-10		5-8

	vmovapd	ymm2, [srcreg+d1+32][rbx]	;; R6 (I2)
	yfnmaddpd ymm4, ymm2, ymm14, ymm4	;; new R2 = A2 - .383*I2	; 7-11		6-9
	yfmaddpd ymm6, ymm2, ymm15, ymm6	;; new I2 = B2 + .924*I2	; 7-11		6-9

	vmovapd	ymm1, [srcreg+d2+d1+32][rbx]	;; R8 (I4)
	yfnmaddpd ymm5, ymm1, ymm15, ymm5	;; new R4 = A4 - .924*I4	; 8-12		7-10
	yfmaddpd ymm7, ymm1, ymm14, ymm7	;; new I4 = B4 + .383*I4	; 8-12		7-10

; two aparts

	vmovapd	ymm0, [srcreg][rbx]		;; R1
	yfmsubpd ymm2, ymm0, ymm12, ymm9	;; R1 - R3 (new R3)		; 11-15		9-12
	yfmaddpd ymm0, ymm0, ymm12, ymm9	;; R1 + R3 (new R1)		; 11-15		9-12

	vmovapd	ymm9, [srcreg+32][rbx]		;; R5 (I1)
	yfmsubpd ymm10, ymm9, ymm12, ymm8	;; I1 - I3 (new I3)		; 12-16		10-13
	yfmaddpd ymm9, ymm9, ymm12, ymm8	;; I1 + I3 (new I1)		; 12-16		10-13

	vsubpd	ymm3, ymm4, ymm5		;; R2 - R4 (new R4)		; 13-15		11-14
	vaddpd	ymm4, ymm4, ymm5		;; R2 + R4 (new R2)		; 14-16		11-14

	vsubpd	ymm5, ymm6, ymm7		;; I2 - I4 (new I4)		; 15-17		12-15
	vaddpd	ymm6, ymm6, ymm7		;; I2 + I4 (new I2)		; 16-18		12-15

; one aparts

	yfmsubpd ymm7, ymm10, ymm12, ymm3	;; I3 - R4 (new I4)		; 17-21		15-18
	yfmaddpd ymm10, ymm10, ymm12, ymm3	;; I3 + R4 (new I3)		; 17-21		15-18
	yfmsubpd ymm3, ymm2, ymm12, ymm5	;; R3 - I4 (new R3)		; 18-22		16-19
	yfmaddpd ymm2, ymm2, ymm12, ymm5	;; R3 + I4 (new R4)		; 18-22		16-19
	yfmsubpd ymm5, ymm0, ymm12, ymm4	;; R1 - R2 (new R2)		; 19-23		17-20
	yfmsubpd ymm1, ymm9, ymm12, ymm6	;; I1 - I2 (new I2)		; 19-23		17-20
	yfmaddpd ymm0, ymm0, ymm12, ymm4	;; R1 + R2 (new R1)		; 20-24		18-21
	yfmaddpd ymm9, ymm9, ymm12, ymm6	;; I1 + I2 (new I1)		; 20-24		18-21

	vmovapd	ymm8, [screg+64+32]		;; cosine/sine
	yfmsubpd ymm4, ymm3, ymm8, ymm10	;; A3 = R3 * cosine/sine - I3	; 23-27		20-23
	yfmaddpd ymm10, ymm10, ymm8, ymm3	;; B3 = I3 * cosine/sine + R3	; 23-27		20-23

	vmovapd	ymm8, [screg+192+32]		;; cosine/sine
	yfmsubpd ymm3, ymm2, ymm8, ymm7		;; A4 = R4 * cosine/sine - I4	; 24-28		21-24
	yfmaddpd ymm7, ymm7, ymm8, ymm2		;; B4 = I4 * cosine/sine + R4	; 24-28		21-24

	vmovapd	ymm8, [screg+128+32]		;; cosine/sine
	yfmsubpd ymm2, ymm5, ymm8, ymm1		;; A2 = R2 * cosine/sine - I2	; 25-29		22-25
	yfmaddpd ymm1, ymm1, ymm8, ymm5		;; B2 = I2 * cosine/sine + R2	; 25-29		22-25

	vmovapd	ymm8, [screg+0+32]		;; cosine/sine
	yfmsubpd ymm5, ymm0, ymm8, ymm9		;; A1 = R1 * cosine/sine - I1	; 26-30		23-26
	yfmaddpd ymm9, ymm9, ymm8, ymm0		;; B1 = I1 * cosine/sine + R1	; 26-30		23-26

;preloads
	vmovapd	ymm11, [screg+64]		;; sine
	vmulpd	ymm4, ymm4, ymm11		;; A3 = A3 * sine (final R3)	; 28-32		24-27
	vmulpd	ymm10, ymm10, ymm11		;; B3 = B3 * sine (final I3)	; 28-32		24-27

	vmovapd	ymm8, [screg+192]		;; sine
	vmulpd	ymm3, ymm3, ymm8		;; A4 = A4 * sine (final R4)	; 29-33		25-28
	vmulpd	ymm7, ymm7, ymm8		;; B4 = B4 * sine (final I4)	; 29-33		25-28

	vmovapd	ymm11, [screg+128]		;; sine
	vmulpd	ymm2, ymm2, ymm11		;; A2 = A2 * sine (final R2)	; 30-34		26-29
	vmulpd	ymm1, ymm1, ymm11		;; B2 = B2 * sine (final I2)	; 30-34		26-29

	vmovapd	ymm8, [screg+0]		;; sine
	vmulpd	ymm5, ymm5, ymm8		;; A1 = A1 * sine (final R1)	; 31-35		27-30
	vmulpd	ymm9, ymm9, ymm8		;; B1 = B1 * sine (final I1)	; 31-35		27-30
	bump	screg, scinc

	L1prefetchw srcreg+d2+L1pd, L1pt
	vshufpd	ymm0, ymm5, ymm2, 0		;; Shuffle R1 and R2 to create 0 8 2 10
	vshufpd	ymm5, ymm5, ymm2, 15		;; Shuffle R1 and R2 to create 1 9 3 11

	L1prefetchw srcreg+d1+L1pd, L1pt
	vshufpd	ymm2, ymm4, ymm3, 0		;; Shuffle R3 and R4 to create 16 24 18 26
	vshufpd	ymm4, ymm4, ymm3, 15		;; Shuffle R3 and R4 to create 17 25 19 27

	L1prefetchw srcreg+d2+d1+L1pd, L1pt
	ylow128s ymm3, ymm0, ymm2		;; Shuffle R1/R2 low and R3/R4 low (0 8 16 24)
	yhigh128s ymm0, ymm0, ymm2		;; Shuffle R1/R2 low and R3/R4 low (2 10 18 26)

	L1prefetchw srcreg+L1pd, L1pt
	ylow128s ymm2, ymm5, ymm4		;; Shuffle R1/R2 hi and R3/R4 hi (1 9 17 25)
	yhigh128s ymm5, ymm5, ymm4		;; Shuffle R1/R2 hi and R3/R4 hi (3 11 19 27)

	ystore	[srcreg], ymm3			;; Save (0 8 16 24)
	ystore	[srcreg+d2], ymm0		;; Save (2 10 18 26)
	ystore	[srcreg+d1], ymm2		;; Save (1 9 17 25)
	ystore	[srcreg+d2+d1], ymm5		;; Save (3 11 19 27)

	vshufpd	ymm0, ymm9, ymm1, 0		;; Shuffle I1 and I2 to create 4 12 6 14
	vshufpd	ymm9, ymm9, ymm1, 15		;; Shuffle I1 and I2 to create 5 13 7 15

	vshufpd	ymm1, ymm10, ymm7, 0		;; Shuffle I3 and I4 to create 16 28 22 30
	vshufpd	ymm10, ymm10, ymm7, 15		;; Shuffle I3 and I4 to create 21 29 23 31

	ylow128s ymm7, ymm0, ymm1		;; Shuffle I1/I2 low and I3/I4 low (4 12 20 28)
	yhigh128s ymm0, ymm0, ymm1		;; Shuffle I1/I2 low and I3/I4 low (6 14 22 30)

	ylow128s ymm1, ymm9, ymm10		;; Shuffle I1/I2 hi and I3/I4 hi (5 13 21 29)
	yhigh128s ymm9, ymm9, ymm10		;; Shuffle I1/I2 hi and I3/I4 hi (7 15 23 31)

	ystore	[srcreg+32], ymm7		;; Save (4 12 20 28)
	ystore	[srcreg+d2+32], ymm0		;; Save (6 14 22 30)
	ystore	[srcreg+d1+32], ymm1		;; Save (5 13 21 29)
	ystore	[srcreg+d2+d1+32], ymm9		;; Save (7 15 23 31)
	bump	srcreg, srcinc
	ENDM

yr4_s4cl_four_complex_last_unfft_preload MACRO
	vbroadcastsd ymm15, YMM_P924
	vbroadcastsd ymm14, YMM_P383
	vbroadcastsd ymm13, YMM_SQRTHALF
	ENDM

yr4_s4cl_four_complex_last_unfft MACRO srcreg,srcinc,d1,d2,screg,scinc,maxrpt,L1pt,L1pd

	;; Swizzle using 8 shufpds and 8 perm2f128  (1 or 3 clocks each on port 5, throughput 1)

	vmovapd	ymm1, [srcreg]			;; R1							; Haswell	KabyLake
	vmovapd	ymm7, [srcreg+d1]		;; R2
	vshufpd	ymm0, ymm1, ymm7, 15		;; Shuffle R1 and R2 to create 8 9 24 25		; 1
	vshufpd	ymm1, ymm1, ymm7, 0		;; Shuffle R1 and R2 to create 0 1 16 17		; 2

	vmovapd	ymm2, [srcreg+d2]		;; R3
	vmovapd	ymm7, [srcreg+d2+d1]		;; R4
	vshufpd	ymm3, ymm2, ymm7, 15		;; Shuffle R3 and R4 to create 10 11 26 27		; 3
	vshufpd	ymm2, ymm2, ymm7, 0		;; Shuffle R3 and R4 to create 2 3 18 19		; 4

	ylow128s ymm4, ymm0, ymm3		;; Shuffle R1/R2 hi and R3/R4 hi (8 9 10 11)		; 5-7
	yhigh128s ymm0, ymm0, ymm3		;; Shuffle R1/R2 hi and R3/R4 hi (24 25 26 27)		; 6-8

	ylow128s ymm3, ymm1, ymm2		;; Shuffle R1/R2 low and R3/R4 low (0 1 2 3)		; 7-9
	yhigh128s ymm1, ymm1, ymm2		;; Shuffle R1/R2 low and R3/R4 low (16 17 18 19)	; 8-10

	vmovapd	ymm6, [srcreg+32]		;; I1
	vmovapd	ymm2, [srcreg+d1+32]		;; I2
	vshufpd	ymm5, ymm6, ymm2, 15		;; Shuffle I1 and I2 to create 12 13 28 29		; 9
	vshufpd	ymm6, ymm6, ymm2, 0		;; Shuffle I1 and I2 to create 4 5 20 21		; 10

	vmovapd	ymm2, [srcreg+d2+32]		;; I3
	vmovapd	ymm7, [srcreg+d2+d1+32]		;; I4
	vshufpd	ymm8, ymm2, ymm7, 15		;; Shuffle I3 and I4 to create 14 15 30 31		; 11
	vshufpd	ymm2, ymm2, ymm7, 0		;; Shuffle I3 and I4 to create 6 7 22 23		; 12

	vmovapd	ymm12, [screg+128+32]		;; cosine/sine						; n 16
	ylow128s ymm7, ymm5, ymm8		;; Shuffle I1/I2 hi and I3/I4 hi (12 13 14 15)		; 13-15
	yhigh128s ymm5, ymm5, ymm8		;; Shuffle I1/I2 hi and I3/I4 hi (28 29 30 31)		; 14-16

	vmovapd	ymm11, [screg+192+32]		;; cosine/sine						; n 17
	yhigh128s ymm8, ymm6, ymm2		;; Shuffle I1/I2 low and I3/I4 low (20 21 22 23)	; 15-17
	ylow128s ymm6, ymm6, ymm2		;; Shuffle I1/I2 low and I3/I4 low (4 5 6 7)		; 16-18

	vmovapd	ymm10, [screg+64+32]		;; cosine/sine						; n 18
	yfmaddpd ymm2, ymm4, ymm12, ymm7	;; A2 = R2 * cosine/sine + I2				; 16-20		16-19
	yfmsubpd ymm7, ymm7, ymm12, ymm4	;; B2 = I2 * cosine/sine - R2				; 16-20		16-19

	vmovapd	ymm9, [screg+0+32]		;; cosine/sine						; n 19
	yfmaddpd ymm4, ymm0, ymm11, ymm5	;; A4 = R4 * cosine/sine + I4				; 17-21		17-20
	yfmsubpd ymm5, ymm5, ymm11, ymm0	;; B4 = I4 * cosine/sine - R4				; 17-21		17-20

	vmovapd	ymm12, [screg+128]		;; sine							; n 21		20
	yfmaddpd ymm0, ymm1, ymm10, ymm8	;; A3 = R3 * cosine/sine + I3				; 18-22		18-21
	yfmsubpd ymm8, ymm8, ymm10, ymm1	;; B3 = I3 * cosine/sine - R3				; 18-22		18-21

	vmovapd	ymm11, [screg+192]		;; sine							; n 22		21
	yfmaddpd ymm1, ymm3, ymm9, ymm6		;; A1 = R1 * cosine/sine + I1				; 19-23		19-22
	yfmsubpd ymm6, ymm6, ymm9, ymm3		;; B1 = I1 * cosine/sine - R1				; 19-23		19-22

	vmovapd	ymm10, [screg+0]		;; sine							; n 26		24
	vmulpd	ymm2, ymm2, ymm12		;; A2 = A2 * sine (new R2)				; 21-25		20-23
	vmulpd	ymm7, ymm7, ymm12		;; B2 = B2 * sine (new I2)				; 21-25		20-23

	vmovapd	ymm9, [screg+64]		;; sine							; n 27		25
	vmulpd	ymm4, ymm4, ymm11		;; A4 = A4 * sine (new R4)				; 22-26		21-24
	vmulpd	ymm5, ymm5, ymm11		;; B4 = B4 * sine (new I4)				; 22-26		21-24
	bump	screg, scinc

	yfmsubpd ymm3, ymm1, ymm10, ymm2	;; R1*sine - R2 (new R2)				; 26-30		24-27
	yfmaddpd ymm1, ymm1, ymm10, ymm2	;; R1*sine + R2 (new R1)				; 26-30		24-27
	L1prefetchw srcreg+L1pd, L1pt

	yfnmaddpd ymm2, ymm0, ymm9, ymm4	;; R4 = R4 - R3*sine (new I4)				; 27-31		25-28
	yfmaddpd ymm4, ymm0, ymm9, ymm4		;; R3 = R4 + R3*sine (new R3)				; 27-31		25-28
	L1prefetchw srcreg+d1+L1pd, L1pt

	yfmsubpd ymm0, ymm8, ymm9, ymm5		;; I3 = I3*sine - I4 (new R4)				; 28-32		26-29
	yfmaddpd ymm8, ymm8, ymm9, ymm5		;; I4 = I3*sine + I4 (new I3)				; 28-32		26-29
	L1prefetchw srcreg+d2+L1pd, L1pt

	yfmsubpd ymm5, ymm6, ymm10, ymm7	;; I1*sine - I2 (new I2)				; 29-33		27-30
	yfmaddpd ymm6, ymm6, ymm10, ymm7	;; I1*sine + I2 (new I1)				; 29-33		27-30
	L1prefetchw srcreg+d2+d1+L1pd, L1pt

	vsubpd	ymm7, ymm1, ymm4		;; R1 - R3 (new R3)					; 32-34		29-32
	vaddpd	ymm1, ymm1, ymm4		;; R1 + R3 (final R1)					; 33-35		29-32

	vsubpd	ymm4, ymm3, ymm0		;; R2 - R4 (new R4)					; 34-36		30-33
	vaddpd	ymm3, ymm3, ymm0		;; R2 + R4 (new R2)					; 35-37		30-33

	vsubpd	ymm0, ymm6, ymm8		;; I1 - I3 (new I3)					; 36-38		31-34
	vaddpd	ymm6, ymm6, ymm8		;; I1 + I3 (final I1)					; 37-39		31-34

	vsubpd	ymm8, ymm5, ymm2		;; I2 - I4 (new I4)					; 38-40		32-35
	vaddpd	ymm5, ymm5, ymm2		;; I2 + I4 (new I2)					; 39-41		32-35

	vmulpd	ymm7, ymm7, ymm13		;; A3 = R3 * SQRTHALF					; 35-39		33-36
	ystore	[srcreg], ymm1			;; Save R1						; 36		33

	vmulpd	ymm2, ymm4, ymm14		;; A4 = R4 * .383					; 37-41		34-37
	vmulpd	ymm4, ymm4, ymm15		;; B4 = R4 * .924					; 37-41		34-37

	vmulpd	ymm1, ymm3, ymm15		;; A2 = R2 * .924					; 38-42		35-38
	vmulpd	ymm3, ymm3, ymm14		;; B2 = R2 * .383					; 38-42		35-38
	ystore	[srcreg+32], ymm6		;; Save I1						; 40		35

	yfmsubpd ymm9, ymm0, ymm13, ymm7	;; final I3 = I3*SQRTHALF - A3				; 40-44		37-40
	yfmaddpd ymm0, ymm0, ymm13, ymm7	;; final R3 = I3*SQRTHALF + A3				; 40-44		37-40

	yfmaddpd ymm2, ymm8, ymm15, ymm2	;; final R4 = A4 + .924*I4				; 42-46		38-41
	yfmsubpd ymm4, ymm8, ymm14, ymm4	;; final I4 = .383*I4 - B4				; 42-46		38-41

	yfmaddpd ymm1, ymm5, ymm14, ymm1	;; final R2 = A2 + .383*I2				; 43-47		39-42
	yfmsubpd ymm3, ymm5, ymm15, ymm3	;; final I2 = .924*I2 - B2				; 43-47		39-42

	ystore	[srcreg+d2], ymm0		;; Save R3
	ystore	[srcreg+d2+32], ymm9		;; Save I3
	ystore	[srcreg+d2+d1], ymm2		;; Save R4
	ystore	[srcreg+d2+d1+32], ymm4		;; Save I4
	ystore	[srcreg+d1], ymm1		;; Save R2
	ystore	[srcreg+d1+32], ymm3		;; Save I2
	bump	srcreg, srcinc
	ENDM

;; Eliminate noswizzle unrolling.  Nice improvement for small FFTs.

yr4_4c_djbfft_noswiz_unroll_preload MACRO
	vbroadcastsd ymm15, YMM_ONE
	ENDM

yr4_4c_djbfft_noswiz_cmn MACRO srcreg,srcinc,srcoff,d1,d2,bcast,screg1,screg2,scoff,screg,scinc,maxrpt,L1pt,L1pd
	vmovapd	ymm1, [srcreg+srcoff]		;; R1
	vmovapd	ymm8, [srcreg+srcoff+d2]	;; R3					; Haswell	Kaby
	yfmsubpd ymm0, ymm1, ymm15, ymm8	;; R1 - R3 (new R3)			; 1-5		1-4
	vmovapd	ymm3, [srcreg+srcoff+32]	;; I1
	vmovapd	ymm9, [srcreg+srcoff+d2+32]	;; I3
	yfmsubpd ymm2, ymm3, ymm15, ymm9	;; I1 - I3 (new I3)			; 1-5		1-4

	yfmaddpd ymm1, ymm1, ymm15, ymm8	;; R1 + R3 (new R1)			; 2-6		2-5
	yfmaddpd ymm3, ymm3, ymm15, ymm9	;; I1 + I3 (new I1)			; 2-6		2-5

	vmovapd	ymm5, [srcreg+srcoff+d1+32]	;; I2
	vmovapd	ymm8, [srcreg+srcoff+d2+d1+32]	;; I4
	yfmaddpd ymm4, ymm5, ymm15, ymm8	;; I2 + I4 (new I2)			; 3-7		3-6
	vmovapd	ymm7, [srcreg+srcoff+d1]	;; R2
	vmovapd	ymm9, [srcreg+srcoff+d2+d1]	;; R4
	yfmaddpd ymm6, ymm7, ymm15, ymm9	;; R2 + R4 (new R2)			; 3-7		3-6

	yfmsubpd ymm5, ymm5, ymm15, ymm8	;; I2 - I4 (new I4)			; 4-8		4-7
	yfmsubpd ymm7, ymm7, ymm15, ymm9	;; R2 - R4 (new R4)			; 4-8		4-7

no bcast vmovapd ymm14, [screg1]		;; Sine
bcast	vbroadcastsd ymm14, Q [screg1]		;; Sine					; n 6		n 5
	vmulpd	ymm0, ymm0, ymm14		;; R3S = R3 * sine			; 6-10		5-8
	vmulpd	ymm2, ymm2, ymm14		;; I3S = I3 * sine			; 6-10		5-8

no bcast vmovapd ymm13, [screg2]		;; Sine
bcast	vbroadcastsd ymm13, Q [screg2]		;; Sine					; n 13		n 11
	yfmsubpd ymm8, ymm3, ymm15, ymm4	;; I1 - I2 (newer I2)			; 8-12		7-10
	yfmsubpd ymm9, ymm1, ymm15, ymm6	;; R1 - R2 (newer R2)			; 8-12		7-10

no bcast vmovapd ymm12, [screg1+32]		;; cosine/sine
bcast	vbroadcastsd ymm12, Q [screg1+scoff]	;; cosine/sine				; n 16		n 13
	yfmaddpd ymm3, ymm3, ymm15, ymm4	;; I1 + I2 (newer & final I1)		; 9-13		8-11
	yfmaddpd ymm1, ymm1, ymm15, ymm6	;; R1 + R2 (newer & final R1)		; 9-13		8-11

no bcast vmovapd ymm11, [screg2+32]		;; cosine/sine
bcast	vbroadcastsd ymm11, Q [screg2+scoff]	;; cosine/sine				; n 18		n 15
	yfnmaddpd ymm4, ymm5, ymm14, ymm0	;; A3 = R3S - I4 * sine (newer R3*sine)	; 11-15		9-12
	yfmaddpd ymm6, ymm7, ymm14, ymm2	;; B3 = I3S + R4 * sine (newer I3*sine)	; 11-15		9-12
	bump	screg, scinc

	L1prefetchw srcreg+L1pd, L1pt
	yfmaddpd ymm5, ymm5, ymm14, ymm0	;; A4 = R3S + I4 * sine (newer R4*sine)	; 12-16		10-13
	yfnmaddpd ymm7, ymm7, ymm14, ymm2	;; B4 = I3S - R4 * sine (newer I4*sine)	; 12-16		10-13

	L1prefetchw srcreg+d2+L1pd, L1pt
	vmulpd	ymm9, ymm9, ymm13		;; R2s = R2 * sine			; 13-17		11-14
	vmulpd	ymm8, ymm8, ymm13		;; I2s = I2 * sine			; 13-17		11-14
	ystore	[srcreg], ymm1			;; Save R1				; 13		11

	L1prefetchw srcreg+d1+L1pd, L1pt
	yfmsubpd ymm1, ymm4, ymm12, ymm6	;; final R3 = A3 * cosine/sine - B3	; 16-20		13-16
	yfmaddpd ymm6, ymm6, ymm12, ymm4	;; final I3 = B3 * cosine/sine + A3	; 16-20		13-16
	ystore	[srcreg+32], ymm3		;; Save I1				; 14		12

	L1prefetchw srcreg+d2+d1+L1pd, L1pt
	yfmaddpd ymm4, ymm5, ymm12, ymm7	;; final R4 = A4 * cosine/sine + B4	; 17-21		14-17
	yfmsubpd ymm7, ymm7, ymm12, ymm5	;; final I4 = B4 * cosine/sine - A4	; 17-21		14-17

	yfmsubpd ymm5, ymm9, ymm11, ymm8	;; final R2 = R2 * cosine/sine - I2	; 18-22		15-18
	yfmaddpd ymm8, ymm8, ymm11, ymm9	;; final I2 = I2 * cosine/sine + R2	; 18-22		15-18

	ystore	[srcreg+d2], ymm1		;; Save R3
	ystore	[srcreg+d2+32], ymm6		;; Save I3
	ystore	[srcreg+d2+d1], ymm4		;; Save R4
	ystore	[srcreg+d2+d1+32], ymm7		;; Save I4
	ystore	[srcreg+d1], ymm5		;; Save R2
	ystore	[srcreg+d1+32], ymm8		;; Save I2
	bump	srcreg, srcinc
	ENDM


yr4_4c_djbunfft_noswiz_unroll_preload MACRO
	vmovapd	ymm15, YMM_ONE
	ENDM

yr4_4c_djbunfft_noswiz_cmn MACRO srcreg,srcinc,d1,d2,bcast,screg1,screg2,scoff,screg,scinc,maxrpt,L1pt,L1pd
no bcast vmovapd ymm14, [screg2+32]		;; cosine/sine
bcast	vbroadcastsd ymm14, Q [screg2+scoff]	;; cosine/sine
	vmovapd	ymm8, [srcreg+d1]		;; R2
	vmovapd	ymm1, [srcreg+d1+32]		;; I2					; Haswell	Kaby
	yfmaddpd ymm0, ymm8, ymm14, ymm1	;; R2/sine = R2 * cosine/sine + I2	; 1-5		1-4
	yfmsubpd ymm1, ymm1, ymm14, ymm8	;; I2/sine = I2 * cosine/sine - R2	; 1-5		1-4

no bcast vmovapd ymm14, [screg1+32]		;; cosine/sine
bcast	vbroadcastsd ymm14, Q [screg1+scoff]	;; cosine/sine
	vmovapd	ymm8, [srcreg+d2]		;; R3
	vmovapd	ymm3, [srcreg+d2+32]		;; I3
	yfmaddpd ymm2, ymm8, ymm14, ymm3	;; R3/sine = R3 * cosine/sine + I3	; 2-6		2-5
	vmovapd	ymm9, [srcreg+d2+d1]		;; R4
	vmovapd	ymm5, [srcreg+d2+d1+32]		;; I4
	yfmsubpd ymm4, ymm9, ymm14, ymm5	;; R4/sine = R4 * cosine/sine - I4	; 2-6		2-5

	vmovapd	ymm6, [srcreg]			;; R1					; n 6		n 5
	yfmsubpd ymm3, ymm3, ymm14, ymm8	;; I3/sine = I3 * cosine/sine - R3	; 3-7		3-6
	yfmaddpd ymm5, ymm5, ymm14, ymm9	;; I4/sine = I4 * cosine/sine + R4	; 3-7		3-6

no bcast vmovapd ymm13, [screg2]		;; Sine
bcast	vbroadcastsd ymm13, Q [screg2]		;; Sine					; n 6		n 5
	yfmaddpd ymm8, ymm0, ymm13, ymm6	;; R1 + R2/sine * sine (new R1)		; 6-10		5-8
	yfnmaddpd ymm0, ymm0, ymm13, ymm6	;; R1 - R2/sine * sine (new R2)		; 6-10		5-8

	vmovapd	ymm7, [srcreg+32]		;; I1					; n 8		n 7
	yfmaddpd ymm6, ymm4, ymm15, ymm2	;; R4/sine + R3/sine (new R3/sine)	; 7-11		6-9
	yfmsubpd ymm4, ymm4, ymm15, ymm2	;; R4/sine - R3/sine (new I4/sine)	; 7-11		6-9

no bcast vmovapd ymm12, [screg1]		;; Sine
bcast	vbroadcastsd ymm12, Q [screg1]		;; Sine					; n 12		n 10
	yfnmaddpd ymm2, ymm1, ymm13, ymm7	;; I1 - I2/sine * sine (new I2)		; 8-12		7-10
	yfmaddpd ymm1, ymm1, ymm13, ymm7	;; I1 + I2/sine * sine (new I1)		; 8-12		7-10
	bump	screg, scinc

	L1prefetchw srcreg+d1+L1pd, L1pt
	yfmsubpd ymm7, ymm3, ymm15, ymm5	;; I3/sine - I4/sine (new R4/sine)	; 9-13		8-11
	yfmaddpd ymm3, ymm3, ymm15, ymm5	;; I3/sine + I4/sine (new I3/sine)	; 9-13		8-11

	L1prefetchw srcreg+d2+L1pd, L1pt
	yfnmaddpd ymm5, ymm6, ymm12, ymm8	;; R1 - R3/sine * sine (final R3)	; 12-16		10-13
	yfmaddpd ymm6, ymm6, ymm12, ymm8	;; R1 + R3/sine * sine (final R1)	; 12-16		10-13

	L1prefetchw srcreg+d2+d1+L1pd, L1pt
	yfnmaddpd ymm8, ymm4, ymm12, ymm2	;; I2 - I4/sine * sine (final I4)	; 13-17		11-14
	yfmaddpd ymm4, ymm4, ymm12, ymm2	;; I2 + I4/sine * sine (final I2)	; 13-17		11-14

	L1prefetchw srcreg+L1pd, L1pt
	yfnmaddpd ymm2, ymm7, ymm12, ymm0	;; R2 - R4/sine * sine (final R4)	; 14-18		12-15
	yfmaddpd ymm7, ymm7, ymm12, ymm0	;; R2 + R4/sine * sine (final R2)	; 14-18		12-15

	yfnmaddpd ymm0, ymm3, ymm12, ymm1	;; I1 - I3/sine * sine (final I3)	; 15-19		13-16
	yfmaddpd ymm3, ymm3, ymm12, ymm1	;; I1 + I3/sine * sine (final I1)	; 15-19		13-16

	ystore	[srcreg+d2], ymm5		;; Save R3
	ystore	[srcreg], ymm6			;; Save R1
	ystore	[srcreg+d2+d1+32], ymm8		;; Save I4
	ystore	[srcreg+d1+32], ymm4		;; Save I2
	ystore	[srcreg+d2+d1], ymm2		;; Save R4
	ystore	[srcreg+d1], ymm7		;; Save R2
	ystore	[srcreg+d2+32], ymm0		;; Save I3
	ystore	[srcreg+32], ymm3		;; Save I1
	bump	srcreg, srcinc
	ENDM

;; Combine yr4_fs4cl_four_complex_first_djbfft and yr4_b4cl_four_complex_djbfft

;; This was a small winner for SkyLake, KabyLake, etc.  
yr4_fs4cl_sixteen_complex_first_djbfft_preload MACRO
	vbroadcastsd ymm15, YMM_ONE
	ENDM
yr4_fs4cl_sixteen_complex_first_djbfft MACRO srcreg,srcinc,d1,d2,screg,scinc,sc2reg,sc2inc,maxrpt,L1pt,L1pd
	vmovapd	ymm8, [srcreg+d2][rbx]		;; R3						; Haswell	KabyLake
	vbroadcastsd ymm12, YMM_SQRTHALF
	vmulpd	ymm8, ymm8, ymm12		;; A3 = R3 * SQRTHALF (new R3)			; 1-5		1-4

	vmovapd	ymm6, [srcreg+d1][rbx]		;; R2
	vbroadcastsd ymm14, YMM_P924
	vbroadcastsd ymm13, YMM_P383
	vmulpd	ymm4, ymm6, ymm14		;; A2 = R2 * .924				; 2-6		2-5
	vmulpd	ymm6, ymm6, ymm13		;; B2 = R2 * .383				; 2-6		2-5

	vmovapd	ymm7, [srcreg+d2+d1][rbx]	;; R4
	vmulpd	ymm5, ymm7, ymm13		;; A4 = R4 * .383				; 3-7		3-6
	vmulpd	ymm7, ymm7, ymm14		;; B4 = R4 * .924				; 3-7		3-6

	vmovapd	ymm3, [srcreg+d2+32][rbx]	;; R7 (I3)
	yfnmaddpd ymm9, ymm3, ymm12, ymm8	;; new R3 = A3 - I3*SQRTHALF			; 6-10		5-8
	yfmaddpd ymm3, ymm3, ymm12, ymm8	;; new I3 = A3 + I3*SQRTHALF			; 6-10		5-8

	vmovapd	ymm2, [srcreg+d1+32][rbx]	;; R6 (I2)
	yfnmaddpd ymm4, ymm2, ymm13, ymm4	;; new R2 = A2 - .383*I2			; 7-11		6-9
	yfmaddpd ymm6, ymm2, ymm14, ymm6	;; new I2 = B2 + .924*I2			; 7-11		6-9

	vmovapd	ymm1, [srcreg+d2+d1+32][rbx]	;; R8 (I4)
	yfnmaddpd ymm5, ymm1, ymm14, ymm5	;; new R4 = A4 - .924*I4			; 8-12		7-10
	yfmaddpd ymm7, ymm1, ymm13, ymm7	;; new I4 = B4 + .383*I4			; 8-12		7-10

; two aparts

	vmovapd	ymm0, [srcreg][rbx]		;; R1
	yfmsubpd ymm2, ymm0, ymm15, ymm9	;; R1 - R3 (new R3)				; 11-15		9-12
	yfmaddpd ymm0, ymm0, ymm15, ymm9	;; R1 + R3 (new R1)				; 11-15		9-12

	vmovapd	ymm9, [srcreg+32][rbx]		;; R5 (I1)
	yfmsubpd ymm8, ymm9, ymm15, ymm3	;; I1 - I3 (new I3)				; 12-16		10-13
	yfmaddpd ymm9, ymm9, ymm15, ymm3	;; I1 + I3 (new I1)				; 12-16		10-13

	vmovapd	ymm14, [screg+64+32]		;; cosine/sine					; n 23		n 20
	vsubpd	ymm3, ymm4, ymm5		;; R2 - R4 (new R4)				; 13-15		11-14
	vaddpd	ymm4, ymm4, ymm5		;; R2 + R4 (new R2)				; 14-16		11-14

	vmovapd	ymm13, [screg+0+32]		;; cosine/sine					; n 24		n 21
	vsubpd	ymm5, ymm6, ymm7		;; I2 - I4 (new I4)				; 15-17		12-15
	vaddpd	ymm6, ymm6, ymm7		;; I2 + I4 (new I2)				; 16-18		12-15

; one aparts

	vmovapd	ymm12, [screg+192+32]		;; cosine/sine					; n 25		n 22
	yfmsubpd ymm7, ymm8, ymm15, ymm3	;; I3 - R4 (new I4)				; 17-21		15-18
	yfmaddpd ymm8, ymm8, ymm15, ymm3	;; I3 + R4 (new I3)				; 17-21		15-18

	vmovapd	ymm11, [screg+128+32]		;; cosine/sine					; n 26		n 23
	yfmsubpd ymm3, ymm2, ymm15, ymm5	;; R3 - I4 (new R3)				; 18-22		16-19
	yfmaddpd ymm2, ymm2, ymm15, ymm5	;; R3 + I4 (new R4)				; 18-22		16-19

	vmovapd	ymm10, [screg+64]		;; sine						; n 28		n 24
	yfmaddpd ymm5, ymm0, ymm15, ymm4	;; R1 + R2 (new R1)				; 19-23		17-20
	yfmaddpd ymm1, ymm9, ymm15, ymm6	;; I1 + I2 (new I1)				; 19-23		17-20

	yfmsubpd ymm0, ymm0, ymm15, ymm4	;; R1 - R2 (new R2)				; 20-24		18-21
	yfmsubpd ymm9, ymm9, ymm15, ymm6	;; I1 - I2 (new I2)				; 20-24		18-21

	vmovapd	ymm6, [screg+0]			;; sine						; n 31		n 27
	yfmsubpd ymm4, ymm3, ymm14, ymm8	;; A3 = R3 * cosine/sine - I3			; 23-27		20-23
	yfmaddpd ymm8, ymm8, ymm14, ymm3	;; B3 = I3 * cosine/sine + R3			; 23-27		20-23

	vmovapd	ymm14, [screg+192]		;; sine						, n 29		n 25
	yfmsubpd ymm3, ymm5, ymm13, ymm1	;; A1 = R1 * cosine/sine - I1			; 24-28		21-24
	yfmaddpd ymm1, ymm1, ymm13, ymm5	;; B1 = I1 * cosine/sine + R1			; 24-28		21-24

	vmovapd	ymm13, [screg+128]		;; sine						; n 30		n 26
	yfmsubpd ymm5, ymm2, ymm12, ymm7	;; A4 = R4 * cosine/sine - I4			; 25-29		22-25
	yfmaddpd ymm7, ymm7, ymm12, ymm2	;; B4 = I4 * cosine/sine + R4			; 25-29		22-25
	bump	screg, scinc

	vbroadcastsd ymm12, Q [sc2reg]		;; sine						; n 46		n 42
	yfmsubpd ymm2, ymm0, ymm11, ymm9	;; A2 = R2 * cosine/sine - I2			; 26-30		23-26
	yfmaddpd ymm9, ymm9, ymm11, ymm0	;; B2 = I2 * cosine/sine + R2			; 26-30		23-26

	vbroadcastsd ymm11, Q [sc2reg+16]	;; sine						; n 55		n 50
	vmulpd	ymm4, ymm4, ymm10		;; A3 = A3 * sine (final R3)			; 28-32		24-27
	vmulpd	ymm8, ymm8, ymm10		;; B3 = B3 * sine (final I3)			; 28-32		24-27

	vmulpd	ymm3, ymm3, ymm6		;; A1 = A1 * sine (final R1)			; 29-33		25-28
	vmulpd	ymm1, ymm1, ymm6		;; B1 = B1 * sine (final I1)			; 29-33		25-28

	vmulpd	ymm5, ymm5, ymm14		;; A4 = A4 * sine (final R4)			; 30-34		26-29
	vmulpd	ymm7, ymm7, ymm14		;; B4 = B4 * sine (final I4)			; 30-34		26-29

	vbroadcastsd ymm14, Q [sc2reg+8]	;; cosine/sine					; n 58		n 52
	vmulpd	ymm2, ymm2, ymm13		;; A2 = A2 * sine (final R2)			; 31-35		27-30
	vmulpd	ymm9, ymm9, ymm13		;; B2 = B2 * sine (final I2)			; 31-35		27-30

	;; We can do the initial adds/subs of the next FFT level half way into the shuffling process.
	;; This gives the FPU pipes something to do during the shufling process.

	vbroadcastsd ymm13, Q [sc2reg+16+8]	;; cosine/sine					; n 60		n 54
	ylow128s ymm0, ymm3, ymm4		;; Shuffle R1 lo and R3 lo (0 1 16 17)		; 34-36		29-31	r 34  29
	bump	sc2reg, sc2inc

	L1prefetchw srcreg+d2+L1pd, L1pt
	yhigh128s ymm3, ymm3, ymm4		;; Shuffle R1 hi and R3 hi (2 3 18 19)		; 35-37		30-32

	L1prefetchw srcreg+d1+L1pd, L1pt
	ylow128s ymm4, ymm2, ymm5		;; Shuffle R2 lo and R4 lo (8 9 24 25)		; 36-38		31-33	r 36  31

	L1prefetchw srcreg+d2+d1+L1pd, L1pt
	yhigh128s ymm2, ymm2, ymm5		;; Shuffle R2 hi and R4 hi (10 11 26 27)	; 37-39		32-34

	L1prefetchw srcreg+L1pd, L1pt
	ylow128s ymm5, ymm1, ymm8		;; Shuffle I1 lo and I3 lo (4 5 20 21)		; 38-40		33-35	r 34  29
	vaddpd	ymm6, ymm0, ymm3		;; r1/r3+ = lo+hi (0+2 1+3 16+18 17+19)		; 38-40		33-36

	yhigh128s ymm1, ymm1, ymm8		;; Shuffle I1 hi and I3 hi (6 7 22 23)		; 39-41		34-36
	vsubpd	ymm0, ymm0, ymm3		;; r1/r3- = lo-hi (0-2 1-3 16-18 17-19)		; 39-41		34-37

	ylow128s ymm8, ymm9, ymm7		;; Shuffle I2 lo and I4 lo (12 13 28 29)	; 40-42		35-37	r 36  31
	vaddpd	ymm3, ymm4, ymm2		;; r2/r4+ = lo+hi (8+10 9+11 24+26 25+27)	; 40-42		35-38

	yhigh128s ymm9, ymm9, ymm7		;; Shuffle I2 hi and I4 hi (14 15 30 31)	; 41-43		36-38
	vsubpd	ymm4, ymm4, ymm2		;; r2/r4- = lo-hi (8-10 9-11 24-26 25-27)	; 41-43		36-39

	vsubpd	ymm7, ymm5, ymm1		;; i1/i3- = lo-hi (4-6 5-7 20-22 21-23)		; 42-44		37-40				CRIT

	vshufpd	ymm2, ymm6, ymm3, 0		;; R1+R3 = Shuf(R1/R3+, R2/R4+) (0+ 8+ 16+ 24+)	; 43		39		r 43 / 39 
	vaddpd	ymm5, ymm5, ymm1		;; i1/i3+ = lo+hi (4+6 5+7 20+22 21+23)		; 43-45		37-40

	vshufpd	ymm6, ymm6, ymm3, 15		;; R2+R4 = Shuf(R1/R3+, R2/R4+) (1+ 9+ 17+ 25+)	; 44		40
	vsubpd	ymm1, ymm8, ymm9		;; i2/i4- = lo-hi (12-14 13-15 28-30 29-31)	; 44-46		39-42				CRIT

	vshufpd	ymm3, ymm0, ymm4, 0		;; R1-R3 = Shuf(R1/R3-, R2/R4-) (0- 8- 16- 24-)	; 45		41		r 44 / 40-
	vaddpd	ymm8, ymm8, ymm9		;; i2/i4+ = lo+hi (12+14 13+15 28+30 29+31)	; 45-47		39-42

	vshufpd	ymm0, ymm0, ymm4, 15		;; R2-R4 = Shuf(R1/R3-, R2/R4-) (1- 9- 17- 25-)	; 46		42
	vmulpd	ymm3, ymm3, ymm12		;; R3S = R3 * sine				; 46-50		42-45		r 46  42

	vshufpd	ymm4, ymm7, ymm1, 0		;; I1-I3 = Shuf(I1/I3-, I2/I4-) (4- 12- 20- 28-); 47		43		r 47 / 43	CRIT
	vsubpd	ymm9, ymm2, ymm6		;; R1 - R2 (newer R2)				; 47-49		43-46		r 45  41

	vshufpd	ymm10, ymm5, ymm8, 0		;; I1+I3 = Shuf(I1/I3+, I2/I4+) (4+ 12+ 20+ 28+); 48		44		r 47 / 43
	vmulpd	ymm4, ymm4, ymm12		;; I3S = I3 * sine				; 48-52		44-47		r 48  44

	vshufpd	ymm5, ymm5, ymm8, 15		;; I2+I4 = Shuf(I1/I3+, I2/I4+) (5+ 13+ 21+ 29+); 49		45
	vaddpd	ymm2, ymm2, ymm6		;; R1 + R2 (newer & final R1)			; 49-51		45-48		r 45  41

	vshufpd	ymm7, ymm7, ymm1, 15		;; I2-I4 = Shuf(I1/I3-, I2/I4-) (5- 13- 21- 29-); 50		46

	; The first FFT level (and part of the second FFT level) was done mid-shuffle above
	; New R1 = R1+R3, new R3 = R1-R3, new R2 = R2+R4, new R4 = R2-R4
	; New I1 = I1+I3, new I3 = I1-I3, new I2 = I2+I4, new I4 = I2-I4

	vsubpd	ymm1, ymm10, ymm5		;; I1 - I2 (newer I2)				; 50-52		46-49		r 50  46

	vaddpd	ymm10, ymm10, ymm5		;; I1 + I2 (newer & final I1)			; 51-53		46-49		r 51  46

	yfnmaddpd ymm5, ymm7, ymm12, ymm3	;; A3 = R3S - I4 * sine (newer R3*sine)		; 53-57		48-51		53   48
	yfmaddpd ymm6, ymm0, ymm12, ymm4	;; B3 = I3S + R4 * sine (newer I3*sine)		; 53-57		48-51		53   48

	yfmaddpd ymm7, ymm7, ymm12, ymm3	;; A4 = R3S + I4 * sine (newer R4*sine)		; 54-58		49-52		54   49
	yfnmaddpd ymm0, ymm0, ymm12, ymm4	;; B4 = I3S - R4 * sine (newer I4*sine)		; 54-58		49-52		54   49
	ystore	[srcreg], ymm2			;; Save R1					; 52		49

	vmulpd	ymm9, ymm9, ymm11		;; R2s = R2 * sine				; 55-59		50-53		55   50
	vmulpd	ymm1, ymm1, ymm11		;; I2s = I2 * sine				; 55-59		50-53		
	ystore	[srcreg+32], ymm10		;; Save I1					; 54		50

	yfmsubpd ymm4, ymm5, ymm14, ymm6	;; final R3 = A3 * cosine/sine - B3		; 58-62		52-55
	yfmaddpd ymm6, ymm6, ymm14, ymm5	;; final I3 = B3 * cosine/sine + A3		; 58-62		52-55

	yfmaddpd ymm5, ymm7, ymm14, ymm0	;; final R4 = A4 * cosine/sine + B4		; 59-63		53-56
	yfmsubpd ymm0, ymm0, ymm14, ymm7	;; final I4 = B4 * cosine/sine - A4		; 59-63		53-56

	yfmsubpd ymm7, ymm9, ymm13, ymm1	;; final R2 = R2s * cosine/sine - I2s		; 60-64		54-57
	yfmaddpd ymm1, ymm1, ymm13, ymm9	;; final I2 = I2s * cosine/sine + R2s		; 60-64		54-57

	ystore	[srcreg+d2], ymm4		;; Save R3
	ystore	[srcreg+d2+32], ymm6		;; Save I3
	ystore	[srcreg+d2+d1], ymm5		;; Save R4
	ystore	[srcreg+d2+d1+32], ymm0		;; Save I4
	ystore	[srcreg+d1], ymm7		;; Save R2
	ystore	[srcreg+d1+32], ymm1		;; Save I2
	bump	srcreg, srcinc
	ENDM

;; Combine yr4_b4cl_four_complex_djbunfft and yr4_s4cl_four_complex_last_unfft

;; Generally worse performance!  The loop merging optimization was a big winner for AVX-512.
yr4_s4cl_sixteen_complex_last_unfft_preload MACRO
	vmovapd	ymm15, YMM_ONE
	ENDM
yr4_s4cl_sixteen_complex_last_unfft MACRO srcreg,srcinc,d1,d2,screg,scinc,sc2reg,sc2inc,maxrpt,L1pt,L1pd
	vbroadcastsd ymm14, Q [screg+16+8]	;; cosine/sine
	vmovapd	ymm8, [srcreg+d1]		;; R2
	vmovapd	ymm1, [srcreg+d1+32]		;; I2					; Haswell	Kaby
	yfmaddpd ymm0, ymm8, ymm14, ymm1	;; R2/sine = R2 * cosine/sine + I2	; 1-5		1-4
	yfmsubpd ymm1, ymm1, ymm14, ymm8	;; I2/sine = I2 * cosine/sine - R2	; 1-5		1-4

	vbroadcastsd ymm14, Q [screg+8]		;; cosine/sine
	vmovapd	ymm8, [srcreg+d2]		;; R3
	vmovapd	ymm3, [srcreg+d2+32]		;; I3
	yfmaddpd ymm2, ymm8, ymm14, ymm3	;; R3/sine = R3 * cosine/sine + I3	; 2-6		2-5
	vmovapd	ymm9, [srcreg+d2+d1]		;; R4
	vmovapd	ymm5, [srcreg+d2+d1+32]		;; I4
	yfmsubpd ymm4, ymm9, ymm14, ymm5	;; R4/sine = R4 * cosine/sine - I4	; 2-6		2-5

	vmovapd	ymm6, [srcreg]			;; R1					; n 6		n 5
	yfmsubpd ymm3, ymm3, ymm14, ymm8	;; I3/sine = I3 * cosine/sine - R3	; 3-7		3-6
	yfmaddpd ymm5, ymm5, ymm14, ymm9	;; I4/sine = I4 * cosine/sine + R4	; 3-7		3-6

	vbroadcastsd ymm13, Q [screg+16]	;; Sine					; n 6		n 5
	yfmaddpd ymm8, ymm0, ymm13, ymm6	;; R1 + R2/sine * sine (new R1)		; 6-10		5-8
	yfnmaddpd ymm0, ymm0, ymm13, ymm6	;; R1 - R2/sine * sine (new R2)		; 6-10		5-8

	vmovapd	ymm7, [srcreg+32]		;; I1					; n 8		n 7
	yfmaddpd ymm6, ymm4, ymm15, ymm2	;; R4/sine + R3/sine (new R3/sine)	; 7-11		6-9
	yfmsubpd ymm4, ymm4, ymm15, ymm2	;; R4/sine - R3/sine (new I4/sine)	; 7-11		6-9

	vbroadcastsd ymm12, Q [screg]		;; Sine					; n 12		n 10
	yfnmaddpd ymm2, ymm1, ymm13, ymm7	;; I1 - I2/sine * sine (new I2)		; 8-12		7-10
	yfmaddpd ymm1, ymm1, ymm13, ymm7	;; I1 + I2/sine * sine (new I1)		; 8-12		7-10
	bump	screg, scinc

	L1prefetchw srcreg+d1+L1pd, L1pt
	yfmsubpd ymm7, ymm3, ymm15, ymm5	;; I3/sine - I4/sine (new R4/sine)	; 9-13		8-11		do at 8
	yfmaddpd ymm3, ymm3, ymm15, ymm5	;; I3/sine + I4/sine (new I3/sine)	; 9-13		8-11

	L1prefetchw srcreg+d2+L1pd, L1pt
	yfnmaddpd ymm5, ymm6, ymm12, ymm8	;; R1 - R3/sine * sine (final R3)	; 12-16		10-13
	yfmaddpd ymm6, ymm6, ymm12, ymm8	;; R1 + R3/sine * sine (final R1)	; 12-16		10-13

	L1prefetchw srcreg+d2+d1+L1pd, L1pt
	yfnmaddpd ymm8, ymm4, ymm12, ymm2	;; I2 - I4/sine * sine (final I4)	; 13-17		11-14
	yfmaddpd ymm4, ymm4, ymm12, ymm2	;; I2 + I4/sine * sine (final I2)	; 13-17		11-14

	L1prefetchw srcreg+L1pd, L1pt
	yfnmaddpd ymm2, ymm7, ymm12, ymm0	;; R2 - R4/sine * sine (final R4)	; 14-18		12-15		do at 13
	yfmaddpd ymm7, ymm7, ymm12, ymm0	;; R2 + R4/sine * sine (final R2)	; 14-18		12-15

	yfnmaddpd ymm0, ymm3, ymm12, ymm1	;; I1 - I3/sine * sine (final I3)	; 15-19		13-16
	yfmaddpd ymm3, ymm3, ymm12, ymm1	;; I1 + I3/sine * sine (final I1)	; 15-19		13-16

	;; Swizzle using 8 shufpds and 8 perm2f128  (1 or 3 clocks each on port 5, throughput 1)

	vshufpd	ymm1, ymm6, ymm7, 15		;; Shuffle R1 and R2 to create 8 9 24 25		; 1		start on clk earlier
	vshufpd	ymm6, ymm6, ymm7, 0		;; Shuffle R1 and R2 to create 0 1 16 17		; 2

	vshufpd	ymm7, ymm5, ymm2, 15		;; Shuffle R3 and R4 to create 10 11 26 27		; 3
	vshufpd	ymm5, ymm5, ymm2, 0		;; Shuffle R3 and R4 to create 2 3 18 19		; 4

	ylow128s ymm2, ymm1, ymm7		;; Shuffle R1/R2 hi and R3/R4 hi (8 9 10 11)		; 5-7
	yhigh128s ymm1, ymm1, ymm7		;; Shuffle R1/R2 hi and R3/R4 hi (24 25 26 27)		; 6-8

	ylow128s ymm7, ymm6, ymm5		;; Shuffle R1/R2 low and R3/R4 low (0 1 2 3)		; 7-9
	yhigh128s ymm6, ymm6, ymm5		;; Shuffle R1/R2 low and R3/R4 low (16 17 18 19)	; 8-10

	vshufpd	ymm5, ymm3, ymm4, 15		;; Shuffle I1 and I2 to create 12 13 28 29		; 9
	vshufpd	ymm3, ymm3, ymm4, 0		;; Shuffle I1 and I2 to create 4 5 20 21		; 10

	vshufpd	ymm4, ymm0, ymm8, 15		;; Shuffle I3 and I4 to create 14 15 30 31		; 11
	vshufpd	ymm0, ymm0, ymm8, 0		;; Shuffle I3 and I4 to create 6 7 22 23		; 12

	vmovapd	ymm14, [sc2reg+128+32]		;; cosine/sine						; n 16
	ylow128s ymm8, ymm5, ymm4		;; Shuffle I1/I2 hi and I3/I4 hi (12 13 14 15)		; 13-15
	yhigh128s ymm5, ymm5, ymm4		;; Shuffle I1/I2 hi and I3/I4 hi (28 29 30 31)		; 14-16

	vmovapd	ymm13, [sc2reg+192+32]		;; cosine/sine						; n 17
	yhigh128s ymm4, ymm3, ymm0		;; Shuffle I1/I2 low and I3/I4 low (20 21 22 23)	; 15-17		accelerate calc of R2/I2 and R4/I4
	ylow128s ymm3, ymm3, ymm0		;; Shuffle I1/I2 low and I3/I4 low (4 5 6 7)		; 16-18

	vmovapd	ymm12, [sc2reg+64+32]		;; cosine/sine						; n 18
	yfmaddpd ymm0, ymm2, ymm14, ymm8	;; A2 = R2 * cosine/sine + I2				; 16-20		16-19
	yfmsubpd ymm8, ymm8, ymm14, ymm2	;; B2 = I2 * cosine/sine - R2				; 16-20		16-19

	vmovapd	ymm11, [sc2reg+0+32]		;; cosine/sine						; n 19
	yfmaddpd ymm2, ymm1, ymm13, ymm5	;; A4 = R4 * cosine/sine + I4				; 17-21		17-20
	yfmsubpd ymm5, ymm5, ymm13, ymm1	;; B4 = I4 * cosine/sine - R4				; 17-21		17-20

	vmovapd	ymm14, [sc2reg+128]		;; sine							; n 21		20
	yfmaddpd ymm1, ymm6, ymm12, ymm4	;; A3 = R3 * cosine/sine + I3				; 18-22		18-21
	yfmsubpd ymm4, ymm4, ymm12, ymm6	;; B3 = I3 * cosine/sine - R3				; 18-22		18-21

	vmovapd	ymm13, [sc2reg+192]		;; sine							; n 22		21
	yfmaddpd ymm6, ymm7, ymm11, ymm3	;; A1 = R1 * cosine/sine + I1				; 19-23		19-22
	yfmsubpd ymm3, ymm3, ymm11, ymm7	;; B1 = I1 * cosine/sine - R1				; 19-23		19-22

	vmovapd	ymm12, [sc2reg+0]		;; sine							; n 26		24
	vmulpd	ymm0, ymm0, ymm14		;; A2 = A2 * sine (new R2)				; 21-25		20-23
	vmulpd	ymm8, ymm8, ymm14		;; B2 = B2 * sine (new I2)				; 21-25		20-23

	vmovapd	ymm11, [sc2reg+64]		;; sine							; n 27		25
	vmulpd	ymm2, ymm2, ymm13		;; A4 = A4 * sine (new R4)				; 22-26		21-24
	vmulpd	ymm5, ymm5, ymm13		;; B4 = B4 * sine (new I4)				; 22-26		21-24
	bump	sc2reg, sc2inc

	yfmsubpd ymm7, ymm6, ymm12, ymm0	;; R1*sine - R2 (new R2)				; 26-30		24-27
	yfmaddpd ymm6, ymm6, ymm12, ymm0	;; R1*sine + R2 (new R1)				; 26-30		24-27

	yfnmaddpd ymm0, ymm1, ymm11, ymm2	;; R4 = R4 - R3*sine (new I4)				; 27-31		25-28
	yfmaddpd ymm2, ymm1, ymm11, ymm2	;; R3 = R4 + R3*sine (new R3)				; 27-31		25-28

	yfmsubpd ymm1, ymm4, ymm11, ymm5	;; I3 = I3*sine - I4 (new R4)				; 28-32		26-29
	yfmaddpd ymm4, ymm4, ymm11, ymm5	;; I4 = I3*sine + I4 (new I3)				; 28-32		26-29

	yfmsubpd ymm5, ymm3, ymm12, ymm8	;; I1*sine - I2 (new I2)				; 29-33		27-30
	yfmaddpd ymm3, ymm3, ymm12, ymm8	;; I1*sine + I2 (new I1)				; 29-33		27-30

	vsubpd	ymm8, ymm6, ymm2		;; R1 - R3 (new R3)					; 32-34		29-32
	vaddpd	ymm6, ymm6, ymm2		;; R1 + R3 (final R1)					; 33-35		29-32

	vsubpd	ymm2, ymm7, ymm1		;; R2 - R4 (new R4)					; 34-36		30-33
	vaddpd	ymm7, ymm7, ymm1		;; R2 + R4 (new R2)					; 35-37		30-33

	vsubpd	ymm1, ymm3, ymm4		;; I1 - I3 (new I3)					; 36-38		31-34
	vaddpd	ymm3, ymm3, ymm4		;; I1 + I3 (final I1)					; 37-39		31-34

	vsubpd	ymm4, ymm5, ymm0		;; I2 - I4 (new I4)					; 38-40		32-35
	vaddpd	ymm5, ymm5, ymm0		;; I2 + I4 (new I2)					; 39-41		32-35

	vbroadcastsd ymm10, YMM_SQRTHALF
	vmulpd	ymm8, ymm8, ymm10		;; A3 = R3 * SQRTHALF					; 35-39		33-36
	ystore	[srcreg], ymm6			;; Save R1						; 36		33

	vbroadcastsd ymm11, YMM_P924
	vbroadcastsd ymm12, YMM_P383
	vmulpd	ymm0, ymm2, ymm12		;; A4 = R4 * .383					; 37-41		34-37
	vmulpd	ymm2, ymm2, ymm11		;; B4 = R4 * .924					; 37-41		34-37

	vmulpd	ymm6, ymm7, ymm11		;; A2 = R2 * .924					; 38-42		35-38
	vmulpd	ymm7, ymm7, ymm12		;; B2 = R2 * .383					; 38-42		35-38
	ystore	[srcreg+32], ymm3		;; Save I1						; 40		35

	yfmsubpd ymm14, ymm1, ymm10, ymm8	;; final I3 = I3*SQRTHALF - A3				; 40-44		37-40
	yfmaddpd ymm1, ymm1, ymm10, ymm8	;; final R3 = I3*SQRTHALF + A3				; 40-44		37-40

	yfmaddpd ymm0, ymm4, ymm11, ymm0	;; final R4 = A4 + .924*I4				; 42-46		38-41
	yfmsubpd ymm2, ymm4, ymm12, ymm2	;; final I4 = .383*I4 - B4				; 42-46		38-41

	yfmaddpd ymm6, ymm5, ymm12, ymm6	;; final R2 = A2 + .383*I2				; 43-47		39-42
	yfmsubpd ymm7, ymm5, ymm11, ymm7	;; final I2 = .924*I2 - B2				; 43-47		39-42

	ystore	[srcreg+d2], ymm1		;; Save R3
	ystore	[srcreg+d2+32], ymm14		;; Save I3
	ystore	[srcreg+d2+d1], ymm0		;; Save R4
	ystore	[srcreg+d2+d1+32], ymm2		;; Save I4
	ystore	[srcreg+d1], ymm6		;; Save R2
	ystore	[srcreg+d1+32], ymm7		;; Save I2
	bump	srcreg, srcinc
	ENDM

; Minor tweaks to start shuffling earlier

; Mildly worse for KabyLake, ever so slightly better for Haswell
NOTUSED_yr4_s8r_fft_cmn_preload MACRO
	vmovapd	ymm15, YMM_ONE
	ENDM

NOTUSED_yr4_s8r_fft_cmn MACRO srcreg,srcinc,srcoff,d1,d2,screg1,screg2,screg5,scregA,scincA,scregB,scincB,maxrpt,L1pt,L1pd
	vmovapd	ymm0, [srcreg+srcoff+d2+d1]	;; R4
	vmovapd	ymm7, [srcreg+srcoff+d2+d1+32]	;; R8						; Haswell	Kaby
	vsubpd	ymm3, ymm0, ymm7		;; new R8 = R4 - R8				; 1-3		1-4
	yfmaddpd ymm7, ymm0, ymm15, ymm7	;; new R4 = R4 + R8				;	1-5	1-4

	vmovapd	ymm5, [srcreg+srcoff+d1]	;; R2
	vmovapd	ymm4, [srcreg+srcoff+d1+32]	;; R6
	vsubpd	ymm1, ymm5, ymm4		;; new R6 = R2 - R6				; 2-4		2-5
	yfmaddpd ymm5, ymm5, ymm15, ymm4	;; new R2 = R2 + R6				;	2-6	2-5

	vmovapd	ymm0, [srcreg+srcoff]		;; R1
	vmovapd	ymm2, [srcreg+srcoff+32]	;; R5
	vaddpd	ymm4, ymm0, ymm2		;; new R1 = R1 + R5				; 3-5		3-6
	yfmsubpd ymm0, ymm0, ymm15, ymm2	;; new R5 = R1 - R5				;	3-7	3-6

	vmovapd	ymm6, [srcreg+srcoff+d2]	;; R3
	vmovapd	ymm8, [srcreg+srcoff+d2+32]	;; R7
	vaddpd	ymm2, ymm6, ymm8		;; new R3 = R3 + R7				; 4-6		4-7
	yfmsubpd ymm6, ymm6, ymm15, ymm8	;; new R7 = R3 - R7				;	4-8	4-7

	vbroadcastsd ymm14, YMM_SQRTHALF							; n 9		n 10
	vsubpd	ymm8, ymm1, ymm3		;; R6 = R6 - R8					; 5-7		6-9
	yfmaddpd ymm1, ymm1, ymm15, ymm3	;; R8 = R6 + R8					;	5-9	6-9

	vmovapd	ymm13, [screg2+32]		;; cosine/sine for w^2n				; n 11		n 12
	vsubpd	ymm3, ymm5, ymm7		;; R2 - R4 (final I2)				; 7-9		7-10
	yfmaddpd ymm5, ymm5, ymm15, ymm7	;; R2 + R4 (final I1, a.k.a 2nd real result)	;	7-11	7-10

	vmovapd	ymm12, [screg5+32]		;; cosine/sine for w^5n				; n 15		n 15
	vsubpd	ymm7, ymm4, ymm2		;; R1 - R3 (final R2)				; 8-10		8-11
	yfmaddpd ymm4, ymm4, ymm15, ymm2	;; R1 + R3 (final R1)				;	8-12	8-11

	vmovapd	ymm11, [screg1+32]		;; cosine/sine for w^n				; n 15		n 15
	yfnmaddpd ymm2, ymm8, ymm14, ymm0	;; R5 - R6 * square root half (final R4)	; 9-13		10-13
	yfmaddpd ymm8, ymm8, ymm14, ymm0	;; R5 + R6 * square root half (final R3)	;	9-13	10-13

	vmovapd	ymm10, [screg2]			;; sine for w^2n				; n 16		n 16
	yfnmaddpd ymm0, ymm1, ymm14, ymm6	;; R7 - R8 * square root half (final I4)	; 10-14		11-14
	yfmaddpd ymm1, ymm1, ymm14, ymm6	;; R7 + R8 * square root half (final I3)	;	10-14	11-14

	vmovapd	ymm14, [screg5]			;; sine for w^5n				; n 20		n 19
	yfmsubpd ymm6, ymm7, ymm13, ymm3	;; A2 = R2 * cosine/sine for w^2n - I2		; 11-15		12-15
	yfmaddpd ymm3, ymm3, ymm13, ymm7	;; B2 = I2 * cosine/sine for w^2n + R2		;	11-15	12-15

	vmovapd	ymm13, [screg1]			;; sine for w^n					; n 20		n 19
	yfmsubpd ymm7, ymm2, ymm12, ymm0	;; A4 = R4 * cosine/sine for w^5n - I4		; 15-19		15-18
	yfmsubpd ymm9, ymm8, ymm11, ymm1	;; A3 = R3 * cosine/sine for w^n - I3		;	15-19	15-18
	bump	scregA, scincA

	L1prefetchw srcreg+d2+d1+L1pd, L1pt
	vmulpd	ymm6, ymm6, ymm10		;; A2 = A2 * sine (final R2)			; 16-20		16-19
	vmulpd	ymm3, ymm3, ymm10		;; B2 = B2 * sine (final I2)			;	16-20	16-19
	bump	scregB, scincB

	L1prefetchw srcreg+d1+L1pd, L1pt
	yfmaddpd ymm0, ymm0, ymm12, ymm2	;; B4 = I4 * cosine/sine for w^5n + R4		; 17-21		17-20
	yfmaddpd ymm1, ymm1, ymm11, ymm8	;; B3 = I3 * cosine/sine for w^n + R3		;	17-21	17-20

	L1prefetchw srcreg+L1pd, L1pt
	vmulpd	ymm7, ymm7, ymm14		;; A4 = A4 * sine (final R4)			; 20-24		19-22
	vmulpd	ymm9, ymm9, ymm13		;; A3 = A3 * sine (final R3)			; 20-24		19-22

	L1prefetchw srcreg+d2+L1pd, L1pt
	vmulpd	ymm0, ymm0, ymm14		;; B4 = B4 * sine (final I4)			;	22-26	21-24
	vmulpd	ymm1, ymm1, ymm13		;; B3 = B3 * sine (final I3)			;	22-26	21-24

	;; Swizzle using 8 shufpds and 8 perm2f128  (1 or 3 clocks each on port 5, throughput 1)

	vshufpd	ymm8, ymm4, ymm6, 0		;; Shuffle R1 and R2 to create 0 8 2 10		; 21		20
	vshufpd	ymm4, ymm4, ymm6, 15		;; Shuffle R1 and R2 to create 1 9 3 11		; 22		21

	vshufpd	ymm6, ymm5, ymm3, 0		;; Shuffle I1 and I2 to create 4 12 6 14	; 23		22
	vshufpd	ymm5, ymm5, ymm3, 15		;; Shuffle I1 and I2 to create 5 13 7 15	; 24		23

	vshufpd	ymm3, ymm9, ymm7, 0		;; Shuffle R3 and R4 to create 16 24 18 26	; 25		24
	vshufpd	ymm9, ymm9, ymm7, 15		;; Shuffle R3 and R4 to create 17 25 19 27	; 26		25

	vshufpd	ymm7, ymm1, ymm0, 0		;; Shuffle I3 and I4 to create 16 28 22 30	; 27		26
	vshufpd	ymm1, ymm1, ymm0, 15		;; Shuffle I3 and I4 to create 21 29 23 31	; 28		27

	ylow128s ymm0, ymm8, ymm3		;; Shuffle R1/R2 low and R3/R4 low (0 8 16 24)	; 29-31		28-30
	yhigh128s ymm8, ymm8, ymm3		;; Shuffle R1/R2 low and R3/R4 low (2 10 18 26)	; 30-32		29-31

	ylow128s ymm3, ymm4, ymm9		;; Shuffle R1/R2 hi and R3/R4 hi (1 9 17 25)	; 31-33		30-32
	yhigh128s ymm4, ymm4, ymm9		;; Shuffle R1/R2 hi and R3/R4 hi (3 11 19 27)	; 32-34		31-33
	ystore	[srcreg], ymm0			;; Save (0 8 16 24)				; 32		33
	ystore	[srcreg+d2], ymm8		;; Save (2 10 18 26)				; 33		34

	ylow128s ymm9, ymm6, ymm7		;; Shuffle I1/I2 low and I3/I4 low (4 12 20 28)	; 33-35		32-34
	yhigh128s ymm6, ymm6, ymm7		;; Shuffle I1/I2 low and I3/I4 low (6 14 22 30)	; 34-36		33-35
	ystore	[srcreg+d1], ymm3		;; Save (1 9 17 25)				; 34		35
	ystore	[srcreg+d2+d1], ymm4		;; Save (3 11 19 27)				; 35		36

	ylow128s ymm7, ymm5, ymm1		;; Shuffle I1/I2 hi and I3/I4 hi (5 13 21 29)	; 35-37		34-36
	yhigh128s ymm5, ymm5, ymm1		;; Shuffle I1/I2 hi and I3/I4 hi (7 15 23 31)	; 36-38		35-37

	ystore	[srcreg+32], ymm9		;; Save (4 12 20 28)
	ystore	[srcreg+d2+32], ymm6		;; Save (6 14 22 30)
	ystore	[srcreg+d1+32], ymm7		;; Save (5 13 21 29)
	ystore	[srcreg+d2+d1+32], ymm5		;; Save (7 15 23 31)
	bump	srcreg, srcinc
	ENDM
ENDIF

