; Copyright 2009-2024 - Mersenne Research, Inc.  All rights reserved
; Author:  George Woltman
; Email: woltman@alum.mit.edu
;

;; Include other macro files needed by our primarily radix-4 traditional FFTs

INCLUDE r2.mac
INCLUDE r3.mac
INCLUDE r5.mac
INCLUDE r7.mac
INCLUDE r8.mac

;;
;;
;; All new macros for version 26 of gwnum where we do a very traditional, primarily
;; radix-4, FFT.  The forward FFT macros multiply by the sin/cos values at the end
;; of the macro and the inverse FFTs multiply by the sin/cos values at the start of
;; the macro.  We also implement the Daniel J. Bernstein (DJB) "exponent-1" idea to
;; save sin/cos memory.
;;
;;

;; In forward FFTs,
;;    input values are R1+R5i, R2+R6i, R3+R7i, R4+R8i
;;    output values are R1+R2i, R3+R4i, R5+R6i, R7+R8i

;; In inverse FFTs,
;;    input values are R1+R2i, R3+R4i, R5+R6i, R7+R8i
;;    output values are R1+R5i, R2+R6i, R3+R7i, R4+R8i

;;
;; ************************************* four-complex-djbfft variants ******************************************
;;
;; Macros to do Daniel J. Bernstein's exponent-1 butterflies.  The
;; difference with a standard four-complex-fft is in the postmultiply step.
;; Instead of multiplying by w^2x, w^x, w^3x, we multiply by w^2x, w^x, w^-x.
;;

r4_x4cl_four_complex_djbfft_preload MACRO
	r4_x4cl_2sc_four_complex_djbfft_preload
	ENDM

r4_x4cl_four_complex_djbfft MACRO srcreg,srcinc,d1,d2,screg
	r4_x4cl_2sc_four_complex_djbfft srcreg,srcinc,d1,d2,screg,screg+32
	ENDM

;; Special version used in pass 2 when first levels are radix-3.

r4_x4cl_2sc_four_complex_djbfft_preload MACRO
	r4_x4c_2sc_djbfft_mem_preload
	;r4_x4c_2sc_djbfft_partial_mem_preload -- assumed to be identical to r4_x4c_2sc_djbfft_mem_preload
	ENDM

r4_x4cl_2sc_four_complex_djbfft MACRO srcreg,srcinc,d1,d2,screg1,screg2
	d3 = d2 + d1
	r4_x4c_2sc_djbfft_mem [srcreg],[srcreg+d1],[srcreg+d2],[srcreg+d2+d1],[srcreg+16],[srcreg+d1+16],[srcreg+d2+16],[srcreg+d2+d1+16],screg1,screg2,srcreg+srcinc,d1,[srcreg],[srcreg+16]
;;	xstore	[srcreg], xmm0		;; Save R1
;;	xstore	[srcreg+16], xmm0	;; Save I1
	xload	xmm0, [srcreg+32]	;; R1
	xload	xmm5, [srcreg+48]	;; R5
	xstore	[srcreg+32], xmm7	;; Save R2
	xstore	[srcreg+48], xmm6	;; Save I2
	xload	xmm7, [srcreg+d1+32]	;; R2
	xload	xmm6, [srcreg+d1+48]	;; R6
	xstore	[srcreg+d1], xmm3	;; Save R3
	xstore	[srcreg+d1+16], xmm1	;; Save I3
	xstore	[srcreg+d1+32], xmm2	;; Save R4
	xstore	[srcreg+d1+48], xmm4	;; Save I4
	r4_x4c_2sc_djbfft_partial_mem xmm0,xmm7,xmm3,xmm2,xmm5,xmm6,xmm1,xmm4,[srcreg+d2+32],[srcreg+d3+32],[srcreg+d2+48],[srcreg+d3+48],screg1,screg2,srcreg+srcinc+d2,d1,[srcreg+d2],[srcreg+d2+16]
;;	xstore	[srcreg+d2], xmm0	;; Save R1
;;	xstore	[srcreg+d2+16], xmm0	;; Save I1
	xstore	[srcreg+d2+32], xmm4	;; Save R2
	xstore	[srcreg+d2+48], xmm1	;; Save I2
	xstore	[srcreg+d2+d1], xmm2	;; Save R3
	xstore	[srcreg+d2+d1+16], xmm7	;; Save I3
	xstore	[srcreg+d2+d1+32], xmm0	;; Save R4
	xstore	[srcreg+d2+d1+48], xmm5	;; Save I4
	bump	srcreg, srcinc
	ENDM

;; Used in last levels of an r4 FFT pass 1 (not in r4delay FFT).  No swizzling.
IFDEF UNUSED
r4_g4cl_four_complex_djbfft_preload MACRO
	r4_x4c_djbfft_mem_preload
	;r4_x4c_djbfft_partial_mem_preload -- assumed same as r4_x4c_djbfft_mem_preload
	ENDM
r4_g4cl_four_complex_djbfft MACRO srcreg,srcinc,d1,d2,dstreg,dstinc,e1,e2,screg
	xprefetch [srcreg+srcinc]
	r4_x4c_djbfft_mem [srcreg],[srcreg+d1],[srcreg+d2],[srcreg+d2+d1],[srcreg+16],[srcreg+d1+16],[srcreg+d2+16],[srcreg+d2+d1+16],screg,0,dstreg+dstinc,e1,[dstreg],[dstreg+16]
	xprefetch [srcreg+srcinc+d1]
;;	xstore	[dstreg], xmm0		;; Save R1
;;	xstore	[dstreg+16], xmm0	;; Save I1
	xload	xmm0, [srcreg+32]	;; R1
	xload	xmm5, [srcreg+48]	;; R5
	xstore	[dstreg+32], xmm7	;; Save R2
	xstore	[dstreg+48], xmm6	;; Save I2
	xload	xmm7, [srcreg+d1+32]	;; R2
	xload	xmm6, [srcreg+d1+48]	;; R6
	xstore	[dstreg+e1], xmm3	;; Save R3
	xstore	[dstreg+e1+16], xmm1	;; Save I3
	xstore	[dstreg+e1+32], xmm2	;; Save R4
	xstore	[dstreg+e1+48], xmm4	;; Save I4
	xprefetch [srcreg+srcinc+d2]
	r4_x4c_djbfft_partial_mem xmm0,xmm7,xmm3,xmm2,xmm5,xmm6,xmm1,xmm4,[srcreg+d2+32],[srcreg+d2+d1+32],[srcreg+d2+48],[srcreg+d2+d1+48],screg,0,dstreg+dstinc+e2,e1,[dstreg+e2],[dstreg+e2+16]
	xprefetch [srcreg+srcinc+d2+d1]
	bump	srcreg, srcinc
;;	xstore	[dstreg+e2], xmm0	;; Save R1
;;	xstore	[dstreg+e2+16], xmm0	;; Save I1
	xstore	[dstreg+e2+32], xmm4	;; Save R2
	xstore	[dstreg+e2+48], xmm1	;; Save I2
	xstore	[dstreg+e2+e1], xmm2	;; Save R3
	xstore	[dstreg+e2+e1+16], xmm7	;; Save I3
	xstore	[dstreg+e2+e1+32], xmm0	;; Save R4
	xstore	[dstreg+e2+e1+48], xmm5	;; Save I4
	bump	dstreg, dstinc
	ENDM
ENDIF

;; Used in last levels of an r4 FFT pass 1 (not in r4delay FFT).  Swizzling.

r4_sg4cl_four_complex_djbfft_preload MACRO
	r4_x4c_djbfft_mem_preload
	;r4_x4c_djbfft_partial_mem_preload -- assumed to be identical to r4_x4c_djbfft_mem_preload
	ENDM

r4_sg4cl_four_complex_djbfft MACRO srcreg,srcinc,d1,d2,dstreg,dstinc,e1,e2,screg
	xprefetch [srcreg+srcinc]
	r4_x4c_djbfft_mem [srcreg],[srcreg+d1],[srcreg+d2],[srcreg+d2+d1],[srcreg+16],[srcreg+d1+16],[srcreg+d2+16],[srcreg+d2+d1+16],screg,0,dstreg+dstinc,e1,[dstreg],[dstreg+32]
	xprefetch [srcreg+srcinc+d1]
	shuffle_store_partial [dstreg], [dstreg+16], xmm0, xmm7				;; Save R1,R2
	shuffle_store_partial [dstreg+32], [dstreg+48], xmm5, xmm6			;; Save I1,I2
	xload	xmm0, [srcreg+32]	;; R1
	xload	xmm7, [srcreg+d1+32]	;; R2
	xload	xmm5, [srcreg+48]	;; R5
	shuffle_store_with_temp [dstreg+e1], [dstreg+e1+16], xmm3, xmm2, xmm6		;; Save R3,R4
	shuffle_store_with_temp [dstreg+e1+32], [dstreg+e1+48], xmm1, xmm4, xmm6	;; Save I3,I4
	xload	xmm6, [srcreg+d1+48]	;; R6
	xprefetch [srcreg+srcinc+d2]
	r4_x4c_djbfft_partial_mem xmm0,xmm7,xmm3,xmm2,xmm5,xmm6,xmm1,xmm4,[srcreg+d2+32],[srcreg+d2+d1+32],[srcreg+d2+48],[srcreg+d2+d1+48],screg,0,dstreg+dstinc+e2,e1,[dstreg+e2],[dstreg+e2+32]
	xprefetch [srcreg+srcinc+d2+d1]
	bump	srcreg, srcinc
	shuffle_store_partial [dstreg+e2], [dstreg+e2+16], xmm3, xmm4			;; Save R1,R2
	shuffle_store_partial [dstreg+e2+32], [dstreg+e2+48], xmm6, xmm1		;; Save I1,I2
	shuffle_store_with_temp [dstreg+e2+e1], [dstreg+e2+e1+16], xmm2, xmm0, xmm3	;; Save R3,R4
	shuffle_store_with_temp [dstreg+e2+e1+32], [dstreg+e2+e1+48], xmm7, xmm5, xmm3	;; Save I3,I4
	bump	dstreg, dstinc
	ENDM

;; Used in first levels of pass 2 when last levels of pass 1 did not swizzle
IFDEF UNUSED
r4_s2cl_four_complex_djbfft MACRO srcreg,srcinc,d1,screg
	shuffle_load xmm0, xmm2, [srcreg][rbx], [srcreg+32][rbx] ;; R1,R3
	shuffle_load xmm1, xmm3, [srcreg+d1][rbx], [srcreg+d1+32][rbx] ;; R2,R4
	xprefetch [srcreg+srcinc][rbx]
	shuffle_load xmm4, xmm6, [srcreg+16][rbx], [srcreg+48][rbx] ;; R5,R7
	shuffle_load xmm5, xmm7, [srcreg+d1+16][rbx], [srcreg+d1+48][rbx] ;; R6,R8
	xprefetch [srcreg+srcinc+d1][rbx]
	r4_x4c_djbfft xmm0,xmm1,xmm2,xmm3,xmm4,xmm5,xmm6,xmm7,screg,0,srcreg+srcinc,d1,[srcreg],[srcreg+16]
;;	xstore	[srcreg], xmm0		;; Save R1
;;	xstore	[srcreg+16], xmm0	;; Save I1
	xstore	[srcreg+32], xmm7	;; Save R2
	xstore	[srcreg+48], xmm6	;; Save I2
	xstore	[srcreg+d1], xmm3	;; Save R3
	xstore	[srcreg+d1+16], xmm1	;; Save I3
	xstore	[srcreg+d1+32], xmm0	;; Save R4
	xstore	[srcreg+d1+48], xmm4	;; Save I4
	bump	srcreg, srcinc
	ENDM
ENDIF

;; Used in first levels of pass 2 when last levels of pass 1 swizzled.

r4_f2cl_four_complex_djbfft_preload MACRO
	r4_x4c_djbfft_mem_preload
	ENDM

r4_f2cl_four_complex_djbfft MACRO srcreg,srcinc,d1,screg
	r4_x4c_djbfft_mem [srcreg][rbx],[srcreg+d1][rbx],[srcreg+16][rbx],[srcreg+d1+16][rbx],[srcreg+32][rbx],[srcreg+d1+32][rbx],[srcreg+48][rbx],[srcreg+d1+48][rbx],screg,0,srcreg+srcinc+rbx,d1,[srcreg],[srcreg+16]
;;	xstore	[srcreg], xmm0		;; Save R1
;;	xstore	[srcreg+16], xmm0	;; Save I1
	xstore	[srcreg+32], xmm7	;; Save R2
	xstore	[srcreg+48], xmm6	;; Save I2
	xstore	[srcreg+d1], xmm3	;; Save R3
	xstore	[srcreg+d1+16], xmm1	;; Save I3
	xstore	[srcreg+d1+32], xmm2	;; Save R4
	xstore	[srcreg+d1+48], xmm4	;; Save I4
	bump	srcreg, srcinc
	ENDM

;; Used in later levels of pass 2.

r4_nf2cl_four_complex_djbfft_preload MACRO
	r4_x4c_djbfft_mem_preload
	ENDM

r4_nf2cl_four_complex_djbfft MACRO srcreg,srcinc,d1,screg
	r4_x4c_djbfft_mem [srcreg],[srcreg+d1],[srcreg+16],[srcreg+d1+16],[srcreg+32],[srcreg+d1+32],[srcreg+48],[srcreg+d1+48],screg,0,srcreg+srcinc,d1,[srcreg],[srcreg+16]
;;	xstore	[srcreg], xmm0		;; Save R1
;;	xstore	[srcreg+16], xmm0	;; Save I1
	xstore	[srcreg+32], xmm7	;; Save R2
	xstore	[srcreg+48], xmm6	;; Save I2
	xstore	[srcreg+d1], xmm3	;; Save R3
	xstore	[srcreg+d1+16], xmm1	;; Save I3
	xstore	[srcreg+d1+32], xmm2	;; Save R4
	xstore	[srcreg+d1+48], xmm4	;; Save I4
	bump	srcreg, srcinc
	ENDM

;; Used in pass 2 after radix-3 levels.  Uses two sin/cos ptrs instead of one.

r4_x2cl_2sc_four_complex_djbfft_preload MACRO
	r4_x4c_2sc_djbfft_mem_preload
	ENDM

r4_x2cl_2sc_four_complex_djbfft MACRO srcreg,srcinc,d1,screg1,screg2
	r4_x4c_2sc_djbfft_mem [srcreg],[srcreg+d1],[srcreg+16],[srcreg+d1+16],[srcreg+32],[srcreg+d1+32],[srcreg+48],[srcreg+d1+48],screg1,screg2,srcreg+srcinc,d1,[srcreg],[srcreg+16]
;;	xstore	[srcreg], xmm0		;; Save R1
;;	xstore	[srcreg+16], xmm0	;; Save I1
	xstore	[srcreg+32], xmm7	;; Save R2
	xstore	[srcreg+48], xmm6	;; Save I2
	xstore	[srcreg+d1], xmm3	;; Save R3
	xstore	[srcreg+d1+16], xmm1	;; Save I3
	xstore	[srcreg+d1+32], xmm2	;; Save R4
	xstore	[srcreg+d1+48], xmm4	;; Save I4
	bump	srcreg, srcinc
	ENDM

;; Used in pass 2 after radix-5 levels.

r4_x2cl_four_complex_djbfft_preload MACRO
	r4_x4c_2sc_djbfft_mem_preload
	ENDM

r4_x2cl_four_complex_djbfft MACRO srcreg,srcinc,d1,screg1
	r4_x4c_2sc_djbfft_mem [srcreg],[srcreg+d1],[srcreg+16],[srcreg+d1+16],[srcreg+32],[srcreg+d1+32],[srcreg+48],[srcreg+d1+48],screg1,screg1+32,srcreg+srcinc,d1,[srcreg],[srcreg+16]
;;	xstore	[srcreg], xmm0		;; Save R1
;;	xstore	[srcreg+16], xmm0	;; Save I1
	xstore	[srcreg+32], xmm7	;; Save R2
	xstore	[srcreg+48], xmm6	;; Save I2
	xstore	[srcreg+d1], xmm3	;; Save R3
	xstore	[srcreg+d1+16], xmm1	;; Save I3
	xstore	[srcreg+d1+32], xmm2	;; Save R4
	xstore	[srcreg+d1+48], xmm4	;; Save I4
	bump	srcreg, srcinc
	ENDM

;; The common macros to get the four complex djbfft job done

r4_x4c_djbfft_mem_preload MACRO
	r4_x4c_2sc_djbfft_mem_preload
	ENDM

r4_x4c_djbfft_mem MACRO R1,R2,R3,R4,R5,R6,R7,R8,screg,off,pre1,pre2,dst1,dst2
	r4_x4c_2sc_djbfft_mem R1,R2,R3,R4,R5,R6,R7,R8,screg+off,screg+off+32,pre1,pre2,dst1,dst2
	ENDM

r4_x4c_2sc_djbfft_mem_preload MACRO
	ENDM

r4_x4c_2sc_djbfft_mem MACRO R1,R2,R3,R4,R5,R6,R7,R8,screg1,screg2,pre1,pre2,dst1,dst2
	xload	xmm0, R1		;; R1
	xload	xmm2, R3		;; R3
	addpd	xmm2, xmm0		;; R3 = R1 + R3 (new R1)

	xload	xmm1, R2		;; R2
	xload	xmm3, R4		;; R4
	addpd	xmm3, xmm1		;; R4 = R2 + R4 (new R2)

	xload	xmm4, R5		;; I1
	xload	xmm6, R7		;; I3
	addpd	xmm6, xmm4		;; I3 = I1 + I3 (new I1)

	 subpd	xmm0, R3		;; R1 = R1 - R3 (new R3)
	 subpd	xmm1, R4		;; R2 = R2 - R4 (new R4)

	xprefetchw [pre1]

	xcopy	xmm5, xmm3		;; Copy R2
	addpd	xmm3, xmm2		;; R2 = R1 + R2 (final R1)
	 subpd	xmm2, xmm5		;; R1 = R1 - R2 (final R2)

	xload	xmm5, R6		;; I2
	xload	xmm7, R8		;; I4
	addpd	xmm7, xmm5		;; I4 = I2 + I4 (new I2)

	 subpd	xmm4, R7		;; I1 = I1 - I3 (new I3)
	 subpd	xmm5, R8		;; I2 = I2 - I4 (new I4)

	xstore	dst1, xmm3		;; Save R1

	xcopy	xmm3, xmm7		;; Copy I2
	addpd	xmm7, xmm6		;; I2 = I1 + I2 (final I1)

	 subpd	xmm6, xmm3		;; I1 = I1 - I2 (final I2)

	xstore	dst2, xmm7		;; Save I1

	xcopy	xmm3, xmm1
	addpd	xmm1, xmm4		;; R4 = I3 + R4 (final I3)

	xcopy	xmm7, xmm0		;; Copy R3
	subpd	xmm0, xmm5		;; R3 = R3 - I4 (final R3)

	 subpd	xmm4, xmm3		;; I3 = I3 - R4 (final I4)
	addpd	xmm5, xmm7		;; I4 = R3 + I4 (final R4)

	xprefetchw [pre1][pre2]

	xload	xmm7, [screg2+16]	;; cosine/sine
	mulpd	xmm7, xmm2		;; A2 = R2 * cosine/sine
	subpd	xmm7, xmm6		;; A2 = A2 - I2

	xload	xmm3, [screg1+16]	;; cosine/sine
	mulpd	xmm3, xmm0		;; A3 = R3 * cosine/sine
	subpd	xmm3, xmm1		;; A3 = A3 - I3

	mulpd	xmm6, [screg2+16]	;; B2 = I2 * cosine/sine
	addpd	xmm6, xmm2		;; B2 = B2 + R2

	xload	xmm2, [screg1+16]	;; cosine/sine
	mulpd	xmm2, xmm5		;; A4 = R4 * cosine/sine
	addpd	xmm2, xmm4		;; A4 = A4 + I4

	mulpd	xmm1, [screg1+16]	;; B3 = I3 * cosine/sine
	addpd	xmm1, xmm0		;; B3 = B3 + R3

	mulpd	xmm4, [screg1+16]	;; B4 = I4 * cosine/sine
	subpd	xmm4, xmm5		;; B4 = B4 - R4

	mulpd	xmm7, [screg2]		;; A2 = A2 * sine (final R2)
	mulpd	xmm6, [screg2]		;; B2 = B2 * sine (final I2)

	xload	xmm5, [screg1]		;; Sine
	mulpd	xmm3, xmm5		;; A3 = A3 * sine (final R3)
	mulpd	xmm1, xmm5		;; B3 = B3 * sine (final I3)

	mulpd	xmm2, xmm5		;; A4 = A4 * sine (final R4)
	mulpd	xmm4, xmm5		;; B4 = B4 * sine (final I4)
	ENDM

r4_x4c_djbfft_partial_mem_preload MACRO
	r4_x4c_2sc_djbfft_partial_mem_preload
	ENDM

r4_x4c_djbfft_partial_mem MACRO r1,r2,r3,r4,r5,r6,r7,r8,mem3,mem4,mem7,mem8,screg,off,pre1,pre2,dst1,dst2
	r4_x4c_2sc_djbfft_partial_mem r1,r2,r3,r4,r5,r6,r7,r8,mem3,mem4,mem7,mem8,screg+off,screg+off+32,pre1,pre2,dst1,dst2
	ENDM

r4_x4c_2sc_djbfft_partial_mem_preload MACRO
	ENDM

r4_x4c_2sc_djbfft_partial_mem MACRO r1,r2,r3,r4,r5,r6,r7,r8,mem3,mem4,mem7,mem8,screg1,screg2,pre1,pre2,dst1,dst2
	xload	r3, mem3
	addpd	r3, r1			;; R3 = R1 + R3 (new R1)
	subpd	r1, mem3		;; R1 = R1 - R3 (new R3)

	xload	r4, mem4
	addpd	r4, r2			;; R4 = R2 + R4 (new R2)
	subpd	r2, mem4		;; R2 = R2 - R4 (new R4)

	xcopy	r7, r3
	subpd	r3, r4			;; R1 = R1 - R2 (final R2)
	addpd	r4, r7			;; R2 = R1 + R2 (final R1)

	xstore	dst1, r4		;; Save R1

	xload	r7, mem7
	xload	r8, mem8

	xcopy	r4, r6
	subpd	r6, r8			;; I2 = I2 - I4 (new I4)
	addpd	r8, r4			;; I4 = I2 + I4 (new I2)

	xcopy	r4, r5
	subpd	r5, r7			;; I1 = I1 - I3 (new I3)
	addpd	r7, r4			;; I3 = I1 + I3 (new I1)

	xcopy	r4, r1
	subpd	r1, r6			;; R3 = R3 - I4 (final R3)
	addpd	r6, r4			;; I4 = R3 + I4 (final R4)

	xcopy	r4, r2
	addpd	r2, r5			;; R4 = I3 + R4 (final I3)
	subpd	r5, r4			;; I3 = I3 - R4 (final I4)

	xprefetchw [pre1]

	xcopy	r4, r7
	subpd	r7, r8			;; I1 = I1 - I2 (final I2)
	addpd	r8, r4			;; I2 = I1 + I2 (final I1)

	xload	r4, [screg1+16]		;; cosine/sine
	mulpd	r4, r1			;; A3 = R3 * cosine/sine

	xstore	dst2, r8		;; Save I1

	xload	r8, [screg2+16]		;; cosine/sine
	mulpd	r8, r3			;; A2 = R2 * cosine/sine

	subpd	r4, r2			;; A3 = A3 - I3
	mulpd	r2, [screg1+16]		;; B3 = I3 * cosine/sine
	addpd	r2, r1			;; B3 = B3 + R3

	xload	r1, [screg1+16]		;; cosine/sine
	mulpd	r1, r6			;; A4 = R4 * cosine/sine

	subpd	r8, r7			;; A2 = A2 - I2
	mulpd	r7, [screg2+16]		;; B2 = I2 * cosine/sine

	xprefetchw [pre1][pre2]

	addpd	r1, r5			;; A4 = A4 + I4
	mulpd	r5, [screg1+16]		;; B4 = I4 * cosine/sine

	addpd	r7, r3			;; B2 = B2 + R2
	subpd	r5, r6			;; B4 = B4 - R4

	xload	r3, [screg1]		;; sine
	mulpd	r4, r3			;; A3 = A3 * sine (final R3)
	mulpd	r2, r3			;; B3 = B3 * sine (final I3)
	mulpd	r8, [screg2]		;; A2 = A2 * sine (final R2)
	mulpd	r1, r3			;; A4 = A4 * sine (final R4)
	mulpd	r7, [screg2]		;; B2 = B2 * sine (final I2)
	mulpd	r5, r3			;; B4 = B4 * sine (final I4)
	ENDM

IFDEF UNUSED
r4_x4c_djbfft MACRO r1,r2,r3,r4,r5,r6,r7,r8,screg,off,pre1,pre2,dst1,dst2
	subpd	r1, r3			;; R1 = R1 - R3 (new R3)
	multwo	r3
	addpd	r3, r1			;; R3 = R1 + R3 (new R1)

	subpd	r2, r4			;; R2 = R2 - R4 (new R4)
	multwo	r4			;; R4 = R4 * 2
	addpd	r4, r2			;; R4 = R2 + R4 (new R2)

	subpd	r3, r4			;; R1 = R1 - R2 (final R2)
	multwo	r4			;; R2 = R2 * 2
	addpd	r4, r3			;; R2 = R1 + R2 (final R1)

	xstore	dst1, r4		;; Save R1

	xcopy	r4, r6
	subpd	r6, r8			;; I2 = I2 - I4 (new I4)
	addpd	r8, r4			;; I4 = I2 + I4 (new I2)

	xcopy	r4, r5
	subpd	r5, r7			;; I1 = I1 - I3 (new I3)
	addpd	r7, r4			;; I3 = I1 + I3 (new I1)

	xcopy	r4, r1
	subpd	r1, r6			;; R3 = R3 - I4 (final R3)
	addpd	r6, r4			;; I4 = R3 + I4 (final R4)

	xcopy	r4, r2
	addpd	r2, r5			;; R4 = I3 + R4 (final I3)
	subpd	r5, r4			;; I3 = I3 - R4 (final I4)

	xprefetchw [pre1]

	xcopy	r4, r7
	subpd	r7, r8			;; I1 = I1 - I2 (final I2)
	addpd	r8, r4			;; I2 = I1 + I2 (final I1)

	xload	r4, [screg+off+0+16]	;; cosine/sine
	mulpd	r4, r1			;; A3 = R3 * cosine/sine

	xstore	dst2, r8		;; Save I1

	xload	r8, [screg+off+32+16]	;; cosine/sine
	mulpd	r8, r3			;; A2 = R2 * cosine/sine

	subpd	r4, r2			;; A3 = A3 - I3
	mulpd	r2, [screg+off+0+16]	;; B3 = I3 * cosine/sine
	addpd	r2, r1			;; B3 = B3 + R3

	xload	r1, [screg+off+0+16]	;; cosine/sine
	mulpd	r1, r6			;; A4 = R4 * cosine/sine

	subpd	r8, r7			;; A2 = A2 - I2
	mulpd	r7, [screg+off+32+16]	;; B2 = I2 * cosine/sine

	xprefetchw [pre1][pre2]

	addpd	r1, r5			;; A4 = A4 + I4
	mulpd	r5, [screg+off+0+16]	;; B4 = I4 * cosine/sine

	addpd	r7, r3			;; B2 = B2 + R2
	subpd	r5, r6			;; B4 = B4 - R4

	xload	r3, [screg+off+0]	;; sine
	mulpd	r4, r3			;; A3 = A3 * sine (final R3)
	mulpd	r2, r3			;; B3 = B3 * sine (final I3)
	mulpd	r8, [screg+off+32]	;; A2 = A2 * sine (final R2)
	mulpd	r1, r3			;; A4 = A4 * sine (final R4)
	mulpd	r7, [screg+off+32]	;; B2 = B2 * sine (final I2)
	mulpd	r5, r3			;; B4 = B4 * sine (final I4)
	ENDM
ENDIF

;; 32-bit AMD K8 optimized versions of the above macros

;; These macros could be improved further if we didn't keep output register compatibility
;; with the Core2/P4 macros.  That is, we'd have to rewrite all the callers of these
;; macros like we do for the Core2/P4 64-bit macros.  One optimization revolves around
;; noting that subpd/multwo/addpd can be changed to addpd/multwo/subpd giving us greater
;; flexibility in ordering the computation of intermediate results.

IF (@INSTR(,%xarch,<K8>) NE 0)

;; K8 Cheat sheet for scheduling dependency chains
;;	      12345678901234567890123456789012345678901234567890123456789012345678901234567890
;;r13(r3)     AAAA
;;i24(i4)       AAAA
;;i13(i3)         AAAA
;;r24(r4)           AAAA
;;r13(r1)             AAAA
;;r24(r2)               AAAA
;;i13(i1)                 AAAA
;;i24(i2)                   AAAA
;;mR3(depR3-I4)               AAAA
;;mI4(depI3-R4)	                AAAA			
;;mR2(depR1-R2)	                  AAAA
;;mI2(depI1-I2) 	            AAAA
;;mR4(depR3+I4)               MMMM    AAAA
;;mI3(depI3+R4)	                MMMM    AAAA				
;;mR1(depR1+R2)	                  MMMM    AAAA
;;mI1(depI1+I2)	                    MMMM    AAAA
;;A3		                      MMMM      AAAAMMMM
;;A2                                    MMMM  AAAAMMMM
;;B2		                          MMMM    AAAAMMMM
;;B3		                            MMMM      AAAAMMMM
;;A4                                          MMMM  AAAAMMMM
;;B4		                                MMMM    AAAAMMMM

r4_x4c_2sc_djbfft_mem MACRO R1,R2,R3,R4,R5,R6,R7,R8,screg1,screg2,pre1,pre2,dst1,dst2
	xload	xmm0, R1		;; R1
	xload	xmm2, R3		;; R3
	subpd	xmm0, xmm2		;; R1 = R1 - R3 (new R3)		;; 1-4

	xload	xmm5, R6		;; I2
	xload	xmm7, R8		;; I4
	subpd	xmm5, xmm7		;; I2 = I2 - I4 (new I4)		;; 3-6

	xload	xmm4, R5		;; I1
	xload	xmm6, R7		;; I3
	subpd	xmm4, xmm6		;; I1 = I1 - I3 (new I3)		;; 5-8

	xload	xmm1, R2		;; R2
	xload	xmm3, R4		;; R4
	subpd	xmm1, xmm3		;; R2 = R2 - R4 (new R4)		;; 7-10

	addpd	xmm2, R1		;; R3 = R1 + R3 (new R1)		;; 9-12

	addpd	xmm3, R2		;; R4 = R2 + R4 (new R2)		;; 11-14

	addpd	xmm6, R5		;; I3 = I1 + I3 (new I1)		;; 13-16

	addpd	xmm7, R6		;; I4 = I2 + I4 (new I2)		;; 15-18

	xprefetchw [pre1]

	 subpd	xmm0, xmm5		;; R3 = R3 - I4 (final R3)		;; 17-20
	 multwo	xmm5			;; I4 = I4 * 2				;; 17-20

	 subpd	xmm4, xmm1		;; I3 = I3 - R4 (final I4)		;; 19-22
	 multwo	xmm1			;; R4 = R4 * 2				;; 19-22

	 subpd	xmm2, xmm3		;; R1 = R1 - R2 (final R2)		;; 21-24
	 multwo	xmm3			;; R2 = R2 * 2				;; 21-24

	 subpd	xmm6, xmm7		;; I1 = I1 - I2 (final I2)		;; 23-26
	 multwo	xmm7			;; I2 = I2 * 2				;; 23-26

	 addpd	xmm5, xmm0		;; I4 = R3 + I4 (final R4)		;; 25-28
	 addpd	xmm1, xmm4		;; R4 = I3 + R4 (final I3)		;; 27-30
	xprefetchw [pre1][pre2]
	 addpd	xmm3, xmm2		;; R2 = R1 + R2 (final R1)		;; 29-32
	 addpd	xmm7, xmm6		;; I2 = I1 + I2 (final I1)		;; 31-34

	xstore	dst1, xmm3		;; Save R1				;; 33

	xload	xmm3, [screg1+16]	;; cosine/sine
	mulpd	xmm3, xmm0		;; A3 = R3 * cosine/sine		;; 25-28

	xstore	dst2, xmm7		;; Save I1				;; 35

	xload	xmm7, [screg2+16]	;; cosine/sine
	mulpd	xmm7, xmm2		;; A2 = R2 * cosine/sine		;; 27-30

	subpd	xmm7, xmm6		;; A2 = A2 - I2				;; 33-36
	mulpd	xmm6, [screg2+16]	;; B2 = I2 * cosine/sine		;; 29-32

	subpd	xmm3, xmm1		;; A3 = A3 - I3				;; 35-38
	mulpd	xmm1, [screg1+16]	;; B3 = I3 * cosine/sine		;; 31-34

	addpd	xmm6, xmm2		;; B2 = B2 + R2				;; 37-40

	xload	xmm2, [screg1+16]	;; cosine/sine
	mulpd	xmm2, xmm5		;; A4 = R4 * cosine/sine		;; 33-36

	addpd	xmm2, xmm4		;; A4 = A4 + I4				;; 39-42
	mulpd	xmm4, [screg1+16]	;; B4 = I4 * cosine/sine		;; 35-38

	mulpd	xmm7, [screg2]		;; A2 = A2 * sine (final R2)		;; 37-40

	mulpd	xmm6, [screg2]		;; B2 = B2 * sine (final I2)		;; 41-44

	addpd	xmm1, xmm0		;; B3 = B3 + R3				;; 41-44
	xload	xmm0, [screg1]		;; Sine
	mulpd	xmm3, xmm0		;; A3 = A3 * sine (final R3)		;; 39-42

	subpd	xmm4, xmm5		;; B4 = B4 - R4				;; 43-46
	mulpd	xmm2, xmm0		;; A4 = A4 * sine (final R4)		;; 43-46

	mulpd	xmm1, xmm0		;; B3 = B3 * sine (final I3)		;; 45-48
	mulpd	xmm4, xmm0		;; B4 = B4 * sine (final I4)		;; 47-50
	ENDM

;; K8 Cheat sheet for scheduling dependency chains
;;	      12345678901234567890123456789012345678901234567890123456789012345678901234567890
;;i13(i1)     AAAA
;;i13(i3)       AAAA
;;r24(r2)         AAAA
;;r24(r4)           AAAA
;;r13(r1)             AAAA
;;r13(r3)               AAAA
;;i24(i2)                 AAAA
;;i24(i4)                   AAAA
;;mI4(depI3-R4)	              AAAA			
;;mR3(depR3-I4)                 AAAA
;;mR2(depR1-R2)	                  AAAA
;;mI2(depI1-I2) 	            AAAA
;;mI3(depI3+R4)	              MMMM    AAAA				
;;mR4(depR3+I4)                 MMMM    AAAA
;;mR1(depR1+R2)	                  MMMM    AAAA
;;mI1(depI1+I2)	                    MMMM    AAAA
;;A3		                      MMMM    AAAAMMMM
;;A2                                    MMMM      AAAAMMMM
;;B3		                          MMMM  AAAAMMMM
;;A4		                            MMMM    AAAAMMMM
;;B2                                          MMMM    AAAAMMMM
;;B4		                                MMMM    AAAAMMMM

r4_x4c_2sc_djbfft_partial_mem MACRO r1,r2,r3,r4,r5,r6,r7,r8,mem3,mem4,mem7,mem8,screg1,screg2,pre1,pre2,dst1,dst2
	xload	r7, mem7						;; K8
	addpd	r7, r5			;; I3 = I1 + I3 (new I1)	;; 1-4
	subpd	r5, mem7		;; I1 = I1 - I3 (new I3)	;; 3-6

	xload	r4, mem4
	addpd	r4, r2			;; R4 = R2 + R4 (new R2)	;; 5-8
	subpd	r2, mem4		;; R2 = R2 - R4 (new R4)	;; 7-10

	xload	r3, mem3
	addpd	r3, r1			;; R3 = R1 + R3 (new R1)	;; 9-12
	subpd	r1, mem3		;; R1 = R1 - R3 (new R3)	;; 11-14

	xload	r8, mem8
	addpd	r8, r6			;; I4 = I2 + I4 (new I2)	;; 13-16
	subpd	r6, mem8		;; I2 = I2 - I4 (new I4)	;; 15-18

	xprefetchw [pre1]

	subpd	r5, r2			;; I3 = I3 - R4 (final I4)	;; 17-20
	multwo	r2			;; R4 = R4 * 2			;; 17-20

	subpd	r1, r6			;; R3 = R3 - I4 (final R3)	;; 19-22
	multwo	r6			;; I4 = I4 * 2			;; 19-22

	subpd	r3, r4			;; R1 = R1 - R2 (final R2)	;; 21-24
	multwo	r4			;; R2 = R2 * 2			;; 21-24

	subpd	r7, r8			;; I1 = I1 - I2 (final I2)	;; 23-26
	multwo	r8			;; I2 = I2 * 2			;; 23-26

	addpd	r6, r1			;; I4 = R3 + I4 (final R4)	;; 25-28
	addpd	r2, r5			;; R4 = I3 + R4 (final I3)	;; 27-30
	xprefetchw [pre1][pre2]
	addpd	r4, r3			;; R2 = R1 + R2 (final R1)	;; 29-32
	addpd	r8, r7			;; I2 = I1 + I2 (final I1)	;; 31-34

	xstore	dst1, r4		;; Save R1

	xload	r4, [screg1+16]		;; cosine/sine
	mulpd	r4, r1			;; A3 = R3 * cosine/sine	;; 25-28

	xstore	dst2, r8		;; Save I1

	xload	r8, [screg2+16]		;; cosine/sine
	mulpd	r8, r3			;; A2 = R2 * cosine/sine	;; 27-30

	subpd	r4, r2			;; A3 = A3 - I3			;; 33-36
	mulpd	r2, [screg1+16]		;; B3 = I3 * cosine/sine	;; 29-32

	addpd	r2, r1			;; B3 = B3 + R3			;; 35-38

	xload	r1, [screg1+16]		;; cosine/sine
	mulpd	r1, r6			;; A4 = R4 * cosine/sine	;; 31-34

	subpd	r8, r7			;; A2 = A2 - I2			;; 37-40
	mulpd	r7, [screg2+16]		;; B2 = I2 * cosine/sine	;; 33-36

	addpd	r1, r5			;; A4 = A4 + I4			;; 39-42
	mulpd	r5, [screg1+16]		;; B4 = I4 * cosine/sine	;; 35-38

	addpd	r7, r3			;; B2 = B2 + R2			;; 41-44
	xload	r3, [screg1]		;; sine
	mulpd	r4, r3			;; A3 = A3 * sine (final R3)	;; 37-40

	mulpd	r2, r3			;; B3 = B3 * sine (final I3)	;; 39-42

	mulpd	r8, [screg2]		;; A2 = A2 * sine (final R2)	;; 41-44

	subpd	r5, r6			;; B4 = B4 - R4			;; 43-46
	mulpd	r1, r3			;; A4 = A4 * sine (final R4)	;; 43-46

	mulpd	r7, [screg2]		;; B2 = B2 * sine (final I2)	;; 45-48

	mulpd	r5, r3			;; B4 = B4 * sine (final I4)	;; 47-50

	ENDM

ENDIF

;; 64-bit Intel and AMD K10 implementations of the above - use the extra XMM registers

IF (@INSTR(,%xarch,<CORE>) NE 0) OR (@INSTR(,%xarch,<P4>) NE 0) OR (@INSTR(,%xarch,<BLEND>) NE 0) OR (@INSTR(,%xarch,<K10>) NE 0)
IFDEF X86_64

; Theoretical best case is 44 clocks on a Core 2.  Now at 47.2 clocks.

r4_x4cl_2sc_four_complex_djbfft_preload MACRO
	ENDM

r4_x4cl_2sc_four_complex_djbfft MACRO srcreg,srcinc,d1,d2,screg1,screg2
	xload	xmm0, [srcreg]		;; R1
	xload	xmm2, [srcreg+d2]	;; R3
	xload	xmm5, [srcreg+d1+16]	;; I2
	xload	xmm7, [srcreg+d2+d1+16]	;; I4
	xload	xmm1, [srcreg+d1]	;; R2
	xload	xmm3, [srcreg+d2+d1]	;; R4
	xload	xmm4, [srcreg+16]	;; I1
	xload	xmm6, [srcreg+d2+16]	;; I3

	xcopy	xmm8, xmm0		;; Copy R1				; 1-3
	subpd	xmm0, xmm2		;; R1 = R1 - R3 (new R3)		; 1-3

	xcopy	xmm9, xmm5		;; Copy I2				; 2-4
	subpd	xmm5, xmm7		;; I2 = I2 - I4 (new I4)		; 2-4

	xcopy	xmm10, xmm1		;; Copy R2				; 3-5
	subpd	xmm1, xmm3		;; R2 = R2 - R4 (new R4)		; 3-5

	xcopy	xmm11, xmm4		;; Copy I1				; 4-6
	subpd	xmm4, xmm6		;; I1 = I1 - I3 (new I3)		; 4-6	avail 12+
	xload	xmm12, [screg1+16]	;; cosine/sine				; 4

	addpd	xmm2, xmm8		;; R3 = R1 + R3 (new R1)		; 5-7	avail 8,13+
	xcopy	xmm13, xmm12							; 5-7

	addpd	xmm3, xmm10		;; R4 = R2 + R4 (new R2)		; 6-8	avail 8,10,14+
	xcopy	xmm14, xmm12							; 6-8

	addpd	xmm7, xmm9		;; I4 = I2 + I4 (new I2)		; 7-9	avail 8.10.9,15
 	 xcopy	xmm8, xmm5		;; Copy I4				; 7-9

	addpd	xmm6, xmm11		;; I3 = I1 + I3 (new I1)		; 8-10	avail 10.9,15,11
	xcopy	xmm15, xmm12							; 8-10

	 addpd	xmm5, xmm0		;; I4 = R3 + I4 (final R4)		; 9-11	avail 10.9,11
	 xcopy	xmm9, xmm4		;; Copy I3				; 9-11
	xload	xmm10, [screg2+16]	;; cosine/sine				; 9

	 subpd	xmm0, xmm8		;; R3 = R3 - I4 (final R3)		; 10-12	avail 11,8

	xprefetchw [srcreg+srcinc]

	 subpd	xmm4, xmm1		;; I3 = I3 - R4 (final I4)		; 11-13	avail 11,8
	xcopy	xmm11, xmm10							; 11-13

	 addpd	xmm1, xmm9		;; R4 = I3 + R4 (final I3)		; 12-14	avail 8,9
	mulpd	xmm12, xmm5		;; A4 = R4 * cosine/sine		; 12-16
	 xcopy	xmm8, xmm2		;; Copy R1				; 12-14

	 subpd	xmm2, xmm3		;; R1 = R1 - R2 (final R2)		; 13-15	avail 9
	mulpd	xmm13, xmm0		;; A3 = R3 * cosine/sine		; 13-17
	 xcopy	xmm9, xmm6		;; Copy I1				; 13-15

	 subpd	xmm6, xmm7		;; I1 = I1 - I2 (final I2)		; 14-16	avail none
	mulpd	xmm14, xmm4		;; B4 = I4 * cosine/sine		; 14-18

	 addpd	xmm3, xmm8		;; R2 = R1 + R2 (final R1)		; 15-17	avail 8
	mulpd	xmm15, xmm1		;; B3 = I3 * cosine/sine		; 15-19
	xload	xmm8, [screg1]		;; Sine					; 15

	 addpd	xmm7, xmm9		;; I2 = I1 + I2 (final I1)		; 16-18	avail 9
	mulpd	xmm10, xmm2		;; A2 = R2 * cosine/sine		; 16-20
	xload	xmm9, [screg2]							; 16

	addpd	xmm12, xmm4		;; A4 = A4 + I4				; 17-19	avail 4
	mulpd	xmm11, xmm6		;; B2 = I2 * cosine/sine		; 17-21
	xload	xmm4, [srcreg+32]	;;#2 R1					; 17

	subpd	xmm13, xmm1		;; A3 = A3 - I3				; 18-20	avail 1
	xload	xmm1, [srcreg+d2+32]	;; next R3
	xstore	[srcreg], xmm3		;; Save R1				; 18	avail 3

	xprefetchw [srcreg+srcinc+d1]

	subpd	xmm14, xmm5		;; B4 = B4 - R4				; 19-21	avail 3,5
	xload	xmm3, [srcreg+d1+48]	;;#2 I2					; 19
	xstore	[srcreg+16], xmm7	;; Save I1				; 19	avail 5,7

	addpd	xmm15, xmm0		;; B3 = B3 + R3				; 20-22	avail 5,7,0
	mulpd	xmm12, xmm8		;; A4 = A4 * sine (final R4)		; 20-24
	xload	xmm5, [srcreg+d2+d1+48]	;;#2 I4

	subpd	xmm10, xmm6		;; A2 = A2 - I2				; 21-23	avail 7,0,6
	mulpd	xmm13, xmm8		;; A3 = A3 * sine (final R3)		; 21-25
	xload	xmm7, [srcreg+d1+32]	;;#2 R2

	addpd	xmm11, xmm2		;; B2 = B2 + R2				; 22-24	avail 0,6,2
	mulpd	xmm14, xmm8		;; B4 = B4 * sine (final I4)		; 22-26
	xload	xmm0, [srcreg+d2+d1+32]	;;#2 R4

	xcopy	xmm6, xmm4		;;#2 Copy R1				; 23-25	avail 2
	subpd	xmm4, xmm1		;;#2 R1 = R1 - R3 (new R3)		; 23-25
	mulpd	xmm15, xmm8		;; B3 = B3 * sine (final I3)		; 23-27	avail 2,8

	xcopy	xmm2, xmm3		;;#2 Copy I2				; 24-26	avail 8
	subpd	xmm3, xmm5		;;#2 I2 = I2 - I4 (new I4)		; 24-26
	mulpd	xmm10, xmm9		;; A2 = A2 * sine (final R2)		; 24-28

	xcopy	xmm8, xmm7		;;#2 Copy R2				; 25-27	avail none
	subpd	xmm7, xmm0		;;#2 R2 = R2 - R4 (new R4)		; 25-27
	mulpd	xmm11, xmm9		;; B2 = B2 * sine (final I2)		; 25-29	avail 9
	xload	xmm9, [srcreg+48]	;;#2 I1					; 25
	xstore	[srcreg+d1+32], xmm12	;; Save R4				; 25	avail 12

	xload	xmm12, [srcreg+d2+48]	;;#2 I3					; 26	avail none
	xstore	[srcreg+d1], xmm13	;; Save R3				; 26	avail 13
	xcopy	xmm13, xmm9		;;#2 Copy I1				; 26-28	avail none
	subpd	xmm9, xmm12		;;#2 I1 = I1 - I3 (new I3)		; 26-28

	addpd	xmm1, xmm6		;;#2 R3 = R1 + R3 (new R1)		; 27-29	avail 6
	xstore	[srcreg+d1+48], xmm14	;; Save I4				; 27	avail 6,14
	xload	xmm14, [screg1+16]	;;#2 cosine/sine			; 27	avail 6

	addpd	xmm0, xmm8		;;#2 R4 = R2 + R4 (new R2)		; 28-30	avail 6,8
	xstore	[srcreg+d1+16], xmm15	;; Save I3				; 28	avail 6,8,15
	xcopy	xmm15, xmm14							; 28-30	avail 6,8

	addpd	xmm5, xmm2		;;#2 I4 = I2 + I4 (new I2)		; 29-31	avail 6,8,2
	xstore	[srcreg+32], xmm10	;; Save R2				; 29	avail 6,8,2,10
 	 xcopy	xmm6, xmm3		;;#2 Copy I4				; 29-31	avail 8,2,10

	addpd	xmm12, xmm13		;;#2 I3 = I1 + I3 (new I1)		; 30-32	avail 8,2,10,13
	xstore	[srcreg+48], xmm11	;; Save I2				; 30	avail 8,2,10,13,11
	xcopy	xmm10, xmm14							; 30-32	avail 8,2,13,11

	xprefetchw [srcreg+srcinc+d2]

	 addpd	xmm3, xmm4		;;#2 I4 = R3 + I4 (final R4)		; 31-33
	xcopy	xmm11, xmm14							; 31-33	avail 8,2,13
	 xcopy	xmm2, xmm9		;;#2 Copy I3				; 31-33	avail 8,13

	 subpd	xmm4, xmm6		;;#2 R3 = R3 - I4 (final R3)		; 32-34	avail 8,13,6
	xload	xmm8, [screg2+16]	;;#2 cosine/sine			; 32	avail 13,6

	 subpd	xmm9, xmm7		;;#2 I3 = I3 - R4 (final I4)		; 33-35
	xcopy	xmm13, xmm8							; 33-35	avail 6

	 addpd	xmm7, xmm2		;;#2 R4 = I3 + R4 (final I3)		; 34-36	avail 6,2
	mulpd	xmm14, xmm3		;;#2 A4 = R4 * cosine/sine		; 34-38
	 xcopy	xmm6, xmm1		;;#2 Copy R1				; 34-36	avail 2

	 subpd	xmm1, xmm0		;;#2 R1 = R1 - R2 (final R2)		; 35-37
	mulpd	xmm15, xmm4		;;#2 A3 = R3 * cosine/sine		; 35-39
	 xcopy	xmm2, xmm12		;;#2 Copy I1				; 35-37	avail none

	 subpd	xmm12, xmm5		;;#2 I1 = I1 - I2 (final I2)		; 36-38
	mulpd	xmm10, xmm9		;;#2 B4 = I4 * cosine/sine		; 36-40

	 addpd	xmm0, xmm6		;;#2 R2 = R1 + R2 (final R1)		; 37-39	avail 6
	mulpd	xmm11, xmm7		;;#2 B3 = I3 * cosine/sine		; 37-41
	xload	xmm6, [screg1]		;;#2 Sine				; 37	avail none

	 addpd	xmm5, xmm2		;;#2 I2 = I1 + I2 (final I1)		; 38-40	avail 2
	mulpd	xmm8, xmm1		;;#2 A2 = R2 * cosine/sine		; 38-42
	xload	xmm2, [screg2]		;;#2 Sine				; 38	avail none

	addpd	xmm14, xmm9		;;#2 A4 = A4 + I4			; 39-41	avail 9
	mulpd	xmm13, xmm12		;;#2 B2 = I2 * cosine/sine		; 39-43

	subpd	xmm15, xmm7		;;#2 A3 = A3 - I3			; 40-42	avail 9,7
	xstore	[srcreg+d2], xmm0	;;#2 Save R1				; 40	avail 9,7,0

	xprefetchw [srcreg+srcinc+d2+d1]

	subpd	xmm10, xmm3		;;#2 B4 = B4 - R4			; 41-43	avail 9,7,0,3
	xstore	[srcreg+d2+16], xmm5	;;#2 Save I1				; 41	avail 9,7,0,3,5

	addpd	xmm11, xmm4		;;#2 B3 = B3 + R3			; 42-44	avail 9,7,0,3,5,4
	mulpd	xmm14, xmm6		;;#2 A4 = A4 * sine (final R4)		; 42-46

	subpd	xmm8, xmm12		;;#2 A2 = A2 - I2			; 43-45	avail 9,7,0,3,5,4,12
	mulpd	xmm15, xmm6		;;#2 A3 = A3 * sine (final R3)		; 43-47

	addpd	xmm13, xmm1		;;#2 B2 = B2 + R2			; 44-46	avail 9,7,0,3,5,4,12,1
	mulpd	xmm10, xmm6		;;#2 B4 = B4 * sine (final I4)		; 44-48

	mulpd	xmm11, xmm6		;;#2 B3 = B3 * sine (final I3)		; 45-49
	mulpd	xmm8, xmm2		;;#2 A2 = A2 * sine (final R2)		; 46-50
	mulpd	xmm13, xmm2		;;#2 B2 = B2 * sine (final I2)		; 47-51

	xstore	[srcreg+d2+d1+32], xmm14 ;;#2 Save R4				; 47
	xstore	[srcreg+d2+d1], xmm15	;;#2 Save R3				; 48
	xstore	[srcreg+d2+d1+48], xmm10 ;;#2 Save I4				; 49
	xstore	[srcreg+d2+d1+16], xmm11 ;;#2 Save I3				; 50
	xstore	[srcreg+d2+32], xmm8	;;#2 Save R2				; 51
	xstore	[srcreg+d2+48], xmm13	;;#2 Save I2				; 52

	bump	srcreg, srcinc
	ENDM

r4_f2cl_four_complex_djbfft_preload MACRO
	ENDM

r4_f2cl_four_complex_djbfft MACRO srcreg,srcinc,d1,screg
	r4_x4c_djbfft_mem64 [srcreg][rbx],[srcreg+d1][rbx],[srcreg+16][rbx],[srcreg+d1+16][rbx],[srcreg+32][rbx],[srcreg+d1+32][rbx],[srcreg+48][rbx],[srcreg+d1+48][rbx],screg,0,srcreg+srcinc+rbx,d1,[srcreg],[srcreg+16]
	xstore	[srcreg+d1+32], xmm12	;; Save R4
	xstore	[srcreg+d1], xmm13	;; Save R3
	xstore	[srcreg+d1+48], xmm14	;; Save I4
	xstore	[srcreg+d1+16], xmm15	;; Save I3
	xstore	[srcreg+32], xmm10	;; Save R2
	xstore	[srcreg+48], xmm11	;; Save I2
	bump	srcreg, srcinc
	ENDM

r4_nf2cl_four_complex_djbfft_preload MACRO
	ENDM

r4_nf2cl_four_complex_djbfft MACRO srcreg,srcinc,d1,screg
	r4_x4c_djbfft_mem64 [srcreg],[srcreg+d1],[srcreg+16],[srcreg+d1+16],[srcreg+32],[srcreg+d1+32],[srcreg+48],[srcreg+d1+48],screg,0,srcreg+srcinc,d1,[srcreg],[srcreg+16]
	xstore	[srcreg+d1+32], xmm12	;; Save R4
	xstore	[srcreg+d1], xmm13	;; Save R3
	xstore	[srcreg+d1+48], xmm14	;; Save I4
	xstore	[srcreg+d1+16], xmm15	;; Save I3
	xstore	[srcreg+32], xmm10	;; Save R2
	xstore	[srcreg+48], xmm11	;; Save I2
	bump	srcreg, srcinc
	ENDM

r4_x2cl_2sc_four_complex_djbfft_preload MACRO
	ENDM

r4_x2cl_2sc_four_complex_djbfft MACRO srcreg,srcinc,d1,screg1,screg2
	r4_x4c_2sc_djbfft_mem64 [srcreg],[srcreg+d1],[srcreg+16],[srcreg+d1+16],[srcreg+32],[srcreg+d1+32],[srcreg+48],[srcreg+d1+48],screg1,screg2,srcreg+srcinc,d1,[srcreg],[srcreg+16]
	xstore	[srcreg+d1+32], xmm12	;; Save R4
	xstore	[srcreg+d1], xmm13	;; Save R3
	xstore	[srcreg+d1+48], xmm14	;; Save I4
	xstore	[srcreg+d1+16], xmm15	;; Save I3
	xstore	[srcreg+32], xmm10	;; Save R2
	xstore	[srcreg+48], xmm11	;; Save I2
	bump	srcreg, srcinc
	ENDM

r4_x2cl_four_complex_djbfft_preload MACRO
	ENDM

r4_x2cl_four_complex_djbfft MACRO srcreg,srcinc,d1,screg1
	r4_x4c_2sc_djbfft_mem64 [srcreg],[srcreg+d1],[srcreg+16],[srcreg+d1+16],[srcreg+32],[srcreg+d1+32],[srcreg+48],[srcreg+d1+48],screg1,screg1+32,srcreg+srcinc,d1,[srcreg],[srcreg+16]
	xstore	[srcreg+d1+32], xmm12	;; Save R4
	xstore	[srcreg+d1], xmm13	;; Save R3
	xstore	[srcreg+d1+48], xmm14	;; Save I4
	xstore	[srcreg+d1+16], xmm15	;; Save I3
	xstore	[srcreg+32], xmm10	;; Save R2
	xstore	[srcreg+48], xmm11	;; Save I2
	bump	srcreg, srcinc
	ENDM

; Core 2 optimal is 22 clocks, currently at 24.4 clocks.

r4_x4c_djbfft_mem64 MACRO R1,R2,R3,R4,R5,R6,R7,R8,screg,off,pre1,pre2,dst1,dst2
	r4_x4c_2sc_djbfft_mem64 R1,R2,R3,R4,R5,R6,R7,R8,screg+off,screg+off+32,pre1,pre2,dst1,dst2
	ENDM
r4_x4c_2sc_djbfft_mem64 MACRO R1,R2,R3,R4,R5,R6,R7,R8,screg1,screg2,pre1,pre2,dst1,dst2
	xload	xmm0, R1		;; R1
	xload	xmm2, R3		;; R3
	xcopy	xmm8, xmm0		;; Copy R1				; 1-3
	subpd	xmm0, xmm2		;; R1 = R1 - R3 (new R3)		; 1-3
	xload	xmm5, R6		;; I2
	xload	xmm7, R8		;; I4
	xcopy	xmm9, xmm5		;; Copy I2				; 2-4
	subpd	xmm5, xmm7		;; I2 = I2 - I4 (new I4)		; 2-4
	xload	xmm1, R2		;; R2
	xload	xmm3, R4		;; R4
	xcopy	xmm10, xmm1		;; Copy R2				; 3-5
	subpd	xmm1, xmm3		;; R2 = R2 - R4 (new R4)		; 3-5
	xload	xmm4, R5		;; I1
	xload	xmm6, R7		;; I3
	xcopy	xmm11, xmm4		;; Copy I1				; 4-6
	subpd	xmm4, xmm6		;; I1 = I1 - I3 (new I3)		; 4-6
	xload	xmm12, [screg1+16]	;; cosine/sine				; 4

	addpd	xmm2, xmm8		;; R3 = R1 + R3 (new R1)		; 5-7
	xcopy	xmm13, xmm12							; 5-7
	addpd	xmm3, xmm10		;; R4 = R2 + R4 (new R2)		; 6-8
	xcopy	xmm14, xmm12							; 6-8
	addpd	xmm7, xmm9		;; I4 = I2 + I4 (new I2)		; 7-9
 	 xcopy	xmm8, xmm5		;; Copy I4				; 7-9
	addpd	xmm6, xmm11		;; I3 = I1 + I3 (new I1)		; 8-10
	xcopy	xmm15, xmm12							; 8-10

	xprefetchw [pre1]

	 addpd	xmm5, xmm0		;; I4 = R3 + I4 (final R4)		; 9-11
	 xcopy	xmm9, xmm4		;; Copy I3				; 9-11
	xload	xmm10, [screg2+16]	;; cosine/sine				; 9

	 subpd	xmm0, xmm8		;; R3 = R3 - I4 (final R3)		; 10-12

	 subpd	xmm4, xmm1		;; I3 = I3 - R4 (final I4)		; 11-13
	xcopy	xmm11, xmm10							; 11-13

	 addpd	xmm1, xmm9		;; R4 = I3 + R4 (final I3)		; 12-14
	mulpd	xmm12, xmm5		;; A4 = R4 * cosine/sine		; 12-16
	 xcopy	xmm8, xmm2		;; Copy R1				; 12-14

	 subpd	xmm2, xmm3		;; R1 = R1 - R2 (final R2)		; 13-15
	mulpd	xmm13, xmm0		;; A3 = R3 * cosine/sine		; 13-17
	 xcopy	xmm9, xmm6		;; Copy I1				; 13-15

	 subpd	xmm6, xmm7		;; I1 = I1 - I2 (final I2)		; 14-16
	mulpd	xmm14, xmm4		;; B4 = I4 * cosine/sine		; 14-18

	 addpd	xmm3, xmm8		;; R2 = R1 + R2 (final R1)		; 15-17
	mulpd	xmm15, xmm1		;; B3 = I3 * cosine/sine		; 15-19

	 addpd	xmm7, xmm9		;; I2 = I1 + I2 (final I1)		; 16-18
	mulpd	xmm10, xmm2		;; A2 = R2 * cosine/sine		; 16-20
	xload	xmm8, [screg1]		;; Sine					; 16

	addpd	xmm12, xmm4		;; A4 = A4 + I4				; 17-19
	mulpd	xmm11, xmm6		;; B2 = I2 * cosine/sine		; 17-21
	xload	xmm9, [screg2]							; 17

	subpd	xmm13, xmm1		;; A3 = A3 - I3				; 18-20
	xstore	dst1, xmm3		;; Save R1				; 18

	xprefetchw [pre1][pre2]

	subpd	xmm14, xmm5		;; B4 = B4 - R4				; 19-21
	xstore	dst2, xmm7		;; Save I1				; 19

	addpd	xmm15, xmm0		;; B3 = B3 + R3				; 20-22
	mulpd	xmm12, xmm8		;; A4 = A4 * sine (final R4)		; 20-24

	subpd	xmm10, xmm6		;; A2 = A2 - I2				; 21-23
	mulpd	xmm13, xmm8		;; A3 = A3 * sine (final R3)		; 21-25

	addpd	xmm11, xmm2		;; B2 = B2 + R2				; 22-24
	mulpd	xmm14, xmm8		;; B4 = B4 * sine (final I4)		; 22-26

	mulpd	xmm15, xmm8		;; B3 = B3 * sine (final I3)		; 23-27
	mulpd	xmm10, xmm9		;; A2 = A2 * sine (final R2)		; 24-28
	mulpd	xmm11, xmm9		;; B2 = B2 * sine (final I2)		; 25-29
	ENDM

ENDIF
ENDIF

;; 64-bit AMD K8 optimized versions of the above macros.  Same as 32-bit macros with some
;; constants preloaded.  We can probably do better than that.

IF (@INSTR(,%xarch,<K8>) NE 0)
IFDEF X86_64

r4_x4c_2sc_djbfft_mem_preload MACRO
	xload	xmm15, XMM_TWO
	ENDM

r4_x4c_2sc_djbfft_mem MACRO R1,R2,R3,R4,R5,R6,R7,R8,screg1,screg2,pre1,pre2,dst1,dst2
	xload	xmm0, R1		;; R1
	xload	xmm2, R3		;; R3
	subpd	xmm0, xmm2		;; R1 = R1 - R3 (new R3)		;; 1-4

	xload	xmm5, R6		;; I2
	xload	xmm7, R8		;; I4
	subpd	xmm5, xmm7		;; I2 = I2 - I4 (new I4)		;; 3-6

	xload	xmm4, R5		;; I1
	xload	xmm6, R7		;; I3
	subpd	xmm4, xmm6		;; I1 = I1 - I3 (new I3)		;; 5-8

	xload	xmm1, R2		;; R2
	xload	xmm3, R4		;; R4
	subpd	xmm1, xmm3		;; R2 = R2 - R4 (new R4)		;; 7-10

	addpd	xmm2, R1		;; R3 = R1 + R3 (new R1)		;; 9-12

	addpd	xmm3, R2		;; R4 = R2 + R4 (new R2)		;; 11-14

	addpd	xmm6, R5		;; I3 = I1 + I3 (new I1)		;; 13-16

	addpd	xmm7, R6		;; I4 = I2 + I4 (new I2)		;; 15-18

	xprefetchw [pre1]

	 subpd	xmm0, xmm5		;; R3 = R3 - I4 (final R3)		;; 17-20
	 mulpd	xmm5, xmm15		;; I4 = I4 * 2				;; 17-20

	 subpd	xmm4, xmm1		;; I3 = I3 - R4 (final I4)		;; 19-22
	 mulpd	xmm1, xmm15		;; R4 = R4 * 2				;; 19-22

	 subpd	xmm2, xmm3		;; R1 = R1 - R2 (final R2)		;; 21-24
	 mulpd	xmm3, xmm15		;; R2 = R2 * 2				;; 21-24

	 subpd	xmm6, xmm7		;; I1 = I1 - I2 (final I2)		;; 23-26
	 mulpd	xmm7, xmm15		;; I2 = I2 * 2				;; 23-26

	 addpd	xmm5, xmm0		;; I4 = R3 + I4 (final R4)		;; 25-28
	 addpd	xmm1, xmm4		;; R4 = I3 + R4 (final I3)		;; 27-30
	xprefetchw [pre1][pre2]
	 addpd	xmm3, xmm2		;; R2 = R1 + R2 (final R1)		;; 29-32
	 addpd	xmm7, xmm6		;; I2 = I1 + I2 (final I1)		;; 31-34

	xstore	dst1, xmm3		;; Save R1				;; 33

	xload	xmm3, [screg1+16]	;; cosine/sine
	mulpd	xmm3, xmm0		;; A3 = R3 * cosine/sine		;; 25-28

	xstore	dst2, xmm7		;; Save I1				;; 35

	xload	xmm7, [screg2+16]	;; cosine/sine
	mulpd	xmm7, xmm2		;; A2 = R2 * cosine/sine		;; 27-30

	subpd	xmm7, xmm6		;; A2 = A2 - I2				;; 33-36
	mulpd	xmm6, [screg2+16]	;; B2 = I2 * cosine/sine		;; 29-32

	subpd	xmm3, xmm1		;; A3 = A3 - I3				;; 35-38
	mulpd	xmm1, [screg1+16]	;; B3 = I3 * cosine/sine		;; 31-34

	addpd	xmm6, xmm2		;; B2 = B2 + R2				;; 37-40

	xload	xmm2, [screg1+16]	;; cosine/sine
	mulpd	xmm2, xmm5		;; A4 = R4 * cosine/sine		;; 33-36

	addpd	xmm2, xmm4		;; A4 = A4 + I4				;; 39-42
	mulpd	xmm4, [screg1+16]	;; B4 = I4 * cosine/sine		;; 35-38

	mulpd	xmm7, [screg2]		;; A2 = A2 * sine (final R2)		;; 37-40

	mulpd	xmm6, [screg2]		;; B2 = B2 * sine (final I2)		;; 41-44

	addpd	xmm1, xmm0		;; B3 = B3 + R3				;; 41-44
	xload	xmm0, [screg1]		;; Sine
	mulpd	xmm3, xmm0		;; A3 = A3 * sine (final R3)		;; 39-42

	subpd	xmm4, xmm5		;; B4 = B4 - R4				;; 43-46
	mulpd	xmm2, xmm0		;; A4 = A4 * sine (final R4)		;; 43-46

	mulpd	xmm1, xmm0		;; B3 = B3 * sine (final I3)		;; 45-48
	mulpd	xmm4, xmm0		;; B4 = B4 * sine (final I4)		;; 47-50
	ENDM

r4_x4c_2sc_djbfft_partial_mem_preload MACRO
	xload	xmm15, XMM_TWO
	ENDM

r4_x4c_2sc_djbfft_partial_mem MACRO r1,r2,r3,r4,r5,r6,r7,r8,mem3,mem4,mem7,mem8,screg1,screg2,pre1,pre2,dst1,dst2
	xload	r7, mem7						;; K8
	addpd	r7, r5			;; I3 = I1 + I3 (new I1)	;; 1-4
	subpd	r5, mem7		;; I1 = I1 - I3 (new I3)	;; 3-6

	xload	r4, mem4
	addpd	r4, r2			;; R4 = R2 + R4 (new R2)	;; 5-8
	subpd	r2, mem4		;; R2 = R2 - R4 (new R4)	;; 7-10

	xload	r3, mem3
	addpd	r3, r1			;; R3 = R1 + R3 (new R1)	;; 9-12
	subpd	r1, mem3		;; R1 = R1 - R3 (new R3)	;; 11-14

	xload	r8, mem8
	addpd	r8, r6			;; I4 = I2 + I4 (new I2)	;; 13-16
	subpd	r6, mem8		;; I2 = I2 - I4 (new I4)	;; 15-18

	xprefetchw [pre1]

	subpd	r5, r2			;; I3 = I3 - R4 (final I4)	;; 17-20
	mulpd	r2, xmm15		;; R4 = R4 * 2			;; 17-20

	subpd	r1, r6			;; R3 = R3 - I4 (final R3)	;; 19-22
	mulpd	r6, xmm15		;; I4 = I4 * 2			;; 19-22

	subpd	r3, r4			;; R1 = R1 - R2 (final R2)	;; 21-24
	mulpd	r4, xmm15		;; R2 = R2 * 2			;; 21-24

	subpd	r7, r8			;; I1 = I1 - I2 (final I2)	;; 23-26
	mulpd	r8, xmm15		;; I2 = I2 * 2			;; 23-26

	addpd	r6, r1			;; I4 = R3 + I4 (final R4)	;; 25-28
	addpd	r2, r5			;; R4 = I3 + R4 (final I3)	;; 27-30
	xprefetchw [pre1][pre2]
	addpd	r4, r3			;; R2 = R1 + R2 (final R1)	;; 29-32
	addpd	r8, r7			;; I2 = I1 + I2 (final I1)	;; 31-34

	xstore	dst1, r4		;; Save R1

	xload	r4, [screg1+16]		;; cosine/sine
	mulpd	r4, r1			;; A3 = R3 * cosine/sine	;; 25-28

	xstore	dst2, r8		;; Save I1

	xload	r8, [screg2+16]		;; cosine/sine
	mulpd	r8, r3			;; A2 = R2 * cosine/sine	;; 27-30

	subpd	r4, r2			;; A3 = A3 - I3			;; 33-36
	mulpd	r2, [screg1+16]		;; B3 = I3 * cosine/sine	;; 29-32

	addpd	r2, r1			;; B3 = B3 + R3			;; 35-38

	xload	r1, [screg1+16]		;; cosine/sine
	mulpd	r1, r6			;; A4 = R4 * cosine/sine	;; 31-34

	subpd	r8, r7			;; A2 = A2 - I2			;; 37-40
	mulpd	r7, [screg2+16]		;; B2 = I2 * cosine/sine	;; 33-36

	addpd	r1, r5			;; A4 = A4 + I4			;; 39-42
	mulpd	r5, [screg1+16]		;; B4 = I4 * cosine/sine	;; 35-38

	addpd	r7, r3			;; B2 = B2 + R2			;; 41-44
	xload	r3, [screg1]		;; sine
	mulpd	r4, r3			;; A3 = A3 * sine (final R3)	;; 37-40

	mulpd	r2, r3			;; B3 = B3 * sine (final I3)	;; 39-42

	mulpd	r8, [screg2]		;; A2 = A2 * sine (final R2)	;; 41-44

	subpd	r5, r6			;; B4 = B4 - R4			;; 43-46
	mulpd	r1, r3			;; A4 = A4 * sine (final R4)	;; 43-46

	mulpd	r7, [screg2]		;; B2 = B2 * sine (final I2)	;; 45-48

	mulpd	r5, r3			;; B4 = B4 * sine (final I4)	;; 47-50

	ENDM

ENDIF
ENDIF

;;
;; ************************************* four-complex-djbunfft variants ******************************************
;;

r4_x4cl_four_complex_djbunfft_preload MACRO
	r4_x4c_djbunfft_mem_preload
	;r4_x4c_djbunfft_partial_mem -- assume same as r4_x4c_djbunfft_mem_preload
	ENDM

r4_x4cl_four_complex_djbunfft MACRO srcreg,srcinc,d1,d2,screg,scoff
	d3 = d2 + d1
	r4_x4c_djbunfft_mem [srcreg+16],[srcreg+48],[srcreg+d1+16],[srcreg+d1+48],[srcreg+d2+16],[srcreg+d2+48],[srcreg+d3+16],[srcreg+d3+48],[srcreg+d1+16],screg,scoff,srcreg+srcinc+d2,d1
;;	xstore	[srcreg+d1+16], xmm6	;; Save R3
	xstore	[srcreg+d1+48], xmm3	;; Save I3
	xload	xmm6, [srcreg+d1]	;; R3
	xload	xmm3, [srcreg+d1+32]	;; R4
	xstore	[srcreg+d3+16], xmm0	;; Save R4
	xstore	[srcreg+d3+48], xmm1	;; Save I4
	xload	xmm0, [srcreg+d3]	;; R7
	xload	xmm1, [srcreg+d3+32]	;; R8
	xstore	[srcreg+d1], xmm7	;; Save R1
	xstore	[srcreg+d1+32], xmm4	;; Save I1
	xstore	[srcreg+d3], xmm5	;; Save R2
	xstore	[srcreg+d3+32], xmm2	;; Save I2
	r4_x4c_djbunfft_partial_mem xmm7,xmm4,xmm6,xmm3,xmm5,xmm2,xmm0,xmm1,[srcreg],[srcreg+32],[srcreg+d2],[srcreg+d2+32],[srcreg],screg,0,srcreg+srcinc,d1
;;	xstore	[srcreg], xmm4		;; Save R1
	xstore	[srcreg+32], xmm5	;; Save I1
	xstore	[srcreg+16], xmm6	;; Save R3
	xstore	[srcreg+48], xmm7	;; Save I3
	xstore	[srcreg+d2], xmm3	;; Save R2
	xstore	[srcreg+d2+32], xmm0	;; Save I2
	xstore	[srcreg+d2+16], xmm2	;; Save R4
	xstore	[srcreg+d2+48], xmm1	;; Save I4
	bump	srcreg, srcinc
	ENDM

;; Used in pass 2 when there are radix-3 levels.  Uses 2 sin/cos ptrs

r4_x4cl_2sc_four_complex_djbunfft_preload MACRO
	r4_x4c_2sc_djbunfft_mem_preload
	;r4_x4c_2sc_djbunfft_partial_mem_preload -- assume same as r4_x4c_2sc_djbunfft_mem_preload
	ENDM

r4_x4cl_2sc_four_complex_djbunfft MACRO srcreg,srcinc,d1,d2,screg1,scoff1,screg2,scoff2
	d3 = d2 + d1
	r4_x4c_2sc_djbunfft_mem [srcreg+16],[srcreg+48],[srcreg+d1+16],[srcreg+d1+48],[srcreg+d2+16],[srcreg+d2+48],[srcreg+d3+16],[srcreg+d3+48],[srcreg+d1+16],screg1+scoff1,screg2+scoff2,srcreg+srcinc+d2,d1
;;	xstore	[srcreg+d1+16], xmm6	;; Save R3
	xstore	[srcreg+d1+48], xmm3	;; Save I3
	xload	xmm6, [srcreg+d1]	;; R3
	xload	xmm3, [srcreg+d1+32]	;; R4
	xstore	[srcreg+d3+16], xmm0	;; Save R4
	xstore	[srcreg+d3+48], xmm1	;; Save I4
	xload	xmm0, [srcreg+d3]	;; R7
	xload	xmm1, [srcreg+d3+32]	;; R8
	xstore	[srcreg+d1], xmm7	;; Save R1
	xstore	[srcreg+d1+32], xmm4	;; Save I1
	xstore	[srcreg+d3], xmm5	;; Save R2
	xstore	[srcreg+d3+32], xmm2	;; Save I2
	r4_x4c_2sc_djbunfft_partial_mem xmm7,xmm4,xmm6,xmm3,xmm5,xmm2,xmm0,xmm1,[srcreg],[srcreg+32],[srcreg+d2],[srcreg+d2+32],[srcreg],screg1,screg2,srcreg+srcinc,d1
;;	xstore	[srcreg], xmm4		;; Save R1
	xstore	[srcreg+32], xmm5	;; Save I1
	xstore	[srcreg+16], xmm6	;; Save R3
	xstore	[srcreg+48], xmm7	;; Save I3
	xstore	[srcreg+d2], xmm3	;; Save R2
	xstore	[srcreg+d2+32], xmm0	;; Save I2
	xstore	[srcreg+d2+16], xmm2	;; Save R4
	xstore	[srcreg+d2+48], xmm1	;; Save I4
	bump	srcreg, srcinc
	ENDM

;; Used in last levels of an r4 FFT pass 1 (not in r4delay FFT).  No swizzling.
IFDEF UNUSED
r4_g2cl_four_complex_djbunfft_preload MACRO
	r4_x4c_djbunfft_mem_preload
	ENDM
r4_g2cl_four_complex_djbunfft MACRO srcreg,srcinc,d1,dstreg,dstinc,e1,screg
	r4_x4c_djbunfft_mem [srcreg],[srcreg+16],[srcreg+32],[srcreg+48],[srcreg+d1],[srcreg+d1+16],[srcreg+d1+32],[srcreg+d1+48],[dstreg+16],screg,0,dstreg+dstinc,e1
	bump	srcreg, srcinc
	xstore	[dstreg], xmm7		;; Save R1
;;	xstore	[dstreg+16], xmm6	;; Save R3
	xstore	[dstreg+32], xmm4	;; Save I1
	xstore	[dstreg+48], xmm3	;; Save I3
	xstore	[dstreg+e1], xmm5	;; Save R2
	xstore	[dstreg+e1+16], xmm0	;; Save R4
	xstore	[dstreg+e1+32], xmm2	;; Save I2
	xstore	[dstreg+e1+48], xmm1	;; Save I4
	bump	dstreg, dstinc
	ENDM

;; Used in first levels of pass 2.  Swizzling.
r4_s4cl_four_complex_djbunfft MACRO srcreg,srcinc,d1,d2,screg,scdist
	r4_x4c_djbunfft_mem_shuffle [srcreg+16],[srcreg+48],[srcreg+d1+16],[srcreg+d1+48],[srcreg+d2+16],[srcreg+d2+48],[srcreg+d2+d1+16],[srcreg+d2+d1+48],[srcreg+d1+16],[srcreg+d1+48],screg,scdist,srcreg+srcinc+d2,d1
;;	shuffle_store [srcreg+d1+16], [srcreg+d1+48], xmm2, xmm6 ;; Save I1, I3
	xload	xmm2, [srcreg+d1]	;; R3
	xload	xmm6, [srcreg+d1+32]	;; R4
	shuffle_store [srcreg+d1], [srcreg+d1+32], xmm5, xmm3 ;; Save R1, R3
	xload	xmm5, [srcreg+d2+d1]	;; R7
	xload	xmm3, [srcreg+d2+d1+32]	;; R8
	shuffle_store [srcreg+d2+d1], [srcreg+d2+d1+32], xmm7, xmm0 ;; Save R2, R4
	shuffle_store [srcreg+d2+d1+16], [srcreg+d2+d1+48], xmm4, xmm1 ;; Save I2, I4
	r4_x4c_djbunfft_partial_mem_shuffle xmm7,xmm0,xmm2,xmm6,xmm4,xmm1,xmm5,xmm3,[srcreg],[srcreg+32],[srcreg+d2],[srcreg+d2+32],[srcreg],[srcreg+32],screg,0,srcreg+srcinc,d1
;;	shuffle_store [srcreg], [srcreg+32], xmm0, xmm2 ;; Save R1, R3
	shuffle_store [srcreg+16], [srcreg+48], xmm4, xmm7 ;; Save I1, I3
	shuffle_store [srcreg+d2], [srcreg+d2+32], xmm6, xmm1 ;; Save R2, R4
	shuffle_store [srcreg+d2+16], [srcreg+d2+48], xmm5, xmm3 ;; Save I2, I4
	bump	srcreg, srcinc
	ENDM
ENDIF

;; Used in last levels of an r4 FFT pass 1 (not in r4delay FFT).  Swizzling.
r4_sg2cl_four_complex_djbunfft MACRO srcreg,srcinc,d1,dstreg,dstinc,e1,screg
	shuffle_load_with_temp xmm7, xmm2, [srcreg+d1], [srcreg+d1+16], xmm1 ;; R3,R4
	xload	xmm1, [screg+16]	;; cosine/sine
	xcopy	xmm3, xmm7		;; Copy R3
	mulpd	xmm7, xmm1		;; A3 = R3 * cosine/sine
	xcopy	xmm0, xmm2		;; Copy R4
	mulpd	xmm2, xmm1		;; A4 = R4 * cosine/sine

	shuffle_load_with_temp xmm5, xmm4, [srcreg+d1+32], [srcreg+d1+48], xmm6 ;; I3,I4
	addpd	xmm7, xmm5		;; A3 = A3 + I3
	mulpd	xmm5, xmm1		;; B3 = I3 * cosine/sine
	subpd	xmm2, xmm4		;; A4 = A4 - I4
	mulpd	xmm4, xmm1		;; B4 = I4 * cosine/sine
	subpd	xmm5, xmm3		;; B3 = B3 - R3
	addpd	xmm4, xmm0		;; B4 = B4 + R4

	shuffle_load_with_temp xmm0, xmm6, [srcreg], [srcreg+16], xmm1	;; R1,R2
	xcopy	xmm1, xmm6		;; Copy R2
	mulpd	xmm6, [screg+48]	;; A2 = R2 * cosine/sine

	xload	xmm3, [screg]		;; Sine
	mulpd	xmm7, xmm3		;; A3 = A3 * sine (new R3)
	mulpd	xmm5, xmm3		;; B3 = B3 * sine (new I3)
	mulpd	xmm2, xmm3		;; A4 = A4 * sine (new R4)
	mulpd	xmm4, xmm3		;; B4 = B4 * sine (new I4)

	xstore	[dstreg], xmm5		;; Temporarily save I3

	shuffle_load xmm5, xmm3, [srcreg+32], [srcreg+48] ;; I1,I2
	addpd	xmm6, xmm3		;; A2 = A2 + I2
	mulpd	xmm3, [screg+48]	;; B2 = I2 * cosine/sine
	subpd	xmm3, xmm1		;; B2 = B2 - R2
	xload	xmm1, [screg+32]
	mulpd	xmm6, xmm1		;; A2 = A2 * sine (new R2)
	mulpd	xmm3, xmm1		;; B2 = B2 * sine (new I2)

	xprefetchw [dstreg+dstinc]
	bump	srcreg, srcinc

	xcopy	xmm1, xmm0		;; Copy R1
	subpd	xmm0, xmm6		;; R1 = R1 - R2 (new R2)
	addpd	xmm6, xmm1		;; R2 = R1 + R2 (new R1)

	xcopy	xmm1, xmm2		;; Copy R4
	subpd	xmm2, xmm7		;; R4 = R4 - R3 (new I4)
	addpd	xmm7, xmm1		;; R3 = R4 + R3 (new R3)

	xcopy	xmm1, xmm5		;; Copy I1
	subpd	xmm5, xmm3		;; I1 = I1 - I2 (new I2)
	addpd	xmm3, xmm1		;; I2 = I1 + I2 (new I1)

	xcopy	xmm1, xmm7		;; Copy R3
	addpd	xmm7, xmm6		;; R3 = R1 + R3 (final R1)
	subpd	xmm6, xmm1		;; R1 = R1 - R3 (final R3)

	xload	xmm1, [dstreg]		;; Reload I3
	xstore	[dstreg], xmm7		;; Save R1

	xcopy	xmm7, xmm1		;; Copy I3
	subpd	xmm1, xmm4		;; I3 = I3 - I4 (new R4)
	addpd	xmm4, xmm7		;; I4 = I3 + I4 (new I3)

	xprefetchw [dstreg+dstinc+e1]

	xcopy	xmm7, xmm0		;; Copy R2
	subpd	xmm0, xmm1		;; R2 = R2 - R4 (final R4)
	addpd	xmm1, xmm7		;; R4 = R2 + R4 (final R2)

	xcopy	xmm7, xmm5		;; Copy I2
	subpd	xmm5, xmm2		;; I2 = I2 - I4 (final I4)
	addpd	xmm2, xmm7		;; I4 = I2 + I4 (final I2)

	xcopy	xmm7, xmm3		;; Copy I1
	subpd	xmm3, xmm4		;; I1 = I1 - I3 (final I3)
	addpd	xmm4, xmm7		;; I3 = I1 + I3 (final I1)

	xstore	[dstreg+16], xmm6	;; Save R3
	xstore	[dstreg+32], xmm4	;; Save I1
	xstore	[dstreg+48], xmm3	;; Save I3
	xstore	[dstreg+e1], xmm1	;; Save R2
	xstore	[dstreg+e1+16], xmm0	;; Save R4
	xstore	[dstreg+e1+32], xmm2	;; Save I2
	xstore	[dstreg+e1+48], xmm5	;; Save I4
	bump	dstreg, dstinc
	ENDM

r4_x4c_djbunfft_mem_preload MACRO
	r4_x4c_2sc_djbunfft_mem_preload
	ENDM

r4_x4c_djbunfft_mem MACRO mem1,mem2,mem3,mem4,mem5,mem6,mem7,mem8,dst3,screg,off,pre1,pre2
	r4_x4c_2sc_djbunfft_mem mem1,mem2,mem3,mem4,mem5,mem6,mem7,mem8,dst3,screg+off,screg+off+32,pre1,pre2
	ENDM

r4_x4c_2sc_djbunfft_mem_preload MACRO
	ENDM

r4_x4c_2sc_djbunfft_mem MACRO mem1,mem2,mem3,mem4,mem5,mem6,mem7,mem8,dst3,screg1,screg2,pre1,pre2
	xload	xmm3, [screg2+16]	;; cosine/sine
	xload	xmm6, mem3		;; R2
	mulpd	xmm6, xmm3		;; A2 = R2 * cosine/sine
	xload	xmm0, mem4		;; I2
	mulpd	xmm3, xmm0		;; B2 = I2 * cosine/sine

	xload	xmm5, [screg1+16]	;; cosine/sine
	xload	xmm7, mem5		;; R3
	mulpd	xmm7, xmm5		;; A3 = R3 * cosine/sine
	xload	xmm1, mem6		;; I3
	xcopy	xmm4, xmm5		;; cosine/sine
	mulpd	xmm5, xmm1		;; B3 = I3 * cosine/sine

	xload	xmm2, mem7		;; R4
	mulpd	xmm2, xmm4		;; A4 = R4 * cosine/sine
	addpd	xmm6, xmm0		;; A2 = A2 + I2
	xload	xmm0, mem8		;; I4
	mulpd	xmm4, xmm0		;; B4 = I4 * cosine/sine

	subpd	xmm3, mem3		;; B2 = B2 - R2
	addpd	xmm7, xmm1		;; A3 = A3 + I3
	subpd	xmm5, mem5		;; B3 = B3 - R3
	subpd	xmm2, xmm0		;; A4 = A4 - I4
	addpd	xmm4, mem7		;; B4 = B4 + R4

	mulpd	xmm6, [screg2]		;; A2 = A2 * sine (new R2)
	mulpd	xmm3, [screg2]		;; B2 = B2 * sine (new I2)
	xload	xmm1, [screg1]		;; Sine
	mulpd	xmm7, xmm1		;; A3 = A3 * sine (new R3)
	mulpd	xmm5, xmm1		;; B3 = B3 * sine (new I3)
	mulpd	xmm2, xmm1		;; A4 = A4 * sine (new R4)
	mulpd	xmm4, xmm1		;; B4 = B4 * sine (new I4)

	xprefetchw [pre1]

	xload	xmm0, mem1		;; R1
	subpd	xmm0, xmm6		;; R1 = R1 - R2 (new R2)
	addpd	xmm6, mem1		;; R2 = R1 + R2 (new R1)

	xcopy	xmm1, xmm2		;; Copy R4
	subpd	xmm2, xmm7		;; R4 = R4 - R3 (new I4)
	addpd	xmm7, xmm1		;; R3 = R4 + R3 (new R3)

	xcopy	xmm1, xmm5		;; Copy I3
	subpd	xmm5, xmm4		;; I3 = I3 - I4 (new R4)
	addpd	xmm4, xmm1		;; I4 = I3 + I4 (new I3)

	xcopy	xmm1, xmm6		;; Copy R1
	subpd	xmm6, xmm7		;; R1 = R1 - R3 (final R3)
	addpd	xmm7, xmm1		;; R3 = R1 + R3 (final R1)

	xload	xmm1, mem2		;; I1
	subpd	xmm1, xmm3		;; I1 = I1 - I2 (new I2)
	addpd	xmm3, mem2		;; I2 = I1 + I2 (new I1)

	xprefetchw [pre1][pre2]

	IFB <dst3>

	subpd	xmm0, xmm5		;; R2 = R2 - R4 (final R4)
	multwo	xmm5			;; R4 = R4 * 2
	addpd	xmm5, xmm0		;; R4 = R2 + R4 (final R2)

	subpd	xmm1, xmm2		;; I2 = I2 - I4 (final I4)
	multwo	xmm2			;; I4 = I4 * 2
	addpd	xmm2, xmm1		;; I4 = I2 + I4 (final I2)

	subpd	xmm3, xmm4		;; I1 = I1 - I3 (final I3)
	multwo	xmm4			;; I3 = I3 * 2
	addpd	xmm4, xmm3		;; I3 = I1 + I3 (final I1)

	ELSE

	xstore	dst3, xmm6

	xcopy	xmm6, xmm0		;; Copy R2
	subpd	xmm0, xmm5		;; R2 = R2 - R4 (final R4)
	addpd	xmm5, xmm6		;; R4 = R2 + R4 (final R2)

	xcopy	xmm6, xmm1		;; Copy I2
	subpd	xmm1, xmm2		;; I2 = I2 - I4 (final I4)
	addpd	xmm2, xmm6		;; I4 = I2 + I4 (final I2)

	xcopy	xmm6, xmm3		;; Copy I1
	subpd	xmm3, xmm4		;; I1 = I1 - I3 (final I3)
	addpd	xmm4, xmm6		;; I3 = I1 + I3 (final I1)

	ENDIF
	ENDM

r4_x4c_djbunfft_partial_mem_preload MACRO
	r4_x4c_2sc_djbunfft_partial_mem_preload
	ENDM

r4_x4c_djbunfft_partial_mem MACRO r1,r2,r3,r4,r5,r6,r7,r8,mem1,mem2,mem5,mem6,dst1,screg,off,pre1,pre2
	r4_x4c_2sc_djbunfft_partial_mem r1,r2,r3,r4,r5,r6,r7,r8,mem1,mem2,mem5,mem6,dst1,screg+off,screg+off+32,pre1,pre2
	ENDM

r4_x4c_2sc_djbunfft_partial_mem_preload MACRO
	ENDM

r4_x4c_2sc_djbunfft_partial_mem MACRO r1,r2,r3,r4,r5,r6,r7,r8,mem1,mem2,mem5,mem6,dst1,screg1,screg2,pre1,pre2
	xload	r1, [screg2+16]		;; cosine/sine
	xcopy	r2, r3			;; Copy R2
	mulpd	r3, r1			;; A2 = R2 * cosine/sine
	mulpd	r1, r4			;; B2 = I2 * cosine/sine

	xload	r5, [screg1+16]		;; cosine/sine
	xcopy	r6, r7			;; Copy R4
	mulpd	r7, r5			;; A4 = R4 * cosine/sine
	addpd	r3, r4			;; A2 = A2 + I2
	xcopy	r4, r5			;; cosine/sine
	mulpd	r5, r8			;; B4 = I4 * cosine/sine
	subpd	r1, r2			;; B2 = B2 - R2

	xload	r2, mem5		;; R3
	mulpd	r2, r4			;; A3 = R3 * cosine/sine
	mulpd	r4, mem6		;; B3 = I3 * cosine/sine

	subpd	r7, r8			;; A4 = A4 - I4
	addpd	r5, r6			;; B4 = B4 + R4
	addpd	r2, mem6		;; A3 = A3 + I3
	subpd	r4, mem5		;; B3 = B3 - R3

	mulpd	r3, [screg2]		;; A2 = A2 * sine (new R2)
	mulpd	r1, [screg2]		;; B2 = B2 * sine (new I2)
	xload	r8, [screg1]		;; Sine
	mulpd	r7, r8			;; A4 = A4 * sine (new R4)
	mulpd	r5, r8			;; B4 = B4 * sine (new I4)
	mulpd	r2, r8			;; A3 = A3 * sine (new R3)
	mulpd	r4, r8			;; B3 = B3 * sine (new I3)

	xprefetchw [pre1]

	xload	r6, mem1		;; R1
	subpd	r6, r3			;; R1 = R1 - R2 (new R2)
	addpd	r3, mem1		;; R2 = R1 + R2 (new R1)

	xcopy	r8, r7			;; Copy R4
	subpd	r7, r2			;; R4 = R4 - R3 (new I4)
	addpd	r2, r8			;; R3 = R4 + R3 (new R3)

	xcopy	r8, r4			;; Copy I3
	subpd	r4, r5			;; I3 = I3 - I4 (new R4)
	addpd	r5, r8			;; I4 = I3 + I4 (new I3)

	xcopy	r8, r3			;; Copy R1
	subpd	r3, r2			;; R1 = R1 - R3 (final R3)
	addpd	r2, r8			;; R3 = R1 + R3 (final R1)

	xload	r8, mem2		;; I1
	subpd	r8, r1			;; I1 = I1 - I2 (new I2)
	addpd	r1, mem2		;; I2 = I1 + I2 (new I1)

	xprefetchw [pre1][pre2]

	IFB <dst1>

	subpd	r6, r4			;; R2 = R2 - R4 (final R4)
	multwo	r4			;; R4 = R4 * 2
	addpd	r4, r6			;; R4 = R2 + R4 (final R2)

	subpd	r8, r7			;; I2 = I2 - I4 (final I4)
	multwo	r7			;; I4 = I4 * 2
	addpd	r7, r8			;; I4 = I2 + I4 (final I2)

	subpd	r1, r5			;; I1 = I1 - I3 (final I3)
	multwo	r5			;; I3 = I3 * 2
	addpd	r5, r1			;; I3 = I1 + I3 (final I1)

	ELSE

	xstore	dst1, r2

	xcopy	r2, r6			;; Copy R2
	subpd	r6, r4			;; R2 = R2 - R4 (final R4)
	addpd	r4, r2			;; R4 = R2 + R4 (final R2)

	xcopy	r2, r8			;; Copy I2
	subpd	r8, r7			;; I2 = I2 - I4 (final I4)
	addpd	r7, r2			;; I4 = I2 + I4 (final I2)

	xcopy	r2, r1			;; Copy I1
	subpd	r1, r5			;; I1 = I1 - I3 (final I3)
	addpd	r5, r2			;; I3 = I1 + I3 (final I1)

	ENDIF
	ENDM

IFDEF UNUSED
r4_x4c_djbunfft_mem_shuffle MACRO mem1,mem2,mem3,mem4,mem5,mem6,mem7,mem8,dst5,dst7,screg,off,pre1,pre2
	xload	xmm3, [screg+off+32+16]	;; cosine/sine
	xload	xmm6, mem4		;; I2
	mulpd	xmm6, xmm3		;; B2 = I2 * cosine/sine
	xload	xmm0, mem3		;; R2
	mulpd	xmm3, xmm0		;; A2 = R2 * cosine/sine

	xload	xmm5, [screg+off+0+16]	;; cosine/sine
	xload	xmm7, mem6		;; I3
	mulpd	xmm7, xmm5		;; B3 = I3 * cosine/sine
	xload	xmm1, mem5		;; R3
	xcopy	xmm4, xmm5		;; cosine/sine
	mulpd	xmm5, xmm1		;; A3 = R3 * cosine/sine

	xload	xmm2, mem8		;; I4
	mulpd	xmm2, xmm4		;; B4 = I4 * cosine/sine
	subpd	xmm6, xmm0		;; B2 = B2 - R2
	xload	xmm0, mem7		;; R4
	mulpd	xmm4, xmm0		;; A4 = R4 * cosine/sine

	subpd	xmm7, xmm1		;; B3 = B3 - R3
	addpd	xmm2, xmm0		;; B4 = B4 + R4
	addpd	xmm3, mem4		;; A2 = A2 + I2
	addpd	xmm5, mem6		;; A3 = A3 + I3
	subpd	xmm4, mem8		;; A4 = A4 - I4

	mulpd	xmm6, [screg+off+32]	;; B2 = B2 * sine (new I2)
	xload	xmm1, [screg+off+0]	;; Sine
	mulpd	xmm7, xmm1		;; B3 = B3 * sine (new I3)
	mulpd	xmm2, xmm1		;; B4 = B4 * sine (new I4)
	mulpd	xmm3, [screg+off+32]	;; A2 = A2 * sine (new R2)
	mulpd	xmm5, xmm1		;; A3 = A3 * sine (new R3)
	mulpd	xmm4, xmm1		;; A4 = A4 * sine (new R4)

	xprefetchw [pre1]

	xload	xmm1, mem2		;; I1
	subpd	xmm1, xmm6		;; I1 = I1 - I2 (new I2)
	addpd	xmm6, mem2		;; I2 = I1 + I2 (new I1)

	xcopy	xmm0, xmm7		;; Copy I3
	subpd	xmm7, xmm2		;; I3 = I3 - I4 (new R4)
	addpd	xmm2, xmm0		;; I4 = I3 + I4 (new I3)

	xcopy	xmm0, xmm4		;; Copy R4
	subpd	xmm4, xmm5		;; R4 = R4 - R3 (new I4)
	addpd	xmm5, xmm0		;; R3 = R4 + R3 (new R3)

	xcopy	xmm0, xmm6		;; Copy I1
	subpd	xmm6, xmm2		;; I1 = I1 - I3 (final I3)
	addpd	xmm2, xmm0		;; I3 = I1 + I3 (final I1)

	xload	xmm0, mem1		;; R1
	subpd	xmm0, xmm3		;; R1 = R1 - R2 (new R2)
	addpd	xmm3, mem1		;; R2 = R1 + R2 (new R1)

	xprefetchw [pre1][pre2]

	shuffle_store dst5, dst7, xmm2, xmm6 ;; Save I1, I3

	xcopy	xmm6, xmm1		;; Copy I2
	subpd	xmm1, xmm4		;; I2 = I2 - I4 (final I4)
	addpd	xmm4, xmm6		;; I4 = I2 + I4 (final I2)

	xcopy	xmm2, xmm3		;; Copy R1
	subpd	xmm3, xmm5		;; R1 = R1 - R3 (final R3)
	addpd	xmm5, xmm2		;; R3 = R1 + R3 (final R1)

	xcopy	xmm6, xmm0		;; Copy R2
	subpd	xmm0, xmm7		;; R2 = R2 - R4 (final R4)
	addpd	xmm7, xmm6		;; R4 = R2 + R4 (final R2)
	ENDM

r4_x4c_djbunfft_partial_mem_shuffle MACRO r1,r2,r3,r4,r5,r6,r7,r8,mem1,mem2,mem5,mem6,dst1,dst3,screg,off,pre1,pre2
	xload	r1, [screg+off+32+16]	;; cosine/sine
	xcopy	r2, r3			;; Copy R2
	mulpd	r3, r1			;; A2 = R2 * cosine/sine
	mulpd	r1, r4			;; B2 = I2 * cosine/sine

	xload	r5, [screg+off+0+16]	;; cosine/sine
	xcopy	r6, r7			;; Copy R4
	mulpd	r7, r5			;; A4 = R4 * cosine/sine
	addpd	r3, r4			;; A2 = A2 + I2
	xcopy	r4, r5			;; cosine/sine
	mulpd	r5, r8			;; B4 = I4 * cosine/sine
	subpd	r1, r2			;; B2 = B2 - R2

	xload	r2, mem5		;; R3
	mulpd	r2, r4			;; A3 = R3 * cosine/sine
	mulpd	r4, mem6		;; B3 = I3 * cosine/sine

	subpd	r7, r8			;; A4 = A4 - I4
	addpd	r5, r6			;; B4 = B4 + R4
	addpd	r2, mem6		;; A3 = A3 + I3
	subpd	r4, mem5		;; B3 = B3 - R3

	mulpd	r3, [screg+off+32]	;; A2 = A2 * sine (new R2)
	mulpd	r1, [screg+off+32]	;; B2 = B2 * sine (new I2)
	xload	r8, [screg+off+0]	;; Sine
	mulpd	r7, r8			;; A4 = A4 * sine (new R4)
	mulpd	r5, r8			;; B4 = B4 * sine (new I4)
	mulpd	r2, r8			;; A3 = A3 * sine (new R3)
	mulpd	r4, r8			;; B3 = B3 * sine (new I3)

	xprefetchw [pre1]

	xload	r6, mem1		;; R1
	subpd	r6, r3			;; R1 = R1 - R2 (new R2)
	addpd	r3, mem1		;; R2 = R1 + R2 (new R1)

	xcopy	r8, r7			;; Copy R4
	subpd	r7, r2			;; R4 = R4 - R3 (new I4)
	addpd	r2, r8			;; R3 = R4 + R3 (new R3)

	xcopy	r8, r4			;; Copy I3
	subpd	r4, r5			;; I3 = I3 - I4 (new R4)
	addpd	r5, r8			;; I4 = I3 + I4 (new I3)

	xcopy	r8, r3			;; Copy R1
	subpd	r3, r2			;; R1 = R1 - R3 (final R3)
	addpd	r2, r8			;; R3 = R1 + R3 (final R1)

	xload	r8, mem2		;; I1
	subpd	r8, r1			;; I1 = I1 - I2 (new I2)
	addpd	r1, mem2		;; I2 = I1 + I2 (new I1)

	xprefetchw [pre1][pre2]

	shuffle_store dst1, dst3, r2, r3	 ;; Save R1 and R3

	xcopy	r2, r6			;; Copy R2
	subpd	r6, r4			;; R2 = R2 - R4 (final R4)
	addpd	r4, r2			;; R4 = R2 + R4 (final R2)

	xcopy	r3, r8			;; Copy I2
	subpd	r8, r7			;; I2 = I2 - I4 (final I4)
	addpd	r7, r3			;; I4 = I2 + I4 (final I2)

	xcopy	r2, r1			;; Copy I1
	subpd	r1, r5			;; I1 = I1 - I3 (final I3)
	addpd	r5, r2			;; I3 = I1 + I3 (final I1)
	ENDM
ENDIF

IFDEF UNUSED
r4_x4c_djbunfft MACRO r1,r2,r3,r4,r5,r6,r7,r8,mem7,mem8,dst1,screg,off,pre1,pre2
	xload	r7, [screg+off+32+16]	;; cosine/sine
	mulpd	r7, r3			;; A2 = R2 * cosine/sine
	addpd	r7, r4			;; A2 = A2 + I2

	xload	r8, [screg+off+0+16]	;; cosine/sine
	mulpd	r8, r5			;; A3 = R3 * cosine/sine
	addpd	r8, r6			;; A3 = A3 + I3

	  mulpd	r4, [screg+off+32+16]	;; B2 = I2 * cosine/sine
	  subpd	r4, r3			;; B2 = B2 - R2

	xload	r3, mem7		;; R4
	mulpd	r3, [screg+off+0+16]	;; A4 = R4 * cosine/sine
	subpd	r3, mem8		;; A4 = A4 - I4

	mulpd	r7, [screg+off+32]	;; A2 = A2 * sine (new R2)
	mulpd	r8, [screg+off+0]	;; A3 = A3 * sine (new R3)
	mulpd	r3, [screg+off+0]	;; A4 = A4 * sine (new R4)

	xprefetchw [pre1]

	  mulpd	r6, [screg+off+0+16]	;; B3 = I3 * cosine/sine
	  subpd	r6, r5			;; B3 = B3 - R3

	xcopy	r5, r7			;; R2 = R2 * 2
	addpd	r7, r1			;; R2 = R1 + R2 (new R1)
	  subpd	r1, r5			;; R1 = R1 - R2 (new R2)

	xcopy	r5, r8 			;; R3 = R3 * 2
	addpd	r8, r3			;; R3 = R4 + R3 (new R3)
	  subpd	r3, r5			;; R4 = R4 - R3 (new I4)

	xcopy	r5, r8
	addpd	r8, r7			;; R3 = R1 + R3 (final R1)
	  subpd	r7, r5			;; R1 = R1 - R3 (final R3)

	xload	r5, mem8		;; I4
	mulpd	r5, [screg+off+0+16]	;; B4 = I4 * cosine/sine
	addpd	r5, mem7		;; B4 = B4 + R4

	mulpd	r4, [screg+off+32]	;; B2 = B2 * sine (new I2)
	mulpd	r6, [screg+off+0]	;; B3 = B3 * sine (new I3)
	mulpd	r5, [screg+off+0]	;; B4 = B4 * sine (new I4)

	xprefetchw [pre1][pre2]

	IFB <dst1>

	subpd	r2, r4			;; I1 = I1 - I2 (new I2)
	multwo	r4			;; I2 = I2 * 2
	addpd	r4, r2			;; I2 = I1 + I2 (new I1)

	subpd	r6, r5			;; I3 = I3 - I4 (new R4)
	multwo	r5			;; I4 = I4 * 2
	addpd	r5, r6			;; I4 = I3 + I4 (new I3)

	subpd	r1, r6			;; R2 = R2 - R4 (final R4)
	multwo	r6			;; R4 = R4 * 2
	addpd	r6, r1			;; R4 = R2 + R4 (final R2)

	subpd	r2, r3			;; I2 = I2 - I4 (final I4)
	multwo	r3			;; I4 = I4 * 2
	addpd	r3, r2			;; I4 = I2 + I4 (final I2)

	subpd	r4, r5			;; I1 = I1 - I3 (final I3)
	multwo	r5			;; I3 = I3 * 2
	addpd	r5, r4			;; I3 = I1 + I3 (final I1)

	ELSE

	xstore	dst1, r8

	xcopy	r8, r2
	subpd	r2, r4			;; I1 = I1 - I2 (new I2)
	addpd	r4, r8			;; I2 = I1 + I2 (new I1)

	xcopy	r8, r6
	subpd	r6, r5			;; I3 = I3 - I4 (new R4)
	addpd	r5, r8			;; I4 = I3 + I4 (new I3)

	xcopy	r8, r1
	subpd	r1, r6			;; R2 = R2 - R4 (final R4)
	addpd	r6, r8			;; R4 = R2 + R4 (final R2)

	xcopy	r8, r2
	subpd	r2, r3			;; I2 = I2 - I4 (final I4)
	addpd	r3, r8			;; I4 = I2 + I4 (final I2)

	xcopy	r8, r4
	subpd	r4, r5			;; I1 = I1 - I3 (final I3)
	addpd	r5, r8			;; I3 = I1 + I3 (final I1)

	ENDIF
	ENDM
ENDIF


;; 32-bit AMD optimized versions of the above macros

IF (@INSTR(,%xarch,<K8>) NE 0)

;; K8 Cheat sheet for scheduling dependency chains
;;	      12345678901234567890123456789012345678901234567890123456789012345678901234567890
;;A2	      MMMMAAAA    MMMM
;;B2	        MMMMAAAA    MMMM
;;A4              MMMMAAAA
;;A3                MMMMAAAA
;;B3		      MMMMAAAA
;;B4		        MMMMAAAA

;;i4(r4-r3)                   AAAAMMMM
;;r3(r4+r3)                   MMMMAAAAMMMM
;;r4(i3-i4)                     AAAAMMMM
;;i3(i3+i4)                     MMMMAAAAMMMM
;;i2(i1-i2)                           AAAA
;;r2(r1-r2)                             AAAA
;;r1(r1+r2)                               AAAA
;;i1(i1+i2)                                 AAAA

;;I4(I2-I4)	                              AAAA
;;I2(I2+I4)			              MMMM    AAAA
;;R4(R2-R4)	                                AAAA			
;;R2(R2+R4)			                MMMM    AAAA
;;R3(R1-R3)	                                  AAAA
;;R1(R1+R3)	                                  MMMM    AAAA
;;I3(I1-I3)	                                    AAAA				
;;I1(I1+I3)                                         MMMM    AAAA

r4_x4c_2sc_djbunfft_mem MACRO mem1,mem2,mem3,mem4,mem5,mem6,mem7,mem8,dst3,screg1,screg2,pre1,pre2
	xload	xmm3, [screg2+16]	;; cosine/sine			;; K8
	xload	xmm6, mem3		;; R2
	mulpd	xmm6, xmm3		;; A2 = R2 * cosine/sine	;; 1-4
	xload	xmm0, mem4		;; I2
	mulpd	xmm3, xmm0		;; B2 = I2 * cosine/sine	;; 3-6

	xload	xmm5, [screg1+16]	;; cosine/sine
	xload	xmm2, mem7		;; R4
	mulpd	xmm2, xmm5		;; A4 = R4 * cosine/sine	;; 5-8
	addpd	xmm6, xmm0		;; A2 = A2 + I2			;; 5-8

	xload	xmm7, mem5		;; R3
	mulpd	xmm7, xmm5		;; A3 = R3 * cosine/sine	;; 7-10
	subpd	xmm3, mem3		;; B2 = B2 - R2			;; 7-10

	xload	xmm1, mem6		;; I3
	mulpd	xmm5, xmm1		;; B3 = I3 * cosine/sine	;; 9-12
	xload	xmm4, mem8		;; I4
	subpd	xmm2, xmm4		;; A4 = A4 - I4			;; 9-12

	mulpd	xmm4, [screg1+16]	;; B4 = I4 * cosine/sine	;; 11-14
	addpd	xmm7, xmm1		;; A3 = A3 + I3			;; 11-14

	mulpd	xmm6, [screg2]		;; A2 = A2 * sine (new R2)	;; 13-16
	subpd	xmm5, mem5		;; B3 = B3 - R3			;; 13-16

	mulpd	xmm3, [screg2]		;; B2 = B2 * sine (new I2)	;; 15-18
	addpd	xmm4, mem7		;; B4 = B4 + R4			;; 15-18

	xprefetchw [pre1]

	subpd	xmm2, xmm7		;; R4 = R4 - R3 (new I4)	;; 17-20
	multwo	xmm7			;; R3 = R3 * 2			;; 17-20

	subpd	xmm5, xmm4		;; I3 = I3 - I4 (new R4)	;; 19-22
	multwo	xmm4			;; I4 = I4 * 2			;; 19-22

	addpd	xmm7, xmm2		;; R3 = R4 + R3 (new R3)	;; 21-24
	xload	xmm0, [screg1]		;; Sine
	mulpd	xmm2, xmm0		;; new I4 = new I4 * sine	;; 21-24

	addpd	xmm4, xmm5		;; I4 = I3 + I4 (new I3)	;; 23-26
	mulpd	xmm5, xmm0		;; new R4 = new R4 * sine	;; 23-26

	xload	xmm1, mem2		;; I1
	subpd	xmm1, xmm3		;; I1 = I1 - I2 (new I2)	;; 25-28
	mulpd	xmm7, xmm0		;; new R3 = new R3 * sine	;; 25-28

	mulpd	xmm4, xmm0		;; new I3 = new I3 * sine	;; 27-30
	xload	xmm0, mem1		;; R1
	subpd	xmm0, xmm6		;; R1 = R1 - R2 (new R2)	;; 27-30

	addpd	xmm6, mem1		;; R2 = R1 + R2 (new R1)	;; 29-32

	addpd	xmm3, mem2		;; I2 = I1 + I2 (new I1)	;; 31-34

	xprefetchw [pre1][pre2]

	subpd	xmm1, xmm2		;; I2 = I2 - I4 (final I4)	;; 33-36
	multwo	xmm2			;; I4 = I4 * 2			;; 33-36

	subpd	xmm0, xmm5		;; R2 = R2 - R4 (final R4)	;; 35-38
	multwo	xmm5			;; R4 = R4 * 2			;; 35-38

	subpd	xmm6, xmm7		;; R1 = R1 - R3 (final R3)	;; 37-40
	multwo	xmm7			;; R3 = R3 * 2			;; 37-40

	subpd	xmm3, xmm4		;; I1 = I1 - I3 (final I3)	;; 39-42
	multwo	xmm4			;; I3 = I3 * 2			;; 39-42

	addpd	xmm2, xmm1		;; I4 = I2 + I4 (final I2)	;; 41-44
	IFNB <dst3>
	xstore	dst3, xmm6						;; 41
	ENDIF

	addpd	xmm5, xmm0		;; R4 = R2 + R4 (final R2)	;; 43-46
	addpd	xmm7, xmm6		;; R3 = R1 + R3 (final R1)	;; 45-48
	addpd	xmm4, xmm3		;; I3 = I1 + I3 (final I1)	;; 47-50

	ENDM

;; K8 Cheat sheet for scheduling dependency chains
;;	      12345678901234567890123456789012345678901234567890123456789012345678901234567890
;;B2	      MMMMAAAA    MMMM
;;A2	        MMMMAAAA    MMMM
;;B4		  MMMMAAAA
;;A4                MMMMAAAA
;;A3                  MMMMAAAA
;;B3		        MMMMAAAA

;;i4(r4-r3)                   AAAAMMMM
;;r3(r4+r3)                   MMMMAAAAMMMM
;;r4(i3-i4)                     AAAAMMMM
;;i3(i3+i4)                     MMMMAAAAMMMM
;;i2(i1-i2)                           AAAA
;;r2(r1-r2)                             AAAA
;;r1(r1+r2)                               AAAA
;;i1(i1+i2)                                 AAAA

;;I4(I2-I4)	                              AAAA
;;I2(I2+I4)			              MMMM    AAAA
;;R4(R2-R4)	                                AAAA			
;;R2(R2+R4)			                MMMM    AAAA
;;R3(R1-R3)	                                  AAAA
;;R1(R1+R3)	                                  MMMM    AAAA
;;I3(I1-I3)	                                    AAAA				
;;I1(I1+I3)                                         MMMM    AAAA

r4_x4c_2sc_djbunfft_partial_mem MACRO r1,r2,r3,r4,r5,r6,r7,r8,mem1,mem2,mem5,mem6,dst1,screg1,screg2,pre1,pre2
	xload	r1, [screg2+16]		;; cosine/sine
	mulpd	r1, r4			;; B2 = I2 * cosine/sine	;; 1-4

	subpd	r1, r3			;; B2 = B2 - R2			;; 5-8

	mulpd	r3, [screg2+16]		;; A2 = R2 * cosine/sine	;; 3-6

	xload	r5, [screg1+16]		;; cosine/sine
	mulpd	r5, r8			;; B4 = I4 * cosine/sine	;; 5-8

	addpd	r3, r4			;; A2 = A2 + I2			;; 7-10

	addpd	r5, r7			;; B4 = B4 + R4			;; 9-12

	xload	r4, [screg1+16]		;; cosine/sine
	mulpd	r7, r4			;; A4 = R4 * cosine/sine	;; 7-10

	xload	r2, mem5		;; R3
	mulpd	r2, r4			;; A3 = R3 * cosine/sine	;; 9-12

	subpd	r7, r8			;; A4 = A4 - I4			;; 11-14
	mulpd	r4, mem6		;; B3 = I3 * cosine/sine	;; 11-14

	addpd	r2, mem6		;; A3 = A3 + I3			;; 13-16
	mulpd	r1, [screg2]		;; B2 = B2 * sine (new I2)	;; 13-16

	subpd	r4, mem5		;; B3 = B3 - R3			;; 15-18
	mulpd	r3, [screg2]		;; A2 = A2 * sine (new R2)	;; 15-18

	xprefetchw [pre1]

	subpd	r7, r2			;; R4 = R4 - R3 (new I4)	;; 17-20
	multwo	r2			;; R3 = R3 * 2			;; 17-20

	subpd	r4, r5			;; I3 = I3 - I4 (new R4)	;; 19-22
	multwo	r5			;; I4 = I4 * 2			;; 19-22

	addpd	r2, r7			;; R3 = R4 + R3 (new R3)	;; 21-24
	xload	r6, [screg1]		;; Sine
	mulpd	r7, r6			;; new I4 = new I4 * sine	;; 21-24

	addpd	r5, r4			;; I4 = I3 + I4 (new I3)	;; 23-26
	mulpd	r4, r6			;; new R4 = new R4 * sine	;; 23-26

	xload	r8, mem2		;; I1
	subpd	r8, r1			;; I1 = I1 - I2 (new I2)	;; 25-28
	mulpd	r2, r6			;; new R3 = new R3 * sine	;; 25-28

	mulpd	r5, r6			;; new I3 = new I3 * sine	;; 27-30
	xload	r6, mem1		;; R1
	subpd	r6, r3			;; R1 = R1 - R2 (new R2)	;; 27-30

	addpd	r3, mem1		;; R2 = R1 + R2 (new R1)	;; 29-32

	xprefetchw [pre1][pre2]

	addpd	r1, mem2		;; I2 = I1 + I2 (new I1)	;; 31-34

	subpd	r8, r7			;; I2 = I2 - I4 (final I4)	;; 33-36
	multwo	r7			;; I4 = I4 * 2			;; 33-36

	subpd	r6, r4			;; R2 = R2 - R4 (final R4)	;; 35-38
	multwo	r4			;; R4 = R4 * 2			;; 35-38

	subpd	r3, r2			;; R1 = R1 - R3 (final R3)	;; 37-40
	multwo	r2			;; R3 = R3 * 2			;; 37-40

	subpd	r1, r5			;; I1 = I1 - I3 (final I3)	;; 39-42
	multwo	r5			;; I3 = I3 * 2			;; 39-42

	addpd	r7, r8			;; I4 = I2 + I4 (final I2)	;; 41-44
	addpd	r4, r6			;; R4 = R2 + R4 (final R2)	;; 43-46
	addpd	r2, r3			;; R3 = R1 + R3 (final R1)	;; 45-48
	addpd	r5, r1			;; I3 = I1 + I3 (final I1)	;; 47-50

	IFNB <dst1>
	xstore	dst1, r2
	ENDIF

	ENDM

ENDIF

;; 64-bit Intel and K10 implementations of the above - use the extra XMM registers
;; For some reason this implementation is worse on a Pentium 4.

IF (@INSTR(,%xarch,<CORE>) NE 0) OR (@INSTR(,%xarch,<BLEND>) NE 0) OR (@INSTR(,%xarch,<K10>) NE 0)
IFDEF X86_64

; Theoretical best case is 44 clocks on a Core 2.  Now at 51.4 clocks.
r4_x4cl_four_complex_djbunfft MACRO srcreg,srcinc,d1,d2,screg,scoff
	xload	xmm0, [srcreg+d2+d1+16]	;; R4
	xload	xmm1, [screg+scoff+16] ;; cosine/sine
	xcopy	xmm2, xmm0		;; Copy R4				; 1-3
	mulpd	xmm0, xmm1		;; A4 = R4 * cosine/sine		; 1-5

	xload	xmm5, [srcreg+d1+48]	;; I2
	xload	xmm6, [screg+scoff+32+16] ;; cosine/sine
	xcopy	xmm7, xmm5		;; Copy I2				; 2-4
	mulpd	xmm5, xmm6		;; B2 = I2 * cosine/sine		; 2-6

	xload	xmm3, [srcreg+d2+16]	;; R3
	xcopy	xmm4, xmm3		;; Copy R3				; 3-5
	mulpd	xmm3, xmm1		;; A3 = R3 * cosine/sine		; 3-7

	xload	xmm8, [srcreg+d1+16]	;; R2
	xcopy	xmm9, xmm8		;; Copy R2				; 4-6
	mulpd	xmm8, xmm6		;; A2 = R2 * cosine/sine		; 4-8	avail 6,10+

	xload	xmm6, [srcreg+d2+48]	;; I3
	xcopy	xmm10, xmm6		;; Copy I3				; 5-7	avail 11+
	mulpd	xmm6, xmm1		;; B3 = I3 * cosine/sine		; 5-9

	xload	xmm11, [srcreg+d2+d1+48];; I4
	subpd	xmm0, xmm11		;; A4 = A4 - I4 (new R4 / sine)		; 6-8
	mulpd	xmm11, xmm1		;; B4 = I4 * cosine/sine		; 6-10	avail 1,12+

	subpd	xmm5, xmm9		;; B2 = B2 - R2				; 7-9	avail 1,9,12+
	xload	xmm1, [screg+scoff+32]	;; sine					; 7 

	addpd	xmm3, xmm10		;; A3 = A3 + I3 (new R3 / sine)		; 8-10	avail 9,10,12+

	addpd	xmm8, xmm7		;; A2 = A2 + I2				; 9-11	avail 9,10,7,12+
	xload	xmm7, [srcreg+48]	;; I1					; 9

	subpd	xmm6, xmm4		;; B3 = B3 - R3 (new I3 / sine)		; 10-12	avail 9,10,4,12+
	mulpd	xmm5, xmm1		;; B2 = B2 * sine (new I2)		; 10-14
	xcopy	xmm4, xmm0		;; Copy R4 / sine			; 10-12	avail 9,10,12+

	addpd	xmm11, xmm2		;; B4 = B4 + R4 (new I4 / sine)		; 11-13	avail 9,10,2,12+

	subpd	xmm0, xmm3		;; R4 = R4 - R3 (newer I4 / sine)	; 12-14
	mulpd	xmm8, xmm1		;; A2 = A2 * sine (new R2)		; 12-16	avail 9,10,2,1,12+
	xload	xmm1, [screg+scoff]	;; sine					; 12	avail 9,10,2,12+

	addpd	xmm3, xmm4		;; R3 = R4 + R3 (newer R3 / sine)	; 13-15	avail 9,10,2,4,12+
	xcopy	xmm4, xmm6		;; Copy I3 / sine			; 13-15
	xload	xmm2, [srcreg+16]	;; R1					; 13	avail 9,10,12+

	xprefetchw [srcreg+srcinc+d2]

	subpd	xmm6, xmm11		;; I3 = I3 - I4 (newer R4 / sine)	; 14-16
	xcopy	xmm10, xmm7		;; Copy I1				; 14-16 avail 9,12+
	xload	xmm9, [srcreg+d2+d1]	;;#2 R4					; 14	avail 12+

	subpd	xmm7, xmm5		;; I1 = I1 - I2 (newer I2)		; 15-17
	mulpd	xmm0, xmm1		;; newer I4 * sine			; 15-19
	xload	xmm12, [screg+16]	;;#2 cosine/sine			; 15	avail 13+

	addpd	xmm11, xmm4		;; I4 = I3 + I4 (newer I3 / sine)	; 16-18 avail 4,13+
	mulpd	xmm3, xmm1		;; newer R3 * sine			; 16-20
	xcopy	xmm4, xmm2		;; Copy R1				; 16-18	avail 13+

	addpd	xmm5, xmm10		;; I2 = I1 + I2 (newer I1)		; 17-19	avail 10,13+
	mulpd	xmm6, xmm1		;; newer R4 * sine			; 17-21
	xload	xmm13, [srcreg+d1+32]	;;#2 I2					; 17	avail 10,14+

	addpd	xmm2, xmm8		;; R2 = R1 + R2 (newer R1)		; 18-20
	xload	xmm14, [screg+32+16]	;;#2 cosine/sine			; 18	avail 10,15

	subpd	xmm4, xmm8		;; R1 = R1 - R2 (newer R2)		; 19-21	avail 10,8,15
	mulpd	xmm11, xmm1		;; newer I3 * sine			; 19-23	avail 10,8,1,15
	xcopy	xmm1, xmm7		;; Copy I2				; 19-21	avail 10,8,15

	xprefetchw [srcreg+srcinc+d2+d1]

	subpd	xmm7, xmm0		;; I2 = I2 - I4 (final I4)		; 20-22	avail 10,8,15 storable 7
	xcopy	xmm15, xmm9		;;#2 Copy R4				; 20-22	avail 10,8 storable 7

	xcopy	xmm8, xmm2		;; Copy R1				; 21-23	avail 10 storable 7
	subpd	xmm2, xmm3		;; R1 = R1 - R3 (final R3)		; 21-23	avail 10 storable 7,2

	addpd	xmm0, xmm1		;; I4 = I2 + I4 (final I2)		; 22-24	avail 10,1 storable 7,2,0
	xcopy	xmm1, xmm4		;; Copy R2				; 22-24	avail 10 storable 7,2,0

	subpd	xmm4, xmm6		;; R2 = R2 - R4 (final R4)		; 23-25	avail 10 storable 7,2,0,4
	mulpd	xmm9, xmm12		;;#2 A4 = R4 * cosine/sine		; 23-27
	xcopy	xmm10, xmm13		;;#2 Copy I2				; 23-25	avail none storable 7,2,0,4
	xstore	[srcreg+d2+d1+48], xmm7	;; Save I4				; 23	avail 7 storable 2,0,4

	addpd	xmm3, xmm8		;; R3 = R1 + R3 (final R1)		; 24-26	avail 7,8 storable 2,0,4,3
	mulpd	xmm13, xmm14		;;#2 B2 = I2 * cosine/sine		; 24-28
	xcopy	xmm7, xmm5		;; Copy I1				; 24-26	avail 8 storable 2,0,4,3
	xstore	[srcreg+d1+16], xmm2	;; Save R3				; 24	avail 8,2 storable 0,4,3

	addpd	xmm6, xmm1		;; R4 = R2 + R4 (final R2)		; 25-27	avail 8,2,1 storable 0,4,3,6
	xload	xmm2, [srcreg+d2]	;;#2 R3					; 25	avail 8,1 storable 0,4,3,6
	xcopy	xmm8, xmm2		;;#2 Copy R3				; 25-27	avail 1 storable 0,4,3,6
	mulpd	xmm2, xmm12		;;#2 A3 = R3 * cosine/sine		; 25-29

	subpd	xmm5, xmm11		;; I1 = I1 - I3 (final I3)		; 26-28	avail 1 storable 0,4,3,6,5
	xload	xmm1, [srcreg+d1]	;;#2 R2					; 26	avail none storable 0,4,3,6,5
	xstore	[srcreg+d2+d1+16], xmm4	;; Save R4				; 26	avail 4 storable 0,3,6,5
	xcopy	xmm4, xmm1		;;#2 Copy R2				; 26-28	avail none storable 0,3,6,5
	mulpd	xmm1, xmm14		;;#2 A2 = R2 * cosine/sine		; 26-30	avail 14 storable 0,3,6,5

	addpd	xmm11, xmm7		;; I3 = I1 + I3 (final I1)		; 27-29	avail 14,7 storable 0,3,6,5,11
	xload	xmm14, [srcreg+d2+32]	;;#2 I3					; 27	avail 7 storable 0,3,6,5,11
	xcopy	xmm7, xmm14		;;#2 Copy I3				; 27-29	avail none storable 0,3,6,5,11
	mulpd	xmm14, xmm12		;;#2 B3 = I3 * cosine/sine		; 27-31
	xstore	[srcreg+d1], xmm3	;; Save R1				; 27	avail 3 storable 0,6,5,11

	xload	xmm3, [srcreg+d2+d1+32] ;;#2 I4					; 28	avail none storable 0,6,5,11
	subpd	xmm9, xmm3		;;#2 A4 = A4 - I4 (new R4 / sine)	; 28-30
	mulpd	xmm3, xmm12		;;#2 B4 = I4 * cosine/sine		; 28-32	avail 12 storable 0,6,5,11
	xstore	[srcreg+d2+d1], xmm6	;; Save R2				; 28	avail 12,6 storable 0,5,11

	subpd	xmm13, xmm4		;;#2 B2 = B2 - R2			; 29-31	avail 12,6,4 storable 0,5,11
	xload	xmm6, [screg+32]	;;#2 sine				; 29	avail 12,4 storable 0,5,11
	xstore	[srcreg+d1+48], xmm5	;; Save I3				; 29	avail 12,4,5 storable 0,11

	addpd	xmm2, xmm7		;;#2 A3 = A3 + I3 (new R3 / sine)	; 30-32	avail 12,4,5,7 storable 0,11
	xload	xmm5, [srcreg+32]	;;#2 I1					; 30	avail 12,4,7 storable 0,11
	xstore	[srcreg+d1+32], xmm11	;; Save I1				; 30	avail 12,4,7,11 storable 0

	addpd	xmm1, xmm10		;;#2 A2 = A2 + I2			; 31-33	avail 12,4,7,11,10 storable 0
	xstore	[srcreg+d2+d1+32], xmm0	;; Save I2				; 25	avail 12,4,7,11,10,0

	subpd	xmm14, xmm8		;;#2 B3 = B3 - R3 (new I3 / sine)	; 32-34	avail 12,4,7,11,10,0,8
	mulpd	xmm13, xmm6		;;#2 B2 = B2 * sine (new I2)		; 32-36
	xcopy	xmm4, xmm9		;;#2 Copy R4 / sine			; 32-34	avail 12,7,11,10,0,8

	addpd	xmm3, xmm15		;;#2 B4 = B4 + R4 (new I4 / sine)	; 33-35	avail 12,7,11,10,0,8,15

	subpd	xmm9, xmm2		;;#2 R4 = R4 - R3 (newer I4 / sine)	; 34-36
	mulpd	xmm1, xmm6		;;#2 A2 = A2 * sine (new R2)		; 34-38	avail 12,7,11,10,0,8,15,6
	xload	xmm6, [screg]		;;#2 sine				; 34	avail 12,7,11,10,0,8,15

	addpd	xmm2, xmm4		;;#2 R3 = R4 + R3 (newer R3 / sine)	; 35-37	avail 12,7,11,10,0,8,15,4
	xcopy	xmm4, xmm14		;;#2 Copy I3 / sine			; 35-37	avail 12,7,11,10,0,8,15
	xload	xmm7, [srcreg]		;;#2 R1					; 35	avail 12,11,10,0,8,15

	xprefetchw [srcreg+srcinc]

	subpd	xmm14, xmm3		;;#2 I3 = I3 - I4 (newer R4 / sine)	; 36-38
	xcopy	xmm0, xmm5		;;#2 Copy I1				; 36-38	avail 12,11,10,8,15

	subpd	xmm5, xmm13		;;#2 I1 = I1 - I2 (newer I2)		; 37-39
	mulpd	xmm9, xmm6		;;#2 newer I4 * sine			; 37-41

	addpd	xmm3, xmm4		;;#2 I4 = I3 + I4 (newer I3 / sine)	; 38-40	avail 12,11,10,8,15,0
	mulpd	xmm2, xmm6		;;#2 newer R3 * sine			; 38-42
	xcopy	xmm4, xmm7		;;#2 Copy R1				; 38-40	avail 12,11,10,8,15

	addpd	xmm13, xmm0		;;#2 I2 = I1 + I2 (newer I1)		; 39-41	avail 12,11,10,8,15,0
	mulpd	xmm14, xmm6		;;#2 newer R4 * sine			; 39-43

	addpd	xmm7, xmm1		;;#2 R2 = R1 + R2 (newer R1)		; 40-42
	xcopy	xmm0, xmm5		;;#2 Copy I2				; 40-42	avail 12,11,10,8,15

	subpd	xmm4, xmm1		;;#2 R1 = R1 - R2 (newer R2)		; 41-43	avail 12,11,10,8,15,1
	mulpd	xmm3, xmm6		;;#2 newer I3 * sine			; 41-45	avail 12,11,10,8,15,1,6

	xprefetchw [srcreg+srcinc+d1]

	subpd	xmm5, xmm9		;;#2 I2 = I2 - I4 (final I4)		; 42-44	avail 12,11,10,8,15,6 storable 5
	xcopy	xmm1, xmm13		;;#2 Copy I1				; 42-44	avail 12,11,10,8,15,6 storable 5

	xcopy	xmm6, xmm7		;;#2 Copy R1				; 43-45	avail 12,11,10,8,15 storable 5
	subpd	xmm7, xmm2		;;#2 R1 = R1 - R3 (final R3)		; 43-45	avail 12,11,10,8,15 storable 5,7

	addpd	xmm9, xmm0		;;#2 I4 = I2 + I4 (final I2)		; 44-46	avail 12,11,10,8,15,0 storable 5,7,9
	xcopy	xmm0, xmm4		;;#2 Copy R2				; 44-46	avail 12,11,10,8,15 storable 5,7,9

	subpd	xmm4, xmm14		;;#2 R2 = R2 - R4 (final R4)		; 45-47	avail 12,11,10,8,15 storable 5,7,9,4
	xstore	[srcreg+d2+48], xmm5	;;#2 Save I4				; 45	avail 12,11,10,8,15,5 storable 7,9,4

	addpd	xmm2, xmm6		;;#2 R3 = R1 + R3 (final R1)		; 46-48	avail 12,11,10,8,15,5,6 storable 7,9,4,2
	xstore	[srcreg+16], xmm7	;;#2 Save R3				; 46	avail 12,11,10,8,15,5,6,7 storable 9,4,2

	addpd	xmm14, xmm0		;;#2 R4 = R2 + R4 (final R2)		; 47-49	avail 12,11,10,8,15,5,6,7,0 storable 9,4,2,14
	xstore	[srcreg+d2+32], xmm9	;;#2 Save I2				; 47	avail 12,11,10,8,15,5,6,7,0,9 storable 4,2,14

	subpd	xmm13, xmm3		;;#2 I1 = I1 - I3 (final I3)		; 48-50	avail 12,11,10,8,15,5,6,7,0,9 storable 4,2,14,13
	xstore	[srcreg+d2+16], xmm4	;;#2 Save R4				; 48	avail 12,11,10,8,15,5,6,7,0,9,4 storable 2,14,13

	addpd	xmm3, xmm1		;;#2 I3 = I1 + I3 (final I1)		; 49-51	avail 12,11,10,8,15,5,6,7,0,9,4,1 storable 2,14,13,3
	xstore	[srcreg], xmm2		;;#2 Save R1				; 49	avail 12,11,10,8,15,5,6,7,0,9,4,1,2 storable 14,13,3

	xstore	[srcreg+d2], xmm14	;;#2 Save R2				; 50
	xstore	[srcreg+48], xmm13	;;#2 Save I3				; 51
	xstore	[srcreg+32], xmm3	;;#2 Save I1				; 52

	bump	srcreg, srcinc
	ENDM
ENDIF
ENDIF

;; 64-bit AMD K8 optimized versions of the above macros.  The same as the 32-bit
;; version with some constants preloaded.  We could probably do better.

IF (@INSTR(,%xarch,<K8>) NE 0)
IFDEF X86_64

r4_x4c_2sc_djbunfft_mem_preload MACRO
	xload	xmm15, XMM_TWO
	ENDM

r4_x4c_2sc_djbunfft_mem MACRO mem1,mem2,mem3,mem4,mem5,mem6,mem7,mem8,dst3,screg1,screg2,pre1,pre2
	xload	xmm3, [screg2+16]	;; cosine/sine			;; K8
	xload	xmm6, mem3		;; R2
	mulpd	xmm6, xmm3		;; A2 = R2 * cosine/sine	;; 1-4
	xload	xmm0, mem4		;; I2
	mulpd	xmm3, xmm0		;; B2 = I2 * cosine/sine	;; 3-6

	xload	xmm5, [screg1+16]	;; cosine/sine
	xload	xmm2, mem7		;; R4
	mulpd	xmm2, xmm5		;; A4 = R4 * cosine/sine	;; 5-8
	addpd	xmm6, xmm0		;; A2 = A2 + I2			;; 5-8

	xload	xmm7, mem5		;; R3
	mulpd	xmm7, xmm5		;; A3 = R3 * cosine/sine	;; 7-10
	subpd	xmm3, mem3		;; B2 = B2 - R2			;; 7-10

	xload	xmm1, mem6		;; I3
	mulpd	xmm5, xmm1		;; B3 = I3 * cosine/sine	;; 9-12
	xload	xmm4, mem8		;; I4
	subpd	xmm2, xmm4		;; A4 = A4 - I4			;; 9-12

	mulpd	xmm4, [screg1+16]	;; B4 = I4 * cosine/sine	;; 11-14
	addpd	xmm7, xmm1		;; A3 = A3 + I3			;; 11-14

	mulpd	xmm6, [screg2]		;; A2 = A2 * sine (new R2)	;; 13-16
	subpd	xmm5, mem5		;; B3 = B3 - R3			;; 13-16

	mulpd	xmm3, [screg2]		;; B2 = B2 * sine (new I2)	;; 15-18
	addpd	xmm4, mem7		;; B4 = B4 + R4			;; 15-18

	xprefetchw [pre1]

	subpd	xmm2, xmm7		;; R4 = R4 - R3 (new I4)	;; 17-20
	mulpd	xmm7, xmm15		;; R3 = R3 * 2			;; 17-20

	subpd	xmm5, xmm4		;; I3 = I3 - I4 (new R4)	;; 19-22
	mulpd	xmm4, xmm15		;; I4 = I4 * 2			;; 19-22

	addpd	xmm7, xmm2		;; R3 = R4 + R3 (new R3)	;; 21-24
	xload	xmm0, [screg1]		;; Sine
	mulpd	xmm2, xmm0		;; new I4 = new I4 * sine	;; 21-24

	addpd	xmm4, xmm5		;; I4 = I3 + I4 (new I3)	;; 23-26
	mulpd	xmm5, xmm0		;; new R4 = new R4 * sine	;; 23-26

	xload	xmm1, mem2		;; I1
	subpd	xmm1, xmm3		;; I1 = I1 - I2 (new I2)	;; 25-28
	mulpd	xmm7, xmm0		;; new R3 = new R3 * sine	;; 25-28

	mulpd	xmm4, xmm0		;; new I3 = new I3 * sine	;; 27-30
	xload	xmm0, mem1		;; R1
	subpd	xmm0, xmm6		;; R1 = R1 - R2 (new R2)	;; 27-30

	addpd	xmm6, mem1		;; R2 = R1 + R2 (new R1)	;; 29-32

	addpd	xmm3, mem2		;; I2 = I1 + I2 (new I1)	;; 31-34

	xprefetchw [pre1][pre2]

	subpd	xmm1, xmm2		;; I2 = I2 - I4 (final I4)	;; 33-36
	mulpd	xmm2, xmm15		;; I4 = I4 * 2			;; 33-36

	subpd	xmm0, xmm5		;; R2 = R2 - R4 (final R4)	;; 35-38
	mulpd	xmm5, xmm15		;; R4 = R4 * 2			;; 35-38

	subpd	xmm6, xmm7		;; R1 = R1 - R3 (final R3)	;; 37-40
	mulpd	xmm7, xmm15		;; R3 = R3 * 2			;; 37-40

	subpd	xmm3, xmm4		;; I1 = I1 - I3 (final I3)	;; 39-42
	mulpd	xmm4, xmm15		;; I3 = I3 * 2			;; 39-42

	addpd	xmm2, xmm1		;; I4 = I2 + I4 (final I2)	;; 41-44
	IFNB <dst3>
	xstore	dst3, xmm6						;; 41
	ENDIF

	addpd	xmm5, xmm0		;; R4 = R2 + R4 (final R2)	;; 43-46
	addpd	xmm7, xmm6		;; R3 = R1 + R3 (final R1)	;; 45-48
	addpd	xmm4, xmm3		;; I3 = I1 + I3 (final I1)	;; 47-50

	ENDM

r4_x4c_2sc_djbunfft_partial_mem_preload MACRO
	xload	xmm15, XMM_TWO
	ENDM

r4_x4c_2sc_djbunfft_partial_mem MACRO r1,r2,r3,r4,r5,r6,r7,r8,mem1,mem2,mem5,mem6,dst1,screg1,screg2,pre1,pre2
	xload	r1, [screg2+16]		;; cosine/sine
	mulpd	r1, r4			;; B2 = I2 * cosine/sine	;; 1-4

	subpd	r1, r3			;; B2 = B2 - R2			;; 5-8

	mulpd	r3, [screg2+16]		;; A2 = R2 * cosine/sine	;; 3-6

	xload	r5, [screg1+16]		;; cosine/sine
	mulpd	r5, r8			;; B4 = I4 * cosine/sine	;; 5-8

	addpd	r3, r4			;; A2 = A2 + I2			;; 7-10

	addpd	r5, r7			;; B4 = B4 + R4			;; 9-12

	xload	r4, [screg1+16]		;; cosine/sine
	mulpd	r7, r4			;; A4 = R4 * cosine/sine	;; 7-10

	xload	r2, mem5		;; R3
	mulpd	r2, r4			;; A3 = R3 * cosine/sine	;; 9-12

	subpd	r7, r8			;; A4 = A4 - I4			;; 11-14
	mulpd	r4, mem6		;; B3 = I3 * cosine/sine	;; 11-14

	addpd	r2, mem6		;; A3 = A3 + I3			;; 13-16
	mulpd	r1, [screg2]		;; B2 = B2 * sine (new I2)	;; 13-16

	subpd	r4, mem5		;; B3 = B3 - R3			;; 15-18
	mulpd	r3, [screg2]		;; A2 = A2 * sine (new R2)	;; 15-18

	xprefetchw [pre1]

	subpd	r7, r2			;; R4 = R4 - R3 (new I4)	;; 17-20
	mulpd	r2, xmm15		;; R3 = R3 * 2			;; 17-20

	subpd	r4, r5			;; I3 = I3 - I4 (new R4)	;; 19-22
	mulpd	r5, xmm15		;; I4 = I4 * 2			;; 19-22

	addpd	r2, r7			;; R3 = R4 + R3 (new R3)	;; 21-24
	xload	r6, [screg1]		;; Sine
	mulpd	r7, r6			;; new I4 = new I4 * sine	;; 21-24

	addpd	r5, r4			;; I4 = I3 + I4 (new I3)	;; 23-26
	mulpd	r4, r6			;; new R4 = new R4 * sine	;; 23-26

	xload	r8, mem2		;; I1
	subpd	r8, r1			;; I1 = I1 - I2 (new I2)	;; 25-28
	mulpd	r2, r6			;; new R3 = new R3 * sine	;; 25-28

	mulpd	r5, r6			;; new I3 = new I3 * sine	;; 27-30
	xload	r6, mem1		;; R1
	subpd	r6, r3			;; R1 = R1 - R2 (new R2)	;; 27-30

	addpd	r3, mem1		;; R2 = R1 + R2 (new R1)	;; 29-32

	xprefetchw [pre1][pre2]

	addpd	r1, mem2		;; I2 = I1 + I2 (new I1)	;; 31-34

	subpd	r8, r7			;; I2 = I2 - I4 (final I4)	;; 33-36
	mulpd	r7, xmm15		;; I4 = I4 * 2			;; 33-36

	subpd	r6, r4			;; R2 = R2 - R4 (final R4)	;; 35-38
	mulpd	r4, xmm15		;; R4 = R4 * 2			;; 35-38

	subpd	r3, r2			;; R1 = R1 - R3 (final R3)	;; 37-40
	mulpd	r2, xmm15		;; R3 = R3 * 2			;; 37-40

	subpd	r1, r5			;; I1 = I1 - I3 (final I3)	;; 39-42
	mulpd	r5, xmm15		;; I3 = I3 * 2			;; 39-42

	addpd	r7, r8			;; I4 = I2 + I4 (final I2)	;; 41-44
	addpd	r4, r6			;; R4 = R2 + R4 (final R2)	;; 43-46
	addpd	r2, r3			;; R3 = R1 + R3 (final R1)	;; 45-48
	addpd	r5, r1			;; I3 = I1 + I3 (final I1)	;; 47-50

	IFNB <dst1>
	xstore	dst1, r2
	ENDIF

	ENDM

ENDIF
ENDIF

;;
;; ************************************* four-complex-first-fft variants ******************************************
;;

;; This code applies the roots-of-minus-1 premultipliers in a negacyclic FFT.
;; It also applies the three sin/cos multipliers after a radix-4 butterfly.  We save memory
;; by splitting the roots-of-minus-1 premultipliers such that every macro uses the 
;; same premultiplier data and we have 4 sin/cos postmultipliers.  Every sin/cos postmultiplier,
;; a very big table anyway, is multiplied by the other part of the split roots-of-minus-1.
;; The common premuliplier data is 1, .924+.383i, SQRTHALF+SQRTHALFi, .383+.924i.
;; This scheme is used by our simple radix-4 DJB "r4" FFTs.

;r4_x2cl_four_complex_first_fft4 MACRO srcreg,srcinc,d1
;	r4_x2cl_four_complex_first_fft4_cmn srcreg,rbx,srcinc,d1,rdi
;	ENDM
r4_x2cl_four_complex_first_fft4_scratch MACRO srcreg,srcinc,d1,screg
	r4_x2cl_four_complex_first_fft4_cmn srcreg,0,srcinc,d1,screg
	ENDM
r4_x2cl_four_complex_first_fft4_cmn MACRO srcreg,off,srcinc,d1,screg
	xload	xmm7, XMM_P924
	xload	xmm1, XMM_P383

	xload	xmm4, [srcreg+off+d1]		;; R2
	mulpd	xmm4, xmm7			;; A2 = R2 * .924
	xload	xmm0, [srcreg+off+d1+32]	;; R6 (I2)
	mulpd	xmm0, xmm1			;; C2 = I2 * .383
	subpd	xmm4, xmm0			;; A2 = A2 - C2 (new R2)

	xload	xmm6, [srcreg+off+d1]		;; R2
	mulpd	xmm6, xmm1			;; B2 = R2 * .383
	xload	xmm2, [srcreg+off+d1+32]	;; R6 (I2)
	mulpd	xmm2, xmm7			;; D2 = I2 * .924
	addpd	xmm6, xmm2			;; B2 = B2 + D2 (new I2)

	xload	xmm5, [srcreg+off+d1+16]	;; R4
	mulpd	xmm5, xmm1			;; A4 = R4 * .383
	xload	xmm0, [srcreg+off+d1+48]	;; R8 (I4)
	mulpd	xmm0, xmm7			;; C4 = I4 * .924
	subpd	xmm5, xmm0			;; A4 = A4 - C4 (new R4)

	mulpd	xmm7, [srcreg+off+d1+16]	;; B4 = R4 * .924
	mulpd	xmm1, [srcreg+off+d1+48]	;; D4 = I4 * .383
	addpd	xmm7, xmm1			;; B4 = B4 + D4 (new I4)

	xprefetchw [srcreg+srcinc]

	xload	xmm1, [srcreg+off+16]		;; R3
	xload	xmm3, [srcreg+off+48]		;; R7 (I3)
	subpd	xmm1, xmm3			;; A3 = R3 - I3
	addpd	xmm3, [srcreg+off+16]		;; B3 = R3 + I3
	mulpd	xmm1, XMM_SQRTHALF		;; A3 = A3 * SQRTHALF (new R3)
	mulpd	xmm3, XMM_SQRTHALF		;; B3 = B3 * SQRTHALF (new I3)

	xprefetchw [srcreg+srcinc+d1]

	xcopy	xmm0, xmm4			;; Copy R2
	subpd	xmm4, xmm5			;; R2 = R2 - R4 (new R4)
	addpd	xmm5, xmm0			;; R4 = R2 + R4 (new R2)

	xcopy	xmm2, xmm6			;; Copy I2
	subpd	xmm6, xmm7			;; I2 = I2 - I4 (new I4)
	addpd	xmm7, xmm2			;; I4 = I2 + I4 (new I2)

	xload	xmm0, [srcreg+off]		;; R1
	subpd	xmm0, xmm1			;; R1 = R1 - R3 (new R3)
	addpd	xmm1, [srcreg+off]		;; R3 = R1 + R3 (new R1)

	xload	xmm2, [srcreg+off+32]		;; R5 (I1)
	subpd	xmm2, xmm3			;; I1 = I1 - I3 (new I3)
	addpd	xmm3, [srcreg+off+32]		;; I3 = I1 + I3 (new I1)

	subpd	xmm0, xmm6			;; R3 = R3 - I4 (new R3)
	multwo	xmm6				;; I4 = I4 * 2
	subpd	xmm2, xmm4			;; I3 = I3 - R4 (new I4)
	multwo	xmm4				;; R4 = R4 * 2
	subpd	xmm1, xmm5			;; R1 = R1 - R2 (new R2)
	multwo	xmm5				;; R2 = R2 * 2
	subpd	xmm3, xmm7			;; I1 = I1 - I2 (new I2)
	multwo	xmm7				;; I2 = I2 * 2
	addpd	xmm6, xmm0			;; I4 = R3 + I4 (new R4)
	addpd	xmm4, xmm2			;; R4 = I3 + R4 (new I3)
	addpd	xmm5, xmm1			;; R2 = R1 + R2 (new R1)
	addpd	xmm7, xmm3			;; I2 = I1 + I2 (new I1)

	xstore	[srcreg], xmm5			;; Save R1
	xstore	[srcreg+16], xmm7		;; Save I1

	xload	xmm5, [screg+32+16]		;; cosine/sine
	mulpd	xmm5, xmm0			;; A3 = R3 * cosine/sine
	subpd	xmm5, xmm4			;; A3 = A3 - I3
	mulpd	xmm4, [screg+32+16]		;; B3 = I3 * cosine/sine
	addpd	xmm4, xmm0			;; B3 = B3 + R3

	xload	xmm7, [screg+64+16]		;; cosine/sine
	mulpd	xmm7, xmm1			;; A2 = R2 * cosine/sine
	subpd	xmm7, xmm3			;; A2 = A2 - I2
	mulpd	xmm3, [screg+64+16]		;; B2 = I2 * cosine/sine
	addpd	xmm3, xmm1			;; B2 = B2 + R2

	xload	xmm0, [screg+96+16]		;; cosine/sine
	mulpd	xmm0, xmm6			;; A4 = R4 * cosine/sine
	subpd	xmm0, xmm2			;; A4 = A4 - I4
	mulpd	xmm2, [screg+96+16]		;; B4 = I4 * cosine/sine
	addpd	xmm2, xmm6			;; B4 = B4 + R4

	xload	xmm1, [screg+0+16]		;; cosine/sine
	mulpd	xmm1, [srcreg]			;; A1 = R1 * cosine/sine
	xload	xmm6, [srcreg+16]		;; Restore I1
	subpd	xmm1, xmm6			;; A1 = A1 - I1
	mulpd	xmm6, [screg+0+16]		;; B1 = I1 * cosine/sine
	addpd	xmm6, [srcreg]			;; B1 = B1 + R1

	mulpd	xmm5, [screg+32]		;; A3 = A3 * sine (final R3)
	mulpd	xmm4, [screg+32]		;; B3 = B3 * sine (final I3)
	mulpd	xmm7, [screg+64]		;; A2 = A2 * sine (final R2)
	mulpd	xmm3, [screg+64]		;; B2 = B2 * sine (final I2)
	mulpd	xmm0, [screg+96]		;; A4 = A4 * sine (final R4)
	mulpd	xmm2, [screg+96]		;; B4 = B4 * sine (final I4)

	mulpd	xmm1, [screg+0]			;; A1 = A1 * sine (final R1)
	mulpd	xmm6, [screg+0]			;; B1 = B1 * sine (final I1)

	xstore	[srcreg+d1], xmm5		;; Save R3
	xstore	[srcreg+d1+16], xmm4		;; Save I3
	xstore	[srcreg+d1+32], xmm0		;; Save R4
	xstore	[srcreg+d1+48], xmm2		;; Save I4
	xstore	[srcreg], xmm1			;; Save R1
	xstore	[srcreg+16], xmm6		;; Save I1
	xstore	[srcreg+32], xmm7		;; Save R2
	xstore	[srcreg+48], xmm3		;; Save I2
	bump	srcreg, srcinc
	ENDM


;; This code applies the roots-of-minus-1 premultipliers in a negacyclic FFT.
;; It also applies the three sin/cos multipliers after the first radix-4 butterfly.
;; We save memory by splitting the roots-of-minus-1 premultipliers such that every
;; macro uses the same premultiplier data.  The rest of the premultiplier is applied
;; at the end of pass 1.  This scheme is used by the r4delay FFTs.

;r4_x2cl_four_complex_first_djbfft MACRO srcreg,srcinc,d1,screg,pmreg
;	r4_x2cl_four_complex_first_djbfft_cmn srcreg,rbx,srcinc,d1,screg,pmreg
;	ENDM
r4_x2cl_four_complex_first_djbfft_scratch MACRO srcreg,srcinc,d1,screg,pmreg
	r4_x2cl_four_complex_first_djbfft_cmn srcreg,0,srcinc,d1,screg,pmreg
	ENDM
r4_x2cl_four_complex_first_djbfft_cmn MACRO srcreg,off,srcinc,d1,screg,pmreg
	xload	xmm7, [pmreg+0+16]		;; cosine/sine
	xload	xmm1, [srcreg+off]		;; R1
	mulpd	xmm7, xmm1			;; A1 = R1 * cosine/sine
	xload	xmm2, [srcreg+off+32]		;; R5 (I1)
	subpd	xmm7, xmm2			;; A1 = A1 - I1
	mulpd	xmm2, [pmreg+0+16]		;; B1 = I1 * cosine/sine
	addpd	xmm2, xmm1			;; B1 = B1 + R1

	xload	xmm6, [pmreg+32+16]		;; cosine/sine
	xload	xmm3, [srcreg+off+d1]		;; R2
	mulpd	xmm6, xmm3			;; A2 = R2 * cosine/sine
	xload	xmm4, [srcreg+off+d1+32]	;; R6 (I2)
	subpd	xmm6, xmm4			;; A2 = A2 - I2
	mulpd	xmm4, [pmreg+32+16]		;; B2 = I2 * cosine/sine
	addpd	xmm4, xmm3			;; B2 = B2 + R2

	xload	xmm5, [pmreg+64+16]		;; cosine/sine
	xload	xmm0, [srcreg+off+16]		;; R3
	mulpd	xmm5, xmm0			;; A3 = R3 * cosine/sine
	xload	xmm1, [srcreg+off+48]		;; R7 (I3)
	subpd	xmm5, xmm1			;; A3 = A3 - I3
	mulpd	xmm1, [pmreg+64+16]		;; B3 = I3 * cosine/sine
	addpd	xmm1, xmm0			;; B3 = B3 + R3

	xload	xmm3, [pmreg+96+16]		;; cosine/sine
	mulpd	xmm3, [srcreg+off+d1+16]	;; A4 = R4 * cosine/sine
	xload	xmm0, [srcreg+off+d1+48]	;; R8 (I4)
	subpd	xmm3, xmm0			;; A4 = A4 - I4
	mulpd	xmm0, [pmreg+96+16]		;; B4 = I4 * cosine/sine
	addpd	xmm0, [srcreg+off+d1+16]	;; B4 = B4 + R4

	mulpd	xmm7, [pmreg+0]			;; A1 = A1 * sine
	mulpd	xmm2, [pmreg+0]			;; B1 = B1 * sine
	mulpd	xmm6, [pmreg+32]		;; A2 = A2 * sine
	mulpd	xmm4, [pmreg+32]		;; B2 = B2 * sine
	mulpd	xmm5, [pmreg+64]		;; A3 = A3 * sine
	mulpd	xmm1, [pmreg+64]		;; B3 = B3 * sine
	mulpd	xmm3, [pmreg+96]		;; A4 = A4 * sine
	mulpd	xmm0, [pmreg+96]		;; B4 = B4 * sine

	xprefetchw [srcreg+srcinc]
	xprefetchw [srcreg+srcinc+d1]

	subpd	xmm6, xmm3			;; R2 = R2 - R4 (new R4)
	multwo	xmm3				;; R4 = R4 * 2
	addpd	xmm3, xmm6			;; R4 = R2 + R4 (new R2)

	subpd	xmm4, xmm0			;; I2 = I2 - I4 (new I4)
	multwo	xmm0				;; I4 = I4 * 2
	addpd	xmm0, xmm4			;; I4 = I2 + I4 (new I2)

	subpd	xmm7, xmm5			;; R1 = R1 - R3 (new R3)
	multwo	xmm5				;; R3 = R3 * 2
	addpd	xmm5, xmm7			;; R3 = R1 + R3 (new R1)

	subpd	xmm2, xmm1			;; I1 = I1 - I3 (new I3)
	multwo	xmm1				;; I3 = I3 * 2
	addpd	xmm1, xmm2			;; I3 = I1 + I3 (new I1)

	subpd	xmm7, xmm4			;; R3 = R3 - I4 (new R3)
	multwo	xmm4				;; I4 = I4 * 2
	subpd	xmm2, xmm6			;; I3 = I3 - R4 (new I4)
	multwo	xmm6				;; R4 = R4 * 2
	subpd	xmm5, xmm3			;; R1 = R1 - R2 (new R2)
	multwo	xmm3				;; R2 = R2 * 2
	subpd	xmm1, xmm0			;; I1 = I1 - I2 (new I2)
	multwo	xmm0				;; I2 = I2 * 2
	addpd	xmm4, xmm7			;; I4 = R3 + I4 (new R4)
	addpd	xmm6, xmm2			;; R4 = I3 + R4 (new I3)
	addpd	xmm3, xmm5			;; R2 = R1 + R2 (new R1)
	addpd	xmm0, xmm1			;; I2 = I1 + I2 (new I1)

	xstore	[srcreg], xmm3			;; Save R1
	xstore	[srcreg+16], xmm0		;; Save I1

	xload	xmm3, [screg+0+16]		;; cosine/sine
	mulpd	xmm3, xmm7			;; A3 = R3 * cosine/sine
	subpd	xmm3, xmm6			;; A3 = A3 - I3
	mulpd	xmm6, [screg+0+16]		;; B3 = I3 * cosine/sine
	addpd	xmm6, xmm7			;; B3 = B3 + R3

	xload	xmm0, [screg+32+16]		;; cosine/sine
	mulpd	xmm0, xmm5			;; A2 = R2 * cosine/sine
	subpd	xmm0, xmm1			;; A2 = A2 - I2
	mulpd	xmm1, [screg+32+16]		;; B2 = I2 * cosine/sine
	addpd	xmm1, xmm5			;; B2 = B2 + R2

	xload	xmm7, [screg+0+16]		;; cosine/sine
	mulpd	xmm7, xmm4			;; A4 = R4 * cosine/sine
	addpd	xmm7, xmm2			;; A4 = A4 + I4
	mulpd	xmm2, [screg+0+16]		;; B4 = I4 * cosine/sine
	subpd	xmm2, xmm4			;; B4 = B4 - R4

	xload	xmm5, [screg+0]
	mulpd	xmm3, xmm5			;; A3 = A3 * sine (final R3)
	mulpd	xmm6, xmm5			;; B3 = B3 * sine (final I3)
	mulpd	xmm0, [screg+32]		;; A2 = A2 * sine (final R2)
	mulpd	xmm1, [screg+32]		;; B2 = B2 * sine (final I2)
	mulpd	xmm7, xmm5			;; A4 = A4 * sine (final R4)
	mulpd	xmm2, xmm5			;; B4 = B4 * sine (final I4)

	xstore	[srcreg+d1], xmm3		;; Save R3
	xstore	[srcreg+d1+16], xmm6		;; Save I3
	xstore	[srcreg+d1+32], xmm7		;; Save R4
	xstore	[srcreg+d1+48], xmm2		;; Save I4
	xstore	[srcreg+32], xmm0		;; Save R2
	xstore	[srcreg+48], xmm1		;; Save I2
	bump	srcreg, srcinc
	ENDM

IFDEF X86_64

;r4_x2cl_four_complex_first_djbfft MACRO srcreg,srcinc,d1,screg,pmreg
;	r4_x2cl_four_complex_first_djbfft_cmn srcreg,rbx,srcinc,d1,screg,pmreg
;	ENDM
r4_x2cl_four_complex_first_djbfft_scratch MACRO srcreg,srcinc,d1,screg,pmreg
	r4_x2cl_four_complex_first_djbfft_cmn srcreg,0,srcinc,d1,screg,pmreg
	ENDM
r4_x2cl_four_complex_first_djbfft_cmn MACRO srcreg,off,srcinc,d1,screg,pmreg

	xload	xmm0, [srcreg+off+d1]		;; R2
	xload	xmm1, [pmreg+32+16]		;; cosine/sine 2
	xcopy	xmm2, xmm0			;; Copy R2			; 1-3
	mulpd	xmm0, xmm1			;; A2 = R2 * cosine/sine	; 1-5

	xload	xmm3, [srcreg+off+d1+16]	;; R4
	xload	xmm4, [pmreg+96+16]		;; cosine/sine 4
	xcopy	xmm5, xmm3			;; Copy R4			; 2-4
	mulpd	xmm3, xmm4			;; A4 = R4 * cosine/sine	; 2-6

	xload	xmm6, [srcreg+off+d1+32]	;; I2
	mulpd	xmm1, xmm6			;; B2 = I2 * cosine/sine	; 3-7

	xload	xmm7, [srcreg+off+d1+48]	;; I4
	mulpd	xmm4, xmm7			;; B4 = I4 * cosine/sine	; 4-8

	xload	xmm8, [srcreg+off]		;; R1
	xload	xmm9, [pmreg+0+16]		;; cosine/sine 1
	xcopy	xmm10, xmm8			;; Copy R1			; 5-7
	mulpd	xmm8, xmm9			;; A1 = R1 * cosine/sine	; 5-9

	subpd	xmm0, xmm6			;; A2 = A2 - I2			; 6-8
	xload	xmm11, [srcreg+off+16]		;; R3
	xload	xmm12, [pmreg+64+16]		;; cosine/sine 3
	xcopy	xmm13, xmm11			;; Copy R3			; 6-8
	mulpd	xmm11, xmm12			;; A3 = R3 * cosine/sine	; 6-10

	subpd	xmm3, xmm7			;; A4 = A4 - I4			; 7-9
	xload	xmm14, [srcreg+off+32]		;; I1
	mulpd	xmm9, xmm14			;; B1 = I1 * cosine/sine	; 7-11

	addpd	xmm1, xmm2			;; B2 = B2 + R2			; 8-10
	xload	xmm15, [srcreg+off+48]		;; I3
	mulpd	xmm12, xmm15			;; B3 = I3 * cosine/sine	; 8-12	avail 6,7,2

	addpd	xmm4, xmm5			;; B4 = B4 + R4			; 9-11	avail 6,7,2,5
	xload	xmm6, [pmreg+32]		;; sine 2			; 9	avail 7,2,5
	mulpd	xmm0, xmm6			;; A2 = A2 * sine		; 9-13

	subpd	xmm8, xmm14			;; A1 = A1 - I1			; 10-12	avail 7,2,5,14
	xload	xmm7, [pmreg+96]		;; sine 4			; 10	avail 2,5,14
	mulpd	xmm3, xmm7			;; A4 = A4 * sine		; 10-14

	subpd	xmm11, xmm15			;; A3 = A3 - I3			; 11-13	avail 2,5,14,15
	mulpd	xmm1, xmm6			;; B2 = B2 * sine		; 11-15	avail 2,5,14,15,6
	xload	xmm2, [pmreg+0]			;; sine 1			; 11	avail 5,14,15,6

	addpd	xmm9, xmm10			;; B1 = B1 + R1			; 12-14	avail 5,14,15,6,10
	mulpd	xmm4, xmm7			;; B4 = B4 * sine		; 12-16	avail 5,14,15,6,10,7
	xload	xmm5, [pmreg+64]		;; sine 3			; 12	avail 14,15,6,10,7

	addpd	xmm12, xmm13			;; B3 = B3 + R3			; 13-15	avail 14,15,6,10,7,13
	mulpd	xmm8, xmm2			;; A1 = A1 * sine		; 13-17

	mulpd	xmm11, xmm5			;; A3 = A3 * sine		; 14-18
	xcopy	xmm6, xmm0			;; Copy R2			; 14 on Core i7, 14-16 on Core 2 avail 14,15,10,7,13

	xprefetchw [srcreg+srcinc]

	subpd	xmm0, xmm3			;; R2 = R2 - R4 (new R4)	; 15-17
	mulpd	xmm9, xmm2			;; B1 = B1 * sine		; 15-19	avail 14,15,10,7,13,2

	addpd	xmm3, xmm6			;; R4 = R2 + R4 (new R2)	; 16-18	avail 14,15,10,7,13,2,6
	mulpd	xmm12, xmm5			;; B3 = B3 * sine		; 16-20	avail 14,15,10,7,13,2,6,5

	xcopy	xmm7, xmm1			;; Copy I2			; 16 on Core i7, 16-18 on Core 2
	subpd	xmm1, xmm4			;; I2 = I2 - I4 (new I4)	; 17-19
	addpd	xmm4, xmm7			;; I4 = I2 + I4 (new I2)	; 18-20

	xcopy	xmm7, xmm8			;; Copy R1			; 18 on Core i7, 16-18 on Core 2
	subpd	xmm8, xmm11			;; R1 = R1 - R3 (new R3)	; 19-21
	addpd	xmm11, xmm7			;; R3 = R1 + R3 (new R1)	; 20-22

	xcopy	xmm7, xmm9			;; Copy I1			; 20 on Core i7, 16-18 on Core 2

	subpd	xmm9, xmm12			;; I1 = I1 - I3 (new I3)	; 21-23
	xcopy	xmm2, xmm1			;; Copy I4			; 21-23 (20) avail 14,15,10,13,6,5

	xprefetchw [srcreg+srcinc+d1]

	addpd	xmm12, xmm7			;; I3 = I1 + I3 (new I1)	; 22-24 avail 14,15,10,13,6,5,7
	xload	xmm6, [screg+0+16]		;; cosine/sine 3&4		; 22	avail 14,15,10,13,5,7

	addpd	xmm1, xmm8			;; I4 = R3 + I4 (newer R4)	; 23-25
	xcopy	xmm7, xmm0			;; Copy R4			; 23-25 (18) avail 14,15,10,13,5

	subpd	xmm8, xmm2			;; R3 = R3 - I4 (newer R3)	; 24-26 avail 14,15,10,13,5,2
	xcopy	xmm2, xmm11			;; Copy R1			; 24-26 (23) avail 14,15,10,13,5

	addpd	xmm0, xmm9			;; R4 = I3 + R4 (newer I3)	; 25-27
	xcopy	xmm10, xmm12			;; Copy I1			; 27-29 (25) avail 15,13,5

	subpd	xmm9, xmm7			;; I3 = I3 - R4 (newer I4)	; 26-28 avail 14,15,13,5,7
	xcopy	xmm14, xmm1			;; Copy R4			; 26-28 avail 15,13,5,7
	mulpd	xmm1, xmm6			;; A4 = R4 * cosine/sine	; 26-30

	subpd	xmm11, xmm3			;; R1 = R1 - R2 (newer R2)	; 27-29
	xcopy	xmm15, xmm8			;; Copy R3			; 27-29 avail 13,5,7
	mulpd	xmm8, xmm6			;; A3 = R3 * cosine/sine	; 27-31

	subpd	xmm12, xmm4			;; I1 = I1 - I2 (newer I2)	; 28-30
	xcopy	xmm13, xmm0			;; Copy I3			; 28-30 avail 5,7
	mulpd	xmm0, xmm6			;; B3 = I3 * cosine/sine	; 28-32

	addpd	xmm3, xmm2			;; R2 = R1 + R2 (final R1)	; 29-31	avail 5,7,2
	mulpd	xmm6, xmm9			;; B4 = I4 * cosine/sine	; 29-33
	xload	xmm5, [screg+32+16]		;; cosine/sine 2		; 29	avail 7,2

	addpd	xmm4, xmm10			;; I2 = I1 + I2 (final I1)	; 30-32	avail 7,2,10
	xcopy	xmm7, xmm11			;; Copy R2			; 30-32	avail 2,10
	mulpd	xmm11, xmm5			;; A2 = R2 * cosine/sine	; 30-34

	addpd	xmm1, xmm9			;; A4 = A4 + I4			; 31-33	avail 2,10,9
	mulpd	xmm5, xmm12			;; B2 = I2 * cosine/sine	; 31-35
	xload	xmm2, [screg+0]			;; sine 3&4			; 31	avail 10,9

	subpd	xmm8, xmm13			;; A3 = A3 - I3			; 32-34	avail 10,9,13
	xstore	[srcreg], xmm3			;; Save R1			; 32	avail 10,9,13,3

	addpd	xmm0, xmm15			;; B3 = B3 + R3			; 33-35	avail 10,9,13,3,15
	xstore	[srcreg+16], xmm4		;; Save I1			; 33	avail 10,9,13,3,15,4

	subpd	xmm6, xmm14			;; B4 = B4 - R4			; 34-36	avail 10,9,13,3,15,4,14
	mulpd	xmm1, xmm2			;; A4 = A4 * sine (final R4)	; 34-38
	xload	xmm3, [screg+32]		;; sine 2			; 34	avail 10,9,13,15,4,14

	subpd	xmm11, xmm12			;; A2 = A2 - I2			; 35-37	avail 10,9,13,15,4,14,12
	mulpd	xmm8, xmm2			;; A3 = A3 * sine (final R3)	; 35-39

	addpd	xmm5, xmm7			;; B2 = B2 + R2			; 36-38	avail 10,9,13,15,4,14,12,7
	mulpd	xmm0, xmm2			;; B3 = B3 * sine (final I3)	; 36-40

	mulpd	xmm6, xmm2			;; B4 = B4 * sine (final I4)	; 37-41	avail 10,9,13,15,4,14,12,7,2

	mulpd	xmm11, xmm3			;; A2 = A2 * sine (final R2)	; 38-42

	mulpd	xmm5, xmm3			;; B2 = B2 * sine (final I2)	; 39-43	avail 10,9,13,15,4,14,12,7,2,3

	xstore	[srcreg+d1+32], xmm1		;; Save R4			; 39
	xstore	[srcreg+d1], xmm8		;; Save R3			; 40
	xstore	[srcreg+d1+16], xmm0		;; Save I3			; 41
	xstore	[srcreg+d1+48], xmm6		;; Save I4			; 42
	xstore	[srcreg+32], xmm11		;; Save R2			; 43
	xstore	[srcreg+48], xmm5		;; Save I2			; 44
	bump	srcreg, srcinc
	ENDM

ENDIF

;;
;; ************************************* four-complex-last-unfft variants ******************************************
;;

;; This code applies the sin/cos multipliers before a radix-4 butterfly.
;; Then it applies the premultipliers since the negacyclic inverse FFT is complete.
;; We save memory by splitting the roots-of-minus-1 premultipliers such that every
;; macro uses the same premultiplier data.  Every sin/cos postmultiplier, a very big
;; table anyway, are then multiplied by the other part of the split roots-of-minus-1.
;; The common postmuliplier data is 1, .924-.383i, SQRTHALF-SQRTHALFi, .383-.924i.

r4_x4cl_four_complex_last_unfft4 MACRO srcreg,srcinc,d1,d2,screg,off
	xload	xmm0, [srcreg]		;; R1
	xload	xmm1, [srcreg+32]	;; R2
	xload	xmm4, [srcreg+d2]	;; R5
	xload	xmm5, [srcreg+d2+32]	;; R6
	r4_x4c_unfft_postmult xmm0,xmm1,xmm2,xmm3,xmm4,xmm5,xmm6,xmm7,[srcreg+d1],[srcreg+d1+32],[srcreg+d2+d1],[srcreg+d2+d1+32],[srcreg],[srcreg+32],screg,0,srcreg+srcinc,d1
;;	xstore	[srcreg], xmm		;; Save R1
;;	xstore	[srcreg+32], xmm	;; Save R5
	xload	xmm5, [srcreg+16]	;; R1
	xload	xmm6, [srcreg+d2+16]	;; R5
	xstore	[srcreg+d2], xmm7	;; Save R2
	xstore	[srcreg+d2+32], xmm0	;; Save R6
	xload	xmm7, [srcreg+48]	;; R2
	xload	xmm0, [srcreg+d2+48]	;; R6
	xstore	[srcreg+d2+16], xmm4	;; Save R4
	xstore	[srcreg+d2+48], xmm1	;; Save R8
	xstore	[srcreg+16], xmm2	;; Save R3
	xstore	[srcreg+48], xmm3	;; Save R7
	r4_x4c_unfft_postmult xmm5,xmm7,xmm2,xmm3,xmm6,xmm0,xmm4,xmm1,[srcreg+d1+16],[srcreg+d1+48],[srcreg+d2+d1+16],[srcreg+d2+d1+48],[srcreg+d1],[srcreg+d1+32],screg,off,srcreg+srcinc+d2,d1
;;	xstore	[srcreg+d1], xmm	;; Save R1
;;	xstore	[srcreg+d1+32], xmm	;; Save R5
	xstore	[srcreg+d2+d1], xmm1	;; Save R2
	xstore	[srcreg+d2+d1+32], xmm5	;; Save R6
	xstore	[srcreg+d2+d1+16], xmm6	;; Save R4
	xstore	[srcreg+d2+d1+48], xmm7	;; Save R8
	xstore	[srcreg+d1+16], xmm2	;; Save R3
	xstore	[srcreg+d1+48], xmm3	;; Save R7
	bump	srcreg, srcinc
	ENDM

r4_x4c_unfft_postmult MACRO r1, r2, r3, r4, r5, r6, r7, r8, mem3, mem4, mem7, mem8, dest1, dest2, screg, off, pre1, pre2
	xload	r7, [screg+off+0+16]	;; cosine/sine
	mulpd	r7, r1			;; A1 = R1 * cosine/sine
	addpd	r7, r2			;; A1 = A1 + I1
	mulpd	r2, [screg+off+0+16]	;; B1 = I1 * cosine/sine
	subpd	r2, r1			;; B1 = B1 - R1

	xload	r8, [screg+off+32+16]	;; cosine/sine
	mulpd	r8, r5			;; A3 = R3 * cosine/sine
	addpd	r8, r6			;; A3 = A3 + I3
	mulpd	r6, [screg+off+32+16]	;; B3 = I3 * cosine/sine
	subpd	r6, r5			;; B3 = B3 - R3

	xload	r3, [screg+off+64+16]	;; cosine/sine
	mulpd	r3, mem3		;; A2 = R2 * cosine/sine
	xload	r4, mem4		;; R4 (I2)
	addpd	r3, r4			;; A2 = A2 + I2
	mulpd	r4, [screg+off+64+16]	;; B2 = I2 * cosine/sine
	subpd	r4, mem3		;; B2 = B2 - R2

	xload	r1, [screg+off+96+16]	;; cosine/sine
	mulpd	r1, mem7		;; A4 = R4 * cosine/sine
	xload	r5, mem8		;; I4
	addpd	r1, r5	 		;; A4 = A4 + I4
	mulpd	r5, [screg+off+96+16]	;; B4 = I4 * cosine/sine
	subpd	r5, mem7		;; B4 = B4 - R4

	mulpd	r7, [screg+off+0]	;; A1 = A1 * sine (new R1)
	mulpd	r2, [screg+off+0]	;; B1 = B1 * sine (new I1)

	mulpd	r8, [screg+off+32]	;; A3 = A3 * sine (new R3)
	mulpd	r6, [screg+off+32]	;; B3 = B3 * sine (new I3)

	mulpd	r3, [screg+off+64]	;; A2 = A2 * sine (new R2)
	mulpd	r4, [screg+off+64]	;; B2 = B2 * sine (new I2)

	mulpd	r1, [screg+off+96]	;; A4 = A4 * sine (new R4)
	mulpd	r5, [screg+off+96]	;; B4 = B4 * sine (new I4)

	xprefetchw [pre1]

	subpd	r7, r3			;; R1 = R1 - R2 (new R2)
	multwo	r3			;; R2 = R2 * 2
	addpd	r3, r7			;; R2 = R1 + R2 (new R1)

	subpd	r2, r4			;; I1 = I1 - I2 (new I2)
	multwo	r4			;; I2 = I2 * 2
	addpd	r4, r2			;; I2 = I1 + I2 (new I1)

	subpd	r1, r8			;; R4 = R4 - R3 (new I4)
	multwo	r8			;; R3 = R3 * 2
	addpd	r8, r1			;; R3 = R4 + R3 (new R3)

	subpd	r6, r5			;; I3 = I3 - I4 (new R4)
	multwo	r5			;; I4 = I4 * 2
	addpd	r5, r6			;; I4 = I3 + I4 (new I3)

	xprefetchw [pre1][pre2]

	subpd	r7, r6			;; R2 = R2 - R4 (new R4)
	multwo	r6			;; R4 = R4 * 2
	addpd	r6, r7			;; R4 = R2 + R4 (new R2)

	subpd	r2, r1			;; I2 = I2 - I4 (new I4)
	multwo	r1			;; I4 = I4 * 2
	addpd	r1, r2			;; I4 = I2 + I4 (new I2)

	subpd	r3, r8			;; R1 = R1 - R3 (new R3)
	multwo	r8			;; R3 = R3 * 2
	addpd	r8, r3			;; R3 = R1 + R3 (final R1)

	subpd	r4, r5			;; I1 = I1 - I3 (new I3)
	multwo	r5			;; I3 = I3 * 2
	addpd	r5, r4			;; I3 = I1 + I3 (final I1)

	xstore	dest1, r8			;; Save R1
	xstore	dest2, r5			;; Save I1

	mulpd	r3, XMM_SQRTHALF		;; A3 = R3 * SQRTHALF
	mulpd	r4, XMM_SQRTHALF		;; B3 = I3 * SQRTHALF
	xcopy	r8, r4				;; Copy B3
	subpd	r4, r3				;; B3 = B3 - A3 (final I3)
	addpd	r3, r8				;; A3 = A3 + B3	(final R3)

	xload	r8, XMM_P924
	mulpd	r8, r6				;; A2 = R2 * .924
	xload	r5, XMM_P383
	mulpd	r5, r1				;; C2 = I2 * .383
	addpd	r8, r5				;; A2 = A2 + C2 (final R2)
	mulpd	r6, XMM_P383			;; B2 = R2 * .383
	mulpd	r1, XMM_P924			;; D2 = I2 * .924
	subpd	r1, r6				;; D2 = D2 - B2 (final I2)

	xload	r5, XMM_P383
	mulpd	r5, r7				;; A4 = R4 * .383
	xload	r6, XMM_P924
	mulpd	r6, r2				;; C4 = I4 * .924
	addpd	r5, r6				;; A4 = A4 + C4	(final R4)
	mulpd	r7, XMM_P924			;; B4 = R4 * .924
	mulpd	r2, XMM_P383			;; D4 = I4 * .383
	subpd	r2, r7				;; D4 = D4 - B4	(final I4)
	ENDM

;; This code applies the sin/cos multipliers before a radix-4 butterfly.
;; After the butterfly, it applies the negacyclic premultipliers since the inverse FFT is complete.

r4_x4cl_four_complex_last_djbunfft MACRO srcreg,srcinc,d1,d2,screg,off,pmreg,pmoff
	d3 = d2+d1
	xload	xmm2, [srcreg+d1+16]	;; R2
	xload	xmm6, [srcreg+d3+16]	;; R4
	xload	xmm3, [srcreg+d1+48]	;; I2
	xload	xmm7, [srcreg+d3+48]	;; I4
	r4_x4c_djbunfft_postmult xmm0,xmm1,xmm2,xmm3,xmm4,xmm5,xmm6,xmm7,[srcreg+16],[srcreg+48],[srcreg+d2+16],[srcreg+d2+48],[srcreg+d1+16],[srcreg+d1+48],screg,off,pmreg,pmoff,srcreg+srcinc+d2,d1
	xstore	[srcreg+d1+16], xmm2	;; Save R3
	xstore	[srcreg+d1+48], xmm3	;; Save I3
	xload	xmm2, [srcreg+d1]	;; R2
	xload	xmm3, [srcreg+d3]	;; R4
	xstore	[srcreg+d2+d1+16], xmm4	;; Save R4
	xstore	[srcreg+d2+d1+48], xmm0	;; Save I4
	xload	xmm4, [srcreg+d1+32]	;; I2
	xload	xmm0, [srcreg+d3+32]	;; I4
	xstore	[srcreg+d1], xmm5	;; Save R1
	xstore	[srcreg+d1+32], xmm7	;; Save I1
	xstore	[srcreg+d2+d1], xmm1	;; Save R2
	xstore	[srcreg+d2+d1+32], xmm6	;; Save I2
	r4_x4c_djbunfft_postmult xmm1,xmm5,xmm2,xmm4,xmm6,xmm7,xmm3,xmm0,[srcreg],[srcreg+32],[srcreg+d2],[srcreg+d2+32],[srcreg],[srcreg+32],screg,0,pmreg,0,srcreg+srcinc,d1
	xstore	[srcreg], xmm7		;; Save R1
	xstore	[srcreg+32], xmm0	;; Save I1
	xstore	[srcreg+16], xmm2	;; Save R3
	xstore	[srcreg+48], xmm4	;; Save I3
	xstore	[srcreg+d2], xmm5	;; Save R2
	xstore	[srcreg+d2+32], xmm3	;; Save I2
	xstore	[srcreg+d2+16], xmm6	;; Save R4
	xstore	[srcreg+d2+48], xmm1	;; Save I4
	bump	srcreg, srcinc
	ENDM

r4_x4c_djbunfft_postmult MACRO r1,r2,r3,r4,r5,r6,r7,r8,mem1,mem2,mem5,mem6,dest1,dest2,screg,off,pmreg,pmoff,pre1,pre2
	xload	r5, [screg+off+32+16]		;; cosine/sine
	mulpd	r5, r3				;; A2 = R2 * cosine/sine
	addpd	r5, r4				;; A2 = A2 + I2
	mulpd	r4, [screg+off+32+16]		;; B2 = I2 * cosine/sine
	subpd	r4, r3				;; B2 = B2 - R2

	xcopy	r6, r7				;; Copy R4
	xload	r1, [screg+off+0+16]		;; cosine/sine
	mulpd	r7, r1				;; A4 = R4 * cosine/sine
	subpd	r7, r8				;; A4 = A4 - I4
	mulpd	r8, r1				;; B4 = I4 * cosine/sine
	addpd	r8, r6				;; B4 = B4 + R4

	xload	r2, mem5			;; R3
	mulpd	r2, r1				;; A3 = R3 * cosine/sine
	addpd	r2, mem6			;; A3 = A3 + I3
	mulpd	r1, mem6			;; B3 = I3 * cosine/sine
	subpd	r1, mem5			;; B3 = B3 - R3

	mulpd	r5, [screg+off+32]		;; A2 = A2 * sine (new R2)
	mulpd	r4, [screg+off+32]		;; B2 = B2 * sine (new I2)

	xload	r3, [screg+off+0]
	mulpd	r7, r3				;; A4 = A4 * sine (new R4)
	mulpd	r8, r3				;; B4 = B4 * sine (new I4)

	mulpd	r2, r3				;; A3 = A3 * sine (new R3)
	mulpd	r1, r3				;; B3 = B3 * sine (new I3)

	xprefetchw [pre1]

	xload	r6, mem1			;; R1
	subpd	r6, r5				;; R1 = R1 - R2 (new R2)
	addpd	r5, mem1			;; R2 = R1 + R2 (new R1)
	xcopy	r3, r7				;; Copy R4
	subpd	r7, r2				;; R4 = R4 - R3 (new I4)
	addpd	r2, r3				;; R3 = R4 + R3 (new R3)
	xcopy	r3, r1				;; Copy I3
	subpd	r1, r8				;; I3 = I3 - I4 (new R4)
	addpd	r8, r3				;; I4 = I3 + I4 (new I3)

	xprefetchw [pre1][pre2]

	xcopy	r3, r6				;; Copy R2
	subpd	r6, r1				;; R2 = R2 - R4 (new R4)
	addpd	r1, r3				;; R4 = R2 + R4 (new R2)

	xload	r3, mem2			;; I1
	subpd	r3, r4				;; I1 = I1 - I2 (new I2)
	addpd	r4, mem2			;; I2 = I1 + I2 (new I1)

	xstore	dest1, r6			;; Save R4

	xcopy	r6, r5				;; Copy R1
	subpd	r5, r2				;; R1 = R1 - R3 (new R3)
	addpd	r2, r6				;; R3 = R1 + R3 (new R1)

	xcopy	r6, r7				;; Copy I4
	addpd	r7, r3				;; I4 = I2 + I4 (new I2)
	subpd	r3, r6				;; I2 = I2 - I4 (new I4)

	xcopy	r6, r4				;; Copy I1
	subpd	r4, r8				;; I1 = I1 - I3 (new I3)
	addpd	r8, r6				;; I3 = I1 + I3 (new I1)

	xstore	dest2, r3			;; Save I4

	xload	r6, [pmreg+pmoff+0+16]		;; cosine/sine
	mulpd	r6, r2				;; A1 = R1 * cosine/sine

	xload	r3, [pmreg+pmoff+64+16]		;; cosine/sine
	mulpd	r3, r5				;; A3 = R3 * cosine/sine
	addpd	r6, r8				;; A1 = A1 + I1
	mulpd	r8, [pmreg+pmoff+0+16]		;; B1 = I1 * cosine/sine
	subpd	r8, r2				;; B1 = B1 - R1

	xload	r2, [pmreg+pmoff+32+16]		;; cosine/sine
	mulpd	r2, r1				;; A2 = R2 * cosine/sine
	addpd	r3, r4				;; A3 = A3 + I3
	mulpd	r4, [pmreg+pmoff+64+16]		;; B3 = I3 * cosine/sine
	subpd	r4, r5				;; B3 = B3 - R3

	xload	r5, [pmreg+pmoff+96+16]		;; cosine/sine
	mulpd	r5, dest1			;; A4 = R4 * cosine/sine
	addpd	r2, r7				;; A2 = A2 + I2
	mulpd	r7, [pmreg+pmoff+32+16]		;; B2 = I2 * cosine/sine
	subpd	r7, r1				;; B2 = B2 - R2

	xload	r1, dest2			;; Restore I4
	addpd	r5, r1				;; A4 = A4 + I4
	mulpd	r1, [pmreg+pmoff+96+16]		;; B4 = I4 * cosine/sine
	subpd	r1, dest1			;; B4 = B4 - R4

	mulpd	r6, [pmreg+pmoff+0]		;; A1 = A1 * sine
	mulpd	r8, [pmreg+pmoff+0]		;; B1 = B1 * sine
	mulpd	r3, [pmreg+pmoff+64]		;; A3 = A3 * sine
	mulpd	r4, [pmreg+pmoff+64]		;; B3 = B3 * sine
	mulpd	r2, [pmreg+pmoff+32]		;; A2 = A2 * sine
	mulpd	r7, [pmreg+pmoff+32]		;; B2 = B2 * sine
	mulpd	r5, [pmreg+pmoff+96]		;; A4 = A4 * sine
	mulpd	r1, [pmreg+pmoff+96]		;; B4 = B4 * sine
	ENDM

IFDEF X86_64

r4_x4cl_four_complex_last_djbunfft MACRO srcreg,srcinc,d1,d2,screg,scoff,pmreg,pmoff

	xload	xmm0, [srcreg+d1+16]		;; R2
	xload	xmm1, [screg+scoff+32+16]	;; cosine/sine 2
	xcopy	xmm2, xmm0			;; Copy R2
	mulpd	xmm0, xmm1			;; A2 = R2 * cosine/sine	; 1-5
	xload	xmm3, [srcreg+d1+48]		;; I2
	mulpd	xmm1, xmm3			;; B2 = I2 * cosine/sine	; 2-6

	xload	xmm4, [srcreg+d2+d1+16]		;; R4
	xload	xmm5, [screg+scoff+0+16]	;; cosine/sine 3&4
	xcopy	xmm6, xmm4			;; Copy R4
	mulpd	xmm4, xmm5			;; A4 = R4 * cosine/sine	; 3-7
	xload	xmm7, [srcreg+d2+d1+48]		;; I4
	xcopy	xmm8, xmm7			;; Copy I4
	mulpd	xmm7, xmm5			;; B4 = I4 * cosine/sine	; 4-8

	xload	xmm9, [srcreg+d2+16]		;; R3
	xcopy	xmm10, xmm9			;; Copy R3
	mulpd	xmm9, xmm5			;; A3 = R3 * cosine/sine	; 5-7

	addpd	xmm0, xmm3			;; A2 = A2 + I2			; 6-8	avail 3,11+
	xload	xmm11, [srcreg+d2+48]		;; I3				; 6	avail 3,12+
	mulpd	xmm5, xmm11			;; B3 = I3 * cosine/sine	; 6-10

	subpd	xmm1, xmm2			;; B2 = B2 - R2			; 7-9	avail 2,3,12+
	xload	xmm12, [srcreg+d1]		;;#2 R2
	xload	xmm13, [screg+32+16]		;;#2 cosine/sine 2
	xcopy	xmm14, xmm12			;;#2 Copy R2
	mulpd	xmm12, xmm13			;;#2 A2 = R2 * cosine/sine	; 7-11	avail 2,3,15

	subpd	xmm4, xmm8			;; A4 = A4 - I4			; 8-10	avail 2,3,8,15
	xload	xmm15, [srcreg+d1+32]		;;#2 I2				; 8	avail 2,3,8
	mulpd	xmm13, xmm15			;;#2 B2 = I2 * cosine/sine	; 8-12

	addpd	xmm7, xmm6			;; B4 = B4 + R4			; 9-11	avail 2,3,8,6
	xload	xmm2, [screg+scoff+32]		;; sine 2			; 9	avail 3,8,6
	mulpd	xmm0, xmm2			;; A2 = A2 * sine (R2)		; 9-13

	addpd	xmm9, xmm11			;; A3 = A3 + I3			; 10-12	avail 3,8,6,11
	mulpd	xmm1, xmm2			;; B2 = B2 * sine (I2)		; 10-14	avail 3,8,6,11,2
	xload	xmm3, [screg+scoff+0]		;; sine 3&4			; 10	avail 8,6,11,2

	subpd	xmm5, xmm10			;; B3 = B3 - R3			; 11-13	avail 8,6,11,2,10
	xload	xmm6, [srcreg+16]		;; R1				; 11	avail 8,11,2,10
	mulpd	xmm4, xmm3			;; A4 = A4 * sine (R4)		; 11-15

	addpd	xmm12, xmm15			;;#2 A2 = A2 + I2		; 12-14	avail 8,11,2,10,15
	mulpd	xmm7, xmm3			;; B4 = B4 * sine (I4)		; 12-16
	xcopy	xmm2, xmm6			;; Copy R1			; 12-14	avail 8,11,10,15
	xload	xmm8, [screg+32]		;;#2 sine 2			; 12	avail 11,10,15

	subpd	xmm13, xmm14			;;#2 B2 = B2 - R2		; 13-15	avail 11,10,15,14
	mulpd	xmm9, xmm3			;; A3 = A3 * sine (R3)		; 13-17
	xload	xmm11, [srcreg+48]		;; I1				; 13	avail 10,15,14

	subpd	xmm6, xmm0			;; R1 = R1 - R2 (new R2)	; 14-16
	mulpd	xmm5, xmm3			;; B3 = B3 * sine (I3)		; 14-18	avail 10,15,14,3
	xcopy	xmm3, xmm11			;; Copy I1			; 14-16	avail 10,15,14

	addpd	xmm0, xmm2			;; R2 = R1 + R2 (new R1)	; 15-17	avail 10,15,14,2
	mulpd	xmm12, xmm8			;;#2 A2 = A2 * sine (R2)	; 15-19 (can safely delay)

	subpd	xmm11, xmm1			;; I1 = I1 - I2 (new I2)	; 16-18
	mulpd	xmm13, xmm8			;;#2 B2 = B2 * sine (I2)	; 16-20	(can safely delay) avail 10,15,14,2,8
	xcopy	xmm2, xmm4			;; Copy R4			; 16-18	avail 10,15,14,8

	xprefetchw [srcreg+srcinc+d2]

	addpd	xmm1, xmm3			;; I2 = I1 + I2 (new I1)	; 17-19	avail 10,15,14,8,3

	subpd	xmm4, xmm9			;; R4 = R4 - R3 (new I4)	; 18-20
	xcopy	xmm3, xmm7			;; Copy I4			; 18-20 (17) avail 10,15,14,8

	addpd	xmm9, xmm2			;; R3 = R4 + R3 (new R3)	; 19-21	avail 10,15,14,8,2

	addpd	xmm7, xmm5			;; I4 = I3 + I4 (new I3)	; 20-22
	xcopy	xmm2, xmm11			;; Copy I2			; 20-22 (19) avail 10,15,14,8

	subpd	xmm5, xmm3			;; I3 = I3 - I4 (new R4)	; 21-23	avail 10,15,14,8,3
	xload	xmm10, [pmreg+pmoff+96+16]	;; cosine/sine 4		; 21	avail 15,14,8,3

	xprefetchw [srcreg+srcinc+d2+d1]

	subpd	xmm11, xmm4			;; I2 = I2 - I4 (newer I4)	; 22-24
	xcopy	xmm3, xmm6			;; Copy R2			; 22-24 (17) avail 15,14,8
	xload	xmm15, [pmreg+pmoff+32+16]	;; cosine/sine 2		; 22	avail 14,8

	addpd	xmm4, xmm2			;; I4 = I2 + I4 (newer I2)	; 23-25	avail 14,8,2
	xcopy	xmm2, xmm0			;; Copy R1			; 23-25 (18) avail 14,8

	subpd	xmm6, xmm5			;; R2 = R2 - R4 (newer R4)	; 24-26
	xcopy	xmm8, xmm1			;; Copy I1			; 24-26 (20) avail 14

	addpd	xmm5, xmm3			;; R4 = R2 + R4 (newer R2)	; 25-27	avail 14,3
	xcopy	xmm14, xmm11			;; Copy I4			; 25-27	avail 3
	mulpd	xmm11, xmm10			;; B4 = I4 * cosine/sine	; 25-29

	subpd	xmm0, xmm9			;; R1 = R1 - R3 (newer R3)	; 26-28
	xcopy	xmm3, xmm4			;; Copy I2			; 26-28	avail none
	mulpd	xmm4, xmm15			;; B2 = I2 * cosine/sine	; 26-30

	subpd	xmm1, xmm7			;; I1 = I1 - I3 (newer I3)	; 27-29
	mulpd	xmm10, xmm6			;; A4 = R4 * cosine/sine	; 27-31

	addpd	xmm9, xmm2			;; R3 = R1 + R3 (newer R1)	; 28-30	avail 2
	mulpd	xmm15, xmm5			;; A2 = R2 * cosine/sine	; 28-32
	xload	xmm2, [pmreg+pmoff+64+16]	;; cosine/sine 3		; 28	avail none

	addpd	xmm7, xmm8			;; I3 = I1 + I3 (newer I1)	; 29-31	avail 8
	xcopy	xmm8, xmm0			;; Copy R3			; 29-31	avail none
	mulpd	xmm0, xmm2			;; A3 = R3 * cosine/sine	; 29-33

	subpd	xmm11, xmm6			;; B4 = B4 - R4			; 30-32	avail 6
	mulpd	xmm2, xmm1			;; B3 = I3 * cosine/sine	; 30-34
	xload	xmm6, [pmreg+pmoff+0+16]	;; cosine/sine 1		; 30	avail none

	subpd	xmm4, xmm5			;; B2 = B2 - R2			; 31-33	avail 5
	xcopy	xmm5, xmm9			;; Copy R1			; 31-33	avail none
	mulpd	xmm9, xmm6			;; A1 = R1 * cosine/sine	; 31-35

	addpd	xmm10, xmm14			;; A4 = A4 + I4			; 32-34	avail 14
	mulpd	xmm6, xmm7			;; B1 = I1 * cosine/sine	; 32-36
	xload	xmm14, [pmreg+pmoff+96]		;; sine 4			; 32	avail none

	addpd	xmm15, xmm3			;; A2 = A2 + I2			; 33-35	avail 3
	mulpd	xmm11, xmm14			;; B4 = B4 * sine (final I4)	; 33-37	avail 3 storable 11
	xload	xmm3, [pmreg+pmoff+32]		;; sine 2			; 33	avail none storable 11

	addpd	xmm0, xmm1			;; A3 = A3 + I3			; 34-36	avail 1 storable 11
	mulpd	xmm4, xmm3			;; B2 = B2 * sine (final I2)	; 34-38	avail 1 storable 11,4
	xload	xmm1, [srcreg+d2+d1]		;;#2 R4				; 34	avail none storable 11,4

	subpd	xmm2, xmm8			;; B3 = B3 - R3			; 35-37	avail 8 storable 11,4
	mulpd	xmm10, xmm14			;; A4 = A4 * sine (final R4)	; 35-39	avail 8,14 storable 11,4,10
	xload	xmm8, [screg+0+16]		;;#2 cosine/sine 3&4		; 35	avail 14 storable 11,4,10

	addpd	xmm9, xmm7			;; A1 = A1 + I1			; 36-38	avail 14,7 storable 11,4,10
	mulpd	xmm15, xmm3			;; A2 = A2 * sine (final R2)	; 36-40	avail 14,7,3 storable 11,4,10,15
	xload	xmm7, [srcreg+d2]		;;#2 R3				; 36	avail 14,3 storable 11,4,10,15

	subpd	xmm6, xmm5			;; B1 = B1 - R1			; 37-39	avail 14,3,5 storable 11,4,10,15
	xcopy	xmm3, xmm1			;;#2 Copy R4			; 37-39	avail 14,5 storable 11,4,10,15
	mulpd	xmm1, xmm8			;;#2 A4 = R4 * cosine/sine	; 37-41
	xload	xmm14, [srcreg+d2+d1+32]	;;#2 I4				; 37	avail 5 storable 11,4,10,15

	xcopy	xmm5, xmm7			;;#2 Copy R3			; 38-40	avail none storable 11,4,10,15
	mulpd	xmm7, xmm8			;;#2 A3 = R3 * cosine/sine	; 38-42
	xstore	[srcreg+d2+d1+48], xmm11	;; Save I4			; 38	avail 11 storable 4,10,15

	xcopy	xmm11, xmm14			;;#2 Copy I4			; 39-41	avail none storable 4,10,15
	mulpd	xmm14, xmm8			;;#2 B4 = I4 * cosine/sine	; 39-43
	xstore	[srcreg+d2+d1+32], xmm4		;; Save I2			; 39 (read in at clock 36) avail 4 storable 10,15

	xload	xmm4, [srcreg+d2+32]		;;#2 I3				; 40	avail none storable 10,15
	mulpd	xmm8, xmm4			;;#2 B3 = I3 * cosine/sine	; 40-44
	xstore	[srcreg+d2+d1+16], xmm10	;; Save R4			; 40	avail 10 storable 15

	xload	xmm10, [pmreg+pmoff+64]		;; sine 3			; 41	avail none storable 15
	mulpd	xmm0, xmm10			;; A3 = A3 * sine (final R3)	; 41-45	avail none storable 15,0
	xstore	[srcreg+d2+d1], xmm15		;; Save R2			; 41 (read in at clock 34) avail 15 storable 0

	subpd	xmm1, xmm11			;;#2 A4 = A4 - I4		; 42-44	avail 15,11 storable 0
	mulpd	xmm2, xmm10			;; B3 = B3 * sine (final I3)	; 42-46	avail 15,11,10 storable 0.2
	xload	xmm11, [pmreg+pmoff+0]		;; sine 1			; 42	avail 15,10 storable 0,2

	addpd	xmm7, xmm4			;;#2 A3 = A3 + I3		; 43-45	avail 15,10,4 storable 0,2
	mulpd	xmm9, xmm11			;; A1 = A1 * sine (final R1)	; 43-47	avail 15,10,4 storable 0,2,9
	xload	xmm4, [screg+0]			;;#2 sine 3&4			; 43	avail 15,10 storable 0,2,9

	addpd	xmm14, xmm3			;;#2 B4 = B4 + R4		; 44-46	avail 15,10,3 storable 0,2,9
	mulpd	xmm6, xmm11			;; B1 = B1 * sine (final I1)	; 44-48	avail 15,10,3,11 storable 0,2,9,6
	xcopy	xmm11, xmm12			;;#2 Copy R2			; 44-46 (20) avail 15,10,3 storable 0,2,9,6
	xload	xmm3, [srcreg]			;;#2 R1				; 44	avail 15,10 storable 0,2,9,6

	subpd	xmm8, xmm5			;;#2 B3 = B3 - R3		; 45-47	avail 15,10,5 storable 0,2,9,6
	mulpd	xmm1, xmm4			;;#2 A4 = A4 * sine (R4)	; 45-49
	xload	xmm10, [srcreg+32]		;;#2 I1				; 45	avail 15,5 storable 0,2,9,6

	addpd	xmm12, xmm3			;;#2 R2 = R1 + R2 (new R1)	; 46-48
	mulpd	xmm7, xmm4			;;#2 A3 = A3 * sine (R3)	; 46-50
	xcopy	xmm5, xmm13			;;#2 Copy I2			; 46-48 (21) avail 15 storable 0,2,9,6
	xstore	[srcreg+d1+16], xmm0		;; Save R3			; 46	avail 15,0 storable 2,9,6

	subpd	xmm3, xmm11			;;#2 R1 = R1 - R2 (new R2)	; 47-49	avail 15,0,11 storable 2,9,6
	mulpd	xmm14, xmm4			;;#2 B4 = B4 * sine (I4)	; 47-51
	xstore	[srcreg+d1+48], xmm2		;; Save I3			; 47	avail 15,0,11,2 storable 9,6

	addpd	xmm13, xmm10			;;#2 I2 = I1 + I2 (new I1)	; 48-50
	mulpd	xmm8, xmm4			;;#2 B3 = B3 * sine (I3)	; 48-52	avail 15,0,11,2,4 storable 9,6
	xstore	[srcreg+d1], xmm9		;; Save R1			; 48 (read in at clock 7) avail 15,0,11,2,4,9 storable 6

	subpd	xmm10, xmm5			;;#2 I1 = I1 - I2 (new I2)	; 49-51	avail 15,0,11,2,4,9,5 storable 6
	xstore	[srcreg+d1+32], xmm6		;; Save I1			; 49 (read at clock 8) avail 15,0,11,2,4,9,5,6

	xprefetchw [srcreg+srcinc]

	xcopy	xmm0, xmm1			;;#2 Copy R4			; 50-52 (50 on Core i7) avail 15,11,2,4,9,5,6
										; 50 STALL

	subpd	xmm1, xmm7			;;#2 R4 = R4 - R3 (new I4)	; 51-52

	addpd	xmm7, xmm0			;;#2 R3 = R4 + R3 (new R3)	; 52-53	avail 15,11,2,4,9,5,6,0
	xcopy	xmm2, xmm14			;;#2 Copy I4			; 52-54 (52 on Core i7) avail 15,11,4,9,5,6,0

	addpd	xmm14, xmm8			;;#2 I4 = I3 + I4 (new I3)	; 53-55
	xcopy	xmm4, xmm10			;;#2 Copy I2			; 53-55 (52) avail 15,11,9,5,6,0

	subpd	xmm8, xmm2			;;#2 I3 = I3 - I4 (new R4)	; 54-56	avail 15,11,9,5,6,0,2
	xload	xmm6, [pmreg+96+16]		;;#2 cosine/sine 4		; 54	avail 15,11,9,5,0,2

	xprefetchw [srcreg+srcinc+d1]

	subpd	xmm10, xmm1			;;#2 I2 = I2 - I4 (newer I4)	; 55-57
	xcopy	xmm5, xmm3			;;#2 Copy R2			; 55-57 (50) avail 15,11,9,0,2
	xload	xmm0, [pmreg+32+16]		;;#2 cosine/sine 2		; 55	avail 15,11,9,2

	addpd	xmm1, xmm4			;;#2 I4 = I2 + I4 (newer I2)	; 56-58	avail 15,11,9,2,4
	xcopy	xmm9, xmm12			;;#2 Copy R1			; 56-58 (49) avail 15,11,2,4

	subpd	xmm3, xmm8			;;#2 R2 = R2 - R4 (newer R4)	; 57-59
	xcopy	xmm11, xmm13			;;#2 Copy I1			; 59-61 (52) avail 15,2,4

	addpd	xmm8, xmm5			;;#2 R4 = R2 + R4 (newer R2)	; 58-60	avail 15,2,4,5
	xcopy	xmm2, xmm10			;;#2 Copy I4			; 58-60	avail 15,4,5
	mulpd	xmm10, xmm6			;;#2 B4 = I4 * cosine/sine	; 58-62

	subpd	xmm12, xmm7			;;#2 R1 = R1 - R3 (newer R3)	; 59-61
	xcopy	xmm4, xmm1			;;#2 Copy I2			; 59-61	avail 15,5
	mulpd	xmm1, xmm0			;;#2 B2 = I2 * cosine/sine	; 59-63

	subpd	xmm13, xmm14			;;#2 I1 = I1 - I3 (newer I3)	; 60-62
	mulpd	xmm6, xmm3			;;#2 A4 = R4 * cosine/sine	; 60-64
	xload	xmm5, [pmreg+64+16]		;;#2 cosine/sine 3		; 60	avail 15

	addpd	xmm7, xmm9			;;#2 R3 = R1 + R3 (newer R1)	; 61-63	avail 15,9
	mulpd	xmm0, xmm8			;;#2 A2 = R2 * cosine/sine	; 61-65
	xload	xmm15, [pmreg+0+16]		;;#2 cosine/sine 1		; 61	avail 9

	addpd	xmm14, xmm11			;;#2 I3 = I1 + I3 (newer I1)	; 62-64	avail 9,11
	xcopy	xmm9, xmm12			;;#2 Copy R3			; 62-64	avail 11
	mulpd	xmm12, xmm5			;;#2 A3 = R3 * cosine/sine	; 62-66

	subpd	xmm10, xmm3			;;#2 B4 = B4 - R4		; 63-65	avail 11,3
	mulpd	xmm5, xmm13			;;#2 B3 = I3 * cosine/sine	; 63-67
	xload	xmm11, [pmreg+96]		;;#2 sine 4			; 63	avail 3

	subpd	xmm1, xmm8			;;#2 B2 = B2 - R2		; 64-66	avail 3,8
	xcopy	xmm3, xmm7			;;#2 Copy R1			; 64-66	avail 8
	mulpd	xmm7, xmm15			;;#2 A1 = R1 * cosine/sine	; 64-68

	addpd	xmm6, xmm2			;;#2 A4 = A4 + I4		; 65-67	avail 8,2
	mulpd	xmm15, xmm14			;;#2 B1 = I1 * cosine/sine	; 65-69
	xload	xmm2, [pmreg+32]		;;#2 sine 2			; 65	avail 8

	addpd	xmm0, xmm4			;;#2 A2 = A2 + I2		; 66-68	avail 8,4
	mulpd	xmm10, xmm11			;;#2 B4 = B4 * sine (final I4)	; 66-70	avail 8,4 storable 10

	addpd	xmm12, xmm13			;;#2 A3 = A3 + I3		; 67-69	avail 8,4,13 storable 10
	mulpd	xmm1, xmm2			;;#2 B2 = B2 * sine (final I2)	; 67-71	avail 8,4,13 storable 10,1
	xload	xmm4, [pmreg+64]		;;#2 sine 3			; 67	avail 8,13 storable 10,1

	subpd	xmm5, xmm9			;;#2 B3 = B3 - R3		; 68-70	avail 8,13,9 storable 10,1
	mulpd	xmm6, xmm11			;;#2 A4 = A4 * sine (final R4)	; 68-72	avail 8,13,9,11 storable 10,1,6
	xload	xmm8, [pmreg+0]			;;#2 sine 1			; 68

	addpd	xmm7, xmm14			;;#2 A1 = A1 + I1		; 69-71	avail 13,9,11,14 storable 10,1,6
	mulpd	xmm0, xmm2			;;#2 A2 = A2 * sine (final R2)	; 69-73	avail 13,9,11,14,2 storable 10,1,6,0

	subpd	xmm15, xmm3			;;#2 B1 = B1 - R1		; 70-72	avail 13,9,11,14,2,3 storable 10,1,6,0
	mulpd	xmm12, xmm4			;;#2 A3 = A3 * sine (final R3)	; 70-74	avail 13,9,11,14,2,3 storable 10,1,6,0,12

	mulpd	xmm5, xmm4			;;#2 B3 = B3 * sine (final I3)	; 71-75	avail 13,9,11,14,2,3,4 storable 10,1,6,0,12,5
	xstore	[srcreg+d2+48], xmm10		;;#2 Save I4			; 71	avail 13,9,11,14,2,3,4,10 storable 1,6,0,12,5

	mulpd	xmm7, xmm8			;;#2 A1 = A1 * sine (final R1)	; 72-76	avail 13,9,11,14,2,3,4,10 storable 1,6,0,12,5,7
	xstore	[srcreg+d2+32], xmm1		;;#2 Save I2			; 72	avail 13,9,11,14,2,3,4,10,1 storable 6,0,12,5,7

	mulpd	xmm15, xmm8			;;#2 B1 = B1 * sine (final I1)	; 73-77	avail 13,9,11,14,2,3,4,10,1,8 storable 6,0,12,5,7,15
	xstore	[srcreg+d2+16], xmm6		;;#2 Save R4			; 73

	xstore	[srcreg+d2], xmm0		;;#2 Save R2			; 74
	xstore	[srcreg+16], xmm12		;;#2 Save R3			; 75
	xstore	[srcreg+48], xmm5		;;#2 Save I3			; 76
	xstore	[srcreg], xmm7			;;#2 Save R1			; 77
	xstore	[srcreg+32], xmm15		;;#2 Save I1			; 78

	bump	srcreg, srcinc
	ENDM

ENDIF

;;
;; ******************************* four-complex-with-partial-normalization variants *************************************
;;
;; These macros are used in pass 1 of r4dwpn two pass FFTs.  They are like the standard four-complex
;; DJBFFT macros except that a normalization multiplier has been pre-applied to the sine multiplier.
;; Consequently, the forward FFT and inverse FFT use different sine multipliers.
;; Also, a normalization multiplier must be applied to the final R1/I1 value.
;;

r4_x4cl_wpn_four_complex_djbfft_preload MACRO
	r4_x4c_wpn_djbfft_mem_preload
	; r4_x4c_wpn_djbfft_partial_mem_preload -- assume same as r4_x4c_wpn_djbfft_mem_preload
	ENDM

r4_x4cl_wpn_four_complex_djbfft MACRO srcreg,srcinc,d1,d2,screg
	d3 = d2 + d1
	r4_x4c_wpn_djbfft_mem [srcreg],[srcreg+d1],[srcreg+d2],[srcreg+d2+d1],[srcreg+16],[srcreg+d1+16],[srcreg+d2+16],[srcreg+d2+d1+16],screg+32,screg+80,screg,srcreg+srcinc,d1,[srcreg],[srcreg+16]
;;	xstore	[srcreg], xmm0		;; Save R1
;;	xstore	[srcreg+16], xmm0	;; Save I1
	xload	xmm0, [srcreg+32]	;; R1
	xload	xmm5, [srcreg+48]	;; R5
	xstore	[srcreg+32], xmm7	;; Save R2
	xstore	[srcreg+48], xmm6	;; Save I2
	xload	xmm7, [srcreg+d1+32]	;; R2
	xload	xmm6, [srcreg+d1+48]	;; R6
	xstore	[srcreg+d1], xmm3	;; Save R3
	xstore	[srcreg+d1+16], xmm1	;; Save I3
	xstore	[srcreg+d1+32], xmm2	;; Save R4
	xstore	[srcreg+d1+48], xmm4	;; Save I4
	r4_x4c_wpn_djbfft_partial_mem xmm0,xmm7,xmm3,xmm2,xmm5,xmm6,xmm1,xmm4,[srcreg+d2+32],[srcreg+d3+32],[srcreg+d2+48],[srcreg+d3+48],screg+32,screg+80,screg,srcreg+srcinc+d2,d1,[srcreg+d2],[srcreg+d2+16]
;;	xstore	[srcreg+d2], xmm0	;; Save R1
;;	xstore	[srcreg+d2+16], xmm0	;; Save I1
	xstore	[srcreg+d2+32], xmm4	;; Save R2
	xstore	[srcreg+d2+48], xmm1	;; Save I2
	xstore	[srcreg+d2+d1], xmm2	;; Save R3
	xstore	[srcreg+d2+d1+16], xmm7	;; Save I3
	xstore	[srcreg+d2+d1+32], xmm0	;; Save R4
	xstore	[srcreg+d2+d1+48], xmm5	;; Save I4
	bump	srcreg, srcinc
	ENDM

r4_x4c_wpn_djbfft_mem_preload MACRO
	ENDM

r4_x4c_wpn_djbfft_mem MACRO R1,R2,R3,R4,R5,R6,R7,R8,screg1,screg2,normreg,pre1,pre2,dst1,dst2
	xload	xmm0, R1		;; R1
	xload	xmm2, R3		;; R3
	addpd	xmm2, xmm0		;; R3 = R1 + R3 (new R1)

	xload	xmm1, R2		;; R2
	xload	xmm3, R4		;; R4
	addpd	xmm3, xmm1		;; R4 = R2 + R4 (new R2)

	xload	xmm4, R5		;; I1
	xload	xmm6, R7		;; I3
	addpd	xmm6, xmm4		;; I3 = I1 + I3 (new I1)

	 subpd	xmm0, R3		;; R1 = R1 - R3 (new R3)
	 subpd	xmm1, R4		;; R2 = R2 - R4 (new R4)

	xprefetchw [pre1]

	xcopy	xmm5, xmm3		;; Copy R2
	addpd	xmm3, xmm2		;; R2 = R1 + R2 (final R1)
	 subpd	xmm2, xmm5		;; R1 = R1 - R2 (final R2)

	xload	xmm5, R6		;; I2
	xload	xmm7, R8		;; I4
	addpd	xmm7, xmm5		;; I4 = I2 + I4 (new I2)

	 subpd	xmm4, R7		;; I1 = I1 - I3 (new I3)
	 subpd	xmm5, R8		;; I2 = I2 - I4 (new I4)

	 mulpd	xmm3, [normreg]		;; Apply normalization multiplier to R1
	xstore	dst1, xmm3		;; Save R1

	xcopy	xmm3, xmm7		;; Copy I2
	addpd	xmm7, xmm6		;; I2 = I1 + I2 (final I1)

	 subpd	xmm6, xmm3		;; I1 = I1 - I2 (final I2)

	 mulpd	xmm7, [normreg]		;; Apply normalization multiplier to I1
	xstore	dst2, xmm7		;; Save I1

	xcopy	xmm3, xmm1
	addpd	xmm1, xmm4		;; R4 = I3 + R4 (final I3)

	xcopy	xmm7, xmm0		;; Copy R3
	subpd	xmm0, xmm5		;; R3 = R3 - I4 (final R3)

	 subpd	xmm4, xmm3		;; I3 = I3 - R4 (final I4)
	addpd	xmm5, xmm7		;; I4 = R3 + I4 (final R4)

	xprefetchw [pre1][pre2]

	xload	xmm7, [screg2+16]	;; cosine/sine
	mulpd	xmm7, xmm2		;; A2 = R2 * cosine/sine
	subpd	xmm7, xmm6		;; A2 = A2 - I2

	xload	xmm3, [screg1+16]	;; cosine/sine
	mulpd	xmm3, xmm0		;; A3 = R3 * cosine/sine
	subpd	xmm3, xmm1		;; A3 = A3 - I3

	mulpd	xmm6, [screg2+16]	;; B2 = I2 * cosine/sine
	addpd	xmm6, xmm2		;; B2 = B2 + R2

	xload	xmm2, [screg1+16]	;; cosine/sine
	mulpd	xmm2, xmm5		;; A4 = R4 * cosine/sine
	addpd	xmm2, xmm4		;; A4 = A4 + I4

	mulpd	xmm1, [screg1+16]	;; B3 = I3 * cosine/sine
	addpd	xmm1, xmm0		;; B3 = B3 + R3

	mulpd	xmm4, [screg1+16]	;; B4 = I4 * cosine/sine
	subpd	xmm4, xmm5		;; B4 = B4 - R4

	mulpd	xmm7, [screg2]		;; A2 = A2 * sine (final R2)
	mulpd	xmm6, [screg2]		;; B2 = B2 * sine (final I2)

	xload	xmm5, [screg1]		;; Sine
	mulpd	xmm3, xmm5		;; A3 = A3 * sine (final R3)
	mulpd	xmm1, xmm5		;; B3 = B3 * sine (final I3)

	mulpd	xmm2, xmm5		;; A4 = A4 * sine (final R4)
	mulpd	xmm4, xmm5		;; B4 = B4 * sine (final I4)
	ENDM

r4_x4c_wpn_djbfft_partial_mem_preload MACRO
	ENDM

r4_x4c_wpn_djbfft_partial_mem MACRO r1,r2,r3,r4,r5,r6,r7,r8,mem3,mem4,mem7,mem8,screg1,screg2,normreg,pre1,pre2,dst1,dst2
	xload	r3, mem3
	addpd	r3, r1			;; R3 = R1 + R3 (new R1)
	subpd	r1, mem3		;; R1 = R1 - R3 (new R3)

	xload	r4, mem4
	addpd	r4, r2			;; R4 = R2 + R4 (new R2)
	subpd	r2, mem4		;; R2 = R2 - R4 (new R4)

	xcopy	r7, r3
	subpd	r3, r4			;; R1 = R1 - R2 (final R2)
	addpd	r4, r7			;; R2 = R1 + R2 (final R1)

	mulpd	r4, [normreg]		;; Apply normalization multiplier to R1
	xstore	dst1, r4		;; Save R1

	xload	r7, mem7
	xload	r8, mem8

	xcopy	r4, r6
	subpd	r6, r8			;; I2 = I2 - I4 (new I4)
	addpd	r8, r4			;; I4 = I2 + I4 (new I2)

	xcopy	r4, r5
	subpd	r5, r7			;; I1 = I1 - I3 (new I3)
	addpd	r7, r4			;; I3 = I1 + I3 (new I1)

	xcopy	r4, r1
	subpd	r1, r6			;; R3 = R3 - I4 (final R3)
	addpd	r6, r4			;; I4 = R3 + I4 (final R4)

	xcopy	r4, r2
	addpd	r2, r5			;; R4 = I3 + R4 (final I3)
	subpd	r5, r4			;; I3 = I3 - R4 (final I4)

	xprefetchw [pre1]

	xcopy	r4, r7
	subpd	r7, r8			;; I1 = I1 - I2 (final I2)
	addpd	r8, r4			;; I2 = I1 + I2 (final I1)

	xload	r4, [screg1+16]		;; cosine/sine
	mulpd	r4, r1			;; A3 = R3 * cosine/sine

	mulpd	r8, [normreg]		;; Apply normalization multiplier to I1
	xstore	dst2, r8		;; Save I1

	xload	r8, [screg2+16]		;; cosine/sine
	mulpd	r8, r3			;; A2 = R2 * cosine/sine

	subpd	r4, r2			;; A3 = A3 - I3
	mulpd	r2, [screg1+16]		;; B3 = I3 * cosine/sine
	addpd	r2, r1			;; B3 = B3 + R3

	xload	r1, [screg1+16]		;; cosine/sine
	mulpd	r1, r6			;; A4 = R4 * cosine/sine

	subpd	r8, r7			;; A2 = A2 - I2
	mulpd	r7, [screg2+16]		;; B2 = I2 * cosine/sine

	xprefetchw [pre1][pre2]

	addpd	r1, r5			;; A4 = A4 + I4
	mulpd	r5, [screg1+16]		;; B4 = I4 * cosine/sine

	addpd	r7, r3			;; B2 = B2 + R2
	subpd	r5, r6			;; B4 = B4 - R4

	xload	r3, [screg1]		;; sine
	mulpd	r4, r3			;; A3 = A3 * sine (final R3)
	mulpd	r2, r3			;; B3 = B3 * sine (final I3)
	mulpd	r8, [screg2]		;; A2 = A2 * sine (final R2)
	mulpd	r1, r3			;; A4 = A4 * sine (final R4)
	mulpd	r7, [screg2]		;; B2 = B2 * sine (final I2)
	mulpd	r5, r3			;; B4 = B4 * sine (final I4)
	ENDM

;; 32-bit AMD K8 optimized versions of the above macros

IF (@INSTR(,%xarch,<K8>) NE 0)

r4_x4c_wpn_djbfft_mem MACRO R1,R2,R3,R4,R5,R6,R7,R8,screg1,screg2,normreg,pre1,pre2,dst1,dst2
	xload	xmm0, R1		;; R1
	xload	xmm2, R3		;; R3
	addpd	xmm2, xmm0		;; R3 = R1 + R3 (new R1)

	xload	xmm1, R2		;; R2
	xload	xmm3, R4		;; R4
	addpd	xmm3, xmm1		;; R4 = R2 + R4 (new R2)

	xload	xmm4, R5		;; I1
	xload	xmm6, R7		;; I3
	addpd	xmm6, xmm4		;; I3 = I1 + I3 (new I1)

	 subpd	xmm0, R3		;; R1 = R1 - R3 (new R3)
	 subpd	xmm1, R4		;; R2 = R2 - R4 (new R4)

	xprefetchw [pre1]

	addpd	xmm3, xmm2		;; R2 = R1 + R2 (final R1)
	multwo	xmm2
	 subpd	xmm2, xmm3		;; R1 = R1 - R2 (final R2)

	xload	xmm5, R6		;; I2
	xload	xmm7, R8		;; I4
	addpd	xmm7, xmm5		;; I4 = I2 + I4 (new I2)

	 subpd	xmm4, R7		;; I1 = I1 - I3 (new I3)
	 subpd	xmm5, R8		;; I2 = I2 - I4 (new I4)

	 mulpd	xmm3, [normreg]		;; Apply normalization multiplier to R1
	xstore	dst1, xmm3		;; Save R1

	addpd	xmm7, xmm6		;; I2 = I1 + I2 (final I1)
	multwo	xmm6
	 subpd	xmm6, xmm7		;; I1 = I1 - I2 (final I2)

	 mulpd	xmm7, [normreg]		;; Apply normalization multiplier to I1
	xstore	dst2, xmm7		;; Save I1

	addpd	xmm1, xmm4		;; R4 = I3 + R4 (final I3)
	multwo	xmm4

	subpd	xmm0, xmm5		;; R3 = R3 - I4 (final R3)
	multwo	xmm5
	 subpd	xmm4, xmm1		;; I3 = I3 - R4 (final I4)
	addpd	xmm5, xmm0		;; I4 = R3 + I4 (final R4)

	xprefetchw [pre1][pre2]

	xload	xmm7, [screg2+16]	;; cosine/sine
	mulpd	xmm7, xmm2		;; A2 = R2 * cosine/sine
	subpd	xmm7, xmm6		;; A2 = A2 - I2

	xload	xmm3, [screg1+16]	;; cosine/sine
	mulpd	xmm3, xmm0		;; A3 = R3 * cosine/sine
	subpd	xmm3, xmm1		;; A3 = A3 - I3

	mulpd	xmm6, [screg2+16]	;; B2 = I2 * cosine/sine
	addpd	xmm6, xmm2		;; B2 = B2 + R2

	xload	xmm2, [screg1+16]	;; cosine/sine
	mulpd	xmm2, xmm5		;; A4 = R4 * cosine/sine
	addpd	xmm2, xmm4		;; A4 = A4 + I4

	mulpd	xmm1, [screg1+16]	;; B3 = I3 * cosine/sine
	addpd	xmm1, xmm0		;; B3 = B3 + R3

	mulpd	xmm4, [screg1+16]	;; B4 = I4 * cosine/sine
	subpd	xmm4, xmm5		;; B4 = B4 - R4

	mulpd	xmm7, [screg2]		;; A2 = A2 * sine (final R2)
	mulpd	xmm6, [screg2]		;; B2 = B2 * sine (final I2)

	xload	xmm5, [screg1]		;; Sine
	mulpd	xmm3, xmm5		;; A3 = A3 * sine (final R3)
	mulpd	xmm1, xmm5		;; B3 = B3 * sine (final I3)

	mulpd	xmm2, xmm5		;; A4 = A4 * sine (final R4)
	mulpd	xmm4, xmm5		;; B4 = B4 * sine (final I4)
	ENDM

r4_x4c_wpn_djbfft_partial_mem MACRO r1,r2,r3,r4,r5,r6,r7,r8,mem3,mem4,mem7,mem8,screg1,screg2,normreg,pre1,pre2,dst1,dst2
	xload	r3, mem3
	addpd	r3, r1			;; R3 = R1 + R3 (new R1)
	subpd	r1, mem3		;; R1 = R1 - R3 (new R3)

	xload	r4, mem4
	addpd	r4, r2			;; R4 = R2 + R4 (new R2)
	subpd	r2, mem4		;; R2 = R2 - R4 (new R4)

	subpd	r3, r4			;; R1 = R1 - R2 (final R2)
	multwo	r4
	addpd	r4, r3			;; R2 = R1 + R2 (final R1)

	mulpd	r4, [normreg]		;; Apply normalization multiplier to R1
	xstore	dst1, r4		;; Save R1

	xload	r7, mem7
	xload	r8, mem8

	subpd	r6, r8			;; I2 = I2 - I4 (new I4)
	multwo	r8
	addpd	r8, r6			;; I4 = I2 + I4 (new I2)

	subpd	r5, r7			;; I1 = I1 - I3 (new I3)
	multwo	r7
	addpd	r7, r5			;; I3 = I1 + I3 (new I1)

	subpd	r1, r6			;; R3 = R3 - I4 (final R3)
	multwo	r6
	addpd	r6, r1			;; I4 = R3 + I4 (final R4)

	addpd	r2, r5			;; R4 = I3 + R4 (final I3)
	multwo	r5
	subpd	r5, r2			;; I3 = I3 - R4 (final I4)

	xprefetchw [pre1]

	subpd	r7, r8			;; I1 = I1 - I2 (final I2)
	multwo	r8
	addpd	r8, r7			;; I2 = I1 + I2 (final I1)

	xload	r4, [screg1+16]		;; cosine/sine
	mulpd	r4, r1			;; A3 = R3 * cosine/sine

	mulpd	r8, [normreg]		;; Apply normalization multiplier to I1
	xstore	dst2, r8		;; Save I1

	xload	r8, [screg2+16]		;; cosine/sine
	mulpd	r8, r3			;; A2 = R2 * cosine/sine

	subpd	r4, r2			;; A3 = A3 - I3
	mulpd	r2, [screg1+16]		;; B3 = I3 * cosine/sine
	addpd	r2, r1			;; B3 = B3 + R3

	xload	r1, [screg1+16]		;; cosine/sine
	mulpd	r1, r6			;; A4 = R4 * cosine/sine

	subpd	r8, r7			;; A2 = A2 - I2
	mulpd	r7, [screg2+16]		;; B2 = I2 * cosine/sine

	xprefetchw [pre1][pre2]

	addpd	r1, r5			;; A4 = A4 + I4
	mulpd	r5, [screg1+16]		;; B4 = I4 * cosine/sine

	addpd	r7, r3			;; B2 = B2 + R2
	subpd	r5, r6			;; B4 = B4 - R4

	xload	r3, [screg1]		;; sine
	mulpd	r4, r3			;; A3 = A3 * sine (final R3)
	mulpd	r2, r3			;; B3 = B3 * sine (final I3)
	mulpd	r8, [screg2]		;; A2 = A2 * sine (final R2)
	mulpd	r1, r3			;; A4 = A4 * sine (final R4)
	mulpd	r7, [screg2]		;; B2 = B2 * sine (final I2)
	mulpd	r5, r3			;; B4 = B4 * sine (final I4)
	ENDM

ENDIF

;; 64-bit Intel implementations of the above - use the extra XMM registers

IF (@INSTR(,%xarch,<CORE>) NE 0) OR (@INSTR(,%xarch,<P4>) NE 0) OR (@INSTR(,%xarch,<BLEND>) NE 0) OR (@INSTR(,%xarch,<K10>) NE 0)
IFDEF X86_64

; Theoretical best case is 44 clocks on a Core 2.  Now at ?? clocks.

r4_x4cl_wpn_four_complex_djbfft MACRO srcreg,srcinc,d1,d2,screg
	xload	xmm0, [srcreg]		;; R1
	xload	xmm2, [srcreg+d2]	;; R3
	xload	xmm5, [srcreg+d1+16]	;; I2
	xload	xmm7, [srcreg+d2+d1+16]	;; I4
	xload	xmm1, [srcreg+d1]	;; R2
	xload	xmm3, [srcreg+d2+d1]	;; R4
	xload	xmm4, [srcreg+16]	;; I1
	xload	xmm6, [srcreg+d2+16]	;; I3

	xcopy	xmm8, xmm0		;; Copy R1				; 1-3
	subpd	xmm0, xmm2		;; R1 = R1 - R3 (new R3)		; 1-3

	xcopy	xmm9, xmm5		;; Copy I2				; 2-4
	subpd	xmm5, xmm7		;; I2 = I2 - I4 (new I4)		; 2-4

	xcopy	xmm10, xmm1		;; Copy R2				; 3-5
	subpd	xmm1, xmm3		;; R2 = R2 - R4 (new R4)		; 3-5

	xcopy	xmm11, xmm4		;; Copy I1				; 4-6
	subpd	xmm4, xmm6		;; I1 = I1 - I3 (new I3)		; 4-6	avail 12+
	xload	xmm12, [screg+32+16]	;; cosine/sine				; 4

	addpd	xmm2, xmm8		;; R3 = R1 + R3 (new R1)		; 5-7	avail 8,13+
	xcopy	xmm13, xmm12							; 5-7

	addpd	xmm3, xmm10		;; R4 = R2 + R4 (new R2)		; 6-8	avail 8,10,14+
	xcopy	xmm14, xmm12							; 6-8

	addpd	xmm7, xmm9		;; I4 = I2 + I4 (new I2)		; 7-9	avail 8.10.9,15
 	 xcopy	xmm8, xmm5		;; Copy I4				; 7-9

	addpd	xmm6, xmm11		;; I3 = I1 + I3 (new I1)		; 8-10	avail 10.9,15,11
	xcopy	xmm15, xmm12							; 8-10

	 addpd	xmm5, xmm0		;; I4 = R3 + I4 (final R4)		; 9-11	avail 10.9,11
	 xcopy	xmm9, xmm4		;; Copy I3				; 9-11
	xload	xmm10, [screg+80+16]	;; cosine/sine				; 9

	 subpd	xmm0, xmm8		;; R3 = R3 - I4 (final R3)		; 10-12	avail 11,8

	xprefetchw [srcreg+srcinc]

	 subpd	xmm4, xmm1		;; I3 = I3 - R4 (final I4)		; 11-13	avail 11,8
	xcopy	xmm11, xmm10							; 11-13

	 addpd	xmm1, xmm9		;; R4 = I3 + R4 (final I3)		; 12-14	avail 8,9
	mulpd	xmm12, xmm5		;; A4 = R4 * cosine/sine		; 12-16
	 xcopy	xmm8, xmm2		;; Copy R1				; 12-14

	 subpd	xmm2, xmm3		;; R1 = R1 - R2 (final R2)		; 13-15	avail 9
	mulpd	xmm13, xmm0		;; A3 = R3 * cosine/sine		; 13-17
	 xcopy	xmm9, xmm6		;; Copy I1				; 13-15

	 subpd	xmm6, xmm7		;; I1 = I1 - I2 (final I2)		; 14-16	avail none
	mulpd	xmm14, xmm4		;; B4 = I4 * cosine/sine		; 14-18

	 addpd	xmm3, xmm8		;; R2 = R1 + R2 (final R1)		; 15-17	avail 8
	mulpd	xmm15, xmm1		;; B3 = I3 * cosine/sine		; 15-19
	xload	xmm8, [screg+32]	;; Sine * normalization value		; 15

	 addpd	xmm7, xmm9		;; I2 = I1 + I2 (final I1)		; 16-18	avail 9
	mulpd	xmm10, xmm2		;; A2 = R2 * cosine/sine		; 16-20
	xload	xmm9, [screg+80]	;; Sine * normalization value		; 16

	addpd	xmm12, xmm4		;; A4 = A4 + I4				; 17-19	avail 4
	mulpd	xmm11, xmm6		;; B2 = I2 * cosine/sine		; 17-21
	xload	xmm4, [srcreg+32]	;;#2 R1					; 17

	subpd	xmm13, xmm1		;; A3 = A3 - I3				; 18-20	avail 1
	mulpd	xmm3, [screg]		;; Apply normalization to final R1	; 18-22
	xload	xmm1, [srcreg+d2+32]	;; next R3

	xprefetchw [srcreg+srcinc+d1]

	subpd	xmm14, xmm5		;; B4 = B4 - R4				; 19-21	avail 5
	mulpd	xmm7, [screg]		;; Apply normalization to final I1	; 19-23
	xstore	[srcreg], xmm3		;; Save R1				; 23	avail 3,5
	xload	xmm3, [srcreg+d1+48]	;;#2 I2					; 19

	addpd	xmm15, xmm0		;; B3 = B3 + R3				; 20-22	avail 5,0
	mulpd	xmm12, xmm8		;; A4 = A4 * sine (final R4)		; 20-24
	xload	xmm5, [srcreg+d2+d1+48]	;;#2 I4

	subpd	xmm10, xmm6		;; A2 = A2 - I2				; 21-23	avail 0,6
	mulpd	xmm13, xmm8		;; A3 = A3 * sine (final R3)		; 21-25
	xstore	[srcreg+16], xmm7	;; Save I1				; 24	avail 0,6,7
	xload	xmm7, [srcreg+d1+32]	;;#2 R2

	addpd	xmm11, xmm2		;; B2 = B2 + R2				; 22-24	avail 0,6,2
	mulpd	xmm14, xmm8		;; B4 = B4 * sine (final I4)		; 22-26
	xload	xmm0, [srcreg+d2+d1+32]	;;#2 R4

	xcopy	xmm6, xmm4		;;#2 Copy R1				; 23-25	avail 2
	subpd	xmm4, xmm1		;;#2 R1 = R1 - R3 (new R3)		; 23-25
	mulpd	xmm15, xmm8		;; B3 = B3 * sine (final I3)		; 23-27	avail 2,8

	xcopy	xmm2, xmm3		;;#2 Copy I2				; 24-26	avail 8
	subpd	xmm3, xmm5		;;#2 I2 = I2 - I4 (new I4)		; 24-26
	mulpd	xmm10, xmm9		;; A2 = A2 * sine (final R2)		; 24-28

	xcopy	xmm8, xmm7		;;#2 Copy R2				; 25-27	avail none
	subpd	xmm7, xmm0		;;#2 R2 = R2 - R4 (new R4)		; 25-27
	mulpd	xmm11, xmm9		;; B2 = B2 * sine (final I2)		; 25-29	avail 9
	xload	xmm9, [srcreg+48]	;;#2 I1					; 25
	xstore	[srcreg+d1+32], xmm12	;; Save R4				; 25	avail 12

	xload	xmm12, [srcreg+d2+48]	;;#2 I3					; 26	avail none
	xstore	[srcreg+d1], xmm13	;; Save R3				; 26	avail 13
	xcopy	xmm13, xmm9		;;#2 Copy I1				; 26-28	avail none
	subpd	xmm9, xmm12		;;#2 I1 = I1 - I3 (new I3)		; 26-28

	addpd	xmm1, xmm6		;;#2 R3 = R1 + R3 (new R1)		; 27-29	avail 6
	xstore	[srcreg+d1+48], xmm14	;; Save I4				; 27	avail 6,14
	xload	xmm14, [screg+32+16]	;;#2 cosine/sine			; 27	avail 6

	addpd	xmm0, xmm8		;;#2 R4 = R2 + R4 (new R2)		; 28-30	avail 6,8
	xstore	[srcreg+d1+16], xmm15	;; Save I3				; 28	avail 6,8,15
	xcopy	xmm15, xmm14							; 28-30	avail 6,8

	addpd	xmm5, xmm2		;;#2 I4 = I2 + I4 (new I2)		; 29-31	avail 6,8,2
	xstore	[srcreg+32], xmm10	;; Save R2				; 29	avail 6,8,2,10
 	 xcopy	xmm6, xmm3		;;#2 Copy I4				; 29-31	avail 8,2,10

	addpd	xmm12, xmm13		;;#2 I3 = I1 + I3 (new I1)		; 30-32	avail 8,2,10,13
	xstore	[srcreg+48], xmm11	;; Save I2				; 30	avail 8,2,10,13,11
	xcopy	xmm10, xmm14							; 30-32	avail 8,2,13,11

	xprefetchw [srcreg+srcinc+d2]

	 addpd	xmm3, xmm4		;;#2 I4 = R3 + I4 (final R4)		; 31-33
	xcopy	xmm11, xmm14							; 31-33	avail 8,2,13
	 xcopy	xmm2, xmm9		;;#2 Copy I3				; 31-33	avail 8,13

	 subpd	xmm4, xmm6		;;#2 R3 = R3 - I4 (final R3)		; 32-34	avail 8,13,6
	xload	xmm8, [screg+80+16]	;;#2 cosine/sine			; 32	avail 13,6

	 subpd	xmm9, xmm7		;;#2 I3 = I3 - R4 (final I4)		; 33-35
	xcopy	xmm13, xmm8							; 33-35	avail 6

	 addpd	xmm7, xmm2		;;#2 R4 = I3 + R4 (final I3)		; 34-36	avail 6,2
	mulpd	xmm14, xmm3		;;#2 A4 = R4 * cosine/sine		; 34-38
	 xcopy	xmm6, xmm1		;;#2 Copy R1				; 34-36	avail 2

	 subpd	xmm1, xmm0		;;#2 R1 = R1 - R2 (final R2)		; 35-37
	mulpd	xmm15, xmm4		;;#2 A3 = R3 * cosine/sine		; 35-39
	 xcopy	xmm2, xmm12		;;#2 Copy I1				; 35-37	avail none

	 subpd	xmm12, xmm5		;;#2 I1 = I1 - I2 (final I2)		; 36-38
	mulpd	xmm10, xmm9		;;#2 B4 = I4 * cosine/sine		; 36-40

	 addpd	xmm0, xmm6		;;#2 R2 = R1 + R2 (final R1)		; 37-39	avail 6
	mulpd	xmm11, xmm7		;;#2 B3 = I3 * cosine/sine		; 37-41
	xload	xmm6, [screg+32]	;;#2 Sine * normalization value		; 37	avail none

	 addpd	xmm5, xmm2		;;#2 I2 = I1 + I2 (final I1)		; 38-40	avail 2
	mulpd	xmm8, xmm1		;;#2 A2 = R2 * cosine/sine		; 38-42
	xload	xmm2, [screg+80]	;;#2 Sine * normalization value		; 38	avail none

	addpd	xmm14, xmm9		;;#2 A4 = A4 + I4			; 39-41	avail 9
	mulpd	xmm13, xmm12		;;#2 B2 = I2 * cosine/sine		; 39-43

	subpd	xmm15, xmm7		;;#2 A3 = A3 - I3			; 40-42	avail 9,7
	mulpd	xmm0, [screg]		;;#2 Apply normalization to final R1	; 40-44

	xprefetchw [srcreg+srcinc+d2+d1]

	subpd	xmm10, xmm3		;;#2 B4 = B4 - R4			; 41-43	avail 9,7,3
	mulpd	xmm5, [screg]		;;#2 Apply normalization to final I1	; 41-45

	addpd	xmm11, xmm4		;;#2 B3 = B3 + R3			; 42-44	avail 9,7,3,4
	mulpd	xmm14, xmm6		;;#2 A4 = A4 * sine (final R4)		; 42-46

	subpd	xmm8, xmm12		;;#2 A2 = A2 - I2			; 43-45	avail 9,7,3,4,12
	mulpd	xmm15, xmm6		;;#2 A3 = A3 * sine (final R3)		; 43-47

	addpd	xmm13, xmm1		;;#2 B2 = B2 + R2			; 44-46	avail 9,7,3,4,12,1
	mulpd	xmm10, xmm6		;;#2 B4 = B4 * sine (final I4)		; 44-48

	mulpd	xmm11, xmm6		;;#2 B3 = B3 * sine (final I3)		; 45-49
	xstore	[srcreg+d2], xmm0	;;#2 Save R1				; 45

	mulpd	xmm8, xmm2		;;#2 A2 = A2 * sine (final R2)		; 46-50
	xstore	[srcreg+d2+16], xmm5	;;#2 Save I1				; 46

	mulpd	xmm13, xmm2		;;#2 B2 = B2 * sine (final I2)		; 47-51
	xstore	[srcreg+d2+d1+32], xmm14 ;;#2 Save R4				; 47

	xstore	[srcreg+d2+d1], xmm15	;;#2 Save R3				; 48
	xstore	[srcreg+d2+d1+48], xmm10 ;;#2 Save I4				; 49
	xstore	[srcreg+d2+d1+16], xmm11 ;;#2 Save I3				; 50
	xstore	[srcreg+d2+32], xmm8	;;#2 Save R2				; 51
	xstore	[srcreg+d2+48], xmm13	;;#2 Save I2				; 52

	bump	srcreg, srcinc
	ENDM
ENDIF
ENDIF

;; 64-bit AMD K8 optimized versions of the above macros

IF (@INSTR(,%xarch,<K8>) NE 0)
IFDEF X86_64

r4_x4c_wpn_djbfft_mem_preload MACRO
	xload	xmm15, XMM_TWO
	ENDM

r4_x4c_wpn_djbfft_mem MACRO R1,R2,R3,R4,R5,R6,R7,R8,screg1,screg2,normreg,pre1,pre2,dst1,dst2
	xload	xmm0, R1		;; R1
	xload	xmm2, R3		;; R3
	addpd	xmm2, xmm0		;; R3 = R1 + R3 (new R1)

	xload	xmm1, R2		;; R2
	xload	xmm3, R4		;; R4
	addpd	xmm3, xmm1		;; R4 = R2 + R4 (new R2)

	xload	xmm4, R5		;; I1
	xload	xmm6, R7		;; I3
	addpd	xmm6, xmm4		;; I3 = I1 + I3 (new I1)

	 subpd	xmm0, R3		;; R1 = R1 - R3 (new R3)
	 subpd	xmm1, R4		;; R2 = R2 - R4 (new R4)

	xprefetchw [pre1]

	addpd	xmm3, xmm2		;; R2 = R1 + R2 (final R1)
	mulpd	xmm2, xmm15
	 subpd	xmm2, xmm3		;; R1 = R1 - R2 (final R2)

	xload	xmm5, R6		;; I2
	xload	xmm7, R8		;; I4
	addpd	xmm7, xmm5		;; I4 = I2 + I4 (new I2)

	 subpd	xmm4, R7		;; I1 = I1 - I3 (new I3)
	 subpd	xmm5, R8		;; I2 = I2 - I4 (new I4)

	 mulpd	xmm3, [normreg]		;; Apply normalization multiplier to R1
	xstore	dst1, xmm3		;; Save R1

	addpd	xmm7, xmm6		;; I2 = I1 + I2 (final I1)
	mulpd	xmm6, xmm15
	 subpd	xmm6, xmm7		;; I1 = I1 - I2 (final I2)

	 mulpd	xmm7, [normreg]		;; Apply normalization multiplier to I1
	xstore	dst2, xmm7		;; Save I1

	addpd	xmm1, xmm4		;; R4 = I3 + R4 (final I3)
	mulpd	xmm4, xmm15

	subpd	xmm0, xmm5		;; R3 = R3 - I4 (final R3)
	mulpd	xmm5, xmm15
	 subpd	xmm4, xmm1		;; I3 = I3 - R4 (final I4)
	addpd	xmm5, xmm0		;; I4 = R3 + I4 (final R4)

	xprefetchw [pre1][pre2]

	xload	xmm7, [screg2+16]	;; cosine/sine
	mulpd	xmm7, xmm2		;; A2 = R2 * cosine/sine
	subpd	xmm7, xmm6		;; A2 = A2 - I2

	xload	xmm3, [screg1+16]	;; cosine/sine
	mulpd	xmm3, xmm0		;; A3 = R3 * cosine/sine
	subpd	xmm3, xmm1		;; A3 = A3 - I3

	mulpd	xmm6, [screg2+16]	;; B2 = I2 * cosine/sine
	addpd	xmm6, xmm2		;; B2 = B2 + R2

	xload	xmm2, [screg1+16]	;; cosine/sine
	mulpd	xmm2, xmm5		;; A4 = R4 * cosine/sine
	addpd	xmm2, xmm4		;; A4 = A4 + I4

	mulpd	xmm1, [screg1+16]	;; B3 = I3 * cosine/sine
	addpd	xmm1, xmm0		;; B3 = B3 + R3

	mulpd	xmm4, [screg1+16]	;; B4 = I4 * cosine/sine
	subpd	xmm4, xmm5		;; B4 = B4 - R4

	mulpd	xmm7, [screg2]		;; A2 = A2 * sine (final R2)
	mulpd	xmm6, [screg2]		;; B2 = B2 * sine (final I2)

	xload	xmm5, [screg1]		;; Sine
	mulpd	xmm3, xmm5		;; A3 = A3 * sine (final R3)
	mulpd	xmm1, xmm5		;; B3 = B3 * sine (final I3)

	mulpd	xmm2, xmm5		;; A4 = A4 * sine (final R4)
	mulpd	xmm4, xmm5		;; B4 = B4 * sine (final I4)
	ENDM

r4_x4c_wpn_djbfft_partial_mem_preload MACRO
	xload	xmm15, XMM_TWO
	ENDM

r4_x4c_wpn_djbfft_partial_mem MACRO r1,r2,r3,r4,r5,r6,r7,r8,mem3,mem4,mem7,mem8,screg1,screg2,normreg,pre1,pre2,dst1,dst2
	xload	r3, mem3
	addpd	r3, r1			;; R3 = R1 + R3 (new R1)
	subpd	r1, mem3		;; R1 = R1 - R3 (new R3)

	xload	r4, mem4
	addpd	r4, r2			;; R4 = R2 + R4 (new R2)
	subpd	r2, mem4		;; R2 = R2 - R4 (new R4)

	subpd	r3, r4			;; R1 = R1 - R2 (final R2)
	mulpd	r4, xmm15
	addpd	r4, r3			;; R2 = R1 + R2 (final R1)

	mulpd	r4, [normreg]		;; Apply normalization multiplier to R1
	xstore	dst1, r4		;; Save R1

	xload	r7, mem7
	xload	r8, mem8

	subpd	r6, r8			;; I2 = I2 - I4 (new I4)
	mulpd	r8, xmm15
	addpd	r8, r6			;; I4 = I2 + I4 (new I2)

	subpd	r5, r7			;; I1 = I1 - I3 (new I3)
	mulpd	r7, xmm15
	addpd	r7, r5			;; I3 = I1 + I3 (new I1)

	subpd	r1, r6			;; R3 = R3 - I4 (final R3)
	mulpd	r6, xmm15
	addpd	r6, r1			;; I4 = R3 + I4 (final R4)

	addpd	r2, r5			;; R4 = I3 + R4 (final I3)
	mulpd	r5, xmm15
	subpd	r5, r2			;; I3 = I3 - R4 (final I4)

	xprefetchw [pre1]

	subpd	r7, r8			;; I1 = I1 - I2 (final I2)
	mulpd	r8, xmm15
	addpd	r8, r7			;; I2 = I1 + I2 (final I1)

	xload	r4, [screg1+16]		;; cosine/sine
	mulpd	r4, r1			;; A3 = R3 * cosine/sine

	mulpd	r8, [normreg]		;; Apply normalization multiplier to I1
	xstore	dst2, r8		;; Save I1

	xload	r8, [screg2+16]		;; cosine/sine
	mulpd	r8, r3			;; A2 = R2 * cosine/sine

	subpd	r4, r2			;; A3 = A3 - I3
	mulpd	r2, [screg1+16]		;; B3 = I3 * cosine/sine
	addpd	r2, r1			;; B3 = B3 + R3

	xload	r1, [screg1+16]		;; cosine/sine
	mulpd	r1, r6			;; A4 = R4 * cosine/sine

	subpd	r8, r7			;; A2 = A2 - I2
	mulpd	r7, [screg2+16]		;; B2 = I2 * cosine/sine

	xprefetchw [pre1][pre2]

	addpd	r1, r5			;; A4 = A4 + I4
	mulpd	r5, [screg1+16]		;; B4 = I4 * cosine/sine

	addpd	r7, r3			;; B2 = B2 + R2
	subpd	r5, r6			;; B4 = B4 - R4

	xload	r3, [screg1]		;; sine
	mulpd	r4, r3			;; A3 = A3 * sine (final R3)
	mulpd	r2, r3			;; B3 = B3 * sine (final I3)
	mulpd	r8, [screg2]		;; A2 = A2 * sine (final R2)
	mulpd	r1, r3			;; A4 = A4 * sine (final R4)
	mulpd	r7, [screg2]		;; B2 = B2 * sine (final I2)
	mulpd	r5, r3			;; B4 = B4 * sine (final I4)
	ENDM

ENDIF
ENDIF

r4_x4cl_wpn_four_complex_djbunfft_preload MACRO
	r4_x4c_wpn_djbunfft_mem_preload
	;r4_x4c_wpn_djbunfft_partial_mem_preload -- assume same as r4_x4c_wpn_djbunfft_mem_preload
	ENDM

r4_x4cl_wpn_four_complex_djbunfft MACRO srcreg,srcinc,d1,d2,screg,scoff
	d3 = d2 + d1
	r4_x4c_wpn_djbunfft_mem [srcreg+16],[srcreg+48],[srcreg+d1+16],[srcreg+d1+48],[srcreg+d2+16],[srcreg+d2+48],[srcreg+d3+16],[srcreg+d3+48],[srcreg+d1+16],screg+32+scoff,screg+scoff,srcreg+srcinc+d2,d1
;;	xstore	[srcreg+d1+16], xmm6	;; Save R3
	xstore	[srcreg+d1+48], xmm3	;; Save I3
	xload	xmm6, [srcreg+d1]	;; R3
	xload	xmm3, [srcreg+d1+32]	;; R4
	xstore	[srcreg+d3+16], xmm0	;; Save R4
	xstore	[srcreg+d3+48], xmm1	;; Save I4
	xload	xmm0, [srcreg+d3]	;; R7
	xload	xmm1, [srcreg+d3+32]	;; R8
	xstore	[srcreg+d1], xmm7	;; Save R1
	xstore	[srcreg+d1+32], xmm4	;; Save I1
	xstore	[srcreg+d3], xmm5	;; Save R2
	xstore	[srcreg+d3+32], xmm2	;; Save I2
	r4_x4c_wpn_djbunfft_partial_mem xmm7,xmm4,xmm6,xmm3,xmm5,xmm2,xmm0,xmm1,[srcreg],[srcreg+32],[srcreg+d2],[srcreg+d2+32],[srcreg],screg+32,screg,srcreg+srcinc,d1
;;	xstore	[srcreg], xmm4		;; Save R1
	xstore	[srcreg+32], xmm5	;; Save I1
	xstore	[srcreg+16], xmm6	;; Save R3
	xstore	[srcreg+48], xmm7	;; Save I3
	xstore	[srcreg+d2], xmm3	;; Save R2
	xstore	[srcreg+d2+32], xmm0	;; Save I2
	xstore	[srcreg+d2+16], xmm2	;; Save R4
	xstore	[srcreg+d2+48], xmm1	;; Save I4
	bump	srcreg, srcinc
	ENDM

r4_x4c_wpn_djbunfft_mem_preload MACRO
	ENDM

r4_x4c_wpn_djbunfft_mem MACRO mem1,mem2,mem3,mem4,mem5,mem6,mem7,mem8,dst3,screg,normreg,pre1,pre2
	xload	xmm3, [screg+48+16]	;; cosine/sine
	xload	xmm6, mem3		;; R2
	mulpd	xmm6, xmm3		;; A2 = R2 * cosine/sine
	xload	xmm0, mem4		;; I2
	mulpd	xmm3, xmm0		;; B2 = I2 * cosine/sine

	xload	xmm5, [screg+16]	;; cosine/sine
	xload	xmm7, mem5		;; R3
	mulpd	xmm7, xmm5		;; A3 = R3 * cosine/sine
	xload	xmm1, mem6		;; I3
	xcopy	xmm4, xmm5		;; cosine/sine
	mulpd	xmm5, xmm1		;; B3 = I3 * cosine/sine

	xload	xmm2, mem7		;; R4
	mulpd	xmm2, xmm4		;; A4 = R4 * cosine/sine
	addpd	xmm6, xmm0		;; A2 = A2 + I2
	xload	xmm0, mem8		;; I4
	mulpd	xmm4, xmm0		;; B4 = I4 * cosine/sine

	subpd	xmm3, mem3		;; B2 = B2 - R2
	addpd	xmm7, xmm1		;; A3 = A3 + I3
	subpd	xmm5, mem5		;; B3 = B3 - R3
	subpd	xmm2, xmm0		;; A4 = A4 - I4
	addpd	xmm4, mem7		;; B4 = B4 + R4

	xload	xmm0, mem1		;; R1
	mulpd	xmm0, [normreg+16]	;; R1 * normalization_inverse

	mulpd	xmm6, [screg+48+32]	;; A2 = A2 * sine*normalization_inverse (new R2)
	mulpd	xmm3, [screg+48+32]	;; B2 = B2 * sine*normalization_inverse (new I2)
	xload	xmm1, [screg+32]	;; Sine * normalization_inverse
	mulpd	xmm7, xmm1		;; A3 = A3 * sine*normalization_inverse (new R3)
	mulpd	xmm5, xmm1		;; B3 = B3 * sine*normalization_inverse (new I3)
	mulpd	xmm2, xmm1		;; A4 = A4 * sine*normalization_inverse (new R4)
	mulpd	xmm4, xmm1		;; B4 = B4 * sine*normalization_inverse (new I4)

	xprefetchw [pre1]

	xcopy	xmm1, xmm0		;; Copy R1
	subpd	xmm0, xmm6		;; R1 = R1 - R2 (new R2)
	addpd	xmm6, xmm1		;; R2 = R1 + R2 (new R1)

	xcopy	xmm1, xmm2		;; Copy R4
	subpd	xmm2, xmm7		;; R4 = R4 - R3 (new I4)
	addpd	xmm7, xmm1		;; R3 = R4 + R3 (new R3)

	xcopy	xmm1, xmm5		;; Copy I3
	subpd	xmm5, xmm4		;; I3 = I3 - I4 (new R4)
	addpd	xmm4, xmm1		;; I4 = I3 + I4 (new I3)

	xcopy	xmm1, xmm6		;; Copy R1
	subpd	xmm6, xmm7		;; R1 = R1 - R3 (final R3)
	addpd	xmm7, xmm1		;; R3 = R1 + R3 (final R1)

	xload	xmm1, mem2		;; I1
	mulpd	xmm1, [normreg+16]	;; I1 * normalization_inverse

	xstore	dst3, xmm6

	xcopy	xmm6, xmm1		;; Copy I1
	subpd	xmm1, xmm3		;; I1 = I1 - I2 (new I2)
	addpd	xmm3, xmm6		;; I2 = I1 + I2 (new I1)

	xprefetchw [pre1][pre2]

	xcopy	xmm6, xmm0		;; Copy R2
	subpd	xmm0, xmm5		;; R2 = R2 - R4 (final R4)
	addpd	xmm5, xmm6		;; R4 = R2 + R4 (final R2)

	xcopy	xmm6, xmm1		;; Copy I2
	subpd	xmm1, xmm2		;; I2 = I2 - I4 (final I4)
	addpd	xmm2, xmm6		;; I4 = I2 + I4 (final I2)

	xcopy	xmm6, xmm3		;; Copy I1
	subpd	xmm3, xmm4		;; I1 = I1 - I3 (final I3)
	addpd	xmm4, xmm6		;; I3 = I1 + I3 (final I1)
	ENDM

r4_x4c_wpn_djbunfft_partial_mem_preload MACRO
	ENDM

r4_x4c_wpn_djbunfft_partial_mem MACRO r1,r2,r3,r4,r5,r6,r7,r8,mem1,mem2,mem5,mem6,dst1,screg,normreg,pre1,pre2
	xload	r1, [screg+48+16]	;; cosine/sine
	xcopy	r2, r3			;; Copy R2
	mulpd	r3, r1			;; A2 = R2 * cosine/sine
	mulpd	r1, r4			;; B2 = I2 * cosine/sine

	xload	r5, [screg+16]		;; cosine/sine
	xcopy	r6, r7			;; Copy R4
	mulpd	r7, r5			;; A4 = R4 * cosine/sine
	addpd	r3, r4			;; A2 = A2 + I2
	xcopy	r4, r5			;; cosine/sine
	mulpd	r5, r8			;; B4 = I4 * cosine/sine
	subpd	r1, r2			;; B2 = B2 - R2

	xload	r2, mem5		;; R3
	mulpd	r2, r4			;; A3 = R3 * cosine/sine
	mulpd	r4, mem6		;; B3 = I3 * cosine/sine

	subpd	r7, r8			;; A4 = A4 - I4
	addpd	r5, r6			;; B4 = B4 + R4
	addpd	r2, mem6		;; A3 = A3 + I3
	subpd	r4, mem5		;; B3 = B3 - R3

	xload	r6, mem1		;; R1
	mulpd	r6, [normreg+16]	;; R1 * normalization_inverse

	mulpd	r3, [screg+48+32]	;; A2 = A2 * sine*normalization_inverse (new R2)
	mulpd	r1, [screg+48+32]	;; B2 = B2 * sine*normalization_inverse (new I2)
	xload	r8, [screg+32]		;; Sine * normalization_inverse
	mulpd	r7, r8			;; A4 = A4 * sine*normalization_inverse (new R4)
	mulpd	r5, r8			;; B4 = B4 * sine*normalization_inverse (new I4)
	mulpd	r2, r8			;; A3 = A3 * sine*normalization_inverse (new R3)
	mulpd	r4, r8			;; B3 = B3 * sine*normalization_inverse (new I3)

	xprefetchw [pre1]

	xcopy	r8, r6			;; Copy R1
	subpd	r6, r3			;; R1 = R1 - R2 (new R2)
	addpd	r3, r8			;; R2 = R1 + R2 (new R1)

	xcopy	r8, r7			;; Copy R4
	subpd	r7, r2			;; R4 = R4 - R3 (new I4)
	addpd	r2, r8			;; R3 = R4 + R3 (new R3)

	xcopy	r8, r4			;; Copy I3
	subpd	r4, r5			;; I3 = I3 - I4 (new R4)
	addpd	r5, r8			;; I4 = I3 + I4 (new I3)

	xcopy	r8, r3			;; Copy R1
	subpd	r3, r2			;; R1 = R1 - R3 (final R3)
	addpd	r2, r8			;; R3 = R1 + R3 (final R1)

	xload	r8, mem2		;; I1
	mulpd	r8, [normreg+16]	;; I1 * normalization_inverse

	xstore	dst1, r2

	xcopy	r2, r8			;; Copy I1
	subpd	r8, r1			;; I1 = I1 - I2 (new I2)
	addpd	r1, r2			;; I2 = I1 + I2 (new I1)

	xprefetchw [pre1][pre2]

	xcopy	r2, r6			;; Copy R2
	subpd	r6, r4			;; R2 = R2 - R4 (final R4)
	addpd	r4, r2			;; R4 = R2 + R4 (final R2)

	xcopy	r2, r8			;; Copy I2
	subpd	r8, r7			;; I2 = I2 - I4 (final I4)
	addpd	r7, r2			;; I4 = I2 + I4 (final I2)

	xcopy	r2, r1			;; Copy I1
	subpd	r1, r5			;; I1 = I1 - I3 (final I3)
	addpd	r5, r2			;; I3 = I1 + I3 (final I1)
	ENDM

;; 32-bit AMD K8 optimized versions of the above macros

IF (@INSTR(,%xarch,<K8>) NE 0)

;; K8 Cheat sheet for scheduling dependency chains
;;	      12345678901234567890123456789012345678901234567890123456789012345678901234567890
;;A2	      MMMMAAAA    MMMM
;;B2	        MMMMAAAA    MMMM
;;A4              MMMMAAAA
;;A3                MMMMAAAA
;;B3		      MMMMAAAA
;;B4		        MMMMAAAA

;;i1			          MMMM
;;r1		                    MMMM

;;i4(r4-r3)                   AAAA        MMMM
;;r3(r4+r3)                   MMMMAAAA        MMMM
;;r4(i3-i4)                     AAAA        MMMM
;;i3(i3+i4)                     MMMMAAAA        MMMM
;;i2(i1-i2)                           AAAA
;;i1(i1+i2)                           MMMMAAAA
;;r2(r1-r2)                             AAAA
;;r1(r1+r2)                             MMMMAAAA

;;I4(I2-I4)	                              AAAA
;;I2(I2+I4)			                  MMMMAAAA
;;R4(R2-R4)	                                AAAA			
;;R2(R2+R4)			                    MMMMAAAA
;;R3(R1-R3)	                                  AAAA
;;R1(R1+R3)	                                      MMMMAAAA
;;I3(I1-I3)	                                    AAAA				
;;I1(I1+I3)                                             MMMMAAAA

r4_x4c_wpn_djbunfft_mem MACRO mem1,mem2,mem3,mem4,mem5,mem6,mem7,mem8,dst3,screg,normreg,pre1,pre2
	xload	xmm3, [screg+48+16]	;; cosine/sine
	xload	xmm6, mem3		;; R2
	mulpd	xmm6, xmm3		;; A2 = R2 * cosine/sine	; 1-4

	xload	xmm0, mem4		;; I2
	mulpd	xmm3, xmm0		;; B2 = I2 * cosine/sine	; 3-6

	xload	xmm4, [screg+16]	;; cosine/sine
	xload	xmm2, mem7		;; R4
	mulpd	xmm2, xmm4		;; A4 = R4 * cosine/sine	; 5-8
	addpd	xmm6, xmm0		;; A2 = A2 + I2			; 5-8

	xload	xmm5, [screg+16]	;; cosine/sine
	xload	xmm7, mem5		;; R3
	mulpd	xmm7, xmm5		;; A3 = R3 * cosine/sine	; 7-10
	subpd	xmm3, mem3		;; B2 = B2 - R2			; 7-10

	xload	xmm1, mem6		;; I3
	mulpd	xmm5, xmm1		;; B3 = I3 * cosine/sine	; 9-12
	xload	xmm0, mem8		;; I4
	subpd	xmm2, xmm0		;; A4 = A4 - I4			; 9-12

	mulpd	xmm4, xmm0		;; B4 = I4 * cosine/sine	; 11-14
	addpd	xmm7, xmm1		;; A3 = A3 + I3			; 11-14

	subpd	xmm5, mem5		;; B3 = B3 - R3			; 13-16
	mulpd	xmm6, [screg+48+32]	;; A2 = A2 * sine*normalization_inverse (new R2) ; 13-16

	addpd	xmm4, mem7		;; B4 = B4 + R4			; 15-18
	mulpd	xmm3, [screg+48+32]	;; B2 = B2 * sine*normalization_inverse (new I2) ; 15-18

	subpd	xmm2, xmm7		;; R4 = R4 - R3 (new I4)	; 17-20
	multwo	xmm7							; 17-20

	xprefetchw [pre1]

	subpd	xmm5, xmm4		;; I3 = I3 - I4 (new R4)	; 19-22
	multwo	xmm4							; 19-22

	xload	xmm1, mem2		;; I1
	addpd	xmm7, xmm2		;; R3 = R4 + R3 (new R3)	; 21-24
	mulpd	xmm1, [normreg+16]	;; I1 * normalization_inverse	; 21-24

	xload	xmm0, mem1		;; R1
	addpd	xmm4, xmm5		;; I4 = I3 + I4 (new I3)	; 23-26
	mulpd	xmm0, [normreg+16]	;; R1 * normalization_inverse	; 23-26

	subpd	xmm1, xmm3		;; I1 = I1 - I2 (new I2)	; 25-28
	multwo	xmm3							; 25-28

	subpd	xmm0, xmm6		;; R1 = R1 - R2 (new R2)	; 27-30
	multwo	xmm6							; 27-30

	xprefetchw [pre1][pre2]

	addpd	xmm3, xmm1		;; I2 = I1 + I2 (new I1)	; 29-32
	mulpd	xmm2, [screg+32]	;; I4 = I4 * sine*normalization_inverse

	addpd	xmm6, xmm0		;; R2 = R1 + R2 (new R1)	; 31-34
	mulpd	xmm5, [screg+32]	;; R4 = R4 * sine*normalization_inverse

	subpd	xmm1, xmm2		;; I2 = I2 - I4 (final I4)	; 33-36
	mulpd	xmm7, [screg+32]	;; R3 = R3 * sine*normalization_inverse

	subpd	xmm0, xmm5		;; R2 = R2 - R4 (final R4)	; 35-38
	mulpd	xmm4, [screg+32]	;; I3 = I3 * sine*normalization_inverse

	subpd	xmm6, xmm7		;; R1 = R1 - R3 (final R3)	; 37-40
	multwo	xmm2

	subpd	xmm3, xmm4		;; I1 = I1 - I3 (final I3)	; 39-42
	multwo	xmm5

	addpd	xmm2, xmm1		;; I4 = I2 + I4 (final I2)	; 41-44
	multwo	xmm7

	addpd	xmm5, xmm0		;; R4 = R2 + R4 (final R2)	; 43-46
	multwo	xmm4

	addpd	xmm7, xmm6		;; R3 = R1 + R3 (final R1)	; 45-48
	xstore	dst3, xmm6						; 41

	addpd	xmm4, xmm3		;; I3 = I1 + I3 (final I1)	; 47-50
	ENDM

r4_x4c_wpn_djbunfft_partial_mem MACRO r1,r2,r3,r4,r5,r6,r7,r8,mem1,mem2,mem5,mem6,dst1,screg,normreg,pre1,pre2
	xload	r1, [screg+48+16]	;; cosine/sine
	mulpd	r1, r4			;; B2 = I2 * cosine/sine	; 1-4

	subpd	r1, r3			;; B2 = B2 - R2			; 5-8
	mulpd	r3, [screg+48+16]	;; A2 = R2 * cosine/sine	; 3-6
	xload	r5, [screg+16]		;; cosine/sine

	xload	r2, mem5		;; R3
	mulpd	r5, r8			;; B4 = I4 * cosine/sine	; 5-8

	addpd	r3, r4			;; A2 = A2 + I2			; 7-10
	xload	r4, [screg+16]		;; cosine/sine
	mulpd	r2, r4			;; A3 = R3 * cosine/sine	; 7-10

	addpd	r5, r7			;; B4 = B4 + R4			; 9-12
	mulpd	r7, [screg+16]		;; A4 = R4 * cosine/sine	; 9-12

	addpd	r2, mem6		;; A3 = A3 + I3			; 11-14
	mulpd	r4, mem6		;; B3 = I3 * cosine/sine	; 11-14

	subpd	r7, r8			;; A4 = A4 - I4			; 13-16
	mulpd	r1, [screg+48+32]	;; B2 = B2 * sine*normalization_inverse (new I2)

	subpd	r4, mem5		;; B3 = B3 - R3			; 15-18
	mulpd	r3, [screg+48+32]	;; A2 = A2 * sine*normalization_inverse (new R2)

	subpd	r7, r2			;; R4 = R4 - R3 (new I4)	; 17-20
	multwo	r2							; 17-20

	xprefetchw [pre1]

	subpd	r4, r5			;; I3 = I3 - I4 (new R4)	; 19-22
	multwo	r5							; 19-22

	xload	r8, mem2		;; I1
	addpd	r2, r7			;; R3 = R4 + R3 (new R3)	; 21-24
	mulpd	r8, [normreg+16]	;; I1 * normalization_inverse	; 21-24

	xload	r6, mem1		;; R1
	addpd	r5, r4			;; I4 = I3 + I4 (new I3)	; 23-26
	mulpd	r6, [normreg+16]	;; R1 * normalization_inverse	; 23-26

	subpd	r8, r1			;; I1 = I1 - I2 (new I2)	; 25-28
	multwo	r1							; 25-28

	xprefetchw [pre1][pre2]

	subpd	r6, r3			;; R1 = R1 - R2 (new R2)	; 27-30
	multwo	r3							; 27-30

	addpd	r1, r8			;; I2 = I1 + I2 (new I1)	; 29-32
	mulpd	r7, [screg+32]		;; I4 = I4 * sine*normalization_inverse

	addpd	r3, r6			;; R2 = R1 + R2 (new R1)	; 31-34
	mulpd	r4, [screg+32]		;; R4 = R4 * sine*normalization_inverse

	subpd	r8, r7			;; I2 = I2 - I4 (final I4)	; 33-36
	mulpd	r2, [screg+32]		;; R3 = R3 * sine*normalization_inverse

	subpd	r6, r4			;; R2 = R2 - R4 (final R4)	; 35-38
	mulpd	r5, [screg+32]		;; I3 = I3 * sine*normalization_inverse

	subpd	r3, r2			;; R1 = R1 - R3 (final R3)	; 37-40
	multwo	r7

	subpd	r1, r5			;; I1 = I1 - I3 (final I3)	; 39-42
	multwo	r4

	addpd	r7, r8			;; I4 = I2 + I4 (final I2)	; 41-44
	multwo	r2

	addpd	r4, r6			;; R4 = R2 + R4 (final R2)	; 43-46
	multwo	r5

	addpd	r2, r3			;; R3 = R1 + R3 (final R1)	; 45-48

	addpd	r5, r1			;; I3 = I1 + I3 (final I1)	; 47-50

	xstore	dst1, r2
	ENDM

ENDIF

;; 64-bit Intel and AMD K10 implementations of the above - use the extra XMM registers

IF (@INSTR(,%xarch,<CORE>) NE 0) OR (@INSTR(,%xarch,<P4>) NE 0) OR (@INSTR(,%xarch,<BLEND>) NE 0) OR (@INSTR(,%xarch,<K10>) NE 0)
IFDEF X86_64

; Theoretical best case is 44 clocks on a Core 2.  Now at ?? clocks.
r4_x4cl_wpn_four_complex_djbunfft MACRO srcreg,srcinc,d1,d2,screg,scoff
	xload	xmm0, [srcreg+d2+d1+16]	;; R4
	xload	xmm1, [screg+scoff+32+16] ;; cosine/sine
	xcopy	xmm2, xmm0		;; Copy R4				; 1-3
	mulpd	xmm0, xmm1		;; A4 = R4 * cosine/sine		; 1-5

	xload	xmm5, [srcreg+d1+48]	;; I2
	xload	xmm6, [screg+scoff+80+16] ;; cosine/sine
	xcopy	xmm7, xmm5		;; Copy I2				; 2-4
	mulpd	xmm5, xmm6		;; B2 = I2 * cosine/sine		; 2-6

	xload	xmm3, [srcreg+d2+16]	;; R3
	xcopy	xmm4, xmm3		;; Copy R3				; 3-5
	mulpd	xmm3, xmm1		;; A3 = R3 * cosine/sine		; 3-7

	xload	xmm8, [srcreg+d1+16]	;; R2
	xcopy	xmm9, xmm8		;; Copy R2				; 4-6
	mulpd	xmm8, xmm6		;; A2 = R2 * cosine/sine		; 4-8	avail 6,10+

	xload	xmm6, [srcreg+d2+48]	;; I3
	xcopy	xmm10, xmm6		;; Copy I3				; 5-7	avail 11+
	mulpd	xmm6, xmm1		;; B3 = I3 * cosine/sine		; 5-9

	xload	xmm11, [srcreg+d2+d1+48];; I4
	subpd	xmm0, xmm11		;; A4 = A4 - I4 (new R4 / sine)		; 6-8
	mulpd	xmm11, xmm1		;; B4 = I4 * cosine/sine		; 6-10	avail 1,12+

	subpd	xmm5, xmm9		;; B2 = B2 - R2				; 7-9	avail 1,9,12+
	xload	xmm1, [screg+scoff+80+32] ;; sine * normalization_inverse	; 7

	addpd	xmm3, xmm10		;; A3 = A3 + I3 (new R3 / sine)		; 8-10	avail 9,10,12+

	addpd	xmm8, xmm7		;; A2 = A2 + I2				; 9-11	avail 9,10,7,12+
	xload	xmm7, [srcreg+48]	;; I1					; 9
	mulpd	xmm7, [screg+scoff+16]	;; I1 * normalization_inverse		; 9-13

	subpd	xmm6, xmm4		;; B3 = B3 - R3 (new I3 / sine)		; 10-12	avail 9,10,4,12+
	mulpd	xmm5, xmm1		;; B2 = B2 * sine (new I2)		; 10-14
	xcopy	xmm4, xmm0		;; Copy R4 / sine			; 10-12	avail 9,10,12+

	addpd	xmm11, xmm2		;; B4 = B4 + R4 (new I4 / sine)		; 11-13	avail 9,10,2,12+
	xload	xmm2, [srcreg+16]	;; R1					; 11	avail 9,10,12+
	mulpd	xmm2, [screg+scoff+16]	;; R1 * normalization_inverse		; 11-15

	subpd	xmm0, xmm3		;; R4 = R4 - R3 (newer I4 / sine)	; 12-14
	mulpd	xmm8, xmm1		;; A2 = A2 * sine (new R2)		; 12-16	avail 9,10,1,12+
	xload	xmm1, [screg+scoff+32+32] ;; sine * normalization_inverse	; 12	avail 9,10,12+

	addpd	xmm3, xmm4		;; R3 = R4 + R3 (newer R3 / sine)	; 13-15	avail 9,10,4,12+
	xcopy	xmm4, xmm6		;; Copy I3 / sine			; 13-15

	xprefetchw [srcreg+srcinc+d2]

	subpd	xmm6, xmm11		;; I3 = I3 - I4 (newer R4 / sine)	; 14-16
	xcopy	xmm10, xmm7		;; Copy I1				; 14-16 avail 9,12+
	xload	xmm9, [srcreg+d2+d1]	;;#2 R4					; 14	avail 12+

	subpd	xmm7, xmm5		;; I1 = I1 - I2 (newer I2)		; 15-17
	mulpd	xmm0, xmm1		;; newer I4 * sine			; 15-19
	xload	xmm12, [screg+32+16]	;;#2 cosine/sine			; 15	avail 13+

	addpd	xmm11, xmm4		;; I4 = I3 + I4 (newer I3 / sine)	; 16-18 avail 4,13+
	mulpd	xmm3, xmm1		;; newer R3 * sine			; 16-20
	xcopy	xmm4, xmm2		;; Copy R1				; 16-18	avail 13+

	addpd	xmm5, xmm10		;; I2 = I1 + I2 (newer I1)		; 17-19	avail 10,13+
	mulpd	xmm6, xmm1		;; newer R4 * sine			; 17-21
	xload	xmm13, [srcreg+d1+32]	;;#2 I2					; 17	avail 10,14+

	addpd	xmm2, xmm8		;; R2 = R1 + R2 (newer R1)		; 18-20
	xload	xmm14, [screg+80+16]	;;#2 cosine/sine			; 18	avail 10,15

	subpd	xmm4, xmm8		;; R1 = R1 - R2 (newer R2)		; 19-21	avail 10,8,15
	mulpd	xmm11, xmm1		;; newer I3 * sine			; 19-23	avail 10,8,1,15
	xcopy	xmm1, xmm7		;; Copy I2				; 19-21	avail 10,8,15

	xprefetchw [srcreg+srcinc+d2+d1]

	subpd	xmm7, xmm0		;; I2 = I2 - I4 (final I4)		; 20-22	avail 10,8,15 storable 7
	xcopy	xmm15, xmm9		;;#2 Copy R4				; 20-22	avail 10,8 storable 7

	xcopy	xmm8, xmm2		;; Copy R1				; 21-23	avail 10 storable 7
	subpd	xmm2, xmm3		;; R1 = R1 - R3 (final R3)		; 21-23	avail 10 storable 7,2

	addpd	xmm0, xmm1		;; I4 = I2 + I4 (final I2)		; 22-24	avail 10,1 storable 7,2,0
	xcopy	xmm1, xmm4		;; Copy R2				; 22-24	avail 10 storable 7,2,0

	subpd	xmm4, xmm6		;; R2 = R2 - R4 (final R4)		; 23-25	avail 10 storable 7,2,0,4
	mulpd	xmm9, xmm12		;;#2 A4 = R4 * cosine/sine		; 23-27
	xcopy	xmm10, xmm13		;;#2 Copy I2				; 23-25	avail none storable 7,2,0,4
	xstore	[srcreg+d2+d1+48], xmm7	;; Save I4				; 23	avail 7 storable 2,0,4

	addpd	xmm3, xmm8		;; R3 = R1 + R3 (final R1)		; 24-26	avail 7,8 storable 2,0,4,3
	mulpd	xmm13, xmm14		;;#2 B2 = I2 * cosine/sine		; 24-28
	xcopy	xmm7, xmm5		;; Copy I1				; 24-26	avail 8 storable 2,0,4,3
	xstore	[srcreg+d1+16], xmm2	;; Save R3				; 24	avail 8,2 storable 0,4,3

	addpd	xmm6, xmm1		;; R4 = R2 + R4 (final R2)		; 25-27	avail 8,2,1 storable 0,4,3,6
	xload	xmm2, [srcreg+d2]	;;#2 R3					; 25	avail 8,1 storable 0,4,3,6
	xcopy	xmm8, xmm2		;;#2 Copy R3				; 25-27	avail 1 storable 0,4,3,6
	mulpd	xmm2, xmm12		;;#2 A3 = R3 * cosine/sine		; 25-29

	subpd	xmm5, xmm11		;; I1 = I1 - I3 (final I3)		; 26-28	avail 1 storable 0,4,3,6,5
	xload	xmm1, [srcreg+d1]	;;#2 R2					; 26	avail none storable 0,4,3,6,5
	xstore	[srcreg+d2+d1+16], xmm4	;; Save R4				; 26	avail 4 storable 0,3,6,5
	xcopy	xmm4, xmm1		;;#2 Copy R2				; 26-28	avail none storable 0,3,6,5
	mulpd	xmm1, xmm14		;;#2 A2 = R2 * cosine/sine		; 26-30	avail 14 storable 0,3,6,5

	addpd	xmm11, xmm7		;; I3 = I1 + I3 (final I1)		; 27-29	avail 14,7 storable 0,3,6,5,11
	xload	xmm14, [srcreg+d2+32]	;;#2 I3					; 27	avail 7 storable 0,3,6,5,11
	xcopy	xmm7, xmm14		;;#2 Copy I3				; 27-29	avail none storable 0,3,6,5,11
	mulpd	xmm14, xmm12		;;#2 B3 = I3 * cosine/sine		; 27-31
	xstore	[srcreg+d1], xmm3	;; Save R1				; 27	avail 3 storable 0,6,5,11

	xload	xmm3, [srcreg+d2+d1+32] ;;#2 I4					; 28	avail none storable 0,6,5,11
	subpd	xmm9, xmm3		;;#2 A4 = A4 - I4 (new R4 / sine)	; 28-30
	mulpd	xmm3, xmm12		;;#2 B4 = I4 * cosine/sine		; 28-32	avail 12 storable 0,6,5,11
	xstore	[srcreg+d2+d1], xmm6	;; Save R2				; 28	avail 12,6 storable 0,5,11

	subpd	xmm13, xmm4		;;#2 B2 = B2 - R2			; 29-31	avail 12,6,4 storable 0,5,11
	xload	xmm6, [screg+80+32]	;;#2 sine * normalization_inverse	; 29	avail 12,4 storable 0,5,11
	xstore	[srcreg+d1+48], xmm5	;; Save I3				; 29	avail 12,4,5 storable 0,11

	addpd	xmm2, xmm7		;;#2 A3 = A3 + I3 (new R3 / sine)	; 30-32	avail 12,4,5,7 storable 0,11
	xload	xmm5, [srcreg+32]	;;#2 I1					; 30	avail 12,4,7 storable 0,11
	mulpd	xmm5, [screg+16]	;;#2 I1 * normalization_inverse		; 30-34
	xstore	[srcreg+d1+32], xmm11	;; Save I1				; 30	avail 12,4,7,11 storable 0

	addpd	xmm1, xmm10		;;#2 A2 = A2 + I2			; 31-33	avail 12,4,7,11,10 storable 0
	xstore	[srcreg+d2+d1+32], xmm0	;; Save I2				; 25	avail 12,4,7,11,10,0
	xload	xmm7, [srcreg]		;;#2 R1					; 31	avail 12,4,11,10,0

	subpd	xmm14, xmm8		;;#2 B3 = B3 - R3 (new I3 / sine)	; 32-34	avail 12,4,11,10,0,8
	mulpd	xmm13, xmm6		;;#2 B2 = B2 * sine (new I2)		; 32-36
	xcopy	xmm4, xmm9		;;#2 Copy R4 / sine			; 32-34	avail 12,11,10,0,8

	addpd	xmm3, xmm15		;;#2 B4 = B4 + R4 (new I4 / sine)	; 33-35	avail 12,11,10,0,8,15
	mulpd	xmm7, [screg+16]	;;#2 R1 * normalization_inverse		; 33-37

	subpd	xmm9, xmm2		;;#2 R4 = R4 - R3 (newer I4 / sine)	; 34-36
	mulpd	xmm1, xmm6		;;#2 A2 = A2 * sine (new R2)		; 34-38	avail 12,11,10,0,8,15,6
	xload	xmm6, [screg+32++32]	;;#2 sine * normalization_inverse	; 34	avail 12,11,10,0,8,15

	addpd	xmm2, xmm4		;;#2 R3 = R4 + R3 (newer R3 / sine)	; 35-37	avail 12,11,10,0,8,15,4
	xcopy	xmm4, xmm14		;;#2 Copy I3 / sine			; 35-37	avail 12,11,10,0,8,15

	xprefetchw [srcreg+srcinc]

	subpd	xmm14, xmm3		;;#2 I3 = I3 - I4 (newer R4 / sine)	; 36-38
	xcopy	xmm0, xmm5		;;#2 Copy I1				; 36-38	avail 12,11,10,8,15

	subpd	xmm5, xmm13		;;#2 I1 = I1 - I2 (newer I2)		; 37-39
	mulpd	xmm9, xmm6		;;#2 newer I4 * sine			; 37-41

	addpd	xmm3, xmm4		;;#2 I4 = I3 + I4 (newer I3 / sine)	; 38-40	avail 12,11,10,8,15,0
	mulpd	xmm2, xmm6		;;#2 newer R3 * sine			; 38-42
	xcopy	xmm4, xmm7		;;#2 Copy R1				; 38-40	avail 12,11,10,8,15

	addpd	xmm13, xmm0		;;#2 I2 = I1 + I2 (newer I1)		; 39-41	avail 12,11,10,8,15,0
	mulpd	xmm14, xmm6		;;#2 newer R4 * sine			; 39-43

	addpd	xmm7, xmm1		;;#2 R2 = R1 + R2 (newer R1)		; 40-42
	xcopy	xmm0, xmm5		;;#2 Copy I2				; 40-42	avail 12,11,10,8,15

	subpd	xmm4, xmm1		;;#2 R1 = R1 - R2 (newer R2)		; 41-43	avail 12,11,10,8,15,1
	mulpd	xmm3, xmm6		;;#2 newer I3 * sine			; 41-45	avail 12,11,10,8,15,1,6

	xprefetchw [srcreg+srcinc+d1]

	subpd	xmm5, xmm9		;;#2 I2 = I2 - I4 (final I4)		; 42-44	avail 12,11,10,8,15,6 storable 5
	xcopy	xmm1, xmm13		;;#2 Copy I1				; 42-44	avail 12,11,10,8,15,6 storable 5

	xcopy	xmm6, xmm7		;;#2 Copy R1				; 43-45	avail 12,11,10,8,15 storable 5
	subpd	xmm7, xmm2		;;#2 R1 = R1 - R3 (final R3)		; 43-45	avail 12,11,10,8,15 storable 5,7

	addpd	xmm9, xmm0		;;#2 I4 = I2 + I4 (final I2)		; 44-46	avail 12,11,10,8,15,0 storable 5,7,9
	xcopy	xmm0, xmm4		;;#2 Copy R2				; 44-46	avail 12,11,10,8,15 storable 5,7,9

	subpd	xmm4, xmm14		;;#2 R2 = R2 - R4 (final R4)		; 45-47	avail 12,11,10,8,15 storable 5,7,9,4
	xstore	[srcreg+d2+48], xmm5	;;#2 Save I4				; 45	avail 12,11,10,8,15,5 storable 7,9,4

	addpd	xmm2, xmm6		;;#2 R3 = R1 + R3 (final R1)		; 46-48	avail 12,11,10,8,15,5,6 storable 7,9,4,2
	xstore	[srcreg+16], xmm7	;;#2 Save R3				; 46	avail 12,11,10,8,15,5,6,7 storable 9,4,2

	addpd	xmm14, xmm0		;;#2 R4 = R2 + R4 (final R2)		; 47-49	avail 12,11,10,8,15,5,6,7,0 storable 9,4,2,14
	xstore	[srcreg+d2+32], xmm9	;;#2 Save I2				; 47	avail 12,11,10,8,15,5,6,7,0,9 storable 4,2,14

	subpd	xmm13, xmm3		;;#2 I1 = I1 - I3 (final I3)		; 48-50	avail 12,11,10,8,15,5,6,7,0,9 storable 4,2,14,13
	xstore	[srcreg+d2+16], xmm4	;;#2 Save R4				; 48	avail 12,11,10,8,15,5,6,7,0,9,4 storable 2,14,13

	addpd	xmm3, xmm1		;;#2 I3 = I1 + I3 (final I1)		; 49-51	avail 12,11,10,8,15,5,6,7,0,9,4,1 storable 2,14,13,3
	xstore	[srcreg], xmm2		;;#2 Save R1				; 49	avail 12,11,10,8,15,5,6,7,0,9,4,1,2 storable 14,13,3

	xstore	[srcreg+d2], xmm14	;;#2 Save R2				; 50
	xstore	[srcreg+48], xmm13	;;#2 Save I3				; 51
	xstore	[srcreg+32], xmm3	;;#2 Save I1				; 52

	bump	srcreg, srcinc
	ENDM
ENDIF
ENDIF

;; 64-bit AMD K8 optimized versions of the above macros

IF (@INSTR(,%xarch,<K8>) NE 0)
IFDEF X86_64

r4_x4c_wpn_djbunfft_mem_preload MACRO
	xload	xmm15, XMM_TWO
	ENDM

r4_x4c_wpn_djbunfft_mem MACRO mem1,mem2,mem3,mem4,mem5,mem6,mem7,mem8,dst3,screg,normreg,pre1,pre2
	xload	xmm3, [screg+48+16]	;; cosine/sine
	xload	xmm6, mem3		;; R2
	mulpd	xmm6, xmm3		;; A2 = R2 * cosine/sine	; 1-4

	xload	xmm0, mem4		;; I2
	mulpd	xmm3, xmm0		;; B2 = I2 * cosine/sine	; 3-6

	xload	xmm4, [screg+16]	;; cosine/sine
	xload	xmm2, mem7		;; R4
	mulpd	xmm2, xmm4		;; A4 = R4 * cosine/sine	; 5-8
	addpd	xmm6, xmm0		;; A2 = A2 + I2			; 5-8

	xload	xmm5, [screg+16]	;; cosine/sine
	xload	xmm7, mem5		;; R3
	mulpd	xmm7, xmm5		;; A3 = R3 * cosine/sine	; 7-10
	subpd	xmm3, mem3		;; B2 = B2 - R2			; 7-10

	xload	xmm1, mem6		;; I3
	mulpd	xmm5, xmm1		;; B3 = I3 * cosine/sine	; 9-12
	xload	xmm0, mem8		;; I4
	subpd	xmm2, xmm0		;; A4 = A4 - I4			; 9-12

	mulpd	xmm4, xmm0		;; B4 = I4 * cosine/sine	; 11-14
	addpd	xmm7, xmm1		;; A3 = A3 + I3			; 11-14

	subpd	xmm5, mem5		;; B3 = B3 - R3			; 13-16
	mulpd	xmm6, [screg+48+32]	;; A2 = A2 * sine*normalization_inverse (new R2) ; 13-16

	addpd	xmm4, mem7		;; B4 = B4 + R4			; 15-18
	mulpd	xmm3, [screg+48+32]	;; B2 = B2 * sine*normalization_inverse (new I2) ; 15-18

	subpd	xmm2, xmm7		;; R4 = R4 - R3 (new I4)	; 17-20
	mulpd	xmm7, xmm15						; 17-20

	xprefetchw [pre1]

	subpd	xmm5, xmm4		;; I3 = I3 - I4 (new R4)	; 19-22
	mulpd	xmm4, xmm15						; 19-22

	xload	xmm1, mem2		;; I1
	addpd	xmm7, xmm2		;; R3 = R4 + R3 (new R3)	; 21-24
	mulpd	xmm1, [normreg+16]	;; I1 * normalization_inverse	; 21-24

	xload	xmm0, mem1		;; R1
	addpd	xmm4, xmm5		;; I4 = I3 + I4 (new I3)	; 23-26
	mulpd	xmm0, [normreg+16]	;; R1 * normalization_inverse	; 23-26

	subpd	xmm1, xmm3		;; I1 = I1 - I2 (new I2)	; 25-28
	mulpd	xmm3, xmm15						; 25-28

	subpd	xmm0, xmm6		;; R1 = R1 - R2 (new R2)	; 27-30
	mulpd	xmm6, xmm15						; 27-30

	xprefetchw [pre1][pre2]

	addpd	xmm3, xmm1		;; I2 = I1 + I2 (new I1)	; 29-32
	mulpd	xmm2, [screg+32]	;; I4 = I4 * sine*normalization_inverse

	addpd	xmm6, xmm0		;; R2 = R1 + R2 (new R1)	; 31-34
	mulpd	xmm5, [screg+32]	;; R4 = R4 * sine*normalization_inverse

	subpd	xmm1, xmm2		;; I2 = I2 - I4 (final I4)	; 33-36
	mulpd	xmm7, [screg+32]	;; R3 = R3 * sine*normalization_inverse

	subpd	xmm0, xmm5		;; R2 = R2 - R4 (final R4)	; 35-38
	mulpd	xmm4, [screg+32]	;; I3 = I3 * sine*normalization_inverse

	subpd	xmm6, xmm7		;; R1 = R1 - R3 (final R3)	; 37-40
	mulpd	xmm2, xmm15

	subpd	xmm3, xmm4		;; I1 = I1 - I3 (final I3)	; 39-42
	mulpd	xmm5, xmm15

	addpd	xmm2, xmm1		;; I4 = I2 + I4 (final I2)	; 41-44
	mulpd	xmm7, xmm15

	addpd	xmm5, xmm0		;; R4 = R2 + R4 (final R2)	; 43-46
	mulpd	xmm4, xmm15

	addpd	xmm7, xmm6		;; R3 = R1 + R3 (final R1)	; 45-48
	xstore	dst3, xmm6						; 41

	addpd	xmm4, xmm3		;; I3 = I1 + I3 (final I1)	; 47-50
	ENDM

r4_x4c_wpn_djbunfft_partial_mem_preload MACRO
	xload	xmm15, XMM_TWO
	ENDM

r4_x4c_wpn_djbunfft_partial_mem MACRO r1,r2,r3,r4,r5,r6,r7,r8,mem1,mem2,mem5,mem6,dst1,screg,normreg,pre1,pre2
	xload	r1, [screg+48+16]	;; cosine/sine
	mulpd	r1, r4			;; B2 = I2 * cosine/sine	; 1-4

	subpd	r1, r3			;; B2 = B2 - R2			; 5-8
	mulpd	r3, [screg+48+16]	;; A2 = R2 * cosine/sine	; 3-6
	xload	r5, [screg+16]		;; cosine/sine

	xload	r2, mem5		;; R3
	mulpd	r5, r8			;; B4 = I4 * cosine/sine	; 5-8

	addpd	r3, r4			;; A2 = A2 + I2			; 7-10
	xload	r4, [screg+16]		;; cosine/sine
	mulpd	r2, r4			;; A3 = R3 * cosine/sine	; 7-10

	addpd	r5, r7			;; B4 = B4 + R4			; 9-12
	mulpd	r7, [screg+16]		;; A4 = R4 * cosine/sine	; 9-12

	addpd	r2, mem6		;; A3 = A3 + I3			; 11-14
	mulpd	r4, mem6		;; B3 = I3 * cosine/sine	; 11-14

	subpd	r7, r8			;; A4 = A4 - I4			; 13-16
	mulpd	r1, [screg+48+32]	;; B2 = B2 * sine*normalization_inverse (new I2)

	subpd	r4, mem5		;; B3 = B3 - R3			; 15-18
	mulpd	r3, [screg+48+32]	;; A2 = A2 * sine*normalization_inverse (new R2)

	subpd	r7, r2			;; R4 = R4 - R3 (new I4)	; 17-20
	mulpd	r2, xmm15						; 17-20

	xprefetchw [pre1]

	subpd	r4, r5			;; I3 = I3 - I4 (new R4)	; 19-22
	mulpd	r5, xmm15						; 19-22

	xload	r8, mem2		;; I1
	addpd	r2, r7			;; R3 = R4 + R3 (new R3)	; 21-24
	mulpd	r8, [normreg+16]	;; I1 * normalization_inverse	; 21-24

	xload	r6, mem1		;; R1
	addpd	r5, r4			;; I4 = I3 + I4 (new I3)	; 23-26
	mulpd	r6, [normreg+16]	;; R1 * normalization_inverse	; 23-26

	subpd	r8, r1			;; I1 = I1 - I2 (new I2)	; 25-28
	mulpd	r1, xmm15						; 25-28

	xprefetchw [pre1][pre2]

	subpd	r6, r3			;; R1 = R1 - R2 (new R2)	; 27-30
	mulpd	r3, xmm15						; 27-30

	addpd	r1, r8			;; I2 = I1 + I2 (new I1)	; 29-32
	mulpd	r7, [screg+32]		;; I4 = I4 * sine*normalization_inverse

	addpd	r3, r6			;; R2 = R1 + R2 (new R1)	; 31-34
	mulpd	r4, [screg+32]		;; R4 = R4 * sine*normalization_inverse

	subpd	r8, r7			;; I2 = I2 - I4 (final I4)	; 33-36
	mulpd	r2, [screg+32]		;; R3 = R3 * sine*normalization_inverse

	subpd	r6, r4			;; R2 = R2 - R4 (final R4)	; 35-38
	mulpd	r5, [screg+32]		;; I3 = I3 * sine*normalization_inverse

	subpd	r3, r2			;; R1 = R1 - R3 (final R3)	; 37-40
	mulpd	r7, xmm15

	subpd	r1, r5			;; I1 = I1 - I3 (final I3)	; 39-42
	mulpd	r4, xmm15

	addpd	r7, r8			;; I4 = I2 + I4 (final I2)	; 41-44
	mulpd	r2, xmm15

	addpd	r4, r6			;; R4 = R2 + R4 (final R2)	; 43-46
	mulpd	r5, xmm15

	addpd	r2, r3			;; R3 = R1 + R3 (final R1)	; 45-48

	addpd	r5, r1			;; I3 = I1 + I3 (final I1)	; 47-50

	xstore	dst1, r2
	ENDM

ENDIF
ENDIF


;;
;; ************************************* four-complex-with_square and variants ******************************************
;;
;; These macros are used in the last levels of pass 2 in two pass FFTs.
;;

;;
;; The last two levels of the forward FFT are performed.
;; No sin/cos multipliers are needed.
;;

r4_x4cl_four_complex_fft_final MACRO srcreg,srcinc,d1,d2
	r4_x4c_simple_fft_mem [srcreg],[srcreg+d1],[srcreg+d2],[srcreg+d2+d1],[srcreg+16],[srcreg+d1+16],[srcreg+d2+16],[srcreg+d2+d1+16],srcreg+srcinc,d1
	xstore	[srcreg], xmm3		;; Save R1
	xstore	[srcreg+16], xmm7	;; Save R2
	xload	xmm3, [srcreg+32]	;; R1
	xload	xmm7, [srcreg+48]	;; R5
	xstore	[srcreg+32], xmm2	;; Save R3
	xstore	[srcreg+48], xmm6	;; Save R4
	xstore	[srcreg+d1], xmm0	;; Save R5
	xstore	[srcreg+d1+16], xmm1	;; Save R6
	xload	xmm2, [srcreg+d1+32]	;; R2
	xload	xmm6, [srcreg+d1+48]	;; R6
	xstore	[srcreg+d1+32], xmm5	;; Save R7
	xstore	[srcreg+d1+48], xmm4	;; Save R8
	r4_x4c_simple_fft_partial_mem xmm3,xmm2,xmm0,xmm5,xmm7,xmm6,xmm1,xmm4,[srcreg+d2+32],[srcreg+d2+d1+32],[srcreg+d2+48],[srcreg+d2+d1+48],srcreg+srcinc+d2,d1
	xstore	[srcreg+d2], xmm5	;; Save R1
	xstore	[srcreg+d2+16], xmm4	;; Save R2
	xstore	[srcreg+d2+32], xmm0	;; Save R3
	xstore	[srcreg+d2+48], xmm1	;; Save R4
	xstore	[srcreg+d2+d1], xmm3	;; Save R5
	xstore	[srcreg+d2+d1+16], xmm2	;; Save R6
	xstore	[srcreg+d2+d1+32], xmm6	;; Save R7
	xstore	[srcreg+d2+d1+48], xmm7	;; Save R8
	bump	srcreg, srcinc
	ENDM

;;
;; The last two levels of the forward FFT are performed, point-wise
;; squaring, and first two levels of the inverse FFT are performed.
;; No sin/cos multipliers are needed.
;;

r4_x4cl_four_complex_with_square_preload MACRO
	ENDM

r4_x4cl_four_complex_with_square MACRO srcreg,srcinc,d1,d2
	r4_x4c_simple_fft_mem [srcreg],[srcreg+d1],[srcreg+d2],[srcreg+d2+d1],[srcreg+16],[srcreg+d1+16],[srcreg+d2+16],[srcreg+d2+d1+16],srcreg+srcinc,d1
	xstore	[srcreg], xmm0			;; R5
	xp_complex_square xmm3, xmm7, xmm0	;; Square R1, R2
	xp_complex_square xmm2, xmm6, xmm0	;; Square R3, R4
	xp_complex_square xmm5, xmm4, xmm0	;; Square R7, R8
	xload	xmm0, [srcreg]
	xstore	[srcreg], xmm4
	xp_complex_square xmm0, xmm1, xmm4	;; Square R5, R6
	r4_x4c_simple_unfft xmm3,xmm7,xmm2,xmm6,xmm0,xmm1,xmm5,xmm4,[srcreg],,,[srcreg]
;;	xstore	[srcreg], xmm0			;; Save R1
	xstore	[srcreg+16], xmm2		;; Save R3
	xload	xmm0, [srcreg+32]		;; R1
	xload	xmm2, [srcreg+48]		;; R5
	xstore	[srcreg+32], xmm4		;; Save R5
	xstore	[srcreg+48], xmm6		;; Save R7
	xload	xmm4, [srcreg+d1+32]		;; R2
	xload	xmm6, [srcreg+d1+48]		;; R6
	xstore	[srcreg+d1], xmm1		;; Save R2
	xstore	[srcreg+d1+16], xmm3		;; Save R4
	xstore	[srcreg+d1+32], xmm5		;; Save R6
	xstore	[srcreg+d1+48], xmm7		;; Save R8
	r4_x4c_simple_fft_partial_mem xmm0,xmm4,xmm1,xmm3,xmm2,xmm6,xmm5,xmm7,[srcreg+d2+32],[srcreg+d2+d1+32],[srcreg+d2+48],[srcreg+d2+d1+48],srcreg+srcinc+d2,d1
	xstore	[srcreg+d2], xmm0		;; R5
	xp_complex_square xmm3, xmm7, xmm0	;; Square R1, R2
	xp_complex_square xmm1, xmm5, xmm0	;; Square R3, R4
	xp_complex_square xmm6, xmm2, xmm0	;; Square R7, R8
	xload	xmm0, [srcreg+d2]		;; R5
	xstore	[srcreg+d2], xmm2		;; R8
	xp_complex_square xmm0, xmm4, xmm2	;; Square R5, R6
	r4_x4c_simple_unfft xmm3,xmm7,xmm1,xmm5,xmm0,xmm4,xmm6,xmm2,[srcreg+d2],,,[srcreg+d2]
;;	xstore	[srcreg+d2], xmm0		;; Save R1
	xstore	[srcreg+d2+16], xmm1		;; Save R3
	xstore	[srcreg+d2+32], xmm2		;; Save R5
	xstore	[srcreg+d2+48], xmm5		;; Save R7
	xstore	[srcreg+d2+d1], xmm4		;; Save R2
	xstore	[srcreg+d2+d1+16], xmm3		;; Save R4
	xstore	[srcreg+d2+d1+32], xmm6		;; Save R6
	xstore	[srcreg+d2+d1+48], xmm7		;; Save R8
	bump	srcreg, srcinc
	ENDM

;;
;; The last two levels of the forward FFT are performed, point-wise
;; multiplication, and first two levels of the inverse FFT are performed.
;; No sin/cos multipliers are needed.
;;

r4_x4cl_four_complex_with_mult MACRO srcreg,srcinc,d1,d2
	r4_x4c_simple_fft_mem [srcreg],[srcreg+d1],[srcreg+d2],[srcreg+d2+d1],[srcreg+16],[srcreg+d1+16],[srcreg+d2+16],[srcreg+d2+d1+16],srcreg+srcinc,d1
	xp4c_mulf xmm3,xmm7,xmm2,xmm6,xmm0,xmm1,xmm5,xmm4,[srcreg],[srcreg+16],[srcreg+32],[srcreg+48],[srcreg+d1],[srcreg+d1+16],[srcreg+d1+32],[srcreg+d1+48]
	xstore	[srcreg], xmm4
	r4_x4c_simple_unfft xmm3,xmm7,xmm2,xmm6,xmm0,xmm1,xmm5,xmm4,[srcreg],,,[srcreg]
;;	xstore	[srcreg], xmm0			;; Save R1
	xstore	[srcreg+16], xmm2		;; Save R3
	xload	xmm0, [srcreg+32]		;; R1
	xload	xmm2, [srcreg+48]		;; R5
	xstore	[srcreg+32], xmm4		;; Save R5
	xstore	[srcreg+48], xmm6		;; Save R7
	xload	xmm4, [srcreg+d1+32]		;; R2
	xload	xmm6, [srcreg+d1+48]		;; R6
	xstore	[srcreg+d1], xmm1		;; Save R2
	xstore	[srcreg+d1+16], xmm3		;; Save R4
	xstore	[srcreg+d1+32], xmm5		;; Save R6
	xstore	[srcreg+d1+48], xmm7		;; Save R8
	r4_x4c_simple_fft_partial_mem xmm0,xmm4,xmm1,xmm3,xmm2,xmm6,xmm5,xmm7,[srcreg+d2+32],[srcreg+d2+d1+32],[srcreg+d2+48],[srcreg+d2+d1+48],srcreg+srcinc+d2,d1
	xp4c_mulf xmm3,xmm7,xmm1,xmm5,xmm0,xmm4,xmm6,xmm2,[srcreg+d2],[srcreg+d2+16],[srcreg+d2+32],[srcreg+d2+48],[srcreg+d2+d1],[srcreg+d2+d1+16],[srcreg+d2+d1+32],[srcreg+d2+d1+48]
	xstore	[srcreg+d2], xmm2
	r4_x4c_simple_unfft xmm3,xmm7,xmm1,xmm5,xmm0,xmm4,xmm6,xmm2,[srcreg+d2],,,[srcreg+d2]
;;	xstore	[srcreg+d2], xmm0		;; Save R1
	xstore	[srcreg+d2+16], xmm1		;; Save R3
	xstore	[srcreg+d2+32], xmm2		;; Save R5
	xstore	[srcreg+d2+48], xmm5		;; Save R7
	xstore	[srcreg+d2+d1], xmm4		;; Save R2
	xstore	[srcreg+d2+d1+16], xmm3		;; Save R4
	xstore	[srcreg+d2+d1+32], xmm6		;; Save R6
	xstore	[srcreg+d2+d1+48], xmm7		;; Save R8
	bump	srcreg, srcinc
	ENDM

;;
;; Point-wise multiplication and first two levels of the inverse FFT are performed.
;; No sin/cos multipliers are needed.
;;

r4_x4cl_four_complex_with_mulf MACRO srcreg,srcinc,d1,d2
	xload	xmm3, [srcreg][rbx]		;; R1
	xload	xmm7, [srcreg+16][rbx]		;; R2
	xload	xmm2, [srcreg+32][rbx]		;; R3
	xload	xmm6, [srcreg+48][rbx]		;; R4
	xload	xmm0, [srcreg+d1][rbx]		;; R5
	xload	xmm1, [srcreg+d1+16][rbx]	;; R6
	xload	xmm5, [srcreg+d1+32][rbx]	;; R7
	xload	xmm4, [srcreg+d1+48][rbx]	;; R8
	xp4c_mulf xmm3,xmm7,xmm2,xmm6,xmm0,xmm1,xmm5,xmm4,[srcreg],[srcreg+16],[srcreg+32],[srcreg+48],[srcreg+d1],[srcreg+d1+16],[srcreg+d1+32],[srcreg+d1+48]
	xstore	[srcreg], xmm4
	r4_x4c_simple_unfft xmm3,xmm7,xmm2,xmm6,xmm0,xmm1,xmm5,xmm4,[srcreg],srcreg+srcinc,d1,[srcreg]
;;	xstore	[srcreg], xmm0			;; Save R1
	xstore	[srcreg+16], xmm2		;; Save R3
	xstore	[srcreg+32], xmm4		;; Save R5
	xstore	[srcreg+48], xmm6		;; Save R7
	xstore	[srcreg+d1], xmm1		;; Save R2
	xstore	[srcreg+d1+16], xmm3		;; Save R4
	xstore	[srcreg+d1+32], xmm5		;; Save R6
	xstore	[srcreg+d1+48], xmm7		;; Save R8
	xload	xmm3, [srcreg+d2][rbx]		;; R1
	xload	xmm7, [srcreg+d2+16][rbx]	;; R2
	xload	xmm1, [srcreg+d2+32][rbx]	;; R3
	xload	xmm5, [srcreg+d2+48][rbx]	;; R4
	xload	xmm0, [srcreg+d2+d1][rbx]	;; R5
	xload	xmm4, [srcreg+d2+d1+16][rbx]	;; R6
	xload	xmm6, [srcreg+d2+d1+32][rbx]	;; R7
	xload	xmm2, [srcreg+d2+d1+48][rbx]	;; R8
	xp4c_mulf xmm3,xmm7,xmm1,xmm5,xmm0,xmm4,xmm6,xmm2,[srcreg+d2],[srcreg+d2+16],[srcreg+d2+32],[srcreg+d2+48],[srcreg+d2+d1],[srcreg+d2+d1+16],[srcreg+d2+d1+32],[srcreg+d2+d1+48]
	xstore	[srcreg+d2], xmm2
	r4_x4c_simple_unfft xmm3,xmm7,xmm1,xmm5,xmm0,xmm4,xmm6,xmm2,[srcreg+d2],srcreg+srcinc+d2,d1,[srcreg+d2]
;;	xstore	[srcreg+d2], xmm0		;; Save R1
	xstore	[srcreg+d2+16], xmm1		;; Save R3
	xstore	[srcreg+d2+32], xmm2		;; Save R5
	xstore	[srcreg+d2+48], xmm5		;; Save R7
	xstore	[srcreg+d2+d1], xmm4		;; Save R2
	xstore	[srcreg+d2+d1+16], xmm3		;; Save R4
	xstore	[srcreg+d2+d1+32], xmm6		;; Save R6
	xstore	[srcreg+d2+d1+48], xmm7		;; Save R8
	bump	srcreg, srcinc
	ENDM


r4_x4c_simple_fft_mem MACRO R1,R2,R3,R4,R5,R6,R7,R8,pre1,pre2
	xload	xmm0, R1		;; R1
	xload	xmm2, R3		;; R3
	xcopy	xmm5, xmm0		;; Copy R1
	subpd	xmm0, xmm2		;; R1 = R1 - R3 (new R3)
	addpd	xmm2, xmm5		;; R3 = R1 + R3 (new R1)

	xload	xmm4, R5		;; I1
	xload	xmm6, R7		;; I3
	xcopy	xmm5, xmm4		;; Copy I1
	subpd	xmm4, xmm6		;; I1 = I1 - I3 (new I3)
	addpd	xmm6, xmm5		;; I3 = I1 + I3 (new I1)

	xprefetchw [pre1]

	xload	xmm1, R2		;; R2
	xload	xmm3, R4		;; R4
	xcopy	xmm5, xmm1		;; Copy R2
	subpd	xmm1, xmm3		;; R2 = R2 - R4 (new R4)
	addpd	xmm3, xmm5		;; R4 = R2 + R4 (new R2)

	xload	xmm5, R6		;; I2
	xload	xmm7, R8		;; I4
	subpd	xmm5, xmm7		;; I2 = I2 - I4 (new I4)
	addpd	xmm7, R6		;; I4 = I2 + I4 (new I2)

	xprefetchw [pre1][pre2]

	subpd	xmm4, xmm1		;; I3 = I3 - R4 (final I4)
	multwo	xmm1			;; R4 = R4 * 2
	addpd	xmm1, xmm4		;; R4 = I3 + R4 (final I3)

	subpd	xmm0, xmm5		;; R3 = R3 - I4 (final R3)
	multwo	xmm5			;; I4 = I4 * 2
	addpd	xmm5, xmm0		;; I4 = R3 + I4 (final R4)

	subpd	xmm2, xmm3		;; R1 = R1 - R2 (final R2)
	multwo	xmm3			;; R2 = R2 * 2
	addpd	xmm3, xmm2		;; R2 = R1 + R2 (final R1)

	subpd	xmm6, xmm7		;; I1 = I1 - I2 (final I2)
	multwo	xmm7			;; I2 = I2 * 2
	addpd	xmm7, xmm6		;; I2 = I1 + I2 (final I1)
	ENDM

r4_x4c_simple_fft_partial_mem MACRO r1, r2, r3, r4, r5, r6, r7, r8, mem3, mem4, mem7, mem8, pre1, pre2
	xload	r3, mem3		;; R3
	xcopy	r8, r3			;; Copy R3
	addpd	r3, r1			;; R3 = R1 + R3 (new R1)
	subpd	r1, r8			;; R1 = R1 - R3 (new R3)

	xload	r7, mem7		;; I3
	xcopy	r8, r7			;; Copy I3
	addpd	r7, r5			;; I3 = I1 + I3 (new I1)
	subpd	r5, r8			;; I1 = I1 - I3 (new I3)

	xprefetchw [pre1]

	xload	r4, mem4		;; R4
	xcopy	r8, r4			;; Copy R4
	addpd	r4, r2			;; R4 = R2 + R4 (new R2)
	subpd	r2, r8			;; R2 = R2 - R4 (new R4)

	xload	r8, mem8		;; I4
	addpd	r8, r6			;; I4 = I2 + I4 (new I2)
	subpd	r6, mem8		;; I2 = I2 - I4 (new I4)

	subpd	r5, r2			;; I3 = I3 - R4 (final I4)
	multwo	r2			;; R4 = R4 * 2
	addpd	r2, r5			;; R4 = I3 + R4 (final I3)

	xprefetchw [pre1][pre2]

	subpd	r1, r6			;; R3 = R3 - I4 (final R3)
	multwo	r6			;; I4 = I4 * 2
	addpd	r6, r1			;; I4 = R3 + I4 (final R4)

	subpd	r3, r4			;; R1 = R1 - R2 (final R2)
	multwo	r4			;; R2 = R2 * 2
	addpd	r4, r3			;; R2 = R1 + R2 (final R1)

	subpd	r7, r8			;; I1 = I1 - I2 (final I2)
	multwo	r8			;; I2 = I2 * 2
	addpd	r8, r7			;; I2 = I1 + I2 (final I1)
	ENDM

r4_x4c_simple_unfft MACRO r1, r2, r3, r4, r5, r6, r7, r8, mem8, pre1, pre2, dest1
	xcopy	r8, r1			;; R1
	subpd	r1, r3			;; R1 = R1 - R2 (new R2)
	addpd	r3, r8			;; R2 = R1 + R2 (new R1)

	xcopy	r8, r7			;; R4
	subpd	r7, r5			;; R4 = R4 - R3 (new I4)
	addpd	r5, r8			;; R3 = R4 + R3 (new R3)

	xcopy	r8, r2			;; I2
	subpd	r2, r4			;; I1 = I1 - I2 (new I2)
	addpd	r4, r8			;; I2 = I1 + I2 (new I1)

	xcopy	r8, r3
	subpd	r3, r5			;; R1 = R1 - R3 (final R3)
	addpd	r5, r8			;; R3 = R1 + R3 (final R1)

	IFNB <pre1>
	xprefetchw [pre1]
	ENDIF

	xload	r8, mem8		;; I4
	addpd	r8, r6			;; I4 = I3 + I4 (new I3)
	subpd	r6, mem8		;; I3 = I3 - I4 (new R4)

	xstore	dest1, r5

	xcopy	r5, r1			;; Copy R2
	subpd	r1, r6			;; R2 = R2 - R4 (final R4)
	addpd	r6, r5			;; R4 = R2 + R4 (final R2)

	xcopy	r5, r2			;; Copy I2
	subpd	r2, r7			;; I2 = I2 - I4 (final I4)
	addpd	r7, r5			;; I4 = I2 + I4 (final I2)

	IFNB <pre1>
	xprefetchw [pre1][pre2]
	ENDIF

	xcopy	r5, r4			;; Copy I1
	subpd	r4, r8			;; I1 = I1 - I3 (final I3)
	addpd	r8, r5			;; I3 = I1 + I3 (final I1)
	ENDM

;; 32-bit AMD K8 optimized versions of the above macros

IF (@INSTR(,%xarch,<K8>) NE 0)

r4_x4c_simple_fft_mem MACRO R1,R2,R3,R4,R5,R6,R7,R8,pre1,pre2
	xload	xmm0, R1		;; R1
	xload	xmm2, R3		;; R3
	subpd	xmm0, xmm2		;; R1 = R1 - R3 (new R3)
	addpd	xmm2, R1		;; R3 = R1 + R3 (new R1)

	xload	xmm4, R5		;; I1
	xload	xmm6, R7		;; I3
	subpd	xmm4, xmm6		;; I1 = I1 - I3 (new I3)
	addpd	xmm6, R5		;; I3 = I1 + I3 (new I1)

	xprefetchw [pre1]

	xload	xmm1, R2		;; R2
	xload	xmm3, R4		;; R4
	subpd	xmm1, xmm3		;; R2 = R2 - R4 (new R4)
	addpd	xmm3, R2		;; R4 = R2 + R4 (new R2)

	xload	xmm5, R6		;; I2
	xload	xmm7, R8		;; I4
	subpd	xmm5, xmm7		;; I2 = I2 - I4 (new I4)
	addpd	xmm7, R6		;; I4 = I2 + I4 (new I2)

	xprefetchw [pre1][pre2]

	subpd	xmm4, xmm1		;; I3 = I3 - R4 (final I4)
	multwo	xmm1			;; R4 = R4 * 2

	subpd	xmm0, xmm5		;; R3 = R3 - I4 (final R3)
	multwo	xmm5			;; I4 = I4 * 2

	subpd	xmm2, xmm3		;; R1 = R1 - R2 (final R2)
	multwo	xmm3			;; R2 = R2 * 2

	subpd	xmm6, xmm7		;; I1 = I1 - I2 (final I2)
	multwo	xmm7			;; I2 = I2 * 2

	addpd	xmm1, xmm4		;; R4 = I3 + R4 (final I3)
	addpd	xmm5, xmm0		;; I4 = R3 + I4 (final R4)
	addpd	xmm3, xmm2		;; R2 = R1 + R2 (final R1)
	addpd	xmm7, xmm6		;; I2 = I1 + I2 (final I1)
	ENDM

r4_x4c_simple_fft_partial_mem MACRO r1, r2, r3, r4, r5, r6, r7, r8, mem3, mem4, mem7, mem8, pre1, pre2
	xload	r3, mem3		;; R3
	addpd	r3, r1			;; R3 = R1 + R3 (new R1)
	subpd	r1, mem3		;; R1 = R1 - R3 (new R3)

	xload	r7, mem7		;; I3
	addpd	r7, r5			;; I3 = I1 + I3 (new I1)
	subpd	r5, mem7		;; I1 = I1 - I3 (new I3)

	xprefetchw [pre1]

	xload	r4, mem4		;; R4
	addpd	r4, r2			;; R4 = R2 + R4 (new R2)
	subpd	r2, mem4		;; R2 = R2 - R4 (new R4)

	xload	r8, mem8		;; I4
	addpd	r8, r6			;; I4 = I2 + I4 (new I2)
	subpd	r6, mem8		;; I2 = I2 - I4 (new I4)

	xprefetchw [pre1][pre2]

	subpd	r5, r2			;; I3 = I3 - R4 (final I4)
	multwo	r2			;; R4 = R4 * 2

	subpd	r1, r6			;; R3 = R3 - I4 (final R3)
	multwo	r6			;; I4 = I4 * 2

	subpd	r3, r4			;; R1 = R1 - R2 (final R2)
	multwo	r4			;; R2 = R2 * 2

	subpd	r7, r8			;; I1 = I1 - I2 (final I2)
	multwo	r8			;; I2 = I2 * 2

	addpd	r2, r5			;; R4 = I3 + R4 (final I3)
	addpd	r6, r1			;; I4 = R3 + I4 (final R4)
	addpd	r4, r3			;; R2 = R1 + R2 (final R1)
	addpd	r8, r7			;; I2 = I1 + I2 (final I1)
	ENDM

r4_x4c_simple_unfft MACRO r1, r2, r3, r4, r5, r6, r7, r8, mem8, pre1, pre2, dest1
	subpd	r1, r3			;; R1 = R1 - R2 (new R2)
	multwo	r3			;; R2 = R2 * 2
	addpd	r3, r1			;; R2 = R1 + R2 (new R1)

	subpd	r7, r5			;; R4 = R4 - R3 (new I4)
	multwo	r5			;; R3 = R3 * 2
	addpd	r5, r7			;; R3 = R4 + R3 (new R3)

	subpd	r2, r4			;; I1 = I1 - I2 (new I2)
	multwo	r4			;; I2 = I2 * 2
	addpd	r4, r2			;; I2 = I1 + I2 (new I1)

	xload	r8, mem8		;; I4
	addpd	r8, r6			;; I4 = I3 + I4 (new I3)
	subpd	r6, mem8		;; I3 = I3 - I4 (new R4)

	IFNB <pre1>
	xprefetchw [pre1]
	ENDIF

	subpd	r3, r5			;; R1 = R1 - R3 (final R3)
	multwo	r5			;; R3 = R3 * 2

	subpd	r1, r6			;; R2 = R2 - R4 (final R4)
	multwo	r6			;; R4 = R4 * 2

	subpd	r2, r7			;; I2 = I2 - I4 (final I4)
	multwo	r7			;; R4 = R4 * 2

	subpd	r4, r8			;; I1 = I1 - I3 (final I3)
	multwo	r8			;; I3 = I3 * 2

	IFNB <pre1>
	xprefetchw [pre1][pre2]
	ENDIF

	addpd	r5, r3			;; R3 = R1 + R3 (final R1)
	addpd	r6, r1			;; R4 = R2 + R4 (final R2)
	xstore	dest1, r5
	addpd	r7, r2			;; I4 = I2 + I4 (final I2)
	addpd	r8, r4			;; I3 = I1 + I3 (final I1)
	ENDM

ENDIF

;; 64-bit Intel and AMD K10 optimized versions of the above macros

IF (@INSTR(,%xarch,<CORE>) NE 0) OR (@INSTR(,%xarch,<P4>) NE 0) OR (@INSTR(,%xarch,<K10>) NE 0) OR (@INSTR(,%xarch,<BLEND>) NE 0)
IFDEF X86_64

r4_x4c_simple_fft_mem MACRO R1,R2,R3,R4,R5,R6,R7,R8,pre1,pre2
	xload	xmm0, R1		;; R1
	xload	xmm2, R3		;; R3
	xcopy	xmm5, xmm0		;; Copy R1
	subpd	xmm0, xmm2		;; R1 = R1 - R3 (new R3)
	addpd	xmm2, xmm5		;; R3 = R1 + R3 (new R1)

	xload	xmm4, R5		;; I1
	xload	xmm6, R7		;; I3
	xcopy	xmm5, xmm4		;; Copy I1
	subpd	xmm4, xmm6		;; I1 = I1 - I3 (new I3)
	addpd	xmm6, xmm5		;; I3 = I1 + I3 (new I1)

	xprefetchw [pre1]

	xload	xmm1, R2		;; R2
	xload	xmm3, R4		;; R4
	xcopy	xmm5, xmm1		;; Copy R2
	subpd	xmm1, xmm3		;; R2 = R2 - R4 (new R4)
	addpd	xmm3, xmm5		;; R4 = R2 + R4 (new R2)

	xload	xmm5, R6		;; I2
	xload	xmm7, R8		;; I4
	xcopy	xmm8, xmm5		;; Copy I2
	subpd	xmm5, xmm7		;; I2 = I2 - I4 (new I4)
	addpd	xmm7, xmm8		;; I4 = I2 + I4 (new I2)

	xprefetchw [pre1][pre2]

	xcopy	xmm8, xmm4		;; Copy I3
	subpd	xmm4, xmm1		;; I3 = I3 - R4 (final I4)
	addpd	xmm1, xmm8		;; R4 = I3 + R4 (final I3)

	xcopy	xmm8, xmm0		;; Copy R3
	subpd	xmm0, xmm5		;; R3 = R3 - I4 (final R3)
	addpd	xmm5, xmm8		;; I4 = R3 + I4 (final R4)

	xcopy	xmm8, xmm2		;; Copy R1
	subpd	xmm2, xmm3		;; R1 = R1 - R2 (final R2)
	addpd	xmm3, xmm8		;; R2 = R1 + R2 (final R1)

	xcopy	xmm8, xmm6		;; Copy I1
	subpd	xmm6, xmm7		;; I1 = I1 - I2 (final I2)
	addpd	xmm7, xmm8		;; I2 = I1 + I2 (final I1)
	ENDM

r4_x4c_simple_fft_partial_mem MACRO r1, r2, r3, r4, r5, r6, r7, r8, mem3, mem4, mem7, mem8, pre1, pre2
	xload	r3, mem3		;; R3
	xcopy	r8, r3			;; Copy R3
	addpd	r3, r1			;; R3 = R1 + R3 (new R1)
	subpd	r1, r8			;; R1 = R1 - R3 (new R3)

	xload	r7, mem7		;; I3
	xcopy	r8, r7			;; Copy I3
	addpd	r7, r5			;; I3 = I1 + I3 (new I1)
	subpd	r5, r8			;; I1 = I1 - I3 (new I3)

	xprefetchw [pre1]

	xload	r4, mem4		;; R4
	xcopy	r8, r4			;; Copy R4
	addpd	r4, r2			;; R4 = R2 + R4 (new R2)
	subpd	r2, r8			;; R2 = R2 - R4 (new R4)

	xload	r8, mem8		;; I4
	xcopy	xmm8, r8		;; Copy I4
	addpd	r8, r6			;; I4 = I2 + I4 (new I2)
	subpd	r6, xmm8		;; I2 = I2 - I4 (new I4)

	xcopy	xmm8, r5		;; Copy I3
	subpd	r5, r2			;; I3 = I3 - R4 (final I4)
	addpd	r2, xmm8		;; R4 = I3 + R4 (final I3)

	xprefetchw [pre1][pre2]

	xcopy	xmm8, r1		;; Copy R3
	subpd	r1, r6			;; R3 = R3 - I4 (final R3)
	addpd	r6, xmm8		;; I4 = R3 + I4 (final R4)

	xcopy	xmm8, r3		;; Copy R1
	subpd	r3, r4			;; R1 = R1 - R2 (final R2)
	addpd	r4, xmm8		;; R2 = R1 + R2 (final R1)

	xcopy	xmm8, r7		;; Copy I1
	subpd	r7, r8			;; I1 = I1 - I2 (final I2)
	addpd	r8, xmm8		;; I2 = I1 + I2 (final I1)
	ENDM

r4_x4c_simple_unfft MACRO r1, r2, r3, r4, r5, r6, r7, r8, mem8, pre1, pre2, dest1
	xcopy	r8, r1			;; R1
	subpd	r1, r3			;; R1 = R1 - R2 (new R2)
	addpd	r3, r8			;; R2 = R1 + R2 (new R1)

	xcopy	r8, r7			;; R4
	subpd	r7, r5			;; R4 = R4 - R3 (new I4)
	addpd	r5, r8			;; R3 = R4 + R3 (new R3)

	xcopy	r8, r2			;; I2
	subpd	r2, r4			;; I1 = I1 - I2 (new I2)
	addpd	r4, r8			;; I2 = I1 + I2 (new I1)

	xcopy	r8, r3
	subpd	r3, r5			;; R1 = R1 - R3 (final R3)
	addpd	r5, r8			;; R3 = R1 + R3 (final R1)

	IFNB <pre1>
	xprefetchw [pre1]
	ENDIF

	xload	r8, mem8		;; I4
	xcopy	xmm8, r8		;; Copy I4
	addpd	r8, r6			;; I4 = I3 + I4 (new I3)
	subpd	r6, xmm8		;; I3 = I3 - I4 (new R4)

	xstore	dest1, r5

	xcopy	r5, r1			;; Copy R2
	subpd	r1, r6			;; R2 = R2 - R4 (final R4)
	addpd	r6, r5			;; R4 = R2 + R4 (final R2)

	xcopy	r5, r2			;; Copy I2
	subpd	r2, r7			;; I2 = I2 - I4 (final I4)
	addpd	r7, r5			;; I4 = I2 + I4 (final I2)

	IFNB <pre1>
	xprefetchw [pre1][pre2]
	ENDIF

	xcopy	r5, r4			;; Copy I1
	subpd	r4, r8			;; I1 = I1 - I3 (final I3)
	addpd	r8, r5			;; I3 = I1 + I3 (final I1)
	ENDM

ENDIF
ENDIF

;; 64-bit AMD K8 optimized versions of the above macros.  Derived from Intel 64-bit
;; version -- could probably be optimized further.

IF (@INSTR(,%xarch,<K8>) NE 0)
IFDEF X86_64

r4_x4cl_four_complex_with_square_preload MACRO
	xload	xmm15, XMM_TWO
	ENDM

r4_x4cl_four_complex_with_square MACRO srcreg,srcinc,d1,d2
	xload	xmm0, [srcreg+16]	;; I1
	xload	xmm2, [srcreg+d2+16]	;; I3
	subpd	xmm0, xmm2		;; I1 = I1 - I3 (new I3)		; 1-3

	xload	xmm3, [srcreg+d1]	;; R2
	xload	xmm5, [srcreg+d2+d1]	;; R4
	subpd	xmm3, xmm5		;; R2 = R2 - R4 (new R4)		; 2-4

	xload	xmm6, [srcreg]		;; R1
	xload	xmm8, [srcreg+d2]	;; R3
	subpd	xmm6, xmm8		;; R1 = R1 - R3 (new R3)		; 3-5

	xload	xmm9, [srcreg+d1+16]	;; I2
	xload	xmm11, [srcreg+d2+d1+16];; I4
	subpd	xmm9, xmm11		;; I2 = I2 - I4 (new I4)		; 4-6

	subpd	xmm0, xmm3		;; I3 = I3 - R4 (final I4)		; 5-7
	mulpd	xmm3, xmm15		;; R4 = R4 * 2
	addpd	xmm3, xmm0		;; R4 = I3 + R4 (final I3)		; 6-8

	subpd	xmm6, xmm9		;; R3 = R3 - I4 (final R3)		; 7-9
	mulpd	xmm9, xmm15		;; I4 = I4 * 2
	addpd	xmm9, xmm6		;; I4 = R3 + I4 (final R4)		; 8-10

	xcopy	xmm13, xmm0		;; Copy I4
	mulpd	xmm0, xmm0		;; I4 * I4				; 8-12

	addpd	xmm2, [srcreg+16]	;; I3 = I1 + I3 (new I1)		; 9-11	avail 1,14,12
	xcopy	xmm1, xmm3		;; Copy I3
	mulpd	xmm3, xmm3		;; I3 * I3				; 9-13	avail 14,12

	addpd	xmm11, [srcreg+d1+16]	;; I4 = I2 + I4 (new I2)		; 10-12	avail 14,12,10
	mulpd	xmm1, xmm6		;; I3 * R3				; 10-14

	addpd	xmm8, [srcreg]		;; R3 = R1 + R3 (new R1)		; 11-13	avail 14,12,10,7
	mulpd	xmm13, xmm9		;; I4 * R4				; 11-15

	addpd	xmm5, [srcreg+d1]	;; R4 = R2 + R4 (new R2)		; 12-14	avail 14,12,10,7,4
	mulpd	xmm6, xmm6		;; R3 * R3				; 12-16

	xprefetchw [srcreg+srcinc]

	subpd	xmm2, xmm11		;; I1 = I1 - I2 (final I2)		; 13-15	avail 14,12,10,7
	mulpd	xmm11, xmm15		;; I2 = I2 * 2
	mulpd	xmm9, xmm9		;; R4 * R4				; 13-17

	addpd	xmm11, xmm2		;; I2 = I1 + I2 (final I1)		; 14-16	avail 14,12,10,7,4
	xload	xmm10, [srcreg+48]	;;#2 I1

	subpd	xmm8, xmm5		;; R1 = R1 - R2 (final R2)		; 15-17	avail 14,12,7
	mulpd	xmm5, xmm15		;; R2 = R2 * 2
	mulpd	xmm1, xmm15		;; I3 * R3 * 2 (new I3)			; 15-19

	addpd	xmm5, xmm8		;; R2 = R1 + R2 (final R1)		; 16-18	avail 14,12,7,4
	xcopy	xmm4, xmm2		;; Copy I2
	mulpd	xmm2, xmm2		;; I2 * I2				; 16-20	avail 14,12,7

	subpd	xmm6, xmm3		;; R3^2 - I3^2 (new R3)			; 17-19	avail 14,12,7,3
	xcopy	xmm3, xmm11		;; Copy I1
	mulpd	xmm11, xmm11		;; I1 * I1				; 17-21	avail 14,12,7

	subpd	xmm9, xmm0		;; R4^2 - I4^2 (new R4)			; 18-20	avail 14,12,7,0
	mulpd	xmm4, xmm8		;; I2 * R2				; 18-22
	xload	xmm14, [srcreg+d2+48]	;;#2 I3

	xprefetchw [srcreg+srcinc+d1]

	xcopy	xmm0, xmm10		;;#2 Copy I1
	subpd	xmm10, xmm14		;;#2 I1 = I1 - I3 (new I3)		; 19-21	avail 12,7
	mulpd	xmm3, xmm5		;; I1 * R1				; 19-23

	addpd	xmm14, xmm0		;;#2 I3 = I1 + I3 (new I1)		; 20-22	avail 12,7,0
	mulpd	xmm8, xmm8		;; R2 * R2				; 20-24
	xload	xmm12, [srcreg+d1+32]	;;#2 R2

	subpd	xmm9, xmm6		;; R4 = R4 - R3 (new I4)		; 21-23	avail 7
	mulpd	xmm6, xmm15		;; R3 = R3 * 2
	mulpd	xmm5, xmm5		;; R1 * R1				; 21-25

	addpd	xmm6, xmm9		;; R3 = R4 + R3 (new R3)		; 22-24	avail 7,0
	mulpd	xmm13, xmm15		;; I4 * R4 * 2 (new I4)			; 22-26 (16)
	xload	xmm7, [srcreg+32]	;;#2 R1

	subpd	xmm12, [srcreg+d2+d1+32];;#2 R2 = R2 - R4 (new R4)		; 23-25
	mulpd	xmm4, xmm15		;; I2 * R2 * 2 (new I2)			; 23-27
	xload	xmm0, [srcreg+d2+32]	;;#2 R3

	subpd	xmm7, xmm0		;;#2 R1 = R1 - R3 (new R3)		; 24-26	avail none
	mulpd	xmm3, xmm15		;; I1 * R1 * 2 (new I1)			; 24-28

	subpd	xmm8, xmm2		;; R2^2 - I2^2 (new R2)			; 25-27	avail 2

	xprefetchw [srcreg+srcinc+d2]

	subpd	xmm5, xmm11		;; R1^2 - I1^2 (new R1)			; 26-28	avail 2,11

	addpd	xmm13, xmm1		;; I4 = I3 + I4 (new I3)		; 27-29
	mulpd	xmm1, xmm15		;; I4 = I4 * 2
	subpd	xmm1, xmm13		;; I3 = I3 - I4 (new R4)		; 28-30
	xload	xmm11, [srcreg+d1+48]	;;#2 I2						avail 2

	addpd	xmm8, xmm5		;; R2 = R1 + R2 (new R1)		; 29-31
	mulpd	xmm5, xmm15
	subpd	xmm5, xmm8		;; R1 = R1 - R2 (new R2)		; 30-32

	subpd	xmm3, xmm4		;; I1 = I1 - I2 (new I2)		; 31-33
	mulpd	xmm4, xmm15		;; I2 = I2 * 2
	addpd	xmm4, xmm3		;; I2 = I1 + I2 (new I1)		; 32-34

	subpd	xmm8, xmm6		;; R1 = R1 - R3 (final R3)		; 33-35
	mulpd	xmm6, xmm15		;; R3 = R3 * 2
	addpd	xmm6, xmm8		;; R3 = R1 + R3 (final R1)		; 34-36	avail 2 storable 8,6

	xcopy	xmm2, xmm5		;; Copy R2
	subpd	xmm5, xmm1		;; R2 = R2 - R4 (final R4)		; 35-37	avail none storable 8,6,5

	xstore	[srcreg+16], xmm8	;; Save R3				; 36
	xload	xmm8, [srcreg+d2+d1+48]	;;#2 I4
	xstore	[srcreg], xmm6		;; Save R1				; 37
	xcopy	xmm6, xmm11		;;#2 Copy I2
	subpd	xmm11, xmm8		;;#2 I2 = I2 - I4 (new I4)		; 36-38	avail none storable 5

	xstore	[srcreg+d1+16], xmm5	;; Save R4				; 38
	subpd	xmm10, xmm12		;;#2 I3 = I3 - R4 (final I4)		; 37-39	avail none
	mulpd	xmm12, xmm15		;;#2 R4 = R4 * 2

	addpd	xmm12, xmm10		;;#2 R4 = I3 + R4 (final I3)		; 38-40	avail 5

	xprefetchw [srcreg+srcinc+d2+d1]

	subpd	xmm7, xmm11		;;#2 R3 = R3 - I4 (final R3)		; 39-41	avail none
	mulpd	xmm11, xmm15		;;#2 I4 = I4 * 2

	addpd	xmm11, xmm7		;;#2 I4 = R3 + I4 (final R4)		; 40-42 avail 5
	xcopy	xmm5, xmm10		;;#2 Copy I4
	mulpd	xmm10, xmm10		;;#2 I4 * I4				; 40-44 avail none

	addpd	xmm1, xmm2		;; R4 = R2 + R4 (final R2)		; 41-43	avail 2 storable 1
	xcopy	xmm2, xmm12		;;#2 Copy I3
	mulpd	xmm12, xmm12		;;#2 I3 * I3				; 41-45	storable 1

	addpd	xmm6, xmm8		;;#2 I4 = I2 + I4 (new I2)		; 42-44	avail 8 storable 1
	mulpd	xmm2, xmm7		;;#2 I3 * R3				; 42-46

	addpd	xmm0, [srcreg+32]	;;#2 R3 = R1 + R3 (new R1)		; 43-45
	mulpd	xmm5, xmm11		;;#2 I4 * R4				; 43-47

	xload	xmm8, [srcreg+d1+32]	;;#2 R2
	addpd	xmm8, [srcreg+d2+d1+32]	;;#2 R4 = R2 + R4 (new R2)		; 44-46
	mulpd	xmm7, xmm7		;;#2 R3 * R3				; 44-48
	xstore	[srcreg+d1], xmm1	;; Save R2				; 44	avail 1

	subpd	xmm14, xmm6		;;#2 I1 = I1 - I2 (final I2)		; 45-47	avail none
	mulpd	xmm6, xmm15		;;#2 I2 = I2 * 2
	mulpd	xmm11, xmm11		;;#2 R4 * R4				; 45-49

	addpd	xmm6, xmm14		;;#2 I2 = I1 + I2 (final I1)		; 46-48	avail 1

	subpd	xmm0, xmm8		;;#2 R1 = R1 - R2 (final R2)		; 47-49	avail none
	mulpd	xmm8, xmm15		;;#2 R2 = R2 * 2
	mulpd	xmm2, xmm15		;;#2 I3 * R3 * 2 (new I3)		; 47-51

	addpd	xmm8, xmm0		;;#2 R2 = R1 + R2 (final R1)		; 48-50	avail 1
	xcopy	xmm1, xmm14		;;#2 Copy I2
	mulpd	xmm14, xmm14		;;#2 I2 * I2				; 48-52	avail none

	subpd	xmm7, xmm12		;;#2 R3^2 - I3^2 (new R3)		; 49-51	avail 12
	xcopy	xmm12, xmm6		;;#2 Copy I1
	mulpd	xmm6, xmm6		;;#2 I1 * I1				; 49-53	avail none

	subpd	xmm11, xmm10		;;#2 R4^2 - I4^2 (new R4)		; 50-52	avail 10
	mulpd	xmm1, xmm0		;;#2 I2 * R2				; 50-54

	subpd	xmm3, xmm9		;; I2 = I2 - I4 (final I4)		; 51-53	avail none storable 3
	mulpd	xmm9, xmm15		;; I4 = I4 * 2
	mulpd	xmm12, xmm8		;;#2 I1 * R1				; 51-55

	addpd	xmm9, xmm3		;; I4 = I2 + I4 (final I2)		; 52-54	avail 10 storable 3,9
	mulpd	xmm0, xmm0		;;#2 R2 * R2				; 52-56

	subpd	xmm11, xmm7		;;#2 R4 = R4 - R3 (new I4)		; 53-55	avail none storable 3,9
	mulpd	xmm7, xmm15		;;#2 R3 = R3 * 2
	mulpd	xmm8, xmm8		;;#2 R1 * R1				; 53-57

	addpd	xmm7, xmm11		;;#2 R3 = R4 + R3 (new R3)		; 54-56	avail 10 storable 3,9
	mulpd	xmm5, xmm15		;;#2 I4 * R4 * 2 (new I4)		; 54-58
	xstore	[srcreg+d1+48], xmm3	;; Save I4				; 54	avail 10,3 storable 9

	subpd	xmm4, xmm13		;; I1 = I1 - I3 (final I3)		; 55-57	avail 10 storable 9,4
	mulpd	xmm13, xmm15		;; I3 = I3 * 2
	mulpd	xmm1, xmm15		;;#2 I2 * R2 * 2 (new I2)		; 55-59
	xstore	[srcreg+d1+32], xmm9	;; Save I2				; 55	avail 10,9 storable 4

	addpd	xmm13, xmm4		;; I3 = I1 + I3 (final I1)		; 56-58	avail 10,9,3 storable 4,13
	mulpd	xmm12, xmm15		;;#2 I1 * R1 * 2 (new I1)		; 56-60

	subpd	xmm0, xmm14		;;#2 R2^2 - I2^2 (new R2)		; 57-59	avail 10,9,3,14 storable 4,13

	subpd	xmm8, xmm6		;;#2 R1^2 - I1^2 (new R1)		; 58-60	avail 10,9,3,14,6 storable 4,13
	xstore	[srcreg+48], xmm4	;; Save I3				; 58	avail 10,9,3,14,6,4 storable 13

	addpd	xmm5, xmm2		;;#2 I4 = I3 + I4 (new I3)		; 59-61
	mulpd	xmm2, xmm15		;;#2 I4 = I4 * 2
	subpd	xmm2, xmm5		;;#2 I3 = I3 - I4 (new R4)		; 60-62
	xstore	[srcreg+32], xmm13	;; Save I1				; 59	avail 10,9,3,14,6,4,13

	addpd	xmm0, xmm8		;;#2 R2 = R1 + R2 (new R1)		; 61-63
	mulpd	xmm8, xmm15		;;#2 R2 = R2 * 2
	subpd	xmm8, xmm0		;;#2 R1 = R1 - R2 (new R2)		; 62-64

	subpd	xmm12, xmm1		;;#2 I1 = I1 - I2 (new I2)		; 63-65
	mulpd	xmm1, xmm15		;;#2 I2 = I2 * 2
	addpd	xmm1, xmm12		;;#2 I2 = I1 + I2 (new I1)		; 64-66

	subpd	xmm0, xmm7		;;#2 R1 = R1 - R3 (final R3)		; 65-67
	mulpd	xmm7, xmm15		;;#2 R3 = R3 * 2
	addpd	xmm7, xmm0		;;#2 R3 = R1 + R3 (final R1)		; 66-68	avail 10,9,3,14,6,4,13 storable 0,7

	subpd	xmm8, xmm2		;;#2 R2 = R2 - R4 (final R4)		; 67-69
	mulpd	xmm2, xmm15		;;#2 R4 = R4 * 2
	addpd	xmm2, xmm8		;;#2 R4 = R2 + R4 (final R2)		; 68-70	avail 10,9,3,14,6,4,13 storable 0,7,8,2
	xstore	[srcreg+d2+16], xmm0	;;#2 Save R3				; 68

	subpd	xmm12, xmm11		;;#2 I2 = I2 - I4 (final I4)		; 69-71	avail 10,9,14,6,4,13,0 storable 7,8,2,12
	mulpd	xmm11, xmm15		;;#2 I4 = I4 * 2
	xstore	[srcreg+d2], xmm7	;;#2 Save R1				; 69
	addpd	xmm11, xmm12		;;#2 I4 = I2 + I4 (final I2)		; 70-72	avail 10,9,3,14,6,4,13,0,7 storable 8,2,12,11
	xstore	[srcreg+d2+d1+16], xmm8	;;#2 Save R4				; 70

	subpd	xmm1, xmm5		;;#2 I1 = I1 - I3 (final I3)		; 71-73
	mulpd	xmm5, xmm15		;;#2 I3 = I3 * 2
	xstore	[srcreg+d2+d1], xmm2	;;#2 Save R2				; 71
	addpd	xmm5, xmm1		;;#2 I3 = I1 + I3 (final I1)		; 72-74	avail 10,9,3,14,6,4,13,0,7,8,2 storable,12,11,1,5
	xstore	[srcreg+d2+d1+48], xmm12 ;;#2 Save I4				; 72

	xstore	[srcreg+d2+d1+32], xmm11 ;;#2 Save I2				; 73
	xstore	[srcreg+d2+48], xmm1	;;#2 Save I3				; 74
	xstore	[srcreg+d2+32], xmm5	;;#2 Save I1				; 75

	bump	srcreg, srcinc
	ENDM

ENDIF
ENDIF

;; 64-bit Intel and AMD K10 optimized versions of the above macros.

IF (@INSTR(,%xarch,<CORE>) NE 0) OR (@INSTR(,%xarch,<P4>) NE 0) OR (@INSTR(,%xarch,<K10>) NE 0) OR (@INSTR(,%xarch,<BLEND>) NE 0)
IFDEF X86_64

r4_x4cl_four_complex_fft_final MACRO srcreg,srcinc,d1,d2
	xload	xmm15, [srcreg]		;; R1
	xload	xmm2, [srcreg+d2]	;; R3
	xcopy	xmm0, xmm15		;; Copy R1
	subpd	xmm15, xmm2		;; R1 = R1 - R3 (new R3)		; 1-3
	addpd	xmm2, xmm0		;; R3 = R1 + R3 (new R1)		; 2-4

	xload	xmm4, [srcreg+16]	;; I1
	xload	xmm6, [srcreg+d2+16]	;; I3
	xcopy	xmm0, xmm4		;; Copy I1
	subpd	xmm4, xmm6		;; I1 = I1 - I3 (new I3)		; 3-5
	addpd	xmm6, xmm0		;; I3 = I1 + I3 (new I1)		; 4-6

	xprefetchw [srcreg+srcinc]

	xload	xmm1, [srcreg+d1]	;; R2
	xload	xmm3, [srcreg+d2+d1]	;; R4
	xcopy	xmm0, xmm1		;; Copy R2
	subpd	xmm1, xmm3		;; R2 = R2 - R4 (new R4)		; 5-7
	addpd	xmm3, xmm0		;; R4 = R2 + R4 (new R2)		; 6-8

	xload	xmm5, [srcreg+d1+16]	;; I2
	xload	xmm7, [srcreg+d2+d1+16]	;; I4
	xcopy	xmm0, xmm5		;; Copy I2
	subpd	xmm5, xmm7		;; I2 = I2 - I4 (new I4)		; 7-9
	addpd	xmm7, xmm0		;; I4 = I2 + I4 (new I2)		; 8-10

	xprefetchw [srcreg+srcinc+d1]

	xcopy	xmm0, xmm4		;; Copy I3
	subpd	xmm4, xmm1		;; I3 = I3 - R4 (final I4)		; 9-11
	addpd	xmm1, xmm0		;; R4 = I3 + R4 (final I3)		; 10-12

	xload	xmm12, [srcreg+d2+d1+48];; I4
	xload	xmm13, [srcreg+d1+48]	;; R6
	xcopy	xmm0, xmm12		;; Copy I4
	addpd	xmm12, xmm13		;; I4 = I2 + I4 (new I2)		; 11-13
	subpd	xmm13, xmm0		;; I2 = I2 - I4 (new I4)		; 12-14
	xstore	[srcreg+d1+48], xmm4	;; Save I4				; 12

	xcopy	xmm0, xmm2		;; Copy R1
	subpd	xmm2, xmm3		;; R1 = R1 - R2 (final R2)		; 13-15
	xstore	[srcreg+d1+16], xmm1	;; Save I3				; 13
	addpd	xmm3, xmm0		;; R2 = R1 + R2 (final R1)		; 14-16

	xload	xmm4, [srcreg+d2+32]	;; R3
	xload	xmm1, [srcreg+32]	;; R1
	xcopy	xmm0, xmm4		;; Copy R3
	addpd	xmm4, xmm1		;; R3 = R1 + R3 (new R1)		; 15-17
	subpd	xmm1, xmm0		;; R1 = R1 - R3 (new R3)		; 16-18
	xstore	[srcreg+32], xmm2	;; Save R2				; 16

	xprefetchw [srcreg+srcinc+d2]

	xcopy	xmm0, xmm6		;; Copy I1
	subpd	xmm6, xmm7		;; I1 = I1 - I2 (final I2)		; 17-19
	xstore	[srcreg], xmm3		;; Save R1				; 17
	addpd	xmm7, xmm0		;; I2 = I1 + I2 (final I1)		; 18-20

	xload	xmm2, [srcreg+d2+48]	;; I3
	xload	xmm3, [srcreg+48]	;; R5
	xcopy	xmm0, xmm2		;; Copy I3
	addpd	xmm2, xmm3		;; I3 = I1 + I3 (new I1)		; 19-21
	subpd	xmm3, xmm0		;; I1 = I1 - I3 (new I3)		; 20-22
	xstore	[srcreg+48], xmm6	;; Save I2				; 20

	xcopy	xmm0, xmm15		;; Copy R3
	subpd	xmm15, xmm5		;; R3 = R3 - I4 (final R3)		; 21-23
	xstore	[srcreg+16], xmm7	;; Save I1				; 21
	addpd	xmm5, xmm0		;; I4 = R3 + I4 (final R4)		; 22-24

	xload	xmm6, [srcreg+d2+d1+32]	;; R4
	xload	xmm7, [srcreg+d1+32]	;; R2
	xcopy	xmm0, xmm6		;; Copy R4
	addpd	xmm6, xmm7		;; R4 = R2 + R4 (new R2)		; 23-25
	subpd	xmm7, xmm0		;; R2 = R2 - R4 (new R4)		; 24-26
	xstore	[srcreg+d1], xmm15	;; Save R3				; 24

	xcopy	xmm0, xmm3		;; Copy I3
	subpd	xmm3, xmm7		;; I3 = I3 - R4 (final I4)		; 25-27
	addpd	xmm7, xmm0		;; R4 = I3 + R4 (final I3)		; 26-28
	xstore	[srcreg+d1+32], xmm5	;; Save R4				; 25

	xprefetchw [srcreg+srcinc+d2+d1]

	xcopy	xmm0, xmm1		;; Copy R3
	subpd	xmm1, xmm13		;; R3 = R3 - I4 (final R3)		; 27-29
	addpd	xmm13, xmm0		;; I4 = R3 + I4 (final R4)		; 28-30
	xstore	[srcreg+d2+d1+48], xmm3	;; Save I4				; 28

	xcopy	xmm0, xmm4		;; Copy R1
	subpd	xmm4, xmm6		;; R1 = R1 - R2 (final R2)		; 29-31
	xstore	[srcreg+d2+d1+16], xmm7	;; Save I3				; 29
	addpd	xmm6, xmm0		;; R2 = R1 + R2 (final R1)		; 30-32
	xstore	[srcreg+d2+d1], xmm1	;; Save R3				; 30

	xcopy	xmm0, xmm2		;; Copy I1
	subpd	xmm2, xmm12		;; I1 = I1 - I2 (final I2)		; 31-33
	xstore	[srcreg+d2+d1+32], xmm13 ;; Save R4				; 31
	addpd	xmm12, xmm0		;; I2 = I1 + I2 (final I1)		; 32-34

	xstore	[srcreg+d2+32], xmm4	;; Save R2				; 32
	xstore	[srcreg+d2], xmm6	;; Save R1				; 33
	xstore	[srcreg+d2+48], xmm2	;; Save I2				; 34
	xstore	[srcreg+d2+16], xmm12	;; Save I1				; 35
	bump	srcreg, srcinc
	ENDM

r4_x4cl_four_complex_with_square_preload MACRO
	xload	xmm15, XMM_TWO
	ENDM

r4_x4cl_four_complex_with_square MACRO srcreg,srcinc,d1,d2
	xload	xmm0, [srcreg+16]	;; I1
	xload	xmm1, [srcreg+d2+16]	;; I3
	xcopy	xmm2, xmm0		;; Copy I1
	subpd	xmm0, xmm1		;; I1 = I1 - I3 (new I3)		; 1-3

	xload	xmm3, [srcreg+d1]	;; R2
	xload	xmm4, [srcreg+d2+d1]	;; R4
	xcopy	xmm5, xmm3		;; Copy R2
	subpd	xmm3, xmm4		;; R2 = R2 - R4 (new R4)		; 2-4

	xload	xmm6, [srcreg]		;; R1
	xload	xmm7, [srcreg+d2]	;; R3
	xcopy	xmm8, xmm6		;; Copy R1
	subpd	xmm6, xmm7		;; R1 = R1 - R3 (new R3)		; 3-5

	xload	xmm9, [srcreg+d1+16]	;; I2
	xload	xmm10, [srcreg+d2+d1+16];; I4
	xcopy	xmm11, xmm9		;; Copy I2
	subpd	xmm9, xmm10		;; I2 = I2 - I4 (new I4)		; 4-6

	xcopy	xmm12, xmm0		;; Copy I3
	subpd	xmm0, xmm3		;; I3 = I3 - R4 (final I4)		; 5-7

	addpd	xmm3, xmm12		;; R4 = I3 + R4 (final I3)		; 6-8

	xcopy	xmm12, xmm6		;; Copy R3
	subpd	xmm6, xmm9		;; R3 = R3 - I4 (final R3)		; 7-9

	addpd	xmm9, xmm12		;; I4 = R3 + I4 (final R4)		; 8-10
	xcopy	xmm13, xmm0		;; Copy I4
	mulpd	xmm0, xmm0		;; I4 * I4				; 8-12

	addpd	xmm2, xmm1		;; I3 = I1 + I3 (new I1)		; 9-11	avail 1,14,12
	xcopy	xmm1, xmm3		;; Copy I3
	mulpd	xmm3, xmm3		;; I3 * I3				; 9-13	avail 14,12

	addpd	xmm11, xmm10		;; I4 = I2 + I4 (new I2)		; 10-12	avail 14,12,10
	mulpd	xmm1, xmm6		;; I3 * R3				; 10-14

	addpd	xmm8, xmm7		;; R3 = R1 + R3 (new R1)		; 11-13	avail 14,12,10,7
	mulpd	xmm13, xmm9		;; I4 * R4				; 11-15

	addpd	xmm5, xmm4		;; R4 = R2 + R4 (new R2)		; 12-14	avail 14,12,10,7,4
	mulpd	xmm6, xmm6		;; R3 * R3				; 12-16

	xprefetchw [srcreg+srcinc]

	xcopy	xmm4, xmm2		;; Copy I1
	subpd	xmm2, xmm11		;; I1 = I1 - I2 (final I2)		; 13-15	avail 14,12,10,7
	mulpd	xmm9, xmm9		;; R4 * R4				; 13-17

	addpd	xmm11, xmm4		;; I2 = I1 + I2 (final I1)		; 14-16	avail 14,12,10,7,4
	xload	xmm10, [srcreg+48]	;;#2 I1

	xcopy	xmm4, xmm8		;; Copy R1
	subpd	xmm8, xmm5		;; R1 = R1 - R2 (final R2)		; 15-17	avail 14,12,7
	mulpd	xmm1, xmm15		;; I3 * R3 * 2 (new I3)			; 15-19

	addpd	xmm5, xmm4		;; R2 = R1 + R2 (final R1)		; 16-18	avail 14,12,7,4
	xcopy	xmm4, xmm2		;; Copy I2
	mulpd	xmm2, xmm2		;; I2 * I2				; 16-20	avail 14,12,7

	subpd	xmm6, xmm3		;; R3^2 - I3^2 (new R3)			; 17-19	avail 14,12,7,3
	xcopy	xmm3, xmm11		;; Copy I1
	mulpd	xmm11, xmm11		;; I1 * I1				; 17-21	avail 14,12,7

	subpd	xmm9, xmm0		;; R4^2 - I4^2 (new R4)			; 18-20	avail 14,12,7,0
	mulpd	xmm4, xmm8		;; I2 * R2				; 18-22
	xload	xmm14, [srcreg+d2+48]	;;#2 I3

	xprefetchw [srcreg+srcinc+d1]

	xcopy	xmm0, xmm10		;;#2 Copy I1
	subpd	xmm10, xmm14		;;#2 I1 = I1 - I3 (new I3)		; 19-21	avail 12,7
	mulpd	xmm3, xmm5		;; I1 * R1				; 19-23

	addpd	xmm14, xmm0		;;#2 I3 = I1 + I3 (new I1)		; 20-22	avail 12,7,0
	mulpd	xmm8, xmm8		;; R2 * R2				; 20-24
	xload	xmm12, [srcreg+d1+32]	;;#2 R2

	xcopy	xmm0, xmm9		;; Copy R4
	subpd	xmm9, xmm6		;; R4 = R4 - R3 (new I4)		; 21-23	avail 7
	mulpd	xmm5, xmm5		;; R1 * R1				; 21-25

	addpd	xmm6, xmm0		;; R3 = R4 + R3 (new R3)		; 22-24	avail 7,0
	mulpd	xmm13, xmm15		;; I4 * R4 * 2 (new I4)			; 22-26 (16)
	xload	xmm7, [srcreg+32]	;;#2 R1

	subpd	xmm12, [srcreg+d2+d1+32];;#2 R2 = R2 - R4 (new R4)		; 23-25
	mulpd	xmm4, xmm15		;; I2 * R2 * 2 (new I2)			; 23-27
	xload	xmm0, [srcreg+d2+32]	;;#2 R3

	subpd	xmm7, xmm0		;;#2 R1 = R1 - R3 (new R3)		; 24-26	avail none
	mulpd	xmm3, xmm15		;; I1 * R1 * 2 (new I1)			; 24-28

	subpd	xmm8, xmm2		;; R2^2 - I2^2 (new R2)			; 25-27	avail 2

	xprefetchw [srcreg+srcinc+d2]

	subpd	xmm5, xmm11		;; R1^2 - I1^2 (new R1)			; 26-28	avail 2,11

	xcopy	xmm2, xmm13		;; Copy I4
	addpd	xmm13, xmm1		;; I4 = I3 + I4 (new I3)		; 27-29
	subpd	xmm1, xmm2		;; I3 = I3 - I4 (new R4)		; 28-30
	xload	xmm11, [srcreg+d1+48]	;;#2 I2						avail 2

	xcopy	xmm2, xmm8		;; Copy R2
	addpd	xmm8, xmm5		;; R2 = R1 + R2 (new R1)		; 29-31
	subpd	xmm5, xmm2		;; R1 = R1 - R2 (new R2)		; 30-32

	xcopy	xmm2, xmm3		;; Copy I1
	subpd	xmm3, xmm4		;; I1 = I1 - I2 (new I2)		; 31-33
	addpd	xmm4, xmm2		;; I2 = I1 + I2 (new I1)		; 32-34

	xcopy	xmm2, xmm8		;; Copy R1
	subpd	xmm8, xmm6		;; R1 = R1 - R3 (final R3)		; 33-35
	addpd	xmm6, xmm2		;; R3 = R1 + R3 (final R1)		; 34-36	avail 2 storable 8,6

	xcopy	xmm2, xmm5		;; Copy R2
	subpd	xmm5, xmm1		;; R2 = R2 - R4 (final R4)		; 35-37	avail none storable 8,6,5

	xstore	[srcreg+16], xmm8	;; Save R3				; 36
	xload	xmm8, [srcreg+d2+d1+48]	;;#2 I4
	xstore	[srcreg], xmm6		;; Save R1				; 37
	xcopy	xmm6, xmm11		;;#2 Copy I2
	subpd	xmm11, xmm8		;;#2 I2 = I2 - I4 (new I4)		; 36-38	avail none storable 5

	xstore	[srcreg+d1+16], xmm5	;; Save R4				; 38
	xcopy	xmm5, xmm10		;;#2 Copy I3
	subpd	xmm10, xmm12		;;#2 I3 = I3 - R4 (final I4)		; 37-39	avail none

	addpd	xmm12, xmm5		;;#2 R4 = I3 + R4 (final I3)		; 38-40	avail 5

	xprefetchw [srcreg+srcinc+d2+d1]

	xcopy	xmm5, xmm7		;;#2 Copy R3
	subpd	xmm7, xmm11		;;#2 R3 = R3 - I4 (final R3)		; 39-41	avail none

	addpd	xmm11, xmm5		;;#2 I4 = R3 + I4 (final R4)		; 40-42 avail 5
	xcopy	xmm5, xmm10		;;#2 Copy I4
	mulpd	xmm10, xmm10		;;#2 I4 * I4				; 40-44 avail none

	addpd	xmm1, xmm2		;; R4 = R2 + R4 (final R2)		; 41-43	avail 2 storable 1
	xcopy	xmm2, xmm12		;;#2 Copy I3
	mulpd	xmm12, xmm12		;;#2 I3 * I3				; 41-45	storable 1

	addpd	xmm6, xmm8		;;#2 I4 = I2 + I4 (new I2)		; 42-44	avail 8 storable 1
	mulpd	xmm2, xmm7		;;#2 I3 * R3				; 42-46

	addpd	xmm0, [srcreg+32]	;;#2 R3 = R1 + R3 (new R1)		; 43-45
	mulpd	xmm5, xmm11		;;#2 I4 * R4				; 43-47

	xload	xmm8, [srcreg+d1+32]	;;#2 R2
	addpd	xmm8, [srcreg+d2+d1+32]	;;#2 R4 = R2 + R4 (new R2)		; 44-46
	mulpd	xmm7, xmm7		;;#2 R3 * R3				; 44-48
	xstore	[srcreg+d1], xmm1	;; Save R2				; 44	avail 1

	xcopy	xmm1, xmm14		;;#2 Copy I1
	subpd	xmm14, xmm6		;;#2 I1 = I1 - I2 (final I2)		; 45-47	avail none
	mulpd	xmm11, xmm11		;;#2 R4 * R4				; 45-49

	addpd	xmm6, xmm1		;;#2 I2 = I1 + I2 (final I1)		; 46-48	avail 1

	xcopy	xmm1, xmm0		;;#2 Copy R1
	subpd	xmm0, xmm8		;;#2 R1 = R1 - R2 (final R2)		; 47-49	avail none
	mulpd	xmm2, xmm15		;;#2 I3 * R3 * 2 (new I3)		; 47-51

	addpd	xmm8, xmm1		;;#2 R2 = R1 + R2 (final R1)		; 48-50	avail 1
	xcopy	xmm1, xmm14		;;#2 Copy I2
	mulpd	xmm14, xmm14		;;#2 I2 * I2				; 48-52	avail none

	subpd	xmm7, xmm12		;;#2 R3^2 - I3^2 (new R3)		; 49-51	avail 12
	xcopy	xmm12, xmm6		;;#2 Copy I1
	mulpd	xmm6, xmm6		;;#2 I1 * I1				; 49-53	avail none

	subpd	xmm11, xmm10		;;#2 R4^2 - I4^2 (new R4)		; 50-52	avail 10
	mulpd	xmm1, xmm0		;;#2 I2 * R2				; 50-54

	xcopy	xmm10, xmm3		;; Copy I2
	subpd	xmm3, xmm9		;; I2 = I2 - I4 (final I4)		; 51-53	avail none storable 3
	mulpd	xmm12, xmm8		;;#2 I1 * R1				; 51-55

	addpd	xmm9, xmm10		;; I4 = I2 + I4 (final I2)		; 52-54	avail 10 storable 3,9
	mulpd	xmm0, xmm0		;;#2 R2 * R2				; 52-56

	xcopy	xmm10, xmm11		;;#2 Copy R4
	subpd	xmm11, xmm7		;;#2 R4 = R4 - R3 (new I4)		; 53-55	avail none storable 3,9
	mulpd	xmm8, xmm8		;;#2 R1 * R1				; 53-57

	addpd	xmm7, xmm10		;;#2 R3 = R4 + R3 (new R3)		; 54-56	avail 10 storable 3,9
	mulpd	xmm5, xmm15		;;#2 I4 * R4 * 2 (new I4)		; 54-58
	xstore	[srcreg+d1+48], xmm3	;; Save I4				; 54	avail 10,3 storable 9

	xcopy	xmm3, xmm4		;; Copy I1
	subpd	xmm4, xmm13		;; I1 = I1 - I3 (final I3)		; 55-57	avail 10 storable 9,4
	mulpd	xmm1, xmm15		;;#2 I2 * R2 * 2 (new I2)		; 55-59
	xstore	[srcreg+d1+32], xmm9	;; Save I2				; 55	avail 10,9 storable 4

	addpd	xmm13, xmm3		;; I3 = I1 + I3 (final I1)		; 56-58	avail 10,9,3 storable 4,13
	mulpd	xmm12, xmm15		;;#2 I1 * R1 * 2 (new I1)		; 56-60

	subpd	xmm0, xmm14		;;#2 R2^2 - I2^2 (new R2)		; 57-59	avail 10,9,3,14 storable 4,13

	subpd	xmm8, xmm6		;;#2 R1^2 - I1^2 (new R1)		; 58-60	avail 10,9,3,14,6 storable 4,13
	xstore	[srcreg+48], xmm4	;; Save I3				; 58	avail 10,9,3,14,6,4 storable 13

	xcopy	xmm3, xmm5		;;#2 Copy I4
	addpd	xmm5, xmm2		;;#2 I4 = I3 + I4 (new I3)		; 59-61
	subpd	xmm2, xmm3		;;#2 I3 = I3 - I4 (new R4)		; 60-62
	xstore	[srcreg+32], xmm13	;; Save I1				; 59	avail 10,9,3,14,6,4,13

	xcopy	xmm3, xmm0		;;#2 Copy R2
	addpd	xmm0, xmm8		;;#2 R2 = R1 + R2 (new R1)		; 61-63
	subpd	xmm8, xmm3		;;#2 R1 = R1 - R2 (new R2)		; 62-64

	xcopy	xmm3, xmm12		;;#2 Copy I1
	subpd	xmm12, xmm1		;;#2 I1 = I1 - I2 (new I2)		; 63-65
	addpd	xmm1, xmm3		;;#2 I2 = I1 + I2 (new I1)		; 64-66

	xcopy	xmm3, xmm0		;;#2 Copy R1
	subpd	xmm0, xmm7		;;#2 R1 = R1 - R3 (final R3)		; 65-67
	addpd	xmm7, xmm3		;;#2 R3 = R1 + R3 (final R1)		; 66-68	avail 10,9,3,14,6,4,13 storable 0,7

	xcopy	xmm3, xmm8		;;#2 Copy R2
	subpd	xmm8, xmm2		;;#2 R2 = R2 - R4 (final R4)		; 67-69
	addpd	xmm2, xmm3		;;#2 R4 = R2 + R4 (final R2)		; 68-70	avail 10,9,3,14,6,4,13 storable 0,7,8,2
	xstore	[srcreg+d2+16], xmm0	;;#2 Save R3				; 68

	xcopy	xmm3, xmm12		;;#2 Copy I2
	subpd	xmm12, xmm11		;;#2 I2 = I2 - I4 (final I4)		; 69-71	avail 10,9,14,6,4,13,0 storable 7,8,2,12
	xstore	[srcreg+d2], xmm7	;;#2 Save R1				; 69
	addpd	xmm11, xmm3		;;#2 I4 = I2 + I4 (final I2)		; 70-72	avail 10,9,3,14,6,4,13,0,7 storable 8,2,12,11
	xstore	[srcreg+d2+d1+16], xmm8	;;#2 Save R4				; 70

	xcopy	xmm3, xmm1		;;#2 Copy I1
	subpd	xmm1, xmm5		;;#2 I1 = I1 - I3 (final I3)		; 71-73
	xstore	[srcreg+d2+d1], xmm2	;;#2 Save R2				; 71
	addpd	xmm5, xmm3		;;#2 I3 = I1 + I3 (final I1)		; 72-74	avail 10,9,3,14,6,4,13,0,7,8,2 storable,12,11,1,5
	xstore	[srcreg+d2+d1+48], xmm12 ;;#2 Save I4				; 72

	xstore	[srcreg+d2+d1+32], xmm11 ;;#2 Save I2				; 73
	xstore	[srcreg+d2+48], xmm1	;;#2 Save I3				; 74
	xstore	[srcreg+d2+32], xmm5	;;#2 Save I1				; 75

	bump	srcreg, srcinc
	ENDM

r4_x4cl_four_complex_with_mult MACRO srcreg,srcinc,d1,d2
	xload	xmm1, [srcreg+16]		;; I1
	xload	xmm2, [srcreg+d2+16]		;; I3
	xcopy	xmm3, xmm1			;; Copy I1
	subpd	xmm1, xmm2			;; I1 = I1 - I3 (new I3)	; 1-3

	xload	xmm4, [srcreg+d1]		;; R2
	xload	xmm5, [srcreg+d2+d1]		;; R4
	xcopy	xmm6, xmm4			;; Copy R2
	subpd	xmm4, xmm5			;; R2 = R2 - R4 (new R4)	; 2-4

	xload	xmm7, [srcreg]			;; R1
	xload	xmm8, [srcreg+d2]		;; R3
	xcopy	xmm9, xmm7			;; Copy R1
	subpd	xmm7, xmm8			;; R1 = R1 - R3 (new R3)	; 3-5

	xload	xmm10, [srcreg+d1+16]		;; I2
	xload	xmm11, [srcreg+d2+d1+16]	;; I4
	xcopy	xmm12, xmm10			;; Copy I2
	subpd	xmm10, xmm11			;; I2 = I2 - I4 (new I4)	; 4-6	avail 0,13+

	xcopy	xmm0, xmm1			;; Copy I3
	addpd	xmm1, xmm4			;; R4 = I3 + R4 (final I3)	; 5-7

	subpd	xmm0, xmm4			;; I3 = I3 - R4 (final I4)	; 6-8	avail 4,13+
	xload	xmm13, [srcreg+d1+16][rbp]	;; Load I3-from-mem

	xcopy	xmm4, xmm7			;; Copy R3
	subpd	xmm7, xmm10			;; R3 = R3 - I4 (final R3)	; 7-9
	xload	xmm14, [srcreg+d1][rbp]		;; Load R3-from-mem

	addpd	xmm10, xmm4			;; I4 = R3 + I4 (final R4)	; 8-10
	xcopy	xmm15, xmm1			;; Copy I3
	mulpd	xmm1, xmm13			;; I3 * I3-from-mem		; 8-12	avail 4

	addpd	xmm6, xmm5			;; R4 = R2 + R4 (new R2)	; 9-11	avail 4,5
	mulpd	xmm15, xmm14			;; I3 * R3-from-mem		; 9-13

	addpd	xmm9, xmm8			;; R3 = R1 + R3 (new R1)	; 10-12	avail 4,5,8
	mulpd	xmm14, xmm7			;; R3 * R3-from-mem		; 10-14
	xload	xmm5, [srcreg+d1+48][rbp]	;; Load I4-from-mem

	addpd	xmm12, xmm11			;; I4 = I2 + I4 (new I2)	; 11-13	avail 4,8,11
	mulpd	xmm7, xmm13			;; R3 * I3-from-mem		; 11-15	avail 4,8,11,13
	xload	xmm11, [srcreg+d1+32][rbp]	;; Load R4-from-mem

	addpd	xmm3, xmm2			;; I3 = I1 + I3 (new I1)	; 12-14	avail 4,8,13,2
	xcopy	xmm8, xmm0			;; Copy I4
	mulpd	xmm0, xmm5			;; I4 * I4-from-mem		; 12-16	avail 4,13,2

	xcopy	xmm4, xmm9			;; Copy R1
	subpd	xmm9, xmm6			;; R1 = R1 - R2 (final R2)	; 13-15	avail 13,2
	mulpd	xmm8, xmm11			;; I4 * R4-from-mem		; 13-17

	addpd	xmm6, xmm4			;; R2 = R1 + R2 (final R1)	; 14-16	avail 4,13,2
	mulpd	xmm11, xmm10			;; R4 * R4-from-mem		; 14-18
	xload	xmm13, [srcreg+32][rbp]		;; Load R2-from-mem

	xcopy	xmm4, xmm3			;; Copy I1
	subpd	xmm3, xmm12			;; I1 = I1 - I2 (final I2)	; 15-17	avail 2
	mulpd	xmm10, xmm5			;; R4 * I4-from-mem		; 15-19	avail 2,5

	xprefetchw [srcreg+srcinc]

	addpd	xmm12, xmm4			;; I2 = I1 + I2 (final I1)	; 16-18	avail 4,2,5
	xcopy	xmm2, xmm9			;; Copy R2
	mulpd	xmm9, xmm13			;; R2 * R2-from-mem		; 16-20	avail 4,5

	subpd	xmm14, xmm1			;; R3*R3-I3*I3 (new R3)		; 17-19	avail 4,5,1
	xload	xmm5, [srcreg+48][rbp]		;; Load I2-from-mem
	mulpd	xmm2, xmm5			;; R2 * I2-from-mem		; 17-21	avail 4,1

	addpd	xmm7, xmm15			;; R3*I3+I3*R3 (new I3)		; 18-20	avail 4,1,15
	mulpd	xmm5, xmm3			;; I2 * I2-from-mem		; 18-22
	xload	xmm1, [srcreg][rbp]		;; Load R1-from-mem

	subpd	xmm11, xmm0			;; R4*R4-I4*I4 (new R4)		; 19-21	avail 4,15,0
	mulpd	xmm3, xmm13			;; I2 * R2-from-mem		; 19-23	avail 4,15,0,13
	xload	xmm15, [srcreg+48]		;;#2 I1

	addpd	xmm10, xmm8			;; R4*I4+I4*R4 (new I4)		; 20-22	avail 4,0,13,8
	xcopy	xmm0, xmm6			;; Copy R1
	mulpd	xmm6, xmm1			;; R1 * R1-from-mem		; 20-24	avail 4,13,8

	xprefetchw [srcreg+srcinc+d1]

	xload	xmm13, [srcreg+d2+48]		;;#2 I3
	xcopy	xmm8, xmm15			;;#2 Copy I1
	subpd	xmm15, xmm13			;;#2 I1 = I1 - I3 (new I3)	; 21-23	avail 4
	xload	xmm4, [srcreg+16][rbp]		;; Load I1-from-mem
	mulpd	xmm0, xmm4			;; R1 * I1-from-mem		; 21-25	avail none

	addpd	xmm8, xmm13			;;#2 I3 = I1 + I3 (new I1)	; 22-24 avail 13
	mulpd	xmm4, xmm12			;; I1 * I1-from-mem		; 22-26

	subpd	xmm9, xmm5			;; R2*R2-I2*I2 (new R2)		; 23-25 avail 13,5
	mulpd	xmm1, xmm12			;; I1 * R1-from-mem		; 23-27 avail 13,5,12

	addpd	xmm2, xmm3			;; R2*I2+I2*R2 (new I2)		; 24-26 avail 13,5,12,3

	xcopy	xmm5, xmm11			;; Copy R4
	subpd	xmm11, xmm14			;; R4 = R4 - R3 (new I4)	; 25-27 avail 13,12,3

	addpd	xmm14, xmm5			;; R3 = R4 + R3 (new R3)	; 26-28 avail 5,12,3
	xload	xmm13, [srcreg+d1+32]		;;#2 R2

	subpd	xmm6, xmm4			;; R1*R1-I1*I1 (new R1)		; 27-29 avail 5,12,3,4
	xload	xmm12, [srcreg+d2+d1+32]	;;#2 R4

	addpd	xmm0, xmm1			;; R1*I1+I1*R1 (new I1)		; 28-30 avail 5,3,4,1
	xload	xmm3, [srcreg+32]		;;#2 R1

	xcopy	xmm5, xmm10			;; Copy I4
	addpd	xmm10, xmm7			;; I4 = I3 + I4 (new I3)	; 29-31 avail 4,1

	subpd	xmm7, xmm5			;; I3 = I3 - I4 (new R4)	; 30-32 avail 5,4,1
	xload	xmm4, [srcreg+d2+32]		;;#2 R3

	xcopy	xmm5, xmm6			;; Copy R1
	subpd	xmm6, xmm9			;; R1 = R1 - R2 (new R2)	; 31-33	avail 1

	addpd	xmm9, xmm5			;; R2 = R1 + R2 (new R1)	; 32-34	avail 5,1

	xcopy	xmm5, xmm0			;; Copy I1
	subpd	xmm0, xmm2			;; I1 = I1 - I2 (new I2)	; 33-35	avail 1

	addpd	xmm2, xmm5			;; I2 = I1 + I2 (new I1)	; 34-36	avail 5,1

	xcopy	xmm5, xmm9			;; Copy R1
	subpd	xmm9, xmm14			;; R1 = R1 - R3 (final R3)	; 35-37	avail 1 storable 9

	addpd	xmm14, xmm5			;; R3 = R1 + R3 (final R1)	; 36-38	avail 5,1 storable 9,14

	xcopy	xmm5, xmm6			;; Copy R2
	subpd	xmm6, xmm7			;; R2 = R2 - R4 (final R4)	; 37-39	avail 1 storable 9,14,6

	addpd	xmm7, xmm5			;; R4 = R2 + R4 (final R2)	; 38-40	avail 5,1 storable 9,14,6,7
	xstore	[srcreg+16], xmm9		;; Save R3			; 38

	xcopy	xmm1, xmm13			;;#2 Copy R2
	subpd	xmm13, xmm12			;;#2 R2 = R2 - R4 (new R4)	; 39-41	avail 5,9 storable 14,6,7
	xstore	[srcreg], xmm14			;; Save R1			; 39
	xload	xmm14, [srcreg+d1+48]		;;#2 I2

	xcopy	xmm9, xmm3			;;#2 Copy R1
	subpd	xmm3, xmm4			;;#2 R1 = R1 - R3 (new R3)	; 40-42	avail 5 storable 6,7
	xstore	[srcreg+d1+16], xmm6		;; Save R4			; 40

	xload	xmm5, [srcreg+d2+d1+48]		;;#2 I4
	xcopy	xmm6, xmm14			;;#2 Copy I2
	subpd	xmm14, xmm5			;;#2 I2 = I2 - I4 (new I4)	; 41-43	avail none storable 7
	xstore	[srcreg+d1], xmm7		;; Save R2			; 41	avail 7

	xcopy	xmm7, xmm15			;;#2 Copy I3
	addpd	xmm15, xmm13			;;#2 R4 = I3 + R4 (final I3)	; 42-44	avail none

	subpd	xmm7, xmm13			;;#2 I3 = I3 - R4 (final I4)	; 43-45	avail 13

	xcopy	xmm13, xmm3			;;#2 Copy R3
	subpd	xmm3, xmm14			;;#2 R3 = R3 - I4 (final R3)	; 44-46

	addpd	xmm14, xmm13			;;#2 I4 = R3 + I4 (final R4)	; 45-47
	xload	xmm13, [srcreg+d2+d1+16][rbp]	;;#2 Load I3-from-mem

	addpd	xmm1, xmm12			;;#2 R4 = R2 + R4 (new R2)	; 46-48	avail 12
	xcopy	xmm12, xmm15			;;#2 Copy I3
	mulpd	xmm15, xmm13			;;#2 I3 * I3-from-mem		; 46-50	avail none

	addpd	xmm4, xmm9			;;#2 R3 = R1 + R3 (new R1)	; 47-49	avail 9
	xload	xmm9, [srcreg+d2+d1][rbp]	;;#2 Load R3-from-mem
	mulpd	xmm12, xmm9			;;#2 I3 * R3-from-mem		; 47-51	avail none

	addpd	xmm6, xmm5			;;#2 I4 = I2 + I4 (new I2)	; 48-50	avail 5
	mulpd	xmm9, xmm3			;;#2 R3 * R3-from-mem		; 48-52

	xcopy	xmm5, xmm0			;; Copy I2
	subpd	xmm0, xmm11			;; I2 = I2 - I4 (final I4)	; 49-51	avail none storable 0
	mulpd	xmm3, xmm13			;;#2 R3 * I3-from-mem		; 49-53	avail 13 storable 0
	xload	xmm13, [srcreg+d2+d1+48][rbp]	;;#2 Load I4-from-mem

	addpd	xmm11, xmm5			;; I4 = I2 + I4 (final I2)	; 50-52	avail 5 storable 0,11
	xcopy	xmm5, xmm7			;;#2 Copy I4
	mulpd	xmm7, xmm13			;;#2 I4 * I4-from-mem		; 50-54	avail none storable 0,11

	xstore	[srcreg+d1+48], xmm0		;; Save I4			; 52
	xcopy	xmm0, xmm4			;;#2 Copy R1
	subpd	xmm4, xmm1			;;#2 R1 = R1 - R2 (final R2)	; 51-53	avail none storable 11
	xstore	[srcreg+d1+32], xmm11		;; Save I2			; 53
	xload	xmm11, [srcreg+d2+d1+32][rbp]	;;#2 Load R4-from-mem
	mulpd	xmm5, xmm11			;;#2 I4 * R4-from-mem		; 51-55	avail none

	addpd	xmm1, xmm0			;;#2 R2 = R1 + R2 (final R1)	; 52-54	avail 0
	mulpd	xmm11, xmm14			;;#2 R4 * R4-from-mem		; 52-56

	xprefetchw [srcreg+srcinc+d2]

	xcopy	xmm0, xmm8			;;#2 Copy I1
	subpd	xmm8, xmm6			;;#2 I1 = I1 - I2 (final I2)	; 53-55	avail none
	mulpd	xmm14, xmm13			;;#2 R4 * I4-from-mem		; 53-57	avail 13

	addpd	xmm6, xmm0			;;#2 I2 = I1 + I2 (final I1)	; 54-56	avail 13,0
	xload	xmm13, [srcreg+d2+32][rbp]	;;#2 Load R2-from-mem
	xcopy	xmm0, xmm4			;;#2 Copy R2
	mulpd	xmm4, xmm13			;;#2 R2 * R2-from-mem		; 54-58	avail none

	subpd	xmm9, xmm15			;;#2 R3*R3-I3*I3 (new R3)	; 55-57	avail 15
	xload	xmm15, [srcreg+d2+48][rbp]	;;#2 Load I2-from-mem
	mulpd	xmm0, xmm15			;;#2 R2 * I2-from-mem		; 55-59	avail none

	addpd	xmm3, xmm12			;;#2 R3*I3+I3*R3 (new I3)	; 56-58	avail 12
	mulpd	xmm15, xmm8			;;#2 I2 * I2-from-mem		; 56-60

	subpd	xmm11, xmm7			;;#2 R4*R4-I4*I4 (new R4)	; 57-59	avail 12,7
	mulpd	xmm8, xmm13			;;#2 I2 * R2-from-mem		; 57-61	avail 12,7,13
	xload	xmm7, [srcreg+d2][rbp]		;;#2 Load R1-from-mem

	xprefetchw [srcreg+srcinc+d2+d1]

	addpd	xmm14, xmm5			;;#2 R4*I4+I4*R4 (new I4)	; 58-60	avail 12,13,5
	xcopy	xmm13, xmm1			;;#2 Copy R1
	mulpd	xmm1, xmm7			;;#2 R1 * R1-from-mem		; 58-62	avail 12,5

	xcopy	xmm5, xmm2			;; Copy I1
	subpd	xmm2, xmm10			;; I1 = I1 - I3 (final I3)	; 59-61	avail 12 storable 2
	xload	xmm12, [srcreg+d2+16][rbp]	;;#2 Load I1-from-mem	
	mulpd	xmm13, xmm12			;;#2 R1 * I1-from-mem		; 59-63	avail none

	addpd	xmm10, xmm5			;; I3 = I1 + I3 (final I1)	; 60-62	avail 5 storable 2,10
	mulpd	xmm12, xmm6			;;#2 I1 * I1-from-mem		; 60-64

	subpd	xmm4, xmm15			;;#2 R2*R2-I2*I2 (new R2)	; 61-63	avail 5,15 storable 2,10
	mulpd	xmm6, xmm7			;;#2 I1 * R1-from-mem		; 61-65	avail 5,15,7 storable 2,10

	addpd	xmm0, xmm8			;;#2 R2*I2+I2*R2 (new I2)	; 62-64	avail 5,15,7,8 storable 2,10
	xstore	[srcreg+48], xmm2		;; Save I3			; 62

	xcopy	xmm5, xmm11			;;#2 Copy R4
	subpd	xmm11, xmm9			;;#2 R4 = R4 - R3 (new I4)	; 63-65	avail 15,7,8,2 storable 10
	addpd	xmm9, xmm5			;;#2 R3 = R4 + R3 (new R3)	; 64-66	avail 5,15,7,8,2 storable 10
	xstore	[srcreg+32], xmm10		;; Save I1			; 63

	subpd	xmm1, xmm12			;;#2 R1*R1-I1*I1 (new R1)	; 65-67	avail 5,15,7,8,2,10,12

	addpd	xmm13, xmm6			;;#2 R1*I1+I1*R1 (new I1)	; 66-68	avail 5,15,7,8,2,10,12,6

	xcopy	xmm5, xmm14			;;#2 Copy I4
	addpd	xmm14, xmm3			;;#2 I4 = I3 + I4 (new I3)	; 67-69
	subpd	xmm3, xmm5			;;#2 I3 = I3 - I4 (new R4)	; 68-70

	xcopy	xmm5, xmm1			;;#2 Copy R1
	subpd	xmm1, xmm4			;;#2 R1 = R1 - R2 (new R2)	; 69-71
	addpd	xmm4, xmm5			;;#2 R2 = R1 + R2 (new R1)	; 70-72

	xcopy	xmm5, xmm13			;;#2 Copy I1
	subpd	xmm13, xmm0			;;#2 I1 = I1 - I2 (new I2)	; 71-73
	addpd	xmm0, xmm5			;;#2 I2 = I1 + I2 (new I1)	; 72-74

	xcopy	xmm5, xmm1			;;#2 Copy R2
	subpd	xmm1, xmm3			;;#2 R2 = R2 - R4 (final R4)	; 73-75
	addpd	xmm3, xmm5			;;#2 R4 = R2 + R4 (final R2)	; 74-76

	xcopy	xmm5, xmm4			;;#2 Copy R1
	subpd	xmm4, xmm9			;;#2 R1 = R1 - R3 (final R3)	; 75-77
	addpd	xmm9, xmm5			;;#2 R3 = R1 + R3 (final R1)	; 76-78
	xstore	[srcreg+d2+d1+16], xmm1		;;#2 Save R4			; 76

	xcopy	xmm5, xmm13			;;#2 Copy I2
	subpd	xmm13, xmm11			;;#2 I2 = I2 - I4 (final I4)	; 77-79
	xstore	[srcreg+d2+d1], xmm3		;;#2 Save R2			; 77
	addpd	xmm11, xmm5			;;#2 I4 = I2 + I4 (final I2)	; 78-80
	xstore	[srcreg+d2+16], xmm4		;;#2 Save R3			; 78

	xcopy	xmm5, xmm0			;;#2 Copy I1
	subpd	xmm0, xmm14			;;#2 I1 = I1 - I3 (final I3)	; 79-81
	xstore	[srcreg+d2], xmm9		;;#2 Save R1			; 79
	addpd	xmm14, xmm5			;;#2 I3 = I1 + I3 (final I1)	; 80-82
	xstore	[srcreg+d2+d1+48], xmm13	;;#2 Save I4			; 80

	xstore	[srcreg+d2+d1+32], xmm11	;;#2 Save I2			; 81
	xstore	[srcreg+d2+48], xmm0		;;#2 Save I3			; 82
	xstore	[srcreg+d2+32], xmm14		;;#2 Save I1			; 83
	bump	srcreg, srcinc
	ENDM

r4_x4cl_four_complex_with_mulf MACRO srcreg,srcinc,d1,d2
	xload	xmm1, [srcreg+d1+16][rbx]	;; I3
	xload	xmm2, [srcreg+d1+16][rbp]	;; Load I3-other
	xcopy	xmm3, xmm1			;; Copy I3
	mulpd	xmm1, xmm2			;; I3 * I3-other		; 1-5

	xload	xmm4, [srcreg+d1][rbp]		;; Load R3-other
	mulpd	xmm3, xmm4			;; I3 * R3-other		; 2-6

	xload	xmm5, [srcreg+d1][rbx]		;; R3
	mulpd	xmm4, xmm5			;; R3 * R3-other		; 3-7

	mulpd	xmm5, xmm2			;; R3 * I3-other		; 4-8
	xload	xmm6, [srcreg+d1+48][rbx]	;; I4

	xload	xmm7, [srcreg+d1+48][rbp]	;; Load I4-other
	xcopy	xmm2, xmm6			;; Copy I4
	mulpd	xmm6, xmm7			;; I4 * I4-other		; 5-9

	xload	xmm0, [srcreg+d1+32][rbp]	;; Load R4-other
	mulpd	xmm2, xmm0			;; I4 * R4-other		; 6-10

	xload	xmm8, [srcreg+d1+32][rbx]	;; R4
	mulpd	xmm0, xmm8			;; R4 * R4-other		; 7-11

	subpd	xmm4, xmm1			;; R3*R3-I3*I3 (new R3)		; 8-10	avail 1,9+
	mulpd	xmm8, xmm7			;; R4 * I4-other		; 8-12	avail 1,7,9+
	xload	xmm9, [srcreg+32][rbx]		;; R2

	addpd	xmm5, xmm3			;; R3*I3+I3*R3 (new I3)		; 9-11	avail 1,7,3,9+
	xload	xmm10, [srcreg+32][rbp]		;; Load R2-other
	xcopy	xmm11, xmm9			;; Copy R2
	mulpd	xmm9, xmm10			;; R2 * R2-other		; 9-13	avail 1,7,3,12+

	xprefetchw [srcreg+srcinc]

	xload	xmm3, [srcreg+48][rbp]		;; Load I2-other
	mulpd	xmm11, xmm3			;; R2 * I2-other		; 10-14	avail 1,7,12+

	xload	xmm7, [srcreg+48][rbx]		;; I2
	mulpd	xmm3, xmm7			;; I2 * I2-other		; 11-15	avail 1,12+

	subpd	xmm0, xmm6			;; R4*R4-I4*I4 (new R4)		; 12-14	avail 1,6,12+
	mulpd	xmm7, xmm10			;; I2 * R2-other		; 12-16	avail 1,6,10,12+
	xload	xmm6, [srcreg][rbx]		;; R1

	addpd	xmm8, xmm2			;; R4*I4+I4*R4 (new I4)		; 13-15	avail 1,2,10,12+
	xload	xmm1, [srcreg][rbp]		;; Load R1-other
	xcopy	xmm2, xmm6			;; Copy R1
	mulpd	xmm6, xmm1			;; R1 * R1-other		; 13-17	avail 10,12+

	xprefetchw [srcreg+srcinc+d1]

	xload	xmm10, [srcreg+16][rbp]		;; Load I1-other
	mulpd	xmm2, xmm10			;; R1 * I1-other		; 14-18	avail 12+

	xload	xmm12, [srcreg+16][rbx]		;; I1
	mulpd	xmm10, xmm12			;; I1 * I1-other		; 15-19	avail 13+

	subpd	xmm9, xmm3			;; R2*R2-I2*I2 (new R2)		; 16-18	avail 3,13+
	mulpd	xmm12, xmm1			;; I1 * R1-other		; 16-20	avail 3,1,13+

	addpd	xmm11, xmm7			;; R2*I2+I2*R2 (new I2)		; 17-19 avail 3,1,7,13+

	xcopy	xmm7, xmm0			;; Copy R4
	subpd	xmm0, xmm4			;; R4 = R4 - R3 (new I4)	; 18-20
	addpd	xmm4, xmm7			;; R3 = R4 + R3 (new R3)	; 19-21

	subpd	xmm6, xmm10			;; R1*R1-I1*I1 (new R1)		; 20-22 avail 3,1,7,10,13+
	xload	xmm13, [srcreg+d2+d1+16][rbx]	;;#2 I3

	addpd	xmm2, xmm12			;; R1*I1+I1*R1 (new I1)		; 21-23 avail 3,1,7,10,12,14+
	xload	xmm14, [srcreg+d2+d1+16][rbp]	;;#2 Load I3-other

	xcopy	xmm7, xmm8			;; Copy I4
	addpd	xmm8, xmm5			;; I4 = I3 + I4 (new I3)	; 22-24 avail 3,1,7,10,12,15

	subpd	xmm5, xmm7			;; I3 = I3 - I4 (new R4)	; 23-25
	xload	xmm10, [srcreg+d2+d1][rbp]	;;#2 Load R3-other

	xprefetchw [srcreg+srcinc+d2]

	xcopy	xmm7, xmm6			;; Copy R1
	subpd	xmm6, xmm9			;; R1 = R1 - R2 (new R2)	; 24-26
	xload	xmm1, [srcreg+d2+d1][rbx]	;;#2 R3

	addpd	xmm9, xmm7			;; R2 = R1 + R2 (new R1)	; 25-27
	xcopy	xmm12, xmm13			;;#2 Copy I3
	mulpd	xmm13, xmm14			;;#2 I3 * I3-other		; 25-29 avail 3,7,15

	xcopy	xmm7, xmm2			;; Copy I1
	subpd	xmm2, xmm11			;; I1 = I1 - I2 (new I2)	; 26-28 avail 3,15
	mulpd	xmm12, xmm10			;;#2 I3 * R3-other		; 26-30 avail 3,15

	xprefetchw [srcreg+srcinc+d2+d1]

	addpd	xmm11, xmm7			;; I2 = I1 + I2 (new I1)	; 27-29 avail 3,7,15
	mulpd	xmm10, xmm1			;;#2 R3 * R3-other		; 27-31
	xload	xmm15, [srcreg+d2+d1+48][rbx]	;;#2 I4

	xcopy	xmm7, xmm9			;; Copy R1
	subpd	xmm9, xmm4			;; R1 = R1 - R3 (final R3)	; 28-30 avail 3 storable 9
	mulpd	xmm1, xmm14			;;#2 R3 * I3-other		; 28-32 avail 3,14 storable 9

	addpd	xmm4, xmm7			;; R3 = R1 + R3 (final R1)	; 29-31 avail 3,7,14 storable 9,4
	xload	xmm3, [srcreg+d2+d1+48][rbp]	;;#2 Load I4-other
	xcopy	xmm14, xmm15			;;#2 Copy I4
	mulpd	xmm15, xmm3			;;#2 I4 * I4-other		; 29-33 avail 7 storable 9,4

	xcopy	xmm7, xmm6			;; Copy R2
	subpd	xmm6, xmm5			;; R2 = R2 - R4 (final R4)	; 30-32 avail none storable 9,4,6
	xstore	[srcreg+16], xmm9		;; Save R3			; 31
	xload	xmm9, [srcreg+d2+d1+32][rbp]	;;#2 Load R4-other
	mulpd	xmm14, xmm9			;;#2 I4 * R4-other		; 30-34	avail none storable 4,6

	addpd	xmm5, xmm7			;; R4 = R2 + R4 (final R2)	; 31-33	avail 7 storable 4,6,5
	xload	xmm7, [srcreg+d2+d1+32][rbx]	;;#2 R4
	mulpd	xmm9, xmm7			;;#2 R4 * R4-other		; 31-35	avail none storable 4,6,5

	subpd	xmm10, xmm13			;;#2 R3*R3-I3*I3 (new R3)	; 32-34	avail 13 storable 4,6,5
	mulpd	xmm7, xmm3			;;#2 R4 * I4-other		; 32-36	avail 13,3 storable 4,6,5
	xload	xmm13, [srcreg+d2+32][rbx]	;;#2 R2
	xstore	[srcreg], xmm4			;; Save R1			; 32	avail 3,4 storable 6,5

	addpd	xmm1, xmm12			;;#2 R3*I3+I3*R3 (new I3)	; 33-35	avail 3,4,12 storable 6,5
	xload	xmm3, [srcreg+d2+32][rbp]	;;#2 Load R2-other
	xcopy	xmm4, xmm13			;;#2 Copy R2
	mulpd	xmm13, xmm3			;;#2 R2 * R2-other		; 33-37	avail 12 storable 6,5
	xstore	[srcreg+d1+16], xmm6		;; Save R4			; 33	avail 6,12 storable 5

	xcopy	xmm6, xmm2			;; Copy I2
	subpd	xmm2, xmm0			;; I2 = I2 - I4 (final I4)	; 34-36	avail 12 storable 5,2
	xload	xmm12, [srcreg+d2+48][rbp]	;;#2 Load I2-other
	mulpd	xmm4, xmm12			;;#2 R2 * I2-other		; 34-38	avail none storable 5,2
	xstore	[srcreg+d1], xmm5		;; Save R2			; 34	avail 5 storable 2

	addpd	xmm0, xmm6			;; I4 = I2 + I4 (final I2)	; 35-37	avail 5,6 storable 2,0
	xload	xmm5, [srcreg+d2+48][rbx]	;;#2 I2
	mulpd	xmm12, xmm5			;;#2 I2 * I2-other		; 35-39	avail 6 storable 2,0

	subpd	xmm9, xmm15			;;#2 R4*R4-I4*I4 (new R4)	; 36-38	avail 6,15 storable 2,0
	mulpd	xmm5, xmm3			;;#2 I2 * R2-other		; 36-40	avail 3,6,15 storable 2,0
	xload	xmm6, [srcreg+d2][rbx]		;;#2 R1

	addpd	xmm7, xmm14			;;#2 R4*I4+I4*R4 (new I4)	; 37-39	avail 3,15,14 storable 2,0
	xload	xmm15, [srcreg+d2][rbp]		;;#2 Load R1-other
	xcopy	xmm3, xmm6			;;#2 Copy R1
	mulpd	xmm6, xmm15			;;#2 R1 * R1-other		; 37-41	avail 14 storable 2,0
	xstore	[srcreg+d1+48], xmm2		;; Save I4			; 37	avail 2,14 storable 0

	xcopy	xmm2, xmm11			;; Copy I1
	subpd	xmm11, xmm8			;; I1 = I1 - I3 (final I3)	; 38-40	avail 14 storable 0,11
	xload	xmm14, [srcreg+d2+16][rbp]	;;#2 Load I1-other	
	mulpd	xmm3, xmm14			;;#2 R1 * I1-other		; 38-42	avail none storable 0,11
	xstore	[srcreg+d1+32], xmm0		;; Save I2			; 38	avail 0 storable 11

	addpd	xmm8, xmm2			;; I3 = I1 + I3 (final I1)	; 39-41	avail 0,2 storable 11,8
	xload	xmm0, [srcreg+d2+16][rbx]	;;#2 I1
	mulpd	xmm14, xmm0			;;#2 I1 * I1-other		; 39-43	avail 2 storable 11,8

	subpd	xmm13, xmm12			;;#2 R2*R2-I2*I2 (new R2)	; 40-42	avail 2,12 storable 11,8
	mulpd	xmm0, xmm15			;;#2 I1 * R1-other		; 40-44	avail 2,12,15 storable 11,8

	addpd	xmm4, xmm5			;;#2 R2*I2+I2*R2 (new I2)	; 41-43	avail 2,12,15,5 storable 11,8
	xstore	[srcreg+48], xmm11		;; Save I3			; 41	avail 2,12,15,5,11 storable 8

	xcopy	xmm5, xmm9			;;#2 Copy R4
	subpd	xmm9, xmm10			;;#2 R4 = R4 - R3 (new I4)	; 42-44
	addpd	xmm10, xmm5			;;#2 R3 = R4 + R3 (new R3)	; 43-45
	xstore	[srcreg+32], xmm8		;; Save I1			; 42	avail 2,12,15,5,11,8

	subpd	xmm6, xmm14			;;#2 R1*R1-I1*I1 (new R1)	; 44-46	avail 2,12,15,5,11,8,14

	addpd	xmm3, xmm0			;;#2 R1*I1+I1*R1 (new I1)	; 45-47	avail 2,12,15,5,11,8,14,0

	xcopy	xmm0, xmm7			;;#2 Copy I4
	addpd	xmm7, xmm1			;;#2 I4 = I3 + I4 (new I3)	; 46-48
	subpd	xmm1, xmm0			;;#2 I3 = I3 - I4 (new R4)	; 47-49

	xcopy	xmm0, xmm6			;;#2 Copy R1
	subpd	xmm6, xmm13			;;#2 R1 = R1 - R2 (new R2)	; 48-50
	addpd	xmm13, xmm0			;;#2 R2 = R1 + R2 (new R1)	; 49-51

	xcopy	xmm0, xmm3			;;#2 Copy I1
	subpd	xmm3, xmm4			;;#2 I1 = I1 - I2 (new I2)	; 50-52
	addpd	xmm4, xmm0			;;#2 I2 = I1 + I2 (new I1)	; 51-53

	xcopy	xmm0, xmm6			;;#2 Copy R2
	subpd	xmm6, xmm1			;;#2 R2 = R2 - R4 (final R4)	; 52-54
	addpd	xmm1, xmm0			;;#2 R4 = R2 + R4 (final R2)	; 53-55

	xcopy	xmm0, xmm13			;;#2 Copy R1
	subpd	xmm13, xmm10			;;#2 R1 = R1 - R3 (final R3)	; 54-56
	addpd	xmm10, xmm0			;;#2 R3 = R1 + R3 (final R1)	; 55-57
	xstore	[srcreg+d2+d1+16], xmm6		;;#2 Save R4			; 55

	xcopy	xmm0, xmm3			;;#2 Copy I2
	subpd	xmm3, xmm9			;;#2 I2 = I2 - I4 (final I4)	; 56-58
	xstore	[srcreg+d2+d1], xmm1		;;#2 Save R2			; 56
	addpd	xmm9, xmm0			;;#2 I4 = I2 + I4 (final I2)	; 57-59
	xstore	[srcreg+d2+16], xmm13		;;#2 Save R3			; 57

	xcopy	xmm0, xmm4			;;#2 Copy I1
	subpd	xmm4, xmm7			;;#2 I1 = I1 - I3 (final I3)	; 58-60
	xstore	[srcreg+d2], xmm10		;;#2 Save R1			; 58
	addpd	xmm7, xmm0			;;#2 I3 = I1 + I3 (final I1)	; 59-61
	xstore	[srcreg+d2+d1+48], xmm3		;;#2 Save I4			; 59

	xstore	[srcreg+d2+d1+32], xmm9		;;#2 Save I2			; 60
	xstore	[srcreg+d2+48], xmm4		;;#2 Save I3			; 61
	xstore	[srcreg+d2+32], xmm7		;;#2 Save I1			; 62
	bump	srcreg, srcinc
	ENDM

ENDIF
ENDIF

;;
;; ************************************* four-complex-fft4 variants ******************************************
;;
;; These macros are used in the last levels of pass 1.  Four sin/cos multipliers are needed to
;; finish off the partial sin/cos multiplies that were done in the first levels of pass 1.
;; FFTs of type r4delay do this to reduce memory usage at the cost of some extra
;; complex multiplies.


;;
;; In the split premultiplier case, we apply part of the roots of -1 at the
;; end of the first pass.  Thus we have 4 sin/cos/premultipliers instead
;; of the usual 3.
;;

;; Used in last levels of pass 1 (split premultiplier and delay cases).  No swizzling.
IFDEF UNUSED
r4_g4cl_four_complex_fft4 MACRO srcreg,srcinc,d1,d2,dstreg,dstinc,e1,e2,screg
	xprefetch [srcreg+srcinc]
	r4_x4c_fft4_mem [srcreg],[srcreg+d1],[srcreg+d2],[srcreg+d2+d1],[srcreg+16],[srcreg+d1+16],[srcreg+d2+16],[srcreg+d2+d1+16],[dstreg],[dstreg+16],screg,0,dstreg+dstinc,e1
	xprefetch [srcreg+srcinc+d1]
	xstore	[dstreg], xmm2		;; Save R1
	xstore	[dstreg+16], xmm5	;; Save I1
	xload	xmm2, [srcreg+32]	;; R1
	xload	xmm5, [srcreg+48]	;; R5
	xstore	[dstreg+32], xmm7	;; Save R2
	xstore	[dstreg+48], xmm6	;; Save I2
	xload	xmm7, [srcreg+d1+32]	;; R2
	xload	xmm6, [srcreg+d1+48]	;; R6
	xstore	[dstreg+e1], xmm3	;; Save R3
	xstore	[dstreg+e1+16], xmm1	;; Save I3
	xstore	[dstreg+e1+32], xmm0	;; Save R4
	xstore	[dstreg+e1+48], xmm4	;; Save I4
	xprefetch [srcreg+srcinc+d2]
	r4_x4c_fft4_partial_mem xmm2,xmm7,xmm3,xmm0,xmm5,xmm6,xmm1,xmm4,[srcreg+d2+32],[srcreg+d2+d1+32],[srcreg+d2+48],[srcreg+d2+d1+48],[dstreg+e2],[dstreg+e2+16],screg,0,dstreg+dstinc+e2,e1
	xprefetch [srcreg+srcinc+d2+d1]
	bump	srcreg, srcinc
	xstore	[dstreg+e2], xmm3	;; Save R1
	xstore	[dstreg+e2+16], xmm6	;; Save I1
	xstore	[dstreg+e2+32], xmm4	;; Save R2
	xstore	[dstreg+e2+48], xmm1	;; Save I2
	xstore	[dstreg+e2+e1], xmm0	;; Save R3
	xstore	[dstreg+e2+e1+16], xmm7	;; Save I3
	xstore	[dstreg+e2+e1+32], xmm2	;; Save R4
	xstore	[dstreg+e2+e1+48], xmm5	;; Save I4
	bump	dstreg, dstinc
	ENDM

r4_x4c_fft4_partial_mem MACRO r1,r2,r3,r4,r5,r6,r7,r8,mem3,mem4,mem7,mem8,dest1,dest2,screg,off,pre1,pre2
	xload	r3, mem3			;; R3
	addpd	r3, r1				;; R3 = R1 + R3 (new R1)
	subpd	r1, mem3			;; R1 = R1 - R3 (new R3)

	xload	r4, mem4			;; R4
	addpd	r4, r2				;; R4 = R2 + R4 (new R2)
	subpd	r2, mem4			;; R2 = R2 - R4 (new R4)

	xcopy	r7, r3
	subpd	r3, r4				;; R1 = R1 - R2 (final R2)
	addpd	r4, r7				;; R2 = R1 + R2 (final R1)

	xload	r7, mem7			;; I3
	addpd	r7, r5				;; I3 = I1 + I3 (new I1)
	subpd	r5, mem7			;; I1 = I1 - I3 (new I3)

	xload	r8, mem8			;; I4
	addpd	r8, r6				;; I4 = I2 + I4 (new I2)
	subpd	r6, mem8			;; I2 = I2 - I4 (new I4)

	xstore	dest1, r4			;; Save R1

	xcopy	r4, r5
	subpd	r5, r2				;; I3 = I3 - R4 (final I4)
	addpd	r2, r4				;; R4 = I3 + R4 (final I3)

	xcopy	r4, r1
	subpd	r1, r6				;; R3 = R3 - I4 (final R3)
	addpd	r6, r4				;; I4 = R3 + I4 (final R4)

	xprefetchw [pre1]

	xcopy	r4, r7
	subpd	r7, r8				;; I1 = I1 - I2 (final I2)
	addpd	r8, r4				;; I2 = I1 + I2 (final I1)

	xload	r4, [screg+off+32+16]		;; cosine/sine
	mulpd	r4, r1				;; A3 = R3 * cosine/sine
	subpd	r4, r2				;; A3 = A3 - I3
	mulpd	r2, [screg+off+32+16]		;; B3 = I3 * cosine/sine
	addpd	r2, r1				;; B3 = B3 + R3

	xstore	dest2, r8			;; Save I1

	xload	r8, [screg+off+64+16]		;; cosine/sine
	mulpd	r8, r3				;; A2 = R2 * cosine/sine
	subpd	r8, r7				;; A2 = A2 - I2
	mulpd	r7, [screg+off+64+16]		;; B2 = I2 * cosine/sine
	addpd	r7, r3				;; B2 = B2 + R2

	xprefetchw [pre1][pre2]

	xload	r1, [screg+off+96+16]		;; cosine/sine
	mulpd	r1, r6				;; A4 = R4 * cosine/sine
	subpd	r1, r5				;; A4 = A4 - I4
	mulpd	r5, [screg+off+96+16]		;; B4 = I4 * cosine/sine
	addpd	r5, r6				;; B4 = B4 + R4

	xload	r3, [screg+off+32]
	mulpd	r4, r3				;; A3 = A3 * sine (final R3)
	mulpd	r2, r3				;; B3 = B3 * sine (final I3)
	xload	r6, [screg+off+64]
	mulpd	r8, r6				;; A2 = A2 * sine (final R2)
	mulpd	r7, r6				;; B2 = B2 * sine (final I2)

	xload	r3, [screg+off+0+16]		;; cosine/sine
	mulpd	r3, dest1			;; A1 = R1 * cosine/sine
	xload	r6, dest2			;; Restore I1
	subpd	r3, r6				;; A1 = A1 - I1
	mulpd	r6, [screg+off+0+16]		;; B1 = I1 * cosine/sine
	addpd	r6, dest1			;; B1 = B1 + R1

	mulpd	r1, [screg+off+96]		;; A4 = A4 * sine (final R4)
	mulpd	r5, [screg+off+96]		;; B4 = B4 * sine (final I4)
	mulpd	r3, [screg+off+0]		;; A1 = A1 * sine (final R1)
	mulpd	r6, [screg+off+0]		;; B1 = B1 * sine (final I1)
	ENDM
ENDIF

;; Used in last levels of pass 1 (split premultiplier and delay cases).  Swizzling.
r4_sg4cl_four_complex_fft4 MACRO srcreg,srcinc,d1,d2,dstreg,dstinc,e1,e2,screg
	xprefetch [srcreg+srcinc]
	r4_x4c_fft4_mem [srcreg],[srcreg+d1],[srcreg+d2],[srcreg+d2+d1],[srcreg+16],[srcreg+d1+16],[srcreg+d2+16],[srcreg+d2+d1+16],[dstreg],[dstreg+16],screg,0,dstreg+dstinc,e1
	xprefetch [srcreg+srcinc+d1]
	shuffle_store [dstreg], [dstreg+16], xmm2, xmm7					;; Save R1,R2
	shuffle_store_with_temp [dstreg+32], [dstreg+48], xmm5, xmm6, xmm7		;; Save I1,I2
	shuffle_store_with_temp [dstreg+e1], [dstreg+e1+16], xmm3, xmm0, xmm7		;; Save R3,R4
	shuffle_store_with_temp [dstreg+e1+32], [dstreg+e1+48], xmm1, xmm4, xmm7	;; Save I3,I4
	xprefetch [srcreg+srcinc+d2]
	r4_x4c_fft4_mem [srcreg+32],[srcreg+d1+32],[srcreg+d2+32],[srcreg+d2+d1+32],[srcreg+48],[srcreg+d1+48],[srcreg+d2+48],[srcreg+d2+d1+48],[dstreg+e2],[dstreg+e2+16],screg,0,dstreg+dstinc+e2,e1
	xprefetch [srcreg+srcinc+d2+d1]
	bump	srcreg, srcinc
	shuffle_store [dstreg+e2], [dstreg+e2+16], xmm2, xmm7				;; Save R1,R2
	shuffle_store_with_temp [dstreg+e2+32], [dstreg+e2+48], xmm5, xmm6, xmm7	;; Save I1,I2
	shuffle_store_with_temp [dstreg+e2+e1], [dstreg+e2+e1+16], xmm3, xmm0, xmm7	;; Save R3,R4
	shuffle_store_with_temp [dstreg+e2+e1+32], [dstreg+e2+e1+48], xmm1, xmm4, xmm7	;; Save I3,I4
	bump	dstreg, dstinc
	ENDM

r4_x4c_fft4_mem MACRO mem1,mem2,mem3,mem4,mem5,mem6,mem7,mem8,dest1,dest2,screg,off,pre1,pre2
	xload	xmm2, mem3			;; R3
	xload	xmm0, mem1			;; R1
	xcopy	xmm5, xmm2			;; Copy R3
	addpd	xmm2, xmm0			;; R3 = R1 + R3 (new R1)
	subpd	xmm0, xmm5			;; R1 = R1 - R3 (new R3)

	xload	xmm3, mem4			;; R4
	xload	xmm1, mem2			;; R2
	xcopy	xmm5, xmm3			;; Copy R4
	addpd	xmm3, xmm1			;; R4 = R2 + R4 (new R2)
	subpd	xmm1, xmm5			;; R2 = R2 - R4 (new R4)

	xcopy	xmm6, xmm2
	subpd	xmm2, xmm3			;; R1 = R1 - R2 (final R2)
	addpd	xmm3, xmm6			;; R2 = R1 + R2 (final R1)

	xload	xmm6, mem7			;; I3
	xload	xmm4, mem5			;; I1
	xcopy	xmm5, xmm6			;; Copy I3
	addpd	xmm6, xmm4			;; I3 = I1 + I3 (new I1)
	subpd	xmm4, xmm5			;; I1 = I1 - I3 (new I3)

	xload	xmm5, mem6			;; I2
	xload	xmm7, mem8			;; I4
	addpd	xmm7, xmm5			;; I4 = I2 + I4 (new I2)
	subpd	xmm5, mem8			;; I2 = I2 - I4 (new I4)

	xstore	dest1, xmm3			;; Save R1

	xcopy	xmm3, xmm4
	subpd	xmm4, xmm1			;; I3 = I3 - R4 (final I4)
	addpd	xmm1, xmm3			;; R4 = I3 + R4 (final I3)

	xcopy	xmm3, xmm0
	subpd	xmm0, xmm5			;; R3 = R3 - I4 (final R3)
	addpd	xmm5, xmm3			;; I4 = R3 + I4 (final R4)

	xprefetchw [pre1]

	xcopy	xmm3, xmm6
	subpd	xmm6, xmm7			;; I1 = I1 - I2 (final I2)
	addpd	xmm7, xmm3			;; I2 = I1 + I2 (final I1)

	xload	xmm3, [screg+off+32+16]		;; cosine/sine
	mulpd	xmm3, xmm0			;; A3 = R3 * cosine/sine
	subpd	xmm3, xmm1			;; A3 = A3 - I3
	mulpd	xmm1, [screg+off+32+16]		;; B3 = I3 * cosine/sine
	addpd	xmm1, xmm0			;; B3 = B3 + R3

	xstore	dest2, xmm7			;; Save I1

	xload	xmm7, [screg+off+64+16]		;; cosine/sine
	mulpd	xmm7, xmm2			;; A2 = R2 * cosine/sine
	subpd	xmm7, xmm6			;; A2 = A2 - I2
	mulpd	xmm6, [screg+off+64+16]		;; B2 = I2 * cosine/sine
	addpd	xmm6, xmm2			;; B2 = B2 + R2

	xprefetchw [pre1][pre2]

	xload	xmm0, [screg+off+96+16]		;; cosine/sine
	mulpd	xmm0, xmm5			;; A4 = R4 * cosine/sine
	subpd	xmm0, xmm4			;; A4 = A4 - I4
	mulpd	xmm4, [screg+off+96+16]		;; B4 = I4 * cosine/sine
	addpd	xmm4, xmm5			;; B4 = B4 + R4

	xload	xmm2, [screg+off+32]
	mulpd	xmm3, xmm2			;; A3 = A3 * sine (final R3)
	mulpd	xmm1, xmm2			;; B3 = B3 * sine (final I3)
	xload	xmm5, [screg+off+64]
	mulpd	xmm7, xmm5			;; A2 = A2 * sine (final R2)
	mulpd	xmm6, xmm5			;; B2 = B2 * sine (final I2)

	xload	xmm2, [screg+off+0+16]		;; cosine/sine
	mulpd	xmm2, dest1			;; A1 = R1 * cosine/sine
	xload	xmm5, dest2			;; Restore I1
	subpd	xmm2, xmm5			;; A1 = A1 - I1
	mulpd	xmm5, [screg+off+0+16]		;; B1 = I1 * cosine/sine
	addpd	xmm5, dest1			;; B1 = B1 + R1

	mulpd	xmm0, [screg+off+96]		;; A4 = A4 * sine (final R4)
	mulpd	xmm4, [screg+off+96]		;; B4 = B4 * sine (final I4)
	mulpd	xmm2, [screg+off+0]		;; A1 = A1 * sine (final R1)
	mulpd	xmm5, [screg+off+0]		;; B1 = B1 * sine (final I1)
	ENDM

;; 32-bit AMD K8 optimized versions of the above macros.  Derived from Intel 32-bit
;; version -- could probably be optimized further.

IF (@INSTR(,%xarch,<K8>) NE 0)

r4_x4c_fft4_mem MACRO mem1,mem2,mem3,mem4,mem5,mem6,mem7,mem8,dest1,dest2,screg,off,pre1,pre2
	xload	xmm2, mem3			;; R3
	xload	xmm0, mem1			;; R1
	addpd	xmm2, xmm0			;; R3 = R1 + R3 (new R1)
	subpd	xmm0, mem3			;; R1 = R1 - R3 (new R3)

	xload	xmm3, mem4			;; R4
	xload	xmm1, mem2			;; R2
	addpd	xmm3, xmm1			;; R4 = R2 + R4 (new R2)
	subpd	xmm1, mem4			;; R2 = R2 - R4 (new R4)

	xload	xmm6, mem7			;; I3
	xload	xmm4, mem5			;; I1
	addpd	xmm6, xmm4			;; I3 = I1 + I3 (new I1)
	subpd	xmm4, mem7			;; I1 = I1 - I3 (new I3)

	xload	xmm5, mem6			;; I2
	xload	xmm7, mem8			;; I4
	addpd	xmm7, xmm5			;; I4 = I2 + I4 (new I2)
	subpd	xmm5, mem8			;; I2 = I2 - I4 (new I4)

	xprefetchw [pre1]

	subpd	xmm2, xmm3			;; R1 = R1 - R2 (final R2)
	multwo	xmm3				;; R2 = R2 * 2

	subpd	xmm4, xmm1			;; I3 = I3 - R4 (final I4)
	multwo	xmm1				;; R4 = R4 * 2

	subpd	xmm6, xmm7			;; I1 = I1 - I2 (final I2)
	multwo	xmm7				;; I2 = I2 * 2

	subpd	xmm0, xmm5			;; R3 = R3 - I4 (final R3)
	multwo	xmm5				;; I4 = I4 * 2

	xprefetchw [pre1][pre2]

	addpd	xmm3, xmm2			;; R2 = R1 + R2 (final R1)
	addpd	xmm1, xmm4			;; R4 = I3 + R4 (final I3)
	addpd	xmm7, xmm6			;; I2 = I1 + I2 (final I1)
	addpd	xmm5, xmm0			;; I4 = R3 + I4 (final R4)

	xstore	dest1, xmm3			;; Save R1

	xload	xmm3, [screg+off+32+16]		;; cosine/sine
	mulpd	xmm3, xmm0			;; A3 = R3 * cosine/sine

	xstore	dest2, xmm7			;; Save I1

	xload	xmm7, [screg+off+64+16]		;; cosine/sine
	mulpd	xmm7, xmm2			;; A2 = R2 * cosine/sine

	subpd	xmm3, xmm1			;; A3 = A3 - I3
	mulpd	xmm1, [screg+off+32+16]		;; B3 = I3 * cosine/sine

	subpd	xmm7, xmm6			;; A2 = A2 - I2
	mulpd	xmm6, [screg+off+64+16]		;; B2 = I2 * cosine/sine

	addpd	xmm1, xmm0			;; B3 = B3 + R3

	xload	xmm0, [screg+off+96+16]		;; cosine/sine
	mulpd	xmm0, xmm5			;; A4 = R4 * cosine/sine

	addpd	xmm6, xmm2			;; B2 = B2 + R2

	xload	xmm2, [screg+off+0+16]		;; cosine/sine
	mulpd	xmm2, dest1			;; A1 = R1 * cosine/sine

	subpd	xmm0, xmm4			;; A4 = A4 - I4
	mulpd	xmm4, [screg+off+96+16]		;; B4 = I4 * cosine/sine

	addpd	xmm4, xmm5			;; B4 = B4 + R4

	xload	xmm5, dest2			;; Restore I1
	subpd	xmm2, xmm5			;; A1 = A1 - I1
	mulpd	xmm5, [screg+off+0+16]		;; B1 = I1 * cosine/sine

	addpd	xmm5, dest1			;; B1 = B1 + R1

	mulpd	xmm3, [screg+off+32]		;; A3 = A3 * sine (final R3)
	mulpd	xmm1, [screg+off+32]		;; B3 = B3 * sine (final I3)
	mulpd	xmm7, [screg+off+64]		;; A2 = A2 * sine (final R2)
	mulpd	xmm6, [screg+off+64]		;; B2 = B2 * sine (final I2)
	mulpd	xmm0, [screg+off+96]		;; A4 = A4 * sine (final R4)
	mulpd	xmm4, [screg+off+96]		;; B4 = B4 * sine (final I4)
	mulpd	xmm2, [screg+off+0]		;; A1 = A1 * sine (final R1)
	mulpd	xmm5, [screg+off+0]		;; B1 = B1 * sine (final I1)
	ENDM

ENDIF

;; 64-bit versions of the above macros.

IFDEF X86_64

r4_sg4cl_four_complex_fft4 MACRO srcreg,srcinc,d1,d2,dstreg,dstinc,e1,e2,screg
	xload	xmm0, [srcreg]			;; R1
	xload	xmm1, [srcreg+d2]		;; R3
	xcopy	xmm2, xmm0			;; Copy R1			; 1-3
	addpd	xmm0, xmm1			;; R3 = R1 + R3 (new R1)	; 1-3
	xload	xmm3, [srcreg+d1]		;; R2
	xload	xmm4, [srcreg+d2+d1]		;; R4
	xcopy	xmm5, xmm3			;; Copy R2			; 2-4
	addpd	xmm3, xmm4			;; R4 = R2 + R4 (new R2)	; 2-4
	xload	xmm6, [srcreg+16]		;; I1
	xload	xmm7, [srcreg+d2+16]		;; I3
	xcopy	xmm8, xmm6			;; Copy I1			; 3-5
	addpd	xmm6, xmm7			;; I3 = I1 + I3 (new I1)	; 3-5
	xload	xmm9, [srcreg+d1+16]		;; I2
	xload	xmm10, [srcreg+d2+d1+16]	;; I4
	xcopy	xmm11, xmm9			;; Copy I2			; 4-6
	addpd	xmm9, xmm10			;; I4 = I2 + I4 (new I2)	; 4-6	avail 12+

	xprefetch [srcreg+srcinc]

	subpd	xmm2, xmm1			;; R1 = R1 - R3 (new R3)	; 5-7	avail 1,12+

	subpd	xmm5, xmm4			;; R2 = R2 - R4 (new R4)	; 6-8	avail 1,4,12+

	xprefetchw [dstreg+dstinc]

	subpd	xmm8, xmm7			;; I1 = I1 - I3 (new I3)	; 7-9	avail 1,4,7,12+
	xcopy	xmm1, xmm0			;; Copy R1			; 7-9 (4) avail 4,7,12+

	subpd	xmm11, xmm10			;; I2 = I2 - I4 (new I4)	; 8-10	avail 4,7,10,12+
	xload	xmm12, [screg+64+16]		;; cosine/sine 2		; 8	avail 4,7,10,13+

	subpd	xmm0, xmm3			;; R1 = R1 - R2 (final R2)	; 9-11
	xcopy	xmm4, xmm6			;; Copy I1			; 9-11 (6) avail 7,10,13+
	xload	xmm13, [screg+0+16]		;; cosine/sine 1		; 9

	addpd	xmm3, xmm1			;; R2 = R1 + R2 (final R1)	; 10-12 avail 7,10,1,14+
	xcopy	xmm7, xmm8			;; Copy I3			; 10-12 avail 10,1,14+

	xprefetch [srcreg+srcinc+d1]

	subpd	xmm6, xmm9			;; I1 = I1 - I2 (final I2)	; 11-13
	xcopy	xmm1, xmm2			;; Copy R3			; 11-13 (8) avail 10,14+

	addpd	xmm9, xmm4			;; I2 = I1 + I2 (final I1)	; 12-14 avail 10,4,14+
	xcopy	xmm10, xmm0			;; Copy R2			; 12-14 avail 4,14+
	mulpd	xmm0, xmm12			;; A2 = R2 * cosine/sine	; 12-16

	xprefetchw [dstreg+dstinc+e1]

	subpd	xmm8, xmm5			;; I3 = I3 - R4 (final I4)	; 13-15
	xcopy	xmm4, xmm3			;; Copy R1			; 13-15 avail 14+
	mulpd	xmm3, xmm13			;; A1 = R1 * cosine/sine	; 13-17

	addpd	xmm5, xmm7			;; R4 = I3 + R4 (final I3)	; 14-16 avail 7,14+
	mulpd	xmm12, xmm6			;; B2 = I2 * cosine/sine	; 14-18
	xload	xmm14, [screg+96+16]		;; cosine/sine 4		; 14	avail 7,15

	subpd	xmm2, xmm11			;; R3 = R3 - I4 (final R3)	; 15-17
	mulpd	xmm13, xmm9			;; B1 = I1 * cosine/sine	; 15-19
	xload	xmm15, [screg+32+16]		;; cosine/sine 3		; 15	avail 7

	addpd	xmm11, xmm1			;; I4 = R3 + I4 (final R4)	; 16-18 avail 7,1
	xcopy	xmm7, xmm8			;; Copy I4			; 16-18 avail 1
	mulpd	xmm8, xmm14			;; B4 = I4 * cosine/sine	; 16-20

	xprefetch [srcreg+srcinc+d2]

	subpd	xmm0, xmm6			;; A2 = A2 - I2			; 17-19	avail 1,6
	xcopy	xmm1, xmm5			;; Copy I3			; 17-19	avail 6
	mulpd	xmm5, xmm15			;; B3 = I3 * cosine/sine	; 17-21

	subpd	xmm3, xmm9			;; A1 = A1 - I1			; 18-20	avail 6,9
	mulpd	xmm15, xmm2			;; A3 = R3 * cosine/sine	; 18-22
	xload	xmm6, [screg+64]		;; sine 2			; 18	avail 9

	addpd	xmm12, xmm10			;; B2 = B2 + R2			; 19-21	avail 9,10
	mulpd	xmm14, xmm11			;; A4 = R4 * cosine/sine	; 19-23
	xload	xmm9, [screg]			;; sine 1			; 19	avail 10

	addpd	xmm13, xmm4			;; B1 = B1 + R1			; 20-22	avail 10,4
	mulpd	xmm0, xmm6			;; A2 = A2 * sine (final R2)	; 20-24
	xload	xmm4, [screg+96]		;; sine 4			; 20	avail 10,11

	addpd	xmm8, xmm11			;; B4 = B4 + R4			; 21-23	avail 10,11
	mulpd	xmm3, xmm9			;; A1 = A1 * sine (final R1)	; 21-25
	xload	xmm11, [screg+32]		;; sine 3			; 21	avail 10

	addpd	xmm5, xmm2			;; B3 = B3 + R3			; 22-24	avail 10,2
	mulpd	xmm12, xmm6			;; B2 = B2 * sine (final I2)	; 22-26	avail 10,2,6
	xload	xmm10, [srcreg+32]		;;#2 R1				; 22	avail 2,6

	subpd	xmm15, xmm1			;; A3 = A3 - I3			; 23-25	avail 2,6,1
	mulpd	xmm13, xmm9			;; B1 = B1 * sine (final I1)	; 23-27	avail 2,6,1,9
	xcopy	xmm6, xmm10			;;#2 Copy R1			; 1-3	avail 2,1,9
	xload	xmm2, [srcreg+d2+32]		;;#2 R3				; 23	avail 1,9

	subpd	xmm14, xmm7			;; A4 = A4 - I4			; 24-26	avail 1,9,7
	mulpd	xmm8, xmm4			;; B4 = B4 * sine (final I4)	; 24-28
	xload	xmm1, [srcreg+d1+32]		;;#2 R2				; 24	avail 9,7
	xcopy	xmm9, xmm1			;;#2 Copy R2			; 2-4	avail 7

	addpd	xmm10, xmm2			;;#2 R3 = R1 + R3 (new R1)	; 1-3
	mulpd	xmm5, xmm11			;; B3 = B3 * sine (final I3)	; 25-29
	xload	xmm7, [srcreg+d2+d1+32]		;;#2 R4				; 25	avail none

	addpd	xmm1, xmm7			;;#2 R4 = R2 + R4 (new R2)	; 2-4
	mulpd	xmm15, xmm11			;; A3 = A3 * sine (final R3)	; 26-30	avail 11

	shuffle_store_with_temp [dstreg], [dstreg+16], xmm3, xmm0, xmm11	;; Save R1,R2 in clocks 26-30  avail 3,0,11

	xload	xmm0, [srcreg+48]		;;#2 I1
	xload	xmm3, [srcreg+d2+48]		;;#2 I3
	xcopy	xmm11, xmm0			;;#2 Copy I1			; 3-5
	addpd	xmm0, xmm3			;;#2 I3 = I1 + I3 (new I1)	; 3-5
	mulpd	xmm14, xmm4			;; A4 = A4 * sine (final R4)	; 27-31	avail 4

	shuffle_store_with_temp [dstreg+32], [dstreg+48], xmm13, xmm12, xmm4	;; Save I1,I2 in clocks 28-32  avail 13,12,4

	xload	xmm4, [srcreg+d1+48]		;;#2 I2
	xload	xmm12, [srcreg+d2+d1+48]	;;#2 I4
	xcopy	xmm13, xmm4			;;#2 Copy I2			; 4-6
	addpd	xmm4, xmm12			;;#2 I4 = I2 + I4 (new I2)	; 4-6	avail none

	xprefetchw [dstreg+dstinc+e2]

	subpd	xmm6, xmm2			;;#2 R1 = R1 - R3 (new R3)	; 5-7	avail 2

	subpd	xmm9, xmm7			;;#2 R2 = R2 - R4 (new R4)	; 6-8	avail 2,7

	shuffle_store_with_temp [dstreg+e1+32], [dstreg+e1+48], xmm5, xmm8, xmm2 ;; Save I3,I4 in clocks 30-34

	subpd	xmm11, xmm3			;;#2 I1 = I1 - I3 (new I3)	; 7-9
	xcopy	xmm2, xmm10			;;#2 Copy R1			; 7-9 (4)

	xprefetch [srcreg+srcinc+d2+d1]

	subpd	xmm13, xmm12			;;#2 I2 = I2 - I4 (new I4)	; 8-10
	xload	xmm5, [screg+64+16]		;;#2 cosine/sine 2		; 8

	shuffle_store_with_temp [dstreg+e1], [dstreg+e1+16], xmm15, xmm14, xmm3	;; Save R3,R4 in clocks 32-36

	subpd	xmm10, xmm1			;;#2 R1 = R1 - R2 (final R2)	; 9-11
	xcopy	xmm7, xmm0			;;#2 Copy I1			; 9-11 (6)
	xload	xmm8, [screg+0+16]		;;#2 cosine/sine 1		; 9

	addpd	xmm1, xmm2			;;#2 R2 = R1 + R2 (final R1)	; 10-12
	xcopy	xmm3, xmm11			;;#2 Copy I3			; 10-12

	xprefetchw [dstreg+dstinc+e2+e1]

	subpd	xmm0, xmm4			;;#2 I1 = I1 - I2 (final I2)	; 11-13
	xcopy	xmm2, xmm6			;;#2 Copy R3			; 11-13 (8)

	addpd	xmm4, xmm7			;;#2 I2 = I1 + I2 (final I1)	; 12-14
	xcopy	xmm12, xmm10			;;#2 Copy R2			; 12-14
	mulpd	xmm10, xmm5			;;#2 A2 = R2 * cosine/sine	; 12-16

	subpd	xmm11, xmm9			;;#2 I3 = I3 - R4 (final I4)	; 13-15
	xcopy	xmm7, xmm1			;;#2 Copy R1			; 13-15
	mulpd	xmm1, xmm8			;;#2 A1 = R1 * cosine/sine	; 13-17

	addpd	xmm9, xmm3			;;#2 R4 = I3 + R4 (final I3)	; 14-16
	mulpd	xmm5, xmm0			;;#2 B2 = I2 * cosine/sine	; 14-18
	xload	xmm14, [screg+96+16]		;;#2 cosine/sine 4		; 14

	subpd	xmm6, xmm13			;;#2 R3 = R3 - I4 (final R3)	; 15-17
	mulpd	xmm8, xmm4			;;#2 B1 = I1 * cosine/sine	; 15-19
	xload	xmm15, [screg+32+16]		;;#2 cosine/sine 3		; 15

	addpd	xmm13, xmm2			;;#2 I4 = R3 + I4 (final R4)	; 16-18
	xcopy	xmm3, xmm11			;;#2 Copy I4			; 16-18
	mulpd	xmm11, xmm14			;;#2 B4 = I4 * cosine/sine	; 16-20

	subpd	xmm10, xmm0			;;#2 A2 = A2 - I2		; 17-19
	xcopy	xmm2, xmm9			;;#2 Copy I3			; 17-19
	mulpd	xmm9, xmm15			;;#2 B3 = I3 * cosine/sine	; 17-21

	subpd	xmm1, xmm4			;;#2 A1 = A1 - I1		; 18-20
	mulpd	xmm15, xmm6			;;#2 A3 = R3 * cosine/sine	; 18-22
	xload	xmm0, [screg+64]		;;#2 sine 2			; 18

	addpd	xmm5, xmm12			;;#2 B2 = B2 + R2		; 19-21
	mulpd	xmm14, xmm13			;;#2 A4 = R4 * cosine/sine	; 19-23
	xload	xmm4, [screg]			;;#2 sine 1			; 19

	addpd	xmm8, xmm7			;;#2 B1 = B1 + R1		; 20-22
	mulpd	xmm10, xmm0			;;#2 A2 = A2 * sine (final R2)	; 20-24

	bump	srcreg, srcinc

	addpd	xmm11, xmm13			;;#2 B4 = B4 + R4		; 21-23
	mulpd	xmm1, xmm4			;;#2 A1 = A1 * sine (final R1)	; 21-25
	xload	xmm7, [screg+96]		;;#2 sine 4			; 21

	addpd	xmm9, xmm6			;;#2 B3 = B3 + R3		; 22-24
	mulpd	xmm5, xmm0			;;#2 B2 = B2 * sine (final I2)	; 22-26
	xload	xmm6, [screg+32]		;;#2 sine 3			; 22

	subpd	xmm15, xmm2			;;#2 A3 = A3 - I3		; 23-25
	mulpd	xmm8, xmm4			;;#2 B1 = B1 * sine (final I1)	; 23-27

	subpd	xmm14, xmm3			;;#2 A4 = A4 - I4		; 24-26
	mulpd	xmm11, xmm7			;;#2 B4 = B4 * sine (final I4)	; 24-28

	mulpd	xmm9, xmm6			;;#2 B3 = B3 * sine (final I3)	; 25-29

	mulpd	xmm15, xmm6			;;#2 A3 = A3 * sine (final R3)	; 26-30

	mulpd	xmm14, xmm7			;;#2 A4 = A4 * sine (final R4)	; 27-31

	shuffle_store_with_temp [dstreg+e2], [dstreg+e2+16], xmm1, xmm10, xmm2		;;#2 Save R1,R2
	shuffle_store_with_temp [dstreg+e2+32], [dstreg+e2+48], xmm8, xmm5, xmm2	;;#2 Save I1,I2
	shuffle_store_with_temp [dstreg+e2+e1], [dstreg+e2+e1+16], xmm15, xmm14, xmm2	;;#2 Save R3,R4
	shuffle_store_with_temp [dstreg+e2+e1+32], [dstreg+e2+e1+48], xmm9, xmm11, xmm2	;;#2 Save I3,I4
	bump	dstreg, dstinc
	ENDM

ENDIF

;;
;; ************************************* four-complex-unfft4 variants ******************************************
;;

;; Used in last levels of pass 1 (split premultiplier and delay cases).  No swizzling.
IFDEF UNUSED
r4_g2cl_four_complex_unfft4 MACRO srcreg,srcinc,d1,dstreg,dstinc,e1,screg
	xload	xmm0, [srcreg]		;; R1
	xload	xmm1, [srcreg+16]	;; I1
	xload	xmm4, [srcreg+d1]	;; R3
	xload	xmm5, [srcreg+d1+16]	;; I3
	r4_x4c_unfft4_partial_mem xmm0,xmm1,xmm2,xmm3,xmm4,xmm5,xmm6,xmm7,[srcreg+32],[srcreg+48],[srcreg+d1+32],[srcreg+d1+48],[dstreg],[dstreg+32],screg,0,dstreg+dstinc,e1
	bump	srcreg, srcinc
	xstore	[dstreg], xmm7		;; Save R1
	xstore	[dstreg+16], xmm2	;; Save R3
	xstore	[dstreg+32], xmm4	;; Save I1
	xstore	[dstreg+48], xmm3	;; Save I3
	xstore	[dstreg+e1], xmm5	;; Save R2
	xstore	[dstreg+e1+16], xmm6	;; Save R4
	xstore	[dstreg+e1+32], xmm0	;; Save I2
	xstore	[dstreg+e1+48], xmm1	;; Save I4
	bump	dstreg, dstinc
	ENDM

r4_x4c_unfft4_partial_mem MACRO r1,r2,r3,r4,r5,r6,r7,r8,mem3,mem4,mem7,mem8,dest1,dest2,screg,off,pre1,pre2
	xload	r7, [screg+off+0+16]		;; cosine/sine
	mulpd	r7, r1				;; A1 = R1 * cosine/sine
	addpd	r7, r2				;; A1 = A1 + I1
	mulpd	r2, [screg+off+0+16]		;; B1 = I1 * cosine/sine
	subpd	r2, r1				;; B1 = B1 - R1

	xload	r8, [screg+off+32+16]		;; cosine/sine
	mulpd	r8, r5				;; A3 = R3 * cosine/sine
	addpd	r8, r6				;; A3 = A3 + I3
	mulpd	r6, [screg+off+32+16]		;; B3 = I3 * cosine/sine
	subpd	r6, r5				;; B3 = B3 - R3

	xload	r3, [screg+off+64+16]		;; cosine/sine
	mulpd	r3, mem3			;; A2 = R2 * cosine/sine
	xload	r4, mem4			;; R4 (I2)
	addpd	r3, r4				;; A2 = A2 + I2
	mulpd	r4, [screg+off+64+16]		;; B2 = I2 * cosine/sine
	subpd	r4, mem3			;; B2 = B2 - R2

	xload	r1, [screg+off+96+16]		;; cosine/sine
	mulpd	r1, mem7			;; A4 = R4 * cosine/sine
	xload	r5, mem8			;; I4
	addpd	r1, r5				;; A4 = A4 + I4
	mulpd	r5, [screg+off+96+16]		;; B4 = I4 * cosine/sine
	subpd	r5, mem7			;; B4 = B4 - R4

	mulpd	r7, [screg+off+0]		;; A1 = A1 * sine (new R1)
	mulpd	r2, [screg+off+0]		;; B1 = B1 * sine (new I1)

	mulpd	r8, [screg+off+32]		;; A3 = A3 * sine (new R3)
	mulpd	r6, [screg+off+32]		;; B3 = B3 * sine (new I3)

	mulpd	r3, [screg+off+64]		;; A2 = A2 * sine (new R2)
	mulpd	r4, [screg+off+64]		;; B2 = B2 * sine (new I2)

	mulpd	r1, [screg+off+96]		;; A4 = A4 * sine (new R4)
	mulpd	r5, [screg+off+96]		;; B4 = B4 * sine (new I4)

	xprefetchw [pre1]

	subpd	r7, r3				;; R1 = R1 - R2 (new R2)
	multwo	r3				;; R2 = R2 * 2
	addpd	r3, r7				;; R2 = R1 + R2 (new R1)

	subpd	r2, r4				;; I1 = I1 - I2 (new I2)
	multwo	r4				;; I2 = I2 * 2
	addpd	r4, r2				;; I2 = I1 + I2 (new I1)

	subpd	r1, r8				;; R4 = R4 - R3 (new I4)
	multwo	r8				;; R3 = R3 * 2
	addpd	r8, r1				;; R3 = R4 + R3 (new R3)

	subpd	r6, r5				;; I3 = I3 - I4 (new R4)
	multwo	r5				;; I4 = I4 * 2
	addpd	r5, r6				;; I4 = I3 + I4 (new I3)

	xprefetchw [pre1][pre2]

	subpd	r7, r6				;; R2 = R2 - R4 (final R4)
	multwo	r6				;; R4 = R4 * 2
	addpd	r6, r7				;; R4 = R2 + R4 (final R2)

	subpd	r2, r1				;; I2 = I2 - I4 (final I4)
	multwo	r1				;; I4 = I4 * 2
	addpd	r1, r2				;; I4 = I2 + I4 (final I2)

	subpd	r3, r8				;; R1 = R1 - R3 (final R3)
	multwo	r8				;; R3 = R3 * 2
	addpd	r8, r3				;; R3 = R1 + R3 (final R1)

	subpd	r4, r5				;; I1 = I1 - I3 (final I3)
	multwo	r5				;; I3 = I3 * 2
	addpd	r5, r4				;; I3 = I1 + I3 (final I1)
	ENDM
ENDIF

;; Used in last levels of pass 1 (r4delay case).  Swizzling.
r4_sg2cl_four_complex_unfft4 MACRO srcreg,srcinc,d1,dstreg,dstinc,e1,screg
	shuffle_load_with_temp xmm6, xmm2, [srcreg], [srcreg+16], xmm0		;; R1,R2
	xcopy	xmm0, xmm6			;; Copy R1
	mulpd	xmm6, [screg+0+16]		;; A1 = R1 * cosine/sine
	xcopy	xmm4, xmm2			;; Copy R2
	mulpd	xmm2, [screg+64+16]		;; A2 = R2 * cosine/sine

	shuffle_load_with_temp xmm1, xmm3, [srcreg+32], [srcreg+48], xmm5	;; I1,I2
	addpd	xmm6, xmm1			;; A1 = A1 + I1
	mulpd	xmm1, [screg+0+16]		;; B1 = I1 * cosine/sine
	addpd	xmm2, xmm3			;; A2 = A2 + I2
	mulpd	xmm3, [screg+64+16]		;; B2 = I2 * cosine/sine
	subpd	xmm1, xmm0			;; B1 = B1 - R1
	subpd	xmm3, xmm4			;; B2 = B2 - R2

	mulpd	xmm6, [screg+0]			;; A1 = A1 * sine (new R1)
	mulpd	xmm2, [screg+64]		;; A2 = A2 * sine (new R2)

	shuffle_load_with_temp xmm7, xmm0, [srcreg+d1], [srcreg+d1+16], xmm4	;; R3,R4
	xcopy	xmm4, xmm7			;; Copy R3
	mulpd	xmm7, [screg+32+16]		;; A3 = R3 * cosine/sine
	xcopy	xmm5, xmm0			;; Copy R4
	mulpd	xmm0, [screg+96+16]		;; A4 = R4 * cosine/sine

	xstore	[dstreg], xmm6			;; Save new R1 temporarily
	xstore	[dstreg+16], xmm2		;; Save new R2 temporarily

	shuffle_load xmm6, xmm2, [srcreg+d1+32], [srcreg+d1+48]	;; I3,I4

	addpd	xmm7, xmm6			;; A3 = A3 + I3
	mulpd	xmm6, [screg+32+16]		;; B3 = I3 * cosine/sine
	addpd	xmm0, xmm2			;; A4 = A4 + I4
	mulpd	xmm2, [screg+96+16]		;; B4 = I4 * cosine/sine
	subpd	xmm6, xmm4			;; B3 = B3 - R3
	subpd	xmm2, xmm5			;; B4 = B4 - R4

	mulpd	xmm1, [screg+0]			;; B1 = B1 * sine (new I1)
	mulpd	xmm3, [screg+64]		;; B2 = B2 * sine (new I2)
	mulpd	xmm7, [screg+32]		;; A3 = A3 * sine (new R3)
	mulpd	xmm0, [screg+96]		;; A4 = A4 * sine (new R4)
	mulpd	xmm6, [screg+32]		;; B3 = B3 * sine (new I3)
	mulpd	xmm2, [screg+96]		;; B4 = B4 * sine (new I4)

	bump	srcreg, srcinc

	xcopy	xmm4, xmm1			;; Copy I1
	subpd	xmm1, xmm3			;; I1 = I1 - I2 (new I2)
	addpd	xmm3, xmm4			;; I2 = I1 + I2 (new I1)

	xprefetchw [dstreg+dstinc]

	xcopy	xmm4, xmm0			;; Copy R4
	subpd	xmm0, xmm7			;; R4 = R4 - R3 (new I4)
	addpd	xmm7, xmm4			;; R3 = R4 + R3 (new R3)

	xprefetchw [dstreg+dstinc+e1]

	xcopy	xmm4, xmm6			;; Copy I3
	subpd	xmm6, xmm2			;; I3 = I3 - I4 (new R4)
	addpd	xmm2, xmm4			;; I4 = I3 + I4 (new I3)

	xprefetch [srcreg]

	xcopy	xmm4, xmm1			;; Copy I2
	subpd	xmm1, xmm0			;; I2 = I2 - I4 (final I4)
	addpd	xmm0, xmm4			;; I4 = I2 + I4 (final I2)

	xprefetch [srcreg+d1]

	xload	xmm4, [dstreg]			;; Reload new R1
	xload	xmm5, [dstreg+16]		;; Reload new R2
	subpd	xmm4, xmm5			;; R1 = R1 - R2 (new R2)
	addpd	xmm5, [dstreg]			;; R2 = R1 + R2 (new R1)

	xstore	[dstreg+e1+48], xmm1		;; Save I4
	xstore	[dstreg+e1+32], xmm0		;; Save I2

	xcopy	xmm1, xmm3			;; Copy I1
	subpd	xmm3, xmm2			;; I1 = I1 - I3 (final I3)
	addpd	xmm2, xmm1			;; I3 = I1 + I3 (final I1)

	xcopy	xmm1, xmm4			;; Copy R2
	subpd	xmm4, xmm6			;; R2 = R2 - R4 (final R4)
	addpd	xmm6, xmm1			;; R4 = R2 + R4 (final R2)

	xcopy	xmm1, xmm5			;; Copy R1
	subpd	xmm5, xmm7			;; R1 = R1 - R3 (final R3)
	addpd	xmm7, xmm1			;; R3 = R1 + R3 (final R1)

	xstore	[dstreg+32], xmm2		;; Save I1
	xstore	[dstreg+48], xmm3		;; Save I3
	xstore	[dstreg+e1], xmm6		;; Save R2
	xstore	[dstreg+e1+16], xmm4		;; Save R4
	xstore	[dstreg], xmm7			;; Save R1
	xstore	[dstreg+16], xmm5		;; Save R3
	bump	dstreg, dstinc
	ENDM

IFDEF X86_64

r4_sg2cl_four_complex_unfft4 MACRO srcreg,srcinc,d1,dstreg,dstinc,e1,screg
	shuffle_load_with_temp xmm1, xmm2, [srcreg], [srcreg+16], xmm0		;; R1,R2

	xload	xmm3, [screg+0+16]		;; cosine/sine 1		; 1
	xcopy	xmm15, xmm1			;; Copy R1			; 1-3
	mulpd	xmm1, xmm3			;; A1 = R1 * cosine/sine	; 1-5

	xload	xmm4, [screg+64+16]		;; cosine/sine 2		; 2
	xcopy	xmm5, xmm2			;; Copy R2			; 2-4
	mulpd	xmm2, xmm4			;; A2 = R2 * cosine/sine	; 2-6

	shuffle_load_with_temp xmm6, xmm7, [srcreg+32], [srcreg+48], xmm0	;; I1,I2

	mulpd	xmm3, xmm6			;; B1 = I1 * cosine/sine	; 3-7
	mulpd	xmm4, xmm7			;; B2 = I2 * cosine/sine	; 4-8

	shuffle_load_with_temp xmm8, xmm9, [srcreg+d1], [srcreg+d1+16], xmm0	;; R3,R4

	xload	xmm10, [screg+32+16]		;; cosine/sine 3		; 5
	xcopy	xmm11, xmm8			;; Copy R3			; 5-7
	mulpd	xmm8, xmm10			;; A3 = R3 * cosine/sine	; 5-9

	addpd	xmm1, xmm6			;; A1 = A1 + I1			; 6-8	avail 0,6,12-14
	xload	xmm12, [screg+96+16]		;; cosine/sine 4		; 6	avail 0,6,13,14
	xcopy	xmm13, xmm9			;; Copy R4			; 6-8	avail 0,6,14
	mulpd	xmm9, xmm12			;; A4 = R4 * cosine/sine	; 6-10

	shuffle_load_with_temp xmm6, xmm14, [srcreg+d1+32], [srcreg+d1+48], xmm0 ;; I3,I4

	addpd	xmm2, xmm7			;; A2 = A2 + I2			; 7-9	avail 0,7
	mulpd	xmm10, xmm6			;; B3 = I3 * cosine/sine	; 7-11
	xload	xmm0, [screg+0]			;; sine 1			; 7	avail 7

	subpd	xmm3, xmm15			;; B1 = B1 - R1			; 8-10	avail 7,15
	mulpd	xmm12, xmm14			;; B4 = I4 * cosine/sine	; 8-12
	xload	xmm7, [screg+64]		;; sine 2			; 8	avail 15

	subpd	xmm4, xmm5			;; B2 = B2 - R2			; 9-11	avail 15,5
	mulpd	xmm1, xmm0			;; A1 = A1 * sine (new R1)	; 9-13

	addpd	xmm8, xmm6			;; A3 = A3 + I3			; 10-12	avail 15,5,6
	mulpd	xmm2, xmm7			;; A2 = A2 * sine (new R2)	; 10-14
	xload	xmm5, [screg+32]		;; sine 3			; 10	avail 15,6

	addpd	xmm9, xmm14			;; A4 = A4 + I4			; 11-13	avail 15,6,14
	mulpd	xmm3, xmm0			;; B1 = B1 * sine (new I1)	; 11-15	avail 15,6,14,0
	xload	xmm6, [screg+96]		;; sine 4			; 11	avail 15,14,0

	subpd	xmm10, xmm11			;; B3 = B3 - R3			; 12-14	avail 15,14,0,11
	mulpd	xmm4, xmm7			;; B2 = B2 * sine (new I2)	; 12-16	avail 15,14,0,11,7

	subpd	xmm12, xmm13			;; B4 = B4 - R4			; 13-15	avail 15,14,0,11,7,13
	mulpd	xmm8, xmm5			;; A3 = A3 * sine (new R3)	; 13-17

	xcopy	xmm0, xmm1			;; Copy R1			; 14-16 (14 on a Core i7) avail 15,14,11,7,13
	mulpd	xmm9, xmm6			;; A4 = A4 * sine (new R4)	; 14-18
	bump	srcreg, srcinc

	subpd	xmm1, xmm2			;; R1 = R1 - R2 (newer R2)	; 15-17
	mulpd	xmm10, xmm5			;; B3 = B3 * sine (new I3)	; 15-19	avail 15,14,11,7,13,5

	xprefetchw [dstreg+dstinc]

	addpd	xmm2, xmm0			;; R2 = R1 + R2 (newer R1)	; 17-19 (16-18 on a Core i7) avail 15,14,11,7,13,5,0
	mulpd	xmm12, xmm6			;; B4 = B4 * sine (new I4)	; 16-20	avail 15,14,11,7,13,5,0,12
	xcopy	xmm0, xmm3			;; Copy I1			; 16-18	avail 15,14,11,7,13,5,12

	subpd	xmm3, xmm4			;; I1 = I1 - I2 (newer I2)	; 18-20
	xcopy	xmm5, xmm8			;; Copy R3			; 18-20	avail 15,14,11,7,13,12

	addpd	xmm4, xmm0			;; I2 = I1 + I2 (newer I1)	; 19-21	avail 15,14,11,7,13,12,0

	xprefetchw [dstreg+dstinc+e1]

	addpd	xmm8, xmm9			;; R3 = R4 + R3 (newer R3)	; 20-22
	xcopy	xmm0, xmm10			;; Copy I3			; 20-22	avail 15,14,11,7,13,12

	subpd	xmm9, xmm5			;; R4 = R4 - R3 (newer I4)	; 21-23	avail 15,14,11,7,13,12,5
	xcopy	xmm7, xmm2			;; Copy R1			; 21-23 (20) avail 15,14,11,13,12,5

	subpd	xmm10, xmm12			;; I3 = I3 - I4 (newer R4)	; 22-24
	xcopy	xmm5, xmm3			;; Copy I2			; 22-24 (21) avail 15,14,11,13,12

	xprefetch [srcreg]

	addpd	xmm12, xmm0			;; I4 = I3 + I4 (newer I3)	; 23-25	avail 15,14,11,13,12,0
	xcopy	xmm11, xmm1			;; Copy R2			; 23-25 (18) avail 15,14,13,12,0

	subpd	xmm2, xmm8			;; R1 = R1 - R3 (final R3)	; 24-26
	xcopy	xmm0, xmm4			;; Copy I1			; 24-26 (22) avail 15,14,13,12

	addpd	xmm8, xmm7			;; R3 = R1 + R3 (final R1)	; 25-27	avail 15,14,13,12,7

	xprefetch [srcreg+d1]

	subpd	xmm3, xmm9			;; I2 = I2 - I4 (final I4)	; 26-28

	addpd	xmm9, xmm5			;; I4 = I2 + I4 (final I2)	; 27-29	avail 15,14,13,12,7,5
	xstore	[dstreg+16], xmm2		;; Save R3			; 27

	subpd	xmm1, xmm10			;; R2 = R2 - R4 (final R4)	; 28-30
	xstore	[dstreg], xmm8			;; Save R1			; 28

	addpd	xmm10, xmm11			;; R4 = R2 + R4 (final R2)	; 29-31	avail 15,14,13,12,7,5,11
	xstore	[dstreg+e1+48], xmm3		;; Save I4			; 29

	subpd	xmm4, xmm12			;; I1 = I1 - I3 (final I3)	; 30-32
	xstore	[dstreg+e1+32], xmm9		;; Save I2			; 30

	addpd	xmm12, xmm0			;; I3 = I1 + I3 (final I1)	; 31-33	avail 15,14,13,12,7,5,11,0
	xstore	[dstreg+e1+16], xmm1		;; Save R4			; 31

	xstore	[dstreg+e1], xmm10		;; Save R2			; 32
	xstore	[dstreg+48], xmm4		;; Save I3			; 33
	xstore	[dstreg+32], xmm12		;; Save I1			; 34
	bump	dstreg, dstinc
	ENDM
ENDIF

;;
;; ************************************* eight-reals-fft variants ******************************************
;;

;; These macros operate on eight reals doing 2 and 3/4 levels of the FFT and applying
;; the sin/cos multipliers afterwards.  The output is 2 reals (only 2 levels of FFT done)
;; and 3 complex numbers (3 levels of FFT performed).  These macros take a screg
;; that points to twiddles w^n, w^2n, and w^5n.

;;r4_x2cl_eight_reals_first_fft_preload MACRO
;;	r4_x2cl_eight_reals_first_fft_cmn_preload
;;	ENDM
;;r4_x2cl_eight_reals_first_fft MACRO srcreg,srcinc,d1,screg
;;	r4_x2cl_eight_reals_first_fft_cmn srcreg,rbx,srcinc,d1,screg
;;	ENDM

r4_x2cl_eight_reals_first_fft_scratch_preload MACRO
	r4_x2cl_eight_reals_first_fft_cmn_preload
	ENDM
r4_x2cl_eight_reals_first_fft_scratch MACRO srcreg,srcinc,d1,screg
	r4_x2cl_eight_reals_first_fft_cmn srcreg,0,srcinc,d1,screg
	ENDM

r4_x2cl_eight_reals_first_fft_cmn_preload MACRO
	r4_x8r_fft_mem_preload
	ENDM
r4_x2cl_eight_reals_first_fft_cmn MACRO srcreg,off,srcinc,d1,screg
	r4_x8r_fft_mem [srcreg+off],[srcreg+off+d1],[srcreg+off+16],[srcreg+off+d1+16],[srcreg+off+32],[srcreg+off+d1+32],[srcreg+off+48],[srcreg+off+d1+48],screg,screg+32,screg+64,srcreg+srcinc,d1,[srcreg],[srcreg+16]
;;	xstore	[srcreg], xmm0		;; Save real value #1
;;	xstore	[srcreg+16], xmm0	;; Save real value #2
	xstore	[srcreg+32], xmm7	;; Save R2
	xstore	[srcreg+48], xmm5	;; Save I2
	xstore	[srcreg+d1], xmm4	;; Save R3
	xstore	[srcreg+d1+16], xmm3	;; Save I3
	xstore	[srcreg+d1+32], xmm6	;; Save R4
	xstore	[srcreg+d1+48], xmm2	;; Save I4
	bump	srcreg, srcinc
	ENDM

;; Common macro for eight-reals-FFT doing 2 and 3/4 levels.

r4_x8r_fft_mem_preload MACRO
	ENDM
r4_x8r_fft_mem MACRO mem1,mem2,mem3,mem4,mem5,mem6,mem7,mem8,screg1,screg2,screg5,pre1,pre2,dst1,dst2
	xload	xmm3, mem4		;; R4
	xload	xmm7, mem8		;; R8
	xcopy	xmm0, xmm3
	subpd	xmm3, xmm7		;; new R8 = R4 - R8
	addpd	xmm7, xmm0		;; new R4 = R4 + R8

	xload	xmm1, mem2		;; R2
	xload	xmm5, mem6		;; R6
	xcopy	xmm4, xmm1
	subpd	xmm1, xmm5		;; new R6 = R2 - R6
	addpd	xmm5, xmm4		;; new R2 = R2 + R6

	mulpd	xmm3, XMM_SQRTHALF	;; R8 = R8 * square root
	mulpd	xmm1, XMM_SQRTHALF	;; R6 = R6 * square root

	xload	xmm0, mem1		;; R1
	xload	xmm4, mem5		;; R5
	xcopy	xmm2, xmm0
	subpd	xmm0, xmm4		;; new R5 = R1 - R5
	addpd	xmm4, xmm2		;; new R1 = R1 + R5

	xcopy	xmm2, xmm5		;; Copy R2
	subpd	xmm5, xmm7		;; R2 = R2 - R4 (final I2)
	addpd	xmm7, xmm2		;; R4 = R2 + R4 (final I1, a.k.a 2nd real result)

	xcopy	xmm6, xmm1		;; Copy R6
	subpd	xmm1, xmm3		;; R6 = R6 - R8 (Real part)
	addpd	xmm3, xmm6		;; R8 = R6 + R8 (Imaginary part)

	xload	xmm6, mem7		;; R7
	addpd	xmm6, mem3		;; new R3 = R3 + R7

	xcopy	xmm2, xmm4		;; Copy R1
	subpd	xmm4, xmm6		;; R1 = R1 - R3 (final R2)
	addpd	xmm6, xmm2		;; R3 = R1 + R3 (final R1)

	xload	xmm2, mem3		;; R3
	subpd	xmm2, mem7		;; new R7 = R3 - R7

	xprefetchw [pre1]

	xstore	dst1, xmm6		;; Save R1
	xstore	dst2, xmm7		;; Save I1

	xcopy	xmm7, xmm0		;; Copy R5
	subpd	xmm0, xmm1		;; R5 = R5 - R6 (final R4)
	addpd	xmm1, xmm7		;; R6 = R5 + R6 (final R3)

	xcopy	xmm6, xmm2		;; Copy R7
	subpd	xmm2, xmm3		;; R7 = R7 - R8 (final I4)
	addpd	xmm3, xmm6		;; R8 = R7 + R8 (final I3)

	xprefetchw [pre1][pre2]

	xload	xmm7, [screg2+16]	;; cosine/sine for w^2n
	mulpd	xmm7, xmm4		;; A2 = R2 * cosine/sine
	subpd	xmm7, xmm5		;; A2 = A2 - I2
	mulpd	xmm5, [screg2+16]	;; B2 = I2 * cosine/sine
	addpd	xmm5, xmm4		;; B2 = B2 + R2

	xload	xmm6, [screg5+16]	;; cosine/sine for w^5n
	mulpd	xmm6, xmm0		;; A4 = R4 * cosine/sine
	subpd	xmm6, xmm2		;; A4 = A4 - I4
	mulpd	xmm2, [screg5+16]	;; B4 = I4 * cosine/sine
	addpd	xmm2, xmm0		;; B4 = B4 + R4

	xload	xmm4, [screg1+16]	;; cosine/sine for w^n
	mulpd	xmm4, xmm1		;; A3 = R3 * cosine/sine
	subpd	xmm4, xmm3		;; A3 = A3 - I3
	mulpd	xmm3, [screg1+16]	;; B3 = I3 * cosine/sine
	addpd	xmm3, xmm1		;; B3 = B3 + R3

	xload	xmm0, [screg2]
	mulpd	xmm7, xmm0		;; A2 = A2 * sine (final R2)
	mulpd	xmm5, xmm0		;; B2 = B2 * sine (final I2)
	xload	xmm1, [screg5]
	mulpd	xmm6, xmm1		;; A4 = A4 * sine (final R4)
	mulpd	xmm2, xmm1		;; B4 = B4 * sine (final I4)
	xload	xmm0, [screg1]
	mulpd	xmm4, xmm0		;; A3 = A3 * sine (final R3)
	mulpd	xmm3, xmm0		;; B3 = B3 * sine (final I3)
	ENDM

;; 32-bit AMD K8 optimized versions of the above macros

IF (@INSTR(,%xarch,<K8>) NE 0)

;R8	AAAA
;R4	  AAAA
;R6	    AAAA
;R2	      AAAA
;R8*sqrt    MMMM
;R6*sqrt        MMMM
;R1	        AAAA
;R5	          AAAA
;FI2(R2-R4)	    AAAA
;FI1(R2+R4)	    MMMMAAAA
;R6(R6-R8)	      AAAA
;R8(R6+R8)	      MMMMAAAA
;R3		            AAAA
;R7			      AAAA
;FR2(R1-R3)		        AAAA
;FR1(R1+R3)		        MMMMAAAA
;FR4(R5-R6)			  AAAA
;FR3(R5+R6)			  MMMM  AAAA
;FI4(R7-R8)			      AAAA
;FI3(R7+R8)			      MMMMAAAA
;A2	                            MMMMMMMMAAAA  
;A4				          MMMMAAAA  MMMM
;B2	                    MMMM            MMMMAAAA    
;B4					      MMMMAAAAMMMM
;A3					        MMMMAAAAMMMM
;B3					          MMMMAAAAMMMM

r4_x8r_fft_mem MACRO mem1,mem2,mem3,mem4,mem5,mem6,mem7,mem8,screg1,screg2,screg5,pre1,pre2,dst1,dst2
	xload	xmm3, mem4		;; R4				; K8
	xload	xmm7, mem8		;; R8
	subpd	xmm3, xmm7		;; new R8 = R4 - R8		; 1-4
	addpd	xmm7, mem4		;; new R4 = R4 + R8		; 3-6

	xload	xmm1, mem2		;; R2
	xload	xmm5, mem6		;; R6
	subpd	xmm1, xmm5		;; new R6 = R2 - R6		; 5-8
	addpd	xmm5, mem2		;; new R2 = R2 + R6		; 7-10

	mulpd	xmm3, XMM_SQRTHALF	;; R8 = R8 * square root	; 5-8

	xload	xmm0, mem1		;; R1
	xload	xmm4, mem5		;; R5
	subpd	xmm0, xmm4		;; new R5 = R1 - R5		; 9-12
	addpd	xmm4, mem1		;; new R1 = R1 + R5		; 11-14

	mulpd	xmm1, XMM_SQRTHALF	;; R6 = R6 * square root	; 9-12

	subpd	xmm5, xmm7		;; R2 = R2 - R4 (final I2)	; 13-16
	multwo	xmm7			;; R4 = R4 * 2			; 13-16
	subpd	xmm1, xmm3		;; R6 = R6 - R8 (Real part)	; 15-18
	multwo	xmm3			;; R8 = R8 * 2			; 15-18
	addpd	xmm7, xmm5		;; R4 = R2 + R4 (final I1)	; 17-20
	addpd	xmm3, xmm1		;; R8 = R6 + R8 (Imaginary part); 19-22

	xload	xmm6, mem7		;; R7
	addpd	xmm6, mem3		;; new R3 = R3 + R7		; 21-24

	xload	xmm2, mem3		;; R3
	subpd	xmm2, mem7		;; new R7 = R3 - R7		; 23-26
	mulpd	xmm5, [screg2]		;; I2 = I2 * sine		; 23-26

	xstore	dst2, xmm7		;; Save I1			; 21

	subpd	xmm4, xmm6		;; R1 = R1 - R3 (final R2)	; 25-28
	multwo	xmm6			;; R3 = R3 * 2			; 25-28

	xprefetchw [pre1]

	subpd	xmm0, xmm1		;; R5 = R5 - R6 (final R4)	; 27-30
	multwo	xmm1			;; R6 = R6 * 2			; 27-30

	addpd	xmm6, xmm4		;; R3 = R1 + R3 (final R1)	; 29-32
	mulpd	xmm4, [screg2]		;; R2 = R2 * sine		; 29-32

	xprefetchw [pre1][pre2]

	subpd	xmm2, xmm3		;; R7 = R7 - R8 (final I4)	; 31-34
	multwo	xmm3			;; R8 = R8 * 2			; 31-34
	xload	xmm7, [screg2+16]	;; cosine/sine for w^2n

	addpd	xmm1, xmm0		;; R6 = R5 + R6 (final R3)	; 33-36
	mulpd	xmm7, xmm4		;; A2 = R2 * cosine/sine	; 33-36
	xstore	dst1, xmm6		;; Save R1			; 33

	addpd	xmm3, xmm2		;; R8 = R7 + R8 (final I3)	; 35-38
	xload	xmm6, [screg5+16]	;; cosine/sine for w^5n
	mulpd	xmm6, xmm0		;; A4 = R4 * cosine/sine	; 35-38

	subpd	xmm7, xmm5		;; A2 = A2 - I2			; 37-40
	mulpd	xmm5, [screg2+16]	;; B2 = I2 * cosine/sine	; 37-40

	subpd	xmm6, xmm2		;; A4 = A4 - I4			; 39-42
	mulpd	xmm2, [screg5+16]	;; B4 = I4 * cosine/sine	; 39-42

	addpd	xmm5, xmm4		;; B2 = B2 + R2			; 41-44
	xload	xmm4, [screg1+16]	;; cosine/sine for w^n
	mulpd	xmm4, xmm1		;; A3 = R3 * cosine/sine	; 41-44

	addpd	xmm2, xmm0		;; B4 = B4 + R4			; 43-46

	subpd	xmm4, xmm3		;; A3 = A3 - I3			; 45-48
	mulpd	xmm3, [screg1+16]	;; B3 = I3 * cosine/sine	; 43-46
	mulpd	xmm6, [screg5]		;; A4 = A4 * sine (final R4)	; 45-48

	addpd	xmm3, xmm1		;; B3 = B3 + R3			; 47-50
	mulpd	xmm2, [screg5]		;; B4 = B4 * sine (final I4)	; 47-50
	mulpd	xmm4, [screg1]		;; A3 = A3 * sine (final R3)	; 49-52
	mulpd	xmm3, [screg1]		;; B3 = B3 * sine (final I3)	; 51-54
	ENDM

ENDIF

;; 64-bit Intel and AMD K10 optimized versions of the above macros

IF (@INSTR(,%xarch,<CORE>) NE 0) OR (@INSTR(,%xarch,<P4>) NE 0) OR (@INSTR(,%xarch,<K10>) NE 0) OR (@INSTR(,%xarch,<BLEND>) NE 0)
IFDEF X86_64

; Optimal Core 2 timing is 24 clocks, currently at 27 clocks

r4_x8r_fft_mem_preload MACRO
	xload	xmm15, XMM_SQRTHALF
	ENDM

r4_x8r_fft_mem MACRO mem1,mem2,mem3,mem4,mem5,mem6,mem7,mem8,screg1,screg2,screg5,pre1,pre2,dst1,dst2
	xload	xmm14, mem4		;; R4
	xload	xmm11, mem8		;; R8
	xcopy	xmm0, xmm14		;; Copy R4
	subpd	xmm14, xmm11		;; new R8 = R4 - R8			; 1-3

	xload	xmm4, mem2		;; R2
	xload	xmm5, mem6		;; R6
	xcopy	xmm1, xmm4		;; Copy R2
	subpd	xmm4, xmm5		;; new R6 = R2 - R6			; 2-4

	addpd	xmm11, xmm0		;; new R4 = R4 + R8			; 3-5

	addpd	xmm5, xmm1		;; new R2 = R2 + R6			; 4-6
	mulpd	xmm14, xmm15		;; R8 = R8 * square root		; 4-8

	xload	xmm0, mem1		;; R1
	xload	xmm1, mem5		;; R5
	xcopy	xmm6, xmm0		;; Copy R1
	addpd	xmm0, xmm1		;; new R1 = R1 + R5			; 5-7
	mulpd	xmm4, xmm15		;; R6 = R6 * square root		; 5-9

	xload	xmm13, mem3		;; R3
	xload	xmm8, mem7		;; R7
	xcopy	xmm9, xmm13		;; Copy R3
	addpd	xmm13, xmm8 		;; new R3 = R3 + R7			; 6-8

	subpd	xmm6, xmm1		;; new R5 = R1 - R5			; 7-9

	xcopy	xmm1, xmm5		;; Copy R2
	subpd	xmm5, xmm11		;; R2 = R2 - R4 (final I2)		; 8-10

	xcopy	xmm10, xmm0		;; Copy R1
	subpd	xmm0, xmm13		;; R1 = R1 - R3 (final R2)		; 9-11

	xcopy	xmm7, xmm4		;; Copy R6
	subpd	xmm4, xmm14		;; R6 = R6 - R8 (Real part)		; 10-12

	addpd	xmm14, xmm7		;; R8 = R6 + R8 (Imaginary part)	; 11-13
	xload	xmm7, [screg2+16]	;; cosine/sine for w^2n

	subpd	xmm9, xmm8		;; new R7 = R3 - R7			; 12-14
	xload	xmm2, [screg5+16]	;; cosine/sine for w^5n

	xcopy	xmm8, xmm6		;; Copy R5
	subpd	xmm6, xmm4		;; R5 = R5 - R6 (final R4)		; 13-15
	xload	xmm3, [screg1+16]	;; cosine/sine for w^n

	xprefetchw [pre1]

	addpd	xmm4, xmm8		;; R6 = R5 + R6 (final R3)		; 14-16
	xcopy	xmm12, xmm5		;; Copy I2
	mulpd	xmm5, xmm7		;; B2 = I2 * cosine/sine		; 14-18

	xcopy	xmm8, xmm9		;; Copy R7
	subpd	xmm9, xmm14		;; R7 = R7 - R8 (final I4)		; 15-17
	mulpd	xmm7, xmm0		;; A2 = R2 * cosine/sine		; 15-19

	addpd	xmm14, xmm8		;; R8 = R7 + R8 (final I3)		; 16-18
	xcopy	xmm8, xmm6		;; Copy R4
	mulpd	xmm6, xmm2		;; A4 = R4 * cosine/sine		; 16-20

	addpd	xmm11, xmm1		;; R4 = R2 + R4 (final I1, a.k.a 2nd real result) ; 17-19
	xcopy	xmm1, xmm4		;; Copy R3
	mulpd	xmm4, xmm3		;; A3 = R3 * cosine/sine		; 17-21

	addpd	xmm13, xmm10		;; R3 = R1 + R3 (final R1)		; 18-20
	mulpd	xmm2, xmm9		;; B4 = I4 * cosine/sine		; 18-22
	xload	xmm10, [screg2]		;; sine

	addpd	xmm5, xmm0		;; B2 = B2 + R2				; 19-21
	mulpd	xmm3, xmm14		;; B3 = I3 * cosine/sine		; 19-23
	xload	xmm0, [screg5]

	xprefetchw [pre1][pre2]

	subpd	xmm7, xmm12		;; A2 = A2 - I2				; 20-22
	xload	xmm12, [screg1]
	xstore	dst2, xmm11		;; Save I1				; 20

	subpd	xmm6, xmm9		;; A4 = A4 - I4				; 21-23
	xstore	dst1, xmm13		;; Save R1				; 21

	subpd	xmm4, xmm14		;; A3 = A3 - I3				; 22-24
	mulpd	xmm5, xmm10		;; B2 = B2 * sine (final I2)		; 22-26

	addpd	xmm2, xmm8		;; B4 = B4 + R4				; 23-25
	mulpd	xmm7, xmm10		;; A2 = A2 * sine (final R2)		; 23-27

	addpd	xmm3, xmm1		;; B3 = B3 + R3				; 24-26
	mulpd	xmm6, xmm0		;; A4 = A4 * sine (final R4)		; 24-28

	mulpd	xmm4, xmm12		;; A3 = A3 * sine (final R3)		; 25-29
	mulpd	xmm2, xmm0		;; B4 = B4 * sine (final I4)		; 26-30
	mulpd	xmm3, xmm12		;; B3 = B3 * sine (final I3)		; 27-31
	ENDM

ENDIF
ENDIF

;; 64-bit AMD K8 optimized versions of the above macros.  Same as 32-bit version with
;; some constants preloaded.

IF (@INSTR(,%xarch,<K8>) NE 0)
IFDEF X86_64

r4_x8r_fft_mem_preload MACRO
	xload	xmm14, XMM_SQRTHALF
	xload	xmm15, XMM_TWO
	ENDM

r4_x8r_fft_mem MACRO mem1,mem2,mem3,mem4,mem5,mem6,mem7,mem8,screg1,screg2,screg5,pre1,pre2,dst1,dst2
	xload	xmm3, mem4		;; R4				; K8
	xload	xmm7, mem8		;; R8
	subpd	xmm3, xmm7		;; new R8 = R4 - R8		; 1-4
	addpd	xmm7, mem4		;; new R4 = R4 + R8		; 3-6

	xload	xmm1, mem2		;; R2
	xload	xmm5, mem6		;; R6
	subpd	xmm1, xmm5		;; new R6 = R2 - R6		; 5-8
	addpd	xmm5, mem2		;; new R2 = R2 + R6		; 7-10

	mulpd	xmm3, xmm14		;; R8 = R8 * square root	; 5-8

	xload	xmm0, mem1		;; R1
	xload	xmm4, mem5		;; R5
	subpd	xmm0, xmm4		;; new R5 = R1 - R5		; 9-12
	addpd	xmm4, mem1		;; new R1 = R1 + R5		; 11-14

	mulpd	xmm1, xmm14		;; R6 = R6 * square root	; 9-12

	subpd	xmm5, xmm7		;; R2 = R2 - R4 (final I2)	; 13-16
	mulpd	xmm7, xmm15		;; R4 = R4 * 2			; 13-16
	subpd	xmm1, xmm3		;; R6 = R6 - R8 (Real part)	; 15-18
	mulpd	xmm3, xmm15		;; R8 = R8 * 2			; 15-18
	addpd	xmm7, xmm5		;; R4 = R2 + R4 (final I1)	; 17-20
	addpd	xmm3, xmm1		;; R8 = R6 + R8 (Imaginary part); 19-22

	xload	xmm6, mem7		;; R7
	addpd	xmm6, mem3		;; new R3 = R3 + R7		; 21-24

	xload	xmm2, mem3		;; R3
	subpd	xmm2, mem7		;; new R7 = R3 - R7		; 23-26
	mulpd	xmm5, [screg2]		;; I2 = I2 * sine		; 23-26

	xstore	dst2, xmm7		;; Save I1			; 21

	subpd	xmm4, xmm6		;; R1 = R1 - R3 (final R2)	; 25-28
	mulpd	xmm6, xmm15		;; R3 = R3 * 2			; 25-28

	xprefetchw [pre1]

	subpd	xmm0, xmm1		;; R5 = R5 - R6 (final R4)	; 27-30
	mulpd	xmm1, xmm15		;; R6 = R6 * 2			; 27-30

	addpd	xmm6, xmm4		;; R3 = R1 + R3 (final R1)	; 29-32
	mulpd	xmm4, [screg2]		;; R2 = R2 * sine		; 29-32

	xprefetchw [pre1][pre2]

	subpd	xmm2, xmm3		;; R7 = R7 - R8 (final I4)	; 31-34
	mulpd	xmm3, xmm15		;; R8 = R8 * 2			; 31-34
	xload	xmm7, [screg2+16]	;; cosine/sine for w^2n

	addpd	xmm1, xmm0		;; R6 = R5 + R6 (final R3)	; 33-36
	mulpd	xmm7, xmm4		;; A2 = R2 * cosine/sine	; 33-36
	xstore	dst1, xmm6		;; Save R1			; 33

	addpd	xmm3, xmm2		;; R8 = R7 + R8 (final I3)	; 35-38
	xload	xmm6, [screg5+16]	;; cosine/sine for w^5n
	mulpd	xmm6, xmm0		;; A4 = R4 * cosine/sine	; 35-38

	subpd	xmm7, xmm5		;; A2 = A2 - I2			; 37-40
	mulpd	xmm5, [screg2+16]	;; B2 = I2 * cosine/sine	; 37-40

	subpd	xmm6, xmm2		;; A4 = A4 - I4			; 39-42
	mulpd	xmm2, [screg5+16]	;; B4 = I4 * cosine/sine	; 39-42

	addpd	xmm5, xmm4		;; B2 = B2 + R2			; 41-44
	xload	xmm4, [screg1+16]	;; cosine/sine for w^n
	mulpd	xmm4, xmm1		;; A3 = R3 * cosine/sine	; 41-44

	addpd	xmm2, xmm0		;; B4 = B4 + R4			; 43-46

	subpd	xmm4, xmm3		;; A3 = A3 - I3			; 45-48
	mulpd	xmm3, [screg1+16]	;; B3 = I3 * cosine/sine	; 43-46
	mulpd	xmm6, [screg5]		;; A4 = A4 * sine (final R4)	; 45-48

	addpd	xmm3, xmm1		;; B3 = B3 + R3			; 47-50
	mulpd	xmm2, [screg5]		;; B4 = B4 * sine (final I4)	; 47-50
	mulpd	xmm4, [screg1]		;; A3 = A3 * sine (final R3)	; 49-52
	mulpd	xmm3, [screg1]		;; B3 = B3 * sine (final I3)	; 51-54
	ENDM

ENDIF
ENDIF

;;
;; ************************************* eight-reals-unfft variants ******************************************
;;

;; These macros produce eight reals after doing 2 and 3/4 levels of the inverse FFT applying
;; the sin/cos multipliers beforehand.  The input is 2 reals (only 2 levels of inverse FFT done)
;; and 3 complex numbers (3 levels of inverse FFT performed).  These macros take a screg
;; that points to twiddles w^n, w^2n, and w^5n.

r4_x4cl_eight_reals_last_unfft_preload MACRO
	r4_x4cl_eight_reals_unfft_cmn_preload
	ENDM

r4_x4cl_eight_reals_last_unfft MACRO srcreg,srcinc,d1,d2,screg,scoff
	r4_x4cl_eight_reals_unfft_cmn srcreg,srcinc,d1,d2,screg,scoff,screg+32,scoff,screg+64,scoff
	ENDM

;; This macro is similar to r4_x4cl_eight_reals_last_unfft.
;; The difference is this macro takes two screg pointers.
;; One points to w^n and w^5n, the other points to w^2n which is
;; also the screg used by the four-complex-FFT.

r4_x4cl_eight_reals_unfft_preload MACRO
	r4_x4cl_eight_reals_unfft_cmn_preload
	ENDM

r4_x4cl_eight_reals_unfft MACRO srcreg,srcinc,d1,d2,screg1,scoff1,screg2,scoff2
	r4_x4cl_eight_reals_unfft_cmn srcreg,srcinc,d1,d2,screg2,scoff2,screg1,scoff1,screg2+32,scoff2
	ENDM

; Common eight reals unfft macro

r4_x4cl_eight_reals_unfft_cmn_preload MACRO
	ENDM

r4_x4cl_eight_reals_unfft_cmn MACRO srcreg,srcinc,d1,d2,screg1,scoff1,screg2,scoff2,screg5,scoff5
	d3 = d2 + d1
	tmp = srcinc+d2
	r4_x8r_unfft_mem [srcreg+16],[srcreg+48],[srcreg+d1+16],[srcreg+d1+48],[srcreg+d2+16],[srcreg+d2+48],[srcreg+d3+16],[srcreg+d3+48],screg1+scoff1,screg2+scoff2,screg5+scoff5,srcreg+tmp,d1,[srcreg+d3+16],[srcreg+d3+48]
	xstore	[srcreg+d1+16], xmm7	;; Save R3
;;	xstore	[srcreg+d3+16], xmm2	;; Save R4
	xstore	[srcreg+d1+48], xmm6	;; Save R7
;;	xstore	[srcreg+d3+48], xmm5	;; Save R8
	xload	xmm6, [srcreg+d3]	;; Load R4
	xload	xmm7, [srcreg+d3+32]	;; Load I4
	xload	xmm2, [srcreg+d1]	;; Load R2
	xload	xmm5, [srcreg+d1+32]	;; Load I2
	xstore	[srcreg+d1], xmm1	;; Save R1
	xstore	[srcreg+d3], xmm4	;; Save R2
	xstore	[srcreg+d1+32], xmm0	;; Save R5
	xstore	[srcreg+d3+32], xmm3	;; Save R6
	r4_x8r_unfft_partial_mem xmm0,xmm1,xmm2,xmm5,xmm3,xmm4,xmm6,xmm7,[srcreg],[srcreg+32],[srcreg+d2],[srcreg+d2+32],screg1,screg2,screg5,srcreg+srcinc,d1,[srcreg+d2+16],[srcreg+d2+48]
	xstore	[srcreg], xmm1		;; Save R1
	xstore	[srcreg+d2], xmm3	;; Save R2
	xstore	[srcreg+32], xmm0	;; Save R5
	xstore	[srcreg+d2+32], xmm5	;; Save R6
	xstore	[srcreg+16], xmm7	;; Save R3
;;	xstore	[srcreg+d2+16], xmm2	;; Save R4
	xstore	[srcreg+48], xmm6	;; Save R7
;;	xstore	[srcreg+d2+48], xmm4	;; Save R8
	bump	srcreg, srcinc
	ENDM

;; Common macro to do the 2 and 3/4 inverse FFT producing eight reals.

r4_x8r_unfft_mem MACRO mem1,mem2,mem3,mem4,mem5,mem6,mem7,mem8,screg1,screg2,screg5,pre1,pre2,dst4,dst8
	xload	xmm6, [screg5+16]	;; cosine/sine for w^5n
	xload	xmm1, mem7		;; R4
	mulpd	xmm1, xmm6		;; A4 = R4 * cosine/sine
	xload	xmm7, mem8		;; I4
	addpd	xmm1, xmm7		;; A4 = A4 + I4
	mulpd	xmm7, xmm6		;; B4 = I4 * cosine/sine

	xload	xmm2, [screg1+16]	;; cosine/sine for w^n
	xload	xmm4, mem5		;; R3
	mulpd	xmm4, xmm2		;; A3 = R3 * cosine/sine for w^n
	xload	xmm5, mem6		;; I3
	addpd	xmm4, xmm5		;; A3 = A3 + I3
	mulpd	xmm5, xmm2		;; B3 = I3 * cosine/sine

	xload	xmm6, [screg2+16]	;; cosine/sine for w^2n
	xload	xmm0, mem3		;; R2
	mulpd	xmm0, xmm6		;; A2 = R2 * cosine/sine
	xload	xmm3, mem4		;; I2
	addpd	xmm0, xmm3		;; A2 = A2 + I2
	mulpd	xmm3, xmm6		;; B2 = I2 * cosine/sine

	subpd	xmm7, mem7		;; B4 = B4 - R4
	subpd	xmm5, mem5		;; B3 = B3 - R3
	subpd	xmm3, mem3		;; B2 = B2 - R2

	xprefetchw [pre1]

	xload	xmm6, [screg5]
	mulpd	xmm1, xmm6		;; new R7 = A4 * sine (final R4)
	mulpd	xmm7, xmm6		;; new R8 = B4 * sine (final I4)
	xload	xmm2, [screg1]
	mulpd	xmm4, xmm2		;; new R5 = A3 * sine (final R3)
	mulpd	xmm5, xmm2		;; new R6 = B3 * sine (final I3)
	xload	xmm6, [screg2]
	mulpd	xmm0, xmm6		;; new R3 = A2 * sine (final R2)
	mulpd	xmm3, xmm6		;; new R4 = B2 * sine (final I2)

	xprefetchw [pre1][pre2]

	xcopy	xmm6, xmm4		;; Copy R5
	subpd	xmm4, xmm1		;; new R6 = R5 - R7
	addpd	xmm1, xmm6		;; new R5 = R5 + R7

	xcopy	xmm2, xmm5		;; Copy R6
	subpd	xmm5, xmm7		;; new R8 = R6 - R8
	addpd	xmm7, xmm2		;; new R7 = R6 + R8

	xcopy	xmm6, xmm4		;; Copy R6
	addpd	xmm4, xmm5		;; R6 = R6 + R8
	subpd	xmm5, xmm6		;; R8 = R8 - R6

	mulpd	xmm4, XMM_SQRTHALF	;; R6 = R6 * square root of 1/2
	mulpd	xmm5, XMM_SQRTHALF	;; R8 = R8 * square root of 1/2

	xload	xmm2, mem2		;; R2
	subpd	xmm2, xmm3		;; R2 = R2 - R4 (new R4)
	addpd	xmm3, mem2		;; R4 = R2 + R4 (new R2)

	xcopy	xmm6, xmm2		;; Copy R4
	subpd	xmm2, xmm5		;; R4 = R4 - R8 (final R8)
	addpd	xmm5, xmm6		;; R8 = R4 + R8 (final R4)

	xcopy	xmm6, xmm3		;; Copy R2
	subpd	xmm3, xmm4		;; R2 = R2 - R6 (final R6)
	addpd	xmm4, xmm6		;; R6 = R2 + R6 (final R2)

	xload	xmm6, mem1		;; R1
	subpd	xmm6, xmm0		;; R1 = R1 - R3 (new R3)
	addpd	xmm0, mem1		;; R3 = R1 + R3 (new R1)

	xstore	dst8, xmm2		;; Save R8
	xstore	dst4, xmm5		;; Save R4

	xcopy	xmm2, xmm6		;; Copy R3
	subpd	xmm6, xmm7		;; R3 = R3 - R7 (final R7)
	addpd	xmm7, xmm2		;; R7 = R3 + R7 (final R3)

	xcopy	xmm5, xmm0		;; Copy R1
	subpd	xmm0, xmm1		;; R1 = R1 - R5 (final R5)
	addpd	xmm1, xmm5		;; R5 = R1 + R5 (final R1)
	ENDM

r4_x8r_unfft_partial_mem MACRO r1,r2,r3,r4,r5,r6,r7,r8,mem1,mem2,mem5,mem6,screg1,screg2,screg5,pre1,pre2,dst4,dst8
	xload	r2, [screg5+16]		;; cosine/sine for w^5n
	mulpd	r2, r7			;; A4 = R4 * cosine/sine
	addpd	r2, r8			;; A4 = A4 + I4
	mulpd	r8, [screg5+16]		;; B4 = I4 * cosine/sine
	subpd	r8, r7			;; B4 = B4 - R4

	xload	r1, [screg1+16]		;; cosine/sine for w^n
	xload	r5, mem5		;; R3
	mulpd	r5, r1			;; A3 = R3 * cosine/sine for w^n
	xload	r6, mem6		;; I3
	addpd	r5, r6			;; A3 = A3 + I3
	mulpd	r6, r1			;; B3 = I3 * cosine/sine
	subpd	r6, mem5		;; B3 = B3 - R3

	xload	r1, [screg2+16]		;; cosine/sine for w^2n
	mulpd	r1, r3			;; A2 = R2 * cosine/sine
	addpd	r1, r4			;; A2 = A2 + I2
	mulpd	r4, [screg2+16]		;; B2 = I2 * cosine/sine
	subpd	r4, r3			;; B2 = B2 - R2

	xprefetchw [pre1]

	xload	r7, [screg5]
	mulpd	r2, r7			;; new R7 = A4 * sine (final R4)
	mulpd	r8, r7			;; new R8 = B4 * sine (final I4)
	xload	r3, [screg1]
	mulpd	r5, r3			;; new R5 = A3 * sine (final R3)
	mulpd	r6, r3			;; new R6 = B3 * sine (final I3)
	xload	r7, [screg2]
	mulpd	r1, r7			;; new R3 = A2 * sine (final R2)
	mulpd	r4, r7			;; new R4 = B2 * sine (final I2)

	xprefetchw [pre1][pre2]

	xcopy	r7, r5			;; Copy R5
	subpd	r5, r2			;; new R6 = R5 - R7
	addpd	r2, r7			;; new R5 = R5 + R7

	xcopy	r3, r6			;; Copy R6
	subpd	r6, r8			;; new R8 = R6 - R8
	addpd	r8, r3			;; new R7 = R6 + R8

	xcopy	r7, r5			;; Copy R6
	addpd	r5, r6			;; R6 = R6 + R8
	subpd	r6, r7			;; R8 = R8 - R6

	mulpd	r5, XMM_SQRTHALF	;; R6 = R6 * square root of 1/2
	mulpd	r6, XMM_SQRTHALF	;; R8 = R8 * square root of 1/2

	xload	r3, mem2		;; R2
	subpd	r3, r4			;; R2 = R2 - R4 (new R4)
	addpd	r4, mem2		;; R4 = R2 + R4 (new R2)

	xcopy	r7, r3			;; Copy R4
	subpd	r3, r6			;; R4 = R4 - R8 (final R8)
	addpd	r6, r7			;; R8 = R4 + R8 (final R4)

	xcopy	r7, r4			;; Copy R2
	subpd	r4, r5			;; R2 = R2 - R6 (final R6)
	addpd	r5, r7			;; R6 = R2 + R6 (final R2)

	xload	r7, mem1		;; R1
	subpd	r7, r1			;; R1 = R1 - R3 (new R3)
	addpd	r1, mem1		;; R3 = R1 + R3 (new R1)

	xstore	dst8, r3		;; Save R8
	xstore	dst4, r6		;; Save R4

	xcopy	r3, r7			;; Copy R3
	subpd	r7, r8			;; R3 = R3 - R7 (final R7)
	addpd	r8, r3			;; R7 = R3 + R7 (final R3)

	xcopy	r6, r1			;; Copy R1
	subpd	r1, r2			;; R1 = R1 - R5 (final R5)
	addpd	r2, r6			;; R5 = R1 + R5 (final R1)
	ENDM

;; This macro is similar to r4_x4cl_eight_reals_unfft.
;; The difference is this macro has different source and destination.

;; Used in the last levels of an r4 FFT pass 1 (swizzling):
r4_sg2cl_eight_reals_djbunfft MACRO srcreg,srcinc,d1,dstreg,dstinc,e1,screg1,screg2
	r4_s8r_unfft4 [srcreg],[srcreg+32],[srcreg+16],[srcreg+48],[srcreg+d1],[srcreg+d1+32],[srcreg+d1+16],[srcreg+d1+48],screg2,screg1,screg2+32,dstreg+dstinc,e1,[dstreg+e1+16],[dstreg+e1+48]
	bump	srcreg, srcinc
;;	xstore	[dstreg+e1+48], xmm2	;; Save R8
;;	xstore	[dstreg+e1+16], xmm5	;; Save R4
	xstore	[dstreg+16], xmm7	;; Save R3
	xstore	[dstreg+48], xmm6	;; Save R7
	xstore	[dstreg], xmm1		;; Save R1
	xstore	[dstreg+e1], xmm4	;; Save R2
	xstore	[dstreg+32], xmm0	;; Save R5
	xstore	[dstreg+e1+32], xmm3	;; Save R6
	bump	dstreg, dstinc
	ENDM

;; The macros below take an XMM_SCD4 pointer.  We know that the first
;; XMM_SCD4 entry is 1, thus we skip over it to get to the w^1 entry we desire.

;; This macro is used in the last levels of an r4delay FFT pass 1 (no swizzling):
IFDEF UNUSED
r4_g2cl_eight_reals_unfft4 MACRO srcreg,srcinc,d1,dstreg,dstinc,e1,screg1,screg2
	r4_x8r_unfft_mem [srcreg],[srcreg+16],[srcreg+32],[srcreg+48],[srcreg+d1],[srcreg+d1+16],[srcreg+d1+32],[srcreg+d1+48],screg2,screg1+32,screg2+32,dstreg+dstinc,e1,[dstreg+e1+16],[dstreg+e1+48]
	bump	srcreg, srcinc
	xstore	[dstreg+16], xmm7	;; Save R3
;;	xstore	[dstreg+e1+16], xmm2	;; Save R4
	xstore	[dstreg+48], xmm6	;; Save R7
;;	xstore	[dstreg+e1+48], xmm5	;; Save R8
	xstore	[dstreg], xmm1		;; Save R1
	xstore	[dstreg+e1], xmm4	;; Save R2
	xstore	[dstreg+32], xmm0	;; Save R5
	xstore	[dstreg+e1+32], xmm3	;; Save R6
	bump	dstreg, dstinc
	ENDM
ENDIF

;; Used in the last levels of an r4delay FFT pass 1 (swizzling):
r4_sg2cl_eight_reals_unfft4 MACRO srcreg,srcinc,d1,dstreg,dstinc,e1,screg1,screg2
	r4_s8r_unfft4 [srcreg],[srcreg+32],[srcreg+16],[srcreg+48],[srcreg+d1],[srcreg+d1+32],[srcreg+d1+16],[srcreg+d1+48],screg2,screg1+32,screg2+32,dstreg+dstinc,e1,[dstreg+e1+16],[dstreg+e1+48]
	bump	srcreg, srcinc
;;	xstore	[dstreg+e1+48], xmm2	;; Save R8
;;	xstore	[dstreg+e1+16], xmm5	;; Save R4
	xstore	[dstreg+16], xmm7	;; Save R3
	xstore	[dstreg+48], xmm6	;; Save R7
	xstore	[dstreg], xmm1		;; Save R1
	xstore	[dstreg+e1], xmm4	;; Save R2
	xstore	[dstreg+32], xmm0	;; Save R5
	xstore	[dstreg+e1+32], xmm3	;; Save R6
	bump	dstreg, dstinc
	ENDM

r4_s8r_unfft4 MACRO memr1_1,memr1_2,memr2,memi2,memr3,memi3,memr4,memi4,screg1,screg2,screg5,pre1,pre2,dst4,dst8
	shuffle_load_with_temp xmm4, xmm1, memr3, memr4, xmm6 ;; R3,R4
	xload	xmm6, [screg5+16]	;; cosine/sine for w^5n
	xcopy	xmm3, xmm1		;; Copy R4
	mulpd	xmm1, xmm6		;; A4 = R4 * cosine/sine
	xload	xmm2, [screg1+16]	;; cosine/sine for w^n
	xcopy	xmm0, xmm4		;; Copy R3
	mulpd	xmm4, xmm2		;; A3 = R3 * cosine/sine for w^n
	shuffle_load xmm5, xmm7, memi3, memi4 ;; I3,I4
	addpd	xmm1, xmm7		;; A4 = A4 + I4
	mulpd	xmm7, xmm6		;; B4 = I4 * cosine/sine
	addpd	xmm4, xmm5		;; A3 = A3 + I3
	mulpd	xmm5, xmm2		;; B3 = I3 * cosine/sine
	subpd	xmm7, xmm3		;; B4 = B4 - R4
	subpd	xmm5, xmm0		;; B3 = B3 - R3

	shuffle_load_with_temp xmm2, xmm0, memr1_1, memr2, xmm6 ;; Real value #1,R2
	xcopy	xmm6, xmm0		;; Copy R2
	mulpd	xmm0, [screg2+16]	;; A2 = R2 * cosine/sine for w^2n
	xstore	dst8, xmm2		;; Temporarily save real value #1
	shuffle_load xmm2, xmm3, memr1_2, memi2 ;; Real value #2,I2
	addpd	xmm0, xmm3		;; A2 = A2 + I2
	mulpd	xmm3, [screg2+16]	;; B2 = I2 * cosine/sine
	subpd	xmm3, xmm6		;; B2 = B2 - R2

	xprefetchw [pre1]

	xload	xmm6, [screg5]
	mulpd	xmm1, xmm6		;; new R7 = A4 * sine (final R4)
	mulpd	xmm7, xmm6		;; new R8 = B4 * sine (final I4)
	xload	xmm6, [screg1]
	mulpd	xmm4, xmm6		;; new R5 = A3 * sine (final R3)
	mulpd	xmm5, xmm6		;; new R6 = B3 * sine (final I3)
	xload	xmm6, [screg2]
	mulpd	xmm0, xmm6		;; new R3 = A2 * sine (final R2)
	mulpd	xmm3, xmm6		;; new R4 = B2 * sine (final I2)

	xprefetchw [pre1][pre2]

	xcopy	xmm6, xmm4		;; Copy R5
	subpd	xmm4, xmm1		;; new R6 = R5 - R7
	addpd	xmm1, xmm6		;; new R5 = R5 + R7

	xcopy	xmm6, xmm5		;; Copy R6
	subpd	xmm5, xmm7		;; new R8 = R6 - R8
	addpd	xmm7, xmm6		;; new R7 = R6 + R8

	xcopy	xmm6, xmm4		;; Copy R6
	addpd	xmm4, xmm5		;; R6 = R6 + R8
	subpd	xmm5, xmm6		;; R8 = R8 - R6

	mulpd	xmm4, XMM_SQRTHALF	;; R6 = R6 * square root of 1/2
	mulpd	xmm5, XMM_SQRTHALF	;; R8 = R8 * square root of 1/2

	xcopy	xmm6, xmm2		;; Copy R2
	subpd	xmm2, xmm3		;; R2 = R2 - R4 (new R4)
	addpd	xmm3, xmm6		;; R4 = R2 + R4 (new R2)

	xcopy	xmm6, xmm2		;; Copy R4
	subpd	xmm2, xmm5		;; R4 = R4 - R8 (final R8)
	addpd	xmm5, xmm6		;; R8 = R4 + R8 (final R4)

	xcopy	xmm6, xmm3		;; Copy R2
	subpd	xmm3, xmm4		;; R2 = R2 - R6 (final R6)
	addpd	xmm4, xmm6		;; R6 = R2 + R6 (final R2)

	xload	xmm6, dst8		;; R1
	subpd	xmm6, xmm0		;; R1 = R1 - R3 (new R3)
	addpd	xmm0, dst8		;; R3 = R1 + R3 (new R1)

	xstore	dst8, xmm2		;; Save R8
	xstore	dst4, xmm5		;; Save R4

	xcopy	xmm2, xmm6		;; Copy R3
	subpd	xmm6, xmm7		;; R3 = R3 - R7 (final R7)
	addpd	xmm7, xmm2		;; R7 = R3 + R7 (final R3)

	xcopy	xmm5, xmm0		;; Copy R1
	subpd	xmm0, xmm1		;; R1 = R1 - R5 (final R5)
	addpd	xmm1, xmm5		;; R5 = R1 + R5 (final R1)
	ENDM

;; 32-bit AMD K8 optimized versions of the above macros

IF (@INSTR(,%xarch,<K8>) NE 0)

r4_x8r_unfft_mem MACRO mem1,mem2,mem3,mem4,mem5,mem6,mem7,mem8,screg1,screg2,screg5,pre1,pre2,dst4,dst8
	xload	xmm6, [screg5+16]	;; cosine/sine for w^5n
	xload	xmm1, mem7		;; R4
	mulpd	xmm1, xmm6		;; A4 = R4 * cosine/sine
	xload	xmm7, mem8		;; I4
	addpd	xmm1, xmm7		;; A4 = A4 + I4
	mulpd	xmm7, xmm6		;; B4 = I4 * cosine/sine

	xload	xmm2, [screg1+16]	;; cosine/sine for w^n
	xload	xmm4, mem5		;; R3
	mulpd	xmm4, xmm2		;; A3 = R3 * cosine/sine for w^n
	xload	xmm5, mem6		;; I3
	addpd	xmm4, xmm5		;; A3 = A3 + I3
	mulpd	xmm5, xmm2		;; B3 = I3 * cosine/sine

	xload	xmm6, [screg2+16]	;; cosine/sine for w^2n
	xload	xmm0, mem3		;; R2
	mulpd	xmm0, xmm6		;; A2 = R2 * cosine/sine
	xload	xmm3, mem4		;; I2
	addpd	xmm0, xmm3		;; A2 = A2 + I2
	mulpd	xmm3, xmm6		;; B2 = I2 * cosine/sine

	subpd	xmm7, mem7		;; B4 = B4 - R4
	subpd	xmm5, mem5		;; B3 = B3 - R3
	subpd	xmm3, mem3		;; B2 = B2 - R2

	xprefetchw [pre1]

	xload	xmm6, [screg5]
	mulpd	xmm1, xmm6		;; new R7 = A4 * sine (final R4)
	mulpd	xmm7, xmm6		;; new R8 = B4 * sine (final I4)
	xload	xmm2, [screg1]
	mulpd	xmm4, xmm2		;; new R5 = A3 * sine (final R3)
	mulpd	xmm5, xmm2		;; new R6 = B3 * sine (final I3)
	xload	xmm6, [screg2]
	mulpd	xmm0, xmm6		;; new R3 = A2 * sine (final R2)
	mulpd	xmm3, xmm6		;; new R4 = B2 * sine (final I2)

	xprefetchw [pre1][pre2]

	subpd	xmm4, xmm1		;; new R6 = R5 - R7
	multwo	xmm1
	subpd	xmm5, xmm7		;; new R8 = R6 - R8
	multwo	xmm7
	addpd	xmm1, xmm4		;; new R5 = R5 + R7
	addpd	xmm7, xmm5		;; new R7 = R6 + R8

	addpd	xmm4, xmm5		;; R6 = R6 + R8
	multwo	xmm5
	subpd	xmm5, xmm4		;; R8 = R8 - R6

	mulpd	xmm4, XMM_SQRTHALF	;; R6 = R6 * square root of 1/2
	mulpd	xmm5, XMM_SQRTHALF	;; R8 = R8 * square root of 1/2

	xload	xmm2, mem2		;; R2
	subpd	xmm2, xmm3		;; R2 = R2 - R4 (new R4)
	addpd	xmm3, mem2		;; R4 = R2 + R4 (new R2)

	subpd	xmm2, xmm5		;; R4 = R4 - R8 (final R8)
	multwo	xmm5
	subpd	xmm3, xmm4		;; R2 = R2 - R6 (final R6)
	multwo	xmm4
	addpd	xmm5, xmm2		;; R8 = R4 + R8 (final R4)
	addpd	xmm4, xmm3		;; R6 = R2 + R6 (final R2)

	xload	xmm6, mem1		;; R1
	subpd	xmm6, xmm0		;; R1 = R1 - R3 (new R3)
	addpd	xmm0, mem1		;; R3 = R1 + R3 (new R1)

	xstore	dst8, xmm2		;; Save R8
	xstore	dst4, xmm5		;; Save R4

	subpd	xmm6, xmm7		;; R3 = R3 - R7 (final R7)
	multwo	xmm7
	subpd	xmm0, xmm1		;; R1 = R1 - R5 (final R5)
	multwo	xmm1
	addpd	xmm7, xmm6		;; R7 = R3 + R7 (final R3)
	addpd	xmm1, xmm0		;; R5 = R1 + R5 (final R1)
	ENDM

r4_x8r_unfft_partial_mem MACRO r1,r2,r3,r4,r5,r6,r7,r8,mem1,mem2,mem5,mem6,screg1,screg2,screg5,pre1,pre2,dst4,dst8
	xload	r2, [screg5+16]		;; cosine/sine for w^5n
	mulpd	r2, r7			;; A4 = R4 * cosine/sine
	addpd	r2, r8			;; A4 = A4 + I4
	mulpd	r8, [screg5+16]		;; B4 = I4 * cosine/sine
	subpd	r8, r7			;; B4 = B4 - R4

	xload	r1, [screg1+16]		;; cosine/sine for w^n
	xload	r5, mem5		;; R3
	mulpd	r5, r1			;; A3 = R3 * cosine/sine for w^n
	xload	r6, mem6		;; I3
	addpd	r5, r6			;; A3 = A3 + I3
	mulpd	r6, r1			;; B3 = I3 * cosine/sine
	subpd	r6, mem5		;; B3 = B3 - R3

	xload	r1, [screg2+16]		;; cosine/sine for w^2n
	mulpd	r1, r3			;; A2 = R2 * cosine/sine
	addpd	r1, r4			;; A2 = A2 + I2
	mulpd	r4, [screg2+16]		;; B2 = I2 * cosine/sine
	subpd	r4, r3			;; B2 = B2 - R2

	xprefetchw [pre1]

	xload	r7, [screg5]
	mulpd	r2, r7			;; new R7 = A4 * sine (final R4)
	mulpd	r8, r7			;; new R8 = B4 * sine (final I4)
	xload	r3, [screg1]
	mulpd	r5, r3			;; new R5 = A3 * sine (final R3)
	mulpd	r6, r3			;; new R6 = B3 * sine (final I3)
	xload	r7, [screg2]
	mulpd	r1, r7			;; new R3 = A2 * sine (final R2)
	mulpd	r4, r7			;; new R4 = B2 * sine (final I2)

	xprefetchw [pre1][pre2]

	subpd	r5, r2			;; new R6 = R5 - R7
	multwo	r2
	subpd	r6, r8			;; new R8 = R6 - R8
	multwo	r8
	addpd	r2, r5			;; new R5 = R5 + R7
	addpd	r8, r6			;; new R7 = R6 + R8

	addpd	r5, r6			;; R6 = R6 + R8
	multwo	r6
	subpd	r6, r5			;; R8 = R8 - R6

	mulpd	r5, XMM_SQRTHALF	;; R6 = R6 * square root of 1/2
	mulpd	r6, XMM_SQRTHALF	;; R8 = R8 * square root of 1/2

	xload	r3, mem2		;; R2
	subpd	r3, r4			;; R2 = R2 - R4 (new R4)
	addpd	r4, mem2		;; R4 = R2 + R4 (new R2)

	subpd	r3, r6			;; R4 = R4 - R8 (final R8)
	multwo	r6
	subpd	r4, r5			;; R2 = R2 - R6 (final R6)
	multwo	r5
	addpd	r6, r3			;; R8 = R4 + R8 (final R4)
	addpd	r5, r4			;; R6 = R2 + R6 (final R2)

	xload	r7, mem1		;; R1
	subpd	r7, r1			;; R1 = R1 - R3 (new R3)
	addpd	r1, mem1		;; R3 = R1 + R3 (new R1)

	xstore	dst8, r3		;; Save R8
	xstore	dst4, r6		;; Save R4

	subpd	r7, r8			;; R3 = R3 - R7 (final R7)
	multwo	r8
	subpd	r1, r2			;; R1 = R1 - R5 (final R5)
	multwo	r2
	addpd	r8, r7			;; R7 = R3 + R7 (final R3)
	addpd	r2, r1			;; R5 = R1 + R5 (final R1)
	ENDM

ENDIF

;; 64-bit Intel and AMD K10 optimized versions of the above macros

IF (@INSTR(,%xarch,<CORE>) NE 0) OR (@INSTR(,%xarch,<P4>) NE 0) OR (@INSTR(,%xarch,<K10>) NE 0) OR (@INSTR(,%xarch,<BLEND>) NE 0)
IFDEF X86_64

r4_x4cl_eight_reals_unfft_cmn_preload MACRO
	xload	xmm15, XMM_SQRTHALF
	ENDM

; Theoretical best Core 2 timing is 48 clocks.  The empty add slot at clock 14 means
; this macro should run in 49 clocks.  Actual timing is 58 clocks.

r4_x4cl_eight_reals_unfft_cmn MACRO srcreg,srcinc,d1,d2,screg1,scoff1,screg2,scoff2,screg5,scoff5
	xload	xmm0, [srcreg+d2+d1+16]	;; R4
	xload	xmm1, [screg5+scoff5+16];; cosine/sine for w^5n
	xcopy	xmm2, xmm0		;; Copy R4
	mulpd	xmm0, xmm1		;; A4 = R4 * cosine/sine		; 1-5

	xload	xmm3, [srcreg+d2+16]	;; R3
	xload	xmm4, [screg1+scoff1+16];; cosine/sine for w^n
	xcopy	xmm5, xmm3		;; Copy R3
	mulpd	xmm3, xmm4		;; A3 = R3 * cosine/sine for w^n	; 2-6

	xload	xmm6, [srcreg+d2+d1+48]	;; I4
	mulpd	xmm1, xmm6		;; B4 = I4 * cosine/sine		; 3-7

	xload	xmm7, [srcreg+d2+48]	;; I3
	mulpd	xmm4, xmm7		;; B3 = I3 * cosine/sine		; 4-8

	xload	xmm8, [srcreg+d2+d1]	;;#2 R4
	xload	xmm9, [screg5+16]	;;#2 cosine/sine for w^5n
	xcopy	xmm10, xmm8		;;#2 Copy R4
	mulpd	xmm8, xmm9		;;#2 A4 = R4 * cosine/sine		; 5-9

	addpd	xmm0, xmm6		;; A4 = A4 + I4				; 6-8	avail 6,11+
	xload	xmm11, [srcreg+d2]	;;#2 R3
	xload	xmm12, [screg1+16]	;;#2 cosine/sine for w^n
	xcopy	xmm13, xmm11		;;#2 Copy R3
	mulpd	xmm11, xmm12		;;#2 A3 = R3 * cosine/sine for w^n	; 6-10	avail 6,14

	addpd	xmm3, xmm7		;; A3 = A3 + I3				; 7-9	avail 6,7,14
	xload	xmm14, [srcreg+d2+d1+32];;#2 I4
	mulpd	xmm9, xmm14		;;#2 B4 = I4 * cosine/sine		; 7-11	avail 6,7

	subpd	xmm1, xmm2		;; B4 = B4 - R4				; 8-10	avail 6,7,2
	xload	xmm6, [srcreg+d2+32]	;;#2 I3
	mulpd	xmm12, xmm6		;;#2 B3 = I3 * cosine/sine		; 8-12	avail 7,2

	subpd	xmm4, xmm5		;; B3 = B3 - R3				; 9-11	avail 7,2,5
	xload	xmm7, [screg5+scoff5]	;; sine
	mulpd	xmm0, xmm7		;; new R7 = A4 * sine (final R4)	; 9-13	avail 2,5

	addpd	xmm8, xmm14		;;#2 A4 = A4 + I4			; 10-12	avail 2,5,14
	xload	xmm2, [screg1+scoff1]	;; sine
	mulpd	xmm3, xmm2		;; new R5 = A3 * sine (final R3)	; 10-14	avail 5,14

	addpd	xmm11, xmm6		;;#2 A3 = A3 + I3			; 11-13	avail 5,14,6
	mulpd	xmm1, xmm7		;; new R8 = B4 * sine (final I4)	; 11-15	avail 5,14,6,7

	subpd	xmm9, xmm10		;;#2 B4 = B4 - R4			; 12-14	avail 5,14,6,7,10
	mulpd	xmm4, xmm2		;; new R6 = B3 * sine (final I3)	; 12-16	avail 5,14,6,7,10,2

	subpd	xmm12, xmm13		;;#2 B3 = B3 - R3			; 13-15	avail 5,14,6,7,10,2,13
	xload	xmm5, [srcreg+d1+16]	;; R2
	xload	xmm14, [screg2+scoff2+16];; cosine/sine for w^2n
	xcopy	xmm6, xmm5		;; Copy R2
	mulpd	xmm5, xmm14		;; A2 = R2 * cosine/sine		; 13-17	avail 7,10,2,13

	xload	xmm7, [srcreg+d1+48]	;; I2
	mulpd	xmm14, xmm7		;; B2 = I2 * cosine/sine		; 14-18	avail 10,2,13

	xprefetchw [srcreg+srcinc]

	xcopy	xmm10, xmm3		;; Copy R5
	subpd	xmm3, xmm0		;; new R6 = R5 - R7			; 15-17	avail 2,13
	xload	xmm2, [screg5]		;;#2 sine
	mulpd	xmm8, xmm2		;;#2 new R7 = A4 * sine (final R4)	; 15-19	avail 13

	addpd	xmm10, xmm0		;; new R5 = R5 + R7			; 16-18	avail 13,0
	xload	xmm13, [screg1]		;;#2 sine
	mulpd	xmm11, xmm13		;;#2 new R5 = A3 * sine (final R3)	; 16-20	avail 0

	xcopy	xmm0, xmm4		;; Copy R6
	subpd	xmm4, xmm1		;; new R8 = R6 - R8			; 17-19
	mulpd	xmm9, xmm2		;;#2 new R8 = B4 * sine (final I4)	; 17-21 avail 2

	addpd	xmm5, xmm7		;; A2 = A2 + I2				; 18-20	avail 2,7
	mulpd	xmm12, xmm13		;;#2 new R6 = B3 * sine (final I3)	; 18-22	avail 2,7,13

	addpd	xmm0, xmm1		;; new R7 = R6 + R8			; 19-21	avail 2,7,13,1
	mulpd	xmm3, xmm15		;; R6 = R6 * square root of 1/2		; 19-23

	subpd	xmm14, xmm6		;; B2 = B2 - R2				; 20-22	avail 2,7,13,1,6
	mulpd	xmm4, xmm15		;; R8 = R8 * square root of 1/2		; 20-24

	xprefetchw [srcreg+srcinc+d1]

	xcopy	xmm2, xmm11		;;#2 Copy R5
	subpd	xmm11, xmm8		;;#2 new R6 = R5 - R7			; 21-23	avail 7,13,1,6

	addpd	xmm2, xmm8		;;#2 new R5 = R5 + R7			; 22-24	avail 7,13,1,6,8
	xload	xmm7, [screg2+scoff2]	;; sine
	mulpd	xmm5, xmm7		;; new R3 = A2 * sine (final R2)	; 22-26	avail 13,1,6,8

	xcopy	xmm13, xmm12		;;#2 Copy R6
	subpd	xmm12, xmm9		;;#2 new R8 = R6 - R8			; 23-25	avail 1,6,8
	mulpd	xmm14, xmm7		;; new R4 = B2 * sine (final I2)	; 23-27	avail 1,6,8,7

	addpd	xmm13, xmm9		;;#2 new R7 = R6 + R8			; 24-26	avail 1,6,8,7,9
	xload	xmm6, [srcreg+16]	;; R1

	xcopy	xmm1, xmm3		;; Copy R6
	addpd	xmm3, xmm4		;; R6 = R6 + R8				; 25-27	avail 8,7,9
	xload	xmm7, [srcreg+48]	;; R2

	subpd	xmm4, xmm1		;; R8 = R8 - R6				; 26-28	avail 8,9,1

	xprefetchw [srcreg+srcinc+d2]	;;#2

	xcopy	xmm8, xmm6		;; Copy R1
	subpd	xmm6, xmm5		;; R1 = R1 - R3 (new R3)		; 27-29	avail 9,1
	addpd	xmm8, xmm5		;; R3 = R1 + R3 (new R1)		; 28-30	avail 9,1,5

	xcopy	xmm9, xmm7		;; Copy R2
	subpd	xmm7, xmm14		;; R2 = R2 - R4 (new R4)		; 29-31	avail 1,5

	addpd	xmm14, xmm9		;; R4 = R2 + R4 (new R2)		; 30-32	avail 1,5,9
	xload	xmm1, [srcreg+d1]	;;#2 R2
	xload	xmm5, [screg2+16]	;;#2 cosine/sine for w^2n
	mulpd	xmm1, xmm5		;;#2 A2 = R2 * cosine/sine		; 30-34	avail 9

	xcopy	xmm9, xmm6		;; Copy R3
	subpd	xmm6, xmm0		;; R3 = R3 - R7 (final R7)		; 31-33	avail none storable 6
	xstore	[srcreg+d1+48], xmm6	;; Save R7				; 34	avail 6
	xload	xmm6, [srcreg+d1+32]	;;#2 I2
	mulpd	xmm5, xmm6		;;#2 B2 = I2 * cosine/sine		; 31-35	avail none

	addpd	xmm0, xmm9		;; R7 = R3 + R7 (final R3)		; 32-34	avail 9 storable 0

	xcopy	xmm9, xmm8		;; Copy R1
	subpd	xmm8, xmm10		;; R1 = R1 - R5 (final R5)		; 33-35	avail none storable 0,8
	addpd	xmm10, xmm9		;; R5 = R1 + R5 (final R1)		; 34-36	avail 9 storable 0,8,10

	addpd	xmm1, xmm6		;;#2 A2 = A2 + I2			; 35-37	avail 9,6 storable 0,8,10
	mulpd	xmm11, xmm15		;;#2 R6 = R6 * square root of 1/2	; 35-39
	xstore	[srcreg+d1+16], xmm0	;; Save R3				; 35	avail 9,6,0 storable 8,10

	subpd	xmm5, [srcreg+d1]	;;#2 B2 = B2 - R2			; 36-38
	mulpd	xmm12, xmm15		;;#2 R8 = R8 * square root of 1/2	; 36-40
	xstore	[srcreg+d1+32], xmm8	;; Save R5				; 36	avail 9,6,0,8 storable 10

	xcopy	xmm9, xmm7		;; Copy R4
	subpd	xmm7, xmm4		;; R4 = R4 - R8 (final R8)		; 37-39	avail 6,0,8 storable 10,7
	xstore	[srcreg+d1], xmm10	;; Save R1				; 37	avail 6,0,8,10 storable 7

	xprefetchw [srcreg+srcinc+d2+d1];;#2

	addpd	xmm4, xmm9		;; R8 = R4 + R8 (final R4)		; 38-40	avail 6,0,8,10,9 storable 7,4
	xload	xmm6, [screg2]		;;#2 sine
	mulpd	xmm1, xmm6		;;#2 new R3 = A2 * sine (final R2)	; 38-42	avail 0,8,10,9 storable 7,4

	xcopy	xmm0, xmm14		;; Copy R2
	subpd	xmm14, xmm3		;; R2 = R2 - R6 (final R6)		; 39-41	avail 8,10,9 storable 7,4,14
	mulpd	xmm5, xmm6		;;#2 new R4 = B2 * sine (final I2)	; 39-43	avail 8,10,9,6 storable 7,4,14

	addpd	xmm3, xmm0		;; R6 = R2 + R6 (final R2)		; 40-42	avail 8,10,9,6,0 storable 7,4,14,3
	xload	xmm10, [srcreg]		;;#2 R1
	xstore	[srcreg+d2+d1+48], xmm7	;; Save R8				; 40	avail 8,9,6,0,7 storable 4,14,3

	xcopy	xmm8, xmm11		;;#2 Copy R6
	addpd	xmm11, xmm12		;;#2 R6 = R6 + R8			; 41-43	avail 9,6,0,7 storable 4,14,3
	xstore	[srcreg+d2+d1+16], xmm4	;; Save R4				; 41	avail 9,6,0,7,4 storable 14,3

	subpd	xmm12, xmm8		;;#2 R8 = R8 - R6			; 42-44	avail 9,6,0,7,4,8 storable 14,3
	xload	xmm6, [srcreg+32]	;;#2 R2
	xstore	[srcreg+d2+d1+32], xmm14;; Save R6				; 42	avail 9,0,7,4,8,14 storable 3

	xcopy	xmm9, xmm10		;;#2 Copy R1
	subpd	xmm10, xmm1		;;#2 R1 = R1 - R3 (new R3)		; 43-45	avail 0,7,4,8,14 storable 3
	xstore	[srcreg+d2+d1], xmm3	;; Save R2				; 43	avail 0,7,4,8,14,3

	addpd	xmm1, xmm9		;;#2 R3 = R1 + R3 (new R1)		; 44-46	avail 0,7,4,8,14,3,9

	xcopy	xmm0, xmm6		;;#2 Copy R2
	subpd	xmm6, xmm5		;;#2 R2 = R2 - R4 (new R4)		; 45-47	avail 7,4,8,14,3,9

	addpd	xmm5, xmm0		;;#2 R4 = R2 + R4 (new R2)		; 46-48	avail 7,4,8,14,3,9,0

	xcopy	xmm7, xmm10		;;#2 Copy R3
	subpd	xmm10, xmm13		;;#2 R3 = R3 - R7 (final R7)		; 47-49	avail 4,8,14,3,9,0 storable 10
	addpd	xmm13, xmm7		;;#2 R7 = R3 + R7 (final R3)		; 48-50	avail 4,8,14,3,9,0,7 storable 10,13

	xcopy	xmm4, xmm1		;;#2 Copy R1
	subpd	xmm1, xmm2		;;#2 R1 = R1 - R5 (final R5)		; 49-51	avail 8,14,3,9,0,7 storable 10,13,1

	addpd	xmm2, xmm4		;;#2 R5 = R1 + R5 (final R1)		; 50-52	avail 8,14,3,9,0,7,4 storable 10,13,1,2
	xstore	[srcreg+48], xmm10	;;#2 Save R7				; 50	avail 8,14,3,9,0,7,4,10 storable 13,1,2

	xcopy	xmm8, xmm6		;;#2 Copy R4
	subpd	xmm6, xmm12		;;#2 R4 = R4 - R8 (final R8)		; 51-53	avail 14,3,9,0,7,4,10 storable 13,1,2,6
	xstore	[srcreg+16], xmm13	;;#2 Save R3				; 51	avail 14,3,9,0,7,4,10,13 storable 1,2,6

	addpd	xmm12, xmm8		;;#2 R8 = R4 + R8 (final R4)		; 52-54	avail 14,3,9,0,7,4,10,13,8 storable 1,2,6,12
	xstore	[srcreg+32], xmm1	;;#2 Save R5				; 52	avail 14,3,9,0,7,4,10,13,8,1 storable 2,6,12

	xcopy	xmm8, xmm5		;;#2 Copy R2
	subpd	xmm5, xmm11		;;#2 R2 = R2 - R6 (final R6)		; 53-55	avail 14,3,9,0,7,4,10,13,1 storable 2,6,12,5
	xstore	[srcreg], xmm2		;;#2 Save R1				; 53	avail 14,3,9,0,7,4,10,13,1,2 storable 6,12,5

	addpd	xmm11, xmm8		;;#2 R6 = R2 + R6 (final R2)		; 54-56	avail 14,3,9,0,7,4,10,13,1,2,8 storable 6,12,5,11
	xstore	[srcreg+d2+48], xmm6	;;#2 Save R8				; 54
	xstore	[srcreg+d2+16], xmm12	;;#2 Save R4				; 55
	xstore	[srcreg+d2+32], xmm5	;;#2 Save R6				; 56
	xstore	[srcreg+d2], xmm11	;;#2 Save R2				; 57

	bump	srcreg, srcinc
	ENDM

ENDIF
ENDIF

;; 64-bit AMD K8 optimized versions of the above macros.  Same as 32-bit version with
;; some constants preloaded. NOT REALLY ANY BETTER!!  We should try rewritting with
;; combining r4_x8r_unfft_mem and r4_x8r_unfft_partial_mem into one macro.  Should
;; yield significant savings.

IF (@INSTR(,%xarch,<K8>) NE 0)
IFDEF X86_64

r4_x4cl_eight_reals_unfft_cmn_preload MACRO
	xload	xmm14, XMM_SQRTHALF
	xload	xmm15, XMM_TWO
	ENDM

r4_x8r_unfft_mem MACRO mem1,mem2,mem3,mem4,mem5,mem6,mem7,mem8,screg1,screg2,screg5,pre1,pre2,dst4,dst8
	xload	xmm6, [screg5+16]	;; cosine/sine for w^5n
	xload	xmm1, mem7		;; R4
	mulpd	xmm1, xmm6		;; A4 = R4 * cosine/sine
	xload	xmm7, mem8		;; I4
	addpd	xmm1, xmm7		;; A4 = A4 + I4
	mulpd	xmm7, xmm6		;; B4 = I4 * cosine/sine

	xload	xmm2, [screg1+16]	;; cosine/sine for w^n
	xload	xmm4, mem5		;; R3
	mulpd	xmm4, xmm2		;; A3 = R3 * cosine/sine for w^n
	xload	xmm5, mem6		;; I3
	addpd	xmm4, xmm5		;; A3 = A3 + I3
	mulpd	xmm5, xmm2		;; B3 = I3 * cosine/sine

	xload	xmm6, [screg2+16]	;; cosine/sine for w^2n
	xload	xmm0, mem3		;; R2
	mulpd	xmm0, xmm6		;; A2 = R2 * cosine/sine
	xload	xmm3, mem4		;; I2
	addpd	xmm0, xmm3		;; A2 = A2 + I2
	mulpd	xmm3, xmm6		;; B2 = I2 * cosine/sine

	subpd	xmm7, mem7		;; B4 = B4 - R4
	subpd	xmm5, mem5		;; B3 = B3 - R3
	subpd	xmm3, mem3		;; B2 = B2 - R2

	xprefetchw [pre1]

	xload	xmm6, [screg5]
	mulpd	xmm1, xmm6		;; new R7 = A4 * sine (final R4)
	mulpd	xmm7, xmm6		;; new R8 = B4 * sine (final I4)
	xload	xmm2, [screg1]
	mulpd	xmm4, xmm2		;; new R5 = A3 * sine (final R3)
	mulpd	xmm5, xmm2		;; new R6 = B3 * sine (final I3)
	xload	xmm6, [screg2]
	mulpd	xmm0, xmm6		;; new R3 = A2 * sine (final R2)
	mulpd	xmm3, xmm6		;; new R4 = B2 * sine (final I2)

	xprefetchw [pre1][pre2]

	subpd	xmm4, xmm1		;; new R6 = R5 - R7
	mulpd	xmm1, xmm15		;; R7 = R7 * 2
	subpd	xmm5, xmm7		;; new R8 = R6 - R8
	mulpd	xmm7, xmm15		;; R8 = R8 * 2
	addpd	xmm1, xmm4		;; new R5 = R5 + R7
	addpd	xmm7, xmm5		;; new R7 = R6 + R8

	addpd	xmm4, xmm5		;; R6 = R6 + R8
	mulpd	xmm5, xmm15		;; R8 = R8 * 2
	subpd	xmm5, xmm4		;; R8 = R8 - R6

	mulpd	xmm4, xmm14		;; R6 = R6 * square root of 1/2
	mulpd	xmm5, xmm14		;; R8 = R8 * square root of 1/2

	xload	xmm2, mem2		;; R2
	subpd	xmm2, xmm3		;; R2 = R2 - R4 (new R4)
	addpd	xmm3, mem2		;; R4 = R2 + R4 (new R2)

	subpd	xmm2, xmm5		;; R4 = R4 - R8 (final R8)
	mulpd	xmm5, xmm15		;; R8 = R8 * 2
	subpd	xmm3, xmm4		;; R2 = R2 - R6 (final R6)
	mulpd	xmm4, xmm15		;; R6 = R6 * 2
	addpd	xmm5, xmm2		;; R8 = R4 + R8 (final R4)
	addpd	xmm4, xmm3		;; R6 = R2 + R6 (final R2)

	xload	xmm6, mem1		;; R1
	subpd	xmm6, xmm0		;; R1 = R1 - R3 (new R3)
	addpd	xmm0, mem1		;; R3 = R1 + R3 (new R1)

	xstore	dst8, xmm2		;; Save R8
	xstore	dst4, xmm5		;; Save R4

	subpd	xmm6, xmm7		;; R3 = R3 - R7 (final R7)
	mulpd	xmm7, xmm15		;; R7 = R7 * 2
	subpd	xmm0, xmm1		;; R1 = R1 - R5 (final R5)
	mulpd	xmm1, xmm15		;; R5 = R5 * 2
	addpd	xmm7, xmm6		;; R7 = R3 + R7 (final R3)
	addpd	xmm1, xmm0		;; R5 = R1 + R5 (final R1)
	ENDM

r4_x8r_unfft_partial_mem MACRO r1,r2,r3,r4,r5,r6,r7,r8,mem1,mem2,mem5,mem6,screg1,screg2,screg5,pre1,pre2,dst4,dst8
	xload	r2, [screg5+16]		;; cosine/sine for w^5n
	mulpd	r2, r7			;; A4 = R4 * cosine/sine
	addpd	r2, r8			;; A4 = A4 + I4
	mulpd	r8, [screg5+16]		;; B4 = I4 * cosine/sine
	subpd	r8, r7			;; B4 = B4 - R4

	xload	r1, [screg1+16]		;; cosine/sine for w^n
	xload	r5, mem5		;; R3
	mulpd	r5, r1			;; A3 = R3 * cosine/sine for w^n
	xload	r6, mem6		;; I3
	addpd	r5, r6			;; A3 = A3 + I3
	mulpd	r6, r1			;; B3 = I3 * cosine/sine
	subpd	r6, mem5		;; B3 = B3 - R3

	xload	r1, [screg2+16]		;; cosine/sine for w^2n
	mulpd	r1, r3			;; A2 = R2 * cosine/sine
	addpd	r1, r4			;; A2 = A2 + I2
	mulpd	r4, [screg2+16]		;; B2 = I2 * cosine/sine
	subpd	r4, r3			;; B2 = B2 - R2

	xprefetchw [pre1]

	xload	r7, [screg5]
	mulpd	r2, r7			;; new R7 = A4 * sine (final R4)
	mulpd	r8, r7			;; new R8 = B4 * sine (final I4)
	xload	r3, [screg1]
	mulpd	r5, r3			;; new R5 = A3 * sine (final R3)
	mulpd	r6, r3			;; new R6 = B3 * sine (final I3)
	xload	r7, [screg2]
	mulpd	r1, r7			;; new R3 = A2 * sine (final R2)
	mulpd	r4, r7			;; new R4 = B2 * sine (final I2)

	xprefetchw [pre1][pre2]

	subpd	r5, r2			;; new R6 = R5 - R7
	mulpd	r2, xmm15		;; R7 = R7 * 2
	subpd	r6, r8			;; new R8 = R6 - R8
	mulpd	r8, xmm15		;; R8 = R8 * 2
	addpd	r2, r5			;; new R5 = R5 + R7
	addpd	r8, r6			;; new R7 = R6 + R8

	addpd	r5, r6			;; R6 = R6 + R8
	mulpd	r6, xmm15		;; R8 = R8 * 2
	subpd	r6, r5			;; R8 = R8 - R6

	mulpd	r5, xmm14		;; R6 = R6 * square root of 1/2
	mulpd	r6, xmm14		;; R8 = R8 * square root of 1/2

	xload	r3, mem2		;; R2
	subpd	r3, r4			;; R2 = R2 - R4 (new R4)
	addpd	r4, mem2		;; R4 = R2 + R4 (new R2)

	subpd	r3, r6			;; R4 = R4 - R8 (final R8)
	mulpd	r6, xmm15		;; R8 = R8 * 2
	subpd	r4, r5			;; R2 = R2 - R6 (final R6)
	mulpd	r5, xmm15		;; R6 = R6 * 2
	addpd	r6, r3			;; R8 = R4 + R8 (final R4)
	addpd	r5, r4			;; R6 = R2 + R6 (final R2)

	xload	r7, mem1		;; R1
	subpd	r7, r1			;; R1 = R1 - R3 (new R3)
	addpd	r1, mem1		;; R3 = R1 + R3 (new R1)

	xstore	dst8, r3		;; Save R8
	xstore	dst4, r6		;; Save R4

	subpd	r7, r8			;; R3 = R3 - R7 (final R7)
	mulpd	r8, xmm15		;; R7 = R7 * 2
	subpd	r1, r2			;; R1 = R1 - R5 (final R5)
	mulpd	r2, xmm15		;; R5 = R5 * 2
	addpd	r8, r7			;; R7 = R3 + R7 (final R3)
	addpd	r2, r1			;; R5 = R1 + R5 (final R1)
	ENDM

ENDIF
ENDIF

; The with-partial-normalization variant

r4_x4cl_wpn_eight_reals_unfft_preload MACRO
	ENDM

r4_x4cl_wpn_eight_reals_unfft MACRO srcreg,srcinc,d1,d2,screg1,scoff1,screg2,scoff2
	d3 = d2 + d1
	d2a = d2
	d1a = d1
	tmp = srcinc+d2
	r4_x8r_wpn_unfft_mem [srcreg+16],[srcreg+48],[srcreg+d1a+16],[srcreg+d1a+48],[srcreg+d2a+16],[srcreg+d2a+48],[srcreg+d3+16],[srcreg+d3+48],screg2+scoff2,screg1+scoff1+32,screg2+scoff2+48,screg1+scoff1,srcreg+tmp,d1a,[srcreg+d3+16],[srcreg+d3+48]
	xstore	[srcreg+d1+16], xmm7	;; Save R3
;;	xstore	[srcreg+d3+16], xmm2	;; Save R4
	xstore	[srcreg+d1+48], xmm6	;; Save R7
;;	xstore	[srcreg+d3+48], xmm5	;; Save R8
	xload	xmm6, [srcreg+d3]	;; Load R4
	xload	xmm7, [srcreg+d3+32]	;; Load I4
	xload	xmm2, [srcreg+d1]	;; Load R2
	xload	xmm5, [srcreg+d1+32]	;; Load I2
	xstore	[srcreg+d1], xmm1	;; Save R1
	xstore	[srcreg+d3], xmm4	;; Save R2
	xstore	[srcreg+d1+32], xmm0	;; Save R5
	xstore	[srcreg+d3+32], xmm3	;; Save R6
	r4_x8r_wpn_unfft_partial_mem xmm0,xmm1,xmm2,xmm5,xmm3,xmm4,xmm6,xmm7,[srcreg],[srcreg+32],[srcreg+d2],[srcreg+d2+32],screg2,screg1+32,screg2+48,screg1,srcreg+srcinc,d1,[srcreg+d2+16],[srcreg+d2+48]
	xstore	[srcreg], xmm1		;; Save R1
	xstore	[srcreg+d2], xmm3	;; Save R2
	xstore	[srcreg+32], xmm0	;; Save R5
	xstore	[srcreg+d2+32], xmm5	;; Save R6
	xstore	[srcreg+16], xmm7	;; Save R3
;;	xstore	[srcreg+d2+16], xmm2	;; Save R4
	xstore	[srcreg+48], xmm6	;; Save R7
;;	xstore	[srcreg+d2+48], xmm4	;; Save R8
	bump	srcreg, srcinc
	ENDM

r4_x8r_wpn_unfft_mem MACRO mem1,mem2,mem3,mem4,mem5,mem6,mem7,mem8,screg1,screg2,screg5,normreg,pre1,pre2,dst4,dst8
	xload	xmm6, [screg5+16]	;; cosine/sine for w^5n
	xload	xmm1, mem7		;; R4
	mulpd	xmm1, xmm6		;; A4 = R4 * cosine/sine
	xload	xmm7, mem8		;; I4
	addpd	xmm1, xmm7		;; A4 = A4 + I4
	mulpd	xmm7, xmm6		;; B4 = I4 * cosine/sine

	xload	xmm2, [screg1+16]	;; cosine/sine for w^n
	xload	xmm4, mem5		;; R3
	mulpd	xmm4, xmm2		;; A3 = R3 * cosine/sine for w^n
	xload	xmm5, mem6		;; I3
	addpd	xmm4, xmm5		;; A3 = A3 + I3
	mulpd	xmm5, xmm2		;; B3 = I3 * cosine/sine

	xload	xmm6, [screg2+16]	;; cosine/sine for w^2n
	xload	xmm0, mem3		;; R2
	mulpd	xmm0, xmm6		;; A2 = R2 * cosine/sine
	xload	xmm3, mem4		;; I2
	addpd	xmm0, xmm3		;; A2 = A2 + I2
	mulpd	xmm3, xmm6		;; B2 = I2 * cosine/sine

	subpd	xmm7, mem7		;; B4 = B4 - R4
	subpd	xmm5, mem5		;; B3 = B3 - R3
	subpd	xmm3, mem3		;; B2 = B2 - R2

	xprefetchw [pre1]

	xload	xmm6, [screg5+32]	;; Sine * normalization_inverse
	mulpd	xmm1, xmm6		;; new R7 = A4 * sine (final R4)
	mulpd	xmm7, xmm6		;; new R8 = B4 * sine (final I4)
	xload	xmm2, [screg1+32]	;; Sine * normalization_inverse
	mulpd	xmm4, xmm2		;; new R5 = A3 * sine (final R3)
	mulpd	xmm5, xmm2		;; new R6 = B3 * sine (final I3)
	xload	xmm6, [screg2+32]	;; Sine * normalization_inverse
	mulpd	xmm0, xmm6		;; new R3 = A2 * sine (final R2)
	mulpd	xmm3, xmm6		;; new R4 = B2 * sine (final I2)

	xload	xmm2, mem2		;; R2
	mulpd	xmm2, [normreg+16]	;; R2 * normalization_inverse

	xprefetchw [pre1][pre2]

	xcopy	xmm6, xmm4		;; Copy R5
	subpd	xmm4, xmm1		;; new R6 = R5 - R7
	addpd	xmm1, xmm6		;; new R5 = R5 + R7

	xcopy	xmm6, xmm5		;; Copy R6
	subpd	xmm5, xmm7		;; new R8 = R6 - R8
	addpd	xmm7, xmm6		;; new R7 = R6 + R8

	xcopy	xmm6, xmm4		;; Copy R6
	addpd	xmm4, xmm5		;; R6 = R6 + R8
	subpd	xmm5, xmm6		;; R8 = R8 - R6

	mulpd	xmm4, XMM_SQRTHALF	;; R6 = R6 * square root of 1/2
	mulpd	xmm5, XMM_SQRTHALF	;; R8 = R8 * square root of 1/2

	xcopy	xmm6, xmm2		;; Copy R2
	subpd	xmm2, xmm3		;; R2 = R2 - R4 (new R4)
	addpd	xmm3, xmm6		;; R4 = R2 + R4 (new R2)

	xcopy	xmm6, xmm2		;; Copy R4
	subpd	xmm2, xmm5		;; R4 = R4 - R8 (final R8)
	addpd	xmm5, xmm6		;; R8 = R4 + R8 (final R4)

	xcopy	xmm6, xmm3		;; Copy R2
	subpd	xmm3, xmm4		;; R2 = R2 - R6 (final R6)
	addpd	xmm4, xmm6		;; R6 = R2 + R6 (final R2)

	xload	xmm6, mem1		;; R1
	mulpd	xmm6, [normreg+16]	;; R1 * normalization_inverse

	xstore	dst8, xmm2		;; Save R8

	xcopy	xmm2, xmm6		;; Copy R1
	subpd	xmm6, xmm0		;; R1 = R1 - R3 (new R3)
	addpd	xmm0, xmm2		;; R3 = R1 + R3 (new R1)

	xstore	dst4, xmm5		;; Save R4

	xcopy	xmm2, xmm6		;; Copy R3
	subpd	xmm6, xmm7		;; R3 = R3 - R7 (final R7)
	addpd	xmm7, xmm2		;; R7 = R3 + R7 (final R3)

	xcopy	xmm5, xmm0		;; Copy R1
	subpd	xmm0, xmm1		;; R1 = R1 - R5 (final R5)
	addpd	xmm1, xmm5		;; R5 = R1 + R5 (final R1)
	ENDM

r4_x8r_wpn_unfft_partial_mem MACRO r1,r2,r3,r4,r5,r6,r7,r8,mem1,mem2,mem5,mem6,screg1,screg2,screg5,normreg,pre1,pre2,dst4,dst8
	xload	r2, [screg5+16]		;; cosine/sine for w^5n
	mulpd	r2, r7			;; A4 = R4 * cosine/sine
	addpd	r2, r8			;; A4 = A4 + I4
	mulpd	r8, [screg5+16]		;; B4 = I4 * cosine/sine
	subpd	r8, r7			;; B4 = B4 - R4

	xload	r1, [screg1+16]		;; cosine/sine for w^n
	xload	r5, mem5		;; R3
	mulpd	r5, r1			;; A3 = R3 * cosine/sine for w^n
	xload	r6, mem6		;; I3
	addpd	r5, r6			;; A3 = A3 + I3
	mulpd	r6, r1			;; B3 = I3 * cosine/sine
	subpd	r6, mem5		;; B3 = B3 - R3

	xload	r1, [screg2+16]		;; cosine/sine for w^2n
	mulpd	r1, r3			;; A2 = R2 * cosine/sine
	addpd	r1, r4			;; A2 = A2 + I2
	mulpd	r4, [screg2+16]		;; B2 = I2 * cosine/sine
	subpd	r4, r3			;; B2 = B2 - R2

	xprefetchw [pre1]

	xload	r7, [screg5+32]		;; Sine * normalization_inverse
	mulpd	r2, r7			;; new R7 = A4 * sine (final R4)
	mulpd	r8, r7			;; new R8 = B4 * sine (final I4)
	xload	r3, [screg1+32]		;; Sine * normalization_inverse
	mulpd	r5, r3			;; new R5 = A3 * sine (final R3)
	mulpd	r6, r3			;; new R6 = B3 * sine (final I3)
	xload	r7, [screg2+32]		;; Sine * normalization_inverse
	mulpd	r1, r7			;; new R3 = A2 * sine (final R2)
	mulpd	r4, r7			;; new R4 = B2 * sine (final I2)

	xload	r3, mem2		;; R2
	mulpd	r3, [normreg+16]	;; R2 * normalization_inverse

	xprefetchw [pre1][pre2]

	xcopy	r7, r5			;; Copy R5
	subpd	r5, r2			;; new R6 = R5 - R7
	addpd	r2, r7			;; new R5 = R5 + R7

	xcopy	r7, r6			;; Copy R6
	subpd	r6, r8			;; new R8 = R6 - R8
	addpd	r8, r7			;; new R7 = R6 + R8

	xcopy	r7, r5			;; Copy R6
	addpd	r5, r6			;; R6 = R6 + R8
	subpd	r6, r7			;; R8 = R8 - R6

	mulpd	r5, XMM_SQRTHALF	;; R6 = R6 * square root of 1/2
	mulpd	r6, XMM_SQRTHALF	;; R8 = R8 * square root of 1/2

	xcopy	r7, r3			;; Copy R2
	subpd	r3, r4			;; R2 = R2 - R4 (new R4)
	addpd	r4, r7			;; R4 = R2 + R4 (new R2)

	xcopy	r7, r3			;; Copy R4
	subpd	r3, r6			;; R4 = R4 - R8 (final R8)
	addpd	r6, r7			;; R8 = R4 + R8 (final R4)

	xload	r7, mem1		;; R1
	mulpd	r7, [normreg+16]	;; R1 * normalization_inverse

	xstore	dst8, r3		;; Save R8

	xcopy	r3, r4			;; Copy R2
	subpd	r4, r5			;; R2 = R2 - R6 (final R6)
	addpd	r5, r3			;; R6 = R2 + R6 (final R2)

	xcopy	r3, r7			;; Copy R1
	subpd	r7, r1			;; R1 = R1 - R3 (new R3)
	addpd	r1, r3			;; R3 = R1 + R3 (new R1)

	xstore	dst4, r6		;; Save R4

	xcopy	r3, r7			;; Copy R3
	subpd	r7, r8			;; R3 = R3 - R7 (final R7)
	addpd	r8, r3			;; R7 = R3 + R7 (final R3)

	xcopy	r6, r1			;; Copy R1
	subpd	r1, r2			;; R1 = R1 - R5 (final R5)
	addpd	r2, r6			;; R5 = R1 + R5 (final R1)
	ENDM

IFDEF X86_64

; Theoretical best Core 2 timing is 48 clocks.  The empty add slot at clock 14 means
; this macro should run in 49 clocks.  Actual timing is ?? clocks.

r4_x4cl_wpn_eight_reals_unfft_preload MACRO
	xload	xmm15, XMM_SQRTHALF
	ENDM

r4_x4cl_wpn_eight_reals_unfft MACRO srcreg,srcinc,d1,d2,screg1,scoff1,screg2,scoff2
	xload	xmm0, [srcreg+d2+d1+16]	;; R4
	xload	xmm1, [screg2+scoff2+48+16];; cosine/sine for w^5n
	xcopy	xmm2, xmm0		;; Copy R4
	mulpd	xmm0, xmm1		;; A4 = R4 * cosine/sine		; 1-5

	xload	xmm3, [srcreg+d2+16]	;; R3
	xload	xmm4, [screg2+scoff2+16];; cosine/sine for w^n
	xcopy	xmm5, xmm3		;; Copy R3
	mulpd	xmm3, xmm4		;; A3 = R3 * cosine/sine for w^n	; 2-6

	xload	xmm6, [srcreg+d2+d1+48]	;; I4
	mulpd	xmm1, xmm6		;; B4 = I4 * cosine/sine		; 3-7

	xload	xmm7, [srcreg+d2+48]	;; I3
	mulpd	xmm4, xmm7		;; B3 = I3 * cosine/sine		; 4-8

	xload	xmm8, [srcreg+d2+d1]	;;#2 R4
	xload	xmm9, [screg2+48+16]	;;#2 cosine/sine for w^5n
	xcopy	xmm10, xmm8		;;#2 Copy R4
	mulpd	xmm8, xmm9		;;#2 A4 = R4 * cosine/sine		; 5-9

	addpd	xmm0, xmm6		;; A4 = A4 + I4				; 6-8	avail 6,11+
	xload	xmm11, [srcreg+d2]	;;#2 R3
	xload	xmm12, [screg2+16]	;;#2 cosine/sine for w^n
	xcopy	xmm13, xmm11		;;#2 Copy R3
	mulpd	xmm11, xmm12		;;#2 A3 = R3 * cosine/sine for w^n	; 6-10	avail 6,14

	addpd	xmm3, xmm7		;; A3 = A3 + I3				; 7-9	avail 6,7,14
	xload	xmm14, [srcreg+d2+d1+32];;#2 I4
	mulpd	xmm9, xmm14		;;#2 B4 = I4 * cosine/sine		; 7-11	avail 6,7

	subpd	xmm1, xmm2		;; B4 = B4 - R4				; 8-10	avail 6,7,2
	xload	xmm6, [srcreg+d2+32]	;;#2 I3
	mulpd	xmm12, xmm6		;;#2 B3 = I3 * cosine/sine		; 8-12	avail 7,2

	subpd	xmm4, xmm5		;; B3 = B3 - R3				; 9-11	avail 7,2,5
	xload	xmm7, [screg2+scoff2+48+32] ;; sine * normalization_inverse
	mulpd	xmm0, xmm7		;; new R7 = A4 * sine (final R4)	; 9-13	avail 2,5

	addpd	xmm8, xmm14		;;#2 A4 = A4 + I4			; 10-12	avail 2,5,14
	xload	xmm2, [screg2+scoff2+32] ;; sine * normalization_inverse
	mulpd	xmm3, xmm2		;; new R5 = A3 * sine (final R3)	; 10-14	avail 5,14

	addpd	xmm11, xmm6		;;#2 A3 = A3 + I3			; 11-13	avail 5,14,6
	mulpd	xmm1, xmm7		;; new R8 = B4 * sine (final I4)	; 11-15	avail 5,14,6,7

	subpd	xmm9, xmm10		;;#2 B4 = B4 - R4			; 12-14	avail 5,14,6,7,10
	mulpd	xmm4, xmm2		;; new R6 = B3 * sine (final I3)	; 12-16	avail 5,14,6,7,10,2

	subpd	xmm12, xmm13		;;#2 B3 = B3 - R3			; 13-15	avail 5,14,6,7,10,2,13
	xload	xmm5, [srcreg+d1+16]	;; R2
	xload	xmm14, [screg1+scoff1+32+16];; cosine/sine for w^2n
	xcopy	xmm6, xmm5		;; Copy R2
	mulpd	xmm5, xmm14		;; A2 = R2 * cosine/sine		; 13-17	avail 7,10,2,13

	xload	xmm7, [srcreg+d1+48]	;; I2
	mulpd	xmm14, xmm7		;; B2 = I2 * cosine/sine		; 14-18	avail 10,2,13

	xprefetchw [srcreg+srcinc]

	xcopy	xmm10, xmm3		;; Copy R5
	subpd	xmm3, xmm0		;; new R6 = R5 - R7			; 15-17	avail 2,13
	xload	xmm2, [screg2+48+32]	;;#2 sine * normalization_inverse
	mulpd	xmm8, xmm2		;;#2 new R7 = A4 * sine (final R4)	; 15-19	avail 13

	addpd	xmm10, xmm0		;; new R5 = R5 + R7			; 16-18	avail 13,0
	xload	xmm13, [screg2+32]	;;#2 sine * normalization_inverse
	mulpd	xmm11, xmm13		;;#2 new R5 = A3 * sine (final R3)	; 16-20	avail 0

	xcopy	xmm0, xmm4		;; Copy R6
	subpd	xmm4, xmm1		;; new R8 = R6 - R8			; 17-19
	mulpd	xmm9, xmm2		;;#2 new R8 = B4 * sine (final I4)	; 17-21 avail 2

	addpd	xmm5, xmm7		;; A2 = A2 + I2				; 18-20	avail 2,7
	mulpd	xmm12, xmm13		;;#2 new R6 = B3 * sine (final I3)	; 18-22	avail 2,7,13

	addpd	xmm0, xmm1		;; new R7 = R6 + R8			; 19-21	avail 2,7,13,1
	mulpd	xmm3, xmm15		;; R6 = R6 * square root of 1/2		; 19-23

	subpd	xmm14, xmm6		;; B2 = B2 - R2				; 20-22	avail 2,7,13,1,6
	mulpd	xmm4, xmm15		;; R8 = R8 * square root of 1/2		; 20-24
	xload	xmm6, [srcreg+16]	;; R1					; 20

	xprefetchw [srcreg+srcinc+d1]

	xcopy	xmm2, xmm11		;;#2 Copy R5
	subpd	xmm11, xmm8		;;#2 new R6 = R5 - R7			; 21-23	avail 7,13,1
	mulpd	xmm6, [screg1+scoff1+16];; R1 * normalization_inverse		; 21-25

	addpd	xmm2, xmm8		;;#2 new R5 = R5 + R7			; 22-24	avail 7,13,1,8
	xload	xmm7, [screg1+scoff1+32+32] ;; sine * normalization_inverse
	mulpd	xmm5, xmm7		;; new R3 = A2 * sine (final R2)	; 22-26	avail 13,1,8

	xcopy	xmm13, xmm12		;;#2 Copy R6
	subpd	xmm12, xmm9		;;#2 new R8 = R6 - R8			; 23-25	avail 1,8
	mulpd	xmm14, xmm7		;; new R4 = B2 * sine (final I2)	; 23-27	avail 1,8,7
	xload	xmm7, [srcreg+48]	;; R2

	addpd	xmm13, xmm9		;;#2 new R7 = R6 + R8			; 24-26	avail 1,8,9
	mulpd	xmm7, [screg1+scoff1+16];; R2 * normalization_inverse		; 24-28

	xcopy	xmm1, xmm3		;; Copy R6
	addpd	xmm3, xmm4		;; R6 = R6 + R8				; 25-27	avail 8,7,9

	subpd	xmm4, xmm1		;; R8 = R8 - R6				; 26-28	avail 8,9,1

	xprefetchw [srcreg+srcinc+d2]	;;#2

	xcopy	xmm8, xmm6		;; Copy R1
	subpd	xmm6, xmm5		;; R1 = R1 - R3 (new R3)		; 27-29	avail 9,1
	addpd	xmm8, xmm5		;; R3 = R1 + R3 (new R1)		; 28-30	avail 9,1,5

	xcopy	xmm9, xmm7		;; Copy R2
	subpd	xmm7, xmm14		;; R2 = R2 - R4 (new R4)		; 29-31	avail 1,5

	addpd	xmm14, xmm9		;; R4 = R2 + R4 (new R2)		; 30-32	avail 1,5,9
	xload	xmm1, [srcreg+d1]	;;#2 R2
	xload	xmm5, [screg1+32+16]	;;#2 cosine/sine for w^2n
	mulpd	xmm1, xmm5		;;#2 A2 = R2 * cosine/sine		; 30-34	avail 9

	xcopy	xmm9, xmm6		;; Copy R3
	subpd	xmm6, xmm0		;; R3 = R3 - R7 (final R7)		; 31-33	avail none storable 6
	xstore	[srcreg+d1+48], xmm6	;; Save R7				; 34	avail 6
	xload	xmm6, [srcreg+d1+32]	;;#2 I2
	mulpd	xmm5, xmm6		;;#2 B2 = I2 * cosine/sine		; 31-35	avail none

	addpd	xmm0, xmm9		;; R7 = R3 + R7 (final R3)		; 32-34	avail 9 storable 0

	xcopy	xmm9, xmm8		;; Copy R1
	subpd	xmm8, xmm10		;; R1 = R1 - R5 (final R5)		; 33-35	avail none storable 0,8
	addpd	xmm10, xmm9		;; R5 = R1 + R5 (final R1)		; 34-36	avail 9 storable 0,8,10

	addpd	xmm1, xmm6		;;#2 A2 = A2 + I2			; 35-37	avail 9,6 storable 0,8,10
	mulpd	xmm11, xmm15		;;#2 R6 = R6 * square root of 1/2	; 35-39
	xstore	[srcreg+d1+16], xmm0	;; Save R3				; 35	avail 9,6,0 storable 8,10

	subpd	xmm5, [srcreg+d1]	;;#2 B2 = B2 - R2			; 36-38
	mulpd	xmm12, xmm15		;;#2 R8 = R8 * square root of 1/2	; 36-40
	xstore	[srcreg+d1+32], xmm8	;; Save R5				; 36	avail 9,6,0,8 storable 10

	xcopy	xmm9, xmm7		;; Copy R4
	subpd	xmm7, xmm4		;; R4 = R4 - R8 (final R8)		; 37-39	avail 6,0,8 storable 10,7
	xstore	[srcreg+d1], xmm10	;; Save R1				; 37	avail 6,0,8,10 storable 7
	xload	xmm10, [srcreg]		;;#2 R1					; 37
	mulpd	xmm10, [screg1+16]	;;#2 R1 * normalization_inverse		; 37-41

	xprefetchw [srcreg+srcinc+d2+d1];;#2

	addpd	xmm4, xmm9		;; R8 = R4 + R8 (final R4)		; 38-40	avail 6,0,8,9 storable 7,4
	xload	xmm6, [screg1+32+32]	;;#2 sine * normalization_inverse
	mulpd	xmm1, xmm6		;;#2 new R3 = A2 * sine (final R2)	; 38-42	avail 0,8,9 storable 7,4

	xcopy	xmm0, xmm14		;; Copy R2
	subpd	xmm14, xmm3		;; R2 = R2 - R6 (final R6)		; 39-41	avail 8,9 storable 7,4,14
	mulpd	xmm5, xmm6		;;#2 new R4 = B2 * sine (final I2)	; 39-43	avail 8,9,6 storable 7,4,14
	xload	xmm6, [srcreg+32]	;;#2 R2					; 39

	addpd	xmm3, xmm0		;; R6 = R2 + R6 (final R2)		; 40-42	avail 8,9,0 storable 7,4,14,3
	mulpd	xmm6, [screg1+16]	;;#2 R2 * normalization_inverse		; 40-44
	xstore	[srcreg+d2+d1+48], xmm7	;; Save R8				; 40	avail 8,9,0,7 storable 4,14,3

	xcopy	xmm8, xmm11		;;#2 Copy R6
	addpd	xmm11, xmm12		;;#2 R6 = R6 + R8			; 41-43	avail 9,0,7 storable 4,14,3
	xstore	[srcreg+d2+d1+16], xmm4	;; Save R4				; 41	avail 9,0,7,4 storable 14,3

	subpd	xmm12, xmm8		;;#2 R8 = R8 - R6			; 42-44	avail 9,0,7,4,8 storable 14,3
	xstore	[srcreg+d2+d1+32], xmm14;; Save R6				; 42	avail 9,0,7,4,8,14 storable 3

	xcopy	xmm9, xmm10		;;#2 Copy R1
	subpd	xmm10, xmm1		;;#2 R1 = R1 - R3 (new R3)		; 43-45	avail 0,7,4,8,14 storable 3
	xstore	[srcreg+d2+d1], xmm3	;; Save R2				; 43	avail 0,7,4,8,14,3

	addpd	xmm1, xmm9		;;#2 R3 = R1 + R3 (new R1)		; 44-46	avail 0,7,4,8,14,3,9

	xcopy	xmm0, xmm6		;;#2 Copy R2
	subpd	xmm6, xmm5		;;#2 R2 = R2 - R4 (new R4)		; 45-47	avail 7,4,8,14,3,9

	addpd	xmm5, xmm0		;;#2 R4 = R2 + R4 (new R2)		; 46-48	avail 7,4,8,14,3,9,0

	xcopy	xmm7, xmm10		;;#2 Copy R3
	subpd	xmm10, xmm13		;;#2 R3 = R3 - R7 (final R7)		; 47-49	avail 4,8,14,3,9,0 storable 10
	addpd	xmm13, xmm7		;;#2 R7 = R3 + R7 (final R3)		; 48-50	avail 4,8,14,3,9,0,7 storable 10,13

	xcopy	xmm4, xmm1		;;#2 Copy R1
	subpd	xmm1, xmm2		;;#2 R1 = R1 - R5 (final R5)		; 49-51	avail 8,14,3,9,0,7 storable 10,13,1

	addpd	xmm2, xmm4		;;#2 R5 = R1 + R5 (final R1)		; 50-52	avail 8,14,3,9,0,7,4 storable 10,13,1,2
	xstore	[srcreg+48], xmm10	;;#2 Save R7				; 50	avail 8,14,3,9,0,7,4,10 storable 13,1,2

	xcopy	xmm8, xmm6		;;#2 Copy R4
	subpd	xmm6, xmm12		;;#2 R4 = R4 - R8 (final R8)		; 51-53	avail 14,3,9,0,7,4,10 storable 13,1,2,6
	xstore	[srcreg+16], xmm13	;;#2 Save R3				; 51	avail 14,3,9,0,7,4,10,13 storable 1,2,6

	addpd	xmm12, xmm8		;;#2 R8 = R4 + R8 (final R4)		; 52-54	avail 14,3,9,0,7,4,10,13,8 storable 1,2,6,12
	xstore	[srcreg+32], xmm1	;;#2 Save R5				; 52	avail 14,3,9,0,7,4,10,13,8,1 storable 2,6,12

	xcopy	xmm8, xmm5		;;#2 Copy R2
	subpd	xmm5, xmm11		;;#2 R2 = R2 - R6 (final R6)		; 53-55	avail 14,3,9,0,7,4,10,13,1 storable 2,6,12,5
	xstore	[srcreg], xmm2		;;#2 Save R1				; 53	avail 14,3,9,0,7,4,10,13,1,2 storable 6,12,5

	addpd	xmm11, xmm8		;;#2 R6 = R2 + R6 (final R2)		; 54-56	avail 14,3,9,0,7,4,10,13,1,2,8 storable 6,12,5,11
	xstore	[srcreg+d2+48], xmm6	;;#2 Save R8				; 54
	xstore	[srcreg+d2+16], xmm12	;;#2 Save R4				; 55
	xstore	[srcreg+d2+32], xmm5	;;#2 Save R6				; 56
	xstore	[srcreg+d2], xmm11	;;#2 Save R2				; 57

	bump	srcreg, srcinc
	ENDM

ENDIF

;;
;; ************************************* eight-reals-four-complex-djbfft variants ******************************************
;;
;; Because of the way we store FFT data elements in cache lines during the pass 1 of the forward FFT, a cache line
;; will contain data needing an eight-reals-fft as well as data needing a four-complex-fft.
;;

;; Macro to do an eight_reals_fft and a four_complex_djbfft.
;; The eight-reals macro is very similar to the eight-reals-first-FFT.
;; Both do 2 and 3/4 levels of the FFT.  One difference this
;; macro takes two screg pointers.  One points to w^n and w^5n, the other
;; points to w^2n which is also the screg used by the four-complex-FFT.

r4_x4cl_eight_reals_four_complex_djbfft_preload MACRO
	r4_x8r_fft_mem_preload
	;r4_x4c_djbfft_partial_mem_preload -- must be compatible with r4_x8r_fft_mem_preload!!
	ENDM

r4_x4cl_eight_reals_four_complex_djbfft MACRO srcreg,srcinc,d1,d2,screg1,screg2
	r4_x8r_fft_mem [srcreg],[srcreg+d1],[srcreg+d2],[srcreg+d2+d1],[srcreg+16],[srcreg+d1+16],[srcreg+d2+16],[srcreg+d2+d1+16],screg2,screg1,screg2+32,srcreg+srcinc,d1,[srcreg],[srcreg+16]
;;	xstore	[srcreg], xmm0		;; Save real value #1
;;	xstore	[srcreg+16], xmm0	;; Save real value #2
	xload	xmm0, [srcreg+32]	;; R1
	xload	xmm1, [srcreg+48]	;; R5
	xstore	[srcreg+32], xmm7	;; Save R2
	xstore	[srcreg+48], xmm5	;; Save I2
	xload	xmm7, [srcreg+d1+32]	;; R2
	xload	xmm5, [srcreg+d1+48]	;; R6
	xstore	[srcreg+d1], xmm4	;; Save R3
	xstore	[srcreg+d1+16], xmm3	;; Save I3
	xstore	[srcreg+d1+32], xmm6	;; Save R4
	xstore	[srcreg+d1+48], xmm2	;; Save I4
	r4_x4c_djbfft_partial_mem xmm0,xmm7,xmm3,xmm2,xmm1,xmm5,xmm4,xmm6,[srcreg+d2+32],[srcreg+d2+d1+32],[srcreg+d2+48],[srcreg+d2+d1+48],screg1,0,srcreg+srcinc+d2,d1,[srcreg+d2],[srcreg+d2+16]
;;	xstore	[srcreg+d2], xmm0	;; Save real value #1
;;	xstore	[srcreg+d2+16], xmm0	;; Save real value #2
	xstore	[srcreg+d2+32], xmm6	;; Save R2
	xstore	[srcreg+d2+48], xmm4	;; Save I2
	xstore	[srcreg+d2+d1], xmm2	;; Save R3
	xstore	[srcreg+d2+d1+16], xmm7	;; Save I3
	xstore	[srcreg+d2+d1+32], xmm0	;; Save R4
	xstore	[srcreg+d2+d1+48], xmm1	;; Save I4
	bump	srcreg, srcinc
	ENDM

;; Macro to do an eight_reals_fft and a four_complex_djbfft.
;; This macro is used in the last levels of an r4 FFT pass 1 (no swizzling)

r4_sg4cl_eight_reals_four_complex_djbfft_preload MACRO
	r4_x8r_fft_mem_preload
	;r4_x4c_djbfft_partial_mem_preload -- must be compatible with r4_x8r_fft_mem_preload!!
	ENDM

r4_sg4cl_eight_reals_four_complex_djbfft MACRO srcreg,srcinc,d1,d2,dstreg,dstinc,e1,e2,screg1,screg2
	r4_x8r_fft_mem [srcreg],[srcreg+d1],[srcreg+d2],[srcreg+d2+d1],[srcreg+16],[srcreg+d1+16],[srcreg+d2+16],[srcreg+d2+d1+16],screg2,screg1,screg2+32,dstreg+dstinc,e1,[dstreg],[dstreg+32]
	shuffle_store_partial [dstreg], [dstreg+16], xmm0, xmm7				;; Save real value #1,R2
	shuffle_store_partial [dstreg+32], [dstreg+48], xmm1, xmm5			;; Save real value #2,I2
	xload	xmm0, [srcreg+32]	;; R1
	xload	xmm7, [srcreg+d1+32]	;; R2
	xload	xmm5, [srcreg+48]	;; R5
	shuffle_store_with_temp [dstreg+e1], [dstreg+e1+16], xmm4, xmm6, xmm1		;; Save R3,R4
	shuffle_store_with_temp [dstreg+e1+32], [dstreg+e1+48], xmm3, xmm2, xmm1	;; Save I3,I4
	xload	xmm1, [srcreg+d1+48]	;; R6
	xprefetch [srcreg+srcinc+d2]
	r4_x4c_djbfft_partial_mem xmm0,xmm7,xmm3,xmm2,xmm5,xmm1,xmm6,xmm4,[srcreg+d2+32],[srcreg+d2+d1+32],[srcreg+d2+48],[srcreg+d2+d1+48],screg1,0,dstreg+dstinc+e2,e1,[dstreg+e2],[dstreg+e2+32]
	xprefetch [srcreg+srcinc+d2+d1]
	bump	srcreg, srcinc
	shuffle_store_partial [dstreg+e2], [dstreg+e2+16], xmm3, xmm4			;; Save R1,R2
	shuffle_store_partial [dstreg+e2+32], [dstreg+e2+48], xmm1, xmm6		;; Save I1,I2
	shuffle_store_with_temp [dstreg+e2+e1], [dstreg+e2+e1+16], xmm2, xmm0, xmm3	;; Save R3,R4
	shuffle_store_with_temp [dstreg+e2+e1+32], [dstreg+e2+e1+48], xmm7, xmm5, xmm3	;; Save I3,I4
	bump	dstreg, dstinc
	ENDM


;; Macro to do an eight_reals_fft and a four_complex_fft4 at the end of pass 1.
;; We could do a four_complex_djbfft since the 4 twiddle factors belong to the
;; first grouping.  However, this will only work if the corresponding unfft does
;; a djbunfft.  This macro is very similar to r4_x4cl_eight_reals_four_complex_djbfft.

;; This macro is used in the last levels of an r4delay pass 1 (no swizzling)
IFDEF UNUSED
r4_g4cl_eight_reals_four_complex_fft4_preload MACRO
	r4_x8r_fft_mem_preload
	;r4_x4c_djbfft_partial_mem_preload -- must be compatible with r4_x8r_fft_mem_preload!!
	ENDM
r4_g4cl_eight_reals_four_complex_fft4 MACRO srcreg,srcinc,d1,d2,dstreg,dstinc,e1,e2,screg1,screg2
	r4_x8r_fft_mem [srcreg],[srcreg+d1],[srcreg+d2],[srcreg+d2+d1],[srcreg+16],[srcreg+d1+16],[srcreg+d2+16],[srcreg+d2+d1+16],screg2,screg1+32,screg2+32,dstreg+dstinc,e1,[dstreg],[dstreg+16]
;;	xstore	[dstreg], xmm0		;; Save real value #1
;;	xstore	[dstreg+16], xmm0	;; Save real value #2
	xload	xmm0, [srcreg+32]	;; R1
	xload	xmm1, [srcreg+48]	;; R5
	xstore	[dstreg+32], xmm7	;; Save R2
	xstore	[dstreg+48], xmm5	;; Save I2
	xload	xmm7, [srcreg+d1+32]	;; R2
	xload	xmm5, [srcreg+d1+48]	;; R6
	xstore	[dstreg+e1], xmm4	;; Save R3
	xstore	[dstreg+e1+16], xmm3	;; Save I3
	xstore	[dstreg+e1+32], xmm6	;; Save R4
	xstore	[dstreg+e1+48], xmm2	;; Save I4
	r4_x4c_fft_partial_mem xmm0,xmm7,xmm3,xmm2,xmm1,xmm5,xmm4,xmm6,[srcreg+d2+32],[srcreg+d2+d1+32],[srcreg+d2+48],[srcreg+d2+d1+48],screg1,32,dstreg+dstinc+e2,e1,[dstreg+e2],[dstreg+e2+16]
	bump	srcreg, srcinc
;;	xstore	[dstreg+e2], xmm0	;; Save R1
;;	xstore	[dstreg+e2+16], xmm0	;; Save I1
	xstore	[dstreg+e2+32], xmm6	;; Save R2
	xstore	[dstreg+e2+48], xmm4	;; Save I2
	xstore	[dstreg+e2+e1], xmm2	;; Save R3
	xstore	[dstreg+e2+e1+16], xmm7	;; Save I3
	xstore	[dstreg+e2+e1+32], xmm0	;; Save R4
	xstore	[dstreg+e2+e1+48], xmm1	;; Save I4
	bump	dstreg, dstinc
	ENDM
ENDIF

;; Used in the last levels of an r4delay pass 1 (swizzling)
r4_sg4cl_eight_reals_four_complex_fft4_preload MACRO
	r4_x8r_fft_mem_preload
	;r4_x4c_djbfft_mem_preload -- must be compatible with r4_x8r_fft_mem_preload!!
	ENDM
r4_sg4cl_eight_reals_four_complex_fft4 MACRO srcreg,srcinc,d1,d2,dstreg,dstinc,e1,e2,screg1,screg2
	r4_x8r_fft_mem [srcreg],[srcreg+d1],[srcreg+d2],[srcreg+d2+d1],[srcreg+16],[srcreg+d1+16],[srcreg+d2+16],[srcreg+d2+d1+16],screg2,screg1+32,screg2+32,dstreg+dstinc,e1,[dstreg],[dstreg+32]
	shuffle_store_partial [dstreg], [dstreg+16], xmm0, xmm7				;; Save real value #1,R2
	shuffle_store_partial [dstreg+32], [dstreg+48], xmm1, xmm5			;; Save real value #2,I2
	shuffle_store_with_temp [dstreg+e1], [dstreg+e1+16], xmm4, xmm6, xmm7		;; Save R3,R4
	shuffle_store_with_temp [dstreg+e1+32], [dstreg+e1+48], xmm3, xmm2, xmm7	;; Save I3,I4
	xprefetch [srcreg+srcinc+d2]
	r4_x4c_fft_mem [srcreg+32],[srcreg+d1+32],[srcreg+d2+32],[srcreg+d2+d1+32],[srcreg+48],[srcreg+d1+48],[srcreg+d2+48],[srcreg+d2+d1+48],screg1+32,screg1+64,screg1+96,dstreg+dstinc+e2,e1,[dstreg+e2],[dstreg+e2+32]
	xprefetch [srcreg+srcinc+d2+d1]
	bump	srcreg, srcinc
	shuffle_store_partial [dstreg+e2], [dstreg+e2+16], xmm2, xmm7			;; Save R1,R2
	shuffle_store_partial [dstreg+e2+32], [dstreg+e2+48], xmm5, xmm6		;; Save I1,I2
	shuffle_store_with_temp [dstreg+e2+e1], [dstreg+e2+e1+16], xmm3, xmm0, xmm7	;; Save R3,R4
	shuffle_store_with_temp [dstreg+e2+e1+32], [dstreg+e2+e1+48], xmm1, xmm4, xmm7	;; Save I3,I4
	bump	dstreg, dstinc
	ENDM

; The with-partial-normalization variant

r4_x4cl_wpn_eight_reals_four_complex_djbfft_preload MACRO
	r4_x8r_wpn_fft_mem_preload
	;r4_x4c_wpn_djbfft_partial_mem -- assumed to be a subset of r4_x8r_wpn_fft_mem_preload
	ENDM

r4_x4cl_wpn_eight_reals_four_complex_djbfft MACRO srcreg,srcinc,d1,d2,screg1,screg2
	r4_x8r_wpn_fft_mem [srcreg],[srcreg+d1],[srcreg+d2],[srcreg+d2+d1],[srcreg+16],[srcreg+d1+16],[srcreg+d2+16],[srcreg+d2+d1+16],screg2,screg1+32,screg2+48,screg1,srcreg+srcinc,d1,[srcreg],[srcreg+16]
;;	xstore	[srcreg], xmm0		;; Save real value #1
;;	xstore	[srcreg+16], xmm0	;; Save real value #2
	xload	xmm0, [srcreg+32]	;; R1
	xload	xmm1, [srcreg+48]	;; R5
	xstore	[srcreg+32], xmm7	;; Save R2
	xstore	[srcreg+48], xmm5	;; Save I2
	xload	xmm7, [srcreg+d1+32]	;; R2
	xload	xmm5, [srcreg+d1+48]	;; R6
	xstore	[srcreg+d1], xmm4	;; Save R3
	xstore	[srcreg+d1+16], xmm3	;; Save I3
	xstore	[srcreg+d1+32], xmm6	;; Save R4
	xstore	[srcreg+d1+48], xmm2	;; Save I4
	r4_x4c_wpn_djbfft_partial_mem xmm0,xmm7,xmm3,xmm2,xmm1,xmm5,xmm4,xmm6,[srcreg+d2+32],[srcreg+d2+d1+32],[srcreg+d2+48],[srcreg+d2+d1+48],screg1+32,screg1+80,screg1,srcreg+srcinc+d2,d1,[srcreg+d2],[srcreg+d2+16]
;;	xstore	[srcreg+d2], xmm0	;; Save real value #1
;;	xstore	[srcreg+d2+16], xmm0	;; Save real value #2
	xstore	[srcreg+d2+32], xmm6	;; Save R2
	xstore	[srcreg+d2+48], xmm4	;; Save I2
	xstore	[srcreg+d2+d1], xmm2	;; Save R3
	xstore	[srcreg+d2+d1+16], xmm7	;; Save I3
	xstore	[srcreg+d2+d1+32], xmm0	;; Save R4
	xstore	[srcreg+d2+d1+48], xmm1	;; Save I4
	bump	srcreg, srcinc
	ENDM

r4_x8r_wpn_fft_mem_preload MACRO
	ENDM

r4_x8r_wpn_fft_mem MACRO mem1,mem2,mem3,mem4,mem5,mem6,mem7,mem8,screg1,screg2,screg5,normreg,pre1,pre2,dst1,dst2
	xload	xmm3, mem4		;; R4
	xload	xmm7, mem8		;; R8
	xcopy	xmm0, xmm3
	subpd	xmm3, xmm7		;; new R8 = R4 - R8
	addpd	xmm7, xmm0		;; new R4 = R4 + R8

	xload	xmm1, mem2		;; R2
	xload	xmm5, mem6		;; R6
	xcopy	xmm4, xmm1
	subpd	xmm1, xmm5		;; new R6 = R2 - R6
	addpd	xmm5, xmm4		;; new R2 = R2 + R6

	mulpd	xmm3, XMM_SQRTHALF	;; R8 = R8 * square root
	mulpd	xmm1, XMM_SQRTHALF	;; R6 = R6 * square root

	xload	xmm0, mem1		;; R1
	xload	xmm4, mem5		;; R5
	xcopy	xmm2, xmm0
	subpd	xmm0, xmm4		;; new R5 = R1 - R5
	addpd	xmm4, xmm2		;; new R1 = R1 + R5

	xcopy	xmm2, xmm5		;; Copy R2
	subpd	xmm5, xmm7		;; R2 = R2 - R4 (final I2)
	addpd	xmm7, xmm2		;; R4 = R2 + R4 (final I1, a.k.a 2nd real result)

	xcopy	xmm6, xmm1		;; Copy R6
	subpd	xmm1, xmm3		;; R6 = R6 - R8 (Real part)
	addpd	xmm3, xmm6		;; R8 = R6 + R8 (Imaginary part)

	xload	xmm6, mem7		;; R7
	addpd	xmm6, mem3		;; new R3 = R3 + R7

	xcopy	xmm2, xmm4		;; Copy R1
	subpd	xmm4, xmm6		;; R1 = R1 - R3 (final R2)
	addpd	xmm6, xmm2		;; R3 = R1 + R3 (final R1)

	xload	xmm2, mem3		;; R3
	subpd	xmm2, mem7		;; new R7 = R3 - R7

	xprefetchw [pre1]

	mulpd	xmm6, [normreg]		;; R1 * normalization value
	xstore	dst1, xmm6		;; Save R1
	mulpd	xmm7, [normreg]		;; I1 * normalization value
	xstore	dst2, xmm7		;; Save I1

	xcopy	xmm7, xmm0		;; Copy R5
	subpd	xmm0, xmm1		;; R5 = R5 - R6 (final R4)
	addpd	xmm1, xmm7		;; R6 = R5 + R6 (final R3)

	xcopy	xmm6, xmm2		;; Copy R7
	subpd	xmm2, xmm3		;; R7 = R7 - R8 (final I4)
	addpd	xmm3, xmm6		;; R8 = R7 + R8 (final I3)

	xprefetchw [pre1][pre2]

	xload	xmm7, [screg2+16]	;; cosine/sine for w^2n
	mulpd	xmm7, xmm4		;; A2 = R2 * cosine/sine
	subpd	xmm7, xmm5		;; A2 = A2 - I2
	mulpd	xmm5, [screg2+16]	;; B2 = I2 * cosine/sine
	addpd	xmm5, xmm4		;; B2 = B2 + R2

	xload	xmm6, [screg5+16]	;; cosine/sine for w^5n
	mulpd	xmm6, xmm0		;; A4 = R4 * cosine/sine
	subpd	xmm6, xmm2		;; A4 = A4 - I4
	mulpd	xmm2, [screg5+16]	;; B4 = I4 * cosine/sine
	addpd	xmm2, xmm0		;; B4 = B4 + R4

	xload	xmm4, [screg1+16]	;; cosine/sine for w^n
	mulpd	xmm4, xmm1		;; A3 = R3 * cosine/sine
	subpd	xmm4, xmm3		;; A3 = A3 - I3
	mulpd	xmm3, [screg1+16]	;; B3 = I3 * cosine/sine
	addpd	xmm3, xmm1		;; B3 = B3 + R3

	xload	xmm0, [screg2]
	mulpd	xmm7, xmm0		;; A2 = A2 * sine (final R2)
	mulpd	xmm5, xmm0		;; B2 = B2 * sine (final I2)
	xload	xmm1, [screg5]
	mulpd	xmm6, xmm1		;; A4 = A4 * sine (final R4)
	mulpd	xmm2, xmm1		;; B4 = B4 * sine (final I4)
	xload	xmm0, [screg1]
	mulpd	xmm4, xmm0		;; A3 = A3 * sine (final R3)
	mulpd	xmm3, xmm0		;; B3 = B3 * sine (final I3)
	ENDM

; 32-bit AMD K8 version

IF (@INSTR(,%xarch,<K8>) NE 0)

r4_x8r_wpn_fft_mem MACRO mem1,mem2,mem3,mem4,mem5,mem6,mem7,mem8,screg1,screg2,screg5,normreg,pre1,pre2,dst1,dst2
	xload	xmm3, mem4		;; R4
	xload	xmm7, mem8		;; R8
	subpd	xmm3, xmm7		;; new R8 = R4 - R8
	addpd	xmm7, mem4		;; new R4 = R4 + R8

	xload	xmm1, mem2		;; R2
	xload	xmm5, mem6		;; R6
	subpd	xmm1, xmm5		;; new R6 = R2 - R6
	addpd	xmm5, mem2		;; new R2 = R2 + R6

	mulpd	xmm3, XMM_SQRTHALF	;; R8 = R8 * square root
	mulpd	xmm1, XMM_SQRTHALF	;; R6 = R6 * square root

	xload	xmm0, mem1		;; R1
	xload	xmm4, mem5		;; R5
	subpd	xmm0, xmm4		;; new R5 = R1 - R5
	addpd	xmm4, mem1		;; new R1 = R1 + R5

	subpd	xmm5, xmm7		;; R2 = R2 - R4 (final I2)
	multwo	xmm7
	addpd	xmm7, xmm5		;; R4 = R2 + R4 (final I1, a.k.a 2nd real result)

	subpd	xmm1, xmm3		;; R6 = R6 - R8 (Real part)
	multwo	xmm3
	addpd	xmm3, xmm1		;; R8 = R6 + R8 (Imaginary part)

	xload	xmm6, mem7		;; R7
	addpd	xmm6, mem3		;; new R3 = R3 + R7

	subpd	xmm4, xmm6		;; R1 = R1 - R3 (final R2)
	multwo	xmm6
	addpd	xmm6, xmm4		;; R3 = R1 + R3 (final R1)

	xload	xmm2, mem3		;; R3
	subpd	xmm2, mem7		;; new R7 = R3 - R7

	xprefetchw [pre1]

	mulpd	xmm6, [normreg]		;; R1 * normalization value
	xstore	dst1, xmm6		;; Save R1
	mulpd	xmm7, [normreg]		;; I1 * normalization value
	xstore	dst2, xmm7		;; Save I1

	subpd	xmm0, xmm1		;; R5 = R5 - R6 (final R4)
	multwo	xmm1
	addpd	xmm1, xmm0		;; R6 = R5 + R6 (final R3)

	subpd	xmm2, xmm3		;; R7 = R7 - R8 (final I4)
	multwo	xmm3
	addpd	xmm3, xmm2		;; R8 = R7 + R8 (final I3)

	xprefetchw [pre1][pre2]

	xload	xmm7, [screg2+16]	;; cosine/sine for w^2n
	mulpd	xmm7, xmm4		;; A2 = R2 * cosine/sine
	subpd	xmm7, xmm5		;; A2 = A2 - I2
	mulpd	xmm5, [screg2+16]	;; B2 = I2 * cosine/sine
	addpd	xmm5, xmm4		;; B2 = B2 + R2

	xload	xmm6, [screg5+16]	;; cosine/sine for w^5n
	mulpd	xmm6, xmm0		;; A4 = R4 * cosine/sine
	subpd	xmm6, xmm2		;; A4 = A4 - I4
	mulpd	xmm2, [screg5+16]	;; B4 = I4 * cosine/sine
	addpd	xmm2, xmm0		;; B4 = B4 + R4

	xload	xmm4, [screg1+16]	;; cosine/sine for w^n
	mulpd	xmm4, xmm1		;; A3 = R3 * cosine/sine
	subpd	xmm4, xmm3		;; A3 = A3 - I3
	mulpd	xmm3, [screg1+16]	;; B3 = I3 * cosine/sine
	addpd	xmm3, xmm1		;; B3 = B3 + R3

	xload	xmm0, [screg2]
	mulpd	xmm7, xmm0		;; A2 = A2 * sine (final R2)
	mulpd	xmm5, xmm0		;; B2 = B2 * sine (final I2)
	xload	xmm1, [screg5]
	mulpd	xmm6, xmm1		;; A4 = A4 * sine (final R4)
	mulpd	xmm2, xmm1		;; B4 = B4 * sine (final I4)
	xload	xmm0, [screg1]
	mulpd	xmm4, xmm0		;; A3 = A3 * sine (final R3)
	mulpd	xmm3, xmm0		;; B3 = B3 * sine (final I3)
	ENDM

ENDIF

;; 64-bit Intel version.

IF (@INSTR(,%xarch,<CORE>) NE 0) OR (@INSTR(,%xarch,<P4>) NE 0) OR (@INSTR(,%xarch,<K10>) NE 0) OR (@INSTR(,%xarch,<BLEND>) NE 0)
IFDEF X86_64

; Optimal Core 2 timing is 24 clocks, currently at ?? clocks

r4_x8r_wpn_fft_mem_preload MACRO
	xload	xmm15, XMM_SQRTHALF
	ENDM

r4_x8r_wpn_fft_mem MACRO mem1,mem2,mem3,mem4,mem5,mem6,mem7,mem8,screg1,screg2,screg5,normreg,pre1,pre2,dst1,dst2
	xload	xmm14, mem4		;; R4
	xload	xmm11, mem8		;; R8
	xcopy	xmm0, xmm14		;; Copy R4
	subpd	xmm14, xmm11		;; new R8 = R4 - R8			; 1-3

	xload	xmm4, mem2		;; R2
	xload	xmm5, mem6		;; R6
	xcopy	xmm1, xmm4		;; Copy R2
	subpd	xmm4, xmm5		;; new R6 = R2 - R6			; 2-4

	addpd	xmm11, xmm0		;; new R4 = R4 + R8			; 3-5

	addpd	xmm5, xmm1		;; new R2 = R2 + R6			; 4-6
	mulpd	xmm14, xmm15		;; R8 = R8 * square root 1/2		; 4-8

	xload	xmm0, mem1		;; R1
	xload	xmm1, mem5		;; R5
	xcopy	xmm6, xmm0		;; Copy R1
	addpd	xmm0, xmm1		;; new R1 = R1 + R5			; 5-7
	mulpd	xmm4, xmm15		;; R6 = R6 * square root 1/2		; 5-9

	xload	xmm13, mem3		;; R3
	xload	xmm8, mem7		;; R7
	xcopy	xmm9, xmm13		;; Copy R3
	addpd	xmm13, xmm8 		;; new R3 = R3 + R7			; 6-8

	subpd	xmm6, xmm1		;; new R5 = R1 - R5			; 7-9

	xcopy	xmm1, xmm5		;; Copy R2
	subpd	xmm5, xmm11		;; R2 = R2 - R4 (final I2)		; 8-10

	xcopy	xmm10, xmm0		;; Copy R1
	subpd	xmm0, xmm13		;; R1 = R1 - R3 (final R2)		; 9-11

	xcopy	xmm7, xmm4		;; Copy R6
	subpd	xmm4, xmm14		;; R6 = R6 - R8 (Real part)		; 10-12

	addpd	xmm14, xmm7		;; R8 = R6 + R8 (Imaginary part)	; 11-13
	xload	xmm7, [screg2+16]	;; cosine/sine for w^2n

	subpd	xmm9, xmm8		;; new R7 = R3 - R7			; 12-14
	xload	xmm2, [screg5+16]	;; cosine/sine for w^5n

	xcopy	xmm8, xmm6		;; Copy R5
	subpd	xmm6, xmm4		;; R5 = R5 - R6 (final R4)		; 13-15
	xload	xmm3, [screg1+16]	;; cosine/sine for w^n

	xprefetchw [pre1]

	addpd	xmm4, xmm8		;; R6 = R5 + R6 (final R3)		; 14-16
	xcopy	xmm12, xmm5		;; Copy I2
	mulpd	xmm5, xmm7		;; B2 = I2 * cosine/sine		; 14-18

	xcopy	xmm8, xmm9		;; Copy R7
	subpd	xmm9, xmm14		;; R7 = R7 - R8 (final I4)		; 15-17
	mulpd	xmm7, xmm0		;; A2 = R2 * cosine/sine		; 15-19

	addpd	xmm14, xmm8		;; R8 = R7 + R8 (final I3)		; 16-18
	xcopy	xmm8, xmm6		;; Copy R4
	mulpd	xmm6, xmm2		;; A4 = R4 * cosine/sine		; 16-20

	addpd	xmm11, xmm1		;; R4 = R2 + R4 (final I1, a.k.a 2nd real result) ; 17-19
	xcopy	xmm1, xmm4		;; Copy R3
	mulpd	xmm4, xmm3		;; A3 = R3 * cosine/sine		; 17-21

	addpd	xmm13, xmm10		;; R3 = R1 + R3 (final R1)		; 18-20
	mulpd	xmm2, xmm9		;; B4 = I4 * cosine/sine		; 18-22
	xload	xmm10, [screg2]		;; Sine * normalization value

	addpd	xmm5, xmm0		;; B2 = B2 + R2				; 19-21
	mulpd	xmm3, xmm14		;; B3 = I3 * cosine/sine		; 19-23
	xload	xmm0, [screg5]		;; Sine * normalization value

	xprefetchw [pre1][pre2]

	subpd	xmm7, xmm12		;; A2 = A2 - I2				; 20-22
	mulpd	xmm11, [normreg]	;; I1 * normalization value		; 20-24
	xload	xmm12, [screg1]		;; Sine * normalization value

	subpd	xmm6, xmm9		;; A4 = A4 - I4				; 21-23
	mulpd	xmm13, [normreg]	;; R1 * normalization value		; 21-25

	subpd	xmm4, xmm14		;; A3 = A3 - I3				; 22-24
	mulpd	xmm5, xmm10		;; B2 = B2 * sine (final I2)		; 22-26

	addpd	xmm2, xmm8		;; B4 = B4 + R4				; 23-25
	mulpd	xmm7, xmm10		;; A2 = A2 * sine (final R2)		; 23-27

	addpd	xmm3, xmm1		;; B3 = B3 + R3				; 24-26
	mulpd	xmm6, xmm0		;; A4 = A4 * sine (final R4)		; 24-28

	mulpd	xmm4, xmm12		;; A3 = A3 * sine (final R3)		; 25-29
	xstore	dst2, xmm11		;; Save I1				; 25

	mulpd	xmm2, xmm0		;; B4 = B4 * sine (final I4)		; 26-30
	xstore	dst1, xmm13		;; Save R1				; 26

	mulpd	xmm3, xmm12		;; B3 = B3 * sine (final I3)		; 27-31
	ENDM

ENDIF
ENDIF

; 64-bit AMD K8 optimized version.  This is the 32-bit K8 version, but with XMM_TWO and
; XMM_SQRTHALF preloaded.

IF (@INSTR(,%xarch,<K8>) NE 0)
IFDEF X86_64

;; WARNING:  this preload must be compatible with r4_x4c_wpn_djbfft_partial_mem
r4_x8r_wpn_fft_mem_preload MACRO
	xload	xmm14, XMM_SQRTHALF
	xload	xmm15, XMM_TWO
	ENDM

r4_x8r_wpn_fft_mem MACRO mem1,mem2,mem3,mem4,mem5,mem6,mem7,mem8,screg1,screg2,screg5,normreg,pre1,pre2,dst1,dst2
	xload	xmm3, mem4		;; R4
	xload	xmm7, mem8		;; R8
	subpd	xmm3, xmm7		;; new R8 = R4 - R8
	addpd	xmm7, mem4		;; new R4 = R4 + R8

	xload	xmm1, mem2		;; R2
	xload	xmm5, mem6		;; R6
	subpd	xmm1, xmm5		;; new R6 = R2 - R6
	addpd	xmm5, mem2		;; new R2 = R2 + R6

	mulpd	xmm3, xmm14		;; R8 = R8 * square root
	mulpd	xmm1, xmm14		;; R6 = R6 * square root

	xload	xmm0, mem1		;; R1
	xload	xmm4, mem5		;; R5
	subpd	xmm0, xmm4		;; new R5 = R1 - R5
	addpd	xmm4, mem1		;; new R1 = R1 + R5

	subpd	xmm5, xmm7		;; R2 = R2 - R4 (final I2)
	mulpd	xmm7, xmm15
	addpd	xmm7, xmm5		;; R4 = R2 + R4 (final I1, a.k.a 2nd real result)

	subpd	xmm1, xmm3		;; R6 = R6 - R8 (Real part)
	mulpd	xmm3, xmm15
	addpd	xmm3, xmm1		;; R8 = R6 + R8 (Imaginary part)

	xload	xmm6, mem7		;; R7
	addpd	xmm6, mem3		;; new R3 = R3 + R7

	subpd	xmm4, xmm6		;; R1 = R1 - R3 (final R2)
	mulpd	xmm6, xmm15
	addpd	xmm6, xmm4		;; R3 = R1 + R3 (final R1)

	xload	xmm2, mem3		;; R3
	subpd	xmm2, mem7		;; new R7 = R3 - R7

	xprefetchw [pre1]

	mulpd	xmm6, [normreg]		;; R1 * normalization value
	xstore	dst1, xmm6		;; Save R1
	mulpd	xmm7, [normreg]		;; I1 * normalization value
	xstore	dst2, xmm7		;; Save I1

	subpd	xmm0, xmm1		;; R5 = R5 - R6 (final R4)
	mulpd	xmm1, xmm15
	addpd	xmm1, xmm0		;; R6 = R5 + R6 (final R3)

	subpd	xmm2, xmm3		;; R7 = R7 - R8 (final I4)
	mulpd	xmm3, xmm15
	addpd	xmm3, xmm2		;; R8 = R7 + R8 (final I3)

	xprefetchw [pre1][pre2]

	xload	xmm7, [screg2+16]	;; cosine/sine for w^2n
	mulpd	xmm7, xmm4		;; A2 = R2 * cosine/sine
	subpd	xmm7, xmm5		;; A2 = A2 - I2
	mulpd	xmm5, [screg2+16]	;; B2 = I2 * cosine/sine
	addpd	xmm5, xmm4		;; B2 = B2 + R2

	xload	xmm6, [screg5+16]	;; cosine/sine for w^5n
	mulpd	xmm6, xmm0		;; A4 = R4 * cosine/sine
	subpd	xmm6, xmm2		;; A4 = A4 - I4
	mulpd	xmm2, [screg5+16]	;; B4 = I4 * cosine/sine
	addpd	xmm2, xmm0		;; B4 = B4 + R4

	xload	xmm4, [screg1+16]	;; cosine/sine for w^n
	mulpd	xmm4, xmm1		;; A3 = R3 * cosine/sine
	subpd	xmm4, xmm3		;; A3 = A3 - I3
	mulpd	xmm3, [screg1+16]	;; B3 = I3 * cosine/sine
	addpd	xmm3, xmm1		;; B3 = B3 + R3

	xload	xmm0, [screg2]
	mulpd	xmm7, xmm0		;; A2 = A2 * sine (final R2)
	mulpd	xmm5, xmm0		;; B2 = B2 * sine (final I2)
	xload	xmm1, [screg5]
	mulpd	xmm6, xmm1		;; A4 = A4 * sine (final R4)
	mulpd	xmm2, xmm1		;; B4 = B4 * sine (final I4)
	xload	xmm0, [screg1]
	mulpd	xmm4, xmm0		;; A3 = A3 * sine (final R3)
	mulpd	xmm3, xmm0		;; B3 = B3 * sine (final I3)
	ENDM

ENDIF
ENDIF

;;
;; ************************************* half-eight-reals-four-complex-djbfft variants ******************************************
;;
;; Because of the way we store FFT data elements in cache lines during the pass 2 of the forward FFT, a cache line
;; will contain data needing an eight-reals-fft in the low word only as well as data needing a four-complex-ffts.
;;

;; Macro to swizzle and do an eight_reals_fft and a four_complex_djbfft in first levels of pass 2.
;; The eight-reals macro and one of the four-complex only use half the XMM 
;; register.  This isn't very efficient, but this macro isn't called a whole lot.

;; This is used in the first levels of pass 2 if pass 2 does the swizzling
IFDEF UNUSED
r4_s2cl_eight_reals_four_complex_djbfft MACRO srcreg,srcinc,d1,screg1,screg2
	shuffle_load xmm0, xmm2, [srcreg][rbx], [srcreg+32][rbx] ;; R1,R3
	shuffle_load xmm1, xmm3, [srcreg+d1][rbx], [srcreg+d1+32][rbx] ;; R2,R4
	shuffle_load xmm4, xmm6, [srcreg+16][rbx], [srcreg+48][rbx] ;; R5,R7
	shuffle_load xmm5, xmm7, [srcreg+d1+16][rbx], [srcreg+d1+48][rbx] ;; R6,R8
	xstore	[srcreg], xmm0
	xstore	[srcreg+16], xmm1
	xstore	[srcreg+32], xmm2
	xstore	[srcreg+48], xmm3
	xstore	[srcreg+d1], xmm4
	xstore	[srcreg+d1+16], xmm5
	xstore	[srcreg+d1+32], xmm6
	xstore	[srcreg+d1+48], xmm7
	r4_h8r_h4c_djbfft_mem [srcreg],[srcreg+16],[srcreg+32],[srcreg+48],[srcreg+d1],[srcreg+d1+16],[srcreg+d1+32],[srcreg+d1+48],screg1,screg2,srcreg+srcinc,d1,[srcreg],[srcreg+16]
;;	xstore	[srcreg], xmm0		;; Save real value #1
;;	xstore	[srcreg+16], xmm0	;; Save real value #2
	xstore	[srcreg+32], xmm7	;; Save R2
	xstore	[srcreg+48], xmm6	;; Save I2
	xstore	[srcreg+d1], xmm3	;; Save R3
	xstore	[srcreg+d1+16], xmm1	;; Save I3
	xstore	[srcreg+d1+32], xmm0	;; Save R4
	xstore	[srcreg+d1+48], xmm4	;; Save I4
	bump	srcreg, srcinc
	ENDM
ENDIF

;; This is used in the first levels of pass 2 if pass 1 does the swizzling
r4_fh2cl_eight_reals_four_complex_djbfft MACRO srcreg,srcinc,d1,screg1,screg2
	r4_h8r_h4c_djbfft_mem [srcreg][rbx],[srcreg+d1][rbx],[srcreg+16][rbx],[srcreg+d1+16][rbx],[srcreg+32][rbx],[srcreg+d1+32][rbx],[srcreg+48][rbx],[srcreg+d1+48][rbx],screg1,screg2,srcreg+srcinc,d1,[srcreg],[srcreg+16]
;;	xstore	[srcreg], xmm0		;; Save real value #1
;;	xstore	[srcreg+16], xmm0	;; Save real value #2
	xstore	[srcreg+32], xmm7	;; Save R2
	xstore	[srcreg+48], xmm6	;; Save I2
	xstore	[srcreg+d1], xmm3	;; Save R3
	xstore	[srcreg+d1+16], xmm1	;; Save I3
	xstore	[srcreg+d1+32], xmm0	;; Save R4
	xstore	[srcreg+d1+48], xmm4	;; Save I4
	bump	srcreg, srcinc
	ENDM

;; This is used in the later levels of pass 2 if first levels used a radix-3 or radix-5.
r4_h2cl_eight_reals_four_complex_djbfft MACRO srcreg,srcinc,d1,screg1,screg2
	r4_h8r_h4c_djbfft_mem [srcreg],[srcreg+d1],[srcreg+16],[srcreg+d1+16],[srcreg+32],[srcreg+d1+32],[srcreg+48],[srcreg+d1+48],screg1,screg2,srcreg+srcinc,d1,[srcreg],[srcreg+16]
;;	xstore	[srcreg], xmm0		;; Save real value #1
;;	xstore	[srcreg+16], xmm0	;; Save real value #2
	xstore	[srcreg+32], xmm7	;; Save R2
	xstore	[srcreg+48], xmm6	;; Save I2
	xstore	[srcreg+d1], xmm3	;; Save R3
	xstore	[srcreg+d1+16], xmm1	;; Save I3
	xstore	[srcreg+d1+32], xmm0	;; Save R4
	xstore	[srcreg+d1+48], xmm4	;; Save I4
	bump	srcreg, srcinc
	ENDM

;; Like r4_h2cl_eight_reals_four_complex_djbfft except it uses 2 sin/cos ptrs for the four-complex djbfft.
;; This is used in the later levels of pass 2 if first levels used a radix-3.
r4_h2cl_2sc_eight_reals_four_complex_djbfft MACRO srcreg,srcinc,d1,screg1,screg2,screg3
	r4_h8r_h4c_2sc_djbfft_mem [srcreg],[srcreg+d1],[srcreg+16],[srcreg+d1+16],[srcreg+32],[srcreg+d1+32],[srcreg+48],[srcreg+d1+48],screg1,screg2,screg3,srcreg+srcinc,d1,[srcreg],[srcreg+16]
;;	xstore	[srcreg], xmm0		;; Save real value #1
;;	xstore	[srcreg+16], xmm0	;; Save real value #2
	xstore	[srcreg+32], xmm7	;; Save R2
	xstore	[srcreg+48], xmm6	;; Save I2
	xstore	[srcreg+d1], xmm3	;; Save R3
	xstore	[srcreg+d1+16], xmm1	;; Save I3
	xstore	[srcreg+d1+32], xmm0	;; Save R4
	xstore	[srcreg+d1+48], xmm4	;; Save I4
	bump	srcreg, srcinc
	ENDM

;; Macro to do an eight_reals_fft and three four_complex_djbfft in pass 2.
;; The eight-reals macro and one of the four-complex only use half the XMM 
;; register.  This isn't very efficient, but this macro isn't called a whole lot.

r4_h4cl_eight_reals_four_complex_djbfft_preload MACRO
	r4_x4c_djbfft_partial_mem_preload
	ENDM

r4_h4cl_eight_reals_four_complex_djbfft MACRO srcreg,srcinc,d1,d2,screg1,screg2
	r4_h8r_h4c_djbfft_mem [srcreg],[srcreg+d1],[srcreg+d2],[srcreg+d2+d1],[srcreg+16],[srcreg+d1+16],[srcreg+d2+16],[srcreg+d2+d1+16],screg1,screg2,srcreg+srcinc,d1,[srcreg],[srcreg+16]
;;	xstore	[srcreg], xmm0		;; Save real value #1
;;	xstore	[srcreg+16], xmm0	;; Save real value #2
	xstore	[srcreg+d1], xmm3	;; Save R3
	xstore	[srcreg+d1+16], xmm1	;; Save I3
	xload	xmm1, [srcreg+32]	;; R1
	xload	xmm3, [srcreg+48]	;; R5
	xstore	[srcreg+32], xmm7	;; Save R2
	xstore	[srcreg+48], xmm6	;; Save I2
	xload	xmm7, [srcreg+d1+32]	;; R2
	xload	xmm6, [srcreg+d1+48]	;; R6
	xstore	[srcreg+d1+32], xmm0	;; Save R4
	xstore	[srcreg+d1+48], xmm4	;; Save I4
	r4_x4c_djbfft_partial_mem xmm1,xmm7,xmm0,xmm2,xmm3,xmm6,xmm4,xmm5,[srcreg+d2+32],[srcreg+d2+d1+32],[srcreg+d2+48],[srcreg+d2+d1+48],screg1,0,srcreg+srcinc+d2,d1,[srcreg+d2],[srcreg+d2+16]
;;	xstore	[srcreg+d2], xmm0	;; Save R1
;;	xstore	[srcreg+d2+16], xmm0	;; Save I1
	xstore	[srcreg+d2+32], xmm5	;; Save R2
	xstore	[srcreg+d2+48], xmm4	;; Save I2
	xstore	[srcreg+d2+d1], xmm2	;; Save R3
	xstore	[srcreg+d2+d1+16], xmm7	;; Save I3
	xstore	[srcreg+d2+d1+32], xmm1	;; Save R4
	xstore	[srcreg+d2+d1+48], xmm3	;; Save I4
	bump	srcreg, srcinc
	ENDM

;; Like r4_h4cl_eight_reals_four_complex_djbfft but takes 2 sin/cos ptrs for the 4 complex djbfft.
;; Used in pass 2 when the first levels are a three-complex FFT.

r4_h4cl_2sc_eight_reals_four_complex_djbfft_preload MACRO
	r4_x4c_2sc_djbfft_partial_mem_preload
	ENDM

r4_h4cl_2sc_eight_reals_four_complex_djbfft MACRO srcreg,srcinc,d1,d2,screg1,screg2,screg3
	r4_h8r_h4c_2sc_djbfft_mem [srcreg],[srcreg+d1],[srcreg+d2],[srcreg+d2+d1],[srcreg+16],[srcreg+d1+16],[srcreg+d2+16],[srcreg+d2+d1+16],screg1,screg2,screg3,srcreg+srcinc,d1,[srcreg],[srcreg+16]
;;	xstore	[srcreg], xmm0		;; Save real value #1
;;	xstore	[srcreg+16], xmm0	;; Save real value #2
	xstore	[srcreg+d1], xmm3	;; Save R3
	xstore	[srcreg+d1+16], xmm1	;; Save I3
	xload	xmm1, [srcreg+32]	;; R1
	xload	xmm3, [srcreg+48]	;; R5
	xstore	[srcreg+32], xmm7	;; Save R2
	xstore	[srcreg+48], xmm6	;; Save I2
	xload	xmm7, [srcreg+d1+32]	;; R2
	xload	xmm6, [srcreg+d1+48]	;; R6
	xstore	[srcreg+d1+32], xmm0	;; Save R4
	xstore	[srcreg+d1+48], xmm4	;; Save I4
	r4_x4c_2sc_djbfft_partial_mem xmm1,xmm7,xmm0,xmm2,xmm3,xmm6,xmm4,xmm5,[srcreg+d2+32],[srcreg+d2+d1+32],[srcreg+d2+48],[srcreg+d2+d1+48],screg1,screg2,srcreg+srcinc+d2,d1,[srcreg+d2],[srcreg+d2+16]
;;	xstore	[srcreg+d2], xmm0	;; Save R1
;;	xstore	[srcreg+d2+16], xmm0	;; Save I1
	xstore	[srcreg+d2+32], xmm5	;; Save R2
	xstore	[srcreg+d2+48], xmm4	;; Save I2
	xstore	[srcreg+d2+d1], xmm2	;; Save R3
	xstore	[srcreg+d2+d1+16], xmm7	;; Save I3
	xstore	[srcreg+d2+d1+32], xmm1	;; Save R4
	xstore	[srcreg+d2+d1+48], xmm3	;; Save I4
	bump	srcreg, srcinc
	ENDM


;; Does an r4_x8r_fft_mem on the low word of the xmm register
;; Does an r4_x4c_djbfft_mem on the high word of the xmm register
r4_h8r_h4c_djbfft_mem MACRO mem1,mem2,mem3,mem4,mem5,mem6,mem7,mem8,screg1,screg2,pre1,pre2,dst1,dst2
	r4_h8r_h4c_2sc_djbfft_mem mem1,mem2,mem3,mem4,mem5,mem6,mem7,mem8,screg1,screg1+32,screg2,pre1,pre2,dst1,dst2
	ENDM
r4_h8r_h4c_2sc_djbfft_mem MACRO mem1,mem2,mem3,mem4,mem5,mem6,mem7,mem8,screg1,screg2,screg3,pre1,pre2,dst1,dst2

	;; Do the four-complex part

	movsd	xmm0, Q mem1[8]		;; R1
	movsd	xmm2, Q mem3[8]		;; R3
	addsd	xmm2, xmm0		;; R3 = R1 + R3 (new R1)
	subsd	xmm0, Q mem3[8]		;; R1 = R1 - R3 (new R3)

	movsd	xmm1, Q mem2[8]		;; R2
	movsd	xmm3, Q mem4[8]		;; R4
	addsd	xmm3, xmm1		;; R4 = R2 + R4 (new R2)
	subsd	xmm1, Q mem4[8]		;; R2 = R2 - R4 (new R4)

	movsd	xmm6, xmm2
	subsd	xmm2, xmm3		;; R1 = R1 - R2 (final R2)
	addsd	xmm3, xmm6		;; R2 = R1 + R2 (final R1)

	movsd	Q dst1[8], xmm3		;; Save R1

	movsd	xmm5, Q mem6[8]		;; I2
	movsd	xmm7, Q mem8[8]		;; I4
	subsd	xmm5, xmm7		;; I2 = I2 - I4 (new I4)
	addsd	xmm7, Q mem6[8]		;; I4 = I2 + I4 (new I2)

	movsd	xmm4, Q mem5[8]		;; I1
	movsd	xmm6, Q mem7[8]		;; I3
	subsd	xmm4, xmm6		;; I1 = I1 - I3 (new I3)
	addsd	xmm6, Q mem5[8]		;; I3 = I1 + I3 (new I1)

	movsd	xmm3, xmm0
	subsd	xmm0, xmm5		;; R3 = R3 - I4 (final R3)
	addsd	xmm5, xmm3		;; I4 = R3 + I4 (final R4)

	movsd	xmm3, xmm1
	addsd	xmm1, xmm4		;; R4 = I3 + R4 (final I3)
	subsd	xmm4, xmm3		;; I3 = I3 - R4 (final I4)

	movsd	xmm3, xmm6
	subsd	xmm6, xmm7		;; I1 = I1 - I2 (final I2)
	addsd	xmm7, xmm3		;; I2 = I1 + I2 (final I1)

	movsd	xmm3, Q [screg1+16]	;; cosine/sine
	mulsd	xmm3, xmm0		;; A3 = R3 * cosine/sine

	movsd	Q dst2[8], xmm7		;; Save I1

	movsd	xmm7, Q [screg2+16]	;; cosine/sine
	mulsd	xmm7, xmm2		;; A2 = R2 * cosine/sine

	subsd	xmm3, xmm1		;; A3 = A3 - I3
	mulsd	xmm1, Q [screg1+16]	;; B3 = I3 * cosine/sine
	addsd	xmm1, xmm0		;; B3 = B3 + R3

	movsd	xmm0, Q [screg1+16]	;; cosine/sine
	mulsd	xmm0, xmm5		;; A4 = R4 * cosine/sine

	subsd	xmm7, xmm6		;; A2 = A2 - I2
	mulsd	xmm6, Q [screg2+16]	;; B2 = I2 * cosine/sine

	addsd	xmm0, xmm4		;; A4 = A4 + I4
	mulsd	xmm4, Q [screg1+16]	;; B4 = I4 * cosine/sine

	addsd	xmm6, xmm2		;; B2 = B2 + R2
	subsd	xmm4, xmm5		;; B4 = B4 - R4

	movsd	xmm2, Q [screg1]	;; sine
	mulsd	xmm3, xmm2		;; A3 = A3 * sine (final R3)
	mulsd	xmm1, xmm2		;; B3 = B3 * sine (final I3)
	mulsd	xmm7, Q [screg2]	;; A2 = A2 * sine (final R2)
	mulsd	xmm0, xmm2		;; A4 = A4 * sine (final R4)
	mulsd	xmm6, Q [screg2]	;; B2 = B2 * sine (final I2)
	mulsd	xmm4, xmm2		;; B4 = B4 * sine (final I4)

	;; Do the eight reals part

	unpcklo xmm3, xmm3		;; Copy to high part of XMM register
	unpcklo xmm1, xmm1
	unpcklo xmm7, xmm7
	unpcklo xmm0, xmm0
	unpcklo xmm6, xmm6
	unpcklo xmm4, xmm4

	movlpd	xmm1, Q mem4		;; R4
	movlpd	xmm7, Q mem8		;; R8
	subsd	xmm1, xmm7		;; new R8 = R4 - R8
	addsd	xmm7, Q mem4		;; new R4 = R4 + R8
	movsd	xmm5, Q mem2		;; R2
	movlpd	xmm6, Q mem6		;; R6
	subsd	xmm5, xmm6		;; new R6 = R2 - R6
	addsd	xmm6, Q mem2		;; new R2 = R2 + R6

	mulsd	xmm1, Q XMM_SQRTHALF	;; R8 = R8 * square root
	mulsd	xmm5, Q XMM_SQRTHALF	;; R6 = R6 * square root

	movsd	xmm2, Q mem1		;; R1
	movlpd	xmm3, Q mem5		;; R5
	subsd	xmm2, xmm3		;; new R5 = R1 - R5
	addsd	xmm3, Q mem1		;; new R1 = R1 + R5

	movsd	xmm4, xmm6		;; Copy R2
	subsd	xmm6, xmm7		;; R2 = R2 - R4 (final I2)
	addsd	xmm7, xmm4		;; R4 = R2 + R4 (final I1, a.k.a 2nd real result)

	movsd	xmm0, xmm5		;; Copy R6
	subsd	xmm5, xmm1		;; R6 = R6 - R8 (Real part)
	addsd	xmm1, xmm0		;; R8 = R6 + R8 (Imaginary part)

	movlpd	xmm0, Q mem7		;; R7
	addsd	xmm0, Q mem3		;; new R3 = R3 + R7

	movsd	xmm4, xmm3		;; Copy R1
	subsd	xmm3, xmm0		;; R1 = R1 - R3 (final R2)
	addsd	xmm0, xmm4		;; R3 = R1 + R3 (final R1)

	movlpd	xmm4, Q mem3		;; R3
	subsd	xmm4, Q mem7		;; new R7 = R3 - R7

	xprefetchw [pre1]

	movsd	Q dst1, xmm0		;; Save R1
	movsd	Q dst2, xmm7		;; Save I1

	movsd	xmm7, xmm2		;; Copy R5
	subsd	xmm2, xmm5		;; R5 = R5 - R6 (final R4)
	addsd	xmm5, xmm7		;; R6 = R5 + R6 (final R3)

	movsd	xmm0, xmm4		;; Copy R7
	subsd	xmm4, xmm1		;; R7 = R7 - R8 (final I4)
	addsd	xmm1, xmm0		;; R8 = R7 + R8 (final I3)

	xprefetchw [pre1][pre2]

	movlpd	xmm7, Q [screg1+16]	;; cosine/sine for w^2n
	mulsd	xmm7, xmm3		;; A2 = R2 * cosine/sine
	subsd	xmm7, xmm6		;; A2 = A2 - I2
	mulsd	xmm6, Q [screg1+16]	;; B2 = I2 * cosine/sine
	addsd	xmm6, xmm3		;; B2 = B2 + R2

	movlpd	xmm0, Q [screg3+24]	;; cosine/sine for w^5n
	mulsd	xmm0, xmm2		;; A4 = R4 * cosine/sine
	subsd	xmm0, xmm4		;; A4 = A4 - I4
	mulsd	xmm4, Q [screg3+24]	;; B4 = I4 * cosine/sine
	addsd	xmm4, xmm2		;; B4 = B4 + R4

	movlpd	xmm3, Q [screg3+8]	;; cosine/sine for w^n
	mulsd	xmm3, xmm5		;; A3 = R3 * cosine/sine
	subsd	xmm3, xmm1		;; A3 = A3 - I3
	mulsd	xmm1, Q [screg3+8]	;; B3 = I3 * cosine/sine
	addsd	xmm1, xmm5		;; B3 = B3 + R3

	mulsd	xmm7, Q [screg1]	;; A2 = A2 * sine (final R2)
	mulsd	xmm6, Q [screg1]	;; B2 = B2 * sine (final I2)
	mulsd	xmm0, Q [screg3+16]	;; A4 = A4 * sine (final R4)
	mulsd	xmm4, Q [screg3+16]	;; B4 = B4 * sine (final I4)
	mulsd	xmm3, Q [screg3]	;; A3 = A3 * sine (final R3)
	mulsd	xmm1, Q [screg3]	;; B3 = B3 * sine (final I3)
	ENDM

;;
;; ************************************* half-eight-reals-four-complex-djbunfft variants ******************************************
;;

;; Macro to do an eight_reals_unfft and a four_complex_djbunfft 
;; and swizzle the results in the last levels of pass 2.
;; The eight-reals operation is done in the lower half of the XMM
;; register.  This isn't very efficient, but this macro isn't called a whole lot.

IFDEF UNUSED
r4_s4cl_eight_reals_four_complex_djbunfft MACRO srcreg,srcinc,d1,d2,screg1,scoff1,screg2,scoff2
	d3 = d2 + d1
	r4_h8r_h4c_djbunfft_mem [srcreg+16],[srcreg+48],[srcreg+d1+16],[srcreg+d1+48],[srcreg+d2+16],[srcreg+d2+48],[srcreg+d3+16],[srcreg+d3+48],screg1,scoff1,screg2,scoff2,srcreg+srcinc+d2,d1,XMM_TMP1,XMM_TMP2
	xload	xmm2, XMM_TMP1		;; Load R4
	xload	xmm5, XMM_TMP2		;; Load R8
	shuffle_store [srcreg+d1+16], [srcreg+d1+48], xmm0, xmm6 ;; Save R5, R7
	shuffle_store [srcreg+d3+16], [srcreg+d3+48], xmm3, xmm5 ;; Save R6, R8
	xload	xmm0, [srcreg+d1]	;; Load R2
	xload	xmm6, [srcreg+d1+32]	;; Load I2
	xload	xmm3, [srcreg+d3]	;; Load R4
	xload	xmm5, [srcreg+d3+32]	;; Load I4
	shuffle_store [srcreg+d1], [srcreg+d1+32], xmm1, xmm7 ;; Save R1, R3
	shuffle_store [srcreg+d3], [srcreg+d3+32], xmm4, xmm2 ;; Save R2, R4
	xstore XMM_TMP1, xmm0
	xstore XMM_TMP2, xmm6
	xstore XMM_TMP3, xmm3
	xstore XMM_TMP4, xmm5
	r4_h8r_h4c_djbunfft_mem [srcreg],[srcreg+32],XMM_TMP1,XMM_TMP2,[srcreg+d2],[srcreg+d2+32],XMM_TMP3,XMM_TMP4,screg1,0,screg2,0,srcreg+srcinc,d1,XMM_TMP1,XMM_TMP2
	xload	xmm2, XMM_TMP1		;; Load R4
	xload	xmm5, XMM_TMP2		;; Load R8
	shuffle_store [srcreg], [srcreg+32], xmm1, xmm7 ;; Save R1, R3
	shuffle_store [srcreg+16], [srcreg+48], xmm0, xmm6 ;; Save R5, R7
	shuffle_store [srcreg+d2], [srcreg+d2+32], xmm4, xmm2 ;; Save R2, R4
	shuffle_store [srcreg+d2+16], [srcreg+d2+48], xmm3, xmm5 ;; Save R6, R8
	bump	srcreg, srcinc
	ENDM
ENDIF

;; Macro to do an eight_reals_unfft and a four_complex_djbunfft in pass 2.
;; The eight-reals operation is done in the lower half of the XMM
;; register.  This isn't very efficient, but this macro isn't called a whole lot.

r4_h4cl_eight_reals_four_complex_djbunfft MACRO srcreg,srcinc,d1,d2,screg1,scoff1,screg2,scoff2
	d3 = d2 + d1
	tmp = srcinc+d2
	r4_h8r_h4c_djbunfft_mem [srcreg+16],[srcreg+48],[srcreg+d1+16],[srcreg+d1+48],[srcreg+d2+16],[srcreg+d2+48],[srcreg+d3+16],[srcreg+d3+48],screg1,scoff1,screg2,scoff2,srcreg+tmp,d1,[srcreg+d3+16],[srcreg+d3+48]
	xstore	[srcreg+d1+16], xmm7	;; Save R3
;;	xstore	[srcreg+d3+16], xmm5	;; Save R4
	xstore	[srcreg+d1+48], xmm6	;; Save R7
;;	xstore	[srcreg+d3+48], xmm2	;; Save R8
	xload	xmm2, [srcreg+d1]	;; Load R2
	xload	xmm5, [srcreg+d1+32]	;; Load I2
	xload	xmm6, [srcreg+d3]	;; Load R4
	xload	xmm7, [srcreg+d3+32]	;; Load I4
	xstore	[srcreg+d1], xmm1	;; Save R1
	xstore	[srcreg+d3], xmm4	;; Save R2
	xstore	[srcreg+d1+32], xmm0	;; Save R5
	xstore	[srcreg+d3+32], xmm3	;; Save R6
	xstore XMM_TMP1, xmm2
	xstore XMM_TMP2, xmm5
	xstore XMM_TMP3, xmm6
	xstore XMM_TMP4, xmm7
	r4_h8r_h4c_djbunfft_mem [srcreg],[srcreg+32],XMM_TMP1,XMM_TMP2,[srcreg+d2],[srcreg+d2+32],XMM_TMP3,XMM_TMP4,screg1,0,screg2,0,srcreg+srcinc,d1,[srcreg+d2+16],[srcreg+d2+48]
	xstore	[srcreg], xmm1		;; Save R1
	xstore	[srcreg+d2], xmm4	;; Save R2
	xstore	[srcreg+16], xmm7	;; Save R3
;;	xstore	[srcreg+d2+16], xmm5	;; Save R4
	xstore	[srcreg+32], xmm0	;; Save R5
	xstore	[srcreg+d2+32], xmm3	;; Save R6
	xstore	[srcreg+48], xmm6	;; Save R7
;;	xstore	[srcreg+d2+48], xmm2	;; Save R8
	bump	srcreg, srcinc
	ENDM

;; Like r4_h4cl_eight_reals_four_complex_djbunfft but uses 2 sin/cos ptrs.
;; Used in pass 2 when the first levels are radix-3.
r4_h4cl_2sc_eight_reals_four_complex_djbunfft MACRO srcreg,srcinc,d1,d2,screg1,scoff1,screg2,scoff2,screg3,scoff3
	d3 = d2 + d1
	tmp = srcinc+d2
	r4_h8r_h4c_2sc_djbunfft_mem [srcreg+16],[srcreg+48],[srcreg+d1+16],[srcreg+d1+48],[srcreg+d2+16],[srcreg+d2+48],[srcreg+d3+16],[srcreg+d3+48],screg1+scoff1,screg2+scoff2,screg3+scoff3,srcreg+tmp,d1,[srcreg+d3+16],[srcreg+d3+48]
	xstore	[srcreg+d1+16], xmm7	;; Save R3
;;	xstore	[srcreg+d3+16], xmm5	;; Save R4
	xstore	[srcreg+d1+48], xmm6	;; Save R7
;;	xstore	[srcreg+d3+48], xmm2	;; Save R8
	xload	xmm2, [srcreg+d1]	;; Load R2
	xload	xmm5, [srcreg+d1+32]	;; Load I2
	xload	xmm6, [srcreg+d3]	;; Load R4
	xload	xmm7, [srcreg+d3+32]	;; Load I4
	xstore	[srcreg+d1], xmm1	;; Save R1
	xstore	[srcreg+d3], xmm4	;; Save R2
	xstore	[srcreg+d1+32], xmm0	;; Save R5
	xstore	[srcreg+d3+32], xmm3	;; Save R6
	xstore XMM_TMP1, xmm2
	xstore XMM_TMP2, xmm5
	xstore XMM_TMP3, xmm6
	xstore XMM_TMP4, xmm7
	r4_h8r_h4c_2sc_djbunfft_mem [srcreg],[srcreg+32],XMM_TMP1,XMM_TMP2,[srcreg+d2],[srcreg+d2+32],XMM_TMP3,XMM_TMP4,screg1,screg2,screg3,srcreg+srcinc,d1,[srcreg+d2+16],[srcreg+d2+48]
	xstore	[srcreg], xmm1		;; Save R1
	xstore	[srcreg+d2], xmm4	;; Save R2
	xstore	[srcreg+16], xmm7	;; Save R3
;;	xstore	[srcreg+d2+16], xmm5	;; Save R4
	xstore	[srcreg+32], xmm0	;; Save R5
	xstore	[srcreg+d2+32], xmm3	;; Save R6
	xstore	[srcreg+48], xmm6	;; Save R7
;;	xstore	[srcreg+d2+48], xmm2	;; Save R8
	bump	srcreg, srcinc
	ENDM

r4_h8r_h4c_djbunfft_mem MACRO mem1,mem2,mem3,mem4,mem5,mem6,mem7,mem8,screg1,scoff1,screg2,scoff2,pre1,pre2,dst4,dst8
	r4_h8r_h4c_2sc_djbunfft_mem mem1,mem2,mem3,mem4,mem5,mem6,mem7,mem8,screg1+scoff1,screg1+scoff1+32,screg2+scoff2,pre1,pre2,dst4,dst8
	ENDM
r4_h8r_h4c_2sc_djbunfft_mem MACRO mem1,mem2,mem3,mem4,mem5,mem6,mem7,mem8,screg1,screg2,screg3,pre1,pre2,dst4,dst8

	;; Do the four complex part

	movsd	xmm7, Q [screg2+16]	;; cosine/sine
	mulsd	xmm7, Q mem3[8]		;; A2 = R2 * cosine/sine
	movsd	xmm6, Q mem4[8]		;; I2
	addsd	xmm7, xmm6		;; A2 = A2 + I2
	mulsd	xmm6, Q [screg2+16]	;; B2 = I2 * cosine/sine

	movsd	xmm3, Q [screg1+16]	;; cosine/sine
	movsd	xmm4, xmm3		;; cosine/sine
	mulsd	xmm3, Q mem7[8]		;; A4 = R4 * cosine/sine
	movsd	xmm0, Q mem8[8]		;; I4
	subsd	xmm3, xmm0		;; A4 = A4 - I4
	mulsd	xmm0, xmm4		;; B4 = I4 * cosine/sine
	subsd	xmm6, Q mem3[8]		;; B2 = B2 - R2

	movsd	xmm1, Q mem5[8]		;; R3
	mulsd	xmm1, xmm4		;; A3 = R3 * cosine/sine
	mulsd	xmm4, Q mem6[8]		;; B3 = I3 * cosine/sine

	addsd	xmm0, Q mem7[8]		;; B4 = B4 + R4
	addsd	xmm1, Q mem6[8]		;; A3 = A3 + I3
	subsd	xmm4, Q mem5[8]		;; B3 = B3 - R3

	mulsd	xmm7, Q [screg2]	;; A2 = A2 * sine (new R2)
	mulsd	xmm6, Q [screg2]	;; B2 = B2 * sine (new I2)
	movsd	xmm2, Q [screg1]	;; Sine
	mulsd	xmm3, xmm2		;; A4 = A4 * sine (new R4)
	mulsd	xmm0, xmm2		;; B4 = B4 * sine (new I4)
	mulsd	xmm1, xmm2		;; A3 = A3 * sine (new R3)
	mulsd	xmm4, xmm2		;; B3 = B3 * sine (new I3)

	movsd	xmm5, Q mem1[8]		;; R1
	subsd	xmm5, xmm7		;; R1 = R1 - R2 (new R2)
	addsd	xmm7, Q mem1[8]		;; R2 = R1 + R2 (new R1)

	movsd	xmm2, xmm3		;; Copy R4
	subsd	xmm3, xmm1		;; R4 = R4 - R3 (new I4)
	addsd	xmm1, xmm2		;; R3 = R4 + R3 (new R3)

	movsd	xmm2, xmm4		;; Copy I3
	subsd	xmm4, xmm0		;; I3 = I3 - I4 (new R4)
	addsd	xmm0, xmm2		;; I4 = I3 + I4 (new I3)

	movsd	xmm2, xmm7		;; Copy R1
	subsd	xmm7, xmm1		;; R1 = R1 - R3 (final R3)
	addsd	xmm1, xmm2		;; R3 = R1 + R3 (final R1)

	movsd	xmm2, xmm5		;; Copy R2
	subsd	xmm5, xmm4		;; R2 = R2 - R4 (final R4)
	addsd	xmm4, xmm2		;; R4 = R2 + R4 (final R2)

	movsd	xmm2, Q mem2[8]		;; I1
	subsd	xmm2, xmm6		;; I1 = I1 - I2 (new I2)
	addsd	xmm6, Q mem2[8]		;; I2 = I1 + I2 (new I1)

	movsd	Q dst4[8], xmm5

	movsd	xmm5, xmm2		;; Copy I2
	subsd	xmm2, xmm3		;; I2 = I2 - I4 (final I4)
	addsd	xmm3, xmm5		;; I4 = I2 + I4 (final I2)

	movsd	xmm5, xmm6		;; Copy I1
	subsd	xmm6, xmm0		;; I1 = I1 - I3 (final I3)
	addsd	xmm0, xmm5		;; I3 = I1 + I3 (final I1)

	movsd	Q dst8[8], xmm2

	;; Do the eight-reals part

	unpcklo xmm0, xmm0
	unpcklo xmm1, xmm1
	unpcklo xmm3, xmm3
	unpcklo xmm4, xmm4
	unpcklo xmm6, xmm6
	unpcklo xmm7, xmm7

	movlpd	xmm1, Q [screg3+24]	;; cosine/sine for w^5n
	mulsd	xmm1, Q mem7		;; A4 = R4 * cosine/sine
	movlpd	xmm7, Q mem8		;; I4
	addsd	xmm1, xmm7		;; A4 = A4 + I4
	mulsd	xmm7, Q [screg3+24]	;; B4 = I4 * cosine/sine
	subsd	xmm7, Q mem7		;; B4 = B4 - R4

	movlpd	xmm4, Q [screg3+8]	;; cosine/sine for w^n
	mulsd	xmm4, Q mem5		;; A3 = R3 * cosine/sine for w^n
	movsd	xmm5, Q mem6		;; I3
	addsd	xmm4, xmm5		;; A3 = A3 + I3
	mulsd	xmm5, Q [screg3+8]	;; B3 = I3 * cosine/sine
	subsd	xmm5, Q mem5		;; B3 = B3 - R3

	movlpd	xmm0, Q [screg1+16]	;; cosine/sine for w^2n
	mulsd	xmm0, Q mem3		;; A2 = R2 * cosine/sine
	movlpd	xmm3, Q mem4		;; I2
	addsd	xmm0, xmm3		;; A2 = A2 + I2
	mulsd	xmm3, Q [screg1+16]	;; B2 = I2 * cosine/sine
	subsd	xmm3, Q mem3		;; B2 = B2 - R2

	xprefetchw [pre1]

	mulsd	xmm1, Q [screg3+16]	;; new R7 = A4 * sine
	mulsd	xmm7, Q [screg3+16]	;; new R8 = B4 * sine
	mulsd	xmm4, Q [screg3]	;; new R5 = A3 * sine
	mulsd	xmm5, Q [screg3]	;; new R6 = B3 * sine
	mulsd	xmm0, Q [screg1]	;; new R3 = A2 * sine
	mulsd	xmm3, Q [screg1]	;; new R4 = B2 * sine

	xprefetchw [pre1][pre2]

	movsd	xmm6, xmm4		;; Copy R5
	subsd	xmm4, xmm1		;; new R6 = R5 - R7
	addsd	xmm1, xmm6		;; new R5 = R5 + R7

	movsd	xmm2, xmm5		;; Copy R6
	subsd	xmm5, xmm7		;; new R8 = R6 - R8
	addsd	xmm7, xmm2		;; new R7 = R6 + R8

	movsd	xmm6, xmm4		;; Copy R6
	addsd	xmm4, xmm5		;; R6 = R6 + R8
	subsd	xmm5, xmm6		;; R8 = R8 - R6

	mulsd	xmm4, Q XMM_SQRTHALF	;; R6 = R6 * square root of 1/2
	mulsd	xmm5, Q XMM_SQRTHALF	;; R8 = R8 * square root of 1/2

	movsd	xmm2, Q mem2		;; R2
	subsd	xmm2, xmm3		;; R2 = R2 - R4 (new R4)
	addsd	xmm3, Q mem2		;; R4 = R2 + R4 (new R2)

	movsd	xmm6, xmm2		;; Copy R4
	subsd	xmm2, xmm5		;; R4 = R4 - R8 (final R8)
	addsd	xmm5, xmm6		;; R8 = R4 + R8 (final R4)

	movsd	xmm6, xmm3		;; Copy R2
	subsd	xmm3, xmm4		;; R2 = R2 - R6 (final R6)
	addsd	xmm4, xmm6		;; R6 = R2 + R6 (final R2)

	movlpd	xmm6, Q mem1		;; R1
	subsd	xmm6, xmm0		;; R1 = R1 - R3 (new R3)
	addsd	xmm0, Q mem1		;; R3 = R1 + R3 (new R1)

	movsd	Q dst8, xmm2		;; Save R8
	movsd	Q dst4, xmm5		;; Save R4

	movsd	xmm2, xmm6		;; Copy R3
	subsd	xmm6, xmm7		;; R3 = R3 - R7 (final R7)
	addsd	xmm7, xmm2		;; R7 = R3 + R7 (final R3)

	movsd	xmm5, xmm0		;; Copy R1
	subsd	xmm0, xmm1		;; R1 = R1 - R5 (final R5)
	addsd	xmm1, xmm5		;; R5 = R1 + R5 (final R1)
	ENDM


;;
;; ********************************* half-eight-reals-four-complex-fft-with-square variants ***************************************
;;

;; Macro to do an eight_reals_fft and three four_complex_djbfft in the final levels of pass 2.
;; The eight-reals macro and one of the four-complex only use half the XMM 
;; register.  This isn't very efficient, but this macro is called only once.

r4_h4cl_eight_reals_four_complex_fft_final MACRO srcreg,srcinc,d1,d2
	r4_h8r_h4c_simple_fft_mem [srcreg],[srcreg+d1],[srcreg+d2],[srcreg+d2+d1],[srcreg+16],[srcreg+d1+16],[srcreg+d2+16],[srcreg+d2+d1+16],[srcreg],[srcreg+16]
;;	xstore	[srcreg], xmm0		;; Save R1
;;	xstore	[srcreg+16], xmm0	;; Save I1
	xload	xmm3, [srcreg+32]	;; R1
	xload	xmm7, [srcreg+48]	;; R5
	xstore	[srcreg+32], xmm2	;; Save R2
	xstore	[srcreg+48], xmm6	;; Save I2
	xstore	[srcreg+d1], xmm0	;; Save R3
	xstore	[srcreg+d1+16], xmm1	;; Save I3
	xload	xmm2, [srcreg+d1+32]	;; R2
	xload	xmm6, [srcreg+d1+48]	;; R6
	xstore	[srcreg+d1+32], xmm5	;; Save R4
	xstore	[srcreg+d1+48], xmm4	;; Save I4
	r4_x4c_simple_fft_partial_mem xmm3,xmm2,xmm0,xmm5,xmm7,xmm6,xmm1,xmm4,[srcreg+d2+32],[srcreg+d2+d1+32],[srcreg+d2+48],[srcreg+d2+d1+48],srcreg+srcinc+d2,d1
	xstore	[srcreg+d2], xmm5	;; Save R1
	xstore	[srcreg+d2+16], xmm4	;; Save R2
	xstore	[srcreg+d2+32], xmm0	;; Save R3
	xstore	[srcreg+d2+48], xmm1	;; Save R4
	xstore	[srcreg+d2+d1], xmm3	;; Save R5
	xstore	[srcreg+d2+d1+16], xmm2	;; Save R6
	xstore	[srcreg+d2+d1+32], xmm6	;; Save R7
	xstore	[srcreg+d2+d1+48], xmm7	;; Save R8
	bump	srcreg, srcinc
	ENDM

r4_h4cl_eight_reals_four_complex_with_square MACRO srcreg,srcinc,d1,d2
	r4_h8r_h4c_simple_fft_mem [srcreg],[srcreg+d1],[srcreg+d2],[srcreg+d2+d1],[srcreg+16],[srcreg+d1+16],[srcreg+d2+16],[srcreg+d2+d1+16],[srcreg],[srcreg+16]
	xp_complex_square xmm2, xmm6, xmm7	;; Square R2, I2
	xstore	[srcreg+d1], xmm2
	xstore	[srcreg+d1+16], xmm6
	xp_complex_square xmm0, xmm1, xmm7	;; Square R3, I3
	xstore	[srcreg+d2], xmm0
	xstore	[srcreg+d2+16], xmm1
	xp_complex_square xmm5, xmm4, xmm7	;; Square R4, I4
	xstore	[srcreg+d2+d1], xmm5
	xstore	[srcreg+d2+d1+16], xmm4
	movsd	xmm0, Q [srcreg]		;; R1
	movsd	xmm1, Q [srcreg+16]		;; I1
	mulsd	xmm0, xmm0			;; Square R1, I1 low (actually they are both real)
	mulsd	xmm1, xmm1
	movsd	Q [srcreg-16], xmm0		;; Save product of sum of FFT values
	movsd	Q [srcreg], xmm0
	movsd	Q [srcreg+16], xmm1
	movsd	xmm0, Q [srcreg+8]		;; R1
	movsd	xmm1, Q [srcreg+24]		;; I1
	xs_complex_square xmm0, xmm1, xmm2	;; Square R1, I1 high
	movsd	Q [srcreg+8], xmm0
	movsd	Q [srcreg+24], xmm1
	r4_h8r_h4c_simple_unfft_mem [srcreg],[srcreg+16],[srcreg+d1],[srcreg+d1+16],[srcreg+d2],[srcreg+d2+16],[srcreg+d2+d1],[srcreg+d2+d1+16],srcreg+srcinc,d1,[srcreg+d1]
	xstore	[srcreg], xmm1			;; Save R1
	xstore	[srcreg+16], xmm7		;; Save R3
	xload	xmm1, [srcreg+32]		;; R1
	xload	xmm7, [srcreg+48]		;; R5
	xstore	[srcreg+32], xmm0		;; Save R5
	xstore	[srcreg+48], xmm6		;; Save R7
	xload	xmm0, [srcreg+d1+32]		;; R2
	xload	xmm6, [srcreg+d1+48]		;; R6
;;	xstore	[srcreg+d1], xmm4		;; Save R2
	xstore	[srcreg+d1+16], xmm5		;; Save R4
	xstore	[srcreg+d1+32], xmm3		;; Save R6
	xstore	[srcreg+d1+48], xmm2		;; Save R8
	r4_x4c_simple_fft_partial_mem xmm1,xmm0,xmm2,xmm3,xmm7,xmm6,xmm5,xmm4,[srcreg+d2+32],[srcreg+d2+d1+32],[srcreg+d2+48],[srcreg+d2+d1+48],srcreg+srcinc+d2,d1
	xstore	[srcreg+d2], xmm1		;; R5
	xp_complex_square xmm3, xmm4, xmm1	;; Square R1, R2
	xp_complex_square xmm2, xmm5, xmm1	;; Square R3, R4
	xp_complex_square xmm6, xmm7, xmm1	;; Square R7, R8
	xload	xmm1, [srcreg+d2]		;; R5
	xstore	[srcreg+d2], xmm7		;; R8
	xp_complex_square xmm1, xmm0, xmm7	;; Square R5, R6
	r4_x4c_simple_unfft xmm3,xmm4,xmm2,xmm5,xmm1,xmm0,xmm6,xmm7,[srcreg+d2],srcreg+srcinc+d2,d1,[srcreg+d2]
;;	xstore	[srcreg+d2], xmm1		;; Save R1
	xstore	[srcreg+d2+16], xmm2		;; Save R3
	xstore	[srcreg+d2+32], xmm7		;; Save R5
	xstore	[srcreg+d2+48], xmm5		;; Save R7
	xstore	[srcreg+d2+d1], xmm0		;; Save R2
	xstore	[srcreg+d2+d1+16], xmm3		;; Save R4
	xstore	[srcreg+d2+d1+32], xmm6		;; Save R6
	xstore	[srcreg+d2+d1+48], xmm4		;; Save R8
	bump	srcreg, srcinc
	ENDM

r4_h4cl_eight_reals_four_complex_with_mult MACRO srcreg,srcinc,d1,d2
	r4_h8r_h4c_simple_fft_mem [srcreg],[srcreg+d1],[srcreg+d2],[srcreg+d2+d1],[srcreg+16],[srcreg+d1+16],[srcreg+d2+16],[srcreg+d2+d1+16],[srcreg],[srcreg+16]
	xload	xmm3, [srcreg]
	xload	xmm7, [srcreg+16]
	xp4c_mulf xmm3,xmm7,xmm2,xmm6,xmm0,xmm1,xmm5,xmm4,[srcreg],[srcreg+16],[srcreg+32],[srcreg+48],[srcreg+d1],[srcreg+d1+16],[srcreg+d1+32],[srcreg+d1+48]
	xstore	[srcreg+d1], xmm2
	xstore	[srcreg+d1+16], xmm6
	xstore	[srcreg+d2], xmm0
	xstore	[srcreg+d2+16], xmm1
	xstore	[srcreg+d2+d1], xmm5
	xstore	[srcreg+d2+d1+16], xmm4
	movlpd	xmm3, Q [srcreg]
	mulsd	xmm3, Q [srcreg][rbp]		;; Multiply R1 low
	movsd	Q [srcreg-16], xmm3		;; Save product of sum of FFT values
	movlpd	xmm7, Q [srcreg+16]
	mulsd	xmm7, Q [srcreg+16][rbp]	;; Multiply I1 low (actually a real value)
	xstore	[srcreg], xmm3
	xstore	[srcreg+16], xmm7
	r4_h8r_h4c_simple_unfft_mem [srcreg],[srcreg+16],[srcreg+d1],[srcreg+d1+16],[srcreg+d2],[srcreg+d2+16],[srcreg+d2+d1],[srcreg+d2+d1+16],srcreg+srcinc,d1,[srcreg+d1]
	xstore	[srcreg], xmm1			;; Save R1
	xstore	[srcreg+16], xmm7		;; Save R3
	xload	xmm1, [srcreg+32]		;; R1
	xload	xmm7, [srcreg+48]		;; R5
	xstore	[srcreg+32], xmm0		;; Save R5
	xstore	[srcreg+48], xmm6		;; Save R7
	xload	xmm0, [srcreg+d1+32]		;; R2
	xload	xmm6, [srcreg+d1+48]		;; R6
;;	xstore	[srcreg+d1], xmm4		;; Save R2
	xstore	[srcreg+d1+16], xmm5		;; Save R4
	xstore	[srcreg+d1+32], xmm3		;; Save R6
	xstore	[srcreg+d1+48], xmm2		;; Save R8
	r4_x4c_simple_fft_partial_mem xmm1,xmm0,xmm2,xmm3,xmm7,xmm6,xmm5,xmm4,[srcreg+d2+32],[srcreg+d2+d1+32],[srcreg+d2+48],[srcreg+d2+d1+48],srcreg+srcinc+d2,d1
	xp4c_mulf xmm3,xmm4,xmm2,xmm5,xmm1,xmm0,xmm6,xmm7,[srcreg+d2],[srcreg+d2+16],[srcreg+d2+32],[srcreg+d2+48],[srcreg+d2+d1],[srcreg+d2+d1+16],[srcreg+d2+d1+32],[srcreg+d2+d1+48]
	xstore	[srcreg+d2], xmm7
	r4_x4c_simple_unfft xmm3,xmm4,xmm2,xmm5,xmm1,xmm0,xmm6,xmm7,[srcreg+d2],srcreg+srcinc+d2,d1,[srcreg+d2]
;;	xstore	[srcreg+d2], xmm1		;; Save R1
	xstore	[srcreg+d2+16], xmm2		;; Save R3
	xstore	[srcreg+d2+32], xmm7		;; Save R5
	xstore	[srcreg+d2+48], xmm5		;; Save R7
	xstore	[srcreg+d2+d1], xmm0		;; Save R2
	xstore	[srcreg+d2+d1+16], xmm3		;; Save R4
	xstore	[srcreg+d2+d1+32], xmm6		;; Save R6
	xstore	[srcreg+d2+d1+48], xmm4		;; Save R8
	bump	srcreg, srcinc
	ENDM

r4_h4cl_eight_reals_four_complex_with_mulf MACRO srcreg,srcinc,d1,d2
	xload	xmm3, [srcreg][rbx]		;; R1
	xload	xmm7, [srcreg+16][rbx]		;; R2
	xload	xmm2, [srcreg+32][rbx]		;; R3
	xload	xmm6, [srcreg+48][rbx]		;; R4
	xload	xmm0, [srcreg+d1][rbx]		;; R5
	xload	xmm1, [srcreg+d1+16][rbx]	;; R6
	xload	xmm5, [srcreg+d1+32][rbx]	;; R7
	xload	xmm4, [srcreg+d1+48][rbx]	;; R8
	xp4c_mulf xmm3,xmm7,xmm2,xmm6,xmm0,xmm1,xmm5,xmm4,[srcreg],[srcreg+16],[srcreg+32],[srcreg+48],[srcreg+d1],[srcreg+d1+16],[srcreg+d1+32],[srcreg+d1+48]
	xstore	[srcreg+32], xmm2
	xstore	[srcreg+48], xmm6
	xstore	[srcreg+d1], xmm0
	xstore	[srcreg+d1+16], xmm1
	xstore	[srcreg+d1+32], xmm5
	xstore	[srcreg+d1+48], xmm4
	movlpd	xmm3, Q [srcreg][rbx]
	mulsd	xmm3, Q [srcreg][rbp]		;; Multiply R1 low
	movsd	Q [srcreg-16], xmm3		;; Save product of sum of FFT values
	movlpd	xmm7, Q [srcreg+16][rbx]
	mulsd	xmm7, Q [srcreg+16][rbp]	;; Multiply I1 low (actually a real value)
	xstore	[srcreg], xmm3
	xstore	[srcreg+16], xmm7
	r4_h8r_h4c_simple_unfft_mem [srcreg],[srcreg+16],[srcreg+32],[srcreg+48],[srcreg+d1],[srcreg+d1+16],[srcreg+d1+32],[srcreg+d1+48],srcreg+srcinc,d1,[srcreg+d1]
	xstore	[srcreg], xmm1			;; Save R1
	xstore	[srcreg+16], xmm7		;; Save R3
	xstore	[srcreg+32], xmm0		;; Save R5
	xstore	[srcreg+48], xmm6		;; Save R7
;;	xstore	[srcreg+d1], xmm4		;; Save R2
	xstore	[srcreg+d1+16], xmm5		;; Save R4
	xstore	[srcreg+d1+32], xmm3		;; Save R6
	xstore	[srcreg+d1+48], xmm2		;; Save R8
	xload	xmm3, [srcreg+d2][rbx]		;; R1
	xload	xmm7, [srcreg+d2+16][rbx]	;; R2
	xload	xmm1, [srcreg+d2+32][rbx]	;; R3
	xload	xmm5, [srcreg+d2+48][rbx]	;; R4
	xload	xmm0, [srcreg+d2+d1][rbx]	;; R5
	xload	xmm4, [srcreg+d2+d1+16][rbx]	;; R6
	xload	xmm6, [srcreg+d2+d1+32][rbx]	;; R7
	xload	xmm2, [srcreg+d2+d1+48][rbx]	;; R8
	xp4c_mulf xmm3,xmm7,xmm1,xmm5,xmm0,xmm4,xmm6,xmm2,[srcreg+d2],[srcreg+d2+16],[srcreg+d2+32],[srcreg+d2+48],[srcreg+d2+d1],[srcreg+d2+d1+16],[srcreg+d2+d1+32],[srcreg+d2+d1+48]
	xstore	[srcreg+d2], xmm2
	r4_x4c_simple_unfft xmm3,xmm7,xmm1,xmm5,xmm0,xmm4,xmm6,xmm2,[srcreg+d2],srcreg+srcinc+d2,d1,[srcreg+d2]
;;	xstore	[srcreg+d2], xmm0		;; Save R1
	xstore	[srcreg+d2+16], xmm1		;; Save R3
	xstore	[srcreg+d2+32], xmm2		;; Save R5
	xstore	[srcreg+d2+48], xmm5		;; Save R7
	xstore	[srcreg+d2+d1], xmm4		;; Save R2
	xstore	[srcreg+d2+d1+16], xmm3		;; Save R4
	xstore	[srcreg+d2+d1+32], xmm6		;; Save R6
	xstore	[srcreg+d2+d1+48], xmm7		;; Save R8
	bump	srcreg, srcinc
	ENDM

;; Does an r4_x8r_fft_mem on the low word of the xmm register
;; Does an r4_x4c_djbfft_mem on the high word of the xmm register
r4_h8r_h4c_simple_fft_mem MACRO mem1,mem2,mem3,mem4,mem5,mem6,mem7,mem8,dst1,dst2

	;; Do the four-complex part

	movsd	xmm0, Q mem1[8]		;; R1
	movsd	xmm2, Q mem3[8]		;; R3
	addsd	xmm2, xmm0		;; R3 = R1 + R3 (new R1)
	subsd	xmm0, Q mem3[8]		;; R1 = R1 - R3 (new R3)

	movsd	xmm1, Q mem2[8]		;; R2
	movsd	xmm3, Q mem4[8]		;; R4
	addsd	xmm3, xmm1		;; R4 = R2 + R4 (new R2)
	subsd	xmm1, Q mem4[8]		;; R2 = R2 - R4 (new R4)

	movsd	xmm6, xmm2
	subsd	xmm2, xmm3		;; R1 = R1 - R2 (final R2)
	addsd	xmm3, xmm6		;; R2 = R1 + R2 (final R1)

	movsd	Q dst1[8], xmm3		;; Save R1

	movsd	xmm5, Q mem6[8]		;; I2
	movsd	xmm7, Q mem8[8]		;; I4
	subsd	xmm5, xmm7		;; I2 = I2 - I4 (new I4)
	addsd	xmm7, Q mem6[8]		;; I4 = I2 + I4 (new I2)

	movsd	xmm4, Q mem5[8]		;; I1
	movsd	xmm6, Q mem7[8]		;; I3
	subsd	xmm4, xmm6		;; I1 = I1 - I3 (new I3)
	addsd	xmm6, Q mem5[8]		;; I3 = I1 + I3 (new I1)

	movsd	xmm3, xmm0
	subsd	xmm0, xmm5		;; R3 = R3 - I4 (final R3)
	addsd	xmm5, xmm3		;; I4 = R3 + I4 (final R4)

	movsd	xmm3, xmm1
	addsd	xmm1, xmm4		;; R4 = I3 + R4 (final I3)
	subsd	xmm4, xmm3		;; I3 = I3 - R4 (final I4)

	movsd	xmm3, xmm6
	subsd	xmm6, xmm7		;; I1 = I1 - I2 (final I2)
	addsd	xmm7, xmm3		;; I2 = I1 + I2 (final I1)

	movsd	Q dst2[8], xmm7		;; Save I1

	;; Do the eight reals part

	unpcklo xmm2, xmm2		;; Copy to high part of XMM register
	unpcklo xmm6, xmm6
	unpcklo xmm0, xmm0
	unpcklo xmm1, xmm1
	unpcklo xmm5, xmm5
	unpcklo xmm4, xmm4

	movlpd	xmm1, Q mem4		;; R4
	movsd	xmm7, Q mem8		;; R8
	subsd	xmm1, xmm7		;; new R8 = R4 - R8
	addsd	xmm7, Q mem4		;; new R4 = R4 + R8
	movlpd	xmm0, Q mem2		;; R2
	movlpd	xmm6, Q mem6		;; R6
	subsd	xmm0, xmm6		;; new R6 = R2 - R6
	addsd	xmm6, Q mem2		;; new R2 = R2 + R6

	mulsd	xmm1, Q XMM_SQRTHALF	;; R8 = R8 * square root
	mulsd	xmm0, Q XMM_SQRTHALF	;; R6 = R6 * square root

	movlpd	xmm5, Q mem1		;; R1
	movlpd	xmm2, Q mem5		;; R5
	subsd	xmm5, xmm2		;; new R5 = R1 - R5
	addsd	xmm2, Q mem1		;; new R1 = R1 + R5

	movsd	xmm4, xmm6		;; Copy R2
	subsd	xmm6, xmm7		;; R2 = R2 - R4 (final I2)
	addsd	xmm7, xmm4		;; R4 = R2 + R4 (new I1)

	movsd	xmm3, xmm0		;; Copy R6
	subsd	xmm0, xmm1		;; R6 = R6 - R8 (Real part)
	addsd	xmm1, xmm3		;; R8 = R6 + R8 (Imaginary part)

	movsd	xmm3, Q mem7		;; R7
	addsd	xmm3, Q mem3		;; new R3 = R3 + R7

	movsd	xmm4, xmm2		;; Copy R1
	subsd	xmm2, xmm3		;; R1 = R1 - R3 (final R2)
	addsd	xmm3, xmm4		;; R3 = R1 + R3 (new R1)

	movsd	xmm4, xmm3		;; Copy R1
	subsd	xmm3, xmm7		;; R1 = R1 - R2 (final I1)
	addsd	xmm7, xmm4		;; R2 = R1 + R2 (final R1)

	movlpd	xmm4, Q mem3		;; R3
	subsd	xmm4, Q mem7		;; new R7 = R3 - R7

	movsd	Q dst1, xmm7		;; Save R1
	movsd	Q dst2, xmm3		;; Save I1

	movsd	xmm7, xmm5		;; Copy R5
	subsd	xmm5, xmm0		;; R5 = R5 - R6 (final R4)
	addsd	xmm0, xmm7		;; R6 = R5 + R6 (final R3)

	movsd	xmm3, xmm4		;; Copy R7
	subsd	xmm4, xmm1		;; R7 = R7 - R8 (final I4)
	addsd	xmm1, xmm3		;; R8 = R7 + R8 (final I3)
	ENDM

r4_h8r_h4c_simple_unfft_mem MACRO mem1,mem2,mem3,mem4,mem5,mem6,mem7,mem8,pre1,pre2,dst2

	;; Do the four complex part

	movsd	xmm5, Q mem1[8]		;; R1
	movsd	xmm7, Q mem3[8]		;; R2
	subsd	xmm5, xmm7		;; R1 = R1 - R2 (new R2)
	addsd	xmm7, Q mem1[8]		;; R2 = R1 + R2 (new R1)

	movsd	xmm3, Q mem7[8]		;; R4
	movsd	xmm1, Q mem5[8]		;; R3
	subsd	xmm3, xmm1		;; R4 = R4 - R3 (new I4)
	addsd	xmm1, Q mem7[8]		;; R3 = R4 + R3 (new R3)

	xprefetchw [pre1]

	movsd	xmm4, Q mem6[8]		;; I3
	movsd	xmm0, Q mem8[8]		;; I4
	subsd	xmm4, xmm0		;; I3 = I3 - I4 (new R4)
	addsd	xmm0, Q mem6[8]		;; I4 = I3 + I4 (new I3)

	movsd	xmm2, xmm7		;; Copy R1
	subsd	xmm7, xmm1		;; R1 = R1 - R3 (final R3)
	addsd	xmm1, xmm2		;; R3 = R1 + R3 (final R1)

	xprefetchw [pre1][pre2]

	movsd	xmm2, xmm5		;; Copy R2
	subsd	xmm5, xmm4		;; R2 = R2 - R4 (final R4)
	addsd	xmm4, xmm2		;; R4 = R2 + R4 (final R2)

	movsd	xmm2, Q mem2[8]		;; I1
	movsd	xmm6, Q mem4[8]		;; I2
	subsd	xmm2, xmm6		;; I1 = I1 - I2 (new I2)
	addsd	xmm6, Q mem2[8]		;; I2 = I1 + I2 (new I1)

	movsd	Q dst2[8], xmm4

	movsd	xmm4, xmm2		;; Copy I2
	subsd	xmm2, xmm3		;; I2 = I2 - I4 (final I4)
	addsd	xmm3, xmm4		;; I4 = I2 + I4 (final I2)

	movsd	xmm4, xmm6		;; Copy I1
	subsd	xmm6, xmm0		;; I1 = I1 - I3 (final I3)
	addsd	xmm0, xmm4		;; I3 = I1 + I3 (final I1)

	;; Do the eight-reals part

	unpcklo xmm0, xmm0
	unpcklo xmm1, xmm1
	unpcklo xmm2, xmm2
	unpcklo xmm3, xmm3
	unpcklo xmm5, xmm5
	unpcklo xmm6, xmm6
	unpcklo xmm7, xmm7

	movsd	xmm4, Q mem5		;; R5
	movlpd	xmm1, Q mem7		;; R7
	subsd	xmm4, xmm1		;; new R6 = R5 - R7
	addsd	xmm1, Q mem5		;; new R5 = R5 + R7

	movlpd	xmm5, Q mem6		;; R6
	movlpd	xmm7, Q mem8		;; R8
	subsd	xmm5, xmm7		;; new R8 = R6 - R8
	addsd	xmm7, Q mem6		;; new R7 = R6 + R8

	movsd	xmm6, xmm4		;; Copy R6
	addsd	xmm4, xmm5		;; R6 = R6 + R8
	subsd	xmm5, xmm6		;; R8 = R8 - R6

	mulsd	xmm4, Q XMM_SQRTHALF	;; R6 = R6 * square root of 1/2
	mulsd	xmm5, Q XMM_SQRTHALF	;; R8 = R8 * square root of 1/2

	movlpd	xmm2, Q mem1		;; R1
	movlpd	xmm6, Q mem2		;; R2
	subsd	xmm2, xmm6		;; new R2 = R1 - R2
	mulhalfs xmm2			;; Mul R2 by HALF
	addsd	xmm6, xmm2		;; new R1 = R1 + R2

	movlpd	xmm3, Q mem4		;; R4
	addsd	xmm3, xmm2		;; R4 = R2 + R4 (new R2)
	subsd	xmm2, Q mem4		;; R2 = R2 - R4 (new R4)

	movsd	xmm0, xmm2		;; Copy R4
	subsd	xmm2, xmm5		;; R4 = R4 - R8 (final R8)
	addsd	xmm5, xmm0		;; R8 = R4 + R8 (final R4)

	movsd	xmm0, xmm3		;; Copy R2
	subsd	xmm3, xmm4		;; R2 = R2 - R6 (final R6)
	addsd	xmm4, xmm0		;; R6 = R2 + R6 (final R2)

	movlpd	xmm0, Q mem3		;; R3
	addsd	xmm0, xmm6		;; R3 = R1 + R3 (new R1)
	subsd	xmm6, Q mem3		;; R1 = R1 - R3 (new R3)

	movsd	Q dst2, xmm4		;; Save R4

	movsd	xmm4, xmm6		;; Copy R3
	subsd	xmm6, xmm7		;; R3 = R3 - R7 (final R7)
	addsd	xmm7, xmm4		;; R7 = R3 + R7 (final R3)

	movsd	xmm4, xmm0		;; Copy R1
	subsd	xmm0, xmm1		;; R1 = R1 - R5 (final R5)
	addsd	xmm1, xmm4		;; R5 = R1 + R5 (final R1)
	ENDM


;;
;; ************************************* four-complex-fft variants ******************************************
;;

IFDEF UNUSED

;; Macros to operate on 4 64-byte cache lines doing 4 four_complex_ffts
;; with sin/cos complex multiplies after the butterfly.

r4_x4cl_four_complex_fft MACRO srcreg,srcinc,d1,d2,screg
	r4_x4c_fft_mem [srcreg],[srcreg+d1],[srcreg+d2],[srcreg+d2+d1],[srcreg+16],[srcreg+d1+16],[srcreg+d2+16],[srcreg+d2+d1+16],screg+0,screg+32,screg+64,srcreg+srcinc,d1,[srcreg],[srcreg+16]
;	xstore	[srcreg], xmm2		;; Save R1
;	xstore	[srcreg+16], xmm5	;; Save R2
	xload	xmm2, [srcreg+32]	;; R1
	xload	xmm5, [srcreg+48]	;; R5
	xstore	[srcreg+32], xmm7	;; Save R3
	xstore	[srcreg+48], xmm6	;; Save R4
	xstore	[srcreg+d1], xmm3	;; Save R5
	xstore	[srcreg+d1+16], xmm1	;; Save R6
	xload	xmm7, [srcreg+d1+32]	;; R2
	xload	xmm6, [srcreg+d1+48]	;; R6
	xstore	[srcreg+d1+32], xmm0	;; Save R7
	xstore	[srcreg+d1+48], xmm4	;; Save R8
	xload	xmm3, [srcreg+d2+32]	;; R3
	xload	xmm1, [srcreg+d2+48]	;; R7
	xload	xmm0, [srcreg+d2+d1+32]	;; R4
	xload	xmm4, [srcreg+d2+d1+48]	;; R8
	r4_x4c_fft xmm2, xmm7, xmm3, xmm0, xmm5, xmm6, xmm1, xmm4, screg, 0, srcreg+srcinc+d2, d1, [srcreg+d2], [srcreg+d2+16]
;	xstore	[srcreg+d2], xmm3	;; Save R1
;	xstore	[srcreg+d2+16], xmm6	;; Save R2
	xstore	[srcreg+d2+32], xmm4	;; Save R3
	xstore	[srcreg+d2+48], xmm1	;; Save R4
	xstore	[srcreg+d2+d1], xmm0	;; Save R5
	xstore	[srcreg+d2+d1+16], xmm7	;; Save R6
	xstore	[srcreg+d2+d1+32], xmm2	;; Save R7
	xstore	[srcreg+d2+d1+48], xmm5	;; Save R8
	bump	srcreg, srcinc
	ENDM

r4_g4cl_four_complex_fft MACRO srcreg,srcinc,d1,d2,dstreg,dstinc,e1,e2
	xprefetch [srcreg+srcinc]
	r4_x4c_fft_mem [srcreg],[srcreg+d1],[srcreg+d2],[srcreg+d2+d1],[srcreg+16],[srcreg+d1+16],[srcreg+d2+16],[srcreg+d2+d1+16],rdi+0,rdi+32,rdi+64,dstreg+dstinc,e1,[dstreg],[dstreg+16]
	xprefetch [srcreg+srcinc+d1]
;	xstore	[dstreg], xmm2		;; Save R1
;	xstore	[dstreg+16], xmm5	;; Save R2
	xstore	[dstreg+32], xmm7	;; Save R3
	xstore	[dstreg+48], xmm6	;; Save R4
	xstore	[dstreg+e1], xmm3	;; Save R5
	xstore	[dstreg+e1+16], xmm1	;; Save R6
	xstore	[dstreg+e1+32], xmm0	;; Save R7
	xstore	[dstreg+e1+48], xmm4	;; Save R8
	xprefetchw [srcreg+srcinc+d2]
	r4_x4c_fft_mem [srcreg+32],[srcreg+d1+32],[srcreg+d2+32],[srcreg+d2+d1+32],[srcreg+48],[srcreg+d1+48],[srcreg+d2+48],[srcreg+d2+d1+48],rdi+0,rdi+32,rdi+64,dstreg+dstinc+e2,e1,[dstreg+e2],[dstreg+e2+16]
	xprefetchw [srcreg+srcinc+d2+d1]
	bump	srcreg, srcinc
;	xstore	[dstreg+e2], xmm2	;; Save R1
;	xstore	[dstreg+e2+16], xmm5	;; Save R2
	xstore	[dstreg+e2+32], xmm7	;; Save R3
	xstore	[dstreg+e2+48], xmm6	;; Save R4
	xstore	[dstreg+e2+e1], xmm3	;; Save R5
	xstore	[dstreg+e2+e1+16], xmm1	;; Save R6
	xstore	[dstreg+e2+e1+32], xmm0	;; Save R7
	xstore	[dstreg+e2+e1+48], xmm4	;; Save R8
	bump	dstreg, dstinc
	ENDM
ENDIF

r4_x4c_fft_mem MACRO R1,R2,R3,R4,R5,R6,R7,R8,screg1,screg2,screg3,pre1,pre2,dst1,dst2
	xload	xmm0, R1		;; R1
	xload	xmm2, R3		;; R3
	subpd	xmm0, xmm2		;; R1 = R1 - R3 (new R3)
	addpd	xmm2, R1		;; R3 = R1 + R3 (new R1)

	xload	xmm4, R5		;; I1
	xload	xmm6, R7		;; I3
	subpd	xmm4, xmm6		;; I1 = I1 - I3 (new I3)
	addpd	xmm6, R5		;; I3 = I1 + I3 (new I1)

	xload	xmm1, R2		;; R2
	xload	xmm3, R4		;; R4
	subpd	xmm1, xmm3		;; R2 = R2 - R4 (new R4)
	addpd	xmm3, R2		;; R4 = R2 + R4 (new R2)

	xload	xmm5, R6		;; I2
	xload	xmm7, R8		;; I4
	subpd	xmm5, xmm7		;; I2 = I2 - I4 (new I4)
	addpd	xmm7, R6		;; I4 = I2 + I4 (new I2)

	subpd	xmm4, xmm1		;; I3 = I3 - R4 (final I4)
	multwo	xmm1			;; R4 = R4 * 2
	addpd	xmm1, xmm4		;; R4 = I3 + R4 (final I3)

	subpd	xmm0, xmm5		;; R3 = R3 - I4 (final R3)
	multwo	xmm5			;; I4 = I4 * 2
	addpd	xmm5, xmm0		;; I4 = R3 + I4 (final R4)

	xprefetchw [pre1]

	subpd	xmm2, xmm3		;; R1 = R1 - R2 (final R2)
	multwo	xmm3			;; R2 = R2 * 2
	addpd	xmm3, xmm2		;; R2 = R1 + R2 (final R1)

	subpd	xmm6, xmm7		;; I1 = I1 - I2 (final I2)
	multwo	xmm7			;; I2 = I2 * 2
	addpd	xmm7, xmm6		;; I2 = I1 + I2 (final I1)

	xstore	dst1, xmm3		;; Save R1
	xstore	dst2, xmm7		;; Save I1

	xload	xmm3, [screg1+16]	;; cosine/sine
	mulpd	xmm3, xmm0		;; A3 = R3 * cosine/sine
	subpd	xmm3, xmm1		;; A3 = A3 - I3
	mulpd	xmm1, [screg1+16]	;; B3 = I3 * cosine/sine
	addpd	xmm1, xmm0		;; B3 = B3 + R3

	xload	xmm7, [screg2+16]	;; cosine/sine
	mulpd	xmm7, xmm2		;; A2 = R2 * cosine/sine
	subpd	xmm7, xmm6		;; A2 = A2 - I2
	mulpd	xmm6, [screg2+16]	;; B2 = I2 * cosine/sine
	addpd	xmm6, xmm2		;; B2 = B2 + R2

	xprefetchw [pre1][pre2]

	xload	xmm0, [screg3+16]	;; cosine/sine
	mulpd	xmm0, xmm5		;; A4 = R4 * cosine/sine
	subpd	xmm0, xmm4		;; A4 = A4 - I4
	mulpd	xmm4, [screg3+16]	;; B4 = I4 * cosine/sine
	addpd	xmm4, xmm5		;; B4 = B4 + R4

	mulpd	xmm3, [screg1]		;; A3 = A3 * sine (final R3)
	mulpd	xmm1, [screg1]		;; B3 = B3 * sine (final I3)
	mulpd	xmm7, [screg2]		;; A2 = A2 * sine (final R2)
	mulpd	xmm6, [screg2]		;; B2 = B2 * sine (final I2)
	mulpd	xmm0, [screg3]		;; A4 = A4 * sine (final R4)
	mulpd	xmm4, [screg3]		;; B4 = B4 * sine (final I4)
	ENDM

r4_x4c_fft_partial_mem MACRO r1, r2, r3, r4, r5, r6, r7, r8, mem3, mem4, mem7, mem8, screg, off, pre1, pre2, dst1, dst2
	xload	r3, mem3		;; R3
	addpd	r3, r1			;; R3 = R1 + R3 (new R1)
	subpd	r1, mem3		;; R1 = R1 - R3 (new R3)

	xload	r7, mem7		;; I3
	addpd	r7, r5			;; I3 = I1 + I3 (new I1)
	subpd	r5, mem7		;; I1 = I1 - I3 (new I3)

	xload	r4, mem4		;; R4
	addpd	r4, r2			;; R4 = R2 + R4 (new R2)
	subpd	r2, mem4		;; R2 = R2 - R4 (new R4)

	xload	r8, mem8		;; I4
	addpd	r8, r6			;; I4 = I2 + I4 (new I2)
	subpd	r6, mem8		;; I2 = I2 - I4 (new I4)

	subpd	r5, r2			;; I3 = I3 - R4 (final I4)
	multwo	r2			;; R4 = R4 * 2
	addpd	r2, r5			;; R4 = I3 + R4 (final I3)

	subpd	r1, r6			;; R3 = R3 - I4 (final R3)
	multwo	r6			;; I4 = I4 * 2
	addpd	r6, r1			;; I4 = R3 + I4 (final R4)

	xprefetchw [pre1]

	subpd	r3, r4			;; R1 = R1 - R2 (final R2)
	multwo	r4			;; R2 = R2 * 2
	addpd	r4, r3			;; R2 = R1 + R2 (final R1)

	subpd	r7, r8			;; I1 = I1 - I2 (final I2)
	multwo	r8			;; I2 = I2 * 2
	addpd	r8, r7			;; I2 = I1 + I2 (final I1)

	xstore	dst1, r4		;; Save R1
	xstore	dst2, r8		;; Save I1

	xload	r4, [screg+off+0+16]	;; cosine/sine
	mulpd	r4, r1			;; A3 = R3 * cosine/sine
	subpd	r4, r2			;; A3 = A3 - I3
	mulpd	r2, [screg+off+0+16]	;; B3 = I3 * cosine/sine
	addpd	r2, r1			;; B3 = B3 + R3

	xload	r8, [screg+off+32+16]	;; cosine/sine
	mulpd	r8, r3			;; A2 = R2 * cosine/sine
	subpd	r8, r7			;; A2 = A2 - I2
	mulpd	r7, [screg+off+32+16]	;; B2 = I2 * cosine/sine
	addpd	r7, r3			;; B2 = B2 + R2

	xprefetchw [pre1][pre2]

	xload	r1, [screg+off+64+16]	;; cosine/sine
	mulpd	r1, r6			;; A4 = R4 * cosine/sine
	subpd	r1, r5			;; A4 = A4 - I4
	mulpd	r5, [screg+off+64+16]	;; B4 = I4 * cosine/sine
	addpd	r5, r6			;; B4 = B4 + R4

	mulpd	r4, [screg+off+0]	;; A3 = A3 * sine (final R3)
	mulpd	r2, [screg+off+0]	;; B3 = B3 * sine (final I3)
	mulpd	r8, [screg+off+32]	;; A2 = A2 * sine (final R2)
	mulpd	r7, [screg+off+32]	;; B2 = B2 * sine (final I2)
	mulpd	r1, [screg+off+64]	;; A4 = A4 * sine (final R4)
	mulpd	r5, [screg+off+64]	;; B4 = B4 * sine (final I4)
	ENDM

