; Copyright 2009-2024 - Mersenne Research, Inc.  All rights reserved
; Author:  George Woltman
; Email: woltman@alum.mit.edu
;

;;
;; Macros for a radix-8 step in an FFT.  This is used in a radix-4 FFT
;; with an odd number of levels.
;;



;;
;; ************************************* eight-complex-djbfft variants ******************************************
;;

;; This is a standard eight-complex case with 7 sin/cos twiddle factors applied.
;; DJB allows us to do this using only 4 sin/cos twiddle factors.
IFDEF UNUSED
r8_x8cl_eight_complex_djbfft MACRO srcreg,srcinc,d1,d2,d4,screg
bug	r8_x8c_djbfft srcreg+0,d1,d2,d4,dstreg+0,e1,e2,screg,0
bug	r8_x8c_djbfft srcreg+32,d1,d2,d4,dstreg+e4,e1,e2,screg,0
	bump	srcreg, srcinc
	ENDM

r8_g8cl_eight_complex_djbfft MACRO srcreg,srcinc,d1,d2,d4,dstreg,dstinc,e1,e2,e4,screg
untested	r8_x8c_djbfft srcreg+0,d1,d2,d4,dstreg+0,e1,e2,screg,0
untested	r8_x8c_djbfft srcreg+32,d1,d2,d4,dstreg+e4,e1,e2,screg,0
	bump	srcreg, srcinc
	bump	dstreg, dstinc
	ENDM

r8_x8c_djbfft MACRO srcreg,d1,d2,d4,dstreg,e1,e2,screg,scoff
	xload	xmm0, [srcreg]			;; R1
	xload	xmm2, [srcreg+d4]		;; R5
	addpd	xmm2, xmm0			;; R5 = R1 + R5 (new1 R1)

	xload	xmm1, [srcreg+d2]		;; R3
	xload	xmm3, [srcreg+d4+d2]		;; R7
	addpd	xmm3, xmm1			;; R7 = R3 + R7 (new1 R3)

	xload	xmm4, [srcreg+16]		;; I1
	xload	xmm6, [srcreg+d4+16]		;; I5
	addpd	xmm6, xmm4			;; I5 = I1 + I5 (new1 I1)

	subpd	xmm0, [srcreg+d4]		;; R1 = R1 - R5 (new1 R5)
	subpd	xmm1, [srcreg+d4+d2]		;; R3 = R3 - R7 (new1 R7)

	xcopy	xmm5, xmm3			;; Copy R3
	addpd	xmm3, xmm2			;; R3 = R1 + R3 (new2 R1)
	subpd	xmm2, xmm5			;; R1 = R1 - R3 (new2 R3)

	xload	xmm5, [srcreg+d2+16]		;; I3
	xload	xmm7, [srcreg+d4+d2+16]		;; I7
	addpd	xmm7, xmm5			;; I7 = I3 + I7 (new1 I3)

	subpd	xmm4, [srcreg+d4+16]		;; I1 = I1 - I5 (new1 I5)
	subpd	xmm5, [srcreg+d4+d2+16]		;; I3 = I3 - I7 (new1 I7)

	xstore	[dstreg], xmm3			;; Save R1

	xcopy	xmm3, xmm7			;; Copy I3
	addpd	xmm7, xmm6			;; I3 = I1 + I3 (new2 I1)
	subpd	xmm6, xmm3			;; I1 = I1 - I3 (new2 I3)

	xstore	[dstreg+16], xmm7		;; Save I1

	xcopy	xmm3, xmm1			;; Copy R7
	addpd	xmm1, xmm4			;; R7 = I5 + R7 (new2 I5)
	xcopy	xmm7, xmm0			;; Copy R5
	subpd	xmm0, xmm5			;; R5 = R5 - I7 (new2 R5)
	subpd	xmm4, xmm3			;; I5 = I5 - R7 (new2 I7)
	addpd	xmm5, xmm7			;; I7 = R5 + I7 (new2 R7)

	xstore	[dstreg+e1], xmm2		;; Save R3
	xstore	[dstreg+e1+16], xmm6		;; Save I3
	xstore	[dstreg+e2], xmm0		;; Save R5
	xstore	[dstreg+e2+16], xmm1		;; Save I5
	xstore	[dstreg+e2+e1], xmm5		;; Save R7
	xstore	[dstreg+e2+e1+16], xmm4		;; Save I7

	xload	xmm0, [srcreg+d1]		;; R2
	xload	xmm2, [srcreg+d4+d1]		;; R6
	addpd	xmm2, xmm0			;; R6 = R2 + R6 (new1 R2)

	xload	xmm1, [srcreg+d2+d1]		;; R4
	xload	xmm3, [srcreg+d4+d2+d1]		;; R8
	addpd	xmm3, xmm1			;; R8 = R4 + R8 (new1 R4)

	xload	xmm4, [srcreg+d1+16]		;; I2
	xload	xmm6, [srcreg+d4+d1+16]		;; I6
	addpd	xmm6, xmm4			;; I6 = I2 + I6 (new1 I2)

	subpd	xmm0, [srcreg+d4+d1]		;; R2 = R2 - R6 (new1 R6)
	subpd	xmm1, [srcreg+d4+d2+d1]		;; R4 = R4 - R8 (new1 R8)

	xcopy	xmm5, xmm3			;; Copy R4
	addpd	xmm3, xmm2			;; R4 = R2 + R4 (new2 R2)
	subpd	xmm2, xmm5			;; R2 = R2 - R4 (new2 R4)

	xload	xmm5, [srcreg+d2+d1+16]		;; I4
	xload	xmm7, [srcreg+d4+d2+d1+16]	;; I8
	addpd	xmm7, xmm5			;; I8 = I4 + I8 (new1 I4)

	subpd	xmm4, [srcreg+d4+d1+16]		;; I2 = I2 - I6 (new1 I6)
	subpd	xmm5, [srcreg+d4+d2+d1+16]	;; I4 = I4 - I8 (new1 I8)

	xstore	[dstreg+e1+32], xmm2		;; Save R4

	xcopy	xmm2, xmm7			;; Copy I4
	addpd	xmm7, xmm6			;; I4 = I2 + I4 (new2 I2)
	subpd	xmm6, xmm2			;; I2 = I2 - I4 (new2 I4)

	xstore	[dstreg+e1+48], xmm6		;; Save I4

	xcopy	xmm2, xmm1			;; Copy R8
	addpd	xmm1, xmm4			;; R8 = I6 + R8 (new2 I6)
	xcopy	xmm6, xmm0			;; Copy R6
	subpd	xmm0, xmm5			;; R6 = R6 - I8 (new2 R6)
	subpd	xmm4, xmm2			;; I6 = I6 - R8 (new2 I8)
	addpd	xmm5, xmm6			;; I8 = R6 + I8 (new2 R8)

	xcopy	xmm2, xmm0			;; Copy R6
	subpd	xmm0, xmm1			;; R6 = R6 - I6
	addpd	xmm1, xmm2			;; I6 = R6 + I6
	xload	xmm2, XMM_SQRTHALF
	mulpd	xmm0, xmm2			;; R6 = R6 * SQRTHALF (new2 R6)
	mulpd	xmm1, xmm2			;; I6 = I6 * SQRTHALF (new2 I6)

	xcopy	xmm6, xmm5			;; Copy R8
	subpd	xmm5, xmm4			;; R8 = R8 - I8
	addpd	xmm4, xmm6			;; I8 = R8 + I8
	mulpd	xmm5, xmm2			;; R8 = R8 * SQRTHALF (new2 R8)
	mulpd	xmm4, xmm2			;; I8 = I8 * SQRTHALF (new2 I8)

;; the last level

	xload	xmm2, [dstreg]			;; R1
	subpd	xmm2, xmm3			;; R1 = R1 - R2 (new3 R2)
	addpd	xmm3, [dstreg]			;; R2 = R1 + R2 (final R1)
	xload	xmm6, [dstreg+16]		;; I1
	subpd	xmm6, xmm7			;; I1 = I1 - I2 (new3 I2)
	addpd	xmm7, [dstreg+16]		;; I2 = I1 + I2 (final I1)

	xstore	[dstreg], xmm3			;; Save R1
	xstore	[dstreg+16], xmm7		;; Save I1

	xload	xmm3, [screg+scoff+96+16]	;; cosine/sine
	mulpd	xmm3, xmm2			;; A2 = R2 * cosine/sine
	subpd	xmm3, xmm6			;; A2 = A2 - I2
	mulpd	xmm6, [screg+scoff+96+16]	;; B2 = I2 * cosine/sine
	addpd	xmm6, xmm2			;; B2 = B2 + R2

	xstore	[dstreg+e2+e1+32], xmm5		;; Save R8

	mulpd	xmm3, [screg+scoff+96]		;; A2 = A2 * sine (final R2)
	mulpd	xmm6, [screg+scoff+96]		;; B2 = B2 * sine (final I2)
	xstore	[dstreg+32], xmm3		;; Save R2
	xstore	[dstreg+48], xmm6		;; Save I2

	xload	xmm2, [dstreg+e2]		;; R5
	subpd	xmm2, xmm0			;; R5 = R5 - R6 (new3 R6)
	xload	xmm7, [dstreg+e2+16]		;; I5
	subpd	xmm7, xmm1			;; I5 = I5 - I6 (new3 I6)

	xload	xmm5, [screg+scoff+64+16]	;; cosine/sine
	xcopy	xmm3, xmm2			;; Copy R6
	mulpd	xmm2, xmm5			;; A6 = R6 * cosine/sine
	addpd	xmm2, xmm7			;; A6 = A6 + I6
	mulpd	xmm7, xmm5			;; B6 = I6 * cosine/sine
	subpd	xmm7, xmm3			;; B6 = B6 - R6

	xload	xmm3, [dstreg+e2+e1]		;; R7
	subpd	xmm3, xmm4			;; R7 = R7 - I8 (new3 R7)

	xload	xmm6, [screg+scoff+64]
	mulpd	xmm2, xmm6			;; A6 = A6 * sine (final R6)
	mulpd	xmm7, xmm6			;; B6 = B6 * sine (final I6)
	xstore	[dstreg+e2+32], xmm2		;; Save R6
	xstore	[dstreg+e2+48], xmm7		;; Save I6

	xcopy	xmm2, xmm3			;; Copy R7
	mulpd	xmm3, xmm5			;; A7 = R7 * cosine/sine

	xload	xmm7, [dstreg+e2+e1+16]		;; I7
	addpd	xmm7, [dstreg+e2+e1+32]		;; R8 = I7 + R8 (new3 I7)

	subpd	xmm3, xmm7			;; A7 = A7 - I7
	mulpd	xmm7, xmm5			;; B7 = I7 * cosine/sine
	addpd	xmm7, xmm2			;; B7 = B7 + R7

	addpd	xmm0, [dstreg+e2]		;; R6 = R5 + R6 (new3 R5)
	addpd	xmm1, [dstreg+e2+16]		;; I6 = I5 + I6 (new3 I5)
	addpd	xmm4, [dstreg+e2+e1+48]		;; I8 = R7 + I8 (new3 R8)
	xload	xmm5, [dstreg+e2+e1+16]		;; I7
	subpd	xmm5, [dstreg+e2+e1+32]		;; I7 = I7 - R8 (new3 I8)

	mulpd	xmm3, xmm6			;; A7 = A7 * sine (final R7)
	xstore	[dstreg+e2+e1], xmm3		;; Save R7

	xload	xmm2, [screg+scoff+0+16]	;; cosine/sine
	xcopy	xmm3, xmm0			;; Copy R5
	mulpd	xmm0, xmm2			;; A5 = R5 * cosine/sine
	subpd	xmm0, xmm1			;; A5 = A5 - I5
	mulpd	xmm1, xmm2			;; B5 = I5 * cosine/sine

	mulpd	xmm7, xmm6			;; B7 = B7 * sine (final I7)

	xcopy	xmm6, xmm4			;; Copy R8
	mulpd	xmm4, xmm2			;; A8 = R8 * cosine/sine
	addpd	xmm4, xmm5			;; A8 = A8 + I8
	mulpd	xmm5, xmm2			;; B8 = I8 * cosine/sine

	addpd	xmm1, xmm3			;; B5 = B5 + R5
	subpd	xmm5, xmm6			;; B8 = B8 - R8

	xload	xmm6, [dstreg+e1]		;; R3
	xload	xmm2, [dstreg+e1+48]		;; I4
	subpd	xmm6, xmm2			;; R3 = R3 - I4 (new3 R3)
	addpd	xmm2, [dstreg+e1]		;; I4 = R3 + I4 (new3 R4)

	xstore	[dstreg+e2+e1+16], xmm7		;; Save I7

	xload	xmm3, [screg+scoff+0]
	mulpd	xmm0, xmm3			;; A5 = A5 * sine (final R5)
	xstore	[dstreg+e2], xmm0		;; Save R5

	xload	xmm7, [screg+scoff+32+16]	;; cosine/sine
	xcopy	xmm0, xmm6			;; Copy R3
	mulpd	xmm6, xmm7			;; A3 = R3 * cosine/sine

	mulpd	xmm1, xmm3			;; B5 = B5 * sine (final I5)
	xstore	[dstreg+e2+16], xmm1		;; Save I5

	xcopy	xmm1, xmm2			;; Copy R4
	mulpd	xmm2, xmm7			;; A4 = R4 * cosine/sine

	mulpd	xmm4, xmm3			;; A8 = A8 * sine (final R8)
	xstore	[dstreg+e2+e1+32], xmm4		;; Save R8
	mulpd	xmm5, xmm3			;; B8 = B8 * sine (final I8)

	xload	xmm4, [dstreg+e1+16]		;; I3
	xload	xmm3, [dstreg+e1+32]		;; R4
	subpd	xmm4, xmm3			;; I3 = I3 - R4 (new3 I4)
	addpd	xmm3, [dstreg+e1+16]		;; R4 = I3 + R4 (new3 I3)

	subpd	xmm6, xmm3			;; A3 = A3 - I3
	mulpd	xmm3, xmm7			;; B3 = I3 * cosine/sine
	addpd	xmm3, xmm0			;; B3 = B3 + R3

	addpd	xmm2, xmm4			;; A4 = A4 + I4
	mulpd	xmm4, xmm7			;; B4 = I4 * cosine/sine
	subpd	xmm4, xmm1			;; B4 = B4 - R4

	xload	xmm7, [screg+scoff+32]
	mulpd	xmm6, xmm7			;; A3 = A3 * sine (final R3)
	mulpd	xmm3, xmm7			;; B3 = B3 * sine (final I3)
	mulpd	xmm2, xmm7			;; A4 = A4 * sine (final R4)
	mulpd	xmm4, xmm7			;; B4 = B4 * sine (final I4)

	xstore	[dstreg+e2+e1+48], xmm5		;; Save I8
	xstore	[dstreg+e1], xmm6		;; Save R3
	xstore	[dstreg+e1+16], xmm3		;; Save I3
	xstore	[dstreg+e1+32], xmm2		;; Save R4
	xstore	[dstreg+e1+48], xmm4		;; Save I4
	ENDM
ENDIF

;;
;; ************************************* eight-complex-with-square and variants ******************************************
;;

;;
;; These macros are used in the last levels of pass 2 in two pass FFTs.
;;

;;
;; The last three levels of the forward FFT are performed.
;; No sin/cos multipliers are needed.
;;

r8_x8cl_eight_complex_fft_final_preload MACRO
	ENDM

r8_x8cl_eight_complex_fft_final MACRO srcreg,srcinc,d1,d2,d4
	r8_x8c_simple_fft_part1 srcreg+0,d1,d2,d4,XMM_COL_MULTS
	r8_x8c_simple_fft_part1 srcreg+32,d1,d2,d4,XMM_COL_MULTS[256]
	r8_x8c_simple_fft_part2 XMM_COL_MULTS,srcreg+0,d1,d2
	r8_x8c_simple_fft_part2 XMM_COL_MULTS[256],srcreg+d4,d1,d2
	bump	srcreg, srcinc
	ENDM

r8_x8c_simple_fft_part1 MACRO srcreg,d1,d2,d4,dst
	xload	xmm0, [srcreg]			;; R1
	xload	xmm2, [srcreg+d4]		;; R5
	xcopy	xmm7, xmm2			;; Copy R5
	addpd	xmm2, xmm0			;; R5 = R1 + R5 (new R1)
	subpd	xmm0, xmm7			;; R1 = R1 - R5 (new R5)

	xload	xmm1, [srcreg+d2]		;; R3
	xload	xmm3, [srcreg+d4+d2]		;; R7
	xcopy	xmm7, xmm3			;; Copy R7
	addpd	xmm3, xmm1			;; R7 = R3 + R7 (new R3)
	subpd	xmm1, xmm7			;; R3 = R3 - R7 (new R7)

	xload	xmm4, [srcreg+16]		;; I1
	xload	xmm6, [srcreg+d4+16]		;; I5
	xcopy	xmm7, xmm6			;; Copy I5
	addpd	xmm6, xmm4			;; I5 = I1 + I5 (new I1)
	subpd	xmm4, xmm7			;; I1 = I1 - I5 (new I5)

	xcopy	xmm7, xmm3			;; Copy R3
	addpd	xmm3, xmm2			;; R3 = R1 + R3 (final R1)
	subpd	xmm2, xmm7			;; R1 = R1 - R3 (final R3)

	xstore	dst[0], xmm3			;; Save R1
	xstore	dst[d1], xmm2			;; Save R3

	xload	xmm5, [srcreg+d2+16]		;; I3
	xload	xmm3, [srcreg+d4+d2+16]		;; I7
	xcopy	xmm7, xmm3			;; Copy I7
	addpd	xmm3, xmm5			;; I7 = I3 + I7 (new I3)
	subpd	xmm5, xmm7			;; I3 = I3 - I7 (new I7)

	xcopy	xmm7, xmm3			;; Copy I3
	addpd	xmm3, xmm6			;; I3 = I1 + I3 (final I1)
	subpd	xmm6, xmm7			;; I1 = I1 - I3 (final I3)

	xstore	dst[32], xmm3			;; Save I1
	xstore	dst[d1+32], xmm6		;; Save I3

	xcopy	xmm3, xmm1			;; Copy R7
	addpd	xmm1, xmm4			;; R7 = I5 + R7 (final I5)
	xcopy	xmm7, xmm0			;; Copy R5
	subpd	xmm0, xmm5			;; R5 = R5 - I7 (final R5)
	subpd	xmm4, xmm3			;; I5 = I5 - R7 (final I7)
	addpd	xmm5, xmm7			;; I7 = R5 + I7 (final R7)

	xstore	dst[d2+32], xmm1		;; Save I5
	xstore	dst[d2], xmm0			;; Save R5
	xstore	dst[d2+d1+32], xmm4		;; Save I7
	xstore	dst[d2+d1], xmm5		;; Save R7

	xload	xmm0, [srcreg+d1]		;; R2
	xload	xmm2, [srcreg+d4+d1]		;; R6
	xcopy	xmm7, xmm2			;; Copy R6
	addpd	xmm2, xmm0			;; R6 = R2 + R6 (new R2)
	subpd	xmm0, xmm7			;; R2 = R2 - R6 (new R6)

	xload	xmm1, [srcreg+d2+d1]		;; R4
	xload	xmm3, [srcreg+d4+d2+d1]		;; R8
	xcopy	xmm7, xmm3			;; Copy R8
	addpd	xmm3, xmm1			;; R8 = R4 + R8 (new R4)
	subpd	xmm1, xmm7			;; R4 = R4 - R8 (new R8)

	xload	xmm4, [srcreg+d1+16]		;; I2
	xload	xmm6, [srcreg+d4+d1+16]		;; I6
	xcopy	xmm7, xmm6			;; Copy I6
	addpd	xmm6, xmm4			;; I6 = I2 + I6 (new I2)
	subpd	xmm4, xmm7			;; I2 = I2 - I6 (new I6)

	xcopy	xmm7, xmm3			;; Copy R4
	addpd	xmm3, xmm2			;; R4 = R2 + R4 (final R2)
	subpd	xmm2, xmm7			;; R2 = R2 - R4 (final R4)

	xstore	dst[16], xmm3			;; Save R2
	xstore	dst[d1+16], xmm2		;; Save R4

	xload	xmm5, [srcreg+d2+d1+16]		;; I4
	xload	xmm3, [srcreg+d4+d2+d1+16]	;; I8
	xcopy	xmm7, xmm3			;; Copy I8
	addpd	xmm3, xmm5			;; I8 = I4 + I8 (new I4)
	subpd	xmm5, xmm7			;; I4 = I4 - I8 (new I8)

	xcopy	xmm7, xmm3			;; Copy I4
	addpd	xmm3, xmm6			;; I4 = I2 + I4 (final I2)
	subpd	xmm6, xmm7			;; I2 = I2 - I4 (final I4)

	xstore	dst[48], xmm3			;; Save I2
	xstore	dst[d1+48], xmm6		;; Save I4

	xcopy	xmm3, xmm1			;; Copy R8
	addpd	xmm1, xmm4			;; R8 = I6 + R8 (new I6)
	xcopy	xmm7, xmm0			;; Copy R6
	subpd	xmm0, xmm5			;; R6 = R6 - I8 (new R6)
	subpd	xmm4, xmm3			;; I6 = I6 - R8 (new I8)
	addpd	xmm5, xmm7			;; I8 = R6 + I8 (new R8)

	xcopy	xmm7, xmm0			;; Copy R6
	subpd	xmm0, xmm1			;; R6 = R6 - I6
	addpd	xmm1, xmm7			;; I6 = R6 + I6
	xload	xmm3, XMM_SQRTHALF
	mulpd	xmm0, xmm3			;; R6 = R6 * SQRTHALF (final R6)
	mulpd	xmm1, xmm3			;; I6 = I6 * SQRTHALF (final I6)

	xcopy	xmm7, xmm5			;; Copy R8
	subpd	xmm5, xmm4			;; R8 = R8 - I8
	addpd	xmm4, xmm7			;; I8 = R8 + I8
	mulpd	xmm5, xmm3			;; R8 = R8 * SQRTHALF (final R8)
	mulpd	xmm4, xmm3			;; I8 = I8 * SQRTHALF (final I8)

	xstore	dst[d2+16], xmm0		;; Save R6
	xstore	dst[d2+48], xmm1		;; Save I6
	xstore	dst[d2+d1+16], xmm5		;; Save R8
	xstore	dst[d2+d1+48], xmm4		;; Save I8
	ENDM

r8_x8c_simple_fft_part2 MACRO src,dstreg,d1,d2
	xload	xmm2, src[d2]			;; R5
	xload	xmm0, src[d2+16] 		;; R6
	xcopy	xmm7, xmm2			;; Copy R5
	subpd	xmm2, xmm0			;; R5 = R5 - R6 (new R6)
	addpd	xmm0, xmm7			;; R6 = R5 + R6 (new R5)

	xload	xmm3, src[d2+32]		;; I5
	xload	xmm1, src[d2+48] 		;; I6
	xcopy	xmm7, xmm3			;; Copy I5
	subpd	xmm3, xmm1			;; I5 = I5 - I6 (new I6)
	addpd	xmm1, xmm7			;; I6 = I5 + I6 (new I5)

	xload	xmm4, src[d2+d1]		;; R7
	xload	xmm5, src[d2+d1+48]		;; I8
	xcopy	xmm7, xmm4			;; Copy R7
	subpd	xmm4, xmm5			;; R7 = R7 - I8 (new R7)
	addpd	xmm5, xmm7			;; I8 = R7 + I8 (new R8)

	xstore	[dstreg+d2+16], xmm2		;; Save R6
	xstore	[dstreg+d2], xmm0		;; Save R5
	xstore	[dstreg+d2+48], xmm3		;; Save I6
	xstore	[dstreg+d2+32], xmm1		;; Save I5
	xstore	[dstreg+d2+d1], xmm4		;; Save R7
	xstore	[dstreg+d2+d1+16], xmm5		;; Save R8

	xload	xmm0, src[d2+d1+32]		;; I7
	xload	xmm1, src[d2+d1+16]		;; R8
	xcopy	xmm7, xmm0			;; Copy I7
	subpd	xmm0, xmm1			;; I7 = I7 - R8 (new I8)
	addpd	xmm1, xmm7			;; R8 = I7 + R8 (new I7)

	xload	xmm3, src[0]			;; R1
	xload	xmm4, src[16]			;; R2
	xcopy	xmm7, xmm3			;; Copy R1
	subpd	xmm3, xmm4			;; R1 = R1 - R2 (new R2)
	addpd	xmm4, xmm7			;; R2 = R1 + R2 (new R1)

	xload	xmm5, src[32]			;; I1
	xload	xmm6, src[48]			;; I2
	xcopy	xmm7, xmm5			;; Copy I1
	subpd	xmm5, xmm6			;; I1 = I1 - I2 (new I2)
	addpd	xmm6, xmm7			;; I2 = I1 + I2 (new I1)

	xstore	[dstreg+d2+d1+48], xmm0		;; Save I8
	xstore	[dstreg+d2+d1+32], xmm1		;; Save I7
	xstore	[dstreg+16], xmm3		;; Save R2
	xstore	[dstreg], xmm4			;; Save R1
	xstore	[dstreg+48], xmm5		;; Save I2
	xstore	[dstreg+32], xmm6		;; Save I1

	xload	xmm5, src[d1]			;; R3
	xload	xmm6, src[d1+48]		;; I4
	xcopy	xmm7, xmm5			;; Copy R3
	subpd	xmm5, xmm6			;; R3 = R3 - I4 (new R3)
	addpd	xmm6, xmm7			;; I4 = R3 + I4 (new R4)

	xload	xmm3, src[d1+32]		;; I3
	xload	xmm4, src[d1+16]		;; R4
	xcopy	xmm7, xmm3			;; Copy I3
	subpd	xmm3, xmm4			;; I3 = I3 - R4 (new I4)
	addpd	xmm4, xmm7			;; R4 = I3 + R4 (new I3)

	xstore	[dstreg+d1], xmm5		;; Save R3
	xstore	[dstreg+d1+16], xmm6		;; Save R4
	xstore	[dstreg+d1+48], xmm3		;; Save I4
	xstore	[dstreg+d1+32], xmm4		;; Save I3
	ENDM

;;
;; The last three levels of the forward FFT are performed, point-wise
;; squaring, and first three levels of the inverse FFT are performed.
;; No sin/cos multipliers are needed.
;;

r8_x8cl_eight_complex_with_square_preload MACRO
	ENDM

r8_x8cl_eight_complex_with_square MACRO srcreg,srcinc,d1,d2,d4
	r8_x8c_simple_fft_part1 srcreg+0,d1,d2,d4,XMM_COL_MULTS
	r8_x8c_simple_fft_part1 srcreg+32,d1,d2,d4,XMM_COL_MULTS[256]
	r8_x8c_simple_fft_with_square XMM_COL_MULTS,d1,d2
	r8_x8c_simple_fft_with_square XMM_COL_MULTS[256],d1,d2
	r8_x8c_simple_unfft XMM_COL_MULTS,srcreg+0,d1,d2
	r8_x8c_simple_unfft XMM_COL_MULTS[256],srcreg+d4,d1,d2
	bump	srcreg, srcinc
	ENDM

r8_x8c_simple_fft_with_square MACRO src,d1,d2
	xload	xmm2, src[d2]			;; R5
	xload	xmm0, src[d2+16] 		;; R6
	xcopy	xmm7, xmm2			;; Copy R5
	subpd	xmm2, xmm0			;; R5 = R5 - R6 (new R6)
	addpd	xmm0, xmm7			;; R6 = R5 + R6 (new R5)

	xload	xmm3, src[d2+32]		;; I5
	xload	xmm1, src[d2+48] 		;; I6
	xcopy	xmm7, xmm3			;; Copy I5
	subpd	xmm3, xmm1			;; I5 = I5 - I6 (new I6)
	addpd	xmm1, xmm7			;; I6 = I5 + I6 (new I5)

	xp_complex_square xmm2, xmm3, xmm7	;; Square R6/I6
	xp_complex_square xmm0, xmm1, xmm7	;; Square R5/I5

	xcopy	xmm7, xmm0			;; Copy R5
	subpd	xmm0, xmm2			;; R5 = R5 - R6 (new R6)
	addpd	xmm2, xmm7			;; R6 = R5 + R6 (new R5)

	xcopy	xmm7, xmm1			;; Copy I5
	subpd	xmm1, xmm3			;; I5 = I5 - I6 (new I6)
	addpd	xmm3, xmm7			;; I6 = I5 + I6 (new I5)

	xstore	src[d2+16], xmm0		;; Save R6
	xstore	src[d2], xmm2			;; Save R5
	xstore	src[d2+48], xmm1		;; Save I6
	xstore	src[d2+32], xmm3		;; Save I5

	xload	xmm4, src[d2+d1]		;; R7
	xload	xmm5, src[d2+d1+48]		;; I8
	xcopy	xmm7, xmm4			;; Copy R7
	subpd	xmm4, xmm5			;; R7 = R7 - I8 (new R7)
	addpd	xmm5, xmm7			;; I8 = R7 + I8 (new R8)

	xload	xmm0, src[d2+d1+32]		;; I7
	xload	xmm1, src[d2+d1+16]		;; R8
	xcopy	xmm7, xmm0			;; Copy I7
	subpd	xmm0, xmm1			;; I7 = I7 - R8 (new I8)
	addpd	xmm1, xmm7			;; R8 = I7 + R8 (new I7)

	xp_complex_square xmm5, xmm0, xmm7	;; Square R8/I8
	xp_complex_square xmm4, xmm1, xmm7	;; Square R7/I7

	xcopy	xmm7, xmm5			;; Copy R8
	subpd	xmm5, xmm4			;; R8 = R8 - R7 (new I8)
	addpd	xmm4, xmm7			;; R7 = R8 + R7 (new R7)

	xcopy	xmm7, xmm1			;; Copy I7
	subpd	xmm1, xmm0			;; I7 = I7 - I8 (new R8)
	addpd	xmm0, xmm7			;; I8 = I7 + I8 (new I7)

	xstore	src[d2+d1+48], xmm5		;; Save I8
	xstore	src[d2+d1], xmm4		;; Save R7
	xstore	src[d2+d1+16], xmm1		;; Save R8
	xstore	src[d2+d1+32], xmm0		;; Save I7

	xload	xmm3, src[0]			;; R1
	xload	xmm4, src[16]			;; R2
	xcopy	xmm7, xmm3			;; Copy R1
	subpd	xmm3, xmm4			;; R1 = R1 - R2 (new R2)
	addpd	xmm4, xmm7			;; R2 = R1 + R2 (new R1)

	xload	xmm5, src[32]			;; I1
	xload	xmm6, src[48]			;; I2
	xcopy	xmm7, xmm5			;; Copy I1
	subpd	xmm5, xmm6			;; I1 = I1 - I2 (new I2)
	addpd	xmm6, xmm7			;; I2 = I1 + I2 (new I1)

	xp_complex_square xmm3, xmm5, xmm7	;; Square R2/I2
	xp_complex_square xmm4, xmm6, xmm7	;; Square R1/I1

	xcopy	xmm7, xmm4			;; Copy R1
	subpd	xmm4, xmm3			;; R1 = R1 - R2 (new R2)
	addpd	xmm3, xmm7			;; R2 = R1 + R2 (new R1)

	xcopy	xmm7, xmm6			;; Copy I1
	subpd	xmm6, xmm5			;; I1 = I1 - I2 (new I2)
	addpd	xmm5, xmm7			;; I2 = I1 + I2 (new I1)

	xstore	src[16], xmm4			;; Save R2
	xstore	src[0], xmm3			;; Save R1
	xstore	src[48], xmm6			;; Save I2
	xstore	src[32], xmm5			;; Save I1

	xload	xmm5, src[d1]			;; R3
	xload	xmm6, src[d1+48]		;; I4
	xcopy	xmm7, xmm5			;; Copy R3
	subpd	xmm5, xmm6			;; R3 = R3 - I4 (new R3)
	addpd	xmm6, xmm7			;; I4 = R3 + I4 (new R4)

	xload	xmm3, src[d1+32]		;; I3
	xload	xmm4, src[d1+16]		;; R4
	xcopy	xmm7, xmm3			;; Copy I3
	subpd	xmm3, xmm4			;; I3 = I3 - R4 (final I4)
	addpd	xmm4, xmm7			;; R4 = I3 + R4 (final I3)

	xp_complex_square xmm6, xmm3, xmm7	;; Square R4/I4
	xp_complex_square xmm5, xmm4, xmm7	;; Square R3/I3

	xcopy	xmm7, xmm6			;; Copy R4
	subpd	xmm6, xmm5			;; R4 = R4 - R3 (new I4)
	addpd	xmm5, xmm7			;; R3 = R4 + R3 (new R3)

	xcopy	xmm7, xmm4			;; Copy I3
	subpd	xmm4, xmm3			;; I3 = I3 - I4 (new R4)
	addpd	xmm3, xmm7			;; I4 = I3 + I4 (new I3)

	xstore	src[d1+48], xmm6		;; Save I4
	xstore	src[d1], xmm5			;; Save R3
	xstore	src[d1+16], xmm4		;; Save R4
	xstore	src[d1+32], xmm3		;; Save I3
	ENDM

r8_x8c_simple_unfft MACRO src,dstreg,d1,d2
	xload	xmm5, src[d2+32]		;; I5
	xload	xmm1, src[d2+d1+32]		;; I7
	xcopy	xmm7, xmm5			;; Copy I5
	subpd	xmm5, xmm1			;; I5 = I5 - I7 (new R7)
	addpd	xmm1, xmm7			;; I7 = I5 + I7 (new I5)

	xload	xmm4, src[0]			;; R1
	xload	xmm2, src[d1]			;; R3
	xcopy	xmm7, xmm4			;; Copy R1
	subpd	xmm4, xmm2			;; R1 = R1 - R3 (new R3)
	addpd	xmm2, xmm7			;; R3 = R1 + R3 (new R1)

	xcopy	xmm7, xmm4			;; Copy R3
	subpd	xmm4, xmm5			;; R3 = R3 - R7 (final R7)
	addpd	xmm5, xmm7			;; R7 = R3 + R7 (final R3)

	xstore	[dstreg+d2+16], xmm4		;; Save R7
	xstore	[dstreg+d2], xmm5		;; Save R3

	xload	xmm6, src[d2]			;; R5
	xload	xmm0, src[d2+d1]		;; R7
	xcopy	xmm7, xmm6			;; Copy R5
	addpd	xmm6, xmm0			;; R5 = R7 + R5 (new R5)
	subpd	xmm0, xmm7			;; R7 = R7 - R5 (new I7)

	xload	xmm3, src[32]			;; I1
	xload	xmm4, src[d1+32]		;; I3
	xcopy	xmm7, xmm3			;; Copy I1
	subpd	xmm3, xmm4			;; I1 = I1 - I3 (new I3)
	addpd	xmm4, xmm7			;; I3 = I1 + I3 (new I1)

	xcopy	xmm7, xmm3			;; Copy I3
	subpd	xmm3, xmm0			;; I3 = I3 - I7 (final I7)
	addpd	xmm0, xmm7			;; I7 = I3 + I7 (final I3)

	xstore	[dstreg+d2+48], xmm3		;; I7
	xstore	[dstreg+d2+32], xmm0		;; I3

	xcopy	xmm7, xmm2			;; Copy R1
	subpd	xmm2, xmm6			;; R1 = R1 - R5 (final R5)
	addpd	xmm6, xmm7			;; R5 = R1 + R5 (final R1)

	xcopy	xmm7, xmm4			;; Copy I1
	subpd	xmm4, xmm1			;; I1 = I1 - I5 (final I5)
	addpd	xmm1, xmm7			;; I5 = I1 + I5 (final I1)

	xstore	[dstreg+16], xmm2		;; R5
	xstore	[dstreg], xmm6			;; R1
	xstore	[dstreg+48], xmm4		;; I5
	xstore	[dstreg+32], xmm1		;; I1


	;; multiply R6/I6 by SQRTHALF - i*SQRTHALF
	xload	xmm4, src[d2+16]		;; R6
	xload	xmm0, src[d2+48]		;; I6
	xcopy	xmm7, xmm0			;; Copy I6
	subpd	xmm0, xmm4			;; I6 = I6 - R6
	addpd	xmm4, xmm7			;; R6 = R6 + I6

	;; multiply R8/I8 by SQRTHALF - i*SQRTHALF
	xload	xmm2, src[d2+d1+16]		;; R8
	xload	xmm1, src[d2+d1+48]		;; I8
	xcopy	xmm7, xmm1			;; Copy I8
	subpd	xmm1, xmm2			;; I8 = I8 - R8
	addpd	xmm2, xmm7			;; R8 = R8 + I8

	xload	xmm5, XMM_SQRTHALF
	mulpd	xmm0, xmm5			;; I6 = I6 * SQRTHALF (new I6)
	mulpd	xmm4, xmm5			;; R6 = R6 * SQRTHALF (new R6)
	mulpd	xmm1, xmm5			;; I8 = I8 * SQRTHALF (new I8)
	mulpd	xmm2, xmm5			;; R8 = R8 * SQRTHALF (new R8)

	xload	xmm6, src[16]			;; R2
	xload	xmm3, src[d1+16]		;; R4
	xcopy	xmm7, xmm3			;; Copy R4
	addpd	xmm3, xmm6			;; R4 = R2 + R4 (new R2)
	subpd	xmm6, xmm7			;; R2 = R2 - R4 (new R4)

	xcopy	xmm7, xmm2			;; Copy R8
	subpd	xmm2, xmm4			;; R8 = R8 - R6 (new I8)
	addpd	xmm4, xmm7			;; R6 = R8 + R6 (new R6)

	xcopy	xmm7, xmm0			;; Copy I6
	subpd	xmm0, xmm1			;; I6 = I6 - I8 (new R8)
	addpd	xmm1, xmm7			;; I8 = I6 + I8 (new I6)

	xcopy	xmm7, xmm3			;; Copy R2
	subpd	xmm3, xmm4			;; R2 = R2 - R6 (final R6)
	addpd	xmm4, xmm7			;; R6 = R2 + R6 (final R2)

	xstore	[dstreg+d1+16], xmm3		;; R6
	xstore	[dstreg+d1], xmm4		;; R2

	xload	xmm5, src[48]			;; I2
	xload	xmm3, src[d1+48]		;; I4
	xcopy	xmm7, xmm3			;; Copy I4
	addpd	xmm3, xmm5			;; I4 = I2 + I4 (new I2)
	subpd	xmm5, xmm7			;; I2 = I2 - I4 (new I4)

	xcopy	xmm7, xmm6			;; Copy R4
	subpd	xmm6, xmm0			;; R4 = R4 - R8 (final R8)
	addpd	xmm0, xmm7			;; R8 = R4 + R8 (final R4)

	xcopy	xmm7, xmm3			;; Copy I2
	subpd	xmm3, xmm1			;; I2 = I2 - I6 (final I6)
	addpd	xmm1, xmm7			;; I6 = I2 + I6 (final I2)

	xcopy	xmm7, xmm5			;; Copy I4
	subpd	xmm5, xmm2			;; I4 = I4 - I8 (final I8)
	addpd	xmm2, xmm7			;; I8 = I4 + I8 (final I4)

	xstore	[dstreg+d2+d1+16], xmm6		;; R8
	xstore	[dstreg+d2+d1], xmm0		;; R4
	xstore	[dstreg+d1+48], xmm3		;; I6
	xstore	[dstreg+d1+32], xmm1		;; I2
	xstore	[dstreg+d2+d1+48], xmm5		;; I8
	xstore	[dstreg+d2+d1+32], xmm2		;; I4
	ENDM

;;
;; The last three levels of the forward FFT are performed, point-wise
;; multiplication, and first three levels of the inverse FFT are performed.
;; No sin/cos multipliers are needed.
;;

r8_x8cl_eight_complex_with_mult_preload MACRO
	ENDM

r8_x8cl_eight_complex_with_mult MACRO srcreg,srcinc,d1,d2,d4
	r8_x8c_simple_fft_part1 srcreg+0,d1,d2,d4,XMM_COL_MULTS
	r8_x8c_simple_fft_part1 srcreg+32,d1,d2,d4,XMM_COL_MULTS[256]
	r8_x8c_simple_fft_with_mult XMM_COL_MULTS,srcreg+rbp,d1,d2
	r8_x8c_simple_fft_with_mult XMM_COL_MULTS[256],srcreg+d4+rbp,d1,d2
	r8_x8c_simple_unfft XMM_COL_MULTS,srcreg+0,d1,d2
	r8_x8c_simple_unfft XMM_COL_MULTS[256],srcreg+d4,d1,d2
	bump	srcreg, srcinc
	ENDM

r8_x8c_simple_fft_with_mult MACRO src,altsrc,d1,d2
	xload	xmm2, src[d2]			;; R5
	xload	xmm0, src[d2+16] 		;; R6
	xcopy	xmm7, xmm2			;; Copy R5
	subpd	xmm2, xmm0			;; R5 = R5 - R6 (new R6)
	addpd	xmm0, xmm7			;; R6 = R5 + R6 (new R5)

	xload	xmm3, src[d2+32]		;; I5
	xload	xmm1, src[d2+48] 		;; I6
	xcopy	xmm7, xmm3			;; Copy I5
	subpd	xmm3, xmm1			;; I5 = I5 - I6 (new I6)
	addpd	xmm1, xmm7			;; I6 = I5 + I6 (new I5)

	xp_complex_mult xmm2, xmm3, XX [altsrc+d2+16], XX [altsrc+d2+48], xmm6, xmm7 ;; Mult R6/I6
	xp_complex_mult xmm0, xmm1, XX [altsrc+d2], XX [altsrc+d2+32], xmm6, xmm7 ;; Mult R5/I5

	xcopy	xmm7, xmm0			;; Copy R5
	subpd	xmm0, xmm2			;; R5 = R5 - R6 (new R6)
	addpd	xmm2, xmm7			;; R6 = R5 + R6 (new R5)

	xcopy	xmm7, xmm1			;; Copy I5
	subpd	xmm1, xmm3			;; I5 = I5 - I6 (new I6)
	addpd	xmm3, xmm7			;; I6 = I5 + I6 (new I5)

	xstore	src[d2+16], xmm0		;; Save R6
	xstore	src[d2], xmm2			;; Save R5
	xstore	src[d2+48], xmm1		;; Save I6
	xstore	src[d2+32], xmm3		;; Save I5

	xload	xmm4, src[d2+d1]		;; R7
	xload	xmm5, src[d2+d1+48]		;; I8
	xcopy	xmm7, xmm4			;; Copy R7
	subpd	xmm4, xmm5			;; R7 = R7 - I8 (new R7)
	addpd	xmm5, xmm7			;; I8 = R7 + I8 (new R8)

	xload	xmm0, src[d2+d1+32]		;; I7
	xload	xmm1, src[d2+d1+16]		;; R8
	xcopy	xmm7, xmm0			;; Copy I7
	subpd	xmm0, xmm1			;; I7 = I7 - R8 (new I8)
	addpd	xmm1, xmm7			;; R8 = I7 + R8 (new I7)

	xp_complex_mult xmm5, xmm0, XX [altsrc+d2+d1+16], XX [altsrc+d2+d1+48], xmm6, xmm7 ;; Mult R8/I8
	xp_complex_mult xmm4, xmm1, XX [altsrc+d2+d1], XX [altsrc+d2+d1+32], xmm6, xmm7 ;; Mult R7/I7

	xcopy	xmm7, xmm5			;; Copy R8
	subpd	xmm5, xmm4			;; R8 = R8 - R7 (new I8)
	addpd	xmm4, xmm7			;; R7 = R8 + R7 (new R7)

	xcopy	xmm7, xmm1			;; Copy I7
	subpd	xmm1, xmm0			;; I7 = I7 - I8 (new R8)
	addpd	xmm0, xmm7			;; I8 = I7 + I8 (new I7)

	xstore	src[d2+d1+48], xmm5		;; Save I8
	xstore	src[d2+d1], xmm4		;; Save R7
	xstore	src[d2+d1+16], xmm1		;; Save R8
	xstore	src[d2+d1+32], xmm0		;; Save I7

	xload	xmm3, src[0]			;; R1
	xload	xmm4, src[16]			;; R2
	xcopy	xmm7, xmm3			;; Copy R1
	subpd	xmm3, xmm4			;; R1 = R1 - R2 (new R2)
	addpd	xmm4, xmm7			;; R2 = R1 + R2 (new R1)

	xload	xmm5, src[32]			;; I1
	xload	xmm6, src[48]			;; I2
	xcopy	xmm7, xmm5			;; Copy I1
	subpd	xmm5, xmm6			;; I1 = I1 - I2 (new I2)
	addpd	xmm6, xmm7			;; I2 = I1 + I2 (new I1)

	xp_complex_mult xmm3, xmm5, XX [altsrc+16], XX [altsrc+48], xmm0, xmm7 ;; Mult R2/I2
	xp_complex_mult xmm4, xmm6, XX [altsrc], XX [altsrc+32], xmm0, xmm7 ;; Mult R1/I1

	xcopy	xmm7, xmm4			;; Copy R1
	subpd	xmm4, xmm3			;; R1 = R1 - R2 (new R2)
	addpd	xmm3, xmm7			;; R2 = R1 + R2 (new R1)

	xcopy	xmm7, xmm6			;; Copy I1
	subpd	xmm6, xmm5			;; I1 = I1 - I2 (new I2)
	addpd	xmm5, xmm7			;; I2 = I1 + I2 (new I1)

	xstore	src[16], xmm4			;; Save R2
	xstore	src[0], xmm3			;; Save R1
	xstore	src[48], xmm6			;; Save I2
	xstore	src[32], xmm5			;; Save I1

	xload	xmm5, src[d1]			;; R3
	xload	xmm6, src[d1+48]		;; I4
	xcopy	xmm7, xmm5			;; Copy R3
	subpd	xmm5, xmm6			;; R3 = R3 - I4 (new R3)
	addpd	xmm6, xmm7			;; I4 = R3 + I4 (new R4)

	xload	xmm3, src[d1+32]		;; I3
	xload	xmm4, src[d1+16]		;; R4
	xcopy	xmm7, xmm3			;; Copy I3
	subpd	xmm3, xmm4			;; I3 = I3 - R4 (final I4)
	addpd	xmm4, xmm7			;; R4 = I3 + R4 (final I3)

	xp_complex_mult xmm6, xmm3, XX [altsrc+d1+16], XX [altsrc+d1+48], xmm0, xmm7 ;; Mult R4/I4
	xp_complex_mult xmm5, xmm4, XX [altsrc+d1], XX [altsrc+d1+32], xmm0, xmm7 ;; Mult R3/I3

	xcopy	xmm7, xmm6			;; Copy R4
	subpd	xmm6, xmm5			;; R4 = R4 - R3 (new I4)
	addpd	xmm5, xmm7			;; R3 = R4 + R3 (new R3)

	xcopy	xmm7, xmm4			;; Copy I3
	subpd	xmm4, xmm3			;; I3 = I3 - I4 (new R4)
	addpd	xmm3, xmm7			;; I4 = I3 + I4 (new I3)

	xstore	src[d1+48], xmm6		;; Save I4
	xstore	src[d1], xmm5			;; Save R3
	xstore	src[d1+16], xmm4		;; Save R4
	xstore	src[d1+32], xmm3		;; Save I3
	ENDM

;;
;; Point-wise multiplication and first three levels of the inverse FFT are performed.
;; No sin/cos multipliers are needed.
;;

r8_x8cl_eight_complex_with_mulf_preload MACRO
	ENDM

r8_x8cl_eight_complex_with_mulf MACRO srcreg,srcinc,d1,d2,d4
	r8_x8c_simple_fft_with_mulf srcreg,d1,d2,XMM_COL_MULTS
	r8_x8c_simple_fft_with_mulf srcreg+d4,d1,d2,XMM_COL_MULTS[256]
	r8_x8c_simple_unfft XMM_COL_MULTS,srcreg+0,d1,d2
	r8_x8c_simple_unfft XMM_COL_MULTS[256],srcreg+d4,d1,d2
	bump	srcreg, srcinc
	ENDM

r8_x8c_simple_fft_with_mulf MACRO srcreg,d1,d2,dst
	xload	xmm2, [srcreg+d2+16][rbx]	;; R6
	xload	xmm3, [srcreg+d2+48][rbx]	;; I6
	xload	xmm0, [srcreg+d2][rbx]		;; R5
	xload	xmm1, [srcreg+d2+32][rbx]	;; I5

	xp_complex_mult xmm2, xmm3, XX [srcreg+d2+16][rbp], XX [srcreg+d2+48][rbp], xmm6, xmm7 ;; Mult R6/I6
	xp_complex_mult xmm0, xmm1, XX [srcreg+d2][rbp], XX [srcreg+d2+32][rbp], xmm6, xmm7 ;; Mult R5/I5

	xcopy	xmm7, xmm0			;; Copy R5
	subpd	xmm0, xmm2			;; R5 = R5 - R6 (new R6)
	addpd	xmm2, xmm7			;; R6 = R5 + R6 (new R5)

	xcopy	xmm7, xmm1			;; Copy I5
	subpd	xmm1, xmm3			;; I5 = I5 - I6 (new I6)
	addpd	xmm3, xmm7			;; I6 = I5 + I6 (new I5)

	xstore	dst[d2+16], xmm0		;; Save R6
	xstore	dst[d2], xmm2			;; Save R5
	xstore	dst[d2+48], xmm1		;; Save I6
	xstore	dst[d2+32], xmm3		;; Save I5

	xload	xmm5, [srcreg+d2+d1+16][rbx]	;; R8
	xload	xmm0, [srcreg+d2+d1+48][rbx]	;; I8
	xload	xmm4, [srcreg+d2+d1][rbx]	;; R7
	xload	xmm1, [srcreg+d2+d1+32][rbx]	;; I7

	xp_complex_mult xmm5, xmm0, XX [srcreg+d2+d1+16][rbp], XX [srcreg+d2+d1+48][rbp], xmm6, xmm7 ;; Mult R8/I8
	xp_complex_mult xmm4, xmm1, XX [srcreg+d2+d1][rbp], XX [srcreg+d2+d1+32][rbp], xmm6, xmm7 ;; Mult R7/I7

	xcopy	xmm7, xmm5			;; Copy R8
	subpd	xmm5, xmm4			;; R8 = R8 - R7 (new I8)
	addpd	xmm4, xmm7			;; R7 = R8 + R7 (new R7)

	xcopy	xmm7, xmm1			;; Copy I7
	subpd	xmm1, xmm0			;; I7 = I7 - I8 (new R8)
	addpd	xmm0, xmm7			;; I8 = I7 + I8 (new I7)

	xstore	dst[d2+d1+48], xmm5		;; Save I8
	xstore	dst[d2+d1], xmm4		;; Save R7
	xstore	dst[d2+d1+16], xmm1		;; Save R8
	xstore	dst[d2+d1+32], xmm0		;; Save I7

	xload	xmm3, [srcreg+16][rbx]		;; R2
	xload	xmm5, [srcreg+48][rbx]		;; I2
	xload	xmm4, [srcreg][rbx]		;; R1
	xload	xmm6, [srcreg+32][rbx]		;; I1

	xp_complex_mult xmm3, xmm5, XX [srcreg+16][rbp], XX [srcreg+48][rbp], xmm0, xmm7 ;; Mult R2/I2
	xp_complex_mult xmm4, xmm6, XX [srcreg][rbp], XX [srcreg+32][rbp], xmm0, xmm7 ;; Mult R1/I1

	xcopy	xmm7, xmm4			;; Copy R1
	subpd	xmm4, xmm3			;; R1 = R1 - R2 (new R2)
	addpd	xmm3, xmm7			;; R2 = R1 + R2 (new R1)

	xcopy	xmm7, xmm6			;; Copy I1
	subpd	xmm6, xmm5			;; I1 = I1 - I2 (new I2)
	addpd	xmm5, xmm7			;; I2 = I1 + I2 (new I1)

	xstore	dst[16], xmm4			;; Save R2
	xstore	dst[0], xmm3			;; Save R1
	xstore	dst[48], xmm6			;; Save I2
	xstore	dst[32], xmm5			;; Save I1

	xload	xmm6, [srcreg+d1+16][rbx]	;; R4
	xload	xmm3, [srcreg+d1+48][rbx]	;; I4
	xload	xmm5, [srcreg+d1][rbx]		;; R3
	xload	xmm4, [srcreg+d1+32][rbx]	;; I3

	xp_complex_mult xmm6, xmm3, XX [srcreg+d1+16][rbp], XX [srcreg+d1+48][rbp], xmm0, xmm7 ;; Mult R4/I4
	xp_complex_mult xmm5, xmm4, XX [srcreg+d1][rbp], XX [srcreg+d1+32][rbp], xmm0, xmm7 ;; Mult R3/I3

	xcopy	xmm7, xmm6			;; Copy R4
	subpd	xmm6, xmm5			;; R4 = R4 - R3 (new I4)
	addpd	xmm5, xmm7			;; R3 = R4 + R3 (new R3)

	xcopy	xmm7, xmm4			;; Copy I3
	subpd	xmm4, xmm3			;; I3 = I3 - I4 (new R4)
	addpd	xmm3, xmm7			;; I4 = I3 + I4 (new I3)

	xstore	dst[d1+48], xmm6		;; Save I4
	xstore	dst[d1], xmm5			;; Save R3
	xstore	dst[d1+16], xmm4		;; Save R4
	xstore	dst[d1+32], xmm3		;; Save I3
	ENDM

;;
;; ************************************* eight-complex-fft8 variants ******************************************
;;
;; In the negacyclic split premultiplier case, we apply part of the roots of -1 at the
;; end of the first pass.  Also, in the r4delay case we apply part of the first level
;; twiddles at the end of the first pass.  Thus we have 8 sin/cos multiplies instead
;; of the usual 7.
;;

;; Used in the last levels of pass 1.  No swizzling.
IFDEF UNUSED
r8_g8cl_eight_complex_fft8 MACRO srcreg,srcinc,d1,d2,d4,dstreg,dstinc,e1,e2,e4,screg
	r8_x8c_fft8 srcreg+0,d1,d2,d4,dstreg+0,e1,e2,screg,0
	r8_x8c_fft8 srcreg+32,d1,d2,d4,dstreg+e4,e1,e2,screg,0
	bump	srcreg, srcinc
	bump	dstreg, dstinc
	ENDM

r8_x8c_fft8 MACRO srcreg,d1,d2,d4,dstreg,e1,e2,screg,scoff
	untested - likely buggy
	xload	xmm0, [srcreg]			;; R1
	xload	xmm2, [srcreg+d4]		;; R5
	addpd	xmm2, xmm0			;; R5 = R1 + R5 (new R1)

	xload	xmm1, [srcreg+d2]		;; R3
	xload	xmm3, [srcreg+d4+d2]		;; R7
	addpd	xmm3, xmm1			;; R7 = R3 + R7 (new R3)

	xload	xmm4, [srcreg+16]		;; I1
	xload	xmm6, [srcreg+d4+16]		;; I5
	addpd	xmm6, xmm4			;; I5 = I1 + I5 (new I1)

	subpd	xmm0, [srcreg+d4]		;; R1 = R1 - R5 (new R5)
	subpd	xmm1, [srcreg+d4+d2]		;; R3 = R3 - R7 (new R7)

	xcopy	xmm5, xmm3			;; Copy R3
	addpd	xmm3, xmm2			;; R3 = R1 + R3 (final R1)
	subpd	xmm2, xmm5			;; R1 = R1 - R3 (final R3)

	xload	xmm5, [srcreg+d2+16]		;; I3
	xload	xmm7, [srcreg+d4+d2+16]		;; I7
	addpd	xmm7, xmm5			;; I7 = I3 + I7 (new I3)

	subpd	xmm4, [srcreg+d4+16]		;; I1 = I1 - I5 (new I5)
	subpd	xmm5, [srcreg+d4+d2+16]		;; I3 = I3 - I7 (new I7)

	xstore	[dstreg], xmm3			;; Save R1

	xcopy	xmm3, xmm7			;; Copy I3
	addpd	xmm7, xmm6			;; I3 = I1 + I3 (final I1)
	subpd	xmm6, xmm3			;; I1 = I1 - I3 (final I3)

	xstore	[dstreg+16], xmm7		;; Save I1

	xcopy	xmm3, xmm1			;; Copy R7
	addpd	xmm1, xmm4			;; R7 = I5 + R7 (final I5)
	xcopy	xmm7, xmm0			;; Copy R5
	subpd	xmm0, xmm5			;; R5 = R5 - I7 (final R5)
	subpd	xmm4, xmm3			;; I5 = I5 - R7 (final I7)
	addpd	xmm5, xmm7			;; I7 = R5 + I7 (final R7)

	xstore	[dstreg+e1], xmm2		;; Save R3
	xstore	[dstreg+e1+16], xmm6		;; Save I3
	xstore	[dstreg+e2], xmm0		;; Save R5
	xstore	[dstreg+e2+16], xmm1		;; Save I5
	xstore	[dstreg+e2+e1], xmm5		;; Save R7
	xstore	[dstreg+e2+e1+16], xmm4		;; Save I7

	xload	xmm0, [srcreg+d1]		;; R2
	xload	xmm2, [srcreg+d4+d1]		;; R6
	addpd	xmm2, xmm0			;; R6 = R2 + R6 (new R2)

	xload	xmm1, [srcreg+d2+d1]		;; R4
	xload	xmm3, [srcreg+d4+d2+d1]		;; R8
	addpd	xmm3, xmm1			;; R8 = R4 + R8 (new R4)

	xload	xmm4, [srcreg+d1+16]		;; I2
	xload	xmm6, [srcreg+d4+d1+16]		;; I6
	addpd	xmm6, xmm4			;; I6 = I2 + I6 (new I2)

	subpd	xmm0, [srcreg+d4+d1]		;; R2 = R2 - R6 (new R6)
	subpd	xmm1, [srcreg+d4+d2+d1]		;; R4 = R4 - R8 (new R8)

	xcopy	xmm5, xmm3			;; Copy R4
	addpd	xmm3, xmm2			;; R4 = R2 + R4 (final R2)
	subpd	xmm2, xmm5			;; R2 = R2 - R4 (final R4)

	xload	xmm5, [srcreg+d2+d1+16]		;; I4
	xload	xmm7, [srcreg+d4+d2+d1+16]	;; I8
	addpd	xmm7, xmm5			;; I8 = I4 + I8 (new I4)

	subpd	xmm4, [srcreg+d4+d1+16]		;; I2 = I2 - I6 (new I6)
	subpd	xmm5, [srcreg+d4+d2+d1+16]	;; I4 = I4 - I8 (new I8)

	xstore	[dstreg+32], xmm3		;; Save R2

	xcopy	xmm3, xmm7			;; Copy I4
	addpd	xmm7, xmm6			;; I4 = I2 + I4 (final I2)
	subpd	xmm6, xmm3			;; I2 = I2 - I4 (final I4)

	xstore	[dstreg+48], xmm7		;; Save I2

	xcopy	xmm3, xmm1			;; Copy I6
	addpd	xmm1, xmm4			;; R8 = I6 + R8 (new I6)
	xcopy	xmm7, xmm0			;; Copy R6
	subpd	xmm0, xmm5			;; R6 = R6 - I8 (new R6)
	subpd	xmm4, xmm3			;; I6 = I6 - R8 (new I8)
	addpd	xmm5, xmm7			;; I8 = R6 + I8 (new R8)

	xcopy	xmm3, xmm0			;; Copy R6
	subpd	xmm0, xmm1			;; R6 = R6 - I6
	addpd	xmm1, xmm3			;; I6 = R6 + I6
	xload	xmm3, XMM_SQRTHALF
	mulpd	xmm0, xmm3			;; R6 = R6 * SQRTHALF (final R6)
	mulpd	xmm1, xmm3			;; I6 = I6 * SQRTHALF (final I6)

	xcopy	xmm7, xmm5			;; Copy R8
	subpd	xmm5, xmm4			;; R8 = R8 - I8
	addpd	xmm4, xmm7			;; I8 = R8 + I8
	mulpd	xmm5, xmm3			;; R8 = R8 * SQRTHALF (final R8)
	mulpd	xmm4, xmm3			;; I8 = I8 * SQRTHALF (final I8)

	xstore	[dstreg+e1+32], xmm2		;; Save R4
	xstore	[dstreg+e1+48], xmm6		;; Save I4

;; the last level

	xload	xmm2, [dstreg+e2]		;; R5
	subpd	xmm2, xmm0			;; R5 = R5 - R6 (new R6)
	addpd	xmm0, [dstreg+e2]		;; R6 = R5 + R6 (new R5)

	xload	xmm3, [dstreg+e2+16]		;; I5
	subpd	xmm3, xmm1			;; I5 = I5 - I6 (new I6)
	addpd	xmm1, [dstreg+e2+16]		;; I6 = I5 + I6 (new I5)

	xload	xmm6, [screg+scoff+160+16]	;; cosine/sine
	mulpd	xmm6, xmm2			;; A6 = R6 * cosine/sine
	subpd	xmm6, xmm3			;; A6 = A6 - I6
	mulpd	xmm3, [screg+scoff+160+16]	;; B6 = I6 * cosine/sine
	addpd	xmm3, xmm2			;; B6 = B6 + R6

	xload	xmm2, [dstreg+e2+e1]		;; R7
	subpd	xmm2, xmm4			;; R7 = R7 - I8 (new R7)
	addpd	xmm4, [dstreg+e2+e1]		;; I8 = R7 + I8 (new R8)

	xload	xmm7, [screg+scoff+32+16]	;; cosine/sine
	mulpd	xmm7, xmm0			;; A5 = R5 * cosine/sine
	subpd	xmm7, xmm1			;; A5 = A5 - I5
	mulpd	xmm1, [screg+scoff+32+16]	;; B5 = I5 * cosine/sine
	addpd	xmm1, xmm0			;; B5 = B5 + R5

	xload	xmm0, [dstreg+e2+e1+16]		;; I7
	subpd	xmm0, xmm5			;; I7 = I7 - R8 (new I8)
	addpd	xmm5, [dstreg+e2+e1+16]		;; R8 = I7 + R8 (new I7)

	mulpd	xmm6, [screg+scoff+160]		;; A6 = A6 * sine (final R6)
	xstore	[dstreg+e2+32], xmm6		;; Save R6

	xload	xmm6, [screg+scoff+96+16]	;; cosine/sine
	mulpd	xmm6, xmm2			;; A7 = R7 * cosine/sine
	subpd	xmm6, xmm5			;; A7 = A7 - I7
	mulpd	xmm5, [screg+scoff+96+16]	;; B7 = I7 * cosine/sine
	addpd	xmm5, xmm2			;; B7 = B7 + R7

	mulpd	xmm3, [screg+scoff+160]		;; B6 = B6 * sine (final I6)
	xstore	[dstreg+e2+48], xmm3		;; Save I6

	mulpd	xmm7, [screg+scoff+32]		;; A5 = A5 * sine (final R5)
	mulpd	xmm1, [screg+scoff+32]		;; B5 = B5 * sine (final I5)

	xstore	[dstreg+e2], xmm7		;; Save R5
	xstore	[dstreg+e2+16], xmm1		;; Save I5

	xload	xmm3, [screg+scoff+224+16]	;; cosine/sine
	mulpd	xmm3, xmm4			;; A8 = R8 * cosine/sine
	subpd	xmm3, xmm0			;; A8 = A8 - I8
	mulpd	xmm0, [screg+scoff+224+16]	;; B8 = I8 * cosine/sine
	addpd	xmm0, xmm4			;; B8 = B8 + R8

	mulpd	xmm6, [screg+scoff+96]		;; A7 = A7 * sine (new R7)
	mulpd	xmm5, [screg+scoff+96]		;; B7 = B7 * sine (new I7)
	mulpd	xmm3, [screg+scoff+224]		;; A8 = A8 * sine (new R8)
	mulpd	xmm0, [screg+scoff+224]		;; B8 = B8 * sine (new I8)

	xstore	[dstreg+e2+e1], xmm6		;; Save R7
	xstore	[dstreg+e2+e1+16], xmm5		;; Save I7
	xstore	[dstreg+e2+e1+32], xmm3		;; Save R8
	xstore	[dstreg+e2+e1+48], xmm0		;; Save I8

	xload	xmm3, [dstreg]			;; R1
	xload	xmm4, [dstreg+32]		;; R2
	subpd	xmm3, xmm4			;; R1 = R1 - R2 (new R2)
	addpd	xmm4, [dstreg]			;; R2 = R1 + R2 (new R1)

	xload	xmm7, [dstreg+16]		;; I1
	xload	xmm0, [dstreg+48]		;; I2
	subpd	xmm7, xmm0			;; I1 = I1 - I2 (new I2)
	addpd	xmm0, [dstreg+16]		;; I2 = I1 + I2 (new I1)

	xload	xmm1, [screg+scoff+128+16]	;; cosine/sine
	mulpd	xmm1, xmm3			;; A2 = R2 * cosine/sine
	subpd	xmm1, xmm7			;; A2 = A2 - I2
	mulpd	xmm7, [screg+scoff+128+16]	;; B2 = I2 * cosine/sine
	addpd	xmm7, xmm3			;; B2 = B2 + R2

	xload	xmm2, [screg+scoff+0+16]	;; cosine/sine
	mulpd	xmm2, xmm4			;; A1 = R1 * cosine/sine
	subpd	xmm2, xmm0			;; A1 = A1 - I1
	mulpd	xmm0, [screg+scoff+0+16]	;; B1 = I1 * cosine/sine
	addpd	xmm0, xmm4			;; B1 = B1 + R1

	mulpd	xmm1, [screg+scoff+128]		;; A2 = A2 * sine (final R2)
	mulpd	xmm7, [screg+scoff+128]		;; B2 = B2 * sine (final I2)
	mulpd	xmm2, [screg+scoff+0]		;; A1 = A1 * sine (final R1)
	mulpd	xmm0, [screg+scoff+0]		;; B1 = B1 * sine (final I1)

	xload	xmm5, [dstreg+e1]		;; R3
	xload	xmm6, [dstreg+e1+48]		;; I4
	subpd	xmm5, xmm6			;; R3 = R3 - I4 (new R3)
	addpd	xmm6, [dstreg+e1]		;; I4 = R3 + I4 (new R4)

	xload	xmm3, [dstreg+e1+16]		;; I3
	xload	xmm4, [dstreg+e1+32]		;; R4
	subpd	xmm3, xmm4			;; I3 = I3 - R4 (final I4)
	addpd	xmm4, [dstreg+e1+16]		;; R4 = I3 + R4 (final I3)

	xstore	[dstreg+32], xmm1		;; Save R2
	xstore	[dstreg+48], xmm7		;; Save I2
	xstore	[dstreg], xmm2			;; Save R1
	xstore	[dstreg+16], xmm0		;; Save I1

	xload	xmm1, [screg+scoff+64+16]	;; cosine/sine
	mulpd	xmm1, xmm5			;; A3 = R3 * cosine/sine
	subpd	xmm1, xmm4			;; A3 = A3 - I3
	mulpd	xmm4, [screg+scoff+64+16]	;; B3 = I3 * cosine/sine
	addpd	xmm4, xmm5			;; B3 = B3 + R3

	xload	xmm7, [screg+scoff+192+16]	;; cosine/sine
	mulpd	xmm7, xmm6			;; A4 = R4 * cosine/sine
	subpd	xmm7, xmm3			;; A4 = A4 - I4
	mulpd	xmm3, [screg+scoff+192+16]	;; B4 = I4 * cosine/sine
	addpd	xmm3, xmm6			;; B4 = B4 + R4

	mulpd	xmm1, [screg+scoff+64]		;; A3 = A3 * sine (final R3)
	mulpd	xmm4, [screg+scoff+64]		;; B3 = B3 * sine (final I3)
	mulpd	xmm7, [screg+scoff+192]		;; A4 = A4 * sine (final R4)
	mulpd	xmm3, [screg+scoff+192]		;; B4 = B4 * sine (final I4)

	xstore	[dstreg+e1], xmm1		;; Save R3
	xstore	[dstreg+e1+16], xmm4		;; Save I3
	xstore	[dstreg+e1+32], xmm7		;; Save R4
	xstore	[dstreg+e1+48], xmm3		;; Save I4
	ENDM
ENDIF

;; Used in the last levels of pass 1.  Swizzles.
r8_sg8cl_eight_complex_fft8 MACRO srcreg,srcinc,d1,d2,d4,dstreg,dstinc,e1,e2,e4,screg
	r8_s8c_fft8 srcreg+0,d1,d2,d4,dstreg+0,e1,e2,screg,0
	r8_s8c_fft8 srcreg+32,d1,d2,d4,dstreg+e4,e1,e2,screg,0
	bump	srcreg, srcinc
	bump	dstreg, dstinc
	ENDM

r8_s8c_fft8 MACRO srcreg,d1,d2,d4,dstreg,e1,e2,screg,scoff
	xload	xmm0, [srcreg]			;; R1
	xload	xmm2, [srcreg+d4]		;; R5
	xcopy	xmm5, xmm2			;; Copy R5
	addpd	xmm2, xmm0			;; R5 = R1 + R5 (new R1)

	xload	xmm1, [srcreg+d2]		;; R3
	xload	xmm3, [srcreg+d4+d2]		;; R7
	xcopy	xmm7, xmm3			;; Copy R7
	addpd	xmm3, xmm1			;; R7 = R3 + R7 (new R3)

	xload	xmm4, [srcreg+16]		;; I1
	xload	xmm6, [srcreg+d4+16]		;; I5
	addpd	xmm6, xmm4			;; I5 = I1 + I5 (new I1)

	subpd	xmm0, xmm5			;; R1 = R1 - R5 (new R5)
	subpd	xmm1, xmm7			;; R3 = R3 - R7 (new R7)

	xcopy	xmm5, xmm3			;; Copy R3
	addpd	xmm3, xmm2			;; R3 = R1 + R3 (final R1)
	subpd	xmm2, xmm5			;; R1 = R1 - R3 (final R3)

	xload	xmm5, [srcreg+d2+16]		;; I3
	xload	xmm7, [srcreg+d4+d2+16]		;; I7
	addpd	xmm7, xmm5			;; I7 = I3 + I7 (new I3)

	subpd	xmm4, [srcreg+d4+16]		;; I1 = I1 - I5 (new I5)
	subpd	xmm5, [srcreg+d4+d2+16]		;; I3 = I3 - I7 (new I7)

	xstore	[dstreg], xmm3			;; Save R1

	xcopy	xmm3, xmm7			;; Copy I3
	addpd	xmm7, xmm6			;; I3 = I1 + I3 (final I1)
	subpd	xmm6, xmm3			;; I1 = I1 - I3 (final I3)

	xstore	[dstreg+16], xmm7		;; Save I1

	xcopy	xmm3, xmm1			;; Copy R7
	addpd	xmm1, xmm4			;; R7 = I5 + R7 (final I5)
	xcopy	xmm7, xmm0			;; Copy R5
	subpd	xmm0, xmm5			;; R5 = R5 - I7 (final R5)
	subpd	xmm4, xmm3			;; I5 = I5 - R7 (final I7)
	addpd	xmm5, xmm7			;; I7 = R5 + I7 (final R7)

	xstore	[dstreg+e1], xmm2		;; Save R3
	xstore	[dstreg+e1+16], xmm6		;; Save I3
	xstore	[dstreg+e2], xmm0		;; Save R5
	xstore	[dstreg+e2+16], xmm1		;; Save I5
	xstore	[dstreg+e2+e1], xmm5		;; Save R7
	xstore	[dstreg+e2+e1+16], xmm4		;; Save I7

	xload	xmm0, [srcreg+d1]		;; R2
	xload	xmm2, [srcreg+d4+d1]		;; R6
	addpd	xmm2, xmm0			;; R6 = R2 + R6 (new R2)

	xload	xmm1, [srcreg+d2+d1]		;; R4
	xload	xmm3, [srcreg+d4+d2+d1]		;; R8
	addpd	xmm3, xmm1			;; R8 = R4 + R8 (new R4)

	xload	xmm4, [srcreg+d1+16]		;; I2
	xload	xmm6, [srcreg+d4+d1+16]		;; I6
	addpd	xmm6, xmm4			;; I6 = I2 + I6 (new I2)

	subpd	xmm0, [srcreg+d4+d1]		;; R2 = R2 - R6 (new R6)
	subpd	xmm1, [srcreg+d4+d2+d1]		;; R4 = R4 - R8 (new R8)

	xcopy	xmm5, xmm3			;; Copy R4
	addpd	xmm3, xmm2			;; R4 = R2 + R4 (final R2)
	subpd	xmm2, xmm5			;; R2 = R2 - R4 (final R4)

	xload	xmm5, [srcreg+d2+d1+16]		;; I4
	xload	xmm7, [srcreg+d4+d2+d1+16]	;; I8
	addpd	xmm7, xmm5			;; I8 = I4 + I8 (new I4)

	subpd	xmm4, [srcreg+d4+d1+16]		;; I2 = I2 - I6 (new I6)
	subpd	xmm5, [srcreg+d4+d2+d1+16]	;; I4 = I4 - I8 (new I8)

	xstore	[dstreg+32], xmm3		;; Save R2

	xcopy	xmm3, xmm7			;; Copy I4
	addpd	xmm7, xmm6			;; I4 = I2 + I4 (final I2)
	subpd	xmm6, xmm3			;; I2 = I2 - I4 (final I4)

	xstore	[dstreg+48], xmm7		;; Save I2

	xcopy	xmm3, xmm1			;; Copy I6
	addpd	xmm1, xmm4			;; R8 = I6 + R8 (new I6)
	xcopy	xmm7, xmm0			;; Copy R6
	subpd	xmm0, xmm5			;; R6 = R6 - I8 (new R6)
	subpd	xmm4, xmm3			;; I6 = I6 - R8 (new I8)
	addpd	xmm5, xmm7			;; I8 = R6 + I8 (new R8)

	xcopy	xmm3, xmm0			;; Copy R6
	subpd	xmm0, xmm1			;; R6 = R6 - I6
	addpd	xmm1, xmm3			;; I6 = R6 + I6
	xload	xmm3, XMM_SQRTHALF
	mulpd	xmm0, xmm3			;; R6 = R6 * SQRTHALF (final R6)
	mulpd	xmm1, xmm3			;; I6 = I6 * SQRTHALF (final I6)

	xcopy	xmm7, xmm5			;; Copy R8
	subpd	xmm5, xmm4			;; R8 = R8 - I8
	addpd	xmm4, xmm7			;; I8 = R8 + I8
	mulpd	xmm5, xmm3			;; R8 = R8 * SQRTHALF (final R8)
	mulpd	xmm4, xmm3			;; I8 = I8 * SQRTHALF (final I8)

	xstore	[dstreg+e1+32], xmm2		;; Save R4
	xstore	[dstreg+e1+48], xmm6		;; Save I4

;; the last level

	xload	xmm2, [dstreg+e2]		;; R5
	subpd	xmm2, xmm0			;; R5 = R5 - R6 (new R6)
	addpd	xmm0, [dstreg+e2]		;; R6 = R5 + R6 (new R5)

	xload	xmm3, [dstreg+e2+16]		;; I5
	subpd	xmm3, xmm1			;; I5 = I5 - I6 (new I6)
	addpd	xmm1, [dstreg+e2+16]		;; I6 = I5 + I6 (new I5)

	xload	xmm6, [screg+scoff+160+16]	;; cosine/sine
	mulpd	xmm6, xmm2			;; A6 = R6 * cosine/sine
	subpd	xmm6, xmm3			;; A6 = A6 - I6
	mulpd	xmm3, [screg+scoff+160+16]	;; B6 = I6 * cosine/sine
	addpd	xmm3, xmm2			;; B6 = B6 + R6

	xload	xmm2, [dstreg+e2+e1]		;; R7
	subpd	xmm2, xmm4			;; R7 = R7 - I8 (new R7)
	addpd	xmm4, [dstreg+e2+e1]		;; I8 = R7 + I8 (new R8)

	xload	xmm7, [screg+scoff+32+16]	;; cosine/sine
	mulpd	xmm7, xmm0			;; A5 = R5 * cosine/sine
	subpd	xmm7, xmm1			;; A5 = A5 - I5
	mulpd	xmm1, [screg+scoff+32+16]	;; B5 = I5 * cosine/sine
	addpd	xmm1, xmm0			;; B5 = B5 + R5

	xload	xmm0, [dstreg+e2+e1+16]		;; I7
	subpd	xmm0, xmm5			;; I7 = I7 - R8 (new I8)
	addpd	xmm5, [dstreg+e2+e1+16]		;; R8 = I7 + R8 (new I7)

	mulpd	xmm6, [screg+scoff+160]		;; A6 = A6 * sine (final R6)
	mulpd	xmm7, [screg+scoff+32]		;; A5 = A5 * sine (final R5)
	shuffle_store [dstreg+e2], [dstreg+e2+16], xmm7, xmm6	;; Save R5,R6

	xload	xmm6, [screg+scoff+96+16]	;; cosine/sine
	mulpd	xmm6, xmm2			;; A7 = R7 * cosine/sine
	subpd	xmm6, xmm5			;; A7 = A7 - I7
	mulpd	xmm5, [screg+scoff+96+16]	;; B7 = I7 * cosine/sine
	addpd	xmm5, xmm2			;; B7 = B7 + R7

	mulpd	xmm3, [screg+scoff+160]		;; B6 = B6 * sine (final I6)
	mulpd	xmm1, [screg+scoff+32]		;; B5 = B5 * sine (final I5)

	shuffle_store_with_temp [dstreg+e2+32], [dstreg+e2+48], xmm1, xmm3, xmm7 ;; Save I5,I6

	xload	xmm3, [screg+scoff+224+16]	;; cosine/sine
	mulpd	xmm3, xmm4			;; A8 = R8 * cosine/sine
	subpd	xmm3, xmm0			;; A8 = A8 - I8
	mulpd	xmm0, [screg+scoff+224+16]	;; B8 = I8 * cosine/sine
	addpd	xmm0, xmm4			;; B8 = B8 + R8

	mulpd	xmm6, [screg+scoff+96]		;; A7 = A7 * sine (new R7)
	mulpd	xmm5, [screg+scoff+96]		;; B7 = B7 * sine (new I7)
	mulpd	xmm3, [screg+scoff+224]		;; A8 = A8 * sine (new R8)
	mulpd	xmm0, [screg+scoff+224]		;; B8 = B8 * sine (new I8)

	shuffle_store_with_temp [dstreg+e2+e1], [dstreg+e2+e1+16], xmm6, xmm3, xmm4	;; Save R7,R8
	shuffle_store_with_temp [dstreg+e2+e1+32], [dstreg+e2+e1+48], xmm5, xmm0, xmm4	;; Save I7,I8

	xload	xmm3, [dstreg]			;; R1
	xload	xmm4, [dstreg+32]		;; R2
	subpd	xmm3, xmm4			;; R1 = R1 - R2 (new R2)
	addpd	xmm4, [dstreg]			;; R2 = R1 + R2 (new R1)

	xload	xmm7, [dstreg+16]		;; I1
	xload	xmm0, [dstreg+48]		;; I2
	subpd	xmm7, xmm0			;; I1 = I1 - I2 (new I2)
	addpd	xmm0, [dstreg+16]		;; I2 = I1 + I2 (new I1)

	xload	xmm1, [screg+scoff+128+16]	;; cosine/sine
	mulpd	xmm1, xmm3			;; A2 = R2 * cosine/sine
	subpd	xmm1, xmm7			;; A2 = A2 - I2
	mulpd	xmm7, [screg+scoff+128+16]	;; B2 = I2 * cosine/sine
	addpd	xmm7, xmm3			;; B2 = B2 + R2

	xload	xmm2, [screg+scoff+0+16]	;; cosine/sine
	mulpd	xmm2, xmm4			;; A1 = R1 * cosine/sine
	subpd	xmm2, xmm0			;; A1 = A1 - I1
	mulpd	xmm0, [screg+scoff+0+16]	;; B1 = I1 * cosine/sine
	addpd	xmm0, xmm4			;; B1 = B1 + R1

	mulpd	xmm1, [screg+scoff+128]		;; A2 = A2 * sine (final R2)
	mulpd	xmm7, [screg+scoff+128]		;; B2 = B2 * sine (final I2)
	mulpd	xmm2, [screg+scoff+0]		;; A1 = A1 * sine (final R1)
	mulpd	xmm0, [screg+scoff+0]		;; B1 = B1 * sine (final I1)

	xload	xmm5, [dstreg+e1]		;; R3
	xload	xmm6, [dstreg+e1+48]		;; I4
	subpd	xmm5, xmm6			;; R3 = R3 - I4 (new R3)
	addpd	xmm6, [dstreg+e1]		;; I4 = R3 + I4 (new R4)

	xload	xmm3, [dstreg+e1+16]		;; I3
	xload	xmm4, [dstreg+e1+32]		;; R4
	subpd	xmm3, xmm4			;; I3 = I3 - R4 (final I4)
	addpd	xmm4, [dstreg+e1+16]		;; R4 = I3 + R4 (final I3)

	shuffle_store [dstreg], [dstreg+16], xmm2, xmm1				;; Save R1,R2
	shuffle_store_with_temp [dstreg+32], [dstreg+48], xmm0, xmm7, xmm1	;; Save I1,I2

	xload	xmm1, [screg+scoff+64+16]	;; cosine/sine
	mulpd	xmm1, xmm5			;; A3 = R3 * cosine/sine
	subpd	xmm1, xmm4			;; A3 = A3 - I3
	mulpd	xmm4, [screg+scoff+64+16]	;; B3 = I3 * cosine/sine
	addpd	xmm4, xmm5			;; B3 = B3 + R3

	xload	xmm7, [screg+scoff+192+16]	;; cosine/sine
	mulpd	xmm7, xmm6			;; A4 = R4 * cosine/sine
	subpd	xmm7, xmm3			;; A4 = A4 - I4
	mulpd	xmm3, [screg+scoff+192+16]	;; B4 = I4 * cosine/sine
	addpd	xmm3, xmm6			;; B4 = B4 + R4

	mulpd	xmm1, [screg+scoff+64]		;; A3 = A3 * sine (final R3)
	mulpd	xmm4, [screg+scoff+64]		;; B3 = B3 * sine (final I3)
	mulpd	xmm7, [screg+scoff+192]		;; A4 = A4 * sine (final R4)
	mulpd	xmm3, [screg+scoff+192]		;; B4 = B4 * sine (final I4)

	shuffle_store_with_temp [dstreg+e1], [dstreg+e1+16], xmm1, xmm7, xmm0		;; Save R3,R4
	shuffle_store_with_temp [dstreg+e1+32], [dstreg+e1+48], xmm4, xmm3, xmm0	;; Save I3,I4
	ENDM

IFDEF X86_64

r8_sg8cl_eight_complex_fft8 MACRO srcreg,srcinc,d1,d2,d4,dstreg,dstinc,e1,e2,e4,screg
	r8_s8c_fft8 srcreg+0,d1,d2,d4,dstreg+0,e1,e2,screg,0
	r8_s8c_fft8 srcreg+32,d1,d2,d4,dstreg+e4,e1,e2,screg,0
	bump	srcreg, srcinc
	bump	dstreg, dstinc
	ENDM

r8_s8c_fft8 MACRO srcreg,d1,d2,d4,dstreg,e1,e2,screg,scoff
	xload	xmm0, [srcreg]			;; R1
	xload	xmm1, [srcreg+d4]		;; R5
	xcopy	xmm2, xmm0			;; Copy R1			; 1-3
	addpd	xmm0, xmm1			;; R1 = R1 + R5 (new R1)	; 1-3

	xload	xmm3, [srcreg+d2]		;; R3
	xload	xmm4, [srcreg+d4+d2]		;; R7
	xcopy	xmm5, xmm3			;; Copy R3			; 2-4
	addpd	xmm3, xmm4			;; R3 = R3 + R7 (new R3)	; 2-4

	xload	xmm6, [srcreg+16]		;; I1
	xload	xmm7, [srcreg+d4+16]		;; I5
	xcopy	xmm8, xmm6			;; Copy I1			; 3-5
	addpd	xmm6, xmm7			;; I1 = I1 + I5 (new I1)	; 3-5

	xload	xmm9, [srcreg+d2+16]		;; I3
	xload	xmm10, [srcreg+d4+d2+16]	;; I7
	xcopy	xmm11, xmm9			;; Copy I3			; 4-6
	addpd	xmm9, xmm10			;; I3 = I3 + I7 (new I3)	; 4-6

	subpd	xmm2, xmm1			;; R5 = R1 - R5 (new R5)	; 5-7

	subpd	xmm5, xmm4			;; R7 = R3 - R7 (new R7)	; 6-8

	subpd	xmm8, xmm7			;; I5 = I1 - I5 (new I5)	; 7-9
	xcopy	xmm1, xmm0			;; Copy R1			; 7-9 (4)

	subpd	xmm11, xmm10			;; I7 = I3 - I7 (new I7)	; 8-10	avail 4,7,10,12+
	xload	xmm12, [srcreg+d1]		;; R2				; 8

	subpd	xmm0, xmm3			;; R1 = R1 - R3 (final R3)	; 9-11
	xcopy	xmm4, xmm6			;; Copy I1			; 9-11 (6) avail 7,10,13+
	xload	xmm13, [srcreg+d4+d1]		;; R6				; 9

	addpd	xmm3, xmm1			;; R3 = R1 + R3 (final R1)	; 10-12	avail 7,10,1,14+
	xload	xmm14, [srcreg+d2+d1]		;; R4				; 10

	subpd	xmm6, xmm9			;; I1 = I1 - I3 (final I3)	; 11-13
	xcopy	xmm1, xmm8			;; Copy I5			; 11-13 (10) avail 7,10,15
	xload	xmm15, [srcreg+d4+d2+d1]	;; R8				; 11

	addpd	xmm9, xmm4			;; I3 = I1 + I3 (final I1)	; 12-14	avail 7,10,4
	xstore	[dstreg+e1], xmm0		;; Save R3			; 12	avail 7,10,4,0

	subpd	xmm8, xmm5			;; I5 = I5 - R7 (final I7)	; 13-15
	xcopy	xmm4, xmm2			;; Copy R5			; 13-15 (8) avail 7,10,0

	addpd	xmm5, xmm1			;; R7 = I5 + R7 (final I5)	; 14-16	avail 7,10,0,1
	xload	xmm7, [srcreg+d1+16]		;; I2				; 14	avail 10,0,1
	xstore	[dstreg+e1+16], xmm6		;; Save I3			; 14	avail 10,0,1,6

	subpd	xmm2, xmm11			;; R5 = R5 - I7 (final R5)	; 15-17
	xcopy	xmm1, xmm12			;; Copy R2			; 15-17	avail 10,0,6
	xload	xmm10, [srcreg+d4+d1+16]	;; I6				; 15	avail 0,6

	addpd	xmm11, xmm4			;; I7 = R5 + I7 (final R7)	; 16-18	avail 0,6,4
	xload	xmm6, [srcreg+d2+d1+16]		;; I4				; 16	avail 0,4
	xstore	[dstreg+e2+e1+16], xmm8		;; Save I7			; 16	avail 0,4,8

	addpd	xmm12, xmm13			;; R2 = R2 + R6 (new R2)	; 17-19
	xcopy	xmm4, xmm14			;; Copy R4			; 17-19	avail 0,8
	xstore	[dstreg+e2+16], xmm5		;; Save I5			; 17	avail 0,8,5

	subpd	xmm1, xmm13			;; R6 = R2 - R6 (new R6)	; 18-20	avail 0,8,5,13
	xload	xmm0, XMM_SQRTHALF						; 18	avail 8,5,13
	xstore	[dstreg+e2], xmm2		;; Save R5			; 18	avail 8,5,13,2

	addpd	xmm14, xmm15			;; R4 = R4 + R8 (new R4)	; 19-21
	xcopy	xmm2, xmm7			;; Copy I2			; 19-21	avail 8,5,13
	xstore	[dstreg+e2+e1], xmm11		;; Save R7			; 19	avail 8,5,13,11

	subpd	xmm4, xmm15			;; R8 = R4 - R8 (new R8)	; 20-22	avail 8,5,13,11,15
	xload	xmm8, [srcreg+d4+d2+d1+16]	;; I8				; 18	avail 5,13,11,15

	addpd	xmm7, xmm10			;; I2 = I2 + I6 (new I2)	; 21-23
	xcopy	xmm5, xmm6			;; Copy I4			; 21-23	avail 13,11,15
	mulpd	xmm1, xmm0			;; R6 = R6 * SQRTHALF		; 21-25

	subpd	xmm2, xmm10			;; I6 = I2 - I6 (new I6)	; 22-24	avail 13,11,15,10

	addpd	xmm6, xmm8			;; I4 = I4 + I8 (new I4)	; 23-25
	mulpd	xmm4, xmm0			;; R8 = R8 * SQRTHALF		; 23-27

	subpd	xmm5, xmm8			;; I8 = I4 - I8 (new I8)	; 24-26	avail 13,11,15,10,8
	xcopy	xmm8, xmm12			;; Copy R2			; 24-26	avail 13,11,15,10

	addpd	xmm12, xmm14			;; R2 = R2 + R4 (final R2)	; 25-27
	xcopy	xmm10, xmm7			;; Copy I2			; 25-27	avail 13,11,15
	mulpd	xmm2, xmm0			;; I6 = I6 * SQRTHALF		; 25-29

	addpd	xmm7, xmm6			;; I2 = I2 + I4 (final I2)	; 26-28
	xload	xmm11, [screg+scoff+128+16]	;; cosine/sine 2		; 26	avail 13,15

	subpd	xmm8, xmm14			;; R4 = R2 - R4 (final R4)	; 27-29	avail 13,15,14
	mulpd	xmm5, xmm0			;; I8 = I8 * SQRTHALF		; 27-31	avail 13,15,14,0
	xcopy	xmm0, xmm3			;; Copy R1			; 27-29	avail 13,15,14

	subpd	xmm10, xmm6			;; I4 = I2 - I4 (final I4)	; 28-30	avail 13,15,14,6
	xload	xmm15, [screg+scoff+0+16]	;; cosine/sine 1		; 28	avail 13,14,6

	subpd	xmm3, xmm12			;; R1 = R1 - R2 (new R2)	; 29-31
	xcopy	xmm6, xmm9			;; Copy I1			; 29-31	avail 13,14

	addpd	xmm12, xmm0			;; R2 = R1 + R2 (new R1)	; 30-32	avail 13,14,0
	xcopy	xmm0, xmm4			;; Copy R8			; 30-32 (28) avail 13,14

	subpd	xmm9, xmm7			;; I1 = I1 - I2 (new I2)	; 31-33
	xcopy	xmm13, xmm1			;; Copy R6			; 31-33 (26) avail 14

	addpd	xmm7, xmm6			;; I2 = I1 + I2 (new I1)	; 32-34	avail 14,6
	xcopy	xmm6, xmm3			;; Copy R2			; 32-34	avail 14
	mulpd	xmm3, xmm11			;; A2 = R2 * cosine/sine	; 32-36

	addpd	xmm4, xmm2			;; R8 = I6 + R8 (new I6)	; 33-35
	xcopy	xmm14, xmm12			;; Copy R1			; 33-35	avail none
	mulpd	xmm12, xmm15			;; A1 = R1 * cosine/sine	; 33-37

	subpd	xmm1, xmm5			;; R6 = R6 - I8 (new R6)	; 34-36
	mulpd	xmm11, xmm9			;; B2 = I2 * cosine/sine	; 34-38

	subpd	xmm2, xmm0			;; I6 = I6 - R8 (new I8)	; 35-37	avail 0
	mulpd	xmm15, xmm7			;; B1 = I1 * cosine/sine	; 35-39

	addpd	xmm5, xmm13			;; I8 = R6 + I8 (new R8)	; 36-38	avail 0,13
	xload	xmm0, [screg+scoff+128]		;; sine 2			; 36	avail 13

	subpd	xmm3, xmm9			;; A2 = A2 - I2			; 37-39	avail 13,9
	xload	xmm9, [dstreg+e1]		;; Reload R3			; 37	avail 13

	subpd	xmm12, xmm7			;; A1 = A1 - I1			; 38-40	avail 13,7
	xload	xmm13, [screg+scoff+0]		;; sine 1			; 38	avail 7

	addpd	xmm11, xmm6			;; B2 = B2 + R2			; 39-41	avail 7,6
	xcopy	xmm6, xmm10			;; Copy I4			; 39-41	avail 7
	xload	xmm7, [dstreg+e1+16]		;; Reload I3			; 39	avail none

	addpd	xmm15, xmm14			;; B1 = B1 + R1			; 40-42	avail 14
	mulpd	xmm3, xmm0			;; A2 = A2 * sine (final R2)	; 40-44

	addpd	xmm10, xmm9			;; I4 = R3 + I4 (new R4)	; 41-43
	mulpd	xmm12, xmm13			;; A1 = A1 * sine (final R1)	; 41-45
	xcopy	xmm14, xmm8			;; Copy R4			; 41-43	avail none

	subpd	xmm9, xmm6			;; R3 = R3 - I4 (new R3)	; 42-44	avail 6
	mulpd	xmm11, xmm0			;; B2 = B2 * sine (final I2)	; 42-46	avail 6,0
	xcopy	xmm0, xmm1			;; Copy R6			; 42-44 (37) avail 6
	xload	xmm6, [screg+scoff+192+16]	;; cosine/sine 4		; 42	avail none

	addpd	xmm8, xmm7			;; R4 = I3 + R4 (new I3)	; 43-45
	mulpd	xmm15, xmm13			;; B1 = B1 * sine (final I1)	; 43-47	avail 13
	xcopy	xmm13, xmm5			;; Copy R8			; 43-45 (39) avail none

	subpd	xmm7, xmm14			;; I3 = I3 - R4 (new I4)	; 44-46	avail 14
	xcopy	xmm14, xmm10			;; Copy R4			; 44-46	avail 3,12
	mulpd	xmm10, xmm6			;; A4 = R4 * cosine/sine	; 44-48

	subpd	xmm1, xmm4			;; R6 = R6 - I6			; 45-47
	shuffle_store [dstreg], [dstreg+16], xmm12, xmm3			;; Save R1,R2 starting at clock 46
	xload	xmm3, [screg+scoff+64+16]	;; cosine/sine 3		; 45	avail 12
	xcopy	xmm12, xmm9			;; Copy R3			; 45-47	avail none
	mulpd	xmm9, xmm3			;; A3 = R3 * cosine/sine	; 45-49

	addpd	xmm4, xmm0			;; I6 = R6 + I6			; 46-48	avail 0
	mulpd	xmm3, xmm8			;; B3 = I3 * cosine/sine	; 46-50

	subpd	xmm5, xmm2			;; R8 = R8 - I8			; 47-49
	mulpd	xmm6, xmm7			;; B4 = I4 * cosine/sine	; 47-51

	addpd	xmm2, xmm13			;; I8 = R8 + I8			; 48-50	avail 0,13

	subpd	xmm10, xmm7			;; A4 = A4 - I4			; 49-51	avail 0,13,7
	xload	xmm7, [screg+scoff+192]		;; sine 4			; 49	avail 0,13

	subpd	xmm9, xmm8			;; A3 = A3 - I3			; 50-52	avail 0,13,8
	xload	xmm8, [screg+scoff+64]		;; sine 3			; 50	avail 0,13

	shuffle_store_with_temp [dstreg+32], [dstreg+48], xmm15, xmm11, xmm0	;; Save I1,I2 starting at clock 50

	addpd	xmm3, xmm12			;; B3 = B3 + R3			; 51-53	avail 0,13,15,11,12
	xcopy	xmm0, xmm1			;; Copy R6			; 51-53	avail 13,15,11,12
	xload	xmm13, [dstreg+e2]		;; Reload R5

	addpd	xmm6, xmm14			;; B4 = B4 + R4			; 52-54	avail 15,11,12,14
	mulpd	xmm10, xmm7			;; A4 = A4 * sine (final R4)	; 52-56
	xload	xmm15, [dstreg+e2+16]		;; Reload I5			; 52	avail 11,12,14

	addpd	xmm1, xmm13			;; R6 = R5 + R6 (new R5)	; 53-55
	mulpd	xmm9, xmm8			;; A3 = A3 * sine (final R3)	; 53-57
	xcopy	xmm11, xmm4			;; Copy I6			; 53-55	avail 12,14
	xload	xmm14, [screg+scoff+32+16]	;; cosine/sine 5		; 53	avail 12

	subpd	xmm13, xmm0			;; R5 = R5 - R6 (new R6)	; 54-56	avail 12,0
	mulpd	xmm3, xmm8			;; B3 = B3 * sine (final I3)	; 54-58 avail 12,0,8
	xcopy	xmm0, xmm2			;; Copy I8			; 54-56 (51) avail 12,8
	xload	xmm12, [dstreg+e2+e1]		;; Reload R7			; 54	avail 8

	addpd	xmm4, xmm15			;; I6 = I5 + I6 (new I5)	; 55-57
	mulpd	xmm6, xmm7			;; B4 = B4 * sine (final I4)	; 55-59 avail 8,7
	xcopy	xmm7, xmm5			;; Copy R8			; 57-59 (50) avail 8

	subpd	xmm15, xmm11			;; I5 = I5 - I6 (new I6)	; 56-58 avail 8,11
	xcopy	xmm8, xmm1			;; Copy R5			; 56-58 avail 11
	mulpd	xmm1, xmm14			;; A5 = R5 * cosine/sine	; 56-60
	xload	xmm11, [screg+scoff+160+16]	;; cosine/sine 6		; 56	avail none

	addpd	xmm2, xmm12			;; I8 = R7 + I8 (new R8)	; 57-59
	shuffle_store [dstreg+e1], [dstreg+e1+16], xmm9, xmm10			;; Save R3,R4 starting at clock 58
	xcopy	xmm9, xmm13			;; Copy R6			; 57-59	avail 10
	mulpd	xmm13, xmm11			;; A6 = R6 * cosine/sine	; 57-61

	subpd	xmm12, xmm0			;; R7 = R7 - I8 (new R7)	; 58-60 avail 10,0
	mulpd	xmm14, xmm4			;; B5 = I5 * cosine/sine	; 58-62
	xload	xmm10, [dstreg+e2+e1+16]	;; Reload I7			; 58	avail 0

	addpd	xmm5, xmm10			;; R8 = I7 + R8 (new I7)	; 59-61
	mulpd	xmm11, xmm15			;; B6 = I6 * cosine/sine	; 59-63
	xload	xmm0, [screg+scoff+224+16]	;; cosine/sine 8		; 59	avail none

	subpd	xmm10, xmm7			;; I7 = I7 - R8 (new I8)	; 60-62 avail 7
	xcopy	xmm7, xmm2			;; Copy R8
	mulpd	xmm2, xmm0			;; A8 = R8 * cosine/sine	; 60-64	avail none

	shuffle_store [dstreg+e1+32], [dstreg+e1+48], xmm3, xmm6		;; Save I3,I4 starting at clock 60

	subpd	xmm1, xmm4			;; A5 = A5 - I5			; 61-63	avail 3,6,4
	xload	xmm3, [screg+scoff+96+16]	;; cosine/sine 7		; 61	avail 6,4
	xcopy	xmm6, xmm12			;; Copy R7			; 61-63	avail 4
	mulpd	xmm12, xmm3			;; A7 = R7 * cosine/sine	; 61-65

	subpd	xmm13, xmm15			;; A6 = A6 - I6			; 62-64	avail 4,15
	mulpd	xmm3, xmm5			;; B7 = I7 * cosine/sine	; 62-66
	xload	xmm4, [screg+scoff+32]		;; sine 5			; 62	avail 15

	addpd	xmm14, xmm8			;; B5 = B5 + R5			; 63-65	avail 15,8
	mulpd	xmm0, xmm10			;; B8 = I8 * cosine/sine	; 63-67
	xload	xmm15, [screg+scoff+160]	;; sine 6			; 63	avail 8

	addpd	xmm11, xmm9			;; B6 = B6 + R6			; 64-66	avail 8,9
	mulpd	xmm1, xmm4			;; A5 = A5 * sine (final R5)	; 64-68
	xload	xmm8, [screg+scoff+224]		;; sine 8			; 64	avail 9

	subpd	xmm2, xmm10			;; A8 = A8 - I8			; 65-67	avail 9,10
	mulpd	xmm13, xmm15			;; A6 = A6 * sine (final R6)	; 65-69
	xload	xmm9, [screg+scoff+96]		;; sine	7			; 65	avail 10

	subpd	xmm12, xmm5			;; A7 = A7 - I7			; 66-68	avail 10,5
	mulpd	xmm14, xmm4			;; B5 = B5 * sine (final I5)	; 66-70	avail 10,5,4

	addpd	xmm3, xmm6			;; B7 = B7 + R7			; 67-69	avail 10,5,4,6
	mulpd	xmm11, xmm15			;; B6 = B6 * sine (final I6)	; 67-71	avail 10,5,4,6,15

	addpd	xmm0, xmm7			;; B8 = B8 + R8			; 68-70	avail 10,5,4,6,15,7
	mulpd	xmm2, xmm8			;; A8 = A8 * sine (final R8)	; 68-72

	mulpd	xmm12, xmm9			;; A7 = A7 * sine (final R7)	; 69-73
	mulpd	xmm3, xmm9			;; B7 = B7 * sine (final I7)	; 70-74	avail 10,5,4,6,15,7,9
	mulpd	xmm0, xmm8			;; B8 = B8 * sine (final I8)	; 71-75	avail 10,5,4,6,15,7,9,8

	shuffle_store_with_temp [dstreg+e2], [dstreg+e2+16], xmm1, xmm13, xmm4 ;; Save R5,R6
	shuffle_store_with_temp [dstreg+e2+32], [dstreg+e2+48], xmm14, xmm11, xmm4 ;; Save I5,I6
	shuffle_store_with_temp [dstreg+e2+e1], [dstreg+e2+e1+16], xmm12, xmm2, xmm4 ;; Save R7,R8
	shuffle_store_with_temp [dstreg+e2+e1+32], [dstreg+e2+e1+48], xmm3, xmm0, xmm4 ;; Save I7,I8

	ENDM

ENDIF

;;
;; ************************************* eight-complex-unfft8 variants ******************************************
;;

;
; Radix-8 inverse FFT building block for negacyclic delayed twiddle multipliers from the first few levels (r4delay).
; Has 8 premult/sin/cos multipliers.
;

;; Used in the last levels of pass 1.  No swizzling.
IFDEF UNUSED
r8_g4cl_eight_complex_unfft8 MACRO srcreg,srcinc,d1,d2,dstreg,dstinc,e1,e2,screg
	xload	xmm7, [screg+0+16]		;; cosine/sine
	mulpd	xmm7, [srcreg]			;; A1 = R1 * cosine/sine
	xload	xmm3, [screg+128+16]		;; cosine/sine
	mulpd	xmm3, [srcreg+32]		;; A2 = R2 * cosine/sine
	xload	xmm0, [screg+64+16]		;; cosine/sine
	mulpd	xmm0, [srcreg+d1]		;; A3 = R3 * cosine/sine
	xload	xmm1, [screg+192+16]		;; cosine/sine
	mulpd	xmm1, [srcreg+d1+32]		;; A4 = R4 * cosine/sine

	xload	xmm2, [srcreg+16]		;; I1
	addpd	xmm7, xmm2			;; A1 = A1 + I1
	mulpd	xmm2, [screg+0+16]		;; B1 = I1 * cosine/sine
	xload	xmm4, [srcreg+48]		;; I2
	addpd	xmm3, xmm4			;; A2 = A2 + I2
	mulpd	xmm4, [screg+128+16]		;; B2 = I2 * cosine/sine
	xload	xmm6, [srcreg+d1+16]
	addpd	xmm0, xmm6			;; A3 = A3 + I3
	mulpd	xmm6, [screg+64+16]		;; B3 = I3 * cosine/sine

	mulpd	xmm7, [screg+0]			;; A1 = A1 * sine (new R1)
	mulpd	xmm3, [screg+128]		;; A2 = A2 * sine (new R2)

	xcopy	xmm5, xmm7			;; Copy R1
	subpd	xmm7, xmm3			;; R1 = R1 - R2 (new R2)
	addpd	xmm3, xmm5			;; R2 = R1 + R2 (new R1)

	xload	xmm5, [srcreg+d1+48]		;; I4
	addpd	xmm1, xmm5	 		;; A4 = A4 + I4
	mulpd	xmm5, [screg+192+16]		;; B4 = I4 * cosine/sine

	subpd	xmm2, [srcreg]			;; B1 = B1 - R1
	subpd	xmm4, [srcreg+32]		;; B2 = B2 - R2
	subpd	xmm6, [srcreg+d1]		;; B3 = B3 - R3
	mulpd	xmm0, [screg+64]		;; A3 = A3 * sine (new R3)
	subpd	xmm5, [srcreg+d1+32]		;; B4 = B4 - R4
	mulpd	xmm1, [screg+192]		;; A4 = A4 * sine (new R4)

	xstore	[dstreg+e1], xmm7		;; R2

	mulpd	xmm2, [screg+0]			;; B1 = B1 * sine (new I1)
	mulpd	xmm4, [screg+128]		;; B2 = B2 * sine (new I2)
	mulpd	xmm6, [screg+64]		;; B3 = B3 * sine (new I3)
	mulpd	xmm5, [screg+192]		;; B4 = B4 * sine (new I4)

	xcopy	xmm7, xmm1			;; Copy R4
	subpd	xmm1, xmm0			;; R4 = R4 - R3 (new I4)
	addpd	xmm0, xmm7			;; R3 = R4 + R3 (new R3)

	xcopy	xmm7, xmm2			;; Copy I1
	subpd	xmm2, xmm4			;; I1 = I1 - I2 (new I2)
	addpd	xmm4, xmm7			;; I2 = I1 + I2 (new I1)

	xcopy	xmm7, xmm6			;; Copy I3
	subpd	xmm6, xmm5			;; I3 = I3 - I4 (new R4)
	addpd	xmm5, xmm7			;; I4 = I3 + I4 (new I3)

	xload	xmm7, [screg+32+16]		;; cosine/sine
	mulpd	xmm7, [srcreg+d2]		;; A5 = R5 * cosine/sine
	xstore	[dstreg], xmm3			;; R1
	xload	xmm3, [screg+160+16]		;; cosine/sine
	mulpd	xmm3, [srcreg+d2+32]		;; A6 = R6 * cosine/sine
	xstore	[dstreg+e2], xmm0		;; R3
	xload	xmm0, [screg+96+16]		;; cosine/sine
	mulpd	xmm0, [srcreg+d2+d1]		;; A7 = R7 * cosine/sine
	xstore	[dstreg+e2+e1+32], xmm1		;; I4
	xload	xmm1, [screg+224+16]		;; cosine/sine
	mulpd	xmm1, [srcreg+d2+d1+32]		;; A8 = R8 * cosine/sine

	xstore	[dstreg+e1+32], xmm2		;; I2
	xload	xmm2, [srcreg+d2+16]		;; I5
	addpd	xmm7, xmm2			;; A5 = A5 + I5
	xstore	[dstreg+32], xmm4		;; I1
	xload	xmm4, [srcreg+d2+48]		;; I6
	addpd	xmm3, xmm4			;; A6 = A6 + I6
	xstore	[dstreg+e2+e1], xmm6		;; R4
	xload	xmm6, [srcreg+d2+d1+16]		;; I7
	addpd	xmm0, xmm6			;; A7 = A7 + I7

	mulpd	xmm7, [screg+32]		;; A5 = A5 * sine (new R5)
	mulpd	xmm3, [screg+160]		;; A6 = A6 * sine (new R6)

	xstore	[dstreg+e2+32], xmm5		;; I3

	xcopy	xmm5, xmm7			;; Copy R5
	subpd	xmm7, xmm3			;; R5 = R5 - R6 (new R6)
	addpd	xmm3, xmm5			;; R6 = R5 + R6 (new R5)

	xload	xmm5, [srcreg+d2+d1+48]		;; I8
	addpd	xmm1, xmm5	 		;; A8 = A8 + I8

	mulpd	xmm2, [screg+32+16]		;; B5 = I5 * cosine/sine
	subpd	xmm2, [srcreg+d2]		;; B5 = B5 - R5

	mulpd	xmm4, [screg+160+16]		;; B6 = I6 * cosine/sine
	subpd	xmm4, [srcreg+d2+32]		;; B6 = B6 - R6

	mulpd	xmm6, [screg+96+16]		;; B7 = I7 * cosine/sine
	subpd	xmm6, [srcreg+d2+d1]		;; B7 = B7 - R7

	mulpd	xmm5, [screg+224+16]		;; B8 = I8 * cosine/sine
	subpd	xmm5, [srcreg+d2+d1+32]		;; B8 = B8 - R8

	mulpd	xmm2, [screg+32]		;; B5 = B5 * sine (new I5)
	mulpd	xmm4, [screg+160]		;; B6 = B6 * sine (new I6)

	mulpd	xmm0, [screg+96]		;; A7 = A7 * sine (new R7)
	mulpd	xmm6, [screg+96]		;; B7 = B7 * sine (new I7)

	mulpd	xmm1, [screg+224]		;; A8 = A8 * sine (new R8)
	mulpd	xmm5, [screg+224]		;; B8 = B8 * sine (new I8)

	bump	srcreg, srcinc

	xstore	[dstreg+e1+16], xmm7		;; R6

	xcopy	xmm7, xmm2			;; Copy I5
	subpd	xmm2, xmm4			;; I5 = I5 - I6 (new I6)
	addpd	xmm4, xmm7			;; I6 = I5 + I6 (new I5)

	xcopy	xmm7, xmm1			;; Copy R8
	subpd	xmm1, xmm0			;; R8 = R8 - R7 (new I8)
	addpd	xmm0, xmm7			;; R7 = R8 + R7 (new R7)

	xcopy	xmm7, xmm6			;; Copy I7
	subpd	xmm6, xmm5			;; I7 = I7 - I8 (new R8)
	addpd	xmm5, xmm7			;; I8 = I7 + I8 (new I7)

	xcopy	xmm7, xmm0			;; Copy R7
	subpd	xmm0, xmm3			;; R7 = R7 - R5 (new I7)
	addpd	xmm3, xmm7			;; R5 = R7 + R5 (new R5)

	xcopy	xmm7, xmm4			;; Copy I5
	subpd	xmm4, xmm5			;; I5 = I5 - I7 (new R7)
	addpd	xmm5, xmm7			;; I7 = I5 + I7 (new I5)

	xstore	[dstreg+e1+48], xmm2		;; I6

	xload	xmm7, [dstreg]			;; R1
	xload	xmm2, [dstreg+e2]		;; R3
	subpd	xmm7, xmm2			;; R1 = R1 - R3 (new R3)
	addpd	xmm2, [dstreg]			;; R3 = R1 + R3 (new R1)

	xstore	[dstreg+e2+e1+16], xmm6		;; R8

	xcopy	xmm6, xmm7			;; Copy R3
	subpd	xmm7, xmm4			;; R3 = R3 - R7 (final R7)
	addpd	xmm4, xmm6			;; R7 = R3 + R7 (final R3)

	xstore	[dstreg+e2+e1+48], xmm1		;; I8

	xload	xmm6, [dstreg+32]		;; I1
	xload	xmm1, [dstreg+e2+32]		;; I3
	subpd	xmm6, xmm1			;; I1 = I1 - I3 (new I3)
	addpd	xmm1, [dstreg+32]		;; I3 = I1 + I3 (new I1)

	xstore	[dstreg+e2+16], xmm7		;; R7

	xcopy	xmm7, xmm6			;; Copy I3
	subpd	xmm6, xmm0			;; I3 = I3 - I7 (final I7)
	addpd	xmm0, xmm7			;; I7 = I3 + I7 (final I3)

	xcopy	xmm7, xmm2			;; Copy R1
	subpd	xmm2, xmm3			;; R1 = R1 - R5 (final R5)
	addpd	xmm3, xmm7			;; R5 = R1 + R5 (final R1)

	xcopy	xmm7, xmm1			;; Copy I1
	subpd	xmm1, xmm5			;; I1 = I1 - I5 (final I5)
	addpd	xmm5, xmm7			;; I5 = I1 + I5 (final I1)

	xstore	[dstreg+e2], xmm4		;; R3
	xstore	[dstreg+e2+32], xmm0		;; I3

	;; multiply R6/I6 by SQRTHALF - i*SQRTHALF
	xload	xmm4, [dstreg+e1+16]		;; R6
	xload	xmm0, [dstreg+e1+48]		;; I6
	subpd	xmm0, xmm4			;; I6 = I6 - R6
	addpd	xmm4, [dstreg+e1+48]		;; R6 = R6 + I6

	xstore	[dstreg+16], xmm2		;; R5
	xstore	[dstreg+48], xmm1		;; I5

	;; multiply R8/I8 by SQRTHALF - i*SQRTHALF
	xload	xmm2, [dstreg+e2+e1+16]		;; R8
	xload	xmm1, [dstreg+e2+e1+48]		;; I8
	subpd	xmm1, xmm2			;; I8 = I8 - R8
	addpd	xmm2, [dstreg+e2+e1+48]		;; R8 = R8 + I8

	xstore	[dstreg+32], xmm5		;; I1
	xload	xmm5, XMM_SQRTHALF
	mulpd	xmm0, xmm5			;; I6 = I6 * SQRTHALF (new I6)
	mulpd	xmm4, xmm5			;; R6 = R6 * SQRTHALF (new R6)
	mulpd	xmm1, xmm5			;; I8 = I8 * SQRTHALF (new I8)
	mulpd	xmm2, xmm5			;; R8 = R8 * SQRTHALF (new R8)

	xstore	[dstreg], xmm3			;; R1

	xload	xmm3, [dstreg+e1]		;; R2
	xload	xmm7, [dstreg+e2+e1]		;; R4
	addpd	xmm7, xmm3			;; R4 = R2 + R4 (new R2)
	subpd	xmm3, [dstreg+e2+e1]		;; R2 = R2 - R4 (new R4)

	xcopy	xmm5, xmm2			;; Copy R8
	subpd	xmm2, xmm4			;; R8 = R8 - R6 (new I8)
	addpd	xmm4, xmm5			;; R6 = R8 + R6 (new R6)

	xcopy	xmm5, xmm0			;; Copy I6
	subpd	xmm0, xmm1			;; I6 = I6 - I8 (new R8)
	addpd	xmm1, xmm5			;; I8 = I6 + I8 (new I6)

	xcopy	xmm5, xmm7			;; Copy R2
	subpd	xmm7, xmm4			;; R2 = R2 - R6 (final R6)
	addpd	xmm4, xmm5			;; R6 = R2 + R6 (final R2)

	xstore	[dstreg+e2+48], xmm6		;; I7
	xstore	[dstreg+e1+16], xmm7		;; R6

	xload	xmm5, [dstreg+e1+32]		;; I2
	xload	xmm6, [dstreg+e2+e1+32]		;; I4
	addpd	xmm6, xmm5			;; I4 = I2 + I4 (new I2)
	subpd	xmm5, [dstreg+e2+e1+32]		;; I2 = I2 - I4 (new I4)

	xcopy	xmm7, xmm3			;; Copy I4
	subpd	xmm3, xmm0			;; R4 = R4 - R8 (final R8)
	addpd	xmm0, xmm7			;; R8 = R4 + R8 (final R4)

	xcopy	xmm7, xmm6			;; Copy I2
	subpd	xmm6, xmm1			;; I2 = I2 - I6 (final I6)
	addpd	xmm1, xmm7			;; I6 = I2 + I6 (final I2)

	xcopy	xmm7, xmm5			;; Copy I4
	subpd	xmm5, xmm2			;; I4 = I4 - I8 (final I8)
	addpd	xmm2, xmm7			;; I8 = I4 + I8 (final I4)

	xstore	[dstreg+e1], xmm4		;; R2
	xstore	[dstreg+e1+32], xmm1		;; I2
	xstore	[dstreg+e2+e1], xmm0		;; R4
	xstore	[dstreg+e2+e1+32], xmm2		;; I4
	xstore	[dstreg+e1+48], xmm6		;; I6
	xstore	[dstreg+e2+e1+16], xmm3		;; R8
	xstore	[dstreg+e2+e1+48], xmm5		;; I8
	bump	dstreg, dstinc
	ENDM
ENDIF

;; Used in the last levels of pass 1.  Swizzles.
r8_sg4cl_eight_complex_unfft8 MACRO srcreg,srcinc,d1,d2,dstreg,dstinc,e1,e2,screg
	shuffle_load_with_temp xmm7, xmm3, [srcreg], [srcreg+16], xmm0 ;; R1,R2
	xload	xmm5, [screg+0+16]		;; cosine/sine
	xcopy	xmm0, xmm7			;; Copy R1
	mulpd	xmm7, xmm5			;; A1 = R1 * cosine/sine
	xload	xmm6, [screg+128+16]		;; cosine/sine
	xcopy	xmm1, xmm3			;; Copy R2
	mulpd	xmm3, xmm6			;; A2 = R2 * cosine/sine

	shuffle_load xmm2, xmm4, [srcreg+32], [srcreg+48] ;; I1,I2
	addpd	xmm7, xmm2			;; A1 = A1 + I1
	mulpd	xmm2, xmm5			;; B1 = I1 * cosine/sine
	addpd	xmm3, xmm4			;; A2 = A2 + I2
	mulpd	xmm4, xmm6			;; B2 = I2 * cosine/sine
	subpd	xmm2, xmm0			;; B1 = B1 - R1
	subpd	xmm4, xmm1			;; B2 = B2 - R2

	shuffle_load_with_temp xmm5, xmm6, [srcreg+d1], [srcreg+d1+16], xmm0 ;; R3,R4
	xload	xmm0, [screg+64+16]		;; cosine/sine
	mulpd	xmm0, xmm5			;; A3 = R3 * cosine/sine

	mulpd	xmm7, [screg+0]			;; A1 = A1 * sine (new R1)
	mulpd	xmm3, [screg+128]		;; A2 = A2 * sine (new R2)

	xcopy	xmm1, xmm7			;; Copy R1
	subpd	xmm7, xmm3			;; R1 = R1 - R2 (new R2)
	addpd	xmm3, xmm1			;; R2 = R1 + R2 (new R1)

	xload	xmm1, [screg+192+16]		;; cosine/sine
	mulpd	xmm1, xmm6			;; A4 = R4 * cosine/sine

	xstore	[dstreg+e1], xmm7		;; R2
	xstore	[dstreg], xmm3			;; R1

	shuffle_load xmm3, xmm7, [srcreg+d1+32], [srcreg+d1+48]	;; I3,I4
	addpd	xmm0, xmm3			;; A3 = A3 + I3
	mulpd	xmm3, [screg+64+16]		;; B3 = I3 * cosine/sine
	addpd	xmm1, xmm7	 		;; A4 = A4 + I4
	mulpd	xmm7, [screg+192+16]		;; B4 = I4 * cosine/sine
	subpd	xmm3, xmm5			;; B3 = B3 - R3
	subpd	xmm7, xmm6			;; B4 = B4 - R4

	mulpd	xmm2, [screg+0]			;; B1 = B1 * sine (new I1)
	mulpd	xmm4, [screg+128]		;; B2 = B2 * sine (new I2)
	xload	xmm5, [screg+64]
	mulpd	xmm0, xmm5			;; A3 = A3 * sine (new R3)
	mulpd	xmm3, xmm5			;; B3 = B3 * sine (new I3)
	xload	xmm5, [screg+192]
	mulpd	xmm1, xmm5			;; A4 = A4 * sine (new R4)
	mulpd	xmm7, xmm5			;; B4 = B4 * sine (new I4)

	xcopy	xmm5, xmm2			;; Copy I1
	subpd	xmm2, xmm4			;; I1 = I1 - I2 (new I2)
	addpd	xmm4, xmm5			;; I2 = I1 + I2 (new I1)

	xcopy	xmm5, xmm1			;; Copy R4
	subpd	xmm1, xmm0			;; R4 = R4 - R3 (new I4)
	addpd	xmm0, xmm5			;; R3 = R4 + R3 (new R3)

	xcopy	xmm5, xmm3			;; Copy I3
	subpd	xmm3, xmm7			;; I3 = I3 - I4 (new R4)
	addpd	xmm7, xmm5			;; I4 = I3 + I4 (new I3)

	xstore	[dstreg+e2+e1], xmm3		;; R4
	xstore	[dstreg+e2+32], xmm7		;; I3
	shuffle_load_with_temp xmm7, xmm3, [srcreg+d2], [srcreg+d2+16], xmm5 ;; R5,R6
	xload	xmm5, [screg+32+16]		;; cosine/sine
	xcopy	xmm6, xmm7			;; Copy R5
	mulpd	xmm7, xmm5			;; A5 = R5 * cosine/sine
	xstore	[dstreg+e2], xmm0		;; R3
	xload	xmm0, [screg+160+16]		;; cosine/sine
	xstore	[dstreg+e2+e1+32], xmm1		;; I4
	xcopy	xmm1, xmm3			;; Copy R6
	mulpd	xmm3, xmm0			;; A6 = R6 * cosine/sine

	xstore	[dstreg+e1+32], xmm2		;; I2
	xstore	[dstreg+32], xmm4		;; I1
	shuffle_load xmm2, xmm4, [srcreg+d2+32], [srcreg+d2+48] ;; I5,I6
	addpd	xmm7, xmm2			;; A5 = A5 + I5
	mulpd	xmm2, xmm5			;; B5 = I5 * cosine/sine
	addpd	xmm3, xmm4			;; A6 = A6 + I6
	mulpd	xmm4, xmm0			;; B6 = I6 * cosine/sine
	subpd	xmm2, xmm6			;; B5 = B5 - R5
	subpd	xmm4, xmm1			;; B6 = B6 - R6

	shuffle_load_with_temp xmm5, xmm6, [srcreg+d2+d1], [srcreg+d2+d1+16], xmm0 ;; R7,R8
	xload	xmm0, [screg+96+16]		;; cosine/sine
	mulpd	xmm0, xmm5			;; A7 = R7 * cosine/sine

	mulpd	xmm7, [screg+32]		;; A5 = A5 * sine (new R5)
	mulpd	xmm3, [screg+160]		;; A6 = A6 * sine (new R6)

	xcopy	xmm1, xmm7			;; Copy R5
	subpd	xmm7, xmm3			;; R5 = R5 - R6 (new R6)
	addpd	xmm3, xmm1			;; R6 = R5 + R6 (new R5)

	xload	xmm1, [screg+224+16]		;; cosine/sine
	mulpd	xmm1, xmm6			;; A8 = R8 * cosine/sine

	xstore	[dstreg+16], xmm3		;; R5
	xstore	[dstreg+e1+16], xmm7		;; R6

	shuffle_load xmm3, xmm7, [srcreg+d2+d1+32], [srcreg+d2+d1+48] ;; I7,I8
	addpd	xmm0, xmm3			;; A7 = A7 + I7
	mulpd	xmm3, [screg+96+16]		;; B7 = I7 * cosine/sine
	addpd	xmm1, xmm7	 		;; A8 = A8 + I8
	mulpd	xmm7, [screg+224+16]		;; B8 = I8 * cosine/sine
	subpd	xmm3, xmm5			;; B7 = B7 - R7
	subpd	xmm7, xmm6			;; B8 = B8 - R8

	mulpd	xmm2, [screg+32]		;; B5 = B5 * sine (new I5)
	mulpd	xmm4, [screg+160]		;; B6 = B6 * sine (new I6)
	xload	xmm5, [screg+96]
	mulpd	xmm0, xmm5			;; A7 = A7 * sine (new R7)
	mulpd	xmm3, xmm5			;; B7 = B7 * sine (new I7)
	xload	xmm5, [screg+224]
	mulpd	xmm1, xmm5			;; A8 = A8 * sine (new R8)
	mulpd	xmm7, xmm5			;; B8 = B8 * sine (new I8)

	bump	srcreg, srcinc

	xcopy	xmm5, xmm2			;; Copy I5
	subpd	xmm2, xmm4			;; I5 = I5 - I6 (new I6)
	addpd	xmm4, xmm5			;; I6 = I5 + I6 (new I5)

	xcopy	xmm5, xmm1			;; Copy R8
	subpd	xmm1, xmm0			;; R8 = R8 - R7 (new I8)
	addpd	xmm0, xmm5			;; R7 = R8 + R7 (new R7)

	xcopy	xmm5, xmm3			;; Copy I7
	subpd	xmm3, xmm7			;; I7 = I7 - I8 (new R8)
	addpd	xmm7, xmm5			;; I8 = I7 + I8 (new I7)

	xload	xmm6, [dstreg+16]		;; R5
	addpd	xmm6, xmm0			;; R5 = R7 + R5 (new R5)
	subpd	xmm0, [dstreg+16]		;; R7 = R7 - R5 (new I7)

	xcopy	xmm5, xmm4			;; Copy I5
	subpd	xmm4, xmm7			;; I5 = I5 - I7 (new R7)
	addpd	xmm7, xmm5			;; I7 = I5 + I7 (new I5)

	xstore	[dstreg+e1+48], xmm2		;; I6

	xload	xmm5, [dstreg]			;; R1
	xload	xmm2, [dstreg+e2]		;; R3
	subpd	xmm5, xmm2			;; R1 = R1 - R3 (new R3)
	addpd	xmm2, [dstreg]			;; R3 = R1 + R3 (new R1)

	xstore	[dstreg+e2+e1+16], xmm3		;; R8

	xcopy	xmm3, xmm5			;; Copy R3
	subpd	xmm5, xmm4			;; R3 = R3 - R7 (final R7)
	addpd	xmm4, xmm3			;; R7 = R3 + R7 (final R3)

	xstore	[dstreg+e2+e1+48], xmm1		;; I8

	xload	xmm3, [dstreg+32]		;; I1
	xload	xmm1, [dstreg+e2+32]		;; I3
	subpd	xmm3, xmm1			;; I1 = I1 - I3 (new I3)
	addpd	xmm1, [dstreg+32]		;; I3 = I1 + I3 (new I1)

	xstore	[dstreg+e2+16], xmm5		;; R7

	xcopy	xmm5, xmm3			;; Copy I3
	subpd	xmm3, xmm0			;; I3 = I3 - I7 (final I7)
	addpd	xmm0, xmm5			;; I7 = I3 + I7 (final I3)

	xcopy	xmm5, xmm2			;; Copy R1
	subpd	xmm2, xmm6			;; R1 = R1 - R5 (final R5)
	addpd	xmm6, xmm5			;; R5 = R1 + R5 (final R1)

	xcopy	xmm5, xmm1			;; Copy I1
	subpd	xmm1, xmm7			;; I1 = I1 - I5 (final I5)
	addpd	xmm7, xmm5			;; I5 = I1 + I5 (final I1)

	xstore	[dstreg+e2], xmm4		;; R3
	xstore	[dstreg+e2+32], xmm0		;; I3

	;; multiply R6/I6 by SQRTHALF - i*SQRTHALF
	xload	xmm4, [dstreg+e1+16]		;; R6
	xload	xmm0, [dstreg+e1+48]		;; I6
	subpd	xmm0, xmm4			;; I6 = I6 - R6
	addpd	xmm4, [dstreg+e1+48]		;; R6 = R6 + I6

	xstore	[dstreg+16], xmm2		;; R5
	xstore	[dstreg+48], xmm1		;; I5

	;; multiply R8/I8 by SQRTHALF - i*SQRTHALF
	xload	xmm2, [dstreg+e2+e1+16]		;; R8
	xload	xmm1, [dstreg+e2+e1+48]		;; I8
	subpd	xmm1, xmm2			;; I8 = I8 - R8
	addpd	xmm2, [dstreg+e2+e1+48]		;; R8 = R8 + I8

	xstore	[dstreg+32], xmm7		;; I1
	xload	xmm5, XMM_SQRTHALF
	mulpd	xmm0, xmm5			;; I6 = I6 * SQRTHALF (new I6)
	mulpd	xmm4, xmm5			;; R6 = R6 * SQRTHALF (new R6)
	mulpd	xmm1, xmm5			;; I8 = I8 * SQRTHALF (new I8)
	mulpd	xmm2, xmm5			;; R8 = R8 * SQRTHALF (new R8)

	xstore	[dstreg], xmm6			;; R1

	xload	xmm6, [dstreg+e1]		;; R2
	xload	xmm7, [dstreg+e2+e1]		;; R4
	addpd	xmm7, xmm6			;; R4 = R2 + R4 (new R2)
	subpd	xmm6, [dstreg+e2+e1]		;; R2 = R2 - R4 (new R4)

	xcopy	xmm5, xmm2			;; Copy R8
	subpd	xmm2, xmm4			;; R8 = R8 - R6 (new I8)
	addpd	xmm4, xmm5			;; R6 = R8 + R6 (new R6)

	xcopy	xmm5, xmm0			;; Copy I6
	subpd	xmm0, xmm1			;; I6 = I6 - I8 (new R8)
	addpd	xmm1, xmm5			;; I8 = I6 + I8 (new I6)

	xcopy	xmm5, xmm7			;; Copy R2
	subpd	xmm7, xmm4			;; R2 = R2 - R6 (final R6)
	addpd	xmm4, xmm5			;; R6 = R2 + R6 (final R2)

	xstore	[dstreg+e2+48], xmm3		;; I7
	xstore	[dstreg+e1+16], xmm7		;; R6

	xload	xmm5, [dstreg+e1+32]		;; I2
	xload	xmm3, [dstreg+e2+e1+32]		;; I4
	addpd	xmm3, xmm5			;; I4 = I2 + I4 (new I2)
	subpd	xmm5, [dstreg+e2+e1+32]		;; I2 = I2 - I4 (new I4)

	xcopy	xmm7, xmm6			;; Copy R4
	subpd	xmm6, xmm0			;; R4 = R4 - R8 (final R8)
	addpd	xmm0, xmm7			;; R8 = R4 + R8 (final R4)

	xcopy	xmm7, xmm3			;; Copy I2
	subpd	xmm3, xmm1			;; I2 = I2 - I6 (final I6)
	addpd	xmm1, xmm7			;; I6 = I2 + I6 (final I2)

	xcopy	xmm7, xmm5			;; Copy I4
	subpd	xmm5, xmm2			;; I4 = I4 - I8 (final I8)
	addpd	xmm2, xmm7			;; I8 = I4 + I8 (final I4)

	xstore	[dstreg+e1], xmm4		;; R2
	xstore	[dstreg+e1+32], xmm1		;; I2
	xstore	[dstreg+e2+e1], xmm0		;; R4
	xstore	[dstreg+e2+e1+32], xmm2		;; I4
	xstore	[dstreg+e1+48], xmm3		;; I6
	xstore	[dstreg+e2+e1+16], xmm6		;; R8
	xstore	[dstreg+e2+e1+48], xmm5		;; I8
	bump	dstreg, dstinc
	ENDM

IFDEF X86_64

r8_sg4cl_eight_complex_unfft8 MACRO srcreg,srcinc,d1,d2,dstreg,dstinc,e1,e2,screg
	shuffle_load_with_temp xmm1, xmm2, [srcreg], [srcreg+16], xmm0 ;; R1,R2

	xload	xmm3, [screg+0+16]		;; cosine/sine 1		; 1
	xcopy	xmm15, xmm1			;; Copy R1			; 1-3
	mulpd	xmm1, xmm3			;; A1 = R1 * cosine/sine	; 1-5

	xload	xmm4, [screg+128+16]		;; cosine/sine 2		; 2
	xcopy	xmm5, xmm2			;; Copy R2			; 2-4
	mulpd	xmm2, xmm4			;; A2 = R2 * cosine/sine	; 2-6

	shuffle_load_with_temp xmm6, xmm7, [srcreg+32], [srcreg+48], xmm0 ;; I1,I2

	mulpd	xmm3, xmm6			;; B1 = I1 * cosine/sine	; 3-7
	mulpd	xmm4, xmm7			;; B2 = I2 * cosine/sine	; 4-8

	shuffle_load_with_temp xmm8, xmm9, [srcreg+d1], [srcreg+d1+16], xmm0 ;; R3,R4

	xload	xmm10, [screg+64+16]		;; cosine/sine 3		; 5
	xcopy	xmm11, xmm8			;; Copy R3			; 5-7
	mulpd	xmm8, xmm10			;; A3 = R3 * cosine/sine	; 5-9

	addpd	xmm1, xmm6			;; A1 = A1 + I1			; 6-8	avail 0,6,12-14
	xload	xmm12, [screg+192+16]		;; cosine/sine 4		; 6	avail 0,6,13,14
	xcopy	xmm13, xmm9			;; Copy R4			; 6-8	avail 0,6,14
	mulpd	xmm9, xmm12			;; A4 = R4 * cosine/sine	; 6-10

	shuffle_load_with_temp xmm6, xmm14, [srcreg+d1+32], [srcreg+d1+48], xmm0 ;; I3,I4

	addpd	xmm2, xmm7			;; A2 = A2 + I2			; 7-9	avail 0,7
	mulpd	xmm10, xmm6			;; B3 = I3 * cosine/sine	; 7-11
	xload	xmm0, [screg+0]			;; sine 1			; 7	avail 7

	subpd	xmm3, xmm15			;; B1 = B1 - R1			; 8-10	avail 7,15
	mulpd	xmm12, xmm14			;; B4 = I4 * cosine/sine	; 8-12
	xload	xmm7, [screg+128]		;; sine 2			; 8	avail 15

	subpd	xmm4, xmm5			;; B2 = B2 - R2			; 9-11	avail 15,5
	mulpd	xmm1, xmm0			;; A1 = A1 * sine (new R1)	; 9-13

	addpd	xmm8, xmm6			;; A3 = A3 + I3			; 10-12	avail 15,5,6
	mulpd	xmm2, xmm7			;; A2 = A2 * sine (new R2)	; 10-14
	xload	xmm5, [screg+64]		;; sine 3			; 10	avail 15,6

	addpd	xmm9, xmm14			;; A4 = A4 + I4			; 11-13	avail 15,6,14
	mulpd	xmm3, xmm0			;; B1 = B1 * sine (new I1)	; 11-15	avail 15,6,14,0
	xload	xmm6, [screg+192]		;; sine 4			; 11	avail 15,14,0

	subpd	xmm10, xmm11			;; B3 = B3 - R3			; 12-14	avail 15,14,0,11
	mulpd	xmm4, xmm7			;; B2 = B2 * sine (new I2)	; 12-16	avail 15,14,0,11,7

	subpd	xmm12, xmm13			;; B4 = B4 - R4			; 13-15	avail 15,14,0,11,7,13
	mulpd	xmm8, xmm5			;; A3 = A3 * sine (new R3)	; 13-17

	xcopy	xmm0, xmm1			;; Copy R1			; 14-16 (14 on a Core i7) avail 15,14,11,7,13
	mulpd	xmm9, xmm6			;; A4 = A4 * sine (new R4)	; 14-18

	shuffle_load_with_temp xmm15, xmm14, [srcreg+d2], [srcreg+d2+16], xmm7 ;; R5,R6

	subpd	xmm1, xmm2			;; R1 = R1 - R2 (newer R2)	; 15-17
	mulpd	xmm10, xmm5			;; B3 = B3 * sine (new I3)	; 15-19	avail 11,7,13,5
	xload	xmm11, [screg+32+16]		;; cosine/sine 5		; 15

	xprefetchw [dstreg+dstinc]

	addpd	xmm2, xmm0			;; R2 = R1 + R2 (newer R1)	; 17-19 (16-18 on a Core i7) avail 7,13,5,0
	mulpd	xmm12, xmm6			;; B4 = B4 * sine (new I4)	; 16-20	avail 7,13,5,0,6
	xcopy	xmm0, xmm3			;; Copy I1			; 16-18	avail 7,13,5,6

	subpd	xmm3, xmm4			;; I1 = I1 - I2 (newer I2)	; 18-20
	xcopy	xmm5, xmm8			;; Copy R3			; 18-20	avail 7,13,6
	xload	xmm7, [screg+160+16]		;; cosine/sine 6		; 18	avail 13,6
	xstore	[dstreg+e1], xmm1		;; Save R2			; 18	avail 13,6,1

	addpd	xmm4, xmm0			;; I2 = I1 + I2 (newer I1)	; 19-21	avail 13,6,1,0
	xcopy	xmm6, xmm15			;; Copy R5			; 19-21	avail 13,1,0
	mulpd	xmm15, xmm11			;; A5 = R5 * cosine/sine	; 19-23

	addpd	xmm8, xmm9			;; R3 = R4 + R3 (newer R3)	; 20-22
	xcopy	xmm13, xmm14			;; Copy R6			; 20-22	avail 1,0
	mulpd	xmm14, xmm7			;; A6 = R6 * cosine/sine	; 20-24
	xstore	[dstreg], xmm2			;; Save R1			; 20	avail 1,0,2
	shuffle_load_with_temp xmm1, xmm2, [srcreg+d2+32], [srcreg+d2+48], xmm0	;; I5,I6
	xcopy	xmm0, xmm10			;; Copy I3			; 20-22	avail none

	subpd	xmm9, xmm5			;; R4 = R4 - R3 (newer I4)	; 21-23	avail 5
	mulpd	xmm11, xmm1			;; B5 = I5 * cosine/sine	; 21-25
	xstore	[dstreg+e1+32], xmm3		;; Save I2			; 21	avail 5,3

	subpd	xmm10, xmm12			;; I3 = I3 - I4 (newer R4)	; 22-24 avail 5,3
	mulpd	xmm7, xmm2			;; B6 = I6 * cosine/sine	; 22-26
	xstore	[dstreg+32], xmm4		;; Save I1			; 22	avail 5,3,4

	addpd	xmm12, xmm0			;; I4 = I3 + I4 (newer I3)	; 23-25	avail 5,3,4,0
	shuffle_load_with_temp xmm5, xmm3, [srcreg+d2+d1], [srcreg+d2+d1+16], xmm0 ;; R7,R8
	xload	xmm4, [screg+96+16]		;; cosine/sine 7		; 23	avail 0
	xcopy	xmm0, xmm5			;; Copy R7			; 23-25	avail none
	mulpd	xmm5, xmm4			;; A7 = R7 * cosine/sine	; 23-27
	xstore	[dstreg+16], xmm8		;; Save R3			; 23	avail 8

	addpd	xmm15, xmm1			;; A5 = A5 + I5			; 24-26	avail 8,1
	xload	xmm8, [screg+224+16]		;; cosine/sine 8		; 24	avail 1
	xcopy	xmm1, xmm3			;; Copy R8			; 24-26	avail none
	mulpd	xmm3, xmm8			;; A8 = R8 * cosine/sine	; 24-28
	xstore	[dstreg+e1+48], xmm9		;; Save I4			; 24	avail 9

	xprefetchw [dstreg+dstinc+e1]

	addpd	xmm14, xmm2			;; A6 = A6 + I6			; 25-27	avail 9,2
	xstore	[dstreg+e1+16], xmm10		;; Save R4			; 25	avail 9,2,10
	shuffle_load_with_temp xmm9, xmm10, [srcreg+d2+d1+32], [srcreg+d2+d1+48], xmm2 ;; I7,I8
	mulpd	xmm4, xmm9			;; B7 = I7 * cosine/sine	; 25-29	avail 2

	subpd	xmm11, xmm6			;; B5 = B5 - R5			; 26-28	avail 2,6
	mulpd	xmm8, xmm10			;; B8 = I8 * cosine/sine	; 26-30
	xload	xmm6, [screg+32]		;; sine 5			; 26	avail 2

	subpd	xmm7, xmm13			;; B6 = B6 - R6			; 27-29	avail 2,13
	mulpd	xmm15, xmm6			;; A5 = A5 * sine (new R5)	; 27-31
	xload	xmm2, [screg+160]		;; sine 6			; 27	avail 13

	addpd	xmm5, xmm9			;; A7 = A7 + I7			; 28-30	avail 13,9
	mulpd	xmm14, xmm2			;; A6 = A6 * sine (new R6)	; 28-32
	xload	xmm13, [screg+96]		;; sine 7			; 28	avail 9

	addpd	xmm3, xmm10			;; A8 = A8 + I8			; 29-31	avail 9,10
	mulpd	xmm11, xmm6			;; B5 = B5 * sine (new I5)	; 29-33	avail 9,10,6
	xload	xmm9, [screg+224]		;; sine 8			; 29	avail 10,6

	subpd	xmm4, xmm0			;; B7 = B7 - R7			; 30-32	avail 10,6,0
	mulpd	xmm7, xmm2			;; B6 = B6 * sine (new I6)	; 30-34	avail 10,6,0,2
	xcopy	xmm0, xmm12			;; Copy I3			; 30-32	avail 10,6,2
	xload	xmm6, [dstreg+32]		;; Reload I1			; 30	avail 10,2

	subpd	xmm8, xmm1			;; B8 = B8 - R8			; 31-33	avail 10,2,1
	mulpd	xmm5, xmm13			;; A7 = A7 * sine (new R7)	; 31-35

	bump	srcreg, srcinc

	addpd	xmm12, xmm6			;; I3 = I1 + I3 (new I1)	; 32-34
	mulpd	xmm3, xmm9			;; A8 = A8 * sine (new R8)	; 32-36
	xcopy	xmm2, xmm15			;; Copy R5			; 32-34	avail 10,1

	subpd	xmm6, xmm0			;; I1 = I1 - I3 (new I3)	; 33-35	avail 10,1,0
	mulpd	xmm4, xmm13			;; B7 = B7 * sine (new I7)	; 33-37	avail 10,1,0,13

	subpd	xmm15, xmm14			;; R5 = R5 - R6 (newer R6)	; 34-36
	mulpd	xmm8, xmm9			;; B8 = B8 * sine (new I8)	; 34-38	avail 10,1,0,13,9
	xcopy	xmm0, xmm11			;; Copy I5			; 34-36	avail 10,1,13,9

	xprefetchw [dstreg+dstinc+e2]

	addpd	xmm14, xmm2			;; R6 = R5 + R6 (newer R5)	; 35-37	avail 10,1,13,9,2

	subpd	xmm11, xmm7			;; I5 = I5 - I6 (newer I6)	; 36-38
	xcopy	xmm13, xmm5			;; Copy R7			; 36-38	avail 10,1,9,2

	addpd	xmm7, xmm0			;; I6 = I5 + I6 (newer I5)	; 37-39	avail 10,1,9,2,0
	xload	xmm2, XMM_SQRTHALF						; 37	avail 10,1,9,0

	addpd	xmm5, xmm3			;; R7 = R8 + R7 (newer R7)	; 38-40
	xcopy	xmm0, xmm4			;; Copy I7			; 38-40	avail 10,1,9

	subpd	xmm3, xmm13			;; R8 = R8 - R7 (newer I8)	; 39-41	avail 10,1,9,13

	subpd	xmm4, xmm8			;; I7 = I7 - I8 (newer R8)	; 40-42
	xcopy	xmm1, xmm15			;; Copy R6			; 40-42 (37) avail 10,9,13

	addpd	xmm8, xmm0			;; I8 = I7 + I8 (newer I7)	; 41-43	avail 10,9,13,0

	xprefetchw [dstreg+dstinc+e2+e1]

						;; multiply R6/I6 by SQRTHALF - i*SQRTHALF
	addpd	xmm15, xmm11			;; R6 = R6 + I6			; 42-44
	xcopy	xmm0, xmm3			;; Copy I8			; 42-44	avail 10,9,13

	subpd	xmm11, xmm1			;; I6 = I6 - R6			; 43-45	avail 10,9,13,1
	xload	xmm10, [dstreg]			;; Reload R1			; 43	avail 9,13,1

						;; multiply R8/I8 by SQRTHALF - i*SQRTHALF
	subpd	xmm3, xmm4			;; I8 = I8 - R8			; 44-46
	xcopy	xmm1, xmm14			;; Copy R5			; 44-46 (38) avail 9,13
	xload	xmm9, [dstreg+16]		;; Reload R3			; 44	avail 13

	addpd	xmm4, xmm0			;; R8 = R8 + I8			; 45-47	avail 13,0
	mulpd	xmm15, xmm2			;; R6 = R6 * SQRTHALF (newer R6); 45-49

	addpd	xmm14, xmm5			;; R5 = R7 + R5 (new R5)	; 46-48
	mulpd	xmm11, xmm2			;; I6 = I6 * SQRTHALF (newer I6); 46-50
	xcopy	xmm0, xmm7			;; Copy I5			; 46-48 (40) avail 13

	subpd	xmm5, xmm1			;; R7 = R7 - R5 (new I7)	; 47-49	avail 13,1
	mulpd	xmm3, xmm2			;; I8 = I8 * SQRTHALF (newer I8); 47-51

	subpd	xmm7, xmm8			;; I5 = I5 - I7 (new R7)	; 48-50
	mulpd	xmm4, xmm2			;; R8 = R8 * SQRTHALF (newer R8); 48-52	avail 13,1,2
	xcopy	xmm1, xmm10			;; Copy R1			; 48-50	avail 13,2

	addpd	xmm8, xmm0			;; I7 = I5 + I7 (new I5)	; 49-51	avail 13,2,0
	xload	xmm13, [dstreg+e1]		;; Reload R2			; 49	avail 2,0

	subpd	xmm10, xmm9			;; R1 = R1 - R3 (new R3)	; 50-52
	xcopy	xmm2, xmm6			;; Copy I3			; 50-52	avail 0

	addpd	xmm9, xmm1			;; R3 = R1 + R3 (new R1)	; 51-53	avail 0,1
	xload	xmm0, [dstreg+e1+16]		;; Reload R4			; 51	avail 1

	subpd	xmm6, xmm5			;; I3 = I3 - I7 (final I7)	; 52-54	avail 1 storable 6
	xcopy	xmm1, xmm12			;; Copy I1			; 52-54	avail none storable 6

	addpd	xmm5, xmm2			;; I7 = I3 + I7 (final I3)	; 53-55	avail 2 storable 6,5

	subpd	xmm12, xmm8			;; I1 = I1 - I5 (final I5)	; 54-56	avail 2 storable 6,5,12
	xcopy	xmm2, xmm10			;; Copy R3			; 54-56	avail none storable 6,5,12

	addpd	xmm8, xmm1			;; I5 = I1 + I5 (final I1)	; 55-57	avail 1 storable 6,5,12,8
	xstore	[dstreg+e2+48], xmm6		;; I7				; 55	avail 1,6 storable 5,12,8
	xload	xmm6, [dstreg+e1+32]		;; Reload I2			; 55	avail 1 storable 5,12,8

	subpd	xmm10, xmm7			;; R3 = R3 - R7 (final R7)	; 56-58	avail 1 storable 5,12,8,10
	xcopy	xmm1, xmm9			;; Copy R1			; 56-58	avail none storable 5,12,8,10
	xstore	[dstreg+e2+32], xmm5		;; I3				; 56	avail 5 storable 12,8,10

	addpd	xmm7, xmm2			;; R7 = R3 + R7 (final R3)	; 57-59	avail 5,2 storable 12,8,10,7
	xload	xmm5, [dstreg+e1+48]		;; Reload I4			; 57	avail 2 storable 12,8,10,7
	xstore	[dstreg+48], xmm12		;; I5				; 57	avail 2,12 storable 8,10,7

	subpd	xmm9, xmm14			;; R1 = R1 - R5 (final R5)	; 58-60	avail 2,12 storable 8,10,7,9
	xcopy	xmm2, xmm13			;; Copy R2			; 58-60	avail 12 storable 8,10,7,9
	xstore	[dstreg+32], xmm8		;; I1				; 58	avail 12,8 storable 10,7,9

	addpd	xmm14, xmm1			;; R5 = R1 + R5 (final R1)	; 59-61	avail 12,8,1 storable 10,7,9,14
	xstore	[dstreg+e2+16], xmm10		;; R7				; 59	avail 12,8,1,10 storable 7,9,14

	addpd	xmm13, xmm0			;; R4 = R2 + R4 (new R2)	; 60-62
	xcopy	xmm1, xmm4			;; Copy R8			; 60-62	avail 12,8,10 storable 7,9,14
	xstore	[dstreg+e2], xmm7		;; R3				; 60	avail 12,8,10,7 storable 9,14

	subpd	xmm2, xmm0			;; R2 = R2 - R4 (new R4)	; 61-63	avail 12,8,10,7,0 storable 9,14
	xstore	[dstreg+16], xmm9		;; R5				; 61	avail 12,8,10,7,0,9 storable 14

	subpd	xmm4, xmm15			;; R8 = R8 - R6 (new I8)	; 62-64
	xcopy	xmm0, xmm6			;; Copy I2			; 62-64	avail 12,8,10,7,9 storable 14
	xstore	[dstreg], xmm14			;; R1				; 62	avail 12,8,10,7,9,14

	addpd	xmm15, xmm1			;; R6 = R8 + R6 (new R6)	; 63-65	avail 12,8,10,7,9,14,1

	addpd	xmm6, xmm5			;; I4 = I2 + I4 (new I2)	; 64-66
	xcopy	xmm1, xmm11			;; Copy I6			; 64-66	avail 12,8,10,7,9,14

	subpd	xmm0, xmm5			;; I2 = I2 - I4 (new I4)	; 65-67	avail 12,8,10,7,9,14,5

	subpd	xmm11, xmm3			;; I6 = I6 - I8 (new R8)	; 66-68
	xcopy	xmm7, xmm13			;; Copy R2			; 66-68	avail 12,8,10,9,14,5

	addpd	xmm3, xmm1			;; I8 = I6 + I8 (new I6)	; 67-69	avail 12,8,10,9,14,5,1

	subpd	xmm13, xmm15			;; R2 = R2 - R6 (final R6)	; 68-70	avail 12,8,10,9,14,5,1 storable 13
	xcopy	xmm1, xmm4			;; Copy I8			; 68-70	avail 12,8,10,9,14,5 storable 13

	addpd	xmm15, xmm7			;; R6 = R2 + R6 (final R2)	; 69-71	avail 12,8,10,9,14,5,7 storable 13,15

	addpd	xmm4, xmm0			;; I8 = I4 + I8 (final I4)	; 70-72	avail 12,8,10,9,14,5,7 storable 13,15,4
	xcopy	xmm5, xmm2			;; Copy R4			; 70-72	avail 12,8,10,9,14,7 storable 13,15,4

	subpd	xmm0, xmm1			;; I4 = I4 - I8 (final I8)	; 71-73	avail 12,8,10,9,14,7,1 storable 13,15,4,0
	xstore	[dstreg+e1+16], xmm13		;; R6				; 71	avail 12,8,10,9,14,7,1,13 storable 15,4,0

	subpd	xmm2, xmm11			;; R4 = R4 - R8 (final R8)	; 72-74	avail 12,8,10,9,14,7,1,13 storable 15,4,0,2
	xcopy	xmm1, xmm6			;; Copy I2			; 72-74	avail 12,8,10,9,14,7,13 storable 15,4,0,2
	xstore	[dstreg+e1], xmm15		;; R2				; 72	avail 12,8,10,9,14,7,13,15 storable 4,0,2

	addpd	xmm11, xmm5			;; R8 = R4 + R8 (final R4)	; 73-75	avail 12,8,10,9,14,7,13,15,5 storable 4,0,2,11
	xstore	[dstreg+e2+e1+32], xmm4		;; I4				; 73	avail 12,8,10,9,14,7,13,15,5,4 storable 0,2,11

	subpd	xmm6, xmm3			;; I2 = I2 - I6 (final I6)	; 74-76	avail 12,8,10,9,14,7,13,15,5,4 storable 0,2,11,6
	xstore	[dstreg+e2+e1+48], xmm0		;; I8				; 74	avail 12,8,10,9,14,7,13,15,5,4,0 storable 2,11,6

	addpd	xmm3, xmm1			;; I6 = I2 + I6 (final I2)	; 75-77	avail 12,8,10,9,14,7,13,15,5,4,0,1 storable 2,11,6,3
	xstore	[dstreg+e2+e1+16], xmm2		;; R8				; 75

	xstore	[dstreg+e2+e1], xmm11		;; R4				; 76
	xstore	[dstreg+e1+48], xmm6		;; I6				; 77
	xstore	[dstreg+e1+32], xmm3		;; I2				; 78
	bump	dstreg, dstinc
	ENDM

ENDIF
;;
;; ************************************* sixteen-reals-four-complex-fft8 variants ******************************************
;;
;; Because of the way we store FFT data elements in cache lines during the pass 1 of the forward FFT, a cache line
;; will contain data needing an eight-reals-fft as well as data needing a four-complex-fft.

;;
;; Do three-and-7/8 levels of the forward FFT on 16 real data values with 7 sin/cos multipliers.
;; Output is 2 real values needing N+1 more levels and 7 complex values needing N more FFT levels.
;; Also do a forward FFT on 8 complex values.
;;
;; To simplify implementation of the 16 reals case, I first operate on 4 sets of 4 reals doing
;; one-and-1/2 levels.  Each set outputs two real values and one complex value.  The four complex
;; values are multiplied by w^1.  Then I do one eight-reals-FFT which can use the sin/cos multipliers
;; that r8_g8cl_eight_complex_djbfft uses.  Finally, the one four-complex-FFT uses two more
;; sin/cos multipliers.
;;

;; Used in last levels of pass 1 (split premultiplier and delay cases).  No swizzling.
IFDEF UNUSED
r8_g8cl_sixteen_reals_eight_complex_fft8 MACRO srcreg,srcinc,d1,d2,d4,dstreg,dstinc,e1,e2,e4,screg1,screg2
	not written
	ENDM
ENDIF

;; Used in last levels of pass 1 (split premultiplier and delay cases).  Swizzling.

r8_sg8cl_sixteen_reals_eight_complex_fft8_preload MACRO
	r4_x8r_fft_mem_preload
	ENDM
r8_sg8cl_sixteen_reals_eight_complex_fft8 MACRO srcreg,srcinc,d1,d2,d4,dstreg,dstinc,e1,e2,e4,screg1,screg2
	e3 = e2 + e1

;; Do first level on the first 8 reals and 2 levels on the last 8 reals.
;; The second level on the last eight reals is the multiply by i.

	xload	xmm0, [srcreg]		;; R1
	xload	xmm2, [srcreg+16]	;; R9
	subpd	xmm0, xmm2		;; new R9 = R1 - R9
	addpd	xmm2, [srcreg]		;; new R1 = R1 + R9

	xload	xmm1, [srcreg+d4]	;; R5
	xload	xmm3, [srcreg+d4+16]	;; R13
	subpd	xmm1, xmm3		;; new I9 = R5 - R13
	addpd	xmm3, [srcreg+d4]	;; new R5 = R5 + R13

	xload	xmm7, [screg2+16]	;; cosine/sine
	mulpd	xmm7, xmm0		;; A9 = R9 * cosine/sine
	subpd	xmm7, xmm1		;; A9 = A9 - I9
	mulpd	xmm1, [screg2+16]	;; B9 = I9 * cosine/sine
	addpd	xmm1, xmm0		;; B9 = B9 + R9

	mulpd	xmm7, [screg2]		;; A9 = A9 * sine (final R9)
	mulpd	xmm1, [screg2]		;; B9 = B9 * sine (final I9)

	xstore	[dstreg], xmm2		;; Save R1
	xstore	[dstreg+e1], xmm3	;; Save R5
	xstore	[dstreg+e2], xmm7	;; Save R9
	xstore	[dstreg+e2+e1], xmm1	;; Save I9

	xload	xmm4, [srcreg+d1]	;; R2
	xload	xmm6, [srcreg+d1+16]	;; R10
	subpd	xmm4, xmm6		;; new R10 = R2 - R10
	addpd	xmm6, [srcreg+d1]	;; new R2 = R2 + R10

	xload	xmm5, [srcreg+d4+d1]	;; R6
	xload	xmm7, [srcreg+d4+d1+16]	;; R14
	subpd	xmm5, xmm7		;; new I10 = R6 - R14
	addpd	xmm7, [srcreg+d4+d1]	;; new R6 = R6 + R14

	xload	xmm2, [screg2+48]	;; cosine/sine
	mulpd	xmm2, xmm4		;; A10 = R10 * cosine/sine
	subpd	xmm2, xmm5		;; A10 = A10 - I10
	mulpd	xmm5, [screg2+48]	;; B10 = I10 * cosine/sine
	addpd	xmm5, xmm4		;; B10 = B10 + R10

	mulpd	xmm2, [screg2+32]	;; A10 = A10 * sine (final R10)
	mulpd	xmm5, [screg2+32]	;; B10 = B10 * sine (final I10)

	xstore	[dstreg+16], xmm6	;; Save R2
	xstore	[dstreg+e1+16], xmm7	;; Save R6
	xstore	[dstreg+e2+16], xmm2	;; Save R10
	xstore	[dstreg+e2+e1+16], xmm5	;; Save I10

	xload	xmm0, [srcreg+d2]	;; R3
	xload	xmm2, [srcreg+d2+16]	;; R11
	subpd	xmm0, xmm2		;; new R11 = R3 - R11
	addpd	xmm2, [srcreg+d2]	;; new R3 = R3 + R11

	xload	xmm1, [srcreg+d4+d2]	;; R7
	xload	xmm3, [srcreg+d4+d2+16]	;; R15
	subpd	xmm1, xmm3		;; new I11 = R7 - R15
	addpd	xmm3, [srcreg+d4+d2]	;; new R7 = R7 + R15

	xload	xmm7, [screg2+80]	;; cosine/sine
	mulpd	xmm7, xmm0		;; A11 = R11 * cosine/sine
	subpd	xmm7, xmm1		;; A11 = A11 - I11
	mulpd	xmm1, [screg2+80]	;; B11 = I11 * cosine/sine
	addpd	xmm1, xmm0		;; B11 = B11 + R11

	mulpd	xmm7, [screg2+64]	;; A11 = A11 * sine (final R11)
	mulpd	xmm1, [screg2+64]	;; B11 = B11 * sine (final I11)

	xstore	[dstreg+32], xmm2	;; Save R3
	xstore	[dstreg+e1+32], xmm3	;; Save R7
	xstore	[dstreg+e2+32], xmm7	;; Save R11
	xstore	[dstreg+e2+e1+32], xmm1	;; Save I11

	xload	xmm4, [srcreg+d2+d1]	;; R4
	xload	xmm6, [srcreg+d2+d1+16]	;; R12
	subpd	xmm4, xmm6		;; new R12 = R4 - R12
	addpd	xmm6, [srcreg+d2+d1]	;; new R4 = R4 + R12

	xload	xmm5, [srcreg+d4+d2+d1]	;; R8
	xload	xmm7, [srcreg+d4+d2+d1+16] ;; R16
	subpd	xmm5, xmm7		;; new I12 = R8 - R16
	addpd	xmm7, [srcreg+d4+d2+d1]	;; new R8 = R8 + R16

	xload	xmm2, [screg2+112]	;; cosine/sine
	mulpd	xmm2, xmm4		;; A12 = R12 * cosine/sine
	subpd	xmm2, xmm5		;; A12 = A12 - I12
	mulpd	xmm5, [screg2+112]	;; B12 = I12 * cosine/sine
	addpd	xmm5, xmm4		;; B12 = B12 + R12

	mulpd	xmm2, [screg2+96]	;; A12 = A12 * sine (final R12)
	mulpd	xmm5, [screg2+96]	;; B12 = B12 * sine (final I12)

	xstore	[dstreg+48], xmm6	;; Save R4
	xstore	[dstreg+e1+48], xmm7	;; Save R8
	xstore	[dstreg+e2+48], xmm2	;; Save R12
	xstore	[dstreg+e2+e1+48], xmm5	;; Save I12

;; Do last two and 3/4 levels on the remaining real data

	xprefetch [srcreg+srcinc]
	r4_x8r_fft_mem [dstreg],[dstreg+16],[dstreg+32],[dstreg+48],[dstreg+e1],[dstreg+e1+16],[dstreg+e1+32],[dstreg+e1+48],screg1+32,screg1+64,screg1+160,dstreg+dstinc,e1,[dstreg],[dstreg+32]
	xprefetch [srcreg+srcinc+d1]
	shuffle_store_partial [dstreg], [dstreg+16], xmm0, xmm7				;; Save real value #1,R2
	shuffle_store_partial [dstreg+32], [dstreg+48], xmm1, xmm5			;; Save real value #2,I2
	shuffle_store_with_temp [dstreg+e1], [dstreg+e1+16], xmm4, xmm6, xmm7		;; Save R3,R4
	shuffle_store_with_temp [dstreg+e1+32], [dstreg+e1+48], xmm3, xmm2, xmm7	;; Save I3,I4

;; Do last two levels on the remaining complex data.
;; Use sin/cos values for w^2, w^4, w^6 (we're doing two levels here and the sin/cos data is for three levels)

	xprefetch [srcreg+srcinc+d2]
	r4_x4c_fft_mem [dstreg+e2],[dstreg+e2+16],[dstreg+e2+32],[dstreg+e2+48],[dstreg+e3],[dstreg+e3+16],[dstreg+e3+32],[dstreg+e3+48],screg1+64,screg1+128,screg1+192,dstreg+dstinc+e2,e1,[dstreg+e2],[dstreg+e2+32]
	xprefetch [srcreg+srcinc+d2+d1]
	shuffle_store_partial [dstreg+e2], [dstreg+e2+16], xmm2, xmm7			;; Save R1,R2
	shuffle_store_partial [dstreg+e2+32], [dstreg+e2+48], xmm5, xmm6		;; Save I1,I2
	shuffle_store_with_temp [dstreg+e2+e1], [dstreg+e2+e1+16], xmm3, xmm0, xmm7	;; Save R3,R4
	shuffle_store_with_temp [dstreg+e2+e1+32], [dstreg+e2+e1+48], xmm1, xmm4, xmm7	;; Save I3,I4

;; Now do the eight-complex part.   The code below is very similar to r8_s8c_fft8
;; except that we can eliminate the complex multiply of R1/I1 (because we are
;; processing the first grouping and the sin/cos multiplier is 1).

	xload	xmm0, [srcreg+32]		;; R1
	xload	xmm2, [srcreg+d4+32]		;; R5
	addpd	xmm2, xmm0			;; R5 = R1 + R5 (new R1)

	xload	xmm1, [srcreg+d2+32]		;; R3
	xload	xmm3, [srcreg+d4+d2+32]		;; R7
	addpd	xmm3, xmm1			;; R7 = R3 + R7 (new R3)

	xload	xmm4, [srcreg+48]		;; I1
	xload	xmm6, [srcreg+d4+48]		;; I5
	addpd	xmm6, xmm4			;; I5 = I1 + I5 (new I1)

	subpd	xmm0, [srcreg+d4+32]		;; R1 = R1 - R5 (new R5)
	subpd	xmm1, [srcreg+d4+d2+32]		;; R3 = R3 - R7 (new R7)

	xcopy	xmm5, xmm3			;; Copy R3
	addpd	xmm3, xmm2			;; R3 = R1 + R3 (final R1)
	subpd	xmm2, xmm5			;; R1 = R1 - R3 (final R3)

	xload	xmm5, [srcreg+d2+48]		;; I3
	xload	xmm7, [srcreg+d4+d2+48]		;; I7
	addpd	xmm7, xmm5			;; I7 = I3 + I7 (new I3)

	subpd	xmm4, [srcreg+d4+48]		;; I1 = I1 - I5 (new I5)
	subpd	xmm5, [srcreg+d4+d2+48]		;; I3 = I3 - I7 (new I7)

	xstore	[dstreg+e4], xmm3		;; Save R1

	xcopy	xmm3, xmm7			;; Copy I3
	addpd	xmm7, xmm6			;; I3 = I1 + I3 (final I1)
	subpd	xmm6, xmm3			;; I1 = I1 - I3 (final I3)

	xstore	[dstreg+e4+16], xmm7		;; Save I1

	xcopy	xmm3, xmm1			;; Copy R7
	addpd	xmm1, xmm4			;; R7 = I5 + R7 (final I5)
	xcopy	xmm7, xmm0			;; Copy R5
	subpd	xmm0, xmm5			;; R5 = R5 - I7 (final R5)
	subpd	xmm4, xmm3			;; I5 = I5 - R7 (final I7)
	addpd	xmm5, xmm7			;; I7 = R5 + I7 (final R7)

	xstore	[dstreg+e4+e1], xmm2		;; Save R3
	xstore	[dstreg+e4+e1+16], xmm6		;; Save I3
	xstore	[dstreg+e4+e2], xmm0		;; Save R5
	xstore	[dstreg+e4+e2+16], xmm1		;; Save I5
	xstore	[dstreg+e4+e2+e1], xmm5		;; Save R7
	xstore	[dstreg+e4+e2+e1+16], xmm4	;; Save I7

	xload	xmm0, [srcreg+d1+32]		;; R2
	xload	xmm2, [srcreg+d4+d1+32]		;; R6
	addpd	xmm2, xmm0			;; R6 = R2 + R6 (new R2)

	xload	xmm1, [srcreg+d2+d1+32]		;; R4
	xload	xmm3, [srcreg+d4+d2+d1+32]	;; R8
	addpd	xmm3, xmm1			;; R8 = R4 + R8 (new R4)

	xload	xmm4, [srcreg+d1+48]		;; I2
	xload	xmm6, [srcreg+d4+d1+48]		;; I6
	addpd	xmm6, xmm4			;; I6 = I2 + I6 (new I2)

	subpd	xmm0, [srcreg+d4+d1+32]		;; R2 = R2 - R6 (new R6)
	subpd	xmm1, [srcreg+d4+d2+d1+32]	;; R4 = R4 - R8 (new R8)

	xcopy	xmm5, xmm3			;; Copy R4
	addpd	xmm3, xmm2			;; R4 = R2 + R4 (final R2)
	subpd	xmm2, xmm5			;; R2 = R2 - R4 (final R4)

	xload	xmm5, [srcreg+d2+d1+48]		;; I4
	xload	xmm7, [srcreg+d4+d2+d1+48]	;; I8
	addpd	xmm7, xmm5			;; I8 = I4 + I8 (new I4)

	subpd	xmm4, [srcreg+d4+d1+48]		;; I2 = I2 - I6 (new I6)
	subpd	xmm5, [srcreg+d4+d2+d1+48]	;; I4 = I4 - I8 (new I8)

	xstore	[dstreg+e4+32], xmm3		;; Save R2

	xcopy	xmm3, xmm7			;; Copy I4
	addpd	xmm7, xmm6			;; I4 = I2 + I4 (final I2)
	subpd	xmm6, xmm3			;; I2 = I2 - I4 (final I4)

	xstore	[dstreg+e4+48], xmm7		;; Save I2

	xcopy	xmm3, xmm1			;; Copy I6
	addpd	xmm1, xmm4			;; R8 = I6 + R8 (new I6)
	xcopy	xmm7, xmm0			;; Copy R6
	subpd	xmm0, xmm5			;; R6 = R6 - I8 (new R6)
	subpd	xmm4, xmm3			;; I6 = I6 - R8 (new I8)
	addpd	xmm5, xmm7			;; I8 = R6 + I8 (new R8)

	xcopy	xmm3, xmm0			;; Copy R6
	subpd	xmm0, xmm1			;; R6 = R6 - I6
	addpd	xmm1, xmm3			;; I6 = R6 + I6
	xload	xmm3, XMM_SQRTHALF
	mulpd	xmm0, xmm3			;; R6 = R6 * SQRTHALF (final R6)
	mulpd	xmm1, xmm3			;; I6 = I6 * SQRTHALF (final I6)

	xcopy	xmm7, xmm5			;; Copy R8
	subpd	xmm5, xmm4			;; R8 = R8 - I8
	addpd	xmm4, xmm7			;; I8 = R8 + I8
	mulpd	xmm5, xmm3			;; R8 = R8 * SQRTHALF (final R8)
	mulpd	xmm4, xmm3			;; I8 = I8 * SQRTHALF (final I8)

	xstore	[dstreg+e4+e1+32], xmm2		;; Save R4
	xstore	[dstreg+e4+e1+48], xmm6		;; Save I4
	bump	srcreg, srcinc

;; the last level

	xload	xmm2, [dstreg+e4+e2]		;; R5
	subpd	xmm2, xmm0			;; R5 = R5 - R6 (new R6)
	addpd	xmm0, [dstreg+e4+e2]		;; R6 = R5 + R6 (new R5)

	xload	xmm3, [dstreg+e4+e2+16]		;; I5
	subpd	xmm3, xmm1			;; I5 = I5 - I6 (new I6)
	addpd	xmm1, [dstreg+e4+e2+16]		;; I6 = I5 + I6 (new I5)

	xload	xmm6, [screg1+160+16]		;; cosine/sine
	mulpd	xmm6, xmm2			;; A6 = R6 * cosine/sine
	subpd	xmm6, xmm3			;; A6 = A6 - I6
	mulpd	xmm3, [screg1+160+16]		;; B6 = I6 * cosine/sine
	addpd	xmm3, xmm2			;; B6 = B6 + R6

	xload	xmm2, [dstreg+e4+e2+e1]		;; R7
	subpd	xmm2, xmm4			;; R7 = R7 - I8 (new R7)
	addpd	xmm4, [dstreg+e4+e2+e1]		;; I8 = R7 + I8 (new R8)

	xload	xmm7, [screg1+32+16]		;; cosine/sine
	mulpd	xmm7, xmm0			;; A5 = R5 * cosine/sine
	subpd	xmm7, xmm1			;; A5 = A5 - I5
	mulpd	xmm1, [screg1+32+16]		;; B5 = I5 * cosine/sine
	addpd	xmm1, xmm0			;; B5 = B5 + R5

	xload	xmm0, [dstreg+e4+e2+e1+16]	;; I7
	subpd	xmm0, xmm5			;; I7 = I7 - R8 (new I8)
	addpd	xmm5, [dstreg+e4+e2+e1+16]	;; R8 = I7 + R8 (new I7)

	mulpd	xmm6, [screg1+160]		;; A6 = A6 * sine (final R6)
	mulpd	xmm7, [screg1+32]		;; A5 = A5 * sine (final R5)
	shuffle_store [dstreg+e4+e2], [dstreg+e4+e2+16], xmm7, xmm6 ;; Save R5,R6

	xload	xmm6, [screg1+96+16]		;; cosine/sine
	mulpd	xmm6, xmm2			;; A7 = R7 * cosine/sine
	subpd	xmm6, xmm5			;; A7 = A7 - I7
	mulpd	xmm5, [screg1+96+16]		;; B7 = I7 * cosine/sine
	addpd	xmm5, xmm2			;; B7 = B7 + R7

	mulpd	xmm3, [screg1+160]		;; B6 = B6 * sine (final I6)
	mulpd	xmm1, [screg1+32]		;; B5 = B5 * sine (final I5)

	shuffle_store_with_temp [dstreg+e4+e2+32], [dstreg+e4+e2+48], xmm1, xmm3, xmm7 ;; Save I5,I6

	xload	xmm3, [screg1+224+16]		;; cosine/sine
	mulpd	xmm3, xmm4			;; A8 = R8 * cosine/sine
	subpd	xmm3, xmm0			;; A8 = A8 - I8
	mulpd	xmm0, [screg1+224+16]		;; B8 = I8 * cosine/sine
	addpd	xmm0, xmm4			;; B8 = B8 + R8

	mulpd	xmm6, [screg1+96]		;; A7 = A7 * sine (new R7)
	mulpd	xmm5, [screg1+96]		;; B7 = B7 * sine (new I7)
	mulpd	xmm3, [screg1+224]		;; A8 = A8 * sine (new R8)
	mulpd	xmm0, [screg1+224]		;; B8 = B8 * sine (new I8)

	shuffle_store_with_temp [dstreg+e4+e2+e1], [dstreg+e4+e2+e1+16], xmm6, xmm3, xmm4 ;; Save R7,R8
	shuffle_store_with_temp [dstreg+e4+e2+e1+32], [dstreg+e4+e2+e1+48], xmm5, xmm0, xmm4 ;; Save I7,I8

	xload	xmm3, [dstreg+e4]		;; R1
	xload	xmm4, [dstreg+e4+32]		;; R2
	subpd	xmm3, xmm4			;; R1 = R1 - R2 (new R2)
	addpd	xmm4, [dstreg+e4]		;; R2 = R1 + R2 (final R1)

	xload	xmm7, [dstreg+e4+16]		;; I1
	xload	xmm0, [dstreg+e4+48]		;; I2
	subpd	xmm7, xmm0			;; I1 = I1 - I2 (new I2)
	addpd	xmm0, [dstreg+e4+16]		;; I2 = I1 + I2 (final I1)

	xload	xmm1, [screg1+128+16]		;; cosine/sine
	mulpd	xmm1, xmm3			;; A2 = R2 * cosine/sine
	subpd	xmm1, xmm7			;; A2 = A2 - I2
	mulpd	xmm7, [screg1+128+16]		;; B2 = I2 * cosine/sine
	addpd	xmm7, xmm3			;; B2 = B2 + R2

	mulpd	xmm1, [screg1+128]		;; A2 = A2 * sine (final R2)
	mulpd	xmm7, [screg1+128]		;; B2 = B2 * sine (final I2)

	xload	xmm5, [dstreg+e4+e1]		;; R3
	xload	xmm6, [dstreg+e4+e1+48]		;; I4
	subpd	xmm5, xmm6			;; R3 = R3 - I4 (new R3)
	addpd	xmm6, [dstreg+e4+e1]		;; I4 = R3 + I4 (new R4)

	shuffle_store_with_temp [dstreg+e4], [dstreg+e4+16], xmm4, xmm1, xmm3 ;; Save R1,R2

	xload	xmm3, [dstreg+e4+e1+16]		;; I3
	xload	xmm2, [dstreg+e4+e1+32]		;; R4
	subpd	xmm3, xmm2			;; I3 = I3 - R4 (final I4)
	addpd	xmm2, [dstreg+e4+e1+16]		;; R4 = I3 + R4 (final I3)

	shuffle_store_with_temp [dstreg+e4+32], [dstreg+e4+48], xmm0, xmm7, xmm1 ;; Save I1,I2

	xload	xmm1, [screg1+64+16]		;; cosine/sine
	mulpd	xmm1, xmm5			;; A3 = R3 * cosine/sine
	subpd	xmm1, xmm2			;; A3 = A3 - I3
	mulpd	xmm2, [screg1+64+16]		;; B3 = I3 * cosine/sine
	addpd	xmm2, xmm5			;; B3 = B3 + R3

	xload	xmm7, [screg1+192+16]		;; cosine/sine
	mulpd	xmm7, xmm6			;; A4 = R4 * cosine/sine
	subpd	xmm7, xmm3			;; A4 = A4 - I4
	mulpd	xmm3, [screg1+192+16]		;; B4 = I4 * cosine/sine
	addpd	xmm3, xmm6			;; B4 = B4 + R4

	mulpd	xmm1, [screg1+64]		;; A3 = A3 * sine (final R3)
	mulpd	xmm2, [screg1+64]		;; B3 = B3 * sine (final I3)
	mulpd	xmm7, [screg1+192]		;; A4 = A4 * sine (final R4)
	mulpd	xmm3, [screg1+192]		;; B4 = B4 * sine (final I4)

	shuffle_store_with_temp [dstreg+e4+e1], [dstreg+e4+e1+16], xmm1, xmm7, xmm0 ;; Save R3,R4
	shuffle_store_with_temp [dstreg+e4+e1+32], [dstreg+e4+e1+48], xmm2, xmm3, xmm0 ;; Save I3,I4
	bump	dstreg, dstinc
	ENDM

;;
;; ************************************* sixteen-reals-unfft8 variants ******************************************
;;

;; Used in last levels of pass 1 (split premultiplier and delay cases).  No swizzling.
IFDEF UNUSED
r8_g4cl_sixteen_reals_unfft8 MACRO srcreg,srcinc,d1,d2,dstreg,dstinc,e1,e2,screg1,screg2
	not written
	ENDM
ENDIF

;; Used in last levels of pass 1 (split premultiplier and delay cases).  Swizzling.
r8_sg4cl_sixteen_reals_unfft8 MACRO srcreg,srcinc,d1,d2,dstreg,dstinc,e1,e2,screg1,screg2

;; Do 2 3/4 levels of unfft on the first eight values.

	r4_s8r_unfft4 [srcreg],[srcreg+32],[srcreg+16],[srcreg+48],[srcreg+d1],[srcreg+d1+32],[srcreg+d1+16],[srcreg+d1+48],screg1+32,screg1+64,screg1+160,dstreg+dstinc,e1,[dstreg+e2+e1],[dstreg+e2+e1+16]
	xstore	[dstreg], xmm1		;; Save R1
	xstore	[dstreg+16], xmm0	;; Save R5
	xstore	[dstreg+e1], xmm4	;; Save R2
	xstore	[dstreg+e1+16], xmm3	;; Save R6
	xstore	[dstreg+e2], xmm7	;; Save R3
	xstore	[dstreg+e2+16], xmm6	;; Save R7
;;	xstore	[dstreg+e2+e1+16], xmm2	;; Save R8
;;	xstore	[dstreg+e2+e1], xmm5	;; Save R4

;; Do 2 levels of unfft on the second eight values.  Similar to r4_sg2cl_four_complex_unfft4
;; except there are only 3 sin/cos multiplies (by w^2, w^4, w^6).

	shuffle_load_with_temp xmm6, xmm2, [srcreg+d2], [srcreg+d2+16], xmm4 ;; R1,R2
	xcopy	xmm4, xmm2			;; Copy R2
	mulpd	xmm2, [screg1+128+16]		;; A2 = R2 * cosine/sine
	shuffle_load_with_temp xmm1, xmm3, [srcreg+d2+32], [srcreg+d2+48], xmm7 ;; I1,I2
	addpd	xmm2, xmm3			;; A2 = A2 + I2
	mulpd	xmm3, [screg1+128+16]		;; B2 = I2 * cosine/sine
	subpd	xmm3, xmm4			;; B2 = B2 - R2
	mulpd	xmm2, [screg1+128]		;; A2 = A2 * sine (new R2)

	shuffle_load_with_temp xmm7, xmm0, [srcreg+d2+d1], [srcreg+d2+d1+16], xmm4 ;; R3,R4
	xcopy	xmm4, xmm7			;; Copy R3
	mulpd	xmm7, [screg1+64+16]		;; A3 = R3 * cosine/sine
	xcopy	xmm5, xmm0			;; Copy R4
	mulpd	xmm0, [screg1+192+16]		;; A4 = R4 * cosine/sine

	xstore	[dstreg+32], xmm6		;; Save new R1 temporarily
	xstore	[dstreg+48], xmm2		;; Save new R2 temporarily

	shuffle_load xmm6, xmm2, [srcreg+d2+d1+32], [srcreg+d2+d1+48] ;; I3,I4

	addpd	xmm7, xmm6			;; A3 = A3 + I3
	mulpd	xmm6, [screg1+64+16]		;; B3 = I3 * cosine/sine
	addpd	xmm0, xmm2			;; A4 = A4 + I4
	mulpd	xmm2, [screg1+192+16]		;; B4 = I4 * cosine/sine
	subpd	xmm6, xmm4			;; B3 = B3 - R3
	subpd	xmm2, xmm5			;; B4 = B4 - R4

	mulpd	xmm3, [screg1+128]		;; B2 = B2 * sine (new I2)
	mulpd	xmm7, [screg1+64]		;; A3 = A3 * sine (new R3)
	mulpd	xmm0, [screg1+192]		;; A4 = A4 * sine (new R4)
	mulpd	xmm6, [screg1+64]		;; B3 = B3 * sine (new I3)
	mulpd	xmm2, [screg1+192]		;; B4 = B4 * sine (new I4)

	xcopy	xmm4, xmm1			;; Copy I1
	subpd	xmm1, xmm3			;; I1 = I1 - I2 (new I2)
	addpd	xmm3, xmm4			;; I2 = I1 + I2 (new I1)

	xprefetchw [dstreg+dstinc+e2]
	bump	srcreg, srcinc

	xcopy	xmm4, xmm0			;; Copy R4
	subpd	xmm0, xmm7			;; R4 = R4 - R3 (new I4)
	addpd	xmm7, xmm4			;; R3 = R4 + R3 (new R3)

	xcopy	xmm4, xmm6			;; Copy I3
	subpd	xmm6, xmm2			;; I3 = I3 - I4 (new R4)
	addpd	xmm2, xmm4			;; I4 = I3 + I4 (new I3)

	xcopy	xmm4, xmm1			;; Copy I2
	subpd	xmm1, xmm0			;; I2 = I2 - I4 (final I4)
	addpd	xmm0, xmm4			;; I4 = I2 + I4 (final I2)

	xload	xmm4, [dstreg+32]		;; Reload new R1
	xload	xmm5, [dstreg+48]		;; Reload new R2
	subpd	xmm4, xmm5			;; R1 = R1 - R2 (new R2)
	addpd	xmm5, [dstreg+32]		;; R2 = R1 + R2 (new R1)

	xstore	[dstreg+e2+e1+48], xmm1		;; Save I4 (new I12)

	xcopy	xmm1, xmm3			;; Copy I1
	subpd	xmm3, xmm2			;; I1 = I1 - I3 (final I3)
	addpd	xmm2, xmm1			;; I3 = I1 + I3 (final I1)

	xcopy	xmm1, xmm4			;; Copy R2
	subpd	xmm4, xmm6			;; R2 = R2 - R4 (final R4)
	addpd	xmm6, xmm1			;; R4 = R2 + R4 (final R2)

	xprefetchw [dstreg+dstinc+e2+e1]

	xcopy	xmm1, xmm5			;; Copy R1
	subpd	xmm5, xmm7			;; R1 = R1 - R3 (final R3)
	addpd	xmm7, xmm1			;; R3 = R1 + R3 (final R1)

;;	xstore	[dstreg+32], xmm7		;; Save R1 (new R9)
;;	xstore	[dstreg+48], xmm2		;; Save I1 (new I9)
;;	xstore	[dstreg+e1+32], xmm6		;; Save R2 (new R10)
;;	xstore	[dstreg+e1+48], xmm0		;; Save I2 (new I10)
;;	xstore	[dstreg+e2+32], xmm5		;; Save R3 (new R11)
;;	xstore	[dstreg+e2+48], xmm3		;; Save I3 (new I11)
	xstore	[dstreg+e2+e1+32], xmm4		;; Save R4 (new R12)

;; Do first level on the first 8 reals and 2 levels on the last 8 reals.
;; The second level on the last eight reals is the multiply by i (the nop level).

	xload	xmm1, [screg2+16]	;; cosine/sine
	xcopy	xmm4, xmm7		;; Copy R9
	mulpd	xmm7, xmm1		;; A9 = R9 * cosine/sine
	addpd	xmm7, xmm2		;; A9 = A9 + I9
	mulpd	xmm2, xmm1		;; B9 = I9 * cosine/sine
	subpd	xmm2, xmm4		;; B9 = B9 - R9
	mulpd	xmm7, [screg2]		;; A9 = A9 * sine (new R9)
	mulpd	xmm2, [screg2]		;; B9 = B9 * sine (new R13)

	xload	xmm1, [dstreg]		;; R1
	subpd	xmm1, xmm7		;; new R9 = R1 - R9
	addpd	xmm7, [dstreg]		;; new R1 = R1 + R9
	xstore	[dstreg], xmm7		;; Save R1
	xstore	[dstreg+32], xmm1	;; Save R9

	xload	xmm1, [dstreg+16]	;; R5
	subpd	xmm1, xmm2		;; new R13 = R5 - R13
	addpd	xmm2, [dstreg+16]	;; new R5 = R5 + R13
	xstore	[dstreg+16], xmm2	;; Save R5
	xstore	[dstreg+48], xmm1	;; Save R13

	xload	xmm1, [screg2+32+16]	;; cosine/sine
	xcopy	xmm4, xmm6		;; Copy R10
	mulpd	xmm6, xmm1		;; A10 = R10 * cosine/sine
	addpd	xmm6, xmm0		;; A10 = A10 + I10
	mulpd	xmm0, xmm1		;; B10 = I10 * cosine/sine
	subpd	xmm0, xmm4		;; B10 = B10 - R10
	mulpd	xmm6, [screg2+32]	;; A10 = A10 * sine (new R10)
	mulpd	xmm0, [screg2+32]	;; B10 = B10 * sine (new R14)

	xload	xmm1, [dstreg+e1]	;; R2
	subpd	xmm1, xmm6		;; new R10 = R2 - R10
	addpd	xmm6, [dstreg+e1]	;; new R2 = R2 + R10
	xstore	[dstreg+e1], xmm6	;; Save R2
	xstore	[dstreg+e1+32], xmm1	;; Save R10

	xload	xmm1, [dstreg+e1+16]	;; R6
	subpd	xmm1, xmm0		;; new R14 = R6 - R14
	addpd	xmm0, [dstreg+e1+16]	;; new R6 = R6 + R14
	xstore	[dstreg+e1+16], xmm0	;; Save R6
	xstore	[dstreg+e1+48], xmm1	;; Save R14

	xload	xmm1, [screg2+64+16]	;; cosine/sine
	xcopy	xmm4, xmm5		;; Copy R11
	mulpd	xmm5, xmm1		;; A11 = R11 * cosine/sine
	addpd	xmm5, xmm3		;; A11 = A11 + I11
	mulpd	xmm3, xmm1		;; B11 = I11 * cosine/sine
	subpd	xmm3, xmm4		;; B11 = B11 - R11
	mulpd	xmm5, [screg2+64]	;; A11 = A11 * sine (new R11)
	mulpd	xmm3, [screg2+64]	;; B11 = B11 * sine (new R15)

	xload	xmm1, [dstreg+e2]	;; R3
	subpd	xmm1, xmm5		;; new R11 = R3 - R11
	addpd	xmm5, [dstreg+e2]	;; new R3 = R3 + R11
	xstore	[dstreg+e2], xmm5	;; Save R3
	xstore	[dstreg+e2+32], xmm1	;; Save R11

	xload	xmm1, [dstreg+e2+16]	;; R7
	subpd	xmm1, xmm3		;; new R15 = R7 - R15
	addpd	xmm3, [dstreg+e2+16]	;; new R7 = R7 + R15
	xstore	[dstreg+e2+16], xmm3	;; Save R7
	xstore	[dstreg+e2+48], xmm1	;; Save R15

	xload	xmm1, [screg2+96+16]	;; cosine/sine
	mulpd	xmm1, [dstreg+e2+e1+32]	;; A12 = R12 * cosine/sine
	xload	xmm4, [dstreg+e2+e1+48]	;; I12
	addpd	xmm1, xmm4		;; A12 = A12 + I12
	mulpd	xmm4, [screg2+96+16]	;; B12 = I12 * cosine/sine
	subpd	xmm4, [dstreg+e2+e1+32]	;; B12 = B12 - R12
	mulpd	xmm1, [screg2+96]	;; A12 = A12 * sine (new R12)
	mulpd	xmm4, [screg2+96]	;; B12 = B12 * sine (new R16)

	xload	xmm0, [dstreg+e2+e1]	;; R4
	subpd	xmm0, xmm1		;; new R12 = R4 - R12
	addpd	xmm1, [dstreg+e2+e1]	;; new R4 = R4 + R12
	xstore	[dstreg+e2+e1], xmm1	;; Save R4
	xstore	[dstreg+e2+e1+32], xmm0	;; Save R12

	xload	xmm0, [dstreg+e2+e1+16]	;; R8
	subpd	xmm0, xmm4		;; new R16 = R8 - R16
	addpd	xmm4, [dstreg+e2+e1+16]	;; new R8 = R8 + R16
	xstore	[dstreg+e2+e1+16], xmm4	;; Save R8
	xstore	[dstreg+e2+e1+48], xmm0	;; Save R16
	bump	dstreg, dstinc
	ENDM


;;
;; ********************************* half-sixteen-reals-eight-complex-fft-with-square variants ***************************************
;;

;; Macro to do a sixteen_reals_fft and three eight_complex_djbfft in the final levels of pass 2.
;; The sixteen-reals macro and one of the eight-complex only use half the XMM 
;; register.  This isn't very efficient, but this macro is called only once.

r8_h8cl_sixteen_reals_eight_complex_fft_final MACRO srcreg,srcinc,d1,d2,d4
	r8_h16r_simple_fft_part1 srcreg+0,d1,d2,d4,XMM_COL_MULTS
	r8_x8c_simple_fft_part1 srcreg+32,d1,d2,d4,XMM_COL_MULTS[256]
	r8_h16r_simple_fft_part2 XMM_COL_MULTS,srcreg+0,d1,d2
	r8_x8c_simple_fft_part2 XMM_COL_MULTS[256],srcreg+d4,d1,d2
	bump	srcreg, srcinc
	ENDM

r8_h16r_simple_fft_part1 MACRO srcreg,d1,d2,d4,dst

	;
	; Do the 16-reals part
	;

	;; Odd level 1

	movsd	xmm0, Q [srcreg]		;; R1
	movsd	xmm1, Q [srcreg+16]		;; R9
	movsd	xmm7, xmm1			;; Copy R9
	addsd	xmm1, xmm0			;; R9 = R1 + R9 (new R1)
	subsd	xmm0, xmm7			;; R1 = R1 - R9 (new R9)

	movsd	xmm2, Q [srcreg+d4]		;; R5
	movsd	xmm3, Q [srcreg+d4+16]		;; R13
	movsd	xmm7, xmm3			;; Copy R13
	addsd	xmm3, xmm2			;; R13 = R5 + R13 (new R5)
	subsd	xmm2, xmm7			;; R5 = R5 - R13 (new R13)

	movsd	xmm4, Q [srcreg+d2]		;; R3
	movsd	xmm5, Q [srcreg+d2+16]		;; R11
	movsd	xmm7, xmm5			;; Copy R11
	addsd	xmm5, xmm4			;; R11 = R3 + R11 (new R3)
	subsd	xmm4, xmm7			;; R3 = R3 - R11 (new R11)

	movsd	xmm6, Q [srcreg+d4+d2]		;; R7
	movsd	xmm7, Q [srcreg+d4+d2+16]	;; R15
	addsd	xmm7, xmm6			;; R15 = R7 + R15 (new R7)
	subsd	xmm6, Q [srcreg+d4+d2+16]	;; R7 = R7 - R15 (new R15)

	;; Odd level 2

	subsd	xmm1, xmm3			;; R1 = R1 - R5 (newer R5 & final R5)
	multwos	xmm3
	addsd	xmm3, xmm1			;; R5 = R1 + R5 (newer R1)

	subsd	xmm5, xmm7			;; R3 = R3 - R7 (newer R7 & final I5)
	multwos	xmm7
	addsd	xmm7, xmm5			;; R7 = R3 + R7 (newer R3)

						;; R9/R13 morphs into newer R9/I9
						;; R11/R15 morphs into newer R11/I11

	;; Premultipliers for odd level 3

						;; mul R11/I11 by SQRTHALF + i*SQRTHALF
	subsd	xmm4, xmm6			;; R11 = R11 - I11
	multwos	xmm6
	addsd	xmm6, xmm4			;; I11 = R11 + I11
	mulsd	xmm4, Q XMM_SQRTHALF		;; R11 = R11 * SQRTHALF
	mulsd	xmm6, Q XMM_SQRTHALF		;; I11 = I11 * SQRTHALF

	;; Odd level 3

	subsd	xmm3, xmm7			;; R1 = R1 - R3 (final R3)
	multwos	xmm7
	addsd	xmm7, xmm3			;; R3 = R1 + R3 (final R1)

						;; R5/R7 morphs into final R5/I5

	subsd	xmm0, xmm4			;; R9 = R9 - R11 (final R11)
	multwos	xmm4
	addsd	xmm4, xmm0			;; R9 = R9 + R11 (final R9)

	subsd	xmm2, xmm6			;; I9 = I9 - I11 (final I11)
	multwos	xmm6
	addsd	xmm6, xmm2			;; I9 = I9 + I11 (final I9)

	movsd	Q dst[0], xmm7			;; Save R1
	movsd	Q dst[32], xmm3			;; Save R3
	movsd	Q dst[d1], xmm1			;; Save R5
	movsd	Q dst[d1+32], xmm5		;; Save I5
	movsd	Q dst[d2], xmm4			;; Save R9
	movsd	Q dst[d2+32], xmm6		;; Save I9
	movsd	Q dst[d2+d1], xmm0		;; Save R11
	movsd	Q dst[d2+d1+32], xmm2		;; Save I11


	;; Even level 1

	movsd	xmm0, Q [srcreg+d1]		;; R2
	movsd	xmm1, Q [srcreg+d1+16]		;; R10
	movsd	xmm7, xmm1			;; Copy R10
	addsd	xmm1, xmm0			;; R10 = R2 + R10 (new R2)
	subsd	xmm0, xmm7			;; R2 = R2 - R10 (new R10)

	movsd	xmm2, Q [srcreg+d4+d1]		;; R6
	movsd	xmm3, Q [srcreg+d4+d1+16]	;; R14
	movsd	xmm7, xmm3			;; Copy R14
	addsd	xmm3, xmm2			;; R14 = R6 + R14 (new R6)
	subsd	xmm2, xmm7			;; R6 = R6 - R14 (new R14)

	movsd	xmm4, Q [srcreg+d2+d1]		;; R4
	movsd	xmm5, Q [srcreg+d2+d1+16]	;; R12
	movsd	xmm7, xmm5			;; Copy R12
	addsd	xmm5, xmm4			;; R12 = R4 + R12 (new R4)
	subsd	xmm4, xmm7			;; R4 = R4 - R12 (new R12)

	movsd	xmm6, Q [srcreg+d4+d2+d1]	;; R8
	movsd	xmm7, Q [srcreg+d4+d2+d1+16]	;; R16
	addsd	xmm7, xmm6			;; R16 = R8 + R16 (new R8)
	subsd	xmm6, Q [srcreg+d4+d2+d1+16]	;; R8 = R8 - R16 (new R16)

	;; Even level 2

	subsd	xmm1, xmm3			;; R2 = R2 - R6 (newer R6)
	multwos	xmm3
	addsd	xmm3, xmm1			;; R6 = R2 + R6 (newer R2)

	subsd	xmm5, xmm7			;; R4 = R4 - R8 (newer R8)
	multwos	xmm7
	addsd	xmm7, xmm5			;; R8 = R4 + R8 (newer R4)

						;; R10/R14 morphs into newer R10/I10
						;; R12/R16 morphs into newer R12/I12

	;; Premultipliers for even level 3

	movsd	Q dst[16], xmm1
	movsd	Q dst[48], xmm3

						; mul R10/I10 by w^1 = .924 + .383i
	movsd	xmm1, xmm0			; Copy R10
	mulsd	xmm0, Q XMM_P924
	movsd	xmm3, xmm2			; Copy I10
	mulsd	xmm3, Q XMM_P383
	subsd	xmm0, xmm3			; Twiddled R10
	mulsd	xmm1, Q XMM_P383
	mulsd	xmm2, Q XMM_P924
	addsd	xmm2, xmm1			; Twiddled I10

						; mul R12/I12 by w^3 = .383 + .924i
	movsd	xmm1, xmm4			; Copy R12
	mulsd	xmm4, Q XMM_P383
	movsd	xmm3, xmm6			; Copy I12
	mulsd	xmm3, Q XMM_P924
	subsd	xmm4, xmm3			; Twiddled R12
	mulsd	xmm1, Q XMM_P924
	mulsd	xmm6, Q XMM_P383
	addsd	xmm6, xmm1			; Twiddled I12

	movsd	xmm1, Q dst[16]
	movsd	xmm3, Q dst[48]

	;; Even level 3

	subsd	xmm3, xmm7			;; R2 = R2 - R4 (final R4)
	multwos	xmm7
	addsd	xmm7, xmm3			;; R4 = R2 + R4 (final R2)

						;; R6/R8 morph into newer R6/I6

	subsd	xmm0, xmm4			;; R10 = R10 - R12 (final R12)
	multwos	xmm4
	addsd	xmm4, xmm0			;; R10 = R10 + R12 (final R10)

	subsd	xmm2, xmm6			;; I10 = I10 - I12 (final I12)
	multwos	xmm6
	addsd	xmm6, xmm2			;; I10 = I10 + I12 (final I10)

	;; Premultipliers for even level 4

						; mul R6/I6 by w^2 = .707 + .707i
	subsd	xmm1, xmm5			;; R6 = R6 - I6
	multwos	xmm5
	addsd	xmm5, xmm1			;; I6 = R6 + I6
	mulsd	xmm1, Q XMM_SQRTHALF		;; R6 = R6 * SQRTHALF (final R6)
	mulsd	xmm5, Q XMM_SQRTHALF		;; I6 = I6 * SQRTHALF (final I6)

	movsd	Q dst[16], xmm7			;; Save R2
	movsd	Q dst[48], xmm3			;; Save R4
	movsd	Q dst[d1+16], xmm1		;; Save R6
	movsd	Q dst[d1+48], xmm5		;; Save I6
	movsd	Q dst[d2+16], xmm4		;; Save R10
	movsd	Q dst[d2+48], xmm6		;; Save I10
	movsd	Q dst[d2+d1+16], xmm0		;; Save R12
	movsd	Q dst[d2+d1+48], xmm2		;; Save I12

	;
	; Do the eight-complex part
	;

	movsd	xmm0, Q [srcreg][8]		;; R1
	movsd	xmm2, Q [srcreg+d4][8]		;; R5
	movsd	xmm7, xmm2			;; Copy R5
	addsd	xmm2, xmm0			;; R5 = R1 + R5 (new R1)
	subsd	xmm0, xmm7			;; R1 = R1 - R5 (new R5)

	movsd	xmm1, Q [srcreg+d2][8]		;; R3
	movsd	xmm3, Q [srcreg+d4+d2][8]	;; R7
	movsd	xmm7, xmm3			;; Copy R7
	addsd	xmm3, xmm1			;; R7 = R3 + R7 (new R3)
	subsd	xmm1, xmm7			;; R3 = R3 - R7 (new R7)

	movsd	xmm4, Q [srcreg+16][8]		;; I1
	movsd	xmm6, Q [srcreg+d4+16][8]	;; I5
	movsd	xmm7, xmm6			;; Copy I5
	addsd	xmm6, xmm4			;; I5 = I1 + I5 (new I1)
	subsd	xmm4, xmm7			;; I1 = I1 - I5 (new I5)

	movsd	xmm7, xmm3			;; Copy R3
	addsd	xmm3, xmm2			;; R3 = R1 + R3 (final R1)
	subsd	xmm2, xmm7			;; R1 = R1 - R3 (final R3)

	movsd	Q dst[0][8], xmm3		;; Save R1
	movsd	Q dst[d1][8], xmm2		;; Save R3

	movsd	xmm5, Q [srcreg+d2+16][8]	;; I3
	movsd	xmm3, Q [srcreg+d4+d2+16][8]	;; I7
	movsd	xmm7, xmm3			;; Copy I7
	addsd	xmm3, xmm5			;; I7 = I3 + I7 (new I3)
	subsd	xmm5, xmm7			;; I3 = I3 - I7 (new I7)

	movsd	xmm7, xmm3			;; Copy I3
	addsd	xmm3, xmm6			;; I3 = I1 + I3 (final I1)
	subsd	xmm6, xmm7			;; I1 = I1 - I3 (final I3)

	movsd	Q dst[32][8], xmm3		;; Save I1
	movsd	Q dst[d1+32][8], xmm6		;; Save I3

	movsd	xmm3, xmm1			;; Copy R7
	addsd	xmm1, xmm4			;; R7 = I5 + R7 (final I5)
	movsd	xmm7, xmm0			;; Copy R5
	subsd	xmm0, xmm5			;; R5 = R5 - I7 (final R5)
	subsd	xmm4, xmm3			;; I5 = I5 - R7 (final I7)
	addsd	xmm5, xmm7			;; I7 = R5 + I7 (final R7)

	movsd	Q dst[d2+32][8], xmm1		;; Save I5
	movsd	Q dst[d2][8], xmm0		;; Save R5
	movsd	Q dst[d2+d1+32][8], xmm4	;; Save I7
	movsd	Q dst[d2+d1][8], xmm5		;; Save R7

	movsd	xmm0, Q [srcreg+d1][8]		;; R2
	movsd	xmm2, Q [srcreg+d4+d1][8]	;; R6
	movsd	xmm7, xmm2			;; Copy R6
	addsd	xmm2, xmm0			;; R6 = R2 + R6 (new R2)
	subsd	xmm0, xmm7			;; R2 = R2 - R6 (new R6)

	movsd	xmm1, Q [srcreg+d2+d1][8]	;; R4
	movsd	xmm3, Q [srcreg+d4+d2+d1][8]	;; R8
	movsd	xmm7, xmm3			;; Copy R8
	addsd	xmm3, xmm1			;; R8 = R4 + R8 (new R4)
	subsd	xmm1, xmm7			;; R4 = R4 - R8 (new R8)

	movsd	xmm4, Q [srcreg+d1+16][8]	;; I2
	movsd	xmm6, Q [srcreg+d4+d1+16][8]	;; I6
	movsd	xmm7, xmm6			;; Copy I6
	addsd	xmm6, xmm4			;; I6 = I2 + I6 (new I2)
	subsd	xmm4, xmm7			;; I2 = I2 - I6 (new I6)

	movsd	xmm7, xmm3			;; Copy R4
	addsd	xmm3, xmm2			;; R4 = R2 + R4 (final R2)
	subsd	xmm2, xmm7			;; R2 = R2 - R4 (final R4)

	movsd	Q dst[16][8], xmm3		;; Save R2
	movsd	Q dst[d1+16][8], xmm2		;; Save R4

	movsd	xmm5, Q [srcreg+d2+d1+16][8]	;; I4
	movsd	xmm3, Q [srcreg+d4+d2+d1+16][8]	;; I8
	movsd	xmm7, xmm3			;; Copy I8
	addsd	xmm3, xmm5			;; I8 = I4 + I8 (new I4)
	subsd	xmm5, xmm7			;; I4 = I4 - I8 (new I8)

	movsd	xmm7, xmm3			;; Copy I4
	addsd	xmm3, xmm6			;; I4 = I2 + I4 (final I2)
	subsd	xmm6, xmm7			;; I2 = I2 - I4 (final I4)

	movsd	Q dst[48][8], xmm3		;; Save I2
	movsd	Q dst[d1+48][8], xmm6		;; Save I4

	movsd	xmm3, xmm1			;; Copy R8
	addsd	xmm1, xmm4			;; R8 = I6 + R8 (new I6)
	movsd	xmm7, xmm0			;; Copy R6
	subsd	xmm0, xmm5			;; R6 = R6 - I8 (new R6)
	subsd	xmm4, xmm3			;; I6 = I6 - R8 (new I8)
	addsd	xmm5, xmm7			;; I8 = R6 + I8 (new R8)

	movsd	xmm7, xmm0			;; Copy R6
	subsd	xmm0, xmm1			;; R6 = R6 - I6
	addsd	xmm1, xmm7			;; I6 = R6 + I6
	movsd	xmm3, Q XMM_SQRTHALF
	mulsd	xmm0, xmm3			;; R6 = R6 * SQRTHALF (final R6)
	mulsd	xmm1, xmm3			;; I6 = I6 * SQRTHALF (final I6)

	movsd	xmm7, xmm5			;; Copy R8
	subsd	xmm5, xmm4			;; R8 = R8 - I8
	addsd	xmm4, xmm7			;; I8 = R8 + I8
	mulsd	xmm5, xmm3			;; R8 = R8 * SQRTHALF (final R8)
	mulsd	xmm4, xmm3			;; I8 = I8 * SQRTHALF (final I8)

	movsd	Q dst[d2+16][8], xmm0		;; Save R6
	movsd	Q dst[d2+48][8], xmm1		;; Save I6
	movsd	Q dst[d2+d1+16][8], xmm5	;; Save R8
	movsd	Q dst[d2+d1+48][8], xmm4	;; Save I8
	ENDM

r8_h16r_simple_fft_part2 MACRO src,dstreg,d1,d2

	; Do the 16-reals part

	movsd	xmm1, Q src[0]			;; R1
	movsd	xmm0, Q src[16] 		;; R2
	movsd	xmm7, xmm1			;; Copy R2
	subsd	xmm1, xmm0			;; R1 = R1 - R2 (new R2)
	addsd	xmm0, xmm7			;; R2 = R1 + R2 (new R1)

	movsd	xmm2, Q src[32]			;; R3/R4 morphs into R3/I3
	movsd	xmm3, Q src[48]

	movsd	Q [dstreg], xmm0		;; Save R1
	movsd	Q [dstreg+32], xmm1		;; Save R2
	movsd	Q [dstreg+16], xmm2		;; Save R3
	movsd	Q [dstreg+48], xmm3		;; Save I3

	movsd	xmm1, Q src[d1]			;; R5
	movsd	xmm0, Q src[d1+16] 		;; R6
	movsd	xmm7, xmm1			;; Copy R6
	subsd	xmm1, xmm0			;; R5 = R5 - R6 (new R6)
	addsd	xmm0, xmm7			;; R6 = R5 + R6 (new R5)

	movsd	xmm3, Q src[d1+32]		;; I5
	movsd	xmm2, Q src[d1+48] 		;; I6
	movsd	xmm7, xmm3			;; Copy I6
	subsd	xmm3, xmm2			;; I5 = I5 - I6 (new I6)
	addsd	xmm2, xmm7			;; I6 = I5 + I6 (new I5)

	movsd	Q [dstreg+d1], xmm0		;; Save R5
	movsd	Q [dstreg+d1+32], xmm2		;; Save I5
	movsd	Q [dstreg+d1+16], xmm1		;; Save R6
	movsd	Q [dstreg+d1+48], xmm3		;; Save I6

	movsd	xmm1, Q src[d2]			;; R9
	movsd	xmm0, Q src[d2+16] 		;; R10
	movsd	xmm7, xmm1			;; Copy R10
	subsd	xmm1, xmm0			;; R9 = R9 - R10 (new R10)
	addsd	xmm0, xmm7			;; R10 = R9 + R10 (new R9)

	movsd	xmm3, Q src[d2+32]		;; I9
	movsd	xmm2, Q src[d2+48] 		;; I10
	movsd	xmm7, xmm3			;; Copy I10
	subsd	xmm3, xmm2			;; I9 = I9 - I10 (new I10)
	addsd	xmm2, xmm7			;; I10 = I9 + I10 (new I9)

	movsd	Q [dstreg+d2], xmm0		;; Save R9
	movsd	Q [dstreg+d2+32], xmm2		;; Save I9
	movsd	Q [dstreg+d2+16], xmm1		;; Save R10
	movsd	Q [dstreg+d2+48], xmm3		;; Save I10

	movsd	xmm1, Q src[d2+d1]		;; R11
	movsd	xmm0, Q src[d2+d1+48] 		;; I12
	movsd	xmm7, xmm1			;; Copy I12
	subsd	xmm1, xmm0			;; R11 = R11 - I12 (new R11)
	addsd	xmm0, xmm7			;; R12 = R11 + I12 (new R12)

	movsd	xmm3, Q src[d2+d1+32]		;; I11
	movsd	xmm2, Q src[d2+d1+16] 		;; R12
	movsd	xmm7, xmm3			;; Copy R12
	subsd	xmm3, xmm2			;; I11 = I11 - R12 (new I12)
	addsd	xmm2, xmm7			;; I12 = I11 + R12 (new I11)

	movsd	Q [dstreg+d2+d1], xmm1		;; Save R11
	movsd	Q [dstreg+d2+d1+32], xmm2	;; Save I11
	movsd	Q [dstreg+d2+d1+16], xmm0	;; Save R12
	movsd	Q [dstreg+d2+d1+48], xmm3	;; Save I12

	; Do the eight-complex part

	movsd	xmm2, Q src[d2][8]		;; R5
	movsd	xmm0, Q src[d2+16][8] 		;; R6
	movsd	xmm7, xmm2			;; Copy R5
	subsd	xmm2, xmm0			;; R5 = R5 - R6 (new R6)
	addsd	xmm0, xmm7			;; R6 = R5 + R6 (new R5)

	movsd	xmm3, Q src[d2+32][8]		;; I5
	movsd	xmm1, Q src[d2+48][8]		;; I6
	movsd	xmm7, xmm3			;; Copy I5
	subsd	xmm3, xmm1			;; I5 = I5 - I6 (new I6)
	addsd	xmm1, xmm7			;; I6 = I5 + I6 (new I5)

	movsd	xmm4, Q src[d2+d1][8]		;; R7
	movsd	xmm5, Q src[d2+d1+48][8]	;; I8
	movsd	xmm7, xmm4			;; Copy R7
	subsd	xmm4, xmm5			;; R7 = R7 - I8 (new R7)
	addsd	xmm5, xmm7			;; I8 = R7 + I8 (new R8)

	movsd	Q [dstreg+d2+16][8], xmm2	;; Save R6
	movsd	Q [dstreg+d2][8], xmm0		;; Save R5
	movsd	Q [dstreg+d2+48][8], xmm3	;; Save I6
	movsd	Q [dstreg+d2+32][8], xmm1	;; Save I5
	movsd	Q [dstreg+d2+d1][8], xmm4	;; Save R7
	movsd	Q [dstreg+d2+d1+16][8], xmm5	;; Save R8

	movsd	xmm0, Q src[d2+d1+32][8]	;; I7
	movsd	xmm1, Q src[d2+d1+16][8]	;; R8
	movsd	xmm7, xmm0			;; Copy I7
	subsd	xmm0, xmm1			;; I7 = I7 - R8 (new I8)
	addsd	xmm1, xmm7			;; R8 = I7 + R8 (new I7)

	movsd	xmm3, Q src[0][8]		;; R1
	movsd	xmm4, Q src[16][8]		;; R2
	movsd	xmm7, xmm3			;; Copy R1
	subsd	xmm3, xmm4			;; R1 = R1 - R2 (new R2)
	addsd	xmm4, xmm7			;; R2 = R1 + R2 (new R1)

	movsd	xmm5, Q src[32][8]		;; I1
	movsd	xmm6, Q src[48][8]		;; I2
	movsd	xmm7, xmm5			;; Copy I1
	subsd	xmm5, xmm6			;; I1 = I1 - I2 (new I2)
	addsd	xmm6, xmm7			;; I2 = I1 + I2 (new I1)

	movsd	Q [dstreg+d2+d1+48][8], xmm0	;; Save I8
	movsd	Q [dstreg+d2+d1+32][8], xmm1	;; Save I7
	movsd	Q [dstreg+16][8], xmm3		;; Save R2
	movsd	Q [dstreg][8], xmm4		;; Save R1
	movsd	Q [dstreg+48][8], xmm5		;; Save I2
	movsd	Q [dstreg+32][8], xmm6		;; Save I1

	movsd	xmm5, Q src[d1][8]		;; R3
	movsd	xmm6, Q src[d1+48][8]		;; I4
	movsd	xmm7, xmm5			;; Copy R3
	subsd	xmm5, xmm6			;; R3 = R3 - I4 (new R3)
	addsd	xmm6, xmm7			;; I4 = R3 + I4 (new R4)

	movsd	xmm3, Q src[d1+32][8]		;; I3
	movsd	xmm4, Q src[d1+16][8]		;; R4
	movsd	xmm7, xmm3			;; Copy I3
	subsd	xmm3, xmm4			;; I3 = I3 - R4 (new I4)
	addsd	xmm4, xmm7			;; R4 = I3 + R4 (new I3)

	movsd	Q [dstreg+d1][8], xmm5		;; Save R3
	movsd	Q [dstreg+d1+16][8], xmm6	;; Save R4
	movsd	Q [dstreg+d1+48][8], xmm3	;; Save I4
	movsd	Q [dstreg+d1+32][8], xmm4	;; Save I3
	ENDM


r8_h8cl_sixteen_reals_eight_complex_with_square MACRO srcreg,srcinc,d1,d2,d4
	r8_h16r_simple_fft_part1 srcreg+0,d1,d2,d4,XMM_COL_MULTS
	r8_x8c_simple_fft_part1 srcreg+32,d1,d2,d4,XMM_COL_MULTS[256]
	r8_h16r_simple_fft_with_square XMM_COL_MULTS,d1,d2,srcreg
	r8_x8c_simple_fft_with_square XMM_COL_MULTS[256],d1,d2
	r8_h16r_simple_unfft XMM_COL_MULTS,srcreg+0,d1,d2
	r8_x8c_simple_unfft XMM_COL_MULTS[256],srcreg+d4,d1,d2
	bump	srcreg, srcinc
	ENDM

r8_h16r_simple_fft_with_square MACRO src,d1,d2,origsrc

	; Do the 16-reals part

	movsd	xmm1, Q src[0]			;; R1
	movsd	xmm0, Q src[16] 		;; R2
	movsd	xmm7, xmm1			;; Copy R2
	subsd	xmm1, xmm0			;; R1 = R1 - R2 (new R2)
	addsd	xmm0, xmm7			;; R2 = R1 + R2 (new R1)

	mulsd	xmm0, xmm0			;; Square R1
	mulsd	xmm1, xmm1			;; Square R2
	movsd	Q [origsrc-16], xmm0		;; Save square of sum of FFT values

	subsd	xmm0, xmm1			;; R1 = R1 - R2 (final R2)
	mulhalfs xmm0				;; Mul R2 by HALF
	addsd	xmm1, xmm0			;; R2 = R1 + R2 (final R1)

	movsd	xmm2, Q src[32]			;; R3/R4 morphs into R3/I3
	movsd	xmm3, Q src[48]

	xs_complex_square xmm2, xmm3, xmm7	;; Square R3/I3

						;; R3/I3 morphs into R3/R4

	movsd	Q src[0], xmm1			;; Save R1
	movsd	Q src[32], xmm2			;; Save R3
	movsd	Q src[16], xmm0			;; Save R2
	movsd	Q src[48], xmm3			;; Save R4

	movsd	xmm1, Q src[d1]			;; R5
	movsd	xmm0, Q src[d1+16] 		;; R6
	movsd	xmm7, xmm1			;; Copy R6
	subsd	xmm1, xmm0			;; R5 = R5 - R6 (new R6)
	addsd	xmm0, xmm7			;; R6 = R5 + R6 (new R5)

	movsd	xmm3, Q src[d1+32]		;; I5
	movsd	xmm2, Q src[d1+48] 		;; I6
	movsd	xmm7, xmm3			;; Copy I6
	subsd	xmm3, xmm2			;; I5 = I5 - I6 (new I6)
	addsd	xmm2, xmm7			;; I6 = I5 + I6 (new I5)

	xs_complex_square xmm0, xmm2, xmm7	;; Square R5/I5
	xs_complex_square xmm1, xmm3, xmm7	;; Square R6/I6

	subsd	xmm0, xmm1			;; R6 = R5 - R6 (new R6)
	multwos	xmm1
	addsd	xmm1, xmm0			;; R5 = R5 + R6 (new R5)

	subsd	xmm2, xmm3			;; I5 = I5 - I6 (new I6)
	multwos	xmm3
	addsd	xmm3, xmm2			;; I6 = I5 + I6 (new I5)

	movsd	Q src[d1], xmm1			;; Save R5
	movsd	Q src[d1+32], xmm3		;; Save I5
	movsd	Q src[d1+16], xmm0		;; Save R6
	movsd	Q src[d1+48], xmm2		;; Save I6

	movsd	xmm1, Q src[d2]			;; R9
	movsd	xmm0, Q src[d2+16] 		;; R10
	movsd	xmm7, xmm1			;; Copy R10
	subsd	xmm1, xmm0			;; R9 = R9 - R10 (new R10)
	addsd	xmm0, xmm7			;; R10 = R9 + R10 (new R9)

	movsd	xmm3, Q src[d2+32]		;; I9
	movsd	xmm2, Q src[d2+48] 		;; I10
	movsd	xmm7, xmm3			;; Copy I10
	subsd	xmm3, xmm2			;; I9 = I9 - I10 (new I10)
	addsd	xmm2, xmm7			;; I10 = I9 + I10 (new I9)

	xs_complex_square xmm0, xmm2, xmm7	;; Square R9/I9
	xs_complex_square xmm1, xmm3, xmm7	;; Square R10/I10

	subsd	xmm0, xmm1			;; R10 = R9 - R10 (new R10)
	multwos	xmm1
	addsd	xmm1, xmm0			;; R9 = R9 + R10 (new R9)

	subsd	xmm2, xmm3			;; I9 = I9 - I10 (new I10)
	multwos	xmm3
	addsd	xmm3, xmm2			;; I10 = I9 + I10 (new I9)

	movsd	Q src[d2], xmm1			;; Save R9
	movsd	Q src[d2+32], xmm3		;; Save I9
	movsd	Q src[d2+16], xmm0		;; Save R10
	movsd	Q src[d2+48], xmm2		;; Save I10

	movsd	xmm1, Q src[d2+d1]		;; R11
	movsd	xmm0, Q src[d2+d1+48] 		;; I12
	movsd	xmm7, xmm1			;; Copy I12
	subsd	xmm1, xmm0			;; R11 = R11 - I12 (new R11)
	addsd	xmm0, xmm7			;; R12 = R11 + I12 (new R12)

	movsd	xmm3, Q src[d2+d1+32]		;; I11
	movsd	xmm2, Q src[d2+d1+16] 		;; R12
	movsd	xmm7, xmm3			;; Copy R12
	subsd	xmm3, xmm2			;; I11 = I11 - R12 (new I12)
	addsd	xmm2, xmm7			;; I12 = I11 + R12 (new I11)

	xs_complex_square xmm1, xmm2, xmm7	;; Square R11/I11
	xs_complex_square xmm0, xmm3, xmm7	;; Square R12/I12

	subsd	xmm0, xmm1			;; R12 = R12 - R11 (new I12)
	multwos	xmm1
	addsd	xmm1, xmm0			;; R11 = R12 + R11 (new R11)

	subsd	xmm2, xmm3			;; I11 = I11 - I12 (new R12)
	multwos	xmm3
	addsd	xmm3, xmm2			;; I12 = I11 + I12 (new I11)

	movsd	Q src[d2+d1], xmm1		;; Save R11
	movsd	Q src[d2+d1+32], xmm3		;; Save I11
	movsd	Q src[d2+d1+16], xmm2		;; Save R12
	movsd	Q src[d2+d1+48], xmm0		;; Save I12

	; Do the eight-complex part

	movsd	xmm2, Q src[d2][8]		;; R5
	movsd	xmm0, Q src[d2+16][8] 		;; R6
	movsd	xmm7, xmm2			;; Copy R5
	subsd	xmm2, xmm0			;; R5 = R5 - R6 (new R6)
	addsd	xmm0, xmm7			;; R6 = R5 + R6 (new R5)

	movsd	xmm3, Q src[d2+32][8]		;; I5
	movsd	xmm1, Q src[d2+48][8] 		;; I6
	movsd	xmm7, xmm3			;; Copy I5
	subsd	xmm3, xmm1			;; I5 = I5 - I6 (new I6)
	addsd	xmm1, xmm7			;; I6 = I5 + I6 (new I5)

	xs_complex_square xmm2, xmm3, xmm7	;; Square R6/I6
	xs_complex_square xmm0, xmm1, xmm7	;; Square R5/I5

	movsd	xmm7, xmm0			;; Copy R5
	subsd	xmm0, xmm2			;; R5 = R5 - R6 (new R6)
	addsd	xmm2, xmm7			;; R6 = R5 + R6 (new R5)

	movsd	xmm7, xmm1			;; Copy I5
	subsd	xmm1, xmm3			;; I5 = I5 - I6 (new I6)
	addsd	xmm3, xmm7			;; I6 = I5 + I6 (new I5)

	movsd	Q src[d2+16][8], xmm0		;; Save R6
	movsd	Q src[d2][8], xmm2		;; Save R5
	movsd	Q src[d2+48][8], xmm1		;; Save I6
	movsd	Q src[d2+32][8], xmm3		;; Save I5

	movsd	xmm4, Q src[d2+d1][8]		;; R7
	movsd	xmm5, Q src[d2+d1+48][8]	;; I8
	movsd	xmm7, xmm4			;; Copy R7
	subsd	xmm4, xmm5			;; R7 = R7 - I8 (new R7)
	addsd	xmm5, xmm7			;; I8 = R7 + I8 (new R8)

	movsd	xmm0, Q src[d2+d1+32][8]	;; I7
	movsd	xmm1, Q src[d2+d1+16][8]	;; R8
	movsd	xmm7, xmm0			;; Copy I7
	subsd	xmm0, xmm1			;; I7 = I7 - R8 (new I8)
	addsd	xmm1, xmm7			;; R8 = I7 + R8 (new I7)

	xs_complex_square xmm5, xmm0, xmm7	;; Square R8/I8
	xs_complex_square xmm4, xmm1, xmm7	;; Square R7/I7

	movsd	xmm7, xmm5			;; Copy R8
	subsd	xmm5, xmm4			;; R8 = R8 - R7 (new I8)
	addsd	xmm4, xmm7			;; R7 = R8 + R7 (new R7)

	movsd	xmm7, xmm1			;; Copy I7
	subsd	xmm1, xmm0			;; I7 = I7 - I8 (new R8)
	addsd	xmm0, xmm7			;; I8 = I7 + I8 (new I7)

	movsd	Q src[d2+d1+48][8], xmm5	;; Save I8
	movsd	Q src[d2+d1][8], xmm4		;; Save R7
	movsd	Q src[d2+d1+16][8], xmm1	;; Save R8
	movsd	Q src[d2+d1+32][8], xmm0	;; Save I7

	movsd	xmm3, Q src[0][8]		;; R1
	movsd	xmm4, Q src[16][8]		;; R2
	movsd	xmm7, xmm3			;; Copy R1
	subsd	xmm3, xmm4			;; R1 = R1 - R2 (new R2)
	addsd	xmm4, xmm7			;; R2 = R1 + R2 (new R1)

	movsd	xmm5, Q src[32][8]		;; I1
	movsd	xmm6, Q src[48][8]		;; I2
	movsd	xmm7, xmm5			;; Copy I1
	subsd	xmm5, xmm6			;; I1 = I1 - I2 (new I2)
	addsd	xmm6, xmm7			;; I2 = I1 + I2 (new I1)

	xs_complex_square xmm3, xmm5, xmm7	;; Square R2/I2
	xs_complex_square xmm4, xmm6, xmm7	;; Square R1/I1

	movsd	xmm7, xmm4			;; Copy R1
	subsd	xmm4, xmm3			;; R1 = R1 - R2 (new R2)
	addsd	xmm3, xmm7			;; R2 = R1 + R2 (new R1)

	movsd	xmm7, xmm6			;; Copy I1
	subsd	xmm6, xmm5			;; I1 = I1 - I2 (new I2)
	addsd	xmm5, xmm7			;; I2 = I1 + I2 (new I1)

	movsd	Q src[16][8], xmm4		;; Save R2
	movsd	Q src[0][8], xmm3		;; Save R1
	movsd	Q src[48][8], xmm6		;; Save I2
	movsd	Q src[32][8], xmm5		;; Save I1

	movsd	xmm5, Q src[d1][8]		;; R3
	movsd	xmm6, Q src[d1+48][8]		;; I4
	movsd	xmm7, xmm5			;; Copy R3
	subsd	xmm5, xmm6			;; R3 = R3 - I4 (new R3)
	addsd	xmm6, xmm7			;; I4 = R3 + I4 (new R4)

	movsd	xmm3, Q src[d1+32][8]		;; I3
	movsd	xmm4, Q src[d1+16][8]		;; R4
	movsd	xmm7, xmm3			;; Copy I3
	subsd	xmm3, xmm4			;; I3 = I3 - R4 (final I4)
	addsd	xmm4, xmm7			;; R4 = I3 + R4 (final I3)

	xs_complex_square xmm6, xmm3, xmm7	;; Square R4/I4
	xs_complex_square xmm5, xmm4, xmm7	;; Square R3/I3

	movsd	xmm7, xmm6			;; Copy R4
	subsd	xmm6, xmm5			;; R4 = R4 - R3 (new I4)
	addsd	xmm5, xmm7			;; R3 = R4 + R3 (new R3)

	movsd	xmm7, xmm4			;; Copy I3
	subsd	xmm4, xmm3			;; I3 = I3 - I4 (new R4)
	addsd	xmm3, xmm7			;; I4 = I3 + I4 (new I3)

	movsd	Q src[d1+48][8], xmm6		;; Save I4
	movsd	Q src[d1][8], xmm5		;; Save R3
	movsd	Q src[d1+16][8], xmm4		;; Save R4
	movsd	Q src[d1+32][8], xmm3		;; Save I3
	ENDM

r8_h16r_simple_unfft MACRO src,dstreg,d1,d2

	; Do the 16-reals part

	;; Premultipliers for even level 4

						; mul R6/I6 by w^2 = .707 - .707i
	movsd	xmm1, Q src[d1+16] 		;; R6
	movsd	xmm5, Q src[d1+48] 		;; I6
	subsd	xmm5, xmm1			;; I6 = I6 - R6
	multwos	xmm1
	addsd	xmm1, xmm5			;; R6 = I6 + R6
	mulsd	xmm5, Q XMM_SQRTHALF		;; I6 = I6 * SQRTHALF
	mulsd	xmm1, Q XMM_SQRTHALF		;; R6 = R6 * SQRTHALF

	;; Even level 3

	movsd	xmm7, Q src[16]			;; R2
	movsd	xmm3, Q src[48] 		;; R4
	movsd	xmm4, Q src[d2+16] 		;; R10
	movsd	xmm6, Q src[d2+48] 		;; I10
	movsd	xmm0, Q src[d2+d1+16] 		;; R12
	movsd	xmm2, Q src[d2+d1+48] 		;; I12

	subsd	xmm7, xmm3			;; R2 = R2 - R4 (new R4)
	multwos	xmm3
	addsd	xmm3, xmm7			;; R4 = R2 + R4 (new R2)

						;; R6/I6 morph into new R6/R8

	subsd	xmm4, xmm0			;; R10 = R10 - R12 (new R12)
	multwos	xmm0
	addsd	xmm0, xmm4			;; R10 = R10 + R12 (new R10)

	subsd	xmm6, xmm2			;; I10 = I10 - I12 (new I12)
	multwos	xmm2
	addsd	xmm2, xmm6			;; I10 = I10 + I12 (new I10)

	;; Premultipliers for even level 3

	movsd	Q [dstreg], xmm1
	movsd	Q [dstreg+16], xmm3

						; mul R10/I10 by w^1 = .924 - .383i
	movsd	xmm1, xmm0			; Copy R10
	mulsd	xmm0, Q XMM_P924
	movsd	xmm3, xmm2			; Copy I10
	mulsd	xmm3, Q XMM_P383
	addsd	xmm0, xmm3			; Twiddled R10
	mulsd	xmm1, Q XMM_P383
	mulsd	xmm2, Q XMM_P924
	subsd	xmm2, xmm1			; Twiddled I10

						; mul R12/I12 by w^3 = .383 - .924i
	movsd	xmm1, xmm4			; Copy R12
	mulsd	xmm4, Q XMM_P383
	movsd	xmm3, xmm6			; Copy I12
	mulsd	xmm3, Q XMM_P924
	addsd	xmm4, xmm3			; Twiddled R12
	mulsd	xmm1, Q XMM_P924
	mulsd	xmm6, Q XMM_P383
	subsd	xmm6, xmm1			; Twiddled I12

	movsd	xmm1, Q [dstreg]
	movsd	xmm3, Q [dstreg+16]

	;; Even level 2

	subsd	xmm3, xmm1			;; R2 = R2 - R6 (newer R6)
	multwos	xmm1
	addsd	xmm1, xmm3			;; R6 = R2 + R6 (newer R2)

	subsd	xmm7, xmm5			;; R4 = R4 - R8 (newer R8)
	multwos	xmm5
	addsd	xmm5, xmm7			;; R8 = R4 + R8 (newer R4)

						;; R10/I10 morphs into newer R10/R14
						;; R12/I12 morphs into newer R12/R16

	;; Even level 1

	subsd	xmm1, xmm0			;; R2 = R2 - R10 (new R10)
	multwos	xmm0
	addsd	xmm0, xmm1			;; R10 = R2 + R10 (new R2)

	subsd	xmm3, xmm2			;; R6 = R6 - R14 (new R14)
	multwos	xmm2
	addsd	xmm2, xmm3			;; R14 = R6 + R14 (new R6)

	subsd	xmm5, xmm4			;; R4 = R4 - R12 (new R12)
	multwos	xmm4
	addsd	xmm4, xmm5			;; R12 = R4 + R12 (new R4)

	subsd	xmm7, xmm6			;; R8 = R8 - R16 (new R16)
	multwos	xmm6
	addsd	xmm6, xmm7			;; R16 = R8 + R16 (new R8)

	movsd	Q [dstreg+d1+32], xmm1		;; R10
	movsd	Q [dstreg+d1], xmm0		;; R2
	movsd	Q [dstreg+d1+48], xmm3		;; R14
	movsd	Q [dstreg+d1+16], xmm2		;; R6
	movsd	Q [dstreg+d2+d1+32], xmm5	;; R12
	movsd	Q [dstreg+d2+d1], xmm4		;; R4
	movsd	Q [dstreg+d2+d1+48], xmm7	;; R16
	movsd	Q [dstreg+d2+d1+16], xmm6	;; R8

	;; Odd level 3

	movsd	xmm7, Q src[0] 			;; R1
	movsd	xmm3, Q src[32] 		;; R3
	movsd	xmm1, Q src[d1] 		;; R5
	movsd	xmm5, Q src[d1+32] 		;; I5
	movsd	xmm4, Q src[d2]			;; R9
	movsd	xmm6, Q src[d2+32] 		;; I9
	movsd	xmm0, Q src[d2+d1] 		;; R11
	movsd	xmm2, Q src[d2+d1+32] 		;; I11

	subsd	xmm7, xmm3			;; R1 = R1 - R3 (new R3)
	multwos	xmm3
	addsd	xmm3, xmm7			;; R3 = R1 + R3 (new R1)

						;; R5/I5 morphs into new R5/R7

	subsd	xmm4, xmm0			;; R9 = R9 - R11 (new R11)
	multwos	xmm0
	addsd	xmm0, xmm4			;; R9 = R9 + R11 (new R9)

	subsd	xmm6, xmm2			;; I9 = I9 - I11 (new I11)
	multwos	xmm2
	addsd	xmm2, xmm6			;; I9 = I9 + I11 (new I9)

	;; Premultipliers for odd level 3

						;; mul R11/I11 by SQRTHALF - i*SQRTHALF
	subsd	xmm6, xmm4			;; I11 = I11 - R11
	multwos	xmm4
	addsd	xmm4, xmm6			;; R11 = I11 + R11
	mulsd	xmm6, Q XMM_SQRTHALF		;; I11 = I11 * SQRTHALF
	mulsd	xmm4, Q XMM_SQRTHALF		;; R11 = R11 * SQRTHALF

	;; Odd level 2

	subsd	xmm3, xmm1			;; R1 = R1 - R5 (newer R5)
	multwos	xmm1
	addsd	xmm1, xmm3			;; R5 = R1 + R5 (newer R1)

	subsd	xmm7, xmm5			;; R3 = R3 - R7 (newer R7)
	multwos	xmm5
	addsd	xmm5, xmm7			;; R7 = R3 + R7 (newer R3)

						;; R9/I9 morphs into newer R9/R13
						;; R11/I11 morphs into newer R11/R15

	;; Odd level 1

	subsd	xmm1, xmm0			;; R1 = R1 - R9 (new R9)
	multwos	xmm0
	addsd	xmm0, xmm1			;; R9 = R1 + R9 (new R1)

	subsd	xmm3, xmm2			;; R5 = R5 - R13 (new R13)
	multwos	xmm2
	addsd	xmm2, xmm3			;; R13 = R5 + R13 (new R5)

	subsd	xmm5, xmm4			;; R3 = R3 - R11 (new R11)
	multwos	xmm4
	addsd	xmm4, xmm5			;; R11 = R3 + R11 (new R3)

	subsd	xmm7, xmm6			;; R7 = R7 - R15 (new R15)
	multwos	xmm6
	addsd	xmm6, xmm7			;; R15 = R7 + R15 (new R7)

	movsd	Q [dstreg+32], xmm1		;; R9
	movsd	Q [dstreg], xmm0		;; R1
	movsd	Q [dstreg+48], xmm3		;; R13
	movsd	Q [dstreg+16], xmm2		;; R5
	movsd	Q [dstreg+d2+32], xmm5		;; R11
	movsd	Q [dstreg+d2], xmm4		;; R3
	movsd	Q [dstreg+d2+48], xmm7		;; R15
	movsd	Q [dstreg+d2+16], xmm6		;; R7

	; Do the eight-complex part

	movsd	xmm5, Q src[d2+32][8]		;; I5
	movsd	xmm1, Q src[d2+d1+32][8]	;; I7
	movsd	xmm7, xmm5			;; Copy I5
	subsd	xmm5, xmm1			;; I5 = I5 - I7 (new R7)
	addsd	xmm1, xmm7			;; I7 = I5 + I7 (new I5)

	movsd	xmm4, Q src[0][8]		;; R1
	movsd	xmm2, Q src[d1][8]		;; R3
	movsd	xmm7, xmm4			;; Copy R1
	subsd	xmm4, xmm2			;; R1 = R1 - R3 (new R3)
	addsd	xmm2, xmm7			;; R3 = R1 + R3 (new R1)

	movsd	xmm7, xmm4			;; Copy R3
	subsd	xmm4, xmm5			;; R3 = R3 - R7 (final R7)
	addsd	xmm5, xmm7			;; R7 = R3 + R7 (final R3)

	movsd	Q [dstreg+d2+16][8], xmm4	;; Save R7
	movsd	Q [dstreg+d2][8], xmm5		;; Save R3

	movsd	xmm6, Q src[d2][8]		;; R5
	movsd	xmm0, Q src[d2+d1][8]		;; R7
	movsd	xmm7, xmm6			;; Copy R5
	addsd	xmm6, xmm0			;; R5 = R7 + R5 (new R5)
	subsd	xmm0, xmm7			;; R7 = R7 - R5 (new I7)

	movsd	xmm3, Q src[32][8]		;; I1
	movsd	xmm4, Q src[d1+32][8]		;; I3
	movsd	xmm7, xmm3			;; Copy I1
	subsd	xmm3, xmm4			;; I1 = I1 - I3 (new I3)
	addsd	xmm4, xmm7			;; I3 = I1 + I3 (new I1)

	movsd	xmm7, xmm3			;; Copy I3
	subsd	xmm3, xmm0			;; I3 = I3 - I7 (final I7)
	addsd	xmm0, xmm7			;; I7 = I3 + I7 (final I3)

	movsd	Q [dstreg+d2+48][8], xmm3	;; I7
	movsd	Q [dstreg+d2+32][8], xmm0	;; I3

	movsd	xmm7, xmm2			;; Copy R1
	subsd	xmm2, xmm6			;; R1 = R1 - R5 (final R5)
	addsd	xmm6, xmm7			;; R5 = R1 + R5 (final R1)

	movsd	xmm7, xmm4			;; Copy I1
	subsd	xmm4, xmm1			;; I1 = I1 - I5 (final I5)
	addsd	xmm1, xmm7			;; I5 = I1 + I5 (final I1)

	movsd	Q [dstreg+16][8], xmm2		;; R5
	movsd	Q [dstreg][8], xmm6		;; R1
	movsd	Q [dstreg+48][8], xmm4		;; I5
	movsd	Q [dstreg+32][8], xmm1		;; I1


	;; multiply R6/I6 by SQRTHALF - i*SQRTHALF
	movsd	xmm4, Q src[d2+16][8]		;; R6
	movsd	xmm0, Q src[d2+48][8]		;; I6
	movsd	xmm7, xmm0			;; Copy I6
	subsd	xmm0, xmm4			;; I6 = I6 - R6
	addsd	xmm4, xmm7			;; R6 = R6 + I6

	;; multiply R8/I8 by SQRTHALF - i*SQRTHALF
	movsd	xmm2, Q src[d2+d1+16][8]	;; R8
	movsd	xmm1, Q src[d2+d1+48][8]	;; I8
	movsd	xmm7, xmm1			;; Copy I8
	subsd	xmm1, xmm2			;; I8 = I8 - R8
	addsd	xmm2, xmm7			;; R8 = R8 + I8

	movsd	xmm5, Q XMM_SQRTHALF
	mulsd	xmm0, xmm5			;; I6 = I6 * SQRTHALF (new I6)
	mulsd	xmm4, xmm5			;; R6 = R6 * SQRTHALF (new R6)
	mulsd	xmm1, xmm5			;; I8 = I8 * SQRTHALF (new I8)
	mulsd	xmm2, xmm5			;; R8 = R8 * SQRTHALF (new R8)

	movsd	xmm6, Q src[16][8]		;; R2
	movsd	xmm3, Q src[d1+16][8]		;; R4
	movsd	xmm7, xmm3			;; Copy R4
	addsd	xmm3, xmm6			;; R4 = R2 + R4 (new R2)
	subsd	xmm6, xmm7			;; R2 = R2 - R4 (new R4)

	movsd	xmm7, xmm2			;; Copy R8
	subsd	xmm2, xmm4			;; R8 = R8 - R6 (new I8)
	addsd	xmm4, xmm7			;; R6 = R8 + R6 (new R6)

	movsd	xmm7, xmm0			;; Copy I6
	subsd	xmm0, xmm1			;; I6 = I6 - I8 (new R8)
	addsd	xmm1, xmm7			;; I8 = I6 + I8 (new I6)

	movsd	xmm7, xmm3			;; Copy R2
	subsd	xmm3, xmm4			;; R2 = R2 - R6 (final R6)
	addsd	xmm4, xmm7			;; R6 = R2 + R6 (final R2)

	movsd	Q [dstreg+d1+16][8], xmm3	;; R6
	movsd	Q [dstreg+d1][8], xmm4		;; R2

	movsd	xmm5, Q src[48][8]		;; I2
	movsd	xmm3, Q src[d1+48][8]		;; I4
	movsd	xmm7, xmm3			;; Copy I4
	addsd	xmm3, xmm5			;; I4 = I2 + I4 (new I2)
	subsd	xmm5, xmm7			;; I2 = I2 - I4 (new I4)

	movsd	xmm7, xmm6			;; Copy R4
	subsd	xmm6, xmm0			;; R4 = R4 - R8 (final R8)
	addsd	xmm0, xmm7			;; R8 = R4 + R8 (final R4)

	movsd	xmm7, xmm3			;; Copy I2
	subsd	xmm3, xmm1			;; I2 = I2 - I6 (final I6)
	addsd	xmm1, xmm7			;; I6 = I2 + I6 (final I2)

	movsd	xmm7, xmm5			;; Copy I4
	subsd	xmm5, xmm2			;; I4 = I4 - I8 (final I8)
	addsd	xmm2, xmm7			;; I8 = I4 + I8 (final I4)

	movsd	Q [dstreg+d2+d1+16][8], xmm6	;; R8
	movsd	Q [dstreg+d2+d1][8], xmm0	;; R4
	movsd	Q [dstreg+d1+48][8], xmm3	;; I6
	movsd	Q [dstreg+d1+32][8], xmm1	;; I2
	movsd	Q [dstreg+d2+d1+48][8], xmm5	;; I8
	movsd	Q [dstreg+d2+d1+32][8], xmm2	;; I4
	ENDM


r8_h8cl_sixteen_reals_eight_complex_with_mult MACRO srcreg,srcinc,d1,d2,d4
	r8_h16r_simple_fft_part1 srcreg+0,d1,d2,d4,XMM_COL_MULTS
	r8_x8c_simple_fft_part1 srcreg+32,d1,d2,d4,XMM_COL_MULTS[256]
	r8_h16r_simple_fft_with_mult XMM_COL_MULTS,srcreg+rbp,d1,d2,srcreg
	r8_x8c_simple_fft_with_mult XMM_COL_MULTS[256],srcreg+d4+rbp,d1,d2
	r8_h16r_simple_unfft XMM_COL_MULTS,srcreg+0,d1,d2
	r8_x8c_simple_unfft XMM_COL_MULTS[256],srcreg+d4,d1,d2
	bump	srcreg, srcinc
	ENDM

r8_h16r_simple_fft_with_mult MACRO src,altsrc,d1,d2,origsrc

	; Do the 16-reals part

	movsd	xmm1, Q src[0]			;; R1
	movsd	xmm0, Q src[16] 		;; R2
	movsd	xmm7, xmm1			;; Copy R2
	subsd	xmm1, xmm0			;; R1 = R1 - R2 (new R2)
	addsd	xmm0, xmm7			;; R2 = R1 + R2 (new R1)

	mulsd	xmm0, Q [altsrc]		;; R1 * R1-from-mem
	mulsd	xmm1, Q [altsrc+32]		;; R2 * R2-from-mem
	movsd	Q [origsrc-16], xmm0		;; Save product of sum of FFT values

	subsd	xmm0, xmm1			;; R1 = R1 - R2 (final R2)
	mulhalfs xmm0				;; Mul R2 by HALF
	addsd	xmm1, xmm0			;; R2 = R1 + R2 (final R1)

	movsd	xmm2, Q src[32]			;; R3/R4 morphs into R3/I3
	movsd	xmm3, Q src[48]

	xs_complex_mult xmm2, xmm3, Q [altsrc+16], Q [altsrc+48], xmm6, xmm7 ;; Mult R3/I3

						;; R3/I3 morphs into R3/R4

	movsd	Q src[0], xmm1			;; Save R1
	movsd	Q src[32], xmm2			;; Save R3
	movsd	Q src[16], xmm0			;; Save R2
	movsd	Q src[48], xmm3			;; Save R4

	movsd	xmm1, Q src[d1]			;; R5
	movsd	xmm0, Q src[d1+16] 		;; R6
	movsd	xmm7, xmm1			;; Copy R6
	subsd	xmm1, xmm0			;; R5 = R5 - R6 (new R6)
	addsd	xmm0, xmm7			;; R6 = R5 + R6 (new R5)

	movsd	xmm3, Q src[d1+32]		;; I5
	movsd	xmm2, Q src[d1+48] 		;; I6
	movsd	xmm7, xmm3			;; Copy I6
	subsd	xmm3, xmm2			;; I5 = I5 - I6 (new I6)
	addsd	xmm2, xmm7			;; I6 = I5 + I6 (new I5)

	xs_complex_mult xmm0, xmm2, Q [altsrc+d1], Q [altsrc+d1+32], xmm6, xmm7 ;; Mult R5/I5
	xs_complex_mult xmm1, xmm3, Q [altsrc+d1+16], Q [altsrc+d1+48], xmm6, xmm7 ;; Mult R6/I6

	subsd	xmm0, xmm1			;; R6 = R5 - R6 (new R6)
	multwos	xmm1
	addsd	xmm1, xmm0			;; R5 = R5 + R6 (new R5)

	subsd	xmm2, xmm3			;; I5 = I5 - I6 (new I6)
	multwos	xmm3
	addsd	xmm3, xmm2			;; I6 = I5 + I6 (new I5)

	movsd	Q src[d1], xmm1			;; Save R5
	movsd	Q src[d1+32], xmm3		;; Save I5
	movsd	Q src[d1+16], xmm0		;; Save R6
	movsd	Q src[d1+48], xmm2		;; Save I6

	movsd	xmm1, Q src[d2]			;; R9
	movsd	xmm0, Q src[d2+16] 		;; R10
	movsd	xmm7, xmm1			;; Copy R10
	subsd	xmm1, xmm0			;; R9 = R9 - R10 (new R10)
	addsd	xmm0, xmm7			;; R10 = R9 + R10 (new R9)

	movsd	xmm3, Q src[d2+32]		;; I9
	movsd	xmm2, Q src[d2+48] 		;; I10
	movsd	xmm7, xmm3			;; Copy I10
	subsd	xmm3, xmm2			;; I9 = I9 - I10 (new I10)
	addsd	xmm2, xmm7			;; I10 = I9 + I10 (new I9)

	xs_complex_mult xmm0, xmm2, Q [altsrc+d2], Q [altsrc+d2+32], xmm6, xmm7 ;; Mult R9/I9
	xs_complex_mult xmm1, xmm3, Q [altsrc+d2+16], Q [altsrc+d2+48], xmm6, xmm7 ;; Mult R10/I10

	subsd	xmm0, xmm1			;; R10 = R9 - R10 (new R10)
	multwos	xmm1
	addsd	xmm1, xmm0			;; R9 = R9 + R10 (new R9)

	subsd	xmm2, xmm3			;; I9 = I9 - I10 (new I10)
	multwos	xmm3
	addsd	xmm3, xmm2			;; I10 = I9 + I10 (new I9)

	movsd	Q src[d2], xmm1			;; Save R9
	movsd	Q src[d2+32], xmm3		;; Save I9
	movsd	Q src[d2+16], xmm0		;; Save R10
	movsd	Q src[d2+48], xmm2		;; Save I10

	movsd	xmm1, Q src[d2+d1]		;; R11
	movsd	xmm0, Q src[d2+d1+48] 		;; I12
	movsd	xmm7, xmm1			;; Copy I12
	subsd	xmm1, xmm0			;; R11 = R11 - I12 (new R11)
	addsd	xmm0, xmm7			;; R12 = R11 + I12 (new R12)

	movsd	xmm3, Q src[d2+d1+32]		;; I11
	movsd	xmm2, Q src[d2+d1+16] 		;; R12
	movsd	xmm7, xmm3			;; Copy R12
	subsd	xmm3, xmm2			;; I11 = I11 - R12 (new I12)
	addsd	xmm2, xmm7			;; I12 = I11 + R12 (new I11)

	xs_complex_mult xmm1, xmm2, Q [altsrc+d2+d1], Q [altsrc+d2+d1+32], xmm6, xmm7 ;; Mult R11/I11
	xs_complex_mult xmm0, xmm3, Q [altsrc+d2+d1+16], Q [altsrc+d2+d1+48], xmm6, xmm7 ;; Mult R12/I12

	subsd	xmm0, xmm1			;; R12 = R12 - R11 (new I12)
	multwos	xmm1
	addsd	xmm1, xmm0			;; R11 = R12 + R11 (new R11)

	subsd	xmm2, xmm3			;; I11 = I11 - I12 (new R12)
	multwos	xmm3
	addsd	xmm3, xmm2			;; I12 = I11 + I12 (new I11)

	movsd	Q src[d2+d1], xmm1		;; Save R11
	movsd	Q src[d2+d1+32], xmm3		;; Save I11
	movsd	Q src[d2+d1+16], xmm2		;; Save R12
	movsd	Q src[d2+d1+48], xmm0		;; Save I12

	; Do the eight-complex part

	movsd	xmm2, Q src[d2][8]		;; R5
	movsd	xmm0, Q src[d2+16][8] 		;; R6
	movsd	xmm7, xmm2			;; Copy R5
	subsd	xmm2, xmm0			;; R5 = R5 - R6 (new R6)
	addsd	xmm0, xmm7			;; R6 = R5 + R6 (new R5)

	movsd	xmm3, Q src[d2+32][8]		;; I5
	movsd	xmm1, Q src[d2+48][8] 		;; I6
	movsd	xmm7, xmm3			;; Copy I5
	subsd	xmm3, xmm1			;; I5 = I5 - I6 (new I6)
	addsd	xmm1, xmm7			;; I6 = I5 + I6 (new I5)

	xs_complex_mult xmm2, xmm3, Q [altsrc+d2+16][8], Q [altsrc+d2+48][8], xmm6, xmm7 ;; Mult R6/I6
	xs_complex_mult xmm0, xmm1, Q [altsrc+d2][8], Q [altsrc+d2+32][8], xmm6, xmm7 ;; Mult R5/I5

	movsd	xmm7, xmm0			;; Copy R5
	subsd	xmm0, xmm2			;; R5 = R5 - R6 (new R6)
	addsd	xmm2, xmm7			;; R6 = R5 + R6 (new R5)

	movsd	xmm7, xmm1			;; Copy I5
	subsd	xmm1, xmm3			;; I5 = I5 - I6 (new I6)
	addsd	xmm3, xmm7			;; I6 = I5 + I6 (new I5)

	movsd	Q src[d2+16][8], xmm0		;; Save R6
	movsd	Q src[d2][8], xmm2		;; Save R5
	movsd	Q src[d2+48][8], xmm1		;; Save I6
	movsd	Q src[d2+32][8], xmm3		;; Save I5

	movsd	xmm4, Q src[d2+d1][8]		;; R7
	movsd	xmm5, Q src[d2+d1+48][8]	;; I8
	movsd	xmm7, xmm4			;; Copy R7
	subsd	xmm4, xmm5			;; R7 = R7 - I8 (new R7)
	addsd	xmm5, xmm7			;; I8 = R7 + I8 (new R8)

	movsd	xmm0, Q src[d2+d1+32][8]	;; I7
	movsd	xmm1, Q src[d2+d1+16][8]	;; R8
	movsd	xmm7, xmm0			;; Copy I7
	subsd	xmm0, xmm1			;; I7 = I7 - R8 (new I8)
	addsd	xmm1, xmm7			;; R8 = I7 + R8 (new I7)

	xs_complex_mult xmm5, xmm0, Q [altsrc+d2+d1+16][8], Q [altsrc+d2+d1+48][8], xmm6, xmm7 ;; Mult R8/I8
	xs_complex_mult xmm4, xmm1, Q [altsrc+d2+d1][8], Q [altsrc+d2+d1+32][8], xmm6, xmm7 ;; Mult R7/I7

	movsd	xmm7, xmm5			;; Copy R8
	subsd	xmm5, xmm4			;; R8 = R8 - R7 (new I8)
	addsd	xmm4, xmm7			;; R7 = R8 + R7 (new R7)

	movsd	xmm7, xmm1			;; Copy I7
	subsd	xmm1, xmm0			;; I7 = I7 - I8 (new R8)
	addsd	xmm0, xmm7			;; I8 = I7 + I8 (new I7)

	movsd	Q src[d2+d1+48][8], xmm5	;; Save I8
	movsd	Q src[d2+d1][8], xmm4		;; Save R7
	movsd	Q src[d2+d1+16][8], xmm1	;; Save R8
	movsd	Q src[d2+d1+32][8], xmm0	;; Save I7

	movsd	xmm3, Q src[0][8]		;; R1
	movsd	xmm4, Q src[16][8]		;; R2
	movsd	xmm7, xmm3			;; Copy R1
	subsd	xmm3, xmm4			;; R1 = R1 - R2 (new R2)
	addsd	xmm4, xmm7			;; R2 = R1 + R2 (new R1)

	movsd	xmm5, Q src[32][8]		;; I1
	movsd	xmm6, Q src[48][8]		;; I2
	movsd	xmm7, xmm5			;; Copy I1
	subsd	xmm5, xmm6			;; I1 = I1 - I2 (new I2)
	addsd	xmm6, xmm7			;; I2 = I1 + I2 (new I1)

	xs_complex_mult xmm3, xmm5, Q [altsrc+16][8], Q [altsrc+48][8], xmm0, xmm7 ;; Mult R2/I2
	xs_complex_mult xmm4, xmm6, Q [altsrc][8], Q [altsrc+32][8], xmm0, xmm7 ;; Mult R1/I1

	movsd	xmm7, xmm4			;; Copy R1
	subsd	xmm4, xmm3			;; R1 = R1 - R2 (new R2)
	addsd	xmm3, xmm7			;; R2 = R1 + R2 (new R1)

	movsd	xmm7, xmm6			;; Copy I1
	subsd	xmm6, xmm5			;; I1 = I1 - I2 (new I2)
	addsd	xmm5, xmm7			;; I2 = I1 + I2 (new I1)

	movsd	Q src[16][8], xmm4		;; Save R2
	movsd	Q src[0][8], xmm3		;; Save R1
	movsd	Q src[48][8], xmm6		;; Save I2
	movsd	Q src[32][8], xmm5		;; Save I1

	movsd	xmm5, Q src[d1][8]		;; R3
	movsd	xmm6, Q src[d1+48][8]		;; I4
	movsd	xmm7, xmm5			;; Copy R3
	subsd	xmm5, xmm6			;; R3 = R3 - I4 (new R3)
	addsd	xmm6, xmm7			;; I4 = R3 + I4 (new R4)

	movsd	xmm3, Q src[d1+32][8]		;; I3
	movsd	xmm4, Q src[d1+16][8]		;; R4
	movsd	xmm7, xmm3			;; Copy I3
	subsd	xmm3, xmm4			;; I3 = I3 - R4 (final I4)
	addsd	xmm4, xmm7			;; R4 = I3 + R4 (final I3)

	xs_complex_mult xmm6, xmm3, Q [altsrc+d1+16][8], Q [altsrc+d1+48][8], xmm0, xmm7 ;; Mult R4/I4
	xs_complex_mult xmm5, xmm4, Q [altsrc+d1][8], Q [altsrc+d1+32][8], xmm0, xmm7 ;; Mult R3/I3

	movsd	xmm7, xmm6			;; Copy R4
	subsd	xmm6, xmm5			;; R4 = R4 - R3 (new I4)
	addsd	xmm5, xmm7			;; R3 = R4 + R3 (new R3)

	movsd	xmm7, xmm4			;; Copy I3
	subsd	xmm4, xmm3			;; I3 = I3 - I4 (new R4)
	addsd	xmm3, xmm7			;; I4 = I3 + I4 (new I3)

	movsd	Q src[d1+48][8], xmm6		;; Save I4
	movsd	Q src[d1][8], xmm5		;; Save R3
	movsd	Q src[d1+16][8], xmm4		;; Save R4
	movsd	Q src[d1+32][8], xmm3		;; Save I3
	ENDM


r8_h8cl_sixteen_reals_eight_complex_with_mulf MACRO srcreg,srcinc,d1,d2,d4
	r8_h16r_simple_fft_with_mulf srcreg,d1,d2,XMM_COL_MULTS
	r8_x8c_simple_fft_with_mulf srcreg+d4,d1,d2,XMM_COL_MULTS[256]
	r8_h16r_simple_unfft XMM_COL_MULTS,srcreg+0,d1,d2
	r8_x8c_simple_unfft XMM_COL_MULTS[256],srcreg+d4,d1,d2
	bump	srcreg, srcinc
	ENDM

r8_h16r_simple_fft_with_mulf MACRO srcreg,d1,d2,dst

	; Do the 16-reals part

	movsd	xmm0, Q [srcreg][rbx]		;; R1
	movsd	xmm1, Q [srcreg+32][rbx]	;; R2

	mulsd	xmm0, Q [srcreg][rbp]		;; R1 * R1-from-mem
	mulsd	xmm1, Q [srcreg+32][rbp]	;; R2 * R2-from-mem
	movsd	Q [srcreg-16], xmm0		;; Save product of sum of FFT values

	subsd	xmm0, xmm1			;; R1 = R1 - R2 (final R2)
	mulhalfs xmm0				;; Mul R2 by HALF
	addsd	xmm1, xmm0			;; R2 = R1 + R2 (final R1)

	movsd	xmm2, Q [srcreg+16][rbx]	;; R3/R4 morphs into R3/I3
	movsd	xmm3, Q [srcreg+48][rbx]

	xs_complex_mult xmm2, xmm3, Q [srcreg+16][rbp], Q [srcreg+48][rbp], xmm6, xmm7 ;; Mult R3/I3

						;; R3/I3 morphs into R3/R4

	movsd	Q dst[0], xmm1			;; Save R1
	movsd	Q dst[32], xmm2			;; Save R3
	movsd	Q dst[16], xmm0			;; Save R2
	movsd	Q dst[48], xmm3			;; Save R4

	movsd	xmm0, Q [srcreg+d1][rbx]	;; R5
	movsd	xmm2, Q [srcreg+d1+32][rbx]	;; I5
	movsd	xmm1, Q [srcreg+d1+16][rbx]	;; R6
	movsd	xmm3, Q [srcreg+d1+48][rbx]	;; I6

	xs_complex_mult xmm0, xmm2, Q [srcreg+d1][rbp], Q [srcreg+d1+32][rbp], xmm6, xmm7 ;; Mult R5/I5
	xs_complex_mult xmm1, xmm3, Q [srcreg+d1+16][rbp], Q [srcreg+d1+48][rbp], xmm6, xmm7 ;; Mult R6/I6

	subsd	xmm0, xmm1			;; R6 = R5 - R6 (new R6)
	multwos	xmm1
	addsd	xmm1, xmm0			;; R5 = R5 + R6 (new R5)

	subsd	xmm2, xmm3			;; I5 = I5 - I6 (new I6)
	multwos	xmm3
	addsd	xmm3, xmm2			;; I6 = I5 + I6 (new I5)

	movsd	Q dst[d1], xmm1			;; Save R5
	movsd	Q dst[d1+32], xmm3		;; Save I5
	movsd	Q dst[d1+16], xmm0		;; Save R6
	movsd	Q dst[d1+48], xmm2		;; Save I6

	movsd	xmm0, Q [srcreg+d2][rbx]	;; R9
	movsd	xmm2, Q [srcreg+d2+32][rbx]	;; I9
	movsd	xmm1, Q [srcreg+d2+16][rbx]	;; R10
	movsd	xmm3, Q [srcreg+d2+48][rbx]	;; I10

	xs_complex_mult xmm0, xmm2, Q [srcreg+d2][rbp], Q [srcreg+d2+32][rbp], xmm6, xmm7 ;; Mult R9/I9
	xs_complex_mult xmm1, xmm3, Q [srcreg+d2+16][rbp], Q [srcreg+d2+48][rbp], xmm6, xmm7 ;; Mult R10/I10

	subsd	xmm0, xmm1			;; R10 = R9 - R10 (new R10)
	multwos	xmm1
	addsd	xmm1, xmm0			;; R9 = R9 + R10 (new R9)

	subsd	xmm2, xmm3			;; I9 = I9 - I10 (new I10)
	multwos	xmm3
	addsd	xmm3, xmm2			;; I10 = I9 + I10 (new I9)

	movsd	Q dst[d2], xmm1			;; Save R9
	movsd	Q dst[d2+32], xmm3		;; Save I9
	movsd	Q dst[d2+16], xmm0		;; Save R10
	movsd	Q dst[d2+48], xmm2		;; Save I10

	movsd	xmm1, Q [srcreg+d2+d1][rbx]	;; R11
	movsd	xmm2, Q [srcreg+d2+d1+32][rbx]	;; I11
	movsd	xmm0, Q [srcreg+d2+d1+16][rbx] 	;; R12
	movsd	xmm3, Q [srcreg+d2+d1+48][rbx] 	;; I12

	xs_complex_mult xmm1, xmm2, Q [srcreg+d2+d1][rbp], Q [srcreg+d2+d1+32][rbp], xmm6, xmm7 ;; Mult R11/I11
	xs_complex_mult xmm0, xmm3, Q [srcreg+d2+d1+16][rbp], Q [srcreg+d2+d1+48][rbp], xmm6, xmm7 ;; Mult R12/I12

	subsd	xmm0, xmm1			;; R12 = R12 - R11 (new I12)
	multwos	xmm1
	addsd	xmm1, xmm0			;; R11 = R12 + R11 (new R11)

	subsd	xmm2, xmm3			;; I11 = I11 - I12 (new R12)
	multwos	xmm3
	addsd	xmm3, xmm2			;; I12 = I11 + I12 (new I11)

	movsd	Q dst[d2+d1], xmm1		;; Save R11
	movsd	Q dst[d2+d1+32], xmm3		;; Save I11
	movsd	Q dst[d2+d1+16], xmm2		;; Save R12
	movsd	Q dst[d2+d1+48], xmm0		;; Save I12

	; Do the eight-complex part

	movsd	xmm2, Q [srcreg+d2+16][rbx][8]	;; R6
	movsd	xmm3, Q [srcreg+d2+48][rbx][8]	;; I6
	movsd	xmm0, Q [srcreg+d2][rbx][8]	;; R5
	movsd	xmm1, Q [srcreg+d2+32][rbx][8]	;; I5

	xs_complex_mult xmm2, xmm3, Q [srcreg+d2+16][rbp][8], Q [srcreg+d2+48][rbp][8], xmm6, xmm7 ;; Mult R6/I6
	xs_complex_mult xmm0, xmm1, Q [srcreg+d2][rbp][8], Q [srcreg+d2+32][rbp][8], xmm6, xmm7 ;; Mult R5/I5

	movsd	xmm7, xmm0			;; Copy R5
	subsd	xmm0, xmm2			;; R5 = R5 - R6 (new R6)
	addsd	xmm2, xmm7			;; R6 = R5 + R6 (new R5)

	movsd	xmm7, xmm1			;; Copy I5
	subsd	xmm1, xmm3			;; I5 = I5 - I6 (new I6)
	addsd	xmm3, xmm7			;; I6 = I5 + I6 (new I5)

	movsd	Q dst[d2+16][8], xmm0		;; Save R6
	movsd	Q dst[d2][8], xmm2		;; Save R5
	movsd	Q dst[d2+48][8], xmm1		;; Save I6
	movsd	Q dst[d2+32][8], xmm3		;; Save I5

	movsd	xmm5, Q [srcreg+d2+d1+16][rbx][8] ;; R8
	movsd	xmm0, Q [srcreg+d2+d1+48][rbx][8] ;; I8
	movsd	xmm4, Q [srcreg+d2+d1][rbx][8]	;; R7
	movsd	xmm1, Q [srcreg+d2+d1+32][rbx][8] ;; I7

	xs_complex_mult xmm5, xmm0, Q [srcreg+d2+d1+16][rbp][8], Q [srcreg+d2+d1+48][rbp][8], xmm6, xmm7 ;; Mult R8/I8
	xs_complex_mult xmm4, xmm1, Q [srcreg+d2+d1][rbp][8], Q [srcreg+d2+d1+32][rbp][8], xmm6, xmm7 ;; Mult R7/I7

	movsd	xmm7, xmm5			;; Copy R8
	subsd	xmm5, xmm4			;; R8 = R8 - R7 (new I8)
	addsd	xmm4, xmm7			;; R7 = R8 + R7 (new R7)

	movsd	xmm7, xmm1			;; Copy I7
	subsd	xmm1, xmm0			;; I7 = I7 - I8 (new R8)
	addsd	xmm0, xmm7			;; I8 = I7 + I8 (new I7)

	movsd	Q dst[d2+d1+48][8], xmm5	;; Save I8
	movsd	Q dst[d2+d1][8], xmm4		;; Save R7
	movsd	Q dst[d2+d1+16][8], xmm1	;; Save R8
	movsd	Q dst[d2+d1+32][8], xmm0	;; Save I7

	movsd	xmm3, Q [srcreg+16][rbx][8]	;; R2
	movsd	xmm5, Q [srcreg+48][rbx][8]	;; I2
	movsd	xmm4, Q [srcreg][rbx][8]	;; R1
	movsd	xmm6, Q [srcreg+32][rbx][8]	;; I1

	xs_complex_mult xmm3, xmm5, Q [srcreg+16][rbp][8], Q [srcreg+48][rbp][8], xmm0, xmm7 ;; Mult R2/I2
	xs_complex_mult xmm4, xmm6, Q [srcreg][rbp][8], Q [srcreg+32][rbp][8], xmm0, xmm7 ;; Mult R1/I1

	movsd	xmm7, xmm4			;; Copy R1
	subsd	xmm4, xmm3			;; R1 = R1 - R2 (new R2)
	addsd	xmm3, xmm7			;; R2 = R1 + R2 (new R1)

	movsd	xmm7, xmm6			;; Copy I1
	subsd	xmm6, xmm5			;; I1 = I1 - I2 (new I2)
	addsd	xmm5, xmm7			;; I2 = I1 + I2 (new I1)

	movsd	Q dst[16][8], xmm4		;; Save R2
	movsd	Q dst[0][8], xmm3		;; Save R1
	movsd	Q dst[48][8], xmm6		;; Save I2
	movsd	Q dst[32][8], xmm5		;; Save I1

	movsd	xmm6, Q [srcreg+d1+16][rbx][8]	;; R4
	movsd	xmm3, Q [srcreg+d1+48][rbx][8]	;; I4
	movsd	xmm5, Q [srcreg+d1][rbx][8]	;; R3
	movsd	xmm4, Q [srcreg+d1+32][rbx][8]	;; I3

	xs_complex_mult xmm6, xmm3, Q [srcreg+d1+16][rbp][8], Q [srcreg+d1+48][rbp][8], xmm0, xmm7 ;; Mult R4/I4
	xs_complex_mult xmm5, xmm4, Q [srcreg+d1][rbp][8], Q [srcreg+d1+32][rbp][8], xmm0, xmm7 ;; Mult R3/I3

	movsd	xmm7, xmm6			;; Copy R4
	subsd	xmm6, xmm5			;; R4 = R4 - R3 (new I4)
	addsd	xmm5, xmm7			;; R3 = R4 + R3 (new R3)

	movsd	xmm7, xmm4			;; Copy I3
	subsd	xmm4, xmm3			;; I3 = I3 - I4 (new R4)
	addsd	xmm3, xmm7			;; I4 = I3 + I4 (new I3)

	movsd	Q dst[d1+48][8], xmm6		;; Save I4
	movsd	Q dst[d1][8], xmm5		;; Save R3
	movsd	Q dst[d1+16][8], xmm4		;; Save R4
	movsd	Q dst[d1+32][8], xmm3		;; Save I3
	ENDM

