; Copyright 2009-2023 - Mersenne Research, Inc.  All rights reserved
; Author:  George Woltman
; Email: woltman@alum.mit.edu
;
; These macros do the second pass of standard radix-4 2 pass FFTs using SSE2 instructions.
;

;;
;; Estimated memory requirements for complex pass 2 macros:
;;
;; Pass 2 size:		256	768	1024	1280	1536	2048	2560	2304	3072	3840	4096	5120	6400	4608	6144	7680	8192	10240	12800	9216	12288	15360	16384	20480	25600
;;
;; FFT data:		8K	24K	32K	40K	48K	64K	80K	72K	96K	120K	128K	160K	200K	144K	192K	240K	256K	320K	400K	288K	384K	480K	512K	640K	800K
;; Fixed sin/cos data:	4K	12K	20K	20K	20K	36K	36K	28K	52K	52K	68K	84K	84K	52K	100K	100K	132K	164K	164K	100K	196K	196K	260K	324K	324K
;;
;; Min L2 cache size:	20K	60K	84K	100K	116K	164K	196K	172K	244K	292K	324K	404K	484K	340K	484K	580K	644K	804K	964K	676K	964K	1156K	1284K	1604K	1924K
;;
;; NOTE:  The total size is a minimum estimated efficient L2 cache size.
;; We double the data and variable sin/cos data size as both the current
;; data and next block data must be in the cache at the same time.  The full formula is:
;;	(FFT data size) * 2 + Fixed sin/cos data
;; ALSO NOTE:  The real sin/cos data is used only in the first FFT block, so the memory
;; requirements for real data isn't particularly interesting.
;;


;*********************************************************************************************
;			Macros derived from pass2-8-levels
;*********************************************************************************************


; *************** radix4-pass2-192-levels-complex macro ******************
; This macro takes 2 groups of 192 complex values and
; performs the final 7.585 levels of a standard mostly radix-4 FFT, squares the
; results, and does the first 7.585 levels of the inverse FFT.

IFDEF  WONT_WORK_BECAUSE_NORMALIZE_ADD_SUB_CODE_EXPECTS_PASS2_IS_MULTIPLE_OF_8KB

r4_pass2_192_levels_complex MACRO
	LOCAL	b1b, b4b, b5b, b6a, b6b, b6c, b6d
	LOCAL	c1b, c4b, c5b
	LOCAL	xpass2_192_levels_complex_1, xpass2_192_levels_complex_2
	LOCAL	xpass2_192_levels_complex_3, xpass2_192_levels_complex_4
	LOCAL	xpass2_192_levels_complex_unfft, xpass2_192_levels_complex_done

;; Type 4 FFTs skip the forward FFT process

	mov	rsi, DATA_ADDR		;; Load address of FFT data
	mov	rbx, DIST_TO_FFTSRCARG
	cmp	ffttype, 4
	je	xpass2_192_levels_complex_4

;; Do FFT levels 1,2 (actually 1.585 levels)
;;
;; We swizzle the data so that one set of data is in the high word of
;; an xmm register, the other set is in the low word.  This means
;; a distance of 384 data values.
;;
;; On input the 64-byte cache lines hold these data values if pass 2 swizzles:
;;	0	+96	192	+96	384	+96	576	+96
;;	1	...
;;	...
;;	95
;; On input the 64-byte cache lines hold these data values if pass 1 swizzles:
;;	0	+384	96	+384	192	+384	288	+384
;;	1	...
;;	...
;;	95	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+384	32	+384	64	+384	96	+384
;;	1	...
;;	...
;;	31	...
;;	64	...
;;	...

;; Do 128 three_complex_fft operations
;;	distance between fft data elements is 64
;;	do 32 macros each processing 24 data values

	start_timer 4
	mov	rdi, xsincos_complex	;; The sin/cos data
	loops_init 32			;; 32 iterations
	r3_f3cl_three_complex_djbfft_preload
b1b:	r3_f3cl_three_complex_djbfft rsi, 64, 32*64, rdi, 32*XMM_SCD1
	bump	rdi, XMM_SCD1		;; Next sin/cos data
	loops	32, b1b			;; Test loop counter
	bump	rsi, -32*64		;; Restore source pointer
	bump	rdi, 32*XMM_SCD1	;; Next sin/cos data
	end_timer 4

;; Do FFT levels 3,4
;;
;; On input the 64-byte cache lines hold these data values:
;;	0	+384	32	+384	64	+384	96	+384
;;	1	...
;;	...
;;	31	...
;;	64	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+384	16	+384	32	+384	48	+384
;;	1	...
;;	...
;;	15	...
;;	64	...
;;	...
;;
;; Do 96 four_complex_fft operations
;;	distance between fft data elements is 16
;;	do 48 macros each processing 16 data values

	start_timer 5
	mov	rcx, DATA_PREFETCH	;; Prefetch pointer (we need to prefetch
					;; 96 cache lines + 0 pad cache lines)
	xtouch	[rcx+4096-128]		;; Load some prefetch TLBs
	xtouch	[rcx+2*4096-128]

	loops_init_prefetch 48, 64, 2, rcx
	r4_nf2cl_four_complex_djbfft_preload
b4b:	r4_nf2cl_four_complex_djbfft rsi, 64, 16*64, rdi
	bump	rdi, XMM_SCD2		;; Next sine/cosine pointer
	loops	16, b4b			;; Loop 16 times
	bump	rsi, -16*64+32*64	;; Next source pointer
	bump	rdi, -16*XMM_SCD2	;; Restore sine/cosine pointer
	loops	3, b4b			;; Test loop counter
	bump	rsi, -3*32*64		;; Restore source pointer
	end_timer 5

;; Do FFT levels 5,6
;;
;; On input the 64-byte cache lines hold these data values:
;;	0	+384	16	+384	32	+384	48	+384
;;	1	...
;;	...
;;	15	...
;;	64	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+384	4	+384	8	+384	12	+384
;;	1	...
;;	...
;;	3	...
;;	16	...
;;	...
;;
;; Do 96 four_complex_fft operations
;;	distance between fft data elements is 4
;;	do 24 macros each processing 32 data values

	start_timer 6
	loops_init_prefetch 24, 64, 2, rcx
	r4_x4cl_four_complex_djbfft_preload
b5b:	r4_x4cl_four_complex_djbfft rsi, 64, 4*64, 8*64, rdi
	bump	rdi, 4*XMM_SCD2		;; Next sine/cosine pointer
	loops	4, b5b			;; Loop 4 times
	bump	rsi, -4*64+16*64	;; Next source pointer
	bump	rdi, -4*4*XMM_SCD2	;; Restore sine/cosine pointer
	loops	6, b5b			;; Loop 6 times
	bump	rsi, -6*16*64		;; Restore source pointer
	end_timer 6

;; Do FFT levels 7,8 as well as inverse FFT levels 7,8
;;
;; On input the 64-byte cache lines hold these data values:
;;	0	+384	4	+384	8	+384	12	+384
;;	1	...
;;	...
;;	3	...
;;	16	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+384	2	+384	4	+384	6	+384
;;	1	...
;;	8	...
;;	...

;; Do 96 four_complex_fft operations
;;	distance between fft data elements is 1
;;	do 24 macros each processing 32 data values

;; Execute the right middle step

	cmp	ffttype, 2
	jl	xpass2_192_levels_complex_1
	jg	xpass2_192_levels_complex_3

xpass2_192_levels_complex_2:
	start_timer 9
	loops_init_prefetch 24, 64, 2, rcx
	xtouch	[rcx+4096-128]		;; Load prefetch TLBs
	xtouch	[rcx+2*4096-128]
	r4_x4cl_four_complex_with_square_preload
b6a:	r4_x4cl_four_complex_with_square rsi, 4*64, 64, 2*64
	loops	24, b6a			;; Loop 24 times
	bump	rsi, -24*4*64		;; Restore source pointer
	end_timer 9
	jmp	xpass2_192_levels_complex_unfft

xpass2_192_levels_complex_1:
	start_timer 9
	loops_init_prefetch 24, 64, 1, rcx
	xtouch	[rcx+4096-128]		;; Load prefetch TLBs
	xtouch	[rcx+2*4096-128]
b6b:	r4_x4cl_four_complex_fft_final rsi, 4*64, 64, 2*64
	loops	24, b6b			;; Loop 24 times
	end_timer 9
	jmp	xpass2_192_levels_complex_done

xpass2_192_levels_complex_3:
	start_timer 9
	mov	rbp, DIST_TO_MULSRCARG
	loops_init_prefetch 24, 64, 2, rcx
	xtouch	[rcx+4096-128]		;; Load prefetch TLBs
	xtouch	[rcx+2*4096-128]
b6c:	r4_x4cl_four_complex_with_mult rsi, 4*64, 64, 2*64
	loops	24, b6c			;; Loop 24 times
	bump	rsi, -24*4*64		;; Restore source pointer
	end_timer 9
	jmp	xpass2_192_levels_complex_unfft

xpass2_192_levels_complex_4:
	start_timer 9
	mov	rcx, DATA_PREFETCH	;; Prefetch pointer
	mov	rbp, DIST_TO_MULSRCARG
	loops_init_prefetch 24, 64, 1, rcx
	xtouch	[rcx]			;; Load first prefetch TLB
	xtouch	[rcx+4096-128]		;; Load prefetch TLBs
b6d:	r4_x4cl_four_complex_with_mulf rsi, 4*64, 64, 2*64
	loops	24, b6d			;; Loop 24 times
	bump	rsi, -24*4*64		;; Restore source pointer
	end_timer 9

;; Do inverse FFT levels 5,6
;; On input the 64-byte cache lines hold these data values:
;;	0	+384	2	+384	4	+384	6	+384
;;	1	...
;;	8	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+384	8	+384	16	+384	24	+384
;;	1	...
;;	...
;;	7	...
;;	32	...
;;	...
;;
;; Do 96 four_complex_unfft operations
;;	distance between fft data elements is 4
;;	do 24 macros each processing 32 data values

;; Do inverse FFT levels 3,4
;; On input the 64-byte cache lines hold these data values:
;;	0	+384	8	+384	16	+384	24	+384
;;	1	...
;;	...
;;	7	...
;;	32	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+384	32	+384	64	+384	96	+384
;;	1	...
;;	...
;;	31	...
;;	128	...
;;	...
;;
;; Do 96 four_complex_unfft operations
;;	distance between fft data elements is 16
;;	do 24 macros each processing 32 data values

;; Do this 16 cache lines at a time to maximize accessing data
;; while it is in the L1 cache.

xpass2_192_levels_complex_unfft:
	start_timer 10
	mov	rdi, xsincos_complex	;; Load sin/cos pointers
	bump	rdi, 64*XMM_SCD1	;; Next sin/cos data
	mov	rbx, rdi
	loops_init_prefetch 24, 64, 2, rcx ;; 3*2 iters of 16 cache lines
	r4_x4cl_four_complex_djbunfft_preload

	xtouch	[rcx+4096-128]		;; Load prefetch TLBs
	xtouch	[rcx+2*4096-128]

c5b:	r4_x4cl_four_complex_djbunfft rsi, 8*64, 2*64, 4*64, rbx, 2*4*XMM_SCD2
	loops	4, c5b			;; Loop 4 times
	bump	rsi, -4*8*64		;; Restore source pointer
	loops_undo 4			;; Restore loop counter

c4b:	r4_x4cl_four_complex_djbunfft rsi, 2*64, 8*64, 16*64, rdi, 8*1*XMM_SCD2
	bump	rdi, 2*1*XMM_SCD2	;; Next sine/cosine pointer
	loops	4, c4b			;; Loop 4 times
	bump	rdi, -4*2*1*XMM_SCD2	;; Restore sine/cosine pointer

	bump	rsi, -4*2*64+64		;; Next source pointer
	bump	rdi, 1*XMM_SCD2		;; Next sine/cosine pointer
	bump	rbx, 4*XMM_SCD2		;; Next sine/cosine pointer
	loops	2, c5b			;; Loop twice
	bump	rsi, -2*64+32*64	;; Next source pointer
	bump	rdi, -2*1*XMM_SCD2	;; Restore sine/cosine pointer
	bump	rbx, -2*4*XMM_SCD2	;; Restore sine/cosine pointer
	loops	3, c5b			;; Loop 3 times
	bump	rsi, -3*32*64		;; Restore source pointer
	end_timer 10

;; Do inverse FFT levels 1,2 (actually 1.585 levels)
;; On input the 64-byte cache lines hold these data values:
;;	0	+384	32	+384	64	+384	96	+384
;;	1	...
;;	...
;;	31	...
;;	128	...
;;	...
;; On output the 64-byte cache lines hold these data values if pass 1 swizzles:
;;	0	+384	96	+384	192	+384	288	+384
;;	1	...
;;	...
;;	95
;; On output the 64-byte cache lines hold these data values if pass 2 swizzles:
;;	0	+96	192	+96	384	+96	576	+96
;;	1	...
;;	...
;;	95

;; Do 96 four_complex_unfft operations
;;	distance between fft data elements is 64
;;	do 32 macros each processing 24 data values

	start_timer 14
	mov	rdi, xsincos_complex	;; Load sin/cos pointers
	bump	rcx, -10*64		;; Back up prefetch pointer by ?? cache lines
	loops_init_prefetch 32, 64, 4, rcx ;; 32 iters
	xtouch	[rcx+4096-128]		;; Load prefetch TLBs
	r3_x3cl_three_complex_djbunfft_preload
c1b:	r3_x3cl_three_complex_djbunfft rsi, 64, 32*64, rdi, 32*XMM_SCD1
	bump	rdi, XMM_SCD1		;; Next set of sin/cos data
	loops	32, c1b			;; Test loop counter
	end_timer 14

xpass2_192_levels_complex_done:
	ENDM

ENDIF


; *************** radix4-pass2-8-levels-real macro ******************
; This macro takes 512 real values and performs the final 9 levels of
; the FFT.  Independently, this macro also takes 256 complex values and
; performs the final 8 levels of the FFT.  The results are squared, and
; the first 8/9 levels of the inverse FFT are done.

r4_pass2_8_levels_real MACRO
	LOCAL	b1b, b4b, b5b, b6b, b7b, b8b, b9b, baa, bab, bac, bad
	LOCAL	c1b, c4b, c5b, c6b, c7b, c8b, c9b
	LOCAL	xpass2_8_levels_real_1, xpass2_8_levels_real_2
	LOCAL	xpass2_8_levels_real_3, xpass2_8_levels_real_4
	LOCAL	xpass2_8_real_unfft, xpass2_8_real_done

;; We switch to the same format used in one-pass FFTs (that is, the
;; high word of xmm registers being 1 greater than the low word):

;; Type 4 FFTs skip the forward FFT process

	start_timer 2
	mov	rsi, DATA_ADDR		;; Load source address
	mov	rbx, DIST_TO_FFTSRCARG
	cmp	ffttype, 4
	je	xpass2_8_levels_real_4

;; Do FFT level 1,2
;;
;; On input the 64-byte cache lines hold these data values if pass 2 swizzles:
;;	0	+128	256	+128	512	+128	768	+128
;;	1	...
;;	...
;;	127
;; On input the 64-byte cache lines hold these data values if pass 1 swizzles:
;;	0	+512	128	+512	256	+512	384	+512
;;	1	...
;;	...
;;	127	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+512	64	+512	128	+512	192	+512
;;	1	...
;;	...
;;	63	...
;;	256	...
;;	...

;; Do 64 eight_reals_fft operations
;; Do 64 four_complex_djbfft operations
;;	distance between fft data elements is 64
;;	do 64 macros each processing 16 data values

	mov	rdi, xsincos_complex	;; Ptr to the sin/cos data
	mov	rdx, sincos3		;; Ptr to real sin/cos data
	loops_init 64
b1b:	r4_fh2cl_eight_reals_four_complex_djbfft rsi, 64, 64*64, rdi, rdx
	bump	rdi, XMM_SCD2		;; Next sin/cos data
	bump	rdx, XMM_HSCD2		;; Next sin/cos data
	loops	64, b1b			;; Test loop counter
	bump	rsi, -64*64		;; Restore source pointer
	bump	rdi, -64*XMM_SCD2	;; Restore sin/cos pointer
	bump	rdx, -64*XMM_HSCD2	;; Restore sin/cos pointer

;; Do FFT levels 3,4
;;
;; On input the 64-byte cache lines hold these data values:
;;	0	+512	64	+512	128	+512	192	+512
;;	1	...
;;	...
;;	63	...
;;	256	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+512	16	+512	32	+512	48	+512
;;	1	...
;;	...
;;	15	...
;;	64	...
;;	...

;; Do 16 eight_reals_fft operations
;; Do 48 four_complex_djbfft operations
;;	distance between fft data elements is 16
;;	do 16 macros each processing 32 data values

	loops_init_prefetch 32, 64, 2, rcx
	mov	rcx, DATA_PREFETCH	;; Prefetch pointer
	xtouch	[rcx]			;; Preload first TLB
	xtouch	[rcx+4096-64]		;; Preload the TLBs
	xtouch	[rcx+2*4096-64]
	r4_h4cl_eight_reals_four_complex_djbfft_preload
b6b:	r4_h4cl_eight_reals_four_complex_djbfft rsi, 64, 16*64, 32*64, rdi, rdx
	bump	rdi, 4*XMM_SCD2		;; Next sin/cos data
	bump	rdx, 4*XMM_HSCD2	;; Next sin/cos data
	loops	16, b6b			;; Test loop counter
	bump	rsi, -16*64+64*64	;; Next source pointer
	bump	rdi, -16*4*XMM_SCD2	;; Restore sin/cos ptr
	bump	rdx, -16*4*XMM_HSCD2	;; Restore sin/cos ptr

;; Do 64 four_complex_djbfft operations
;;	distance between fft data elements is 16
;;	do 16 macros each processing 32 data values

	loops_reset			;; 1 more iters of 16
	r4_x4cl_four_complex_djbfft_preload
b7b:	r4_x4cl_four_complex_djbfft rsi, 64, 16*64, 32*64, rdi
	bump	rdi, 4*XMM_SCD2		;; Next sin/cos data
	loops	16, b7b			;; Test inner loop counter
	bump	rsi, -16*64+64*64	;; Next source pointer
	bump	rdi, -16*4*XMM_SCD2	;; Restore sin/cos ptr
	loops	2, b7b			;; Test loop counter
	bump	rsi, -2*64*64		;; Next source pointer

;; Do FFT levels 5,6
;;
;; On input the 64-byte cache lines hold these data values:
;;	0	+512	16	+512	32	+512	48	+512
;;	1	...
;;	...
;;	15	...
;;	64	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+512	4	+512	8	+512	12	+512
;;	1	...
;;	...
;;	3	...
;;	16	...
;;	...

;; Do 4 eight_reals_fft operations
;; Do 12 four_complex_djbfft operations
;;	distance between fft data elements is 4
;;	do 4 macros each processing 32 data values

	loops_init_prefetch 32, 64, 2, rcx
	r4_h4cl_eight_reals_four_complex_djbfft_preload
b8b:	r4_h4cl_eight_reals_four_complex_djbfft rsi, 64, 4*64, 8*64, rdi, rdx
	bump	rdi, 16*XMM_SCD2	;; Next sin/cos data
	bump	rdx, 16*XMM_HSCD2	;; Next sin/cos data
	loops	4, b8b			;; Test loop counter
	bump	rsi, -4*64+16*64	;; Next source pointer
	bump	rdi, -4*16*XMM_SCD2	;; Restore sin/cos ptr
	bump	rdx, -4*16*XMM_HSCD2	;; Restore sin/cos ptr

;; Do 112 four_complex_djbfft operations
;;	distance between fft data elements is 4
;;	do 28 macros each processing 32 data values

	loops_reset			;; 7 more iters of 4
	r4_x4cl_four_complex_djbfft_preload
b9b:	r4_x4cl_four_complex_djbfft rsi, 64, 4*64, 8*64, rdi
	bump	rdi, 16*XMM_SCD2	;; Next sin/cos data
	loops	4, b9b			;; Test loop counter
	bump	rsi, -4*64+16*64	;; Next source pointer
	bump	rdi, -4*16*XMM_SCD2	;; Restore sin/cos ptr
	loops	8, b9b			;; Test middle loop counter
	bump	rsi, -8*16*64		;; Next source pointer

;; Do FFT levels 7,8
;;
;; On input the 64-byte cache lines hold these data values:
;;	0	+512	4	+512	8	+512	12	+512
;;	1	...
;;	...
;;	3	...
;;	16	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+512	2	+512	4	+512	6	+512
;;	1	...
;;	8	...
;;	...

;; Do 1 eight_reals_fft operations
;; Do 3 four_complex_fft operations
;;	distance between fft data elements is 1

;; Do 124 four_complex_fft operations
;;	distance between fft data elements is 1
;;	do 31 macros each processing 32 data values

;; Execute the proper middle step

	cmp	ffttype, 2
	jl	xpass2_8_levels_real_1
	jg	xpass2_8_levels_real_3

xpass2_8_levels_real_2:
	r4_h4cl_eight_reals_four_complex_with_square rsi, 4*64, 64, 2*64
	loops_init_prefetch 32, 64, 2, rcx, 1, 32
	r4_x4cl_four_complex_with_square_preload
baa:	r4_x4cl_four_complex_with_square rsi, 4*64, 64, 2*64
	loops	32, baa			;; Test loop counter
	bump	rsi, -32*4*64		;; Next source pointer
	jmp	xpass2_8_real_unfft

xpass2_8_levels_real_1:
	r4_h4cl_eight_reals_four_complex_fft_final rsi, 4*64, 64, 2*64
	loops_init_prefetch 32, 64, 1, rcx, 1, 32
bab:	r4_x4cl_four_complex_fft_final rsi, 4*64, 64, 2*64
	loops	32, bab			;; Test loop counter
	jmp	xpass2_8_real_done

xpass2_8_levels_real_3:
	mov	rbp, DIST_TO_MULSRCARG
	r4_h4cl_eight_reals_four_complex_with_mult rsi, 4*64, 64, 2*64
	loops_init_prefetch 32, 64, 2, rcx, 1, 32
bac:	r4_x4cl_four_complex_with_mult rsi, 4*64, 64, 2*64
	loops	32, bac			;; Test loop counter
	bump	rsi, -32*4*64		;; Next source pointer
	jmp	xpass2_8_real_unfft

xpass2_8_levels_real_4:
	mov	rbp, DIST_TO_MULSRCARG
	mov	rcx, DATA_PREFETCH	;; Load prefetch pointer
	xtouch	[rcx]			;; Preload the TLBs
	xtouch	[rcx+4096-128]
	xtouch	[rcx+2*4096-128]
	r4_h4cl_eight_reals_four_complex_with_mulf rsi, 4*64, 64, 2*64
	loops_init_prefetch 32, 64, 1, rcx, 1, 32
bad:	r4_x4cl_four_complex_with_mulf rsi, 4*64, 64, 2*64
	loops	32, bad			;; Test loop counter
	bump	rsi, -32*4*64		;; Next source pointer

;; Do inverse FFT levels 5,6
;; On input the 64-byte cache lines hold these data values:
;;	0	+512	2	+512	4	+512	6	+512
;;	1	...
;;	8	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+512	8	+512	16	+512	24	+512
;;	1	...
;;	...
;;	7	...
;;	32	...
;;	...

;; Do 4 eight_reals_unfft operations
;; Do 4 four_complex_djbunfft operations
;;	distance between fft data elements is 4
;;	do 2 macros processing 32 data values

xpass2_8_real_unfft:
	mov	rdi, xsincos_complex	;; Load sin/cos pointers
	mov	rdx, sincos3
	loops_init_prefetch 32, 64, 1, rcx
c8b:	r4_h4cl_eight_reals_four_complex_djbunfft rsi, 64, 2*64, 4*64, rdi, 2*16*XMM_SCD2, rdx, 2*16*XMM_HSCD2
	bump	rdi, 16*XMM_SCD2	;; Next sin/cos data
	bump	rdx, 16*XMM_HSCD2	;; Next sin/cos data
	loops	2, c8b			;; Test loop counter
	bump	rsi, -2*64+8*64		;; Next source pointer
	bump	rdi, -2*16*XMM_SCD2	;; Restore sin/cos ptr
	bump	rdx, -2*16*XMM_HSCD2	;; Restore sin/cos ptr

;; Do 120 four_complex_djbunfft operations
;;	distance between fft data elements is 4
;;	do 30 macros each processing 32 data values

	loops_reset			;; 4 iters of 15 or 16 of 2
	r4_x4cl_four_complex_djbunfft_preload
c9b:	r4_x4cl_four_complex_djbunfft rsi, 64, 2*64, 4*64, rdi, 2*16*XMM_SCD2
	bump	rdi, 16*XMM_SCD2	;; Next sine/cosine pointer
	loops	2, c9b			;; Test loop counter
	bump	rsi, -2*64+8*64		;; Next source pointer
	bump	rdi, -2*16*XMM_SCD2	;; Restore sin/cos ptr
	loops	16, c9b			;; Test loop counter
	bump	rsi, -16*8*64		;; Next source pointer

;; Do inverse FFT levels 3,4
;; On input the 64-byte cache lines hold these data values:
;;	0	+512	8	+512	16	+512	24	+512
;;	1	...
;;	...
;;	7	...
;;	32	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+512	32	+512	64	+512	96	+512
;;	1	...
;;	...
;;	31	...
;;	128	...
;;	...

;; Do 16 eight_reals_unfft operations
;; Do 16 four_complex_djbunfft operations
;;	distance between fft data elements is 16
;;	do 8 macros processing 32 data values

	loops_init_prefetch 32, 64, 1, rcx
c6b:	r4_h4cl_eight_reals_four_complex_djbunfft rsi, 64, 8*64, 16*64, rdi, 8*4*XMM_SCD2, rdx, 8*4*XMM_HSCD2
	bump	rdi, 4*XMM_SCD2		;; Next sin/cos data
	bump	rdx, 4*XMM_HSCD2	;; Next sin/cos data
	loops	8, c6b			;; Test inner loop counter
	bump	rsi, -8*64+32*64	;; Next source pointer
	bump	rdi, -8*4*XMM_SCD2	;; Restore sin/cos ptr
	bump	rdx, -8*4*XMM_HSCD2	;; Restore sin/cos ptr

;; Do 96 four_complex_djbunfft operations
;;	distance between fft data elements is 16
;;	do 24 macros each processing 32 data values

	loops_reset			;; 3 more iters of 8
	r4_x4cl_four_complex_djbunfft_preload
c7b:	r4_x4cl_four_complex_djbunfft rsi, 64, 8*64, 16*64, rdi, 8*4*XMM_SCD2
	bump	rdi, 4*XMM_SCD2		;; Next sin/cos data
	loops	8, c7b			;; Test inner loop counter
	bump	rsi, -8*64+32*64	;; Next source pointer
	bump	rdi, -8*4*XMM_SCD2	;; Restore sin/cos ptr
	loops	4, c7b			;; Test loop counter
	bump	rsi, -4*32*64		;; Next source pointer

;; Do inverse FFT level 1,2
;; On input the 64-byte cache lines hold these data values:
;;	0	+512	32	+512	64	+512	96	+512
;;	1	...
;;	...
;;	31	...
;;	128	...
;;	...
;; On output the 64-byte cache lines hold these data values if pass 1 swizzles:
;;	0	+512	128	+512	256	+512	384	+512
;;	1	...
;;	...
;;	127
;; On output the 64-byte cache lines hold these data values if pass 2 swizzles:
;;	0	+128	256	+128	512	+128	768	+128
;;	1	...
;;	...
;;	127

;; Do 64 eight_reals_unfft operations
;; Do 64 four_complex_djbunfft operations
;;	distance between fft data elements is 64
;;	do 32 macros processing 32 data values

	mov	rdi, xsincos_complex	;; Load sin/cos pointers
	loops_init_prefetch 32, 64, 2, rcx
c1b:	r4_h4cl_eight_reals_four_complex_djbunfft rsi, 64, 32*64, 64*64, rdi, 32*XMM_SCD2, rdx, 32*XMM_HSCD2
	bump	rdi, XMM_SCD2		;; Next sin/cos data
	bump	rdx, XMM_HSCD2		;; Next sin/cos data
	loops	32, c1b			;; Test loop counter

xpass2_8_real_done:
	end_timer 2
	ENDM

; *************** radix4-pass2-8-levels-complex macro ******************
; This macro takes 2 groups of 256 complex values and
; performs the final 8 levels of a standard mostly radix-4 FFT, squares the
; results, and does the first 8 levels of the inverse FFT.

r4_pass2_8_levels_complex MACRO
	LOCAL	b1b, b4b, b5b, b6a, b6b, b6c, b6d, c1b, c4b, c5b
	LOCAL	xpass2_8_levels_complex_1, xpass2_8_levels_complex_2
	LOCAL	xpass2_8_levels_complex_3, xpass2_8_levels_complex_4
	LOCAL	xpass2_8_levels_complex_unfft, xpass2_8_levels_complex_done

;; Type 4 FFTs skip the forward FFT process

	mov	rsi, DATA_ADDR		;; Load address of FFT data
	mov	rbx, DIST_TO_FFTSRCARG
	cmp	ffttype, 4
	je	xpass2_8_levels_complex_4

;; Do FFT levels 1,2
;;
;; We swizzle the data so that one set of data is in the high word of
;; an xmm register, the other set is in the low word.  This means
;; a distance of 512 data values.
;;
;; On input the 64-byte cache lines hold these data values if pass 2 swizzles:
;;	0	+128	256	+128	512	+128	768	+128
;;	1	...
;;	...
;;	127
;; On input the 64-byte cache lines hold these data values if pass 1 swizzles:
;;	0	+512	128	+512	256	+512	384	+512
;;	1	...
;;	...
;;	127	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+512	64	+512	128	+512	192	+512
;;	1	...
;;	...
;;	63	...
;;	256	...
;;	...

;; Do 128 four_complex_fft operations
;;	distance between fft data elements is 64
;;	do 64 macros each processing 16 data values

	start_timer 4
	mov	rdi, xsincos_complex	;; The sin/cos data
	loops_init 64			;; 64 iterations
	r4_f2cl_four_complex_djbfft_preload
b1b:	xxprefetch [rdi+XMM_SCD2]	;; Prefetch for next pass
	r4_f2cl_four_complex_djbfft rsi, 64, 64*64, rdi
	bump	rdi, XMM_SCD2		;; Next sin/cos data
	loops	64, b1b			;; Test loop counter
	bump	rsi, -64*64		;; Restore source pointer
	bump	rdi, -64*XMM_SCD2	;; Restore sin/cos data
	end_timer 4

;; Do FFT levels 3,4
;;
;; On input the 64-byte cache lines hold these data values:
;;	0	+512	64	+512	128	+512	192	+512
;;	1	...
;;	...
;;	63	...
;;	256	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+512	16	+512	32	+512	48	+512
;;	1	...
;;	...
;;	15	...
;;	64	...
;;	...
;;
;; Do 128 four_complex_fft operations
;;	distance between fft data elements is 16
;;	do 32 macros each processing 32 data values

;; Do FFT levels 5,6
;;
;; On input the 64-byte cache lines hold these data values:
;;	0	+512	16	+512	32	+512	48	+512
;;	1	...
;;	...
;;	15	...
;;	64	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+512	4	+512	8	+512	12	+512
;;	1	...
;;	...
;;	3	...
;;	16	...
;;	...
;;
;; Do 128 four_complex_fft operations
;;	distance between fft data elements is 4
;;	do 32 macros each processing 32 data values

;; Do this 16 cache lines at a time to maximize accessing data
;; while it is in the L1 cache.

	start_timer 6
	mov	rcx, DATA_PREFETCH	;; Prefetch pointer (we need to prefetch
					;; 128 cache lines + 0 pad cache lines)
	xtouch	[rcx]			;; Load all the prefetch TLBs
	xtouch	[rcx+4096-64]
	xtouch	[rcx+2*4096-64]
	mov	rbx, rdi		;; Copy sin/cos ptr
	loops_init_prefetch 32, 64, 2, rcx ;; 2*4 iters of 16 cache lines
	r4_x4cl_four_complex_djbfft_preload

b4b:	r4_x4cl_four_complex_djbfft rsi, 4*64, 16*64, 32*64, rdi
	xxprefetch [rdi+4*XMM_SCD2]	;; Prefetch for next pass
	bump	rdi, 4*4*XMM_SCD2	;; Next sine/cosine pointer
	loops	4, b4b			;; Loop 4 times
	bump	rsi, -4*4*64		;; Restore source pointer
	bump	rdi, -4*4*4*XMM_SCD2	;; Restore sine/cosine pointer
	loops_undo 4			;; Back up loop counter

b5b:	r4_x4cl_four_complex_djbfft rsi, 16*64, 4*64, 8*64, rbx
	loops	4, b5b			;; Loop 4 times

	bump	rsi, -4*16*64+64	;; Next source pointer
	bump	rdi, 4*XMM_SCD2		;; Next sine/cosine pointer
	bump	rbx, 16*XMM_SCD2	;; Next sine/cosine pointer
	loops	4, b4b			;; Loop 4 times
	bump	rsi, -4*64+64*64	;; Next source pointer
	bump	rdi, -4*4*XMM_SCD2	;; Restore sine/cosine pointer
	bump	rbx, -4*16*XMM_SCD2	;; Restore sine/cosine pointer
	loops	2, b4b			;; Loop twice
	bump	rsi, -2*64*64		;; Restore source pointer
	end_timer 6

;; Do FFT levels 7,8 as well as inverse FFT levels 7,8
;;
;; On input the 64-byte cache lines hold these data values:
;;	0	+512	4	+512	8	+512	12	+512
;;	1	...
;;	...
;;	3	...
;;	16	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+512	2	+512	4	+512	6	+512
;;	1	...
;;	8	...
;;	...

;; Do 128 four_complex_fft operations
;;	distance between fft data elements is 1
;;	do 32 macros each processing 32 data values

;; Execute the right middle step

	cmp	ffttype, 2
	jl	xpass2_8_levels_complex_1
	jg	xpass2_8_levels_complex_3

xpass2_8_levels_complex_2:
	start_timer 9
	loops_init_prefetch 32, 64, 2, rcx
	r4_x4cl_four_complex_with_square_preload
b6a:	r4_x4cl_four_complex_with_square rsi, 4*64, 64, 2*64
	loops	32, b6a			;; Loop 32 times
	bump	rsi, -32*4*64		;; Restore source pointer
	end_timer 9
	jmp	xpass2_8_levels_complex_unfft

xpass2_8_levels_complex_1:
	start_timer 9
	loops_init_prefetch 32, 64, 1, rcx
b6b:	r4_x4cl_four_complex_fft_final rsi, 4*64, 64, 2*64
	loops	32, b6b			;; Loop 32 times
	end_timer 9
	jmp	xpass2_8_levels_complex_done

xpass2_8_levels_complex_3:
	start_timer 9
	mov	rbp, DIST_TO_MULSRCARG
	loops_init_prefetch 32, 64, 2, rcx
b6c:	r4_x4cl_four_complex_with_mult rsi, 4*64, 64, 2*64
	loops	32, b6c			;; Loop 32 times
	bump	rsi, -32*4*64		;; Restore source pointer
	end_timer 9
	jmp	xpass2_8_levels_complex_unfft

xpass2_8_levels_complex_4:
	start_timer 9
	mov	rcx, DATA_PREFETCH	;; Prefetch pointer
	xtouch	[rcx]			;; Load all prefetch TLB
	xtouch	[rcx+4096-64]
	xtouch	[rcx+2*4096-64]
	mov	rbp, DIST_TO_MULSRCARG
	loops_init_prefetch 32, 64, 1, rcx
b6d:	r4_x4cl_four_complex_with_mulf rsi, 4*64, 64, 2*64
	loops	32, b6d			;; Loop 32 times
	bump	rsi, -32*4*64		;; Restore source pointer
	end_timer 9

;; Do inverse FFT levels 5,6
;; On input the 64-byte cache lines hold these data values:
;;	0	+512	2	+512	4	+512	6	+512
;;	1	...
;;	8	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+512	8	+512	16	+512	24	+512
;;	1	...
;;	...
;;	7	...
;;	32	...
;;	...
;;
;; Do 128 four_complex_unfft operations
;;	distance between fft data elements is 4
;;	do 32 macros each processing 32 data values

;; Do inverse FFT levels 3,4
;; On input the 64-byte cache lines hold these data values:
;;	0	+512	8	+512	16	+512	24	+512
;;	1	...
;;	...
;;	7	...
;;	32	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+512	32	+512	64	+512	96	+512
;;	1	...
;;	...
;;	31	...
;;	128	...
;;	...
;;
;; Do 128 four_complex_unfft operations
;;	distance between fft data elements is 16
;;	do 32 macros each processing 32 data values

;; Do this 16 cache lines at a time to maximize accessing data
;; while it is in the L1 cache.

xpass2_8_levels_complex_unfft:
	start_timer 10
	mov	rdi, xsincos_complex	;; Load sin/cos pointers
	mov	rbx, rdi
	loops_init_prefetch 32, 64, 1, rcx ;; 4*2 iters of 16 cache lines
	r4_x4cl_four_complex_djbunfft_preload

c5b:	r4_x4cl_four_complex_djbunfft rsi, 8*64, 2*64, 4*64, rbx, 2*16*XMM_SCD2
	loops	4, c5b			;; Loop 4 times
	bump	rsi, -4*8*64		;; Restore source pointer
	loops_undo 4			;; Restore loop counter

c4b:	r4_x4cl_four_complex_djbunfft rsi, 2*64, 8*64, 16*64, rdi, 8*4*XMM_SCD2
	bump	rdi, 2*4*XMM_SCD2	;; Next sine/cosine pointer
	loops	4, c4b			;; Loop 4 times

	bump	rsi, -4*2*64+64		;; Next source pointer
	bump	rdi, -4*2*4*XMM_SCD2+4*XMM_SCD2 ;; Next sine/cosine pointer
	bump	rbx, 16*XMM_SCD2	;; Next sine/cosine pointer
	loops	2, c5b			;; Loop twice
	bump	rsi, -2*64+32*64	;; Next source pointer
	bump	rdi, -2*4*XMM_SCD2	;; Restore sine/cosine pointer
	bump	rbx, -2*16*XMM_SCD2	;; Restore sine/cosine pointer
	loops	4, c5b			;; Loop 4 times
	bump	rsi, -4*32*64		;; Restore source pointer
	end_timer 10

;; Do inverse FFT levels 1,2
;; On input the 64-byte cache lines hold these data values:
;;	0	+512	32	+512	64	+512	96	+512
;;	1	...
;;	...
;;	31	...
;;	128	...
;;	...
;; On output the 64-byte cache lines hold these data values if pass 1 swizzles:
;;	0	+512	128	+512	256	+512	384	+512
;;	1	...
;;	...
;;	127
;; On output the 64-byte cache lines hold these data values if pass 2 swizzles:
;;	0	+128	256	+128	512	+128	768	+128
;;	1	...
;;	...
;;	127

;; Do 128 four_complex_unfft operations
;;	distance between fft data elements is 64
;;	do 32 macros each processing 32 data values

	start_timer 14
	loops_init_prefetch 32, 64, 2, rcx ;; 32 iters
	r4_x4cl_four_complex_djbunfft_preload
c1b:	r4_x4cl_four_complex_djbunfft rsi, 64, 32*64, 64*64, rdi, 32*XMM_SCD2
	bump	rdi, XMM_SCD2		;; Next set of sin/cos data
	loops	32, c1b			;; Test loop counter
	end_timer 14

xpass2_8_levels_complex_done:
	ENDM


;*********************************************************************************************
;			Macros derived from pass2-10-levels
;*********************************************************************************************


; *************** radix4-pass2-768-levels-real macro ******************
; This macro takes 1536 real values and performs the final 10.585 levels of
; the FFT.  Independently, this macro also takes 768 complex values and
; performs the final 9.585 levels of the FFT.  The results are squared, and
; the first 9.585/10.585 levels of the inverse FFT are done.

r4_pass2_768_levels_real MACRO
	LOCAL	b1b, b4b, b5b, b6b, b7b, b8b, b9b, baa, bab0, bab, bac, bad0, bad
	LOCAL	c1b, c4b, c5b, c6b, c7b, c8b, c9b
	LOCAL	xpass2_768_levels_real_1, xpass2_768_levels_real_2
	LOCAL	xpass2_768_levels_real_3, xpass2_768_levels_real_4
	LOCAL	xpass2_768_real_unfft, xpass2_768_real_done

;; We switch to the same format used in one-pass FFTs (that is, the
;; high word of xmm registers being 1 greater than the low word):

;; Type 4 FFTs skip the forward FFT process

	start_timer 2
	mov	rsi, DATA_ADDR		;; Load source address
	mov	rbx, DIST_TO_FFTSRCARG
	cmp	ffttype, 4
	je	xpass2_768_levels_real_4

;; Do FFT level 1,2 (actually 1.585 levels)
;;
;; On input the 64-byte cache lines hold these data values if pass 2 swizzles:
;;	0	+384	768	+384	1536	+384	2304	+384
;;	1	...
;;	...
;;	383
;; On input the 64-byte cache lines hold these data values if pass 1 swizzles:
;;	0	+1536	384	+1536	768	+1536	1152	+1536
;;	1	...
;;	...
;;	383	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+1536	128	+1536	256	+1536	384	+1536
;;	1	...
;;	...
;;	127	...
;;	512	...
;;	...

;; Do 256 six_reals_fft operations
;; Do 256 three_complex_djbfft operations
;;	distance between fft data elements is 256
;;	do 128 macros each processing 24 data values

	mov	rdi, xsincos_complex	;; Ptr to the sin/cos data
	mov	rdx, sincos3		;; Ptr to real sin/cos data
	loops_init 128
b1b:	r3_fh3cl_six_reals_three_complex_djbfft rsi, 64, dist128, rdi, 128*XMM_SCD1, rdx, 128*XMM_HSCD1
	bump	rdi, XMM_SCD1		;; Next sin/cos data
	bump	rdx, XMM_HSCD1		;; Next sin/cos data
	loops	128, b1b		;; Test loop counter
	bump	rsi, -128*64		;; Restore source pointer
	bump	rdi, 128*XMM_SCD1	;; Next sin/cos ptr
	bump	rdx, 128*XMM_HSCD1	;; Next sin/cos ptr

;; Do FFT levels 3,4
;;
;; On input the 64-byte cache lines hold these data values:
;;	0	+1536	128	+1536	256	+1536	384	+1536
;;	1	...
;;	...
;;	127	...
;;	512	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+1536	64	+1536	128	+1536	192	+1536
;;	1	...
;;	...
;;	63	...
;;	256	...
;;	...

;; Do 64 eight_reals_fft operations
;; Do 64 four_complex_djbfft operations
;;	distance between fft data elements is 64
;;	do 64 macros each processing 16 data values

	mov	rcx, DATA_PREFETCH	;; Prefetch pointer
	xtouch	[rcx]			;; Preload 3 TLBs
	xtouch	[rcx+4096-64]
	xtouch	[rcx+2*4096-64]
	xtouch	[rcx+3*4096-64]
	loops_init_prefetch 192, 64, 4, rcx
b4b:	r4_h2cl_eight_reals_four_complex_djbfft rsi, 64, 64*64, rdi, rdx
	bump	rdi, XMM_SCD2		;; Next sin/cos data
	bump	rdx, XMM_HSCD2		;; Next sin/cos data
	loops	64, b4b			;; Test loop counter
	bump	rsi, -64*64+dist128	;; Next source pointer
	bump	rdi, -64*XMM_SCD2	;; Restore sin/cos ptr
	bump	rdx, -64*XMM_HSCD2	;; Restore sin/cos ptr

;; Do 256 four_complex_djbfft operations
;;	distance between fft data elements is 64
;;	do 128 macros each processing 16 data values

	loops_reset			;; 2 more iters of 64
	r4_nf2cl_four_complex_djbfft_preload
b5b:	r4_nf2cl_four_complex_djbfft rsi, 64, 64*64, rdi
	bump	rdi, XMM_SCD2		;; Next sin/cos data
	loops	64, b5b			;; Test inner loop counter
	bump	rsi, -64*64+dist128	;; Next source pointer
	bump	rdi, -64*XMM_SCD2	;; Restore sin/cos ptr
	loops	3, b5b			;; Test loop counter
	bump	rsi, -3*dist128		;; Next source pointer

;; Do FFT levels 5,6
;;
;; On input the 64-byte cache lines hold these data values:
;;	0	+1536	64	+1536	128	+1536	192	+1536
;;	1	...
;;	...
;;	63	...
;;	256	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+1536	16	+1536	32	+1536	48	+1536
;;	1	...
;;	...
;;	15	...
;;	64	...
;;	...

;; Do 16 eight_reals_fft operations
;; Do 48 four_complex_djbfft operations
;;	distance between fft data elements is 16
;;	do 16 macros each processing 32 data values

	loops_init_prefetch 96, 64, 2, rcx
	r4_h4cl_eight_reals_four_complex_djbfft_preload
b6b:	r4_h4cl_eight_reals_four_complex_djbfft rsi, 64, 16*64, 32*64, rdi, rdx
	bump	rdi, 4*XMM_SCD2		;; Next sin/cos data
	bump	rdx, 4*XMM_HSCD2	;; Next sin/cos data
	loops	16, b6b			;; Test loop counter
	bump	rsi, -16*64+64*64	;; Next source pointer
	bump	rdi, -16*4*XMM_SCD2	;; Restore sin/cos ptr
	bump	rdx, -16*4*XMM_HSCD2	;; Restore sin/cos ptr

;; Do 320 four_complex_djbfft operations
;;	distance between fft data elements is 16
;;	do 80 macros each processing 32 data values

	loops_reset			;; 3 iters of 1 or 2 of 16
	r4_x4cl_four_complex_djbfft_preload
b7b:	r4_x4cl_four_complex_djbfft rsi, 64, 16*64, 32*64, rdi
	bump	rdi, 4*XMM_SCD2		;; Next sin/cos data
	loops	16, b7b			;; Test inner loop counter
	bump	rsi, -16*64+64*64	;; Next source pointer
	bump	rdi, -16*4*XMM_SCD2	;; Restore sin/cos ptr
	loops	2, b7b			;; Test loop counter
	bump	rsi, -2*64*64+dist128	;; Next source pointer
	loops	3, b7b			;; 3 iterations
	bump	rsi, -3*dist128		;; Next source pointer

;; Do FFT levels 7,8
;;
;; On input the 64-byte cache lines hold these data values:
;;	0	+1536	16	+1536	32	+1536	48	+1536
;;	1	...
;;	...
;;	15	...
;;	64	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+1536	4	+1536	8	+1536	12	+1536
;;	1	...
;;	...
;;	3	...
;;	16	...
;;	...

;; Do 4 eight_reals_fft operations
;; Do 12 four_complex_djbfft operations
;;	distance between fft data elements is 4
;;	do 4 macros each processing 32 data values

	loops_init_prefetch 96, 64, 2, rcx
	r4_h4cl_eight_reals_four_complex_djbfft_preload
b8b:	r4_h4cl_eight_reals_four_complex_djbfft rsi, 64, 4*64, 8*64, rdi, rdx
	bump	rdi, 16*XMM_SCD2	;; Next sin/cos data
	bump	rdx, 16*XMM_HSCD2	;; Next sin/cos data
	loops	4, b8b			;; Test loop counter
	bump	rsi, -4*64+16*64	;; Next source pointer
	bump	rdi, -4*16*XMM_SCD2	;; Restore sin/cos ptr
	bump	rdx, -4*16*XMM_HSCD2	;; Restore sin/cos ptr

;; Do 368 four_complex_djbfft operations
;;	distance between fft data elements is 4
;;	do 92 macros each processing 32 data values

	loops_reset			;; 3 iters of 7 or 8 of 4
	r4_x4cl_four_complex_djbfft_preload
b9b:	r4_x4cl_four_complex_djbfft rsi, 64, 4*64, 8*64, rdi
	bump	rdi, 16*XMM_SCD2	;; Next sin/cos data
	loops	4, b9b			;; Test loop counter
	bump	rsi, -4*64+16*64	;; Next source pointer
	bump	rdi, -4*16*XMM_SCD2	;; Restore sin/cos ptr
	loops	8, b9b			;; Test middle loop counter
	bump	rsi, -8*16*64+dist128	;; Next source pointer
	loops	3, b9b			;; 3 iterations
	bump	rsi, -3*dist128		;; Next source pointer

;; Do FFT levels 9,10
;;
;; On input the 64-byte cache lines hold these data values:
;;	0	+1536	4	+1536	8	+1536	12	+1536
;;	1	...
;;	...
;;	3	...
;;	16	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+1536	2	+1536	4	+1536	6	+1536
;;	1	...
;;	8	...
;;	...

;; Do 1 eight_reals_fft operations
;; Do 3 four_complex_fft operations
;;	distance between fft data elements is 1

;; Do 380 four_complex_fft operations
;;	distance between fft data elements is 1
;;	do 95 macros each processing 32 data values

;; Execute the proper middle step

	cmp	ffttype, 2
	jl	xpass2_768_levels_real_1
	jg	xpass2_768_levels_real_3

xpass2_768_levels_real_2:
	r4_h4cl_eight_reals_four_complex_with_square rsi, 4*64, 64, 2*64
	loops_init_prefetch 96, 64, 2, rcx, 1, 32
	r4_x4cl_four_complex_with_square_preload
baa:	r4_x4cl_four_complex_with_square rsi, 4*64, 64, 2*64
	loops	32, baa			;; Test loop counter
	bump	rsi, -32*4*64+dist128	;; Next source pointer
	loops	3, baa			;; Test loop counter
	bump	rsi, -3*dist128		;; Next source pointer
	jmp	xpass2_768_real_unfft

xpass2_768_levels_real_1:
	r4_h4cl_eight_reals_four_complex_fft_final rsi, 4*64, 64, 2*64
	loops_init_prefetch 96, 128, 1, rcx, 1, 32
bab0:	xtouch	[rcx+4096-64]		;; Preload the TLBs
bab:	r4_x4cl_four_complex_fft_final rsi, 4*64, 64, 2*64
	loops	32, bab			;; Test loop counter
	bump	rsi, -32*4*64+dist128	;; Next source pointer
	loops	3, bab0			;; Test loop counter
	jmp	xpass2_768_real_done

xpass2_768_levels_real_3:
	mov	rbp, DIST_TO_MULSRCARG
	r4_h4cl_eight_reals_four_complex_with_mult rsi, 4*64, 64, 2*64
	loops_init_prefetch 96, 64, 2, rcx, 1, 32
bac:	r4_x4cl_four_complex_with_mult rsi, 4*64, 64, 2*64
	loops	32, bac			;; Test loop counter
	bump	rsi, -32*4*64+dist128	;; Next source pointer
	loops	3, bac			;; Test loop counter
	bump	rsi, -3*dist128		;; Next source pointer
	jmp	xpass2_768_real_unfft

xpass2_768_levels_real_4:
	mov	rbp, DIST_TO_MULSRCARG
	mov	rcx, DATA_PREFETCH	;; Load prefetch pointer
	xtouch	[rcx]			;; Preload 3 TLBs
	r4_h4cl_eight_reals_four_complex_with_mulf rsi, 4*64, 64, 2*64
	loops_init_prefetch 96, 128, 1, rcx, 1, 32
bad0:	xtouch	[rcx+4096-64]
bad:	r4_x4cl_four_complex_with_mulf rsi, 4*64, 64, 2*64
	loops	32, bad			;; Test loop counter
	bump	rsi, -32*4*64+dist128	;; Next source pointer
	loops	3, bad0			;; Test loop counter
	bump	rsi, -3*dist128		;; Next source pointer

;; Do inverse FFT levels 7,8
;; On input the 64-byte cache lines hold these data values:
;;	0	+1536	2	+1536	4	+1536	6	+1536
;;	1	...
;;	8	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+1536	8	+1536	16	+1536	24	+1536
;;	1	...
;;	...
;;	7	...
;;	32	...
;;	...

;; Do 4 eight_reals_unfft operations
;; Do 4 four_complex_djbunfft operations
;;	distance between fft data elements is 4
;;	do 2 macros processing 32 data values

xpass2_768_real_unfft:
	mov	rdi, xsincos_complex	;; Load sin/cos pointers
	bump	rdi, 256*XMM_SCD1
	mov	rdx, sincos3
	bump	rdx, 256*XMM_HSCD1
	loops_init_prefetch 96, 64, 2, rcx
	xtouch	[rcx+4096-128]		;; Preload 3 more TLBs
	xtouch	[rcx+2*4096-128]
	xtouch	[rcx+3*4096-128]
c8b:	r4_h4cl_eight_reals_four_complex_djbunfft rsi, 64, 2*64, 4*64, rdi, 2*16*XMM_SCD2, rdx, 2*16*XMM_HSCD2
	bump	rdi, 16*XMM_SCD2	;; Next sin/cos data
	bump	rdx, 16*XMM_HSCD2	;; Next sin/cos data
	loops	2, c8b			;; Test loop counter
	bump	rsi, -2*64+8*64		;; Next source pointer
	bump	rdi, -2*16*XMM_SCD2	;; Restore sin/cos ptr
	bump	rdx, -2*16*XMM_HSCD2	;; Restore sin/cos ptr

;; Do 376 four_complex_djbunfft operations
;;	distance between fft data elements is 4
;;	do 94 macros each processing 32 data values

	loops_reset			;; 3 iters of 15 or 16 of 2
	r4_x4cl_four_complex_djbunfft_preload
c9b:	r4_x4cl_four_complex_djbunfft rsi, 64, 2*64, 4*64, rdi, 2*16*XMM_SCD2
	bump	rdi, 16*XMM_SCD2	;; Next sine/cosine pointer
	loops	2, c9b			;; Test loop counter
	bump	rsi, -2*64+8*64		;; Next source pointer
	bump	rdi, -2*16*XMM_SCD2	;; Restore sin/cos ptr
	loops	16, c9b			;; Test loop counter
	bump	rsi, -16*8*64+dist128	;; Next source pointer
	loops	3, c9b			;; Test loop counter
	bump	rsi, -3*dist128		;; Next source pointer

;; Do inverse FFT levels 5,6
;; On input the 64-byte cache lines hold these data values:
;;	0	+1536	8	+1536	16	+1536	24	+1536
;;	1	...
;;	...
;;	7	...
;;	32	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+1536	32	+1536	64	+1536	96	+1536
;;	1	...
;;	...
;;	31	...
;;	128	...
;;	...

;; Do 16 eight_reals_unfft operations
;; Do 16 four_complex_djbunfft operations
;;	distance between fft data elements is 16
;;	do 8 macros processing 32 data values

	loops_init_prefetch 96, 64, 2, rcx
c6b:	r4_h4cl_eight_reals_four_complex_djbunfft rsi, 64, 8*64, 16*64, rdi, 8*4*XMM_SCD2, rdx, 8*4*XMM_HSCD2
	bump	rdi, 4*XMM_SCD2		;; Next sin/cos data
	bump	rdx, 4*XMM_HSCD2	;; Next sin/cos data
	loops	8, c6b			;; Test inner loop counter
	bump	rsi, -8*64+32*64	;; Next source pointer
	bump	rdi, -8*4*XMM_SCD2	;; Restore sin/cos ptr
	bump	rdx, -8*4*XMM_HSCD2	;; Restore sin/cos ptr

;; Do 352 four_complex_djbunfft operations
;;	distance between fft data elements is 16
;;	do 88 macros each processing 32 data values

	loops_reset			;; 3 iters of 3 or 4 of 4
	r4_x4cl_four_complex_djbunfft_preload
c7b:	r4_x4cl_four_complex_djbunfft rsi, 64, 8*64, 16*64, rdi, 8*4*XMM_SCD2
	bump	rdi, 4*XMM_SCD2		;; Next sin/cos data
	loops	8, c7b			;; Test inner loop counter
	bump	rsi, -8*64+32*64	;; Next source pointer
	bump	rdi, -8*4*XMM_SCD2	;; Restore sin/cos ptr
	loops	4, c7b			;; Test loop counter
	bump	rsi, -4*32*64+dist128	;; Next source pointer
	loops	3, c7b			;; 3 iterations
	bump	rsi, -3*dist128		;; Next source pointer

;; Do inverse FFT levels 3,4
;; On input the 64-byte cache lines hold these data values:
;;	0	+1536	32	+1536	64	+1536	96	+1536
;;	1	...
;;	...
;;	31	...
;;	128	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+1536	128	+1536	256	+1536	384	+1536
;;	1	...
;;	...
;;	127	...
;;	512	...
;;	...

;; Do 64 eight_reals_unfft operations
;; Do 64 four_complex_djbunfft operations
;;	distance between fft data elements is 64
;;	do 32 macros processing 32 data values

	loops_init_prefetch 96, 64, 2, rcx
c4b:	r4_h4cl_eight_reals_four_complex_djbunfft rsi, 64, 32*64, 64*64, rdi, 32*XMM_SCD2, rdx, 32*XMM_HSCD2
	bump	rdi, XMM_SCD2		;; Next sin/cos data
	bump	rdx, XMM_HSCD2		;; Next sin/cos data
	loops	32, c4b			;; Test inner loop counter
	bump	rsi, -32*64+dist128	;; Next source pointer
	bump	rdi, -32*XMM_SCD2	;; Restore sin/cos ptr
	bump	rdx, -32*XMM_HSCD2	;; Restore sin/cos ptr

;; Do 256 four_complex_djbunfft operations
;;	distance between fft data elements is 64
;;	do 64 macros each processing 32 data values

	loops_reset			;; 2 more iters of 32
	r4_x4cl_four_complex_djbunfft_preload
c5b:	r4_x4cl_four_complex_djbunfft rsi, 64, 32*64, 64*64, rdi, 32*XMM_SCD2
	bump	rdi, XMM_SCD2		;; Next sin/cos data
	loops	32, c5b			;; Test inner loop counter
	bump	rsi, -32*64+dist128	;; Next source pointer
	bump	rdi, -32*XMM_SCD2	;; Restore sin/cos ptr
	loops	3, c5b			;; Test loop counter
	bump	rsi, -3*dist128		;; Next source pointer

;; Do inverse FFT level 1,2 (actually 1.585 FFT levels)
;; On input the 64-byte cache lines hold these data values:
;;	0	+1536	128	+1536	256	+1536	384	+1536
;;	1	...
;;	...
;;	127	...
;;	512	...
;;	...
;; On output the 64-byte cache lines hold these data values if pass 1 swizzles:
;;	0	+1536	384	+1536	768	+1536	1152	+1536
;;	1	...
;;	...
;;	383
;; On output the 64-byte cache lines hold these data values if pass 2 swizzles:
;;	0	+384	768	+384	1536	+384	2304	+384
;;	1	...
;;	...
;;	383

;; Do 256 six_reals_unfft operations
;; Do 256 three_complex_djbunfft operations
;;	distance between fft data elements is 256
;;	do 128 macros processing 24 data values

	mov	rdi, xsincos_complex	;; Load sin/cos pointers
	mov	rdx, sincos3
	bump	rcx, -12*64		;; Back up prefetch pointer by 12 cache lines
	loops_init_prefetch 128, 64, 2, rcx
c1b:	r3_h3cl_six_reals_three_complex_djbunfft rsi, 64, dist128, rdi, 128*XMM_SCD1, rdx, 128*XMM_HSCD1
	bump	rdi, XMM_SCD1		;; Next sin/cos data
	bump	rdx, XMM_HSCD1		;; Next sin/cos data
	loops	128, c1b		;; Test loop counter

xpass2_768_real_done:
	end_timer 2
	ENDM

; *************** radix4-pass2-768-levels-complex macro ******************
; This macro takes 2 groups of 768 complex values and
; performs the final 9.585 levels of a standard mostly radix-4 FFT, squares the
; results, and does the first 9.585 levels of the inverse FFT.

r4_pass2_768_levels_complex MACRO
	LOCAL	b1b, b3b, b4b, b5b, b6a, b6b0, b6b, b6c, b6d0, b6d
	LOCAL	c1b, c3b, c4b, c5b
	LOCAL	xpass2_768_levels_complex_1, xpass2_768_levels_complex_2
	LOCAL	xpass2_768_levels_complex_3, xpass2_768_levels_complex_4
	LOCAL	xpass2_768_levels_complex_unfft, xpass2_768_levels_complex_done

;; Type 4 FFTs skip the forward FFT process

	mov	rsi, DATA_ADDR		;; Load address of FFT data
	mov	rbx, DIST_TO_FFTSRCARG
	cmp	ffttype, 4
	je	xpass2_768_levels_complex_4

;; Do FFT levels 1,2 (actually 1.585 levels)
;;
;; We swizzle the data so that one set of data is in the high word of
;; an xmm register, the other set is in the low word.  This means
;; a distance of 1536 data values.
;;
;; On input the 64-byte cache lines hold these data values if pass 2 swizzles:
;;	0	+384	768	+384	1536	+384	2304	+384
;;	1	...
;;	...
;;	383
;; On input the 64-byte cache lines hold these data values if pass 1 swizzles:
;;	0	+1536	384	+1536	768	+1536	1152	+1536
;;	1	...
;;	...
;;	383	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+1536	128	+1536	256	+1536	384	+1536
;;	1	...
;;	...
;;	127	...
;;	512	...
;;	...

;; Do 512 three_complex_fft operations
;;	distance between fft data elements is 256
;;	do 128 macros each processing 24 data values

	start_timer 4
	mov	rdi, xsincos_complex	;; The sin/cos data
	loops_init 128			;; 128 iterations
	r3_f3cl_three_complex_djbfft_preload
b1b:	xxprefetch [rdi+XMM_SCD1]	;; Prefetch for next pass
	xxprefetch [rdi+129*XMM_SCD1]	;; Prefetch for next pass
	r3_f3cl_three_complex_djbfft rsi, 64, dist128, rdi, 128*XMM_SCD1
	bump	rdi, XMM_SCD1		;; Next sin/cos data
	loops	128, b1b		;; Test inner loop counter
	bump	rsi, -128*64		;; Restore source pointer
	bump	rdi, 128*XMM_SCD1	;; Next sin/cos ptr
	end_timer 4

;; Do FFT levels 3,4
;;
;; On input the 64-byte cache lines hold these data values:
;;	0	+1536	128	+1536	256	+1536	384	+1536
;;	1	...
;;	...
;;	127	...
;;	512	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+1536	64	+1536	128	+1536	192	+1536
;;	1	...
;;	...
;;	63	...
;;	256	...
;;	...
;;
;; Do 384 four_complex_fft operations
;;	distance between fft data elements is 64
;;	do 192 macros each processing 16 data values

;; Do FFT levels 5,6
;;
;; On input the 64-byte cache lines hold these data values:
;;	0	+1536	64	+1536	128	+1536	192	+1536
;;	1	...
;;	...
;;	63	...
;;	256	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+1536	16	+1536	32	+1536	48	+1536
;;	1	...
;;	...
;;	15	...
;;	64	...
;;	...
;;
;; Do 384 four_complex_fft operations
;;	distance between fft data elements is 16
;;	do 96 macros each processing 32 data values

;; Do FFT levels 7,8
;;
;; On input the 64-byte cache lines hold these data values:
;;	0	+1536	16	+1536	32	+1536	48	+1536
;;	1	...
;;	...
;;	15	...
;;	64	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+1536	4	+1536	8	+1536	12	+1536
;;	1	...
;;	...
;;	3	...
;;	16	...
;;	...
;;
;; Do 384 four_complex_fft operations
;;	distance between fft data elements is 4
;;	do 96 macros each processing 32 data values

;; Do this 32 cache lines at a time to maximize accessing data
;; while it is in the L1 cache.

	start_timer 6
	mov	rcx, DATA_PREFETCH	;; Prefetch pointer (we need to prefetch 
					;; 384 cache lines + 4 pad cache lines)
	xtouch	[rcx]			;; Load 3 prefetch TLBs
	xtouch	[rcx+4096-64]
	xtouch	[rcx+2*4096-64]
	xtouch	[rcx+3*4096-64]
	mov	rbx, rdi		;; Copy sin/cos pointers
	mov	rbp, rdi
	loops_init_prefetch 192, 64, 4, rcx ;; 3 iters of 4 of 32 cache lines
	;r4_nf2cl_four_complex_djbfft_preload -- assume same as r4_x4cl_four_complex_djbfft_preload
	r4_x4cl_four_complex_djbfft_preload

b3b:	xxprefetch [rdi+4*XMM_SCD2]	;; Prefetch for next pass
	r4_nf2cl_four_complex_djbfft rsi, 4*64, 64*64, rdi
	bump	rdi, 4*XMM_SCD2		;; Next sine/cosine pointer
	loops	16, b3b			;; Loop 16 times
	bump	rsi, -16*4*64		;; Restore source pointer
	bump	rdi, -16*4*XMM_SCD2	;; Restore sine/cosine pointer
	loops_undo 16			;; Back up loop counter

b4b:	r4_x4cl_four_complex_djbfft rsi, 4*64, 16*64, 32*64, rbx
	bump	rbx, 4*4*XMM_SCD2	;; Next sine/cosine pointer
	loops	4, b4b, 2		;; 4 iterations with non-standard increment
	bump	rsi, -4*4*64+64*64	;; Next source pointer
	bump	rbx, -4*4*4*XMM_SCD2	;; Restore sine/cosine pointer
	loops	2, b4b			;; Loop twice
	bump	rsi, -2*64*64		;; Restore source pointer
	loops_undo 2*4*2		;; Back up loop counter

b5b:	r4_x4cl_four_complex_djbfft rsi, 16*64, 4*64, 8*64, rbp
	loops	8, b5b, 2		;; 8 iterations with non-standard increment

	bump	rsi, -8*16*64+64	;; Next source pointer
	bump	rdi, XMM_SCD2		;; Next sine/cosine pointer
	bump	rbx, 4*XMM_SCD2		;; Next sine/cosine pointer
	bump	rbp, 16*XMM_SCD2	;; Next sine/cosine pointer
	loops	4, b3b			;; Loop 4 times
	bump	rsi, -4*64+dist128	;; Next source pointer
	bump	rdi, -4*XMM_SCD2	;; Restore sine/cosine pointer
	bump	rbx, -4*4*XMM_SCD2	;; Restore sine/cosine pointer
	bump	rbp, -4*16*XMM_SCD2	;; Restore sine/cosine pointer
	loops	3, b3b			;; Loop 3 times
	bump	rsi, -3*dist128		;; Restore source pointer
	end_timer 6

;; Do FFT levels 9,10 as well as inverse FFT levels 9,10
;;
;; On input the 64-byte cache lines hold these data values:
;;	0	+1536	4	+1536	8	+1536	12	+1536
;;	1	...
;;	...
;;	3	...
;;	16	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+1536	2	+1536	4	+1536	6	+1536
;;	1	...
;;	8	...
;;	...

;; Do 384 four_complex_fft operations
;;	distance between fft data elements is 1
;;	do 96 macros each processing 32 data values

;; Execute the right middle step

	cmp	ffttype, 2
	jl	xpass2_768_levels_complex_1
	jg	xpass2_768_levels_complex_3

xpass2_768_levels_complex_2:
	start_timer 9
	loops_init_prefetch 96, 64, 2, rcx
	r4_x4cl_four_complex_with_square_preload
b6a:	r4_x4cl_four_complex_with_square rsi, 4*64, 64, 2*64
	loops	32, b6a			;; Loop 32 times
	bump	rsi, -32*4*64+dist128	;; Next source pointer
	loops	3, b6a			;; Loop 3 times
	bump	rsi, -3*dist128		;; Restore source pointer
	end_timer 9
	jmp	xpass2_768_levels_complex_unfft

xpass2_768_levels_complex_1:
	start_timer 9
	loops_init_prefetch 96, 128, 1, rcx
b6b0:	xtouch	[rcx+4096-64]		;; Load next 3 prefetch TLBs
b6b:	r4_x4cl_four_complex_fft_final rsi, 4*64, 64, 2*64
	loops	32, b6b			;; Loop 32 times
	bump	rsi, -32*4*64+dist128	;; Next source pointer
	loops	3, b6b0			;; Loop 3 times
	end_timer 9
	jmp	xpass2_768_levels_complex_done

xpass2_768_levels_complex_3:
	start_timer 9
	mov	rbp, DIST_TO_MULSRCARG
	loops_init_prefetch 96, 64, 2, rcx
b6c:	r4_x4cl_four_complex_with_mult rsi, 4*64, 64, 2*64
	loops	32, b6c			;; Loop 32 times
	bump	rsi, -32*4*64+dist128	;; Next source pointer
	loops	3, b6c			;; Loop 3 times
	bump	rsi, -3*dist128		;; Restore source pointer
	end_timer 9
	jmp	xpass2_768_levels_complex_unfft

xpass2_768_levels_complex_4:
	start_timer 9
	mov	rcx, DATA_PREFETCH	;; Prefetch pointer
	xtouch	[rcx]			;; Load first prefetch TLB
	mov	rbp, DIST_TO_MULSRCARG
	loops_init_prefetch 96, 128, 1, rcx
b6d0:	xtouch	[rcx+4096-64]
b6d:	r4_x4cl_four_complex_with_mulf rsi, 4*64, 64, 2*64
	loops	32, b6d			;; Loop 32 times
	bump	rsi, -32*4*64+dist128	;; Next source pointer
	loops	3, b6d0			;; Loop 3 times
	bump	rsi, -3*dist128		;; Restore source pointer
	end_timer 9

;; Do inverse FFT levels 7,8
;; On input the 64-byte cache lines hold these data values:
;;	0	+1536	2	+1536	4	+1536	6	+1536
;;	1	...
;;	8	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+1536	8	+1536	16	+1536	24	+1536
;;	1	...
;;	...
;;	7	...
;;	32	...
;;	...
;;
;; Do 384 four_complex_unfft operations
;;	distance between fft data elements is 4
;;	do 96 macros each processing 32 data values

;; Do inverse FFT levels 5,6
;; On input the 64-byte cache lines hold these data values:
;;	0	+1536	8	+1536	16	+1536	24	+1536
;;	1	...
;;	...
;;	7	...
;;	32	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+1536	32	+1536	64	+1536	96	+1536
;;	1	...
;;	...
;;	31	...
;;	128	...
;;	...
;;
;; Do 384 four_complex_unfft operations
;;	distance between fft data elements is 16
;;	do 96 macros each processing 32 data values

;; Do inverse FFT levels 3,4
;; On input the 64-byte cache lines hold these data values:
;;	0	+1536	32	+1536	64	+1536	96	+1536
;;	1	...
;;	...
;;	31	...
;;	128	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+1536	128	+1536	256	+1536	384	+1536
;;	1	...
;;	...
;;	127	...
;;	512	...
;;	...
;;
;; Do 384 four_complex_unfft operations
;;	distance between fft data elements is 64
;;	do 96 macros each processing 32 data values

;; Do this 64 cache lines at a time to maximize accessing data
;; while it is in the L1 cache.

xpass2_768_levels_complex_unfft:
	start_timer 10
	mov	rdi, xsincos_complex	;; Load sin/cos pointers
	bump	rdi, 256*XMM_SCD1
	mov	rbx, rdi
	mov	rbp, rdi
	loops_init_prefetch 96, 64, 2, rcx ;; 3*2 iters of 64 cache lines
	r4_x4cl_four_complex_djbunfft_preload

	xtouch	[rcx+4096-64]		;; Load next 3 prefetch TLBs
	xtouch	[rcx+2*4096-64]
	xtouch	[rcx+3*4096-64]

c5b:	r4_x4cl_four_complex_djbunfft rsi, 8*64, 2*64, 4*64, rbp, 2*16*XMM_SCD2
	loops	16, c5b			;; Loop 16 times
	bump	rsi, -16*8*64		;; Restore source pointer
	loops_undo 16			;; Restore loop counter

c4b:	r4_x4cl_four_complex_djbunfft rsi, 2*64, 8*64, 16*64, rbx, 8*4*XMM_SCD2
	bump	rbx, 2*4*XMM_SCD2	;; Next sine/cosine pointer
	loops	4, c4b			;; Loop 4 times
	bump	rsi, -4*2*64+32*64	;; Next source pointer
	bump	rbx, -4*2*4*XMM_SCD2	;; Restore sine/cosine pointer
	loops	4, c4b			;; Loop 4 times
	bump	rsi, -4*32*64		;; Restore source pointer
	loops_undo 4*4			;; Restore loop counter

c3b:	r4_x4cl_four_complex_djbunfft rsi, 2*64, 32*64, 64*64, rdi, 32*1*XMM_SCD2
	bump	rdi, 2*XMM_SCD2		;; Next sine/cosine pointer
	loops	16, c3b			;; Loop 16 times
	bump	rdi, -16*2*XMM_SCD2	;; Restore sine/cosine pointer

	bump	rsi, -16*2*64+64	;; Next source pointer
	bump	rdi, XMM_SCD2		;; Next sine/cosine pointer
	bump	rbx, 4*XMM_SCD2		;; Next sine/cosine pointer
	bump	rbp, 16*XMM_SCD2	;; Next sine/cosine pointer
	loops	2, c5b			;; Loop twice
	bump	rsi, -2*64+dist128	;; Next source pointer
	bump	rdi, -2*XMM_SCD2	;; Restore sine/cosine pointer
	bump	rbx, -2*4*XMM_SCD2	;; Restore sine/cosine pointer
	bump	rbp, -2*16*XMM_SCD2	;; Restore sine/cosine pointer
	loops	3, c5b			;; Loop 3 times
	bump	rsi, -3*dist128		;; Restore source pointer
	end_timer 10

;; Do inverse FFT levels 1,2 (actually 1.585 FFT levels)
;; On input the 64-byte cache lines hold these data values:
;;	0	+1536	128	+1536	256	+1536	384	+1536
;;	1	...
;;	...
;;	127	...
;;	512	...
;;	...
;; On output the 64-byte cache lines hold these data values if pass 1 swizzles:
;;	0	+1536	384	+1536	768	+1536	1152	+1536
;;	1	...
;;	...
;;	383
;; On output the 64-byte cache lines hold these data values if pass 2 swizzles:
;;	0	+384	768	+384	1536	+384	2304	+384
;;	1	...
;;	...
;;	383

;; Do 512 three_complex_unfft operations
;;	distance between fft data elements is 256
;;	do 128 macros each processing 24 data values

	start_timer 14
	mov	rdi, xsincos_complex	;; Load sin/cos pointer
	bump	rcx, -12*64		;; Back up prefetch pointer by 12 cache lines
	loops_init_prefetch 128, 64, 2, rcx ;; 128 iters
	r3_x3cl_three_complex_djbunfft_preload
c1b:	r3_x3cl_three_complex_djbunfft rsi, 64, dist128, rdi, 128*XMM_SCD1
	bump	rdi, XMM_SCD1		;; Next set of sin/cos data
	loops	128, c1b		;; Test loop counter
	end_timer 14

xpass2_768_levels_complex_done:
	ENDM


; *************** radix4-pass2-10-levels-real macro ******************
; This macro takes 2048 real values and performs the final 11 levels of
; the FFT.  Independently, this macro also takes 1024 complex values and
; performs the final 10 levels of the FFT.  The results are squared, and
; the first 10/11 levels of the inverse FFT are done.

r4_pass2_10_levels_real MACRO
	LOCAL	b1b, b4b, b5b, b6b, b7b, b8b, b9b, baa, bab0, bab, bac, bad0, bad
	LOCAL	c1b, c4b, c5b, c6b, c7b, c8b, c9b
	LOCAL	xpass2_10_levels_real_1, xpass2_10_levels_real_2
	LOCAL	xpass2_10_levels_real_3, xpass2_10_levels_real_4
	LOCAL	xpass2_10_real_unfft, xpass2_10_real_done

;; We switch to the same format used in one-pass FFTs (that is, the
;; high word of xmm registers being 1 greater than the low word):

;; Type 4 FFTs skip the forward FFT process

	start_timer 2
	mov	rsi, DATA_ADDR		;; Load source address
	mov	rbx, DIST_TO_FFTSRCARG
	cmp	ffttype, 4
	je	xpass2_10_levels_real_4

;; Do FFT level 1,2
;;
;; On input the 64-byte cache lines hold these data values if pass 2 swizzles:
;;	0	+512	1024	+512	2048	+512	3072	+512
;;	1	...
;;	...
;;	511
;; On input the 64-byte cache lines hold these data values if pass 1 swizzles:
;;	0	+2K	512	+2K	1024	+2K	1536	+2K
;;	1	...
;;	...
;;	511	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+2K	256	+2K	512	+2K	768	+2K
;;	1	...
;;	...
;;	255	...
;;	1024	...
;;	...

;; Do 256 eight_reals_fft operations
;; Do 256 four_complex_djbfft operations
;;	distance between fft data elements is 256
;;	do 256 macros each processing 16 data values

	mov	rdi, xsincos_complex	;; Ptr to the sin/cos data
	mov	rdx, sincos3		;; Ptr to real sin/cos data
	loops_init 256
b1b:	r4_fh2cl_eight_reals_four_complex_djbfft rsi, 64, 2*dist128, rdi, rdx
	bump	rdi, XMM_SCD2		;; Next sin/cos data
	bump	rdx, XMM_HSCD2		;; Next sin/cos data
	loops	128, b1b		;; Test loop counter
	bump	rsi, -128*64+dist128	;; Next source pointer
	loops	2, b1b			;; Test loop counter
	bump	rsi, -2*dist128		;; Restore source pointer
	bump	rdx, -256*XMM_HSCD2	;; Restore sin/cos pointer

;; Do FFT levels 3,4
;;
;; On input the 64-byte cache lines hold these data values:
;;	0	+2K	256	+2K	512	+2K	768	+2K
;;	1	...
;;	...
;;	255	...
;;	1024	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+2K	64	+2K	128	+2K	192	+2K
;;	1	...
;;	...
;;	63	...
;;	256	...
;;	...

;; Do 64 eight_reals_fft operations
;; Do 192 four_complex_djbfft operations
;;	distance between fft data elements is 64
;;	do 64 macros each processing 32 data values

	mov	rcx, DATA_PREFETCH	;; Prefetch pointer
	xtouch	[rcx]			;; Preload first TLB
	loops_init_prefetch 128, 64, 2, rcx
	xtouch	[rcx+4096-64]		;; Preload the TLBs
	r4_h4cl_eight_reals_four_complex_djbfft_preload
b4b:	r4_h4cl_eight_reals_four_complex_djbfft rsi, 64, 64*64, dist128, rdi, rdx
	bump	rdi, XMM_SCD2		;; Next sin/cos data
	bump	rdx, 4*XMM_HSCD2	;; Next sin/cos data
	loops	64, b4b			;; Test loop counter
	bump	rsi, -64*64+2*dist128	;; Next source pointer
	bump	rdi, -64*XMM_SCD2	;; Restore sin/cos ptr
	bump	rdx, -64*4*XMM_HSCD2	;; Restore sin/cos ptr

;; Do 256 four_complex_djbfft operations
;;	distance between fft data elements is 64
;;	do 64 macros each processing 32 data values

	loops_reset			;; 1 more iter of 64
	r4_x4cl_four_complex_djbfft_preload
b5b:	r4_x4cl_four_complex_djbfft rsi, 64, 64*64, dist128, rdi
	bump	rdi, XMM_SCD2		;; Next sin/cos data
	loops	64, b5b			;; Test inner loop counter
	bump	rsi, -64*64+2*dist128	;; Next source pointer
	bump	rdi, -64*XMM_SCD2	;; Restore sin/cos ptr
	loops	2, b5b			;; Test loop counter
	bump	rsi, -2*2*dist128	;; Next source pointer

;; Do FFT levels 5,6
;;
;; On input the 64-byte cache lines hold these data values:
;;	0	+2K	64	+2K	128	+2K	192	+2K
;;	1	...
;;	...
;;	63	...
;;	256	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+2K	16	+2K	32	+2K	48	+2K
;;	1	...
;;	...
;;	15	...
;;	64	...
;;	...

;; Do 16 eight_reals_fft operations
;; Do 48 four_complex_djbfft operations
;;	distance between fft data elements is 16
;;	do 16 macros each processing 32 data values

	loops_init_prefetch 128, 64, 2, rcx
	xtouch	[rcx+4096-64]		;; Preload the TLB
	r4_h4cl_eight_reals_four_complex_djbfft_preload
b6b:	r4_h4cl_eight_reals_four_complex_djbfft rsi, 64, 16*64, 32*64, rdi, rdx
	bump	rdi, 4*XMM_SCD2		;; Next sin/cos data
	bump	rdx, 16*XMM_HSCD2	;; Next sin/cos data
	loops	16, b6b			;; Test loop counter
	bump	rsi, -16*64+64*64	;; Next source pointer
	bump	rdi, -16*4*XMM_SCD2	;; Restore sin/cos ptr
	bump	rdx, -16*16*XMM_HSCD2	;; Restore sin/cos ptr

;; Do 384 four_complex_djbfft operations
;;	distance between fft data elements is 16
;;	do 96 macros each processing 32 data values

	loops_reset			;; 4 iters of 1 or 2 of 16
	r4_x4cl_four_complex_djbfft_preload
b7b:	r4_x4cl_four_complex_djbfft rsi, 64, 16*64, 32*64, rdi
	bump	rdi, 4*XMM_SCD2		;; Next sin/cos data
	loops	16, b7b			;; Test inner loop counter
	bump	rsi, -16*64+64*64	;; Next source pointer
	bump	rdi, -16*4*XMM_SCD2	;; Restore sin/cos ptr
	loops	2, b7b			;; Test loop counter
	bump	rsi, -2*64*64+dist128	;; Next source pointer
	loops	4, b7b			;; 4 iterations
	bump	rsi, -4*dist128		;; Next source pointer
	bump	rcx, 128		;; Skip pad bytes

;; Do FFT levels 7,8
;;
;; On input the 64-byte cache lines hold these data values:
;;	0	+2K	16	+2K	32	+2K	48	+2K
;;	1	...
;;	...
;;	15	...
;;	64	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+2K	4	+2K	8	+2K	12	+2K
;;	1	...
;;	...
;;	3	...
;;	16	...
;;	...

;; Do 4 eight_reals_fft operations
;; Do 12 four_complex_djbfft operations
;;	distance between fft data elements is 4
;;	do 4 macros each processing 32 data values

	loops_init_prefetch 128, 64, 2, rcx
	xtouch	[rcx+4096-64]		;; Preload the TLB
	r4_h4cl_eight_reals_four_complex_djbfft_preload
b8b:	r4_h4cl_eight_reals_four_complex_djbfft rsi, 64, 4*64, 8*64, rdi, rdx
	bump	rdi, 16*XMM_SCD2	;; Next sin/cos data
	bump	rdx, 64*XMM_HSCD2	;; Next sin/cos data
	loops	4, b8b			;; Test loop counter
	bump	rsi, -4*64+16*64	;; Next source pointer
	bump	rdi, -4*16*XMM_SCD2	;; Restore sin/cos ptr
	bump	rdx, -4*64*XMM_HSCD2	;; Restore sin/cos ptr

;; Do 496 four_complex_djbfft operations
;;	distance between fft data elements is 4
;;	do 124 macros each processing 32 data values

	loops_reset			;; 4 iters of 7 or 8 of 4
	r4_x4cl_four_complex_djbfft_preload
b9b:	r4_x4cl_four_complex_djbfft rsi, 64, 4*64, 8*64, rdi
	bump	rdi, 16*XMM_SCD2	;; Next sin/cos data
	loops	4, b9b			;; Test loop counter
	bump	rsi, -4*64+16*64	;; Next source pointer
	bump	rdi, -4*16*XMM_SCD2	;; Restore sin/cos ptr
	loops	8, b9b			;; Test middle loop counter
	bump	rsi, -8*16*64+dist128	;; Next source pointer
	loops	4, b9b			;; 4 iterations
	bump	rsi, -4*dist128		;; Next source pointer

;; Do FFT levels 9,10
;;
;; On input the 64-byte cache lines hold these data values:
;;	0	+2K	4	+2K	8	+2K	12	+2K
;;	1	...
;;	...
;;	3	...
;;	16	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+2K	2	+2K	4	+2K	6	+2K
;;	1	...
;;	8	...
;;	...

;; Do 1 eight_reals_fft operations
;; Do 3 four_complex_fft operations
;;	distance between fft data elements is 1

;; Do 508 four_complex_fft operations
;;	distance between fft data elements is 1
;;	do 127 macros each processing 32 data values

;; Execute the proper middle step

	cmp	ffttype, 2
	jl	xpass2_10_levels_real_1
	jg	xpass2_10_levels_real_3

xpass2_10_levels_real_2:
	r4_h4cl_eight_reals_four_complex_with_square rsi, 4*64, 64, 2*64
	loops_init_prefetch 128, 64, 2, rcx, 1, 32
	xtouch	[rcx+4096-64]		;; Preload the TLB
	r4_x4cl_four_complex_with_square_preload
baa:	r4_x4cl_four_complex_with_square rsi, 4*64, 64, 2*64
	loops	32, baa			;; Test loop counter
	bump	rsi, -32*4*64+dist128	;; Next source pointer
	loops	4, baa			;; Test loop counter
	bump	rsi, -4*dist128		;; Next source pointer
	bump	rcx, 128		;; Skip pad bytes
	jmp	xpass2_10_real_unfft

xpass2_10_levels_real_1:
	r4_h4cl_eight_reals_four_complex_fft_final rsi, 4*64, 64, 2*64
	loops_init_prefetch 128, 128, 1, rcx, 1, 32
bab0:	xtouch	[rcx+4096-64]		;; Preload the TLBs
bab:	r4_x4cl_four_complex_fft_final rsi, 4*64, 64, 2*64
	loops	32, bab			;; Test loop counter
	bump	rsi, -32*4*64+dist128	;; Next source pointer
	loops	4, bab0			;; Test loop counter
	jmp	xpass2_10_real_done

xpass2_10_levels_real_3:
	mov	rbp, DIST_TO_MULSRCARG
	r4_h4cl_eight_reals_four_complex_with_mult rsi, 4*64, 64, 2*64
	loops_init_prefetch 128, 64, 2, rcx, 1, 32
	xtouch	[rcx+4096-64]		;; Preload the TLB
bac:	r4_x4cl_four_complex_with_mult rsi, 4*64, 64, 2*64
	loops	32, bac			;; Test loop counter
	bump	rsi, -32*4*64+dist128	;; Next source pointer
	loops	4, bac			;; Test loop counter
	bump	rsi, -4*dist128		;; Next source pointer
	bump	rcx, 128		;; Skip pad bytes
	jmp	xpass2_10_real_unfft

xpass2_10_levels_real_4:
	mov	rbp, DIST_TO_MULSRCARG
	mov	rcx, DATA_PREFETCH	;; Load prefetch pointer
	xtouch	[rcx]			;; Preload the first TLB
	r4_h4cl_eight_reals_four_complex_with_mulf rsi, 4*64, 64, 2*64
	loops_init_prefetch 128, 128, 1, rcx, 1, 32
bad0:	xtouch	[rcx+4096-64]		;; Preload the TLBs
bad:	r4_x4cl_four_complex_with_mulf rsi, 4*64, 64, 2*64
	loops	32, bad			;; Test loop counter
	bump	rsi, -32*4*64+dist128	;; Next source pointer
	loops	4, bad0			;; Test loop counter
	bump	rsi, -4*dist128		;; Next source pointer
	bump	rcx, 2*128		;; Skip pad bytes

;; Do inverse FFT levels 7,8
;; On input the 64-byte cache lines hold these data values:
;;	0	+2K	2	+2K	4	+2K	6	+2K
;;	1	...
;;	8	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+2K	8	+2K	16	+2K	24	+2K
;;	1	...
;;	...
;;	7	...
;;	32	...
;;	...

;; Do 4 eight_reals_unfft operations
;; Do 4 four_complex_djbunfft operations
;;	distance between fft data elements is 4
;;	do 2 macros processing 32 data values

xpass2_10_real_unfft:
	mov	rdi, xsincos_complex	;; Load sin/cos pointers
	bump	rdi, 256*XMM_SCD2
	mov	rdx, sincos3
	loops_init_prefetch 128, 64, 2, rcx
	xtouch	[rcx+4096-64]		;; Load the prefetch TLB
c8b:	r4_h4cl_eight_reals_four_complex_djbunfft rsi, 64, 2*64, 4*64, rdi, 2*16*XMM_SCD2, rdx, 2*64*XMM_HSCD2
	bump	rdi, 16*XMM_SCD2	;; Next sin/cos data
	bump	rdx, 64*XMM_HSCD2	;; Next sin/cos data
	loops	2, c8b			;; Test loop counter
	bump	rsi, -2*64+8*64		;; Next source pointer
	bump	rdi, -2*16*XMM_SCD2	;; Restore sin/cos ptr
	bump	rdx, -2*64*XMM_HSCD2	;; Restore sin/cos ptr

;; Do 504 four_complex_djbunfft operations
;;	distance between fft data elements is 4
;;	do 126 macros each processing 32 data values

	loops_reset			;; 4 iters of 15 or 16 of 2
	r4_x4cl_four_complex_djbunfft_preload
c9b:	r4_x4cl_four_complex_djbunfft rsi, 64, 2*64, 4*64, rdi, 2*16*XMM_SCD2
	bump	rdi, 16*XMM_SCD2	;; Next sine/cosine pointer
	loops	2, c9b			;; Test loop counter
	bump	rsi, -2*64+8*64		;; Next source pointer
	bump	rdi, -2*16*XMM_SCD2	;; Restore sin/cos ptr
	loops	16, c9b			;; Test loop counter
	bump	rsi, -16*8*64+dist128	;; Next source pointer
	loops	4, c9b			;; Test loop counter
	bump	rsi, -4*dist128		;; Next source pointer

;; Do inverse FFT levels 5,6
;; On input the 64-byte cache lines hold these data values:
;;	0	+2K	8	+2K	16	+2K	24	+2K
;;	1	...
;;	...
;;	7	...
;;	32	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+2K	32	+2K	64	+2K	96	+2K
;;	1	...
;;	...
;;	31	...
;;	128	...
;;	...

;; Do 16 eight_reals_unfft operations
;; Do 16 four_complex_djbunfft operations
;;	distance between fft data elements is 16
;;	do 8 macros processing 32 data values

	loops_init_prefetch 128, 64, 2, rcx
	xtouch	[rcx+4096-64]		;; Load the prefetch TLB
c6b:	r4_h4cl_eight_reals_four_complex_djbunfft rsi, 64, 8*64, 16*64, rdi, 8*4*XMM_SCD2, rdx, 8*16*XMM_HSCD2
	bump	rdi, 4*XMM_SCD2		;; Next sin/cos data
	bump	rdx, 16*XMM_HSCD2	;; Next sin/cos data
	loops	8, c6b			;; Test inner loop counter
	bump	rsi, -8*64+32*64	;; Next source pointer
	bump	rdi, -8*4*XMM_SCD2	;; Restore sin/cos ptr
	bump	rdx, -8*16*XMM_HSCD2	;; Restore sin/cos ptr

;; Do 480 four_complex_djbunfft operations
;;	distance between fft data elements is 16
;;	do 120 macros each processing 32 data values

	loops_reset			;; 4 iters of 3 or 4 iters of 8
	r4_x4cl_four_complex_djbunfft_preload
c7b:	r4_x4cl_four_complex_djbunfft rsi, 64, 8*64, 16*64, rdi, 8*4*XMM_SCD2
	bump	rdi, 4*XMM_SCD2		;; Next sin/cos data
	loops	8, c7b			;; Test inner loop counter
	bump	rsi, -8*64+32*64	;; Next source pointer
	bump	rdi, -8*4*XMM_SCD2	;; Restore sin/cos ptr
	loops	4, c7b			;; Test loop counter
	bump	rsi, -4*32*64+dist128	;; Next source pointer
	loops	4, c7b			;; 4 iterations
	bump	rsi, -4*dist128		;; Next source pointer
	bump	rcx, 128		;; Skip pad bytes

;; Do inverse FFT levels 3,4
;; On input the 64-byte cache lines hold these data values:
;;	0	+2K	32	+2K	64	+2K	96	+2K
;;	1	...
;;	...
;;	31	...
;;	128	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+2K	128	+2K	256	+2K	384	+2K
;;	1	...
;;	...
;;	127	...
;;	512	...
;;	...

;; Do 64 eight_reals_unfft operations
;; Do 64 four_complex_djbunfft operations
;;	distance between fft data elements is 64
;;	do 32 macros processing 32 data values

	loops_init_prefetch 128, 64, 2, rcx
	xtouch	[rcx+4096-64]		;; Load the prefetch TLB
c4b:	r4_h4cl_eight_reals_four_complex_djbunfft rsi, 64, 32*64, 64*64, rdi, 32*XMM_SCD2, rdx, 128*XMM_HSCD2
	bump	rdi, XMM_SCD2		;; Next sin/cos data
	bump	rdx, 4*XMM_HSCD2	;; Next sin/cos data
	loops	32, c4b			;; Test inner loop counter
	bump	rsi, -32*64+dist128	;; Next source pointer
	bump	rdi, -32*XMM_SCD2	;; Restore sin/cos ptr
	bump	rdx, -32*4*XMM_HSCD2	;; Restore sin/cos ptr

;; Do 384 four_complex_djbunfft operations
;;	distance between fft data elements is 64
;;	do 96 macros each processing 32 data values

	loops_reset			;; 3 more iters of 32
	r4_x4cl_four_complex_djbunfft_preload
c5b:	r4_x4cl_four_complex_djbunfft rsi, 64, 32*64, 64*64, rdi, 32*XMM_SCD2
	bump	rdi, XMM_SCD2		;; Next sin/cos data
	loops	32, c5b			;; Test inner loop counter
	bump	rsi, -32*64+dist128	;; Next source pointer
	bump	rdi, -32*XMM_SCD2	;; Restore sin/cos ptr
	loops	4, c5b			;; Test loop counter
	bump	rsi, -4*dist128		;; Next source pointer

;; Do inverse FFT level 1,2
;; On input the 64-byte cache lines hold these data values:
;;	0	+2K	128	+2K	256	+2K	384	+2K
;;	1	...
;;	...
;;	127	...
;;	512	...
;;	...
;; On output the 64-byte cache lines hold these data values if pass 1 swizzles:
;;	0	+2K	512	+2K	1024	+2K	1536	+2K
;;	1	...
;;	...
;;	511
;; On output the 64-byte cache lines hold these data values if pass 2 swizzles:
;;	0	+512	1024	+512	2048	+512	3072	+512
;;	1	...
;;	...
;;	511

;; Do 256 eight_reals_unfft operations
;; Do 256 four_complex_djbunfft operations
;;	distance between fft data elements is 256
;;	do 128 macros processing 32 data values

	mov	rdi, xsincos_complex	;; Load sin/cos pointers
	loops_init_prefetch 128, 64, 4, rcx
	xtouch	[rcx+4096-64]		;; Load the prefetch TLB
c1b:	r4_h4cl_eight_reals_four_complex_djbunfft rsi, 64, dist128, 2*dist128, rdi, 128*XMM_SCD2, rdx, 128*XMM_HSCD2
	bump	rdi, XMM_SCD2		;; Next sin/cos data
	bump	rdx, XMM_HSCD2		;; Next sin/cos data
	loops	128, c1b		;; Test loop counter

xpass2_10_real_done:
	end_timer 2
	ENDM

; *************** radix4-pass2-10-levels-complex macro ******************
; This macro takes 2 groups of 1024 complex values and
; performs the final 10 levels of a standard mostly radix-4 FFT, squares the
; results, and does the first 10 levels of the inverse FFT.

r4_pass2_10_levels_complex MACRO
	LOCAL	b1b, b3b, b4b, b5b, b6a, b6b0, b6b, b6c, b6d0, b6d
	LOCAL	c1b, c3b, c4b, c5b
	LOCAL	xpass2_10_levels_complex_1, xpass2_10_levels_complex_2
	LOCAL	xpass2_10_levels_complex_3, xpass2_10_levels_complex_4
	LOCAL	xpass2_10_levels_complex_unfft, xpass2_10_levels_complex_done

;; Type 4 FFTs skip the forward FFT process

	mov	rsi, DATA_ADDR		;; Load address of FFT data
	mov	rbx, DIST_TO_FFTSRCARG
	cmp	ffttype, 4
	je	xpass2_10_levels_complex_4

;; Do FFT levels 1,2
;;
;; We swizzle the data so that one set of data is in the high word of
;; an xmm register, the other set is in the low word.  This means
;; a distance of 2048 data values.
;;
;; On input the 64-byte cache lines hold these data values if pass 2 swizzles:
;;	0	+512	1024	+512	2048	+512	3072	+512
;;	1	...
;;	...
;;	511
;; On input the 64-byte cache lines hold these data values if pass 1 swizzles:
;;	0	+2K	512	+2K	1024	+2K	1536	+2K
;;	1	...
;;	...
;;	511	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+2K	256	+2K	512	+2K	768	+2K
;;	1	...
;;	...
;;	255	...
;;	1024	...
;;	...

;; Do 512 four_complex_fft operations
;;	distance between fft data elements is 256
;;	do 256 macros each processing 16 data values

	start_timer 4
	mov	rdi, xsincos_complex	;; The sin/cos data
	loops_init 256			;; 256 iterations
	r4_f2cl_four_complex_djbfft_preload
b1b:	xxprefetch [rdi+XMM_SCD2]	;; Prefetch for next pass
	r4_f2cl_four_complex_djbfft rsi, 64, 2*dist128, rdi
	bump	rdi, XMM_SCD2		;; Next sin/cos data
	loops	128, b1b		;; Test inner loop counter
	bump	rsi, -128*64+dist128	;; Next source pointer
	loops	2, b1b			;; Test inner loop counter
	bump	rsi, -2*dist128		;; Restore source pointer
	end_timer 4

;; Do FFT levels 3,4
;;
;; On input the 64-byte cache lines hold these data values:
;;	0	+2K	256	+2K	512	+2K	768	+2K
;;	1	...
;;	...
;;	255	...
;;	1024	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+2K	64	+2K	128	+2K	192	+2K
;;	1	...
;;	...
;;	63	...
;;	256	...
;;	...
;;
;; Do 512 four_complex_fft operations
;;	distance between fft data elements is 64
;;	do 128 macros each processing 32 data values

;; Do FFT levels 5,6
;;
;; On input the 64-byte cache lines hold these data values:
;;	0	+2K	64	+2K	128	+2K	192	+2K
;;	1	...
;;	...
;;	63	...
;;	256	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+2K	16	+2K	32	+2K	48	+2K
;;	1	...
;;	...
;;	15	...
;;	64	...
;;	...
;;
;; Do 512 four_complex_fft operations
;;	distance between fft data elements is 16
;;	do 128 macros each processing 32 data values

;; Do FFT levels 7,8
;;
;; On input the 64-byte cache lines hold these data values:
;;	0	+2K	16	+2K	32	+2K	48	+2K
;;	1	...
;;	...
;;	15	...
;;	64	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+2K	4	+2K	8	+2K	12	+2K
;;	1	...
;;	...
;;	3	...
;;	16	...
;;	...
;;
;; Do 512 four_complex_fft operations
;;	distance between fft data elements is 4
;;	do 128 macros each processing 32 data values

;; Do this 64 cache lines at a time to maximize accessing data
;; while it is in the L1 cache.

	start_timer 6
	mov	rcx, DATA_PREFETCH	;; Prefetch pointer (we need to prefetch 
					;; 512 cache lines + 6 pad cache lines)
	xtouch	[rcx]			;; Load first prefetch TLB
	mov	rbx, rdi
	mov	rbp, rdi
	loops_init_prefetch 128, 64, 2, rcx ;; 2*4 iters of 64 cache lines
	r4_x4cl_four_complex_djbfft_preload

	xtouch	[rcx+4096-64]		;; Load prefetch TLBs
	xtouch	[rcx+2*4096-64]
	xtouch	[rcx+3*4096-64]

b3b:	xxprefetch [rdi+4*XMM_SCD2]	;; Prefetch for next pass
	r4_x4cl_four_complex_djbfft rsi, 4*64, 64*64, dist128, rdi
	bump	rdi, 4*XMM_SCD2		;; Next sine/cosine pointer
	loops	16, b3b			;; Loop 16 times
	bump	rsi, -16*4*64		;; Restore source pointer
	bump	rdi, -16*4*XMM_SCD2	;; Restore sine/cosine pointer
	bump	rcx, -8*64+4096		;; Prefetch from next 4KB page
	loops_undo 16			;; Back up loop counter

b4b:	r4_x4cl_four_complex_djbfft rsi, 4*64, 16*64, 32*64, rbx
	bump	rbx, 4*4*XMM_SCD2	;; Next sine/cosine pointer
	loops	4, b4b			;; Loop 4 times
	bump	rsi, -4*4*64+64*64	;; Next source pointer
	bump	rbx, -4*4*4*XMM_SCD2	;; Restore sine/cosine pointer
	loops	2, b4b			;; Loop twice
	bump	rsi, -2*64*64+dist128	;; Next source pointer
	loops	2, b4b			;; Loop twice
	bump	rsi, -2*dist128		;; Restore source pointer
	bump	rcx, -8*64+4096+128	;; Prefetch from next 4KB page
	loops_undo 2*2*4		;; Back up loop counter

b5b:	r4_x4cl_four_complex_djbfft rsi, 16*64, 4*64, 8*64, rbp
	loops	8, b5b			;; Loop 8 times
	bump	rsi, -8*16*64+dist128	;; Next source pointer
	loops	2, b5b			;; Loop twice
	bump	rcx, -(8192+128)	;; Back to prefetching from 1st 4KB page

	bump	rsi, -2*dist128+64	;; Next source pointer
	bump	rdi, XMM_SCD2		;; Next sine/cosine pointer
	bump	rbx, 4*XMM_SCD2		;; Next sine/cosine pointer
	bump	rbp, 16*XMM_SCD2	;; Next sine/cosine pointer
	loops	4, b3b			;; Loop 4 times
	bump	rsi, -4*64+2*dist128	;; Next source pointer
	bump	rdi, -4*XMM_SCD2	;; Restore sine/cosine pointer
	bump	rbx, -4*4*XMM_SCD2	;; Restore sine/cosine pointer
	bump	rbp, -4*16*XMM_SCD2	;; Restore sine/cosine pointer
	loops	2, b3b			;; Loop twice
	bump	rsi, -2*2*dist128	;; Restore source pointer
	bump	rcx, 8192+128		;; Move prefetch pointer along
	end_timer 6

;; Do FFT levels 9,10 as well as inverse FFT levels 9,10
;;
;; On input the 64-byte cache lines hold these data values:
;;	0	+2K	4	+2K	8	+2K	12	+2K
;;	1	...
;;	...
;;	3	...
;;	16	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+2K	2	+2K	4	+2K	6	+2K
;;	1	...
;;	8	...
;;	...

;; Do 512 four_complex_fft operations
;;	distance between fft data elements is 1
;;	do 128 macros each processing 32 data values

;; Execute the right middle step

	cmp	ffttype, 2
	jl	xpass2_10_levels_complex_1
	jg	xpass2_10_levels_complex_3

xpass2_10_levels_complex_2:
	start_timer 9
	loops_init_prefetch 128, 64, 2, rcx
	xtouch	[rcx+4096-64]		;; Preload the TLB
	r4_x4cl_four_complex_with_square_preload
b6a:	r4_x4cl_four_complex_with_square rsi, 4*64, 64, 2*64
	loops	32, b6a			;; Loop 32 times
	bump	rsi, -32*4*64+dist128	;; Next source pointer
	loops	4, b6a			;; Loop 4 times
	bump	rsi, -4*dist128		;; Restore source pointer
	bump	rcx, 128		;; Skip pad bytes
	end_timer 9
	jmp	xpass2_10_levels_complex_unfft

xpass2_10_levels_complex_1:
	start_timer 9
	loops_init_prefetch 128, 128, 1, rcx
b6b0:	xtouch	[rcx+4096-64]		;; Preload the TLBs
b6b:	r4_x4cl_four_complex_fft_final rsi, 4*64, 64, 2*64
	loops	32, b6b			;; Loop 32 times
	bump	rsi, -32*4*64+dist128	;; Next source pointer
	loops	4, b6b0			;; Loop 4 times
	end_timer 9
	jmp	xpass2_10_levels_complex_done

xpass2_10_levels_complex_3:
	start_timer 9
	mov	rbp, DIST_TO_MULSRCARG
	loops_init_prefetch 128, 64, 2, rcx
	xtouch	[rcx+4096-64]		;; Preload the TLB
b6c:	r4_x4cl_four_complex_with_mult rsi, 4*64, 64, 2*64
	loops	32, b6c			;; Loop 32 times
	bump	rsi, -32*4*64+dist128	;; Next source pointer
	loops	4, b6c			;; Loop 4 times
	bump	rsi, -4*dist128		;; Restore source pointer
	bump	rcx, 128		;; Skip pad bytes
	end_timer 9
	jmp	xpass2_10_levels_complex_unfft

xpass2_10_levels_complex_4:
	start_timer 9
	mov	rcx, DATA_PREFETCH	;; Prefetch pointer
	xtouch	[rcx]			;; Load first prefetch TLB
	mov	rbp, DIST_TO_MULSRCARG
	loops_init_prefetch 128, 128, 1, rcx
b6d0:	xtouch	[rcx+4096-64]		;; Preload the TLBs
b6d:	r4_x4cl_four_complex_with_mulf rsi, 4*64, 64, 2*64
	loops	32, b6d			;; Loop 32 times
	bump	rsi, -32*4*64+dist128	;; Next source pointer
	loops	4, b6d0			;; Loop 4 times
	bump	rsi, -4*dist128		;; Restore source pointer
	bump	rcx, 2*128		;; Skip pad bytes
	end_timer 9

;; Do inverse FFT levels 7,8
;; On input the 64-byte cache lines hold these data values:
;;	0	+2K	2	+2K	4	+2K	6	+2K
;;	1	...
;;	8	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+2K	8	+2K	16	+2K	24	+2K
;;	1	...
;;	...
;;	7	...
;;	32	...
;;	...
;;
;; Do 512 four_complex_unfft operations
;;	distance between fft data elements is 4
;;	do 128 macros each processing 32 data values

;; Do inverse FFT levels 5,6
;; On input the 64-byte cache lines hold these data values:
;;	0	+2K	8	+2K	16	+2K	24	+2K
;;	1	...
;;	...
;;	7	...
;;	32	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+2K	32	+2K	64	+2K	96	+2K
;;	1	...
;;	...
;;	31	...
;;	128	...
;;	...
;;
;; Do 512 four_complex_unfft operations
;;	distance between fft data elements is 16
;;	do 128 macros each processing 32 data values

;; Do inverse FFT levels 3,4
;; On input the 64-byte cache lines hold these data values:
;;	0	+2K	32	+2K	64	+2K	96	+2K
;;	1	...
;;	...
;;	31	...
;;	128	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+2K	128	+2K	256	+2K	384	+2K
;;	1	...
;;	...
;;	127	...
;;	512	...
;;	...
;;
;; Do 512 four_complex_unfft operations
;;	distance between fft data elements is 64
;;	do 128 macros each processing 32 data values

;; Do this 64 cache lines at a time to maximize accessing data
;; while it is in the L1 cache.

xpass2_10_levels_complex_unfft:
	start_timer 10
	mov	rdi, xsincos_complex	;; Load sin/cos pointers
	bump	rdi, 256*XMM_SCD2
	mov	rbx, rdi
	mov	rbp, rdi
	loops_init_prefetch 128, 64, 2, rcx ;; 4*2 iters of 64 cache lines
	r4_x4cl_four_complex_djbunfft_preload

	xtouch	[rcx+4096-64]		;; Load 3 prefetch TLBs
	xtouch	[rcx+2*4096-64]
	xtouch	[rcx+3*4096-64]

c5b:	r4_x4cl_four_complex_djbunfft rsi, 8*64, 2*64, 4*64, rbp, 2*16*XMM_SCD2
	loops	16, c5b			;; Loop 16 times
	bump	rsi, -16*8*64		;; Restore source pointer
	bump	rcx, -8*64+4096		;; Prefetch from next 4KB page
	loops_undo 16			;; Restore loop counter

c4b:	r4_x4cl_four_complex_djbunfft rsi, 2*64, 8*64, 16*64, rbx, 8*4*XMM_SCD2
	bump	rbx, 2*4*XMM_SCD2	;; Next sine/cosine pointer
	loops	4, c4b			;; Loop 4 times
	bump	rsi, -4*2*64+32*64	;; Next source pointer
	bump	rbx, -4*2*4*XMM_SCD2	;; Restore sine/cosine pointer
	loops	4, c4b			;; Loop 4 times
	bump	rsi, -4*32*64		;; Restore source pointer
	bump	rcx, -8*64+4096+128	;; Prefetch from next 4KB page
	loops_undo 4*4			;; Restore loop counter

c3b:	r4_x4cl_four_complex_djbunfft rsi, 2*64, 32*64, 64*64, rdi, 32*1*XMM_SCD2
	bump	rdi, 2*XMM_SCD2		;; Next sine/cosine pointer
	loops	16, c3b			;; Loop 16 times
	bump	rcx, -(8192+128)	;; Back to prefetching from 1st 4KB page
	bump	rdi, -16*2*XMM_SCD2	;; Restore sine/cosine pointer

	bump	rsi, -16*2*64+64	;; Next source pointer
	bump	rdi, XMM_SCD2		;; Next sine/cosine pointer
	bump	rbx, 4*XMM_SCD2		;; Next sine/cosine pointer
	bump	rbp, 16*XMM_SCD2	;; Next sine/cosine pointer
	loops	2, c5b			;; Loop twice
	bump	rsi, -2*64+dist128	;; Next source pointer
	bump	rdi, -2*XMM_SCD2	;; Restore sine/cosine pointer
	bump	rbx, -2*4*XMM_SCD2	;; Restore sine/cosine pointer
	bump	rbp, -2*16*XMM_SCD2	;; Restore sine/cosine pointer
	loops	4, c5b			;; Loop 4 times
	bump	rsi, -4*dist128		;; Restore source pointer
	bump	rcx, 8192+128		;; Move prefetch pointer along
	end_timer 10

;; Do inverse FFT levels 1,2
;; On input the 64-byte cache lines hold these data values:
;;	0	+2K	128	+2K	256	+2K	384	+2K
;;	1	...
;;	...
;;	127	...
;;	512	...
;;	...
;; On output the 64-byte cache lines hold these data values if pass 1 swizzles:
;;	0	+2K	512	+2K	1024	+2K	1536	+2K
;;	1	...
;;	...
;;	511
;; On output the 64-byte cache lines hold these data values if pass 2 swizzles:
;;	0	+512	1024	+512	2048	+512	3072	+512
;;	1	...
;;	...
;;	511

;; Do 512 four_complex_unfft operations
;;	distance between fft data elements is 256
;;	do 128 macros each processing 32 data values

	start_timer 14
	mov	rdi, xsincos_complex	;; Load sin/cos pointer
	loops_init_prefetch 128, 64, 2, rcx ;; 128 iters
	xtouch	[rcx+4096-64]		;; Load prefetch TLBs
	r4_x4cl_four_complex_djbunfft_preload
c1b:	r4_x4cl_four_complex_djbunfft rsi, 64, dist128, 2*dist128, rdi, 128*XMM_SCD2
	bump	rdi, XMM_SCD2		;; Next set of sin/cos data
	loops	128, c1b		;; Test loop counter
	end_timer 14

xpass2_10_levels_complex_done:
	ENDM


; *************** radix4-pass2-1280-levels-real macro ******************
; This macro takes 2560 real values and performs the final 11.322 levels of
; the FFT.  Independently, this macro also takes 1280 complex values and
; performs the final 10.322 levels of the FFT.  The results are squared, and
; the first 10.322/11.322 levels of the inverse FFT are done.

r4_pass2_1280_levels_real MACRO
	LOCAL	b1b, b4b, b5b, b6b, b7b, b8b, b9b, baa, bab0, bab, bac, bad0, bad
	LOCAL	c1b, c4b, c5b, c6b, c7b, c8b, c9b
	LOCAL	xpass2_1280_levels_real_1, xpass2_1280_levels_real_2
	LOCAL	xpass2_1280_levels_real_3, xpass2_1280_levels_real_4
	LOCAL	xpass2_1280_real_unfft, xpass2_1280_real_done

;; We switch to the same format used in one-pass FFTs (that is, the
;; high word of xmm registers being 1 greater than the low word):

;; Type 4 FFTs skip the forward FFT process

	start_timer 2
	mov	rsi, DATA_ADDR		;; Load source address
	mov	rbx, DIST_TO_FFTSRCARG
	cmp	ffttype, 4
	je	xpass2_1280_levels_real_4

;; Do FFT level 1,2 (actually 2.322 levels)
;;
;; On input the 64-byte cache lines hold these data values if pass 2 swizzles:
;;	0	+640	1280	+640	2560	+640	3840	+640
;;	1	...
;;	...
;;	639
;; On input the 64-byte cache lines hold these data values if pass 1 swizzles:
;;	0	+2560	640	+2560	1280	+2560	1920	+2560
;;	1	...
;;	...
;;	639	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+2560	128	+2560	256	+2560	384	+2560
;;	1	...
;;	...
;;	127	...
;;	512	...
;;	...

;; Do 256 ten_reals_fft operations
;; Do 256 five_complex_djbfft operations
;;	distance between fft data elements is 256
;;	do 128 macros each processing 40 data values

	mov	rdi, xsincos_complex	;; Ptr to the sin/cos data
	mov	rdx, sincos3		;; Ptr to real sin/cos data
	loops_init 128
b1b:	r5_fh5cl_ten_reals_five_complex_djbfft rsi, 64, dist128, rdi, 128*XMM_SCD2, rdx, 128*XMM_HSCD2
	bump	rdi, XMM_SCD2		;; Next sin/cos data
	bump	rdx, XMM_HSCD2		;; Next sin/cos data
	loops	128, b1b		;; Test loop counter
	bump	rsi, -128*64		;; Restore source pointer
	bump	rdi, 128*XMM_SCD2	;; Next sin/cos ptr
	bump	rdx, 128*XMM_HSCD2	;; Next sin/cos ptr

;; Do FFT levels 3,4
;;
;; On input the 64-byte cache lines hold these data values:
;;	0	+2560	128	+2560	256	+2560	384	+2560
;;	1	...
;;	...
;;	127	...
;;	512	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+2560	64	+2560	128	+2560	192	+2560
;;	1	...
;;	...
;;	63	...
;;	256	...
;;	...

;; Do 64 eight_reals_fft operations
;; Do 64 four_complex_djbfft operations
;;	distance between fft data elements is 64
;;	do 64 macros each processing 16 data values

	mov	rcx, DATA_PREFETCH	;; Prefetch pointer
	xtouch	[rcx]			;; Load 5 prefetch TLBs
	xtouch	[rcx+4096-64]
	xtouch	[rcx+2*4096-64]
	xtouch	[rcx+3*4096-64]
	xtouch	[rcx+4*4096-64]
	xtouch	[rcx+5*4096-64]
	loops_init_prefetch 320, 64, 4, rcx
b4b:	r4_h2cl_eight_reals_four_complex_djbfft rsi, 64, 64*64, rdi, rdx
	bump	rdi, XMM_SCD2		;; Next sin/cos data
	bump	rdx, XMM_HSCD2		;; Next sin/cos data
	loops	64, b4b			;; Test loop counter
	bump	rsi, -64*64+dist128	;; Next source pointer
	bump	rdi, -64*XMM_SCD2	;; Restore sin/cos ptr
	bump	rdx, -64*XMM_HSCD2	;; Restore sin/cos ptr

;; Do 512 four_complex_djbfft operations
;;	distance between fft data elements is 64
;;	do 256 macros each processing 16 data values

	loops_reset			;; 4 more iters of 64
	r4_nf2cl_four_complex_djbfft_preload
b5b:	r4_nf2cl_four_complex_djbfft rsi, 64, 64*64, rdi
	bump	rdi, XMM_SCD2		;; Next sin/cos data
	loops	64, b5b			;; Test inner loop counter
	bump	rsi, -64*64+dist128	;; Next source pointer
	bump	rdi, -64*XMM_SCD2	;; Restore sin/cos ptr
	loops	5, b5b			;; Test loop counter
	bump	rsi, -5*dist128		;; Next source pointer

;; Do FFT levels 5,6
;;
;; On input the 64-byte cache lines hold these data values:
;;	0	+2560	64	+2560	128	+2560	192	+2560
;;	1	...
;;	...
;;	63	...
;;	256	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+2560	16	+2560	32	+2560	48	+2560
;;	1	...
;;	...
;;	15	...
;;	64	...
;;	...

;; Do 16 eight_reals_fft operations
;; Do 48 four_complex_djbfft operations
;;	distance between fft data elements is 16
;;	do 16 macros each processing 32 data values

	loops_init_prefetch 160, 64, 2, rcx
	r4_h4cl_eight_reals_four_complex_djbfft_preload
b6b:	r4_h4cl_eight_reals_four_complex_djbfft rsi, 64, 16*64, 32*64, rdi, rdx
	bump	rdi, 4*XMM_SCD2		;; Next sin/cos data
	bump	rdx, 4*XMM_HSCD2	;; Next sin/cos data
	loops	16, b6b			;; Test loop counter
	bump	rsi, -16*64+64*64	;; Next source pointer
	bump	rdi, -16*4*XMM_SCD2	;; Restore sin/cos ptr
	bump	rdx, -16*4*XMM_HSCD2	;; Restore sin/cos ptr

;; Do 576 four_complex_djbfft operations
;;	distance between fft data elements is 16
;;	do 144 macros each processing 32 data values

	loops_reset			;; 5 iters of 1 or 2 of 16
	r4_x4cl_four_complex_djbfft_preload
b7b:	r4_x4cl_four_complex_djbfft rsi, 64, 16*64, 32*64, rdi
	bump	rdi, 4*XMM_SCD2		;; Next sin/cos data
	loops	16, b7b			;; Test inner loop counter
	bump	rsi, -16*64+64*64	;; Next source pointer
	bump	rdi, -16*4*XMM_SCD2	;; Restore sin/cos ptr
	loops	2, b7b			;; Test loop counter
	bump	rsi, -2*64*64+dist128	;; Next source pointer
	loops	5, b7b			;; 5 iterations
	bump	rsi, -5*dist128		;; Next source pointer

;; Do FFT levels 7,8
;;
;; On input the 64-byte cache lines hold these data values:
;;	0	+2560	16	+2560	32	+2560	48	+2560
;;	1	...
;;	...
;;	15	...
;;	64	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+2560	4	+2560	8	+2560	12	+2560
;;	1	...
;;	...
;;	3	...
;;	16	...
;;	...

;; Do 4 eight_reals_fft operations
;; Do 12 four_complex_djbfft operations
;;	distance between fft data elements is 4
;;	do 4 macros each processing 32 data values

	loops_init_prefetch 160, 64, 2, rcx
	r4_h4cl_eight_reals_four_complex_djbfft_preload
b8b:	r4_h4cl_eight_reals_four_complex_djbfft rsi, 64, 4*64, 8*64, rdi, rdx
	bump	rdi, 16*XMM_SCD2	;; Next sin/cos data
	bump	rdx, 16*XMM_HSCD2	;; Next sin/cos data
	loops	4, b8b			;; Test loop counter
	bump	rsi, -4*64+16*64	;; Next source pointer
	bump	rdi, -4*16*XMM_SCD2	;; Restore sin/cos ptr
	bump	rdx, -4*16*XMM_HSCD2	;; Restore sin/cos ptr

;; Do 624 four_complex_djbfft operations
;;	distance between fft data elements is 4
;;	do 156 macros each processing 32 data values

	loops_reset			;; 5 iters of 7 or 8 of 4
	r4_x4cl_four_complex_djbfft_preload
b9b:	r4_x4cl_four_complex_djbfft rsi, 64, 4*64, 8*64, rdi
	bump	rdi, 16*XMM_SCD2	;; Next sin/cos data
	loops	4, b9b			;; Test loop counter
	bump	rsi, -4*64+16*64	;; Next source pointer
	bump	rdi, -4*16*XMM_SCD2	;; Restore sin/cos ptr
	loops	8, b9b			;; Test middle loop counter
	bump	rsi, -8*16*64+dist128	;; Next source pointer
	loops	5, b9b			;; 5 iterations
	bump	rsi, -5*dist128		;; Next source pointer

;; Do FFT levels 9,10
;;
;; On input the 64-byte cache lines hold these data values:
;;	0	+2560	4	+2560	8	+2560	12	+2560
;;	1	...
;;	...
;;	3	...
;;	16	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+2560	2	+2560	4	+2560	6	+2560
;;	1	...
;;	8	...
;;	...

;; Do 1 eight_reals_fft operations
;; Do 3 four_complex_fft operations
;;	distance between fft data elements is 1

;; Do 636 four_complex_fft operations
;;	distance between fft data elements is 1
;;	do 159 macros each processing 32 data values

;; Execute the proper middle step

	cmp	ffttype, 2
	jl	xpass2_1280_levels_real_1
	jg	xpass2_1280_levels_real_3

xpass2_1280_levels_real_2:
	r4_h4cl_eight_reals_four_complex_with_square rsi, 4*64, 64, 2*64
	loops_init_prefetch 160, 64, 2, rcx, 1, 32
	r4_x4cl_four_complex_with_square_preload
baa:	r4_x4cl_four_complex_with_square rsi, 4*64, 64, 2*64
	loops	32, baa			;; Test loop counter
	bump	rsi, -32*4*64+dist128	;; Next source pointer
	loops	5, baa			;; Test loop counter
	bump	rsi, -5*dist128		;; Next source pointer
	jmp	xpass2_1280_real_unfft

xpass2_1280_levels_real_1:
	r4_h4cl_eight_reals_four_complex_fft_final rsi, 4*64, 64, 2*64
	loops_init_prefetch 160, 128, 1, rcx, 1, 32
bab0:	xtouch	[rcx+4096-64]		;; Load next 5 prefetch TLBs
bab:	r4_x4cl_four_complex_fft_final rsi, 4*64, 64, 2*64
	loops	32, bab			;; Test loop counter
	bump	rsi, -32*4*64+dist128	;; Next source pointer
	loops	5, bab0			;; Test loop counter
	jmp	xpass2_1280_real_done

xpass2_1280_levels_real_3:
	mov	rbp, DIST_TO_MULSRCARG
	r4_h4cl_eight_reals_four_complex_with_mult rsi, 4*64, 64, 2*64
	loops_init_prefetch 160, 64, 2, rcx, 1, 32
bac:	r4_x4cl_four_complex_with_mult rsi, 4*64, 64, 2*64
	loops	32, bac			;; Test loop counter
	bump	rsi, -32*4*64+dist128	;; Next source pointer
	loops	5, bac			;; Test loop counter
	bump	rsi, -5*dist128		;; Next source pointer
	jmp	xpass2_1280_real_unfft

xpass2_1280_levels_real_4:
	mov	rbp, DIST_TO_MULSRCARG
	mov	rcx, DATA_PREFETCH	;; Load prefetch pointer
	r4_h4cl_eight_reals_four_complex_with_mulf rsi, 4*64, 64, 2*64
	loops_init_prefetch 160, 128, 1, rcx, 1, 32
	xtouch	[rcx]			;; Load first 5 prefetch TLBs
bad0:	xtouch	[rcx+4096-64]
bad:	r4_x4cl_four_complex_with_mulf rsi, 4*64, 64, 2*64
	loops	32, bad			;; Test loop counter
	bump	rsi, -32*4*64+dist128	;; Next source pointer
	loops	5, bad0			;; Test loop counter
	bump	rsi, -5*dist128		;; Next source pointer

;; Do inverse FFT levels 7,8
;; On input the 64-byte cache lines hold these data values:
;;	0	+2560	2	+2560	4	+2560	6	+2560
;;	1	...
;;	8	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+2560	8	+2560	16	+2560	24	+2560
;;	1	...
;;	...
;;	7	...
;;	32	...
;;	...

;; Do 4 eight_reals_unfft operations
;; Do 4 four_complex_djbunfft operations
;;	distance between fft data elements is 4
;;	do 2 macros processing 32 data values

xpass2_1280_real_unfft:
	mov	rdi, xsincos_complex	;; Load sin/cos pointers
	bump	rdi, 256*XMM_SCD2
	mov	rdx, sincos3
	bump	rdx, 256*XMM_HSCD2
	loops_init_prefetch 160, 64, 2, rcx
	xtouch	[rcx+4096-64]		;; Load next 5 prefetch TLBs
	xtouch	[rcx+2*4096-64]
	xtouch	[rcx+3*4096-64]
	xtouch	[rcx+4*4096-64]
	xtouch	[rcx+5*4096-64]
c8b:	r4_h4cl_eight_reals_four_complex_djbunfft rsi, 64, 2*64, 4*64, rdi, 2*16*XMM_SCD2, rdx, 2*16*XMM_HSCD2
	bump	rdi, 16*XMM_SCD2	;; Next sin/cos data
	bump	rdx, 16*XMM_HSCD2	;; Next sin/cos data
	loops	2, c8b			;; Test loop counter
	bump	rsi, -2*64+8*64		;; Next source pointer
	bump	rdi, -2*16*XMM_SCD2	;; Restore sin/cos ptr
	bump	rdx, -2*16*XMM_HSCD2	;; Restore sin/cos ptr

;; Do 632 four_complex_djbunfft operations
;;	distance between fft data elements is 4
;;	do 158 macros each processing 32 data values

	loops_reset			;; 5 iters of 15 or 16 of 2
	r4_x4cl_four_complex_djbunfft_preload
c9b:	r4_x4cl_four_complex_djbunfft rsi, 64, 2*64, 4*64, rdi, 2*16*XMM_SCD2
	bump	rdi, 16*XMM_SCD2	;; Next sine/cosine pointer
	loops	2, c9b			;; Test loop counter
	bump	rsi, -2*64+8*64		;; Next source pointer
	bump	rdi, -2*16*XMM_SCD2	;; Restore sin/cos ptr
	loops	16, c9b			;; Test loop counter
	bump	rsi, -16*8*64+dist128	;; Next source pointer
	loops	5, c9b			;; Test loop counter
	bump	rsi, -5*dist128		;; Next source pointer

;; Do inverse FFT levels 5,6
;; On input the 64-byte cache lines hold these data values:
;;	0	+2560	8	+2560	16	+2560	24	+2560
;;	1	...
;;	...
;;	7	...
;;	32	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+2560	32	+2560	64	+2560	96	+2560
;;	1	...
;;	...
;;	31	...
;;	128	...
;;	...

;; Do 16 eight_reals_unfft operations
;; Do 16 four_complex_djbunfft operations
;;	distance between fft data elements is 16
;;	do 8 macros processing 32 data values

	loops_init_prefetch 160, 64, 2, rcx
c6b:	r4_h4cl_eight_reals_four_complex_djbunfft rsi, 64, 8*64, 16*64, rdi, 8*4*XMM_SCD2, rdx, 8*4*XMM_HSCD2
	bump	rdi, 4*XMM_SCD2		;; Next sin/cos data
	bump	rdx, 4*XMM_HSCD2	;; Next sin/cos data
	loops	8, c6b			;; Test inner loop counter
	bump	rsi, -8*64+32*64	;; Next source pointer
	bump	rdi, -8*4*XMM_SCD2	;; Restore sin/cos ptr
	bump	rdx, -8*4*XMM_HSCD2	;; Restore sin/cos ptr

;; Do 608 four_complex_djbunfft operations
;;	distance between fft data elements is 16
;;	do 152 macros each processing 32 data values

	loops_reset			;; 5 iters of 3 or 4 of 4
	r4_x4cl_four_complex_djbunfft_preload
c7b:	r4_x4cl_four_complex_djbunfft rsi, 64, 8*64, 16*64, rdi, 8*4*XMM_SCD2
	bump	rdi, 4*XMM_SCD2		;; Next sin/cos data
	loops	8, c7b			;; Test inner loop counter
	bump	rsi, -8*64+32*64	;; Next source pointer
	bump	rdi, -8*4*XMM_SCD2	;; Restore sin/cos ptr
	loops	4, c7b			;; Test loop counter
	bump	rsi, -4*32*64+dist128	;; Next source pointer
	loops	5, c7b			;; 5 iterations
	bump	rsi, -5*dist128		;; Next source pointer

;; Do inverse FFT levels 3,4
;; On input the 64-byte cache lines hold these data values:
;;	0	+2560	32	+2560	64	+2560	96	+2560
;;	1	...
;;	...
;;	31	...
;;	128	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+2560	128	+2560	256	+2560	384	+2560
;;	1	...
;;	...
;;	127	...
;;	512	...
;;	...

;; Do 64 eight_reals_unfft operations
;; Do 64 four_complex_djbunfft operations
;;	distance between fft data elements is 64
;;	do 32 macros processing 32 data values

	loops_init_prefetch 160, 64, 4, rcx
c4b:	r4_h4cl_eight_reals_four_complex_djbunfft rsi, 64, 32*64, 64*64, rdi, 32*XMM_SCD2, rdx, 32*XMM_HSCD2
	bump	rdi, XMM_SCD2		;; Next sin/cos data
	bump	rdx, XMM_HSCD2		;; Next sin/cos data
	loops	32, c4b			;; Test inner loop counter
	bump	rsi, -32*64+dist128	;; Next source pointer
	bump	rdi, -32*XMM_SCD2	;; Restore sin/cos ptr
	bump	rdx, -32*XMM_HSCD2	;; Restore sin/cos ptr

;; Do 512 four_complex_djbunfft operations
;;	distance between fft data elements is 64
;;	do 128 macros each processing 32 data values

	loops_reset			;; 4 more iters of 32
	r4_x4cl_four_complex_djbunfft_preload
c5b:	r4_x4cl_four_complex_djbunfft rsi, 64, 32*64, 64*64, rdi, 32*XMM_SCD2
	bump	rdi, XMM_SCD2		;; Next sin/cos data
	loops	32, c5b			;; Test inner loop counter
	bump	rsi, -32*64+dist128	;; Next source pointer
	bump	rdi, -32*XMM_SCD2	;; Restore sin/cos ptr
	loops	5, c5b			;; Test loop counter
	bump	rsi, -5*dist128		;; Next source pointer

;; Do inverse FFT level 1,2 (actually 2.322 FFT levels)
;; On input the 64-byte cache lines hold these data values:
;;	0	+2560	128	+2560	256	+2560	384	+2560
;;	1	...
;;	...
;;	127	...
;;	512	...
;;	...
;; On output the 64-byte cache lines hold these data values if pass 1 swizzles:
;;	0	+2560	640	+2560	1280	+2560	1920	+2560
;;	1	...
;;	...
;;	639
;; On output the 64-byte cache lines hold these data values if pass 2 swizzles:
;;	0	+640	1280	+640	1560	+640	3840	+640
;;	1	...
;;	...
;;	639

;; Do 256 ten_reals_unfft operations
;; Do 256 five_complex_djbunfft operations
;;	distance between fft data elements is 256
;;	do 128 macros processing 40 data values

	mov	rdi, xsincos_complex	;; Load sin/cos pointers
	mov	rdx, sincos3
	loops_init_prefetch 128, 64, 1, rcx
c1b:	r5_h5cl_ten_reals_five_complex_djbunfft rsi, 64, dist128, rdi, 128*XMM_SCD2, rdx, 128*XMM_HSCD2
	bump	rdi, XMM_SCD2		;; Next sin/cos data
	bump	rdx, XMM_HSCD2		;; Next sin/cos data
	loops	128, c1b		;; Test loop counter

xpass2_1280_real_done:
	end_timer 2
	ENDM

; *************** radix4-pass2-1280-levels-complex macro ******************
; This macro takes 2 groups of 1280 complex values and
; performs the final 10.322 levels of a standard mostly radix-4 FFT, squares the
; results, and does the first 10.322 levels of the inverse FFT.

r4_pass2_1280_levels_complex MACRO
	LOCAL	b1b, b3b, b4b, b5b, b6a, b6b0, b6b, b6c, b6d0, b6d
	LOCAL	c1b, c3b, c4b, c5b
	LOCAL	xpass2_1280_levels_complex_1, xpass2_1280_levels_complex_2
	LOCAL	xpass2_1280_levels_complex_3, xpass2_1280_levels_complex_4
	LOCAL	xpass2_1280_levels_complex_unfft, xpass2_1280_levels_complex_done

;; Type 4 FFTs skip the forward FFT process

	mov	rsi, DATA_ADDR		;; Load address of FFT data
	mov	rbx, DIST_TO_FFTSRCARG
	cmp	ffttype, 4
	je	xpass2_1280_levels_complex_4

;; Do FFT levels 1,2 (actually 2.322 levels)
;;
;; We swizzle the data so that one set of data is in the high word of
;; an xmm register, the other set is in the low word.  This means
;; a distance of 2560 data values.
;;
;; On input the 64-byte cache lines hold these data values if pass 2 swizzles:
;;	0	+640	1280	+640	2560	+640	3840	+640
;;	1	...
;;	...
;;	639
;; On input the 64-byte cache lines hold these data values if pass 1 swizzles:
;;	0	+2560	640	+2560	1280	+2560	1920	+2560
;;	1	...
;;	...
;;	639	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+2560	128	+2560	256	+2560	384	+2560
;;	1	...
;;	...
;;	127	...
;;	512	...
;;	...

;; Do 512 five_complex_fft operations
;;	distance between fft data elements is 256
;;	do 128 macros each processing 40 data values

	start_timer 4
	mov	rdi, xsincos_complex	;; The sin/cos data
	loops_init 128			;; 128 iterations
	r5_f5cl_five_complex_djbfft_preload
b1b:	xxprefetch [rdi+XMM_SCD2]	;; Prefetch for next pass
	xxprefetch [rdi+129*XMM_SCD2]	;; Prefetch for next pass
	r5_f5cl_five_complex_djbfft rsi, 64, dist128, rdi, 128*XMM_SCD2
	bump	rdi, XMM_SCD2		;; Next sin/cos data
	loops	128, b1b		;; Test inner loop counter
	bump	rsi, -128*64		;; Restore source pointer
	bump	rdi, 128*XMM_SCD2	;; Next sin/cos ptr
	end_timer 4

;; Do FFT levels 3,4
;;
;; On input the 64-byte cache lines hold these data values:
;;	0	+2560	128	+2560	256	+2560	384	+2560
;;	1	...
;;	...
;;	127	...
;;	512	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+2560	64	+2560	128	+2560	192	+2560
;;	1	...
;;	...
;;	63	...
;;	256	...
;;	...
;;
;; Do 640 four_complex_fft operations
;;	distance between fft data elements is 64
;;	do 320 macros each processing 16 data values

;; Do FFT levels 5,6
;;
;; On input the 64-byte cache lines hold these data values:
;;	0	+2560	64	+2560	128	+2560	192	+2560
;;	1	...
;;	...
;;	63	...
;;	256	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+2560	16	+2560	32	+2560	48	+2560
;;	1	...
;;	...
;;	15	...
;;	64	...
;;	...
;;
;; Do 640 four_complex_fft operations
;;	distance between fft data elements is 16
;;	do 160 macros each processing 32 data values

;; Do FFT levels 7,8
;;
;; On input the 64-byte cache lines hold these data values:
;;	0	+2560	16	+2560	32	+2560	48	+2560
;;	1	...
;;	...
;;	15	...
;;	64	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+2560	4	+2560	8	+2560	12	+2560
;;	1	...
;;	...
;;	3	...
;;	16	...
;;	...
;;
;; Do 640 four_complex_fft operations
;;	distance between fft data elements is 4
;;	do 160 macros each processing 32 data values

;; Do this 32 cache lines at a time to maximize accessing data
;; while it is in the L1 cache.

	start_timer 6
	mov	rcx, DATA_PREFETCH	;; Prefetch pointer (we need to prefetch 
					;; 384 cache lines + 4 pad cache lines)
	xtouch	[rcx]			;; Load first 5 prefetch TLBs
	xtouch	[rcx+4096-64]
	xtouch	[rcx+2*4096-64]
	xtouch	[rcx+3*4096-64]
	xtouch	[rcx+4*4096-64]
	xtouch	[rcx+5*4096-64]
	mov	rbx, rdi		;; Copy sin/cos ptrs
	mov	rbp, rdi
	loops_init_prefetch 320, 64, 4, rcx ;; 5 iters of 4 of 32 cache lines
	;r4_nf2cl_four_complex_djbfft_preload -- assume same as r4_x4cl_four_complex_djbfft_preload
	r4_x4cl_four_complex_djbfft_preload

b3b:	xxprefetch [rdi+4*XMM_SCD2]	;; Prefetch for next pass
	r4_nf2cl_four_complex_djbfft rsi, 4*64, 64*64, rdi
	bump	rdi, 4*XMM_SCD2		;; Next sine/cosine pointer
	loops	16, b3b			;; Loop 16 times
	bump	rsi, -16*4*64		;; Restore source pointer
	bump	rdi, -16*4*XMM_SCD2	;; Restore sine/cosine pointer
	loops_undo 16			;; Back up loop counter

b4b:	r4_x4cl_four_complex_djbfft rsi, 4*64, 16*64, 32*64, rbx
	bump	rbx, 4*4*XMM_SCD2	;; Next sine/cosine pointer
	loops	4, b4b, 2		;; 4 iterations with non-standard increment
	bump	rsi, -4*4*64+64*64	;; Next source pointer
	bump	rbx, -4*4*4*XMM_SCD2	;; Restore sine/cosine pointer
	loops	2, b4b			;; Loop twice
	bump	rsi, -2*64*64		;; Restore source pointer
	loops_undo 2*4*2		;; Back up loop counter

b5b:	r4_x4cl_four_complex_djbfft rsi, 16*64, 4*64, 8*64, rbp
	loops	8, b5b, 2		;; 8 iterations with non-standard increment

	bump	rsi, -8*16*64+64	;; Next source pointer
	bump	rdi, XMM_SCD2		;; Next sine/cosine pointer
	bump	rbx, 4*XMM_SCD2		;; Next sine/cosine pointer
	bump	rbp, 16*XMM_SCD2	;; Next sine/cosine pointer
	loops	4, b3b			;; Loop 4 times
	bump	rsi, -4*64+dist128	;; Next source pointer
	bump	rdi, -4*XMM_SCD2	;; Restore sine/cosine pointer
	bump	rbx, -4*4*XMM_SCD2	;; Restore sine/cosine pointer
	bump	rbp, -4*16*XMM_SCD2	;; Restore sine/cosine pointer
	loops	5, b3b			;; Loop 5 times
	bump	rsi, -5*dist128		;; Restore source pointer
	end_timer 6

;; Do FFT levels 9,10 as well as inverse FFT levels 9,10
;;
;; On input the 64-byte cache lines hold these data values:
;;	0	+2560	4	+2560	8	+2560	12	+2560
;;	1	...
;;	...
;;	3	...
;;	16	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+2560	2	+2560	4	+2560	6	+2560
;;	1	...
;;	8	...
;;	...

;; Do 640 four_complex_fft operations
;;	distance between fft data elements is 1
;;	do 160 macros each processing 32 data values

;; Execute the right middle step

	cmp	ffttype, 2
	jl	xpass2_1280_levels_complex_1
	jg	xpass2_1280_levels_complex_3

xpass2_1280_levels_complex_2:
	start_timer 9
	loops_init_prefetch 160, 64, 4, rcx
	r4_x4cl_four_complex_with_square_preload
b6a:	r4_x4cl_four_complex_with_square rsi, 4*64, 64, 2*64
	loops	32, b6a			;; Loop 32 times
	bump	rsi, -32*4*64+dist128	;; Next source pointer
	loops	5, b6a			;; Loop 5 times
	bump	rsi, -5*dist128		;; Restore source pointer
	end_timer 9
	jmp	xpass2_1280_levels_complex_unfft

xpass2_1280_levels_complex_1:
	start_timer 9
	loops_init_prefetch 160, 128, 1, rcx
b6b0:	xtouch	[rcx+4096-64]		;; Load next 5 prefetch TLBs
b6b:	r4_x4cl_four_complex_fft_final rsi, 4*64, 64, 2*64
	loops	32, b6b			;; Loop 32 times
	bump	rsi, -32*4*64+dist128	;; Next source pointer
	loops	5, b6b0			;; Loop 5 times
	end_timer 9
	jmp	xpass2_1280_levels_complex_done

xpass2_1280_levels_complex_3:
	start_timer 9
	mov	rbp, DIST_TO_MULSRCARG
	loops_init_prefetch 160, 64, 4, rcx
	xtouch	[rcx+4096-128]		;; Load prefetch TLBs
	xtouch	[rcx+2*4096-128]
b6c:	r4_x4cl_four_complex_with_mult rsi, 4*64, 64, 2*64
	loops	32, b6c			;; Loop 32 times
	bump	rsi, -32*4*64+dist128	;; Next source pointer
	loops	5, b6c			;; Loop 5 times
	bump	rsi, -5*dist128		;; Restore source pointer
	end_timer 9
	jmp	xpass2_1280_levels_complex_unfft

xpass2_1280_levels_complex_4:
	start_timer 9
	mov	rcx, DATA_PREFETCH	;; Prefetch pointer
	xtouch	[rcx]			;; Load first 5 prefetch TLBs
	mov	rbp, DIST_TO_MULSRCARG
	loops_init_prefetch 160, 128, 1, rcx
b6d0:	xtouch	[rcx+4096-64]
b6d:	r4_x4cl_four_complex_with_mulf rsi, 4*64, 64, 2*64
	loops	32, b6d			;; Loop 32 times
	bump	rsi, -32*4*64+dist128	;; Next source pointer
	loops	5, b6d0			;; Loop 5 times
	bump	rsi, -5*dist128		;; Restore source pointer
	end_timer 9

;; Do inverse FFT levels 7,8
;; On input the 64-byte cache lines hold these data values:
;;	0	+2560	2	+2560	4	+2560	6	+2560
;;	1	...
;;	8	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+2560	8	+2560	16	+2560	24	+2560
;;	1	...
;;	...
;;	7	...
;;	32	...
;;	...
;;
;; Do 640 four_complex_unfft operations
;;	distance between fft data elements is 4
;;	do 160 macros each processing 32 data values

;; Do inverse FFT levels 5,6
;; On input the 64-byte cache lines hold these data values:
;;	0	+2560	8	+2560	16	+2560	24	+2560
;;	1	...
;;	...
;;	7	...
;;	32	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+2560	32	+2560	64	+2560	96	+2560
;;	1	...
;;	...
;;	31	...
;;	128	...
;;	...
;;
;; Do 640 four_complex_unfft operations
;;	distance between fft data elements is 16
;;	do 160 macros each processing 32 data values

;; Do inverse FFT levels 3,4
;; On input the 64-byte cache lines hold these data values:
;;	0	+2560	32	+2560	64	+2560	96	+2560
;;	1	...
;;	...
;;	31	...
;;	128	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+2560	128	+2560	256	+2560	384	+2560
;;	1	...
;;	...
;;	127	...
;;	512	...
;;	...
;;
;; Do 640 four_complex_unfft operations
;;	distance between fft data elements is 64
;;	do 160 macros each processing 32 data values

;; Do this 64 cache lines at a time to maximize accessing data
;; while it is in the L1 cache.

xpass2_1280_levels_complex_unfft:
	start_timer 10
	mov	rdi, xsincos_complex	;; Load sin/cos pointers
	bump	rdi, 256*XMM_SCD2
	mov	rbx, rdi
	mov	rbp, rdi
	loops_init_prefetch 160, 64, 2, rcx ;; 5*2 iters of 64 cache lines
	r4_x4cl_four_complex_djbunfft_preload

	xtouch	[rcx+4096-64]		;; Load last 5 prefetch TLBs
	xtouch	[rcx+2*4096-64]
	xtouch	[rcx+3*4096-64]
	xtouch	[rcx+4*4096-64]
	xtouch	[rcx+5*4096-64]

c5b:	r4_x4cl_four_complex_djbunfft rsi, 8*64, 2*64, 4*64, rbp, 2*16*XMM_SCD2
	loops	16, c5b			;; Loop 16 times
	bump	rsi, -16*8*64		;; Restore source pointer
	loops_undo 16			;; Restore loop counter

c4b:	r4_x4cl_four_complex_djbunfft rsi, 2*64, 8*64, 16*64, rbx, 8*4*XMM_SCD2
	bump	rbx, 2*4*XMM_SCD2	;; Next sine/cosine pointer
	loops	4, c4b			;; Loop 4 times
	bump	rsi, -4*2*64+32*64	;; Next source pointer
	bump	rbx, -4*2*4*XMM_SCD2	;; Restore sine/cosine pointer
	loops	4, c4b			;; Loop 4 times
	bump	rsi, -4*32*64		;; Restore source pointer
	loops_undo 4*4			;; Restore loop counter

c3b:	r4_x4cl_four_complex_djbunfft rsi, 2*64, 32*64, 64*64, rdi, 32*1*XMM_SCD2
	bump	rdi, 2*XMM_SCD2		;; Next sine/cosine pointer
	loops	16, c3b			;; Loop 16 times
	bump	rdi, -16*2*XMM_SCD2	;; Restore sine/cosine pointer

	bump	rsi, -16*2*64+64	;; Next source pointer
	bump	rdi, XMM_SCD2		;; Next sine/cosine pointer
	bump	rbx, 4*XMM_SCD2		;; Next sine/cosine pointer
	bump	rbp, 16*XMM_SCD2	;; Next sine/cosine pointer
	loops	2, c5b			;; Loop twice
	bump	rsi, -2*64+dist128	;; Next source pointer
	bump	rdi, -2*XMM_SCD2	;; Restore sine/cosine pointer
	bump	rbx, -2*4*XMM_SCD2	;; Restore sine/cosine pointer
	bump	rbp, -2*16*XMM_SCD2	;; Restore sine/cosine pointer
	loops	5, c5b			;; Loop 5 times
	bump	rsi, -5*dist128		;; Restore source pointer
	end_timer 10

;; Do inverse FFT levels 1,2 (actually 2.322 FFT levels)
;; On input the 64-byte cache lines hold these data values:
;;	0	+2560	128	+2560	256	+2560	384	+2560
;;	1	...
;;	...
;;	127	...
;;	512	...
;;	...
;; On output the 64-byte cache lines hold these data values if pass 1 swizzles:
;;	0	+2560	640	+2560	1280	+2560	1920	+2560
;;	1	...
;;	...
;;	639
;; On output the 64-byte cache lines hold these data values if pass 2 swizzles:
;;	0	+640	1280	+640	2560	+640	3840	+640
;;	1	...
;;	...
;;	639

;; Do 512 five_complex_unfft operations
;;	distance between fft data elements is 256
;;	do 128 macros each processing 40 data values

	start_timer 14
	mov	rdi, xsincos_complex	;; Load sin/cos pointer
	loops_init_prefetch 128, 64, 1, rcx ;; 128 iters
	xtouch	[rcx+4096-128]		;; Load prefetch TLBs
	r5_x5cl_five_complex_djbunfft_preload
c1b:	r5_x5cl_five_complex_djbunfft rsi, 64, dist128, rdi, 128*XMM_SCD2
	bump	rdi, XMM_SCD2		;; Next set of sin/cos data
	loops	128, c1b		;; Test loop counter
	end_timer 14

xpass2_1280_levels_complex_done:
	ENDM


;*********************************************************************************************
;			Macros derived from pass2-11-levels
;*********************************************************************************************


; *************** radix4-pass2-1536-levels-real macro ******************
; This macro takes 3072 real values and performs the final 11.585 levels of
; the FFT.  Independently, this macro also takes 3072 complex values and
; performs the final 10.585 levels of the FFT.  The results are squared, and
; the first 10.585/11.585 levels of the inverse FFT are done.

r4_pass2_1536_levels_real MACRO
	LOCAL	b1b, b3b, b4b, b5b, b6b, b7b
	LOCAL	b8b, b9b, baa, bab0, bab, bac, bad0, bad
	LOCAL	c1b, c3b, c4b, c5b, c6b, c7b, c8b, c9b
	LOCAL	xpass2_1536_levels_real_1, xpass2_1536_levels_real_2
	LOCAL	xpass2_1536_levels_real_3, xpass2_1536_levels_real_4
	LOCAL	xpass2_1536_real_unfft, xpass2_1536_real_done

;; We switch to the same format used in one-pass FFTs (that is, the
;; high word of xmm registers being 1 greater than the low word):

;; Type 4 FFTs skip the forward FFT process

	start_timer 2
	mov	rsi, DATA_ADDR		;; Load source address
	mov	rbx, DIST_TO_FFTSRCARG
	cmp	ffttype, 4
	je	xpass2_1536_levels_real_4

;; Do FFT level 1,2 (actually 1.585 levels)
;;
;; On input the 64-byte cache lines hold these data values if pass 2 swizzles:
;;	0	+768	1536	+768	3072	+768	4608	+768
;;	1	...
;;	...
;;	767
;; On input the 64-byte cache lines hold these data values if pass 1 swizzles:
;;	0	+3K	768	+3K	1536	+3K	2304	+3K
;;	1	...
;;	...
;;	767	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+3K	256	+3K	512	+3K	768	+3K
;;	1	...
;;	...
;;	255	...
;;	1024	...
;;	...

;; Do 512 six_reals_fft operations
;; Do 512 three_complex_djbfft operations
;;	distance between fft data elements is 512
;;	do 256 iterations

	mov	rdi, xsincos_complex	;; Ptr to the sin/cos data
	mov	rdx, sincos3		;; Ptr to real sin/cos data
	loops_init 256
b1b:	r3_fh3cl_six_reals_three_complex_djbfft rsi, 64, 2*dist128, rdi, 256*XMM_SCD1, rdx, 256*XMM_HSCD1
	bump	rdi, XMM_SCD1		;; Next sin/cos data
	bump	rdx, XMM_HSCD1		;; Next sin/cos data
	loops	128, b1b		;; Test loop counter
	bump	rsi, -128*64+dist128	;; Next source pointer
	loops	2, b1b			;; Test loop counter
	bump	rsi, -2*dist128		;; Restore source pointer
	bump	rdi, 256*XMM_SCD1	;; Next sin/cos ptr
	bump	rdx, 256*XMM_HSCD1	;; Next sin/cos ptr

;; Do FFT levels 3,4
;;
;; On input the 64-byte cache lines hold these data values:
;;	0	+3K	256	+3K	512	+3K	768	+3K
;;	1	...
;;	...
;;	255	...
;;	1024	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+3K	128	+3K	256	+3K	384	+3K
;;	1	...
;;	...
;;	127	...
;;	512	...
;;	...

;; Do 128 eight_reals_fft operations
;; Do 128 four_complex_djbfft operations
;;	distance between fft data elements is 128
;;	do 128 macros each processing 16 data values

	mov	rcx, DATA_PREFETCH	;; Prefetch pointer
	xtouch	[rcx]			;; Preload first TLB
	loops_init_prefetch 384, 64, 4, rcx
	xtouch	[rcx+4096-64]		;; Preload the TLBs
	xtouch	[rcx+2*4096-64]
b4b:	r4_h2cl_eight_reals_four_complex_djbfft rsi, 64, dist128, rdi, rdx
	bump	rdi, XMM_SCD2		;; Next sin/cos data
	bump	rdx, XMM_HSCD2		;; Next sin/cos data
	loops	128, b4b		;; Test loop counter
	bump	rsi, -128*64+2*dist128	;; Next source pointer
	bump	rdi, -128*XMM_SCD2	;; Restore sin/cos ptr
	bump	rdx, -128*XMM_HSCD2	;; Restore sin/cos ptr

;; Do 512 four_complex_djbfft operations
;;	distance between fft data elements is 128
;;	do 256 macros each processing 16 data values

	loops_reset			;; 2 more iterations of 128
	r4_nf2cl_four_complex_djbfft_preload
b5b:	r4_nf2cl_four_complex_djbfft rsi, 64, dist128, rdi
	bump	rdi, XMM_SCD2		;; Next sin/cos data
	loops	128, b5b		;; Test inner loop counter
	bump	rsi, -128*64+2*dist128	;; Next source pointer
	bump	rdi, -128*XMM_SCD2	;; Restore sin/cos ptr
	loops	3, b5b			;; Test loop counter
	bump	rsi, -3*2*dist128	;; Next source pointer

;; Do FFT levels 5,6
;;
;; On input the 64-byte cache lines hold these data values:
;;	0	+3K	128	+3K	256	+3K	384	+3K
;;	1	...
;;	...
;;	127	...
;;	512	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+3K	32	+3K	64	+3K	96	+3K
;;	1	...
;;	...
;;	31	...
;;	128	...
;;	...

;; Do 32 eight_reals_fft operations
;; Do 96 four_complex_djbfft operations
;;	distance between fft data elements is 32
;;	do 32 macros each processing 32 data values

	loops_init_prefetch 192, 64, 2, rcx
	xtouch	[rcx+4096-64]		;; Preload the TLBs
	xtouch	[rcx+2*4096-64]
	r4_h4cl_eight_reals_four_complex_djbfft_preload
b6b:	r4_h4cl_eight_reals_four_complex_djbfft rsi, 64, 32*64, 64*64, rdi, rdx
	bump	rdi, 4*XMM_SCD2		;; Next sin/cos data
	bump	rdx, 4*XMM_HSCD2	;; Next sin/cos data
	loops	32, b6b			;; Test loop counter
	bump	rsi, -32*64+dist128	;; Next source pointer
	bump	rdi, -32*4*XMM_SCD2	;; Restore sin/cos ptr
	bump	rdx, -32*4*XMM_HSCD2	;; Restore sin/cos ptr

;; Do 640 four_complex_djbfft operations
;;	distance between fft data elements is 32
;;	do 160 macros each processing 32 data values

	loops_reset			;; 5 more iters of 32
	r4_x4cl_four_complex_djbfft_preload
b7b:	r4_x4cl_four_complex_djbfft rsi, 64, 32*64, 64*64, rdi
	bump	rdi, 4*XMM_SCD2		;; Next sin/cos data
	loops	32, b7b			;; Test inner loop counter
	bump	rsi, -32*64+dist128	;; Next source pointer
	bump	rdi, -32*4*XMM_SCD2	;; Restore sin/cos ptr
	loops	6, b7b			;; 6 iterations
	bump	rsi, -6*dist128		;; Next source pointer

;; Do FFT levels 7,8
;;
;; On input the 64-byte cache lines hold these data values:
;;	0	+3K	32	+3K	64	+3K	96	+3K
;;	1	...
;;	...
;;	31	...
;;	128	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+3K	8	+3K	16	+3K	24	+3K
;;	1	...
;;	...
;;	7	...
;;	32	...
;;	...

;; Do 8 eight_reals_fft operations
;; Do 24 four_complex_djbfft operations
;;	distance between fft data elements is 8
;;	do 8 macros each processing 32 data values

	loops_init_prefetch 192, 64, 2, rcx
	xtouch	[rcx+4096-64]		;; Preload the TLBs
	xtouch	[rcx+2*4096-64]
	r4_h4cl_eight_reals_four_complex_djbfft_preload
b8b:	r4_h4cl_eight_reals_four_complex_djbfft rsi, 64, 8*64, 16*64, rdi, rdx
	bump	rdi, 16*XMM_SCD2	;; Next sin/cos data
	bump	rdx, 16*XMM_HSCD2	;; Next sin/cos data
	loops	8, b8b			;; Test loop counter
	bump	rsi, -8*64+32*64	;; Next source pointer
	bump	rdi, -8*16*XMM_SCD2	;; Restore sin/cos ptr
	bump	rdx, -8*16*XMM_HSCD2	;; Restore sin/cos ptr

;; Do 3040 four_complex_djbfft operations
;;	distance between fft data elements is 8
;;	do 760 macros each processing 32 data values

	loops_reset			;; 6 iters of 3 or 4 of 8
	r4_x4cl_four_complex_djbfft_preload
b9b:	r4_x4cl_four_complex_djbfft rsi, 64, 8*64, 16*64, rdi
	bump	rdi, 16*XMM_SCD2	;; Next sin/cos data
	loops	8, b9b			;; Test loop counter
	bump	rsi, -8*64+32*64	;; Next source pointer
	bump	rdi, -8*16*XMM_SCD2	;; Restore sin/cos ptr
	loops	4, b9b			;; Test middle loop counter
	bump	rsi, -4*32*64+dist128	;; Next source pointer
	loops	6, b9b			;; 6 iterations
	bump	rsi, -6*dist128		;; Next source pointer

;; Do FFT levels 9,10,11
;;
;; On input the 64-byte cache lines hold these data values:
;;	0	+3K	8	+3K	16	+3K	24	+3K
;;	1	...
;;	...
;;	7	...
;;	32	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+3K	4	+3K	8	+3K	12	+3K
;;	1	...
;;	...
;;	3	...
;;	16	...
;;	...

;; Do 1 sixteen_reals_fft operations
;; Do 3 eight_complex_fft operations
;;	distance between fft data elements is 1

;; Do 380 eight_complex_fft operations
;;	distance between fft data elements is 1
;;	do 95 macros each processing 64 data values

;; Execute the proper middle step

	cmp	ffttype, 2
	jl	xpass2_1536_levels_real_1
	jg	xpass2_1536_levels_real_3

xpass2_1536_levels_real_2:
	r8_h8cl_sixteen_reals_eight_complex_with_square rsi, 8*64, 64, 2*64, 4*64
	loops_init_prefetch 96, 64, 1, rcx, 1, 16
	r8_x8cl_eight_complex_with_square_preload
	xtouch	[rcx+4096-64]		;; Preload the TLBs
	xtouch	[rcx+2*4096-64]		;; Preload the TLBs
baa:	r8_x8cl_eight_complex_with_square rsi, 8*64, 64, 2*64, 4*64
	loops	16, baa			;; Test loop counter
	bump	rsi, -16*8*64+dist128	;; Next source pointer
	loops	6, baa			;; Test loop counter
	bump	rsi, -6*dist128		;; Next source pointer
	jmp	xpass2_1536_real_unfft

xpass2_1536_levels_real_1:
	r8_h8cl_sixteen_reals_eight_complex_fft_final rsi, 8*64, 64, 2*64, 4*64
	loops_init_prefetch 96, 128, 1, rcx, 1, 16
	r8_x8cl_eight_complex_fft_final_preload
bab0:	xtouch	[rcx+4096-64]		;; Preload the TLBs
bab:	r8_x8cl_eight_complex_fft_final rsi, 8*64, 64, 2*64, 4*64
	loops	16, bab			;; Test loop counter
	bump	rsi, -16*8*64+dist128	;; Next source pointer
	loops_touch 2, bab, 3, bab0		;; Test loop counter
	jmp	xpass2_1536_real_done

xpass2_1536_levels_real_3:
	mov	rbp, DIST_TO_MULSRCARG
	r8_h8cl_sixteen_reals_eight_complex_with_mult rsi, 8*64, 64, 2*64, 4*64
	loops_init_prefetch 96, 64, 1, rcx, 1, 16
	r8_x8cl_eight_complex_with_mult_preload
	xtouch	[rcx+4096-64]		;; Preload the TLBs
	xtouch	[rcx+2*4096-64]
bac:	r8_x8cl_eight_complex_with_mult rsi, 8*64, 64, 2*64, 4*64
	loops	16, bac			;; Test loop counter
	bump	rsi, -16*8*64+dist128	;; Next source pointer
	loops	6, bac			;; Test loop counter
	bump	rsi, -6*dist128		;; Next source pointer
	jmp	xpass2_1536_real_unfft

xpass2_1536_levels_real_4:
	mov	rbp, DIST_TO_MULSRCARG
	mov	rcx, DATA_PREFETCH	;; Load prefetch pointer
	xtouch	[rcx]			;; Preload first TLBs
	r8_h8cl_sixteen_reals_eight_complex_with_mulf rsi, 8*64, 64, 2*64, 4*64
	loops_init_prefetch 96, 128, 1, rcx, 1, 16
	r8_x8cl_eight_complex_with_mulf_preload
bad0:	xtouch	[rcx+4096-64]
bad:	r8_x8cl_eight_complex_with_mulf rsi, 8*64, 64, 2*64, 4*64
	loops	16, bad			;; Test loop counter
	bump	rsi, -16*8*64+dist128	;; Next source pointer
	loops_touch 2, bad, 3, bad0	;; Test loop counter
	bump	rsi, -6*dist128		;; Next source pointer

;; Do inverse FFT levels 7,8
;; On input the 64-byte cache lines hold these data values:
;;	0	+3K	4	+3K	8	+3K	12	+3K
;;	1	...
;;	...
;;	3	...
;;	16	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+3K	16	+3K	32	+3K	48	+3K
;;	1	...
;;	...
;;	15	...
;;	64	...
;;	...

;; Do 8 eight_reals_unfft operations
;; Do 8 four_complex_djbunfft operations
;;	distance between fft data elements is 8
;;	do 4 macros processing 32 data values

xpass2_1536_real_unfft:
	mov	rdi, xsincos_complex	;; Load sin/cos pointers
	bump	rdi, 512*XMM_SCD1
	mov	rdx, sincos3
	bump	rdx, 512*XMM_HSCD1
	loops_init_prefetch 192, 64, 2, rcx
	xtouch	[rcx+4096-64]
	xtouch	[rcx+2*4096-64]
c8b:	r4_h4cl_eight_reals_four_complex_djbunfft rsi, 64, 4*64, 8*64, rdi, 4*16*XMM_SCD2, rdx, 4*16*XMM_HSCD2
	bump	rdi, 16*XMM_SCD2	;; Next sin/cos data
	bump	rdx, 16*XMM_HSCD2	;; Next sin/cos data
	loops	4, c8b			;; Test loop counter
	bump	rsi, -4*64+16*64	;; Next source pointer
	bump	rdi, -4*16*XMM_SCD2	;; Restore sin/cos ptr
	bump	rdx, -4*16*XMM_HSCD2	;; Restore sin/cos ptr

;; Do 752 four_complex_djbunfft operations
;;	distance between fft data elements is 8
;;	do 188 macros each processing 32 data values

	loops_reset			;; 6 iters of 7 or 8 of 4
	r4_x4cl_four_complex_djbunfft_preload
c9b:	r4_x4cl_four_complex_djbunfft rsi, 64, 4*64, 8*64, rdi, 4*16*XMM_SCD2
	bump	rdi, 16*XMM_SCD2	;; Next sine/cosine pointer
	loops	4, c9b			;; Test loop counter
	bump	rsi, -4*64+16*64	;; Next source pointer
	bump	rdi, -4*16*XMM_SCD2	;; Restore sin/cos ptr
	loops	8, c9b			;; Test loop counter
	bump	rsi, -8*16*64+dist128	;; Next source pointer
	loops	6, c9b			;; Test loop counter
	bump	rsi, -6*dist128		;; Next source pointer

;; Do inverse FFT levels 5,6
;; On input the 64-byte cache lines hold these data values:
;;	0	+3K	16	+3K	32	+3K	48	+3K
;;	1	...
;;	...
;;	15	...
;;	64	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+3K	64	+3K	128	+3K	192	+3K
;;	1	...
;;	...
;;	63	...
;;	256	...
;;	...

;; Do 32 eight_reals_unfft operations
;; Do 32 four_complex_djbunfft operations
;;	distance between fft data elements is 32
;;	do 16 macros processing 32 data values

	loops_init_prefetch 192, 64, 2, rcx
	xtouch	[rcx+4096-64]
	xtouch	[rcx+2*4096-64]
c6b:	r4_h4cl_eight_reals_four_complex_djbunfft rsi, 64, 16*64, 32*64, rdi, 16*4*XMM_SCD2, rdx, 16*4*XMM_HSCD2
	bump	rdi, 4*XMM_SCD2		;; Next sin/cos data
	bump	rdx, 4*XMM_HSCD2	;; Next sin/cos data
	loops	16, c6b			;; Test inner loop counter
	bump	rsi, -16*64+64*64	;; Next source pointer
	bump	rdi, -16*4*XMM_SCD2	;; Restore sin/cos ptr
	bump	rdx, -16*4*XMM_HSCD2	;; Restore sin/cos ptr

;; Do 704 four_complex_djbunfft operations
;;	distance between fft data elements is 32
;;	do 176 macros each processing 32 data values

	loops_reset			;; 6 iters of 1 or 2 of 16
	r4_x4cl_four_complex_djbunfft_preload
c7b:	r4_x4cl_four_complex_djbunfft rsi, 64, 16*64, 32*64, rdi, 16*4*XMM_SCD2
	bump	rdi, 4*XMM_SCD2		;; Next sin/cos data
	loops	16, c7b			;; Test inner loop counter
	bump	rsi, -16*64+64*64	;; Next source pointer
	bump	rdi, -16*4*XMM_SCD2	;; Restore sin/cos ptr
	loops	2, c7b			;; Test loop counter
	bump	rsi, -2*64*64+dist128	;; Next source pointer
	loops	6, c7b			;; 6 iterations
	bump	rsi, -6*dist128		;; Next source pointer

;; Do inverse FFT levels 3,4
;; On input the 64-byte cache lines hold these data values:
;;	0	+3K	64	+3K	128	+3K	192	+3K
;;	1	...
;;	...
;;	63	...
;;	256	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+3K	256	+3K	512	+3K	768	+3K
;;	1	...
;;	...
;;	255	...
;;	1024	...
;;	...

;; Do 128 eight_reals_unfft operations
;; Do 128 four_complex_djbunfft operations
;;	distance between fft data elements is 128
;;	do 64 macros processing 32 data values

	loops_init_prefetch 192, 64, 2, rcx
	xtouch	[rcx+4096-64]
	xtouch	[rcx+2*4096-64]
c4b:	r4_h4cl_eight_reals_four_complex_djbunfft rsi, 64, 64*64, dist128, rdi, 64*XMM_SCD2, rdx, 64*XMM_HSCD2
	bump	rdi, XMM_SCD2		;; Next sin/cos data
	bump	rdx, XMM_HSCD2		;; Next sin/cos data
	loops	64, c4b			;; Test inner loop counter
	bump	rsi, -64*64+2*dist128	;; Next source pointer
	bump	rdi, -64*XMM_SCD2	;; Restore sin/cos ptr
	bump	rdx, -64*XMM_HSCD2	;; Restore sin/cos ptr

;; Do 512 four_complex_djbunfft operations
;;	distance between fft data elements is 128
;;	do 128 macros each processing 32 data values

	loops_reset			;; 2 more iters of 64
	r4_x4cl_four_complex_djbunfft_preload
c5b:	r4_x4cl_four_complex_djbunfft rsi, 64, 64*64, dist128, rdi, 64*XMM_SCD2
	bump	rdi, XMM_SCD2		;; Next sin/cos data
	loops	64, c5b			;; Test inner loop counter
	bump	rsi, -64*64+2*dist128	;; Next source pointer
	bump	rdi, -64*XMM_SCD2	;; Restore sin/cos ptr
	loops	3, c5b			;; Test loop counter
	bump	rsi, -3*2*dist128	;; Next source pointer

;; Do inverse FFT level 1,2 (actually 1.585 FFT levels)
;; On input the 64-byte cache lines hold these data values:
;;	0	+3K	256	+3K	512	+3K	768	+3K
;;	1	...
;;	...
;;	255	...
;;	1024	...
;;	...
;; On output the 64-byte cache lines hold these data values if pass 1 swizzles:
;;	0	+3K	768	+3K	1536	+3K	2304	+3K
;;	1	...
;;	...
;;	767
;; On output the 64-byte cache lines hold these data values if pass 2 swizzles:
;;	0	+768	1536	+768	3072	+768	4608	+768
;;	1	...
;;	...
;;	767

;; Do 512 six_reals_unfft operations
;; Do 512 three_complex_djbunfft operations
;;	distance between fft data elements is 512
;;	do 256 macros processing 24 data values

	mov	rdi, xsincos_complex	;; Load sin/cos pointers
	mov	rdx, sincos3
	bump	rcx, -21*64		;; Back up prefetch pointer by 21 cache lines
	loops_init_prefetch 256, 64, 2, rcx
	xtouch	[rcx+4096-64]
	xtouch	[rcx+2*4096-64]
c1b:	r3_h3cl_six_reals_three_complex_djbunfft rsi, 64, 2*dist128, rdi, 256*XMM_SCD1, rdx, 256*XMM_HSCD1
	bump	rdi, XMM_SCD1		;; Next sin/cos data
	bump	rdx, XMM_HSCD1		;; Next sin/cos data
	loops	128, c1b		;; Test loop counter
	bump	rsi, -128*64+dist128	;; Next source pointer
	loops	2, c1b			;; Test loop counter

xpass2_1536_real_done:
	end_timer 2
	ENDM

; *************** radix4-pass2-1536-levels-complex macro ******************
; This macro takes 2 groups of 1536 complex values and
; performs the final 10.585 levels of a standard mostly radix-4 FFT, squares the
; results, and does the first 10.585 levels of the inverse FFT.

r4_pass2_1536_levels_complex MACRO
	LOCAL	b1b, b3a, b3b, b4b, b5b, b6a, b6b0, b6b, b6c, b6d0, b6d
	LOCAL	c1b, c3b, c4b, c5a, c5b
	LOCAL	xpass2_1536_levels_complex_1, xpass2_1536_levels_complex_2
	LOCAL	xpass2_1536_levels_complex_3, xpass2_1536_levels_complex_4
	LOCAL	xpass2_1536_levels_complex_unfft, xpass2_1536_levels_complex_done

;; Type 4 FFTs skip the forward FFT process

	mov	rsi, DATA_ADDR		;; Load address of FFT data
	mov	rbx, DIST_TO_FFTSRCARG
	cmp	ffttype, 4
	je	xpass2_1536_levels_complex_4

;; Do FFT levels 1,2 (actually 1.585 levels)
;;
;; We swizzle the data so that one set of data is in the high word of
;; an xmm register, the other set is in the low word.  This means
;; a distance of 3072 data values.
;;
;; On input the 64-byte cache lines hold these data values if pass 2 swizzles:
;;	0	+768	1536	+768	3072	+768	4608	+768
;;	1	...
;;	...
;;	767
;; On input the 64-byte cache lines hold these data values if pass 1 swizzles:
;;	0	+3K	768	+3K	1536	+3K	2304	+3K
;;	1	...
;;	...
;;	767	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+3K	256	+3K	512	+3K	768	+3K
;;	1	...
;;	...
;;	255	...
;;	1024	...
;;	...

;; Do 1024 three_complex_fft operations
;;	distance between fft data elements is 512
;;	do 256 macros each processing 24 data values

	start_timer 4
	mov	rdi, xsincos_complex	;; The sin/cos data
	loops_init 256			;; 2 iterations of 128
	r3_f3cl_three_complex_djbfft_preload
b1b:	xxprefetch [rdi+XMM_SCD1]	;; Prefetch for next pass
	xxprefetch [rdi+257*XMM_SCD1]	;; Prefetch for next pass
	r3_f3cl_three_complex_djbfft rsi, 64, 2*dist128, rdi, 256*XMM_SCD1
	bump	rdi, XMM_SCD1		;; Next sin/cos data
	loops	128, b1b		;; Test inner loop counter
	bump	rsi, -128*64+dist128	;; Next source pointer
	loops	2, b1b			;; Test outer loop counter
	bump	rsi, -2*dist128		;; Restore source pointer
	bump	rdi, 256*XMM_SCD1	;; Next sin/cos ptr
	end_timer 4

;; Do FFT levels 3,4
;;
;; On input the 64-byte cache lines hold these data values:
;;	0	+3K	128	+3K	256	+3K	384	+3K
;;	1	...
;;	...
;;	127	...
;;	512	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+3K	128	+3K	256	+3K	384	+3K
;;	1	...
;;	...
;;	127	...
;;	512	...
;;	...
;;
;; Do 768 four_complex_fft operations
;;	distance between fft data elements is 128
;;	do 384 macros each processing 16 data values

;; Do FFT levels 5,6
;;
;; On input the 64-byte cache lines hold these data values:
;;	0	+3K	128	+3K	256	+3K	384	+3K
;;	1	...
;;	...
;;	127	...
;;	512	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+3K	32	+3K	64	+3K	96	+3K
;;	1	...
;;	...
;;	31	...
;;	128	...
;;	...
;;
;; Do 768 four_complex_fft operations
;;	distance between fft data elements is 32
;;	do 192 macros each processing 32 data values

;; Do FFT levels 7,8
;;
;; On input the 64-byte cache lines hold these data values:
;;	0	+3K	32	+3K	64	+3K	96	+3K
;;	1	...
;;	...
;;	31	...
;;	128	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+3K	8	+3K	16	+3K	24	+3K
;;	1	...
;;	...
;;	7	...
;;	32	...
;;	...
;;
;; Do 768 four_complex_fft operations
;;	distance between fft data elements is 8
;;	do 192 macros each processing 32 data values

;; Do this 32 cache lines at a time to maximize accessing data
;; while it is in the L1 cache.

	start_timer 6
	mov	rcx, DATA_PREFETCH	;; Prefetch pointer (we need to prefetch 
					;; 768 cache lines + 10 pad cache lines)
	xtouch	[rcx]			;; Load first prefetch TLBs
	mov	rbx, rdi		;; Copy sin/cos pointers
	mov	rbp, rdi
	loops_init_prefetch 384, 64, 4, rcx ;; 3*8 iters of 32 cache lines
	;r4_nf2cl_four_complex_djbfft_preload -- assume same as r4_x4cl_four_complex_djbfft_preload
	r4_x4cl_four_complex_djbfft_preload

b3a:	xtouch	[rcx+4096-64]		;; Load 2 prefetch TLBs
	xtouch	[rcx+2*4096-64]

b3b:	xxprefetch [rdi+8*XMM_SCD2]	;; Prefetch for next pass
	r4_nf2cl_four_complex_djbfft rsi, 8*64, dist128, rdi
	bump	rdi, 8*XMM_SCD2		;; Next sine/cosine pointer
	loops	16, b3b			;; Loop 16 times
	bump	rsi, -16*8*64		;; Restore source pointer
	bump	rdi, -16*8*XMM_SCD2	;; Restore sine/cosine pointer
	loops_undo 16			;; Back up loop counter

b4b:	r4_x4cl_four_complex_djbfft rsi, 8*64, 32*64, 64*64, rbx
	bump	rbx, 4*8*XMM_SCD2	;; Next sine/cosine pointer
	loops	4, b4b, 2		;; 4 iterations with non-standard increment
	bump	rsi, -4*8*64+dist128	;; Next source pointer
	bump	rbx, -4*4*8*XMM_SCD2	;; Restore sine/cosine pointer
	loops	2, b4b			;; Loop twice
	bump	rsi, -2*dist128		;; Restore source pointer
	loops_undo 2*4*2		;; Back up loop counter

b5b:	r4_x4cl_four_complex_djbfft rsi, 32*64, 8*64, 16*64, rbp
	loops	4, b5b, 2 		;; 4 iterations with non-standard increment
	bump	rsi, -4*32*64+dist128	;; Next source pointer
	loops	2, b5b	 		;; 2 iterations

	bump	rsi, -2*dist128+64	;; Next source pointer
	bump	rdi, XMM_SCD2		;; Next sine/cosine pointer
	bump	rbx, 4*XMM_SCD2		;; Next sine/cosine pointer
	bump	rbp, 16*XMM_SCD2	;; Next sine/cosine pointer
	loops	8, b3b			;; Loop 8 times
	bump	rsi, -8*64+2*dist128	;; Next source pointer
	bump	rdi, -8*XMM_SCD2	;; Restore sine/cosine pointer
	bump	rbx, -8*4*XMM_SCD2	;; Restore sine/cosine pointer
	bump	rbp, -8*16*XMM_SCD2	;; Restore sine/cosine pointer
	loops	3, b3a			;; Loop 3 times
	bump	rsi, -3*2*dist128	;; Restore source pointer
	end_timer 6

;; Do FFT levels 9,10,11
;;
;; On input the 64-byte cache lines hold these data values:
;;	0	+3K	8	+3K	16	+3K	24	+3K
;;	1	...
;;	...
;;	7	...
;;	32	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+3K	4	+3K	8	+3K	12	+3K
;;	1	...
;;	...
;;	3	...
;;	16	...
;;	...

;; Do 384 eight_complex_fft operations
;;	distance between fft data elements is 1
;;	do 96 macros each processing 64 data values

;; Execute the right middle step

	cmp	ffttype, 2
	jl	xpass2_1536_levels_complex_1
	jg	xpass2_1536_levels_complex_3

xpass2_1536_levels_complex_2:
	start_timer 9
	loops_init_prefetch 96, 64, 1, rcx
	r8_x8cl_eight_complex_with_square_preload
	xtouch	[rcx+4096-64]		;; Load prefetch TLBs
	xtouch	[rcx+2*4096-64]
b6a:	r8_x8cl_eight_complex_with_square rsi, 8*64, 64, 2*64, 4*64
	loops	16, b6a			;; Loop 16 times
	bump	rsi, -16*8*64+dist128	;; Next source pointer
	loops	6, b6a			;; Loop 6 times
	bump	rsi, -6*dist128		;; Restore source pointer
	end_timer 9
	jmp	xpass2_1536_levels_complex_unfft

xpass2_1536_levels_complex_1:
	start_timer 9
	loops_init_prefetch 96, 128, 1, rcx
	r8_x8cl_eight_complex_fft_final_preload
b6b0:	xtouch	[rcx+4096-64]		;; Load prefetch TLBs
b6b:	r8_x8cl_eight_complex_fft_final rsi, 8*64, 64, 2*64, 4*64
	loops	16, b6b			;; Loop 16 times
	bump	rsi, -16*8*64+dist128	;; Next source pointer
	loops_touch 2, b6b, 3, b6b0	;; Loop 6 times
	end_timer 9
	jmp	xpass2_1536_levels_complex_done

xpass2_1536_levels_complex_3:
	start_timer 9
	mov	rbp, DIST_TO_MULSRCARG
	loops_init_prefetch 96, 64, 1, rcx
	r8_x8cl_eight_complex_with_mult_preload
	xtouch	[rcx+4096-64]		;; Load prefetch TLBs
	xtouch	[rcx+2*4096-64]
b6c:	r8_x8cl_eight_complex_with_mult rsi, 8*64, 64, 2*64, 4*64
	loops	16, b6c			;; Loop 16 times
	bump	rsi, -16*8*64+dist128	;; Next source pointer
	loops	6, b6c			;; Loop 6 times
	bump	rsi, -6*dist128		;; Restore source pointer
	end_timer 9
	jmp	xpass2_1536_levels_complex_unfft

xpass2_1536_levels_complex_4:
	start_timer 9
	mov	rcx, DATA_PREFETCH	;; Prefetch pointer
	xtouch	[rcx]			;; Load first prefetch TLB
	mov	rbp, DIST_TO_MULSRCARG
	loops_init_prefetch 96, 128, 1, rcx
	r8_x8cl_eight_complex_with_mulf_preload
b6d0:	xtouch	[rcx+4096-64]		;; Load prefetch TLBs
b6d:	r8_x8cl_eight_complex_with_mulf rsi, 8*64, 64, 2*64, 4*64
	loops	16, b6d			;; Loop 16 times
	bump	rsi, -16*8*64+dist128	;; Next source pointer
	loops_touch 2, b6d, 3, b6d0	;; Loop 6 times
	bump	rsi, -6*dist128		;; Restore source pointer
	end_timer 9

;; Do inverse FFT levels 7,8
;; On input the 64-byte cache lines hold these data values:
;;	0	+3K	4	+3K	8	+3K	12	+3K
;;	1	...
;;	...
;;	3	...
;;	16	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+3K	16	+3K	32	+3K	48	+3K
;;	1	...
;;	...
;;	15	...
;;	64	...
;;	...
;;
;; Do 768 four_complex_unfft operations
;;	distance between fft data elements is 8
;;	do 192 macros each processing 32 data values

;; Do inverse FFT levels 5,6
;; On input the 64-byte cache lines hold these data values:
;;	0	+3K	16	+3K	32	+3K	48	+3K
;;	1	...
;;	...
;;	15	...
;;	64	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+3K	64	+3K	128	+3K	192	+3K
;;	1	...
;;	...
;;	63	...
;;	256	...
;;	...
;;
;; Do 768 four_complex_unfft operations
;;	distance between fft data elements is 32
;;	do 192 macros each processing 32 data values

;; Do inverse FFT levels 3,4
;; On input the 64-byte cache lines hold these data values:
;;	0	+3K	64	+3K	128	+3K	192	+3K
;;	1	...
;;	...
;;	63	...
;;	256	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+3K	256	+3K	512	+3K	768	+3K
;;	1	...
;;	...
;;	255	...
;;	1024	...
;;	...
;;
;; Do 768 four_complex_unfft operations
;;	distance between fft data elements is 128
;;	do 192 macros each processing 32 data values

;; Do this 64 cache lines at a time to maximize accessing data
;; while it is in the L1 cache.

xpass2_1536_levels_complex_unfft:
	start_timer 10
	mov	rdi, xsincos_complex	;; Load sin/cos pointers
	bump	rdi, 512*XMM_SCD1
	mov	rbx, rdi
	mov	rbp, rdi
	loops_init_prefetch 192, 64, 2, rcx ;; 3*4 iters of 64 cache lines
	r4_x4cl_four_complex_djbunfft_preload

c5a:	xtouch	[rcx+4096-64]		;; Load prefetch TLBs
	xtouch	[rcx+2*4096-64]

c5b:	r4_x4cl_four_complex_djbunfft rsi, 16*64, 4*64, 8*64, rbp, 4*16*XMM_SCD2
	loops	8, c5b			;; Loop 8 times
	bump	rsi, -8*16*64+dist128	;; Next source pointer
	loops	2, c5b			;; Loop 2 times
	bump	rsi, -2*dist128		;; Restore source pointer
	loops_undo 2*8			;; Restore loop counter

c4b:	r4_x4cl_four_complex_djbunfft rsi, 4*64, 16*64, 32*64, rbx, 16*4*XMM_SCD2
	bump	rbx, 4*4*XMM_SCD2	;; Next sine/cosine pointer
	loops	4, c4b			;; Loop 4 times
	bump	rsi, -4*4*64+64*64	;; Next source pointer
	bump	rbx, -4*4*4*XMM_SCD2	;; Restore sine/cosine pointer
	loops	2, c4b			;; Loop twice
	bump	rsi, -2*64*64+dist128	;; Next source pointer
	loops	2, c4b			;; Loop twice
	bump	rsi, -2*dist128		;; Restore source pointer
	loops_undo 2*2*4		;; Restore loop counter

c3b:	r4_x4cl_four_complex_djbunfft rsi, 4*64, 64*64, dist128, rdi, 64*1*XMM_SCD2
	bump	rdi, 4*1*XMM_SCD2	;; Next sine/cosine pointer
	loops	16, c3b			;; Loop 16 times
	bump	rdi, -16*4*1*XMM_SCD2	;; Restore sine/cosine pointer

	bump	rsi, -16*4*64+64	;; Next source pointer
	bump	rdi, 1*XMM_SCD2		;; Next sine/cosine pointer
	bump	rbx, 4*XMM_SCD2		;; Next sine/cosine pointer
	bump	rbp, 16*XMM_SCD2	;; Next sine/cosine pointer
	loops	4, c5b			;; Loop 4 times
	bump	rsi, -4*64+2*dist128	;; Next source pointer
	bump	rdi, -4*1*XMM_SCD2	;; Restore sine/cosine pointer
	bump	rbx, -4*4*XMM_SCD2	;; Restore sine/cosine pointer
	bump	rbp, -4*16*XMM_SCD2	;; Restore sine/cosine pointer
	loops	3, c5a			;; Loop 3 times
	bump	rsi, -3*2*dist128	;; Restore source pointer
	end_timer 10

;; Do inverse FFT levels 1,2 (actually 1.585 FFT levels)
;; On input the 64-byte cache lines hold these data values:
;;	0	+3K	256	+3K	512	+3K	768	+3K
;;	1	...
;;	...
;;	255	...
;;	1024	...
;;	...
;; On output the 64-byte cache lines hold these data values if pass 1 swizzles:
;;	0	+3K	768	+3K	1536	+3K	2304	+3K
;;	1	...
;;	...
;;	767
;; On output the 64-byte cache lines hold these data values if pass 2 swizzles:
;;	0	+768	1536	+768	3072	+768	4608	+768
;;	1	...
;;	...
;;	767

;; Do 1024 three_complex_unfft operations
;;	distance between fft data elements is 512
;;	do 256 macros each processing 24 data values

	start_timer 14
	mov	rdi, xsincos_complex	;; Load group multiplier pointer
	bump	rcx, -22*64		;; Back up prefetch pointer by 22 cache lines
	loops_init_prefetch 256, 64, 2, rcx ;; 2 iters of 128
	xtouch	[rcx+4096-64]		;; Load prefetch TLBs
	xtouch	[rcx+4096-64]		;; Load prefetch TLBs
	r3_x3cl_three_complex_djbunfft_preload
c1b:	r3_x3cl_three_complex_djbunfft rsi, 64, 2*dist128, rdi, 256*XMM_SCD1
	bump	rdi, XMM_SCD1		;; Next set of sin/cos data
	loops	128, c1b		;; Test inner loop counter
	bump	rsi, -128*64+dist128	;; Next source pointer
	loops	2, c1b			;; Test outer loop counter
	end_timer 14

xpass2_1536_levels_complex_done:
	ENDM


; *************** radix4-pass2-11-levels-real macro ******************
; This macro takes 4096 real values and performs the final 12 levels of
; the FFT.  Independently, this macro also takes 2048 complex values and
; performs the final 11 levels of the FFT.  The results are squared, and
; the first 11/12 levels of the inverse FFT are done.

r4_pass2_11_levels_real MACRO
	LOCAL	b1b, b4b, b5b, b6b, b7b
	LOCAL	b8b, b9b, baa, bab0, bab, bac, bad0, bad
	LOCAL	c1b, c4b, c5b, c6b, c7b, c8b, c9b
	LOCAL	xpass2_11_levels_real_1, xpass2_11_levels_real_2
	LOCAL	xpass2_11_levels_real_3, xpass2_11_levels_real_4
	LOCAL	xpass2_11_real_unfft, xpass2_11_real_done

;; We switch to the same format used in one-pass FFTs (that is, the
;; high word of xmm registers being 1 greater than the low word):

;; Type 4 FFTs skip the forward FFT process

	start_timer 2
	mov	rsi, DATA_ADDR		;; Load source address
	mov	rbx, DIST_TO_FFTSRCARG
	cmp	ffttype, 4
	je	xpass2_11_levels_real_4

;; Do FFT level 1,2
;;
;; On input the 64-byte cache lines hold these data values if pass 2 swizzles:
;;	0	+1K	2048	+1K	4096	+1K	6144	+1K
;;	1	...
;;	...
;;	1023
;; On input the 64-byte cache lines hold these data values if pass 1 swizzles:
;;	0	+4K	1024	+4K	2048	+4K	3072	+4K
;;	1	...
;;	...
;;	1023	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+4K	512	+4K	1024	+4K	1536	+4K
;;	1	...
;;	...
;;	511	...
;;	2048	...
;;	...

;; Do 512 eight_reals_fft operations
;; Do 512 four_complex_djbfft operations
;;	distance between fft data elements is 512
;;	do 512 macros each processing 16 data values

	mov	rdi, xsincos_complex	;; Ptr to the sin/cos data
	mov	rdx, sincos3		;; Ptr to real sin/cos data
	loops_init 512
b1b:	r4_fh2cl_eight_reals_four_complex_djbfft rsi, 64, 4*dist128, rdi, rdx
	bump	rdi, XMM_SCD2		;; Next sin/cos data
	bump	rdx, XMM_HSCD2		;; Next sin/cos data
	loops	128, b1b		;; Test loop counter
	bump	rsi, -128*64+dist128	;; Next source pointer
	loops	4, b1b			;; Test loop counter
	bump	rsi, -4*dist128		;; Restore source pointer
	bump	rdx, -512*XMM_HSCD2	;; Restore sin/cos ptr

;; Do FFT levels 3,4
;;
;; On input the 64-byte cache lines hold these data values:
;;	0	+4K	512	+4K	1024	+4K	1536	+4K
;;	1	...
;;	...
;;	511	...
;;	2048	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+4K	128	+4K	256	+4K	384	+4K
;;	1	...
;;	...
;;	127	...
;;	512	...
;;	...

;; Do 128 eight_reals_fft operations
;; Do 384 four_complex_djbfft operations
;;	distance between fft data elements is 128
;;	do 128 macros each processing 32 data values

	mov	rcx, DATA_PREFETCH	;; Prefetch pointer
	xtouch	[rcx]			;; Preload the first TLB
	loops_init_prefetch 256, 64, 2, rcx
	xtouch	[rcx+4096-64]		;; Preload the TLBs
	xtouch	[rcx+2*4096-64]
	r4_h4cl_eight_reals_four_complex_djbfft_preload
b4b:	r4_h4cl_eight_reals_four_complex_djbfft rsi, 64, dist128, 2*dist128, rdi, rdx
	bump	rdi, XMM_SCD2		;; Next sin/cos data
	bump	rdx, 4*XMM_HSCD2	;; Next sin/cos data
	loops	128, b4b		;; Test loop counter
	bump	rsi, -128*64+4*dist128	;; Next source pointer
	bump	rdi, -128*XMM_SCD2	;; Restore sin/cos ptr
	bump	rdx, -128*4*XMM_HSCD2	;; Restore sin/cos ptr

;; Do 512 four_complex_djbfft operations
;;	distance between fft data elements is 128
;;	do 128 macros each processing 32 data values

	loops_reset			;; 1 more iterations of 128
	r4_x4cl_four_complex_djbfft_preload
b5b:	r4_x4cl_four_complex_djbfft rsi, 64, dist128, 2*dist128, rdi
	bump	rdi, XMM_SCD2		;; Next sin/cos data
	loops	128, b5b		;; Test inner loop counter
	bump	rsi, -128*64+4*dist128	;; Next source pointer
	bump	rdi, -128*XMM_SCD2	;; Restore sin/cos ptr
	loops	2, b5b			;; Test loop counter
	bump	rsi, -2*4*dist128	;; Next source pointer
	bump	rcx, 128		;; Skip pad bytes

;; Do FFT levels 5,6
;;
;; On input the 64-byte cache lines hold these data values:
;;	0	+4K	128	+4K	256	+4K	384	+4K
;;	1	...
;;	...
;;	127	...
;;	512	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+4K	32	+4K	64	+4K	96	+4K
;;	1	...
;;	...
;;	31	...
;;	128	...
;;	...

;; Do 32 eight_reals_fft operations
;; Do 96 four_complex_djbfft operations
;;	distance between fft data elements is 32
;;	do 32 macros each processing 32 data values

	loops_init_prefetch 256, 64, 2, rcx
	xtouch	[rcx+4096-64]		;; Preload the TLBs
	xtouch	[rcx+2*4096-64]
	r4_h4cl_eight_reals_four_complex_djbfft_preload
b6b:	r4_h4cl_eight_reals_four_complex_djbfft rsi, 64, 32*64, 64*64, rdi, rdx
	bump	rdi, 4*XMM_SCD2		;; Next sin/cos data
	bump	rdx, 16*XMM_HSCD2	;; Next sin/cos data
	loops	32, b6b			;; Test loop counter
	bump	rsi, -32*64+dist128	;; Next source pointer
	bump	rdi, -32*4*XMM_SCD2	;; Restore sin/cos ptr
	bump	rdx, -32*16*XMM_HSCD2	;; Restore sin/cos ptr

;; Do 896 four_complex_djbfft operations
;;	distance between fft data elements is 32
;;	do 224 macros each processing 32 data values

	loops_reset			;; 7 more iters of 32
	r4_x4cl_four_complex_djbfft_preload
b7b:	r4_x4cl_four_complex_djbfft rsi, 64, 32*64, 64*64, rdi
	bump	rdi, 4*XMM_SCD2		;; Next sin/cos data
	loops	32, b7b			;; Test inner loop counter
	bump	rsi, -32*64+dist128	;; Next source pointer
	bump	rdi, -32*4*XMM_SCD2	;; Restore sin/cos ptr
	loops	8, b7b			;; 8 iterations
	bump	rsi, -8*dist128		;; Next source pointer
	bump	rcx, 128		;; Skip pad bytes

;; Do FFT levels 7,8
;;
;; On input the 64-byte cache lines hold these data values:
;;	0	+4K	32	+4K	64	+4K	96	+4K
;;	1	...
;;	...
;;	31	...
;;	128	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+4K	8	+4K	16	+4K	24	+4K
;;	1	...
;;	...
;;	7	...
;;	32	...
;;	...

;; Do 8 eight_reals_fft operations
;; Do 24 four_complex_djbfft operations
;;	distance between fft data elements is 8
;;	do 8 macros each processing 32 data values

	loops_init_prefetch 256, 64, 2, rcx
	xtouch	[rcx+4096-64]		;; Preload the TLBs
	xtouch	[rcx+2*4096-64]
	r4_h4cl_eight_reals_four_complex_djbfft_preload
b8b:	r4_h4cl_eight_reals_four_complex_djbfft rsi, 64, 8*64, 16*64, rdi, rdx
	bump	rdi, 16*XMM_SCD2	;; Next sin/cos data
	bump	rdx, 64*XMM_HSCD2	;; Next sin/cos data
	loops	8, b8b			;; Test loop counter
	bump	rsi, -8*64+32*64	;; Next source pointer
	bump	rdi, -8*16*XMM_SCD2	;; Restore sin/cos ptr
	bump	rdx, -8*64*XMM_HSCD2	;; Restore sin/cos ptr

;; Do 992 four_complex_djbfft operations
;;	distance between fft data elements is 8
;;	do 248 macros each processing 32 data values

	loops_reset			;; 8 iters of 3 or 4 of 8
	r4_x4cl_four_complex_djbfft_preload
b9b:	r4_x4cl_four_complex_djbfft rsi, 64, 8*64, 16*64, rdi
	bump	rdi, 16*XMM_SCD2	;; Next sin/cos data
	loops	8, b9b			;; Test loop counter
	bump	rsi, -8*64+32*64	;; Next source pointer
	bump	rdi, -8*16*XMM_SCD2	;; Restore sin/cos ptr
	loops	4, b9b			;; Test middle loop counter
	bump	rsi, -4*32*64+dist128	;; Next source pointer
	loops	8, b9b			;; 8 iterations
	bump	rsi, -8*dist128		;; Next source pointer
	bump	rcx, 128		;; Skip pad bytes

;; Do FFT levels 9,10,11
;;
;; On input the 64-byte cache lines hold these data values:
;;	0	+4K	8	+4K	16	+4K	24	+4K
;;	1	...
;;	...
;;	7	...
;;	32	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+4K	4	+4K	8	+4K	12	+4K
;;	1	...
;;	...
;;	3	...
;;	16	...
;;	...

;; Do 1 sixteen_reals_fft operations
;; Do 3 eight_complex_fft operations
;;	distance between fft data elements is 1

;; Do 508 eight_complex_fft operations
;;	distance between fft data elements is 1
;;	do 127 macros each processing 64 data values

;; Execute the proper middle step

	cmp	ffttype, 2
	jl	xpass2_11_levels_real_1
	jg	xpass2_11_levels_real_3

xpass2_11_levels_real_2:
	r8_h8cl_sixteen_reals_eight_complex_with_square rsi, 8*64, 64, 2*64, 4*64
	loops_init_prefetch 128, 64, 1, rcx, 1, 16
	r8_x8cl_eight_complex_with_square_preload
	xtouch	[rcx+4096-128]		;; Preload the TLBs
	xtouch	[rcx+2*4096-128]	;; Preload the TLBs
baa:	r8_x8cl_eight_complex_with_square rsi, 8*64, 64, 2*64, 4*64
	loops	16, baa			;; Test loop counter
	bump	rsi, -16*8*64+dist128	;; Next source pointer
	loops	8, baa			;; Test loop counter
	bump	rsi, -8*dist128		;; Next source pointer
	bump	rcx, 128+128		;; Skip pad bytes and the one missed prefetch
	jmp	xpass2_11_real_unfft

xpass2_11_levels_real_1:
	r8_h8cl_sixteen_reals_eight_complex_fft_final rsi, 8*64, 64, 2*64, 4*64
	loops_init_prefetch 128, 128, 1, rcx, 1, 16
	r8_x8cl_eight_complex_fft_final_preload
bab0:	xtouch	[rcx+4096-64]		;; Preload the TLBs
bab:	r8_x8cl_eight_complex_fft_final rsi, 8*64, 64, 2*64, 4*64
	loops	16, bab			;; Test loop counter
	bump	rsi, -16*8*64+dist128	;; Next source pointer
	loops_touch 2, bab, 4, bab0	;; Test loop counter
	jmp	xpass2_11_real_done

xpass2_11_levels_real_3:
	mov	rbp, DIST_TO_MULSRCARG
	r8_h8cl_sixteen_reals_eight_complex_with_mult rsi, 8*64, 64, 2*64, 4*64
	loops_init_prefetch 128, 64, 1, rcx, 1, 16
	r8_x8cl_eight_complex_with_mult_preload
	xtouch	[rcx+4096-64]		;; Preload the TLBs
	xtouch	[rcx+2*4096-64]
bac:	r8_x8cl_eight_complex_with_mult rsi, 8*64, 64, 2*64, 4*64
	loops	16, bac			;; Test loop counter
	bump	rsi, -16*8*64+dist128	;; Next source pointer
	loops	8, bac			;; Test loop counter
	bump	rsi, -8*dist128		;; Next source pointer
	bump	rcx, 128+128		;; Skip pad bytes and the one missed prefetch
	jmp	xpass2_11_real_unfft

xpass2_11_levels_real_4:
	mov	rbp, DIST_TO_MULSRCARG
	mov	rcx, DATA_PREFETCH	;; Load prefetch pointer
	xtouch	[rcx]			;; Preload the first TLB
	r8_h8cl_sixteen_reals_eight_complex_with_mulf rsi, 8*64, 64, 2*64, 4*64
	loops_init_prefetch 128, 128, 1, rcx, 1, 16
	r8_x8cl_eight_complex_with_mulf_preload
bad0:	xtouch	[rcx+4096-64]		;; Preload the TLBs
bad:	r8_x8cl_eight_complex_with_mulf rsi, 8*64, 64, 2*64, 4*64
	loops	16, bad			;; Test loop counter
	bump	rsi, -16*8*64+dist128	;; Next source pointer
	loops_touch 2, bad, 4, bad0	;; Test loop counter
	bump	rsi, -8*dist128		;; Next source pointer
	bump	rcx, 2*128+128		;; Skip pad bytes and the one missed prefetch

;; Do inverse FFT levels 7,8
;; On input the 64-byte cache lines hold these data values:
;;	0	+4K	4	+4K	8	+4K	12	+4K
;;	1	...
;;	...
;;	3	...
;;	16	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+4K	16	+4K	32	+4K	48	+4K
;;	1	...
;;	...
;;	15	...
;;	64	...
;;	...

;; Do 8 eight_reals_unfft operations
;; Do 8 four_complex_djbunfft operations
;;	distance between fft data elements is 8
;;	do 4 macros processing 32 data values

xpass2_11_real_unfft:
	mov	rdi, xsincos_complex	;; Load sin/cos pointers
	bump	rdi, 512*XMM_SCD2
	mov	rdx, sincos3
	loops_init_prefetch 256, 64, 2, rcx
	xtouch	[rcx+4096-128]
	xtouch	[rcx+2*4096-128]
c8b:	r4_h4cl_eight_reals_four_complex_djbunfft rsi, 64, 4*64, 8*64, rdi, 4*16*XMM_SCD2, rdx, 4*64*XMM_HSCD2
	bump	rdi, 16*XMM_SCD2	;; Next sin/cos data
	bump	rdx, 64*XMM_HSCD2	;; Next sin/cos data
	loops	4, c8b			;; Test loop counter
	bump	rsi, -4*64+16*64	;; Next source pointer
	bump	rdi, -4*16*XMM_SCD2	;; Restore sin/cos ptr
	bump	rdx, -4*64*XMM_HSCD2	;; Restore sin/cos ptr

;; Do 1008 four_complex_djbunfft operations
;;	distance between fft data elements is 8
;;	do 252 macros each processing 32 data values

	loops_reset			;; 8 iters of 7 or 8 of 4
	r4_x4cl_four_complex_djbunfft_preload
c9b:	r4_x4cl_four_complex_djbunfft rsi, 64, 4*64, 8*64, rdi, 4*16*XMM_SCD2
	bump	rdi, 16*XMM_SCD2	;; Next sine/cosine pointer
	loops	4, c9b			;; Test loop counter
	bump	rsi, -4*64+16*64	;; Next source pointer
	bump	rdi, -4*16*XMM_SCD2	;; Restore sin/cos ptr
	loops	8, c9b			;; Test loop counter
	bump	rsi, -8*16*64+dist128	;; Next source pointer
	loops	8, c9b			;; Test loop counter
	bump	rsi, -8*dist128	;; Next source pointer
	bump	rcx, 128		;; Skip pad bytes

;; Do inverse FFT levels 5,6
;; On input the 64-byte cache lines hold these data values:
;;	0	+4K	16	+4K	32	+4K	48	+4K
;;	1	...
;;	...
;;	15	...
;;	64	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+4K	64	+4K	128	+4K	192	+4K
;;	1	...
;;	...
;;	63	...
;;	256	...
;;	...

;; Do 32 eight_reals_unfft operations
;; Do 32 four_complex_djbunfft operations
;;	distance between fft data elements is 32
;;	do 16 macros processing 32 data values

	loops_init_prefetch 256, 64, 2, rcx
	xtouch	[rcx+4096-64]		;; Preload TLBs
	xtouch	[rcx+2*4096-64]
c6b:	r4_h4cl_eight_reals_four_complex_djbunfft rsi, 64, 16*64, 32*64, rdi, 16*4*XMM_SCD2, rdx, 16*16*XMM_HSCD2
	bump	rdi, 4*XMM_SCD2		;; Next sin/cos data
	bump	rdx, 16*XMM_HSCD2	;; Next sin/cos data
	loops	16, c6b			;; Test inner loop counter
	bump	rsi, -16*64+64*64	;; Next source pointer
	bump	rdi, -16*4*XMM_SCD2	;; Restore sin/cos ptr
	bump	rdx, -16*16*XMM_HSCD2	;; Restore sin/cos ptr

;; Do 960 four_complex_djbunfft operations
;;	distance between fft data elements is 32
;;	do 240 macros each processing 32 data values

	loops_reset			;; 8 iters of 1 or 2 of 16
	r4_x4cl_four_complex_djbunfft_preload
c7b:	r4_x4cl_four_complex_djbunfft rsi, 64, 16*64, 32*64, rdi, 16*4*XMM_SCD2
	bump	rdi, 4*XMM_SCD2		;; Next sin/cos data
	loops	16, c7b			;; Test inner loop counter
	bump	rsi, -16*64+64*64	;; Next source pointer
	bump	rdi, -16*4*XMM_SCD2	;; Restore sin/cos ptr
	loops	2, c7b			;; Test loop counter
	bump	rsi, -2*64*64+dist128	;; Next source pointer
	loops	8, c7b			;; 8 iterations
	bump	rsi, -8*dist128		;; Next source pointer
	bump	rcx, 128		;; Skip pad bytes

;; Do inverse FFT levels 3,4
;; On input the 64-byte cache lines hold these data values:
;;	0	+4K	64	+4K	128	+4K	192	+4K
;;	1	...
;;	...
;;	63	...
;;	256	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+4K	256	+4K	512	+4K	768	+4K
;;	1	...
;;	...
;;	255	...
;;	1024	...
;;	...

;; Do 128 eight_reals_unfft operations
;; Do 128 four_complex_djbunfft operations
;;	distance between fft data elements is 128
;;	do 64 macros processing 32 data values

	loops_init_prefetch 256, 64, 2, rcx
	xtouch	[rcx+4096-128]		;; Preload TLBs
	xtouch	[rcx+2*4096-128]
c4b:	r4_h4cl_eight_reals_four_complex_djbunfft rsi, 64, 64*64, dist128, rdi, 64*XMM_SCD2, rdx, 64*4*XMM_HSCD2
	bump	rdi, XMM_SCD2		;; Next sin/cos data
	bump	rdx, 4*XMM_HSCD2	;; Next sin/cos data
	loops	64, c4b			;; Test inner loop counter
	bump	rsi, -64*64+2*dist128	;; Next source pointer
	bump	rdi, -64*XMM_SCD2	;; Restore sin/cos ptr
	bump	rdx, -64*4*XMM_HSCD2	;; Restore sin/cos ptr

;; Do 384 four_complex_djbunfft operations
;;	distance between fft data elements is 128
;;	do 192 macros each processing 32 data values

	loops_reset			;; 3 more iters of 64
	r4_x4cl_four_complex_djbunfft_preload
c5b:	r4_x4cl_four_complex_djbunfft rsi, 64, 64*64, dist128, rdi, 64*XMM_SCD2
	bump	rdi, XMM_SCD2		;; Next sin/cos data
	loops	64, c5b			;; Test inner loop counter
	bump	rsi, -64*64+2*dist128	;; Next source pointer
	bump	rdi, -64*XMM_SCD2	;; Restore sin/cos ptr
	loops	4, c5b			;; Test loop counter
	bump	rsi, -4*2*dist128	;; Next source pointer
	bump	rcx, 128		;; Skip pad bytes

;; Do inverse FFT level 1,2
;; On input the 64-byte cache lines hold these data values:
;;	0	+4K	256	+4K	512	+4K	768	+4K
;;	1	...
;;	...
;;	255	...
;;	1024	...
;;	...
;; On output the 64-byte cache lines hold these data values if pass 1 swizzles:
;;	0	+4K	1024	+4K	2048	+4K	3072	+4K
;;	1	...
;;	...
;;	1023
;; On output the 64-byte cache lines hold these data values if pass 2 swizzles:
;;	0	+1K	2048	+1K	4096	+1K	6144	+1K
;;	1	...
;;	...
;;	1023

;; Do 512 eight_reals_unfft operations
;; Do 512 four_complex_djbunfft operations
;;	distance between fft data elements is 512
;;	do 256 macros processing 32 data values

	mov	rdi, xsincos_complex	;; Load sin/cos pointers
	loops_init_prefetch 256, 64, 2, rcx
	xtouch	[rcx+4096-64]		;; Preload TLBs
	xtouch	[rcx+2*4096-64]
c1b:	r4_h4cl_eight_reals_four_complex_djbunfft rsi, 64, 2*dist128, 4*dist128, rdi, 256*XMM_SCD2, rdx, 256*XMM_HSCD2
	bump	rdi, XMM_SCD2		;; Next sin/cos data
	bump	rdx, XMM_HSCD2		;; Next sin/cos data
	loops	128, c1b		;; Test loop counter
	bump	rsi, -128*64+dist128	;; Next source pointer
	loops	2, c1b			;; Test loop counter

xpass2_11_real_done:
	end_timer 2
	ENDM

; *************** radix4-pass2-11-levels-complex macro ******************
; This macro takes 2 groups of 2048 complex values and
; performs the final 11 levels of a standard mostly radix-4 FFT, squares the
; results, and does the first 11 levels of the inverse FFT.

r4_pass2_11_levels_complex MACRO
	LOCAL	b1b, b3b, b4b, b5b, b6a, b6b0, b6b, b6c, b6d0, b6d
	LOCAL	c1b, c3b, c4b, c5b
	LOCAL	xpass2_11_levels_complex_1, xpass2_11_levels_complex_2
	LOCAL	xpass2_11_levels_complex_3, xpass2_11_levels_complex_4
	LOCAL	xpass2_11_levels_complex_unfft, xpass2_11_levels_complex_done

;; Type 4 FFTs skip the forward FFT process

	mov	rsi, DATA_ADDR		;; Load address of FFT data
	mov	rbx, DIST_TO_FFTSRCARG
	cmp	ffttype, 4
	je	xpass2_11_levels_complex_4

;; Do FFT levels 1,2
;;
;; We swizzle the data so that one set of data is in the high word of
;; an xmm register, the other set is in the low word.  This means
;; a distance of 4096 data values.
;;
;; On input the 64-byte cache lines hold these data values if pass 2 swizzles:
;;	0	+1K	2048	+1K	4096	+1K	6144	+1K
;;	1	...
;;	...
;;	1023
;; On input the 64-byte cache lines hold these data values if pass 1 swizzles:
;;	0	+4K	1024	+4K	2048	+4K	3072	+4K
;;	1	...
;;	...
;;	1023	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+4K	512	+4K	1024	+4K	1536	+4K
;;	1	...
;;	...
;;	255	...
;;	1024	...
;;	...

;; Do 1024 four_complex_fft operations
;;	distance between fft data elements is 512
;;	do 512 macros each processing 16 data values

	start_timer 4
	mov	rdi, xsincos_complex	;; The sin/cos data
	loops_init 512			;; 4 iterations of 128
	r4_f2cl_four_complex_djbfft_preload
b1b:	xxprefetch [rdi+XMM_SCD2]	;; Prefetch for next pass
	r4_f2cl_four_complex_djbfft rsi, 64, 4*dist128, rdi
	bump	rdi, XMM_SCD2		;; Next sin/cos data
	loops	128, b1b		;; Test inner loop counter
	bump	rsi, -128*64+dist128	;; Next source pointer
	loops	4, b1b			;; Test outer loop counter
	bump	rsi, -4*dist128		;; Restore source pointer
	end_timer 4

;; Do FFT levels 3,4
;;
;; On input the 64-byte cache lines hold these data values:
;;	0	+4K	512	+4K	1024	+4K	1536	+4K
;;	1	...
;;	...
;;	255	...
;;	1024	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+4K	128	+4K	256	+4K	384	+4K
;;	1	...
;;	...
;;	127	...
;;	512	...
;;	...
;;
;; Do 1024 four_complex_fft operations
;;	distance between fft data elements is 128
;;	do 256 macros each processing 32 data values

;; Do FFT levels 5,6
;;
;; On input the 64-byte cache lines hold these data values:
;;	0	+4K	128	+4K	256	+4K	384	+4K
;;	1	...
;;	...
;;	127	...
;;	512	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+4K	32	+4K	64	+4K	96	+4K
;;	1	...
;;	...
;;	31	...
;;	128	...
;;	...
;;
;; Do 1024 four_complex_fft operations
;;	distance between fft data elements is 32
;;	do 256 macros each processing 32 data values

;; Do FFT levels 7,8
;;
;; On input the 64-byte cache lines hold these data values:
;;	0	+4K	32	+4K	64	+4K	96	+4K
;;	1	...
;;	...
;;	31	...
;;	128	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+4K	8	+4K	16	+4K	24	+4K
;;	1	...
;;	...
;;	7	...
;;	32	...
;;	...
;;
;; Do 1024 four_complex_fft operations
;;	distance between fft data elements is 8
;;	do 256 macros each processing 32 data values

;; Do this 64 cache lines at a time to maximize accessing data
;; while it is in the L1 cache.

	start_timer 6
	mov	rcx, DATA_PREFETCH	;; Prefetch pointer (we need to prefetch 
					;; 1024 cache lines + 14 pad cache lines)
	xtouch	[rcx]			;; Load first prefetch TLB
	mov	rbx, rdi		;; Copy sin/cos pointers
	mov	rbp, rdi
	loops_init_prefetch 256, 64, 2, rcx ;; 2*8 iters of 64 cache lines
	r4_x4cl_four_complex_djbfft_preload

	xtouch	[rcx+4096-64]		;; Load some prefetch TLBs
	xtouch	[rcx+2*4096-64]
	xtouch	[rcx+3*4096-64]
	xtouch	[rcx+4*4096-64]
	xtouch	[rcx+5*4096-64]
	xtouch	[rcx+6*4096-64]

b3b:	xxprefetch [rdi+8*XMM_SCD2]	;; Prefetch for next pass
	r4_x4cl_four_complex_djbfft rsi, 8*64, dist128, 2*dist128, rdi
	bump	rdi, 8*XMM_SCD2		;; Next sine/cosine pointer
	loops	16, b3b			;; Loop 16 times
	bump	rsi, -16*8*64		;; Restore source pointer
	bump	rdi, -16*8*XMM_SCD2	;; Restore sine/cosine pointer
	bump	rcx, -8*64+8192+128	;; Prefetch from next 8KB page
	loops_undo 16			;; Back up loop counter

b4b:	r4_x4cl_four_complex_djbfft rsi, 8*64, 32*64, 64*64, rbx
	bump	rbx, 8*4*XMM_SCD2	;; Next sine/cosine pointer
	loops	4, b4b			;; Loop 4 times
	bump	rsi, -4*8*64+dist128	;; Next source pointer
	bump	rbx, -4*8*4*XMM_SCD2	;; Restore sine/cosine pointer
	loops	4, b4b			;; Loop 4 times
	bump	rsi, -4*dist128		;; Restore source pointer
	bump	rcx, -8*64+8192+128	;; Prefetch from next 8KB page
	loops_undo 4*4			;; Back up loop counter

b5b:	r4_x4cl_four_complex_djbfft rsi, 32*64, 8*64, 16*64, rbp
	loops	4, b5b			;; Loop 8 times
	bump	rsi, -4*32*64+dist128	;; Next source pointer
	loops	4, b5b			;; Loop twice
	bump	rcx, -2*(8192+128)	;; Back to prefetching from 1st 8KB page

	bump	rsi, -4*dist128+64	;; Next source pointer
	bump	rdi, XMM_SCD2		;; Next sine/cosine pointer
	bump	rbx, 4*XMM_SCD2		;; Next sine/cosine pointer
	bump	rbp, 16*XMM_SCD2	;; Next sine/cosine pointer
	loops	8, b3b			;; Loop 8 times
	bump	rsi, -8*64+4*dist128	;; Next source pointer
	bump	rdi, -8*XMM_SCD2	;; Restore sine/cosine pointer
	bump	rbx, -8*4*XMM_SCD2	;; Restore sine/cosine pointer
	bump	rbp, -8*16*XMM_SCD2	;; Restore sine/cosine pointer
	loops	2, b3b			;; Loop 2 times
	bump	rsi, -2*4*dist128	;; Restore source pointer
	bump	rcx, -2*8*8*64+3*(8192+128) ;; Move prefetch pointer along
	end_timer 6

;; Do FFT levels 9,10,11
;;
;; On input the 64-byte cache lines hold these data values:
;;	0	+4K	8	+4K	16	+4K	24	+4K
;;	1	...
;;	...
;;	7	...
;;	32	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+4K	4	+4K	8	+4K	12	+4K
;;	1	...
;;	...
;;	3	...
;;	16	...
;;	...

;; Do 512 eight_complex_fft operations
;;	distance between fft data elements is 1
;;	do 128 macros each processing 64 data values

;; Execute the right middle step

	cmp	ffttype, 2
	jl	xpass2_11_levels_complex_1
	jg	xpass2_11_levels_complex_3

xpass2_11_levels_complex_2:
	start_timer 9
	loops_init_prefetch 128, 64, 1, rcx
	r8_x8cl_eight_complex_with_square_preload
	xtouch	[rcx+4096-64]		;; Load prefetch TLBs
	xtouch	[rcx+2*4096-64]
b6a:	r8_x8cl_eight_complex_with_square rsi, 8*64, 64, 2*64, 4*64
	loops	16, b6a			;; Loop 16 times
	bump	rsi, -16*8*64+dist128	;; Next source pointer
	loops	8, b6a			;; Loop 8 times
	bump	rsi, -8*dist128		;; Restore source pointer
	bump	rcx, 128		;; Skip pad bytes
	end_timer 9
	jmp	xpass2_11_levels_complex_unfft

xpass2_11_levels_complex_1:
	start_timer 9
	loops_init_prefetch 128, 128, 1, rcx
	r8_x8cl_eight_complex_fft_final_preload
b6b0:	xtouch	[rcx+4096-64]		;; Load prefetch TLBs
b6b:	r8_x8cl_eight_complex_fft_final rsi, 8*64, 64, 2*64, 4*64
	loops	16, b6b			;; Loop 16 times
	bump	rsi, -16*8*64+dist128	;; Next source pointer
	loops_touch 2, b6b, 4, b6b0	;; Loop 8 times
	end_timer 9
	jmp	xpass2_11_levels_complex_done

xpass2_11_levels_complex_3:
	start_timer 9
	mov	rbp, DIST_TO_MULSRCARG
	loops_init_prefetch 128, 64, 1, rcx
	r8_x8cl_eight_complex_with_mult_preload
	xtouch	[rcx+4096-64]		;; Load prefetch TLBs
	xtouch	[rcx+2*4096-64]
b6c:	r8_x8cl_eight_complex_with_mult rsi, 8*64, 64, 2*64, 4*64
	loops	16, b6c			;; Loop 16 times
	bump	rsi, -16*8*64+dist128	;; Next source pointer
	loops	8, b6c			;; Loop 8 times
	bump	rsi, -8*dist128		;; Restore source pointer
	bump	rcx, 128		;; Skip pad bytes
	end_timer 9
	jmp	xpass2_11_levels_complex_unfft

xpass2_11_levels_complex_4:
	start_timer 9
	mov	rcx, DATA_PREFETCH	;; Prefetch pointer
	xtouch	[rcx]			;; Load first prefetch TLB
	mov	rbp, DIST_TO_MULSRCARG
	loops_init_prefetch 128, 128, 1, rcx
	r8_x8cl_eight_complex_with_mulf_preload
b6d0:	xtouch	[rcx+4096-64]		;; Load prefetch TLBs
b6d:	r8_x8cl_eight_complex_with_mulf rsi, 8*64, 64, 2*64, 4*64
	loops	16, b6d			;; Loop 16 times
	bump	rsi, -16*8*64+dist128	;; Next source pointer
	loops_touch 2, b6d, 4, b6d0	;; Loop 8 times
	bump	rsi, -8*dist128		;; Restore source pointer
	bump	rcx, 2*128		;; Skip pad bytes
	end_timer 9

;; Do inverse FFT levels 7,8
;; On input the 64-byte cache lines hold these data values:
;;	0	+4K	4	+4K	8	+4K	12	+4K
;;	1	...
;;	...
;;	3	...
;;	16	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+4K	16	+4K	32	+4K	48	+4K
;;	1	...
;;	...
;;	15	...
;;	64	...
;;	...
;;
;; Do 1024 four_complex_unfft operations
;;	distance between fft data elements is 8
;;	do 256 macros each processing 32 data values

;; Do inverse FFT levels 5,6
;; On input the 64-byte cache lines hold these data values:
;;	0	+4K	16	+4K	32	+4K	48	+4K
;;	1	...
;;	...
;;	15	...
;;	64	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+4K	64	+4K	128	+4K	192	+4K
;;	1	...
;;	...
;;	63	...
;;	256	...
;;	...
;;
;; Do 1024 four_complex_unfft operations
;;	distance between fft data elements is 32
;;	do 256 macros each processing 32 data values

;; Do inverse FFT levels 3,4
;; On input the 64-byte cache lines hold these data values:
;;	0	+4K	64	+4K	128	+4K	192	+4K
;;	1	...
;;	...
;;	63	...
;;	256	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+4K	256	+4K	512	+4K	768	+4K
;;	1	...
;;	...
;;	255	...
;;	1024	...
;;	...
;;
;; Do 1024 four_complex_unfft operations
;;	distance between fft data elements is 128
;;	do 256 macros each processing 32 data values

;; Do this 64 cache lines at a time to maximize accessing data
;; while it is in the L1 cache.

xpass2_11_levels_complex_unfft:
	start_timer 10
	mov	rdi, xsincos_complex	;; Load sin/cos pointers
	bump	rdi, 512*XMM_SCD2
	mov	rbx, rdi
	mov	rbp, rdi
	loops_init_prefetch 256, 64, 2, rcx ;; 4*4 iters of 64 cache lines
	r4_x4cl_four_complex_djbunfft_preload

	xtouch	[rcx+4096-64]		;; Load some prefetch TLBs
	xtouch	[rcx+2*4096-64]
	xtouch	[rcx+3*4096-64]
	xtouch	[rcx+4*4096-64]
	xtouch	[rcx+5*4096-64]
	xtouch	[rcx+6*4096-64]

c5b:	r4_x4cl_four_complex_djbunfft rsi, 16*64, 4*64, 8*64, rbp, 4*16*XMM_SCD2
	loops	8, c5b			;; Loop 8 times
	bump	rsi, -8*16*64+dist128	;; Next source pointer
	loops	2, c5b			;; Loop 2 times
	bump	rsi, -2*dist128		;; Restore source pointer
	bump	rcx, -8*64+8192+128	;; Prefetch from next 8KB page
	loops_undo 2*8			;; Restore loop counter

c4b:	r4_x4cl_four_complex_djbunfft rsi, 4*64, 16*64, 32*64, rbx, 16*4*XMM_SCD2
	bump	rbx, 4*4*XMM_SCD2	;; Next sine/cosine pointer
	loops	4, c4b			;; Loop 4 times
	bump	rsi, -4*4*64+64*64	;; Next source pointer
	bump	rbx, -4*4*4*XMM_SCD2	;; Restore sine/cosine pointer
	loops	2, c4b			;; Loop twice
	bump	rsi, -2*64*64+dist128	;; Next source pointer
	loops	2, c4b			;; Loop twice
	bump	rsi, -2*dist128		;; Restore source pointer
	bump	rcx, -8*64+8192+128	;; Prefetch from next 8KB page
	loops_undo 2*2*4		;; Restore loop counter

c3b:	r4_x4cl_four_complex_djbunfft rsi, 4*64, 64*64, dist128, rdi, 64*1*XMM_SCD2
	bump	rdi, 4*1*XMM_SCD2	;; Next sine/cosine pointer
	loops	16, c3b			;; Loop 16 times
	bump	rdi, -16*4*1*XMM_SCD2	;; Restore sine/cosine pointer
	bump	rcx, -2*(8192+128)	;; Back to prefetching from 1st 8KB page

	bump	rsi, -16*4*64+64	;; Next source pointer
	bump	rdi, 1*XMM_SCD2		;; Next sine/cosine pointer
	bump	rbx, 4*XMM_SCD2		;; Next sine/cosine pointer
	bump	rbp, 16*XMM_SCD2	;; Next sine/cosine pointer
	loops	4, c5b			;; Loop 4 times
	bump	rsi, -4*64+2*dist128	;; Next source pointer
	bump	rdi, -4*1*XMM_SCD2	;; Restore sine/cosine pointer
	bump	rbx, -4*4*XMM_SCD2	;; Restore sine/cosine pointer
	bump	rbp, -4*16*XMM_SCD2	;; Restore sine/cosine pointer
	loops	4, c5b			;; Loop 4 times
	bump	rsi, -4*2*dist128	;; Restore source pointer
	bump	rcx, -4*4*8*64+3*(8192+128) ;; Move prefetch pointer along
	end_timer 10

;; Do inverse FFT levels 1,2
;; On input the 64-byte cache lines hold these data values:
;;	0	+4K	256	+4K	512	+4K	768	+4K
;;	1	...
;;	...
;;	255	...
;;	1024	...
;;	...
;; On output the 64-byte cache lines hold these data values if pass 1 swizzles:
;;	0	+4K	1024	+4K	2048	+4K	3072	+4K
;;	1	...
;;	...
;;	1023
;; On output the 64-byte cache lines hold these data values if pass 2 swizzles:
;;	0	+1K	2048	+1K	4096	+1K	6144	+1K
;;	1	...
;;	...
;;	1023

;; Do 1024 four_complex_unfft operations
;;	distance between fft data elements is 512
;;	do 256 macros each processing 24 data values

	start_timer 14
	mov	rdi, xsincos_complex	;; Load group multiplier pointer
	loops_init_prefetch 256, 64, 2, rcx ;; 2 iters of 128
	xtouch	[rcx+4096-64]		;; Load prefetch TLBs
	xtouch	[rcx+2*4096-64]
	r4_x4cl_four_complex_djbunfft_preload
c1b:	r4_x4cl_four_complex_djbunfft rsi, 64, 2*dist128, 4*dist128, rdi, 256*XMM_SCD2
	bump	rdi, XMM_SCD2		;; Next set of sin/cos data
	loops	128, c1b		;; Test inner loop counter
	bump	rsi, -128*64+dist128	;; Next source pointer
	loops	2, c1b			;; Test outer loop counter
	end_timer 14

xpass2_11_levels_complex_done:
	ENDM


; *************** radix4-pass2-2560-levels-real macro ******************
; This macro takes 5120 real values and performs the final 12.322 levels of
; the FFT.  Independently, this macro also takes 2560 complex values and
; performs the final 11.322 levels of the FFT.  The results are squared, and
; the first 11.322/12.322 levels of the inverse FFT are done.

r4_pass2_2560_levels_real MACRO
	LOCAL	b1b, b3b, b4b, b5b, b6b, b7b
	LOCAL	b8b, b9b, baa, bab0, bab, bac, bad0, bad
	LOCAL	c1a, c1b, c3b, c4b, c5b, c6b, c7b, c8b, c9b
	LOCAL	xpass2_2560_levels_real_1, xpass2_2560_levels_real_2
	LOCAL	xpass2_2560_levels_real_3, xpass2_2560_levels_real_4
	LOCAL	xpass2_2560_real_unfft, xpass2_2560_real_done

;; We switch to the same format used in one-pass FFTs (that is, the
;; high word of xmm registers being 1 greater than the low word):

;; Type 4 FFTs skip the forward FFT process

	start_timer 2
	mov	rsi, DATA_ADDR		;; Load source address
	mov	rbx, DIST_TO_FFTSRCARG
	cmp	ffttype, 4
	je	xpass2_2560_levels_real_4

;; Do FFT level 1,2 (actually 2.322 levels)
;;
;; On input the 64-byte cache lines hold these data values if pass 2 swizzles:
;;	0	+1280	2560	+1280	5120	+1280	7680	+1280
;;	1	...
;;	...
;;	1279
;; On input the 64-byte cache lines hold these data values if pass 1 swizzles:
;;	0	+5K	1280	+5K	2560	+5K	3840	+5K
;;	1	...
;;	...
;;	1279	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+5K	256	+5K	512	+5K	768	+5K
;;	1	...
;;	...
;;	255	...
;;	1024	...
;;	...

;; Do 512 ten_reals_fft operations
;; Do 512 five_complex_djbfft operations
;;	distance between fft data elements is 512
;;	do 256 iterations

	mov	rdi, xsincos_complex	;; Ptr to the sin/cos data
	mov	rdx, sincos3		;; Ptr to real sin/cos data
	loops_init 256
b1b:	r5_fh5cl_ten_reals_five_complex_djbfft rsi, 64, 2*dist128, rdi, 256*XMM_SCD2, rdx, 256*XMM_HSCD2
	bump	rdi, XMM_SCD2		;; Next sin/cos data
	bump	rdx, XMM_HSCD2		;; Next sin/cos data
	loops	128, b1b		;; Test loop counter
	bump	rsi, -128*64+dist128	;; Next source pointer
	loops	2, b1b			;; Test loop counter
	bump	rsi, -2*dist128		;; Restore source pointer
	bump	rdi, 256*XMM_SCD2	;; Next sin/cos ptr
	bump	rdx, 256*XMM_HSCD2	;; Next sin/cos ptr

;; Do FFT levels 3,4
;;
;; On input the 64-byte cache lines hold these data values:
;;	0	+5K	256	+5K	512	+5K	768	+5K
;;	1	...
;;	...
;;	255	...
;;	1024	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+5K	128	+5K	256	+5K	384	+5K
;;	1	...
;;	...
;;	127	...
;;	512	...
;;	...

;; Do 128 eight_reals_fft operations
;; Do 128 four_complex_djbfft operations
;;	distance between fft data elements is 128
;;	do 128 macros each processing 16 data values

	mov	rcx, DATA_PREFETCH	;; Prefetch pointer
	loops_init_prefetch 640, 64, 4, rcx
	xtouch	[rcx+4096-64]		;; Preload the TLBs
	xtouch	[rcx+2*4096-64]
	xtouch	[rcx+3*4096-64]
b4b:	r4_h2cl_eight_reals_four_complex_djbfft rsi, 64, dist128, rdi, rdx
	bump	rdi, XMM_SCD2		;; Next sin/cos data
	bump	rdx, XMM_HSCD2		;; Next sin/cos data
	loops	128, b4b		;; Test loop counter
	bump	rsi, -128*64+2*dist128	;; Next source pointer
	bump	rdi, -128*XMM_SCD2	;; Restore sin/cos ptr
	bump	rdx, -128*XMM_HSCD2	;; Restore sin/cos ptr

;; Do 1024 four_complex_djbfft operations
;;	distance between fft data elements is 128
;;	do 512 macros each processing 16 data values

	loops_reset			;; 4 more iterations of 128
	r4_nf2cl_four_complex_djbfft_preload
b5b:	r4_nf2cl_four_complex_djbfft rsi, 64, dist128, rdi
	bump	rdi, XMM_SCD2		;; Next sin/cos data
	loops	128, b5b		;; Test inner loop counter
	bump	rsi, -128*64+2*dist128	;; Next source pointer
	bump	rdi, -128*XMM_SCD2	;; Restore sin/cos ptr
	loops	5, b5b			;; Test loop counter
	bump	rsi, -5*2*dist128	;; Next source pointer

;; Do FFT levels 5,6
;;
;; On input the 64-byte cache lines hold these data values:
;;	0	+5K	128	+5K	256	+5K	384	+5K
;;	1	...
;;	...
;;	127	...
;;	512	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+5K	32	+5K	64	+5K	96	+5K
;;	1	...
;;	...
;;	31	...
;;	128	...
;;	...

;; Do 32 eight_reals_fft operations
;; Do 96 four_complex_djbfft operations
;;	distance between fft data elements is 32
;;	do 32 macros each processing 32 data values

	loops_init_prefetch 320, 64, 2, rcx
	xtouch	[rcx+4096-64]		;; Preload the TLBs
	xtouch	[rcx+2*4096-64]
	xtouch	[rcx+3*4096-64]
	r4_h4cl_eight_reals_four_complex_djbfft_preload
b6b:	r4_h4cl_eight_reals_four_complex_djbfft rsi, 64, 32*64, 64*64, rdi, rdx
	bump	rdi, 4*XMM_SCD2		;; Next sin/cos data
	bump	rdx, 4*XMM_HSCD2	;; Next sin/cos data
	loops	32, b6b			;; Test loop counter
	bump	rsi, -32*64+dist128	;; Next source pointer
	bump	rdi, -32*4*XMM_SCD2	;; Restore sin/cos ptr
	bump	rdx, -32*4*XMM_HSCD2	;; Restore sin/cos ptr

;; Do 1152 four_complex_djbfft operations
;;	distance between fft data elements is 32
;;	do 288 macros each processing 32 data values

	loops_reset			;; 9 more iters of 32
	r4_x4cl_four_complex_djbfft_preload
b7b:	r4_x4cl_four_complex_djbfft rsi, 64, 32*64, 64*64, rdi
	bump	rdi, 4*XMM_SCD2		;; Next sin/cos data
	loops	32, b7b			;; Test inner loop counter
	bump	rsi, -32*64+dist128	;; Next source pointer
	bump	rdi, -32*4*XMM_SCD2	;; Restore sin/cos ptr
	loops	10, b7b			;; 10 iterations
	bump	rsi, -10*dist128	;; Next source pointer

;; Do FFT levels 7,8
;;
;; On input the 64-byte cache lines hold these data values:
;;	0	+5K	32	+5K	64	+5K	96	+5K
;;	1	...
;;	...
;;	31	...
;;	128	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+5K	8	+5K	16	+5K	24	+5K
;;	1	...
;;	...
;;	7	...
;;	32	...
;;	...

;; Do 8 eight_reals_fft operations
;; Do 24 four_complex_djbfft operations
;;	distance between fft data elements is 8
;;	do 8 macros each processing 32 data values

	loops_init_prefetch 320, 64, 2, rcx
	xtouch	[rcx+4096-64]		;; Preload the TLBs
	xtouch	[rcx+2*4096-64]
	xtouch	[rcx+3*4096-64]
	r4_h4cl_eight_reals_four_complex_djbfft_preload
b8b:	r4_h4cl_eight_reals_four_complex_djbfft rsi, 64, 8*64, 16*64, rdi, rdx
	bump	rdi, 16*XMM_SCD2	;; Next sin/cos data
	bump	rdx, 16*XMM_HSCD2	;; Next sin/cos data
	loops	8, b8b			;; Test loop counter
	bump	rsi, -8*64+32*64	;; Next source pointer
	bump	rdi, -8*16*XMM_SCD2	;; Restore sin/cos ptr
	bump	rdx, -8*16*XMM_HSCD2	;; Restore sin/cos ptr

;; Do 1248 four_complex_djbfft operations
;;	distance between fft data elements is 8
;;	do 312 macros each processing 32 data values

	loops_reset			;; 10 iters of 3 or 4 of 8
	r4_x4cl_four_complex_djbfft_preload
b9b:	r4_x4cl_four_complex_djbfft rsi, 64, 8*64, 16*64, rdi
	bump	rdi, 16*XMM_SCD2	;; Next sin/cos data
	loops	8, b9b			;; Test loop counter
	bump	rsi, -8*64+32*64	;; Next source pointer
	bump	rdi, -8*16*XMM_SCD2	;; Restore sin/cos ptr
	loops	4, b9b			;; Test middle loop counter
	bump	rsi, -4*32*64+dist128	;; Next source pointer
	loops	10, b9b			;; 10 iterations
	bump	rsi, -10*dist128	;; Next source pointer

;; Do FFT levels 9,10,11
;;
;; On input the 64-byte cache lines hold these data values:
;;	0	+5K	8	+5K	16	+5K	24	+5K
;;	1	...
;;	...
;;	7	...
;;	32	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+5K	4	+5K	8	+5K	12	+5K
;;	1	...
;;	...
;;	3	...
;;	16	...
;;	...

;; Do 1 sixteen_reals_fft operations
;; Do 3 eight_complex_fft operations
;;	distance between fft data elements is 1

;; Do 636 eight_complex_fft operations
;;	distance between fft data elements is 1
;;	do 159 macros each processing 64 data values

;; Execute the proper middle step

	cmp	ffttype, 2
	jl	xpass2_2560_levels_real_1
	jg	xpass2_2560_levels_real_3

xpass2_2560_levels_real_2:
	r8_h8cl_sixteen_reals_eight_complex_with_square rsi, 8*64, 64, 2*64, 4*64
	loops_init_prefetch 160, 64, 1, rcx, 1, 16
	r8_x8cl_eight_complex_with_square_preload
	xtouch	[rcx+4096-64]		;; Preload the TLBs
	xtouch	[rcx+2*4096-64]
	xtouch	[rcx+3*4096-64]
baa:	r8_x8cl_eight_complex_with_square rsi, 8*64, 64, 2*64, 4*64
	loops	16, baa			;; Test loop counter
	bump	rsi, -16*8*64+dist128	;; Next source pointer
	loops	10, baa			;; Test loop counter
	bump	rsi, -10*dist128	;; Next source pointer
	jmp	xpass2_2560_real_unfft

xpass2_2560_levels_real_1:
	r8_h8cl_sixteen_reals_eight_complex_fft_final rsi, 8*64, 64, 2*64, 4*64
	loops_init_prefetch 160, 128, 1, rcx, 1, 16
	r8_x8cl_eight_complex_fft_final_preload
bab0:	xtouch	[rcx+4096-64]		;; Preload the TLBs
bab:	r8_x8cl_eight_complex_fft_final rsi, 8*64, 64, 2*64, 4*64
	loops	16, bab			;; Test loop counter
	bump	rsi, -16*8*64+dist128	;; Next source pointer
	loops_touch 2, bab, 5, bab0	;; Test loop counter
	jmp	xpass2_2560_real_done

xpass2_2560_levels_real_3:
	mov	rbp, DIST_TO_MULSRCARG
	r8_h8cl_sixteen_reals_eight_complex_with_mult rsi, 8*64, 64, 2*64, 4*64
	loops_init_prefetch 160, 64, 1, rcx, 1, 16
	r8_x8cl_eight_complex_with_mult_preload
	xtouch	[rcx+4096-64]		;; Preload the TLBs
	xtouch	[rcx+2*4096-64]
	xtouch	[rcx+3*4096-64]
bac:	r8_x8cl_eight_complex_with_mult rsi, 8*64, 64, 2*64, 4*64
	loops	16, bac			;; Test loop counter
	bump	rsi, -16*8*64+dist128	;; Next source pointer
	loops	10, bac			;; Test loop counter
	bump	rsi, -10*dist128	;; Next source pointer
	jmp	xpass2_2560_real_unfft

xpass2_2560_levels_real_4:
	mov	rbp, DIST_TO_MULSRCARG
	mov	rcx, DATA_PREFETCH	;; Load prefetch pointer
	xtouch	[rcx]			;; Preload first TLB
	r8_h8cl_sixteen_reals_eight_complex_with_mulf rsi, 8*64, 64, 2*64, 4*64
	loops_init_prefetch 160, 128, 1, rcx, 1, 16
	r8_x8cl_eight_complex_with_mulf_preload
bad0:	xtouch	[rcx+4096-64]		;; Preload the TLBs
bad:	r8_x8cl_eight_complex_with_mulf rsi, 8*64, 64, 2*64, 4*64
	loops	16, bad			;; Test loop counter
	bump	rsi, -16*8*64+dist128	;; Next source pointer
	loops_touch 2, bad, 5, bad0	;; Test loop counter
	bump	rsi, -10*dist128	;; Next source pointer

;; Do inverse FFT levels 7,8
;; On input the 64-byte cache lines hold these data values:
;;	0	+5K	4	+5K	8	+5K	12	+5K
;;	1	...
;;	...
;;	3	...
;;	16	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+5K	16	+5K	32	+5K	48	+5K
;;	1	...
;;	...
;;	15	...
;;	64	...
;;	...

;; Do 8 eight_reals_unfft operations
;; Do 8 four_complex_djbunfft operations
;;	distance between fft data elements is 8
;;	do 4 macros processing 32 data values

xpass2_2560_real_unfft:
	mov	rdi, xsincos_complex	;; Load sin/cos pointers
	bump	rdi, 512*XMM_SCD2
	mov	rdx, sincos3
	bump	rdx, 512*XMM_HSCD2
	loops_init_prefetch 320, 64, 2, rcx
	xtouch	[rcx+4096-64]		;; Load prefetch TLBs
	xtouch	[rcx+2*4096-64]
	xtouch	[rcx+3*4096-64]
c8b:	r4_h4cl_eight_reals_four_complex_djbunfft rsi, 64, 4*64, 8*64, rdi, 4*16*XMM_SCD2, rdx, 4*16*XMM_HSCD2
	bump	rdi, 16*XMM_SCD2	;; Next sin/cos data
	bump	rdx, 16*XMM_HSCD2	;; Next sin/cos data
	loops	4, c8b			;; Test loop counter
	bump	rsi, -4*64+16*64	;; Next source pointer
	bump	rdi, -4*16*XMM_SCD2	;; Restore sin/cos ptr
	bump	rdx, -4*16*XMM_HSCD2	;; Restore sin/cos ptr

;; Do 1264 four_complex_djbunfft operations
;;	distance between fft data elements is 8
;;	do 316 macros each processing 32 data values

	loops_reset			;; 10 iters of 7 or 8 of 4
	r4_x4cl_four_complex_djbunfft_preload
c9b:	r4_x4cl_four_complex_djbunfft rsi, 64, 4*64, 8*64, rdi, 4*16*XMM_SCD2
	bump	rdi, 16*XMM_SCD2	;; Next sine/cosine pointer
	loops	4, c9b			;; Test loop counter
	bump	rsi, -4*64+16*64	;; Next source pointer
	bump	rdi, -4*16*XMM_SCD2	;; Restore sin/cos ptr
	loops	8, c9b			;; Test loop counter
	bump	rsi, -8*16*64+dist128	;; Next source pointer
	loops	10, c9b			;; Test loop counter
	bump	rsi, -10*dist128	;; Next source pointer

;; Do inverse FFT levels 5,6
;; On input the 64-byte cache lines hold these data values:
;;	0	+5K	16	+5K	32	+5K	48	+5K
;;	1	...
;;	...
;;	15	...
;;	64	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+5K	64	+5K	128	+5K	192	+5K
;;	1	...
;;	...
;;	63	...
;;	256	...
;;	...

;; Do 32 eight_reals_unfft operations
;; Do 32 four_complex_djbunfft operations
;;	distance between fft data elements is 32
;;	do 16 macros processing 32 data values

	loops_init_prefetch 320, 64, 2, rcx
	xtouch	[rcx+4096-64]		;; Load prefetch TLBs
	xtouch	[rcx+2*4096-64]
	xtouch	[rcx+3*4096-64]
c6b:	r4_h4cl_eight_reals_four_complex_djbunfft rsi, 64, 16*64, 32*64, rdi, 16*4*XMM_SCD2, rdx, 16*4*XMM_HSCD2
	bump	rdi, 4*XMM_SCD2		;; Next sin/cos data
	bump	rdx, 4*XMM_HSCD2	;; Next sin/cos data
	loops	16, c6b			;; Test inner loop counter
	bump	rsi, -16*64+64*64	;; Next source pointer
	bump	rdi, -16*4*XMM_SCD2	;; Restore sin/cos ptr
	bump	rdx, -16*4*XMM_HSCD2	;; Restore sin/cos ptr

;; Do 1216 four_complex_djbunfft operations
;;	distance between fft data elements is 32
;;	do 304 macros each processing 32 data values

	loops_reset			;; 10 iters of 1 or 2 of 16
	r4_x4cl_four_complex_djbunfft_preload
c7b:	r4_x4cl_four_complex_djbunfft rsi, 64, 16*64, 32*64, rdi, 16*4*XMM_SCD2
	bump	rdi, 4*XMM_SCD2		;; Next sin/cos data
	loops	16, c7b			;; Test inner loop counter
	bump	rsi, -16*64+64*64	;; Next source pointer
	bump	rdi, -16*4*XMM_SCD2	;; Restore sin/cos ptr
	loops	2, c7b			;; Test loop counter
	bump	rsi, -2*64*64+dist128	;; Next source pointer
	loops	10, c7b			;; 10 iterations
	bump	rsi, -10*dist128	;; Next source pointer

;; Do inverse FFT levels 3,4
;; On input the 64-byte cache lines hold these data values:
;;	0	+5K	64	+5K	128	+5K	192	+5K
;;	1	...
;;	...
;;	63	...
;;	256	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+5K	256	+5K	512	+5K	768	+5K
;;	1	...
;;	...
;;	255	...
;;	1024	...
;;	...

;; Do 128 eight_reals_unfft operations
;; Do 128 four_complex_djbunfft operations
;;	distance between fft data elements is 128
;;	do 64 macros processing 32 data values

	loops_init_prefetch 320, 64, 2, rcx
	xtouch	[rcx+4096-64]		;; Load prefetch TLBs
	xtouch	[rcx+2*4096-64]
	xtouch	[rcx+3*4096-64]
c4b:	r4_h4cl_eight_reals_four_complex_djbunfft rsi, 64, 64*64, dist128, rdi, 64*XMM_SCD2, rdx, 64*XMM_HSCD2
	bump	rdi, XMM_SCD2		;; Next sin/cos data
	bump	rdx, XMM_HSCD2		;; Next sin/cos data
	loops	64, c4b			;; Test inner loop counter
	bump	rsi, -64*64+2*dist128	;; Next source pointer
	bump	rdi, -64*XMM_SCD2	;; Restore sin/cos ptr
	bump	rdx, -64*XMM_HSCD2	;; Restore sin/cos ptr

;; Do 512 four_complex_djbunfft operations
;;	distance between fft data elements is 128
;;	do 128 macros each processing 32 data values

	loops_reset			;; 4 more iters of 64
	r4_x4cl_four_complex_djbunfft_preload
c5b:	r4_x4cl_four_complex_djbunfft rsi, 64, 64*64, dist128, rdi, 64*XMM_SCD2
	bump	rdi, XMM_SCD2		;; Next sin/cos data
	loops	64, c5b			;; Test inner loop counter
	bump	rsi, -64*64+2*dist128	;; Next source pointer
	bump	rdi, -64*XMM_SCD2	;; Restore sin/cos ptr
	loops	5, c5b			;; Test loop counter
	bump	rsi, -5*2*dist128	;; Next source pointer

;; Do inverse FFT level 1,2 (actually 2.322 FFT levels)
;; On input the 64-byte cache lines hold these data values:
;;	0	+5K	256	+5K	512	+5K	768	+5K
;;	1	...
;;	...
;;	255	...
;;	1024	...
;;	...
;; On output the 64-byte cache lines hold these data values if pass 1 swizzles:
;;	0	+5K	1280	+5K	2560	+5K	3840	+5K
;;	1	...
;;	...
;;	1279
;; On output the 64-byte cache lines hold these data values if pass 2 swizzles:
;;	0	+1280	2560	+1280	5120	+1280	7680	+1280
;;	1	...
;;	...
;;	1279

;; Do 512 ten_reals_unfft operations
;; Do 512 five_complex_djbunfft operations
;;	distance between fft data elements is 512
;;	do 256 macros processing 40 data values

	mov	rdi, xsincos_complex	;; Load sin/cos pointers
	mov	rdx, sincos3
	bump	rcx, -77*64		;; Back up prefetch pointer by 77 cache lines
	loops_init_prefetch 256, 64, 1, rcx
c1a:	xtouch	[rcx+4096-64]		;; Load prefetch TLBs
	xtouch	[rcx+2*4096-64]
c1b:	r5_h5cl_ten_reals_five_complex_djbunfft rsi, 64, 2*dist128, rdi, 256*XMM_SCD2, rdx, 256*XMM_HSCD2
	bump	rdi, XMM_SCD2		;; Next sin/cos data
	bump	rdx, XMM_HSCD2		;; Next sin/cos data
	loops	128, c1b		;; Test loop counter
	bump	rsi, -128*64+dist128	;; Next source pointer
	loops	2, c1a			;; Test loop counter

xpass2_2560_real_done:
	end_timer 2
	ENDM

; *************** radix4-pass2-2560-levels-complex macro ******************
; This macro takes 2 groups of 2560 complex values and
; performs the final 11.322 levels of a standard mostly radix-4 FFT, squares the
; results, and does the first 11.322 levels of the inverse FFT.

r4_pass2_2560_levels_complex MACRO
	LOCAL	b1b, b3a, b3b, b4b, b5b, b6a, b6b0, b6b, b6c, b6d0, b6d
	LOCAL	c1a, c1b, c3b, c4b, c5a, c5b
	LOCAL	xpass2_2560_levels_complex_1, xpass2_2560_levels_complex_2
	LOCAL	xpass2_2560_levels_complex_3, xpass2_2560_levels_complex_4
	LOCAL	xpass2_2560_levels_complex_unfft, xpass2_2560_levels_complex_done

;; Type 4 FFTs skip the forward FFT process

	mov	rsi, DATA_ADDR		;; Load address of FFT data
	mov	rbx, DIST_TO_FFTSRCARG
	cmp	ffttype, 4
	je	xpass2_2560_levels_complex_4

;; Do FFT levels 1,2 (actually 2.322 levels)
;;
;; We swizzle the data so that one set of data is in the high word of
;; an xmm register, the other set is in the low word.  This means
;; a distance of 5120 data values.
;;
;; On input the 64-byte cache lines hold these data values if pass 2 swizzles:
;;	0	+1280	2560	+1280	5120	+1280	7680	+1280
;;	1	...
;;	...
;;	1279
;; On input the 64-byte cache lines hold these data values if pass 1 swizzles:
;;	0	+5K	1280	+5K	2560	+5K	3840	+5K
;;	1	...
;;	...
;;	1279	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+5K	256	+5K	512	+5K	768	+5K
;;	1	...
;;	...
;;	255	...
;;	1024	...
;;	...

;; Do 1024 five_complex_fft operations
;;	distance between fft data elements is 512
;;	do 256 macros each processing 24 data values

	start_timer 4
	mov	rdi, xsincos_complex	;; The sin/cos data
	loops_init 256			;; 2 iterations of 128
	r5_f5cl_five_complex_djbfft_preload
b1b:	xxprefetch [rdi+XMM_SCD2]	;; Prefetch for next pass
	xxprefetch [rdi+257*XMM_SCD2]	;; Prefetch for next pass
	r5_f5cl_five_complex_djbfft rsi, 64, 2*dist128, rdi, 256*XMM_SCD2
	bump	rdi, XMM_SCD2		;; Next sin/cos data
	loops	128, b1b		;; Test inner loop counter
	bump	rsi, -128*64+dist128	;; Next source pointer
	loops	2, b1b			;; Test outer loop counter
	bump	rsi, -2*dist128		;; Restore source pointer
	bump	rdi, 256*XMM_SCD2	;; Next sin/cos ptr
	end_timer 4

;; Do FFT levels 3,4
;;
;; On input the 64-byte cache lines hold these data values:
;;	0	+5K	256	+5K	512	+5K	768	+5K
;;	1	...
;;	...
;;	255	...
;;	1024	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+5K	128	+5K	256	+5K	384	+5K
;;	1	...
;;	...
;;	127	...
;;	512	...
;;	...
;;
;; Do 1280 four_complex_fft operations
;;	distance between fft data elements is 128
;;	do 640 macros each processing 16 data values

;; Do FFT levels 5,6
;;
;; On input the 64-byte cache lines hold these data values:
;;	0	+5K	128	+5K	256	+5K	384	+5K
;;	1	...
;;	...
;;	127	...
;;	512	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+5K	32	+5K	64	+5K	96	+5K
;;	1	...
;;	...
;;	31	...
;;	128	...
;;	...
;;
;; Do 1280 four_complex_fft operations
;;	distance between fft data elements is 32
;;	do 320 macros each processing 32 data values

;; Do FFT levels 7,8
;;
;; On input the 64-byte cache lines hold these data values:
;;	0	+5K	32	+5K	64	+5K	96	+5K
;;	1	...
;;	...
;;	31	...
;;	128	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+5K	8	+5K	16	+5K	24	+5K
;;	1	...
;;	...
;;	7	...
;;	32	...
;;	...
;;
;; Do 1280 four_complex_fft operations
;;	distance between fft data elements is 8
;;	do 320 macros each processing 32 data values

;; Do this 32 cache lines at a time to maximize accessing data
;; while it is in the L1 cache.

	start_timer 6
	mov	rcx, DATA_PREFETCH	;; Prefetch pointer (we need to prefetch 
					;; 1280 cache lines + 18 pad cache lines)
	mov	rbx, rdi		;; Copy sin/cos pointers
	mov	rbp, rdi
	loops_init_prefetch 640, 64, 4, rcx ;; 5*8 iters of 32 cache lines
	;r4_nf2cl_four_complex_djbfft_preload -- assume same as r4_x4cl_four_complex_djbfft_preload
	r4_x4cl_four_complex_djbfft_preload

b3a:	xtouch	[rcx+4096-64]		;; Load prefetch TLBs
	xtouch	[rcx+2*4096-64]

b3b:	xxprefetch [rdi+8*XMM_SCD2]	;; Prefetch for next pass
	r4_nf2cl_four_complex_djbfft rsi, 8*64, dist128, rdi
	bump	rdi, 8*XMM_SCD2		;; Next sine/cosine pointer
	loops	16, b3b			;; Loop 16 times
	bump	rsi, -16*8*64		;; Restore source pointer
	bump	rdi, -16*8*XMM_SCD2	;; Restore sine/cosine pointer
	loops_undo 16			;; Back up loop counter

b4b:	r4_x4cl_four_complex_djbfft rsi, 8*64, 32*64, 64*64, rbx
	bump	rbx, 4*8*XMM_SCD2	;; Next sine/cosine pointer
	loops	4, b4b, 2		;; 4 iterations with non-standard increment
	bump	rsi, -4*8*64+dist128	;; Next source pointer
	bump	rbx, -4*4*8*XMM_SCD2	;; Restore sine/cosine pointer
	loops	2, b4b			;; Loop twice
	bump	rsi, -2*dist128		;; Restore source pointer
	loops_undo 2*4*2		;; Back up loop counter

b5b:	r4_x4cl_four_complex_djbfft rsi, 32*64, 8*64, 16*64, rbp
	loops	4, b5b, 2 		;; 4 iterations with non-standard increment
	bump	rsi, -4*32*64+dist128	;; Next source pointer
	loops	2, b5b	 		;; 2 iterations

	bump	rsi, -2*dist128+64	;; Next source pointer
	bump	rdi, XMM_SCD2		;; Next sine/cosine pointer
	bump	rbx, 4*XMM_SCD2		;; Next sine/cosine pointer
	bump	rbp, 16*XMM_SCD2	;; Next sine/cosine pointer
	loops	8, b3b			;; Loop 8 times
	bump	rsi, -8*64+2*dist128	;; Next source pointer
	bump	rdi, -8*XMM_SCD2	;; Restore sine/cosine pointer
	bump	rbx, -8*4*XMM_SCD2	;; Restore sine/cosine pointer
	bump	rbp, -8*16*XMM_SCD2	;; Restore sine/cosine pointer
	loops	5, b3a			;; Loop 5 times
	bump	rsi, -5*2*dist128	;; Restore source pointer
	end_timer 6

;; Do FFT levels 9,10,11
;;
;; On input the 64-byte cache lines hold these data values:
;;	0	+5K	8	+5K	16	+5K	24	+5K
;;	1	...
;;	...
;;	7	...
;;	32	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+5K	4	+5K	8	+5K	12	+5K
;;	1	...
;;	...
;;	3	...
;;	16	...
;;	...

;; Do 640 eight_complex_fft operations
;;	distance between fft data elements is 1
;;	do 160 macros each processing 64 data values

;; Execute the right middle step

	cmp	ffttype, 2
	jl	xpass2_2560_levels_complex_1
	jg	xpass2_2560_levels_complex_3

xpass2_2560_levels_complex_2:
	start_timer 9
	loops_init_prefetch 160, 64, 1, rcx
	r8_x8cl_eight_complex_with_square_preload
	xtouch	[rcx+2048+4096-64]	;; Load prefetch TLBs
	xtouch	[rcx+2048+2*4096-64]
b6a:	r8_x8cl_eight_complex_with_square rsi, 8*64, 64, 2*64, 4*64
	loops	16, b6a			;; Loop 16 times
	bump	rsi, -16*8*64+dist128	;; Next source pointer
	loops	10, b6a			;; Loop 10 times
	bump	rsi, -10*dist128	;; Restore source pointer
	end_timer 9
	jmp	xpass2_2560_levels_complex_unfft

xpass2_2560_levels_complex_1:
	start_timer 9
	loops_init_prefetch 160, 128, 1, rcx
	r8_x8cl_eight_complex_fft_final_preload
b6b0:	xtouch	[rcx+4096-64]		;; Load prefetch TLBs
b6b:	r8_x8cl_eight_complex_fft_final rsi, 8*64, 64, 2*64, 4*64
	loops	16, b6b			;; Loop 16 times
	bump	rsi, -16*8*64+dist128	;; Next source pointer
	loops_touch 2, b6b, 5, b6b0	;; Loop 10 times
	end_timer 9
	jmp	xpass2_2560_levels_complex_done

xpass2_2560_levels_complex_3:
	start_timer 9
	mov	rbp, DIST_TO_MULSRCARG
	loops_init_prefetch 160, 64, 1, rcx
	r8_x8cl_eight_complex_with_mult_preload
	xtouch	[rcx+2048+4096-64]	;; Load prefetch TLBs
	xtouch	[rcx+2048+2*4096-64]
b6c:	r8_x8cl_eight_complex_with_mult rsi, 8*64, 64, 2*64, 4*64
	loops	16, b6c			;; Loop 16 times
	bump	rsi, -16*8*64+dist128	;; Next source pointer
	loops	10, b6c			;; Loop 10 times
	bump	rsi, -10*dist128	;; Restore source pointer
	end_timer 9
	jmp	xpass2_2560_levels_complex_unfft

xpass2_2560_levels_complex_4:
	start_timer 9
	mov	rcx, DATA_PREFETCH	;; Prefetch pointer
	xtouch	[rcx]			;; Load first prefetch TLB
	mov	rbp, DIST_TO_MULSRCARG
	loops_init_prefetch 160, 128, 1, rcx
	r8_x8cl_eight_complex_with_mulf_preload
b6d0:	xtouch	[rcx+4096-64]		;; Load prefetch TLBs
b6d:	r8_x8cl_eight_complex_with_mulf rsi, 8*64, 64, 2*64, 4*64
	loops	16, b6d			;; Loop 16 times
	bump	rsi, -16*8*64+dist128	;; Next source pointer
	loops_touch 2, b6d, 5, b6d0	;; Loop 10 times
	bump	rsi, -10*dist128	;; Restore source pointer
	end_timer 9

;; Do inverse FFT levels 7,8
;; On input the 64-byte cache lines hold these data values:
;;	0	+5K	4	+5K	8	+5K	12	+5K
;;	1	...
;;	...
;;	3	...
;;	16	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+5K	16	+5K	32	+5K	48	+5K
;;	1	...
;;	...
;;	15	...
;;	64	...
;;	...
;;
;; Do 1280 four_complex_unfft operations
;;	distance between fft data elements is 8
;;	do 320 macros each processing 32 data values

;; Do inverse FFT levels 5,6
;; On input the 64-byte cache lines hold these data values:
;;	0	+5K	16	+5K	32	+5K	48	+5K
;;	1	...
;;	...
;;	15	...
;;	64	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+5K	64	+5K	128	+5K	192	+5K
;;	1	...
;;	...
;;	63	...
;;	256	...
;;	...
;;
;; Do 1280 four_complex_unfft operations
;;	distance between fft data elements is 32
;;	do 320 macros each processing 32 data values

;; Do inverse FFT levels 3,4
;; On input the 64-byte cache lines hold these data values:
;;	0	+5K	64	+5K	128	+5K	192	+5K
;;	1	...
;;	...
;;	63	...
;;	256	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+5K	256	+5K	512	+5K	768	+5K
;;	1	...
;;	...
;;	255	...
;;	1024	...
;;	...
;;
;; Do 1280 four_complex_unfft operations
;;	distance between fft data elements is 128
;;	do 320 macros each processing 32 data values

;; Do this 64 cache lines at a time to maximize accessing data
;; while it is in the L1 cache.

xpass2_2560_levels_complex_unfft:
	start_timer 10
	mov	rdi, xsincos_complex	;; Load sin/cos pointers
	bump	rdi, 512*XMM_SCD2
	mov	rbx, rdi
	mov	rbp, rdi
	loops_init_prefetch 320, 64, 2, rcx ;; 5*4 iters of 64 cache lines
	r4_x4cl_four_complex_djbunfft_preload

c5a:	xtouch	[rcx+4096-64]		;; Load some prefetch TLBs
	xtouch	[rcx+2*4096-64]

c5b:	r4_x4cl_four_complex_djbunfft rsi, 16*64, 4*64, 8*64, rbp, 4*16*XMM_SCD2
	loops	8, c5b			;; Loop 8 times
	bump	rsi, -8*16*64+dist128	;; Next source pointer
	loops	2, c5b			;; Loop 2 times
	bump	rsi, -2*dist128		;; Restore source pointer
	loops_undo 2*8			;; Restore loop counter

c4b:	r4_x4cl_four_complex_djbunfft rsi, 4*64, 16*64, 32*64, rbx, 16*4*XMM_SCD2
	bump	rbx, 4*4*XMM_SCD2	;; Next sine/cosine pointer
	loops	4, c4b			;; Loop 4 times
	bump	rsi, -4*4*64+64*64	;; Next source pointer
	bump	rbx, -4*4*4*XMM_SCD2	;; Restore sine/cosine pointer
	loops	2, c4b			;; Loop twice
	bump	rsi, -2*64*64+dist128	;; Next source pointer
	loops	2, c4b			;; Loop twice
	bump	rsi, -2*dist128		;; Restore source pointer
	loops_undo 2*2*4		;; Restore loop counter

c3b:	r4_x4cl_four_complex_djbunfft rsi, 4*64, 64*64, dist128, rdi, 64*1*XMM_SCD2
	bump	rdi, 4*1*XMM_SCD2	;; Next sine/cosine pointer
	loops	16, c3b			;; Loop 16 times
	bump	rdi, -16*4*1*XMM_SCD2	;; Restore sine/cosine pointer

	bump	rsi, -16*4*64+64	;; Next source pointer
	bump	rdi, 1*XMM_SCD2		;; Next sine/cosine pointer
	bump	rbx, 4*XMM_SCD2		;; Next sine/cosine pointer
	bump	rbp, 16*XMM_SCD2	;; Next sine/cosine pointer
	loops	4, c5b			;; Loop 4 times
	bump	rsi, -4*64+2*dist128	;; Next source pointer
	bump	rdi, -4*1*XMM_SCD2	;; Restore sine/cosine pointer
	bump	rbx, -4*4*XMM_SCD2	;; Restore sine/cosine pointer
	bump	rbp, -4*16*XMM_SCD2	;; Restore sine/cosine pointer
	loops	5, c5a			;; Loop 5 times
	bump	rsi, -5*2*dist128	;; Restore source pointer
	end_timer 10

;; Do inverse FFT levels 1,2 (actually 2.322 FFT levels)
;; On input the 64-byte cache lines hold these data values:
;;	0	+5K	256	+5K	512	+5K	768	+5K
;;	1	...
;;	...
;;	255	...
;;	1024	...
;;	...
;; On output the 64-byte cache lines hold these data values if pass 1 swizzles:
;;	0	+5K	1280	+5K	2560	+5K	3840	+5K
;;	1	...
;;	...
;;	1279
;; On output the 64-byte cache lines hold these data values if pass 2 swizzles:
;;	0	+1280	2560	+1280	5120	+1280	7680	+1280
;;	1	...
;;	...
;;	1279

;; Do 1024 five_complex_unfft operations
;;	distance between fft data elements is 512
;;	do 256 macros each processing 24 data values

	start_timer 14
	mov	rdi, xsincos_complex	;; Load group multiplier pointer
	bump	rcx, -78*64		;; Back up prefetch pointer by 78 cache lines
	loops_init_prefetch 256, 64, 1, rcx ;; 2 iters of 128
	r5_x5cl_five_complex_djbunfft_preload
c1a:	xtouch	[rcx+4096-64]		;; Load prefetch TLBs
	xtouch	[rcx+2*4096-64]
c1b:	r5_x5cl_five_complex_djbunfft rsi, 64, 2*dist128, rdi, 256*XMM_SCD2
	bump	rdi, XMM_SCD2		;; Next set of sin/cos data
	loops	128, c1b		;; Test inner loop counter
	bump	rsi, -128*64+dist128	;; Next source pointer
	loops	2, c1a			;; Test outer loop counter
	end_timer 14

xpass2_2560_levels_complex_done:
	ENDM



;*********************************************************************************************
;			Macros derived from pass2-12-levels
;*********************************************************************************************


; *************** radix4-pass2-2304-levels-real macro ******************
; This macro takes 4608 real values and performs the final 12.17 levels of
; the FFT.  Independently, this macro also takes 2304 complex values and
; performs the final 11.17 levels of the FFT.  The results are squared, and
; the first 11.17/12.17 levels of the inverse FFT are done.

r4_pass2_2304_levels_real MACRO
	LOCAL	b1b, b2b, b3b, b4b, b5b, b6b, b7b
	LOCAL	b8b, b9b, baa, bab0, bab, bac, bad0, bad
	LOCAL	c1b, c2b, c3b, c4b, c5b, c6b, c7b, c8b, c9b
	LOCAL	xpass2_2304_levels_real_1, xpass2_2304_levels_real_2
	LOCAL	xpass2_2304_levels_real_3, xpass2_2304_levels_real_4
	LOCAL	xpass2_2304_real_unfft, xpass2_2304_real_done

;; We switch to the same format used in one-pass FFTs (that is, the
;; high word of xmm registers being 1 greater than the low word):

;; Type 4 FFTs skip the forward FFT process

	start_timer 2
	mov	rsi, DATA_ADDR		;; Load source address
	mov	rbx, DIST_TO_FFTSRCARG
	cmp	ffttype, 4
	je	xpass2_2304_levels_real_4

;; Do FFT level 1,2 (actually 1.585 levels)
;;
;; On input the 64-byte cache lines hold these data values if pass 2 swizzles:
;;	0	+1152	2304	+1152	4608	+1152	6912	+1152
;;	1	...
;;	...
;;	1151
;; On input the 64-byte cache lines hold these data values if pass 1 swizzles:
;;	0	+4608	1152	+4608	2304	+4608	3456	+4608
;;	1	...
;;	...
;;	1151	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+4608	384	+4608	768	+4608	1152	+4608
;;	1	...
;;	...
;;	383	...
;;	1536	...
;;	...

;; Do 768 six_reals_fft on values 0 - 4607
;; Do 768 three_complex_djbfft on values 4608 - 9215
;;	distance between fft data elements is 768
;;	do 384 iterations

	mov	rdi, xsincos_complex	;; Ptr to the sin/cos data
	mov	rdx, sincos3		;; Ptr to real sin/cos data
	loops_init 384
b1b:	r3_fh3cl_six_reals_three_complex_djbfft rsi, 64, 3*dist128, rdi, 384*XMM_SCD1, rdx, 384*XMM_HSCD1
	bump	rdi, XMM_SCD1		;; Next sin/cos data
	bump	rdx, XMM_HSCD1		;; Next sin/cos data
	loops	128, b1b		;; Test loop counter
	bump	rsi, -128*64+dist128	;; Next source pointer
	loops	3, b1b			;; Test loop counter
	bump	rsi, -3*dist128		;; Restore source pointer
	bump	rdi, -384*XMM_SCD1	;; Restore sin/cos ptr
	bump	rdx, -384*XMM_HSCD1	;; Restore sin/cos ptr

;; Do FFT levels 3,4 (actually 1.585 levels)
;; Values 0-1535 is real data, 1536-9215 is complex data.
;;
;; On input the 64-byte cache lines hold these data values:
;;	0	+4608	384	+4608	768	+4608	1152	+4608
;;	1	...
;;	...
;;	383	...
;;	1536	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+4608	128	+4608	256	+4608	384	+4608
;;	1	...
;;	...
;;	127	...
;;	512	...
;;	...

;; Do 256 six_reals_fft operations
;; Do 256 three_complex_djbfft operations
;;	distance between fft data elements is 256
;;	do 128 macros each processing 24 data values

	mov	rcx, DATA_PREFETCH	;; Prefetch pointer
	xtouch	[rcx]			;; Preload first TLB
	loops_init_prefetch 384, 64, 8, rcx
	xtouch	[rcx+4096-64]		;; Preload the TLBs
b2b:	r3_h3cl_six_reals_three_complex_djbfft rsi, 64, dist128, rdi, 128*3*XMM_SCD1, rdx, 128*3*XMM_HSCD1
	bump	rdi, 3*XMM_SCD1		;; Next sin/cos data
	bump	rdx, 3*XMM_HSCD1	;; Next sin/cos data
	loops	128, b2b		;; Test inner loop counter
	bump	rsi, -128*64+3*dist128	;; Next source pointer
	bump	rdi, -128*3*XMM_SCD1	;; Restore sin/cos ptr
	bump	rdx, 128*3*XMM_HSCD1	;; Next sin/cos ptr

;; Do 1024 three_complex_fft operations
;;	distance between fft data elements is 256
;;	do 256 macros each processing 24 data values

	loops_reset			;; 2 more iters of 128
	r3_nf3cl_three_complex_djbfft_preload
b3b:	r3_nf3cl_three_complex_djbfft rsi, 64, dist128, rdi, 128*3*XMM_SCD1
	bump	rdi, 3*XMM_SCD1		;; Next sin/cos data
	loops	128, b3b		;; Test inner loop counter
	bump	rsi, -128*64+3*dist128	;; Next source pointer
	bump	rdi, -128*3*XMM_SCD1	;; Restore sin/cos ptr
	loops	3, b3b			;; Test loop counter
	bump	rsi, -3*3*dist128	;; Restore source pointer

;; Do FFT levels 5,6
;; Values 0-511 is real data, 512-9215 is complex data.
;;
;; On input the 64-byte cache lines hold these data values:
;;	0	+4608	128	+4608	256	+4608	384	+4608
;;	1	...
;;	...
;;	127	...
;;	512	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+4608	64	+4608	128	+4608	192	+4608
;;	1	...
;;	...
;;	63	...
;;	256	...
;;	...

;; Do 64 eight_reals_fft operations
;; Do 192 four_complex_djbfft operations
;;	distance between fft data elements is 64
;;	do 64 macros each processing 16 data values

	bump	rdi, 768*XMM_SCD1	;; Load sin/cos pointer
	loops_init_prefetch 576, 64, 4, rcx
	xtouch	[rcx+4096-64]		;; Preload the TLBs
	xtouch	[rcx+2*4096-64]
	xtouch	[rcx+3*4096-64]
b4b:	r4_h2cl_eight_reals_four_complex_djbfft rsi, 64, 64*64, rdi, rdx
	bump	rdi, XMM_SCD2		;; Next sin/cos data
	bump	rdx, XMM_HSCD2		;; Next sin/cos data
	loops	64, b4b			;; Test loop counter
	bump	rsi, -64*64+dist128	;; Next source pointer
	bump	rdi, -64*XMM_SCD2	;; Restore sin/cos ptr
	bump	rdx, -64*XMM_HSCD2	;; Restore sin/cos ptr

;; Do 1024 four_complex_djbfft operations
;;	distance between fft data elements is 64
;;	do 512 macros each processing 16 data values

	loops_reset			;; 8 more iterations of 64
	r4_nf2cl_four_complex_djbfft_preload
b5b:	r4_nf2cl_four_complex_djbfft rsi, 64, 64*64, rdi
	bump	rdi, XMM_SCD2		;; Next sin/cos data
	loops	64, b5b			;; Test inner loop counter
	bump	rsi, -64*64+dist128	;; Next source pointer
	bump	rdi, -64*XMM_SCD2	;; Restore sin/cos ptr
	loops	9, b5b			;; Test loop counter
	bump	rsi, -9*dist128		;; Next source pointer

;; Do FFT levels 7,8
;; Values 0-127 is real data, 128-9215 is complex data.
;;
;; On input the 64-byte cache lines hold these data values:
;;	0	+4608	64	+4608	128	+4608	192	+4608
;;	1	...
;;	...
;;	63	...
;;	256	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+4608	16	+4608	32	+4608	48	+4608
;;	1	...
;;	...
;;	15	...
;;	64	...
;;	...

;; Do 16 eight_reals_fft operations
;; Do 48 four_complex_djbfft operations
;;	distance between fft data elements is 16
;;	do 16 macros each processing 32 data values

	loops_init_prefetch 288, 64, 2, rcx
	xtouch	[rcx+4096-64]		;; Preload the TLBs
	xtouch	[rcx+2*4096-64]
	xtouch	[rcx+3*4096-64]
	r4_h4cl_eight_reals_four_complex_djbfft_preload
b6b:	r4_h4cl_eight_reals_four_complex_djbfft rsi, 64, 16*64, 32*64, rdi, rdx
	bump	rdi, 4*XMM_SCD2		;; Next sin/cos data
	bump	rdx, 4*XMM_HSCD2	;; Next sin/cos data
	loops	16, b6b			;; Test loop counter
	bump	rsi, -16*64+64*64	;; Next source pointer
	bump	rdi, -16*4*XMM_SCD2	;; Restore sin/cos ptr
	bump	rdx, -16*4*XMM_HSCD2	;; Restore sin/cos ptr

;; Do 1088 four_complex_djbfft operations
;;	distance between fft data elements is 16
;;	do 272 macros each processing 32 data values

	loops_reset			;; 9 iters of 1 or 2 of 16
	r4_x4cl_four_complex_djbfft_preload
b7b:	r4_x4cl_four_complex_djbfft rsi, 64, 16*64, 32*64, rdi
	bump	rdi, 4*XMM_SCD2		;; Next sin/cos data
	loops	16, b7b			;; Test inner loop counter
	bump	rsi, -16*64+64*64	;; Next source pointer
	bump	rdi, -16*4*XMM_SCD2	;; Restore sin/cos ptr
	loops	2, b7b			;; Test loop counter
	bump	rsi, -2*64*64+dist128	;; Next source pointer
	loops	9, b7b			;; 9 iterations
	bump	rsi, -9*dist128		;; Next source pointer

;; Do FFT levels 9,10
;; Values 0-31 is real data, 32-9215 is complex data.
;;
;; On input the 64-byte cache lines hold these data values:
;;	0	+4608	16	+4608	32	+4608	48	+4608
;;	1	...
;;	...
;;	15	...
;;	64	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+4608	4	+4608	8	+4608	12	+4608
;;	1	...
;;	...
;;	3	...
;;	16	...
;;	...

;; Do 4 eight_reals_fft operations
;; Do 12 four_complex_djbfft operations
;;	distance between fft data elements is 4
;;	do 4 macros each processing 32 data values

	loops_init_prefetch 288, 64, 2, rcx
	xtouch	[rcx+4096-64]		;; Preload the TLBs
	xtouch	[rcx+2*4096-64]
	xtouch	[rcx+3*4096-64]
	r4_h4cl_eight_reals_four_complex_djbfft_preload
b8b:	r4_h4cl_eight_reals_four_complex_djbfft rsi, 64, 4*64, 8*64, rdi, rdx
	bump	rdi, 16*XMM_SCD2	;; Next sin/cos data
	bump	rdx, 16*XMM_HSCD2	;; Next sin/cos data
	loops	4, b8b			;; Test loop counter
	bump	rsi, -4*64+16*64	;; Next source pointer
	bump	rdi, -4*16*XMM_SCD2	;; Restore sin/cos ptr
	bump	rdx, -4*16*XMM_HSCD2	;; Restore sin/cos ptr

;; Do 1136 four_complex_djbfft operations
;;	distance between fft data elements is 4
;;	do 284 macros each processing 32 data values

	loops_reset			;; 9 iters of 7 or 8 of 4
	r4_x4cl_four_complex_djbfft_preload
b9b:	r4_x4cl_four_complex_djbfft rsi, 64, 4*64, 8*64, rdi
	bump	rdi, 16*XMM_SCD2	;; Next sin/cos data
	loops	4, b9b			;; Test loop counter
	bump	rsi, -4*64+16*64	;; Next source pointer
	bump	rdi, -4*16*XMM_SCD2	;; Restore sin/cos ptr
	loops	8, b9b			;; Test middle loop counter
	bump	rsi, -8*16*64+dist128	;; Next source pointer
	loops	9, b9b			;; 9 iterations
	bump	rsi, -9*dist128		;; Next source pointer

;; Do FFT levels 11,12
;; Values 0-7 is real data, 8-9215 is complex data.
;;
;; On input the 64-byte cache lines hold these data values:
;;	0	+4608	4	+4608	8	+4608	12	+4608
;;	1	...
;;	...
;;	3	...
;;	16	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+4608	2	+4608	4	+4608	6	+4608
;;	1	...
;;	8	...
;;	...

;; Do 1 eight_reals_fft operations
;; Do 3 four_complex_fft operations
;;	distance between fft data elements is 1

;; Do 1148 four_complex_fft operations
;;	distance between fft data elements is 1
;;	do 287 macros each processing 32 data values

;; Execute the proper middle step

	cmp	ffttype, 2
	jl	xpass2_2304_levels_real_1
	jg	xpass2_2304_levels_real_3

xpass2_2304_levels_real_2:
	r4_h4cl_eight_reals_four_complex_with_square rsi, 4*64, 64, 2*64
	loops_init_prefetch 288, 64, 4, rcx, 1, 32
	xtouch	[rcx+4096-64]		;; Preload the TLBs
	r4_x4cl_four_complex_with_square_preload
baa:	r4_x4cl_four_complex_with_square rsi, 4*64, 64, 2*64
	loops	32, baa			;; Test loop counter
	bump	rsi, -32*4*64+dist128	;; Next source pointer
	loops	9, baa			;; Test loop counter
	bump	rsi, -9*dist128		;; Next source pointer
	jmp	xpass2_2304_real_unfft

xpass2_2304_levels_real_1:
	r4_h4cl_eight_reals_four_complex_fft_final rsi, 4*64, 64, 2*64
	loops_init_prefetch 288, 128, 1, rcx, 1, 32
bab0:	xtouch	[rcx+4096-64]		;; Preload the TLBs
bab:	r4_x4cl_four_complex_fft_final rsi, 4*64, 64, 2*64
	loops	32, bab			;; Test loop counter
	bump	rsi, -32*4*64+dist128	;; Next source pointer
	loops	9, bab0			;; Test loop counter
	jmp	xpass2_2304_real_done

xpass2_2304_levels_real_3:
	mov	rbp, DIST_TO_MULSRCARG
	r4_h4cl_eight_reals_four_complex_with_mult rsi, 4*64, 64, 2*64
	loops_init_prefetch 288, 64, 4, rcx, 1, 32
	xtouch	[rcx+4096-128]		;; Preload the TLBs
	xtouch	[rcx+2*4096-128]
bac:	r4_x4cl_four_complex_with_mult rsi, 4*64, 64, 2*64
	loops	32, bac			;; Test loop counter
	bump	rsi, -32*4*64+dist128	;; Next source pointer
	loops	9, bac			;; Test loop counter
	bump	rsi, -9*dist128		;; Next source pointer
	jmp	xpass2_2304_real_unfft

xpass2_2304_levels_real_4:
	mov	rbp, DIST_TO_MULSRCARG
	mov	rcx, DATA_PREFETCH	;; Load prefetch pointer
	xtouch	[rcx]			;; Preload first TLB
	r4_h4cl_eight_reals_four_complex_with_mulf rsi, 4*64, 64, 2*64
	loops_init_prefetch 288, 128, 1, rcx, 1, 32
bad0:	xtouch	[rcx+4096-64]		;; Preload TLB
bad:	r4_x4cl_four_complex_with_mulf rsi, 4*64, 64, 2*64
	loops	32, bad			;; Test loop counter
	bump	rsi, -32*4*64+dist128	;; Next source pointer
	loops	9, bad0			;; Test loop counter
	bump	rsi, -9*dist128		;; Next source pointer
	bump	rcx, -24*64		;; Back up prefetch pointer

;; Do inverse FFT levels 9,10
;; On input the 64-byte cache lines hold these data values:
;;	0	+4608	2	+4608	4	+4608	6	+4608
;;	1	...
;;	8	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+4608	8	+4608	16	+4608	24	+4608
;;	1	...
;;	...
;;	7	...
;;	32	...
;;	...

;; Do 4 eight_reals_unfft operations
;; Do 4 four_complex_djbunfft operations
;;	distance between fft data elements is 4
;;	do 2 macros processing 32 data values

xpass2_2304_real_unfft:
	mov	rdi, xsincos_complex	;; Load sin/cos pointers
	bump	rdi, 768*XMM_SCD1
	mov	rdx, sincos3
	bump	rdx, 768*XMM_HSCD1
	loops_init_prefetch 288, 64, 2, rcx
	xtouch	[rcx+4096-64]		;; Preload the TLBs
	xtouch	[rcx+2*4096-64]
	xtouch	[rcx+3*4096-64]
c8b:	r4_h4cl_eight_reals_four_complex_djbunfft rsi, 64, 2*64, 4*64, rdi, 2*16*XMM_SCD2, rdx, 2*16*XMM_HSCD2
	bump	rdi, 16*XMM_SCD2	;; Next sin/cos data
	bump	rdx, 16*XMM_HSCD2	;; Next sin/cos data
	loops	2, c8b			;; Test loop counter
	bump	rsi, -2*64+8*64		;; Next source pointer
	bump	rdi, -2*16*XMM_SCD2	;; Restore sin/cos ptr
	bump	rdx, -2*16*XMM_HSCD2	;; Restore sin/cos ptr

;; Do 1144 four_complex_djbunfft operations
;;	distance between fft data elements is 4
;;	do 286 macros each processing 32 data values

	loops_reset			;; 9 iters of 15 or 16 of 2
	r4_x4cl_four_complex_djbunfft_preload
c9b:	r4_x4cl_four_complex_djbunfft rsi, 64, 2*64, 4*64, rdi, 2*16*XMM_SCD2
	bump	rdi, 16*XMM_SCD2	;; Next sine/cosine pointer
	loops	2, c9b			;; Test loop counter
	bump	rsi, -2*64+8*64		;; Next source pointer
	bump	rdi, -2*16*XMM_SCD2	;; Restore sin/cos ptr
	loops	16, c9b			;; Test loop counter
	bump	rsi, -16*8*64+dist128	;; Next source pointer
	loops	9, c9b			;; Test loop counter
	bump	rsi, -9*dist128		;; Next source pointer

;; Do inverse FFT levels 7,8
;; On input the 64-byte cache lines hold these data values:
;;	0	+4608	8	+4608	16	+4608	24	+4608
;;	1	...
;;	...
;;	7	...
;;	32	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+4608	32	+4608	64	+4608	96	+4608
;;	1	...
;;	...
;;	31	...
;;	128	...
;;	...

;; Do 16 eight_reals_unfft operations
;; Do 16 four_complex_djbunfft operations
;;	distance between fft data elements is 16
;;	do 8 macros processing 32 data values

	loops_init_prefetch 288, 64, 2, rcx
	xtouch	[rcx+4096-64]		;; Preload the TLBs
	xtouch	[rcx+2*4096-64]
	xtouch	[rcx+3*4096-64]
c6b:	r4_h4cl_eight_reals_four_complex_djbunfft rsi, 64, 8*64, 16*64, rdi, 8*4*XMM_SCD2, rdx, 8*4*XMM_HSCD2
	bump	rdi, 4*XMM_SCD2		;; Next sin/cos data
	bump	rdx, 4*XMM_HSCD2	;; Next sin/cos data
	loops	8, c6b			;; Test inner loop counter
	bump	rsi, -8*64+32*64	;; Next source pointer
	bump	rdi, -8*4*XMM_SCD2	;; Restore sin/cos ptr
	bump	rdx, -8*4*XMM_HSCD2	;; Restore sin/cos ptr

;; Do 1120 four_complex_djbunfft operations
;;	distance between fft data elements is 16
;;	do 280 macros each processing 32 data values

	loops_reset			;; 9 iters of 3 or 4 of 4
	r4_x4cl_four_complex_djbunfft_preload
c7b:	r4_x4cl_four_complex_djbunfft rsi, 64, 8*64, 16*64, rdi, 8*4*XMM_SCD2
	bump	rdi, 4*XMM_SCD2		;; Next sin/cos data
	loops	8, c7b			;; Test inner loop counter
	bump	rsi, -8*64+32*64	;; Next source pointer
	bump	rdi, -8*4*XMM_SCD2	;; Restore sin/cos ptr
	loops	4, c7b			;; Test loop counter
	bump	rsi, -4*32*64+dist128	;; Next source pointer
	loops	9, c7b			;; 9 iterations
	bump	rsi, -9*dist128	;; Next source pointer

;; Do inverse FFT levels 5,6
;; On input the 64-byte cache lines hold these data values:
;;	0	+4608	32	+4608	64	+4608	96	+4608
;;	1	...
;;	...
;;	31	...
;;	128	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+4608	128	+4608	256	+4608	384	+4608
;;	1	...
;;	...
;;	127	...
;;	512	...
;;	...

;; Do 64 eight_reals_unfft operations
;; Do 64 four_complex_djbunfft operations
;;	distance between fft data elements is 64
;;	do 32 macros processing 32 data values

	loops_init_prefetch 288, 64, 2, rcx
	xtouch	[rcx+4096-64]		;; Preload the TLBs
	xtouch	[rcx+2*4096-64]
	xtouch	[rcx+3*4096-64]
c4b:	r4_h4cl_eight_reals_four_complex_djbunfft rsi, 64, 32*64, 64*64, rdi, 32*XMM_SCD2, rdx, 32*XMM_HSCD2
	bump	rdi, XMM_SCD2		;; Next sin/cos data
	bump	rdx, XMM_HSCD2		;; Next sin/cos data
	loops	32, c4b			;; Test inner loop counter
	bump	rsi, -32*64+dist128	;; Next source pointer
	bump	rdi, -32*XMM_SCD2	;; Restore sin/cos ptr
	bump	rdx, -32*XMM_HSCD2-768*XMM_HSCD1 ;; Restore sin/cos ptr

;; Do 1024 four_complex_djbunfft operations
;;	distance between fft data elements is 64
;;	do 256 macros each processing 32 data values

	loops_reset			;; 8 more iters of 32
	r4_x4cl_four_complex_djbunfft_preload
c5b:	r4_x4cl_four_complex_djbunfft rsi, 64, 32*64, 64*64, rdi, 32*XMM_SCD2
	bump	rdi, XMM_SCD2		;; Next sin/cos data
	loops	32, c5b			;; Test inner loop counter
	bump	rsi, -32*64+dist128	;; Next source pointer
	bump	rdi, -32*XMM_SCD2	;; Restore sin/cos ptr
	loops	9, c5b			;; Test loop counter
	bump	rsi, -9*dist128		;; Next source pointer

;; Do inverse FFT levels 3,4 (actually 1.585 FFT levels)
;; On input the 64-byte cache lines hold these data values:
;;	0	+4608	128	+4608	256	+4608	384	+4608
;;	1	...
;;	...
;;	127	...
;;	512	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+4608	384	+4608	768	+4608	1152	+4608
;;	1	...
;;	...
;;	383	...
;;	1536	...
;;	...

;; Do 256 six_reals_unfft operations
;; Do 256 six_complex_djbunfft operations
;;	distance between fft data elements is 256
;;	do 128 macros processing 24 data values

	loops_init_prefetch 384, 64, 4, rcx
	mov	rdi, xsincos_complex	;; Load sin/cos pointer
	xtouch	[rcx+4096-64]		;; Preload the TLBs
	xtouch	[rcx+2*4096-64]
c2b:	r3_h3cl_six_reals_three_complex_djbunfft rsi, 64, dist128, rdi, 128*3*XMM_SCD1, rdx, 128*3*XMM_HSCD1
	bump	rdi, 3*XMM_SCD1		;; Next sin/cos data
	bump	rdx, 3*XMM_HSCD1	;; Next sin/cos data
	loops	128, c2b		;; Test inner loop counter
	bump	rsi, -128*64+3*dist128	;; Next source pointer
	bump	rdi, -128*3*XMM_SCD1	;; Restore sin/cos ptr
	bump	rdx, -128*3*XMM_HSCD1	;; Restore sin/cos ptr

;; Do 1024 three_complex_djbunfft operations
;;	distance between fft data elements is 256
;;	do 256 macros each processing 24 data values

	loops_reset			;; 2 more iters of 128
	r3_x3cl_three_complex_djbunfft_preload
c3b:	r3_x3cl_three_complex_djbunfft rsi, 64, dist128, rdi, 128*3*XMM_SCD1
	bump	rdi, 3*XMM_SCD1		;; Next sin/cos data
	loops	128, c3b		;; Test inner loop counter
	bump	rsi, -128*64+3*dist128	;; Next source pointer
	bump	rdi, -128*3*XMM_SCD1	;; Restore sin/cos ptr
	loops	3, c3b			;; Test loop counter
	bump	rsi, -3*3*dist128	;; Next source pointer

;; Do inverse FFT level 1,2 (actually 1.585 FFT levels)
;; On input the 64-byte cache lines hold these data values:
;;	0	+4608	384	+4608	768	+4608	1152	+4608
;;	1	...
;;	...
;;	383	...
;;	1536	...
;;	...
;; On output the 64-byte cache lines hold these data values if pass 1 swizzles:
;;	0	+4608	1152	+4608	2304	+4608	3456	+4608
;;	1	...
;;	...
;;	1151
;; On output the 64-byte cache lines hold these data values if pass 2 swizzles:
;;	0	+1152	2304	+1152	4608	+1152	6912	+1152
;;	1	...
;;	...
;;	1151

;; Do 768 six_reals_unfft operations
;; Do 768 three_complex_djbunfft operations
;;	distance between fft data elements is 768
;;	do 384 macros processing 24 data values

	bump	rcx, -8*64		;; Back up prefetch pointer by 8 cache lines
	loops_init_prefetch 384, 64, 4, rcx
	xtouch	[rcx+4096-64]		;; Preload the TLBs
	xtouch	[rcx+96*64-64]
c1b:	r3_h3cl_six_reals_three_complex_djbunfft rsi, 64, 3*dist128, rdi, 384*XMM_SCD1, rdx, 384*XMM_HSCD1
	bump	rdi, XMM_SCD1		;; Next sin/cos data
	bump	rdx, XMM_HSCD1		;; Next sin/cos data
	loops	128, c1b		;; Test loop counter
	bump	rsi, -128*64+dist128	;; Next source pointer
	loops	3, c1b			;; Test loop counter

xpass2_2304_real_done:
	end_timer 2
	ENDM

; *************** radix4-pass2-2304-levels-complex macro ******************
; This macro takes 2 groups of 2304 complex values and
; performs the final 11.17 levels of a standard mostly radix-4 FFT, squares the
; results, and does the first 11.17 levels of the inverse FFT.

r4_pass2_2304_levels_complex MACRO
	LOCAL	b1b, b2b, b3a, b3b, b4b, b5b, b6a, b6b0, b6b, b6c, b6d0, b6d
	LOCAL	c1b, c2a, c2b, c3b, c4b, c5a, c5b
	LOCAL	xpass2_2304_levels_complex_1, xpass2_2304_levels_complex_2
	LOCAL	xpass2_2304_levels_complex_3, xpass2_2304_levels_complex_4
	LOCAL	xpass2_2304_levels_complex_unfft, xpass2_2304_levels_complex_done

;; Type 4 FFTs skip the forward FFT process

	mov	rsi, DATA_ADDR		;; Load address of FFT data
	mov	rbx, DIST_TO_FFTSRCARG
	cmp	ffttype, 4
	je	xpass2_2304_levels_complex_4

;; Do FFT levels 1,2 (actually 1.585 levels)
;;
;; We swizzle the data so that one set of data is in the high word of
;; an xmm register, the other set is in the low word.  This means
;; a distance of 4608 data values.
;;
;; On input the 64-byte cache lines hold these data values if pass 2 swizzles:
;;	0	+1152	2304	+1152	4608	+1152	6912	+1152
;;	1	...
;;	...
;;	1151
;; On input the 64-byte cache lines hold these data values if pass 1 swizzles:
;;	0	+4608	1152	+4608	2304	+4608	3456	+4608
;;	1	...
;;	...
;;	1151	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+4608	384	+4608	768	+4608	1152	+4608
;;	1	...
;;	...
;;	383	...
;;	1536	...
;;	...

;; Do 1536 three_complex_fft operations
;;	distance between fft data elements is 768
;;	do 384 macros each processing 24 data values

;; Do FFT levels 3,4 (actually 1.585 levels)
;;
;; On input the 64-byte cache lines hold these data values:
;;	0	+4608	384	+4608	768	+4608	1152	+4608
;;	1	...
;;	...
;;	383	...
;;	1536	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+4608	128	+4608	256	+4608	384	+4608
;;	1	...
;;	...
;;	127	...
;;	512	...
;;	...

;; Do 1536 three_complex_fft operations
;;	distance between fft data elements is 256
;;	do 384 macros each processing 24 data values

IF @INSTR(,%xarch,<CORE>) NE 0		;; Core 2 faster doing separate loops

	start_timer 4
	mov	rdi, xsincos_complex	;; The sin/cos data
	loops_init 384			;; 3 iterations of 128
	r3_f3cl_three_complex_djbfft_preload
b1b:	r3_f3cl_three_complex_djbfft rsi, 64, 3*dist128, rdi, 384*XMM_SCD1
	bump	rdi, XMM_SCD1		;; Next sin/cos data
	loops	128, b1b		;; Test inner loop counter
	bump	rsi, -128*64+dist128	;; Next source pointer
	loops	3, b1b			;; Test outer loop counter
	bump	rsi, -3*dist128		;; Restore source pointer
	end_timer 4

	start_timer 5
	mov	rcx, DATA_PREFETCH	;; Prefetch pointer (we need to prefetch 
					;; 1152 cache lines + 16 pad cache lines)
	xtouch	[rcx]			;; Load first prefetch TLB
	mov	rdi, xsincos_complex	;; Ptr to the sin/cos data
	loops_init_prefetch 384, 64, 8, rcx ;; 3 iters of 128
	xtouch	[rcx+4096-64]		;; Load prefetch TLB
	r3_nf3cl_three_complex_djbfft_preload
b2b:	r3_nf3cl_three_complex_djbfft rsi, 64, dist128, rdi, 128*3*XMM_SCD1
	bump	rdi, 3*XMM_SCD1		;; Next sin/cos data
	loops	128, b2b		;; Test inner loop counter
	bump	rsi, -128*64+3*dist128	;; Next source pointer
	bump	rdi, -128*3*XMM_SCD1	;; Restore sin/cos ptr
	loops	3, b2b			;; Test loop counter
	bump	rsi, -3*3*dist128	;; Restore source pointer
	end_timer 5

ELSE					;; Pentium 4 faster doing merged loops

	start_timer 4
	mov	rdi, xsincos_complex	;; The sin/cos data
	mov	rbp, rdi
	mov	rcx, DATA_PREFETCH	;; Prefetch pointer (we need to prefetch 
					;; 1152 cache lines + 16 pad cache lines)
	xtouch	[rcx]			;; Load first prefetch TLB
	loops_init 384			;; 3 iters of 128

	r3_f3cl_three_complex_djbfft_preload
;;	r3_nf3cl_three_complex_djbfft_preload ;; Risky - assume this preload is identical to r3_f3cl_three_complex_djbfft_preload

b1b:	r3_f3cl_three_complex_djbfft rsi, dist128, 3*dist128, rdi, 384*XMM_SCD1
	xxprefetch [rdi+XMM_SCD1]	;; Prefetch for next pass
	xxprefetch [rdi+385*XMM_SCD1]	;; Prefetch for next pass
	bump	rdi, 128*XMM_SCD1	;; Next sin/cos data
	loops	3, b1b			;; Test loop counter
	bump	rsi, -3*dist128		;; Restore source pointer
	loops_undo 3

b2b:	r3_nf3cl_three_complex_djbfft rsi, 3*dist128, dist128, rbp, 128*3*XMM_SCD1
	loops	3, b2b			;; Test loop counter

	bump	rsi, -3*3*dist128+64	;; Next source pointer
	bump	rdi, -3*128*XMM_SCD1+XMM_SCD1 ;; Next sin/cos ptr
	bump	rbp, 3*XMM_SCD1		;; Next sin/cos ptr
	loops	128, b1b		;; Test loop counter
	bump	rsi, -128*64		;; Restore source pointer
	bump	rdi, -128*XMM_SCD1	;; Restore sin/cos ptr
	end_timer 4

ENDIF

;; Do FFT levels 5,6
;;
;; On input the 64-byte cache lines hold these data values:
;;	0	+4608	128	+4608	256	+4608	384	+4608
;;	1	...
;;	...
;;	127	...
;;	512	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+4608	64	+4608	128	+4608	192	+4608
;;	1	...
;;	...
;;	63	...
;;	256	...
;;	...
;;
;; Do 1152 four_complex_fft operations
;;	distance between fft data elements is 64
;;	do 576 macros each processing 16 data values

;; Do FFT levels 7,8
;;
;; On input the 64-byte cache lines hold these data values:
;;	0	+4608	64	+4608	128	+4608	192	+4608
;;	1	...
;;	...
;;	63	...
;;	256	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+4608	16	+4608	32	+4608	48	+4608
;;	1	...
;;	...
;;	15	...
;;	64	...
;;	...
;;
;; Do 1152 four_complex_fft operations
;;	distance between fft data elements is 16
;;	do 288 macros each processing 32 data values

;; Do FFT levels 9,10
;;
;; On input the 64-byte cache lines hold these data values:
;;	0	+4608	16	+4608	32	+4608	48	+4608
;;	1	...
;;	...
;;	15	...
;;	64	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+4608	4	+4608	8	+4608	12	+4608
;;	1	...
;;	...
;;	3	...
;;	16	...
;;	...
;;
;; Do 1152 four_complex_fft operations
;;	distance between fft data elements is 4
;;	do 288 macros each processing 32 data values

;; Do this 32 cache lines at a time to maximize accessing data
;; while it is in the L1 cache.

	start_timer 6
	bump	rdi, 768*XMM_SCD1	;; Load sin/cos pointers
	mov	rbx, rdi
	mov	rbp, rdi
	loops_init_prefetch 576, 64, 4, rcx ;; 2*4 iters of 4 of 64 cache lines
	;r4_nf2cl_four_complex_djbfft_preload -- assume same as r4_x4cl_four_complex_djbfft_preload
	r4_x4cl_four_complex_djbfft_preload

b3a:	xtouch	[rcx+4096-64]		;; Load six prefetch TLBs

b3b:	xxprefetch [rdi+4*XMM_SCD2]	;; Prefetch for next pass
	r4_nf2cl_four_complex_djbfft rsi, 4*64, 64*64, rdi
	bump	rdi, 4*XMM_SCD2		;; Next sine/cosine pointer
	loops	16, b3b			;; Loop 16 times
	bump	rsi, -16*4*64		;; Restore source pointer
	bump	rdi, -16*4*XMM_SCD2	;; Restore sine/cosine pointer
	loops_undo 16			;; Back up loop counter

b4b:	r4_x4cl_four_complex_djbfft rsi, 4*64, 16*64, 32*64, rbx
	bump	rbx, 4*4*XMM_SCD2	;; Next sine/cosine pointer
	loops	4, b4b, 2		;; 4 iterations with non-standard increment
	bump	rsi, -4*4*64+64*64	;; Next source pointer
	bump	rbx, -4*4*4*XMM_SCD2	;; Restore sine/cosine pointer
	loops	2, b4b			;; Loop twice
	bump	rsi, -2*64*64		;; Restore source pointer
	loops_undo 2*4*2		;; Back up loop counter

b5b:	r4_x4cl_four_complex_djbfft rsi, 16*64, 4*64, 8*64, rbp
	loops	8, b5b, 2 		;; 8 iterations with non-standard increment

	bump	rsi, -8*16*64+64	;; Next source pointer
	bump	rdi, XMM_SCD2		;; Next sine/cosine pointer
	bump	rbx, 4*XMM_SCD2		;; Next sine/cosine pointer
	bump	rbp, 16*XMM_SCD2	;; Next sine/cosine pointer
	loops	4, b3b			;; Loop 4 times
	bump	rsi, -4*64+dist128	;; Next source pointer
	bump	rdi, -4*XMM_SCD2	;; Restore sine/cosine pointer
	bump	rbx, -4*4*XMM_SCD2	;; Restore sine/cosine pointer
	bump	rbp, -4*16*XMM_SCD2	;; Restore sine/cosine pointer
	loops	9, b3a			;; Loop 9 times
	bump	rsi, -9*dist128		;; Restore source pointer
	end_timer 6

;; Do FFT levels 11,12 as well as inverse FFT levels 11,12
;;
;; On input the 64-byte cache lines hold these data values:
;;	0	+4608	4	+4608	8	+4608	12	+4608
;;	1	...
;;	...
;;	3	...
;;	16	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+4608	2	+4608	4	+4608	6	+4608
;;	1	...
;;	8	...
;;	...

;; Do 1152 four_complex_fft operations
;;	distance between fft data elements is 1
;;	do 288 macros each processing 32 data values

;; Execute the right middle step

	cmp	ffttype, 2
	jl	xpass2_2304_levels_complex_1
	jg	xpass2_2304_levels_complex_3

xpass2_2304_levels_complex_2:
	start_timer 9
	loops_init_prefetch 288, 64, 4, rcx
	xtouch	[rcx+4096-64]		;; Load prefetch TLB
	r4_x4cl_four_complex_with_square_preload
b6a:	r4_x4cl_four_complex_with_square rsi, 4*64, 64, 2*64
	loops	32, b6a			;; Loop 32 times
	bump	rsi, -16*2*4*64+dist128	;; Next source pointer
	loops	9, b6a			;; Loop 9 times
	bump	rsi, -9*dist128		;; Restore source pointer
	end_timer 9
	jmp	xpass2_2304_levels_complex_unfft

xpass2_2304_levels_complex_1:
	start_timer 9
	loops_init_prefetch 288, 128, 1, rcx
b6b0:	xtouch	[rcx+4096-64]		;; Load prefetch TLBs
b6b:	r4_x4cl_four_complex_fft_final rsi, 4*64, 64, 2*64
	loops	32, b6b			;; Loop 32 times
	bump	rsi, -16*2*4*64+dist128	;; Next source pointer
	loops	9, b6b0			;; Loop 9 times
	end_timer 9
	jmp	xpass2_2304_levels_complex_done

xpass2_2304_levels_complex_3:
	start_timer 9
	mov	rbp, DIST_TO_MULSRCARG
	loops_init_prefetch 288, 64, 4, rcx
	xtouch	[rcx+4096-128]		;; Load prefetch TLBs
b6c:	r4_x4cl_four_complex_with_mult rsi, 4*64, 64, 2*64
	loops	32, b6c			;; Loop 32 times
	bump	rsi, -16*2*4*64+dist128	;; Next source pointer
	loops	9, b6c			;; Loop 9 times
	bump	rsi, -9*dist128		;; Restore source pointer
	end_timer 9
	jmp	xpass2_2304_levels_complex_unfft

xpass2_2304_levels_complex_4:
	start_timer 9
	mov	rcx, DATA_PREFETCH	;; Prefetch pointer
	xtouch	[rcx]			;; Load first prefetch TLB
	mov	rbp, DIST_TO_MULSRCARG
	loops_init_prefetch 288, 128, 1, rcx
b6d0:	xtouch	[rcx+4096-128]		;; Load prefetch TLBs
b6d:	r4_x4cl_four_complex_with_mulf rsi, 4*64, 64, 2*64
	loops	32, b6d			;; Loop 32 times
	bump	rsi, -16*2*4*64+dist128	;; Next source pointer
	loops	9, b6d0			;; Loop 9 times
	bump	rsi, -9*dist128		;; Restore source pointer
	bump	rcx, -24*64		;; Back up prefetch pointer
	end_timer 9

;; Do inverse FFT levels 9,10
;; On input the 64-byte cache lines hold these data values:
;;	0	+4608	2	+4608	4	+4608	6	+4608
;;	1	...
;;	8	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+4608	8	+4608	16	+4608	24	+4608
;;	1	...
;;	...
;;	7	...
;;	32	...
;;	...
;;
;; Do 1152 four_complex_unfft operations
;;	distance between fft data elements is 4
;;	do 288 macros each processing 32 data values

;; Do inverse FFT levels 7,8
;; On input the 64-byte cache lines hold these data values:
;;	0	+4608	8	+4608	16	+4608	24	+4608
;;	1	...
;;	...
;;	7	...
;;	32	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+4608	32	+4608	64	+4608	96	+4608
;;	1	...
;;	...
;;	31	...
;;	128	...
;;	...
;;
;; Do 1152 four_complex_unfft operations
;;	distance between fft data elements is 16
;;	do 288 macros each processing 32 data values

;; Do inverse FFT levels 5,6
;; On input the 64-byte cache lines hold these data values:
;;	0	+4608	32	+4608	64	+4608	96	+4608
;;	1	...
;;	...
;;	31	...
;;	128	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+4608	128	+4608	256	+4608	384	+4608
;;	1	...
;;	...
;;	127	...
;;	512	...
;;	...
;;
;; Do 1152 four_complex_unfft operations
;;	distance between fft data elements is 64
;;	do 288 macros each processing 32 data values

;; Do this 64 cache lines at a time to maximize accessing data
;; while it is in the L1 cache.

xpass2_2304_levels_complex_unfft:
	start_timer 10
	mov	rdi, xsincos_complex	;; Load sin/cos pointers
	bump	rdi, 768*XMM_SCD1
	mov	rbx, rdi
	mov	rbp, rdi
	loops_init_prefetch 288, 64, 2, rcx ;; 9*2 iters of 64 cache lines
	r4_x4cl_four_complex_djbunfft_preload

c5a:	xtouch	[rcx+4096-64]		;; Load prefetch TLB

c5b:	r4_x4cl_four_complex_djbunfft rsi, 8*64, 2*64, 4*64, rbp, 2*16*XMM_SCD2
	loops	16, c5b			;; Loop 16 times
	bump	rsi, -16*8*64		;; Restore source pointer
	loops_undo 16			;; Restore loop counter

c4b:	r4_x4cl_four_complex_djbunfft rsi, 2*64, 8*64, 16*64, rbx, 8*4*XMM_SCD2
	bump	rbx, 2*4*XMM_SCD2	;; Next sine/cosine pointer
	loops	4, c4b			;; Loop 4 times
	bump	rsi, -4*2*64+32*64	;; Next source pointer
	bump	rbx, -4*2*4*XMM_SCD2	;; Restore sine/cosine pointer
	loops	4, c4b			;; Loop 4 times
	bump	rsi, -4*32*64		;; Restore source pointer
	loops_undo 4*4			;; Restore loop counter

c3b:	r4_x4cl_four_complex_djbunfft rsi, 2*64, 32*64, 64*64, rdi, 32*1*XMM_SCD2
	bump	rdi, 2*XMM_SCD2		;; Next sine/cosine pointer
	loops	16, c3b			;; Loop 16 times
	bump	rdi, -16*2*XMM_SCD2	;; Restore sine/cosine pointer

	bump	rsi, -16*2*64+64	;; Next source pointer
	bump	rdi, XMM_SCD2		;; Next sine/cosine pointer
	bump	rbx, 4*XMM_SCD2		;; Next sine/cosine pointer
	bump	rbp, 16*XMM_SCD2	;; Next sine/cosine pointer
	loops	2, c5b			;; Loop twice
	bump	rsi, -2*64+dist128	;; Next source pointer
	bump	rdi, -2*XMM_SCD2	;; Restore sine/cosine pointer
	bump	rbx, -2*4*XMM_SCD2	;; Restore sine/cosine pointer
	bump	rbp, -2*16*XMM_SCD2	;; Restore sine/cosine pointer
	loops	9, c5a			;; Loop 9 times
	bump	rsi, -9*dist128		;; Restore source pointer
	end_timer 10

;; Do inverse FFT levels 3,4 (actually 1.585 FFT levels)
;; On input the 64-byte cache lines hold these data values:
;;	0	+4608	128	+4608	256	+4608	384	+4608
;;	1	...
;;	...
;;	127	...
;;	512	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+4608	384	+4608	768	+4608	1152	+4608
;;	1	...
;;	...
;;	383	...
;;	1536	...
;;	...

;; Do 1536 three_complex_unfft operations
;;	distance between fft data elements is 256
;;	do 384 macros each processing 24 data values

;; Do inverse FFT levels 1,2 (actually 1.585 FFT levels)
;; On input the 64-byte cache lines hold these data values:
;;	0	+4608	384	+4608	768	+4608	1152	+4608
;;	1	...
;;	...
;;	383	...
;;	1536	...
;;	...
;; On output the 64-byte cache lines hold these data values if pass 1 swizzles:
;;	0	+4608	1152	+4608	2304	+4608	3456	+4608
;;	1	...
;;	...
;;	1151
;; On output the 64-byte cache lines hold these data values if pass 2 swizzles:
;;	0	+1152	2304	+1152	4608	+1152	6912	+1152
;;	1	...
;;	...
;;	1151

;; Do 1536 three_complex_unfft operations
;;	distance between fft data elements is 768
;;	do 384 macros each processing 24 data values

IF @INSTR(,%xarch,<CORE>) NE 0		;; Core 2 faster doing separate loops

	start_timer 13
	mov	rdi, xsincos_complex	;; Load sin/cos pointer
	loops_init_prefetch 384, 64, 4, rcx;; 3 iters of 128
	xtouch	[rcx+4096-64]		;; Load prefetch TLBs
	xtouch	[rcx+2*4096-64]
	r3_x3cl_three_complex_djbunfft_preload
c2b:	r3_x3cl_three_complex_djbunfft rsi, 64, dist128, rdi, 128*3*XMM_SCD1
	bump	rdi, 3*XMM_SCD1		;; Next sine/cosine pointer
	loops	128, c2b		;; Loop 128 times
	bump	rsi, -128*64+3*dist128	;; Next source pointer
	bump	rdi, -128*3*XMM_SCD1	;; Reset sine/cosine pointer
	loops	3, c2b			;; Test loop counter
	bump	rsi, -3*3*dist128	;; Restore source pointer
	end_timer 13

	start_timer 14
	mov	rdi, xsincos_complex	;; Load sin/cos pointer
	bump	rcx, -8*64		;; Back up prefetch pointer by 8 cache lines
	loops_init_prefetch 384, 64, 4, rcx;; 3 iters of 128
	xtouch	[rcx+4096-64]		;; Load prefetch TLBs
	xtouch	[rcx+96*64-64]
	r3_x3cl_three_complex_djbunfft_preload
c1b:	r3_x3cl_three_complex_djbunfft rsi, 64, 3*dist128, rdi, 384*XMM_SCD1
	bump	rdi, XMM_SCD1		;; Next set of sin/cos data
	loops	128, c1b		;; Test inner loop counter
	bump	rsi, -128*64+dist128	;; Next source pointer
	loops	3, c1b			;; Test outer loop counter
	end_timer 14

ELSE					;; Pentium 4 faster doing merged loops

	start_timer 13
	mov	rdi, xsincos_complex	;; Load sin/cos pointer
	mov	rbp, rdi
	bump	rcx, -24*64		;; Back up prefetch pointer by 24 cache lines
	loops_init_prefetch 384, 64, 3, rcx ;; 128 iters of 3, prefetch 128 cache lines in each loop
	r3_x3cl_three_complex_djbunfft_preload

c2a:	xtouch	[rcx+4096-64]		;; Load prefetch TLB

c2b:	r3_x3cl_three_complex_djbunfft rsi, 3*dist128, dist128, rbp, 128*3*XMM_SCD1
	loops	3, c2b			;; Test loop counter
	bump	rsi, -3*3*dist128	;; Restore source pointer
	loops_undo 3			;; Back up loop counter

c1b:	r3_x3cl_three_complex_djbunfft rsi, dist128, 3*dist128, rdi, 384*XMM_SCD1
	bump	rdi, 128*XMM_SCD1	;; Next sin/cos ptr
	loops	3, c1b			;; Test loop counter

	bump	rsi, -3*dist128+64	;; Next source pointer
	bump	rdi, -3*128*XMM_SCD1+XMM_SCD1 ;; Next sin/cos ptr
	bump	rbp, 3*XMM_SCD1		;; Next sine/cosine pointer
	loops_touch 32, c2b, 4, c2a	;; Loop 128 times
	end_timer 13

ENDIF

xpass2_2304_levels_complex_done:
	ENDM


; *************** radix4-pass2-3072-levels-real macro ******************
; This macro takes 6144 real values and performs the final 12.585 levels of
; the FFT.  Independently, this macro also takes 3072 complex values and
; performs the final 11.585 levels of the FFT.  The results are squared, and
; the first 11.585/12.585 levels of the inverse FFT are done.

r4_pass2_3072_levels_real MACRO
	LOCAL	b1b, b2b, b3b, b4b, b5b, b6b, b7b
	LOCAL	b8b, b9b, baa, bab0, bab, bac, bad0, bad
	LOCAL	c1b, c2b, c3b, c4b, c5b, c6b, c7b, c8b, c9b
	LOCAL	xpass2_3072_levels_real_1, xpass2_3072_levels_real_2
	LOCAL	xpass2_3072_levels_real_3, xpass2_3072_levels_real_4
	LOCAL	xpass2_3072_real_unfft, xpass2_3072_real_done

;; We switch to the same format used in one-pass FFTs (that is, the
;; high word of xmm registers being 1 greater than the low word):

;; Type 4 FFTs skip the forward FFT process

	start_timer 2
	mov	rsi, DATA_ADDR		;; Load source address
	mov	rbx, DIST_TO_FFTSRCARG
	cmp	ffttype, 4
	je	xpass2_3072_levels_real_4

;; Do FFT level 1,2 (actually 1.585 levels)
;;
;; On input the 64-byte cache lines hold these data values if pass 2 swizzles:
;;	0	+1.5K	3072	+1.5K	6144	+1.5K	9216	+1.5K
;;	1	...
;;	...
;;	1535
;; On input the 64-byte cache lines hold these data values if pass 1 swizzles:
;;	0	+6K	1536	+6K	3072	+6K	4608	+6K
;;	1	...
;;	...
;;	1535	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+6K	512	+6K	1024	+6K	1536	+6K
;;	1	...
;;	...
;;	511	...
;;	2048	...
;;	...

;; Do 1024 six_reals_fft on values 0 - 6143
;; Do 1024 three_complex_djbfft on values 6144 - 12287
;;	distance between fft data elements is 1K
;;	do 512 iterations

	mov	rdi, xsincos_complex	;; Ptr to the sin/cos data
	mov	rdx, sincos3		;; Ptr to real sin/cos data
	loops_init 512
b1b:	r3_fh3cl_six_reals_three_complex_djbfft rsi, 64, 4*dist128, rdi, 512*XMM_SCD1, rdx, 512*XMM_HSCD1
	bump	rdi, XMM_SCD1		;; Next sin/cos data
	bump	rdx, XMM_HSCD1		;; Next sin/cos data
	loops	128, b1b		;; Test loop counter
	bump	rsi, -128*64+dist128	;; Next source pointer
	loops	4, b1b			;; Test loop counter
	bump	rsi, -4*dist128		;; Restore source pointer
	bump	rdi, -512*XMM_SCD1	;; Restore sin/cos ptr
	bump	rdx, 512*XMM_HSCD1	;; Next sin/cos ptr

;; Do FFT levels 3,4
;; Values 0-2047 is real data, 2048-12287 is complex data.
;;
;; On input the 64-byte cache lines hold these data values:
;;	0	+6K	512	+6K	1024	+6K	1536	+6K
;;	1	...
;;	...
;;	511	...
;;	2048	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+6K	256	+6K	512	+6K	768	+6K
;;	1	...
;;	...
;;	255	...
;;	1024	...
;;	...

;; Do 256 eight_reals_fft operations
;; Do 256 four_complex_djbfft operations
;;	distance between fft data elements is 256
;;	do 256 macros each processing 16 data values

	mov	rcx, DATA_PREFETCH	;; Prefetch pointer
	xtouch	[rcx]			;; Preload first TLB
	mov	rbx, rdi
	loops_init 768
b2b:	r4_h2cl_2sc_eight_reals_four_complex_djbfft rsi, 64, 2*dist128, rdi, rbx, rdx
	bump	rdi, 3*XMM_SCD1		;; Next sin/cos data
	bump	rbx, 6*XMM_SCD1		;; Next sin/cos data
	bump	rdx, XMM_HSCD2		;; Next sin/cos data
	loops	128, b2b		;; Test inner loop counter
	bump	rsi, -128*64+dist128	;; Next source pointer
	loops	2, b2b			;; Test loop counter
	bump	rsi, -2*dist128+4*dist128 ;; Next source pointer
	bump	rdi, -256*3*XMM_SCD1	;; Restore sin/cos ptr
	bump	rbx, -256*6*XMM_SCD1	;; Restore sin/cos ptr
	bump	rdx, -256*XMM_HSCD2	;; Restore sin/cos ptr

;; Do 1024 four_complex_fft operations
;;	distance between fft data elements is 256
;;	do 512 macros each processing 16 data values

	loops_reset			;; 2 more iters of 256
	r4_x2cl_2sc_four_complex_djbfft_preload
b3b:	r4_x2cl_2sc_four_complex_djbfft rsi, 64, 2*dist128, rdi, rbx
	bump	rdi, 3*XMM_SCD1		;; Next sin/cos data
	bump	rbx, 6*XMM_SCD1		;; Next sin/cos data
	loops	128, b3b		;; Test inner loop counter
	bump	rsi, -128*64+dist128	;; Next source pointer
	loops	2, b3b			;; Test loop counter
	bump	rsi, -2*dist128+4*dist128 ;; Next source pointer
	bump	rdi, -256*3*XMM_SCD1	;; Restore sin/cos ptr
	bump	rbx, -256*6*XMM_SCD1	;; Restore sin/cos ptr
	loops	3, b3b			;; Test loop counter
	bump	rsi, -3*4*dist128	;; Restore source pointer

;; Do FFT levels 5,6
;; Values 0-511 is real data, 512-12287 is complex data.
;;
;; On input the 64-byte cache lines hold these data values:
;;	0	+6K	256	+6K	512	+6K	768	+6K
;;	1	...
;;	...
;;	255	...
;;	1024	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+6K	64	+6K	128	+6K	192	+6K
;;	1	...
;;	...
;;	63	...
;;	256	...
;;	...

;; Do 64 eight_reals_fft operations
;; Do 192 four_complex_djbfft operations
;;	distance between fft data elements is 64
;;	do 64 macros each processing 32 data values

	bump	rdi, 1536*XMM_SCD1	;; Load sin/cos pointer
	loops_init_prefetch 384, 64, 2, rcx
	xtouch	[rcx+4096-64]		;; Preload the TLBs
	xtouch	[rcx+2*4096-64]
	xtouch	[rcx+3*4096-64]
	r4_h4cl_eight_reals_four_complex_djbfft_preload
b4b:	r4_h4cl_eight_reals_four_complex_djbfft rsi, 64, 64*64, dist128, rdi, rdx
	bump	rdi, XMM_SCD2		;; Next sin/cos data
	bump	rdx, 4*XMM_HSCD2	;; Next sin/cos data
	loops	64, b4b			;; Test loop counter
	bump	rsi, -64*64+2*dist128	;; Next source pointer
	bump	rdi, -64*XMM_SCD2	;; Restore sin/cos ptr
	bump	rdx, -64*4*XMM_HSCD2	;; Restore sin/cos ptr

;; Do 1280 four_complex_djbfft operations
;;	distance between fft data elements is 64
;;	do 320 macros each processing 32 data values

	loops_reset			;; 5 iterations of 64
	r4_x4cl_four_complex_djbfft_preload
b5b:	r4_x4cl_four_complex_djbfft rsi, 64, 64*64, dist128, rdi
	bump	rdi, XMM_SCD2		;; Next sin/cos data
	loops	64, b5b			;; Test inner loop counter
	bump	rsi, -64*64+2*dist128	;; Next source pointer
	bump	rdi, -64*XMM_SCD2	;; Restore sin/cos ptr
	loops	6, b5b			;; Test loop counter
	bump	rsi, -6*2*dist128	;; Next source pointer

;; Do FFT levels 7,8
;; Values 0-127 is real data, 128-12687 is complex data.
;;
;; On input the 64-byte cache lines hold these data values:
;;	0	+6K	64	+6K	128	+6K	192	+6K
;;	1	...
;;	...
;;	63	...
;;	256	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+6K	16	+6K	32	+6K	48	+6K
;;	1	...
;;	...
;;	15	...
;;	64	...
;;	...

;; Do 16 eight_reals_fft operations
;; Do 48 four_complex_djbfft operations
;;	distance between fft data elements is 16
;;	do 16 macros each processing 32 data values

	loops_init_prefetch 384, 64, 2, rcx
	xtouch	[rcx+4096-64]		;; Preload the TLBs
	xtouch	[rcx+2*4096-64]
	xtouch	[rcx+3*4096-64]
	r4_h4cl_eight_reals_four_complex_djbfft_preload
b6b:	r4_h4cl_eight_reals_four_complex_djbfft rsi, 64, 16*64, 32*64, rdi, rdx
	bump	rdi, 4*XMM_SCD2		;; Next sin/cos data
	bump	rdx, 16*XMM_HSCD2	;; Next sin/cos data
	loops	16, b6b			;; Test loop counter
	bump	rsi, -16*64+64*64	;; Next source pointer
	bump	rdi, -16*4*XMM_SCD2	;; Restore sin/cos ptr
	bump	rdx, -16*16*XMM_HSCD2	;; Restore sin/cos ptr

;; Do 1472 four_complex_djbfft operations
;;	distance between fft data elements is 16
;;	do 368 macros each processing 32 data values

	loops_reset			;; 12 iters of 1 or 2 of 16
	r4_x4cl_four_complex_djbfft_preload
b7b:	r4_x4cl_four_complex_djbfft rsi, 64, 16*64, 32*64, rdi
	bump	rdi, 4*XMM_SCD2		;; Next sin/cos data
	loops	16, b7b			;; Test inner loop counter
	bump	rsi, -16*64+64*64	;; Next source pointer
	bump	rdi, -16*4*XMM_SCD2	;; Restore sin/cos ptr
	loops	2, b7b			;; Test loop counter
	bump	rsi, -2*64*64+dist128	;; Next source pointer
	loops	12, b7b			;; 12 iterations
	bump	rsi, -12*dist128	;; Next source pointer

;; Do FFT levels 9,10
;; Values 0-31 is real data, 32-12687 is complex data.
;;
;; On input the 64-byte cache lines hold these data values:
;;	0	+6K	16	+6K	32	+6K	48	+6K
;;	1	...
;;	...
;;	15	...
;;	64	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+6K	4	+6K	8	+6K	12	+6K
;;	1	...
;;	...
;;	3	...
;;	16	...
;;	...

;; Do 4 eight_reals_fft operations
;; Do 12 four_complex_djbfft operations
;;	distance between fft data elements is 4
;;	do 4 macros each processing 32 data values

	loops_init_prefetch 384, 64, 2, rcx
	xtouch	[rcx+4096-64]		;; Preload the TLBs
	xtouch	[rcx+2*4096-64]
	xtouch	[rcx+3*4096-64]
	r4_h4cl_eight_reals_four_complex_djbfft_preload
b8b:	r4_h4cl_eight_reals_four_complex_djbfft rsi, 64, 4*64, 8*64, rdi, rdx
	bump	rdi, 16*XMM_SCD2	;; Next sin/cos data
	bump	rdx, 64*XMM_HSCD2	;; Next sin/cos data
	loops	4, b8b			;; Test loop counter
	bump	rsi, -4*64+16*64	;; Next source pointer
	bump	rdi, -4*16*XMM_SCD2	;; Restore sin/cos ptr
	bump	rdx, -4*64*XMM_HSCD2	;; Restore sin/cos ptr

;; Do 1520 four_complex_djbfft operations
;;	distance between fft data elements is 4
;;	do 380 macros each processing 32 data values

	loops_reset			;; 12 iters of 7 or 8 of 4
	r4_x4cl_four_complex_djbfft_preload
b9b:	r4_x4cl_four_complex_djbfft rsi, 64, 4*64, 8*64, rdi
	bump	rdi, 16*XMM_SCD2	;; Next sin/cos data
	loops	4, b9b			;; Test loop counter
	bump	rsi, -4*64+16*64	;; Next source pointer
	bump	rdi, -4*16*XMM_SCD2	;; Restore sin/cos ptr
	loops	8, b9b			;; Test middle loop counter
	bump	rsi, -8*16*64+dist128	;; Next source pointer
	loops	12, b9b			;; 12 iterations
	bump	rsi, -12*dist128	;; Next source pointer

;; Do FFT levels 11,12
;; Values 0-7 is real data, 8-12687 is complex data.
;;
;; On input the 64-byte cache lines hold these data values:
;;	0	+6K	4	+6K	8	+6K	12	+6K
;;	1	...
;;	...
;;	3	...
;;	16	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+6K	2	+6K	4	+6K	6	+6K
;;	1	...
;;	8	...
;;	...

;; Do 1 eight_reals_fft operations
;; Do 3 four_complex_fft operations
;;	distance between fft data elements is 1

;; Do 1532 four_complex_fft operations
;;	distance between fft data elements is 1
;;	do 383 macros each processing 32 data values

;; Execute the proper middle step

	cmp	ffttype, 2
	jl	xpass2_3072_levels_real_1
	jg	xpass2_3072_levels_real_3

xpass2_3072_levels_real_2:
	r4_h4cl_eight_reals_four_complex_with_square rsi, 4*64, 64, 2*64
	loops_init_prefetch 384, 64, 4, rcx, 1, 32
	xtouch	[rcx+4096-64]		;; Preload the TLBs
	xtouch	[rcx+2*4096-64]
	r4_x4cl_four_complex_with_square_preload
baa:	r4_x4cl_four_complex_with_square rsi, 4*64, 64, 2*64
	loops	32, baa			;; Test loop counter
	bump	rsi, -32*4*64+dist128	;; Next source pointer
	loops	12, baa			;; Test loop counter
	bump	rsi, -12*dist128	;; Next source pointer
	jmp	xpass2_3072_real_unfft

xpass2_3072_levels_real_1:
	r4_h4cl_eight_reals_four_complex_fft_final rsi, 4*64, 64, 2*64
	loops_init_prefetch 384, 128, 1, rcx, 1, 32
bab0:	xtouch	[rcx+4096-64]		;; Preload the TLBs
bab:	r4_x4cl_four_complex_fft_final rsi, 4*64, 64, 2*64
	loops	32, bab			;; Test loop counter
	bump	rsi, -32*4*64+dist128	;; Next source pointer
	loops	12, bab0		;; Test loop counter
	jmp	xpass2_3072_real_done

xpass2_3072_levels_real_3:
	mov	rbp, DIST_TO_MULSRCARG
	r4_h4cl_eight_reals_four_complex_with_mult rsi, 4*64, 64, 2*64
	loops_init_prefetch 384, 64, 4, rcx, 1, 32
	xtouch	[rcx+4096-64]		;; Preload the TLBs
	xtouch	[rcx+2*4096-64]
bac:	r4_x4cl_four_complex_with_mult rsi, 4*64, 64, 2*64
	loops	32, bac			;; Test loop counter
	bump	rsi, -32*4*64+dist128	;; Next source pointer
	loops	12, bac			;; Test loop counter
	bump	rsi, -12*dist128	;; Next source pointer
	jmp	xpass2_3072_real_unfft

xpass2_3072_levels_real_4:
	mov	rbp, DIST_TO_MULSRCARG
	mov	rcx, DATA_PREFETCH	;; Load prefetch pointer
	xtouch	[rcx]			;; Preload first TLB
	r4_h4cl_eight_reals_four_complex_with_mulf rsi, 4*64, 64, 2*64
	loops_init_prefetch 384, 64, 1, rcx, 1, 32
bad0:	xtouch	[rcx+4096-64]
bad:	r4_x4cl_four_complex_with_mulf rsi, 4*64, 64, 2*64
	loops	32, bad			;; Test loop counter
	bump	rsi, -32*4*64+dist128	;; Next source pointer
	loops	12, bad0		;; Test loop counter
	bump	rsi, -12*dist128	;; Next source pointer
	bump	rcx, -23*64		;; Back up prefetch pointer

;; Do inverse FFT levels 9,10
;; On input the 64-byte cache lines hold these data values:
;;	0	+6K	2	+6K	4	+6K	6	+6K
;;	1	...
;;	8	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+6K	8	+6K	16	+6K	24	+6K
;;	1	...
;;	...
;;	7	...
;;	32	...
;;	...

;; Do 4 eight_reals_unfft operations
;; Do 4 four_complex_djbunfft operations
;;	distance between fft data elements is 4
;;	do 2 macros processing 32 data values

xpass2_3072_real_unfft:
	mov	rdi, xsincos_complex	;; Load sin/cos pointers
	bump	rdi, 1536*XMM_SCD1
	mov	rdx, sincos3
	bump	rdx, 1024*XMM_HSCD1
	loops_init_prefetch 384, 64, 2, rcx
	xtouch	[rcx+4096-64]		;; Preload TLBs
	xtouch	[rcx+2*4096-64]
	xtouch	[rcx+3*4096-64]
c8b:	r4_h4cl_eight_reals_four_complex_djbunfft rsi, 64, 2*64, 4*64, rdi, 2*16*XMM_SCD2, rdx, 2*64*XMM_HSCD2
	bump	rdi, 16*XMM_SCD2	;; Next sin/cos data
	bump	rdx, 64*XMM_HSCD2	;; Next sin/cos data
	loops	2, c8b			;; Test loop counter
	bump	rsi, -2*64+8*64		;; Next source pointer
	bump	rdi, -2*16*XMM_SCD2	;; Restore sin/cos ptr
	bump	rdx, -2*64*XMM_HSCD2	;; Restore sin/cos ptr

;; Do 1528 four_complex_djbunfft operations
;;	distance between fft data elements is 4
;;	do 382 macros each processing 32 data values

	loops_reset			;; 12 iters of 15 or 16 of 2
	r4_x4cl_four_complex_djbunfft_preload
c9b:	r4_x4cl_four_complex_djbunfft rsi, 64, 2*64, 4*64, rdi, 2*16*XMM_SCD2
	bump	rdi, 16*XMM_SCD2	;; Next sine/cosine pointer
	loops	2, c9b			;; Test loop counter
	bump	rsi, -2*64+8*64		;; Next source pointer
	bump	rdi, -2*16*XMM_SCD2	;; Restore sin/cos ptr
	loops	16, c9b			;; Test loop counter
	bump	rsi, -16*8*64+dist128	;; Next source pointer
	loops	12, c9b			;; Test loop counter
	bump	rsi, -12*dist128	;; Next source pointer

;; Do inverse FFT levels 7,8
;; On input the 64-byte cache lines hold these data values:
;;	0	+6K	8	+6K	16	+6K	24	+6K
;;	1	...
;;	...
;;	7	...
;;	32	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+6K	32	+6K	64	+6K	96	+6K
;;	1	...
;;	...
;;	31	...
;;	128	...
;;	...

;; Do 16 eight_reals_unfft operations
;; Do 16 four_complex_djbunfft operations
;;	distance between fft data elements is 16
;;	do 8 macros processing 32 data values

	loops_init_prefetch 384, 64, 2, rcx
	xtouch	[rcx+4096-64]		;; Preload TLBs
	xtouch	[rcx+2*4096-64]
	xtouch	[rcx+3*4096-64]
c6b:	r4_h4cl_eight_reals_four_complex_djbunfft rsi, 64, 8*64, 16*64, rdi, 8*4*XMM_SCD2, rdx, 8*16*XMM_HSCD2
	bump	rdi, 4*XMM_SCD2		;; Next sin/cos data
	bump	rdx, 16*XMM_HSCD2	;; Next sin/cos data
	loops	8, c6b			;; Test inner loop counter
	bump	rsi, -8*64+32*64	;; Next source pointer
	bump	rdi, -8*4*XMM_SCD2	;; Restore sin/cos ptr
	bump	rdx, -8*16*XMM_HSCD2	;; Restore sin/cos ptr

;; Do 1504 four_complex_djbunfft operations
;;	distance between fft data elements is 16
;;	do 376 macros each processing 32 data values

	loops_reset			;; 12 iters of 3 or 4 of 4
	r4_x4cl_four_complex_djbunfft_preload
c7b:	r4_x4cl_four_complex_djbunfft rsi, 64, 8*64, 16*64, rdi, 8*4*XMM_SCD2
	bump	rdi, 4*XMM_SCD2		;; Next sin/cos data
	loops	8, c7b			;; Test inner loop counter
	bump	rsi, -8*64+32*64	;; Next source pointer
	bump	rdi, -8*4*XMM_SCD2	;; Restore sin/cos ptr
	loops	4, c7b			;; Test loop counter
	bump	rsi, -4*32*64+dist128	;; Next source pointer
	loops	12, c7b			;; 12 iterations
	bump	rsi, -12*dist128	;; Next source pointer

;; Do inverse FFT levels 5,6
;; On input the 64-byte cache lines hold these data values:
;;	0	+6K	32	+6K	64	+6K	96	+6K
;;	1	...
;;	...
;;	31	...
;;	128	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+6K	128	+6K	256	+6K	384	+6K
;;	1	...
;;	...
;;	127	...
;;	512	...
;;	...

;; Do 64 eight_reals_unfft operations
;; Do 64 four_complex_djbunfft operations
;;	distance between fft data elements is 64
;;	do 32 macros processing 32 data values

	loops_init_prefetch 384, 64, 2, rcx
	xtouch	[rcx+4096-64]		;; Preload TLBs
	xtouch	[rcx+2*4096-64]
	xtouch	[rcx+3*4096-64]
c4b:	r4_h4cl_eight_reals_four_complex_djbunfft rsi, 64, 32*64, 64*64, rdi, 32*XMM_SCD2, rdx, 32*4*XMM_HSCD2
	bump	rdi, XMM_SCD2		;; Next sin/cos data
	bump	rdx, 4*XMM_HSCD2	;; Next sin/cos data
	loops	32, c4b			;; Test inner loop counter
	bump	rsi, -32*64+dist128	;; Next source pointer
	bump	rdi, -32*XMM_SCD2	;; Restore sin/cos ptr
	bump	rdx, -32*4*XMM_HSCD2	;; Restore sin/cos ptr

;; Do 1408 four_complex_djbunfft operations
;;	distance between fft data elements is 64
;;	do 352 macros each processing 32 data values

	loops_reset			;; 11 iters of 32
	r4_x4cl_four_complex_djbunfft_preload
c5b:	r4_x4cl_four_complex_djbunfft rsi, 64, 32*64, 64*64, rdi, 32*XMM_SCD2
	bump	rdi, XMM_SCD2		;; Next sin/cos data
	loops	32, c5b			;; Test inner loop counter
	bump	rsi, -32*64+dist128	;; Next source pointer
	bump	rdi, -32*XMM_SCD2	;; Restore sin/cos ptr
	loops	12, c5b			;; Test loop counter
	bump	rsi, -12*dist128	;; Next source pointer

;; Do inverse FFT levels 3,4
;; On input the 64-byte cache lines hold these data values:
;;	0	+6K	128	+6K	256	+6K	384	+6K
;;	1	...
;;	...
;;	127	...
;;	512	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+6K	512	+6K	1024	+6K	1536	+6K
;;	1	...
;;	...
;;	511	...
;;	2048	...
;;	...

;; Do 256 eight_reals_unfft operations
;; Do 256 four_complex_djbunfft operations
;;	distance between fft data elements is 256
;;	do 128 macros processing 32 data values

	loops_init_prefetch 384, 64, 2, rcx
	xtouch	[rcx+4096-64]		;; Preload TLBs
	xtouch	[rcx+2*4096-64]
	xtouch	[rcx+3*4096-64]
	mov	rdi, xsincos_complex	;; Load sin/cos pointer
	mov	rbx, rdi
c2b:	r4_h4cl_2sc_eight_reals_four_complex_djbunfft rsi, 64, dist128, 2*dist128, rdi, 128*3*XMM_SCD1, rbx, 128*6*XMM_SCD1, rdx, 128*XMM_HSCD2
	bump	rdi, 3*XMM_SCD1		;; Next sin/cos data
	bump	rbx, 6*XMM_SCD1		;; Next sin/cos data
	bump	rdx, XMM_HSCD2		;; Next sin/cos data
	loops	128, c2b		;; Test inner loop counter
	bump	rsi, -128*64+4*dist128	;; Next source pointer
	bump	rdi, -128*3*XMM_SCD1	;; Restore sin/cos ptr
	bump	rbx, -128*6*XMM_SCD1	;; Restore sin/cos ptr
	bump	rdx, -128*XMM_HSCD2-1024*XMM_HSCD1 ;; Restore sin/cos ptr

;; Do 1024 four_complex_djbunfft operations
;;	distance between fft data elements is 256
;;	do 256 macros each processing 32 data values

	loops_reset			;; 2 iters of 128
	r4_x4cl_2sc_four_complex_djbunfft_preload
c3b:	r4_x4cl_2sc_four_complex_djbunfft rsi, 64, dist128, 2*dist128, rdi, 128*3*XMM_SCD1, rbx, 128*6*XMM_SCD1
	bump	rdi, 3*XMM_SCD1		;; Next sin/cos data
	bump	rbx, 6*XMM_SCD1		;; Next sin/cos data
	loops	128, c3b		;; Test inner loop counter
	bump	rsi, -128*64+4*dist128	;; Next source pointer
	bump	rdi, -128*3*XMM_SCD1	;; Restore sin/cos ptr
	bump	rbx, -128*6*XMM_SCD1	;; Restore sin/cos ptr
	loops	3, c3b			;; Test loop counter
	bump	rsi, -3*4*dist128	;; Next source pointer

;; Do inverse FFT level 1,2 (actually 1.585 FFT levels)
;; On input the 64-byte cache lines hold these data values:
;;	0	+6K	512	+6K	1024	+6K	1536	+6K
;;	1	...
;;	...
;;	511	...
;;	2048	...
;;	...
;; On output the 64-byte cache lines hold these data values if pass 1 swizzles:
;;	0	+6K	1536	+6K	3072	+6K	4608	+6K
;;	1	...
;;	...
;;	1535
;; On output the 64-byte cache lines hold these data values if pass 2 swizzles:
;;	0	+1.5K	3072	+1.5K	6144	+1.5K	9216	+1.5K
;;	1	...
;;	...
;;	1535

;; Do 1024 six_reals_unfft operations
;; Do 1024 three_complex_djbunfft operations
;;	distance between fft data elements is 1024
;;	do 512 macros processing 24 data values

	bump	rcx, -10*64		;; Back up prefetch pointer by 10 cache lines
	loops_init_prefetch 512, 64, 4, rcx
	xtouch	[rcx+4096-64]		;; Preload TLBs
	xtouch	[rcx+2*4096-64]
c1b:	r3_h3cl_six_reals_three_complex_djbunfft rsi, 64, 4*dist128, rdi, 512*XMM_SCD1, rdx, 512*XMM_HSCD1
	bump	rdi, XMM_SCD1		;; Next sin/cos data
	bump	rdx, XMM_HSCD1		;; Next sin/cos data
	loops	128, c1b		;; Test loop counter
	bump	rsi, -128*64+dist128	;; Next source pointer
	loops	4, c1b			;; Test loop counter

xpass2_3072_real_done:
	end_timer 2
	ENDM

; *************** radix4-pass2-3072-levels-complex macro ******************
; This macro takes 2 groups of 3072 complex values and
; performs the final 11.585 levels of a standard mostly radix-4 FFT, squares the
; results, and does the first 11.585 levels of the inverse FFT.

r4_pass2_3072_levels_complex MACRO
	LOCAL	b1a0, b1a, b1b, b2b, b3a, b3b, b4b, b5b, b6a, b6b0, b6b, b6c, b6d0, b6d
	LOCAL	c1b, c2a0, c2a, c2b, c3b, c4b, c5a, c5b
	LOCAL	xpass2_3072_levels_complex_1, xpass2_3072_levels_complex_2
	LOCAL	xpass2_3072_levels_complex_3, xpass2_3072_levels_complex_4
	LOCAL	xpass2_3072_levels_complex_unfft, xpass2_3072_levels_complex_done

;; Type 4 FFTs skip the forward FFT process

	mov	rsi, DATA_ADDR		;; Load address of FFT data
	mov	rbx, DIST_TO_FFTSRCARG
	cmp	ffttype, 4
	je	xpass2_3072_levels_complex_4

;; Do FFT levels 1,2 (actually 1.585 levels)
;;
;; We swizzle the data so that one set of data is in the high word of
;; an xmm register, the other set is in the low word.  This means
;; a distance of 6144 data values.
;;
;; On input the 64-byte cache lines hold these data values if pass 2 swizzles:
;;	0	+1.5K	3072	+1.5K	6144	+1.5K	9216	+1.5K
;;	1	...
;;	...
;;	1535
;; On input the 64-byte cache lines hold these data values if pass 1 swizzles:
;;	0	+6K	1536	+6K	3072	+6K	4608	+6K
;;	1	...
;;	...
;;	1535	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+6K	512	+6K	1024	+6K	1536	+6K
;;	1	...
;;	...
;;	511	...
;;	2048	...
;;	...

;; Do 2048 three_complex_fft operations
;;	distance between fft data elements is 1024
;;	do 512 macros each processing 24 data values

;; Do FFT levels 3,4
;;
;; On input the 64-byte cache lines hold these data values:
;;	0	+6K	512	+6K	1024	+6K	1536	+6K
;;	1	...
;;	...
;;	511	...
;;	2048	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+6K	256	+6K	512	+6K	768	+6K
;;	1	...
;;	...
;;	255	...
;;	1024	...
;;	...

;; Do 1536 four_complex_fft operations
;;	distance between fft data elements is 256
;;	do 768 macros each processing 16 data values

IF @INSTR(,%xarch,<CORE>) NE 0		;; Core 2 faster doing separate loops

	start_timer 4
	mov	rdi, xsincos_complex	;; The sin/cos data
	loops_init 512			;; 4 iterations of 128
	r3_f3cl_three_complex_djbfft_preload
b1b:	r3_f3cl_three_complex_djbfft rsi, 64, 4*dist128, rdi, 512*XMM_SCD1
	bump	rdi, XMM_SCD1		;; Next sin/cos data
	loops	128, b1b		;; Test inner loop counter
	bump	rsi, -128*64+dist128	;; Next source pointer
	loops	4, b1b			;; Test outer loop counter
	bump	rsi, -4*dist128		;; Restore source pointer
	end_timer 4

	start_timer 5
	mov	rdi, xsincos_complex	;; Ptr to the sin/cos data
	mov	rdx, rdi
	mov	rcx, DATA_PREFETCH	;; Prefetch pointer (we need to prefetch 
					;; 1536 cache lines + 22 pad cache lines)
	xtouch	[rcx]			;; Load first prefetch TLB
	loops_init 768			;; 3 iters of 2 iters of 128
	r4_x2cl_2sc_four_complex_djbfft_preload
b2b:	r4_x2cl_2sc_four_complex_djbfft rsi, 64, 2*dist128, rdi, rdx
	bump	rdi, 3*XMM_SCD1		;; Next sin/cos data
	bump	rdx, 6*XMM_SCD1		;; Next sin/cos data
	loops	128, b2b		;; Test inner loop counter
	bump	rsi, -128*64+dist128	;; Next source pointer
	loops	2, b2b			;; Test loop counter
	bump	rsi, -2*dist128+4*dist128 ;; Next source pointer
	bump	rdi, -256*3*XMM_SCD1	;; Restore sin/cos ptr
	bump	rdx, -256*6*XMM_SCD1	;; Restore sin/cos ptr
	loops	3, b2b			;; Test loop counter
	bump	rsi, -3*4*dist128	;; Restore source pointer
	end_timer 5

ELSE					;; Pentium 4 faster doing merged loops

	start_timer 4
	mov	rdi, xsincos_complex	;; The sin/cos data
	mov	rbp, rdi
	mov	rdx, rdi
	mov	rcx, DATA_PREFETCH	;; Prefetch pointer (we need to prefetch 
					;; 1536 cache lines + 22 pad cache lines)
	xtouch	[rcx]			;; Load first prefetch TLB
	loops_init_prefetch 1536, 64, 12, rcx ;; 256 iterations of 3

b1a0:	xtouch	[rcx+4096-64]		;; Load prefetch TLBs
b1a:	r3_f3cl_three_complex_djbfft_preload
b1b:	r3_f3cl_three_complex_djbfft rsi, 2*dist128, 4*dist128, rdi, 512*XMM_SCD1
	xxprefetch [rdi+XMM_SCD1]	;; Prefetch for next pass
	xxprefetch [rdi+513*XMM_SCD1]	;; Prefetch for next pass
	bump	rdi, 256*XMM_SCD1	;; Next sin/cos data
	loops	2, b1b, 3		;; Test loop counter with non-standard incr
	bump	rsi, -4*dist128		;; Restore source pointer
	loops_undo 6			;; Rollback loop counter

	r4_x2cl_2sc_four_complex_djbfft_preload
b2b:	r4_x2cl_2sc_four_complex_djbfft rsi, 4*dist128, 2*dist128, rbp, rdx
	loops	3, b2b, 2		;; Test loop counter with non-standard incr

	bump	rsi, -3*4*dist128+64	;; Next source pointer
	bump	rdi, -2*256*XMM_SCD1+XMM_SCD1 ;; Next sin/cos data
	bump	rbp, 3*XMM_SCD1		;; Next sin/cos data
	bump	rdx, 6*XMM_SCD1		;; Next sin/cos data
	loops	128, b1a		;; Test loop counter
	bump	rsi, -128*64+dist128	;; Next source pointer
	loops	2, b1a0			;; Test loop counter
	bump	rsi, -2*dist128		;; Restore source pointer
	bump	rdi, -256*XMM_SCD1	;; Restore sin/cos ptr
	end_timer 4

ENDIF

;; Do FFT levels 5,6
;;
;; On input the 64-byte cache lines hold these data values:
;;	0	+6K	256	+6K	512	+6K	768	+6K
;;	1	...
;;	...
;;	255	...
;;	1024	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+6K	64	+6K	128	+6K	192	+6K
;;	1	...
;;	...
;;	63	...
;;	256	...
;;	...
;;
;; Do 1536 four_complex_fft operations
;;	distance between fft data elements is 64
;;	do 384 macros each processing 32 data values

;; Do FFT levels 7,8
;;
;; On input the 64-byte cache lines hold these data values:
;;	0	+6K	64	+6K	128	+6K	192	+6K
;;	1	...
;;	...
;;	63	...
;;	256	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+6K	16	+6K	32	+6K	48	+6K
;;	1	...
;;	...
;;	15	...
;;	64	...
;;	...
;;
;; Do 1536 four_complex_fft operations
;;	distance between fft data elements is 16
;;	do 384 macros each processing 32 data values

;; Do FFT levels 9,10
;;
;; On input the 64-byte cache lines hold these data values:
;;	0	+6K	16	+6K	32	+6K	48	+6K
;;	1	...
;;	...
;;	15	...
;;	64	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+6K	4	+6K	8	+6K	12	+6K
;;	1	...
;;	...
;;	3	...
;;	16	...
;;	...
;;
;; Do 1536 four_complex_fft operations
;;	distance between fft data elements is 4
;;	do 384 macros each processing 32 data values

;; Do this 64 cache lines at a time to maximize accessing data
;; while it is in the L1 cache.

	start_timer 6
	bump	rdi, 1536*XMM_SCD1	;; Load sin/cos pointers
	mov	rbx, rdi
	mov	rbp, rdi
	loops_init_prefetch 384, 64, 2, rcx ;; 6 iters of 4 of 64 cache lines
	r4_x4cl_four_complex_djbfft_preload

b3a:	xtouch	[rcx+4096-64]		;; Load prefetch TLBs
	xtouch	[rcx+2*4096-64]

b3b:	xxprefetch [rdi+4*XMM_SCD2]	;; Prefetch for next pass
	r4_x4cl_four_complex_djbfft rsi, 4*64, 64*64, dist128, rdi
	bump	rdi, 4*XMM_SCD2		;; Next sine/cosine pointer
	loops	16, b3b			;; Loop 16 times
	bump	rsi, -16*4*64		;; Restore source pointer
	bump	rdi, -16*4*XMM_SCD2	;; Restore sine/cosine pointer
	loops_undo 16			;; Back up loop counter

b4b:	r4_x4cl_four_complex_djbfft rsi, 4*64, 16*64, 32*64, rbx
	bump	rbx, 4*4*XMM_SCD2	;; Next sine/cosine pointer
	loops	4, b4b			;; Loop 4 times
	bump	rsi, -4*4*64+64*64	;; Next source pointer
	bump	rbx, -4*4*4*XMM_SCD2	;; Restore sine/cosine pointer
	loops	2, b4b			;; Loop twice
	bump	rsi, -2*64*64+dist128	;; Next source pointer
	loops	2, b4b			;; Loop twice
	bump	rsi, -2*dist128		;; Restore source pointer
	loops_undo 2*2*4		;; Back up loop counter

b5b:	r4_x4cl_four_complex_djbfft rsi, 16*64, 4*64, 8*64, rbp
	loops	8, b5b			;; Loop 8 times
	bump	rsi, -8*16*64+dist128	;; Next source pointer
	loops	2, b5b			;; Loop twice

	bump	rsi, -2*dist128+64	;; Next source pointer
	bump	rdi, XMM_SCD2		;; Next sine/cosine pointer
	bump	rbx, 4*XMM_SCD2		;; Next sine/cosine pointer
	bump	rbp, 16*XMM_SCD2	;; Next sine/cosine pointer
	loops	4, b3b			;; Loop 4 times
	bump	rsi, -4*64+2*dist128	;; Next source pointer
	bump	rdi, -4*XMM_SCD2	;; Restore sine/cosine pointer
	bump	rbx, -4*4*XMM_SCD2	;; Restore sine/cosine pointer
	bump	rbp, -4*16*XMM_SCD2	;; Restore sine/cosine pointer
	loops	6, b3a			;; Loop 6 times
	bump	rsi, -6*2*dist128	;; Restore source pointer
	end_timer 6

;; Do FFT levels 11,12 as well as inverse FFT levels 11,12
;;
;; On input the 64-byte cache lines hold these data values:
;;	0	+6K	4	+6K	8	+6K	12	+6K
;;	1	...
;;	...
;;	3	...
;;	16	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+6K	2	+6K	4	+6K	6	+6K
;;	1	...
;;	8	...
;;	...

;; Do 1536 four_complex_fft operations
;;	distance between fft data elements is 1
;;	do 384 macros each processing 32 data values

;; Execute the right middle step

	cmp	ffttype, 2
	jl	xpass2_3072_levels_complex_1
	jg	xpass2_3072_levels_complex_3

xpass2_3072_levels_complex_2:
	start_timer 9
	loops_init_prefetch 384, 64, 4, rcx
	xtouch	[rcx+4096-64]		;; Load prefetch TLBs
	xtouch	[rcx+2*4096-64]
	r4_x4cl_four_complex_with_square_preload
b6a:	r4_x4cl_four_complex_with_square rsi, 4*64, 64, 2*64
	loops	32, b6a			;; Loop 32 times
	bump	rsi, -32*4*64+dist128	;; Next source pointer
	loops	12, b6a			;; Loop 12 times
	bump	rsi, -12*dist128	;; Restore source pointer
	end_timer 9
	jmp	xpass2_3072_levels_complex_unfft

xpass2_3072_levels_complex_1:
	start_timer 9
	loops_init_prefetch 384, 128, 1, rcx
b6b0:	xtouch	[rcx+4096-64]		;; Load prefetch TLBs
b6b:	r4_x4cl_four_complex_fft_final rsi, 4*64, 64, 2*64
	loops	32, b6b			;; Loop 32 times
	bump	rsi, -32*4*64+dist128	;; Next source pointer
	loops	12, b6b0		;; Loop 12 times
	end_timer 9
	jmp	xpass2_3072_levels_complex_done

xpass2_3072_levels_complex_3:
	start_timer 9
	mov	rbp, DIST_TO_MULSRCARG
	loops_init_prefetch 384, 64, 4, rcx
	xtouch	[rcx+4096-64]		;; Load prefetch TLBs
	xtouch	[rcx+2*4096-64]
b6c:	r4_x4cl_four_complex_with_mult rsi, 4*64, 64, 2*64
	loops	32, b6c			;; Loop 32 times
	bump	rsi, -32*4*64+dist128	;; Next source pointer
	loops	12, b6c			;; Loop 12 times
	bump	rsi, -12*dist128	;; Restore source pointer
	end_timer 9
	jmp	xpass2_3072_levels_complex_unfft

xpass2_3072_levels_complex_4:
	start_timer 9
	mov	rcx, DATA_PREFETCH	;; Prefetch pointer
	xtouch	[rcx]			;; Load first prefetch TLB
	mov	rbp, DIST_TO_MULSRCARG
	loops_init_prefetch 384, 128, 1, rcx
b6d0:	xtouch	[rcx+4096-128]		;; Load prefetch TLBs
b6d:	r4_x4cl_four_complex_with_mulf rsi, 4*64, 64, 2*64
	loops	32, b6d			;; Loop 32 times
	bump	rsi, -32*4*64+dist128	;; Next source pointer
	loops	12, b6d0		;; Loop 12 times
	bump	rsi, -12*dist128	;; Restore source pointer
	bump	rcx, -96*64		;; Back up prefetch pointer
	end_timer 9

;; Do inverse FFT levels 9,10
;; On input the 64-byte cache lines hold these data values:
;;	0	+6K	2	+6K	4	+6K	6	+6K
;;	1	...
;;	8	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+6K	8	+6K	16	+6K	24	+6K
;;	1	...
;;	...
;;	7	...
;;	32	...
;;	...
;;
;; Do 1536 four_complex_unfft operations
;;	distance between fft data elements is 4
;;	do 384 macros each processing 32 data values

;; Do inverse FFT levels 7,8
;; On input the 64-byte cache lines hold these data values:
;;	0	+6K	8	+6K	16	+6K	24	+6K
;;	1	...
;;	...
;;	7	...
;;	32	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+6K	32	+6K	64	+6K	96	+6K
;;	1	...
;;	...
;;	31	...
;;	128	...
;;	...
;;
;; Do 1536 four_complex_unfft operations
;;	distance between fft data elements is 16
;;	do 384 macros each processing 32 data values

;; Do inverse FFT levels 5,6
;; On input the 64-byte cache lines hold these data values:
;;	0	+6K	32	+6K	64	+6K	96	+6K
;;	1	...
;;	...
;;	31	...
;;	128	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+6K	128	+6K	256	+6K	384	+6K
;;	1	...
;;	...
;;	127	...
;;	512	...
;;	...
;;
;; Do 1536 four_complex_unfft operations
;;	distance between fft data elements is 64
;;	do 384 macros each processing 32 data values

;; Do this 64 cache lines at a time to maximize accessing data
;; while it is in the L1 cache.

xpass2_3072_levels_complex_unfft:
	start_timer 10
	mov	rdi, xsincos_complex	;; Load sin/cos pointers
	bump	rdi, 1536*XMM_SCD1
	mov	rbx, rdi
	mov	rbp, rdi
	loops_init_prefetch 384, 64, 2, rcx ;; 12*2 iters of 64 cache lines
	r4_x4cl_four_complex_djbunfft_preload

c5a:	xtouch	[rcx+4096-64]		;; Load prefetch TLBs

c5b:	r4_x4cl_four_complex_djbunfft rsi, 8*64, 2*64, 4*64, rbp, 2*16*XMM_SCD2
	loops	16, c5b			;; Loop 16 times
	bump	rsi, -16*8*64		;; Restore source pointer
	loops_undo 16			;; Restore loop counter

c4b:	r4_x4cl_four_complex_djbunfft rsi, 2*64, 8*64, 16*64, rbx, 8*4*XMM_SCD2
	bump	rbx, 2*4*XMM_SCD2	;; Next sine/cosine pointer
	loops	4, c4b			;; Loop 4 times
	bump	rsi, -4*2*64+32*64	;; Next source pointer
	bump	rbx, -4*2*4*XMM_SCD2	;; Restore sine/cosine pointer
	loops	4, c4b			;; Loop 4 times
	bump	rsi, -4*32*64		;; Restore source pointer
	loops_undo 4*4			;; Restore loop counter

c3b:	r4_x4cl_four_complex_djbunfft rsi, 2*64, 32*64, 64*64, rdi, 32*1*XMM_SCD2
	bump	rdi, 2*XMM_SCD2		;; Next sine/cosine pointer
	loops	16, c3b			;; Loop 16 times
	bump	rdi, -16*2*XMM_SCD2	;; Restore sine/cosine pointer

	bump	rsi, -16*2*64+64	;; Next source pointer
	bump	rdi, XMM_SCD2		;; Next sine/cosine pointer
	bump	rbx, 4*XMM_SCD2		;; Next sine/cosine pointer
	bump	rbp, 16*XMM_SCD2	;; Next sine/cosine pointer
	loops	2, c5b			;; Loop twice
	bump	rsi, -2*64+dist128	;; Next source pointer
	bump	rdi, -2*XMM_SCD2	;; Restore sine/cosine pointer
	bump	rbx, -2*4*XMM_SCD2	;; Restore sine/cosine pointer
	bump	rbp, -2*16*XMM_SCD2	;; Restore sine/cosine pointer
	loops	12, c5a			;; Loop 12 times
	bump	rsi, -12*dist128	;; Restore source pointer
	end_timer 10

;; Do inverse FFT levels 3,4
;; On input the 64-byte cache lines hold these data values:
;;	0	+6K	128	+6K	256	+6K	384	+6K
;;	1	...
;;	...
;;	127	...
;;	512	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+6K	512	+6K	1024	+6K	1536	+6K
;;	1	...
;;	...
;;	511	...
;;	2048	...
;;	...

;; Do 1536 four_complex_unfft operations
;;	distance between fft data elements is 256
;;	do 384 macros each processing 32 data values

;; Do inverse FFT levels 1,2 (actually 1.585 FFT levels)
;; On input the 64-byte cache lines hold these data values:
;;	0	+6K	512	+6K	1024	+6K	1536	+6K
;;	1	...
;;	...
;;	511	...
;;	2048	...
;;	...
;; On output the 64-byte cache lines hold these data values if pass 1 swizzles:
;;	0	+6K	1536	+6K	3072	+6K	4608	+6K
;;	1	...
;;	...
;;	1535
;; On output the 64-byte cache lines hold these data values if pass 2 swizzles:
;;	0	+1.5K	3072	+1.5K	6144	+1.5K	9216	+1.5K
;;	1	...
;;	...
;;	1535

;; Do 2048 three_complex_unfft operations
;;	distance between fft data elements is 1024
;;	do 512 macros each processing 24 data values

IF @INSTR(,%xarch,<CORE>) NE 0		;; Core 2 faster doing separate loops

	start_timer 13
	mov	rdi, xsincos_complex	;; Load sin/cos pointer
	mov	rdx, rdi
	loops_init_prefetch 384, 64, 2, rcx;; 3 iters of 128
	xtouch	[rcx+4096-64]		;; Load prefetch TLBs
	xtouch	[rcx+2*4096-64]
	xtouch	[rcx+3*4096-64]
	r4_x4cl_2sc_four_complex_djbunfft_preload
c2b:	r4_x4cl_2sc_four_complex_djbunfft rsi, 64, dist128, 2*dist128, rdi, 128*3*XMM_SCD1, rdx, 128*6*XMM_SCD1
	bump	rdi, 3*XMM_SCD1		;; Next sine/cosine pointer
	bump	rdx, 6*XMM_SCD1		;; Next sine/cosine pointer
	loops	128, c2b		;; Loop 128 times
	bump	rsi, -128*64+4*dist128	;; Next source pointer
	bump	rdi, -128*3*XMM_SCD1	;; Reset sine/cosine pointer
	bump	rdx, -128*6*XMM_SCD1	;; Reset sine/cosine pointer
	loops	3, c2b			;; Test loop counter
	bump	rsi, -3*4*dist128	;; Restore source pointer
	end_timer 13

	start_timer 14
	mov	rdi, xsincos_complex	;; Load sin/cos pointer
	bump	rcx, -10*64		;; Back up prefetch pointer by 10 cache lines
	loops_init_prefetch 512, 64, 4, rcx ;; 4 iters of 128
	xtouch	[rcx+4096-64]		;; Load prefetch TLBs
	xtouch	[rcx+2*4096-64]
	r3_x3cl_three_complex_djbunfft_preload
c1b:	r3_x3cl_three_complex_djbunfft rsi, 64, 4*dist128, rdi, 512*XMM_SCD1
	bump	rdi, XMM_SCD1		;; Next set of sin/cos data
	loops	128, c1b		;; Test inner loop counter
	bump	rsi, -128*64+dist128	;; Next source pointer
	loops	4, c1b			;; Test outer loop counter
	end_timer 14

ELSE					;; Pentium 4 faster doing merged loops

	start_timer 13
	mov	rdi, xsincos_complex	;; Load sin/cos pointer
	mov	rbp, rdi
	mov	rdx, rdi
	bump	rcx, -74*64		;; Back up prefetch pointer by 74 cache lines
	loops_init_prefetch 1536, 64, 12, rcx ;; 4 iters of 128, prefetching 128 cache lines each loop

c2a0:	xtouch	[rcx+4096-64]		;; Load prefetch TLB

c2a:	r4_x4cl_2sc_four_complex_djbunfft_preload
c2b:	r4_x4cl_2sc_four_complex_djbunfft rsi, 4*dist128, dist128, 2*dist128, rbp, 128*3*XMM_SCD1, rdx, 128*6*XMM_SCD1
	loops	3, c2b, 4		;; Test loop counter with non-standard incr
	bump	rsi, -3*4*dist128	;; Restore source pointer
	loops_undo 12

	r3_x3cl_three_complex_djbunfft_preload
c1b:	r3_x3cl_three_complex_djbunfft rsi, dist128, 4*dist128, rdi, 512*XMM_SCD1
	bump	rdi, 128*XMM_SCD1	;; Next set of sin/cos data
	loops	4, c1b, 3		;; Test loop counter with non-standard incr

	bump	rsi, -4*dist128+64	;; Next source pointer
	bump	rdi, -4*128*XMM_SCD1+XMM_SCD1 ;; Next sin/cos ptr
	bump	rbp, 3*XMM_SCD1		;; Next sine/cosine pointer
	bump	rdx, 6*XMM_SCD1		;; Next sine/cosine pointer
	loops_touch 32, c2a, 4, c2a0	;; Test loop counter
	end_timer 13

ENDIF

xpass2_3072_levels_complex_done:
	ENDM


; *************** radix4-pass2-3840-levels-real macro ******************
; This macro takes 7680 real values and performs the final 12.907 levels of
; the FFT.  Independently, this macro also takes 3840 complex values and
; performs the final 11.907 levels of the FFT.  The results are squared, and
; the first 11.907/12.907 levels of the inverse FFT are done.

r4_pass2_3840_levels_real MACRO
	LOCAL	b1b, b2b, b3b, b4b, b5b, b6b, b7b
	LOCAL	b8b, b9b, baa, bab0, bab, bac, bad0, bad
	LOCAL	c1a, c1b, c2b, c3b, c4b, c5b, c6b, c7b, c8b, c9b
	LOCAL	xpass2_3840_levels_real_1, xpass2_3840_levels_real_2
	LOCAL	xpass2_3840_levels_real_3, xpass2_3840_levels_real_4
	LOCAL	xpass2_3840_real_unfft, xpass2_3840_real_done

;; We switch to the same format used in one-pass FFTs (that is, the
;; high word of xmm registers being 1 greater than the low word):

;; Type 4 FFTs skip the forward FFT process

	start_timer 2
	mov	rsi, DATA_ADDR		;; Load source address
	mov	rbx, DIST_TO_FFTSRCARG
	cmp	ffttype, 4
	je	xpass2_3840_levels_real_4

;; Do FFT level 1,2 (actually 1.585 levels)
;;
;; On input the 64-byte cache lines hold these data values if pass 2 swizzles:
;;	0	+1920	3840	+1920	7680	+1920	11520	+1920
;;	1	...
;;	...
;;	1919
;; On input the 64-byte cache lines hold these data values if pass 1 swizzles:
;;	0	+7680	1920	+7680	3840	+7680	5760	+7680
;;	1	...
;;	...
;;	1919	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+7680	640	+7680	1280	+7680	1920	+7680
;;	1	...
;;	...
;;	639	...
;;	2560	...
;;	...

;; Do 1280 six_reals_fft on values 0 - 7679
;; Do 1280 three_complex_djbfft on values 7680 - 15359
;;	distance between fft data elements is 1280
;;	do 640 iterations

	mov	rdi, xsincos_complex	;; Ptr to the sin/cos data
	mov	rdx, sincos3		;; Ptr to real sin/cos data
	loops_init 640
b1b:	r3_fh3cl_six_reals_three_complex_djbfft rsi, 64, 5*dist128, rdi, 640*XMM_SCD1, rdx, 640*XMM_HSCD1
	bump	rdi, XMM_SCD1		;; Next sin/cos data
	bump	rdx, XMM_HSCD1		;; Next sin/cos data
	loops	128, b1b		;; Test loop counter
	bump	rsi, -128*64+dist128	;; Next source pointer
	loops	5, b1b			;; Test loop counter
	bump	rsi, -5*dist128		;; Restore source pointer
	bump	rdi, -640*XMM_SCD1	;; Restore sin/cos ptr
	bump	rdx, 640*XMM_HSCD1	;; Next sin/cos ptr

;; Do FFT levels 3,4 (actually 2.322 FFT levels)
;; Values 0-2559 is real data, 2560-15359 is complex data.
;;
;; On input the 64-byte cache lines hold these data values:
;;	0	+7680	640	+7680	1280	+7680	1920	+7680
;;	1	...
;;	...
;;	639	...
;;	2560	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+7680	128	+7680	256	+7680	384	+7680
;;	1	...
;;	...
;;	127	...
;;	512	...
;;	...

;; Do 256 ten_reals_fft operations
;; Do 256 five_complex_djbfft operations
;;	distance between fft data elements is 256
;;	do 128 macros each processing 40 data values

	mov	rcx, DATA_PREFETCH	;; Prefetch pointer
	xtouch	[rcx]			;; Preload first TLB
	mov	rbx, rdi
	loops_init 384
b2b:	r5_h5cl_2sc_ten_reals_five_complex_djbfft rsi, 64, dist128, rdi, 128*3*XMM_SCD1, rbx, 128*6*XMM_SCD1, rdx, 128*XMM_HSCD2
	bump	rdi, 3*XMM_SCD1		;; Next sin/cos data
	bump	rbx, 6*XMM_SCD1		;; Next sin/cos data
	bump	rdx, XMM_HSCD2		;; Next sin/cos data
	loops	128, b2b		;; Test inner loop counter
	bump	rsi, -128*64+5*dist128	;; Next source pointer
	bump	rdi, -128*3*XMM_SCD1	;; Restore sin/cos ptr
	bump	rbx, -128*6*XMM_SCD1	;; Restore sin/cos ptr
	bump	rdx, 128*XMM_HSCD2	;; Next sin/cos ptr

;; Do 1024 five_complex_fft operations
;;	distance between fft data elements is 256
;;	do 256 macros each processing 40 data values

	loops_reset
	r5_nf5cl_2sc_five_complex_djbfft_preload
b3b:	r5_nf5cl_2sc_five_complex_djbfft rsi, 64, dist128, rdi, 128*3*XMM_SCD1, rbx, 128*6*XMM_SCD1
	bump	rdi, 3*XMM_SCD1		;; Next sin/cos data
	bump	rbx, 6*XMM_SCD1		;; Next sin/cos data
	loops	128, b3b		;; Test inner loop counter
	bump	rsi, -128*64+5*dist128	;; Next source pointer
	bump	rdi, -128*3*XMM_SCD1	;; Restore sin/cos ptr
	bump	rbx, -128*6*XMM_SCD1	;; Restore sin/cos ptr
	loops	3, b3b			;; Test loop counter
	bump	rsi, -3*5*dist128	;; Restore source pointer

;; Do FFT levels 5,6
;; Values 0-511 is real data, 512-15359 is complex data.
;;
;; On input the 64-byte cache lines hold these data values:
;;	0	+7680	128	+7680	256	+7680	384	+7680
;;	1	...
;;	...
;;	127	...
;;	512	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+7680	64	+7680	128	+7680	192	+7680
;;	1	...
;;	...
;;	63	...
;;	256	...
;;	...

;; Do 64 eight_reals_fft operations
;; Do 192 four_complex_djbfft operations
;;	distance between fft data elements is 64
;;	do 64 macros each processing 16 data values

	bump	rdi, 1536*XMM_SCD1	;; Load sin/cos pointer
	loops_init_prefetch 960, 64, 4, rcx
	xtouch	[rcx+4096-128]		;; Preload the TLBs
	xtouch	[rcx+2*4096-128]
	xtouch	[rcx+3*4096-128]
	xtouch	[rcx+4*4096-128]
b4b:	r4_h2cl_eight_reals_four_complex_djbfft rsi, 64, 64*64, rdi, rdx
	bump	rdi, XMM_SCD2		;; Next sin/cos data
	bump	rdx, XMM_HSCD2		;; Next sin/cos data
	loops	64, b4b			;; Test loop counter
	bump	rsi, -64*64+dist128	;; Next source pointer
	bump	rdi, -64*XMM_SCD2	;; Restore sin/cos ptr
	bump	rdx, -64*XMM_HSCD2	;; Restore sin/cos ptr

;; Do 1792 four_complex_djbfft operations
;;	distance between fft data elements is 64
;;	do 896 macros each processing 16 data values

	loops_reset			;; 14 more iterations of 64
	r4_nf2cl_four_complex_djbfft_preload
b5b:	r4_nf2cl_four_complex_djbfft rsi, 64, 64*64, rdi
	bump	rdi, XMM_SCD2		;; Next sin/cos data
	loops	64, b5b			;; Test inner loop counter
	bump	rsi, -64*64+dist128	;; Next source pointer
	bump	rdi, -64*XMM_SCD2	;; Restore sin/cos ptr
	loops	15, b5b			;; Test loop counter
	bump	rsi, -15*dist128	;; Next source pointer

;; Do FFT levels 7,8
;; Values 0-127 is real data, 128-15359 is complex data.
;;
;; On input the 64-byte cache lines hold these data values:
;;	0	+7680	64	+7680	128	+7680	192	+7680
;;	1	...
;;	...
;;	63	...
;;	256	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+7680	16	+7680	32	+7680	48	+7680
;;	1	...
;;	...
;;	15	...
;;	64	...
;;	...

;; Do 16 eight_reals_fft operations
;; Do 48 four_complex_djbfft operations
;;	distance between fft data elements is 16
;;	do 16 macros each processing 32 data values

	loops_init_prefetch 480, 64, 2, rcx
	xtouch	[rcx+4096-128]		;; Preload the TLBs
	xtouch	[rcx+2*4096-128]
	xtouch	[rcx+3*4096-128]
	xtouch	[rcx+4*4096-128]
	r4_h4cl_eight_reals_four_complex_djbfft_preload
b6b:	r4_h4cl_eight_reals_four_complex_djbfft rsi, 64, 16*64, 32*64, rdi, rdx
	bump	rdi, 4*XMM_SCD2		;; Next sin/cos data
	bump	rdx, 4*XMM_HSCD2	;; Next sin/cos data
	loops	16, b6b			;; Test loop counter
	bump	rsi, -16*64+64*64	;; Next source pointer
	bump	rdi, -16*4*XMM_SCD2	;; Restore sin/cos ptr
	bump	rdx, -16*4*XMM_HSCD2	;; Restore sin/cos ptr

;; Do 1856 four_complex_djbfft operations
;;	distance between fft data elements is 16
;;	do 464 macros each processing 32 data values

	loops_reset			;; 15 iters of 1 or 2 of 16
	r4_x4cl_four_complex_djbfft_preload
b7b:	r4_x4cl_four_complex_djbfft rsi, 64, 16*64, 32*64, rdi
	bump	rdi, 4*XMM_SCD2		;; Next sin/cos data
	loops	16, b7b			;; Test inner loop counter
	bump	rsi, -16*64+64*64	;; Next source pointer
	bump	rdi, -16*4*XMM_SCD2	;; Restore sin/cos ptr
	loops	2, b7b			;; Test loop counter
	bump	rsi, -2*64*64+dist128	;; Next source pointer
	loops	15, b7b			;; 15 iterations
	bump	rsi, -15*dist128	;; Next source pointer

;; Do FFT levels 9,10
;; Values 0-31 is real data, 32-15359 is complex data.
;;
;; On input the 64-byte cache lines hold these data values:
;;	0	+7680	16	+7680	32	+7680	48	+7680
;;	1	...
;;	...
;;	15	...
;;	64	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+7680	4	+7680	8	+7680	12	+7680
;;	1	...
;;	...
;;	3	...
;;	16	...
;;	...

;; Do 4 eight_reals_fft operations
;; Do 12 four_complex_djbfft operations
;;	distance between fft data elements is 4
;;	do 4 macros each processing 32 data values

	loops_init_prefetch 480, 64, 2, rcx
	xtouch	[rcx+4096-128]		;; Preload the TLBs
	xtouch	[rcx+2*4096-128]
	xtouch	[rcx+3*4096-128]
	xtouch	[rcx+4*4096-128]
	r4_h4cl_eight_reals_four_complex_djbfft_preload
b8b:	r4_h4cl_eight_reals_four_complex_djbfft rsi, 64, 4*64, 8*64, rdi, rdx
	bump	rdi, 16*XMM_SCD2	;; Next sin/cos data
	bump	rdx, 16*XMM_HSCD2	;; Next sin/cos data
	loops	4, b8b			;; Test loop counter
	bump	rsi, -4*64+16*64	;; Next source pointer
	bump	rdi, -4*16*XMM_SCD2	;; Restore sin/cos ptr
	bump	rdx, -4*16*XMM_HSCD2	;; Restore sin/cos ptr

;; Do 1904 four_complex_djbfft operations
;;	distance between fft data elements is 4
;;	do 476 macros each processing 32 data values

	loops_reset			;; 15 iters of 7 or 8 of 4
	r4_x4cl_four_complex_djbfft_preload
b9b:	r4_x4cl_four_complex_djbfft rsi, 64, 4*64, 8*64, rdi
	bump	rdi, 16*XMM_SCD2	;; Next sin/cos data
	loops	4, b9b			;; Test loop counter
	bump	rsi, -4*64+16*64	;; Next source pointer
	bump	rdi, -4*16*XMM_SCD2	;; Restore sin/cos ptr
	loops	8, b9b			;; Test middle loop counter
	bump	rsi, -8*16*64+dist128	;; Next source pointer
	loops	15, b9b			;; 15 iterations
	bump	rsi, -15*dist128	;; Next source pointer

;; Do FFT levels 11,12
;; Values 0-7 is real data, 8-15359 is complex data.
;;
;; On input the 64-byte cache lines hold these data values:
;;	0	+7680	4	+7680	8	+7680	12	+7680
;;	1	...
;;	...
;;	3	...
;;	16	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+7680	2	+7680	4	+7680	6	+7680
;;	1	...
;;	8	...
;;	...

;; Do 1 eight_reals_fft operations
;; Do 3 four_complex_fft operations
;;	distance between fft data elements is 1

;; Do 1916 four_complex_fft operations
;;	distance between fft data elements is 1
;;	do 479 macros each processing 32 data values

;; Execute the proper middle step

	cmp	ffttype, 2
	jl	xpass2_3840_levels_real_1
	jg	xpass2_3840_levels_real_3

xpass2_3840_levels_real_2:
	r4_h4cl_eight_reals_four_complex_with_square rsi, 4*64, 64, 2*64
	loops_init 480, 1, 32
	r4_x4cl_four_complex_with_square_preload
baa:	r4_x4cl_four_complex_with_square rsi, 4*64, 64, 2*64
	loops	32, baa			;; Test loop counter
	bump	rsi, -32*4*64+dist128	;; Next source pointer
	loops	15, baa			;; Test loop counter
	bump	rsi, -15*dist128	;; Next source pointer
	jmp	xpass2_3840_real_unfft

xpass2_3840_levels_real_1:
	r4_h4cl_eight_reals_four_complex_fft_final rsi, 4*64, 64, 2*64
	loops_init_prefetch 480, 128, 1, rcx, 1, 32
bab0:	xtouch	[rcx+4096-64]		;; Preload the TLBs
bab:	r4_x4cl_four_complex_fft_final rsi, 4*64, 64, 2*64
	loops	32, bab			;; Test loop counter
	bump	rsi, -32*4*64+dist128	;; Next source pointer
	loops	15, bab0			;; Test loop counter
	jmp	xpass2_3840_real_done

xpass2_3840_levels_real_3:
	mov	rbp, DIST_TO_MULSRCARG
	r4_h4cl_eight_reals_four_complex_with_mult rsi, 4*64, 64, 2*64
	loops_init 480, 1, 32
bac:	r4_x4cl_four_complex_with_mult rsi, 4*64, 64, 2*64
	loops	32, bac			;; Test loop counter
	bump	rsi, -32*4*64+dist128	;; Next source pointer
	loops	15, bac			;; Test loop counter
	bump	rsi, -15*dist128	;; Next source pointer
	jmp	xpass2_3840_real_unfft

xpass2_3840_levels_real_4:
	mov	rbp, DIST_TO_MULSRCARG
	mov	rcx, DATA_PREFETCH	;; Load prefetch pointer
	xtouch	[rcx]			;; Preload first TLB
	r4_h4cl_eight_reals_four_complex_with_mulf rsi, 4*64, 64, 2*64
	loops_init_prefetch 480, 64, 1, rcx, 1, 32
bad0:	xtouch	[rcx+4096-64]		;; Preload the TLBs
bad:	r4_x4cl_four_complex_with_mulf rsi, 4*64, 64, 2*64
	loops	32, bad			;; Test loop counter
	bump	rsi, -32*4*64+dist128	;; Next source pointer
	loops	15, bad0		;; Test loop counter
	bump	rsi, -15*dist128	;; Next source pointer

;; Do inverse FFT levels 9,10
;; On input the 64-byte cache lines hold these data values:
;;	0	+7680	2	+7680	4	+7680	6	+7680
;;	1	...
;;	8	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+7680	8	+7680	16	+7680	24	+7680
;;	1	...
;;	...
;;	7	...
;;	32	...
;;	...

;; Do 4 eight_reals_unfft operations
;; Do 4 four_complex_djbunfft operations
;;	distance between fft data elements is 4
;;	do 2 macros processing 32 data values

xpass2_3840_real_unfft:
	mov	rdi, xsincos_complex	;; Load sin/cos pointers
	bump	rdi, 1536*XMM_SCD1
	mov	rdx, sincos3
	bump	rdx, 1280*XMM_HSCD1+256*XMM_HSCD2
	loops_init_prefetch 480, 64, 2, rcx
	xtouch	[rcx+4096-128]		;; Preload the TLBs
	xtouch	[rcx+2*4096-128]
	xtouch	[rcx+3*4096-128]
	xtouch	[rcx+4*4096-128]
c8b:	r4_h4cl_eight_reals_four_complex_djbunfft rsi, 64, 2*64, 4*64, rdi, 2*16*XMM_SCD2, rdx, 2*16*XMM_HSCD2
	bump	rdi, 16*XMM_SCD2	;; Next sin/cos data
	bump	rdx, 16*XMM_HSCD2	;; Next sin/cos data
	loops	2, c8b			;; Test loop counter
	bump	rsi, -2*64+8*64		;; Next source pointer
	bump	rdi, -2*16*XMM_SCD2	;; Restore sin/cos ptr
	bump	rdx, -2*16*XMM_HSCD2	;; Restore sin/cos ptr

;; Do 1912 four_complex_djbunfft operations
;;	distance between fft data elements is 4
;;	do 478 macros each processing 32 data values

	loops_reset			;; 15 iters of 15 or 16 of 2
	r4_x4cl_four_complex_djbunfft_preload
c9b:	r4_x4cl_four_complex_djbunfft rsi, 64, 2*64, 4*64, rdi, 2*16*XMM_SCD2
	bump	rdi, 16*XMM_SCD2	;; Next sine/cosine pointer
	loops	2, c9b			;; Test loop counter
	bump	rsi, -2*64+8*64		;; Next source pointer
	bump	rdi, -2*16*XMM_SCD2	;; Restore sin/cos ptr
	loops	16, c9b			;; Test loop counter
	bump	rsi, -16*8*64+dist128	;; Next source pointer
	loops	15, c9b			;; Test loop counter
	bump	rsi, -15*dist128	;; Next source pointer

;; Do inverse FFT levels 7,8
;; On input the 64-byte cache lines hold these data values:
;;	0	+7680	8	+7680	16	+7680	24	+7680
;;	1	...
;;	...
;;	7	...
;;	32	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+7680	32	+7680	64	+7680	96	+7680
;;	1	...
;;	...
;;	31	...
;;	128	...
;;	...

;; Do 16 eight_reals_unfft operations
;; Do 16 four_complex_djbunfft operations
;;	distance between fft data elements is 16
;;	do 8 macros processing 32 data values

	loops_init_prefetch 480, 64, 2, rcx
	xtouch	[rcx+4096-128]		;; Preload the TLBs
	xtouch	[rcx+2*4096-128]
	xtouch	[rcx+3*4096-128]
	xtouch	[rcx+4*4096-128]
c6b:	r4_h4cl_eight_reals_four_complex_djbunfft rsi, 64, 8*64, 16*64, rdi, 8*4*XMM_SCD2, rdx, 8*4*XMM_HSCD2
	bump	rdi, 4*XMM_SCD2		;; Next sin/cos data
	bump	rdx, 4*XMM_HSCD2	;; Next sin/cos data
	loops	8, c6b			;; Test inner loop counter
	bump	rsi, -8*64+32*64	;; Next source pointer
	bump	rdi, -8*4*XMM_SCD2	;; Restore sin/cos ptr
	bump	rdx, -8*4*XMM_HSCD2	;; Restore sin/cos ptr

;; Do 1888 four_complex_djbunfft operations
;;	distance between fft data elements is 16
;;	do 472 macros each processing 32 data values

	loops_reset			;; 15 iters of 3 or 4 of 4
	r4_x4cl_four_complex_djbunfft_preload
c7b:	r4_x4cl_four_complex_djbunfft rsi, 64, 8*64, 16*64, rdi, 8*4*XMM_SCD2
	bump	rdi, 4*XMM_SCD2		;; Next sin/cos data
	loops	8, c7b			;; Test inner loop counter
	bump	rsi, -8*64+32*64	;; Next source pointer
	bump	rdi, -8*4*XMM_SCD2	;; Restore sin/cos ptr
	loops	4, c7b			;; Test loop counter
	bump	rsi, -4*32*64+dist128	;; Next source pointer
	loops	15, c7b			;; 15 iterations
	bump	rsi, -15*dist128	;; Next source pointer

;; Do inverse FFT levels 5,6
;; On input the 64-byte cache lines hold these data values:
;;	0	+7680	32	+7680	64	+7680	96	+7680
;;	1	...
;;	...
;;	31	...
;;	128	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+7680	128	+7680	256	+7680	384	+7680
;;	1	...
;;	...
;;	127	...
;;	512	...
;;	...

;; Do 64 eight_reals_unfft operations
;; Do 64 four_complex_djbunfft operations
;;	distance between fft data elements is 64
;;	do 32 macros processing 32 data values

	loops_init_prefetch 480, 64, 2, rcx
	xtouch	[rcx+4096-128]		;; Preload the TLBs
	xtouch	[rcx+2*4096-128]
	xtouch	[rcx+3*4096-128]
	xtouch	[rcx+4*4096-128]
c4b:	r4_h4cl_eight_reals_four_complex_djbunfft rsi, 64, 32*64, 64*64, rdi, 32*XMM_SCD2, rdx, 32*XMM_HSCD2
	bump	rdi, XMM_SCD2		;; Next sin/cos data
	bump	rdx, XMM_HSCD2		;; Next sin/cos data
	loops	32, c4b			;; Test inner loop counter
	bump	rsi, -32*64+dist128	;; Next source pointer
	bump	rdi, -32*XMM_SCD2	;; Restore sin/cos ptr
	bump	rdx, -32*XMM_HSCD2-256*XMM_HSCD2 ;; Restore sin/cos ptr

;; Do 1792 four_complex_djbunfft operations
;;	distance between fft data elements is 64
;;	do 448 macros each processing 32 data values

	loops_reset			;; 14 more iters of 32
	r4_x4cl_four_complex_djbunfft_preload
c5b:	r4_x4cl_four_complex_djbunfft rsi, 64, 32*64, 64*64, rdi, 32*XMM_SCD2
	bump	rdi, XMM_SCD2		;; Next sin/cos data
	loops	32, c5b			;; Test inner loop counter
	bump	rsi, -32*64+dist128	;; Next source pointer
	bump	rdi, -32*XMM_SCD2	;; Restore sin/cos ptr
	loops	15, c5b			;; Test loop counter
	bump	rsi, -15*dist128	;; Next source pointer

;; Do inverse FFT levels 3,4 (actually 2.322 FFT levels)
;; On input the 64-byte cache lines hold these data values:
;;	0	+7680	128	+7680	256	+7680	384	+7680
;;	1	...
;;	...
;;	127	...
;;	512	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+7680	640	+7680	1280	+7680	1920	+7680
;;	1	...
;;	...
;;	639	...
;;	2560	...
;;	...

;; Do 256 ten_reals_unfft operations
;; Do 256 ten_complex_djbunfft operations
;;	distance between fft data elements is 256
;;	do 128 macros processing 40 data values

	loops_init_prefetch 384, 64, 2, rcx
	xtouch	[rcx+4096-128]		;; Preload the TLBs
	xtouch	[rcx+2*4096-128]
	xtouch	[rcx+3*4096-128]
	xtouch	[rcx+4*4096-128]
	mov	rdi, xsincos_complex	;; Load sin/cos pointer
	mov	rbx, rdi
c2b:	r5_h5cl_2sc_ten_reals_five_complex_djbunfft rsi, 64, dist128, rdi, 128*3*XMM_SCD1, rbx, 128*6*XMM_SCD1, rdx, 128*XMM_HSCD2
	bump	rdi, 3*XMM_SCD1		;; Next sin/cos data
	bump	rbx, 6*XMM_SCD1		;; Next sin/cos data
	bump	rdx, XMM_HSCD2		;; Next sin/cos data
	loops	128, c2b		;; Test inner loop counter
	bump	rsi, -128*64+5*dist128	;; Next source pointer
	bump	rdi, -128*3*XMM_SCD1	;; Restore sin/cos ptr
	bump	rbx, -128*6*XMM_SCD1	;; Restore sin/cos ptr
	bump	rdx, -128*XMM_HSCD2-1280*XMM_HSCD1 ;; Next sin/cos ptr

;; Do 1024 five_complex_djbunfft operations
;;	distance between fft data elements is 256
;;	do 256 macros each processing 32 data values

	loops_reset			;; 2 more iters of 128
	r5_x5cl_2sc_five_complex_djbunfft_preload
c3b:	r5_x5cl_2sc_five_complex_djbunfft rsi, 64, dist128, rdi, 128*3*XMM_SCD1, rbx, 128*6*XMM_SCD1
	bump	rdi, 3*XMM_SCD1		;; Next sin/cos data
	bump	rbx, 6*XMM_SCD1		;; Next sin/cos data
	loops	128, c3b		;; Test inner loop counter
	bump	rsi, -128*64+5*dist128	;; Next source pointer
	bump	rdi, -128*3*XMM_SCD1	;; Restore sin/cos ptr
	bump	rbx, -128*6*XMM_SCD1	;; Restore sin/cos ptr
	loops	3, c3b			;; Test loop counter
	bump	rsi, -3*5*dist128	;; Next source pointer

;; Do inverse FFT level 1,2 (actually 1.585 FFT levels)
;; On input the 64-byte cache lines hold these data values:
;;	0	+7680	640	+7680	1280	+7680	1920	+7680
;;	1	...
;;	...
;;	639	...
;;	2560	...
;;	...
;; On output the 64-byte cache lines hold these data values if pass 1 swizzles:
;;	0	+7680	1920	+7680	3840	+7680	5760	+7680
;;	1	...
;;	...
;;	1919
;; On output the 64-byte cache lines hold these data values if pass 2 swizzles:
;;	0	+1920	3840	+1920	7680	+1920	11520	+1920
;;	1	...
;;	...
;;	1919

;; Do 1280 six_reals_unfft operations
;; Do 1280 three_complex_djbunfft operations
;;	distance between fft data elements is 1280
;;	do 640 macros processing 24 data values

	bump	rcx, -4*64		;; Back up prefetch pointer by 4 cache lines
	loops_init_prefetch 640, 64, 2, rcx
c1a:	xtouch	[rcx+4096-64]		;; Preload the TLBs
c1b:	r3_h3cl_six_reals_three_complex_djbunfft rsi, 64, 5*dist128, rdi, 640*XMM_SCD1, rdx, 640*XMM_HSCD1
	bump	rdi, XMM_SCD1		;; Next sin/cos data
	bump	rdx, XMM_HSCD1		;; Next sin/cos data
	loops	128, c1b		;; Test loop counter
	bump	rsi, -128*64+dist128	;; Next source pointer
	loops	5, c1a			;; Test loop counter

xpass2_3840_real_done:
	end_timer 2
	ENDM

; *************** radix4-pass2-3840-levels-complex macro ******************
; This macro takes 2 groups of 3840 complex values and
; performs the final 11.907 levels of a standard mostly radix-4 FFT, squares the
; results, and does the first 11.907 levels of the inverse FFT.

r4_pass2_3840_levels_complex MACRO
	LOCAL	b1a, b1b, b2b, b3a, b3b, b4b, b5b, b6a, b6b0, b6b, b6c, b6d0, b6d
	LOCAL	c1a, c1b, c2a0, c2a, c2b, c3b, c4b, c5a, c5b
	LOCAL	xpass2_3840_levels_complex_1, xpass2_3840_levels_complex_2
	LOCAL	xpass2_3840_levels_complex_3, xpass2_3840_levels_complex_4
	LOCAL	xpass2_3840_levels_complex_unfft, xpass2_3840_levels_complex_done

;; Type 4 FFTs skip the forward FFT process

	mov	rsi, DATA_ADDR		;; Load address of FFT data
	mov	rbx, DIST_TO_FFTSRCARG
	cmp	ffttype, 4
	je	xpass2_3840_levels_complex_4

;; Do FFT levels 1,2 (actually 1.585 levels)
;;
;; We swizzle the data so that one set of data is in the high word of
;; an xmm register, the other set is in the low word.  This means
;; a distance of 4608 data values.
;;
;; On input the 64-byte cache lines hold these data values if pass 2 swizzles:
;;	0	+1920	3840	+1920	7680	+1920	11520	+1920
;;	1	...
;;	...
;;	1919
;; On input the 64-byte cache lines hold these data values if pass 1 swizzles:
;;	0	+7680	1920	+7680	3840	+7680	5760	+7680
;;	1	...
;;	...
;;	1919	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+7680	640	+7680	1280	+7680	1920	+7680
;;	1	...
;;	...
;;	639	...
;;	2560	...
;;	...

;; Do 2560 three_complex_fft operations
;;	distance between fft data elements is 1280
;;	do 640 macros each processing 24 data values

;; Do FFT levels 3,4 (actually 2.322 levels)
;;
;; On input the 64-byte cache lines hold these data values:
;;	0	+7680	640	+7680	1280	+7680	1920	+7680
;;	1	...
;;	...
;;	639	...
;;	2560	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+7680	128	+7680	256	+7680	384	+7680
;;	1	...
;;	...
;;	127	...
;;	512	...
;;	...

;; Do 1536 five_complex_fft operations
;;	distance between fft data elements is 256
;;	do 384 macros each processing 40 data values

IF @INSTR(,%xarch,<CORE>) NE 0		;; Core 2 faster doing separate loops

	start_timer 4
	mov	rdi, xsincos_complex	;; The sin/cos data
	loops_init 640			;; 5 iterations of 128
	r3_f3cl_three_complex_djbfft_preload
b1b:	r3_f3cl_three_complex_djbfft rsi, 64, 5*dist128, rdi, 640*XMM_SCD1
	bump	rdi, XMM_SCD1		;; Next sin/cos data
	loops	128, b1b		;; Test inner loop counter
	bump	rsi, -128*64+dist128	;; Next source pointer
	loops	5, b1b			;; Test outer loop counter
	bump	rsi, -5*dist128		;; Restore source pointer
	end_timer 4

	start_timer 5
	mov	rdi, xsincos_complex	;; Ptr to the sin/cos data
	mov	rdx, rdi
	mov	rcx, DATA_PREFETCH	;; Prefetch pointer (we need to prefetch
					;; 1920 cache lines + 28 pad cache lines)
	xtouch	[rcx]			;; Preload first TLB
	loops_init 384
	r5_nf5cl_2sc_five_complex_djbfft_preload
b2b:	r5_nf5cl_2sc_five_complex_djbfft rsi, 64, dist128, rdi, 128*3*XMM_SCD1, rdx, 128*6*XMM_SCD1
	bump	rdi, 3*XMM_SCD1		;; Next sin/cos data
	bump	rdx, 6*XMM_SCD1		;; Next sin/cos data
	loops	128, b2b		;; Test inner loop counter
	bump	rsi, -128*64+5*dist128	;; Next source pointer
	bump	rdi, -128*3*XMM_SCD1	;; Restore sin/cos ptr
	bump	rdx, -128*6*XMM_SCD1	;; Restore sin/cos ptr
	loops	3, b2b			;; Test loop counter
	bump	rsi, -3*5*dist128	;; Restore source pointer
	end_timer 5

ELSE					;; Pentium 4 faster doing merged loops

	start_timer 4
	mov	rdi, xsincos_complex	;; The sin/cos data
	mov	rbp, rdi
	mov	rdx, rdi
	mov	rcx, DATA_PREFETCH	;; Prefetch pointer (we need to prefetch
					;; 1920 cache lines + 28 pad cache lines)
	xtouch	[rcx]			;; Preload first TLB
	loops_init 1920			;; 128 iters of 3 or 5

b1a:	r3_f3cl_three_complex_djbfft_preload
b1b:	r3_f3cl_three_complex_djbfft rsi, dist128, 5*dist128, rdi, 640*XMM_SCD1
	xxprefetch [rdi+XMM_SCD1]	;; Prefetch for next pass
	xxprefetch [rdi+641*XMM_SCD1]	;; Prefetch for next pass
	bump	rdi, 128*XMM_SCD1	;; Next sin/cos data
	loops	5, b1b, 3		;; Test loop counter with non-standard incr
	bump	rsi, -5*dist128		;; Restore source pointer
	loops_undo 15

	r5_nf5cl_2sc_five_complex_djbfft_preload
b2b:	r5_nf5cl_2sc_five_complex_djbfft rsi, 5*dist128, dist128, rbp, 128*3*XMM_SCD1, rdx, 128*6*XMM_SCD1
	loops	3, b2b, 5		;; Test loop counter with non-standard incr

	bump	rsi, -3*5*dist128+64	;; Next source pointer
	bump	rdi, -5*128*XMM_SCD1+XMM_SCD1 ;; Next sin/cos data
	bump	rbp, 3*XMM_SCD1		;; Next sin/cos data
	bump	rdx, 6*XMM_SCD1		;; Next sin/cos data
	loops	128, b1a		;; Test loop counter
	bump	rsi, -128*64		;; Restore source pointer
	bump	rdi, -128*XMM_SCD1	;; Restore sin/cos ptr
	end_timer 4

ENDIF

;; Do FFT levels 5,6
;;
;; On input the 64-byte cache lines hold these data values:
;;	0	+7680	128	+7680	256	+7680	384	+7680
;;	1	...
;;	...
;;	127	...
;;	512	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+7680	64	+7680	128	+7680	192	+7680
;;	1	...
;;	...
;;	63	...
;;	256	...
;;	...
;;
;; Do 1920 four_complex_fft operations
;;	distance between fft data elements is 64
;;	do 960 macros each processing 16 data values

;; Do FFT levels 7,8
;;
;; On input the 64-byte cache lines hold these data values:
;;	0	+7680	64	+7680	128	+7680	192	+7680
;;	1	...
;;	...
;;	63	...
;;	256	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+7680	16	+7680	32	+7680	48	+7680
;;	1	...
;;	...
;;	15	...
;;	64	...
;;	...
;;
;; Do 1920 four_complex_fft operations
;;	distance between fft data elements is 16
;;	do 480 macros each processing 32 data values

;; Do FFT levels 9,10
;;
;; On input the 64-byte cache lines hold these data values:
;;	0	+7680	16	+7680	32	+7680	48	+7680
;;	1	...
;;	...
;;	15	...
;;	64	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+7680	4	+7680	8	+7680	12	+7680
;;	1	...
;;	...
;;	3	...
;;	16	...
;;	...
;;
;; Do 1920 four_complex_fft operations
;;	distance between fft data elements is 4
;;	do 480 macros each processing 32 data values

;; Do this 32 cache lines at a time to maximize accessing data
;; while it is in the L1 cache.

	start_timer 6
	bump	rdi, 1536*XMM_SCD1	;; Load sin/cos pointers
	mov	rbx, rdi
	mov	rbp, rdi
	loops_init_prefetch 960, 64, 4, rcx ;; 2*4 iters of 4 of 64 cache lines
	;r4_nf2cl_four_complex_djbfft_preload -- assume same as r4_x4cl_four_complex_djbfft_preload
	r4_x4cl_four_complex_djbfft_preload

b3a:	xtouch	[rcx+4096-64]		;; Load 12 prefetch TLBs

b3b:	xxprefetch [rdi+4*XMM_SCD2]	;; Prefetch for next pass
	r4_nf2cl_four_complex_djbfft rsi, 4*64, 64*64, rdi
	bump	rdi, 4*XMM_SCD2		;; Next sine/cosine pointer
	loops	16, b3b			;; Loop 16 times
	bump	rsi, -16*4*64		;; Restore source pointer
	bump	rdi, -16*4*XMM_SCD2	;; Restore sine/cosine pointer
	loops_undo 16			;; Back up loop counter

b4b:	r4_x4cl_four_complex_djbfft rsi, 4*64, 16*64, 32*64, rbx
	bump	rbx, 4*4*XMM_SCD2	;; Next sine/cosine pointer
	loops	4, b4b, 2		;; 4 iterations with non-standard increment
	bump	rsi, -4*4*64+64*64	;; Next source pointer
	bump	rbx, -4*4*4*XMM_SCD2	;; Restore sine/cosine pointer
	loops	2, b4b			;; Loop twice
	bump	rsi, -2*64*64		;; Restore source pointer
	loops_undo 2*4*2		;; Back up loop counter

b5b:	r4_x4cl_four_complex_djbfft rsi, 16*64, 4*64, 8*64, rbp
	loops	8, b5b, 2 		;; 8 iterations with non-standard increment

	bump	rsi, -8*16*64+64	;; Next source pointer
	bump	rdi, XMM_SCD2		;; Next sine/cosine pointer
	bump	rbx, 4*XMM_SCD2		;; Next sine/cosine pointer
	bump	rbp, 16*XMM_SCD2	;; Next sine/cosine pointer
	loops	4, b3b			;; Loop 4 times
	bump	rsi, -4*64+dist128	;; Next source pointer
	bump	rdi, -4*XMM_SCD2	;; Restore sine/cosine pointer
	bump	rbx, -4*4*XMM_SCD2	;; Restore sine/cosine pointer
	bump	rbp, -4*16*XMM_SCD2	;; Restore sine/cosine pointer
	loops	15, b3a			;; Loop 15 times
	bump	rsi, -15*dist128	;; Restore source pointer
	end_timer 6

;; Do FFT levels 11,12 as well as inverse FFT levels 11,12
;;
;; On input the 64-byte cache lines hold these data values:
;;	0	+7680	4	+7680	8	+7680	12	+7680
;;	1	...
;;	...
;;	3	...
;;	16	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+7680	2	+7680	4	+7680	6	+7680
;;	1	...
;;	8	...
;;	...

;; Do 1920 four_complex_fft operations
;;	distance between fft data elements is 1
;;	do 480 macros each processing 32 data values

;; Execute the right middle step

	cmp	ffttype, 2
	jl	xpass2_3840_levels_complex_1
	jg	xpass2_3840_levels_complex_3

xpass2_3840_levels_complex_2:
	start_timer 9
	loops_init 480
	r4_x4cl_four_complex_with_square_preload
b6a:	r4_x4cl_four_complex_with_square rsi, 4*64, 64, 2*64
	loops	32, b6a			;; Loop 32 times
	bump	rsi, -16*2*4*64+dist128	;; Next source pointer
	loops	15, b6a			;; Loop 15 times
	bump	rsi, -15*dist128	;; Restore source pointer
	end_timer 9
	jmp	xpass2_3840_levels_complex_unfft

xpass2_3840_levels_complex_1:
	start_timer 9
	loops_init_prefetch 480, 128, 1, rcx
b6b0:	xtouch	[rcx+4096-64]		;; Load prefetch TLBs
b6b:	r4_x4cl_four_complex_fft_final rsi, 4*64, 64, 2*64
	loops	32, b6b			;; Loop 32 times
	bump	rsi, -16*2*4*64+dist128	;; Next source pointer
	loops	15, b6b0		;; Loop 15 times
	end_timer 9
	jmp	xpass2_3840_levels_complex_done

xpass2_3840_levels_complex_3:
	start_timer 9
	mov	rbp, DIST_TO_MULSRCARG
	loops_init 480
b6c:	r4_x4cl_four_complex_with_mult rsi, 4*64, 64, 2*64
	loops	32, b6c			;; Loop 32 times
	bump	rsi, -16*2*4*64+dist128	;; Next source pointer
	loops	15, b6c			;; Loop 15 times
	bump	rsi, -15*dist128	;; Restore source pointer
	end_timer 9
	jmp	xpass2_3840_levels_complex_unfft

xpass2_3840_levels_complex_4:
	start_timer 9
	mov	rcx, DATA_PREFETCH	;; Prefetch pointer
	xtouch	[rcx]			;; Load first prefetch TLB
	mov	rbp, DIST_TO_MULSRCARG
	loops_init_prefetch 480, 64, 1, rcx
b6d0:	xtouch	[rcx+4096-64]		;; Load prefetch TLBs
b6d:	r4_x4cl_four_complex_with_mulf rsi, 4*64, 64, 2*64
	loops	32, b6d			;; Loop 32 times
	bump	rsi, -16*2*4*64+dist128	;; Next source pointer
	loops	15, b6d0		;; Loop 15 times
	bump	rsi, -15*dist128	;; Restore source pointer
	end_timer 9

;; Do inverse FFT levels 9,10
;; On input the 64-byte cache lines hold these data values:
;;	0	+7680	2	+7680	4	+7680	6	+7680
;;	1	...
;;	8	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+7680	8	+7680	16	+7680	24	+7680
;;	1	...
;;	...
;;	7	...
;;	32	...
;;	...
;;
;; Do 1920 four_complex_unfft operations
;;	distance between fft data elements is 4
;;	do 480 macros each processing 32 data values

;; Do inverse FFT levels 7,8
;; On input the 64-byte cache lines hold these data values:
;;	0	+7680	8	+7680	16	+7680	24	+7680
;;	1	...
;;	...
;;	7	...
;;	32	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+7680	32	+7680	64	+7680	96	+7680
;;	1	...
;;	...
;;	31	...
;;	128	...
;;	...
;;
;; Do 1920 four_complex_unfft operations
;;	distance between fft data elements is 16
;;	do 480 macros each processing 32 data values

;; Do inverse FFT levels 5,6
;; On input the 64-byte cache lines hold these data values:
;;	0	+7680	32	+7680	64	+7680	96	+7680
;;	1	...
;;	...
;;	31	...
;;	128	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+7680	128	+7680	256	+7680	384	+7680
;;	1	...
;;	...
;;	127	...
;;	512	...
;;	...
;;
;; Do 1920 four_complex_unfft operations
;;	distance between fft data elements is 64
;;	do 480 macros each processing 32 data values

;; Do this 64 cache lines at a time to maximize accessing data
;; while it is in the L1 cache.

xpass2_3840_levels_complex_unfft:
	start_timer 10
	mov	rdi, xsincos_complex	;; Load sin/cos pointers
	bump	rdi, 1536*XMM_SCD1
	mov	rbx, rdi
	mov	rbp, rdi
	loops_init_prefetch 480, 64, 2, rcx ;; 2*2*8 iters of 64 cache lines
	r4_x4cl_four_complex_djbunfft_preload

c5a:	xtouch	[rcx+4096-64]		;; Load prefetch TLBs

c5b:	r4_x4cl_four_complex_djbunfft rsi, 8*64, 2*64, 4*64, rbp, 2*16*XMM_SCD2
	loops	16, c5b			;; Loop 16 times
	bump	rsi, -16*8*64		;; Restore source pointer
	loops_undo 16			;; Restore loop counter

c4b:	r4_x4cl_four_complex_djbunfft rsi, 2*64, 8*64, 16*64, rbx, 8*4*XMM_SCD2
	bump	rbx, 2*4*XMM_SCD2	;; Next sine/cosine pointer
	loops	4, c4b			;; Loop 4 times
	bump	rsi, -4*2*64+32*64	;; Next source pointer
	bump	rbx, -4*2*4*XMM_SCD2	;; Restore sine/cosine pointer
	loops	4, c4b			;; Loop 4 times
	bump	rsi, -4*32*64		;; Restore source pointer
	loops_undo 4*4			;; Restore loop counter

c3b:	r4_x4cl_four_complex_djbunfft rsi, 2*64, 32*64, 64*64, rdi, 32*1*XMM_SCD2
	bump	rdi, 2*XMM_SCD2		;; Next sine/cosine pointer
	loops	16, c3b			;; Loop 16 times
	bump	rdi, -16*2*XMM_SCD2	;; Restore sine/cosine pointer

	bump	rsi, -16*2*64+64	;; Next source pointer
	bump	rdi, XMM_SCD2		;; Next sine/cosine pointer
	bump	rbx, 4*XMM_SCD2		;; Next sine/cosine pointer
	bump	rbp, 16*XMM_SCD2	;; Next sine/cosine pointer
	loops	2, c5b			;; Loop twice
	bump	rsi, -2*64+dist128	;; Next source pointer
	bump	rdi, -2*XMM_SCD2	;; Restore sine/cosine pointer
	bump	rbx, -2*4*XMM_SCD2	;; Restore sine/cosine pointer
	bump	rbp, -2*16*XMM_SCD2	;; Restore sine/cosine pointer
	loops	15, c5a			;; Loop 15 times
	bump	rsi, -15*dist128	;; Restore source pointer
	end_timer 10

;; Do inverse FFT levels 3,4 (actually 2.322 FFT levels)
;; On input the 64-byte cache lines hold these data values:
;;	0	+7680	128	+7680	256	+7680	384	+7680
;;	1	...
;;	...
;;	127	...
;;	512	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+7680	640	+7680	1280	+7680	1920	+7680
;;	1	...
;;	...
;;	639	...
;;	2560	...
;;	...

;; Do 1536 five_complex_unfft operations
;;	distance between fft data elements is 256
;;	do 384 macros each processing 40 data values

;; Do inverse FFT levels 1,2 (actually 1.585 FFT levels)
;; On input the 64-byte cache lines hold these data values:
;;	0	+7680	640	+7680	1280	+7680	1920	+7680
;;	1	...
;;	...
;;	639	...
;;	2560	...
;;	...
;; On output the 64-byte cache lines hold these data values if pass 1 swizzles:
;;	0	+7680	1920	+7680	3840	+7680	5760	+7680
;;	1	...
;;	...
;;	1919
;; On output the 64-byte cache lines hold these data values if pass 2 swizzles:
;;	0	+1920	3840	+1920	7680	+1920	11520	+1920
;;	1	...
;;	...
;;	1919

;; Do 2560 three_complex_unfft operations
;;	distance between fft data elements is 1280
;;	do 640 macros each processing 24 data values

IF @INSTR(,%xarch,<CORE>) NE 0		;; Core 2 faster doing separate loops

	start_timer 13
	mov	rdi, xsincos_complex	;; Load sin/cos pointer
	mov	rdx, rdi
	loops_init_prefetch 384, 64, 2, rcx;; 3 iters of 128
	xtouch	[rcx+4096-64]		;; Load prefetch TLBs
	xtouch	[rcx+2*4096-64]
	xtouch	[rcx+3*4096-64]
	r5_x5cl_2sc_five_complex_djbunfft_preload
c2b:	r5_x5cl_2sc_five_complex_djbunfft rsi, 64, dist128, rdi, 128*3*XMM_SCD1, rdx, 128*6*XMM_SCD1
	bump	rdi, 3*XMM_SCD1		;; Next sine/cosine pointer
	bump	rdx, 6*XMM_SCD1		;; Next sine/cosine pointer
	loops	128, c2b		;; Loop 128 times
	bump	rsi, -128*64+5*dist128	;; Next source pointer
	bump	rdi, -128*3*XMM_SCD1	;; Reset sine/cosine pointer
	bump	rdx, -128*6*XMM_SCD1	;; Reset sine/cosine pointer
	loops	3, c2b			;; Test loop counter
	bump	rsi, -3*5*dist128	;; Restore source pointer
	end_timer 13

	start_timer 14
	mov	rdi, xsincos_complex	;; Load sin/cos pointer
	bump	rcx, -4*64		;; Back up prefetch pointer by 4 cache lines
	loops_init_prefetch 640, 64, 2, rcx;; 5 iters of 128
	r3_x3cl_three_complex_djbunfft_preload
c1a:	xtouch	[rcx+4096-64]		;; Load prefetch TLBs
c1b:	r3_x3cl_three_complex_djbunfft rsi, 64, 5*dist128, rdi, 640*XMM_SCD1
	bump	rdi, XMM_SCD1		;; Next set of sin/cos data
	loops	128, c1b		;; Test inner loop counter
	bump	rsi, -128*64+dist128	;; Next source pointer
	loops	5, c1a			;; Test outer loop counter
	end_timer 14

ELSE					;; Pentium 4 faster doing merged loops

	start_timer 13
	mov	rdi, xsincos_complex	;; Load sin/cos pointer
	mov	rbp, rdi
	mov	rdx, rdi
	bump	rcx, -4*64		;; Back up prefetch pointer by 4 cache lines
	loops_init_prefetch 1920, 128, 15, rcx ;; 128 iters of 3 or 5, prefetching 256 cache lines each loop

c2a0:	xtouch	[rcx+4096-64]		;; Load prefetch TLB

c2a:	r5_x5cl_2sc_five_complex_djbunfft_preload
c2b:	r5_x5cl_2sc_five_complex_djbunfft rsi, 5*dist128, dist128, rbp, 128*3*XMM_SCD1, rdx, 128*6*XMM_SCD1
	loops	3, c2b, 5		;; Test loop counter with non-standard incr
	bump	rsi, -3*5*dist128	;; Restore source pointer
	loops_undo 15

	r3_x3cl_three_complex_djbunfft_preload
c1b:	r3_x3cl_three_complex_djbunfft rsi, dist128, 5*dist128, rdi, 640*XMM_SCD1
	bump	rdi, 128*XMM_SCD1	;; Next sin/cos ptr
	loops	5, c1b, 3		;; Test loop counter with non-standard incr

	bump	rsi, -5*dist128+64	;; Next source pointer
	bump	rdi, -5*128*XMM_SCD1+XMM_SCD1 ;; Next sine/cosine pointer
	bump	rbp, 3*XMM_SCD1		;; Next sine/cosine pointer
	bump	rdx, 6*XMM_SCD1		;; Next sine/cosine pointer
	loops_touch 16, c2a, 8, c2a0	;; Loop 128 times
	end_timer 13

ENDIF
xpass2_3840_levels_complex_done:
	ENDM


; *************** radix4-pass2-12-levels-real macro ******************
; This macro takes 8192 real values and performs the final 13 levels of
; the FFT.  Independently, this macro also takes 4096 complex values and
; performs the final 12 levels of the FFT.  The results are squared, and
; the first 12/13 levels of the inverse FFT are done.

r4_pass2_12_levels_real MACRO
	LOCAL	b1b, b2b0, b2b, b3b0, b3b, b4b, b5a, b5b, b6b, b7b
	LOCAL	b8b, b9b, baa, bab0, bab, bac, bad0, bad
	LOCAL	c1b, c2b, c3b, c4b, c5a, c5b, c6b, c7a, c7b, c8b, c9b
	LOCAL	xpass2_12_levels_real_1, xpass2_12_levels_real_2
	LOCAL	xpass2_12_levels_real_3, xpass2_12_levels_real_4
	LOCAL	xpass2_12_real_unfft, xpass2_12_real_done

;; We switch to the same format used in one-pass FFTs (that is, the
;; high word of xmm registers being 1 greater than the low word):

;; Type 4 FFTs skip the forward FFT process

	start_timer 2
	mov	rsi, DATA_ADDR		;; Load source address
	mov	rbx, DIST_TO_FFTSRCARG
	cmp	ffttype, 4
	je	xpass2_12_levels_real_4

;; Do FFT level 1,2
;;
;; On input the 64-byte cache lines hold these data values if pass 2 swizzles:
;;	0	+2K	4096	+2K	8192	+2K	12288	+2K
;;	1	...
;;	...
;;	2047	...
;; On input the 64-byte cache lines hold these data values if pass 1 swizzles:
;;	0	+8K	2048	+8K	4096	+8K	6144	+8K
;;	1	...
;;	...
;;	2047	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+8K	1024	+8K	2048	+8K	3072	+8K
;;	1	...
;;	...
;;	1023	...
;;	4096	...
;;	...

;; Do 1024 eight_reals_fft on values 0 - 8191
;; Do 1024 four_complex_djbfft on values 8192 - 16383
;;	distance between fft data elements is 1K
;;	do 1024 iterations

	mov	rdi, xsincos_complex	;; Ptr to the sin/cos data
	mov	rdx, sincos3		;; Ptr to real sin/cos data
	loops_init 1024
b1b:	r4_fh2cl_eight_reals_four_complex_djbfft rsi, 64, 8*dist128, rdi, rdx
	bump	rdi, XMM_SCD2		;; Next sin/cos data
	bump	rdx, XMM_HSCD2		;; Next sin/cos data
	loops	128, b1b		;; Test loop counter
	bump	rsi, -128*64+dist128	;; Next source pointer
	loops	8, b1b			;; Test loop counter
	bump	rsi, -8*dist128	;; Restore source pointer
	bump	rdi, -1024*XMM_SCD2	;; Restore sin/cos ptr
	bump	rdx, -1024*XMM_HSCD2	;; Restore sin/cos ptr

;; Do FFT levels 3,4
;; Values 0-2047 is real data, 2048-16383 is complex data.
;;
;; On input the 64-byte cache lines hold these data values:
;;	0	+8K	1024	+8K	2048	+8K	3072	+8K
;;	1	...
;;	...
;;	1023	...
;;	4096	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+8K	256	+8K	512	+8K	768	+8K
;;	1	...
;;	...
;;	255	...
;;	1024	...
;;	...

;; Do 256 eight_reals_fft operations
;; Do 768 four_complex_djbfft operations
;;	distance between fft data elements is 256
;;	do 256 macros each processing 32 data values

	mov	rcx, DATA_PREFETCH	;; Prefetch pointer
	xtouch	[rcx]			;; Preload first TLB
	loops_init_prefetch 256, 64, 1, rcx
	r4_h4cl_eight_reals_four_complex_djbfft_preload
b2b0:	xtouch	[rcx+4096-128]		;; Preload the TLBs
	xtouch	[rcx+2*4096-128]
b2b:	r4_h4cl_eight_reals_four_complex_djbfft rsi, 64, 2*dist128, 4*dist128, rdi, rdx
	bump	rdi, 4*XMM_SCD2		;; Next sin/cos data
	bump	rdx, 4*XMM_HSCD2	;; Next sin/cos data
	loops	128, b2b		;; Test inner loop counter
	bump	rsi, -128*64+dist128	;; Next source pointer
	bump	rcx, 128		;; Skip pad bytes
	loops	2, b2b0			;; Test loop counter
	bump	rsi, -2*dist128+8*dist128 ;; Next source pointer
	bump	rdi, -256*4*XMM_SCD2	;; Restore sin/cos ptr
	bump	rdx, -256*4*XMM_HSCD2	;; Restore sin/cos ptr

;; Do 1024 four_complex_fft operations
;;	distance between fft data elements is 256
;;	do 256 macros each processing 32 data values

	loops_init_prefetch 256, 64, 1, rcx
	r4_x4cl_four_complex_djbfft_preload
b3b0:	xtouch	[rcx+4096-128]		;; Preload the TLBs
	xtouch	[rcx+2*4096-128]
b3b:	r4_x4cl_four_complex_djbfft rsi, 64, 2*dist128, 4*dist128, rdi
	bump	rdi, 4*XMM_SCD2		;; Next sin/cos data
	loops	128, b3b		;; Test inner loop counter
	bump	rsi, -128*64+dist128	;; Next source pointer
	bump	rcx, 128		;; Skip pad bytes
	loops	2, b3b0			;; Test loop counter
	bump	rsi, -2*dist128-8*dist128 ;; Restore source pointer
	;; don't restore rdi, we'll fix it in next section

;; Do FFT levels 5,6
;; Values 0-511 is real data, 512-16384 is complex data.
;;
;; On input the 64-byte cache lines hold these data values:
;;	0	+8K	256	+8K	512	+8K	768	+8K
;;	1	...
;;	...
;;	255	...
;;	1024	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+8K	64	+8K	128	+8K	192	+8K
;;	1	...
;;	...
;;	63	...
;;	256	...
;;	...

;; Do 64 eight_reals_fft operations
;; Do 192 four_complex_djbfft operations
;;	distance between fft data elements is 64
;;	do 64 macros each processing 32 data values

	bump	rdi, -256*4*XMM_SCD2+1024*XMM_SCD2 ;; Load sin/cos pointer
	loops_init_prefetch 512, 64, 1, rcx
	xtouch	[rcx+4096-64]		;; Preload the TLBs
	r4_h4cl_eight_reals_four_complex_djbfft_preload
b4b:	r4_h4cl_eight_reals_four_complex_djbfft rsi, 64, 64*64, dist128, rdi, rdx
	bump	rdi, XMM_SCD2		;; Next sin/cos data
	bump	rdx, 16*XMM_HSCD2	;; Next sin/cos data
	loops	64, b4b			;; Test loop counter
	bump	rsi, -64*64+2*dist128	;; Next source pointer
	bump	rdi, -64*XMM_SCD2	;; Restore sin/cos ptr
	bump	rdx, -64*16*XMM_HSCD2	;; Restore sin/cos ptr

;; Do 1792 four_complex_djbfft operations
;;	distance between fft data elements is 64
;;	do 448 macros each processing 32 data values

	loops_reset			;; 7 iterations of 64
	r4_x4cl_four_complex_djbfft_preload
b5a:	xtouch	[rcx+4096-64]		;; Preload the TLBs
b5b:	r4_x4cl_four_complex_djbfft rsi, 64, 64*64, dist128, rdi
	bump	rdi, XMM_SCD2		;; Next sin/cos data
	loops	64, b5b			;; Test inner loop counter
	bump	rsi, -64*64+2*dist128	;; Next source pointer
	bump	rdi, -64*XMM_SCD2	;; Restore sin/cos ptr
	loops	2, b5a			;; Test loop counter
	bump	rcx, 128		;; Skip pad bytes
	loops	4, b5a			;; Test loop counter
	bump	rsi, -8*2*dist128	;; Next source pointer

;; Do FFT levels 7,8
;; Values 0-127 is real data, 128-16384 is complex data.
;;
;; On input the 64-byte cache lines hold these data values:
;;	0	+8K	64	+8K	128	+8K	192	+8K
;;	1	...
;;	...
;;	63	...
;;	256	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+8K	16	+8K	32	+8K	48	+8K
;;	1	...
;;	...
;;	15	...
;;	64	...
;;	...

;; Do 16 eight_reals_fft operations
;; Do 48 four_complex_djbfft operations
;;	distance between fft data elements is 16
;;	do 16 macros each processing 32 data values

	loops_init 512
	r4_h4cl_eight_reals_four_complex_djbfft_preload
b6b:	r4_h4cl_eight_reals_four_complex_djbfft rsi, 64, 16*64, 32*64, rdi, rdx
	bump	rdi, 4*XMM_SCD2		;; Next sin/cos data
	bump	rdx, 64*XMM_HSCD2	;; Next sin/cos data
	loops	16, b6b			;; Test loop counter
	bump	rsi, -16*64+64*64	;; Next source pointer
	bump	rdi, -16*4*XMM_SCD2	;; Restore sin/cos ptr
	bump	rdx, -16*64*XMM_HSCD2	;; Restore sin/cos ptr

;; Do 1984 four_complex_djbfft operations
;;	distance between fft data elements is 16
;;	do 496 macros each processing 32 data values

	loops_reset			;; 16 iters of 1 or 2 of 16
	r4_x4cl_four_complex_djbfft_preload
b7b:	r4_x4cl_four_complex_djbfft rsi, 64, 16*64, 32*64, rdi
	bump	rdi, 4*XMM_SCD2		;; Next sin/cos data
	loops	16, b7b			;; Test inner loop counter
	bump	rsi, -16*64+64*64	;; Next source pointer
	bump	rdi, -16*4*XMM_SCD2	;; Restore sin/cos ptr
	loops	2, b7b			;; Test loop counter
	bump	rsi, -2*64*64+dist128	;; Next source pointer
	loops	16, b7b			;; 16 iterations
	bump	rsi, -16*dist128	;; Next source pointer

;; Do FFT levels 9,10
;; Values 0-31 is real data, 32-16383 is complex data.
;;
;; On input the 64-byte cache lines hold these data values:
;;	0	+8K	16	+8K	32	+8K	48	+8K
;;	1	...
;;	...
;;	15	...
;;	64	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+8K	4	+8K	8	+8K	12	+8K
;;	1	...
;;	...
;;	3	...
;;	16	...
;;	...

;; Do 4 eight_reals_fft operations
;; Do 12 four_complex_djbfft operations
;;	distance between fft data elements is 4
;;	do 4 macros each processing 32 data values

	loops_init 512
	r4_h4cl_eight_reals_four_complex_djbfft_preload
b8b:	r4_h4cl_eight_reals_four_complex_djbfft rsi, 64, 4*64, 8*64, rdi, rdx
	bump	rdi, 16*XMM_SCD2	;; Next sin/cos data
	bump	rdx, 256*XMM_HSCD2	;; Next sin/cos data
	loops	4, b8b			;; Test loop counter
	bump	rsi, -4*64+16*64	;; Next source pointer
	bump	rdi, -4*16*XMM_SCD2	;; Restore sin/cos ptr
	bump	rdx, -4*256*XMM_HSCD2	;; Restore sin/cos ptr

;; Do 2032 four_complex_djbfft operations
;;	distance between fft data elements is 4
;;	do 510 macros each processing 32 data values

	loops_reset			;; 16 iters of 7 or 8 of 4
	r4_x4cl_four_complex_djbfft_preload
b9b:	r4_x4cl_four_complex_djbfft rsi, 64, 4*64, 8*64, rdi
	bump	rdi, 16*XMM_SCD2	;; Next sin/cos data
	loops	4, b9b			;; Test loop counter
	bump	rsi, -4*64+16*64	;; Next source pointer
	bump	rdi, -4*16*XMM_SCD2	;; Restore sin/cos ptr
	loops	8, b9b			;; Test middle loop counter
	bump	rsi, -8*16*64+dist128	;; Next source pointer
	loops	16, b9b			;; 16 iterations
	bump	rsi, -16*dist128	;; Next source pointer

;; Do FFT levels 11,12
;; Values 0-7 is real data, 8-16383 is complex data.
;;
;; On input the 64-byte cache lines hold these data values:
;;	0	+8K	4	+8K	8	+8K	12	+8K
;;	1	...
;;	...
;;	3	...
;;	16	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+8K	2	+8K	4	+8K	6	+8K
;;	1	...
;;	8	...
;;	...

;; Do 1 eight_reals_fft operations
;; Do 3 four_complex_fft operations
;;	distance between fft data elements is 1

;; Do 2044 four_complex_fft operations
;;	distance between fft data elements is 1
;;	do 511 macros each processing 32 data values

;; Execute the proper middle step

	cmp	ffttype, 2
	jl	xpass2_12_levels_real_1
	jg	xpass2_12_levels_real_3

xpass2_12_levels_real_2:
	r4_h4cl_eight_reals_four_complex_with_square rsi, 4*64, 64, 2*64
	loops_init 512, 1, 32
	r4_x4cl_four_complex_with_square_preload
baa:	r4_x4cl_four_complex_with_square rsi, 4*64, 64, 2*64
	loops	32, baa			;; Test loop counter
	bump	rsi, -32*4*64+dist128	;; Next source pointer
	loops	16, baa			;; Test loop counter
	bump	rsi, -16*dist128	;; Next source pointer
	jmp	xpass2_12_real_unfft

xpass2_12_levels_real_1:
	r4_h4cl_eight_reals_four_complex_fft_final rsi, 4*64, 64, 2*64
	L2prefetch128 [rcx]
	bump	rcx, 128
	loops_init_prefetch 512, 128, 1, rcx, 1, 32
bab0:	xtouch	[rcx+4096-64]		;; Preload the TLBs
bab:	r4_x4cl_four_complex_fft_final rsi, 4*64, 64, 2*64
	loops	32, bab			;; Test loop counter
	bump	rsi, -32*4*64+dist128	;; Next source pointer
	loops	2, bab0			;; Test loop counter
	bump	rcx, 128		;; Skip pad bytes
	loops	8, bab0			;; Test loop counter
	jmp	xpass2_12_real_done

xpass2_12_levels_real_3:
	mov	rbp, DIST_TO_MULSRCARG
	r4_h4cl_eight_reals_four_complex_with_mult rsi, 4*64, 64, 2*64
	loops_init 512, 1, 32
bac:	r4_x4cl_four_complex_with_mult rsi, 4*64, 64, 2*64
	loops	32, bac			;; Test loop counter
	bump	rsi, -32*4*64+dist128	;; Next source pointer
	loops	16, bac			;; Test loop counter
	bump	rsi, -16*dist128	;; Next source pointer
	jmp	xpass2_12_real_unfft

xpass2_12_levels_real_4:
	mov	rbp, DIST_TO_MULSRCARG
	mov	rcx, DATA_PREFETCH	;; Load prefetch pointer
	xtouch	[rcx]			;; Preload the TLBs
	r4_h4cl_eight_reals_four_complex_with_mulf rsi, 4*64, 64, 2*64
	L2prefetch128 [rcx]
	bump	rcx, 128
	loops_init_prefetch 512, 128, 1, rcx, 1, 32
bad0:	xtouch	[rcx+4096-64]		;; Preload the TLBs
bad:	r4_x4cl_four_complex_with_mulf rsi, 4*64, 64, 2*64
	loops	32, bad			;; Test loop counter
	bump	rsi, -32*4*64+dist128	;; Next source pointer
	loops	2, bad0			;; Test loop counter
	bump	rcx, 128		;; Skip pad bytes
	loops	8, bad0			;; Test loop counter
	bump	rsi, -16*dist128	;; Next source pointer

;; Do inverse FFT levels 9,10
;; On input the 64-byte cache lines hold these data values:
;;	0	+8K	2	+8K	4	+8K	6	+8K
;;	1	...
;;	8	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+8K	8	+8K	16	+8K	24	+8K
;;	1	...
;;	...
;;	7	...
;;	32	...
;;	...

;; Do 4 eight_reals_unfft operations
;; Do 4 four_complex_djbunfft operations
;;	distance between fft data elements is 4
;;	do 2 macros processing 32 data values

xpass2_12_real_unfft:
	mov	rdi, xsincos_complex	;; Load sin/cos pointers
	bump	rdi, 1024*XMM_SCD2
	mov	rdx, sincos3
	loops_init 512
c8b:	r4_h4cl_eight_reals_four_complex_djbunfft rsi, 64, 2*64, 4*64, rdi, 2*16*XMM_SCD2, rdx, 2*256*XMM_HSCD2
	bump	rdi, 16*XMM_SCD2	;; Next sin/cos data
	bump	rdx, 256*XMM_HSCD2	;; Next sin/cos data
	loops	2, c8b			;; Test loop counter
	bump	rsi, -2*64+8*64		;; Next source pointer
	bump	rdi, -2*16*XMM_SCD2	;; Restore sin/cos ptr
	bump	rdx, -2*256*XMM_HSCD2	;; Restore sin/cos ptr

;; Do 2040 four_complex_djbunfft operations
;;	distance between fft data elements is 4
;;	do 510 macros each processing 32 data values

	loops_reset			;; 16 iters of 15 or 16 of 2
	r4_x4cl_four_complex_djbunfft_preload
c9b:	r4_x4cl_four_complex_djbunfft rsi, 64, 2*64, 4*64, rdi, 2*16*XMM_SCD2
	bump	rdi, 16*XMM_SCD2	;; Next sine/cosine pointer
	loops	2, c9b			;; Test loop counter
	bump	rsi, -2*64+8*64		;; Next source pointer
	bump	rdi, -2*16*XMM_SCD2	;; Restore sin/cos ptr
	loops	16, c9b			;; Test loop counter
	bump	rsi, -16*8*64+dist128	;; Next source pointer
	loops	16, c9b			;; Test loop counter
	bump	rsi, -16*dist128	;; Next source pointer

;; Do inverse FFT levels 7,8
;; On input the 64-byte cache lines hold these data values:
;;	0	+8K	8	+8K	16	+8K	24	+8K
;;	1	...
;;	...
;;	7	...
;;	32	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+8K	32	+8K	64	+8K	96	+8K
;;	1	...
;;	...
;;	31	...
;;	128	...
;;	...

;; Do 16 eight_reals_unfft operations
;; Do 16 four_complex_djbunfft operations
;;	distance between fft data elements is 16
;;	do 8 macros processing 32 data values

	loops_init_prefetch 512, 64, 1, rcx
	xtouch	[rcx+4096-64]		;; Preload the TLBs
c6b:	r4_h4cl_eight_reals_four_complex_djbunfft rsi, 64, 8*64, 16*64, rdi, 8*4*XMM_SCD2, rdx, 8*64*XMM_HSCD2
	bump	rdi, 4*XMM_SCD2		;; Next sin/cos data
	bump	rdx, 64*XMM_HSCD2	;; Next sin/cos data
	loops	8, c6b			;; Test inner loop counter
	bump	rsi, -8*64+32*64	;; Next source pointer
	bump	rdi, -8*4*XMM_SCD2	;; Restore sin/cos ptr
	bump	rdx, -8*64*XMM_HSCD2	;; Restore sin/cos ptr

;; Do 2016 four_complex_djbunfft operations
;;	distance between fft data elements is 16
;;	do 504 macros each processing 32 data values

	loops_reset			;; 16 iters of 3 or 4 of 4
	r4_x4cl_four_complex_djbunfft_preload
c7a:	xtouch	[rcx+4096-64]		;; Preload the TLBs
c7b:	r4_x4cl_four_complex_djbunfft rsi, 64, 8*64, 16*64, rdi, 8*4*XMM_SCD2
	bump	rdi, 4*XMM_SCD2		;; Next sin/cos data
	loops	8, c7b			;; Test inner loop counter
	bump	rsi, -8*64+32*64	;; Next source pointer
	bump	rdi, -8*4*XMM_SCD2	;; Restore sin/cos ptr
	loops	4, c7b			;; Test loop counter
	bump	rsi, -4*32*64+dist128	;; Next source pointer
	loops_touch 2, c7b, 2, c7a	;; Test loop counter
	bump	rcx, 128		;; Skip pad bytes
	loops	4, c7a			;; Test loop counter
	bump	rsi, -16*dist128	;; Next source pointer

;; Do inverse FFT levels 5,6
;; On input the 64-byte cache lines hold these data values:
;;	0	+8K	32	+8K	64	+8K	96	+8K
;;	1	...
;;	...
;;	31	...
;;	128	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+8K	128	+8K	256	+8K	384	+8K
;;	1	...
;;	...
;;	127	...
;;	512	...
;;	...

;; Do 64 eight_reals_unfft operations
;; Do 64 four_complex_djbunfft operations
;;	distance between fft data elements is 64
;;	do 32 macros processing 32 data values

	loops_init_prefetch 512, 64, 1, rcx
	xtouch	[rcx+4096-64]		;; Preload the TLBs
c4b:	r4_h4cl_eight_reals_four_complex_djbunfft rsi, 64, 32*64, 64*64, rdi, 32*XMM_SCD2, rdx, 32*16*XMM_HSCD2
	bump	rdi, XMM_SCD2		;; Next sin/cos data
	bump	rdx, 16*XMM_HSCD2	;; Next sin/cos data
	loops	32, c4b			;; Test inner loop counter
	bump	rsi, -32*64+dist128	;; Next source pointer
	bump	rdi, -32*XMM_SCD2	;; Restore sin/cos ptr
	bump	rdx, -32*16*XMM_HSCD2	;; Restore sin/cos ptr

;; Do 1920 four_complex_djbunfft operations
;;	distance between fft data elements is 64
;;	do 480 macros each processing 32 data values

	loops_reset			;; 15 iters of 32
	r4_x4cl_four_complex_djbunfft_preload
c5a:	xtouch	[rcx+4096-64]		;; Preload the TLBs
c5b:	r4_x4cl_four_complex_djbunfft rsi, 64, 32*64, 64*64, rdi, 32*XMM_SCD2
	bump	rdi, XMM_SCD2		;; Next sin/cos data
	loops	32, c5b			;; Test inner loop counter
	bump	rsi, -32*64+dist128	;; Next source pointer
	bump	rdi, -32*XMM_SCD2	;; Restore sin/cos ptr
	loops_touch 2, c5b, 2, c5a	;; Test loop counter
	bump	rcx, 128		;; Skip pad bytes
	loops	4, c5a			;; Test loop counter
	bump	rsi, -16*dist128	;; Next source pointer

;; Do inverse FFT levels 3,4
;; On input the 64-byte cache lines hold these data values:
;;	0	+8K	128	+8K	256	+8K	384	+8K
;;	1	...
;;	...
;;	127	...
;;	512	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+8K	512	+8K	1024	+8K	1536	+8K
;;	1	...
;;	...
;;	511	...
;;	2048	...
;;	...

;; Do 256 eight_reals_unfft operations
;; Do 256 four_complex_djbunfft operations
;;	distance between fft data elements is 256
;;	do 128 macros processing 32 data values

	loops_init 512
	mov	rdi, xsincos_complex	;; Load sin/cos pointer
c2b:	r4_h4cl_eight_reals_four_complex_djbunfft rsi, 64, dist128, 2*dist128, rdi, 128*4*XMM_SCD2, rdx, 128*4*XMM_HSCD2
	bump	rdi, 4*XMM_SCD2		;; Next sin/cos data
	bump	rdx, 4*XMM_HSCD2	;; Next sin/cos data
	loops	128, c2b		;; Test inner loop counter
	bump	rsi, -128*64+4*dist128	;; Next source pointer
	bump	rdi, -128*4*XMM_SCD2	;; Restore sin/cos ptr
	bump	rdx, -128*4*XMM_HSCD2	;; Restore sin/cos ptr

;; Do 1536 four_complex_djbunfft operations
;;	distance between fft data elements is 256
;;	do 384 macros each processing 32 data values

	loops_reset			;; 3 iters of 128
	r4_x4cl_four_complex_djbunfft_preload
c3b:	r4_x4cl_four_complex_djbunfft rsi, 64, dist128, 2*dist128, rdi, 128*4*XMM_SCD2
	bump	rdi, 4*XMM_SCD2		;; Next sin/cos data
	loops	128, c3b		;; Test inner loop counter
	bump	rsi, -128*64+4*dist128	;; Next source pointer
	bump	rdi, -128*4*XMM_SCD2	;; Restore sin/cos ptr
	loops	4, c3b			;; Test loop counter
	bump	rsi, -4*4*dist128	;; Next source pointer

;; Do inverse FFT level 1,2
;; On input the 64-byte cache lines hold these data values:
;;	0	+8K	512	+8K	1024	+8K	1536	+8K
;;	1	...
;;	...
;;	511	...
;;	2048	...
;;	...
;; On output the 64-byte cache lines hold these data values if pass 1 swizzles:
;;	0	+8K	2048	+8K	4096	+8K	6144	+8K
;;	1	...
;;	...
;;	2047
;; On output the 64-byte cache lines hold these data values if pass 2 swizzles:
;;	0	+2K	4096	+2K	8192	+2K	12288	+2K
;;	1	...
;;	...
;;	2047

;; Do 1024 eight_reals_unfft operations
;; Do 1024 four_complex_djbunfft operations
;;	distance between fft data elements is 1024
;;	do 512 macros processing 32 data values

	loops_init 512
c1b:	r4_h4cl_eight_reals_four_complex_djbunfft rsi, 64, 4*dist128, 8*dist128, rdi, 512*XMM_SCD2, rdx, 512*XMM_HSCD2
	bump	rdi, XMM_SCD2		;; Next sin/cos data
	bump	rdx, XMM_HSCD2		;; Next sin/cos data
	loops	128, c1b		;; Test loop counter
	bump	rsi, -128*64+dist128	;; Next source pointer
	loops	4, c1b			;; Test loop counter

xpass2_12_real_done:
	end_timer 2
	ENDM


; *************** radix4-pass2-12-levels-complex macro ******************
; This macro takes 2 groups of 4096 complex values and
; performs the final 12 levels of a standard radix-4 FFT, squares the
; results, and does the first 12 levels of the inverse FFT.

r4_pass2_12_levels_complex MACRO
	LOCAL	b1b, b2b, b3a, b3b, b4a, b4b, b5a, b5b
	LOCAL	b6a0, b6a, b6b0, b6b, b6c0, b6c, b6d0, b6d
	LOCAL	c1b, c2a, c2b, c3a, c3b, c4a, c4b, c5a, c5b
	LOCAL	xpass2_12_levels_complex_1, xpass2_12_levels_complex_2
	LOCAL	xpass2_12_levels_complex_3, xpass2_12_levels_complex_4
	LOCAL	xpass2_12_levels_complex_unfft, xpass2_12_levels_complex_done

;; Type 4 FFTs skip the forward FFT process

	mov	rsi, DATA_ADDR		;; Load address of FFT data
	mov	rbx, DIST_TO_FFTSRCARG
	cmp	ffttype, 4
	je	xpass2_12_levels_complex_4

;; Do FFT level 1,2
;;
;; We swizzle the data so that one set of data is in the high word of
;; an xmm register, the other set is in the low word.  This means
;; a distance of 8192 data values.
;;
;; On input the 64-byte cache lines hold these data values if pass 2 swizzles:
;;	0	+2K	4096	+2K	8192	+2K	12288	+2K
;;	1	...
;;	...
;;	2047
;; On input the 64-byte cache lines hold these data values if pass 1 swizzles:
;;	0	+8K	2048	+8K	4096	+8K	6144	+8K
;;	1	...
;;	...
;;	2047
;; On output the 64-byte cache lines hold these data values:
;;	0	+8K	1024	+8K	2048	+8K	3072	+8K
;;	1	...
;;	...
;;	1023	...
;;	4096	...
;;	...

;; Do 2048 four_complex_fft operations
;;	distance between fft data elements is 1024
;;	do 1024 macros each processing 16 data values

;; Do FFT levels 3,4
;;
;; On input the 64-byte cache lines hold these data values:
;;	0	+8K	1024	+8K	2048	+8K	3072	+8K
;;	1	...
;;	...
;;	1023	...
;;	4096	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+8K	256	+8K	512	+8K	768	+8K
;;	1	...
;;	...
;;	255	...
;;	1024	...
;;	...

;; Do 2048 four_complex_fft operations
;;	distance between fft data elements is 256
;;	do 512 macros each processing 32 data values

IF @INSTR(,%xarch,<CORE>) NE 0		;; Core 2 faster doing separate loops

	start_timer 4
	mov	rdi, xsincos_complex	;; The sin/cos data
	loops_init 1024			;; 8 iterations of 128
	r4_f2cl_four_complex_djbfft_preload
b1b:	r4_f2cl_four_complex_djbfft rsi, 64, 8*dist128, rdi
	bump	rdi, XMM_SCD2		;; Next sin/cos data
	loops	128, b1b		;; Test inner loop counter
	bump	rsi, -128*64+dist128	;; Next source pointer
	loops	8, b1b			;; Test outer loop counter
	bump	rsi, -8*dist128		;; Restore source pointer
	end_timer 4

	start_timer 5
	mov	rdi, xsincos_complex	;; Ptr to the sin/cos data
	loops_init 512			;; 2 iters of 2 iters of 128
	r4_x4cl_four_complex_djbfft_preload
b2b:	r4_x4cl_four_complex_djbfft rsi, 64, 2*dist128, 4*dist128, rdi
	bump	rdi, 4*XMM_SCD2		;; Next sin/cos data
	loops	128, b2b		;; Test inner loop counter
	bump	rsi, -128*64+dist128	;; Next source pointer
	loops	2, b2b			;; Test loop counter
	bump	rsi, -2*dist128+8*dist128 ;; Next source pointer
	bump	rdi, -256*4*XMM_SCD2	;; Restore sin/cos ptr
	loops	2, b2b			;; Test loop counter
	bump	rsi, -2*8*dist128	;; Restore source pointer
	end_timer 5

ELSE					;; Pentium 4 faster doing merged loops

	start_timer 4
	mov	rdi, xsincos_complex	;; The sin/cos data
	mov	rbp, rdi
	loops_init 1024			;; 8 iterations of 128
	;r4_f2cl_four_complex_djbfft_preload -- assume same as r4_x4cl_four_complex_djbfft_preload
	r4_x4cl_four_complex_djbfft_preload

b1b:	r4_f2cl_four_complex_djbfft rsi, 2*dist128, 8*dist128, rdi
	xxprefetch [rdi+XMM_SCD2]	;; Prefetch for next pass
	bump	rdi, 256*XMM_SCD2	;; Next sin/cos data
	loops	4, b1b			;; Test inner loop counter
	bump	rsi, -4*2*dist128	;; Restore source pointer
	loops_undo 4			;; Back up loop counter

b2b:	r4_x4cl_four_complex_djbfft rsi, 8*dist128, 2*dist128, 4*dist128, rbp
	loops	2, b2b, 2		;; Test inner loop counter with non-standard increment

	bump	rsi, -2*8*dist128+64	;; Next source pointer
	bump	rdi, -4*256*XMM_SCD2+XMM_SCD2 ;; Next sin/cos data
	bump	rbp, 4*XMM_SCD2		;; Next sin/cos data
	loops	128, b1b		;; Test loop counter
	bump	rsi, -128*64+dist128	;; Next source pointer
	loops	2, b1b			;; Test loop counter
	bump	rsi, -2*dist128		;; Restore source pointer
	bump	rdi, -256*XMM_SCD2	;; Restore sin/cos pointer
	end_timer 4

ENDIF

;; Do FFT levels 5,6
;;
;; On input the 64-byte cache lines hold these data values:
;;	0	+8K	256	+8K	512	+8K	768	+8K
;;	1	...
;;	...
;;	255	...
;;	1024	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+8K	64	+8K	128	+8K	192	+8K
;;	1	...
;;	...
;;	63	...
;;	256	...
;;	...
;;
;; Do 2048 four_complex_fft operations
;;	distance between fft data elements is 64
;;	do 512 macros each processing 32 data values

;; Do FFT levels 7,8
;;
;; On input the 64-byte cache lines hold these data values:
;;	0	+8K	64	+8K	128	+8K	192	+8K
;;	1	...
;;	...
;;	63	...
;;	256	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+8K	16	+8K	32	+8K	48	+8K
;;	1	...
;;	...
;;	15	...
;;	64	...
;;	...
;;
;; Do 2048 four_complex_fft operations
;;	distance between fft data elements is 16
;;	do 512 macros each processing 32 data values

;; Do FFT levels 9,10
;;
;; On input the 64-byte cache lines hold these data values:
;;	0	+8K	16	+8K	32	+8K	48	+8K
;;	1	...
;;	...
;;	15	...
;;	64	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+8K	4	+8K	8	+8K	12	+8K
;;	1	...
;;	...
;;	3	...
;;	16	...
;;	...
;;
;; Do 2048 four_complex_fft operations
;;	distance between fft data elements is 4
;;	do 512 macros each processing 32 data values

;; Do this 64 cache lines at a time to maximize accessing data
;; while it is in the L1 cache.

IFDEF NEVER				;; Never faster to do separate loops

	start_timer 6
	mov	rcx, DATA_PREFETCH	;; Prefetch pointer
	xtouch	[rcx]			;; Load first TLB
	bump	rdi, 1024*XMM_SCD2	;; Load sin/cos pointer
	loops_init_prefetch 512, 64, 2, rcx ;; 2*4 iters of 64
	r4_x4cl_four_complex_djbfft_preload
b3a:	xtouch	[rcx+4096-128]		;; Load two prefetch TLBs
	xtouch	[rcx+2*4096-128]
b3b:	r4_x4cl_four_complex_djbfft rsi, 64, 64*64, dist128, rdi
	bump	rdi, XMM_SCD2		;; Next sine/cosine pointer
	loops	64, b3b			;; Loop 64 times
	bump	rsi, -64*64+2*dist128	;; Next source pointer
	bump	rdi, -64*XMM_SCD2	;; Restore sine/cosine pointer
	loops	4, b3b			;; Test loop counter
	bump	rcx, -128*64+8192+128	;; Next 8KB section to prefetch
	loops	2, b3a			;; Test loop counter
	bump	rsi, -2*4*2*dist128	;; Restore source pointer
	end_timer 6

	start_timer 7
	loops_init_prefetch 512, 64, 2, rcx ;; 2*8*2 iters of 16
	r4_x4cl_four_complex_djbfft_preload
b4a:	xtouch	[rcx+4096-128]		;; Load two prefetch TLBs
	xtouch	[rcx+2*4096-128]
b4b:	r4_x4cl_four_complex_djbfft rsi, 64, 16*64, 32*64, rdi
	bump	rdi, 4*XMM_SCD2		;; Next sine/cosine pointer
	loops	16, b4b			;; Loop 16 times
	bump	rsi, -16*64+64*64	;; Next source pointer
	bump	rdi, -16*4*XMM_SCD2	;; Restore sine/cosine pointer
	loops	2, b4b			;; Test loop counter
	bump	rsi, -2*64*64+dist128	;; Adjust source pointer
	loops	8, b4b			;; Test loop counter
	bump	rcx, -128*64+8192+128	;; Next 8KB section to prefetch
	loops	2, b4a			;; Test loop counter
	bump	rsi, -2*8*dist128	;; Restore source pointer
	end_timer 7

	start_timer 8
	loops_init_prefetch 512, 64, 2, rcx ;; 2*8*8 iters of 4
	r4_x4cl_four_complex_djbfft_preload
b5a:	xtouch	[rcx+4096-128]		;; Load two prefetch TLBs
	xtouch	[rcx+2*4096-128]
b5b:	r4_x4cl_four_complex_djbfft rsi, 64, 4*64, 8*64, rdi
	bump	rdi, 16*XMM_SCD2	;; Next sine/cosine pointer
	loops	4, b5b			;; Loop 4 times
	bump	rsi, -4*64+16*64	;; Next source pointer
	bump	rdi, -4*16*XMM_SCD2	;; Restore sine/cosine pointer
	loops	8, b5b			;; Test loop counter
	bump	rsi, -8*16*64+dist128	;; Adjust source pointer
	loops	8, b5b			;; Test loop counter
	bump	rcx, -128*64+8192+128	;; Next 8KB section to prefetch
	loops	2, b5a			;; Test loop counter
	bump	rsi, -2*8*dist128	;; Restore source pointer
	end_timer 8

ELSE					;; Always faster to do merged loops

	start_timer 6
	mov	rcx, DATA_PREFETCH	;; Prefetch pointer
	xtouch	[rcx]			;; Load first TLB
	bump	rdi, 1024*XMM_SCD2	;; Load sin/cos pointers
	mov	rbx, rdi
	mov	rbp, rdi
	loops_init_prefetch 512, 64, 2, rcx ;; 2*4 iters of 4 of 64 cache lines
	r4_x4cl_four_complex_djbfft_preload

b3a:	xtouch	[rcx+4096-128]		;; Load six prefetch TLBs
	xtouch	[rcx+2*4096-128]
	xtouch	[rcx+3*4096-128]
	xtouch	[rcx+4*4096-128]
	xtouch	[rcx+5*4096-128]
	xtouch	[rcx+6*4096-128]

b3b:	xxprefetch [rdi+4*XMM_SCD2]	;; Prefetch for next pass
	r4_x4cl_four_complex_djbfft rsi, 4*64, 64*64, dist128, rdi
	bump	rdi, 4*XMM_SCD2		;; Next sine/cosine pointer
	loops	16, b3b			;; Loop 16 times
	bump	rsi, -16*4*64		;; Restore source pointer
	bump	rcx, -8*64+8192+128	;; Prefetch from next section
	loops_undo 16			;; Back up loop counter

b4b:	r4_x4cl_four_complex_djbfft rsi, 4*64, 16*64, 32*64, rbx
	bump	rbx, 4*4*XMM_SCD2	;; Next sine/cosine pointer
	loops	4, b4b			;; Loop 4 times
	bump	rsi, -4*4*64+64*64	;; Next source pointer
	bump	rbx, -4*4*4*XMM_SCD2	;; Restore sine/cosine pointer
	loops	2, b4b			;; Loop twice
	bump	rsi, -2*64*64+dist128	;; Next source pointer
	loops	2, b4b			;; Loop twice
	bump	rsi, -2*dist128		;; Restore source pointer
	bump	rcx, -8*64+8192+128	;; Prefetch from next section
	loops_undo 2*2*4		;; Back up loop counter

b5b:	r4_x4cl_four_complex_djbfft rsi, 16*64, 4*64, 8*64, rbp
	loops	8, b5b			;; Loop 8 times
	bump	rsi, -8*16*64+dist128	;; Next source pointer
	loops	2, b5b			;; Loop twice
	bump	rcx, -2*(8192+128)	;; Back to prefetching from 1st section

	bump	rsi, -2*dist128+64	;; Next source pointer
	bump	rdi, -16*4*XMM_SCD2+XMM_SCD2 ;; Next sine/cosine pointer
	bump	rbx, 4*XMM_SCD2		;; Next sine/cosine pointer
	bump	rbp, 16*XMM_SCD2	;; Next sine/cosine pointer
	loops	4, b3b			;; Loop 4 times
	bump	rsi, -4*64+2*dist128	;; Next source pointer
	bump	rdi, -4*XMM_SCD2	;; Restore sine/cosine pointer
	bump	rbx, -4*4*XMM_SCD2	;; Restore sine/cosine pointer
	bump	rbp, -4*16*XMM_SCD2	;; Restore sine/cosine pointer
	loops	4, b3b			;; Loop 4 times
	bump	rcx, -16*8*64+3*(8192+128) ;; Prefetch next 3 sections
	loops	2, b3a			;; Loop twice
	bump	rsi, -8*2*dist128	;; Restore source pointer
	end_timer 6
ENDIF

;; Do FFT levels 11,12 as well as inverse FFT levels 11,12
;;
;; On input the 64-byte cache lines hold these data values:
;;	0	+8K	4	+8K	8	+8K	12	+8K
;;	1	...
;;	...
;;	3	...
;;	16	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+8K	2	+8K	4	+8K	6	+8K
;;	1	...
;;	8	...
;;	...

;; Do 2048 four_complex_fft operations
;;	distance between fft data elements is 1
;;	do 512 macros each processing 32 data values

;; Execute the right middle step

	cmp	ffttype, 2
	jl	xpass2_12_levels_complex_1
	jg	xpass2_12_levels_complex_3

xpass2_12_levels_complex_2:
	start_timer 9
	loops_init_prefetch 512, 64, 2, rcx
	r4_x4cl_four_complex_with_square_preload
b6a0:	xtouch	[rcx+4096-128]		;; Load prefetch TLBs
	xtouch	[rcx+2*4096-128]
b6a:	r4_x4cl_four_complex_with_square rsi, 4*64, 64, 2*64
	loops	32, b6a			;; Loop 32 times
	bump	rsi, -32*4*64+dist128	;; Next source pointer
	loops	8, b6a			;; Loop 8 times
	bump	rcx, -64*128+(8192+128)	;; Prefetch next section
	loops	2, b6a0			;; Loop twice
	bump	rsi, -16*dist128	;; Restore source pointer
	end_timer 9
	jmp	xpass2_12_levels_complex_unfft

xpass2_12_levels_complex_1:
	start_timer 9
	loops_init_prefetch 512, 128, 1, rcx
b6b0:	xtouch	[rcx+4096-128]		;; Load prefetch TLBs
	xtouch	[rcx+2*4096-128]
b6b:	r4_x4cl_four_complex_fft_final rsi, 4*64, 64, 2*64
	loops	32, b6b			;; Loop 32 times
	bump	rsi, -32*4*64+dist128	;; Next source pointer
	loops	2, b6b			;; Loop 2 times
	bump	rcx, 128		;; Prefetch next section
	loops	8, b6b0			;; Loop 8 times
	end_timer 9
	jmp	xpass2_12_levels_complex_done

xpass2_12_levels_complex_3:
	start_timer 9
	mov	rbp, DIST_TO_MULSRCARG
	loops_init_prefetch 512, 64, 2, rcx
b6c0:	xtouch	[rcx+4096-128]		;; Load prefetch TLBs
	xtouch	[rcx+2*4096-128]
b6c:	r4_x4cl_four_complex_with_mult rsi, 4*64, 64, 2*64
	loops	32, b6c			;; Loop 32 times
	bump	rsi, -32*4*64+dist128	;; Next source pointer
	loops	8, b6c			;; Loop 8 times
	bump	rcx, -64*128+(8192+128)	;; Prefetch next section
	loops	2, b6c0			;; Loop twice
	bump	rsi, -16*dist128	;; Restore source pointer
	end_timer 9
	jmp	xpass2_12_levels_complex_unfft

xpass2_12_levels_complex_4:
	start_timer 9
	mov	rcx, DATA_PREFETCH	;; Prefetch pointer
	xtouch	[rcx]			;; Load first prefetch TLB
	mov	rbp, DIST_TO_MULSRCARG
	loops_init_prefetch 512, 128, 1, rcx
b6d0:	xtouch	[rcx+4096-128]		;; Load prefetch TLBs
	xtouch	[rcx+2*4096-128]
b6d:	r4_x4cl_four_complex_with_mulf rsi, 4*64, 64, 2*64
	loops	32, b6d			;; Loop 32 times
	bump	rsi, -32*4*64+dist128	;; Next source pointer
	loops	2, b6d			;; Loop 2 times
	bump	rcx, 128		;; Prefetch next section
	loops	8, b6d0			;; Loop 8 times
	bump	rsi, -16*dist128	;; Restore source pointer
	end_timer 9

;; Do inverse FFT levels 9,10
;; On input the 64-byte cache lines hold these data values:
;;	0	+8K	2	+8K	4	+8K	6	+8K
;;	1	...
;;	8	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+8K	8	+8K	16	+8K	24	+8K
;;	1	...
;;	...
;;	7	...
;;	32	...
;;	...
;;
;; Do 2048 four_complex_unfft operations
;;	distance between fft data elements is 4
;;	do 512 macros each processing 32 data values

;; Do inverse FFT levels 7,8
;; On input the 64-byte cache lines hold these data values:
;;	0	+8K	8	+8K	16	+8K	24	+8K
;;	1	...
;;	...
;;	7	...
;;	32	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+8K	32	+8K	64	+8K	96	+8K
;;	1	...
;;	...
;;	31	...
;;	128	...
;;	...
;;
;; Do 2048 four_complex_unfft operations
;;	distance between fft data elements is 16
;;	do 512 macros each processing 32 data values

;; Do inverse FFT levels 5,6
;; On input the 64-byte cache lines hold these data values:
;;	0	+8K	32	+8K	64	+8K	96	+8K
;;	1	...
;;	...
;;	31	...
;;	128	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+8K	128	+8K	256	+8K	384	+8K
;;	1	...
;;	...
;;	127	...
;;	512	...
;;	...
;;
;; Do 2048 four_complex_unfft operations
;;	distance between fft data elements is 64
;;	do 512 macros each processing 32 data values

;; Do this 64 cache lines at a time to maximize accessing data
;; while it is in the L1 cache.

xpass2_12_levels_complex_unfft:

IFDEF NEVER				;; Never faster to do separate loops

	start_timer 10
	mov	rdi, xsincos_complex	;; Load sin/cos pointers
	bump	rdi, 1024*XMM_SCD2
	loops_init_prefetch 512, 64, 2, rcx;; 2 iters of 8 iters of 16 iters of 2
	r4_x4cl_four_complex_djbunfft_preload
c5a:	xtouch	[rcx+4096-128]		;; Load prefetch TLBs
	xtouch	[rcx+2*4096-128]
c5b:	r4_x4cl_four_complex_djbunfft rsi, 64, 2*64, 4*64, rdi, 2*16*XMM_SCD2
	bump	rdi, 16*XMM_SCD2	;; Next sine/cosine pointer
	loops	2, c5b			;; Loop 2 times
	bump	rsi, -2*64+8*64		;; Next source pointer
	bump	rdi, -2*16*XMM_SCD2	;; Reset sine/cosine pointer
	loops	16, c5b			;; Test loop counter
	bump	rsi, -16*8*64+dist128	;; Next source pointer
	loops	8, c5b			;; Loop 8 times
	bump	rcx, -128*64+8192+128	;; Next 8KB section to prefetch
	loops	2, c5a			;; Test loop counter
	bump	rsi, -2*8*dist128	;; Restore source pointer
	end_timer 10

	start_timer 11
	loops_init_prefetch 512, 64, 2, rcx;; 2 iters of 8 iters of 4 iters of 8
	r4_x4cl_four_complex_djbunfft_preload
c4a:	xtouch	[rcx+4096-128]		;; Load prefetch TLBs
	xtouch	[rcx+2*4096-128]
c4b:	r4_x4cl_four_complex_djbunfft rsi, 64, 8*64, 16*64, rdi, 8*4*XMM_SCD2
	bump	rdi, 4*XMM_SCD2		;; Next sine/cosine pointer
	loops	8, c4b			;; Loop 8 times
	bump	rsi, -8*64+32*64	;; Next source pointer
	bump	rdi, -8*4*XMM_SCD2	;; Reset sine/cosine pointer
	loops	4, c4b			;; Test loop counter
	bump	rsi, -4*32*64+dist128	;; Next source pointer
	loops	8, c4b			;; Loop 8 times
	bump	rcx, -128*64+8192+128	;; Next 8KB section to prefetch
	loops	2, c4a			;; Test loop counter
	bump	rsi, -2*8*dist128	;; Restore source pointer
	end_timer 11

	start_timer 12
	loops_init_prefetch 512, 64, 2, rcx;; 2 iters of 8 iters of 32
	r4_x4cl_four_complex_djbunfft_preload
c3a:	xtouch	[rcx+4096-128]		;; Load prefetch TLBs
	xtouch	[rcx+2*4096-128]
c3b:	r4_x4cl_four_complex_djbunfft rsi, 64, 32*64, 64*64, rdi, 32*1*XMM_SCD2
	bump	rdi, 1*XMM_SCD2		;; Next sine/cosine pointer
	loops	32, c3b			;; Loop 32 times
	bump	rsi, -32*64+dist128	;; Next source pointer
	bump	rdi, -32*1*XMM_SCD2	;; Reset sine/cosine pointer
	loops	8, c3b			;; Loop 8 times
	bump	rcx, -128*64+8192+128	;; Next 8KB section to prefetch
	loops	2, c3a			;; Test loop counter
	bump	rsi, -2*8*dist128	;; Restore source pointer
	end_timer 12

ELSE					;; Always faster to do merged loops

	start_timer 10
	mov	rdi, xsincos_complex	;; Load sin/cos pointers
	bump	rdi, 1024*XMM_SCD2
	mov	rbx, rdi
	mov	rbp, rdi
	loops_init_prefetch 512, 64, 2, rcx ;; 2*2*8 iters of 64 cache lines
	r4_x4cl_four_complex_djbunfft_preload

c5a:	xtouch	[rcx+4096-128]		;; Load six prefetch TLBs
	xtouch	[rcx+2*4096-128]
	xtouch	[rcx+3*4096-128]
	xtouch	[rcx+4*4096-128]
	xtouch	[rcx+5*4096-128]
	xtouch	[rcx+6*4096-128]

c5b:	r4_x4cl_four_complex_djbunfft rsi, 8*64, 2*64, 4*64, rbp, 2*16*XMM_SCD2
	loops	16, c5b			;; Loop 16 times
	bump	rsi, -16*8*64		;; Restore source pointer
	bump	rcx, -8*64+8192+128	;; Prefetch from next section
	loops_undo 16			;; Restore loop counter

c4b:	r4_x4cl_four_complex_djbunfft rsi, 2*64, 8*64, 16*64, rbx, 8*4*XMM_SCD2
	bump	rbx, 2*4*XMM_SCD2	;; Next sine/cosine pointer
	loops	4, c4b			;; Loop 4 times
	bump	rsi, -4*2*64+32*64	;; Next source pointer
	bump	rbx, -4*2*4*XMM_SCD2	;; Restore sine/cosine pointer
	loops	4, c4b			;; Loop 4 times
	bump	rsi, -4*32*64		;; Restore source pointer
	bump	rcx, -8*64+8192+128	;; Prefetch from next section
	loops_undo 4*4			;; Restore loop counter

c3b:	;;xxprefetch [rdi+XMM_SCD2+2*XMM_SCD2] ;; Prefetch for next pass
	;;xxprefetch [rdi+XMM_SCD2+34*XMM_SCD2] ;; Prefetch for next pass
	r4_x4cl_four_complex_djbunfft rsi, 2*64, 32*64, 64*64, rdi, 32*1*XMM_SCD2
	bump	rdi, 2*XMM_SCD2		;; Next sine/cosine pointer
	loops	16, c3b			;; Loop 16 times
	bump	rcx, -2*(8192+128)	;; Back to prefetching from 1st section

	bump	rsi, -16*2*64+64	;; Next source pointer
	bump	rdi, -16*2*XMM_SCD2+XMM_SCD2 ;; Next sine/cosine pointer
	bump	rbx, 4*XMM_SCD2		;; Next sine/cosine pointer
	bump	rbp, 16*XMM_SCD2	;; Next sine/cosine pointer
	loops	2, c5b			;; Loop twice
	bump	rsi, -2*64+dist128	;; Next source pointer
	bump	rdi, -2*XMM_SCD2	;; Restore sine/cosine pointer
	bump	rbx, -2*4*XMM_SCD2	;; Restore sine/cosine pointer
	bump	rbp, -2*16*XMM_SCD2	;; Restore sine/cosine pointer
	loops	8, c5b			;; Loop 8 times
	bump	rcx, -16*8*64+3*(8192+128) ;; Prefetch next 3 sections
	loops	2, c5a			;; Loop twice
	bump	rsi, -16*dist128	;; Restore source pointer
	end_timer 10
ENDIF

;; Do inverse FFT levels 3,4
;; On input the 64-byte cache lines hold these data values:
;;	0	+8K	128	+8K	256	+8K	384	+8K
;;	1	...
;;	...
;;	127	...
;;	512	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+8K	512	+8K	1024	+8K	1536	+8K
;;	1	...
;;	...
;;	511	...
;;	2048	...
;;	...

;; Do 2048 four_complex_unfft operations
;;	distance between fft data elements is 256
;;	do 512 macros each processing 32 data values

;; Do inverse FFT levels 1,2
;; On input the 64-byte cache lines hold these data values:
;;	0	+8K	512	+8K	1024	+8K	1536	+8K
;;	1	...
;;	...
;;	511	...
;;	2048	...
;;	...
;; On output the 64-byte cache lines hold these data values if pass 1 swizzles:
;;	0	+8K	2048	+8K	4096	+8K	6144	+8K
;;	1	...
;;	...
;;	2047
;; On output the 64-byte cache lines hold these data values if pass 2 swizzles:
;;	0	+2K	4096	+2K	8192	+2K	12288	+2K
;;	1	...
;;	...
;;	2047

;; Do 2048 four_complex_unfft operations
;;	distance between fft data elements is 1024
;;	do 512 macros each processing 32 data values

IF @INSTR(,%xarch,<CORE>) NE 0		;; Core 2 faster doing separate loops

	start_timer 13
	mov	rdi, xsincos_complex	;; Load sin/cos pointer
	loops_init_prefetch 512, 64, 2, rcx;; 2 iters of 2 iters of 128
	r4_x4cl_four_complex_djbunfft_preload
c2a:	xtouch	[rcx+4096-128]		;; Load prefetch TLBs
	xtouch	[rcx+2*4096-128]
c2b:	r4_x4cl_four_complex_djbunfft rsi, 64, dist128, 2*dist128, rdi, 128*4*XMM_SCD2
	bump	rdi, 4*XMM_SCD2		;; Next sine/cosine pointer
	loops	128, c2b		;; Loop 128 times
	bump	rsi, -128*64+4*dist128	;; Next source pointer
	bump	rdi, -128*4*XMM_SCD2	;; Reset sine/cosine pointer
	loops	2, c2b			;; Test loop counter
	bump	rcx, -128*64+8192+128	;; Next 8KB section to prefetch
	loops	2, c2a			;; Test loop counter
	bump	rsi, -2*2*4*dist128	;; Restore source pointer
	end_timer 13

	start_timer 14
	mov	rdi, xsincos_complex	;; Load sin/cos pointer
	loops_init 512			;; 4 iters of 128
	r4_x4cl_four_complex_djbunfft_preload
c1b:	r4_x4cl_four_complex_djbunfft rsi, 64, 4*dist128, 8*dist128, rdi, 512*XMM_SCD2
	bump	rdi, XMM_SCD2		;; Next set of sin/cos data
	loops	128, c1b		;; Test inner loop counter
	bump	rsi, -128*64+dist128	;; Next source pointer
	loops	4, c1b			;; Test outer loop counter
	end_timer 14

ELSE					;; Pentium 4 faster doing merged loops

	start_timer 13
	mov	rdi, xsincos_complex	;; Load sin/cos pointer
	mov	rbp, rdi
	loops_init_prefetch 512, 64, 4, rcx;; 128 iters of 4, prefetching 128 cache lines in each loop
	r4_x4cl_four_complex_djbunfft_preload

c2a:	xtouch	[rcx+4096-128]		;; Load prefetch TLBs
	xtouch	[rcx+2*4096-128]

c2b:	r4_x4cl_four_complex_djbunfft rsi, 4*dist128, dist128, 2*dist128, rbp, 128*4*XMM_SCD2
	loops	4, c2b			;; Loop 4 times
	bump	rsi, -4*4*dist128	;; Restore source pointer
	loops_undo 4			;; Back up loop counter

c1b:	;;xxprefetch [rdi+XMM_SCD2+XMM_SCD2] ;; Prefetch for next pass
	r4_x4cl_four_complex_djbunfft rsi, dist128, 4*dist128, 8*dist128, rdi, 512*XMM_SCD2
	;;xxprefetch [rdi+XMM_SCD2+513*XMM_SCD2] ;; Prefetch for next pass
	bump	rdi, 128*XMM_SCD2	;; Next sin/cos data
	loops	4, c1b			;; Test loop counter

	bump	rsi, -4*dist128+64	;; Next source pointer
	bump	rdi, -4*128*XMM_SCD2+XMM_SCD2 ;; Next set of sin/cos data
	bump	rbp, 4*XMM_SCD2		;; Next sine/cosine pointer
	loops	64, c2b			;; Test loop counter
	bump	rcx, -128*64+8192+128	;; Next 8KB section to prefetch
	loops	2, c2a			;; Test loop counter
	end_timer 13

ENDIF

xpass2_12_levels_complex_done:
	ENDM


; *************** radix4-pass2-5120-levels-real macro ******************
; This macro takes 10240 real values and performs the final 13.322 levels of
; the FFT.  Independently, this macro also takes 5120 complex values and
; performs the final 12.322 levels of the FFT.  The results are squared, and
; the first 12.322/13.322 levels of the inverse FFT are done.

r4_pass2_5120_levels_real MACRO
	LOCAL	b1b, b2b, b3a, b3b, b4b, b5b, b6b, b7b
	LOCAL	b8b, b9b, baa, bab0, bab, bac, bad0, bad
	LOCAL	c1b, c2b, c3b, c4b, c5b, c6b, c7b, c8b, c9b
	LOCAL	xpass2_5120_levels_real_1, xpass2_5120_levels_real_2
	LOCAL	xpass2_5120_levels_real_3, xpass2_5120_levels_real_4
	LOCAL	xpass2_5120_real_unfft, xpass2_5120_real_done

;; We switch to the same format used in one-pass FFTs (that is, the
;; high word of xmm registers being 1 greater than the low word):

;; Type 4 FFTs skip the forward FFT process

	start_timer 2
	mov	rsi, DATA_ADDR		;; Load source address
	mov	rbx, DIST_TO_FFTSRCARG
	cmp	ffttype, 4
	je	xpass2_5120_levels_real_4

;; Do FFT level 1,2 (actually 2.322 levels)
;;
;; On input the 64-byte cache lines hold these data values if pass 2 swizzles:
;;	0	+2.5K	5120	+2.5K	10240	+2.5K	15360	+2.5K
;;	1	...
;;	...
;;	2559
;; On input the 64-byte cache lines hold these data values if pass 1 swizzles:
;;	0	+10K	2560	+10K	5120	+10K	7680	+10K
;;	1	...
;;	...
;;	2559	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+10K	512	+10K	1024	+10K	1536	+10K
;;	1	...
;;	...
;;	511	...
;;	2048	...
;;	...

;; Do 1024 ten_reals_fft on values 0 - 10239
;; Do 1024 five_complex_djbfft on values 10240 - 20479
;;	distance between fft data elements is 1K
;;	do 512 iterations

	mov	rdi, xsincos_complex	;; Ptr to the sin/cos data
	mov	rdx, sincos3		;; Ptr to real sin/cos data
	loops_init 512
b1b:	r5_fh5cl_ten_reals_five_complex_djbfft rsi, 64, 4*dist128, rdi, 512*XMM_SCD2, rdx, 512*XMM_HSCD2
	bump	rdi, XMM_SCD2		;; Next sin/cos data
	bump	rdx, XMM_HSCD2		;; Next sin/cos data
	loops	128, b1b		;; Test loop counter
	bump	rsi, -128*64+dist128	;; Next source pointer
	loops	4, b1b			;; Test loop counter
	bump	rsi, -4*dist128		;; Restore source pointer
	bump	rdi, -512*XMM_SCD2	;; Restore sin/cos ptr
	bump	rdx, 512*XMM_HSCD2	;; Next sin/cos ptr

;; Do FFT levels 3,4
;; Values 0-2047 is real data, 2048-20479 is complex data.
;;
;; On input the 64-byte cache lines hold these data values:
;;	0	+10K	512	+10K	1024	+10K	1536	+10K
;;	1	...
;;	...
;;	511	...
;;	2048	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+10K	256	+10K	512	+10K	768	+10K
;;	1	...
;;	...
;;	255	...
;;	1024	...
;;	...

;; Do 256 eight_reals_fft operations
;; Do 256 four_complex_djbfft operations
;;	distance between fft data elements is 256
;;	do 256 macros each processing 16 data values

	mov	rcx, DATA_PREFETCH	;; Prefetch pointer
	xtouch	[rcx]			;; Preload first TLB
	loops_init_prefetch 256, 64, 4, rcx
	xtouch	[rcx+4096-128]		;; Preload the TLBs
b2b:	r4_h2cl_eight_reals_four_complex_djbfft rsi, 64, 2*dist128, rdi, rdx
	bump	rdi, 5*XMM_SCD2		;; Next sin/cos data
	bump	rdx, XMM_HSCD2		;; Next sin/cos data
	loops	128, b2b		;; Test inner loop counter
	bump	rsi, -128*64+dist128	;; Next source pointer
	loops	2, b2b			;; Test loop counter
	bump	rsi, -2*dist128+4*dist128 ;; Next source pointer
	bump	rdi, -256*5*XMM_SCD2	;; Restore sin/cos ptr
	bump	rdx, -256*XMM_HSCD2	;; Restore sin/cos ptr

;; Do 2048 four_complex_fft operations
;;	distance between fft data elements is 512
;;	do 1024 macros each processing 16 data values

	loops_init_prefetch 1024, 64, 4, rcx
	r4_x2cl_four_complex_djbfft_preload
b3a:	xtouch	[rcx+4096-64]		;; Preload the TLBs
b3b:	r4_x2cl_four_complex_djbfft rsi, 64, 2*dist128, rdi
	bump	rdi, 5*XMM_SCD2		;; Next sin/cos data
	loops	128, b3b		;; Test inner loop counter
	bump	rsi, -128*64+dist128	;; Next source pointer
	bump	rcx, 128		;; Skip pad bytes
	loops	2, b3b			;; Test loop counter
	bump	rsi, -2*dist128+4*dist128 ;; Next source pointer
	bump	rdi, -256*5*XMM_SCD2	;; Restore sin/cos ptr
	loops	4, b3a			;; Test loop counter
	bump	rsi, -4*4*dist128-4*dist128 ;; Restore source pointer

;; Do FFT levels 5,6
;; Values 0-511 is real data, 512-20479 is complex data.
;;
;; On input the 64-byte cache lines hold these data values:
;;	0	+10K	256	+10K	512	+10K	768	+10K
;;	1	...
;;	...
;;	255	...
;;	1024	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+10K	64	+10K	128	+10K	192	+10K
;;	1	...
;;	...
;;	63	...
;;	256	...
;;	...

;; Do 64 eight_reals_fft operations
;; Do 192 four_complex_djbfft operations
;;	distance between fft data elements is 64
;;	do 64 macros each processing 32 data values

	bump	rdi, 1280*XMM_SCD2	;; Load sin/cos pointer
	loops_init_prefetch 640, 64, 2, rcx
	xtouch	[rcx+4096-128]		;; Preload the TLBs
	xtouch	[rcx+2*4096-128]
	xtouch	[rcx+3*4096-128]
	xtouch	[rcx+4*4096-128]
	xtouch	[rcx+5*4096-128]
	r4_h4cl_eight_reals_four_complex_djbfft_preload
b4b:	r4_h4cl_eight_reals_four_complex_djbfft rsi, 64, 64*64, dist128, rdi, rdx
	bump	rdi, XMM_SCD2		;; Next sin/cos data
	bump	rdx, 4*XMM_HSCD2	;; Next sin/cos data
	loops	64, b4b			;; Test loop counter
	bump	rsi, -64*64+2*dist128	;; Next source pointer
	bump	rdi, -64*XMM_SCD2	;; Restore sin/cos ptr
	bump	rdx, -64*4*XMM_HSCD2	;; Restore sin/cos ptr

;; Do 2304 four_complex_djbfft operations
;;	distance between fft data elements is 64
;;	do 576 macros each processing 32 data values

	loops_reset			;; 9 iterations of 64
	r4_x4cl_four_complex_djbfft_preload
b5b:	r4_x4cl_four_complex_djbfft rsi, 64, 64*64, dist128, rdi
	bump	rdi, XMM_SCD2		;; Next sin/cos data
	loops	64, b5b			;; Test inner loop counter
	bump	rsi, -64*64+2*dist128	;; Next source pointer
	bump	rdi, -64*XMM_SCD2	;; Restore sin/cos ptr
	loops	10, b5b			;; Test loop counter
	bump	rsi, -10*2*dist128	;; Next source pointer

;; Do FFT levels 7,8
;; Values 0-127 is real data, 128-20479 is complex data.
;;
;; On input the 64-byte cache lines hold these data values:
;;	0	+10K	64	+10K	128	+10K	192	+10K
;;	1	...
;;	...
;;	63	...
;;	256	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+10K	16	+10K	32	+10K	48	+10K
;;	1	...
;;	...
;;	15	...
;;	64	...
;;	...

;; Do 16 eight_reals_fft operations
;; Do 48 four_complex_djbfft operations
;;	distance between fft data elements is 16
;;	do 16 macros each processing 32 data values

	loops_init_prefetch 640, 64, 2, rcx
	xtouch	[rcx+4096-128]		;; Preload the TLBs
	xtouch	[rcx+2*4096-128]
	xtouch	[rcx+3*4096-128]
	xtouch	[rcx+4*4096-128]
	xtouch	[rcx+5*4096-128]
	r4_h4cl_eight_reals_four_complex_djbfft_preload
b6b:	r4_h4cl_eight_reals_four_complex_djbfft rsi, 64, 16*64, 32*64, rdi, rdx
	bump	rdi, 4*XMM_SCD2		;; Next sin/cos data
	bump	rdx, 16*XMM_HSCD2	;; Next sin/cos data
	loops	16, b6b			;; Test loop counter
	bump	rsi, -16*64+64*64	;; Next source pointer
	bump	rdi, -16*4*XMM_SCD2	;; Restore sin/cos ptr
	bump	rdx, -16*16*XMM_HSCD2	;; Restore sin/cos ptr

;; Do 1472 four_complex_djbfft operations
;;	distance between fft data elements is 16
;;	do 368 macros each processing 32 data values

	loops_reset			;; 20 iters of 1 or 2 of 16
	r4_x4cl_four_complex_djbfft_preload
b7b:	r4_x4cl_four_complex_djbfft rsi, 64, 16*64, 32*64, rdi
	bump	rdi, 4*XMM_SCD2		;; Next sin/cos data
	loops	16, b7b			;; Test inner loop counter
	bump	rsi, -16*64+64*64	;; Next source pointer
	bump	rdi, -16*4*XMM_SCD2	;; Restore sin/cos ptr
	loops	2, b7b			;; Test loop counter
	bump	rsi, -2*64*64+dist128	;; Next source pointer
	loops	20, b7b			;; 20 iterations
	bump	rsi, -20*dist128	;; Next source pointer

;; Do FFT levels 9,10
;; Values 0-31 is real data, 32-20479 is complex data.
;;
;; On input the 64-byte cache lines hold these data values:
;;	0	+10K	16	+10K	32	+10K	48	+10K
;;	1	...
;;	...
;;	15	...
;;	64	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+10K	4	+10K	8	+10K	12	+10K
;;	1	...
;;	...
;;	3	...
;;	16	...
;;	...

;; Do 4 eight_reals_fft operations
;; Do 12 four_complex_djbfft operations
;;	distance between fft data elements is 4
;;	do 4 macros each processing 32 data values

	loops_init_prefetch 640, 64, 2, rcx
	xtouch	[rcx+4096-128]		;; Preload the TLBs
	xtouch	[rcx+2*4096-128]
	xtouch	[rcx+3*4096-128]
	xtouch	[rcx+4*4096-128]
	xtouch	[rcx+5*4096-128]
	r4_h4cl_eight_reals_four_complex_djbfft_preload
b8b:	r4_h4cl_eight_reals_four_complex_djbfft rsi, 64, 4*64, 8*64, rdi, rdx
	bump	rdi, 16*XMM_SCD2	;; Next sin/cos data
	bump	rdx, 64*XMM_HSCD2	;; Next sin/cos data
	loops	4, b8b			;; Test loop counter
	bump	rsi, -4*64+16*64	;; Next source pointer
	bump	rdi, -4*16*XMM_SCD2	;; Restore sin/cos ptr
	bump	rdx, -4*64*XMM_HSCD2	;; Restore sin/cos ptr

;; Do 2544 four_complex_djbfft operations
;;	distance between fft data elements is 4
;;	do 636 macros each processing 32 data values

	loops_reset			;; 20 iters of 7 or 8 of 4
	r4_x4cl_four_complex_djbfft_preload
b9b:	r4_x4cl_four_complex_djbfft rsi, 64, 4*64, 8*64, rdi
	bump	rdi, 16*XMM_SCD2	;; Next sin/cos data
	loops	4, b9b			;; Test loop counter
	bump	rsi, -4*64+16*64	;; Next source pointer
	bump	rdi, -4*16*XMM_SCD2	;; Restore sin/cos ptr
	loops	8, b9b			;; Test middle loop counter
	bump	rsi, -8*16*64+dist128	;; Next source pointer
	loops	20, b9b			;; 20 iterations
	bump	rsi, -20*dist128	;; Next source pointer

;; Do FFT levels 11,12
;; Values 0-7 is real data, 8-20479 is complex data.
;;
;; On input the 64-byte cache lines hold these data values:
;;	0	+10K	4	+10K	8	+10K	12	+10K
;;	1	...
;;	...
;;	3	...
;;	16	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+10K	2	+10K	4	+10K	6	+10K
;;	1	...
;;	8	...
;;	...

;; Do 1 eight_reals_fft operations
;; Do 3 four_complex_fft operations
;;	distance between fft data elements is 1

;; Do 2556 four_complex_fft operations
;;	distance between fft data elements is 1
;;	do 639 macros each processing 32 data values

;; Execute the proper middle step

	cmp	ffttype, 2
	jl	xpass2_5120_levels_real_1
	jg	xpass2_5120_levels_real_3

xpass2_5120_levels_real_2:
	r4_h4cl_eight_reals_four_complex_with_square rsi, 4*64, 64, 2*64
	loops_init_prefetch 640, 64, 8, rcx, 1, 32
	xtouch	[rcx+4096-128]		;; Preload the TLBs
	xtouch	[rcx+2*4096-128]
	r4_x4cl_four_complex_with_square_preload
baa:	r4_x4cl_four_complex_with_square rsi, 4*64, 64, 2*64
	loops	32, baa			;; Test loop counter
	bump	rsi, -32*4*64+dist128	;; Next source pointer
	loops	20, baa			;; Test loop counter
	bump	rsi, -20*dist128	;; Next source pointer
	jmp	xpass2_5120_real_unfft

xpass2_5120_levels_real_1:
	r4_h4cl_eight_reals_four_complex_fft_final rsi, 4*64, 64, 2*64
	loops_init_prefetch 640, 128, 1, rcx, 1, 32
bab0:	xtouch	[rcx+4096-64]		;; Preload the TLBs
bab:	r4_x4cl_four_complex_fft_final rsi, 4*64, 64, 2*64
	loops	32, bab			;; Test loop counter
	bump	rsi, -32*4*64+dist128	;; Next source pointer
	loops	20, bab0		;; Test loop counter
	jmp	xpass2_5120_real_done

xpass2_5120_levels_real_3:
	mov	rbp, DIST_TO_MULSRCARG
	r4_h4cl_eight_reals_four_complex_with_mult rsi, 4*64, 64, 2*64
	loops_init_prefetch 640, 64, 8, rcx, 1, 32
	xtouch	[rcx+4096-128]		;; Preload the TLBs
	xtouch	[rcx+2*4096-128]
bac:	r4_x4cl_four_complex_with_mult rsi, 4*64, 64, 2*64
	loops	32, bac			;; Test loop counter
	bump	rsi, -32*4*64+dist128	;; Next source pointer
	loops	20, bac			;; Test loop counter
	bump	rsi, -20*dist128	;; Next source pointer
	jmp	xpass2_5120_real_unfft

xpass2_5120_levels_real_4:
	mov	rbp, DIST_TO_MULSRCARG
	mov	rcx, DATA_PREFETCH	;; Load prefetch pointer
	xtouch	[rcx]			;; Preload first TLB
	r4_h4cl_eight_reals_four_complex_with_mulf rsi, 4*64, 64, 2*64
	loops_init_prefetch 640, 128, 1, rcx, 1, 32
bad0:	xtouch	[rcx+4096-64]		;; Preload the TLBs
bad:	r4_x4cl_four_complex_with_mulf rsi, 4*64, 64, 2*64
	loops	32, bad			;; Test loop counter
	bump	rsi, -32*4*64+dist128	;; Next source pointer
	loops	20, bad0		;; Test loop counter
	bump	rsi, -20*dist128	;; Next source pointer

;; Do inverse FFT levels 9,10
;; On input the 64-byte cache lines hold these data values:
;;	0	+10K	2	+10K	4	+10K	6	+10K
;;	1	...
;;	8	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+10K	8	+10K	16	+10K	24	+10K
;;	1	...
;;	...
;;	7	...
;;	32	...
;;	...

;; Do 4 eight_reals_unfft operations
;; Do 4 four_complex_djbunfft operations
;;	distance between fft data elements is 4
;;	do 2 macros processing 32 data values

xpass2_5120_real_unfft:
	mov	rdi, xsincos_complex	;; Load sin/cos pointers
	bump	rdi, 1280*XMM_SCD2
	mov	rdx, sincos3
	bump	rdx, 1024*XMM_HSCD2
	loops_init_prefetch 640, 64, 2, rcx
	xtouch	[rcx+4096-128]		;; Preload the TLBs
	xtouch	[rcx+2*4096-128]
	xtouch	[rcx+3*4096-128]
	xtouch	[rcx+4*4096-128]
	xtouch	[rcx+5*4096-128]
c8b:	r4_h4cl_eight_reals_four_complex_djbunfft rsi, 64, 2*64, 4*64, rdi, 2*16*XMM_SCD2, rdx, 2*64*XMM_HSCD2
	bump	rdi, 16*XMM_SCD2	;; Next sin/cos data
	bump	rdx, 64*XMM_HSCD2	;; Next sin/cos data
	loops	2, c8b			;; Test loop counter
	bump	rsi, -2*64+8*64		;; Next source pointer
	bump	rdi, -2*16*XMM_SCD2	;; Restore sin/cos ptr
	bump	rdx, -2*64*XMM_HSCD2	;; Restore sin/cos ptr

;; Do 2552 four_complex_djbunfft operations
;;	distance between fft data elements is 4
;;	do 638 macros each processing 32 data values

	loops_reset			;; 20 iters of 15 or 16 of 2
	r4_x4cl_four_complex_djbunfft_preload
c9b:	r4_x4cl_four_complex_djbunfft rsi, 64, 2*64, 4*64, rdi, 2*16*XMM_SCD2
	bump	rdi, 16*XMM_SCD2	;; Next sine/cosine pointer
	loops	2, c9b			;; Test loop counter
	bump	rsi, -2*64+8*64		;; Next source pointer
	bump	rdi, -2*16*XMM_SCD2	;; Restore sin/cos ptr
	loops	16, c9b			;; Test loop counter
	bump	rsi, -16*8*64+dist128	;; Next source pointer
	loops	20, c9b			;; Test loop counter
	bump	rsi, -20*dist128	;; Next source pointer

;; Do inverse FFT levels 7,8
;; On input the 64-byte cache lines hold these data values:
;;	0	+10K	8	+10K	16	+10K	24	+10K
;;	1	...
;;	...
;;	7	...
;;	32	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+10K	32	+10K	64	+10K	96	+10K
;;	1	...
;;	...
;;	31	...
;;	128	...
;;	...

;; Do 16 eight_reals_unfft operations
;; Do 16 four_complex_djbunfft operations
;;	distance between fft data elements is 16
;;	do 8 macros processing 32 data values

	loops_init_prefetch 640, 64, 2, rcx
	xtouch	[rcx+4096-128]		;; Preload the TLBs
	xtouch	[rcx+2*4096-128]
	xtouch	[rcx+3*4096-128]
	xtouch	[rcx+4*4096-128]
	xtouch	[rcx+5*4096-128]
c6b:	r4_h4cl_eight_reals_four_complex_djbunfft rsi, 64, 8*64, 16*64, rdi, 8*4*XMM_SCD2, rdx, 8*16*XMM_HSCD2
	bump	rdi, 4*XMM_SCD2		;; Next sin/cos data
	bump	rdx, 16*XMM_HSCD2	;; Next sin/cos data
	loops	8, c6b			;; Test inner loop counter
	bump	rsi, -8*64+32*64	;; Next source pointer
	bump	rdi, -8*4*XMM_SCD2	;; Restore sin/cos ptr
	bump	rdx, -8*16*XMM_HSCD2	;; Restore sin/cos ptr

;; Do 2528 four_complex_djbunfft operations
;;	distance between fft data elements is 16
;;	do 632 macros each processing 32 data values

	loops_reset			;; 20 iters of 3 or 4 of 4
	r4_x4cl_four_complex_djbunfft_preload
c7b:	r4_x4cl_four_complex_djbunfft rsi, 64, 8*64, 16*64, rdi, 8*4*XMM_SCD2
	bump	rdi, 4*XMM_SCD2		;; Next sin/cos data
	loops	8, c7b			;; Test inner loop counter
	bump	rsi, -8*64+32*64	;; Next source pointer
	bump	rdi, -8*4*XMM_SCD2	;; Restore sin/cos ptr
	loops	4, c7b			;; Test loop counter
	bump	rsi, -4*32*64+dist128	;; Next source pointer
	loops	20, c7b			;; 20 iterations
	bump	rsi, -20*dist128	;; Next source pointer

;; Do inverse FFT levels 5,6
;; On input the 64-byte cache lines hold these data values:
;;	0	+10K	32	+10K	64	+10K	96	+10K
;;	1	...
;;	...
;;	31	...
;;	128	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+10K	128	+10K	256	+10K	384	+10K
;;	1	...
;;	...
;;	127	...
;;	512	...
;;	...

;; Do 64 eight_reals_unfft operations
;; Do 64 four_complex_djbunfft operations
;;	distance between fft data elements is 64
;;	do 32 macros processing 32 data values

	loops_init_prefetch 640, 64, 2, rcx
	xtouch	[rcx+4096-128]		;; Preload the TLBs
	xtouch	[rcx+2*4096-128]
	xtouch	[rcx+3*4096-128]
	xtouch	[rcx+4*4096-128]
	xtouch	[rcx+5*4096-128]
c4b:	r4_h4cl_eight_reals_four_complex_djbunfft rsi, 64, 32*64, 64*64, rdi, 32*XMM_SCD2, rdx, 32*4*XMM_HSCD2
	bump	rdi, XMM_SCD2		;; Next sin/cos data
	bump	rdx, 4*XMM_HSCD2	;; Next sin/cos data
	loops	32, c4b			;; Test inner loop counter
	bump	rsi, -32*64+dist128	;; Next source pointer
	bump	rdi, -32*XMM_SCD2	;; Restore sin/cos ptr
	bump	rdx, -32*4*XMM_HSCD2	;; Restore sin/cos ptr

;; Do 1408 four_complex_djbunfft operations
;;	distance between fft data elements is 64
;;	do 352 macros each processing 32 data values

	loops_reset			;; 19 iters of 32
	r4_x4cl_four_complex_djbunfft_preload
c5b:	r4_x4cl_four_complex_djbunfft rsi, 64, 32*64, 64*64, rdi, 32*XMM_SCD2
	bump	rdi, XMM_SCD2		;; Next sin/cos data
	loops	32, c5b			;; Test inner loop counter
	bump	rsi, -32*64+dist128	;; Next source pointer
	bump	rdi, -32*XMM_SCD2	;; Restore sin/cos ptr
	loops	20, c5b			;; Test loop counter
	bump	rsi, -20*dist128	;; Next source pointer

;; Do inverse FFT levels 3,4
;; On input the 64-byte cache lines hold these data values:
;;	0	+10K	128	+10K	256	+10K	384	+10K
;;	1	...
;;	...
;;	127	...
;;	512	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+10K	512	+10K	1024	+10K	1536	+10K
;;	1	...
;;	...
;;	511	...
;;	2048	...
;;	...

;; Do 256 eight_reals_unfft operations
;; Do 256 four_complex_djbunfft operations
;;	distance between fft data elements is 256
;;	do 128 macros processing 32 data values

	mov	rdi, xsincos_complex	;; Load sin/cos pointer
	loops_init_prefetch 640, 64, 4, rcx
	xtouch	[rcx+4096-64]		;; Preload the TLBs
	xtouch	[rcx+2*4096-64]
	xtouch	[rcx+3*4096-64]
c2b:	r4_h4cl_eight_reals_four_complex_djbunfft rsi, 64, dist128, 2*dist128, rdi, 128*5*XMM_SCD2, rdx, 128*XMM_HSCD2
	bump	rdi, 5*XMM_SCD2		;; Next sin/cos data
	bump	rdx, XMM_HSCD2		;; Next sin/cos data
	loops	128, c2b		;; Test inner loop counter
	bump	rsi, -128*64+4*dist128	;; Next source pointer
	bump	rdi, -128*5*XMM_SCD2	;; Restore sin/cos ptr
	bump	rdx, -128*XMM_HSCD2-1024*XMM_HSCD2 ;; Restore sin/cos ptr

;; Do 2048 four_complex_djbunfft operations
;;	distance between fft data elements is 256
;;	do 512 macros each processing 32 data values

	loops_reset			;; 4 iters of 128
	r4_x4cl_four_complex_djbunfft_preload
c3b:	r4_x4cl_four_complex_djbunfft rsi, 64, dist128, 2*dist128, rdi, 128*5*XMM_SCD2
	bump	rdi, 5*XMM_SCD2		;; Next sin/cos data
	loops	128, c3b		;; Test inner loop counter
	bump	rsi, -128*64+4*dist128	;; Next source pointer
	bump	rdi, -128*5*XMM_SCD2	;; Restore sin/cos ptr
	loops	5, c3b			;; Test loop counter
	bump	rsi, -5*4*dist128	;; Next source pointer

;; Do inverse FFT level 1,2 (actually 2.322 FFT levels)
;; On input the 64-byte cache lines hold these data values:
;;	0	+10K	512	+10K	1024	+10K	1536	+10K
;;	1	...
;;	...
;;	511	...
;;	2048	...
;;	...
;; On output the 64-byte cache lines hold these data values if pass 1 swizzles:
;;	0	+10K	2560	+10K	5120	+10K	7680	+10K
;;	1	...
;;	...
;;	2559
;; On output the 64-byte cache lines hold these data values if pass 2 swizzles:
;;	0	+2.5K	5120	+2.5K	10240	+2.5K	15360	+2.5K
;;	1	...
;;	...
;;	2559

;; Do 1024 ten_reals_unfft operations
;; Do 1024 five_complex_djbunfft operations
;;	distance between fft data elements is 1024
;;	do 512 macros processing 40 data values

	bump	rcx, -10*64		;; Back up prefetch pointer by 10 cache lines
	loops_init_prefetch 512, 64, 4, rcx
	xtouch	[rcx+4096-64]		;; Preload the TLBs
	xtouch	[rcx+2*4096-64]
c1b:	r5_h5cl_ten_reals_five_complex_djbunfft rsi, 64, 4*dist128, rdi, 512*XMM_SCD2, rdx, 512*XMM_HSCD2
	bump	rdi, XMM_SCD2		;; Next sin/cos data
	bump	rdx, XMM_HSCD2		;; Next sin/cos data
	loops	128, c1b		;; Test loop counter
	bump	rsi, -128*64+dist128	;; Next source pointer
	loops	4, c1b			;; Test loop counter

xpass2_5120_real_done:
	end_timer 2
	ENDM

; *************** radix4-pass2-5120-levels-complex macro ******************
; This macro takes 2 groups of 5120 complex values and
; performs the final 12.322 levels of a standard mostly radix-4 FFT, squares the
; results, and does the first 12.322 levels of the inverse FFT.

r4_pass2_5120_levels_complex MACRO
	LOCAL	b1a, b1b, b2a, b2b, b3a, b3b, b4b, b5b, b6a, b6b0, b6b, b6c, b6d0, b6d
	LOCAL	c1b, c2a0, c2a, c2b, c3b, c4b, c5a, c5b
	LOCAL	xpass2_5120_levels_complex_1, xpass2_5120_levels_complex_2
	LOCAL	xpass2_5120_levels_complex_3, xpass2_5120_levels_complex_4
	LOCAL	xpass2_5120_levels_complex_unfft, xpass2_5120_levels_complex_done

;; Type 4 FFTs skip the forward FFT process

	mov	rsi, DATA_ADDR		;; Load address of FFT data
	mov	rbx, DIST_TO_FFTSRCARG
	cmp	ffttype, 4
	je	xpass2_5120_levels_complex_4

;; Do FFT levels 1,2 (actually 2.322 levels)
;;
;; We swizzle the data so that one set of data is in the high word of
;; an xmm register, the other set is in the low word.  This means
;; a distance of 10240 data values.
;;
;; On input the 64-byte cache lines hold these data values if pass 2 swizzles:
;;	0	+2.5K	5120	+2.5K	10240	+2.5K	15360	+2.5K
;;	1	...
;;	...
;;	2559
;; On input the 64-byte cache lines hold these data values if pass 1 swizzles:
;;	0	+10K	2560	+10K	5120	+10K	7680	+10K
;;	1	...
;;	...
;;	2559	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+10K	512	+10K	1024	+10K	1536	+10K
;;	1	...
;;	...
;;	511	...
;;	2048	...
;;	...

;; Do 2048 five_complex_fft operations
;;	distance between fft data elements is 1024
;;	do 512 macros each processing 40 data values

;; Do FFT levels 3,4
;;
;; On input the 64-byte cache lines hold these data values:
;;	0	+10K	512	+10K	1024	+10K	1536	+10K
;;	1	...
;;	...
;;	511	...
;;	2048	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+10K	256	+10K	512	+10K	768	+10K
;;	1	...
;;	...
;;	255	...
;;	1024	...
;;	...

;; Do 2560 four_complex_fft operations
;;	distance between fft data elements is 256
;;	do 1280 macros each processing 16 data values

IF @INSTR(,%xarch,<CORE>) NE 0		;; Core 2 faster doing separate loops

	start_timer 4
	mov	rdi, xsincos_complex	;; The sin/cos data
	loops_init 512			;; 4 iterations of 128
	r5_f5cl_five_complex_djbfft_preload
b1b:	r5_f5cl_five_complex_djbfft rsi, 64, 4*dist128, rdi, 512*XMM_SCD2
	bump	rdi, XMM_SCD2		;; Next sin/cos data
	loops	128, b1b		;; Test inner loop counter
	bump	rsi, -128*64+dist128	;; Next source pointer
	loops	4, b1b			;; Test outer loop counter
	bump	rsi, -4*dist128		;; Restore source pointer
	end_timer 4

	start_timer 5
	mov	rdi, xsincos_complex	;; Ptr to the sin/cos data
	mov	rcx, DATA_PREFETCH	;; Prefetch pointer (we need to prefetch
					;; 2560 cache lines + 38 pad cache lines)
	xtouch	[rcx]			;; Preload first TLB
	loops_init_prefetch 1280, 64, 4, rcx ;; 3 iters of 2 iters of 128
	r4_x2cl_four_complex_djbfft_preload
b2a:	xtouch	[rcx+4096-64]		;; Preload the TLBs
b2b:	r4_x2cl_four_complex_djbfft rsi, 64, 2*dist128, rdi
	bump	rdi, 5*XMM_SCD2		;; Next sin/cos data
	loops	128, b2b		;; Test inner loop counter
	bump	rsi, -128*64+dist128	;; Next source pointer
	loops	2, b2b			;; Test loop counter
	bump	rsi, -2*dist128+4*dist128 ;; Next source pointer
	bump	rdi, -256*5*XMM_SCD2	;; Restore sin/cos ptr
	loops	5, b2a			;; Test loop counter
	bump	rsi, -5*4*dist128	;; Restore source pointer
	end_timer 5

ELSE					;; Pentium 4 faster doing merged loops

	start_timer 4
	mov	rdi, xsincos_complex	;; The sin/cos data
	mov	rbp, rdi
	mov	rcx, DATA_PREFETCH	;; Prefetch pointer (we need to prefetch
					;; 2560 cache lines + 38 pad cache lines)
	xtouch	[rcx]			;; Preload first TLB
	loops_init_prefetch 2560, 64, 20, rcx ;; 256 iters of 2 or 5

	xtouch	[rcx+4096-64]		;; Preload the TLBs
	xtouch	[rcx+2*4096-64]

b1a:	r5_f5cl_five_complex_djbfft_preload
b1b:	r5_f5cl_five_complex_djbfft rsi, 2*dist128, 4*dist128, rdi, 512*XMM_SCD2
	xxprefetch [rdi+XMM_SCD2]	;; Prefetch for next pass
	xxprefetch [rdi+513*XMM_SCD2]	;; Prefetch for next pass
	bump	rdi, 256*XMM_SCD2	;; Next sin/cos data
	loops	2, b1b, 5		;; Test loop counter with non-standard incr
	bump	rsi, -2*2*dist128	;; Restore source pointer
	loops_undo 10

	r4_x2cl_four_complex_djbfft_preload
b2b:	r4_x2cl_four_complex_djbfft rsi, 4*dist128, 2*dist128, rbp
	loops	5, b2b, 2		;; Test loop counter with non-standard incr

	bump	rsi, -5*4*dist128+64	;; Next source pointer
	bump	rdi, -2*256*XMM_SCD2+XMM_SCD2 ;; Next sin/cos ptr
	bump	rbp, 5*XMM_SCD2		;; Next sin/cos ptr
	loops	128, b1a		;; Test loop counter
	bump	rsi, -128*64+dist128	;; Next source pointer
	loops	2, b1a			;; Test loop counter
	bump	rsi, -2*dist128		;; Restore source pointer
	bump	rdi, -256*XMM_SCD2	;; Restore sin/cos ptr
	end_timer 4

ENDIF

;; Do FFT levels 5,6
;;
;; On input the 64-byte cache lines hold these data values:
;;	0	+10K	256	+10K	512	+10K	768	+10K
;;	1	...
;;	...
;;	255	...
;;	1024	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+10K	64	+10K	128	+10K	192	+10K
;;	1	...
;;	...
;;	63	...
;;	256	...
;;	...
;;
;; Do 2560 four_complex_fft operations
;;	distance between fft data elements is 64
;;	do 640 macros each processing 32 data values

;; Do FFT levels 7,8
;;
;; On input the 64-byte cache lines hold these data values:
;;	0	+10K	64	+10K	128	+10K	192	+10K
;;	1	...
;;	...
;;	63	...
;;	256	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+10K	16	+10K	32	+10K	48	+10K
;;	1	...
;;	...
;;	15	...
;;	64	...
;;	...
;;
;; Do 2560 four_complex_fft operations
;;	distance between fft data elements is 16
;;	do 640 macros each processing 32 data values

;; Do FFT levels 9,10
;;
;; On input the 64-byte cache lines hold these data values:
;;	0	+10K	16	+10K	32	+10K	48	+10K
;;	1	...
;;	...
;;	15	...
;;	64	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+10K	4	+10K	8	+10K	12	+10K
;;	1	...
;;	...
;;	3	...
;;	16	...
;;	...
;;
;; Do 2560 four_complex_fft operations
;;	distance between fft data elements is 4
;;	do 640 macros each processing 32 data values

;; Do this 64 cache lines at a time to maximize accessing data
;; while it is in the L1 cache.

	start_timer 6
	bump	rdi, 1280*XMM_SCD2	;; Load sin/cos pointers
	mov	rbx, rdi
	mov	rbp, rdi
	loops_init_prefetch 640, 64, 2, rcx ;; 10 iters of 4 of 64 cache lines
	r4_x4cl_four_complex_djbfft_preload

b3a:	xtouch	[rcx+4096-64]		;; Load prefetch TLBs
	xtouch	[rcx+2*4096-64]

b3b:	xxprefetch [rdi+4*XMM_SCD2]	;; Prefetch for next pass
	r4_x4cl_four_complex_djbfft rsi, 4*64, 64*64, dist128, rdi
	bump	rdi, 4*XMM_SCD2		;; Next sine/cosine pointer
	loops	16, b3b			;; Loop 16 times
	bump	rsi, -16*4*64		;; Restore source pointer
	bump	rdi, -16*4*XMM_SCD2	;; Restore sine/cosine pointer
	loops_undo 16			;; Back up loop counter

b4b:	r4_x4cl_four_complex_djbfft rsi, 4*64, 16*64, 32*64, rbx
	bump	rbx, 4*4*XMM_SCD2	;; Next sine/cosine pointer
	loops	4, b4b			;; Loop 4 times
	bump	rsi, -4*4*64+64*64	;; Next source pointer
	bump	rbx, -4*4*4*XMM_SCD2	;; Restore sine/cosine pointer
	loops	2, b4b			;; Loop twice
	bump	rsi, -2*64*64+dist128	;; Next source pointer
	loops	2, b4b			;; Loop twice
	bump	rsi, -2*dist128		;; Restore source pointer
	loops_undo 2*2*4		;; Back up loop counter

b5b:	r4_x4cl_four_complex_djbfft rsi, 16*64, 4*64, 8*64, rbp
	loops	8, b5b			;; Loop 8 times
	bump	rsi, -8*16*64+dist128	;; Next source pointer
	loops	2, b5b			;; Loop twice

	bump	rsi, -2*dist128+64	;; Next source pointer
	bump	rdi, XMM_SCD2		;; Next sine/cosine pointer
	bump	rbx, 4*XMM_SCD2		;; Next sine/cosine pointer
	bump	rbp, 16*XMM_SCD2	;; Next sine/cosine pointer
	loops	4, b3b			;; Loop 4 times
	bump	rsi, -4*64+2*dist128	;; Next source pointer
	bump	rdi, -4*XMM_SCD2	;; Restore sine/cosine pointer
	bump	rbx, -4*4*XMM_SCD2	;; Restore sine/cosine pointer
	bump	rbp, -4*16*XMM_SCD2	;; Restore sine/cosine pointer
	loops	10, b3a			;; Loop 10 times
	bump	rsi, -10*2*dist128	;; Restore source pointer
	end_timer 6

;; Do FFT levels 11,12 as well as inverse FFT levels 11,12
;;
;; On input the 64-byte cache lines hold these data values:
;;	0	+10K	4	+10K	8	+10K	12	+10K
;;	1	...
;;	...
;;	3	...
;;	16	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+10K	2	+10K	4	+10K	6	+10K
;;	1	...
;;	8	...
;;	...

;; Do 2560 four_complex_fft operations
;;	distance between fft data elements is 1
;;	do 640 macros each processing 32 data values

;; Execute the right middle step

	cmp	ffttype, 2
	jl	xpass2_5120_levels_complex_1
	jg	xpass2_5120_levels_complex_3

xpass2_5120_levels_complex_2:
	start_timer 9
	loops_init_prefetch 640, 64, 8, rcx
	xtouch	[rcx+4096-64]		;; Load prefetch TLBs
	xtouch	[rcx+2*4096-64]
	r4_x4cl_four_complex_with_square_preload
b6a:	r4_x4cl_four_complex_with_square rsi, 4*64, 64, 2*64
	loops	32, b6a			;; Loop 32 times
	bump	rsi, -16*2*4*64+dist128	;; Next source pointer
	loops	20, b6a			;; Loop 20 times
	bump	rsi, -20*dist128	;; Restore source pointer
	end_timer 9
	jmp	xpass2_5120_levels_complex_unfft

xpass2_5120_levels_complex_1:
	start_timer 9
	loops_init_prefetch 640, 128, 1, rcx
b6b0:	xtouch	[rcx+4096-128]		;; Load prefetch TLBs
b6b:	r4_x4cl_four_complex_fft_final rsi, 4*64, 64, 2*64
	loops	32, b6b			;; Loop 32 times
	bump	rsi, -16*2*4*64+dist128	;; Next source pointer
	loops	20, b6b0		;; Loop 20 times
	end_timer 9
	jmp	xpass2_5120_levels_complex_done

xpass2_5120_levels_complex_3:
	start_timer 9
	mov	rbp, DIST_TO_MULSRCARG
	loops_init_prefetch 640, 64, 8, rcx
	xtouch	[rcx+4096-128]		;; Load prefetch TLBs
	xtouch	[rcx+2*4096-128]
b6c:	r4_x4cl_four_complex_with_mult rsi, 4*64, 64, 2*64
	loops	32, b6c			;; Loop 32 times
	bump	rsi, -16*2*4*64+dist128	;; Next source pointer
	loops	20, b6c			;; Loop 20 times
	bump	rsi, -20*dist128	;; Restore source pointer
	end_timer 9
	jmp	xpass2_5120_levels_complex_unfft

xpass2_5120_levels_complex_4:
	start_timer 9
	mov	rcx, DATA_PREFETCH	;; Prefetch pointer
	xtouch	[rcx]			;; Preload first TLB
	mov	rbp, DIST_TO_MULSRCARG
	loops_init_prefetch 640, 128, 1, rcx
b6d0:	xtouch	[rcx+4096-64]		;; Load prefetch TLBs
b6d:	r4_x4cl_four_complex_with_mulf rsi, 4*64, 64, 2*64
	loops	32, b6d			;; Loop 32 times
	bump	rsi, -16*2*4*64+dist128	;; Next source pointer
	loops	20, b6d0		;; Loop 20 times
	bump	rsi, -20*dist128	;; Restore source pointer
	end_timer 9

;; Do inverse FFT levels 9,10
;; On input the 64-byte cache lines hold these data values:
;;	0	+10K	2	+10K	4	+10K	6	+10K
;;	1	...
;;	8	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+10K	8	+10K	16	+10K	24	+10K
;;	1	...
;;	...
;;	7	...
;;	32	...
;;	...
;;
;; Do 2560 four_complex_unfft operations
;;	distance between fft data elements is 4
;;	do 640 macros each processing 32 data values

;; Do inverse FFT levels 7,8
;; On input the 64-byte cache lines hold these data values:
;;	0	+10K	8	+10K	16	+10K	24	+10K
;;	1	...
;;	...
;;	7	...
;;	32	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+10K	32	+10K	64	+10K	96	+10K
;;	1	...
;;	...
;;	31	...
;;	128	...
;;	...
;;
;; Do 2560 four_complex_unfft operations
;;	distance between fft data elements is 16
;;	do 640 macros each processing 32 data values

;; Do inverse FFT levels 5,6
;; On input the 64-byte cache lines hold these data values:
;;	0	+10K	32	+10K	64	+10K	96	+10K
;;	1	...
;;	...
;;	31	...
;;	128	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+10K	128	+10K	256	+10K	384	+10K
;;	1	...
;;	...
;;	127	...
;;	512	...
;;	...
;;
;; Do 2560 four_complex_unfft operations
;;	distance between fft data elements is 64
;;	do 640 macros each processing 32 data values

;; Do this 64 cache lines at a time to maximize accessing data
;; while it is in the L1 cache.

xpass2_5120_levels_complex_unfft:
	start_timer 10
	mov	rdi, xsincos_complex	;; Load sin/cos pointers
	bump	rdi, 1280*XMM_SCD2
	mov	rbx, rdi
	mov	rbp, rdi
	loops_init_prefetch 640, 64, 2, rcx ;; 20*2 iters of 64 cache lines
	r4_x4cl_four_complex_djbunfft_preload

c5a:	xtouch	[rcx+4096-64]		;; Load prefetch TLBs

c5b:	r4_x4cl_four_complex_djbunfft rsi, 8*64, 2*64, 4*64, rbp, 2*16*XMM_SCD2
	loops	16, c5b			;; Loop 16 times
	bump	rsi, -16*8*64		;; Restore source pointer
	loops_undo 16			;; Restore loop counter

c4b:	r4_x4cl_four_complex_djbunfft rsi, 2*64, 8*64, 16*64, rbx, 8*4*XMM_SCD2
	bump	rbx, 2*4*XMM_SCD2	;; Next sine/cosine pointer
	loops	4, c4b			;; Loop 4 times
	bump	rsi, -4*2*64+32*64	;; Next source pointer
	bump	rbx, -4*2*4*XMM_SCD2	;; Restore sine/cosine pointer
	loops	4, c4b			;; Loop 4 times
	bump	rsi, -4*32*64		;; Restore source pointer
	loops_undo 4*4			;; Restore loop counter

c3b:	r4_x4cl_four_complex_djbunfft rsi, 2*64, 32*64, 64*64, rdi, 32*1*XMM_SCD2
	bump	rdi, 2*XMM_SCD2		;; Next sine/cosine pointer
	loops	16, c3b			;; Loop 16 times
	bump	rdi, -16*2*XMM_SCD2	;; Restore sine/cosine pointer

	bump	rsi, -16*2*64+64	;; Next source pointer
	bump	rdi, XMM_SCD2		;; Next sine/cosine pointer
	bump	rbx, 4*XMM_SCD2		;; Next sine/cosine pointer
	bump	rbp, 16*XMM_SCD2	;; Next sine/cosine pointer
	loops	2, c5b			;; Loop twice
	bump	rsi, -2*64+dist128	;; Next source pointer
	bump	rdi, -2*XMM_SCD2	;; Restore sine/cosine pointer
	bump	rbx, -2*4*XMM_SCD2	;; Restore sine/cosine pointer
	bump	rbp, -2*16*XMM_SCD2	;; Restore sine/cosine pointer
	loops	20, c5a			;; Loop 20 times
	bump	rsi, -20*dist128	;; Restore source pointer
	end_timer 10

;; Do inverse FFT levels 3,4
;; On input the 64-byte cache lines hold these data values:
;;	0	+10K	128	+10K	256	+10K	384	+10K
;;	1	...
;;	...
;;	127	...
;;	512	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+10K	512	+10K	1024	+10K	1536	+10K
;;	1	...
;;	...
;;	511	...
;;	2048	...
;;	...

;; Do 2560 four_complex_unfft operations
;;	distance between fft data elements is 256
;;	do 640 macros each processing 32 data values

;; Do inverse FFT levels 1,2 (actually 2.322 FFT levels)
;; On input the 64-byte cache lines hold these data values:
;;	0	+10K	512	+10K	1024	+10K	1536	+10K
;;	1	...
;;	...
;;	511	...
;;	2048	...
;;	...
;; On output the 64-byte cache lines hold these data values if pass 1 swizzles:
;;	0	+10K	2560	+10K	5120	+10K	7680	+10K
;;	1	...
;;	...
;;	2559
;; On output the 64-byte cache lines hold these data values if pass 2 swizzles:
;;	0	+2.5K	5120	+2.5K	10240	+2.5K	15360	+2.5K
;;	1	...
;;	...
;;	2559

;; Do 2048 five_complex_unfft operations
;;	distance between fft data elements is 1024
;;	do 512 macros each processing 40 data values

IF @INSTR(,%xarch,<CORE>) NE 0		;; Core 2 faster doing separate loops

	start_timer 13
	mov	rdi, xsincos_complex	;; Load sin/cos pointer
	loops_init_prefetch 640, 64, 4, rcx;; 5 iters of 128
	xtouch	[rcx+4096-64]		;; Load prefetch TLBs
	xtouch	[rcx+2*4096-64]
	xtouch	[rcx+3*4096-64]
	r4_x4cl_four_complex_djbunfft_preload
c2b:	r4_x4cl_four_complex_djbunfft rsi, 64, dist128, 2*dist128, rdi, 128*5*XMM_SCD2
	bump	rdi, 5*XMM_SCD2		;; Next sine/cosine pointer
	loops	128, c2b		;; Loop 128 times
	bump	rsi, -128*64+4*dist128	;; Next source pointer
	bump	rdi, -128*5*XMM_SCD2	;; Reset sine/cosine pointer
	loops	5, c2b			;; Test loop counter
	bump	rsi, -5*4*dist128	;; Restore source pointer
	end_timer 13

	start_timer 14
	mov	rdi, xsincos_complex	;; Load sin/cos pointer
	bump	rcx, -10*64		;; Back up prefetch pointer by 10 cache lines
	loops_init_prefetch 512, 64, 4, rcx;; 4 iters of 128
	xtouch	[rcx+4096-64]		;; Load prefetch TLBs
	xtouch	[rcx+2*4096-64]
	r5_x5cl_five_complex_djbunfft_preload
c1b:	r5_x5cl_five_complex_djbunfft rsi, 64, 4*dist128, rdi, 512*XMM_SCD2
	bump	rdi, XMM_SCD2		;; Next set of sin/cos data
	loops	128, c1b		;; Test inner loop counter
	bump	rsi, -128*64+dist128	;; Next source pointer
	loops	4, c1b			;; Test outer loop counter
	end_timer 14

ELSE					;; Pentium 4 faster doing merged loops

	start_timer 13
	mov	rdi, xsincos_complex	;; Load sin/cos pointer
	mov	rbp, rdi
	bump	rcx, -42*64		;; Back up prefetch pointer by 42 cache lines
	loops_init_prefetch 2560, 64, 5, rcx;; 128 iters of 4 or 5

c2a0:	xtouch	[rcx+4096-64]		;; Load prefetch TLB

c2a:	r4_x4cl_four_complex_djbunfft_preload
c2b:	r4_x4cl_four_complex_djbunfft rsi, 4*dist128, dist128, 2*dist128, rbp, 128*5*XMM_SCD2
	loops_no_prefetch 5, c2b, 4	;; Test loop counter with non-standard incr
	bump	rsi, -5*4*dist128	;; Restore source pointer
	loops_undo 20

	r5_x5cl_five_complex_djbunfft_preload
c1b:	r5_x5cl_five_complex_djbunfft rsi, dist128, 4*dist128, rdi, 512*XMM_SCD2
	bump	rdi, 128*XMM_SCD2	;; Next sin/cos ptr
	loops	4, c1b, 5		;; Test loop counter with non-standard incr

	bump	rsi, -4*dist128+64	;; Next source pointer
	bump	rdi, -4*128*XMM_SCD2+XMM_SCD2 ;; Next sine/cosine pointer
	bump	rbp, 5*XMM_SCD2		;; Next sine/cosine pointer
	loops_touch 16, c2a, 8, c2a0	;; Loop 128 times
	end_timer 13

ENDIF

xpass2_5120_levels_complex_done:
	ENDM


; *************** radix4-pass2-6400-levels-real macro ******************
; This macro takes 12800 real values and performs the final 13.644 levels of
; the FFT.  Independently, this macro also takes 6400 complex values and
; performs the final 12.644 levels of the FFT.  The results are squared, and
; the first 12.644/13.644 levels of the inverse FFT are done.

r4_pass2_6400_levels_real MACRO
	LOCAL	b1b, b2b, b3a, b3b, b4b, b5a, b5b, b6b, b7a, b7b
	LOCAL	b8b, b9a, b9b, baa, bab0, bab, bac, bad0, bad
	LOCAL	c1b, c2b, c3a, c3b, c4b, c5a, c5b, c6b, c7a, c7b, c8b, c9a, c9b
	LOCAL	xpass2_6400_levels_real_1, xpass2_6400_levels_real_2
	LOCAL	xpass2_6400_levels_real_3, xpass2_6400_levels_real_4
	LOCAL	xpass2_6400_real_unfft, xpass2_6400_real_done

;; We switch to the same format used in one-pass FFTs (that is, the
;; high word of xmm registers being 1 greater than the low word):

;; Type 4 FFTs skip the forward FFT process

	start_timer 2
	mov	rsi, DATA_ADDR		;; Load source address
	mov	rbx, DIST_TO_FFTSRCARG
	cmp	ffttype, 4
	je	xpass2_6400_levels_real_4

;; Do FFT level 1,2 (actually 2.322 levels)
;;
;; On input the 64-byte cache lines hold these data values if pass 2 swizzles:
;;	0	+3.2K	6400	+3.2K	12800	+3.2K	19200	+3.2K
;;	1	...
;;	...
;;	3199
;; On input the 64-byte cache lines hold these data values if pass 1 swizzles:
;;	0	+12.5K	3200	+12.5K	6400	+12.5K	9600	+12.5K
;;	1	...
;;	...
;;	3199	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+12.5K	640	+12.5K	1280	+12.5K	1920	+12.5K
;;	1	...
;;	...
;;	639	...
;;	2560	...
;;	...

;; Do 1280 ten_reals_fft on values 0 - 12799
;; Do 1280 five_complex_djbfft on values 12800 - 25599
;;	distance between fft data elements is 1280
;;	do 640 iterations

	mov	rdi, xsincos_complex	;; Ptr to the sin/cos data
	mov	rdx, sincos3		;; Ptr to real sin/cos data
	loops_init 640
b1b:	r5_fh5cl_ten_reals_five_complex_djbfft rsi, 64, 5*dist128, rdi, 640*XMM_SCD2, rdx, 640*XMM_HSCD2
	bump	rdi, XMM_SCD2		;; Next sin/cos data
	bump	rdx, XMM_HSCD2		;; Next sin/cos data
	loops	128, b1b		;; Test loop counter
	bump	rsi, -128*64+dist128	;; Next source pointer
	loops	5, b1b			;; Test loop counter
	bump	rsi, -5*dist128		;; Restore source pointer
	bump	rdi, -640*XMM_SCD2	;; Restore sin/cos ptr
	bump	rdx, -640*XMM_HSCD2	;; Restore sin/cos ptr

;; Do FFT levels 3,4 (actually 2.322 levels)
;; Values 0-2559 is real data, 2560-25599 is complex data.
;;
;; On input the 64-byte cache lines hold these data values:
;;	0	+12.5K	640	+12.5K	1280	+12.5K	1920	+12.5K
;;	1	...
;;	...
;;	639	...
;;	2560	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+12.5K	128	+12.5K	256	+12.5K	384	+12.5K
;;	1	...
;;	...
;;	127	...
;;	512	...
;;	...

;; Do 256 ten_reals_fft operations
;; Do 256 five_complex_djbfft operations
;;	distance between fft data elements is 256
;;	do 128 macros each processing 40 data values

	mov	rcx, DATA_PREFETCH	;; Prefetch pointer
	xtouch	[rcx]			;; Preload first TLB
	loops_init_prefetch 640, 64, 2, rcx
	xtouch	[rcx+4096-64]		;; Preload the TLBs
b2b:	r5_h5cl_ten_reals_five_complex_djbfft rsi, 64, dist128, rdi, 128*5*XMM_SCD2, rdx, 128*5*XMM_HSCD2
	bump	rdi, 5*XMM_SCD2		;; Next sin/cos data
	bump	rdx, 5*XMM_HSCD2	;; Next sin/cos data
	loops	128, b2b		;; Test inner loop counter
	bump	rsi, -128*64+5*dist128	;; Next source pointer
	bump	rdi, -128*5*XMM_SCD2	;; Restore sin/cos ptr
	bump	rdx, -128*5*XMM_HSCD2+1280*XMM_HSCD2 ;; Next sin/cos ptr

;; Do 2048 five_complex_fft operations
;;	distance between fft data elements is 256
;;	do 512 macros each processing 40 data values

	loops_reset
	r5_nf5cl_five_complex_djbfft_preload
b3a:	xtouch	[rcx+4096-64]		;; Preload the TLBs
b3b:	r5_nf5cl_five_complex_djbfft rsi, 64, dist128, rdi, 128*5*XMM_SCD2
	bump	rdi, 5*XMM_SCD2		;; Next sin/cos data
	loops	128, b3b		;; Test inner loop counter
	bump	rsi, -128*64+5*dist128	;; Next source pointer
	bump	rdi, -128*5*XMM_SCD2	;; Restore sin/cos ptr
	bump	rcx, 128		;; Skip pad bytes
	loops	5, b3a			;; Test loop counter
	bump	rsi, -4*5*dist128-5*dist128 ;; Restore source pointer

;; Do FFT levels 5,6
;; Values 0-511 is real data, 512-25599 is complex data.
;;
;; On input the 64-byte cache lines hold these data values:
;;	0	+12.5K	128	+12.5K	256	+12.5K	384	+12.5K
;;	1	...
;;	...
;;	127	...
;;	512	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+12.5K	64	+12.5K	128	+12.5K	192	+12.5K
;;	1	...
;;	...
;;	63	...
;;	256	...
;;	...

;; Do 64 eight_reals_fft operations
;; Do 64 four_complex_djbfft operations
;;	distance between fft data elements is 64
;;	do 64 macros each processing 16 data values

	bump	rdi, 1280*XMM_SCD2	;; Load sin/cos pointer
	loops_init_prefetch 1600, 64, 4, rcx
	xtouch	[rcx+4096-64]		;; Preload the TLBs
b4b:	r4_h2cl_eight_reals_four_complex_djbfft rsi, 64, 64*64, rdi, rdx
	bump	rdi, XMM_SCD2		;; Next sin/cos data
	bump	rdx, XMM_HSCD2		;; Next sin/cos data
	loops	64, b4b			;; Test loop counter
	bump	rsi, -64*64+dist128	;; Next source pointer
	bump	rdi, -64*XMM_SCD2	;; Restore sin/cos ptr
	bump	rdx, -64*XMM_HSCD2	;; Restore sin/cos ptr

;; Do 3072 four_complex_djbfft operations
;;	distance between fft data elements is 64
;;	do 1536 macros each processing 16 data values

	loops_reset			;; 24 more iterations of 64
	r4_nf2cl_four_complex_djbfft_preload
b5a:	xtouch	[rcx+4096-64]		;; Preload the TLBs
	xtouch	[rcx+80*64-64]
b5b:	r4_nf2cl_four_complex_djbfft rsi, 64, 64*64, rdi
	bump	rdi, XMM_SCD2		;; Next sin/cos data
	loops	64, b5b			;; Test inner loop counter
	bump	rsi, -64*64+dist128	;; Next source pointer
	bump	rdi, -64*XMM_SCD2	;; Restore sin/cos ptr
	loops_touch 5, b5b, 5, b5a	;; Test loop counter
	bump	rsi, -25*dist128	;; Next source pointer

;; Do FFT levels 7,8
;; Values 0-127 is real data, 128-25599 is complex data.
;;
;; On input the 64-byte cache lines hold these data values:
;;	0	+12.5K	64	+12.5K	128	+12.5K	192	+12.5K
;;	1	...
;;	...
;;	63	...
;;	256	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+12.5K	16	+12.5K	32	+12.5K	48	+12.5K
;;	1	...
;;	...
;;	15	...
;;	64	...
;;	...

;; Do 16 eight_reals_fft operations
;; Do 48 four_complex_djbfft operations
;;	distance between fft data elements is 16
;;	do 16 macros each processing 32 data values

	loops_init_prefetch 800, 64, 2, rcx
	xtouch	[rcx+4096-64]		;; Preload the TLBs
	r4_h4cl_eight_reals_four_complex_djbfft_preload
b6b:	r4_h4cl_eight_reals_four_complex_djbfft rsi, 64, 16*64, 32*64, rdi, rdx
	bump	rdi, 4*XMM_SCD2		;; Next sin/cos data
	bump	rdx, 4*XMM_HSCD2	;; Next sin/cos data
	loops	16, b6b			;; Test loop counter
	bump	rsi, -16*64+64*64	;; Next source pointer
	bump	rdi, -16*4*XMM_SCD2	;; Restore sin/cos ptr
	bump	rdx, -16*4*XMM_HSCD2	;; Restore sin/cos ptr

;; Do 3136 four_complex_djbfft operations
;;	distance between fft data elements is 16
;;	do 784 macros each processing 32 data values

	loops_reset			;; 25 iters of 1 or 2 of 16
	r4_x4cl_four_complex_djbfft_preload
b7a:	xtouch	[rcx+4096-64]		;; Preload the TLBs
	xtouch	[rcx+80*64-64]
b7b:	r4_x4cl_four_complex_djbfft rsi, 64, 16*64, 32*64, rdi
	bump	rdi, 4*XMM_SCD2		;; Next sin/cos data
	loops	16, b7b			;; Test inner loop counter
	bump	rsi, -16*64+64*64	;; Next source pointer
	bump	rdi, -16*4*XMM_SCD2	;; Restore sin/cos ptr
	loops	2, b7b			;; Test loop counter
	bump	rsi, -2*64*64+dist128	;; Next source pointer
	loops_touch 5, b7b, 5, b7a	;; 25 iterations
	bump	rsi, -25*dist128	;; Next source pointer

;; Do FFT levels 9,10
;; Values 0-31 is real data, 32-25599 is complex data.
;;
;; On input the 64-byte cache lines hold these data values:
;;	0	+12.5K	16	+12.5K	32	+12.5K	48	+12.5K
;;	1	...
;;	...
;;	15	...
;;	64	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+12.5K	4	+12.5K	8	+12.5K	12	+12.5K
;;	1	...
;;	...
;;	3	...
;;	16	...
;;	...

;; Do 4 eight_reals_fft operations
;; Do 12 four_complex_djbfft operations
;;	distance between fft data elements is 4
;;	do 4 macros each processing 32 data values

	loops_init_prefetch 800, 64, 2, rcx
	r4_h4cl_eight_reals_four_complex_djbfft_preload
b8b:	r4_h4cl_eight_reals_four_complex_djbfft rsi, 64, 4*64, 8*64, rdi, rdx
	bump	rdi, 16*XMM_SCD2	;; Next sin/cos data
	bump	rdx, 16*XMM_HSCD2	;; Next sin/cos data
	loops	4, b8b			;; Test loop counter
	bump	rsi, -4*64+16*64	;; Next source pointer
	bump	rdi, -4*16*XMM_SCD2	;; Restore sin/cos ptr
	bump	rdx, -4*16*XMM_HSCD2	;; Restore sin/cos ptr

;; Do 3184 four_complex_djbfft operations
;;	distance between fft data elements is 4
;;	do 796 macros each processing 32 data values

	loops_reset			;; 25 iters of 7 or 8 of 4
	r4_x4cl_four_complex_djbfft_preload
b9a:	xtouch	[rcx+4096-64]		;; Preload the TLBs
	xtouch	[rcx+80*64-64]
b9b:	r4_x4cl_four_complex_djbfft rsi, 64, 4*64, 8*64, rdi
	bump	rdi, 16*XMM_SCD2	;; Next sin/cos data
	loops	4, b9b			;; Test loop counter
	bump	rsi, -4*64+16*64	;; Next source pointer
	bump	rdi, -4*16*XMM_SCD2	;; Restore sin/cos ptr
	loops	8, b9b			;; Test middle loop counter
	bump	rsi, -8*16*64+dist128	;; Next source pointer
	loops_touch 5, b9b, 5, b9a	;; 25 iterations
	bump	rsi, -25*dist128	;; Restore source pointer

;; Do FFT levels 11,12
;; Values 0-7 is real data, 8-25599 is complex data.
;;
;; On input the 64-byte cache lines hold these data values:
;;	0	+12.5K	4	+12.5K	8	+12.5K	12	+12.5K
;;	1	...
;;	...
;;	3	...
;;	16	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+12.5K	2	+12.5K	4	+12.5K	6	+12.5K
;;	1	...
;;	8	...
;;	...

;; Do 1 eight_reals_fft operations
;; Do 3 four_complex_fft operations
;;	distance between fft data elements is 1

;; Do 3196 four_complex_fft operations
;;	distance between fft data elements is 1
;;	do 799 macros each processing 32 data values

;; Execute the proper middle step

	cmp	ffttype, 2
	jl	xpass2_6400_levels_real_1
	jg	xpass2_6400_levels_real_3

xpass2_6400_levels_real_2:
	r4_h4cl_eight_reals_four_complex_with_square rsi, 4*64, 64, 2*64
	loops_init_prefetch 800, 64, 16, rcx, 1, 32
	xtouch	[rcx+4096-64]		;; Preload the TLBs
	r4_x4cl_four_complex_with_square_preload
baa:	r4_x4cl_four_complex_with_square rsi, 4*64, 64, 2*64
	loops	32, baa			;; Test loop counter
	bump	rsi, -32*4*64+dist128	;; Next source pointer
	loops	25, baa			;; Test loop counter
	bump	rsi, -25*dist128	;; Next source pointer
	jmp	xpass2_6400_real_unfft

xpass2_6400_levels_real_1:
	r4_h4cl_eight_reals_four_complex_fft_final rsi, 4*64, 64, 2*64
	loops_init_prefetch 800, 128, 1, rcx, 1, 32
bab0:	xtouch	[rcx+4096-64]		;; Preload the TLBs
bab:	r4_x4cl_four_complex_fft_final rsi, 4*64, 64, 2*64
	loops	32, bab			;; Test loop counter
	bump	rsi, -32*4*64+dist128	;; Next source pointer
	loops	25, bab0		;; Test loop counter
	jmp	xpass2_6400_real_done

xpass2_6400_levels_real_3:
	mov	rbp, DIST_TO_MULSRCARG
	r4_h4cl_eight_reals_four_complex_with_mult rsi, 4*64, 64, 2*64
	loops_init_prefetch 800, 64, 16, rcx, 1, 32
	xtouch	[rcx+4096-64]		;; Preload the TLBs
bac:	r4_x4cl_four_complex_with_mult rsi, 4*64, 64, 2*64
	loops	32, bac			;; Test loop counter
	bump	rsi, -32*4*64+dist128	;; Next source pointer
	loops	25, bac			;; Test loop counter
	bump	rsi, -25*dist128	;; Next source pointer
	jmp	xpass2_6400_real_unfft

xpass2_6400_levels_real_4:
	mov	rbp, DIST_TO_MULSRCARG
	mov	rcx, DATA_PREFETCH	;; Load prefetch pointer
	xtouch	[rcx]			;; Preload first TLB
	r4_h4cl_eight_reals_four_complex_with_mulf rsi, 4*64, 64, 2*64
	loops_init_prefetch 800, 128, 1, rcx, 1, 32
bad0:	xtouch	[rcx+4096-64]		;; Preload the TLBs
bad:	r4_x4cl_four_complex_with_mulf rsi, 4*64, 64, 2*64
	loops	32, bad			;; Test loop counter
	bump	rsi, -32*4*64+dist128	;; Next source pointer
	loops	25, bad0		;; Test loop counter
	bump	rsi, -25*dist128	;; Next source pointer
	bump	rcx, -28*64		;; Backup the prefetch pointer

;; Do inverse FFT levels 9,10
;; On input the 64-byte cache lines hold these data values:
;;	0	+12.5K	2	+12.5K	4	+12.5K	6	+12.5K
;;	1	...
;;	8	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+12.5K	8	+12.5K	16	+12.5K	24	+12.5K
;;	1	...
;;	...
;;	7	...
;;	32	...
;;	...

;; Do 4 eight_reals_unfft operations
;; Do 4 four_complex_djbunfft operations
;;	distance between fft data elements is 4
;;	do 2 macros processing 32 data values

xpass2_6400_real_unfft:
	mov	rdi, xsincos_complex	;; Load sin/cos pointers
	bump	rdi, 1280*XMM_SCD2
	mov	rdx, sincos3
	bump	rdx, 1280*XMM_HSCD2
	loops_init_prefetch 800, 64, 2, rcx
c8b:	r4_h4cl_eight_reals_four_complex_djbunfft rsi, 64, 2*64, 4*64, rdi, 2*16*XMM_SCD2, rdx, 2*16*XMM_HSCD2
	bump	rdi, 16*XMM_SCD2	;; Next sin/cos data
	bump	rdx, 16*XMM_HSCD2	;; Next sin/cos data
	loops	2, c8b			;; Test loop counter
	bump	rsi, -2*64+8*64		;; Next source pointer
	bump	rdi, -2*16*XMM_SCD2	;; Restore sin/cos ptr
	bump	rdx, -2*16*XMM_HSCD2	;; Restore sin/cos ptr

;; Do 3192 four_complex_djbunfft operations
;;	distance between fft data elements is 4
;;	do 798 macros each processing 32 data values

	loops_reset			;; 20 iters of 15 or 16 of 2
	r4_x4cl_four_complex_djbunfft_preload
c9a:	xtouch	[rcx+4096-64]		;; Preload the TLBs
	xtouch	[rcx+80*64-64]
c9b:	r4_x4cl_four_complex_djbunfft rsi, 64, 2*64, 4*64, rdi, 2*16*XMM_SCD2
	bump	rdi, 16*XMM_SCD2	;; Next sine/cosine pointer
	loops	2, c9b			;; Test loop counter
	bump	rsi, -2*64+8*64		;; Next source pointer
	bump	rdi, -2*16*XMM_SCD2	;; Restore sin/cos ptr
	loops	16, c9b			;; Test loop counter
	bump	rsi, -16*8*64+dist128	;; Next source pointer
	loops_touch 5, c9b, 5, c9a	;; Test loop counter
	bump	rsi, -25*dist128	;; Next source pointer

;; Do inverse FFT levels 7,8
;; On input the 64-byte cache lines hold these data values:
;;	0	+12.5K	8	+12.5K	16	+12.5K	24	+12.5K
;;	1	...
;;	...
;;	7	...
;;	32	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+12.5K	32	+12.5K	64	+12.5K	96	+12.5K
;;	1	...
;;	...
;;	31	...
;;	128	...
;;	...

;; Do 16 eight_reals_unfft operations
;; Do 16 four_complex_djbunfft operations
;;	distance between fft data elements is 16
;;	do 8 macros processing 32 data values

	loops_init_prefetch 800, 64, 2, rcx
c6b:	r4_h4cl_eight_reals_four_complex_djbunfft rsi, 64, 8*64, 16*64, rdi, 8*4*XMM_SCD2, rdx, 8*4*XMM_HSCD2
	bump	rdi, 4*XMM_SCD2		;; Next sin/cos data
	bump	rdx, 4*XMM_HSCD2	;; Next sin/cos data
	loops	8, c6b			;; Test inner loop counter
	bump	rsi, -8*64+32*64	;; Next source pointer
	bump	rdi, -8*4*XMM_SCD2	;; Restore sin/cos ptr
	bump	rdx, -8*4*XMM_HSCD2	;; Restore sin/cos ptr

;; Do 3168 four_complex_djbunfft operations
;;	distance between fft data elements is 16
;;	do 792 macros each processing 32 data values

	loops_reset			;; 25 iters of 3 or 4 of 4
	r4_x4cl_four_complex_djbunfft_preload
c7a:	xtouch	[rcx+4096-64]		;; Preload the TLBs
	xtouch	[rcx+80*64-64]
c7b:	r4_x4cl_four_complex_djbunfft rsi, 64, 8*64, 16*64, rdi, 8*4*XMM_SCD2
	bump	rdi, 4*XMM_SCD2		;; Next sin/cos data
	loops	8, c7b			;; Test inner loop counter
	bump	rsi, -8*64+32*64	;; Next source pointer
	bump	rdi, -8*4*XMM_SCD2	;; Restore sin/cos ptr
	loops	4, c7b			;; Test loop counter
	bump	rsi, -4*32*64+dist128	;; Next source pointer
	loops_touch 5, c7b, 5, c7a	;; 25 iterations
	bump	rsi, -25*dist128	;; Next source pointer

;; Do inverse FFT levels 5,6
;; On input the 64-byte cache lines hold these data values:
;;	0	+12.5K	32	+12.5K	64	+12.5K	96	+12.5K
;;	1	...
;;	...
;;	31	...
;;	128	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+12.5K	128	+12.5K	256	+12.5K	384	+12.5K
;;	1	...
;;	...
;;	127	...
;;	512	...
;;	...

;; Do 64 eight_reals_unfft operations
;; Do 64 four_complex_djbunfft operations
;;	distance between fft data elements is 64
;;	do 32 macros processing 32 data values

	loops_init_prefetch 800, 64, 2, rcx
	xtouch	[rcx+4096-64]		;; Preload the TLBs
c4b:	r4_h4cl_eight_reals_four_complex_djbunfft rsi, 64, 32*64, 64*64, rdi, 32*XMM_SCD2, rdx, 32*XMM_HSCD2
	bump	rdi, XMM_SCD2		;; Next sin/cos data
	bump	rdx, XMM_HSCD2		;; Next sin/cos data
	loops	32, c4b			;; Test inner loop counter
	bump	rsi, -32*64+dist128	;; Next source pointer
	bump	rdi, -32*XMM_SCD2	;; Restore sin/cos ptr
	bump	rdx, -32*XMM_HSCD2-1280*XMM_HSCD2 ;; Restore sin/cos ptr

;; Do 3072 four_complex_djbunfft operations
;;	distance between fft data elements is 64
;;	do 768 macros each processing 32 data values

	loops_reset			;; 24 more iters of 32
	r4_x4cl_four_complex_djbunfft_preload
c5a:	xtouch	[rcx+4096-64]		;; Preload the TLBs
	xtouch	[rcx+80*64-64]
c5b:	r4_x4cl_four_complex_djbunfft rsi, 64, 32*64, 64*64, rdi, 32*XMM_SCD2
	bump	rdi, XMM_SCD2		;; Next sin/cos data
	loops	32, c5b			;; Test inner loop counter
	bump	rsi, -32*64+dist128	;; Next source pointer
	bump	rdi, -32*XMM_SCD2	;; Restore sin/cos ptr
	loops_touch 5, c5b, 5, c5a	;; Test loop counter
	bump	rsi, -25*dist128	;; Restore source pointer

;; Do inverse FFT levels 3,4 (actually 2.322 FFT levels)
;; On input the 64-byte cache lines hold these data values:
;;	0	+12.5K	128	+12.5K	256	+12.5K	384	+12.5K
;;	1	...
;;	...
;;	127	...
;;	512	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+12.5K	640	+12.5K	1280	+12.5K	1920	+12.5K
;;	1	...
;;	...
;;	639	...
;;	2560	...
;;	...

;; Do 256 ten_reals_unfft operations
;; Do 256 five_complex_djbunfft operations
;;	distance between fft data elements is 256
;;	do 128 macros processing 40 data values

	mov	rdi, xsincos_complex	;; Load sin/cos pointer
	loops_init_prefetch 640, 64, 2, rcx
	xtouch	[rcx+4096-64]		;; Preload the TLBs
c2b:	r5_h5cl_ten_reals_five_complex_djbunfft rsi, 64, dist128, rdi, 128*5*XMM_SCD2, rdx, 128*5*XMM_HSCD2
	bump	rdi, 5*XMM_SCD2		;; Next sin/cos data
	bump	rdx, 5*XMM_HSCD2	;; Next sin/cos data
	loops	128, c2b		;; Test inner loop counter
	bump	rsi, -128*64+5*dist128	;; Next source pointer
	bump	rdi, -128*5*XMM_SCD2	;; Restore sin/cos ptr
	bump	rdx, -128*5*XMM_HSCD2	;; Restore sin/cos ptr

;; Do 2048 five_complex_djbunfft operations
;;	distance between fft data elements is 256
;;	do 512 macros each processing 40 data values

	loops_reset			;; 4 iters of 128
	r5_x5cl_five_complex_djbunfft_preload
c3a:	xtouch	[rcx+4096-64]		;; Preload the TLBs
c3b:	r5_x5cl_five_complex_djbunfft rsi, 64, dist128, rdi, 128*5*XMM_SCD2
	bump	rdi, 5*XMM_SCD2		;; Next sin/cos data
	loops	128, c3b		;; Test inner loop counter
	bump	rsi, -128*64+5*dist128	;; Next source pointer
	bump	rdi, -128*5*XMM_SCD2	;; Restore sin/cos ptr
	loops	5, c3a			;; Test loop counter
	bump	rsi, -5*5*dist128	;; Next source pointer

;; Do inverse FFT level 1,2 (actually 2.322 FFT levels)
;; On input the 64-byte cache lines hold these data values:
;;	0	+12.5K	640	+12.5K	1280	+12.5K	1920	+12.5K
;;	1	...
;;	...
;;	639	...
;;	2560	...
;;	...
;; On output the 64-byte cache lines hold these data values if pass 1 swizzles:
;;	0	+12.5K	3200	+12.5K	6400	+12.5K	9600	+12.5K
;;	1	...
;;	...
;;	3199
;; On output the 64-byte cache lines hold these data values if pass 2 swizzles:
;;	0	+3.2K	6400	+3.2K	12800	+3.2K	19200	+3.2K
;;	1	...
;;	...
;;	3199

;; Do 1280 ten_reals_unfft operations
;; Do 1280 five_complex_djbunfft operations
;;	distance between fft data elements is 1280
;;	do 640 macros processing 40 data values

	bump	rcx, -2*64		;; Back up prefetch pointer by 2 cache lines
	loops_init_prefetch 640, 64, 4, rcx
	xtouch	[rcx+4096-64]		;; Preload the TLBs
	xtouch	[rcx+2*4096-64]
	xtouch	[rcx+160*64-64]
c1b:	r5_h5cl_ten_reals_five_complex_djbunfft rsi, 64, 5*dist128, rdi, 640*XMM_SCD2, rdx, 640*XMM_HSCD2
	bump	rdi, XMM_SCD2		;; Next sin/cos data
	bump	rdx, XMM_HSCD2		;; Next sin/cos data
	loops	128, c1b		;; Test loop counter
	bump	rsi, -128*64+dist128	;; Next source pointer
	loops	5, c1b			;; Test loop counter

xpass2_6400_real_done:
	end_timer 2
	ENDM

; *************** radix4-pass2-6400-levels-complex macro ******************
; This macro takes 2 groups of 6400 complex values and
; performs the final 12.644 levels of a standard mostly radix-4 FFT, squares the
; results, and does the first 12.644 levels of the inverse FFT.

r4_pass2_6400_levels_complex MACRO
	LOCAL	b1a, b1b, b2a, b2b, b3a, b3b, b4b, b5b, b6a, b6b0, b6b, b6c, b6d0, b6d
	LOCAL	c1b, c2a, c2b, c3b, c4b, c5a, c5b
	LOCAL	xpass2_6400_levels_complex_1, xpass2_6400_levels_complex_2
	LOCAL	xpass2_6400_levels_complex_3, xpass2_6400_levels_complex_4
	LOCAL	xpass2_6400_levels_complex_unfft, xpass2_6400_levels_complex_done

;; Type 4 FFTs skip the forward FFT process

	mov	rsi, DATA_ADDR		;; Load address of FFT data
	mov	rbx, DIST_TO_FFTSRCARG
	cmp	ffttype, 4
	je	xpass2_6400_levels_complex_4

;; Do FFT levels 1,2 (actually 2.322 levels)
;;
;; We swizzle the data so that one set of data is in the high word of
;; an xmm register, the other set is in the low word.  This means
;; a distance of 12800 data values.
;;
;; On input the 64-byte cache lines hold these data values if pass 2 swizzles:
;;	0	+3.2K	6400	+3.2K	12800	+3.2K	19200	+3.2K
;;	1	...
;;	...
;;	3199
;; On input the 64-byte cache lines hold these data values if pass 1 swizzles:
;;	0	+12.5K	3200	+12.5K	6400	+12.5K	9600	+12.5K
;;	1	...
;;	...
;;	3199	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+12.5K	640	+12.5K	1280	+12.5K	1920	+12.5K
;;	1	...
;;	...
;;	639	...
;;	2560	...
;;	...

;; Do 2560 five_complex_fft operations
;;	distance between fft data elements is 1280
;;	do 640 macros each processing 40 data values

;; Do FFT levels 3,4 (actually 2.322 levels)
;;
;; On input the 64-byte cache lines hold these data values:
;;	0	+12.5K	640	+12.5K	1280	+12.5K	1920	+12.5K
;;	1	...
;;	...
;;	639	...
;;	2560	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+12.5K	128	+12.5K	256	+12.5K	384	+12.5K
;;	1	...
;;	...
;;	127	...
;;	512	...
;;	...

;; Do 2560 five_complex_fft operations
;;	distance between fft data elements is 256
;;	do 640 macros each processing 40 data values

IF @INSTR(,%xarch,<CORE>) NE 0		;; Core 2 faster doing separate loops

	start_timer 4
	mov	rdi, xsincos_complex	;; The sin/cos data
	loops_init 640			;; 5 iterations of 128
	r5_f5cl_five_complex_djbfft_preload
b1b:	r5_f5cl_five_complex_djbfft rsi, 64, 5*dist128, rdi, 640*XMM_SCD2
	bump	rdi, XMM_SCD2		;; Next sin/cos data
	loops	128, b1b		;; Test inner loop counter
	bump	rsi, -128*64+dist128	;; Next source pointer
	loops	5, b1b			;; Test outer loop counter
	bump	rsi, -5*dist128		;; Restore source pointer
	end_timer 4

	start_timer 5
	mov	rdi, xsincos_complex	;; Ptr to the sin/cos data
	mov	rcx, DATA_PREFETCH	;; Prefetch pointer (we need to prefetch
					;; 3200 cache lines + 48 pad cache lines)
	xtouch	[rcx]			;; Load first TLB
	loops_init_prefetch 640, 64, 2, rcx ;; 5 iters of 128
	r5_nf5cl_five_complex_djbfft_preload
b2a:	xtouch	[rcx+4096-64]		;; Preload the TLBs
b2b:	r5_nf5cl_five_complex_djbfft rsi, 64, dist128, rdi, 128*5*XMM_SCD2
	bump	rdi, 5*XMM_SCD2		;; Next sin/cos data
	loops	128, b2b		;; Test inner loop counter
	bump	rsi, -128*64+5*dist128	;; Next source pointer
	bump	rdi, -128*5*XMM_SCD2	;; Restore sin/cos ptr
	loops	5, b2a			;; Test loop counter
	bump	rsi, -5*5*dist128	;; Restore source pointer
	end_timer 5

ELSE					;; Pentium 4 faster doing merged loops

	start_timer 4
	mov	rdi, xsincos_complex	;; The sin/cos data
	mov	rbp, rdi
	mov	rcx, DATA_PREFETCH	;; Prefetch pointer (we need to prefetch
					;; 3200 cache lines + 48 pad cache lines)
	xtouch	[rcx]			;; Load first TLB
	loops_init_prefetch 640, 64, 5, rcx ;; 128 iters of 5, prefetching 128 cache lines in first loop

	r5_f5cl_five_complex_djbfft_preload
;;	r5_nf5cl_five_complex_djbfft_preload	;; Dangerous - assume same as r5_f5cl_five_complex_djbfft_preload

b1a:	xtouch	[rcx+4096-64]		;; Preload the TLBs
	xtouch	[rcx+2*4096-64]
	xtouch	[rcx+3*4096-64]

b1b:	r5_f5cl_five_complex_djbfft rsi, dist128, 5*dist128, rdi, 640*XMM_SCD2
	xxprefetch [rdi+XMM_SCD2]	;; Prefetch for next pass
	xxprefetch [rdi+641*XMM_SCD2]	;; Prefetch for next pass
	bump	rdi, 128*XMM_SCD2	;; Next sin/cos ptr
	loops	5, b1b			;; Test loop counter
	bump	rsi, -5*dist128		;; Restore source pointer
	loops_undo 5

	loops_set_prefetch 128, 5, rcx	;; prefetching 256 cache lines in second loop
b2b:	r5_nf5cl_five_complex_djbfft rsi, 5*dist128, dist128, rbp, 128*5*XMM_SCD2
	loops	5, b2b			;; Test loop counter

	bump	rsi, -5*5*dist128+64	;; Next source pointer
	bump	rdi, -5*128*XMM_SCD2+XMM_SCD2 ;; Next sin/cos ptr
	bump	rbp, 5*XMM_SCD2		;; Next sin/cos data
	loops_touch 64, b1b, 2, b1a	;; Loop 128 times
	bump	rsi, -128*64		;; Restore source pointer
	bump	rdi, -128*XMM_SCD2	;; Restore sin/cos ptr
	end_timer 4

ENDIF

;; Do FFT levels 5,6
;;
;; On input the 64-byte cache lines hold these data values:
;;	0	+12.5K	128	+12.5K	256	+12.5K	384	+12.5K
;;	1	...
;;	...
;;	127	...
;;	512	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+12.5K	64	+12.5K	128	+12.5K	192	+12.5K
;;	1	...
;;	...
;;	63	...
;;	256	...
;;	...
;;
;; Do 3200 four_complex_fft operations
;;	distance between fft data elements is 64
;;	do 1600 macros each processing 16 data values

;; Do FFT levels 7,8
;;
;; On input the 64-byte cache lines hold these data values:
;;	0	+12.5K	64	+12.5K	128	+12.5K	192	+12.5K
;;	1	...
;;	...
;;	63	...
;;	256	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+12.5K	16	+12.5K	32	+12.5K	48	+12.5K
;;	1	...
;;	...
;;	15	...
;;	64	...
;;	...
;;
;; Do 3200 four_complex_fft operations
;;	distance between fft data elements is 16
;;	do 800 macros each processing 32 data values

;; Do FFT levels 9,10
;;
;; On input the 64-byte cache lines hold these data values:
;;	0	+12.5K	16	+12.5K	32	+12.5K	48	+12.5K
;;	1	...
;;	...
;;	15	...
;;	64	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+12.5K	4	+12.5K	8	+12.5K	12	+12.5K
;;	1	...
;;	...
;;	3	...
;;	16	...
;;	...
;;
;; Do 3200 four_complex_fft operations
;;	distance between fft data elements is 4
;;	do 800 macros each processing 32 data values

;; Do this 32 cache lines at a time to maximize accessing data
;; while it is in the L1 cache.

	start_timer 6
	bump	rdi, 1280*XMM_SCD2	;; Load sin/cos pointers
	mov	rbx, rdi
	mov	rbp, rdi
	loops_init_prefetch 1600, 64, 4, rcx ;; 25 iters of 4 of 32 cache lines
	;r4_nf2cl_four_complex_djbfft_preload -- assume same as r4_x4cl_four_complex_djbfft_preload
	r4_x4cl_four_complex_djbfft_preload

b3a:	xtouch	[rcx+3072-64]		;; Load prefetch TLB

b3b:	xxprefetch [rdi+4*XMM_SCD2]	;; Prefetch for next pass
	r4_nf2cl_four_complex_djbfft rsi, 4*64, 64*64, rdi
	bump	rdi, 4*XMM_SCD2		;; Next sine/cosine pointer
	loops	16, b3b			;; Loop 16 times
	bump	rsi, -16*4*64		;; Restore source pointer
	bump	rdi, -16*4*XMM_SCD2	;; Restore sine/cosine pointer
	loops_undo 16			;; Back up loop counter

b4b:	r4_x4cl_four_complex_djbfft rsi, 4*64, 16*64, 32*64, rbx
	bump	rbx, 4*4*XMM_SCD2	;; Next sine/cosine pointer
	loops	4, b4b, 2		;; 4 iterations with non-standard increment
	bump	rsi, -4*4*64+64*64	;; Next source pointer
	bump	rbx, -4*4*4*XMM_SCD2	;; Restore sine/cosine pointer
	loops	2, b4b			;; Loop twice
	bump	rsi, -2*64*64		;; Restore source pointer
	loops_undo 2*4*2		;; Back up loop counter

b5b:	r4_x4cl_four_complex_djbfft rsi, 16*64, 4*64, 8*64, rbp
	loops	8, b5b, 2		;; 8 iterations with non-standard increment

	bump	rsi, -8*16*64+64	;; Next source pointer
	bump	rdi, XMM_SCD2		;; Next sine/cosine pointer
	bump	rbx, 4*XMM_SCD2		;; Next sine/cosine pointer
	bump	rbp, 16*XMM_SCD2	;; Next sine/cosine pointer
	loops	4, b3b			;; Loop 4 times
	bump	rsi, -4*64+dist128	;; Next source pointer
	bump	rdi, -4*XMM_SCD2	;; Restore sine/cosine pointer
	bump	rbx, -4*4*XMM_SCD2	;; Restore sine/cosine pointer
	bump	rbp, -4*16*XMM_SCD2	;; Restore sine/cosine pointer
	loops	25, b3a			;; Loop 25 times
	bump	rsi, -25*dist128	;; Restore source pointer
	end_timer 6

;; Do FFT levels 11,12 as well as inverse FFT levels 11,12
;;
;; On input the 64-byte cache lines hold these data values:
;;	0	+12.5K	4	+12.5K	8	+12.5K	12	+12.5K
;;	1	...
;;	...
;;	3	...
;;	16	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+12.5K	2	+12.5K	4	+12.5K	6	+12.5K
;;	1	...
;;	8	...
;;	...

;; Do 3200 four_complex_fft operations
;;	distance between fft data elements is 1
;;	do 800 macros each processing 32 data values

;; Execute the right middle step

	cmp	ffttype, 2
	jl	xpass2_6400_levels_complex_1
	jg	xpass2_6400_levels_complex_3

xpass2_6400_levels_complex_2:
	start_timer 9
	loops_init_prefetch 800, 64, 16, rcx
	xtouch	[rcx+4096-64]		;; Load prefetch TLBs
	r4_x4cl_four_complex_with_square_preload
b6a:	r4_x4cl_four_complex_with_square rsi, 4*64, 64, 2*64
	loops	32, b6a			;; Loop 32 times
	bump	rsi, -32*4*64+dist128	;; Next source pointer
	loops	25, b6a			;; Loop 25 times
	bump	rsi, -25*dist128	;; Restore source pointer
	end_timer 9
	jmp	xpass2_6400_levels_complex_unfft

xpass2_6400_levels_complex_1:
	start_timer 9
	loops_init_prefetch 800, 128, 1, rcx
b6b0:	xtouch	[rcx+4096-64]		;; Load prefetch TLBs
b6b:	r4_x4cl_four_complex_fft_final rsi, 4*64, 64, 2*64
	loops	32, b6b			;; Loop 32 times
	bump	rsi, -32*4*64+dist128	;; Next source pointer
	loops	25, b6b0		;; Loop 25 times
	end_timer 9
	jmp	xpass2_6400_levels_complex_done

xpass2_6400_levels_complex_3:
	start_timer 9
	mov	rbp, DIST_TO_MULSRCARG
	loops_init_prefetch 800, 64, 16, rcx
	xtouch	[rcx+4096-64]		;; Load prefetch TLBs
b6c:	r4_x4cl_four_complex_with_mult rsi, 4*64, 64, 2*64
	loops	32, b6c			;; Loop 32 times
	bump	rsi, -32*4*64+dist128	;; Next source pointer
	loops	25, b6c			;; Loop 25 times
	bump	rsi, -25*dist128	;; Restore source pointer
	end_timer 9
	jmp	xpass2_6400_levels_complex_unfft

xpass2_6400_levels_complex_4:
	start_timer 9
	mov	rcx, DATA_PREFETCH	;; Prefetch pointer
	xtouch	[rcx]			;; Load first prefetch TLB
	mov	rbp, DIST_TO_MULSRCARG
	loops_init_prefetch 800, 128, 1, rcx
b6d0:	xtouch	[rcx+4096-64]		;; Load prefetch TLBs
b6d:	r4_x4cl_four_complex_with_mulf rsi, 4*64, 64, 2*64
	loops	32, b6d			;; Loop 32 times
	bump	rsi, -32*4*64+dist128	;; Next source pointer
	loops	25, b6d			;; Loop 25 times
	bump	rsi, -25*dist128	;; Restore source pointer
	bump	rcx, -30*64		;; Back up prefetch pointer
	end_timer 9

;; Do inverse FFT levels 9,10
;; On input the 64-byte cache lines hold these data values:
;;	0	+12.5K	2	+12.5K	4	+12.5K	6	+12.5K
;;	1	...
;;	8	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+12.5K	8	+12.5K	16	+12.5K	24	+12.5K
;;	1	...
;;	...
;;	7	...
;;	32	...
;;	...
;;
;; Do 3200 four_complex_unfft operations
;;	distance between fft data elements is 4
;;	do 800 macros each processing 32 data values

;; Do inverse FFT levels 7,8
;; On input the 64-byte cache lines hold these data values:
;;	0	+12.5K	8	+12.5K	16	+12.5K	24	+12.5K
;;	1	...
;;	...
;;	7	...
;;	32	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+12.5K	32	+12.5K	64	+12.5K	96	+12.5K
;;	1	...
;;	...
;;	31	...
;;	128	...
;;	...
;;
;; Do 3200 four_complex_unfft operations
;;	distance between fft data elements is 16
;;	do 800 macros each processing 32 data values

;; Do inverse FFT levels 5,6
;; On input the 64-byte cache lines hold these data values:
;;	0	+12.5K	32	+12.5K	64	+12.5K	96	+12.5K
;;	1	...
;;	...
;;	31	...
;;	128	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+12.5K	128	+12.5K	256	+12.5K	384	+12.5K
;;	1	...
;;	...
;;	127	...
;;	512	...
;;	...
;;
;; Do 3200 four_complex_unfft operations
;;	distance between fft data elements is 64
;;	do 800 macros each processing 32 data values

;; Do this 64 cache lines at a time to maximize accessing data
;; while it is in the L1 cache.

xpass2_6400_levels_complex_unfft:
	start_timer 10
	mov	rdi, xsincos_complex	;; Load sin/cos pointers
	bump	rdi, 1280*XMM_SCD2
	mov	rbx, rdi
	mov	rbp, rdi
	loops_init_prefetch 800, 64, 2, rcx ;; 25*2 iters of 64 cache lines
	r4_x4cl_four_complex_djbunfft_preload

c5a:	xtouch	[rcx+3072-64]		;; Load prefetch TLB

c5b:	r4_x4cl_four_complex_djbunfft rsi, 8*64, 2*64, 4*64, rbp, 2*16*XMM_SCD2
	loops	16, c5b			;; Loop 16 times
	bump	rsi, -16*8*64		;; Restore source pointer
	loops_undo 16			;; Restore loop counter

c4b:	r4_x4cl_four_complex_djbunfft rsi, 2*64, 8*64, 16*64, rbx, 8*4*XMM_SCD2
	bump	rbx, 2*4*XMM_SCD2	;; Next sine/cosine pointer
	loops	4, c4b			;; Loop 4 times
	bump	rsi, -4*2*64+32*64	;; Next source pointer
	bump	rbx, -4*2*4*XMM_SCD2	;; Restore sine/cosine pointer
	loops	4, c4b			;; Loop 4 times
	bump	rsi, -4*32*64		;; Restore source pointer
	loops_undo 4*4			;; Restore loop counter

c3b:	r4_x4cl_four_complex_djbunfft rsi, 2*64, 32*64, 64*64, rdi, 32*1*XMM_SCD2
	bump	rdi, 2*XMM_SCD2		;; Next sine/cosine pointer
	loops	16, c3b			;; Loop 16 times
	bump	rdi, -16*2*XMM_SCD2	;; Restore sine/cosine pointer

	bump	rsi, -16*2*64+64	;; Next source pointer
	bump	rdi, XMM_SCD2		;; Next sine/cosine pointer
	bump	rbx, 4*XMM_SCD2		;; Next sine/cosine pointer
	bump	rbp, 16*XMM_SCD2	;; Next sine/cosine pointer
	loops	2, c5b			;; Loop twice
	bump	rsi, -2*64+dist128	;; Next source pointer
	bump	rdi, -2*XMM_SCD2	;; Restore sine/cosine pointer
	bump	rbx, -2*4*XMM_SCD2	;; Restore sine/cosine pointer
	bump	rbp, -2*16*XMM_SCD2	;; Restore sine/cosine pointer
	loops	25, c5a			;; Loop 25 times
	bump	rsi, -25*dist128	;; Restore source pointer
	end_timer 10

;; Do inverse FFT levels 3,4 (actually 2.322 levels)
;; On input the 64-byte cache lines hold these data values:
;;	0	+12.5K	128	+12.5K	256	+12.5K	384	+12.5K
;;	1	...
;;	...
;;	127	...
;;	512	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+12.5K	640	+12.5K	1280	+12.5K	1920	+12.5K
;;	1	...
;;	...
;;	639	...
;;	2560	...
;;	...

;; Do 2560 five_complex_unfft operations
;;	distance between fft data elements is 256
;;	do 640 macros each processing 40 data values

;; Do inverse FFT levels 1,2 (actually 2.322 FFT levels)
;; On input the 64-byte cache lines hold these data values:
;;	0	+12.5K	640	+12.5K	1280	+12.5K	1920	+12.5K
;;	1	...
;;	...
;;	639	...
;;	2560	...
;;	...
;; On output the 64-byte cache lines hold these data values if pass 1 swizzles:
;;	0	+12.5K	3200	+12.5K	6400	+12.5K	9600	+12.5K
;;	1	...
;;	...
;;	3199
;; On output the 64-byte cache lines hold these data values if pass 2 swizzles:
;;	0	+3.2K	6400	+3.2K	12800	+3.2K	19200	+3.2K
;;	1	...
;;	...
;;	3199

;; Do 2560 five_complex_unfft operations
;;	distance between fft data elements is 1280
;;	do 640 macros each processing 40 data values

IF @INSTR(,%xarch,<CORE>) NE 0		;; Core 2 faster doing separate loops

	start_timer 13
	mov	rdi, xsincos_complex	;; Load sin/cos pointer
	loops_init_prefetch 640, 64, 2, rcx;; 5 iters of 128
	r5_x5cl_five_complex_djbunfft_preload
c2a:	xtouch	[rcx+4096-128]		;; Load prefetch TLBs
c2b:	r5_x5cl_five_complex_djbunfft rsi, 64, dist128, rdi, 128*5*XMM_SCD2
	bump	rdi, 5*XMM_SCD2		;; Next sine/cosine pointer
	loops	128, c2b		;; Loop 128 times
	bump	rsi, -128*64+5*dist128	;; Next source pointer
	bump	rdi, -128*5*XMM_SCD2	;; Reset sine/cosine pointer
	loops	5, c2a			;; Test loop counter
	bump	rsi, -5*5*dist128	;; Restore source pointer
	end_timer 13

	start_timer 14
	mov	rdi, xsincos_complex	;; Load sin/cos pointer
	bump	rcx, -2*64		;; Back up prefetch pointer by 2 cache lines
	loops_init_prefetch 640, 64, 4, rcx;; 5 iters of 128
	xtouch	[rcx+4096-64]		;; Load prefetch TLBs
	xtouch	[rcx+2*4096-64]
	xtouch	[rcx+160*64-64]
	r5_x5cl_five_complex_djbunfft_preload
c1b:	r5_x5cl_five_complex_djbunfft rsi, 64, 5*dist128, rdi, 640*XMM_SCD2
	bump	rdi, XMM_SCD2		;; Next set of sin/cos data
	loops	128, c1b		;; Test inner loop counter
	bump	rsi, -128*64+dist128	;; Next source pointer
	loops	5, c1b			;; Test outer loop counter
	end_timer 14

ELSE					;; Pentium 4 faster doing merged loops

	start_timer 13
	mov	rdi, xsincos_complex	;; Load sin/cos pointer
	mov	rbp, rdi
	bump	rcx, -98*64		;; Back up prefetch pointer by 98 cache lines
	loops_init_prefetch 640, 128, 5, rcx ;; 128 iters of 5, prefetching 256 cache lines in each loop

	r5_x5cl_five_complex_djbunfft_preload

c2a:	xtouch	[rcx+4096-64]		;; Load prefetch TLBs

c2b:	r5_x5cl_five_complex_djbunfft rsi, 5*dist128, dist128, rbp, 128*5*XMM_SCD2
	loops	5, c2b			;; Test loop counter
	bump	rsi, -5*5*dist128	;; Restore source pointer
	loops_undo 5

c1b:	r5_x5cl_five_complex_djbunfft rsi, dist128, 5*dist128, rdi, 640*XMM_SCD2
	bump	rdi, 128*XMM_SCD2	;; Next sine/cosine pointer
	loops	5, c1b			;; Test loop counter

	bump	rsi, -5*dist128+64	;; Next source pointer
	bump	rdi, -5*128*XMM_SCD2+XMM_SCD2 ;; Next sin/cos ptr
	bump	rbp, 5*XMM_SCD2		;; Next sine/cosine pointer
	loops_touch 16, c2b, 8, c2a	;; Loop 128 times
	end_timer 13

ENDIF

xpass2_6400_levels_complex_done:
	ENDM



;*********************************************************************************************
;			Macros derived from pass2-13-levels
;*********************************************************************************************


; *************** radix4-pass2-4608-levels-real macro ******************
; This macro takes 9216 real values and performs the final 13.17 levels of
; the FFT.  Independently, this macro also takes 4608 complex values and
; performs the final 12.17 levels of the FFT.  The results are squared, and
; the first 12.17/13.17 levels of the inverse FFT are done.

r4_pass2_4608_levels_real MACRO
	LOCAL	b1b, b2b, b3b, b4b, b5b, b6b, b7b
	LOCAL	b8b, b9b, baa, bab0, bab, bac, bad0, bad
	LOCAL	c1b, c2b, c3b, c4b, c5b, c6b, c7b, c8b, c9b
	LOCAL	xpass2_4608_levels_real_1, xpass2_4608_levels_real_2
	LOCAL	xpass2_4608_levels_real_3, xpass2_4608_levels_real_4
	LOCAL	xpass2_4608_real_unfft, xpass2_4608_real_done

;; We switch to the same format used in one-pass FFTs (that is, the
;; high word of xmm registers being 1 greater than the low word):

;; Type 4 FFTs skip the forward FFT process

	start_timer 2
	mov	rsi, DATA_ADDR		;; Load source address
	mov	rbx, DIST_TO_FFTSRCARG
	cmp	ffttype, 4
	je	xpass2_4608_levels_real_4

;; Do FFT level 1,2 (actually 1.585 levels)
;;
;; On input the 64-byte cache lines hold these data values if pass 2 swizzles:
;;	0	+2304	4608	+2304	9216	+2304	13824	+2304
;;	1	...
;;	...
;;	2303
;; On input the 64-byte cache lines hold these data values if pass 1 swizzles:
;;	0	+9216	2304	+9216	4608	+9216	6912	+9216
;;	1	...
;;	...
;;	2303	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+9216	768	+9216	1536	+9216	2304	+9216
;;	1	...
;;	...
;;	767	...
;;	3072	...
;;	...

;; Do 1536 six_reals_fft operations
;; Do 1536 three_complex_djbfft operations
;;	distance between fft data elements is 1536
;;	do 768 iterations

	mov	rdi, xsincos_complex	;; Ptr to the sin/cos data
	mov	rdx, sincos3		;; Ptr to real sin/cos data
	loops_init 768
b1b:	r3_fh3cl_six_reals_three_complex_djbfft rsi, 64, 6*dist128, rdi, 768*XMM_SCD1, rdx, 768*XMM_HSCD1
	bump	rdi, XMM_SCD1		;; Next sin/cos data
	bump	rdx, XMM_HSCD1		;; Next sin/cos data
	loops	128, b1b		;; Test loop counter
	bump	rsi, -128*64+dist128	;; Next source pointer
	loops	6, b1b			;; Test loop counter
	bump	rsi, -6*dist128		;; Restore source pointer
	bump	rdi, -768*XMM_SCD1	;; Restore sin/cos ptr
	bump	rdx, -768*XMM_HSCD1	;; Restore sin/cos ptr

;; Do FFT levels 3,4 (actually 1.585 levels)
;;
;; On input the 64-byte cache lines hold these data values:
;;	0	+9216	768	+9216	1536	+9216	2304	+9216
;;	1	...
;;	...
;;	767	...
;;	3072	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+9216	256	+9216	512	+9216	768	+9216
;;	1	...
;;	...
;;	255	...
;;	1024	...
;;	...

;; Do 512 six_reals_fft operations
;; Do 512 three_complex_djbfft operations
;;	distance between fft data elements is 512
;;	do 256 macros each processing 24 data values

	mov	rcx, DATA_PREFETCH	;; Prefetch pointer
	xtouch	[rcx]			;; Preload first TLB
	loops_init_prefetch 768, 64, 4, rcx
	xtouch	[rcx+4096-64]		;; Preload the TLBs
b2b:	r3_h3cl_six_reals_three_complex_djbfft rsi, 64, 2*dist128, rdi, 256*3*XMM_SCD1, rdx, 256*3*XMM_HSCD1
	bump	rdi, 3*XMM_SCD1		;; Next sin/cos data
	bump	rdx, 3*XMM_HSCD1	;; Next sin/cos data
	loops	128, b2b		;; Test inner loop counter
	bump	rsi, -128*64+dist128	;; Next source pointer
	loops	2, b2b			;; Test loop counter
	bump	rsi, -2*dist128+6*dist128 ;; Next source pointer
	bump	rdi, -256*3*XMM_SCD1	;; Restore sin/cos ptr
	bump	rdx, 256*3*XMM_HSCD1	;; Next sin/cos ptr

;; Do 2048 three_complex_fft operations
;;	distance between fft data elements is 512
;;	do 512 macros each processing 24 data values

	loops_reset
	xtouch	[rcx+4096-64]		;; Preload the TLBs
	xtouch	[rcx+2*4096-64]
	r3_nf3cl_three_complex_djbfft_preload
b3b:	r3_nf3cl_three_complex_djbfft rsi, 64, 2*dist128, rdi, 256*3*XMM_SCD1
	bump	rdi, 3*XMM_SCD1		;; Next sin/cos data
	loops	128, b3b		;; Test inner loop counter
	bump	rsi, -128*64+dist128	;; Next source pointer
	loops	2, b3b			;; Test loop counter
	bump	rsi, -2*dist128+6*dist128 ;; Next source pointer
	bump	rdi, -256*3*XMM_SCD1	;; Restore sin/cos ptr
	loops	3, b3b			;; Test loop counter
	bump	rsi, -3*6*dist128	;; Restore source pointer

;; Do FFT levels 5,6
;;
;; On input the 64-byte cache lines hold these data values:
;;	0	+9216	256	+9216	512	+9216	768	+9216
;;	1	...
;;	...
;;	255	...
;;	1024	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+9216	128	+9216	256	+9216	384	+9216
;;	1	...
;;	...
;;	127	...
;;	512	...
;;	...

;; Do 128 eight_reals_fft operations
;; Do 384 four_complex_djbfft operations
;;	distance between fft data elements is 128
;;	do 128 macros each processing 16 data values

	bump	rdi, 1536*XMM_SCD1	;; Load sin/cos pointer
	loops_init_prefetch 1152, 64, 4, rcx
	xtouch	[rcx+4096-64]		;; Preload the TLBs
	xtouch	[rcx+2*4096-64]
	xtouch	[rcx+3*4096-64]
	xtouch	[rcx+4*4096-64]
	xtouch	[rcx+5*4096-64]
b4b:	r4_h2cl_eight_reals_four_complex_djbfft rsi, 64, dist128, rdi, rdx
	bump	rdi, XMM_SCD2		;; Next sin/cos data
	bump	rdx, XMM_HSCD2		;; Next sin/cos data
	loops	128, b4b		;; Test loop counter
	bump	rsi, -128*64+2*dist128	;; Next source pointer
	bump	rdi, -128*XMM_SCD2	;; Restore sin/cos ptr
	bump	rdx, -128*XMM_HSCD2	;; Restore sin/cos ptr

;; Do 2048 four_complex_djbfft operations
;;	distance between fft data elements is 128
;;	do 1024 macros each processing 16 data values

	loops_reset			;; 8 more iterations of 128
	r4_nf2cl_four_complex_djbfft_preload
b5b:	r4_nf2cl_four_complex_djbfft rsi, 64, dist128, rdi
	bump	rdi, XMM_SCD2		;; Next sin/cos data
	loops	128, b5b		;; Test inner loop counter
	bump	rsi, -128*64+2*dist128	;; Next source pointer
	bump	rdi, -128*XMM_SCD2	;; Restore sin/cos ptr
	loops	9, b5b			;; Test loop counter
	bump	rsi, -9*2*dist128	;; Next source pointer

;; Do FFT levels 7,8
;;
;; On input the 64-byte cache lines hold these data values:
;;	0	+9216	128	+9216	256	+9216	384	+9216
;;	1	...
;;	...
;;	127	...
;;	512	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+9216	32	+9216	64	+9216	96	+9216
;;	1	...
;;	...
;;	31	...
;;	128	...
;;	...

;; Do 32 eight_reals_fft operations
;; Do 96 four_complex_djbfft operations
;;	distance between fft data elements is 32
;;	do 32 macros each processing 32 data values

	loops_init_prefetch 576, 64, 2, rcx
	xtouch	[rcx+4096-64]		;; Preload the TLBs
	xtouch	[rcx+2*4096-64]
	xtouch	[rcx+3*4096-64]
	xtouch	[rcx+4*4096-64]
	xtouch	[rcx+5*4096-64]
	r4_h4cl_eight_reals_four_complex_djbfft_preload
b6b:	r4_h4cl_eight_reals_four_complex_djbfft rsi, 64, 32*64, 64*64, rdi, rdx
	bump	rdi, 4*XMM_SCD2		;; Next sin/cos data
	bump	rdx, 4*XMM_HSCD2	;; Next sin/cos data
	loops	32, b6b			;; Test loop counter
	bump	rsi, -32*64+dist128	;; Next source pointer
	bump	rdi, -32*4*XMM_SCD2	;; Restore sin/cos ptr
	bump	rdx, -32*4*XMM_HSCD2	;; Restore sin/cos ptr

;; Do 2176 four_complex_djbfft operations
;;	distance between fft data elements is 32
;;	do 544 macros each processing 32 data values

	loops_reset			;; 17 more iters of 32
	r4_x4cl_four_complex_djbfft_preload
b7b:	r4_x4cl_four_complex_djbfft rsi, 64, 32*64, 64*64, rdi
	bump	rdi, 4*XMM_SCD2		;; Next sin/cos data
	loops	32, b7b			;; Test inner loop counter
	bump	rsi, -32*64+dist128	;; Next source pointer
	bump	rdi, -32*4*XMM_SCD2	;; Restore sin/cos ptr
	loops	18, b7b			;; 18 iterations
	bump	rsi, -18*dist128	;; Next source pointer

;; Do FFT levels 9,10
;;
;; On input the 64-byte cache lines hold these data values:
;;	0	+9216	32	+9216	64	+9216	96	+9216
;;	1	...
;;	...
;;	31	...
;;	128	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+9216	8	+9216	16	+9216	24	+9216
;;	1	...
;;	...
;;	7	...
;;	32	...
;;	...

;; Do 8 eight_reals_fft operations
;; Do 24 four_complex_djbfft operations
;;	distance between fft data elements is 8
;;	do 8 macros each processing 32 data values

	loops_init_prefetch 576, 64, 2, rcx
	xtouch	[rcx+4096-64]		;; Preload the TLBs
	xtouch	[rcx+2*4096-64]
	xtouch	[rcx+3*4096-64]
	xtouch	[rcx+4*4096-64]
	xtouch	[rcx+5*4096-64]
	r4_h4cl_eight_reals_four_complex_djbfft_preload
b8b:	r4_h4cl_eight_reals_four_complex_djbfft rsi, 64, 8*64, 16*64, rdi, rdx
	bump	rdi, 16*XMM_SCD2	;; Next sin/cos data
	bump	rdx, 16*XMM_HSCD2	;; Next sin/cos data
	loops	8, b8b			;; Test loop counter
	bump	rsi, -8*64+32*64	;; Next source pointer
	bump	rdi, -8*16*XMM_SCD2	;; Restore sin/cos ptr
	bump	rdx, -8*16*XMM_HSCD2	;; Restore sin/cos ptr

;; Do 2272 four_complex_djbfft operations
;;	distance between fft data elements is 8
;;	do 568 macros each processing 32 data values

	loops_reset			;; 18 iters of 3 or 4 of 8
	r4_x4cl_four_complex_djbfft_preload
b9b:	r4_x4cl_four_complex_djbfft rsi, 64, 8*64, 16*64, rdi
	bump	rdi, 16*XMM_SCD2	;; Next sin/cos data
	loops	8, b9b			;; Test loop counter
	bump	rsi, -8*64+32*64	;; Next source pointer
	bump	rdi, -8*16*XMM_SCD2	;; Restore sin/cos ptr
	loops	4, b9b			;; Test middle loop counter
	bump	rsi, -4*32*64+dist128	;; Next source pointer
	loops	18, b9b			;; 18 iterations
	bump	rsi, -18*dist128	;; Next source pointer

;; Do FFT levels 11,12,13
;; Values 0-7 is real data, 8-9215 is complex data.
;;
;; On input the 64-byte cache lines hold these data values:
;;	0	+9216	8	+9216	16	+9216	24	+9216
;;	1	...
;;	...
;;	7	...
;;	32	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+9216	4	+9216	8	+9216	12	+9216
;;	1	...
;;	...
;;	3	...
;;	16	...
;;	...

;; Do 1 sixteen_reals_fft operations
;; Do 3 eight_complex_fft operations
;;	distance between fft data elements is 1

;; Do 1148 eight_complex_fft operations
;;	distance between fft data elements is 1
;;	do 287 macros each processing 64 data values

;; Execute the proper middle step

	cmp	ffttype, 2
	jl	xpass2_4608_levels_real_1
	jg	xpass2_4608_levels_real_3

xpass2_4608_levels_real_2:
	r8_h8cl_sixteen_reals_eight_complex_with_square rsi, 8*64, 64, 2*64, 4*64
	loops_init_prefetch 288, 64, 8, rcx, 1, 16
	xtouch	[rcx+4096-64]		;; Preload the TLBs
	r8_x8cl_eight_complex_with_square_preload
baa:	r8_x8cl_eight_complex_with_square rsi, 8*64, 64, 2*64, 4*64
	loops	16, baa			;; Test loop counter
	bump	rsi, -16*8*64+dist128	;; Next source pointer
	loops	18, baa			;; Test loop counter
	bump	rsi, -18*dist128	;; Next source pointer
	jmp	xpass2_4608_real_unfft

xpass2_4608_levels_real_1:
	r8_h8cl_sixteen_reals_eight_complex_fft_final rsi, 8*64, 64, 2*64, 4*64
	loops_init_prefetch 288, 128, 1, rcx, 1, 16
	r8_x8cl_eight_complex_fft_final_preload
bab0:	xtouch	[rcx+4096-64]		;; Preload the TLBs
bab:	r8_x8cl_eight_complex_fft_final rsi, 8*64, 64, 2*64, 4*64
	loops	16, bab			;; Test loop counter
	bump	rsi, -16*8*64+dist128	;; Next source pointer
	loops_touch 2, bab, 9, bab0	;; Test loop counter
	jmp	xpass2_4608_real_done

xpass2_4608_levels_real_3:
	mov	rbp, DIST_TO_MULSRCARG
	r8_h8cl_sixteen_reals_eight_complex_with_mult rsi, 8*64, 64, 2*64, 4*64
	loops_init_prefetch 288, 64, 8, rcx, 1, 16
	xtouch	[rcx+4096-64]		;; Preload the TLBs
	r8_x8cl_eight_complex_with_mult_preload
bac:	r8_x8cl_eight_complex_with_mult rsi, 8*64, 64, 2*64, 4*64
	loops	16, bac			;; Test loop counter
	bump	rsi, -16*8*64+dist128	;; Next source pointer
	loops	18, bac			;; Test loop counter
	bump	rsi, -18*dist128	;; Next source pointer
	jmp	xpass2_4608_real_unfft

xpass2_4608_levels_real_4:
	mov	rbp, DIST_TO_MULSRCARG
	mov	rcx, DATA_PREFETCH	;; Load prefetch pointer
	xtouch	[rcx]			;; Preload first TLB
	r8_h8cl_sixteen_reals_eight_complex_with_mulf rsi, 8*64, 64, 2*64, 4*64
	loops_init_prefetch 288, 64, 1, rcx, 1, 16
	r8_x8cl_eight_complex_with_mulf_preload
bad0:	xtouch	[rcx+4096-64]		;; Preload the TLBs
bad:	r8_x8cl_eight_complex_with_mulf rsi, 8*64, 64, 2*64, 4*64
	loops	16, bad			;; Test loop counter
	bump	rsi, -16*8*64+dist128	;; Next source pointer
	loops_touch 2, bad, 9, bad0	;; Test loop counter
	bump	rsi, -18*dist128	;; Next source pointer

;; Do inverse FFT levels 9,10
;; On input the 64-byte cache lines hold these data values:
;;	0	+9216	4	+9216	8	+9216	12	+9216
;;	1	...
;;	...
;;	3	...
;;	16	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+9216	16	+9216	32	+4608	48	+9216
;;	1	...
;;	...
;;	15	...
;;	64	...
;;	...

;; Do 8 eight_reals_unfft operations
;; Do 8 four_complex_djbunfft operations
;;	distance between fft data elements is 8
;;	do 4 macros processing 32 data values

xpass2_4608_real_unfft:
	mov	rdi, xsincos_complex	;; Load sin/cos pointers
	bump	rdi, 1536*XMM_SCD1
	mov	rdx, sincos3
	bump	rdx, 1536*XMM_HSCD1
	loops_init_prefetch 576, 64, 2, rcx
	xtouch	[rcx+4096-64]		;; Preload the TLBs
	xtouch	[rcx+2*4096-64]
	xtouch	[rcx+3*4096-64]
	xtouch	[rcx+4*4096-64]
	xtouch	[rcx+5*4096-64]
c8b:	r4_h4cl_eight_reals_four_complex_djbunfft rsi, 64, 4*64, 8*64, rdi, 4*16*XMM_SCD2, rdx, 4*16*XMM_HSCD2
	bump	rdi, 16*XMM_SCD2	;; Next sin/cos data
	bump	rdx, 16*XMM_HSCD2	;; Next sin/cos data
	loops	4, c8b			;; Test loop counter
	bump	rsi, -4*64+16*64	;; Next source pointer
	bump	rdi, -4*16*XMM_SCD2	;; Restore sin/cos ptr
	bump	rdx, -4*16*XMM_HSCD2	;; Restore sin/cos ptr

;; Do 2288 four_complex_djbunfft operations
;;	distance between fft data elements is 8
;;	do 572 macros each processing 32 data values

	loops_reset			;; 18 iters of 7 or 8 of 4
	r4_x4cl_four_complex_djbunfft_preload
c9b:	r4_x4cl_four_complex_djbunfft rsi, 64, 4*64, 8*64, rdi, 4*16*XMM_SCD2
	bump	rdi, 16*XMM_SCD2	;; Next sine/cosine pointer
	loops	4, c9b			;; Test loop counter
	bump	rsi, -4*64+16*64	;; Next source pointer
	bump	rdi, -4*16*XMM_SCD2	;; Restore sin/cos ptr
	loops	8, c9b			;; Test loop counter
	bump	rsi, -8*16*64+dist128	;; Next source pointer
	loops	18, c9b			;; Test loop counter
	bump	rsi, -18*dist128	;; Next source pointer

;; Do inverse FFT levels 7,8
;; On input the 64-byte cache lines hold these data values:
;;	0	+9216	16	+9216	32	+4608	48	+9216
;;	1	...
;;	...
;;	15	...
;;	64	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+9216	64	+9216	128	+4608	192	+9216
;;	1	...
;;	...
;;	63	...
;;	256	...
;;	...

;; Do 32 eight_reals_unfft operations
;; Do 32 four_complex_djbunfft operations
;;	distance between fft data elements is 32
;;	do 16 macros processing 32 data values

	loops_init_prefetch 576, 64, 2, rcx
	xtouch	[rcx+4096-64]		;; Preload the TLBs
	xtouch	[rcx+2*4096-64]
	xtouch	[rcx+3*4096-64]
	xtouch	[rcx+4*4096-64]
	xtouch	[rcx+5*4096-64]
c6b:	r4_h4cl_eight_reals_four_complex_djbunfft rsi, 64, 16*64, 32*64, rdi, 16*4*XMM_SCD2, rdx, 16*4*XMM_HSCD2
	bump	rdi, 4*XMM_SCD2		;; Next sin/cos data
	bump	rdx, 4*XMM_HSCD2	;; Next sin/cos data
	loops	16, c6b			;; Test inner loop counter
	bump	rsi, -16*64+64*64	;; Next source pointer
	bump	rdi, -16*4*XMM_SCD2	;; Restore sin/cos ptr
	bump	rdx, -16*4*XMM_HSCD2	;; Restore sin/cos ptr

;; Do 2240 four_complex_djbunfft operations
;;	distance between fft data elements is 16
;;	do 560 macros each processing 32 data values

	loops_reset			;; 18 iters of 1 or 2 of 16
	r4_x4cl_four_complex_djbunfft_preload
c7b:	r4_x4cl_four_complex_djbunfft rsi, 64, 16*64, 32*64, rdi, 16*4*XMM_SCD2
	bump	rdi, 4*XMM_SCD2		;; Next sin/cos data
	loops	16, c7b			;; Test inner loop counter
	bump	rsi, -16*64+64*64	;; Next source pointer
	bump	rdi, -16*4*XMM_SCD2	;; Restore sin/cos ptr
	loops	2, c7b			;; Test loop counter
	bump	rsi, -2*64*64+dist128	;; Next source pointer
	loops	18, c7b			;; 18 iterations
	bump	rsi, -18*dist128	;; Next source pointer

;; Do inverse FFT levels 5,6
;; On input the 64-byte cache lines hold these data values:
;;	0	+9216	64	+9216	128	+4608	192	+9216
;;	1	...
;;	...
;;	63	...
;;	256	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+9216	256	+9216	512	+4608	768	+9216
;;	1	...
;;	...
;;	255	...
;;	1024	...
;;	...

;; Do 128 eight_reals_unfft operations
;; Do 128 four_complex_djbunfft operations
;;	distance between fft data elements is 128
;;	do 64 macros processing 32 data values

	loops_init_prefetch 576, 64, 2, rcx
	xtouch	[rcx+4096-64]		;; Preload the TLBs
	xtouch	[rcx+2*4096-64]
	xtouch	[rcx+3*4096-64]
	xtouch	[rcx+4*4096-64]
	xtouch	[rcx+5*4096-64]
c4b:	r4_h4cl_eight_reals_four_complex_djbunfft rsi, 64, 64*64, dist128, rdi, 64*XMM_SCD2, rdx, 64*XMM_HSCD2
	bump	rdi, XMM_SCD2		;; Next sin/cos data
	bump	rdx, XMM_HSCD2		;; Next sin/cos data
	loops	64, c4b			;; Test inner loop counter
	bump	rsi, -64*64+2*dist128	;; Next source pointer
	bump	rdi, -64*XMM_SCD2	;; Restore sin/cos ptr
	bump	rdx, -64*XMM_HSCD2-1536*XMM_HSCD1 ;; Restore sin/cos ptr

;; Do 2048 four_complex_djbunfft operations
;;	distance between fft data elements is 128
;;	do 512 macros each processing 32 data values

	loops_reset			;; 8 more iters of 64
	r4_x4cl_four_complex_djbunfft_preload
c5b:	r4_x4cl_four_complex_djbunfft rsi, 64, 64*64, dist128, rdi, 64*XMM_SCD2
	bump	rdi, XMM_SCD2		;; Next sin/cos data
	loops	64, c5b			;; Test inner loop counter
	bump	rsi, -64*64+2*dist128	;; Next source pointer
	bump	rdi, -64*XMM_SCD2	;; Restore sin/cos ptr
	loops	9, c5b			;; Test loop counter
	bump	rsi, -9*2*dist128	;; Next source pointer

;; Do inverse FFT levels 3,4 (actually 1.585 FFT levels)
;; On input the 64-byte cache lines hold these data values:
;;	0	+9216	256	+9216	512	+4608	768	+9216
;;	1	...
;;	...
;;	255	...
;;	1024	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+9216	768	+9216	1536	+4608	2304	+9216
;;	1	...
;;	...
;;	767	...
;;	3072	...
;;	...

;; Do 512 six_reals_unfft operations
;; Do 512 six_complex_djbunfft operations
;;	distance between fft data elements is 512
;;	do 256 macros processing 32 data values

	loops_init_prefetch 768, 64, 4, rcx
	xtouch	[rcx+4096-64]		;; Preload the TLBs
	xtouch	[rcx+2*4096-64]
	xtouch	[rcx+3*4096-64]
	mov	rdi, xsincos_complex	;; Load sin/cos pointer
c2b:	r3_h3cl_six_reals_three_complex_djbunfft rsi, 64, 2*dist128, rdi, 256*3*XMM_SCD1, rdx, 256*3*XMM_HSCD1
	bump	rdi, 3*XMM_SCD1		;; Next sin/cos data
	bump	rdx, 3*XMM_HSCD1	;; Next sin/cos data
	loops	128, c2b		;; Test inner loop counter
	bump	rsi, -128*64+dist128	;; Next source pointer
	loops	2, c2b			;; Test loop counter
	bump	rsi, -2*dist128+6*dist128 ;; Next source pointer
	bump	rdi, -256*3*XMM_SCD1	;; Restore sin/cos ptr
	bump	rdx, -256*3*XMM_HSCD1	;; Restore sin/cos ptr

;; Do 2048 three_complex_djbunfft operations
;;	distance between fft data elements is 512
;;	do 512 macros each processing 32 data values

	loops_reset			;; 2 more iters of 256
	r3_x3cl_three_complex_djbunfft_preload
c3b:	r3_x3cl_three_complex_djbunfft rsi, 64, 2*dist128, rdi, 256*3*XMM_SCD1
	bump	rdi, 3*XMM_SCD1		;; Next sin/cos data
	loops	128, c3b		;; Test inner loop counter
	bump	rsi, -128*64+dist128	;; Next source pointer
	loops	2, c3b			;; Test loop counter
	bump	rsi, -2*dist128+6*dist128 ;; Next source pointer
	bump	rdi, -256*3*XMM_SCD1	;; Restore sin/cos ptr
	loops	3, c3b			;; Test loop counter
	bump	rsi, -3*6*dist128	;; Next source pointer

;; Do inverse FFT level 1,2 (actually 1.585 FFT levels)
;; On input the 64-byte cache lines hold these data values:
;;	0	+9216	768	+9216	1536	+4608	2304	+9216
;;	1	...
;;	...
;;	767	...
;;	3072	...
;;	...
;; On output the 64-byte cache lines hold these data values if pass 1 swizzles:
;;	0	+9216	2304	+9216	4608	+4608	6912	+9216
;;	1	...
;;	...
;;	2303
;; On output the 64-byte cache lines hold these data values if pass 2 swizzles:
;;	0	+2304	4608	+2304	9216	+2304	13824	+2304
;;	1	...
;;	...
;;	2303

;; Do 1536 six_reals_unfft operations
;; Do 1536 three_complex_djbunfft operations
;;	distance between fft data elements is 1536
;;	do 768 macros processing 24 data values

	bump	rcx, -2*64		;; Back up prefetch pointer by 2 cache lines
	loops_init_prefetch 768, 64, 4, rcx
	xtouch	[rcx+4096-64]		;; Preload the TLBs
	xtouch	[rcx+2*4096-64]
	xtouch	[rcx+3*4096-64]
c1b:	r3_h3cl_six_reals_three_complex_djbunfft rsi, 64, 6*dist128, rdi, 768*XMM_SCD1, rdx, 768*XMM_HSCD1
	bump	rdi, XMM_SCD1		;; Next sin/cos data
	bump	rdx, XMM_HSCD1		;; Next sin/cos data
	loops	128, c1b		;; Test loop counter
	bump	rsi, -128*64+dist128	;; Next source pointer
	loops	6, c1b			;; Test loop counter

xpass2_4608_real_done:
	end_timer 2
	ENDM

; *************** radix4-pass2-4608-levels-complex macro ******************
; This macro takes 2 groups of 4608 complex values and
; performs the final 12.17 levels of a standard mostly radix-4 FFT, squares the
; results, and does the first 12.17 levels of the inverse FFT.

r4_pass2_4608_levels_complex MACRO
	LOCAL	b1b, b2a, b2b, b3a, b3b, b4b, b5b
	LOCAL	b6a, b6b0, b6b, b6c, b6d0, b6d
	LOCAL	c1a, c1b, c2a, c2b, c3b, c4b, c5a, c5b
	LOCAL	xpass2_4608_levels_complex_1, xpass2_4608_levels_complex_2
	LOCAL	xpass2_4608_levels_complex_3, xpass2_4608_levels_complex_4
	LOCAL	xpass2_4608_levels_complex_unfft, xpass2_4608_levels_complex_done

;; Type 4 FFTs skip the forward FFT process

	mov	rsi, DATA_ADDR		;; Load address of FFT data
	mov	rbx, DIST_TO_FFTSRCARG
	cmp	ffttype, 4
	je	xpass2_4608_levels_complex_4

;; Do FFT levels 1,2 (actually 1.585 levels)
;;
;; We swizzle the data so that one set of data is in the high word of
;; an xmm register, the other set is in the low word.  This means
;; a distance of 9216 data values.
;;
;; On input the 64-byte cache lines hold these data values if pass 2 swizzles:
;;	0	+2304	4608	+2304	9216	+2304	13824	+2304
;;	1	...
;;	...
;;	2303
;; On input the 64-byte cache lines hold these data values if pass 1 swizzles:
;;	0	+9216	2304	+9216	4608	+9216	6912	+9216
;;	1	...
;;	...
;;	2303	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+9216	768	+9216	1536	+9216	2304	+9216
;;	1	...
;;	...
;;	767	...
;;	3072	...
;;	...

;; Do 3072 three_complex_fft operations
;;	distance between fft data elements is 1536
;;	do 768 macros each processing 24 data values

;; Do FFT levels 3,4 (actually 1.585 levels)
;;
;; On input the 64-byte cache lines hold these data values:
;;	0	+9216	768	+9216	1536	+9216	2304	+9216
;;	1	...
;;	...
;;	767	...
;;	3072	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+9216	256	+9216	512	+9216	768	+9216
;;	1	...
;;	...
;;	255	...
;;	1024	...
;;	...

;; Do 2304 three_complex_fft operations
;;	distance between fft data elements is 512
;;	do 768 macros each processing 24 data values

IF @INSTR(,%xarch,<CORE>) NE 0		;; Core 2 faster doing separate loops

	start_timer 4
	mov	rdi, xsincos_complex	;; The sin/cos data
	loops_init 768			;; 6 iterations of 128
	r3_f3cl_three_complex_djbfft_preload
b1b:	r3_f3cl_three_complex_djbfft rsi, 64, 6*dist128, rdi, 768*XMM_SCD1
	bump	rdi, XMM_SCD1		;; Next sin/cos data
	loops	128, b1b		;; Test inner loop counter
	bump	rsi, -128*64+dist128	;; Next source pointer
	loops	6, b1b			;; Test outer loop counter
	bump	rsi, -6*dist128		;; Restore source pointer
	end_timer 4

	start_timer 5
	mov	rcx, DATA_PREFETCH	;; Prefetch pointer (we need to prefetch 
					;; 2304 cache lines + 34 pad cache lines)
	xtouch	[rcx]			;; Load first prefetch TLB
	mov	rdi, xsincos_complex	;; Ptr to the sin/cos data
	loops_init_prefetch 768, 64, 4, rcx ;; 3 iters of 256
	r3_nf3cl_three_complex_djbfft_preload
b2a:	xtouch	[rcx+4096-64]		;; Load prefetch TLBs
b2b:	r3_nf3cl_three_complex_djbfft rsi, 64, 2*dist128, rdi, 256*3*XMM_SCD1
	bump	rdi, 3*XMM_SCD1		;; Next sin/cos data
	loops	128, b2b		;; Test inner loop counter
	bump	rsi, -128*64+dist128	;; Next source pointer
	loops	2, b2b			;; Test loop counter
	bump	rsi, -2*dist128+6*dist128 ;; Next source pointer
	bump	rdi, -256*3*XMM_SCD1	;; Restore sin/cos ptr
	loops	3, b2a			;; Test loop counter
	bump	rsi, -3*6*dist128	;; Restore source pointer
	end_timer 5

ELSE					;; Pentium 4 faster doing merged loops

	start_timer 4
	mov	rdi, xsincos_complex	;; The sin/cos data
	mov	rbp, rdi
	mov	rcx, DATA_PREFETCH	;; Prefetch pointer (we need to prefetch 
					;; 2304 cache lines + 34 pad cache lines)
	xtouch	[rcx]			;; Load first prefetch TLB
	mov	rdi, xsincos_complex	;; Ptr to the sin/cos data
	loops_init_prefetch 768, 64, 12, rcx ;; 256 iters of 3, prefetching 64 cache lines

	r3_f3cl_three_complex_djbfft_preload
;;	r3_nf3cl_three_complex_djbfft_preload	;; Dangerous - assume same as r3_f3cl_three_complex_djbfft_preload

	xtouch	[rcx+4096-64]		;; Load prefetch TLBs

b1b:	r3_f3cl_three_complex_djbfft rsi, 2*dist128, 6*dist128, rdi, 768*XMM_SCD1
	xxprefetch [rdi+XMM_SCD1]	;; Prefetch for next pass
	xxprefetch [rdi+769*XMM_SCD1]	;; Prefetch for next pass
	bump	rdi, 256*XMM_SCD1	;; Next sin/cos ptr
	loops	3, b1b			;; Test loop counter
	bump	rsi, -3*2*dist128	;; Restore source pointer
	loops_undo 3

b2b:	r3_nf3cl_three_complex_djbfft rsi, 6*dist128, 2*dist128, rbp, 256*3*XMM_SCD1
	loops	3, b2b			;; Test loop counter

	bump	rsi, -3*6*dist128+64	;; Next source pointer
	bump	rdi, -3*256*XMM_SCD1+XMM_SCD1 ;; Next sin/cos ptr
	bump	rbp, 3*XMM_SCD1		;; Next sin/cos ptr
	loops	128, b1b		;; Test inner loop counter
	bump	rsi, -128*64+dist128	;; Next source pointer
	loops	2, b1b			;; Test loop counter
	bump	rsi, -2*dist128		;; Restore source pointer
	bump	rdi, -256*XMM_SCD1	;; Restore sin/cos ptr
	end_timer 4

ENDIF

;; Do FFT levels 5,6
;;
;; On input the 64-byte cache lines hold these data values:
;;	0	+9216	256	+9216	512	+9216	768	+9216
;;	1	...
;;	...
;;	255	...
;;	1024	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+9216	128	+9216	256	+9216	384	+9216
;;	1	...
;;	...
;;	127	...
;;	512	...
;;	...
;;
;; Do 2304 four_complex_fft operations
;;	distance between fft data elements is 128
;;	do 1152 macros each processing 16 data values

;; Do FFT levels 7,8
;;
;; On input the 64-byte cache lines hold these data values:
;;	0	+9216	128	+9216	256	+9216	384	+9216
;;	1	...
;;	...
;;	127	...
;;	512	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+9216	32	+9216	64	+9216	96	+9216
;;	1	...
;;	...
;;	31	...
;;	128	...
;;	...
;;
;; Do 2304 four_complex_fft operations
;;	distance between fft data elements is 32
;;	do 576 macros each processing 32 data values

;; Do FFT levels 9,10
;;
;; On input the 64-byte cache lines hold these data values:
;;	0	+9216	32	+9216	64	+9216	96	+9216
;;	1	...
;;	...
;;	31	...
;;	128	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+9216	8	+9216	16	+9216	24	+9216
;;	1	...
;;	...
;;	7	...
;;	32	...
;;	...
;;
;; Do 2304 four_complex_fft operations
;;	distance between fft data elements is 8
;;	do 576 macros each processing 32 data values

;; Do this 32 cache lines at a time to maximize accessing data
;; while it is in the L1 cache.

	start_timer 6
	bump	rdi, 1536*XMM_SCD1	;; Load sin/cos pointers
	mov	rbx, rdi
	mov	rbp, rdi
	loops_init_prefetch 1152, 64, 4, rcx ;; 9 * 8 iters of 32 cache lines
	;r4_nf2cl_four_complex_djbfft_preload -- assume same as r4_x4cl_four_complex_djbfft_preload
	r4_x4cl_four_complex_djbfft_preload

b3a:	xtouch	[rcx+4096-64]		;; Load prefetch TLBs
	xtouch	[rcx+2*4096-64]

b3b:	xxprefetch [rdi+8*XMM_SCD2]	;; Prefetch for next pass
	r4_nf2cl_four_complex_djbfft rsi, 8*64, dist128, rdi
	bump	rdi, 8*XMM_SCD2		;; Next sine/cosine pointer
	loops	16, b3b			;; Loop 16 times
	bump	rsi, -16*8*64		;; Restore source pointer
	bump	rdi, -16*8*XMM_SCD2	;; Restore sine/cosine pointer
	loops_undo 16			;; Back up loop counter

b4b:	r4_x4cl_four_complex_djbfft rsi, 8*64, 32*64, 64*64, rbx
	bump	rbx, 4*8*XMM_SCD2	;; Next sine/cosine pointer
	loops	4, b4b, 2		;; 4 iterations with non-standard increment
	bump	rsi, -4*8*64+dist128	;; Next source pointer
	bump	rbx, -4*4*8*XMM_SCD2	;; Restore sine/cosine pointer
	loops	2, b4b			;; Loop twice
	bump	rsi, -2*dist128		;; Restore source pointer
	loops_undo 2*4*2		;; Back up loop counter

b5b:	r4_x4cl_four_complex_djbfft rsi, 32*64, 8*64, 16*64, rbp
	loops	4, b5b, 2 		;; 4 iterations with non-standard increment
	bump	rsi, -4*32*64+dist128	;; Next source pointer
	loops	2, b5b	 		;; 2 iterations

	bump	rsi, -2*dist128+64	;; Next source pointer
	bump	rdi, XMM_SCD2		;; Next sine/cosine pointer
	bump	rbx, 4*XMM_SCD2		;; Next sine/cosine pointer
	bump	rbp, 16*XMM_SCD2	;; Next sine/cosine pointer
	loops	8, b3b			;; Loop 8 times
	bump	rsi, -8*64+2*dist128	;; Next source pointer
	bump	rdi, -8*XMM_SCD2	;; Restore sine/cosine pointer
	bump	rbx, -8*4*XMM_SCD2	;; Restore sine/cosine pointer
	bump	rbp, -8*16*XMM_SCD2	;; Restore sine/cosine pointer
	loops	9, b3a			;; Loop 9 times
	bump	rsi, -9*2*dist128	;; Restore source pointer
	end_timer 6

;; Do FFT levels 11,12,13 as well as inverse FFT levels 11,12,13
;;
;; On input the 64-byte cache lines hold these data values:
;;	0	+9216	8	+9216	16	+9216	24	+9216
;;	1	...
;;	...
;;	7	...
;;	32	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+9216	4	+9216	8	+9216	12	+9216
;;	1	...
;;	...
;;	3	...
;;	16	...
;;	...

;; Do 1152 eight_complex_fft operations
;;	distance between fft data elements is 1
;;	do 288 macros each processing 64 data values

;; Execute the right middle step

	cmp	ffttype, 2
	jl	xpass2_4608_levels_complex_1
	jg	xpass2_4608_levels_complex_3

xpass2_4608_levels_complex_2:
	start_timer 9
	loops_init_prefetch 288, 64, 8, rcx
	xtouch	[rcx+4096-64]		;; Load prefetch TLB
	r8_x8cl_eight_complex_with_square_preload
b6a:	r8_x8cl_eight_complex_with_square rsi, 8*64, 64, 2*64, 4*64
	loops	16, b6a			;; Loop 16 times
	bump	rsi, -16*8*64+dist128	;; Next source pointer
	loops	18, b6a			;; Loop 18 times
	bump	rsi, -18*dist128	;; Restore source pointer
	end_timer 9
	jmp	xpass2_4608_levels_complex_unfft

xpass2_4608_levels_complex_1:
	start_timer 9
	loops_init_prefetch 288, 128, 1, rcx
	r8_x8cl_eight_complex_fft_final_preload
b6b0:	xtouch	[rcx+4096-64]		;; Load prefetch TLBs
b6b:	r8_x8cl_eight_complex_fft_final rsi, 8*64, 64, 2*64, 4*64
	loops	16, b6b			;; Loop 16 times
	bump	rsi, -16*8*64+dist128	;; Next source pointer
	loops_touch 2, b6b, 9, b6b0	;; Loop 18 times
	end_timer 9
	jmp	xpass2_4608_levels_complex_done

xpass2_4608_levels_complex_3:
	start_timer 9
	mov	rbp, DIST_TO_MULSRCARG
	loops_init_prefetch 288, 64, 8, rcx
	r8_x8cl_eight_complex_with_mult_preload
	xtouch	[rcx+4096-64]		;; Load prefetch TLB
b6c:	r8_x8cl_eight_complex_with_mult rsi, 8*64, 64, 2*64, 4*64
	loops	16, b6c			;; Loop 16 times
	bump	rsi, -16*8*64+dist128	;; Next source pointer
	loops	18, b6c			;; Loop 18 times
	bump	rsi, -18*dist128	;; Restore source pointer
	end_timer 9
	jmp	xpass2_4608_levels_complex_unfft

xpass2_4608_levels_complex_4:
	start_timer 9
	mov	rcx, DATA_PREFETCH	;; Prefetch pointer
	xtouch	[rcx]			;; Load first prefetch TLB
	mov	rbp, DIST_TO_MULSRCARG
	loops_init_prefetch 288, 128, 1, rcx
	r8_x8cl_eight_complex_with_mulf_preload
b6d0:	xtouch	[rcx+4096-64]		;; Load prefetch TLBs
b6d:	r8_x8cl_eight_complex_with_mulf rsi, 8*64, 64, 2*64, 4*64
	loops	16, b6d			;; Loop 16 times
	bump	rsi, -16*8*64+dist128	;; Next source pointer
	loops_touch 2, b6d, 9, b6d0	;; Loop 18 times
	bump	rsi, -18*dist128	;; Restore source pointer
	end_timer 9

;; Do inverse FFT levels 9,10
;; On input the 64-byte cache lines hold these data values:
;;	0	+9216	4	+9216	8	+9216	12	+9216
;;	1	...
;;	...
;;	3	...
;;	16	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+9216	16	+9216	32	+4608	48	+9216
;;	1	...
;;	...
;;	15	...
;;	64	...
;;	...
;;
;; Do 2304 four_complex_unfft operations
;;	distance between fft data elements is 8
;;	do 576 macros each processing 32 data values

;; Do inverse FFT levels 7,8
;; On input the 64-byte cache lines hold these data values:
;;	0	+9216	16	+9216	32	+4608	48	+9216
;;	1	...
;;	...
;;	15	...
;;	64	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+9216	64	+9216	128	+4608	192	+9216
;;	1	...
;;	...
;;	63	...
;;	256	...
;;	...
;;
;; Do 2304 four_complex_unfft operations
;;	distance between fft data elements is 32
;;	do 576 macros each processing 32 data values

;; Do inverse FFT levels 5,6
;; On input the 64-byte cache lines hold these data values:
;;	0	+9216	64	+9216	128	+4608	192	+9216
;;	1	...
;;	...
;;	63	...
;;	256	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+9216	256	+9216	512	+4608	768	+9216
;;	1	...
;;	...
;;	255	...
;;	1024	...
;;	...
;;
;; Do 2304 four_complex_unfft operations
;;	distance between fft data elements is 128
;;	do 576 macros each processing 32 data values

;; Do this 64 cache lines at a time to maximize accessing data
;; while it is in the L1 cache.

xpass2_4608_levels_complex_unfft:
	start_timer 10
	mov	rdi, xsincos_complex	;; Load sin/cos pointers
	bump	rdi, 1536*XMM_SCD1
	mov	rbx, rdi
	mov	rbp, rdi
	loops_init_prefetch 576, 64, 2, rcx ;; 9*4 iters of 64 cache lines
	r4_x4cl_four_complex_djbunfft_preload

c5a:	xtouch	[rcx+4096-128]		;; Load prefetch TLBs
	xtouch	[rcx+2*4096-128]

c5b:	r4_x4cl_four_complex_djbunfft rsi, 16*64, 4*64, 8*64, rbp, 4*16*XMM_SCD2
	loops	8, c5b			;; Loop 8 times
	bump	rsi, -8*16*64+dist128	;; Next source pointer
	loops	2, c5b			;; Loop 2 times
	bump	rsi, -2*dist128		;; Restore source pointer
	loops_undo 2*8			;; Restore loop counter

c4b:	r4_x4cl_four_complex_djbunfft rsi, 4*64, 16*64, 32*64, rbx, 16*4*XMM_SCD2
	bump	rbx, 4*4*XMM_SCD2	;; Next sine/cosine pointer
	loops	4, c4b			;; Loop 4 times
	bump	rsi, -4*4*64+64*64	;; Next source pointer
	bump	rbx, -4*4*4*XMM_SCD2	;; Restore sine/cosine pointer
	loops	2, c4b			;; Loop twice
	bump	rsi, -2*64*64+dist128	;; Next source pointer
	loops	2, c4b			;; Loop twice
	bump	rsi, -2*dist128		;; Restore source pointer
	loops_undo 2*2*4		;; Restore loop counter

c3b:	r4_x4cl_four_complex_djbunfft rsi, 4*64, 64*64, dist128, rdi, 64*1*XMM_SCD2
	bump	rdi, 4*1*XMM_SCD2	;; Next sine/cosine pointer
	loops	16, c3b			;; Loop 16 times
	bump	rdi, -16*4*1*XMM_SCD2	;; Restore sine/cosine pointer

	bump	rsi, -16*4*64+64	;; Next source pointer
	bump	rdi, XMM_SCD2		;; Next sine/cosine pointer
	bump	rbx, 4*XMM_SCD2		;; Next sine/cosine pointer
	bump	rbp, 16*XMM_SCD2	;; Next sine/cosine pointer
	loops	4, c5b			;; Loop 4 times
	bump	rsi, -4*64+2*dist128	;; Next source pointer
	bump	rdi, -4*XMM_SCD2	;; Restore sine/cosine pointer
	bump	rbx, -4*4*XMM_SCD2	;; Restore sine/cosine pointer
	bump	rbp, -4*16*XMM_SCD2	;; Restore sine/cosine pointer
	loops	9, c5a			;; Loop 9 times
	bump	rsi, -9*2*dist128	;; Restore source pointer
	end_timer 10

;; Do inverse FFT levels 3,4 (actually 1.585 FFT levels)
;; On input the 64-byte cache lines hold these data values:
;;	0	+9216	256	+9216	512	+4608	768	+9216
;;	1	...
;;	...
;;	255	...
;;	1024	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+9216	768	+9216	1536	+4608	2304	+9216
;;	1	...
;;	...
;;	767	...
;;	3072	...
;;	...

;; Do 3072 three_complex_unfft operations
;;	distance between fft data elements is 512
;;	do 768 macros each processing 32 data values

;; Do inverse FFT levels 1,2 (actually 1.585 FFT levels)
;; On input the 64-byte cache lines hold these data values:
;;	0	+9216	768	+9216	1536	+4608	2304	+9216
;;	1	...
;;	...
;;	767	...
;;	3072	...
;;	...
;; On output the 64-byte cache lines hold these data values if pass 1 swizzles:
;;	0	+9216	2304	+9216	4608	+4608	6912	+9216
;;	1	...
;;	...
;;	2303
;; On output the 64-byte cache lines hold these data values if pass 2 swizzles:
;;	0	+2304	4608	+2304	9216	+2304	13824	+2304
;;	1	...
;;	...
;;	2303

;; Do 3072 three_complex_unfft operations
;;	distance between fft data elements is 1536
;;	do 768 macros each processing 24 data values

IF @INSTR(,%xarch,<CORE>) NE 0		;; Core 2 faster doing separate loops

	start_timer 13
	mov	rdi, xsincos_complex	;; Load sin/cos pointer
	loops_init_prefetch 768, 64, 4, rcx;; 3 iters of 128
	r3_x3cl_three_complex_djbunfft_preload
c2a:	xtouch	[rcx+4096-64]		;; Load prefetch TLBs
c2b:	r3_x3cl_three_complex_djbunfft rsi, 64, 2*dist128, rdi, 256*3*XMM_SCD1
	bump	rdi, 3*XMM_SCD1		;; Next sine/cosine pointer
	loops	128, c2b		;; Loop 128 times
	bump	rsi, -128*64+dist128	;; Next source pointer
	loops	2, c2b			;; Loop 2 times
	bump	rsi, -2*dist128+6*dist128 ;; Next source pointer
	bump	rdi, -256*3*XMM_SCD1	;; Reset sine/cosine pointer
	loops	3, c2a			;; Test loop counter
	bump	rsi, -3*6*dist128	;; Restore source pointer
	end_timer 13

	start_timer 14
	mov	rdi, xsincos_complex	;; Load sin/cos pointer
	bump	rcx, -2*64		;; Back up prefetch pointer by 2 cache lines
	loops_init_prefetch 768, 64, 4, rcx;; 6 iters of 128
	r3_x3cl_three_complex_djbunfft_preload
c1a:	xtouch	[rcx+4096-64]		;; Load prefetch TLBs
c1b:	r3_x3cl_three_complex_djbunfft rsi, 64, 6*dist128, rdi, 768*XMM_SCD1
	bump	rdi, XMM_SCD1		;; Next set of sin/cos data
	loops	128, c1b		;; Test inner loop counter
	bump	rsi, -128*64+dist128	;; Next source pointer
	loops_touch 2, c1b, 3, c1a	;; Test outer loop counter
	end_timer 14

ELSE					;; Pentium 4 faster doing merged loops

	start_timer 13
	mov	rdi, xsincos_complex	;; Load sin/cos pointer
	mov	rbp, rdi
	bump	rcx, -2*64		;; Back up prefetch pointer by 2 cache lines
	loops_init_prefetch 768, 64, 3, rcx ;; 256 iters of 3, prefetching 256 cache lines each loop
	r3_x3cl_three_complex_djbunfft_preload

c2a:	xtouch	[rcx+4096-64]		;; Load prefetch TLBs

c2b:	r3_x3cl_three_complex_djbunfft rsi, 6*dist128, 2*dist128, rbp, 256*3*XMM_SCD1
	loops	3, c2b			;; Test loop counter
	bump	rsi, -3*6*dist128	;; Restore source pointer
	loops_undo 3

c1b:	r3_x3cl_three_complex_djbunfft rsi, 2*dist128, 6*dist128, rdi, 768*XMM_SCD1
	bump	rdi, 256*XMM_SCD1	;; Next sin/cos ptr
	loops	3, c1b			;; Test loop counter

	bump	rsi, -3*2*dist128+64	;; Next source pointer
	bump	rdi, -3*256*XMM_SCD1+XMM_SCD1 ;; Next sin/cos ptr
	bump	rbp, 3*XMM_SCD1		;; Next sine/cosine pointer
	loops_touch 32, c2b, 4, c2a	;; Loop 128 times
	bump	rsi, -128*64+dist128	;; Next source pointer
	loops	2, c2a			;; Loop 2 times
	end_timer 13

ENDIF

xpass2_4608_levels_complex_done:
	ENDM


; *************** radix4-pass2-6144-levels-real macro ******************
; This macro takes 12288 real values and performs the final 13.585 levels of
; the FFT.  Independently, this macro also takes 6144 complex values and
; performs the final 12.585 levels of the FFT.  The results are squared, and
; the first 12.585/13.585 levels of the inverse FFT are done.

r4_pass2_6144_levels_real MACRO
	LOCAL	b1b, b2b, b3a, b3b, b4b, b5a, b5b, b6b, b7a, b7b
	LOCAL	b8b, b9b, baa, bab0, bab, bac, bad0, bad
	LOCAL	c1b, c2b, c3a, c3b, c4b, c5a, c5b, c6b, c7a, c7b, c8b, c9a, c9b
	LOCAL	xpass2_6144_levels_real_1, xpass2_6144_levels_real_2
	LOCAL	xpass2_6144_levels_real_3, xpass2_6144_levels_real_4
	LOCAL	xpass2_6144_real_unfft, xpass2_6144_real_done

;; We switch to the same format used in one-pass FFTs (that is, the
;; high word of xmm registers being 1 greater than the low word):

;; Type 4 FFTs skip the forward FFT process

	start_timer 2
	mov	rsi, DATA_ADDR		;; Load source address
	mov	rbx, DIST_TO_FFTSRCARG
	cmp	ffttype, 4
	je	xpass2_6144_levels_real_4

;; Do FFT level 1,2 (actually 1.585 levels)
;;
;; On input the 64-byte cache lines hold these data values if pass 2 swizzles:
;;	0	+3K	6144	+3K	12288	+3K	18432	+3K
;;	1	...
;;	...
;;	3071
;; On input the 64-byte cache lines hold these data values if pass 1 swizzles:
;;	0	+12K	3072	+12K	6144	+12K	9216	+12K
;;	1	...
;;	...
;;	3071	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+12K	1024	+12K	2048	+12K	3072	+12K
;;	1	...
;;	...
;;	1023	...
;;	4096	...
;;	...

;; Do 2048 six_reals_fft operations
;; Do 2048 three_complex_djbfft operations
;;	distance between fft data elements is 2K
;;	do 1024 iterations

	mov	rdi, xsincos_complex	;; Ptr to the sin/cos data
	mov	rdx, sincos3		;; Ptr to real sin/cos data
	loops_init 1024
b1b:	r3_fh3cl_six_reals_three_complex_djbfft rsi, 64, 8*dist128, rdi, 1024*XMM_SCD1, rdx, 1024*XMM_HSCD1
	bump	rdi, XMM_SCD1		;; Next sin/cos data
	bump	rdx, XMM_HSCD1		;; Next sin/cos data
	loops	128, b1b		;; Test loop counter
	bump	rsi, -128*64+dist128	;; Next source pointer
	loops	8, b1b			;; Test loop counter
	bump	rsi, -8*dist128		;; Restore source pointer
	bump	rdi, -1024*XMM_SCD1	;; Restore sin/cos ptr
	bump	rdx, 1024*XMM_HSCD1	;; Next sin/cos ptr

;; Do FFT levels 3,4
;;
;; On input the 64-byte cache lines hold these data values:
;;	0	+12K	1024	+12K	2048	+12K	3072	+12K
;;	1	...
;;	...
;;	1023	...
;;	4096	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+12K	512	+12K	1024	+12K	1536	+12K
;;	1	...
;;	...
;;	511	...
;;	2048	...
;;	...

;; Do 512 eight_reals_fft operations
;; Do 512 four_complex_djbfft operations
;;	distance between fft data elements is 512
;;	do 512 macros each processing 16 data values

	mov	rcx, DATA_PREFETCH	;; Prefetch pointer
	xtouch	[rcx]			;; Preload first TLB
	mov	rbx, rdi
	loops_init_prefetch 1536, 64, 4, rcx
	xtouch	[rcx+4096-64]		;; Preload the TLBs
b2b:	r4_h2cl_2sc_eight_reals_four_complex_djbfft rsi, 64, 4*dist128, rdi, rbx, rdx
	bump	rdi, 3*XMM_SCD1		;; Next sin/cos data
	bump	rbx, 6*XMM_SCD1		;; Next sin/cos data
	bump	rdx, XMM_HSCD2		;; Next sin/cos data
	loops	128, b2b		;; Test inner loop counter
	bump	rsi, -128*64+dist128	;; Next source pointer
	loops	4, b2b			;; Test loop counter
	bump	rsi, -4*dist128+8*dist128 ;; Next source pointer
	bump	rdi, -512*3*XMM_SCD1	;; Restore sin/cos ptr
	bump	rbx, -512*6*XMM_SCD1	;; Restore sin/cos ptr
	bump	rdx, -512*XMM_HSCD2	;; Restore sin/cos ptr

;; Do 2048 four_complex_fft operations
;;	distance between fft data elements is 512
;;	do 1024 macros each processing 16 data values

	loops_reset
	r4_x2cl_2sc_four_complex_djbfft_preload
b3a:	xtouch	[rcx+4096-64]		;; Preload the TLBs
b3b:	r4_x2cl_2sc_four_complex_djbfft rsi, 64, 4*dist128, rdi, rbx
	bump	rdi, 3*XMM_SCD1		;; Next sin/cos data
	bump	rbx, 6*XMM_SCD1		;; Next sin/cos data
	loops	128, b3b		;; Test inner loop counter
	bump	rsi, -128*64+dist128	;; Next source pointer
	loops_touch 2, b3b, 2, b3a	;; Test loop counter
	bump	rsi, -4*dist128+8*dist128 ;; Next source pointer
	bump	rdi, -512*3*XMM_SCD1	;; Restore sin/cos ptr
	bump	rbx, -512*6*XMM_SCD1	;; Restore sin/cos ptr
	loops	3, b3a			;; Test loop counter
	bump	rsi, -3*8*dist128	;; Restore source pointer

;; Do FFT levels 5,6
;;
;; On input the 64-byte cache lines hold these data values:
;;	0	+12K	512	+12K	1024	+12K	1536	+12K
;;	1	...
;;	...
;;	511	...
;;	2048	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+12K	128	+12K	256	+12K	384	+12K
;;	1	...
;;	...
;;	127	...
;;	512	...
;;	...

;; Do 128 eight_reals_fft operations
;; Do 384 four_complex_djbfft operations
;;	distance between fft data elements is 128
;;	do 128 macros each processing 32 data values

	bump	rdi, 3072*XMM_SCD1	;; Load sin/cos pointer
	loops_init_prefetch 768, 64, 2, rcx
	xtouch	[rcx+4096-64]		;; Preload the TLBs
	r4_h4cl_eight_reals_four_complex_djbfft_preload
b4b:	r4_h4cl_eight_reals_four_complex_djbfft rsi, 64, dist128, 2*dist128, rdi, rdx
	bump	rdi, XMM_SCD2		;; Next sin/cos data
	bump	rdx, 4*XMM_HSCD2	;; Next sin/cos data
	loops	128, b4b		;; Test loop counter
	bump	rsi, -128*64+4*dist128	;; Next source pointer
	bump	rdi, -128*XMM_SCD2	;; Restore sin/cos ptr
	bump	rdx, -128*4*XMM_HSCD2	;; Restore sin/cos ptr

;; Do 2560 four_complex_djbfft operations
;;	distance between fft data elements is 128
;;	do 640 macros each processing 32 data values

	loops_reset			;; 5 iterations of 128
	r4_x4cl_four_complex_djbfft_preload
b5a:	xtouch	[rcx+4096-64]		;; Preload the TLBs
b5b:	r4_x4cl_four_complex_djbfft rsi, 64, dist128, 2*dist128, rdi
	bump	rdi, XMM_SCD2		;; Next sin/cos data
	loops	128, b5b		;; Test inner loop counter
	bump	rsi, -128*64+4*dist128	;; Next source pointer
	bump	rdi, -128*XMM_SCD2	;; Restore sin/cos ptr
	loops	6, b5a			;; Test loop counter
	bump	rsi, -6*4*dist128	;; Next source pointer

;; Do FFT levels 7,8
;;
;; On input the 64-byte cache lines hold these data values:
;;	0	+12K	128	+12K	256	+12K	384	+12K
;;	1	...
;;	...
;;	127	...
;;	512	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+12K	32	+12K	64	+12K	96	+12K
;;	1	...
;;	...
;;	31	...
;;	128	...
;;	...

;; Do 32 eight_reals_fft operations
;; Do 96 four_complex_djbfft operations
;;	distance between fft data elements is 32
;;	do 32 macros each processing 32 data values

	loops_init_prefetch 768, 64, 2, rcx
	xtouch	[rcx+4096-64]		;; Preload the TLBs
	r4_h4cl_eight_reals_four_complex_djbfft_preload
b6b:	r4_h4cl_eight_reals_four_complex_djbfft rsi, 64, 32*64, 64*64, rdi, rdx
	bump	rdi, 4*XMM_SCD2		;; Next sin/cos data
	bump	rdx, 16*XMM_HSCD2	;; Next sin/cos data
	loops	32, b6b			;; Test loop counter
	bump	rsi, -32*64+dist128	;; Next source pointer
	bump	rdi, -32*4*XMM_SCD2	;; Restore sin/cos ptr
	bump	rdx, -32*16*XMM_HSCD2	;; Restore sin/cos ptr

;; Do 2944 four_complex_djbfft operations
;;	distance between fft data elements is 32
;;	do 736 macros each processing 32 data values

	loops_reset			;; 23 more iters of 32
	r4_x4cl_four_complex_djbfft_preload
b7a:	xtouch	[rcx+4096-64]		;; Preload the TLBs
b7b:	r4_x4cl_four_complex_djbfft rsi, 64, 32*64, 64*64, rdi
	bump	rdi, 4*XMM_SCD2		;; Next sin/cos data
	loops	32, b7b			;; Test inner loop counter
	bump	rsi, -32*64+dist128	;; Next source pointer
	bump	rdi, -32*4*XMM_SCD2	;; Restore sin/cos ptr
	loops_touch 4, b7b, 6, b7a	;; 24 iterations
	bump	rsi, -24*dist128	;; Next source pointer

;; Do FFT levels 9,10
;;
;; On input the 64-byte cache lines hold these data values:
;;	0	+12K	32	+12K	64	+12K	96	+12K
;;	1	...
;;	...
;;	31	...
;;	128	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+12K	8	+12K	16	+12K	24	+12K
;;	1	...
;;	...
;;	7	...
;;	32	...
;;	...

;; Do 8 eight_reals_fft operations
;; Do 24 four_complex_djbfft operations
;;	distance between fft data elements is 8
;;	do 8 macros each processing 32 data values

	loops_init_prefetch 768, 64, 2, rcx
	xtouch	[rcx+4096-64]		;; Preload the TLBs
	r4_h4cl_eight_reals_four_complex_djbfft_preload
b8b:	r4_h4cl_eight_reals_four_complex_djbfft rsi, 64, 8*64, 16*64, rdi, rdx
	bump	rdi, 16*XMM_SCD2	;; Next sin/cos data
	bump	rdx, 64*XMM_HSCD2	;; Next sin/cos data
	loops	8, b8b			;; Test loop counter
	bump	rsi, -8*64+32*64	;; Next source pointer
	bump	rdi, -8*16*XMM_SCD2	;; Restore sin/cos ptr
	bump	rdx, -8*64*XMM_HSCD2	;; Restore sin/cos ptr

;; Do 3040 four_complex_djbfft operations
;;	distance between fft data elements is 8
;;	do 760 macros each processing 32 data values

	loops_reset			;; 24 iters of 3 or 4 of 8
	r4_x4cl_four_complex_djbfft_preload
b9a:	xtouch	[rcx+4096-64]		;; Preload the TLBs
b9b:	r4_x4cl_four_complex_djbfft rsi, 64, 8*64, 16*64, rdi
	bump	rdi, 16*XMM_SCD2	;; Next sin/cos data
	loops	8, b9b			;; Test loop counter
	bump	rsi, -8*64+32*64	;; Next source pointer
	bump	rdi, -8*16*XMM_SCD2	;; Restore sin/cos ptr
	loops	4, b9b			;; Test middle loop counter
	bump	rsi, -4*32*64+dist128	;; Next source pointer
	loops_touch 4, b9b, 6, b9a	;; 24 iterations
	bump	rsi, -24*dist128	;; Next source pointer

;; Do FFT levels 11,12
;;
;; On input the 64-byte cache lines hold these data values:
;;	0	+12K	8	+12K	16	+12K	24	+12K
;;	1	...
;;	...
;;	7	...
;;	32	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+12K	4	+12K	8	+12K	12	+12K
;;	1	...
;;	...
;;	3	...
;;	16	...
;;	...

;; Do 1 sixteen_reals_fft operations
;; Do 3 eight_complex_fft operations
;;	distance between fft data elements is 1

;; Do 1532 eight_complex_fft operations
;;	distance between fft data elements is 1
;;	do 383 macros each processing 64 data values

;; Execute the proper middle step

	cmp	ffttype, 2
	jl	xpass2_6144_levels_real_1
	jg	xpass2_6144_levels_real_3

xpass2_6144_levels_real_2:
	r8_h8cl_sixteen_reals_eight_complex_with_square rsi, 8*64, 64, 2*64, 4*64
	loops_init_prefetch 384, 64, 8, rcx, 1, 16
	r8_x8cl_eight_complex_with_square_preload
	xtouch	[rcx+4096-64]		;; Preload the TLBs
baa:	r8_x8cl_eight_complex_with_square rsi, 8*64, 64, 2*64, 4*64
	loops	16, baa			;; Test loop counter
	bump	rsi, -16*8*64+dist128	;; Next source pointer
	loops	24, baa			;; Test loop counter
	bump	rsi, -24*dist128	;; Next source pointer
	jmp	xpass2_6144_real_unfft

xpass2_6144_levels_real_1:
	r8_h8cl_sixteen_reals_eight_complex_fft_final rsi, 8*64, 64, 2*64, 4*64
	loops_init_prefetch 384, 128, 1, rcx, 1, 16
	r8_x8cl_eight_complex_fft_final_preload
bab0:	xtouch	[rcx+4096-64]		;; Preload the TLBs
bab:	r8_x8cl_eight_complex_fft_final rsi, 8*64, 64, 2*64, 4*64
	loops	16, bab			;; Test loop counter
	bump	rsi, -16*8*64+dist128	;; Next source pointer
	loops_touch 2, bab, 12, bab0	;; Test loop counter
	jmp	xpass2_6144_real_done

xpass2_6144_levels_real_3:
	mov	rbp, DIST_TO_MULSRCARG
	r8_h8cl_sixteen_reals_eight_complex_with_mult rsi, 8*64, 64, 2*64, 4*64
	loops_init_prefetch 384, 64, 8, rcx, 1, 16
	r8_x8cl_eight_complex_with_mult_preload
	xtouch	[rcx+4096-64]		;; Preload the TLBs
bac:	r8_x8cl_eight_complex_with_mult rsi, 8*64, 64, 2*64, 4*64
	loops	16, bac			;; Test loop counter
	bump	rsi, -16*8*64+dist128	;; Next source pointer
	loops	24, bac			;; Test loop counter
	bump	rsi, -24*dist128	;; Next source pointer
	jmp	xpass2_6144_real_unfft

xpass2_6144_levels_real_4:
	mov	rbp, DIST_TO_MULSRCARG
	mov	rcx, DATA_PREFETCH	;; Load prefetch pointer
	xtouch	[rcx]			;; Preload the TLBs
	r8_h8cl_sixteen_reals_eight_complex_with_mulf rsi, 8*64, 64, 2*64, 4*64
	loops_init_prefetch 384, 128, 1, rcx, 1, 16
	r8_x8cl_eight_complex_with_mulf_preload
bad0:	xtouch	[rcx+4096-64]
bad:	r8_x8cl_eight_complex_with_mulf rsi, 8*64, 64, 2*64, 4*64
	loops	16, bad			;; Test loop counter
	bump	rsi, -16*8*64+dist128	;; Next source pointer
	loops_touch 2, bad, 12, bad0	;; Test loop counter
	bump	rsi, -24*dist128	;; Next source pointer

;; Do inverse FFT levels 9,10
;; On input the 64-byte cache lines hold these data values:
;;	0	+12K	4	+12K	8	+12K	12	+12K
;;	1	...
;;	...
;;	3	...
;;	16	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+12K	16	+12K	32	+12K	48	+12K
;;	1	...
;;	...
;;	15	...
;;	64	...
;;	...

;; Do 8 eight_reals_unfft operations
;; Do 8 four_complex_djbunfft operations
;;	distance between fft data elements is 8
;;	do 4 macros processing 32 data values

xpass2_6144_real_unfft:
	mov	rdi, xsincos_complex	;; Load sin/cos pointers
	bump	rdi, 3072*XMM_SCD1
	mov	rdx, sincos3
	bump	rdx, 2048*XMM_HSCD1
	loops_init_prefetch 768, 64, 2, rcx
	xtouch	[rcx+4096-64]
c8b:	r4_h4cl_eight_reals_four_complex_djbunfft rsi, 64, 4*64, 8*64, rdi, 4*16*XMM_SCD2, rdx, 4*64*XMM_HSCD2
	bump	rdi, 16*XMM_SCD2	;; Next sin/cos data
	bump	rdx, 64*XMM_HSCD2	;; Next sin/cos data
	loops	4, c8b			;; Test loop counter
	bump	rsi, -4*64+16*64	;; Next source pointer
	bump	rdi, -4*16*XMM_SCD2	;; Restore sin/cos ptr
	bump	rdx, -4*64*XMM_HSCD2	;; Restore sin/cos ptr

;; Do 3056 four_complex_djbunfft operations
;;	distance between fft data elements is 8
;;	do 764 macros each processing 32 data values

	loops_reset			;; 24 iters of 7 or 8 of 4
	r4_x4cl_four_complex_djbunfft_preload
c9a:	xtouch	[rcx+4096-64]
c9b:	r4_x4cl_four_complex_djbunfft rsi, 64, 4*64, 8*64, rdi, 4*16*XMM_SCD2
	bump	rdi, 16*XMM_SCD2	;; Next sine/cosine pointer
	loops	4, c9b			;; Test loop counter
	bump	rsi, -4*64+16*64	;; Next source pointer
	bump	rdi, -4*16*XMM_SCD2	;; Restore sin/cos ptr
	loops	8, c9b			;; Test loop counter
	bump	rsi, -8*16*64+dist128	;; Next source pointer
	loops_touch 4, c9b, 6, c9a	;; Test loop counter
	bump	rsi, -24*dist128	;; Next source pointer

;; Do inverse FFT levels 7,8
;; On input the 64-byte cache lines hold these data values:
;;	0	+12K	16	+12K	32	+12K	48	+12K
;;	1	...
;;	...
;;	15	...
;;	64	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+12K	64	+12K	128	+12K	192	+12K
;;	1	...
;;	...
;;	63	...
;;	256	...
;;	...

;; Do 32 eight_reals_unfft operations
;; Do 32 four_complex_djbunfft operations
;;	distance between fft data elements is 32
;;	do 16 macros processing 32 data values

	loops_init_prefetch 768, 64, 2, rcx
	xtouch	[rcx+4096-64]
c6b:	r4_h4cl_eight_reals_four_complex_djbunfft rsi, 64, 16*64, 32*64, rdi, 16*4*XMM_SCD2, rdx, 16*16*XMM_HSCD2
	bump	rdi, 4*XMM_SCD2		;; Next sin/cos data
	bump	rdx, 16*XMM_HSCD2	;; Next sin/cos data
	loops	16, c6b			;; Test inner loop counter
	bump	rsi, -16*64+64*64	;; Next source pointer
	bump	rdi, -16*4*XMM_SCD2	;; Restore sin/cos ptr
	bump	rdx, -16*16*XMM_HSCD2	;; Restore sin/cos ptr

;; Do 3008 four_complex_djbunfft operations
;;	distance between fft data elements is 32
;;	do 752 macros each processing 32 data values

	loops_reset			;; 24 iters of 1 or 2 of 16
	r4_x4cl_four_complex_djbunfft_preload
c7a:	xtouch	[rcx+4096-64]
c7b:	r4_x4cl_four_complex_djbunfft rsi, 64, 16*64, 32*64, rdi, 16*4*XMM_SCD2
	bump	rdi, 4*XMM_SCD2		;; Next sin/cos data
	loops	16, c7b			;; Test inner loop counter
	bump	rsi, -16*64+64*64	;; Next source pointer
	bump	rdi, -16*4*XMM_SCD2	;; Restore sin/cos ptr
	loops	2, c7b			;; Test loop counter
	bump	rsi, -2*64*64+dist128	;; Next source pointer
	loops_touch 4, c7b, 6, c7a	;; 24 iterations
	bump	rsi, -24*dist128	;; Next source pointer

;; Do inverse FFT levels 5,6
;; On input the 64-byte cache lines hold these data values:
;;	0	+12K	64	+12K	128	+12K	192	+12K
;;	1	...
;;	...
;;	63	...
;;	256	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+12K	256	+12K	512	+12K	768	+12K
;;	1	...
;;	...
;;	255	...
;;	1024	...
;;	...

;; Do 128 eight_reals_unfft operations
;; Do 128 four_complex_djbunfft operations
;;	distance between fft data elements is 128
;;	do 64 macros processing 32 data values

	loops_init_prefetch 768, 64, 2, rcx
	xtouch	[rcx+4096-64]
c4b:	r4_h4cl_eight_reals_four_complex_djbunfft rsi, 64, 64*64, dist128, rdi, 64*XMM_SCD2, rdx, 64*4*XMM_HSCD2
	bump	rdi, XMM_SCD2		;; Next sin/cos data
	bump	rdx, 4*XMM_HSCD2	;; Next sin/cos data
	loops	64, c4b			;; Test inner loop counter
	bump	rsi, -64*64+2*dist128	;; Next source pointer
	bump	rdi, -64*XMM_SCD2	;; Restore sin/cos ptr
	bump	rdx, -64*4*XMM_HSCD2	;; Restore sin/cos ptr

;; Do 2816 four_complex_djbunfft operations
;;	distance between fft data elements is 128
;;	do 704 macros each processing 32 data values

	loops_reset			;; 11 more iters of 64
	r4_x4cl_four_complex_djbunfft_preload
c5a:	xtouch	[rcx+4096-64]
c5b:	r4_x4cl_four_complex_djbunfft rsi, 64, 64*64, dist128, rdi, 64*XMM_SCD2
	bump	rdi, XMM_SCD2		;; Next sin/cos data
	loops	64, c5b			;; Test inner loop counter
	bump	rsi, -64*64+2*dist128	;; Next source pointer
	bump	rdi, -64*XMM_SCD2	;; Restore sin/cos ptr
	loops_touch 2, c5b, 6, c5a	;; Test loop counter
	bump	rsi, -12*2*dist128	;; Next source pointer

;; Do inverse FFT levels 3,4
;; On input the 64-byte cache lines hold these data values:
;;	0	+12K	256	+12K	512	+12K	768	+12K
;;	1	...
;;	...
;;	255	...
;;	1024	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+12K	1024	+12K	2048	+12K	3072	+12K
;;	1	...
;;	...
;;	1023	...
;;	4096	...
;;	...

;; Do 512 eight_reals_unfft operations
;; Do 512 four_complex_djbunfft operations
;;	distance between fft data elements is 512
;;	do 256 macros processing 32 data values

	loops_init_prefetch 768, 64, 2, rcx
	bump	rcx, -2*64		;; Back up prefetch pointer by 2 cache lines
	xtouch	[rcx+4096-64]
	mov	rdi, xsincos_complex	;; Load sin/cos pointer
	mov	rbx, rdi
c2b:	r4_h4cl_2sc_eight_reals_four_complex_djbunfft rsi, 64, 2*dist128, 4*dist128, rdi, 256*3*XMM_SCD1, rbx, 256*6*XMM_SCD1, rdx, 256*XMM_HSCD2
	bump	rdi, 3*XMM_SCD1		;; Next sin/cos data
	bump	rbx, 6*XMM_SCD1		;; Next sin/cos data
	bump	rdx, XMM_HSCD2		;; Next sin/cos data
	loops	128, c2b		;; Test inner loop counter
	bump	rsi, -128*64+dist128	;; Next source pointer
	loops	2, c2b			;; Test loop counter
	bump	rsi, -2*dist128+8*dist128 ;; Next source pointer
	bump	rdi, -256*3*XMM_SCD1	;; Restore sin/cos ptr
	bump	rbx, -256*6*XMM_SCD1	;; Restore sin/cos ptr
	bump	rdx, -256*XMM_HSCD2-2048*XMM_HSCD1 ;; Restore sin/cos ptr

;; Do 2048 four_complex_djbunfft operations
;;	distance between fft data elements is 512
;;	do 512 macros each processing 32 data values

	loops_reset			;; 2 more iters of 256
	r4_x4cl_2sc_four_complex_djbunfft_preload
c3a:	xtouch	[rcx+4096-64]
c3b:	r4_x4cl_2sc_four_complex_djbunfft rsi, 64, 2*dist128, 4*dist128, rdi, 256*3*XMM_SCD1, rbx, 256*6*XMM_SCD1
	bump	rdi, 3*XMM_SCD1		;; Next sin/cos data
	bump	rbx, 6*XMM_SCD1		;; Next sin/cos data
	loops	128, c3b		;; Test inner loop counter
	bump	rsi, -128*64+dist128	;; Next source pointer
	loops	2, c3a			;; Test loop counter
	bump	rsi, -2*dist128+8*dist128 ;; Next source pointer
	bump	rdi, -256*3*XMM_SCD1	;; Restore sin/cos ptr
	bump	rbx, -256*6*XMM_SCD1	;; Restore sin/cos ptr
	loops	3, c3a			;; Test loop counter
	bump	rsi, -3*8*dist128	;; Next source pointer

;; Do inverse FFT level 1,2 (actually 1.585 FFT levels)
;; On input the 64-byte cache lines hold these data values:
;;	0	+12K	1024	+12K	2048	+12K	3072	+12K
;;	1	...
;;	...
;;	1023	...
;;	4096	...
;;	...
;; On output the 64-byte cache lines hold these data values if pass 1 swizzles:
;;	0	+12K	3072	+12K	6144	+12K	9216	+12K
;;	1	...
;;	...
;;	3071
;; On output the 64-byte cache lines hold these data values if pass 2 swizzles:
;;	0	+3K	6144	+3K	12288	+3K	18432	+3K
;;	1	...
;;	...
;;	3071

;; Do 2048 six_reals_unfft operations
;; Do 2048 three_complex_djbunfft operations
;;	distance between fft data elements is 2048
;;	do 1024 macros processing 24 data values

	loops_init 1024
c1b:	r3_h3cl_six_reals_three_complex_djbunfft rsi, 64, 8*dist128, rdi, 1024*XMM_SCD1, rdx, 1024*XMM_HSCD1
	bump	rdi, XMM_SCD1		;; Next sin/cos data
	bump	rdx, XMM_HSCD1		;; Next sin/cos data
	loops	128, c1b		;; Test loop counter
	bump	rsi, -128*64+dist128	;; Next source pointer
	loops	8, c1b			;; Test loop counter

xpass2_6144_real_done:
	end_timer 2
	ENDM

; *************** radix4-pass2-6144-levels-complex macro ******************
; This macro takes 2 groups of 6144 complex values and
; performs the final 12.585 levels of a standard mostly radix-4 FFT, squares the
; results, and does the first 12.585 levels of the inverse FFT.

r4_pass2_6144_levels_complex MACRO
	LOCAL	b1a0, b1a, b1b, b2a, b2b, b3a, b3b, b4b, b5b, b6a, b6b0, b6b, b6c, b6d0, b6d
	LOCAL	c1b, c2a0, c2a, c2b, c3b, c4b, c5a, c5b
	LOCAL	xpass2_6144_levels_complex_1, xpass2_6144_levels_complex_2
	LOCAL	xpass2_6144_levels_complex_3, xpass2_6144_levels_complex_4
	LOCAL	xpass2_6144_levels_complex_unfft, xpass2_6144_levels_complex_done

;; Type 4 FFTs skip the forward FFT process

	mov	rsi, DATA_ADDR		;; Load address of FFT data
	mov	rbx, DIST_TO_FFTSRCARG
	cmp	ffttype, 4
	je	xpass2_6144_levels_complex_4

;; Do FFT levels 1,2 (actually 1.585 levels)
;;
;; We swizzle the data so that one set of data is in the high word of
;; an xmm register, the other set is in the low word.  This means
;; a distance of 12288 data values.
;;
;; On input the 64-byte cache lines hold these data values if pass 2 swizzles:
;;	0	+3K	6144	+3K	12288	+3K	18432	+3K
;;	1	...
;;	...
;;	3071
;; On input the 64-byte cache lines hold these data values if pass 1 swizzles:
;;	0	+12K	3072	+12K	6144	+12K	9216	+12K
;;	1	...
;;	...
;;	3071	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+12K	1024	+12K	2048	+12K	3072	+12K
;;	1	...
;;	...
;;	1023	...
;;	4096	...
;;	...

;; Do 4096 three_complex_fft operations
;;	distance between fft data elements is 2048
;;	do 1024 macros each processing 24 data values

;; Do FFT levels 3,4
;;
;; On input the 64-byte cache lines hold these data values:
;;	0	+12K	1024	+12K	2048	+12K	3072	+12K
;;	1	...
;;	...
;;	1023	...
;;	4096	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+12K	512	+12K	1024	+12K	1536	+12K
;;	1	...
;;	...
;;	511	...
;;	2048	...
;;	...

;; Do 3072 four_complex_fft operations
;;	distance between fft data elements is 512
;;	do 1536 macros each processing 16 data values

IF @INSTR(,%xarch,<CORE>) NE 0		;; Core 2 faster doing separate loops

	start_timer 4
	mov	rdi, xsincos_complex	;; The sin/cos data
	loops_init 1024			;; 8 iterations of 128
	r3_f3cl_three_complex_djbfft_preload
b1b:	r3_f3cl_three_complex_djbfft rsi, 64, 8*dist128, rdi, 1024*XMM_SCD1
	bump	rdi, XMM_SCD1		;; Next sin/cos data
	loops	128, b1b		;; Test inner loop counter
	bump	rsi, -128*64+dist128	;; Next source pointer
	loops	8, b1b			;; Test outer loop counter
	bump	rsi, -8*dist128		;; Restore source pointer
	end_timer 4

	start_timer 5
	mov	rcx, DATA_PREFETCH	;; Prefetch pointer (we need to prefetch 
					;; 3072 cache lines + 46 pad cache lines)
	xtouch	[rcx]			;; Load first prefetch TLB
	mov	rdi, xsincos_complex	;; Ptr to the sin/cos data
	mov	rdx, rdi
	loops_init_prefetch 1536, 64, 4, rcx ;; 3 iters of 4 iters of 128
	r4_x2cl_2sc_four_complex_djbfft_preload
b2a:	xtouch	[rcx+4096-64]		;; Load prefetch TLBs
b2b:	r4_x2cl_2sc_four_complex_djbfft rsi, 64, 4*dist128, rdi, rdx
	bump	rdi, 3*XMM_SCD1		;; Next sin/cos data
	bump	rdx, 6*XMM_SCD1		;; Next sin/cos data
	loops	128, b2b		;; Test inner loop counter
	bump	rsi, -128*64+dist128	;; Next source pointer
	loops_touch 2, b2b, 2, b2a	;; Test loop counter
	bump	rsi, -4*dist128+8*dist128 ;; Next source pointer
	bump	rdi, -512*3*XMM_SCD1	;; Restore sin/cos ptr
	bump	rdx, -512*6*XMM_SCD1	;; Restore sin/cos ptr
	loops	3, b2a			;; Test loop counter
	bump	rsi, -3*8*dist128	;; Restore source pointer
	end_timer 5

ELSE					;; Pentium 4 faster doing merged loops

	start_timer 4
	mov	rdi, xsincos_complex	;; The sin/cos data
	mov	rbp, rdi
	mov	rdx, rdi
	mov	rcx, DATA_PREFETCH	;; Prefetch pointer (we need to prefetch 
					;; 3072 cache lines + 46 pad cache lines)
	xtouch	[rcx]			;; Load first prefetch TLB
	loops_init_prefetch 3072, 64, 12, rcx ;; 512 iters of 2 or 3

b1a0:	xtouch	[rcx+4096-64]		;; Load prefetch TLB

b1a:	r3_f3cl_three_complex_djbfft_preload
b1b:	r3_f3cl_three_complex_djbfft rsi, 4*dist128, 8*dist128, rdi, 1024*XMM_SCD1
	xxprefetch [rdi+XMM_SCD1]	;; Prefetch for next pass
	xxprefetch [rdi+1025*XMM_SCD1]	;; Prefetch for next pass
	bump	rdi, 512*XMM_SCD1	;; Next sin/cos data
	loops	2, b1b, 3		;; Test loop counter with non-standard incr
	bump	rsi, -2*4*dist128	;; Restore source pointer
	loops_undo 6

	r4_x2cl_2sc_four_complex_djbfft_preload
b2b:	r4_x2cl_2sc_four_complex_djbfft rsi, 8*dist128, 4*dist128, rbp, rdx
	loops	3, b2b, 2		;; Test loop counter with non-standard incr

	bump	rsi, -3*8*dist128+64	;; Next source pointer
	bump	rdi, -2*512*XMM_SCD1+XMM_SCD1 ;; Next sin/cos data
	bump	rbp, 3*XMM_SCD1		;; Next sin/cos data
	bump	rdx, 6*XMM_SCD1		;; Next sin/cos data
	loops	128, b1a		;; Test loop counter
	bump	rsi, -128*64+dist128	;; Next source pointer
	loops	4, b1a0			;; Test loop counter
	bump	rsi, -4*dist128		;; Restore source pointer
	bump	rdi, -512*XMM_SCD1	;; Restore sin/cos ptr
	end_timer 4

ENDIF

;; Do FFT levels 5,6
;;
;; On input the 64-byte cache lines hold these data values:
;;	0	+12K	512	+12K	1024	+12K	1536	+12K
;;	1	...
;;	...
;;	511	...
;;	2048	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+12K	128	+12K	256	+12K	384	+12K
;;	1	...
;;	...
;;	127	...
;;	512	...
;;	...
;;
;; Do 3072 four_complex_fft operations
;;	distance between fft data elements is 128
;;	do 768 macros each processing 32 data values

;; Do FFT levels 7,8
;;
;; On input the 64-byte cache lines hold these data values:
;;	0	+12K	128	+12K	256	+12K	384	+12K
;;	1	...
;;	...
;;	127	...
;;	512	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+12K	32	+12K	64	+12K	96	+12K
;;	1	...
;;	...
;;	31	...
;;	128	...
;;	...
;;
;; Do 3072 four_complex_fft operations
;;	distance between fft data elements is 32
;;	do 768 macros each processing 32 data values

;; Do FFT levels 9,10
;;
;; On input the 64-byte cache lines hold these data values:
;;	0	+12K	32	+12K	64	+12K	96	+12K
;;	1	...
;;	...
;;	31	...
;;	128	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+12K	8	+12K	16	+12K	24	+12K
;;	1	...
;;	...
;;	7	...
;;	32	...
;;	...
;;
;; Do 3072 four_complex_fft operations
;;	distance between fft data elements is 8
;;	do 768 macros each processing 32 data values

;; Do this 64 cache lines at a time to maximize accessing data
;; while it is in the L1 cache.

	start_timer 6
	bump	rdi, 3072*XMM_SCD1	;; Load sin/cos pointers
	mov	rbx, rdi
	mov	rbp, rdi
	loops_init_prefetch 768, 64, 2, rcx ;; 6*8 iters of 64 cache lines
	r4_x4cl_four_complex_djbfft_preload

b3a:	xtouch	[rcx+4096-64]		;; Load prefetch TLBs
	xtouch	[rcx+2*4096-64]
	xtouch	[rcx+3*4096-64]

b3b:	xxprefetch [rdi+8*XMM_SCD2]	;; Prefetch for next pass
	r4_x4cl_four_complex_djbfft rsi, 8*64, dist128, 2*dist128, rdi
	bump	rdi, 8*XMM_SCD2		;; Next sine/cosine pointer
	loops	16, b3b			;; Loop 16 times
	bump	rsi, -16*8*64		;; Restore source pointer
	bump	rdi, -16*8*XMM_SCD2	;; Restore sine/cosine pointer
	loops_undo 16			;; Back up loop counter

b4b:	r4_x4cl_four_complex_djbfft rsi, 8*64, 32*64, 64*64, rbx
	bump	rbx, 4*8*XMM_SCD2	;; Next sine/cosine pointer
	loops	4, b4b			;; Loop 4 times
	bump	rsi, -4*8*64+dist128	;; Next source pointer
	bump	rbx, -4*4*8*XMM_SCD2	;; Restore sine/cosine pointer
	loops	4, b4b			;; Loop twice
	bump	rsi, -4*dist128		;; Restore source pointer
	loops_undo 4*4			;; Back up loop counter

b5b:	r4_x4cl_four_complex_djbfft rsi, 32*64, 8*64, 16*64, rbp
	loops	4, b5b			;; Loop 4 times
	bump	rsi, -4*32*64+dist128	;; Next source pointer
	loops	4, b5b			;; Loop 4 times

	bump	rsi, -4*dist128+64	;; Next source pointer
	bump	rdi, XMM_SCD2		;; Next sine/cosine pointer
	bump	rbx, 4*XMM_SCD2		;; Next sine/cosine pointer
	bump	rbp, 16*XMM_SCD2	;; Next sine/cosine pointer
	loops	8, b3b			;; Loop 8 times
	bump	rsi, -8*64+4*dist128	;; Next source pointer
	bump	rdi, -8*XMM_SCD2	;; Restore sine/cosine pointer
	bump	rbx, -8*4*XMM_SCD2	;; Restore sine/cosine pointer
	bump	rbp, -8*16*XMM_SCD2	;; Restore sine/cosine pointer
	loops	6, b3a			;; Loop 6 times
	bump	rsi, -6*4*dist128	;; Restore source pointer
	end_timer 6

;; Do FFT levels 11,12,13 as well as inverse FFT levels 11,12,13
;;
;; On input the 64-byte cache lines hold these data values:
;;	0	+12K	8	+12K	16	+12K	24	+12K
;;	1	...
;;	...
;;	7	...
;;	32	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+12K	4	+12K	8	+12K	12	+12K
;;	1	...
;;	...
;;	3	...
;;	16	...
;;	...

;; Do 1536 eight_complex_fft operations
;;	distance between fft data elements is 1
;;	do 384 macros each processing 64 data values

;; Execute the right middle step

	cmp	ffttype, 2
	jl	xpass2_6144_levels_complex_1
	jg	xpass2_6144_levels_complex_3

xpass2_6144_levels_complex_2:
	start_timer 9
	loops_init_prefetch 384, 64, 8, rcx
	r8_x8cl_eight_complex_with_square_preload
	xtouch	[rcx+4096-64]		;; Load prefetch TLB
b6a:	r8_x8cl_eight_complex_with_square rsi, 8*64, 64, 2*64, 4*64
	loops	16, b6a			;; Loop 16 times
	bump	rsi, -16*8*64+dist128	;; Next source pointer
	loops	24, b6a			;; Loop 24 times
	bump	rsi, -24*dist128	;; Restore source pointer
	end_timer 9
	jmp	xpass2_6144_levels_complex_unfft

xpass2_6144_levels_complex_1:
	start_timer 9
	loops_init_prefetch 384, 128, 1, rcx
	r8_x8cl_eight_complex_fft_final_preload
b6b0:	xtouch	[rcx+4096-64]		;; Load prefetch TLBs
b6b:	r8_x8cl_eight_complex_fft_final rsi, 8*64, 64, 2*64, 4*64
	loops	16, b6b			;; Loop 16 times
	bump	rsi, -16*8*64+dist128	;; Next source pointer
	loops_touch 2, b6b, 12, b6b0	;; Loop 24 times
	end_timer 9
	jmp	xpass2_6144_levels_complex_done

xpass2_6144_levels_complex_3:
	start_timer 9
	mov	rbp, DIST_TO_MULSRCARG
	loops_init_prefetch 384, 64, 8, rcx
	r8_x8cl_eight_complex_with_mult_preload
	xtouch	[rcx+4096-64]		;; Load prefetch TLBs
b6c:	r8_x8cl_eight_complex_with_mult rsi, 8*64, 64, 2*64, 4*64
	loops	16, b6c			;; Loop 16 times
	bump	rsi, -16*8*64+dist128	;; Next source pointer
	loops	24, b6c			;; Loop 24 times
	bump	rsi, -24*dist128	;; Restore source pointer
	end_timer 9
	jmp	xpass2_6144_levels_complex_unfft

xpass2_6144_levels_complex_4:
	start_timer 9
	mov	rcx, DATA_PREFETCH	;; Prefetch pointer
	xtouch	[rcx]			;; Load first prefetch TLB
	mov	rbp, DIST_TO_MULSRCARG
	loops_init_prefetch 384, 128, 1, rcx
	r8_x8cl_eight_complex_with_mulf_preload
b6d0:	xtouch	[rcx+4096-64]		;; Load prefetch TLBs
b6d:	r8_x8cl_eight_complex_with_mulf rsi, 8*64, 64, 2*64, 4*64
	loops	16, b6d			;; Loop 16 times
	bump	rsi, -16*8*64+dist128	;; Next source pointer
	loops_touch 2, b6d, 12, b6d0	;; Loop 24 times
	bump	rsi, -24*dist128	;; Restore source pointer
	end_timer 9

;; Do inverse FFT levels 9,10
;; On input the 64-byte cache lines hold these data values:
;;	0	+12K	4	+12K	8	+12K	12	+12K
;;	1	...
;;	...
;;	3	...
;;	16	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+12K	16	+12K	32	+12K	48	+12K
;;	1	...
;;	...
;;	15	...
;;	64	...
;;	...
;;
;; Do 3072 four_complex_unfft operations
;;	distance between fft data elements is 8
;;	do 768 macros each processing 32 data values

;; Do inverse FFT levels 7,8
;; On input the 64-byte cache lines hold these data values:
;;	0	+12K	16	+12K	32	+12K	48	+12K
;;	1	...
;;	...
;;	15	...
;;	64	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+12K	64	+12K	128	+12K	192	+12K
;;	1	...
;;	...
;;	63	...
;;	256	...
;;	...
;;
;; Do 3072 four_complex_unfft operations
;;	distance between fft data elements is 32
;;	do 768 macros each processing 32 data values

;; Do inverse FFT levels 5,6
;; On input the 64-byte cache lines hold these data values:
;;	0	+12K	64	+12K	128	+12K	192	+12K
;;	1	...
;;	...
;;	63	...
;;	256	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+12K	256	+12K	512	+12K	768	+12K
;;	1	...
;;	...
;;	255	...
;;	1024	...
;;	...
;;
;; Do 3072 four_complex_unfft operations
;;	distance between fft data elements is 128
;;	do 768 macros each processing 32 data values

;; Do this 64 cache lines at a time to maximize accessing data
;; while it is in the L1 cache.

xpass2_6144_levels_complex_unfft:
	start_timer 10
	mov	rdi, xsincos_complex	;; Load sin/cos pointers
	bump	rdi, 3072*XMM_SCD1
	mov	rbx, rdi
	mov	rbp, rdi
	loops_init_prefetch 768, 64, 2, rcx ;; 12*4 iters of 64 cache lines
	r4_x4cl_four_complex_djbunfft_preload

c5a:	xtouch	[rcx+4096-64]		;; Load prefetch TLBs
	xtouch	[rcx+2*4096-64]
	xtouch	[rcx+3*4096-64]

c5b:	r4_x4cl_four_complex_djbunfft rsi, 16*64, 4*64, 8*64, rbp, 4*16*XMM_SCD2
	loops	8, c5b			;; Loop 8 times
	bump	rsi, -8*16*64+dist128	;; Next source pointer
	loops	2, c5b			;; Loop 2 times
	bump	rsi, -2*dist128		;; Restore source pointer
	loops_undo 2*8			;; Restore loop counter

c4b:	r4_x4cl_four_complex_djbunfft rsi, 4*64, 16*64, 32*64, rbx, 16*4*XMM_SCD2
	bump	rbx, 4*4*XMM_SCD2	;; Next sine/cosine pointer
	loops	4, c4b			;; Loop 4 times
	bump	rsi, -4*4*64+64*64	;; Next source pointer
	bump	rbx, -4*4*4*XMM_SCD2	;; Restore sine/cosine pointer
	loops	2, c4b			;; Loop twice
	bump	rsi, -2*64*64+dist128	;; Next source pointer
	loops	2, c4b			;; Loop twice
	bump	rsi, -2*dist128		;; Restore source pointer
	loops_undo 2*2*4		;; Restore loop counter

c3b:	r4_x4cl_four_complex_djbunfft rsi, 4*64, 64*64, dist128, rdi, 64*1*XMM_SCD2
	bump	rdi, 4*1*XMM_SCD2	;; Next sine/cosine pointer
	loops	16, c3b			;; Loop 16 times
	bump	rdi, -16*4*1*XMM_SCD2	;; Restore sine/cosine pointer

	bump	rsi, -16*4*64+64	;; Next source pointer
	bump	rdi, 1*XMM_SCD2		;; Next sine/cosine pointer
	bump	rbx, 4*XMM_SCD2		;; Next sine/cosine pointer
	bump	rbp, 16*XMM_SCD2	;; Next sine/cosine pointer
	loops	4, c5b			;; Loop 4 times
	bump	rsi, -4*64+2*dist128	;; Next source pointer
	bump	rdi, -4*1*XMM_SCD2	;; Restore sine/cosine pointer
	bump	rbx, -4*4*XMM_SCD2	;; Restore sine/cosine pointer
	bump	rbp, -4*16*XMM_SCD2	;; Restore sine/cosine pointer
	loops_touch 2, c5b, 6, c5a	;; Loop 12 times
	bump	rsi, -12*2*dist128	;; Restore source pointer
	end_timer 10

;; Do inverse FFT levels 3,4
;; On input the 64-byte cache lines hold these data values:
;;	0	+12K	256	+12K	512	+12K	768	+12K
;;	1	...
;;	...
;;	255	...
;;	1024	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+12K	1024	+12K	2048	+12K	3072	+12K
;;	1	...
;;	...
;;	1023	...
;;	4096	...
;;	...

;; Do 3072 four_complex_unfft operations
;;	distance between fft data elements is 512
;;	do 768 macros each processing 32 data values

;; Do inverse FFT levels 1,2 (actually 1.585 FFT levels)
;; On input the 64-byte cache lines hold these data values:
;;	0	+12K	1024	+12K	2048	+12K	3072	+12K
;;	1	...
;;	...
;;	1023	...
;;	4096	...
;;	...
;; On output the 64-byte cache lines hold these data values if pass 1 swizzles:
;;	0	+12K	3072	+12K	6144	+12K	9216	+12K
;;	1	...
;;	...
;;	3071
;; On output the 64-byte cache lines hold these data values if pass 2 swizzles:
;;	0	+3K	6144	+3K	12288	+3K	18432	+3K
;;	1	...
;;	...
;;	3071

;; Do 4096 three_complex_unfft operations
;;	distance between fft data elements is 2048
;;	do 1024 macros each processing 24 data values

IF @INSTR(,%xarch,<CORE>) NE 0		;; Core 2 faster doing separate loops

	start_timer 13
	mov	rdi, xsincos_complex	;; Load sin/cos pointer
	mov	rdx, rdi
	loops_init_prefetch 768, 64, 2, rcx;; 3 iters of 128
	bump	rcx, -2*64		;; Back up prefetch pointer by 2 cache lines
	r4_x4cl_2sc_four_complex_djbunfft_preload
c2a:	xtouch	[rcx+4096-64]		;; Load prefetch TLBs
c2b:	r4_x4cl_2sc_four_complex_djbunfft rsi, 64, 2*dist128, 4*dist128, rdi, 256*3*XMM_SCD1, rdx, 256*6*XMM_SCD1
	bump	rdi, 3*XMM_SCD1		;; Next sine/cosine pointer
	bump	rdx, 6*XMM_SCD1		;; Next sine/cosine pointer
	loops	128, c2b		;; Loop 128 times
	bump	rsi, -128*64+dist128	;; Next source pointer
	loops	2, c2a			;; Loop 2 times
	bump	rsi, -2*dist128+8*dist128 ;; Next source pointer
	bump	rdi, -256*3*XMM_SCD1	;; Reset sine/cosine pointer
	bump	rdx, -256*6*XMM_SCD1	;; Reset sine/cosine pointer
	loops	3, c2a			;; Test loop counter
	bump	rsi, -3*8*dist128	;; Restore source pointer
	end_timer 13

	start_timer 14
	mov	rdi, xsincos_complex	;; Load group multiplier pointer
	loops_init 1024			;; 8 iters of 128
	xtouch	[rcx+4096-128]		;; Load prefetch TLBs
	r3_x3cl_three_complex_djbunfft_preload
c1b:	r3_x3cl_three_complex_djbunfft rsi, 64, 8*dist128, rdi, 1024*XMM_SCD1
	bump	rdi, XMM_SCD1		;; Next set of sin/cos data
	loops	128, c1b		;; Test inner loop counter
	bump	rsi, -128*64+dist128	;; Next source pointer
	loops	8, c1b			;; Test outer loop counter
	end_timer 14

ELSE					;; Pentium 4 faster doing merged loops

	start_timer 13
	mov	rdi, xsincos_complex	;; Load sin/cos pointer
	mov	rbp, rdi
	mov	rdx, rdi
	bump	rcx, -2*64		;; Back up prefetch pointer by 2 cache lines
	loops_init_prefetch 3072, 64, 6, rcx ;; 256 iters of 3 or 4

c2a0:	xtouch	[rcx+4096-64]		;; Load prefetch TLBs

	loops_set_prefetch 64, 3072, rcx ;; prefetch 0 cache lines in first loop
c2a:	r4_x4cl_2sc_four_complex_djbunfft_preload
c2b:	r4_x4cl_2sc_four_complex_djbunfft rsi, 8*dist128, 2*dist128, 4*dist128, rbp, 256*3*XMM_SCD1, rdx, 256*6*XMM_SCD1
	loops_no_prefetch 3, c2b, 4	;; Test loop counter with non-standard incr
	bump	rsi, -3*8*dist128	;; Restore source pointer
	loops_undo 12

	loops_set_prefetch 64, 6, rcx	;; prefetch 512 cache lines in second loop
	r3_x3cl_three_complex_djbunfft_preload
c1b:	r3_x3cl_three_complex_djbunfft rsi, 2*dist128, 8*dist128, rdi, 1024*XMM_SCD1
	bump	rdi, 256*XMM_SCD1	;; Next sin/cos ptr
	loops	4, c1b, 3		;; Test loop counter with non-standard incr

	bump	rsi, -4*2*dist128+64	;; Next source pointer
	bump	rdi, -4*256*XMM_SCD1+XMM_SCD1 ;; Next sin/cos ptr
	bump	rbp, 3*XMM_SCD1		;; Next sine/cosine pointer
	bump	rdx, 6*XMM_SCD1		;; Next sine/cosine pointer
	loops_touch 32, c2a, 4, c2a0	;; Loop 128 times
	bump	rsi, -128*64+dist128	;; Next source pointer
	loops	2, c2a0			;; Loop 2 times
	end_timer 13

ENDIF

xpass2_6144_levels_complex_done:
	ENDM


; *************** radix4-pass2-7680-levels-real macro ******************
; This macro takes 15360 real values and performs the final 13.907 levels of
; the FFT.  Independently, this macro also takes 7680 complex values and
; performs the final 12.907 levels of the FFT.  The results are squared, and
; the first 12.907/13.907 levels of the inverse FFT are done.

r4_pass2_7680_levels_real MACRO
	LOCAL	b1b, b2a, b2b, b3a, b3b, b4b, b5a, b5b, b6b, b7a, b7b
	LOCAL	b8b, b9a, b9b, baa, bab0, bab, bac, bad0, bad
	LOCAL	c1b, c2b, c3a, c3b, c4b, c5a, c5b, c6b, c7a, c7b, c8b, c9a, c9b
	LOCAL	xpass2_7680_levels_real_1, xpass2_7680_levels_real_2
	LOCAL	xpass2_7680_levels_real_3, xpass2_7680_levels_real_4
	LOCAL	xpass2_7680_real_unfft, xpass2_7680_real_done

;; We switch to the same format used in one-pass FFTs (that is, the
;; high word of xmm registers being 1 greater than the low word):

;; Type 4 FFTs skip the forward FFT process

	start_timer 2
	mov	rsi, DATA_ADDR		;; Load source address
	mov	rbx, DIST_TO_FFTSRCARG
	cmp	ffttype, 4
	je	xpass2_7680_levels_real_4

;; Do FFT level 1,2 (actually 1.585 levels)
;;
;; On input the 64-byte cache lines hold these data values if pass 2 swizzles:
;;	0	+3840	7680	+3840	15360	+3840	23040	+3840
;;	1	...
;;	...
;;	3839
;; On input the 64-byte cache lines hold these data values if pass 1 swizzles:
;;	0	+15360	3840	+15360	7680	+15360	11520	+15360
;;	1	...
;;	...
;;	3839	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+15360	1280	+15360	2560	+15360	3840	+15360
;;	1	...
;;	...
;;	1279	...
;;	5120	...
;;	...

;; Do 2560 six_reals_fft operations
;; Do 2560 three_complex_djbfft operations
;;	distance between fft data elements is 2560
;;	do 1280 iterations

	mov	rdi, xsincos_complex	;; Ptr to the sin/cos data
	mov	rdx, sincos3		;; Ptr to real sin/cos data
	loops_init 1280
b1b:	r3_fh3cl_six_reals_three_complex_djbfft rsi, 64, 10*dist128, rdi, 1280*XMM_SCD1, rdx, 1280*XMM_HSCD1
	bump	rdi, XMM_SCD1		;; Next sin/cos data
	bump	rdx, XMM_HSCD1		;; Next sin/cos data
	loops	128, b1b		;; Test loop counter
	bump	rsi, -128*64+dist128	;; Next source pointer
	loops	10, b1b			;; Test loop counter
	bump	rsi, -10*dist128	;; Restore source pointer
	bump	rdi, -1280*XMM_SCD1	;; Restore sin/cos ptr
	bump	rdx, 1280*XMM_HSCD1	;; Next sin/cos ptr

;; Do FFT levels 3,4 (actually 2.322 FFT levels)
;;
;; On input the 64-byte cache lines hold these data values:
;;	0	+15360	1280	+15360	2560	+15360	3840	+15360
;;	1	...
;;	...
;;	1279	...
;;	5120	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+15360	256	+15360	512	+15360	768	+15360
;;	1	...
;;	...
;;	255	...
;;	1024	...
;;	...

;; Do 512 ten_reals_fft operations
;; Do 512 five_complex_djbfft operations
;;	distance between fft data elements is 512
;;	do 256 macros each processing 40 data values

	mov	rcx, DATA_PREFETCH	;; Prefetch pointer
	xtouch	[rcx]			;; Preload first TLB
	mov	rbx, rdi
	loops_init_prefetch 768, 64, 2, rcx
b2a:	xtouch	[rcx+4096-64]		;; Preload the TLBs
b2b:	r5_h5cl_2sc_ten_reals_five_complex_djbfft rsi, 64, 2*dist128, rdi, 256*3*XMM_SCD1, rbx, 256*6*XMM_SCD1, rdx, 256*XMM_HSCD2
	bump	rdi, 3*XMM_SCD1		;; Next sin/cos data
	bump	rbx, 6*XMM_SCD1		;; Next sin/cos data
	bump	rdx, XMM_HSCD2		;; Next sin/cos data
	loops	128, b2b		;; Test inner loop counter
	bump	rsi, -128*64+dist128	;; Next source pointer
	loops	2, b2a			;; Test loop counter
	bump	rsi, -2*dist128+10*dist128 ;; Next source pointer
	bump	rdi, -256*3*XMM_SCD1	;; Restore sin/cos ptr
	bump	rbx, -256*6*XMM_SCD1	;; Restore sin/cos ptr
	bump	rdx, 256*XMM_HSCD2	;; Next sin/cos ptr

;; Do 2048 five_complex_fft operations
;;	distance between fft data elements is 512
;;	do 512 macros each processing 40 data values

	loops_reset
	r5_nf5cl_2sc_five_complex_djbfft_preload
b3a:	xtouch	[rcx+4096-64]		;; Preload the TLBs
b3b:	r5_nf5cl_2sc_five_complex_djbfft rsi, 64, 2*dist128, rdi, 256*3*XMM_SCD1, rbx, 256*6*XMM_SCD1
	bump	rdi, 3*XMM_SCD1		;; Next sin/cos data
	bump	rbx, 6*XMM_SCD1		;; Next sin/cos data
	loops	128, b3b		;; Test inner loop counter
	bump	rsi, -128*64+dist128	;; Next source pointer
	loops	2, b3a			;; Test loop counter
	bump	rsi, -2*dist128+10*dist128 ;; Next source pointer
	bump	rdi, -256*3*XMM_SCD1	;; Restore sin/cos ptr
	bump	rbx, -256*6*XMM_SCD1	;; Restore sin/cos ptr
	loops	3, b3a			;; Test loop counter
	bump	rsi, -3*10*dist128	;; Restore source pointer

;; Do FFT levels 5,6
;;
;; On input the 64-byte cache lines hold these data values:
;;	0	+15360	256	+15360	512	+15360	768	+15360
;;	1	...
;;	...
;;	255	...
;;	1024	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+15360	128	+15360	256	+15360	384	+15360
;;	1	...
;;	...
;;	127	...
;;	512	...
;;	...

;; Do 128 eight_reals_fft operations
;; Do 384 four_complex_djbfft operations
;;	distance between fft data elements is 128
;;	do 128 macros each processing 16 data values

	bump	rdi, 3072*XMM_SCD1	;; Load sin/cos pointer
	loops_init_prefetch 1920, 64, 4, rcx
	xtouch	[rcx+4096-64]		;; Preload the TLBs
b4b:	r4_h2cl_eight_reals_four_complex_djbfft rsi, 64, dist128, rdi, rdx
	bump	rdi, XMM_SCD2		;; Next sin/cos data
	bump	rdx, XMM_HSCD2		;; Next sin/cos data
	loops	128, b4b		;; Test loop counter
	bump	rsi, -128*64+2*dist128	;; Next source pointer
	bump	rdi, -128*XMM_SCD2	;; Restore sin/cos ptr
	bump	rdx, -128*XMM_HSCD2	;; Restore sin/cos ptr

;; Do 3584 four_complex_djbfft operations
;;	distance between fft data elements is 128
;;	do 1792 macros each processing 16 data values

	loops_reset			;; 14 more iterations of 128
	r4_nf2cl_four_complex_djbfft_preload
b5a:	xtouch	[rcx+4096-64]		;; Preload the TLBs
b5b:	r4_nf2cl_four_complex_djbfft rsi, 64, dist128, rdi
	bump	rdi, XMM_SCD2		;; Next sin/cos data
	loops	128, b5b		;; Test inner loop counter
	bump	rsi, -128*64+2*dist128	;; Next source pointer
	bump	rdi, -128*XMM_SCD2	;; Restore sin/cos ptr
	loops	15, b5a			;; Test loop counter
	bump	rsi, -15*2*dist128	;; Next source pointer

;; Do FFT levels 7,8
;;
;; On input the 64-byte cache lines hold these data values:
;; On input the 64-byte cache lines hold these data values:
;;	0	+15360	128	+15360	256	+15360	384	+15360
;;	1	...
;;	...
;;	127	...
;;	512	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+15360	32	+15360	64	+15360	96	+15360
;;	1	...
;;	...
;;	31	...
;;	128	...
;;	...

;; Do 32 eight_reals_fft operations
;; Do 96 four_complex_djbfft operations
;;	distance between fft data elements is 32
;;	do 32 macros each processing 32 data values

	loops_init_prefetch 960, 64, 2, rcx
	xtouch	[rcx+4096-64]		;; Preload the TLBs
	r4_h4cl_eight_reals_four_complex_djbfft_preload
b6b:	r4_h4cl_eight_reals_four_complex_djbfft rsi, 64, 32*64, 64*64, rdi, rdx
	bump	rdi, 4*XMM_SCD2		;; Next sin/cos data
	bump	rdx, 4*XMM_HSCD2	;; Next sin/cos data
	loops	32, b6b			;; Test loop counter
	bump	rsi, -32*64+dist128	;; Next source pointer
	bump	rdi, -32*4*XMM_SCD2	;; Restore sin/cos ptr
	bump	rdx, -32*4*XMM_HSCD2	;; Restore sin/cos ptr

;; Do 3712 four_complex_djbfft operations
;;	distance between fft data elements is 32
;;	do 928 macros each processing 32 data values

	loops_reset			;; 29 more iters of 32
	r4_x4cl_four_complex_djbfft_preload
b7a:	xtouch	[rcx+4096-64]		;; Preload the TLBs
b7b:	r4_x4cl_four_complex_djbfft rsi, 64, 32*64, 64*64, rdi
	bump	rdi, 4*XMM_SCD2		;; Next sin/cos data
	loops	32, b7b			;; Test inner loop counter
	bump	rsi, -32*64+dist128	;; Next source pointer
	bump	rdi, -32*4*XMM_SCD2	;; Restore sin/cos ptr
	loops_touch 2, b7b, 15, b7a	;; 30 iterations
	bump	rsi, -30*dist128	;; Next source pointer

;; Do FFT levels 9,10
;;
;; On input the 64-byte cache lines hold these data values:
;;	0	+15360	32	+15360	64	+15360	96	+15360
;;	1	...
;;	...
;;	31	...
;;	128	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+15360	8	+15360	16	+15360	24	+15360
;;	1	...
;;	...
;;	7	...
;;	32	...
;;	...

;; Do 8 eight_reals_fft operations
;; Do 24 four_complex_djbfft operations
;;	distance between fft data elements is 8
;;	do 8 macros each processing 32 data values

	loops_init_prefetch 960, 64, 2, rcx
	xtouch	[rcx+4096-64]		;; Preload the TLBs
	r4_h4cl_eight_reals_four_complex_djbfft_preload
b8b:	r4_h4cl_eight_reals_four_complex_djbfft rsi, 64, 8*64, 16*64, rdi, rdx
	bump	rdi, 16*XMM_SCD2	;; Next sin/cos data
	bump	rdx, 16*XMM_HSCD2	;; Next sin/cos data
	loops	8, b8b			;; Test loop counter
	bump	rsi, -8*64+32*64	;; Next source pointer
	bump	rdi, -8*16*XMM_SCD2	;; Restore sin/cos ptr
	bump	rdx, -8*16*XMM_HSCD2	;; Restore sin/cos ptr

;; Do 3808 four_complex_djbfft operations
;;	distance between fft data elements is 8
;;	do 952 macros each processing 32 data values

	loops_reset			;; 30 iters of 3 or 4 of 8
	r4_x4cl_four_complex_djbfft_preload
b9a:	xtouch	[rcx+4096-64]		;; Preload the TLBs
b9b:	r4_x4cl_four_complex_djbfft rsi, 64, 8*64, 16*64, rdi
	bump	rdi, 16*XMM_SCD2	;; Next sin/cos data
	loops	8, b9b			;; Test loop counter
	bump	rsi, -8*64+32*64	;; Next source pointer
	bump	rdi, -8*16*XMM_SCD2	;; Restore sin/cos ptr
	loops	4, b9b			;; Test middle loop counter
	bump	rsi, -4*32*64+dist128	;; Next source pointer
	loops_touch 2, b9b, 15, b9a	;; 30 iterations
	bump	rsi, -30*dist128	;; Next source pointer

;; Do FFT levels 11,12,13
;;
;; On input the 64-byte cache lines hold these data values:
;;	0	+15360	8	+15360	16	+15360	24	+15360
;;	1	...
;;	...
;;	7	...
;;	32	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+15360	4	+15360	8	+15360	12	+15360
;;	1	...
;;	...
;;	3	...
;;	16	...
;;	...

;; Do 1 sixteen_reals_fft operations
;; Do 3 eight_complex_fft operations
;;	distance between fft data elements is 1

;; Do 1916 eight_complex_fft operations
;;	distance between fft data elements is 1
;;	do 479 macros each processing 64 data values

;; Execute the proper middle step

	cmp	ffttype, 2
	jl	xpass2_7680_levels_real_1
	jg	xpass2_7680_levels_real_3

xpass2_7680_levels_real_2:
	r8_h8cl_sixteen_reals_eight_complex_with_square rsi, 8*64, 64, 2*64, 4*64
	loops_init_prefetch 480, 64, 4, rcx, 1, 16
	r8_x8cl_eight_complex_with_square_preload
	xtouch	[rcx+4096-64]		;; Preload the TLBs
	xtouch	[rcx+2*4096-64]
baa:	r8_x8cl_eight_complex_with_square rsi, 8*64, 64, 2*64, 4*64
	loops	16, baa			;; Test loop counter
	bump	rsi, -16*8*64+dist128	;; Next source pointer
	loops	30, baa			;; Test loop counter
	bump	rsi, -30*dist128	;; Next source pointer
	jmp	xpass2_7680_real_unfft

xpass2_7680_levels_real_1:
	r8_h8cl_sixteen_reals_eight_complex_fft_final rsi, 8*64, 64, 2*64, 4*64
	loops_init_prefetch 480, 128, 1, rcx, 1, 16
	r8_x8cl_eight_complex_fft_final_preload
bab0:	xtouch	[rcx+4096-64]		;; Preload the TLBs
bab:	r8_x8cl_eight_complex_fft_final rsi, 8*64, 64, 2*64, 4*64
	loops	16, bab			;; Test loop counter
	bump	rsi, -16*8*64+dist128	;; Next source pointer
	loops_touch 2, bab, 15, bab0	;; Test loop counter
	jmp	xpass2_7680_real_done

xpass2_7680_levels_real_3:
	mov	rbp, DIST_TO_MULSRCARG
	r8_h8cl_sixteen_reals_eight_complex_with_mult rsi, 8*64, 64, 2*64, 4*64
	loops_init_prefetch 480, 64, 4, rcx, 1, 16
	r8_x8cl_eight_complex_with_mult_preload
	xtouch	[rcx+4096-64]		;; Preload the TLBs
	xtouch	[rcx+2*4096-64]
bac:	r8_x8cl_eight_complex_with_mult rsi, 8*64, 64, 2*64, 4*64
	loops	16, bac			;; Test loop counter
	bump	rsi, -16*8*64+dist128	;; Next source pointer
	loops	30, bac			;; Test loop counter
	bump	rsi, -30*dist128	;; Next source pointer
	jmp	xpass2_7680_real_unfft

xpass2_7680_levels_real_4:
	mov	rbp, DIST_TO_MULSRCARG
	mov	rcx, DATA_PREFETCH	;; Load prefetch pointer
	xtouch	[rcx]			;; Preload first TLB
	r8_h8cl_sixteen_reals_eight_complex_with_mulf rsi, 8*64, 64, 2*64, 4*64
	loops_init_prefetch 480, 64, 1, rcx, 1, 16
	r8_x8cl_eight_complex_with_mulf_preload
bad0:	xtouch	[rcx+4096-64]		;; Preload the TLBs
bad:	r8_x8cl_eight_complex_with_mulf rsi, 8*64, 64, 2*64, 4*64
	loops	16, bad			;; Test loop counter
	bump	rsi, -16*8*64+dist128	;; Next source pointer
	loops_touch 2, bad, 15, bad0	;; Test loop counter
	bump	rsi, -30*dist128	;; Next source pointer

;; Do inverse FFT levels 9,10
;; On input the 64-byte cache lines hold these data values:
;;	0	+15360	4	+15360	8	+15360	12	+15360
;;	1	...
;;	...
;;	3	...
;;	16	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+15360	16	+15360	32	+15360	48	+15360
;;	1	...
;;	...
;;	15	...
;;	64	...
;;	...

;; Do 8 eight_reals_unfft operations
;; Do 8 four_complex_djbunfft operations
;;	distance between fft data elements is 8
;;	do 4 macros processing 32 data values

xpass2_7680_real_unfft:
	mov	rdi, xsincos_complex	;; Load sin/cos pointers
	bump	rdi, 3072*XMM_SCD1
	mov	rdx, sincos3
	bump	rdx, 2560*XMM_HSCD1+512*XMM_HSCD2
	loops_init_prefetch 960, 64, 2, rcx
	xtouch	[rcx+4096-64]		;; Preload the TLBs
c8b:	r4_h4cl_eight_reals_four_complex_djbunfft rsi, 64, 4*64, 8*64, rdi, 4*16*XMM_SCD2, rdx, 4*16*XMM_HSCD2
	bump	rdi, 16*XMM_SCD2	;; Next sin/cos data
	bump	rdx, 16*XMM_HSCD2	;; Next sin/cos data
	loops	4, c8b			;; Test loop counter
	bump	rsi, -4*64+16*64	;; Next source pointer
	bump	rdi, -4*16*XMM_SCD2	;; Restore sin/cos ptr
	bump	rdx, -4*16*XMM_HSCD2	;; Restore sin/cos ptr

;; Do 3824 four_complex_djbunfft operations
;;	distance between fft data elements is 8
;;	do 956 macros each processing 32 data values

	loops_reset			;; 30 iters of 7 or 8 of 4
	r4_x4cl_four_complex_djbunfft_preload
c9a:	xtouch	[rcx+4096-64]		;; Preload the TLBs
c9b:	r4_x4cl_four_complex_djbunfft rsi, 64, 4*64, 8*64, rdi, 4*16*XMM_SCD2
	bump	rdi, 16*XMM_SCD2	;; Next sine/cosine pointer
	loops	4, c9b			;; Test loop counter
	bump	rsi, -4*64+16*64	;; Next source pointer
	bump	rdi, -4*16*XMM_SCD2	;; Restore sin/cos ptr
	loops	8, c9b			;; Test loop counter
	bump	rsi, -8*16*64+dist128	;; Next source pointer
	loops_touch 2, c9b, 15, c9a	;; Test loop counter
	bump	rsi, -30*dist128	;; Next source pointer

;; Do inverse FFT levels 7,8
;; On input the 64-byte cache lines hold these data values:
;;	0	+15360	16	+15360	32	+15360	48	+15360
;;	1	...
;;	...
;;	15	...
;;	64	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+15360	64	+15360	128	+15360	192	+15360
;;	1	...
;;	...
;;	63	...
;;	256	...
;;	...

;; Do 32 eight_reals_unfft operations
;; Do 32 four_complex_djbunfft operations
;;	distance between fft data elements is 32
;;	do 16 macros processing 32 data values

	loops_init_prefetch 960, 64, 2, rcx
	xtouch	[rcx+4096-64]		;; Preload the TLBs
c6b:	r4_h4cl_eight_reals_four_complex_djbunfft rsi, 64, 16*64, 32*64, rdi, 16*4*XMM_SCD2, rdx, 16*4*XMM_HSCD2
	bump	rdi, 4*XMM_SCD2		;; Next sin/cos data
	bump	rdx, 4*XMM_HSCD2	;; Next sin/cos data
	loops	16, c6b			;; Test inner loop counter
	bump	rsi, -16*64+64*64	;; Next source pointer
	bump	rdi, -16*4*XMM_SCD2	;; Restore sin/cos ptr
	bump	rdx, -16*4*XMM_HSCD2	;; Restore sin/cos ptr

;; Do 3776 four_complex_djbunfft operations
;;	distance between fft data elements is 32
;;	do 944 macros each processing 32 data values

	loops_reset			;; 30 iters of 1 or 2 of 8
	r4_x4cl_four_complex_djbunfft_preload
c7a:	xtouch	[rcx+4096-64]		;; Preload the TLBs
c7b:	r4_x4cl_four_complex_djbunfft rsi, 64, 16*64, 32*64, rdi, 16*4*XMM_SCD2
	bump	rdi, 4*XMM_SCD2		;; Next sin/cos data
	loops	16, c7b			;; Test inner loop counter
	bump	rsi, -16*64+64*64	;; Next source pointer
	bump	rdi, -16*4*XMM_SCD2	;; Restore sin/cos ptr
	loops	2, c7b			;; Test loop counter
	bump	rsi, -2*64*64+dist128	;; Next source pointer
	loops_touch 2, c7b, 15, c7a	;; 30 iterations
	bump	rsi, -30*dist128	;; Next source pointer

;; Do inverse FFT levels 5,6
;; On input the 64-byte cache lines hold these data values:
;;	0	+15360	64	+15360	128	+15360	192	+15360
;;	1	...
;;	...
;;	63	...
;;	256	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+15360	256	+15360	512	+15360	768	+15360
;;	1	...
;;	...
;;	255	...
;;	1024	...
;;	...

;; Do 128 eight_reals_unfft operations
;; Do 128 four_complex_djbunfft operations
;;	distance between fft data elements is 128
;;	do 64 macros processing 32 data values

	loops_init_prefetch 960, 64, 2, rcx
	xtouch	[rcx+4096-64]		;; Preload the TLBs
c4b:	r4_h4cl_eight_reals_four_complex_djbunfft rsi, 64, 64*64, dist128, rdi, 64*XMM_SCD2, rdx, 64*XMM_HSCD2
	bump	rdi, XMM_SCD2		;; Next sin/cos data
	bump	rdx, XMM_HSCD2		;; Next sin/cos data
	loops	64, c4b			;; Test inner loop counter
	bump	rsi, -64*64+2*dist128	;; Next source pointer
	bump	rdi, -64*XMM_SCD2	;; Restore sin/cos ptr
	bump	rdx, -64*XMM_HSCD2-512*XMM_HSCD2 ;; Restore sin/cos ptr

;; Do 3584 four_complex_djbunfft operations
;;	distance between fft data elements is 128
;;	do 896 macros each processing 32 data values

	loops_reset			;; 14 more iters of 64
	r4_x4cl_four_complex_djbunfft_preload
c5a:	xtouch	[rcx+4096-64]		;; Preload the TLBs
c5b:	r4_x4cl_four_complex_djbunfft rsi, 64, 64*64, dist128, rdi, 64*XMM_SCD2
	bump	rdi, XMM_SCD2		;; Next sin/cos data
	loops	64, c5b			;; Test inner loop counter
	bump	rsi, -64*64+2*dist128	;; Next source pointer
	bump	rdi, -64*XMM_SCD2	;; Restore sin/cos ptr
	loops	15, c5a			;; Test loop counter
	bump	rsi, -15*2*dist128	;; Next source pointer

;; Do inverse FFT levels 3,4 (actually 2.322 FFT levels)
;; On input the 64-byte cache lines hold these data values:
;;	0	+15360	256	+15360	512	+15360	768	+15360
;;	1	...
;;	...
;;	255	...
;;	1024	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+15360	1280	+15360	2560	+15360	3840	+15360
;;	1	...
;;	...
;;	1279	...
;;	5120	...
;;	...

;; Do 512 ten_reals_unfft operations
;; Do 512 ten_complex_djbunfft operations
;;	distance between fft data elements is 512
;;	do 256 macros processing 40 data values

	loops_init_prefetch 768, 64, 2, rcx
	xtouch	[rcx+4096-64]		;; Preload the TLBs
	mov	rdi, xsincos_complex	;; Load sin/cos pointer
	mov	rbx, rdi
c2b:	r5_h5cl_2sc_ten_reals_five_complex_djbunfft rsi, 64, 2*dist128, rdi, 256*3*XMM_SCD1, rbx, 256*6*XMM_SCD1, rdx, 256*XMM_HSCD2
	bump	rdi, 3*XMM_SCD1		;; Next sin/cos data
	bump	rbx, 6*XMM_SCD1		;; Next sin/cos data
	bump	rdx, XMM_HSCD2		;; Next sin/cos data
	loops	128, c2b		;; Test inner loop counter
	bump	rsi, -128*64+dist128	;; Next source pointer
	loops	2, c2b			;; Test loop counter
	bump	rsi, -2*dist128+10*dist128 ;; Next source pointer
	bump	rdi, -256*3*XMM_SCD1	;; Restore sin/cos ptr
	bump	rbx, -256*6*XMM_SCD1	;; Restore sin/cos ptr
	bump	rdx, -256*XMM_HSCD2-2560*XMM_HSCD1 ;; Next sin/cos ptr

;; Do 2048 five_complex_djbunfft operations
;;	distance between fft data elements is 512
;;	do 512 macros each processing 32 data values

	loops_reset			;; 2 more iters of 256
	r5_x5cl_2sc_five_complex_djbunfft_preload
c3a:	xtouch	[rcx+4096-64]		;; Preload the TLBs
c3b:	r5_x5cl_2sc_five_complex_djbunfft rsi, 64, 2*dist128, rdi, 256*3*XMM_SCD1, rbx, 256*6*XMM_SCD1
	bump	rdi, 3*XMM_SCD1		;; Next sin/cos data
	bump	rbx, 6*XMM_SCD1		;; Next sin/cos data
	loops	128, c3b		;; Test inner loop counter
	bump	rsi, -128*64+dist128	;; Next source pointer
	loops	2, c3a			;; Test loop counter
	bump	rsi, -2*dist128+10*dist128 ;; Next source pointer
	bump	rdi, -256*3*XMM_SCD1	;; Restore sin/cos ptr
	bump	rbx, -256*6*XMM_SCD1	;; Restore sin/cos ptr
	loops	3, c3a			;; Test loop counter
	bump	rsi, -3*10*dist128	;; Next source pointer

;; Do inverse FFT level 1,2 (actually 1.585 FFT levels)
;; On input the 64-byte cache lines hold these data values:
;;	0	+15360	1280	+15360	2560	+15360	3840	+15360
;;	1	...
;;	...
;;	1279	...
;;	5120	...
;;	...
;; On output the 64-byte cache lines hold these data values if pass 1 swizzles:
;;	0	+15360	3840	+15360	7680	+15360	11520	+15360
;;	1	...
;;	...
;;	3839
;; On output the 64-byte cache lines hold these data values if pass 2 swizzles:
;;	0	+1920	7680	+1920	15360	+1920	23040	+1920
;;	1	...
;;	...
;;	3839

;; Do 2560 six_reals_unfft operations
;; Do 2560 three_complex_djbunfft operations
;;	distance between fft data elements is 2560
;;	do 1280 macros processing 24 data values

	bump	rcx, -30*64		;; Back up prefetch pointer by 30 cache lines
	loops_init_prefetch 1280, 64, 8, rcx
	xtouch	[rcx+4096-64]		;; Preload the TLBs
	xtouch	[rcx+2*4096-64]
	xtouch	[rcx+160*64-64]
c1b:	r3_h3cl_six_reals_three_complex_djbunfft rsi, 64, 10*dist128, rdi, 1280*XMM_SCD1, rdx, 1280*XMM_HSCD1
	bump	rdi, XMM_SCD1		;; Next sin/cos data
	bump	rdx, XMM_HSCD1		;; Next sin/cos data
	loops	128, c1b		;; Test loop counter
	bump	rsi, -128*64+dist128	;; Next source pointer
	loops	10, c1b			;; Test loop counter

xpass2_7680_real_done:
	end_timer 2
	ENDM

; *************** radix4-pass2-7680-levels-complex macro ******************
; This macro takes 2 groups of 7680 complex values and
; performs the final 12.907 levels of a standard mostly radix-4 FFT, squares the
; results, and does the first 12.907 levels of the inverse FFT.

r4_pass2_7680_levels_complex MACRO
	LOCAL	b1a0, b1a, b1b, b2a, b2b, b3a, b3b, b4b, b5b
	LOCAL	b6a, b6b0, b6b, b6c, b6d0, b6d
	LOCAL	c1b, c2a0, c2a, c2b, c3b, c4b, c5a, c5b
	LOCAL	xpass2_7680_levels_complex_1, xpass2_7680_levels_complex_2
	LOCAL	xpass2_7680_levels_complex_3, xpass2_7680_levels_complex_4
	LOCAL	xpass2_7680_levels_complex_unfft, xpass2_7680_levels_complex_done

;; Type 4 FFTs skip the forward FFT process

	mov	rsi, DATA_ADDR		;; Load address of FFT data
	mov	rbx, DIST_TO_FFTSRCARG
	cmp	ffttype, 4
	je	xpass2_7680_levels_complex_4

;; Do FFT levels 1,2 (actually 1.585 levels)
;;
;; We swizzle the data so that one set of data is in the high word of
;; an xmm register, the other set is in the low word.  This means
;; a distance of 15360 data values.
;;
;; On input the 64-byte cache lines hold these data values if pass 2 swizzles:
;;	0	+3840	7680	+3840	15360	+3840	23040	+3840
;;	1	...
;;	...
;;	3839
;; On input the 64-byte cache lines hold these data values if pass 1 swizzles:
;;	0	+15360	3840	+15360	7680	+15360	11520	+15360
;;	1	...
;;	...
;;	3839	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+15360	1280	+15360	2560	+15360	3840	+15360
;;	1	...
;;	...
;;	1279	...
;;	5120	...
;;	...

;; Do 5120 three_complex_fft operations
;;	distance between fft data elements is 2560
;;	do 1280 macros each processing 24 data values

;; Do FFT levels 3,4 (actually 2.322 levels)
;;
;; On input the 64-byte cache lines hold these data values:
;;	0	+15360	1280	+15360	2560	+15360	3840	+15360
;;	1	...
;;	...
;;	1279	...
;;	5120	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+15360	256	+15360	512	+15360	768	+15360
;;	1	...
;;	...
;;	255	...
;;	1024	...
;;	...

;; Do 3072 five_complex_fft operations
;;	distance between fft data elements is 512
;;	do 768 macros each processing 40 data values

IF @INSTR(,%xarch,<CORE>) NE 0		;; Core 2 faster doing separate loops

	start_timer 4
	mov	rdi, xsincos_complex	;; The sin/cos data
	loops_init 1280			;; 5 iterations of 128
	r3_f3cl_three_complex_djbfft_preload
b1b:	r3_f3cl_three_complex_djbfft rsi, 64, 10*dist128, rdi, 1280*XMM_SCD1
	bump	rdi, XMM_SCD1		;; Next sin/cos data
	loops	128, b1b		;; Test inner loop counter
	bump	rsi, -128*64+dist128	;; Next source pointer
	loops	10, b1b			;; Test outer loop counter
	bump	rsi, -10*dist128	;; Restore source pointer
	end_timer 4

	start_timer 5
	mov	rdi, xsincos_complex	;; Ptr to the sin/cos data
	mov	rdx, rdi
	mov	rcx, DATA_PREFETCH	;; Prefetch pointer (we need to prefetch
					;; 3840 cache lines + 58 pad cache lines)
	xtouch	[rcx]			;; Preload first TLB
	loops_init_prefetch 768, 64, 2, rcx
	r5_nf5cl_2sc_five_complex_djbfft_preload
b2a:	xtouch	[rcx+4096-64]		;; Preload the TLBs
b2b:	r5_nf5cl_2sc_five_complex_djbfft rsi, 64, 2*dist128, rdi, 256*3*XMM_SCD1, rdx, 256*6*XMM_SCD1
	bump	rdi, 3*XMM_SCD1		;; Next sin/cos data
	bump	rdx, 6*XMM_SCD1		;; Next sin/cos data
	loops	128, b2b		;; Test inner loop counter
	bump	rsi, -128*64+dist128	;; Next source pointer
	loops	2, b2a			;; Test loop counter
	bump	rsi, -2*dist128+10*dist128 ;; Next source pointer
	bump	rdi, -256*3*XMM_SCD1	;; Restore sin/cos ptr
	bump	rdx, -256*6*XMM_SCD1	;; Restore sin/cos ptr
	loops	3, b2a			;; Test loop counter
	bump	rsi, -3*10*dist128	;; Restore source pointer
	end_timer 5

ELSE					;; Pentium 4 faster doing merged loops

	start_timer 4
	mov	rdi, xsincos_complex	;; The sin/cos data
	mov	rbp, rdi
	mov	rdx, rdi
	mov	rcx, DATA_PREFETCH	;; Prefetch pointer (we need to prefetch
					;; 3840 cache lines + 58 pad cache lines)
	xtouch	[rcx]			;; Preload first TLB
	loops_init_prefetch 3840, 64, 5, rcx ;; 256 iters of 5 or 3

b1a0:	xtouch	[rcx+4096-64]		;; Preload TLBs
	xtouch	[rcx+2*4096-64]
	xtouch	[rcx+3*4096-64]

b1a:	r3_f3cl_three_complex_djbfft_preload
b1b:	r3_f3cl_three_complex_djbfft rsi, 2*dist128, 10*dist128, rdi, 1280*XMM_SCD1
	xxprefetch [rdi+XMM_SCD1]	;; Prefetch for next pass
	xxprefetch [rdi+1281*XMM_SCD1]	;; Prefetch for next pass
	bump	rdi, 256*XMM_SCD1	;; Next sin/cos ptr
	loops_no_prefetch 5, b1b, 3	;; Test loop counter with non-standard incr
	bump	rsi, -5*2*dist128	;; Restore source pointer
	loops_undo 15

	r5_nf5cl_2sc_five_complex_djbfft_preload
b2b:	r5_nf5cl_2sc_five_complex_djbfft rsi, 10*dist128, 2*dist128, rbp, 256*3*XMM_SCD1, rdx, 256*6*XMM_SCD1
	loops	3, b2b, 5		;; Test loop counter with non-standard incr

	bump	rsi, -3*10*dist128+64	;; Next source pointer
	bump	rdi, -5*256*XMM_SCD1+XMM_SCD1 ;; Next sin/cos ptr
	bump	rbp, 3*XMM_SCD1		;; Next sin/cos data
	bump	rdx, 6*XMM_SCD1		;; Next sin/cos data
	loops_touch 64, b1a, 2, b1a0	;; Loop 128 times
	bump	rsi, -128*64+dist128	;; Next source pointer
	loops	2, b1a0			;; Test loop counter
	bump	rsi, -2*dist128		;; Restore source pointer
	bump	rdi, -256*XMM_SCD1	;; Restore sin/cos ptr
	end_timer 4

ENDIF

;; Do FFT levels 5,6
;;
;; On input the 64-byte cache lines hold these data values:
;;	0	+15360	256	+15360	512	+15360	768	+15360
;;	1	...
;;	...
;;	255	...
;;	1024	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+15360	128	+15360	256	+15360	384	+15360
;;	1	...
;;	...
;;	127	...
;;	512	...
;;	...
;;
;; Do 3840 four_complex_fft operations
;;	distance between fft data elements is 128
;;	do 1920 macros each processing 16 data values

;; Do FFT levels 7,8
;;
;; On input the 64-byte cache lines hold these data values:
;;	0	+15360	128	+15360	256	+15360	384	+15360
;;	1	...
;;	...
;;	127	...
;;	512	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+15360	32	+15360	64	+15360	96	+15360
;;	1	...
;;	...
;;	31	...
;;	128	...
;;	...
;;
;; Do 3840 four_complex_fft operations
;;	distance between fft data elements is 32
;;	do 960 macros each processing 32 data values

;; Do FFT levels 9,10
;;
;; On input the 64-byte cache lines hold these data values:
;;	0	+15360	32	+15360	64	+15360	96	+15360
;;	1	...
;;	...
;;	31	...
;;	128	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+15360	8	+15360	16	+15360	24	+15360
;;	1	...
;;	...
;;	7	...
;;	32	...
;;	...
;;
;; Do 3840 four_complex_fft operations
;;	distance between fft data elements is 8
;;	do 960 macros each processing 32 data values

;; Do this 32 cache lines at a time to maximize accessing data
;; while it is in the L1 cache.

	start_timer 6
	bump	rdi, 3072*XMM_SCD1	;; Load sin/cos pointers
	mov	rbx, rdi
	mov	rbp, rdi
	loops_init_prefetch 1920, 64, 4, rcx ;; 15*8 iters of 32 cache lines
	;r4_nf2cl_four_complex_djbfft_preload -- assume same as r4_x4cl_four_complex_djbfft_preload
	r4_x4cl_four_complex_djbfft_preload

b3a:	xtouch	[rcx+4096-64]		;; Load 12 prefetch TLBs
	xtouch	[rcx+2*4096-64]

b3b:	xxprefetch [rdi+8*XMM_SCD2]	;; Prefetch for next pass
	r4_nf2cl_four_complex_djbfft rsi, 8*64, dist128, rdi
	bump	rdi, 8*XMM_SCD2		;; Next sine/cosine pointer
	loops	16, b3b			;; Loop 16 times
	bump	rsi, -16*8*64		;; Restore source pointer
	bump	rdi, -16*8*XMM_SCD2	;; Restore sine/cosine pointer
	loops_undo 16			;; Back up loop counter

b4b:	r4_x4cl_four_complex_djbfft rsi, 8*64, 32*64, 64*64, rbx
	bump	rbx, 8*4*XMM_SCD2	;; Next sine/cosine pointer
	loops	4, b4b, 2		;; 4 iterations with non-standard increment
	bump	rsi, -4*8*64+dist128	;; Next source pointer
	bump	rbx, -4*8*4*XMM_SCD2	;; Restore sine/cosine pointer
	loops	2, b4b			;; Loop twice
	bump	rsi, -2*dist128		;; Restore source pointer
	loops_undo 2*4*2		;; Back up loop counter

b5b:	r4_x4cl_four_complex_djbfft rsi, 32*64, 8*64, 16*64, rbp
	loops	4, b5b, 2 		;; 4 iterations with non-standard increment
	bump	rsi, -4*32*64+dist128	;; Next source pointer
	loops	2, b5b			;; 2 iterations

	bump	rsi, -2*dist128+64	;; Next source pointer
	bump	rdi, XMM_SCD2		;; Next sine/cosine pointer
	bump	rbx, 4*XMM_SCD2		;; Next sine/cosine pointer
	bump	rbp, 16*XMM_SCD2	;; Next sine/cosine pointer
	loops	8, b3b			;; Loop 4 times
	bump	rsi, -8*64+2*dist128	;; Next source pointer
	bump	rdi, -8*XMM_SCD2	;; Restore sine/cosine pointer
	bump	rbx, -8*4*XMM_SCD2	;; Restore sine/cosine pointer
	bump	rbp, -8*16*XMM_SCD2	;; Restore sine/cosine pointer
	loops	15, b3a			;; Loop 15 times
	bump	rsi, -15*2*dist128	;; Restore source pointer
	end_timer 6

;; Do FFT levels 11,12,13 as well as inverse FFT levels 11,12,13
;;
;; On input the 64-byte cache lines hold these data values:
;;	0	+15360	8	+15360	16	+15360	24	+15360
;;	1	...
;;	...
;;	7	...
;;	32	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+15360	4	+15360	8	+15360	12	+15360
;;	1	...
;;	...
;;	3	...
;;	16	...
;;	...

;; Do 1920 eight_complex_fft operations
;;	distance between fft data elements is 1
;;	do 480 macros each processing 64 data values

;; Execute the right middle step

	cmp	ffttype, 2
	jl	xpass2_7680_levels_complex_1
	jg	xpass2_7680_levels_complex_3

xpass2_7680_levels_complex_2:
	start_timer 9
	loops_init_prefetch 480, 64, 4, rcx
	r8_x8cl_eight_complex_with_square_preload
	xtouch	[rcx+4096-64]		;; Load prefetch TLBs
	xtouch	[rcx+2*4096-64]
b6a:	r8_x8cl_eight_complex_with_square rsi, 8*64, 64, 2*64, 4*64
	loops	16, b6a			;; Loop 16 times
	bump	rsi, -16*8*64+dist128	;; Next source pointer
	loops	30, b6a			;; Loop 30 times
	bump	rsi, -30*dist128	;; Restore source pointer
	end_timer 9
	jmp	xpass2_7680_levels_complex_unfft

xpass2_7680_levels_complex_1:
	start_timer 9
	loops_init_prefetch 480, 128, 1, rcx
	r8_x8cl_eight_complex_fft_final_preload
b6b0:	xtouch	[rcx+4096-64]		;; Load prefetch TLBs
b6b:	r8_x8cl_eight_complex_fft_final rsi, 8*64, 64, 2*64, 4*64
	loops	16, b6b			;; Loop 16 times
	bump	rsi, -16*8*64+dist128	;; Next source pointer
	loops_touch 2, b6b, 15, b6b0	;; Loop 30 times
	end_timer 9
	jmp	xpass2_7680_levels_complex_done

xpass2_7680_levels_complex_3:
	start_timer 9
	mov	rbp, DIST_TO_MULSRCARG
	loops_init_prefetch 480, 64, 4, rcx
	r8_x8cl_eight_complex_with_mult_preload
	xtouch	[rcx+4096-64]		;; Load prefetch TLBs
	xtouch	[rcx+2*4096-64]
b6c:	r8_x8cl_eight_complex_with_mult rsi, 8*64, 64, 2*64, 4*64
	loops	16, b6c			;; Loop 16 times
	bump	rsi, -16*8*64+dist128	;; Next source pointer
	loops	30, b6c			;; Loop 30 times
	bump	rsi, -30*dist128	;; Restore source pointer
	end_timer 9
	jmp	xpass2_7680_levels_complex_unfft

xpass2_7680_levels_complex_4:
	start_timer 9
	mov	rcx, DATA_PREFETCH	;; Prefetch pointer
	xtouch	[rcx]			;; Load first prefetch TLB
	mov	rbp, DIST_TO_MULSRCARG
	loops_init_prefetch 480, 128, 1, rcx
	r8_x8cl_eight_complex_with_mulf_preload
b6d0:	xtouch	[rcx+4096-64]		;; Load prefetch TLBs
b6d:	r8_x8cl_eight_complex_with_mulf rsi, 8*64, 64, 2*64, 4*64
	loops	16, b6d			;; Loop 16 times
	bump	rsi, -16*8*64+dist128	;; Next source pointer
	loops_touch 2, b6d, 15, b6d0	;; Loop 30 times
	bump	rsi, -30*dist128	;; Restore source pointer
	end_timer 9

;; Do inverse FFT levels 9,10
;; On input the 64-byte cache lines hold these data values:
;;	0	+15360	4	+15360	8	+15360	12	+15360
;;	1	...
;;	...
;;	3	...
;;	16	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+15360	16	+15360	32	+15360	48	+15360
;;	1	...
;;	...
;;	15	...
;;	64	...
;;	...
;;
;; Do 3840 four_complex_unfft operations
;;	distance between fft data elements is 8
;;	do 960 macros each processing 32 data values

;; Do inverse FFT levels 7,8
;; On input the 64-byte cache lines hold these data values:
;;	0	+15360	16	+15360	32	+15360	48	+15360
;;	1	...
;;	...
;;	15	...
;;	64	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+15360	64	+15360	128	+15360	192	+15360
;;	1	...
;;	...
;;	63	...
;;	256	...
;;	...
;;
;; Do 3840 four_complex_unfft operations
;;	distance between fft data elements is 32
;;	do 960 macros each processing 32 data values

;; Do inverse FFT levels 5,6
;; On input the 64-byte cache lines hold these data values:
;;	0	+15360	64	+15360	128	+15360	192	+15360
;;	1	...
;;	...
;;	63	...
;;	256	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+15360	256	+15360	512	+15360	768	+15360
;;	1	...
;;	...
;;	255	...
;;	1024	...
;;	...
;;
;; Do 3840 four_complex_unfft operations
;;	distance between fft data elements is 128
;;	do 960 macros each processing 32 data values

;; Do this 64 cache lines at a time to maximize accessing data
;; while it is in the L1 cache.

xpass2_7680_levels_complex_unfft:
	start_timer 10
	mov	rdi, xsincos_complex	;; Load sin/cos pointers
	bump	rdi, 3072*XMM_SCD1
	mov	rbx, rdi
	mov	rbp, rdi
	loops_init_prefetch 960, 64, 2, rcx ;; 15*4 iters of 64 cache lines
	r4_x4cl_four_complex_djbunfft_preload

c5a:	xtouch	[rcx+4096-64]		;; Load prefetch TLBs
	xtouch	[rcx+2*4096-64]

c5b:	r4_x4cl_four_complex_djbunfft rsi, 16*64, 4*64, 8*64, rbp, 4*16*XMM_SCD2
	loops	8, c5b			;; Loop 8 times
	bump	rsi, -8*16*64+dist128	;; Next source pointer
	loops	2, c5b			;; Loop 2 times
	bump	rsi, -2*dist128		;; Restore source pointer
	loops_undo 2*8			;; Restore loop counter

c4b:	r4_x4cl_four_complex_djbunfft rsi, 4*64, 16*64, 32*64, rbx, 16*4*XMM_SCD2
	bump	rbx, 4*4*XMM_SCD2	;; Next sine/cosine pointer
	loops	4, c4b			;; Loop 4 times
	bump	rsi, -4*4*64+64*64	;; Next source pointer
	bump	rbx, -4*4*4*XMM_SCD2	;; Restore sine/cosine pointer
	loops	2, c4b			;; Loop twice
	bump	rsi, -2*64*64+dist128	;; Next source pointer
	loops	2, c4b			;; Loop twice
	bump	rsi, -2*dist128		;; Restore source pointer
	loops_undo 2*2*4		;; Restore loop counter

c3b:	r4_x4cl_four_complex_djbunfft rsi, 4*64, 64*64, dist128, rdi, 64*1*XMM_SCD2
	bump	rdi, 4*1*XMM_SCD2	;; Next sine/cosine pointer
	loops	16, c3b			;; Loop 16 times
	bump	rdi, -16*4*1*XMM_SCD2	;; Restore sine/cosine pointer

	bump	rsi, -16*4*64+64	;; Next source pointer
	bump	rdi, 1*XMM_SCD2		;; Next sine/cosine pointer
	bump	rbx, 4*XMM_SCD2		;; Next sine/cosine pointer
	bump	rbp, 16*XMM_SCD2	;; Next sine/cosine pointer
	loops	4, c5b			;; Loop 4 times
	bump	rsi, -4*64+2*dist128	;; Next source pointer
	bump	rdi, -4*1*XMM_SCD2	;; Restore sine/cosine pointer
	bump	rbx, -4*4*XMM_SCD2	;; Restore sine/cosine pointer
	bump	rbp, -4*16*XMM_SCD2	;; Restore sine/cosine pointer
	loops	15, c5a			;; Loop 15 times
	bump	rsi, -15*2*dist128	;; Restore source pointer
	end_timer 10

;; Do inverse FFT levels 3,4 (actually 2.322 FFT levels)
;; On input the 64-byte cache lines hold these data values:
;;	0	+15360	256	+15360	512	+15360	768	+15360
;;	1	...
;;	...
;;	255	...
;;	1024	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+15360	1280	+15360	2560	+15360	3840	+15360
;;	1	...
;;	...
;;	1279	...
;;	5120	...
;;	...

;; Do 3072 five_complex_unfft operations
;;	distance between fft data elements is 512
;;	do 768 macros each processing 40 data values

;; Do inverse FFT levels 1,2 (actually 1.585 FFT levels)
;; On input the 64-byte cache lines hold these data values:
;;	0	+15360	1280	+15360	2560	+15360	3840	+15360
;;	1	...
;;	...
;;	1279	...
;;	5120	...
;;	...
;; On output the 64-byte cache lines hold these data values if pass 1 swizzles:
;;	0	+15360	3840	+15360	7680	+15360	11520	+15360
;;	1	...
;;	...
;;	3839
;; On output the 64-byte cache lines hold these data values if pass 2 swizzles:
;;	0	+1920	7680	+1920	15360	+1920	23040	+1920
;;	1	...
;;	...
;;	3839

;; Do 5120 three_complex_unfft operations
;;	distance between fft data elements is 2560
;;	do 1280 macros each processing 24 data values

IF @INSTR(,%xarch,<CORE>) NE 0		;; Core 2 faster doing separate loops

	start_timer 13
	mov	rdi, xsincos_complex	;; Load sin/cos pointer
	mov	rdx, rdi
	loops_init_prefetch 768, 64, 2, rcx;; 3 iters of 256
	r5_x5cl_2sc_five_complex_djbunfft_preload
c2a:	xtouch	[rcx+4096-64]		;; Load prefetch TLBs
c2b:	r5_x5cl_2sc_five_complex_djbunfft rsi, 64, 2*dist128, rdi, 256*3*XMM_SCD1, rdx, 256*6*XMM_SCD1
	bump	rdi, 3*XMM_SCD1		;; Next sine/cosine pointer
	bump	rdx, 6*XMM_SCD1		;; Next sine/cosine pointer
	loops	128, c2b		;; Loop 128 times
	bump	rsi, -128*64+dist128	;; Next source pointer
	loops	2, c2a			;; Loop twice
	bump	rsi, -2*dist128+10*dist128 ;; Next source pointer
	bump	rdi, -256*3*XMM_SCD1	;; Reset sine/cosine pointer
	bump	rdx, -256*6*XMM_SCD1	;; Reset sine/cosine pointer
	loops	3, c2a			;; Test loop counter
	bump	rsi, -3*10*dist128	;; Restore source pointer
	end_timer 13

	start_timer 14
	mov	rdi, xsincos_complex	;; Load group multiplier pointer
	bump	rcx, -30*64		;; Back up prefetch pointer by 30 cache lines
	loops_init_prefetch 1280, 64, 8, rcx;; 10 iters of 128
	xtouch	[rcx+4096-64]		;; Load prefetch TLBs
	xtouch	[rcx+2*4096-64]
	xtouch	[rcx+160*64-64]
	r3_x3cl_three_complex_djbunfft_preload
c1b:	r3_x3cl_three_complex_djbunfft rsi, 64, 10*dist128, rdi, 1280*XMM_SCD1
	bump	rdi, XMM_SCD1		;; Next set of sin/cos data
	loops	128, c1b		;; Test inner loop counter
	bump	rsi, -128*64+dist128	;; Next source pointer
	loops	10, c1b			;; Test outer loop counter
	end_timer 14

ELSE					;; Pentium 4 faster doing merged loops

	start_timer 13
	mov	rdi, xsincos_complex	;; Load sin/cos pointer
	mov	rbp, rdi
	mov	rdx, rdi
	loops_init_prefetch 3840, 64, 30, rcx ;; 256 iters of 3 or 5

c2a0:	xtouch	[rcx+4096-64]		;; Load prefetch TLB

c2a:	r5_x5cl_2sc_five_complex_djbunfft_preload
c2b:	r5_x5cl_2sc_five_complex_djbunfft rsi, 10*dist128, 2*dist128, rbp, 256*3*XMM_SCD1, rdx, 256*6*XMM_SCD1
	loops	3, c2b, 5		;; Test loop counter with non-standard incr
	bump	rsi, -3*10*dist128	;; Restore source pointer
	loops_undo 15

	r3_x3cl_three_complex_djbunfft_preload
c1b:	r3_x3cl_three_complex_djbunfft rsi, 2*dist128, 10*dist128, rdi, 1280*XMM_SCD1
	bump	rdi, 256*XMM_SCD1	;; Next sin/cos ptr
	loops	5, c1b, 3		;; Test loop counter with non-standard incr

	bump	rsi, -5*2*dist128+64	;; Next source pointer
	bump	rdi, -5*256*XMM_SCD1+XMM_SCD1 ;; Next sin/cos ptr
	bump	rbp, 3*XMM_SCD1		;; Next sine/cosine pointer
	bump	rdx, 6*XMM_SCD1		;; Next sine/cosine pointer
	loops	128, c2a		;; Loop 128 times
	bump	rsi, -128*64+dist128	;; Next source pointer
	loops	2, c2a0			;; Loop twice
	end_timer 13

ENDIF

xpass2_7680_levels_complex_done:
	ENDM


; *************** radix4-pass2-13-levels-real macro ******************
; This macro takes 16384 real values and performs the final 14 levels of
; the FFT.  Independently, this macro also takes 8192 complex values and
; performs the final 13 levels of the FFT.  The results are squared, and
; the first 13/14 levels of the inverse FFT are done.

r4_pass2_13_levels_real MACRO
	LOCAL	b1b, b2b0, b2b, b3b0, b3b, b4b, b5b, b6b, b7b
	LOCAL	b8b, b9b, baa0, baa, bab0, bab, bac0, bac, bad0, bad
	LOCAL	c1b, c2b, c3b, c4b, c5b, c6b, c7b, c8b, c9b
	LOCAL	xpass2_13_levels_real_1, xpass2_13_levels_real_2
	LOCAL	xpass2_13_levels_real_3, xpass2_13_levels_real_4
	LOCAL	xpass2_13_real_unfft, xpass2_13_real_done

;; We switch to the same format used in one-pass FFTs (that is, the
;; high word of xmm registers being 1 greater than the low word):

;; Type 4 FFTs skip the forward FFT process

	start_timer 2
	mov	rsi, DATA_ADDR		;; Load source address
	mov	rbx, DIST_TO_FFTSRCARG
	cmp	ffttype, 4
	je	xpass2_13_levels_real_4

;; Do FFT level 1,2
;;
;; On input the 64-byte cache lines hold these data values if pass 2 swizzles:
;;	0	+4K	8192	+4K	16384	+4K	24576	+4K
;;	1	...
;;	...
;;	4095
;; On input the 64-byte cache lines hold these data values if pass 1 swizzles:
;;	0	+16K	4096	+16K	8192	+16K	12288	+16K
;;	1	...
;;	...
;;	4095
;; On output the 64-byte cache lines hold these data values:
;;	0	+16K	2048	+16K	4096	+16K	6144	+16K
;;	1	...
;;	...
;;	2047	...
;;	8192	...
;;	...

;; Do 2048 eight_reals_fft on values 0 - 16383
;; Do 2048 four_complex_djbfft on values 16384 - 32767
;;	distance between fft data elements is 2K
;;	do 2048 iterations

	mov	rdi, xsincos_complex	;; Ptr to the sin/cos data
	mov	rdx, sincos3		;; Ptr to real sin/cos data
	loops_init 2048
b1b:	r4_fh2cl_eight_reals_four_complex_djbfft rsi, 64, 16*dist128, rdi, rdx
	bump	rdi, XMM_SCD2		;; Next sin/cos data
	bump	rdx, XMM_HSCD2		;; Next sin/cos data
	loops	128, b1b		;; Test loop counter
	bump	rsi, -128*64+dist128	;; Next source pointer
	loops	16, b1b			;; Test loop counter
	bump	rsi, -16*dist128	;; Restore source pointer
	bump	rdi, -16*128*XMM_SCD2	;; Restore sin/cos ptr
	bump	rdx, -16*128*XMM_HSCD2	;; Restore sin/cos ptr

;; Do FFT levels 3,4
;; Values 0-4095 is real data, 4096-32767 is complex data.
;;
;; On input the 64-byte cache lines hold these data values:
;;	0	+16K	2048	+16K	4096	+16K	6144	+16K
;;	1	...
;;	...
;;	2047	...
;;	8192	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+16K	512	+16K	1024	+16K	1536	+16K
;;	1	...
;;	...
;;	511	...
;;	2048	...
;;	...

;; Do 512 eight_reals_fft operations
;; Do 1536 four_complex_djbfft operations
;;	distance between fft data elements is 512
;;	do 512 macros each processing 32 data values

	mov	rcx, DATA_PREFETCH	;; Prefetch pointer
	xtouch	[rcx]			;; Preload first TLB
	loops_init_prefetch 512, 64, 1, rcx
	r4_h4cl_eight_reals_four_complex_djbfft_preload
b2b0:	xtouch	[rcx+4096-128]		;; Preload the TLBs
	xtouch	[rcx+2*4096-128]
b2b:	r4_h4cl_eight_reals_four_complex_djbfft rsi, 64, 4*dist128, 8*dist128, rdi, rdx
	bump	rdi, 4*XMM_SCD2		;; Next sin/cos data
	bump	rdx, 4*XMM_HSCD2	;; Next sin/cos data
	loops	128, b2b		;; Test inner loop counter
	bump	rsi, -128*64+dist128	;; Next source pointer
	bump	rcx, 128		;; Skip pad bytes
	loops	4, b2b0			;; Test loop counter
	bump	rsi, -4*dist128+16*dist128 ;; Next source pointer
	bump	rdi, -4*128*4*XMM_SCD2	;; Restore sin/cos ptr
	bump	rdx, -4*128*4*XMM_HSCD2	;; Restore sin/cos ptr

;; Do 2048 four_complex_fft operations
;;	distance between fft data elements is 512
;;	do 512 macros each processing 32 data values

	loops_init_prefetch 512, 64, 1, rcx
	r4_x4cl_four_complex_djbfft_preload
b3b0:	xtouch	[rcx+4096-128]		;; Preload the TLBs
	xtouch	[rcx+2*4096-128]
b3b:	r4_x4cl_four_complex_djbfft rsi, 64, 4*dist128, 8*dist128, rdi
	bump	rdi, 4*XMM_SCD2		;; Next sin/cos data
	loops	128, b3b		;; Test inner loop counter
	bump	rsi, -128*64+dist128	;; Next source pointer
	bump	rcx, 128		;; Skip pad bytes
	loops	4, b3b0			;; Test loop counter
	bump	rsi, -4*dist128-16*dist128 ;; Restore source pointer
	;; don't restore rdi, we'll fix it in next section

;; Do FFT levels 5,6
;; Values 0-1023 is real data, 1024-32767 is complex data.
;;
;; On input the 64-byte cache lines hold these data values:
;;	0	+16K	512	+16K	1024	+16K	1536	+16K
;;	1	...
;;	...
;;	511	...
;;	2048	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+16K	128	+16K	256	+16K	384	+16K
;;	1	...
;;	...
;;	127	...
;;	512	...
;;	...

;; Do 128 eight_reals_fft operations
;; Do 384 four_complex_djbfft operations
;;	distance between fft data elements is 128
;;	do 128 macros each processing 32 data values

	bump	rdi, -4*128*4*XMM_SCD2+2048*XMM_SCD2 ;; Load sin/cos pointer
	loops_init 1024
	r4_h4cl_eight_reals_four_complex_djbfft_preload
b4b:	r4_h4cl_eight_reals_four_complex_djbfft rsi, 64, dist128, 2*dist128, rdi, rdx
	bump	rdi, XMM_SCD2		;; Next sin/cos data
	bump	rdx, 16*XMM_HSCD2	;; Next sin/cos data
	loops	128, b4b		;; Test loop counter
	bump	rsi, -128*64+4*dist128	;; Next source pointer
	bump	rdi, -128*XMM_SCD2	;; Restore sin/cos ptr
	bump	rdx, -128*16*XMM_HSCD2	;; Restore sin/cos ptr

;; Do 3584 four_complex_djbfft operations
;;	distance between fft data elements is 128
;;	do 896 macros each processing 32 data values

	loops_reset			;; 7 iterations of 128
	r4_x4cl_four_complex_djbfft_preload
b5b:	r4_x4cl_four_complex_djbfft rsi, 64, dist128, 2*dist128, rdi
	bump	rdi, XMM_SCD2		;; Next sin/cos data
	loops	128, b5b		;; Test inner loop counter
	bump	rsi, -128*64+4*dist128	;; Next source pointer
	bump	rdi, -128*XMM_SCD2	;; Restore sin/cos ptr
	loops	8, b5b			;; Test loop counter
	bump	rsi, -8*4*dist128	;; Next source pointer

;; Do FFT levels 7,8
;; Values 0-255 is real data, 256-32767 is complex data.
;;
;; On input the 64-byte cache lines hold these data values:
;;	0	+16K	128	+16K	256	+16K	384	+16K
;;	1	...
;;	...
;;	127	...
;;	512	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+16K	32	+16K	64	+16K	96	+16K
;;	1	...
;;	...
;;	31	...
;;	128	...
;;	...

;; Do 32 eight_reals_fft operations
;; Do 96 four_complex_djbfft operations
;;	distance between fft data elements is 32
;;	do 32 macros each processing 32 data values

	loops_init 1024
	r4_h4cl_eight_reals_four_complex_djbfft_preload
b6b:	r4_h4cl_eight_reals_four_complex_djbfft rsi, 64, 32*64, 64*64, rdi, rdx
	bump	rdi, 4*XMM_SCD2		;; Next sin/cos data
	bump	rdx, 64*XMM_HSCD2	;; Next sin/cos data
	loops	32, b6b			;; Test loop counter
	bump	rsi, -32*64+dist128	;; Next source pointer
	bump	rdi, -32*4*XMM_SCD2	;; Restore sin/cos ptr
	bump	rdx, -32*64*XMM_HSCD2	;; Restore sin/cos ptr

;; Do 3978 four_complex_djbfft operations
;;	distance between fft data elements is 32
;;	do 992 macros each processing 32 data values

	loops_reset			;; 31 iters of 32
	r4_x4cl_four_complex_djbfft_preload
b7b:	r4_x4cl_four_complex_djbfft rsi, 64, 32*64, 64*64, rdi
	bump	rdi, 4*XMM_SCD2		;; Next sin/cos data
	loops	32, b7b			;; Test inner loop counter
	bump	rsi, -32*64+dist128	;; Next source pointer
	bump	rdi, -32*4*XMM_SCD2	;; Restore sin/cos ptr
	loops	32, b7b			;; 32 iterations
	bump	rsi, -32*dist128	;; Next source pointer

;; Do FFT levels 9,10
;; Values 0-31 is real data, 32-16383 is complex data.
;;
;; On input the 64-byte cache lines hold these data values:
;;	0	+16K	32	+16K	64	+16K	96	+16K
;;	1	...
;;	...
;;	31	...
;;	128	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+16K	8	+16K	16	+16K	24	+16K
;;	1	...
;;	...
;;	7	...
;;	32	...
;;	...

;; Do 8 eight_reals_fft operations
;; Do 24 four_complex_djbfft operations
;;	distance between fft data elements is 8
;;	do 8 macros each processing 32 data values

	loops_init 1024
	r4_h4cl_eight_reals_four_complex_djbfft_preload
b8b:	r4_h4cl_eight_reals_four_complex_djbfft rsi, 64, 8*64, 16*64, rdi, rdx
	bump	rdi, 16*XMM_SCD2	;; Next sin/cos data
	bump	rdx, 256*XMM_HSCD2	;; Next sin/cos data
	loops	8, b8b			;; Test loop counter
	bump	rsi, -8*64+32*64	;; Next source pointer
	bump	rdi, -8*16*XMM_SCD2	;; Restore sin/cos ptr
	bump	rdx, -8*256*XMM_HSCD2	;; Restore sin/cos ptr

;; Do 4064 four_complex_djbfft operations
;;	distance between fft data elements is 8
;;	do 1020 macros each processing 32 data values

	loops_reset			;; 32 iters of 7 or 8 of 4
	r4_x4cl_four_complex_djbfft_preload
b9b:	r4_x4cl_four_complex_djbfft rsi, 64, 8*64, 16*64, rdi
	bump	rdi, 16*XMM_SCD2	;; Next sin/cos data
	loops	8, b9b			;; Test loop counter
	bump	rsi, -8*64+32*64	;; Next source pointer
	bump	rdi, -8*16*XMM_SCD2	;; Restore sin/cos ptr
	loops	4, b9b			;; Test middle loop counter
	bump	rsi, -4*32*64+dist128	;; Next source pointer
	loops	32, b9b			;; 32 iterations
	bump	rsi, -32*dist128	;; Next source pointer

;; Do FFT levels 11,12,13
;; Values 0-7 is real data, 8-32767 is complex data.
;;
;; On input the 64-byte cache lines hold these data values:
;;	0	+16K	8	+16K	16	+16K	24	+16K
;;	1	...
;;	...
;;	7	...
;;	32	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+16K	4	+16K	8	+16K	12	+16K
;;	...
;;	3	...
;;	16	...
;;	...

;; Do 1 sixteen_reals_fft operations
;; Do 3 eight_complex_fft operations
;;	distance between fft data elements is 1

;; Do 2044 eight_complex_fft operations
;;	distance between fft data elements is 1
;;	do 511 macros each processing 64 data values

;; Execute the proper middle step

	cmp	ffttype, 2
	jl	xpass2_13_levels_real_1
	jg	xpass2_13_levels_real_3

xpass2_13_levels_real_2:
	r8_h8cl_sixteen_reals_eight_complex_with_square rsi, 8*64, 64, 2*64, 4*64
	loops_init_prefetch 512, 128, 1, rcx, 1, 16
	r8_x8cl_eight_complex_with_square_preload
baa0:	xtouch	[rcx+4096-128]		;; Preload the TLBs
baa:	r8_x8cl_eight_complex_with_square rsi, 8*64, 64, 2*64, 4*64
	loops	16, baa			;; Test loop counter
	bump	rsi, -16*8*64+dist128	;; Next source pointer
	loops_touch 2, baa, 16, baa0	;; Test loop counter
	bump	rsi, -16*2*dist128	;; Next source pointer
	jmp	xpass2_13_real_unfft

xpass2_13_levels_real_1:
	r8_h8cl_sixteen_reals_eight_complex_fft_final rsi, 8*64, 64, 2*64, 4*64
	loops_init_prefetch 512, 128, 1, rcx, 1, 16
	r8_x8cl_eight_complex_fft_final_preload
bab0:	xtouch	[rcx+4096-128]		;; Preload the TLBs
bab:	r8_x8cl_eight_complex_fft_final rsi, 8*64, 64, 2*64, 4*64
	loops	16, bab			;; Test loop counter
	bump	rsi, -16*8*64+dist128	;; Next source pointer
	loops_touch 2, bab, 16, bab0	;; Test loop counter
	jmp	xpass2_13_real_done

xpass2_13_levels_real_3:
	mov	rbp, DIST_TO_MULSRCARG
	r8_h8cl_sixteen_reals_eight_complex_with_mult rsi, 8*64, 64, 2*64, 4*64
	loops_init_prefetch 512, 128, 1, rcx, 1, 16
	r8_x8cl_eight_complex_with_mult_preload
bac0:	xtouch	[rcx+4096-128]		;; Preload the TLBs
bac:	r8_x8cl_eight_complex_with_mult rsi, 8*64, 64, 2*64, 4*64
	loops	16, bac			;; Test loop counter
	bump	rsi, -16*8*64+dist128	;; Next source pointer
	loops_touch 2, bac, 16, bac0	;; Test loop counter
	bump	rsi, -16*2*dist128	;; Next source pointer
	jmp	xpass2_13_real_unfft

xpass2_13_levels_real_4:
	mov	rbp, DIST_TO_MULSRCARG
	mov	rcx, DATA_PREFETCH	;; Load prefetch pointer
	r8_h8cl_sixteen_reals_eight_complex_with_mulf rsi, 8*64, 64, 2*64, 4*64
	loops_init_prefetch 512, 128, 1, rcx, 1, 16
	r8_x8cl_eight_complex_with_mulf_preload
	xtouch	[rcx]			;; Preload the TLBs
bad0:	xtouch	[rcx+4096-128]		;; Preload the TLBs
bad:	r8_x8cl_eight_complex_with_mulf rsi, 8*64, 64, 2*64, 4*64
	loops	16, bad			;; Test loop counter
	bump	rsi, -16*8*64+dist128	;; Next source pointer
	loops_touch 2, bad, 16, bad0	;; Test loop counter
	bump	rsi, -16*2*dist128	;; Next source pointer

;; Do inverse FFT levels 9,10
;; On input the 64-byte cache lines hold these data values:
;;	0	+16K	4	+16K	8	+16K	12	+16K
;;	...
;;	3	...
;;	16	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+16K	16	+16K	32	+16K	48	+16K
;;	1	...
;;	...
;;	15	...
;;	64	...
;;	...

;; Do 8 eight_reals_unfft operations
;; Do 8 four_complex_djbunfft operations
;;	distance between fft data elements is 8
;;	do 4 macros processing 32 data values

xpass2_13_real_unfft:
	mov	rdi, xsincos_complex	;; Load sin/cos pointers
	bump	rdi, 2048*XMM_SCD2
	mov	rdx, sincos3
	loops_init 1024
c8b:	r4_h4cl_eight_reals_four_complex_djbunfft rsi, 64, 4*64, 8*64, rdi, 4*16*XMM_SCD2, rdx, 4*256*XMM_HSCD2
	bump	rdi, 16*XMM_SCD2	;; Next sin/cos data
	bump	rdx, 256*XMM_HSCD2	;; Next sin/cos data
	loops	4, c8b			;; Test loop counter
	bump	rsi, -4*64+16*64	;; Next source pointer
	bump	rdi, -4*16*XMM_SCD2	;; Restore sin/cos ptr
	bump	rdx, -4*256*XMM_HSCD2	;; Restore sin/cos ptr

;; Do 4080 four_complex_djbunfft operations
;;	distance between fft data elements is 8
;;	do 1020 macros each processing 32 data values

	loops_reset			;; 32 iters of 7 or 8 of 4
	r4_x4cl_four_complex_djbunfft_preload
c9b:	r4_x4cl_four_complex_djbunfft rsi, 64, 4*64, 8*64, rdi, 4*16*XMM_SCD2
	bump	rdi, 16*XMM_SCD2	;; Next sine/cosine pointer
	loops	4, c9b			;; Test loop counter
	bump	rsi, -4*64+16*64	;; Next source pointer
	bump	rdi, -4*16*XMM_SCD2	;; Restore sin/cos ptr
	loops	8, c9b			;; Test loop counter
	bump	rsi, -8*16*64+dist128	;; Next source pointer
	loops	32, c9b			;; Test loop counter
	bump	rsi, -32*dist128	;; Next source pointer

;; Do inverse FFT levels 7,8
;; On input the 64-byte cache lines hold these data values:
;;	0	+16K	16	+16K	32	+16K	48	+16K
;;	1	...
;;	...
;;	15	...
;;	64	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+16K	64	+16K	128	+16K	192	+16K
;;	1	...
;;	...
;;	63	...
;;	256	...
;;	...

;; Do 32 eight_reals_unfft operations
;; Do 32 four_complex_djbunfft operations
;;	distance between fft data elements is 32
;;	do 16 macros processing 32 data values

	loops_init_prefetch 1024, 64, 1, rcx
c6b:	r4_h4cl_eight_reals_four_complex_djbunfft rsi, 64, 16*64, 32*64, rdi, 16*4*XMM_SCD2, rdx, 16*64*XMM_HSCD2
	bump	rdi, 4*XMM_SCD2		;; Next sin/cos data
	bump	rdx, 64*XMM_HSCD2	;; Next sin/cos data
	loops	16, c6b			;; Test inner loop counter
	bump	rsi, -16*64+64*64	;; Next source pointer
	bump	rdi, -16*4*XMM_SCD2	;; Restore sin/cos ptr
	bump	rdx, -16*64*XMM_HSCD2	;; Restore sin/cos ptr

;; Do 4032 four_complex_djbunfft operations
;;	distance between fft data elements is 32
;;	do 1008 macros each processing 32 data values

	loops_reset			;; 32 iters of 1 or 2 of 16
	r4_x4cl_four_complex_djbunfft_preload
c7b:	r4_x4cl_four_complex_djbunfft rsi, 64, 16*64, 32*64, rdi, 16*4*XMM_SCD2
	bump	rdi, 4*XMM_SCD2		;; Next sin/cos data
	loops	16, c7b			;; Test inner loop counter
	bump	rsi, -16*64+64*64	;; Next source pointer
	bump	rdi, -16*4*XMM_SCD2	;; Restore sin/cos ptr
	loops	2, c7b			;; Test loop counter
	bump	rsi, -2*64*64+dist128	;; Next source pointer
	loops	32, c7b			;; 32 iterations
	bump	rsi, -32*dist128	;; Next source pointer

;; Do inverse FFT levels 5,6
;; On input the 64-byte cache lines hold these data values:
;;	0	+16K	64	+16K	128	+16K	192	+16K
;;	1	...
;;	...
;;	63	...
;;	256	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+16K	256	+16K	512	+16K	768	+16K
;;	1	...
;;	...
;;	255	...
;;	1024	...
;;	...

;; Do 128 eight_reals_unfft operations
;; Do 128 four_complex_djbunfft operations
;;	distance between fft data elements is 128
;;	do 64 macros processing 32 data values

	loops_init_prefetch 1024, 64, 1, rcx
c4b:	r4_h4cl_eight_reals_four_complex_djbunfft rsi, 64, 64*64, dist128, rdi, 64*XMM_SCD2, rdx, 64*16*XMM_HSCD2
	bump	rdi, XMM_SCD2		;; Next sin/cos data
	bump	rdx, 16*XMM_HSCD2	;; Next sin/cos data
	loops	64, c4b			;; Test inner loop counter
	bump	rsi, -64*64+2*dist128	;; Next source pointer
	bump	rdi, -64*XMM_SCD2	;; Restore sin/cos ptr
	bump	rdx, -64*16*XMM_HSCD2	;; Restore sin/cos ptr

;; Do 3840 four_complex_djbunfft operations
;;	distance between fft data elements is 128
;;	do 960 macros each processing 32 data values

	loops_reset			;; 15 iters of 64
	r4_x4cl_four_complex_djbunfft_preload
c5b:	r4_x4cl_four_complex_djbunfft rsi, 64, 64*64, dist128, rdi, 64*XMM_SCD2
	bump	rdi, XMM_SCD2		;; Next sin/cos data
	loops	64, c5b			;; Test inner loop counter
	bump	rsi, -64*64+2*dist128	;; Next source pointer
	bump	rdi, -64*XMM_SCD2	;; Restore sin/cos ptr
	loops	16, c5b			;; Test loop counter
	bump	rsi, -16*2*dist128	;; Next source pointer

;; Do inverse FFT levels 3,4
;; On input the 64-byte cache lines hold these data values:
;;	0	+16K	256	+16K	512	+16K	768	+16K
;;	1	...
;;	...
;;	255	...
;;	1024	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+16K	1024	+16K	2048	+16K	3072	+16K
;;	1	...
;;	...
;;	1023	...
;;	4096	...
;;	...

;; Do 512 eight_reals_unfft operations
;; Do 512 four_complex_djbunfft operations
;;	distance between fft data elements is 512
;;	do 256 macros processing 32 data values

	loops_init 1024
	mov	rdi, xsincos_complex	;; Load sin/cos pointer
c2b:	r4_h4cl_eight_reals_four_complex_djbunfft rsi, 64, 2*dist128, 4*dist128, rdi, 256*4*XMM_SCD2, rdx, 256*4*XMM_HSCD2
	bump	rdi, 4*XMM_SCD2		;; Next sin/cos data
	bump	rdx, 4*XMM_HSCD2	;; Next sin/cos data
	loops	128, c2b		;; Test inner loop counter
	bump	rsi, -128*64+dist128	;; Next source pointer
	loops	2, c2b			;; Test inner loop counter
	bump	rsi, -2*dist128+8*dist128 ;; Next source pointer
	bump	rdi, -2*128*4*XMM_SCD2	;; Restore sin/cos ptr
	bump	rdx, -2*128*4*XMM_HSCD2	;; Restore sin/cos ptr

;; Do 3072 four_complex_djbunfft operations
;;	distance between fft data elements is 512
;;	do 768 macros each processing 32 data values

	loops_reset			;; 3 iters of 256
	r4_x4cl_four_complex_djbunfft_preload
c3b:	r4_x4cl_four_complex_djbunfft rsi, 64, 2*dist128, 4*dist128, rdi, 256*4*XMM_SCD2
	bump	rdi, 4*XMM_SCD2		;; Next sin/cos data
	loops	128, c3b		;; Test inner loop counter
	bump	rsi, -128*64+dist128	;; Next source pointer
	loops	2, c3b			;; Test inner loop counter
	bump	rsi, -2*dist128+8*dist128 ;; Next source pointer
	bump	rdi, -2*128*4*XMM_SCD2	;; Restore sin/cos ptr
	loops	4, c3b			;; Test loop counter
	bump	rsi, -4*8*dist128	;; Next source pointer

;; Do inverse FFT level 1,2
;; On input the 64-byte cache lines hold these data values:
;;	0	+16K	1024	+16K	2048	+16K	3072	+16K
;;	1	...
;;	...
;;	1023	...
;;	4096	...
;;	...
;; On output the 64-byte cache lines hold these data values if pass 1 swizzles:
;;	0	+16K	4096	+16K	8192	+16K	12288	+16K
;;	1	...
;;	...
;;	4095
;; On output the 64-byte cache lines hold these data values if pass 2 swizzles:
;;	0	+4K	8192	+4K	16384	+4K	24576	+4K
;;	1	...
;;	...
;;	4095

;; Do 2048 eight_reals_unfft operations
;; Do 2048 four_complex_djbunfft operations
;;	distance between fft data elements is 2048
;;	do 1024 macros processing 32 data values

	loops_init 1024
c1b:	r4_h4cl_eight_reals_four_complex_djbunfft rsi, 64, 8*dist128, 16*dist128, rdi, 1024*XMM_SCD2, rdx, 1024*XMM_HSCD2
	bump	rdi, XMM_SCD2		;; Next sin/cos data
	bump	rdx, XMM_HSCD2		;; Next sin/cos data
	loops	128, c1b		;; Test loop counter
	bump	rsi, -128*64+dist128	;; Next source pointer
	loops	8, c1b			;; Test loop counter

xpass2_13_real_done:
	end_timer 2
	ENDM


; *************** radix4-pass2-13-levels-complex macro ******************
; This macro takes 2 groups of 8192 complex values and
; performs the final 13 levels of a standard radix-4 FFT, squares the
; results, and does the first 13 levels of the inverse FFT.

r4_pass2_13_levels_complex MACRO
	LOCAL	b1b, b2b, b3a, b3b, b4a, b4b, b5a, b5b
	LOCAL	b6a0, b6a, b6b0, b6b, b6c0, b6c, b6d0, b6d
	LOCAL	c1b, c2a, c2b, c3a, c3b, c4a, c4b, c5a, c5b
	LOCAL	xpass2_13_levels_complex_1, xpass2_13_levels_complex_2
	LOCAL	xpass2_13_levels_complex_3, xpass2_13_levels_complex_4
	LOCAL	xpass2_13_levels_complex_unfft, xpass2_13_levels_complex_done

;; Type 4 FFTs skip the forward FFT process

	mov	rsi, DATA_ADDR		;; Load address of FFT data
	mov	rbx, DIST_TO_FFTSRCARG
	cmp	ffttype, 4
	je	xpass2_13_levels_complex_4

;; Do FFT level 1,2
;;
;; We swizzle the data so that one set of data is in the high word of
;; an xmm register, the other set is in the low word.  This means
;; a distance of 16384 data values.
;;
;; On input the 64-byte cache lines hold these data values if pass 2 swizzles:
;;	0	+4K	8192	+4K	16384	+4K	24576	+4K
;;	1	...
;;	...
;;	4095
;; On input the 64-byte cache lines hold these data values if pass 1 swizzles:
;;	0	+16K	4096	+16K	8192	+16K	12288	+16K
;;	1	...
;;	...
;;	4095
;; On output the 64-byte cache lines hold these data values:
;;	0	+16K	2048	+16K	4096	+16K	6144	+16K
;;	1	...
;;	...
;;	2047	...
;;	8192	...
;;	...

;; Do 4096 four_complex_fft operations
;;	distance between fft data elements is 2048
;;	do 2048 macros each processing 16 data values

;; Do FFT levels 3,4
;;
;; On input the 64-byte cache lines hold these data values:
;; On output the 64-byte cache lines hold these data values:
;;	0	+16K	2048	+16K	4096	+16K	6144	+16K
;;	1	...
;;	...
;;	2047	...
;;	8192	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+16K	512	+16K	1024	+16K	1536	+16K
;;	1	...
;;	...
;;	511	...
;;	2048	...
;;	...

;; Do 4096 four_complex_fft operations
;;	distance between fft data elements is 512
;;	do 1024 macros each processing 32 data values

IF @INSTR(,%xarch,<CORE>) NE 0		;; Core 2 faster doing separate loops

	start_timer 4
	mov	rdi, xsincos_complex	;; The sin/cos data
	loops_init 2048			;; 16 iterations of 128
	r4_f2cl_four_complex_djbfft_preload
b1b:	r4_f2cl_four_complex_djbfft rsi, 64, 16*dist128, rdi
	bump	rdi, XMM_SCD2		;; Next sin/cos data
	loops	128, b1b		;; Test inner loop counter
	bump	rsi, -128*64+dist128	;; Next source pointer
	loops	16, b1b			;; Test outer loop counter
	bump	rsi, -16*dist128	;; Restore source pointer
	end_timer 4

	start_timer 5
	mov	rdi, xsincos_complex	;; Ptr to the sin/cos data
	loops_init 1024			;; 2 iters of 4 iters of 128
	r4_x4cl_four_complex_djbfft_preload
b2b:	r4_x4cl_four_complex_djbfft rsi, 64, 4*dist128, 8*dist128, rdi
	bump	rdi, 4*XMM_SCD2		;; Next sin/cos data
	loops	128, b2b		;; Test inner loop counter
	bump	rsi, -128*64+dist128	;; Next source pointer
	loops	4, b2b			;; Test loop counter
	bump	rsi, -4*dist128+16*dist128 ;; Next source pointer
	bump	rdi, -512*4*XMM_SCD2	;; Restore sin/cos ptr
	loops	2, b2b			;; Test loop counter
	bump	rsi, -2*16*dist128	;; Restore source pointer
	end_timer 5

ELSE					;; Pentium 4 faster doing merged loops

	start_timer 4
	mov	rdi, xsincos_complex	;; The sin/cos data
	mov	rbp, rdi
	loops_init 2048			;; 16 iterations of 128
	;r4_f2cl_four_complex_djbfft_preload -- assume same as r4_x4cl_four_complex_djbfft_preload
	r4_x4cl_four_complex_djbfft_preload

b1b:	r4_f2cl_four_complex_djbfft rsi, 4*dist128, 16*dist128, rdi
	xxprefetch [rdi+XMM_SCD2]	;; Prefetch for next pass
	bump	rdi, 512*XMM_SCD2	;; Next sin/cos ptr
	loops	4, b1b			;; Test loop counter
	bump	rsi, -4*4*dist128	;; Restore source pointer
	loops_undo 4

b2b:	r4_x4cl_four_complex_djbfft rsi, 16*dist128, 4*dist128, 8*dist128, rbp
	loops	2, b2b, 2		;; Test loop counter with non-standard incr

	bump	rsi, -2*16*dist128+64	;; Next source pointer
	bump	rdi, -4*512*XMM_SCD2+XMM_SCD2 ;; Next sin/cos ptr
	bump	rbp, 4*XMM_SCD2		;; Next sin/cos data
	loops	128, b1b		;; Test loop counter
	bump	rsi, -128*64+dist128	;; Next source pointer
	loops	4, b1b			;; Test loop counter
	bump	rsi, -4*dist128		;; Restore source pointer
	bump	rdi, -512*XMM_SCD2	;; Restore sin/cos ptr
	end_timer 4

ENDIF

;; Do FFT levels 5,6
;;
;; On input the 64-byte cache lines hold these data values:
;;	0	+16K	512	+16K	1024	+16K	1536	+16K
;;	1	...
;;	...
;;	511	...
;;	2048	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+16K	128	+16K	256	+16K	384	+16K
;;	1	...
;;	...
;;	127	...
;;	512	...
;;	...
;;
;; Do 4096 four_complex_fft operations
;;	distance between fft data elements is 128
;;	do 1024 macros each processing 32 data values

;; Do FFT levels 7,8
;;
;; On input the 64-byte cache lines hold these data values:
;;	0	+16K	128	+16K	256	+16K	384	+16K
;;	1	...
;;	...
;;	127	...
;;	512	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+16K	32	+16K	64	+16K	96	+16K
;;	1	...
;;	...
;;	31	...
;;	128	...
;;	...
;;
;; Do 4096 four_complex_fft operations
;;	distance between fft data elements is 32
;;	do 1024 macros each processing 32 data values

;; Do FFT levels 9,10
;;
;; On input the 64-byte cache lines hold these data values:
;;	0	+16K	32	+16K	64	+16K	96	+16K
;;	1	...
;;	...
;;	31	...
;;	128	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+16K	8	+16K	16	+16K	24	+16K
;;	1	...
;;	...
;;	7	...
;;	32	...
;;	...
;;
;; Do 4096 four_complex_fft operations
;;	distance between fft data elements is 8
;;	do 1024 macros each processing 32 data values

;; Do this 64 cache lines at a time to maximize accessing data
;; while it is in the L1 cache.

IFDEF NEVER				;; Never faster to do separate loops

	start_timer 6
	mov	rcx, DATA_PREFETCH	;; Prefetch pointer
	xtouch	[rcx]			;; Load first TLB
	bump	rdi, 2048*XMM_SCD2	;; Load sin/cos pointer
	loops_init_prefetch 1024, 64, 2, rcx ;; 4*2 iters of 128
	r4_x4cl_four_complex_djbfft_preload
b3a:	xtouch	[rcx+4096-64]		;; Load prefetch TLBs
b3b:	r4_x4cl_four_complex_djbfft rsi, 64, dist128, 2*dist128, rdi
	bump	rdi, XMM_SCD2		;; Next sine/cosine pointer
	loops	128, b3b		;; Loop 128 times
	bump	rsi, -128*64+4*dist128	;; Next source pointer
	bump	rdi, -128*XMM_SCD2	;; Restore sine/cosine pointer
	loops	2, b3a			;; Test loop counter
	bump	rcx, -128*64+8192+128	;; Next 8KB section to prefetch
	loops	4, b3a			;; Test loop counter
	bump	rsi, -4*2*4*dist128	;; Restore source pointer
	end_timer 6

	start_timer 7
	loops_init_prefetch 1024, 64, 2, rcx ;; 4*8 iters of 32
	r4_x4cl_four_complex_djbfft_preload
b4a:	xtouch	[rcx+4096-64]		;; Load two prefetch TLBs
b4b:	r4_x4cl_four_complex_djbfft rsi, 64, 32*64, 64*64, rdi
	bump	rdi, 4*XMM_SCD2		;; Next sine/cosine pointer
	loops	32, b4b			;; Loop 32 times
	bump	rsi, -32*64+dist128	;; Next source pointer
	bump	rdi, -32*4*XMM_SCD2	;; Restore sine/cosine pointer
	loops_touch 4, b4b, 2, b4a	;; Test loop counter
	bump	rcx, -128*64+8192+128	;; Next 8KB section to prefetch
	loops	4, b4a			;; Test loop counter
	bump	rsi, -4*8*dist128	;; Restore source pointer
	end_timer 7

	start_timer 8
	loops_init_prefetch 1024, 64, 2, rcx ;; 4*8*4 iters of 8
	r4_x4cl_four_complex_djbfft_preload
b5a:	xtouch	[rcx+4096-64]		;; Load two prefetch TLBs
b5b:	r4_x4cl_four_complex_djbfft rsi, 64, 8*64, 16*64, rdi
	bump	rdi, 16*XMM_SCD2	;; Next sine/cosine pointer
	loops	8, b5b			;; Loop 8 times
	bump	rsi, -8*64+32*64	;; Next source pointer
	bump	rdi, -8*16*XMM_SCD2	;; Restore sine/cosine pointer
	loops	4, b5b			;; Test loop counter
	bump	rsi, -4*32*64+dist128	;; Adjust source pointer
	loops_touch 4, b5b, 2, b5a	;; Test loop counter
	bump	rcx, -128*64+8192+128	;; Next 8KB section to prefetch
	loops	4, b5a			;; Test loop counter
	bump	rsi, -4*8*dist128	;; Restore source pointer
	end_timer 8

ELSE					;; Always faster to do merged loops

	start_timer 6
	mov	rcx, DATA_PREFETCH	;; Prefetch pointer
	xtouch	[rcx]			;; Load first TLB
	bump	rdi, 2048*XMM_SCD2	;; Load sin/cos pointers
	mov	rbx, rdi		;; Copy sin/cos ptrs
	mov	rbp, rdi
	loops_init_prefetch 1024, 64, 2, rcx ;; 4*2*8 iters of 64 cache lines
	r4_x4cl_four_complex_djbfft_preload

b3a:	xtouch	[rcx+4096-128]		;; Load six prefetch TLBs
	xtouch	[rcx+2*4096-128]
	xtouch	[rcx+3*4096-128]
	xtouch	[rcx+4*4096-128]
	xtouch	[rcx+5*4096-128]
	xtouch	[rcx+6*4096-128]

b3b:	xxprefetch [rdi+8*XMM_SCD2]	;; Prefetch for next pass
	r4_x4cl_four_complex_djbfft rsi, 8*64, dist128, 2*dist128, rdi
	bump	rdi, 8*XMM_SCD2		;; Next sine/cosine pointer
	loops	16, b3b			;; Loop 16 times
	bump	rsi, -16*8*64		;; Restore source pointer
	bump	rcx, -8*64+8192+128	;; Prefetch from next section
	loops_undo 16			;; Back up loop counter

b4b:	r4_x4cl_four_complex_djbfft rsi, 8*64, 32*64, 64*64, rbx
	bump	rbx, 8*4*XMM_SCD2	;; Next sine/cosine pointer
	loops	4, b4b			;; Loop 4 times
	bump	rsi, -4*8*64+dist128	;; Next source pointer
	bump	rbx, -4*8*4*XMM_SCD2	;; Restore sine/cosine pointer
	loops	4, b4b			;; Loop 4 times
	bump	rsi, -4*dist128		;; Restore source pointer
	bump	rcx, -8*64+8192+128	;; Prefetch from next section
	loops_undo 4*4			;; Back up loop counter

b5b:	r4_x4cl_four_complex_djbfft rsi, 32*64, 8*64, 16*64, rbp
	loops	4, b5b			;; Loop 8 times
	bump	rsi, -4*32*64+dist128	;; Next source pointer
	loops	4, b5b			;; Loop twice
	bump	rcx, -2*(8192+128)	;; Back to prefetching from 1st section

	bump	rsi, -4*dist128+64	;; Next source pointer
	bump	rdi, -16*8*XMM_SCD2+XMM_SCD2 ;; Next sine/cosine pointer
	bump	rbx, 4*XMM_SCD2		;; Next sine/cosine pointer
	bump	rbp, 16*XMM_SCD2	;; Next sine/cosine pointer
	loops	8, b3b			;; Loop 8 times
	bump	rsi, -8*64+4*dist128	;; Next source pointer
	bump	rdi, -8*XMM_SCD2	;; Restore sine/cosine pointer
	bump	rbx, -8*4*XMM_SCD2	;; Restore sine/cosine pointer
	bump	rbp, -8*16*XMM_SCD2	;; Restore sine/cosine pointer
	loops	2, b3b			;; Loop twice
	bump	rcx, -16*8*64+3*(8192+128) ;; Prefetch next 3 sections
	loops	4, b3a			;; Loop 4 times
	bump	rsi, -4*2*4*dist128	;; Restore source pointer
	end_timer 6
ENDIF

;; Do FFT levels 11,12,13 as well as inverse FFT levels 11,12,13
;;
;; On input the 64-byte cache lines hold these data values:
;;	0	+16K	8	+16K	16	+16K	24	+16K
;;	1	...
;;	...
;;	7	...
;;	32	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+16K	4	+16K	8	+16K	12	+16K
;;	...
;;	3	...
;;	16	...
;;	...

;; Do 2048 eight_complex_fft operations
;;	distance between fft data elements is 1
;;	do 512 macros each processing 64 data values

;; Execute the right middle step

	cmp	ffttype, 2
	jl	xpass2_13_levels_complex_1
	jg	xpass2_13_levels_complex_3

xpass2_13_levels_complex_2:
	start_timer 9
	loops_init_prefetch 512, 64, 1, rcx
	r8_x8cl_eight_complex_with_square_preload
b6a0:	xtouch	[rcx+4096-64]		;; Load prefetch TLBs
b6a:	r8_x8cl_eight_complex_with_square rsi, 8*64, 64, 2*64, 4*64
	loops	16, b6a			;; Loop 16 times
	bump	rsi, -16*8*64+dist128	;; Next source pointer
	loops_touch 4, b6a, 2, b6a0	;; Loop 8 times
	bump	rcx, -64*128+(8192+128)	;; Prefetch next section
	loops	4, b6a0			;; Loop 4 times
	bump	rsi, -4*8*dist128	;; Restore source pointer
	end_timer 9
	jmp	xpass2_13_levels_complex_unfft

xpass2_13_levels_complex_1:
	start_timer 9
	loops_init_prefetch 512, 128, 1, rcx
	r8_x8cl_eight_complex_fft_final_preload
b6b0:	xtouch	[rcx+4096-64]		;; Load prefetch TLBs
b6b:	r8_x8cl_eight_complex_fft_final rsi, 8*64, 64, 2*64, 4*64
	loops	16, b6b			;; Loop 16 times
	bump	rsi, -16*8*64+dist128	;; Next source pointer
	loops_touch 2, b6b, 2, b6b0	;; Loop 4 times
	bump	rcx, 128		;; Prefetch next section
	loops	8, b6b0			;; Loop 8 times
	end_timer 9
	jmp	xpass2_13_levels_complex_done

xpass2_13_levels_complex_3:
	start_timer 9
	mov	rbp, DIST_TO_MULSRCARG
	loops_init_prefetch 512, 64, 1, rcx
	r8_x8cl_eight_complex_with_mult_preload
b6c0:	xtouch	[rcx+4096-64]		;; Load prefetch TLBs
b6c:	r8_x8cl_eight_complex_with_mult rsi, 8*64, 64, 2*64, 4*64
	loops	16, b6c			;; Loop 16 times
	bump	rsi, -16*8*64+dist128	;; Next source pointer
	loops_touch 4, b6c, 2, b6c0	;; Loop 8 times
	bump	rcx, -64*128+(8192+128)	;; Prefetch next section
	loops	4, b6c0			;; Loop 4 times
	bump	rsi, -4*8*dist128	;; Restore source pointer
	end_timer 9
	jmp	xpass2_13_levels_complex_unfft

xpass2_13_levels_complex_4:
	start_timer 9
	mov	rcx, DATA_PREFETCH	;; Prefetch pointer
	xtouch	[rcx]			;; Load first prefetch TLB
	mov	rbp, DIST_TO_MULSRCARG
	loops_init_prefetch 512, 128, 1, rcx
	r8_x8cl_eight_complex_with_mulf_preload
b6d0:	xtouch	[rcx+4096-64]		;; Load prefetch TLBs
b6d:	r8_x8cl_eight_complex_with_mulf rsi, 8*64, 64, 2*64, 4*64
	loops	16, b6d			;; Loop 16 times
	bump	rsi, -16*8*64+dist128	;; Next source pointer
	loops_touch 2, b6d, 2, b6d0	;; Loop 4 times
	bump	rcx, 128		;; Prefetch next section
	loops	8, b6d0			;; Loop 8 times
	bump	rsi, -8*4*dist128	;; Restore source pointer
	end_timer 9

;; Do inverse FFT levels 9,10
;; On input the 64-byte cache lines hold these data values:
;;	0	+16K	4	+16K	8	+16K	12	+16K
;;	...
;;	3	...
;;	16	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+16K	16	+16K	32	+16K	48	+16K
;;	1	...
;;	...
;;	15	...
;;	64	...
;;	...
;;
;; Do 4096 four_complex_unfft operations
;;	distance between fft data elements is 8
;;	do 1024 macros each processing 32 data values

;; Do inverse FFT levels 7,8
;; On input the 64-byte cache lines hold these data values:
;;	0	+16K	16	+16K	32	+16K	48	+16K
;;	1	...
;;	...
;;	15	...
;;	64	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+16K	64	+16K	128	+16K	192	+16K
;;	1	...
;;	...
;;	63	...
;;	256	...
;;	...
;;
;; Do 4096 four_complex_unfft operations
;;	distance between fft data elements is 32
;;	do 1024 macros each processing 32 data values

;; Do inverse FFT levels 5,6
;; On input the 64-byte cache lines hold these data values:
;;	0	+16K	64	+16K	128	+16K	192	+16K
;;	1	...
;;	...
;;	63	...
;;	256	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+16K	256	+16K	512	+16K	768	+16K
;;	1	...
;;	...
;;	255	...
;;	1024	...
;;	...
;;
;; Do 4096 four_complex_unfft operations
;;	distance between fft data elements is 128
;;	do 1024 macros each processing 32 data values

;; Do this 64 cache lines at a time to maximize accessing data
;; while it is in the L1 cache.

xpass2_13_levels_complex_unfft:

IFDEF NEVER				;; Never faster to do separate loops

	start_timer 10
	mov	rdi, xsincos_complex	;; Load sin/cos pointers
	bump	rdi, 2048*XMM_SCD2
	loops_init_prefetch 1024, 64, 2, rcx;; 4 iters of 8 iters of 8 iters of 4
	r4_x4cl_four_complex_djbunfft_preload
c5a:	xtouch	[rcx+4096-64]		;; Load prefetch TLBs
c5b:	r4_x4cl_four_complex_djbunfft rsi, 64, 4*64, 8*64, rdi, 4*16*XMM_SCD2
	bump	rdi, 16*XMM_SCD2	;; Next sine/cosine pointer
	loops	4, c5b			;; Loop 4 times
	bump	rsi, -4*64+16*64	;; Next source pointer
	bump	rdi, -4*16*XMM_SCD2	;; Reset sine/cosine pointer
	loops	8, c5b			;; Test loop counter
	bump	rsi, -8*16*64+dist128	;; Next source pointer
	loops_touch 4, c5b, 2, c5a	;; Loop 8 times
	bump	rcx, -128*64+8192+128	;; Next 8KB section to prefetch
	loops	4, c5a			;; Test loop counter
	bump	rsi, -4*8*dist128	;; Restore source pointer
	end_timer 10

	start_timer 11
	loops_init_prefetch 1024, 64, 2, rcx;; 4 iters of 8 iters of 2 iters of 16
	r4_x4cl_four_complex_djbunfft_preload
c4a:	xtouch	[rcx+4096-64]		;; Load prefetch TLBs
c4b:	r4_x4cl_four_complex_djbunfft rsi, 64, 16*64, 32*64, rdi, 16*4*XMM_SCD2
	bump	rdi, 4*XMM_SCD2		;; Next sine/cosine pointer
	loops	16, c4b			;; Loop 16 times
	bump	rsi, -16*64+64*64	;; Next source pointer
	bump	rdi, -16*4*XMM_SCD2	;; Reset sine/cosine pointer
	loops	2, c4b			;; Test loop counter
	bump	rsi, -2*64*64+dist128	;; Next source pointer
	loops_touch 4, c4b, 2, c4a	;; Loop 8 times
	bump	rcx, -128*64+8192+128	;; Next 8KB section to prefetch
	loops	4, c4a			;; Test loop counter
	bump	rsi, -4*8*dist128	;; Restore source pointer
	end_timer 11

	start_timer 12
	loops_init_prefetch 1024, 64, 2, rcx;; 4 iters of 4 iters of 64
	r4_x4cl_four_complex_djbunfft_preload
c3a:	xtouch	[rcx+4096-64]		;; Load prefetch TLBs
c3b:	r4_x4cl_four_complex_djbunfft rsi, 64, 64*64, dist128, rdi, 64*1*XMM_SCD2
	bump	rdi, 1*XMM_SCD2		;; Next sine/cosine pointer
	loops	64, c3b			;; Loop 64 times
	bump	rsi, -64*64+2*dist128	;; Next source pointer
	bump	rdi, -64*1*XMM_SCD2	;; Reset sine/cosine pointer
	loops_touch 2, c3b, 2, c3a	;; Loop 4 times
	bump	rcx, -128*64+8192+128	;; Next 8KB section to prefetch
	loops	4, c3a			;; Test loop counter
	bump	rsi, -4*4*2*dist128	;; Restore source pointer
	end_timer 12

ELSE					;; Always faster to do merged loops

	start_timer 10
	mov	rdi, xsincos_complex	;; Load sin/cos pointers
	bump	rdi, 2048*XMM_SCD2
	mov	rbx, rdi		;; Copy sin/cos ptrs
	mov	rbp, rdi
	loops_init_prefetch 1024, 64, 2, rcx ;; 4*2*8 iters of 64 cache lines
	r4_x4cl_four_complex_djbunfft_preload

c5a:	xtouch	[rcx+4096-128]		;; Load six prefetch TLBs
	xtouch	[rcx+2*4096-128]
	xtouch	[rcx+3*4096-128]
	xtouch	[rcx+4*4096-128]
	xtouch	[rcx+5*4096-128]
	xtouch	[rcx+6*4096-128]

c5b:	r4_x4cl_four_complex_djbunfft rsi, 16*64, 4*64, 8*64, rbp, 4*16*XMM_SCD2
	loops	8, c5b			;; Loop 8 times
	bump	rsi, -8*16*64+dist128	;; Next source pointer
	loops	2, c5b			;; Loop 2 times
	bump	rsi, -2*dist128		;; Restore source pointer
	bump	rcx, -8*64+8192+128	;; Prefetch from next section
	loops_undo 2*8			;; Restore loop counter

c4b:	r4_x4cl_four_complex_djbunfft rsi, 4*64, 16*64, 32*64, rbx, 16*4*XMM_SCD2
	bump	rbx, 4*4*XMM_SCD2	;; Next sine/cosine pointer
	loops	4, c4b			;; Loop 4 times
	bump	rsi, -4*4*64+64*64	;; Next source pointer
	bump	rbx, -4*4*4*XMM_SCD2	;; Restore sine/cosine pointer
	loops	2, c4b			;; Loop twice
	bump	rsi, -2*64*64+dist128	;; Next source pointer
	loops	2, c4b			;; Loop twice
	bump	rsi, -2*dist128		;; Restore source pointer
	bump	rcx, -8*64+8192+128	;; Prefetch from next section
	loops_undo 2*2*4		;; Restore loop counter

c3b:	r4_x4cl_four_complex_djbunfft rsi, 4*64, 64*64, dist128, rdi, 64*1*XMM_SCD2
	bump	rdi, 4*1*XMM_SCD2	;; Next sine/cosine pointer
	loops	16, c3b			;; Loop 16 times
	bump	rcx, -2*(8192+128)	;; Back to prefetching from 1st section

	bump	rsi, -16*4*64+64	;; Next source pointer
	bump	rdi, -16*4*1*XMM_SCD2+XMM_SCD2 ;; Next sine/cosine pointer
	bump	rbx, 4*XMM_SCD2		;; Next sine/cosine pointer
	bump	rbp, 16*XMM_SCD2	;; Next sine/cosine pointer
	loops	4, c5b			;; Loop 4 times
	bump	rsi, -4*64+2*dist128	;; Next source pointer
	bump	rdi, -4*XMM_SCD2	;; Restore sine/cosine pointer
	bump	rbx, -4*4*XMM_SCD2	;; Restore sine/cosine pointer
	bump	rbp, -4*16*XMM_SCD2	;; Restore sine/cosine pointer
	loops	4, c5b			;; Loop 4 times
	bump	rcx, -16*8*64+3*(8192+128) ;; Prefetch next 3 sections
	loops	4, c5a			;; Loop 4 times
	bump	rsi, -4*4*2*dist128	;; Restore source pointer
	end_timer 10
ENDIF

;; Do inverse FFT levels 3,4
;; On input the 64-byte cache lines hold these data values:
;;	0	+16K	256	+16K	512	+16K	768	+16K
;;	1	...
;;	...
;;	255	...
;;	1024	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+16K	1024	+16K	2048	+16K	3072	+16K
;;	1	...
;;	...
;;	1023	...
;;	4096	...
;;	...

;; Do 4096 four_complex_unfft operations
;;	distance between fft data elements is 512
;;	do 1024 macros each processing 32 data values

;; Do inverse FFT levels 1,2
;; On input the 64-byte cache lines hold these data values:
;;	0	+16K	1024	+16K	2048	+16K	3072	+16K
;;	1	...
;;	...
;;	1023	...
;;	4096	...
;;	...
;; On output the 64-byte cache lines hold these data values if pass 1 swizzles:
;;	0	+16K	4096	+16K	8192	+16K	12288	+16K
;;	1	...
;;	...
;;	4095
;; On output the 64-byte cache lines hold these data values if pass 2 swizzles:
;;	0	+4K	8192	+4K	16384	+4K	24576	+4K
;;	1	...
;;	...
;;	4095

;; Do 4096 four_complex_unfft operations
;;	distance between fft data elements is 2048
;;	do 1024 macros each processing 32 data values

IF @INSTR(,%xarch,<CORE>) NE 0		;; Core 2 faster doing separate loops

	start_timer 13
	mov	rdi, xsincos_complex	;; Load sin/cos pointer
	loops_init_prefetch 1024, 64, 2, rcx;; 4 iters of 2 iters of 128
	r4_x4cl_four_complex_djbunfft_preload
c2a:	xtouch	[rcx+4096-64]		;; Load prefetch TLBs
c2b:	r4_x4cl_four_complex_djbunfft rsi, 64, 2*dist128, 4*dist128, rdi, 256*4*XMM_SCD2
	bump	rdi, 4*XMM_SCD2		;; Next sine/cosine pointer
	loops	128, c2b		;; Loop 128 times
	bump	rsi, -128*64+dist128	;; Next source pointer
	loops	2, c2a			;; Loop 2 times
	bump	rsi, -2*dist128+8*dist128 ;; Next source pointer
	bump	rdi, -256*4*XMM_SCD2	;; Reset sine/cosine pointer
	bump	rcx, -128*64+8192+128	;; Next 8KB section to prefetch
	loops	4, c2a			;; Test loop counter
	bump	rsi, -4*8*dist128	;; Restore source pointer
	end_timer 13

	start_timer 14
	mov	rdi, xsincos_complex	;; Load sin/cos pointer
	loops_init 1024			;; 8 iters of 128
	r4_x4cl_four_complex_djbunfft_preload
c1b:	r4_x4cl_four_complex_djbunfft rsi, 64, 8*dist128, 16*dist128, rdi, 1024*XMM_SCD2
	bump	rdi, XMM_SCD2		;; Next set of sin/cos data
	loops	128, c1b		;; Test inner loop counter
	bump	rsi, -128*64+dist128	;; Next source pointer
	loops	8, c1b			;; Test outer loop counter
	end_timer 14

ELSE					;; Pentium 4 faster doing merged loops

	start_timer 13
	mov	rdi, xsincos_complex	;; Load sin/cos pointer
	mov	rbp, rdi
	loops_init_prefetch 1024, 64, 2, rcx;; 4 iters of 2 iters of 128
	r4_x4cl_four_complex_djbunfft_preload

c2a:	xtouch	[rcx+4096-64]		;; Load prefetch TLBs
	xtouch	[rcx+2*4096-64]

c2b:	r4_x4cl_four_complex_djbunfft rsi, 8*dist128, 2*dist128, 4*dist128, rbp, 256*4*XMM_SCD2
	loops	4, c2b			;; Test loop counter
	bump	rsi, -4*8*dist128	;; Restore source pointer
	loops_undo 4

	loops_set_prefetch 64, 1024, rcx ;; Disable prefetching in this section
c1b:	r4_x4cl_four_complex_djbunfft rsi, 2*dist128, 8*dist128, 16*dist128, rdi, 1024*XMM_SCD2
	bump	rdi, 256*XMM_SCD2	;; Next sin/cos ptr
	loops	4, c1b			;; Test loop counter

	bump	rsi, -4*2*dist128+64	;; Next source pointer
	bump	rdi, -4*256*XMM_SCD2+XMM_SCD2 ;; Next sin/cos ptr
	bump	rbp, 4*XMM_SCD2		;; Next sine/cosine pointer
	loops	64, c2b			;; Loop 64 times
	bump	rcx, -128*64+8192+128	;; Next 8KB section to prefetch
	loops	2, c2a			;; Loop 2 times
	bump	rsi, -128*64+dist128	;; Next source pointer
	loops	2, c2a			;; Loop 2 times
	end_timer 13

ENDIF

xpass2_13_levels_complex_done:
	ENDM


; *************** radix4-pass2-10240-levels-real macro ******************
; This macro takes 20480 real values and performs the final 14.322 levels of
; the FFT.  Independently, this macro also takes 10240 complex values and
; performs the final 13.322 levels of the FFT.  The results are squared, and
; the first 13.322/14.322 levels of the inverse FFT are done.

r4_pass2_10240_levels_real MACRO
	LOCAL	b1b, b2a, b2b, b3a, b3b, b4b, b5a, b5b, b6b, b7a, b7b
	LOCAL	b8b, b9a, b9b, baa, bab0, bab, bac, bad0, bad
	LOCAL	c1b, c2a, c2b, c3a, c3b, c4b, c5a, c5b, c6b, c7a, c7b, c8b, c9a, c9b
	LOCAL	xpass2_10240_levels_real_1, xpass2_10240_levels_real_2
	LOCAL	xpass2_10240_levels_real_3, xpass2_10240_levels_real_4
	LOCAL	xpass2_10240_real_unfft, xpass2_10240_real_done

;; We switch to the same format used in one-pass FFTs (that is, the
;; high word of xmm registers being 1 greater than the low word):

;; Type 4 FFTs skip the forward FFT process

	start_timer 2
	mov	rsi, DATA_ADDR		;; Load source address
	mov	rbx, DIST_TO_FFTSRCARG
	cmp	ffttype, 4
	je	xpass2_10240_levels_real_4

;; Do FFT level 1,2 (actually 2.322 levels)
;;
;; On input the 64-byte cache lines hold these data values if pass 2 swizzles:
;;	0	+5K	10240	+5K	20480	+5K	30720	+5K
;;	1	...
;;	...
;;	5119
;; On input the 64-byte cache lines hold these data values if pass 1 swizzles:
;;	0	+20K	5120	+20K	10240	+20K	15360	+20K
;;	1	...
;;	...
;;	5119	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+20K	1024	+20K	2048	+20K	3072	+20K
;;	1	...
;;	...
;;	1023	...
;;	4096	...
;;	...

;; Do 2048 ten_reals_fft operations
;; Do 2048 five_complex_djbfft operations
;;	distance between fft data elements is 2K
;;	do 1024 iterations

	mov	rdi, xsincos_complex	;; Ptr to the sin/cos data
	mov	rdx, sincos3		;; Ptr to real sin/cos data
	loops_init 1024
b1b:	r5_fh5cl_ten_reals_five_complex_djbfft rsi, 64, 8*dist128, rdi, 1024*XMM_SCD2, rdx, 1024*XMM_HSCD2
	bump	rdi, XMM_SCD2		;; Next sin/cos data
	bump	rdx, XMM_HSCD2		;; Next sin/cos data
	loops	128, b1b		;; Test loop counter
	bump	rsi, -128*64+dist128	;; Next source pointer
	loops	8, b1b			;; Test loop counter
	bump	rsi, -8*dist128		;; Restore source pointer
	bump	rdi, -1024*XMM_SCD2	;; Restore sin/cos ptr
	bump	rdx, 1024*XMM_HSCD2	;; Next sin/cos ptr

;; Do FFT levels 3,4
;;
;; On input the 64-byte cache lines hold these data values:
;;	0	+20K	1024	+20K	2048	+20K	3072	+20K
;;	1	...
;;	...
;;	1023	...
;;	4096	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+20K	512	+20K	1024	+20K	1536	+20K
;;	1	...
;;	...
;;	511	...
;;	2048	...
;;	...

;; Do 512 eight_reals_fft operations
;; Do 512 four_complex_djbfft operations
;;	distance between fft data elements is 512
;;	do 512 macros each processing 16 data values

	mov	rcx, DATA_PREFETCH	;; Prefetch pointer
	xtouch	[rcx]			;; Preload first TLB
	loops_init_prefetch 2560, 64, 4, rcx
b2a:	xtouch	[rcx+4096-64]		;; Preload the TLBs
b2b:	r4_h2cl_eight_reals_four_complex_djbfft rsi, 64, 4*dist128, rdi, rdx
	bump	rdi, 5*XMM_SCD2		;; Next sin/cos data
	bump	rdx, XMM_HSCD2		;; Next sin/cos data
	loops	128, b2b		;; Test inner loop counter
	bump	rsi, -128*64+dist128	;; Next source pointer
	loops_touch 2, b2b, 2, b2a	;; Test loop counter
	bump	rsi, -4*dist128+8*dist128 ;; Next source pointer
	bump	rdi, -512*5*XMM_SCD2	;; Restore sin/cos ptr
	bump	rdx, -512*XMM_HSCD2	;; Restore sin/cos ptr

;; Do 4096 four_complex_fft operations
;;	distance between fft data elements is 512
;;	do 2048 macros each processing 16 data values

	loops_reset
	r4_x2cl_four_complex_djbfft_preload
b3a:	xtouch	[rcx+4096-64]		;; Preload the TLBs
b3b:	r4_x2cl_four_complex_djbfft rsi, 64, 4*dist128, rdi
	bump	rdi, 5*XMM_SCD2		;; Next sin/cos data
	loops	128, b3b		;; Test inner loop counter
	bump	rsi, -128*64+dist128	;; Next source pointer
	loops_touch 2, b3b, 2, b3a	;; Test loop counter
	bump	rsi, -4*dist128+8*dist128 ;; Next source pointer
	bump	rdi, -512*5*XMM_SCD2	;; Restore sin/cos ptr
	loops	5, b3a			;; Test loop counter
	bump	rsi, -5*8*dist128	;; Restore source pointer

;; Do FFT levels 5,6
;;
;; On input the 64-byte cache lines hold these data values:
;;	0	+20K	512	+20K	1024	+20K	1536	+20K
;;	1	...
;;	...
;;	511	...
;;	2048	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+20K	128	+20K	256	+20K	384	+20K
;;	1	...
;;	...
;;	127	...
;;	512	...
;;	...

;; Do 128 eight_reals_fft operations
;; Do 384 four_complex_djbfft operations
;;	distance between fft data elements is 128
;;	do 128 macros each processing 32 data values

	bump	rdi, 2560*XMM_SCD2	;; Load sin/cos pointer
	loops_init_prefetch 1280, 64, 2, rcx
	xtouch	[rcx+4096-64]		;; Preload the TLBs
	r4_h4cl_eight_reals_four_complex_djbfft_preload
b4b:	r4_h4cl_eight_reals_four_complex_djbfft rsi, 64, dist128, 2*dist128, rdi, rdx
	bump	rdi, XMM_SCD2		;; Next sin/cos data
	bump	rdx, 4*XMM_HSCD2	;; Next sin/cos data
	loops	128, b4b		;; Test loop counter
	bump	rsi, -128*64+4*dist128	;; Next source pointer
	bump	rdi, -128*XMM_SCD2	;; Restore sin/cos ptr
	bump	rdx, -128*4*XMM_HSCD2	;; Restore sin/cos ptr

;; Do 4608 four_complex_djbfft operations
;;	distance between fft data elements is 128
;;	do 1152 macros each processing 32 data values

	loops_reset			;; 9 more iterations of 128
	r4_x4cl_four_complex_djbfft_preload
b5a:	xtouch	[rcx+4096-64]		;; Preload the TLBs
b5b:	r4_x4cl_four_complex_djbfft rsi, 64, dist128, 2*dist128, rdi
	bump	rdi, XMM_SCD2		;; Next sin/cos data
	loops	128, b5b		;; Test inner loop counter
	bump	rsi, -128*64+4*dist128	;; Next source pointer
	bump	rdi, -128*XMM_SCD2	;; Restore sin/cos ptr
	loops	10, b5a			;; Test loop counter
	bump	rsi, -10*4*dist128	;; Next source pointer

;; Do FFT levels 7,8
;;
;; On input the 64-byte cache lines hold these data values:
;;	0	+20K	128	+20K	256	+20K	384	+20K
;;	1	...
;;	...
;;	127	...
;;	512	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+20K	32	+20K	64	+20K	96	+20K
;;	1	...
;;	...
;;	31	...
;;	128	...
;;	...

;; Do 32 eight_reals_fft operations
;; Do 96 four_complex_djbfft operations
;;	distance between fft data elements is 32
;;	do 32 macros each processing 32 data values

	loops_init_prefetch 1280, 64, 2, rcx
	xtouch	[rcx+4096-64]		;; Preload the TLBs
	r4_h4cl_eight_reals_four_complex_djbfft_preload
b6b:	r4_h4cl_eight_reals_four_complex_djbfft rsi, 64, 32*64, 64*64, rdi, rdx
	bump	rdi, 4*XMM_SCD2		;; Next sin/cos data
	bump	rdx, 16*XMM_HSCD2	;; Next sin/cos data
	loops	32, b6b			;; Test loop counter
	bump	rsi, -32*64+dist128	;; Next source pointer
	bump	rdi, -32*4*XMM_SCD2	;; Restore sin/cos ptr
	bump	rdx, -32*16*XMM_HSCD2	;; Restore sin/cos ptr

;; Do 2944 four_complex_djbfft operations
;;	distance between fft data elements is 32
;;	do 736 macros each processing 32 data values

	loops_reset			;; 40 iters of 32
	r4_x4cl_four_complex_djbfft_preload
b7a:	xtouch	[rcx+4096-64]		;; Preload the TLBs
b7b:	r4_x4cl_four_complex_djbfft rsi, 64, 32*64, 64*64, rdi
	bump	rdi, 4*XMM_SCD2		;; Next sin/cos data
	loops	32, b7b			;; Test inner loop counter
	bump	rsi, -32*64+dist128	;; Next source pointer
	bump	rdi, -32*4*XMM_SCD2	;; Restore sin/cos ptr
	loops_touch 4, b7b, 10, b7a	;; 40 iterations
	bump	rsi, -40*dist128	;; Next source pointer

;; Do FFT levels 9,10
;;
;; On input the 64-byte cache lines hold these data values:
;;	0	+20K	32	+20K	64	+20K	96	+20K
;;	1	...
;;	...
;;	31	...
;;	128	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+20K	8	+20K	16	+20K	24	+20K
;;	1	...
;;	...
;;	7	...
;;	32	...
;;	...

;; Do 8 eight_reals_fft operations
;; Do 24 four_complex_djbfft operations
;;	distance between fft data elements is 8
;;	do 8 macros each processing 32 data values

	loops_init_prefetch 1280, 64, 2, rcx
	xtouch	[rcx+4096-64]		;; Preload the TLBs
	r4_h4cl_eight_reals_four_complex_djbfft_preload
b8b:	r4_h4cl_eight_reals_four_complex_djbfft rsi, 64, 8*64, 16*64, rdi, rdx
	bump	rdi, 16*XMM_SCD2	;; Next sin/cos data
	bump	rdx, 64*XMM_HSCD2	;; Next sin/cos data
	loops	8, b8b			;; Test loop counter
	bump	rsi, -8*64+32*64	;; Next source pointer
	bump	rdi, -8*16*XMM_SCD2	;; Restore sin/cos ptr
	bump	rdx, -8*64*XMM_HSCD2	;; Restore sin/cos ptr

;; Do 5088 four_complex_djbfft operations
;;	distance between fft data elements is 8
;;	do 1272 macros each processing 32 data values

	loops_reset			;; 40 iters of 3 or 4 of 8
	r4_x4cl_four_complex_djbfft_preload
b9a:	xtouch	[rcx+4096-64]		;; Preload the TLBs
b9b:	r4_x4cl_four_complex_djbfft rsi, 64, 8*64, 16*64, rdi
	bump	rdi, 16*XMM_SCD2	;; Next sin/cos data
	loops	8, b9b			;; Test loop counter
	bump	rsi, -8*64+32*64	;; Next source pointer
	bump	rdi, -8*16*XMM_SCD2	;; Restore sin/cos ptr
	loops	4, b9b			;; Test middle loop counter
	bump	rsi, -4*32*64+dist128	;; Next source pointer
	loops_touch 4, b9b, 10, b9a	;; 40 iterations
	bump	rsi, -40*dist128	;; Next source pointer

;; Do FFT levels 11,12,13
;;
;; On input the 64-byte cache lines hold these data values:
;;	0	+20K	8	+20K	16	+20K	24	+20K
;;	1	...
;;	...
;;	7	...
;;	32	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+20K	4	+20K	8	+20K	12	+20K
;;	1	...
;;	...
;;	3	...
;;	16	...
;;	...

;; Do 1 sixteen_reals_fft operations
;; Do 3 eight_complex_fft operations
;;	distance between fft data elements is 1

;; Do 2556 eight_complex_fft operations
;;	distance between fft data elements is 1
;;	do 639 macros each processing 64 data values

;; Execute the proper middle step

	cmp	ffttype, 2
	jl	xpass2_10240_levels_real_1
	jg	xpass2_10240_levels_real_3

xpass2_10240_levels_real_2:
	r8_h8cl_sixteen_reals_eight_complex_with_square rsi, 8*64, 64, 2*64, 4*64
	loops_init_prefetch 640, 64, 8, rcx, 1, 16
	r8_x8cl_eight_complex_with_square_preload
	xtouch	[rcx+4096-64]		;; Preload the TLBs
	xtouch	[rcx+2*4096-64]
baa:	r8_x8cl_eight_complex_with_square rsi, 8*64, 64, 2*64, 4*64
	loops	16, baa			;; Test loop counter
	bump	rsi, -16*8*64+dist128	;; Next source pointer
	loops	40, baa			;; Test loop counter
	bump	rsi, -40*dist128	;; Next source pointer
	jmp	xpass2_10240_real_unfft

xpass2_10240_levels_real_1:
	r8_h8cl_sixteen_reals_eight_complex_fft_final rsi, 8*64, 64, 2*64, 4*64
	loops_init_prefetch 640, 128, 1, rcx, 1, 16
	r8_x8cl_eight_complex_fft_final_preload
bab0:	xtouch	[rcx+4096-64]		;; Preload the TLBs
bab:	r8_x8cl_eight_complex_fft_final rsi, 8*64, 64, 2*64, 4*64
	loops	16, bab			;; Test loop counter
	bump	rsi, -16*8*64+dist128	;; Next source pointer
	loops_touch 2, bab, 20, bab0	;; Test loop counter
	jmp	xpass2_10240_real_done

xpass2_10240_levels_real_3:
	mov	rbp, DIST_TO_MULSRCARG
	r8_h8cl_sixteen_reals_eight_complex_with_mult rsi, 8*64, 64, 2*64, 4*64
	loops_init_prefetch 640, 64, 8, rcx, 1, 16
	r8_x8cl_eight_complex_with_mult_preload
	xtouch	[rcx+4096-64]		;; Preload the TLBs
	xtouch	[rcx+2*4096-64]
bac:	r8_x8cl_eight_complex_with_mult rsi, 8*64, 64, 2*64, 4*64
	loops	16, bac			;; Test loop counter
	bump	rsi, -16*8*64+dist128	;; Next source pointer
	loops	40, bac			;; Test loop counter
	bump	rsi, -40*dist128	;; Next source pointer
	jmp	xpass2_10240_real_unfft

xpass2_10240_levels_real_4:
	mov	rbp, DIST_TO_MULSRCARG
	mov	rcx, DATA_PREFETCH	;; Load prefetch pointer
	xtouch	[rcx]			;; Preload first TLB
	r8_h8cl_sixteen_reals_eight_complex_with_mulf rsi, 8*64, 64, 2*64, 4*64
	loops_init_prefetch 640, 128, 1, rcx, 1, 16
	r8_x8cl_eight_complex_with_mulf_preload
bad0:	xtouch	[rcx+4096-64]		;; Preload the TLBs
bad:	r8_x8cl_eight_complex_with_mulf rsi, 8*64, 64, 2*64, 4*64
	loops	16, bad			;; Test loop counter
	bump	rsi, -16*8*64+dist128	;; Next source pointer
	loops_touch 2, bad, 20, bad0	;; Test loop counter
	bump	rsi, -40*dist128	;; Next source pointer

;; Do inverse FFT levels 9,10
;; On input the 64-byte cache lines hold these data values:
;;	0	+20K	4	+20K	8	+20K	12	+20K
;;	1	...
;;	...
;;	3	...
;;	16	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+20K	16	+20K	32	+20K	48	+20K
;;	1	...
;;	...
;;	15	...
;;	64	...
;;	...

;; Do 8 eight_reals_unfft operations
;; Do 8 four_complex_djbunfft operations
;;	distance between fft data elements is 8
;;	do 4 macros processing 32 data values

xpass2_10240_real_unfft:
	mov	rdi, xsincos_complex	;; Load sin/cos pointers
	bump	rdi, 2560*XMM_SCD2
	mov	rdx, sincos3
	bump	rdx, 2048*XMM_HSCD2
	loops_init_prefetch 1280, 64, 2, rcx
	xtouch	[rcx+4096-64]		;; Preload the TLBs
c8b:	r4_h4cl_eight_reals_four_complex_djbunfft rsi, 64, 4*64, 8*64, rdi, 4*16*XMM_SCD2, rdx, 4*64*XMM_HSCD2
	bump	rdi, 16*XMM_SCD2	;; Next sin/cos data
	bump	rdx, 64*XMM_HSCD2	;; Next sin/cos data
	loops	4, c8b			;; Test loop counter
	bump	rsi, -4*64+16*64	;; Next source pointer
	bump	rdi, -4*16*XMM_SCD2	;; Restore sin/cos ptr
	bump	rdx, -4*64*XMM_HSCD2	;; Restore sin/cos ptr

;; Do 5104 four_complex_djbunfft operations
;;	distance between fft data elements is 8
;;	do 1278 macros each processing 32 data values

	loops_reset			;; 40 iters of 7 or 8 of 4
	r4_x4cl_four_complex_djbunfft_preload
c9a:	xtouch	[rcx+4096-64]		;; Preload the TLBs
c9b:	r4_x4cl_four_complex_djbunfft rsi, 64, 4*64, 8*64, rdi, 4*16*XMM_SCD2
	bump	rdi, 16*XMM_SCD2	;; Next sine/cosine pointer
	loops	4, c9b			;; Test loop counter
	bump	rsi, -4*64+16*64	;; Next source pointer
	bump	rdi, -4*16*XMM_SCD2	;; Restore sin/cos ptr
	loops	8, c9b			;; Test loop counter
	bump	rsi, -8*16*64+dist128	;; Next source pointer
	loops_touch 4, c9b, 10, c9a	;; Test loop counter
	bump	rsi, -40*dist128	;; Next source pointer

;; Do inverse FFT levels 7,8
;; On input the 64-byte cache lines hold these data values:
;;	0	+20K	16	+20K	32	+20K	48	+20K
;;	1	...
;;	...
;;	15	...
;;	64	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+20K	64	+20K	128	+20K	192	+20K
;;	1	...
;;	...
;;	64	...
;;	256	...
;;	...

;; Do 32 eight_reals_unfft operations
;; Do 32 four_complex_djbunfft operations
;;	distance between fft data elements is 32
;;	do 16 macros processing 32 data values

	loops_init_prefetch 1280, 64, 2, rcx
	xtouch	[rcx+4096-64]		;; Preload the TLBs
c6b:	r4_h4cl_eight_reals_four_complex_djbunfft rsi, 64, 16*64, 32*64, rdi, 16*4*XMM_SCD2, rdx, 16*16*XMM_HSCD2
	bump	rdi, 4*XMM_SCD2		;; Next sin/cos data
	bump	rdx, 16*XMM_HSCD2	;; Next sin/cos data
	loops	16, c6b			;; Test inner loop counter
	bump	rsi, -16*64+64*64	;; Next source pointer
	bump	rdi, -16*4*XMM_SCD2	;; Restore sin/cos ptr
	bump	rdx, -16*16*XMM_HSCD2	;; Restore sin/cos ptr

;; Do 5056 four_complex_djbunfft operations
;;	distance between fft data elements is 32
;;	do 1264 macros each processing 32 data values

	loops_reset			;; 40 iters of 1 or 2 of 8
	r4_x4cl_four_complex_djbunfft_preload
c7a:	xtouch	[rcx+4096-64]		;; Preload the TLBs
c7b:	r4_x4cl_four_complex_djbunfft rsi, 64, 16*64, 32*64, rdi, 16*4*XMM_SCD2
	bump	rdi, 4*XMM_SCD2		;; Next sin/cos data
	loops	16, c7b			;; Test inner loop counter
	bump	rsi, -16*64+64*64	;; Next source pointer
	bump	rdi, -16*4*XMM_SCD2	;; Restore sin/cos ptr
	loops	2, c7b			;; Test loop counter
	bump	rsi, -2*64*64+dist128	;; Next source pointer
	loops_touch 4, c7b, 10, c7a	;; 40 iterations
	bump	rsi, -40*dist128	;; Next source pointer

;; Do inverse FFT levels 5,6
;; On input the 64-byte cache lines hold these data values:
;;	0	+20K	64	+20K	128	+20K	192	+20K
;;	1	...
;;	...
;;	64	...
;;	256	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+20K	256	+20K	512	+20K	768	+20K
;;	1	...
;;	...
;;	255	...
;;	1024	...
;;	...

;; Do 128 eight_reals_unfft operations
;; Do 128 four_complex_djbunfft operations
;;	distance between fft data elements is 128
;;	do 64 macros processing 32 data values

	loops_init_prefetch 1280, 64, 2, rcx
	xtouch	[rcx+4096-64]		;; Preload the TLBs
c4b:	r4_h4cl_eight_reals_four_complex_djbunfft rsi, 64, 64*64, dist128, rdi, 64*XMM_SCD2, rdx, 64*4*XMM_HSCD2
	bump	rdi, XMM_SCD2		;; Next sin/cos data
	bump	rdx, 4*XMM_HSCD2	;; Next sin/cos data
	loops	64, c4b			;; Test inner loop counter
	bump	rsi, -64*64+2*dist128	;; Next source pointer
	bump	rdi, -64*XMM_SCD2	;; Restore sin/cos ptr
	bump	rdx, -64*4*XMM_HSCD2	;; Restore sin/cos ptr

;; Do 2816 four_complex_djbunfft operations
;;	distance between fft data elements is 128
;;	do 704 macros each processing 32 data values

	loops_reset			;; 19 more iters of 64
	r4_x4cl_four_complex_djbunfft_preload
c5a:	xtouch	[rcx+4096-64]		;; Preload the TLBs
c5b:	r4_x4cl_four_complex_djbunfft rsi, 64, 64*64, dist128, rdi, 64*XMM_SCD2
	bump	rdi, XMM_SCD2		;; Next sin/cos data
	loops	64, c5b			;; Test inner loop counter
	bump	rsi, -64*64+2*dist128	;; Next source pointer
	bump	rdi, -64*XMM_SCD2	;; Restore sin/cos ptr
	loops_touch 2, c5b, 10, c5a	;; Test loop counter
	bump	rsi, -20*2*dist128	;; Next source pointer

;; Do inverse FFT levels 3,4
;; On input the 64-byte cache lines hold these data values:
;;	0	+20K	256	+20K	512	+20K	768	+20K
;;	1	...
;;	...
;;	255	...
;;	1024	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+20K	1024	+20K	2048	+20K	3072	+20K
;;	1	...
;;	...
;;	1023	...
;;	4096	...
;;	...

;; Do 512 eight_reals_unfft operations
;; Do 512 four_complex_djbunfft operations
;;	distance between fft data elements is 512
;;	do 256 macros processing 32 data values

	mov	rdi, xsincos_complex	;; Load sin/cos pointer
	loops_init_prefetch 1280, 64, 2, rcx
	bump	rcx, -2*64		;; Back up prefetch pointer by 2 cache lines
c2a:	xtouch	[rcx+4096-64]		;; Preload the TLBs
c2b:	r4_h4cl_eight_reals_four_complex_djbunfft rsi, 64, 2*dist128, 4*dist128, rdi, 256*5*XMM_SCD2, rdx, 256*XMM_HSCD2
	bump	rdi, 5*XMM_SCD2		;; Next sin/cos data
	bump	rdx, XMM_HSCD2		;; Next sin/cos data
	loops	128, c2b		;; Test inner loop counter
	bump	rsi, -128*64+dist128	;; Next source pointer
	loops	2, c2a			;; Test loop counter
	bump	rsi, -2*dist128+8*dist128 ;; Next source pointer
	bump	rdi, -256*5*XMM_SCD2	;; Restore sin/cos ptr
	bump	rdx, -256*XMM_HSCD2-2048*XMM_HSCD2 ;; Restore sin/cos ptr

;; Do 4096 four_complex_djbunfft operations
;;	distance between fft data elements is 512
;;	do 1024 macros each processing 32 data values

	loops_reset			;; 4 iters of 256
	r4_x4cl_four_complex_djbunfft_preload
c3a:	xtouch	[rcx+4096-64]		;; Preload the TLBs
c3b:	r4_x4cl_four_complex_djbunfft rsi, 64, 2*dist128, 4*dist128, rdi, 256*5*XMM_SCD2
	bump	rdi, 5*XMM_SCD2		;; Next sin/cos data
	loops	128, c3b		;; Test inner loop counter
	bump	rsi, -128*64+dist128	;; Next source pointer
	loops	2, c3a			;; Test loop counter
	bump	rsi, -2*dist128+8*dist128 ;; Next source pointer
	bump	rdi, -256*5*XMM_SCD2	;; Restore sin/cos ptr
	loops	5, c3a			;; Test loop counter
	bump	rsi, -5*8*dist128	;; Next source pointer

;; Do inverse FFT level 1,2 (actually 2.322 FFT levels)
;; On input the 64-byte cache lines hold these data values:
;;	0	+20K	1024	+20K	2048	+20K	3072	+20K
;;	1	...
;;	...
;;	1023	...
;;	4096	...
;;	...
;; On output the 64-byte cache lines hold these data values if pass 1 swizzles:
;;	0	+20K	5120	+20K	10240	+20K	15360	+20K
;;	1	...
;;	...
;;	5119
;; On output the 64-byte cache lines hold these data values if pass 2 swizzles:
;;	0	+5K	10240	+5K	20480	+5K	30720	+5K
;;	1	...
;;	...
;;	5119

;; Do 2048 ten_reals_unfft operations
;; Do 2048 five_complex_djbunfft operations
;;	distance between fft data elements is 2048
;;	do 1024 macros processing 40 data values

	loops_init 1024
c1b:	r5_h5cl_ten_reals_five_complex_djbunfft rsi, 64, 8*dist128, rdi, 1024*XMM_SCD2, rdx, 1024*XMM_HSCD2
	bump	rdi, XMM_SCD2		;; Next sin/cos data
	bump	rdx, XMM_HSCD2		;; Next sin/cos data
	loops	128, c1b		;; Test loop counter
	bump	rsi, -128*64+dist128	;; Next source pointer
	loops	8, c1b			;; Test loop counter

xpass2_10240_real_done:
	end_timer 2
	ENDM

; *************** radix4-pass2-10240-levels-complex macro ******************
; This macro takes 2 groups of 10240 complex values and
; performs the final 13.322 levels of a standard mostly radix-4 FFT, squares the
; results, and does the first 13.322 levels of the inverse FFT.

r4_pass2_10240_levels_complex MACRO
	LOCAL	b1a0, b1a, b1b, b2a, b2b, b3a, b3b, b4b, b5b, b6a, b6b0, b6b, b6c, b6d0, b6d
	LOCAL	c1b, c2a0, c2a, c2b, c3b, c4b, c5a, c5b
	LOCAL	xpass2_10240_levels_complex_1, xpass2_10240_levels_complex_2
	LOCAL	xpass2_10240_levels_complex_3, xpass2_10240_levels_complex_4
	LOCAL	xpass2_10240_levels_complex_unfft, xpass2_10240_levels_complex_done

;; Type 4 FFTs skip the forward FFT process

	mov	rsi, DATA_ADDR		;; Load address of FFT data
	mov	rbx, DIST_TO_FFTSRCARG
	cmp	ffttype, 4
	je	xpass2_10240_levels_complex_4

;; Do FFT levels 1,2 (actually 2.322 levels)
;;
;; We swizzle the data so that one set of data is in the high word of
;; an xmm register, the other set is in the low word.  This means
;; a distance of 20480 data values.
;;
;; On input the 64-byte cache lines hold these data values if pass 2 swizzles:
;;	0	+5K	10240	+5K	20480	+5K	30720	+5K
;;	1	...
;;	...
;;	5119
;; On input the 64-byte cache lines hold these data values if pass 1 swizzles:
;;	0	+20K	5120	+20K	10240	+20K	15360	+20K
;;	1	...
;;	...
;;	5119	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+20K	1024	+20K	2048	+20K	3072	+20K
;;	1	...
;;	...
;;	1023	...
;;	4096	...
;;	...

;; Do 4096 five_complex_fft operations
;;	distance between fft data elements is 2048
;;	do 1024 macros each processing 40 data values

;; Do FFT levels 3,4
;;
;; On input the 64-byte cache lines hold these data values:
;;	0	+20K	1024	+20K	2048	+20K	3072	+20K
;;	1	...
;;	...
;;	1023	...
;;	4096	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+20K	512	+20K	1024	+20K	1536	+20K
;;	1	...
;;	...
;;	511	...
;;	2048	...
;;	...

;; Do 5120 four_complex_fft operations
;;	distance between fft data elements is 512
;;	do 2560 macros each processing 16 data values

IF @INSTR(,%xarch,<CORE>) NE 0		;; Core 2 faster doing separate loops

	start_timer 4
	mov	rdi, xsincos_complex	;; The sin/cos data
	loops_init 1024			;; 8 iterations of 128
	r5_f5cl_five_complex_djbfft_preload
b1b:	r5_f5cl_five_complex_djbfft rsi, 64, 8*dist128, rdi, 1024*XMM_SCD2
	bump	rdi, XMM_SCD2		;; Next sin/cos data
	loops	128, b1b		;; Test inner loop counter
	bump	rsi, -128*64+dist128	;; Next source pointer
	loops	8, b1b			;; Test outer loop counter
	bump	rsi, -8*dist128		;; Restore source pointer
	end_timer 4

	start_timer 5
	mov	rdi, xsincos_complex	;; Ptr to the sin/cos data
	mov	rcx, DATA_PREFETCH	;; Prefetch pointer (we need to prefetch
					;; 5120 cache lines + 78 pad cache lines)
	xtouch	[rcx]			;; Preload first TLB
	loops_init_prefetch 2560, 64, 4, rcx ;; 5 iters of 4 iters of 128
	r4_x2cl_four_complex_djbfft_preload
b2a:	xtouch	[rcx+4096-64]		;; Preload the TLBs
b2b:	r4_x2cl_four_complex_djbfft rsi, 64, 4*dist128, rdi
	bump	rdi, 5*XMM_SCD2		;; Next sin/cos data
	loops	128, b2b		;; Test inner loop counter
	bump	rsi, -128*64+dist128	;; Next source pointer
	loops_touch 2, b2b, 2, b2a	;; Test loop counter
	bump	rsi, -4*dist128+8*dist128 ;; Next source pointer
	bump	rdi, -512*5*XMM_SCD2	;; Restore sin/cos ptr
	loops	5, b2a			;; Test loop counter
	bump	rsi, -5*8*dist128	;; Restore source pointer
	end_timer 5

ELSE					;; Pentium 4 faster doing merged loops

	start_timer 4
	mov	rdi, xsincos_complex	;; The sin/cos data
	mov	rbp, rdi
	mov	rcx, DATA_PREFETCH	;; Prefetch pointer (we need to prefetch
					;; 5120 cache lines + 78 pad cache lines)
	xtouch	[rcx]			;; Preload first TLB
	loops_init_prefetch 5120, 64, 20, rcx ;; 512 iters of 2 or 5

b1a0:	xtouch	[rcx+4096-64]		;; Preload TLB

b1a:	r5_f5cl_five_complex_djbfft_preload
b1b:	r5_f5cl_five_complex_djbfft rsi, 4*dist128, 8*dist128, rdi, 1024*XMM_SCD2
	xxprefetch [rdi+XMM_SCD2]	;; Prefetch for next pass
	xxprefetch [rdi+1025*XMM_SCD2]	;; Prefetch for next pass
	bump	rdi, 512*XMM_SCD2	;; Next sin/cos ptr
	loops	2, b1b, 5		;; Test loop counter with non-standard incr
	bump	rsi, -2*4*dist128	;; Restore source pointer
	loops_undo 10

	r4_x2cl_four_complex_djbfft_preload
b2b:	r4_x2cl_four_complex_djbfft rsi, 8*dist128, 4*dist128, rbp
	loops	5, b2b, 2		;; Test loop counter with non-standard incr

	bump	rsi, -5*8*dist128+64	;; Next source pointer
	bump	rdi, -2*512*XMM_SCD2+XMM_SCD2 ;; Next sin/cos ptr
	bump	rbp, 5*XMM_SCD2		;; Next sin/cos data
	loops	128, b1a		;; Test loop counter
	bump	rsi, -128*64+dist128	;; Next source pointer
	loops	4, b1a0			;; Test loop counter
	bump	rsi, -4*dist128		;; Restore source pointer
	bump	rdi, -512*XMM_SCD2	;; Restore sin/cos ptr
	end_timer 4

ENDIF

;; Do FFT levels 5,6
;;
;; On input the 64-byte cache lines hold these data values:
;;	0	+20K	512	+20K	1024	+20K	1536	+20K
;;	1	...
;;	...
;;	511	...
;;	2048	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+20K	128	+20K	256	+20K	384	+20K
;;	1	...
;;	...
;;	127	...
;;	512	...
;;	...
;;
;; Do 5120 four_complex_fft operations
;;	distance between fft data elements is 128
;;	do 1280 macros each processing 32 data values

;; Do FFT levels 7,8
;;
;; On input the 64-byte cache lines hold these data values:
;;	0	+20K	128	+20K	256	+20K	384	+20K
;;	1	...
;;	...
;;	127	...
;;	512	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+20K	32	+20K	64	+20K	96	+20K
;;	1	...
;;	...
;;	31	...
;;	128	...
;;	...
;;
;; Do 5120 four_complex_fft operations
;;	distance between fft data elements is 32
;;	do 1280 macros each processing 32 data values

;; Do FFT levels 9,10
;;
;; On input the 64-byte cache lines hold these data values:
;;	0	+20K	32	+20K	64	+20K	96	+20K
;;	1	...
;;	...
;;	31	...
;;	128	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+20K	8	+20K	16	+20K	24	+20K
;;	1	...
;;	...
;;	7	...
;;	32	...
;;	...
;;
;; Do 5120 four_complex_fft operations
;;	distance between fft data elements is 8
;;	do 1280 macros each processing 32 data values

;; Do this 64 cache lines at a time to maximize accessing data
;; while it is in the L1 cache.

	start_timer 6
	bump	rdi, 2560*XMM_SCD2	;; Load sin/cos pointers
	mov	rbx, rdi		;; Copy sin/cos ptrs
	mov	rbp, rdi
	loops_init_prefetch 1280, 64, 2, rcx ;; 10*8 iters of 64 cache lines
	r4_x4cl_four_complex_djbfft_preload

b3a:	xtouch	[rcx+4096-64]		;; Load prefetch TLBs
	xtouch	[rcx+2*4096-64]
	xtouch	[rcx+3*4096-64]

b3b:	xxprefetch [rdi+8*XMM_SCD2]	;; Prefetch for next pass
	r4_x4cl_four_complex_djbfft rsi, 8*64, dist128, 2*dist128, rdi
	bump	rdi, 8*XMM_SCD2		;; Next sine/cosine pointer
	loops	16, b3b			;; Loop 16 times
	bump	rsi, -16*8*64		;; Restore source pointer
	bump	rdi, -16*8*XMM_SCD2	;; Restore sine/cosine pointer
	loops_undo 16			;; Back up loop counter

b4b:	r4_x4cl_four_complex_djbfft rsi, 8*64, 32*64, 64*64, rbx
	bump	rbx, 4*8*XMM_SCD2	;; Next sine/cosine pointer
	loops	4, b4b			;; Loop 4 times
	bump	rsi, -4*8*64+dist128	;; Next source pointer
	bump	rbx, -4*4*8*XMM_SCD2	;; Restore sine/cosine pointer
	loops	4, b4b			;; Loop 4 times
	bump	rsi, -4*dist128		;; Restore source pointer
	loops_undo 4*4			;; Back up loop counter

b5b:	r4_x4cl_four_complex_djbfft rsi, 32*64, 8*64, 16*64, rbp
	loops	4, b5b			;; Loop 4 times
	bump	rsi, -4*32*64+dist128	;; Next source pointer
	loops	4, b5b			;; Loop 4 times

	bump	rsi, -4*dist128+64	;; Next source pointer
	bump	rdi, XMM_SCD2		;; Next sine/cosine pointer
	bump	rbx, 4*XMM_SCD2		;; Next sine/cosine pointer
	bump	rbp, 16*XMM_SCD2	;; Next sine/cosine pointer
	loops	8, b3b			;; Loop 8 times
	bump	rsi, -8*64+4*dist128	;; Next source pointer
	bump	rdi, -8*XMM_SCD2	;; Restore sine/cosine pointer
	bump	rbx, -8*4*XMM_SCD2	;; Restore sine/cosine pointer
	bump	rbp, -8*16*XMM_SCD2	;; Restore sine/cosine pointer
	loops	10, b3a			;; Loop 10 times
	bump	rsi, -10*4*dist128	;; Restore source pointer
	end_timer 6

;; Do FFT levels 11,12,13 as well as inverse FFT levels 11,12,13
;;
;; On input the 64-byte cache lines hold these data values:
;;	0	+20K	8	+20K	16	+20K	24	+20K
;;	1	...
;;	...
;;	7	...
;;	32	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+20K	4	+20K	8	+20K	12	+20K
;;	1	...
;;	...
;;	3	...
;;	16	...
;;	...

;; Do 2560 eight_complex_fft operations
;;	distance between fft data elements is 1
;;	do 640 macros each processing 64 data values

;; Execute the right middle step

	cmp	ffttype, 2
	jl	xpass2_10240_levels_complex_1
	jg	xpass2_10240_levels_complex_3

xpass2_10240_levels_complex_2:
	start_timer 9
	loops_init_prefetch 640, 64, 8, rcx
	r8_x8cl_eight_complex_with_square_preload
	xtouch	[rcx+4096-64]		;; Load prefetch TLBs
	xtouch	[rcx+2*4096-64]
b6a:	r8_x8cl_eight_complex_with_square rsi, 8*64, 64, 2*64, 4*64
	loops	16, b6a			;; Loop 16 times
	bump	rsi, -16*8*64+dist128	;; Next source pointer
	loops	40, b6a			;; Loop 40 times
	bump	rsi, -40*dist128	;; Restore source pointer
	end_timer 9
	jmp	xpass2_10240_levels_complex_unfft

xpass2_10240_levels_complex_1:
	start_timer 9
	loops_init_prefetch 640, 128, 1, rcx
	r8_x8cl_eight_complex_fft_final_preload
b6b0:	xtouch	[rcx+4096-64]		;; Load prefetch TLBs
b6b:	r8_x8cl_eight_complex_fft_final rsi, 8*64, 64, 2*64, 4*64
	loops	16, b6b			;; Loop 16 times
	bump	rsi, -16*8*64+dist128	;; Next source pointer
	loops_touch 2, b6b, 20, b6b0	;; Loop 40 times
	end_timer 9
	jmp	xpass2_10240_levels_complex_done

xpass2_10240_levels_complex_3:
	start_timer 9
	mov	rbp, DIST_TO_MULSRCARG
	loops_init_prefetch 640, 64, 8, rcx
	r8_x8cl_eight_complex_with_mult_preload
	xtouch	[rcx+4096-64]		;; Load prefetch TLBs
	xtouch	[rcx+2*4096-64]
b6c:	r8_x8cl_eight_complex_with_mult rsi, 8*64, 64, 2*64, 4*64
	loops	16, b6c			;; Loop 16 times
	bump	rsi, -16*8*64+dist128	;; Next source pointer
	loops	40, b6c			;; Loop 40 times
	bump	rsi, -40*dist128	;; Restore source pointer
	end_timer 9
	jmp	xpass2_10240_levels_complex_unfft

xpass2_10240_levels_complex_4:
	start_timer 9
	mov	rcx, DATA_PREFETCH	;; Prefetch pointer
	xtouch	[rcx]			;; Load first prefetch TLB
	mov	rbp, DIST_TO_MULSRCARG
	loops_init_prefetch 640, 128, 1, rcx
	r8_x8cl_eight_complex_with_mulf_preload
b6d0:	xtouch	[rcx+4096-64]		;; Load prefetch TLBs
b6d:	r8_x8cl_eight_complex_with_mulf rsi, 8*64, 64, 2*64, 4*64
	loops	16, b6d			;; Loop 16 times
	bump	rsi, -16*8*64+dist128	;; Next source pointer
	loops_touch 2, b6d, 20, b6d0	;; Loop 40 times
	bump	rsi, -40*dist128	;; Restore source pointer
	end_timer 9

;; Do inverse FFT levels 9,10
;; On input the 64-byte cache lines hold these data values:
;;	0	+20K	4	+20K	8	+20K	12	+20K
;;	1	...
;;	...
;;	3	...
;;	16	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+20K	16	+20K	32	+20K	48	+20K
;;	1	...
;;	...
;;	15	...
;;	64	...
;;	...
;;
;; Do 5120 four_complex_unfft operations
;;	distance between fft data elements is 8
;;	do 1280 macros each processing 32 data values

;; Do inverse FFT levels 7,8
;; On input the 64-byte cache lines hold these data values:
;;	0	+20K	16	+20K	32	+20K	48	+20K
;;	1	...
;;	...
;;	15	...
;;	64	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+20K	64	+20K	128	+20K	192	+20K
;;	1	...
;;	...
;;	64	...
;;	256	...
;;	...
;;
;; Do 5120 four_complex_unfft operations
;;	distance between fft data elements is 32
;;	do 1280 macros each processing 32 data values

;; Do inverse FFT levels 5,6
;; On input the 64-byte cache lines hold these data values:
;;	0	+20K	64	+20K	128	+20K	192	+20K
;;	1	...
;;	...
;;	64	...
;;	256	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+20K	256	+20K	512	+20K	768	+20K
;;	1	...
;;	...
;;	255	...
;;	1024	...
;;	...
;;
;; Do 5120 four_complex_unfft operations
;;	distance between fft data elements is 128
;;	do 1280 macros each processing 32 data values

;; Do this 64 cache lines at a time to maximize accessing data
;; while it is in the L1 cache.

xpass2_10240_levels_complex_unfft:
	start_timer 10
	mov	rdi, xsincos_complex	;; Load sin/cos pointers
	bump	rdi, 2560*XMM_SCD2
	mov	rbx, rdi
	mov	rbp, rdi
	loops_init_prefetch 1280, 64, 2, rcx ;; 20*4 iters of 64 cache lines
	r4_x4cl_four_complex_djbunfft_preload

c5a:	xtouch	[rcx+4096-64]		;; Load prefetch TLBs
	xtouch	[rcx+2*4096-64]
	xtouch	[rcx+3*4096-64]

c5b:	r4_x4cl_four_complex_djbunfft rsi, 16*64, 4*64, 8*64, rbp, 4*16*XMM_SCD2
	loops	8, c5b			;; Loop 8 times
	bump	rsi, -8*16*64+dist128	;; Next source pointer
	loops	2, c5b			;; Loop 2 times
	bump	rsi, -2*dist128		;; Restore source pointer
	loops_undo 2*8			;; Restore loop counter

c4b:	r4_x4cl_four_complex_djbunfft rsi, 4*64, 16*64, 32*64, rbx, 16*4*XMM_SCD2
	bump	rbx, 4*4*XMM_SCD2	;; Next sine/cosine pointer
	loops	4, c4b			;; Loop 4 times
	bump	rsi, -4*4*64+64*64	;; Next source pointer
	bump	rbx, -4*4*4*XMM_SCD2	;; Restore sine/cosine pointer
	loops	2, c4b			;; Loop twice
	bump	rsi, -2*64*64+dist128	;; Next source pointer
	loops	2, c4b			;; Loop twice
	bump	rsi, -2*dist128		;; Restore source pointer
	loops_undo 2*2*4		;; Restore loop counter

c3b:	r4_x4cl_four_complex_djbunfft rsi, 4*64, 64*64, dist128, rdi, 64*1*XMM_SCD2
	bump	rdi, 4*1*XMM_SCD2	;; Next sine/cosine pointer
	loops	16, c3b			;; Loop 16 times
	bump	rdi, -16*4*1*XMM_SCD2	;; Restore sine/cosine pointer

	bump	rsi, -16*4*64+64	;; Next source pointer
	bump	rdi, 1*XMM_SCD2		;; Next sine/cosine pointer
	bump	rbx, 4*XMM_SCD2		;; Next sine/cosine pointer
	bump	rbp, 16*XMM_SCD2	;; Next sine/cosine pointer
	loops	4, c5b			;; Loop 4 times
	bump	rsi, -4*64+2*dist128	;; Next source pointer
	bump	rdi, -4*1*XMM_SCD2	;; Restore sine/cosine pointer
	bump	rbx, -4*4*XMM_SCD2	;; Restore sine/cosine pointer
	bump	rbp, -4*16*XMM_SCD2	;; Restore sine/cosine pointer
	loops_touch 2, c5b, 10, c5a	;; Loop 20 times
	bump	rsi, -20*2*dist128	;; Restore source pointer
	end_timer 10

;; Do inverse FFT levels 3,4
;; On input the 64-byte cache lines hold these data values:
;;	0	+20K	256	+20K	512	+20K	768	+20K
;;	1	...
;;	...
;;	255	...
;;	1024	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+20K	1024	+20K	2048	+20K	3072	+20K
;;	1	...
;;	...
;;	1023	...
;;	4096	...
;;	...

;; Do 5120 four_complex_unfft operations
;;	distance between fft data elements is 512
;;	do 1280 macros each processing 32 data values

;; Do inverse FFT levels 1,2 (actually 2.322 FFT levels)
;; On input the 64-byte cache lines hold these data values:
;;	0	+20K	1024	+20K	2048	+20K	3072	+20K
;;	1	...
;;	...
;;	1023	...
;;	4096	...
;;	...
;; On output the 64-byte cache lines hold these data values if pass 1 swizzles:
;;	0	+20K	5120	+20K	10240	+20K	15360	+20K
;;	1	...
;;	...
;;	5119
;; On output the 64-byte cache lines hold these data values if pass 2 swizzles:
;;	0	+5K	10240	+5K	20480	+5K	30720	+5K
;;	1	...
;;	...
;;	5119

;; Do 4096 five_complex_unfft operations
;;	distance between fft data elements is 2048
;;	do 1024 macros each processing 40 data values

IF @INSTR(,%xarch,<CORE>) NE 0		;; Core 2 faster doing separate loops

	start_timer 13
	mov	rdi, xsincos_complex	;; Load sin/cos pointer
	loops_init_prefetch 1280, 64, 2, rcx;; 5 iters of 256
	bump	rcx, -2*64		;; Back up prefetch pointer by 2 cache lines
	r4_x4cl_four_complex_djbunfft_preload
c2a:	xtouch	[rcx+4096-64]		;; Load prefetch TLBs
c2b:	r4_x4cl_four_complex_djbunfft rsi, 64, 2*dist128, 4*dist128, rdi, 256*5*XMM_SCD2
	bump	rdi, 5*XMM_SCD2		;; Next sine/cosine pointer
	loops	128, c2b		;; Loop 128 times
	bump	rsi, -128*64+dist128	;; Next source pointer
	loops	2, c2a			;; Loop twice
	bump	rsi, -2*dist128+8*dist128 ;; Next source pointer
	bump	rdi, -256*5*XMM_SCD2	;; Reset sine/cosine pointer
	loops	5, c2a			;; Test loop counter
	bump	rsi, -5*8*dist128	;; Restore source pointer
	end_timer 13

	start_timer 14
	mov	rdi, xsincos_complex	;; Load group multiplier pointer
	loops_init 1024			;; 8 iters of 128
	r5_x5cl_five_complex_djbunfft_preload
c1b:	r5_x5cl_five_complex_djbunfft rsi, 64, 8*dist128, rdi, 1024*XMM_SCD2
	bump	rdi, XMM_SCD2		;; Next set of sin/cos data
	loops	128, c1b		;; Test inner loop counter
	bump	rsi, -128*64+dist128	;; Next source pointer
	loops	8, c1b			;; Test outer loop counter
	end_timer 14

ELSE					;; Pentium 4 faster doing merged loops

	start_timer 13
	mov	rdi, xsincos_complex	;; Load sin/cos pointer
	mov	rbp, rdi
	bump	rcx, -2*64		;; Back up prefetch pointer by 2 cache lines
	loops_init_prefetch 5120, 64, 5, rcx ;; 256 iters of 5 or 4

c2a0:	xtouch	[rcx+4096-64]		;; Load prefetch TLBs

c2a:	r4_x4cl_four_complex_djbunfft_preload
c2b:	r4_x4cl_four_complex_djbunfft rsi, 8*dist128, 2*dist128, 4*dist128, rbp, 256*5*XMM_SCD2
	loops_no_prefetch 5, c2b, 4	;; Test loop counter with non-standard incr
	bump	rsi, -5*8*dist128	;; Restore source pointer
	loops_undo 20

	r5_x5cl_five_complex_djbunfft_preload
c1b:	r5_x5cl_five_complex_djbunfft rsi, 2*dist128, 8*dist128, rdi, 1024*XMM_SCD2
	bump	rdi, 256*XMM_SCD2	;; Next sin/cos ptr
	loops	4, c1b, 5		;; Test loop counter with non-standard incr

	bump	rsi, -4*2*dist128+64	;; Next source pointer
	bump	rdi, -4*256*XMM_SCD2+XMM_SCD2 ;; Next sin/cos ptr
	bump	rbp, 5*XMM_SCD2		;; Next sine/cosine pointer
	loops_touch 16, c2a, 8, c2a0	;; Loop 128 times
	bump	rsi, -128*64+dist128	;; Next source pointer
	loops	2, c2a0			;; Loop twice
	end_timer 13

ENDIF

xpass2_10240_levels_complex_done:
	ENDM


; *************** radix4-pass2-12800-levels-real macro ******************
; This macro takes 25600 real values and performs the final 14.644 levels of
; the FFT.  Independently, this macro also takes 12800 complex values and
; performs the final 13.644 levels of the FFT.  The results are squared, and
; the first 13.644/14.644 levels of the inverse FFT are done.

r4_pass2_12800_levels_real MACRO
	LOCAL	b1b, b2a, b2b, b3a, b3b, b4b, b5a, b5b, b6b, b7a, b7b
	LOCAL	b8b, b9a, b9b, baa, bab0, bab, bac, bad0, bad
	LOCAL	c1a, c1b, c2a, c2b, c3a, c3b, c4b, c5a, c5b, c6b, c7a, c7b, c8b, c9a, c9b
	LOCAL	xpass2_12800_levels_real_1, xpass2_12800_levels_real_2
	LOCAL	xpass2_12800_levels_real_3, xpass2_12800_levels_real_4
	LOCAL	xpass2_12800_real_unfft, xpass2_12800_real_done

;; We switch to the same format used in one-pass FFTs (that is, the
;; high word of xmm registers being 1 greater than the low word):

;; Type 4 FFTs skip the forward FFT process

	start_timer 2
	mov	rsi, DATA_ADDR		;; Load source address
	mov	rbx, DIST_TO_FFTSRCARG
	cmp	ffttype, 4
	je	xpass2_12800_levels_real_4

;; Do FFT level 1,2 (actually 2.322 levels)
;;
;; On input the 64-byte cache lines hold these data values if pass 2 swizzles:
;;	0	+6.4K	12800	+6.4K	25600	+6.4K	38400	+6.4K
;;	1	...
;;	...
;;	6399
;; On input the 64-byte cache lines hold these data values if pass 1 swizzles:
;;	0	+25K	6400	+25K	12800	+25K	19200	+25K
;;	1	...
;;	...
;;	6399	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+25K	1280	+25K	2560	+25K	3840	+25K
;;	1	...
;;	...
;;	1279	...
;;	5120	...
;;	...

;; Do 2560 ten_reals_fft operations
;; Do 2560 five_complex_djbfft operations
;;	distance between fft data elements is 2560
;;	do 1280 iterations

	mov	rdi, xsincos_complex	;; Ptr to the sin/cos data
	mov	rdx, sincos3		;; Ptr to real sin/cos data
	loops_init 1280
b1b:	r5_fh5cl_ten_reals_five_complex_djbfft rsi, 64, 10*dist128, rdi, 1280*XMM_SCD2, rdx, 1280*XMM_HSCD2
	bump	rdi, XMM_SCD2		;; Next sin/cos data
	bump	rdx, XMM_HSCD2		;; Next sin/cos data
	loops	128, b1b		;; Test loop counter
	bump	rsi, -128*64+dist128	;; Next source pointer
	loops	10, b1b			;; Test loop counter
	bump	rsi, -10*dist128	;; Restore source pointer
	bump	rdi, -1280*XMM_SCD2	;; Restore sin/cos ptr
	bump	rdx, -1280*XMM_HSCD2	;; Restore sin/cos ptr

;; Do FFT levels 3,4 (actually 2.322 levels)
;;
;; On input the 64-byte cache lines hold these data values:
;;	0	+25K	1280	+25K	2560	+25K	3840	+25K
;;	1	...
;;	...
;;	1279	...
;;	5120	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+25K	256	+25K	512	+25K	768	+25K
;;	1	...
;;	...
;;	255	...
;;	1024	...
;;	...

;; Do 512 ten_reals_fft operations
;; Do 512 five_complex_djbfft operations
;;	distance between fft data elements is 512
;;	do 256 macros each processing 40 data values

	mov	rcx, DATA_PREFETCH	;; Prefetch pointer
	xtouch	[rcx]			;; Preload first TLB
	loops_init_prefetch 1280, 64, 2, rcx
b2a:	xtouch	[rcx+4096-64]		;; Preload the TLBs
b2b:	r5_h5cl_ten_reals_five_complex_djbfft rsi, 64, 2*dist128, rdi, 256*5*XMM_SCD2, rdx, 256*5*XMM_HSCD2
	bump	rdi, 5*XMM_SCD2		;; Next sin/cos data
	bump	rdx, 5*XMM_HSCD2	;; Next sin/cos data
	loops	128, b2b		;; Test inner loop counter
	bump	rsi, -128*64+dist128	;; Next source pointer
	loops	2, b2a			;; Test loop counter
	bump	rsi, -2*dist128+10*dist128 ;; Next source pointer
	bump	rdi, -256*5*XMM_SCD2	;; Restore sin/cos ptr
	bump	rdx, -256*5*XMM_HSCD2+2560*XMM_HSCD2 ;; Next sin/cos ptr

;; Do 4096 five_complex_fft operations
;;	distance between fft data elements is 512
;;	do 1024 macros each processing 40 data values

	loops_reset
	r5_nf5cl_five_complex_djbfft_preload
b3a:	xtouch	[rcx+4096-64]		;; Preload the TLBs
b3b:	r5_nf5cl_five_complex_djbfft rsi, 64, 2*dist128, rdi, 256*5*XMM_SCD2
	bump	rdi, 5*XMM_SCD2		;; Next sin/cos data
	loops	128, b3b		;; Test inner loop counter
	bump	rsi, -128*64+dist128	;; Next source pointer
	loops	2, b3a			;; Test loop counter
	bump	rsi, -2*dist128+10*dist128 ;; Next source pointer
	bump	rdi, -256*5*XMM_SCD2	;; Restore sin/cos ptr
	loops	5, b3a			;; Test loop counter
	bump	rsi, -5*10*dist128	;; Restore source pointer

;; Do FFT levels 5,6
;;
;; On input the 64-byte cache lines hold these data values:
;;	0	+25K	256	+25K	512	+25K	768	+25K
;;	1	...
;;	...
;;	255	...
;;	1024	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+25K	128	+25K	256	+25K	384	+25K
;;	1	...
;;	...
;;	127	...
;;	512	...
;;	...

;; Do 128 eight_reals_fft operations
;; Do 128 four_complex_djbfft operations
;;	distance between fft data elements is 128
;;	do 128 macros each processing 16 data values

	bump	rdi, 2560*XMM_SCD2	;; Load sin/cos pointer
	loops_init_prefetch 3200, 64, 4, rcx
	xtouch	[rcx+4096-64]		;; Preload the TLBs
b4b:	r4_h2cl_eight_reals_four_complex_djbfft rsi, 64, dist128, rdi, rdx
	bump	rdi, XMM_SCD2		;; Next sin/cos data
	bump	rdx, XMM_HSCD2		;; Next sin/cos data
	loops	128, b4b		;; Test loop counter
	bump	rsi, -128*64+2*dist128	;; Next source pointer
	bump	rdi, -128*XMM_SCD2	;; Restore sin/cos ptr
	bump	rdx, -128*XMM_HSCD2	;; Restore sin/cos ptr

;; Do 6144 four_complex_djbfft operations
;;	distance between fft data elements is 128
;;	do 3072 macros each processing 16 data values

	loops_reset			;; 24 more iterations of 128
	r4_nf2cl_four_complex_djbfft_preload
b5a:	xtouch	[rcx+4096-64]		;; Preload the TLBs
b5b:	r4_nf2cl_four_complex_djbfft rsi, 64, dist128, rdi
	bump	rdi, XMM_SCD2		;; Next sin/cos data
	loops	128, b5b		;; Test inner loop counter
	bump	rsi, -128*64+2*dist128	;; Next source pointer
	bump	rdi, -128*XMM_SCD2	;; Restore sin/cos ptr
	loops	25, b5a			;; Test loop counter
	bump	rsi, -25*2*dist128	;; Next source pointer

;; Do FFT levels 7,8
;;
;; On input the 64-byte cache lines hold these data values:
;;	0	+25K	128	+25K	256	+25K	384	+25K
;;	1	...
;;	...
;;	127	...
;;	512	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+25K	32	+25K	64	+25K	96	+25K
;;	1	...
;;	...
;;	31	...
;;	128	...
;;	...

;; Do 32 eight_reals_fft operations
;; Do 96 four_complex_djbfft operations
;;	distance between fft data elements is 32
;;	do 32 macros each processing 32 data values

	loops_init_prefetch 1600, 64, 2, rcx
	xtouch	[rcx+4096-64]		;; Preload the TLBs
	r4_h4cl_eight_reals_four_complex_djbfft_preload
b6b:	r4_h4cl_eight_reals_four_complex_djbfft rsi, 64, 32*64, 64*64, rdi, rdx
	bump	rdi, 4*XMM_SCD2		;; Next sin/cos data
	bump	rdx, 4*XMM_HSCD2	;; Next sin/cos data
	loops	32, b6b			;; Test loop counter
	bump	rsi, -32*64+dist128	;; Next source pointer
	bump	rdi, -32*4*XMM_SCD2	;; Restore sin/cos ptr
	bump	rdx, -32*4*XMM_HSCD2	;; Restore sin/cos ptr

;; Do 6272 four_complex_djbfft operations
;;	distance between fft data elements is 32
;;	do 1568 macros each processing 32 data values

	loops_reset			;; 50 iters of 32
	r4_x4cl_four_complex_djbfft_preload
b7a:	xtouch	[rcx+4096-64]		;; Preload the TLBs
b7b:	r4_x4cl_four_complex_djbfft rsi, 64, 32*64, 64*64, rdi
	bump	rdi, 4*XMM_SCD2		;; Next sin/cos data
	loops	32, b7b			;; Test inner loop counter
	bump	rsi, -32*64+dist128	;; Next source pointer
	bump	rdi, -32*4*XMM_SCD2	;; Restore sin/cos ptr
	loops_touch 2, b7b, 25, b7a	;; 50 iterations
	bump	rsi, -50*dist128	;; Next source pointer

;; Do FFT levels 9,10
;;
;; On input the 64-byte cache lines hold these data values:
;;	0	+25K	32	+25K	64	+25K	96	+25K
;;	1	...
;;	...
;;	31	...
;;	128	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+25K	8	+25K	16	+25K	24	+25K
;;	1	...
;;	...
;;	7	...
;;	32	...
;;	...

;; Do 8 eight_reals_fft operations
;; Do 24 four_complex_djbfft operations
;;	distance between fft data elements is 8
;;	do 8 macros each processing 32 data values

	loops_init_prefetch 1600, 64, 2, rcx
	xtouch	[rcx+4096-64]		;; Preload the TLBs
	r4_h4cl_eight_reals_four_complex_djbfft_preload
b8b:	r4_h4cl_eight_reals_four_complex_djbfft rsi, 64, 8*64, 16*64, rdi, rdx
	bump	rdi, 16*XMM_SCD2	;; Next sin/cos data
	bump	rdx, 16*XMM_HSCD2	;; Next sin/cos data
	loops	8, b8b			;; Test loop counter
	bump	rsi, -8*64+32*64	;; Next source pointer
	bump	rdi, -8*16*XMM_SCD2	;; Restore sin/cos ptr
	bump	rdx, -8*16*XMM_HSCD2	;; Restore sin/cos ptr

;; Do 6378 four_complex_djbfft operations
;;	distance between fft data elements is 8
;;	do 1592 macros each processing 32 data values

	loops_reset			;; 50 iters of 3 or 4 of 8
	r4_x4cl_four_complex_djbfft_preload
b9a:	xtouch	[rcx+4096-64]		;; Preload the TLBs
b9b:	r4_x4cl_four_complex_djbfft rsi, 64, 8*64, 16*64, rdi
	bump	rdi, 16*XMM_SCD2	;; Next sin/cos data
	loops	8, b9b			;; Test loop counter
	bump	rsi, -8*64+32*64	;; Next source pointer
	bump	rdi, -8*16*XMM_SCD2	;; Restore sin/cos ptr
	loops	4, b9b			;; Test middle loop counter
	bump	rsi, -4*32*64+dist128	;; Next source pointer
	loops_touch 2, b9b, 25, b9a	;; 50 iterations
	bump	rsi, -50*dist128	;; Restore source pointer

;; Do FFT levels 11,12
;;
;; On input the 64-byte cache lines hold these data values:
;;	0	+25K	8	+25K	16	+25K	24	+25K
;;	1	...
;;	...
;;	7	...
;;	32	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+25K	4	+25K	8	+25K	12	+25K
;;	1	...
;;	...
;;	3	...
;;	16	...
;;	...

;; Do 1 sixteen_reals_fft operations
;; Do 3 eight_complex_fft operations
;;	distance between fft data elements is 1

;; Do 3196 eight_complex_fft operations
;;	distance between fft data elements is 1
;;	do 799 macros each processing 64 data values

;; Execute the proper middle step

	cmp	ffttype, 2
	jl	xpass2_12800_levels_real_1
	jg	xpass2_12800_levels_real_3

xpass2_12800_levels_real_2:
	r8_h8cl_sixteen_reals_eight_complex_with_square rsi, 8*64, 64, 2*64, 4*64
	loops_init_prefetch 800, 64, 8, rcx, 1, 16
	r8_x8cl_eight_complex_with_square_preload
	xtouch	[rcx+4096-64]		;; Preload the TLBs
	xtouch	[rcx+2*4096-64]
baa:	r8_x8cl_eight_complex_with_square rsi, 8*64, 64, 2*64, 4*64
	loops	16, baa			;; Test loop counter
	bump	rsi, -16*8*64+dist128	;; Next source pointer
	loops	50, baa			;; Test loop counter
	bump	rsi, -50*dist128	;; Next source pointer
	jmp	xpass2_12800_real_unfft

xpass2_12800_levels_real_1:
	r8_h8cl_sixteen_reals_eight_complex_fft_final rsi, 8*64, 64, 2*64, 4*64
	loops_init_prefetch 800, 128, 1, rcx, 1, 16
	r8_x8cl_eight_complex_fft_final_preload
bab0:	xtouch	[rcx+4096-64]		;; Preload the TLBs
bab:	r8_x8cl_eight_complex_fft_final rsi, 8*64, 64, 2*64, 4*64
	loops	16, bab			;; Test loop counter
	bump	rsi, -16*8*64+dist128	;; Next source pointer
	loops_touch 2, bab, 25, bab0	;; Test loop counter
	jmp	xpass2_12800_real_done

xpass2_12800_levels_real_3:
	mov	rbp, DIST_TO_MULSRCARG
	r8_h8cl_sixteen_reals_eight_complex_with_mult rsi, 8*64, 64, 2*64, 4*64
	loops_init_prefetch 800, 64, 8, rcx, 1, 16
	r8_x8cl_eight_complex_with_mult_preload
	xtouch	[rcx+4096-64]		;; Preload the TLBs
	xtouch	[rcx+2*4096-64]
bac:	r8_x8cl_eight_complex_with_mult rsi, 8*64, 64, 2*64, 4*64
	loops	16, bac			;; Test loop counter
	bump	rsi, -16*8*64+dist128	;; Next source pointer
	loops	50, bac			;; Test loop counter
	bump	rsi, -50*dist128	;; Next source pointer
	jmp	xpass2_12800_real_unfft

xpass2_12800_levels_real_4:
	mov	rbp, DIST_TO_MULSRCARG
	mov	rcx, DATA_PREFETCH	;; Load prefetch pointer
	xtouch	[rcx]			;; Preload first TLB
	r8_h8cl_sixteen_reals_eight_complex_with_mulf rsi, 8*64, 64, 2*64, 4*64
	loops_init_prefetch 800, 128, 1, rcx, 1, 16
	r8_x8cl_eight_complex_with_mulf_preload
bad0:	xtouch	[rcx+4096-64]		;; Preload the TLBs
bad:	r8_x8cl_eight_complex_with_mulf rsi, 8*64, 64, 2*64, 4*64
	loops	16, bad			;; Test loop counter
	bump	rsi, -16*8*64+dist128	;; Next source pointer
	loops_touch 2, bad, 25, bad0	;; Test loop counter
	bump	rsi, -50*dist128	;; Next source pointer

;; Do inverse FFT levels 9,10
;; On input the 64-byte cache lines hold these data values:
;;	0	+25K	4	+25K	8	+25K	12	+25K
;;	1	...
;;	...
;;	3	...
;;	16	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+25K	16	+25K	32	+25K	48	+25K
;;	1	...
;;	...
;;	15	...
;;	64	...
;;	...

;; Do 8 eight_reals_unfft operations
;; Do 8 four_complex_djbunfft operations
;;	distance between fft data elements is 8
;;	do 4 macros processing 32 data values

xpass2_12800_real_unfft:
	mov	rdi, xsincos_complex	;; Load sin/cos pointers
	bump	rdi, 2560*XMM_SCD2
	mov	rdx, sincos3
	bump	rdx, 2560*XMM_HSCD2
	loops_init_prefetch 1600, 64, 2, rcx
	xtouch	[rcx+4096-64]		;; Preload the TLBs
c8b:	r4_h4cl_eight_reals_four_complex_djbunfft rsi, 64, 4*64, 8*64, rdi, 4*16*XMM_SCD2, rdx, 4*16*XMM_HSCD2
	bump	rdi, 16*XMM_SCD2	;; Next sin/cos data
	bump	rdx, 16*XMM_HSCD2	;; Next sin/cos data
	loops	4, c8b			;; Test loop counter
	bump	rsi, -4*64+16*64	;; Next source pointer
	bump	rdi, -4*16*XMM_SCD2	;; Restore sin/cos ptr
	bump	rdx, -4*16*XMM_HSCD2	;; Restore sin/cos ptr

;; Do 6384 four_complex_djbunfft operations
;;	distance between fft data elements is 8
;;	do 1596 macros each processing 32 data values

	loops_reset			;; 50 iters of 7 or 8 of 4
	r4_x4cl_four_complex_djbunfft_preload
c9a:	xtouch	[rcx+4096-64]		;; Preload the TLBs
c9b:	r4_x4cl_four_complex_djbunfft rsi, 64, 4*64, 8*64, rdi, 4*16*XMM_SCD2
	bump	rdi, 16*XMM_SCD2	;; Next sine/cosine pointer
	loops	4, c9b			;; Test loop counter
	bump	rsi, -4*64+16*64	;; Next source pointer
	bump	rdi, -4*16*XMM_SCD2	;; Restore sin/cos ptr
	loops	8, c9b			;; Test loop counter
	bump	rsi, -8*16*64+dist128	;; Next source pointer
	loops_touch 2, c9b, 25, c9a	;; Test loop counter
	bump	rsi, -50*dist128	;; Next source pointer

;; Do inverse FFT levels 7,8
;; On input the 64-byte cache lines hold these data values:
;;	0	+25K	16	+25K	32	+25K	48	+25K
;;	1	...
;;	...
;;	15	...
;;	64	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+25K	64	+25K	128	+25K	192	+25K
;;	1	...
;;	...
;;	63	...
;;	256	...
;;	...

;; Do 32 eight_reals_unfft operations
;; Do 32 four_complex_djbunfft operations
;;	distance between fft data elements is 32
;;	do 16 macros processing 32 data values

	loops_init_prefetch 1600, 64, 2, rcx
	xtouch	[rcx+4096-64]		;; Preload the TLBs
c6b:	r4_h4cl_eight_reals_four_complex_djbunfft rsi, 64, 16*64, 32*64, rdi, 16*4*XMM_SCD2, rdx, 16*4*XMM_HSCD2
	bump	rdi, 4*XMM_SCD2		;; Next sin/cos data
	bump	rdx, 4*XMM_HSCD2	;; Next sin/cos data
	loops	16, c6b			;; Test inner loop counter
	bump	rsi, -16*64+64*64	;; Next source pointer
	bump	rdi, -16*4*XMM_SCD2	;; Restore sin/cos ptr
	bump	rdx, -16*4*XMM_HSCD2	;; Restore sin/cos ptr

;; Do 6336 four_complex_djbunfft operations
;;	distance between fft data elements is 32
;;	do 1584 macros each processing 32 data values

	loops_reset			;; 50 iters of 1 or 2 of 16
	r4_x4cl_four_complex_djbunfft_preload
c7a:	xtouch	[rcx+4096-64]		;; Preload the TLBs
c7b:	r4_x4cl_four_complex_djbunfft rsi, 64, 16*64, 32*64, rdi, 16*4*XMM_SCD2
	bump	rdi, 4*XMM_SCD2		;; Next sin/cos data
	loops	16, c7b			;; Test inner loop counter
	bump	rsi, -16*64+64*64	;; Next source pointer
	bump	rdi, -16*4*XMM_SCD2	;; Restore sin/cos ptr
	loops	2, c7b			;; Test loop counter
	bump	rsi, -2*64*64+dist128	;; Next source pointer
	loops_touch 2, c7b, 25, c7a	;; 50 iterations
	bump	rsi, -50*dist128	;; Next source pointer

;; Do inverse FFT levels 5,6
;; On input the 64-byte cache lines hold these data values:
;;	0	+25K	64	+25K	128	+25K	192	+25K
;;	1	...
;;	...
;;	63	...
;;	256	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+25K	256	+25K	512	+25K	768	+25K
;;	1	...
;;	...
;;	255	...
;;	1024	...
;;	...

;; Do 128 eight_reals_unfft operations
;; Do 128 four_complex_djbunfft operations
;;	distance between fft data elements is 128
;;	do 64 macros processing 32 data values

	loops_init_prefetch 1600, 64, 2, rcx
	xtouch	[rcx+4096-64]		;; Preload the TLBs
c4b:	r4_h4cl_eight_reals_four_complex_djbunfft rsi, 64, 64*64, dist128, rdi, 64*XMM_SCD2, rdx, 64*XMM_HSCD2
	bump	rdi, XMM_SCD2		;; Next sin/cos data
	bump	rdx, XMM_HSCD2		;; Next sin/cos data
	loops	64, c4b			;; Test inner loop counter
	bump	rsi, -64*64+2*dist128	;; Next source pointer
	bump	rdi, -64*XMM_SCD2	;; Restore sin/cos ptr
	bump	rdx, -64*XMM_HSCD2-2560*XMM_HSCD2 ;; Restore sin/cos ptr

;; Do 6144 four_complex_djbunfft operations
;;	distance between fft data elements is 128
;;	do 1536 macros each processing 32 data values

	loops_reset			;; 24 more iters of 64
	r4_x4cl_four_complex_djbunfft_preload
c5a:	xtouch	[rcx+4096-64]		;; Preload the TLBs
c5b:	r4_x4cl_four_complex_djbunfft rsi, 64, 64*64, dist128, rdi, 64*XMM_SCD2
	bump	rdi, XMM_SCD2		;; Next sin/cos data
	loops	64, c5b			;; Test inner loop counter
	bump	rsi, -64*64+2*dist128	;; Next source pointer
	bump	rdi, -64*XMM_SCD2	;; Restore sin/cos ptr
	loops	25, c5a			;; Test loop counter
	bump	rsi, -25*2*dist128	;; Restore source pointer

;; Do inverse FFT levels 3,4 (actually 2.322 FFT levels)
;; On input the 64-byte cache lines hold these data values:
;;	0	+25K	256	+25K	512	+25K	768	+25K
;;	1	...
;;	...
;;	255	...
;;	1024	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+25K	1280	+25K	2560	+25K	3840	+25K
;;	1	...
;;	...
;;	1279	...
;;	5120	...
;;	...

;; Do 512 ten_reals_unfft operations
;; Do 512 five_complex_djbunfft operations
;;	distance between fft data elements is 512
;;	do 256 macros processing 40 data values

	mov	rdi, xsincos_complex	;; Load sin/cos pointer
	loops_init_prefetch 1280, 64, 2, rcx
c2a:	xtouch	[rcx+4096-64]		;; Preload the TLBs
c2b:	r5_h5cl_ten_reals_five_complex_djbunfft rsi, 64, 2*dist128, rdi, 256*5*XMM_SCD2, rdx, 256*5*XMM_HSCD2
	bump	rdi, 5*XMM_SCD2		;; Next sin/cos data
	bump	rdx, 5*XMM_HSCD2	;; Next sin/cos data
	loops	128, c2b		;; Test inner loop counter
	bump	rsi, -128*64+dist128	;; Next source pointer
	loops	2, c2a			;; Test loop counter
	bump	rsi, -2*dist128+10*dist128 ;; Next source pointer
	bump	rdi, -256*5*XMM_SCD2	;; Restore sin/cos ptr
	bump	rdx, -256*5*XMM_HSCD2	;; Restore sin/cos ptr

;; Do 4096 five_complex_djbunfft operations
;;	distance between fft data elements is 512
;;	do 1024 macros each processing 40 data values

	loops_reset			;; 4 iters of 256
	r5_x5cl_five_complex_djbunfft_preload
c3a:	xtouch	[rcx+4096-64]		;; Preload the TLBs
c3b:	r5_x5cl_five_complex_djbunfft rsi, 64, 2*dist128, rdi, 256*5*XMM_SCD2
	bump	rdi, 5*XMM_SCD2		;; Next sin/cos data
	loops	128, c3b		;; Test inner loop counter
	bump	rsi, -128*64+dist128	;; Next source pointer
	loops	2, c3a			;; Test loop counter
	bump	rsi, -2*dist128+10*dist128 ;; Next source pointer
	bump	rdi, -256*5*XMM_SCD2	;; Restore sin/cos ptr
	loops	5, c3a			;; Test loop counter
	bump	rsi, -5*10*dist128	;; Next source pointer

;; Do inverse FFT level 1,2 (actually 2.322 FFT levels)
;; On input the 64-byte cache lines hold these data values:
;;	0	+25K	1280	+25K	2560	+25K	3840	+25K
;;	1	...
;;	...
;;	1279	...
;;	5120	...
;;	...
;; On output the 64-byte cache lines hold these data values if pass 1 swizzles:
;;	0	+25K	6400	+25K	12800	+25K	19200	+25K
;;	1	...
;;	...
;;	6399
;; On output the 64-byte cache lines hold these data values if pass 2 swizzles:
;;	0	+6.4K	12800	+6.4K	25600	+6.4K	38400	+6.4K
;;	1	...
;;	...
;;	6399

;; Do 2560 ten_reals_unfft operations
;; Do 2560 five_complex_djbunfft operations
;;	distance between fft data elements is 2560
;;	do 1280 macros processing 40 data values

	bump	rcx, -2*64		;; Back up prefetch pointer by 2 cache lines
	loops_init_prefetch 1280, 64, 4, rcx
c1a:	xtouch	[rcx+4096-64]		;; Preload the TLBs
c1b:	r5_h5cl_ten_reals_five_complex_djbunfft rsi, 64, 10*dist128, rdi, 1280*XMM_SCD2, rdx, 1280*XMM_HSCD2
	bump	rdi, XMM_SCD2		;; Next sin/cos data
	bump	rdx, XMM_HSCD2		;; Next sin/cos data
	loops	128, c1b		;; Test loop counter
	bump	rsi, -128*64+dist128	;; Next source pointer
	loops_touch 2, c1b, 5, c1a	;; Test loop counter

xpass2_12800_real_done:
	end_timer 2
	ENDM

; *************** radix4-pass2-12800-levels-complex macro ******************
; This macro takes 2 groups of 12800 complex values and
; performs the final 13.644 levels of a standard mostly radix-4 FFT, squares the
; results, and does the first 13.644 levels of the inverse FFT.

r4_pass2_12800_levels_complex MACRO
	LOCAL	b1a, b1b, b2a, b2b, b3a, b3b, b4b, b5b,b6a, b6b0, b6b, b6c, b6d0, b6d
	LOCAL	c1a, c1b, c2a, c2b, c3b, c4b, c5a, c5b
	LOCAL	xpass2_12800_levels_complex_1, xpass2_12800_levels_complex_2
	LOCAL	xpass2_12800_levels_complex_3, xpass2_12800_levels_complex_4
	LOCAL	xpass2_12800_levels_complex_unfft, xpass2_12800_levels_complex_done

;; Type 4 FFTs skip the forward FFT process

	mov	rsi, DATA_ADDR		;; Load address of FFT data
	mov	rbx, DIST_TO_FFTSRCARG
	cmp	ffttype, 4
	je	xpass2_12800_levels_complex_4

;; Do FFT levels 1,2 (actually 2.322 levels)
;;
;; We swizzle the data so that one set of data is in the high word of
;; an xmm register, the other set is in the low word.  This means
;; a distance of 25600 data values.
;;
;; On input the 64-byte cache lines hold these data values if pass 2 swizzles:
;;	0	+6.4K	12800	+6.4K	25600	+6.4K	38400	+6.4K
;;	1	...
;;	...
;;	6399
;; On input the 64-byte cache lines hold these data values if pass 1 swizzles:
;;	0	+25K	6400	+25K	12800	+25K	19200	+25K
;;	1	...
;;	...
;;	6399	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+25K	1280	+25K	2560	+25K	3840	+25K
;;	1	...
;;	...
;;	1279	...
;;	5120	...
;;	...

;; Do 5120 five_complex_fft operations
;;	distance between fft data elements is 2560
;;	do 1280 macros each processing 40 data values

;; Do FFT levels 3,4 (actually 2.322 levels)
;;
;; On input the 64-byte cache lines hold these data values:
;;	0	+25K	1280	+25K	2560	+25K	3840	+25K
;;	1	...
;;	...
;;	1279	...
;;	5120	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+25K	256	+25K	512	+25K	768	+25K
;;	1	...
;;	...
;;	255	...
;;	1024	...
;;	...

;; Do 5120 five_complex_fft operations
;;	distance between fft data elements is 512
;;	do 1280 macros each processing 40 data values

IF @INSTR(,%xarch,<CORE>) NE 0		;; Core 2 faster doing separate loops

	start_timer 4
	mov	rdi, xsincos_complex	;; The sin/cos data
	loops_init 1280			;; 10 iterations of 128
	r5_f5cl_five_complex_djbfft_preload
b1b:	r5_f5cl_five_complex_djbfft rsi, 64, 10*dist128, rdi, 1280*XMM_SCD2
	bump	rdi, XMM_SCD2		;; Next sin/cos data
	loops	128, b1b		;; Test inner loop counter
	bump	rsi, -128*64+dist128	;; Next source pointer
	loops	10, b1b			;; Test outer loop counter
	bump	rsi, -10*dist128	;; Restore source pointer
	end_timer 4

	start_timer 5
	mov	rdi, xsincos_complex	;; Ptr to the sin/cos data
	mov	rcx, DATA_PREFETCH	;; Prefetch pointer (we need to prefetch
					;; 6400 cache lines + 98 pad cache lines)
	xtouch	[rcx]			;; Load first TLB
	loops_init_prefetch 1280, 64, 2, rcx ;; 5 iters of 256
	r5_nf5cl_five_complex_djbfft_preload
b2a:	xtouch	[rcx+4096-64]		;; Preload the TLBs
b2b:	r5_nf5cl_five_complex_djbfft rsi, 64, 2*dist128, rdi, 256*5*XMM_SCD2
	bump	rdi, 5*XMM_SCD2		;; Next sin/cos data
	loops	128, b2b		;; Test inner loop counter
	bump	rsi, -128*64+dist128	;; Next source pointer
	loops	2, b2a			;; Test loop counter
	bump	rsi, -2*dist128+10*dist128 ;; Next source pointer
	bump	rdi, -256*5*XMM_SCD2	;; Restore sin/cos ptr
	loops	5, b2a			;; Test loop counter
	bump	rsi, -5*10*dist128	;; Restore source pointer
	end_timer 5

ELSE					;; Pentium 4 faster doing merged loops

	start_timer 4
	mov	rdi, xsincos_complex	;; The sin/cos data
	mov	rbp, rdi
	mov	rcx, DATA_PREFETCH	;; Prefetch pointer (we need to prefetch
					;; 6400 cache lines + 98 pad cache lines)
	xtouch	[rcx]			;; Load first TLB
	loops_init_prefetch 1280, 64, 5, rcx ;; 256 iters of 5, prefetching 256 cache lines each loop

	r5_f5cl_five_complex_djbfft_preload
;	r5_nf5cl_five_complex_djbfft_preload	;; Dangerous - assume same as r5_f5cl_five_complex_djbfft_preload

b1a:	xtouch	[rcx+4096-64]		;; Preload the TLBs

b1b:	r5_f5cl_five_complex_djbfft rsi, 2*dist128, 10*dist128, rdi, 1280*XMM_SCD2
	xxprefetch [rdi+XMM_SCD2]	;; Prefetch for next pass
	xxprefetch [rdi+1281*XMM_SCD2]	;; Prefetch for next pass
	bump	rdi, 256*XMM_SCD2	;; Next sin/cos ptr
	loops	5, b1b			;; Test loop counter
	bump	rsi, -5*2*dist128	;; Restore source pointer
	loops_undo 5

b2b:	r5_nf5cl_five_complex_djbfft rsi, 10*dist128, 2*dist128, rbp, 256*5*XMM_SCD2
	loops	5, b2b			;; Test loop counter

	bump	rsi, -5*10*dist128+64	;; Next source pointer
	bump	rdi, -5*256*XMM_SCD2+XMM_SCD2 ;; Next sin/cos ptr
	bump	rbp, 5*XMM_SCD2		;; Next sin/cos data
	loops_touch 32, b1b, 4, b1a	;; Test loop counter
	bump	rsi, -128*64+dist128	;; Next source pointer
	loops	2, b1a			;; Test loop counter
	bump	rsi, -2*dist128		;; Restore source pointer
	bump	rdi, -256*XMM_SCD2	;; Restore sin/cos ptr
	end_timer 4

ENDIF

;; Do FFT levels 5,6
;;
;; On input the 64-byte cache lines hold these data values:
;;	0	+25K	256	+25K	512	+25K	768	+25K
;;	1	...
;;	...
;;	255	...
;;	1024	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+25K	128	+25K	256	+25K	384	+25K
;;	1	...
;;	...
;;	127	...
;;	512	...
;;	...
;;
;; Do 6400 four_complex_fft operations
;;	distance between fft data elements is 128
;;	do 3200 macros each processing 16 data values

;; Do FFT levels 7,8
;;
;; On input the 64-byte cache lines hold these data values:
;;	0	+25K	128	+25K	256	+25K	384	+25K
;;	1	...
;;	...
;;	127	...
;;	512	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+25K	32	+25K	64	+25K	96	+25K
;;	1	...
;;	...
;;	31	...
;;	128	...
;;	...
;;
;; Do 6400 four_complex_fft operations
;;	distance between fft data elements is 32
;;	do 1600 macros each processing 32 data values

;; Do FFT levels 9,10
;;
;; On input the 64-byte cache lines hold these data values:
;;	0	+25K	32	+25K	64	+25K	96	+25K
;;	1	...
;;	...
;;	31	...
;;	128	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+25K	8	+25K	16	+25K	24	+25K
;;	1	...
;;	...
;;	7	...
;;	32	...
;;	...
;;
;; Do 6400 four_complex_fft operations
;;	distance between fft data elements is 8
;;	do 1600 macros each processing 32 data values

;; Do this 32 cache lines at a time to maximize accessing data
;; while it is in the L1 cache.

	start_timer 6
	bump	rdi, 2560*XMM_SCD2	;; Load sin/cos pointers
	mov	rbx, rdi		;; Copy sin/cos ptrs
	mov	rbp, rdi
	loops_init_prefetch 3200, 64, 4, rcx ;; 25 iters of 8 of 32 cache lines
	;r4_nf2cl_four_complex_djbfft_preload -- assume same as r4_x4cl_four_complex_djbfft_preload
	r4_x4cl_four_complex_djbfft_preload

b3a:	xtouch	[rcx+4096-64]		;; Load prefetch TLB
	xtouch	[rcx+2*4096-64]

b3b:	xxprefetch [rdi+8*XMM_SCD2]	;; Prefetch for next pass
	r4_nf2cl_four_complex_djbfft rsi, 8*64, dist128, rdi
	bump	rdi, 8*XMM_SCD2		;; Next sine/cosine pointer
	loops	16, b3b			;; Loop 16 times
	bump	rsi, -16*8*64		;; Restore source pointer
	bump	rdi, -16*8*XMM_SCD2	;; Restore sine/cosine pointer
	loops_undo 16			;; Back up loop counter

b4b:	r4_x4cl_four_complex_djbfft rsi, 8*64, 32*64, 64*64, rbx
	bump	rbx, 8*4*XMM_SCD2	;; Next sine/cosine pointer
	loops	4, b4b, 2		;; 4 iterations with non-standard increment
	bump	rsi, -4*8*64+dist128	;; Next source pointer
	bump	rbx, -4*8*4*XMM_SCD2	;; Restore sine/cosine pointer
	loops	2, b4b			;; Loop twice
	bump	rsi, -2*dist128		;; Restore source pointer
	loops_undo 2*4*2		;; Back up loop counter

b5b:	r4_x4cl_four_complex_djbfft rsi, 32*64, 8*64, 16*64, rbp
	loops	4, b5b, 2 		;; 4 iterations with non-standard increment
	bump	rsi, -4*32*64+dist128	;; Next source pointer
	loops	2, b5b			;; 2 iterations

	bump	rsi, -2*dist128+64	;; Next source pointer
	bump	rdi, XMM_SCD2		;; Next sine/cosine pointer
	bump	rbx, 4*XMM_SCD2		;; Next sine/cosine pointer
	bump	rbp, 16*XMM_SCD2	;; Next sine/cosine pointer
	loops	8, b3b			;; Loop 4 times
	bump	rsi, -8*64+2*dist128	;; Next source pointer
	bump	rdi, -8*XMM_SCD2	;; Restore sine/cosine pointer
	bump	rbx, -8*4*XMM_SCD2	;; Restore sine/cosine pointer
	bump	rbp, -8*16*XMM_SCD2	;; Restore sine/cosine pointer
	loops	25, b3a			;; Loop 25 times
	bump	rsi, -25*2*dist128	;; Restore source pointer
	end_timer 6

;; Do FFT levels 11,12,13 as well as inverse FFT levels 11,12,13
;;
;; On input the 64-byte cache lines hold these data values:
;;	0	+25K	8	+25K	16	+25K	24	+25K
;;	1	...
;;	...
;;	7	...
;;	32	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+25K	4	+25K	8	+25K	12	+25K
;;	1	...
;;	...
;;	3	...
;;	16	...
;;	...

;; Do 3200 eight_complex_fft operations
;;	distance between fft data elements is 1
;;	do 800 macros each processing 64 data values

;; Execute the right middle step

	cmp	ffttype, 2
	jl	xpass2_12800_levels_complex_1
	jg	xpass2_12800_levels_complex_3

xpass2_12800_levels_complex_2:
	start_timer 9
	loops_init_prefetch 800, 64, 8, rcx
	r8_x8cl_eight_complex_with_square_preload
	xtouch	[rcx+4096-64]		;; Load prefetch TLBs
	xtouch	[rcx+2*4096-64]
b6a:	r8_x8cl_eight_complex_with_square rsi, 8*64, 64, 2*64, 4*64
	loops	16, b6a			;; Loop 16 times
	bump	rsi, -16*8*64+dist128	;; Next source pointer
	loops	50, b6a			;; Loop 50 times
	bump	rsi, -50*dist128	;; Restore source pointer
	end_timer 9
	jmp	xpass2_12800_levels_complex_unfft

xpass2_12800_levels_complex_1:
	start_timer 9
	loops_init_prefetch 800, 128, 1, rcx
	r8_x8cl_eight_complex_fft_final_preload
b6b0:	xtouch	[rcx+4096-64]		;; Load prefetch TLBs
b6b:	r8_x8cl_eight_complex_fft_final rsi, 8*64, 64, 2*64, 4*64
	loops	16, b6b			;; Loop 16 times
	bump	rsi, -16*8*64+dist128	;; Next source pointer
	loops_touch 2, b6b, 25, b6b0	;; Loop 50 times
	end_timer 9
	jmp	xpass2_12800_levels_complex_done

xpass2_12800_levels_complex_3:
	start_timer 9
	mov	rbp, DIST_TO_MULSRCARG
	loops_init_prefetch 800, 64, 8, rcx
	r8_x8cl_eight_complex_with_mult_preload
	xtouch	[rcx+4096-64]		;; Load prefetch TLBs
	xtouch	[rcx+2*4096-64]
b6c:	r8_x8cl_eight_complex_with_mult rsi, 8*64, 64, 2*64, 4*64
	loops	16, b6c			;; Loop 16 times
	bump	rsi, -16*8*64+dist128	;; Next source pointer
	loops	50, b6c			;; Loop 50 times
	bump	rsi, -50*dist128	;; Restore source pointer
	end_timer 9
	jmp	xpass2_12800_levels_complex_unfft

xpass2_12800_levels_complex_4:
	start_timer 9
	mov	rcx, DATA_PREFETCH	;; Prefetch pointer
	xtouch	[rcx]			;; Load first prefetch TLB
	mov	rbp, DIST_TO_MULSRCARG
	loops_init_prefetch 800, 128, 1, rcx
	r8_x8cl_eight_complex_with_mulf_preload
b6d0:	xtouch	[rcx+4096-64]		;; Load prefetch TLBs
b6d:	r8_x8cl_eight_complex_with_mulf rsi, 8*64, 64, 2*64, 4*64
	loops	16, b6d			;; Loop 16 times
	bump	rsi, -16*8*64+dist128	;; Next source pointer
	loops_touch 2, b6d, 25, b6d0	;; Loop 50 times
	bump	rsi, -50*dist128	;; Restore source pointer
	end_timer 9

;; Do inverse FFT levels 9,10
;; On input the 64-byte cache lines hold these data values:
;;	0	+25K	4	+25K	8	+25K	12	+25K
;;	1	...
;;	...
;;	3	...
;;	16	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+25K	16	+25K	32	+25K	48	+25K
;;	1	...
;;	...
;;	15	...
;;	64	...
;;	...
;;
;; Do 6400 four_complex_unfft operations
;;	distance between fft data elements is 8
;;	do 1600 macros each processing 32 data values

;; Do inverse FFT levels 7,8
;; On input the 64-byte cache lines hold these data values:
;;	0	+25K	16	+25K	32	+25K	48	+25K
;;	1	...
;;	...
;;	15	...
;;	64	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+25K	64	+25K	128	+25K	192	+25K
;;	1	...
;;	...
;;	63	...
;;	256	...
;;	...
;;
;; Do 6400 four_complex_unfft operations
;;	distance between fft data elements is 32
;;	do 1600 macros each processing 32 data values

;; Do inverse FFT levels 5,6
;; On input the 64-byte cache lines hold these data values:
;;	0	+25K	64	+25K	128	+25K	192	+25K
;;	1	...
;;	...
;;	63	...
;;	256	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+25K	256	+25K	512	+25K	768	+25K
;;	1	...
;;	...
;;	255	...
;;	1024	...
;;	...
;;
;; Do 6400 four_complex_unfft operations
;;	distance between fft data elements is 128
;;	do 1600 macros each processing 32 data values

;; Do this 64 cache lines at a time to maximize accessing data
;; while it is in the L1 cache.

xpass2_12800_levels_complex_unfft:
	start_timer 10
	mov	rdi, xsincos_complex	;; Load sin/cos pointers
	bump	rdi, 2560*XMM_SCD2
	mov	rbx, rdi
	mov	rbp, rdi
	loops_init_prefetch 1600, 64, 2, rcx ;; 25*4 iters of 64 cache lines
	r4_x4cl_four_complex_djbunfft_preload

c5a:	xtouch	[rcx+4096-64]		;; Load prefetch TLB
	xtouch	[rcx+2*4096-64]

c5b:	r4_x4cl_four_complex_djbunfft rsi, 16*64, 4*64, 8*64, rbp, 4*16*XMM_SCD2
	loops	8, c5b			;; Loop 8 times
	bump	rsi, -8*16*64+dist128	;; Next source pointer
	loops	2, c5b			;; Loop 2 times
	bump	rsi, -2*dist128		;; Restore source pointer
	loops_undo 2*8			;; Restore loop counter

c4b:	r4_x4cl_four_complex_djbunfft rsi, 4*64, 16*64, 32*64, rbx, 16*4*XMM_SCD2
	bump	rbx, 4*4*XMM_SCD2	;; Next sine/cosine pointer
	loops	4, c4b			;; Loop 4 times
	bump	rsi, -4*4*64+64*64	;; Next source pointer
	bump	rbx, -4*4*4*XMM_SCD2	;; Restore sine/cosine pointer
	loops	2, c4b			;; Loop twice
	bump	rsi, -2*64*64+dist128	;; Next source pointer
	loops	2, c4b			;; Loop twice
	bump	rsi, -2*dist128		;; Restore source pointer
	loops_undo 2*2*4		;; Restore loop counter

c3b:	r4_x4cl_four_complex_djbunfft rsi, 4*64, 64*64, dist128, rdi, 64*1*XMM_SCD2
	bump	rdi, 4*1*XMM_SCD2	;; Next sine/cosine pointer
	loops	16, c3b			;; Loop 16 times
	bump	rdi, -16*4*1*XMM_SCD2	;; Restore sine/cosine pointer

	bump	rsi, -16*4*64+64	;; Next source pointer
	bump	rdi, 1*XMM_SCD2		;; Next sine/cosine pointer
	bump	rbx, 4*XMM_SCD2		;; Next sine/cosine pointer
	bump	rbp, 16*XMM_SCD2	;; Next sine/cosine pointer
	loops	4, c5b			;; Loop 4 times
	bump	rsi, -4*64+2*dist128	;; Next source pointer
	bump	rdi, -4*1*XMM_SCD2	;; Restore sine/cosine pointer
	bump	rbx, -4*4*XMM_SCD2	;; Restore sine/cosine pointer
	bump	rbp, -4*16*XMM_SCD2	;; Restore sine/cosine pointer
	loops	25, c5a			;; Loop 25 times
	bump	rsi, -25*2*dist128	;; Restore source pointer
	end_timer 10

;; Do inverse FFT levels 3,4 (actually 2.322 levels)
;; On input the 64-byte cache lines hold these data values:
;;	0	+25K	256	+25K	512	+25K	768	+25K
;;	1	...
;;	...
;;	255	...
;;	1024	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+25K	1280	+25K	2560	+25K	3840	+25K
;;	1	...
;;	...
;;	1279	...
;;	5120	...
;;	...

;; Do 5120 five_complex_unfft operations
;;	distance between fft data elements is 512
;;	do 1280 macros each processing 40 data values

;; Do inverse FFT levels 1,2 (actually 2.322 FFT levels)
;; On input the 64-byte cache lines hold these data values:
;;	0	+25K	1280	+25K	2560	+25K	3840	+25K
;;	1	...
;;	...
;;	1279	...
;;	5120	...
;;	...
;; On output the 64-byte cache lines hold these data values if pass 1 swizzles:
;;	0	+25K	6400	+25K	12800	+25K	19200	+25K
;;	1	...
;;	...
;;	6399
;; On output the 64-byte cache lines hold these data values if pass 2 swizzles:
;;	0	+6.4K	12800	+6.4K	25600	+6.4K	38400	+6.4K
;;	1	...
;;	...
;;	6399

;; Do 5120 five_complex_unfft operations
;;	distance between fft data elements is 2560
;;	do 1280 macros each processing 40 data values

IF @INSTR(,%xarch,<CORE>) NE 0		;; Core 2 faster doing separate loops

	start_timer 13
	mov	rdi, xsincos_complex	;; Load sin/cos pointer
	loops_init_prefetch 1280, 64, 2, rcx;; 5 iters of 256
	r5_x5cl_five_complex_djbunfft_preload
c2a:	xtouch	[rcx+4096-64]		;; Load prefetch TLBs
c2b:	r5_x5cl_five_complex_djbunfft rsi, 64, 2*dist128, rdi, 256*5*XMM_SCD2
	bump	rdi, 5*XMM_SCD2		;; Next sine/cosine pointer
	loops	128, c2b		;; Loop 128 times
	bump	rsi, -128*64+dist128	;; Next source pointer
	loops	2, c2a			;; Loop twice
	bump	rsi, -2*dist128+10*dist128 ;; Next source pointer
	bump	rdi, -256*5*XMM_SCD2	;; Reset sine/cosine pointer
	loops	5, c2a			;; Test loop counter
	bump	rsi, -5*10*dist128	;; Restore source pointer
	end_timer 13

	start_timer 14
	mov	rdi, xsincos_complex	;; Load group multiplier pointer
	bump	rcx, -2*64		;; Back up prefetch pointer by 2 cache lines
	loops_init_prefetch 1280, 64, 4, rcx;; 10 iters of 128
	r5_x5cl_five_complex_djbunfft_preload
c1a:	xtouch	[rcx+4096-64]		;; Load prefetch TLBs
c1b:	r5_x5cl_five_complex_djbunfft rsi, 64, 10*dist128, rdi, 1280*XMM_SCD2
	bump	rdi, XMM_SCD2		;; Next set of sin/cos data
	loops	128, c1b		;; Test inner loop counter
	bump	rsi, -128*64+dist128	;; Next source pointer
	loops_touch 2, c1b, 5, c1a	;; Test outer loop counter
	end_timer 14

ELSE					;; Pentium 4 faster doing merged loops

	start_timer 13
	mov	rdi, xsincos_complex	;; Load sin/cos pointer
	mov	rbp, rdi
	bump	rcx, -194*64		;; Back up prefetch pointer by 194 cache lines
	loops_init_prefetch 1280, 64, 1, rcx ;; 256 iters of 5
	r5_x5cl_five_complex_djbunfft_preload

c2a:	xtouch	[rcx+4096-64]		;; Load prefetch TLBs
	xtouch	[rcx+2*4096-64]
	xtouch	[rcx+3*4096-64]
	xtouch	[rcx+4*4096-64]
	xtouch	[rcx+5*4096-64]

c2b:	r5_x5cl_five_complex_djbunfft rsi, 10*dist128, 2*dist128, rbp, 256*5*XMM_SCD2
	loops_no_prefetch 5, c2b	;; Test loop counter
	bump	rsi, -5*10*dist128	;; Restore source pointer
	loops_undo 5

c1b:	r5_x5cl_five_complex_djbunfft rsi, 2*dist128, 10*dist128, rdi, 1280*XMM_SCD2
	bump	rdi, 256*XMM_SCD2	;; Next sin/cos ptr
	loops	5, c1b			;; Test loop counter

	bump	rsi, -5*2*dist128+64	;; Next source pointer
	bump	rdi, -5*256*XMM_SCD2+XMM_SCD2 ;; Next sine/cosine pointer
	bump	rbp, 5*XMM_SCD2		;; Next sine/cosine pointer
	loops_touch 64, c2b, 2, c2a	;; Loop 128 times
	bump	rsi, -128*64+dist128	;; Next source pointer
	loops	2, c2a			;; Loop twice
	end_timer 13

ENDIF

xpass2_12800_levels_complex_done:
	ENDM


;*********************************************************************************************
;			Macros derived from pass2-14-levels
;*********************************************************************************************


; *************** radix4-pass2-9216-levels-real macro ******************
; This macro takes 18432 real values and performs the final 14.17 levels of
; the FFT.  Independently, this macro also takes 9216 complex values and
; performs the final 13.17 levels of the FFT.  The results are squared, and
; the first 13.17/14.17 levels of the inverse FFT are done.

r4_pass2_9216_levels_real MACRO
	LOCAL	b1b, d2b, d3b, b2b, b3b, b4b, b5a, b5b, b6b, b7a, b7b
	LOCAL	b8b, b9b, baa, bab0, bab, bac, bad0, bad
	LOCAL	c1b, e2b, e3b, c2b, c3b, c4b, c5a, c5b, c6b, c7a, c7b, c8b, c9b
	LOCAL	xpass2_9216_levels_real_1, xpass2_9216_levels_real_2
	LOCAL	xpass2_9216_levels_real_3, xpass2_9216_levels_real_4
	LOCAL	xpass2_9216_real_unfft, xpass2_9216_real_done

;; We switch to the same format used in one-pass FFTs (that is, the
;; high word of xmm registers being 1 greater than the low word):

;; Type 4 FFTs skip the forward FFT process

	start_timer 2
	mov	rsi, DATA_ADDR		;; Load source address
	mov	rbx, DIST_TO_FFTSRCARG
	cmp	ffttype, 4
	je	xpass2_9216_levels_real_4

;; Do FFT level 1,2 (actually 1.585 levels)
;;
;; On input the 64-byte cache lines hold these data values if pass 2 swizzles:
;;	0	+4608	9216	+4608	18432	+4608	27648	+4608
;;	1	...
;;	...
;;	4607	...
;; On input the 64-byte cache lines hold these data values if pass 1 swizzles:
;;	0	+18K	4608	+18K	9216	+18K	13824	+18K
;;	1	...
;;	...
;;	4607	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+18K	1536	+18K	3072	+18K	4608	+18K
;;	1	...
;;	...
;;	1535	...
;;	6144	...
;;	...

;; Do 3072 six_reals_fft operations
;; Do 3072 three_complex_djbfft operations
;;	distance between fft data elements is 3072
;;	do 1536 iterations

	mov	rdi, xsincos_complex	;; Ptr to the sin/cos data
	mov	rdx, sincos3		;; Ptr to real sin/cos data
	loops_init 1536
b1b:	r3_fh3cl_six_reals_three_complex_djbfft rsi, 64, 12*dist128, rdi, 1536*XMM_SCD1, rdx, 1536*XMM_HSCD1
	bump	rdi, XMM_SCD1		;; Next sin/cos data
	bump	rdx, XMM_HSCD1		;; Next sin/cos data
	loops	128, b1b		;; Test loop counter
	bump	rsi, -128*64+dist128	;; Next source pointer
	loops	12, b1b			;; Test loop counter
	bump	rsi, -12*dist128	;; Restore source pointer
	bump	rdi, -1536*XMM_SCD1	;; Restore sin/cos ptr
	bump	rdx, -1536*XMM_HSCD1	;; Restore sin/cos ptr

;; Do FFT levels 3,4 (actually 1.585 levels)
;;
;; On input the 64-byte cache lines hold these data values:
;;	0	+18K	1536	+18K	3072	+18K	4608	+18K
;;	1	...
;;	...
;;	1535	...
;;	6144	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+18K	512	+18K	1024	+18K	1536	+18K
;;	1	...
;;	...
;;	511	...
;;	2048	...
;;	...

;; Do 1024 six_reals_fft operations
;; Do 1024 three_complex_djbfft operations
;;	distance between fft data elements is 1024
;;	do 512 macros each processing 24 data values

	mov	rcx, DATA_PREFETCH	;; Prefetch pointer
	xtouch	[rcx]			;; Preload first TLB
	loops_init 1536
d2b:	r3_h3cl_six_reals_three_complex_djbfft rsi, 64, 4*dist128, rdi, 512*3*XMM_SCD1, rdx, 512*3*XMM_HSCD1
	bump	rdi, 3*XMM_SCD1		;; Next sin/cos data
	bump	rdx, 3*XMM_HSCD1	;; Next sin/cos data
	loops	128, d2b		;; Test inner loop counter
	bump	rsi, -128*64+dist128	;; Next source pointer
	loops	4, d2b			;; Test loop counter
	bump	rsi, -4*dist128+12*dist128 ;; Next source pointer
	bump	rdi, -512*3*XMM_SCD1	;; Restore sin/cos ptr
	bump	rdx, 512*3*XMM_HSCD1	;; Next sin/cos ptr

;; Do 4096 three_complex_fft operations
;;	distance between fft data elements is 1024
;;	do 1024 macros each processing 24 data values

	loops_reset			;; 2 more iters of 512
	r3_nf3cl_three_complex_djbfft_preload
d3b:	r3_nf3cl_three_complex_djbfft rsi, 64, 4*dist128, rdi, 512*3*XMM_SCD1
	bump	rdi, 3*XMM_SCD1		;; Next sin/cos data
	loops	128, d3b		;; Test inner loop counter
	bump	rsi, -128*64+dist128	;; Next source pointer
	loops	4, d3b			;; Test loop counter
	bump	rsi, -4*dist128+12*dist128 ;; Next source pointer
	bump	rdi, -512*3*XMM_SCD1	;; Restore sin/cos ptr
	loops	3, d3b			;; Test loop counter
	bump	rsi, -3*12*dist128	;; Restore source pointer

;; Do FFT levels 5,6
;;
;; On input the 64-byte cache lines hold these data values:
;;	0	+18K	512	+18K	1024	+18K	1536	+18K
;;	1	...
;;	...
;;	511	...
;;	2048	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+18K	256	+18K	512	+18K	768	+18K
;;	1	...
;;	...
;;	255	...
;;	1024	...
;;	...

;; Do 256 eight_reals_fft operations
;; Do 256 four_complex_djbfft operations
;;	distance between fft data elements is 256
;;	do 256 macros each processing 16 data values

	mov	rbx, rdi
	loops_init 2304
b2b:	r4_h2cl_2sc_eight_reals_four_complex_djbfft rsi, 64, 2*dist128, rdi, rbx, rdx
	bump	rdi, 9*XMM_SCD1		;; Next sin/cos data
	bump	rbx, 18*XMM_SCD1	;; Next sin/cos data
	bump	rdx, XMM_HSCD2		;; Next sin/cos data
	loops	128, b2b		;; Test inner loop counter
	bump	rsi, -128*64+dist128	;; Next source pointer
	loops	2, b2b			;; Test loop counter
	bump	rsi, -2*dist128+4*dist128 ;; Next source pointer
	bump	rdi, -256*9*XMM_SCD1	;; Restore sin/cos ptr
	bump	rbx, -256*18*XMM_SCD1	;; Restore sin/cos ptr
	bump	rdx, -256*XMM_HSCD2	;; Restore sin/cos ptr

;; Do 4096 four_complex_fft operations
;;	distance between fft data elements is 256
;;	do 2048 macros each processing 16 data values

	loops_reset
	r4_x2cl_2sc_four_complex_djbfft_preload
b3b:	r4_x2cl_2sc_four_complex_djbfft rsi, 64, 2*dist128, rdi, rbx
	bump	rdi, 9*XMM_SCD1		;; Next sin/cos data
	bump	rbx, 18*XMM_SCD1	;; Next sin/cos data
	loops	128, b3b		;; Test inner loop counter
	bump	rsi, -128*64+dist128	;; Next source pointer
	loops	2, b3b			;; Test loop counter
	bump	rsi, -2*dist128+4*dist128 ;; Next source pointer
	bump	rdi, -256*9*XMM_SCD1	;; Restore sin/cos ptr
	bump	rbx, -256*18*XMM_SCD1	;; Restore sin/cos ptr
	loops	9, b3b			;; Test loop counter
	bump	rsi, -9*4*dist128	;; Restore source pointer

;; Do FFT levels 7,8
;;
;; On input the 64-byte cache lines hold these data values:
;;	0	+18K	256	+18K	512	+18K	768	+18K
;;	1	...
;;	...
;;	255	...
;;	1024	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+18K	64	+18K	128	+18K	192	+18K
;;	1	...
;;	...
;;	63	...
;;	256	...
;;	...

;; Do 64 eight_reals_fft operations
;; Do 192 four_complex_djbfft operations
;;	distance between fft data elements is 64
;;	do 64 macros each processing 32 data values

	bump	rdi, 4608*XMM_SCD1	;; Load sin/cos pointer
	loops_init_prefetch 1152, 64, 1, rcx
	xtouch	[rcx+4096-64]		;; Preload the TLBs
	r4_h4cl_eight_reals_four_complex_djbfft_preload
b4b:	r4_h4cl_eight_reals_four_complex_djbfft rsi, 64, 64*64, dist128, rdi, rdx
	bump	rdi, XMM_SCD2		;; Next sin/cos data
	bump	rdx, 4*XMM_HSCD2	;; Next sin/cos data
	loops	64, b4b			;; Test loop counter
	bump	rsi, -64*64+2*dist128	;; Next source pointer
	bump	rdi, -64*XMM_SCD2	;; Restore sin/cos ptr
	bump	rdx, -64*4*XMM_HSCD2	;; Restore sin/cos ptr

;; Do 4352 four_complex_djbfft operations
;;	distance between fft data elements is 64
;;	do 1088 macros each processing 32 data values

	loops_reset			;; 17 more iterations of 64
	r4_x4cl_four_complex_djbfft_preload
b5a:	xtouch	[rcx+4096-64]		;; Preload the TLBs
b5b:	r4_x4cl_four_complex_djbfft rsi, 64, 64*64, dist128, rdi
	bump	rdi, XMM_SCD2		;; Next sin/cos data
	loops	64, b5b			;; Test inner loop counter
	bump	rsi, -64*64+2*dist128	;; Next source pointer
	bump	rdi, -64*XMM_SCD2	;; Restore sin/cos ptr
	loops	18, b5a			;; Test loop counter
	bump	rsi, -18*2*dist128	;; Restore source pointer

;; Do FFT levels 9,10
;;
;; On input the 64-byte cache lines hold these data values:
;;	0	+18K	64	+18K	128	+18K	192	+18K
;;	1	...
;;	...
;;	63	...
;;	256	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+18K	16	+18K	32	+18K	48	+18K
;;	1	...
;;	...
;;	15	...
;;	64	...
;;	...

;; Do 16 eight_reals_fft operations
;; Do 48 four_complex_djbfft operations
;;	distance between fft data elements is 16
;;	do 16 macros each processing 32 data values

	loops_init_prefetch 1152, 64, 1, rcx
	xtouch	[rcx+4096-64]		;; Preload the TLBs
	r4_h4cl_eight_reals_four_complex_djbfft_preload
b6b:	r4_h4cl_eight_reals_four_complex_djbfft rsi, 64, 16*64, 32*64, rdi, rdx
	bump	rdi, 4*XMM_SCD2		;; Next sin/cos data
	bump	rdx, 16*XMM_HSCD2	;; Next sin/cos data
	loops	16, b6b			;; Test loop counter
	bump	rsi, -16*64+64*64	;; Next source pointer
	bump	rdi, -16*4*XMM_SCD2	;; Restore sin/cos ptr
	bump	rdx, -16*16*XMM_HSCD2	;; Restore sin/cos ptr

;; Do 4544 four_complex_djbfft operations
;;	distance between fft data elements is 16
;;	do 1136 macros each processing 32 data values

	loops_reset			;; 36 iters of 1 or 2 of 16
	r4_x4cl_four_complex_djbfft_preload
b7a:	xtouch	[rcx+4096-64]		;; Preload the TLBs
b7b:	r4_x4cl_four_complex_djbfft rsi, 64, 16*64, 32*64, rdi
	bump	rdi, 4*XMM_SCD2		;; Next sin/cos data
	loops	16, b7b			;; Test inner loop counter
	bump	rsi, -16*64+64*64	;; Next source pointer
	bump	rdi, -16*4*XMM_SCD2	;; Restore sin/cos ptr
	loops	2, b7b			;; Test loop counter
	bump	rsi, -2*64*64+dist128	;; Next source pointer
	loops_touch 2, b7b, 18, b7a	;; 36 iterations
	bump	rsi, -36*dist128	;; Next source pointer

;; Do FFT levels 11,12
;;
;; On input the 64-byte cache lines hold these data values:
;;	0	+18K	16	+18K	32	+18K	48	+18K
;;	1	...
;;	...
;;	15	...
;;	64	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+18K	4	+18K	8	+18K	12	+18K
;;	1	...
;;	...
;;	3	...
;;	16	...
;;	...

;; Do 4 eight_reals_fft operations
;; Do 12 four_complex_djbfft operations
;;	distance between fft data elements is 4
;;	do 4 macros each processing 32 data values

	loops_init 1152
	r4_h4cl_eight_reals_four_complex_djbfft_preload
b8b:	r4_h4cl_eight_reals_four_complex_djbfft rsi, 64, 4*64, 8*64, rdi, rdx
	bump	rdi, 16*XMM_SCD2	;; Next sin/cos data
	bump	rdx, 64*XMM_HSCD2	;; Next sin/cos data
	loops	4, b8b			;; Test loop counter
	bump	rsi, -4*64+16*64	;; Next source pointer
	bump	rdi, -4*16*XMM_SCD2	;; Restore sin/cos ptr
	bump	rdx, -4*64*XMM_HSCD2	;; Restore sin/cos ptr

;; Do 4592 four_complex_djbfft operations
;;	distance between fft data elements is 4
;;	do 1148 macros each processing 32 data values

	loops_reset			;; 36 iters of 7 or 8 of 4
	r4_x4cl_four_complex_djbfft_preload
b9b:	r4_x4cl_four_complex_djbfft rsi, 64, 4*64, 8*64, rdi
	bump	rdi, 16*XMM_SCD2	;; Next sin/cos data
	loops	4, b9b			;; Test loop counter
	bump	rsi, -4*64+16*64	;; Next source pointer
	bump	rdi, -4*16*XMM_SCD2	;; Restore sin/cos ptr
	loops	8, b9b			;; Test middle loop counter
	bump	rsi, -8*16*64+dist128	;; Next source pointer
	loops	36, b9b			;; 36 iterations
	bump	rsi, -36*dist128	;; Restore source pointer

;; Do FFT levels 13,14
;;
;; On input the 64-byte cache lines hold these data values:
;;	0	+18K	4	+18K	8	+18K	12	+18K
;;	1	...
;;	...
;;	3	...
;;	16	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+18K	2	+18K	4	+18K	6	+18K
;;	1	...
;;	8	...
;;	...

;; Do 1 eight_reals_fft operations
;; Do 3 four_complex_fft operations
;;	distance between fft data elements is 1

;; Do 4604 four_complex_fft operations
;;	distance between fft data elements is 1
;;	do 1151 macros each processing 32 data values

;; Execute the proper middle step

	cmp	ffttype, 2
	jl	xpass2_9216_levels_real_1
	jg	xpass2_9216_levels_real_3

xpass2_9216_levels_real_2:
	r4_h4cl_eight_reals_four_complex_with_square rsi, 4*64, 64, 2*64
	loops_init 1152, 1, 32
	r4_x4cl_four_complex_with_square_preload
baa:	r4_x4cl_four_complex_with_square rsi, 4*64, 64, 2*64
	loops	32, baa			;; Test loop counter
	bump	rsi, -32*4*64+dist128	;; Next source pointer
	loops	36, baa			;; Test loop counter
	bump	rsi, -36*dist128	;; Restore source pointer
	jmp	xpass2_9216_real_unfft

xpass2_9216_levels_real_1:
	r4_h4cl_eight_reals_four_complex_fft_final rsi, 4*64, 64, 2*64
	L2prefetch128 [rcx]
	bump	rcx, 128
	loops_init_prefetch 1152, 128, 1, rcx, 1, 32
bab0:	xtouch	[rcx+4096-64]		;; Preload the TLBs
bab:	r4_x4cl_four_complex_fft_final rsi, 4*64, 64, 2*64
	loops	32, bab			;; Test loop counter
	bump	rsi, -32*4*64+dist128	;; Next source pointer
	loops	36, bab0		;; Test loop counter
	jmp	xpass2_9216_real_done

xpass2_9216_levels_real_3:
	mov	rbp, DIST_TO_MULSRCARG
	r4_h4cl_eight_reals_four_complex_with_mult rsi, 4*64, 64, 2*64
	loops_init 1152, 1, 32
bac:	r4_x4cl_four_complex_with_mult rsi, 4*64, 64, 2*64
	loops	32, bac			;; Test loop counter
	bump	rsi, -32*4*64+dist128	;; Next source pointer
	loops	36, bac			;; Test loop counter
	bump	rsi, -36*dist128	;; Restore source pointer
	jmp	xpass2_9216_real_unfft

xpass2_9216_levels_real_4:
	mov	rbp, DIST_TO_MULSRCARG
	mov	rcx, DATA_PREFETCH	;; Load prefetch pointer
	xtouch	[rcx]			;; Preload the TLBs
	r4_h4cl_eight_reals_four_complex_with_mulf rsi, 4*64, 64, 2*64
	L2prefetch128 [rcx]
	bump	rcx, 128
	loops_init_prefetch 1152, 128, 1, rcx, 1, 32
bad0:	xtouch	[rcx+4096-64]		;; Preload the TLBs
bad:	r4_x4cl_four_complex_with_mulf rsi, 4*64, 64, 2*64
	loops	32, bad			;; Test loop counter
	bump	rsi, -32*4*64+dist128	;; Next source pointer
	loops	36, bad0		;; Test loop counter
	bump	rsi, -36*dist128	;; Restore source pointer

;; Do inverse FFT levels 11,12
;; On input the 64-byte cache lines hold these data values:
;;	0	+18K	2	+18K	4	+18K	6	+18K
;;	1	...
;;	8	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+18K	8	+18K	16	+18K	24	+18K
;;	1	...
;;	...
;;	7	...
;;	32	...
;;	...

;; Do 4 eight_reals_unfft operations
;; Do 4 four_complex_djbunfft operations
;;	distance between fft data elements is 4
;;	do 2 macros processing 32 data values

xpass2_9216_real_unfft:
	mov	rdi, xsincos_complex	;; Load sin/cos pointers
	bump	rdi, 4608*XMM_SCD1
	mov	rdx, sincos3
	bump	rdx, 3072*XMM_HSCD1
	loops_init 1152
c8b:	r4_h4cl_eight_reals_four_complex_djbunfft rsi, 64, 2*64, 4*64, rdi, 2*16*XMM_SCD2, rdx, 2*64*XMM_HSCD2
	bump	rdi, 16*XMM_SCD2	;; Next sin/cos data
	bump	rdx, 64*XMM_HSCD2	;; Next sin/cos data
	loops	2, c8b			;; Test loop counter
	bump	rsi, -2*64+8*64		;; Next source pointer
	bump	rdi, -2*16*XMM_SCD2	;; Restore sin/cos ptr
	bump	rdx, -2*64*XMM_HSCD2	;; Restore sin/cos ptr

;; Do 4600 four_complex_djbunfft operations
;;	distance between fft data elements is 4
;;	do 1150 macros each processing 32 data values

	loops_reset			;; 36 iters of 15 or 16 of 2
	r4_x4cl_four_complex_djbunfft_preload
c9b:	r4_x4cl_four_complex_djbunfft rsi, 64, 2*64, 4*64, rdi, 2*16*XMM_SCD2
	bump	rdi, 16*XMM_SCD2	;; Next sine/cosine pointer
	loops	2, c9b			;; Test loop counter
	bump	rsi, -2*64+8*64		;; Next source pointer
	bump	rdi, -2*16*XMM_SCD2	;; Restore sin/cos ptr
	loops	16, c9b			;; Test loop counter
	bump	rsi, -16*8*64+dist128	;; Next source pointer
	loops	36, c9b			;; Test loop counter
	bump	rsi, -36*dist128	;; Restore source pointer

;; Do inverse FFT levels 9,10
;; On input the 64-byte cache lines hold these data values:
;;	0	+18K	8	+18K	16	+18K	24	+18K
;;	1	...
;;	...
;;	7	...
;;	32	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+18K	32	+18K	64	+18K	96	+18K
;;	1	...
;;	...
;;	31	...
;;	128	...
;;	...

;; Do 16 eight_reals_unfft operations
;; Do 16 four_complex_djbunfft operations
;;	distance between fft data elements is 16
;;	do 8 macros processing 32 data values

	loops_init_prefetch 1152, 64, 1, rcx
	xtouch	[rcx+4096-64]		;; Preload the TLBs
c6b:	r4_h4cl_eight_reals_four_complex_djbunfft rsi, 64, 8*64, 16*64, rdi, 8*4*XMM_SCD2, rdx, 8*16*XMM_HSCD2
	bump	rdi, 4*XMM_SCD2		;; Next sin/cos data
	bump	rdx, 16*XMM_HSCD2	;; Next sin/cos data
	loops	8, c6b			;; Test inner loop counter
	bump	rsi, -8*64+32*64	;; Next source pointer
	bump	rdi, -8*4*XMM_SCD2	;; Restore sin/cos ptr
	bump	rdx, -8*16*XMM_HSCD2	;; Restore sin/cos ptr

;; Do 4576 four_complex_djbunfft operations
;;	distance between fft data elements is 16
;;	do 1144 macros each processing 32 data values

	loops_reset			;; 36 iters of 3 or 4 of 4
	r4_x4cl_four_complex_djbunfft_preload
c7a:	xtouch	[rcx+4096-64]		;; Preload the TLBs
c7b:	r4_x4cl_four_complex_djbunfft rsi, 64, 8*64, 16*64, rdi, 8*4*XMM_SCD2
	bump	rdi, 4*XMM_SCD2		;; Next sin/cos data
	loops	8, c7b			;; Test inner loop counter
	bump	rsi, -8*64+32*64	;; Next source pointer
	bump	rdi, -8*4*XMM_SCD2	;; Restore sin/cos ptr
	loops	4, c7b			;; Test loop counter
	bump	rsi, -4*32*64+dist128	;; Next source pointer
	loops_touch 2, c7b, 18, c7a	;; Test loop counter
	bump	rsi, -36*dist128	;; Restore source pointer

;; Do inverse FFT levels 7,8
;; On input the 64-byte cache lines hold these data values:
;;	0	+18K	32	+18K	64	+18K	96	+18K
;;	1	...
;;	...
;;	31	...
;;	128	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+18K	128	+18K	256	+18K	384	+18K
;;	1	...
;;	...
;;	127	...
;;	512	...
;;	...

;; Do 64 eight_reals_unfft operations
;; Do 64 four_complex_djbunfft operations
;;	distance between fft data elements is 64
;;	do 32 macros processing 32 data values

	loops_init_prefetch 1152, 64, 1, rcx
	xtouch	[rcx+4096-64]		;; Preload the TLBs
c4b:	r4_h4cl_eight_reals_four_complex_djbunfft rsi, 64, 32*64, 64*64, rdi, 32*XMM_SCD2, rdx, 32*4*XMM_HSCD2
	bump	rdi, XMM_SCD2		;; Next sin/cos data
	bump	rdx, 4*XMM_HSCD2	;; Next sin/cos data
	loops	32, c4b			;; Test inner loop counter
	bump	rsi, -32*64+dist128	;; Next source pointer
	bump	rdi, -32*XMM_SCD2	;; Restore sin/cos ptr
	bump	rdx, -32*4*XMM_HSCD2	;; Restore sin/cos ptr

;; Do 4480 four_complex_djbunfft operations
;;	distance between fft data elements is 64
;;	do 1120 macros each processing 32 data values

	loops_reset			;; 35 more iters of 32
	r4_x4cl_four_complex_djbunfft_preload
c5a:	xtouch	[rcx+4096-64]		;; Preload the TLBs
c5b:	r4_x4cl_four_complex_djbunfft rsi, 64, 32*64, 64*64, rdi, 32*XMM_SCD2
	bump	rdi, XMM_SCD2		;; Next sin/cos data
	loops	32, c5b			;; Test inner loop counter
	bump	rsi, -32*64+dist128	;; Next source pointer
	bump	rdi, -32*XMM_SCD2	;; Restore sin/cos ptr
	loops_touch 2, c5b, 18, c5a	;; Test loop counter
	bump	rsi, -36*dist128	;; Next source pointer

;; Do inverse FFT levels 5,6
;; On input the 64-byte cache lines hold these data values:
;;	0	+18K	128	+18K	256	+18K	384	+18K
;;	1	...
;;	...
;;	127	...
;;	512	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+18K	512	+18K	1024	+18K	1536	+18K
;;	1	...
;;	...
;;	511	...
;;	2048	...
;;	...

;; Do 256 eight_reals_unfft operations
;; Do 256 four_complex_djbunfft operations
;;	distance between fft data elements is 256
;;	do 128 macros processing 32 data values

	mov	rdi, xsincos_complex	;; Load sin/cos pointer
	mov	rbx, rdi
	bump	rcx, -4*64		;; Back up prefetch pointer by 4 cache lines
	loops_init_prefetch 1152, 64, 16, rcx
	xtouch	[rcx+4096-64]		;; Preload the TLBs
	xtouch	[rcx+72*64-64]		;; Preload the TLBs
c2b:	r4_h4cl_2sc_eight_reals_four_complex_djbunfft rsi, 64, dist128, 2*dist128, rdi, 128*9*XMM_SCD1, rbx, 128*18*XMM_SCD1, rdx, 128*XMM_HSCD2
	bump	rdi, 9*XMM_SCD1		;; Next sin/cos data
	bump	rbx, 18*XMM_SCD1	;; Next sin/cos data
	bump	rdx, XMM_HSCD2		;; Next sin/cos data
	loops	128, c2b		;; Test inner loop counter
	bump	rsi, -128*64+4*dist128	;; Next source pointer
	bump	rdi, -128*9*XMM_SCD1	;; Restore sin/cos ptr
	bump	rbx, -128*18*XMM_SCD1	;; Restore sin/cos ptr
	bump	rdx, -128*XMM_HSCD2-3072*XMM_HSCD1 ;; Next sin/cos ptr

;; Do 4096 four_complex_djbunfft operations
;;	distance between fft data elements is 256
;;	do 1024 macros each processing 32 data values

	loops_reset			;; 8 more iters of 128
	r4_x4cl_2sc_four_complex_djbunfft_preload
c3b:	r4_x4cl_2sc_four_complex_djbunfft rsi, 64, dist128, 2*dist128, rdi, 128*9*XMM_SCD1, rbx, 128*18*XMM_SCD1
	bump	rdi, 9*XMM_SCD1		;; Next sin/cos data
	bump	rbx, 18*XMM_SCD1	;; Next sin/cos data
	loops	128, c3b		;; Test inner loop counter
	bump	rsi, -128*64+4*dist128	;; Next source pointer
	bump	rdi, -128*9*XMM_SCD1	;; Restore sin/cos ptr
	bump	rbx, -128*18*XMM_SCD1	;; Restore sin/cos ptr
	loops	9, c3b			;; Test loop counter
	bump	rsi, -9*4*dist128	;; Next source pointer

;; Do inverse FFT levels 3,4 (actually 1.585 FFT levels)
;; On input the 64-byte cache lines hold these data values:
;;	0	+18K	512	+18K	1024	+18K	1536	+18K
;;	1	...
;;	...
;;	511	...
;;	2048	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+18K	1536	+18K	3072	+18K	4608	+18K
;;	1	...
;;	...
;;	1535	...
;;	6144	...
;;	...

;; Do 1024 six_reals_unfft operations
;; Do 1024 six_complex_djbunfft operations
;;	distance between fft data elements is 1024
;;	do 512 macros processing 24 data values

	loops_init 1536
e2b:	r3_h3cl_six_reals_three_complex_djbunfft rsi, 64, 4*dist128, rdi, 512*3*XMM_SCD1, rdx, 512*3*XMM_HSCD1
	bump	rdi, 3*XMM_SCD1		;; Next sin/cos data
	bump	rdx, 3*XMM_HSCD1	;; Next sin/cos data
	loops	128, e2b		;; Test inner loop counter
	bump	rsi, -128*64+dist128	;; Next source pointer
	loops	4, e2b			;; Test loop counter
	bump	rsi, -4*dist128+12*dist128 ;; Next source pointer
	bump	rdi, -512*3*XMM_SCD1	;; Restore sin/cos ptr
	bump	rdx, -512*3*XMM_HSCD1	;; Restore sin/cos ptr

;; Do 4096 three_complex_djbunfft operations
;;	distance between fft data elements is 1024
;;	do 1024 macros each processing 24 data values

	loops_reset			;; 2 more iters of 512
	r3_x3cl_three_complex_djbunfft_preload
e3b:	r3_x3cl_three_complex_djbunfft rsi, 64, 4*dist128, rdi, 512*3*XMM_SCD1
	bump	rdi, 3*XMM_SCD1		;; Next sin/cos data
	loops	128, e3b		;; Test inner loop counter
	bump	rsi, -128*64+dist128	;; Next source pointer
	loops	4, e3b			;; Test loop counter
	bump	rsi, -4*dist128+12*dist128 ;; Next source pointer
	bump	rdi, -512*3*XMM_SCD1	;; Restore sin/cos ptr
	loops	3, e3b			;; Test loop counter
	bump	rsi, -3*12*dist128	;; Restore source pointer

;; Do inverse FFT levels 1,2 (actually 1.585 FFT levels)
;; On input the 64-byte cache lines hold these data values:
;;	0	+18K	1536	+18K	3072	+18K	4608	+18K
;;	1	...
;;	...
;;	1535	...
;;	6144	...
;;	...
;; On output the 64-byte cache lines hold these data values if pass 1 swizzles:
;;	0	+18K	4608	+18K	9216	+18K	13824	+18K
;;	1	...
;;	...
;;	4607	...
;; On output the 64-byte cache lines hold these data values if pass 2 swizzles:
;;	0	+4608	9216	+4608	18432	+4608	27648	+4608
;;	1	...
;;	...
;;	4607	...

;; Do 3072 six_reals_unfft operations
;; Do 3072 three_complex_djbunfft operations
;;	distance between fft data elements is 3072
;;	do 1536 macros processing 24 data values

	loops_init 1536
c1b:	r3_h3cl_six_reals_three_complex_djbunfft rsi, 64, 12*dist128, rdi, 1536*XMM_SCD1, rdx, 1536*XMM_HSCD1
	bump	rdi, XMM_SCD1		;; Next sin/cos data
	bump	rdx, XMM_HSCD1		;; Next sin/cos data
	loops	128, c1b		;; Test loop counter
	bump	rsi, -128*64+dist128	;; Next source pointer
	loops	12, c1b			;; Test loop counter

xpass2_9216_real_done:
	end_timer 2
	ENDM

; *************** radix4-pass2-9216-levels-complex macro ******************
; This macro takes 2 groups of 9216 complex values and
; performs the final 13.17 levels of a standard radix-4 FFT, squares the
; results, and does the first 13.17 levels of the inverse FFT.

r4_pass2_9216_levels_complex MACRO
	LOCAL	b1a, b1b, b7a, b7b, b2b, b3a, b3b, b4a, b4b, b5a, b5b
	LOCAL	b6a0, b6a, b6b0, b6b, b6c0, b6c, b6d0, b6d
	LOCAL	c1a, c1b, c7a, c7b, c2a, c2b, c3a, c3b, c4a, c4b, c5a, c5b
	LOCAL	xpass2_9216_levels_complex_1, xpass2_9216_levels_complex_2
	LOCAL	xpass2_9216_levels_complex_3, xpass2_9216_levels_complex_4
	LOCAL	xpass2_9216_levels_complex_unfft, xpass2_9216_levels_complex_done

;; Type 4 FFTs skip the forward FFT process

	mov	rsi, DATA_ADDR		;; Load address of FFT data
	mov	rbx, DIST_TO_FFTSRCARG
	cmp	ffttype, 4
	je	xpass2_9216_levels_complex_4

;; Do FFT level 1,2 (actually 1.585 levels)
;;
;; We swizzle the data so that one set of data is in the high word of
;; an xmm register, the other set is in the low word.  This means
;; a distance of 4608 data values.
;;
;; On input the 64-byte cache lines hold these data values if pass 2 swizzles:
;;	0	+4608	9216	+4608	18432	+4608	27648	+4608
;;	1	...
;;	...
;;	4607	...
;; On input the 64-byte cache lines hold these data values if pass 1 swizzles:
;;	0	+18K	4608	+18K	9216	+18K	13824	+18K
;;	1	...
;;	...
;;	4607	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+18K	1536	+18K	3072	+18K	4608	+18K
;;	1	...
;;	...
;;	1535	...
;;	6144	...
;;	...

;; Do 6144 three_complex_fft operations
;;	distance between fft data elements is 3072
;;	do 1536 macros each processing 24 data values

;; Do FFT levels 3,4 (actually 1.585 levels)
;;
;; On input the 64-byte cache lines hold these data values:
;;	0	+18K	1536	+18K	3072	+18K	4608	+18K
;;	1	...
;;	...
;;	1535	...
;;	6144	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+18K	512	+18K	1024	+18K	1536	+18K
;;	1	...
;;	...
;;	511	...
;;	2048	...
;;	...

;; Do 6144 three_complex_fft operations
;;	distance between fft data elements is 1024
;;	do 1536 macros each processing 24 data values

;; Do FFT levels 5,6
;;
;; On input the 64-byte cache lines hold these data values:
;;	0	+18K	512	+18K	1024	+18K	1536	+18K
;;	1	...
;;	...
;;	511	...
;;	2048	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+18K	256	+18K	512	+18K	768	+18K
;;	1	...
;;	...
;;	255	...
;;	1024	...
;;	...

;; Do 4608 four_complex_fft operations
;;	distance between fft data elements is 256
;;	do 2304 macros each processing 16 data values

IF @INSTR(,%xarch,<CORE>) NE 0		;; Core 2 faster doing separate loops

	start_timer 3
	mov	rdi, xsincos_complex	;; The sin/cos data
	loops_init 1536			;; 12 iterations of 128
	r3_f3cl_three_complex_djbfft_preload
b1b:	r3_f3cl_three_complex_djbfft rsi, 64, 12*dist128, rdi, 1536*XMM_SCD1
	bump	rdi, XMM_SCD1		;; Next sin/cos data
	loops	128, b1b		;; Test inner loop counter
	bump	rsi, -128*64+dist128	;; Next source pointer
	loops	12, b1b			;; Test outer loop counter
	bump	rsi, -12*dist128	;; Restore source pointer
	end_timer 3

	start_timer 4
	mov	rcx, DATA_PREFETCH	;; Prefetch pointer (we need to prefetch 
					;; 4608 cache lines + 68 pad cache lines)
	xtouch	[rcx]			;; Load first prefetch TLB
	mov	rdi, xsincos_complex	;; Ptr to the sin/cos data
	loops_init_prefetch 1536, 64, 4, rcx ;; 3 iters of 4 iters of 128
	r3_nf3cl_three_complex_djbfft_preload
b7a:	xtouch	[rcx+4096-64]		;; Load prefetch TLB
b7b:	r3_nf3cl_three_complex_djbfft rsi, 64, 4*dist128, rdi, 512*3*XMM_SCD1
	bump	rdi, 3*XMM_SCD1		;; Next sin/cos data
	loops	128, b7b		;; Test inner loop counter
	bump	rsi, -128*64+dist128	;; Next source pointer
	loops_touch 2, b7b, 2, b7a	;; Test loop counter
	bump	rsi, -4*dist128+12*dist128 ;; Next source pointer
	bump	rdi, -512*3*XMM_SCD1	;; Restore sin/cos ptr
	loops	3, b7b			;; Test loop counter
	bump	rsi, -3*12*dist128	;; Restore source pointer
	end_timer 4

	start_timer 5
	mov	rdx, rdi
	loops_init 2304			;; 9 iters of 2 iters of 128
	r4_x2cl_2sc_four_complex_djbfft_preload
b2b:	r4_x2cl_2sc_four_complex_djbfft rsi, 64, 2*dist128, rdi, rdx
	bump	rdi, 9*XMM_SCD1		;; Next sin/cos data
	bump	rdx, 18*XMM_SCD1	;; Next sin/cos data
	loops	128, b2b		;; Test inner loop counter
	bump	rsi, -128*64+dist128	;; Next source pointer
	loops	2, b2b			;; Test loop counter
	bump	rsi, -2*dist128+4*dist128 ;; Next source pointer
	bump	rdi, -256*9*XMM_SCD1	;; Restore sin/cos ptr
	bump	rdx, -256*18*XMM_SCD1	;; Restore sin/cos ptr
	loops	9, b2b			;; Test loop counter
	bump	rsi, -9*4*dist128	;; Restore source pointer
	end_timer 5

ELSE					;; Pentium 4 faster doing merged loops

	start_timer 3
	mov	rdi, xsincos_complex	;; The sin/cos data
	mov	rbp, rdi
	mov	rcx, rdi
	mov	rdx, rdi
	loops_init 4608			;; 256 iters of 6 or 9

b1a:	r3_f3cl_three_complex_djbfft_preload

b1b:	r3_f3cl_three_complex_djbfft rsi, 2*dist128, 12*dist128, rdi, 1536*XMM_SCD1
	xxprefetch [rdi+XMM_SCD1]	;; Prefetch for next pass
	xxprefetch [rdi+1537*XMM_SCD1]	;; Prefetch for next pass
	bump	rdi, 256*XMM_SCD1	;; Next sin/cos ptr
	loops	6, b1b, 3		;; Test loop counter with non-standard incr
	bump	rsi, -6*2*dist128	;; Restore source pointer
	loops_undo 18

;;	r3_nf3cl_three_complex_djbfft_preload ;; Assume same as r3_f3cl_three_complex_djbfft_preload
b7b:	r3_nf3cl_three_complex_djbfft rsi, 2*dist128, 4*dist128, rbp, 512*3*XMM_SCD1
	bump	rbp, 256*3*XMM_SCD1	;; Next sin/cos data
	loops	2, b7b, 3		;; Test loop counter with non-standard incr
	bump	rsi, -2*2*dist128+12*dist128 ;; Next source pointer
	bump	rbp, -2*256*3*XMM_SCD1	;; Restore sin/cos ptr
	loops	3, b7b			;; Test loop counter
	bump	rsi, -3*12*dist128	;; Restore source pointer
	loops_undo 18

	r4_x2cl_2sc_four_complex_djbfft_preload
b2b:	r4_x2cl_2sc_four_complex_djbfft rsi, 4*dist128, 2*dist128, rcx, rdx
	loops	9, b2b, 2		;; Test loop counter with non-standard incr

	bump	rsi, -9*4*dist128+64	;; Next source pointer
	bump	rdi, -6*256*XMM_SCD1+XMM_SCD1 ;; Next sin/cos ptr
	bump	rbp, 3*XMM_SCD1		;; Next sin/cos ptr
	bump	rcx, 9*XMM_SCD1		;; Next sin/cos ptr
	bump	rdx, 18*XMM_SCD1	;; Next sin/cos ptr
	loops	128, b1a		;; Test loop counter
	bump	rsi, -128*64+dist128	;; Next source pointer
	loops	2, b1a			;; Test loop counter
	bump	rsi, -2*dist128		;; Restore source pointer
	bump	rdi, -256*XMM_SCD1	;; Restore sin/cos ptr
	mov	rcx, DATA_PREFETCH	;; Prefetch pointer (we need to prefetch
					;; 4608 cache lines + 68 pad cache lines)
	xtouch	[rcx]			;; Load first prefetch TLB
	end_timer 3

ENDIF

;; Do FFT levels 7,8
;;
;; On input the 64-byte cache lines hold these data values:
;;	0	+18K	256	+18K	512	+18K	768	+18K
;;	1	...
;;	...
;;	255	...
;;	1024	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+18K	64	+18K	128	+18K	192	+18K
;;	1	...
;;	...
;;	63	...
;;	256	...
;;	...
;;
;; Do 4608 four_complex_fft operations
;;	distance between fft data elements is 64
;;	do 1152 macros each processing 32 data values

;; Do FFT levels 9,10
;;
;; On input the 64-byte cache lines hold these data values:
;;	0	+18K	64	+18K	128	+18K	192	+18K
;;	1	...
;;	...
;;	63	...
;;	256	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+18K	16	+18K	32	+18K	48	+18K
;;	1	...
;;	...
;;	15	...
;;	64	...
;;	...
;;
;; Do 4608 four_complex_fft operations
;;	distance between fft data elements is 16
;;	do 1152 macros each processing 32 data values

;; Do FFT levels 11,12
;;
;; On input the 64-byte cache lines hold these data values:
;;	0	+18K	16	+18K	32	+18K	48	+18K
;;	1	...
;;	...
;;	15	...
;;	64	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+18K	4	+18K	8	+18K	12	+18K
;;	1	...
;;	...
;;	3	...
;;	16	...
;;	...
;;
;; Do 4608 four_complex_fft operations
;;	distance between fft data elements is 4
;;	do 1152 macros each processing 32 data values

;; Do this 64 cache lines at a time to maximize accessing data
;; while it is in the L1 cache.

	start_timer 6
	bump	rdi, 4608*XMM_SCD1	;; Load sin/cos pointers
	mov	rbx, rdi
	mov	rbp, rdi
	loops_init_prefetch 1152, 64, 2, rcx ;; 2*4 iters of 4 of 64 cache lines
	r4_x4cl_four_complex_djbfft_preload

b3a:	xtouch	[rcx+48*64-64]		;; Load prefetch TLB

b3b:	xxprefetch [rdi+4*XMM_SCD2]	;; Prefetch for next pass
	r4_x4cl_four_complex_djbfft rsi, 4*64, 64*64, dist128, rdi
	bump	rdi, 4*XMM_SCD2		;; Next sine/cosine pointer
	loops	16, b3b			;; Loop 16 times
	bump	rsi, -16*4*64		;; Restore source pointer
	bump	rdi, -16*4*XMM_SCD2	;; Restore sine/cosine pointer
	loops_undo 16			;; Back up loop counter

b4b:	r4_x4cl_four_complex_djbfft rsi, 4*64, 16*64, 32*64, rbx
	bump	rbx, 4*4*XMM_SCD2	;; Next sine/cosine pointer
	loops	4, b4b			;; Loop 4 times
	bump	rsi, -4*4*64+64*64	;; Next source pointer
	bump	rbx, -4*4*4*XMM_SCD2	;; Restore sine/cosine pointer
	loops	2, b4b			;; Loop twice
	bump	rsi, -2*64*64+dist128	;; Next source pointer
	loops	2, b4b			;; Loop twice
	bump	rsi, -2*dist128		;; Restore source pointer
	loops_undo 2*2*4		;; Back up loop counter

b5b:	r4_x4cl_four_complex_djbfft rsi, 16*64, 4*64, 8*64, rbp
	loops	8, b5b			;; Loop 8 times
	bump	rsi, -8*16*64+dist128	;; Next source pointer
	loops	2, b5b			;; Loop twice

	bump	rsi, -2*dist128+64	;; Next source pointer
	bump	rdi, XMM_SCD2		;; Next sine/cosine pointer
	bump	rbx, 4*XMM_SCD2		;; Next sine/cosine pointer
	bump	rbp, 16*XMM_SCD2	;; Next sine/cosine pointer
	loops_touch 2, b3b, 2, b3a	;; Loop 4 times
	bump	rsi, -4*64+2*dist128	;; Next source pointer
	bump	rdi, -4*XMM_SCD2	;; Restore sine/cosine pointer
	bump	rbx, -4*4*XMM_SCD2	;; Restore sine/cosine pointer
	bump	rbp, -4*16*XMM_SCD2	;; Restore sine/cosine pointer
	loops	18, b3a			;; Test outer loop counter
	bump	rsi, -18*2*dist128	;; Restore source pointer
	end_timer 6

;; Do FFT levels 13,14 as well as inverse FFT levels 13,14
;;
;; On input the 64-byte cache lines hold these data values:
;;	0	+18K	4	+18K	8	+18K	12	+18K
;;	1	...
;;	...
;;	3	...
;;	16	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+18K	2	+18K	4	+18K	6	+18K
;;	1	...
;;	8	...
;;	...

;; Do 4608 four_complex_fft operations
;;	distance between fft data elements is 1
;;	do 1152 macros each processing 32 data values

;; Execute the right middle step

	cmp	ffttype, 2
	jl	xpass2_9216_levels_complex_1
	jg	xpass2_9216_levels_complex_3

xpass2_9216_levels_complex_2:
	start_timer 9
	loops_init 1152
	r4_x4cl_four_complex_with_square_preload
b6a:	r4_x4cl_four_complex_with_square rsi, 4*64, 64, 2*64
	loops	32, b6a			;; Loop 32 times
	bump	rsi, -32*4*64+dist128	;; Next source pointer
	loops	36, b6a			;; Loop 36 times
	bump	rsi, -36*dist128	;; Restore source pointer
	end_timer 9
	jmp	xpass2_9216_levels_complex_unfft

xpass2_9216_levels_complex_1:
	start_timer 9
	loops_init_prefetch 1152, 128, 1, rcx
b6b0:	xtouch	[rcx+4096-128]		;; Load prefetch TLBs
b6b:	r4_x4cl_four_complex_fft_final rsi, 4*64, 64, 2*64
	loops	32, b6b			;; Loop 32 times
	bump	rsi, -32*4*64+dist128	;; Next source pointer
	loops	36, b6b0		;; Loop 36 times
	end_timer 9
	jmp	xpass2_9216_levels_complex_done

xpass2_9216_levels_complex_3:
	start_timer 9
	mov	rbp, DIST_TO_MULSRCARG
	loops_init 1152
b6c:	r4_x4cl_four_complex_with_mult rsi, 4*64, 64, 2*64
	loops	32, b6c			;; Loop 32 times
	bump	rsi, -32*4*64+dist128	;; Next source pointer
	loops	36, b6c			;; Loop 36 times
	bump	rsi, -36*dist128	;; Restore source pointer
	end_timer 9
	jmp	xpass2_9216_levels_complex_unfft

xpass2_9216_levels_complex_4:
	start_timer 9
	mov	rcx, DATA_PREFETCH	;; Prefetch pointer
	xtouch	[rcx]			;; Load first prefetch TLB
	mov	rbp, DIST_TO_MULSRCARG
	loops_init_prefetch 1152, 128, 1, rcx
b6d0:	xtouch	[rcx+4096-128]		;; Load prefetch TLBs
b6d:	r4_x4cl_four_complex_with_mulf rsi, 4*64, 64, 2*64
	loops	32, b6d			;; Loop 32 times
	bump	rsi, -32*4*64+dist128	;; Next source pointer
	loops	36, b6d0		;; Loop 32 times
	bump	rsi, -36*dist128	;; Restore source pointer
	bump	rcx, -192*64		;; We've prefetched 192 blocks too many
	end_timer 9

;; Do inverse FFT levels 11,12
;; On input the 64-byte cache lines hold these data values:
;;	0	+18K	2	+18K	4	+18K	6	+18K
;;	1	...
;;	8	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+18K	8	+18K	16	+18K	24	+18K
;;	1	...
;;	...
;;	7	...
;;	32	...
;;	...
;;
;; Do 4608 four_complex_unfft operations
;;	distance between fft data elements is 4
;;	do 1152 macros each processing 32 data values

;; Do inverse FFT levels 9,10
;; On input the 64-byte cache lines hold these data values:
;;	0	+18K	8	+18K	16	+18K	24	+18K
;;	1	...
;;	...
;;	7	...
;;	32	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+18K	32	+18K	64	+18K	96	+18K
;;	1	...
;;	...
;;	31	...
;;	128	...
;;	...
;;
;; Do 4608 four_complex_unfft operations
;;	distance between fft data elements is 16
;;	do 1152 macros each processing 32 data values

;; Do inverse FFT levels 7,8
;; On input the 64-byte cache lines hold these data values:
;;	0	+18K	32	+18K	64	+18K	96	+18K
;;	1	...
;;	...
;;	31	...
;;	128	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+18K	128	+18K	256	+18K	384	+18K
;;	1	...
;;	...
;;	127	...
;;	512	...
;;	...
;;
;; Do 4608 four_complex_unfft operations
;;	distance between fft data elements is 64
;;	do 1152 macros each processing 32 data values

;; Do this 64 cache lines at a time to maximize accessing data
;; while it is in the L1 cache.

xpass2_9216_levels_complex_unfft:

	start_timer 10
	mov	rdi, xsincos_complex	;; Load sin/cos pointers
	bump	rdi, 4608*XMM_SCD1
	mov	rbx, rdi
	mov	rbp, rdi
	loops_init_prefetch 1152, 64, 2, rcx ;; 8*2*8 iters of 64 cache lines
	r4_x4cl_four_complex_djbunfft_preload

c5a:	xtouch	[rcx+48*64-128]		;; Load prefetch TLB

c5b:	r4_x4cl_four_complex_djbunfft rsi, 8*64, 2*64, 4*64, rbp, 2*16*XMM_SCD2
	loops	16, c5b			;; Loop 16 times
	bump	rsi, -16*8*64		;; Restore source pointer
	loops_undo 16			;; Restore loop counter

c4b:	r4_x4cl_four_complex_djbunfft rsi, 2*64, 8*64, 16*64, rbx, 8*4*XMM_SCD2
	bump	rbx, 2*4*XMM_SCD2	;; Next sine/cosine pointer
	loops	4, c4b			;; Loop 4 times
	bump	rsi, -4*2*64+32*64	;; Next source pointer
	bump	rbx, -4*2*4*XMM_SCD2	;; Restore sine/cosine pointer
	loops	4, c4b			;; Loop 4 times
	bump	rsi, -4*32*64		;; Restore source pointer
	loops_undo 4*4			;; Restore loop counter

c3b:	r4_x4cl_four_complex_djbunfft rsi, 2*64, 32*64, 64*64, rdi, 32*1*XMM_SCD2
	bump	rdi, 2*XMM_SCD2		;; Next sine/cosine pointer
	loops	16, c3b			;; Loop 16 times
	bump	rdi, -16*2*XMM_SCD2	;; Restore sine/cosine pointer

	bump	rsi, -16*2*64+64	;; Next source pointer
	bump	rdi, XMM_SCD2		;; Next sine/cosine pointer
	bump	rbx, 4*XMM_SCD2		;; Next sine/cosine pointer
	bump	rbp, 16*XMM_SCD2	;; Next sine/cosine pointer
	loops	2, c5b			;; Loop twice
	bump	rsi, -2*64+dist128	;; Next source pointer
	bump	rdi, -2*XMM_SCD2	;; Restore sine/cosine pointer
	bump	rbx, -2*4*XMM_SCD2	;; Restore sine/cosine pointer
	bump	rbp, -2*16*XMM_SCD2	;; Restore sine/cosine pointer
	loops	36, c5a			;; Loop 36 times
	bump	rsi, -36*dist128	;; Restore source pointer
	end_timer 10

;; Do inverse FFT levels 5,6
;; On input the 64-byte cache lines hold these data values:
;;	0	+18K	128	+18K	256	+18K	384	+18K
;;	1	...
;;	...
;;	127	...
;;	512	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+18K	512	+18K	1024	+18K	1536	+18K
;;	1	...
;;	...
;;	511	...
;;	2048	...
;;	...

;; Do 4608 four_complex_unfft operations
;;	distance between fft data elements is 256
;;	do 1152 macros each processing 32 data values

;; Do inverse FFT levels 3,4 (actually 1.585 FFT levels)
;; On input the 64-byte cache lines hold these data values:
;;	0	+18K	512	+18K	1024	+18K	1536	+18K
;;	1	...
;;	...
;;	511	...
;;	2048	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+18K	1536	+18K	3072	+18K	4608	+18K
;;	1	...
;;	...
;;	1535	...
;;	6144	...
;;	...

;; Do 6144 three_complex_unfft operations
;;	distance between fft data elements is 1024
;;	do 1536 macros each processing 24 data values

;; Do inverse FFT levels 1,2 (actually 1.585 FFT levels)
;; On input the 64-byte cache lines hold these data values:
;;	0	+18K	1536	+18K	3072	+18K	4608	+18K
;;	1	...
;;	...
;;	1535	...
;;	6144	...
;;	...
;; On output the 64-byte cache lines hold these data values if pass 1 swizzles:
;;	0	+18K	4608	+18K	9216	+18K	13824	+18K
;;	1	...
;;	...
;;	4607	...
;; On output the 64-byte cache lines hold these data values if pass 2 swizzles:
;;	0	+4608	9216	+4608	18432	+4608	27648	+4608
;;	1	...
;;	...
;;	4607	...

;; Do 6144 three_complex_unfft operations
;;	distance between fft data elements is 3072
;;	do 1536 macros each processing 24 data values

IF @INSTR(,%xarch,<CORE>) NE 0		;; Core 2 faster doing separate loops

	start_timer 13
	mov	rdi, xsincos_complex	;; Load sin/cos pointer
	mov	rdx, rdi
	loops_init_prefetch 1152, 64, 16, rcx ;; 8 iters of 2 iters of 128
	xtouch	[rcx+4096-64]		;; Load prefetch TLBs
	xtouch	[rcx+72*64-64]
	r4_x4cl_2sc_four_complex_djbunfft_preload
c2b:	r4_x4cl_2sc_four_complex_djbunfft rsi, 64, dist128, 2*dist128, rdi, 128*9*XMM_SCD1, rdx, 128*18*XMM_SCD1
	bump	rdi, 9*XMM_SCD1		;; Next sine/cosine pointer
	bump	rdx, 18*XMM_SCD1	;; Next sine/cosine pointer
	loops	128, c2b		;; Loop 128 times
	bump	rsi, -128*64+4*dist128	;; Next source pointer
	bump	rdi, -128*9*XMM_SCD1	;; Reset sine/cosine pointer
	bump	rdx, -128*18*XMM_SCD1	;; Reset sine/cosine pointer
	loops	9, c2b			;; Test loop counter
	bump	rsi, -9*4*dist128	;; Restore source pointer
	end_timer 13

	start_timer 14
	loops_init_prefetch 1536, 64, 4, rcx ;; 8 iters of 2 iters of 128
	r3_x3cl_three_complex_djbunfft_preload
c7a:	xtouch	[rcx+4096-64]		;; Load prefetch TLBs
c7b:	r3_x3cl_three_complex_djbunfft rsi, 64, 4*dist128, rdi, 512*3*XMM_SCD1
	bump	rdi, 3*XMM_SCD1		;; Next sine/cosine pointer
	loops	128, c7b		;; Loop 128 times
	bump	rsi, -128*64+dist128	;; Next source pointer
	loops_touch 2, c7b, 2, c7a	;; Loop 4 times
	bump	rsi, -4*dist128+12*dist128 ;; Next source pointer
	bump	rdi, -512*3*XMM_SCD1	;; Reset sine/cosine pointer
	loops	3, c7b			;; Test loop counter
	bump	rsi, -3*12*dist128	;; Restore source pointer
	end_timer 14

	start_timer 15
	bump	rcx, -4*64		;; Back up prefetch pointer by 4 cache lines
	loops_init_prefetch 1536, 64, 4, rcx ;; 12 iters of 128
	r3_x3cl_three_complex_djbunfft_preload
c1a:	xtouch	[rcx+4096-64]		;; Load prefetch TLBs
c1b:	r3_x3cl_three_complex_djbunfft rsi, 64, 12*dist128, rdi, 1536*XMM_SCD1
	bump	rdi, XMM_SCD1		;; Next set of sin/cos data
	loops	128, c1b		;; Test inner loop counter
	bump	rsi, -128*64+dist128	;; Next source pointer
	loops_touch 2, c1b, 6, c1a	;; Test outer loop counter
	end_timer 15

ELSE					;; Pentium 4 faster doing merged loops

	start_timer 13
	mov	rdi, xsincos_complex	;; Load sin/cos pointer
	mov	rbp, rdi
	mov	rbx, rdi
	mov	rdx, rdi
	bump	rcx, -60*64		;; Back up prefetch pointer by 60 cache lines
	loops_init_prefetch 4608, 64, 12, rcx ;; 128 iters of 9 or 12

c2a:	xtouch	[rcx+4096-64]		;; Load prefetch TLBs
	xtouch	[rcx+2*4096-64]
	xtouch	[rcx+3*4096-64]
	xtouch	[rcx+4*4096-64]
	xtouch	[rcx+5*4096-64]

	loops_set_prefetch 128, 12, rcx	;; Fetch 768 cache lines in this loop
	r4_x4cl_2sc_four_complex_djbunfft_preload
c2b:	r4_x4cl_2sc_four_complex_djbunfft rsi, 4*dist128, dist128, 2*dist128, rbx, 128*9*XMM_SCD1, rdx, 128*18*XMM_SCD1
	loops	9, c2b, 4		;; Test loop counter with non-standard incr
	bump	rsi, -9*4*dist128	;; Restore source pointer
	loops_undo 36

	loops_set_prefetch 128, 4608, rcx ;; Fetch 0 cache lines in this loop
	r3_x3cl_three_complex_djbunfft_preload
c7b:	r3_x3cl_three_complex_djbunfft rsi, dist128, 4*dist128, rbp, 512*3*XMM_SCD1
	bump	rbp, 128*3*XMM_SCD1	;; Next sine/cosine pointer
	loops	4, c7b, 3		;; Test loop counter with non-standard incr
	bump	rsi, -4*dist128+12*dist128 ;; Next source pointer
	bump	rbp, -4*128*3*XMM_SCD1	;; Reset sine/cosine pointer
	loops	3, c7b			;; Test loop counter
	bump	rsi, -3*12*dist128	;; Restore source pointer
	loops_undo 36

	loops_set_prefetch 64, 9, rcx	;; Fetch 512 cache lines in this loop
c1b:	r3_x3cl_three_complex_djbunfft rsi, dist128, 12*dist128, rdi, 1536*XMM_SCD1
	bump	rdi, 128*XMM_SCD1	;; Next sin/cos ptr
	loops	12, c1b, 3		;; Test loop counter with non-standard incr 

	bump	rsi, -12*dist128+64	;; Next source pointer
	bump	rdi, -12*128*XMM_SCD1+XMM_SCD1 ;; Next sin/cos ptr
	bump	rbp, 3*XMM_SCD1		;; Next sine/cosine pointer
	bump	rbx, 9*XMM_SCD1		;; Next sine/cosine pointer
	bump	rdx, 18*XMM_SCD1	;; Next sine/cosine pointer
	loops_touch 32, c2b, 4, c2a	;; Loop 128 times
	end_timer 13

ENDIF

xpass2_9216_levels_complex_done:
	ENDM


; *************** radix4-pass2-12288-levels-real macro ******************
; This macro takes 24576 real values and performs the final 14.585 levels of
; the FFT.  Independently, this macro also takes 12288 complex values and
; performs the final 13.585 levels of the FFT.  The results are squared, and
; the first 13.585/14.585 levels of the inverse FFT are done.

r4_pass2_12288_levels_real MACRO
	LOCAL	b1b, d2b, d3b, b2b0, b2b, b3b0, b3b, b4b, b5a, b5b, b6b, b7b
	LOCAL	b8b, b9b, baa, bab0, bab, bac, bad0, bad
	LOCAL	c1b, e2b, e3b, c2b, c3b, c4b, c5a, c5b, c6b, c7a, c7b, c8b, c9b
	LOCAL	xpass2_12288_levels_real_1, xpass2_12288_levels_real_2
	LOCAL	xpass2_12288_levels_real_3, xpass2_12288_levels_real_4
	LOCAL	xpass2_12288_real_unfft, xpass2_12288_real_done

;; We switch to the same format used in one-pass FFTs (that is, the
;; high word of xmm registers being 1 greater than the low word):

;; Type 4 FFTs skip the forward FFT process

	start_timer 2
	mov	rsi, DATA_ADDR		;; Load source address
	mov	rbx, DIST_TO_FFTSRCARG
	cmp	ffttype, 4
	je	xpass2_12288_levels_real_4

;; Do FFT level 1,2 (actually 1.585 levels)
;;
;; On input the 64-byte cache lines hold these data values if pass 2 swizzles:
;;	0	+6K	12288	+6K	24576	+6K	36864	+6K
;;	1	...
;;	...
;;	6143	...
;; On input the 64-byte cache lines hold these data values if pass 1 swizzles:
;;	0	+24K	6144	+24K	12288	+24K	18432	+24K
;;	1	...
;;	...
;;	6143	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+24K	2048	+24K	4096	+24K	6144	+24K
;;	1	...
;;	...
;;	2047	...
;;	8192	...
;;	...

;; Do 4096 six_reals_fft operations
;; Do 4096 three_complex_djbfft operations
;;	distance between fft data elements is 4096
;;	do 2048 iterations

	mov	rdi, xsincos_complex	;; Ptr to the sin/cos data
	mov	rdx, sincos3		;; Ptr to real sin/cos data
	loops_init 2048
b1b:	r3_fh3cl_six_reals_three_complex_djbfft rsi, 64, 16*dist128, rdi, 2048*XMM_SCD1, rdx, 2048*XMM_HSCD1
	bump	rdi, XMM_SCD1		;; Next sin/cos data
	bump	rdx, XMM_HSCD1		;; Next sin/cos data
	loops	128, b1b		;; Test loop counter
	bump	rsi, -128*64+dist128	;; Next source pointer
	loops	16, b1b			;; Test loop counter
	bump	rsi, -16*dist128	;; Restore source pointer
	bump	rdi, -2048*XMM_SCD1	;; Restore sin/cos ptr
	bump	rdx, -2048*XMM_HSCD1+4096*XMM_HSCD1 ;; Next sin/cos ptr

;; Do FFT levels 3,4
;;
;; On input the 64-byte cache lines hold these data values:
;;	0	+24K	2048	+24K	4096	+24K	6144	+24K
;;	1	...
;;	...
;;	2047	...
;;	8192	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+24K	1024	+24K	2048	+24K	3072	+24K
;;	1	...
;;	...
;;	1023	...
;;	4096	...
;;	...

;; Do 1024 eight_reals_fft operations
;; Do 1024 four_complex_djbfft operations
;;	distance between fft data elements is 1024
;;	do 1024 macros each processing 16 data values

	mov	rcx, DATA_PREFETCH	;; Prefetch pointer
	xtouch	[rcx]			;; Preload first TLB
	mov	rbx, rdi
	loops_init 3072
d2b:	r4_h2cl_2sc_eight_reals_four_complex_djbfft rsi, 64, 8*dist128, rdi, rbx, rdx
	bump	rdi, 3*XMM_SCD1		;; Next sin/cos data
	bump	rbx, 6*XMM_SCD1		;; Next sin/cos data
	bump	rdx, XMM_HSCD2		;; Next sin/cos data
	loops	128, d2b		;; Test inner loop counter
	bump	rsi, -128*64+dist128	;; Next source pointer
	loops	8, d2b			;; Test loop counter
	bump	rsi, -8*dist128+16*dist128 ;; Next source pointer
	bump	rdi, -1024*3*XMM_SCD1	;; Restore sin/cos ptr
	bump	rbx, -1024*6*XMM_SCD1	;; Restore sin/cos ptr
	bump	rdx, -1024*XMM_HSCD2	;; Restore sin/cos ptr

;; Do 4096 four_complex_fft operations
;;	distance between fft data elements is 1024
;;	do 2048 macros each processing 16 data values

	loops_reset			;; 2 more iters of 1024
	r4_x2cl_2sc_four_complex_djbfft_preload
d3b:	r4_x2cl_2sc_four_complex_djbfft rsi, 64, 8*dist128, rdi, rbx
	bump	rdi, 3*XMM_SCD1		;; Next sin/cos data
	bump	rbx, 6*XMM_SCD1		;; Next sin/cos data
	loops	128, d3b		;; Test inner loop counter
	bump	rsi, -128*64+dist128	;; Next source pointer
	loops	8, d3b			;; Test loop counter
	bump	rsi, -8*dist128+16*dist128 ;; Next source pointer
	bump	rdi, -1024*3*XMM_SCD1	;; Restore sin/cos ptr
	bump	rbx, -1024*6*XMM_SCD1	;; Restore sin/cos ptr
	loops	3, d3b			;; Test loop counter
	bump	rsi, -3*16*dist128	;; Restore source pointer

;; Do FFT levels 5,6
;;
;; On input the 64-byte cache lines hold these data values:
;;	0	+24K	1024	+24K	2048	+24K	3072	+24K
;;	1	...
;;	...
;;	1023	...
;;	4096	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+24K	256	+24K	512	+24K	768	+24K
;;	1	...
;;	...
;;	255	...
;;	1024	...
;;	...

;; Do 256 eight_reals_fft operations
;; Do 768 four_complex_djbfft operations
;;	distance between fft data elements is 256
;;	do 256 macros each processing 32 data values

	loops_init_prefetch 1536, 64, 1, rcx
	r4_h4cl_2sc_eight_reals_four_complex_djbfft_preload
b2b0:	xtouch	[rcx+4096-64]		;; Preload the TLB
b2b:	r4_h4cl_2sc_eight_reals_four_complex_djbfft rsi, 64, 2*dist128, 4*dist128, rdi, rbx, rdx
	bump	rdi, 12*XMM_SCD1	;; Next sin/cos data
	bump	rbx, 24*XMM_SCD1	;; Next sin/cos data
	bump	rdx, 4*XMM_HSCD2	;; Next sin/cos data
	loops_touch 64, b2b, 2, b2b0	;; Test inner loop counter
	bump	rsi, -128*64+dist128	;; Next source pointer
	loops	2, b2b0			;; Test loop counter
	bump	rsi, -2*dist128+8*dist128 ;; Next source pointer
	bump	rdi, -256*12*XMM_SCD1	;; Restore sin/cos ptr
	bump	rbx, -256*24*XMM_SCD1	;; Restore sin/cos ptr
	bump	rdx, -256*4*XMM_HSCD2	;; Restore sin/cos ptr

;; Do 5120 four_complex_fft operations
;;	distance between fft data elements is 256
;;	do 1280 macros each processing 32 data values

	loops_reset
	r4_x4cl_2sc_four_complex_djbfft_preload
b3b0:	xtouch	[rcx+4096-64]		;; Preload the TLB
b3b:	r4_x4cl_2sc_four_complex_djbfft rsi, 64, 2*dist128, 4*dist128, rdi, rbx
	bump	rdi, 12*XMM_SCD1	;; Next sin/cos data
	bump	rbx, 24*XMM_SCD1	;; Next sin/cos data
	loops_touch 64, b3b, 2, b3b0	;; Test inner loop counter
	bump	rsi, -128*64+dist128	;; Next source pointer
	loops	2, b3b0			;; Test loop counter
	bump	rsi, -2*dist128+8*dist128 ;; Next source pointer
	bump	rdi, -256*12*XMM_SCD1	;; Restore sin/cos ptr
	bump	rbx, -256*24*XMM_SCD1	;; Restore sin/cos ptr
	loops	6, b3b0			;; Test loop counter
	bump	rsi, -6*8*dist128	;; Restore source pointer

;; Do FFT levels 7,8
;;
;; On input the 64-byte cache lines hold these data values:
;;	0	+24K	256	+24K	512	+24K	768	+24K
;;	1	...
;;	...
;;	255	...
;;	1024	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+24K	64	+24K	128	+24K	192	+24K
;;	1	...
;;	...
;;	63	...
;;	256	...
;;	...

;; Do 64 eight_reals_fft operations
;; Do 192 four_complex_djbfft operations
;;	distance between fft data elements is 64
;;	do 64 macros each processing 32 data values

	bump	rdi, 6144*XMM_SCD1	;; Load sin/cos pointer
	loops_init_prefetch 1536, 64, 1, rcx
	xtouch	[rcx+4096-64]		;; Preload the TLB
	r4_h4cl_eight_reals_four_complex_djbfft_preload
b4b:	r4_h4cl_eight_reals_four_complex_djbfft rsi, 64, 64*64, dist128, rdi, rdx
	bump	rdi, XMM_SCD2		;; Next sin/cos data
	bump	rdx, 16*XMM_HSCD2	;; Next sin/cos data
	loops	64, b4b			;; Test loop counter
	bump	rsi, -64*64+2*dist128	;; Next source pointer
	bump	rdi, -64*XMM_SCD2	;; Restore sin/cos ptr
	bump	rdx, -64*16*XMM_HSCD2	;; Restore sin/cos ptr

;; Do 5888 four_complex_djbfft operations
;;	distance between fft data elements is 64
;;	do 1472 macros each processing 32 data values

	loops_reset			;; 23 more iterations of 64
	r4_x4cl_four_complex_djbfft_preload
b5a:	xtouch	[rcx+4096-64]		;; Preload the TLB
b5b:	r4_x4cl_four_complex_djbfft rsi, 64, 64*64, dist128, rdi
	bump	rdi, XMM_SCD2		;; Next sin/cos data
	loops	64, b5b			;; Test inner loop counter
	bump	rsi, -64*64+2*dist128	;; Next source pointer
	bump	rdi, -64*XMM_SCD2	;; Restore sin/cos ptr
	loops	24, b5a			;; Test loop counter
	bump	rsi, -24*2*dist128	;; Restore source pointer

;; Do FFT levels 9,10
;;
;; On input the 64-byte cache lines hold these data values:
;;	0	+24K	64	+24K	128	+24K	192	+24K
;;	1	...
;;	...
;;	63	...
;;	256	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+24K	16	+24K	32	+24K	48	+24K
;;	1	...
;;	...
;;	15	...
;;	64	...
;;	...

;; Do 16 eight_reals_fft operations
;; Do 48 four_complex_djbfft operations
;;	distance between fft data elements is 16
;;	do 16 macros each processing 32 data values

	loops_init 1536
	r4_h4cl_eight_reals_four_complex_djbfft_preload
b6b:	r4_h4cl_eight_reals_four_complex_djbfft rsi, 64, 16*64, 32*64, rdi, rdx
	bump	rdi, 4*XMM_SCD2		;; Next sin/cos data
	bump	rdx, 64*XMM_HSCD2	;; Next sin/cos data
	loops	16, b6b			;; Test loop counter
	bump	rsi, -16*64+64*64	;; Next source pointer
	bump	rdi, -16*4*XMM_SCD2	;; Restore sin/cos ptr
	bump	rdx, -16*64*XMM_HSCD2	;; Restore sin/cos ptr

;; Do 6080 four_complex_djbfft operations
;;	distance between fft data elements is 16
;;	do 1520 macros each processing 32 data values

	loops_reset			;; 48 iters of 1 or 2 of 16
	r4_x4cl_four_complex_djbfft_preload
b7b:	r4_x4cl_four_complex_djbfft rsi, 64, 16*64, 32*64, rdi
	bump	rdi, 4*XMM_SCD2		;; Next sin/cos data
	loops	16, b7b			;; Test inner loop counter
	bump	rsi, -16*64+64*64	;; Next source pointer
	bump	rdi, -16*4*XMM_SCD2	;; Restore sin/cos ptr
	loops	2, b7b			;; Test loop counter
	bump	rsi, -2*64*64+dist128	;; Next source pointer
	loops	48, b7b			;; 48 iterations
	bump	rsi, -48*dist128	;; Next source pointer

;; Do FFT levels 11,12
;;
;; On input the 64-byte cache lines hold these data values:
;;	0	+24K	16	+24K	32	+24K	48	+24K
;;	1	...
;;	...
;;	15	...
;;	64	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+24K	4	+24K	8	+24K	12	+24K
;;	1	...
;;	...
;;	3	...
;;	16	...
;;	...

;; Do 4 eight_reals_fft operations
;; Do 12 four_complex_djbfft operations
;;	distance between fft data elements is 4
;;	do 4 macros each processing 32 data values

	loops_init 1536
	r4_h4cl_eight_reals_four_complex_djbfft_preload
b8b:	r4_h4cl_eight_reals_four_complex_djbfft rsi, 64, 4*64, 8*64, rdi, rdx
	bump	rdi, 16*XMM_SCD2	;; Next sin/cos data
	bump	rdx, 256*XMM_HSCD2	;; Next sin/cos data
	loops	4, b8b			;; Test loop counter
	bump	rsi, -4*64+16*64	;; Next source pointer
	bump	rdi, -4*16*XMM_SCD2	;; Restore sin/cos ptr
	bump	rdx, -4*256*XMM_HSCD2	;; Restore sin/cos ptr

;; Do 6128 four_complex_djbfft operations
;;	distance between fft data elements is 4
;;	do 1532 macros each processing 32 data values

	loops_reset			;; 48 iters of 7 or 8 of 4
	r4_x4cl_four_complex_djbfft_preload
b9b:	r4_x4cl_four_complex_djbfft rsi, 64, 4*64, 8*64, rdi
	bump	rdi, 16*XMM_SCD2	;; Next sin/cos data
	loops	4, b9b			;; Test loop counter
	bump	rsi, -4*64+16*64	;; Next source pointer
	bump	rdi, -4*16*XMM_SCD2	;; Restore sin/cos ptr
	loops	8, b9b			;; Test middle loop counter
	bump	rsi, -8*16*64+dist128	;; Next source pointer
	loops	48, b9b			;; 48 iterations
	bump	rsi, -48*dist128	;; Restore source pointer

;; Do FFT levels 13,14
;;
;; On input the 64-byte cache lines hold these data values:
;;	0	+24K	4	+24K	8	+24K	12	+24K
;;	1	...
;;	...
;;	3	...
;;	16	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+24K	2	+24K	4	+24K	6	+24K
;;	1	...
;;	8	...
;;	...

;; Do 1 eight_reals_fft operations
;; Do 3 four_complex_fft operations
;;	distance between fft data elements is 1

;; Do 6140 four_complex_fft operations
;;	distance between fft data elements is 1
;;	do 1535 macros each processing 32 data values

;; Execute the proper middle step

	cmp	ffttype, 2
	jl	xpass2_12288_levels_real_1
	jg	xpass2_12288_levels_real_3

xpass2_12288_levels_real_2:
	r4_h4cl_eight_reals_four_complex_with_square rsi, 4*64, 64, 2*64
	loops_init 1536, 1, 32
	r4_x4cl_four_complex_with_square_preload
baa:	r4_x4cl_four_complex_with_square rsi, 4*64, 64, 2*64
	loops	32, baa			;; Test loop counter
	bump	rsi, -32*4*64+dist128	;; Next source pointer
	loops	48, baa			;; Test loop counter
	bump	rsi, -48*dist128	;; Restore source pointer
	jmp	xpass2_12288_real_unfft

xpass2_12288_levels_real_1:
	r4_h4cl_eight_reals_four_complex_fft_final rsi, 4*64, 64, 2*64
	L2prefetch128 [rcx]
	bump	rcx, 128
	loops_init_prefetch 1536, 128, 1, rcx, 1, 32
bab0:	xtouch	[rcx+4096-64]		;; Preload the TLBs
bab:	r4_x4cl_four_complex_fft_final rsi, 4*64, 64, 2*64
	loops	32, bab			;; Test loop counter
	bump	rsi, -32*4*64+dist128	;; Next source pointer
	loops	48, bab0		;; Test loop counter
	jmp	xpass2_12288_real_done

xpass2_12288_levels_real_3:
	mov	rbp, DIST_TO_MULSRCARG
	r4_h4cl_eight_reals_four_complex_with_mult rsi, 4*64, 64, 2*64
	loops_init 1536, 1, 32
bac:	r4_x4cl_four_complex_with_mult rsi, 4*64, 64, 2*64
	loops	32, bac			;; Test loop counter
	bump	rsi, -32*4*64+dist128	;; Next source pointer
	loops	48, bac			;; Test loop counter
	bump	rsi, -48*dist128	;; Restore source pointer
	jmp	xpass2_12288_real_unfft

xpass2_12288_levels_real_4:
	mov	rbp, DIST_TO_MULSRCARG
	mov	rcx, DATA_PREFETCH	;; Load prefetch pointer
	xtouch	[rcx]			;; Preload the TLBs
	r4_h4cl_eight_reals_four_complex_with_mulf rsi, 4*64, 64, 2*64
	L2prefetch128 [rcx]
	bump	rcx, 128
	loops_init_prefetch 1536, 128, 1, rcx, 1, 32
bad0:	xtouch	[rcx+4096-64]		;; Preload the TLBs
bad:	r4_x4cl_four_complex_with_mulf rsi, 4*64, 64, 2*64
	loops	32, bad			;; Test loop counter
	bump	rsi, -32*4*64+dist128	;; Next source pointer
	loops	48, bad0		;; Test loop counter
	bump	rsi, -48*dist128	;; Restore source pointer

;; Do inverse FFT levels 11,12
;; On input the 64-byte cache lines hold these data values:
;;	0	+24K	2	+24K	4	+24K	6	+24K
;;	1	...
;;	8	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+24K	8	+24K	16	+24K	24	+24K
;;	1	...
;;	...
;;	7	...
;;	32	...
;;	...

;; Do 4 eight_reals_unfft operations
;; Do 4 four_complex_djbunfft operations
;;	distance between fft data elements is 4
;;	do 2 macros processing 32 data values

xpass2_12288_real_unfft:
	mov	rdi, xsincos_complex	;; Load sin/cos pointers
	bump	rdi, 6144*XMM_SCD1
	mov	rdx, sincos3
	bump	rdx, 4096*XMM_HSCD1
	loops_init 1536
c8b:	r4_h4cl_eight_reals_four_complex_djbunfft rsi, 64, 2*64, 4*64, rdi, 2*16*XMM_SCD2, rdx, 2*256*XMM_HSCD2
	bump	rdi, 16*XMM_SCD2	;; Next sin/cos data
	bump	rdx, 256*XMM_HSCD2	;; Next sin/cos data
	loops	2, c8b			;; Test loop counter
	bump	rsi, -2*64+8*64		;; Next source pointer
	bump	rdi, -2*16*XMM_SCD2	;; Restore sin/cos ptr
	bump	rdx, -2*256*XMM_HSCD2	;; Restore sin/cos ptr

;; Do 6136 four_complex_djbunfft operations
;;	distance between fft data elements is 4
;;	do 1534 macros each processing 32 data values

	loops_reset			;; 48 iters of 15 or 16 of 2
	r4_x4cl_four_complex_djbunfft_preload
c9b:	r4_x4cl_four_complex_djbunfft rsi, 64, 2*64, 4*64, rdi, 2*16*XMM_SCD2
	bump	rdi, 16*XMM_SCD2	;; Next sine/cosine pointer
	loops	2, c9b			;; Test loop counter
	bump	rsi, -2*64+8*64		;; Next source pointer
	bump	rdi, -2*16*XMM_SCD2	;; Restore sin/cos ptr
	loops	16, c9b			;; Test loop counter
	bump	rsi, -16*8*64+dist128	;; Next source pointer
	loops	48, c9b			;; Test loop counter
	bump	rsi, -48*dist128	;; Restore source pointer

;; Do inverse FFT levels 9,10
;; On input the 64-byte cache lines hold these data values:
;;	0	+24K	8	+24K	16	+24K	24	+24K
;;	1	...
;;	...
;;	7	...
;;	32	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+24K	32	+24K	64	+24K	96	+24K
;;	1	...
;;	...
;;	31	...
;;	128	...
;;	...

;; Do 16 eight_reals_unfft operations
;; Do 16 four_complex_djbunfft operations
;;	distance between fft data elements is 16
;;	do 8 macros processing 32 data values

	loops_init_prefetch 1536, 64, 1, rcx
	xtouch	[rcx+4096-64]		;; Preload the TLB
c6b:	r4_h4cl_eight_reals_four_complex_djbunfft rsi, 64, 8*64, 16*64, rdi, 8*4*XMM_SCD2, rdx, 8*64*XMM_HSCD2
	bump	rdi, 4*XMM_SCD2		;; Next sin/cos data
	bump	rdx, 64*XMM_HSCD2	;; Next sin/cos data
	loops	8, c6b			;; Test inner loop counter
	bump	rsi, -8*64+32*64	;; Next source pointer
	bump	rdi, -8*4*XMM_SCD2	;; Restore sin/cos ptr
	bump	rdx, -8*64*XMM_HSCD2	;; Restore sin/cos ptr

;; Do 6112 four_complex_djbunfft operations
;;	distance between fft data elements is 16
;;	do 1528 macros each processing 32 data values

	loops_reset			;; 48 iters of 3 or 4 of 4
	r4_x4cl_four_complex_djbunfft_preload
c7a:	xtouch	[rcx+4096-64]		;; Preload the TLB
c7b:	r4_x4cl_four_complex_djbunfft rsi, 64, 8*64, 16*64, rdi, 8*4*XMM_SCD2
	bump	rdi, 4*XMM_SCD2		;; Next sin/cos data
	loops	8, c7b			;; Test inner loop counter
	bump	rsi, -8*64+32*64	;; Next source pointer
	bump	rdi, -8*4*XMM_SCD2	;; Restore sin/cos ptr
	loops	4, c7b			;; Test loop counter
	bump	rsi, -4*32*64+dist128	;; Next source pointer
	loops_touch 2, c7b, 24, c7a	;; Test loop counter
	bump	rsi, -48*dist128	;; Restore source pointer

;; Do inverse FFT levels 7,8
;; On input the 64-byte cache lines hold these data values:
;;	0	+24K	32	+24K	64	+24K	96	+24K
;;	1	...
;;	...
;;	31	...
;;	128	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+24K	128	+24K	256	+24K	384	+24K
;;	1	...
;;	...
;;	127	...
;;	512	...
;;	...

;; Do 64 eight_reals_unfft operations
;; Do 64 four_complex_djbunfft operations
;;	distance between fft data elements is 64
;;	do 32 macros processing 32 data values

	loops_init_prefetch 1536, 64, 1, rcx
	xtouch	[rcx+4096-64]		;; Preload the TLBs
c4b:	r4_h4cl_eight_reals_four_complex_djbunfft rsi, 64, 32*64, 64*64, rdi, 32*XMM_SCD2, rdx, 32*16*XMM_HSCD2
	bump	rdi, XMM_SCD2		;; Next sin/cos data
	bump	rdx, 16*XMM_HSCD2	;; Next sin/cos data
	loops	32, c4b			;; Test inner loop counter
	bump	rsi, -32*64+dist128	;; Next source pointer
	bump	rdi, -32*XMM_SCD2	;; Restore sin/cos ptr
	bump	rdx, -32*16*XMM_HSCD2	;; Restore sin/cos ptr

;; Do 6016 four_complex_djbunfft operations
;;	distance between fft data elements is 64
;;	do 1504 macros each processing 32 data values

	loops_reset			;; 47 more iters of 32
	r4_x4cl_four_complex_djbunfft_preload
c5a:	xtouch	[rcx+4096-64]		;; Preload the TLBs
c5b:	r4_x4cl_four_complex_djbunfft rsi, 64, 32*64, 64*64, rdi, 32*XMM_SCD2
	bump	rdi, XMM_SCD2		;; Next sin/cos data
	loops	32, c5b			;; Test inner loop counter
	bump	rsi, -32*64+dist128	;; Next source pointer
	bump	rdi, -32*XMM_SCD2	;; Restore sin/cos ptr
	loops_touch 2, c5b, 24, c5a	;; Test loop counter
	bump	rsi, -48*dist128	;; Next source pointer

;; Do inverse FFT levels 5,6
;; On input the 64-byte cache lines hold these data values:
;;	0	+24K	128	+24K	256	+24K	384	+24K
;;	1	...
;;	...
;;	127	...
;;	512	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+24K	512	+24K	1024	+24K	1536	+24K
;;	1	...
;;	...
;;	511	...
;;	2048	...
;;	...

;; Do 256 eight_reals_unfft operations
;; Do 256 four_complex_djbunfft operations
;;	distance between fft data elements is 256
;;	do 128 macros processing 32 data values

	mov	rdi, xsincos_complex	;; Load sin/cos pointer
	mov	rbx, rdi
	bump	rcx, -2*64		;; Back up prefetch pointer by 2 cache lines
	loops_init_prefetch 1536, 64, 16, rcx
	xtouch	[rcx+4096-64]		;; Preload the TLBs
	xtouch	[rcx+96*64-64]
c2b:	r4_h4cl_2sc_eight_reals_four_complex_djbunfft rsi, 64, dist128, 2*dist128, rdi, 128*12*XMM_SCD1, rbx, 128*24*XMM_SCD1, rdx, 128*4*XMM_HSCD2
	bump	rdi, 12*XMM_SCD1	;; Next sin/cos data
	bump	rbx, 24*XMM_SCD1	;; Next sin/cos data
	bump	rdx, 4*XMM_HSCD2	;; Next sin/cos data
	loops	128, c2b		;; Test inner loop counter
	bump	rsi, -128*64+4*dist128	;; Next source pointer
	bump	rdi, -128*12*XMM_SCD1	;; Restore sin/cos ptr
	bump	rbx, -128*24*XMM_SCD1	;; Restore sin/cos ptr
	bump	rdx, -128*4*XMM_HSCD2	;; Restore sin/cos ptr

;; Do 5632 four_complex_djbunfft operations
;;	distance between fft data elements is 256
;;	do 1408 macros each processing 32 data values

	loops_reset			;; 11 more iters of 128
	r4_x4cl_2sc_four_complex_djbunfft_preload
c3b:	r4_x4cl_2sc_four_complex_djbunfft rsi, 64, dist128, 2*dist128, rdi, 128*12*XMM_SCD1, rbx, 128*24*XMM_SCD1
	bump	rdi, 12*XMM_SCD1	;; Next sin/cos data
	bump	rbx, 24*XMM_SCD1	;; Next sin/cos data
	loops	128, c3b		;; Test inner loop counter
	bump	rsi, -128*64+4*dist128	;; Next source pointer
	bump	rdi, -128*12*XMM_SCD1	;; Restore sin/cos ptr
	bump	rbx, -128*24*XMM_SCD1	;; Restore sin/cos ptr
	loops	12, c3b			;; Test loop counter
	bump	rsi, -12*4*dist128	;; Next source pointer

;; Do inverse FFT levels 3,4
;; On input the 64-byte cache lines hold these data values:
;;	0	+24K	512	+24K	1024	+24K	1536	+24K
;;	1	...
;;	...
;;	511	...
;;	2048	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+24K	2048	+24K	4096	+24K	6144	+24K
;;	1	...
;;	...
;;	2047	...
;;	8192	...
;;	...

;; Do 1024 eight_reals_unfft operations
;; Do 1024 four_complex_djbunfft operations
;;	distance between fft data elements is 1024
;;	do 512 macros processing 32 data values

	loops_init 1536
e2b:	r4_h4cl_2sc_eight_reals_four_complex_djbunfft rsi, 64, 4*dist128, 8*dist128, rdi, 512*3*XMM_SCD1, rbx, 512*6*XMM_SCD1, rdx, 512*XMM_HSCD2
	bump	rdi, 3*XMM_SCD1		;; Next sin/cos data
	bump	rbx, 6*XMM_SCD1		;; Next sin/cos data
	bump	rdx, XMM_HSCD2		;; Next sin/cos data
	loops	128, e2b		;; Test inner loop counter
	bump	rsi, -128*64+dist128	;; Next source pointer
	loops	4, e2b			;; Test loop counter
	bump	rsi, -4*dist128+16*dist128 ;; Next source pointer
	bump	rdi, -512*3*XMM_SCD1	;; Restore sin/cos ptr
	bump	rbx, -512*6*XMM_SCD1	;; Restore sin/cos ptr
	bump	rdx, -512*XMM_HSCD2-4096*XMM_HSCD1 ;; Next sin/cos ptr

;; Do 4096 four_complex_djbunfft operations
;;	distance between fft data elements is 1024
;;	do 1024 macros each processing 32 data values

	loops_reset			;; 2 more iters of 512
	r4_x4cl_2sc_four_complex_djbunfft_preload
e3b:	r4_x4cl_2sc_four_complex_djbunfft rsi, 64, 4*dist128, 8*dist128, rdi, 512*3*XMM_SCD1, rbx, 512*6*XMM_SCD1
	bump	rdi, 3*XMM_SCD1		;; Next sin/cos data
	bump	rbx, 6*XMM_SCD1		;; Next sin/cos data
	loops	128, e3b		;; Test inner loop counter
	bump	rsi, -128*64+dist128	;; Next source pointer
	loops	4, e3b			;; Test loop counter
	bump	rsi, -4*dist128+16*dist128 ;; Next source pointer
	bump	rdi, -512*3*XMM_SCD1	;; Restore sin/cos ptr
	bump	rbx, -512*6*XMM_SCD1	;; Restore sin/cos ptr
	loops	3, e3b			;; Test loop counter
	bump	rsi, -3*16*dist128	;; Restore source pointer

;; Do inverse FFT levels 1,2 (actually 1.585 FFT levels)
;; On input the 64-byte cache lines hold these data values:
;;	0	+24K	2048	+24K	4096	+24K	6144	+24K
;;	1	...
;;	...
;;	2047	...
;;	8192	...
;;	...
;; On output the 64-byte cache lines hold these data values if pass 1 swizzles:
;;	0	+24K	6144	+24K	12288	+24K	18432	+24K
;;	1	...
;;	...
;;	6143	...
;; On output the 64-byte cache lines hold these data values if pass 2 swizzles:
;;	0	+6K	12288	+6K	24576	+6K	36864	+6K
;;	1	...
;;	...
;;	6143	...

;; Do 4096 six_reals_unfft operations
;; Do 4096 three_complex_djbunfft operations
;;	distance between fft data elements is 4096
;;	do 2048 macros processing 24 data values

	loops_init 2048
c1b:	r3_h3cl_six_reals_three_complex_djbunfft rsi, 64, 16*dist128, rdi, 2048*XMM_SCD1, rdx, 2048*XMM_HSCD1
	bump	rdi, XMM_SCD1		;; Next sin/cos data
	bump	rdx, XMM_HSCD1		;; Next sin/cos data
	loops	128, c1b		;; Test loop counter
	bump	rsi, -128*64+dist128	;; Next source pointer
	loops	16, c1b			;; Test loop counter

xpass2_12288_real_done:
	end_timer 2
	ENDM

; *************** radix4-pass2-12288-levels-complex macro ******************
; This macro takes 2 groups of 12288 complex values and
; performs the final 13.585 levels of a standard radix-4 FFT, squares the
; results, and does the first 13.585 levels of the inverse FFT.

r4_pass2_12288_levels_complex MACRO
	LOCAL	b1b, b7b, b2a, b2b, b3a, b3b, b4a, b4b, b5a, b5b
	LOCAL	b6a, b6b0, b6b, b6c, b6d0, b6d
	LOCAL	c1b, c7b, c2a, c2b, c3a, c3b, c4a, c4b, c5a, c5b
	LOCAL	xpass2_12288_levels_complex_1, xpass2_12288_levels_complex_2
	LOCAL	xpass2_12288_levels_complex_3, xpass2_12288_levels_complex_4
	LOCAL	xpass2_12288_levels_complex_unfft, xpass2_12288_levels_complex_done

;; Type 4 FFTs skip the forward FFT process

	mov	rsi, DATA_ADDR		;; Load address of FFT data
	mov	rbx, DIST_TO_FFTSRCARG
	cmp	ffttype, 4
	je	xpass2_12288_levels_complex_4

;; Do FFT level 1,2 (actually 1.585 levels)
;;
;; We swizzle the data so that one set of data is in the high word of
;; an xmm register, the other set is in the low word.  This means
;; a distance of 6144 data values.
;;
;; On input the 64-byte cache lines hold these data values if pass 2 swizzles:
;;	0	+6K	12288	+6K	24576	+6K	36864	+6K
;;	1	...
;;	...
;;	6143	...
;; On input the 64-byte cache lines hold these data values if pass 1 swizzles:
;;	0	+24K	6144	+24K	12288	+24K	18432	+24K
;;	1	...
;;	...
;;	6143	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+24K	2048	+24K	4096	+24K	6144	+24K
;;	1	...
;;	...
;;	2047	...
;;	8192	...
;;	...

;; Do 8192 three_complex_fft operations
;;	distance between fft data elements is 4096
;;	do 2048 macros each processing 24 data values

	start_timer 3
	mov	rdi, xsincos_complex	;; The sin/cos data
	loops_init 2048			;; 16 iterations of 128
	r3_f3cl_three_complex_djbfft_preload
b1b:	xxprefetch [rdi+XMM_SCD1]	;; Prefetch for next pass
	xxprefetch [rdi+2049*XMM_SCD1]	;; Prefetch for next pass
	r3_f3cl_three_complex_djbfft rsi, 64, 16*dist128, rdi, 2048*XMM_SCD1
	bump	rdi, XMM_SCD1		;; Next sin/cos data
	loops	128, b1b		;; Test inner loop counter
	bump	rsi, -128*64+dist128	;; Next source pointer
	loops	16, b1b			;; Test outer loop counter
	bump	rsi, -16*dist128	;; Restore source pointer
	end_timer 3

;; Do FFT levels 3,4
;;
;; On input the 64-byte cache lines hold these data values:
;;	0	+24K	2048	+24K	4096	+24K	6144	+24K
;;	1	...
;;	...
;;	2047	...
;;	8192	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+24K	1024	+24K	2048	+24K	3072	+24K
;;	1	...
;;	...
;;	1023	...
;;	4096	...
;;	...

;; Do 6144 four_complex_fft operations
;;	distance between fft data elements is 1024
;;	do 3072 macros each processing 16 data values

	start_timer 4
	mov	rcx, DATA_PREFETCH	;; Prefetch pointer (we need to prefetch 
					;; 6144 cache lines + 94 pad cache lines)
	xtouch	[rcx]			;; Load first prefetch TLB
	mov	rdi, xsincos_complex	;; Ptr to the sin/cos data
	mov	rbx, rdi
	loops_init 3072			;; 3 iters of 8 iters of 128
	r4_x2cl_2sc_four_complex_djbfft_preload
b7b:	r4_x2cl_2sc_four_complex_djbfft rsi, 64, 8*dist128, rdi, rbx
	bump	rdi, 3*XMM_SCD1		;; Next sin/cos data
	bump	rbx, 6*XMM_SCD1		;; Next sin/cos data
	loops	128, b7b		;; Test inner loop counter
	bump	rsi, -128*64+dist128	;; Next source pointer
	loops	8, b7b			;; Test inner loop counter
	bump	rsi, -8*dist128+16*dist128 ;; Next source pointer
	bump	rdi, -1024*3*XMM_SCD1	;; Restore sin/cos ptr
	bump	rbx, -1024*6*XMM_SCD1	;; Restore sin/cos ptr
	loops	3, b7b			;; Test loop counter
	bump	rsi, -3*16*dist128	;; Restore source pointer
	end_timer 4

;; Do FFT levels 5,6
;;
;; On input the 64-byte cache lines hold these data values:
;;	0	+24K	1024	+24K	2048	+24K	3072	+24K
;;	1	...
;;	...
;;	1023	...
;;	4096	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+24K	256	+24K	512	+24K	768	+24K
;;	1	...
;;	...
;;	255	...
;;	1024	...
;;	...

;; Do 6144 four_complex_fft operations
;;	distance between fft data elements is 256
;;	do 1536 macros each processing 16 data values

	start_timer 5
	loops_init_prefetch 1536, 64, 2, rcx ;; 6 iters of 2 iters of 128
	r4_x4cl_2sc_four_complex_djbfft_preload
b2a:	xtouch	[rcx+4096-64]		;; Load prefetch TLB
b2b:	r4_x4cl_2sc_four_complex_djbfft rsi, 64, 2*dist128, 4*dist128, rdi, rbx
	bump	rdi, 12*XMM_SCD1	;; Next sin/cos data
	bump	rbx, 24*XMM_SCD1	;; Next sin/cos data
	loops	128, b2b		;; Test inner loop counter
	bump	rsi, -128*64+dist128	;; Next source pointer
	loops	2, b2a			;; Test loop counter
	bump	rsi, -2*dist128+8*dist128 ;; Next source pointer
	bump	rdi, -256*12*XMM_SCD1	;; Restore sin/cos ptr
	bump	rbx, -256*24*XMM_SCD1	;; Restore sin/cos ptr
	loops	6, b2a			;; Test loop counter
	bump	rsi, -6*8*dist128	;; Restore source pointer
	end_timer 5

;; Do FFT levels 7,8
;;
;; On input the 64-byte cache lines hold these data values:
;;	0	+24K	256	+24K	512	+24K	768	+24K
;;	1	...
;;	...
;;	255	...
;;	1024	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+24K	64	+24K	128	+24K	192	+24K
;;	1	...
;;	...
;;	63	...
;;	256	...
;;	...
;;
;; Do 6144 four_complex_fft operations
;;	distance between fft data elements is 64
;;	do 1536 macros each processing 32 data values

;; Do FFT levels 9,10
;;
;; On input the 64-byte cache lines hold these data values:
;;	0	+24K	64	+24K	128	+24K	192	+24K
;;	1	...
;;	...
;;	63	...
;;	256	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+24K	16	+24K	32	+24K	48	+24K
;;	1	...
;;	...
;;	15	...
;;	64	...
;;	...
;;
;; Do 6144 four_complex_fft operations
;;	distance between fft data elements is 16
;;	do 1536 macros each processing 32 data values

;; Do FFT levels 11,12
;;
;; On input the 64-byte cache lines hold these data values:
;;	0	+24K	16	+24K	32	+24K	48	+24K
;;	1	...
;;	...
;;	15	...
;;	64	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+24K	4	+24K	8	+24K	12	+24K
;;	1	...
;;	...
;;	3	...
;;	16	...
;;	...
;;
;; Do 6144 four_complex_fft operations
;;	distance between fft data elements is 4
;;	do 1536 macros each processing 32 data values

;; Do this 64 cache lines at a time to maximize accessing data
;; while it is in the L1 cache.

	start_timer 6
	bump	rdi, 6144*XMM_SCD1	;; Load sin/cos pointers
	mov	rbx, rdi
	mov	rbp, rdi
	loops_init_prefetch 1536, 64, 2, rcx ;; 24 iters of 4 of 64 cache lines
	r4_x4cl_four_complex_djbfft_preload

b3a:	xtouch	[rcx+48*64-64]		;; Load prefetch TLB

b3b:	xxprefetch [rdi+4*XMM_SCD2]	;; Prefetch for next pass
	r4_x4cl_four_complex_djbfft rsi, 4*64, 64*64, dist128, rdi
	bump	rdi, 4*XMM_SCD2		;; Next sine/cosine pointer
	loops	16, b3b			;; Loop 16 times
	bump	rsi, -16*4*64		;; Restore source pointer
	bump	rdi, -16*4*XMM_SCD2	;; Restore sine/cosine pointer
	loops_undo 16			;; Back up loop counter

b4b:	r4_x4cl_four_complex_djbfft rsi, 4*64, 16*64, 32*64, rbx
	bump	rbx, 4*4*XMM_SCD2	;; Next sine/cosine pointer
	loops	4, b4b			;; Loop 4 times
	bump	rsi, -4*4*64+64*64	;; Next source pointer
	bump	rbx, -4*4*4*XMM_SCD2	;; Restore sine/cosine pointer
	loops	2, b4b			;; Loop twice
	bump	rsi, -2*64*64+dist128	;; Next source pointer
	loops	2, b4b			;; Loop twice
	bump	rsi, -2*dist128		;; Restore source pointer
	loops_undo 2*2*4		;; Back up loop counter

b5b:	r4_x4cl_four_complex_djbfft rsi, 16*64, 4*64, 8*64, rbp
	loops	8, b5b			;; Loop 8 times
	bump	rsi, -8*16*64+dist128	;; Next source pointer
	loops	2, b5b			;; Loop twice

	bump	rsi, -2*dist128+64	;; Next source pointer
	bump	rdi, XMM_SCD2		;; Next sine/cosine pointer
	bump	rbx, 4*XMM_SCD2		;; Next sine/cosine pointer
	bump	rbp, 16*XMM_SCD2	;; Next sine/cosine pointer
	loops_touch 2, b3b, 2, b3a	;; Loop 4 times
	bump	rsi, -4*64+2*dist128	;; Next source pointer
	bump	rdi, -4*XMM_SCD2	;; Restore sine/cosine pointer
	bump	rbx, -4*4*XMM_SCD2	;; Restore sine/cosine pointer
	bump	rbp, -4*16*XMM_SCD2	;; Restore sine/cosine pointer
	loops	24, b3a			;; Test outer loop counter
	bump	rsi, -24*2*dist128	;; Restore source pointer
	end_timer 6

;; Do FFT levels 13,14 as well as inverse FFT levels 13,14
;;
;; On input the 64-byte cache lines hold these data values:
;;	0	+24K	4	+24K	8	+24K	12	+24K
;;	1	...
;;	...
;;	3	...
;;	16	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+24K	2	+24K	4	+24K	6	+24K
;;	1	...
;;	8	...
;;	...

;; Do 6144 four_complex_fft operations
;;	distance between fft data elements is 1
;;	do 1536 macros each processing 32 data values

;; Execute the right middle step

	cmp	ffttype, 2
	jl	xpass2_12288_levels_complex_1
	jg	xpass2_12288_levels_complex_3

xpass2_12288_levels_complex_2:
	start_timer 9
	loops_init 1536
	r4_x4cl_four_complex_with_square_preload
b6a:	r4_x4cl_four_complex_with_square rsi, 4*64, 64, 2*64
	loops	32, b6a			;; Loop 32 times
	bump	rsi, -32*4*64+dist128	;; Next source pointer
	loops	48, b6a			;; Loop 48 times
	bump	rsi, -48*dist128	;; Restore source pointer
	end_timer 9
	jmp	xpass2_12288_levels_complex_unfft

xpass2_12288_levels_complex_1:
	start_timer 9
	loops_init_prefetch 1536, 128, 1, rcx
b6b0:	xtouch	[rcx+4096-128]		;; Load prefetch TLBs
b6b:	r4_x4cl_four_complex_fft_final rsi, 4*64, 64, 2*64
	loops	32, b6b			;; Loop 32 times
	bump	rsi, -32*4*64+dist128	;; Next source pointer
	loops	48, b6b0		;; Loop 48 times
	end_timer 9
	jmp	xpass2_12288_levels_complex_done

xpass2_12288_levels_complex_3:
	start_timer 9
	mov	rbp, DIST_TO_MULSRCARG
	loops_init 1536
b6c:	r4_x4cl_four_complex_with_mult rsi, 4*64, 64, 2*64
	loops	32, b6c			;; Loop 32 times
	bump	rsi, -32*4*64+dist128	;; Next source pointer
	loops	48, b6c			;; Loop 48 times
	bump	rsi, -48*dist128	;; Restore source pointer
	end_timer 9
	jmp	xpass2_12288_levels_complex_unfft

xpass2_12288_levels_complex_4:
	start_timer 9
	mov	rcx, DATA_PREFETCH	;; Prefetch pointer
	xtouch	[rcx]			;; Load first prefetch TLB
	mov	rbp, DIST_TO_MULSRCARG
	loops_init_prefetch 1536, 128, 1, rcx
b6d0:	xtouch	[rcx+4096-128]		;; Load prefetch TLBs
b6d:	r4_x4cl_four_complex_with_mulf rsi, 4*64, 64, 2*64
	loops	32, b6d			;; Loop 32 times
	bump	rsi, -32*4*64+dist128	;; Next source pointer
	loops	48, b6d0		;; Loop 48 times
	bump	rsi, -48*dist128	;; Restore source pointer
	end_timer 9

;; Do inverse FFT levels 11,12
;; On input the 64-byte cache lines hold these data values:
;;	0	+24K	2	+24K	4	+24K	6	+24K
;;	1	...
;;	8	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+24K	8	+24K	16	+24K	24	+24K
;;	1	...
;;	...
;;	7	...
;;	32	...
;;	...
;;
;; Do 6144 four_complex_unfft operations
;;	distance between fft data elements is 4
;;	do 1536 macros each processing 32 data values

;; Do inverse FFT levels 9,10
;; On input the 64-byte cache lines hold these data values:
;;	0	+24K	8	+24K	16	+24K	24	+24K
;;	1	...
;;	...
;;	7	...
;;	32	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+24K	32	+24K	64	+24K	96	+24K
;;	1	...
;;	...
;;	31	...
;;	128	...
;;	...
;;
;; Do 6144 four_complex_unfft operations
;;	distance between fft data elements is 16
;;	do 1536 macros each processing 32 data values

;; Do inverse FFT levels 7,8
;; On input the 64-byte cache lines hold these data values:
;;	0	+24K	32	+24K	64	+24K	96	+24K
;;	1	...
;;	...
;;	31	...
;;	128	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+24K	128	+24K	256	+24K	384	+24K
;;	1	...
;;	...
;;	127	...
;;	512	...
;;	...
;;
;; Do 6144 four_complex_unfft operations
;;	distance between fft data elements is 64
;;	do 1536 macros each processing 32 data values

;; Do this 64 cache lines at a time to maximize accessing data
;; while it is in the L1 cache.

xpass2_12288_levels_complex_unfft:

	start_timer 10
	mov	rdi, xsincos_complex	;; Load sin/cos pointers
	bump	rdi, 6144*XMM_SCD1
	mov	rbx, rdi
	mov	rbp, rdi
	loops_init_prefetch 1536, 64, 2, rcx ;; 48 iters of 2 iters of 64 cache lines
	r4_x4cl_four_complex_djbunfft_preload

c5a:	xtouch	[rcx+48*64-64]		;; Load prefetch TLB

c5b:	r4_x4cl_four_complex_djbunfft rsi, 8*64, 2*64, 4*64, rbp, 2*16*XMM_SCD2
	loops	16, c5b			;; Loop 16 times
	bump	rsi, -16*8*64		;; Restore source pointer
	loops_undo 16			;; Restore loop counter

c4b:	r4_x4cl_four_complex_djbunfft rsi, 2*64, 8*64, 16*64, rbx, 8*4*XMM_SCD2
	bump	rbx, 2*4*XMM_SCD2	;; Next sine/cosine pointer
	loops	4, c4b			;; Loop 4 times
	bump	rsi, -4*2*64+32*64	;; Next source pointer
	bump	rbx, -4*2*4*XMM_SCD2	;; Restore sine/cosine pointer
	loops	4, c4b			;; Loop 4 times
	bump	rsi, -4*32*64		;; Restore source pointer
	loops_undo 4*4			;; Restore loop counter

c3b:	r4_x4cl_four_complex_djbunfft rsi, 2*64, 32*64, 64*64, rdi, 32*1*XMM_SCD2
	bump	rdi, 2*XMM_SCD2		;; Next sine/cosine pointer
	loops	16, c3b			;; Loop 16 times
	bump	rdi, -16*2*XMM_SCD2	;; Restore sine/cosine pointer

	bump	rsi, -16*2*64+64	;; Next source pointer
	bump	rdi, XMM_SCD2		;; Next sine/cosine pointer
	bump	rbx, 4*XMM_SCD2		;; Next sine/cosine pointer
	bump	rbp, 16*XMM_SCD2	;; Next sine/cosine pointer
	loops	2, c5b			;; Loop twice
	bump	rsi, -2*64+dist128	;; Next source pointer
	bump	rdi, -2*XMM_SCD2	;; Restore sine/cosine pointer
	bump	rbx, -2*4*XMM_SCD2	;; Restore sine/cosine pointer
	bump	rbp, -2*16*XMM_SCD2	;; Restore sine/cosine pointer
	loops	48, c5a			;; Loop 48 times
	bump	rsi, -48*dist128	;; Restore source pointer
	end_timer 10

;; Do inverse FFT levels 5,6
;; On input the 64-byte cache lines hold these data values:
;;	0	+24K	128	+24K	256	+24K	384	+24K
;;	1	...
;;	...
;;	127	...
;;	512	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+24K	512	+24K	1024	+24K	1536	+24K
;;	1	...
;;	...
;;	511	...
;;	2048	...
;;	...

;; Do 6144 four_complex_unfft operations
;;	distance between fft data elements is 256
;;	do 1536 macros each processing 32 data values

	start_timer 13
	mov	rdi, xsincos_complex	;; Load sin/cos pointer
	mov	rbx, rdi
	loops_init_prefetch 1536, 64, 2, rcx;; 12 iters of 128
	r4_x4cl_2sc_four_complex_djbunfft_preload
c2a:	xtouch	[rcx+4096-64]		;; Load prefetch TLBs
c2b:	r4_x4cl_2sc_four_complex_djbunfft rsi, 64, dist128, 2*dist128, rdi, 128*12*XMM_SCD1, rbx, 128*24*XMM_SCD1
	bump	rdi, 12*XMM_SCD1	;; Next sin/cos data
	bump	rbx, 24*XMM_SCD1	;; Next sin/cos data
	loops	128, c2b		;; Loop 128 times
	bump	rsi, -128*64+4*dist128	;; Next source pointer
	bump	rdi, -128*12*XMM_SCD1	;; Restore sin/cos ptr
	bump	rbx, -128*24*XMM_SCD1	;; Restore sin/cos ptr
	loops	12, c2a			;; Test loop counter
	bump	rsi, -12*4*dist128	;; Restore source pointer
	end_timer 13

;; Do inverse FFT levels 3,4
;; On input the 64-byte cache lines hold these data values:
;;	0	+24K	512	+24K	1024	+24K	1536	+24K
;;	1	...
;;	...
;;	511	...
;;	2048	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+24K	2048	+24K	4096	+24K	6144	+24K
;;	1	...
;;	...
;;	2047	...
;;	8192	...
;;	...

;; Do 6144 four_complex_unfft operations
;;	distance between fft data elements is 1024
;;	do 1536 macros each processing 32 data values

	start_timer 14
	bump	rcx, -2*64		;; Back up prefetch pointer by 2 cache lines
	loops_init_prefetch 1536, 64, 16, rcx ;; 3 iters of 4 iters of 128
	xtouch	[rcx+4096-64]		;; Load prefetch TLBs
	xtouch	[rcx+96*64-64]
	r4_x4cl_2sc_four_complex_djbunfft_preload
c7b:	r4_x4cl_2sc_four_complex_djbunfft rsi, 64, 4*dist128, 8*dist128, rdi, 512*3*XMM_SCD1, rbx, 512*6*XMM_SCD1
	bump	rdi, 3*XMM_SCD1		;; Next sin/cos data
	bump	rbx, 6*XMM_SCD1		;; Next sin/cos data
	loops	128, c7b		;; Loop 128 times
	bump	rsi, -128*64+dist128	;; Next source pointer
	loops	4, c7b			;; Loop 4 times
	bump	rsi, -4*dist128+16*dist128 ;; Next source pointer
	bump	rdi, -512*3*XMM_SCD1	;; Restore sin/cos ptr
	bump	rbx, -512*6*XMM_SCD1	;; Restore sin/cos ptr
	loops	3, c7b			;; Test loop counter
	bump	rsi, -3*16*dist128	;; Restore source pointer
	end_timer 14

;; Do inverse FFT levels 1,2 (actually 1.585 FFT levels)
;; On input the 64-byte cache lines hold these data values:
;;	0	+24K	2048	+24K	4096	+24K	6144	+24K
;;	1	...
;;	...
;;	2047	...
;;	8192	...
;;	...
;; On output the 64-byte cache lines hold these data values if pass 1 swizzles:
;;	0	+24K	6144	+24K	12288	+24K	18432	+24K
;;	1	...
;;	...
;;	6143	...
;; On output the 64-byte cache lines hold these data values if pass 2 swizzles:
;;	0	+6K	12288	+6K	24576	+6K	36864	+6K
;;	1	...
;;	...
;;	6143	...

;; Do 8192 three_complex_unfft operations
;;	distance between fft data elements is 4096
;;	do 2048 macros each processing 24 data values

	start_timer 15
	loops_init 2048			;; 16 iters of 128
	r3_x3cl_three_complex_djbunfft_preload
c1b:	r3_x3cl_three_complex_djbunfft rsi, 64, 16*dist128, rdi, 2048*XMM_SCD1
	bump	rdi, XMM_SCD1		;; Next set of sin/cos data
	loops	128, c1b		;; Test inner loop counter
	bump	rsi, -128*64+dist128	;; Next source pointer
	loops	16, c1b			;; Test outer loop counter
	end_timer 15

xpass2_12288_levels_complex_done:
	ENDM


; *************** radix4-pass2-15360-levels-real macro ******************
; This macro takes 30720 real values and performs the final 14.907 levels of
; the FFT.  Independently, this macro also takes 15360 complex values and
; performs the final 13.907 levels of the FFT.  The results are squared, and
; the first 13.907/14.907 levels of the inverse FFT are done.

r4_pass2_15360_levels_real MACRO
	LOCAL	b1b, d2b, d3b, b2b0, b2b, b3b0, b3b, b4b, b5a, b5b, b6b, b7b
	LOCAL	b8b, b9b, baa, bab0, bab, bac, bad0, bad
	LOCAL	c1b, e2b, e3b, c2b, c3b, c4b, c5a, c5b, c6b, c7a, c7b, c8b, c9b
	LOCAL	xpass2_15360_levels_real_1, xpass2_15360_levels_real_2
	LOCAL	xpass2_15360_levels_real_3, xpass2_15360_levels_real_4
	LOCAL	xpass2_15360_real_unfft, xpass2_15360_real_done

;; We switch to the same format used in one-pass FFTs (that is, the
;; high word of xmm registers being 1 greater than the low word):

;; Type 4 FFTs skip the forward FFT process

	start_timer 2
	mov	rsi, DATA_ADDR		;; Load source address
	mov	rbx, DIST_TO_FFTSRCARG
	cmp	ffttype, 4
	je	xpass2_15360_levels_real_4

;; Do FFT level 1,2 (actually 1.585 levels)
;;
;; On input the 64-byte cache lines hold these data values if pass 2 swizzles:
;;	0	+7680	15360	+7680	30720	+7680	46080	+7680
;;	1	...
;;	...
;;	7679	...
;; On input the 64-byte cache lines hold these data values if pass 1 swizzles:
;;	0	+30K	7680	+30K	15360	+30K	23040	+30K
;;	1	...
;;	...
;;	7679	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+30K	2560	+30K	5120	+30K	7680	+30K
;;	1	...
;;	...
;;	2559	...
;;	10240	...
;;	...

;; Do 5120 six_reals_fft operations
;; Do 5120 three_complex_djbfft operations
;;	distance between fft data elements is 5120
;;	do 2560 iterations

	mov	rdi, xsincos_complex	;; Ptr to the sin/cos data
	mov	rdx, sincos3		;; Ptr to real sin/cos data
	loops_init 2560
b1b:	r3_fh3cl_six_reals_three_complex_djbfft rsi, 64, 20*dist128, rdi, 2560*XMM_SCD1, rdx, 2560*XMM_HSCD1
	bump	rdi, XMM_SCD1		;; Next sin/cos data
	bump	rdx, XMM_HSCD1		;; Next sin/cos data
	loops	128, b1b		;; Test loop counter
	bump	rsi, -128*64+dist128	;; Next source pointer
	loops	20, b1b			;; Test loop counter
	bump	rsi, -20*dist128	;; Restore source pointer
	bump	rdi, -2560*XMM_SCD1	;; Restore sin/cos ptr
	bump	rdx, 2560*XMM_HSCD1	;; Next sin/cos ptr

;; Do FFT levels 3,4 (actually 2.322 levels)
;;
;; On input the 64-byte cache lines hold these data values:
;;	0	+30K	2560	+30K	5120	+30K	7680	+30K
;;	1	...
;;	...
;;	2559	...
;;	10240	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+30K	512	+30K	1024	+30K	1536	+30K
;;	1	...
;;	...
;;	511	...
;;	2048	...
;;	...

;; Do 1024 ten_reals_fft operations
;; Do 1024 five_complex_djbfft operations
;;	distance between fft data elements is 1024
;;	do 512 macros each processing 40 data values

	mov	rcx, DATA_PREFETCH	;; Prefetch pointer
	xtouch	[rcx]			;; Preload first TLB
	mov	rbx, rdi
	loops_init_prefetch 1536, 64, 16, rcx
	xtouch	[rcx+4096-64]		;; Preload the TLBs
	xtouch	[rcx+96*64-64]
d2b:	r5_h5cl_2sc_ten_reals_five_complex_djbfft rsi, 64, 4*dist128, rdi, 512*3*XMM_SCD1, rbx, 512*6*XMM_SCD1, rdx, 512*XMM_HSCD2
	bump	rdi, 3*XMM_SCD1		;; Next sin/cos data
	bump	rbx, 6*XMM_SCD1		;; Next sin/cos data
	bump	rdx, XMM_HSCD2		;; Next sin/cos data
	loops	128, d2b		;; Test inner loop counter
	bump	rsi, -128*64+dist128	;; Next source pointer
	loops	4, d2b			;; Test loop counter
	bump	rsi, -4*dist128+20*dist128 ;; Next source pointer
	bump	rdi, -512*3*XMM_SCD1	;; Restore sin/cos ptr
	bump	rbx, -512*6*XMM_SCD1	;; Restore sin/cos ptr
	bump	rdx, 512*XMM_HSCD2	;; Next sin/cos ptr

;; Do 4096 five_complex_fft operations
;;	distance between fft data elements is 1024
;;	do 1024 macros each processing 40 data values

	loops_reset			;; 2 more iters of 512
	r5_nf5cl_2sc_five_complex_djbfft_preload
d3b:	r5_nf5cl_2sc_five_complex_djbfft rsi, 64, 4*dist128, rdi, 512*3*XMM_SCD1, rbx, 512*6*XMM_SCD1
	bump	rdi, 3*XMM_SCD1		;; Next sin/cos data
	bump	rbx, 6*XMM_SCD1		;; Next sin/cos data
	loops	128, d3b		;; Test inner loop counter
	bump	rsi, -128*64+dist128	;; Next source pointer
	loops	4, d3b			;; Test loop counter
	bump	rsi, -4*dist128+20*dist128 ;; Next source pointer
	bump	rdi, -512*3*XMM_SCD1	;; Restore sin/cos ptr
	bump	rbx, -512*6*XMM_SCD1	;; Restore sin/cos ptr
	loops	3, d3b			;; Test loop counter
	bump	rsi, -3*20*dist128	;; Restore source pointer

;; Do FFT levels 5,6
;;
;; On input the 64-byte cache lines hold these data values:
;;	0	+30K	512	+30K	1024	+30K	1536	+30K
;;	1	...
;;	...
;;	511	...
;;	2048	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+30K	256	+30K	512	+30K	768	+30K
;;	1	...
;;	...
;;	255	...
;;	1024	...
;;	...

;; Do 256 eight_reals_fft operations
;; Do 256 four_complex_djbfft operations
;;	distance between fft data elements is 256
;;	do 256 macros each processing 16 data values

	loops_init_prefetch 3840, 64, 2, rcx
b2b0:	xtouch	[rcx+4096-64]		;; Preload the TLB
b2b:	r4_h2cl_2sc_eight_reals_four_complex_djbfft rsi, 64, 2*dist128, rdi, rbx, rdx
	bump	rdi, 15*XMM_SCD1	;; Next sin/cos data
	bump	rbx, 30*XMM_SCD1	;; Next sin/cos data
	bump	rdx, XMM_HSCD2		;; Next sin/cos data
	loops	128, b2b		;; Test inner loop counter
	bump	rsi, -128*64+dist128	;; Next source pointer
	loops	2, b2b0			;; Test loop counter
	bump	rsi, -2*dist128+4*dist128 ;; Next source pointer
	bump	rdi, -256*15*XMM_SCD1	;; Restore sin/cos ptr
	bump	rbx, -256*30*XMM_SCD1	;; Restore sin/cos ptr
	bump	rdx, -256*XMM_HSCD2	;; Restore sin/cos ptr

;; Do 7168 four_complex_fft operations
;;	distance between fft data elements is 256
;;	do 3584 macros each processing 16 data values

	loops_reset
	r4_x2cl_2sc_four_complex_djbfft_preload
b3b0:	xtouch	[rcx+4096-64]		;; Preload the TLBs
b3b:	r4_x2cl_2sc_four_complex_djbfft rsi, 64, 2*dist128, rdi, rbx
	bump	rdi, 15*XMM_SCD1	;; Next sin/cos data
	bump	rbx, 30*XMM_SCD1	;; Next sin/cos data
	loops	128, b3b		;; Test inner loop counter
	bump	rsi, -128*64+dist128	;; Next source pointer
	loops	2, b3b0			;; Test loop counter
	bump	rsi, -2*dist128+4*dist128 ;; Next source pointer
	bump	rdi, -256*15*XMM_SCD1	;; Restore sin/cos ptr
	bump	rbx, -256*30*XMM_SCD1	;; Restore sin/cos ptr
	loops	15, b3b0		;; Test loop counter
	bump	rsi, -15*4*dist128	;; Restore source pointer

;; Do FFT levels 7,8
;;
;; On input the 64-byte cache lines hold these data values:
;;	0	+30K	256	+30K	512	+30K	768	+30K
;;	1	...
;;	...
;;	255	...
;;	1024	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+30K	64	+30K	128	+30K	192	+30K
;;	1	...
;;	...
;;	63	...
;;	256	...
;;	...

;; Do 64 eight_reals_fft operations
;; Do 192 four_complex_djbfft operations
;;	distance between fft data elements is 64
;;	do 64 macros each processing 32 data values

	bump	rdi, 7680*XMM_SCD1	;; Load sin/cos pointer
	loops_init_prefetch 1920, 64, 1, rcx
	xtouch	[rcx+4096-64]		;; Preload the TLBs
	r4_h4cl_eight_reals_four_complex_djbfft_preload
b4b:	r4_h4cl_eight_reals_four_complex_djbfft rsi, 64, 64*64, dist128, rdi, rdx
	bump	rdi, XMM_SCD2		;; Next sin/cos data
	bump	rdx, 4*XMM_HSCD2	;; Next sin/cos data
	loops	64, b4b			;; Test loop counter
	bump	rsi, -64*64+2*dist128	;; Next source pointer
	bump	rdi, -64*XMM_SCD2	;; Restore sin/cos ptr
	bump	rdx, -64*4*XMM_HSCD2	;; Restore sin/cos ptr

;; Do 7424 four_complex_djbfft operations
;;	distance between fft data elements is 64
;;	do 1856 macros each processing 32 data values

	loops_reset			;; 29 more iterations of 64
	r4_x4cl_four_complex_djbfft_preload
b5a:	xtouch	[rcx+4096-64]		;; Preload the TLBs
b5b:	r4_x4cl_four_complex_djbfft rsi, 64, 64*64, dist128, rdi
	bump	rdi, XMM_SCD2		;; Next sin/cos data
	loops	64, b5b			;; Test inner loop counter
	bump	rsi, -64*64+2*dist128	;; Next source pointer
	bump	rdi, -64*XMM_SCD2	;; Restore sin/cos ptr
	loops	30, b5a			;; Test loop counter
	bump	rsi, -30*2*dist128	;; Restore source pointer

;; Do FFT levels 9,10
;;
;; On input the 64-byte cache lines hold these data values:
;;	0	+30K	64	+30K	128	+30K	192	+30K
;;	1	...
;;	...
;;	63	...
;;	256	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+30K	16	+30K	32	+30K	48	+30K
;;	1	...
;;	...
;;	15	...
;;	64	...
;;	...

;; Do 16 eight_reals_fft operations
;; Do 48 four_complex_djbfft operations
;;	distance between fft data elements is 16
;;	do 16 macros each processing 32 data values

	loops_init 1920
	r4_h4cl_eight_reals_four_complex_djbfft_preload
b6b:	r4_h4cl_eight_reals_four_complex_djbfft rsi, 64, 16*64, 32*64, rdi, rdx
	bump	rdi, 4*XMM_SCD2		;; Next sin/cos data
	bump	rdx, 16*XMM_HSCD2	;; Next sin/cos data
	loops	16, b6b			;; Test loop counter
	bump	rsi, -16*64+64*64	;; Next source pointer
	bump	rdi, -16*4*XMM_SCD2	;; Restore sin/cos ptr
	bump	rdx, -16*16*XMM_HSCD2	;; Restore sin/cos ptr

;; Do 7616 four_complex_djbfft operations
;;	distance between fft data elements is 16
;;	do 1904 macros each processing 32 data values

	loops_reset			;; 60 iters of 1 or 2 of 16
	r4_x4cl_four_complex_djbfft_preload
b7b:	r4_x4cl_four_complex_djbfft rsi, 64, 16*64, 32*64, rdi
	bump	rdi, 4*XMM_SCD2		;; Next sin/cos data
	loops	16, b7b			;; Test inner loop counter
	bump	rsi, -16*64+64*64	;; Next source pointer
	bump	rdi, -16*4*XMM_SCD2	;; Restore sin/cos ptr
	loops	2, b7b			;; Test loop counter
	bump	rsi, -2*64*64+dist128	;; Next source pointer
	loops	60, b7b			;; 60 iterations
	bump	rsi, -60*dist128	;; Next source pointer

;; Do FFT levels 11,12
;;
;; On input the 64-byte cache lines hold these data values:
;;	0	+30K	16	+30K	32	+30K	48	+30K
;;	1	...
;;	...
;;	15	...
;;	64	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+30K	4	+30K	8	+30K	12	+30K
;;	1	...
;;	...
;;	3	...
;;	16	...
;;	...

;; Do 4 eight_reals_fft operations
;; Do 12 four_complex_djbfft operations
;;	distance between fft data elements is 4
;;	do 4 macros each processing 32 data values

	loops_init 1920
	r4_h4cl_eight_reals_four_complex_djbfft_preload
b8b:	r4_h4cl_eight_reals_four_complex_djbfft rsi, 64, 4*64, 8*64, rdi, rdx
	bump	rdi, 16*XMM_SCD2	;; Next sin/cos data
	bump	rdx, 64*XMM_HSCD2	;; Next sin/cos data
	loops	4, b8b			;; Test loop counter
	bump	rsi, -4*64+16*64	;; Next source pointer
	bump	rdi, -4*16*XMM_SCD2	;; Restore sin/cos ptr
	bump	rdx, -4*64*XMM_HSCD2	;; Restore sin/cos ptr

;; Do 7664 four_complex_djbfft operations
;;	distance between fft data elements is 4
;;	do 1916 macros each processing 32 data values

	loops_reset			;; 60 iters of 7 or 8 of 4
	r4_x4cl_four_complex_djbfft_preload
b9b:	r4_x4cl_four_complex_djbfft rsi, 64, 4*64, 8*64, rdi
	bump	rdi, 16*XMM_SCD2	;; Next sin/cos data
	loops	4, b9b			;; Test loop counter
	bump	rsi, -4*64+16*64	;; Next source pointer
	bump	rdi, -4*16*XMM_SCD2	;; Restore sin/cos ptr
	loops	8, b9b			;; Test middle loop counter
	bump	rsi, -8*16*64+dist128	;; Next source pointer
	loops	60, b9b			;; 60 iterations
	bump	rsi, -60*dist128	;; Restore source pointer

;; Do FFT levels 13,14
;;
;; On input the 64-byte cache lines hold these data values:
;;	0	+30K	4	+30K	8	+30K	12	+30K
;;	1	...
;;	...
;;	3	...
;;	16	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+30K	2	+30K	4	+30K	6	+30K
;;	1	...
;;	8	...
;;	...

;; Do 1 eight_reals_fft operations
;; Do 3 four_complex_fft operations
;;	distance between fft data elements is 1

;; Do 7676 four_complex_fft operations
;;	distance between fft data elements is 1
;;	do 1919 macros each processing 32 data values

;; Execute the proper middle step

	cmp	ffttype, 2
	jl	xpass2_15360_levels_real_1
	jg	xpass2_15360_levels_real_3

xpass2_15360_levels_real_2:
	r4_h4cl_eight_reals_four_complex_with_square rsi, 4*64, 64, 2*64
	loops_init 1920, 1, 32
	r4_x4cl_four_complex_with_square_preload
baa:	r4_x4cl_four_complex_with_square rsi, 4*64, 64, 2*64
	loops	32, baa			;; Test loop counter
	bump	rsi, -32*4*64+dist128	;; Next source pointer
	loops	60, baa			;; Test loop counter
	bump	rsi, -60*dist128	;; Restore source pointer
	jmp	xpass2_15360_real_unfft

xpass2_15360_levels_real_1:
	r4_h4cl_eight_reals_four_complex_fft_final rsi, 4*64, 64, 2*64
	L2prefetch128 [rcx]
	bump	rcx, 128
	loops_init_prefetch 1920, 128, 1, rcx, 1, 32
bab0:	xtouch	[rcx+4096-64]		;; Preload the TLBs
bab:	r4_x4cl_four_complex_fft_final rsi, 4*64, 64, 2*64
	loops	32, bab			;; Test loop counter
	bump	rsi, -32*4*64+dist128	;; Next source pointer
	loops	60, bab0		;; Test loop counter
	jmp	xpass2_15360_real_done

xpass2_15360_levels_real_3:
	mov	rbp, DIST_TO_MULSRCARG
	r4_h4cl_eight_reals_four_complex_with_mult rsi, 4*64, 64, 2*64
	loops_init 1920, 1, 32
bac:	r4_x4cl_four_complex_with_mult rsi, 4*64, 64, 2*64
	loops	32, bac			;; Test loop counter
	bump	rsi, -32*4*64+dist128	;; Next source pointer
	loops	60, bac			;; Test loop counter
	bump	rsi, -60*dist128	;; Restore source pointer
	jmp	xpass2_15360_real_unfft

xpass2_15360_levels_real_4:
	mov	rbp, DIST_TO_MULSRCARG
	mov	rcx, DATA_PREFETCH	;; Load prefetch pointer
	xtouch	[rcx]			;; Preload the TLBs
	r4_h4cl_eight_reals_four_complex_with_mulf rsi, 4*64, 64, 2*64
	L2prefetch128 [rcx]
	bump	rcx, 128
	loops_init_prefetch 1920, 128, 1, rcx, 1, 32
bad0:	xtouch	[rcx+4096-64]		;; Preload the TLBs
bad:	r4_x4cl_four_complex_with_mulf rsi, 4*64, 64, 2*64
	loops	32, bad			;; Test loop counter
	bump	rsi, -32*4*64+dist128	;; Next source pointer
	loops	60, bad0		;; Test loop counter
	bump	rsi, -60*dist128	;; Restore source pointer

;; Do inverse FFT levels 11,12
;; On input the 64-byte cache lines hold these data values:
;;	0	+30K	2	+30K	4	+30K	6	+30K
;;	1	...
;;	8	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+30K	8	+30K	16	+30K	24	+30K
;;	1	...
;;	...
;;	7	...
;;	32	...
;;	...

;; Do 4 eight_reals_unfft operations
;; Do 4 four_complex_djbunfft operations
;;	distance between fft data elements is 4
;;	do 2 macros processing 32 data values

xpass2_15360_real_unfft:
	mov	rdi, xsincos_complex	;; Load sin/cos pointers
	bump	rdi, 7680*XMM_SCD1
	mov	rdx, sincos3
	bump	rdx, 5120*XMM_HSCD1+1024*XMM_HSCD2
	loops_init 1920
c8b:	r4_h4cl_eight_reals_four_complex_djbunfft rsi, 64, 2*64, 4*64, rdi, 2*16*XMM_SCD2, rdx, 2*64*XMM_HSCD2
	bump	rdi, 16*XMM_SCD2	;; Next sin/cos data
	bump	rdx, 64*XMM_HSCD2	;; Next sin/cos data
	loops	2, c8b			;; Test loop counter
	bump	rsi, -2*64+8*64		;; Next source pointer
	bump	rdi, -2*16*XMM_SCD2	;; Restore sin/cos ptr
	bump	rdx, -2*64*XMM_HSCD2	;; Restore sin/cos ptr

;; Do 7672 four_complex_djbunfft operations
;;	distance between fft data elements is 4
;;	do 1918 macros each processing 32 data values

	loops_reset			;; 60 iters of 15 or 16 of 2
	r4_x4cl_four_complex_djbunfft_preload
c9b:	r4_x4cl_four_complex_djbunfft rsi, 64, 2*64, 4*64, rdi, 2*16*XMM_SCD2
	bump	rdi, 16*XMM_SCD2	;; Next sine/cosine pointer
	loops	2, c9b			;; Test loop counter
	bump	rsi, -2*64+8*64		;; Next source pointer
	bump	rdi, -2*16*XMM_SCD2	;; Restore sin/cos ptr
	loops	16, c9b			;; Test loop counter
	bump	rsi, -16*8*64+dist128	;; Next source pointer
	loops	60, c9b			;; Test loop counter
	bump	rsi, -60*dist128	;; Restore source pointer

;; Do inverse FFT levels 9,10
;; On input the 64-byte cache lines hold these data values:
;;	0	+30K	8	+30K	16	+30K	24	+30K
;;	1	...
;;	...
;;	7	...
;;	32	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+30K	32	+30K	64	+30K	96	+30K
;;	1	...
;;	...
;;	31	...
;;	128	...
;;	...

;; Do 16 eight_reals_unfft operations
;; Do 16 four_complex_djbunfft operations
;;	distance between fft data elements is 16
;;	do 8 macros processing 32 data values

	loops_init_prefetch 1920, 64, 1, rcx
	xtouch	[rcx+4096-64]		;; Preload the TLBs
c6b:	r4_h4cl_eight_reals_four_complex_djbunfft rsi, 64, 8*64, 16*64, rdi, 8*4*XMM_SCD2, rdx, 8*16*XMM_HSCD2
	bump	rdi, 4*XMM_SCD2		;; Next sin/cos data
	bump	rdx, 16*XMM_HSCD2	;; Next sin/cos data
	loops	8, c6b			;; Test inner loop counter
	bump	rsi, -8*64+32*64	;; Next source pointer
	bump	rdi, -8*4*XMM_SCD2	;; Restore sin/cos ptr
	bump	rdx, -8*16*XMM_HSCD2	;; Restore sin/cos ptr

;; Do 7648 four_complex_djbunfft operations
;;	distance between fft data elements is 16
;;	do 1912 macros each processing 32 data values

	loops_reset			;; 60 iters of 3 or 4 of 4
	r4_x4cl_four_complex_djbunfft_preload
c7a:	xtouch	[rcx+4096-64]		;; Preload the TLBs
c7b:	r4_x4cl_four_complex_djbunfft rsi, 64, 8*64, 16*64, rdi, 8*4*XMM_SCD2
	bump	rdi, 4*XMM_SCD2		;; Next sin/cos data
	loops	8, c7b			;; Test inner loop counter
	bump	rsi, -8*64+32*64	;; Next source pointer
	bump	rdi, -8*4*XMM_SCD2	;; Restore sin/cos ptr
	loops	4, c7b			;; Test loop counter
	bump	rsi, -4*32*64+dist128	;; Next source pointer
	loops_touch 2, c7b, 30, c7a	;; Test loop counter
	bump	rsi, -60*dist128	;; Restore source pointer

;; Do inverse FFT levels 7,8
;; On input the 64-byte cache lines hold these data values:
;;	0	+30K	32	+30K	64	+30K	96	+30K
;;	1	...
;;	...
;;	31	...
;;	128	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+30K	128	+30K	256	+30K	384	+30K
;;	1	...
;;	...
;;	127	...
;;	512	...
;;	...

;; Do 64 eight_reals_unfft operations
;; Do 64 four_complex_djbunfft operations
;;	distance between fft data elements is 64
;;	do 32 macros processing 32 data values

	loops_init_prefetch 1920, 64, 1, rcx
	xtouch	[rcx+4096-64]		;; Preload the TLBs
c4b:	r4_h4cl_eight_reals_four_complex_djbunfft rsi, 64, 32*64, 64*64, rdi, 32*XMM_SCD2, rdx, 32*4*XMM_HSCD2
	bump	rdi, XMM_SCD2		;; Next sin/cos data
	bump	rdx, 4*XMM_HSCD2	;; Next sin/cos data
	loops	32, c4b			;; Test inner loop counter
	bump	rsi, -32*64+dist128	;; Next source pointer
	bump	rdi, -32*XMM_SCD2	;; Restore sin/cos ptr
	bump	rdx, -32*4*XMM_HSCD2	;; Restore sin/cos ptr

;; Do 7552 four_complex_djbunfft operations
;;	distance between fft data elements is 64
;;	do 1888 macros each processing 32 data values

	loops_reset			;; 59 more iters of 32
	r4_x4cl_four_complex_djbunfft_preload
c5a:	xtouch	[rcx+4096-64]		;; Preload the TLBs
c5b:	r4_x4cl_four_complex_djbunfft rsi, 64, 32*64, 64*64, rdi, 32*XMM_SCD2
	bump	rdi, XMM_SCD2		;; Next sin/cos data
	loops	32, c5b			;; Test inner loop counter
	bump	rsi, -32*64+dist128	;; Next source pointer
	bump	rdi, -32*XMM_SCD2	;; Restore sin/cos ptr
	loops_touch 2, c5b, 30, c5a	;; Test loop counter
	bump	rsi, -60*dist128	;; Next source pointer

;; Do inverse FFT levels 5,6
;; On input the 64-byte cache lines hold these data values:
;;	0	+30K	128	+30K	256	+30K	384	+30K
;;	1	...
;;	...
;;	127	...
;;	512	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+30K	512	+30K	1024	+30K	1536	+30K
;;	1	...
;;	...
;;	511	...
;;	2048	...
;;	...

;; Do 256 eight_reals_unfft operations
;; Do 256 four_complex_djbunfft operations
;;	distance between fft data elements is 256
;;	do 128 macros processing 32 data values

	loops_init 1920
	mov	rdi, xsincos_complex	;; Load sin/cos pointer
	mov	rbx, rdi
c2b:	r4_h4cl_2sc_eight_reals_four_complex_djbunfft rsi, 64, dist128, 2*dist128, rdi, 128*15*XMM_SCD1, rbx, 128*30*XMM_SCD1, rdx, 128*XMM_HSCD2
	bump	rdi, 15*XMM_SCD1	;; Next sin/cos data
	bump	rbx, 30*XMM_SCD1	;; Next sin/cos data
	bump	rdx, XMM_HSCD2		;; Next sin/cos data
	loops	128, c2b		;; Test inner loop counter
	bump	rsi, -128*64+4*dist128	;; Next source pointer
	bump	rdi, -128*15*XMM_SCD1	;; Restore sin/cos ptr
	bump	rbx, -128*30*XMM_SCD1	;; Restore sin/cos ptr
	bump	rdx, -128*XMM_HSCD2-1024*XMM_HSCD2 ;; Next sin/cos ptr

;; Do 7168 four_complex_djbunfft operations
;;	distance between fft data elements is 256
;;	do 1792 macros each processing 32 data values

	loops_reset			;; 14 more iters of 128
	r4_x4cl_2sc_four_complex_djbunfft_preload
c3b:	r4_x4cl_2sc_four_complex_djbunfft rsi, 64, dist128, 2*dist128, rdi, 128*15*XMM_SCD1, rbx, 128*30*XMM_SCD1
	bump	rdi, 15*XMM_SCD1	;; Next sin/cos data
	bump	rbx, 30*XMM_SCD1	;; Next sin/cos data
	loops	128, c3b		;; Test inner loop counter
	bump	rsi, -128*64+4*dist128	;; Next source pointer
	bump	rdi, -128*15*XMM_SCD1	;; Restore sin/cos ptr
	bump	rbx, -128*30*XMM_SCD1	;; Restore sin/cos ptr
	loops	15, c3b			;; Test loop counter
	bump	rsi, -15*4*dist128	;; Next source pointer

;; Do inverse FFT levels 3,4 (actually 2.322 FFT levels)
;; On input the 64-byte cache lines hold these data values:
;;	0	+30K	512	+30K	1024	+30K	1536	+30K
;;	1	...
;;	...
;;	511	...
;;	2048	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+30K	2560	+30K	5120	+30K	7680	+30K
;;	1	...
;;	...
;;	2559	...
;;	10240	...
;;	...

;; Do 1024 ten_reals_unfft operations
;; Do 1024 ten_complex_djbunfft operations
;;	distance between fft data elements is 1024
;;	do 512 macros processing 40 data values

	bump	rcx, -2*64		;; Back up prefetch pointer by 2 cache lines
	loops_init_prefetch 1536, 64, 64, rcx
	xtouch	[rcx+24*64-64]		;; Load prefetch TLB
e2b:	r5_h5cl_2sc_ten_reals_five_complex_djbunfft rsi, 64, 4*dist128, rdi, 512*3*XMM_SCD1, rbx, 512*6*XMM_SCD1, rdx, 512*XMM_HSCD2
	bump	rdi, 3*XMM_SCD1		;; Next sin/cos data
	bump	rbx, 6*XMM_SCD1		;; Next sin/cos data
	bump	rdx, XMM_HSCD2		;; Next sin/cos data
	loops	128, e2b		;; Test inner loop counter
	bump	rsi, -128*64+dist128	;; Next source pointer
	loops	4, e2b			;; Test loop counter
	bump	rsi, -4*dist128+20*dist128 ;; Next source pointer
	bump	rdi, -512*3*XMM_SCD1	;; Restore sin/cos ptr
	bump	rbx, -512*6*XMM_SCD1	;; Restore sin/cos ptr
	bump	rdx, -512*XMM_HSCD2-5120*XMM_HSCD1 ;; Next sin/cos ptr

;; Do 4096 five_complex_djbunfft operations
;;	distance between fft data elements is 1024
;;	do 1024 macros each processing 40 data values

	loops_reset			;; 2 more iters of 512
	r5_x5cl_2sc_five_complex_djbunfft_preload
e3b:	r5_x5cl_2sc_five_complex_djbunfft rsi, 64, 4*dist128, rdi, 512*3*XMM_SCD1, rbx, 512*6*XMM_SCD1
	bump	rdi, 3*XMM_SCD1		;; Next sin/cos data
	bump	rbx, 6*XMM_SCD1		;; Next sin/cos data
	loops	128, e3b		;; Test inner loop counter
	bump	rsi, -128*64+dist128	;; Next source pointer
	loops	4, e3b			;; Test loop counter
	bump	rsi, -4*dist128+20*dist128 ;; Next source pointer
	bump	rdi, -512*3*XMM_SCD1	;; Restore sin/cos ptr
	bump	rbx, -512*6*XMM_SCD1	;; Restore sin/cos ptr
	loops	3, e3b			;; Test loop counter
	bump	rsi, -3*20*dist128	;; Restore source pointer

;; Do inverse FFT levels 1,2 (actually 1.585 FFT levels)
;; On input the 64-byte cache lines hold these data values:
;;	0	+30K	2560	+30K	5120	+30K	7680	+30K
;;	1	...
;;	...
;;	2559	...
;;	10240	...
;;	...
;; On output the 64-byte cache lines hold these data values if pass 1 swizzles:
;;	0	+30K	7680	+30K	15360	+30K	23040	+30K
;;	1	...
;;	...
;;	7679	...
;; On output the 64-byte cache lines hold these data values if pass 2 swizzles:
;;	0	+7680	15360	+7680	30720	+7680	46080	+7680
;;	1	...
;;	...
;;	7679	...

;; Do 5120 six_reals_unfft operations
;; Do 5120 three_complex_djbunfft operations
;;	distance between fft data elements is 5120
;;	do 2560 macros processing 24 data values

	loops_init 2560
c1b:	r3_h3cl_six_reals_three_complex_djbunfft rsi, 64, 20*dist128, rdi, 2560*XMM_SCD1, rdx, 2560*XMM_HSCD1
	bump	rdi, XMM_SCD1		;; Next sin/cos data
	bump	rdx, XMM_HSCD1		;; Next sin/cos data
	loops	128, c1b		;; Test loop counter
	bump	rsi, -128*64+dist128	;; Next source pointer
	loops	20, c1b			;; Test loop counter

xpass2_15360_real_done:
	end_timer 2
	ENDM

; *************** radix4-pass2-15360-levels-complex macro ******************
; This macro takes 2 groups of 15360 complex values and
; performs the final 13.907 levels of a standard radix-4 FFT, squares the
; results, and does the first 13.907 levels of the inverse FFT.

r4_pass2_15360_levels_complex MACRO
	LOCAL	b1b, b7b, b2a, b2b, b3a, b3b, b4a, b4b, b5a, b5b
	LOCAL	b6a, b6b0, b6b, b6c, b6d0, b6d
	LOCAL	c1b, c7b, c2a, c2b, c3a, c3b, c4a, c4b, c5a, c5b
	LOCAL	xpass2_15360_levels_complex_1, xpass2_15360_levels_complex_2
	LOCAL	xpass2_15360_levels_complex_3, xpass2_15360_levels_complex_4
	LOCAL	xpass2_15360_levels_complex_unfft, xpass2_15360_levels_complex_done

;; Type 4 FFTs skip the forward FFT process

	mov	rsi, DATA_ADDR		;; Load address of FFT data
	mov	rbx, DIST_TO_FFTSRCARG
	cmp	ffttype, 4
	je	xpass2_15360_levels_complex_4

;; Do FFT level 1,2 (actually 1.585 levels)
;;
;; We swizzle the data so that one set of data is in the high word of
;; an xmm register, the other set is in the low word.  This means
;; a distance of 7680 data values.
;;
;; On input the 64-byte cache lines hold these data values if pass 2 swizzles:
;;	0	+7680	15360	+7680	30720	+7680	46080	+7680
;;	1	...
;;	...
;;	7679	...
;; On input the 64-byte cache lines hold these data values if pass 1 swizzles:
;;	0	+30K	7680	+30K	15360	+30K	23040	+30K
;;	1	...
;;	...
;;	7679	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+30K	2560	+30K	5120	+30K	7680	+30K
;;	1	...
;;	...
;;	2559	...
;;	10240	...
;;	...

;; Do 10240 three_complex_fft operations
;;	distance between fft data elements is 5120
;;	do 2560 macros each processing 24 data values

	start_timer 3
	mov	rdi, xsincos_complex	;; The sin/cos data
	loops_init 2560			;; 20 iterations of 128
	r3_f3cl_three_complex_djbfft_preload
b1b:	xxprefetch [rdi+XMM_SCD1]	;; Prefetch for next pass
	xxprefetch [rdi+2561*XMM_SCD1]	;; Prefetch for next pass
	r3_f3cl_three_complex_djbfft rsi, 64, 20*dist128, rdi, 2560*XMM_SCD1
	bump	rdi, XMM_SCD1		;; Next sin/cos data
	loops	128, b1b		;; Test inner loop counter
	bump	rsi, -128*64+dist128	;; Next source pointer
	loops	20, b1b			;; Test outer loop counter
	bump	rsi, -20*dist128	;; Restore source pointer
	end_timer 3

;; Do FFT levels 3,4 (actually 2.322 levels)
;;
;; On input the 64-byte cache lines hold these data values:
;;	0	+30K	2560	+30K	5120	+30K	7680	+30K
;;	1	...
;;	...
;;	2559	...
;;	10240	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+30K	512	+30K	1024	+30K	1536	+30K
;;	1	...
;;	...
;;	511	...
;;	2048	...
;;	...

;; Do 6144 five_complex_fft operations
;;	distance between fft data elements is 1024
;;	do 1536 macros each processing 40 data values

	start_timer 4
	mov	rcx, DATA_PREFETCH	;; Prefetch pointer (we need to prefetch 
					;; 7680 cache lines + 118 pad cache lines)
	xtouch	[rcx]			;; Load first prefetch TLB
	mov	rdi, xsincos_complex	;; Ptr to the sin/cos data
	mov	rdx, rdi
	loops_init_prefetch 1536, 64, 16, rcx ;; 3 iters of 4 iters of 128
	xtouch	[rcx+4096-64]		;; Load prefetch TLBs
	xtouch	[rcx+96*64-64]
	r5_nf5cl_2sc_five_complex_djbfft_preload
b7b:	r5_nf5cl_2sc_five_complex_djbfft rsi, 64, 4*dist128, rdi, 512*3*XMM_SCD1, rdx, 512*6*XMM_SCD1
	bump	rdi, 3*XMM_SCD1		;; Next sin/cos data
	bump	rdx, 6*XMM_SCD1		;; Next sin/cos data
	loops	128, b7b		;; Test inner loop counter
	bump	rsi, -128*64+dist128	;; Next source pointer
	loops	4, b7b			;; Test inner loop counter
	bump	rsi, -4*dist128+20*dist128 ;; Next source pointer
	bump	rdi, -512*3*XMM_SCD1	;; Restore sin/cos ptr
	bump	rdx, -512*6*XMM_SCD1	;; Restore sin/cos ptr
	loops	3, b7b			;; Test loop counter
	bump	rsi, -3*20*dist128	;; Restore source pointer
	end_timer 4

;; Do FFT levels 5,6
;;
;; On input the 64-byte cache lines hold these data values:
;;	0	+30K	512	+30K	1024	+30K	1536	+30K
;;	1	...
;;	...
;;	511	...
;;	2048	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+30K	256	+30K	512	+30K	768	+30K
;;	1	...
;;	...
;;	255	...
;;	1024	...
;;	...

;; Do 7680 four_complex_fft operations
;;	distance between fft data elements is 256
;;	do 3840 macros each processing 16 data values

	start_timer 5
	loops_init_prefetch 3840, 64, 4, rcx ;; 15 iters of 2 iters of 128
	r4_x2cl_2sc_four_complex_djbfft_preload
b2a:	xtouch	[rcx+4096-64]		;; Load prefetch TLBs
b2b:	r4_x2cl_2sc_four_complex_djbfft rsi, 64, 2*dist128, rdi, rdx
	bump	rdi, 15*XMM_SCD1	;; Next sin/cos data
	bump	rdx, 30*XMM_SCD1	;; Next sin/cos data
	loops	128, b2b		;; Test inner loop counter
	bump	rsi, -128*64+dist128	;; Next source pointer
	loops	2, b2b			;; Test loop counter
	bump	rsi, -2*dist128+4*dist128 ;; Next source pointer
	bump	rdi, -256*15*XMM_SCD1	;; Restore sin/cos ptr
	bump	rdx, -256*30*XMM_SCD1	;; Restore sin/cos ptr
	loops	15, b2a			;; Test loop counter
	bump	rsi, -15*4*dist128	;; Restore source pointer
	end_timer 5

;; Do FFT levels 7,8
;;
;; On input the 64-byte cache lines hold these data values:
;;	0	+30K	256	+30K	512	+30K	768	+30K
;;	1	...
;;	...
;;	255	...
;;	1024	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+30K	64	+30K	128	+30K	192	+30K
;;	1	...
;;	...
;;	63	...
;;	256	...
;;	...
;;
;; Do 7680 four_complex_fft operations
;;	distance between fft data elements is 64
;;	do 1920 macros each processing 32 data values

;; Do FFT levels 9,10
;;
;; On input the 64-byte cache lines hold these data values:
;;	0	+30K	64	+30K	128	+30K	192	+30K
;;	1	...
;;	...
;;	63	...
;;	256	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+30K	16	+30K	32	+30K	48	+30K
;;	1	...
;;	...
;;	15	...
;;	64	...
;;	...
;;
;; Do 7680 four_complex_fft operations
;;	distance between fft data elements is 16
;;	do 1920 macros each processing 32 data values

;; Do FFT levels 11,12
;;
;; On input the 64-byte cache lines hold these data values:
;;	0	+30K	16	+30K	32	+30K	48	+30K
;;	1	...
;;	...
;;	15	...
;;	64	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+30K	4	+30K	8	+30K	12	+30K
;;	1	...
;;	...
;;	3	...
;;	16	...
;;	...
;;
;; Do 7680 four_complex_fft operations
;;	distance between fft data elements is 4
;;	do 1920 macros each processing 32 data values

;; Do this 64 cache lines at a time to maximize accessing data
;; while it is in the L1 cache.

	start_timer 6
	bump	rdi, 7680*XMM_SCD1	;; Load sin/cos pointers
	mov	rbx, rdi
	mov	rbp, rdi
	loops_init_prefetch 1920, 64, 2, rcx ;; 2*4 iters of 4 of 64 cache lines
	r4_x4cl_four_complex_djbfft_preload

b3a:	xtouch	[rcx+48*64-128]		;; Load prefetch TLB

b3b:	xxprefetch [rdi+4*XMM_SCD2]	;; Prefetch for next pass
	r4_x4cl_four_complex_djbfft rsi, 4*64, 64*64, dist128, rdi
	bump	rdi, 4*XMM_SCD2		;; Next sine/cosine pointer
	loops	16, b3b			;; Loop 16 times
	bump	rsi, -16*4*64		;; Restore source pointer
	bump	rdi, -16*4*XMM_SCD2	;; Restore sine/cosine pointer
	loops_undo 16			;; Back up loop counter

b4b:	r4_x4cl_four_complex_djbfft rsi, 4*64, 16*64, 32*64, rbx
	bump	rbx, 4*4*XMM_SCD2	;; Next sine/cosine pointer
	loops	4, b4b			;; Loop 4 times
	bump	rsi, -4*4*64+64*64	;; Next source pointer
	bump	rbx, -4*4*4*XMM_SCD2	;; Restore sine/cosine pointer
	loops	2, b4b			;; Loop twice
	bump	rsi, -2*64*64+dist128	;; Next source pointer
	loops	2, b4b			;; Loop twice
	bump	rsi, -2*dist128		;; Restore source pointer
	loops_undo 2*2*4		;; Back up loop counter

b5b:	r4_x4cl_four_complex_djbfft rsi, 16*64, 4*64, 8*64, rbp
	loops	8, b5b			;; Loop 8 times
	bump	rsi, -8*16*64+dist128	;; Next source pointer
	loops	2, b5b			;; Loop twice

	bump	rsi, -2*dist128+64	;; Next source pointer
	bump	rdi, XMM_SCD2		;; Next sine/cosine pointer
	bump	rbx, 4*XMM_SCD2		;; Next sine/cosine pointer
	bump	rbp, 16*XMM_SCD2	;; Next sine/cosine pointer
	loops_touch 2, b3b, 2, b3a	;; Loop 4 times
	bump	rsi, -4*64+2*dist128	;; Next source pointer
	bump	rdi, -4*XMM_SCD2	;; Restore sine/cosine pointer
	bump	rbx, -4*4*XMM_SCD2	;; Restore sine/cosine pointer
	bump	rbp, -4*16*XMM_SCD2	;; Restore sine/cosine pointer
	loops	30, b3a			;; Test outer loop counter
	bump	rsi, -30*2*dist128	;; Restore source pointer
	end_timer 6

;; Do FFT levels 13,14 as well as inverse FFT levels 13,14
;;
;; On input the 64-byte cache lines hold these data values:
;;	0	+30K	4	+30K	8	+30K	12	+30K
;;	1	...
;;	...
;;	3	...
;;	16	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+30K	2	+30K	4	+30K	6	+30K
;;	1	...
;;	8	...
;;	...

;; Do 7680 four_complex_fft operations
;;	distance between fft data elements is 1
;;	do 1920 macros each processing 32 data values

;; Execute the right middle step

	cmp	ffttype, 2
	jl	xpass2_15360_levels_complex_1
	jg	xpass2_15360_levels_complex_3

xpass2_15360_levels_complex_2:
	start_timer 9
	loops_init 1920
	r4_x4cl_four_complex_with_square_preload
b6a:	r4_x4cl_four_complex_with_square rsi, 4*64, 64, 2*64
	loops	32, b6a			;; Loop 32 times
	bump	rsi, -32*4*64+dist128	;; Next source pointer
	loops	60, b6a			;; Loop 60 times
	bump	rsi, -60*dist128	;; Restore source pointer
	end_timer 9
	jmp	xpass2_15360_levels_complex_unfft

xpass2_15360_levels_complex_1:
	start_timer 9
	loops_init_prefetch 1920, 128, 1, rcx
b6b0:	xtouch	[rcx+4096-128]		;; Load prefetch TLBs
b6b:	r4_x4cl_four_complex_fft_final rsi, 4*64, 64, 2*64
	loops	32, b6b			;; Loop 32 times
	bump	rsi, -32*4*64+dist128	;; Next source pointer
	loops	60, b6b0		;; Loop 60 times
	end_timer 9
	jmp	xpass2_15360_levels_complex_done

xpass2_15360_levels_complex_3:
	start_timer 9
	mov	rbp, DIST_TO_MULSRCARG
	loops_init 1920
b6c:	r4_x4cl_four_complex_with_mult rsi, 4*64, 64, 2*64
	loops	32, b6c			;; Loop 32 times
	bump	rsi, -32*4*64+dist128	;; Next source pointer
	loops	60, b6c			;; Loop 60 times
	bump	rsi, -60*dist128	;; Restore source pointer
	end_timer 9
	jmp	xpass2_15360_levels_complex_unfft

xpass2_15360_levels_complex_4:
	start_timer 9
	mov	rcx, DATA_PREFETCH	;; Prefetch pointer
	xtouch	[rcx]			;; Load first prefetch TLB
	mov	rbp, DIST_TO_MULSRCARG
	loops_init_prefetch 1920, 128, 1, rcx
b6d0:	xtouch	[rcx+4096-128]		;; Load prefetch TLBs
b6d:	r4_x4cl_four_complex_with_mulf rsi, 4*64, 64, 2*64
	loops	32, b6d			;; Loop 32 times
	bump	rsi, -32*4*64+dist128	;; Next source pointer
	loops	60, b6d0		;; Loop 60 times
	bump	rsi, -60*dist128	;; Restore source pointer
	end_timer 9

;; Do inverse FFT levels 11,12
;; On input the 64-byte cache lines hold these data values:
;;	0	+30K	2	+30K	4	+30K	6	+30K
;;	1	...
;;	8	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+30K	8	+30K	16	+30K	24	+30K
;;	1	...
;;	...
;;	7	...
;;	32	...
;;	...
;;
;; Do 7680 four_complex_unfft operations
;;	distance between fft data elements is 4
;;	do 1920 macros each processing 32 data values

;; Do inverse FFT levels 9,10
;; On input the 64-byte cache lines hold these data values:
;;	0	+30K	8	+30K	16	+30K	24	+30K
;;	1	...
;;	...
;;	7	...
;;	32	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+30K	32	+30K	64	+30K	96	+30K
;;	1	...
;;	...
;;	31	...
;;	128	...
;;	...
;;
;; Do 7680 four_complex_unfft operations
;;	distance between fft data elements is 16
;;	do 1920 macros each processing 32 data values

;; Do inverse FFT levels 7,8
;; On input the 64-byte cache lines hold these data values:
;;	0	+30K	32	+30K	64	+30K	96	+30K
;;	1	...
;;	...
;;	31	...
;;	128	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+30K	128	+30K	256	+30K	384	+30K
;;	1	...
;;	...
;;	127	...
;;	512	...
;;	...
;;
;; Do 7680 four_complex_unfft operations
;;	distance between fft data elements is 64
;;	do 1920 macros each processing 32 data values

;; Do this 64 cache lines at a time to maximize accessing data
;; while it is in the L1 cache.

xpass2_15360_levels_complex_unfft:

	start_timer 10
	mov	rdi, xsincos_complex	;; Load sin/cos pointers
	bump	rdi, 7680*XMM_SCD1
	mov	rbx, rdi
	mov	rbp, rdi
	loops_init_prefetch 1920, 64, 2, rcx ;; 8*2*8 iters of 64 cache lines
	r4_x4cl_four_complex_djbunfft_preload

c5a:	xtouch	[rcx+48*64-64]		;; Load prefetch TLB

c5b:	r4_x4cl_four_complex_djbunfft rsi, 8*64, 2*64, 4*64, rbp, 2*16*XMM_SCD2
	loops	16, c5b			;; Loop 16 times
	bump	rsi, -16*8*64		;; Restore source pointer
	loops_undo 16			;; Restore loop counter

c4b:	r4_x4cl_four_complex_djbunfft rsi, 2*64, 8*64, 16*64, rbx, 8*4*XMM_SCD2
	bump	rbx, 2*4*XMM_SCD2	;; Next sine/cosine pointer
	loops	4, c4b			;; Loop 4 times
	bump	rsi, -4*2*64+32*64	;; Next source pointer
	bump	rbx, -4*2*4*XMM_SCD2	;; Restore sine/cosine pointer
	loops	4, c4b			;; Loop 4 times
	bump	rsi, -4*32*64		;; Restore source pointer
	loops_undo 4*4			;; Restore loop counter

c3b:	r4_x4cl_four_complex_djbunfft rsi, 2*64, 32*64, 64*64, rdi, 32*1*XMM_SCD2
	bump	rdi, 2*XMM_SCD2		;; Next sine/cosine pointer
	loops	16, c3b			;; Loop 16 times
	bump	rdi, -16*2*XMM_SCD2	;; Restore sine/cosine pointer

	bump	rsi, -16*2*64+64	;; Next source pointer
	bump	rdi, XMM_SCD2		;; Next sine/cosine pointer
	bump	rbx, 4*XMM_SCD2		;; Next sine/cosine pointer
	bump	rbp, 16*XMM_SCD2	;; Next sine/cosine pointer
	loops	2, c5b			;; Loop twice
	bump	rsi, -2*64+dist128	;; Next source pointer
	bump	rdi, -2*XMM_SCD2	;; Restore sine/cosine pointer
	bump	rbx, -2*4*XMM_SCD2	;; Restore sine/cosine pointer
	bump	rbp, -2*16*XMM_SCD2	;; Restore sine/cosine pointer
	loops	60, c5a			;; Loop 60 times
	bump	rsi, -60*dist128	;; Restore source pointer
	end_timer 10

;; Do inverse FFT levels 5,6
;; On input the 64-byte cache lines hold these data values:
;;	0	+30K	128	+30K	256	+30K	384	+30K
;;	1	...
;;	...
;;	127	...
;;	512	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+30K	512	+30K	1024	+30K	1536	+30K
;;	1	...
;;	...
;;	511	...
;;	2048	...
;;	...

;; Do 7680 four_complex_unfft operations
;;	distance between fft data elements is 256
;;	do 1920 macros each processing 32 data values

	start_timer 13
	mov	rdi, xsincos_complex	;; Load sin/cos pointer
	mov	rdx, rdi
	loops_init_prefetch 1920, 64, 2, rcx;; 15 iters of 128
	r4_x4cl_2sc_four_complex_djbunfft_preload
c2a:	xtouch	[rcx+4096-64]		;; Load prefetch TLBs
c2b:	r4_x4cl_2sc_four_complex_djbunfft rsi, 64, dist128, 2*dist128, rdi, 128*15*XMM_SCD1, rdx, 128*30*XMM_SCD1
	bump	rdi, 15*XMM_SCD1	;; Next sine/cosine pointer
	bump	rdx, 30*XMM_SCD1	;; Next sine/cosine pointer
	loops	128, c2b		;; Loop 128 times
	bump	rsi, -128*64+4*dist128	;; Next source pointer
	bump	rdi, -128*15*XMM_SCD1	;; Reset sine/cosine pointer
	bump	rdx, -128*30*XMM_SCD1	;; Reset sine/cosine pointer
	loops	15, c2a			;; Test loop counter
	bump	rsi, -15*4*dist128	;; Restore source pointer
	end_timer 13

;; Do inverse FFT levels 3,4 (actually 2.322 FFT levels)
;; On input the 64-byte cache lines hold these data values:
;;	0	+30K	512	+30K	1024	+30K	1536	+30K
;;	1	...
;;	...
;;	511	...
;;	2048	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+30K	2560	+30K	5120	+30K	7680	+30K
;;	1	...
;;	...
;;	2559	...
;;	10240	...
;;	...

;; Do 6144 five_complex_unfft operations
;;	distance between fft data elements is 1024
;;	do 1536 macros each processing 40 data values

	start_timer 14
	bump	rcx, -2*64		;; Back up prefetch pointer by 2 cache lines
	loops_init_prefetch 1536, 64, 64, rcx ;; 3 iters of 4 iters of 128
	xtouch	[rcx+24*64-64]		;; Load prefetch TLBs
	r5_x5cl_2sc_five_complex_djbunfft_preload
c7b:	r5_x5cl_2sc_five_complex_djbunfft rsi, 64, 4*dist128, rdi, 512*3*XMM_SCD1, rdx, 512*6*XMM_SCD1
	bump	rdi, 3*XMM_SCD1		;; Next sine/cosine pointer
	bump	rdx, 6*XMM_SCD1		;; Next sine/cosine pointer
	loops	128, c7b		;; Loop 128 times
	bump	rsi, -128*64+dist128	;; Next source pointer
	loops	4, c7b			;; Loop 4 times
	bump	rsi, -4*dist128+20*dist128 ;; Next source pointer
	bump	rdi, -512*3*XMM_SCD1	;; Reset sine/cosine pointer
	bump	rdx, -512*6*XMM_SCD1	;; Reset sine/cosine pointer
	loops	3, c7b			;; Test loop counter
	bump	rsi, -3*20*dist128	;; Restore source pointer
	end_timer 14

;; Do inverse FFT levels 1,2 (actually 1.585 FFT levels)
;; On input the 64-byte cache lines hold these data values:
;;	0	+30K	2560	+30K	5120	+30K	7680	+30K
;;	1	...
;;	...
;;	2559	...
;;	10240	...
;;	...
;; On output the 64-byte cache lines hold these data values if pass 1 swizzles:
;;	0	+30K	7680	+30K	15360	+30K	23040	+30K
;;	1	...
;;	...
;;	7679	...
;; On output the 64-byte cache lines hold these data values if pass 2 swizzles:
;;	0	+7680	15360	+7680	30720	+7680	46080	+7680
;;	1	...
;;	...
;;	7679	...

;; Do 10240 three_complex_unfft operations
;;	distance between fft data elements is 5120
;;	do 2560 macros each processing 24 data values

	start_timer 15
	loops_init 2560			;; 20 iters of 128
	r3_x3cl_three_complex_djbunfft_preload
c1b:	r3_x3cl_three_complex_djbunfft rsi, 64, 20*dist128, rdi, 2560*XMM_SCD1
	bump	rdi, XMM_SCD1		;; Next set of sin/cos data
	loops	128, c1b		;; Test inner loop counter
	bump	rsi, -128*64+dist128	;; Next source pointer
	loops	20, c1b			;; Test outer loop counter
	end_timer 15

xpass2_15360_levels_complex_done:
	ENDM


; *************** radix4-pass2-14-levels-real macro ******************
; This macro takes 32768 real values and performs the final 15 levels of
; the FFT.  Independently, this macro also takes 16384 complex values and
; performs the final 14 levels of the FFT.  The results are squared, and
; the first 14/15 levels of the inverse FFT are done.

r4_pass2_14_levels_real MACRO
	LOCAL	b1b, d2b, d3b, b2b0, b2b, b3b0, b3b, b4b, b5a, b5b, b6b, b7b
	LOCAL	b8b, b9b, baa, bab0, bab, bac, bad0, bad
	LOCAL	c1b, e2b, e3b, c2b, c3b, c4b, c5a, c5b, c6b, c7a, c7b, c8b, c9b
	LOCAL	xpass2_14_levels_real_1, xpass2_14_levels_real_2
	LOCAL	xpass2_14_levels_real_3, xpass2_14_levels_real_4
	LOCAL	xpass2_14_real_unfft, xpass2_14_real_done

;; We switch to the same format used in one-pass FFTs (that is, the
;; high word of xmm registers being 1 greater than the low word):

;; Type 4 FFTs skip the forward FFT process

	start_timer 2
	mov	rsi, DATA_ADDR		;; Load source address
	mov	rbx, DIST_TO_FFTSRCARG
	cmp	ffttype, 4
	je	xpass2_14_levels_real_4

;; Do FFT level 1,2
;;
;; On input the 64-byte cache lines hold these data values if pass 2 swizzles:
;;	0	+8K	16384	+8K	32768	+8K	49152	+8K
;;	1	...
;;	...
;;	8191	...
;; On input the 64-byte cache lines hold these data values if pass 1 swizzles:
;;	0	+32K	8192	+32K	16384	+32K	24576	+32K
;;	1	...
;;	...
;;	8191	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+32K	4096	+32K	8192	+32K	12288	+32K
;;	1	...
;;	...
;;	4095	...
;;	16384	...
;;	...

;; Do 4096 eight_reals_fft operations
;; Do 4096 four_complex_djbfft operations
;;	distance between fft data elements is 4K
;;	do 4096 iterations

	mov	rdi, xsincos_complex	;; Ptr to the sin/cos data
	mov	rdx, sincos3		;; Ptr to real sin/cos data
	loops_init 4096
b1b:	r4_fh2cl_eight_reals_four_complex_djbfft rsi, 64, 32*dist128, rdi, rdx
	bump	rdi, XMM_SCD2		;; Next sin/cos data
	bump	rdx, XMM_HSCD2		;; Next sin/cos data
	loops	128, b1b		;; Test loop counter
	bump	rsi, -128*64+dist128	;; Next source pointer
	loops	32, b1b			;; Test loop counter
	bump	rsi, -32*dist128	;; Restore source pointer
	bump	rdi, -4096*XMM_SCD2	;; Restore sin/cos ptr
	bump	rdx, -4096*XMM_HSCD2	;; Restore sin/cos ptr

;; Do FFT levels 3,4
;;
;; On input the 64-byte cache lines hold these data values:
;;	0	+32K	4096	+32K	8192	+32K	12288	+32K
;;	1	...
;;	...
;;	4095	...
;;	16384	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+32K	1024	+32K	2048	+32K	3072	+32K
;;	1	...
;;	...
;;	1023	...
;;	4096	...
;;	...

;; Do 1024 eight_reals_fft operations
;; Do 3072 four_complex_djbfft operations
;;	distance between fft data elements is 1024
;;	do 1024 macros each processing 32 data values

	mov	rcx, DATA_PREFETCH	;; Prefetch pointer
	xtouch	[rcx]			;; Preload first TLB
	loops_init 1024
	r4_h4cl_eight_reals_four_complex_djbfft_preload
d2b:	r4_h4cl_eight_reals_four_complex_djbfft rsi, 64, 8*dist128, 16*dist128, rdi, rdx
	bump	rdi, 4*XMM_SCD2		;; Next sin/cos data
	bump	rdx, 4*XMM_HSCD2	;; Next sin/cos data
	loops	128, d2b		;; Test inner loop counter
	bump	rsi, -128*64+dist128	;; Next source pointer
	loops	8, d2b			;; Test loop counter
	bump	rsi, -8*dist128+32*dist128 ;; Next source pointer
	bump	rdi, -1024*4*XMM_SCD2	;; Restore sin/cos ptr
	bump	rdx, -1024*4*XMM_HSCD2	;; Restore sin/cos ptr

;; Do 4096 four_complex_fft operations
;;	distance between fft data elements is 1024
;;	do 1024 macros each processing 32 data values

	loops_init 1024
	r4_x4cl_four_complex_djbfft_preload
d3b:	r4_x4cl_four_complex_djbfft rsi, 64, 8*dist128, 16*dist128, rdi
	bump	rdi, 4*XMM_SCD2		;; Next sin/cos data
	loops	128, d3b		;; Test inner loop counter
	bump	rsi, -128*64+dist128	;; Next source pointer
	loops	8, d3b			;; Test loop counter
	bump	rsi, -8*dist128-32*dist128 ;; Restore source pointer
	bump	rdi, -1024*4*XMM_SCD2	;; Restore sin/cos pointer

;; Do FFT levels 5,6
;;
;; On input the 64-byte cache lines hold these data values:
;;	0	+32K	1024	+32K	2048	+32K	3072	+32K
;;	1	...
;;	...
;;	1023	...
;;	4096	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+32K	256	+32K	512	+32K	768	+32K
;;	1	...
;;	...
;;	255	...
;;	1024	...
;;	...

;; Do 256 eight_reals_fft operations
;; Do 768 four_complex_djbfft operations
;;	distance between fft data elements is 256
;;	do 256 macros each processing 32 data values

	loops_init_prefetch 2048, 64, 1, rcx
	r4_h4cl_eight_reals_four_complex_djbfft_preload
b2b0:	xtouch	[rcx+4096-64]		;; Preload the TLBs
b2b:	r4_h4cl_eight_reals_four_complex_djbfft rsi, 64, 2*dist128, 4*dist128, rdi, rdx
	bump	rdi, 16*XMM_SCD2	;; Next sin/cos data
	bump	rdx, 16*XMM_HSCD2	;; Next sin/cos data
	loops_touch 64, b2b, 2, b2b0	;; Test inner loop counter
	bump	rsi, -128*64+dist128	;; Next source pointer
	bump	rcx, 128		;; Skip pad bytes
	loops	2, b2b0			;; Test loop counter
	bump	rsi, -2*dist128+8*dist128 ;; Next source pointer
	bump	rdi, -256*16*XMM_SCD2	;; Restore sin/cos ptr
	bump	rdx, -256*16*XMM_HSCD2	;; Restore sin/cos ptr

;; Do 7168 four_complex_fft operations
;;	distance between fft data elements is 256
;;	do 1792 macros each processing 32 data values

	loops_reset
	r4_x4cl_four_complex_djbfft_preload
b3b0:	xtouch	[rcx+4096-64]		;; Preload the TLBs
b3b:	r4_x4cl_four_complex_djbfft rsi, 64, 2*dist128, 4*dist128, rdi
	bump	rdi, 16*XMM_SCD2	;; Next sin/cos data
	loops_touch 64, b3b, 2, b3b0	;; Test inner loop counter
	bump	rsi, -128*64+dist128	;; Next source pointer
	bump	rcx, 128		;; Skip pad bytes
	loops	2, b3b0			;; Test loop counter
	bump	rsi, -2*dist128+8*dist128 ;; Next source pointer
	bump	rdi, -256*16*XMM_SCD2	;; Restore sin/cos ptr
	loops	8, b3b0			;; Test loop counter
	bump	rsi, -8*8*dist128	;; Restore source pointer

;; Do FFT levels 7,8
;;
;; On input the 64-byte cache lines hold these data values:
;;	0	+32K	256	+32K	512	+32K	768	+32K
;;	1	...
;;	...
;;	255	...
;;	1024	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+32K	64	+32K	128	+32K	192	+32K
;;	1	...
;;	...
;;	63	...
;;	256	...
;;	...

;; Do 64 eight_reals_fft operations
;; Do 192 four_complex_djbfft operations
;;	distance between fft data elements is 64
;;	do 64 macros each processing 32 data values

	bump	rdi, 4096*XMM_SCD2	;; Load sin/cos pointer
	loops_init_prefetch 2048, 64, 1, rcx
	xtouch	[rcx+4096-64]		;; Preload the TLBs
	r4_h4cl_eight_reals_four_complex_djbfft_preload
b4b:	r4_h4cl_eight_reals_four_complex_djbfft rsi, 64, 64*64, dist128, rdi, rdx
	bump	rdi, XMM_SCD2		;; Next sin/cos data
	bump	rdx, 64*XMM_HSCD2	;; Next sin/cos data
	loops	64, b4b			;; Test loop counter
	bump	rsi, -64*64+2*dist128	;; Next source pointer
	bump	rdi, -64*XMM_SCD2	;; Restore sin/cos ptr
	bump	rdx, -64*64*XMM_HSCD2	;; Restore sin/cos ptr

;; Do 7944 four_complex_djbfft operations
;;	distance between fft data elements is 64
;;	do 1986 macros each processing 32 data values

	loops_reset			;; 31 iterations of 64
	r4_x4cl_four_complex_djbfft_preload
b5a:	xtouch	[rcx+4096-64]		;; Preload the TLBs
b5b:	r4_x4cl_four_complex_djbfft rsi, 64, 64*64, dist128, rdi
	bump	rdi, XMM_SCD2		;; Next sin/cos data
	loops	64, b5b			;; Test inner loop counter
	bump	rsi, -64*64+2*dist128	;; Next source pointer
	bump	rdi, -64*XMM_SCD2	;; Restore sin/cos ptr
	loops	2, b5a			;; Test loop counter
	bump	rcx, 128		;; Skip pad bytes
	loops	16, b5a			;; Test loop counter
	bump	rsi, -32*2*dist128	;; Restore source pointer

;; Do FFT levels 9,10
;;
;; On input the 64-byte cache lines hold these data values:
;;	0	+32K	64	+32K	128	+32K	192	+32K
;;	1	...
;;	...
;;	63	...
;;	256	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+32K	16	+32K	32	+32K	48	+32K
;;	1	...
;;	...
;;	15	...
;;	64	...
;;	...

;; Do 16 eight_reals_fft operations
;; Do 48 four_complex_djbfft operations
;;	distance between fft data elements is 16
;;	do 16 macros each processing 32 data values

	loops_init 2048
	r4_h4cl_eight_reals_four_complex_djbfft_preload
b6b:	r4_h4cl_eight_reals_four_complex_djbfft rsi, 64, 16*64, 32*64, rdi, rdx
	bump	rdi, 4*XMM_SCD2		;; Next sin/cos data
	bump	rdx, 256*XMM_HSCD2	;; Next sin/cos data
	loops	16, b6b			;; Test loop counter
	bump	rsi, -16*64+64*64	;; Next source pointer
	bump	rdi, -16*4*XMM_SCD2	;; Restore sin/cos ptr
	bump	rdx, -16*256*XMM_HSCD2	;; Restore sin/cos ptr

;; Do 8128 four_complex_djbfft operations
;;	distance between fft data elements is 16
;;	do 2032 macros each processing 32 data values

	loops_reset			;; 64 iters of 1 or 2 of 16
	r4_x4cl_four_complex_djbfft_preload
b7b:	r4_x4cl_four_complex_djbfft rsi, 64, 16*64, 32*64, rdi
	bump	rdi, 4*XMM_SCD2		;; Next sin/cos data
	loops	16, b7b			;; Test inner loop counter
	bump	rsi, -16*64+64*64	;; Next source pointer
	bump	rdi, -16*4*XMM_SCD2	;; Restore sin/cos ptr
	loops	2, b7b			;; Test loop counter
	bump	rsi, -2*64*64+dist128	;; Next source pointer
	loops	64, b7b			;; 64 iterations
	bump	rsi, -64*dist128	;; Next source pointer

;; Do FFT levels 11,12
;;
;; On input the 64-byte cache lines hold these data values:
;;	0	+32K	16	+32K	32	+32K	48	+32K
;;	1	...
;;	...
;;	15	...
;;	64	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+32K	4	+32K	8	+32K	12	+32K
;;	1	...
;;	...
;;	3	...
;;	16	...
;;	...

;; Do 4 eight_reals_fft operations
;; Do 12 four_complex_djbfft operations
;;	distance between fft data elements is 4
;;	do 4 macros each processing 32 data values

	loops_init 2048
	r4_h4cl_eight_reals_four_complex_djbfft_preload
b8b:	r4_h4cl_eight_reals_four_complex_djbfft rsi, 64, 4*64, 8*64, rdi, rdx
	bump	rdi, 16*XMM_SCD2	;; Next sin/cos data
	bump	rdx, 1024*XMM_HSCD2	;; Next sin/cos data
	loops	4, b8b			;; Test loop counter
	bump	rsi, -4*64+16*64	;; Next source pointer
	bump	rdi, -4*16*XMM_SCD2	;; Restore sin/cos ptr
	bump	rdx, -4*1024*XMM_HSCD2	;; Restore sin/cos ptr

;; Do 8176 four_complex_djbfft operations
;;	distance between fft data elements is 4
;;	do 2044 macros each processing 32 data values

	loops_reset			;; 64 iters of 7 or 8 of 4
	r4_x4cl_four_complex_djbfft_preload
b9b:	r4_x4cl_four_complex_djbfft rsi, 64, 4*64, 8*64, rdi
	bump	rdi, 16*XMM_SCD2	;; Next sin/cos data
	loops	4, b9b			;; Test loop counter
	bump	rsi, -4*64+16*64	;; Next source pointer
	bump	rdi, -4*16*XMM_SCD2	;; Restore sin/cos ptr
	loops	8, b9b			;; Test middle loop counter
	bump	rsi, -8*16*64+dist128	;; Next source pointer
	loops	64, b9b			;; 16 iterations
	bump	rsi, -64*dist128	;; Restore source pointer

;; Do FFT levels 13,14
;;
;; On input the 64-byte cache lines hold these data values:
;;	0	+32K	4	+32K	8	+32K	12	+32K
;;	1	...
;;	...
;;	3	...
;;	16	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+32K	2	+32K	4	+32K	6	+32K
;;	1	...
;;	8	...
;;	...

;; Do 1 eight_reals_fft operations
;; Do 3 four_complex_fft operations
;;	distance between fft data elements is 1

;; Do 8188 four_complex_fft operations
;;	distance between fft data elements is 1
;;	do 2047 macros each processing 32 data values

;; Execute the proper middle step

	cmp	ffttype, 2
	jl	xpass2_14_levels_real_1
	jg	xpass2_14_levels_real_3

xpass2_14_levels_real_2:
	r4_h4cl_eight_reals_four_complex_with_square rsi, 4*64, 64, 2*64
	loops_init 2048, 1, 32
	r4_x4cl_four_complex_with_square_preload
baa:	r4_x4cl_four_complex_with_square rsi, 4*64, 64, 2*64
	loops	32, baa			;; Test loop counter
	bump	rsi, -32*4*64+dist128	;; Next source pointer
	loops	64, baa			;; Test loop counter
	bump	rsi, -64*dist128	;; Restore source pointer
	jmp	xpass2_14_real_unfft

xpass2_14_levels_real_1:
	r4_h4cl_eight_reals_four_complex_fft_final rsi, 4*64, 64, 2*64
	L2prefetch128 [rcx]
	bump	rcx, 128
	loops_init_prefetch 2048, 128, 1, rcx, 1, 32
bab0:	xtouch	[rcx+4096-64]		;; Preload the TLBs
bab:	r4_x4cl_four_complex_fft_final rsi, 4*64, 64, 2*64
	loops	32, bab			;; Test loop counter
	bump	rsi, -32*4*64+dist128	;; Next source pointer
	loops	2, bab0			;; Test loop counter
	bump	rcx, 128		;; Skip pad bytes
	loops	32, bab0		;; Test loop counter
	jmp	xpass2_14_real_done

xpass2_14_levels_real_3:
	mov	rbp, DIST_TO_MULSRCARG
	r4_h4cl_eight_reals_four_complex_with_mult rsi, 4*64, 64, 2*64
	loops_init 2048, 1, 32
bac:	r4_x4cl_four_complex_with_mult rsi, 4*64, 64, 2*64
	loops	32, bac			;; Test loop counter
	bump	rsi, -32*4*64+dist128	;; Next source pointer
	loops	64, bac			;; Test loop counter
	bump	rsi, -64*dist128	;; Restore source pointer
	jmp	xpass2_14_real_unfft

xpass2_14_levels_real_4:
	mov	rbp, DIST_TO_MULSRCARG
	mov	rcx, DATA_PREFETCH	;; Load prefetch pointer
	xtouch	[rcx]			;; Preload the TLBs
	r4_h4cl_eight_reals_four_complex_with_mulf rsi, 4*64, 64, 2*64
	L2prefetch128 [rcx]
	bump	rcx, 128
	loops_init_prefetch 2048, 128, 1, rcx, 1, 32
bad0:	xtouch	[rcx+4096-64]		;; Preload the TLBs
bad:	r4_x4cl_four_complex_with_mulf rsi, 4*64, 64, 2*64
	loops	32, bad			;; Test loop counter
	bump	rsi, -32*4*64+dist128	;; Next source pointer
	loops	2, bad0			;; Test loop counter
	bump	rcx, 128		;; Skip pad bytes
	loops	32, bad0		;; Test loop counter
	bump	rsi, -64*dist128	;; Restore source pointer

;; Do inverse FFT levels 11,12
;; On input the 64-byte cache lines hold these data values:
;;	0	+32K	2	+32K	4	+32K	6	+32K
;;	1	...
;;	8	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+32K	8	+32K	16	+32K	24	+32K
;;	1	...
;;	...
;;	7	...
;;	32	...
;;	...

;; Do 4 eight_reals_unfft operations
;; Do 4 four_complex_djbunfft operations
;;	distance between fft data elements is 4
;;	do 2 macros processing 32 data values

xpass2_14_real_unfft:
	mov	rdi, xsincos_complex	;; Load sin/cos pointers
	bump	rdi, 4096*XMM_SCD2
	mov	rdx, sincos3
	loops_init 2048
c8b:	r4_h4cl_eight_reals_four_complex_djbunfft rsi, 64, 2*64, 4*64, rdi, 2*16*XMM_SCD2, rdx, 2*1024*XMM_HSCD2
	bump	rdi, 16*XMM_SCD2	;; Next sin/cos data
	bump	rdx, 1024*XMM_HSCD2	;; Next sin/cos data
	loops	2, c8b			;; Test loop counter
	bump	rsi, -2*64+8*64		;; Next source pointer
	bump	rdi, -2*16*XMM_SCD2	;; Restore sin/cos ptr
	bump	rdx, -2*1024*XMM_HSCD2	;; Restore sin/cos ptr

;; Do 8184 four_complex_djbunfft operations
;;	distance between fft data elements is 4
;;	do 2046 macros each processing 32 data values

	loops_reset			;; 64 iters of 15 or 16 of 2
	r4_x4cl_four_complex_djbunfft_preload
c9b:	r4_x4cl_four_complex_djbunfft rsi, 64, 2*64, 4*64, rdi, 2*16*XMM_SCD2
	bump	rdi, 16*XMM_SCD2	;; Next sine/cosine pointer
	loops	2, c9b			;; Test loop counter
	bump	rsi, -2*64+8*64		;; Next source pointer
	bump	rdi, -2*16*XMM_SCD2	;; Restore sin/cos ptr
	loops	16, c9b			;; Test loop counter
	bump	rsi, -16*8*64+dist128	;; Next source pointer
	loops	64, c9b			;; Test loop counter
	bump	rsi, -64*dist128	;; Restore source pointer

;; Do inverse FFT levels 9,10
;; On input the 64-byte cache lines hold these data values:
;;	0	+32K	8	+32K	16	+32K	24	+32K
;;	1	...
;;	...
;;	7	...
;;	32	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+32K	32	+32K	64	+32K	96	+32K
;;	1	...
;;	...
;;	31	...
;;	128	...
;;	...

;; Do 16 eight_reals_unfft operations
;; Do 16 four_complex_djbunfft operations
;;	distance between fft data elements is 16
;;	do 8 macros processing 32 data values

	loops_init_prefetch 2048, 64, 1, rcx
	xtouch	[rcx+4096-64]		;; Preload the TLBs
c6b:	r4_h4cl_eight_reals_four_complex_djbunfft rsi, 64, 8*64, 16*64, rdi, 8*4*XMM_SCD2, rdx, 8*256*XMM_HSCD2
	bump	rdi, 4*XMM_SCD2		;; Next sin/cos data
	bump	rdx, 256*XMM_HSCD2	;; Next sin/cos data
	loops	8, c6b			;; Test inner loop counter
	bump	rsi, -8*64+32*64	;; Next source pointer
	bump	rdi, -8*4*XMM_SCD2	;; Restore sin/cos ptr
	bump	rdx, -8*256*XMM_HSCD2	;; Restore sin/cos ptr

;; Do 8160 four_complex_djbunfft operations
;;	distance between fft data elements is 16
;;	do 2040 macros each processing 32 data values

	loops_reset			;; 64 iters of 3 or 4 of 4
	r4_x4cl_four_complex_djbunfft_preload
c7a:	xtouch	[rcx+4096-64]		;; Preload the TLBs
c7b:	r4_x4cl_four_complex_djbunfft rsi, 64, 8*64, 16*64, rdi, 8*4*XMM_SCD2
	bump	rdi, 4*XMM_SCD2		;; Next sin/cos data
	loops	8, c7b			;; Test inner loop counter
	bump	rsi, -8*64+32*64	;; Next source pointer
	bump	rdi, -8*4*XMM_SCD2	;; Restore sin/cos ptr
	loops	4, c7b			;; Test loop counter
	bump	rsi, -4*32*64+dist128	;; Next source pointer
	loops_touch 2, c7b, 2, c7a	;; Test loop counter
	bump	rcx, 128		;; Skip pad bytes
	loops	16, c7a			;; Test loop counter
	bump	rsi, -64*dist128	;; Restore source pointer

;; Do inverse FFT levels 7,8
;; On input the 64-byte cache lines hold these data values:
;;	0	+32K	32	+32K	64	+32K	96	+32K
;;	1	...
;;	...
;;	31	...
;;	128	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+32K	128	+32K	256	+32K	384	+32K
;;	1	...
;;	...
;;	127	...
;;	512	...
;;	...

;; Do 64 eight_reals_unfft operations
;; Do 64 four_complex_djbunfft operations
;;	distance between fft data elements is 64
;;	do 32 macros processing 32 data values

	loops_init_prefetch 2048, 64, 1, rcx
	xtouch	[rcx+4096-64]		;; Preload the TLBs
c4b:	r4_h4cl_eight_reals_four_complex_djbunfft rsi, 64, 32*64, 64*64, rdi, 32*XMM_SCD2, rdx, 32*64*XMM_HSCD2
	bump	rdi, XMM_SCD2		;; Next sin/cos data
	bump	rdx, 64*XMM_HSCD2	;; Next sin/cos data
	loops	32, c4b			;; Test inner loop counter
	bump	rsi, -32*64+dist128	;; Next source pointer
	bump	rdi, -32*XMM_SCD2	;; Restore sin/cos ptr
	bump	rdx, -32*64*XMM_HSCD2	;; Restore sin/cos ptr

;; Do 8064 four_complex_djbunfft operations
;;	distance between fft data elements is 64
;;	do 2016 macros each processing 32 data values

	loops_reset			;; 63 iters of 32
	r4_x4cl_four_complex_djbunfft_preload
c5a:	xtouch	[rcx+4096-64]		;; Preload the TLBs
c5b:	r4_x4cl_four_complex_djbunfft rsi, 64, 32*64, 64*64, rdi, 32*XMM_SCD2
	bump	rdi, XMM_SCD2		;; Next sin/cos data
	loops	32, c5b			;; Test inner loop counter
	bump	rsi, -32*64+dist128	;; Next source pointer
	bump	rdi, -32*XMM_SCD2	;; Restore sin/cos ptr
	loops_touch 2, c5b, 2, c5a	;; Test loop counter
	bump	rcx, 128		;; Skip pad bytes
	loops	16, c5a			;; Test loop counter
	bump	rsi, -64*dist128	;; Next source pointer

;; Do inverse FFT levels 5,6
;; On input the 64-byte cache lines hold these data values:
;;	0	+32K	128	+32K	256	+32K	384	+32K
;;	1	...
;;	...
;;	127	...
;;	512	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+32K	512	+32K	1024	+32K	1536	+32K
;;	1	...
;;	...
;;	511	...
;;	2048	...
;;	...

;; Do 256 eight_reals_unfft operations
;; Do 256 four_complex_djbunfft operations
;;	distance between fft data elements is 256
;;	do 128 macros processing 32 data values

	loops_init 2048
	mov	rdi, xsincos_complex	;; Load sin/cos pointer
c2b:	r4_h4cl_eight_reals_four_complex_djbunfft rsi, 64, dist128, 2*dist128, rdi, 128*16*XMM_SCD2, rdx, 128*16*XMM_HSCD2
	bump	rdi, 16*XMM_SCD2	;; Next sin/cos data
	bump	rdx, 16*XMM_HSCD2	;; Next sin/cos data
	loops	128, c2b		;; Test inner loop counter
	bump	rsi, -128*64+4*dist128	;; Next source pointer
	bump	rdi, -128*16*XMM_SCD2	;; Restore sin/cos ptr
	bump	rdx, -128*16*XMM_HSCD2	;; Restore sin/cos ptr

;; Do 7680 four_complex_djbunfft operations
;;	distance between fft data elements is 256
;;	do 1920 macros each processing 32 data values

	loops_reset			;; 15 iters of 128
	r4_x4cl_four_complex_djbunfft_preload
c3b:	r4_x4cl_four_complex_djbunfft rsi, 64, dist128, 2*dist128, rdi, 128*16*XMM_SCD2
	bump	rdi, 16*XMM_SCD2	;; Next sin/cos data
	loops	128, c3b		;; Test inner loop counter
	bump	rsi, -128*64+4*dist128	;; Next source pointer
	bump	rdi, -128*16*XMM_SCD2	;; Restore sin/cos ptr
	loops	16, c3b			;; Test loop counter
	bump	rsi, -16*4*dist128	;; Next source pointer

;; Do inverse FFT levels 3,4
;; On input the 64-byte cache lines hold these data values:
;;	0	+32K	512	+32K	1024	+32K	1536	+32K
;;	1	...
;;	...
;;	511	...
;;	2048	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+32K	2048	+32K	4096	+32K	6144	+32K
;;	1	...
;;	...
;;	2047	...
;;	8192	...
;;	...

;; Do 1024 eight_reals_unfft operations
;; Do 1024 four_complex_djbunfft operations
;;	distance between fft data elements is 1024
;;	do 512 macros processing 32 data values

	loops_init 2048
	mov	rdi, xsincos_complex	;; Load sin/cos pointer
e2b:	r4_h4cl_eight_reals_four_complex_djbunfft rsi, 64, 4*dist128, 8*dist128, rdi, 512*4*XMM_SCD2, rdx, 512*4*XMM_HSCD2
	bump	rdi, 4*XMM_SCD2		;; Next sin/cos data
	bump	rdx, 4*XMM_HSCD2	;; Next sin/cos data
	loops	128, e2b		;; Test inner loop counter
	bump	rsi, -128*64+dist128	;; Next source pointer
	loops	4, e2b			;; Test loop counter
	bump	rsi, -4*dist128+16*dist128 ;; Next source pointer
	bump	rdi, -512*4*XMM_SCD2	;; Restore sin/cos ptr
	bump	rdx, -512*4*XMM_HSCD2	;; Restore sin/cos ptr

;; Do 6144 four_complex_djbunfft operations
;;	distance between fft data elements is 1024
;;	do 1536 macros each processing 32 data values

	loops_reset			;; 3 iters of 512
	r4_x4cl_four_complex_djbunfft_preload
e3b:	r4_x4cl_four_complex_djbunfft rsi, 64, 4*dist128, 8*dist128, rdi, 512*4*XMM_SCD2
	bump	rdi, 4*XMM_SCD2		;; Next sin/cos data
	loops	128, e3b		;; Test inner loop counter
	bump	rsi, -128*64+dist128	;; Next source pointer
	loops	4, e3b			;; Test loop counter
	bump	rsi, -4*dist128+16*dist128 ;; Next source pointer
	bump	rdi, -512*4*XMM_SCD2	;; Restore sin/cos ptr
	loops	4, e3b			;; Test loop counter
	bump	rsi, -4*16*dist128	;; Restore source pointer

;; Do inverse FFT levels 1,2
;; On input the 64-byte cache lines hold these data values:
;;	0	+32K	2048	+32K	4096	+32K	6144	+32K
;;	1	...
;;	...
;;	2047	...
;;	8192	...
;;	...
;; On output the 64-byte cache lines hold these data values if pass 1 swizzles:
;;	0	+32K	8192	+32K	16384	+32K	24576	+32K
;;	1	...
;;	...
;;	8191	...
;; On output the 64-byte cache lines hold these data values if pass 2 swizzles:
;;	0	+8K	16384	+8K	32768	+8K	49152	+8K
;;	1	...
;;	...
;;	8191	...

;; Do 4096 eight_reals_unfft operations
;; Do 4096 four_complex_djbunfft operations
;;	distance between fft data elements is 4096
;;	do 2048 macros processing 32 data values

	loops_init 2048
c1b:	r4_h4cl_eight_reals_four_complex_djbunfft rsi, 64, 16*dist128, 32*dist128, rdi, 2048*XMM_SCD2, rdx, 2048*XMM_HSCD2
	bump	rdi, XMM_SCD2		;; Next sin/cos data
	bump	rdx, XMM_HSCD2		;; Next sin/cos data
	loops	128, c1b		;; Test loop counter
	bump	rsi, -128*64+dist128	;; Next source pointer
	loops	16, c1b			;; Test loop counter

xpass2_14_real_done:
	end_timer 2
	ENDM


; *************** radix4-pass2-14-levels-complex macro ******************
; This macro takes 2 groups of 16384 complex values and
; performs the final 14 levels of a standard radix-4 FFT, squares the
; results, and does the first 14 levels of the inverse FFT.

r4_pass2_14_levels_complex MACRO
	LOCAL	b1a, b1b, b7a, b7b, b2b, b3a, b3b, b4a, b4b, b5a, b5b
	LOCAL	b6a0, b6a, b6b0, b6b, b6c0, b6c, b6d0, b6d
	LOCAL	c1b, c7b, c2a, c2b, c3a, c3b, c4a, c4b, c5a, c5b
	LOCAL	xpass2_14_levels_complex_1, xpass2_14_levels_complex_2
	LOCAL	xpass2_14_levels_complex_3, xpass2_14_levels_complex_4
	LOCAL	xpass2_14_levels_complex_unfft, xpass2_14_levels_complex_done

;; Type 4 FFTs skip the forward FFT process

	mov	rsi, DATA_ADDR		;; Load address of FFT data
	mov	rbx, DIST_TO_FFTSRCARG
	cmp	ffttype, 4
	je	xpass2_14_levels_complex_4

;; Do FFT level 1,2
;;
;; We swizzle the data so that one set of data is in the high word of
;; an xmm register, the other set is in the low word.  This means
;; a distance of 8192 data values.
;;
;; On input the 64-byte cache lines hold these data values if pass 2 swizzles:
;;	0	+8K	16384	+8K	32768	+8K	49152	+8K
;;	1	...
;;	...
;;	8191	...
;; On input the 64-byte cache lines hold these data values if pass 1 swizzles:
;;	0	+32K	8192	+32K	16384	+32K	24576	+32K
;;	1	...
;;	...
;;	8191	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+32K	4096	+32K	8192	+32K	12288	+32K
;;	1	...
;;	...
;;	4095	...
;;	16384	...
;;	...

;; Do 8192 four_complex_fft operations
;;	distance between fft data elements is 4096
;;	do 4096 macros each processing 16 data values

;; Do FFT levels 3,4
;;
;; On input the 64-byte cache lines hold these data values:
;;	0	+32K	4096	+32K	8192	+32K	12288	+32K
;;	1	...
;;	...
;;	4095	...
;;	16384	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+32K	1024	+32K	2048	+32K	3072	+32K
;;	1	...
;;	...
;;	1023	...
;;	4096	...
;;	...

;; Do 8192 four_complex_fft operations
;;	distance between fft data elements is 1024
;;	do 2048 macros each processing 32 data values

;; Do FFT levels 5,6
;;
;; On input the 64-byte cache lines hold these data values:
;;	0	+32K	1024	+32K	2048	+32K	3072	+32K
;;	1	...
;;	...
;;	1023	...
;;	4096	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+32K	256	+32K	512	+32K	768	+32K
;;	1	...
;;	...
;;	255	...
;;	1024	...
;;	...

;; Do 8192 four_complex_fft operations
;;	distance between fft data elements is 256
;;	do 2048 macros each processing 32 data values

IF @INSTR(,%xarch,<CORE>) NE 0		;; Core 2 faster doing separate loops

	start_timer 3
	mov	rdi, xsincos_complex	;; The sin/cos data
	loops_init 4096			;; 8 iterations of 128
	r4_f2cl_four_complex_djbfft_preload
b1b:	r4_f2cl_four_complex_djbfft rsi, 64, 32*dist128, rdi
	bump	rdi, XMM_SCD2		;; Next sin/cos data
	loops	128, b1b		;; Test inner loop counter
	bump	rsi, -128*64+dist128	;; Next source pointer
	loops	32, b1b			;; Test outer loop counter
	bump	rsi, -32*dist128	;; Restore source pointer
	end_timer 3

	start_timer 4
	mov	rdi, xsincos_complex	;; Ptr to the sin/cos data
	mov	rcx, DATA_PREFETCH	;; Prefetch pointer
	xtouch	[rcx]			;; Load first TLB
	loops_init_prefetch 2048, 64, 2, rcx ;; 2 iters of 8 iters of 128
	r4_x4cl_four_complex_djbfft_preload
b7a:	xtouch	[rcx+4096-64]		;; Load prefetch TLB
b7b:	r4_x4cl_four_complex_djbfft rsi, 64, 8*dist128, 16*dist128, rdi
	bump	rdi, 4*XMM_SCD2		;; Next sin/cos data
	loops	128, b7b		;; Test inner loop counter
	bump	rsi, -128*64+dist128	;; Next source pointer
	loops	2, b7a			;; Test loop counter
	bump	rcx, 128		;; Skip pad bytes
	loops	4, b7a			;; Test loop counter
	bump	rsi, -8*dist128+32*dist128 ;; Next source pointer
	bump	rdi, -1024*4*XMM_SCD2	;; Restore sin/cos ptr
	loops	2, b7a			;; Test loop counter
	bump	rsi, -2*32*dist128	;; Restore source pointer
	end_timer 4

	start_timer 5
	loops_init 2048			;; 8 iters of 2 iters of 128
	r4_x4cl_four_complex_djbfft_preload
b2b:	r4_x4cl_four_complex_djbfft rsi, 64, 2*dist128, 4*dist128, rdi
	bump	rdi, 16*XMM_SCD2	;; Next sin/cos data
	loops	128, b2b		;; Test inner loop counter
	bump	rsi, -128*64+dist128	;; Next source pointer
	loops	2, b2b			;; Test loop counter
	bump	rsi, -2*dist128+8*dist128 ;; Next source pointer
	bump	rdi, -256*16*XMM_SCD2	;; Restore sin/cos ptr
	loops	8, b2b			;; Test loop counter
	bump	rsi, -8*8*dist128	;; Restore source pointer
	end_timer 5

ELSE					;; Pentium 4 faster doing merged loops

	start_timer 3
	mov	rdi, xsincos_complex	;; The sin/cos data
	mov	rbp, rdi
	mov	rdx, rdi
	mov	rcx, DATA_PREFETCH	;; Prefetch pointer
	xtouch	[rcx]			;; Load first TLB

	loops_init_prefetch 4096, 64, 4, rcx ;; 256 iterations of 32
	;r4_f2cl_four_complex_djbfft_preload -- assume same as r4_x4cl_four_complex_djbfft_preload
	r4_x4cl_four_complex_djbfft_preload

b1a:	xtouch	[rcx+4096-64]		;; Load prefetch TLB
b1b:	r4_f2cl_four_complex_djbfft rsi, 2*dist128, 32*dist128, rdi
	xxprefetch [rdi+XMM_SCD2]	;; Prefetch for next pass
	bump	rdi, 256*XMM_SCD2	;; Next sin/cos data
	loops	16, b1b			;; Test loop counter
	bump	rsi, -16*2*dist128	;; Restore source pointer
	loops_undo 16

b7b:	r4_x4cl_four_complex_djbfft rsi, 2*dist128, 8*dist128, 16*dist128, rbp
	bump	rbp, 256*4*XMM_SCD2	;; Next sin/cos data
	loops_no_prefetch 4, b7b, 2	;; Test loop counter with non-standard incr
	bump	rsi, -4*2*dist128+32*dist128 ;; Next source pointer
	bump	rbp, -4*256*4*XMM_SCD2	;; Restore sin/cos ptr
	loops_no_prefetch 2, b7b	;; Test loop counter
	bump	rsi, -2*32*dist128	;; Restore source pointer
	loops_undo 16

b2b:	r4_x4cl_four_complex_djbfft rsi, 8*dist128, 2*dist128, 4*dist128, rdx
	loops_no_prefetch 8, b2b, 2	;; Test loop counter with non-standard incr

	bump	rsi, -8*8*dist128+64	;; Next source pointer
	bump	rdi, -16*256*XMM_SCD2+XMM_SCD2 ;; Next sin/cos data
	bump	rbp, 4*XMM_SCD2		;; Next sin/cos data
	bump	rdx, 16*XMM_SCD2	;; Next sin/cos data
	loops	32, b1b			;; Test loop counter
	bump	rcx, 128		;; Skip pad bytes
	loops	4, b1a			;; Test loop counter
	bump	rsi, -128*64+dist128	;; Next source pointer
	loops	2, b1a			;; Test loop counter
	bump	rsi, -2*dist128		;; Restore source pointer
	bump	rdi, -256*XMM_SCD2	;; Restore sin/cos data
	end_timer 3

ENDIF

;; Do FFT levels 7,8
;;
;; On input the 64-byte cache lines hold these data values:
;;	0	+32K	256	+32K	512	+32K	768	+32K
;;	1	...
;;	...
;;	255	...
;;	1024	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+32K	64	+32K	128	+32K	192	+32K
;;	1	...
;;	...
;;	63	...
;;	256	...
;;	...
;;
;; Do 8192 four_complex_fft operations
;;	distance between fft data elements is 64
;;	do 2048 macros each processing 32 data values

;; Do FFT levels 9,10
;;
;; On input the 64-byte cache lines hold these data values:
;;	0	+32K	64	+32K	128	+32K	192	+32K
;;	1	...
;;	...
;;	63	...
;;	256	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+32K	16	+32K	32	+32K	48	+32K
;;	1	...
;;	...
;;	15	...
;;	64	...
;;	...
;;
;; Do 8192 four_complex_fft operations
;;	distance between fft data elements is 16
;;	do 2048 macros each processing 32 data values

;; Do FFT levels 11,12
;;
;; On input the 64-byte cache lines hold these data values:
;;	0	+32K	16	+32K	32	+32K	48	+32K
;;	1	...
;;	...
;;	15	...
;;	64	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+32K	4	+32K	8	+32K	12	+32K
;;	1	...
;;	...
;;	3	...
;;	16	...
;;	...
;;
;; Do 8192 four_complex_fft operations
;;	distance between fft data elements is 4
;;	do 2048 macros each processing 32 data values

;; Do this 64 cache lines at a time to maximize accessing data
;; while it is in the L1 cache.

IFDEF NEVER				;; Never faster to do separate loops

	start_timer 6
	bump	rdi, 4096*XMM_SCD2	;; Load sin/cos pointer
	loops_init_prefetch 2048, 64, 2, rcx ;; 8*4 iters of 64
	r4_x4cl_four_complex_djbfft_preload
b3a:	xtouch	[rcx+4096-128]		;; Load two prefetch TLBs
	xtouch	[rcx+2*4096-128]
b3b:	r4_x4cl_four_complex_djbfft rsi, 64, 64*64, dist128, rdi
	bump	rdi, XMM_SCD2		;; Next sine/cosine pointer
	loops	64, b3b			;; Loop 64 times
	bump	rsi, -64*64+2*dist128	;; Next source pointer
	bump	rdi, -64*XMM_SCD2	;; Restore sine/cosine pointer
	loops	4, b3b			;; Test loop counter
	bump	rcx, -128*64+8192+128	;; Next 8KB section to prefetch
	loops	8, b3a			;; Test loop counter
	bump	rsi, -8*4*2*dist128	;; Restore source pointer
	end_timer 6

	start_timer 7
	loops_init_prefetch 2048, 64, 2, rcx ;; 8*8*2 iters of 16
	r4_x4cl_four_complex_djbfft_preload
b4a:	xtouch	[rcx+4096-128]		;; Load two prefetch TLBs
	xtouch	[rcx+2*4096-128]
b4b:	r4_x4cl_four_complex_djbfft rsi, 64, 16*64, 32*64, rdi
	bump	rdi, 4*XMM_SCD2		;; Next sine/cosine pointer
	loops	16, b4b			;; Loop 16 times
	bump	rsi, -16*64+64*64	;; Next source pointer
	bump	rdi, -16*4*XMM_SCD2	;; Restore sine/cosine pointer
	loops	2, b4b			;; Test loop counter
	bump	rsi, -2*64*64+dist128	;; Adjust source pointer
	loops	8, b4b			;; Test loop counter
	bump	rcx, -128*64+8192+128	;; Next 8KB section to prefetch
	loops	8, b4a			;; Test loop counter
	bump	rsi, -8*8*dist128	;; Restore source pointer
	end_timer 7

	start_timer 8
	loops_init_prefetch 2048, 64, 2, rcx ;; 8*8*8 iters of 4
	r4_x4cl_four_complex_djbfft_preload
b5a:	xtouch	[rcx+4096-128]		;; Load two prefetch TLBs
	xtouch	[rcx+2*4096-128]
b5b:	r4_x4cl_four_complex_djbfft rsi, 64, 4*64, 8*64, rdi
	bump	rdi, 16*XMM_SCD2	;; Next sine/cosine pointer
	loops	4, b5b			;; Loop 4 times
	bump	rsi, -4*64+16*64	;; Next source pointer
	bump	rdi, -4*16*XMM_SCD2	;; Restore sine/cosine pointer
	loops	8, b5b			;; Test loop counter
	bump	rsi, -8*16*64+dist128	;; Adjust source pointer
	loops	8, b5b			;; Test loop counter
	bump	rcx, -128*64+8192+128	;; Next 8KB section to prefetch
	loops	8, b5a			;; Test loop counter
	bump	rsi, -8*8*dist128	;; Restore source pointer
	end_timer 8

ELSE					;; Always faster to do merged loops

	start_timer 6
	bump	rdi, 4096*XMM_SCD2	;; Load sin/cos pointers
	mov	rbx, rdi
	mov	rbp, rdi
	loops_init_prefetch 2048, 64, 2, rcx ;; 2*4 iters of 4 of 64 cache lines
	r4_x4cl_four_complex_djbfft_preload

b3a:	xtouch	[rcx+4096-128]		;; Load six prefetch TLBs
	xtouch	[rcx+2*4096-128]
	xtouch	[rcx+3*4096-128]
	xtouch	[rcx+4*4096-128]
	xtouch	[rcx+5*4096-128]
	xtouch	[rcx+6*4096-128]

b3b:	xxprefetch [rdi+4*XMM_SCD2]	;; Prefetch for next pass
	r4_x4cl_four_complex_djbfft rsi, 4*64, 64*64, dist128, rdi
	bump	rdi, 4*XMM_SCD2		;; Next sine/cosine pointer
	loops	16, b3b			;; Loop 16 times
	bump	rsi, -16*4*64		;; Restore source pointer
	bump	rcx, -8*64+8192+128	;; Prefetch from next section
	loops_undo 16			;; Back up loop counter

b4b:	r4_x4cl_four_complex_djbfft rsi, 4*64, 16*64, 32*64, rbx
	bump	rbx, 4*4*XMM_SCD2	;; Next sine/cosine pointer
	loops	4, b4b			;; Loop 4 times
	bump	rsi, -4*4*64+64*64	;; Next source pointer
	bump	rbx, -4*4*4*XMM_SCD2	;; Restore sine/cosine pointer
	loops	2, b4b			;; Loop twice
	bump	rsi, -2*64*64+dist128	;; Next source pointer
	loops	2, b4b			;; Loop twice
	bump	rsi, -2*dist128		;; Restore source pointer
	bump	rcx, -8*64+8192+128	;; Prefetch from next section
	loops_undo 2*2*4		;; Back up loop counter

b5b:	r4_x4cl_four_complex_djbfft rsi, 16*64, 4*64, 8*64, rbp
	loops	8, b5b			;; Loop 8 times
	bump	rsi, -8*16*64+dist128	;; Next source pointer
	loops	2, b5b			;; Loop twice
	bump	rcx, -2*(8192+128)	;; Back to prefetching from 1st section

	bump	rsi, -2*dist128+64	;; Next source pointer
	bump	rdi, -16*4*XMM_SCD2+XMM_SCD2 ;; Next sine/cosine pointer
	bump	rbx, 4*XMM_SCD2		;; Next sine/cosine pointer
	bump	rbp, 16*XMM_SCD2	;; Next sine/cosine pointer
	loops	4, b3b			;; Loop 4 times
	bump	rsi, -4*64+2*dist128	;; Next source pointer
	bump	rdi, -4*XMM_SCD2	;; Restore sine/cosine pointer
	bump	rbx, -4*4*XMM_SCD2	;; Restore sine/cosine pointer
	bump	rbp, -4*16*XMM_SCD2	;; Restore sine/cosine pointer
	loops	4, b3b			;; Loop 4 times
	bump	rcx, -16*8*64+3*(8192+128) ;; Prefetch next 3 sections
	loops	8, b3a			;; Loop 8 times
	bump	rsi, -32*2*dist128	;; Restore source pointer
	end_timer 6
ENDIF

;; Do FFT levels 13,14 as well as inverse FFT levels 13,14
;;
;; On input the 64-byte cache lines hold these data values:
;;	0	+32K	4	+32K	8	+32K	12	+32K
;;	1	...
;;	...
;;	3	...
;;	16	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+32K	2	+32K	4	+32K	6	+32K
;;	1	...
;;	8	...
;;	...

;; Do 8192 four_complex_fft operations
;;	distance between fft data elements is 1
;;	do 2048 macros each processing 32 data values

;; Execute the right middle step

	cmp	ffttype, 2
	jl	xpass2_14_levels_complex_1
	jg	xpass2_14_levels_complex_3

xpass2_14_levels_complex_2:
	start_timer 9
	loops_init 2048
	r4_x4cl_four_complex_with_square_preload
b6a:	r4_x4cl_four_complex_with_square rsi, 4*64, 64, 2*64
	loops	32, b6a			;; Loop 32 times
	bump	rsi, -32*4*64+dist128	;; Next source pointer
	loops	64, b6a			;; Loop 64 times
	bump	rsi, -64*dist128	;; Restore source pointer
	end_timer 9
	jmp	xpass2_14_levels_complex_unfft

xpass2_14_levels_complex_1:
	start_timer 9
	loops_init_prefetch 2048, 128, 1, rcx
b6b0:	xtouch	[rcx+4096-128]		;; Load prefetch TLB
b6b:	r4_x4cl_four_complex_fft_final rsi, 4*64, 64, 2*64
	loops	32, b6b			;; Loop 32 times
	bump	rsi, -32*4*64+dist128	;; Next source pointer
	loops	2, b6b0			;; Loop 2 times
	bump	rcx, 128		;; Prefetch next section
	loops	32, b6b0		;; Loop 32 times
	end_timer 9
	jmp	xpass2_14_levels_complex_done

xpass2_14_levels_complex_3:
	start_timer 9
	mov	rbp, DIST_TO_MULSRCARG
	loops_init 2048
b6c:	r4_x4cl_four_complex_with_mult rsi, 4*64, 64, 2*64
	loops	32, b6c			;; Loop 32 times
	bump	rsi, -32*4*64+dist128	;; Next source pointer
	loops	64, b6c			;; Loop 64 times
	bump	rsi, -64*dist128	;; Restore source pointer
	end_timer 9
	jmp	xpass2_14_levels_complex_unfft

xpass2_14_levels_complex_4:
	start_timer 9
	mov	rcx, DATA_PREFETCH	;; Prefetch pointer
	xtouch	[rcx]			;; Load first prefetch TLB
	mov	rbp, DIST_TO_MULSRCARG
	loops_init_prefetch 2048, 128, 1, rcx
b6d0:	xtouch	[rcx+4096-64]		;; Load prefetch TLB
b6d:	r4_x4cl_four_complex_with_mulf rsi, 4*64, 64, 2*64
	loops	32, b6d			;; Loop 32 times
	bump	rsi, -32*4*64+dist128	;; Next source pointer
	loops	2, b6d0			;; Loop 2 times
	bump	rcx, 128		;; Prefetch next section
	loops	32, b6d0		;; Loop 32 times
	bump	rsi, -64*dist128	;; Restore source pointer
	end_timer 9

;; Do inverse FFT levels 11,12
;; On input the 64-byte cache lines hold these data values:
;;	0	+32K	2	+32K	4	+32K	6	+32K
;;	1	...
;;	8	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+32K	8	+32K	16	+32K	24	+32K
;;	1	...
;;	...
;;	7	...
;;	32	...
;;	...
;;
;; Do 8192 four_complex_unfft operations
;;	distance between fft data elements is 4
;;	do 2048 macros each processing 32 data values

;; Do inverse FFT levels 9,10
;; On input the 64-byte cache lines hold these data values:
;;	0	+32K	8	+32K	16	+32K	24	+32K
;;	1	...
;;	...
;;	7	...
;;	32	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+32K	32	+32K	64	+32K	96	+32K
;;	1	...
;;	...
;;	31	...
;;	128	...
;;	...
;;
;; Do 8192 four_complex_unfft operations
;;	distance between fft data elements is 16
;;	do 2048 macros each processing 32 data values

;; Do inverse FFT levels 7,8
;; On input the 64-byte cache lines hold these data values:
;;	0	+32K	32	+32K	64	+32K	96	+32K
;;	1	...
;;	...
;;	31	...
;;	128	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+32K	128	+32K	256	+32K	384	+32K
;;	1	...
;;	...
;;	127	...
;;	512	...
;;	...
;;
;; Do 8192 four_complex_unfft operations
;;	distance between fft data elements is 64
;;	do 2048 macros each processing 32 data values

;; Do this 64 cache lines at a time to maximize accessing data
;; while it is in the L1 cache.

xpass2_14_levels_complex_unfft:

IFDEF NEVER				;; Never faster to do separate loops

	start_timer 10
	mov	rdi, xsincos_complex	;; Load sin/cos pointers
	bump	rdi, 4096*XMM_SCD2
	loops_init_prefetch 2048, 64, 2, rcx;; 8 iters of 8 iters of 16 iters of 2
	r4_x4cl_four_complex_djbunfft_preload
c5a:	xtouch	[rcx+4096-128]		;; Load prefetch TLBs
	xtouch	[rcx+2*4096-128]
c5b:	r4_x4cl_four_complex_djbunfft rsi, 64, 2*64, 4*64, rdi, 2*16*XMM_SCD2
	bump	rdi, 16*XMM_SCD2	;; Next sine/cosine pointer
	loops	2, c5b			;; Loop 2 times
	bump	rsi, -2*64+8*64		;; Next source pointer
	bump	rdi, -2*16*XMM_SCD2	;; Reset sine/cosine pointer
	loops	16, c5b			;; Test loop counter
	bump	rsi, -16*8*64+dist128	;; Next source pointer
	loops	8, c5b			;; Loop 8 times
	bump	rcx, -128*64+8192+128	;; Next 8KB section to prefetch
	loops	8, c5a			;; Test loop counter
	bump	rsi, -8*8*dist128	;; Restore source pointer
	end_timer 10

	start_timer 11
	loops_init_prefetch 2048, 64, 2, rcx;; 8 iters of 8 iters of 4 iters of 8
	r4_x4cl_four_complex_djbunfft_preload
c4a:	xtouch	[rcx+4096-128]		;; Load prefetch TLBs
	xtouch	[rcx+2*4096-128]
c4b:	r4_x4cl_four_complex_djbunfft rsi, 64, 8*64, 16*64, rdi, 8*4*XMM_SCD2
	bump	rdi, 4*XMM_SCD2		;; Next sine/cosine pointer
	loops	8, c4b			;; Loop 8 times
	bump	rsi, -8*64+32*64	;; Next source pointer
	bump	rdi, -8*4*XMM_SCD2	;; Reset sine/cosine pointer
	loops	4, c4b			;; Test loop counter
	bump	rsi, -4*32*64+dist128	;; Next source pointer
	loops	8, c4b			;; Loop 8 times
	bump	rcx, -128*64+8192+128	;; Next 8KB section to prefetch
	loops	8, c4a			;; Test loop counter
	bump	rsi, -8*8*dist128	;; Restore source pointer
	end_timer 11

	start_timer 12
	loops_init_prefetch 2048, 64, 2, rcx;; 8 iters of 8 iters of 32
	r4_x4cl_four_complex_djbunfft_preload
c3a:	xtouch	[rcx+4096-128]		;; Load prefetch TLBs
	xtouch	[rcx+2*4096-128]
c3b:	r4_x4cl_four_complex_djbunfft rsi, 64, 32*64, 64*64, rdi, 32*1*XMM_SCD2
	bump	rdi, 1*XMM_SCD2		;; Next sine/cosine pointer
	loops	32, c3b			;; Loop 32 times
	bump	rsi, -32*64+dist128	;; Next source pointer
	bump	rdi, -32*1*XMM_SCD2	;; Reset sine/cosine pointer
	loops	8, c3b			;; Loop 8 times
	bump	rcx, -128*64+8192+128	;; Next 8KB section to prefetch
	loops	8, c3a			;; Test loop counter
	bump	rsi, -8*8*dist128	;; Restore source pointer
	end_timer 12

ELSE					;; Always faster to do merged loops

	start_timer 10
	mov	rdi, xsincos_complex	;; Load sin/cos pointers
	bump	rdi, 4096*XMM_SCD2
	mov	rbx, rdi
	mov	rbp, rdi
	loops_init_prefetch 2048, 64, 2, rcx ;; 8*2*8 iters of 64 cache lines
	r4_x4cl_four_complex_djbunfft_preload

c5a:	xtouch	[rcx+4096-128]		;; Load six prefetch TLBs
	xtouch	[rcx+2*4096-128]
	xtouch	[rcx+3*4096-128]
	xtouch	[rcx+4*4096-128]
	xtouch	[rcx+5*4096-128]
	xtouch	[rcx+6*4096-128]

c5b:	r4_x4cl_four_complex_djbunfft rsi, 8*64, 2*64, 4*64, rbp, 2*16*XMM_SCD2
	loops	16, c5b			;; Loop 16 times
	bump	rsi, -16*8*64		;; Restore source pointer
	bump	rcx, -8*64+8192+128	;; Prefetch from next section
	loops_undo 16			;; Restore loop counter

c4b:	r4_x4cl_four_complex_djbunfft rsi, 2*64, 8*64, 16*64, rbx, 8*4*XMM_SCD2
	bump	rbx, 2*4*XMM_SCD2	;; Next sine/cosine pointer
	loops	4, c4b			;; Loop 4 times
	bump	rsi, -4*2*64+32*64	;; Next source pointer
	bump	rbx, -4*2*4*XMM_SCD2	;; Restore sine/cosine pointer
	loops	4, c4b			;; Loop 4 times
	bump	rsi, -4*32*64		;; Restore source pointer
	bump	rcx, -8*64+8192+128	;; Prefetch from next section
	loops_undo 4*4			;; Restore loop counter

c3b:	r4_x4cl_four_complex_djbunfft rsi, 2*64, 32*64, 64*64, rdi, 32*1*XMM_SCD2
	bump	rdi, 2*XMM_SCD2		;; Next sine/cosine pointer
	loops	16, c3b			;; Loop 16 times
	bump	rcx, -2*(8192+128)	;; Back to prefetching from 1st section

	bump	rsi, -16*2*64+64	;; Next source pointer
	bump	rdi, -16*2*XMM_SCD2+XMM_SCD2 ;; Next sine/cosine pointer
	bump	rbx, 4*XMM_SCD2		;; Next sine/cosine pointer
	bump	rbp, 16*XMM_SCD2	;; Next sine/cosine pointer
	loops	2, c5b			;; Loop twice
	bump	rsi, -2*64+dist128	;; Next source pointer
	bump	rdi, -2*XMM_SCD2	;; Restore sine/cosine pointer
	bump	rbx, -2*4*XMM_SCD2	;; Restore sine/cosine pointer
	bump	rbp, -2*16*XMM_SCD2	;; Restore sine/cosine pointer
	loops	8, c5b			;; Loop 8 times
	bump	rcx, -16*8*64+3*(8192+128) ;; Prefetch next 3 sections
	loops	8, c5a			;; Loop 8 times
	bump	rsi, -64*dist128	;; Restore source pointer
	end_timer 10
ENDIF

;; Do inverse FFT levels 5,6
;; On input the 64-byte cache lines hold these data values:
;;	0	+32K	128	+32K	256	+32K	384	+32K
;;	1	...
;;	...
;;	127	...
;;	512	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+32K	512	+32K	1024	+32K	1536	+32K
;;	1	...
;;	...
;;	511	...
;;	2048	...
;;	...

;; Do 8192 four_complex_unfft operations
;;	distance between fft data elements is 256
;;	do 2048 macros each processing 32 data values

;; Do inverse FFT levels 3,4
;; On input the 64-byte cache lines hold these data values:
;;	0	+32K	512	+32K	1024	+32K	1536	+32K
;;	1	...
;;	...
;;	511	...
;;	2048	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+32K	2048	+32K	4096	+32K	6144	+32K
;;	1	...
;;	...
;;	2047	...
;;	8192	...
;;	...

;; Do 8192 four_complex_unfft operations
;;	distance between fft data elements is 1024
;;	do 2048 macros each processing 32 data values

;; Do inverse FFT levels 1,2
;; On input the 64-byte cache lines hold these data values:
;;	0	+32K	2048	+32K	4096	+32K	6144	+32K
;;	1	...
;;	...
;;	2047	...
;;	8192	...
;;	...
;; On output the 64-byte cache lines hold these data values if pass 1 swizzles:
;;	0	+32K	8192	+32K	16384	+32K	24576	+32K
;;	1	...
;;	...
;;	8191	...
;; On output the 64-byte cache lines hold these data values if pass 2 swizzles:
;;	0	+8K	16384	+8K	32768	+8K	49152	+8K
;;	1	...
;;	...
;;	8191	...

;; Do 8192 four_complex_unfft operations
;;	distance between fft data elements is 1024
;;	do 2048 macros each processing 32 data values

IF @INSTR(,%xarch,<CORE>) NE 0		;; Core 2 faster doing separate loops

	start_timer 13
	mov	rdi, xsincos_complex	;; Load sin/cos pointer
	loops_init_prefetch 2048, 64, 2, rcx;; 8 iters of 2 iters of 128
	r4_x4cl_four_complex_djbunfft_preload
c2a:	xtouch	[rcx+4096-64]		;; Load prefetch TLB
c2b:	r4_x4cl_four_complex_djbunfft rsi, 64, dist128, 2*dist128, rdi, 128*16*XMM_SCD2
	bump	rdi, 16*XMM_SCD2	;; Next sine/cosine pointer
	loops	128, c2b		;; Loop 128 times
	bump	rsi, -128*64+4*dist128	;; Next source pointer
	bump	rdi, -128*16*XMM_SCD2	;; Reset sine/cosine pointer
	loops	2, c2a			;; Test loop counter
	bump	rcx, -128*64+8192+128	;; Next 8KB section to prefetch
	loops	8, c2a			;; Test loop counter
	bump	rsi, -8*2*4*dist128	;; Restore source pointer
	end_timer 13

	start_timer 14
	loops_init 2048			;; 8 iters of 2 iters of 128
	r4_x4cl_four_complex_djbunfft_preload
c7b:	r4_x4cl_four_complex_djbunfft rsi, 64, 4*dist128, 8*dist128, rdi, 512*4*XMM_SCD2
	bump	rdi, 4*XMM_SCD2		;; Next sine/cosine pointer
	loops	128, c7b		;; Loop 128 times
	bump	rsi, -128*64+dist128	;; Next source pointer
	loops	4, c7b			;; Loop 4 times
	bump	rsi, -4*dist128+16*dist128 ;; Next source pointer
	bump	rdi, -512*4*XMM_SCD2	;; Reset sine/cosine pointer
	loops	4, c7b			;; Test loop counter
	bump	rsi, -4*16*dist128	;; Restore source pointer
	end_timer 14

	start_timer 15
	mov	rdi, xsincos_complex	;; Load sin/cos pointer
	loops_init 2048			;; 16 iters of 128
	r4_x4cl_four_complex_djbunfft_preload
c1b:	r4_x4cl_four_complex_djbunfft rsi, 64, 16*dist128, 32*dist128, rdi, 2048*XMM_SCD2
	bump	rdi, XMM_SCD2		;; Next set of sin/cos data
	loops	128, c1b		;; Test inner loop counter
	bump	rsi, -128*64+dist128	;; Next source pointer
	loops	16, c1b			;; Test outer loop counter
	end_timer 15

ELSE					;; Pentium 4 faster doing merged loops

	start_timer 13
	mov	rdi, xsincos_complex	;; Load sin/cos pointer
	mov	rbp, rdi
	mov	rdx, rdi
	loops_init_prefetch 2048, 64, 2, rcx ;; 128 iters of 16
	r4_x4cl_four_complex_djbunfft_preload

c2a:	xtouch	[rcx+4096-64]		;; Load prefetch TLB
c2b:	r4_x4cl_four_complex_djbunfft rsi, 4*dist128, dist128, 2*dist128, rdx, 128*16*XMM_SCD2
	loops	16, c2b			;; Test loop counter
	bump	rsi, -16*4*dist128	;; Restore source pointer
	loops_undo 16

c7b:	r4_x4cl_four_complex_djbunfft rsi, dist128, 4*dist128, 8*dist128, rbp, 512*4*XMM_SCD2
	bump	rbp, 128*4*XMM_SCD2	;; Next sine/cosine pointer
	loops_no_prefetch 4, c7b	;; Loop 4 times
	bump	rsi, -4*dist128+16*dist128 ;; Next source pointer
	bump	rbp, -4*128*4*XMM_SCD2	;; Reset sine/cosine pointer
	loops_no_prefetch 4, c7b	;; Test loop counter
	bump	rsi, -4*16*dist128	;; Restore source pointer
	loops_undo 16

c1b:	r4_x4cl_four_complex_djbunfft rsi, dist128, 16*dist128, 32*dist128, rdi, 2048*XMM_SCD2
	bump	rdi, 128*XMM_SCD2	;; Next set of sin/cos data
	loops_no_prefetch 16, c1b	;; Test loop counter

	bump	rsi, -16*dist128+64	;; Next source pointer
	bump	rdi, -16*128*XMM_SCD2+XMM_SCD2 ;; Next set of sin/cos data
	bump	rbp, 4*XMM_SCD2		;; Next sine/cosine pointer
	bump	rdx, 16*XMM_SCD2	;; Next sine/cosine pointer
	loops	16, c2b			;; Loop 128 times
	bump	rcx, 128		;; Next 8KB section to prefetch
	loops	8, c2a			;; Loop 128 times
	end_timer 13

ENDIF

xpass2_14_levels_complex_done:
	ENDM


; *************** radix4-pass2-20480-levels-real macro ******************
; This macro takes 40960 real values and performs the final 15.322 levels of
; the FFT.  Independently, this macro also takes 20480 complex values and
; performs the final 14.322 levels of the FFT.  The results are squared, and
; the first 14.322/15.322 levels of the inverse FFT are done.

r4_pass2_20480_levels_real MACRO
	LOCAL	b1b, d2b, d3b, b2b0, b2b, b3b0, b3b, b4b, b5a, b5b, b6b, b7b
	LOCAL	b8b, b9b, baa, bab0, bab, bac, bad0, bad
	LOCAL	c1b, e2b, e3b, c2b, c3b, c4b, c5a, c5b, c6b, c7a, c7b, c8b, c9b
	LOCAL	xpass2_20480_levels_real_1, xpass2_20480_levels_real_2
	LOCAL	xpass2_20480_levels_real_3, xpass2_20480_levels_real_4
	LOCAL	xpass2_20480_real_unfft, xpass2_20480_real_done

;; We switch to the same format used in one-pass FFTs (that is, the
;; high word of xmm registers being 1 greater than the low word):

;; Type 4 FFTs skip the forward FFT process

	start_timer 2
	mov	rsi, DATA_ADDR		;; Load source address
	mov	rbx, DIST_TO_FFTSRCARG
	cmp	ffttype, 4
	je	xpass2_20480_levels_real_4

;; Do FFT level 1,2 (actually 2.322 levels)
;;
;; On input the 64-byte cache lines hold these data values if pass 2 swizzles:
;;	0	+10K	20480	+10K	40960	+10K	61440	+10K
;;	1	...
;;	...
;;	10239	...
;; On input the 64-byte cache lines hold these data values if pass 1 swizzles:
;;	0	+40K	10240	+40K	20480	+40K	30720	+40K
;;	1	...
;;	...
;;	10239	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+40K	2048	+40K	4096	+40K	6144	+40K
;;	1	...
;;	...
;;	2047	...
;;	8192	...
;;	...

;; Do 4096 ten_reals_fft operations
;; Do 4096 five_complex_djbfft operations
;;	distance between fft data elements is 4096
;;	do 2048 iterations

	mov	rdi, xsincos_complex	;; Ptr to the sin/cos data
	mov	rdx, sincos3		;; Ptr to real sin/cos data
	loops_init 2048
b1b:	r5_fh5cl_ten_reals_five_complex_djbfft rsi, 64, 16*dist128, rdi, 2048*XMM_SCD2, rdx, 2048*XMM_HSCD2
	bump	rdi, XMM_SCD2		;; Next sin/cos data
	bump	rdx, XMM_HSCD2		;; Next sin/cos data
	loops	128, b1b		;; Test loop counter
	bump	rsi, -128*64+dist128	;; Next source pointer
	loops	16, b1b			;; Test loop counter
	bump	rsi, -16*dist128	;; Restore source pointer
	bump	rdi, -2048*XMM_SCD2	;; Restore sin/cos ptr
	bump	rdx, -2048*XMM_HSCD2+4096*XMM_HSCD2 ;; Next sin/cos ptr

;; Do FFT levels 3,4
;;
;; On input the 64-byte cache lines hold these data values:
;;	0	+40K	2048	+40K	4096	+40K	6144	+40K
;;	1	...
;;	...
;;	2047	...
;;	8192	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+40K	1024	+40K	2048	+40K	3072	+40K
;;	1	...
;;	...
;;	1023	...
;;	4096	...
;;	...

;; Do 1024 eight_reals_fft operations
;; Do 1024 four_complex_djbfft operations
;;	distance between fft data elements is 1024
;;	do 1024 macros each processing 16 data values

	mov	rcx, DATA_PREFETCH	;; Prefetch pointer
	xtouch	[rcx]			;; Preload first TLB
	loops_init 5120
d2b:	r4_h2cl_eight_reals_four_complex_djbfft rsi, 64, 8*dist128, rdi, rdx
	bump	rdi, 5*XMM_SCD2		;; Next sin/cos data
	bump	rdx, XMM_HSCD2		;; Next sin/cos data
	loops	128, d2b		;; Test inner loop counter
	bump	rsi, -128*64+dist128	;; Next source pointer
	loops	8, d2b			;; Test loop counter
	bump	rsi, -8*dist128+16*dist128 ;; Next source pointer
	bump	rdi, -1024*5*XMM_SCD2	;; Restore sin/cos ptr
	bump	rdx, -1024*XMM_HSCD2	;; Restore sin/cos ptr

;; Do 8192 four_complex_fft operations
;;	distance between fft data elements is 1024
;;	do 4096 macros each processing 16 data values

	loops_reset			;; 4 more iters of 1024
	r4_x2cl_four_complex_djbfft_preload
d3b:	r4_x2cl_four_complex_djbfft rsi, 64, 8*dist128, rdi
	bump	rdi, 5*XMM_SCD2		;; Next sin/cos data
	loops	128, d3b		;; Test inner loop counter
	bump	rsi, -128*64+dist128	;; Next source pointer
	loops	8, d3b			;; Test loop counter
	bump	rsi, -8*dist128+16*dist128 ;; Next source pointer
	bump	rdi, -1024*5*XMM_SCD2	;; Restore sin/cos ptr
	loops	5, d3b			;; Test loop counter
	bump	rsi, -5*16*dist128	;; Restore source pointer

;; Do FFT levels 5,6
;;
;; On input the 64-byte cache lines hold these data values:
;;	0	+40K	1024	+40K	2048	+40K	3072	+40K
;;	1	...
;;	...
;;	1023	...
;;	4096	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+40K	256	+40K	512	+40K	768	+40K
;;	1	...
;;	...
;;	255	...
;;	1024	...
;;	...

;; Do 256 eight_reals_fft operations
;; Do 768 four_complex_djbfft operations
;;	distance between fft data elements is 256
;;	do 256 macros each processing 32 data values

	loops_init_prefetch 2560, 64, 1, rcx
	r4_h4cl_eight_reals_four_complex_djbfft_preload
b2b0:	xtouch	[rcx+4096-64]		;; Preload the TLB
b2b:	r4_h4cl_eight_reals_four_complex_djbfft rsi, 64, 2*dist128, 4*dist128, rdi, rdx
	bump	rdi, 20*XMM_SCD2	;; Next sin/cos data
	bump	rdx, 4*XMM_HSCD2	;; Next sin/cos data
	loops_touch 64, b2b, 2, b2b0	;; Test inner loop counter
	bump	rsi, -128*64+dist128	;; Next source pointer
	loops	2, b2b0			;; Test loop counter
	bump	rsi, -2*dist128+8*dist128 ;; Next source pointer
	bump	rdi, -256*20*XMM_SCD2	;; Restore sin/cos ptr
	bump	rdx, -256*4*XMM_HSCD2	;; Restore sin/cos ptr

;; Do 9216 four_complex_fft operations
;;	distance between fft data elements is 256
;;	do 2304 macros each processing 32 data values

	loops_reset
	r4_x4cl_four_complex_djbfft_preload
b3b0:	xtouch	[rcx+4096-64]		;; Preload the TLB
b3b:	r4_x4cl_four_complex_djbfft rsi, 64, 2*dist128, 4*dist128, rdi
	bump	rdi, 20*XMM_SCD2	;; Next sin/cos data
	loops_touch 64, b3b, 2, b3b0	;; Test inner loop counter
	bump	rsi, -128*64+dist128	;; Next source pointer
	loops	2, b3b0			;; Test loop counter
	bump	rsi, -2*dist128+8*dist128 ;; Next source pointer
	bump	rdi, -256*20*XMM_SCD2	;; Restore sin/cos ptr
	loops	10, b3b0		;; Test loop counter
	bump	rsi, -10*8*dist128	;; Restore source pointer

;; Do FFT levels 7,8
;;
;; On input the 64-byte cache lines hold these data values:
;;	0	+40K	256	+40K	512	+40K	768	+40K
;;	1	...
;;	...
;;	255	...
;;	1024	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+40K	64	+40K	128	+40K	192	+40K
;;	1	...
;;	...
;;	63	...
;;	256	...
;;	...

;; Do 64 eight_reals_fft operations
;; Do 192 four_complex_djbfft operations
;;	distance between fft data elements is 64
;;	do 64 macros each processing 32 data values

	bump	rdi, 5120*XMM_SCD2	;; Load sin/cos pointer
	loops_init_prefetch 2560, 64, 1, rcx
	xtouch	[rcx+4096-64]		;; Preload the TLBs
	r4_h4cl_eight_reals_four_complex_djbfft_preload
b4b:	r4_h4cl_eight_reals_four_complex_djbfft rsi, 64, 64*64, dist128, rdi, rdx
	bump	rdi, XMM_SCD2		;; Next sin/cos data
	bump	rdx, 16*XMM_HSCD2	;; Next sin/cos data
	loops	64, b4b			;; Test loop counter
	bump	rsi, -64*64+2*dist128	;; Next source pointer
	bump	rdi, -64*XMM_SCD2	;; Restore sin/cos ptr
	bump	rdx, -64*16*XMM_HSCD2	;; Restore sin/cos ptr

;; Do 9984 four_complex_djbfft operations
;;	distance between fft data elements is 64
;;	do 2496 macros each processing 32 data values

	loops_reset			;; 39 more iterations of 64
	r4_x4cl_four_complex_djbfft_preload
b5a:	xtouch	[rcx+4096-64]		;; Preload the TLBs
b5b:	r4_x4cl_four_complex_djbfft rsi, 64, 64*64, dist128, rdi
	bump	rdi, XMM_SCD2		;; Next sin/cos data
	loops	64, b5b			;; Test inner loop counter
	bump	rsi, -64*64+2*dist128	;; Next source pointer
	bump	rdi, -64*XMM_SCD2	;; Restore sin/cos ptr
	loops	40, b5a			;; Test loop counter
	bump	rsi, -40*2*dist128	;; Restore source pointer

;; Do FFT levels 9,10
;;
;; On input the 64-byte cache lines hold these data values:
;;	0	+40K	64	+40K	128	+40K	192	+40K
;;	1	...
;;	...
;;	63	...
;;	256	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+40K	16	+40K	32	+40K	48	+40K
;;	1	...
;;	...
;;	15	...
;;	64	...
;;	...

;; Do 16 eight_reals_fft operations
;; Do 48 four_complex_djbfft operations
;;	distance between fft data elements is 16
;;	do 16 macros each processing 32 data values

	loops_init 2560
	r4_h4cl_eight_reals_four_complex_djbfft_preload
b6b:	r4_h4cl_eight_reals_four_complex_djbfft rsi, 64, 16*64, 32*64, rdi, rdx
	bump	rdi, 4*XMM_SCD2		;; Next sin/cos data
	bump	rdx, 64*XMM_HSCD2	;; Next sin/cos data
	loops	16, b6b			;; Test loop counter
	bump	rsi, -16*64+64*64	;; Next source pointer
	bump	rdi, -16*4*XMM_SCD2	;; Restore sin/cos ptr
	bump	rdx, -16*64*XMM_HSCD2	;; Restore sin/cos ptr

;; Do 10176 four_complex_djbfft operations
;;	distance between fft data elements is 16
;;	do 2544 macros each processing 32 data values

	loops_reset			;; 80 iters of 1 or 2 of 16
	r4_x4cl_four_complex_djbfft_preload
b7b:	r4_x4cl_four_complex_djbfft rsi, 64, 16*64, 32*64, rdi
	bump	rdi, 4*XMM_SCD2		;; Next sin/cos data
	loops	16, b7b			;; Test inner loop counter
	bump	rsi, -16*64+64*64	;; Next source pointer
	bump	rdi, -16*4*XMM_SCD2	;; Restore sin/cos ptr
	loops	2, b7b			;; Test loop counter
	bump	rsi, -2*64*64+dist128	;; Next source pointer
	loops	80, b7b			;; 80 iterations
	bump	rsi, -80*dist128	;; Next source pointer

;; Do FFT levels 11,12
;;
;; On input the 64-byte cache lines hold these data values:
;;	0	+40K	16	+40K	32	+40K	48	+40K
;;	1	...
;;	...
;;	15	...
;;	64	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+40K	4	+40K	8	+40K	12	+40K
;;	1	...
;;	...
;;	3	...
;;	16	...
;;	...

;; Do 4 eight_reals_fft operations
;; Do 12 four_complex_djbfft operations
;;	distance between fft data elements is 4
;;	do 4 macros each processing 32 data values

	loops_init 2560
	r4_h4cl_eight_reals_four_complex_djbfft_preload
b8b:	r4_h4cl_eight_reals_four_complex_djbfft rsi, 64, 4*64, 8*64, rdi, rdx
	bump	rdi, 16*XMM_SCD2	;; Next sin/cos data
	bump	rdx, 256*XMM_HSCD2	;; Next sin/cos data
	loops	4, b8b			;; Test loop counter
	bump	rsi, -4*64+16*64	;; Next source pointer
	bump	rdi, -4*16*XMM_SCD2	;; Restore sin/cos ptr
	bump	rdx, -4*256*XMM_HSCD2	;; Restore sin/cos ptr

;; Do 10224 four_complex_djbfft operations
;;	distance between fft data elements is 4
;;	do 2556 macros each processing 32 data values

	loops_reset			;; 80 iters of 7 or 8 of 4
	r4_x4cl_four_complex_djbfft_preload
b9b:	r4_x4cl_four_complex_djbfft rsi, 64, 4*64, 8*64, rdi
	bump	rdi, 16*XMM_SCD2	;; Next sin/cos data
	loops	4, b9b			;; Test loop counter
	bump	rsi, -4*64+16*64	;; Next source pointer
	bump	rdi, -4*16*XMM_SCD2	;; Restore sin/cos ptr
	loops	8, b9b			;; Test middle loop counter
	bump	rsi, -8*16*64+dist128	;; Next source pointer
	loops	80, b9b			;; 80 iterations
	bump	rsi, -80*dist128	;; Restore source pointer

;; Do FFT levels 13,14
;;
;; On input the 64-byte cache lines hold these data values:
;;	0	+40K	4	+40K	8	+40K	12	+40K
;;	1	...
;;	...
;;	3	...
;;	16	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+40K	2	+40K	4	+40K	6	+40K
;;	1	...
;;	8	...
;;	...

;; Do 1 eight_reals_fft operations
;; Do 3 four_complex_fft operations
;;	distance between fft data elements is 1

;; Do 10236 four_complex_fft operations
;;	distance between fft data elements is 1
;;	do 2559 macros each processing 32 data values

;; Execute the proper middle step

	cmp	ffttype, 2
	jl	xpass2_20480_levels_real_1
	jg	xpass2_20480_levels_real_3

xpass2_20480_levels_real_2:
	r4_h4cl_eight_reals_four_complex_with_square rsi, 4*64, 64, 2*64
	loops_init 2560, 1, 32
	r4_x4cl_four_complex_with_square_preload
baa:	r4_x4cl_four_complex_with_square rsi, 4*64, 64, 2*64
	loops	32, baa			;; Test loop counter
	bump	rsi, -32*4*64+dist128	;; Next source pointer
	loops	80, baa			;; Test loop counter
	bump	rsi, -80*dist128	;; Restore source pointer
	jmp	xpass2_20480_real_unfft

xpass2_20480_levels_real_1:
	r4_h4cl_eight_reals_four_complex_fft_final rsi, 4*64, 64, 2*64
	L2prefetch128 [rcx]
	bump	rcx, 128
	loops_init_prefetch 2560, 128, 1, rcx, 1, 32
bab0:	xtouch	[rcx+4096-64]		;; Preload the TLBs
bab:	r4_x4cl_four_complex_fft_final rsi, 4*64, 64, 2*64
	loops	32, bab			;; Test loop counter
	bump	rsi, -32*4*64+dist128	;; Next source pointer
	loops	80, bab0		;; Test loop counter
	jmp	xpass2_20480_real_done

xpass2_20480_levels_real_3:
	mov	rbp, DIST_TO_MULSRCARG
	r4_h4cl_eight_reals_four_complex_with_mult rsi, 4*64, 64, 2*64
	loops_init 2560, 1, 32
bac:	r4_x4cl_four_complex_with_mult rsi, 4*64, 64, 2*64
	loops	32, bac			;; Test loop counter
	bump	rsi, -32*4*64+dist128	;; Next source pointer
	loops	80, bac			;; Test loop counter
	bump	rsi, -80*dist128	;; Restore source pointer
	jmp	xpass2_20480_real_unfft

xpass2_20480_levels_real_4:
	mov	rbp, DIST_TO_MULSRCARG
	mov	rcx, DATA_PREFETCH	;; Load prefetch pointer
	xtouch	[rcx]			;; Preload the TLBs
	r4_h4cl_eight_reals_four_complex_with_mulf rsi, 4*64, 64, 2*64
	L2prefetch128 [rcx]
	bump	rcx, 128
	loops_init_prefetch 2560, 128, 1, rcx, 1, 32
bad0:	xtouch	[rcx+4096-64]		;; Preload the TLBs
bad:	r4_x4cl_four_complex_with_mulf rsi, 4*64, 64, 2*64
	loops	32, bad			;; Test loop counter
	bump	rsi, -32*4*64+dist128	;; Next source pointer
	loops	80, bad0		;; Test loop counter
	bump	rsi, -80*dist128	;; Restore source pointer

;; Do inverse FFT levels 11,12
;; On input the 64-byte cache lines hold these data values:
;;	0	+40K	2	+40K	4	+40K	6	+40K
;;	1	...
;;	8	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+40K	8	+40K	16	+40K	24	+40K
;;	1	...
;;	...
;;	7	...
;;	32	...
;;	...

;; Do 4 eight_reals_unfft operations
;; Do 4 four_complex_djbunfft operations
;;	distance between fft data elements is 4
;;	do 2 macros processing 32 data values

xpass2_20480_real_unfft:
	mov	rdi, xsincos_complex	;; Load sin/cos pointers
	bump	rdi, 5120*XMM_SCD2
	mov	rdx, sincos3
	bump	rdx, 4096*XMM_HSCD2
	loops_init 2560
c8b:	r4_h4cl_eight_reals_four_complex_djbunfft rsi, 64, 2*64, 4*64, rdi, 2*16*XMM_SCD2, rdx, 2*256*XMM_HSCD2
	bump	rdi, 16*XMM_SCD2	;; Next sin/cos data
	bump	rdx, 256*XMM_HSCD2	;; Next sin/cos data
	loops	2, c8b			;; Test loop counter
	bump	rsi, -2*64+8*64		;; Next source pointer
	bump	rdi, -2*16*XMM_SCD2	;; Restore sin/cos ptr
	bump	rdx, -2*256*XMM_HSCD2	;; Restore sin/cos ptr

;; Do 10232 four_complex_djbunfft operations
;;	distance between fft data elements is 4
;;	do 2558 macros each processing 32 data values

	loops_reset			;; 48 iters of 15 or 16 of 2
	r4_x4cl_four_complex_djbunfft_preload
c9b:	r4_x4cl_four_complex_djbunfft rsi, 64, 2*64, 4*64, rdi, 2*16*XMM_SCD2
	bump	rdi, 16*XMM_SCD2	;; Next sine/cosine pointer
	loops	2, c9b			;; Test loop counter
	bump	rsi, -2*64+8*64		;; Next source pointer
	bump	rdi, -2*16*XMM_SCD2	;; Restore sin/cos ptr
	loops	16, c9b			;; Test loop counter
	bump	rsi, -16*8*64+dist128	;; Next source pointer
	loops	80, c9b			;; Test loop counter
	bump	rsi, -80*dist128	;; Restore source pointer

;; Do inverse FFT levels 9,10
;; On input the 64-byte cache lines hold these data values:
;;	0	+40K	8	+40K	16	+40K	24	+40K
;;	1	...
;;	...
;;	7	...
;;	32	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+40K	32	+40K	64	+40K	96	+40K
;;	1	...
;;	...
;;	31	...
;;	128	...
;;	...

;; Do 16 eight_reals_unfft operations
;; Do 16 four_complex_djbunfft operations
;;	distance between fft data elements is 16
;;	do 8 macros processing 32 data values

	loops_init_prefetch 2560, 64, 1, rcx
	xtouch	[rcx+4096-64]		;; Preload the TLBs
c6b:	r4_h4cl_eight_reals_four_complex_djbunfft rsi, 64, 8*64, 16*64, rdi, 8*4*XMM_SCD2, rdx, 8*64*XMM_HSCD2
	bump	rdi, 4*XMM_SCD2		;; Next sin/cos data
	bump	rdx, 64*XMM_HSCD2	;; Next sin/cos data
	loops	8, c6b			;; Test inner loop counter
	bump	rsi, -8*64+32*64	;; Next source pointer
	bump	rdi, -8*4*XMM_SCD2	;; Restore sin/cos ptr
	bump	rdx, -8*64*XMM_HSCD2	;; Restore sin/cos ptr

;; Do 10208 four_complex_djbunfft operations
;;	distance between fft data elements is 16
;;	do 2552 macros each processing 32 data values

	loops_reset			;; 80 iters of 3 or 4 of 4
	r4_x4cl_four_complex_djbunfft_preload
c7a:	xtouch	[rcx+4096-64]		;; Preload the TLBs
c7b:	r4_x4cl_four_complex_djbunfft rsi, 64, 8*64, 16*64, rdi, 8*4*XMM_SCD2
	bump	rdi, 4*XMM_SCD2		;; Next sin/cos data
	loops	8, c7b			;; Test inner loop counter
	bump	rsi, -8*64+32*64	;; Next source pointer
	bump	rdi, -8*4*XMM_SCD2	;; Restore sin/cos ptr
	loops	4, c7b			;; Test loop counter
	bump	rsi, -4*32*64+dist128	;; Next source pointer
	loops_touch 2, c7b, 40, c7a	;; Test loop counter
	bump	rsi, -80*dist128	;; Restore source pointer

;; Do inverse FFT levels 7,8
;; On input the 64-byte cache lines hold these data values:
;;	0	+40K	32	+40K	64	+40K	96	+40K
;;	1	...
;;	...
;;	31	...
;;	128	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+40K	128	+40K	256	+40K	384	+40K
;;	1	...
;;	...
;;	127	...
;;	512	...
;;	...

;; Do 64 eight_reals_unfft operations
;; Do 64 four_complex_djbunfft operations
;;	distance between fft data elements is 64
;;	do 32 macros processing 32 data values

	loops_init_prefetch 2560, 64, 1, rcx
	xtouch	[rcx+4096-64]		;; Preload the TLBs
c4b:	r4_h4cl_eight_reals_four_complex_djbunfft rsi, 64, 32*64, 64*64, rdi, 32*XMM_SCD2, rdx, 32*16*XMM_HSCD2
	bump	rdi, XMM_SCD2		;; Next sin/cos data
	bump	rdx, 16*XMM_HSCD2	;; Next sin/cos data
	loops	32, c4b			;; Test inner loop counter
	bump	rsi, -32*64+dist128	;; Next source pointer
	bump	rdi, -32*XMM_SCD2	;; Restore sin/cos ptr
	bump	rdx, -32*16*XMM_HSCD2	;; Restore sin/cos ptr

;; Do 10112 four_complex_djbunfft operations
;;	distance between fft data elements is 64
;;	do 2528 macros each processing 32 data values

	loops_reset			;; 79 more iters of 32
	r4_x4cl_four_complex_djbunfft_preload
c5a:	xtouch	[rcx+4096-64]		;; Preload the TLBs
c5b:	r4_x4cl_four_complex_djbunfft rsi, 64, 32*64, 64*64, rdi, 32*XMM_SCD2
	bump	rdi, XMM_SCD2		;; Next sin/cos data
	loops	32, c5b			;; Test inner loop counter
	bump	rsi, -32*64+dist128	;; Next source pointer
	bump	rdi, -32*XMM_SCD2	;; Restore sin/cos ptr
	loops_touch 2, c5b, 40, c5a	;; Test loop counter
	bump	rsi, -80*dist128	;; Next source pointer

;; Do inverse FFT levels 5,6
;; On input the 64-byte cache lines hold these data values:
;;	0	+40K	128	+40K	256	+40K	384	+40K
;;	1	...
;;	...
;;	127	...
;;	512	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+40K	512	+40K	1024	+40K	1536	+40K
;;	1	...
;;	...
;;	511	...
;;	2048	...
;;	...

;; Do 256 eight_reals_unfft operations
;; Do 256 four_complex_djbunfft operations
;;	distance between fft data elements is 256
;;	do 128 macros processing 32 data values

	mov	rdi, xsincos_complex	;; Load sin/cos pointer
	mov	rbx, rdi
	bump	rcx, -2*64		;; Back up prefetch pointer by 2 cache lines
	loops_init_prefetch 2560, 64, 16, rcx
	xtouch	[rcx+4096-64]		;; Load prefetch TLBs
	xtouch	[rcx+2*4096-64]
	xtouch	[rcx+160*64-64]
c2b:	r4_h4cl_eight_reals_four_complex_djbunfft rsi, 64, dist128, 2*dist128, rdi, 128*20*XMM_SCD2, rdx, 128*4*XMM_HSCD2
	bump	rdi, 20*XMM_SCD2	;; Next sin/cos data
	bump	rdx, 4*XMM_HSCD2	;; Next sin/cos data
	loops	128, c2b		;; Test inner loop counter
	bump	rsi, -128*64+4*dist128	;; Next source pointer
	bump	rdi, -128*20*XMM_SCD2	;; Restore sin/cos ptr
	bump	rdx, -128*4*XMM_HSCD2	;; Restore sin/cos ptr

;; Do 9728 four_complex_djbunfft operations
;;	distance between fft data elements is 256
;;	do 2432 macros each processing 32 data values

	loops_reset			;; 19 more iters of 128
	r4_x4cl_four_complex_djbunfft_preload
c3b:	r4_x4cl_four_complex_djbunfft rsi, 64, dist128, 2*dist128, rdi, 128*20*XMM_SCD2
	bump	rdi, 20*XMM_SCD2	;; Next sin/cos data
	loops	128, c3b		;; Test inner loop counter
	bump	rsi, -128*64+4*dist128	;; Next source pointer
	bump	rdi, -128*20*XMM_SCD2	;; Restore sin/cos ptr
	loops	20, c3b			;; Test loop counter
	bump	rsi, -20*4*dist128	;; Next source pointer

;; Do inverse FFT levels 3,4
;; On input the 64-byte cache lines hold these data values:
;;	0	+40K	512	+40K	1024	+40K	1536	+40K
;;	1	...
;;	...
;;	511	...
;;	2048	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+40K	2048	+40K	4096	+40K	6144	+40K
;;	1	...
;;	...
;;	2047	...
;;	8192	...
;;	...

;; Do 1024 eight_reals_unfft operations
;; Do 1024 four_complex_djbunfft operations
;;	distance between fft data elements is 1024
;;	do 512 macros processing 32 data values

	loops_init 2560
e2b:	r4_h4cl_eight_reals_four_complex_djbunfft rsi, 64, 4*dist128, 8*dist128, rdi, 512*5*XMM_SCD2, rdx, 512*XMM_HSCD2
	bump	rdi, 5*XMM_SCD2		;; Next sin/cos data
	bump	rdx, XMM_HSCD2		;; Next sin/cos data
	loops	128, e2b		;; Test inner loop counter
	bump	rsi, -128*64+dist128	;; Next source pointer
	loops	4, e2b			;; Test loop counter
	bump	rsi, -4*dist128+16*dist128 ;; Next source pointer
	bump	rdi, -512*5*XMM_SCD2	;; Restore sin/cos ptr
	bump	rdx, -512*XMM_HSCD2-4096*XMM_HSCD2 ;; Next sin/cos ptr

;; Do 8192 four_complex_djbunfft operations
;;	distance between fft data elements is 1024
;;	do 2048 macros each processing 32 data values

	loops_reset			;; 4 more iters of 512
	r4_x4cl_four_complex_djbunfft_preload
e3b:	r4_x4cl_four_complex_djbunfft rsi, 64, 4*dist128, 8*dist128, rdi, 512*5*XMM_SCD2
	bump	rdi, 5*XMM_SCD2		;; Next sin/cos data
	loops	128, e3b		;; Test inner loop counter
	bump	rsi, -128*64+dist128	;; Next source pointer
	loops	4, e3b			;; Test loop counter
	bump	rsi, -4*dist128+16*dist128 ;; Next source pointer
	bump	rdi, -512*5*XMM_SCD2	;; Restore sin/cos ptr
	loops	5, e3b			;; Test loop counter
	bump	rsi, -5*16*dist128	;; Restore source pointer

;; Do inverse FFT levels 1,2 (actually 2.322 FFT levels)
;; On input the 64-byte cache lines hold these data values:
;;	0	+40K	2048	+40K	4096	+40K	6144	+40K
;;	1	...
;;	...
;;	2047	...
;;	8192	...
;;	...
;; On output the 64-byte cache lines hold these data values if pass 1 swizzles:
;;	0	+40K	10240	+40K	20480	+40K	30720	+40K
;;	1	...
;;	...
;;	10239	...
;; On output the 64-byte cache lines hold these data values if pass 2 swizzles:
;;	0	+10K	20480	+10K	40960	+10K	61440	+10K
;;	1	...
;;	...
;;	10239	...

;; Do 4096 ten_reals_unfft operations
;; Do 4096 five_complex_djbunfft operations
;;	distance between fft data elements is 4096
;;	do 2048 macros processing 40 data values

	loops_init 2048
c1b:	r5_h5cl_ten_reals_five_complex_djbunfft rsi, 64, 16*dist128, rdi, 2048*XMM_SCD2, rdx, 2048*XMM_HSCD2
	bump	rdi, XMM_SCD2		;; Next sin/cos data
	bump	rdx, XMM_HSCD2		;; Next sin/cos data
	loops	128, c1b		;; Test loop counter
	bump	rsi, -128*64+dist128	;; Next source pointer
	loops	16, c1b			;; Test loop counter

xpass2_20480_real_done:
	end_timer 2
	ENDM

; *************** radix4-pass2-20480-levels-complex macro ******************
; This macro takes 2 groups of 20480 complex values and
; performs the final 14.322 levels of a standard radix-4 FFT, squares the
; results, and does the first 14.322 levels of the inverse FFT.

r4_pass2_20480_levels_complex MACRO
	LOCAL	b1b, b7b, b2a, b2b, b3a, b3b, b4a, b4b, b5a, b5b
	LOCAL	b6a0, b6a, b6b0, b6b, b6c0, b6c, b6d0, b6d
	LOCAL	c1b, c7b, c2a, c2b, c3a, c3b, c4a, c4b, c5a, c5b
	LOCAL	xpass2_20480_levels_complex_1, xpass2_20480_levels_complex_2
	LOCAL	xpass2_20480_levels_complex_3, xpass2_20480_levels_complex_4
	LOCAL	xpass2_20480_levels_complex_unfft, xpass2_20480_levels_complex_done

;; Type 4 FFTs skip the forward FFT process

	mov	rsi, DATA_ADDR		;; Load address of FFT data
	mov	rbx, DIST_TO_FFTSRCARG
	cmp	ffttype, 4
	je	xpass2_20480_levels_complex_4

;; Do FFT level 1,2 (actually 2.322 levels)
;;
;; We swizzle the data so that one set of data is in the high word of
;; an xmm register, the other set is in the low word.  This means
;; a distance of 10240 data values.
;;
;; On input the 64-byte cache lines hold these data values if pass 2 swizzles:
;;	0	+10K	20480	+10K	40960	+10K	61440	+10K
;;	1	...
;;	...
;;	10239	...
;; On input the 64-byte cache lines hold these data values if pass 1 swizzles:
;;	0	+40K	10240	+40K	20480	+40K	30720	+40K
;;	1	...
;;	...
;;	10239	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+40K	2048	+40K	4096	+40K	6144	+40K
;;	1	...
;;	...
;;	2047	...
;;	8192	...
;;	...

;; Do 8192 five_complex_fft operations
;;	distance between fft data elements is 4096
;;	do 2048 macros each processing 40 data values

	start_timer 3
	mov	rdi, xsincos_complex	;; The sin/cos data
	loops_init 2048			;; 16 iterations of 128
	r5_f5cl_five_complex_djbfft_preload
b1b:	xxprefetch [rdi+XMM_SCD2]	;; Prefetch for next pass
	xxprefetch [rdi+2049*XMM_SCD2]	;; Prefetch for next pass
	r5_f5cl_five_complex_djbfft rsi, 64, 16*dist128, rdi, 2048*XMM_SCD2
	bump	rdi, XMM_SCD2		;; Next sin/cos data
	loops	128, b1b		;; Test inner loop counter
	bump	rsi, -128*64+dist128	;; Next source pointer
	loops	16, b1b			;; Test outer loop counter
	bump	rsi, -16*dist128	;; Restore source pointer
	end_timer 3

;; Do FFT levels 3,4
;;
;; On input the 64-byte cache lines hold these data values:
;;	0	+40K	2048	+40K	4096	+40K	6144	+40K
;;	1	...
;;	...
;;	2047	...
;;	8192	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+40K	1024	+40K	2048	+40K	3072	+40K
;;	1	...
;;	...
;;	1023	...
;;	4096	...
;;	...

;; Do 10240 four_complex_fft operations
;;	distance between fft data elements is 1024
;;	do 5120 macros each processing 16 data values

	start_timer 4
	mov	rcx, DATA_PREFETCH	;; Prefetch pointer (we need to prefetch 
					;; 10240 cache lines + 158 pad cache lines)
	xtouch	[rcx]			;; Load first prefetch TLB
	mov	rdi, xsincos_complex	;; Ptr to the sin/cos data
	loops_init 5120			;; 3 iters of 8 iters of 128
	r4_x2cl_four_complex_djbfft_preload
b7b:	r4_x2cl_four_complex_djbfft rsi, 64, 8*dist128, rdi
	bump	rdi, 5*XMM_SCD2		;; Next sin/cos data
	loops	128, b7b		;; Test inner loop counter
	bump	rsi, -128*64+dist128	;; Next source pointer
	loops	8, b7b			;; Test inner loop counter
	bump	rsi, -8*dist128+16*dist128 ;; Next source pointer
	bump	rdi, -1024*5*XMM_SCD2	;; Restore sin/cos ptr
	loops	5, b7b			;; Test loop counter
	bump	rsi, -5*16*dist128	;; Restore source pointer
	end_timer 4

;; Do FFT levels 5,6
;;
;; On input the 64-byte cache lines hold these data values:
;;	0	+40K	1024	+40K	2048	+40K	3072	+40K
;;	1	...
;;	...
;;	1023	...
;;	4096	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+40K	256	+40K	512	+40K	768	+40K
;;	1	...
;;	...
;;	255	...
;;	1024	...
;;	...

;; Do 10240 four_complex_fft operations
;;	distance between fft data elements is 256
;;	do 2560 macros each processing 16 data values

	start_timer 5
	loops_init_prefetch 2560, 64, 2, rcx ;; 10 iters of 2 iters of 128
	r4_x4cl_four_complex_djbfft_preload
b2a:	xtouch	[rcx+4096-64]		;; Load prefetch TLB
b2b:	r4_x4cl_four_complex_djbfft rsi, 64, 2*dist128, 4*dist128, rdi
	bump	rdi, 20*XMM_SCD2	;; Next sin/cos data
	loops	128, b2b		;; Test inner loop counter
	bump	rsi, -128*64+dist128	;; Next source pointer
	loops	2, b2a			;; Test loop counter
	bump	rsi, -2*dist128+8*dist128 ;; Next source pointer
	bump	rdi, -256*20*XMM_SCD2	;; Restore sin/cos ptr
	loops	10, b2a			;; Test loop counter
	bump	rsi, -10*8*dist128	;; Restore source pointer
	end_timer 5

;; Do FFT levels 7,8
;;
;; On input the 64-byte cache lines hold these data values:
;;	0	+40K	256	+40K	512	+40K	768	+40K
;;	1	...
;;	...
;;	255	...
;;	1024	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+40K	64	+40K	128	+40K	192	+40K
;;	1	...
;;	...
;;	63	...
;;	256	...
;;	...
;;
;; Do 10240 four_complex_fft operations
;;	distance between fft data elements is 64
;;	do 2560 macros each processing 32 data values

;; Do FFT levels 9,10
;;
;; On input the 64-byte cache lines hold these data values:
;;	0	+40K	64	+40K	128	+40K	192	+40K
;;	1	...
;;	...
;;	63	...
;;	256	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+40K	16	+40K	32	+40K	48	+40K
;;	1	...
;;	...
;;	15	...
;;	64	...
;;	...
;;
;; Do 10240 four_complex_fft operations
;;	distance between fft data elements is 16
;;	do 2560 macros each processing 32 data values

;; Do FFT levels 11,12
;;
;; On input the 64-byte cache lines hold these data values:
;;	0	+40K	16	+40K	32	+40K	48	+40K
;;	1	...
;;	...
;;	15	...
;;	64	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+40K	4	+40K	8	+40K	12	+40K
;;	1	...
;;	...
;;	3	...
;;	16	...
;;	...
;;
;; Do 10240 four_complex_fft operations
;;	distance between fft data elements is 4
;;	do 2560 macros each processing 32 data values

;; Do this 64 cache lines at a time to maximize accessing data
;; while it is in the L1 cache.

	start_timer 6
	bump	rdi, 5120*XMM_SCD2	;; Load sin/cos pointers
	mov	rbx, rdi
	mov	rbp, rdi
	loops_init_prefetch 2560, 64, 2, rcx ;; 40 iters of 4 of 64 cache lines
	r4_x4cl_four_complex_djbfft_preload

b3a:	xtouch	[rcx+48*64-64]		;; Load prefetch TLB

b3b:	xxprefetch [rdi+4*XMM_SCD2]	;; Prefetch for next pass
	r4_x4cl_four_complex_djbfft rsi, 4*64, 64*64, dist128, rdi
	bump	rdi, 4*XMM_SCD2		;; Next sine/cosine pointer
	loops	16, b3b			;; Loop 16 times
	bump	rsi, -16*4*64		;; Restore source pointer
	bump	rdi, -16*4*XMM_SCD2	;; Restore sine/cosine pointer
	loops_undo 16			;; Back up loop counter

b4b:	r4_x4cl_four_complex_djbfft rsi, 4*64, 16*64, 32*64, rbx
	bump	rbx, 4*4*XMM_SCD2	;; Next sine/cosine pointer
	loops	4, b4b			;; Loop 4 times
	bump	rsi, -4*4*64+64*64	;; Next source pointer
	bump	rbx, -4*4*4*XMM_SCD2	;; Restore sine/cosine pointer
	loops	2, b4b			;; Loop twice
	bump	rsi, -2*64*64+dist128	;; Next source pointer
	loops	2, b4b			;; Loop twice
	bump	rsi, -2*dist128		;; Restore source pointer
	loops_undo 2*2*4		;; Back up loop counter

b5b:	r4_x4cl_four_complex_djbfft rsi, 16*64, 4*64, 8*64, rbp
	loops	8, b5b			;; Loop 8 times
	bump	rsi, -8*16*64+dist128	;; Next source pointer
	loops	2, b5b			;; Loop twice

	bump	rsi, -2*dist128+64	;; Next source pointer
	bump	rdi, XMM_SCD2		;; Next sine/cosine pointer
	bump	rbx, 4*XMM_SCD2		;; Next sine/cosine pointer
	bump	rbp, 16*XMM_SCD2	;; Next sine/cosine pointer
	loops_touch 2, b3b, 2, b3a	;; Loop 4 times
	bump	rsi, -4*64+2*dist128	;; Next source pointer
	bump	rdi, -4*XMM_SCD2	;; Restore sine/cosine pointer
	bump	rbx, -4*4*XMM_SCD2	;; Restore sine/cosine pointer
	bump	rbp, -4*16*XMM_SCD2	;; Restore sine/cosine pointer
	loops	40, b3a			;; Test outer loop counter
	bump	rsi, -40*2*dist128	;; Restore source pointer
	end_timer 6

;; Do FFT levels 13,14 as well as inverse FFT levels 13,14
;;
;; On input the 64-byte cache lines hold these data values:
;;	0	+40K	4	+40K	8	+40K	12	+40K
;;	1	...
;;	...
;;	3	...
;;	16	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+40K	2	+40K	4	+40K	6	+40K
;;	1	...
;;	8	...
;;	...

;; Do 10240 four_complex_fft operations
;;	distance between fft data elements is 1
;;	do 2560 macros each processing 32 data values

;; Execute the right middle step

	cmp	ffttype, 2
	jl	xpass2_20480_levels_complex_1
	jg	xpass2_20480_levels_complex_3

xpass2_20480_levels_complex_2:
	start_timer 9
	loops_init 2560
	r4_x4cl_four_complex_with_square_preload
b6a:	r4_x4cl_four_complex_with_square rsi, 4*64, 64, 2*64
	loops	32, b6a			;; Loop 32 times
	bump	rsi, -32*4*64+dist128	;; Next source pointer
	loops	80, b6a			;; Loop 80 times
	bump	rsi, -80*dist128	;; Restore source pointer
	end_timer 9
	jmp	xpass2_20480_levels_complex_unfft

xpass2_20480_levels_complex_1:
	start_timer 9
	loops_init_prefetch 2560, 128, 1, rcx
b6b0:	xtouch	[rcx+4096-128]		;; Load prefetch TLBs
b6b:	r4_x4cl_four_complex_fft_final rsi, 4*64, 64, 2*64
	loops	32, b6b			;; Loop 32 times
	bump	rsi, -32*4*64+dist128	;; Next source pointer
	loops	80, b6b0		;; Loop 80 times
	end_timer 9
	jmp	xpass2_20480_levels_complex_done

xpass2_20480_levels_complex_3:
	start_timer 9
	mov	rbp, DIST_TO_MULSRCARG
	loops_init 2560
b6c:	r4_x4cl_four_complex_with_mult rsi, 4*64, 64, 2*64
	loops	32, b6c			;; Loop 32 times
	bump	rsi, -32*4*64+dist128	;; Next source pointer
	loops	80, b6c			;; Loop 80 times
	bump	rsi, -80*dist128	;; Restore source pointer
	end_timer 9
	jmp	xpass2_20480_levels_complex_unfft

xpass2_20480_levels_complex_4:
	start_timer 9
	mov	rcx, DATA_PREFETCH	;; Prefetch pointer
	xtouch	[rcx]			;; Load first prefetch TLB
	mov	rbp, DIST_TO_MULSRCARG
	loops_init_prefetch 2560, 128, 1, rcx
b6d0:	xtouch	[rcx+4096-128]		;; Load prefetch TLBs
b6d:	r4_x4cl_four_complex_with_mulf rsi, 4*64, 64, 2*64
	loops	32, b6d			;; Loop 32 times
	bump	rsi, -32*4*64+dist128	;; Next source pointer
	loops	80, b6d0		;; Loop 80 times
	bump	rsi, -80*dist128	;; Restore source pointer
	end_timer 9

;; Do inverse FFT levels 11,12
;; On input the 64-byte cache lines hold these data values:
;;	0	+40K	2	+40K	4	+40K	6	+40K
;;	1	...
;;	8	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+40K	8	+40K	16	+40K	24	+40K
;;	1	...
;;	...
;;	7	...
;;	32	...
;;	...
;;
;; Do 10240 four_complex_unfft operations
;;	distance between fft data elements is 4
;;	do 2560 macros each processing 32 data values

;; Do inverse FFT levels 9,10
;; On input the 64-byte cache lines hold these data values:
;;	0	+40K	8	+40K	16	+40K	24	+40K
;;	1	...
;;	...
;;	7	...
;;	32	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+40K	32	+40K	64	+40K	96	+40K
;;	1	...
;;	...
;;	31	...
;;	128	...
;;	...
;;
;; Do 10240 four_complex_unfft operations
;;	distance between fft data elements is 16
;;	do 2560 macros each processing 32 data values

;; Do inverse FFT levels 7,8
;; On input the 64-byte cache lines hold these data values:
;;	0	+40K	32	+40K	64	+40K	96	+40K
;;	1	...
;;	...
;;	31	...
;;	128	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+40K	128	+40K	256	+40K	384	+40K
;;	1	...
;;	...
;;	127	...
;;	512	...
;;	...
;;
;; Do 10240 four_complex_unfft operations
;;	distance between fft data elements is 64
;;	do 2560 macros each processing 32 data values

;; Do this 64 cache lines at a time to maximize accessing data
;; while it is in the L1 cache.

xpass2_20480_levels_complex_unfft:

	start_timer 10
	mov	rdi, xsincos_complex	;; Load sin/cos pointers
	bump	rdi, 5120*XMM_SCD2
	mov	rbx, rdi
	mov	rbp, rdi
	loops_init_prefetch 2560, 64, 2, rcx ;; 80 iters of 2 iters of 64 cache lines
	r4_x4cl_four_complex_djbunfft_preload

c5a:	xtouch	[rcx+48*64-64]		;; Load prefetch TLB

c5b:	r4_x4cl_four_complex_djbunfft rsi, 8*64, 2*64, 4*64, rbp, 2*16*XMM_SCD2
	loops	16, c5b			;; Loop 16 times
	bump	rsi, -16*8*64		;; Restore source pointer
	loops_undo 16			;; Restore loop counter

c4b:	r4_x4cl_four_complex_djbunfft rsi, 2*64, 8*64, 16*64, rbx, 8*4*XMM_SCD2
	bump	rbx, 2*4*XMM_SCD2	;; Next sine/cosine pointer
	loops	4, c4b			;; Loop 4 times
	bump	rsi, -4*2*64+32*64	;; Next source pointer
	bump	rbx, -4*2*4*XMM_SCD2	;; Restore sine/cosine pointer
	loops	4, c4b			;; Loop 4 times
	bump	rsi, -4*32*64		;; Restore source pointer
	loops_undo 4*4			;; Restore loop counter

c3b:	r4_x4cl_four_complex_djbunfft rsi, 2*64, 32*64, 64*64, rdi, 32*1*XMM_SCD2
	bump	rdi, 2*XMM_SCD2		;; Next sine/cosine pointer
	loops	16, c3b			;; Loop 16 times
	bump	rdi, -16*2*XMM_SCD2	;; Restore sine/cosine pointer

	bump	rsi, -16*2*64+64	;; Next source pointer
	bump	rdi, XMM_SCD2		;; Next sine/cosine pointer
	bump	rbx, 4*XMM_SCD2		;; Next sine/cosine pointer
	bump	rbp, 16*XMM_SCD2	;; Next sine/cosine pointer
	loops	2, c5b			;; Loop twice
	bump	rsi, -2*64+dist128	;; Next source pointer
	bump	rdi, -2*XMM_SCD2	;; Restore sine/cosine pointer
	bump	rbx, -2*4*XMM_SCD2	;; Restore sine/cosine pointer
	bump	rbp, -2*16*XMM_SCD2	;; Restore sine/cosine pointer
	loops	80, c5a			;; Loop 80 times
	bump	rsi, -80*dist128	;; Restore source pointer
	end_timer 10

;; Do inverse FFT levels 5,6
;; On input the 64-byte cache lines hold these data values:
;;	0	+40K	128	+40K	256	+40K	384	+40K
;;	1	...
;;	...
;;	127	...
;;	512	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+40K	512	+40K	1024	+40K	1536	+40K
;;	1	...
;;	...
;;	511	...
;;	2048	...
;;	...

;; Do 6144 four_complex_unfft operations
;;	distance between fft data elements is 256
;;	do 1536 macros each processing 32 data values

	start_timer 13
	mov	rdi, xsincos_complex	;; Load sin/cos pointer
	loops_init_prefetch 2560, 64, 2, rcx ;; 20 iters of 128
	r4_x4cl_four_complex_djbunfft_preload
c2a:	xtouch	[rcx+4096-64]		;; Load prefetch TLB
c2b:	r4_x4cl_four_complex_djbunfft rsi, 64, dist128, 2*dist128, rdi, 128*20*XMM_SCD2
	bump	rdi, 20*XMM_SCD2	;; Next sin/cos data
	loops	128, c2b		;; Loop 128 times
	bump	rsi, -128*64+4*dist128	;; Next source pointer
	bump	rdi, -128*20*XMM_SCD2	;; Restore sin/cos ptr
	loops	20, c2a			;; Test loop counter
	bump	rsi, -20*4*dist128	;; Restore source pointer
	end_timer 13

;; Do inverse FFT levels 3,4
;; On input the 64-byte cache lines hold these data values:
;;	0	+40K	512	+40K	1024	+40K	1536	+40K
;;	1	...
;;	...
;;	511	...
;;	2048	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+40K	2048	+40K	4096	+40K	6144	+40K
;;	1	...
;;	...
;;	2047	...
;;	8192	...
;;	...

;; Do 10240 four_complex_unfft operations
;;	distance between fft data elements is 1024
;;	do 2560 macros each processing 32 data values

	start_timer 14
	bump	rcx, -2*64		;; Back up prefetch pointer by 2 cache lines
	loops_init_prefetch 2560, 64, 16, rcx ;; 5 iters of 4 iters of 128
	xtouch	[rcx+4096-64]		;; Load prefetch TLBs
	xtouch	[rcx+2*4096-64]
	xtouch	[rcx+160*64-64]
	r4_x4cl_four_complex_djbunfft_preload
c7b:	r4_x4cl_four_complex_djbunfft rsi, 64, 4*dist128, 8*dist128, rdi, 512*5*XMM_SCD2
	bump	rdi, 5*XMM_SCD2		;; Next sin/cos data
	loops	128, c7b		;; Loop 128 times
	bump	rsi, -128*64+dist128	;; Next source pointer
	loops	4, c7b			;; Loop 4 times
	bump	rsi, -4*dist128+16*dist128 ;; Next source pointer
	bump	rdi, -512*5*XMM_SCD2	;; Restore sin/cos ptr
	loops	5, c7b			;; Test loop counter
	bump	rsi, -5*16*dist128	;; Restore source pointer
	end_timer 14

;; Do inverse FFT levels 1,2 (actually 2.322 FFT levels)
;; On input the 64-byte cache lines hold these data values:
;;	0	+40K	2048	+40K	4096	+40K	6144	+40K
;;	1	...
;;	...
;;	2047	...
;;	8192	...
;;	...
;; On output the 64-byte cache lines hold these data values if pass 1 swizzles:
;;	0	+40K	10240	+40K	20480	+40K	30720	+40K
;;	1	...
;;	...
;;	10239	...
;; On output the 64-byte cache lines hold these data values if pass 2 swizzles:
;;	0	+10K	20480	+10K	40960	+10K	61440	+10K
;;	1	...
;;	...
;;	10239	...

;; Do 8192 five_complex_unfft operations
;;	distance between fft data elements is 4096
;;	do 2048 macros each processing 40 data values

	start_timer 15
	loops_init 2048			;; 16 iters of 128
	r5_x5cl_five_complex_djbunfft_preload
c1b:	r5_x5cl_five_complex_djbunfft rsi, 64, 16*dist128, rdi, 2048*XMM_SCD2
	bump	rdi, XMM_SCD2		;; Next set of sin/cos data
	loops	128, c1b		;; Test inner loop counter
	bump	rsi, -128*64+dist128	;; Next source pointer
	loops	16, c1b			;; Test outer loop counter
	end_timer 15

xpass2_20480_levels_complex_done:
	ENDM


; *************** radix4-pass2-25600-levels-real macro ******************
; This macro takes 51200 real values and performs the final 15.644 levels of
; the FFT.  Independently, this macro also takes 25600 complex values and
; performs the final 14.644 levels of the FFT.  The results are squared, and
; the first 14.644/15.644 levels of the inverse FFT are done.

r4_pass2_25600_levels_real MACRO
	LOCAL	b1b, d2b, d3b, b2b0, b2b, b3b0, b3b, b4b, b5a, b5b, b6b, b7b
	LOCAL	b8b, b9b, baa, bab0, bab, bac, bad0, bad
	LOCAL	c1b, e2b, e3b, c2b, c3b, c4b, c5a, c5b, c6b, c7a, c7b, c8b, c9b
	LOCAL	xpass2_25600_levels_real_1, xpass2_25600_levels_real_2
	LOCAL	xpass2_25600_levels_real_3, xpass2_25600_levels_real_4
	LOCAL	xpass2_25600_real_unfft, xpass2_25600_real_done

;; We switch to the same format used in one-pass FFTs (that is, the
;; high word of xmm registers being 1 greater than the low word):

;; Type 4 FFTs skip the forward FFT process

	start_timer 2
	mov	rsi, DATA_ADDR		;; Load source address
	mov	rbx, DIST_TO_FFTSRCARG
	cmp	ffttype, 4
	je	xpass2_25600_levels_real_4

;; Do FFT level 1,2 (actually 2.322 levels)
;;
;; On input the 64-byte cache lines hold these data values if pass 2 swizzles:
;;	0	+12800	25600	+12800	51200	+12800	76800	+12800
;;	1	...
;;	...
;;	12799	...
;; On input the 64-byte cache lines hold these data values if pass 1 swizzles:
;;	0	+50K	12800	+50K	25600	+50K	38400	+50K
;;	1	...
;;	...
;;	12799	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+50K	2560	+50K	5120	+50K	7680	+50K
;;	1	...
;;	...
;;	2559	...
;;	10240	...
;;	...

;; Do 5120 ten_reals_fft operations
;; Do 5120 five_complex_djbfft operations
;;	distance between fft data elements is 5120
;;	do 2560 iterations

	mov	rdi, xsincos_complex	;; Ptr to the sin/cos data
	mov	rdx, sincos3		;; Ptr to real sin/cos data
	loops_init 2560
b1b:	r5_fh5cl_ten_reals_five_complex_djbfft rsi, 64, 20*dist128, rdi, 2560*XMM_SCD2, rdx, 2560*XMM_HSCD2
	bump	rdi, XMM_SCD2		;; Next sin/cos data
	bump	rdx, XMM_HSCD2		;; Next sin/cos data
	loops	128, b1b		;; Test loop counter
	bump	rsi, -128*64+dist128	;; Next source pointer
	loops	20, b1b			;; Test loop counter
	bump	rsi, -20*dist128	;; Restore source pointer
	bump	rdi, -2560*XMM_SCD2	;; Restore sin/cos ptr
	bump	rdx, -2560*XMM_HSCD2	;; Next sin/cos ptr

;; Do FFT levels 3,4 (actually 2.322 levels)
;;
;; On input the 64-byte cache lines hold these data values:
;;	0	+50K	2560	+50K	5120	+50K	7680	+50K
;;	1	...
;;	...
;;	2559	...
;;	10240	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+50K	512	+50K	1024	+50K	1536	+50K
;;	1	...
;;	...
;;	511	...
;;	2048	...
;;	...

;; Do 1024 ten_reals_fft operations
;; Do 1024 five_complex_djbfft operations
;;	distance between fft data elements is 1024
;;	do 512 macros each processing 40 data values

	mov	rcx, DATA_PREFETCH	;; Prefetch pointer
	xtouch	[rcx]			;; Preload first TLB
	loops_init_prefetch 2560, 64, 16, rcx
	xtouch	[rcx+4096-64]		;; Preload the TLBs
	xtouch	[rcx+2*4096-64]
	xtouch	[rcx+160*64-64]
d2b:	r5_h5cl_ten_reals_five_complex_djbfft rsi, 64, 4*dist128, rdi, 512*5*XMM_SCD2, rdx, 512*5*XMM_HSCD2
	bump	rdi, 5*XMM_SCD2		;; Next sin/cos data
	bump	rdx, 5*XMM_HSCD2	;; Next sin/cos data
	loops	128, d2b		;; Test inner loop counter
	bump	rsi, -128*64+dist128	;; Next source pointer
	loops	4, d2b			;; Test loop counter
	bump	rsi, -4*dist128+20*dist128 ;; Next source pointer
	bump	rdi, -512*5*XMM_SCD2	;; Restore sin/cos ptr
	bump	rdx, -512*5*XMM_HSCD2+5120*XMM_HSCD2 ;; Next sin/cos ptr

;; Do 8192 five_complex_fft operations
;;	distance between fft data elements is 1024
;;	do 2048 macros each processing 40 data values

	loops_reset			;; 4 more iters of 512
	r5_nf5cl_five_complex_djbfft_preload
d3b:	r5_nf5cl_five_complex_djbfft rsi, 64, 4*dist128, rdi, 512*5*XMM_SCD2
	bump	rdi, 5*XMM_SCD2		;; Next sin/cos data
	loops	128, d3b		;; Test inner loop counter
	bump	rsi, -128*64+dist128	;; Next source pointer
	loops	4, d3b			;; Test loop counter
	bump	rsi, -4*dist128+20*dist128 ;; Next source pointer
	bump	rdi, -512*5*XMM_SCD2	;; Restore sin/cos ptr
	loops	5, d3b			;; Test loop counter
	bump	rsi, -5*20*dist128	;; Restore source pointer

;; Do FFT levels 5,6
;;
;; On input the 64-byte cache lines hold these data values:
;;	0	+50K	512	+50K	1024	+50K	1536	+50K
;;	1	...
;;	...
;;	511	...
;;	2048	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+50K	256	+50K	512	+50K	768	+50K
;;	1	...
;;	...
;;	255	...
;;	1024	...
;;	...

;; Do 256 eight_reals_fft operations
;; Do 256 four_complex_djbfft operations
;;	distance between fft data elements is 256
;;	do 256 macros each processing 16 data values

	loops_init_prefetch 6400, 64, 2, rcx
b2b0:	xtouch	[rcx+4096-64]		;; Preload the TLB
b2b:	r4_h2cl_eight_reals_four_complex_djbfft rsi, 64, 2*dist128, rdi, rdx
	bump	rdi, 25*XMM_SCD2	;; Next sin/cos data
	bump	rdx, XMM_HSCD2		;; Next sin/cos data
	loops	128, b2b		;; Test inner loop counter
	bump	rsi, -128*64+dist128	;; Next source pointer
	loops	2, b2b0			;; Test loop counter
	bump	rsi, -2*dist128+4*dist128 ;; Next source pointer
	bump	rdi, -256*25*XMM_SCD2	;; Restore sin/cos ptr
	bump	rdx, -256*XMM_HSCD2	;; Restore sin/cos ptr

;; Do 12288 four_complex_fft operations
;;	distance between fft data elements is 256
;;	do 6144 macros each processing 16 data values

	loops_reset
	r4_x2cl_four_complex_djbfft_preload
b3b0:	xtouch	[rcx+4096-64]		;; Preload the TLB
b3b:	r4_x2cl_four_complex_djbfft rsi, 64, 2*dist128, rdi
	bump	rdi, 25*XMM_SCD2	;; Next sin/cos data
	loops	128, b3b		;; Test inner loop counter
	bump	rsi, -128*64+dist128	;; Next source pointer
	loops	2, b3b0			;; Test loop counter
	bump	rsi, -2*dist128+4*dist128 ;; Next source pointer
	bump	rdi, -256*25*XMM_SCD2	;; Restore sin/cos ptr
	loops	25, b3b0		;; Test loop counter
	bump	rsi, -25*4*dist128	;; Restore source pointer

;; Do FFT levels 7,8
;;
;; On input the 64-byte cache lines hold these data values:
;;	0	+50K	256	+50K	512	+50K	768	+50K
;;	1	...
;;	...
;;	255	...
;;	1024	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+50K	64	+50K	128	+50K	192	+50K
;;	1	...
;;	...
;;	63	...
;;	256	...
;;	...

;; Do 64 eight_reals_fft operations
;; Do 192 four_complex_djbfft operations
;;	distance between fft data elements is 64
;;	do 64 macros each processing 32 data values

	bump	rdi, 6400*XMM_SCD2	;; Load sin/cos pointer
	loops_init_prefetch 3200, 64, 1, rcx
	xtouch	[rcx+4096-64]		;; Preload the TLB
	r4_h4cl_eight_reals_four_complex_djbfft_preload
b4b:	r4_h4cl_eight_reals_four_complex_djbfft rsi, 64, 64*64, dist128, rdi, rdx
	bump	rdi, XMM_SCD2		;; Next sin/cos data
	bump	rdx, 4*XMM_HSCD2	;; Next sin/cos data
	loops	64, b4b			;; Test loop counter
	bump	rsi, -64*64+2*dist128	;; Next source pointer
	bump	rdi, -64*XMM_SCD2	;; Restore sin/cos ptr
	bump	rdx, -64*4*XMM_HSCD2	;; Restore sin/cos ptr

;; Do 12544 four_complex_djbfft operations
;;	distance between fft data elements is 64
;;	do 3136 macros each processing 32 data values

	loops_reset			;; 49 more iterations of 64
	r4_x4cl_four_complex_djbfft_preload
b5a:	xtouch	[rcx+4096-64]		;; Preload the TLB
b5b:	r4_x4cl_four_complex_djbfft rsi, 64, 64*64, dist128, rdi
	bump	rdi, XMM_SCD2		;; Next sin/cos data
	loops	64, b5b			;; Test inner loop counter
	bump	rsi, -64*64+2*dist128	;; Next source pointer
	bump	rdi, -64*XMM_SCD2	;; Restore sin/cos ptr
	loops	50, b5a			;; Test loop counter
	bump	rsi, -50*2*dist128	;; Restore source pointer

;; Do FFT levels 9,10
;;
;; On input the 64-byte cache lines hold these data values:
;;	0	+50K	64	+50K	128	+50K	192	+50K
;;	1	...
;;	...
;;	63	...
;;	256	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+50K	16	+50K	32	+50K	48	+50K
;;	1	...
;;	...
;;	15	...
;;	64	...
;;	...

;; Do 16 eight_reals_fft operations
;; Do 48 four_complex_djbfft operations
;;	distance between fft data elements is 16
;;	do 16 macros each processing 32 data values

	loops_init 3200
	r4_h4cl_eight_reals_four_complex_djbfft_preload
b6b:	r4_h4cl_eight_reals_four_complex_djbfft rsi, 64, 16*64, 32*64, rdi, rdx
	bump	rdi, 4*XMM_SCD2		;; Next sin/cos data
	bump	rdx, 16*XMM_HSCD2	;; Next sin/cos data
	loops	16, b6b			;; Test loop counter
	bump	rsi, -16*64+64*64	;; Next source pointer
	bump	rdi, -16*4*XMM_SCD2	;; Restore sin/cos ptr
	bump	rdx, -16*16*XMM_HSCD2	;; Restore sin/cos ptr

;; Do 12736 four_complex_djbfft operations
;;	distance between fft data elements is 16
;;	do 3184 macros each processing 32 data values

	loops_reset			;; 100 iters of 1 or 2 of 16
	r4_x4cl_four_complex_djbfft_preload
b7b:	r4_x4cl_four_complex_djbfft rsi, 64, 16*64, 32*64, rdi
	bump	rdi, 4*XMM_SCD2		;; Next sin/cos data
	loops	16, b7b			;; Test inner loop counter
	bump	rsi, -16*64+64*64	;; Next source pointer
	bump	rdi, -16*4*XMM_SCD2	;; Restore sin/cos ptr
	loops	2, b7b			;; Test loop counter
	bump	rsi, -2*64*64+dist128	;; Next source pointer
	loops	100, b7b		;; 100 iterations
	bump	rsi, -100*dist128	;; Next source pointer

;; Do FFT levels 11,12
;;
;; On input the 64-byte cache lines hold these data values:
;;	0	+50K	16	+50K	32	+50K	48	+50K
;;	1	...
;;	...
;;	15	...
;;	64	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+50K	4	+50K	8	+50K	12	+50K
;;	1	...
;;	...
;;	3	...
;;	16	...
;;	...

;; Do 4 eight_reals_fft operations
;; Do 12 four_complex_djbfft operations
;;	distance between fft data elements is 4
;;	do 4 macros each processing 32 data values

	loops_init 3200
	r4_h4cl_eight_reals_four_complex_djbfft_preload
b8b:	r4_h4cl_eight_reals_four_complex_djbfft rsi, 64, 4*64, 8*64, rdi, rdx
	bump	rdi, 16*XMM_SCD2	;; Next sin/cos data
	bump	rdx, 64*XMM_HSCD2	;; Next sin/cos data
	loops	4, b8b			;; Test loop counter
	bump	rsi, -4*64+16*64	;; Next source pointer
	bump	rdi, -4*16*XMM_SCD2	;; Restore sin/cos ptr
	bump	rdx, -4*64*XMM_HSCD2	;; Restore sin/cos ptr

;; Do 12784 four_complex_djbfft operations
;;	distance between fft data elements is 4
;;	do 3196 macros each processing 32 data values

	loops_reset			;; 100 iters of 7 or 8 of 4
	r4_x4cl_four_complex_djbfft_preload
b9b:	r4_x4cl_four_complex_djbfft rsi, 64, 4*64, 8*64, rdi
	bump	rdi, 16*XMM_SCD2	;; Next sin/cos data
	loops	4, b9b			;; Test loop counter
	bump	rsi, -4*64+16*64	;; Next source pointer
	bump	rdi, -4*16*XMM_SCD2	;; Restore sin/cos ptr
	loops	8, b9b			;; Test middle loop counter
	bump	rsi, -8*16*64+dist128	;; Next source pointer
	loops	100, b9b		;; 100 iterations
	bump	rsi, -100*dist128	;; Restore source pointer

;; Do FFT levels 13,14
;;
;; On input the 64-byte cache lines hold these data values:
;;	0	+50K	4	+50K	8	+50K	12	+50K
;;	1	...
;;	...
;;	3	...
;;	16	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+50K	2	+50K	4	+50K	6	+50K
;;	1	...
;;	8	...
;;	...

;; Do 1 eight_reals_fft operations
;; Do 3 four_complex_fft operations
;;	distance between fft data elements is 1

;; Do 12796 four_complex_fft operations
;;	distance between fft data elements is 1
;;	do 3199 macros each processing 32 data values

;; Execute the proper middle step

	cmp	ffttype, 2
	jl	xpass2_25600_levels_real_1
	jg	xpass2_25600_levels_real_3

xpass2_25600_levels_real_2:
	r4_h4cl_eight_reals_four_complex_with_square rsi, 4*64, 64, 2*64
	loops_init 3200, 1, 32
	r4_x4cl_four_complex_with_square_preload
baa:	r4_x4cl_four_complex_with_square rsi, 4*64, 64, 2*64
	loops	32, baa			;; Test loop counter
	bump	rsi, -32*4*64+dist128	;; Next source pointer
	loops	100, baa		;; Test loop counter
	bump	rsi, -100*dist128	;; Restore source pointer
	jmp	xpass2_25600_real_unfft

xpass2_25600_levels_real_1:
	r4_h4cl_eight_reals_four_complex_fft_final rsi, 4*64, 64, 2*64
	L2prefetch128 [rcx]
	bump	rcx, 128
	loops_init_prefetch 3200, 128, 1, rcx, 1, 32
bab0:	xtouch	[rcx+4096-64]		;; Preload the TLBs
bab:	r4_x4cl_four_complex_fft_final rsi, 4*64, 64, 2*64
	loops	32, bab			;; Test loop counter
	bump	rsi, -32*4*64+dist128	;; Next source pointer
	loops	100, bab0		;; Test loop counter
	jmp	xpass2_25600_real_done

xpass2_25600_levels_real_3:
	mov	rbp, DIST_TO_MULSRCARG
	r4_h4cl_eight_reals_four_complex_with_mult rsi, 4*64, 64, 2*64
	loops_init 3200, 1, 32
bac:	r4_x4cl_four_complex_with_mult rsi, 4*64, 64, 2*64
	loops	32, bac			;; Test loop counter
	bump	rsi, -32*4*64+dist128	;; Next source pointer
	loops	100, bac		;; Test loop counter
	bump	rsi, -100*dist128	;; Restore source pointer
	jmp	xpass2_25600_real_unfft

xpass2_25600_levels_real_4:
	mov	rbp, DIST_TO_MULSRCARG
	mov	rcx, DATA_PREFETCH	;; Load prefetch pointer
	xtouch	[rcx]			;; Preload the TLBs
	r4_h4cl_eight_reals_four_complex_with_mulf rsi, 4*64, 64, 2*64
	L2prefetch128 [rcx]
	bump	rcx, 128
	loops_init_prefetch 3200, 128, 1, rcx, 1, 32
bad0:	xtouch	[rcx+4096-64]		;; Preload the TLBs
bad:	r4_x4cl_four_complex_with_mulf rsi, 4*64, 64, 2*64
	loops	32, bad			;; Test loop counter
	bump	rsi, -32*4*64+dist128	;; Next source pointer
	loops	100, bad0		;; Test loop counter
	bump	rsi, -100*dist128	;; Restore source pointer

;; Do inverse FFT levels 11,12
;; On input the 64-byte cache lines hold these data values:
;;	0	+50K	2	+50K	4	+50K	6	+50K
;;	1	...
;;	8	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+50K	8	+50K	16	+50K	24	+50K
;;	1	...
;;	...
;;	7	...
;;	32	...
;;	...

;; Do 4 eight_reals_unfft operations
;; Do 4 four_complex_djbunfft operations
;;	distance between fft data elements is 4
;;	do 2 macros processing 32 data values

xpass2_25600_real_unfft:
	mov	rdi, xsincos_complex	;; Load sin/cos pointers
	bump	rdi, 6400*XMM_SCD2
	mov	rdx, sincos3
	bump	rdx, 5120*XMM_HSCD2
	loops_init 3200
c8b:	r4_h4cl_eight_reals_four_complex_djbunfft rsi, 64, 2*64, 4*64, rdi, 2*16*XMM_SCD2, rdx, 2*64*XMM_HSCD2
	bump	rdi, 16*XMM_SCD2	;; Next sin/cos data
	bump	rdx, 64*XMM_HSCD2	;; Next sin/cos data
	loops	2, c8b			;; Test loop counter
	bump	rsi, -2*64+8*64		;; Next source pointer
	bump	rdi, -2*16*XMM_SCD2	;; Restore sin/cos ptr
	bump	rdx, -2*64*XMM_HSCD2	;; Restore sin/cos ptr

;; Do 12792 four_complex_djbunfft operations
;;	distance between fft data elements is 4
;;	do 3198 macros each processing 32 data values

	loops_reset			;; 100 iters of 15 or 16 of 2
	r4_x4cl_four_complex_djbunfft_preload
c9b:	r4_x4cl_four_complex_djbunfft rsi, 64, 2*64, 4*64, rdi, 2*16*XMM_SCD2
	bump	rdi, 16*XMM_SCD2	;; Next sine/cosine pointer
	loops	2, c9b			;; Test loop counter
	bump	rsi, -2*64+8*64		;; Next source pointer
	bump	rdi, -2*16*XMM_SCD2	;; Restore sin/cos ptr
	loops	16, c9b			;; Test loop counter
	bump	rsi, -16*8*64+dist128	;; Next source pointer
	loops	100, c9b		;; Test loop counter
	bump	rsi, -100*dist128	;; Restore source pointer

;; Do inverse FFT levels 9,10
;; On input the 64-byte cache lines hold these data values:
;;	0	+50K	8	+50K	16	+50K	24	+50K
;;	1	...
;;	...
;;	7	...
;;	32	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+50K	32	+50K	64	+50K	96	+50K
;;	1	...
;;	...
;;	31	...
;;	128	...
;;	...

;; Do 16 eight_reals_unfft operations
;; Do 16 four_complex_djbunfft operations
;;	distance between fft data elements is 16
;;	do 8 macros processing 32 data values

	loops_init_prefetch 3200, 64, 1, rcx
	xtouch	[rcx+4096-64]		;; Preload the TLBs
c6b:	r4_h4cl_eight_reals_four_complex_djbunfft rsi, 64, 8*64, 16*64, rdi, 8*4*XMM_SCD2, rdx, 8*16*XMM_HSCD2
	bump	rdi, 4*XMM_SCD2		;; Next sin/cos data
	bump	rdx, 16*XMM_HSCD2	;; Next sin/cos data
	loops	8, c6b			;; Test inner loop counter
	bump	rsi, -8*64+32*64	;; Next source pointer
	bump	rdi, -8*4*XMM_SCD2	;; Restore sin/cos ptr
	bump	rdx, -8*16*XMM_HSCD2	;; Restore sin/cos ptr

;; Do 12768 four_complex_djbunfft operations
;;	distance between fft data elements is 16
;;	do 3192 macros each processing 32 data values

	loops_reset			;; 100 iters of 3 or 4 of 4
	r4_x4cl_four_complex_djbunfft_preload
c7a:	xtouch	[rcx+4096-64]		;; Preload the TLBs
c7b:	r4_x4cl_four_complex_djbunfft rsi, 64, 8*64, 16*64, rdi, 8*4*XMM_SCD2
	bump	rdi, 4*XMM_SCD2		;; Next sin/cos data
	loops	8, c7b			;; Test inner loop counter
	bump	rsi, -8*64+32*64	;; Next source pointer
	bump	rdi, -8*4*XMM_SCD2	;; Restore sin/cos ptr
	loops	4, c7b			;; Test loop counter
	bump	rsi, -4*32*64+dist128	;; Next source pointer
	loops_touch 2, c7b, 50, c7a	;; Test loop counter
	bump	rsi, -100*dist128	;; Restore source pointer

;; Do inverse FFT levels 7,8
;; On input the 64-byte cache lines hold these data values:
;;	0	+50K	32	+50K	64	+50K	96	+50K
;;	1	...
;;	...
;;	31	...
;;	128	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+50K	128	+50K	256	+50K	384	+50K
;;	1	...
;;	...
;;	127	...
;;	512	...
;;	...

;; Do 64 eight_reals_unfft operations
;; Do 64 four_complex_djbunfft operations
;;	distance between fft data elements is 64
;;	do 32 macros processing 32 data values

	loops_init_prefetch 3200, 64, 1, rcx
	xtouch	[rcx+4096-64]		;; Preload the TLB
c4b:	r4_h4cl_eight_reals_four_complex_djbunfft rsi, 64, 32*64, 64*64, rdi, 32*XMM_SCD2, rdx, 32*4*XMM_HSCD2
	bump	rdi, XMM_SCD2		;; Next sin/cos data
	bump	rdx, 4*XMM_HSCD2	;; Next sin/cos data
	loops	32, c4b			;; Test inner loop counter
	bump	rsi, -32*64+dist128	;; Next source pointer
	bump	rdi, -32*XMM_SCD2	;; Restore sin/cos ptr
	bump	rdx, -32*4*XMM_HSCD2	;; Restore sin/cos ptr

;; Do 12672 four_complex_djbunfft operations
;;	distance between fft data elements is 64
;;	do 3168 macros each processing 32 data values

	loops_reset			;; 99 more iters of 32
	r4_x4cl_four_complex_djbunfft_preload
c5a:	xtouch	[rcx+4096-64]		;; Preload the TLB
c5b:	r4_x4cl_four_complex_djbunfft rsi, 64, 32*64, 64*64, rdi, 32*XMM_SCD2
	bump	rdi, XMM_SCD2		;; Next sin/cos data
	loops	32, c5b			;; Test inner loop counter
	bump	rsi, -32*64+dist128	;; Next source pointer
	bump	rdi, -32*XMM_SCD2	;; Restore sin/cos ptr
	loops_touch 2, c5b, 50, c5a	;; Test loop counter
	bump	rsi, -100*dist128	;; Next source pointer

;; Do inverse FFT levels 5,6
;; On input the 64-byte cache lines hold these data values:
;;	0	+50K	128	+50K	256	+50K	384	+50K
;;	1	...
;;	...
;;	127	...
;;	512	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+50K	512	+50K	1024	+50K	1536	+50K
;;	1	...
;;	...
;;	511	...
;;	2048	...
;;	...

;; Do 256 eight_reals_unfft operations
;; Do 256 four_complex_djbunfft operations
;;	distance between fft data elements is 256
;;	do 128 macros processing 32 data values

	loops_init 3200
	mov	rdi, xsincos_complex	;; Load sin/cos pointer
c2b:	r4_h4cl_eight_reals_four_complex_djbunfft rsi, 64, dist128, 2*dist128, rdi, 128*25*XMM_SCD2, rdx, 128*XMM_HSCD2
	bump	rdi, 25*XMM_SCD2	;; Next sin/cos data
	bump	rdx, XMM_HSCD2		;; Next sin/cos data
	loops	128, c2b		;; Test inner loop counter
	bump	rsi, -128*64+4*dist128	;; Next source pointer
	bump	rdi, -128*25*XMM_SCD2	;; Restore sin/cos ptr
	bump	rdx, -128*XMM_HSCD2-5120*XMM_HSCD2 ;; Next sin/cos ptr

;; Do 12288 four_complex_djbunfft operations
;;	distance between fft data elements is 256
;;	do 3072 macros each processing 32 data values

	loops_reset			;; 24 more iters of 128
	r4_x4cl_four_complex_djbunfft_preload
c3b:	r4_x4cl_four_complex_djbunfft rsi, 64, dist128, 2*dist128, rdi, 128*25*XMM_SCD2
	bump	rdi, 25*XMM_SCD2	;; Next sin/cos data
	loops	128, c3b		;; Test inner loop counter
	bump	rsi, -128*64+4*dist128	;; Next source pointer
	bump	rdi, -128*25*XMM_SCD2	;; Restore sin/cos ptr
	loops	25, c3b			;; Test loop counter
	bump	rsi, -25*4*dist128	;; Next source pointer

;; Do inverse FFT levels 3,4 (actually 2.322 FFT levels)
;; On input the 64-byte cache lines hold these data values:
;;	0	+50K	512	+50K	1024	+50K	1536	+50K
;;	1	...
;;	...
;;	511	...
;;	2048	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+50K	2560	+50K	5120	+50K	7680	+50K
;;	1	...
;;	...
;;	2559	...
;;	10240	...
;;	...

;; Do 1024 ten_reals_unfft operations
;; Do 1024 ten_complex_djbunfft operations
;;	distance between fft data elements is 1024
;;	do 512 macros processing 40 data values

	bump	rcx, -2*64		;; Back up prefetch pointer by 2 cache lines
	loops_init_prefetch 2560, 64, 64, rcx
	xtouch	[rcx+40*64-64]		;; Preload the TLB
e2b:	r5_h5cl_ten_reals_five_complex_djbunfft rsi, 64, 4*dist128, rdi, 512*5*XMM_SCD2, rdx, 512*5*XMM_HSCD2
	bump	rdi, 5*XMM_SCD2		;; Next sin/cos data
	bump	rdx, 5*XMM_HSCD2	;; Next sin/cos data
	loops	128, e2b		;; Test inner loop counter
	bump	rsi, -128*64+dist128	;; Next source pointer
	loops	4, e2b			;; Test loop counter
	bump	rsi, -4*dist128+20*dist128 ;; Next source pointer
	bump	rdi, -512*5*XMM_SCD2	;; Restore sin/cos ptr
	bump	rdx, -512*5*XMM_HSCD2	;; Next sin/cos ptr

;; Do 8192 five_complex_djbunfft operations
;;	distance between fft data elements is 1024
;;	do 2048 macros each processing 40 data values

	loops_reset			;; 4 more iters of 512
	r5_x5cl_five_complex_djbunfft_preload
e3b:	r5_x5cl_five_complex_djbunfft rsi, 64, 4*dist128, rdi, 512*5*XMM_SCD2
	bump	rdi, 5*XMM_SCD2		;; Next sin/cos data
	loops	128, e3b		;; Test inner loop counter
	bump	rsi, -128*64+dist128	;; Next source pointer
	loops	4, e3b			;; Test loop counter
	bump	rsi, -4*dist128+20*dist128 ;; Next source pointer
	bump	rdi, -512*5*XMM_SCD2	;; Restore sin/cos ptr
	loops	5, e3b			;; Test loop counter
	bump	rsi, -5*20*dist128	;; Restore source pointer

;; Do inverse FFT levels 1,2 (actually 2.322 FFT levels)
;; On input the 64-byte cache lines hold these data values:
;;	0	+50K	2560	+50K	5120	+50K	7680	+50K
;;	1	...
;;	...
;;	2559	...
;;	10240	...
;;	...
;; On output the 64-byte cache lines hold these data values if pass 1 swizzles:
;;	0	+50K	12800	+50K	25600	+50K	38400	+50K
;;	1	...
;;	...
;;	12799	...
;; On output the 64-byte cache lines hold these data values if pass 2 swizzles:
;;	0	+12800	25600	+12800	51200	+12800	76800	+12800
;;	1	...
;;	...
;;	12799	...

;; Do 5120 ten_reals_unfft operations
;; Do 5120 five_complex_djbunfft operations
;;	distance between fft data elements is 5120
;;	do 2560 macros processing 40 data values

	loops_init 2560
c1b:	r5_h5cl_ten_reals_five_complex_djbunfft rsi, 64, 20*dist128, rdi, 2560*XMM_SCD2, rdx, 2560*XMM_HSCD2
	bump	rdi, XMM_SCD2		;; Next sin/cos data
	bump	rdx, XMM_HSCD2		;; Next sin/cos data
	loops	128, c1b		;; Test loop counter
	bump	rsi, -128*64+dist128	;; Next source pointer
	loops	20, c1b			;; Test loop counter

xpass2_25600_real_done:
	end_timer 2
	ENDM

; *************** radix4-pass2-25600-levels-complex macro ******************
; This macro takes 2 groups of 25600 complex values and
; performs the final 14.644 levels of a standard radix-4 FFT, squares the
; results, and does the first 14.644 levels of the inverse FFT.

r4_pass2_25600_levels_complex MACRO
	LOCAL	b1b, b7b, b2a, b2b, b3a, b3b, b4a, b4b, b5a, b5b
	LOCAL	b6a0, b6a, b6b0, b6b, b6c0, b6c, b6d0, b6d
	LOCAL	c1b, c7b, c2a, c2b, c3a, c3b, c4a, c4b, c5a, c5b
	LOCAL	xpass2_25600_levels_complex_1, xpass2_25600_levels_complex_2
	LOCAL	xpass2_25600_levels_complex_3, xpass2_25600_levels_complex_4
	LOCAL	xpass2_25600_levels_complex_unfft, xpass2_25600_levels_complex_done

;; Type 4 FFTs skip the forward FFT process

	mov	rsi, DATA_ADDR		;; Load address of FFT data
	mov	rbx, DIST_TO_FFTSRCARG
	cmp	ffttype, 4
	je	xpass2_25600_levels_complex_4

;; Do FFT level 1,2 (actually 2.322 levels)
;;
;; We swizzle the data so that one set of data is in the high word of
;; an xmm register, the other set is in the low word.  This means
;; a distance of 12800 data values.
;;
;; On input the 64-byte cache lines hold these data values if pass 2 swizzles:
;;	0	+3.2K	6400	+3.2K	12800	+3.2K	19200	+3.2K
;;	1	...
;;	...
;;	3199
;; On input the 64-byte cache lines hold these data values if pass 1 swizzles:
;;	0	+12.5K	3200	+12.5K	6400	+12.5K	9600	+12.5K
;;	1	...
;;	...
;;	3199	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+12.5K	640	+12.5K	1280	+12.5K	1920	+12.5K
;;	1	...
;;	...
;;	639	...
;;	2560	...
;;	...

;; Do 10240 five_complex_fft operations
;;	distance between fft data elements is 5120
;;	do 2560 macros each processing 40 data values

	start_timer 3
	mov	rdi, xsincos_complex	;; The sin/cos data
	loops_init 2560			;; 20 iterations of 128
	r5_f5cl_five_complex_djbfft_preload
b1b:	xxprefetch [rdi+XMM_SCD2]	;; Prefetch for next pass
	xxprefetch [rdi+2561*XMM_SCD2]	;; Prefetch for next pass
	r5_f5cl_five_complex_djbfft rsi, 64, 20*dist128, rdi, 2560*XMM_SCD2
	bump	rdi, XMM_SCD2		;; Next sin/cos data
	loops	128, b1b		;; Test inner loop counter
	bump	rsi, -128*64+dist128	;; Next source pointer
	loops	20, b1b			;; Test outer loop counter
	bump	rsi, -20*dist128	;; Restore source pointer
	end_timer 3

;; Do FFT levels 3,4 (actually 2.322 levels)
;;
;; On input the 64-byte cache lines hold these data values:
;;	0	+50K	2560	+50K	5120	+50K	7680	+50K
;;	1	...
;;	...
;;	2559	...
;;	10240	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+50K	512	+50K	1024	+50K	1536	+50K
;;	1	...
;;	...
;;	511	...
;;	2048	...
;;	...

;; Do 10240 five_complex_fft operations
;;	distance between fft data elements is 1024
;;	do 2560 macros each processing 40 data values

	start_timer 4
	mov	rcx, DATA_PREFETCH	;; Prefetch pointer (we need to prefetch 
					;; 12800 cache lines + 198 pad cache lines)
	xtouch	[rcx]			;; Load first prefetch TLB
	mov	rdi, xsincos_complex	;; Ptr to the sin/cos data
	loops_init_prefetch 2560, 64, 16, rcx ;; 5 iters of 4 iters of 128
	xtouch	[rcx+4096-64]		;; Load prefetch TLBs
	xtouch	[rcx+2*4096-64]
	xtouch	[rcx+160*64-64]
	r5_nf5cl_five_complex_djbfft_preload
b7b:	r5_nf5cl_five_complex_djbfft rsi, 64, 4*dist128, rdi, 512*5*XMM_SCD2
	bump	rdi, 5*XMM_SCD2		;; Next sin/cos data
	loops	128, b7b		;; Test inner loop counter
	bump	rsi, -128*64+dist128	;; Next source pointer
	loops	4, b7b			;; Test inner loop counter
	bump	rsi, -4*dist128+20*dist128 ;; Next source pointer
	bump	rdi, -512*5*XMM_SCD2	;; Restore sin/cos ptr
	loops	5, b7b			;; Test loop counter
	bump	rsi, -5*20*dist128	;; Restore source pointer
	end_timer 4

;; Do FFT levels 5,6
;;
;; On input the 64-byte cache lines hold these data values:
;;	0	+50K	512	+50K	1024	+50K	1536	+50K
;;	1	...
;;	...
;;	511	...
;;	2048	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+50K	256	+50K	512	+50K	768	+50K
;;	1	...
;;	...
;;	255	...
;;	1024	...
;;	...

;; Do 12800 four_complex_fft operations
;;	distance between fft data elements is 256
;;	do 6400 macros each processing 16 data values

	start_timer 5
	loops_init_prefetch 6400, 64, 4, rcx ;; 25 iters of 2 iters of 128
	r4_x2cl_four_complex_djbfft_preload
b2a:	xtouch	[rcx+4096-64]		;; Load prefetch TLB
b2b:	r4_x2cl_four_complex_djbfft rsi, 64, 2*dist128, rdi
	bump	rdi, 25*XMM_SCD2	;; Next sin/cos data
	loops	128, b2b		;; Test inner loop counter
	bump	rsi, -128*64+dist128	;; Next source pointer
	loops	2, b2b			;; Test loop counter
	bump	rsi, -2*dist128+4*dist128 ;; Next source pointer
	bump	rdi, -256*25*XMM_SCD2	;; Restore sin/cos ptr
	loops	25, b2a			;; Test loop counter
	bump	rsi, -25*4*dist128	;; Restore source pointer
	end_timer 5

;; Do FFT levels 7,8
;;
;; On input the 64-byte cache lines hold these data values:
;;	0	+50K	256	+50K	512	+50K	768	+50K
;;	1	...
;;	...
;;	255	...
;;	1024	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+50K	64	+50K	128	+50K	192	+50K
;;	1	...
;;	...
;;	63	...
;;	256	...
;;	...
;;
;; Do 12800 four_complex_fft operations
;;	distance between fft data elements is 64
;;	do 3200 macros each processing 32 data values

;; Do FFT levels 9,10
;;
;; On input the 64-byte cache lines hold these data values:
;;	0	+50K	64	+50K	128	+50K	192	+50K
;;	1	...
;;	...
;;	63	...
;;	256	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+50K	16	+50K	32	+50K	48	+50K
;;	1	...
;;	...
;;	15	...
;;	64	...
;;	...
;;
;; Do 12800 four_complex_fft operations
;;	distance between fft data elements is 16
;;	do 3200 macros each processing 32 data values

;; Do FFT levels 11,12
;;
;; On input the 64-byte cache lines hold these data values:
;;	0	+50K	16	+50K	32	+50K	48	+50K
;;	1	...
;;	...
;;	15	...
;;	64	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+50K	4	+50K	8	+50K	12	+50K
;;	1	...
;;	...
;;	3	...
;;	16	...
;;	...
;;
;; Do 12800 four_complex_fft operations
;;	distance between fft data elements is 4
;;	do 3200 macros each processing 32 data values

;; Do this 64 cache lines at a time to maximize accessing data
;; while it is in the L1 cache.

	start_timer 6
	bump	rdi, 6400*XMM_SCD2	;; Load sin/cos pointers
	mov	rbx, rdi
	mov	rbp, rdi
	loops_init_prefetch 3200, 64, 2, rcx ;; 2*4 iters of 4 of 64 cache lines
	r4_x4cl_four_complex_djbfft_preload

b3a:	xtouch	[rcx+48*64-64]		;; Load prefetch TLB

b3b:	xxprefetch [rdi+4*XMM_SCD2]	;; Prefetch for next pass
	r4_x4cl_four_complex_djbfft rsi, 4*64, 64*64, dist128, rdi
	bump	rdi, 4*XMM_SCD2		;; Next sine/cosine pointer
	loops	16, b3b			;; Loop 16 times
	bump	rsi, -16*4*64		;; Restore source pointer
	bump	rdi, -16*4*XMM_SCD2	;; Restore sine/cosine pointer
	loops_undo 16			;; Back up loop counter

b4b:	r4_x4cl_four_complex_djbfft rsi, 4*64, 16*64, 32*64, rbx
	bump	rbx, 4*4*XMM_SCD2	;; Next sine/cosine pointer
	loops	4, b4b			;; Loop 4 times
	bump	rsi, -4*4*64+64*64	;; Next source pointer
	bump	rbx, -4*4*4*XMM_SCD2	;; Restore sine/cosine pointer
	loops	2, b4b			;; Loop twice
	bump	rsi, -2*64*64+dist128	;; Next source pointer
	loops	2, b4b			;; Loop twice
	bump	rsi, -2*dist128		;; Restore source pointer
	loops_undo 2*2*4		;; Back up loop counter

b5b:	r4_x4cl_four_complex_djbfft rsi, 16*64, 4*64, 8*64, rbp
	loops	8, b5b			;; Loop 8 times
	bump	rsi, -8*16*64+dist128	;; Next source pointer
	loops	2, b5b			;; Loop twice

	bump	rsi, -2*dist128+64	;; Next source pointer
	bump	rdi, XMM_SCD2		;; Next sine/cosine pointer
	bump	rbx, 4*XMM_SCD2		;; Next sine/cosine pointer
	bump	rbp, 16*XMM_SCD2	;; Next sine/cosine pointer
	loops_touch 2, b3b, 2, b3a	;; Loop 4 times
	bump	rsi, -4*64+2*dist128	;; Next source pointer
	bump	rdi, -4*XMM_SCD2	;; Restore sine/cosine pointer
	bump	rbx, -4*4*XMM_SCD2	;; Restore sine/cosine pointer
	bump	rbp, -4*16*XMM_SCD2	;; Restore sine/cosine pointer
	loops	50, b3a			;; Test outer loop counter
	bump	rsi, -50*2*dist128	;; Restore source pointer
	end_timer 6

;; Do FFT levels 13,14 as well as inverse FFT levels 13,14
;;
;; On input the 64-byte cache lines hold these data values:
;;	0	+50K	4	+50K	8	+50K	12	+50K
;;	1	...
;;	...
;;	3	...
;;	16	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+50K	2	+50K	4	+50K	6	+50K
;;	1	...
;;	8	...
;;	...

;; Do 12800 four_complex_fft operations
;;	distance between fft data elements is 1
;;	do 3200 macros each processing 32 data values

;; Execute the right middle step

	cmp	ffttype, 2
	jl	xpass2_25600_levels_complex_1
	jg	xpass2_25600_levels_complex_3

xpass2_25600_levels_complex_2:
	start_timer 9
	loops_init 3200
	r4_x4cl_four_complex_with_square_preload
b6a:	r4_x4cl_four_complex_with_square rsi, 4*64, 64, 2*64
	loops	32, b6a			;; Loop 32 times
	bump	rsi, -32*4*64+dist128	;; Next source pointer
	loops	100, b6a		;; Loop 100 times
	bump	rsi, -100*dist128	;; Restore source pointer
	end_timer 9
	jmp	xpass2_25600_levels_complex_unfft

xpass2_25600_levels_complex_1:
	start_timer 9
	loops_init_prefetch 3200, 128, 1, rcx
b6b0:	xtouch	[rcx+4096-128]		;; Load prefetch TLBs
b6b:	r4_x4cl_four_complex_fft_final rsi, 4*64, 64, 2*64
	loops	32, b6b			;; Loop 32 times
	bump	rsi, -32*4*64+dist128	;; Next source pointer
	loops	100, b6b0		;; Loop 100 times
	end_timer 9
	jmp	xpass2_25600_levels_complex_done

xpass2_25600_levels_complex_3:
	start_timer 9
	mov	rbp, DIST_TO_MULSRCARG
	loops_init 3200
b6c:	r4_x4cl_four_complex_with_mult rsi, 4*64, 64, 2*64
	loops	32, b6c			;; Loop 32 times
	bump	rsi, -32*4*64+dist128	;; Next source pointer
	loops	100, b6c		;; Loop 100 times
	bump	rsi, -100*dist128	;; Restore source pointer
	end_timer 9
	jmp	xpass2_25600_levels_complex_unfft

xpass2_25600_levels_complex_4:
	start_timer 9
	mov	rcx, DATA_PREFETCH	;; Prefetch pointer
	xtouch	[rcx]			;; Load first prefetch TLB
	mov	rbp, DIST_TO_MULSRCARG
	loops_init_prefetch 3200, 128, 1, rcx
b6d0:	xtouch	[rcx+4096-128]		;; Load prefetch TLBs
b6d:	r4_x4cl_four_complex_with_mulf rsi, 4*64, 64, 2*64
	loops	32, b6d			;; Loop 32 times
	bump	rsi, -32*4*64+dist128	;; Next source pointer
	loops	100, b6d0		;; Loop 100 times
	bump	rsi, -100*dist128	;; Restore source pointer
	end_timer 9

;; Do inverse FFT levels 11,12
;; On input the 64-byte cache lines hold these data values:
;;	0	+50K	2	+50K	4	+50K	6	+50K
;;	1	...
;;	8	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+50K	8	+50K	16	+50K	24	+50K
;;	1	...
;;	...
;;	7	...
;;	32	...
;;	...
;;
;; Do 12800 four_complex_unfft operations
;;	distance between fft data elements is 4
;;	do 3200 macros each processing 32 data values

;; Do inverse FFT levels 9,10
;; On input the 64-byte cache lines hold these data values:
;;	0	+50K	8	+50K	16	+50K	24	+50K
;;	1	...
;;	...
;;	7	...
;;	32	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+50K	32	+50K	64	+50K	96	+50K
;;	1	...
;;	...
;;	31	...
;;	128	...
;;	...
;;
;; Do 12800 four_complex_unfft operations
;;	distance between fft data elements is 16
;;	do 3200 macros each processing 32 data values

;; Do inverse FFT levels 7,8
;; On input the 64-byte cache lines hold these data values:
;;	0	+50K	32	+50K	64	+50K	96	+50K
;;	1	...
;;	...
;;	31	...
;;	128	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+50K	128	+50K	256	+50K	384	+50K
;;	1	...
;;	...
;;	127	...
;;	512	...
;;	...
;;
;; Do 12800 four_complex_unfft operations
;;	distance between fft data elements is 64
;;	do 3200 macros each processing 32 data values

;; Do this 64 cache lines at a time to maximize accessing data
;; while it is in the L1 cache.

xpass2_25600_levels_complex_unfft:

	start_timer 10
	mov	rdi, xsincos_complex	;; Load sin/cos pointers
	bump	rdi, 6400*XMM_SCD2
	mov	rbx, rdi
	mov	rbp, rdi
	loops_init_prefetch 3200, 64, 2, rcx ;; 8*2*8 iters of 64 cache lines
	r4_x4cl_four_complex_djbunfft_preload

c5a:	xtouch	[rcx+48*64-64]		;; Load prefetch TLB

c5b:	r4_x4cl_four_complex_djbunfft rsi, 8*64, 2*64, 4*64, rbp, 2*16*XMM_SCD2
	loops	16, c5b			;; Loop 16 times
	bump	rsi, -16*8*64		;; Restore source pointer
	loops_undo 16			;; Restore loop counter

c4b:	r4_x4cl_four_complex_djbunfft rsi, 2*64, 8*64, 16*64, rbx, 8*4*XMM_SCD2
	bump	rbx, 2*4*XMM_SCD2	;; Next sine/cosine pointer
	loops	4, c4b			;; Loop 4 times
	bump	rsi, -4*2*64+32*64	;; Next source pointer
	bump	rbx, -4*2*4*XMM_SCD2	;; Restore sine/cosine pointer
	loops	4, c4b			;; Loop 4 times
	bump	rsi, -4*32*64		;; Restore source pointer
	loops_undo 4*4			;; Restore loop counter

c3b:	r4_x4cl_four_complex_djbunfft rsi, 2*64, 32*64, 64*64, rdi, 32*1*XMM_SCD2
	bump	rdi, 2*XMM_SCD2		;; Next sine/cosine pointer
	loops	16, c3b			;; Loop 16 times
	bump	rdi, -16*2*XMM_SCD2	;; Restore sine/cosine pointer

	bump	rsi, -16*2*64+64	;; Next source pointer
	bump	rdi, XMM_SCD2		;; Next sine/cosine pointer
	bump	rbx, 4*XMM_SCD2		;; Next sine/cosine pointer
	bump	rbp, 16*XMM_SCD2	;; Next sine/cosine pointer
	loops	2, c5b			;; Loop twice
	bump	rsi, -2*64+dist128	;; Next source pointer
	bump	rdi, -2*XMM_SCD2	;; Restore sine/cosine pointer
	bump	rbx, -2*4*XMM_SCD2	;; Restore sine/cosine pointer
	bump	rbp, -2*16*XMM_SCD2	;; Restore sine/cosine pointer
	loops	100, c5a		;; Loop 100 times
	bump	rsi, -100*dist128	;; Restore source pointer
	end_timer 10

;; Do inverse FFT levels 5,6
;; On input the 64-byte cache lines hold these data values:
;;	0	+50K	128	+50K	256	+50K	384	+50K
;;	1	...
;;	...
;;	127	...
;;	512	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+50K	512	+50K	1024	+50K	1536	+50K
;;	1	...
;;	...
;;	511	...
;;	2048	...
;;	...

;; Do 12800 four_complex_unfft operations
;;	distance between fft data elements is 256
;;	do 3200 macros each processing 32 data values

	start_timer 13
	mov	rdi, xsincos_complex	;; Load sin/cos pointer
	loops_init_prefetch 3200, 64, 2, rcx;; 25 iters of 128
	r4_x4cl_four_complex_djbunfft_preload
c2a:	xtouch	[rcx+4096-64]		;; Load prefetch TLB
c2b:	r4_x4cl_four_complex_djbunfft rsi, 64, dist128, 2*dist128, rdi, 128*25*XMM_SCD2
	bump	rdi, 25*XMM_SCD2	;; Next sine/cosine pointer
	loops	128, c2b		;; Loop 128 times
	bump	rsi, -128*64+4*dist128	;; Next source pointer
	bump	rdi, -128*25*XMM_SCD2	;; Reset sine/cosine pointer
	loops	25, c2a			;; Test loop counter
	bump	rsi, -25*4*dist128	;; Restore source pointer
	end_timer 13

;; Do inverse FFT levels 3,4 (actually 2.322 FFT levels)
;; On input the 64-byte cache lines hold these data values:
;;	0	+50K	512	+50K	1024	+50K	1536	+50K
;;	1	...
;;	...
;;	511	...
;;	2048	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+50K	2560	+50K	5120	+50K	7680	+50K
;;	1	...
;;	...
;;	2559	...
;;	10240	...
;;	...

;; Do 10240 five_complex_unfft operations
;;	distance between fft data elements is 1024
;;	do 2560 macros each processing 40 data values

	start_timer 14
	bump	rcx, -2*64		;; Back up prefetch pointer by 2 cache lines
	loops_init_prefetch 2560, 64, 64, rcx ;; 5 iters of 4 iters of 128
	xtouch	[rcx+40*64-64]		;; Load prefetch TLB
	r5_x5cl_five_complex_djbunfft_preload
c7b:	r5_x5cl_five_complex_djbunfft rsi, 64, 4*dist128, rdi, 512*5*XMM_SCD2
	bump	rdi, 5*XMM_SCD2		;; Next sine/cosine pointer
	loops	128, c7b		;; Loop 128 times
	bump	rsi, -128*64+dist128	;; Next source pointer
	loops	4, c7b			;; Loop 4 times
	bump	rsi, -4*dist128+20*dist128 ;; Next source pointer
	bump	rdi, -512*5*XMM_SCD2	;; Reset sine/cosine pointer
	loops	5, c7b			;; Test loop counter
	bump	rsi, -5*20*dist128	;; Restore source pointer
	end_timer 14

;; Do inverse FFT levels 1,2 (actually 2.322 FFT levels)
;; On input the 64-byte cache lines hold these data values:
;;	0	+50K	2560	+50K	5120	+50K	7680	+50K
;;	1	...
;;	...
;;	2559	...
;;	10240	...
;;	...
;; On output the 64-byte cache lines hold these data values if pass 1 swizzles:
;;	0	+50K	12800	+50K	25600	+50K	38400	+50K
;;	1	...
;;	...
;;	12799	...
;; On output the 64-byte cache lines hold these data values if pass 2 swizzles:
;;	0	+12800	25600	+12800	51200	+12800	76800	+12800
;;	1	...
;;	...
;;	12799	...

;; Do 10240 five_complex_unfft operations
;;	distance between fft data elements is 5120
;;	do 2560 macros each processing 40 data values

	start_timer 15
	loops_init 2560			;; 20 iters of 128
	r5_x5cl_five_complex_djbunfft_preload
c1b:	r5_x5cl_five_complex_djbunfft rsi, 64, 20*dist128, rdi, 2560*XMM_SCD2
	bump	rdi, XMM_SCD2		;; Next set of sin/cos data
	loops	128, c1b		;; Test inner loop counter
	bump	rsi, -128*64+dist128	;; Next source pointer
	loops	20, c1b			;; Test outer loop counter
	end_timer 15

xpass2_25600_levels_complex_done:
	ENDM


