; Copyright 2011-2023 - Mersenne Research, Inc.  All rights reserved
; Author:  George Woltman
; Email: woltman@alum.mit.edu
;
; These macros do the second pass of a radix-4 2 pass FFT using AVX instructions.
;

;;
;; Estimated memory requirements for complex pass 2 macros:
;;
;; Pass 2 size:		48	64	80	192	256	320	768	1024	1280	1536	2048	2560	2304	3072
;; FFT data:		<1K	1K	2K	3K	4K	5K	12K	16K	20K	24K	32K	40K	36K	48K
;; Fixed sin/cos data:	<1K	<1K	<1K	2K	2K	3K	6K	9K	10K	13K	17K	21K	18K	26K
;; Min L2 cache size:	2K	3K	4K	8K	10K	13K	30K	41K	50K	61K	81K	101K	90K	122K
;;
;; Pass 2 size:		3840	4096	5120	6400	4608	6144	7680	8192	10240	12800	9216	12288	15360	16384	20480	25600
;; FFT data:		60K	64K	80K	100K	72K	96K	120K	128K	160K	200K	144K	192K	240K	256K	320K	400K
;; Fixed sin/cos data:	30K	34K	42K	50K	36K	49K	60K	65K	81K	100K	72K	98K	120K	130K	162K	200K
;; Min L2 cache size:	150K	162K	202K	250K	180K	241K	300K	321K	401K	500K	360K	482K	600K	644K	802K	1000K
;;
;; NOTE:  The total size is a minimum estimated efficient L2 cache size.
;; We double the data size as both the current data and next block data must
;; be in the cache at the same time.  The full formula is:
;;	(FFT data size) * 2 + fixed sin/cos data
;; ALSO NOTE:  The real sin/cos data is used only in the first FFT block, so the memory
;; requirements for real data aren't particularly interesting.
;;


;*********************************************************************************************
;			Macros derived from pass2-6-levels
;*********************************************************************************************


; *************** radix4-pass2-48-real macro ******************
; This macro takes 96 real values and performs the final 6.585 levels of the FFT.
; The results are squared, and the first 6.585 levels of the inverse FFT are done.

yr4dwpn_pass2_48_real MACRO
	LOCAL	b1b, b2b, b3a, b3b, b3c, b3d, c1b, c2b
	LOCAL	ypass2_48_real_1, ypass2_48_real_2
	LOCAL	ypass2_48_real_3, ypass2_48_real_4
	LOCAL	ypass2_48_real_unfft, ypass2_48_real_done

;; Type 4 FFTs skip the forward FFT process

	start_timer 2
	mov	rsi, DATA_ADDR		;; Load source address
	mov	rbx, DIST_TO_FFTSRCARG
	mov	rcx, DATA_PREFETCH	;; Prefetch pointer (we need to prefetch 
					;; 12 cache lines + 0 pad cache lines)
	cmp	ffttype, 4
	je	ypass2_48_real_4

;; Do FFT levels 1,2,3 which correspond to complex levels 1,2
;; On input the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	48	+1	+1	+1
;;	4	...
;;	...
;;	44
;; On output the 64-byte cache lines are swizzled to hold these data values:
;;	0	+1	+1	+1	16	+1	+1	+1
;;	4	...
;;	...
;;	12	...
;;	32	...
;;	...

;; Do 16 six_reals_fft operations
;;	distance between fft data elements is 16
;;	do 4 macros each processing 24 data values

	mov	rdi, sincos3		;; Real sin/cos data
	mov	rbp, xsincos_complex	;; Complex sin/cos data
	yloop_init 4
	yloop_set_prefetch YSINGLE	;; Prefetch 4 cache lines
	yr3_f3cl_2sc_six_reals_fft_preload
b1b:	yr3_f3cl_2sc_six_reals_fft rsi, 64, 4*64, rbp, YMM_SCD1, rdi, YMM_SCD1, 4
	yloop	4, b1b, -4*64		;; Test loop counter, restore source pointer

;; Do FFT levels 3,4
;; On input the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	16	+1	+1	+1
;;	4	...
;;	...
;;	12	...
;;	32	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+8	+8	+8	4	+8	+8	+8
;;	1	...
;;	...
;;	32	...
;;	...

;; Do 4 eight_reals_fft operations
;; Do 8 four_complex_djbfft operations
;;	distance between fft data elements is 4
;;	do 3 macros each processing 32 data values

	yr4_s4cl_2sc_eight_reals_fft_preload
	yr4_s4cl_2sc_eight_reals_fft rsi, 4*64, 64, 2*64, rbp, 0, rdi, 0, 1

	yloop_init 2			;; 2 more iters
	yloop_set_prefetch YSINGLE	;; Prefetch 2 cache lines
	yr4_s4cl_four_complex_djbfft_preload
b2b:	yr4_s4cl_four_complex_djbfft rsi, 4*64, 64, 2*64, rbp, 0, 2
	yloop	2, b2b, -3*4*64		;; Loop 2 times, restore source pointer

;; Do FFT levels 5,6
;; On input and output the 64-byte cache lines hold these data values:
;;	0	+8	+8	+8	4	+8	+8	+8
;;	1	...
;;	...
;;	32	...
;;	...

;; Do 1 eight_reals_fft operations
;; Do 11 four_complex_fft operations
;;	distance between fft data elements is 1
;;	do 3 macros each processing 32 data values

;; Execute the proper middle step

	cmp	ffttype, 2
	jl	ypass2_48_real_1
	jg	ypass2_48_real_3

ypass2_48_real_2:
;;	yr4_4cl_eight_reals_four_complex_with_square rsi, 4*64, 64, 2*64
	yr4_dispatch call, yreal_square, 64, 2*64
	yloop_init 2			;; 2 more iterations
	yloop_set_prefetch YDOUBLE	;; Prefetch 4 cache lines
	yr4_4cl_four_complex_with_square_preload
b3b:	yr4_4cl_four_complex_with_square rsi, 4*64, 64, 2*64, 1
	yloop	2, b3b, -3*4*64		;; Test loop counter, restore source pointer
	jmp	ypass2_48_real_unfft

ypass2_48_real_1:
;;	yr4_4cl_eight_reals_four_complex_fft_final rsi, 4*64, 64, 2*64
	yr4_dispatch call, yreal_fft_final, 64, 2*64
	mov	rcx, xsincos_complex	;; Prefetch complex sin/cos data, 6 cache lines
	yloop_init 2			;; 2 more iterations
	yloop_set_prefetch YDOUBLE	;; Prefetch 4 cache lines
b3a:	yr4_4cl_four_complex_fft_final rsi, 4*64, 64, 2*64
	yloop	2, b3a			;; Test loop counter
	jmp	ypass2_48_real_done

ypass2_48_real_3:
x64	mov	r8, DEST2ARG
	mov	rbp, DIST_TO_MULSRCARG
;;	yr4_4cl_eight_reals_four_complex_with_mult rsi, 4*64, 64, 2*64
	yr4_dispatch call, yreal_mult, 64, 2*64
	yloop_init 2			;; 2 more iterations
	yloop_set_prefetch YDOUBLE	;; Prefetch 4 cache lines
	yr4_4cl_four_complex_with_mult_preload
b3c:	yr4_4cl_four_complex_with_mult rsi, 4*64, 64, 2*64
	yloop	2, b3c, -3*4*64		;; Test loop counter, restore source pointer
x64	add	rsi, r8
	jmp	ypass2_48_real_unfft

ypass2_48_real_4:
	mov	rbp, DIST_TO_MULSRCARG
;;	yr4_4cl_eight_reals_four_complex_with_mulf rsi, 4*64, 64, 2*64
	yr4_dispatch call, yreal_mulf, 64, 2*64
	yloop_init 2			;; 2 more iterations
	yloop_set_prefetch YDOUBLE	;; Prefetch 4 cache lines
	yr4_4cl_four_complex_with_mulf_preload
b3d:	yr4_4cl_four_complex_with_mulf rsi, 4*64, 64, 2*64
	yloop	2, b3d, -3*4*64		;; Test loop counter, restore source pointer
;;	bump	rcx, (-4+4+2+4)*64	;; Move prefetch pointer back to where squaring would be

;; Do inverse FFT levels 3,4
;; On input the 64-byte cache lines hold these data values:
;;	0	+8	+8	+8	4	+8	+8	+8
;;	1	...
;;	...
;;	32	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	16	+1	+1	+1
;;	4	...
;;	...
;;	12	...
;;	32	...
;;	...

;; Do 1 eight_reals_unfft operations
;; Do 11 four_complex_djbunfft operations
;;	distance between fft data elements is 4
;;	do 4 macros processing 32 data values

ypass2_48_real_unfft:
	mov	rdi, sincos3		;; Real sin/cos pointer
	bump	rdi, 4*YMM_SCD1
	mov	rbp, xsincos_complex	;; Complex sin/cos data
	bump	rbp, 4*YMM_SCD1
	mov	rcx, xsincos_complex	;; Prefetch complex sin/cos data, 6 cache lines

	yr4_s4cl_2sc_eight_reals_unfft_preload
	yr4_s4cl_2sc_eight_reals_unfft rsi, 4*64, 64, 2*64, rbp, 0, rdi, 0, 1

	yloop_init 2			;; 2 more iters
	yloop_set_prefetch YSINGLE	;; Prefetch 2 cache lines
	yr4_s4cl_four_complex_djbunfft_preload
c2b:	yr4_s4cl_four_complex_djbunfft rsi, 4*64, 64, 2*64, rbp, 0, 2
	yloop	2, c2b, -3*4*64		;; Test loop counter, restore source pointer

;; Do inverse FFT levels 1,2,3 which correspond to complex levels 1,2
;; On input the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	16	+1	+1	+1
;;	4	...
;;	...
;;	12	...
;;	32	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	48	+1	+1	+1
;;	4	...
;;	...
;;	44

;; Do 16 six_reals_unfft operations
;;	distance between fft data elements is 16
;;	do 4 macros processing 24 data values

	bump	rdi, -4*YMM_SCD1	;; Real sin/cos ptr
	bump	rbp, -4*YMM_SCD1	;; Complex sin/cos data
;;	bump	rcx, (6-2-4)*64		;; Move prefetch pointer along
	yloop_init 4			;; 4 iterations
	yloop_set_prefetch YSINGLE	;; Prefetch 4 cache lines
	yr3_3cl_2sc_six_reals_unfft_preload
c1b:	yr3_3cl_2sc_six_reals_unfft rsi, 64, 4*64, rbp, YMM_SCD1, rdi, YMM_SCD1, 4
	yloop	4, c1b			;; Test loop counter

ypass2_48_real_done:
	end_timer 2
	ENDM

; *************** radix4-pass2-48-complex macro ******************
; This macro takes 48 complex values and performs the final 5.585 levels of a standard
; mostly radix-4 FFT, squares the results, and does the first 5.585 levels of the inverse FFT.

yr4dwpn_pass2_48_complex MACRO
	LOCAL	b1b, b2b, b3a, b3b, b3c, b3d, c1b, c2b
	LOCAL	ypass2_48_complex_1, ypass2_48_complex_2
	LOCAL	ypass2_48_complex_3, ypass2_48_complex_4
	LOCAL	ypass2_48_complex_unfft, ypass2_48_complex_done

;; Type 4 FFTs skip the forward FFT process

	mov	rsi, DATA_ADDR		;; Load address of FFT data
	mov	rbx, DIST_TO_FFTSRCARG
	mov	rcx, DATA_PREFETCH	;; Prefetch pointer (we need to prefetch 
					;; 12 cache lines + 0 pad cache lines)
	cmp	ffttype, 4
	je	ypass2_48_complex_4

;; Do FFT levels 1,2
;; On input the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	48	+1	+1	+1
;;	4	...
;;	...
;;	44
;; On output the 64-byte cache lines are swizzled to hold these data values:
;;	0	+1	+1	+1	16	+1	+1	+1
;;	4	...
;;	...
;;	12	...
;;	32	...
;;	...

;; Do 16 three_complex_fft operations
;;	distance between fft data elements is 16
;;	do 4 macros each processing 24 data values

	start_timer 4
	mov	rdi, xsincos_complex	;; The sin/cos data
	yloop_init 4			;; 4 iterations
	yr3_f3cl_three_complex_djbfft_preload
b1b:	yr3_f3cl_three_complex_djbfft rsi, 64, 4*64, rdi, YMM_SCD1, 4
	yloop	4, b1b, -4*64		;; Test inner loop counter, restore source pointer
	end_timer 4

;; Do FFT levels 3,4
;; On input the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	16	+1	+1	+1
;;	4	...
;;	...
;;	12	...
;;	32	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+8	+8	+8	4	+8	+8	+8
;;	1	...
;;	...
;;	32	...
;;	...

;; Do 12 four_complex_fft operations
;;	distance between fft data elements is 4
;;	do 3 macros each processing 32 data values

	start_timer 6
	yloop_init 3			;; 3 iters
	yloop_set_prefetch YSINGLE	;; Prefetch 3 cache lines
	yr4_s4cl_four_complex_djbfft_preload
b2b:	yr4_s4cl_four_complex_djbfft rsi, 4*64, 64, 2*64, rdi, 0, 3
	yloop	3, b2b, -3*4*64		;; Loop 3 times, restore source pointer
	end_timer 6

;; Do FFT levels 5,6 as well as inverse FFT levels 5,6
;; On input and output the 64-byte cache lines hold these data values:
;;	0	+8	+8	+8	4	+8	+8	+8
;;	1	...
;;	...
;;	32	...
;;	...

;; Do 12 four_complex_fft operations
;;	distance between fft data elements is 1
;;	do 3 macros each processing 32 data values

;; Execute the right middle step

	cmp	ffttype, 2
	jl	ypass2_48_complex_1
	jg	ypass2_48_complex_3

ypass2_48_complex_2:
	start_timer 9
	yloop_init 3
	yloop_set_prefetch YSINGLE	;; Prefetch 3 cache lines
	yr4_4cl_four_complex_with_square_preload
b3b:	yr4_4cl_four_complex_with_square rsi, 4*64, 64, 2*64, 3
	yloop	3, b3b, -3*4*64		;; Loop 3 times, restore source pointer
	end_timer 9
	jmp	ypass2_48_complex_unfft

ypass2_48_complex_1:
	start_timer 9
	yloop_init 3
	yloop_set_prefetch YDOUBLE	;; Prefetch 6 cache lines
b3a:	yr4_4cl_four_complex_fft_final rsi, 4*64, 64, 2*64
	yloop	3, b3a			;; Loop 3 times
	end_timer 9
	jmp	ypass2_48_complex_done

ypass2_48_complex_3:
	start_timer 9
x64	mov	r8, DEST2ARG
	mov	rbp, DIST_TO_MULSRCARG
	yloop_init 3
	yloop_set_prefetch YSINGLE	;; Prefetch 3 cache lines
	yr4_4cl_four_complex_with_mult_preload
b3c:	yr4_4cl_four_complex_with_mult rsi, 4*64, 64, 2*64
	yloop	3, b3c, -3*4*64		;; Loop 3 times, restore source pointer
x64	add	rsi, r8
	end_timer 9
	jmp	ypass2_48_complex_unfft

ypass2_48_complex_4:
	start_timer 9
	mov	rbp, DIST_TO_MULSRCARG
	yloop_init 3
	yloop_set_prefetch YDOUBLE	;; Prefetch 6 cache lines
	yr4_4cl_four_complex_with_mulf_preload
b3d:	yr4_4cl_four_complex_with_mulf rsi, 4*64, 64, 2*64
	yloop	3, b3d, -3*4*64		;; Loop 3 times, restore source pointer
	end_timer 9

;; Do inverse FFT levels 3,4
;; On input the 64-byte cache lines hold these data values:
;;	0	+8	+8	+8	4	+8	+8	+8
;;	1	...
;;	...
;;	32	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	16	+1	+1	+1
;;	4	...
;;	...
;;	12	...
;;	32	...
;;	...

;; Do 12 four_complex_unfft operations
;;	distance between fft data elements is 4
;;	do 3 macros each processing 32 data values

ypass2_48_complex_unfft:
	start_timer 11
	mov	rdi, xsincos_complex	;; Load sin/cos pointers
	bump	rdi, 4*YMM_SCD1
	yloop_init 3			;; 3 iters
	yloop_set_prefetch YSINGLE	;; Prefetch 3 cache lines
	yr4_s4cl_four_complex_djbunfft_preload
c2b:	yr4_s4cl_four_complex_djbunfft rsi, 4*64, 64, 2*64, rdi, 0, 3
	yloop	3, c2b, -3*4*64		;; Loop 3 times, restore source pointer
	end_timer 11

;; Do inverse FFT levels 1,2
;; On input the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	16	+1	+1	+1
;;	4	...
;;	...
;;	12	...
;;	32	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	48	+1	+1	+1
;;	4	...
;;	...
;;	44

;; Do 16 three_complex_unfft operations
;;	distance between fft data elements is 16
;;	do 4 macros each processing 24 data values

	start_timer 14
	bump	rdi, -4*YMM_SCD1	;; Load sin/cos ptr
	bump	rcx, (12+0-3*3-4)*64	;; Move prefetch pointer along
	yloop_init 4			;; 4 iters
	yloop_set_prefetch YSINGLE	;; Prefetch 4 cache lines
	yr3_3cl_three_complex_djbunfft_preload
c1b:	yr3_3cl_three_complex_djbunfft rsi, 64, 4*64, rdi, YMM_SCD1, 4
	yloop	4, c1b			;; Test loop counter
	end_timer 14

ypass2_48_complex_done:
	ENDM


; *************** radix4-pass2-6-levels-real macro ******************
; This macro takes 128 real values and performs the final 7 levels of the FFT.
; The results are squared, and the first 7 levels of the inverse FFT are done.

yr4dwpn_pass2_6_levels_real MACRO
	LOCAL	b1b, b2b, b3a, b3b, b3c, b3d, c1b, c2b
	LOCAL	ypass2_6_levels_real_1, ypass2_6_levels_real_2
	LOCAL	ypass2_6_levels_real_3, ypass2_6_levels_real_4
	LOCAL	ypass2_6_levels_real_unfft, ypass2_6_levels_real_done

;; Type 4 FFTs skip the forward FFT process

	start_timer 2
	mov	rsi, DATA_ADDR		;; Load source address
	mov	rbx, DIST_TO_FFTSRCARG
	mov	rcx, DATA_PREFETCH	;; Prefetch pointer (we need to prefetch 
					;; 16 cache lines + 0 pad cache lines)
	cmp	ffttype, 4
	je	ypass2_6_levels_real_4

;; Do FFT levels 1,2,3 which correspond to complex levels 1,2
;; On input the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	64	+1	+1	+1
;;	4	...
;;	...
;;	60
;; On output the 64-byte cache lines are swizzled to hold these data values:
;;	0	+32	+32	+32	16	+32	+32	+32
;;	4	...
;;	...
;;	12	...
;;	1	...
;;	...

;; Do 16 eight_reals_fft operations
;;	distance between fft data elements is 16
;;	do 4 macros each processing 32 data values

	mov	rdi, sincos3		;; Real sin/cos data
	mov	rbp, xsincos_complex	;; Complex sin/cos data
	yloop_init 4
	yloop_set_prefetch YSINGLE
	yr4_fs4cl_2sc_eight_reals_fft_preload
b1b:	yr4_fs4cl_2sc_eight_reals_fft rsi, 64, 4*64, 8*64, rbp, YMM_SCD2, rdi, YMM_SCD2, 4
	yloop	4, b1b, -4*64		;; Test loop counter, restore source pointer

;; Do FFT levels 3,4
;; On input the 64-byte cache lines hold these data values:
;;	0	+32	+32	+32	16	+32	+32	+32
;;	4	...
;;	...
;;	12	...
;;	1	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+32	+32	+32	4	+32	+32	+32
;;	8	...
;;	...
;;	1	...
;;	...

;; Do 4 eight_reals_fft operations
;; Do 12 four_complex_djbfft operations
;;	distance between fft data elements is 4
;;	do 4 macros each processing 32 data values

	yloop_init 4			
	yloop_set_prefetch YSINGLE
	yr4_4cl_eight_reals_four_complex_djbfft_preload
b2b:	yr4_4cl_eight_reals_four_complex_djbfft rsi, 4*64, 64, 2*64, rdi, YMM_SCD3, 4
	yloop	4, b2b, -4*4*64, rdi, -4*YMM_SCD3 ;; Test loop counter, restore source pointer, restore sin/cos ptr

;; Do FFT levels 5,6
;; On input and output the 64-byte cache lines hold these data values:
;;	0	+32	+32	+32	4	+32	+32	+32
;;	8	...
;;	...
;;	1	...
;;	...

;; Do 1 eight_reals_fft operations
;; Do 15 four_complex_fft operations
;;	distance between fft data elements is 1
;;	do 4 macros each processing 32 data values

;; Execute the proper middle step

	cmp	ffttype, 2
	jl	ypass2_6_levels_real_1
	jg	ypass2_6_levels_real_3

ypass2_6_levels_real_2:
;;	yr4_4cl_eight_reals_four_complex_with_square rsi, 64, 4*64, 8*64
	yr4_dispatch call, yreal_square, 4*64, 8*64
	yloop_init 4			;; 3 more iterations
	yloop_adjust 1
	yloop_set_prefetch YDOUBLE
	yr4_4cl_four_complex_with_square_preload
b3b:	yr4_4cl_four_complex_with_square rsi, 64, 4*64, 8*64, 1
	yloop	4, b3b, -4*64		;; Test loop counter, restore source pointer
	jmp	ypass2_6_levels_real_unfft

ypass2_6_levels_real_1:
;;	yr4_4cl_eight_reals_four_complex_fft_final rsi, 64, 4*64, 8*64
	yr4_dispatch call, yreal_fft_final, 4*64, 8*64
	mov	rcx, xsincos_complex	;; Prefetch complex sin/cos data, 10 cache lines
	yloop_init 4			;; 3 more iterations
	yloop_adjust 1
	yloop_set_prefetch YDOUBLE
b3a:	yr4_4cl_four_complex_fft_final rsi, 64, 4*64, 8*64
	yloop	4, b3a			;; Test loop counter
	jmp	ypass2_6_levels_real_done

ypass2_6_levels_real_3:
x64	mov	r8, DEST2ARG
	mov	rbp, DIST_TO_MULSRCARG
;;	yr4_4cl_eight_reals_four_complex_with_mult rsi, 64, 4*64, 8*64
	yr4_dispatch call, yreal_mult, 4*64, 8*64
	yloop_init 4			;; 3 more iterations
	yloop_adjust 1
	yloop_set_prefetch YDOUBLE
	yr4_4cl_four_complex_with_mult_preload
b3c:	yr4_4cl_four_complex_with_mult rsi, 64, 4*64, 8*64
	yloop	4, b3c, -4*64		;; Test loop counter, restore source pointer
x64	add	rsi, r8
	jmp	ypass2_6_levels_real_unfft

ypass2_6_levels_real_4:
	mov	rbp, DIST_TO_MULSRCARG
;;	yr4_4cl_eight_reals_four_complex_with_mulf rsi, 64, 4*64, 8*64
	yr4_dispatch call, yreal_mulf, 4*64, 8*64
	yloop_init 4			;; 3 more iterations
	yloop_adjust 1
	yloop_set_prefetch YDOUBLE
	yr4_4cl_four_complex_with_mulf_preload
b3d:	yr4_4cl_four_complex_with_mulf rsi, 64, 4*64, 8*64
	yloop	4, b3d, -4*64		;; Test loop counter, restore source pointer
;;	bump	rcx, (-6+2*4-6)*64	;; Move prefetch pointer back to where squaring would be

;; Do inverse FFT levels 3,4
;; On input the 64-byte cache lines hold these data values:
;;	0	+32	+32	+32	4	+32	+32	+32
;;	8	...
;;	...
;;	1	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+32	+32	+32	16	+32	+32	+32
;;	4	...
;;	...
;;	12	...
;;	1	...
;;	...

;; Do 1 eight_reals_unfft operations
;; Do 15 four_complex_djbunfft operations
;;	distance between fft data elements is 4
;;	do 4 macros processing 32 data values

ypass2_6_levels_real_unfft:
	mov	rdi, sincos3		;; Load sin/cos pointer
	bump	rdi, 4*YMM_SCD2
	mov	rcx, xsincos_complex	;; Prefetch complex sin/cos data, 10 cache lines
	yloop_init 4			;; 4 iterations
	yloop_set_prefetch YSINGLE
	yr4_4cl_eight_reals_four_complex_djbunfft_preload
c2b:	yr4_4cl_eight_reals_four_complex_djbunfft rsi, 4*64, 64, 2*64, rdi, YMM_SCD3, 4
	yloop	4, c2b, -4*4*64		;; Test loop counter, restore source pointer

;; Do inverse FFT levels 1,2,3 which correspond to complex levels 1,2
;; On input the 64-byte cache lines hold these data values:
;;	0	+32	+32	+32	16	+32	+32	+32
;;	4	...
;;	...
;;	12	...
;;	1	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	64	+1	+1	+1
;;	4	...
;;	...
;;	60

;; Do 16 eight_reals_unfft operations
;;	distance between fft data elements is 16
;;	do 4 macros processing 32 data values

	bump	rdi, -4*YMM_SCD3-4*YMM_SCD2 ;; Real sin/cos pointer
	mov	rbp, xsincos_complex	;; Complex sin/cos data
	yloop_init 4			;; 4 iterations
	yloop_set_prefetch YSINGLE
	yr4_s4cl_2sc_eight_reals_unfft_preload
c1b:	yr4_s4cl_2sc_eight_reals_unfft rsi, 64, 4*64, 8*64, rbp, YMM_SCD2, rdi, YMM_SCD2, 4
	yloop	4, c1b			;; Test loop counter

ypass2_6_levels_real_done:
	end_timer 2
	ENDM

; *************** radix4-pass2-6-levels-complex macro ******************
; This macro takes 64 complex values and performs the final 6 levels of a standard
; mostly radix-4 FFT, squares the results, and does the first 6 levels of the inverse FFT.

yr4dwpn_pass2_6_levels_complex MACRO
	LOCAL	b1b, b2b, b3a, b3b, b3c, b3d, c1b, c2b
	LOCAL	ypass2_6_levels_complex_1, ypass2_6_levels_complex_2
	LOCAL	ypass2_6_levels_complex_3, ypass2_6_levels_complex_4
	LOCAL	ypass2_6_levels_complex_unfft, ypass2_6_levels_complex_done

;; Type 4 FFTs skip the forward FFT process

	mov	rsi, DATA_ADDR		;; Load address of FFT data
	mov	rbx, DIST_TO_FFTSRCARG
	mov	rcx, DATA_PREFETCH	;; Prefetch pointer (we need to prefetch 
					;; 16 cache lines + 0 pad cache lines)
	cmp	ffttype, 4
	je	ypass2_6_levels_complex_4

;; Do FFT levels 1,2
;; On input the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	64	+1	+1	+1
;;	4	...
;;	...
;;	60
;; On output the 64-byte cache lines are swizzled to hold these data values:
;;	0	+32	+32	+32	16	+32	+32	+32
;;	4	...
;;	...
;;	12	...
;;	1	...
;;	...

;; Do 16 four_complex_fft operations
;;	distance between fft data elements is 16
;;	do 4 macros each processing 32 data values

	start_timer 4
	mov	rdi, xsincos_complex	;; The sin/cos data
	yloop_init 4			;; 4 iterations
	yr4_fs4cl_four_complex_djbfft_preload
b1b:	yr4_fs4cl_four_complex_djbfft rsi, 64, 4*64, 8*64, rdi, YMM_SCD2, 4
	yloop	4, b1b, -4*64		;; Test inner loop counter, restore source pointer
	end_timer 4

;; Do FFT levels 3,4
;; On input the 64-byte cache lines hold these data values:
;;	0	+32	+32	+32	16	+32	+32	+32
;;	4	...
;;	...
;;	12	...
;;	1	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+32	+32	+32	4	+32	+32	+32
;;	8	...
;;	...
;;	1	...
;;	...

;; Do 16 four_complex_fft operations
;;	distance between fft data elements is 4
;;	do 4 macros each processing 32 data values

	start_timer 6
	yloop_init 4			;; 4 iters
	yloop_set_prefetch YSINGLE
	yr4_b4cl_four_complex_djbfft_preload
b2b:	yr4_b4cl_four_complex_djbfft rsi, 4*64, 64, 2*64, rdi, YMM_SCD2/4, 4
	yloop	4, b2b, -4*4*64, rdi, -4*YMM_SCD2/4 ;; Loop 4 times, restore source pointer, restore sine/cosine pointer
	end_timer 6

;; Do FFT levels 5,6 as well as inverse FFT levels 5,6
;; On input and output the 64-byte cache lines hold these data values:
;;	0	+32	+32	+32	4	+32	+32	+32
;;	8	...
;;	...
;;	1	...
;;	...

;; Do 16 four_complex_fft operations
;;	distance between fft data elements is 1
;;	do 4 macros each processing 32 data values

;; Execute the right middle step

	cmp	ffttype, 2
	jl	ypass2_6_levels_complex_1
	jg	ypass2_6_levels_complex_3

ypass2_6_levels_complex_2:
	start_timer 9
	yloop_init 4
	yloop_set_prefetch YSINGLE
	yr4_4cl_four_complex_with_square_preload
b3b:	yr4_4cl_four_complex_with_square rsi, 64, 4*64, 8*64, 4
	yloop	4, b3b, -4*64		;; Loop 4 times, restore source pointer
	end_timer 9
	jmp	ypass2_6_levels_complex_unfft

ypass2_6_levels_complex_1:
	start_timer 9
	yloop_init 4
	yloop_set_prefetch YDOUBLE
b3a:	yr4_4cl_four_complex_fft_final rsi, 64, 4*64, 8*64
	yloop	4, b3a			;; Loop 4 times
	end_timer 9
	jmp	ypass2_6_levels_complex_done

ypass2_6_levels_complex_3:
	start_timer 9
x64	mov	r8, DEST2ARG
	mov	rbp, DIST_TO_MULSRCARG
	yloop_init 4
	yloop_set_prefetch YSINGLE
	yr4_4cl_four_complex_with_mult_preload
b3c:	yr4_4cl_four_complex_with_mult rsi, 64, 4*64, 8*64
	yloop	4, b3c, -4*64		;; Loop 4 times, restore source pointer
x64	add	rsi, r8
	end_timer 9
	jmp	ypass2_6_levels_complex_unfft

ypass2_6_levels_complex_4:
	start_timer 9
	mov	rbp, DIST_TO_MULSRCARG
	yloop_init 4
	yloop_set_prefetch YDOUBLE
	yr4_4cl_four_complex_with_mulf_preload
b3d:	yr4_4cl_four_complex_with_mulf rsi, 64, 4*64, 8*64
	yloop	4, b3d, -4*64		;; Loop 4 times, restore source pointer
	end_timer 9

;; Do inverse FFT levels 3,4
;; On input the 64-byte cache lines hold these data values:
;;	0	+32	+32	+32	4	+32	+32	+32
;;	8	...
;;	...
;;	1	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+32	+32	+32	16	+32	+32	+32
;;	4	...
;;	...
;;	12	...
;;	1	...
;;	...

;; Do 16 four_complex_unfft operations
;;	distance between fft data elements is 4
;;	do 4 macros each processing 32 data values

ypass2_6_levels_complex_unfft:
	start_timer 11
	mov	rdi, xsincos_complex	;; Load sin/cos pointers
	bump	rdi, 4*YMM_SCD2

	yloop_init 4			;; 4 iters
	yloop_set_prefetch YSINGLE
	yr4_b4cl_four_complex_djbunfft_preload
c2b:	yr4_b4cl_four_complex_djbunfft rsi, 4*64, 64, 2*64, rdi, YMM_SCD2/4, 4
	yloop	4, c2b, -4*4*64		;; Loop 4 times, restore source pointer
	end_timer 11

;; Do inverse FFT levels 1,2
;; On input the 64-byte cache lines hold these data values:
;;	0	+32	+32	+32	16	+32	+32	+32
;;	4	...
;;	...
;;	12	...
;;	1	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	64	+1	+1	+1
;;	4	...
;;	...
;;	60

;; Do 16 four_complex_unfft operations
;;	distance between fft data elements is 16
;;	do 4 macros each processing 32 data values

	start_timer 14
	mov	rdi, xsincos_complex	;; Load sin/cos pointer
	yloop_init 4			;; 4 iters
	yloop_set_prefetch YSINGLE
	yr4_s4cl_four_complex_djbunfft_preload
c1b:	yr4_s4cl_four_complex_djbunfft rsi, 64, 4*64, 8*64, rdi, YMM_SCD2, 4
	yloop	4, c1b			;; Test loop counter
	end_timer 14

ypass2_6_levels_complex_done:
	ENDM


; *************** radix4-pass2-80-real macro ******************
; This macro takes 160 real values and performs the final 7.322 levels of the FFT.
; The results are squared, and the first 7.322 levels of the inverse FFT are done.

yr4dwpn_pass2_80_real MACRO
	LOCAL	b1b, b2b, b3a, b3b, b3c, b3d, c1b, c2b
	LOCAL	ypass2_80_real_1, ypass2_80_real_2
	LOCAL	ypass2_80_real_3, ypass2_80_real_4
	LOCAL	ypass2_80_real_unfft, ypass2_80_real_done

;; Type 4 FFTs skip the forward FFT process

	start_timer 2
	mov	rsi, DATA_ADDR		;; Load source address
	mov	rbx, DIST_TO_FFTSRCARG
	mov	rcx, DATA_PREFETCH	;; Prefetch pointer (we need to prefetch 
					;; 20 cache lines + 0 pad cache lines)
	cmp	ffttype, 4
	je	ypass2_80_real_4

;; Do FFT levels 1,2,3 which correspond to complex levels 1,2
;; On input the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	80	+1	+1	+1
;;	4	...
;;	...
;;	76
;; On output the 64-byte cache lines are swizzled to hold these data values:
;;	0	+1	+1	+1	16	+1	+1	+1
;;	4	...
;;	...
;;	12	...
;;	32	...
;;	...

;; Do 16 ten_reals_fft operations
;;	distance between fft data elements is 16
;;	do 4 macros each processing 40 data values

	mov	rdi, sincos3		;; Real sin/cos data
	mov	rbp, xsincos_complex	;; Complex sin/cos data
	yloop_init 4
	yloop_set_prefetch YSINGLE
	yr5_f5cl_2sc_ten_reals_fft_preload
b1b:	yr5_f5cl_2sc_ten_reals_fft rsi, 64, 4*64, rbp, YMM_SCD2, rdi, YMM_SCD2, 4
	yloop	4, b1b, -4*64		;; Test loop counter, restore source pointer

;; Do FFT levels 3,4
;; On input the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	16	+1	+1	+1
;;	4	...
;;	...
;;	12	...
;;	32	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+8	+8	+8	4	+8	+8	+8
;;	1	...
;;	...
;;	32	...
;;	...

;; Do 4 eight_reals_fft operations
;; Do 16 four_complex_djbfft operations
;;	distance between fft data elements is 4
;;	do 5 macros each processing 32 data values

	yr4_s4cl_2sc_eight_reals_fft_preload
	yr4_s4cl_2sc_eight_reals_fft rsi, 4*64, 64, 2*64, rbp, 0, rdi, 0, 1

	yloop_init 4			;; 4 more iters
	yloop_set_prefetch YDOUBLE
	yr4_s4cl_four_complex_djbfft_preload
b2b:	yr4_s4cl_four_complex_djbfft rsi, 4*64, 64, 2*64, rbp, 0, 4
	yloop	4, b2b, -5*4*64		;; Loop 4 times, restore source pointer

;; Do FFT levels 5,6
;; On input and output the 64-byte cache lines hold these data values:
;;	0	+8	+8	+8	4	+8	+8	+8
;;	1	...
;;	...
;;	32	...
;;	...

;; Do 1 eight_reals_fft operations
;; Do 19 four_complex_fft operations
;;	distance between fft data elements is 1
;;	do 5 macros each processing 32 data values

;; Execute the proper middle step

	cmp	ffttype, 2
	jl	ypass2_80_real_1
	jg	ypass2_80_real_3

ypass2_80_real_2:
;;	yr4_4cl_eight_reals_four_complex_with_square rsi, 4*64, 64, 2*64
	yr4_dispatch call, yreal_square, 64, 2*64
	yloop_init 4			;; 4 more iterations
	yloop_set_prefetch YDOUBLE
	yr4_4cl_four_complex_with_square_preload
b3b:	yr4_4cl_four_complex_with_square rsi, 4*64, 64, 2*64, 1
	yloop	4, b3b, -5*4*64		;; Test loop counter, restore source pointer
	jmp	ypass2_80_real_unfft

ypass2_80_real_1:
;;	yr4_4cl_eight_reals_four_complex_fft_final rsi, 4*64, 64, 2*64
	yr4_dispatch call, yreal_fft_final, 64, 2*64
	mov	rcx, xsincos_complex	;; Prefetch complex sin/cos data, 10 cache lines
	yloop_init 4			;; 4 more iterations
	yloop_set_prefetch YDOUBLE
b3a:	yr4_4cl_four_complex_fft_final rsi, 4*64, 64, 2*64
	yloop	4, b3a			;; Test loop counter
	jmp	ypass2_80_real_done

ypass2_80_real_3:
x64	mov	r8, DEST2ARG
	mov	rbp, DIST_TO_MULSRCARG
;;	yr4_4cl_eight_reals_four_complex_with_mult rsi, 4*64, 64, 2*64
	yr4_dispatch call, yreal_mult, 64, 2*64
	yloop_init 4			;; 4 more iterations
	yloop_set_prefetch YDOUBLE
	yr4_4cl_four_complex_with_mult_preload
b3c:	yr4_4cl_four_complex_with_mult rsi, 4*64, 64, 2*64
	yloop	4, b3c, -5*4*64		;; Test loop counter, restore source pointer
x64	add	rsi, r8
	jmp	ypass2_80_real_unfft

ypass2_80_real_4:
	mov	rbp, DIST_TO_MULSRCARG
;;	yr4_4cl_eight_reals_four_complex_with_mulf rsi, 4*64, 64, 2*64
	yr4_dispatch call, yreal_mulf, 64, 2*64
	yloop_init 4			;; 4 more iterations
	yloop_set_prefetch YDOUBLE
	yr4_4cl_four_complex_with_mulf_preload
b3d:	yr4_4cl_four_complex_with_mulf rsi, 4*64, 64, 2*64
	yloop	4, b3d, -5*4*64		;; Test loop counter, restore source pointer
;;	bump	rcx, (-8+4+2*8)*64	;; Move prefetch pointer back to where squaring would be

;; Do inverse FFT levels 3,4
;; On input the 64-byte cache lines hold these data values:
;;	0	+8	+8	+8	4	+8	+8	+8
;;	1	...
;;	...
;;	32	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	16	+1	+1	+1
;;	4	...
;;	...
;;	12	...
;;	32	...
;;	...

;; Do 1 eight_reals_unfft operations
;; Do 19 four_complex_djbunfft operations
;;	distance between fft data elements is 4
;;	do 4 macros processing 32 data values

ypass2_80_real_unfft:
	mov	rdi, sincos3		;; Real sin/cos pointer
	bump	rdi, 4*YMM_SCD2
	mov	rbp, xsincos_complex	;; Complex sin/cos data
	bump	rbp, 4*YMM_SCD2
	mov	rcx, xsincos_complex	;; Prefetch complex sin/cos data, 10 cache lines

	yr4_s4cl_2sc_eight_reals_unfft_preload
	yr4_s4cl_2sc_eight_reals_unfft rsi, 4*64, 64, 2*64, rbp, 0, rdi, 0, 1

	yloop_init 4			;; 4 more iters
	yloop_set_prefetch YSINGLE
	yr4_s4cl_four_complex_djbunfft_preload
c2b:	yr4_s4cl_four_complex_djbunfft rsi, 4*64, 64, 2*64, rbp, 0, 4
	yloop	4, c2b, -5*4*64		;; Test loop counter, restore source pointer

;; Do inverse FFT levels 1,2,3 which correspond to complex levels 1,2
;; On input the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	16	+1	+1	+1
;;	4	...
;;	...
;;	12	...
;;	32	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	80	+1	+1	+1
;;	4	...
;;	...
;;	76

;; Do 16 ten_reals_unfft operations
;;	distance between fft data elements is 16
;;	do 4 macros processing 40 data values

	bump	rdi, -4*YMM_SCD2	;; Real sin/cos ptr
	bump	rbp, -4*YMM_SCD2	;; Complex sin/cos data
;;	bump	rcx, (10-4-4)*64	;; Move prefetch pointer along
	yloop_init 4			;; 4 iterations
	yloop_set_prefetch YSINGLE
	yr5_5cl_2sc_ten_reals_unfft_preload
c1b:	yr5_5cl_2sc_ten_reals_unfft rsi, 64, 4*64, rbp, YMM_SCD2, rdi, YMM_SCD2, 4
	yloop	4, c1b			;; Test loop counter

ypass2_80_real_done:
	end_timer 2
	ENDM

; *************** radix4-pass2-80-complex macro ******************
; This macro takes 80 complex values and performs the final 6.322 levels of a standard
; mostly radix-4 FFT, squares the results, and does the first 6.322 levels of the inverse FFT.

yr4dwpn_pass2_80_complex MACRO
	LOCAL	b1b, b2b, b3a, b3b, b3c, b3d, c1b, c2b
	LOCAL	ypass2_80_complex_1, ypass2_80_complex_2
	LOCAL	ypass2_80_complex_3, ypass2_80_complex_4
	LOCAL	ypass2_80_complex_unfft, ypass2_80_complex_done

;; Type 4 FFTs skip the forward FFT process

	mov	rsi, DATA_ADDR		;; Load address of FFT data
	mov	rbx, DIST_TO_FFTSRCARG
	mov	rcx, DATA_PREFETCH	;; Prefetch pointer (we need to prefetch 
					;; 20 cache lines + 0 pad cache lines)
	cmp	ffttype, 4
	je	ypass2_80_complex_4

;; Do FFT levels 1,2
;; On input the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	80	+1	+1	+1
;;	4	...
;;	...
;;	76
;; On output the 64-byte cache lines are swizzled to hold these data values:
;;	0	+1	+1	+1	16	+1	+1	+1
;;	4	...
;;	...
;;	12	...
;;	32	...
;;	...

;; Do 16 five_complex_fft operations
;;	distance between fft data elements is 16
;;	do 4 macros each processing 40 data values

	start_timer 4
	mov	rdi, xsincos_complex	;; The sin/cos data
	yloop_init 4			;; 4 iterations
	yr5_f5cl_five_complex_djbfft_preload
b1b:	yr5_f5cl_five_complex_djbfft rsi, 64, 4*64, rdi, YMM_SCD2, 4
	yloop	4, b1b, -4*64		;; Test inner loop counter, restore source pointer
	end_timer 4

;; Do FFT levels 3,4
;; On input the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	16	+1	+1	+1
;;	4	...
;;	...
;;	12	...
;;	32	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+8	+8	+8	4	+8	+8	+8
;;	1	...
;;	...
;;	32	...
;;	...

;; Do 20 four_complex_fft operations
;;	distance between fft data elements is 4
;;	do 5 macros each processing 32 data values

	start_timer 6
	yloop_init 5			;; 5 iters
	yloop_set_prefetch YSINGLE
	yr4_s4cl_four_complex_djbfft_preload
b2b:	yr4_s4cl_four_complex_djbfft rsi, 4*64, 64, 2*64, rdi, 0, 5
	yloop	5, b2b, -5*4*64		;; Loop 5 times, restore source pointer
	end_timer 6

;; Do FFT levels 5,6 as well as inverse FFT levels 5,6
;; On input and output the 64-byte cache lines hold these data values:
;;	0	+8	+8	+8	4	+8	+8	+8
;;	1	...
;;	...
;;	32	...
;;	...

;; Do 20 four_complex_fft operations
;;	distance between fft data elements is 1
;;	do 5 macros each processing 32 data values

;; Execute the right middle step

	cmp	ffttype, 2
	jl	ypass2_80_complex_1
	jg	ypass2_80_complex_3

ypass2_80_complex_2:
	start_timer 9
	yloop_init 5			
	yloop_set_prefetch YSINGLE
	yr4_4cl_four_complex_with_square_preload
b3b:	yr4_4cl_four_complex_with_square rsi, 4*64, 64, 2*64, 5
	yloop	5, b3b, -5*4*64		;; Loop 5 times, restore source pointer
	end_timer 9
	jmp	ypass2_80_complex_unfft

ypass2_80_complex_1:
	start_timer 9
	yloop_init 5
	yloop_set_prefetch YDOUBLE
b3a:	yr4_4cl_four_complex_fft_final rsi, 4*64, 64, 2*64
	yloop	5, b3a			;; Loop 5 times
	end_timer 9
	jmp	ypass2_80_complex_done

ypass2_80_complex_3:
	start_timer 9
x64	mov	r8, DEST2ARG
	mov	rbp, DIST_TO_MULSRCARG
	yloop_init 5
	yloop_set_prefetch YSINGLE
	yr4_4cl_four_complex_with_mult_preload
b3c:	yr4_4cl_four_complex_with_mult rsi, 4*64, 64, 2*64
	yloop	5, b3c, -5*4*64		;; Loop 5 times, restore source pointer
x64	add	rsi, r8
	end_timer 9
	jmp	ypass2_80_complex_unfft

ypass2_80_complex_4:
	start_timer 9
	mov	rbp, DIST_TO_MULSRCARG
	yloop_init 5
	yloop_set_prefetch YDOUBLE
	yr4_4cl_four_complex_with_mulf_preload
b3d:	yr4_4cl_four_complex_with_mulf rsi, 4*64, 64, 2*64
	yloop	5, b3d, -5*4*64		;; Loop 5 times, restore source pointer
	end_timer 9

;; Do inverse FFT levels 3,4
;; On input the 64-byte cache lines hold these data values:
;;	0	+8	+8	+8	4	+8	+8	+8
;;	1	...
;;	...
;;	32	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	16	+1	+1	+1
;;	4	...
;;	...
;;	12	...
;;	32	...
;;	...

;; Do 20 four_complex_unfft operations
;;	distance between fft data elements is 4
;;	do 5 macros each processing 32 data values

ypass2_80_complex_unfft:
	start_timer 11
	mov	rdi, xsincos_complex	;; Load sin/cos pointers
	bump	rdi, 4*YMM_SCD2
	yloop_init 5			;; 5 iters
	yloop_set_prefetch YSINGLE
	yr4_s4cl_four_complex_djbunfft_preload
c2b:	yr4_s4cl_four_complex_djbunfft rsi, 4*64, 64, 2*64, rdi, 0, 5
	yloop	5, c2b, -5*4*64		;; Loop 5 times, restore source pointer
	end_timer 11

;; Do inverse FFT levels 1,2
;; On input the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	16	+1	+1	+1
;;	4	...
;;	...
;;	12	...
;;	32	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	80	+1	+1	+1
;;	4	...
;;	...
;;	76

;; Do 16 five_complex_unfft operations
;;	distance between fft data elements is 16
;;	do 4 macros each processing 40 data values

	start_timer 14
	bump	rdi, -4*YMM_SCD2	;; Load sin/cos ptr
;;	bump	rcx, (20+0-3*5-4)*64	;; Move prefetch pointer along
	yloop_init 4			;; 4 iters
	yloop_set_prefetch YSINGLE
	yr5_5cl_five_complex_djbunfft_preload
c1b:	yr5_5cl_five_complex_djbunfft rsi, 64, 4*64, rdi, YMM_SCD2, 4
	yloop	4, c1b			;; Test loop counter
	end_timer 14

ypass2_80_complex_done:
	ENDM


;*********************************************************************************************
;			Macros derived from pass2-8-levels
;*********************************************************************************************


; *************** radix4-pass2-192-real macro ******************
; This macro takes 384 real values and performs the final 8.585 levels of the FFT.
; The results are squared, and the first 8.585 levels of the inverse FFT are done.

yr4dwpn_pass2_192_real MACRO
	LOCAL	b1b, b2a, b2b, b3a, b3b, b4a, b4b, b4c, b4d, c1b, c2a, c2b, c3a, c3b
	LOCAL	ypass2_192_real_1, ypass2_192_real_2
	LOCAL	ypass2_192_real_3, ypass2_192_real_4
	LOCAL	ypass2_192_real_unfft, ypass2_192_real_done

;; Type 4 FFTs skip the forward FFT process

	start_timer 2
	mov	rsi, DATA_ADDR		;; Load source address
	mov	rbx, DIST_TO_FFTSRCARG
	mov	rcx, DATA_PREFETCH	;; Prefetch pointer (we need to prefetch
					;; 48 cache lines + 0 pad cache lines)
	cmp	ffttype, 4
	je	ypass2_192_real_4

;; Do FFT levels 1,2,3 which correspond to complex levels 1,2
;; On input the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	192	+1	+1	+1
;;	4	...
;;	...
;;	188
;; On output the 64-byte cache lines are swizzled to hold these data values:
;;	0	+1	+1	+1	64	+1	+1	+1
;;	4	...
;;	...
;;	60	...
;;	128	...
;;	...

;; Do 64 six_reals_fft operations
;;	distance between fft data elements is 64
;;	do 16 macros each processing 24 data values

	mov	rdi, sincos3		;; Real sin/cos data
	mov	rbp, xsincos_complex	;; Complex sin/cos data
	yloop_init 16
	yr3_f3cl_2sc_six_reals_fft_preload
b1b:	yr3_f3cl_2sc_six_reals_fft rsi, 64, 16*64, rbp, YMM_SCD1, rdi, YMM_SCD1, 16
	yloop	16, b1b, -16*64		;; Test loop counter, restore source pointer

;; Do FFT levels 3,4
;; On input the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	64	+1	+1	+1
;;	4	...
;;	...
;;	60	...
;;	128	...
;;	...
;; On output the 64-byte cache lines are swizzled to hold these data values:
;;	0	+32	+32	+32	16	+32	+32	+32
;;	4	...
;;	...
;;	12	...
;;	1	...
;;	...
;;	128	...
;;	...

;; Do 16 eight_reals_fft operations
;; Do 32 four_complex_djbfft operations
;;	distance between fft data elements is 16
;;	do 12 macros each processing 32 data values

	yloop_init 12			;; 3 iterations of 4
	yloop_set_prefetch YSINGLE
	yr4_s4cl_2sc_eight_reals_fft_preload
b2a:	yr4_s4cl_2sc_eight_reals_fft rsi, 64, 4*64, 8*64, rbp, YMM_SCD2, rdi, YMM_SCD2, 4
	yloop	4, b2a, -4*64+16*64, rbp, -4*YMM_SCD2 ;; Test loop counter, next source pointer, restore sine/cosine pointer

	loops_reset			;; 2 more iters of 4
	yloop_set_prefetch YSINGLE
	yr4_s4cl_four_complex_djbfft_preload
b2b:	yr4_s4cl_four_complex_djbfft rsi, 64, 4*64, 8*64, rbp, YMM_SCD2, 4
	yloop	4, b2b, -4*64+16*64, rbp, -4*YMM_SCD2 ;; Test loop counter, next source pointer, restore sin/cos ptr
	yloop	3, b2b, -3*16*64	;; Loop 3 times, restore source pointer

;; Do FFT levels 5,6
;; On input the 64-byte cache lines hold these data values:
;;	0	+32	+32	+32	16	+32	+32	+32
;;	4	...
;;	...
;;	12	...
;;	1	...
;;	...
;;	128	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+32	+32	+32	4	+32	+32	+32
;;	8	...
;;	...
;;	1	...
;;	...
;;	128	...
;;	...

;; Do 4 eight_reals_fft operations
;; Do 44 four_complex_djbfft operations
;;	distance between fft data elements is 4
;;	do 12 macros each processing 32 data values

	yloop_init 12			;; 3 iterations of 4
	yloop_set_prefetch YSINGLE
	yr4_4cl_eight_reals_four_complex_djbfft_preload
b3a:	yr4_4cl_eight_reals_four_complex_djbfft rsi, 4*64, 64, 2*64, rdi, YMM_SCD3, 4
	yloop	4, b3a, 0, rdi, -4*YMM_SCD3 ;; Test loop counter, restore sin/cos pointer

	loops_reset			;; 2 more iters of 4
	yloop_set_prefetch YSINGLE
	yr4_rb4cl_four_complex_djbfft_preload
b3b:	yr4_rb4cl_four_complex_djbfft rsi, 4*64, 64, 2*64, rdi, YMM_SCD3, 4
	yloop	4, b3b, 0, rdi, -4*YMM_SCD3 ;; Test loop counter, restore sin/cos pointer
	yloop	3, b3b, -12*4*64	;; Test loop counter, restore source pointer

;; Do FFT levels 7,8
;; On input and output the 64-byte cache lines hold these data values:
;;	0	+32	+32	+32	4	+32	+32	+32
;;	8	...
;;	...
;;	1	...
;;	...
;;	128	...
;;	...

;; Do 1 eight_reals_fft operations
;; Do 47 four_complex_fft operations
;;	distance between fft data elements is 1
;;	do 12 macros each processing 32 data values

;; Execute the proper middle step

	cmp	ffttype, 2
	jl	ypass2_192_real_1
	jg	ypass2_192_real_3

ypass2_192_real_2:
;;	yr4_4cl_eight_reals_four_complex_with_square rsi, 64, 4*64, 8*64
	yr4_dispatch call, yreal_square, 4*64, 8*64
	yloop_init 12			;; 11 more iterations
	yloop_adjust 1
	yloop_set_prefetch YSINGLE
	yr4_4cl_four_complex_with_square_preload
b4b:	yr4_4cl_four_complex_with_square rsi, 64, 4*64, 8*64, 1
	yloop	4, b4b, -4*64+16*64		;; Test loop counter, next source pointer
	yloop	3, b4b, -3*16*64		;; Test loop counter, restore source pointer
	jmp	ypass2_192_real_unfft

ypass2_192_real_1:
;;	yr4_4cl_eight_reals_four_complex_fft_final rsi, 64, 4*64, 8*64
	yr4_dispatch call, yreal_fft_final, 4*64, 8*64
	mov	rcx, xsincos_complex	;; Prefetch complex sin/cos data, 26 cache lines
	yloop_init 12			;; 11 more iterations
	yloop_adjust 1
	yloop_set_prefetch YDOUBLE
b4a:	yr4_4cl_four_complex_fft_final rsi, 64, 4*64, 8*64
	yloop	4, b4a, -4*64+16*64		;; Test loop counter, next source pointer
	yloop	3, b4a			;; Test loop counter
	jmp	ypass2_192_real_done

ypass2_192_real_3:
x64	mov	r8, DEST2ARG
	mov	rbp, DIST_TO_MULSRCARG
;;	yr4_4cl_eight_reals_four_complex_with_mult rsi, 64, 4*64, 8*64
	yr4_dispatch call, yreal_mult, 4*64, 8*64
	yloop_init 12			;; 11 more iterations
	yloop_adjust 1
	yloop_set_prefetch YSINGLE
	yr4_4cl_four_complex_with_mult_preload
b4c:	yr4_4cl_four_complex_with_mult rsi, 64, 4*64, 8*64
	yloop	4, b4c, -4*64+16*64		;; Test loop counter, next source pointer
	yloop	3, b4c, -3*16*64		;; Test loop counter, restore source pointer
x64	add	rsi, r8
	jmp	ypass2_192_real_unfft

ypass2_192_real_4:
	mov	rbp, DIST_TO_MULSRCARG
;;	yr4_4cl_eight_reals_four_complex_with_mulf rsi, 64, 4*64, 8*64
	yr4_dispatch call, yreal_mulf, 4*64, 8*64
	yloop_init 12			;; 11 more iterations
	yloop_adjust 1
	yloop_set_prefetch YDOUBLE
	yr4_4cl_four_complex_with_mulf_preload
b4d:	yr4_4cl_four_complex_with_mulf rsi, 64, 4*64, 8*64
	yloop	4, b4d, -4*64+16*64		;; Test loop counter, next source pointer
	yloop	3, b4d, -3*16*64		;; Test loop counter, restore source pointer
;;	bump	rcx, (-22+3*12)*64	;; Move prefetch pointer back to where squaring would be

;; Do inverse FFT levels 5,6
;; On input the 64-byte cache lines hold these data values:
;;	0	+32	+32	+32	4	+32	+32	+32
;;	8	...
;;	...
;;	1	...
;;	...
;;	128	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+32	+32	+32	16	+32	+32	+32
;;	4	...
;;	...
;;	12	...
;;	1	...
;;	...
;;	128	...
;;	...

;; Do 4 eight_reals_unfft operations
;; Do 44 four_complex_djbunfft operations
;;	distance between fft data elements is 4
;;	do 12 macros processing 32 data values

ypass2_192_real_unfft:
	mov	rdi, sincos3		;; Load sin/cos pointer
	bump	rdi, 4*YMM_SCD2+16*YMM_SCD1
;;	bump	rcx, (48+0-3*12-12)*64	;; Move prefetch pointer along
	yloop_init 12			;; 3 iterations of 4
	yloop_set_prefetch YSINGLE
	yr4_4cl_eight_reals_four_complex_djbunfft_preload
c3a:	yr4_4cl_eight_reals_four_complex_djbunfft rsi, 4*64, 64, 2*64, rdi, YMM_SCD3, 4
	yloop	4, c3a, 0, rdi, -4*YMM_SCD3 ;; Test loop counter, restore sine/cosine pointer

	loops_reset			;; 2 more iters of 4
	yloop_set_prefetch YSINGLE
	yr4_rb4cl_four_complex_djbunfft_preload
c3b:	yr4_rb4cl_four_complex_djbunfft rsi, 4*64, 64, 2*64, rdi, YMM_SCD3, 4
	yloop	4, c3b, 0, rdi, -4*YMM_SCD3 ;; Test loop counter, restore sine/cosine pointer
	yloop	3, c3b, -12*4*64		;; Test loop counter, restore source pointer

;; Do inverse FFT levels 3,4
;; On input the 64-byte cache lines hold these data values:
;;	0	+32	+32	+32	16	+32	+32	+32
;;	4	...
;;	...
;;	12	...
;;	1	...
;;	...
;;	128	...
;;	...
;; On output the 64-byte cache lines are swizzled to hold these data values:
;;	0	+1	+1	+1	64	+1	+1	+1
;;	4	...
;;	...
;;	60	...
;;	128	...
;;	...

;; Do 16 eight_reals_unfft operations
;; Do 32 four_complex_djbunfft operations
;;	distance between fft data elements is 16
;;	do 12 macros processing 32 data values

	bump	rdi, -4*YMM_SCD2	;; Real sin/cos ptrs
	mov	rbp, xsincos_complex	;; Complex sin/cos data
	bump	rbp, 16*YMM_SCD1
	mov	rcx, xsincos_complex	;; Prefetch complex sin/cos data, 26 cache lines
	yloop_init 12			;; 3 iterations of 4
	yloop_set_prefetch YSINGLE
	yr4_s4cl_2sc_eight_reals_unfft_preload
c2a:	yr4_s4cl_2sc_eight_reals_unfft rsi, 64, 4*64, 8*64, rbp, YMM_SCD2, rdi, YMM_SCD2, 4
	yloop	4, c2a, -4*64+16*64, rbp, -4*YMM_SCD2 ;; Test loop counter, next source pointer, restore sine/cosine pointer

	loops_reset			;; 2 more iters of 4
	yloop_set_prefetch YSINGLE
	yr4_s4cl_four_complex_djbunfft_preload
c2b:	yr4_s4cl_four_complex_djbunfft rsi, 64, 4*64, 8*64, rbp, YMM_SCD2, 4
	yloop	4, c2b, -4*64+16*64, rbp, -4*YMM_SCD2 ;; Test loop counter, next source pointer, restore sine/cosine pointer
	yloop	3, c2b, -3*16*64		;; Test loop counter, restore source pointer

;; Do inverse FFT levels 1,2,3 which correspond to complex levels 1,2
;; On input the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	64	+1	+1	+1
;;	4	...
;;	...
;;	60	...
;;	128	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	192	+1	+1	+1
;;	4	...
;;	...
;;	188

;; Do 64 six_reals_unfft operations
;;	distance between fft data elements is 64
;;	do 16 macros processing 24 data values

	bump	rdi, -4*YMM_SCD2-16*YMM_SCD1 ;; Real sin/cos ptr
	bump	rbp, -16*YMM_SCD1	;; Complex sin/cos data
	bump	rcx, (26-12-16)*64	;; Move prefetch pointer along
	yloop_init 16			;; 16 iterations
	yloop_set_prefetch YSINGLE
	yr3_3cl_2sc_six_reals_unfft_preload
c1b:	yr3_3cl_2sc_six_reals_unfft rsi, 64, 16*64, rbp, YMM_SCD1, rdi, YMM_SCD1, 16
	yloop	16, c1b			;; Test loop counter

ypass2_192_real_done:
	end_timer 2
	ENDM

; *************** radix4-pass2-192-complex macro ******************
; This macro takes 192 complex values and performs the final 7.585 levels of a standard
; mostly radix-4 FFT, squares the results, and does the first 7.585 levels of the inverse FFT.

yr4dwpn_pass2_192_complex MACRO
	LOCAL	b1b, b2b, b3b, b4a, b4b, b4c, b4d, c1b, c2b, c3b
	LOCAL	ypass2_192_complex_1, ypass2_192_complex_2
	LOCAL	ypass2_192_complex_3, ypass2_192_complex_4
	LOCAL	ypass2_192_complex_unfft, ypass2_192_complex_done

;; Type 4 FFTs skip the forward FFT process

	mov	rsi, DATA_ADDR		;; Load address of FFT data
	mov	rbx, DIST_TO_FFTSRCARG
	mov	rcx, DATA_PREFETCH	;; Prefetch pointer (we need to prefetch 
					;; 48 cache lines + 0 pad cache lines)
	cmp	ffttype, 4
	je	ypass2_192_complex_4

;; Do FFT levels 1,2
;; On input the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	192	+1	+1	+1
;;	4	...
;;	...
;;	188
;; On output the 64-byte cache lines are swizzled to hold these data values:
;;	0	+1	+1	+1	64	+1	+1	+1
;;	4	...
;;	...
;;	60	...
;;	128	...
;;	...

;; Do 64 three_complex_fft operations
;;	distance between fft data elements is 64
;;	do 16 macros each processing 24 data values

	start_timer 4
	mov	rdi, xsincos_complex	;; The sin/cos data
	yloop_init 16			;; 16 iterations
	yr3_f3cl_three_complex_djbfft_preload
b1b:	yr3_f3cl_three_complex_djbfft rsi, 64, 16*64, rdi, YMM_SCD1, 16
	yloop	16, b1b, -16*64		;; Test inner loop counter, restore source pointer
	end_timer 4

;; Do FFT levels 3,4
;; On input the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	64	+1	+1	+1
;;	4	...
;;	...
;;	60	...
;;	128	...
;;	...
;; On output the 64-byte cache lines are swizzled to hold these data values:
;;	0	+32	+32	+32	16	+32	+32	+32
;;	4	...
;;	...
;;	12	...
;;	1	...
;;	...
;;	128	...
;;	...

;; Do 48 four_complex_fft operations
;;	distance between fft data elements is 16
;;	do 12 macros each processing 32 data values

	start_timer 6
	yloop_init 12			;; 3 iters of 4
	yloop_set_prefetch YSINGLE
	yr4_s4cl_four_complex_djbfft_preload
b2b:	yr4_s4cl_four_complex_djbfft rsi, 64, 4*64, 8*64, rdi, YMM_SCD2, 4
	yloop	4, b2b, -4*64+16*64, rdi, -4*YMM_SCD2 ;; Loop 4 times, next source pointer, restore sine/cosine pointer
	yloop	3, b2b, -3*16*64, rdi, 4*YMM_SCD2 ;; Loop 3 times, restore source pointer, next sine/cosine pointer
	end_timer 6

;; Do FFT levels 5,6
;; On input the 64-byte cache lines hold these data values:
;;	0	+32	+32	+32	16	+32	+32	+32
;;	4	...
;;	...
;;	12	...
;;	1	...
;;	...
;;	128	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+32	+32	+32	4	+32	+32	+32
;;	8	...
;;	...
;;	1	...
;;	...
;;	128	...
;;	...

;; Do 48 four_complex_fft operations
;;	distance between fft data elements is 4
;;	do 12 macros each processing 32 data values

	start_timer 7
	yloop_init 12			;; 3 iters of 4
	yloop_set_prefetch YHALF
	yr4_b4cl_four_complex_djbfft_preload
b3b:	yr4_b4cl_four_complex_djbfft rsi, 4*64, 64, 2*64, rdi, YMM_SCD2/4, 4
	yloop	4, b3b, 0, rdi, -4*YMM_SCD2/4 ;; Loop 4 times, restore sine/cosine pointer
	yloop	3, b3b, -12*4*64		;; Loop 3 times, restore source pointer
	end_timer 7

;; Do FFT levels 7,8 as well as inverse FFT levels 7,8
;; On input and output the 64-byte cache lines hold these data values:
;;	0	+32	+32	+32	4	+32	+32	+32
;;	8	...
;;	...
;;	1	...
;;	...
;;	128	...
;;	...

;; Do 48 four_complex_fft operations
;;	distance between fft data elements is 1
;;	do 12 macros each processing 32 data values

;; Execute the right middle step

	cmp	ffttype, 2
	jl	ypass2_192_complex_1
	jg	ypass2_192_complex_3

ypass2_192_complex_2:
	start_timer 9
	yloop_init 12
	yloop_set_prefetch YHALF
	yr4_4cl_four_complex_with_square_preload
b4b:	yr4_4cl_four_complex_with_square rsi, 64, 4*64, 8*64, 4
	yloop	4, b4b, -4*64+16*64		;; Loop 4 times, next source pointer
	yloop	3, b4b, -3*16*64		;; Loop 3 times, restore source pointer
	end_timer 9
	jmp	ypass2_192_complex_unfft

ypass2_192_complex_1:
	start_timer 9
;;	bump	rcx, (48+0-12-6-24)*64	;; Move prefetch pointer along
	yloop_init 12
	yloop_set_prefetch YDOUBLE
b4a:	yr4_4cl_four_complex_fft_final rsi, 64, 4*64, 8*64
	yloop	4, b4a, -4*64+16*64		;; Loop 4 times, next source pointer
	yloop	3, b4a			;; Loop 3 times
	end_timer 9
	jmp	ypass2_192_complex_done

ypass2_192_complex_3:
	start_timer 9
x64	mov	r8, DEST2ARG
	mov	rbp, DIST_TO_MULSRCARG
	yloop_init 12
	yloop_set_prefetch YHALF
	yr4_4cl_four_complex_with_mult_preload
b4c:	yr4_4cl_four_complex_with_mult rsi, 64, 4*64, 8*64
	yloop	4, b4c, -4*64+16*64		;; Loop 4 times, next source pointer
	yloop	3, b4c, -3*16*64		;; Loop 3 times, restore source pointer
x64	add	rsi, r8
	end_timer 9
	jmp	ypass2_192_complex_unfft

ypass2_192_complex_4:
	start_timer 9
	mov	rbp, DIST_TO_MULSRCARG
	yloop_init 12			
	yloop_set_prefetch YDOUBLE
	yr4_4cl_four_complex_with_mulf_preload
b4d:	yr4_4cl_four_complex_with_mulf rsi, 64, 4*64, 8*64
	yloop	4, b4d, -4*64+16*64		;; Loop 4 times, next source pointer
	yloop	3, b4d, -3*16*64		;; Loop 3 times, restore source pointer
;;	bump	rcx, (-24+12+2*6)*64	;; Move prefetch pointer to where squaring would be
	end_timer 9

;; Do inverse FFT levels 5,6
;; On input the 64-byte cache lines hold these data values:
;;	0	+32	+32	+32	4	+32	+32	+32
;;	8	...
;;	...
;;	1	...
;;	...
;;	128	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+32	+32	+32	16	+32	+32	+32
;;	4	...
;;	...
;;	12	...
;;	1	...
;;	...
;;	128	...
;;	...

;; Do 48 four_complex_unfft operations
;;	distance between fft data elements is 4
;;	do 12 macros each processing 32 data values

ypass2_192_complex_unfft:
	start_timer 10
	mov	rdi, xsincos_complex	;; Load sin/cos pointers
	bump	rdi, 16*YMM_SCD1+4*YMM_SCD2
	yloop_init 12			;; 3 iters of 4
	yloop_set_prefetch YHALF
	yr4_b4cl_four_complex_djbunfft_preload
c3b:	yr4_b4cl_four_complex_djbunfft rsi, 4*64, 64, 2*64, rdi, YMM_SCD2/4, 4
	yloop	4, c3b, 0, rdi, -4*YMM_SCD2/4 ;; Loop 4 times, restore sine/cosine pointer
	yloop	3, c3b, -12*4*64		;; Loop 3 times, restore source pointer
	end_timer 10

;; Do inverse FFT levels 3,4
;; On input the 64-byte cache lines hold these data values:
;;	0	+32	+32	+32	16	+32	+32	+32
;;	4	...
;;	...
;;	12	...
;;	1	...
;;	...
;;	128	...
;;	...
;; On output the 64-byte cache lines are swizzled to hold these data values:
;;	0	+1	+1	+1	64	+1	+1	+1
;;	4	...
;;	...
;;	60	...
;;	128	...
;;	...

;; Do 48 four_complex_unfft operations
;;	distance between fft data elements is 16
;;	do 12 macros each processing 32 data values

	start_timer 11
	bump	rdi, -4*YMM_SCD2	;; Load sin/cos ptr
	yloop_init 12			;; 3 iters of 4
	yloop_set_prefetch YSINGLE
	yr4_s4cl_four_complex_djbunfft_preload
c2b:	yr4_s4cl_four_complex_djbunfft rsi, 64, 4*64, 8*64, rdi, YMM_SCD2, 4
	yloop	4, c2b, -4*64+16*64, rdi, -4*YMM_SCD2 ;; Loop 4 times, next source pointer, restore sine/cosine pointer
	yloop	3, c2b, -3*16*64		;; Loop 3 times, restore source pointer
	end_timer 11

;; Do inverse FFT levels 1,2
;; On input the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	64	+1	+1	+1
;;	4	...
;;	...
;;	60	...
;;	128	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	192	+1	+1	+1
;;	4	...
;;	...
;;	188

;; Do 64 three_complex_unfft operations
;;	distance between fft data elements is 64
;;	do 16 macros each processing 24 data values

	start_timer 14
	bump	rdi, -16*YMM_SCD1	;; Load sin/cos ptr
	bump	rcx, (48+0-12-3*6-12-8)*64 ;; Move prefetch pointer along
	yloop_init 16			;; 16 iters
	yloop_set_prefetch YHALF
	yr3_3cl_three_complex_djbunfft_preload
c1b:	yr3_3cl_three_complex_djbunfft rsi, 64, 16*64, rdi, YMM_SCD1, 16
	yloop	16, c1b			;; Test loop counter
	end_timer 14

ypass2_192_complex_done:
	ENDM


; *************** radix4-pass2-8-levels-real macro ******************
; This macro takes 512 real values and performs the final 9 levels of the FFT.
; The results are squared, and the first 9 levels of the inverse FFT are done.

yr4dwpn_pass2_8_levels_real MACRO
	LOCAL	b1b, b2b, b3a, b3b, b4a, b4b, b4c, b4d, c1b, c2b, c3a, c3b
	LOCAL	ypass2_8_levels_real_1, ypass2_8_levels_real_2
	LOCAL	ypass2_8_levels_real_3, ypass2_8_levels_real_4
	LOCAL	ypass2_8_levels_real_unfft, ypass2_8_levels_real_done

;; Type 4 FFTs skip the forward FFT process

	start_timer 2
	mov	rsi, DATA_ADDR		;; Load source address
	mov	rbx, DIST_TO_FFTSRCARG
	mov	rcx, DATA_PREFETCH	;; Prefetch pointer (we need to prefetch 
					;; 64 cache lines + 0 pad cache lines)
	cmp	ffttype, 4
	je	ypass2_8_levels_real_4

;; Do FFT levels 1,2,3 which correspond to complex levels 1,2
;; On input the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	256	+1	+1	+1
;;	4	...
;;	...
;;	252
;; On output the 64-byte cache lines are swizzled to hold these data values:
;;	0	+128	+128	+128	64	+128	+128	+128
;;	4	...
;;	...
;;	60	...
;;	1	...
;;	...

;; Do 64 eight_reals_fft operations
;;	distance between fft data elements is 64
;;	do 16 macros each processing 32 data values

	mov	rdi, sincos3		;; Real sin/cos data
	mov	rbp, xsincos_complex	;; Complex sin/cos data
	yloop_init 16
	yr4_fs4cl_2sc_eight_reals_fft_preload
b1b:	yr4_fs4cl_2sc_eight_reals_fft rsi, 64, 16*64, 32*64, rbp, YMM_SCD2, rdi, YMM_SCD2, 16
	yloop	16, b1b, -16*64		;; Test loop counter, restore source pointer

;; Do FFT levels 3,4
;; On input the 64-byte cache lines hold these data values:
;;	0	+128	+128	+128	64	+128	+128	+128
;;	4	...
;;	...
;;	60	...
;;	1	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+128	+128	+128	16	+128	+128	+128
;;	4	...
;;	...
;;	12	...
;;	32	...
;;	...
;;	1	...
;;	...

;; Do 16 eight_reals_fft operations
;; Do 48 four_complex_djbfft operations
;;	distance between fft data elements is 16
;;	do 16 macros each processing 32 data values

	yloop_init 16
	yloop_set_prefetch YSINGLE
	yr4_4cl_eight_reals_four_complex_djbfft_preload
b2b:	yr4_4cl_eight_reals_four_complex_djbfft rsi, 16*64, 4*64, 8*64, rdi, YMM_SCD3, 4
	yloop	4, b2b, -4*16*64+64		;; Test loop counter, next source pointer
	yloop	4, b2b, -4*64, rdi, -16*YMM_SCD3 ;; Test loop counter, restore source pointer, restore sin/cos ptr

;; Do FFT levels 5,6
;; On input the 64-byte cache lines hold these data values:
;;	0	+128	+128	+128	16	+128	+128	+128
;;	4	...
;;	...
;;	12	...
;;	32	...
;;	...
;;	1	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+128	+128	+128	4	+128	+128	+128
;;	8	...
;;	...
;;	1	...
;;	...

;; Do 4 eight_reals_fft operations
;; Do 60 four_complex_djbfft operations
;;	distance between fft data elements is 4
;;	do 16 macros each processing 32 data values

	yloop_init 16			;; 4 iterations of 4
	yloop_set_prefetch YSINGLE
	yr4_4cl_eight_reals_four_complex_djbfft_preload
b3a:	yr4_4cl_eight_reals_four_complex_djbfft rsi, 16*64, 64, 2*64, rdi, 4*YMM_SCD3, 4
	yloop	4, b3a, -4*16*64+4*64, rdi, -4*4*YMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer

	loops_reset			;; 3 more iters of 4
	yr4_rb4cl_four_complex_djbfft_preload
b3b:	yr4_rb4cl_four_complex_djbfft rsi, 16*64, 64, 2*64, rdi, 4*YMM_SCD3, 4
	yloop	4, b3b, -4*16*64+4*64, rdi, -4*4*YMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer
	yloop	4, b3b, -4*4*64		;; Test loop counter, restore source pointer

;; Do FFT levels 7,8
;; On input and output the 64-byte cache lines hold these data values:
;;	0	+128	+128	+128	4	+128	+128	+128
;;	8	...
;;	...
;;	1	...
;;	...

;; Do 1 eight_reals_fft operations
;; Do 63 four_complex_fft operations
;;	distance between fft data elements is 1
;;	do 16 macros each processing 32 data values

;; Execute the proper middle step

	cmp	ffttype, 2
	jl	ypass2_8_levels_real_1
	jg	ypass2_8_levels_real_3

ypass2_8_levels_real_2:
;;	yr4_4cl_eight_reals_four_complex_with_square rsi, 64, 16*64, 32*64
	yr4_dispatch call, yreal_square, 16*64, 32*64
	yloop_init 16			;; 15 more iterations
	yloop_adjust 1
	yloop_set_prefetch YSINGLE
	yr4_4cl_four_complex_with_square_preload
b4b:	yr4_4cl_four_complex_with_square rsi, 64, 16*64, 32*64, 1
	yloop	16, b4b, -16*64		;; Test loop counter, restore source pointer
	jmp	ypass2_8_levels_real_unfft

ypass2_8_levels_real_1:
;;	yr4_4cl_eight_reals_four_complex_fft_final rsi, 64, 16*64, 32*64
	yr4_dispatch call, yreal_fft_final, 16*64, 32*64
	mov	rcx, xsincos_complex	;; Prefetch complex sin/cos data, 34 cache lines
	yloop_init 16			;; 15 more iterations
	yloop_adjust 1
	yloop_set_prefetch YDOUBLE
b4a:	yr4_4cl_four_complex_fft_final rsi, 64, 16*64, 32*64
	yloop	16, b4a			;; Test loop counter
	jmp	ypass2_8_levels_real_done

ypass2_8_levels_real_3:
x64	mov	r8, DEST2ARG
	mov	rbp, DIST_TO_MULSRCARG
;;	yr4_4cl_eight_reals_four_complex_with_mult rsi, 64, 16*64, 32*64
	yr4_dispatch call, yreal_mult, 16*64, 32*64
	yloop_init 16			;; 15 more iterations
	yloop_adjust 1
	yloop_set_prefetch YSINGLE
	yr4_4cl_four_complex_with_mult_preload
b4c:	yr4_4cl_four_complex_with_mult rsi, 64, 16*64, 32*64
	yloop	16, b4c, -16*64		;; Test loop counter, restore source pointer
x64	add	rsi, r8
	jmp	ypass2_8_levels_real_unfft

ypass2_8_levels_real_4:
	mov	rbp, DIST_TO_MULSRCARG
;;	yr4_4cl_eight_reals_four_complex_with_mulf rsi, 64, 16*64, 32*64
	yr4_dispatch call, yreal_mulf, 16*64, 32*64
	yloop_init 16			;; 15 more iterations
	yloop_adjust 1
	yloop_set_prefetch YDOUBLE
	yr4_4cl_four_complex_with_mulf_preload
b4d:	yr4_4cl_four_complex_with_mulf rsi, 64, 16*64, 32*64
	yloop	16, b4d, -16*64		;; Test loop counter, restore source pointer
;;	bump	rcx, (-32+3*16)*64	;; Move prefetch pointer back to where squaring would be

;; Do inverse FFT levels 5,6
;; On input the 64-byte cache lines hold these data values:
;;	0	+128	+128	+128	4	+128	+128	+128
;;	8	...
;;	...
;;	1	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+128	+128	+128	16	+128	+128	+128
;;	4	...
;;	...
;;	12	...
;;	32	...
;;	...
;;	1	...
;;	...

;; Do 4 eight_reals_unfft operations
;; Do 60 four_complex_djbunfft operations
;;	distance between fft data elements is 4
;;	do 16 macros processing 32 data values

ypass2_8_levels_real_unfft:
	mov	rdi, sincos3		;; Load sin/cos pointer
	bump	rdi, 16*YMM_SCD2
	yloop_init 16			;; 4 iterations of 4
	yloop_set_prefetch YSINGLE
	yr4_4cl_eight_reals_four_complex_djbunfft_preload
c3a:	yr4_4cl_eight_reals_four_complex_djbunfft rsi, 16*64, 64, 2*64, rdi, 4*YMM_SCD3, 4
	yloop	4, c3a, -4*16*64+4*64, rdi, -4*4*YMM_SCD3 ;; Test loop counter, next source pointer, restore sine/cosine pointer

	loops_reset			;; 3 more iters of 4
	yloop_set_prefetch YSINGLE
	yr4_rb4cl_four_complex_djbunfft_preload
c3b:	yr4_rb4cl_four_complex_djbunfft rsi, 16*64, 64, 2*64, rdi, 4*YMM_SCD3, 4
	yloop	4, c3b, -4*16*64+4*64, rdi, -4*4*YMM_SCD3 ;; Test loop counter, next source pointer, restore sine/cosine pointer
	yloop	4, c3b, -4*4*64		;; Test loop counter, restore source pointer

;; Do inverse FFT levels 3,4
;; On input the 64-byte cache lines hold these data values:
;;	0	+128	+128	+128	16	+128	+128	+128
;;	4	...
;;	...
;;	12	...
;;	32	...
;;	...
;;	1	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+128	+128	+128	64	+128	+128	+128
;;	4	...
;;	...
;;	60	...
;;	1	...
;;	...

;; Do 16 eight_reals_unfft operations
;; Do 48 four_complex_djbunfft operations
;;	distance between fft data elements is 16
;;	do 16 macros processing 32 data values

	mov	rcx, xsincos_complex	;; Prefetch complex sin/cos data, 34 cache lines
	yloop_init 16			;; 16 iterations
	yloop_set_prefetch YSINGLE
	yr4_4cl_eight_reals_four_complex_djbunfft_preload
c2b:	yr4_4cl_eight_reals_four_complex_djbunfft rsi, 16*64, 4*64, 8*64, rdi, YMM_SCD3, 4
	yloop	4, c2b, -4*16*64+64		;; Test loop counter, next source pointer
	yloop	4, c2b, -4*64		;; Test loop counter, restore source pointer

;; Do inverse FFT levels 1,2,3 which correspond to complex levels 1,2
;; On input the 64-byte cache lines hold these data values:
;;	0	+128	+128	+128	64	+128	+128	+128
;;	4	...
;;	...
;;	60	...
;;	1	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	256	+1	+1	+1
;;	4	...
;;	...
;;	252

;; Do 64 eight_reals_unfft operations
;;	distance between fft data elements is 64
;;	do 16 macros processing 32 data values

	bump	rdi, -16*YMM_SCD3-16*YMM_SCD2 ;; Real sin/cos pointer
	mov	rbp, xsincos_complex	;; Complex sin/cos data
	yloop_init 16			;; 16 iterations
	yloop_set_prefetch YSINGLE
	yr4_s4cl_2sc_eight_reals_unfft_preload
c1b:	yr4_s4cl_2sc_eight_reals_unfft rsi, 64, 16*64, 32*64, rbp, YMM_SCD2, rdi, YMM_SCD2, 16
	yloop	16, c1b			;; Test loop counter

ypass2_8_levels_real_done:
	end_timer 2
	ENDM

; *************** radix4-pass2-8-levels-complex macro ******************
; This macro takes 256 complex values and performs the final 8 levels of a standard
; mostly radix-4 FFT, squares the results, and does the first 8 levels of the inverse FFT.

yr4dwpn_pass2_8_levels_complex MACRO
	LOCAL	b1b, b2b, b3b, b4a, b4b, b4c, b4d, c1b, c2b, c3b
	LOCAL	ypass2_8_levels_complex_1, ypass2_8_levels_complex_2
	LOCAL	ypass2_8_levels_complex_3, ypass2_8_levels_complex_4
	LOCAL	ypass2_8_levels_complex_unfft, ypass2_8_levels_complex_done

;; Type 4 FFTs skip the forward FFT process

	mov	rsi, DATA_ADDR		;; Load address of FFT data
	mov	rbx, DIST_TO_FFTSRCARG
	mov	rcx, DATA_PREFETCH	;; Prefetch pointer (we need to prefetch 
					;; 64 cache lines + 0 pad cache lines)
	cmp	ffttype, 4
	je	ypass2_8_levels_complex_4

;; Do FFT levels 1,2
;; On input the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	256	+1	+1	+1
;;	4	...
;;	...
;;	252
;; On output the 64-byte cache lines are swizzled to hold these data values:
;;	0	+128	+128	+128	64	+128	+128	+128
;;	4	...
;;	...
;;	60	...
;;	1	...
;;	...

;; Do 64 four_complex_fft operations
;;	distance between fft data elements is 64
;;	do 16 macros each processing 32 data values

	start_timer 4
	mov	rdi, xsincos_complex	;; The sin/cos data
	yloop_init 16			;; 16 iterations
	yr4_fs4cl_four_complex_djbfft_preload
b1b:	yr4_fs4cl_four_complex_djbfft rsi, 64, 16*64, 32*64, rdi, YMM_SCD2, 16
	yloop	16, b1b, -16*64		;; Test inner loop counter, restore source pointer
	end_timer 4

;; Do FFT levels 3,4
;; On input the 64-byte cache lines hold these data values:
;;	0	+128	+128	+128	64	+128	+128	+128
;;	4	...
;;	...
;;	60	...
;;	1	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+128	+128	+128	16	+128	+128	+128
;;	4	...
;;	...
;;	12	...
;;	32	...
;;	...
;;	1	...
;;	...

;; Do 64 four_complex_fft operations
;;	distance between fft data elements is 16
;;	do 16 macros each processing 32 data values

	start_timer 6
	yloop_init 16			;; 16 iters
	yloop_set_prefetch YSINGLE
	yr4_b4cl_four_complex_djbfft_preload
b2b:	yr4_b4cl_four_complex_djbfft rsi, 64, 4*64, 8*64, rdi, YMM_SCD2/4, 4
	yloop	4, b2b, -4*64+16*64		;; Loop 4 times, next source pointer
	yloop	4, b2b, -4*16*64, rdi, -16*YMM_SCD2/4 ;; Loop 4 times, restore source pointer, restore sine/cosine pointer
	end_timer 6

;; Do FFT levels 5,6
;; On input the 64-byte cache lines hold these data values:
;;	0	+128	+128	+128	16	+128	+128	+128
;;	4	...
;;	...
;;	12	...
;;	32	...
;;	...
;;	1	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+128	+128	+128	4	+128	+128	+128
;;	8	...
;;	...
;;	1	...
;;	...

;; Do 64 four_complex_fft operations
;;	distance between fft data elements is 4
;;	do 16 macros each processing 32 data values

	start_timer 7
	yloop_init 16			;; 16 iters
	yloop_set_prefetch YSINGLE
	yr4_b4cl_four_complex_djbfft_preload
b3b:	yr4_b4cl_four_complex_djbfft rsi, 4*64, 64, 2*64, rdi, 0, 4
	yloop	4, b3b, 0, rdi, YMM_SCD2/4 ;; Loop 4 times, next sine/cosine pointer
	yloop	4, b3b, -16*4*64		;; Loop 4 times, restore source pointer
	end_timer 7

;; Do FFT levels 7,8 as well as inverse FFT levels 7,8
;; On input and output the 64-byte cache lines hold these data values:
;;	0	+128	+128	+128	4	+128	+128	+128
;;	8	...
;;	...
;;	1	...
;;	...

;; Do 64 four_complex_fft operations
;;	distance between fft data elements is 1
;;	do 16 macros each processing 32 data values

;; Execute the right middle step

	cmp	ffttype, 2
	jl	ypass2_8_levels_complex_1
	jg	ypass2_8_levels_complex_3

ypass2_8_levels_complex_2:
	start_timer 9
	yloop_init 16
	yloop_set_prefetch YHALF
	yr4_4cl_four_complex_with_square_preload
b4b:	yr4_4cl_four_complex_with_square rsi, 64, 16*64, 32*64, 16
	yloop	16, b4b, -16*64		;; Loop 16 times, restore source pointer
	end_timer 9
	jmp	ypass2_8_levels_complex_unfft

ypass2_8_levels_complex_1:
	start_timer 9
	yloop_init 16
	yloop_set_prefetch YDOUBLE
b4a:	yr4_4cl_four_complex_fft_final rsi, 64, 16*64, 32*64
	yloop	16, b4a			;; Loop 16 times
	end_timer 9
	jmp	ypass2_8_levels_complex_done

ypass2_8_levels_complex_3:
	start_timer 9
x64	mov	r8, DEST2ARG
	mov	rbp, DIST_TO_MULSRCARG
	yloop_init 16
	yloop_set_prefetch YHALF
	yr4_4cl_four_complex_with_mult_preload
b4c:	yr4_4cl_four_complex_with_mult rsi, 64, 16*64, 32*64
	yloop	16, b4c, -16*64		;; Loop 16 times, restore source pointer
x64	add	rsi, r8
	end_timer 9
	jmp	ypass2_8_levels_complex_unfft

ypass2_8_levels_complex_4:
	start_timer 9
	mov	rbp, DIST_TO_MULSRCARG
	yloop_init 16
	yloop_set_prefetch YDOUBLE
	yr4_4cl_four_complex_with_mulf_preload
b4d:	yr4_4cl_four_complex_with_mulf rsi, 64, 16*64, 32*64
	yloop	16, b4d, -16*64		;; Loop 16 times, restore source pointer
	end_timer 9

;; Do inverse FFT levels 5,6
;; On input the 64-byte cache lines hold these data values:
;;	0	+128	+128	+128	4	+128	+128	+128
;;	8	...
;;	...
;;	1	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+128	+128	+128	16	+128	+128	+128
;;	4	...
;;	...
;;	12	...
;;	32	...
;;	...
;;	1	...
;;	...

;; Do 64 four_complex_unfft operations
;;	distance between fft data elements is 4
;;	do 16 macros each processing 32 data values

ypass2_8_levels_complex_unfft:
	start_timer 10
	mov	rdi, xsincos_complex	;; Load sin/cos pointers
	bump	rdi, 16*YMM_SCD2

	yloop_init 16			;; 16 iters
	yloop_set_prefetch YHALF
	yr4_b4cl_four_complex_djbunfft_preload
c3b:	yr4_b4cl_four_complex_djbunfft rsi, 4*64, 64, 2*64, rdi, 0, 4
	yloop	4, c3b, 0, rdi, YMM_SCD2/4 ;; Loop 4 times, next sine/cosine pointer
	yloop	4, c3b, -16*4*64, rdi, -4*YMM_SCD2/4 ;; Loop 4 times, restore source pointer, restore sine/cosine pointer
	end_timer 10

;; Do inverse FFT levels 3,4
;; On input the 64-byte cache lines hold these data values:
;;	0	+128	+128	+128	16	+128	+128	+128
;;	4	...
;;	...
;;	12	...
;;	32	...
;;	...
;;	1	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+128	+128	+128	64	+128	+128	+128
;;	4	...
;;	...
;;	60	...
;;	1	...
;;	...

;; Do 64 four_complex_unfft operations
;;	distance between fft data elements is 16
;;	do 16 macros each processing 32 data values

	start_timer 11
	yloop_init 16			;; 16 iters
	yloop_set_prefetch YHALF
	yr4_b4cl_four_complex_djbunfft_preload
c2b:	yr4_b4cl_four_complex_djbunfft rsi, 64, 4*64, 8*64, rdi, YMM_SCD2/4, 4
	yloop	4, c2b, -4*64+16*64		;; Loop 4 times, next source pointer
	yloop	4, c2b, -4*16*64		;; Loop 4 times, restore source pointer
	end_timer 11

;; Do inverse FFT levels 1,2
;; On input the 64-byte cache lines hold these data values:
;;	0	+128	+128	+128	64	+128	+128	+128
;;	4	...
;;	...
;;	60	...
;;	1	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	256	+1	+1	+1
;;	4	...
;;	...
;;	252

;; Do 64 four_complex_unfft operations
;;	distance between fft data elements is 64
;;	do 16 macros each processing 32 data values

	start_timer 14
	mov	rdi, xsincos_complex	;; Load sin/cos pointer
	yloop_init 16			;; 16 iters
	yloop_set_prefetch YHALF
	yr4_s4cl_four_complex_djbunfft_preload
c1b:	yr4_s4cl_four_complex_djbunfft rsi, 64, 16*64, 32*64, rdi, YMM_SCD2, 16
	yloop	16, c1b			;; Test loop counter
	end_timer 14

ypass2_8_levels_complex_done:
	ENDM


; *************** radix4-pass2-320-real macro ******************
; This macro takes 640 real values and performs the final 9.322 levels of the FFT.
; The results are squared, and the first 9.322 levels of the inverse FFT are done.

yr4dwpn_pass2_320_real MACRO
	LOCAL	b1b, b2a, b2b, b3a, b3b, b4a, b4b, b4c, b4d, c1b, c2a, c2b, c3a, c3b
	LOCAL	ypass2_320_real_1, ypass2_320_real_2
	LOCAL	ypass2_320_real_3, ypass2_320_real_4
	LOCAL	ypass2_320_real_unfft, ypass2_320_real_done

;; Type 4 FFTs skip the forward FFT process

	start_timer 2
	mov	rsi, DATA_ADDR		;; Load source address
	mov	rbx, DIST_TO_FFTSRCARG
	mov	rcx, DATA_PREFETCH	;; Prefetch pointer (we need to prefetch
					;; 80 cache lines + 0 pad cache lines)
	cmp	ffttype, 4
	je	ypass2_320_real_4

;; Do FFT levels 1,2,3 which correspond to complex levels 1,2
;; On input the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	320	+1	+1	+1
;;	4	...
;;	...
;;	316
;; On output the 64-byte cache lines are swizzled to hold these data values:
;;	0	+1	+1	+1	64	+1	+1	+1
;;	4	...
;;	...
;;	60	...
;;	128	...
;;	...

;; Do 64 ten_reals_fft operations
;;	distance between fft data elements is 64
;;	do 16 macros each processing 40 data values

	mov	rdi, sincos3		;; Real sin/cos data
	mov	rbp, xsincos_complex	;; Complex sin/cos data
	yloop_init 16
	yr5_f5cl_2sc_ten_reals_fft_preload
b1b:	yr5_f5cl_2sc_ten_reals_fft rsi, 64, 16*64, rbp, YMM_SCD2, rdi, YMM_SCD2, 16
	yloop	16, b1b, -16*64		;; Test loop counter, restore source pointer

;; Do FFT levels 3,4
;; On input the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	64	+1	+1	+1
;;	4	...
;;	...
;;	60	...
;;	128	...
;;	...
;; On output the 64-byte cache lines are swizzled to hold these data values:
;;	0	+32	+32	+32	16	+32	+32	+32
;;	4	...
;;	...
;;	12	...
;;	1	...
;;	...
;;	128	...
;;	...

;; Do 16 eight_reals_fft operations
;; Do 64 four_complex_djbfft operations
;;	distance between fft data elements is 16
;;	do 20 macros each processing 32 data values

	yloop_init 20			;; 5 iterations of 4
	yloop_set_prefetch YSINGLE
	yr4_s4cl_2sc_eight_reals_fft_preload
b2a:	yr4_s4cl_2sc_eight_reals_fft rsi, 64, 4*64, 8*64, rbp, YMM_SCD2, rdi, YMM_SCD2, 4
	yloop	4, b2a, -4*64+16*64, rbp, -4*YMM_SCD2 ;; Test loop counter, next source pointer, restore sine/cosine pointer

	loops_reset			;; 4 more iters of 4
	yloop_set_prefetch YSINGLE
	yr4_s4cl_four_complex_djbfft_preload
b2b:	yr4_s4cl_four_complex_djbfft rsi, 64, 4*64, 8*64, rbp, YMM_SCD2, 4
	yloop	4, b2b, -4*64+16*64, rbp, -4*YMM_SCD2 ;; Test loop counter, next source pointer, restore sin/cos ptr
	yloop	5, b2b, -5*16*64	;; Loop 5 times, restore source pointer

;; Do FFT levels 5,6
;; On input the 64-byte cache lines hold these data values:
;;	0	+32	+32	+32	16	+32	+32	+32
;;	4	...
;;	...
;;	12	...
;;	1	...
;;	...
;;	128	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+32	+32	+32	4	+32	+32	+32
;;	8	...
;;	...
;;	1	...
;;	...
;;	128	...
;;	...

;; Do 4 eight_reals_fft operations
;; Do 76 four_complex_djbfft operations
;;	distance between fft data elements is 4
;;	do 20 macros each processing 32 data values

	yloop_init 20			;; 5 iterations of 4
	yloop_set_prefetch YSINGLE
	yr4_4cl_eight_reals_four_complex_djbfft_preload
b3a:	yr4_4cl_eight_reals_four_complex_djbfft rsi, 4*64, 64, 2*64, rdi, YMM_SCD3, 4
	yloop	4, b3a, 0, rdi, -4*YMM_SCD3 ;; Test loop counter, restore sin/cos pointer

	loops_reset			;; 4 more iters of 4
	yloop_set_prefetch YSINGLE
	yr4_rb4cl_four_complex_djbfft_preload
b3b:	yr4_rb4cl_four_complex_djbfft rsi, 4*64, 64, 2*64, rdi, YMM_SCD3, 4
	yloop	4, b3b, 0, rdi, -4*YMM_SCD3 ;; Test loop counter, restore sin/cos pointer
	yloop	5, b3b, -20*4*64	;; Test loop counter, restore source pointer

;; Do FFT levels 7,8
;; On input and output the 64-byte cache lines hold these data values:
;;	0	+32	+32	+32	4	+32	+32	+32
;;	8	...
;;	...
;;	1	...
;;	...
;;	128	...
;;	...

;; Do 1 eight_reals_fft operations
;; Do 79 four_complex_fft operations
;;	distance between fft data elements is 1
;;	do 20 macros each processing 32 data values

;; Execute the proper middle step

	cmp	ffttype, 2
	jl	ypass2_320_real_1
	jg	ypass2_320_real_3

ypass2_320_real_2:
;;	yr4_4cl_eight_reals_four_complex_with_square rsi, 64, 4*64, 8*64
	yr4_dispatch call, yreal_square, 4*64, 8*64
	yloop_init 20			;; 19 more iterations
	yloop_adjust 1
	yloop_set_prefetch YSINGLE
	yr4_4cl_four_complex_with_square_preload
b4b:	yr4_4cl_four_complex_with_square rsi, 64, 4*64, 8*64, 1
	yloop	4, b4b, -4*64+16*64		;; Test loop counter, next source pointer
	yloop	5, b4b, -5*16*64		;; Test loop counter, restore source pointer
	jmp	ypass2_320_real_unfft

ypass2_320_real_1:
;;	yr4_4cl_eight_reals_four_complex_fft_final rsi, 64, 4*64, 8*64
	yr4_dispatch call, yreal_fft_final, 4*64, 8*64
	mov	rcx, xsincos_complex	;; Prefetch complex sin/cos data, 42 cache lines
	yloop_init 20			;; 19 more iterations
	yloop_adjust 1
	yloop_set_prefetch YDOUBLE
b4a:	yr4_4cl_four_complex_fft_final rsi, 64, 4*64, 8*64
	yloop	4, b4a, -4*64+16*64		;; Test loop counter, next source pointer
	yloop	5, b4a			;; Test loop counter
	jmp	ypass2_320_real_done

ypass2_320_real_3:
x64	mov	r8, DEST2ARG
	mov	rbp, DIST_TO_MULSRCARG
;;	yr4_4cl_eight_reals_four_complex_with_mult rsi, 64, 4*64, 8*64
	yr4_dispatch call, yreal_mult, 4*64, 8*64
	yloop_init 20			;; 19 more iterations
	yloop_adjust 1
	yloop_set_prefetch YSINGLE
	yr4_4cl_four_complex_with_mult_preload
b4c:	yr4_4cl_four_complex_with_mult rsi, 64, 4*64, 8*64
	yloop	4, b4c, -4*64+16*64		;; Test loop counter, next source pointer
	yloop	5, b4c, -5*16*64		;; Test loop counter, restore source pointer
x64	add	rsi, r8
	jmp	ypass2_320_real_unfft

ypass2_320_real_4:
	mov	rbp, DIST_TO_MULSRCARG
;;	yr4_4cl_eight_reals_four_complex_with_mulf rsi, 64, 4*64, 8*64
	yr4_dispatch call, yreal_mulf, 4*64, 8*64
	yloop_init 20			;; 19 more iterations
	yloop_adjust 1
	yloop_set_prefetch YDOUBLE
	yr4_4cl_four_complex_with_mulf_preload
b4d:	yr4_4cl_four_complex_with_mulf rsi, 64, 4*64, 8*64
	yloop	4, b4d, -4*64+16*64		;; Test loop counter, next source pointer
	yloop	5, b4d, -5*16*64		;; Test loop counter, restore source pointer
;;	bump	rcx, (-38+3*20)*64	;; Move prefetch pointer back to where squaring would be

;; Do inverse FFT levels 5,6
;; On input the 64-byte cache lines hold these data values:
;;	0	+32	+32	+32	4	+32	+32	+32
;;	8	...
;;	...
;;	1	...
;;	...
;;	128	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+32	+32	+32	16	+32	+32	+32
;;	4	...
;;	...
;;	12	...
;;	1	...
;;	...
;;	128	...
;;	...

;; Do 4 eight_reals_unfft operations
;; Do 76 four_complex_djbunfft operations
;;	distance between fft data elements is 4
;;	do 20 macros processing 32 data values

ypass2_320_real_unfft:
	mov	rdi, sincos3		;; Load sin/cos pointer
	bump	rdi, 4*YMM_SCD2+16*YMM_SCD2
;;	bump	rcx, (80+0-3*20-20)*64	;; Move prefetch pointer along
	yloop_init 20			;; 5 iterations of 4
	yloop_set_prefetch YSINGLE
	yr4_4cl_eight_reals_four_complex_djbunfft_preload
c3a:	yr4_4cl_eight_reals_four_complex_djbunfft rsi, 4*64, 64, 2*64, rdi, YMM_SCD3, 4
	yloop	4, c3a, 0, rdi, -4*YMM_SCD3 ;; Test loop counter, restore sine/cosine pointer

	loops_reset			;; 4 more iters of 4
	yloop_set_prefetch YSINGLE
	yr4_rb4cl_four_complex_djbunfft_preload
c3b:	yr4_rb4cl_four_complex_djbunfft rsi, 4*64, 64, 2*64, rdi, YMM_SCD3, 4
	yloop	4, c3b, 0, rdi, -4*YMM_SCD3 ;; Test loop counter, restore sine/cosine pointer
	yloop	5, c3b, -20*4*64		;; Test loop counter, restore source pointer

;; Do inverse FFT levels 3,4
;; On input the 64-byte cache lines hold these data values:
;;	0	+32	+32	+32	16	+32	+32	+32
;;	4	...
;;	...
;;	12	...
;;	1	...
;;	...
;;	128	...
;;	...
;; On output the 64-byte cache lines are swizzled to hold these data values:
;;	0	+1	+1	+1	64	+1	+1	+1
;;	4	...
;;	...
;;	60	...
;;	128	...
;;	...

;; Do 16 eight_reals_unfft operations
;; Do 64 four_complex_djbunfft operations
;;	distance between fft data elements is 16
;;	do 20 macros processing 32 data values

	bump	rdi, -4*YMM_SCD2	;; Real sin/cos ptr
	mov	rbp, xsincos_complex	;; Complex sin/cos data
	bump	rbp, 16*YMM_SCD2
	mov	rcx, xsincos_complex	;; Prefetch complex sin/cos data, 42 cache lines
	yloop_init 20			;; 5 iterations of 4
	yloop_set_prefetch YSINGLE
	yr4_s4cl_2sc_eight_reals_unfft_preload
c2a:	yr4_s4cl_2sc_eight_reals_unfft rsi, 64, 4*64, 8*64, rbp, YMM_SCD2, rdi, YMM_SCD2, 4
	yloop	4, c2a, -4*64+16*64, rbp, -4*YMM_SCD2 ;; Test loop counter, next source pointer, restore sine/cosine pointer

	loops_reset			;; 4 more iters of 4
	yloop_set_prefetch YSINGLE
	yr4_s4cl_four_complex_djbunfft_preload
c2b:	yr4_s4cl_four_complex_djbunfft rsi, 64, 4*64, 8*64, rbp, YMM_SCD2, 4
	yloop	4, c2b, -4*64+16*64, rbp, -4*YMM_SCD2 ;; Test loop counter, next source pointer, restore sine/cosine pointer
	yloop	5, c2b, -5*16*64	;; Test loop counter, restore source pointer

;; Do inverse FFT levels 1,2,3 which correspond to complex levels 1,2
;; On input the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	64	+1	+1	+1
;;	4	...
;;	...
;;	60	...
;;	128	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	320	+1	+1	+1
;;	4	...
;;	...
;;	316

;; Do 64 ten_reals_unfft operations
;;	distance between fft data elements is 64
;;	do 16 macros processing 40 data values

	bump	rdi, -4*YMM_SCD2-16*YMM_SCD2 ;; Real sin/cos ptr
	bump	rbp, -16*YMM_SCD2	;; Complex sin/cos data
;;	bump	rcx, (42-20-16)*64	;; Move prefetch pointer along
	yloop_init 16			;; 16 iterations
	yloop_set_prefetch YSINGLE
	yr5_5cl_2sc_ten_reals_unfft_preload
c1b:	yr5_5cl_2sc_ten_reals_unfft rsi, 64, 16*64, rbp, YMM_SCD2, rdi, YMM_SCD2, 16
	yloop	16, c1b			;; Test loop counter

ypass2_320_real_done:
	end_timer 2
	ENDM

; *************** radix4-pass2-320-complex macro ******************
; This macro takes 320 complex values and performs the final 8.322 levels of a standard
; mostly radix-4 FFT, squares the results, and does the first 8.322 levels of the inverse FFT.

yr4dwpn_pass2_320_complex MACRO
	LOCAL	b1b, b2b, b3b, b4a, b4b, b4c, b4d, c1b, c2b, c3b
	LOCAL	ypass2_320_complex_1, ypass2_320_complex_2
	LOCAL	ypass2_320_complex_3, ypass2_320_complex_4
	LOCAL	ypass2_320_complex_unfft, ypass2_320_complex_done

;; Type 4 FFTs skip the forward FFT process

	mov	rsi, DATA_ADDR		;; Load address of FFT data
	mov	rbx, DIST_TO_FFTSRCARG
	mov	rcx, DATA_PREFETCH	;; Prefetch pointer (we need to prefetch 
					;; 80 cache lines + 0 pad cache lines)
	cmp	ffttype, 4
	je	ypass2_320_complex_4

;; Do FFT levels 1,2
;; On input the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	320	+1	+1	+1
;;	4	...
;;	...
;;	316
;; On output the 64-byte cache lines are swizzled to hold these data values:
;;	0	+1	+1	+1	64	+1	+1	+1
;;	4	...
;;	...
;;	60	...
;;	128	...
;;	...

;; Do 64 five_complex_fft operations
;;	distance between fft data elements is 64
;;	do 16 macros each processing 40 data values

	start_timer 4
	mov	rdi, xsincos_complex	;; The sin/cos data
	yloop_init 16			;; 16 iterations
	yr5_f5cl_five_complex_djbfft_preload
b1b:	yr5_f5cl_five_complex_djbfft rsi, 64, 16*64, rdi, YMM_SCD2, 16
	yloop	16, b1b, -16*64		;; Test inner loop counter, restore source pointer
	end_timer 4

;; Do FFT levels 3,4
;; On input the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	64	+1	+1	+1
;;	4	...
;;	...
;;	60	...
;;	128	...
;;	...
;; On output the 64-byte cache lines are swizzled to hold these data values:
;;	0	+32	+32	+32	16	+32	+32	+32
;;	4	...
;;	...
;;	12	...
;;	1	...
;;	...
;;	128	...
;;	...

;; Do 80 four_complex_fft operations
;;	distance between fft data elements is 16
;;	do 20 macros each processing 32 data values

	start_timer 6
	yloop_init 20			;; 5 iters of 4
	yloop_set_prefetch YSINGLE
	yr4_s4cl_four_complex_djbfft_preload
b2b:	yr4_s4cl_four_complex_djbfft rsi, 64, 4*64, 8*64, rdi, YMM_SCD2, 4
	yloop	4, b2b, -4*64+16*64, rdi, -4*YMM_SCD2 ;; Loop 4 times, next source pointer, restore sine/cosine pointer
	yloop	5, b2b, -5*16*64, rdi, 4*YMM_SCD2 ;; Loop 5 times, restore source pointer, next sine/cosine pointer
	end_timer 6

;; Do FFT levels 5,6
;; On input the 64-byte cache lines hold these data values:
;;	0	+32	+32	+32	16	+32	+32	+32
;;	4	...
;;	...
;;	12	...
;;	1	...
;;	...
;;	128	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+32	+32	+32	4	+32	+32	+32
;;	8	...
;;	...
;;	1	...
;;	...
;;	128	...
;;	...

;; Do 80 four_complex_fft operations
;;	distance between fft data elements is 4
;;	do 20 macros each processing 32 data values

	start_timer 7
	yloop_init 20			;; 5 iters of 4
	yloop_set_prefetch YHALF
	yr4_b4cl_four_complex_djbfft_preload
b3b:	yr4_b4cl_four_complex_djbfft rsi, 4*64, 64, 2*64, rdi, YMM_SCD2/4, 4
	yloop	4, b3b, 0, rdi, -4*YMM_SCD2/4 ;; Loop 4 times, restore sine/cosine pointer
	yloop	5, b3b, -20*4*64		;; Loop 5 times, restore source pointer
	end_timer 7

;; Do FFT levels 7,8 as well as inverse FFT levels 7,8
;; On input and output the 64-byte cache lines hold these data values:
;;	0	+32	+32	+32	4	+32	+32	+32
;;	8	...
;;	...
;;	1	...
;;	...
;;	128	...
;;	...

;; Do 80 four_complex_fft operations
;;	distance between fft data elements is 1
;;	do 20 macros each processing 32 data values

;; Execute the right middle step

	cmp	ffttype, 2
	jl	ypass2_320_complex_1
	jg	ypass2_320_complex_3

ypass2_320_complex_2:
	start_timer 9
	yloop_init 20
	yloop_set_prefetch YHALF
	yr4_4cl_four_complex_with_square_preload
b4b:	yr4_4cl_four_complex_with_square rsi, 64, 4*64, 8*64, 4
	yloop	4, b4b, -4*64+16*64		;; Loop 4 times, next source pointer
	yloop	5, b4b, -5*16*64		;; Loop 5 times, restore source pointer
	end_timer 9
	jmp	ypass2_320_complex_unfft

ypass2_320_complex_1:
	start_timer 9
;;	bump	rcx, (80+0-20-10-40)*64	;; Move prefetch pointer along
	yloop_init 20
	yloop_set_prefetch YDOUBLE
b4a:	yr4_4cl_four_complex_fft_final rsi, 64, 4*64, 8*64
	yloop	4, b4a, -4*64+16*64		;; Loop 4 times, next source pointer
	yloop	5, b4a			;; Loop 5 times
	end_timer 9
	jmp	ypass2_320_complex_done

ypass2_320_complex_3:
	start_timer 9
x64	mov	r8, DEST2ARG
	mov	rbp, DIST_TO_MULSRCARG
	yloop_init 20
	yloop_set_prefetch YHALF
	yr4_4cl_four_complex_with_mult_preload
b4c:	yr4_4cl_four_complex_with_mult rsi, 64, 4*64, 8*64
	yloop	4, b4c, -4*64+16*64		;; Loop 4 times, next source pointer
	yloop	5, b4c, -5*16*64		;; Loop 5 times, restore source pointer
x64	add	rsi, r8
	end_timer 9
	jmp	ypass2_320_complex_unfft

ypass2_320_complex_4:
	start_timer 9
	mov	rbp, DIST_TO_MULSRCARG
	yloop_init 20
	yloop_set_prefetch YDOUBLE
	yr4_4cl_four_complex_with_mulf_preload
b4d:	yr4_4cl_four_complex_with_mulf rsi, 64, 4*64, 8*64
	yloop	4, b4d, -4*64+16*64		;; Loop 4 times, next source pointer
	yloop	5, b4d, -5*16*64		;; Loop 5 times, restore source pointer
;;	bump	rcx, (-40+20+2*10)*64	;; Move prefetch pointer to where squaring would be
	end_timer 9

;; Do inverse FFT levels 5,6
;; On input the 64-byte cache lines hold these data values:
;;	0	+32	+32	+32	4	+32	+32	+32
;;	8	...
;;	...
;;	1	...
;;	...
;;	128	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+32	+32	+32	16	+32	+32	+32
;;	4	...
;;	...
;;	12	...
;;	1	...
;;	...
;;	128	...
;;	...

;; Do 80 four_complex_unfft operations
;;	distance between fft data elements is 4
;;	do 20 macros each processing 32 data values

ypass2_320_complex_unfft:
	start_timer 10
	mov	rdi, xsincos_complex	;; Load sin/cos pointers
	bump	rdi, 16*YMM_SCD2+4*YMM_SCD2
	yloop_init 20			;; 5 iters of 4
	yloop_set_prefetch YHALF
	yr4_b4cl_four_complex_djbunfft_preload
c3b:	yr4_b4cl_four_complex_djbunfft rsi, 4*64, 64, 2*64, rdi, YMM_SCD2/4, 4
	yloop	4, c3b, 0, rdi, -4*YMM_SCD2/4 ;; Loop 4 times, restore sine/cosine pointer
	yloop	5, c3b, -20*4*64		;; Loop 5 times, restore source pointer
	end_timer 10

;; Do inverse FFT levels 3,4
;; On input the 64-byte cache lines hold these data values:
;;	0	+32	+32	+32	16	+32	+32	+32
;;	4	...
;;	...
;;	12	...
;;	1	...
;;	...
;;	128	...
;;	...
;; On output the 64-byte cache lines are swizzled to hold these data values:
;;	0	+1	+1	+1	64	+1	+1	+1
;;	4	...
;;	...
;;	60	...
;;	128	...
;;	...

;; Do 80 four_complex_unfft operations
;;	distance between fft data elements is 16
;;	do 20 macros each processing 32 data values

	start_timer 11
	bump	rdi, -4*YMM_SCD2	;; Load sin/cos ptr
	yloop_init 20			;; 5 iters of 4
	yloop_set_prefetch YSINGLE
	yr4_s4cl_four_complex_djbunfft_preload
c2b:	yr4_s4cl_four_complex_djbunfft rsi, 64, 4*64, 8*64, rdi, YMM_SCD2, 4
	yloop	4, c2b, -4*64+16*64, rdi, -4*YMM_SCD2 ;; Loop 4 times, next source pointer, restore sine/cosine pointer
	yloop	5, c2b, -5*16*64		;; Loop 5 times, restore source pointer
	end_timer 11

;; Do inverse FFT levels 1,2
;; On input the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	64	+1	+1	+1
;;	4	...
;;	...
;;	60	...
;;	128	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	320	+1	+1	+1
;;	4	...
;;	...
;;	316

;; Do 64 five_complex_unfft operations
;;	distance between fft data elements is 64
;;	do 16 macros each processing 40 data values

	start_timer 14
	bump	rdi, -16*YMM_SCD2	;; Load sin/cos ptr
	bump	rcx, (80+0-20-3*10-20-8)*64 ;; Move prefetch pointer along
	yloop_init 16			;; 16 iters
	yloop_set_prefetch YHALF
	yr5_5cl_five_complex_djbunfft_preload
c1b:	yr5_5cl_five_complex_djbunfft rsi, 64, 16*64, rdi, YMM_SCD2, 16
	yloop	16, c1b			;; Test loop counter
	end_timer 14

ypass2_320_complex_done:
	ENDM


;*********************************************************************************************
;			Macros derived from pass2-10-levels
;*********************************************************************************************


; *************** radix4-pass2-768-real macro ******************
; This macro takes 1536 real values and performs the final 10.585 levels of the FFT.
; The results are squared, and the first 10.585 levels of the inverse FFT are done.

yr4dwpn_pass2_768_real MACRO
	LOCAL	b1b, b2a, b2b, b3a, b3b, b4a, b4b, b5a, b5b, b5c, b5d
	LOCAL	c1b, c2a, c2b, c3a, c3b, c4a, c4b
	LOCAL	ypass2_768_real_1, ypass2_768_real_2
	LOCAL	ypass2_768_real_3, ypass2_768_real_4
	LOCAL	ypass2_768_real_unfft, ypass2_768_real_done

;; Type 4 FFTs skip the forward FFT process

	start_timer 2
	mov	rsi, DATA_ADDR		;; Load source address
	mov	rbx, DIST_TO_FFTSRCARG
	mov	rcx, DATA_PREFETCH	;; Prefetch pointer (we need to prefetch 
					;; 192 cache lines + 2 pad cache lines)
	cmp	ffttype, 4
	je	ypass2_768_real_4

;; Do FFT levels 1,2,3 which correspond to complex levels 1,2
;; On input the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	768	+1	+1	+1
;;	4	...
;;	...
;;	764
;; On output the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	256	+1	+1	+1
;;	4	...
;;	...
;;	252	...
;;	512	...
;;	...

;; Do 256 six_reals_fft operations
;;	distance between fft data elements is 256
;;	do 64 macros each processing 24 data values

	mov	rdi, sincos3		;; Real sin/cos data
	mov	rbp, xsincos_complex	;; Complex sin/cos data
	yloop_init 64			;; 64 iterations
	yloop_set_prefetch YHALF
	yr3_f3cl_2sc_six_reals_fft_preload
b1b:	yr3_f3cl_2sc_six_reals_fft rsi, 64, dist64, rbp, YMM_SCD1, rdi, YMM_SCD1, 64
	yloop	64, b1b, -64*64		;; Test loop counter, restore source pointer

;; Do FFT levels 3,4
;; On input the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	256	+1	+1	+1
;;	4	...
;;	...
;;	252	...
;;	512	...
;;	...
;; On output the 64-byte cache lines are swizzled to hold these data values:
;;	0	+128	+128	+128	64	+128	+128	+128
;;	4	...
;;	...
;;	60	...
;;	1	...
;;	...
;;	512	...
;;	...

;; Do 64 eight_reals_fft operations
;; Do 128 four_complex_djbfft operations
;;	distance between fft data elements is 64
;;	do 48 macros each processing 32 data values

	yloop_init 48			;; 3 iterations of 16
	yloop_set_prefetch YHALF
	yr4_s4cl_2sc_eight_reals_fft_preload
b2a:	yr4_s4cl_2sc_eight_reals_fft rsi, 64, 16*64, 32*64, rbp, YMM_SCD2, rdi, YMM_SCD2, 16
	yloop	16, b2a, -16*64+dist64, rbp, -16*YMM_SCD2 ;; Test loop counter, next source pointer, restore sine/cosine pointer

	loops_reset			;; 2 more iters of 16
	yloop_set_prefetch YHALF
	yr4_s4cl_four_complex_djbfft_preload
b2b:	yr4_s4cl_four_complex_djbfft rsi, 64, 16*64, 32*64, rbp, YMM_SCD2, 16
	yloop	16, b2b, -16*64+dist64, rbp, -16*YMM_SCD2 ;; Test loop counter, next source pointer, restore sin/cos ptr
	yloop	3, b2b, -3*dist64	;; Loop 3 times, restore source pointer

;; Do FFT levels 5,6
;; On input the 64-byte cache lines hold these data values:
;;	0	+128	+128	+128	64	+128	+128	+128
;;	4	...
;;	...
;;	60	...
;;	1	...
;;	...
;;	512	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+128	+128	+128	16	+128	+128	+128
;;	4	...
;;	...
;;	12	...
;;	32	...
;;	...
;;	1	...
;;	...
;;	512	...
;;	...

;; Do 16 eight_reals_fft operations
;; Do 176 four_complex_djbfft operations
;;	distance between fft data elements is 16
;;	do 48 macros each processing 32 data values

	yloop_init 48			;; 3 iterations of 16
	yloop_set_prefetch YHALF
	yr4_4cl_eight_reals_four_complex_djbfft_preload
b3a:	yr4_4cl_eight_reals_four_complex_djbfft rsi, 16*64, 4*64, 8*64, rdi, YMM_SCD3, 4
	yloop	4, b3a, -4*16*64+64		;; Test loop counter, next source pointer
	yloop	4, b3a, -4*64+dist64, rdi, -16*YMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer

	loops_reset			;; 2 more iters of 16
	yloop_set_prefetch YHALF
	yr4_rb4cl_four_complex_djbfft_preload
b3b:	yr4_rb4cl_four_complex_djbfft rsi, 16*64, 4*64, 8*64, rdi, YMM_SCD3, 4
	yloop	4, b3b, -4*16*64+64	;; Test loop counter, next source pointer
	yloop	4, b3b, -4*64+dist64, rdi, -16*YMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer
	yloop	3, b3b, -3*dist64	;; Test loop counter, restore source pointer

;; Do FFT levels 7,8
;; On input the 64-byte cache lines hold these data values:
;;	0	+128	+128	+128	16	+128	+128	+128
;;	4	...
;;	...
;;	12	...
;;	32	...
;;	...
;;	1	...
;;	...
;;	512	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+128	+128	+128	4	+128	+128	+128
;;	8	...
;;	...
;;	1	...
;;	...
;;	512	...
;;	...

;; Do 4 eight_reals_fft operations
;; Do 188 four_complex_djbfft operations
;;	distance between fft data elements is 4
;;	do 48 macros each processing 32 data values

	yloop_init 48			;; 12 iterations of 4
	yloop_set_prefetch YHALF
	yr4_4cl_eight_reals_four_complex_djbfft_preload
b4a:	yr4_4cl_eight_reals_four_complex_djbfft rsi, 16*64, 64, 2*64, rdi, 4*YMM_SCD3, 4
	yloop	4, b4a, -4*16*64+4*64, rdi, -4*4*YMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer

	loops_reset			;; 11 more iters of 4
	yloop_set_prefetch YHALF
	yr4_rb4cl_four_complex_djbfft_preload
b4b:	yr4_rb4cl_four_complex_djbfft rsi, 16*64, 64, 2*64, rdi, 4*YMM_SCD3, 4
	yloop	4, b4b, -4*16*64+4*64, rdi, -4*4*YMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer
	yloop	4, b4b, -4*4*64+dist64	;; Test loop counter, next source pointer
	yloop	3, b4b, -3*dist64	;; Test loop counter, restore source pointer

;; Do FFT levels 9,10
;; On input and output the 64-byte cache lines hold these data values:
;;	0	+128	+128	+128	4	+128	+128	+128
;;	8	...
;;	...
;;	1	...
;;	...
;;	512	...
;;	...

;; Do 1 eight_reals_fft operations
;; Do 191 four_complex_fft operations
;;	distance between fft data elements is 1
;;	do 48 macros each processing 32 data values

;; Execute the proper middle step

	cmp	ffttype, 2
	jl	ypass2_768_real_1
	jg	ypass2_768_real_3

ypass2_768_real_2:
;;	yr4_4cl_eight_reals_four_complex_with_square rsi, 64, 16*64, 32*64
	yr4_dispatch call, yreal_square, 16*64, 32*64
	yloop_init 48			;; 47 more iterations
	yloop_adjust 1
	yloop_set_prefetch YHALF
	yr4_4cl_four_complex_with_square_preload
b5b:	yr4_4cl_four_complex_with_square rsi, 64, 16*64, 32*64, 1
	yloop	16, b5b, -16*64+dist64	;; Loop 16 times, next source pointer
	yloop	3, b5b, -3*dist64	;; Loop 3 times, restore source pointer
	jmp	ypass2_768_real_unfft

ypass2_768_real_1:
;;	yr4_4cl_eight_reals_four_complex_fft_final rsi, 64, 16*64, 32*64
	yr4_dispatch call, yreal_fft_final, 16*64, 32*64
	mov	rcx, xsincos_complex	;; Prefetch complex sin/cos data, 98 cache lines
	yloop_init 48			;; 47 more iterations
	yloop_adjust 1
	yloop_set_prefetch YDOUBLE
b5a:	yr4_4cl_four_complex_fft_final rsi, 64, 16*64, 32*64
	yloop	16, b5a, -16*64+dist64	;; Loop 16 times, next source pointer
	yloop	3, b5a			;; Loop 3 times
	jmp	ypass2_768_real_done

ypass2_768_real_3:
x64	mov	r8, DEST2ARG
	mov	rbp, DIST_TO_MULSRCARG
;;	yr4_4cl_eight_reals_four_complex_with_mult rsi, 64, 16*64, 32*64
	yr4_dispatch call, yreal_mult, 16*64, 32*64
	yloop_init 48			;; 47 more iterations
	yloop_adjust 1
	yloop_set_prefetch YHALF
	yr4_4cl_four_complex_with_mult_preload
b5c:	yr4_4cl_four_complex_with_mult rsi, 64, 16*64, 32*64
	yloop	16, b5c, -16*64+dist64	;; Loop 16 times, next source pointer
	yloop	3, b5c, -3*dist64	;; Loop 3 times, restore source pointer
x64	add	rsi, r8
	jmp	ypass2_768_real_unfft

ypass2_768_real_4:
	mov	rbp, DIST_TO_MULSRCARG
;;	yr4_4cl_eight_reals_four_complex_with_mulf rsi, 64, 16*64, 32*64
	yr4_dispatch call, yreal_mulf, 16*64, 32*64
	yloop_init 48			;; 47 more iterations
	yloop_adjust 1
	yloop_set_prefetch YDOUBLE
	yr4_4cl_four_complex_with_mulf_preload
b5d:	yr4_4cl_four_complex_with_mulf rsi, 64, 16*64, 32*64
	yloop	16, b5d, -16*64+dist64	;; Loop 16 times, next source pointer
	yloop	3, b5d, -3*dist64	;; Loop 3 times, restore source pointer
;;	bump	rcx, (-96+32+4*24)*64	;; Move prefetch pointer back to where squaring would be

;; Do inverse FFT levels 7,8
;; On input the 64-byte cache lines hold these data values:
;;	0	+128	+128	+128	4	+128	+128	+128
;;	8	...
;;	...
;;	1	...
;;	...
;;	512	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+128	+128	+128	16	+128	+128	+128
;;	4	...
;;	...
;;	12	...
;;	32	...
;;	...
;;	1	...
;;	...
;;	512	...
;;	...

;; Do 4 eight_reals_unfft operations
;; Do 188 four_complex_djbunfft operations
;;	distance between fft data elements is 4
;;	do 48 macros processing 32 data values

ypass2_768_real_unfft:
	mov	rdi, sincos3		;; Load sin/cos pointer
	bump	rdi, 16*YMM_SCD2+64*YMM_SCD1
	yloop_init 48			;; 12 iterations of 4
	yloop_set_prefetch YHALF
	yr4_4cl_eight_reals_four_complex_djbunfft_preload
c4a:	yr4_4cl_eight_reals_four_complex_djbunfft rsi, 16*64, 64, 2*64, rdi, 4*YMM_SCD3, 4
	yloop	4, c4a, -4*16*64+4*64, rdi, -4*4*YMM_SCD3 ;; Test loop counter, next source pointer, restore sine/cosine pointer

	loops_reset			;; 11 more iters of 4
	yloop_set_prefetch YHALF
	yr4_rb4cl_four_complex_djbunfft_preload
c4b:	yr4_rb4cl_four_complex_djbunfft rsi, 16*64, 64, 2*64, rdi, 4*YMM_SCD3, 4
	yloop	4, c4b, -4*16*64+4*64, rdi, -4*4*YMM_SCD3 ;; Test loop counter, next source pointer, restore sine/cosine pointer
	yloop	4, c4b, -4*4*64+dist64	;; Test loop counter, next source pointer
	yloop	3, c4b, -3*dist64	;; Test loop counter, restore source pointer

;; Do inverse FFT levels 5,6
;; On input the 64-byte cache lines hold these data values:
;;	0	+128	+128	+128	16	+128	+128	+128
;;	4	...
;;	...
;;	12	...
;;	32	...
;;	...
;;	1	...
;;	...
;;	512	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+128	+128	+128	64	+128	+128	+128
;;	4	...
;;	...
;;	60	...
;;	1	...
;;	...
;;	512	...
;;	...

;; Do 16 eight_reals_unfft operations
;; Do 176 four_complex_djbunfft operations
;;	distance between fft data elements is 16
;;	do 48 macros processing 32 data values

	bump	rcx, (192+2-32-5*24-48)*64 ;; Move prefetch pointer along
	yloop_init 48			;; 3 iterations of 16
	yloop_set_prefetch YSINGLE
	yr4_4cl_eight_reals_four_complex_djbunfft_preload
c3a:	yr4_4cl_eight_reals_four_complex_djbunfft rsi, 16*64, 4*64, 8*64, rdi, YMM_SCD3, 4
	yloop	4, c3a, -4*16*64+64	;; Test loop counter, next source pointer
	yloop	4, c3a, -4*64+dist64, rdi, -16*YMM_SCD3 ;; Test loop counter, next source pointer, restore sine/cosine pointer

	loops_reset			;; 2 more iters of 16
	yloop_set_prefetch YSINGLE
	yr4_rb4cl_four_complex_djbunfft_preload
c3b:	yr4_rb4cl_four_complex_djbunfft rsi, 16*64, 4*64, 8*64, rdi, YMM_SCD3, 4
	yloop	4, c3b, -4*16*64+64	;; Test loop counter, next source pointer
	yloop	4, c3b, -4*64+dist64, rdi, -16*YMM_SCD3 ;; Test loop counter, next source pointer, restore sine/cosine pointer
	yloop	3, c3b, -3*dist64	;; Test loop counter, restore source pointer

;; Do inverse FFT levels 3,4
;; On input the 64-byte cache lines hold these data values:
;;	0	+128	+128	+128	64	+128	+128	+128
;;	4	...
;;	...
;;	60	...
;;	1	...
;;	...
;;	512	...
;;	...
;; On output the 64-byte cache lines are swizzled to hold these data values:
;;	0	+1	+1	+1	256	+1	+1	+1
;;	4	...
;;	...
;;	252	...
;;	512	...
;;	...

;; Do 64 eight_reals_unfft operations
;; Do 128 four_complex_djbunfft operations
;;	distance between fft data elements is 64
;;	do 48 macros processing 32 data values

	bump	rdi, -16*YMM_SCD2	;; Real sin/cos ptrs
	mov	rbp, xsincos_complex	;; Complex sin/cos data
	bump	rbp, 64*YMM_SCD1
	mov	rcx, xsincos_complex	;; Prefetch complex sin/cos data, 98 cache lines
	yloop_init 48			;; 3 iterations of 16
	yloop_set_prefetch YHALF
	yr4_s4cl_2sc_eight_reals_unfft_preload
c2a:	yr4_s4cl_2sc_eight_reals_unfft rsi, 64, 16*64, 32*64, rbp, YMM_SCD2, rdi, YMM_SCD2, 16
	yloop	16, c2a, -16*64+dist64, rbp, -16*YMM_SCD2 ;; Test loop counter, next source pointer, restore sine/cosine pointer

	loops_reset			;; 2 more iters of 16
	yloop_set_prefetch YHALF
	yr4_s4cl_four_complex_djbunfft_preload
c2b:	yr4_s4cl_four_complex_djbunfft rsi, 64, 16*64, 32*64, rbp, YMM_SCD2, 16
	yloop	16, c2b, -16*64+dist64, rbp, -16*YMM_SCD2 ;; Test loop counter, next source pointer, restore sine/cosine pointer
	yloop	3, c2b, -3*dist64	;; Test loop counter, restore source pointer

;; Do inverse FFT levels 1,2,3 which correspond to complex levels 1,2
;; On input the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	256	+1	+1	+1
;;	4	...
;;	...
;;	252	...
;;	512	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	768	+1	+1	+1
;;	4	...
;;	...
;;	764

;; Do 256 six_reals_unfft operations
;;	distance between fft data elements is 256
;;	do 64 macros processing 24 data values

	bump	rdi, -16*YMM_SCD2-64*YMM_SCD1 ;; Real sin/cos ptr
	bump	rbp, -64*YMM_SCD1	;; Complex sin/cos data
	bump	rcx, (98-24-64)*64	;; Move prefetch pointer along
	yloop_init 64			;; 64 iterations
	yloop_set_prefetch YSINGLE
	yr3_3cl_2sc_six_reals_unfft_preload
c1b:	yr3_3cl_2sc_six_reals_unfft rsi, 64, dist64, rbp, YMM_SCD1, rdi, YMM_SCD1, 64
	yloop	64, c1b			;; Test loop counter

ypass2_768_real_done:
	end_timer 2
	ENDM

; *************** radix4-pass2-768-complex macro ******************
; This macro takes 768 complex values and performs the final 9.585 levels of a standard
; mostly radix-4 FFT, squares the results, and does the first 9.585 levels of the inverse FFT.

yr4dwpn_pass2_768_complex MACRO
	LOCAL	b1b, b2b, b3b, b4b, b5a, b5b, b5c, b5d, c1b, c2b, c3b, c4b
	LOCAL	ypass2_768_complex_1, ypass2_768_complex_2
	LOCAL	ypass2_768_complex_3, ypass2_768_complex_4
	LOCAL	ypass2_768_complex_unfft, ypass2_768_complex_done

;; Type 4 FFTs skip the forward FFT process

	mov	rsi, DATA_ADDR		;; Load address of FFT data
	mov	rbx, DIST_TO_FFTSRCARG
	mov	rcx, DATA_PREFETCH	;; Prefetch pointer (we need to prefetch 
					;; 192 cache lines + 2 pad cache lines)
	cmp	ffttype, 4
	je	ypass2_768_complex_4

;; Do FFT levels 1,2 (actually 1.585 levels)
;; On input the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	768	+1	+1	+1
;;	4	...
;;	...
;;	764
;; On output the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	256	+1	+1	+1
;;	4	...
;;	...
;;	252	...
;;	512	...
;;	...

;; Do 256 three_complex_fft operations
;;	distance between fft data elements is 256
;;	do 64 macros each processing 24 data values

	start_timer 4
	mov	rdi, xsincos_complex	;; The sin/cos data
	yloop_init 64			;; 64 iterations
	yr3_f3cl_three_complex_djbfft_preload
;; Loop below reworked to avoid 4KB distances
;;b1b:	yr3_f3cl_three_complex_djbfft rsi, 64, dist64, rdi, YMM_SCD1, 64
;;	yloop	64, b1b, -64*64		;; Test inner loop counter, restore source pointer
b1b:	yr3_f3cl_three_complex_djbfft rsi, 4*64, dist64, rdi, 4*YMM_SCD1, 16
	yloop	16, b1b, -16*4*64+64, rdi, -16*4*YMM_SCD1+YMM_SCD1 ;; Test inner loop counter, next source pointer, next sin/cos data
	yloop	4, b1b, -4*64, rdi, -4*YMM_SCD1+64*YMM_SCD1 ;; Test loop counter, restore source pointer, next sin/cos data
	end_timer 4

;; Do FFT levels 3,4
;; On input the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	256	+1	+1	+1
;;	4	...
;;	...
;;	252	...
;;	512	...
;;	...
;; On output the 64-byte cache lines are swizzled to hold these data values:
;;	0	+128	+128	+128	64	+128	+128	+128
;;	4	...
;;	...
;;	60	...
;;	1	...
;;	...
;;	512	...
;;	...

;; Do 192 four_complex_fft operations
;;	distance between fft data elements is 64
;;	do 48 macros each processing 32 data values

	start_timer 6
	yloop_init 48			;; 3 iters of 16
	yloop_set_prefetch YHALF
	yr4_s4cl_four_complex_djbfft_preload
b2b:	yr4_s4cl_four_complex_djbfft rsi, 64, 16*64, 32*64, rdi, YMM_SCD2, 16
	yloop	16, b2b, -16*64+dist64, rdi, -16*YMM_SCD2 ;; Loop 16 times, next source pointer, restore sine/cosine pointer
	yloop	3, b2b, -3*dist64, rdi, 16*YMM_SCD2 ;; Loop 3 times, restore source pointer, next sine/cosine pointer
	end_timer 6

;; Do FFT levels 5,6
;; On input the 64-byte cache lines hold these data values:
;;	0	+128	+128	+128	64	+128	+128	+128
;;	4	...
;;	...
;;	60	...
;;	1	...
;;	...
;;	512	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+128	+128	+128	16	+128	+128	+128
;;	4	...
;;	...
;;	12	...
;;	32	...
;;	...
;;	1	...
;;	...
;;	512	...
;;	...

;; Do 192 four_complex_fft operations
;;	distance between fft data elements is 16
;;	do 48 macros each processing 32 data values

	start_timer 7
	yloop_init 48			;; 3 iters of 16
	yloop_set_prefetch YHALF
	yr4_b4cl_four_complex_djbfft_preload
b3b:	yr4_b4cl_four_complex_djbfft rsi, 64, 4*64, 8*64, rdi, YMM_SCD2/4, 4
	yloop	4, b3b, -4*64+16*64		;; Loop 4 times, next source pointer
	yloop	4, b3b, -4*16*64+dist64, rdi, -16*YMM_SCD2/4 ;; Loop 4 times, next source pointer, restore sine/cosine pointer
	yloop	3, b3b, -3*dist64		;; Loop 3 times, restore source pointer
	end_timer 7

;; Do FFT levels 7,8
;; On input the 64-byte cache lines hold these data values:
;;	0	+128	+128	+128	16	+128	+128	+128
;;	4	...
;;	...
;;	12	...
;;	32	...
;;	...
;;	1	...
;;	...
;;	512	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+128	+128	+128	4	+128	+128	+128
;;	8	...
;;	...
;;	1	...
;;	...
;;	512	...
;;	...

;; Do 192 four_complex_fft operations
;;	distance between fft data elements is 4
;;	do 48 macros each processing 32 data values

	start_timer 8
	yloop_init 48			;; 12 iters of 4
	yloop_set_prefetch YHALF
	yr4_b4cl_four_complex_djbfft_preload
b4b:	yr4_b4cl_four_complex_djbfft rsi, 4*64, 64, 2*64, rdi, 0, 4
	yloop	4, b4b, 0, rdi, YMM_SCD2/4 ;; Loop 4 times, next sine/cosine pointer
	yloop	4, b4b, -16*4*64+dist64, rdi, -4*YMM_SCD2/4 ;; Loop 4 times, next source pointer, restore sine/cosine pointer
	yloop	3, b4b, -3*dist64		;; Loop 3 times, restore source pointer
	end_timer 8

;; Do FFT levels 9,10 as well as inverse FFT levels 9,10
;; On input and output the 64-byte cache lines hold these data values:
;;	0	+128	+128	+128	4	+128	+128	+128
;;	8	...
;;	...
;;	1	...
;;	...
;;	512	...
;;	...

;; Do 192 four_complex_fft operations
;;	distance between fft data elements is 1
;;	do 48 macros each processing 32 data values

;; Execute the right middle step

	cmp	ffttype, 2
	jl	ypass2_768_complex_1
	jg	ypass2_768_complex_3

ypass2_768_complex_2:
	start_timer 9
	yloop_init 48
	yloop_set_prefetch YHALF
	yr4_4cl_four_complex_with_square_preload
b5b:	yr4_4cl_four_complex_with_square rsi, 64, 16*64, 32*64, 16
	yloop	16, b5b, -16*64+dist64		;; Loop 16 times, next source pointer
	yloop	3, b5b, -3*dist64		;; Loop 3 times, restore source pointer
	end_timer 9
	jmp	ypass2_768_complex_unfft

ypass2_768_complex_1:
	start_timer 9
;;	bump	rcx, (192+2-3*24-96)*64 ;; Move prefetch pointer along
	yloop_init 48
	yloop_set_prefetch YDOUBLE
b5a:	yr4_4cl_four_complex_fft_final rsi, 64, 16*64, 32*64
	yloop	16, b5a, -16*64+dist64		;; Loop 16 times, next source pointer
	yloop	3, b5a			;; Loop 3 times
	end_timer 9
	jmp	ypass2_768_complex_done

ypass2_768_complex_3:
	start_timer 9
x64	mov	r8, DEST2ARG
	mov	rbp, DIST_TO_MULSRCARG
	yloop_init 48
	yloop_set_prefetch YHALF
	yr4_4cl_four_complex_with_mult_preload
b5c:	yr4_4cl_four_complex_with_mult rsi, 64, 16*64, 32*64
	yloop	16, b5c, -16*64+dist64		;; Loop 16 times, next source pointer
	yloop	3, b5c, -3*dist64		;; Loop 3 times, restore source pointer
x64	add	rsi, r8
	end_timer 9
	jmp	ypass2_768_complex_unfft

ypass2_768_complex_4:
	start_timer 9
	mov	rbp, DIST_TO_MULSRCARG
	yloop_init 48
	yloop_set_prefetch YDOUBLE
	yr4_4cl_four_complex_with_mulf_preload
b5d:	yr4_4cl_four_complex_with_mulf rsi, 64, 16*64, 32*64
	yloop	16, b5d, -16*64+dist64		;; Loop 16 times, next source pointer
	yloop	3, b5d, -3*dist64		;; Loop 3 times, restore source pointer
;;	bump	rcx, (-96+4*24)*64	;; Move prefetch pointer to where squaring would be
	end_timer 9

;; Do inverse FFT levels 7,8
;; On input the 64-byte cache lines hold these data values:
;;	0	+128	+128	+128	4	+128	+128	+128
;;	8	...
;;	...
;;	1	...
;;	...
;;	512	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+128	+128	+128	16	+128	+128	+128
;;	4	...
;;	...
;;	12	...
;;	32	...
;;	...
;;	1	...
;;	...
;;	512	...
;;	...

;; Do 192 four_complex_unfft operations
;;	distance between fft data elements is 4
;;	do 48 macros each processing 32 data values

ypass2_768_complex_unfft:
	start_timer 10
	mov	rdi, xsincos_complex	;; Load sin/cos pointers
	bump	rdi, 64*YMM_SCD1+16*YMM_SCD2
	yloop_init 48			;; 12 iters of 4
	yloop_set_prefetch YHALF
	yr4_b4cl_four_complex_djbunfft_preload
c4b:	yr4_b4cl_four_complex_djbunfft rsi, 4*64, 64, 2*64, rdi, 0, 4
	yloop	4, c4b, 0, rdi, YMM_SCD2/4 ;; Loop 4 times, next sine/cosine pointer
	yloop	4, c4b, -16*4*64+dist64, rdi, -4*YMM_SCD2/4 ;; Loop 4 times, next source pointer, restore sine/cosine pointer
	yloop	3, c4b, -3*dist64	;; Loop 3 times, restore source pointer
	end_timer 10

;; Do inverse FFT levels 5,6
;; On input the 64-byte cache lines hold these data values:
;;	0	+128	+128	+128	16	+128	+128	+128
;;	4	...
;;	...
;;	12	...
;;	32	...
;;	...
;;	1	...
;;	...
;;	512	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+128	+128	+128	64	+128	+128	+128
;;	4	...
;;	...
;;	60	...
;;	1	...
;;	...
;;	512	...
;;	...

;; Do 192 four_complex_unfft operations
;;	distance between fft data elements is 16
;;	do 48 macros each processing 32 data values

	start_timer 11
	yloop_init 48			;; 3 iters of 16
	yloop_set_prefetch YHALF
	yr4_b4cl_four_complex_djbunfft_preload
c3b:	yr4_b4cl_four_complex_djbunfft rsi, 64, 4*64, 8*64, rdi, YMM_SCD2/4, 4
	yloop	4, c3b, -4*64+16*64	;; Loop 4 times, next source pointer
	yloop	4, c3b, -4*16*64+dist64, rdi, -16*YMM_SCD2/4 ;; Loop 4 times, next source pointer, restore sine/cosine pointer
	yloop	3, c3b, -3*dist64	;; Loop 3 times, restore source pointer
	end_timer 11

;; Do inverse FFT levels 3,4
;; On input the 64-byte cache lines hold these data values:
;;	0	+128	+128	+128	64	+128	+128	+128
;;	4	...
;;	...
;;	60	...
;;	1	...
;;	...
;;	512	...
;;	...
;; On output the 64-byte cache lines are swizzled to hold these data values:
;;	0	+1	+1	+1	256	+1	+1	+1
;;	4	...
;;	...
;;	252	...
;;	512	...
;;	...

;; Do 192 four_complex_unfft operations
;;	distance between fft data elements is 64
;;	do 48 macros each processing 32 data values

	start_timer 12
	bump	rdi, -16*YMM_SCD2	;; Load sin/cos ptr
	yloop_init 48			;; 3 iters of 16
	yloop_set_prefetch YHALF
	yr4_s4cl_four_complex_djbunfft_preload
c2b:	yr4_s4cl_four_complex_djbunfft rsi, 64, 16*64, 32*64, rdi, YMM_SCD2, 16
	yloop	16, c2b, -16*64+dist64, rdi, -16*YMM_SCD2 ;; Loop 16 times, next source pointer, restore sine/cosine pointer
	yloop	3, c2b, -3*dist64		;; Loop 3 times, restore source pointer
	end_timer 12

;; Do inverse FFT levels 1,2
;; On input the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	256	+1	+1	+1
;;	4	...
;;	...
;;	252	...
;;	512	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	768	+1	+1	+1
;;	4	...
;;	...
;;	764

;; Do 256 three_complex_unfft operations
;;	distance between fft data elements is 256
;;	do 64 macros each processing 24 data values

	start_timer 14
	bump	rdi, -64*YMM_SCD1	;; Load sin/cos ptr
	bump	rcx, (192+2-7*24-32)*64 ;; Move prefetch pointer along
	yloop_init 64			;; 64 iters
	yloop_set_prefetch YHALF
	yr3_3cl_three_complex_djbunfft_preload
;; Loop below reworked to avoid 4KB distances
;;c1b:	yr3_3cl_three_complex_djbunfft rsi, 64, dist64, rdi, YMM_SCD1, 64
;;	yloop	64, c1b			;; Test loop counter
c1b:	yr3_3cl_three_complex_djbunfft rsi, 4*64, dist64, rdi, 4*YMM_SCD1, 16
	yloop	16, c1b, -16*4*64+64, rdi, -16*4*YMM_SCD1+YMM_SCD1 ;; Test loop counter, next source pointer, next set of sin/cos data
	yloop	4, c1b			;; Test loop counter
	end_timer 14

ypass2_768_complex_done:
	ENDM


; *************** radix4-pass2-10-levels-real macro ******************
; This macro takes 2048 real values and performs the final 11 levels of the FFT.
; The results are squared, and the first 11 levels of the inverse FFT are done.

yr4dwpn_pass2_10_levels_real MACRO
	LOCAL	b1b, b2b, b3a, b3b, b4a, b4b, b5a, b5b, b5c, b5d
	LOCAL	c1b, c2b, c3a, c3b, c4a, c4b
	LOCAL	ypass2_10_levels_real_1, ypass2_10_levels_real_2
	LOCAL	ypass2_10_levels_real_3, ypass2_10_levels_real_4
	LOCAL	ypass2_10_levels_real_unfft, ypass2_10_levels_real_done

;; Type 4 FFTs skip the forward FFT process

	start_timer 2
	mov	rsi, DATA_ADDR		;; Load source address
	mov	rbx, DIST_TO_FFTSRCARG
	mov	rcx, DATA_PREFETCH	;; Prefetch pointer (we need to prefetch 
					;; 256 cache lines + 2*3 pad cache lines)
	cmp	ffttype, 4
	je	ypass2_10_levels_real_4

;; Do FFT levels 1,2,3 which correspond to complex levels 1,2
;; On input the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	1024	+1	+1	+1
;;	4	...
;;	...
;;	1020
;; On output the 64-byte cache lines are swizzled to hold these data values:
;;	0	+512	+512	+512	256	+512	+512	+512
;;	4	...
;;	...
;;	252	...
;;	1	...
;;	...

;; Do 256 eight_reals_fft operations
;;	distance between fft data elements is 256
;;	do 64 macros each processing 32 data values

	mov	rdi, sincos3		;; Real sin/cos data
	mov	rbp, xsincos_complex	;; Complex sin/cos data
	yloop_init 64
	yloop_set_prefetch YSINGLE
	yr4_fs4cl_2sc_eight_reals_fft_preload
b1b:	yr4_fs4cl_2sc_eight_reals_fft rsi, 64, dist64, 2*dist64, rbp, YMM_SCD2, rdi, YMM_SCD2, 64
	yloop	64, b1b, -64*64		;; Test loop counter, restore source pointer
	bump	rcx, 64			;; Skip pad bytes

;; Do FFT levels 3,4
;; On input the 64-byte cache lines hold these data values:
;;	0	+512	+512	+512	256	+512	+512	+512
;;	4	...
;;	...
;;	252	...
;;	1	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+512	+512	+512	64	+512	+512	+512
;;	4	...
;;	...
;;	60	...
;;	128	...
;;	...
;;	1	...
;;	...

;; Do 64 eight_reals_fft operations
;; Do 192 four_complex_djbfft operations
;;	distance between fft data elements is 64
;;	do 64 macros each processing 32 data values

	yloop_init 64
	yloop_set_prefetch YSINGLE
	yr4_4cl_eight_reals_four_complex_djbfft_preload
b2b:	yr4_4cl_eight_reals_four_complex_djbfft rsi, dist64, 16*64, 32*64, rdi, YMM_SCD3, 4
	yloop	4, b2b, -4*dist64+64		;; Test loop counter, next source pointer
	yloop	16, b2b, -16*64, rdi, -64*YMM_SCD3 ;; Test loop counter, restore source pointer, restore sin/cos ptr
	bump	rcx, 64			;; Skip pad bytes

;; Do FFT levels 5,6
;; On input the 64-byte cache lines hold these data values:
;;	0	+512	+512	+512	64	+512	+512	+512
;;	4	...
;;	...
;;	60	...
;;	128	...
;;	...
;;	1	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+512	+512	+512	16	+512	+512	+512
;;	4	...
;;	...
;;	12	...
;;	32	...
;;	...
;;	1	...
;;	...

;; Do 16 eight_reals_fft operations
;; Do 240 four_complex_djbfft operations
;;	distance between fft data elements is 16
;;	do 64 macros each processing 32 data values

	yloop_init 64			;; 4 iterations of 16
	yloop_set_prefetch YHALF
	yr4_4cl_eight_reals_four_complex_djbfft_preload
b3a:	yr4_4cl_eight_reals_four_complex_djbfft rsi, dist64, 4*64, 8*64, rdi, 4*YMM_SCD3, 4
	yloop	4, b3a, -4*dist64+64		;; Test loop counter, next source pointer
	yloop	4, b3a, -4*64+16*64, rdi, -16*4*YMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer

	loops_reset			;; 3 more iters of 16
	yloop_set_prefetch YHALF
	yr4_rb4cl_four_complex_djbfft_preload
b3b:	yr4_rb4cl_four_complex_djbfft rsi, dist64, 4*64, 8*64, rdi, 4*YMM_SCD3, 4
	yloop	4, b3b, -4*dist64+64	;; Test loop counter, next source pointer
	yloop	4, b3b, -4*64+16*64, rdi, -16*4*YMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer
	yloop	4, b3b, -4*16*64	;; Test loop counter, restore source pointer

;; Do FFT levels 7,8
;; On input the 64-byte cache lines hold these data values:
;;	0	+512	+512	+512	16	+512	+512	+512
;;	4	...
;;	...
;;	12	...
;;	32	...
;;	...
;;	1	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+512	+512	+512	4	+512	+512	+512
;;	8	...
;;	...
;;	1	...
;;	...

;; Do 4 eight_reals_fft operations
;; Do 252 four_complex_djbfft operations
;;	distance between fft data elements is 4
;;	do 64 macros each processing 32 data values

	yloop_init 64			;; 16 iterations of 4
	yloop_set_prefetch YHALF
	yr4_4cl_eight_reals_four_complex_djbfft_preload
b4a:	yr4_4cl_eight_reals_four_complex_djbfft rsi, dist64, 64, 2*64, rdi, 16*YMM_SCD3, 4
	yloop	4, b4a, -4*dist64+4*64, rdi, -4*16*YMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer

	loops_reset			;; 15 more iters of 4
	yloop_set_prefetch YHALF
	yr4_rb4cl_four_complex_djbfft_preload
b4b:	yr4_rb4cl_four_complex_djbfft rsi, dist64, 64, 2*64, rdi, 16*YMM_SCD3, 4
	yloop	4, b4b, -4*dist64+4*64, rdi, -4*16*YMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer
	yloop	16, b4b, -16*4*64	;; Test loop counter, restore source pointer
	bump	rcx, 64			;; Skip pad bytes

;; Do FFT levels 9,10
;; On input and output the 64-byte cache lines hold these data values:
;;	0	+512	+512	+512	4	+512	+512	+512
;;	8	...
;;	...
;;	1	...
;;	...

;; Do 1 eight_reals_fft operations
;; Do 255 four_complex_fft operations
;;	distance between fft data elements is 1
;;	do 64 macros each processing 32 data values

;; Execute the proper middle step

	cmp	ffttype, 2
	jl	ypass2_10_levels_real_1
	jg	ypass2_10_levels_real_3

ypass2_10_levels_real_2:
;;	yr4_4cl_eight_reals_four_complex_with_square rsi, 64, dist64, 2*dist64
	yr4_dispatch call, yreal_square, dist64, 2*dist64
	yloop_init 64			;; 63 more iterations
	yloop_adjust 1
	yloop_set_prefetch YHALF
	yr4_4cl_four_complex_with_square_preload
b5b:	yr4_4cl_four_complex_with_square rsi, 64, dist64, 2*dist64, 1
	yloop	64, b5b, -64*64		;; Test loop counter, restore source pointer
	jmp	ypass2_10_levels_real_unfft

ypass2_10_levels_real_1:
;;	yr4_4cl_eight_reals_four_complex_fft_final rsi, 64, dist64, 2*dist64
	yr4_dispatch call, yreal_fft_final, dist64, 2*dist64
	mov	rcx, xsincos_complex	;; Prefetch complex sin/cos data, 136 cache lines
	yloop_init 64			;; 63 more iterations
	yloop_adjust 1
	yloop_set_prefetch YDOUBLE
b5a:	yr4_4cl_four_complex_fft_final rsi, 64, dist64, 2*dist64
	yloop	64, b5a			;; Test loop counter
	jmp	ypass2_10_levels_real_done

ypass2_10_levels_real_3:
x64	mov	r8, DEST2ARG
	mov	rbp, DIST_TO_MULSRCARG
;;	yr4_4cl_eight_reals_four_complex_with_mult rsi, 64, dist64, 2*dist64
	yr4_dispatch call, yreal_mult, dist64, 2*dist64
	yloop_init 64			;; 63 more iterations
	yloop_adjust 1
	yloop_set_prefetch YHALF
	yr4_4cl_four_complex_with_mult_preload
b5c:	yr4_4cl_four_complex_with_mult rsi, 64, dist64, 2*dist64
	yloop	64, b5c, -64*64		;; Test loop counter, restore source pointer
x64	add	rsi, r8
	jmp	ypass2_10_levels_real_unfft

ypass2_10_levels_real_4:
	mov	rbp, DIST_TO_MULSRCARG
;;	yr4_4cl_eight_reals_four_complex_with_mulf rsi, 64, dist64, 2*dist64
	yr4_dispatch call, yreal_mulf, dist64, 2*dist64
	yloop_init 64			;; 63 more iterations
	yloop_adjust 1
	yloop_set_prefetch YDOUBLE
	yr4_4cl_four_complex_with_mulf_preload
b5d:	yr4_4cl_four_complex_with_mulf rsi, 64, dist64, 2*dist64
	yloop	64, b5d, -64*64		;; Test loop counter, restore source pointer
;;	bump	rcx, (-128+2*64+3*32)*64 ;; Move prefetch pointer back to where squaring would be

;; Do inverse FFT levels 7,8
;; On input the 64-byte cache lines hold these data values:
;;	0	+512	+512	+512	4	+512	+512	+512
;;	8	...
;;	...
;;	1	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+512	+512	+512	16	+512	+512	+512
;;	4	...
;;	...
;;	12	...
;;	32	...
;;	...
;;	1	...
;;	...

;; Do 4 eight_reals_unfft operations
;; Do 252 four_complex_djbunfft operations
;;	distance between fft data elements is 4
;;	do 64 macros processing 32 data values

ypass2_10_levels_real_unfft:
	mov	rdi, sincos3		;; Load sin/cos pointer
	bump	rdi, 64*YMM_SCD2
;;	bump	rcx, (256+2*3-2*64-3*32-64)*64 ;; Move prefetch pointer back to where squaring would be
	yloop_init 64			;; 16 iterations of 4
	yloop_set_prefetch YSINGLE
	yr4_4cl_eight_reals_four_complex_djbunfft_preload
c4a:	yr4_4cl_eight_reals_four_complex_djbunfft rsi, dist64, 64, 2*64, rdi, 16*YMM_SCD3, 4
	yloop	4, c4a, -4*dist64+4*64, rdi, -4*16*YMM_SCD3 ;; Test loop counter, next source pointer, restore sine/cosine pointer

	loops_reset			;; 15 more iters of 4
	yr4_rb4cl_four_complex_djbunfft_preload
c4b:	yr4_rb4cl_four_complex_djbunfft rsi, dist64, 64, 2*64, rdi, 16*YMM_SCD3, 4
	yloop	4, c4b, -4*dist64+4*64, rdi, -4*16*YMM_SCD3 ;; Test loop counter, next source pointer, restore sine/cosine pointer
	yloop	16, c4b, -16*4*64		;; Test loop counter, restore source pointer

;; Do inverse FFT levels 5,6
;; On input the 64-byte cache lines hold these data values:
;;	0	+512	+512	+512	16	+512	+512	+512
;;	4	...
;;	...
;;	12	...
;;	32	...
;;	...
;;	1	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+512	+512	+512	64	+512	+512	+512
;;	4	...
;;	...
;;	60	...
;;	128	...
;;	...
;;	1	...
;;	...

;; Do 16 eight_reals_unfft operations
;; Do 240 four_complex_djbunfft operations
;;	distance between fft data elements is 16
;;	do 64 macros processing 32 data values

	mov	rcx, xsincos_complex	;; Prefetch complex sin/cos data, 136 cache lines
	yloop_init 64			;; 4 iterations of 16
	yloop_set_prefetch YOTHER, 8
	yr4_4cl_eight_reals_four_complex_djbunfft_preload
c3a:	yr4_4cl_eight_reals_four_complex_djbunfft rsi, dist64, 4*64, 8*64, rdi, 4*YMM_SCD3, 4
	yloop	4, c3a, -4*dist64+64		;; Test loop counter, next source pointer
	yloop	4, c3a, -4*64+16*64, rdi, -16*4*YMM_SCD3 ;; Test loop counter, next source pointer, restore sine/cosine pointer

	loops_reset			;; 3 more iters of 16
	yloop_set_prefetch YOTHER, 8
	yr4_rb4cl_four_complex_djbunfft_preload
c3b:	yr4_rb4cl_four_complex_djbunfft rsi, dist64, 4*64, 8*64, rdi, 4*YMM_SCD3, 4
	yloop	4, c3b, -4*dist64+64		;; Test loop counter, next source pointer
	yloop	4, c3b, -4*64+16*64, rdi, -16*4*YMM_SCD3 ;; Test loop counter, next source pointer, restore sine/cosine pointer
	yloop	4, c3b, -4*16*64		;; Test loop counter, restore source pointer

;; Do inverse FFT levels 3,4
;; On input the 64-byte cache lines hold these data values:
;;	0	+512	+512	+512	64	+512	+512	+512
;;	4	...
;;	...
;;	60	...
;;	128	...
;;	...
;;	1	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+512	+512	+512	256	+512	+512	+512
;;	4	...
;;	...
;;	252	...
;;	1	...
;;	...

;; Do 64 eight_reals_unfft operations
;; Do 192 four_complex_djbunfft operations
;;	distance between fft data elements is 64
;;	do 64 macros processing 32 data values

	yloop_init 64			;; 64 iterations
	yloop_set_prefetch YSINGLE
	yr4_4cl_eight_reals_four_complex_djbunfft_preload
c2b:	yr4_4cl_eight_reals_four_complex_djbunfft rsi, dist64, 16*64, 32*64, rdi, YMM_SCD3, 4
	yloop	4, c2b, -4*dist64+64		;; Test loop counter, next source pointer
	yloop	16, c2b, -16*64		;; Test loop counter, restore source pointer

;; Do inverse FFT levels 1,2,3 which correspond to complex levels 1,2
;; On input the 64-byte cache lines hold these data values:
;;	0	+512	+512	+512	256	+512	+512	+512
;;	4	...
;;	...
;;	252	...
;;	1	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	1K	+1	+1	+1
;;	4	...
;;	...
;;	1020

;; Do 256 eight_reals_unfft operations
;;	distance between fft data elements is 256
;;	do 64 macros processing 32 data values

	bump	rdi, -64*YMM_SCD3-64*YMM_SCD2 ;; Real sin/cos pointer
	mov	rbp, xsincos_complex	;; Complex sin/cos data
	bump	rcx, (136-8-64-64)*64	;; Move prefetch pointer along
	yloop_init 64			;; 64 iterations
	yloop_set_prefetch YSINGLE
	yr4_s4cl_2sc_eight_reals_unfft_preload
c1b:	yr4_s4cl_2sc_eight_reals_unfft rsi, 64, dist64, 2*dist64, rbp, YMM_SCD2, rdi, YMM_SCD2, 64
	yloop	64, c1b			;; Test loop counter

ypass2_10_levels_real_done:
	end_timer 2
	ENDM

; *************** radix4-pass2-10-levels-complex macro ******************
; This macro takes 1024 complex values and performs the final 10 levels of a standard
; mostly radix-4 FFT, squares the results, and does the first 10 levels of the inverse FFT.

yr4dwpn_pass2_10_levels_complex MACRO
	LOCAL	b1b, b2a, b2b, b3b, b4b, b5a, b5b, b5c, b5d, c1b, c2b, c3b, c4a, c4b
	LOCAL	ypass2_10_levels_complex_1, ypass2_10_levels_complex_2
	LOCAL	ypass2_10_levels_complex_3, ypass2_10_levels_complex_4
	LOCAL	ypass2_10_levels_complex_unfft, ypass2_10_levels_complex_done

;; Type 4 FFTs skip the forward FFT process

	mov	rsi, DATA_ADDR		;; Load address of FFT data
	mov	rbx, DIST_TO_FFTSRCARG
	mov	rcx, DATA_PREFETCH	;; Prefetch pointer (we need to prefetch 
					;; 256 cache lines + 2*3 pad cache lines)
	cmp	ffttype, 4
	je	ypass2_10_levels_complex_4

;; Do FFT levels 1,2
;; On input the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	1024	+1	+1	+1
;;	4	...
;;	...
;;	1020
;; On output the 64-byte cache lines are swizzled to hold these data values:
;;	0	+512	+512	+512	256	+512	+512	+512
;;	4	...
;;	...
;;	252	...
;;	1	...
;;	...

;; Do 256 four_complex_fft operations
;;	distance between fft data elements is 256
;;	do 64 macros each processing 32 data values

	start_timer 4
	mov	rdi, xsincos_complex	;; The sin/cos data
	yloop_init 64			;; 64 iterations
	yloop_set_prefetch YHALF
	yr4_fs4cl_four_complex_djbfft_preload
b1b:	yr4_fs4cl_four_complex_djbfft rsi, 64, dist64, 2*dist64, rdi, YMM_SCD2, 64
	yloop	64, b1b, -64*64		;; Test inner loop counter, restore source pointer
	end_timer 4

;; Do FFT levels 3,4
;; On input the 64-byte cache lines hold these data values:
;;	0	+512	+512	+512	256	+512	+512	+512
;;	4	...
;;	...
;;	252	...
;;	1	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+512	+512	+512	64	+512	+512	+512
;;	4	...
;;	...
;;	60	...
;;	128	...
;;	...
;;	1	...
;;	...
;;
;; Do 256 four_complex_fft operations
;;	distance between fft data elements is 64
;;	do 64 macros each processing 32 data values

;; Do FFT levels 5,6
;; On input the 64-byte cache lines hold these data values:
;;	0	+512	+512	+512	64	+512	+512	+512
;;	4	...
;;	...
;;	60	...
;;	128	...
;;	...
;;	1	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+512	+512	+512	16	+512	+512	+512
;;	4	...
;;	...
;;	12	...
;;	32	...
;;	...
;;	1	...
;;	...
;;
;; Do 256 four_complex_fft operations
;;	distance between fft data elements is 16
;;	do 64 macros each processing 32 data values

;; Do FFT levels 7,8
;; On input the 64-byte cache lines hold these data values:
;;	0	+512	+512	+512	16	+512	+512	+512
;;	4	...
;;	...
;;	12	...
;;	32	...
;;	...
;;	1	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+512	+512	+512	4	+512	+512	+512
;;	8	...
;;	...
;;	1	...
;;	...
;;
;; Do 256 four_complex_fft operations
;;	distance between fft data elements is 4
;;	do 64 macros each processing 32 data values

;; Do this 64 cache lines at a time to maximize accessing data
;; while it is in the L1 cache.

	mov	rbx, rdi
	mov	rbp, rdi
	yloop_init 64			;; 4 iters of 64 cache lines
	yloop_set_prefetch YHALF
	yr4_b4cl_four_complex_djbfft_preload

b2a:	start_timer 6
b2b:	yr4_b4cl_four_complex_djbfft rsi, 64, 16*64, 32*64, rdi, YMM_SCD2/4, 16
	yloop	16, b2b, -16*64		;; Loop 16 times, restore source pointer
	loops_undo 16			;; Back up loop counter
	end_timer 6

	start_timer 7
b3b:	yr4_b4cl_four_complex_djbfft rsi, 64, 4*64, 8*64, rbx, 4*YMM_SCD2/4, 4
	yloop	4, b3b, -4*64+16*64, rbx, -4*4*YMM_SCD2/4 ;; Loop 4 times, next source pointer, restore sine/cosine pointer
	yloop	4, b3b, -4*16*64, rbx, YMM_SCD2/4 ;; Loop 4 times, restore source pointer, next sine/cosine pointer
	loops_undo 4*4			;; Back up loop counter
	end_timer 7

	start_timer 8
b4b:	yr4_b4cl_four_complex_djbfft rsi, 4*64, 64, 2*64, rbp, 0, 16
	yloop	16, b4b, -16*4*64+dist64, rbp, 4*YMM_SCD2/4 ;; Loop 16 times, next source pointer, next sine/cosine pointer
	end_timer 8

	yloop	4, b2a, -4*dist64		;; Loop 4 times, restore source pointer

;; Do FFT levels 9,10 as well as inverse FFT levels 9,10
;; On input and output the 64-byte cache lines hold these data values:
;;	0	+512	+512	+512	4	+512	+512	+512
;;	8	...
;;	...
;;	1	...
;;	...

;; Do 256 four_complex_fft operations
;;	distance between fft data elements is 1
;;	do 64 macros each processing 32 data values

;; Execute the right middle step

	cmp	ffttype, 2
	jl	ypass2_10_levels_complex_1
	jg	ypass2_10_levels_complex_3

ypass2_10_levels_complex_2:
	start_timer 9
	yloop_init 64
	yloop_set_prefetch YHALF
	yr4_4cl_four_complex_with_square_preload
b5b:	yr4_4cl_four_complex_with_square rsi, 64, dist64, 2*dist64, 64
	yloop	64, b5b, -64*64		;; Loop 64 times, restore source pointer
	end_timer 9
	jmp	ypass2_10_levels_complex_unfft

ypass2_10_levels_complex_1:
	start_timer 9
;;	bump	rcx, (256+2*3-4*32-128)*64 ;; Move prefetch pointer along
	yloop_init 64
	yloop_set_prefetch YDOUBLE
b5a:	yr4_4cl_four_complex_fft_final rsi, 64, dist64, 2*dist64
	yloop	64, b5a			;; Loop 64 times
	end_timer 9
	jmp	ypass2_10_levels_complex_done

ypass2_10_levels_complex_3:
	start_timer 9
x64	mov	r8, DEST2ARG
	mov	rbp, DIST_TO_MULSRCARG
	yloop_init 64
	yloop_set_prefetch YHALF
	yr4_4cl_four_complex_with_mult_preload
b5c:	yr4_4cl_four_complex_with_mult rsi, 64, dist64, 2*dist64
	yloop	64, b5c, -64*64		;; Loop 64 times, restore source pointer
x64	add	rsi, r8
	end_timer 9
	jmp	ypass2_10_levels_complex_unfft

ypass2_10_levels_complex_4:
	start_timer 9
	mov	rbp, DIST_TO_MULSRCARG
	yloop_init 64
	yloop_set_prefetch YDOUBLE
	yr4_4cl_four_complex_with_mulf_preload
b5d:	yr4_4cl_four_complex_with_mulf rsi, 64, dist64, 2*dist64
	yloop	64, b5d, -64*64		;; Loop 64 times, restore source pointer
;;	bump	rcx, (-128+5*32)*64	;; Move prefetch pointer back to where squaring would be
	end_timer 9

;; Do inverse FFT levels 7,8
;; On input the 64-byte cache lines hold these data values:
;;	0	+512	+512	+512	4	+512	+512	+512
;;	8	...
;;	...
;;	1	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+512	+512	+512	16	+512	+512	+512
;;	4	...
;;	...
;;	12	...
;;	32	...
;;	...
;;	1	...
;;	...
;;
;; Do 256 four_complex_unfft operations
;;	distance between fft data elements is 4
;;	do 64 macros each processing 32 data values

;; Do inverse FFT levels 5,6
;; On input the 64-byte cache lines hold these data values:
;;	0	+512	+512	+512	16	+512	+512	+512
;;	4	...
;;	...
;;	12	...
;;	32	...
;;	...
;;	1	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+512	+512	+512	64	+512	+512	+512
;;	4	...
;;	...
;;	60	...
;;	128	...
;;	...
;;	1	...
;;	...
;;
;; Do 256 four_complex_unfft operations
;;	distance between fft data elements is 16
;;	do 64 macros each processing 32 data values

;; Do inverse FFT levels 3,4
;; On input the 64-byte cache lines hold these data values:
;;	0	+512	+512	+512	64	+512	+512	+512
;;	4	...
;;	...
;;	60	...
;;	128	...
;;	...
;;	1	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+512	+512	+512	256	+512	+512	+512
;;	4	...
;;	...
;;	252	...
;;	1	...
;;	...
;;
;; Do 256 four_complex_unfft operations
;;	distance between fft data elements is 64
;;	do 64 macros each processing 32 data values

;; Do this 64 cache lines at a time to maximize accessing data
;; while it is in the L1 cache.

ypass2_10_levels_complex_unfft:
	mov	rdi, xsincos_complex	;; Load sin/cos pointers
	bump	rdi, 64*YMM_SCD2
	mov	rbx, rdi
	mov	rbp, rdi
	yloop_init 64			;; 4 iters of 64 cache lines
	yloop_set_prefetch YHALF

c4a:	start_timer 10
	yr4_b4cl_four_complex_djbunfft_preload
c4b:	yr4_b4cl_four_complex_djbunfft rsi, 4*64, 64, 2*64, rbp, 0, 16
	yloop	16, c4b, -16*4*64	;; Loop 16 times, restore source pointer
	loops_undo 16			;; Restore loop counter
	end_timer 10

	start_timer 11
;;	yr4_b4cl_four_complex_djbunfft_preload
c3b:	yr4_b4cl_four_complex_djbunfft rsi, 64, 4*64, 8*64, rbx, 4*YMM_SCD2/4, 4
	yloop	4, c3b, -4*64+16*64, rbx, -4*4*YMM_SCD2/4 ;; Loop 4 times, next source pointer, restore sine/cosine pointer
	yloop	4, c3b, -4*16*64, rbx, YMM_SCD2/4 ;; Loop 4 times, restore source pointer, next sine/cosine pointer
	loops_undo 4*4			;; Restore loop counter
	end_timer 11

	start_timer 12
;;	yr4_b4cl_four_complex_djbunfft_preload
c2b:	yr4_b4cl_four_complex_djbunfft rsi, 64, 16*64, 32*64, rdi, YMM_SCD2/4, 16
	yloop	16, c2b, -16*64+dist64, rbp, 4*YMM_SCD2/4 ;; Loop 16 times, next source pointer, next sine/cosine pointer
	end_timer 12

	yloop	4, c4a, -4*dist64	;; Loop 4 times, restore source pointer

;; Do inverse FFT levels 1,2
;; On input the 64-byte cache lines hold these data values:
;;	0	+512	+512	+512	256	+512	+512	+512
;;	4	...
;;	...
;;	252	...
;;	1	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	1K	+1	+1	+1
;;	4	...
;;	...
;;	1020

;; Do 256 four_complex_unfft operations
;;	distance between fft data elements is 256
;;	do 64 macros each processing 32 data values

	start_timer 14
	mov	rdi, xsincos_complex	;; Load sin/cos pointer
	bump	rcx, (256+2*3-8*32-8)*64 ;; Move prefetch pointer along
	yloop_init 64			;; 64 iters
	yloop_set_prefetch YOTHER, 8	;; Prefetch 8 cache lines
	yr4_s4cl_four_complex_djbunfft_preload
c1b:	yr4_s4cl_four_complex_djbunfft rsi, 64, dist64, 2*dist64, rdi, YMM_SCD2, 64
	yloop	64, c1b			;; Test loop counter
	end_timer 14

ypass2_10_levels_complex_done:
	ENDM


; *************** radix4-pass2-1280-real macro ******************
; This macro takes 2560 real values and performs the final 11.322 levels of the FFT.
; The results are squared, and the first 11.322 levels of the inverse FFT are done.

yr4dwpn_pass2_1280_real MACRO
	LOCAL	b1b, b2a, b2b, b3a, b3b, b4a, b4b, b5a, b5b, b5c, b5d
	LOCAL	c1b, c2a, c2b, c3a, c3b, c4a, c4b
	LOCAL	ypass2_1280_real_1, ypass2_1280_real_2
	LOCAL	ypass2_1280_real_3, ypass2_1280_real_4
	LOCAL	ypass2_1280_real_unfft, ypass2_1280_real_done

;; Type 4 FFTs skip the forward FFT process

	start_timer 2
	mov	rsi, DATA_ADDR		;; Load source address
	mov	rbx, DIST_TO_FFTSRCARG
	mov	rcx, DATA_PREFETCH	;; Prefetch pointer (we need to prefetch 
					;; 320 cache lines + 2*4 pad cache lines)
	cmp	ffttype, 4
	je	ypass2_1280_real_4

;; Do FFT levels 1,2,3 which correspond to complex levels 1,2
;; On input the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	1280	+1	+1	+1
;;	4	...
;;	...
;;	1276
;; On output the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	256	+1	+1	+1
;;	4	...
;;	...
;;	252	...
;;	512	...
;;	...

;; Do 256 ten_reals_fft operations
;;	distance between fft data elements is 256
;;	do 64 macros each processing 40 data values

	mov	rdi, sincos3		;; Real sin/cos data
	mov	rbp, xsincos_complex	;; Complex sin/cos data
	yloop_init 64			;; 64 iterations
	yloop_set_prefetch YOTHER, 8
	yr5_f5cl_2sc_ten_reals_fft_preload
b1b:	yr5_f5cl_2sc_ten_reals_fft rsi, 64, dist64, rbp, YMM_SCD2, rdi, YMM_SCD2, 64
	yloop	64, b1b, -64*64	;; Test loop counter, restore source pointer

;; Do FFT levels 3,4
;; On input the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	256	+1	+1	+1
;;	4	...
;;	...
;;	252	...
;;	512	...
;;	...
;; On output the 64-byte cache lines are swizzled to hold these data values:
;;	0	+128	+128	+128	64	+128	+128	+128
;;	4	...
;;	...
;;	60	...
;;	1	...
;;	...
;;	512	...
;;	...

;; Do 64 eight_reals_fft operations
;; Do 256 four_complex_djbfft operations
;;	distance between fft data elements is 64
;;	do 80 macros each processing 32 data values

	yloop_init 80			;; 5 iterations of 16
	yloop_set_prefetch YSINGLE
	yr4_s4cl_2sc_eight_reals_fft_preload
b2a:	yr4_s4cl_2sc_eight_reals_fft rsi, 64, 16*64, 32*64, rbp, YMM_SCD2, rdi, YMM_SCD2, 16
	yloop	16, b2a, -16*64+dist64, rbp, -16*YMM_SCD2 ;; Test loop counter, next source pointer, restore sine/cosine pointer

	loops_reset			;; 4 more iters of 16
	yr4_s4cl_four_complex_djbfft_preload
b2b:	yr4_s4cl_four_complex_djbfft rsi, 64, 16*64, 32*64, rbp, YMM_SCD2, 16
	yloop	16, b2b, -16*64+dist64, rbp, -16*YMM_SCD2 ;; Test loop counter, next source pointer, restore sin/cos ptr
	yloop	5, b2b, -5*dist64	;; Loop 5 times, restore source pointer

;; Do FFT levels 5,6
;; On input the 64-byte cache lines hold these data values:
;;	0	+128	+128	+128	64	+128	+128	+128
;;	4	...
;;	...
;;	60	...
;;	1	...
;;	...
;;	512	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+128	+128	+128	16	+128	+128	+128
;;	4	...
;;	...
;;	12	...
;;	32	...
;;	...
;;	1	...
;;	...
;;	512	...
;;	...

;; Do 16 eight_reals_fft operations
;; Do 304 four_complex_djbfft operations
;;	distance between fft data elements is 16
;;	do 80 macros each processing 32 data values

	yloop_init 80			;; 5 iterations of 16
	yloop_set_prefetch YSINGLE
	yr4_4cl_eight_reals_four_complex_djbfft_preload
b3a:	yr4_4cl_eight_reals_four_complex_djbfft rsi, 16*64, 4*64, 8*64, rdi, YMM_SCD3, 4
	yloop	4, b3a, -4*16*64+64	;; Test loop counter, next source pointer
	yloop	4, b3a, -4*64+dist64, rdi, -16*YMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer

	loops_reset			;; 4 more iters of 16
	yr4_rb4cl_four_complex_djbfft_preload
b3b:	yr4_rb4cl_four_complex_djbfft rsi, 16*64, 4*64, 8*64, rdi, YMM_SCD3, 4
	yloop	4, b3b, -4*16*64+64	;; Test loop counter, next source pointer
	yloop	4, b3b, -4*64+dist64, rdi, -16*YMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer
	yloop	5, b3b, -5*dist64	;; Test loop counter, restore source pointer

;; Do FFT levels 7,8
;; On input the 64-byte cache lines hold these data values:
;;	0	+128	+128	+128	16	+128	+128	+128
;;	4	...
;;	...
;;	12	...
;;	32	...
;;	...
;;	1	...
;;	...
;;	512	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+128	+128	+128	4	+128	+128	+128
;;	8	...
;;	...
;;	1	...
;;	...
;;	512	...
;;	...

;; Do 4 eight_reals_fft operations
;; Do 316 four_complex_djbfft operations
;;	distance between fft data elements is 4
;;	do 80 macros each processing 32 data values

	yloop_init 80			;; 20 iterations of 4
	yloop_set_prefetch YSINGLE
	yr4_4cl_eight_reals_four_complex_djbfft_preload
b4a:	yr4_4cl_eight_reals_four_complex_djbfft rsi, 16*64, 64, 2*64, rdi, 4*YMM_SCD3, 4
	yloop	4, b4a, -4*16*64+4*64, rdi, -4*4*YMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer

	loops_reset			;; 19 more iters of 4
	yr4_rb4cl_four_complex_djbfft_preload
b4b:	yr4_rb4cl_four_complex_djbfft rsi, 16*64, 64, 2*64, rdi, 4*YMM_SCD3, 4
	yloop	4, b4b, -4*16*64+4*64, rdi, -4*4*YMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer
	yloop	4, b4b, -4*4*64+dist64	;; Test loop counter, next source pointer
	yloop	5, b4b, -5*dist64	;; Test loop counter, restore source pointer

;; Do FFT levels 9,10
;; On input and output the 64-byte cache lines hold these data values:
;;	0	+128	+128	+128	4	+128	+128	+128
;;	8	...
;;	...
;;	1	...
;;	...
;;	512	...
;;	...

;; Do 1 eight_reals_fft operations
;; Do 319 four_complex_fft operations
;;	distance between fft data elements is 1
;;	do 80 macros each processing 32 data values

;; Execute the proper middle step

	cmp	ffttype, 2
	jl	ypass2_1280_real_1
	jg	ypass2_1280_real_3

ypass2_1280_real_2:
;;	yr4_4cl_eight_reals_four_complex_with_square rsi, 64, 16*64, 32*64
	yr4_dispatch call, yreal_square, 16*64, 32*64
	yloop_init 80			;; 79 more iterations
	yloop_adjust 1
	yloop_set_prefetch YHALF
	yr4_4cl_four_complex_with_square_preload
b5b:	yr4_4cl_four_complex_with_square rsi, 64, 16*64, 32*64, 1
	yloop	16, b5b, -16*64+dist64	;; Loop 16 times, next source pointer
	yloop	5, b5b, -5*dist64	;; Loop 5 times, restore source pointer
	jmp	ypass2_1280_real_unfft

ypass2_1280_real_1:
;;	yr4_4cl_eight_reals_four_complex_fft_final rsi, 64, 16*64, 32*64
	yr4_dispatch call, yreal_fft_final, 16*64, 32*64
	mov	rcx, xsincos_complex	;; Prefetch complex sin/cos data, 162 cache lines
	yloop_init 80			;; 79 more iterations
	yloop_adjust 1
	yloop_set_prefetch YDOUBLE
b5a:	yr4_4cl_four_complex_fft_final rsi, 64, 16*64, 32*64
	yloop	16, b5a, -16*64+dist64	;; Loop 16 times, next source pointer
	yloop	5, b5a			;; Loop 5 times
	jmp	ypass2_1280_real_done

ypass2_1280_real_3:
x64	mov	r8, DEST2ARG
	mov	rbp, DIST_TO_MULSRCARG
;;	yr4_4cl_eight_reals_four_complex_with_mult rsi, 64, 16*64, 32*64
	yr4_dispatch call, yreal_mult, 16*64, 32*64
	yloop_init 80			;; 79 more iterations
	yloop_adjust 1
	yloop_set_prefetch YHALF
	yr4_4cl_four_complex_with_mult_preload
b5c:	yr4_4cl_four_complex_with_mult rsi, 64, 16*64, 32*64
	yloop	16, b5c, -16*64+dist64	;; Loop 16 times, next source pointer
	yloop	5, b5c, -5*dist64	;; Loop 5 times, restore source pointer
x64	add	rsi, r8
	jmp	ypass2_1280_real_unfft

ypass2_1280_real_4:
	mov	rbp, DIST_TO_MULSRCARG
;;	yr4_4cl_eight_reals_four_complex_with_mulf rsi, 64, 16*64, 32*64
	yr4_dispatch call, yreal_mulf, 16*64, 32*64
	yloop_init 80			;; 79 more iterations
	yloop_adjust 1
	yloop_set_prefetch YDOUBLE
	yr4_4cl_four_complex_with_mulf_preload
b5d:	yr4_4cl_four_complex_with_mulf rsi, 64, 16*64, 32*64
	yloop	16, b5d, -16*64+dist64	;; Loop 16 times, next source pointer
	yloop	5, b5d, -5*dist64	;; Loop 5 times, restore source pointer
;;	bump	rcx, (-128+8+3*80+40)*64 ;; Move prefetch pointer back to where squaring would be

;; Do inverse FFT levels 7,8
;; On input the 64-byte cache lines hold these data values:
;;	0	+128	+128	+128	4	+128	+128	+128
;;	8	...
;;	...
;;	1	...
;;	...
;;	512	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+128	+128	+128	16	+128	+128	+128
;;	4	...
;;	...
;;	12	...
;;	32	...
;;	...
;;	1	...
;;	...
;;	512	...
;;	...

;; Do 4 eight_reals_unfft operations
;; Do 316 four_complex_djbunfft operations
;;	distance between fft data elements is 4
;;	do 80 macros processing 32 data values

ypass2_1280_real_unfft:
	mov	rdi, sincos3		;; Load sin/cos pointer
	bump	rdi, 16*YMM_SCD2+64*YMM_SCD2
;;	bump	rcx, (320+2*4-8-3*80-40-40)*64 ;; Move prefetch pointer back to where squaring would be
	yloop_init 80			;; 20 iterations of 4
	yloop_set_prefetch YSINGLE
	yr4_4cl_eight_reals_four_complex_djbunfft_preload
c4a:	yr4_4cl_eight_reals_four_complex_djbunfft rsi, 16*64, 64, 2*64, rdi, 4*YMM_SCD3, 4
	yloop	4, c4a, -4*16*64+4*64, rdi, -4*4*YMM_SCD3 ;; Test loop counter, next source pointer, restore sine/cosine pointer

	loops_reset			;; 19 more iters of 4
	yr4_rb4cl_four_complex_djbunfft_preload
c4b:	yr4_rb4cl_four_complex_djbunfft rsi, 16*64, 64, 2*64, rdi, 4*YMM_SCD3, 4
	yloop	4, c4b, -4*16*64+4*64, rdi, -4*4*YMM_SCD3 ;; Test loop counter, next source pointer, restore sine/cosine pointer
	yloop	4, c4b, -4*4*64+dist64	;; Test loop counter, next source pointer
	yloop	5, c4b, -5*dist64	;; Test loop counter, restore source pointer

;; Do inverse FFT levels 5,6
;; On input the 64-byte cache lines hold these data values:
;;	0	+128	+128	+128	16	+128	+128	+128
;;	4	...
;;	...
;;	12	...
;;	32	...
;;	...
;;	1	...
;;	...
;;	512	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+128	+128	+128	64	+128	+128	+128
;;	4	...
;;	...
;;	60	...
;;	1	...
;;	...
;;	512	...
;;	...

;; Do 16 eight_reals_unfft operations
;; Do 304 four_complex_djbunfft operations
;;	distance between fft data elements is 16
;;	do 80 macros processing 32 data values

	mov	rcx, xsincos_complex	;; Prefetch complex sin/cos data, 162 cache lines
	yloop_init 80			;; 5 iterations of 16
	yloop_set_prefetch YQUARTER
	yr4_4cl_eight_reals_four_complex_djbunfft_preload
c3a:	yr4_4cl_eight_reals_four_complex_djbunfft rsi, 16*64, 4*64, 8*64, rdi, YMM_SCD3, 4
	yloop	4, c3a, -4*16*64+64	;; Test loop counter, next source pointer
	yloop	4, c3a, -4*64+dist64, rdi, -16*YMM_SCD3 ;; Test loop counter, next source pointer, restore sine/cosine pointer

	loops_reset			;; 4 more iters of 16
	yr4_rb4cl_four_complex_djbunfft_preload
c3b:	yr4_rb4cl_four_complex_djbunfft rsi, 16*64, 4*64, 8*64, rdi, YMM_SCD3, 4
	yloop	4, c3b, -4*16*64+64	;; Test loop counter, next source pointer
	yloop	4, c3b, -4*64+dist64, rdi, -16*YMM_SCD3 ;; Test loop counter, next source pointer, restore sine/cosine pointer
	yloop	5, c3b, -5*dist64	;; Test loop counter, restore source pointer

;; Do inverse FFT levels 3,4
;; On input the 64-byte cache lines hold these data values:
;;	0	+128	+128	+128	64	+128	+128	+128
;;	4	...
;;	...
;;	60	...
;;	1	...
;;	...
;;	512	...
;;	...
;; On output the 64-byte cache lines are swizzled to hold these data values:
;;	0	+1	+1	+1	256	+1	+1	+1
;;	4	...
;;	...
;;	252	...
;;	512	...
;;	...

;; Do 64 eight_reals_unfft operations
;; Do 256 four_complex_djbunfft operations
;;	distance between fft data elements is 64
;;	do 80 macros processing 32 data values

	bump	rdi, -16*YMM_SCD2	;; Real sin/cos ptr
	mov	rbp, xsincos_complex	;; Complex sin/cos data
	bump	rbp, 64*YMM_SCD2
	yloop_init 80			;; 5 iterations of 16
	yloop_set_prefetch YSINGLE
	yr4_s4cl_2sc_eight_reals_unfft_preload
c2a:	yr4_s4cl_2sc_eight_reals_unfft rsi, 64, 16*64, 32*64, rbp, YMM_SCD2, rdi, YMM_SCD2, 16
	yloop	16, c2a, -16*64+dist64, rbp, -16*YMM_SCD2 ;; Test loop counter, next source pointer, restore sine/cosine pointer

	loops_reset			;; 4 more iters of 16
	yr4_s4cl_four_complex_djbunfft_preload
c2b:	yr4_s4cl_four_complex_djbunfft rsi, 64, 16*64, 32*64, rbp, YMM_SCD2, 16
	yloop	16, c2b, -16*64+dist64, rbp, -16*YMM_SCD2 ;; Test loop counter, next source pointer, restore sine/cosine pointer
	yloop	5, c2b, -5*dist64	;; Test loop counter, restore source pointer

;; Do inverse FFT levels 1,2,3 which correspond to complex levels 1,2
;; On input the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	256	+1	+1	+1
;;	4	...
;;	...
;;	252	...
;;	512	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	1280	+1	+1	+1
;;	4	...
;;	...
;;	1276

;; Do 256 ten_reals_unfft operations
;;	distance between fft data elements is 256
;;	do 64 macros processing 40 data values

	bump	rdi, -16*YMM_SCD2-64*YMM_SCD2 ;; Real sin/cos ptr
	bump	rbp, -64*YMM_SCD2	;; Complex sin/cos data
	bump	rcx, (162-20-80-64)*64	;; Move prefetch pointer along
	yloop_init 64			;; 64 iterations
	yloop_set_prefetch YSINGLE
	yr5_5cl_2sc_ten_reals_unfft_preload
c1b:	yr5_5cl_2sc_ten_reals_unfft rsi, 64, dist64, rbp, YMM_SCD2, rdi, YMM_SCD2, 64
	yloop	64, c1b			;; Test loop counter

ypass2_1280_real_done:
	end_timer 2
	ENDM

; *************** radix4-pass2-1280-complex macro ******************
; This macro takes 1280 complex values and performs the final 10.322 levels of a standard
; mostly radix-4 FFT, squares the results, and does the first 10.322 levels of the inverse FFT.

yr4dwpn_pass2_1280_complex MACRO
	LOCAL	b1b, b2b, b3b, b4b, b5a, b5b, b5c, b5d, c1b, c2b, c3b, c4b
	LOCAL	ypass2_1280_complex_1, ypass2_1280_complex_2
	LOCAL	ypass2_1280_complex_3, ypass2_1280_complex_4
	LOCAL	ypass2_1280_complex_unfft, ypass2_1280_complex_done

;; Type 4 FFTs skip the forward FFT process

	mov	rsi, DATA_ADDR		;; Load address of FFT data
	mov	rbx, DIST_TO_FFTSRCARG
	mov	rcx, DATA_PREFETCH	;; Prefetch pointer (we need to prefetch 
					;; 320 cache lines + 2*4 pad cache lines)
	cmp	ffttype, 4
	je	ypass2_1280_complex_4

;; Do FFT levels 1,2 (actually 2.322 levels)
;; On input the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	1280	+1	+1	+1
;;	4	...
;;	...
;;	1276
;; On output the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	256	+1	+1	+1
;;	4	...
;;	...
;;	252	...
;;	512	...
;;	...

;; Do 256 five_complex_fft operations
;;	distance between fft data elements is 256
;;	do 64 macros each processing 40 data values

	start_timer 4
	mov	rdi, xsincos_complex	;; The sin/cos data
	yloop_init 64			;; 64 iterations
	yloop_set_prefetch YQUARTER
	yr5_f5cl_five_complex_djbfft_preload
b1b:	yr5_f5cl_five_complex_djbfft rsi, 64, dist64, rdi, YMM_SCD2, 64
	yloop	64, b1b, -64*64	;; Test inner loop counter, restore source pointer
	end_timer 4

;; Do FFT levels 3,4
;; On input the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	256	+1	+1	+1
;;	4	...
;;	...
;;	252	...
;;	512	...
;;	...
;; On output the 64-byte cache lines are swizzled to hold these data values:
;;	0	+128	+128	+128	64	+128	+128	+128
;;	4	...
;;	...
;;	60	...
;;	1	...
;;	...
;;	512	...
;;	...

;; Do 320 four_complex_fft operations
;;	distance between fft data elements is 64
;;	do 80 macros each processing 32 data values

	start_timer 6
	yloop_init 80			;; 5 iters of 16
	yloop_set_prefetch YHALF
	yr4_s4cl_four_complex_djbfft_preload
b2b:	yr4_s4cl_four_complex_djbfft rsi, 64, 16*64, 32*64, rdi, YMM_SCD2, 16
	yloop	16, b2b, -16*64+dist64, rdi, -16*YMM_SCD2 ;; Loop 16 times, next source pointer, restore sine/cosine pointer
	yloop	5, b2b, -5*dist64, rdi, 16*YMM_SCD2 ;; Loop 5 times, restore source pointer, next sine/cosine pointer
	end_timer 6

;; Do FFT levels 5,6
;; On input the 64-byte cache lines hold these data values:
;;	0	+128	+128	+128	64	+128	+128	+128
;;	4	...
;;	...
;;	60	...
;;	1	...
;;	...
;;	512	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+128	+128	+128	16	+128	+128	+128
;;	4	...
;;	...
;;	12	...
;;	32	...
;;	...
;;	1	...
;;	...
;;	512	...
;;	...

;; Do 320 four_complex_fft operations
;;	distance between fft data elements is 16
;;	do 80 macros each processing 32 data values

	start_timer 7
	yloop_init 80			;; 5 iters of 16
	yloop_set_prefetch YHALF
	yr4_b4cl_four_complex_djbfft_preload
b3b:	yr4_b4cl_four_complex_djbfft rsi, 64, 4*64, 8*64, rdi, YMM_SCD2/4, 4
	yloop	4, b3b, -4*64+16*64	;; Loop 4 times, next source pointer
	yloop	4, b3b, -4*16*64+dist64, rdi, -16*YMM_SCD2/4 ;; Loop 4 times, next source pointer, restore sine/cosine pointer
	yloop	5, b3b, -5*dist64	;; Loop 5 times, restore source pointer
	end_timer 7

;; Do FFT levels 7,8
;; On input the 64-byte cache lines hold these data values:
;;	0	+128	+128	+128	16	+128	+128	+128
;;	4	...
;;	...
;;	12	...
;;	32	...
;;	...
;;	1	...
;;	...
;;	512	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+128	+128	+128	4	+128	+128	+128
;;	8	...
;;	...
;;	1	...
;;	...
;;	512	...
;;	...

;; Do 320 four_complex_fft operations
;;	distance between fft data elements is 4
;;	do 80 macros each processing 32 data values

	start_timer 8
	yloop_init 80			;; 20 iters of 4
	yloop_set_prefetch YHALF
	yr4_b4cl_four_complex_djbfft_preload
b4b:	yr4_b4cl_four_complex_djbfft rsi, 4*64, 64, 2*64, rdi, 0, 4
	yloop	4, b4b, 0, rdi, YMM_SCD2/4 ;; Loop 4 times, next sine/cosine pointer
	yloop	4, b4b, -16*4*64+dist64, rdi, -4*YMM_SCD2/4 ;; Loop 4 times, next source pointer, restore sine/cosine pointer
	yloop	5, b4b, -5*dist64	;; Loop 5 times, restore source pointer
	end_timer 8

;; Do FFT levels 9,10 as well as inverse FFT levels 9,10
;; On input and output the 64-byte cache lines hold these data values:
;;	0	+128	+128	+128	4	+128	+128	+128
;;	8	...
;;	...
;;	1	...
;;	...
;;	512	...
;;	...

;; Do 320 four_complex_fft operations
;;	distance between fft data elements is 1
;;	do 80 macros each processing 32 data values

;; Execute the right middle step

	cmp	ffttype, 2
	jl	ypass2_1280_complex_1
	jg	ypass2_1280_complex_3

ypass2_1280_complex_2:
	start_timer 9
	yloop_init 80
	yloop_set_prefetch YHALF
	yr4_4cl_four_complex_with_square_preload
b5b:	yr4_4cl_four_complex_with_square rsi, 64, 16*64, 32*64, 16
	yloop	16, b5b, -16*64+dist64	;; Loop 16 times, next source pointer
	yloop	5, b5b, -5*dist64	;; Loop 5 times, restore source pointer
	end_timer 9
	jmp	ypass2_1280_complex_unfft

ypass2_1280_complex_1:
	start_timer 9
;;	bump	rcx, (320+4-16-3*40-160)*64 ;; Move prefetch pointer along
	yloop_init 80
	yloop_set_prefetch YDOUBLE
b5a:	yr4_4cl_four_complex_fft_final rsi, 64, 16*64, 32*64
	yloop	16, b5a, -16*64+dist64	;; Loop 16 times, next source pointer
	yloop	5, b5a			;; Loop 5 times
	end_timer 9
	jmp	ypass2_1280_complex_done

ypass2_1280_complex_3:
	start_timer 9
x64	mov	r8, DEST2ARG
	mov	rbp, DIST_TO_MULSRCARG
	yloop_init 80
	yloop_set_prefetch YHALF
	yr4_4cl_four_complex_with_mult_preload
b5c:	yr4_4cl_four_complex_with_mult rsi, 64, 16*64, 32*64
	yloop	16, b5c, -16*64+dist64	;; Loop 16 times, next source pointer
	yloop	5, b5c, -5*dist64	;; Loop 5 times, restore source pointer
x64	add	rsi, r8
	end_timer 9
	jmp	ypass2_1280_complex_unfft

ypass2_1280_complex_4:
	start_timer 9
	mov	rbp, DIST_TO_MULSRCARG
	yloop_init 80
	yloop_set_prefetch YDOUBLE
	yr4_4cl_four_complex_with_mulf_preload
b5d:	yr4_4cl_four_complex_with_mulf rsi, 64, 16*64, 32*64
	yloop	16, b5d, -16*64+dist64	;; Loop 16 times, next source pointer
	yloop	5, b5d, -5*dist64	;; Loop 5 times, restore source pointer
;;	bump	rcx, (-160+16+4*40)*64	;; Move prefetch pointer to where squaring would be
	end_timer 9

;; Do inverse FFT levels 7,8
;; On input the 64-byte cache lines hold these data values:
;;	0	+128	+128	+128	4	+128	+128	+128
;;	8	...
;;	...
;;	1	...
;;	...
;;	512	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+128	+128	+128	16	+128	+128	+128
;;	4	...
;;	...
;;	12	...
;;	32	...
;;	...
;;	1	...
;;	...
;;	512	...
;;	...

;; Do 320 four_complex_unfft operations
;;	distance between fft data elements is 4
;;	do 80 macros each processing 32 data values

ypass2_1280_complex_unfft:
	start_timer 10
	mov	rdi, xsincos_complex	;; Load sin/cos pointers
	bump	rdi, 64*YMM_SCD2+16*YMM_SCD2
	yloop_init 80			;; 20 iters of 4
	yloop_set_prefetch YHALF
	yr4_b4cl_four_complex_djbunfft_preload
c4b:	yr4_b4cl_four_complex_djbunfft rsi, 4*64, 64, 2*64, rdi, 0, 4
	yloop	4, c4b, 0, rdi, YMM_SCD2/4 ;; Loop 4 times, next sine/cosine pointer
	yloop	4, c4b, -16*4*64+dist64, rdi, -4*YMM_SCD2/4 ;; Loop 4 times, next source pointer, restore sine/cosine pointer
	yloop	5, c4b, -5*dist64	;; Loop 5 times, restore source pointer
	end_timer 10

;; Do inverse FFT levels 5,6
;; On input the 64-byte cache lines hold these data values:
;;	0	+128	+128	+128	16	+128	+128	+128
;;	4	...
;;	...
;;	12	...
;;	32	...
;;	...
;;	1	...
;;	...
;;	512	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+128	+128	+128	64	+128	+128	+128
;;	4	...
;;	...
;;	60	...
;;	1	...
;;	...
;;	512	...
;;	...

;; Do 320 four_complex_unfft operations
;;	distance between fft data elements is 16
;;	do 80 macros each processing 32 data values

	start_timer 11
	yloop_init 80			;; 5 iters of 16
	yloop_set_prefetch YHALF
	yr4_b4cl_four_complex_djbunfft_preload
c3b:	yr4_b4cl_four_complex_djbunfft rsi, 64, 4*64, 8*64, rdi, YMM_SCD2/4, 4
	yloop	4, c3b, -4*64+16*64	;; Loop 4 times, next source pointer
	yloop	4, c3b, -4*16*64+dist64, rdi, -16*YMM_SCD2/4 ;; Loop 4 times, next source pointer, restore sine/cosine pointer
	yloop	5, c3b, -5*dist64	;; Loop 5 times, restore source pointer
	end_timer 11

;; Do inverse FFT levels 3,4
;; On input the 64-byte cache lines hold these data values:
;;	0	+128	+128	+128	64	+128	+128	+128
;;	4	...
;;	...
;;	60	...
;;	1	...
;;	...
;;	512	...
;;	...
;; On output the 64-byte cache lines are swizzled to hold these data values:
;;	0	+1	+1	+1	256	+1	+1	+1
;;	4	...
;;	...
;;	252	...
;;	512	...
;;	...

;; Do 320 four_complex_unfft operations
;;	distance between fft data elements is 64
;;	do 80 macros each processing 32 data values

	start_timer 12
	bump	rdi, -16*YMM_SCD2	;; Load sin/cos ptr
	yloop_init 80			;; 5 iters of 16
	yloop_set_prefetch YHALF
	yr4_s4cl_four_complex_djbunfft_preload
c2b:	yr4_s4cl_four_complex_djbunfft rsi, 64, 16*64, 32*64, rdi, YMM_SCD2, 16
	yloop	16, c2b, -16*64+dist64, rdi, -16*YMM_SCD2 ;; Loop 16 times, next source pointer, restore sine/cosine pointer
	yloop	5, c2b, -5*dist64	;; Loop 5 times, restore source pointer
	end_timer 12

;; Do inverse FFT levels 1,2
;; On input the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	256	+1	+1	+1
;;	4	...
;;	...
;;	252	...
;;	512	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	1280	+1	+1	+1
;;	4	...
;;	...
;;	1276

;; Do 256 five_complex_unfft operations
;;	distance between fft data elements is 256
;;	do 64 macros each processing 40 data values

	start_timer 14
	bump	rdi, -64*YMM_SCD2	;; Load sin/cos ptr
	bump	rcx, (320+2*4-16-7*40-32)*64 ;; Move prefetch pointer along
	yloop_init 64			;; 64 iters
	yloop_set_prefetch YHALF
	yr5_5cl_five_complex_djbunfft_preload
c1b:	yr5_5cl_five_complex_djbunfft rsi, 64, dist64, rdi, YMM_SCD2, 64
	yloop	64, c1b			;; Test loop counter
	end_timer 14

ypass2_1280_complex_done:
	ENDM


;*********************************************************************************************
;			Macros derived from pass2-11-levels
;*********************************************************************************************


; *************** radix4-pass2-1536-real macro ******************
; This macro takes 3072 real values and performs the final 11.585 levels of the FFT.
; The results are squared, and the first 11.585 levels of the inverse FFT are done.

yr4dwpn_pass2_1536_real MACRO
	LOCAL	b1b, b2a, b2b, b3a, b3b, b4a, b4b, b5a, b5b, b5c, b5d
	LOCAL	c1b, c2a, c2b, c3a, c3b, c4a, c4b
	LOCAL	ypass2_1536_real_1, ypass2_1536_real_2
	LOCAL	ypass2_1536_real_3, ypass2_1536_real_4
	LOCAL	ypass2_1536_real_unfft, ypass2_1536_real_done

;; Type 4 FFTs skip the forward FFT process

	start_timer 2
	mov	rsi, DATA_ADDR		;; Load source address
	mov	rbx, DIST_TO_FFTSRCARG
	mov	rcx, DATA_PREFETCH	;; Prefetch pointer (we need to prefetch 
					;; 384 cache lines + 2*5 pad cache lines)
	cmp	ffttype, 4
	je	ypass2_1536_real_4

;; Do FFT levels 1,2,3 which correspond to complex levels 1,2
;; On input the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	1536	+1	+1	+1
;;	4	...
;;	...
;;	1532
;; On output the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	512	+1	+1	+1
;;	4	...
;;	...
;;	508	...
;;	1024	...
;;	...

;; Do 512 six_reals_fft operations
;;	distance between fft data elements is 512
;;	do 128 macros each processing 24 data values

	mov	rdi, sincos3		;; Real sin/cos data
	mov	rbp, xsincos_complex	;; Complex sin/cos data
	yloop_init 128			;; 128 iterations
	yloop_set_prefetch YHALF
	yr3_f3cl_2sc_six_reals_fft_preload
b1b:	yr3_f3cl_2sc_six_reals_fft rsi, 64, 2*dist64, rbp, YMM_SCD1, rdi, YMM_SCD1, 64
	yloop	64, b1b, -64*64+dist64	;; Test loop counter, next source pointer
	yloop	2, b1b, -2*dist64	;; Test loop counter, restore source pointer

;; Do FFT levels 3,4
;; On input the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	512	+1	+1	+1
;;	4	...
;;	...
;;	508	...
;;	1024	...
;;	...
;; On output the 64-byte cache lines are swizzled to hold these data values:
;;	0	+256	+256	+256	128	+256	+256	+256
;;	4	...
;;	...
;;	124	...
;;	1	...
;;	...
;;	1024	...
;;	...

;; Do 128 eight_reals_fft operations
;; Do 256 four_complex_djbfft operations
;;	distance between fft data elements is 128
;;	do 96 macros each processing 32 data values

	yloop_init 96			;; 3 iters of 32
	yloop_set_prefetch YSINGLE
	yr4_s4cl_2sc_eight_reals_fft_preload
b2a:	yr4_s4cl_2sc_eight_reals_fft rsi, 64, 32*64, dist64, rbp, YMM_SCD2, rdi, YMM_SCD2, 32
	yloop	32, b2a, -32*64+2*dist64, rbp, -32*YMM_SCD2 ;; Test loop counter, next source pointer, restore sine/cosine pointer

	loops_reset			;; 2 more iters of 32
	yr4_s4cl_four_complex_djbfft_preload
b2b:	yr4_s4cl_four_complex_djbfft rsi, 64, 32*64, dist64, rbp, YMM_SCD2, 32
	yloop	32, b2b, -32*64+2*dist64, rbp, -32*YMM_SCD2 ;; Test loop counter, next source pointer, restore sin/cos ptr
	yloop	3, b2b, -3*2*dist64	;; Loop 3 times, restore source pointer

;; Do FFT levels 5,6
;; On input the 64-byte cache lines hold these data values:
;;	0	+256	+256	+256	128	+256	+256	+256
;;	4	...
;;	...
;;	124	...
;;	1	...
;;	...
;;	1024	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+256	+256	+256	32	+256	+256	+256
;;	4	...
;;	...
;;	28	...
;;	64	...
;;	...
;;	1	...
;;	...
;;	1024	...
;;	...

;; Do 32 eight_reals_fft operations
;; Do 352 four_complex_djbfft operations
;;	distance between fft data elements is 32
;;	do 96 macros each processing 32 data values

	yloop_init 96			;; 3 iterations of 32
	yloop_set_prefetch YHALF
	yr4_4cl_eight_reals_four_complex_djbfft_preload
b3a:	yr4_4cl_eight_reals_four_complex_djbfft rsi, 32*64, 8*64, 16*64, rdi, YMM_SCD3, 2
	yloop	2, b3a, -2*32*64+dist64	;; Test loop counter, next source pointer
	yloop	2, b3a, -2*dist64+64	;; Test loop counter, next source pointer
	yloop	8, b3a, -8*64+2*dist64, rdi, -32*YMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer

	loops_reset			;; 2 more iters of 32
	yr4_rb4cl_four_complex_djbfft_preload
b3b:	yr4_rb4cl_four_complex_djbfft rsi, 32*64, 8*64, 16*64, rdi, YMM_SCD3, 2
	yloop	2, b3b, -2*32*64+dist64	;; Test loop counter, next source pointer
	yloop	2, b3b, -2*dist64+64	;; Test loop counter, next source pointer
	yloop	8, b3b, -8*64+2*dist64, rdi, -32*YMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer
	yloop	3, b3b, -3*2*dist64	;; Test loop counter, restore source pointer

;; Do FFT levels 7,8
;; On input the 64-byte cache lines hold these data values:
;;	0	+256	+256	+256	32	+256	+256	+256
;;	4	...
;;	...
;;	28	...
;;	64	...
;;	...
;;	1	...
;;	...
;;	1024	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+256	+256	+256	8	+256	+256	+256
;;	4	...
;;	16
;;	...
;;	1	...
;;	...
;;	1024	...
;;	...

;; Do 8 eight_reals_fft operations
;; Do 376 four_complex_djbfft operations
;;	distance between fft data elements is 8
;;	do 96 macros each processing 32 data values

	yloop_init 96			;; 12 iterations of 8
	yloop_set_prefetch YHALF
	yr4_4cl_eight_reals_four_complex_djbfft_preload
b4a:	yr4_4cl_eight_reals_four_complex_djbfft rsi, 32*64, 2*64, 4*64, rdi, 4*YMM_SCD3, 2
	yloop	2, b4a, -2*32*64+dist64	;; Test loop counter, next source pointer
	yloop	2, b4a, -2*dist64+64	;; Test loop counter, next source pointer
	yloop	2, b4a, -2*64+8*64, rdi, -8*4*YMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer

	loops_reset			;; 11 more iters of 8
	yr4_rb4cl_four_complex_djbfft_preload
b4b:	yr4_rb4cl_four_complex_djbfft rsi, 32*64, 2*64, 4*64, rdi, 4*YMM_SCD3, 2
	yloop	2, b4b, -2*32*64+dist64	;; Test loop counter, next source pointer
	yloop	2, b4b, -2*dist64+64	;; Test loop counter, next source pointer
	yloop	2, b4b, -2*64+8*64, rdi, -8*4*YMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer
	yloop	4, b4b, -4*8*64+2*dist64 ;; Test loop counter, next source pointer
	yloop	3, b4b, -3*2*dist64	;; Test loop counter, restore source pointer

;; Do FFT levels 9,10,11 as well as inverse FFT levels 9,10,11
;; On input and output the 64-byte cache lines hold these data values:
;;	0	+256	+256	+256	8	+256	+256	+256
;;	4	...
;;	16
;;	...
;;	1	...
;;	...
;;	1024	...
;;	...

;; Do 1 sixteen_reals_fft operations
;; Do 192 eight_complex_fft operations
;;	distance between fft data elements is 1
;;	do 48 macros each processing 64 data values

;; Execute the proper middle step

	cmp	ffttype, 2
	jl	ypass2_1536_real_1
	jg	ypass2_1536_real_3

ypass2_1536_real_2:
;;	yr8_8cl_sixteen_reals_eight_complex_with_square rsi, 2*64, 32*64, dist64, 64
	yr8_dispatch call, y8real_square, 32*64, dist64, 64
	yloop_init 48			;; 47 more iterations
	yloop_adjust 1
	yloop_set_prefetch YSINGLE
	yr8_8cl_eight_complex_with_square_preload
b5b:	yr8_8cl_eight_complex_with_square rsi, 2*64, 32*64, dist64, 64, 1
	yloop	16, b5b, -16*2*64+2*dist64	;; Loop 16 times, next source pointer
	yloop	3, b5b, -3*2*dist64	;; Loop 3 times, restore source pointer
	jmp	ypass2_1536_real_unfft

ypass2_1536_real_1:
;;	yr8_8cl_sixteen_reals_eight_complex_fft_final rsi, 2*64, 32*64, dist64, 64
	yr8_dispatch call, y8real_fft_final, 32*64, dist64, 64
	mov	rcx, xsincos_complex	;; Prefetch complex sin/cos data, 208 cache lines
	yloop_init 48			;; 47 more iterations
	yloop_adjust 1
	yloop_set_prefetch YDOUBLE
b5a:	yr8_8cl_eight_complex_fft_final rsi, 2*64, 32*64, dist64, 64
	yloop	16, b5a, -16*2*64+2*dist64	;; Loop 16 times, next source pointer
	yloop	3, b5a			;; Loop 3 times
	jmp	ypass2_1536_real_done

ypass2_1536_real_3:
x64	mov	r8, DEST2ARG
	mov	rbp, DIST_TO_MULSRCARG
;;	yr8_8cl_sixteen_reals_eight_complex_with_mult rsi, 2*64, 32*64, dist64, 64
	yr8_dispatch call, y8real_mult, 32*64, dist64, 64
	yloop_init 48			;; 47 more iterations
	yloop_adjust 1
	yloop_set_prefetch YSINGLE
	yr8_8cl_eight_complex_with_mult_preload
b5c:	yr8_8cl_eight_complex_with_mult rsi, 2*64, 32*64, dist64, 64
	yloop	16, b5c, -16*2*64+2*dist64	;; Loop 16 times, next source pointer
	yloop	3, b5c, -3*2*dist64	;; Loop 3 times, restore source pointer
x64	add	rsi, r8
	jmp	ypass2_1536_real_unfft

ypass2_1536_real_4:
	mov	rbp, DIST_TO_MULSRCARG
;;	yr8_8cl_sixteen_reals_eight_complex_with_mulf rsi, 2*64, 32*64, dist64, 64
	yr8_dispatch call, y8real_mulf, 32*64, dist64, 64
	yloop_init 48			;; 47 more iterations
	yloop_adjust 1
	yloop_set_prefetch YDOUBLE
	yr8_8cl_eight_complex_with_mulf_preload
b5d:	yr8_8cl_eight_complex_with_mulf rsi, 2*64, 32*64, dist64, 64
	yloop	16, b5d, -16*2*64+2*dist64	;; Loop 16 times, next source pointer
	yloop	3, b5d, -3*2*dist64	;; Loop 3 times, restore source pointer
;;	bump	rcx, (-96+64+96+2*96+47)*64 ;; Move prefetch pointer back to where squaring would be

;; Do inverse FFT levels 7,8
;; On input the 64-byte cache lines hold these data values:
;;	0	+256	+256	+256	8	+256	+256	+256
;;	4	...
;;	16
;;	...
;;	1	...
;;	...
;;	1024	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+256	+256	+256	32	+256	+256	+256
;;	4	...
;;	...
;;	28	...
;;	64	...
;;	...
;;	1	...
;;	...
;;	1024	...
;;	...

;; Do 8 eight_reals_unfft operations
;; Do 376 four_complex_djbunfft operations
;;	distance between fft data elements is 8
;;	do 96 macros processing 32 data values

ypass2_1536_real_unfft:
	mov	rdi, sincos3		;; Load sin/cos pointer
	bump	rdi, 32*YMM_SCD2+128*YMM_SCD1
	yloop_init 96			;; 12 iterations of 8
	yloop_set_prefetch YHALF
	yr4_4cl_eight_reals_four_complex_djbunfft_preload
c4a:	yr4_4cl_eight_reals_four_complex_djbunfft rsi, 32*64, 2*64, 4*64, rdi, 4*YMM_SCD3, 2
	yloop	2, c4a, -2*32*64+dist64	;; Test loop counter, next source pointer
	yloop	2, c4a, -2*dist64+64	;; Test loop counter, next source pointer
	yloop	2, c4a, -2*64+8*64, rdi, -8*4*YMM_SCD3 ;; Test loop counter, next source pointer, restore sine/cosine pointer

	loops_reset			;; 11 more iters of 8
	yr4_rb4cl_four_complex_djbunfft_preload
c4b:	yr4_rb4cl_four_complex_djbunfft rsi, 32*64, 2*64, 4*64, rdi, 4*YMM_SCD3, 2
	yloop	2, c4b, -2*32*64+dist64	;; Test loop counter, next source pointer
	yloop	2, c4b, -2*dist64+64	;; Test loop counter, next source pointer
	yloop	2, c4b, -2*64+8*64, rdi, -8*4*YMM_SCD3 ;; Test loop counter, next source pointer, restore sine/cosine pointer
	yloop	4, c4b, -4*8*64+2*dist64	;; Test loop counter, next source pointer
	yloop	3, c4b, -3*2*dist64	;; Test loop counter, restore source pointer

;; Do inverse FFT levels 5,6
;; On input the 64-byte cache lines hold these data values:
;;	0	+256	+256	+256	32	+256	+256	+256
;;	4	...
;;	...
;;	28	...
;;	64	...
;;	...
;;	1	...
;;	...
;;	1024	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+256	+256	+256	128	+256	+256	+256
;;	4	...
;;	...
;;	124	...
;;	1	...
;;	...
;;	1024	...
;;	...

;; Do 32 eight_reals_unfft operations
;; Do 352 four_complex_djbunfft operations
;;	distance between fft data elements is 32
;;	do 96 macros processing 32 data values

	bump	rcx, (384+2*5-64-96-2*48-47-48-48)*64 ;; Move prefetch pointer along
	yloop_init 96			;; 3 iterations of 32
	yloop_set_prefetch YHALF
	yr4_4cl_eight_reals_four_complex_djbunfft_preload
c3a:	yr4_4cl_eight_reals_four_complex_djbunfft rsi, 32*64, 8*64, 16*64, rdi, YMM_SCD3, 2
	yloop	2, c3a, -2*32*64+dist64	;; Test loop counter, next source pointer
	yloop	2, c3a, -2*dist64+64	;; Test loop counter, next source pointer
	yloop	8, c3a, -8*64+2*dist64, rdi, -32*YMM_SCD3 ;; Test loop counter, next source pointer, restore sine/cosine pointer

	loops_reset			;; 2 more iters of 32
	yr4_rb4cl_four_complex_djbunfft_preload
c3b:	yr4_rb4cl_four_complex_djbunfft rsi, 32*64, 8*64, 16*64, rdi, YMM_SCD3, 2
	yloop	2, c3b, -2*32*64+dist64	;; Test loop counter, next source pointer
	yloop	2, c3b, -2*dist64+64	;; Test loop counter, next source pointer
	yloop	8, c3b, -8*64+2*dist64, rdi, -32*YMM_SCD3 ;; Test loop counter, next source pointer, restore sine/cosine pointer
	yloop	3, c3b, -3*2*dist64	;; Test loop counter, restore source pointer

;; Do inverse FFT levels 3,4
;; On input the 64-byte cache lines hold these data values:
;;	0	+256	+256	+256	128	+256	+256	+256
;;	4	...
;;	...
;;	124	...
;;	1	...
;;	...
;;	1024	...
;;	...
;; On output the 64-byte cache lines are swizzled to hold these data values:
;;	0	+1	+1	+1	512	+1	+1	+1
;;	4	...
;;	...
;;	508	...
;;	1024	...
;;	...

;; Do 128 eight_reals_unfft operations
;; Do 256 four_complex_djbunfft operations
;;	distance between fft data elements is 128
;;	do 96 macros processing 32 data values

	bump	rdi, -32*YMM_SCD2	;; Real sin/cos ptr
	mov	rbp, xsincos_complex	;; Complex sin/cos data
	bump	rbp, 128*YMM_SCD1
	mov	rcx, xsincos_complex	;; Prefetch complex sin/cos data, 208 cache lines
	yloop_init 96			;; 3 iterations of 32
	yloop_set_prefetch YSINGLE
	yr4_s4cl_2sc_eight_reals_unfft_preload
c2a:	yr4_s4cl_2sc_eight_reals_unfft rsi, 64, 32*64, dist64, rbp, YMM_SCD2, rdi, YMM_SCD2, 32
	yloop	32, c2a, -32*64+2*dist64, rbp, -32*YMM_SCD2 ;; Test loop counter, next source pointer, restore sine/cosine pointer

	loops_reset			;; 2 more iters of 32
	yr4_s4cl_four_complex_djbunfft_preload
c2b:	yr4_s4cl_four_complex_djbunfft rsi, 64, 32*64, dist64, rbp, YMM_SCD2, 32
	yloop	32, c2b, -32*64+2*dist64, rbp, -32*YMM_SCD2 ;; Test loop counter, next source pointer, restore sine/cosine pointer
	yloop	3, c2b, -3*2*dist64	;; Test loop counter, restore source pointer

;; Do inverse FFT levels 1,2,3 which correspond to complex levels 1,2
;; On input the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	512	+1	+1	+1
;;	4	...
;;	...
;;	508	...
;;	1024	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	1536	+1	+1	+1
;;	4	...
;;	...
;;	1532

;; Do 512 six_reals_unfft operations
;;	distance between fft data elements is 512
;;	do 128 macros processing 24 data values

	bump	rdi, -32*YMM_SCD2-128*YMM_SCD1 ;; Real sin/cos ptr
	bump	rbp, -128*YMM_SCD1	;; Complex sin/cos data
	bump	rcx, (208-96-128)*64	;; Move prefetch pointer along
	yloop_init 128			;; 128 iterations
	yloop_set_prefetch YSINGLE
	yr3_3cl_2sc_six_reals_unfft_preload
c1b:	yr3_3cl_2sc_six_reals_unfft rsi, 64, 2*dist64, rbp, YMM_SCD1, rdi, YMM_SCD1, 64
	yloop	64, c1b, -64*64+dist64	;; Test loop counter, next source pointer
	yloop	2, c1b			;; Test loop counter

ypass2_1536_real_done:
	end_timer 2
	ENDM

; *************** radix4-pass2-1536-complex macro ******************
; This macro takes 1536 complex values and performs the final 10.585 levels of a standard
; mostly radix-4 FFT, squares the results, and does the first 10.585 levels of the inverse FFT.

yr4dwpn_pass2_1536_complex MACRO
	LOCAL	b1a, b1b, b2b, b3a0, b3a, b3b0, b3b, b3c0, b3c, b4a, b4b, b4c, b5a, b5b, b5c, b5d0, b5d
	LOCAL	c1b, c2a, c2b, c3b, c3c, c3d, c4b, c4c, c4d
	LOCAL	ypass2_1536_complex_1, ypass2_1536_complex_2
	LOCAL	ypass2_1536_complex_3, ypass2_1536_complex_4
	LOCAL	ypass2_1536_complex_unfft, ypass2_1536_complex_done

;; Type 4 FFTs skip the forward FFT process

	mov	rsi, DATA_ADDR		;; Load address of FFT data
	mov	rbx, DIST_TO_FFTSRCARG
	mov	rcx, DATA_PREFETCH	;; Prefetch pointer (we need to prefetch 
					;; 384 cache lines + 5 pad cache lines)
	cmp	ffttype, 4
	je	ypass2_1536_complex_4

;; Do FFT levels 1,2 (actually 1.585 levels)
;; On input the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	1536	+1	+1	+1
;;	4	...
;;	...
;;	1532
;; On output the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	512	+1	+1	+1
;;	4	...
;;	...
;;	508	...
;;	1024	...
;;	...

;; Do 512 three_complex_fft operations
;;	distance between fft data elements is 512
;;	do 128 macros each processing 24 data values

;; Do FFT levels 3,4
;; On input the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	512	+1	+1	+1
;;	4	...
;;	...
;;	508	...
;;	1024	...
;;	...
;; On output the 64-byte cache lines are swizzled to hold these data values:
;;	0	+256	+256	+256	128	+256	+256	+256
;;	4	...
;;	...
;;	124	...
;;	1	...
;;	...
;;	1024	...
;;	...

;; Do 384 four_complex_fft operations
;;	distance between fft data elements is 128
;;	do 96 macros each processing 32 data values

	mov	rdi, xsincos_complex	;; The sin/cos ptrs
	lea	rbp, [rdi+128*YMM_SCD1]
	yloop_init 128			;; 128 iterations or 96 iterations

b1a:	start_timer 4
	yloop_set_prefetch YQUARTER	;; Prefetch 32 cache lines
	yr3_f3cl_three_complex_djbfft_preload
b1b:	yr3_f3cl_three_complex_djbfft rsi, 32*64, 2*dist64, rdi, 32*YMM_SCD1, 2
	yloop	2, b1b, -2*32*64+dist64	;; Test loop counter, next source pointer
	yloop	2, b1b, -2*dist64, rdi, -4*32*YMM_SCD1+YMM_SCD1 ;; Test loop counter, restore source pointer, next sin/cos data
	loops_undo 4			;; Restore loop counter
	end_timer 4

	start_timer 5
	yloop_adjust 1			;; One fewer iterations in this loop
	yloop_set_prefetch YQUARTER	;; Prefetch 32 cache lines
	yr4_s4cl_four_complex_djbfft_preload
b2b:	yr4_s4cl_four_complex_djbfft rsi, 2*dist64, 32*64, dist64, rbp, 0, 3, L1PREFETCH_ALL, 64
	yloop	4, b2b, -3*2*dist64+64, rbp, YMM_SCD2 ;; 3 iterations!!  Next source pointer, next sin/cos data
	end_timer 5

	yloop	32, b1a, -32*64, rdi, -32*YMM_SCD1+128*YMM_SCD1+32*YMM_SCD2 ;; Test loop counter, restore source pointer, next sin/cos ptr

;; Do FFT levels 5,6
;; On input the 64-byte cache lines hold these data values:
;;	0	+256	+256	+256	128	+256	+256	+256
;;	4	...
;;	...
;;	124	...
;;	1	...
;;	...
;;	1024	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+256	+256	+256	32	+256	+256	+256
;;	4	...
;;	...
;;	28	...
;;	64	...
;;	...
;;	1	...
;;	...
;;	1024	...
;;	...

;; Do 384 four_complex_fft operations
;;	distance between fft data elements is 32
;;	do 96 macros each processing 32 data values

;; Do FFT levels 7,8
;; On input the 64-byte cache lines hold these data values:
;;	0	+256	+256	+256	32	+256	+256	+256
;;	4	...
;;	...
;;	28	...
;;	64	...
;;	...
;;	1	...
;;	...
;;	1024	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+256	+256	+256	8	+256	+256	+256
;;	4	...
;;	16
;;	...
;;	1	...
;;	...
;;	1024	...
;;	...

;; Do 384 four_complex_fft operations
;;	distance between fft data elements is 8
;;	do 96 macros each processing 32 data values

;; Do FFT levels 9,10,11 as well as inverse FFT levels 9,10,11
;; On input and output the 64-byte cache lines hold these data values:
;;	0	+256	+256	+256	8	+256	+256	+256
;;	4	...
;;	16
;;	...
;;	1	...
;;	...
;;	1024	...
;;	...

;; Do 192 eight_complex_fft operations
;;	distance between fft data elements is 1
;;	do 48 macros each processing 64 data values

;; Do inverse FFT levels 7,8
;; On input the 64-byte cache lines hold these data values:
;;	0	+256	+256	+256	8	+256	+256	+256
;;	4	...
;;	16
;;	...
;;	1	...
;;	...
;;	1024	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+256	+256	+256	32	+256	+256	+256
;;	4	...
;;	...
;;	28	...
;;	64	...
;;	...
;;	1	...
;;	...
;;	1024	...
;;	...

;; Do 384 four_complex_unfft operations
;;	distance between fft data elements is 8
;;	do 96 macros each processing 32 data values

;; Do inverse FFT levels 5,6
;; On input the 64-byte cache lines hold these data values:
;;	0	+256	+256	+256	32	+256	+256	+256
;;	4	...
;;	...
;;	28	...
;;	64	...
;;	...
;;	1	...
;;	...
;;	1024	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+256	+256	+256	128	+256	+256	+256
;;	4	...
;;	...
;;	124	...
;;	1	...
;;	...
;;	1024	...
;;	...

;; Do 384 four_complex_unfft operations
;;	distance between fft data elements is 32
;;	do 96 macros each processing 32 data values

;; Do this 128 cache lines (8KB) at a time to maximize accessing data
;; while it is in the L1 cache.

;; Execute the right middle step

	cmp	ffttype, 2
	jl	ypass2_1536_complex_1
	jg	ypass2_1536_complex_3

ypass2_1536_complex_2:
	yloop_init 96			;; 3 iters of 128 cache lines

b3b0:	start_timer 6
	yloop_set_prefetch YHALF	;; Prefetch 48 cache lines in this loop
	yr4_b4cl_four_complex_djbfft_preload
b3b:	yr4_b4cl_four_complex_djbfft rsi, 64, 8*64, 16*64, rdi, YMM_SCD2/4, 8
	yloop	8, b3b, -8*64+32*64	;; Loop 8 times, next source pointer
	yloop	2, b3b, -2*32*64+dist64	;; Loop 2 times, next source pointer
	yloop	2, b3b, -2*dist64, rdi, -32*YMM_SCD2/4 ;; Loop 2 times, restore source pointer, restore sine/cosine pointer
	loops_undo 4*8			;; Back up loop counter
	end_timer 6

	start_timer 7
	yloop_set_prefetch YHALF	;; Prefetch 48 cache lines in this loop
;;	yr4_b4cl_four_complex_djbfft_preload
b4b:	yr4_b4cl_four_complex_djbfft rsi, 64, 2*64, 4*64, rdi, 4*YMM_SCD2/4, 2
	yloop	2, b4b, -2*64+8*64, rdi, -2*4*YMM_SCD2/4 ;; Loop 2 times, next source pointer, restore sine/cosine pointer
	yloop	4, b4b, -4*8*64+32*64, rdi, YMM_SCD2/4 ;; Loop 4 times, next source pointer, next sine/cosine pointer
	yloop	2, b4b, -2*32*64+dist64	;; Loop 2 times, next source pointer
	yloop	2, b4b, -2*dist64, rdi, -4*YMM_SCD2/4 ;; Loop 2 times, restore source pointer, restore sine/cosine pointer
	loops_undo 4*4*2		;; Back up loop counter
	end_timer 7

	start_timer 9
	yloop_adjust 2*8		;; 16 fewer iterations in this loop
	yloop_set_prefetch YSINGLE	;; Prefetch 48 cache lines in this loop
	yr8_8cl_eight_complex_with_square_preload
;; Loop below reworked to avoid 4KB distances
;;b5b:	yr8_8cl_eight_complex_with_square rsi, 2*64, 32*64, dist64, 64, 32
;;	yloop	32, b5b, -16*2*64	;; 16 iterations!!  Restore source pointer
b5b:	yr8_8cl_eight_complex_with_square rsi, 4*64, 32*64, dist64, 64, 8
	yloop	8, b5b, -8*4*64+2*64	;; Loop 8 times, next source pointer
	yloop	4, b5b, -2*2*64		;; 2 iterations!!  Restore source pointer
	loops_undo 4*8			;; Back up loop counter
	end_timer 9

	start_timer 10
	yloop_set_prefetch YHALF	;; Prefetch 48 cache lines in this loop
	yr4_b4cl_four_complex_djbunfft_preload
c4b:	yr4_b4cl_four_complex_djbunfft rsi, 64, 2*64, 4*64, rdi, 4*YMM_SCD2/4, 2
	yloop	2, c4b, -2*64+8*64, rdi, -2*4*YMM_SCD2/4 ;; Loop 2 times, next source pointer, restore sine/cosine pointer
	yloop	4, c4b, -4*8*64+32*64, rdi, YMM_SCD2/4 ;; Loop 4 times, next source pointer, next sine/cosine pointer
	yloop	2, c4b, -2*32*64+dist64	;; Loop 2 times, next source pointer
	yloop	2, c4b, -2*dist64, rdi, -4*YMM_SCD2/4 ;; Loop 2 times, restore source pointer, restore sine/cosine pointer
	loops_undo 4*4*2		;; Back up loop counter
	end_timer 10

	start_timer 11
	yloop_set_prefetch YHALF	;; Prefetch 48 cache lines in this loop
;;	yr4_b4cl_four_complex_djbunfft_preload
c3b:	yr4_b4cl_four_complex_djbunfft rsi, 64, 8*64, 16*64, rdi, YMM_SCD2/4, 8, L1PREFETCH_ALL, 2*dist64
	yloop	8, c3b, -8*64+32*64	;; Loop 8 times, next source pointer
	yloop	2, c3b, -2*32*64+dist64	;; Loop 2 times, next source pointer
	yloop	2, c3b, -2*dist64+2*dist64, rdi, -32*YMM_SCD2/4 ;; Loop 2 times, next source pointer, restore sine/cosine pointer
	end_timer 11

	yloop	3, b3b0, -3*2*dist64	;; Loop 3 times, restore source pointer
	jmp	ypass2_1536_complex_unfft


ypass2_1536_complex_1:
	start_timer 6
;;	bump	rcx, (384+2*5-2*32-3*96)*64 ;; Move prefetch pointer along
	yloop_init 96			;; 3 iters of 128 cache lines

b3a0:	yloop_set_prefetch YSINGLE	;; Prefetch 96 cache lines in this loop
	yr4_b4cl_four_complex_djbfft_preload
b3a:	yr4_b4cl_four_complex_djbfft rsi, 64, 8*64, 16*64, rdi, YMM_SCD2/4, 8
	yloop	8, b3a, -8*64+32*64	;; Loop 8 times, next source pointer
	yloop	2, b3a, -2*32*64+dist64	;; Loop 2 times, next source pointer
	yloop	2, b3a, -2*dist64, rdi, -32*YMM_SCD2/4 ;; Loop 2 times, restore source pointer, restore sine/cosine pointer
	loops_undo 4*8			;; Back up loop counter

	yloop_set_prefetch YSINGLE	;; Prefetch 96 cache lines in this loop
;;	yr4_b4cl_four_complex_djbfft_preload
b4a:	yr4_b4cl_four_complex_djbfft rsi, 64, 2*64, 4*64, rdi, 4*YMM_SCD2/4, 2, L1PREFETCH_ALL, 2*dist64
	yloop	2, b4a, -2*64+8*64, rdi, -2*4*YMM_SCD2/4 ;; Loop 2 times, next source pointer, restore sine/cosine pointer
	yloop	4, b4a, -4*8*64+32*64, rdi, YMM_SCD2/4 ;; Loop 4 times, next source pointer, next sine/cosine pointer
	yloop	2, b4a, -2*32*64+dist64	;; Loop 2 times, next source pointer
	yloop	2, b4a, -2*dist64, rdi, -4*YMM_SCD2/4 ;; Loop 2 times, restore source pointer, restore sine/cosine pointer
	loops_undo 4*4*2		;; Back up loop counter

	yloop_adjust 2*8		;; 16 fewer iterations in this loop
	yloop_set_prefetch YDOUBLE	;; Prefetch 96 cache lines in this loop
	yr8_8cl_eight_complex_fft_final_preload
;; Loop below reworked to avoid 4KB distances
;;b5a:	yr8_8cl_eight_complex_fft_final rsi, 2*64, 32*64, dist64, 64
;;	yloop	32, b5a, -16*2*64	;; 16 iterations!!  Restore source pointer
b5a:	yr8_8cl_eight_complex_fft_final rsi, 4*64, 32*64, dist64, 64
	yloop	8, b5a, -8*4*64+2*64	;; Loop 8 times, next source pointer
	yloop	4, b5a, -2*2*64+2*dist64 ;; 2 iterations!!  Next source pointer

	yloop	3, b3a0			;; Loop 3 times
	end_timer 6
	jmp	ypass2_1536_complex_done


ypass2_1536_complex_3:
	start_timer 6
x64	mov	r8, DEST2ARG
	mov	rbp, DIST_TO_MULSRCARG
	yloop_init 96			;; 3 iters of 128 cache lines

b3c0:	yloop_set_prefetch YHALF	;; Prefetch 48 cache lines in this loop
	yr4_b4cl_four_complex_djbfft_preload
b3c:	yr4_b4cl_four_complex_djbfft rsi, 64, 8*64, 16*64, rdi, YMM_SCD2/4, 8
	yloop	8, b3c, -8*64+32*64	;; Loop 8 times, next source pointer
	yloop	2, b3c, -2*32*64+dist64	;; Loop 2 times, next source pointer
	yloop	2, b3c, -2*dist64, rdi, -32*YMM_SCD2/4 ;; Loop 2 times, restore source pointer, restore sine/cosine pointer
	loops_undo 4*8			;; Back up loop counter

	yloop_set_prefetch YHALF	;; Prefetch 48 cache lines in this loop
;;	yr4_b4cl_four_complex_djbfft_preload
b4c:	yr4_b4cl_four_complex_djbfft rsi, 64, 2*64, 4*64, rdi, 4*YMM_SCD2/4, 2
	yloop	2, b4c, -2*64+8*64, rdi, -2*4*YMM_SCD2/4 ;; Loop 2 times, next source pointer, restore sine/cosine pointer
	yloop	4, b4c, -4*8*64+32*64, rdi, YMM_SCD2/4 ;; Loop 4 times, next source pointer, next sine/cosine pointer
	yloop	2, b4c, -2*32*64+dist64	;; Loop 2 times, next source pointer
	yloop	2, b4c, -2*dist64, rdi, -4*YMM_SCD2/4 ;; Loop 2 times, restore source pointer, restore sine/cosine pointer
	loops_undo 4*4*2		;; Back up loop counter

	yloop_adjust 2*8		;; 16 fewer iterations in this loop
	yloop_set_prefetch YSINGLE	;; Prefetch 48 cache lines in this loop
	yr8_8cl_eight_complex_with_mult_preload
;; Loop below reworked to avoid 4KB distances
;;b5c:	yr8_8cl_eight_complex_with_mult rsi, 2*64, 32*64, dist64, 64
;;	yloop	32, b5c, -16*2*64	;; 16 iterations!!  Restore source pointer
b5c:	yr8_8cl_eight_complex_with_mult rsi, 4*64, 32*64, dist64, 64
	yloop	8, b5c, -8*4*64+2*64	;; Loop 8 times, next source pointer
	yloop	4, b5c, -2*2*64		;; 2 iterations!!  Restore source pointer
x64	add	rsi, r8
	loops_undo 4*8			;; Back up loop counter

	yloop_set_prefetch YHALF	;; Prefetch 48 cache lines in this loop
	yr4_b4cl_four_complex_djbunfft_preload
c4c:	yr4_b4cl_four_complex_djbunfft rsi, 64, 2*64, 4*64, rdi, 4*YMM_SCD2/4, 2
	yloop	2, c4c, -2*64+8*64, rdi, -2*4*YMM_SCD2/4 ;; Loop 2 times, next source pointer, restore sine/cosine pointer
	yloop	4, c4c, -4*8*64+32*64, rdi, YMM_SCD2/4 ;; Loop 4 times, next source pointer, next sine/cosine pointer
	yloop	2, c4c, -2*32*64+dist64	;; Loop 2 times, next source pointer
	yloop	2, c4c, -2*dist64, rdi, -4*YMM_SCD2/4 ;; Loop 2 times, restore source pointer, restore sine/cosine pointer
	loops_undo 4*4*2		;; Back up loop counter

	yloop_set_prefetch YHALF	;; Prefetch 48 cache lines in this loop
;;	yr4_b4cl_four_complex_djbunfft_preload
c3c:	yr4_b4cl_four_complex_djbunfft rsi, 64, 8*64, 16*64, rdi, YMM_SCD2/4, 8, L1PREFETCH_ALL, 2*dist64
	yloop	8, c3c, -8*64+32*64	;; Loop 8 times, next source pointer
	yloop	2, c3c, -2*32*64+dist64	;; Loop 2 times, next source pointer
	yloop	2, c3c, -2*dist64+2*dist64, rdi, -32*YMM_SCD2/4 ;; Loop 2 times, next source pointer, restore sine/cosine pointer

x64	sub	rsi, r8
	yloop	3, b3c0, -3*2*dist64	;; Loop 3 times, restore source pointer
x64	add	rsi, r8
	end_timer 6
	jmp	ypass2_1536_complex_unfft


ypass2_1536_complex_4:
	start_timer 6
	mov	rdi, xsincos_complex	;; The sin/cos ptrs
	bump	rdi, 128*YMM_SCD1+32*YMM_SCD2
	mov	rbp, DIST_TO_MULSRCARG
	yloop_init 96			;; 3 iters of 128 cache lines

b5d0:	yloop_adjust 2*8		;; 16 fewer iterations in this loop
	yloop_set_prefetch YDOUBLE	;; Prefetch 96 cache lines in this loop
	yr8_8cl_eight_complex_with_mulf_preload
;; Loop below reworked to avoid 4KB distances
;;b5d:	yr8_8cl_eight_complex_with_mulf rsi, 2*64, 32*64, dist64, 64
;;	yloop	32, b5d, -16*2*64	;; 16 iterations!!  Restore source pointer
b5d:	yr8_8cl_eight_complex_with_mulf rsi, 4*64, 32*64, dist64, 64
	yloop	8, b5d, -8*4*64+2*64	;; Loop 8 times, next source pointer
	yloop	4, b5d, -2*2*64		;; 2 iterations!!  Restore source pointer
	loops_undo 4*8			;; Back up loop counter

	yloop_set_prefetch YSINGLE	;; Prefetch 96 cache lines in this loop
	yr4_b4cl_four_complex_djbunfft_preload
c4d:	yr4_b4cl_four_complex_djbunfft rsi, 64, 2*64, 4*64, rdi, 4*YMM_SCD2/4, 2
	yloop	2, c4d, -2*64+8*64, rdi, -2*4*YMM_SCD2/4 ;; Loop 2 times, next source pointer, restore sine/cosine pointer
	yloop	4, c4d, -4*8*64+32*64, rdi, YMM_SCD2/4 ;; Loop 4 times, next source pointer, next sine/cosine pointer
	yloop	2, c4d, -2*32*64+dist64	;; Loop 2 times, next source pointer
	yloop	2, c4d, -2*dist64, rdi, -4*YMM_SCD2/4 ;; Loop 2 times, restore source pointer, restore sine/cosine pointer
	loops_undo 4*4*2		;; Back up loop counter

	yloop_set_prefetch YSINGLE	;; Prefetch 96 cache lines in this loop
;;	yr4_b4cl_four_complex_djbunfft_preload
c3d:	yr4_b4cl_four_complex_djbunfft rsi, 64, 8*64, 16*64, rdi, YMM_SCD2/4, 8, L1PREFETCH_ALL, 2*dist64
	yloop	8, c3d, -8*64+32*64	;; Loop 8 times, next source pointer
	yloop	2, c3d, -2*32*64+dist64	;; Loop 2 times, next source pointer
	yloop	2, c3d, -2*dist64+2*dist64, rdi, -32*YMM_SCD2/4 ;; Loop 2 times, next source pointer, restore sine/cosine pointer

	yloop	3, b5d0, -3*2*dist64	;; Loop 3 times, restore source pointer
;;	bump	rcx, (-3*96+2*32+5*48)*64 ;; Move prefetch pointer to where squaring would be
	end_timer 6

;; Do inverse FFT levels 3,4
;; On input the 64-byte cache lines hold these data values:
;;	0	+256	+256	+256	128	+256	+256	+256
;;	4	...
;;	...
;;	124	...
;;	1	...
;;	...
;;	1024	...
;;	...
;; On output the 64-byte cache lines are swizzled to hold these data values:
;;	0	+1	+1	+1	512	+1	+1	+1
;;	4	...
;;	...
;;	508	...
;;	1024	...
;;	...

;; Do 384 four_complex_unfft operations
;;	distance between fft data elements is 128
;;	do 96 macros each processing 32 data values

;; Do inverse FFT levels 1,2
;; On input the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	512	+1	+1	+1
;;	4	...
;;	...
;;	508	...
;;	1024	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	1536	+1	+1	+1
;;	4	...
;;	...
;;	1532

;; Do 512 three_complex_unfft operations
;;	distance between fft data elements is 512
;;	do 128 macros each processing 24 data values

ypass2_1536_complex_unfft:
	mov	rdi, xsincos_complex	;; Load sin/cos pointers
	lea	rbp, [rdi+128*YMM_SCD1]
	bump	rcx, (384+2*5-2*32-5*48-32-64)*64 ;; Move prefetch pointer along
	yloop_init 128			;; 128 iterations or 96 iterations

c2a:	start_timer 13
	yloop_adjust 1			;; One fewer iteration this loop
	yloop_set_prefetch YQUARTER	;; Prefetch 32 cache lines in this loop
	yr4_s4cl_four_complex_djbunfft_preload
c2b:	yr4_s4cl_four_complex_djbunfft rsi, 2*dist64, 32*64, dist64, rbp, 0, 3, L1PREFETCH_ALL, 64
	yloop	4, c2b, -3*2*dist64, rbp, YMM_SCD2 ;; 3 iterations!!  Restore source pointer, next sine/cosine pointer
	loops_undo 4			;; Restore loop counter
	end_timer 13

	start_timer 14
	yloop_set_prefetch YHALF	;; Prefetch 64 cache lines in this loop
	yr3_3cl_three_complex_djbunfft_preload
c1b:	yr3_3cl_three_complex_djbunfft rsi, 32*64, 2*dist64, rdi, 32*YMM_SCD1, 2
	yloop	2, c1b, -2*32*64+dist64	;; Test loop counter, next source pointer
	yloop	2, c1b, -2*dist64+64, rdi, -4*32*YMM_SCD1+YMM_SCD1 ;; Test loop counter, next source pointer, next sine/cosine pointer
	end_timer 14

	yloop	32, c2a			;; Loop 32 times

ypass2_1536_complex_done:
	ENDM


; *************** radix4-pass2-11-levels-real macro ******************
; This macro takes 4096 real values and performs the final 12 levels of the FFT.
; The results are squared, and the first 12 levels of the inverse FFT are done.

yr4dwpn_pass2_11_levels_real MACRO
	LOCAL	b1b, b2b, b3a, b3b, b4a, b4b, b5a, b5b, b5c, b5d
	LOCAL	c1b, c2b, c3a, c3b, c4a, c4b
	LOCAL	ypass2_11_levels_real_1, ypass2_11_levels_real_2
	LOCAL	ypass2_11_levels_real_3, ypass2_11_levels_real_4
	LOCAL	ypass2_11_levels_real_unfft, ypass2_11_levels_real_done

;; Type 4 FFTs skip the forward FFT process

	start_timer 2
	mov	rsi, DATA_ADDR		;; Load source address
	mov	rbx, DIST_TO_FFTSRCARG
	mov	rcx, DATA_PREFETCH	;; Prefetch pointer (we need to prefetch 
					;; 512 cache lines + 7 pad cache lines)
	cmp	ffttype, 4
	je	ypass2_11_levels_real_4

;; Do FFT levels 1,2,3 which correspond to complex levels 1,2
;; On input the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	2048	+1	+1	+1
;;	4	...
;;	...
;;	2044
;; On output the 64-byte cache lines are swizzled to hold these data values:
;;	0	+1K	+1K	+1K	512	+1K	+1K	+1K
;;	4	...
;;	...
;;	508	...
;;	1	...
;;	...

;; Do 512 eight_reals_fft operations
;;	distance between fft data elements is 512
;;	do 128 macros each processing 32 data values

	mov	rdi, sincos3		;; Real sin/cos pointer
	mov	rbp, xsincos_complex	;; Complex sin/cos data
	yloop_init 128			;; 128 iterations
	yloop_set_prefetch YSINGLE
	yr4_fs4cl_2sc_eight_reals_fft_preload
b1b:	yr4_fs4cl_2sc_eight_reals_fft rsi, 64, 2*dist64, 4*dist64, rbp, YMM_SCD2, rdi, YMM_SCD2, 64
	yloop	64, b1b, -64*64+dist64	;; Test loop counter, next source pointer
	yloop	2, b1b, -2*dist64	;; Test loop counter, restore source pointer

;; Do FFT levels 3,4
;; On input the 64-byte cache lines hold these data values:
;;	0	+1K	+1K	+1K	512	+1K	+1K	+1K
;;	4	...
;;	...
;;	508	...
;;	1	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+1K	+1K	+1K	128	+1K	+1K	+1K
;;	4	...
;;	...
;;	124	...
;;	256	...
;;	...
;;	1	...
;;	...

;; Do 128 eight_reals_fft operations
;; Do 384 four_complex_fft operations
;;	distance between fft data elements is 128
;;	do 128 macros each processing 32 data values

	yloop_init 128			;; 128 iterations
	yloop_set_prefetch YSINGLE
	yr4_4cl_eight_reals_four_complex_djbfft_preload
b2b:	yr4_4cl_eight_reals_four_complex_djbfft rsi, 2*dist64, 32*64, dist64, rdi, YMM_SCD3, 4
	yloop	4, b2b, -4*2*dist64+64	;; Test loop counter, next source pointer
	yloop	32, b2b, -32*64, rdi, -128*YMM_SCD3 ;; Test loop counter, restore source pointer, restore sin/cos ptr

;; Do FFT levels 5,6
;; On input the 64-byte cache lines hold these data values:
;;	0	+1K	+1K	+1K	128	+1K	+1K	+1K
;;	4	...
;;	...
;;	124	...
;;	256	...
;;	...
;;	1	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+1K	+1K	+1K	32	+1K	+1K	+1K
;;	4	...
;;	...
;;	28	...
;;	64	...
;;	...
;;	1	...
;;	...

;; Do 32 eight_reals_fft operations
;; Do 480 four_complex_fft operations
;;	distance between fft data elements is 32
;;	do 128 macros each processing 32 data values

	yloop_init 128			;; 4 iterations of 32
	yloop_set_prefetch YSINGLE
	yr4_4cl_eight_reals_four_complex_djbfft_preload
b3a:	yr4_4cl_eight_reals_four_complex_djbfft rsi, 2*dist64, 8*64, 16*64, rdi, 4*YMM_SCD3, 4
	yloop	4, b3a, -4*2*dist64+64	;; Test loop counter, next source pointer
	yloop	8, b3a, -8*64+32*64, rdi, -32*4*YMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer

	loops_reset			;; 3 more iters of 32
	yr4_rb4cl_four_complex_djbfft_preload
b3b:	yr4_rb4cl_four_complex_djbfft rsi, 2*dist64, 8*64, 16*64, rdi, 4*YMM_SCD3, 4
	yloop	4, b3b, -4*2*dist64+64	;; Test loop counter, next source pointer
	yloop	8, b3b, -8*64+32*64, rdi, -32*4*YMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer
	yloop	2, b3b, -2*32*64+dist64	;; Test loop counter, next source pointer
	yloop	2, b3b, -2*dist64	;; Test loop counter, restore source pointer

;; Do FFT levels 7,8
;; On input the 64-byte cache lines hold these data values:
;;	0	+1K	+1K	+1K	32	+1K	+1K	+1K
;;	4	...
;;	...
;;	28	...
;;	64	...
;;	...
;;	1	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+1K	+1K	+1K	8	+1K	+1K	+1K
;;	4	...
;;	16	...
;;	...
;;	1	...
;;	...

;; Do 8 eight_reals_fft operations
;; Do 504 four_complex_fft operations
;;	distance between fft data elements is 8
;;	do 128 macros each processing 32 data values

	yloop_init 128			;; 16 iterations of 8
	yloop_set_prefetch YHALF
	yr4_4cl_eight_reals_four_complex_djbfft_preload
b4a:	yr4_4cl_eight_reals_four_complex_djbfft rsi, 2*dist64, 2*64, 4*64, rdi, 16*YMM_SCD3, 4
	yloop	4, b4a, -4*2*dist64+64	;; Test loop counter, next source pointer
	yloop	2, b4a, -2*64+8*64, rdi, -8*16*YMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer

	loops_reset			;; 15 more iters of 8
	yr4_rb4cl_four_complex_djbfft_preload
b4b:	yr4_rb4cl_four_complex_djbfft rsi, 2*dist64, 2*64, 4*64, rdi, 16*YMM_SCD3, 4
	yloop	4, b4b, -4*2*dist64+64	;; Test loop counter, next source pointer
	yloop	2, b4b, -2*64+8*64, rdi, -8*16*YMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer
	yloop	8, b4b, -8*8*64+dist64	;; Test loop counter, next source pointer
	yloop	2, b4b, -2*dist64	;; Test loop counter, restore source pointer

;; Do FFT levels 9,10,11 as well as inverse FFT levels 9,10,11
;; On input and output the 64-byte cache lines hold these data values:
;;	0	+1K	+1K	+1K	8	+1K	+1K	+1K
;;	4	...
;;	16	...
;;	...
;;	1	...
;;	...

;; Do 1 sixteen_reals_fft operation
;; Do 255 eight_complex_fft operations
;;	distance between fft data elements is 1
;;	do 64 macros each processing 64 data values

	cmp	ffttype, 2
	jl	ypass2_11_levels_real_1
	jg	ypass2_11_levels_real_3

ypass2_11_levels_real_2:
;;	yr8_8cl_sixteen_reals_eight_complex_with_square rsi, 2*64, 2*dist64, 4*dist64, 64
	yr8_dispatch call, y8real_square, 2*dist64, 4*dist64, 64
	yloop_init 64			;; 63 more iterations
	yloop_adjust 1
	yloop_set_prefetch YOTHER, 8
	yr8_8cl_eight_complex_with_square_preload
b5b:	yr8_8cl_eight_complex_with_square rsi, 2*64, 2*dist64, 4*dist64, 64, 1
	yloop	32, b5b, -32*2*64+dist64	;; Test loop counter, next source pointer
	yloop	2, b5b, -2*dist64	;; Test loop counter, restore source pointer
	jmp	ypass2_11_levels_real_unfft

ypass2_11_levels_real_1:
;;	yr8_8cl_sixteen_reals_eight_complex_fft_final rsi, 2*64, 2*dist64, 4*dist64, 64
	yr8_dispatch call, y8real_fft_final, 2*dist64, 4*dist64, 64
	mov	rcx, xsincos_complex	;; Prefetch complex sin/cos data, 272 cache lines
	yloop_init 64			;; 63 more iterations
	yloop_adjust 1
	yloop_set_prefetch YDOUBLE
b5a:	yr8_8cl_eight_complex_fft_final rsi, 2*64, 2*dist64, 4*dist64, 64
	yloop	32, b5a, -32*2*64+dist64	;; Test loop counter, next source pointer
	yloop	2, b5a			;; Test loop counter
	jmp	ypass2_11_levels_real_done

ypass2_11_levels_real_3:
x64	mov	r8, DEST2ARG
	mov	rbp, DIST_TO_MULSRCARG
;;	yr8_8cl_sixteen_reals_eight_complex_with_mult rsi, 2*64, 2*dist64, 4*dist64, 64
	yr8_dispatch call, y8real_mult, 2*dist64, 4*dist64, 64
	yloop_init 64			;; 63 more iterations
	yloop_adjust 1
	yloop_set_prefetch YOTHER, 8
	yr8_8cl_eight_complex_with_mult_preload
b5c:	yr8_8cl_eight_complex_with_mult rsi, 2*64, 2*dist64, 4*dist64, 64
	yloop	32, b5c, -32*2*64+dist64	;; Test loop counter, next source pointer
	yloop	2, b5c, -2*dist64	;; Test loop counter, restore source pointer
x64	add	rsi, r8
	jmp	ypass2_11_levels_real_unfft

ypass2_11_levels_real_4:
	mov	rbp, DIST_TO_MULSRCARG
;;	yr8_8cl_sixteen_reals_eight_complex_with_mulf rsi, 2*64, 2*dist64, 4*dist64, 64
	yr8_dispatch call, y8real_mulf, 2*dist64, 4*dist64, 64
	yloop_init 64			;; 63 more iterations
	yloop_adjust 1
	yloop_set_prefetch YDOUBLE
	yr8_8cl_eight_complex_with_mulf_preload
b5d:	yr8_8cl_eight_complex_with_mulf rsi, 2*64, 2*dist64, 4*dist64, 64
	yloop	32, b5d, -32*2*64+dist64	;; Test loop counter, next source pointer
	yloop	2, b5d, -2*dist64	;; Test loop counter, restore source pointer
;;	bump	rcx, (-128+3*128+64+8)*64 ;; Move prefetch pointer back to where squaring would be

;; Do inverse FFT levels 7,8
;; On input the 64-byte cache lines hold these data values:
;;	0	+1K	+1K	+1K	8	+1K	+1K	+1K
;;	4	...
;;	16	...
;;	...
;;	1	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+1K	+1K	+1K	32	+1K	+1K	+1K
;;	4	...
;;	...
;;	28	...
;;	64	...
;;	...
;;	1	...
;;	...

;; Do 8 eight_reals_unfft operations
;; Do 504 four_complex_unfft operations
;;	distance between fft data elements is 8
;;	do 128 macros each processing 32 data values

ypass2_11_levels_real_unfft:
	mov	rdi, sincos3		;; Load sin/cos pointer
	bump	rdi, 128*YMM_SCD2
	bump	rcx, (512+7-3*128-64-8-64)*64 ;; Move prefetch pointer along
	yloop_init 128			;; 16 iterations of 8
	yloop_set_prefetch YHALF
	yr4_4cl_eight_reals_four_complex_djbunfft_preload
c4a:	yr4_4cl_eight_reals_four_complex_djbunfft rsi, 2*dist64, 2*64, 4*64, rdi, 16*YMM_SCD3, 4
	yloop	4, c4a, -4*2*dist64+64	;; Test loop counter, next source pointer
	yloop	2, c4a, -2*64+8*64, rdi, -8*16*YMM_SCD3 ;; Test loop counter, next source pointer, restore sine/cosine pointer

	loops_reset			;; 15 more iters of 8
	yr4_rb4cl_four_complex_djbunfft_preload
c4b:	yr4_rb4cl_four_complex_djbunfft rsi, 2*dist64, 2*64, 4*64, rdi, 16*YMM_SCD3, 4
	yloop	4, c4b, -4*2*dist64+64	;; Test loop counter, next source pointer
	yloop	2, c4b, -2*64+8*64, rdi, -8*16*YMM_SCD3 ;; Test loop counter, next source pointer, restore sine/cosine pointer
	yloop	8, c4b, -8*8*64+dist64	;; Test loop counter, next source pointer
	yloop	2, c4b, -2*dist64	;; Test loop counter, restore source pointer

;; Do inverse FFT levels 5,6
;; On input the 64-byte cache lines hold these data values:
;;	0	+1K	+1K	+1K	32	+1K	+1K	+1K
;;	4	...
;;	...
;;	28	...
;;	64	...
;;	...
;;	1	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+1K	+1K	+1K	128	+1K	+1K	+1K
;;	4	...
;;	...
;;	124	...
;;	256	...
;;	...
;;	1	...
;;	...

;; Do 32 eight_reals_unfft operations
;; Do 480 four_complex_unfft operations
;;	distance between fft data elements is 32
;;	do 128 macros each processing 32 data values

	mov	rcx, xsincos_complex	;; Prefetch complex sin/cos data, 272 cache lines
	yloop_init 128			;; 4 iterations of 32
	yloop_set_prefetch YOTHER, 8
	yr4_4cl_eight_reals_four_complex_djbunfft_preload
c3a:	yr4_4cl_eight_reals_four_complex_djbunfft rsi, 2*dist64, 8*64, 16*64, rdi, 4*YMM_SCD3, 4
	yloop	4, c3a, -4*2*dist64+64	;; Test loop counter, next source pointer
	yloop	8, c3a, -8*64+32*64, rdi, -32*4*YMM_SCD3 ;; Test loop counter, next source pointer, restore sine/cosine pointer

	loops_reset			;; 3 more iters of 32
	yr4_rb4cl_four_complex_djbunfft_preload
c3b:	yr4_rb4cl_four_complex_djbunfft rsi, 2*dist64, 8*64, 16*64, rdi, 4*YMM_SCD3, 4
	yloop	4, c3b, -4*2*dist64+64	;; Test loop counter, next source pointer
	yloop	8, c3b, -8*64+32*64, rdi, -32*4*YMM_SCD3 ;; Test loop counter, next source pointer, restore sine/cosine pointer
	yloop	2, c3b, -2*32*64+dist64	;; Test loop counter, next source pointer
	yloop	2, c3b, -2*dist64	;; Test loop counter, restore source pointer

;; Do inverse FFT levels 3,4
;; On input the 64-byte cache lines hold these data values:
;;	0	+1K	+1K	+1K	128	+1K	+1K	+1K
;;	4	...
;;	...
;;	124	...
;;	256	...
;;	...
;;	1	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+1K	+1K	+1K	512	+1K	+1K	+1K
;;	4	...
;;	...
;;	508	...
;;	1	...
;;	...

;; Do 128 eight_reals_unfft operations
;; Do 384 four_complex_unfft operations
;;	distance between fft data elements is 128
;;	do 128 macros each processing 32 data values

	yloop_init 128			;; 128 iterations
	yloop_set_prefetch YSINGLE
	yr4_4cl_eight_reals_four_complex_djbunfft_preload
c2b:	yr4_4cl_eight_reals_four_complex_djbunfft rsi, 2*dist64, 32*64, dist64, rdi, YMM_SCD3, 4
	yloop	4, c2b, -4*2*dist64+64	;; Test loop counter, next source pointer
	yloop	32, c2b, -32*64	;; Test loop counter, restore source pointer

;; Do inverse FFT levels 1,2,3 which correspond to complex levels 1,2
;; On input the 64-byte cache lines hold these data values:
;;	0	+1K	+1K	+1K	512	+1K	+1K	+1K
;;	4	...
;;	...
;;	508	...
;;	1	...
;;	...
;; On output the 64-byte cache lines are swizzled to hold these data values:
;;	0	+1	+1	+1	2048	+1	+1	+1
;;	4	...
;;	...
;;	2044

;; Do 512 eight_reals_unfft operations
;;	distance between fft data elements is 512
;;	do 128 macros each processing 32 data values

	bump	rdi, -128*YMM_SCD3-128*YMM_SCD2 ;; Real sin/cos pointer
	mov	rbp, xsincos_complex	;; Complex sin/cos data
	bump	rcx, (272-16-128-128)*64 ;; Move prefetch pointer along
	yloop_init 128			;; 128 iterations
	yloop_set_prefetch YSINGLE
	yr4_s4cl_2sc_eight_reals_unfft_preload
c1b:	yr4_s4cl_2sc_eight_reals_unfft rsi, 64, 2*dist64, 4*dist64, rbp, YMM_SCD2, rdi, YMM_SCD2, 64
	yloop	64, c1b, -64*64+dist64	;; Test loop counter, next source pointer
	yloop	2, c1b			;; Test loop counter

ypass2_11_levels_real_done:
	end_timer 2
	ENDM

; *************** radix4-pass2-11-levels-complex macro ******************
; This macro takes 2048 complex values and performs the final 11 levels of a standard
; radix-4 FFT, squares the results, and does the first 11 levels of the inverse FFT.

yr4dwpn_pass2_11_levels_complex MACRO
	LOCAL	b1a, b1b, b2b, b3a0, b3a, b3b0, b3b, b3c0, b3c, b4a, b4b, b4c, b5a, b5b, b5c, b5d0, b5d
	LOCAL	c1b, c2a, c2b, c3b, c3c, c3d, c4b, c4c, c4d
	LOCAL	ypass2_11_levels_complex_1, ypass2_11_levels_complex_2
	LOCAL	ypass2_11_levels_complex_3, ypass2_11_levels_complex_4
	LOCAL	ypass2_11_levels_complex_unfft, ypass2_11_levels_complex_done

;; Type 4 FFTs skip the forward FFT process

	mov	rsi, DATA_ADDR		;; Load address of FFT data
	mov	rbx, DIST_TO_FFTSRCARG
	mov	rcx, DATA_PREFETCH	;; Prefetch pointer (we need to prefetch 
					;; 512 cache lines + 7 pad cache lines)
	cmp	ffttype, 4
	je	ypass2_11_levels_complex_4

;; Do FFT level 1,2
;; On input the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	2048	+1	+1	+1
;;	4	...
;;	...
;;	2044
;; On output the 64-byte cache lines are swizzled to hold these data values:
;;	0	+1K	+1K	+1K	512	+1K	+1K	+1K
;;	4	...
;;	...
;;	508	...
;;	1	...
;;	...

;; Do 512 four_complex_fft operations
;;	distance between fft data elements is 512
;;	do 128 macros each processing 32 data values

;; Do FFT levels 3,4
;; On input the 64-byte cache lines hold these data values:
;;	0	+1K	+1K	+1K	512	+1K	+1K	+1K
;;	4	...
;;	...
;;	508	...
;;	1	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+1K	+1K	+1K	128	+1K	+1K	+1K
;;	4	...
;;	...
;;	124	...
;;	256	...
;;	...
;;	1	...
;;	...

;; Do 512 four_complex_fft operations
;;	distance between fft data elements is 128
;;	do 128 macros each processing 32 data values

	mov	rdi, xsincos_complex	;; The sin/cos ptrs
	mov	rbp, rdi
	yloop_init 128			;; 128 iterations
	yloop_set_prefetch YHALF

b1a:	start_timer 4
	yr4_fs4cl_four_complex_djbfft_preload
b1b:	yr4_fs4cl_four_complex_djbfft rsi, 32*64, 2*dist64, 4*dist64, rdi, 32*YMM_SCD2, 2
	yloop	2, b1b, -2*32*64+dist64	;; Test loop counter, next source pointer
	yloop	2, b1b, -2*dist64, rdi, -4*32*YMM_SCD2+YMM_SCD2 ;; Test loop counter, restore source pointer, next sin/cos data
	loops_undo 4
	end_timer 4

	start_timer 5
	yr4_eb4cl_four_complex_djbfft_preload
b2b:	yr4_eb4cl_four_complex_djbfft rsi, 2*dist64, 32*64, dist64, rbp, YMM_SCD2, 4, L1PREFETCH_ALL, 64
	yloop	4, b2b, -4*2*dist64+64	;; Test loop counter, next source pointer
	end_timer 5

	yloop	32, b1a, -32*64, rdi, -32*YMM_SCD2+128*YMM_SCD2 ;; Test loop counter, restore source pointer, next sin/cos ptr

;; Do FFT levels 5,6
;; On input the 64-byte cache lines hold these data values:
;;	0	+1K	+1K	+1K	128	+1K	+1K	+1K
;;	4	...
;;	...
;;	124	...
;;	256	...
;;	...
;;	1	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+1K	+1K	+1K	32	+1K	+1K	+1K
;;	4	...
;;	...
;;	28	...
;;	64	...
;;	...
;;	1	...
;;	...

;; Do 512 four_complex_fft operations
;;	distance between fft data elements is 32
;;	do 128 macros each processing 32 data values

;; Do FFT levels 7,8
;; On input the 64-byte cache lines hold these data values:
;;	0	+1K	+1K	+1K	32	+1K	+1K	+1K
;;	4	...
;;	...
;;	28	...
;;	64	...
;;	...
;;	1	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+1K	+1K	+1K	8	+1K	+1K	+1K
;;	4	...
;;	16	...
;;	...
;;	1	...
;;	...

;; Do 512 four_complex_fft operations
;;	distance between fft data elements is 8
;;	do 128 macros each processing 32 data values

;; Do FFT levels 9,10,11 as well as inverse FFT levels 9,10,11
;; On input and output the 64-byte cache lines hold these data values:
;;	0	+1K	+1K	+1K	8	+1K	+1K	+1K
;;	4	...
;;	16	...
;;	...
;;	1	...
;;	...

;; Do 256 eight_complex_fft operations
;;	distance between fft data elements is 1
;;	do 64 macros each processing 64 data values

;; Do inverse FFT levels 7,8
;; On input the 64-byte cache lines hold these data values:
;;	0	+1K	+1K	+1K	8	+1K	+1K	+1K
;;	4	...
;;	16	...
;;	...
;;	1	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+1K	+1K	+1K	32	+1K	+1K	+1K
;;	4	...
;;	...
;;	28	...
;;	64	...
;;	...
;;	1	...
;;	...

;; Do 512 four_complex_unfft operations
;;	distance between fft data elements is 8
;;	do 128 macros each processing 32 data values

;; Do inverse FFT levels 5,6
;; On input the 64-byte cache lines hold these data values:
;;	0	+1K	+1K	+1K	32	+1K	+1K	+1K
;;	4	...
;;	...
;;	28	...
;;	64	...
;;	...
;;	1	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+1K	+1K	+1K	128	+1K	+1K	+1K
;;	4	...
;;	...
;;	124	...
;;	256	...
;;	...
;;	1	...
;;	...

;; Do 512 four_complex_unfft operations
;;	distance between fft data elements is 32
;;	do 128 macros each processing 32 data values

;; Do this 128 cache lines (8KB) at a time to maximize accessing data
;; while it is in the L1 cache.

;; Execute the right middle step

	cmp	ffttype, 2
	jl	ypass2_11_levels_complex_1
	jg	ypass2_11_levels_complex_3

ypass2_11_levels_complex_2:
	yloop_init 128			;; 4 iters of 128 cache lines

b3b0:	start_timer 6
	yloop_set_prefetch YHALF	;; Prefetch 64 cache lines in this loop
	yr4_b4cl_four_complex_djbfft_preload
b3b:	yr4_b4cl_four_complex_djbfft rsi, 64, 8*64, 16*64, rdi, YMM_SCD2/4, 8
	yloop	8, b3b, -8*64+2*dist64	;; Loop 8 times, next source pointer
	yloop	4, b3b, -4*2*dist64, rdi, -32*YMM_SCD2/4 ;; Loop 4 times, restore source pointer, restore sine/cosine pointer
	loops_undo 4*8			;; Back up loop counter
	end_timer 6

	start_timer 7
	yloop_set_prefetch YHALF	;; Prefetch 64 cache lines in this loop
;;	yr4_b4cl_four_complex_djbfft_preload
b4b:	yr4_b4cl_four_complex_djbfft rsi, 64, 2*64, 4*64, rdi, 4*YMM_SCD2/4, 2
	yloop	2, b4b, -2*64+8*64, rdi, -2*4*YMM_SCD2/4 ;; Loop 2 times, next source pointer, restore sine/cosine pointer
	yloop	4, b4b, -4*8*64+2*dist64, rdi, YMM_SCD2/4 ;; Loop 4 times, next source pointer, next sine/cosine pointer
	yloop	4, b4b, -4*2*dist64, rdi, -4*YMM_SCD2/4 ;; Loop 4 times, restore source pointer, restore sine/cosine pointer
	loops_undo 4*4*2		;; Back up loop counter
	end_timer 7

	start_timer 9
	yloop_adjust 4*4		;; 16 fewer iterations this loop
	yloop_set_prefetch YOTHER, 16	;; Prefetch 8 cache lines in this loop
	yr8_8cl_eight_complex_with_square_preload
;; Loop below reworked to avoid 4KB distances
;;b5b:	yr8_8cl_eight_complex_with_square rsi, 2*64, 2*dist64, 4*dist64, 64, 32
;;	yloop	32, b5b, -16*2*64	;; 16 iterations!!  Restore source pointer
b5b:	yr8_8cl_eight_complex_with_square rsi, 8*64, 2*dist64, 4*dist64, 64, 4
	yloop	4, b5b, -4*8*64+2*64	;; Loop 4 times, next source pointer
	yloop	8, b5b, -4*2*64		;; 8 iterations!!  Restore source pointer
	loops_undo 8*4			;; Back up loop counter
	end_timer 9

	start_timer 10
	yloop_set_prefetch YHALF	;; Prefetch 64 cache lines in this loop
	yr4_b4cl_four_complex_djbunfft_preload
c4b:	yr4_b4cl_four_complex_djbunfft rsi, 64, 2*64, 4*64, rdi, 4*YMM_SCD2/4, 2
	yloop	2, c4b, -2*64+8*64, rdi, -2*4*YMM_SCD2/4 ;; Loop 2 times, next source pointer, restore sine/cosine pointer
	yloop	4, c4b, -4*8*64+2*dist64, rdi, YMM_SCD2/4 ;; Loop 4 times, next source pointer, next sine/cosine pointer
	yloop	4, c4b, -4*2*dist64, rdi, -4*YMM_SCD2/4 ;; Loop 4 times, restore source pointer, restore sine/cosine pointer
	loops_undo 4*4*2		;; Back up loop counter
	end_timer 10

	start_timer 11
	yloop_set_prefetch YHALF	;; Prefetch 64 cache lines in this loop
;;	yr4_b4cl_four_complex_djbunfft_preload
c3b:	yr4_b4cl_four_complex_djbunfft rsi, 64, 8*64, 16*64, rdi, YMM_SCD2/4, 8, L1PREFETCH_ALL, 32*64
	yloop	8, c3b, -8*64+2*dist64	;; Loop 8 times, next source pointer
	yloop	4, c3b, -4*2*dist64+32*64, rdi, -32*YMM_SCD2/4 ;; Loop 4 times, next source pointer, restore sine/cosine pointer
	end_timer 11

	yloop	2, b3b0, -2*32*64+dist64 ;; Loop 2 times, next source pointer
	yloop	2, b3b0, -2*dist64	;; Loop 2 times, restore source pointer
	jmp	ypass2_11_levels_complex_unfft


ypass2_11_levels_complex_1:
	start_timer 6
;;	bump	rcx, (512+7-2*64-3*128)*64 ;; Move prefetch pointer along
	yloop_init 128			;; 4 iters of 128 cache lines

b3a0:	yloop_set_prefetch YSINGLE	;; Prefetch 128 cache lines in this loop
	yr4_b4cl_four_complex_djbfft_preload
b3a:	yr4_b4cl_four_complex_djbfft rsi, 64, 8*64, 16*64, rdi, YMM_SCD2/4, 8
	yloop	8, b3a, -8*64+2*dist64	;; Loop 8 times, next source pointer
	yloop	4, b3a, -4*2*dist64, rdi, -32*YMM_SCD2/4 ;; Loop 4 times, restore source pointer, restore sine/cosine pointer
	loops_undo 4*8			;; Back up loop counter

	yloop_set_prefetch YSINGLE	;; Prefetch 128 cache lines in this loop
;;	yr4_b4cl_four_complex_djbfft_preload
b4a:	yr4_b4cl_four_complex_djbfft rsi, 64, 2*64, 4*64, rdi, 4*YMM_SCD2/4, 2, L1PREFETCH_ALL, 32*64
	yloop	2, b4a, -2*64+8*64, rdi, -2*4*YMM_SCD2/4 ;; Loop 2 times, next source pointer, restore sine/cosine pointer
	yloop	4, b4a, -4*8*64+2*dist64, rdi, YMM_SCD2/4 ;; Loop 4 times, next source pointer, next sine/cosine pointer
	yloop	4, b4a, -4*2*dist64, rdi, -4*YMM_SCD2/4 ;; Loop 4 times, restore source pointer, restore sine/cosine pointer
	loops_undo 4*4*2		;; Back up loop counter

	yloop_adjust 4*4		;; 16 fewer iterations this loop
	yloop_set_prefetch YDOUBLE	;; Prefetch 128 cache lines in this loop
	yr8_8cl_eight_complex_fft_final_preload
;; Loop below reworked to avoid 4KB distances
;;b5a:	yr8_8cl_eight_complex_fft_final rsi, 2*64, 2*dist64, 4*dist64, 64
;;	yloop	32, b5a, -16*2*64	;; 16 iterations!!  Restore source pointer
b5a:	yr8_8cl_eight_complex_fft_final rsi, 8*64, 2*dist64, 4*dist64, 64
	yloop	4, b5a, -4*8*64+2*64	;; Loop 4 times, next source pointer
	yloop	8, b5a, -4*2*64+32*64	;; 4 iterations!!  Next source pointer

	yloop	2, b3a0, -2*32*64+dist64 ;; Loop 2 times, next source pointer
	yloop	2, b3a0			;; Loop 2 times
	end_timer 6
	jmp	ypass2_11_levels_complex_done


ypass2_11_levels_complex_3:
	start_timer 6
x64	mov	r8, DEST2ARG
	mov	rbp, DIST_TO_MULSRCARG
	yloop_init 128			;; 4 iters of 128 cache lines

b3c0:	yloop_set_prefetch YHALF	;; Prefetch 64 cache lines in this loop
	yr4_b4cl_four_complex_djbfft_preload
b3c:	yr4_b4cl_four_complex_djbfft rsi, 64, 8*64, 16*64, rdi, YMM_SCD2/4, 8
	yloop	8, b3c, -8*64+2*dist64	;; Loop 8 times, next source pointer
	yloop	4, b3c, -4*2*dist64, rdi, -32*YMM_SCD2/4 ;; Loop 4 times, restore source pointer, restore sine/cosine pointer
	loops_undo 4*8			;; Back up loop counter

	yloop_set_prefetch YHALF	;; Prefetch 64 cache lines in this loop
;;	yr4_b4cl_four_complex_djbfft_preload
b4c:	yr4_b4cl_four_complex_djbfft rsi, 64, 2*64, 4*64, rdi, 4*YMM_SCD2/4, 2
	yloop	2, b4c, -2*64+8*64, rdi, -2*4*YMM_SCD2/4 ;; Loop 2 times, next source pointer, restore sine/cosine pointer
	yloop	4, b4c, -4*8*64+2*dist64, rdi, YMM_SCD2/4 ;; Loop 4 times, next source pointer, next sine/cosine pointer
	yloop	4, b4c, -4*2*dist64, rdi, -4*YMM_SCD2/4 ;; Loop 4 times, restore source pointer, restore sine/cosine pointer
	loops_undo 4*4*2		;; Back up loop counter

	yloop_adjust 4*4		;; 16 fewer iterations this loop
	yloop_set_prefetch YOTHER, 16	;; Prefetch 8 cache lines in this loop
	yr8_8cl_eight_complex_with_mult_preload
;; Loop below reworked to avoid 4KB distances
;;b5c:	yr8_8cl_eight_complex_with_mult rsi, 2*64, 2*dist64, 4*dist64, 64
;;	yloop	32, b5c, -16*2*64	;; 16 iterations!!  Restore source pointer
b5c:	yr8_8cl_eight_complex_with_mult rsi, 8*64, 2*dist64, 4*dist64, 64
	yloop	4, b5c, -4*8*64+2*64	;; Loop 4 times, next source pointer
	yloop	8, b5c, -4*2*64		;; 4 iterations!!  Restore source pointer
x64	add	rsi, r8
	loops_undo 8*4			;; Back up loop counter

	yloop_set_prefetch YHALF	;; Prefetch 64 cache lines in this loop
	yr4_b4cl_four_complex_djbunfft_preload
c4c:	yr4_b4cl_four_complex_djbunfft rsi, 64, 2*64, 4*64, rdi, 4*YMM_SCD2/4, 2
	yloop	2, c4c, -2*64+8*64, rdi, -2*4*YMM_SCD2/4 ;; Loop 2 times, next source pointer, restore sine/cosine pointer
	yloop	4, c4c, -4*8*64+2*dist64, rdi, YMM_SCD2/4 ;; Loop 4 times, next source pointer, next sine/cosine pointer
	yloop	4, c4c, -4*2*dist64, rdi, -4*YMM_SCD2/4 ;; Loop 4 times, restore source pointer, restore sine/cosine pointer
	loops_undo 4*4*2		;; Back up loop counter

	yloop_set_prefetch YHALF	;; Prefetch 64 cache lines in this loop
;;	yr4_b4cl_four_complex_djbunfft_preload
c3c:	yr4_b4cl_four_complex_djbunfft rsi, 64, 8*64, 16*64, rdi, YMM_SCD2/4, 8, L1PREFETCH_ALL, 32*64
	yloop	8, c3c, -8*64+2*dist64	;; Loop 8 times, next source pointer
	yloop	4, c3c, -4*2*dist64+32*64, rdi, -32*YMM_SCD2/4 ;; Loop 4 times, next source pointer, restore sine/cosine pointer

x64	sub	rsi, r8
	yloop	2, b3c0, -2*32*64+dist64 ;; Loop 2 times, next source pointer
	yloop	2, b3c0, -2*dist64	;; Loop 2 times, restore source pointer
	end_timer 6
x64	add	rsi, r8
	jmp	ypass2_11_levels_complex_unfft


ypass2_11_levels_complex_4:
	start_timer 6
	mov	rdi, xsincos_complex	;; The sin/cos ptrs
	bump	rdi, 128*YMM_SCD2
	mov	rbp, DIST_TO_MULSRCARG
	yloop_init 128			;; 4 iters of 128 cache lines

b5d0:	yloop_adjust 4*4		;; 16 fewer iterations this loop
	yloop_set_prefetch YDOUBLE	;; Prefetch 128 cache lines in this loop
	yr8_8cl_eight_complex_with_mulf_preload
;; Loop below reworked to avoid 4KB distances
;;b5d:	yr8_8cl_eight_complex_with_mulf rsi, 2*64, 2*dist64, 4*dist64, 64
;;	yloop	32, b5d, -16*2*64	;; 16 iterations!!  Restore source pointer
b5d:	yr8_8cl_eight_complex_with_mulf rsi, 8*64, 2*dist64, 4*dist64, 64
	yloop	4, b5d, -4*8*64+2*64	;; Loop 4 times, next source pointer
	yloop	8, b5d, -4*2*64		;; 4 iterations!!  Restore source pointer
	loops_undo 8*4			;; Back up loop counter

	yloop_set_prefetch YSINGLE	;; Prefetch 128 cache lines in this loop
	yr4_b4cl_four_complex_djbunfft_preload
c4d:	yr4_b4cl_four_complex_djbunfft rsi, 64, 2*64, 4*64, rdi, 4*YMM_SCD2/4, 2
	yloop	2, c4d, -2*64+8*64, rdi, -2*4*YMM_SCD2/4 ;; Loop 2 times, next source pointer, restore sine/cosine pointer
	yloop	4, c4d, -4*8*64+2*dist64, rdi, YMM_SCD2/4 ;; Loop 4 times, next source pointer, next sine/cosine pointer
	yloop	4, c4d, -4*2*dist64, rdi, -4*YMM_SCD2/4 ;; Loop 4 times, restore source pointer, restore sine/cosine pointer
	loops_undo 4*4*2		;; Back up loop counter

	yloop_set_prefetch YSINGLE	;; Prefetch 128 cache lines in this loop
;;	yr4_b4cl_four_complex_djbunfft_preload
c3d:	yr4_b4cl_four_complex_djbunfft rsi, 64, 8*64, 16*64, rdi, YMM_SCD2/4, 8, L1PREFETCH_ALL, 32*64
	yloop	8, c3d, -8*64+2*dist64	;; Loop 8 times, next source pointer
	yloop	4, c3d, -4*2*dist64+32*64, rdi, -32*YMM_SCD2/4 ;; Loop 4 times, next source pointer, restore sine/cosine pointer

	yloop	2, b5d0, -2*32*64+dist64	;; Loop 2 times, next source pointer
	yloop	2, b5d0, -2*dist64	;; Loop 2 times, restore source pointer
;;	bump	rcx, (-8-3*128+4*64+8+2*64)*64 ;; Move prefetch pointer to where squaring would be
	end_timer 6

;; Do inverse FFT levels 3,4
;; On input the 64-byte cache lines hold these data values:
;;	0	+1K	+1K	+1K	128	+1K	+1K	+1K
;;	4	...
;;	...
;;	124	...
;;	256	...
;;	...
;;	1	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+1K	+1K	+1K	512	+1K	+1K	+1K
;;	4	...
;;	...
;;	508	...
;;	1	...
;;	...

;; Do 512 four_complex_unfft operations
;;	distance between fft data elements is 128
;;	do 128 macros each processing 32 data values

;; Do inverse FFT levels 1,2
;; On input the 64-byte cache lines hold these data values:
;;	0	+1K	+1K	+1K	512	+1K	+1K	+1K
;;	4	...
;;	...
;;	508	...
;;	1	...
;;	...
;; On output the 64-byte cache lines are swizzled to hold these data values:
;;	0	+1	+1	+1	2048	+1	+1	+1
;;	4	...
;;	...
;;	2044

;; Do 512 four_complex_unfft operations
;;	distance between fft data elements is 512
;;	do 128 macros each processing 32 data values

ypass2_11_levels_complex_unfft:
	mov	rdi, xsincos_complex	;; Load sin/cos pointers
	mov	rbp, rdi
	bump	rcx, (512+7-4*64-8-2*64-2*64)*64 ;; Move prefetch pointer along
	yloop_init 128			;; 128 iterations
	yloop_set_prefetch YHALF

c2a:	start_timer 13
	yr4_eb4cl_four_complex_djbunfft_preload
c2b:	yr4_eb4cl_four_complex_djbunfft rsi, 2*dist64, 32*64, dist64, rbp, YMM_SCD2, 4
	yloop	4, c2b, -4*2*dist64	;; Test loop counter, restore source pointer
	loops_undo 4			;; Restore loop counter
	end_timer 13

	start_timer 14
	yr4_s4cl_four_complex_djbunfft_preload
c1b:	yr4_s4cl_four_complex_djbunfft rsi, 32*64, 2*dist64, 4*dist64, rdi, 32*YMM_SCD2, 2, L1PREFETCH_ALL, 64
	yloop	2, c1b, -2*32*64+dist64	;; Test loop counter, next source pointer
	yloop	2, c1b, -2*dist64+64, rdi, -4*32*YMM_SCD2+YMM_SCD2 ;; Test loop counter, next source pointer, next sine/cosine pointer
	end_timer 14

	yloop	32, c2a			;; Loop 32 times

ypass2_11_levels_complex_done:
	ENDM


; *************** radix4-pass2-2560-real macro ******************
; This macro takes 5120 real values and performs the final 12.322 levels of the FFT.
; The results are squared, and the first 12.322 levels of the inverse FFT are done.

yr4dwpn_pass2_2560_real MACRO
	LOCAL	b1b, b2a, b2b, b3a, b3b, b4a, b4b, b5a, b5b, b5c, b5d
	LOCAL	c1b, c2a, c2b, c3a, c3b, c4a, c4b
	LOCAL	ypass2_2560_real_1, ypass2_2560_real_2
	LOCAL	ypass2_2560_real_3, ypass2_2560_real_4
	LOCAL	ypass2_2560_real_unfft, ypass2_2560_real_done

;; Type 4 FFTs skip the forward FFT process

	start_timer 2
	mov	rsi, DATA_ADDR		;; Load source address
	mov	rbx, DIST_TO_FFTSRCARG
	mov	rcx, DATA_PREFETCH	;; Prefetch pointer (we need to prefetch 
					;; 640 cache lines + 2*9 pad cache lines)
	cmp	ffttype, 4
	je	ypass2_2560_real_4

;; Do FFT levels 1,2,3 which correspond to complex levels 1,2
;; On input the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	2560	+1	+1	+1
;;	4	...
;;	...
;;	2556
;; On output the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	512	+1	+1	+1
;;	4	...
;;	...
;;	508	...
;;	1024	...
;;	...

;; Do 512 ten_reals_fft operations
;;	distance between fft data elements is 512
;;	do 128 macros each processing 40 data values

	mov	rdi, sincos3		;; Real sin/cos data
	mov	rbp, xsincos_complex	;; Complex sin/cos data
	yloop_init 128			;; 128 iterations
	yloop_set_prefetch YSINGLE
	yr5_f5cl_2sc_ten_reals_fft_preload
b1b:	yr5_f5cl_2sc_ten_reals_fft rsi, 64, 2*dist64, rbp, YMM_SCD2, rdi, YMM_SCD2, 64
	yloop	64, b1b, -64*64+dist64	;; Test loop counter, next source pointer
	yloop	2, b1b, -2*dist64	;; Test loop counter, restore source pointer

;; Do FFT levels 3,4
;; On input the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	512	+1	+1	+1
;;	4	...
;;	...
;;	508	...
;;	1024	...
;;	...
;; On output the 64-byte cache lines are swizzled to hold these data values:
;;	0	+256	+256	+256	128	+256	+256	+256
;;	4	...
;;	...
;;	124	...
;;	1	...
;;	...
;;	1024	...
;;	...

;; Do 128 eight_reals_fft operations
;; Do 512 four_complex_djbfft operations
;;	distance between fft data elements is 128
;;	do 160 macros each processing 32 data values

	yloop_init 160			;; 5 iterations of 32
	yloop_set_prefetch YSINGLE
	yr4_s4cl_2sc_eight_reals_fft_preload
b2a:	yr4_s4cl_2sc_eight_reals_fft rsi, 64, 32*64, dist64, rbp, YMM_SCD2, rdi, YMM_SCD2, 32
	yloop	32, b2a, -32*64+2*dist64, rbp, -32*YMM_SCD2 ;; Test loop counter, next source pointer, restore sine/cosine pointer

	loops_reset			;; 4 more iters of 32
	yr4_s4cl_four_complex_djbfft_preload
b2b:	yr4_s4cl_four_complex_djbfft rsi, 64, 32*64, dist64, rbp, YMM_SCD2, 32
	yloop	32, b2b, -32*64+2*dist64, rbp, -32*YMM_SCD2 ;; Test loop counter, next source pointer, restore sin/cos ptr
	yloop	5, b2b, -5*2*dist64	;; Loop 5 times, restore source pointer

;; Do FFT levels 5,6
;; On input the 64-byte cache lines hold these data values:
;;	0	+256	+256	+256	128	+256	+256	+256
;;	4	...
;;	...
;;	124	...
;;	1	...
;;	...
;;	1024	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+256	+256	+256	32	+256	+256	+256
;;	4	...
;;	...
;;	28	...
;;	64	...
;;	...
;;	1	...
;;	...
;;	1024	...
;;	...

;; Do 32 eight_reals_fft operations
;; Do 608 four_complex_djbfft operations
;;	distance between fft data elements is 32
;;	do 160 macros each processing 32 data values

	yloop_init 160			;; 5 iterations of 32
	yloop_set_prefetch YHALF
	yr4_4cl_eight_reals_four_complex_djbfft_preload
b3a:	yr4_4cl_eight_reals_four_complex_djbfft rsi, 32*64, 8*64, 16*64, rdi, YMM_SCD3, 2
	yloop	2, b3a, -2*32*64+dist64	;; Test loop counter, next source pointer
	yloop	2, b3a, -2*dist64+64	;; Test loop counter, next source pointer
	yloop	8, b3a, -8*64+2*dist64, rdi, -32*YMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer

	loops_reset			;; 4 more iters of 32
	yr4_rb4cl_four_complex_djbfft_preload
b3b:	yr4_rb4cl_four_complex_djbfft rsi, 32*64, 8*64, 16*64, rdi, YMM_SCD3, 2
	yloop	2, b3b, -2*32*64+dist64	;; Test loop counter, next source pointer
	yloop	2, b3b, -2*dist64+64	;; Test loop counter, next source pointer
	yloop	8, b3b, -8*64+2*dist64, rdi, -32*YMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer
	yloop	5, b3b, -5*2*dist64	;; Test loop counter, restore source pointer

;; Do FFT levels 7,8
;; On input the 64-byte cache lines hold these data values:
;;	0	+256	+256	+256	32	+256	+256	+256
;;	4	...
;;	...
;;	28	...
;;	64	...
;;	...
;;	1	...
;;	...
;;	1024	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+256	+256	+256	8	+256	+256	+256
;;	4	...
;;	16
;;	...
;;	1	...
;;	...
;;	1024	...
;;	...

;; Do 8 eight_reals_fft operations
;; Do 632 four_complex_djbfft operations
;;	distance between fft data elements is 8
;;	do 160 macros each processing 32 data values

	yloop_init 160			;; 20 iterations of 8
	yloop_set_prefetch YHALF
	yr4_4cl_eight_reals_four_complex_djbfft_preload
b4a:	yr4_4cl_eight_reals_four_complex_djbfft rsi, 32*64, 2*64, 4*64, rdi, 4*YMM_SCD3, 2
	yloop	2, b4a, -2*32*64+dist64	;; Test loop counter, next source pointer
	yloop	2, b4a, -2*dist64+64	;; Test loop counter, next source pointer
	yloop	2, b4a, -2*64+8*64, rdi, -8*4*YMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer

	loops_reset			;; 19 more iters of 8
	yr4_rb4cl_four_complex_djbfft_preload
b4b:	yr4_rb4cl_four_complex_djbfft rsi, 32*64, 2*64, 4*64, rdi, 4*YMM_SCD3, 2
	yloop	2, b4b, -2*32*64+dist64	;; Test loop counter, next source pointer
	yloop	2, b4b, -2*dist64+64	;; Test loop counter, next source pointer
	yloop	2, b4b, -2*64+8*64, rdi, -8*4*YMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer
	yloop	4, b4b, -4*8*64+2*dist64 ;; Test loop counter, next source pointer
	yloop	5, b4b, -5*2*dist64	;; Test loop counter, restore source pointer

;; Do FFT levels 9,10,11 as well as inverse FFT levels 9,10,11
;; On input and output the 64-byte cache lines hold these data values:
;;	0	+256	+256	+256	8	+256	+256	+256
;;	4	...
;;	16
;;	...
;;	1	...
;;	...
;;	1024	...
;;	...

;; Do 1 sixteen_reals_fft operations
;; Do 319 eight_complex_fft operations
;;	distance between fft data elements is 1
;;	do 80 macros each processing 64 data values

;; Execute the proper middle step

	cmp	ffttype, 2
	jl	ypass2_2560_real_1
	jg	ypass2_2560_real_3

ypass2_2560_real_2:
;;	yr8_8cl_sixteen_reals_eight_complex_with_square rsi, 2*64, 32*64, dist64, 64
	yr8_dispatch call, y8real_square, 32*64, dist64, 64
	yloop_init 80			;; 79 more iterations
	yloop_adjust 1
	yloop_set_prefetch YSINGLE
	yr8_8cl_eight_complex_with_square_preload
b5b:	yr8_8cl_eight_complex_with_square rsi, 2*64, 32*64, dist64, 64, 1
	yloop	16, b5b, -16*2*64+2*dist64	;; Loop 16 times, next source pointer
	yloop	5, b5b, -5*2*dist64	;; Loop 5 times, restore source pointer
	jmp	ypass2_2560_real_unfft

ypass2_2560_real_1:
;;	yr8_8cl_sixteen_reals_eight_complex_fft_final rsi, 2*64, 32*64, dist64, 64
	yr8_dispatch call, y8real_fft_final, 32*64, dist64, 64
	mov	rcx, xsincos_complex	;; Prefetch complex sin/cos data, 336 cache lines
	yloop_init 80			;; 79 more iterations
	yloop_adjust 1
	yloop_set_prefetch YDOUBLE
b5a:	yr8_8cl_eight_complex_fft_final rsi, 2*64, 32*64, dist64, 64
	yloop	16, b5a, -16*2*64+2*dist64	;; Loop 16 times, next source pointer
	yloop	5, b5a			;; Loop 5 times
	jmp	ypass2_2560_real_done

ypass2_2560_real_3:
x64	mov	r8, DEST2ARG
	mov	rbp, DIST_TO_MULSRCARG
;;	yr8_8cl_sixteen_reals_eight_complex_with_mult rsi, 2*64, 32*64, dist64, 64
	yr8_dispatch call, y8real_mult, 32*64, dist64, 64
	yloop_init 80			;; 79 more iterations
	yloop_adjust 1
	yloop_set_prefetch YSINGLE
	yr8_8cl_eight_complex_with_mult_preload
b5c:	yr8_8cl_eight_complex_with_mult rsi, 2*64, 32*64, dist64, 64
	yloop	16, b5c, -16*2*64+2*dist64	;; Loop 16 times, next source pointer
	yloop	5, b5c, -5*2*dist64	;; Loop 5 times, restore source pointer
x64	add	rsi, r8
	jmp	ypass2_2560_real_unfft

ypass2_2560_real_4:
	mov	rbp, DIST_TO_MULSRCARG
;;	yr8_8cl_sixteen_reals_eight_complex_with_mulf rsi, 2*64, 32*64, dist64, 64
	yr8_dispatch call, y8real_mulf, 32*64, dist64, 64
	yloop_init 80			;; 79 more iterations
	yloop_adjust 1
	yloop_set_prefetch YDOUBLE
	yr8_8cl_eight_complex_with_mulf_preload
b5d:	yr8_8cl_eight_complex_with_mulf rsi, 2*64, 32*64, dist64, 64
	yloop	16, b5d, -16*2*64+2*dist64	;; Loop 16 times, next source pointer
	yloop	5, b5d, -5*2*dist64	;; Loop 5 times, restore source pointer
;;	bump	rcx, (-158+128+160+2*80+79)*64 ;; Move prefetch pointer back to where squaring would be

;; Do inverse FFT levels 7,8
;; On input the 64-byte cache lines hold these data values:
;;	0	+256	+256	+256	8	+256	+256	+256
;;	4	...
;;	16
;;	...
;;	1	...
;;	...
;;	1024	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+256	+256	+256	32	+256	+256	+256
;;	4	...
;;	...
;;	28	...
;;	64	...
;;	...
;;	1	...
;;	...
;;	1024	...
;;	...

;; Do 8 eight_reals_unfft operations
;; Do 632 four_complex_djbunfft operations
;;	distance between fft data elements is 8
;;	do 160 macros processing 32 data values

ypass2_2560_real_unfft:
	mov	rdi, sincos3		;; Load sin/cos pointer
	bump	rdi, 32*YMM_SCD2+128*YMM_SCD2
	bump	rcx, (640+2*9-128-160-2*80-79-160)*64 ;; Move prefetch pointer along
	yloop_init 160			;; 20 iterations of 8
	yloop_set_prefetch YSINGLE
	yr4_4cl_eight_reals_four_complex_djbunfft_preload
c4a:	yr4_4cl_eight_reals_four_complex_djbunfft rsi, 32*64, 2*64, 4*64, rdi, 4*YMM_SCD3, 2
	yloop	2, c4a, -2*32*64+dist64	;; Test loop counter, next source pointer
	yloop	2, c4a, -2*dist64+64	;; Test loop counter, next source pointer
	yloop	2, c4a, -2*64+8*64, rdi, -8*4*YMM_SCD3 ;; Test loop counter, next source pointer, restore sine/cosine pointer

	loops_reset			;; 19 more iters of 8
	yr4_rb4cl_four_complex_djbunfft_preload
c4b:	yr4_rb4cl_four_complex_djbunfft rsi, 32*64, 2*64, 4*64, rdi, 4*YMM_SCD3, 2
	yloop	2, c4b, -2*32*64+dist64	;; Test loop counter, next source pointer
	yloop	2, c4b, -2*dist64+64	;; Test loop counter, next source pointer
	yloop	2, c4b, -2*64+8*64, rdi, -8*4*YMM_SCD3 ;; Test loop counter, next source pointer, restore sine/cosine pointer
	yloop	4, c4b, -4*8*64+2*dist64	;; Test loop counter, next source pointer
	yloop	5, c4b, -5*2*dist64	;; Test loop counter, restore source pointer

;; Do inverse FFT levels 5,6
;; On input the 64-byte cache lines hold these data values:
;;	0	+256	+256	+256	32	+256	+256	+256
;;	4	...
;;	...
;;	28	...
;;	64	...
;;	...
;;	1	...
;;	...
;;	1024	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+256	+256	+256	128	+256	+256	+256
;;	4	...
;;	...
;;	124	...
;;	1	...
;;	...
;;	1024	...
;;	...

;; Do 32 eight_reals_unfft operations
;; Do 608 four_complex_djbunfft operations
;;	distance between fft data elements is 32
;;	do 160 macros processing 32 data values

	mov	rcx, xsincos_complex	;; Prefetch complex sin/cos data, 336 cache lines
	yloop_init 160			;; 5 iterations of 32
	yloop_set_prefetch YHALF
	yr4_4cl_eight_reals_four_complex_djbunfft_preload
c3a:	yr4_4cl_eight_reals_four_complex_djbunfft rsi, 32*64, 8*64, 16*64, rdi, YMM_SCD3, 2
	yloop	2, c3a, -2*32*64+dist64	;; Test loop counter, next source pointer
	yloop	2, c3a, -2*dist64+64	;; Test loop counter, next source pointer
	yloop	8, c3a, -8*64+2*dist64, rdi, -32*YMM_SCD3 ;; Test loop counter, next source pointer, restore sine/cosine pointer

	loops_reset			;; 4 more iters of 32
	yr4_rb4cl_four_complex_djbunfft_preload
c3b:	yr4_rb4cl_four_complex_djbunfft rsi, 32*64, 8*64, 16*64, rdi, YMM_SCD3, 2
	yloop	2, c3b, -2*32*64+dist64	;; Test loop counter, next source pointer
	yloop	2, c3b, -2*dist64+64	;; Test loop counter, next source pointer
	yloop	8, c3b, -8*64+2*dist64, rdi, -32*YMM_SCD3 ;; Test loop counter, next source pointer, restore sine/cosine pointer
	yloop	5, c3b, -5*2*dist64	;; Test loop counter, restore source pointer

;; Do inverse FFT levels 3,4
;; On input the 64-byte cache lines hold these data values:
;;	0	+256	+256	+256	128	+256	+256	+256
;;	4	...
;;	...
;;	124	...
;;	1	...
;;	...
;;	1024	...
;;	...
;; On output the 64-byte cache lines are swizzled to hold these data values:
;;	0	+1	+1	+1	512	+1	+1	+1
;;	4	...
;;	...
;;	508	...
;;	1024	...
;;	...

;; Do 128 eight_reals_unfft operations
;; Do 512 four_complex_djbunfft operations
;;	distance between fft data elements is 128
;;	do 160 macros processing 32 data values

	bump	rdi, -32*YMM_SCD2	;; Real sin/cos ptr
	mov	rbp, xsincos_complex	;; Complex sin/cos data
	bump	rbp, 128*YMM_SCD2
	yloop_init 160			;; 5 iterations of 32
	yloop_set_prefetch YSINGLE
	yr4_s4cl_2sc_eight_reals_unfft_preload
c2a:	yr4_s4cl_2sc_eight_reals_unfft rsi, 64, 32*64, dist64, rbp, YMM_SCD2, rdi, YMM_SCD2, 32
	yloop	32, c2a, -32*64+2*dist64, rbp, -32*YMM_SCD2 ;; Test loop counter, next source pointer, restore sine/cosine pointer

	loops_reset			;; 4 more iters of 32
	yr4_s4cl_four_complex_djbunfft_preload
c2b:	yr4_s4cl_four_complex_djbunfft rsi, 64, 32*64, dist64, rbp, YMM_SCD2, 32
	yloop	32, c2b, -32*64+2*dist64, rbp, -32*YMM_SCD2 ;; Test loop counter, next source pointer, restore sine/cosine pointer
	yloop	5, c2b, -5*2*dist64	;; Test loop counter, restore source pointer

;; Do inverse FFT levels 1,2,3 which correspond to complex levels 1,2
;; On input the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	512	+1	+1	+1
;;	4	...
;;	...
;;	508	...
;;	1024	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	2560	+1	+1	+1
;;	4	...
;;	...
;;	2556

;; Do 512 ten_reals_unfft operations
;;	distance between fft data elements is 512
;;	do 128 macros processing 40 data values

	bump	rdi, -32*YMM_SCD2-128*YMM_SCD2 ;; Real sin/cos ptr
	bump	rbp, -128*YMM_SCD2	;; Complex sin/cos data
	bump	rcx, (336-80-160-128)*64 ;; Move prefetch pointer along
	yloop_init 128			;; 128 iterations
	yloop_set_prefetch YSINGLE
	yr5_5cl_2sc_ten_reals_unfft_preload
c1b:	yr5_5cl_2sc_ten_reals_unfft rsi, 64, 2*dist64, rbp, YMM_SCD2, rdi, YMM_SCD2, 64
	yloop	64, c1b, -64*64+dist64	;; Test loop counter, next source pointer
	yloop	2, c1b			;; Test loop counter

ypass2_2560_real_done:
	end_timer 2
	ENDM

; *************** radix4-pass2-2560-complex macro ******************
; This macro takes 2560 complex values and performs the final 11.322 levels of a standard
; mostly radix-4 FFT, squares the results, and does the first 11.322 levels of the inverse FFT.

yr4dwpn_pass2_2560_complex MACRO
	LOCAL	b1a, b1b, b2b, b3a0, b3a, b3b0, b3b, b3c0, b3c, b4a, b4b, b4c, b5a, b5b, b5c, b5d0, b5d
	LOCAL	c1b, c2a, c2b, c3b, c3c, c3d, c4b, c4c, c4d
	LOCAL	ypass2_2560_complex_1, ypass2_2560_complex_2
	LOCAL	ypass2_2560_complex_3, ypass2_2560_complex_4
	LOCAL	ypass2_2560_complex_unfft, ypass2_2560_complex_done

;; Type 4 FFTs skip the forward FFT process

	mov	rsi, DATA_ADDR		;; Load address of FFT data
	mov	rbx, DIST_TO_FFTSRCARG
	mov	rcx, DATA_PREFETCH	;; Prefetch pointer (we need to prefetch 
					;; 640 cache lines + 2*9 pad cache lines)
	cmp	ffttype, 4
	je	ypass2_2560_complex_4

;; Do FFT levels 1,2 (actually 2.322 levels)
;; On input the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	2560	+1	+1	+1
;;	4	...
;;	...
;;	2556
;; On output the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	512	+1	+1	+1
;;	4	...
;;	...
;;	508	...
;;	1024	...
;;	...

;; Do 512 five_complex_fft operations
;;	distance between fft data elements is 512
;;	do 128 macros each processing 40 data values

;; Do FFT levels 3,4
;; On input the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	512	+1	+1	+1
;;	4	...
;;	...
;;	508	...
;;	1024	...
;;	...
;; On output the 64-byte cache lines are swizzled to hold these data values:
;;	0	+256	+256	+256	128	+256	+256	+256
;;	4	...
;;	...
;;	124	...
;;	1	...
;;	...
;;	1024	...
;;	...

;; Do 640 four_complex_fft operations
;;	distance between fft data elements is 128
;;	do 160 macros each processing 32 data values

	mov	rdi, xsincos_complex	;; The sin/cos ptrs
	lea	rbp, [rdi+128*YMM_SCD2]
	yloop_init 256			;; 128 iterations or 160 iterations

b1a:	start_timer 4
	yloop_adjust 2*2		;; Four fewer iterations this loop
	yloop_set_prefetch YSINGLE	;; Prefetch 128 cache lines in this loop
	yr5_f5cl_five_complex_djbfft_preload
b1b:	yr5_f5cl_five_complex_djbfft rsi, 32*64, 2*dist64, rdi, 32*YMM_SCD2, 2
	yloop	2, b1b, -2*32*64+dist64	;; Test loop counter, next source pointer
	yloop	4, b1b, -2*dist64, rdi, -4*32*YMM_SCD2+YMM_SCD2 ;; 2 iterations!!  Restore source pointer, next sin/cos data
	loops_undo 4*2			;; Restore loop counter
	end_timer 4

	start_timer 5
	yloop_adjust 3			;; Three fewer iterations this loop
	yloop_set_prefetch YOTHER, 8	;; Prefetch 32 cache lines in this loop
	yr4_s4cl_four_complex_djbfft_preload
b2b:	yr4_s4cl_four_complex_djbfft rsi, 2*dist64, 32*64, dist64, rbp, 0, 5, L1PREFETCH_ALL, 64
	yloop	8, b2b, -5*2*dist64+64, rbp, YMM_SCD2 ;; 5 iterations!!  Next source pointer, next sin/cos data
	end_timer 5

	yloop	32, b1a, -32*64, rdi, -32*YMM_SCD2+128*YMM_SCD2+32*YMM_SCD2 ;; Test loop counter, restore source pointer, next sin/cos ptr

;; Do FFT levels 5,6
;; On input the 64-byte cache lines hold these data values:
;;	0	+256	+256	+256	128	+256	+256	+256
;;	4	...
;;	...
;;	124	...
;;	1	...
;;	...
;;	1024	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+256	+256	+256	32	+256	+256	+256
;;	4	...
;;	...
;;	28	...
;;	64	...
;;	...
;;	1	...
;;	...
;;	1024	...
;;	...

;; Do 640 four_complex_fft operations
;;	distance between fft data elements is 32
;;	do 160 macros each processing 32 data values

;; Do FFT levels 7,8
;; On input the 64-byte cache lines hold these data values:
;;	0	+256	+256	+256	32	+256	+256	+256
;;	4	...
;;	...
;;	28	...
;;	64	...
;;	...
;;	1	...
;;	...
;;	1024	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+256	+256	+256	8	+256	+256	+256
;;	4	...
;;	16
;;	...
;;	1	...
;;	...
;;	1024	...
;;	...

;; Do 640 four_complex_fft operations
;;	distance between fft data elements is 8
;;	do 160 macros each processing 32 data values

;; Do FFT levels 9,10,11 as well as inverse FFT levels 9,10,11
;; On input and output the 64-byte cache lines hold these data values:
;;	0	+256	+256	+256	8	+256	+256	+256
;;	4	...
;;	16
;;	...
;;	1	...
;;	...
;;	1024	...
;;	...

;; Do 320 eight_complex_fft operations
;;	distance between fft data elements is 1
;;	do 80 macros each processing 64 data values

;; Do inverse FFT levels 7,8
;; On input the 64-byte cache lines hold these data values:
;;	0	+256	+256	+256	8	+256	+256	+256
;;	4	...
;;	16
;;	...
;;	1	...
;;	...
;;	1024	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+256	+256	+256	32	+256	+256	+256
;;	4	...
;;	...
;;	28	...
;;	64	...
;;	...
;;	1	...
;;	...
;;	1024	...
;;	...

;; Do 640 four_complex_unfft operations
;;	distance between fft data elements is 8
;;	do 160 macros each processing 32 data values

;; Do inverse FFT levels 5,6
;; On input the 64-byte cache lines hold these data values:
;;	0	+256	+256	+256	32	+256	+256	+256
;;	4	...
;;	...
;;	28	...
;;	64	...
;;	...
;;	1	...
;;	...
;;	1024	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+256	+256	+256	128	+256	+256	+256
;;	4	...
;;	...
;;	124	...
;;	1	...
;;	...
;;	1024	...
;;	...

;; Do 640 four_complex_unfft operations
;;	distance between fft data elements is 32
;;	do 160 macros each processing 32 data values

;; Do this 128 cache lines (8KB) at a time to maximize accessing data
;; while it is in the L1 cache.

;; Execute the right middle step

	cmp	ffttype, 2
	jl	ypass2_2560_complex_1
	jg	ypass2_2560_complex_3

ypass2_2560_complex_2:
	yloop_init 160			;; 5 iters of 128 cache lines

b3b0:	start_timer 6
	yloop_set_prefetch YHALF	;; Prefetch 80 cache lines in this loop
	yr4_b4cl_four_complex_djbfft_preload
b3b:	yr4_b4cl_four_complex_djbfft rsi, 64, 8*64, 16*64, rdi, YMM_SCD2/4, 8
	yloop	8, b3b, -8*64+32*64	;; Loop 8 times, next source pointer
	yloop	2, b3b, -2*32*64+dist64	;; Loop 2 times, next source pointer
	yloop	2, b3b, -2*dist64, rdi, -32*YMM_SCD2/4 ;; Loop 2 times, restore source pointer, restore sine/cosine pointer
	loops_undo 4*8			;; Back up loop counter
	end_timer 6

	start_timer 7
	yloop_set_prefetch YHALF	;; Prefetch 80 cache lines in this loop
;;	yr4_b4cl_four_complex_djbfft_preload
b4b:	yr4_b4cl_four_complex_djbfft rsi, 64, 2*64, 4*64, rdi, 4*YMM_SCD2/4, 2
	yloop	2, b4b, -2*64+8*64, rdi, -2*4*YMM_SCD2/4 ;; Loop 2 times, next source pointer, restore sine/cosine pointer
	yloop	4, b4b, -4*8*64+32*64, rdi, YMM_SCD2/4 ;; Loop 4 times, next source pointer, next sine/cosine pointer
	yloop	2, b4b, -2*32*64+dist64	;; Loop 2 times, next source pointer
	yloop	2, b4b, -2*dist64, rdi, -4*YMM_SCD2/4 ;; Loop 2 times, restore source pointer, restore sine/cosine pointer
	loops_undo 4*4*2		;; Back up loop counter
	end_timer 7

	start_timer 9
	yloop_adjust 2*8		;; 16 fewer iterations this loop
	yloop_set_prefetch YOTHER, 8	;; Prefetch 20 cache lines in this loop
	yr8_8cl_eight_complex_with_square_preload
;; Loop below reworked to avoid 4KB distances
;;b5b:	yr8_8cl_eight_complex_with_square rsi, 2*64, 32*64, dist64, 64, 32
;;	yloop	32, b5b, -16*2*64	;; 16 iterations!!  Restore source pointer
b5b:	yr8_8cl_eight_complex_with_square rsi, 4*64, 32*64, dist64, 64, 8
	yloop	8, b5b, -8*4*64+2*64	;; Loop 8 times, next source pointer
	yloop	4, b5b, -2*2*64		;; 2 iterations!!  Restore source pointer
	loops_undo 4*8			;; Back up loop counter
	end_timer 9

	start_timer 10
	yloop_set_prefetch YHALF	;; Prefetch 80 cache lines in this loop
	yr4_b4cl_four_complex_djbunfft_preload
c4b:	yr4_b4cl_four_complex_djbunfft rsi, 64, 2*64, 4*64, rdi, 4*YMM_SCD2/4, 2
	yloop	2, c4b, -2*64+8*64, rdi, -2*4*YMM_SCD2/4 ;; Loop 2 times, next source pointer, restore sine/cosine pointer
	yloop	4, c4b, -4*8*64+32*64, rdi, YMM_SCD2/4 ;; Loop 4 times, next source pointer, next sine/cosine pointer
	yloop	2, c4b, -2*32*64+dist64	;; Loop 2 times, next source pointer
	yloop	2, c4b, -2*dist64, rdi, -4*YMM_SCD2/4 ;; Loop 2 times, restore source pointer, restore sine/cosine pointer
	loops_undo 4*4*2		;; Back up loop counter
	end_timer 10

	start_timer 11
	yloop_set_prefetch YHALF	;; Prefetch 80 cache lines in this loop
;;	yr4_b4cl_four_complex_djbunfft_preload
c3b:	yr4_b4cl_four_complex_djbunfft rsi, 64, 8*64, 16*64, rdi, YMM_SCD2/4, 8, L1PREFETCH_ALL, 2*dist64
	yloop	8, c3b, -8*64+32*64	;; Loop 8 times, next source pointer
	yloop	2, c3b, -2*32*64+dist64	;; Loop 2 times, next source pointer
	yloop	2, c3b, -2*dist64+2*dist64, rdi, -32*YMM_SCD2/4 ;; Loop 2 times, next source pointer, restore sine/cosine pointer
	end_timer 11

	yloop	5, b3b0, -5*2*dist64	;; Loop 5 times, restore source pointer
	jmp	ypass2_2560_complex_unfft


ypass2_2560_complex_1:
	start_timer 6
;;	bump	rcx, (640+2*9-128-32-3*160)*64 ;; Move prefetch pointer along
	yloop_init 160			;; 5 iters of 128 cache lines

b3a0:	yloop_set_prefetch YSINGLE	;; Prefetch 160 cache lines in this loop
	yr4_b4cl_four_complex_djbfft_preload
b3a:	yr4_b4cl_four_complex_djbfft rsi, 64, 8*64, 16*64, rdi, YMM_SCD2/4, 8
	yloop	8, b3a, -8*64+32*64	;; Loop 8 times, next source pointer
	yloop	2, b3a, -2*32*64+dist64	;; Loop 2 times, next source pointer
	yloop	2, b3a, -2*dist64, rdi, -32*YMM_SCD2/4 ;; Loop 2 times, restore source pointer, restore sine/cosine pointer
	loops_undo 4*8			;; Back up loop counter

	yloop_set_prefetch YSINGLE	;; Prefetch 160 cache lines in this loop
;;	yr4_b4cl_four_complex_djbfft_preload
b4a:	yr4_b4cl_four_complex_djbfft rsi, 64, 2*64, 4*64, rdi, 4*YMM_SCD2/4, 2, L1PREFETCH_ALL, 2*dist64
	yloop	2, b4a, -2*64+8*64, rdi, -2*4*YMM_SCD2/4 ;; Loop 2 times, next source pointer, restore sine/cosine pointer
	yloop	4, b4a, -4*8*64+32*64, rdi, YMM_SCD2/4 ;; Loop 4 times, next source pointer, next sine/cosine pointer
	yloop	2, b4a, -2*32*64+dist64	;; Loop 2 times, next source pointer
	yloop	2, b4a, -2*dist64, rdi, -4*YMM_SCD2/4 ;; Loop 2 times, restore source pointer, restore sine/cosine pointer
	loops_undo 4*4*2		;; Back up loop counter

	yloop_adjust 2*8		;; 16 fewer iterations this loop
	yloop_set_prefetch YDOUBLE	;; Prefetch 160 cache lines in this loop
	yr8_8cl_eight_complex_fft_final_preload
;; Loop below reworked to avoid 4KB distances
;;b5a:	yr8_8cl_eight_complex_fft_final rsi, 2*64, 32*64, dist64, 64
;;	yloop	32, b5a, -16*2*64	;; 16 iterations!!  Restore source pointer
b5a:	yr8_8cl_eight_complex_fft_final rsi, 4*64, 32*64, dist64, 64
	yloop	8, b5a, -8*4*64+2*64	;; Loop 8 times, next source pointer
	yloop	4, b5a, -2*2*64+2*dist64 ;; 2 iterations!!  Next source pointer

	yloop	5, b3a0, -5*2*dist64	;; Loop 5 times, restore source pointer
	end_timer 6
	jmp	ypass2_2560_complex_done


ypass2_2560_complex_3:
	start_timer 6
x64	mov	r8, DEST2ARG
	mov	rbp, DIST_TO_MULSRCARG
	yloop_init 160			;; 5 iters of 128 cache lines

b3c0:	yloop_set_prefetch YHALF	;; Prefetch 80 cache lines in this loop
	yr4_b4cl_four_complex_djbfft_preload
b3c:	yr4_b4cl_four_complex_djbfft rsi, 64, 8*64, 16*64, rdi, YMM_SCD2/4, 8
	yloop	8, b3c, -8*64+32*64	;; Loop 8 times, next source pointer
	yloop	2, b3c, -2*32*64+dist64	;; Loop 2 times, next source pointer
	yloop	2, b3c, -2*dist64, rdi, -32*YMM_SCD2/4 ;; Loop 2 times, restore source pointer, restore sine/cosine pointer
	loops_undo 4*8			;; Back up loop counter

	yloop_set_prefetch YHALF	;; Prefetch 80 cache lines in this loop
;;	yr4_b4cl_four_complex_djbfft_preload
b4c:	yr4_b4cl_four_complex_djbfft rsi, 64, 2*64, 4*64, rdi, 4*YMM_SCD2/4, 2
	yloop	2, b4c, -2*64+8*64, rdi, -2*4*YMM_SCD2/4 ;; Loop 2 times, next source pointer, restore sine/cosine pointer
	yloop	4, b4c, -4*8*64+32*64, rdi, YMM_SCD2/4 ;; Loop 4 times, next source pointer, next sine/cosine pointer
	yloop	2, b4c, -2*32*64+dist64	;; Loop 2 times, next source pointer
	yloop	2, b4c, -2*dist64, rdi, -4*YMM_SCD2/4 ;; Loop 2 times, restore source pointer, restore sine/cosine pointer
	loops_undo 4*4*2		;; Back up loop counter

	yloop_adjust 2*8		;; 16 fewer iterations this loop
	yloop_set_prefetch YOTHER, 8	;; Prefetch 20 cache lines in this loop
	yr8_8cl_eight_complex_with_mult_preload
;; Loop below reworked to avoid 4KB distances
;;b5c:	yr8_8cl_eight_complex_with_mult rsi, 2*64, 32*64, dist64, 64
;;	yloop	32, b5c, -16*2*64	;; 16 iterations!!  Restore source pointer
b5c:	yr8_8cl_eight_complex_with_mult rsi, 4*64, 32*64, dist64, 64
	yloop	8, b5c, -8*4*64+2*64	;; Loop 8 times, next source pointer
	yloop	4, b5c, -2*2*64		;; 2 iterations!!  Restore source pointer
x64	add	rsi, r8
	loops_undo 4*8			;; Back up loop counter

	yloop_set_prefetch YHALF	;; Prefetch 80 cache lines in this loop
	yr4_b4cl_four_complex_djbunfft_preload
c4c:	yr4_b4cl_four_complex_djbunfft rsi, 64, 2*64, 4*64, rdi, 4*YMM_SCD2/4, 2
	yloop	2, c4c, -2*64+8*64, rdi, -2*4*YMM_SCD2/4 ;; Loop 2 times, next source pointer, restore sine/cosine pointer
	yloop	4, c4c, -4*8*64+32*64, rdi, YMM_SCD2/4 ;; Loop 4 times, next source pointer, next sine/cosine pointer
	yloop	2, c4c, -2*32*64+dist64	;; Loop 2 times, next source pointer
	yloop	2, c4c, -2*dist64, rdi, -4*YMM_SCD2/4 ;; Loop 2 times, restore source pointer, restore sine/cosine pointer
	loops_undo 4*4*2		;; Back up loop counter

	yloop_set_prefetch YHALF	;; Prefetch 80 cache lines in this loop
;;	yr4_b4cl_four_complex_djbunfft_preload
c3c:	yr4_b4cl_four_complex_djbunfft rsi, 64, 8*64, 16*64, rdi, YMM_SCD2/4, 8, L1PREFETCH_ALL, 2*dist64
	yloop	8, c3c, -8*64+32*64	;; Loop 8 times, next source pointer
	yloop	2, c3c, -2*32*64+dist64	;; Loop 2 times, next source pointer
	yloop	2, c3c, -2*dist64+2*dist64, rdi, -32*YMM_SCD2/4 ;; Loop 2 times, next source pointer, restore sine/cosine pointer

x64	sub	rsi, r8
	yloop	5, b3c0, -5*2*dist64	;; Loop 5 times, restore source pointer
x64	add	rsi, r8
	end_timer 6
	jmp	ypass2_2560_complex_unfft


ypass2_2560_complex_4:
	start_timer 6
	mov	rdi, xsincos_complex	;; The sin/cos ptrs
	bump	rdi, 128*YMM_SCD2+32*YMM_SCD2
	mov	rbp, DIST_TO_MULSRCARG
	yloop_init 160			;; 5 iters of 128 cache lines

b5d0:	yloop_adjust 2*8		;; 16 fewer iterations this loop
	yloop_set_prefetch YDOUBLE	;; Prefetch 160 cache lines in this loop
	yr8_8cl_eight_complex_with_mulf_preload
;; Loop below reworked to avoid 4KB distances
;;b5d:	yr8_8cl_eight_complex_with_mulf rsi, 2*64, 32*64, dist64, 64
;;	yloop	32, b5d, -16*2*64	;; 16 iterations!!  Restore source pointer
b5d:	yr8_8cl_eight_complex_with_mulf rsi, 4*64, 32*64, dist64, 64
	yloop	8, b5d, -8*4*64+2*64	;; Loop 8 times, next source pointer
	yloop	4, b5d, -2*2*64		;; 2 iterations!!  Restore source pointer
	loops_undo 4*8			;; Back up loop counter

	yloop_set_prefetch YSINGLE	;; Prefetch 160 cache lines in this loop
	yr4_b4cl_four_complex_djbunfft_preload
c4d:	yr4_b4cl_four_complex_djbunfft rsi, 64, 2*64, 4*64, rdi, 4*YMM_SCD2/4, 2
	yloop	2, c4d, -2*64+8*64, rdi, -2*4*YMM_SCD2/4 ;; Loop 2 times, next source pointer, restore sine/cosine pointer
	yloop	4, c4d, -4*8*64+32*64, rdi, YMM_SCD2/4 ;; Loop 4 times, next source pointer, next sine/cosine pointer
	yloop	2, c4d, -2*32*64+dist64	;; Loop 2 times, next source pointer
	yloop	2, c4d, -2*dist64, rdi, -4*YMM_SCD2/4 ;; Loop 2 times, restore source pointer, restore sine/cosine pointer
	loops_undo 4*4*2		;; Back up loop counter

	yloop_set_prefetch YSINGLE	;; Prefetch 160 cache lines in this loop
;;	yr4_b4cl_four_complex_djbunfft_preload
c3d:	yr4_b4cl_four_complex_djbunfft rsi, 64, 8*64, 16*64, rdi, YMM_SCD2/4, 8, L1PREFETCH_ALL, 2*dist64
	yloop	8, c3d, -8*64+32*64	;; Loop 8 times, next source pointer
	yloop	2, c3d, -2*32*64+dist64	;; Loop 2 times, next source pointer
	yloop	2, c3d, -2*dist64+2*dist64, rdi, -32*YMM_SCD2/4 ;; Loop 2 times, next source pointer, restore sine/cosine pointer

	yloop	5, b5d0, -5*2*dist64	;; Loop 5 times, restore source pointer
;;	bump	rcx, (-3*160+128+32+2*80+10+2*80)*64 ;; Move prefetch pointer to where squaring would be
	end_timer 6

;; Do inverse FFT levels 3,4
;; On input the 64-byte cache lines hold these data values:
;;	0	+256	+256	+256	128	+256	+256	+256
;;	4	...
;;	...
;;	124	...
;;	1	...
;;	...
;;	1024	...
;;	...
;; On output the 64-byte cache lines are swizzled to hold these data values:
;;	0	+1	+1	+1	512	+1	+1	+1
;;	4	...
;;	...
;;	508	...
;;	1024	...
;;	...

;; Do 640 four_complex_unfft operations
;;	distance between fft data elements is 128
;;	do 160 macros each processing 32 data values

;; Do inverse FFT levels 1,2
;; On input the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	512	+1	+1	+1
;;	4	...
;;	...
;;	508	...
;;	1024	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	2560	+1	+1	+1
;;	4	...
;;	...
;;	1532

;; Do 512 five_complex_unfft operations
;;	distance between fft data elements is 512
;;	do 128 macros each processing 40 data values

ypass2_2560_complex_unfft:
	mov	rdi, xsincos_complex	;; Load sin/cos pointers
	lea	rbp, [rdi+128*YMM_SCD2]
	bump	rcx, (640+2*9-128-32-2*80-20-2*80-32-128)*64 ;; Move prefetch pointer along
	yloop_init 256			;; 128 iterations or 160 iterations

c2a:	start_timer 13
	yloop_adjust 3			;; Three fewer iterations in this loop
	yloop_set_prefetch YOTHER, 8	;; Prefetch 32 cache lines in this loop
	yr4_s4cl_four_complex_djbunfft_preload
c2b:	yr4_s4cl_four_complex_djbunfft rsi, 2*dist64, 32*64, dist64, rbp, 0, 5, L1PREFETCH_ALL, 64
	yloop	8, c2b, -5*2*dist64, rbp, YMM_SCD2 ;; 5 iterations!!  Restore source pointer, next sine/cosine pointer
	loops_undo 8			;; Restore loop counter
	end_timer 13

	start_timer 14
	yloop_adjust 4			;; Four fewer iterations in this loop
	yloop_set_prefetch YSINGLE	;; Prefetch 128 cache lines in this loop
	yr5_5cl_five_complex_djbunfft_preload
c1b:	yr5_5cl_five_complex_djbunfft rsi, 32*64, 2*dist64, rdi, 32*YMM_SCD2, 2
	yloop	2, c1b, -2*32*64+dist64	;; Test loop counter, next source pointer
	yloop	4, c1b, -2*dist64+64, rdi, -4*32*YMM_SCD2+YMM_SCD2 ;; 2 iterations!!  Next source pointer, next sine/cosine pointer
	end_timer 14

	yloop	32, c2a			;; Loop 32 times

ypass2_2560_complex_done:
	ENDM


;*********************************************************************************************
;			Macros derived from pass2-12-levels
;*********************************************************************************************


; *************** radix4-pass2-2304-real macro ******************
; This macro takes 4608 real values and performs the final 12.17 levels of the FFT.
; The results are squared, and the first 12.17 levels of the inverse FFT are done.

yr4dwpn_pass2_2304_real MACRO
	LOCAL	b1b, b2a, b2b, b3a, b3b, b4a, b4b, b5a, b5b, b6a, b6b, b6c, b6d
	LOCAL	c1b, c2a, c2b, c3a, c3b, c4a, c4b, c5a, c5b
	LOCAL	ypass2_2304_real_1, ypass2_2304_real_2
	LOCAL	ypass2_2304_real_3, ypass2_2304_real_4
 	LOCAL	ypass2_2304_real_unfft, ypass2_2304_real_done

;; Type 4 FFTs skip the forward FFT process

	start_timer 2
	mov	rsi, DATA_ADDR		;; Load source address
	mov	rbx, DIST_TO_FFTSRCARG
	mov	rcx, DATA_PREFETCH	;; Prefetch pointer (we need to prefetch 
					;; 576 cache lines + 8 pad cache lines)
	cmp	ffttype, 4
	je	ypass2_2304_real_4

;; Do FFT levels 1,2,3 which correspond to complex levels 1,2
;; On input the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	2304	+1	+1	+1
;;	4	...
;;	...
;;	2300
;; On output the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	768	+1	+1	+1
;;	4	...
;;	...
;;	764	...
;;	1536	...
;;	...

;; Do 768 six_reals_fft operations
;;	distance between fft data elements is 768
;;	do 192 macros each processing 24 data values

	mov	rdi, sincos3		;; Real sin/cos pointer
	mov	rbp, xsincos_complex	;; Complex sin/cos data
	yloop_init 192			;; 192 iterations
	yloop_set_prefetch YHALF
	yr3_f3cl_2sc_six_reals_fft_preload
b1b:	yr3_f3cl_2sc_six_reals_fft rsi, 64, 3*dist64, rbp, YMM_SCD1, rdi, YMM_SCD1, 64
	yloop	64, b1b, -64*64+dist64	;; Test loop counter, next source pointer
	yloop	3, b1b, -3*dist64	;; Test loop counter, restore source pointer

;; Do FFT levels 3,4
;; On input the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	768	+1	+1	+1
;;	4	...
;;	...
;;	764	...
;;	1536	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	256	+1	+1	+1
;;	4	...
;;	...
;;	252	...
;;	512	...
;;	...

;; Do 256 six_reals_fft operations
;; Do 512 three_complex_fft operations
;;	distance between fft data elements is 256
;;	do 192 macros each processing 24 data values

	yloop_init 192			;; 3 iterations of 64
	yloop_set_prefetch YHALF
	yr3_3cl_2sc_six_reals_fft_preload
b2a:	yr3_3cl_2sc_six_reals_fft rsi, 64, dist64, rbp, YMM_SCD1, rdi, YMM_SCD1, 64
	yloop	64, b2a, -64*64+3*dist64, rbp, -64*YMM_SCD1 ;; Test loop counter, next source pointer, restore sin/cos pointer

	loops_reset			;; 2 more iters of 64
	yr3_3cl_three_complex_djbfft_preload
b2b:	yr3_3cl_three_complex_djbfft rsi, 64, dist64, rbp, YMM_SCD1, 64
	yloop	64, b2b, -64*64+3*dist64, rbp, -64*YMM_SCD1 ;; Test loop counter, next source pointer, restore sin/cos pointer
	yloop	3, b2b, -3*3*dist64, rbp, 64*YMM_SCD1 ;; Test loop counter, restore source pointer, next sin/cos pointer

;; Do FFT levels 5,6
;; On input the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	256	+1	+1	+1
;;	4	...
;;	...
;;	252	...
;;	512	...
;;	...
;; On output the 64-byte cache lines are swizzled to hold these data values:
;;	0	+128	+128	+128	64	+128	+128	+128
;;	4	...
;;	...
;;	60	...
;;	1	...
;;	...
;;	512	...
;;	...

;; Do 64 eight_reals_fft operations
;; Do 512 four_complex_fft operations
;;	distance between fft data elements is 64
;;	do 144 macros each processing 32 data values

	yloop_init 144			;; 9 iters of 16
	yloop_set_prefetch YHALF
	yr4_s4cl_2sc_eight_reals_fft_preload
b3a:	yr4_s4cl_2sc_eight_reals_fft rsi, 64, 16*64, 32*64, rbp, YMM_SCD2, rdi, YMM_SCD2, 16
	yloop	16, b3a, -16*64+dist64, rbp, -16*YMM_SCD2 ;; Test loop counter, next source pointer, restore sine/cosine pointer

	loops_reset			;; 8 more iters of 16
	yr4_s4cl_four_complex_djbfft_preload
b3b:	yr4_s4cl_four_complex_djbfft rsi, 64, 16*64, 32*64, rbp, YMM_SCD2, 16
	yloop	16, b3b, -16*64+dist64, rbp, -16*YMM_SCD2 ;; Test loop counter, next source pointer, restore sin/cos ptr
	yloop	9, b3b, -9*dist64	;; Loop 9 times, restore source pointer

;; Do FFT levels 7,8
;; On input the 64-byte cache lines hold these data values:
;;	0	+128	+128	+128	64	+128	+128	+128
;;	4	...
;;	...
;;	60	...
;;	1	...
;;	...
;;	512	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+128	+128	+128	16	+128	+128	+128
;;	4	...
;;	...
;;	12	...
;;	32	...
;;	...
;;	1	...
;;	...
;;	512	...
;;	...

;; Do 16 eight_reals_fft operations
;; Do 560 four_complex_fft operations
;;	distance between fft data elements is 16
;;	do 144 macros each processing 32 data values

	yloop_init 144			;; 9 iterations of 16
	yloop_set_prefetch YHALF
	yr4_4cl_eight_reals_four_complex_djbfft_preload
b4a:	yr4_4cl_eight_reals_four_complex_djbfft rsi, 16*64, 4*64, 8*64, rdi, YMM_SCD3, 4
	yloop	4, b4a, -4*16*64+64	;; Test loop counter, next source pointer
	yloop	4, b4a, -4*64+dist64, rdi, -16*YMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer

	loops_reset			;; 8 more iters of 16
	yr4_rb4cl_four_complex_djbfft_preload
b4b:	yr4_rb4cl_four_complex_djbfft rsi, 16*64, 4*64, 8*64, rdi, YMM_SCD3, 4
	yloop	4, b4b, -4*16*64+64	;; Test loop counter, next source pointer
	yloop	4, b4b, -4*64+dist64, rdi, -16*YMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer
	yloop	9, b4b, -9*dist64	;; Test loop counter, restore source pointer

;; Do FFT levels 9,10
;; On input the 64-byte cache lines hold these data values:
;;	0	+128	+128	+128	16	+128	+128	+128
;;	4	...
;;	...
;;	12	...
;;	32	...
;;	...
;;	1	...
;;	...
;;	512	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+128	+128	+128	4	+128	+128	+128
;;	8	...
;;	...
;;	1	...
;;	...
;;	512	...
;;	...

;; Do 4 eight_reals_fft operations
;; Do 572 four_complex_fft operations
;;	distance between fft data elements is 4
;;	do 144 macros each processing 32 data values

	yloop_init 144			;; 36 iterations of 4
	yloop_set_prefetch YHALF
	yr4_4cl_eight_reals_four_complex_djbfft_preload
b5a:	yr4_4cl_eight_reals_four_complex_djbfft rsi, 16*64, 64, 2*64, rdi, 4*YMM_SCD3, 4
	yloop	4, b5a, -4*16*64+4*64, rdi, -4*4*YMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer

	loops_reset			;; 35 more iters of 4
	yr4_rb4cl_four_complex_djbfft_preload
b5b:	yr4_rb4cl_four_complex_djbfft rsi, 16*64, 64, 2*64, rdi, 4*YMM_SCD3, 4
	yloop	4, b5b, -4*16*64+4*64, rdi, -4*4*YMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer
	yloop	4, b5b, -4*4*64+dist64	;; Test loop counter, next source pointer
	yloop	9, b5b, -9*dist64	;; Test loop counter, restore source pointer

;; Do FFT levels 11,12 as well as inverse FFT levels 11,12
;; On input and output the 64-byte cache lines hold these data values:
;;	0	+128	+128	+128	4	+128	+128	+128
;;	8	...
;;	...
;;	1	...
;;	...
;;	512	...
;;	...

;; Do 1 eight_reals_fft operation
;; Do 575 four_complex_fft operations
;;	distance between fft data elements is 1
;;	do 144 macros each processing 32 data values

	cmp	ffttype, 2
	jl	ypass2_2304_real_1
	jg	ypass2_2304_real_3

ypass2_2304_real_2:
;;	yr4_4cl_eight_reals_four_complex_with_square rsi, 64, 16*64, 32*64
	yr4_dispatch call, yreal_square, 16*64, 32*64
	yloop_init 144			;; 143 more iterations
	yloop_adjust 1
	yloop_set_prefetch YQUARTER
	yr4_4cl_four_complex_with_square_preload
b6b:	yr4_4cl_four_complex_with_square rsi, 64, 16*64, 32*64, 1
	yloop	16, b6b, -16*64+dist64	;; Test loop counter, next source pointer
	yloop	9, b6b, -9*dist64	;; Test loop counter, restore source pointer
	jmp	ypass2_2304_real_unfft

ypass2_2304_real_1:
;;	yr4_4cl_eight_reals_four_complex_fft_final rsi, 64, 16*64, 32*64
	yr4_dispatch call, yreal_fft_final, 16*64, 32*64
	mov	rcx, xsincos_complex	;; Prefetch complex sin/cos data, 288 cache lines
	yloop_init 144			;; 143 more iterations
	yloop_adjust 1
	yloop_set_prefetch YDOUBLE
b6a:	yr4_4cl_four_complex_fft_final rsi, 64, 16*64, 32*64
	yloop	16, b6a, -16*64+dist64	;; Test loop counter, next source pointer
	yloop	9, b6a			;; Test loop counter
	jmp	ypass2_2304_real_done

ypass2_2304_real_3:
x64	mov	r8, DEST2ARG
	mov	rbp, DIST_TO_MULSRCARG
;;	yr4_4cl_eight_reals_four_complex_with_mult rsi, 64, 16*64, 32*64
	yr4_dispatch call, yreal_mult, 16*64, 32*64
	yloop_init 144			;; 143 more iterations
	yloop_adjust 1
	yloop_set_prefetch YQUARTER
	yr4_4cl_four_complex_with_mult_preload
b6c:	yr4_4cl_four_complex_with_mult rsi, 64, 16*64, 32*64
	yloop	16, b6c, -16*64+dist64	;; Test loop counter, next source pointer
	yloop	9, b6c, -9*dist64	;; Test loop counter, restore source pointer
x64	add	rsi, r8
	jmp	ypass2_2304_real_unfft

ypass2_2304_real_4:
	mov	rbp, DIST_TO_MULSRCARG
;;	yr4_4cl_eight_reals_four_complex_with_mulf rsi, 64, 16*64, 32*64
	yr4_dispatch call, yreal_mulf, 16*64, 32*64
	yloop_init 144			;; 143 more iterations
	yloop_adjust 1
	yloop_set_prefetch YDOUBLE
	yr4_4cl_four_complex_with_mulf_preload
b6d:	yr4_4cl_four_complex_with_mulf rsi, 64, 16*64, 32*64
	yloop	16, b6d, -16*64+dist64	;; Test loop counter, next source pointer
	yloop	9, b6d, -9*dist64	;; Test loop counter, restore source pointer
;;	bump	rcx, (-288+2*96+3*72+36)*64 ;; Move prefetch pointer back to where squaring would be

;; Do inverse FFT levels 9,10
;; On input the 64-byte cache lines hold these data values:
;;	0	+128	+128	+128	4	+128	+128	+128
;;	8	...
;;	...
;;	1	...
;;	...
;;	512	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+128	+128	+128	16	+128	+128	+128
;;	4	...
;;	...
;;	12	...
;;	32	...
;;	...
;;	1	...
;;	...
;;	512	...
;;	...

;; Do 4 eight_reals_unfft operations
;; Do 572 four_complex_unfft operations
;;	distance between fft data elements is 4
;;	do 192 macros each processing 32 data values

ypass2_2304_real_unfft:
	mov	rdi, sincos3		;; Load sin/cos pointer
	bump	rdi, 16*YMM_SCD2+64*YMM_SCD1+192*YMM_SCD1
	yloop_init 144			;; 36 iterations of 4
	yloop_set_prefetch YHALF
	yr4_4cl_eight_reals_four_complex_djbunfft_preload
c5a:	yr4_4cl_eight_reals_four_complex_djbunfft rsi, 16*64, 64, 2*64, rdi, 4*YMM_SCD3, 4
	yloop	4, c5a, -4*16*64+4*64, rdi, -4*4*YMM_SCD3 ;; Test loop counter, next source pointer, restore sine/cosine pointer

	loops_reset			;; 35 more iters of 4
	yr4_rb4cl_four_complex_djbunfft_preload
c5b:	yr4_rb4cl_four_complex_djbunfft rsi, 16*64, 64, 2*64, rdi, 4*YMM_SCD3, 4
	yloop	4, c5b, -4*16*64+4*64, rdi, -4*4*YMM_SCD3 ;; Test loop counter, next source pointer, restore sine/cosine pointer
	yloop	4, c5b, -4*4*64+dist64	;; Test loop counter, next source pointer
	yloop	9, c5b, -9*dist64	;; Test loop counter, restore source pointer

;; Do inverse FFT levels 7,8
;; On input the 64-byte cache lines hold these data values:
;;	0	+128	+128	+128	16	+128	+128	+128
;;	4	...
;;	...
;;	12	...
;;	32	...
;;	...
;;	1	...
;;	...
;;	512	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+128	+128	+128	64	+128	+128	+128
;;	4	...
;;	...
;;	60	...
;;	1	...
;;	...
;;	512	...
;;	...

;; Do 16 eight_reals_unfft operations
;; Do 560 four_complex_unfft operations
;;	distance between fft data elements is 16
;;	do 144 macros each processing 32 data values

	bump	rcx, (576+8-2*96-3*72-36-72-72)*64 ;; Move prefetch pointer along
	yloop_init 144			;; 9 iterations of 16
	yloop_set_prefetch YHALF
	yr4_4cl_eight_reals_four_complex_djbunfft_preload
c4a:	yr4_4cl_eight_reals_four_complex_djbunfft rsi, 16*64, 4*64, 8*64, rdi, YMM_SCD3, 4
	yloop	4, c4a, -4*16*64+64	;; Test loop counter, next source pointer
	yloop	4, c4a, -4*64+dist64, rdi, -16*YMM_SCD3 ;; Test loop counter, next source pointer, restore sine/cosine pointer

	loops_reset			;; 8 more iters of 16
	yr4_rb4cl_four_complex_djbunfft_preload
c4b:	yr4_rb4cl_four_complex_djbunfft rsi, 16*64, 4*64, 8*64, rdi, YMM_SCD3, 4
	yloop	4, c4b, -4*16*64+64	;; Test loop counter, next source pointer
	yloop	4, c4b, -4*64+dist64, rdi, -16*YMM_SCD3 ;; Test loop counter, next source pointer, restore sine/cosine pointer
	yloop	9, c4b, -9*dist64	;; Test loop counter, restore source pointer

;; Do inverse FFT levels 5,6
;; On input the 64-byte cache lines hold these data values:
;;	0	+128	+128	+128	64	+128	+128	+128
;;	4	...
;;	...
;;	60	...
;;	1	...
;;	...
;;	512	...
;;	...
;; On output the 64-byte cache lines are swizzled to hold these data values:
;;	0	+1	+1	+1	256	+1	+1	+1
;;	4	...
;;	...
;;	252	...
;;	512	...
;;	...

;; Do 64 eight_reals_unfft operations
;; Do 512 four_complex_unfft operations
;;	distance between fft data elements is 64
;;	do 144 macros each processing 32 data values

	bump	rdi, -16*YMM_SCD2	;; Real sin/cos ptr
	mov	rbp, xsincos_complex	;; Complex sin/cos data
	bump	rbp, 64*YMM_SCD1+192*YMM_SCD1
	mov	rcx, xsincos_complex	;; Prefetch complex sin/cos data, 288 cache lines
	yloop_init 144			;; 9 iters of 16
	yloop_set_prefetch YSINGLE
	yr4_s4cl_2sc_eight_reals_unfft_preload
c3a:	yr4_s4cl_2sc_eight_reals_unfft rsi, 64, 16*64, 32*64, rbp, YMM_SCD2, rdi, YMM_SCD2, 16
	yloop	16, c3a, -16*64+dist64, rbp, -16*YMM_SCD2 ;; Test loop counter, next source pointer, restore sine/cosine pointer

	loops_reset			;; 8 more iters of 16
	yr4_s4cl_four_complex_djbunfft_preload
c3b:	yr4_s4cl_four_complex_djbunfft rsi, 64, 16*64, 32*64, rbp, YMM_SCD2, 16
	yloop	16, c3b, -16*64+dist64, rbp, -16*YMM_SCD2 ;; Test loop counter, next source pointer, restore sine/cosine pointer
	yloop	9, c3b, -9*dist64	;; Test loop counter, restore source pointer

;; Do inverse FFT levels 3,4
;; On input the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	256	+1	+1	+1
;;	4	...
;;	...
;;	252	...
;;	512	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	768	+1	+1	+1
;;	4	...
;;	...
;;	764	...
;;	1536	...
;;	...

;; Do 256 six_reals_unfft operations
;; Do 512 three_complex_unfft operations
;;	distance between fft data elements is 256
;;	do 192 macros each processing 24 data values

	bump	rdi, -16*YMM_SCD2-64*YMM_SCD1 ;; Real sin/cos ptr
	bump	rbp, -64*YMM_SCD1	;; Complex sin/cos data
	yloop_init 192			;; 3 iters of 64
	yloop_set_prefetch YHALF
	yr3_3cl_2sc_six_reals_unfft_preload
c2a:	yr3_3cl_2sc_six_reals_unfft rsi, 64, dist64, rbp, YMM_SCD1, rdi, YMM_SCD1, 64
	yloop	64, c2a, -64*64+3*dist64, rbp, -64*YMM_SCD1 ;; Test loop counter, next source pointer, restore sine/cosine pointer

	loops_reset			;; 2 more iters of 64
	yr3_3cl_three_complex_djbunfft_preload
c2b:	yr3_3cl_three_complex_djbunfft rsi, 64, dist64, rbp, YMM_SCD1, 64
	yloop	64, c2b, -64*64+3*dist64, rbp, -64*YMM_SCD1 ;; Test loop counter, next source pointer, restore sine/cosine pointer
	yloop	3, c2b, -3*3*dist64	;; Test loop counter, restore source pointer

;; Do inverse FFT levels 1,2,3 which correspond to complex levels 1,2
;; On input the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	768	+1	+1	+1
;;	4	...
;;	...
;;	764	...
;;	1536	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	2304	+1	+1	+1
;;	4	...
;;	...
;;	2300

;; Do 768 six_reals_unfft operations
;;	distance between fft data elements is 768
;;	do 192 macros each processing 24 data values

	bump	rdi, -64*YMM_SCD1-192*YMM_SCD1 ;; Real sin/cos ptr
	bump	rbp, -192*YMM_SCD1	;; Complex sin/cos ptr
	bump	rcx, (288-144-96-48)*64 ;; Move prefetch pointer along
	yloop_init 192			;; 192 iterations
	yloop_set_prefetch YQUARTER
	yr3_3cl_2sc_six_reals_unfft_preload
c1b:	yr3_3cl_2sc_six_reals_unfft rsi, 64, 3*dist64, rbp, YMM_SCD1, rdi, YMM_SCD1, 64
	yloop	64, c1b, -64*64+dist64	;; Test loop counter, next source pointer
	yloop	3, c1b			;; Test loop counter

ypass2_2304_real_done:
	end_timer 2
	ENDM

; *************** radix4-pass2-2304-complex macro ******************
; This macro takes 2304 complex values and performs the final 11.17 levels of a standard
; radix-4 FFT, squares the results, and does the first 11.17 levels of the inverse FFT.

yr4dwpn_pass2_2304_complex MACRO
	LOCAL	b1a, b1b, b2b, b3a0, b3a, b3b0, b3b, b3c0, b3c, b4a, b4b, b4c, b5a, b5b, b5c, b6a, b6b, b6c, b6d
	LOCAL	c1b, c2a, c2b, c3b, c3c, c3d, c4b, c4c, c4d, c5b, c5c, c5d
	LOCAL	ypass2_2304_complex_1, ypass2_2304_complex_2
	LOCAL	ypass2_2304_complex_3, ypass2_2304_complex_4
	LOCAL	ypass2_2304_complex_unfft, ypass2_2304_complex_done

;; Type 4 FFTs skip the forward FFT process

	mov	rsi, DATA_ADDR		;; Load address of FFT data
	mov	rbx, DIST_TO_FFTSRCARG
	mov	rcx, DATA_PREFETCH	;; Prefetch pointer (we need to prefetch 
					;; 576 cache lines + 8 pad cache lines)
	cmp	ffttype, 4
	je	ypass2_2304_complex_4

;; Do FFT level 1,2 (actually 1.585 levels)
;; On input the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	2304	+1	+1	+1
;;	4	...
;;	...
;;	2300
;; On output the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	768	+1	+1	+1
;;	4	...
;;	...
;;	764	...
;;	1536	...
;;	...

;; Do 768 three_complex_fft operations
;;	distance between fft data elements is 768
;;	do 192 macros each processing 24 data values

;; Do FFT levels 3,4 (actually 1.585 levels)
;; On input the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	768	+1	+1	+1
;;	4	...
;;	...
;;	764	...
;;	1536	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	256	+1	+1	+1
;;	4	...
;;	...
;;	252	...
;;	512	...
;;	...

;; Do 768 three_complex_fft operations
;;	distance between fft data elements is 256
;;	do 192 macros each processing 24 data values

	mov	rdi, xsincos_complex	;; The sin/cos ptrs
	lea	rbp, [rdi+192*YMM_SCD1]
	yloop_init 256			;; 192 iterations

	yr3_f3cl_three_complex_djbfft_preload
b1a:	start_timer 4
	yloop_adjust 1
	yloop_set_prefetch YQUARTER	;; Prefetch 64 cache lines
b1b:	yr3_f3cl_three_complex_djbfft rsi, dist64, 3*dist64, rdi, 64*YMM_SCD1, 3
	yloop	4, b1b, -3*dist64, rdi, -3*64*YMM_SCD1+YMM_SCD1 ;; 3 iterations!!  Restore source pointer, next sin/cos data
	loops_undo 4
	end_timer 4

	start_timer 5
	yloop_adjust 1
	yloop_set_prefetch YQUARTER	;; Prefetch 64 cache lines
b2b:	yr3_3cl_three_complex_djbfft rsi, 3*dist64, dist64, rbp, 0, 3, L1PREFETCH_ALL, 64
	yloop	4, b2b, -3*3*dist64+64, rbp, YMM_SCD1 ;; 3 iterations!!  Next source pointer, next sin/cos data
	end_timer 5

	yloop	64, b1a, -64*64, rdi, -64*YMM_SCD1+192*YMM_SCD1+64*YMM_SCD1 ;; Test loop counter, restore source pointer, next sin/cos ptr

;; Do FFT levels 5,6
;; On input the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	256	+1	+1	+1
;;	4	...
;;	...
;;	252	...
;;	512	...
;;	...
;; On output the 64-byte cache lines are swizzled to hold these data values:
;;	0	+128	+128	+128	64	+128	+128	+128
;;	4	...
;;	...
;;	60	...
;;	1	...
;;	...
;;	512	...
;;	...

;; Do 576 four_complex_fft operations
;;	distance between fft data elements is 64
;;	do 144 macros each processing 32 data values

;; Do FFT levels 7,8
;; On input the 64-byte cache lines hold these data values:
;;	0	+128	+128	+128	64	+128	+128	+128
;;	4	...
;;	...
;;	60	...
;;	1	...
;;	...
;;	512	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+128	+128	+128	16	+128	+128	+128
;;	4	...
;;	...
;;	12	...
;;	32	...
;;	...
;;	1	...
;;	...
;;	512	...
;;	...

;; Do 576 four_complex_fft operations
;;	distance between fft data elements is 16
;;	do 144 macros each processing 32 data values

;; Do FFT levels 9,10
;; On input the 64-byte cache lines hold these data values:
;;	0	+128	+128	+128	16	+128	+128	+128
;;	4	...
;;	...
;;	12	...
;;	32	...
;;	...
;;	1	...
;;	...
;;	512	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+128	+128	+128	4	+128	+128	+128
;;	8	...
;;	...
;;	1	...
;;	...
;;	512	...
;;	...

;; Do 576 four_complex_fft operations
;;	distance between fft data elements is 4
;;	do 144 macros each processing 32 data values

;; Do FFT levels 11,12 as well as inverse FFT levels 11,12
;; On input and output the 64-byte cache lines hold these data values:
;;	0	+128	+128	+128	4	+128	+128	+128
;;	8	...
;;	...
;;	1	...
;;	...
;;	512	...
;;	...

;; Do 576 four_complex_fft operations
;;	distance between fft data elements is 1
;;	do 144 macros each processing 32 data values

;; Do inverse FFT levels 9,10
;; On input the 64-byte cache lines hold these data values:
;;	0	+128	+128	+128	4	+128	+128	+128
;;	8	...
;;	...
;;	1	...
;;	...
;;	512	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+128	+128	+128	16	+128	+128	+128
;;	4	...
;;	...
;;	12	...
;;	32	...
;;	...
;;	1	...
;;	...
;;	512	...
;;	...

;; Do 576 four_complex_unfft operations
;;	distance between fft data elements is 4
;;	do 144 macros each processing 32 data values

;; Do inverse FFT levels 7,8
;; On input the 64-byte cache lines hold these data values:
;;	0	+128	+128	+128	16	+128	+128	+128
;;	4	...
;;	...
;;	12	...
;;	32	...
;;	...
;;	1	...
;;	...
;;	512	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+128	+128	+128	64	+128	+128	+128
;;	4	...
;;	...
;;	60	...
;;	1	...
;;	...
;;	512	...
;;	...

;; Do 576 four_complex_unfft operations
;;	distance between fft data elements is 16
;;	do 144 macros each processing 32 data values

;; Do inverse FFT levels 5,6
;; On input the 64-byte cache lines hold these data values:
;;	0	+128	+128	+128	64	+128	+128	+128
;;	4	...
;;	...
;;	60	...
;;	1	...
;;	...
;;	512	...
;;	...
;; On output the 64-byte cache lines are swizzled to hold these data values:
;;	0	+1	+1	+1	256	+1	+1	+1
;;	4	...
;;	...
;;	252	...
;;	512	...
;;	...

;; Do 576 four_complex_unfft operations
;;	distance between fft data elements is 64
;;	do 144 macros each processing 32 data values

;; Do this 64 cache lines (4KB) at a time to maximize accessing data
;; while it is in the L1 cache.

;; Execute the right middle step

	cmp	ffttype, 2
	jl	ypass2_2304_complex_1
	jg	ypass2_2304_complex_3

ypass2_2304_complex_2:
	yloop_init 144			;; 9 iters of 64 cache lines

b3b0:	start_timer 6
	yloop_set_prefetch YHALF	;; Prefetch 72 cache lines in this loop
	yr4_s4cl_four_complex_djbfft_preload
b3b:	yr4_s4cl_four_complex_djbfft rsi, 64, 16*64, 32*64, rdi, YMM_SCD2, 16
	yloop	16, b3b, -16*64, rdi, -16*YMM_SCD2 ;; Loop 16 times, restore source pointer, restore sine/cosine pointer
	loops_undo 16			;; Back up loop counter
	end_timer 6

	start_timer 7
	yloop_set_prefetch YQUARTER	;; Prefetch 36 cache lines in this loop
	yr4_eb4cl_four_complex_djbfft_preload
b4b:	yr4_eb4cl_four_complex_djbfft rsi, 64, 4*64, 8*64, rdi, 4*YMM_SCD2, 4
	yloop	4, b4b, -4*64+16*64, rdi, -4*4*YMM_SCD2+YMM_SCD2 ;; Loop 4 times, next source pointer, next sine/cosine pointer
	yloop	4, b4b, -4*16*64, rdi, -4*YMM_SCD2 ;; Loop 4 times, restore source pointer, restore sine/cosine pointer
	loops_undo 4*4			;; Back up loop counter
	end_timer 7

	start_timer 8
	yloop_set_prefetch YQUARTER	;; Prefetch 36 cache lines in this loop
;;	yr4_eb4cl_four_complex_djbfft_preload
b5b:	yr4_eb4cl_four_complex_djbfft rsi, 4*64, 64, 2*64, rdi, 0, 4
	yloop	4, b5b, 0, rdi, 4*YMM_SCD2 ;; Loop 4 times, next sine/cosine pointer
	yloop	4, b5b, -16*4*64, rdi, -4*4*YMM_SCD2 ;; Loop 4 times, restore source pointer, restore sine/cosine pointer
	loops_undo 4*4			;; Back up loop counter
	end_timer 8

	start_timer 9
	yloop_set_prefetch YQUARTER	;; Prefetch 36 cache lines in this loop
	yr4_4cl_four_complex_with_square_preload
b6b:	yr4_4cl_four_complex_with_square rsi, 64, 16*64, 32*64, 16
	yloop	16, b6b, -16*64		;; Loop 16 times, restore source pointer
	loops_undo 16			;; Back up loop counter
	end_timer 9

	start_timer 10
	yloop_set_prefetch YQUARTER	;; Prefetch 36 cache lines in this loop
	yr4_eb4cl_four_complex_djbunfft_preload
c5b:	yr4_eb4cl_four_complex_djbunfft rsi, 4*64, 64, 2*64, rdi, 0, 4
	yloop	4, c5b, 0, rdi, 4*YMM_SCD2 ;; Loop 4 times, next sine/cosine pointer
	yloop	4, c5b, -16*4*64, rdi, -4*4*YMM_SCD2 ;; Loop 4 times, restore source pointer, restore sine/cosine pointer
	loops_undo 4*4			;; Restore loop counter
	end_timer 10

	start_timer 11
	yloop_set_prefetch YQUARTER	;; Prefetch 36 cache lines in this loop
;;	yr4_eb4cl_four_complex_djbunfft_preload
c4b:	yr4_eb4cl_four_complex_djbunfft rsi, 64, 4*64, 8*64, rdi, 4*YMM_SCD2, 4
	yloop	4, c4b, -4*64+16*64, rdi, -4*4*YMM_SCD2+YMM_SCD2 ;; Loop 4 times, next source pointer, next sine/cosine pointer
	yloop	4, c4b, -4*16*64, rdi, -4*YMM_SCD2 ;; Loop 4 times, restore source pointer, restore sine/cosine pointer
	loops_undo 4*4			;; Back up loop counter
	end_timer 11

	start_timer 12
	yloop_set_prefetch YHALF	;; Prefetch 72 cache lines in this loop
	yr4_s4cl_four_complex_djbunfft_preload
c3b:	yr4_s4cl_four_complex_djbunfft rsi, 64, 16*64, 32*64, rdi, YMM_SCD2, 16, L1PREFETCH_ALL, dist64
	yloop	16, c3b, -16*64+dist64, rdi, -16*YMM_SCD2 ;; Loop 16 times, next source pointer, restore sine/cosine pointer
	end_timer 12

	yloop	9, b3b0, -9*dist64	;; Loop 9 times, restore source pointer
	jmp	ypass2_2304_complex_unfft


ypass2_2304_complex_1:
	start_timer 6
	bump	rcx, (576+8-2*64-144-36-2*144)*64 ;; Move prefetch pointer along
	yloop_init 144			;; 9 iters of 64 cache lines

b3a0:	yloop_set_prefetch YSINGLE	;; Prefetch 144 cache lines in this loop
	yr4_s4cl_four_complex_djbfft_preload
b3a:	yr4_s4cl_four_complex_djbfft rsi, 64, 16*64, 32*64, rdi, YMM_SCD2, 16
	yloop	16, b3a, -16*64, rdi, -16*YMM_SCD2 ;; Loop 16 times, restore source pointer, restore sine/cosine pointer
	loops_undo 16			;; Back up loop counter

	yloop_set_prefetch YQUARTER	;; Prefetch 36 cache lines in this loop
	yr4_eb4cl_four_complex_djbfft_preload
b4a:	yr4_eb4cl_four_complex_djbfft rsi, 64, 4*64, 8*64, rdi, 4*YMM_SCD2, 4
	yloop	4, b4a, -4*64+16*64, rdi, -4*4*YMM_SCD2+YMM_SCD2 ;; Loop 4 times, next source pointer, next sine/cosine pointer
	yloop	4, b4a, -4*16*64, rdi, -4*YMM_SCD2 ;; Loop 4 times, restore source pointer, restore sine/cosine pointer
	loops_undo 4*4			;; Back up loop counter

	yloop_set_prefetch YSINGLE	;; Prefetch 144 cache lines in this loop
;;	yr4_eb4cl_four_complex_djbfft_preload
b5a:	yr4_eb4cl_four_complex_djbfft rsi, 4*64, 64, 2*64, rdi, 0, 4, L1PREFETCH_ALL, dist64
	yloop	4, b5a, 0, rdi, 4*YMM_SCD2 ;; Loop 4 times, next sine/cosine pointer
	yloop	4, b5a, -16*4*64, rdi, -4*4*YMM_SCD2 ;; Loop 4 times, restore source pointer, restore sine/cosine pointer
	loops_undo 4*4			;; Back up loop counter

	yloop_set_prefetch YSINGLE	;; Prefetch 144 cache lines in this loop
	yr4_4cl_four_complex_fft_final_preload
b6a:	yr4_4cl_four_complex_fft_final rsi, 64, 16*64, 32*64
	yloop	16, b6a, -16*64+dist64	;; Loop 16 times, next source pointer

	yloop	9, b3a0			;; Loop 9 times
	end_timer 6
	jmp	ypass2_2304_complex_done


ypass2_2304_complex_3:
	start_timer 6
x64	mov	r8, DEST2ARG
	mov	rbp, DIST_TO_MULSRCARG
	yloop_init 144			;; 9 iters of 64 cache lines

b3c0:	yloop_set_prefetch YHALF	;; Prefetch 72 cache lines in this loop
	yr4_s4cl_four_complex_djbfft_preload
b3c:	yr4_s4cl_four_complex_djbfft rsi, 64, 16*64, 32*64, rdi, YMM_SCD2, 16
	yloop	16, b3c, -16*64, rdi, -16*YMM_SCD2 ;; Loop 16 times, restore source pointer, restore sine/cosine pointer
	loops_undo 16			;; Back up loop counter

	yloop_set_prefetch YQUARTER	;; Prefetch 36 cache lines in this loop
	yr4_eb4cl_four_complex_djbfft_preload
b4c:	yr4_eb4cl_four_complex_djbfft rsi, 64, 4*64, 8*64, rdi, 4*YMM_SCD2, 4
	yloop	4, b4c, -4*64+16*64, rdi, -4*4*YMM_SCD2+YMM_SCD2 ;; Loop 4 times, next source pointer, next sine/cosine pointer
	yloop	4, b4c, -4*16*64, rdi, -4*YMM_SCD2 ;; Loop 4 times, restore source pointer, restore sine/cosine pointer
	loops_undo 4*4			;; Back up loop counter

	yloop_set_prefetch YQUARTER	;; Prefetch 36 cache lines in this loop
;;	yr4_eb4cl_four_complex_djbfft_preload
b5c:	yr4_eb4cl_four_complex_djbfft rsi, 4*64, 64, 2*64, rdi, 0, 4
	yloop	4, b5c, 0, rdi, 4*YMM_SCD2 ;; Loop 4 times, next sine/cosine pointer
	yloop	4, b5c, -16*4*64, rdi, -4*4*YMM_SCD2 ;; Loop 4 times, restore source pointer, restore sine/cosine pointer
	loops_undo 4*4			;; Back up loop counter

	yloop_set_prefetch YQUARTER	;; Prefetch 36 cache lines in this loop
	yr4_4cl_four_complex_with_mult_preload
b6c:	yr4_4cl_four_complex_with_mult rsi, 64, 16*64, 32*64
	yloop	16, b6c, -16*64	;; Loop 16 times, restore source pointer
x64	add	rsi, r8
	loops_undo 16			;; Back up loop counter

	yloop_set_prefetch YQUARTER	;; Prefetch 36 cache lines in this loop
	yr4_eb4cl_four_complex_djbunfft_preload
c5c:	yr4_eb4cl_four_complex_djbunfft rsi, 4*64, 64, 2*64, rdi, 0, 4
	yloop	4, c5c, 0, rdi, 4*YMM_SCD2 ;; Loop 4 times, next sine/cosine pointer
	yloop	4, c5c, -16*4*64, rdi, -4*4*YMM_SCD2 ;; Loop 4 times, restore source pointer, restore sine/cosine pointer
	loops_undo 4*4			;; Restore loop counter

	yloop_set_prefetch YQUARTER	;; Prefetch 36 cache lines in this loop
;;	yr4_eb4cl_four_complex_djbunfft_preload
c4c:	yr4_eb4cl_four_complex_djbunfft rsi, 64, 4*64, 8*64, rdi, 4*YMM_SCD2, 4
	yloop	4, c4c, -4*64+16*64, rdi, -4*4*YMM_SCD2+YMM_SCD2 ;; Loop 4 times, next source pointer, next sine/cosine pointer
	yloop	4, c4c, -4*16*64, rdi, -4*YMM_SCD2 ;; Loop 4 times, restore source pointer, restore sine/cosine pointer
	loops_undo 4*4			;; Back up loop counter

	yloop_set_prefetch YHALF	;; Prefetch 72 cache lines in this loop
	yr4_s4cl_four_complex_djbunfft_preload
c3c:	yr4_s4cl_four_complex_djbunfft rsi, 64, 16*64, 32*64, rdi, YMM_SCD2, 16, L1PREFETCH_ALL, dist64
	yloop	16, c3c, -16*64+dist64, rdi, -16*YMM_SCD2 ;; Loop 16 times, next source pointer, restore sine/cosine pointer

x64	sub	rsi, r8
	yloop	9, b3c0, -9*dist64	;; Loop 9 times, restore source pointer
x64	add	rsi, r8
	end_timer 6
	jmp	ypass2_2304_complex_unfft


ypass2_2304_complex_4:
	start_timer 6
	mov	rdi, xsincos_complex	;; The sin/cos ptrs
	bump	rdi, 192*YMM_SCD1+64*YMM_SCD1
	mov	rbp, DIST_TO_MULSRCARG
	yloop_init 144			;; 9 iters of 64 cache lines

	yloop_set_prefetch YSINGLE	;; Prefetch 144 cache lines in this loop
	yr4_4cl_four_complex_with_mulf_preload
b6d:	yr4_4cl_four_complex_with_mulf rsi, 64, 16*64, 32*64
	yloop	16, b6d, -16*64	;; Loop 16 times, restore source pointer
	loops_undo 16			;; Back up loop counter

	yloop_set_prefetch YSINGLE	;; Prefetch 144 cache lines in this loop
	yr4_eb4cl_four_complex_djbunfft_preload
c5d:	yr4_eb4cl_four_complex_djbunfft rsi, 4*64, 64, 2*64, rdi, 0, 4
	yloop	4, c5d, 0, rdi, 4*YMM_SCD2 ;; Loop 4 times, next sine/cosine pointer
	yloop	4, c5d, -16*4*64, rdi, -4*4*YMM_SCD2 ;; Loop 4 times, restore source pointer, restore sine/cosine pointer
	loops_undo 4*4			;; Back up loop counter

	yloop_set_prefetch YQUARTER	;; Prefetch 36 cache lines in this loop
;;	yr4_eb4cl_four_complex_djbunfft_preload
c4d:	yr4_eb4cl_four_complex_djbunfft rsi, 64, 4*64, 8*64, rdi, 4*YMM_SCD2, 4
	yloop	4, c4d, -4*64+16*64, rdi, -4*4*YMM_SCD2+YMM_SCD2 ;; Loop 4 times, next source pointer, next sine/cosine pointer
	yloop	4, c4d, -4*16*64, rdi, -4*YMM_SCD2 ;; Loop 4 times, restore source pointer, restore sine/cosine pointer
	loops_undo 4*4			;; Back up loop counter

	yloop_set_prefetch YSINGLE	;; Prefetch 144 cache lines in this loop
	yr4_s4cl_four_complex_djbunfft_preload
c3d:	yr4_s4cl_four_complex_djbunfft rsi, 64, 16*64, 32*64, rdi, YMM_SCD2, 16, L1PREFETCH_ALL, dist64
	yloop	16, c3d, -16*64+dist64, rdi, -16*YMM_SCD2 ;; Loop 16 times, next source pointer, restore sine/cosine pointer

	yloop	9, b6d, -9*dist64	;; Loop 9 times, restore source pointer
	bump	rcx, (-2*144-36-144+2*64+72+5*36+72)*64 ;; Move prefetch pointer to where squaring would be
	end_timer 6

;; Do inverse FFT levels 3,4
;; On input the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	256	+1	+1	+1
;;	4	...
;;	...
;;	252	...
;;	512	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	768	+1	+1	+1
;;	4	...
;;	...
;;	764	...
;;	1536	...
;;	...

;; Do 768 three_complex_unfft operations
;;	distance between fft data elements is 256
;;	do 192 macros each processing 24 data values

;; Do inverse FFT levels 1,2
;; On input the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	768	+1	+1	+1
;;	4	...
;;	...
;;	764	...
;;	1536	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	2304	+1	+1	+1
;;	4	...
;;	...
;;	2300

;; Do 768 three_complex_unfft operations
;;	distance between fft data elements is 768
;;	do 192 macros each processing 24 data values

ypass2_2304_complex_unfft:
	mov	rdi, xsincos_complex	;; Load sin/cos pointers
	lea	rbp, [rdi+192*YMM_SCD1]
	bump	rcx, (576+8-2*64-72-5*36-72-2*64)*64 ;; Move prefetch pointer along
	yloop_init 256			;; 192 iterations

	yr3_3cl_three_complex_djbunfft_preload
c2a:	start_timer 13
	yloop_adjust 1
	yloop_set_prefetch YQUARTER	;; Prefetch 64 cache lines
c2b:	yr3_3cl_three_complex_djbunfft rsi, 3*dist64, dist64, rbp, 0, 3
	yloop	4, c2b, -3*3*dist64, rbp, YMM_SCD1 ;; 3 iterations!!  Restore source pointer, next sine/cosine pointer
	loops_undo 4			;; Restore loop counter
	end_timer 13

	start_timer 14
	yloop_adjust 1
	yloop_set_prefetch YQUARTER	;; Prefetch 64 cache lines
c1b:	yr3_3cl_three_complex_djbunfft rsi, dist64, 3*dist64, rdi, 64*YMM_SCD1, 3, L1PREFETCH_ALL, 64
	yloop	4, c1b, -3*dist64+64, rdi, -3*64*YMM_SCD1+YMM_SCD1 ;; 3 iterations!!  Next source pointer, next sine/cosine pointer
	end_timer 14

	yloop	64, c2a			;; Loop 64 times

ypass2_2304_complex_done:
	ENDM


; *************** radix4-pass2-3072-real macro ******************
; This macro takes 6144 real values and performs the final 12.585 levels of the FFT.
; The results are squared, and the first 12.585 levels of the inverse FFT are done.

yr4dwpn_pass2_3072_real MACRO
	LOCAL	b1b, b2a, b2b, b3a, b3b, b4a, b4b, b5a, b5b, b6a, b6b, b6c, b6d
	LOCAL	c1b, c2a, c2b, c3a, c3b, c4a, c4b, c5a, c5b
	LOCAL	ypass2_3072_real_1, ypass2_3072_real_2
	LOCAL	ypass2_3072_real_3, ypass2_3072_real_4
	LOCAL	ypass2_3072_real_unfft, ypass2_3072_real_done

;; Type 4 FFTs skip the forward FFT process

	start_timer 2
	mov	rsi, DATA_ADDR		;; Load source address
	mov	rbx, DIST_TO_FFTSRCARG
	mov	rcx, DATA_PREFETCH	;; Prefetch pointer (we need to prefetch 
					;; 768 cache lines + 11 pad cache lines)
	cmp	ffttype, 4
	je	ypass2_3072_real_4

;; Do FFT levels 1,2,3 which correspond to complex levels 1,2
;; On input the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	3072	+1	+1	+1
;;	4	...
;;	...
;;	3068
;; On output the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	1024	+1	+1	+1
;;	4	...
;;	...
;;	1020	...
;;	2048	...
;;	...

;; Do 1024 six_reals_fft operations
;;	distance between fft data elements is 1024
;;	do 256 macros each processing 24 data values

	mov	rdi, sincos3		;; Real sin/cos pointer
	mov	rbp, xsincos_complex	;; Complex sin/cos data
	yloop_init 256			;; 256 iterations
	yloop_set_prefetch YHALF
	yr3_f3cl_2sc_six_reals_fft_preload
b1b:	yr3_f3cl_2sc_six_reals_fft rsi, 64, 4*dist64, rbp, YMM_SCD1, rdi, YMM_SCD1, 64
	yloop	64, b1b, -64*64+dist64	;; Test loop counter, next source pointer
	yloop	4, b1b, -4*dist64	;; Test loop counter, restore source pointer

;; Do FFT levels 3,4
;; On input the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	1024	+1	+1	+1
;;	4	...
;;	...
;;	1020	...
;;	2048	...
;;	...
;; On output the 64-byte cache lines are swizzled to hold these data values:
;;	0	+512	+512	+512	256	+512	+512	+512
;;	4	...
;;	...
;;	252	...
;;	1	...
;;	...
;;	2048	...
;;	...

;; Do 256 eight_reals_fft operations
;; Do 512 four_complex_fft operations
;;	distance between fft data elements is 256
;;	do 192 macros each processing 32 data values

	yloop_init 192			;; 3 iters of 64
	yloop_set_prefetch YHALF
	yr4_s4cl_2sc_eight_reals_fft_preload
b2a:	yr4_s4cl_2sc_eight_reals_fft rsi, 64, dist64, 2*dist64, rbp, YMM_SCD2, rdi, YMM_SCD2, 64
	yloop	64, b2a, -64*64+4*dist64, rbp, -64*YMM_SCD2 ;; Test loop counter, next source pointer, restore sine/cosine pointer

	loops_reset			;; 2 more iters of 64
	yr4_s4cl_four_complex_djbfft_preload
b2b:	yr4_s4cl_four_complex_djbfft rsi, 64, dist64, 2*dist64, rbp, YMM_SCD2, 64
	yloop	64, b2b, -64*64+4*dist64, rbp, -64*YMM_SCD2 ;; Test loop counter, next source pointer, restore sin/cos ptr
	yloop	3, b2b, -3*4*dist64	;; Loop 3 times, restore source pointer

;; Do FFT levels 5,6
;; On input the 64-byte cache lines hold these data values:
;;	0	+512	+512	+512	256	+512	+512	+512
;;	4	...
;;	...
;;	252	...
;;	1	...
;;	...
;;	2048	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+512	+512	+512	64	+512	+512	+512
;;	4	...
;;	...
;;	60	...
;;	128	...
;;	...
;;	1	...
;;	...
;;	2048	...
;;	...

;; Do 64 eight_reals_fft operations
;; Do 704 four_complex_fft operations
;;	distance between fft data elements is 64
;;	do 192 macros each processing 32 data values

	yloop_init 192			;; 3 iterations of 64
	yloop_set_prefetch YHALF
	yr4_4cl_eight_reals_four_complex_djbfft_preload
b3a:	yr4_4cl_eight_reals_four_complex_djbfft rsi, dist64, 16*64, 32*64, rdi, YMM_SCD3, 4
	yloop	4, b3a, -4*dist64+64	;; Test loop counter, next source pointer
	yloop	16, b3a, -16*64+4*dist64, rdi, -64*YMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer

	loops_reset			;; 3 more iters of 64
	yr4_rb4cl_four_complex_djbfft_preload
b3b:	yr4_rb4cl_four_complex_djbfft rsi, dist64, 16*64, 32*64, rdi, YMM_SCD3, 4
	yloop	4, b3b, -4*dist64+64	;; Test loop counter, next source pointer
	yloop	16, b3b, -16*64+4*dist64, rdi, -64*YMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer
	yloop	3, b3b, -3*4*dist64	;; Test loop counter, restore source pointer

;; Do FFT levels 7,8
;; On input the 64-byte cache lines hold these data values:
;;	0	+512	+512	+512	64	+512	+512	+512
;;	4	...
;;	...
;;	60	...
;;	128	...
;;	...
;;	1	...
;;	...
;;	2048	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+512	+512	+512	16	+512	+512	+512
;;	4	...
;;	...
;;	12	...
;;	32	...
;;	...
;;	1	...
;;	...
;;	2048	...
;;	...

;; Do 16 eight_reals_fft operations
;; Do 752 four_complex_fft operations
;;	distance between fft data elements is 16
;;	do 192 macros each processing 32 data values

	yloop_init 192			;; 12 iterations of 16
	yloop_set_prefetch YHALF
	yr4_4cl_eight_reals_four_complex_djbfft_preload
b4a:	yr4_4cl_eight_reals_four_complex_djbfft rsi, dist64, 4*64, 8*64, rdi, 4*YMM_SCD3, 4
	yloop	4, b4a, -4*dist64+64	;; Test loop counter, next source pointer
	yloop	4, b4a, -4*64+16*64, rdi, -16*4*YMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer

	loops_reset			;; 11 more iters of 16
	yr4_rb4cl_four_complex_djbfft_preload
b4b:	yr4_rb4cl_four_complex_djbfft rsi, dist64, 4*64, 8*64, rdi, 4*YMM_SCD3, 4
	yloop	4, b4b, -4*dist64+64	;; Test loop counter, next source pointer
	yloop	4, b4b, -4*64+16*64, rdi, -16*4*YMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer
	yloop	4, b4b, -4*16*64+4*dist64 ;; Test loop counter, next source pointer
	yloop	3, b4b, -3*4*dist64	;; Test loop counter, restore source pointer

;; Do FFT levels 9,10
;; On input the 64-byte cache lines hold these data values:
;;	0	+512	+512	+512	16	+512	+512	+512
;;	4	...
;;	...
;;	12	...
;;	32	...
;;	...
;;	1	...
;;	...
;;	2048	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+512	+512	+512	4	+512	+512	+512
;;	8	...
;;	...
;;	1	...
;;	...
;;	2048	...
;;	...

;; Do 4 eight_reals_fft operations
;; Do 764 four_complex_fft operations
;;	distance between fft data elements is 4
;;	do 192 macros each processing 32 data values

	yloop_init 192			;; 48 iterations of 4
	yloop_set_prefetch YHALF
	yr4_4cl_eight_reals_four_complex_djbfft_preload
b5a:	yr4_4cl_eight_reals_four_complex_djbfft rsi, dist64, 64, 2*64, rdi, 16*YMM_SCD3, 4
	yloop	4, b5a, -4*dist64+4*64, rdi, -4*16*YMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer

	loops_reset			;; 47 more iters of 4
	yr4_rb4cl_four_complex_djbfft_preload
b5b:	yr4_rb4cl_four_complex_djbfft rsi, dist64, 64, 2*64, rdi, 16*YMM_SCD3, 4
	yloop	4, b5b, -4*dist64+4*64, rdi, -4*16*YMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer
	yloop	16, b5b, -16*4*64+4*dist64 ;; Test loop counter, next source pointer
	yloop	3, b5b, -3*4*dist64	;; Test loop counter, restore source pointer

;; Do FFT levels 11,12 as well as inverse FFT levels 11,12
;; On input and output the 64-byte cache lines hold these data values:
;;	0	+512	+512	+512	4	+512	+512	+512
;;	8	...
;;	...
;;	1	...
;;	...
;;	2048	...
;;	...

;; Do 1 eight_reals_fft operation
;; Do 767 four_complex_fft operations
;;	distance between fft data elements is 1
;;	do 192 macros each processing 32 data values

	cmp	ffttype, 2
	jl	ypass2_3072_real_1
	jg	ypass2_3072_real_3

ypass2_3072_real_2:
;;	yr4_4cl_eight_reals_four_complex_with_square rsi, 64, dist64, 2*dist64
	yr4_dispatch call, yreal_square, dist64, 2*dist64
	yloop_init 192			;; 191 more iterations
	yloop_adjust 1
	yloop_set_prefetch YHALF
	yr4_4cl_four_complex_with_square_preload
b6b:	yr4_4cl_four_complex_with_square rsi, 64, dist64, 2*dist64, 1
	yloop	64, b6b, -64*64+4*dist64	;; Test loop counter, next source pointer
	yloop	3, b6b, -3*4*dist64	;; Test loop counter, restore source pointer
	jmp	ypass2_3072_real_unfft

ypass2_3072_real_1:
;;	yr4_4cl_eight_reals_four_complex_fft_final rsi, 64, dist64, 2*dist64
	yr4_dispatch call, yreal_fft_final, dist64, 2*dist64
	mov	rcx, xsincos_complex	;; Prefetch complex sin/cos data, 416 cache lines
	yloop_init 192			;; 191 more iterations
	yloop_adjust 1
	yloop_set_prefetch YDOUBLE
b6a:	yr4_4cl_four_complex_fft_final rsi, 64, dist64, 2*dist64
	yloop	64, b6a, -64*64+4*dist64	;; Test loop counter, next source pointer
	yloop	3, b6a			;; Test loop counter
	jmp	ypass2_3072_real_done

ypass2_3072_real_3:
x64	mov	r8, DEST2ARG
	mov	rbp, DIST_TO_MULSRCARG
;;	yr4_4cl_eight_reals_four_complex_with_mult rsi, 64, dist64, 2*dist64
	yr4_dispatch call, yreal_mult, dist64, 2*dist64
	yloop_init 192			;; 191 more iterations
	yloop_adjust 1
	yloop_set_prefetch YHALF
	yr4_4cl_four_complex_with_mult_preload
b6c:	yr4_4cl_four_complex_with_mult rsi, 64, dist64, 2*dist64
	yloop	64, b6c, -64*64+4*dist64	;; Test loop counter, next source pointer
	yloop	3, b6c, -3*4*dist64	;; Test loop counter, restore source pointer
x64	add	rsi, r8
	jmp	ypass2_3072_real_unfft

ypass2_3072_real_4:
	mov	rbp, DIST_TO_MULSRCARG
;;	yr4_4cl_eight_reals_four_complex_with_mulf rsi, 64, dist64, 2*dist64
	yr4_dispatch call, yreal_mulf, dist64, 2*dist64
	yloop_init 192			;; 191 more iterations
	yloop_adjust 1
	yloop_set_prefetch YDOUBLE
	yr4_4cl_four_complex_with_mulf_preload
b6d:	yr4_4cl_four_complex_with_mulf rsi, 64, dist64, 2*dist64
	yloop	64, b6d, -64*64+4*dist64	;; Test loop counter, next source pointer
	yloop	3, b6d, -3*4*dist64	;; Test loop counter, restore source pointer
;;	bump	rcx, (-384+128+5*96)*64 ;; Move prefetch pointer back to where squaring would be

;; Do inverse FFT levels 9,10
;; On input the 64-byte cache lines hold these data values:
;;	0	+512	+512	+512	4	+512	+512	+512
;;	8	...
;;	...
;;	1	...
;;	...
;;	2048	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+512	+512	+512	16	+512	+512	+512
;;	4	...
;;	...
;;	12	...
;;	32	...
;;	...
;;	1	...
;;	...
;;	2048	...
;;	...

;; Do 4 eight_reals_unfft operations
;; Do 764 four_complex_unfft operations
;;	distance between fft data elements is 4
;;	do 192 macros each processing 32 data values

ypass2_3072_real_unfft:
	mov	rdi, sincos3		;; Load sin/cos pointer
	bump	rdi, 64*YMM_SCD2+256*YMM_SCD1
	yloop_init 192			;; 48 iterations of 4
	yloop_set_prefetch YHALF
	yr4_4cl_eight_reals_four_complex_djbunfft_preload
c5a:	yr4_4cl_eight_reals_four_complex_djbunfft rsi, dist64, 64, 2*64, rdi, 16*YMM_SCD3, 4
	yloop	4, c5a, -4*dist64+4*64, rdi, -4*16*YMM_SCD3 ;; Test loop counter, next source pointer, restore sine/cosine pointer

	loops_reset			;; 47 more iters of 4
	yr4_rb4cl_four_complex_djbunfft_preload
c5b:	yr4_rb4cl_four_complex_djbunfft rsi, dist64, 64, 2*64, rdi, 16*YMM_SCD3, 4
	yloop	4, c5b, -4*dist64+4*64, rdi, -4*16*YMM_SCD3 ;; Test loop counter, next source pointer, restore sine/cosine pointer
	yloop	16, c5b, -16*4*64+4*dist64	;; Test loop counter, next source pointer
	yloop	3, c5b, -3*4*dist64	;; Test loop counter, restore source pointer

;; Do inverse FFT levels 7,8
;; On input the 64-byte cache lines hold these data values:
;;	0	+512	+512	+512	16	+512	+512	+512
;;	4	...
;;	...
;;	12	...
;;	32	...
;;	...
;;	1	...
;;	...
;;	2048	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+512	+512	+512	64	+512	+512	+512
;;	4	...
;;	...
;;	60	...
;;	128	...
;;	...
;;	1	...
;;	...
;;	2048	...
;;	...

;; Do 16 eight_reals_unfft operations
;; Do 752 four_complex_unfft operations
;;	distance between fft data elements is 16
;;	do 192 macros each processing 32 data values

	bump	rcx, (768+11-128-6*96-96)*64 ;; Move prefetch pointer along
	yloop_init 192			;; 12 iterations of 16
	yloop_set_prefetch YHALF
	yr4_4cl_eight_reals_four_complex_djbunfft_preload
c4a:	yr4_4cl_eight_reals_four_complex_djbunfft rsi, dist64, 4*64, 8*64, rdi, 4*YMM_SCD3, 4
	yloop	4, c4a, -4*dist64+64	;; Test loop counter, next source pointer
	yloop	4, c4a, -4*64+16*64, rdi, -16*4*YMM_SCD3 ;; Test loop counter, next source pointer, restore sine/cosine pointer

	loops_reset			;; 11 more iters of 16
	yr4_rb4cl_four_complex_djbunfft_preload
c4b:	yr4_rb4cl_four_complex_djbunfft rsi, dist64, 4*64, 8*64, rdi, 4*YMM_SCD3, 4
	yloop	4, c4b, -4*dist64+64	;; Test loop counter, next source pointer
	yloop	4, c4b, -4*64+16*64, rdi, -16*4*YMM_SCD3 ;; Test loop counter, next source pointer, restore sine/cosine pointer
	yloop	4, c4b, -4*16*64+4*dist64	;; Test loop counter, next source pointer
	yloop	3, c4b, -3*4*dist64	;; Test loop counter, restore source pointer

;; Do inverse FFT levels 5,6
;; On input the 64-byte cache lines hold these data values:
;;	0	+512	+512	+512	64	+512	+512	+512
;;	4	...
;;	...
;;	60	...
;;	128	...
;;	...
;;	1	...
;;	...
;;	2048	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+512	+512	+512	256	+512	+512	+512
;;	4	...
;;	...
;;	252	...
;;	1	...
;;	...
;;	2048	...
;;	...

;; Do 64 eight_reals_unfft operations
;; Do 704 four_complex_unfft operations
;;	distance between fft data elements is 64
;;	do 192 macros each processing 32 data values

	mov	rcx, xsincos_complex	;; Prefetch complex sin/cos data, 208 cache lines
	yloop_init 192			;; 3 iterations of 64
	yloop_set_prefetch YSINGLE
	yr4_4cl_eight_reals_four_complex_djbunfft_preload
c3a:	yr4_4cl_eight_reals_four_complex_djbunfft rsi, dist64, 16*64, 32*64, rdi, YMM_SCD3, 4
	yloop	4, c3a, -4*dist64+64	;; Test loop counter, next source pointer
	yloop	16, c3a, -16*64+4*dist64, rdi, -64*YMM_SCD3 ;; Test loop counter, next source pointer, restore sine/cosine pointer

	loops_reset			;; 2 more iters of 64
	yr4_rb4cl_four_complex_djbunfft_preload
c3b:	yr4_rb4cl_four_complex_djbunfft rsi, dist64, 16*64, 32*64, rdi, YMM_SCD3, 4
	yloop	4, c3b, -4*dist64+64	;; Test loop counter, next source pointer
	yloop	16, c3b, -16*64+4*dist64, rdi, -64*YMM_SCD3 ;; Test loop counter, next source pointer, restore sine/cosine pointer
	yloop	3, c3b, -3*4*dist64	;; Test loop counter, restore source pointer

;; Do inverse FFT levels 3,4
;; On input the 64-byte cache lines hold these data values:
;;	0	+512	+512	+512	256	+512	+512	+512
;;	4	...
;;	...
;;	252	...
;;	1	...
;;	...
;;	2048	...
;;	...
;; On output the 64-byte cache lines are swizzled to hold these data values:
;;	0	+1	+1	+1	1024	+1	+1	+1
;;	4	...
;;	...
;;	1020	...
;;	2048	...
;;	...

;; Do 256 eight_reals_unfft operations
;; Do 512 four_complex_unfft operations
;;	distance between fft data elements is 256
;;	do 192 macros each processing 32 data values

	bump	rdi, -64*YMM_SCD2	;; Real sin/cos ptr
	mov	rbp, xsincos_complex	;; Complex sin/cos data
	bump	rbp, 256*YMM_SCD1
	yloop_init 192			;; 3 iters of 64
	yloop_set_prefetch YHALF
	yr4_s4cl_2sc_eight_reals_unfft_preload
c2a:	yr4_s4cl_2sc_eight_reals_unfft rsi, 64, dist64, 2*dist64, rbp, YMM_SCD2, rdi, YMM_SCD2, 64
	yloop	64, c2a, -64*64+4*dist64, rbp, -64*YMM_SCD2 ;; Test loop counter, next source pointer, restore sine/cosine pointer

	loops_reset			;; 2 more iters of 64
	yr4_s4cl_four_complex_djbunfft_preload
c2b:	yr4_s4cl_four_complex_djbunfft rsi, 64, dist64, 2*dist64, rbp, YMM_SCD2, 64
	yloop	64, c2b, -64*64+4*dist64, rbp, -64*YMM_SCD2 ;; Test loop counter, next source pointer, restore sine/cosine pointer
	yloop	3, c2b, -3*4*dist64	;; Test loop counter, restore source pointer

;; Do inverse FFT levels 1,2,3 which correspond to complex levels 1,2
;; On input the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	1024	+1	+1	+1
;;	4	...
;;	...
;;	1020	...
;;	2048	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	3072	+1	+1	+1
;;	4	...
;;	...
;;	3068

;; Do 1024 six_reals_unfft operations
;;	distance between fft data elements is 1024
;;	do 256 macros each processing 24 data values

	bump	rdi, -64*YMM_SCD2-256*YMM_SCD1 ;; Real sin/cos ptr
	bump	rbp, -256*YMM_SCD1	;; Complex sin/cos data
	bump	rcx, (416-192-96-128)*64 ;; Move prefetch pointer along
	yloop_init 256			;; 256 iterations
	yloop_set_prefetch YHALF
	yr3_3cl_2sc_six_reals_unfft_preload
c1b:	yr3_3cl_2sc_six_reals_unfft rsi, 64, 4*dist64, rbp, YMM_SCD1, rdi, YMM_SCD1, 64
	yloop	64, c1b, -64*64+dist64	;; Test loop counter, next source pointer
	yloop	4, c1b			;; Test loop counter

ypass2_3072_real_done:
	end_timer 2
	ENDM

; *************** radix4-pass2-3072-complex macro ******************
; This macro takes 3072 complex values and performs the final 11.585 levels of a standard
; radix-4 FFT, squares the results, and does the first 11.585 levels of the inverse FFT.

yr4dwpn_pass2_3072_complex MACRO
	LOCAL	b1a, b1b, b2b, b3a0, b3a, b3b0, b3b, b3c0, b3c, b4a, b4b, b4c, b5a, b5b, b5c, b6a, b6b, b6c, b6d
	LOCAL	c1b, c2a, c2b, c3b, c3c, c3d, c4b, c4c, c4d, c5b, c5c, c5d
	LOCAL	ypass2_3072_complex_1, ypass2_3072_complex_2
	LOCAL	ypass2_3072_complex_3, ypass2_3072_complex_4
	LOCAL	ypass2_3072_complex_unfft, ypass2_3072_complex_done

;; Type 4 FFTs skip the forward FFT process

	mov	rsi, DATA_ADDR		;; Load address of FFT data
	mov	rbx, DIST_TO_FFTSRCARG
	mov	rcx, DATA_PREFETCH	;; Prefetch pointer (we need to prefetch 
					;; 768 cache lines + 11 pad cache lines)
	cmp	ffttype, 4
	je	ypass2_3072_complex_4

;; Do FFT level 1,2
;; On input the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	3072	+1	+1	+1
;;	4	...
;;	...
;;	3068
;; On output the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	1024	+1	+1	+1
;;	4	...
;;	...
;;	1020	...
;;	2048	...
;;	...

;; Do 1024 three_complex_fft operations
;;	distance between fft data elements is 1024
;;	do 256 macros each processing 24 data values

;; Do FFT levels 3,4
;; On input the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	1024	+1	+1	+1
;;	4	...
;;	...
;;	1020	...
;;	2048	...
;;	...
;; On output the 64-byte cache lines are swizzled to hold these data values:
;;	0	+512	+512	+512	256	+512	+512	+512
;;	4	...
;;	...
;;	252	...
;;	1	...
;;	...
;;	2048	...
;;	...

;; Do 768 four_complex_fft operations
;;	distance between fft data elements is 256
;;	do 192 macros each processing 32 data values

	mov	rdi, xsincos_complex	;; The sin/cos ptrs
	lea	rbp, [rdi+256*YMM_SCD1]
	yloop_init 256			;; 256 iterations or 192 iterations

b1a:	start_timer 4
	yloop_set_prefetch YQUARTER	;; Prefetch 64 cache lines
	yr3_f3cl_three_complex_djbfft_preload
b1b:	yr3_f3cl_three_complex_djbfft rsi, dist64, 4*dist64, rdi, 64*YMM_SCD1, 4
	yloop	4, b1b, -4*dist64, rdi, -4*64*YMM_SCD1+YMM_SCD1 ;; Test loop counter, restore source pointer, next sin/cos data
	loops_undo 4
	end_timer 4

	start_timer 5
	yloop_adjust 1			;; One fewer iteration this loop
	yloop_set_prefetch YQUARTER	;; Prefetch 64 cache lines
	yr4_s4cl_four_complex_djbfft_preload
b2b:	yr4_s4cl_four_complex_djbfft rsi, 4*dist64, dist64, 2*dist64, rbp, 0, 3, L1PREFETCH_ALL, 64
	yloop	4, b2b, -3*4*dist64+64, rbp, YMM_SCD2 ;; 3 iterations!!  Next source pointer, next sin/cos data
	end_timer 5

	yloop	64, b1a, -64*64, rdi, -64*YMM_SCD1+256*YMM_SCD1+64*YMM_SCD2 ;; Test loop counter, restore source pointer, next sin/cos ptr

;; Do FFT levels 5,6
;; On input the 64-byte cache lines hold these data values:
;;	0	+512	+512	+512	256	+512	+512	+512
;;	4	...
;;	...
;;	252	...
;;	1	...
;;	...
;;	2048	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+512	+512	+512	64	+512	+512	+512
;;	4	...
;;	...
;;	60	...
;;	128	...
;;	...
;;	1	...
;;	...
;;	2048	...
;;	...

;; Do 768 four_complex_fft operations
;;	distance between fft data elements is 64
;;	do 192 macros each processing 32 data values

;; Do FFT levels 7,8
;; On input the 64-byte cache lines hold these data values:
;;	0	+512	+512	+512	64	+512	+512	+512
;;	4	...
;;	...
;;	60	...
;;	128	...
;;	...
;;	1	...
;;	...
;;	2048	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+512	+512	+512	16	+512	+512	+512
;;	4	...
;;	...
;;	12	...
;;	32	...
;;	...
;;	1	...
;;	...
;;	2048	...
;;	...

;; Do 768 four_complex_fft operations
;;	distance between fft data elements is 16
;;	do 192 macros each processing 32 data values

;; Do FFT levels 9,10
;; On input the 64-byte cache lines hold these data values:
;;	0	+512	+512	+512	16	+512	+512	+512
;;	4	...
;;	...
;;	12	...
;;	32	...
;;	...
;;	1	...
;;	...
;;	2048	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+512	+512	+512	4	+512	+512	+512
;;	8	...
;;	...
;;	1	...
;;	...
;;	2048	...
;;	...

;; Do 768 four_complex_fft operations
;;	distance between fft data elements is 4
;;	do 192 macros each processing 32 data values

;; Do FFT levels 11,12 as well as inverse FFT levels 11,12
;; On input and output the 64-byte cache lines hold these data values:
;;	0	+512	+512	+512	4	+512	+512	+512
;;	8	...
;;	...
;;	1	...
;;	...
;;	2048	...
;;	...

;; Do 768 four_complex_fft operations
;;	distance between fft data elements is 1
;;	do 192 macros each processing 32 data values

;; Do inverse FFT levels 9,10
;; On input the 64-byte cache lines hold these data values:
;;	0	+512	+512	+512	4	+512	+512	+512
;;	8	...
;;	...
;;	1	...
;;	...
;;	2048	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+512	+512	+512	16	+512	+512	+512
;;	4	...
;;	...
;;	12	...
;;	32	...
;;	...
;;	1	...
;;	...
;;	2048	...
;;	...

;; Do 768 four_complex_unfft operations
;;	distance between fft data elements is 4
;;	do 192 macros each processing 32 data values

;; Do inverse FFT levels 7,8
;; On input the 64-byte cache lines hold these data values:
;;	0	+512	+512	+512	16	+512	+512	+512
;;	4	...
;;	...
;;	12	...
;;	32	...
;;	...
;;	1	...
;;	...
;;	2048	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+512	+512	+512	64	+512	+512	+512
;;	4	...
;;	...
;;	60	...
;;	128	...
;;	...
;;	1	...
;;	...
;;	2048	...
;;	...

;; Do 768 four_complex_unfft operations
;;	distance between fft data elements is 16
;;	do 192 macros each processing 32 data values

;; Do inverse FFT levels 5,6
;; On input the 64-byte cache lines hold these data values:
;;	0	+512	+512	+512	64	+512	+512	+512
;;	4	...
;;	...
;;	60	...
;;	128	...
;;	...
;;	1	...
;;	...
;;	2048	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+512	+512	+512	256	+512	+512	+512
;;	4	...
;;	...
;;	252	...
;;	1	...
;;	...
;;	2048	...
;;	...

;; Do 768 four_complex_unfft operations
;;	distance between fft data elements is 64
;;	do 192 macros each processing 32 data values

;; Do this 256 cache lines (16KB) at a time to maximize accessing data
;; while it is in the L1 cache.

;; Execute the right middle step

	cmp	ffttype, 2
	jl	ypass2_3072_complex_1
	jg	ypass2_3072_complex_3

ypass2_3072_complex_2:
	yloop_init 192			;; 3 iters of 256 cache lines

b3b0:	start_timer 6
	yloop_set_prefetch YHALF	;; Prefetch 96 cache lines in this loop
	yr4_b4cl_four_complex_djbfft_preload
b3b:	yr4_b4cl_four_complex_djbfft rsi, 64, 16*64, 32*64, rdi, YMM_SCD2/4, 16
	yloop	16, b3b, -16*64+dist64	;; Loop 16 times, next source pointer
	yloop	4, b3b, -4*dist64, rdi, -64*YMM_SCD2/4 ;; Loop 4 times, restore source pointer, restore sine/cosine pointer
	loops_undo 4*16			;; Back up loop counter
	end_timer 6

	start_timer 7
	yloop_set_prefetch YHALF	;; Prefetch 96 cache lines in this loop
;;	yr4_b4cl_four_complex_djbfft_preload
b4b:	yr4_b4cl_four_complex_djbfft rsi, 64, 4*64, 8*64, rdi, 4*YMM_SCD2/4, 4
	yloop	4, b4b, -4*64+16*64, rdi, -4*4*YMM_SCD2/4 ;; Loop 4 times, next source pointer, restore sine/cosine pointer
	yloop	4, b4b, -4*16*64+dist64, rdi, YMM_SCD2/4 ;; Loop 4 times, next source pointer, next sine/cosine pointer
	yloop	4, b4b, -4*dist64, rdi, -4*YMM_SCD2/4 ;; Loop 4 times, restore source pointer, restore sine/cosine pointer
	loops_undo 4*4*4		;; Back up loop counter
	end_timer 7

	start_timer 8
	yloop_set_prefetch YQUARTER	;; Prefetch 48 cache lines in this loop
;;	yr4_b4cl_four_complex_djbfft_preload
b5b:	yr4_b4cl_four_complex_djbfft rsi, 4*64, 64, 2*64, rdi, 0, 16
	yloop	16, b5b, -16*4*64+dist64, rdi, 4*YMM_SCD2/4 ;; Loop 16 times, next source pointer, next sine/cosine pointer
	yloop	4, b5b, -4*dist64, rdi, -4*4*YMM_SCD2/4 ;; Loop 4 times, restore source pointer, restore sine/cosine pointer
	loops_undo 4*16			;; Back up loop counter
	end_timer 8

	start_timer 9
	yloop_set_prefetch YQUARTER	;; Prefetch 48 cache lines in this loop
	yr4_4cl_four_complex_with_square_preload
;; Loop below reworked to avoid 4KB distances
;;b6b:	yr4_4cl_four_complex_with_square rsi, 64, dist64, 2*dist64
;;	yloop	64, b6b, -64*64	;; Loop 64 times, restore source pointer
b6b:	yr4_4cl_four_complex_with_square rsi, 4*64, dist64, 2*dist64, 16
	yloop	16, b6b, -16*4*64+64	;; Loop 16 times, next source pointer
	yloop	4, b6b, -4*64	;; Loop 4 times, restore source pointer
	loops_undo 64			;; Back up loop counter
	end_timer 9

	start_timer 10
	yloop_set_prefetch YQUARTER	;; Prefetch 48 cache lines in this loop
	yr4_b4cl_four_complex_djbunfft_preload
c5b:	yr4_b4cl_four_complex_djbunfft rsi, 4*64, 64, 2*64, rdi, 0, 16
	yloop	16, c5b, -16*4*64+dist64, rdi, 4*YMM_SCD2/4 ;; Loop 16 times, next source pointer, next sine/cosine pointer
	yloop	4, c5b, -4*dist64, rdi, -4*4*YMM_SCD2/4 ;; Loop 4 times, restore source pointer, restore sine/cosine pointer
	loops_undo 4*16			;; Restore loop counter
	end_timer 10

	start_timer 11
	yloop_set_prefetch YHALF	;; Prefetch 96 cache lines in this loop
;;	yr4_b4cl_four_complex_djbunfft_preload
c4b:	yr4_b4cl_four_complex_djbunfft rsi, 64, 4*64, 8*64, rdi, 4*YMM_SCD2/4, 4
	yloop	4, c4b, -4*64+16*64, rdi, -4*4*YMM_SCD2/4 ;; Loop 4 times, next source pointer, restore sine/cosine pointer
	yloop	4, c4b, -4*16*64+dist64, rdi, YMM_SCD2/4 ;; Loop 4 times, next source pointer, next sine/cosine pointer
	yloop	4, c4b, -4*dist64, rdi, -4*YMM_SCD2/4 ;; Loop 4 times, restore source pointer, restore sine/cosine pointer
	loops_undo 4*4*4		;; Back up loop counter
	end_timer 11

	start_timer 12
	yloop_set_prefetch YHALF	;; Prefetch 96 cache lines in this loop
;;	yr4_b4cl_four_complex_djbunfft_preload
c3b:	yr4_b4cl_four_complex_djbunfft rsi, 64, 16*64, 32*64, rdi, YMM_SCD2/4, 16
	yloop	16, c3b, -16*64+dist64	;; Loop 16 times, next source pointer
	yloop	4, c3b, -4*dist64+4*dist64, rdi, -64*YMM_SCD2/4 ;; Loop 4 times, next source pointer, restore sine/cosine pointer
	end_timer 12

	yloop	3, b3b0, -3*4*dist64	;; Loop 3 times, restore source pointer
	jmp	ypass2_3072_complex_unfft


ypass2_3072_complex_1:
	start_timer 6
	bump	rcx, (768+11-2*64-3*192-96)*64 ;; Move prefetch pointer along
	yloop_init 192			;; 3 iters of 256 cache lines

b3a0:	yloop_set_prefetch YSINGLE	;; Prefetch 192 cache lines in this loop
	yr4_b4cl_four_complex_djbfft_preload
b3a:	yr4_b4cl_four_complex_djbfft rsi, 64, 16*64, 32*64, rdi, YMM_SCD2/4, 16
	yloop	16, b3a, -16*64+dist64	;; Loop 16 times, next source pointer
	yloop	4, b3a, -4*dist64, rdi, -64*YMM_SCD2/4 ;; Loop 4 times, restore source pointer, restore sine/cosine pointer
	loops_undo 4*16			;; Back up loop counter

	yloop_set_prefetch YSINGLE	;; Prefetch 192 cache lines in this loop
;;	yr4_b4cl_four_complex_djbfft_preload
b4a:	yr4_b4cl_four_complex_djbfft rsi, 64, 4*64, 8*64, rdi, 4*YMM_SCD2/4, 4
	yloop	4, b4a, -4*64+16*64, rdi, -4*4*YMM_SCD2/4 ;; Loop 4 times, next source pointer, restore sine/cosine pointer
	yloop	4, b4a, -4*16*64+dist64, rdi, YMM_SCD2/4 ;; Loop 4 times, next source pointer, next sine/cosine pointer
	yloop	4, b4a, -4*dist64, rdi, -4*YMM_SCD2/4 ;; Loop 4 times, restore source pointer, restore sine/cosine pointer
	loops_undo 4*4*4		;; Back up loop counter

	yloop_set_prefetch YSINGLE	;; Prefetch 192 cache lines in this loop
;;	yr4_b4cl_four_complex_djbfft_preload
b5a:	yr4_b4cl_four_complex_djbfft rsi, 4*64, 64, 2*64, rdi, 0, 16
	yloop	16, b5a, -16*4*64+dist64, rdi, 4*YMM_SCD2/4 ;; Loop 16 times, next source pointer, next sine/cosine pointer
	yloop	4, b5a, -4*dist64, rdi, -4*4*YMM_SCD2/4 ;; Loop 4 times, restore source pointer, restore sine/cosine pointer
	loops_undo 4*16			;; Back up loop counter

	yloop_set_prefetch YHALF	;; Prefetch 96 cache lines in this loop
;; Loop below reworked to avoid 4KB distances
;;b6a:	yr4_4cl_four_complex_fft_final rsi, 64, dist64, 2*dist64
;;	yloop	64, b6a, -64*64	;; Loop 64 times, restore source pointer
b6a:	yr4_4cl_four_complex_fft_final rsi, 4*64, dist64, 2*dist64
	yloop	16, b6a, -16*4*64+64	;; Loop 16 times, next source pointer
	yloop	4, b6a, -4*64+4*dist64	;; Loop 4 times, next source pointer

	yloop	3, b3a0			;; Loop 3 times
	end_timer 6
	jmp	ypass2_3072_complex_done


ypass2_3072_complex_3:
	start_timer 6
x64	mov	r8, DEST2ARG
	mov	rbp, DIST_TO_MULSRCARG
	yloop_init 192			;; 3 iters of 256 cache lines

b3c0:	yloop_set_prefetch YHALF	;; Prefetch 96 cache lines in this loop
	yr4_b4cl_four_complex_djbfft_preload
b3c:	yr4_b4cl_four_complex_djbfft rsi, 64, 16*64, 32*64, rdi, YMM_SCD2/4, 16
	yloop	16, b3c, -16*64+dist64	;; Loop 16 times, next source pointer
	yloop	4, b3c, -4*dist64, rdi, -64*YMM_SCD2/4 ;; Loop 4 times, restore source pointer, restore sine/cosine pointer
	loops_undo 4*16			;; Back up loop counter

	yloop_set_prefetch YHALF	;; Prefetch 96 cache lines in this loop
;;	yr4_b4cl_four_complex_djbfft_preload
b4c:	yr4_b4cl_four_complex_djbfft rsi, 64, 4*64, 8*64, rdi, 4*YMM_SCD2/4, 4
	yloop	4, b4c, -4*64+16*64, rdi, -4*4*YMM_SCD2/4 ;; Loop 4 times, next source pointer, restore sine/cosine pointer
	yloop	4, b4c, -4*16*64+dist64, rdi, YMM_SCD2/4 ;; Loop 4 times, next source pointer, next sine/cosine pointer
	yloop	4, b4c, -4*dist64, rdi, -4*YMM_SCD2/4 ;; Loop 4 times, restore source pointer, restore sine/cosine pointer
	loops_undo 4*4*4		;; Back up loop counter

	yloop_set_prefetch YQUARTER	;; Prefetch 48 cache lines in this loop
;;	yr4_b4cl_four_complex_djbfft_preload
b5c:	yr4_b4cl_four_complex_djbfft rsi, 4*64, 64, 2*64, rdi, 0, 16
	yloop	16, b5c, -16*4*64+dist64, rdi, 4*YMM_SCD2/4 ;; Loop 16 times, next source pointer, next sine/cosine pointer
	yloop	4, b5c, -4*dist64, rdi, -4*4*YMM_SCD2/4 ;; Loop 4 times, restore source pointer, restore sine/cosine pointer
	loops_undo 4*16			;; Back up loop counter

	yloop_set_prefetch YQUARTER	;; Prefetch 48 cache lines in this loop
	yr4_4cl_four_complex_with_mult_preload
;; Loop below reworked to avoid 4KB distances
;;b6c:	yr4_4cl_four_complex_with_mult rsi, 64, dist64, 2*dist64
;;	yloop	64, b6c, -64*64	;; Loop 64 times, restore source pointer
b6c:	yr4_4cl_four_complex_with_mult rsi, 4*64, dist64, 2*dist64
	yloop	16, b6c, -16*4*64+64	;; Loop 16 times, next source pointer
	yloop	4, b6c, -4*64		;; Loop 4 times, restore source pointer
x64	add	rsi, r8
	loops_undo 64			;; Back up loop counter

	yloop_set_prefetch YQUARTER	;; Prefetch 48 cache lines in this loop
	yr4_b4cl_four_complex_djbunfft_preload
c5c:	yr4_b4cl_four_complex_djbunfft rsi, 4*64, 64, 2*64, rdi, 0, 16
	yloop	16, c5c, -16*4*64+dist64, rdi, 4*YMM_SCD2/4 ;; Loop 16 times, next source pointer, next sine/cosine pointer
	yloop	4, c5c, -4*dist64, rdi, -4*4*YMM_SCD2/4 ;; Loop 4 times, restore source pointer, restore sine/cosine pointer
	loops_undo 4*16			;; Restore loop counter

	yloop_set_prefetch YHALF	;; Prefetch 96 cache lines in this loop
;;	yr4_b4cl_four_complex_djbunfft_preload
c4c:	yr4_b4cl_four_complex_djbunfft rsi, 64, 4*64, 8*64, rdi, 4*YMM_SCD2/4, 4
	yloop	4, c4c, -4*64+16*64, rdi, -4*4*YMM_SCD2/4 ;; Loop 4 times, next source pointer, restore sine/cosine pointer
	yloop	4, c4c, -4*16*64+dist64, rdi, YMM_SCD2/4 ;; Loop 4 times, next source pointer, next sine/cosine pointer
	yloop	4, c4c, -4*dist64, rdi, -4*YMM_SCD2/4 ;; Loop 4 times, restore source pointer, restore sine/cosine pointer
	loops_undo 4*4*4		;; Back up loop counter

	yloop_set_prefetch YHALF	;; Prefetch 96 cache lines in this loop
;;	yr4_b4cl_four_complex_djbunfft_preload
c3c:	yr4_b4cl_four_complex_djbunfft rsi, 64, 16*64, 32*64, rdi, YMM_SCD2/4, 16
	yloop	16, c3c, -16*64+dist64	;; Loop 16 times, next source pointer
	yloop	4, c3c, -4*dist64+4*dist64, rdi, -64*YMM_SCD2/4 ;; Loop 4 times, next source pointer, restore sine/cosine pointer

x64	sub	rsi, r8
	yloop	3, b3c0, -3*4*dist64	;; Loop 3 times, restore source pointer
x64	add	rsi, r8
	end_timer 6
	jmp	ypass2_3072_complex_unfft


ypass2_3072_complex_4:
	start_timer 6
	mov	rdi, xsincos_complex	;; The sin/cos ptrs
	bump	rdi, 256*YMM_SCD1+64*YMM_SCD2
	mov	rbp, DIST_TO_MULSRCARG
	yloop_init 192			;; 3 iters of 256 cache lines

	yloop_set_prefetch YHALF	;; Prefetch 96 cache lines in this loop
	yr4_4cl_four_complex_with_mulf_preload
;; Loop below reworked to avoid 4KB distances
;;b6d:	yr4_4cl_four_complex_with_mulf rsi, 64, dist64, 2*dist64
;;	yloop	64, b6d, -64*64	;; Loop 64 times, restore source pointer
b6d:	yr4_4cl_four_complex_with_mulf rsi, 4*64, dist64, 2*dist64
	yloop	16, b6d, -16*4*64+64	;; Loop 16 times, next source pointer
	yloop	4, b6d, -4*64		;; Loop 4 times, restore source pointer
	loops_undo 64			;; Back up loop counter

	yloop_set_prefetch YSINGLE	;; Prefetch 192 cache lines in this loop
	yr4_b4cl_four_complex_djbunfft_preload
c5d:	yr4_b4cl_four_complex_djbunfft rsi, 4*64, 64, 2*64, rdi, 0, 16
	yloop	16, c5d, -16*4*64+dist64, rdi, 4*YMM_SCD2/4 ;; Loop 16 times, next source pointer, next sine/cosine pointer
	yloop	4, c5d, -4*dist64, rdi, -4*4*YMM_SCD2/4 ;; Loop 4 times, restore source pointer, restore sine/cosine pointer
	loops_undo 4*16			;; Restore loop counter

	yloop_set_prefetch YSINGLE	;; Prefetch 192 cache lines in this loop
;;	yr4_b4cl_four_complex_djbunfft_preload
c4d:	yr4_b4cl_four_complex_djbunfft rsi, 64, 4*64, 8*64, rdi, 4*YMM_SCD2/4, 4
	yloop	4, c4d, -4*64+16*64, rdi, -4*4*YMM_SCD2/4 ;; Loop 4 times, next source pointer, restore sine/cosine pointer
	yloop	4, c4d, -4*16*64+dist64, rdi, YMM_SCD2/4 ;; Loop 4 times, next source pointer, next sine/cosine pointer
	yloop	4, c4d, -4*dist64, rdi, -4*YMM_SCD2/4 ;; Loop 4 times, restore source pointer, restore sine/cosine pointer
	loops_undo 4*4*4		;; Back up loop counter

	yloop_set_prefetch YSINGLE	;; Prefetch 192 cache lines in this loop
;;	yr4_b4cl_four_complex_djbunfft_preload
c3d:	yr4_b4cl_four_complex_djbunfft rsi, 64, 16*64, 32*64, rdi, YMM_SCD2/4, 16
	yloop	16, c3d, -16*64+dist64	;; Loop 16 times, next source pointer
	yloop	4, c3d, -4*dist64+4*dist64, rdi, -64*YMM_SCD2/4 ;; Loop 4 times, next source pointer, restore sine/cosine pointer

	yloop	3, b6d, -3*4*dist64	;; Loop 3 times, restore source pointer
	bump	rcx, (-96-3*192+2*64+2*96+3*48+2*96)*64 ;; Move prefetch pointer to where squaring would be
	end_timer 6

;; Do inverse FFT levels 3,4
;; On input the 64-byte cache lines hold these data values:
;;	0	+512	+512	+512	256	+512	+512	+512
;;	4	...
;;	...
;;	252	...
;;	1	...
;;	...
;;	2048	...
;;	...
;; On output the 64-byte cache lines are swizzled to hold these data values:
;;	0	+1	+1	+1	1024	+1	+1	+1
;;	4	...
;;	...
;;	1020	...
;;	2048	...
;;	...

;; Do 768 four_complex_unfft operations
;;	distance between fft data elements is 256
;;	do 192 macros each processing 32 data values

;; Do inverse FFT levels 1,2
;; On input the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	1024	+1	+1	+1
;;	4	...
;;	...
;;	1020	...
;;	2048	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	3072	+1	+1	+1
;;	4	...
;;	...
;;	3068

;; Do 1024 three_complex_unfft operations
;;	distance between fft data elements is 1024
;;	do 256 macros each processing 24 data values

ypass2_3072_complex_unfft:
	mov	rdi, xsincos_complex	;; Load sin/cos pointers
	lea	rbp, [rdi+256*YMM_SCD1]
	bump	rcx, (768+11-2*64-2*96-3*48-2*96-2*64)*64 ;; Move prefetch pointer along
	yloop_init 256			;; 256 iterations or 192 iterations

c2a:	start_timer 13
	yloop_adjust 1			;; One fewer iteration this loop
	yloop_set_prefetch YQUARTER	;; Prefetch 64 cache lines
	yr4_s4cl_four_complex_djbunfft_preload
c2b:	yr4_s4cl_four_complex_djbunfft rsi, 4*dist64, dist64, 2*dist64, rbp, 0, 3, L1PREFETCH_ALL, 64
	yloop	4, c2b, -3*4*dist64, rbp, YMM_SCD2 ;; 3 iterations!!  Restore source pointer, next sine/cosine pointer
	loops_undo 4			;; Restore loop counter
	end_timer 13

	start_timer 14
	yloop_set_prefetch YQUARTER	;; Prefetch 64 cache lines
	yr3_3cl_three_complex_djbunfft_preload
c1b:	yr3_3cl_three_complex_djbunfft rsi, dist64, 4*dist64, rdi, 64*YMM_SCD1, 4
	yloop	4, c1b, -4*dist64+64, rdi, -4*64*YMM_SCD1+YMM_SCD1 ;; Test loop counter, next source pointer, next sine/cosine pointer
	end_timer 14

	yloop	64, c2a			;; Loop 64 times

ypass2_3072_complex_done:
	ENDM


; *************** radix4-pass2-3840-real macro ******************
; This macro takes 7680 real values and performs the final 12.907 levels of the FFT.
; The results are squared, and the first 12.907 levels of the inverse FFT are done.

yr4dwpn_pass2_3840_real MACRO
	LOCAL	b1b, b2a, b2b, b3a, b3b, b4a, b4b, b5a, b5b, b6a, b6b, b6c, b6d
	LOCAL	c1b, c2a, c2b, c3a, c3b, c4a, c4b, c5a, c5b
	LOCAL	ypass2_3840_real_1, ypass2_3840_real_2
	LOCAL	ypass2_3840_real_3, ypass2_3840_real_4
 	LOCAL	ypass2_3840_real_unfft, ypass2_3840_real_done

;; Type 4 FFTs skip the forward FFT process

	start_timer 2
	mov	rsi, DATA_ADDR		;; Load source address
	mov	rbx, DIST_TO_FFTSRCARG
	mov	rcx, DATA_PREFETCH	;; Prefetch pointer (we need to prefetch 
					;; 960 cache lines + 14 pad cache lines)
	cmp	ffttype, 4
	je	ypass2_3840_real_4

;; Do FFT levels 1,2,3 which correspond to complex levels 1,2
;; On input the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	3840	+1	+1	+1
;;	4	...
;;	...
;;	3836
;; On output the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	1280	+1	+1	+1
;;	4	...
;;	...
;;	1276	...
;;	2560	...
;;	...

;; Do 1280 six_reals_fft operations
;;	distance between fft data elements is 1280
;;	do 320 macros each processing 24 data values

	mov	rdi, sincos3		;; Real sin/cos pointer
	mov	rbp, xsincos_complex	;; Complex sin/cos data
	yloop_init 320			;; 320 iterations
	yloop_set_prefetch YHALF
	yr3_f3cl_2sc_six_reals_fft_preload
b1b:	yr3_f3cl_2sc_six_reals_fft rsi, 64, 5*dist64, rbp, YMM_SCD1, rdi, YMM_SCD1, 64
	yloop	64, b1b, -64*64+dist64	;; Test loop counter, next source pointer
	yloop	5, b1b, -5*dist64	;; Test loop counter, restore source pointer

;; Do FFT levels 3,4
;; On input the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	1280	+1	+1	+1
;;	4	...
;;	...
;;	1276	...
;;	2560	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	256	+1	+1	+1
;;	4	...
;;	...
;;	252	...
;;	512	...
;;	...

;; Do 256 ten_reals_fft operations
;; Do 512 five_complex_fft operations
;;	distance between fft data elements is 256
;;	do 192 macros each processing 40 data values

	yloop_init 192			;; 3 iterations of 64
	yloop_set_prefetch YHALF
	yr5_5cl_2sc_ten_reals_fft_preload
b2a:	yr5_5cl_2sc_ten_reals_fft rsi, 64, dist64, rbp, YMM_SCD2, rdi, YMM_SCD2, 64
	yloop	64, b2a, -64*64+5*dist64, rbp, -64*YMM_SCD2 ;; Test loop counter, next source pointer, restore sin/cos pointer

	loops_reset			;; 2 more iters of 64
	yr5_5cl_five_complex_djbfft_preload
b2b:	yr5_5cl_five_complex_djbfft rsi, 64, dist64, rbp, YMM_SCD2, 64
	yloop	64, b2b, -64*64+5*dist64, rbp, -64*YMM_SCD2 ;; Test loop counter, next source pointer, restore sin/cos pointer
	yloop	3, b2b, -3*5*dist64, rbp, 64*YMM_SCD2 ;; Test loop counter, restore source pointer, next sin/cos pointer

;; Do FFT levels 5,6
;; On input the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	256	+1	+1	+1
;;	4	...
;;	...
;;	252	...
;;	512	...
;;	...
;; On output the 64-byte cache lines are swizzled to hold these data values:
;;	0	+128	+128	+128	64	+128	+128	+128
;;	4	...
;;	...
;;	60	...
;;	1	...
;;	...
;;	512	...
;;	...

;; Do 64 eight_reals_fft operations
;; Do 896 four_complex_fft operations
;;	distance between fft data elements is 64
;;	do 240 macros each processing 32 data values

	yloop_init 240			;; 15 iters of 16
	yloop_set_prefetch YHALF
	yr4_s4cl_2sc_eight_reals_fft_preload
b3a:	yr4_s4cl_2sc_eight_reals_fft rsi, 64, 16*64, 32*64, rbp, YMM_SCD2, rdi, YMM_SCD2, 16
	yloop	16, b3a, -16*64+dist64, rbp, -16*YMM_SCD2 ;; Test loop counter, next source pointer, restore sine/cosine pointer

	loops_reset			;; 14 more iters of 16
	yr4_s4cl_four_complex_djbfft_preload
b3b:	yr4_s4cl_four_complex_djbfft rsi, 64, 16*64, 32*64, rbp, YMM_SCD2, 16
	yloop	16, b3b, -16*64+dist64, rbp, -16*YMM_SCD2 ;; Test loop counter, next source pointer, restore sin/cos ptr
	yloop	15, b3b, -15*dist64	;; Loop 15 times, restore source pointer

;; Do FFT levels 7,8
;; On input the 64-byte cache lines hold these data values:
;;	0	+128	+128	+128	64	+128	+128	+128
;;	4	...
;;	...
;;	60	...
;;	1	...
;;	...
;;	512	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+128	+128	+128	16	+128	+128	+128
;;	4	...
;;	...
;;	12	...
;;	32	...
;;	...
;;	1	...
;;	...
;;	512	...
;;	...

;; Do 16 eight_reals_fft operations
;; Do 944 four_complex_fft operations
;;	distance between fft data elements is 16
;;	do 240 macros each processing 32 data values

	yloop_init 240			;; 15 iterations of 16
	yloop_set_prefetch YHALF
	yr4_4cl_eight_reals_four_complex_djbfft_preload
b4a:	yr4_4cl_eight_reals_four_complex_djbfft rsi, 16*64, 4*64, 8*64, rdi, YMM_SCD3, 4
	yloop	4, b4a, -4*16*64+64	;; Test loop counter, next source pointer
	yloop	4, b4a, -4*64+dist64, rdi, -16*YMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer

	loops_reset			;; 14 more iters of 16
	yr4_rb4cl_four_complex_djbfft_preload
b4b:	yr4_rb4cl_four_complex_djbfft rsi, 16*64, 4*64, 8*64, rdi, YMM_SCD3, 4
	yloop	4, b4b, -4*16*64+64	;; Test loop counter, next source pointer
	yloop	4, b4b, -4*64+dist64, rdi, -16*YMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer
	yloop	15, b4b, -15*dist64	;; Test loop counter, restore source pointer

;; Do FFT levels 9,10
;; On input the 64-byte cache lines hold these data values:
;;	0	+128	+128	+128	16	+128	+128	+128
;;	4	...
;;	...
;;	12	...
;;	32	...
;;	...
;;	1	...
;;	...
;;	512	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+128	+128	+128	4	+128	+128	+128
;;	8	...
;;	...
;;	1	...
;;	...
;;	512	...
;;	...

;; Do 4 eight_reals_fft operations
;; Do 956 four_complex_fft operations
;;	distance between fft data elements is 4
;;	do 240 macros each processing 32 data values

	yloop_init 240			;; 60 iterations of 4
	yloop_set_prefetch YHALF
	yr4_4cl_eight_reals_four_complex_djbfft_preload
b5a:	yr4_4cl_eight_reals_four_complex_djbfft rsi, 16*64, 64, 2*64, rdi, 4*YMM_SCD3, 4
	yloop	4, b5a, -4*16*64+4*64, rdi, -4*4*YMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer

	loops_reset			;; 59 more iters of 4
	yr4_rb4cl_four_complex_djbfft_preload
b5b:	yr4_rb4cl_four_complex_djbfft rsi, 16*64, 64, 2*64, rdi, 4*YMM_SCD3, 4
	yloop	4, b5b, -4*16*64+4*64, rdi, -4*4*YMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer
	yloop	4, b5b, -4*4*64+dist64	;; Test loop counter, next source pointer
	yloop	15, b5b, -15*dist64	;; Test loop counter, restore source pointer

;; Do FFT levels 11,12 as well as inverse FFT levels 11,12
;; On input and output the 64-byte cache lines hold these data values:
;;	0	+128	+128	+128	4	+128	+128	+128
;;	8	...
;;	...
;;	1	...
;;	...
;;	512	...
;;	...

;; Do 1 eight_reals_fft operation
;; Do 959 four_complex_fft operations
;;	distance between fft data elements is 1
;;	do 240 macros each processing 32 data values

	cmp	ffttype, 2
	jl	ypass2_3840_real_1
	jg	ypass2_3840_real_3

ypass2_3840_real_2:
;;	yr4_4cl_eight_reals_four_complex_with_square rsi, 64, 16*64, 32*64
	yr4_dispatch call, yreal_square, 16*64, 32*64
	yloop_init 240			;; 239 more iterations
	yloop_adjust 1
	yloop_set_prefetch YHALF
	yr4_4cl_four_complex_with_square_preload
b6b:	yr4_4cl_four_complex_with_square rsi, 64, 16*64, 32*64, 1
	yloop	16, b6b, -16*64+dist64	;; Test loop counter, next source pointer
	yloop	15, b6b, -15*dist64	;; Test loop counter, restore source pointer
	jmp	ypass2_3840_real_unfft

ypass2_3840_real_1:
;;	yr4_4cl_eight_reals_four_complex_fft_final rsi, 64, 16*64, 32*64
	yr4_dispatch call, yreal_fft_final, 16*64, 32*64
	mov	rcx, xsincos_complex	;; Prefetch complex sin/cos data, 480 cache lines
	yloop_init 240			;; 239 more iterations
	yloop_adjust 1
	yloop_set_prefetch YDOUBLE
b6a:	yr4_4cl_four_complex_fft_final rsi, 64, 16*64, 32*64
	yloop	16, b6a, -16*64+dist64	;; Test loop counter, next source pointer
	yloop	15, b6a			;; Test loop counter
	jmp	ypass2_3840_real_done

ypass2_3840_real_3:
x64	mov	r8, DEST2ARG
	mov	rbp, DIST_TO_MULSRCARG
;;	yr4_4cl_eight_reals_four_complex_with_mult rsi, 64, 16*64, 32*64
	yr4_dispatch call, yreal_mult, 16*64, 32*64
	yloop_init 240			;; 239 more iterations
	yloop_adjust 1
	yloop_set_prefetch YHALF
	yr4_4cl_four_complex_with_mult_preload
b6c:	yr4_4cl_four_complex_with_mult rsi, 64, 16*64, 32*64
	yloop	16, b6c, -16*64+dist64	;; Test loop counter, next source pointer
	yloop	15, b6c, -15*dist64	;; Test loop counter, restore source pointer
x64	add	rsi, r8
	jmp	ypass2_3840_real_unfft

ypass2_3840_real_4:
	mov	rbp, DIST_TO_MULSRCARG
;;	yr4_4cl_eight_reals_four_complex_with_mulf rsi, 64, 16*64, 32*64
	yr4_dispatch call, yreal_mulf, 16*64, 32*64
	yloop_init 240			;; 239 more iterations
	yloop_adjust 1
	yloop_set_prefetch YDOUBLE
	yr4_4cl_four_complex_with_mulf_preload
b6d:	yr4_4cl_four_complex_with_mulf rsi, 64, 16*64, 32*64
	yloop	16, b6d, -16*64+dist64	;; Test loop counter, next source pointer
	yloop	15, b6d, -15*dist64	;; Test loop counter, restore source pointer
;;	bump	rcx, (-478+160+96+4*120)*64 ;; Move prefetch pointer back to where squaring would be

;; Do inverse FFT levels 9,10
;; On input the 64-byte cache lines hold these data values:
;;	0	+128	+128	+128	4	+128	+128	+128
;;	8	...
;;	...
;;	1	...
;;	...
;;	512	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+128	+128	+128	16	+128	+128	+128
;;	4	...
;;	...
;;	12	...
;;	32	...
;;	...
;;	1	...
;;	...
;;	512	...
;;	...

;; Do 4 eight_reals_unfft operations
;; Do 956 four_complex_unfft operations
;;	distance between fft data elements is 4
;;	do 240 macros each processing 32 data values

ypass2_3840_real_unfft:
	mov	rdi, sincos3		;; Load sin/cos pointer
	bump	rdi, 16*YMM_SCD2+64*YMM_SCD2+320*YMM_SCD1
	yloop_init 240			;; 60 iterations of 4
	yloop_set_prefetch YHALF
	yr4_4cl_eight_reals_four_complex_djbunfft_preload
c5a:	yr4_4cl_eight_reals_four_complex_djbunfft rsi, 16*64, 64, 2*64, rdi, 4*YMM_SCD3, 4
	yloop	4, c5a, -4*16*64+4*64, rdi, -4*4*YMM_SCD3 ;; Test loop counter, next source pointer, restore sine/cosine pointer

	loops_reset			;; 59 more iters of 4
	yr4_rb4cl_four_complex_djbunfft_preload
c5b:	yr4_rb4cl_four_complex_djbunfft rsi, 16*64, 64, 2*64, rdi, 4*YMM_SCD3, 4
	yloop	4, c5b, -4*16*64+4*64, rdi, -4*4*YMM_SCD3 ;; Test loop counter, next source pointer, restore sine/cosine pointer
	yloop	4, c5b, -4*4*64+dist64	;; Test loop counter, next source pointer
	yloop	15, c5b, -15*dist64	;; Test loop counter, restore source pointer

;; Do inverse FFT levels 7,8
;; On input the 64-byte cache lines hold these data values:
;;	0	+128	+128	+128	16	+128	+128	+128
;;	4	...
;;	...
;;	12	...
;;	32	...
;;	...
;;	1	...
;;	...
;;	512	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+128	+128	+128	64	+128	+128	+128
;;	4	...
;;	...
;;	60	...
;;	1	...
;;	...
;;	512	...
;;	...

;; Do 16 eight_reals_unfft operations
;; Do 944 four_complex_unfft operations
;;	distance between fft data elements is 16
;;	do 240 macros each processing 32 data values

	bump	rcx, (960+14-160-96-5*120-120)*64 ;; Move prefetch pointer along
	yloop_init 240			;; 15 iterations of 16
	yloop_set_prefetch YHALF
	yr4_4cl_eight_reals_four_complex_djbunfft_preload
c4a:	yr4_4cl_eight_reals_four_complex_djbunfft rsi, 16*64, 4*64, 8*64, rdi, YMM_SCD3, 4
	yloop	4, c4a, -4*16*64+64	;; Test loop counter, next source pointer
	yloop	4, c4a, -4*64+dist64, rdi, -16*YMM_SCD3 ;; Test loop counter, next source pointer, restore sine/cosine pointer

	loops_reset			;; 14 more iters of 16
	yr4_rb4cl_four_complex_djbunfft_preload
c4b:	yr4_rb4cl_four_complex_djbunfft rsi, 16*64, 4*64, 8*64, rdi, YMM_SCD3, 4
	yloop	4, c4b, -4*16*64+64	;; Test loop counter, next source pointer
	yloop	4, c4b, -4*64+dist64, rdi, -16*YMM_SCD3 ;; Test loop counter, next source pointer, restore sine/cosine pointer
	yloop	15, c4b, -15*dist64	;; Test loop counter, restore source pointer

;; Do inverse FFT levels 5,6
;; On input the 64-byte cache lines hold these data values:
;;	0	+128	+128	+128	64	+128	+128	+128
;;	4	...
;;	...
;;	60	...
;;	1	...
;;	...
;;	512	...
;;	...
;; On output the 64-byte cache lines are swizzled to hold these data values:
;;	0	+1	+1	+1	256	+1	+1	+1
;;	4	...
;;	...
;;	252	...
;;	512	...
;;	...

;; Do 64 eight_reals_unfft operations
;; Do 896 four_complex_unfft operations
;;	distance between fft data elements is 64
;;	do 240 macros each processing 32 data values

	bump	rdi, -16*YMM_SCD2	;; Real sin/cos ptr
	mov	rbp, xsincos_complex	;; Complex sin/cos data
	bump	rbp, 64*YMM_SCD2+320*YMM_SCD1
	mov	rcx, xsincos_complex	;; Prefetch complex sin/cos data, 480 cache lines
	yloop_init 240			;; 15 iters of 16
	yloop_set_prefetch YSINGLE
	yr4_s4cl_2sc_eight_reals_unfft_preload
c3a:	yr4_s4cl_2sc_eight_reals_unfft rsi, 64, 16*64, 32*64, rbp, YMM_SCD2, rdi, YMM_SCD2, 16
	yloop	16, c3a, -16*64+dist64, rbp, -16*YMM_SCD2 ;; Test loop counter, next source pointer, restore sine/cosine pointer

	loops_reset			;; 14 more iters of 16
	yr4_s4cl_four_complex_djbunfft_preload
c3b:	yr4_s4cl_four_complex_djbunfft rsi, 64, 16*64, 32*64, rbp, YMM_SCD2, 16
	yloop	16, c3b, -16*64+dist64, rbp, -16*YMM_SCD2 ;; Test loop counter, next source pointer, restore sine/cosine pointer
	yloop	15, c3b, -15*dist64	;; Test loop counter, restore source pointer

;; Do inverse FFT levels 3,4
;; On input the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	256	+1	+1	+1
;;	4	...
;;	...
;;	252	...
;;	512	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	1280	+1	+1	+1
;;	4	...
;;	...
;;	1276	...
;;	2560	...
;;	...

;; Do 256 ten_reals_unfft operations
;; Do 512 five_complex_unfft operations
;;	distance between fft data elements is 256
;;	do 192 macros each processing 40 data values

	bump	rdi, -16*YMM_SCD2-64*YMM_SCD2 ;; Real sin/cos ptr
	bump	rbp, -64*YMM_SCD2	;; Complex sin/cos data
	yloop_init 192			;; 3 iters of 64
	yloop_set_prefetch YHALF
	yr5_5cl_2sc_ten_reals_unfft_preload
c2a:	yr5_5cl_2sc_ten_reals_unfft rsi, 64, dist64, rbp, YMM_SCD2, rdi, YMM_SCD2, 64
	yloop	64, c2a, -64*64+5*dist64, rbp, -64*YMM_SCD2 ;; Test loop counter, next source pointer, restore sine/cosine pointer

	loops_reset			;; 2 more iters of 64
	yr5_5cl_five_complex_djbunfft_preload
c2b:	yr5_5cl_five_complex_djbunfft rsi, 64, dist64, rbp, YMM_SCD2, 64
	yloop	64, c2b, -64*64+5*dist64, rbp, -64*YMM_SCD2 ;; Test loop counter, next source pointer, restore sine/cosine pointer
	yloop	3, c2b, -3*5*dist64	;; Test loop counter, restore source pointer

;; Do inverse FFT levels 1,2,3 which correspond to complex levels 1,2
;; On input the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	1280	+1	+1	+1
;;	4	...
;;	...
;;	1276	...
;;	2560	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	3840	+1	+1	+1
;;	4	...
;;	...
;;	3836

;; Do 1280 six_reals_unfft operations
;;	distance between fft data elements is 1280
;;	do 320 macros each processing 24 data values

	bump	rdi, -64*YMM_SCD2-320*YMM_SCD1 ;; Real sin/cos ptr
	bump	rbp, -320*YMM_SCD1	;; Complex sin/cos ptr
	bump	rcx, (480-240-96-160)*64 ;; Move prefetch pointer along
	yloop_init 320			;; 320 iterations
	yloop_set_prefetch YHALF
	yr3_3cl_2sc_six_reals_unfft_preload
c1b:	yr3_3cl_2sc_six_reals_unfft rsi, 64, 5*dist64, rbp, YMM_SCD1, rdi, YMM_SCD1, 64
	yloop	64, c1b, -64*64+dist64	;; Test loop counter, next source pointer
	yloop	5, c1b			;; Test loop counter

ypass2_3840_real_done:
	end_timer 2
	ENDM

; *************** radix4-pass2-3840-complex macro ******************
; This macro takes 3840 complex values and performs the final 11.907 levels of a standard
; radix-4 FFT, squares the results, and does the first 11.907 levels of the inverse FFT.

yr4dwpn_pass2_3840_complex MACRO
	LOCAL	b1a, b1b, b2b, b3a0, b3a, b3b0, b3b, b3c0, b3c, b4a, b4b, b4c, b5a, b5b, b5c, b6a, b6b, b6c, b6d
	LOCAL	c1b, c2a, c2b, c3b, c3c, c3d, c4b, c4c, c4d, c5b, c5c, c5d
	LOCAL	ypass2_3840_complex_1, ypass2_3840_complex_2
	LOCAL	ypass2_3840_complex_3, ypass2_3840_complex_4
	LOCAL	ypass2_3840_complex_unfft, ypass2_3840_complex_done

;; Type 4 FFTs skip the forward FFT process

	mov	rsi, DATA_ADDR		;; Load address of FFT data
	mov	rbx, DIST_TO_FFTSRCARG
	mov	rcx, DATA_PREFETCH	;; Prefetch pointer (we need to prefetch 
					;; 960 cache lines + 14 pad cache lines)
	cmp	ffttype, 4
	je	ypass2_3840_complex_4

;; Do FFT level 1,2 (actually 1.585 levels)
;; On input the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	3840	+1	+1	+1
;;	4	...
;;	...
;;	3836
;; On output the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	1280	+1	+1	+1
;;	4	...
;;	...
;;	1276	...
;;	2560	...
;;	...

;; Do 1280 three_complex_fft operations
;;	distance between fft data elements is 1280
;;	do 320 macros each processing 24 data values

;; Do FFT levels 3,4 (actually 2.322 levels)
;; On input the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	1280	+1	+1	+1
;;	4	...
;;	...
;;	1276	...
;;	2560	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	256	+1	+1	+1
;;	4	...
;;	...
;;	252	...
;;	512	...
;;	...

;; Do 768 five_complex_fft operations
;;	distance between fft data elements is 256
;;	do 192 macros each processing 40 data values

	mov	rdi, xsincos_complex	;; The sin/cos ptrs
	lea	rbp, [rdi+320*YMM_SCD1]
	yloop_init 512			;; 320 iterations or 192 iterations

b1a:	start_timer 4
	yloop_adjust 3			;; Three fewer iterations this loop
	yloop_set_prefetch YOTHER, 8	;; Prefetch 64 cache lines
	yr3_f3cl_three_complex_djbfft_preload
b1b:	yr3_f3cl_three_complex_djbfft rsi, dist64, 5*dist64, rdi, 64*YMM_SCD1, 5
	yloop	8, b1b, -5*dist64, rdi, -5*64*YMM_SCD1+YMM_SCD1 ;; 5 iterations!!  Restore source pointer, next sin/cos data
	loops_undo 8
	end_timer 4

	start_timer 5
	yloop_adjust 5			;; Five fewer iterations this loop
	yloop_set_prefetch YOTHER, 8	;; Prefetch 64 cache lines
	yr5_5cl_five_complex_djbfft_preload
b2b:	yr5_5cl_five_complex_djbfft rsi, 5*dist64, dist64, rbp, 0, 3, L1PREFETCH_ALL, 64
	yloop	8, b2b, -3*5*dist64+64, rbp, YMM_SCD2 ;; 3 iterations!!  Next source pointer, next sin/cos data
	end_timer 5

	yloop	64, b1a, -64*64, rdi, -64*YMM_SCD1+320*YMM_SCD1+64*YMM_SCD2 ;; Test loop counter, restore source pointer, next sin/cos ptr

;; Do FFT levels 5,6
;; On input the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	256	+1	+1	+1
;;	4	...
;;	...
;;	252	...
;;	512	...
;;	...
;; On output the 64-byte cache lines are swizzled to hold these data values:
;;	0	+128	+128	+128	64	+128	+128	+128
;;	4	...
;;	...
;;	60	...
;;	1	...
;;	...
;;	512	...
;;	...

;; Do 960 four_complex_fft operations
;;	distance between fft data elements is 64
;;	do 240 macros each processing 32 data values

;; Do FFT levels 7,8
;; On input the 64-byte cache lines hold these data values:
;;	0	+128	+128	+128	64	+128	+128	+128
;;	4	...
;;	...
;;	60	...
;;	1	...
;;	...
;;	512	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+128	+128	+128	16	+128	+128	+128
;;	4	...
;;	...
;;	12	...
;;	32	...
;;	...
;;	1	...
;;	...
;;	512	...
;;	...

;; Do 960 four_complex_fft operations
;;	distance between fft data elements is 16
;;	do 240 macros each processing 32 data values

;; Do FFT levels 9,10
;; On input the 64-byte cache lines hold these data values:
;;	0	+128	+128	+128	16	+128	+128	+128
;;	4	...
;;	...
;;	12	...
;;	32	...
;;	...
;;	1	...
;;	...
;;	512	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+128	+128	+128	4	+128	+128	+128
;;	8	...
;;	...
;;	1	...
;;	...
;;	512	...
;;	...

;; Do 960 four_complex_fft operations
;;	distance between fft data elements is 4
;;	do 240 macros each processing 32 data values

;; Do FFT levels 11,12 as well as inverse FFT levels 11,12
;; On input and output the 64-byte cache lines hold these data values:
;;	0	+128	+128	+128	4	+128	+128	+128
;;	8	...
;;	...
;;	1	...
;;	...
;;	512	...
;;	...

;; Do 960 four_complex_fft operations
;;	distance between fft data elements is 1
;;	do 240 macros each processing 32 data values

;; Do inverse FFT levels 9,10
;; On input the 64-byte cache lines hold these data values:
;;	0	+128	+128	+128	4	+128	+128	+128
;;	8	...
;;	...
;;	1	...
;;	...
;;	512	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+128	+128	+128	16	+128	+128	+128
;;	4	...
;;	...
;;	12	...
;;	32	...
;;	...
;;	1	...
;;	...
;;	512	...
;;	...

;; Do 960 four_complex_unfft operations
;;	distance between fft data elements is 4
;;	do 240 macros each processing 32 data values

;; Do inverse FFT levels 7,8
;; On input the 64-byte cache lines hold these data values:
;;	0	+128	+128	+128	16	+128	+128	+128
;;	4	...
;;	...
;;	12	...
;;	32	...
;;	...
;;	1	...
;;	...
;;	512	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+128	+128	+128	64	+128	+128	+128
;;	4	...
;;	...
;;	60	...
;;	1	...
;;	...
;;	512	...
;;	...

;; Do 960 four_complex_unfft operations
;;	distance between fft data elements is 16
;;	do 240 macros each processing 32 data values

;; Do inverse FFT levels 5,6
;; On input the 64-byte cache lines hold these data values:
;;	0	+128	+128	+128	64	+128	+128	+128
;;	4	...
;;	...
;;	60	...
;;	1	...
;;	...
;;	512	...
;;	...
;; On output the 64-byte cache lines are swizzled to hold these data values:
;;	0	+1	+1	+1	256	+1	+1	+1
;;	4	...
;;	...
;;	252	...
;;	512	...
;;	...

;; Do 960 four_complex_unfft operations
;;	distance between fft data elements is 64
;;	do 240 macros each processing 32 data values

;; Do this 64 cache lines (4KB) at a time to maximize accessing data
;; while it is in the L1 cache.

;; Execute the right middle step

	cmp	ffttype, 2
	jl	ypass2_3840_complex_1
	jg	ypass2_3840_complex_3

ypass2_3840_complex_2:
	yloop_init 240			;; 15 iters of 64 cache lines

b3b0:	start_timer 6
	yloop_set_prefetch YHALF	;; Prefetch 120 cache lines in this loop
	yr4_s4cl_four_complex_djbfft_preload
b3b:	yr4_s4cl_four_complex_djbfft rsi, 64, 16*64, 32*64, rdi, YMM_SCD2, 16
	yloop	16, b3b, -16*64, rdi, -16*YMM_SCD2 ;; Loop 16 times, restore source pointer, restore sine/cosine pointer
	loops_undo 16			;; Back up loop counter
	end_timer 6

	start_timer 7
	yloop_set_prefetch YHALF	;; Prefetch 120 cache lines in this loop
	yr4_eb4cl_four_complex_djbfft_preload
b4b:	yr4_eb4cl_four_complex_djbfft rsi, 64, 4*64, 8*64, rdi, 4*YMM_SCD2, 4
	yloop	4, b4b, -4*64+16*64, rdi, -4*4*YMM_SCD2+YMM_SCD2 ;; Loop 4 times, next source pointer, next sine/cosine pointer
	yloop	4, b4b, -4*16*64, rdi, -4*YMM_SCD2 ;; Loop 4 times, restore source pointer, restore sine/cosine pointer
	loops_undo 4*4			;; Back up loop counter
	end_timer 7

	start_timer 8
	yloop_set_prefetch YHALF	;; Prefetch 120 cache lines in this loop
;;	yr4_eb4cl_four_complex_djbfft_preload
b5b:	yr4_eb4cl_four_complex_djbfft rsi, 4*64, 64, 2*64, rdi, 0, 4
	yloop	4, b5b, 0, rdi, 4*YMM_SCD2 ;; Loop 4 times, next sine/cosine pointer
	yloop	4, b5b, -16*4*64, rdi, -4*4*YMM_SCD2 ;; Loop 4 times, restore source pointer, restore sine/cosine pointer
	loops_undo 4*4			;; Back up loop counter
	end_timer 8

	start_timer 9
	yloop_set_prefetch YQUARTER	;; Prefetch 60 cache lines in this loop
	yr4_4cl_four_complex_with_square_preload
b6b:	yr4_4cl_four_complex_with_square rsi, 64, 16*64, 32*64, 16
	yloop	16, b6b, -16*64	;; Loop 16 times, restore source pointer
	loops_undo 16			;; Back up loop counter
	end_timer 9

	start_timer 10
	yloop_set_prefetch YQUARTER	;; Prefetch 60 cache lines in this loop
	yr4_eb4cl_four_complex_djbunfft_preload
c5b:	yr4_eb4cl_four_complex_djbunfft rsi, 4*64, 64, 2*64, rdi, 0, 4
	yloop	4, c5b, 0, rdi, 4*YMM_SCD2 ;; Loop 4 times, next sine/cosine pointer
	yloop	4, c5b, -16*4*64, rdi, -4*4*YMM_SCD2 ;; Loop 4 times, restore source pointer, restore sine/cosine pointer
	loops_undo 4*4			;; Restore loop counter
	end_timer 10

	start_timer 11
	yloop_set_prefetch YHALF	;; Prefetch 120 cache lines in this loop
;;	yr4_eb4cl_four_complex_djbunfft_preload
c4b:	yr4_eb4cl_four_complex_djbunfft rsi, 64, 4*64, 8*64, rdi, 4*YMM_SCD2, 4
	yloop	4, c4b, -4*64+16*64, rdi, -4*4*YMM_SCD2+YMM_SCD2 ;; Loop 4 times, next source pointer, next sine/cosine pointer
	yloop	4, c4b, -4*16*64, rdi, -4*YMM_SCD2 ;; Loop 4 times, restore source pointer, restore sine/cosine pointer
	loops_undo 4*4			;; Back up loop counter
	end_timer 11

	start_timer 12
	yloop_set_prefetch YHALF	;; Prefetch 120 cache lines in this loop
	yr4_s4cl_four_complex_djbunfft_preload
c3b:	yr4_s4cl_four_complex_djbunfft rsi, 64, 16*64, 32*64, rdi, YMM_SCD2, 16, L1PREFETCH_ALL, dist64
	yloop	16, c3b, -16*64+dist64, rdi, -16*YMM_SCD2 ;; Loop 16 times, next source pointer, restore sine/cosine pointer
	end_timer 12

	yloop	15, b3b0, -15*dist64	;; Loop 15 times, restore source pointer
	jmp	ypass2_3840_complex_unfft


ypass2_3840_complex_1:
	start_timer 6
;;	bump	rcx, (960+14-2*64-3*240-120)*64 ;; Move prefetch pointer along
	yloop_init 240			;; 15 iters of 64 cache lines

b3a0:	yloop_set_prefetch YSINGLE	;; Prefetch 240 cache lines in this loop
	yr4_s4cl_four_complex_djbfft_preload
b3a:	yr4_s4cl_four_complex_djbfft rsi, 64, 16*64, 32*64, rdi, YMM_SCD2, 16
	yloop	16, b3a, -16*64, rdi, -16*YMM_SCD2 ;; Loop 16 times, restore source pointer, restore sine/cosine pointer
	loops_undo 16			;; Back up loop counter

	yloop_set_prefetch YSINGLE	;; Prefetch 240 cache lines in this loop
	yr4_eb4cl_four_complex_djbfft_preload
b4a:	yr4_eb4cl_four_complex_djbfft rsi, 64, 4*64, 8*64, rdi, 4*YMM_SCD2, 4
	yloop	4, b4a, -4*64+16*64, rdi, -4*4*YMM_SCD2+YMM_SCD2 ;; Loop 4 times, next source pointer, next sine/cosine pointer
	yloop	4, b4a, -4*16*64, rdi, -4*YMM_SCD2 ;; Loop 4 times, restore source pointer, restore sine/cosine pointer
	loops_undo 4*4			;; Back up loop counter

	yloop_set_prefetch YSINGLE	;; Prefetch 240 cache lines in this loop
;;	yr4_eb4cl_four_complex_djbfft_preload
b5a:	yr4_eb4cl_four_complex_djbfft rsi, 4*64, 64, 2*64, rdi, 0, 4, L1PREFETCH_ALL, dist64
	yloop	4, b5a, 0, rdi, 4*YMM_SCD2 ;; Loop 4 times, next sine/cosine pointer
	yloop	4, b5a, -16*4*64, rdi, -4*4*YMM_SCD2 ;; Loop 4 times, restore source pointer, restore sine/cosine pointer
	loops_undo 4*4			;; Back up loop counter

	yloop_set_prefetch YHALF	;; Prefetch 120 cache lines in this loop
	yr4_4cl_four_complex_fft_final_preload
b6a:	yr4_4cl_four_complex_fft_final rsi, 64, 16*64, 32*64
	yloop	16, b6a, -16*64+dist64	;; Loop 16 times, next source pointer

	yloop	15, b3a0		;; Loop 15 times
	end_timer 6
	jmp	ypass2_3840_complex_done


ypass2_3840_complex_3:
	start_timer 6
x64	mov	r8, DEST2ARG
	mov	rbp, DIST_TO_MULSRCARG
	yloop_init 240			;; 15 iters of 64 cache lines

b3c0:	yloop_set_prefetch YHALF	;; Prefetch 120 cache lines in this loop
	yr4_s4cl_four_complex_djbfft_preload
b3c:	yr4_s4cl_four_complex_djbfft rsi, 64, 16*64, 32*64, rdi, YMM_SCD2, 16
	yloop	16, b3c, -16*64, rdi, -16*YMM_SCD2 ;; Loop 16 times, restore source pointer, restore sine/cosine pointer
	loops_undo 16			;; Back up loop counter

	yloop_set_prefetch YHALF	;; Prefetch 120 cache lines in this loop
	yr4_eb4cl_four_complex_djbfft_preload
b4c:	yr4_eb4cl_four_complex_djbfft rsi, 64, 4*64, 8*64, rdi, 4*YMM_SCD2, 4
	yloop	4, b4c, -4*64+16*64, rdi, -4*4*YMM_SCD2+YMM_SCD2 ;; Loop 4 times, next source pointer, next sine/cosine pointer
	yloop	4, b4c, -4*16*64, rdi, -4*YMM_SCD2 ;; Loop 4 times, restore source pointer, restore sine/cosine pointer
	loops_undo 4*4			;; Back up loop counter

	yloop_set_prefetch YHALF	;; Prefetch 120 cache lines in this loop
;;	yr4_eb4cl_four_complex_djbfft_preload
b5c:	yr4_eb4cl_four_complex_djbfft rsi, 4*64, 64, 2*64, rdi, 0, 4
	yloop	4, b5c, 0, rdi, 4*YMM_SCD2 ;; Loop 4 times, next sine/cosine pointer
	yloop	4, b5c, -16*4*64, rdi, -4*4*YMM_SCD2 ;; Loop 4 times, restore source pointer, restore sine/cosine pointer
	loops_undo 4*4			;; Back up loop counter

	yloop_set_prefetch YQUARTER	;; Prefetch 60 cache lines in this loop
	yr4_4cl_four_complex_with_mult_preload
b6c:	yr4_4cl_four_complex_with_mult rsi, 64, 16*64, 32*64
	yloop	16, b6c, -16*64		;; Loop 16 times, restore source pointer
x64	add	rsi, r8
	loops_undo 16			;; Back up loop counter

	yloop_set_prefetch YQUARTER	;; Prefetch 60 cache lines in this loop
	yr4_eb4cl_four_complex_djbunfft_preload
c5c:	yr4_eb4cl_four_complex_djbunfft rsi, 4*64, 64, 2*64, rdi, 0, 4
	yloop	4, c5c, 0, rdi, 4*YMM_SCD2 ;; Loop 4 times, next sine/cosine pointer
	yloop	4, c5c, -16*4*64, rdi, -4*4*YMM_SCD2 ;; Loop 4 times, restore source pointer, restore sine/cosine pointer
	loops_undo 4*4			;; Restore loop counter

	yloop_set_prefetch YHALF	;; Prefetch 120 cache lines in this loop
;;	yr4_eb4cl_four_complex_djbunfft_preload
c4c:	yr4_eb4cl_four_complex_djbunfft rsi, 64, 4*64, 8*64, rdi, 4*YMM_SCD2, 4
	yloop	4, c4c, -4*64+16*64, rdi, -4*4*YMM_SCD2+YMM_SCD2 ;; Loop 4 times, next source pointer, next sine/cosine pointer
	yloop	4, c4c, -4*16*64, rdi, -4*YMM_SCD2 ;; Loop 4 times, restore source pointer, restore sine/cosine pointer
	loops_undo 4*4			;; Back up loop counter

	yloop_set_prefetch YHALF	;; Prefetch 120 cache lines in this loop
	yr4_s4cl_four_complex_djbunfft_preload
c3c:	yr4_s4cl_four_complex_djbunfft rsi, 64, 16*64, 32*64, rdi, YMM_SCD2, 16, L1PREFETCH_ALL, dist64
	yloop	16, c3c, -16*64+dist64, rdi, -16*YMM_SCD2 ;; Loop 16 times, next source pointer, restore sine/cosine pointer

x64	sub	rsi, r8
	yloop	15, b3c0, -15*dist64	;; Loop 15 times, restore source pointer
x64	add	rsi, r8
	end_timer 6
	jmp	ypass2_3840_complex_unfft


ypass2_3840_complex_4:
	start_timer 6
	mov	rdi, xsincos_complex	;; The sin/cos ptrs
	bump	rdi, 320*YMM_SCD1+64*YMM_SCD2
	mov	rbp, DIST_TO_MULSRCARG
	yloop_init 240			;; 15 iters of 64 cache lines

	yloop_set_prefetch YHALF	;; Prefetch 120 cache lines in this loop
	yr4_4cl_four_complex_with_mulf_preload
b6d:	yr4_4cl_four_complex_with_mulf rsi, 64, 16*64, 32*64
	yloop	16, b6d, -16*64	;; Loop 16 times, restore source pointer
	loops_undo 16			;; Back up loop counter

	yloop_set_prefetch YSINGLE	;; Prefetch 240 cache lines in this loop
	yr4_eb4cl_four_complex_djbunfft_preload
c5d:	yr4_eb4cl_four_complex_djbunfft rsi, 4*64, 64, 2*64, rdi, 0, 4
	yloop	4, c5d, 0, rdi, 4*YMM_SCD2 ;; Loop 4 times, next sine/cosine pointer
	yloop	4, c5d, -16*4*64, rdi, -4*4*YMM_SCD2 ;; Loop 4 times, restore source pointer, restore sine/cosine pointer
	loops_undo 4*4			;; Back up loop counter

	yloop_set_prefetch YSINGLE	;; Prefetch 240 cache lines in this loop
;;	yr4_eb4cl_four_complex_djbunfft_preload
c4d:	yr4_eb4cl_four_complex_djbunfft rsi, 64, 4*64, 8*64, rdi, 4*YMM_SCD2, 4
	yloop	4, c4d, -4*64+16*64, rdi, -4*4*YMM_SCD2+YMM_SCD2 ;; Loop 4 times, next source pointer, next sine/cosine pointer
	yloop	4, c4d, -4*16*64, rdi, -4*YMM_SCD2 ;; Loop 4 times, restore source pointer, restore sine/cosine pointer
	loops_undo 4*4			;; Back up loop counter

	yloop_set_prefetch YSINGLE	;; Prefetch 240 cache lines in this loop
	yr4_s4cl_four_complex_djbunfft_preload
c3d:	yr4_s4cl_four_complex_djbunfft rsi, 64, 16*64, 32*64, rdi, YMM_SCD2, 16, L1PREFETCH_ALL, dist64
	yloop	16, c3d, -16*64+dist64, rdi, -16*YMM_SCD2 ;; Loop 16 times, next source pointer, restore sine/cosine pointer

	yloop	15, b6d, -15*dist64	;; Loop 15 times, restore source pointer
;;	bump	rcx, (-120-3*240+2*64+3*120+2*60+2*120)*64 ;; Move prefetch pointer to where squaring would be
	end_timer 6

;; Do inverse FFT levels 3,4
;; On input the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	256	+1	+1	+1
;;	4	...
;;	...
;;	252	...
;;	512	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	1280	+1	+1	+1
;;	4	...
;;	...
;;	1276	...
;;	2560	...
;;	...

;; Do 768 five_complex_unfft operations
;;	distance between fft data elements is 256
;;	do 192 macros each processing 40 data values

;; Do inverse FFT levels 1,2
;; On input the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	1280	+1	+1	+1
;;	4	...
;;	...
;;	1276	...
;;	2560	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	3840	+1	+1	+1
;;	4	...
;;	...
;;	3836

;; Do 1280 three_complex_unfft operations
;;	distance between fft data elements is 1280
;;	do 320 macros each processing 24 data values

ypass2_3840_complex_unfft:
	mov	rdi, xsincos_complex	;; Load sin/cos pointers
	lea	rbp, [rdi+320*YMM_SCD1]
	bump	rcx, (960+14-2*64-3*120-2*60-2*120-2*64)*64 ;; Move prefetch pointer along
	yloop_init 512			;; 192 iterations or 320 iterations

c2a:	start_timer 13
	yloop_adjust 5			;; Five fewer iterations in this loop
	yloop_set_prefetch YOTHER, 8	;; Prefetch 64 cache lines
	yr5_5cl_five_complex_djbunfft_preload
c2b:	yr5_5cl_five_complex_djbunfft rsi, 5*dist64, dist64, rbp, 0, 3
	yloop	8, c2b, -3*5*dist64, rbp, YMM_SCD2 ;; 3 iterations!!  Restore source pointer, next sine/cosine pointer
	loops_undo 8			;; Restore loop counter
	end_timer 13

	start_timer 14
	yloop_adjust 3			;; Three fewer iterations in this loop
	yloop_set_prefetch YOTHER, 8	;; Prefetch 64 cache lines
	yr3_3cl_three_complex_djbunfft_preload
c1b:	yr3_3cl_three_complex_djbunfft rsi, dist64, 5*dist64, rdi, 64*YMM_SCD1, 5, L1PREFETCH_ALL, 64
	yloop	8, c1b, -5*dist64+64, rdi, -5*64*YMM_SCD1+YMM_SCD1 ;; 5 iterations!!  Next source pointer, next sine/cosine pointer
	end_timer 14

	yloop	64, c2a			;; Loop 64 times

ypass2_3840_complex_done:
	ENDM


; *************** radix4-pass2-12-levels-real macro ******************
; This macro takes 8192 real values and performs the final 13 levels of the FFT.
; The results are squared, and the first 13 levels of the inverse FFT are done.

yr4dwpn_pass2_12_levels_real MACRO
	LOCAL	b1b, b2b, b3a, b3b, b4a, b4b, b5a, b5b, b6a, b6b, b6c, b6d
	LOCAL	c1b, c2b, c3a, c3b, c4a, c4b, c5a, c5b
	LOCAL	ypass2_12_levels_real_1, ypass2_12_levels_real_2
	LOCAL	ypass2_12_levels_real_3, ypass2_12_levels_real_4
	LOCAL	ypass2_12_levels_real_unfft, ypass2_12_levels_real_done

;; Type 4 FFTs skip the forward FFT process

	start_timer 2
	mov	rsi, DATA_ADDR		;; Load source address
	mov	rbx, DIST_TO_FFTSRCARG
	mov	rcx, DATA_PREFETCH	;; Prefetch pointer (we need to prefetch 
					;; 1024 cache lines + 15 pad cache lines)
	cmp	ffttype, 4
	je	ypass2_12_levels_real_4

;; Do FFT levels 1,2,3 which correspond to complex levels 1,2
;; On input the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	4096	+1	+1	+1
;;	4	...
;;	...
;;	4092
;; On output these are swizzled to:
;;	0	+2K	+2K	+2K	1024	+2K	+2K	+2K
;;	4	...
;;	...
;;	1020	...
;;	1	...
;;	...

;; Do 1024 eight_reals_fft operations
;;	distance between fft data elements is 1024
;;	do 256 macros each processing 32 data values

	mov	rdi, sincos3		;; Real sin/cos data
	mov	rbp, xsincos_complex	;; Complex sin/cos data
	yloop_init 256			;; 256 iterations
	yloop_set_prefetch YHALF
	yr4_fs4cl_2sc_eight_reals_fft_preload
b1b:	yr4_fs4cl_2sc_eight_reals_fft rsi, 64, 4*dist64, 8*dist64, rbp, YMM_SCD2, rdi, YMM_SCD2, 64
	yloop	64, b1b, -64*64+dist64	;; Test loop counter, next source pointer
	yloop	4, b1b, -4*dist64	;; Test loop counter, restore source pointer

;; Do FFT levels 3,4
;; On input the 64-byte cache lines hold these data values:
;;	0	+2K	+2K	+2K	1024	+2K	+2K	+2K
;;	4	...
;;	...
;;	1020	...
;;	1	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+2K	+2K	+2K	256	+2K	+2K	+2K
;;	4	...
;;	...
;;	252	...
;;	512	...
;;	...
;;	1	...
;;	...

;; Do 256 eight_reals_fft operations
;; Do 768 four_complex_fft operations
;;	distance between fft data elements is 256
;;	do 256 macros each processing 32 data values

	yloop_init 256			;; 256 iterations
	yloop_set_prefetch YHALF
	yr4_4cl_eight_reals_four_complex_djbfft_preload
b2b:	yr4_4cl_eight_reals_four_complex_djbfft rsi, 4*dist64, dist64, 2*dist64, rdi, YMM_SCD3, 4
	yloop	4, b2b, -4*4*dist64+64	;; Test loop counter, next source pointer
	yloop	64, b2b, -64*64, rdi, -256*YMM_SCD3 ;; Test loop counter, restore source pointer, restore sin/cos ptr

;; Do FFT levels 5,6
;; On input the 64-byte cache lines hold these data values:
;;	0	+2K	+2K	+2K	256	+2K	+2K	+2K
;;	4	...
;;	...
;;	252	...
;;	512	...
;;	...
;;	1	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+2K	+2K	+2K	64	+2K	+2K	+2K
;;	4	...
;;	...
;;	60	...
;;	128	...
;;	...
;;	1	...
;;	...

;; Do 64 eight_reals_fft operations
;; Do 960 four_complex_fft operations
;;	distance between fft data elements is 64
;;	do 256 macros each processing 32 data values

	yloop_init 256			;; 4 iterations of 64
	yloop_set_prefetch YHALF
	yr4_4cl_eight_reals_four_complex_djbfft_preload
b3a:	yr4_4cl_eight_reals_four_complex_djbfft rsi, 4*dist64, 16*64, 32*64, rdi, 4*YMM_SCD3, 4
	yloop	4, b3a, -4*4*dist64+64	;; Test loop counter, next source pointer
	yloop	16, b3a, -16*64+dist64, rdi, -64*4*YMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer

	loops_reset			;; 3 more iters of 64
	yr4_rb4cl_four_complex_djbfft_preload
b3b:	yr4_rb4cl_four_complex_djbfft rsi, 4*dist64, 16*64, 32*64, rdi, 4*YMM_SCD3, 4
	yloop	4, b3b, -4*4*dist64+64	;; Test loop counter, next source pointer
	yloop	16, b3b, -16*64+dist64, rdi, -64*4*YMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer
	yloop	4, b3b, -4*dist64	;; Test loop counter, restore source pointer

;; Do FFT levels 7,8
;; On input the 64-byte cache lines hold these data values:
;;	0	+2K	+2K	+2K	64	+2K	+2K	+2K
;;	4	...
;;	...
;;	60	...
;;	128	...
;;	...
;;	1	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+2K	+2K	+2K	16	+2K	+2K	+2K
;;	4	...
;;	...
;;	12	...
;;	32	...
;;	...
;;	1	...
;;	...

;; Do 16 eight_reals_fft operations
;; Do 1008 four_complex_fft operations
;;	distance between fft data elements is 16
;;	do 256 macros each processing 32 data values

	yloop_init 256			;; 16 iterations of 16
	yloop_set_prefetch YHALF
	yr4_4cl_eight_reals_four_complex_djbfft_preload
b4a:	yr4_4cl_eight_reals_four_complex_djbfft rsi, 4*dist64, 4*64, 8*64, rdi, 16*YMM_SCD3, 4
	yloop	4, b4a, -4*4*dist64+64	;; Test loop counter, next source pointer
	yloop	4, b4a, -4*64+16*64, rdi, -16*16*YMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer

	loops_reset			;; 15 more iters of 16
	yr4_rb4cl_four_complex_djbfft_preload
b4b:	yr4_rb4cl_four_complex_djbfft rsi, 4*dist64, 4*64, 8*64, rdi, 16*YMM_SCD3, 4
	yloop	4, b4b, -4*4*dist64+64	;; Test loop counter, next source pointer
	yloop	4, b4b, -4*64+16*64, rdi, -16*16*YMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer
	yloop	4, b4b, -4*16*64+dist64	;; Test loop counter, next source pointer
	yloop	4, b4b, -4*dist64	;; Test loop counter, restore source pointer

;; Do FFT levels 9,10
;; On input the 64-byte cache lines hold these data values:
;;	0	+2K	+2K	+2K	16	+2K	+2K	+2K
;;	4	...
;;	...
;;	12	...
;;	32	...
;;	...
;;	1	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+2K	+2K	+2K	4	+2K	+2K	+2K
;;	8	...
;;	...
;;	1	...
;;	...

;; Do 4 eight_reals_fft operations
;; Do 1020 four_complex_fft operations
;;	distance between fft data elements is 4
;;	do 256 macros each processing 32 data values

	yloop_init 256			;; 64 iterations of 4
	yloop_set_prefetch YHALF
	yr4_4cl_eight_reals_four_complex_djbfft_preload
b5a:	yr4_4cl_eight_reals_four_complex_djbfft rsi, 4*dist64, 64, 2*64, rdi, 64*YMM_SCD3, 4
	yloop	4, b5a, -4*4*dist64+4*64, rdi, -4*64*YMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer

	loops_reset			;; 63 more iters of 4
	yr4_rb4cl_four_complex_djbfft_preload
b5b:	yr4_rb4cl_four_complex_djbfft rsi, 4*dist64, 64, 2*64, rdi, 64*YMM_SCD3, 4
	yloop	4, b5b, -4*4*dist64+4*64, rdi, -4*64*YMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer
	yloop	16, b5b, -16*4*64+dist64 ;; Test loop counter, next source pointer
	yloop	4, b5b, -4*dist64	;; Test loop counter, restore source pointer

;; Do FFT levels 11,12 as well as inverse FFT levels 11,12
;; On input and output the 64-byte cache lines hold these data values:
;;	0	+2K	+2K	+2K	4	+2K	+2K	+2K
;;	8	...
;;	...
;;	1	...
;;	...

;; Do 1 eight_reals_fft operation
;; Do 1023 four_complex_fft operations
;;	distance between fft data elements is 1
;;	do 256 macros each processing 32 data values

	cmp	ffttype, 2
	jl	ypass2_12_levels_real_1
	jg	ypass2_12_levels_real_3

ypass2_12_levels_real_2:
;;	yr4_4cl_eight_reals_four_complex_with_square rsi, 64, 4*dist64, 8*dist64
	yr4_dispatch call, yreal_square, 4*dist64, 8*dist64
	yloop_init 256			;; 255 more iterations
	yloop_adjust 1
	yloop_set_prefetch YOTHER, 16
	yr4_4cl_four_complex_with_square_preload
b6b:	yr4_4cl_four_complex_with_square rsi, 64, 4*dist64, 8*dist64, 1
	yloop	64, b6b, -64*64+dist64	;; Test loop counter, next source pointer
	yloop	4, b6b, -4*dist64	;; Test loop counter, restore source pointer
	jmp	ypass2_12_levels_real_unfft

ypass2_12_levels_real_1:
;;	yr4_4cl_eight_reals_four_complex_fft_final rsi, 64, 4*dist64, 8*dist64
	yr4_dispatch call, yreal_fft_final, 4*dist64, 8*dist64
	mov	rcx, xsincos_complex	;; Prefetch complex sin/cos data, 544 cache lines
	yloop_init 256			;; 255 more iterations
	yloop_adjust 1
	yloop_set_prefetch YDOUBLE
b6a:	yr4_4cl_four_complex_fft_final rsi, 64, 4*dist64, 8*dist64
	yloop	64, b6a, -64*64+dist64	;; Test loop counter, next source pointer
	yloop	4, b6a			;; Test loop counter
	jmp	ypass2_12_levels_real_done

ypass2_12_levels_real_3:
x64	mov	r8, DEST2ARG
	mov	rbp, DIST_TO_MULSRCARG
;;	yr4_4cl_eight_reals_four_complex_with_mult rsi, 64, 4*dist64, 8*dist64
	yr4_dispatch call, yreal_mult, 4*dist64, 8*dist64
	yloop_init 256			;; 255 more iterations
	yloop_adjust 1
	yloop_set_prefetch YOTHER, 16
	yr4_4cl_four_complex_with_mult_preload
b6c:	yr4_4cl_four_complex_with_mult rsi, 64, 4*dist64, 8*dist64
	yloop	64, b6c, -64*64+dist64	;; Test loop counter, next source pointer
	yloop	4, b6c, -4*dist64	;; Test loop counter, restore source pointer
x64	add	rsi, r8
	jmp	ypass2_12_levels_real_unfft

ypass2_12_levels_real_4:
	mov	rbp, DIST_TO_MULSRCARG
;;	yr4_4cl_eight_reals_four_complex_with_mulf rsi, 64, 4*dist64, 8*dist64
	yr4_dispatch call, yreal_mulf, 4*dist64, 8*dist64
	yloop_init 256			;; 255 more iterations
	yloop_adjust 1
	yloop_set_prefetch YDOUBLE
	yr4_4cl_four_complex_with_mulf_preload
b6d:	yr4_4cl_four_complex_with_mulf rsi, 64, 4*dist64, 8*dist64
	yloop	64, b6d, -64*64+dist64	;; Test loop counter, next source pointer
	yloop	4, b6d, -4*dist64	;; Test loop counter, restore source pointer
;;	bump	rcx, (-512+5*128+16)*64 ;; Move prefetch pointer back to where squaring would be

;; Do inverse FFT levels 9,10
;; On input the 64-byte cache lines hold these data values:
;;	0	+2K	+2K	+2K	4	+2K	+2K	+2K
;;	8	...
;;	...
;;	1	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+2K	+2K	+2K	16	+2K	+2K	+2K
;;	4	...
;;	...
;;	12	...
;;	32	...
;;	...
;;	1	...
;;	...

;; Do 4 eight_reals_unfft operations
;; Do 1020 four_complex_unfft operations
;;	distance between fft data elements is 4
;;	do 256 macros each processing 32 data values

ypass2_12_levels_real_unfft:
	mov	rdi, sincos3		;; Load sin/cos pointer
	bump	rdi, 256*YMM_SCD2
	yloop_init 256			;; 64 iterations of 4
	yloop_set_prefetch YHALF
	yr4_4cl_eight_reals_four_complex_djbunfft_preload
c5a:	yr4_4cl_eight_reals_four_complex_djbunfft rsi, 4*dist64, 64, 2*64, rdi, 64*YMM_SCD3, 4
	yloop	4, c5a, -4*4*dist64+4*64, rdi, -4*64*YMM_SCD3 ;; Test loop counter, next source pointer, restore sine/cosine pointer

	loops_reset			;; 63 more iters of 4
	yr4_rb4cl_four_complex_djbunfft_preload
c5b:	yr4_rb4cl_four_complex_djbunfft rsi, 4*dist64, 64, 2*64, rdi, 64*YMM_SCD3, 4
	yloop	4, c5b, -4*4*dist64+4*64, rdi, -4*64*YMM_SCD3 ;; Test loop counter, next source pointer, restore sine/cosine pointer
	yloop	16, c5b, -16*4*64+dist64	;; Test loop counter, next source pointer
	yloop	4, c5b, -4*dist64	;; Test loop counter, restore source pointer

;; Do inverse FFT levels 7,8
;; On input the 64-byte cache lines hold these data values:
;;	0	+2K	+2K	+2K	16	+2K	+2K	+2K
;;	4	...
;;	...
;;	12	...
;;	32	...
;;	...
;;	1	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+2K	+2K	+2K	64	+2K	+2K	+2K
;;	4	...
;;	...
;;	60	...
;;	128	...
;;	...
;;	1	...
;;	...

;; Do 16 eight_reals_unfft operations
;; Do 1008 four_complex_unfft operations
;;	distance between fft data elements is 16
;;	do 256 macros each processing 32 data values

	bump	rcx, (1024+15-5*128-16-128-256)*64 ;; Move prefetch pointer along
	yloop_init 256			;; 16 iterations of 16
	yloop_set_prefetch YSINGLE
	yr4_4cl_eight_reals_four_complex_djbunfft_preload
c4a:	yr4_4cl_eight_reals_four_complex_djbunfft rsi, 4*dist64, 4*64, 8*64, rdi, 16*YMM_SCD3, 4
	yloop	4, c4a, -4*4*dist64+64	;; Test loop counter, next source pointer
	yloop	4, c4a, -4*64+16*64, rdi, -16*16*YMM_SCD3 ;; Test loop counter, next source pointer, restore sine/cosine pointer

	loops_reset			;; 15 more iters of 16
	yr4_rb4cl_four_complex_djbunfft_preload
c4b:	yr4_rb4cl_four_complex_djbunfft rsi, 4*dist64, 4*64, 8*64, rdi, 16*YMM_SCD3, 4
	yloop	4, c4b, -4*4*dist64+64	;; Test loop counter, next source pointer
	yloop	4, c4b, -4*64+16*64, rdi, -16*16*YMM_SCD3 ;; Test loop counter, next source pointer, restore sine/cosine pointer
	yloop	4, c4b, -4*16*64+dist64	;; Test loop counter, next source pointer
	yloop	4, c4b, -4*dist64	;; Test loop counter, restore source pointer

;; Do inverse FFT levels 5,6
;; On input the 64-byte cache lines hold these data values:
;;	0	+2K	+2K	+2K	64	+2K	+2K	+2K
;;	4	...
;;	...
;;	60	...
;;	128	...
;;	...
;;	1	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+2K	+2K	+2K	256	+2K	+2K	+2K
;;	4	...
;;	...
;;	252	...
;;	512	...
;;	...
;;	1	...
;;	...

;; Do 64 eight_reals_unfft operations
;; Do 1960 four_complex_unfft operations
;;	distance between fft data elements is 64
;;	do 256 macros each processing 32 data values

	mov	rcx, xsincos_complex	;; Prefetch complex sin/cos data, 544 cache lines
	yloop_init 256			;; 4 iterations of 64
	yloop_set_prefetch YSINGLE
	yr4_4cl_eight_reals_four_complex_djbunfft_preload
c3a:	yr4_4cl_eight_reals_four_complex_djbunfft rsi, 4*dist64, 16*64, 32*64, rdi, 4*YMM_SCD3, 4
	yloop	4, c3a, -4*4*dist64+64	;; Test loop counter, next source pointer
	yloop	16, c3a, -16*64+dist64, rdi, -64*4*YMM_SCD3 ;; Test loop counter, next source pointer, restore sine/cosine pointer

	loops_reset			;; 3 more iters of 64
	yr4_rb4cl_four_complex_djbunfft_preload
c3b:	yr4_rb4cl_four_complex_djbunfft rsi, 4*dist64, 16*64, 32*64, rdi, 4*YMM_SCD3, 4
	yloop	4, c3b, -4*4*dist64+64	;; Test loop counter, next source pointer
	yloop	16, c3b, -16*64+dist64, rdi, -64*4*YMM_SCD3 ;; Test loop counter, next source pointer, restore sine/cosine pointer
	yloop	4, c3b, -4*dist64	;; Test loop counter, restore source pointer

;; Do inverse FFT levels 3,4
;; On input the 64-byte cache lines hold these data values:
;;	0	+2K	+2K	+2K	256	+2K	+2K	+2K
;;	4	...
;;	...
;;	252	...
;;	512	...
;;	...
;;	1	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+2K	+2K	+2K	1024	+2K	+2K	+2K
;;	4	...
;;	...
;;	1020	...
;;	1	...
;;	...

;; Do 256 eight_reals_unfft operations
;; Do 768 four_complex_unfft operations
;;	distance between fft data elements is 256
;;	do 256 macros each processing 32 data values

	yloop_init 256			;; 256 iterations
	yloop_set_prefetch YSINGLE
	yr4_4cl_eight_reals_four_complex_djbunfft_preload
c2b:	yr4_4cl_eight_reals_four_complex_djbunfft rsi, 4*dist64, dist64, 2*dist64, rdi, YMM_SCD3, 4
	yloop	4, c2b, -4*4*dist64+64	;; Test loop counter, next source pointer
	yloop	64, c2b, -64*64	;; Test loop counter, restore source pointer

;; Do inverse FFT levels 1,2,3 which correspond to complex levels 1,2
;; On input the 64-byte cache lines hold these data values:
;;	0	+2K	+2K	+2K	1024	+2K	+2K	+2K
;;	4	...
;;	...
;;	1020	...
;;	1	...
;;	...
;; On output the 64-byte cache lines are swizzled to hold these data values:
;;	0	+1	+1	+1	4096	+1	+1	+1
;;	4	...
;;	...
;;	4092

;; Do 1024 eight_reals_unfft operations
;;	distance between fft data elements is 1024
;;	do 256 macros each processing 32 data values

	bump	rdi, -256*YMM_SCD3-256*YMM_SCD2 ;; Real sin/cos pointer
	mov	rbp, xsincos_complex	;; Complex sin/cos data
	bump	rcx, (544-2*256-32)*64	;; Move prefetch pointer along
	yloop_init 256			;; 256 iterations
	yloop_set_prefetch YOTHER, 8
	yr4_s4cl_2sc_eight_reals_unfft_preload
c1b:	yr4_s4cl_2sc_eight_reals_unfft rsi, 64, 4*dist64, 8*dist64, rbp, YMM_SCD2, rdi, YMM_SCD2, 64
	yloop	64, c1b, -64*64+dist64	;; Test loop counter, next source pointer
	yloop	4, c1b			;; Test loop counter

ypass2_12_levels_real_done:
	end_timer 2
	ENDM

; *************** radix4-pass2-12-levels-complex macro ******************
; This macro takes 4096 complex values and performs the final 12 levels of a standard
; radix-4 FFT, squares the results, and does the first 12 levels of the inverse FFT.

yr4dwpn_pass2_12_levels_complex MACRO
	LOCAL	b1a, b1b, b2b, b3a0, b3a, b3b0, b3b, b3c0, b3c, b4a, b4b, b4c, b5a, b5b, b5c, b6a, b6b, b6c, b6d
	LOCAL	c1b, c2a, c2b, c3b, c3c, c3d, c4b, c4c, c4d, c5b, c5c, c5d
	LOCAL	ypass2_12_levels_complex_1, ypass2_12_levels_complex_2
	LOCAL	ypass2_12_levels_complex_3, ypass2_12_levels_complex_4
	LOCAL	ypass2_12_levels_complex_unfft, ypass2_12_levels_complex_done

;; Type 4 FFTs skip the forward FFT process

	mov	rsi, DATA_ADDR		;; Load address of FFT data
	mov	rbx, DIST_TO_FFTSRCARG
	mov	rcx, DATA_PREFETCH	;; Prefetch pointer (we need to prefetch 
					;; 1024 cache lines + 15 pad cache lines)
	cmp	ffttype, 4
	je	ypass2_12_levels_complex_4

;; Do FFT level 1,2
;; On input the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	4096	+1	+1	+1
;;	4	...
;;	...
;;	4092
;; On output the 64-byte cache lines are swizzled to hold these data values:
;;	0	+2K	+2K	+2K	1024	+2K	+2K	+2K
;;	4	...
;;	...
;;	1020	...
;;	1	...
;;	...

;; Do 1024 four_complex_fft operations
;;	distance between fft data elements is 1024
;;	do 256 macros each processing 32 data values

;; Do FFT levels 3,4
;; On input the 64-byte cache lines hold these data values:
;;	0	+2K	+2K	+2K	1024	+2K	+2K	+2K
;;	4	...
;;	...
;;	1020	...
;;	1	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+2K	+2K	+2K	256	+2K	+2K	+2K
;;	4	...
;;	...
;;	252	...
;;	512	...
;;	...
;;	1	...
;;	...

;; Do 1024 four_complex_fft operations
;;	distance between fft data elements is 256
;;	do 256 macros each processing 32 data values

	mov	rdi, xsincos_complex	;; The sin/cos ptrs
	mov	rbp, rdi
	yloop_init 256			;; 256 iterations
	yloop_set_prefetch YHALF

b1a:	start_timer 4
	yr4_fs4cl_four_complex_djbfft_preload
b1b:	yr4_fs4cl_four_complex_djbfft rsi, dist64, 4*dist64, 8*dist64, rdi, 64*YMM_SCD2, 4
	yloop	4, b1b, -4*dist64, rdi, -4*64*YMM_SCD2+YMM_SCD2 ;; Test loop counter, restore source pointer, next sin/cos data
	loops_undo 4
	end_timer 4

	start_timer 5
	yr4_eb4cl_four_complex_djbfft_preload
b2b:	yr4_eb4cl_four_complex_djbfft rsi, 4*dist64, dist64, 2*dist64, rbp, YMM_SCD2, 4, L1PREFETCH_ALL, 64
	yloop	4, b2b, -4*4*dist64+64	;; Test loop counter, next source pointer
	end_timer 5

	yloop	64, b1a, -64*64, rdi, -64*YMM_SCD2+256*YMM_SCD2 ;; Test loop counter, restore source pointer, next sin/cos ptr

;; Do FFT levels 5,6
;; On input the 64-byte cache lines hold these data values:
;;	0	+2K	+2K	+2K	256	+2K	+2K	+2K
;;	4	...
;;	...
;;	252	...
;;	512	...
;;	...
;;	1	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+2K	+2K	+2K	64	+2K	+2K	+2K
;;	4	...
;;	...
;;	60	...
;;	128	...
;;	...
;;	1	...
;;	...
;;
;; Do 1024 four_complex_fft operations
;;	distance between fft data elements is 64
;;	do 256 macros each processing 32 data values

;; Do FFT levels 7,8
;; On input the 64-byte cache lines hold these data values:
;;	0	+2K	+2K	+2K	64	+2K	+2K	+2K
;;	4	...
;;	...
;;	60	...
;;	128	...
;;	...
;;	1	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+2K	+2K	+2K	16	+2K	+2K	+2K
;;	4	...
;;	...
;;	12	...
;;	32	...
;;	...
;;	1	...
;;	...
;;
;; Do 1024 four_complex_fft operations
;;	distance between fft data elements is 16
;;	do 256 macros each processing 32 data values

;; Do FFT levels 9,10
;; On input the 64-byte cache lines hold these data values:
;;	0	+2K	+2K	+2K	16	+2K	+2K	+2K
;;	4	...
;;	...
;;	12	...
;;	32	...
;;	...
;;	1	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+2K	+2K	+2K	4	+2K	+2K	+2K
;;	8	...
;;	...
;;	1	...
;;	...
;;
;; Do 1024 four_complex_fft operations
;;	distance between fft data elements is 4
;;	do 256 macros each processing 32 data values

;; Do FFT levels 11,12 as well as inverse FFT levels 11,12
;; On input and output the 64-byte cache lines hold these data values:
;;	0	+2K	+2K	+2K	4	+2K	+2K	+2K
;;	8	...
;;	...
;;	1	...
;;	...

;; Do 1024 four_complex_fft operations
;;	distance between fft data elements is 1
;;	do 256 macros each processing 32 data values

;; Do inverse FFT levels 9,10
;; On input the 64-byte cache lines hold these data values:
;;	0	+2K	+2K	+2K	4	+2K	+2K	+2K
;;	8	...
;;	...
;;	1	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+2K	+2K	+2K	16	+2K	+2K	+2K
;;	4	...
;;	...
;;	12	...
;;	32	...
;;	...
;;	1	...
;;	...
;;
;; Do 1024 four_complex_unfft operations
;;	distance between fft data elements is 4
;;	do 256 macros each processing 32 data values

;; Do inverse FFT levels 7,8
;; On input the 64-byte cache lines hold these data values:
;;	0	+2K	+2K	+2K	16	+2K	+2K	+2K
;;	4	...
;;	...
;;	12	...
;;	32	...
;;	...
;;	1	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+2K	+2K	+2K	64	+2K	+2K	+2K
;;	4	...
;;	...
;;	60	...
;;	128	...
;;	...
;;	1	...
;;	...
;;
;; Do 1024 four_complex_unfft operations
;;	distance between fft data elements is 16
;;	do 256 macros each processing 32 data values

;; Do inverse FFT levels 5,6
;; On input the 64-byte cache lines hold these data values:
;;	0	+2K	+2K	+2K	64	+2K	+2K	+2K
;;	4	...
;;	...
;;	60	...
;;	128	...
;;	...
;;	1	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+2K	+2K	+2K	256	+2K	+2K	+2K
;;	4	...
;;	...
;;	252	...
;;	512	...
;;	...
;;	1	...
;;	...
;;
;; Do 1024 four_complex_unfft operations
;;	distance between fft data elements is 64
;;	do 256 macros each processing 32 data values

;; Do this 256 cache lines (16KB) at a time to maximize accessing data
;; while it is in the L1 cache.

;; Execute the right middle step

	cmp	ffttype, 2
	jl	ypass2_12_levels_complex_1
	jg	ypass2_12_levels_complex_3

ypass2_12_levels_complex_2:
	yloop_init 256			;; 4 iters of 256 cache lines

b3b0:	start_timer 6
	yloop_set_prefetch YHALF	;; Prefetch 128 cache lines in this loop
	yr4_b4cl_four_complex_djbfft_preload
b3b:	yr4_b4cl_four_complex_djbfft rsi, 64, 16*64, 32*64, rdi, YMM_SCD2/4, 16
	yloop	16, b3b, -16*64+4*dist64 ;; Loop 16 times, next source pointer
	yloop	4, b3b, -4*4*dist64, rdi, -64*YMM_SCD2/4 ;; Loop 4 times, restore source pointer, restore sine/cosine pointer
	loops_undo 4*16			;; Back up loop counter
	end_timer 6

	start_timer 7
	yloop_set_prefetch YQUARTER	;; Prefetch 64 cache lines in this loop
;;	yr4_b4cl_four_complex_djbfft_preload
b4b:	yr4_b4cl_four_complex_djbfft rsi, 64, 4*64, 8*64, rdi, 4*YMM_SCD2/4, 4
	yloop	4, b4b, -4*64+16*64, rdi, -4*4*YMM_SCD2/4 ;; Loop 4 times, next source pointer, restore sine/cosine pointer
	yloop	4, b4b, -4*16*64+4*dist64, rdi, YMM_SCD2/4 ;; Loop 4 times, next source pointer, next sine/cosine pointer
	yloop	4, b4b, -4*4*dist64, rdi, -4*YMM_SCD2/4 ;; Loop 4 times, restore source pointer, restore sine/cosine pointer
	loops_undo 4*4*4		;; Back up loop counter
	end_timer 7

	start_timer 8
	yloop_set_prefetch YQUARTER	;; Prefetch 64 cache lines in this loop
;;	yr4_b4cl_four_complex_djbfft_preload
b5b:	yr4_b4cl_four_complex_djbfft rsi, 4*64, 64, 2*64, rdi, 0, 16
	yloop	16, b5b, -16*4*64+4*dist64, rdi, 4*YMM_SCD2/4 ;; Loop 16 times, next source pointer, next sine/cosine pointer
	yloop	4, b5b, -4*4*dist64, rdi, -4*4*YMM_SCD2/4 ;; Loop 4 times, restore source pointer, restore sine/cosine pointer
	loops_undo 4*16			;; Back up loop counter
	end_timer 8

	start_timer 9
	yloop_set_prefetch YOTHER, 16	;; Prefetch 16 cache lines in this loop
	yr4_4cl_four_complex_with_square_preload
b6b:	yr4_4cl_four_complex_with_square rsi, 64, 4*dist64, 8*dist64, 64
	yloop	64, b6b, -64*64		;; Loop 64 times, restore source pointer
	loops_undo 64			;; Back up loop counter
	end_timer 9

	start_timer 10
	yloop_set_prefetch YQUARTER	;; Prefetch 64 cache lines in this loop
	yr4_b4cl_four_complex_djbunfft_preload
c5b:	yr4_b4cl_four_complex_djbunfft rsi, 4*64, 64, 2*64, rdi, 0, 16
	yloop	16, c5b, -16*4*64+4*dist64, rdi, 4*YMM_SCD2/4 ;; Loop 16 times, next source pointer, next sine/cosine pointer
	yloop	4, c5b, -4*4*dist64, rdi, -4*4*YMM_SCD2/4 ;; Loop 4 times, restore source pointer, restore sine/cosine pointer
	loops_undo 4*16			;; Restore loop counter
	end_timer 10

	start_timer 11
	yloop_set_prefetch YQUARTER	;; Prefetch 64 cache lines in this loop
;;	yr4_b4cl_four_complex_djbunfft_preload
c4b:	yr4_b4cl_four_complex_djbunfft rsi, 64, 4*64, 8*64, rdi, 4*YMM_SCD2/4, 4
	yloop	4, c4b, -4*64+16*64, rdi, -4*4*YMM_SCD2/4 ;; Loop 4 times, next source pointer, restore sine/cosine pointer
	yloop	4, c4b, -4*16*64+4*dist64, rdi, YMM_SCD2/4 ;; Loop 4 times, next source pointer, next sine/cosine pointer
	yloop	4, c4b, -4*4*dist64, rdi, -4*YMM_SCD2/4 ;; Loop 4 times, restore source pointer, restore sine/cosine pointer
	loops_undo 4*4*4		;; Back up loop counter
	end_timer 11

	start_timer 12
	yloop_set_prefetch YHALF	;; Prefetch 128 cache lines in this loop
;;	yr4_b4cl_four_complex_djbunfft_preload
c3b:	yr4_b4cl_four_complex_djbunfft rsi, 64, 16*64, 32*64, rdi, YMM_SCD2/4, 16
	yloop	16, c3b, -16*64+4*dist64 ;; Loop 16 times, next source pointer
	yloop	4, c3b, -4*4*dist64+dist64, rdi, -64*YMM_SCD2/4 ;; Loop 4 times, next source pointer, restore sine/cosine pointer
	end_timer 12

	yloop	4, b3b0, -4*dist64	;; Loop 4 times, restore source pointer
	jmp	ypass2_12_levels_complex_unfft


ypass2_12_levels_complex_1:
	start_timer 6
	bump	rcx, (1024+15-2*128-3*256-16)*64 ;; Move prefetch pointer along
	yloop_init 256			;; 4 iters of 256 cache lines

b3a0:	yloop_set_prefetch YSINGLE	;; Prefetch 256 cache lines in this loop
	yr4_b4cl_four_complex_djbfft_preload
b3a:	yr4_b4cl_four_complex_djbfft rsi, 64, 16*64, 32*64, rdi, YMM_SCD2/4, 16
	yloop	16, b3a, -16*64+4*dist64 ;; Loop 16 times, next source pointer
	yloop	4, b3a, -4*4*dist64, rdi, -64*YMM_SCD2/4 ;; Loop 4 times, restore source pointer, restore sine/cosine pointer
	loops_undo 4*16			;; Back up loop counter

	yloop_set_prefetch YSINGLE	;; Prefetch 256 cache lines in this loop
;;	yr4_b4cl_four_complex_djbfft_preload
b4a:	yr4_b4cl_four_complex_djbfft rsi, 64, 4*64, 8*64, rdi, 4*YMM_SCD2/4, 4
	yloop	4, b4a, -4*64+16*64, rdi, -4*4*YMM_SCD2/4 ;; Loop 4 times, next source pointer, restore sine/cosine pointer
	yloop	4, b4a, -4*16*64+4*dist64, rdi, YMM_SCD2/4 ;; Loop 4 times, next source pointer, next sine/cosine pointer
	yloop	4, b4a, -4*4*dist64, rdi, -4*YMM_SCD2/4 ;; Loop 4 times, restore source pointer, restore sine/cosine pointer
	loops_undo 4*4*4		;; Back up loop counter

	yloop_set_prefetch YSINGLE	;; Prefetch 256 cache lines in this loop
;;	yr4_b4cl_four_complex_djbfft_preload
b5a:	yr4_b4cl_four_complex_djbfft rsi, 4*64, 64, 2*64, rdi, 0, 16
	yloop	16, b5a, -16*4*64+4*dist64, rdi, 4*YMM_SCD2/4 ;; Loop 16 times, next source pointer, next sine/cosine pointer
	yloop	4, b5a, -4*4*dist64, rdi, -4*4*YMM_SCD2/4 ;; Loop 4 times, restore source pointer, restore sine/cosine pointer
	loops_undo 4*16			;; Back up loop counter

	yloop_set_prefetch YOTHER, 16	;; Prefetch 16 cache lines in this loop
	yr4_4cl_four_complex_fft_final_preload
b6a:	yr4_4cl_four_complex_fft_final rsi, 64, 4*dist64, 8*dist64
	yloop	64, b6a, -64*64+dist64	;; Loop 64 times, next source pointer

	yloop	4, b3a0, -4*dist64	;; Loop 4 times, restore source pointer
	end_timer 6
	jmp	ypass2_12_levels_complex_done


ypass2_12_levels_complex_3:
	start_timer 6
x64	mov	r8, DEST2ARG
	mov	rbp, DIST_TO_MULSRCARG
	yloop_init 256			;; 4 iters of 256 cache lines

b3c0:	yloop_set_prefetch YHALF	;; Prefetch 128 cache lines in this loop
	yr4_b4cl_four_complex_djbfft_preload
b3c:	yr4_b4cl_four_complex_djbfft rsi, 64, 16*64, 32*64, rdi, YMM_SCD2/4, 16
	yloop	16, b3c, -16*64+4*dist64	;; Loop 16 times, next source pointer
	yloop	4, b3c, -4*4*dist64, rdi, -64*YMM_SCD2/4 ;; Loop 4 times, restore source pointer, restore sine/cosine pointer
	loops_undo 4*16			;; Back up loop counter

	yloop_set_prefetch YQUARTER	;; Prefetch 64 cache lines in this loop
;;	yr4_b4cl_four_complex_djbfft_preload
b4c:	yr4_b4cl_four_complex_djbfft rsi, 64, 4*64, 8*64, rdi, 4*YMM_SCD2/4, 4
	yloop	4, b4c, -4*64+16*64, rdi, -4*4*YMM_SCD2/4 ;; Loop 4 times, next source pointer, restore sine/cosine pointer
	yloop	4, b4c, -4*16*64+4*dist64, rdi, YMM_SCD2/4 ;; Loop 4 times, next source pointer, next sine/cosine pointer
	yloop	4, b4c, -4*4*dist64, rdi, -4*YMM_SCD2/4 ;; Loop 4 times, restore source pointer, restore sine/cosine pointer
	loops_undo 4*4*4		;; Back up loop counter

	yloop_set_prefetch YQUARTER	;; Prefetch 64 cache lines in this loop
;;	yr4_b4cl_four_complex_djbfft_preload
b5c:	yr4_b4cl_four_complex_djbfft rsi, 4*64, 64, 2*64, rdi, 0, 16
	yloop	16, b5c, -16*4*64+4*dist64, rdi, 4*YMM_SCD2/4 ;; Loop 16 times, next source pointer, next sine/cosine pointer
	yloop	4, b5c, -4*4*dist64, rdi, -4*4*YMM_SCD2/4 ;; Loop 4 times, restore source pointer, restore sine/cosine pointer
	loops_undo 4*16			;; Back up loop counter

	yloop_set_prefetch YOTHER, 16	;; Prefetch 16 cache lines in this loop
	yr4_4cl_four_complex_with_mult_preload
b6c:	yr4_4cl_four_complex_with_mult rsi, 64, 4*dist64, 8*dist64
	yloop	64, b6c, -64*64		;; Loop 64 times, restore source pointer
x64	add	rsi, r8
	loops_undo 64			;; Back up loop counter

	yloop_set_prefetch YQUARTER	;; Prefetch 64 cache lines in this loop
	yr4_b4cl_four_complex_djbunfft_preload
c5c:	yr4_b4cl_four_complex_djbunfft rsi, 4*64, 64, 2*64, rdi, 0, 16
	yloop	16, c5c, -16*4*64+4*dist64, rdi, 4*YMM_SCD2/4 ;; Loop 16 times, next source pointer, next sine/cosine pointer
	yloop	4, c5c, -4*4*dist64, rdi, -4*4*YMM_SCD2/4 ;; Loop 4 times, restore source pointer, restore sine/cosine pointer
	loops_undo 4*16			;; Restore loop counter

	yloop_set_prefetch YQUARTER	;; Prefetch 64 cache lines in this loop
;;	yr4_b4cl_four_complex_djbunfft_preload
c4c:	yr4_b4cl_four_complex_djbunfft rsi, 64, 4*64, 8*64, rdi, 4*YMM_SCD2/4, 4
	yloop	4, c4c, -4*64+16*64, rdi, -4*4*YMM_SCD2/4 ;; Loop 4 times, next source pointer, restore sine/cosine pointer
	yloop	4, c4c, -4*16*64+4*dist64, rdi, YMM_SCD2/4 ;; Loop 4 times, next source pointer, next sine/cosine pointer
	yloop	4, c4c, -4*4*dist64, rdi, -4*YMM_SCD2/4 ;; Loop 4 times, restore source pointer, restore sine/cosine pointer
	loops_undo 4*4*4		;; Back up loop counter

	yloop_set_prefetch YHALF	;; Prefetch 128 cache lines in this loop
;;	yr4_b4cl_four_complex_djbunfft_preload
c3c:	yr4_b4cl_four_complex_djbunfft rsi, 64, 16*64, 32*64, rdi, YMM_SCD2/4, 16
	yloop	16, c3c, -16*64+4*dist64	;; Loop 16 times, next source pointer
	yloop	4, c3c, -4*4*dist64+dist64, rdi, -64*YMM_SCD2/4 ;; Loop 4 times, next source pointer, restore sine/cosine pointer

x64	sub	rsi, r8
	yloop	4, b3c0, -4*dist64	;; Loop 4 times, restore source pointer
x64	add	rsi, r8
	end_timer 6
	jmp	ypass2_12_levels_complex_unfft


ypass2_12_levels_complex_4:
	start_timer 6
	mov	rdi, xsincos_complex	;; The sin/cos ptrs
	bump	rdi, 256*YMM_SCD2
	mov	rbp, DIST_TO_MULSRCARG
	yloop_init 256			;; 4 iters of 256 cache lines

	yloop_set_prefetch YOTHER, 16	;; Prefetch 16 cache lines in this loop
	yr4_4cl_four_complex_with_mulf_preload
b6d:	yr4_4cl_four_complex_with_mulf rsi, 64, 4*dist64, 8*dist64
	yloop	64, b6d, -64*64		;; Loop 64 times, restore source pointer
	loops_undo 64			;; Back up loop counter

	yloop_set_prefetch YSINGLE	;; Prefetch 256 cache lines in this loop
	yr4_b4cl_four_complex_djbunfft_preload
c5d:	yr4_b4cl_four_complex_djbunfft rsi, 4*64, 64, 2*64, rdi, 0, 16
	yloop	16, c5d, -16*4*64+4*dist64, rdi, 4*YMM_SCD2/4 ;; Loop 16 times, next source pointer, next sine/cosine pointer
	yloop	4, c5d, -4*4*dist64, rdi, -4*4*YMM_SCD2/4 ;; Loop 4 times, restore source pointer, restore sine/cosine pointer
	loops_undo 4*16			;; Restore loop counter

	yloop_set_prefetch YSINGLE	;; Prefetch 256 cache lines in this loop
;;	yr4_b4cl_four_complex_djbunfft_preload
c4d:	yr4_b4cl_four_complex_djbunfft rsi, 64, 4*64, 8*64, rdi, 4*YMM_SCD2/4, 4
	yloop	4, c4d, -4*64+16*64, rdi, -4*4*YMM_SCD2/4 ;; Loop 4 times, next source pointer, restore sine/cosine pointer
	yloop	4, c4d, -4*16*64+4*dist64, rdi, YMM_SCD2/4 ;; Loop 4 times, next source pointer, next sine/cosine pointer
	yloop	4, c4d, -4*4*dist64, rdi, -4*YMM_SCD2/4 ;; Loop 4 times, restore source pointer, restore sine/cosine pointer
	loops_undo 4*4*4		;; Back up loop counter

	yloop_set_prefetch YSINGLE	;; Prefetch 256 cache lines in this loop
;;	yr4_b4cl_four_complex_djbunfft_preload
c3d:	yr4_b4cl_four_complex_djbunfft rsi, 64, 16*64, 32*64, rdi, YMM_SCD2/4, 16
	yloop	16, c3d, -16*64+4*dist64	;; Loop 16 times, next source pointer
	yloop	4, c3d, -4*4*dist64+dist64, rdi, -64*YMM_SCD2/4 ;; Loop 4 times, next source pointer, restore sine/cosine pointer

	yloop	4, b6d, -4*dist64	;; Loop 4 times, restore source pointer
	bump	rcx, (-16-3*256+3*128+2*64+16+2*64+128)*64 ;; Move prefetch pointer to where squaring would be
	end_timer 6

;; Do inverse FFT levels 3,4
;; On input the 64-byte cache lines hold these data values:
;;	0	+2K	+2K	+2K	256	+2K	+2K	+2K
;;	4	...
;;	...
;;	252	...
;;	512	...
;;	...
;;	1	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+2K	+2K	+2K	1024	+2K	+2K	+2K
;;	4	...
;;	...
;;	1020	...
;;	1	...
;;	...

;; Do 1024 four_complex_unfft operations
;;	distance between fft data elements is 256
;;	do 256 macros each processing 32 data values

;; Do inverse FFT levels 1,2
;; On input the 64-byte cache lines hold these data values:
;;	0	+2K	+2K	+2K	1024	+2K	+2K	+2K
;;	4	...
;;	...
;;	1020	...
;;	1	...
;;	...
;; On output the 64-byte cache lines are swizzled to hold these data values:
;;	0	+1	+1	+1	4096	+1	+1	+1
;;	4	...
;;	...
;;	4092

;; Do 1024 four_complex_unfft operations
;;	distance between fft data elements is 1024
;;	do 256 macros each processing 32 data values

ypass2_12_levels_complex_unfft:
	mov	rdi, xsincos_complex	;; Load sin/cos pointers
	mov	rbp, rdi
	bump	rcx, (1024+15-3*128-2*64-16-2*64-128-2*128)*64 ;; Move prefetch pointer along
	yloop_init 256			;; 256 iterations
	yloop_set_prefetch YHALF

c2a:	start_timer 13
	yr4_eb4cl_four_complex_djbunfft_preload
c2b:	yr4_eb4cl_four_complex_djbunfft rsi, 4*dist64, dist64, 2*dist64, rbp, YMM_SCD2, 4
	yloop	4, c2b, -4*4*dist64	;; Test loop counter, restore source pointer
	loops_undo 4			;; Restore loop counter
	end_timer 13

	start_timer 14
	yr4_s4cl_four_complex_djbunfft_preload
c1b:	yr4_s4cl_four_complex_djbunfft rsi, dist64, 4*dist64, 8*dist64, rdi, 64*YMM_SCD2, 4, L1PREFETCH_ALL, 64
	yloop	4, c1b, -4*dist64+64, rdi, -4*64*YMM_SCD2+YMM_SCD2 ;; Test loop counter, next source pointer, next sine/cosine pointer
	end_timer 14

	yloop	64, c2a			;; Loop 64 times

ypass2_12_levels_complex_done:
	ENDM


; *************** radix4-pass2-5120-real macro ******************
; This macro takes 10240 real values and performs the final 13.322 levels of the FFT.
; The results are squared, and the first 13.322 levels of the inverse FFT are done.

yr4dwpn_pass2_5120_real MACRO
	LOCAL	b1b, b2a, b2b, b3a, b3b, b4a, b4b, b5a, b5b, b6a, b6b, b6c, b6d
	LOCAL	c1b, c2a, c2b, c3a, c3b, c4a, c4b, c5a, c5b
	LOCAL	ypass2_5120_real_1, ypass2_5120_real_2
	LOCAL	ypass2_5120_real_3, ypass2_5120_real_4
	LOCAL	ypass2_5120_real_unfft, ypass2_5120_real_done

;; Type 4 FFTs skip the forward FFT process

	start_timer 2
	mov	rsi, DATA_ADDR		;; Load source address
	mov	rbx, DIST_TO_FFTSRCARG
	mov	rcx, DATA_PREFETCH	;; Prefetch pointer (we need to prefetch 
					;; 1280 cache lines + 19 pad cache lines)
	cmp	ffttype, 4
	je	ypass2_5120_real_4

;; Do FFT levels 1,2,3 which correspond to complex levels 1,2
;; On input the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	5120	+1	+1	+1
;;	4	...
;;	...
;;	5016
;; On output the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	1024	+1	+1	+1
;;	4	...
;;	...
;;	1020	...
;;	2048	...
;;	...

;; Do 1024 ten_reals_fft operations
;;	distance between fft data elements is 1024
;;	do 256 macros each processing 40 data values

	mov	rdi, sincos3		;; Real sin/cos pointer
	mov	rbp, xsincos_complex	;; Complex sin/cos data
	yloop_init 256			;; 256 iterations
	yloop_set_prefetch YHALF
	yr5_f5cl_2sc_ten_reals_fft_preload
b1b:	yr5_f5cl_2sc_ten_reals_fft rsi, 64, 4*dist64, rbp, YMM_SCD2, rdi, YMM_SCD2, 64
	yloop	64, b1b, -64*64+dist64	;; Test loop counter, next source pointer
	yloop	4, b1b, -4*dist64	;; Test loop counter, restore source pointer

;; Do FFT levels 3,4
;; On input the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	1024	+1	+1	+1
;;	4	...
;;	...
;;	1020	...
;;	2048	...
;;	...
;; On output the 64-byte cache lines are swizzled to hold these data values:
;;	0	+512	+512	+512	256	+512	+512	+512
;;	4	...
;;	...
;;	252	...
;;	1	...
;;	...
;;	2048	...
;;	...

;; Do 256 eight_reals_fft operations
;; Do 1024 four_complex_fft operations
;;	distance between fft data elements is 256
;;	do 320 macros each processing 32 data values

	yloop_init 320			;; 5 iters of 64
	yloop_set_prefetch YHALF
	yr4_s4cl_2sc_eight_reals_fft_preload
b2a:	yr4_s4cl_2sc_eight_reals_fft rsi, 64, dist64, 2*dist64, rbp, YMM_SCD2, rdi, YMM_SCD2, 64
	yloop	64, b2a, -64*64+4*dist64, rbp, -64*YMM_SCD2 ;; Test loop counter, next source pointer, restore sine/cosine pointer

	loops_reset			;; 4 more iters of 64
	yr4_s4cl_four_complex_djbfft_preload
b2b:	yr4_s4cl_four_complex_djbfft rsi, 64, dist64, 2*dist64, rbp, YMM_SCD2, 64
	yloop	64, b2b, -64*64+4*dist64, rbp, -64*YMM_SCD2 ;; Test loop counter, next source pointer, restore sin/cos ptr
	yloop	5, b2b, -5*4*dist64	;; Loop 5 times, restore source pointer

;; Do FFT levels 5,6
;; On input the 64-byte cache lines hold these data values:
;;	0	+512	+512	+512	256	+512	+512	+512
;;	4	...
;;	...
;;	252	...
;;	1	...
;;	...
;;	2048	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+512	+512	+512	64	+512	+512	+512
;;	4	...
;;	...
;;	60	...
;;	128	...
;;	...
;;	1	...
;;	...
;;	2048	...
;;	...

;; Do 64 eight_reals_fft operations
;; Do 1216 four_complex_fft operations
;;	distance between fft data elements is 64
;;	do 320 macros each processing 32 data values

	yloop_init 320			;; 5 iterations of 64
	yloop_set_prefetch YHALF
	yr4_4cl_eight_reals_four_complex_djbfft_preload
b3a:	yr4_4cl_eight_reals_four_complex_djbfft rsi, dist64, 16*64, 32*64, rdi, YMM_SCD3, 4
	yloop	4, b3a, -4*dist64+64	;; Test loop counter, next source pointer
	yloop	16, b3a, -16*64+4*dist64, rdi, -64*YMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer

	loops_reset			;; 4 more iters of 64
	yr4_rb4cl_four_complex_djbfft_preload
b3b:	yr4_rb4cl_four_complex_djbfft rsi, dist64, 16*64, 32*64, rdi, YMM_SCD3, 4
	yloop	4, b3b, -4*dist64+64	;; Test loop counter, next source pointer
	yloop	16, b3b, -16*64+4*dist64, rdi, -64*YMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer
	yloop	5, b3b, -5*4*dist64	;; Test loop counter, restore source pointer

;; Do FFT levels 7,8
;; On input the 64-byte cache lines hold these data values:
;;	0	+512	+512	+512	64	+512	+512	+512
;;	4	...
;;	...
;;	60	...
;;	128	...
;;	...
;;	1	...
;;	...
;;	2048	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+512	+512	+512	16	+512	+512	+512
;;	4	...
;;	...
;;	12	...
;;	32	...
;;	...
;;	1	...
;;	...
;;	2048	...
;;	...

;; Do 16 eight_reals_fft operations
;; Do 1264 four_complex_fft operations
;;	distance between fft data elements is 16
;;	do 320 macros each processing 32 data values

	yloop_init 320			;; 20 iterations of 16
	yloop_set_prefetch YHALF
	yr4_4cl_eight_reals_four_complex_djbfft_preload
b4a:	yr4_4cl_eight_reals_four_complex_djbfft rsi, dist64, 4*64, 8*64, rdi, 4*YMM_SCD3, 4
	yloop	4, b4a, -4*dist64+64	;; Test loop counter, next source pointer
	yloop	4, b4a, -4*64+16*64, rdi, -16*4*YMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer

	loops_reset			;; 19 more iters of 16
	yr4_rb4cl_four_complex_djbfft_preload
b4b:	yr4_rb4cl_four_complex_djbfft rsi, dist64, 4*64, 8*64, rdi, 4*YMM_SCD3, 4
	yloop	4, b4b, -4*dist64+64	;; Test loop counter, next source pointer
	yloop	4, b4b, -4*64+16*64, rdi, -16*4*YMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer
	yloop	4, b4b, -4*16*64+4*dist64 ;; Test loop counter, next source pointer
	yloop	5, b4b, -5*4*dist64	;; Test loop counter, restore source pointer

;; Do FFT levels 9,10
;; On input the 64-byte cache lines hold these data values:
;;	0	+512	+512	+512	16	+512	+512	+512
;;	4	...
;;	...
;;	12	...
;;	32	...
;;	...
;;	1	...
;;	...
;;	2048	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+512	+512	+512	4	+512	+512	+512
;;	8	...
;;	...
;;	1	...
;;	...
;;	2048	...
;;	...

;; Do 4 eight_reals_fft operations
;; Do 1276 four_complex_fft operations
;;	distance between fft data elements is 4
;;	do 320 macros each processing 32 data values

	yloop_init 320			;; 80 iterations of 4
	yloop_set_prefetch YHALF
	yr4_4cl_eight_reals_four_complex_djbfft_preload
b5a:	yr4_4cl_eight_reals_four_complex_djbfft rsi, dist64, 64, 2*64, rdi, 16*YMM_SCD3, 4
	yloop	4, b5a, -4*dist64+4*64, rdi, -4*16*YMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer

	loops_reset			;; 79 more iters of 4
	yr4_rb4cl_four_complex_djbfft_preload
b5b:	yr4_rb4cl_four_complex_djbfft rsi, dist64, 64, 2*64, rdi, 16*YMM_SCD3, 4
	yloop	4, b5b, -4*dist64+4*64, rdi, -4*16*YMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer
	yloop	16, b5b, -16*4*64+4*dist64 ;; Test loop counter, next source pointer
	yloop	5, b5b, -5*4*dist64	;; Test loop counter, restore source pointer

;; Do FFT levels 11,12 as well as inverse FFT levels 11,12
;; On input and output the 64-byte cache lines hold these data values:
;;	0	+512	+512	+512	4	+512	+512	+512
;;	8	...
;;	...
;;	1	...
;;	...
;;	2048	...
;;	...

;; Do 1 eight_reals_fft operation
;; Do 1279 four_complex_fft operations
;;	distance between fft data elements is 1
;;	do 320 macros each processing 32 data values

	cmp	ffttype, 2
	jl	ypass2_5120_real_1
	jg	ypass2_5120_real_3

ypass2_5120_real_2:
;;	yr4_4cl_eight_reals_four_complex_with_square rsi, 64, dist64, 2*dist64
	yr4_dispatch call, yreal_square, dist64, 2*dist64
	yloop_init 320			;; 319 more iterations
	yloop_adjust 1
	yloop_set_prefetch YQUARTER
	yr4_4cl_four_complex_with_square_preload
b6b:	yr4_4cl_four_complex_with_square rsi, 64, dist64, 2*dist64, 1
	yloop	64, b6b, -64*64+4*dist64	;; Test loop counter, next source pointer
	yloop	5, b6b, -5*4*dist64	;; Test loop counter, restore source pointer
	jmp	ypass2_5120_real_unfft

ypass2_5120_real_1:
;;	yr4_4cl_eight_reals_four_complex_fft_final rsi, 64, dist64, 2*dist64
	yr4_dispatch call, yreal_fft_final, dist64, 2*dist64
	mov	rcx, xsincos_complex	;; Prefetch complex sin/cos data, 672 cache lines
	yloop_init 320			;; 319 more iterations
	yloop_adjust 1
	yloop_set_prefetch YDOUBLE
b6a:	yr4_4cl_four_complex_fft_final rsi, 64, dist64, 2*dist64
	yloop	64, b6a, -64*64+4*dist64	;; Test loop counter, next source pointer
	yloop	5, b6a			;; Test loop counter
	jmp	ypass2_5120_real_done

ypass2_5120_real_3:
x64	mov	r8, DEST2ARG
	mov	rbp, DIST_TO_MULSRCARG
;;	yr4_4cl_eight_reals_four_complex_with_mult rsi, 64, dist64, 2*dist64
	yr4_dispatch call, yreal_mult, dist64, 2*dist64
	yloop_init 320			;; 319 more iterations
	yloop_adjust 1
	yloop_set_prefetch YQUARTER
	yr4_4cl_four_complex_with_mult_preload
b6c:	yr4_4cl_four_complex_with_mult rsi, 64, dist64, 2*dist64
	yloop	64, b6c, -64*64+4*dist64	;; Test loop counter, next source pointer
	yloop	5, b6c, -5*4*dist64	;; Test loop counter, restore source pointer
x64	add	rsi, r8
	jmp	ypass2_5120_real_unfft

ypass2_5120_real_4:
	mov	rbp, DIST_TO_MULSRCARG
;;	yr4_4cl_eight_reals_four_complex_with_mulf rsi, 64, dist64, 2*dist64
	yr4_dispatch call, yreal_mulf, dist64, 2*dist64
	yloop_init 320			;; 319 more iterations
	yloop_adjust 1
	yloop_set_prefetch YDOUBLE
	yr4_4cl_four_complex_with_mulf_preload
b6d:	yr4_4cl_four_complex_with_mulf rsi, 64, dist64, 2*dist64
	yloop	64, b6d, -64*64+4*dist64	;; Test loop counter, next source pointer
	yloop	5, b6d, -5*4*dist64	;; Test loop counter, restore source pointer
;;	bump	rcx, (-638+128+4*160+80)*64 ;; Move prefetch pointer back to where squaring would be

;; Do inverse FFT levels 9,10
;; On input the 64-byte cache lines hold these data values:
;;	0	+512	+512	+512	4	+512	+512	+512
;;	8	...
;;	...
;;	1	...
;;	...
;;	2048	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+512	+512	+512	16	+512	+512	+512
;;	4	...
;;	...
;;	12	...
;;	32	...
;;	...
;;	1	...
;;	...
;;	2048	...
;;	...

;; Do 4 eight_reals_unfft operations
;; Do 1276 four_complex_unfft operations
;;	distance between fft data elements is 4
;;	do 320 macros each processing 32 data values

ypass2_5120_real_unfft:
	mov	rdi, sincos3		;; Load sin/cos pointer
	bump	rdi, 64*YMM_SCD2+256*YMM_SCD2
	yloop_init 320			;; 80 iterations of 4
	yloop_set_prefetch YHALF
	yr4_4cl_eight_reals_four_complex_djbunfft_preload
c5a:	yr4_4cl_eight_reals_four_complex_djbunfft rsi, dist64, 64, 2*64, rdi, 16*YMM_SCD3, 4
	yloop	4, c5a, -4*dist64+4*64, rdi, -4*16*YMM_SCD3 ;; Test loop counter, next source pointer, restore sine/cosine pointer

	loops_reset			;; 79 more iters of 4
	yr4_rb4cl_four_complex_djbunfft_preload
c5b:	yr4_rb4cl_four_complex_djbunfft rsi, dist64, 64, 2*64, rdi, 16*YMM_SCD3, 4
	yloop	4, c5b, -4*dist64+4*64, rdi, -4*16*YMM_SCD3 ;; Test loop counter, next source pointer, restore sine/cosine pointer
	yloop	16, c5b, -16*4*64+4*dist64	;; Test loop counter, next source pointer
	yloop	5, c5b, -5*4*dist64	;; Test loop counter, restore source pointer

;; Do inverse FFT levels 7,8
;; On input the 64-byte cache lines hold these data values:
;;	0	+512	+512	+512	16	+512	+512	+512
;;	4	...
;;	...
;;	12	...
;;	32	...
;;	...
;;	1	...
;;	...
;;	2048	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+512	+512	+512	64	+512	+512	+512
;;	4	...
;;	...
;;	60	...
;;	128	...
;;	...
;;	1	...
;;	...
;;	2048	...
;;	...

;; Do 16 eight_reals_unfft operations
;; Do 1264 four_complex_unfft operations
;;	distance between fft data elements is 16
;;	do 320 macros each processing 32 data values

	bump	rcx, (1280+19-128-4*160-80-160-320)*64 ;; Move prefetch pointer along
	yloop_init 320			;; 20 iterations of 16
	yloop_set_prefetch YSINGLE
	yr4_4cl_eight_reals_four_complex_djbunfft_preload
c4a:	yr4_4cl_eight_reals_four_complex_djbunfft rsi, dist64, 4*64, 8*64, rdi, 4*YMM_SCD3, 4
	yloop	4, c4a, -4*dist64+64	;; Test loop counter, next source pointer
	yloop	4, c4a, -4*64+16*64, rdi, -16*4*YMM_SCD3 ;; Test loop counter, next source pointer, restore sine/cosine pointer

	loops_reset			;; 19 more iters of 16
	yr4_rb4cl_four_complex_djbunfft_preload
c4b:	yr4_rb4cl_four_complex_djbunfft rsi, dist64, 4*64, 8*64, rdi, 4*YMM_SCD3, 4
	yloop	4, c4b, -4*dist64+64	;; Test loop counter, next source pointer
	yloop	4, c4b, -4*64+16*64, rdi, -16*4*YMM_SCD3 ;; Test loop counter, next source pointer, restore sine/cosine pointer
	yloop	4, c4b, -4*16*64+4*dist64	;; Test loop counter, next source pointer
	yloop	5, c4b, -5*4*dist64	;; Test loop counter, restore source pointer

;; Do inverse FFT levels 5,6
;; On input the 64-byte cache lines hold these data values:
;;	0	+512	+512	+512	64	+512	+512	+512
;;	4	...
;;	...
;;	60	...
;;	128	...
;;	...
;;	1	...
;;	...
;;	2048	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+512	+512	+512	256	+512	+512	+512
;;	4	...
;;	...
;;	252	...
;;	1	...
;;	...
;;	2048	...
;;	...

;; Do 64 eight_reals_unfft operations
;; Do 1216 four_complex_unfft operations
;;	distance between fft data elements is 64
;;	do 320 macros each processing 32 data values

	mov	rcx, xsincos_complex	;; Prefetch complex sin/cos data, 672 cache lines
	yloop_init 320			;; 5 iterations of 64
	yloop_set_prefetch YSINGLE
	yr4_4cl_eight_reals_four_complex_djbunfft_preload
c3a:	yr4_4cl_eight_reals_four_complex_djbunfft rsi, dist64, 16*64, 32*64, rdi, YMM_SCD3, 4
	yloop	4, c3a, -4*dist64+64	;; Test loop counter, next source pointer
	yloop	16, c3a, -16*64+4*dist64, rdi, -64*YMM_SCD3 ;; Test loop counter, next source pointer, restore sine/cosine pointer

	loops_reset			;; 4 more iters of 64
	yr4_rb4cl_four_complex_djbunfft_preload
c3b:	yr4_rb4cl_four_complex_djbunfft rsi, dist64, 16*64, 32*64, rdi, YMM_SCD3, 4
	yloop	4, c3b, -4*dist64+64	;; Test loop counter, next source pointer
	yloop	16, c3b, -16*64+4*dist64, rdi, -64*YMM_SCD3 ;; Test loop counter, next source pointer, restore sine/cosine pointer
	yloop	5, c3b, -5*4*dist64	;; Test loop counter, restore source pointer

;; Do inverse FFT levels 3,4
;; On input the 64-byte cache lines hold these data values:
;;	0	+512	+512	+512	256	+512	+512	+512
;;	4	...
;;	...
;;	252	...
;;	1	...
;;	...
;;	2048	...
;;	...
;; On output the 64-byte cache lines are swizzled to hold these data values:
;;	0	+1	+1	+1	1024	+1	+1	+1
;;	4	...
;;	...
;;	1020	...
;;	2048	...
;;	...

;; Do 256 eight_reals_unfft operations
;; Do 1024 four_complex_unfft operations
;;	distance between fft data elements is 256
;;	do 320 macros each processing 32 data values

	bump	rdi, -64*YMM_SCD2	;; Real sin/cos ptr
	mov	rbp, xsincos_complex	;; Complex sin/cos data
	bump	rbp, 256*YMM_SCD2
	yloop_init 320			;; 5 iters of 64
	yloop_set_prefetch YSINGLE
	yr4_s4cl_2sc_eight_reals_unfft_preload
c2a:	yr4_s4cl_2sc_eight_reals_unfft rsi, 64, dist64, 2*dist64, rbp, YMM_SCD2, rdi, YMM_SCD2, 64
	yloop	64, c2a, -64*64+4*dist64, rbp, -64*YMM_SCD2 ;; Test loop counter, next source pointer, restore sine/cosine pointer

	loops_reset			;; 4 more iters of 64
	yr4_s4cl_four_complex_djbunfft_preload
c2b:	yr4_s4cl_four_complex_djbunfft rsi, 64, dist64, 2*dist64, rbp, YMM_SCD2, 64
	yloop	64, c2b, -64*64+4*dist64, rbp, -64*YMM_SCD2 ;; Test loop counter, next source pointer, restore sine/cosine pointer
	yloop	5, c2b, -5*4*dist64	;; Test loop counter, restore source pointer

;; Do inverse FFT levels 1,2,3 which correspond to complex levels 1,2
;; On input the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	1024	+1	+1	+1
;;	4	...
;;	...
;;	1020	...
;;	2048	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	5120	+1	+1	+1
;;	4	...
;;	...
;;	5116

;; Do 1024 ten_reals_unfft operations
;;	distance between fft data elements is 1024
;;	do 256 macros each processing 40 data values

	bump	rdi, -64*YMM_SCD2-256*YMM_SCD2 ;; Real sin/cos ptr
	bump	rbp, -256*YMM_SCD2	;; Complex sin/cos data
	bump	rcx, (672-2*320-32)*64	;; Move prefetch pointer along
	yloop_init 256			;; 256 iterations
	yloop_set_prefetch YOTHER, 8
	yr5_5cl_2sc_ten_reals_unfft_preload
c1b:	yr5_5cl_2sc_ten_reals_unfft rsi, 64, 4*dist64, rbp, YMM_SCD2, rdi, YMM_SCD2, 64
	yloop	64, c1b, -64*64+dist64	;; Test loop counter, next source pointer
	yloop	4, c1b			;; Test loop counter

ypass2_5120_real_done:
	end_timer 2
	ENDM

; *************** radix4-pass2-5120-complex macro ******************
; This macro takes 5120 complex values and performs the final 12.322 levels of a standard
; radix-4 FFT, squares the results, and does the first 12.322 levels of the inverse FFT.

yr4dwpn_pass2_5120_complex MACRO
	LOCAL	b1a, b1b, b2b, b3a0, b3a, b3b0, b3b, b3c0, b3c, b4a, b4b, b4c, b5a, b5b, b5c, b6a, b6b, b6c, b6d
	LOCAL	c1b, c2a, c2b, c3b, c3c, c3d, c4b, c4c, c4d, c5b, c5c, c5d
	LOCAL	ypass2_5120_complex_1, ypass2_5120_complex_2
	LOCAL	ypass2_5120_complex_3, ypass2_5120_complex_4
	LOCAL	ypass2_5120_complex_unfft, ypass2_5120_complex_done

;; Type 4 FFTs skip the forward FFT process

	mov	rsi, DATA_ADDR		;; Load address of FFT data
	mov	rbx, DIST_TO_FFTSRCARG
	mov	rcx, DATA_PREFETCH	;; Prefetch pointer (we need to prefetch 
					;; 1280 cache lines + 19 pad cache lines)
	cmp	ffttype, 4
	je	ypass2_5120_complex_4

;; Do FFT level 1,2
;; On input the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	5120	+1	+1	+1
;;	4	...
;;	...
;;	5116
;; On output the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	1024	+1	+1	+1
;;	4	...
;;	...
;;	1020	...
;;	2048	...
;;	...

;; Do 1024 five_complex_fft operations
;;	distance between fft data elements is 1024
;;	do 256 macros each processing 40 data values

;; Do FFT levels 3,4
;; On input the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	1024	+1	+1	+1
;;	4	...
;;	...
;;	1020	...
;;	2048	...
;;	...
;; On output the 64-byte cache lines are swizzled to hold these data values:
;;	0	+512	+512	+512	256	+512	+512	+512
;;	4	...
;;	...
;;	252	...
;;	1	...
;;	...
;;	2048	...
;;	...

;; Do 1280 four_complex_fft operations
;;	distance between fft data elements is 256
;;	do 320 macros each processing 32 data values

	mov	rdi, xsincos_complex	;; The sin/cos ptrs
	lea	rbp, [rdi+256*YMM_SCD2]
	yloop_init 512			;; 256 iterations or 320 iterations

b1a:	start_timer 4
	yloop_adjust 4			;; Four fewer iterations this loop
	yloop_set_prefetch YOTHER, 8	;; Prefetch 64 cache lines in this loop
	yr5_f5cl_five_complex_djbfft_preload
b1b:	yr5_f5cl_five_complex_djbfft rsi, dist64, 4*dist64, rdi, 64*YMM_SCD2, 4
	yloop	8, b1b, -4*dist64, rdi, -4*64*YMM_SCD2+YMM_SCD2 ;; 4 iterations!!  Restore source pointer, next sin/cos data
	loops_undo 8
	end_timer 4

	start_timer 5
	yloop_adjust 3			;; Three fewer iterations this loop
	yloop_set_prefetch YOTHER, 16	;; Prefetch 32 cache lines in this loop
	yr4_s4cl_four_complex_djbfft_preload
b2b:	yr4_s4cl_four_complex_djbfft rsi, 4*dist64, dist64, 2*dist64, rbp, 0, 5, L1PREFETCH_ALL, 64
	yloop	8, b2b, -5*4*dist64+64, rbp, YMM_SCD2 ;; 5 iterations!!  Next source pointer, next sin/cos data
	end_timer 5

	yloop	64, b1a, -64*64, rdi, -64*YMM_SCD2+256*YMM_SCD2+64*YMM_SCD2 ;; Test loop counter, restore source pointer, next sin/cos ptr

;; Do FFT levels 5,6
;; On input the 64-byte cache lines hold these data values:
;;	0	+512	+512	+512	256	+512	+512	+512
;;	4	...
;;	...
;;	252	...
;;	1	...
;;	...
;;	2048	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+512	+512	+512	64	+512	+512	+512
;;	4	...
;;	...
;;	60	...
;;	128	...
;;	...
;;	1	...
;;	...
;;	2048	...
;;	...

;; Do 1280 four_complex_fft operations
;;	distance between fft data elements is 64
;;	do 320 macros each processing 32 data values

;; Do FFT levels 7,8
;; On input the 64-byte cache lines hold these data values:
;;	0	+512	+512	+512	64	+512	+512	+512
;;	4	...
;;	...
;;	60	...
;;	128	...
;;	...
;;	1	...
;;	...
;;	2048	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+512	+512	+512	16	+512	+512	+512
;;	4	...
;;	...
;;	12	...
;;	32	...
;;	...
;;	1	...
;;	...
;;	2048	...
;;	...

;; Do 1280 four_complex_fft operations
;;	distance between fft data elements is 16
;;	do 320 macros each processing 32 data values

;; Do FFT levels 9,10
;; On input the 64-byte cache lines hold these data values:
;;	0	+512	+512	+512	16	+512	+512	+512
;;	4	...
;;	...
;;	12	...
;;	32	...
;;	...
;;	1	...
;;	...
;;	2048	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+512	+512	+512	4	+512	+512	+512
;;	8	...
;;	...
;;	1	...
;;	...
;;	2048	...
;;	...

;; Do 1280 four_complex_fft operations
;;	distance between fft data elements is 4
;;	do 320 macros each processing 32 data values

;; Do FFT levels 11,12 as well as inverse FFT levels 11,12
;; On input and output the 64-byte cache lines hold these data values:
;;	0	+512	+512	+512	4	+512	+512	+512
;;	8	...
;;	...
;;	1	...
;;	...
;;	2048	...
;;	...

;; Do 1280 four_complex_fft operations
;;	distance between fft data elements is 1
;;	do 320 macros each processing 32 data values

;; Do inverse FFT levels 9,10
;; On input the 64-byte cache lines hold these data values:
;;	0	+512	+512	+512	4	+512	+512	+512
;;	8	...
;;	...
;;	1	...
;;	...
;;	2048	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+512	+512	+512	16	+512	+512	+512
;;	4	...
;;	...
;;	12	...
;;	32	...
;;	...
;;	1	...
;;	...
;;	2048	...
;;	...

;; Do 1280 four_complex_unfft operations
;;	distance between fft data elements is 4
;;	do 320 macros each processing 32 data values

;; Do inverse FFT levels 7,8
;; On input the 64-byte cache lines hold these data values:
;;	0	+512	+512	+512	16	+512	+512	+512
;;	4	...
;;	...
;;	12	...
;;	32	...
;;	...
;;	1	...
;;	...
;;	2048	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+512	+512	+512	64	+512	+512	+512
;;	4	...
;;	...
;;	60	...
;;	128	...
;;	...
;;	1	...
;;	...
;;	2048	...
;;	...

;; Do 1280 four_complex_unfft operations
;;	distance between fft data elements is 16
;;	do 320 macros each processing 32 data values

;; Do inverse FFT levels 5,6
;; On input the 64-byte cache lines hold these data values:
;;	0	+512	+512	+512	64	+512	+512	+512
;;	4	...
;;	...
;;	60	...
;;	128	...
;;	...
;;	1	...
;;	...
;;	2048	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+512	+512	+512	256	+512	+512	+512
;;	4	...
;;	...
;;	252	...
;;	1	...
;;	...
;;	2048	...
;;	...

;; Do 1280 four_complex_unfft operations
;;	distance between fft data elements is 64
;;	do 320 macros each processing 32 data values

;; Do this 256 cache lines (16KB) at a time to maximize accessing data
;; while it is in the L1 cache.

;; Execute the right middle step

	cmp	ffttype, 2
	jl	ypass2_5120_complex_1
	jg	ypass2_5120_complex_3

ypass2_5120_complex_2:
	yloop_init 320			;; 5 iters of 256 cache lines
	yloop_set_prefetch YHALF

b3b0:	start_timer 6
	yr4_b4cl_four_complex_djbfft_preload
b3b:	yr4_b4cl_four_complex_djbfft rsi, 64, 16*64, 32*64, rdi, YMM_SCD2/4, 16
	yloop	16, b3b, -16*64+dist64	;; Loop 16 times, next source pointer
	yloop	4, b3b, -4*dist64, rdi, -64*YMM_SCD2/4 ;; Loop 4 times, restore source pointer, restore sine/cosine pointer
	loops_undo 4*16			;; Back up loop counter
	end_timer 6

	start_timer 7
;;	yr4_b4cl_four_complex_djbfft_preload
b4b:	yr4_b4cl_four_complex_djbfft rsi, 64, 4*64, 8*64, rdi, 4*YMM_SCD2/4, 4
	yloop	4, b4b, -4*64+16*64, rdi, -4*4*YMM_SCD2/4 ;; Loop 4 times, next source pointer, restore sine/cosine pointer
	yloop	4, b4b, -4*16*64+dist64, rdi, YMM_SCD2/4 ;; Loop 4 times, next source pointer, next sine/cosine pointer
	yloop	4, b4b, -4*dist64, rdi, -4*YMM_SCD2/4 ;; Loop 4 times, restore source pointer, restore sine/cosine pointer
	loops_undo 4*4*4		;; Back up loop counter
	end_timer 7

	start_timer 8
;;	yr4_b4cl_four_complex_djbfft_preload
b5b:	yr4_b4cl_four_complex_djbfft rsi, 4*64, 64, 2*64, rdi, 0, 16
	yloop	16, b5b, -16*4*64+dist64, rdi, 4*YMM_SCD2/4 ;; Loop 16 times, next source pointer, next sine/cosine pointer
	yloop	4, b5b, -4*dist64, rdi, -4*4*YMM_SCD2/4 ;; Loop 4 times, restore source pointer, restore sine/cosine pointer
	loops_undo 4*16			;; Back up loop counter
	end_timer 8

	start_timer 9
	yr4_4cl_four_complex_with_square_preload
;; Loop below reworked to avoid 4KB distances
;;b6b:	yr4_4cl_four_complex_with_square rsi, 64, dist64, 2*dist64
;;	yloop	64, b6b, -64*64	;; Loop 64 times, restore source pointer
b6b:	yr4_4cl_four_complex_with_square rsi, 4*64, dist64, 2*dist64, 16
	yloop	16, b6b, -16*4*64+64	;; Loop 16 times, next source pointer
	yloop	4, b6b, -4*64	;; Loop 4 times, restore source pointer
	loops_undo 64			;; Back up loop counter
	end_timer 9

	start_timer 10
	yr4_b4cl_four_complex_djbunfft_preload
c5b:	yr4_b4cl_four_complex_djbunfft rsi, 4*64, 64, 2*64, rdi, 0, 16
	yloop	16, c5b, -16*4*64+dist64, rdi, 4*YMM_SCD2/4 ;; Loop 16 times, next source pointer, next sine/cosine pointer
	yloop	4, c5b, -4*dist64, rdi, -4*4*YMM_SCD2/4 ;; Loop 4 times, restore source pointer, restore sine/cosine pointer
	loops_undo 4*16			;; Restore loop counter
	end_timer 10

	start_timer 11
;;	yr4_b4cl_four_complex_djbunfft_preload
c4b:	yr4_b4cl_four_complex_djbunfft rsi, 64, 4*64, 8*64, rdi, 4*YMM_SCD2/4, 4
	yloop	4, c4b, -4*64+16*64, rdi, -4*4*YMM_SCD2/4 ;; Loop 4 times, next source pointer, restore sine/cosine pointer
	yloop	4, c4b, -4*16*64+dist64, rdi, YMM_SCD2/4 ;; Loop 4 times, next source pointer, next sine/cosine pointer
	yloop	4, c4b, -4*dist64, rdi, -4*YMM_SCD2/4 ;; Loop 4 times, restore source pointer, restore sine/cosine pointer
	loops_undo 4*4*4		;; Back up loop counter
	end_timer 11

	start_timer 12
;;	yr4_b4cl_four_complex_djbunfft_preload
c3b:	yr4_b4cl_four_complex_djbunfft rsi, 64, 16*64, 32*64, rdi, YMM_SCD2/4, 16
	yloop	16, c3b, -16*64+dist64	;; Loop 16 times, next source pointer
	yloop	4, c3b, -4*dist64+4*dist64, rdi, -64*YMM_SCD2/4 ;; Loop 4 times, next source pointer, restore sine/cosine pointer
	end_timer 12

	yloop	5, b3b0, -5*4*dist64	;; Loop 5 times, restore source pointer
	jmp	ypass2_5120_complex_unfft


ypass2_5120_complex_1:
	start_timer 6
	bump	rcx, (1280+19-64-32-4*320)*64 ;; Move prefetch pointer along
	yloop_init 320			;; 5 iters of 256 cache lines
	yloop_set_prefetch YSINGLE

b3a0:	yr4_b4cl_four_complex_djbfft_preload
b3a:	yr4_b4cl_four_complex_djbfft rsi, 64, 16*64, 32*64, rdi, YMM_SCD2/4, 16
	yloop	16, b3a, -16*64+dist64	;; Loop 16 times, next source pointer
	yloop	4, b3a, -4*dist64, rdi, -64*YMM_SCD2/4 ;; Loop 4 times, restore source pointer, restore sine/cosine pointer
	loops_undo 4*16			;; Back up loop counter

;;	yr4_b4cl_four_complex_djbfft_preload
b4a:	yr4_b4cl_four_complex_djbfft rsi, 64, 4*64, 8*64, rdi, 4*YMM_SCD2/4, 4
	yloop	4, b4a, -4*64+16*64, rdi, -4*4*YMM_SCD2/4 ;; Loop 4 times, next source pointer, restore sine/cosine pointer
	yloop	4, b4a, -4*16*64+dist64, rdi, YMM_SCD2/4 ;; Loop 4 times, next source pointer, next sine/cosine pointer
	yloop	4, b4a, -4*dist64, rdi, -4*YMM_SCD2/4 ;; Loop 4 times, restore source pointer, restore sine/cosine pointer
	loops_undo 4*4*4		;; Back up loop counter

;;	yr4_b4cl_four_complex_djbfft_preload
b5a:	yr4_b4cl_four_complex_djbfft rsi, 4*64, 64, 2*64, rdi, 0, 16
	yloop	16, b5a, -16*4*64+dist64, rdi, 4*YMM_SCD2/4 ;; Loop 16 times, next source pointer, next sine/cosine pointer
	yloop	4, b5a, -4*dist64, rdi, -4*4*YMM_SCD2/4 ;; Loop 4 times, restore source pointer, restore sine/cosine pointer
	loops_undo 4*16			;; Back up loop counter

	yr4_4cl_four_complex_fft_final_preload
;; Loop below reworked to avoid 4KB distances
;;b6a:	yr4_4cl_four_complex_fft_final rsi, 64, dist64, 2*dist64
;;	yloop	64, b6a, -64*64	;; Loop 64 times, restore source pointer
b6a:	yr4_4cl_four_complex_fft_final rsi, 4*64, dist64, 2*dist64
	yloop	16, b6a, -16*4*64+64	;; Loop 16 times, next source pointer
	yloop	4, b6a, -4*64+4*dist64	;; Loop 4 times, next source pointer

	yloop	5, b3a0			;; Loop 5 times
	end_timer 6
	jmp	ypass2_5120_complex_done


ypass2_5120_complex_3:
	start_timer 6
x64	mov	r8, DEST2ARG
	mov	rbp, DIST_TO_MULSRCARG
	yloop_init 320			;; 5 iters of 256 cache lines
	yloop_set_prefetch YHALF

b3c0:	yr4_b4cl_four_complex_djbfft_preload
b3c:	yr4_b4cl_four_complex_djbfft rsi, 64, 16*64, 32*64, rdi, YMM_SCD2/4, 16
	yloop	16, b3c, -16*64+dist64	;; Loop 16 times, next source pointer
	yloop	4, b3c, -4*dist64, rdi, -64*YMM_SCD2/4 ;; Loop 4 times, restore source pointer, restore sine/cosine pointer
	loops_undo 4*16			;; Back up loop counter

;;	yr4_b4cl_four_complex_djbfft_preload
b4c:	yr4_b4cl_four_complex_djbfft rsi, 64, 4*64, 8*64, rdi, 4*YMM_SCD2/4, 4
	yloop	4, b4c, -4*64+16*64, rdi, -4*4*YMM_SCD2/4 ;; Loop 4 times, next source pointer, restore sine/cosine pointer
	yloop	4, b4c, -4*16*64+dist64, rdi, YMM_SCD2/4 ;; Loop 4 times, next source pointer, next sine/cosine pointer
	yloop	4, b4c, -4*dist64, rdi, -4*YMM_SCD2/4 ;; Loop 4 times, restore source pointer, restore sine/cosine pointer
	loops_undo 4*4*4		;; Back up loop counter

;;	yr4_b4cl_four_complex_djbfft_preload
b5c:	yr4_b4cl_four_complex_djbfft rsi, 4*64, 64, 2*64, rdi, 0, 16
	yloop	16, b5c, -16*4*64+dist64, rdi, 4*YMM_SCD2/4 ;; Loop 16 times, next source pointer, next sine/cosine pointer
	yloop	4, b5c, -4*dist64, rdi, -4*4*YMM_SCD2/4 ;; Loop 4 times, restore source pointer, restore sine/cosine pointer
	loops_undo 4*16			;; Back up loop counter

	yr4_4cl_four_complex_with_mult_preload
;; Loop below reworked to avoid 4KB distances
;;b6c:	yr4_4cl_four_complex_with_mult rsi, 64, dist64, 2*dist64
;;	yloop	64, b6c, -64*64	;; Loop 64 times, restore source pointer
b6c:	yr4_4cl_four_complex_with_mult rsi, 4*64, dist64, 2*dist64
	yloop	16, b6c, -16*4*64+64	;; Loop 16 times, next source pointer
	yloop	4, b6c, -4*64		;; Loop 4 times, restore source pointer
x64	add	rsi, r8
	loops_undo 64			;; Back up loop counter

	yr4_b4cl_four_complex_djbunfft_preload
c5c:	yr4_b4cl_four_complex_djbunfft rsi, 4*64, 64, 2*64, rdi, 0, 16
	yloop	16, c5c, -16*4*64+dist64, rdi, 4*YMM_SCD2/4 ;; Loop 16 times, next source pointer, next sine/cosine pointer
	yloop	4, c5c, -4*dist64, rdi, -4*4*YMM_SCD2/4 ;; Loop 4 times, restore source pointer, restore sine/cosine pointer
	loops_undo 4*16			;; Restore loop counter

;;	yr4_b4cl_four_complex_djbunfft_preload
c4c:	yr4_b4cl_four_complex_djbunfft rsi, 64, 4*64, 8*64, rdi, 4*YMM_SCD2/4, 4
	yloop	4, c4c, -4*64+16*64, rdi, -4*4*YMM_SCD2/4 ;; Loop 4 times, next source pointer, restore sine/cosine pointer
	yloop	4, c4c, -4*16*64+dist64, rdi, YMM_SCD2/4 ;; Loop 4 times, next source pointer, next sine/cosine pointer
	yloop	4, c4c, -4*dist64, rdi, -4*YMM_SCD2/4 ;; Loop 4 times, restore source pointer, restore sine/cosine pointer
	loops_undo 4*4*4		;; Back up loop counter

;;	yr4_b4cl_four_complex_djbunfft_preload
c3c:	yr4_b4cl_four_complex_djbunfft rsi, 64, 16*64, 32*64, rdi, YMM_SCD2/4, 16
	yloop	16, c3c, -16*64+dist64	;; Loop 16 times, next source pointer
	yloop	4, c3c, -4*dist64+4*dist64, rdi, -64*YMM_SCD2/4 ;; Loop 4 times, next source pointer, restore sine/cosine pointer

x64	sub	rsi, r8
	yloop	5, b3c0, -5*4*dist64	;; Loop 5 times, restore source pointer
x64	add	rsi, r8
	end_timer 6
	jmp	ypass2_5120_complex_unfft


ypass2_5120_complex_4:
	start_timer 6
	mov	rdi, xsincos_complex	;; The sin/cos ptrs
	bump	rdi, 256*YMM_SCD2+64*YMM_SCD2
	mov	rbp, DIST_TO_MULSRCARG
	yloop_init 320			;; 5 iters of 256 cache lines
	yloop_set_prefetch YSINGLE

	yr4_4cl_four_complex_with_mulf_preload
;; Loop below reworked to avoid 4KB distances
;;b6d:	yr4_4cl_four_complex_with_mulf rsi, 64, dist64, 2*dist64
;;	yloop	64, b6d, -64*64	;; Loop 64 times, restore source pointer
b6d:	yr4_4cl_four_complex_with_mulf rsi, 4*64, dist64, 2*dist64
	yloop	16, b6d, -16*4*64+64	;; Loop 16 times, next source pointer
	yloop	4, b6d, -4*64		;; Loop 4 times, restore source pointer
	loops_undo 64			;; Back up loop counter

	yr4_b4cl_four_complex_djbunfft_preload
c5d:	yr4_b4cl_four_complex_djbunfft rsi, 4*64, 64, 2*64, rdi, 0, 16
	yloop	16, c5d, -16*4*64+dist64, rdi, 4*YMM_SCD2/4 ;; Loop 16 times, next source pointer, next sine/cosine pointer
	yloop	4, c5d, -4*dist64, rdi, -4*4*YMM_SCD2/4 ;; Loop 4 times, restore source pointer, restore sine/cosine pointer
	loops_undo 4*16			;; Restore loop counter

;;	yr4_b4cl_four_complex_djbunfft_preload
c4d:	yr4_b4cl_four_complex_djbunfft rsi, 64, 4*64, 8*64, rdi, 4*YMM_SCD2/4, 4
	yloop	4, c4d, -4*64+16*64, rdi, -4*4*YMM_SCD2/4 ;; Loop 4 times, next source pointer, restore sine/cosine pointer
	yloop	4, c4d, -4*16*64+dist64, rdi, YMM_SCD2/4 ;; Loop 4 times, next source pointer, next sine/cosine pointer
	yloop	4, c4d, -4*dist64, rdi, -4*YMM_SCD2/4 ;; Loop 4 times, restore source pointer, restore sine/cosine pointer
	loops_undo 4*4*4		;; Back up loop counter

;;	yr4_b4cl_four_complex_djbunfft_preload
c3d:	yr4_b4cl_four_complex_djbunfft rsi, 64, 16*64, 32*64, rdi, YMM_SCD2/4, 16
	yloop	16, c3d, -16*64+dist64	;; Loop 16 times, next source pointer
	yloop	4, c3d, -4*dist64+4*dist64, rdi, -64*YMM_SCD2/4 ;; Loop 4 times, next source pointer, restore sine/cosine pointer

	yloop	5, b6d, -5*4*dist64	;; Loop 5 times, restore source pointer
	bump	rcx, (-4*320+64+32+7*160)*64 ;; Move prefetch pointer to where squaring would be
	end_timer 6

;; Do inverse FFT levels 3,4
;; On input the 64-byte cache lines hold these data values:
;;	0	+512	+512	+512	256	+512	+512	+512
;;	4	...
;;	...
;;	252	...
;;	1	...
;;	...
;;	2048	...
;;	...
;; On output the 64-byte cache lines are swizzled to hold these data values:
;;	0	+1	+1	+1	1024	+1	+1	+1
;;	4	...
;;	...
;;	1020	...
;;	2048	...
;;	...

;; Do 1280 four_complex_unfft operations
;;	distance between fft data elements is 256
;;	do 320 macros each processing 32 data values

;; Do inverse FFT levels 1,2
;; On input the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	1024	+1	+1	+1
;;	4	...
;;	...
;;	1020	...
;;	2048	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	5120	+1	+1	+1
;;	4	...
;;	...
;;	5116

;; Do 1024 five_complex_unfft operations
;;	distance between fft data elements is 1024
;;	do 256 macros each processing 40 data values

ypass2_5120_complex_unfft:
	mov	rdi, xsincos_complex	;; Load sin/cos pointers
	lea	rbp, [rdi+256*YMM_SCD2]
	bump	rcx, (1280+19-64-32-7*160-64-32)*64 ;; Move prefetch pointer along
	yloop_init 512			;; 256 iterations or 320 iterations

c2a:	start_timer 13
	yloop_adjust 3			;; Three fewer iteration this loop
	yloop_set_prefetch YOTHER, 8	;; Prefetch 64 cache lines in this loop
	yr4_s4cl_four_complex_djbunfft_preload
c2b:	yr4_s4cl_four_complex_djbunfft rsi, 4*dist64, dist64, 2*dist64, rbp, 0, 5, L1PREFETCH_ALL, 64
	yloop	8, c2b, -5*4*dist64, rbp, YMM_SCD2 ;; 5 iterations!!  Restore source pointer, next sine/cosine pointer
	loops_undo 8			;; Restore loop counter
	end_timer 13

	start_timer 14
	yloop_adjust 4			;; Four fewer iteration this loop
	yloop_set_prefetch YOTHER, 16	;; Prefetch 32 cache lines in this loop
	yr5_5cl_five_complex_djbunfft_preload
c1b:	yr5_5cl_five_complex_djbunfft rsi, dist64, 4*dist64, rdi, 64*YMM_SCD2, 4
	yloop	8, c1b, -4*dist64+64, rdi, -4*64*YMM_SCD2+YMM_SCD2 ;; 4 iterations!!  Next source pointer, next sine/cosine pointer
	end_timer 14

	yloop	64, c2a			;; Loop 64 times

ypass2_5120_complex_done:
	ENDM


; *************** radix4-pass2-6400-real macro ******************
; This macro takes 12800 real values and performs the final 13.644 levels of the FFT.
; The results are squared, and the first 13.644 levels of the inverse FFT are done.

yr4dwpn_pass2_6400_real MACRO
	LOCAL	b1b, b2a, b2b, b3a, b3b, b4a, b4b, b5a, b5b, b6a, b6b, b6c, b6d
	LOCAL	c1b, c2a, c2b, c3a, c3b, c4a, c4b, c5a, c5b
	LOCAL	ypass2_6400_real_1, ypass2_6400_real_2
	LOCAL	ypass2_6400_real_3, ypass2_6400_real_4
 	LOCAL	ypass2_6400_real_unfft, ypass2_6400_real_done

;; Type 4 FFTs skip the forward FFT process

	start_timer 2
	mov	rsi, DATA_ADDR		;; Load source address
	mov	rbx, DIST_TO_FFTSRCARG
	mov	rcx, DATA_PREFETCH	;; Prefetch pointer (we need to prefetch 
					;; 1600 cache lines + 24 pad cache lines)
	cmp	ffttype, 4
	je	ypass2_6400_real_4

;; Do FFT levels 1,2,3 which correspond to complex levels 1,2
;; On input the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	6400	+1	+1	+1
;;	4	...
;;	...
;;	6396
;; On output the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	1280	+1	+1	+1
;;	4	...
;;	...
;;	1276	...
;;	2560	...
;;	...

;; Do 1280 ten_reals_fft operations
;;	distance between fft data elements is 1280
;;	do 320 macros each processing 40 data values

	mov	rdi, sincos3		;; Real sin/cos pointer
	mov	rbp, xsincos_complex	;; Complex sin/cos data
	yloop_init 320			;; 320 iterations
	yloop_set_prefetch YHALF
	yr5_f5cl_2sc_ten_reals_fft_preload
b1b:	yr5_f5cl_2sc_ten_reals_fft rsi, 64, 5*dist64, rbp, YMM_SCD2, rdi, YMM_SCD2, 64
	yloop	64, b1b, -64*64+dist64	;; Test loop counter, next source pointer
	yloop	5, b1b, -5*dist64	;; Test loop counter, restore source pointer

;; Do FFT levels 3,4
;; On input the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	1280	+1	+1	+1
;;	4	...
;;	...
;;	1276	...
;;	2560	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	256	+1	+1	+1
;;	4	...
;;	...
;;	252	...
;;	512	...
;;	...

;; Do 256 ten_reals_fft operations
;; Do 512 five_complex_fft operations
;;	distance between fft data elements is 256
;;	do 320 macros each processing 40 data values

	yloop_init 320			;; 5 iterations of 64
	yloop_set_prefetch YQUARTER
	yr5_5cl_2sc_ten_reals_fft_preload
b2a:	yr5_5cl_2sc_ten_reals_fft rsi, 64, dist64, rbp, YMM_SCD2, rdi, YMM_SCD2, 64
	yloop	64, b2a, -64*64+5*dist64, rbp, -64*YMM_SCD2 ;; Test loop counter, next source pointer, restore sin/cos pointer

	loops_reset			;; 4 more iters of 64
	yr5_5cl_five_complex_djbfft_preload
b2b:	yr5_5cl_five_complex_djbfft rsi, 64, dist64, rbp, YMM_SCD2, 64
	yloop	64, b2b, -64*64+5*dist64, rbp, -64*YMM_SCD2 ;; Test loop counter, next source pointer, restore sin/cos pointer
	yloop	5, b2b, -5*5*dist64, rbp, 64*YMM_SCD2 ;; Test loop counter, restore source pointer, next sin/cos pointer

;; Do FFT levels 5,6
;; On input the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	256	+1	+1	+1
;;	4	...
;;	...
;;	252	...
;;	512	...
;;	...
;; On output the 64-byte cache lines are swizzled to hold these data values:
;;	0	+128	+128	+128	64	+128	+128	+128
;;	4	...
;;	...
;;	60	...
;;	1	...
;;	...
;;	512	...
;;	...

;; Do 64 eight_reals_fft operations
;; Do 1536 four_complex_fft operations
;;	distance between fft data elements is 64
;;	do 400 macros each processing 32 data values

	yloop_init 400			;; 25 iters of 16
	yloop_set_prefetch YHALF
	yr4_s4cl_2sc_eight_reals_fft_preload
b3a:	yr4_s4cl_2sc_eight_reals_fft rsi, 64, 16*64, 32*64, rbp, YMM_SCD2, rdi, YMM_SCD2, 16
	yloop	16, b3a, -16*64+dist64, rbp, -16*YMM_SCD2 ;; Test loop counter, next source pointer, restore sine/cosine pointer

	loops_reset			;; 24 more iters of 16
	yr4_s4cl_four_complex_djbfft_preload
b3b:	yr4_s4cl_four_complex_djbfft rsi, 64, 16*64, 32*64, rbp, YMM_SCD2, 16
	yloop	16, b3b, -16*64+dist64, rbp, -16*YMM_SCD2 ;; Test loop counter, next source pointer, restore sin/cos ptr
	yloop	25, b3b, -25*dist64	;; Loop 25 times, restore source pointer

;; Do FFT levels 7,8
;; On input the 64-byte cache lines hold these data values:
;;	0	+128	+128	+128	64	+128	+128	+128
;;	4	...
;;	...
;;	60	...
;;	1	...
;;	...
;;	512	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+128	+128	+128	16	+128	+128	+128
;;	4	...
;;	...
;;	12	...
;;	32	...
;;	...
;;	1	...
;;	...
;;	512	...
;;	...

;; Do 16 eight_reals_fft operations
;; Do 1584 four_complex_fft operations
;;	distance between fft data elements is 16
;;	do 400 macros each processing 32 data values

	yloop_init 400			;; 25 iterations of 16
	yloop_set_prefetch YHALF
	yr4_4cl_eight_reals_four_complex_djbfft_preload
b4a:	yr4_4cl_eight_reals_four_complex_djbfft rsi, 16*64, 4*64, 8*64, rdi, YMM_SCD3, 4
	yloop	4, b4a, -4*16*64+64	;; Test loop counter, next source pointer
	yloop	4, b4a, -4*64+dist64, rdi, -16*YMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer

	loops_reset			;; 24 more iters of 16
	yr4_rb4cl_four_complex_djbfft_preload
b4b:	yr4_rb4cl_four_complex_djbfft rsi, 16*64, 4*64, 8*64, rdi, YMM_SCD3, 4
	yloop	4, b4b, -4*16*64+64	;; Test loop counter, next source pointer
	yloop	4, b4b, -4*64+dist64, rdi, -16*YMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer
	yloop	25, b4b, -25*dist64	;; Test loop counter, restore source pointer

;; Do FFT levels 9,10
;; On input the 64-byte cache lines hold these data values:
;;	0	+128	+128	+128	16	+128	+128	+128
;;	4	...
;;	...
;;	12	...
;;	32	...
;;	...
;;	1	...
;;	...
;;	512	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+128	+128	+128	4	+128	+128	+128
;;	8	...
;;	...
;;	1	...
;;	...
;;	512	...
;;	...

;; Do 4 eight_reals_fft operations
;; Do 1596 four_complex_fft operations
;;	distance between fft data elements is 4
;;	do 400 macros each processing 32 data values

	yloop_init 400			;; 100 iterations of 4
	yloop_set_prefetch YHALF
	yr4_4cl_eight_reals_four_complex_djbfft_preload
b5a:	yr4_4cl_eight_reals_four_complex_djbfft rsi, 16*64, 64, 2*64, rdi, 4*YMM_SCD3, 4
	yloop	4, b5a, -4*16*64+4*64, rdi, -4*4*YMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer

	loops_reset			;; 99 more iters of 4
	yr4_rb4cl_four_complex_djbfft_preload
b5b:	yr4_rb4cl_four_complex_djbfft rsi, 16*64, 64, 2*64, rdi, 4*YMM_SCD3, 4
	yloop	4, b5b, -4*16*64+4*64, rdi, -4*4*YMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer
	yloop	4, b5b, -4*4*64+dist64	;; Test loop counter, next source pointer
	yloop	25, b5b, -25*dist64	;; Test loop counter, restore source pointer

;; Do FFT levels 11,12 as well as inverse FFT levels 11,12
;; On input and output the 64-byte cache lines hold these data values:
;;	0	+128	+128	+128	4	+128	+128	+128
;;	8	...
;;	...
;;	1	...
;;	...
;;	512	...
;;	...

;; Do 1 eight_reals_fft operation
;; Do 1659 four_complex_fft operations
;;	distance between fft data elements is 1
;;	do 400 macros each processing 32 data values

	cmp	ffttype, 2
	jl	ypass2_6400_real_1
	jg	ypass2_6400_real_3

ypass2_6400_real_2:
;;	yr4_4cl_eight_reals_four_complex_with_square rsi, 64, 16*64, 32*64
	yr4_dispatch call, yreal_square, 16*64, 32*64
	yloop_init 400			;; 399 more iterations
	yloop_adjust 1
	yloop_set_prefetch YHALF
	yr4_4cl_four_complex_with_square_preload
b6b:	yr4_4cl_four_complex_with_square rsi, 64, 16*64, 32*64, 1
	yloop	16, b6b, -16*64+dist64	;; Test loop counter, next source pointer
	yloop	25, b6b, -25*dist64	;; Test loop counter, restore source pointer
	jmp	ypass2_6400_real_unfft

ypass2_6400_real_1:
;;	yr4_4cl_eight_reals_four_complex_fft_final rsi, 64, 16*64, 32*64
	yr4_dispatch call, yreal_fft_final, 16*64, 32*64
	mov	rcx, xsincos_complex	;; Prefetch complex sin/cos data, 800 cache lines
	yloop_init 400			;; 399 more iterations
	yloop_adjust 1
	yloop_set_prefetch YDOUBLE
b6a:	yr4_4cl_four_complex_fft_final rsi, 64, 16*64, 32*64
	yloop	16, b6a, -16*64+dist64	;; Test loop counter, next source pointer
	yloop	25, b6a			;; Test loop counter
	jmp	ypass2_6400_real_done

ypass2_6400_real_3:
x64	mov	r8, DEST2ARG
	mov	rbp, DIST_TO_MULSRCARG
;;	yr4_4cl_eight_reals_four_complex_with_mult rsi, 64, 16*64, 32*64
	yr4_dispatch call, yreal_mult, 16*64, 32*64
	yloop_init 400			;; 399 more iterations
	yloop_adjust 1
	yloop_set_prefetch YHALF
	yr4_4cl_four_complex_with_mult_preload
b6c:	yr4_4cl_four_complex_with_mult rsi, 64, 16*64, 32*64
	yloop	16, b6c, -16*64+dist64	;; Test loop counter, next source pointer
	yloop	25, b6c, -25*dist64	;; Test loop counter, restore source pointer
x64	add	rsi, r8
	jmp	ypass2_6400_real_unfft

ypass2_6400_real_4:
	mov	rbp, DIST_TO_MULSRCARG
;;	yr4_4cl_eight_reals_four_complex_with_mulf rsi, 64, 16*64, 32*64
	yr4_dispatch call, yreal_mulf, 16*64, 32*64
	yloop_init 400			;; 399 more iterations
	yloop_adjust 1
	yloop_set_prefetch YDOUBLE
	yr4_4cl_four_complex_with_mulf_preload
b6d:	yr4_4cl_four_complex_with_mulf rsi, 64, 16*64, 32*64
	yloop	16, b6d, -16*64+dist64	;; Test loop counter, next source pointer
	yloop	25, b6d, -25*dist64	;; Test loop counter, restore source pointer
;;	bump	rcx, (-798+160+80+4*200)*64 ;; Move prefetch pointer back to where squaring would be

;; Do inverse FFT levels 9,10
;; On input the 64-byte cache lines hold these data values:
;;	0	+128	+128	+128	4	+128	+128	+128
;;	8	...
;;	...
;;	1	...
;;	...
;;	512	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+128	+128	+128	16	+128	+128	+128
;;	4	...
;;	...
;;	12	...
;;	32	...
;;	...
;;	1	...
;;	...
;;	512	...
;;	...

;; Do 4 eight_reals_unfft operations
;; Do 1596 four_complex_unfft operations
;;	distance between fft data elements is 4
;;	do 400 macros each processing 32 data values

ypass2_6400_real_unfft:
	mov	rdi, sincos3		;; Load sin/cos pointer
	bump	rdi, 16*YMM_SCD2+64*YMM_SCD2+320*YMM_SCD2
	yloop_init 400			;; 100 iterations of 4
	yloop_set_prefetch YHALF
	yr4_4cl_eight_reals_four_complex_djbunfft_preload
c5a:	yr4_4cl_eight_reals_four_complex_djbunfft rsi, 16*64, 64, 2*64, rdi, 4*YMM_SCD3, 4
	yloop	4, c5a, -4*16*64+4*64, rdi, -4*4*YMM_SCD3 ;; Test loop counter, next source pointer, restore sine/cosine pointer

	loops_reset			;; 99 more iters of 4
	yr4_rb4cl_four_complex_djbunfft_preload
c5b:	yr4_rb4cl_four_complex_djbunfft rsi, 16*64, 64, 2*64, rdi, 4*YMM_SCD3, 4
	yloop	4, c5b, -4*16*64+4*64, rdi, -4*4*YMM_SCD3 ;; Test loop counter, next source pointer, restore sine/cosine pointer
	yloop	4, c5b, -4*4*64+dist64	;; Test loop counter, next source pointer
	yloop	25, c5b, -25*dist64	;; Test loop counter, restore source pointer

;; Do inverse FFT levels 7,8
;; On input the 64-byte cache lines hold these data values:
;;	0	+128	+128	+128	16	+128	+128	+128
;;	4	...
;;	...
;;	12	...
;;	32	...
;;	...
;;	1	...
;;	...
;;	512	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+128	+128	+128	64	+128	+128	+128
;;	4	...
;;	...
;;	60	...
;;	1	...
;;	...
;;	512	...
;;	...

;; Do 16 eight_reals_unfft operations
;; Do 1584 four_complex_unfft operations
;;	distance between fft data elements is 16
;;	do 400 macros each processing 32 data values

	bump	rcx, (1600+24-160-80-5*200-400)*64 ;; Move prefetch pointer along
	yloop_init 400			;; 25 iterations of 16
	yloop_set_prefetch YSINGLE
	yr4_4cl_eight_reals_four_complex_djbunfft_preload
c4a:	yr4_4cl_eight_reals_four_complex_djbunfft rsi, 16*64, 4*64, 8*64, rdi, YMM_SCD3, 4
	yloop	4, c4a, -4*16*64+64	;; Test loop counter, next source pointer
	yloop	4, c4a, -4*64+dist64, rdi, -16*YMM_SCD3 ;; Test loop counter, next source pointer, restore sine/cosine pointer

	loops_reset			;; 24 more iters of 16
	yr4_rb4cl_four_complex_djbunfft_preload
c4b:	yr4_rb4cl_four_complex_djbunfft rsi, 16*64, 4*64, 8*64, rdi, YMM_SCD3, 4
	yloop	4, c4b, -4*16*64+64	;; Test loop counter, next source pointer
	yloop	4, c4b, -4*64+dist64, rdi, -16*YMM_SCD3 ;; Test loop counter, next source pointer, restore sine/cosine pointer
	yloop	25, c4b, -25*dist64	;; Test loop counter, restore source pointer

;; Do inverse FFT levels 5,6
;; On input the 64-byte cache lines hold these data values:
;;	0	+128	+128	+128	64	+128	+128	+128
;;	4	...
;;	...
;;	60	...
;;	1	...
;;	...
;;	512	...
;;	...
;; On output the 64-byte cache lines are swizzled to hold these data values:
;;	0	+1	+1	+1	256	+1	+1	+1
;;	4	...
;;	...
;;	252	...
;;	512	...
;;	...

;; Do 64 eight_reals_unfft operations
;; Do 1536 four_complex_unfft operations
;;	distance between fft data elements is 64
;;	do 400 macros each processing 32 data values

	bump	rdi, -16*YMM_SCD2	;; Real sin/cos ptr
	mov	rbp, xsincos_complex	;; Complex sin/cos data
	bump	rbp, 64*YMM_SCD2+320*YMM_SCD2
	mov	rcx, xsincos_complex	;; Prefetch complex sin/cos data, 800 cache lines
	yloop_init 400			;; 25 iters of 16
	yloop_set_prefetch YSINGLE
	yr4_s4cl_2sc_eight_reals_unfft_preload
c3a:	yr4_s4cl_2sc_eight_reals_unfft rsi, 64, 16*64, 32*64, rbp, YMM_SCD2, rdi, YMM_SCD2, 16
	yloop	16, c3a, -16*64+dist64, rbp, -16*YMM_SCD2 ;; Test loop counter, next source pointer, restore sine/cosine pointer

	loops_reset			;; 24 more iters of 16
	yr4_s4cl_four_complex_djbunfft_preload
c3b:	yr4_s4cl_four_complex_djbunfft rsi, 64, 16*64, 32*64, rbp, YMM_SCD2, 16
	yloop	16, c3b, -16*64+dist64, rbp, -16*YMM_SCD2 ;; Test loop counter, next source pointer, restore sine/cosine pointer
	yloop	25, c3b, -25*dist64	;; Test loop counter, restore source pointer

;; Do inverse FFT levels 3,4
;; On input the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	256	+1	+1	+1
;;	4	...
;;	...
;;	252	...
;;	512	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	1280	+1	+1	+1
;;	4	...
;;	...
;;	1276	...
;;	2560	...
;;	...

;; Do 256 ten_reals_unfft operations
;; Do 512 five_complex_unfft operations
;;	distance between fft data elements is 256
;;	do 320 macros each processing 40 data values

	bump	rdi, -16*YMM_SCD2-64*YMM_SCD2 ;; Real sin/cos ptr
	bump	rbp, -64*YMM_SCD2	;; Complex sin/cos data
	yloop_init 320			;; 5 iters of 64
	yloop_set_prefetch YSINGLE
	yr5_5cl_2sc_ten_reals_unfft_preload
c2a:	yr5_5cl_2sc_ten_reals_unfft rsi, 64, dist64, rbp, YMM_SCD2, rdi, YMM_SCD2, 64
	yloop	64, c2a, -64*64+5*dist64, rbp, -64*YMM_SCD2 ;; Test loop counter, next source pointer, restore sine/cosine pointer

	loops_reset			;; 4 more iters of 64
	yr5_5cl_five_complex_djbunfft_preload
c2b:	yr5_5cl_five_complex_djbunfft rsi, 64, dist64, rbp, YMM_SCD2, 64
	yloop	64, c2b, -64*64+5*dist64, rbp, -64*YMM_SCD2 ;; Test loop counter, next source pointer, restore sine/cosine pointer
	yloop	5, c2b, -5*5*dist64	;; Test loop counter, restore source pointer

;; Do inverse FFT levels 1,2,3 which correspond to complex levels 1,2
;; On input the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	1280	+1	+1	+1
;;	4	...
;;	...
;;	1276	...
;;	2560	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	6400	+1	+1	+1
;;	4	...
;;	...
;;	6396

;; Do 1280 ten_reals_unfft operations
;;	distance between fft data elements is 1280
;;	do 320 macros each processing 40 data values

	bump	rdi, -64*YMM_SCD2-320*YMM_SCD2 ;; Real sin/cos ptr
	bump	rbp, -320*YMM_SCD2	;; Complex sin/cos ptr
	bump	rcx, (800-400-320-80)*64 ;; Move prefetch pointer along
	yloop_init 320			;; 320 iterations
	yloop_set_prefetch YQUARTER
	yr5_5cl_2sc_ten_reals_unfft_preload
c1b:	yr5_5cl_2sc_ten_reals_unfft rsi, 64, 5*dist64, rbp, YMM_SCD2, rdi, YMM_SCD2, 64
	yloop	64, c1b, -64*64+dist64	;; Test loop counter, next source pointer
	yloop	5, c1b			;; Test loop counter

ypass2_6400_real_done:
	end_timer 2
	ENDM

; *************** radix4-pass2-6400-complex macro ******************
; This macro takes 6400 complex values and performs the final 12.644 levels of a standard
; radix-4 FFT, squares the results, and does the first 12.644 levels of the inverse FFT.

yr4dwpn_pass2_6400_complex MACRO
	LOCAL	b1a, b1b, b2b, b3a0, b3a, b3b0, b3b, b3c0, b3c, b4a, b4b, b4c, b5a, b5b, b5c, b6a, b6b, b6c, b6d
	LOCAL	c1b, c2a, c2b, c3b, c3c, c3d, c4b, c4c, c4d, c5b, c5c, c5d
	LOCAL	ypass2_6400_complex_1, ypass2_6400_complex_2
	LOCAL	ypass2_6400_complex_3, ypass2_6400_complex_4
	LOCAL	ypass2_6400_complex_unfft, ypass2_6400_complex_done

;; Type 4 FFTs skip the forward FFT process

	mov	rsi, DATA_ADDR		;; Load address of FFT data
	mov	rbx, DIST_TO_FFTSRCARG
	mov	rcx, DATA_PREFETCH	;; Prefetch pointer (we need to prefetch 
					;; 1600 cache lines + 24 pad cache lines)
	cmp	ffttype, 4
	je	ypass2_6400_complex_4

;; Do FFT level 1,2 (actually 2.322 levels)
;; On input the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	6400	+1	+1	+1
;;	4	...
;;	...
;;	6396
;; On output the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	1280	+1	+1	+1
;;	4	...
;;	...
;;	1276	...
;;	2560	...
;;	...

;; Do 1280 five_complex_fft operations
;;	distance between fft data elements is 1280
;;	do 320 macros each processing 40 data values

;; Do FFT levels 3,4 (actually 2.322 levels)
;; On input the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	1280	+1	+1	+1
;;	4	...
;;	...
;;	1276	...
;;	2560	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	256	+1	+1	+1
;;	4	...
;;	...
;;	252	...
;;	512	...
;;	...

;; Do 1280 five_complex_fft operations
;;	distance between fft data elements is 256
;;	do 320 macros each processing 40 data values

	mov	rdi, xsincos_complex	;; The sin/cos ptrs
	lea	rbp, [rdi+320*YMM_SCD2]
	yloop_init 512			;; 320 iterations

	yr5_f5cl_five_complex_djbfft_preload
b1a:	start_timer 4
	yloop_adjust 3			;; Three fewer iterations this loop
	yloop_set_prefetch YOTHER, 8	;; Prefetch 64 cache lines
b1b:	yr5_f5cl_five_complex_djbfft rsi, dist64, 5*dist64, rdi, 64*YMM_SCD2, 5
	yloop	8, b1b, -5*dist64, rdi, -5*64*YMM_SCD2+YMM_SCD2 ;; 5 iterations!!  Restore source pointer, next sin/cos data
	loops_undo 8
	end_timer 4

	start_timer 5
	yloop_adjust 3			;; Three fewer iterations this loop
	yloop_set_prefetch YOTHER, 8	;; Prefetch 64 cache lines
;;	yr5_5cl_five_complex_djbfft_preload
b2b:	yr5_5cl_five_complex_djbfft rsi, 5*dist64, dist64, rbp, 0, 5, L1PREFETCH_ALL, 64
	yloop	8, b2b, -5*5*dist64+64, rbp, YMM_SCD2 ;; 5 iterations!!  Next source pointer, next sin/cos data
	end_timer 5

	yloop	64, b1a, -64*64, rdi, -64*YMM_SCD2+320*YMM_SCD2+64*YMM_SCD2 ;; Test loop counter, restore source pointer, next sin/cos ptr

;; Do FFT levels 5,6
;; On input the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	256	+1	+1	+1
;;	4	...
;;	...
;;	252	...
;;	512	...
;;	...
;; On output the 64-byte cache lines are swizzled to hold these data values:
;;	0	+128	+128	+128	64	+128	+128	+128
;;	4	...
;;	...
;;	60	...
;;	1	...
;;	...
;;	512	...
;;	...

;; Do 1600 four_complex_fft operations
;;	distance between fft data elements is 64
;;	do 400 macros each processing 32 data values

;; Do FFT levels 7,8
;; On input the 64-byte cache lines hold these data values:
;;	0	+128	+128	+128	64	+128	+128	+128
;;	4	...
;;	...
;;	60	...
;;	1	...
;;	...
;;	512	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+128	+128	+128	16	+128	+128	+128
;;	4	...
;;	...
;;	12	...
;;	32	...
;;	...
;;	1	...
;;	...
;;	512	...
;;	...

;; Do 1600 four_complex_fft operations
;;	distance between fft data elements is 16
;;	do 400 macros each processing 32 data values

;; Do FFT levels 9,10
;; On input the 64-byte cache lines hold these data values:
;;	0	+128	+128	+128	16	+128	+128	+128
;;	4	...
;;	...
;;	12	...
;;	32	...
;;	...
;;	1	...
;;	...
;;	512	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+128	+128	+128	4	+128	+128	+128
;;	8	...
;;	...
;;	1	...
;;	...
;;	512	...
;;	...

;; Do 1600 four_complex_fft operations
;;	distance between fft data elements is 4
;;	do 400 macros each processing 32 data values

;; Do FFT levels 11,12 as well as inverse FFT levels 11,12
;; On input and output the 64-byte cache lines hold these data values:
;;	0	+128	+128	+128	4	+128	+128	+128
;;	8	...
;;	...
;;	1	...
;;	...
;;	512	...
;;	...

;; Do 1600 four_complex_fft operations
;;	distance between fft data elements is 1
;;	do 400 macros each processing 32 data values

;; Do inverse FFT levels 9,10
;; On input the 64-byte cache lines hold these data values:
;;	0	+128	+128	+128	4	+128	+128	+128
;;	8	...
;;	...
;;	1	...
;;	...
;;	512	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+128	+128	+128	16	+128	+128	+128
;;	4	...
;;	...
;;	12	...
;;	32	...
;;	...
;;	1	...
;;	...
;;	512	...
;;	...

;; Do 1600 four_complex_unfft operations
;;	distance between fft data elements is 4
;;	do 400 macros each processing 32 data values

;; Do inverse FFT levels 7,8
;; On input the 64-byte cache lines hold these data values:
;;	0	+128	+128	+128	16	+128	+128	+128
;;	4	...
;;	...
;;	12	...
;;	32	...
;;	...
;;	1	...
;;	...
;;	512	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+128	+128	+128	64	+128	+128	+128
;;	4	...
;;	...
;;	60	...
;;	1	...
;;	...
;;	512	...
;;	...

;; Do 1600 four_complex_unfft operations
;;	distance between fft data elements is 16
;;	do 400 macros each processing 32 data values

;; Do inverse FFT levels 5,6
;; On input the 64-byte cache lines hold these data values:
;;	0	+128	+128	+128	64	+128	+128	+128
;;	4	...
;;	...
;;	60	...
;;	1	...
;;	...
;;	512	...
;;	...
;; On output the 64-byte cache lines are swizzled to hold these data values:
;;	0	+1	+1	+1	256	+1	+1	+1
;;	4	...
;;	...
;;	252	...
;;	512	...
;;	...

;; Do 1600 four_complex_unfft operations
;;	distance between fft data elements is 64
;;	do 400 macros each processing 32 data values

;; Do this 64 cache lines (4KB) at a time to maximize accessing data
;; while it is in the L1 cache.

;; Execute the right middle step

	cmp	ffttype, 2
	jl	ypass2_6400_complex_1
	jg	ypass2_6400_complex_3

ypass2_6400_complex_2:
	yloop_init 400			;; 25 iters of 64 cache lines
	yloop_set_prefetch YHALF

b3b0:	start_timer 6
	yr4_s4cl_four_complex_djbfft_preload
b3b:	yr4_s4cl_four_complex_djbfft rsi, 64, 16*64, 32*64, rdi, YMM_SCD2, 16
	yloop	16, b3b, -16*64, rdi, -16*YMM_SCD2 ;; Loop 16 times, restore source pointer, restore sine/cosine pointer
	loops_undo 16			;; Back up loop counter
	end_timer 6

	start_timer 7
	yr4_eb4cl_four_complex_djbfft_preload
b4b:	yr4_eb4cl_four_complex_djbfft rsi, 64, 4*64, 8*64, rdi, 4*YMM_SCD2, 4
	yloop	4, b4b, -4*64+16*64, rdi, -4*4*YMM_SCD2+YMM_SCD2 ;; Loop 4 times, next source pointer, next sine/cosine pointer
	yloop	4, b4b, -4*16*64, rdi, -4*YMM_SCD2 ;; Loop 4 times, restore source pointer, restore sine/cosine pointer
	loops_undo 4*4			;; Back up loop counter
	end_timer 7

	start_timer 8
;;	yr4_eb4cl_four_complex_djbfft_preload
b5b:	yr4_eb4cl_four_complex_djbfft rsi, 4*64, 64, 2*64, rdi, 0, 4
	yloop	4, b5b, 0, rdi, 4*YMM_SCD2 ;; Loop 4 times, next sine/cosine pointer
	yloop	4, b5b, -16*4*64, rdi, -4*4*YMM_SCD2 ;; Loop 4 times, restore source pointer, restore sine/cosine pointer
	loops_undo 4*4			;; Back up loop counter
	end_timer 8

	start_timer 9
	yr4_4cl_four_complex_with_square_preload
b6b:	yr4_4cl_four_complex_with_square rsi, 64, 16*64, 32*64, 16
	yloop	16, b6b, -16*64	;; Loop 16 times, restore source pointer
	loops_undo 16			;; Back up loop counter
	end_timer 9

	start_timer 10
	yr4_eb4cl_four_complex_djbunfft_preload
c5b:	yr4_eb4cl_four_complex_djbunfft rsi, 4*64, 64, 2*64, rdi, 0, 4
	yloop	4, c5b, 0, rdi, 4*YMM_SCD2 ;; Loop 4 times, next sine/cosine pointer
	yloop	4, c5b, -16*4*64, rdi, -4*4*YMM_SCD2 ;; Loop 4 times, restore source pointer, restore sine/cosine pointer
	loops_undo 4*4			;; Restore loop counter
	end_timer 10

	start_timer 11
;;	yr4_eb4cl_four_complex_djbunfft_preload
c4b:	yr4_eb4cl_four_complex_djbunfft rsi, 64, 4*64, 8*64, rdi, 4*YMM_SCD2, 4
	yloop	4, c4b, -4*64+16*64, rdi, -4*4*YMM_SCD2+YMM_SCD2 ;; Loop 4 times, next source pointer, next sine/cosine pointer
	yloop	4, c4b, -4*16*64, rdi, -4*YMM_SCD2 ;; Loop 4 times, restore source pointer, restore sine/cosine pointer
	loops_undo 4*4			;; Back up loop counter
	end_timer 11

	start_timer 12
	yr4_s4cl_four_complex_djbunfft_preload
c3b:	yr4_s4cl_four_complex_djbunfft rsi, 64, 16*64, 32*64, rdi, YMM_SCD2, 16, L1PREFETCH_ALL, dist64
	yloop	16, c3b, -16*64+dist64, rdi, -16*YMM_SCD2 ;; Loop 16 times, next source pointer, restore sine/cosine pointer
	end_timer 12

	yloop	25, b3b0, -25*dist64	;; Loop 25 times, restore source pointer
	jmp	ypass2_6400_complex_unfft


ypass2_6400_complex_1:
	start_timer 6
	bump	rcx, (1600+24-2*64-4*400)*64 ;; Move prefetch pointer along
	yloop_init 400			;; 25 iters of 64 cache lines
	yloop_set_prefetch YSINGLE

b3a0:	yr4_s4cl_four_complex_djbfft_preload
b3a:	yr4_s4cl_four_complex_djbfft rsi, 64, 16*64, 32*64, rdi, YMM_SCD2, 16
	yloop	16, b3a, -16*64, rdi, -16*YMM_SCD2 ;; Loop 16 times, restore source pointer, restore sine/cosine pointer
	loops_undo 16			;; Back up loop counter

	yr4_eb4cl_four_complex_djbfft_preload
b4a:	yr4_eb4cl_four_complex_djbfft rsi, 64, 4*64, 8*64, rdi, 4*YMM_SCD2, 4
	yloop	4, b4a, -4*64+16*64, rdi, -4*4*YMM_SCD2+YMM_SCD2 ;; Loop 4 times, next source pointer, next sine/cosine pointer
	yloop	4, b4a, -4*16*64, rdi, -4*YMM_SCD2 ;; Loop 4 times, restore source pointer, restore sine/cosine pointer
	loops_undo 4*4			;; Back up loop counter

;;	yr4_eb4cl_four_complex_djbfft_preload
b5a:	yr4_eb4cl_four_complex_djbfft rsi, 4*64, 64, 2*64, rdi, 0, 4, L1PREFETCH_ALL, dist64
	yloop	4, b5a, 0, rdi, 4*YMM_SCD2 ;; Loop 4 times, next sine/cosine pointer
	yloop	4, b5a, -16*4*64, rdi, -4*4*YMM_SCD2 ;; Loop 4 times, restore source pointer, restore sine/cosine pointer
	loops_undo 4*4			;; Back up loop counter

	yr4_4cl_four_complex_fft_final_preload
b6a:	yr4_4cl_four_complex_fft_final rsi, 64, 16*64, 32*64
	yloop	16, b6a, -16*64+dist64	;; Loop 16 times, next source pointer

	yloop	25, b3a0		;; Loop 25 times
	end_timer 6
	jmp	ypass2_6400_complex_done


ypass2_6400_complex_3:
	start_timer 6
x64	mov	r8, DEST2ARG
	mov	rbp, DIST_TO_MULSRCARG
	yloop_init 400			;; 25 iters of 64 cache lines
	yloop_set_prefetch YHALF

b3c0:	yr4_s4cl_four_complex_djbfft_preload
b3c:	yr4_s4cl_four_complex_djbfft rsi, 64, 16*64, 32*64, rdi, YMM_SCD2, 16
	yloop	16, b3c, -16*64, rdi, -16*YMM_SCD2 ;; Loop 16 times, restore source pointer, restore sine/cosine pointer
	loops_undo 16			;; Back up loop counter

	yr4_eb4cl_four_complex_djbfft_preload
b4c:	yr4_eb4cl_four_complex_djbfft rsi, 64, 4*64, 8*64, rdi, 4*YMM_SCD2, 4
	yloop	4, b4c, -4*64+16*64, rdi, -4*4*YMM_SCD2+YMM_SCD2 ;; Loop 4 times, next source pointer, next sine/cosine pointer
	yloop	4, b4c, -4*16*64, rdi, -4*YMM_SCD2 ;; Loop 4 times, restore source pointer, restore sine/cosine pointer
	loops_undo 4*4			;; Back up loop counter

;;	yr4_eb4cl_four_complex_djbfft_preload
b5c:	yr4_eb4cl_four_complex_djbfft rsi, 4*64, 64, 2*64, rdi, 0, 4
	yloop	4, b5c, 0, rdi, 4*YMM_SCD2 ;; Loop 4 times, next sine/cosine pointer
	yloop	4, b5c, -16*4*64, rdi, -4*4*YMM_SCD2 ;; Loop 4 times, restore source pointer, restore sine/cosine pointer
	loops_undo 4*4			;; Back up loop counter

	yr4_4cl_four_complex_with_mult_preload
b6c:	yr4_4cl_four_complex_with_mult rsi, 64, 16*64, 32*64
	yloop	16, b6c, -16*64		;; Loop 16 times, restore source pointer
x64	add	rsi, r8
	loops_undo 16			;; Back up loop counter

	yr4_eb4cl_four_complex_djbunfft_preload
c5c:	yr4_eb4cl_four_complex_djbunfft rsi, 4*64, 64, 2*64, rdi, 0, 4
	yloop	4, c5c, 0, rdi, 4*YMM_SCD2 ;; Loop 4 times, next sine/cosine pointer
	yloop	4, c5c, -16*4*64, rdi, -4*4*YMM_SCD2 ;; Loop 4 times, restore source pointer, restore sine/cosine pointer
	loops_undo 4*4			;; Restore loop counter

;;	yr4_eb4cl_four_complex_djbunfft_preload
c4c:	yr4_eb4cl_four_complex_djbunfft rsi, 64, 4*64, 8*64, rdi, 4*YMM_SCD2, 4
	yloop	4, c4c, -4*64+16*64, rdi, -4*4*YMM_SCD2+YMM_SCD2 ;; Loop 4 times, next source pointer, next sine/cosine pointer
	yloop	4, c4c, -4*16*64, rdi, -4*YMM_SCD2 ;; Loop 4 times, restore source pointer, restore sine/cosine pointer
	loops_undo 4*4			;; Back up loop counter

	yr4_s4cl_four_complex_djbunfft_preload
c3c:	yr4_s4cl_four_complex_djbunfft rsi, 64, 16*64, 32*64, rdi, YMM_SCD2, 16, L1PREFETCH_ALL, dist64
	yloop	16, c3c, -16*64+dist64, rdi, -16*YMM_SCD2 ;; Loop 16 times, next source pointer, restore sine/cosine pointer

x64	sub	rsi, r8
	yloop	25, b3c0, -25*dist64	;; Loop 25 times, restore source pointer
x64	add	rsi, r8
	end_timer 6
	jmp	ypass2_6400_complex_unfft


ypass2_6400_complex_4:
	start_timer 6
	mov	rdi, xsincos_complex	;; The sin/cos ptrs
	bump	rdi, 320*YMM_SCD2+64*YMM_SCD2
	mov	rbp, DIST_TO_MULSRCARG
	yloop_init 400			;; 25 iters of 64 cache lines
	yloop_set_prefetch YSINGLE

	yr4_4cl_four_complex_with_mulf_preload
b6d:	yr4_4cl_four_complex_with_mulf rsi, 64, 16*64, 32*64
	yloop	16, b6d, -16*64		;; Loop 16 times, restore source pointer
	loops_undo 16			;; Back up loop counter

	yr4_eb4cl_four_complex_djbunfft_preload
c5d:	yr4_eb4cl_four_complex_djbunfft rsi, 4*64, 64, 2*64, rdi, 0, 4
	yloop	4, c5d, 0, rdi, 4*YMM_SCD2 ;; Loop 4 times, next sine/cosine pointer
	yloop	4, c5d, -16*4*64, rdi, -4*4*YMM_SCD2 ;; Loop 4 times, restore source pointer, restore sine/cosine pointer
	loops_undo 4*4			;; Back up loop counter

;;	yr4_eb4cl_four_complex_djbunfft_preload
c4d:	yr4_eb4cl_four_complex_djbunfft rsi, 64, 4*64, 8*64, rdi, 4*YMM_SCD2, 4
	yloop	4, c4d, -4*64+16*64, rdi, -4*4*YMM_SCD2+YMM_SCD2 ;; Loop 4 times, next source pointer, next sine/cosine pointer
	yloop	4, c4d, -4*16*64, rdi, -4*YMM_SCD2 ;; Loop 4 times, restore source pointer, restore sine/cosine pointer
	loops_undo 4*4			;; Back up loop counter

	yr4_s4cl_four_complex_djbunfft_preload
c3d:	yr4_s4cl_four_complex_djbunfft rsi, 64, 16*64, 32*64, rdi, YMM_SCD2, 16, L1PREFETCH_ALL, dist64
	yloop	16, c3d, -16*64+dist64, rdi, -16*YMM_SCD2 ;; Loop 16 times, next source pointer, restore sine/cosine pointer

	yloop	25, b6d, -25*dist64	;; Loop 25 times, restore source pointer
	bump	rcx, (-4*400+2*64+7*200)*64 ;; Move prefetch pointer to where squaring would be
	end_timer 6

;; Do inverse FFT levels 3,4
;; On input the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	256	+1	+1	+1
;;	4	...
;;	...
;;	252	...
;;	512	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	1280	+1	+1	+1
;;	4	...
;;	...
;;	1276	...
;;	2560	...
;;	...

;; Do 1280 five_complex_unfft operations
;;	distance between fft data elements is 256
;;	do 320 macros each processing 40 data values

;; Do inverse FFT levels 1,2
;; On input the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	1280	+1	+1	+1
;;	4	...
;;	...
;;	1276	...
;;	2560	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	6400	+1	+1	+1
;;	4	...
;;	...
;;	6396

;; Do 1280 five_complex_unfft operations
;;	distance between fft data elements is 1280
;;	do 320 macros each processing 40 data values

ypass2_6400_complex_unfft:
	mov	rdi, xsincos_complex	;; Load sin/cos pointers
	lea	rbp, [rdi+320*YMM_SCD2]
	bump	rcx, (1600+24-2*64-7*200-64-32)*64 ;; Move prefetch pointer along
	yloop_init 512			;; 320 iterations

	yr5_5cl_five_complex_djbunfft_preload
c2a:	start_timer 13
	yloop_adjust 3			;; Three fewer iterations this loop
	yloop_set_prefetch YOTHER, 8	;; Prefetch 64 cache lines in this loop
c2b:	yr5_5cl_five_complex_djbunfft rsi, 5*dist64, dist64, rbp, 0, 5
	yloop	8, c2b, -5*5*dist64, rbp, YMM_SCD2 ;; 5 iterations!!  Restore source pointer, next sine/cosine pointer
	loops_undo 8			;; Restore loop counter
	end_timer 13

	start_timer 14
	yloop_adjust 3			;; Three fewer iterations this loop
	yloop_set_prefetch YOTHER, 16	;; Prefetch 32 cache lines in this loop
;;	yr5_5cl_five_complex_djbunfft_preload
c1b:	yr5_5cl_five_complex_djbunfft rsi, dist64, 5*dist64, rdi, 64*YMM_SCD2, 5, L1PREFETCH_ALL, 64
	yloop	8, c1b, -5*dist64+64, rdi, -5*64*YMM_SCD2+YMM_SCD2 ;; 5 iterations!!  Next source pointer, next sine/cosine pointer
	end_timer 14

	yloop	64, c2a			;; Loop 64 times

ypass2_6400_complex_done:
	ENDM


;*********************************************************************************************
;			Macros derived from pass2-13-levels
;*********************************************************************************************


; *************** radix4-pass2-4608-real macro ******************
; This macro takes 9216 real values and performs the final 13.17 levels of the FFT.
; The results are squared, and the first 13.17 levels of the inverse FFT are done.

yr4dwpn_pass2_4608_real MACRO
	LOCAL	b1b, b2a, b2b, b3a, b3b, b4a, b4b, b5a, b5b, b6a, b6b, b6c, b6d
	LOCAL	c1b, c2a, c2b, c3a, c3b, c4a, c4b, c5a, c5b
	LOCAL	ypass2_4608_real_1, ypass2_4608_real_2
	LOCAL	ypass2_4608_real_3, ypass2_4608_real_4
	LOCAL	ypass2_4608_real_unfft, ypass2_4608_real_done

;; Type 4 FFTs skip the forward FFT process

	start_timer 2
	mov	rsi, DATA_ADDR		;; Load source address
	mov	rbx, DIST_TO_FFTSRCARG
	mov	rcx, DATA_PREFETCH	;; Prefetch pointer (we need to prefetch 
					;; 1152 cache lines + 2*17 pad cache lines)
	cmp	ffttype, 4
	je	ypass2_4608_real_4

;; Do FFT levels 1,2,3 which correspond to complex levels 1,2
;; On input the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	4608	+1	+1	+1
;;	4	...
;;	...
;;	4604
;; On output the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	1536	+1	+1	+1
;;	4	...
;;	...
;;	1532	...
;;	3072	...
;;	...

;; Do 1536 six_reals_fft operations
;;	distance between fft data elements is 1536
;;	do 384 macros each processing 24 data values

	mov	rdi, sincos3		;; Real sin/cos pointer
	mov	rbp, xsincos_complex	;; Complex sin/cos data
	yloop_init 384			;; 384 iterations
	yloop_set_prefetch YQUARTER
	yr3_f3cl_2sc_six_reals_fft_preload
b1b:	yr3_f3cl_2sc_six_reals_fft rsi, 64, 6*dist64, rbp, YMM_SCD1, rdi, YMM_SCD1, 64
	yloop	64, b1b, -64*64+dist64	;; Test loop counter, next source pointer
	yloop	6, b1b, -6*dist64	;; Test loop counter, restore source pointer

;; Do FFT levels 3,4
;; On input the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	1536	+1	+1	+1
;;	4	...
;;	...
;;	1532	...
;;	3072	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	512	+1	+1	+1
;;	4	...
;;	...
;;	508	...
;;	1024	...
;;	...

;; Do 512 six_reals_fft operations
;; Do 1024 three_complex_fft operations
;;	distance between fft data elements is 512
;;	do 384 macros each processing 24 data values

	yloop_init 384			;; 3 iters of 128
	yloop_set_prefetch YQUARTER
	yr3_3cl_2sc_six_reals_fft_preload
b2a:	yr3_3cl_2sc_six_reals_fft rsi, 64, 2*dist64, rbp, YMM_SCD1, rdi, YMM_SCD1, 64
	yloop	64, b2a, -64*64+dist64	;; Test loop counter, next source pointer
	yloop	2, b2a, -2*dist64+6*dist64, rbp, -128*YMM_SCD1 ;; Test loop counter, next source pointer, restore sine/cosine pointer

	loops_reset			;; 2 more iters of 128
	yr3_3cl_three_complex_djbfft_preload
b2b:	yr3_3cl_three_complex_djbfft rsi, 64, 2*dist64, rbp, YMM_SCD1, 64
	yloop	64, b2b, -64*64+dist64	;; Test loop counter, next source pointer
	yloop	2, b2b, -2*dist64+6*dist64, rbp, -128*YMM_SCD1 ;; Test loop counter, next source pointer, restore sin/cos ptr
	yloop	3, b2b, -3*6*dist64, rbp, 128*YMM_SCD1 ;; Loop 3 times, restore source pointer, next sin/cos pointer

;; Do FFT levels 5,6
;; On input the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	512	+1	+1	+1
;;	4	...
;;	...
;;	508	...
;;	1024	...
;;	...
;; On output the 64-byte cache lines are swizzled to hold these data values:
;;	0	+256	+256	+256	128	+256	+256	+256
;;	4	...
;;	...
;;	124	...
;;	1	...
;;	...
;;	1024	...
;;	...

;; Do 128 eight_reals_fft operations
;; Do 1024 four_complex_fft operations
;;	distance between fft data elements is 128
;;	do 288 macros each processing 32 data values

	yloop_init 288			;; 9 iterations of 32
	yloop_set_prefetch YHALF
	yr4_s4cl_2sc_eight_reals_fft_preload
b3a:	yr4_s4cl_2sc_eight_reals_fft rsi, 64, 32*64, dist64, rbp, YMM_SCD2, rdi, YMM_SCD2, 32
	yloop	32, b3a, -32*64+2*dist64, rbp, -32*YMM_SCD2 ;; Test loop counter, next source pointer, restore sine/cosine pointer

	loops_reset			;; 8 more iters of 32
	yr4_s4cl_four_complex_djbfft_preload
b3b:	yr4_s4cl_four_complex_djbfft rsi, 64, 32*64, dist64, rbp, YMM_SCD2, 32
	yloop	32, b3b, -32*64+2*dist64, rbp, -32*YMM_SCD2 ;; Test loop counter, next source pointer, restore sin/cos ptr
	yloop	9, b3b, -9*2*dist64	;; Loop 9 times, restore source pointer

;; Do FFT levels 7,8
;; On input the 64-byte cache lines hold these data values:
;;	0	+256	+256	+256	128	+256	+256	+256
;;	4	...
;;	...
;;	124	...
;;	1	...
;;	...
;;	1024	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+256	+256	+256	32	+256	+256	+256
;;	4	...
;;	...
;;	28	...
;;	64	...
;;	...
;;	1	...
;;	...
;;	1024	...
;;	...

;; Do 32 eight_reals_fft operations
;; Do 1120 four_complex_fft operations
;;	distance between fft data elements is 32
;;	do 288 macros each processing 32 data values

	yloop_init 288			;; 9 iterations of 32
	yloop_set_prefetch YHALF
	yr4_4cl_eight_reals_four_complex_djbfft_preload
b4a:	yr4_4cl_eight_reals_four_complex_djbfft rsi, 32*64, 8*64, 16*64, rdi, YMM_SCD3, 2
	yloop	2, b4a, -2*32*64+dist64	;; Test loop counter, next source pointer
	yloop	2, b4a, -2*dist64+64	;; Test loop counter, next source pointer
	yloop	8, b4a, -8*64+2*dist64, rdi, -32*YMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer

	loops_reset			;; 8 more iters of 32
	yr4_rb4cl_four_complex_djbfft_preload
b4b:	yr4_rb4cl_four_complex_djbfft rsi, 32*64, 8*64, 16*64, rdi, YMM_SCD3, 2
	yloop	2, b4b, -2*32*64+dist64	;; Test loop counter, next source pointer
	yloop	2, b4b, -2*dist64+64	;; Test loop counter, next source pointer
	yloop	8, b4b, -8*64+2*dist64, rdi, -32*YMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer
	yloop	9, b4b, -9*2*dist64	;; Test loop counter, restore source pointer

;; Do FFT levels 9,10
;; On input the 64-byte cache lines hold these data values:
;;	0	+256	+256	+256	32	+256	+256	+256
;;	4	...
;;	...
;;	28	...
;;	64	...
;;	...
;;	1	...
;;	...
;;	1024	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+256	+256	+256	8	+256	+256	+256
;;	4	...
;;	16	...
;;	...
;;	1	...
;;	...
;;	1024	...
;;	...

;; Do 8 eight_reals_fft operations
;; Do 1144 four_complex_fft operations
;;	distance between fft data elements is 8
;;	do 288 macros each processing 32 data values

	yloop_init 288			;; 36 iterations of 8
	yloop_set_prefetch YHALF
	yr4_4cl_eight_reals_four_complex_djbfft_preload
b5a:	yr4_4cl_eight_reals_four_complex_djbfft rsi, 32*64, 2*64, 4*64, rdi, 4*YMM_SCD3, 2
	yloop	2, b5a, -2*32*64+dist64	;; Test loop counter, next source pointer
	yloop	2, b5a, -2*dist64+64	;; Test loop counter, next source pointer
	yloop	2, b5a, -2*64+8*64, rdi, -8*4*YMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer

	loops_reset			;; 35 more iters of 8
	yr4_rb4cl_four_complex_djbfft_preload
b5b:	yr4_rb4cl_four_complex_djbfft rsi, 32*64, 2*64, 4*64, rdi, 4*YMM_SCD3, 2
	yloop	2, b5b, -2*32*64+dist64	;; Test loop counter, next source pointer
	yloop	2, b5b, -2*dist64+64	;; Test loop counter, next source pointer
	yloop	2, b5b, -2*64+8*64, rdi, -8*4*YMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer
	yloop	4, b5b, -4*8*64+2*dist64 ;; Test loop counter, next source pointer
	yloop	9, b5b, -9*2*dist64	;; Test loop counter, restore source pointer

;; Do FFT levels 11,12,13 as well as inverse FFT levels 11,12,13
;; On input and output the 64-byte cache lines hold these data values:
;;	0	+256	+256	+256	8	+256	+256	+256
;;	4	...
;;	16	...
;;	...
;;	1	...
;;	...
;;	1024	...
;;	...

;; Do 1 sixteen_reals_fft operation
;; Do 575 eight_complex_fft operations
;;	distance between fft data elements is 1
;;	do 144 macros each processing 64 data values

	cmp	ffttype, 2
	jl	ypass2_4608_real_1
	jg	ypass2_4608_real_3

ypass2_4608_real_2:
;;	yr8_8cl_sixteen_reals_eight_complex_with_square rsi, 2*64, 32*64, dist64, 64
	yr8_dispatch call, y8real_square, 32*64, dist64, 64
	yloop_init 144			;; 143 more iterations
	yloop_adjust 1
	yloop_set_prefetch YSINGLE
	yr8_8cl_eight_complex_with_square_preload
b6b:	yr8_8cl_eight_complex_with_square rsi, 2*64, 32*64, dist64, 64, 1
	yloop	16, b6b, -16*2*64+2*dist64	;; Test loop counter, next source pointer
	yloop	9, b6b, -9*2*dist64	;; Test loop counter, restore source pointer
	jmp	ypass2_4608_real_unfft

ypass2_4608_real_1:
;;	yr8_8cl_sixteen_reals_eight_complex_fft_final rsi, 2*64, 32*64, dist64, 64
	yr8_dispatch call, y8real_fft_final, 32*64, dist64, 64
	mov	rcx, xsincos_complex	;; Prefetch complex sin/cos data, 576 cache lines
	yloop_init 144			;; 143 more iterations
	yloop_adjust 1
	yloop_set_prefetch YDOUBLE
b6a:	yr8_8cl_eight_complex_fft_final rsi, 2*64, 32*64, dist64, 64
	yloop	16, b6a, -16*2*64+2*dist64	;; Test loop counter, next source pointer
	yloop	9, b6a			;; Test loop counter
	jmp	ypass2_4608_real_done

ypass2_4608_real_3:
x64	mov	r8, DEST2ARG
	mov	rbp, DIST_TO_MULSRCARG
;;	yr8_8cl_sixteen_reals_eight_complex_with_mult rsi, 2*64, 32*64, dist64, 64
	yr8_dispatch call, y8real_mult, 32*64, dist64, 64
	yloop_init 144			;; 143 more iterations
	yloop_adjust 1
	yloop_set_prefetch YSINGLE
	yr8_8cl_eight_complex_with_mult_preload
b6c:	yr8_8cl_eight_complex_with_mult rsi, 2*64, 32*64, dist64, 64
	yloop	16, b6c, -16*2*64+2*dist64	;; Test loop counter, next source pointer
	yloop	9, b6c, -9*2*dist64	;; Test loop counter, restore source pointer
x64	add	rsi, r8
	jmp	ypass2_4608_real_unfft

ypass2_4608_real_4:
	mov	rbp, DIST_TO_MULSRCARG
;;	yr8_8cl_sixteen_reals_eight_complex_with_mulf rsi, 2*64, 32*64, dist64, 64
	yr8_dispatch call, y8real_mulf, 32*64, dist64, 64
	yloop_init 144			;; 143 more iterations
	yloop_adjust 1
	yloop_set_prefetch YDOUBLE
	yr8_8cl_eight_complex_with_mulf_preload
b6d:	yr8_8cl_eight_complex_with_mulf rsi, 2*64, 32*64, dist64, 64
	yloop	16, b6d, -16*2*64+2*dist64	;; Test loop counter, next source pointer
	yloop	9, b6d, -9*2*dist64	;; Test loop counter, restore source pointer
;;	bump	rcx, (-288+2*96+3*144+143)*64 ;; Move prefetch pointer back to where squaring would be

;; Do inverse FFT levels 9,10
;; On input the 64-byte cache lines hold these data values:
;;	0	+256	+256	+256	8	+256	+256	+256
;;	4	...
;;	16	...
;;	...
;;	1	...
;;	...
;;	1024	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+256	+256	+256	32	+256	+256	+256
;;	4	...
;;	...
;;	28	...
;;	64	...
;;	...
;;	1	...
;;	...
;;	1024	...
;;	...

;; Do 8 eight_reals_unfft operations
;; Do 1144 four_complex_unfft operations
;;	distance between fft data elements is 8
;;	do 288 macros each processing 32 data values

ypass2_4608_real_unfft:
	mov	rdi, sincos3		;; Load sin/cos pointer
	bump	rdi, 32*YMM_SCD2+128*YMM_SCD1+384*YMM_SCD1
	yloop_init 288			;; 36 iterations of 8
	yloop_set_prefetch YHALF
	yr4_4cl_eight_reals_four_complex_djbunfft_preload
c5a:	yr4_4cl_eight_reals_four_complex_djbunfft rsi, 32*64, 2*64, 4*64, rdi, 4*YMM_SCD3, 2
	yloop	2, c5a, -2*32*64+dist64	;; Test loop counter, next source pointer
	yloop	2, c5a, -2*dist64+64	;; Test loop counter, next source pointer
	yloop	2, c5a, -2*64+8*64, rdi, -8*4*YMM_SCD3 ;; Test loop counter, next source pointer, restore sine/cosine pointer

	loops_reset			;; 35 more iters of 8
	yr4_rb4cl_four_complex_djbunfft_preload
c5b:	yr4_rb4cl_four_complex_djbunfft rsi, 32*64, 2*64, 4*64, rdi, 4*YMM_SCD3, 2
	yloop	2, c5b, -2*32*64+dist64	;; Test loop counter, next source pointer
	yloop	2, c5b, -2*dist64+64	;; Test loop counter, next source pointer
	yloop	2, c5b, -2*64+8*64, rdi, -8*4*YMM_SCD3 ;; Test loop counter, next source pointer, restore sine/cosine pointer
	yloop	4, c5b, -4*8*64+2*dist64	;; Test loop counter, next source pointer
	yloop	9, c5b, -9*2*dist64	;; Test loop counter, restore source pointer

;; Do inverse FFT levels 7,8
;; On input the 64-byte cache lines hold these data values:
;;	0	+256	+256	+256	32	+256	+256	+256
;;	4	...
;;	...
;;	28	...
;;	64	...
;;	...
;;	1	...
;;	...
;;	1024	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+256	+256	+256	128	+256	+256	+256
;;	4	...
;;	...
;;	124	...
;;	1	...
;;	...
;;	1024	...
;;	...

;; Do 32 eight_reals_unfft operations
;; Do 1120 four_complex_unfft operations
;;	distance between fft data elements is 32
;;	do 288 macros each processing 32 data values

	yloop_init 288			;; 9 iterations of 32
	yloop_set_prefetch YHALF
	yr4_4cl_eight_reals_four_complex_djbunfft_preload
c4a:	yr4_4cl_eight_reals_four_complex_djbunfft rsi, 32*64, 8*64, 16*64, rdi, YMM_SCD3, 2
	yloop	2, c4a, -2*32*64+dist64	;; Test loop counter, next source pointer
	yloop	2, c4a, -2*dist64+64	;; Test loop counter, next source pointer
	yloop	8, c4a, -8*64+2*dist64, rdi, -32*YMM_SCD3 ;; Test loop counter, next source pointer, restore sine/cosine pointer

	loops_reset			;; 8 more iters of 32
	yr4_rb4cl_four_complex_djbunfft_preload
c4b:	yr4_rb4cl_four_complex_djbunfft rsi, 32*64, 8*64, 16*64, rdi, YMM_SCD3, 2
	yloop	2, c4b, -2*32*64+dist64	;; Test loop counter, next source pointer
	yloop	2, c4b, -2*dist64+64	;; Test loop counter, next source pointer
	yloop	8, c4b, -8*64+2*dist64, rdi, -32*YMM_SCD3 ;; Test loop counter, next source pointer, restore sine/cosine pointer
	yloop	9, c4b, -9*2*dist64	;; Test loop counter, restore source pointer

;; Do inverse FFT levels 5,6
;; On input the 64-byte cache lines hold these data values:
;;	0	+256	+256	+256	128	+256	+256	+256
;;	4	...
;;	...
;;	124	...
;;	1	...
;;	...
;;	1024	...
;;	...
;; On output the 64-byte cache lines are swizzled to hold these data values:
;;	0	+1	+1	+1	512	+1	+1	+1
;;	4	...
;;	...
;;	508	...
;;	1024	...
;;	...

;; Do 128 eight_reals_unfft operations
;; Do 1024 four_complex_unfft operations
;;	distance between fft data elements is 128
;;	do 288 macros each processing 32 data values

	bump	rdi, -32*YMM_SCD2	;; Real sin/cos ptr
	mov	rbp, xsincos_complex	;; Complex sin/cos data
	bump	rbp, 128*YMM_SCD1+384*YMM_SCD1
	bump	rcx, (1152+2*17-2*96-3*144-143-2*144-144)*64 ;; Move prefetch pointer along
	yloop_init 288			;; 9 iters of 32
	yloop_set_prefetch YHALF
	yr4_s4cl_2sc_eight_reals_unfft_preload
c3a:	yr4_s4cl_2sc_eight_reals_unfft rsi, 64, 32*64, dist64, rbp, YMM_SCD2, rdi, YMM_SCD2, 32
	yloop	32, c3a, -32*64+2*dist64, rbp, -32*YMM_SCD2 ;; Test loop counter, next source pointer, restore sine/cosine pointer

	loops_reset			;; 8 more iters of 32
	yr4_s4cl_four_complex_djbunfft_preload
c3b:	yr4_s4cl_four_complex_djbunfft rsi, 64, 32*64, dist64, rbp, YMM_SCD2, 32
	yloop	32, c3b, -32*64+2*dist64, rbp, -32*YMM_SCD2 ;; Test loop counter, next source pointer, restore sine/cosine pointer
	yloop	9, c3b, -9*2*dist64	;; Test loop counter, restore source pointer

;; Do inverse FFT levels 3,4
;; On output the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	512	+1	+1	+1
;;	4	...
;;	...
;;	508	...
;;	1024	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	1536	+1	+1	+1
;;	4	...
;;	...
;;	1532	...
;;	3072	...
;;	...

;; Do 512 six_reals_unfft operations
;; Do 1024 three_complex_unfft operations
;;	distance between fft data elements is 512
;;	do 384 macros each processing 24 data values

	bump	rdi, -32*YMM_SCD2-128*YMM_SCD1 ;; Real sin/cos ptr
	bump	rbp, -128*YMM_SCD1	;; Complex sin/cos data
	mov	rcx, xsincos_complex	;; Prefetch complex sin/cos data, 576 cache lines
	yloop_init 384			;; 3 iters of 128
	yloop_set_prefetch YSINGLE
	yr3_3cl_2sc_six_reals_unfft_preload
c2a:	yr3_3cl_2sc_six_reals_unfft rsi, 64, 2*dist64, rbp, YMM_SCD1, rdi, YMM_SCD1, 64
	yloop	64, c2a, -64*64+dist64	;; Test loop counter, next source pointer
	yloop	2, c2a, -2*dist64+6*dist64, rbp, -128*YMM_SCD1 ;; Test loop counter, next source pointer, restore sine/cosine pointer

	loops_reset			;; 2 more iters of 128
	yr3_3cl_three_complex_djbunfft_preload
c2b:	yr3_3cl_three_complex_djbunfft rsi, 64, 2*dist64, rbp, YMM_SCD1, 64
	yloop	64, c2b, -64*64+dist64	;; Test loop counter, next source pointer
	yloop	2, c2b, -2*dist64+6*dist64, rbp, -128*YMM_SCD1 ;; Test loop counter, next source pointer, restore sine/cosine pointer
	yloop	3, c2b, -3*6*dist64	;; Test loop counter, restore source pointer

;; Do inverse FFT levels 1,2,3 which correspond to complex levels 1,2
;; On input the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	1536	+1	+1	+1
;;	4	...
;;	...
;;	1532	...
;;	3072	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	4608	+1	+1	+1
;;	4	...
;;	...
;;	4604

;; Do 1536 six_reals_unfft operations
;;	distance between fft data elements is 1536
;;	do 384 macros each processing 24 data values

	bump	rdi, -128*YMM_SCD1-384*YMM_SCD1 ;; Real sin/cos ptr
	bump	rbp, -384*YMM_SCD1	;; Complex sin/cos ptr
	bump	rcx, (576-192-384)*64	;; Move prefetch pointer along
	yloop_init 384			;; 384 iterations
	yloop_set_prefetch YHALF
	yr3_3cl_2sc_six_reals_unfft_preload
c1b:	yr3_3cl_2sc_six_reals_unfft rsi, 64, 6*dist64, rbp, YMM_SCD1, rdi, YMM_SCD1, 64
	yloop	64, c1b, -64*64+dist64	;; Test loop counter, next source pointer
	yloop	6, c1b			;; Test loop counter

ypass2_4608_real_done:
	end_timer 2
	ENDM

; *************** radix4-pass2-4608-complex macro ******************
; This macro takes 4608 complex values and performs the final 12.17 levels of a standard
; radix-4 FFT, squares the results, and does the first 12.17 levels of the inverse FFT.

yr4dwpn_pass2_4608_complex MACRO
	LOCAL	b1a, b1b, b2b, b3a0, b3a, b3b0, b3b, b3c0, b3c, b4a, b4b, b4c, b5a, b5b, b5c, b6a, b6b, b6c, b6d0, b6d
	LOCAL	c1b, c2a, c2b, c3b, c3c, c3d, c4b, c4c, c4d, c5b, c5c, c5d
	LOCAL	ypass2_4608_complex_1, ypass2_4608_complex_2
	LOCAL	ypass2_4608_complex_3, ypass2_4608_complex_4
	LOCAL	ypass2_4608_complex_unfft, ypass2_4608_complex_done

;; Type 4 FFTs skip the forward FFT process

	mov	rsi, DATA_ADDR		;; Load address of FFT data
	mov	rbx, DIST_TO_FFTSRCARG
	mov	rcx, DATA_PREFETCH	;; Prefetch pointer (we need to prefetch 
					;; 1152 cache lines + 2*17 pad cache lines)
	cmp	ffttype, 4
	je	ypass2_4608_complex_4

;; Do FFT level 1,2 (actually 1.585 levels)
;; On input the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	4608	+1	+1	+1
;;	4	...
;;	...
;;	4604
;; On output the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	1536	+1	+1	+1
;;	4	...
;;	...
;;	1532	...
;;	3072	...
;;	...

;; Do 1536 three_complex_fft operations
;;	distance between fft data elements is 1536
;;	do 384 macros each processing 24 data values

;; Do FFT levels 3,4 (actually 1.585 levels)
;; On input the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	1536	+1	+1	+1
;;	4	...
;;	...
;;	1532	...
;;	3072	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	512	+1	+1	+1
;;	4	...
;;	...
;;	508	...
;;	1024	...
;;	...

;; Do 1536 three_complex_fft operations
;;	distance between fft data elements is 512
;;	do 384 macros each processing 24 data values

	mov	rdi, xsincos_complex	;; The sin/cos ptrs
	lea	rbp, [rdi+384*YMM_SCD1]
	yloop_init 512			;; 384 iterations

	yr3_f3cl_three_complex_djbfft_preload
b1a:	start_timer 4
	yloop_adjust 1
	yloop_set_prefetch YQUARTER	;; Prefetch 128 cache lines
b1b:	yr3_f3cl_three_complex_djbfft rsi, 2*dist64, 6*dist64, rdi, 128*YMM_SCD1, 3
	yloop	4, b1b, -3*2*dist64, rdi, -3*128*YMM_SCD1+YMM_SCD1 ;; 3 iterations!!  Restore source pointer, next sin/cos data
	loops_undo 4			;; Back up loop counter
	end_timer 4

	start_timer 5
	yloop_adjust 1
	yloop_set_prefetch YQUARTER	;; Prefetch 128 cache lines
b2b:	yr3_3cl_three_complex_djbfft rsi, 6*dist64, 2*dist64, rbp, 0, 3, L1PREFETCH_ALL, 64
	yloop	4, b2b, -3*6*dist64+64, rbp, YMM_SCD1 ;; 3 iterations!!  Next source pointer, next sin/cos data
	end_timer 5

	yloop	64, b1a, -64*64+dist64	;; Test loop counter, next source pointer
	yloop	2, b1a, -2*dist64, rdi, -128*YMM_SCD1+384*YMM_SCD1+128*YMM_SCD1 ;; Test loop counter, restore source pointer, next sin/cos ptr

;; Do FFT levels 5,6
;; On input the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	512	+1	+1	+1
;;	4	...
;;	...
;;	508	...
;;	1024	...
;;	...
;; On output the 64-byte cache lines are swizzled to hold these data values:
;;	0	+256	+256	+256	128	+256	+256	+256
;;	4	...
;;	...
;;	124	...
;;	1	...
;;	...
;;	1024	...
;;	...

;; Do 1152 four_complex_fft operations
;;	distance between fft data elements is 128
;;	do 288 macros each processing 32 data values

;; Do FFT levels 7,8
;; On input the 64-byte cache lines hold these data values:
;;	0	+256	+256	+256	128	+256	+256	+256
;;	4	...
;;	...
;;	124	...
;;	1	...
;;	...
;;	1024	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+256	+256	+256	32	+256	+256	+256
;;	4	...
;;	...
;;	28	...
;;	64	...
;;	...
;;	1	...
;;	...
;;	1024	...
;;	...

;; Do 1152 four_complex_fft operations
;;	distance between fft data elements is 32
;;	do 288 macros each processing 32 data values

;; Do FFT levels 9,10
;; On input the 64-byte cache lines hold these data values:
;;	0	+256	+256	+256	32	+256	+256	+256
;;	4	...
;;	...
;;	28	...
;;	64	...
;;	...
;;	1	...
;;	...
;;	1024	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+256	+256	+256	8	+256	+256	+256
;;	4	...
;;	16	...
;;	...
;;	1	...
;;	...
;;	1024	...
;;	...

;; Do 1152 four_complex_fft operations
;;	distance between fft data elements is 8
;;	do 288 macros each processing 32 data values

;; Do FFT levels 11,12,13 as well as inverse FFT levels 11,12,13
;; On input and output the 64-byte cache lines hold these data values:
;;	0	+256	+256	+256	8	+256	+256	+256
;;	4	...
;;	16	...
;;	...
;;	1	...
;;	...
;;	1024	...
;;	...

;; Do 576 eight_complex_fft operations
;;	distance between fft data elements is 1
;;	do 144 macros each processing 64 data values

;; Do inverse FFT levels 9,10
;; On input the 64-byte cache lines hold these data values:
;;	0	+256	+256	+256	8	+256	+256	+256
;;	4	...
;;	16	...
;;	...
;;	1	...
;;	...
;;	1024	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+256	+256	+256	32	+256	+256	+256
;;	4	...
;;	...
;;	28	...
;;	64	...
;;	...
;;	1	...
;;	...
;;	1024	...
;;	...

;; Do 1152 four_complex_unfft operations
;;	distance between fft data elements is 8
;;	do 288 macros each processing 32 data values

;; Do inverse FFT levels 7,8
;; On input the 64-byte cache lines hold these data values:
;;	0	+256	+256	+256	32	+256	+256	+256
;;	4	...
;;	...
;;	28	...
;;	64	...
;;	...
;;	1	...
;;	...
;;	1024	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+256	+256	+256	128	+256	+256	+256
;;	4	...
;;	...
;;	124	...
;;	1	...
;;	...
;;	1024	...
;;	...

;; Do 1152 four_complex_unfft operations
;;	distance between fft data elements is 32
;;	do 288 macros each processing 32 data values

;; Do inverse FFT levels 5,6
;; On input the 64-byte cache lines hold these data values:
;;	0	+256	+256	+256	128	+256	+256	+256
;;	4	...
;;	...
;;	124	...
;;	1	...
;;	...
;;	1024	...
;;	...
;; On output the 64-byte cache lines are swizzled to hold these data values:
;;	0	+1	+1	+1	512	+1	+1	+1
;;	4	...
;;	...
;;	508	...
;;	1024	...
;;	...

;; Do 1152 four_complex_unfft operations
;;	distance between fft data elements is 128
;;	do 288 macros each processing 32 data values

;; Do this 128 cache lines (8KB) at a time to maximize accessing data
;; while it is in the L1 cache.

;; Execute the right middle step

	cmp	ffttype, 2
	jl	ypass2_4608_complex_1
	jg	ypass2_4608_complex_3

ypass2_4608_complex_2:
	yloop_init 288			;; 9 iters of 128 cache lines

b3b0:	start_timer 6
	yloop_set_prefetch YHALF	;; Prefetch 144 cache lines in this loop
	yr4_s4cl_four_complex_djbfft_preload
b3b:	yr4_s4cl_four_complex_djbfft rsi, 64, 32*64, dist64, rdi, YMM_SCD2, 32
	yloop	32, b3b, -32*64, rdi, -32*YMM_SCD2 ;; Loop 32 times, restore source pointer, restore sine/cosine pointer
	loops_undo 32			;; Back up loop counter
	end_timer 6

	start_timer 7
	yloop_set_prefetch YQUARTER	;; Prefetch 72 cache lines in this loop
	yr4_eb4cl_four_complex_djbfft_preload
b4b:	yr4_eb4cl_four_complex_djbfft rsi, 64, 8*64, 16*64, rdi, 4*YMM_SCD2, 8
	yloop	8, b4b, -8*64+32*64, rdi, -8*4*YMM_SCD2+YMM_SCD2 ;; Loop 8 times, next source pointer, next sine/cosine pointer
	yloop	2, b4b, -2*32*64+dist64	;; Loop 2 times, next source pointer
	yloop	2, b4b, -2*dist64, rdi, -4*YMM_SCD2 ;; Loop 2 times, restore source pointer, restore sine/cosine pointer
	loops_undo 4*8			;; Back up loop counter
	end_timer 7

	start_timer 8
	yloop_set_prefetch YQUARTER	;; Prefetch 72 cache lines in this loop
;;	yr4_eb4cl_four_complex_djbfft_preload
b5b:	yr4_eb4cl_four_complex_djbfft rsi, 64, 2*64, 4*64, rdi, 16*YMM_SCD2, 2
	yloop	2, b5b, -2*64+8*64, rdi, -2*16*YMM_SCD2 ;; Loop 2 times, next source pointer, restore sine/cosine pointer
	yloop	4, b5b, -4*8*64+32*64, rdi, 4*YMM_SCD2 ;; Loop 4 times, next source pointer, next sine/cosine pointer
	yloop	2, b5b, -2*32*64+dist64	;; Test loop counter, next source pointer
	yloop	2, b5b, -2*dist64, rdi, -4*4*YMM_SCD2 ;; Test loop counter, restore source pointer, restore sine/cosine pointer
	loops_undo 4*4*2		;; Back up loop counter
	end_timer 8

	start_timer 9
	yloop_adjust 2*8		;; 16 fewer iterations this loop
	yloop_set_prefetch YOTHER, 8	;; Prefetch 36 cache lines in this loop
	yr8_8cl_eight_complex_with_square_preload
;; Loop below reworked to avoid 4KB distances
;;b6b:	yr8_8cl_eight_complex_with_square rsi, 2*64, 32*64, dist64, 64, 32
;;	yloop	32, b6b, -16*2*64	;; 16 iterations!!  Restore source pointer
b6b:	yr8_8cl_eight_complex_with_square rsi, 4*64, 32*64, dist64, 64, 8
	yloop	8, b6b, -8*4*64+2*64	;; Loop 8 times, next source pointer
	yloop	4, b6b, -2*2*64		;; 2 iterations!!  Restore source pointer
	loops_undo 4*8			;; Back up loop counter
	end_timer 9

	start_timer 10
	yloop_set_prefetch YQUARTER	;; Prefetch 72 cache lines in this loop
	yr4_eb4cl_four_complex_djbunfft_preload
c5b:	yr4_eb4cl_four_complex_djbunfft rsi, 64, 2*64, 4*64, rdi, 16*YMM_SCD2, 2
	yloop	2, c5b, -2*64+8*64, rdi, -2*16*YMM_SCD2 ;; Loop 2 times, next source pointer, restore sine/cosine pointer
	yloop	4, c5b, -4*8*64+32*64, rdi, 4*YMM_SCD2 ;; Loop 4 times, next source pointer, next sine/cosine pointer
	yloop	2, c5b, -2*32*64+dist64	;; Test loop counter, next source pointer
	yloop	2, c5b, -2*dist64, rdi, -4*4*YMM_SCD2 ;; Test loop counter, restore source pointer, restore sine/cosine pointer
	loops_undo 4*4*2		;; Back up loop counter
	end_timer 10

	start_timer 11
	yloop_set_prefetch YHALF	;; Prefetch 144 cache lines in this loop
;;	yr4_eb4cl_four_complex_djbunfft_preload
c4b:	yr4_eb4cl_four_complex_djbunfft rsi, 64, 8*64, 16*64, rdi, 4*YMM_SCD2, 8
	yloop	8, c4b, -8*64+32*64, rdi, -8*4*YMM_SCD2+YMM_SCD2 ;; Loop 8 times, next source pointer, next sine/cosine pointer
	yloop	2, c4b, -2*32*64+dist64	;; Loop 2 times, next source pointer
	yloop	2, c4b, -2*dist64, rdi, -4*YMM_SCD2 ;; Loop 2 times, restore source pointer, restore sine/cosine pointer
	loops_undo 4*8			;; Back up loop counter
	end_timer 11

	start_timer 12
	yloop_set_prefetch YHALF	;; Prefetch 144 cache lines in this loop
	yr4_s4cl_four_complex_djbunfft_preload
c3b:	yr4_s4cl_four_complex_djbunfft rsi, 64, 32*64, dist64, rdi, YMM_SCD2, 32, L1PREFETCH_ALL, 2*dist64
	yloop	32, c3b, -32*64+2*dist64, rdi, -32*YMM_SCD2 ;; Loop 32 times, next source pointer, restore sine/cosine pointer
	end_timer 12

	yloop	9, b3b0, -9*2*dist64	;; Loop 9 times, restore source pointer
	jmp	ypass2_4608_complex_unfft


ypass2_4608_complex_1:
	start_timer 6
	bump	rcx, (1152+2*17-2*128-3*288-144)*64 ;; Move prefetch pointer along
	yloop_init 288			;; 9 iters of 128 cache lines

b3a0:	yloop_set_prefetch YSINGLE	;; Prefetch 288 cache lines in this loop
	yr4_s4cl_four_complex_djbfft_preload
b3a:	yr4_s4cl_four_complex_djbfft rsi, 64, 32*64, dist64, rdi, YMM_SCD2, 32
	yloop	32, b3a, -32*64, rdi, -32*YMM_SCD2 ;; Loop 32 times, restore source pointer, restore sine/cosine pointer
	loops_undo 32			;; Back up loop counter

	yloop_set_prefetch YSINGLE	;; Prefetch 288 cache lines in this loop
	yr4_eb4cl_four_complex_djbfft_preload
b4a:	yr4_eb4cl_four_complex_djbfft rsi, 64, 8*64, 16*64, rdi, 4*YMM_SCD2, 8
	yloop	8, b4a, -8*64+32*64, rdi, -8*4*YMM_SCD2+YMM_SCD2 ;; Loop 8 times, next source pointer, next sine/cosine pointer
	yloop	2, b4a, -2*32*64+dist64	;; Loop 2 times, next source pointer
	yloop	2, b4a, -2*dist64, rdi, -4*YMM_SCD2 ;; Loop 2 times, restore source pointer, restore sine/cosine pointer
	loops_undo 4*8			;; Back up loop counter

	yloop_set_prefetch YSINGLE	;; Prefetch 288 cache lines in this loop
;;	yr4_eb4cl_four_complex_djbfft_preload
b5a:	yr4_eb4cl_four_complex_djbfft rsi, 64, 2*64, 4*64, rdi, 16*YMM_SCD2, 2, L1PREFETCH_ALL, 2*dist64
	yloop	2, b5a, -2*64+8*64, rdi, -2*16*YMM_SCD2 ;; Loop 2 times, next source pointer, restore sine/cosine pointer
	yloop	4, b5a, -4*8*64+32*64, rdi, 4*YMM_SCD2 ;; Loop 4 times, next source pointer, next sine/cosine pointer
	yloop	2, b5a, -2*32*64+dist64	;; Test loop counter, next source pointer
	yloop	2, b5a, -2*dist64, rdi, -4*4*YMM_SCD2 ;; Test loop counter, restore source pointer, restore sine/cosine pointer
	loops_undo 4*4*2		;; Back up loop counter

	yloop_adjust 2*8		;; 16 fewer iterations this loop
	yloop_set_prefetch YHALF	;; Prefetch 144 cache lines in this loop
	yr8_8cl_eight_complex_fft_final_preload
;; Loop below reworked to avoid 4KB distances
;;b6a:	yr8_8cl_eight_complex_fft_final rsi, 2*64, 32*64, dist64, 64
;;	yloop	32, b6a, -16*2*64	;; 16 iterations!!  Restore source pointer
b6a:	yr8_8cl_eight_complex_fft_final rsi, 4*64, 32*64, dist64, 64
	yloop	8, b6a, -8*4*64+2*64	;; Loop 8 times, next source pointer
	yloop	4, b6a, -2*2*64+2*dist64 ;; 2 iterations!!  Next source pointer

	yloop	9, b3a0			;; Loop 9 times
	end_timer 6
	jmp	ypass2_4608_complex_done


ypass2_4608_complex_3:
	start_timer 6
x64	mov	r8, DEST2ARG
	mov	rbp, DIST_TO_MULSRCARG
	yloop_init 288			;; 9 iters of 128 cache lines

b3c0:	yloop_set_prefetch YHALF	;; Prefetch 144 cache lines in this loop
	yr4_s4cl_four_complex_djbfft_preload
b3c:	yr4_s4cl_four_complex_djbfft rsi, 64, 32*64, dist64, rdi, YMM_SCD2, 32
	yloop	32, b3c, -32*64, rdi, -32*YMM_SCD2 ;; Loop 32 times, restore source pointer, restore sine/cosine pointer
	loops_undo 32			;; Back up loop counter

	yloop_set_prefetch YQUARTER	;; Prefetch 72 cache lines in this loop
	yr4_eb4cl_four_complex_djbfft_preload
b4c:	yr4_eb4cl_four_complex_djbfft rsi, 64, 8*64, 16*64, rdi, 4*YMM_SCD2, 8
	yloop	8, b4c, -8*64+32*64, rdi, -8*4*YMM_SCD2+YMM_SCD2 ;; Loop 8 times, next source pointer, next sine/cosine pointer
	yloop	2, b4c, -2*32*64+dist64	;; Loop 2 times, next source pointer
	yloop	2, b4c, -2*dist64, rdi, -4*YMM_SCD2 ;; Loop 2 times, restore source pointer, restore sine/cosine pointer
	loops_undo 4*8			;; Back up loop counter

	yloop_set_prefetch YQUARTER	;; Prefetch 72 cache lines in this loop
;;	yr4_eb4cl_four_complex_djbfft_preload
b5c:	yr4_eb4cl_four_complex_djbfft rsi, 64, 2*64, 4*64, rdi, 16*YMM_SCD2, 2
	yloop	2, b5c, -2*64+8*64, rdi, -2*16*YMM_SCD2 ;; Loop 2 times, next source pointer, restore sine/cosine pointer
	yloop	4, b5c, -4*8*64+32*64, rdi, 4*YMM_SCD2 ;; Loop 4 times, next source pointer, next sine/cosine pointer
	yloop	2, b5c, -2*32*64+dist64	;; Test loop counter, next source pointer
	yloop	2, b5c, -2*dist64, rdi, -4*4*YMM_SCD2 ;; Test loop counter, restore source pointer, restore sine/cosine pointer
	loops_undo 4*4*2		;; Back up loop counter

	yloop_adjust 2*8		;; 16 fewer iterations this loop
	yloop_set_prefetch YOTHER, 8	;; Prefetch 36 cache lines in this loop
	yr8_8cl_eight_complex_with_mult_preload
;; Loop below reworked to avoid 4KB distances
;;b6c:	yr8_8cl_eight_complex_with_mult rsi, 2*64, 32*64, dist64, 64
;;	yloop	32, b6c, -16*2*64	;; 16 iterations!!  Restore source pointer
b6c:	yr8_8cl_eight_complex_with_mult rsi, 4*64, 32*64, dist64, 64
	yloop	8, b6c, -8*4*64+2*64	;; Loop 8 times, next source pointer
	yloop	4, b6c, -2*2*64		;; 2 iterations!!  Restore source pointer
x64	add	rsi, r8
	loops_undo 4*8			;; Back up loop counter

	yloop_set_prefetch YQUARTER	;; Prefetch 72 cache lines in this loop
	yr4_eb4cl_four_complex_djbunfft_preload
c5c:	yr4_eb4cl_four_complex_djbunfft rsi, 64, 2*64, 4*64, rdi, 16*YMM_SCD2, 2
	yloop	2, c5c, -2*64+8*64, rdi, -2*16*YMM_SCD2 ;; Loop 2 times, next source pointer, restore sine/cosine pointer
	yloop	4, c5c, -4*8*64+32*64, rdi, 4*YMM_SCD2 ;; Loop 4 times, next source pointer, next sine/cosine pointer
	yloop	2, c5c, -2*32*64+dist64	;; Test loop counter, next source pointer
	yloop	2, c5c, -2*dist64, rdi, -4*4*YMM_SCD2 ;; Test loop counter, restore source pointer, restore sine/cosine pointer
	loops_undo 4*4*2		;; Back up loop counter

	yloop_set_prefetch YHALF	;; Prefetch 144 cache lines in this loop
;;	yr4_eb4cl_four_complex_djbunfft_preload
c4c:	yr4_eb4cl_four_complex_djbunfft rsi, 64, 8*64, 16*64, rdi, 4*YMM_SCD2, 8
	yloop	8, c4c, -8*64+32*64, rdi, -8*4*YMM_SCD2+YMM_SCD2 ;; Loop 8 times, next source pointer, next sine/cosine pointer
	yloop	2, c4c, -2*32*64+dist64	;; Loop 2 times, next source pointer
	yloop	2, c4c, -2*dist64, rdi, -4*YMM_SCD2 ;; Loop 2 times, restore source pointer, restore sine/cosine pointer
	loops_undo 4*8			;; Back up loop counter

	yloop_set_prefetch YHALF	;; Prefetch 144 cache lines in this loop
	yr4_s4cl_four_complex_djbunfft_preload
c3c:	yr4_s4cl_four_complex_djbunfft rsi, 64, 32*64, dist64, rdi, YMM_SCD2, 32, L1PREFETCH_ALL, 2*dist64
	yloop	32, c3c, -32*64+2*dist64, rdi, -32*YMM_SCD2 ;; Loop 32 times, next source pointer, restore sine/cosine pointer

x64	sub	rsi, r8
	yloop	9, b3c0, -9*2*dist64	;; Loop 9 times, restore source pointer
x64	add	rsi, r8
	end_timer 6
	jmp	ypass2_4608_complex_unfft


ypass2_4608_complex_4:
	start_timer 6
	mov	rdi, xsincos_complex	;; The sin/cos ptrs
	bump	rdi, 384*YMM_SCD1+128*YMM_SCD1
	mov	rbp, DIST_TO_MULSRCARG
	yloop_init 288			;; 9 iters of 128 cache lines

b6d0:	yloop_adjust 2*8		;; 16 fewer iterations this loop
	yloop_set_prefetch YHALF	;; Prefetch 144 cache lines in this loop
	yr8_8cl_eight_complex_with_mulf_preload
;; Loop below reworked to avoid 4KB distances
;;b6d:	yr8_8cl_eight_complex_with_mulf rsi, 2*64, 32*64, dist64, 64
;;	yloop	32, b6d, -16*2*64	;; 16 iterations!!  Restore source pointer
b6d:	yr8_8cl_eight_complex_with_mulf rsi, 4*64, 32*64, dist64, 64
	yloop	8, b6d, -8*4*64+2*64	;; Loop 8 times, next source pointer
	yloop	4, b6d, -2*2*64		;; 2 itertions!!  Restore source pointer
	loops_undo 4*8			;; Back up loop counter

	yloop_set_prefetch YSINGLE	;; Prefetch 288 cache lines in this loop
	yr4_eb4cl_four_complex_djbunfft_preload
c5d:	yr4_eb4cl_four_complex_djbunfft rsi, 64, 2*64, 4*64, rdi, 16*YMM_SCD2, 2
	yloop	2, c5d, -2*64+8*64, rdi, -2*16*YMM_SCD2 ;; Loop 2 times, next source pointer, restore sine/cosine pointer
	yloop	4, c5d, -4*8*64+32*64, rdi, 4*YMM_SCD2 ;; Loop 4 times, next source pointer, next sine/cosine pointer
	yloop	2, c5d, -2*32*64+dist64	;; Test loop counter, next source pointer
	yloop	2, c5d, -2*dist64, rdi, -4*4*YMM_SCD2 ;; Test loop counter, restore source pointer, restore sine/cosine pointer
	loops_undo 4*4*2		;; Back up loop counter

	yloop_set_prefetch YSINGLE	;; Prefetch 288 cache lines in this loop
;;	yr4_eb4cl_four_complex_djbunfft_preload
c4d:	yr4_eb4cl_four_complex_djbunfft rsi, 64, 8*64, 16*64, rdi, 4*YMM_SCD2, 8
	yloop	8, c4d, -8*64+32*64, rdi, -8*4*YMM_SCD2+YMM_SCD2 ;; Loop 8 times, next source pointer, next sine/cosine pointer
	yloop	2, c4d, -2*32*64+dist64	;; Loop 2 times, next source pointer
	yloop	2, c4d, -2*dist64, rdi, -4*YMM_SCD2 ;; Loop 2 times, restore source pointer, restore sine/cosine pointer
	loops_undo 4*8			;; Back up loop counter

	yloop_set_prefetch YSINGLE	;; Prefetch 288 cache lines in this loop
	yr4_s4cl_four_complex_djbunfft_preload
c3d:	yr4_s4cl_four_complex_djbunfft rsi, 64, 32*64, dist64, rdi, YMM_SCD2, 32, L1PREFETCH_ALL, 2*dist64
	yloop	32, c3d, -32*64+2*dist64, rdi, -32*YMM_SCD2 ;; Loop 32 times, next source pointer, restore sine/cosine pointer

	yloop	9, b6d0, -9*2*dist64	;; Loop 9 times, restore source pointer
	bump	rcx, (-144-3*288+2*128+144+2*72+36+72+2*144)*64 ;; Move prefetch pointer to where squaring would be
	end_timer 6

;; Do inverse FFT levels 3,4
;; On output the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	512	+1	+1	+1
;;	4	...
;;	...
;;	508	...
;;	1024	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	1536	+1	+1	+1
;;	4	...
;;	...
;;	1532	...
;;	3072	...
;;	...

;; Do 1536 three_complex_unfft operations
;;	distance between fft data elements is 512
;;	do 384 macros each processing 24 data values

;; Do inverse FFT levels 1,2
;; On input the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	1536	+1	+1	+1
;;	4	...
;;	...
;;	1532	...
;;	3072	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	4608	+1	+1	+1
;;	4	...
;;	...
;;	4604

;; Do 1536 three_complex_unfft operations
;;	distance between fft data elements is 1536
;;	do 384 macros each processing 24 data values

ypass2_4608_complex_unfft:
	mov	rdi, xsincos_complex	;; Load sin/cos pointers
	lea	rbp, [rdi+384*YMM_SCD1]
	bump	rcx, (1152+2*17-2*128-144-2*72-36-72-2*144-2*128)*64 ;; Move prefetch pointer along
	yloop_init 512			;; 384 iterations

	yr3_3cl_three_complex_djbunfft_preload
c2a:	start_timer 13
	yloop_adjust 1
	yloop_set_prefetch YQUARTER	;; Prefetch 128 cache lines
c2b:	yr3_3cl_three_complex_djbunfft rsi, 6*dist64, 2*dist64, rbp, 0, 3
	yloop	4, c2b, -3*6*dist64, rbp, YMM_SCD1 ;; 3 iterations!!  Restore source pointer, next sine/cosine pointer
	loops_undo 4			;; Restore loop counter
	end_timer 13

	start_timer 14
	yloop_adjust 1
	yloop_set_prefetch YQUARTER	;; Prefetch 128 cache lines
c1b:	yr3_3cl_three_complex_djbunfft rsi, 2*dist64, 6*dist64, rdi, 128*YMM_SCD1, 3, L1PREFETCH_ALL, 64
	yloop	4, c1b, -3*2*dist64+64, rdi, -3*128*YMM_SCD1+YMM_SCD1 ;; 3 iterations!!  Next source pointer, next sine/cosine pointer
	end_timer 14

	yloop	64, c2a, -64*64+dist64	;; Loop 64 times, next source pointer
	yloop	2, c2a			;; Loop 2 times

ypass2_4608_complex_done:
	ENDM


; *************** radix4-pass2-6144-real macro ******************
; This macro takes 12288 real values and performs the final 13.585 levels of the FFT.
; The results are squared, and the first 13.585 levels of the inverse FFT are done.

yr4dwpn_pass2_6144_real MACRO
	LOCAL	b1b, b2a, b2b, b3a, b3b, b4a, b4b, b5a, b5b, b6a, b6b, b6c, b6d
	LOCAL	c1b, c2a, c2b, c3a, c3b, c4a, c4b, c5a, c5b
	LOCAL	ypass2_6144_real_1, ypass2_6144_real_2
	LOCAL	ypass2_6144_real_3, ypass2_6144_real_4
	LOCAL	ypass2_6144_real_unfft, ypass2_6144_real_done

;; Type 4 FFTs skip the forward FFT process

	start_timer 2
	mov	rsi, DATA_ADDR		;; Load source address
	mov	rbx, DIST_TO_FFTSRCARG
	mov	rcx, DATA_PREFETCH	;; Prefetch pointer (we need to prefetch 
					;; 1536 cache lines + 23 pad cache lines)
	cmp	ffttype, 4
	je	ypass2_6144_real_4

;; Do FFT levels 1,2,3 which correspond to complex levels 1,2
;; On input the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	6144	+1	+1	+1
;;	4	...
;;	...
;;	6140
;; On output the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	2048	+1	+1	+1
;;	4	...
;;	...
;;	2044	...
;;	4096	...
;;	...

;; Do 2048 six_reals_fft operations
;;	distance between fft data elements is 2048
;;	do 512 macros each processing 24 data values

	mov	rdi, sincos3		;; Real sin/cos pointer
	mov	rbp, xsincos_complex	;; Complex sin/cos data
	yloop_init 512			;; 512 iterations
	yloop_set_prefetch YHALF
	yr3_f3cl_2sc_six_reals_fft_preload
b1b:	yr3_f3cl_2sc_six_reals_fft rsi, 64, 8*dist64, rbp, YMM_SCD1, rdi, YMM_SCD1, 64
	yloop	64, b1b, -64*64+dist64	;; Test loop counter, next source pointer
	yloop	8, b1b, -8*dist64	;; Test loop counter, restore source pointer

;; Do FFT levels 3,4
;; On input the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	2048	+1	+1	+1
;;	4	...
;;	...
;;	2044	...
;;	4096	...
;;	...
;; On output the 64-byte cache lines are swizzled to hold these data values:
;;	0	+1K	+1K	+1K	512	+1K	+1K	+1K
;;	4	...
;;	...
;;	508	...
;;	1	...
;;	...
;;	4096	...
;;	...

;; Do 512 eight_reals_fft operations
;; Do 1024 four_complex_fft operations
;;	distance between fft data elements is 512
;;	do 384 macros each processing 32 data values

	yloop_init 384			;; 3 iterations of 128
	yloop_set_prefetch YHALF
	yr4_s4cl_2sc_eight_reals_fft_preload
b2a:	yr4_s4cl_2sc_eight_reals_fft rsi, 64, 2*dist64, 4*dist64, rbp, YMM_SCD2, rdi, YMM_SCD2, 64
	yloop	64, b2a, -64*64+dist64	;; Test loop counter, next source pointer
	yloop	2, b2a, -2*dist64+8*dist64, rbp, -128*YMM_SCD2 ;; Test loop counter, next source pointer, restore sine/cosine pointer

	loops_reset			;; 2 more iters of 128
	yr4_s4cl_four_complex_djbfft_preload
b2b:	yr4_s4cl_four_complex_djbfft rsi, 64, 2*dist64, 4*dist64, rbp, YMM_SCD2, 64
	yloop	64, b2b, -64*64+dist64	;; Test loop counter, next source pointer
	yloop	2, b2b, -2*dist64+8*dist64, rbp, -128*YMM_SCD2 ;; Test loop counter, next source pointer, restore sin/cos ptr
	yloop	3, b2b, -3*8*dist64	;; Loop 3 times, restore source pointer

;; Do FFT levels 5,6
;; On input the 64-byte cache lines hold these data values:
;;	0	+1K	+1K	+1K	512	+1K	+1K	+1K
;;	4	...
;;	...
;;	508	...
;;	1	...
;;	...
;;	4096	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+1K	+1K	+1K	128	+1K	+1K	+1K
;;	4	...
;;	...
;;	124	...
;;	256	...
;;	...
;;	1	...
;;	...
;;	4096	...
;;	...

;; Do 128 eight_reals_fft operations
;; Do 1408 four_complex_fft operations
;;	distance between fft data elements is 128
;;	do 384 macros each processing 32 data values

	yloop_init 384			;; 3 iterations of 128
	yloop_set_prefetch YHALF
	yr4_4cl_eight_reals_four_complex_djbfft_preload
b3a:	yr4_4cl_eight_reals_four_complex_djbfft rsi, 2*dist64, 32*64, dist64, rdi, YMM_SCD3, 4
	yloop	4, b3a, -4*2*dist64+64	;; Test loop counter, next source pointer
	yloop	32, b3a, -32*64+8*dist64, rdi, -128*YMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer

	loops_reset			;; 2 more iters of 128
	yr4_rb4cl_four_complex_djbfft_preload
b3b:	yr4_rb4cl_four_complex_djbfft rsi, 2*dist64, 32*64, dist64, rdi, YMM_SCD3, 4
	yloop	4, b3b, -4*2*dist64+64	;; Test loop counter, next source pointer
	yloop	32, b3b, -32*64+8*dist64, rdi, -128*YMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer
	yloop	3, b3b, -3*8*dist64	;; Test loop counter, restore source pointer

;; Do FFT levels 7,8
;; On input the 64-byte cache lines hold these data values:
;;	0	+1K	+1K	+1K	128	+1K	+1K	+1K
;;	4	...
;;	...
;;	124	...
;;	256	...
;;	...
;;	1	...
;;	...
;;	4096	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+1K	+1K	+1K	32	+1K	+1K	+1K
;;	4	...
;;	...
;;	28	...
;;	64	...
;;	...
;;	1	...
;;	...
;;	4096	...
;;	...

;; Do 32 eight_reals_fft operations
;; Do 1504 four_complex_fft operations
;;	distance between fft data elements is 32
;;	do 384 macros each processing 32 data values

	yloop_init 384			;; 12 iterations of 32
	yloop_set_prefetch YHALF
	yr4_4cl_eight_reals_four_complex_djbfft_preload
b4a:	yr4_4cl_eight_reals_four_complex_djbfft rsi, 2*dist64, 8*64, 16*64, rdi, 4*YMM_SCD3, 4
	yloop	4, b4a, -4*2*dist64+64	;; Test loop counter, next source pointer
	yloop	8, b4a, -8*64+32*64, rdi, -32*4*YMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer

	loops_reset			;; 11 more iters of 32
	yr4_rb4cl_four_complex_djbfft_preload
b4b:	yr4_rb4cl_four_complex_djbfft rsi, 2*dist64, 8*64, 16*64, rdi, 4*YMM_SCD3, 4
	yloop	4, b4b, -4*2*dist64+64	;; Test loop counter, next source pointer
	yloop	8, b4b, -8*64+32*64, rdi, -32*4*YMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer
	yloop	2, b4b, -2*32*64+dist64	;; Test loop counter, next source pointer
	yloop	2, b4b, -2*dist64+8*dist64 ;; Test loop counter, next source pointer
	yloop	3, b4b, -3*8*dist64	;; Test loop counter, restore source pointer

;; Do FFT levels 9,10
;; On input the 64-byte cache lines hold these data values:
;;	0	+1K	+1K	+1K	32	+1K	+1K	+1K
;;	4	...
;;	...
;;	28	...
;;	64	...
;;	...
;;	1	...
;;	...
;;	4096	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+1K	+1K	+1K	8	+1K	+1K	+1K
;;	4	...
;;	16	...
;;	...
;;	1	...
;;	...
;;	4096	...
;;	...

;; Do 8 eight_reals_fft operations
;; Do 1528 four_complex_fft operations
;;	distance between fft data elements is 8
;;	do 384 macros each processing 32 data values

	yloop_init 384			;; 48 iterations of 8
	yloop_set_prefetch YHALF
	yr4_4cl_eight_reals_four_complex_djbfft_preload
b5a:	yr4_4cl_eight_reals_four_complex_djbfft rsi, 2*dist64, 2*64, 4*64, rdi, 16*YMM_SCD3, 4
	yloop	4, b5a, -4*2*dist64+64	;; Test loop counter, next source pointer
	yloop	2, b5a, -2*64+8*64, rdi, -8*16*YMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer

	loops_reset			;; 47 more iters of 8
	yr4_rb4cl_four_complex_djbfft_preload
b5b:	yr4_rb4cl_four_complex_djbfft rsi, 2*dist64, 2*64, 4*64, rdi, 16*YMM_SCD3, 4
	yloop	4, b5b, -4*2*dist64+64	;; Test loop counter, next source pointer
	yloop	2, b5b, -2*64+8*64, rdi, -8*16*YMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer
	yloop	8, b5b, -8*8*64+dist64	;; Test loop counter, next source pointer
	yloop	2, b5b, -2*dist64+8*dist64 ;; Test loop counter, next source pointer
	yloop	3, b5b, -3*8*dist64	;; Test loop counter, restore source pointer

;; Do FFT levels 11,12,13 as well as inverse FFT levels 11,12,13
;; On input and output the 64-byte cache lines hold these data values:
;;	0	+1K	+1K	+1K	8	+1K	+1K	+1K
;;	4	...
;;	16	...
;;	...
;;	1	...
;;	...
;;	4096	...
;;	...

;; Do 1 sixteen_reals_fft operation
;; Do 767 eight_complex_fft operations
;;	distance between fft data elements is 1
;;	do 192 macros each processing 64 data values

	cmp	ffttype, 2
	jl	ypass2_6144_real_1
	jg	ypass2_6144_real_3

ypass2_6144_real_2:
;;	yr8_8cl_sixteen_reals_eight_complex_with_square rsi, 2*64, 2*dist64, 4*dist64, 64
	yr8_dispatch call, y8real_square, 2*dist64, 4*dist64, 64
	yloop_init 192			;; 191 more iterations
	yloop_adjust 1
	yloop_set_prefetch YSINGLE
	yr8_8cl_eight_complex_with_square_preload
b6b:	yr8_8cl_eight_complex_with_square rsi, 2*64, 2*dist64, 4*dist64, 64, 1
	yloop	32, b6b, -32*2*64+dist64	;; Test loop counter, next source pointer
	yloop	2, b6b, -2*dist64+8*dist64	;; Test loop counter, next source pointer
	yloop	3, b6b, -3*8*dist64	;; Test loop counter, restore source pointer
	jmp	ypass2_6144_real_unfft

ypass2_6144_real_1:
;;	yr8_8cl_sixteen_reals_eight_complex_fft_final rsi, 2*64, 2*dist64, 4*dist64, 64
	yr8_dispatch call, y8real_fft_final, 2*dist64, 4*dist64, 64
	mov	rcx, xsincos_complex	;; Prefetch complex sin/cos data, 784 cache lines
	yloop_init 192			;; 191 more iterations
	yloop_adjust 1
	yloop_set_prefetch YDOUBLE
b6a:	yr8_8cl_eight_complex_fft_final rsi, 2*64, 2*dist64, 4*dist64, 64
	yloop	32, b6a, -32*2*64+dist64	;; Test loop counter, next source pointer
	yloop	2, b6a, -2*dist64+8*dist64	;; Test loop counter, next source pointer
	yloop	3, b6a			;; Test loop counter
	jmp	ypass2_6144_real_done

ypass2_6144_real_3:
x64	mov	r8, DEST2ARG
	mov	rbp, DIST_TO_MULSRCARG
;;	yr8_8cl_sixteen_reals_eight_complex_with_mult rsi, 2*64, 2*dist64, 4*dist64, 64
	yr8_dispatch call, y8real_mult, 2*dist64, 4*dist64, 64
	yloop_init 192			;; 191 more iterations
	yloop_adjust 1
	yloop_set_prefetch YSINGLE
	yr8_8cl_eight_complex_with_mult_preload
b6c:	yr8_8cl_eight_complex_with_mult rsi, 2*64, 2*dist64, 4*dist64, 64
	yloop	32, b6c, -32*2*64+dist64	;; Test loop counter, next source pointer
	yloop	2, b6c, -2*dist64+8*dist64	;; Test loop counter, next source pointer
	yloop	3, b6c, -3*8*dist64	;; Test loop counter, restore source pointer
x64	add	rsi, r8
	jmp	ypass2_6144_real_unfft

ypass2_6144_real_4:
	mov	rbp, DIST_TO_MULSRCARG
;;	yr8_8cl_sixteen_reals_eight_complex_with_mulf rsi, 2*64, 2*dist64, 4*dist64, 64
	yr8_dispatch call, y8real_mulf, 2*dist64, 4*dist64, 64
	yloop_init 192			;; 191 more iterations
	yloop_adjust 1
	yloop_set_prefetch YDOUBLE
	yr8_8cl_eight_complex_with_mulf_preload
b6d:	yr8_8cl_eight_complex_with_mulf rsi, 2*64, 2*dist64, 4*dist64, 64
	yloop	32, b6d, -32*2*64+dist64	;; Test loop counter, next source pointer
	yloop	2, b6d, -2*dist64+8*dist64	;; Test loop counter, next source pointer
	yloop	3, b6d, -3*8*dist64	;; Test loop counter, restore source pointer
;;	bump	rcx, (-384+256+4*192+191)*64 ;; Move prefetch pointer back to where squaring would be

;; Do inverse FFT levels 9,10
;; On input the 64-byte cache lines hold these data values:
;;	0	+1K	+1K	+1K	8	+1K	+1K	+1K
;;	4	...
;;	16	...
;;	...
;;	1	...
;;	...
;;	4096	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+1K	+1K	+1K	32	+1K	+1K	+1K
;;	4	...
;;	...
;;	28	...
;;	64	...
;;	...
;;	1	...
;;	...
;;	4096	...
;;	...

;; Do 8 eight_reals_unfft operations
;; Do 1528 four_complex_unfft operations
;;	distance between fft data elements is 8
;;	do 384 macros each processing 32 data values

ypass2_6144_real_unfft:
	mov	rdi, sincos3		;; Load sin/cos pointer
	bump	rdi, 128*YMM_SCD2+512*YMM_SCD1
	yloop_init 384			;; 48 iterations of 8
	yloop_set_prefetch YHALF
	yr4_4cl_eight_reals_four_complex_djbunfft_preload
c5a:	yr4_4cl_eight_reals_four_complex_djbunfft rsi, 2*dist64, 2*64, 4*64, rdi, 16*YMM_SCD3, 4
	yloop	4, c5a, -4*2*dist64+64	;; Test loop counter, next source pointer
	yloop	2, c5a, -2*64+8*64, rdi, -8*16*YMM_SCD3 ;; Test loop counter, next source pointer, restore sine/cosine pointer

	loops_reset			;; 47 more iters of 8
	yr4_rb4cl_four_complex_djbunfft_preload
c5b:	yr4_rb4cl_four_complex_djbunfft rsi, 2*dist64, 2*64, 4*64, rdi, 16*YMM_SCD3, 4
	yloop	4, c5b, -4*2*dist64+64	;; Test loop counter, next source pointer
	yloop	2, c5b, -2*64+8*64, rdi, -8*16*YMM_SCD3 ;; Test loop counter, next source pointer, restore sine/cosine pointer
	yloop	8, c5b, -8*8*64+dist64	;; Test loop counter, next source pointer
	yloop	2, c5b, -2*dist64+8*dist64	;; Test loop counter, next source pointer
	yloop	3, c5b, -3*8*dist64	;; Test loop counter, restore source pointer

;; Do inverse FFT levels 7,8
;; On input the 64-byte cache lines hold these data values:
;;	0	+1K	+1K	+1K	32	+1K	+1K	+1K
;;	4	...
;;	...
;;	28	...
;;	64	...
;;	...
;;	1	...
;;	...
;;	4096	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+1K	+1K	+1K	128	+1K	+1K	+1K
;;	4	...
;;	...
;;	124	...
;;	256	...
;;	...
;;	1	...
;;	...
;;	4096	...
;;	...

;; Do 32 eight_reals_unfft operations
;; Do 1504 four_complex_unfft operations
;;	distance between fft data elements is 32
;;	do 384 macros each processing 32 data values

	bump	rcx, (1536+23-256-4*192-191-192-192)*64 ;; Move prefetch pointer along
	yloop_init 384			;; 12 iterations of 32
	yloop_set_prefetch YHALF
	yr4_4cl_eight_reals_four_complex_djbunfft_preload
c4a:	yr4_4cl_eight_reals_four_complex_djbunfft rsi, 2*dist64, 8*64, 16*64, rdi, 4*YMM_SCD3, 4
	yloop	4, c4a, -4*2*dist64+64	;; Test loop counter, next source pointer
	yloop	8, c4a, -8*64+32*64, rdi, -32*4*YMM_SCD3 ;; Test loop counter, next source pointer, restore sine/cosine pointer

	loops_reset			;; 11 more iters of 32
	yr4_rb4cl_four_complex_djbunfft_preload
c4b:	yr4_rb4cl_four_complex_djbunfft rsi, 2*dist64, 8*64, 16*64, rdi, 4*YMM_SCD3, 4
	yloop	4, c4b, -4*2*dist64+64	;; Test loop counter, next source pointer
	yloop	8, c4b, -8*64+32*64, rdi, -32*4*YMM_SCD3 ;; Test loop counter, next source pointer, restore sine/cosine pointer
	yloop	2, c4b, -2*32*64+dist64	;; Test loop counter, next source pointer
	yloop	2, c4b, -2*dist64+8*dist64	;; Test loop counter, next source pointer
	yloop	3, c4b, -3*8*dist64	;; Test loop counter, restore source pointer

;; Do inverse FFT levels 5,6
;; On input the 64-byte cache lines hold these data values:
;;	0	+1K	+1K	+1K	128	+1K	+1K	+1K
;;	4	...
;;	...
;;	124	...
;;	256	...
;;	...
;;	1	...
;;	...
;;	4096	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+1K	+1K	+1K	512	+1K	+1K	+1K
;;	4	...
;;	...
;;	508	...
;;	1	...
;;	...
;;	4096	...
;;	...

;; Do 128 eight_reals_unfft operations
;; Do 1408 four_complex_unfft operations
;;	distance between fft data elements is 128
;;	do 384 macros each processing 32 data values

	mov	rcx, xsincos_complex	;; Prefetch complex sin/cos data, 784 cache lines
	yloop_init 384			;; 3 iterations of 128
	yloop_set_prefetch YHALF
	yr4_4cl_eight_reals_four_complex_djbunfft_preload
c3a:	yr4_4cl_eight_reals_four_complex_djbunfft rsi, 2*dist64, 32*64, dist64, rdi, YMM_SCD3, 4
	yloop	4, c3a, -4*2*dist64+64	;; Test loop counter, next source pointer
	yloop	32, c3a, -32*64+8*dist64, rdi, -128*YMM_SCD3 ;; Test loop counter, next source pointer, restore sine/cosine pointer

	loops_reset			;; 2 more iters of 128
	yr4_rb4cl_four_complex_djbunfft_preload
c3b:	yr4_rb4cl_four_complex_djbunfft rsi, 2*dist64, 32*64, dist64, rdi, YMM_SCD3, 4
	yloop	4, c3b, -4*2*dist64+64	;; Test loop counter, next source pointer
	yloop	32, c3b, -32*64+8*dist64, rdi, -128*YMM_SCD3 ;; Test loop counter, next source pointer, restore sine/cosine pointer
	yloop	3, c3b, -3*8*dist64	;; Test loop counter, restore source pointer

;; Do inverse FFT levels 3,4
;; On input the 64-byte cache lines hold these data values:
;;	0	+1K	+1K	+1K	512	+1K	+1K	+1K
;;	4	...
;;	...
;;	508	...
;;	1	...
;;	...
;;	4096	...
;;	...
;; On output the 64-byte cache lines are swizzled to hold these data values:
;;	0	+1	+1	+1	2048	+1	+1	+1
;;	4	...
;;	...
;;	2044	...
;;	4096	...
;;	...

;; Do 512 eight_reals_unfft operations
;; Do 1024 four_complex_unfft operations
;;	distance between fft data elements is 512
;;	do 384 macros each processing 32 data values

	bump	rdi, -128*YMM_SCD2	;; Real sin/cos ptr
	mov	rbp, xsincos_complex	;; Complex sin/cos data
	bump	rbp, 512*YMM_SCD1
	yloop_init 384			;; 3 iterations of 128
	yloop_set_prefetch YSINGLE
	yr4_s4cl_2sc_eight_reals_unfft_preload
c2a:	yr4_s4cl_2sc_eight_reals_unfft rsi, 64, 2*dist64, 4*dist64, rbp, YMM_SCD2, rdi, YMM_SCD2, 64
	yloop	64, c2a, -64*64+dist64	;; Test loop counter, next source pointer
	yloop	2, c2a, -2*dist64+8*dist64, rbp, -128*YMM_SCD2 ;; Test loop counter, next source pointer, restore sine/cosine pointer

	loops_reset			;; 2 more iters of 128
	yr4_s4cl_four_complex_djbunfft_preload
c2b:	yr4_s4cl_four_complex_djbunfft rsi, 64, 2*dist64, 4*dist64, rbp, YMM_SCD2, 64
	yloop	64, c2b, -64*64+dist64	;; Test loop counter, next source pointer
	yloop	2, c2b, -2*dist64+8*dist64, rbp, -128*YMM_SCD2 ;; Test loop counter, next source pointer, restore sine/cosine pointer
	yloop	3, c2b, -3*8*dist64	;; Test loop counter, restore source pointer

;; Do inverse FFT levels 1,2,3 which correspond to complex levels 1,2
;; On input the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	2048	+1	+1	+1
;;	4	...
;;	...
;;	2044	...
;;	4096	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	6144	+1	+1	+1
;;	4	...
;;	...
;;	6140

;; Do 2048 six_reals_unfft operations
;;	distance between fft data elements is 2048
;;	do 512 macros each processing 24 data values

	bump	rdi, -128*YMM_SCD2-512*YMM_SCD1	;; Real sin/cos ptr
	bump	rbp, -512*YMM_SCD1	;; Complex sin/cos data
	bump	rcx, (784-192-384-256)*64 ;; Move prefetch pointer along
	yloop_init 512			;; 512 iterations
	yloop_set_prefetch YHALF
	yr3_3cl_2sc_six_reals_unfft_preload
c1b:	yr3_3cl_2sc_six_reals_unfft rsi, 64, 8*dist64, rbp, YMM_SCD1, rdi, YMM_SCD1, 64
	yloop	64, c1b, -64*64+dist64	;; Test loop counter, next source pointer
	yloop	8, c1b			;; Test loop counter

ypass2_6144_real_done:
	end_timer 2
	ENDM

; *************** radix4-pass2-6144-complex macro ******************
; This macro takes 6144 complex values and performs the final 12.585 levels of a standard
; radix-4 FFT, squares the results, and does the first 12.585 levels of the inverse FFT.

yr4dwpn_pass2_6144_complex MACRO
	LOCAL	b1a, b1b, b2b, b3b, b4a0, b4a, b4b0, b4b, b4c0, b4c, b5a, b5b, b5c, b6a, b6b, b6c, b6d0, b6d
	LOCAL	c1b, c2b, c3a, c3b, c4b, c4c, c4d, c5b, c5c, c5d
	LOCAL	ypass2_6144_complex_1, ypass2_6144_complex_2
	LOCAL	ypass2_6144_complex_3, ypass2_6144_complex_4
	LOCAL	ypass2_6144_complex_unfft, ypass2_6144_complex_done

;; Type 4 FFTs skip the forward FFT process

	mov	rsi, DATA_ADDR		;; Load address of FFT data
	mov	rbx, DIST_TO_FFTSRCARG
	mov	rcx, DATA_PREFETCH	;; Prefetch pointer (we need to prefetch 
					;; 1536 cache lines + 23 pad cache lines)
	cmp	ffttype, 4
	je	ypass2_6144_complex_4

;; Do FFT level 1,2 (actually 1.585 levels)
;; On input the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	6144	+1	+1	+1
;;	4	...
;;	...
;;	6140
;; On output the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	2048	+1	+1	+1
;;	4	...
;;	...
;;	2044	...
;;	4096	...
;;	...

;; Do 2048 three_complex_fft operations
;;	distance between fft data elements is 2048
;;	do 512 macros each processing 24 data values

;; Do FFT levels 3,4
;; On input the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	2048	+1	+1	+1
;;	4	...
;;	...
;;	2044	...
;;	4096	...
;;	...
;; On output the 64-byte cache lines are swizzled to hold these data values:
;;	0	+1K	+1K	+1K	512	+1K	+1K	+1K
;;	4	...
;;	...
;;	508	...
;;	1	...
;;	...
;;	4096	...
;;	...

;; Do 1536 four_complex_fft operations
;;	distance between fft data elements is 512
;;	do 384 macros each processing 32 data values

;; Do FFT levels 5,6
;; On input the 64-byte cache lines hold these data values:
;;	0	+1K	+1K	+1K	512	+1K	+1K	+1K
;;	4	...
;;	...
;;	508	...
;;	1	...
;;	...
;;	4096	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+1K	+1K	+1K	128	+1K	+1K	+1K
;;	4	...
;;	...
;;	124	...
;;	256	...
;;	...
;;	1	...
;;	...
;;	4096	...
;;	...

;; Do 1536 four_complex_fft operations
;;	distance between fft data elements is 128
;;	do 384 macros each processing 32 data values

	mov	rdi, xsincos_complex	;; The sin/cos ptrs
	lea	rbp, [rdi+512*YMM_SCD1]
	mov	rdx, rbp
	yloop_init 512			;; 512 iterations or 384 iterations

b1a:	start_timer 4
	yloop_set_prefetch YQUARTER	;; Prefetch 128 cache lines in this loop
	yr3_f3cl_three_complex_djbfft_preload
b1b:	yr3_f3cl_three_complex_djbfft rsi, 32*64, 8*dist64, rdi, 32*YMM_SCD1, 2
	yloop	2, b1b, -2*32*64+dist64	;; Test loop counter, next source pointer
	yloop	8, b1b, -8*dist64, rdi, -16*32*YMM_SCD1+YMM_SCD1 ;; Test loop counter, restore source pointer, next sin/cos data
	loops_undo 16			;; Back up loop counter
	end_timer 4

	start_timer 5
	yloop_adjust 4			;; Four fewer iterations this loop
	yloop_set_prefetch YHALF	;; Prefetch 192 cache lines in this loop
	yr4_s4cl_four_complex_djbfft_preload
b2b:	yr4_s4cl_four_complex_djbfft rsi, 32*64, 2*dist64, 4*dist64, rbp, 32*YMM_SCD2, 2
	yloop	2, b2b, -2*32*64+dist64	;; Test loop counter, next source pointer
	yloop	2, b2b, -2*dist64+8*dist64, rbp, -4*32*YMM_SCD2 ;; Test loop counter, next source pointer, restore sin/cos data
	yloop	4, b2b, -3*8*dist64, rbp, YMM_SCD2 ;; 3 iterations!!  Restore source pointer, next sin/cos data
	loops_undo 4*4			;; Back up loop counter
	end_timer 5

	start_timer 6
	yloop_adjust 4			;; Four fewer iterations this loop
	yloop_set_prefetch YHALF	;; Prefetch 192 cache lines in this loop
	yr4_eb4cl_four_complex_djbfft_preload
b3b:	yr4_eb4cl_four_complex_djbfft rsi, 2*dist64, 32*64, dist64, rdx, YMM_SCD2, 4, L1PREFETCH_ALL, 64
	yloop	4, b3b, 0, rdx, -4*YMM_SCD2 ;; Loop 4 times, restore sine/cosine pointer
	yloop	4, b3b, -3*4*2*dist64+64, rdx, 4*YMM_SCD2 ;; 3 iterations!!  Next source pointer, next sine/cosine pointer
	end_timer 6

	yloop	32, b1a, -32*64, rdi, -32*YMM_SCD1+512*YMM_SCD1+128*YMM_SCD2 ;; Test loop counter, restore source pointer, next sin/cos ptr

;; Do FFT levels 7,8
;; On input the 64-byte cache lines hold these data values:
;;	0	+1K	+1K	+1K	128	+1K	+1K	+1K
;;	4	...
;;	...
;;	124	...
;;	256	...
;;	...
;;	1	...
;;	...
;;	4096	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+1K	+1K	+1K	32	+1K	+1K	+1K
;;	4	...
;;	...
;;	28	...
;;	64	...
;;	...
;;	1	...
;;	...
;;	4096	...
;;	...

;; Do 1536 four_complex_fft operations
;;	distance between fft data elements is 32
;;	do 384 macros each processing 32 data values

;; Do FFT levels 9,10
;; On input the 64-byte cache lines hold these data values:
;;	0	+1K	+1K	+1K	32	+1K	+1K	+1K
;;	4	...
;;	...
;;	28	...
;;	64	...
;;	...
;;	1	...
;;	...
;;	4096	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+1K	+1K	+1K	8	+1K	+1K	+1K
;;	4	...
;;	16	...
;;	...
;;	1	...
;;	...
;;	4096	...
;;	...

;; Do 1536 four_complex_fft operations
;;	distance between fft data elements is 8
;;	do 384 macros each processing 32 data values

;; Do FFT levels 11,12,13 as well as inverse FFT levels 11,12,13
;; On input and output the 64-byte cache lines hold these data values:
;;	0	+1K	+1K	+1K	8	+1K	+1K	+1K
;;	4	...
;;	16	...
;;	...
;;	1	...
;;	...
;;	4096	...
;;	...

;; Do 768 eight_complex_fft operations
;;	distance between fft data elements is 1
;;	do 192 macros each processing 64 data values

;; Do inverse FFT levels 9,10
;; On input the 64-byte cache lines hold these data values:
;;	0	+1K	+1K	+1K	8	+1K	+1K	+1K
;;	4	...
;;	16	...
;;	...
;;	1	...
;;	...
;;	4096	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+1K	+1K	+1K	32	+1K	+1K	+1K
;;	4	...
;;	...
;;	28	...
;;	64	...
;;	...
;;	1	...
;;	...
;;	4096	...
;;	...

;; Do 1536 four_complex_unfft operations
;;	distance between fft data elements is 8
;;	do 384 macros each processing 32 data values

;; Do inverse FFT levels 7,8
;; On input the 64-byte cache lines hold these data values:
;;	0	+1K	+1K	+1K	32	+1K	+1K	+1K
;;	4	...
;;	...
;;	28	...
;;	64	...
;;	...
;;	1	...
;;	...
;;	4096	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+1K	+1K	+1K	128	+1K	+1K	+1K
;;	4	...
;;	...
;;	124	...
;;	256	...
;;	...
;;	1	...
;;	...
;;	4096	...
;;	...

;; Do 1536 four_complex_unfft operations
;;	distance between fft data elements is 32
;;	do 384 macros each processing 32 data values

;; Do this 128 cache lines (8KB) at a time to maximize accessing data
;; while it is in the L1 cache.

;; Execute the right middle step

	cmp	ffttype, 2
	jl	ypass2_6144_complex_1
	jg	ypass2_6144_complex_3

ypass2_6144_complex_2:
	yloop_init 384			;; 12 iters of 128 cache lines

b4b0:	start_timer 7
	yloop_set_prefetch YQUARTER	;; Prefetch 96 cache lines in this loop
	yr4_b4cl_four_complex_djbfft_preload
b4b:	yr4_b4cl_four_complex_djbfft rsi, 64, 8*64, 16*64, rdi, YMM_SCD2/4, 8
	yloop	8, b4b, -8*64+2*dist64	;; Loop 8 times, next source pointer
	yloop	4, b4b, -4*2*dist64, rdi, -32*YMM_SCD2/4 ;; Loop 4 times, restore source pointer, restore sine/cosine pointer
	loops_undo 4*8			;; Back up loop counter
	end_timer 7

	start_timer 8
	yloop_set_prefetch YQUARTER	;; Prefetch 96 cache lines in this loop
;;	yr4_b4cl_four_complex_djbfft_preload
b5b:	yr4_b4cl_four_complex_djbfft rsi, 64, 2*64, 4*64, rdi, 4*YMM_SCD2/4, 2
	yloop	2, b5b, -2*64+8*64, rdi, -2*4*YMM_SCD2/4 ;; Loop 2 times, next source pointer, restore sine/cosine pointer
	yloop	4, b5b, -4*8*64+2*dist64, rdi, YMM_SCD2/4 ;; Loop 4 times, next source pointer, next sine/cosine pointer
	yloop	4, b5b, -4*2*dist64, rdi, -4*YMM_SCD2/4 ;; Loop 4 times, restore source pointer, restore sine/cosine pointer
	loops_undo 4*4*2		;; Back up loop counter
	end_timer 8

	start_timer 9
	yloop_adjust 4*4		;; 16 fewer iterations this loop
	yloop_set_prefetch YHALF	;; Prefetch 192 cache lines in this loop
	yr8_8cl_eight_complex_with_square_preload
;; Loop below reworked to avoid 4KB distances
;;b6b:	yr8_8cl_eight_complex_with_square rsi, 2*64, 2*dist64, 4*dist64, 64, 32
;;	yloop	32, b6b, -16*2*64	;; 16 iterations!!  Restore source pointer
b6b:	yr8_8cl_eight_complex_with_square rsi, 8*64, 2*dist64, 4*dist64, 64, 4
	yloop	4, b6b, -4*8*64+2*64	;; Loop 4 times, next source pointer
	yloop	8, b6b, -4*2*64		;; 4 iterations!!  Restore source pointer
	loops_undo 8*4			;; Back up loop counter
	end_timer 9

	start_timer 10
	yloop_set_prefetch YQUARTER	;; Prefetch 96 cache lines in this loop
	yr4_b4cl_four_complex_djbunfft_preload
c5b:	yr4_b4cl_four_complex_djbunfft rsi, 64, 2*64, 4*64, rdi, 4*YMM_SCD2/4, 2
	yloop	2, c5b, -2*64+8*64, rdi, -2*4*YMM_SCD2/4 ;; Loop 2 times, next source pointer, restore sine/cosine pointer
	yloop	4, c5b, -4*8*64+2*dist64, rdi, YMM_SCD2/4 ;; Loop 4 times, next source pointer, next sine/cosine pointer
	yloop	4, c5b, -4*2*dist64, rdi, -4*YMM_SCD2/4 ;; Loop 4 times, restore source pointer, restore sine/cosine pointer
	loops_undo 4*4*2		;; Back up loop counter
	end_timer 10

	start_timer 11
	yloop_set_prefetch YQUARTER	;; Prefetch 96 cache lines in this loop
;;	yr4_b4cl_four_complex_djbunfft_preload
c4b:	yr4_b4cl_four_complex_djbunfft rsi, 64, 8*64, 16*64, rdi, YMM_SCD2/4, 8, L1PREFETCH_ALL, 32*64
	yloop	8, c4b, -8*64+2*dist64	;; Loop 8 times, next source pointer
	yloop	4, c4b, -4*2*dist64+32*64, rdi, -32*YMM_SCD2/4 ;; Loop 4 times, next source pointer, restore sine/cosine pointer
	end_timer 11

	yloop	2, b4b0, -2*32*64+dist64 ;; Loop 2 times, next source pointer
	yloop	2, b4b0, -2*dist64+8*dist64 ;; Loop 2 times, next source pointer
	yloop	3, b4b0, -3*8*dist64	;; Loop 3 times, restore source pointer
	jmp	ypass2_6144_complex_unfft


ypass2_6144_complex_1:
	start_timer 6
	bump	rcx, (1536+23-128-2*192-3*384)*64 ;; Move prefetch pointer along
	yloop_init 384			;; 12 iters of 128 cache lines

b4a0:	yloop_set_prefetch YSINGLE	;; Prefetch 384 cache lines in this loop
	yr4_b4cl_four_complex_djbfft_preload
b4a:	yr4_b4cl_four_complex_djbfft rsi, 64, 8*64, 16*64, rdi, YMM_SCD2/4, 8
	yloop	8, b4a, -8*64+2*dist64	;; Loop 8 times, next source pointer
	yloop	4, b4a, -4*2*dist64, rdi, -32*YMM_SCD2/4 ;; Loop 4 times, restore source pointer, restore sine/cosine pointer
	loops_undo 4*8			;; Back up loop counter

	yloop_set_prefetch YSINGLE	;; Prefetch 384 cache lines in this loop
;;	yr4_b4cl_four_complex_djbfft_preload
b5a:	yr4_b4cl_four_complex_djbfft rsi, 64, 2*64, 4*64, rdi, 4*YMM_SCD2/4, 2, L1PREFETCH_ALL, 32*64
	yloop	2, b5a, -2*64+8*64, rdi, -2*4*YMM_SCD2/4 ;; Loop 2 times, next source pointer, restore sine/cosine pointer
	yloop	4, b5a, -4*8*64+2*dist64, rdi, YMM_SCD2/4 ;; Loop 4 times, next source pointer, next sine/cosine pointer
	yloop	4, b5a, -4*2*dist64, rdi, -4*YMM_SCD2/4 ;; Loop 4 times, restore source pointer, restore sine/cosine pointer
	loops_undo 4*4*2		;; Back up loop counter

	yloop_adjust 4*4		;; 16 fewer iterations this loop
	yloop_set_prefetch YDOUBLE	;; Prefetch 384 cache lines in this loop
	yr8_8cl_eight_complex_fft_final_preload
;; Loop below reworked to avoid 4KB distances
;;b6a:	yr8_8cl_eight_complex_fft_final rsi, 2*64, 2*dist64, 4*dist64, 64
;;	yloop	32, b6a, -16*2*64	;; 16 iterations!!  Restore source pointer
b6a:	yr8_8cl_eight_complex_fft_final rsi, 8*64, 2*dist64, 4*dist64, 64
	yloop	4, b6a, -4*8*64+2*64	;; Loop 4 times, next source pointer
	yloop	8, b6a, -4*2*64+32*64	;; 4 iterations!!  Next source pointer

	yloop	2, b4a0, -2*32*64+dist64 ;; Loop 2 times, next source pointer
	yloop	2, b4a0, -2*dist64+8*dist64 ;; Loop 2 times, next source pointer
	yloop	3, b4a0			;; Loop 3 times
	end_timer 6
	jmp	ypass2_6144_complex_done


ypass2_6144_complex_3:
	start_timer 6
x64	mov	r8, DEST2ARG
	mov	rbp, DIST_TO_MULSRCARG
	yloop_init 384			;; 12 iters of 128 cache lines

b4c0:	yloop_set_prefetch YQUARTER	;; Prefetch 96 cache lines in this loop
	yr4_b4cl_four_complex_djbfft_preload
b4c:	yr4_b4cl_four_complex_djbfft rsi, 64, 8*64, 16*64, rdi, YMM_SCD2/4, 8
	yloop	8, b4c, -8*64+2*dist64	;; Loop 8 times, next source pointer
	yloop	4, b4c, -4*2*dist64, rdi, -32*YMM_SCD2/4 ;; Loop 4 times, restore source pointer, restore sine/cosine pointer
	loops_undo 4*8			;; Back up loop counter

	yloop_set_prefetch YQUARTER	;; Prefetch 96 cache lines in this loop
;;	yr4_b4cl_four_complex_djbfft_preload
b5c:	yr4_b4cl_four_complex_djbfft rsi, 64, 2*64, 4*64, rdi, 4*YMM_SCD2/4, 2
	yloop	2, b5c, -2*64+8*64, rdi, -2*4*YMM_SCD2/4 ;; Loop 2 times, next source pointer, restore sine/cosine pointer
	yloop	4, b5c, -4*8*64+2*dist64, rdi, YMM_SCD2/4 ;; Loop 4 times, next source pointer, next sine/cosine pointer
	yloop	4, b5c, -4*2*dist64, rdi, -4*YMM_SCD2/4 ;; Loop 4 times, restore source pointer, restore sine/cosine pointer
	loops_undo 4*4*2		;; Back up loop counter

	yloop_adjust 4*4		;; 16 fewer iterations this loop
	yloop_set_prefetch YHALF	;; Prefetch 192 cache lines in this loop
	yr8_8cl_eight_complex_with_mult_preload
;; Loop below reworked to avoid 4KB distances
;;b6c:	yr8_8cl_eight_complex_with_mult rsi, 2*64, 2*dist64, 4*dist64, 64
;;	yloop	32, b6c, -16*2*64	;; 16 iterations!!  Restore source pointer
b6c:	yr8_8cl_eight_complex_with_mult rsi, 8*64, 2*dist64, 4*dist64, 64
	yloop	4, b6c, -4*8*64+2*64	;; Loop 4 times, next source pointer
	yloop	8, b6c, -4*2*64		;; 4 iterations!!  Restore source pointer
x64	add	rsi, r8
	loops_undo 8*4			;; Back up loop counter

	yloop_set_prefetch YQUARTER	;; Prefetch 96 cache lines in this loop
	yr4_b4cl_four_complex_djbunfft_preload
c5c:	yr4_b4cl_four_complex_djbunfft rsi, 64, 2*64, 4*64, rdi, 4*YMM_SCD2/4, 2
	yloop	2, c5c, -2*64+8*64, rdi, -2*4*YMM_SCD2/4 ;; Loop 2 times, next source pointer, restore sine/cosine pointer
	yloop	4, c5c, -4*8*64+2*dist64, rdi, YMM_SCD2/4 ;; Loop 4 times, next source pointer, next sine/cosine pointer
	yloop	4, c5c, -4*2*dist64, rdi, -4*YMM_SCD2/4 ;; Loop 4 times, restore source pointer, restore sine/cosine pointer
	loops_undo 4*4*2		;; Back up loop counter

	yloop_set_prefetch YQUARTER	;; Prefetch 96 cache lines in this loop
;;	yr4_b4cl_four_complex_djbunfft_preload
c4c:	yr4_b4cl_four_complex_djbunfft rsi, 64, 8*64, 16*64, rdi, YMM_SCD2/4, 8, L1PREFETCH_ALL, 32*64
	yloop	8, c4c, -8*64+2*dist64	;; Loop 8 times, next source pointer
	yloop	4, c4c, -4*2*dist64+32*64, rdi, -32*YMM_SCD2/4 ;; Loop 4 times, next source pointer, restore sine/cosine pointer

x64	sub	rsi, r8
	yloop	2, b4c0, -2*32*64+dist64 ;; Loop 2 times, next source pointer
	yloop	2, b4c0, -2*dist64+8*dist64 ;; Loop 2 times, next source pointer
	yloop	3, b4c0, -3*8*dist64	;; Loop 3 times, restore source pointer
x64	add	rsi, r8
	end_timer 6
	jmp	ypass2_6144_complex_unfft


ypass2_6144_complex_4:
	start_timer 6
	mov	rdi, xsincos_complex	;; The sin/cos ptrs
	bump	rdi, 512*YMM_SCD1+128*YMM_SCD2
	mov	rbp, DIST_TO_MULSRCARG
	yloop_init 384			;; 12 iters of 128 cache lines

b6d0:	yloop_adjust 4*4		;; 16 fewer iterations this loop
	yloop_set_prefetch YDOUBLE	;; Prefetch 384 cache lines in this loop
	yr8_8cl_eight_complex_with_mulf_preload
;; Loop below reworked to avoid 4KB distances
;;b6d:	yr8_8cl_eight_complex_with_mulf rsi, 2*64, 2*dist64, 4*dist64, 64
;;	yloop	32, b6d, -16*2*64	;; 16 iterations!!  Restore source pointer
b6d:	yr8_8cl_eight_complex_with_mulf rsi, 8*64, 2*dist64, 4*dist64, 64
	yloop	4, b6d, -4*8*64+2*64	;; Loop 4 times, next source pointer
	yloop	8, b6d, -4*2*64		;; 4 iterations!!  Restore source pointer
	loops_undo 8*4			;; Back up loop counter

	yloop_set_prefetch YSINGLE	;; Prefetch 384 cache lines in this loop
	yr4_b4cl_four_complex_djbunfft_preload
c5d:	yr4_b4cl_four_complex_djbunfft rsi, 64, 2*64, 4*64, rdi, 4*YMM_SCD2/4, 2
	yloop	2, c5d, -2*64+8*64, rdi, -2*4*YMM_SCD2/4 ;; Loop 2 times, next source pointer, restore sine/cosine pointer
	yloop	4, c5d, -4*8*64+2*dist64, rdi, YMM_SCD2/4 ;; Loop 4 times, next source pointer, next sine/cosine pointer
	yloop	4, c5d, -4*2*dist64, rdi, -4*YMM_SCD2/4 ;; Loop 4 times, restore source pointer, restore sine/cosine pointer
	loops_undo 4*4*2		;; Back up loop counter

	yloop_set_prefetch YSINGLE	;; Prefetch 384 cache lines in this loop
;;	yr4_b4cl_four_complex_djbunfft_preload
c4d:	yr4_b4cl_four_complex_djbunfft rsi, 64, 8*64, 16*64, rdi, YMM_SCD2/4, 8, L1PREFETCH_ALL, 32*64
	yloop	8, c4d, -8*64+2*dist64	;; Loop 8 times, next source pointer
	yloop	4, c4d, -4*2*dist64+32*64, rdi, -32*YMM_SCD2/4 ;; Loop 4 times, next source pointer, restore sine/cosine pointer

	yloop	2, b6d0, -2*32*64+dist64 ;; Loop 2 times, next source pointer
	yloop	2, b6d0, -2*dist64+8*dist64 ;; Loop 2 times, next source pointer
	yloop	3, b6d0, -3*8*dist64	;; Loop 3 times, restore source pointer
	bump	rcx, (-3*384+128+2*192+2*96+192+2*96)*64 ;; Move prefetch pointer to where squaring would be
	end_timer 6

;; Do inverse FFT levels 5,6
;; On input the 64-byte cache lines hold these data values:
;;	0	+1K	+1K	+1K	128	+1K	+1K	+1K
;;	4	...
;;	...
;;	124	...
;;	256	...
;;	...
;;	1	...
;;	...
;;	4096	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+1K	+1K	+1K	512	+1K	+1K	+1K
;;	4	...
;;	...
;;	508	...
;;	1	...
;;	...
;;	4096	...
;;	...

;; Do 1536 four_complex_unfft operations
;;	distance between fft data elements is 128
;;	do 384 macros each processing 32 data values

;; Do inverse FFT levels 3,4
;; On input the 64-byte cache lines hold these data values:
;;	0	+1K	+1K	+1K	512	+1K	+1K	+1K
;;	4	...
;;	...
;;	508	...
;;	1	...
;;	...
;;	4096	...
;;	...
;; On output the 64-byte cache lines are swizzled to hold these data values:
;;	0	+1	+1	+1	2048	+1	+1	+1
;;	4	...
;;	...
;;	2044	...
;;	4096	...
;;	...

;; Do 1536 four_complex_unfft operations
;;	distance between fft data elements is 512
;;	do 384 macros each processing 32 data values

;; Do inverse FFT levels 1,2
;; On input the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	2048	+1	+1	+1
;;	4	...
;;	...
;;	2044	...
;;	4096	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	6144	+1	+1	+1
;;	4	...
;;	...
;;	6140

;; Do 2048 three_complex_unfft operations
;;	distance between fft data elements is 2048
;;	do 512 macros each processing 24 data values

ypass2_6144_complex_unfft:
	mov	rdi, xsincos_complex	;; Load sin/cos pointers
	lea	rbp, [rdi+512*YMM_SCD1]
	mov	rdx, rbp
	bump	rcx, (1536+23-128-2*192-2*96-192-2*96-2*192-128)*64 ;; Move prefetch pointer along
	yloop_init 512			;; 512 iterations or 384 iterations

c3a:	start_timer 12
	yloop_adjust 4			;; Four fewer iterations this loop
	yloop_set_prefetch YHALF	;; Prefetch 192 cache lines in this loop
	yr4_eb4cl_four_complex_djbunfft_preload
c3b:	yr4_eb4cl_four_complex_djbunfft rsi, 2*dist64, 32*64, dist64, rdx, YMM_SCD2, 4
	yloop	4, c3b, 0, rdx, -4*YMM_SCD2 ;; Loop 4 times, restore sine/cosine pointer
	yloop	4, c3b, -3*4*2*dist64, rdx, 4*YMM_SCD2 ;; 3 iterations!!  Restore source pointer, next sine/cosine pointer
	loops_undo 4*4			;; Back up loop counter
	end_timer 12

	start_timer 13
	yloop_adjust 4			;; Four fewer iterations this loop
	yloop_set_prefetch YHALF	;; Prefetch 192 cache lines in this loop
	yr4_s4cl_four_complex_djbunfft_preload
c2b:	yr4_s4cl_four_complex_djbunfft rsi, 32*64, 2*dist64, 4*dist64, rbp, 32*YMM_SCD2, 2, L1PREFETCH_ALL, 64
	yloop	2, c2b, -2*32*64+dist64	;; Test loop counter, next source pointer
	yloop	2, c2b, -2*dist64+8*dist64, rbp, -4*32*YMM_SCD2 ;; Test loop counter, next source pointer, restore sine/cosine pointer
	yloop	4, c2b, -3*8*dist64, rbp, YMM_SCD2 ;; 3 iterations!!  Restore source pointer, next sine/cosine pointer
	loops_undo 4*4			;; Back up loop counter
	end_timer 13

	start_timer 14
	yloop_set_prefetch YQUARTER	;; Prefetch 128 cache lines in this loop
	yr3_3cl_three_complex_djbunfft_preload
c1b:	yr3_3cl_three_complex_djbunfft rsi, 32*64, 8*dist64, rdi, 32*YMM_SCD1, 2
	yloop	2, c1b, -2*32*64+dist64	;; Test loop counter, next source pointer
	yloop	8, c1b, -8*dist64+64, rdi, -16*32*YMM_SCD1+YMM_SCD1 ;; Test loop counter, next source pointer, next sine/cosine pointer
	end_timer 14

	yloop	32, c3a			;; Loop 32 times

ypass2_6144_complex_done:
	ENDM


; *************** radix4-pass2-7680-real macro ******************
; This macro takes 15360 real values and performs the final 13.907 levels of the FFT.
; The results are squared, and the first 13.907 levels of the inverse FFT are done.

yr4dwpn_pass2_7680_real MACRO
	LOCAL	b1b, b2a, b2b, b3a, b3b, b4a, b4b, b5a, b5b, b6a, b6b, b6c, b6d
	LOCAL	c1b, c2a, c2b, c3a, c3b, c4a, c4b, c5a, c5b
	LOCAL	ypass2_7680_real_1, ypass2_7680_real_2
	LOCAL	ypass2_7680_real_3, ypass2_7680_real_4
	LOCAL	ypass2_7680_real_unfft, ypass2_7680_real_done

;; Type 4 FFTs skip the forward FFT process

	start_timer 2
	mov	rsi, DATA_ADDR		;; Load source address
	mov	rbx, DIST_TO_FFTSRCARG
	mov	rcx, DATA_PREFETCH	;; Prefetch pointer (we need to prefetch 
					;; 1920 cache lines + 2*29 pad cache lines)
	cmp	ffttype, 4
	je	ypass2_7680_real_4

;; Do FFT levels 1,2,3 which correspond to complex levels 1,2
;; On input the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	7680	+1	+1	+1
;;	4	...
;;	...
;;	7676
;; On output the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	2560	+1	+1	+1
;;	4	...
;;	...
;;	2556	...
;;	5120	...
;;	...

;; Do 2560 six_reals_fft operations
;;	distance between fft data elements is 2560
;;	do 640 macros each processing 24 data values

	mov	rdi, sincos3		;; Real sin/cos pointer
	mov	rbp, xsincos_complex	;; Complex sin/cos data
	yloop_init 640			;; 640 iterations
	yloop_set_prefetch YHALF
	yr3_f3cl_2sc_six_reals_fft_preload
b1b:	yr3_f3cl_2sc_six_reals_fft rsi, 64, 10*dist64, rbp, YMM_SCD1, rdi, YMM_SCD1, 64
	yloop	64, b1b, -64*64+dist64	;; Test loop counter, next source pointer
	yloop	10, b1b, -10*dist64	;; Test loop counter, restore source pointer

;; Do FFT levels 3,4
;; On input the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	2560	+1	+1	+1
;;	4	...
;;	...
;;	2556	...
;;	5120	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	512	+1	+1	+1
;;	4	...
;;	...
;;	508	...
;;	1024	...
;;	...

;; Do 512 ten_reals_fft operations
;; Do 1024 five_complex_fft operations
;;	distance between fft data elements is 512
;;	do 384 macros each processing 40 data values

	yloop_init 384			;; 3 iters of 128
	yloop_set_prefetch YSINGLE
	yr5_5cl_2sc_ten_reals_fft_preload
b2a:	yr5_5cl_2sc_ten_reals_fft rsi, 64, 2*dist64, rbp, YMM_SCD2, rdi, YMM_SCD2, 64
	yloop	64, b2a, -64*64+dist64	;; Test loop counter, next source pointer
	yloop	2, b2a, -2*dist64+10*dist64, rbp, -128*YMM_SCD2 ;; Test loop counter, next source pointer, restore sine/cosine pointer

	loops_reset			;; 2 more iters of 128
	yr5_5cl_five_complex_djbfft_preload
b2b:	yr5_5cl_five_complex_djbfft rsi, 64, 2*dist64, rbp, YMM_SCD2, 64
	yloop	64, b2b, -64*64+dist64	;; Test loop counter, next source pointer
	yloop	2, b2b, -2*dist64+10*dist64, rbp, -128*YMM_SCD2 ;; Test loop counter, next source pointer, restore sin/cos ptr
	yloop	3, b2b, -3*10*dist64, rbp, 128*YMM_SCD2 ;; Loop 3 times, restore source pointer, next sin/cos pointer

;; Do FFT levels 5,6
;; On input the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	512	+1	+1	+1
;;	4	...
;;	...
;;	508	...
;;	1024	...
;;	...
;; On output the 64-byte cache lines are swizzled to hold these data values:
;;	0	+256	+256	+256	128	+256	+256	+256
;;	4	...
;;	...
;;	124	...
;;	1	...
;;	...
;;	1024	...
;;	...

;; Do 128 eight_reals_fft operations
;; Do 1792 four_complex_fft operations
;;	distance between fft data elements is 128
;;	do 480 macros each processing 32 data values

	yloop_init 480			;; 15 iterations of 32
	yloop_set_prefetch YHALF
	yr4_s4cl_2sc_eight_reals_fft_preload
b3a:	yr4_s4cl_2sc_eight_reals_fft rsi, 64, 32*64, dist64, rbp, YMM_SCD2, rdi, YMM_SCD2, 32
	yloop	32, b3a, -32*64+2*dist64, rbp, -32*YMM_SCD2 ;; Test loop counter, next source pointer, restore sine/cosine pointer

	loops_reset			;; 14 more iters of 32
	yr4_s4cl_four_complex_djbfft_preload
b3b:	yr4_s4cl_four_complex_djbfft rsi, 64, 32*64, dist64, rbp, YMM_SCD2, 32
	yloop	32, b3b, -32*64+2*dist64, rbp, -32*YMM_SCD2 ;; Test loop counter, next source pointer, restore sin/cos ptr
	yloop	15, b3b, -15*2*dist64	;; Loop 15 times, restore source pointer

;; Do FFT levels 7,8
;; On input the 64-byte cache lines hold these data values:
;;	0	+256	+256	+256	128	+256	+256	+256
;;	4	...
;;	...
;;	124	...
;;	1	...
;;	...
;;	1024	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+256	+256	+256	32	+256	+256	+256
;;	4	...
;;	...
;;	28	...
;;	64	...
;;	...
;;	1	...
;;	...
;;	1024	...
;;	...

;; Do 32 eight_reals_fft operations
;; Do 1888 four_complex_fft operations
;;	distance between fft data elements is 32
;;	do 480 macros each processing 32 data values

	yloop_init 480			;; 15 iterations of 32
	yloop_set_prefetch YHALF
	yr4_4cl_eight_reals_four_complex_djbfft_preload
b4a:	yr4_4cl_eight_reals_four_complex_djbfft rsi, 32*64, 8*64, 16*64, rdi, YMM_SCD3, 2
	yloop	2, b4a, -2*32*64+dist64	;; Test loop counter, next source pointer
	yloop	2, b4a, -2*dist64+64	;; Test loop counter, next source pointer
	yloop	8, b4a, -8*64+2*dist64, rdi, -32*YMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer

	loops_reset			;; 14 more iters of 32
	yr4_rb4cl_four_complex_djbfft_preload
b4b:	yr4_rb4cl_four_complex_djbfft rsi, 32*64, 8*64, 16*64, rdi, YMM_SCD3, 2
	yloop	2, b4b, -2*32*64+dist64	;; Test loop counter, next source pointer
	yloop	2, b4b, -2*dist64+64	;; Test loop counter, next source pointer
	yloop	8, b4b, -8*64+2*dist64, rdi, -32*YMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer
	yloop	15, b4b, -15*2*dist64	;; Test loop counter, restore source pointer

;; Do FFT levels 9,10
;; On input the 64-byte cache lines hold these data values:
;;	0	+256	+256	+256	32	+256	+256	+256
;;	4	...
;;	...
;;	28	...
;;	64	...
;;	...
;;	1	...
;;	...
;;	1024	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+256	+256	+256	8	+256	+256	+256
;;	4	...
;;	16	...
;;	...
;;	1	...
;;	...
;;	1024	...
;;	...

;; Do 8 eight_reals_fft operations
;; Do 1912 four_complex_fft operations
;;	distance between fft data elements is 8
;;	do 480 macros each processing 32 data values

	yloop_init 480			;; 60 iterations of 8
	yloop_set_prefetch YHALF
	yr4_4cl_eight_reals_four_complex_djbfft_preload
b5a:	yr4_4cl_eight_reals_four_complex_djbfft rsi, 32*64, 2*64, 4*64, rdi, 4*YMM_SCD3, 2
	yloop	2, b5a, -2*32*64+dist64	;; Test loop counter, next source pointer
	yloop	2, b5a, -2*dist64+64	;; Test loop counter, next source pointer
	yloop	2, b5a, -2*64+8*64, rdi, -8*4*YMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer

	loops_reset			;; 59 more iters of 8
	yr4_rb4cl_four_complex_djbfft_preload
b5b:	yr4_rb4cl_four_complex_djbfft rsi, 32*64, 2*64, 4*64, rdi, 4*YMM_SCD3, 2
	yloop	2, b5b, -2*32*64+dist64	;; Test loop counter, next source pointer
	yloop	2, b5b, -2*dist64+64	;; Test loop counter, next source pointer
	yloop	2, b5b, -2*64+8*64, rdi, -8*4*YMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer
	yloop	4, b5b, -4*8*64+2*dist64 ;; Test loop counter, next source pointer
	yloop	15, b5b, -15*2*dist64	;; Test loop counter, restore source pointer

;; Do FFT levels 11,12,13 as well as inverse FFT levels 11,12,13
;; On input and output the 64-byte cache lines hold these data values:
;;	0	+256	+256	+256	8	+256	+256	+256
;;	4	...
;;	16	...
;;	...
;;	1	...
;;	...
;;	1024	...
;;	...

;; Do 1 sixteen_reals_fft operation
;; Do 959 eight_complex_fft operations
;;	distance between fft data elements is 1
;;	do 240 macros each processing 64 data values

	cmp	ffttype, 2
	jl	ypass2_7680_real_1
	jg	ypass2_7680_real_3

ypass2_7680_real_2:
;;	yr8_8cl_sixteen_reals_eight_complex_with_square rsi, 2*64, 32*64, dist64, 64
	yr8_dispatch call, y8real_square, 32*64, dist64, 64
	yloop_init 240			;; 239 more iterations
	yloop_adjust 1
	yloop_set_prefetch YHALF
	yr8_8cl_eight_complex_with_square_preload
b6b:	yr8_8cl_eight_complex_with_square rsi, 2*64, 32*64, dist64, 64, 1
	yloop	16, b6b, -16*2*64+2*dist64	;; Test loop counter, next source pointer
	yloop	15, b6b, -15*2*dist64	;; Test loop counter, restore source pointer
	jmp	ypass2_7680_real_unfft

ypass2_7680_real_1:
;;	yr8_8cl_sixteen_reals_eight_complex_fft_final rsi, 2*64, 32*64, dist64, 64
	yr8_dispatch call, y8real_fft_final, 32*64, dist64, 64
	mov	rcx, xsincos_complex	;; Prefetch complex sin/cos data, 960 cache lines
	yloop_init 240			;; 239 more iterations
	yloop_adjust 1
	yloop_set_prefetch YDOUBLE
b6a:	yr8_8cl_eight_complex_fft_final rsi, 2*64, 32*64, dist64, 64
	yloop	16, b6a, -16*2*64+2*dist64	;; Test loop counter, next source pointer
	yloop	15, b6a			;; Test loop counter
	jmp	ypass2_7680_real_done

ypass2_7680_real_3:
x64	mov	r8, DEST2ARG
	mov	rbp, DIST_TO_MULSRCARG
;;	yr8_8cl_sixteen_reals_eight_complex_with_mult rsi, 2*64, 32*64, dist64, 64
	yr8_dispatch call, y8real_mult, 32*64, dist64, 64
	yloop_init 240			;; 239 more iterations
	yloop_adjust 1
	yloop_set_prefetch YHALF
	yr8_8cl_eight_complex_with_mult_preload
b6c:	yr8_8cl_eight_complex_with_mult rsi, 2*64, 32*64, dist64, 64
	yloop	16, b6c, -16*2*64+2*dist64	;; Test loop counter, next source pointer
	yloop	15, b6c, -15*2*dist64	;; Test loop counter, restore source pointer
x64	add	rsi, r8
	jmp	ypass2_7680_real_unfft

ypass2_7680_real_4:
	mov	rbp, DIST_TO_MULSRCARG
;;	yr8_8cl_sixteen_reals_eight_complex_with_mulf rsi, 2*64, 32*64, dist64, 64
	yr8_dispatch call, y8real_mulf, 32*64, dist64, 64
	yloop_init 240			;; 239 more iterations
	yloop_adjust 1
	yloop_set_prefetch YDOUBLE
	yr8_8cl_eight_complex_with_mulf_preload
b6d:	yr8_8cl_eight_complex_with_mulf rsi, 2*64, 32*64, dist64, 64
	yloop	16, b6d, -16*2*64+2*dist64	;; Test loop counter, next source pointer
	yloop	15, b6d, -15*2*dist64	;; Test loop counter, restore source pointer
;;	bump	rcx, (-478+320+384+3*240+120)*64 ;; Move prefetch pointer back to where squaring would be

;; Do inverse FFT levels 9,10
;; On input the 64-byte cache lines hold these data values:
;;	0	+256	+256	+256	8	+256	+256	+256
;;	4	...
;;	16	...
;;	...
;;	1	...
;;	...
;;	1024	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+256	+256	+256	32	+256	+256	+256
;;	4	...
;;	...
;;	28	...
;;	64	...
;;	...
;;	1	...
;;	...
;;	1024	...
;;	...

;; Do 8 eight_reals_unfft operations
;; Do 1912 four_complex_unfft operations
;;	distance between fft data elements is 8
;;	do 480 macros each processing 32 data values

ypass2_7680_real_unfft:
	mov	rdi, sincos3		;; Load sin/cos pointer
	bump	rdi, 32*YMM_SCD2+128*YMM_SCD2+640*YMM_SCD1
	yloop_init 480			;; 60 iterations of 8
	yloop_set_prefetch YHALF
	yr4_4cl_eight_reals_four_complex_djbunfft_preload
c5a:	yr4_4cl_eight_reals_four_complex_djbunfft rsi, 32*64, 2*64, 4*64, rdi, 4*YMM_SCD3, 2
	yloop	2, c5a, -2*32*64+dist64	;; Test loop counter, next source pointer
	yloop	2, c5a, -2*dist64+64	;; Test loop counter, next source pointer
	yloop	2, c5a, -2*64+8*64, rdi, -8*4*YMM_SCD3 ;; Test loop counter, next source pointer, restore sine/cosine pointer

	loops_reset			;; 59 more iters of 8
	yr4_rb4cl_four_complex_djbunfft_preload
c5b:	yr4_rb4cl_four_complex_djbunfft rsi, 32*64, 2*64, 4*64, rdi, 4*YMM_SCD3, 2
	yloop	2, c5b, -2*32*64+dist64	;; Test loop counter, next source pointer
	yloop	2, c5b, -2*dist64+64	;; Test loop counter, next source pointer
	yloop	2, c5b, -2*64+8*64, rdi, -8*4*YMM_SCD3 ;; Test loop counter, next source pointer, restore sine/cosine pointer
	yloop	4, c5b, -4*8*64+2*dist64	;; Test loop counter, next source pointer
	yloop	15, c5b, -15*2*dist64	;; Test loop counter, restore source pointer

;; Do inverse FFT levels 7,8
;; On input the 64-byte cache lines hold these data values:
;;	0	+256	+256	+256	32	+256	+256	+256
;;	4	...
;;	...
;;	28	...
;;	64	...
;;	...
;;	1	...
;;	...
;;	1024	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+256	+256	+256	128	+256	+256	+256
;;	4	...
;;	...
;;	124	...
;;	1	...
;;	...
;;	1024	...
;;	...

;; Do 32 eight_reals_unfft operations
;; Do 1888 four_complex_unfft operations
;;	distance between fft data elements is 32
;;	do 480 macros each processing 32 data values

	bump	rcx, (1920+2*29-320-384-3*240-120-240-240)*64 ;; Move prefetch pointer along
	yloop_init 480			;; 15 iterations of 32
	yloop_set_prefetch YHALF
	yr4_4cl_eight_reals_four_complex_djbunfft_preload
c4a:	yr4_4cl_eight_reals_four_complex_djbunfft rsi, 32*64, 8*64, 16*64, rdi, YMM_SCD3, 2
	yloop	2, c4a, -2*32*64+dist64	;; Test loop counter, next source pointer
	yloop	2, c4a, -2*dist64+64	;; Test loop counter, next source pointer
	yloop	8, c4a, -8*64+2*dist64, rdi, -32*YMM_SCD3 ;; Test loop counter, next source pointer, restore sine/cosine pointer

	loops_reset			;; 14 more iters of 32
	yr4_rb4cl_four_complex_djbunfft_preload
c4b:	yr4_rb4cl_four_complex_djbunfft rsi, 32*64, 8*64, 16*64, rdi, YMM_SCD3, 2
	yloop	2, c4b, -2*32*64+dist64	;; Test loop counter, next source pointer
	yloop	2, c4b, -2*dist64+64	;; Test loop counter, next source pointer
	yloop	8, c4b, -8*64+2*dist64, rdi, -32*YMM_SCD3 ;; Test loop counter, next source pointer, restore sine/cosine pointer
	yloop	15, c4b, -15*2*dist64	;; Test loop counter, restore source pointer

;; Do inverse FFT levels 5,6
;; On input the 64-byte cache lines hold these data values:
;;	0	+256	+256	+256	128	+256	+256	+256
;;	4	...
;;	...
;;	124	...
;;	1	...
;;	...
;;	1024	...
;;	...
;; On output the 64-byte cache lines are swizzled to hold these data values:
;;	0	+1	+1	+1	512	+1	+1	+1
;;	4	...
;;	...
;;	508	...
;;	1024	...
;;	...

;; Do 128 eight_reals_unfft operations
;; Do 1792 four_complex_unfft operations
;;	distance between fft data elements is 128
;;	do 480 macros each processing 32 data values

	bump	rdi, -32*YMM_SCD2	;; Real sin/cos ptr
	mov	rbp, xsincos_complex	;; Complex sin/cos data
	bump	rbp, 128*YMM_SCD2+640*YMM_SCD1
	mov	rcx, xsincos_complex	;; Prefetch complex sin/cos data, 960 cache lines
	yloop_init 480			;; 15 iters of 32
	yloop_set_prefetch YSINGLE
	yr4_s4cl_2sc_eight_reals_unfft_preload
c3a:	yr4_s4cl_2sc_eight_reals_unfft rsi, 64, 32*64, dist64, rbp, YMM_SCD2, rdi, YMM_SCD2, 32
	yloop	32, c3a, -32*64+2*dist64, rbp, -32*YMM_SCD2 ;; Test loop counter, next source pointer, restore sine/cosine pointer

	loops_reset			;; 14 more iters of 32
	yr4_s4cl_four_complex_djbunfft_preload
c3b:	yr4_s4cl_four_complex_djbunfft rsi, 64, 32*64, dist64, rbp, YMM_SCD2, 32
	yloop	32, c3b, -32*64+2*dist64, rbp, -32*YMM_SCD2 ;; Test loop counter, next source pointer, restore sine/cosine pointer
	yloop	15, c3b, -15*2*dist64	;; Test loop counter, restore source pointer

;; Do inverse FFT levels 3,4
;; On output the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	512	+1	+1	+1
;;	4	...
;;	...
;;	508	...
;;	1024	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	2560	+1	+1	+1
;;	4	...
;;	...
;;	2556	...
;;	5120	...
;;	...

;; Do 512 ten_reals_unfft operations
;; Do 1024 five_complex_unfft operations
;;	distance between fft data elements is 512
;;	do 384 macros each processing 40 data values

	bump	rdi, -32*YMM_SCD2-128*YMM_SCD2 ;; Real sin/cos ptr
	bump	rbp, -128*YMM_SCD2	;; Complex sin/cos data
	yloop_init 384			;; 3 iters of 128
	yloop_set_prefetch YHALF
	yr5_5cl_2sc_ten_reals_unfft_preload
c2a:	yr5_5cl_2sc_ten_reals_unfft rsi, 64, 2*dist64, rbp, YMM_SCD2, rdi, YMM_SCD2, 64
	yloop	64, c2a, -64*64+dist64	;; Test loop counter, next source pointer
	yloop	2, c2a, -2*dist64+10*dist64, rbp, -128*YMM_SCD2 ;; Test loop counter, next source pointer, restore sine/cosine pointer

	loops_reset			;; 2 more iters of 128
	yr5_5cl_five_complex_djbunfft_preload
c2b:	yr5_5cl_five_complex_djbunfft rsi, 64, 2*dist64, rbp, YMM_SCD2, 64
	yloop	64, c2b, -64*64+dist64	;; Test loop counter, next source pointer
	yloop	2, c2b, -2*dist64+10*dist64, rbp, -128*YMM_SCD2 ;; Test loop counter, next source pointer, restore sine/cosine pointer
	yloop	3, c2b, -3*10*dist64	;; Test loop counter, restore source pointer

;; Do inverse FFT levels 1,2,3 which correspond to complex levels 1,2
;; On input the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	2560	+1	+1	+1
;;	4	...
;;	...
;;	2556	...
;;	5120	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	7680	+1	+1	+1
;;	4	...
;;	...
;;	7676

;; Do 2560 six_reals_unfft operations
;;	distance between fft data elements is 2560
;;	do 640 macros each processing 24 data values

	bump	rdi, -128*YMM_SCD2-640*YMM_SCD1 ;; Real sin/cos ptr
	bump	rbp, -640*YMM_SCD1	;; Complex sin/cos ptr
	bump	rcx, (960-480-192-320)*64 ;; Move prefetch pointer along
	yloop_init 640			;; 640 iterations
	yloop_set_prefetch YHALF
	yr3_3cl_2sc_six_reals_unfft_preload
c1b:	yr3_3cl_2sc_six_reals_unfft rsi, 64, 10*dist64, rbp, YMM_SCD1, rdi, YMM_SCD1, 64
	yloop	64, c1b, -64*64+dist64	;; Test loop counter, next source pointer
	yloop	10, c1b			;; Test loop counter

ypass2_7680_real_done:
	end_timer 2
	ENDM

; *************** radix4-pass2-7680-complex macro ******************
; This macro takes 7680 complex values and performs the final 12.907 levels of a standard
; radix-4 FFT, squares the results, and does the first 12.907 levels of the inverse FFT.

yr4dwpn_pass2_7680_complex MACRO
	LOCAL	b1a, b1b, b2b, b3a0, b3a, b3b0, b3b, b3c0, b3c, b4a, b4b, b4c, b5a, b5b, b5c, b6a, b6b, b6c, b6d0, b6d
	LOCAL	c1b, c2a, c2b, c3b, c3c, c3d, c4b, c4c, c4d, c5b, c5c, c5d
	LOCAL	ypass2_7680_complex_1, ypass2_7680_complex_2
	LOCAL	ypass2_7680_complex_3, ypass2_7680_complex_4
	LOCAL	ypass2_7680_complex_unfft, ypass2_7680_complex_done

;; Type 4 FFTs skip the forward FFT process

	mov	rsi, DATA_ADDR		;; Load address of FFT data
	mov	rbx, DIST_TO_FFTSRCARG
	mov	rcx, DATA_PREFETCH	;; Prefetch pointer (we need to prefetch 
					;; 1920 cache lines + 2*29 pad cache lines)
	cmp	ffttype, 4
	je	ypass2_7680_complex_4

;; Do FFT level 1,2 (actually 1.585 levels)
;; On input the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	7680	+1	+1	+1
;;	4	...
;;	...
;;	7676
;; On output the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	2560	+1	+1	+1
;;	4	...
;;	...
;;	2556	...
;;	5120	...
;;	...

;; Do 2560 three_complex_fft operations
;;	distance between fft data elements is 2560
;;	do 640 macros each processing 24 data values

;; Do FFT levels 3,4 (actually 2.322 levels)
;; On input the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	2560	+1	+1	+1
;;	4	...
;;	...
;;	2556	...
;;	5120	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	512	+1	+1	+1
;;	4	...
;;	...
;;	508	...
;;	1024	...
;;	...

;; Do 1536 five_complex_fft operations
;;	distance between fft data elements is 512
;;	do 384 macros each processing 40 data values

	mov	rdi, xsincos_complex	;; The sin/cos ptrs
	lea	rbp, [rdi+640*YMM_SCD1]
	yloop_init 1024			;; 640 iterations or 384 iterations

b1a:	start_timer 4
	yloop_adjust 3			;; Three fewer iterations this loop
	yloop_set_prefetch YOTHER, 8	;; Prefetch 128 cache lines
	yr3_f3cl_three_complex_djbfft_preload
b1b:	yr3_f3cl_three_complex_djbfft rsi, 2*dist64, 10*dist64, rdi, 128*YMM_SCD1, 5
	yloop	8, b1b, -5*2*dist64, rdi, -5*128*YMM_SCD1+YMM_SCD1 ;; 5 iterations!!  Restore source pointer, next sin/cos data
	loops_undo 8			;; Back up loop counter
	end_timer 4

	start_timer 5
	yloop_adjust 5			;; Five fewer iterations this loop
	yloop_set_prefetch YOTHER, 8	;; Prefetch 128 cache lines
	yr5_5cl_five_complex_djbfft_preload
b2b:	yr5_5cl_five_complex_djbfft rsi, 10*dist64, 2*dist64, rbp, 0, 3, L1PREFETCH_ALL, 64
	yloop	8, b2b, -3*10*dist64+64, rbp, YMM_SCD2 ;; 3 iterations!!  Next source pointer, next sin/cos data
	end_timer 5

	yloop	64, b1a, -64*64+dist64	;; Test loop counter, next source pointer
	yloop	2, b1a, -2*dist64, rdi, -128*YMM_SCD1+640*YMM_SCD1+128*YMM_SCD2 ;; Test loop counter, restore source pointer, next sin/cos ptr

;; Do FFT levels 5,6
;; On input the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	512	+1	+1	+1
;;	4	...
;;	...
;;	508	...
;;	1024	...
;;	...
;; On output the 64-byte cache lines are swizzled to hold these data values:
;;	0	+256	+256	+256	128	+256	+256	+256
;;	4	...
;;	...
;;	124	...
;;	1	...
;;	...
;;	1024	...
;;	...

;; Do 1920 four_complex_fft operations
;;	distance between fft data elements is 128
;;	do 480 macros each processing 32 data values

;; Do FFT levels 7,8
;; On input the 64-byte cache lines hold these data values:
;;	0	+256	+256	+256	128	+256	+256	+256
;;	4	...
;;	...
;;	124	...
;;	1	...
;;	...
;;	1024	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+256	+256	+256	32	+256	+256	+256
;;	4	...
;;	...
;;	28	...
;;	64	...
;;	...
;;	1	...
;;	...
;;	1024	...
;;	...

;; Do 1920 four_complex_fft operations
;;	distance between fft data elements is 32
;;	do 480 macros each processing 32 data values

;; Do FFT levels 9,10
;; On input the 64-byte cache lines hold these data values:
;;	0	+256	+256	+256	32	+256	+256	+256
;;	4	...
;;	...
;;	28	...
;;	64	...
;;	...
;;	1	...
;;	...
;;	1024	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+256	+256	+256	8	+256	+256	+256
;;	4	...
;;	16	...
;;	...
;;	1	...
;;	...
;;	1024	...
;;	...

;; Do 1920 four_complex_fft operations
;;	distance between fft data elements is 8
;;	do 480 macros each processing 32 data values

;; Do FFT levels 11,12,13 as well as inverse FFT levels 11,12,13
;; On input and output the 64-byte cache lines hold these data values:
;;	0	+256	+256	+256	8	+256	+256	+256
;;	4	...
;;	16	...
;;	...
;;	1	...
;;	...
;;	1024	...
;;	...

;; Do 960 eight_complex_fft operations
;;	distance between fft data elements is 1
;;	do 240 macros each processing 64 data values

;; Do inverse FFT levels 9,10
;; On input the 64-byte cache lines hold these data values:
;;	0	+256	+256	+256	8	+256	+256	+256
;;	4	...
;;	16	...
;;	...
;;	1	...
;;	...
;;	1024	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+256	+256	+256	32	+256	+256	+256
;;	4	...
;;	...
;;	28	...
;;	64	...
;;	...
;;	1	...
;;	...
;;	1024	...
;;	...

;; Do 1920 four_complex_unfft operations
;;	distance between fft data elements is 8
;;	do 480 macros each processing 32 data values

;; Do inverse FFT levels 7,8
;; On input the 64-byte cache lines hold these data values:
;;	0	+256	+256	+256	32	+256	+256	+256
;;	4	...
;;	...
;;	28	...
;;	64	...
;;	...
;;	1	...
;;	...
;;	1024	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+256	+256	+256	128	+256	+256	+256
;;	4	...
;;	...
;;	124	...
;;	1	...
;;	...
;;	1024	...
;;	...

;; Do 1920 four_complex_unfft operations
;;	distance between fft data elements is 32
;;	do 480 macros each processing 32 data values

;; Do inverse FFT levels 5,6
;; On input the 64-byte cache lines hold these data values:
;;	0	+256	+256	+256	128	+256	+256	+256
;;	4	...
;;	...
;;	124	...
;;	1	...
;;	...
;;	1024	...
;;	...
;; On output the 64-byte cache lines are swizzled to hold these data values:
;;	0	+1	+1	+1	512	+1	+1	+1
;;	4	...
;;	...
;;	508	...
;;	1024	...
;;	...

;; Do 1920 four_complex_unfft operations
;;	distance between fft data elements is 128
;;	do 480 macros each processing 32 data values

;; Do this 128 cache lines (8KB) at a time to maximize accessing data
;; while it is in the L1 cache.

;; Execute the right middle step

	cmp	ffttype, 2
	jl	ypass2_7680_complex_1
	jg	ypass2_7680_complex_3

ypass2_7680_complex_2:
	yloop_init 480			;; 15 iters of 128 cache lines

b3b0:	start_timer 6
	yloop_set_prefetch YHALF	;; Prefetch 240 cache lines in this loop
	yr4_s4cl_four_complex_djbfft_preload
b3b:	yr4_s4cl_four_complex_djbfft rsi, 64, 32*64, dist64, rdi, YMM_SCD2, 32
	yloop	32, b3b, -32*64, rdi, -32*YMM_SCD2 ;; Loop 32 times, restore source pointer, restore sine/cosine pointer
	loops_undo 32			;; Back up loop counter
	end_timer 6

	start_timer 7
	yloop_set_prefetch YHALF	;; Prefetch 240 cache lines in this loop
	yr4_eb4cl_four_complex_djbfft_preload
b4b:	yr4_eb4cl_four_complex_djbfft rsi, 64, 8*64, 16*64, rdi, 4*YMM_SCD2, 8
	yloop	8, b4b, -8*64+32*64, rdi, -8*4*YMM_SCD2+YMM_SCD2 ;; Loop 8 times, next source pointer, next sine/cosine pointer
	yloop	2, b4b, -2*32*64+dist64	;; Loop 2 times, next source pointer
	yloop	2, b4b, -2*dist64, rdi, -4*YMM_SCD2 ;; Loop 2 times, restore source pointer, restore sine/cosine pointer
	loops_undo 4*8			;; Back up loop counter
	end_timer 7

	start_timer 8
	yloop_set_prefetch YHALF	;; Prefetch 240 cache lines in this loop
;;	yr4_eb4cl_four_complex_djbfft_preload
b5b:	yr4_eb4cl_four_complex_djbfft rsi, 64, 2*64, 4*64, rdi, 16*YMM_SCD2, 2
	yloop	2, b5b, -2*64+8*64, rdi, -2*16*YMM_SCD2 ;; Loop 2 times, next source pointer, restore sine/cosine pointer
	yloop	4, b5b, -4*8*64+32*64, rdi, 4*YMM_SCD2 ;; Loop 4 times, next source pointer, next sine/cosine pointer
	yloop	2, b5b, -2*32*64+dist64	;; Test loop counter, next source pointer
	yloop	2, b5b, -2*dist64, rdi, -4*4*YMM_SCD2 ;; Test loop counter, restore source pointer, restore sine/cosine pointer
	loops_undo 4*4*2		;; Back up loop counter
	end_timer 8

	start_timer 9
	yloop_adjust 2*8		;; 16 fewer iterations this loop
	yloop_set_prefetch YQUARTER	;; Prefetch 120 cache lines in this loop
	yr8_8cl_eight_complex_with_square_preload
;; Loop below reworked to avoid 4KB distances
;;b6b:	yr8_8cl_eight_complex_with_square rsi, 2*64, 32*64, dist64, 64, 32
;;	yloop	32, b6b, -16*2*64	;; 16 iterations!!  Restore source pointer
b6b:	yr8_8cl_eight_complex_with_square rsi, 4*64, 32*64, dist64, 64, 8
	yloop	8, b6b, -8*4*64+2*64	;; Loop 8 times, next source pointer
	yloop	4, b6b, -2*2*64		;; 2 iterations!!  Restore source pointer
	loops_undo 4*8			;; Back up loop counter
	end_timer 9

	start_timer 10
	yloop_set_prefetch YHALF	;; Prefetch 240 cache lines in this loop
	yr4_eb4cl_four_complex_djbunfft_preload
c5b:	yr4_eb4cl_four_complex_djbunfft rsi, 64, 2*64, 4*64, rdi, 16*YMM_SCD2, 2
	yloop	2, c5b, -2*64+8*64, rdi, -2*16*YMM_SCD2 ;; Loop 2 times, next source pointer, restore sine/cosine pointer
	yloop	4, c5b, -4*8*64+32*64, rdi, 4*YMM_SCD2 ;; Loop 4 times, next source pointer, next sine/cosine pointer
	yloop	2, c5b, -2*32*64+dist64	;; Test loop counter, next source pointer
	yloop	2, c5b, -2*dist64, rdi, -4*4*YMM_SCD2 ;; Test loop counter, restore source pointer, restore sine/cosine pointer
	loops_undo 4*4*2		;; Back up loop counter
	end_timer 10

	start_timer 11
	yloop_set_prefetch YHALF	;; Prefetch 240 cache lines in this loop
;;	yr4_eb4cl_four_complex_djbunfft_preload
c4b:	yr4_eb4cl_four_complex_djbunfft rsi, 64, 8*64, 16*64, rdi, 4*YMM_SCD2, 8
	yloop	8, c4b, -8*64+32*64, rdi, -8*4*YMM_SCD2+YMM_SCD2 ;; Loop 8 times, next source pointer, next sine/cosine pointer
	yloop	2, c4b, -2*32*64+dist64	;; Loop 2 times, next source pointer
	yloop	2, c4b, -2*dist64, rdi, -4*YMM_SCD2 ;; Loop 2 times, restore source pointer, restore sine/cosine pointer
	loops_undo 4*8			;; Back up loop counter
	end_timer 11

	start_timer 12
	yloop_set_prefetch YHALF	;; Prefetch 240 cache lines in this loop
	yr4_s4cl_four_complex_djbunfft_preload
c3b:	yr4_s4cl_four_complex_djbunfft rsi, 64, 32*64, dist64, rdi, YMM_SCD2, 32, L1PREFETCH_ALL, 2*dist64
	yloop	32, c3b, -32*64+2*dist64, rdi, -32*YMM_SCD2 ;; Loop 32 times, next source pointer, restore sine/cosine pointer
	end_timer 12

	yloop	15, b3b0, -15*2*dist64	;; Loop 15 times, restore source pointer
	jmp	ypass2_7680_complex_unfft


ypass2_7680_complex_1:
	start_timer 6
;;	bump	rcx, (1920+2*29-2*128-3*480-240)*64 ;; Move prefetch pointer along
	yloop_init 480			;; 15 iters of 128 cache lines

b3a0:	yloop_set_prefetch YSINGLE	;; Prefetch 480 cache lines in this loop
	yr4_s4cl_four_complex_djbfft_preload
b3a:	yr4_s4cl_four_complex_djbfft rsi, 64, 32*64, dist64, rdi, YMM_SCD2, 32
	yloop	32, b3a, -32*64, rdi, -32*YMM_SCD2 ;; Loop 32 times, restore source pointer, restore sine/cosine pointer
	loops_undo 32			;; Back up loop counter

	yloop_set_prefetch YSINGLE	;; Prefetch 480 cache lines in this loop
	yr4_eb4cl_four_complex_djbfft_preload
b4a:	yr4_eb4cl_four_complex_djbfft rsi, 64, 8*64, 16*64, rdi, 4*YMM_SCD2, 8
	yloop	8, b4a, -8*64+32*64, rdi, -8*4*YMM_SCD2+YMM_SCD2 ;; Loop 8 times, next source pointer, next sine/cosine pointer
	yloop	2, b4a, -2*32*64+dist64	;; Loop 2 times, next source pointer
	yloop	2, b4a, -2*dist64, rdi, -4*YMM_SCD2 ;; Loop 2 times, restore source pointer, restore sine/cosine pointer
	loops_undo 4*8			;; Back up loop counter

	yloop_set_prefetch YSINGLE	;; Prefetch 480 cache lines in this loop
;;	yr4_eb4cl_four_complex_djbfft_preload
b5a:	yr4_eb4cl_four_complex_djbfft rsi, 64, 2*64, 4*64, rdi, 16*YMM_SCD2, 2, L1PREFETCH_ALL, 2*dist64
	yloop	2, b5a, -2*64+8*64, rdi, -2*16*YMM_SCD2 ;; Loop 2 times, next source pointer, restore sine/cosine pointer
	yloop	4, b5a, -4*8*64+32*64, rdi, 4*YMM_SCD2 ;; Loop 4 times, next source pointer, next sine/cosine pointer
	yloop	2, b5a, -2*32*64+dist64	;; Test loop counter, next source pointer
	yloop	2, b5a, -2*dist64, rdi, -4*4*YMM_SCD2 ;; Test loop counter, restore source pointer, restore sine/cosine pointer
	loops_undo 4*4*2		;; Back up loop counter

	yloop_adjust 2*8		;; 16 fewer iterations this loop
	yloop_set_prefetch YHALF	;; Prefetch 240 cache lines in this loop
	yr8_8cl_eight_complex_fft_final_preload
;; Loop below reworked to avoid 4KB distances
;;b6a:	yr8_8cl_eight_complex_fft_final rsi, 2*64, 32*64, dist64, 64
;;	yloop	32, b6a, -16*2*64	;; 16 iterations!!  Restore source pointer
b6a:	yr8_8cl_eight_complex_fft_final rsi, 4*64, 32*64, dist64, 64
	yloop	8, b6a, -8*4*64+2*64	;; Loop 8 times, next source pointer
	yloop	4, b6a, -2*2*64+2*dist64 ;; 2 iterations!!  Next source pointer

	yloop	15, b3a0		;; Loop 15 times
	end_timer 6
	jmp	ypass2_7680_complex_done


ypass2_7680_complex_3:
	start_timer 6
x64	mov	r8, DEST2ARG
	mov	rbp, DIST_TO_MULSRCARG
	yloop_init 480			;; 15 iters of 128 cache lines

b3c0:	yloop_set_prefetch YHALF	;; Prefetch 240 cache lines in this loop
	yr4_s4cl_four_complex_djbfft_preload
b3c:	yr4_s4cl_four_complex_djbfft rsi, 64, 32*64, dist64, rdi, YMM_SCD2, 32
	yloop	32, b3c, -32*64, rdi, -32*YMM_SCD2 ;; Loop 32 times, restore source pointer, restore sine/cosine pointer
	loops_undo 32			;; Back up loop counter

	yloop_set_prefetch YHALF	;; Prefetch 240 cache lines in this loop
	yr4_eb4cl_four_complex_djbfft_preload
b4c:	yr4_eb4cl_four_complex_djbfft rsi, 64, 8*64, 16*64, rdi, 4*YMM_SCD2, 8
	yloop	8, b4c, -8*64+32*64, rdi, -8*4*YMM_SCD2+YMM_SCD2 ;; Loop 8 times, next source pointer, next sine/cosine pointer
	yloop	2, b4c, -2*32*64+dist64	;; Loop 2 times, next source pointer
	yloop	2, b4c, -2*dist64, rdi, -4*YMM_SCD2 ;; Loop 2 times, restore source pointer, restore sine/cosine pointer
	loops_undo 4*8			;; Back up loop counter

	yloop_set_prefetch YHALF	;; Prefetch 240 cache lines in this loop
;;	yr4_eb4cl_four_complex_djbfft_preload
b5c:	yr4_eb4cl_four_complex_djbfft rsi, 64, 2*64, 4*64, rdi, 16*YMM_SCD2, 2
	yloop	2, b5c, -2*64+8*64, rdi, -2*16*YMM_SCD2 ;; Loop 2 times, next source pointer, restore sine/cosine pointer
	yloop	4, b5c, -4*8*64+32*64, rdi, 4*YMM_SCD2 ;; Loop 4 times, next source pointer, next sine/cosine pointer
	yloop	2, b5c, -2*32*64+dist64	;; Test loop counter, next source pointer
	yloop	2, b5c, -2*dist64, rdi, -4*4*YMM_SCD2 ;; Test loop counter, restore source pointer, restore sine/cosine pointer
	loops_undo 4*4*2		;; Back up loop counter

	yloop_adjust 2*8		;; 16 fewer iterations this loop
	yloop_set_prefetch YQUARTER	;; Prefetch 120 cache lines in this loop
	yr8_8cl_eight_complex_with_mult_preload
;; Loop below reworked to avoid 4KB distances
;;b6c:	yr8_8cl_eight_complex_with_mult rsi, 2*64, 32*64, dist64, 64
;;	yloop	32, b6c, -16*2*64	;; 16 iterations!!  Restore source pointer
b6c:	yr8_8cl_eight_complex_with_mult rsi, 4*64, 32*64, dist64, 64
	yloop	8, b6c, -8*4*64+2*64	;; Loop 8 times, next source pointer
	yloop	4, b6c, -2*2*64		;; 2 iterations!!  Restore source pointer
x64	add	rsi, r8
	loops_undo 4*8			;; Back up loop counter

	yloop_set_prefetch YHALF	;; Prefetch 240 cache lines in this loop
	yr4_eb4cl_four_complex_djbunfft_preload
c5c:	yr4_eb4cl_four_complex_djbunfft rsi, 64, 2*64, 4*64, rdi, 16*YMM_SCD2, 2
	yloop	2, c5c, -2*64+8*64, rdi, -2*16*YMM_SCD2 ;; Loop 2 times, next source pointer, restore sine/cosine pointer
	yloop	4, c5c, -4*8*64+32*64, rdi, 4*YMM_SCD2 ;; Loop 4 times, next source pointer, next sine/cosine pointer
	yloop	2, c5c, -2*32*64+dist64	;; Test loop counter, next source pointer
	yloop	2, c5c, -2*dist64, rdi, -4*4*YMM_SCD2 ;; Test loop counter, restore source pointer, restore sine/cosine pointer
	loops_undo 4*4*2		;; Back up loop counter

	yloop_set_prefetch YHALF	;; Prefetch 240 cache lines in this loop
;;	yr4_eb4cl_four_complex_djbunfft_preload
c4c:	yr4_eb4cl_four_complex_djbunfft rsi, 64, 8*64, 16*64, rdi, 4*YMM_SCD2, 8
	yloop	8, c4c, -8*64+32*64, rdi, -8*4*YMM_SCD2+YMM_SCD2 ;; Loop 8 times, next source pointer, next sine/cosine pointer
	yloop	2, c4c, -2*32*64+dist64	;; Loop 2 times, next source pointer
	yloop	2, c4c, -2*dist64, rdi, -4*YMM_SCD2 ;; Loop 2 times, restore source pointer, restore sine/cosine pointer
	loops_undo 4*8			;; Back up loop counter

	yloop_set_prefetch YHALF	;; Prefetch 240 cache lines in this loop
	yr4_s4cl_four_complex_djbunfft_preload
c3c:	yr4_s4cl_four_complex_djbunfft rsi, 64, 32*64, dist64, rdi, YMM_SCD2, 32, L1PREFETCH_ALL, 2*dist64
	yloop	32, c3c, -32*64+2*dist64, rdi, -32*YMM_SCD2 ;; Loop 32 times, next source pointer, restore sine/cosine pointer

x64	sub	rsi, r8
	yloop	15, b3c0, -15*2*dist64	;; Loop 15 times, restore source pointer
x64	add	rsi, r8
	end_timer 6
	jmp	ypass2_7680_complex_unfft


ypass2_7680_complex_4:
	start_timer 6
	mov	rdi, xsincos_complex	;; The sin/cos ptrs
	bump	rdi, 640*YMM_SCD1+128*YMM_SCD2
	mov	rbp, DIST_TO_MULSRCARG
	yloop_init 480			;; 15 iters of 128 cache lines

b6d0:	yloop_adjust 2*8		;; 16 fewer iterations this loop
	yloop_set_prefetch YHALF	;; Prefetch 240 cache lines in this loop
	yr8_8cl_eight_complex_with_mulf_preload
;; Loop below reworked to avoid 4KB distances
;;b6d:	yr8_8cl_eight_complex_with_mulf rsi, 2*64, 32*64, dist64, 64
;;	yloop	32, b6d, -16*2*64	;; 16 iterations!!  Restore source pointer
b6d:	yr8_8cl_eight_complex_with_mulf rsi, 4*64, 32*64, dist64, 64
	yloop	8, b6d, -8*4*64+2*64	;; Loop 8 times, next source pointer
	yloop	4, b6d, -2*2*64		;; 2 iterations!!  Restore source pointer
	loops_undo 4*8			;; Back up loop counter

	yloop_set_prefetch YSINGLE	;; Prefetch 480 cache lines in this loop
	yr4_eb4cl_four_complex_djbunfft_preload
c5d:	yr4_eb4cl_four_complex_djbunfft rsi, 64, 2*64, 4*64, rdi, 16*YMM_SCD2, 2
	yloop	2, c5d, -2*64+8*64, rdi, -2*16*YMM_SCD2 ;; Loop 2 times, next source pointer, restore sine/cosine pointer
	yloop	4, c5d, -4*8*64+32*64, rdi, 4*YMM_SCD2 ;; Loop 4 times, next source pointer, next sine/cosine pointer
	yloop	2, c5d, -2*32*64+dist64	;; Test loop counter, next source pointer
	yloop	2, c5d, -2*dist64, rdi, -4*4*YMM_SCD2 ;; Test loop counter, restore source pointer, restore sine/cosine pointer
	loops_undo 4*4*2		;; Back up loop counter

	yloop_set_prefetch YSINGLE	;; Prefetch 480 cache lines in this loop
;;	yr4_eb4cl_four_complex_djbunfft_preload
c4d:	yr4_eb4cl_four_complex_djbunfft rsi, 64, 8*64, 16*64, rdi, 4*YMM_SCD2, 8
	yloop	8, c4d, -8*64+32*64, rdi, -8*4*YMM_SCD2+YMM_SCD2 ;; Loop 8 times, next source pointer, next sine/cosine pointer
	yloop	2, c4d, -2*32*64+dist64	;; Loop 2 times, next source pointer
	yloop	2, c4d, -2*dist64, rdi, -4*YMM_SCD2 ;; Loop 2 times, restore source pointer, restore sine/cosine pointer
	loops_undo 4*8			;; Back up loop counter

	yloop_set_prefetch YSINGLE	;; Prefetch 480 cache lines in this loop
	yr4_s4cl_four_complex_djbunfft_preload
c3d:	yr4_s4cl_four_complex_djbunfft rsi, 64, 32*64, dist64, rdi, YMM_SCD2, 32, L1PREFETCH_ALL, 2*dist64
	yloop	32, c3d, -32*64+2*dist64, rdi, -32*YMM_SCD2 ;; Loop 32 times, next source pointer, restore sine/cosine pointer

	yloop	15, b6d0, -15*2*dist64	;; Loop 15 times, restore source pointer
;;	bump	rcx, (-240-3*480+2*128+2*240+2*120+3*240)*64 ;; Move prefetch pointer to where squaring would be
	end_timer 6

;; Do inverse FFT levels 3,4
;; On output the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	512	+1	+1	+1
;;	4	...
;;	...
;;	508	...
;;	1024	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	2560	+1	+1	+1
;;	4	...
;;	...
;;	2556	...
;;	5120	...
;;	...

;; Do 1536 five_complex_unfft operations
;;	distance between fft data elements is 512
;;	do 384 macros each processing 40 data values

;; Do inverse FFT levels 1,2
;; On input the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	2560	+1	+1	+1
;;	4	...
;;	...
;;	2556	...
;;	5120	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	7680	+1	+1	+1
;;	4	...
;;	...
;;	7676

;; Do 2560 three_complex_unfft operations
;;	distance between fft data elements is 2560
;;	do 640 macros each processing 24 data values

ypass2_7680_complex_unfft:
	mov	rdi, xsincos_complex	;; Load sin/cos pointers
	lea	rbp, [rdi+640*YMM_SCD1]
	bump	rcx, (1920+2*29-2*128-3*240-120-3*240-128-64)*64 ;; Move prefetch pointer along
	yloop_init 1024			;; 384 iterations or 640 iterations

c2a:	start_timer 13
	yloop_adjust 5			;; Five fewer iterations this loop
	yloop_set_prefetch YOTHER, 8	;; Prefetch 128 cache lines in this loop
	yr5_5cl_five_complex_djbunfft_preload
c2b:	yr5_5cl_five_complex_djbunfft rsi, 10*dist64, 2*dist64, rbp, 0, 3
	yloop	8, c2b, -3*10*dist64, rbp, YMM_SCD2 ;; 3 iterations!!  Restore source pointer, next sine/cosine pointer
	loops_undo 8			;; Restore loop counter
	end_timer 13

	start_timer 14
	yloop_adjust 3			;; Three fewer iterations this loop
	yloop_set_prefetch YOTHER, 16	;; Prefetch 64 cache lines in this loop
	yr3_3cl_three_complex_djbunfft_preload
c1b:	yr3_3cl_three_complex_djbunfft rsi, 2*dist64, 10*dist64, rdi, 128*YMM_SCD1, 5, L1PREFETCH_ALL, 64
	yloop	8, c1b, -5*2*dist64+64, rdi, -5*128*YMM_SCD1+YMM_SCD1 ;; 5 iterations!!  Next source pointer, next sine/cosine pointer
	end_timer 14

	yloop	64, c2a, -64*64+dist64	;; Loop 64 times, next source pointer
	yloop	2, c2a			;; Loop 2 times

ypass2_7680_complex_done:
	ENDM


; *************** radix4-pass2-13-levels-real macro ******************
; This macro takes 16384 real values and performs the final 14 levels of the FFT.
; The results are squared, and the first 14 levels of the inverse FFT are done.

yr4dwpn_pass2_13_levels_real MACRO
	LOCAL	b1b, b2b, b3a, b3b, b4a, b4b, b5a, b5b, b6a, b6b, b6c, b6d
	LOCAL	c1b, c2b, c3a, c3b, c4a, c4b, c5a, c5b
	LOCAL	ypass2_13_levels_real_1, ypass2_13_levels_real_2
	LOCAL	ypass2_13_levels_real_3, ypass2_13_levels_real_4
	LOCAL	ypass2_13_levels_real_unfft, ypass2_13_levels_real_done

;; Type 4 FFTs skip the forward FFT process

	start_timer 2
	mov	rsi, DATA_ADDR		;; Load source address
	mov	rbx, DIST_TO_FFTSRCARG
	mov	rcx, DATA_PREFETCH	;; Prefetch pointer (we need to prefetch 
					;; 2048 cache lines + 31 pad cache lines)
	cmp	ffttype, 4
	je	ypass2_13_levels_real_4

;; Do FFT levels 1,2,3 which correspond to complex levels 1,2
;; On input the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	8192	+1	+1	+1
;;	4	...
;;	...
;;	8188
;; On output the 64-byte cache lines are swizzled to hold these data values:
;;	0	+4K	+4K	+4K	2048	+4K	+4K	+4K
;;	4	...
;;	...
;;	2044	...
;;	1	...
;;	...

;; Do 2048 eight_reals_fft operations
;;	distance between fft data elements is 2048
;;	do 512 macros each processing 32 data values

	mov	rdi, sincos3		;; Real sin/cos pointer
	mov	rbp, xsincos_complex	;; Complex sin/cos data
	yloop_init 512			;; 512 iterations
	yloop_set_prefetch YSINGLE
	yr4_fs4cl_2sc_eight_reals_fft_preload
b1b:	yr4_fs4cl_2sc_eight_reals_fft rsi, 64, 8*dist64, 16*dist64, rbp, YMM_SCD2, rdi, YMM_SCD2, 64
	yloop	64, b1b, -64*64+dist64	;; Test loop counter, next source pointer
	yloop	8, b1b, -8*dist64	;; Test loop counter, restore source pointer

;; Do FFT levels 3,4
;; On input the 64-byte cache lines hold these data values:
;;	0	+4K	+4K	+4K	2048	+4K	+4K	+4K
;;	4	...
;;	...
;;	2044	...
;;	1	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+4K	+4K	+4K	512	+4K	+4K	+4K
;;	4	...
;;	...
;;	508	...
;;	1024	...
;;	...
;;	1	...
;;	...

;; Do 512 eight_reals_fft operations
;; Do 1536 four_complex_fft operations
;;	distance between fft data elements is 512
;;	do 512 macros each processing 32 data values

	yloop_init 512			;; 512 iterations
	yloop_set_prefetch YHALF
	yr4_4cl_eight_reals_four_complex_djbfft_preload
b2b:	yr4_4cl_eight_reals_four_complex_djbfft rsi, 8*dist64, 2*dist64, 4*dist64, rdi, YMM_SCD3, 4
	yloop	4, b2b, -4*8*dist64+64	;; Test loop counter, next source pointer
	yloop	64, b2b, -64*64+dist64	;; Test loop counter, next source pointer
	yloop	2, b2b, -2*dist64, rdi, -512*YMM_SCD3 ;; Test loop counter, restore source pointer, restore sin/cos ptr

;; Do FFT levels 5,6
;; On input the 64-byte cache lines hold these data values:
;;	0	+1K	+1K	+1K	128	+1K	+1K	+1K
;;	4	...
;;	...
;;	124	...
;;	256	...
;;	...
;;	1	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+1K	+1K	+1K	32	+1K	+1K	+1K
;;	4	...
;;	...
;;	28	...
;;	64	...
;;	...
;;	1	...
;;	...

;; Do 128 eight_reals_fft operations
;; Do 1920 four_complex_fft operations
;;	distance between fft data elements is 128
;;	do 512 macros each processing 32 data values

	yloop_init 512			;; 4 iterations of 128
	yloop_set_prefetch YHALF
	yr4_4cl_eight_reals_four_complex_djbfft_preload
b3a:	yr4_4cl_eight_reals_four_complex_djbfft rsi, 8*dist64, 32*64, dist64, rdi, 4*YMM_SCD3, 4
	yloop	4, b3a, -4*8*dist64+64	;; Test loop counter, next source pointer
	yloop	32, b3a, -32*64+2*dist64, rdi, -128*4*YMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer

	loops_reset			;; 3 more iters of 128
	yr4_rb4cl_four_complex_djbfft_preload
b3b:	yr4_rb4cl_four_complex_djbfft rsi, 8*dist64, 32*64, dist64, rdi, 4*YMM_SCD3, 4
	yloop	4, b3b, -4*8*dist64+64	;; Test loop counter, next source pointer
	yloop	32, b3b, -32*64+2*dist64, rdi, -128*4*YMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer
	yloop	4, b3b, -4*2*dist64	;; Test loop counter, restore source pointer

;; Do FFT levels 7,8
;; On input the 64-byte cache lines hold these data values:
;;	0	+4K	+4K	+4K	128	+4K	+4K	+4K
;;	4	...
;;	...
;;	124	...
;;	256	...
;;	...
;;	1	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+4K	+4K	+4K	32	+4K	+4K	+4K
;;	4	...
;;	...
;;	28	...
;;	64	...
;;	...
;;	1	...
;;	...

;; Do 32 eight_reals_fft operations
;; Do 2016 four_complex_fft operations
;;	distance between fft data elements is 32
;;	do 512 macros each processing 32 data values

	yloop_init 512			;; 16 iterations of 32
	yloop_set_prefetch YHALF
	yr4_4cl_eight_reals_four_complex_djbfft_preload
b4a:	yr4_4cl_eight_reals_four_complex_djbfft rsi, 8*dist64, 8*64, 16*64, rdi, 16*YMM_SCD3, 4
	yloop	4, b4a, -4*8*dist64+64	;; Test loop counter, next source pointer
	yloop	8, b4a, -8*64+32*64, rdi, -32*16*YMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer

	loops_reset			;; 15 more iters of 32
	yr4_rb4cl_four_complex_djbfft_preload
b4b:	yr4_rb4cl_four_complex_djbfft rsi, 8*dist64, 8*64, 16*64, rdi, 16*YMM_SCD3, 4
	yloop	4, b4b, -4*8*dist64+64	;; Test loop counter, next source pointer
	yloop	8, b4b, -8*64+32*64, rdi, -32*16*YMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer
	yloop	2, b4b, -2*32*64+dist64	;; Test loop counter, next source pointer
	yloop	8, b4b, -8*dist64	;; Test loop counter, restore source pointer

;; Do FFT levels 9,10
;; On input the 64-byte cache lines hold these data values:
;;	0	+4K	+4K	+4K	32	+4K	+4K	+4K
;;	4	...
;;	...
;;	28	...
;;	64	...
;;	...
;;	1	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+4K	+4K	+4K	8	+4K	+4K	+4K
;;	4	...
;;	16	...
;;	...
;;	1	...
;;	...

;; Do 8 eight_reals_fft operations
;; Do 2040 four_complex_fft operations
;;	distance between fft data elements is 8
;;	do 512 macros each processing 32 data values

	yloop_init 512			;; 64 iterations of 8
	yloop_set_prefetch YHALF
	yr4_4cl_eight_reals_four_complex_djbfft_preload
b5a:	yr4_4cl_eight_reals_four_complex_djbfft rsi, 8*dist64, 2*64, 4*64, rdi, 64*YMM_SCD3, 4
	yloop	4, b5a, -4*8*dist64+64	;; Test loop counter, next source pointer
	yloop	2, b5a, -2*64+8*64, rdi, -8*64*YMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer

	loops_reset			;; 63 more iters of 8
	yr4_rb4cl_four_complex_djbfft_preload
b5b:	yr4_rb4cl_four_complex_djbfft rsi, 8*dist64, 2*64, 4*64, rdi, 64*YMM_SCD3, 4
	yloop	4, b5b, -4*8*dist64+64	;; Test loop counter, next source pointer
	yloop	2, b5b, -2*64+8*64, rdi, -8*64*YMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer
	yloop	8, b5b, -8*8*64+dist64	;; Test loop counter, next source pointer
	yloop	8, b5b, -8*dist64	;; Test loop counter, restore source pointer

;; Do FFT levels 11,12,13 as well as inverse FFT levels 11,12,13
;; On input and output the 64-byte cache lines hold these data values:
;;	0	+4K	+4K	+4K	8	+4K	+4K	+4K
;;	4	...
;;	16	...
;;	...
;;	1	...
;;	...

;; Do 1 sixteen_reals_fft operation
;; Do 1023 eight_complex_fft operations
;;	distance between fft data elements is 1
;;	do 256 macros each processing 64 data values

	cmp	ffttype, 2
	jl	ypass2_13_levels_real_1
	jg	ypass2_13_levels_real_3

ypass2_13_levels_real_2:
;;	yr8_8cl_sixteen_reals_eight_complex_with_square rsi, 2*64, 8*dist64, 16*dist64, 64
	yr8_dispatch call, y8real_square, 8*dist64, 16*dist64, 64
	yloop_init 256			;; 255 more iterations
	yloop_adjust 1
	yloop_set_prefetch YOTHER, 16
	yr8_8cl_eight_complex_with_square_preload
b6b:	yr8_8cl_eight_complex_with_square rsi, 2*64, 8*dist64, 16*dist64, 64, 1
	yloop	32, b6b, -32*2*64+dist64	;; Test loop counter, next source pointer
	yloop	8, b6b, -8*dist64	;; Test loop counter, restore source pointer
	jmp	ypass2_13_levels_real_unfft

ypass2_13_levels_real_1:
;;	yr8_8cl_sixteen_reals_eight_complex_fft_final rsi, 2*64, 8*dist64, 16*dist64, 64
	yr8_dispatch call, y8real_fft_final, 8*dist64, 16*dist64, 64
	mov	rcx, xsincos_complex	;; Prefetch complex sin/cos data, 1040 cache lines
	yloop_init 256			;; 255 more iterations
	yloop_adjust 1
	yloop_set_prefetch YDOUBLE
b6a:	yr8_8cl_eight_complex_fft_final rsi, 2*64, 8*dist64, 16*dist64, 64
	yloop	32, b6a, -32*2*64+dist64	;; Test loop counter, next source pointer
	yloop	8, b6a			;; Test loop counter
	jmp	ypass2_13_levels_real_done

ypass2_13_levels_real_3:
x64	mov	r8, DEST2ARG
	mov	rbp, DIST_TO_MULSRCARG
;;	yr8_8cl_sixteen_reals_eight_complex_with_mult rsi, 2*64, 8*dist64, 16*dist64, 64
	yr8_dispatch call, y8real_mult, 8*dist64, 16*dist64, 64
	yloop_init 256			;; 255 more iterations
	yloop_adjust 1
	yloop_set_prefetch YOTHER, 16
	yr8_8cl_eight_complex_with_mult_preload
b6c:	yr8_8cl_eight_complex_with_mult rsi, 2*64, 8*dist64, 16*dist64, 64
	yloop	32, b6c, -32*2*64+dist64	;; Test loop counter, next source pointer
	yloop	8, b6c, -8*dist64	;; Test loop counter, restore source pointer
x64	add	rsi, r8
	jmp	ypass2_13_levels_real_unfft

ypass2_13_levels_real_4:
	mov	rbp, DIST_TO_MULSRCARG
;;	yr8_8cl_sixteen_reals_eight_complex_with_mulf rsi, 2*64, 8*dist64, 16*dist64, 64
	yr8_dispatch call, y8real_mulf, 8*dist64, 16*dist64, 64
	yloop_init 256			;; 255 more iterations
	yloop_adjust 1
	yloop_set_prefetch YDOUBLE
	yr8_8cl_eight_complex_with_mulf_preload
b6d:	yr8_8cl_eight_complex_with_mulf rsi, 2*64, 8*dist64, 16*dist64, 64
	yloop	32, b6d, -32*2*64+dist64	;; Test loop counter, next source pointer
	yloop	8, b6d, -8*dist64	;; Test loop counter, restore source pointer
;;	bump	rcx, (-512+512+4*256+32)*64 ;; Move prefetch pointer back to where squaring would be

;; Do inverse FFT levels 9,10
;; On input the 64-byte cache lines hold these data values:
;;	0	+4K	+4K	+4K	8	+4K	+4K	+4K
;;	4	...
;;	16	...
;;	...
;;	1	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+4K	+4K	+4K	32	+4K	+4K	+4K
;;	4	...
;;	...
;;	28	...
;;	64	...
;;	...
;;	1	...
;;	...

;; Do 8 eight_reals_unfft operations
;; Do 2040 four_complex_unfft operations
;;	distance between fft data elements is 8
;;	do 512 macros each processing 32 data values

ypass2_13_levels_real_unfft:
	mov	rdi, sincos3		;; Load sin/cos pointer
	bump	rdi, 512*YMM_SCD2
	yloop_init 512			;; 64 iterations of 8
	yloop_set_prefetch YHALF
	yr4_4cl_eight_reals_four_complex_djbunfft_preload
c5a:	yr4_4cl_eight_reals_four_complex_djbunfft rsi, 8*dist64, 2*64, 4*64, rdi, 64*YMM_SCD3, 4
	yloop	4, c5a, -4*8*dist64+64	;; Test loop counter, next source pointer
	yloop	2, c5a, -2*64+8*64, rdi, -8*64*YMM_SCD3 ;; Test loop counter, next source pointer, restore sine/cosine pointer

	loops_reset			;; 63 more iters of 8
	yr4_rb4cl_four_complex_djbunfft_preload
c5b:	yr4_rb4cl_four_complex_djbunfft rsi, 8*dist64, 2*64, 4*64, rdi, 64*YMM_SCD3, 4
	yloop	4, c5b, -4*8*dist64+64	;; Test loop counter, next source pointer
	yloop	2, c5b, -2*64+8*64, rdi, -8*64*YMM_SCD3 ;; Test loop counter, next source pointer, restore sine/cosine pointer
	yloop	8, c5b, -8*8*64+dist64	;; Test loop counter, next source pointer
	yloop	8, c5b, -8*dist64	;; Test loop counter, restore source pointer

;; Do inverse FFT levels 7,8
;; On input the 64-byte cache lines hold these data values:
;;	0	+4K	+4K	+4K	32	+4K	+4K	+4K
;;	4	...
;;	...
;;	28	...
;;	64	...
;;	...
;;	1	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+4K	+4K	+4K	128	+4K	+4K	+4K
;;	4	...
;;	...
;;	124	...
;;	256	...
;;	...
;;	1	...
;;	...

;; Do 32 eight_reals_unfft operations
;; Do 2016 four_complex_unfft operations
;;	distance between fft data elements is 32
;;	do 512 macros each processing 32 data values

	bump	rcx, (2048+31-512-4*256-32-256-256)*64 ;; Move prefetch pointer along
	yloop_init 512			;; 16 iterations of 32
	yloop_set_prefetch YHALF
	yr4_4cl_eight_reals_four_complex_djbunfft_preload
c4a:	yr4_4cl_eight_reals_four_complex_djbunfft rsi, 8*dist64, 8*64, 16*64, rdi, 16*YMM_SCD3, 4
	yloop	4, c4a, -4*8*dist64+64	;; Test loop counter, next source pointer
	yloop	8, c4a, -8*64+32*64, rdi, -32*16*YMM_SCD3 ;; Test loop counter, next source pointer, restore sine/cosine pointer

	loops_reset			;; 15 more iters of 32
	yr4_rb4cl_four_complex_djbunfft_preload
c4b:	yr4_rb4cl_four_complex_djbunfft rsi, 8*dist64, 8*64, 16*64, rdi, 16*YMM_SCD3, 4
	yloop	4, c4b, -4*8*dist64+64	;; Test loop counter, next source pointer
	yloop	8, c4b, -8*64+32*64, rdi, -32*16*YMM_SCD3 ;; Test loop counter, next source pointer, restore sine/cosine pointer
	yloop	2, c4b, -2*32*64+dist64	;; Test loop counter, next source pointer
	yloop	8, c4b, -8*dist64	;; Test loop counter, restore source pointer

;; Do inverse FFT levels 5,6
;; On input the 64-byte cache lines hold these data values:
;;	0	+4K	+4K	+4K	128	+4K	+4K	+4K
;;	4	...
;;	...
;;	124	...
;;	256	...
;;	...
;;	1	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+4K	+4K	+4K	512	+4K	+4K	+4K
;;	4	...
;;	...
;;	508	...
;;	1024	...
;;	...
;;	1	...
;;	...

;; Do 128 eight_reals_unfft operations
;; Do 1920 four_complex_unfft operations
;;	distance between fft data elements is 128
;;	do 512 macros each processing 32 data values

	mov	rcx, xsincos_complex	;; Prefetch complex sin/cos data, 1040 cache lines
	yloop_init 512			;; 4 iterations of 128
	yloop_set_prefetch YOTHER, 32
	yr4_4cl_eight_reals_four_complex_djbunfft_preload
c3a:	yr4_4cl_eight_reals_four_complex_djbunfft rsi, 8*dist64, 32*64, dist64, rdi, 4*YMM_SCD3, 4
	yloop	4, c3a, -4*8*dist64+64	;; Test loop counter, next source pointer
	yloop	32, c3a, -32*64+2*dist64, rdi, -128*4*YMM_SCD3 ;; Test loop counter, next source pointer, restore sine/cosine pointer

	loops_reset			;; 3 more iters of 128
	yr4_rb4cl_four_complex_djbunfft_preload
c3b:	yr4_rb4cl_four_complex_djbunfft rsi, 8*dist64, 32*64, dist64, rdi, 4*YMM_SCD3, 4
	yloop	4, c3b, -4*8*dist64+64	;; Test loop counter, next source pointer
	yloop	32, c3b, -32*64+2*dist64, rdi, -128*4*YMM_SCD3 ;; Test loop counter, next source pointer, restore sine/cosine pointer
	yloop	4, c3b, -4*2*dist64	;; Test loop counter, restore source pointer

;; Do inverse FFT levels 3,4
;; On input the 64-byte cache lines hold these data values:
;;	0	+4K	+4K	+4K	512	+4K	+4K	+4K
;;	4	...
;;	...
;;	508	...
;;	1024	...
;;	...
;;	1	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+4K	+4K	+4K	2048	+4K	+4K	+4K
;;	4	...
;;	...
;;	2044	...
;;	1	...
;;	...

;; Do 512 eight_reals_unfft operations
;; Do 1536 four_complex_unfft operations
;;	distance between fft data elements is 512
;;	do 512 macros each processing 32 data values

	yloop_init 512			;; 512 iterations
	yloop_set_prefetch YSINGLE
	yr4_4cl_eight_reals_four_complex_djbunfft_preload
c2b:	yr4_4cl_eight_reals_four_complex_djbunfft rsi, 8*dist64, 2*dist64, 4*dist64, rdi, YMM_SCD3, 4
	yloop	4, c2b, -4*8*dist64+64	;; Test loop counter, next source pointer
	yloop	64, c2b, -64*64+dist64	;; Test loop counter, next source pointer
	yloop	2, c2b, -2*dist64	;; Test loop counter, restore source pointer

;; Do inverse FFT levels 1,2,3 which correspond to complex levels 1,2
;; On input the 64-byte cache lines hold these data values:
;;	0	+4K	+4K	+4K	2048	+4K	+4K	+4K
;;	4	...
;;	...
;;	2044	...
;;	1	...
;;	...
;; On output the 64-byte cache lines are swizzled to hold these data values:
;;	0	+1	+1	+1	8192	+1	+1	+1
;;	4	...
;;	...
;;	8188

;; Do 2048 eight_reals_unfft operations
;;	distance between fft data elements is 2048
;;	do 512 macros each processing 32 data values

	bump	rdi, -512*YMM_SCD3-512*YMM_SCD2 ;; Real sin/cos pointer
	mov	rbp, xsincos_complex	;; Complex sin/cos data
	bump	rcx, (1040-16-512-512)*64 ;; Move prefetch pointer along
	yloop_init 512			;; 512 iterations
	yloop_set_prefetch YSINGLE
	yr4_s4cl_2sc_eight_reals_unfft_preload
c1b:	yr4_s4cl_2sc_eight_reals_unfft rsi, 64, 8*dist64, 16*dist64, rbp, YMM_SCD2, rdi, YMM_SCD2, 64
	yloop	64, c1b, -64*64+dist64	;; Test loop counter, next source pointer
	yloop	8, c1b			;; Test loop counter

ypass2_13_levels_real_done:
	end_timer 2
	ENDM

; *************** radix4-pass2-13-levels-complex macro ******************
; This macro takes 8192 complex values and performs the final 13 levels of a standard
; radix-4 FFT, squares the results, and does the first 13 levels of the inverse FFT.

yr4dwpn_pass2_13_levels_complex MACRO
	LOCAL	b1a, b1b, b2b, b3b, b4a0, b4a, b4b0, b4b, b4c0, b4c, b5a, b5b, b5c, b6a, b6b, b6c, b6d0, b6d
	LOCAL	c1b, c2b, c3a, c3b, c4b, c4c, c4d, c5b, c5c, c5d
	LOCAL	ypass2_13_levels_complex_1, ypass2_13_levels_complex_2
	LOCAL	ypass2_13_levels_complex_3, ypass2_13_levels_complex_4
	LOCAL	ypass2_13_levels_complex_unfft, ypass2_13_levels_complex_done

;; Type 4 FFTs skip the forward FFT process

	mov	rsi, DATA_ADDR		;; Load address of FFT data
	mov	rbx, DIST_TO_FFTSRCARG
	mov	rcx, DATA_PREFETCH	;; Prefetch pointer (we need to prefetch 
					;; 2048 cache lines + 31 pad cache lines)
	cmp	ffttype, 4
	je	ypass2_13_levels_complex_4

;; Do FFT level 1,2
;; On input the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	8192	+1	+1	+1
;;	4	...
;;	...
;;	8188
;; On output the 64-byte cache lines are swizzled to hold these data values:
;;	0	+4K	+4K	+4K	2048	+4K	+4K	+4K
;;	4	...
;;	...
;;	2044	...
;;	1	...
;;	...

;; Do 2048 four_complex_fft operations
;;	distance between fft data elements is 2048
;;	do 512 macros each processing 32 data values

;; Do FFT levels 3,4
;; On input the 64-byte cache lines hold these data values:
;;	0	+4K	+4K	+4K	2048	+4K	+4K	+4K
;;	4	...
;;	...
;;	2044	...
;;	1	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+4K	+4K	+4K	512	+4K	+4K	+4K
;;	4	...
;;	...
;;	508	...
;;	1024	...
;;	...
;;	1	...
;;	...

;; Do 2048 four_complex_fft operations
;;	distance between fft data elements is 512
;;	do 512 macros each processing 32 data values

;; Do FFT levels 5,6
;; On input the 64-byte cache lines hold these data values:
;;	0	+4K	+4K	+4K	512	+4K	+4K	+4K
;;	4	...
;;	...
;;	508	...
;;	1024	...
;;	...
;;	1	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+4K	+4K	+4K	128	+4K	+4K	+4K
;;	4	...
;;	...
;;	124	...
;;	256	...
;;	...
;;	1	...
;;	...

;; Do 2048 four_complex_fft operations
;;	distance between fft data elements is 128
;;	do 512 macros each processing 32 data values

	mov	rdi, xsincos_complex	;; The sin/cos ptrs
	mov	rbp, rdi
	mov	rdx, rdi
	yloop_init 512			;; 512 iterations

b1a:	start_timer 4
	yloop_set_prefetch YHALF	;; Prefetch 256 cache lines in this loop
	yr4_fs4cl_four_complex_djbfft_preload
b1b:	yr4_fs4cl_four_complex_djbfft rsi, 32*64, 8*dist64, 16*dist64, rdi, 32*YMM_SCD2, 2
	yloop	2, b1b, -2*32*64+dist64	;; Test loop counter, next source pointer
	yloop	8, b1b, -8*dist64, rdi, -16*32*YMM_SCD2+YMM_SCD2 ;; Test loop counter, restore source pointer, next sin/cos data
	loops_undo 16			;; Back up loop counter
	end_timer 4

	start_timer 5
	yloop_set_prefetch YHALF	;; Prefetch 256 cache lines in this loop
	yr4_eb4cl_four_complex_djbfft_preload
b2b:	yr4_eb4cl_four_complex_djbfft rsi, 32*64, 2*dist64, 4*dist64, rbp, 128*YMM_SCD2, 2
	yloop	2, b2b, -2*32*64+dist64	;; Test loop counter, next source pointer
	yloop	2, b2b, -2*dist64+8*dist64, rbp, -4*128*YMM_SCD2+YMM_SCD2 ;; Test loop counter, next source pointer, next sin/cos data
	yloop	4, b2b, -4*8*dist64	;; Test loop counter, restore source pointer
	loops_undo 4*4			;; Back up loop counter
	end_timer 5

	start_timer 6
	yloop_set_prefetch YQUARTER	;; Prefetch 128 cache lines in this loop
;;	yr4_eb4cl_four_complex_djbfft_preload
b3b:	yr4_eb4cl_four_complex_djbfft rsi, 2*dist64, 32*64, dist64, rdx, 0, 4, L1PREFETCH_ALL, 64
	yloop	4, b3b, 0, rdx, 4*YMM_SCD2 ;; Loop 4 times, next sine/cosine pointer
	yloop	4, b3b, -4*4*2*dist64+64	;; Loop 4 times, next source pointer
	end_timer 6

	yloop	32, b1a, -32*64, rdi, -32*YMM_SCD2+512*YMM_SCD2 ;; Test loop counter, restore source pointer, next sin/cos ptr

;; Do FFT levels 7,8
;; On input the 64-byte cache lines hold these data values:
;;	0	+4K	+4K	+4K	128	+4K	+4K	+4K
;;	4	...
;;	...
;;	124	...
;;	256	...
;;	...
;;	1	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+4K	+4K	+4K	32	+4K	+4K	+4K
;;	4	...
;;	...
;;	28	...
;;	64	...
;;	...
;;	1	...
;;	...

;; Do 2048 four_complex_fft operations
;;	distance between fft data elements is 32
;;	do 512 macros each processing 32 data values

;; Do FFT levels 9,10
;; On input the 64-byte cache lines hold these data values:
;;	0	+4K	+4K	+4K	32	+4K	+4K	+4K
;;	4	...
;;	...
;;	28	...
;;	64	...
;;	...
;;	1	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+4K	+4K	+4K	8	+4K	+4K	+4K
;;	4	...
;;	16	...
;;	...
;;	1	...
;;	...

;; Do 2048 four_complex_fft operations
;;	distance between fft data elements is 8
;;	do 512 macros each processing 32 data values

;; Do FFT levels 11,12,13 as well as inverse FFT levels 11,12,13
;; On input and output the 64-byte cache lines hold these data values:
;;	0	+4K	+4K	+4K	8	+4K	+4K	+4K
;;	4	...
;;	16	...
;;	...
;;	1	...
;;	...

;; Do 1024 eight_complex_fft operations
;;	distance between fft data elements is 1
;;	do 256 macros each processing 64 data values

;; Do inverse FFT levels 9,10
;; On input the 64-byte cache lines hold these data values:
;;	0	+4K	+4K	+4K	8	+4K	+4K	+4K
;;	4	...
;;	16	...
;;	...
;;	1	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+4K	+4K	+4K	32	+4K	+4K	+4K
;;	4	...
;;	...
;;	28	...
;;	64	...
;;	...
;;	1	...
;;	...

;; Do 2048 four_complex_unfft operations
;;	distance between fft data elements is 8
;;	do 512 macros each processing 32 data values

;; Do inverse FFT levels 7,8
;; On input the 64-byte cache lines hold these data values:
;;	0	+4K	+4K	+4K	32	+4K	+4K	+4K
;;	4	...
;;	...
;;	28	...
;;	64	...
;;	...
;;	1	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+4K	+4K	+4K	128	+4K	+4K	+4K
;;	4	...
;;	...
;;	124	...
;;	256	...
;;	...
;;	1	...
;;	...

;; Do 2048 four_complex_unfft operations
;;	distance between fft data elements is 32
;;	do 512 macros each processing 32 data values

;; Do this 128 cache lines (8KB) at a time to maximize accessing data
;; while it is in the L1 cache.

;; Execute the right middle step

	cmp	ffttype, 2
	jl	ypass2_13_levels_complex_1
	jg	ypass2_13_levels_complex_3

ypass2_13_levels_complex_2:
	yloop_init 512			;; 16 iters of 128 cache lines

b4b0:	start_timer 7
	yloop_set_prefetch YQUARTER	;; Prefetch 128 cache lines in this loop
	yr4_b4cl_four_complex_djbfft_preload
b4b:	yr4_b4cl_four_complex_djbfft rsi, 64, 8*64, 16*64, rdi, YMM_SCD2/4, 8
	yloop	8, b4b, -8*64+8*dist64	;; Loop 8 times, next source pointer
	yloop	4, b4b, -4*8*dist64, rdi, -32*YMM_SCD2/4 ;; Loop 4 times, restore source pointer, restore sine/cosine pointer
	loops_undo 4*8			;; Back up loop counter
	end_timer 7

	start_timer 8
	yloop_set_prefetch YQUARTER	;; Prefetch 128 cache lines in this loop
;;	yr4_b4cl_four_complex_djbfft_preload
b5b:	yr4_b4cl_four_complex_djbfft rsi, 64, 2*64, 4*64, rdi, 4*YMM_SCD2/4, 2
	yloop	2, b5b, -2*64+8*64, rdi, -2*4*YMM_SCD2/4 ;; Loop 2 times, next source pointer, restore sine/cosine pointer
	yloop	4, b5b, -4*8*64+8*dist64, rdi, YMM_SCD2/4 ;; Loop 4 times, next source pointer, next sine/cosine pointer
	yloop	4, b5b, -4*8*dist64, rdi, -4*YMM_SCD2/4 ;; Loop 4 times, restore source pointer, restore sine/cosine pointer
	loops_undo 4*4*2		;; Back up loop counter
	end_timer 8

	start_timer 9
	yloop_adjust 16			;; 16 fewer iterations this loop
	yloop_set_prefetch YOTHER, 16	;; Prefetch 32 cache lines in this loop
	yr8_8cl_eight_complex_with_square_preload
b6b:	yr8_8cl_eight_complex_with_square rsi, 2*64, 8*dist64, 16*dist64, 64, 32
	yloop	32, b6b, -16*2*64	;; 16 iterations!!  Restore source pointer
	loops_undo 32			;; Back up loop counter
	end_timer 9

	start_timer 10
	yloop_set_prefetch YQUARTER	;; Prefetch 128 cache lines in this loop
	yr4_b4cl_four_complex_djbunfft_preload
c5b:	yr4_b4cl_four_complex_djbunfft rsi, 64, 2*64, 4*64, rdi, 4*YMM_SCD2/4, 2
	yloop	2, c5b, -2*64+8*64, rdi, -2*4*YMM_SCD2/4 ;; Loop 2 times, next source pointer, restore sine/cosine pointer
	yloop	4, c5b, -4*8*64+8*dist64, rdi, YMM_SCD2/4 ;; Loop 4 times, next source pointer, next sine/cosine pointer
	yloop	4, c5b, -4*8*dist64, rdi, -4*YMM_SCD2/4 ;; Loop 4 times, restore source pointer, restore sine/cosine pointer
	loops_undo 4*4*2		;; Back up loop counter
	end_timer 10

	start_timer 11
	yloop_set_prefetch YQUARTER	;; Prefetch 128 cache lines in this loop
;;	yr4_b4cl_four_complex_djbunfft_preload
c4b:	yr4_b4cl_four_complex_djbunfft rsi, 64, 8*64, 16*64, rdi, YMM_SCD2/4, 8, L1PREFETCH_ALL, 32*64
	yloop	8, c4b, -8*64+8*dist64	;; Loop 8 times, next source pointer
	yloop	4, c4b, -4*8*dist64+32*64, rdi, -32*YMM_SCD2/4 ;; Loop 4 times, next source pointer, restore sine/cosine pointer
	end_timer 11

	yloop	2, b4b0, -2*32*64+dist64 ;; Loop 2 times, next source pointer
	yloop	8, b4b0, -8*dist64	;; Loop 8 times, restore source pointer
	jmp	ypass2_13_levels_complex_unfft


ypass2_13_levels_complex_1:
	start_timer 6
	bump	rcx, (2048+31-2*256-128-3*512)*64 ;; Move prefetch pointer along
	yloop_init 512			;; 16 iters of 128 cache lines

b4a0:	yloop_set_prefetch YSINGLE	;; Prefetch 512 cache lines in this loop
	yr4_b4cl_four_complex_djbfft_preload
b4a:	yr4_b4cl_four_complex_djbfft rsi, 64, 8*64, 16*64, rdi, YMM_SCD2/4, 8
	yloop	8, b4a, -8*64+8*dist64	;; Loop 8 times, next source pointer
	yloop	4, b4a, -4*8*dist64, rdi, -32*YMM_SCD2/4 ;; Loop 4 times, restore source pointer, restore sine/cosine pointer
	loops_undo 4*8			;; Back up loop counter

	yloop_set_prefetch YSINGLE	;; Prefetch 512 cache lines in this loop
;;	yr4_b4cl_four_complex_djbfft_preload
b5a:	yr4_b4cl_four_complex_djbfft rsi, 64, 2*64, 4*64, rdi, 4*YMM_SCD2/4, 2, L1PREFETCH_ALL, 32*64
	yloop	2, b5a, -2*64+8*64, rdi, -2*4*YMM_SCD2/4 ;; Loop 2 times, next source pointer, restore sine/cosine pointer
	yloop	4, b5a, -4*8*64+8*dist64, rdi, YMM_SCD2/4 ;; Loop 4 times, next source pointer, next sine/cosine pointer
	yloop	4, b5a, -4*8*dist64, rdi, -4*YMM_SCD2/4 ;; Loop 4 times, restore source pointer, restore sine/cosine pointer
	loops_undo 4*4*2		;; Back up loop counter

	yloop_adjust 16			;; 16 fewer iterations this loop
	yloop_set_prefetch YDOUBLE	;; Prefetch 512 cache lines in this loop
	yr8_8cl_eight_complex_fft_final_preload
b6a:	yr8_8cl_eight_complex_fft_final rsi, 2*64, 8*dist64, 16*dist64, 64
	yloop	32, b6a, -16*2*64+32*64	;; 16 iterations!!  Next source pointer

	yloop	2, b4a0, -2*32*64+dist64 ;; Loop 2 times, next source pointer
	yloop	8, b4a0			;; Loop 8 times
	end_timer 6
	jmp	ypass2_13_levels_complex_done


ypass2_13_levels_complex_3:
	start_timer 6
x64	mov	r8, DEST2ARG
	mov	rbp, DIST_TO_MULSRCARG
	yloop_init 512			;; 16 iters of 128 cache lines

b4c0:	yloop_set_prefetch YQUARTER	;; Prefetch 128 cache lines in this loop
	yr4_b4cl_four_complex_djbfft_preload
b4c:	yr4_b4cl_four_complex_djbfft rsi, 64, 8*64, 16*64, rdi, YMM_SCD2/4, 8
	yloop	8, b4c, -8*64+8*dist64	;; Loop 8 times, next source pointer
	yloop	4, b4c, -4*8*dist64, rdi, -32*YMM_SCD2/4 ;; Loop 4 times, restore source pointer, restore sine/cosine pointer
	loops_undo 4*8			;; Back up loop counter

	yloop_set_prefetch YQUARTER	;; Prefetch 128 cache lines in this loop
;;	yr4_b4cl_four_complex_djbfft_preload
b5c:	yr4_b4cl_four_complex_djbfft rsi, 64, 2*64, 4*64, rdi, 4*YMM_SCD2/4, 2
	yloop	2, b5c, -2*64+8*64, rdi, -2*4*YMM_SCD2/4 ;; Loop 2 times, next source pointer, restore sine/cosine pointer
	yloop	4, b5c, -4*8*64+8*dist64, rdi, YMM_SCD2/4 ;; Loop 4 times, next source pointer, next sine/cosine pointer
	yloop	4, b5c, -4*8*dist64, rdi, -4*YMM_SCD2/4 ;; Loop 4 times, restore source pointer, restore sine/cosine pointer
	loops_undo 4*4*2		;; Back up loop counter

	yloop_adjust 16			;; 16 fewer iterations this loop
	yloop_set_prefetch YOTHER, 16	;; Prefetch 32 cache lines in this loop
	yr8_8cl_eight_complex_with_mult_preload
b6c:	yr8_8cl_eight_complex_with_mult rsi, 2*64, 8*dist64, 16*dist64, 64
	yloop	32, b6c, -16*2*64	;; 16 iterations!!  Restore source pointer
x64	add	rsi, r8
	loops_undo 32			;; Back up loop counter

	yloop_set_prefetch YQUARTER	;; Prefetch 128 cache lines in this loop
	yr4_b4cl_four_complex_djbunfft_preload
c5c:	yr4_b4cl_four_complex_djbunfft rsi, 64, 2*64, 4*64, rdi, 4*YMM_SCD2/4, 2
	yloop	2, c5c, -2*64+8*64, rdi, -2*4*YMM_SCD2/4 ;; Loop 2 times, next source pointer, restore sine/cosine pointer
	yloop	4, c5c, -4*8*64+8*dist64, rdi, YMM_SCD2/4 ;; Loop 4 times, next source pointer, next sine/cosine pointer
	yloop	4, c5c, -4*8*dist64, rdi, -4*YMM_SCD2/4 ;; Loop 4 times, restore source pointer, restore sine/cosine pointer
	loops_undo 4*4*2		;; Back up loop counter

	yloop_set_prefetch YQUARTER	;; Prefetch 128 cache lines in this loop
;;	yr4_b4cl_four_complex_djbunfft_preload
c4c:	yr4_b4cl_four_complex_djbunfft rsi, 64, 8*64, 16*64, rdi, YMM_SCD2/4, 8, L1PREFETCH_ALL, 32*64
	yloop	8, c4c, -8*64+8*dist64	;; Loop 8 times, next source pointer
	yloop	4, c4c, -4*8*dist64+32*64, rdi, -32*YMM_SCD2/4 ;; Loop 4 times, next source pointer, restore sine/cosine pointer

x64	sub	rsi, r8
	yloop	2, b4c0, -2*32*64+dist64 ;; Loop 2 times, next source pointer
	yloop	8, b4c0, -8*dist64	;; Loop 8 times, restore source pointer
x64	add	rsi, r8
	end_timer 6
	jmp	ypass2_13_levels_complex_unfft


ypass2_13_levels_complex_4:
	start_timer 6
	mov	rdi, xsincos_complex	;; The sin/cos ptrs
	bump	rdi, 512*YMM_SCD2
	mov	rbp, DIST_TO_MULSRCARG
	yloop_init 512			;; 16 iters of 128 cache lines

b6d0:	yloop_adjust 16			;; 16 fewer iterations this loop
	yloop_set_prefetch YHALF	;; Prefetch 256 cache lines in this loop
	yr8_8cl_eight_complex_with_mulf_preload
b6d:	yr8_8cl_eight_complex_with_mulf rsi, 2*64, 8*dist64, 16*dist64, 64
	yloop	32, b6d, -16*2*64	;; 16 iterations!!  Restore source pointer
	loops_undo 32			;; Back up loop counter

	yloop_set_prefetch YSINGLE	;; Prefetch 512 cache lines in this loop
	yr4_b4cl_four_complex_djbunfft_preload
c5d:	yr4_b4cl_four_complex_djbunfft rsi, 64, 2*64, 4*64, rdi, 4*YMM_SCD2/4, 2
	yloop	2, c5d, -2*64+8*64, rdi, -2*4*YMM_SCD2/4 ;; Loop 2 times, next source pointer, restore sine/cosine pointer
	yloop	4, c5d, -4*8*64+8*dist64, rdi, YMM_SCD2/4 ;; Loop 4 times, next source pointer, next sine/cosine pointer
	yloop	4, c5d, -4*8*dist64, rdi, -4*YMM_SCD2/4 ;; Loop 4 times, restore source pointer, restore sine/cosine pointer
	loops_undo 4*4*2		;; Back up loop counter

	yloop_set_prefetch YSINGLE	;; Prefetch 512 cache lines in this loop
;;	yr4_b4cl_four_complex_djbunfft_preload
c4d:	yr4_b4cl_four_complex_djbunfft rsi, 64, 8*64, 16*64, rdi, YMM_SCD2/4, 8, L1PREFETCH_ALL, 32*64
	yloop	8, c4d, -8*64+8*dist64	;; Loop 8 times, next source pointer
	yloop	4, c4d, -4*8*dist64+32*64, rdi, -32*YMM_SCD2/4 ;; Loop 4 times, next source pointer, restore sine/cosine pointer

	yloop	2, b6d0, -2*32*64+dist64	;; Loop 2 times, next source pointer
	yloop	8, b6d0, -8*dist64	;; Loop 8 times, restore source pointer
	bump	rcx, (-256-2*512+2*256+3*128+32+2*128)*64 ;; Move prefetch pointer to where squaring would be
	end_timer 6

;; Do inverse FFT levels 5,6
;; On input the 64-byte cache lines hold these data values:
;;	0	+4K	+4K	+4K	128	+4K	+4K	+4K
;;	4	...
;;	...
;;	124	...
;;	256	...
;;	...
;;	1	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+4K	+4K	+4K	512	+4K	+4K	+4K
;;	4	...
;;	...
;;	508	...
;;	1024	...
;;	...
;;	1	...
;;	...

;; Do 2048 four_complex_unfft operations
;;	distance between fft data elements is 128
;;	do 512 macros each processing 32 data values

;; Do inverse FFT levels 3,4
;; On input the 64-byte cache lines hold these data values:
;;	0	+4K	+4K	+4K	512	+4K	+4K	+4K
;;	4	...
;;	...
;;	508	...
;;	1024	...
;;	...
;;	1	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+4K	+4K	+4K	2048	+4K	+4K	+4K
;;	4	...
;;	...
;;	2044	...
;;	1	...
;;	...

;; Do 2048 four_complex_unfft operations
;;	distance between fft data elements is 512
;;	do 512 macros each processing 32 data values

;; Do inverse FFT levels 1,2
;; On input the 64-byte cache lines hold these data values:
;;	0	+4K	+4K	+4K	2048	+4K	+4K	+4K
;;	4	...
;;	...
;;	2044	...
;;	1	...
;;	...
;; On output the 64-byte cache lines are swizzled to hold these data values:
;;	0	+1	+1	+1	8192	+1	+1	+1
;;	4	...
;;	...
;;	8188

;; Do 2048 four_complex_unfft operations
;;	distance between fft data elements is 2048
;;	do 512 macros each processing 32 data values

ypass2_13_levels_complex_unfft:
	mov	rdi, xsincos_complex	;; Load sin/cos pointers
	mov	rbp, rdi
	mov	rdx, rdi
	bump	rcx, (2048+31-2*256-3*128-32-2*128-128-2*256)*64 ;; Move prefetch pointer along
	yloop_init 512			;; 512 iterations

c3a:	start_timer 12
	yloop_set_prefetch YQUARTER	;; Prefetch 128 cache lines in this loop
	yr4_eb4cl_four_complex_djbunfft_preload
c3b:	yr4_eb4cl_four_complex_djbunfft rsi, 2*dist64, 32*64, dist64, rdx, 0, 4
	yloop	4, c3b, 0, rdx, 4*YMM_SCD2 ;; Loop 4 times, next sine/cosine pointer
	yloop	4, c3b, -4*4*2*dist64	;; Loop 4 times, restore source pointer
	loops_undo 4*4			;; Back up loop counter
	end_timer 12

	start_timer 13
	yloop_set_prefetch YHALF	;; Prefetch 256 cache lines in this loop
;;	yr4_eb4cl_four_complex_djbunfft_preload
c2b:	yr4_eb4cl_four_complex_djbunfft rsi, 32*64, 2*dist64, 4*dist64, rbp, 128*YMM_SCD2, 2
	yloop	2, c2b, -2*32*64+dist64	;; Test loop counter, next source pointer
	yloop	2, c2b, -2*dist64+8*dist64, rbp, -4*128*YMM_SCD2+YMM_SCD2 ;; Test loop counter, next source pointer, next sine/cosine pointer
	yloop	4, c2b, -4*8*dist64	;; Test loop counter, restore source pointer
	loops_undo 4*4			;; Back up loop counter
	end_timer 13

	start_timer 14
	yloop_set_prefetch YHALF	;; Prefetch 256 cache lines in this loop
	yr4_s4cl_four_complex_djbunfft_preload
c1b:	yr4_s4cl_four_complex_djbunfft rsi, 32*64, 8*dist64, 16*dist64, rdi, 32*YMM_SCD2, 2, L1PREFETCH_ALL, 64
	yloop	2, c1b, -2*32*64+dist64	;; Test loop counter, next source pointer
	yloop	8, c1b, -8*dist64+64, rdi, -16*32*YMM_SCD2+YMM_SCD2 ;; Test loop counter, next source pointer, next sine/cosine pointer
	end_timer 14

	yloop	32, c3a			;; Loop 32 times

ypass2_13_levels_complex_done:
	ENDM


; *************** radix4-pass2-10240-real macro ******************
; This macro takes 20480 real values and performs the final 14.322 levels of the FFT.
; The results are squared, and the first 14.322 levels of the inverse FFT are done.

yr4dwpn_pass2_10240_real MACRO
	LOCAL	b1b, b2a, b2b, b3a, b3b, b4a, b4b, b5a, b5b, b6a, b6b, b6c, b6d
	LOCAL	c1b, c2a, c2b, c3a, c3b, c4a, c4b, c5a, c5b
	LOCAL	ypass2_10240_real_1, ypass2_10240_real_2
	LOCAL	ypass2_10240_real_3, ypass2_10240_real_4
	LOCAL	ypass2_10240_real_unfft, ypass2_10240_real_done

;; Type 4 FFTs skip the forward FFT process

	start_timer 2
	mov	rsi, DATA_ADDR		;; Load source address
	mov	rbx, DIST_TO_FFTSRCARG
	mov	rcx, DATA_PREFETCH	;; Prefetch pointer (we need to prefetch 
					;; 2560 cache lines + 39 pad cache lines)
	cmp	ffttype, 4
	je	ypass2_10240_real_4

;; Do FFT levels 1,2,3 which correspond to complex levels 1,2
;; On input the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	10240	+1	+1	+1
;;	4	...
;;	...
;;	10236
;; On output the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	2048	+1	+1	+1
;;	4	...
;;	...
;;	2044	...
;;	4096	...
;;	...

;; Do 2048 ten_reals_fft operations
;;	distance between fft data elements is 2048
;;	do 512 macros each processing 40 data values

	mov	rdi, sincos3		;; Real sin/cos pointer
	mov	rbp, xsincos_complex	;; Complex sin/cos data
	yloop_init 512			;; 512 iterations
	yloop_set_prefetch YHALF
	yr5_f5cl_2sc_ten_reals_fft_preload
b1b:	yr5_f5cl_2sc_ten_reals_fft rsi, 64, 8*dist64, rbp, YMM_SCD2, rdi, YMM_SCD2, 64
	yloop	64, b1b, -64*64+dist64	;; Test loop counter, next source pointer
	yloop	8, b1b, -8*dist64	;; Test loop counter, restore source pointer

;; Do FFT levels 3,4
;; On input the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	2048	+1	+1	+1
;;	4	...
;;	...
;;	2044	...
;;	4096	...
;;	...
;; On output the 64-byte cache lines are swizzled to hold these data values:
;;	0	+1K	+1K	+1K	512	+1K	+1K	+1K
;;	4	...
;;	...
;;	508	...
;;	1	...
;;	...
;;	4096	...
;;	...

;; Do 512 eight_reals_fft operations
;; Do 2048 four_complex_fft operations
;;	distance between fft data elements is 512
;;	do 640 macros each processing 32 data values

	yloop_init 640			;; 5 iterations of 128
	yloop_set_prefetch YSINGLE
	yr4_s4cl_2sc_eight_reals_fft_preload
b2a:	yr4_s4cl_2sc_eight_reals_fft rsi, 64, 2*dist64, 4*dist64, rbp, YMM_SCD2, rdi, YMM_SCD2, 64
	yloop	64, b2a, -64*64+dist64	;; Test loop counter, next source pointer
	yloop	2, b2a, -2*dist64+8*dist64, rbp, -128*YMM_SCD2 ;; Test loop counter, next source pointer, restore sine/cosine pointer

	loops_reset			;; 4 more iters of 128
	yr4_s4cl_four_complex_djbfft_preload
b2b:	yr4_s4cl_four_complex_djbfft rsi, 64, 2*dist64, 4*dist64, rbp, YMM_SCD2, 64
	yloop	64, b2b, -64*64+dist64	;; Test loop counter, next source pointer
	yloop	2, b2b, -2*dist64+8*dist64, rbp, -128*YMM_SCD2 ;; Test loop counter, next source pointer, restore sin/cos ptr
	yloop	5, b2b, -5*8*dist64	;; Loop 5 times, restore source pointer

;; Do FFT levels 5,6
;; On input the 64-byte cache lines hold these data values:
;;	0	+1K	+1K	+1K	512	+1K	+1K	+1K
;;	4	...
;;	...
;;	508	...
;;	1	...
;;	...
;;	4096	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+1K	+1K	+1K	128	+1K	+1K	+1K
;;	4	...
;;	...
;;	124	...
;;	256	...
;;	...
;;	1	...
;;	...
;;	4096	...
;;	...

;; Do 128 eight_reals_fft operations
;; Do 2432 four_complex_fft operations
;;	distance between fft data elements is 128
;;	do 640 macros each processing 32 data values

	yloop_init 640			;; 5 iterations of 128
	yloop_set_prefetch YHALF
	yr4_4cl_eight_reals_four_complex_djbfft_preload
b3a:	yr4_4cl_eight_reals_four_complex_djbfft rsi, 2*dist64, 32*64, dist64, rdi, YMM_SCD3, 4
	yloop	4, b3a, -4*2*dist64+64	;; Test loop counter, next source pointer
	yloop	32, b3a, -32*64+8*dist64, rdi, -128*YMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer

	loops_reset			;; 5 more iters of 128
	yr4_rb4cl_four_complex_djbfft_preload
b3b:	yr4_rb4cl_four_complex_djbfft rsi, 2*dist64, 32*64, dist64, rdi, YMM_SCD3, 4
	yloop	4, b3b, -4*2*dist64+64	;; Test loop counter, next source pointer
	yloop	32, b3b, -32*64+8*dist64, rdi, -128*YMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer
	yloop	5, b3b, -5*8*dist64	;; Test loop counter, restore source pointer

;; Do FFT levels 7,8
;; On input the 64-byte cache lines hold these data values:
;;	0	+1K	+1K	+1K	128	+1K	+1K	+1K
;;	4	...
;;	...
;;	124	...
;;	256	...
;;	...
;;	1	...
;;	...
;;	4096	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+1K	+1K	+1K	32	+1K	+1K	+1K
;;	4	...
;;	...
;;	28	...
;;	64	...
;;	...
;;	1	...
;;	...
;;	4096	...
;;	...

;; Do 32 eight_reals_fft operations
;; Do 2528 four_complex_fft operations
;;	distance between fft data elements is 32
;;	do 640 macros each processing 32 data values

	yloop_init 640			;; 20 iterations of 32
	yloop_set_prefetch YHALF
	yr4_4cl_eight_reals_four_complex_djbfft_preload
b4a:	yr4_4cl_eight_reals_four_complex_djbfft rsi, 2*dist64, 8*64, 16*64, rdi, 4*YMM_SCD3, 4
	yloop	4, b4a, -4*2*dist64+64	;; Test loop counter, next source pointer
	yloop	8, b4a, -8*64+32*64, rdi, -32*4*YMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer

	loops_reset			;; 19 more iters of 32
	yr4_rb4cl_four_complex_djbfft_preload
b4b:	yr4_rb4cl_four_complex_djbfft rsi, 2*dist64, 8*64, 16*64, rdi, 4*YMM_SCD3, 4
	yloop	4, b4b, -4*2*dist64+64	;; Test loop counter, next source pointer
	yloop	8, b4b, -8*64+32*64, rdi, -32*4*YMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer
	yloop	2, b4b, -2*32*64+dist64	;; Test loop counter, next source pointer
	yloop	2, b4b, -2*dist64+8*dist64 ;; Test loop counter, next source pointer
	yloop	5, b4b, -5*8*dist64	;; Test loop counter, restore source pointer

;; Do FFT levels 9,10
;; On input the 64-byte cache lines hold these data values:
;;	0	+1K	+1K	+1K	32	+1K	+1K	+1K
;;	4	...
;;	...
;;	28	...
;;	64	...
;;	...
;;	1	...
;;	...
;;	4096	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+1K	+1K	+1K	8	+1K	+1K	+1K
;;	4	...
;;	16	...
;;	...
;;	1	...
;;	...
;;	4096	...
;;	...

;; Do 8 eight_reals_fft operations
;; Do 2552 four_complex_fft operations
;;	distance between fft data elements is 8
;;	do 640 macros each processing 32 data values

	yloop_init 640			;; 80 iterations of 8
	yloop_set_prefetch YHALF
	yr4_4cl_eight_reals_four_complex_djbfft_preload
b5a:	yr4_4cl_eight_reals_four_complex_djbfft rsi, 2*dist64, 2*64, 4*64, rdi, 16*YMM_SCD3, 4
	yloop	4, b5a, -4*2*dist64+64	;; Test loop counter, next source pointer
	yloop	2, b5a, -2*64+8*64, rdi, -8*16*YMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer

	loops_reset			;; 79 more iters of 8
	yr4_rb4cl_four_complex_djbfft_preload
b5b:	yr4_rb4cl_four_complex_djbfft rsi, 2*dist64, 2*64, 4*64, rdi, 16*YMM_SCD3, 4
	yloop	4, b5b, -4*2*dist64+64	;; Test loop counter, next source pointer
	yloop	2, b5b, -2*64+8*64, rdi, -8*16*YMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer
	yloop	8, b5b, -8*8*64+dist64	;; Test loop counter, next source pointer
	yloop	2, b5b, -2*dist64+8*dist64 ;; Test loop counter, next source pointer
	yloop	5, b5b, -5*8*dist64	;; Test loop counter, restore source pointer

;; Do FFT levels 11,12,13 as well as inverse FFT levels 11,12,13
;; On input and output the 64-byte cache lines hold these data values:
;;	0	+1K	+1K	+1K	8	+1K	+1K	+1K
;;	4	...
;;	16	...
;;	...
;;	1	...
;;	...
;;	4096	...
;;	...

;; Do 1 sixteen_reals_fft operation
;; Do 1279 eight_complex_fft operations
;;	distance between fft data elements is 1
;;	do 319 macros each processing 64 data values

	cmp	ffttype, 2
	jl	ypass2_10240_real_1
	jg	ypass2_10240_real_3

ypass2_10240_real_2:
;;	yr8_8cl_sixteen_reals_eight_complex_with_square rsi, 2*64, 2*dist64, 4*dist64, 64
	yr8_dispatch call, y8real_square, 2*dist64, 4*dist64, 64
	yloop_init 320			;; 319 more iterations
	yloop_adjust 1
	yloop_set_prefetch YQUARTER
	yr8_8cl_eight_complex_with_square_preload
b6b:	yr8_8cl_eight_complex_with_square rsi, 2*64, 2*dist64, 4*dist64, 64, 1
	yloop	32, b6b, -32*2*64+dist64	;; Test loop counter, next source pointer
	yloop	2, b6b, -2*dist64+8*dist64	;; Test loop counter, next source pointer
	yloop	5, b6b, -5*8*dist64	;; Test loop counter, restore source pointer
	jmp	ypass2_10240_real_unfft

ypass2_10240_real_1:
;;	yr8_8cl_sixteen_reals_eight_complex_fft_final rsi, 2*64, 2*dist64, 4*dist64, 64
	yr8_dispatch call, y8real_fft_final, 2*dist64, 4*dist64, 64
	mov	rcx, xsincos_complex	;; Prefetch complex sin/cos data, 1296 cache lines
	yloop_init 320			;; 319 more iterations
	yloop_adjust 1
	yloop_set_prefetch YDOUBLE
b6a:	yr8_8cl_eight_complex_fft_final rsi, 2*64, 2*dist64, 4*dist64, 64
	yloop	32, b6a, -32*2*64+dist64	;; Test loop counter, next source pointer
	yloop	2, b6a, -2*dist64+8*dist64	;; Test loop counter, next source pointer
	yloop	5, b6a			;; Test loop counter
	jmp	ypass2_10240_real_done

ypass2_10240_real_3:
x64	mov	r8, DEST2ARG
	mov	rbp, DIST_TO_MULSRCARG
;;	yr8_8cl_sixteen_reals_eight_complex_with_mult rsi, 2*64, 2*dist64, 4*dist64, 64
	yr8_dispatch call, y8real_mult, 2*dist64, 4*dist64, 64
	yloop_init 320			;; 319 more iterations
	yloop_adjust 1
	yloop_set_prefetch YQUARTER
	yr8_8cl_eight_complex_with_mult_preload
b6c:	yr8_8cl_eight_complex_with_mult rsi, 2*64, 2*dist64, 4*dist64, 64
	yloop	32, b6c, -32*2*64+dist64	;; Test loop counter, next source pointer
	yloop	2, b6c, -2*dist64+8*dist64	;; Test loop counter, next source pointer
	yloop	5, b6c, -5*8*dist64	;; Test loop counter, restore source pointer
x64	add	rsi, r8
	jmp	ypass2_10240_real_unfft

ypass2_10240_real_4:
	mov	rbp, DIST_TO_MULSRCARG
;;	yr8_8cl_sixteen_reals_eight_complex_with_mulf rsi, 2*64, 2*dist64, 4*dist64, 64
	yr8_dispatch call, y8real_mulf, 2*dist64, 4*dist64, 64
	yloop_init 320			;; 319 more iterations
	yloop_adjust 1
	yloop_set_prefetch YDOUBLE
	yr8_8cl_eight_complex_with_mulf_preload
b6d:	yr8_8cl_eight_complex_with_mulf rsi, 2*64, 2*dist64, 4*dist64, 64
	yloop	32, b6d, -32*2*64+dist64	;; Test loop counter, next source pointer
	yloop	2, b6d, -2*dist64+8*dist64	;; Test loop counter, next source pointer
	yloop	5, b6d, -5*8*dist64	;; Test loop counter, restore source pointer
;;	bump	rcx, (-638+256+640+3*320+160)*64 ;; Move prefetch pointer back to where squaring would be

;; Do inverse FFT levels 9,10
;; On input the 64-byte cache lines hold these data values:
;;	0	+1K	+1K	+1K	8	+1K	+1K	+1K
;;	4	...
;;	16	...
;;	...
;;	1	...
;;	...
;;	4096	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+1K	+1K	+1K	32	+1K	+1K	+1K
;;	4	...
;;	...
;;	28	...
;;	64	...
;;	...
;;	1	...
;;	...
;;	4096	...
;;	...

;; Do 8 eight_reals_unfft operations
;; Do 2552 four_complex_unfft operations
;;	distance between fft data elements is 8
;;	do 640 macros each processing 32 data values

ypass2_10240_real_unfft:
	mov	rdi, sincos3		;; Load sin/cos pointer
	bump	rdi, 128*YMM_SCD2+512*YMM_SCD2
	yloop_init 640			;; 80 iterations of 8
	yloop_set_prefetch YHALF
	yr4_4cl_eight_reals_four_complex_djbunfft_preload
c5a:	yr4_4cl_eight_reals_four_complex_djbunfft rsi, 2*dist64, 2*64, 4*64, rdi, 16*YMM_SCD3, 4
	yloop	4, c5a, -4*2*dist64+64	;; Test loop counter, next source pointer
	yloop	2, c5a, -2*64+8*64, rdi, -8*16*YMM_SCD3 ;; Test loop counter, next source pointer, restore sine/cosine pointer

	loops_reset			;; 79 more iters of 8
	yr4_rb4cl_four_complex_djbunfft_preload
c5b:	yr4_rb4cl_four_complex_djbunfft rsi, 2*dist64, 2*64, 4*64, rdi, 16*YMM_SCD3, 4
	yloop	4, c5b, -4*2*dist64+64	;; Test loop counter, next source pointer
	yloop	2, c5b, -2*64+8*64, rdi, -8*16*YMM_SCD3 ;; Test loop counter, next source pointer, restore sine/cosine pointer
	yloop	8, c5b, -8*8*64+dist64	;; Test loop counter, next source pointer
	yloop	2, c5b, -2*dist64+8*dist64	;; Test loop counter, next source pointer
	yloop	5, c5b, -5*8*dist64	;; Test loop counter, restore source pointer

;; Do inverse FFT levels 7,8
;; On input the 64-byte cache lines hold these data values:
;;	0	+1K	+1K	+1K	32	+1K	+1K	+1K
;;	4	...
;;	...
;;	28	...
;;	64	...
;;	...
;;	1	...
;;	...
;;	4096	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+1K	+1K	+1K	128	+1K	+1K	+1K
;;	4	...
;;	...
;;	124	...
;;	256	...
;;	...
;;	1	...
;;	...
;;	4096	...
;;	...

;; Do 32 eight_reals_unfft operations
;; Do 2528 four_complex_unfft operations
;;	distance between fft data elements is 32
;;	do 640 macros each processing 32 data values

	bump	rcx, (2560+39-256-640-3*320-160-320-320)*64 ;; Move prefetch pointer along
	yloop_init 640			;; 20 iterations of 32
	yloop_set_prefetch YHALF
	yr4_4cl_eight_reals_four_complex_djbunfft_preload
c4a:	yr4_4cl_eight_reals_four_complex_djbunfft rsi, 2*dist64, 8*64, 16*64, rdi, 4*YMM_SCD3, 4
	yloop	4, c4a, -4*2*dist64+64	;; Test loop counter, next source pointer
	yloop	8, c4a, -8*64+32*64, rdi, -32*4*YMM_SCD3 ;; Test loop counter, next source pointer, restore sine/cosine pointer

	loops_reset			;; 19 more iters of 32
	yr4_rb4cl_four_complex_djbunfft_preload
c4b:	yr4_rb4cl_four_complex_djbunfft rsi, 2*dist64, 8*64, 16*64, rdi, 4*YMM_SCD3, 4
	yloop	4, c4b, -4*2*dist64+64	;; Test loop counter, next source pointer
	yloop	8, c4b, -8*64+32*64, rdi, -32*4*YMM_SCD3 ;; Test loop counter, next source pointer, restore sine/cosine pointer
	yloop	2, c4b, -2*32*64+dist64	;; Test loop counter, next source pointer
	yloop	2, c4b, -2*dist64+8*dist64	;; Test loop counter, next source pointer
	yloop	5, c4b, -5*8*dist64	;; Test loop counter, restore source pointer

;; Do inverse FFT levels 5,6
;; On input the 64-byte cache lines hold these data values:
;;	0	+1K	+1K	+1K	128	+1K	+1K	+1K
;;	4	...
;;	...
;;	124	...
;;	256	...
;;	...
;;	1	...
;;	...
;;	4096	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+1K	+1K	+1K	512	+1K	+1K	+1K
;;	4	...
;;	...
;;	508	...
;;	1	...
;;	...
;;	4096	...
;;	...

;; Do 128 eight_reals_unfft operations
;; Do 2432 four_complex_unfft operations
;;	distance between fft data elements is 128
;;	do 640 macros each processing 32 data values

	mov	rcx, xsincos_complex	;; Prefetch complex sin/cos data, 1296 cache lines
	yloop_init 640			;; 5 iterations of 128
	yloop_set_prefetch YSINGLE
	yr4_4cl_eight_reals_four_complex_djbunfft_preload
c3a:	yr4_4cl_eight_reals_four_complex_djbunfft rsi, 2*dist64, 32*64, dist64, rdi, YMM_SCD3, 4
	yloop	4, c3a, -4*2*dist64+64	;; Test loop counter, next source pointer
	yloop	32, c3a, -32*64+8*dist64, rdi, -128*YMM_SCD3 ;; Test loop counter, next source pointer, restore sine/cosine pointer

	loops_reset			;; 4 more iters of 128
	yr4_rb4cl_four_complex_djbunfft_preload
c3b:	yr4_rb4cl_four_complex_djbunfft rsi, 2*dist64, 32*64, dist64, rdi, YMM_SCD3, 4
	yloop	4, c3b, -4*2*dist64+64	;; Test loop counter, next source pointer
	yloop	32, c3b, -32*64+8*dist64, rdi, -128*YMM_SCD3 ;; Test loop counter, next source pointer, restore sine/cosine pointer
	yloop	5, c3b, -5*8*dist64	;; Test loop counter, restore source pointer

;; Do inverse FFT levels 3,4
;; On input the 64-byte cache lines hold these data values:
;;	0	+1K	+1K	+1K	512	+1K	+1K	+1K
;;	4	...
;;	...
;;	508	...
;;	1	...
;;	...
;;	4096	...
;;	...
;; On output the 64-byte cache lines are swizzled to hold these data values:
;;	0	+1	+1	+1	2048	+1	+1	+1
;;	4	...
;;	...
;;	2044	...
;;	4096	...
;;	...

;; Do 512 eight_reals_unfft operations
;; Do 2048 four_complex_unfft operations
;;	distance between fft data elements is 512
;;	do 640 macros each processing 32 data values

	bump	rdi, -128*YMM_SCD2	;; Real sin/cos ptr
	mov	rbp, xsincos_complex	;; Complex sin/cos data
	bump	rbp, 512*YMM_SCD2
	yloop_init 640			;; 5 iterations of 128
	yloop_set_prefetch YSINGLE
	yr4_s4cl_2sc_eight_reals_unfft_preload
c2a:	yr4_s4cl_2sc_eight_reals_unfft rsi, 64, 2*dist64, 4*dist64, rbp, YMM_SCD2, rdi, YMM_SCD2, 64
	yloop	64, c2a, -64*64+dist64	;; Test loop counter, next source pointer
	yloop	2, c2a, -2*dist64+8*dist64, rbp, -128*YMM_SCD2 ;; Test loop counter, next source pointer, restore sine/cosine pointer

	loops_reset			;; 4 more iters of 128
	yr4_s4cl_four_complex_djbunfft_preload
c2b:	yr4_s4cl_four_complex_djbunfft rsi, 64, 2*dist64, 4*dist64, rbp, YMM_SCD2, 64
	yloop	64, c2b, -64*64+dist64	;; Test loop counter, next source pointer
	yloop	2, c2b, -2*dist64+8*dist64, rbp, -128*YMM_SCD2 ;; Test loop counter, next source pointer, restore sine/cosine pointer
	yloop	5, c2b, -5*8*dist64	;; Test loop counter, restore source pointer

;; Do inverse FFT levels 1,2,3 which correspond to complex levels 1,2
;; On input the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	2048	+1	+1	+1
;;	4	...
;;	...
;;	2044	...
;;	4096	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	10240	+1	+1	+1
;;	4	...
;;	...
;;	10236

;; Do 2048 ten_reals_unfft operations
;;	distance between fft data elements is 2048
;;	do 512 macros each processing 40 data values

	bump	rdi, -128*YMM_SCD2-512*YMM_SCD2	;; Real sin/cos ptr
	bump	rbp, -512*YMM_SCD2	;; Complex sin/cos data
	bump	rcx, (1296-2*640-16)*64 ;; Move prefetch pointer along
	yloop_init 512			;; 512 iterations
	yloop_set_prefetch YOTHER, 32
	yr5_5cl_2sc_ten_reals_unfft_preload
c1b:	yr5_5cl_2sc_ten_reals_unfft rsi, 64, 8*dist64, rbp, YMM_SCD2, rdi, YMM_SCD2, 64
	yloop	64, c1b, -64*64+dist64	;; Test loop counter, next source pointer
	yloop	8, c1b			;; Test loop counter

ypass2_10240_real_done:
	end_timer 2
	ENDM

; *************** radix4-pass2-10240-complex macro ******************
; This macro takes 10240 complex values and performs the final 13.322 levels of a standard
; radix-4 FFT, squares the results, and does the first 13.322 levels of the inverse FFT.

yr4dwpn_pass2_10240_complex MACRO
	LOCAL	b1a, b1b, b2b, b3b, b4a0, b4a, b4b0, b4b, b4c0, b4c, b5a, b5b, b5c, b6a, b6b, b6c, b6d0, b6d
	LOCAL	c1b, c2b, c3a, c3b, c4b, c4c, c4d, c5b, c5c, c5d
	LOCAL	ypass2_10240_complex_1, ypass2_10240_complex_2
	LOCAL	ypass2_10240_complex_3, ypass2_10240_complex_4
	LOCAL	ypass2_10240_complex_unfft, ypass2_10240_complex_done

;; Type 4 FFTs skip the forward FFT process

	mov	rsi, DATA_ADDR		;; Load address of FFT data
	mov	rbx, DIST_TO_FFTSRCARG
	mov	rcx, DATA_PREFETCH	;; Prefetch pointer (we need to prefetch 
					;; 2560 cache lines + 39 pad cache lines)
	cmp	ffttype, 4
	je	ypass2_10240_complex_4

;; Do FFT level 1,2 (actually 2.322 levels)
;; On input the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	10240	+1	+1	+1
;;	4	...
;;	...
;;	10236
;; On output the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	2048	+1	+1	+1
;;	4	...
;;	...
;;	2044	...
;;	4096	...
;;	...

;; Do 2048 five_complex_fft operations
;;	distance between fft data elements is 2048
;;	do 512 macros each processing 40 data values

;; Do FFT levels 3,4
;; On input the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	2048	+1	+1	+1
;;	4	...
;;	...
;;	2044	...
;;	4096	...
;;	...
;; On output the 64-byte cache lines are swizzled to hold these data values:
;;	0	+1K	+1K	+1K	512	+1K	+1K	+1K
;;	4	...
;;	...
;;	508	...
;;	1	...
;;	...
;;	4096	...
;;	...

;; Do 2560 four_complex_fft operations
;;	distance between fft data elements is 512
;;	do 640 macros each processing 32 data values

;; Do FFT levels 5,6
;; On input the 64-byte cache lines hold these data values:
;;	0	+1K	+1K	+1K	512	+1K	+1K	+1K
;;	4	...
;;	...
;;	508	...
;;	1	...
;;	...
;;	4096	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+1K	+1K	+1K	128	+1K	+1K	+1K
;;	4	...
;;	...
;;	124	...
;;	256	...
;;	...
;;	1	...
;;	...
;;	4096	...
;;	...

;; Do 2560 four_complex_fft operations
;;	distance between fft data elements is 128
;;	do 640 macros each processing 32 data values

	mov	rdi, xsincos_complex	;; The sin/cos ptrs
	lea	rbp, [rdi+512*YMM_SCD2]
	mov	rdx, rbp
	yloop_init 640			;; 512 iterations or 640 iterations

b1a:	start_timer 4
	yloop_adjust 4			;; Four fewer iterations this loop
	yloop_set_prefetch YQUARTER 	;; Prefetch 128 cache lines in this loop
	yr5_f5cl_five_complex_djbfft_preload
b1b:	yr5_f5cl_five_complex_djbfft rsi, 32*64, 8*dist64, rdi, 32*YMM_SCD2, 2
	yloop	2, b1b, -2*32*64+dist64	;; Test loop counter, next source pointer
	yloop	10, b1b, -8*dist64, rdi, -16*32*YMM_SCD2+YMM_SCD2 ;; 8 iterations!!  Restore source pointer, next sin/cos data
	loops_undo 20			;; Back up loop counter
	end_timer 4

	start_timer 5
	yloop_set_prefetch YQUARTER	;; Prefetch 160 cache lines in this loop
	yr4_s4cl_four_complex_djbfft_preload
b2b:	yr4_s4cl_four_complex_djbfft rsi, 32*64, 2*dist64, 4*dist64, rbp, 32*YMM_SCD2, 2
	yloop	2, b2b, -2*32*64+dist64	;; Test loop counter, next source pointer
	yloop	2, b2b, -2*dist64+8*dist64, rbp, -4*32*YMM_SCD2 ;; Test loop counter, next source pointer, restore sin/cos data
	yloop	5, b2b, -5*8*dist64, rbp, YMM_SCD2 ;; Test loop counter, restore source pointer, next sin/cos data
	loops_undo 5*4			;; Back up loop counter
	end_timer 5

	start_timer 6
	yloop_set_prefetch YQUARTER	;; Prefetch 160 cache lines in this loop
	yr4_eb4cl_four_complex_djbfft_preload
b3b:	yr4_eb4cl_four_complex_djbfft rsi, 2*dist64, 32*64, dist64, rdx, YMM_SCD2, 4, L1PREFETCH_ALL, 64
	yloop	4, b3b, 0, rdx, -4*YMM_SCD2 ;; Loop 4 times, restore sine/cosine pointer
	yloop	5, b3b, -5*4*2*dist64+64, rdx, 4*YMM_SCD2 ;; Loop 5 times, next source pointer, next sine/cosine pointer
	end_timer 6

	yloop	32, b1a, -32*64, rdi, -32*YMM_SCD2+512*YMM_SCD2+128*YMM_SCD2 ;; Test loop counter, restore source pointer, next sin/cos ptr

;; Do FFT levels 7,8
;; On input the 64-byte cache lines hold these data values:
;;	0	+1K	+1K	+1K	128	+1K	+1K	+1K
;;	4	...
;;	...
;;	124	...
;;	256	...
;;	...
;;	1	...
;;	...
;;	4096	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+1K	+1K	+1K	32	+1K	+1K	+1K
;;	4	...
;;	...
;;	28	...
;;	64	...
;;	...
;;	1	...
;;	...
;;	4096	...
;;	...

;; Do 2560 four_complex_fft operations
;;	distance between fft data elements is 32
;;	do 640 macros each processing 32 data values

;; Do FFT levels 9,10
;; On input the 64-byte cache lines hold these data values:
;;	0	+1K	+1K	+1K	32	+1K	+1K	+1K
;;	4	...
;;	...
;;	28	...
;;	64	...
;;	...
;;	1	...
;;	...
;;	4096	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+1K	+1K	+1K	8	+1K	+1K	+1K
;;	4	...
;;	16	...
;;	...
;;	1	...
;;	...
;;	4096	...
;;	...

;; Do 2560 four_complex_fft operations
;;	distance between fft data elements is 8
;;	do 640 macros each processing 32 data values

;; Do FFT levels 11,12,13 as well as inverse FFT levels 11,12,13
;; On input and output the 64-byte cache lines hold these data values:
;;	0	+1K	+1K	+1K	8	+1K	+1K	+1K
;;	4	...
;;	16	...
;;	...
;;	1	...
;;	...
;;	4096	...
;;	...

;; Do 1280 eight_complex_fft operations
;;	distance between fft data elements is 1
;;	do 320 macros each processing 64 data values

;; Do inverse FFT levels 9,10
;; On input the 64-byte cache lines hold these data values:
;;	0	+1K	+1K	+1K	8	+1K	+1K	+1K
;;	4	...
;;	16	...
;;	...
;;	1	...
;;	...
;;	4096	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+1K	+1K	+1K	32	+1K	+1K	+1K
;;	4	...
;;	...
;;	28	...
;;	64	...
;;	...
;;	1	...
;;	...
;;	4096	...
;;	...

;; Do 2560 four_complex_unfft operations
;;	distance between fft data elements is 8
;;	do 640 macros each processing 32 data values

;; Do inverse FFT levels 7,8
;; On input the 64-byte cache lines hold these data values:
;;	0	+1K	+1K	+1K	32	+1K	+1K	+1K
;;	4	...
;;	...
;;	28	...
;;	64	...
;;	...
;;	1	...
;;	...
;;	4096	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+1K	+1K	+1K	128	+1K	+1K	+1K
;;	4	...
;;	...
;;	124	...
;;	256	...
;;	...
;;	1	...
;;	...
;;	4096	...
;;	...

;; Do 2560 four_complex_unfft operations
;;	distance between fft data elements is 32
;;	do 640 macros each processing 32 data values

;; Do this 128 cache lines (8KB) at a time to maximize accessing data
;; while it is in the L1 cache.

;; Execute the right middle step

	cmp	ffttype, 2
	jl	ypass2_10240_complex_1
	jg	ypass2_10240_complex_3

ypass2_10240_complex_2:
	yloop_init 640			;; 20 iters of 128 cache lines

b4b0:	start_timer 7
	yloop_set_prefetch YHALF	;; Prefetch 320 cache lines in this loop
	yr4_b4cl_four_complex_djbfft_preload
b4b:	yr4_b4cl_four_complex_djbfft rsi, 64, 8*64, 16*64, rdi, YMM_SCD2/4, 8
	yloop	8, b4b, -8*64+2*dist64	;; Loop 8 times, next source pointer
	yloop	4, b4b, -4*2*dist64, rdi, -32*YMM_SCD2/4 ;; Loop 4 times, restore source pointer, restore sine/cosine pointer
	loops_undo 4*8			;; Back up loop counter
	end_timer 7

	start_timer 8
	yloop_set_prefetch YHALF	;; Prefetch 320 cache lines in this loop
;;	yr4_b4cl_four_complex_djbfft_preload
b5b:	yr4_b4cl_four_complex_djbfft rsi, 64, 2*64, 4*64, rdi, 4*YMM_SCD2/4, 2
	yloop	2, b5b, -2*64+8*64, rdi, -2*4*YMM_SCD2/4 ;; Loop 2 times, next source pointer, restore sine/cosine pointer
	yloop	4, b5b, -4*8*64+2*dist64, rdi, YMM_SCD2/4 ;; Loop 4 times, next source pointer, next sine/cosine pointer
	yloop	4, b5b, -4*2*dist64, rdi, -4*YMM_SCD2/4 ;; Loop 4 times, restore source pointer, restore sine/cosine pointer
	loops_undo 4*4*2		;; Back up loop counter
	end_timer 8

	start_timer 9
	yloop_adjust 4*4		;; 16 fewer iterations this loop
	yloop_set_prefetch YHALF	;; Prefetch 320 cache lines in this loop
	yr8_8cl_eight_complex_with_square_preload
;; Loop below reworked to avoid 4KB distances
;;b6b:	yr8_8cl_eight_complex_with_square rsi, 2*64, 2*dist64, 4*dist64, 64, 32
;;	yloop	32, b6b, -16*2*64	;; 16 iterations!!  Restore source pointer
b6b:	yr8_8cl_eight_complex_with_square rsi, 8*64, 2*dist64, 4*dist64, 64, 4
	yloop	4, b6b, -4*8*64+2*64	;; Loop 4 times, next source pointer
	yloop	8, b6b, -4*2*64		;; 4 iterations!!  Restore source pointer
	loops_undo 8*4			;; Back up loop counter
	end_timer 9

	start_timer 10
	yloop_set_prefetch YHALF	;; Prefetch 320 cache lines in this loop
	yr4_b4cl_four_complex_djbunfft_preload
c5b:	yr4_b4cl_four_complex_djbunfft rsi, 64, 2*64, 4*64, rdi, 4*YMM_SCD2/4, 2
	yloop	2, c5b, -2*64+8*64, rdi, -2*4*YMM_SCD2/4 ;; Loop 2 times, next source pointer, restore sine/cosine pointer
	yloop	4, c5b, -4*8*64+2*dist64, rdi, YMM_SCD2/4 ;; Loop 4 times, next source pointer, next sine/cosine pointer
	yloop	4, c5b, -4*2*dist64, rdi, -4*YMM_SCD2/4 ;; Loop 4 times, restore source pointer, restore sine/cosine pointer
	loops_undo 4*4*2		;; Back up loop counter
	end_timer 10

	start_timer 11
	yloop_set_prefetch YHALF	;; Prefetch 320 cache lines in this loop
;;	yr4_b4cl_four_complex_djbunfft_preload
c4b:	yr4_b4cl_four_complex_djbunfft rsi, 64, 8*64, 16*64, rdi, YMM_SCD2/4, 8, L1PREFETCH_ALL, 32*64
	yloop	8, c4b, -8*64+2*dist64	;; Loop 8 times, next source pointer
	yloop	4, c4b, -4*2*dist64+32*64, rdi, -32*YMM_SCD2/4 ;; Loop 4 times, next source pointer, restore sine/cosine pointer
	end_timer 11

	yloop	2, b4b0, -2*32*64+dist64 ;; Loop 2 times, next source pointer
	yloop	2, b4b0, -2*dist64+8*dist64 ;; Loop 2 times, next source pointer
	yloop	5, b4b0, -5*8*dist64	;; Loop 5 times, restore source pointer
	jmp	ypass2_10240_complex_unfft


ypass2_10240_complex_1:
	start_timer 6
;;	bump	rcx, (2560+39-128-2*160-3*640)*64 ;; Move prefetch pointer along
	yloop_init 640			;; 20 iters of 128 cache lines

b4a0:	yloop_set_prefetch YSINGLE	;; Prefetch 640 cache lines in this loop
	yr4_b4cl_four_complex_djbfft_preload
b4a:	yr4_b4cl_four_complex_djbfft rsi, 64, 8*64, 16*64, rdi, YMM_SCD2/4, 8
	yloop	8, b4a, -8*64+2*dist64	;; Loop 8 times, next source pointer
	yloop	4, b4a, -4*2*dist64, rdi, -32*YMM_SCD2/4 ;; Loop 4 times, restore source pointer, restore sine/cosine pointer
	loops_undo 4*8			;; Back up loop counter

	yloop_set_prefetch YSINGLE	;; Prefetch 640 cache lines in this loop
;;	yr4_b4cl_four_complex_djbfft_preload
b5a:	yr4_b4cl_four_complex_djbfft rsi, 64, 2*64, 4*64, rdi, 4*YMM_SCD2/4, 2, L1PREFETCH_ALL, 32*64
	yloop	2, b5a, -2*64+8*64, rdi, -2*4*YMM_SCD2/4 ;; Loop 2 times, next source pointer, restore sine/cosine pointer
	yloop	4, b5a, -4*8*64+2*dist64, rdi, YMM_SCD2/4 ;; Loop 4 times, next source pointer, next sine/cosine pointer
	yloop	4, b5a, -4*2*dist64, rdi, -4*YMM_SCD2/4 ;; Loop 4 times, restore source pointer, restore sine/cosine pointer
	loops_undo 4*4*2		;; Back up loop counter

	yloop_adjust 4*4		;; 16 fewer iterations this loop
	yloop_set_prefetch YDOUBLE	;; Prefetch 640 cache lines in this loop
	yr8_8cl_eight_complex_fft_final_preload
;; Loop below reworked to avoid 4KB distances
;;b6a:	yr8_8cl_eight_complex_fft_final rsi, 2*64, 2*dist64, 4*dist64, 64
;;	yloop	32, b6a, -16*2*64	;; 16 iterations!!  Restore source pointer
b6a:	yr8_8cl_eight_complex_fft_final rsi, 8*64, 2*dist64, 4*dist64, 64
	yloop	4, b6a, -4*8*64+2*64	;; Loop 4 times, next source pointer
	yloop	8, b6a, -4*2*64+32*64	;; 4 iterations!!  Next source pointer

	yloop	2, b4a0, -2*32*64+dist64 ;; Loop 2 times, next source pointer
	yloop	2, b4a0, -2*dist64+8*dist64 ;; Loop 2 times, next source pointer
	yloop	5, b4a0			;; Loop 5 times
	end_timer 6
	jmp	ypass2_10240_complex_done


ypass2_10240_complex_3:
	start_timer 6
x64	mov	r8, DEST2ARG
	mov	rbp, DIST_TO_MULSRCARG
	yloop_init 640			;; 20 iters of 128 cache lines

b4c0:	yloop_set_prefetch YHALF	;; Prefetch 320 cache lines in this loop
	yr4_b4cl_four_complex_djbfft_preload
b4c:	yr4_b4cl_four_complex_djbfft rsi, 64, 8*64, 16*64, rdi, YMM_SCD2/4, 8
	yloop	8, b4c, -8*64+2*dist64	;; Loop 8 times, next source pointer
	yloop	4, b4c, -4*2*dist64, rdi, -32*YMM_SCD2/4 ;; Loop 4 times, restore source pointer, restore sine/cosine pointer
	loops_undo 4*8			;; Back up loop counter

	yloop_set_prefetch YHALF	;; Prefetch 320 cache lines in this loop
;;	yr4_b4cl_four_complex_djbfft_preload
b5c:	yr4_b4cl_four_complex_djbfft rsi, 64, 2*64, 4*64, rdi, 4*YMM_SCD2/4, 2
	yloop	2, b5c, -2*64+8*64, rdi, -2*4*YMM_SCD2/4 ;; Loop 2 times, next source pointer, restore sine/cosine pointer
	yloop	4, b5c, -4*8*64+2*dist64, rdi, YMM_SCD2/4 ;; Loop 4 times, next source pointer, next sine/cosine pointer
	yloop	4, b5c, -4*2*dist64, rdi, -4*YMM_SCD2/4 ;; Loop 4 times, restore source pointer, restore sine/cosine pointer
	loops_undo 4*4*2		;; Back up loop counter

	yloop_adjust 4*4		;; 16 fewer iterations this loop
	yloop_set_prefetch YHALF	;; Prefetch 320 cache lines in this loop
	yr8_8cl_eight_complex_with_mult_preload
;; Loop below reworked to avoid 4KB distances
;;b6c:	yr8_8cl_eight_complex_with_mult rsi, 2*64, 2*dist64, 4*dist64, 64
;;	yloop	32, b6c, -16*2*64	;; 16 iterations!!  Restore source pointer
b6c:	yr8_8cl_eight_complex_with_mult rsi, 8*64, 2*dist64, 4*dist64, 64
	yloop	4, b6c, -4*8*64+2*64	;; Loop 4 times, next source pointer
	yloop	8, b6c, -4*2*64		;; 4 iterations!!  Restore source pointer
x64	add	rsi, r8
	loops_undo 8*4			;; Back up loop counter

	yloop_set_prefetch YHALF	;; Prefetch 320 cache lines in this loop
	yr4_b4cl_four_complex_djbunfft_preload
c5c:	yr4_b4cl_four_complex_djbunfft rsi, 64, 2*64, 4*64, rdi, 4*YMM_SCD2/4, 2
	yloop	2, c5c, -2*64+8*64, rdi, -2*4*YMM_SCD2/4 ;; Loop 2 times, next source pointer, restore sine/cosine pointer
	yloop	4, c5c, -4*8*64+2*dist64, rdi, YMM_SCD2/4 ;; Loop 4 times, next source pointer, next sine/cosine pointer
	yloop	4, c5c, -4*2*dist64, rdi, -4*YMM_SCD2/4 ;; Loop 4 times, restore source pointer, restore sine/cosine pointer
	loops_undo 4*4*2		;; Back up loop counter

	yloop_set_prefetch YHALF	;; Prefetch 320 cache lines in this loop
;;	yr4_b4cl_four_complex_djbunfft_preload
c4c:	yr4_b4cl_four_complex_djbunfft rsi, 64, 8*64, 16*64, rdi, YMM_SCD2/4, 8, L1PREFETCH_ALL, 32*64
	yloop	8, c4c, -8*64+2*dist64	;; Loop 8 times, next source pointer
	yloop	4, c4c, -4*2*dist64+32*64, rdi, -32*YMM_SCD2/4 ;; Loop 4 times, next source pointer, restore sine/cosine pointer

x64	sub	rsi, r8
	yloop	2, b4c0, -2*32*64+dist64 ;; Loop 2 times, next source pointer
	yloop	2, b4c0, -2*dist64+8*dist64 ;; Loop 2 times, next source pointer
	yloop	5, b4c0, -5*8*dist64	;; Loop 5 times, restore source pointer
x64	add	rsi, r8
	end_timer 6
	jmp	ypass2_10240_complex_unfft


ypass2_10240_complex_4:
	start_timer 6
	mov	rdi, xsincos_complex	;; The sin/cos ptrs
	bump	rdi, 512*YMM_SCD2+128*YMM_SCD2
	mov	rbp, DIST_TO_MULSRCARG
	yloop_init 640			;; 20 iters of 128 cache lines

b6d0:	yloop_adjust 4*4		;; 16 fewer iterations this loop
	yloop_set_prefetch YDOUBLE	;; Prefetch 640 cache lines in this loop
	yr8_8cl_eight_complex_with_mulf_preload
;; Loop below reworked to avoid 4KB distances
;;b6d:	yr8_8cl_eight_complex_with_mulf rsi, 2*64, 2*dist64, 4*dist64, 64
;;	yloop	32, b6d, -16*2*64	;; 16 iterations!!  Restore source pointer
b6d:	yr8_8cl_eight_complex_with_mulf rsi, 8*64, 2*dist64, 4*dist64, 64
	yloop	4, b6d, -4*8*64+2*64	;; Loop 4 times, next source pointer
	yloop	8, b6d, -4*2*64		;; 4 iterations!!  Restore source pointer
	loops_undo 8*4			;; Back up loop counter

	yloop_set_prefetch YSINGLE	;; Prefetch 640 cache lines in this loop
	yr4_b4cl_four_complex_djbunfft_preload
c5d:	yr4_b4cl_four_complex_djbunfft rsi, 64, 2*64, 4*64, rdi, 4*YMM_SCD2/4, 2
	yloop	2, c5d, -2*64+8*64, rdi, -2*4*YMM_SCD2/4 ;; Loop 2 times, next source pointer, restore sine/cosine pointer
	yloop	4, c5d, -4*8*64+2*dist64, rdi, YMM_SCD2/4 ;; Loop 4 times, next source pointer, next sine/cosine pointer
	yloop	4, c5d, -4*2*dist64, rdi, -4*YMM_SCD2/4 ;; Loop 4 times, restore source pointer, restore sine/cosine pointer
	loops_undo 4*4*2		;; Back up loop counter

	yloop_set_prefetch YSINGLE	;; Prefetch 640 cache lines in this loop
;;	yr4_b4cl_four_complex_djbunfft_preload
c4d:	yr4_b4cl_four_complex_djbunfft rsi, 64, 8*64, 16*64, rdi, YMM_SCD2/4, 8, L1PREFETCH_ALL, 32*64
	yloop	8, c4d, -8*64+2*dist64	;; Loop 8 times, next source pointer
	yloop	4, c4d, -4*2*dist64+32*64, rdi, -32*YMM_SCD2/4 ;; Loop 4 times, next source pointer, restore sine/cosine pointer

	yloop	2, b6d0, -2*32*64+dist64 ;; Loop 2 times, next source pointer
	yloop	2, b6d0, -2*dist64+8*dist64 ;; Loop 2 times, next source pointer
	yloop	5, b6d0, -5*8*dist64	;; Loop 5 times, restore source pointer
;;	bump	rcx, (-3*640+128+2*160+5*320)*64 ;; Move prefetch pointer to where squaring would be
	end_timer 6

;; Do inverse FFT levels 5,6
;; On input the 64-byte cache lines hold these data values:
;;	0	+1K	+1K	+1K	128	+1K	+1K	+1K
;;	4	...
;;	...
;;	124	...
;;	256	...
;;	...
;;	1	...
;;	...
;;	4096	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+1K	+1K	+1K	512	+1K	+1K	+1K
;;	4	...
;;	...
;;	508	...
;;	1	...
;;	...
;;	4096	...
;;	...

;; Do 2560 four_complex_unfft operations
;;	distance between fft data elements is 128
;;	do 640 macros each processing 32 data values

;; Do inverse FFT levels 3,4
;; On input the 64-byte cache lines hold these data values:
;;	0	+1K	+1K	+1K	512	+1K	+1K	+1K
;;	4	...
;;	...
;;	508	...
;;	1	...
;;	...
;;	4096	...
;;	...
;; On output the 64-byte cache lines are swizzled to hold these data values:
;;	0	+1	+1	+1	2048	+1	+1	+1
;;	4	...
;;	...
;;	2044	...
;;	4096	...
;;	...

;; Do 2560 four_complex_unfft operations
;;	distance between fft data elements is 512
;;	do 640 macros each processing 32 data values

;; Do inverse FFT levels 1,2
;; On input the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	2048	+1	+1	+1
;;	4	...
;;	...
;;	2044	...
;;	4096	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	10240	+1	+1	+1
;;	4	...
;;	...
;;	6140

;; Do 2048 five_complex_unfft operations
;;	distance between fft data elements is 2048
;;	do 512 macros each processing 40 data values

ypass2_10240_complex_unfft:
	mov	rdi, xsincos_complex	;; Load sin/cos pointers
	lea	rbp, [rdi+512*YMM_SCD2]
	mov	rdx, rbp
	bump	rcx, (2560+39-128-2*160-5*320-2*160-256)*64 ;; Move prefetch pointer along
	yloop_init 640			;; 512 iterations or 640 iterations

c3a:	start_timer 12
	yloop_set_prefetch YQUARTER	;; Prefetch 160 cache lines in this loop
	yr4_eb4cl_four_complex_djbunfft_preload
c3b:	yr4_eb4cl_four_complex_djbunfft rsi, 2*dist64, 32*64, dist64, rdx, YMM_SCD2, 4
	yloop	4, c3b, 0, rdx, -4*YMM_SCD2 ;; Loop 4 times, restore sine/cosine pointer
	yloop	5, c3b, -5*4*2*dist64, rdx, 4*YMM_SCD2 ;; Loop 3 times, restore source pointer, next sine/cosine pointer
	loops_undo 5*4			;; Back up loop counter
	end_timer 12

	start_timer 13
	yloop_set_prefetch YQUARTER	;; Prefetch 160 cache lines in this loop
	yr4_s4cl_four_complex_djbunfft_preload
c2b:	yr4_s4cl_four_complex_djbunfft rsi, 32*64, 2*dist64, 4*dist64, rbp, 32*YMM_SCD2, 2, L1PREFETCH_ALL, 64
	yloop	2, c2b, -2*32*64+dist64	;; Test loop counter, next source pointer
	yloop	2, c2b, -2*dist64+8*dist64, rbp, -4*32*YMM_SCD2 ;; Test loop counter, next source pointer, restore sine/cosine pointer
	yloop	5, c2b, -5*8*dist64, rbp, YMM_SCD2 ;; Test loop counter, restore source pointer, next sine/cosine pointer
	loops_undo 5*4			;; Back up loop counter
	end_timer 13

	start_timer 14
	yloop_adjust 4			;; Four fewer iterations this loop
	yloop_set_prefetch YHALF	;; Prefetch 256 cache lines in this loop
	yr5_5cl_five_complex_djbunfft_preload
c1b:	yr5_5cl_five_complex_djbunfft rsi, 32*64, 8*dist64, rdi, 32*YMM_SCD2, 2
	yloop	2, c1b, -2*32*64+dist64	;; Test loop counter, next source pointer
	yloop	10, c1b, -8*dist64+64, rdi, -16*32*YMM_SCD2+YMM_SCD2 ;; 8 iterations!!  Next source pointer, next sine/cosine pointer
	end_timer 14

	yloop	32, c3a			;; Loop 32 times

ypass2_10240_complex_done:
	ENDM


; *************** radix4-pass2-12800-real macro ******************
; This macro takes 25600 real values and performs the final 14.644 levels of the FFT.
; The results are squared, and the first 14.644 levels of the inverse FFT are done.

yr4dwpn_pass2_12800_real MACRO
	LOCAL	b1b, b2a, b2b, b3a, b3b, b4a, b4b, b5a, b5b, b6a, b6b, b6c, b6d
	LOCAL	c1b, c2a, c2b, c3a, c3b, c4a, c4b, c5a, c5b
	LOCAL	ypass2_12800_real_1, ypass2_12800_real_2
	LOCAL	ypass2_12800_real_3, ypass2_12800_real_4
	LOCAL	ypass2_12800_real_unfft, ypass2_12800_real_done

;; Type 4 FFTs skip the forward FFT process

	start_timer 2
	mov	rsi, DATA_ADDR		;; Load source address
	mov	rbx, DIST_TO_FFTSRCARG
	mov	rcx, DATA_PREFETCH	;; Prefetch pointer (we need to prefetch 
					;; 3200 cache lines + 2*49 pad cache lines)
	cmp	ffttype, 4
	je	ypass2_12800_real_4

;; Do FFT levels 1,2,3 which correspond to complex levels 1,2
;; On input the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	12800	+1	+1	+1
;;	4	...
;;	...
;;	12796
;; On output the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	2560	+1	+1	+1
;;	4	...
;;	...
;;	2556	...
;;	5120	...
;;	...

;; Do 2560 ten_reals_fft operations
;;	distance between fft data elements is 2560
;;	do 640 macros each processing 40 data values

	mov	rdi, sincos3		;; Real sin/cos pointer
	mov	rbp, xsincos_complex	;; Complex sin/cos data
	yloop_init 640			;; 640 iterations
	yloop_set_prefetch YHALF
	yr5_f5cl_2sc_ten_reals_fft_preload
b1b:	yr5_f5cl_2sc_ten_reals_fft rsi, 64, 10*dist64, rbp, YMM_SCD2, rdi, YMM_SCD2, 64
	yloop	64, b1b, -64*64+dist64	;; Test loop counter, next source pointer
	yloop	10, b1b, -10*dist64	;; Test loop counter, restore source pointer

;; Do FFT levels 3,4
;; On input the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	2560	+1	+1	+1
;;	4	...
;;	...
;;	2556	...
;;	5120	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	512	+1	+1	+1
;;	4	...
;;	...
;;	508	...
;;	1024	...
;;	...

;; Do 512 ten_reals_fft operations
;; Do 1024 five_complex_fft operations
;;	distance between fft data elements is 512
;;	do 640 macros each processing 40 data values

	yloop_init 640			;; 5 iters of 128
	yloop_set_prefetch YHALF
	yr5_5cl_2sc_ten_reals_fft_preload
b2a:	yr5_5cl_2sc_ten_reals_fft rsi, 64, 2*dist64, rbp, YMM_SCD2, rdi, YMM_SCD2, 64
	yloop	64, b2a, -64*64+dist64	;; Test loop counter, next source pointer
	yloop	2, b2a, -2*dist64+10*dist64, rbp, -128*YMM_SCD2 ;; Test loop counter, next source pointer, restore sine/cosine pointer

	loops_reset			;; 4 more iters of 128
	yr5_5cl_five_complex_djbfft_preload
b2b:	yr5_5cl_five_complex_djbfft rsi, 64, 2*dist64, rbp, YMM_SCD2, 64
	yloop	64, b2b, -64*64+dist64	;; Test loop counter, next source pointer
	yloop	2, b2b, -2*dist64+10*dist64, rbp, -128*YMM_SCD2 ;; Test loop counter, next source pointer, restore sin/cos ptr
	yloop	5, b2b, -5*10*dist64, rbp, 128*YMM_SCD2 ;; Loop 5 times, restore source pointer, next sin/cos pointer

;; Do FFT levels 5,6
;; On input the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	512	+1	+1	+1
;;	4	...
;;	...
;;	508	...
;;	1024	...
;;	...
;; On output the 64-byte cache lines are swizzled to hold these data values:
;;	0	+256	+256	+256	128	+256	+256	+256
;;	4	...
;;	...
;;	124	...
;;	1	...
;;	...
;;	1024	...
;;	...

;; Do 128 eight_reals_fft operations
;; Do 3072 four_complex_fft operations
;;	distance between fft data elements is 128
;;	do 800 macros each processing 32 data values

	yloop_init 800			;; 25 iterations of 32
	yloop_set_prefetch YHALF
	yr4_s4cl_2sc_eight_reals_fft_preload
b3a:	yr4_s4cl_2sc_eight_reals_fft rsi, 64, 32*64, dist64, rbp, YMM_SCD2, rdi, YMM_SCD2, 32
	yloop	32, b3a, -32*64+2*dist64, rbp, -32*YMM_SCD2 ;; Test loop counter, next source pointer, restore sine/cosine pointer

	loops_reset			;; 24 more iters of 32
	yr4_s4cl_four_complex_djbfft_preload
b3b:	yr4_s4cl_four_complex_djbfft rsi, 64, 32*64, dist64, rbp, YMM_SCD2, 32
	yloop	32, b3b, -32*64+2*dist64, rbp, -32*YMM_SCD2 ;; Test loop counter, next source pointer, restore sin/cos ptr
	yloop	25, b3b, -25*2*dist64	;; Loop 25 times, restore source pointer

;; Do FFT levels 7,8
;; On input the 64-byte cache lines hold these data values:
;;	0	+256	+256	+256	128	+256	+256	+256
;;	4	...
;;	...
;;	124	...
;;	1	...
;;	...
;;	1024	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+256	+256	+256	32	+256	+256	+256
;;	4	...
;;	...
;;	28	...
;;	64	...
;;	...
;;	1	...
;;	...
;;	1024	...
;;	...

;; Do 32 eight_reals_fft operations
;; Do 3168 four_complex_fft operations
;;	distance between fft data elements is 32
;;	do 800 macros each processing 32 data values

	yloop_init 800			;; 25 iterations of 32
	yloop_set_prefetch YHALF
	yr4_4cl_eight_reals_four_complex_djbfft_preload
b4a:	yr4_4cl_eight_reals_four_complex_djbfft rsi, 32*64, 8*64, 16*64, rdi, YMM_SCD3, 2
	yloop	2, b4a, -2*32*64+dist64	;; Test loop counter, next source pointer
	yloop	2, b4a, -2*dist64+64	;; Test loop counter, next source pointer
	yloop	8, b4a, -8*64+2*dist64, rdi, -32*YMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer

	loops_reset			;; 24 more iters of 32
	yr4_rb4cl_four_complex_djbfft_preload
b4b:	yr4_rb4cl_four_complex_djbfft rsi, 32*64, 8*64, 16*64, rdi, YMM_SCD3, 2
	yloop	2, b4b, -2*32*64+dist64	;; Test loop counter, next source pointer
	yloop	2, b4b, -2*dist64+64	;; Test loop counter, next source pointer
	yloop	8, b4b, -8*64+2*dist64, rdi, -32*YMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer
	yloop	25, b4b, -25*2*dist64	;; Test loop counter, restore source pointer

;; Do FFT levels 9,10
;; On input the 64-byte cache lines hold these data values:
;;	0	+256	+256	+256	32	+256	+256	+256
;;	4	...
;;	...
;;	28	...
;;	64	...
;;	...
;;	1	...
;;	...
;;	1024	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+256	+256	+256	8	+256	+256	+256
;;	4	...
;;	16	...
;;	...
;;	1	...
;;	...
;;	1024	...
;;	...

;; Do 8 eight_reals_fft operations
;; Do 3192 four_complex_fft operations
;;	distance between fft data elements is 8
;;	do 800 macros each processing 32 data values

	yloop_init 800			;; 100 iterations of 8
	yloop_set_prefetch YHALF
	yr4_4cl_eight_reals_four_complex_djbfft_preload
b5a:	yr4_4cl_eight_reals_four_complex_djbfft rsi, 32*64, 2*64, 4*64, rdi, 4*YMM_SCD3, 2
	yloop	2, b5a, -2*32*64+dist64	;; Test loop counter, next source pointer
	yloop	2, b5a, -2*dist64+64	;; Test loop counter, next source pointer
	yloop	2, b5a, -2*64+8*64, rdi, -8*4*YMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer

	loops_reset			;; 99 more iters of 8
	yr4_rb4cl_four_complex_djbfft_preload
b5b:	yr4_rb4cl_four_complex_djbfft rsi, 32*64, 2*64, 4*64, rdi, 4*YMM_SCD3, 2
	yloop	2, b5b, -2*32*64+dist64	;; Test loop counter, next source pointer
	yloop	2, b5b, -2*dist64+64	;; Test loop counter, next source pointer
	yloop	2, b5b, -2*64+8*64, rdi, -8*4*YMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer
	yloop	4, b5b, -4*8*64+2*dist64 ;; Test loop counter, next source pointer
	yloop	25, b5b, -25*2*dist64	;; Test loop counter, restore source pointer

;; Do FFT levels 11,12,13 as well as inverse FFT levels 11,12,13
;; On input and output the 64-byte cache lines hold these data values:
;;	0	+256	+256	+256	8	+256	+256	+256
;;	4	...
;;	16	...
;;	...
;;	1	...
;;	...
;;	1024	...
;;	...

;; Do 1 sixteen_reals_fft operation
;; Do 1659 eight_complex_fft operations
;;	distance between fft data elements is 1
;;	do 400 macros each processing 64 data values

	cmp	ffttype, 2
	jl	ypass2_12800_real_1
	jg	ypass2_12800_real_3

ypass2_12800_real_2:
;;	yr8_8cl_sixteen_reals_eight_complex_with_square rsi, 2*64, 32*64, dist64, 64
	yr8_dispatch call, y8real_square, 32*64, dist64, 64
	yloop_init 400			;; 399 more iterations
	yloop_adjust 1
	yloop_set_prefetch YSINGLE
	yr8_8cl_eight_complex_with_square_preload
b6b:	yr8_8cl_eight_complex_with_square rsi, 2*64, 32*64, dist64, 64, 1
	yloop	16, b6b, -16*2*64+2*dist64	;; Test loop counter, next source pointer
	yloop	25, b6b, -25*2*dist64	;; Test loop counter, restore source pointer
	jmp	ypass2_12800_real_unfft

ypass2_12800_real_1:
;;	yr8_8cl_sixteen_reals_eight_complex_fft_final rsi, 2*64, 32*64, dist64, 64
	yr8_dispatch call, y8real_fft_final, 32*64, dist64, 64
	mov	rcx, xsincos_complex	;; Prefetch complex sin/cos data, 1600 cache lines
	yloop_init 400			;; 399 more iterations
	yloop_adjust 1
	yloop_set_prefetch YDOUBLE
b6a:	yr8_8cl_eight_complex_fft_final rsi, 2*64, 32*64, dist64, 64
	yloop	16, b6a, -16*2*64+2*dist64	;; Test loop counter, next source pointer
	yloop	25, b6a			;; Test loop counter
	jmp	ypass2_12800_real_done

ypass2_12800_real_3:
x64	mov	r8, DEST2ARG
	mov	rbp, DIST_TO_MULSRCARG
;;	yr8_8cl_sixteen_reals_eight_complex_with_mult rsi, 2*64, 32*64, dist64, 64
	yr8_dispatch call, y8real_mult, 32*64, dist64, 64
	yloop_init 400			;; 399 more iterations
	yloop_adjust 1
	yloop_set_prefetch YSINGLE
	yr8_8cl_eight_complex_with_mult_preload
b6c:	yr8_8cl_eight_complex_with_mult rsi, 2*64, 32*64, dist64, 64
	yloop	16, b6c, -16*2*64+2*dist64	;; Test loop counter, next source pointer
	yloop	25, b6c, -25*2*dist64	;; Test loop counter, restore source pointer
x64	add	rsi, r8
	jmp	ypass2_12800_real_unfft

ypass2_12800_real_4:
	mov	rbp, DIST_TO_MULSRCARG
;;	yr8_8cl_sixteen_reals_eight_complex_with_mulf rsi, 2*64, 32*64, dist64, 64
	yr8_dispatch call, y8real_mulf, 32*64, dist64, 64
	yloop_init 400			;; 399 more iterations
	yloop_adjust 1
	yloop_set_prefetch YDOUBLE
	yr8_8cl_eight_complex_with_mulf_preload
b6d:	yr8_8cl_eight_complex_with_mulf rsi, 2*64, 32*64, dist64, 64
	yloop	16, b6d, -16*2*64+2*dist64	;; Test loop counter, next source pointer
	yloop	25, b6d, -25*2*dist64	;; Test loop counter, restore source pointer
;;	bump	rcx, (-798+2*320+3*400+399)*64 ;; Move prefetch pointer back to where squaring would be

;; Do inverse FFT levels 9,10
;; On input the 64-byte cache lines hold these data values:
;;	0	+256	+256	+256	8	+256	+256	+256
;;	4	...
;;	16	...
;;	...
;;	1	...
;;	...
;;	1024	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+256	+256	+256	32	+256	+256	+256
;;	4	...
;;	...
;;	28	...
;;	64	...
;;	...
;;	1	...
;;	...
;;	1024	...
;;	...

;; Do 8 eight_reals_unfft operations
;; Do 3192 four_complex_unfft operations
;;	distance between fft data elements is 8
;;	do 800 macros each processing 32 data values

ypass2_12800_real_unfft:
	mov	rdi, sincos3		;; Load sin/cos pointer
	bump	rdi, 32*YMM_SCD2+128*YMM_SCD2+640*YMM_SCD2
	yloop_init 800			;; 100 iterations of 8
	yloop_set_prefetch YHALF
	yr4_4cl_eight_reals_four_complex_djbunfft_preload
c5a:	yr4_4cl_eight_reals_four_complex_djbunfft rsi, 32*64, 2*64, 4*64, rdi, 4*YMM_SCD3, 2
	yloop	2, c5a, -2*32*64+dist64	;; Test loop counter, next source pointer
	yloop	2, c5a, -2*dist64+64	;; Test loop counter, next source pointer
	yloop	2, c5a, -2*64+8*64, rdi, -8*4*YMM_SCD3 ;; Test loop counter, next source pointer, restore sine/cosine pointer

	loops_reset			;; 99 more iters of 8
	yr4_rb4cl_four_complex_djbunfft_preload
c5b:	yr4_rb4cl_four_complex_djbunfft rsi, 32*64, 2*64, 4*64, rdi, 4*YMM_SCD3, 2
	yloop	2, c5b, -2*32*64+dist64	;; Test loop counter, next source pointer
	yloop	2, c5b, -2*dist64+64	;; Test loop counter, next source pointer
	yloop	2, c5b, -2*64+8*64, rdi, -8*4*YMM_SCD3 ;; Test loop counter, next source pointer, restore sine/cosine pointer
	yloop	4, c5b, -4*8*64+2*dist64	;; Test loop counter, next source pointer
	yloop	25, c5b, -25*2*dist64	;; Test loop counter, restore source pointer

;; Do inverse FFT levels 7,8
;; On input the 64-byte cache lines hold these data values:
;;	0	+256	+256	+256	32	+256	+256	+256
;;	4	...
;;	...
;;	28	...
;;	64	...
;;	...
;;	1	...
;;	...
;;	1024	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+256	+256	+256	128	+256	+256	+256
;;	4	...
;;	...
;;	124	...
;;	1	...
;;	...
;;	1024	...
;;	...

;; Do 32 eight_reals_unfft operations
;; Do 3168 four_complex_unfft operations
;;	distance between fft data elements is 32
;;	do 800 macros each processing 32 data values

	bump	rcx, (3200+2*49-2*320-3*400-399-400-800)*64 ;; Move prefetch pointer along
	yloop_init 800			;; 25 iterations of 32
	yloop_set_prefetch YSINGLE
	yr4_4cl_eight_reals_four_complex_djbunfft_preload
c4a:	yr4_4cl_eight_reals_four_complex_djbunfft rsi, 32*64, 8*64, 16*64, rdi, YMM_SCD3, 2
	yloop	2, c4a, -2*32*64+dist64	;; Test loop counter, next source pointer
	yloop	2, c4a, -2*dist64+64	;; Test loop counter, next source pointer
	yloop	8, c4a, -8*64+2*dist64, rdi, -32*YMM_SCD3 ;; Test loop counter, next source pointer, restore sine/cosine pointer

	loops_reset			;; 24 more iters of 32
	yr4_rb4cl_four_complex_djbunfft_preload
c4b:	yr4_rb4cl_four_complex_djbunfft rsi, 32*64, 8*64, 16*64, rdi, YMM_SCD3, 2
	yloop	2, c4b, -2*32*64+dist64	;; Test loop counter, next source pointer
	yloop	2, c4b, -2*dist64+64	;; Test loop counter, next source pointer
	yloop	8, c4b, -8*64+2*dist64, rdi, -32*YMM_SCD3 ;; Test loop counter, next source pointer, restore sine/cosine pointer
	yloop	25, c4b, -25*2*dist64	;; Test loop counter, restore source pointer

;; Do inverse FFT levels 5,6
;; On input the 64-byte cache lines hold these data values:
;;	0	+256	+256	+256	128	+256	+256	+256
;;	4	...
;;	...
;;	124	...
;;	1	...
;;	...
;;	1024	...
;;	...
;; On output the 64-byte cache lines are swizzled to hold these data values:
;;	0	+1	+1	+1	512	+1	+1	+1
;;	4	...
;;	...
;;	508	...
;;	1024	...
;;	...

;; Do 128 eight_reals_unfft operations
;; Do 3072 four_complex_unfft operations
;;	distance between fft data elements is 128
;;	do 800 macros each processing 32 data values

	bump	rdi, -32*YMM_SCD2	;; Real sin/cos ptr
	mov	rbp, xsincos_complex	;; Complex sin/cos data
	bump	rbp, 128*YMM_SCD2+640*YMM_SCD2
	mov	rcx, xsincos_complex	;; Prefetch complex sin/cos data, 1600 cache lines
	yloop_init 800			;; 25 iters of 32
	yloop_set_prefetch YHALF
	yr4_s4cl_2sc_eight_reals_unfft_preload
c3a:	yr4_s4cl_2sc_eight_reals_unfft rsi, 64, 32*64, dist64, rbp, YMM_SCD2, rdi, YMM_SCD2, 32
	yloop	32, c3a, -32*64+2*dist64, rbp, -32*YMM_SCD2 ;; Test loop counter, next source pointer, restore sine/cosine pointer

	loops_reset			;; 24 more iters of 32
	yr4_s4cl_four_complex_djbunfft_preload
c3b:	yr4_s4cl_four_complex_djbunfft rsi, 64, 32*64, dist64, rbp, YMM_SCD2, 32
	yloop	32, c3b, -32*64+2*dist64, rbp, -32*YMM_SCD2 ;; Test loop counter, next source pointer, restore sine/cosine pointer
	yloop	25, c3b, -25*2*dist64	;; Test loop counter, restore source pointer

;; Do inverse FFT levels 3,4
;; On output the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	512	+1	+1	+1
;;	4	...
;;	...
;;	508	...
;;	1024	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	2560	+1	+1	+1
;;	4	...
;;	...
;;	2556	...
;;	5120	...
;;	...

;; Do 512 ten_reals_unfft operations
;; Do 1024 five_complex_unfft operations
;;	distance between fft data elements is 512
;;	do 640 macros each processing 40 data values

	bump	rdi, -32*YMM_SCD2-128*YMM_SCD2 ;; Real sin/cos ptr
	bump	rbp, -128*YMM_SCD2	;; Complex sin/cos data
	yloop_init 640			;; 5 iters of 128
	yloop_set_prefetch YSINGLE
	yr5_5cl_2sc_ten_reals_unfft_preload
c2a:	yr5_5cl_2sc_ten_reals_unfft rsi, 64, 2*dist64, rbp, YMM_SCD2, rdi, YMM_SCD2, 64
	yloop	64, c2a, -64*64+dist64	;; Test loop counter, next source pointer
	yloop	2, c2a, -2*dist64+10*dist64, rbp, -128*YMM_SCD2 ;; Test loop counter, next source pointer, restore sine/cosine pointer

	loops_reset			;; 4 more iters of 128
	yr5_5cl_five_complex_djbunfft_preload
c2b:	yr5_5cl_five_complex_djbunfft rsi, 64, 2*dist64, rbp, YMM_SCD2, 64
	yloop	64, c2b, -64*64+dist64	;; Test loop counter, next source pointer
	yloop	2, c2b, -2*dist64+10*dist64, rbp, -128*YMM_SCD2 ;; Test loop counter, next source pointer, restore sine/cosine pointer
	yloop	5, c2b, -5*10*dist64	;; Test loop counter, restore source pointer

;; Do inverse FFT levels 1,2,3 which correspond to complex levels 1,2
;; On input the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	2560	+1	+1	+1
;;	4	...
;;	...
;;	2556	...
;;	5120	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	12800	+1	+1	+1
;;	4	...
;;	...
;;	12796

;; Do 2560 ten_reals_unfft operations
;;	distance between fft data elements is 2560
;;	do 640 macros each processing 40 data values

	bump	rdi, -128*YMM_SCD2-640*YMM_SCD2	;; Real sin/cos ptr
	bump	rbp, -640*YMM_SCD2	;; Complex sin/cos data
	bump	rcx, (1600-400-640-640)*64 ;; Move prefetch pointer along
	yloop_init 640			;; 640 iterations
	yloop_set_prefetch YSINGLE
	yr5_5cl_2sc_ten_reals_unfft_preload
c1b:	yr5_5cl_2sc_ten_reals_unfft rsi, 64, 10*dist64, rbp, YMM_SCD2, rdi, YMM_SCD2, 64
	yloop	64, c1b, -64*64+dist64	;; Test loop counter, next source pointer
	yloop	10, c1b			;; Test loop counter

ypass2_12800_real_done:
	end_timer 2
	ENDM

; *************** radix4-pass2-12800-complex macro ******************
; This macro takes 12800 complex values and performs the final 13.644 levels of a standard
; radix-4 FFT, squares the results, and does the first 13.644 levels of the inverse FFT.

yr4dwpn_pass2_12800_complex MACRO
	LOCAL	b1a, b1b, b2b, b3a0, b3a, b3b0, b3b, b3c0, b3c, b4a, b4b, b4c, b5a, b5b, b5c, b6a, b6b, b6c, b6d0, b6d
	LOCAL	c1b, c2a, c2b, c3b, c3c, c3d, c4b, c4c, c4d, c5b, c5c, c5d
	LOCAL	ypass2_12800_complex_1, ypass2_12800_complex_2
	LOCAL	ypass2_12800_complex_3, ypass2_12800_complex_4
	LOCAL	ypass2_12800_complex_unfft, ypass2_12800_complex_done

;; Type 4 FFTs skip the forward FFT process

	mov	rsi, DATA_ADDR		;; Load address of FFT data
	mov	rbx, DIST_TO_FFTSRCARG
	mov	rcx, DATA_PREFETCH	;; Prefetch pointer (we need to prefetch 
					;; 3200 cache lines + 2*49 pad cache lines)
	cmp	ffttype, 4
	je	ypass2_12800_complex_4

;; Do FFT level 1,2 (actually 2.322 levels)
;; On input the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	12800	+1	+1	+1
;;	4	...
;;	...
;;	12796
;; On output the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	2560	+1	+1	+1
;;	4	...
;;	...
;;	2556	...
;;	5120	...
;;	...

;; Do 2560 five_complex_fft operations
;;	distance between fft data elements is 2560
;;	do 640 macros each processing 40 data values

;; Do FFT levels 3,4 (actually 2.322 levels)
;; On input the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	2560	+1	+1	+1
;;	4	...
;;	...
;;	2556	...
;;	5120	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	512	+1	+1	+1
;;	4	...
;;	...
;;	508	...
;;	1024	...
;;	...

;; Do 2560 five_complex_fft operations
;;	distance between fft data elements is 512
;;	do 640 macros each processing 40 data values

	mov	rdi, xsincos_complex	;; The sin/cos ptrs
	lea	rbp, [rdi+640*YMM_SCD2]
	yloop_init 1024			;; 640 iterations

	yr5_f5cl_five_complex_djbfft_preload
b1a:	start_timer 4
	yloop_adjust 3			;; Three fewer iterations this loop
	yloop_set_prefetch YOTHER, 8	;; Prefetch 128 cache lines
b1b:	yr5_f5cl_five_complex_djbfft rsi, 2*dist64, 10*dist64, rdi, 128*YMM_SCD2, 5
	yloop	8, b1b, -5*2*dist64, rdi, -5*128*YMM_SCD2+YMM_SCD2 ;; 5 iterations!!  Restore source pointer, next sin/cos data
	loops_undo 8			;; Back up loop counter
	end_timer 4

	start_timer 5
	yloop_adjust 3			;; Three fewer iterations this loop
	yloop_set_prefetch YOTHER, 8	;; Prefetch 128 cache lines
;;	yr5_5cl_five_complex_djbfft_preload
b2b:	yr5_5cl_five_complex_djbfft rsi, 10*dist64, 2*dist64, rbp, 0, 5, L1PREFETCH_ALL, 64
	yloop	8, b2b, -5*10*dist64+64, rbp, YMM_SCD2 ;; 5 iterations!!  Next source pointer, next sin/cos data
	end_timer 5

	yloop	64, b1a, -64*64+dist64	;; Test loop counter, next source pointer
	yloop	2, b1a, -2*dist64, rdi, -128*YMM_SCD2+640*YMM_SCD2+128*YMM_SCD2 ;; Test loop counter, restore source pointer, next sin/cos ptr

;; Do FFT levels 5,6
;; On input the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	512	+1	+1	+1
;;	4	...
;;	...
;;	508	...
;;	1024	...
;;	...
;; On output the 64-byte cache lines are swizzled to hold these data values:
;;	0	+256	+256	+256	128	+256	+256	+256
;;	4	...
;;	...
;;	124	...
;;	1	...
;;	...
;;	1024	...
;;	...

;; Do 3200 four_complex_fft operations
;;	distance between fft data elements is 128
;;	do 800 macros each processing 32 data values

;; Do FFT levels 7,8
;; On input the 64-byte cache lines hold these data values:
;;	0	+256	+256	+256	128	+256	+256	+256
;;	4	...
;;	...
;;	124	...
;;	1	...
;;	...
;;	1024	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+256	+256	+256	32	+256	+256	+256
;;	4	...
;;	...
;;	28	...
;;	64	...
;;	...
;;	1	...
;;	...
;;	1024	...
;;	...

;; Do 3200 four_complex_fft operations
;;	distance between fft data elements is 32
;;	do 800 macros each processing 32 data values

;; Do FFT levels 9,10
;; On input the 64-byte cache lines hold these data values:
;;	0	+256	+256	+256	32	+256	+256	+256
;;	4	...
;;	...
;;	28	...
;;	64	...
;;	...
;;	1	...
;;	...
;;	1024	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+256	+256	+256	8	+256	+256	+256
;;	4	...
;;	16	...
;;	...
;;	1	...
;;	...
;;	1024	...
;;	...

;; Do 3200 four_complex_fft operations
;;	distance between fft data elements is 8
;;	do 800 macros each processing 32 data values

;; Do FFT levels 11,12,13 as well as inverse FFT levels 11,12,13
;; On input and output the 64-byte cache lines hold these data values:
;;	0	+256	+256	+256	8	+256	+256	+256
;;	4	...
;;	16	...
;;	...
;;	1	...
;;	...
;;	1024	...
;;	...

;; Do 1600 eight_complex_fft operations
;;	distance between fft data elements is 1
;;	do 400 macros each processing 64 data values

;; Do inverse FFT levels 9,10
;; On input the 64-byte cache lines hold these data values:
;;	0	+256	+256	+256	8	+256	+256	+256
;;	4	...
;;	16	...
;;	...
;;	1	...
;;	...
;;	1024	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+256	+256	+256	32	+256	+256	+256
;;	4	...
;;	...
;;	28	...
;;	64	...
;;	...
;;	1	...
;;	...
;;	1024	...
;;	...

;; Do 3200 four_complex_unfft operations
;;	distance between fft data elements is 8
;;	do 800 macros each processing 32 data values

;; Do inverse FFT levels 7,8
;; On input the 64-byte cache lines hold these data values:
;;	0	+256	+256	+256	32	+256	+256	+256
;;	4	...
;;	...
;;	28	...
;;	64	...
;;	...
;;	1	...
;;	...
;;	1024	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+256	+256	+256	128	+256	+256	+256
;;	4	...
;;	...
;;	124	...
;;	1	...
;;	...
;;	1024	...
;;	...

;; Do 3200 four_complex_unfft operations
;;	distance between fft data elements is 32
;;	do 800 macros each processing 32 data values

;; Do inverse FFT levels 5,6
;; On input the 64-byte cache lines hold these data values:
;;	0	+256	+256	+256	128	+256	+256	+256
;;	4	...
;;	...
;;	124	...
;;	1	...
;;	...
;;	1024	...
;;	...
;; On output the 64-byte cache lines are swizzled to hold these data values:
;;	0	+1	+1	+1	512	+1	+1	+1
;;	4	...
;;	...
;;	508	...
;;	1024	...
;;	...

;; Do 3200 four_complex_unfft operations
;;	distance between fft data elements is 128
;;	do 800 macros each processing 32 data values

;; Do this 128 cache lines (8KB) at a time to maximize accessing data
;; while it is in the L1 cache.

;; Execute the right middle step

	cmp	ffttype, 2
	jl	ypass2_12800_complex_1
	jg	ypass2_12800_complex_3

ypass2_12800_complex_2:
	yloop_init 800			;; 25 iters of 128 cache lines

b3b0:	start_timer 6
	yloop_set_prefetch YHALF	;; Prefetch 400 cache lines
	yr4_s4cl_four_complex_djbfft_preload
b3b:	yr4_s4cl_four_complex_djbfft rsi, 64, 32*64, dist64, rdi, YMM_SCD2, 32
	yloop	32, b3b, -32*64, rdi, -32*YMM_SCD2 ;; Loop 32 times, restore source pointer, restore sine/cosine pointer
	loops_undo 32			;; Back up loop counter
	end_timer 6

	start_timer 7
	yloop_set_prefetch YHALF	;; Prefetch 400 cache lines
	yr4_eb4cl_four_complex_djbfft_preload
b4b:	yr4_eb4cl_four_complex_djbfft rsi, 64, 8*64, 16*64, rdi, 4*YMM_SCD2, 8
	yloop	8, b4b, -8*64+32*64, rdi, -8*4*YMM_SCD2+YMM_SCD2 ;; Loop 8 times, next source pointer, next sine/cosine pointer
	yloop	2, b4b, -2*32*64+dist64	;; Loop 2 times, next source pointer
	yloop	2, b4b, -2*dist64, rdi, -4*YMM_SCD2 ;; Loop 2 times, restore source pointer, restore sine/cosine pointer
	loops_undo 4*8			;; Back up loop counter
	end_timer 7

	start_timer 8
	yloop_set_prefetch YHALF	;; Prefetch 400 cache lines
;;	yr4_eb4cl_four_complex_djbfft_preload
b5b:	yr4_eb4cl_four_complex_djbfft rsi, 64, 2*64, 4*64, rdi, 16*YMM_SCD2, 2
	yloop	2, b5b, -2*64+8*64, rdi, -2*16*YMM_SCD2 ;; Loop 2 times, next source pointer, restore sine/cosine pointer
	yloop	4, b5b, -4*8*64+32*64, rdi, 4*YMM_SCD2 ;; Loop 4 times, next source pointer, next sine/cosine pointer
	yloop	2, b5b, -2*32*64+dist64	;; Test loop counter, next source pointer
	yloop	2, b5b, -2*dist64, rdi, -4*4*YMM_SCD2 ;; Test loop counter, restore source pointer, restore sine/cosine pointer
	loops_undo 4*4*2		;; Back up loop counter
	end_timer 8

	start_timer 9
	yloop_adjust 2*8		;; 16 fewer iterations this loop
	yloop_set_prefetch YSINGLE	;; Prefetch 400 cache lines
	yr8_8cl_eight_complex_with_square_preload
;; Loop below reworked to avoid 4KB distances
;;b6b:	yr8_8cl_eight_complex_with_square rsi, 2*64, 32*64, dist64, 64, 32
;;	yloop	32, b6b, -16*2*64	;; 16 iterations!!  Restore source pointer
b6b:	yr8_8cl_eight_complex_with_square rsi, 4*64, 32*64, dist64, 64, 8
	yloop	8, b6b, -8*4*64+2*64	;; Loop 8 times, next source pointer
	yloop	4, b6b, -2*2*64		;; 2 iterations!!  Restore source pointer
	loops_undo 4*8			;; Back up loop counter
	end_timer 9

	start_timer 10
	yloop_set_prefetch YHALF	;; Prefetch 400 cache lines
	yr4_eb4cl_four_complex_djbunfft_preload
c5b:	yr4_eb4cl_four_complex_djbunfft rsi, 64, 2*64, 4*64, rdi, 16*YMM_SCD2, 2
	yloop	2, c5b, -2*64+8*64, rdi, -2*16*YMM_SCD2 ;; Loop 2 times, next source pointer, restore sine/cosine pointer
	yloop	4, c5b, -4*8*64+32*64, rdi, 4*YMM_SCD2 ;; Loop 4 times, next source pointer, next sine/cosine pointer
	yloop	2, c5b, -2*32*64+dist64	;; Test loop counter, next source pointer
	yloop	2, c5b, -2*dist64, rdi, -4*4*YMM_SCD2 ;; Test loop counter, restore source pointer, restore sine/cosine pointer
	loops_undo 4*4*2		;; Back up loop counter
	end_timer 10

	start_timer 11
	yloop_set_prefetch YHALF	;; Prefetch 400 cache lines
;;	yr4_eb4cl_four_complex_djbunfft_preload
c4b:	yr4_eb4cl_four_complex_djbunfft rsi, 64, 8*64, 16*64, rdi, 4*YMM_SCD2, 8
	yloop	8, c4b, -8*64+32*64, rdi, -8*4*YMM_SCD2+YMM_SCD2 ;; Loop 8 times, next source pointer, next sine/cosine pointer
	yloop	2, c4b, -2*32*64+dist64	;; Loop 2 times, next source pointer
	yloop	2, c4b, -2*dist64, rdi, -4*YMM_SCD2 ;; Loop 2 times, restore source pointer, restore sine/cosine pointer
	loops_undo 4*8			;; Back up loop counter
	end_timer 11

	start_timer 12
	yloop_set_prefetch YHALF	;; Prefetch 400 cache lines
	yr4_s4cl_four_complex_djbunfft_preload
c3b:	yr4_s4cl_four_complex_djbunfft rsi, 64, 32*64, dist64, rdi, YMM_SCD2, 32, L1PREFETCH_ALL, 2*dist64
	yloop	32, c3b, -32*64+2*dist64, rdi, -32*YMM_SCD2 ;; Loop 32 times, next source pointer, restore sine/cosine pointer
	end_timer 12

	yloop	25, b3b0, -25*2*dist64	;; Loop 25 times, restore source pointer
	jmp	ypass2_12800_complex_unfft


ypass2_12800_complex_1:
	start_timer 6
	bump	rcx, (3200+2*49-2*128-4*800)*64 ;; Move prefetch pointer along
	yloop_init 800			;; 25 iters of 128 cache lines

b3a0:	yloop_set_prefetch YSINGLE	;; Prefetch 800 cache lines in this loop
	yr4_s4cl_four_complex_djbfft_preload
b3a:	yr4_s4cl_four_complex_djbfft rsi, 64, 32*64, dist64, rdi, YMM_SCD2, 32
	yloop	32, b3a, -32*64, rdi, -32*YMM_SCD2 ;; Loop 32 times, restore source pointer, restore sine/cosine pointer
	loops_undo 32			;; Back up loop counter

	yloop_set_prefetch YSINGLE	;; Prefetch 800 cache lines in this loop
	yr4_eb4cl_four_complex_djbfft_preload
b4a:	yr4_eb4cl_four_complex_djbfft rsi, 64, 8*64, 16*64, rdi, 4*YMM_SCD2, 8
	yloop	8, b4a, -8*64+32*64, rdi, -8*4*YMM_SCD2+YMM_SCD2 ;; Loop 8 times, next source pointer, next sine/cosine pointer
	yloop	2, b4a, -2*32*64+dist64	;; Loop 2 times, next source pointer
	yloop	2, b4a, -2*dist64, rdi, -4*YMM_SCD2 ;; Loop 2 times, restore source pointer, restore sine/cosine pointer
	loops_undo 4*8			;; Back up loop counter

	yloop_set_prefetch YSINGLE	;; Prefetch 800 cache lines in this loop
;;	yr4_eb4cl_four_complex_djbfft_preload
b5a:	yr4_eb4cl_four_complex_djbfft rsi, 64, 2*64, 4*64, rdi, 16*YMM_SCD2, 2, L1PREFETCH_ALL, 2*dist64
	yloop	2, b5a, -2*64+8*64, rdi, -2*16*YMM_SCD2 ;; Loop 2 times, next source pointer, restore sine/cosine pointer
	yloop	4, b5a, -4*8*64+32*64, rdi, 4*YMM_SCD2 ;; Loop 4 times, next source pointer, next sine/cosine pointer
	yloop	2, b5a, -2*32*64+dist64	;; Test loop counter, next source pointer
	yloop	2, b5a, -2*dist64, rdi, -4*4*YMM_SCD2 ;; Test loop counter, restore source pointer, restore sine/cosine pointer
	loops_undo 4*4*2		;; Back up loop counter

	yloop_adjust 2*8		;; 16 fewer iterations this loop
	yloop_set_prefetch YDOUBLE	;; Prefetch 800 cache lines in this loop
	yr8_8cl_eight_complex_fft_final_preload
;; Loop below reworked to avoid 4KB distances
;;b6a:	yr8_8cl_eight_complex_fft_final rsi, 2*64, 32*64, dist64, 64
;;	yloop	32, b6a, -16*2*64	;; 16 iterations!!  Restore source pointer
b6a:	yr8_8cl_eight_complex_fft_final rsi, 4*64, 32*64, dist64, 64
	yloop	8, b6a, -8*4*64+2*64	;; Loop 8 times, next source pointer
	yloop	4, b6a, -2*2*64+2*dist64 ;; 2 iterations!!  Next source pointer

	yloop	25, b3a0		;; Loop 25 times
	end_timer 6
	jmp	ypass2_12800_complex_done


ypass2_12800_complex_3:
	start_timer 6
x64	mov	r8, DEST2ARG
	mov	rbp, DIST_TO_MULSRCARG
	yloop_init 800			;; 25 iters of 128 cache lines

b3c0:	yloop_set_prefetch YHALF	;; Prefetch 400 cache lines
	yr4_s4cl_four_complex_djbfft_preload
b3c:	yr4_s4cl_four_complex_djbfft rsi, 64, 32*64, dist64, rdi, YMM_SCD2, 32
	yloop	32, b3c, -32*64, rdi, -32*YMM_SCD2 ;; Loop 32 times, restore source pointer, restore sine/cosine pointer
	loops_undo 32			;; Back up loop counter

	yloop_set_prefetch YHALF	;; Prefetch 400 cache lines
	yr4_eb4cl_four_complex_djbfft_preload
b4c:	yr4_eb4cl_four_complex_djbfft rsi, 64, 8*64, 16*64, rdi, 4*YMM_SCD2, 8
	yloop	8, b4c, -8*64+32*64, rdi, -8*4*YMM_SCD2+YMM_SCD2 ;; Loop 8 times, next source pointer, next sine/cosine pointer
	yloop	2, b4c, -2*32*64+dist64	;; Loop 2 times, next source pointer
	yloop	2, b4c, -2*dist64, rdi, -4*YMM_SCD2 ;; Loop 2 times, restore source pointer, restore sine/cosine pointer
	loops_undo 4*8			;; Back up loop counter

	yloop_set_prefetch YHALF	;; Prefetch 400 cache lines
;;	yr4_eb4cl_four_complex_djbfft_preload
b5c:	yr4_eb4cl_four_complex_djbfft rsi, 64, 2*64, 4*64, rdi, 16*YMM_SCD2, 2
	yloop	2, b5c, -2*64+8*64, rdi, -2*16*YMM_SCD2 ;; Loop 2 times, next source pointer, restore sine/cosine pointer
	yloop	4, b5c, -4*8*64+32*64, rdi, 4*YMM_SCD2 ;; Loop 4 times, next source pointer, next sine/cosine pointer
	yloop	2, b5c, -2*32*64+dist64	;; Test loop counter, next source pointer
	yloop	2, b5c, -2*dist64, rdi, -4*4*YMM_SCD2 ;; Test loop counter, restore source pointer, restore sine/cosine pointer
	loops_undo 4*4*2		;; Back up loop counter

	yloop_adjust 2*8		;; 16 fewer iterations this loop
	yloop_set_prefetch YSINGLE	;; Prefetch 400 cache lines
	yr8_8cl_eight_complex_with_mult_preload
;; Loop below reworked to avoid 4KB distances
;;b6c:	yr8_8cl_eight_complex_with_mult rsi, 2*64, 32*64, dist64, 64
;;	yloop	32, b6c, -16*2*64	;; 16 iterations!!  Restore source pointer
b6c:	yr8_8cl_eight_complex_with_mult rsi, 4*64, 32*64, dist64, 64
	yloop	8, b6c, -8*4*64+2*64	;; Loop 8 times, next source pointer
	yloop	4, b6c, -2*2*64		;; 2 iterations!!  Restore source pointer
x64	add	rsi, r8
	loops_undo 4*8			;; Back up loop counter

	yloop_set_prefetch YHALF	;; Prefetch 400 cache lines
	yr4_eb4cl_four_complex_djbunfft_preload
c5c:	yr4_eb4cl_four_complex_djbunfft rsi, 64, 2*64, 4*64, rdi, 16*YMM_SCD2, 2
	yloop	2, c5c, -2*64+8*64, rdi, -2*16*YMM_SCD2 ;; Loop 2 times, next source pointer, restore sine/cosine pointer
	yloop	4, c5c, -4*8*64+32*64, rdi, 4*YMM_SCD2 ;; Loop 4 times, next source pointer, next sine/cosine pointer
	yloop	2, c5c, -2*32*64+dist64	;; Test loop counter, next source pointer
	yloop	2, c5c, -2*dist64, rdi, -4*4*YMM_SCD2 ;; Test loop counter, restore source pointer, restore sine/cosine pointer
	loops_undo 4*4*2		;; Back up loop counter

	yloop_set_prefetch YHALF	;; Prefetch 400 cache lines
;;	yr4_eb4cl_four_complex_djbunfft_preload
c4c:	yr4_eb4cl_four_complex_djbunfft rsi, 64, 8*64, 16*64, rdi, 4*YMM_SCD2, 8
	yloop	8, c4c, -8*64+32*64, rdi, -8*4*YMM_SCD2+YMM_SCD2 ;; Loop 8 times, next source pointer, next sine/cosine pointer
	yloop	2, c4c, -2*32*64+dist64	;; Loop 2 times, next source pointer
	yloop	2, c4c, -2*dist64, rdi, -4*YMM_SCD2 ;; Loop 2 times, restore source pointer, restore sine/cosine pointer
	loops_undo 4*8			;; Back up loop counter

	yloop_set_prefetch YHALF	;; Prefetch 400 cache lines
	yr4_s4cl_four_complex_djbunfft_preload
c3c:	yr4_s4cl_four_complex_djbunfft rsi, 64, 32*64, dist64, rdi, YMM_SCD2, 32, L1PREFETCH_ALL, 2*dist64
	yloop	32, c3c, -32*64+2*dist64, rdi, -32*YMM_SCD2 ;; Loop 32 times, next source pointer, restore sine/cosine pointer

x64	sub	rsi, r8
	yloop	25, b3c0, -25*2*dist64	;; Loop 25 times, restore source pointer
x64	add	rsi, r8
	end_timer 6
	jmp	ypass2_12800_complex_unfft


ypass2_12800_complex_4:
	start_timer 6
	mov	rdi, xsincos_complex	;; The sin/cos ptrs
	bump	rdi, 640*YMM_SCD2+128*YMM_SCD2
	mov	rbp, DIST_TO_MULSRCARG
	yloop_init 800			;; 25 iters of 128 cache lines

b6d0:	yloop_adjust 2*8		;; 16 fewer iterations this loop
	yloop_set_prefetch YDOUBLE	;; Prefetch 800 cache lines in this loop
	yr8_8cl_eight_complex_with_mulf_preload
;; Loop below reworked to avoid 4KB distances
;;b6d:	yr8_8cl_eight_complex_with_mulf rsi, 2*64, 32*64, dist64, 64
;;	yloop	32, b6d, -16*2*64	;; 16 iterations!!  Restore source pointer
b6d:	yr8_8cl_eight_complex_with_mulf rsi, 4*64, 32*64, dist64, 64
	yloop	8, b6d, -8*4*64+2*64	;; Loop 8 times, next source pointer
	yloop	4, b6d, -2*2*64		;; 2 iterations!!  Restore source pointer
	loops_undo 4*8			;; Back up loop counter

	yloop_set_prefetch YSINGLE	;; Prefetch 800 cache lines in this loop
	yr4_eb4cl_four_complex_djbunfft_preload
c5d:	yr4_eb4cl_four_complex_djbunfft rsi, 64, 2*64, 4*64, rdi, 16*YMM_SCD2, 2
	yloop	2, c5d, -2*64+8*64, rdi, -2*16*YMM_SCD2 ;; Loop 2 times, next source pointer, restore sine/cosine pointer
	yloop	4, c5d, -4*8*64+32*64, rdi, 4*YMM_SCD2 ;; Loop 4 times, next source pointer, next sine/cosine pointer
	yloop	2, c5d, -2*32*64+dist64	;; Test loop counter, next source pointer
	yloop	2, c5d, -2*dist64, rdi, -4*4*YMM_SCD2 ;; Test loop counter, restore source pointer, restore sine/cosine pointer
	loops_undo 4*4*2		;; Back up loop counter

	yloop_set_prefetch YSINGLE	;; Prefetch 800 cache lines in this loop
;;	yr4_eb4cl_four_complex_djbunfft_preload
c4d:	yr4_eb4cl_four_complex_djbunfft rsi, 64, 8*64, 16*64, rdi, 4*YMM_SCD2, 8
	yloop	8, c4d, -8*64+32*64, rdi, -8*4*YMM_SCD2+YMM_SCD2 ;; Loop 8 times, next source pointer, next sine/cosine pointer
	yloop	2, c4d, -2*32*64+dist64	;; Loop 2 times, next source pointer
	yloop	2, c4d, -2*dist64, rdi, -4*YMM_SCD2 ;; Loop 2 times, restore source pointer, restore sine/cosine pointer
	loops_undo 4*8			;; Back up loop counter

	yloop_set_prefetch YSINGLE	;; Prefetch 800 cache lines in this loop
	yr4_s4cl_four_complex_djbunfft_preload
c3d:	yr4_s4cl_four_complex_djbunfft rsi, 64, 32*64, dist64, rdi, YMM_SCD2, 32, L1PREFETCH_ALL, 2*dist64
	yloop	32, c3d, -32*64+2*dist64, rdi, -32*YMM_SCD2 ;; Loop 32 times, next source pointer, restore sine/cosine pointer

	yloop	25, b6d0, -25*2*dist64	;; Loop 25 times, restore source pointer
	bump	rcx, (-4*800+2*128+7*400)*64 ;; Move prefetch pointer to where squaring would be
	end_timer 6

;; Do inverse FFT levels 3,4
;; On output the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	512	+1	+1	+1
;;	4	...
;;	...
;;	508	...
;;	1024	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	2560	+1	+1	+1
;;	4	...
;;	...
;;	2556	...
;;	5120	...
;;	...

;; Do 2560 five_complex_unfft operations
;;	distance between fft data elements is 512
;;	do 640 macros each processing 40 data values

;; Do inverse FFT levels 1,2
;; On input the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	2560	+1	+1	+1
;;	4	...
;;	...
;;	2556	...
;;	5120	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	12800	+1	+1	+1
;;	4	...
;;	...
;;	12796

;; Do 2560 three_complex_unfft operations
;;	distance between fft data elements is 2560
;;	do 640 macros each processing 40 data values

ypass2_12800_complex_unfft:
	mov	rdi, xsincos_complex	;; Load sin/cos pointers
	lea	rbp, [rdi+640*YMM_SCD2]
	bump	rcx, (3200+2*49-2*128-7*400-2*128)*64 ;; Move prefetch pointer along
	yloop_init 1024			;; 640 iterations

	yr5_5cl_five_complex_djbunfft_preload
c2a:	start_timer 13
	yloop_adjust 3			;; Three fewer iterations this loop
	yloop_set_prefetch YOTHER, 8	;; Prefetch 128 cache lines
c2b:	yr5_5cl_five_complex_djbunfft rsi, 10*dist64, 2*dist64, rbp, 0, 5
	yloop	8, c2b, -5*10*dist64, rbp, YMM_SCD2 ;; 5 iterations!!  Restore source pointer, next sine/cosine pointer
	loops_undo 8			;; Restore loop counter
	end_timer 13

	start_timer 14
	yloop_adjust 3			;; Three fewer iterations this loop
	yloop_set_prefetch YOTHER, 8	;; Prefetch 128 cache lines
;;	yr5_5cl_five_complex_djbunfft_preload
c1b:	yr5_5cl_five_complex_djbunfft rsi, 2*dist64, 10*dist64, rdi, 128*YMM_SCD2, 5, L1PREFETCH_ALL, 64
	yloop	8, c1b, -5*2*dist64+64, rdi, -5*128*YMM_SCD2+YMM_SCD2 ;; 5 iterations!!  Next source pointer, next sine/cosine pointer
	end_timer 14

	yloop	64, c2a, -64*64+dist64	;; Loop 64 times, next source pointer
	yloop	2, c2a			;; Loop 2 times

ypass2_12800_complex_done:
	ENDM


;*********************************************************************************************
;			Macros derived from pass2-14-levels
;*********************************************************************************************


; *************** radix4-pass2-9216-real macro ******************
; This macro takes 18432 real values and performs the final 14.17 levels of the FFT.
; The results are squared, and the first 14.17 levels of the inverse FFT are done.

yr4dwpn_pass2_9216_real MACRO
	LOCAL	b1b, b2a, b2b, b3a, b3b, b4a, b4b, b5a, b5b, b6a, b6b, b7a, b7b, b7c, b7d
	LOCAL	c1b, c2a, c2b, c3a, c3b, c4a, c4b, c5a, c5b, c6a, c6b
	LOCAL	ypass2_9216_real_1, ypass2_9216_real_2
	LOCAL	ypass2_9216_real_3, ypass2_9216_real_4
	LOCAL	ypass2_9216_real_unfft, ypass2_9216_real_done

;; Type 4 FFTs skip the forward FFT process

	start_timer 2
	mov	rsi, DATA_ADDR		;; Load source address
	mov	rbx, DIST_TO_FFTSRCARG
	mov	rcx, DATA_PREFETCH	;; Prefetch pointer (we need to prefetch 
					;; 2304 cache lines + 35 pad cache lines)
	cmp	ffttype, 4
	je	ypass2_9216_real_4

;; Do FFT levels 1,2,3 which correspond to complex levels 1,2
;; On input the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	9216	+1	+1	+1
;;	4	...
;;	...
;;	9212
;; On output the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	3072	+1	+1	+1
;;	4	...
;;	...
;;	3068	...
;;	6144	...
;;	...

;; Do 3072 six_reals_fft operations
;;	distance between fft data elements is 3072
;;	do 768 macros each processing 24 data values

	mov	rdi, sincos3		;; Real sin/cos pointer
	mov	rbp, xsincos_complex	;; Complex sin/cos data
	yloop_init 768			;; 768 iterations
	yloop_set_prefetch YHALF
	yr3_f3cl_2sc_six_reals_fft_preload
b1b:	yr3_f3cl_2sc_six_reals_fft rsi, 64, 12*dist64, rbp, YMM_SCD1, rdi, YMM_SCD1, 64
	yloop	64, b1b, -64*64+dist64	;; Test loop counter, next source pointer
	yloop	12, b1b, -12*dist64	;; Test loop counter, restore source pointer

;; Do FFT levels 3,4
;; On input the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	3072	+1	+1	+1
;;	4	...
;;	...
;;	3068	...
;;	6144	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	1024	+1	+1	+1
;;	4	...
;;	...
;;	1020	...
;;	2048	...
;;	...

;; Do 1024 six_reals_fft operations
;; Do 2048 three_complex_fft operations
;;	distance between fft data elements is 1024
;;	do 768 macros each processing 24 data values

	yloop_init 768			;; 3 iters of 256
	yloop_set_prefetch YHALF
	yr3_3cl_2sc_six_reals_fft_preload
b2a:	yr3_3cl_2sc_six_reals_fft rsi, 64, 4*dist64, rbp, YMM_SCD1, rdi, YMM_SCD1, 64
	yloop	64, b2a, -64*64+dist64	;; Test loop counter, next source pointer
	yloop	4, b2a, -4*dist64+12*dist64, rbp, -256*YMM_SCD1 ;; Test loop counter, next source pointer, restore sine/cosine pointer

	loops_reset			;; 2 more iters of 256
	yr3_3cl_three_complex_djbfft_preload
b2b:	yr3_3cl_three_complex_djbfft rsi, 64, 4*dist64, rbp, YMM_SCD1, 64
	yloop	64, b2b, -64*64+dist64	;; Test loop counter, next source pointer
	yloop	4, b2b, -4*dist64+12*dist64, rbp, -256*YMM_SCD1 ;; Test loop counter, next source pointer, restore sin/cos ptr
	yloop	3, b2b, -3*12*dist64, rbp, 256*YMM_SCD1 ;; Loop 3 times, restore source pointer, next sin/cos pointer

;; Do FFT levels 5,6
;; On input the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	1024	+1	+1	+1
;;	4	...
;;	...
;;	1020	...
;;	2048	...
;;	...
;; On output the 64-byte cache lines are swizzled to hold these data values:
;;	0	+512	+512	+512	256	+512	+512	+512
;;	4	...
;;	...
;;	508	...
;;	1	...
;;	...
;;	2048	...
;;	...

;; Do 256 eight_reals_fft operations
;; Do 2048 four_complex_fft operations
;;	distance between fft data elements is 256
;;	do 576 macros each processing 32 data values

	yloop_init 576			;; 9 iterations of 64
	yloop_set_prefetch YHALF
	yr4_s4cl_2sc_eight_reals_fft_preload
b3a:	yr4_s4cl_2sc_eight_reals_fft rsi, 64, dist64, 2*dist64, rbp, YMM_SCD2, rdi, YMM_SCD2, 64
	yloop	64, b3a, -64*64+4*dist64, rbp, -64*YMM_SCD2 ;; Test loop counter, next source pointer, restore sine/cosine pointer

	loops_reset			;; 8 more iters of 64
	yr4_s4cl_four_complex_djbfft_preload
b3b:	yr4_s4cl_four_complex_djbfft rsi, 64, dist64, 2*dist64, rbp, YMM_SCD2, 64
	yloop	64, b3b, -64*64+4*dist64, rbp, -64*YMM_SCD2 ;; Test loop counter, next source pointer, restore sin/cos ptr
	yloop	9, b3b, -9*4*dist64	;; Loop 9 times, restore source pointer

;; Do FFT levels 7,8
;; On input the 64-byte cache lines hold these data values:
;;	0	+512	+512	+512	256	+512	+512	+512
;;	4	...
;;	...
;;	508	...
;;	1	...
;;	...
;;	2048	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+512	+512	+512	64	+512	+512	+512
;;	4	...
;;	...
;;	60	...
;;	128	...
;;	...
;;	1	...
;;	...
;;	2048	...
;;	...

;; Do 64 eight_reals_fft operations
;; Do 2240 four_complex_fft operations
;;	distance between fft data elements is 64
;;	do 576 macros each processing 32 data values

	yloop_init 576			;; 9 iterations of 64
	yloop_set_prefetch YHALF
	yr4_4cl_eight_reals_four_complex_djbfft_preload
b4a:	yr4_4cl_eight_reals_four_complex_djbfft rsi, dist64, 16*64, 32*64, rdi, YMM_SCD3, 4
	yloop	4, b4a, -4*dist64+64	;; Test loop counter, next source pointer
	yloop	16, b4a, -16*64+4*dist64, rdi, -64*YMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer

	loops_reset			;; 8 more iters of 64
	yr4_rb4cl_four_complex_djbfft_preload
b4b:	yr4_rb4cl_four_complex_djbfft rsi, dist64, 16*64, 32*64, rdi, YMM_SCD3, 4
	yloop	4, b4b, -4*dist64+64	;; Test loop counter, next source pointer
	yloop	16, b4b, -16*64+4*dist64, rdi, -64*YMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer
	yloop	9, b4b, -9*4*dist64	;; Test loop counter, restore source pointer

;; Do FFT levels 9,10
;; On input the 64-byte cache lines hold these data values:
;;	0	+512	+512	+512	64	+512	+512	+512
;;	4	...
;;	...
;;	60	...
;;	128	...
;;	...
;;	1	...
;;	...
;;	2048	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+512	+512	+512	16	+512	+512	+512
;;	4	...
;;	...
;;	12	...
;;	32	...
;;	...
;;	1	...
;;	...
;;	2048	...
;;	...

;; Do 16 eight_reals_fft operations
;; Do 2288 four_complex_fft operations
;;	distance between fft data elements is 16
;;	do 576 macros each processing 32 data values

	yloop_init 576			;; 36 iterations of 16
	yloop_set_prefetch YQUARTER
	yr4_4cl_eight_reals_four_complex_djbfft_preload
b5a:	yr4_4cl_eight_reals_four_complex_djbfft rsi, dist64, 4*64, 8*64, rdi, 4*YMM_SCD3, 4
	yloop	4, b5a, -4*dist64+64	;; Test loop counter, next source pointer
	yloop	4, b5a, -4*64+16*64, rdi, -16*4*YMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer

	loops_reset			;; 35 more iters of 16
	yr4_rb4cl_four_complex_djbfft_preload
b5b:	yr4_rb4cl_four_complex_djbfft rsi, dist64, 4*64, 8*64, rdi, 4*YMM_SCD3, 4
	yloop	4, b5b, -4*dist64+64	;; Test loop counter, next source pointer
	yloop	4, b5b, -4*64+16*64, rdi, -16*4*YMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer
	yloop	4, b5b, -4*16*64+4*dist64 ;; Test loop counter, next source pointer
	yloop	9, b5b, -9*4*dist64	;; Test loop counter, restore source pointer

;; Do FFT levels 11,12
;; On input the 64-byte cache lines hold these data values:
;;	0	+512	+512	+512	16	+512	+512	+512
;;	4	...
;;	...
;;	12	...
;;	32	...
;;	...
;;	1	...
;;	...
;;	2048	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+512	+512	+512	4	+512	+512	+512
;;	8	...
;;	...
;;	1	...
;;	...
;;	2048	...
;;	...

;; Do 4 eight_reals_fft operations
;; Do 2300 four_complex_fft operations
;;	distance between fft data elements is 4
;;	do 576 macros each processing 32 data values

	yloop_init 576			;; 144 iterations of 4
	yloop_set_prefetch YQUARTER
	yr4_4cl_eight_reals_four_complex_djbfft_preload
b6a:	yr4_4cl_eight_reals_four_complex_djbfft rsi, dist64, 64, 2*64, rdi, 16*YMM_SCD3, 4
	yloop	4, b6a, -4*dist64+4*64, rdi, -4*16*YMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer

	loops_reset			;; 143 more iters of 4
	yr4_rb4cl_four_complex_djbfft_preload
b6b:	yr4_rb4cl_four_complex_djbfft rsi, dist64, 64, 2*64, rdi, 16*YMM_SCD3, 4
	yloop	4, b6b, -4*dist64+4*64, rdi, -4*16*YMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer
	yloop	16, b6b, -16*4*64+4*dist64 ;; Test loop counter, next source pointer
	yloop	9, b6b, -9*4*dist64	;; Test loop counter, restore source pointer

;; Do FFT levels 13,14 as well as inverse FFT levels 13,14
;; On input and output the 64-byte cache lines hold these data values:
;;	0	+512	+512	+512	4	+512	+512	+512
;;	8	...
;;	...
;;	1	...
;;	...
;;	2048	...
;;	...

;; Do 1 eight_reals_fft operation
;; Do 2303 four_complex_fft operations
;;	distance between fft data elements is 1
;;	do 576 macros each processing 32 data values

	cmp	ffttype, 2
	jl	ypass2_9216_real_1
	jg	ypass2_9216_real_3

ypass2_9216_real_2:
;;	yr4_4cl_eight_reals_four_complex_with_square rsi, 64, dist64, 2*dist64
	yr4_dispatch call, yreal_square, dist64, 2*dist64
	yloop_init 576			;; 575 more iterations
	yloop_adjust 1
	yloop_set_prefetch YQUARTER
	yr4_4cl_four_complex_with_square_preload
b7b:	yr4_4cl_four_complex_with_square rsi, 64, dist64, 2*dist64, 1
	yloop	64, b7b, -64*64+4*dist64	;; Test loop counter, next source pointer
	yloop	9, b7b, -9*4*dist64	;; Test loop counter, restore source pointer
	jmp	ypass2_9216_real_unfft

ypass2_9216_real_1:
;;	yr4_4cl_eight_reals_four_complex_fft_final rsi, 64, dist64, 2*dist64
	yr4_dispatch call, yreal_fft_final, dist64, 2*dist64
	mov	rcx, xsincos_complex	;; Prefetch complex sin/cos data, 1152 cache lines
	yloop_init 576			;; 575 more iterations
	yloop_adjust 1
	yloop_set_prefetch YDOUBLE
b7a:	yr4_4cl_four_complex_fft_final rsi, 64, dist64, 2*dist64
	yloop	64, b7a, -64*64+4*dist64	;; Test loop counter, next source pointer
	yloop	9, b7a			;; Test loop counter
	jmp	ypass2_9216_real_done

ypass2_9216_real_3:
x64	mov	r8, DEST2ARG
	mov	rbp, DIST_TO_MULSRCARG
;;	yr4_4cl_eight_reals_four_complex_with_mult rsi, 64, dist64, 2*dist64
	yr4_dispatch call, yreal_mult, dist64, 2*dist64
	yloop_init 576			;; 575 more iterations
	yloop_adjust 1
	yloop_set_prefetch YQUARTER
	yr4_4cl_four_complex_with_mult_preload
b7c:	yr4_4cl_four_complex_with_mult rsi, 64, dist64, 2*dist64
	yloop	64, b7c, -64*64+4*dist64	;; Test loop counter, next source pointer
	yloop	9, b7c, -9*4*dist64	;; Test loop counter, restore source pointer
x64	add	rsi, r8
	jmp	ypass2_9216_real_unfft

ypass2_9216_real_4:
	mov	rbp, DIST_TO_MULSRCARG
;;	yr4_4cl_eight_reals_four_complex_with_mulf rsi, 64, dist64, 2*dist64
	yr4_dispatch call, yreal_mulf, dist64, 2*dist64
	yloop_init 576			;; 575 more iterations
	yloop_adjust 1
	yloop_set_prefetch YDOUBLE
	yr4_4cl_four_complex_with_mulf_preload
b7d:	yr4_4cl_four_complex_with_mulf rsi, 64, dist64, 2*dist64
	yloop	64, b7d, -64*64+4*dist64	;; Test loop counter, next source pointer
	yloop	9, b7d, -9*4*dist64	;; Test loop counter, restore source pointer
;;	bump	rcx, (-1152+2*384+2*288+3*144)*64 ;; Move prefetch pointer back to where squaring would be

;; Do inverse FFT levels 11,12
;; On input the 64-byte cache lines hold these data values:
;;	0	+512	+512	+512	4	+512	+512	+512
;;	8	...
;;	...
;;	1	...
;;	...
;;	2048	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+512	+512	+512	16	+512	+512	+512
;;	4	...
;;	...
;;	12	...
;;	32	...
;;	...
;;	1	...
;;	...
;;	2048	...
;;	...

;; Do 4 eight_reals_unfft operations
;; Do 2300 four_complex_unfft operations
;;	distance between fft data elements is 4
;;	do 576 macros each processing 32 data values

ypass2_9216_real_unfft:
	mov	rdi, sincos3		;; Load sin/cos pointer
	bump	rdi, 64*YMM_SCD2+256*YMM_SCD1+768*YMM_SCD1
	yloop_init 576			;; 144 iterations of 4
	yloop_set_prefetch YQUARTER
	yr4_4cl_eight_reals_four_complex_djbunfft_preload
c6a:	yr4_4cl_eight_reals_four_complex_djbunfft rsi, dist64, 64, 2*64, rdi, 16*YMM_SCD3, 4
	yloop	4, c6a, -4*dist64+4*64, rdi, -4*16*YMM_SCD3 ;; Test loop counter, next source pointer, restore sine/cosine pointer

	loops_reset			;; 143 more iters of 4
	yr4_rb4cl_four_complex_djbunfft_preload
c6b:	yr4_rb4cl_four_complex_djbunfft rsi, dist64, 64, 2*64, rdi, 16*YMM_SCD3, 4
	yloop	4, c6b, -4*dist64+4*64, rdi, -4*16*YMM_SCD3 ;; Test loop counter, next source pointer, restore sine/cosine pointer
	yloop	16, c6b, -16*4*64+4*dist64	;; Test loop counter, next source pointer
	yloop	9, c6b, -9*4*dist64	;; Test loop counter, restore source pointer

;; Do inverse FFT levels 9,10
;; On input the 64-byte cache lines hold these data values:
;;	0	+512	+512	+512	16	+512	+512	+512
;;	4	...
;;	...
;;	12	...
;;	32	...
;;	...
;;	1	...
;;	...
;;	2048	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+512	+512	+512	64	+512	+512	+512
;;	4	...
;;	...
;;	60	...
;;	128	...
;;	...
;;	1	...
;;	...
;;	2048	...
;;	...

;; Do 16 eight_reals_unfft operations
;; Do 2288 four_complex_unfft operations
;;	distance between fft data elements is 16
;;	do 576 macros each processing 32 data values

	yloop_init 576			;; 36 iterations of 16
	yloop_set_prefetch YQUARTER
	yr4_4cl_eight_reals_four_complex_djbunfft_preload
c5a:	yr4_4cl_eight_reals_four_complex_djbunfft rsi, dist64, 4*64, 8*64, rdi, 4*YMM_SCD3, 4
	yloop	4, c5a, -4*dist64+64	;; Test loop counter, next source pointer
	yloop	4, c5a, -4*64+16*64, rdi, -16*4*YMM_SCD3 ;; Test loop counter, next source pointer, restore sine/cosine pointer

	loops_reset			;; 35 more iters of 16
	yr4_rb4cl_four_complex_djbunfft_preload
c5b:	yr4_rb4cl_four_complex_djbunfft rsi, dist64, 4*64, 8*64, rdi, 4*YMM_SCD3, 4
	yloop	4, c5b, -4*dist64+64	;; Test loop counter, next source pointer
	yloop	4, c5b, -4*64+16*64, rdi, -16*4*YMM_SCD3 ;; Test loop counter, next source pointer, restore sine/cosine pointer
	yloop	4, c5b, -4*16*64+4*dist64	;; Test loop counter, next source pointer
	yloop	9, c5b, -9*4*dist64	;; Test loop counter, restore source pointer

;; Do inverse FFT levels 7,8
;; On input the 64-byte cache lines hold these data values:
;;	0	+512	+512	+512	64	+512	+512	+512
;;	4	...
;;	...
;;	60	...
;;	128	...
;;	...
;;	1	...
;;	...
;;	2048	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+512	+512	+512	256	+512	+512	+512
;;	4	...
;;	...
;;	508	...
;;	1	...
;;	...
;;	2048	...
;;	...

;; Do 64 eight_reals_unfft operations
;; Do 2240 four_complex_unfft operations
;;	distance between fft data elements is 64
;;	do 576 macros each processing 32 data values

	bump	rcx, (2304+35-2*384-2*288-5*144-288)*64 ;; Move prefetch pointer along
	yloop_init 576			;; 9 iterations of 64
	yloop_set_prefetch YHALF
	yr4_4cl_eight_reals_four_complex_djbunfft_preload
c4a:	yr4_4cl_eight_reals_four_complex_djbunfft rsi, dist64, 16*64, 32*64, rdi, YMM_SCD3, 4
	yloop	4, c4a, -4*dist64+64	;; Test loop counter, next source pointer
	yloop	16, c4a, -16*64+4*dist64, rdi, -64*YMM_SCD3 ;; Test loop counter, next source pointer, restore sine/cosine pointer

	loops_reset			;; 8 more iters of 64
	yr4_rb4cl_four_complex_djbunfft_preload
c4b:	yr4_rb4cl_four_complex_djbunfft rsi, dist64, 16*64, 32*64, rdi, YMM_SCD3, 4
	yloop	4, c4b, -4*dist64+64	;; Test loop counter, next source pointer
	yloop	16, c4b, -16*64+4*dist64, rdi, -64*YMM_SCD3 ;; Test loop counter, next source pointer, restore sine/cosine pointer
	yloop	9, c4b, -9*4*dist64	;; Test loop counter, restore source pointer

;; Do inverse FFT levels 5,6
;; On input the 64-byte cache lines hold these data values:
;;	0	+512	+512	+512	256	+512	+512	+512
;;	4	...
;;	...
;;	508	...
;;	1	...
;;	...
;;	2048	...
;;	...
;; On output the 64-byte cache lines are swizzled to hold these data values:
;;	0	+1	+1	+1	1024	+1	+1	+1
;;	4	...
;;	...
;;	1020	...
;;	2048	...
;;	...

;; Do 256 eight_reals_unfft operations
;; Do 2048 four_complex_unfft operations
;;	distance between fft data elements is 256
;;	do 576 macros each processing 32 data values

	bump	rdi, -64*YMM_SCD2	;; Real sin/cos ptr
	mov	rbp, xsincos_complex	;; Complex sin/cos data
	bump	rbp, 256*YMM_SCD1+768*YMM_SCD1
	mov	rcx, xsincos_complex	;; Prefetch complex sin/cos data, 1152 cache lines
	yloop_init 576			;; 9 iters of 64
	yloop_set_prefetch YSINGLE
	yr4_s4cl_2sc_eight_reals_unfft_preload
c3a:	yr4_s4cl_2sc_eight_reals_unfft rsi, 64, dist64, 2*dist64, rbp, YMM_SCD2, rdi, YMM_SCD2, 64
	yloop	64, c3a, -64*64+4*dist64, rbp, -64*YMM_SCD2 ;; Test loop counter, next source pointer, restore sine/cosine pointer

	loops_reset			;; 8 more iters of 64
	yr4_s4cl_four_complex_djbunfft_preload
c3b:	yr4_s4cl_four_complex_djbunfft rsi, 64, dist64, 2*dist64, rbp, YMM_SCD2, 64
	yloop	64, c3b, -64*64+4*dist64, rbp, -64*YMM_SCD2 ;; Test loop counter, next source pointer, restore sine/cosine pointer
	yloop	9, c3b, -9*4*dist64	;; Test loop counter, restore source pointer

;; Do inverse FFT levels 3,4
;; On input the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	1024	+1	+1	+1
;;	4	...
;;	...
;;	1020	...
;;	2048	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	3072	+1	+1	+1
;;	4	...
;;	...
;;	3068	...
;;	6144	...
;;	...

;; Do 1024 six_reals_unfft operations
;; Do 2048 three_complex_unfft operations
;;	distance between fft data elements is 1024
;;	do 768 macros each processing 24 data values

	bump	rdi, -64*YMM_SCD2-256*YMM_SCD1 ;; Real sin/cos ptr
	bump	rbp, -256*YMM_SCD1	;; Complex sin/cos data
	yloop_init 768			;; 3 iters of 256
	yloop_set_prefetch YHALF
	yr3_3cl_2sc_six_reals_unfft_preload
c2a:	yr3_3cl_2sc_six_reals_unfft rsi, 64, 4*dist64, rbp, YMM_SCD1, rdi, YMM_SCD1, 64
	yloop	64, c2a, -64*64+dist64	;; Test loop counter, next source pointer
	yloop	4, c2a, -4*dist64+12*dist64, rbp, -256*YMM_SCD1 ;; Test loop counter, next source pointer, restore sine/cosine pointer

	loops_reset			;; 2 more iters of 256
	yr3_3cl_three_complex_djbunfft_preload
c2b:	yr3_3cl_three_complex_djbunfft rsi, 64, 4*dist64, rbp, YMM_SCD1, 64
	yloop	64, c2b, -64*64+dist64	;; Test loop counter, next source pointer
	yloop	4, c2b, -4*dist64+12*dist64, rbp, -256*YMM_SCD1 ;; Test loop counter, next source pointer, restore sine/cosine pointer
	yloop	3, c2b, -3*12*dist64	;; Test loop counter, restore source pointer

;; Do inverse FFT levels 1,2,3 which correspond to complex levels 1,2
;; On input the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	3072	+1	+1	+1
;;	4	...
;;	...
;;	3068	...
;;	6144	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	9216	+1	+1	+1
;;	4	...
;;	...
;;	9212

;; Do 3072 six_reals_unfft operations
;;	distance between fft data elements is 3072
;;	do 768 macros each processing 24 data values

	bump	rdi, -256*YMM_SCD1-768*YMM_SCD1 ;; Real sin/cos ptr
	bump	rbp, -768*YMM_SCD1	;; Complex sin/cos ptr
	bump	rcx, (1152-576-384-384)*64 ;; Move prefetch pointer along
	yloop_init 768			;; 768 iterations
	yloop_set_prefetch YHALF
	yr3_3cl_2sc_six_reals_unfft_preload
c1b:	yr3_3cl_2sc_six_reals_unfft rsi, 64, 12*dist64, rbp, YMM_SCD1, rdi, YMM_SCD1, 64
	yloop	64, c1b, -64*64+dist64	;; Test loop counter, next source pointer
	yloop	12, c1b			;; Test loop counter

ypass2_9216_real_done:
	end_timer 2
	ENDM

; *************** radix4-pass2-9216-complex macro ******************
; This macro takes 9216 complex values and performs the final 13.585 levels of a standard
; radix-4 FFT, squares the results, and does the first 13.585 levels of the inverse FFT.

yr4dwpn_pass2_9216_complex MACRO
	LOCAL	b1a, b1b, b2b, b3a0, b3a, b3b0, b3b, b3c0, b3c, b4a, b4b, b4c, b5a, b5b, b5c, b6a, b6b, b6c, b7a, b7b, b7c, b7d
	LOCAL	c1b, c2a, c2b, c3b, c3c, c3d, c4b, c4c, c4d, c5b, c5c, c5d, c6b, c6c, c6d
	LOCAL	ypass2_9216_complex_1, ypass2_9216_complex_2
	LOCAL	ypass2_9216_complex_3, ypass2_9216_complex_4
	LOCAL	ypass2_9216_complex_unfft, ypass2_9216_complex_done

;; Type 4 FFTs skip the forward FFT process

	mov	rsi, DATA_ADDR		;; Load address of FFT data
	mov	rbx, DIST_TO_FFTSRCARG
	mov	rcx, DATA_PREFETCH	;; Prefetch pointer (we need to prefetch 
					;; 2304 cache lines + 35 pad cache lines)
	cmp	ffttype, 4
	je	ypass2_9216_complex_4

;; Do FFT level 1,2 (actually 1.585 levels)
;; On input the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	9216	+1	+1	+1
;;	4	...
;;	...
;;	9212
;; On output the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	3072	+1	+1	+1
;;	4	...
;;	...
;;	3068	...
;;	6144	...
;;	...

;; Do 3072 three_complex_fft operations
;;	distance between fft data elements is 3072
;;	do 768 macros each processing 24 data values

;; Do FFT levels 3,4 (actually 1.585 levels)
;; On input the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	3072	+1	+1	+1
;;	4	...
;;	...
;;	3068	...
;;	6144	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	1024	+1	+1	+1
;;	4	...
;;	...
;;	1020	...
;;	2048	...
;;	...

;; Do 3072 three_complex_fft operations
;;	distance between fft data elements is 1024
;;	do 768 macros each processing 24 data values

	mov	rdi, xsincos_complex	;; The sin/cos ptrs
	lea	rbp, [rdi+768*YMM_SCD1]
	yloop_init 1024			;; 768 iterations

	yr3_f3cl_three_complex_djbfft_preload
b1a:	start_timer 3
	yloop_adjust 1
	yloop_set_prefetch YQUARTER	;; Prefetch 256 cache lines
b1b:	yr3_f3cl_three_complex_djbfft rsi, 4*dist64, 12*dist64, rdi, 256*YMM_SCD1, 3
	yloop	4, b1b, -3*4*dist64, rdi, -3*256*YMM_SCD1+YMM_SCD1 ;; 3 iterations!!  Restore source pointer, next sin/cos data
	loops_undo 4			;; Back up loop counter
	end_timer 3

	start_timer 4
	yloop_adjust 1
	yloop_set_prefetch YQUARTER	;; Prefetch 256 cache lines
;;	yr3_3cl_three_complex_djbfft_preload
b2b:	yr3_3cl_three_complex_djbfft rsi, 12*dist64, 4*dist64, rbp, 0, 3, L1PREFETCH_ALL, 64
	yloop	4, b2b, -3*12*dist64+64, rbp, YMM_SCD1 ;; 3 iterations!!  Next source pointer, next sin/cos data
	end_timer 4

	yloop	64, b1a, -64*64+dist64	;; Test loop counter, next source pointer
	yloop	4, b1a, -4*dist64, rdi, -256*YMM_SCD1+768*YMM_SCD1+256*YMM_SCD1 ;; Test loop counter, restore source pointer, next sin/cos ptr

;; Do FFT levels 5,6
;; On input the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	1024	+1	+1	+1
;;	4	...
;;	...
;;	1020	...
;;	2048	...
;;	...
;; On output the 64-byte cache lines are swizzled to hold these data values:
;;	0	+512	+512	+512	256	+512	+512	+512
;;	4	...
;;	...
;;	508	...
;;	1	...
;;	...
;;	2048	...
;;	...

;; Do 2304 four_complex_fft operations
;;	distance between fft data elements is 256
;;	do 576 macros each processing 32 data values

;; Do FFT levels 7,8
;; On input the 64-byte cache lines hold these data values:
;;	0	+512	+512	+512	256	+512	+512	+512
;;	4	...
;;	...
;;	508	...
;;	1	...
;;	...
;;	2048	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+512	+512	+512	64	+512	+512	+512
;;	4	...
;;	...
;;	60	...
;;	128	...
;;	...
;;	1	...
;;	...
;;	2048	...
;;	...

;; Do 2304 four_complex_fft operations
;;	distance between fft data elements is 64
;;	do 576 macros each processing 32 data values

;; Do FFT levels 9,10
;; On input the 64-byte cache lines hold these data values:
;;	0	+512	+512	+512	64	+512	+512	+512
;;	4	...
;;	...
;;	60	...
;;	128	...
;;	...
;;	1	...
;;	...
;;	2048	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+512	+512	+512	16	+512	+512	+512
;;	4	...
;;	...
;;	12	...
;;	32	...
;;	...
;;	1	...
;;	...
;;	2048	...
;;	...

;; Do 2304 four_complex_fft operations
;;	distance between fft data elements is 16
;;	do 576 macros each processing 32 data values

;; Do FFT levels 11,12
;; On input the 64-byte cache lines hold these data values:
;;	0	+512	+512	+512	16	+512	+512	+512
;;	4	...
;;	...
;;	12	...
;;	32	...
;;	...
;;	1	...
;;	...
;;	2048	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+512	+512	+512	4	+512	+512	+512
;;	8	...
;;	...
;;	1	...
;;	...
;;	2048	...
;;	...

;; Do 2304 four_complex_fft operations
;;	distance between fft data elements is 4
;;	do 576 macros each processing 32 data values

;; Do FFT levels 13,14 as well as inverse FFT levels 13,14
;; On input and output the 64-byte cache lines hold these data values:
;;	0	+512	+512	+512	4	+512	+512	+512
;;	8	...
;;	...
;;	1	...
;;	...
;;	2048	...
;;	...

;; Do 2304 four_complex_fft operations
;;	distance between fft data elements is 1
;;	do 576 macros each processing 32 data values

;; Do inverse FFT levels 11,12
;; On input the 64-byte cache lines hold these data values:
;;	0	+512	+512	+512	4	+512	+512	+512
;;	8	...
;;	...
;;	1	...
;;	...
;;	2048	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+512	+512	+512	16	+512	+512	+512
;;	4	...
;;	...
;;	12	...
;;	32	...
;;	...
;;	1	...
;;	...
;;	2048	...
;;	...

;; Do 2304 four_complex_unfft operations
;;	distance between fft data elements is 4
;;	do 576 macros each processing 32 data values

;; Do inverse FFT levels 9,10
;; On input the 64-byte cache lines hold these data values:
;;	0	+512	+512	+512	16	+512	+512	+512
;;	4	...
;;	...
;;	12	...
;;	32	...
;;	...
;;	1	...
;;	...
;;	2048	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+512	+512	+512	64	+512	+512	+512
;;	4	...
;;	...
;;	60	...
;;	128	...
;;	...
;;	1	...
;;	...
;;	2048	...
;;	...

;; Do 2304 four_complex_unfft operations
;;	distance between fft data elements is 16
;;	do 576 macros each processing 32 data values

;; Do inverse FFT levels 7,8
;; On input the 64-byte cache lines hold these data values:
;;	0	+512	+512	+512	64	+512	+512	+512
;;	4	...
;;	...
;;	60	...
;;	128	...
;;	...
;;	1	...
;;	...
;;	2048	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+512	+512	+512	256	+512	+512	+512
;;	4	...
;;	...
;;	508	...
;;	1	...
;;	...
;;	2048	...
;;	...

;; Do 2304 four_complex_unfft operations
;;	distance between fft data elements is 64
;;	do 576 macros each processing 32 data values

;; Do inverse FFT levels 5,6
;; On input the 64-byte cache lines hold these data values:
;;	0	+512	+512	+512	256	+512	+512	+512
;;	4	...
;;	...
;;	508	...
;;	1	...
;;	...
;;	2048	...
;;	...
;; On output the 64-byte cache lines are swizzled to hold these data values:
;;	0	+1	+1	+1	1024	+1	+1	+1
;;	4	...
;;	...
;;	1020	...
;;	2048	...
;;	...

;; Do 2304 four_complex_unfft operations
;;	distance between fft data elements is 256
;;	do 576 macros each processing 32 data values

;; Do this 256 cache lines (16KB) at a time to maximize accessing data
;; while it is in the L1 cache.

;; Execute the right middle step

	cmp	ffttype, 2
	jl	ypass2_9216_complex_1
	jg	ypass2_9216_complex_3

ypass2_9216_complex_2:
	yloop_init 576			;; 9 iters of 256 cache lines

b3b0:	start_timer 5
	yloop_set_prefetch YQUARTER	;; Prefetch 144 cache lines in this loop
	yr4_s4cl_four_complex_djbfft_preload
;; Loop below reworked to avoid 4KB distances
;;b3b:	yr4_s4cl_four_complex_djbfft rsi, 64, dist64, 2*dist64, rdi, YMM_SCD2, 64
;;	yloop	64, b3b, -64*64, rdi, -64*YMM_SCD2 ;; Loop 64 times, restore source pointer, restore sine/cosine pointer
b3b:	yr4_s4cl_four_complex_djbfft rsi, 4*64, dist64, 2*dist64, rdi, 4*YMM_SCD2, 16
	yloop	16, b3b, -16*4*64+64, rdi, -16*4*YMM_SCD2+YMM_SCD2 ;; Loop 16 times, next source pointer, next sine/cosine pointer
	yloop	4, b3b, -4*64, rdi, -4*YMM_SCD2 ;; Loop 4 times, restore source pointer, restore sine/cosine pointer
	loops_undo 64			;; Back up loop counter
	end_timer 5

	start_timer 6
	yloop_set_prefetch YQUARTER	;; Prefetch 144 cache lines in this loop
	yr4_eb4cl_four_complex_djbfft_preload
b4b:	yr4_eb4cl_four_complex_djbfft rsi, 64, 16*64, 32*64, rdi, 4*YMM_SCD2, 16
	yloop	16, b4b, -16*64+dist64, rdi, -16*4*YMM_SCD2+YMM_SCD2 ;; Loop 16 times, next source pointer, next sine/cosine pointer
	yloop	4, b4b, -4*dist64, rdi, -4*YMM_SCD2 ;; Loop 4 times, restore source pointer, restore sine/cosine pointer
	loops_undo 4*16			;; Back up loop counter
	end_timer 6

	start_timer 7
	yloop_set_prefetch YQUARTER	;; Prefetch 144 cache lines in this loop
;;	yr4_eb4cl_four_complex_djbfft_preload
b5b:	yr4_eb4cl_four_complex_djbfft rsi, 64, 4*64, 8*64, rdi, 16*YMM_SCD2, 4
	yloop	4, b5b, -4*64+16*64, rdi, -4*16*YMM_SCD2 ;; Loop 4 times, next source pointer, restore sine/cosine pointer
	yloop	4, b5b, -4*16*64+dist64, rdi, 4*YMM_SCD2 ;; Loop 4 times, next source pointer, next sine/cosine pointer
	yloop	4, b5b, -4*dist64, rdi, -4*4*YMM_SCD2 ;; Test loop counter, restore source pointer, restore sine/cosine pointer
	loops_undo 4*4*4		;; Back up loop counter
	end_timer 7

	start_timer 8
	yloop_set_prefetch YQUARTER	;; Prefetch 144 cache lines in this loop
;;	yr4_eb4cl_four_complex_djbfft_preload
b6b:	yr4_eb4cl_four_complex_djbfft rsi, 4*64, 64, 2*64, rdi, 0, 16
	yloop	16, b6b, -16*4*64+dist64, rdi, 16*YMM_SCD2 ;; Loop 4 times, next source pointer, next sine/cosine pointer
	yloop	4, b6b, -4*dist64, rdi, -4*16*YMM_SCD2 ;; Test loop counter, restore source pointer, restore sine/cosine pointer
	loops_undo 4*16			;; Back up loop counter
	end_timer 8

	start_timer 9
	yloop_set_prefetch YOTHER, 16	;; Prefetch 36 cache lines in this loop
	yr4_4cl_four_complex_with_square_preload
;; Loop below reworked to avoid 4KB distances
;;b7b:	yr4_4cl_four_complex_with_square rsi, 64, dist64, 2*dist64
;;	yloop	64, b7b, -64*64	;; Loop 64 times, restore source pointer
b7b:	yr4_4cl_four_complex_with_square rsi, 4*64, dist64, 2*dist64, 16
	yloop	16, b7b, -16*4*64+64	;; Loop 16 times, next source pointer
	yloop	4, b7b, -4*64	;; Loop 4 times, restore source pointer
	loops_undo 64			;; Back up loop counter
	end_timer 9

	start_timer 10
	yloop_set_prefetch YQUARTER	;; Prefetch 144 cache lines in this loop
	yr4_eb4cl_four_complex_djbunfft_preload
c6b:	yr4_eb4cl_four_complex_djbunfft rsi, 4*64, 64, 2*64, rdi, 0, 16
	yloop	16, c6b, -16*4*64+dist64, rdi, 16*YMM_SCD2 ;; Loop 4 times, next source pointer, next sine/cosine pointer
	yloop	4, c6b, -4*dist64, rdi, -4*16*YMM_SCD2 ;; Test loop counter, restore source pointer, restore sine/cosine pointer
	loops_undo 4*16			;; Back up loop counter
	end_timer 10

	start_timer 11
	yloop_set_prefetch YQUARTER	;; Prefetch 144 cache lines in this loop
;;	yr4_eb4cl_four_complex_djbunfft_preload
c5b:	yr4_eb4cl_four_complex_djbunfft rsi, 64, 4*64, 8*64, rdi, 16*YMM_SCD2, 4
	yloop	4, c5b, -4*64+16*64, rdi, -4*16*YMM_SCD2 ;; Loop 4 times, next source pointer, restore sine/cosine pointer
	yloop	4, c5b, -4*16*64+dist64, rdi, 4*YMM_SCD2 ;; Loop 4 times, next source pointer, next sine/cosine pointer
	yloop	4, c5b, -4*dist64, rdi, -4*4*YMM_SCD2 ;; Test loop counter, restore source pointer, restore sine/cosine pointer
	loops_undo 4*4*4		;; Back up loop counter
	end_timer 11

	start_timer 12
	yloop_set_prefetch YQUARTER	;; Prefetch 144 cache lines in this loop
;;	yr4_eb4cl_four_complex_djbunfft_preload
c4b:	yr4_eb4cl_four_complex_djbunfft rsi, 64, 16*64, 32*64, rdi, 4*YMM_SCD2, 16
	yloop	16, c4b, -16*64+dist64, rdi, -16*4*YMM_SCD2+YMM_SCD2 ;; Loop 16 times, next source pointer, next sine/cosine pointer
	yloop	4, c4b, -4*dist64, rdi, -4*YMM_SCD2 ;; Loop 4 times, restore source pointer, restore sine/cosine pointer
	loops_undo 4*16			;; Back up loop counter
	end_timer 12

	start_timer 13
	yloop_set_prefetch YHALF	;; Prefetch 288 cache lines in this loop
	yr4_s4cl_four_complex_djbunfft_preload
;; Loop below reworked to avoid 4KB distances
;;c3b:	yr4_s4cl_four_complex_djbunfft rsi, 4*64, dist64, 2*dist64, rdi, YMM_SCD2, 64
;;	yloop	64, c3b, -64*64, rdi, -64*YMM_SCD2 ;; Loop 64 times, restore source pointer, restore sine/cosine pointer
c3b:	yr4_s4cl_four_complex_djbunfft rsi, 4*64, dist64, 2*dist64, rdi, 4*YMM_SCD2, 16
	yloop	16, c3b, -16*4*64+64, rdi, -16*4*YMM_SCD2+YMM_SCD2 ;; Loop 16 times, next source pointer, next sine/cosine pointer
	yloop	4, c3b, -4*64+4*dist64, rdi, -4*YMM_SCD2 ;; Loop 4 times, next source pointer, restore sine/cosine pointer
	end_timer 13

	yloop	9, b3b0, -9*4*dist64	;; Loop 9 times, restore source pointer
	jmp	ypass2_9216_complex_unfft


ypass2_9216_complex_1:
	start_timer 6
	bump	rcx, (2304+35-2*256-2*576-2*288-144)*64 ;; Move prefetch pointer along
	yloop_init 576			;; 9 iters of 256 cache lines

b3a0:	yloop_set_prefetch YSINGLE	;; Prefetch 576 cache lines in this loop
	yr4_s4cl_four_complex_djbfft_preload
;; Loop below reworked to avoid 4KB distances
;;b3a:	yr4_s4cl_four_complex_djbfft rsi, 64, dist64, 2*dist64, rdi, YMM_SCD2, 64
;;	yloop	64, b3a, -64*64, rdi, -64*YMM_SCD2 ;; Loop 64 times, restore source pointer, restore sine/cosine pointer
b3a:	yr4_s4cl_four_complex_djbfft rsi, 4*64, dist64, 2*dist64, rdi, 4*YMM_SCD2, 16
	yloop	16, b3a, -16*4*64+64, rdi, -16*4*YMM_SCD2+YMM_SCD2 ;; Loop 16 times, next source pointer, next sine/cosine pointer
	yloop	4, b3a, -4*64, rdi, -4*YMM_SCD2 ;; Loop 4 times, restore source pointer, restore sine/cosine pointer
	loops_undo 64			;; Back up loop counter

	yloop_set_prefetch YSINGLE	;; Prefetch 576 cache lines in this loop
	yr4_eb4cl_four_complex_djbfft_preload
b4a:	yr4_eb4cl_four_complex_djbfft rsi, 64, 16*64, 32*64, rdi, 4*YMM_SCD2, 16
	yloop	16, b4a, -16*64+dist64, rdi, -16*4*YMM_SCD2+YMM_SCD2 ;; Loop 16 times, next source pointer, next sine/cosine pointer
	yloop	4, b4a, -4*dist64, rdi, -4*YMM_SCD2 ;; Loop 4 times, restore source pointer, restore sine/cosine pointer
	loops_undo 4*16			;; Back up loop counter

	yloop_set_prefetch YHALF	;; Prefetch 288 cache lines in this loop
;;	yr4_eb4cl_four_complex_djbfft_preload
b5a:	yr4_eb4cl_four_complex_djbfft rsi, 64, 4*64, 8*64, rdi, 16*YMM_SCD2, 4
	yloop	4, b5a, -4*64+16*64, rdi, -4*16*YMM_SCD2 ;; Loop 4 times, next source pointer, restore sine/cosine pointer
	yloop	4, b5a, -4*16*64+dist64, rdi, 4*YMM_SCD2 ;; Loop 4 times, next source pointer, next sine/cosine pointer
	yloop	4, b5a, -4*dist64, rdi, -4*4*YMM_SCD2 ;; Test loop counter, restore source pointer, restore sine/cosine pointer
	loops_undo 4*4*4		;; Back up loop counter

	yloop_set_prefetch YHALF	;; Prefetch 288 cache lines in this loop
;;	yr4_eb4cl_four_complex_djbfft_preload
b6a:	yr4_eb4cl_four_complex_djbfft rsi, 4*64, 64, 2*64, rdi, 0, 16
	yloop	16, b6a, -16*4*64+dist64, rdi, 16*YMM_SCD2 ;; Loop 4 times, next source pointer, next sine/cosine pointer
	yloop	4, b6a, -4*dist64, rdi, -4*16*YMM_SCD2 ;; Test loop counter, restore source pointer, restore sine/cosine pointer
	loops_undo 4*16			;; Back up loop counter

	yloop_set_prefetch YQUARTER	;; Prefetch 144 cache lines in this loop
	yr4_4cl_four_complex_fft_final_preload
;; Loop below reworked to avoid 4KB distances
;;b7a:	yr4_4cl_four_complex_fft_final rsi, 64, dist64, 2*dist64
;;	yloop	64, b7a, -64*64	;; Loop 64 times, restore source pointer
b7a:	yr4_4cl_four_complex_fft_final rsi, 4*64, dist64, 2*dist64
	yloop	16, b7a, -16*4*64+64	;; Loop 16 times, next source pointer
	yloop	4, b7a, -4*64+4*dist64	;; Loop 4 times, next source pointer

	yloop	9, b3a0			;; Loop 9 times
	end_timer 6
	jmp	ypass2_9216_complex_done


ypass2_9216_complex_3:
	start_timer 6
x64	mov	r8, DEST2ARG
	mov	rbp, DIST_TO_MULSRCARG
	yloop_init 576			;; 9 iters of 256 cache lines

b3c0:	yloop_set_prefetch YQUARTER	;; Prefetch 144 cache lines in this loop
	yr4_s4cl_four_complex_djbfft_preload
;; Loop below reworked to avoid 4KB distances
;;b3c:	yr4_s4cl_four_complex_djbfft rsi, 64, dist64, 2*dist64, rdi, YMM_SCD2, 64
;;	yloop	64, b3c, -64*64, rdi, -64*YMM_SCD2 ;; Loop 64 times, restore source pointer, restore sine/cosine pointer
b3c:	yr4_s4cl_four_complex_djbfft rsi, 4*64, dist64, 2*dist64, rdi, 4*YMM_SCD2, 16
	yloop	16, b3c, -16*4*64+64, rdi, -16*4*YMM_SCD2+YMM_SCD2 ;; Loop 16 times, next source pointer, next sine/cosine pointer
	yloop	4, b3c, -4*64, rdi, -4*YMM_SCD2 ;; Loop 4 times, restore source pointer, restore sine/cosine pointer
	loops_undo 64			;; Back up loop counter

	yloop_set_prefetch YQUARTER	;; Prefetch 144 cache lines in this loop
	yr4_eb4cl_four_complex_djbfft_preload
b4c:	yr4_eb4cl_four_complex_djbfft rsi, 64, 16*64, 32*64, rdi, 4*YMM_SCD2, 16
	yloop	16, b4c, -16*64+dist64, rdi, -16*4*YMM_SCD2+YMM_SCD2 ;; Loop 16 times, next source pointer, next sine/cosine pointer
	yloop	4, b4c, -4*dist64, rdi, -4*YMM_SCD2 ;; Loop 4 times, restore source pointer, restore sine/cosine pointer
	loops_undo 4*16			;; Back up loop counter

	yloop_set_prefetch YQUARTER	;; Prefetch 144 cache lines in this loop
;;	yr4_eb4cl_four_complex_djbfft_preload
b5c:	yr4_eb4cl_four_complex_djbfft rsi, 64, 4*64, 8*64, rdi, 16*YMM_SCD2, 4
	yloop	4, b5c, -4*64+16*64, rdi, -4*16*YMM_SCD2 ;; Loop 4 times, next source pointer, restore sine/cosine pointer
	yloop	4, b5c, -4*16*64+dist64, rdi, 4*YMM_SCD2 ;; Loop 4 times, next source pointer, next sine/cosine pointer
	yloop	4, b5c, -4*dist64, rdi, -4*4*YMM_SCD2 ;; Test loop counter, restore source pointer, restore sine/cosine pointer
	loops_undo 4*4*4		;; Back up loop counter

	yloop_set_prefetch YQUARTER	;; Prefetch 144 cache lines in this loop
;;	yr4_eb4cl_four_complex_djbfft_preload
b6c:	yr4_eb4cl_four_complex_djbfft rsi, 4*64, 64, 2*64, rdi, 0, 16
	yloop	16, b6c, -16*4*64+dist64, rdi, 16*YMM_SCD2 ;; Loop 4 times, next source pointer, next sine/cosine pointer
	yloop	4, b6c, -4*dist64, rdi, -4*16*YMM_SCD2 ;; Test loop counter, restore source pointer, restore sine/cosine pointer
	loops_undo 4*16			;; Back up loop counter

	yloop_set_prefetch YOTHER, 16	;; Prefetch 36 cache lines in this loop
	yr4_4cl_four_complex_with_mult_preload
;; Loop below reworked to avoid 4KB distances
;;b7c:	yr4_4cl_four_complex_with_mult rsi, 64, dist64, 2*dist64
;;	yloop	64, b7c, -64*64	;; Loop 64 times, restore source pointer
b7c:	yr4_4cl_four_complex_with_mult rsi, 4*64, dist64, 2*dist64
	yloop	16, b7c, -16*4*64+64	;; Loop 16 times, next source pointer
	yloop	4, b7c, -4*64		;; Loop 4 times, restore source pointer
x64	add	rsi, r8
	loops_undo 64			;; Back up loop counter

	yloop_set_prefetch YQUARTER	;; Prefetch 144 cache lines in this loop
	yr4_eb4cl_four_complex_djbunfft_preload
c6c:	yr4_eb4cl_four_complex_djbunfft rsi, 4*64, 64, 2*64, rdi, 0, 16
	yloop	16, c6c, -16*4*64+dist64, rdi, 16*YMM_SCD2 ;; Loop 16 times, next source pointer, next sine/cosine pointer
	yloop	4, c6c, -4*dist64, rdi, -4*16*YMM_SCD2 ;; Test loop counter, restore source pointer, restore sine/cosine pointer
	loops_undo 4*16			;; Back up loop counter

	yloop_set_prefetch YQUARTER	;; Prefetch 144 cache lines in this loop
;;	yr4_eb4cl_four_complex_djbunfft_preload
c5c:	yr4_eb4cl_four_complex_djbunfft rsi, 64, 4*64, 8*64, rdi, 16*YMM_SCD2, 4
	yloop	4, c5c, -4*64+16*64, rdi, -4*16*YMM_SCD2 ;; Loop 4 times, next source pointer, restore sine/cosine pointer
	yloop	4, c5c, -4*16*64+dist64, rdi, 4*YMM_SCD2 ;; Loop 4 times, next source pointer, next sine/cosine pointer
	yloop	4, c5c, -4*dist64, rdi, -4*4*YMM_SCD2 ;; Test loop counter, restore source pointer, restore sine/cosine pointer
	loops_undo 4*4*4		;; Back up loop counter

	yloop_set_prefetch YQUARTER	;; Prefetch 144 cache lines in this loop
;;	yr4_eb4cl_four_complex_djbunfft_preload
c4c:	yr4_eb4cl_four_complex_djbunfft rsi, 64, 16*64, 32*64, rdi, 4*YMM_SCD2, 16
	yloop	16, c4c, -16*64+dist64, rdi, -16*4*YMM_SCD2+YMM_SCD2 ;; Loop 16 times, next source pointer, next sine/cosine pointer
	yloop	4, c4c, -4*dist64, rdi, -4*YMM_SCD2 ;; Loop 4 times, restore source pointer, restore sine/cosine pointer
	loops_undo 4*16			;; Back up loop counter

	yloop_set_prefetch YHALF	;; Prefetch 288 cache lines in this loop
	yr4_s4cl_four_complex_djbunfft_preload
;; Loop below reworked to avoid 4KB distances
;;c3c:	yr4_s4cl_four_complex_djbunfft rsi, 64, dist64, 2*dist64, rdi, YMM_SCD2, 64
;;	yloop	64, c3c, -64*64, rdi, -64*YMM_SCD2 ;; Loop 64 times, restore source pointer, restore sine/cosine pointer
c3c:	yr4_s4cl_four_complex_djbunfft rsi, 4*64, dist64, 2*dist64, rdi, 4*YMM_SCD2, 16
	yloop	16, c3c, -16*4*64+64, rdi, -16*4*YMM_SCD2+YMM_SCD2 ;; Loop 16 times, next source pointer, next sine/cosine pointer
	yloop	4, c3c, -4*64+4*dist64, rdi, -4*YMM_SCD2 ;; Loop 4 times, next source pointer, restore sine/cosine pointer

x64	sub	rsi, r8
	yloop	9, b3c0, -9*4*dist64	;; Loop 9 times, restore source pointer
x64	add	rsi, r8
	end_timer 6
	jmp	ypass2_9216_complex_unfft


ypass2_9216_complex_4:
	start_timer 6
	mov	rdi, xsincos_complex	;; The sin/cos ptrs
	bump	rdi, 768*YMM_SCD1+256*YMM_SCD1
	mov	rbp, DIST_TO_MULSRCARG
	yloop_init 576			;; 9 iters of 256 cache lines

	yloop_set_prefetch YQUARTER	;; Prefetch 144 cache lines in this loop
	yr4_4cl_four_complex_with_mulf_preload
;; Loop below reworked to avoid 4KB distances
;;b7d:	yr4_4cl_four_complex_with_mulf rsi, 64, dist64, 2*dist64
;;	yloop	64, b7d, -64*64	;; Loop 64 times, restore source pointer
b7d:	yr4_4cl_four_complex_with_mulf rsi, 4*64, dist64, 2*dist64
	yloop	16, b7d, -16*4*64+64	;; Loop 16 times, next source pointer
	yloop	4, b7d, -4*64		;; Loop 4 times, restore source pointer
	loops_undo 64			;; Back up loop counter

	yloop_set_prefetch YHALF	;; Prefetch 288 cache lines in this loop
	yr4_eb4cl_four_complex_djbunfft_preload
c6d:	yr4_eb4cl_four_complex_djbunfft rsi, 4*64, 64, 2*64, rdi, 0, 16
	yloop	16, c6d, -16*4*64+dist64, rdi, 16*YMM_SCD2 ;; Loop 4 times, next source pointer, next sine/cosine pointer
	yloop	4, c6d, -4*dist64, rdi, -4*16*YMM_SCD2 ;; Test loop counter, restore source pointer, restore sine/cosine pointer
	loops_undo 4*16			;; Back up loop counter

	yloop_set_prefetch YHALF	;; Prefetch 288 cache lines in this loop
;;	yr4_eb4cl_four_complex_djbunfft_preload
c5d:	yr4_eb4cl_four_complex_djbunfft rsi, 64, 4*64, 8*64, rdi, 16*YMM_SCD2, 4
	yloop	4, c5d, -4*64+16*64, rdi, -4*16*YMM_SCD2 ;; Loop 4 times, next source pointer, restore sine/cosine pointer
	yloop	4, c5d, -4*16*64+dist64, rdi, 4*YMM_SCD2 ;; Loop 4 times, next source pointer, next sine/cosine pointer
	yloop	4, c5d, -4*dist64, rdi, -4*4*YMM_SCD2 ;; Test loop counter, restore source pointer, restore sine/cosine pointer
	loops_undo 4*4*4		;; Back up loop counter

	yloop_set_prefetch YSINGLE	;; Prefetch 576 cache lines in this loop
;;	yr4_eb4cl_four_complex_djbunfft_preload
c4d:	yr4_eb4cl_four_complex_djbunfft rsi, 64, 16*64, 32*64, rdi, 4*YMM_SCD2, 16
	yloop	16, c4d, -16*64+dist64, rdi, -16*4*YMM_SCD2+YMM_SCD2 ;; Loop 16 times, next source pointer, next sine/cosine pointer
	yloop	4, c4d, -4*dist64, rdi, -4*YMM_SCD2 ;; Loop 4 times, restore source pointer, restore sine/cosine pointer
	loops_undo 4*16			;; Back up loop counter

	yloop_set_prefetch YSINGLE	;; Prefetch 576 cache lines in this loop
	yr4_s4cl_four_complex_djbunfft_preload
;; Loop below reworked to avoid 4KB distances
;;c3d:	yr4_s4cl_four_complex_djbunfft rsi, 64, dist64, 2*dist64, rdi, YMM_SCD2, 64
;;	yloop	64, c3d, -64*64, rdi, -64*YMM_SCD2 ;; Loop 64 times, restore source pointer, restore sine/cosine pointer
c3d:	yr4_s4cl_four_complex_djbunfft rsi, 4*64, dist64, 2*dist64, rdi, 4*YMM_SCD2, 16
	yloop	16, c3d, -16*4*64+64, rdi, -16*4*YMM_SCD2+YMM_SCD2 ;; Loop 16 times, next source pointer, next sine/cosine pointer
	yloop	4, c3d, -4*64+4*dist64, rdi, -4*YMM_SCD2 ;; Loop 4 times, next source pointer, restore sine/cosine pointer

	yloop	9, b7d, -9*4*dist64	;; Loop 9 times, restore source pointer
	bump	rcx, (-2*576-2*288-144+2*256+4*144+36+3*144+288)*64 ;; Move prefetch pointer to where squaring would be
	end_timer 6

;; Do inverse FFT levels 3,4
;; On input the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	1024	+1	+1	+1
;;	4	...
;;	...
;;	1020	...
;;	2048	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	3072	+1	+1	+1
;;	4	...
;;	...
;;	3068	...
;;	6144	...
;;	...

;; Do 3072 three_complex_unfft operations
;;	distance between fft data elements is 1024
;;	do 768 macros each processing 24 data values

;; Do inverse FFT levels 1,2
;; On input the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	3072	+1	+1	+1
;;	4	...
;;	...
;;	3068	...
;;	6144	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	9216	+1	+1	+1
;;	4	...
;;	...
;;	9212

;; Do 3072 three_complex_unfft operations
;;	distance between fft data elements is 3072
;;	do 768 macros each processing 24 data values

ypass2_9216_complex_unfft:
	mov	rdi, xsincos_complex	;; Load sin/cos pointers
	lea	rbp, [rdi+768*YMM_SCD1]
	bump	rcx, (2304+35-2*256-4*144-36-3*144-288-2*256)*64 ;; Move prefetch pointer along
	yloop_init 1024			;; 768 iterations

	yr3_3cl_three_complex_djbunfft_preload
c2a:	start_timer 14
	yloop_adjust 1
	yloop_set_prefetch YQUARTER	;; Prefetch 256 cache lines
c2b:	yr3_3cl_three_complex_djbunfft rsi, 12*dist64, 4*dist64, rbp, 0, 3
	yloop	4, c2b, -3*12*dist64, rbp, YMM_SCD1 ;; 3 iterations!!  Restore source pointer, next sine/cosine pointer
	loops_undo 4			;; Restore loop counter
	end_timer 14

	start_timer 15
	yloop_adjust 1
	yloop_set_prefetch YQUARTER	;; Prefetch 256 cache lines
;;	yr3_3cl_three_complex_djbunfft_preload
c1b:	yr3_3cl_three_complex_djbunfft rsi, 4*dist64, 12*dist64, rdi, 256*YMM_SCD1, 3, L1PREFETCH_ALL, 64
	yloop	4, c1b, -3*4*dist64+64, rdi, -3*256*YMM_SCD1+YMM_SCD1 ;; 3 iterations!!  Next source pointer, next sine/cosine pointer
	end_timer 15

	yloop	64, c2a, -64*64+dist64	;; Loop 64 times, next source pointer
	yloop	4, c2a			;; Loop 4 times

ypass2_9216_complex_done:
	ENDM


; *************** radix4-pass2-12288-real macro ******************
; This macro takes 24576 real values and performs the final 14.585 levels of the FFT.
; The results are squared, and the first 14.585 levels of the inverse FFT are done.

yr4dwpn_pass2_12288_real MACRO
	LOCAL	b1b, b2a, b2b, b3a, b3b, b4a, b4b, b5a, b5b, b6a, b6b, b7a, b7b, b7c, b7d
	LOCAL	c1b, c2a, c2b, c3a, c3b, c4a, c4b, c5a, c5b, c6a, c6b
	LOCAL	ypass2_12288_real_1, ypass2_12288_real_2
	LOCAL	ypass2_12288_real_3, ypass2_12288_real_4
	LOCAL	ypass2_12288_real_unfft, ypass2_12288_real_done

;; Type 4 FFTs skip the forward FFT process

	start_timer 2
	mov	rsi, DATA_ADDR		;; Load source address
	mov	rbx, DIST_TO_FFTSRCARG
	mov	rcx, DATA_PREFETCH	;; Prefetch pointer (we need to prefetch 
					;; 3072 cache lines + 47 pad cache lines)
	cmp	ffttype, 4
	je	ypass2_12288_real_4

;; Do FFT levels 1,2,3 which correspond to complex levels 1,2
;; On input the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	12288	+1	+1	+1
;;	4	...
;;	...
;;	12284
;; On output the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	4096	+1	+1	+1
;;	4	...
;;	...
;;	4092	...
;;	8192	...
;;	...

;; Do 4096 six_reals_fft operations
;;	distance between fft data elements is 4096
;;	do 1024 macros each processing 24 data values

	mov	rdi, sincos3		;; Real sin/cos pointer
	mov	rbp, xsincos_complex	;; Complex sin/cos data
	yloop_init 1024			;; 1024 iterations
	yloop_set_prefetch YHALF
	yr3_f3cl_2sc_six_reals_fft_preload
b1b:	yr3_f3cl_2sc_six_reals_fft rsi, 64, 16*dist64, rbp, YMM_SCD1, rdi, YMM_SCD1, 64
	yloop	64, b1b, -64*64+dist64	;; Test loop counter, next source pointer
	yloop	16, b1b, -16*dist64	;; Test loop counter, restore source pointer

;; Do FFT levels 3,4
;; On input the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	4096	+1	+1	+1
;;	4	...
;;	...
;;	4092	...
;;	8192	...
;;	...
;; On output the 64-byte cache lines are swizzled to hold these data values:
;;	0	+2K	+2K	+2K	1024	+2K	+2K	+2K
;;	4	...
;;	...
;;	1020	...
;;	1	...
;;	...
;;	8192	...
;;	...

;; Do 1024 eight_reals_fft operations
;; Do 2048 four_complex_fft operations
;;	distance between fft data elements is 1024
;;	do 768 macros each processing 32 data values

	yloop_init 768			;; 3 iterations of 256
	yloop_set_prefetch YHALF
	yr4_s4cl_2sc_eight_reals_fft_preload
b2a:	yr4_s4cl_2sc_eight_reals_fft rsi, 64, 4*dist64, 8*dist64, rbp, YMM_SCD2, rdi, YMM_SCD2, 64
	yloop	64, b2a, -64*64+dist64	;; Test loop counter, next source pointer
	yloop	4, b2a, -4*dist64+16*dist64, rbp, -256*YMM_SCD2 ;; Test loop counter, next source pointer, restore sine/cosine pointer

	loops_reset			;; 2 more iters of 256
	yr4_s4cl_four_complex_djbfft_preload
b2b:	yr4_s4cl_four_complex_djbfft rsi, 64, 4*dist64, 8*dist64, rbp, YMM_SCD2, 64
	yloop	64, b2b, -64*64+dist64	;; Test loop counter, next source pointer
	yloop	4, b2b, -4*dist64+16*dist64, rbp, -256*YMM_SCD2 ;; Test loop counter, next source pointer, restore sin/cos ptr
	yloop	3, b2b, -3*16*dist64	;; Loop 3 times, restore source pointer

;; Do FFT levels 5,6
;; On input the 64-byte cache lines hold these data values:
;;	0	+2K	+2K	+2K	1024	+2K	+2K	+2K
;;	4	...
;;	...
;;	1020	...
;;	1	...
;;	...
;;	8192	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+2K	+2K	+2K	256	+2K	+2K	+2K
;;	4	...
;;	...
;;	252	...
;;	512	...
;;	...
;;	1	...
;;	...
;;	8192	...
;;	...

;; Do 256 eight_reals_fft operations
;; Do 2816 four_complex_fft operations
;;	distance between fft data elements is 256
;;	do 768 macros each processing 32 data values

	yloop_init 768			;; 3 iterations of 256
	yloop_set_prefetch YHALF
	yr4_4cl_eight_reals_four_complex_djbfft_preload
b3a:	yr4_4cl_eight_reals_four_complex_djbfft rsi, 4*dist64, dist64, 2*dist64, rdi, YMM_SCD3, 4
	yloop	4, b3a, -4*4*dist64+64	;; Test loop counter, next source pointer
	yloop	64, b3a, -64*64+16*dist64, rdi, -256*YMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer

	loops_reset			;; 2 more iters of 256
	yr4_rb4cl_four_complex_djbfft_preload
b3b:	yr4_rb4cl_four_complex_djbfft rsi, 4*dist64, dist64, 2*dist64, rdi, YMM_SCD3, 4
	yloop	4, b3b, -4*4*dist64+64	;; Test loop counter, next source pointer
	yloop	64, b3b, -64*64+16*dist64, rdi, -256*YMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer
	yloop	3, b3b, -3*16*dist64	;; Test loop counter, restore source pointer

;; Do FFT levels 7,8
;; On input the 64-byte cache lines hold these data values:
;;	0	+2K	+2K	+2K	256	+2K	+2K	+2K
;;	4	...
;;	...
;;	252	...
;;	512	...
;;	...
;;	1	...
;;	...
;;	8192	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+2K	+2K	+2K	64	+2K	+2K	+2K
;;	4	...
;;	...
;;	60	...
;;	128	...
;;	...
;;	1	...
;;	...
;;	8192	...
;;	...

;; Do 64 eight_reals_fft operations
;; Do 3008 four_complex_fft operations
;;	distance between fft data elements is 64
;;	do 768 macros each processing 32 data values

	yloop_init 768			;; 12 iterations of 64
	yloop_set_prefetch YHALF
	yr4_4cl_eight_reals_four_complex_djbfft_preload
b4a:	yr4_4cl_eight_reals_four_complex_djbfft rsi, 4*dist64, 16*64, 32*64, rdi, 4*YMM_SCD3, 4
	yloop	4, b4a, -4*4*dist64+64	;; Test loop counter, next source pointer
	yloop	16, b4a, -16*64+dist64, rdi, -64*4*YMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer

	loops_reset			;; 11 more iters of 64
	yr4_rb4cl_four_complex_djbfft_preload
b4b:	yr4_rb4cl_four_complex_djbfft rsi, 4*dist64, 16*64, 32*64, rdi, 4*YMM_SCD3, 4
	yloop	4, b4b, -4*4*dist64+64	;; Test loop counter, next source pointer
	yloop	16, b4b, -16*64+dist64, rdi, -64*4*YMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer
	yloop	4, b4b, -4*dist64+16*dist64 ;; Test loop counter, next source pointer
	yloop	3, b4b, -3*16*dist64	;; Test loop counter, restore source pointer

;; Do FFT levels 9,10
;; On input the 64-byte cache lines hold these data values:
;;	0	+2K	+2K	+2K	64	+2K	+2K	+2K
;;	4	...
;;	...
;;	60	...
;;	128	...
;;	...
;;	1	...
;;	...
;;	8192	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+2K	+2K	+2K	16	+2K	+2K	+2K
;;	4	...
;;	...
;;	12	...
;;	32	...
;;	...
;;	1	...
;;	...
;;	8192	...
;;	...

;; Do 16 eight_reals_fft operations
;; Do 3056 four_complex_fft operations
;;	distance between fft data elements is 16
;;	do 768 macros each processing 32 data values

	yloop_init 768			;; 48 iterations of 16
	yloop_set_prefetch YQUARTER
	yr4_4cl_eight_reals_four_complex_djbfft_preload
b5a:	yr4_4cl_eight_reals_four_complex_djbfft rsi, 4*dist64, 4*64, 8*64, rdi, 16*YMM_SCD3, 4
	yloop	4, b5a, -4*4*dist64+64	;; Test loop counter, next source pointer
	yloop	4, b5a, -4*64+16*64, rdi, -16*16*YMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer

	loops_reset			;; 47 more iters of 16
	yr4_rb4cl_four_complex_djbfft_preload
b5b:	yr4_rb4cl_four_complex_djbfft rsi, 4*dist64, 4*64, 8*64, rdi, 16*YMM_SCD3, 4
	yloop	4, b5b, -4*4*dist64+64	;; Test loop counter, next source pointer
	yloop	4, b5b, -4*64+16*64, rdi, -16*16*YMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer
	yloop	4, b5b, -4*16*64+dist64	;; Test loop counter, next source pointer
	yloop	4, b5b, -4*dist64+16*dist64 ;; Test loop counter, next source pointer
	yloop	3, b5b, -3*16*dist64	;; Test loop counter, restore source pointer

;; Do FFT levels 11,12
;; On input the 64-byte cache lines hold these data values:
;;	0	+2K	+2K	+2K	16	+2K	+2K	+2K
;;	4	...
;;	...
;;	12	...
;;	32	...
;;	...
;;	1	...
;;	...
;;	8192	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+2K	+2K	+2K	4	+2K	+2K	+2K
;;	8	...
;;	...
;;	1	...
;;	...
;;	8192	...
;;	...

;; Do 4 eight_reals_fft operations
;; Do 3068 four_complex_fft operations
;;	distance between fft data elements is 4
;;	do 768 macros each processing 32 data values

	yloop_init 768			;; 192 iterations of 4
	yloop_set_prefetch YQUARTER
	yr4_4cl_eight_reals_four_complex_djbfft_preload
b6a:	yr4_4cl_eight_reals_four_complex_djbfft rsi, 4*dist64, 64, 2*64, rdi, 64*YMM_SCD3, 4
	yloop	4, b6a, -4*4*dist64+4*64, rdi, -4*64*YMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer

	loops_reset			;; 191 more iters of 4
	yr4_rb4cl_four_complex_djbfft_preload
b6b:	yr4_rb4cl_four_complex_djbfft rsi, 4*dist64, 64, 2*64, rdi, 64*YMM_SCD3, 4
	yloop	4, b6b, -4*4*dist64+4*64, rdi, -4*64*YMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer
	yloop	16, b6b, -16*4*64+dist64 ;; Test loop counter, next source pointer
	yloop	4, b6b, -4*dist64+16*dist64 ;; Test loop counter, next source pointer
	yloop	3, b6b, -3*16*dist64	;; Test loop counter, restore source pointer

;; Do FFT levels 13,14 as well as inverse FFT levels 13,14
;; On input and output the 64-byte cache lines hold these data values:
;;	0	+2K	+2K	+2K	4	+2K	+2K	+2K
;;	8	...
;;	...
;;	1	...
;;	...
;;	8192	...
;;	...

;; Do 1 eight_reals_fft operation
;; Do 3071 four_complex_fft operations
;;	distance between fft data elements is 1
;;	do 768 macros each processing 32 data values

	cmp	ffttype, 2
	jl	ypass2_12288_real_1
	jg	ypass2_12288_real_3

ypass2_12288_real_2:
;;	yr4_4cl_eight_reals_four_complex_with_square rsi, 64, 4*dist64, 8*dist64
	yr4_dispatch call, yreal_square, 4*dist64, 8*dist64
	yloop_init 768			;; 767 more iterations
	yloop_adjust 1
	yloop_set_prefetch YQUARTER
	yr4_4cl_four_complex_with_square_preload
b7b:	yr4_4cl_four_complex_with_square rsi, 64, 4*dist64, 8*dist64, 1
	yloop	64, b7b, -64*64+dist64	;; Test loop counter, next source pointer
	yloop	4, b7b, -4*dist64+16*dist64	;; Test loop counter, next source pointer
	yloop	3, b7b, -3*16*dist64	;; Test loop counter, restore source pointer
	jmp	ypass2_12288_real_unfft

ypass2_12288_real_1:
;;	yr4_4cl_eight_reals_four_complex_fft_final rsi, 64, 4*dist64, 8*dist64
	yr4_dispatch call, yreal_fft_final, 4*dist64, 8*dist64
	mov	rcx, xsincos_complex	;; Prefetch complex sin/cos data, 1568 cache lines
	yloop_init 768			;; 767 more iterations
	yloop_adjust 1
	yloop_set_prefetch YDOUBLE
b7a:	yr4_4cl_four_complex_fft_final rsi, 64, 4*dist64, 8*dist64
	yloop	64, b7a, -64*64+dist64	;; Test loop counter, next source pointer
	yloop	4, b7a, -4*dist64+16*dist64	;; Test loop counter, next source pointer
	yloop	3, b7a			;; Test loop counter
	jmp	ypass2_12288_real_done

ypass2_12288_real_3:
x64	mov	r8, DEST2ARG
	mov	rbp, DIST_TO_MULSRCARG
;;	yr4_4cl_eight_reals_four_complex_with_mult rsi, 64, 4*dist64, 8*dist64
	yr4_dispatch call, yreal_mult, 4*dist64, 8*dist64
	yloop_init 768			;; 767 more iterations
	yloop_adjust 1
	yloop_set_prefetch YQUARTER
	yr4_4cl_four_complex_with_mult_preload
b7c:	yr4_4cl_four_complex_with_mult rsi, 64, 4*dist64, 8*dist64
	yloop	64, b7c, -64*64+dist64	;; Test loop counter, next source pointer
	yloop	4, b7c, -4*dist64+16*dist64	;; Test loop counter, next source pointer
	yloop	3, b7c, -3*16*dist64	;; Test loop counter, restore source pointer
x64	add	rsi, r8
	jmp	ypass2_12288_real_unfft

ypass2_12288_real_4:
	mov	rbp, DIST_TO_MULSRCARG
;;	yr4_4cl_eight_reals_four_complex_with_mulf rsi, 64, 4*dist64, 8*dist64
	yr4_dispatch call, yreal_mulf, 4*dist64, 8*dist64
	yloop_init 768			;; 767 more iterations
	yloop_adjust 1
	yloop_set_prefetch YDOUBLE
	yr4_4cl_four_complex_with_mulf_preload
b7d:	yr4_4cl_four_complex_with_mulf rsi, 64, 4*dist64, 8*dist64
	yloop	64, b7d, -64*64+dist64	;; Test loop counter, next source pointer
	yloop	4, b7d, -4*dist64+16*dist64	;; Test loop counter, next source pointer
	yloop	3, b7d, -3*16*dist64	;; Test loop counter, restore source pointer
;;	bump	rcx, (-1536+512+3*384+3*192)*64 ;; Move prefetch pointer back to where squaring would be

;; Do inverse FFT levels 11,12
;; On input the 64-byte cache lines hold these data values:
;;	0	+2K	+2K	+2K	4	+2K	+2K	+2K
;;	8	...
;;	...
;;	1	...
;;	...
;;	8192	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+2K	+2K	+2K	16	+2K	+2K	+2K
;;	4	...
;;	...
;;	12	...
;;	32	...
;;	...
;;	1	...
;;	...
;;	8192	...
;;	...

;; Do 4 eight_reals_unfft operations
;; Do 3068 four_complex_unfft operations
;;	distance between fft data elements is 4
;;	do 768 macros each processing 32 data values

ypass2_12288_real_unfft:
	mov	rdi, sincos3		;; Load sin/cos pointer
	bump	rdi, 256*YMM_SCD2+1024*YMM_SCD1
	yloop_init 768			;; 192 iterations of 4
	yloop_set_prefetch YQUARTER
	yr4_4cl_eight_reals_four_complex_djbunfft_preload
c6a:	yr4_4cl_eight_reals_four_complex_djbunfft rsi, 4*dist64, 64, 2*64, rdi, 64*YMM_SCD3, 4
	yloop	4, c6a, -4*4*dist64+4*64, rdi, -4*64*YMM_SCD3 ;; Test loop counter, next source pointer, restore sine/cosine pointer

	loops_reset			;; 191 more iters of 4
	yr4_rb4cl_four_complex_djbunfft_preload
c6b:	yr4_rb4cl_four_complex_djbunfft rsi, 4*dist64, 64, 2*64, rdi, 64*YMM_SCD3, 4
	yloop	4, c6b, -4*4*dist64+4*64, rdi, -4*64*YMM_SCD3 ;; Test loop counter, next source pointer, restore sine/cosine pointer
	yloop	16, c6b, -16*4*64+dist64	;; Test loop counter, next source pointer
	yloop	4, c6b, -4*dist64+16*dist64	;; Test loop counter, next source pointer
	yloop	3, c6b, -3*16*dist64	;; Test loop counter, restore source pointer

;; Do inverse FFT levels 9,10
;; On input the 64-byte cache lines hold these data values:
;;	0	+2K	+2K	+2K	16	+2K	+2K	+2K
;;	4	...
;;	...
;;	12	...
;;	32	...
;;	...
;;	1	...
;;	...
;;	8192	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+2K	+2K	+2K	64	+2K	+2K	+2K
;;	4	...
;;	...
;;	60	...
;;	128	...
;;	...
;;	1	...
;;	...
;;	8192	...
;;	...

;; Do 16 eight_reals_unfft operations
;; Do 3056 four_complex_unfft operations
;;	distance between fft data elements is 16
;;	do 768 macros each processing 32 data values

	yloop_init 768			;; 48 iterations of 16
	yloop_set_prefetch YHALF
	yr4_4cl_eight_reals_four_complex_djbunfft_preload
c5a:	yr4_4cl_eight_reals_four_complex_djbunfft rsi, 4*dist64, 4*64, 8*64, rdi, 16*YMM_SCD3, 4
	yloop	4, c5a, -4*4*dist64+64	;; Test loop counter, next source pointer
	yloop	4, c5a, -4*64+16*64, rdi, -16*16*YMM_SCD3 ;; Test loop counter, next source pointer, restore sine/cosine pointer

	loops_reset			;; 47 more iters of 16
	yr4_rb4cl_four_complex_djbunfft_preload
c5b:	yr4_rb4cl_four_complex_djbunfft rsi, 4*dist64, 4*64, 8*64, rdi, 16*YMM_SCD3, 4
	yloop	4, c5b, -4*4*dist64+64	;; Test loop counter, next source pointer
	yloop	4, c5b, -4*64+16*64, rdi, -16*16*YMM_SCD3 ;; Test loop counter, next source pointer, restore sine/cosine pointer
	yloop	4, c5b, -4*16*64+dist64	;; Test loop counter, next source pointer
	yloop	4, c5b, -4*dist64+16*dist64	;; Test loop counter, next source pointer
	yloop	3, c5b, -3*16*dist64	;; Test loop counter, restore source pointer

;; Do inverse FFT levels 7,8
;; On input the 64-byte cache lines hold these data values:
;;	0	+2K	+2K	+2K	64	+2K	+2K	+2K
;;	4	...
;;	...
;;	60	...
;;	128	...
;;	...
;;	1	...
;;	...
;;	8192	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+2K	+2K	+2K	256	+2K	+2K	+2K
;;	4	...
;;	...
;;	252	...
;;	512	...
;;	...
;;	1	...
;;	...
;;	8192	...
;;	...

;; Do 64 eight_reals_unfft operations
;; Do 3008 four_complex_unfft operations
;;	distance between fft data elements is 64
;;	do 768 macros each processing 32 data values

	bump	rcx, (3072+47-512-3*384-4*192-384-384)*64 ;; Move prefetch pointer along
	yloop_init 768			;; 12 iterations of 64
	yloop_set_prefetch YHALF
	yr4_4cl_eight_reals_four_complex_djbunfft_preload
c4a:	yr4_4cl_eight_reals_four_complex_djbunfft rsi, 4*dist64, 16*64, 32*64, rdi, 4*YMM_SCD3, 4
	yloop	4, c4a, -4*4*dist64+64	;; Test loop counter, next source pointer
	yloop	16, c4a, -16*64+dist64, rdi, -64*4*YMM_SCD3 ;; Test loop counter, next source pointer, restore sine/cosine pointer

	loops_reset			;; 11 more iters of 64
	yr4_rb4cl_four_complex_djbunfft_preload
c4b:	yr4_rb4cl_four_complex_djbunfft rsi, 4*dist64, 16*64, 32*64, rdi, 4*YMM_SCD3, 4
	yloop	4, c4b, -4*4*dist64+64	;; Test loop counter, next source pointer
	yloop	16, c4b, -16*64+dist64, rdi, -64*4*YMM_SCD3 ;; Test loop counter, next source pointer, restore sine/cosine pointer
	yloop	4, c4b, -4*dist64+16*dist64	;; Test loop counter, next source pointer
	yloop	3, c4b, -3*16*dist64	;; Test loop counter, restore source pointer

;; Do inverse FFT levels 5,6
;; On input the 64-byte cache lines hold these data values:
;;	0	+2K	+2K	+2K	256	+2K	+2K	+2K
;;	4	...
;;	...
;;	252	...
;;	512	...
;;	...
;;	1	...
;;	...
;;	8192	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+2K	+2K	+2K	1024	+2K	+2K	+2K
;;	4	...
;;	...
;;	1020	...
;;	1	...
;;	...
;;	8192	...
;;	...

;; Do 256 eight_reals_unfft operations
;; Do 2816 four_complex_unfft operations
;;	distance between fft data elements is 256
;;	do 768 macros each processing 32 data values

	mov	rcx, xsincos_complex	;; Prefetch complex sin/cos data, 1568 cache lines
	yloop_init 768			;; 3 iterations of 256
	yloop_set_prefetch YSINGLE
	yr4_4cl_eight_reals_four_complex_djbunfft_preload
c3a:	yr4_4cl_eight_reals_four_complex_djbunfft rsi, 4*dist64, dist64, 2*dist64, rdi, YMM_SCD3, 4
	yloop	4, c3a, -4*4*dist64+64	;; Test loop counter, next source pointer
	yloop	64, c3a, -64*64+16*dist64, rdi, -256*YMM_SCD3 ;; Test loop counter, next source pointer, restore sine/cosine pointer

	loops_reset			;; 2 more iters of 256
	yr4_rb4cl_four_complex_djbunfft_preload
c3b:	yr4_rb4cl_four_complex_djbunfft rsi, 4*dist64, dist64, 2*dist64, rdi, YMM_SCD3, 4
	yloop	4, c3b, -4*4*dist64+64	;; Test loop counter, next source pointer
	yloop	64, c3b, -64*64+16*dist64, rdi, -256*YMM_SCD3 ;; Test loop counter, next source pointer, restore sine/cosine pointer
	yloop	3, c3b, -3*16*dist64	;; Test loop counter, restore source pointer

;; Do inverse FFT levels 3,4
;; On input the 64-byte cache lines hold these data values:
;;	0	+2K	+2K	+2K	1024	+2K	+2K	+2K
;;	4	...
;;	...
;;	1020	...
;;	1	...
;;	...
;;	8192	...
;;	...
;; On output the 64-byte cache lines are swizzled to hold these data values:
;;	0	+1	+1	+1	4096	+1	+1	+1
;;	4	...
;;	...
;;	4092	...
;;	8192	...
;;	...

;; Do 1024 eight_reals_unfft operations
;; Do 2048 four_complex_unfft operations
;;	distance between fft data elements is 1024
;;	do 768 macros each processing 32 data values

	bump	rdi, -256*YMM_SCD2	;; Real sin/cos ptr
	mov	rbp, xsincos_complex	;; Complex sin/cos data
	bump	rbp, 1024*YMM_SCD1
	yloop_init 768			;; 3 iterations of 256
	yloop_set_prefetch YHALF
	yr4_s4cl_2sc_eight_reals_unfft_preload
c2a:	yr4_s4cl_2sc_eight_reals_unfft rsi, 64, 4*dist64, 8*dist64, rbp, YMM_SCD2, rdi, YMM_SCD2, 64
	yloop	64, c2a, -64*64+dist64	;; Test loop counter, next source pointer
	yloop	4, c2a, -4*dist64+16*dist64, rbp, -256*YMM_SCD2 ;; Test loop counter, next source pointer, restore sine/cosine pointer

	loops_reset			;; 2 more iters of 256
	yr4_s4cl_four_complex_djbunfft_preload
c2b:	yr4_s4cl_four_complex_djbunfft rsi, 64, 4*dist64, 8*dist64, rbp, YMM_SCD2, 64
	yloop	64, c2b, -64*64+dist64	;; Test loop counter, next source pointer
	yloop	4, c2b, -4*dist64+16*dist64, rbp, -256*YMM_SCD2 ;; Test loop counter, next source pointer, restore sine/cosine pointer
	yloop	3, c2b, -3*16*dist64	;; Test loop counter, restore source pointer

;; Do inverse FFT levels 1,2,3 which correspond to complex levels 1,2
;; On input the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	4096	+1	+1	+1
;;	4	...
;;	...
;;	4092	...
;;	8192	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	12288	+1	+1	+1
;;	4	...
;;	...
;;	12284

;; Do 4096 six_reals_unfft operations
;;	distance between fft data elements is 4096
;;	do 1024 macros each processing 24 data values

	bump	rdi, -256*YMM_SCD2-1024*YMM_SCD1 ;; Real sin/cos ptr
	bump	rbp, -1024*YMM_SCD1	;; Complex sin/cos data
	bump	rcx, (1568-768-384-512)*64 ;; Move prefetch pointer along
	yloop_init 1024			;; 1024 iterations
	yloop_set_prefetch YHALF
	yr3_3cl_2sc_six_reals_unfft_preload
c1b:	yr3_3cl_2sc_six_reals_unfft rsi, 64, 16*dist64, rbp, YMM_SCD1, rdi, YMM_SCD1, 64
	yloop	64, c1b, -64*64+dist64	;; Test loop counter, next source pointer
	yloop	16, c1b			;; Test loop counter

ypass2_12288_real_done:
	end_timer 2
	ENDM

; *************** radix4-pass2-12288-complex macro ******************
; This macro takes 12288 complex values and performs the final 13.585 levels of a standard
; radix-4 FFT, squares the results, and does the first 13.585 levels of the inverse FFT.

yr4dwpn_pass2_12288_complex MACRO
	LOCAL	b1a, b1b, b2b, b3b, b4a0, b4a, b4b0, b4b, b4c0, b4c, b5a, b5b, b5c, b6a, b6b, b6c, b7a, b7b, b7c, b7d
	LOCAL	c1b, c2b, c3a, c3b, c4b, c4c, c4d, c5b, c5c, c5d, c6b, c6c, c6d
	LOCAL	ypass2_12288_complex_1, ypass2_12288_complex_2
	LOCAL	ypass2_12288_complex_3, ypass2_12288_complex_4
	LOCAL	ypass2_12288_complex_unfft, ypass2_12288_complex_done

;; Type 4 FFTs skip the forward FFT process

	mov	rsi, DATA_ADDR		;; Load address of FFT data
	mov	rbx, DIST_TO_FFTSRCARG
	mov	rcx, DATA_PREFETCH	;; Prefetch pointer (we need to prefetch 
					;; 3072 cache lines + 47 pad cache lines)
	cmp	ffttype, 4
	je	ypass2_12288_complex_4

;; Do FFT level 1,2
;; On input the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	12288	+1	+1	+1
;;	4	...
;;	...
;;	12284
;; On output the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	4096	+1	+1	+1
;;	4	...
;;	...
;;	4092	...
;;	8192	...
;;	...

;; Do 4096 three_complex_fft operations
;;	distance between fft data elements is 4096
;;	do 1024 macros each processing 24 data values

;; Do FFT levels 3,4
;; On input the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	4096	+1	+1	+1
;;	4	...
;;	...
;;	4092	...
;;	8192	...
;;	...
;; On output the 64-byte cache lines are swizzled to hold these data values:
;;	0	+2K	+2K	+2K	1024	+2K	+2K	+2K
;;	4	...
;;	...
;;	1020	...
;;	1	...
;;	...
;;	8192	...
;;	...

;; Do 3072 four_complex_fft operations
;;	distance between fft data elements is 1024
;;	do 768 macros each processing 32 data values

;; Do FFT levels 5,6
;; On input the 64-byte cache lines hold these data values:
;;	0	+2K	+2K	+2K	1024	+2K	+2K	+2K
;;	4	...
;;	...
;;	1020	...
;;	1	...
;;	...
;;	8192	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+2K	+2K	+2K	256	+2K	+2K	+2K
;;	4	...
;;	...
;;	252	...
;;	512	...
;;	...
;;	1	...
;;	...
;;	8192	...
;;	...

;; Do 3072 four_complex_fft operations
;;	distance between fft data elements is 256
;;	do 768 macros each processing 32 data values

	mov	rdi, xsincos_complex	;; The sin/cos ptrs
	lea	rbp, [rdi+1024*YMM_SCD1]
	mov	rdx, rbp
	yloop_init 1024			;; 1024 iterations or 768 iterations

b1a:	start_timer 3
	yloop_set_prefetch YQUARTER	;; Prefetch 256 cache lines in this loop
	yr3_f3cl_three_complex_djbfft_preload
b1b:	yr3_f3cl_three_complex_djbfft rsi, dist64, 16*dist64, rdi, 64*YMM_SCD1, 16
	yloop	16, b1b, -16*dist64, rdi, -16*64*YMM_SCD1+YMM_SCD1 ;; Test loop counter, restore source pointer, next sin/cos data
	loops_undo 16			;; Back up loop counter
	end_timer 3

	start_timer 4
	yloop_adjust 4			;; Four fewer iterations this loop
	yloop_set_prefetch YQUARTER	;; Prefetch 192 cache lines in this loop
	yr4_s4cl_four_complex_djbfft_preload
b2b:	yr4_s4cl_four_complex_djbfft rsi, dist64, 4*dist64, 8*dist64, rbp, 64*YMM_SCD2, 4
	yloop	4, b2b, -4*dist64+16*dist64, rbp, -4*64*YMM_SCD2 ;; Test loop counter, next source pointer, restore sin/cos data
	yloop	4, b2b, -3*16*dist64, rbp, YMM_SCD2 ;; 3 iterations!!  Restore source pointer, next sin/cos data
	loops_undo 4*4			;; Back up loop counter
	end_timer 4

	start_timer 5
	yloop_adjust 4			;; Four fewer iterations this loop
	yloop_set_prefetch YQUARTER	;; Prefetch 192 cache lines in this loop
	yr4_eb4cl_four_complex_djbfft_preload
b3b:	yr4_eb4cl_four_complex_djbfft rsi, 4*dist64, dist64, 2*dist64, rdx, YMM_SCD2, 4, L1PREFETCH_ALL, 64
	yloop	4, b3b, 0, rdx, -4*YMM_SCD2 ;; Loop 4 times, restore sine/cosine pointer
	yloop	4, b3b, -3*4*4*dist64+64, rdx, 4*YMM_SCD2 ;; 3 iterations!!  Next source pointer, next sine/cosine pointer
	end_timer 5

	yloop	64, b1a, -64*64, rdi, -64*YMM_SCD1+1024*YMM_SCD1+256*YMM_SCD2 ;; Test loop counter, restore source pointer, next sin/cos ptr

;; Do FFT levels 7,8
;; On input the 64-byte cache lines hold these data values:
;;	0	+2K	+2K	+2K	256	+2K	+2K	+2K
;;	4	...
;;	...
;;	252	...
;;	512	...
;;	...
;;	1	...
;;	...
;;	8192	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+2K	+2K	+2K	64	+2K	+2K	+2K
;;	4	...
;;	...
;;	60	...
;;	128	...
;;	...
;;	1	...
;;	...
;;	8192	...
;;	...

;; Do 3072 four_complex_fft operations
;;	distance between fft data elements is 64
;;	do 768 macros each processing 32 data values

;; Do FFT levels 9,10
;; On input the 64-byte cache lines hold these data values:
;;	0	+2K	+2K	+2K	64	+2K	+2K	+2K
;;	4	...
;;	...
;;	60	...
;;	128	...
;;	...
;;	1	...
;;	...
;;	8192	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+2K	+2K	+2K	16	+2K	+2K	+2K
;;	4	...
;;	...
;;	12	...
;;	32	...
;;	...
;;	1	...
;;	...
;;	8192	...
;;	...

;; Do 3072 four_complex_fft operations
;;	distance between fft data elements is 16
;;	do 768 macros each processing 32 data values

;; Do FFT levels 11,12
;; On input the 64-byte cache lines hold these data values:
;;	0	+2K	+2K	+2K	16	+2K	+2K	+2K
;;	4	...
;;	...
;;	12	...
;;	32	...
;;	...
;;	1	...
;;	...
;;	8192	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+2K	+2K	+2K	4	+2K	+2K	+2K
;;	8	...
;;	...
;;	1	...
;;	...
;;	8192	...
;;	...

;; Do 3072 four_complex_fft operations
;;	distance between fft data elements is 4
;;	do 768 macros each processing 32 data values

;; Do FFT levels 13,14 as well as inverse FFT levels 13,14
;; On input and output the 64-byte cache lines hold these data values:
;;	0	+2K	+2K	+2K	4	+2K	+2K	+2K
;;	8	...
;;	...
;;	1	...
;;	...
;;	8192	...
;;	...

;; Do 3072 four_complex_fft operations
;;	distance between fft data elements is 1
;;	do 768 macros each processing 32 data values

;; Do inverse FFT levels 11,12
;; On input the 64-byte cache lines hold these data values:
;;	0	+2K	+2K	+2K	4	+2K	+2K	+2K
;;	8	...
;;	...
;;	1	...
;;	...
;;	8192	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+2K	+2K	+2K	16	+2K	+2K	+2K
;;	4	...
;;	...
;;	12	...
;;	32	...
;;	...
;;	1	...
;;	...
;;	8192	...
;;	...

;; Do 3072 four_complex_unfft operations
;;	distance between fft data elements is 4
;;	do 768 macros each processing 32 data values

;; Do inverse FFT levels 9,10
;; On input the 64-byte cache lines hold these data values:
;;	0	+2K	+2K	+2K	16	+2K	+2K	+2K
;;	4	...
;;	...
;;	12	...
;;	32	...
;;	...
;;	1	...
;;	...
;;	8192	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+2K	+2K	+2K	64	+2K	+2K	+2K
;;	4	...
;;	...
;;	60	...
;;	128	...
;;	...
;;	1	...
;;	...
;;	8192	...
;;	...

;; Do 3072 four_complex_unfft operations
;;	distance between fft data elements is 16
;;	do 768 macros each processing 32 data values

;; Do inverse FFT levels 7,8
;; On input the 64-byte cache lines hold these data values:
;;	0	+2K	+2K	+2K	64	+2K	+2K	+2K
;;	4	...
;;	...
;;	60	...
;;	128	...
;;	...
;;	1	...
;;	...
;;	8192	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+2K	+2K	+2K	256	+2K	+2K	+2K
;;	4	...
;;	...
;;	252	...
;;	512	...
;;	...
;;	1	...
;;	...
;;	8192	...
;;	...

;; Do 3072 four_complex_unfft operations
;;	distance between fft data elements is 64
;;	do 768 macros each processing 32 data values

;; Do this 256 cache lines (16KB) at a time to maximize accessing data
;; while it is in the L1 cache.

;; Execute the right middle step

	cmp	ffttype, 2
	jl	ypass2_12288_complex_1
	jg	ypass2_12288_complex_3

ypass2_12288_complex_2:
	yloop_init 768			;; 12 iters of 256 cache lines

b4b0:	start_timer 6
	yloop_set_prefetch YHALF	;; Prefetch 384 cache lines in this loop
	yr4_b4cl_four_complex_djbfft_preload
b4b:	yr4_b4cl_four_complex_djbfft rsi, 64, 16*64, 32*64, rdi, YMM_SCD2/4, 16
	yloop	16, b4b, -16*64+4*dist64 ;; Loop 16 times, next source pointer
	yloop	4, b4b, -4*4*dist64, rdi, -64*YMM_SCD2/4 ;; Loop 4 times, restore source pointer, restore sine/cosine pointer
	loops_undo 4*16			;; Back up loop counter
	end_timer 6

	start_timer 7
	yloop_set_prefetch YQUARTER	;; Prefetch 192 cache lines in this loop
;;	yr4_b4cl_four_complex_djbfft_preload
b5b:	yr4_b4cl_four_complex_djbfft rsi, 64, 4*64, 8*64, rdi, 4*YMM_SCD2/4, 4
	yloop	4, b5b, -4*64+16*64, rdi, -4*4*YMM_SCD2/4 ;; Loop 4 times, next source pointer, restore sine/cosine pointer
	yloop	4, b5b, -4*16*64+4*dist64, rdi, YMM_SCD2/4 ;; Loop 4 times, next source pointer, next sine/cosine pointer
	yloop	4, b5b, -4*4*dist64, rdi, -4*YMM_SCD2/4 ;; Loop 4 times, restore source pointer, next sine/cosine pointer
	loops_undo 4*4*4		;; Back up loop counter
	end_timer 7

	start_timer 8
	yloop_set_prefetch YQUARTER	;; Prefetch 192 cache lines in this loop
;;	yr4_b4cl_four_complex_djbfft_preload
b6b:	yr4_b4cl_four_complex_djbfft rsi, 4*64, 64, 2*64, rdi, 0, 16
	yloop	16, b6b, -16*4*64+4*dist64, rdi, 4*YMM_SCD2/4 ;; Loop 16 times, next source pointer, next sine/cosine pointer
	yloop	4, b6b, -4*4*dist64, rdi, -4*4*YMM_SCD2/4 ;; Loop 4 times, restore source pointer, restore sine/cosine pointer
	loops_undo 4*16			;; Back up loop counter
	end_timer 8

	start_timer 9
	yloop_set_prefetch YQUARTER	;; Prefetch 192 cache lines in this loop
	yr4_4cl_four_complex_with_square_preload
b7b:	yr4_4cl_four_complex_with_square rsi, 64, 4*dist64, 8*dist64, 64
	yloop	64, b7b, -64*64		;; Loop 64 times, restore source pointer
	loops_undo 64			;; Back up loop counter
	end_timer 9

	start_timer 10
	yloop_set_prefetch YQUARTER	;; Prefetch 192 cache lines in this loop
	yr4_b4cl_four_complex_djbunfft_preload
c6b:	yr4_b4cl_four_complex_djbunfft rsi, 4*64, 64, 2*64, rdi, 0, 16
	yloop	16, c6b, -16*4*64+4*dist64, rdi, 4*YMM_SCD2/4 ;; Loop 16 times, next source pointer, next sine/cosine pointer
	yloop	4, c6b, -4*4*dist64, rdi, -4*4*YMM_SCD2/4 ;; Loop 4 times, restore source pointer, restore sine/cosine pointer
	loops_undo 4*16			;; Restore loop counter
	end_timer 10

	start_timer 11
	yloop_set_prefetch YHALF	;; Prefetch 384 cache lines in this loop
;;	yr4_b4cl_four_complex_djbunfft_preload
c5b:	yr4_b4cl_four_complex_djbunfft rsi, 64, 4*64, 8*64, rdi, 4*YMM_SCD2/4, 4
	yloop	4, c5b, -4*64+16*64, rdi, -4*4*YMM_SCD2/4 ;; Loop 4 times, next source pointer, restore sine/cosine pointer
	yloop	4, c5b, -4*16*64+4*dist64, rdi, YMM_SCD2/4 ;; Loop 4 times, next source pointer, next sine/cosine pointer
	yloop	4, c5b, -4*4*dist64, rdi, -4*YMM_SCD2/4 ;; Loop 4 times, restore source pointer, next sine/cosine pointer
	loops_undo 4*4*4		;; Back up loop counter
	end_timer 11

	start_timer 12
	yloop_set_prefetch YHALF	;; Prefetch 384 cache lines in this loop
;;	yr4_b4cl_four_complex_djbunfft_preload
c4b:	yr4_b4cl_four_complex_djbunfft rsi, 64, 16*64, 32*64, rdi, YMM_SCD2/4, 16
	yloop	16, c4b, -16*64+4*dist64 ;; Loop 16 times, next source pointer
	yloop	4, c4b, -4*4*dist64+dist64, rdi, -64*YMM_SCD2/4 ;; Loop 4 times, next source pointer, restore sine/cosine pointer
	end_timer 12

	yloop	4, b4b0, -4*dist64+16*dist64 ;; Loop 4 times, next source pointer
	yloop	3, b4b0, -3*16*dist64	;; Loop 3 times, restore source pointer
	jmp	ypass2_12288_complex_unfft


ypass2_12288_complex_1:
	start_timer 6
	bump	rcx, (3072+47-256-2*192-3*768-192)*64 ;; Move prefetch pointer along
	yloop_init 768			;; 12 iters of 256 cache lines

b4a0:	yloop_set_prefetch YSINGLE	;; Prefetch 768 cache lines in this loop
	yr4_b4cl_four_complex_djbfft_preload
b4a:	yr4_b4cl_four_complex_djbfft rsi, 64, 16*64, 32*64, rdi, YMM_SCD2/4, 16
	yloop	16, b4a, -16*64+4*dist64 ;; Loop 16 times, next source pointer
	yloop	4, b4a, -4*4*dist64, rdi, -64*YMM_SCD2/4 ;; Loop 4 times, restore source pointer, restore sine/cosine pointer
	loops_undo 4*16			;; Back up loop counter

	yloop_set_prefetch YSINGLE	;; Prefetch 768 cache lines in this loop
;;	yr4_b4cl_four_complex_djbfft_preload
b5a:	yr4_b4cl_four_complex_djbfft rsi, 64, 4*64, 8*64, rdi, 4*YMM_SCD2/4, 4
	yloop	4, b5a, -4*64+16*64, rdi, -4*4*YMM_SCD2/4 ;; Loop 4 times, next source pointer, restore sine/cosine pointer
	yloop	4, b5a, -4*16*64+4*dist64, rdi, YMM_SCD2/4 ;; Loop 4 times, next source pointer, next sine/cosine pointer
	yloop	4, b5a, -4*4*dist64, rdi, -4*YMM_SCD2/4 ;; Loop 4 times, restore source pointer, next sine/cosine pointer
	loops_undo 4*4*4		;; Back up loop counter

	yloop_set_prefetch YSINGLE	;; Prefetch 768 cache lines in this loop
;;	yr4_b4cl_four_complex_djbfft_preload
b6a:	yr4_b4cl_four_complex_djbfft rsi, 4*64, 64, 2*64, rdi, 0, 16
	yloop	16, b6a, -16*4*64+4*dist64, rdi, 4*YMM_SCD2/4 ;; Loop 16 times, next source pointer, next sine/cosine pointer
	yloop	4, b6a, -4*4*dist64, rdi, -4*4*YMM_SCD2/4 ;; Loop 4 times, restore source pointer, restore sine/cosine pointer
	loops_undo 4*16			;; Back up loop counter

	yloop_set_prefetch YQUARTER	;; Prefetch 192 cache lines in this loop
	yr4_4cl_four_complex_fft_final_preload
b7a:	yr4_4cl_four_complex_fft_final rsi, 64, 4*dist64, 8*dist64
	yloop	64, b7a, -64*64+dist64	;; Loop 64 times, next source pointer

	yloop	4, b4a0, -4*dist64+16*dist64 ;; Loop 4 times, next source pointer
	yloop	3, b4a0			;; Loop 3 times
	end_timer 6
	jmp	ypass2_12288_complex_done


ypass2_12288_complex_3:
	start_timer 6
x64	mov	r8, DEST2ARG
	mov	rbp, DIST_TO_MULSRCARG
	yloop_init 768			;; 12 iters of 256 cache lines

b4c0:	yloop_set_prefetch YHALF	;; Prefetch 384 cache lines in this loop
	yr4_b4cl_four_complex_djbfft_preload
b4c:	yr4_b4cl_four_complex_djbfft rsi, 64, 16*64, 32*64, rdi, YMM_SCD2/4, 16
	yloop	16, b4c, -16*64+4*dist64	;; Loop 16 times, next source pointer
	yloop	4, b4c, -4*4*dist64, rdi, -64*YMM_SCD2/4 ;; Loop 4 times, restore source pointer, restore sine/cosine pointer
	loops_undo 4*16			;; Back up loop counter

	yloop_set_prefetch YQUARTER	;; Prefetch 192 cache lines in this loop
;;	yr4_b4cl_four_complex_djbfft_preload
b5c:	yr4_b4cl_four_complex_djbfft rsi, 64, 4*64, 8*64, rdi, 4*YMM_SCD2/4, 4
	yloop	4, b5c, -4*64+16*64, rdi, -4*4*YMM_SCD2/4 ;; Loop 4 times, next source pointer, restore sine/cosine pointer
	yloop	4, b5c, -4*16*64+4*dist64, rdi, YMM_SCD2/4 ;; Loop 4 times, next source pointer, next sine/cosine pointer
	yloop	4, b5c, -4*4*dist64, rdi, -4*YMM_SCD2/4 ;; Loop 4 times, restore source pointer, next sine/cosine pointer
	loops_undo 4*4*4		;; Back up loop counter

	yloop_set_prefetch YQUARTER	;; Prefetch 192 cache lines in this loop
;;	yr4_b4cl_four_complex_djbfft_preload
b6c:	yr4_b4cl_four_complex_djbfft rsi, 4*64, 64, 2*64, rdi, 0, 16
	yloop	16, b6c, -16*4*64+4*dist64, rdi, 4*YMM_SCD2/4 ;; Loop 16 times, next source pointer, next sine/cosine pointer
	yloop	4, b6c, -4*4*dist64, rdi, -4*4*YMM_SCD2/4 ;; Loop 4 times, restore source pointer, restore sine/cosine pointer
	loops_undo 4*16			;; Back up loop counter

	yloop_set_prefetch YQUARTER	;; Prefetch 192 cache lines in this loop
	yr4_4cl_four_complex_with_mult_preload
b7c:	yr4_4cl_four_complex_with_mult rsi, 64, 4*dist64, 8*dist64
	yloop	64, b7c, -64*64		;; Loop 64 times, restore source pointer
x64	add	rsi, r8
	loops_undo 64			;; Back up loop counter

	yloop_set_prefetch YQUARTER	;; Prefetch 192 cache lines in this loop
	yr4_b4cl_four_complex_djbunfft_preload
c6c:	yr4_b4cl_four_complex_djbunfft rsi, 4*64, 64, 2*64, rdi, 0, 16
	yloop	16, c6c, -16*4*64+4*dist64, rdi, 4*YMM_SCD2/4 ;; Loop 16 times, next source pointer, next sine/cosine pointer
	yloop	4, c6c, -4*4*dist64, rdi, -4*4*YMM_SCD2/4 ;; Loop 4 times, restore source pointer, restore sine/cosine pointer
	loops_undo 4*16			;; Restore loop counter

	yloop_set_prefetch YHALF	;; Prefetch 384 cache lines in this loop
;;	yr4_b4cl_four_complex_djbunfft_preload
c5c:	yr4_b4cl_four_complex_djbunfft rsi, 64, 4*64, 8*64, rdi, 4*YMM_SCD2/4, 4
	yloop	4, c5c, -4*64+16*64, rdi, -4*4*YMM_SCD2/4 ;; Loop 4 times, next source pointer, restore sine/cosine pointer
	yloop	4, c5c, -4*16*64+4*dist64, rdi, YMM_SCD2/4 ;; Loop 4 times, next source pointer, next sine/cosine pointer
	yloop	4, c5c, -4*4*dist64, rdi, -4*YMM_SCD2/4 ;; Loop 4 times, restore source pointer, next sine/cosine pointer
	loops_undo 4*4*4		;; Back up loop counter

	yloop_set_prefetch YHALF	;; Prefetch 384 cache lines in this loop
;;	yr4_b4cl_four_complex_djbunfft_preload
c4c:	yr4_b4cl_four_complex_djbunfft rsi, 64, 16*64, 32*64, rdi, YMM_SCD2/4, 16
	yloop	16, c4c, -16*64+4*dist64 ;; Loop 16 times, next source pointer
	yloop	4, c4c, -4*4*dist64+dist64, rdi, -64*YMM_SCD2/4 ;; Loop 4 times, next source pointer, restore sine/cosine pointer

x64	sub	rsi, r8
	yloop	4, b4c0, -4*dist64+16*dist64 ;; Loop 4 times, next source pointer
	yloop	3, b4c0, -3*16*dist64	;; Loop 3 times, restore source pointer
x64	add	rsi, r8
	end_timer 6
	jmp	ypass2_12288_complex_unfft


ypass2_12288_complex_4:
	start_timer 6
	mov	rdi, xsincos_complex	;; The sin/cos ptrs
	bump	rdi, 1024*YMM_SCD1+256*YMM_SCD2
	mov	rbp, DIST_TO_MULSRCARG
	yloop_init 768			;; 12 iters of 256 cache lines

	yloop_set_prefetch YHALF	;; Prefetch 384 cache lines in this loop
	yr4_4cl_four_complex_with_mulf_preload
b7d:	yr4_4cl_four_complex_with_mulf rsi, 64, 4*dist64, 8*dist64
	yloop	64, b7d, -64*64		;; Loop 64 times, restore source pointer
	loops_undo 64			;; Back up loop counter

	yloop_set_prefetch YSINGLE	;; Prefetch 768 cache lines in this loop
	yr4_b4cl_four_complex_djbunfft_preload
c6d:	yr4_b4cl_four_complex_djbunfft rsi, 4*64, 64, 2*64, rdi, 0, 16
	yloop	16, c6d, -16*4*64+4*dist64, rdi, 4*YMM_SCD2/4 ;; Loop 16 times, next source pointer, next sine/cosine pointer
	yloop	4, c6d, -4*4*dist64, rdi, -4*4*YMM_SCD2/4 ;; Loop 4 times, restore source pointer, restore sine/cosine pointer
	loops_undo 4*16			;; Restore loop counter

	yloop_set_prefetch YSINGLE	;; Prefetch 768 cache lines in this loop
;;	yr4_b4cl_four_complex_djbunfft_preload
c5d:	yr4_b4cl_four_complex_djbunfft rsi, 64, 4*64, 8*64, rdi, 4*YMM_SCD2/4, 4
	yloop	4, c5d, -4*64+16*64, rdi, -4*4*YMM_SCD2/4 ;; Loop 4 times, next source pointer, restore sine/cosine pointer
	yloop	4, c5d, -4*16*64+4*dist64, rdi, YMM_SCD2/4 ;; Loop 4 times, next source pointer, next sine/cosine pointer
	yloop	4, c5d, -4*4*dist64, rdi, -4*YMM_SCD2/4 ;; Loop 4 times, restore source pointer, next sine/cosine pointer
	loops_undo 4*4*4		;; Back up loop counter

	yloop_set_prefetch YSINGLE	;; Prefetch 768 cache lines in this loop
;;	yr4_b4cl_four_complex_djbunfft_preload
c4d:	yr4_b4cl_four_complex_djbunfft rsi, 64, 16*64, 32*64, rdi, YMM_SCD2/4, 16
	yloop	16, c4d, -16*64+4*dist64	;; Loop 16 times, next source pointer
	yloop	4, c4d, -4*4*dist64+dist64, rdi, -64*YMM_SCD2/4 ;; Loop 4 times, next source pointer, restore sine/cosine pointer

	yloop	4, b7d, -4*dist64+16*dist64	;; Loop 4 times, next source pointer
	yloop	3, b7d, -3*16*dist64	;; Loop 3 times, restore source pointer
	bump	rcx, (-384-3*768+256+2*192+384+4*192+2*384)*64 ;; Move prefetch pointer to where squaring would be
	end_timer 6

;; Do inverse FFT levels 5,6
;; On input the 64-byte cache lines hold these data values:
;;	0	+2K	+2K	+2K	256	+2K	+2K	+2K
;;	4	...
;;	...
;;	252	...
;;	512	...
;;	...
;;	1	...
;;	...
;;	8192	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+2K	+2K	+2K	1024	+2K	+2K	+2K
;;	4	...
;;	...
;;	1020	...
;;	1	...
;;	...
;;	8192	...
;;	...

;; Do 3072 four_complex_unfft operations
;;	distance between fft data elements is 256
;;	do 768 macros each processing 32 data values

;; Do inverse FFT levels 3,4
;; On input the 64-byte cache lines hold these data values:
;;	0	+2K	+2K	+2K	1024	+2K	+2K	+2K
;;	4	...
;;	...
;;	1020	...
;;	1	...
;;	...
;;	8192	...
;;	...
;; On output the 64-byte cache lines are swizzled to hold these data values:
;;	0	+1	+1	+1	4096	+1	+1	+1
;;	4	...
;;	...
;;	4092	...
;;	8192	...
;;	...

;; Do 3072 four_complex_unfft operations
;;	distance between fft data elements is 1024
;;	do 768 macros each processing 32 data values

;; Do inverse FFT levels 1,2
;; On input the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	4096	+1	+1	+1
;;	4	...
;;	...
;;	4092	...
;;	8192	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	12288	+1	+1	+1
;;	4	...
;;	...
;;	12284

;; Do 4096 three_complex_unfft operations
;;	distance between fft data elements is 4096
;;	do 1024 macros each processing 24 data values

ypass2_12288_complex_unfft:
	mov	rdi, xsincos_complex	;; Load sin/cos pointers
	lea	rbp, [rdi+1024*YMM_SCD1]
	mov	rdx, rbp
	bump	rcx, (3072+47-256-2*192-384-4*192-2*384-2*192-256)*64 ;; Move prefetch pointer along
	yloop_init 1024			;; 1024 iterations or 768 iterations

c3a:	start_timer 13
	yloop_adjust 4			;; Four fewer iterations this loop
	yloop_set_prefetch YQUARTER	;; Prefetch 192 cache lines in this loop
	yr4_eb4cl_four_complex_djbunfft_preload
c3b:	yr4_eb4cl_four_complex_djbunfft rsi, 4*dist64, dist64, 2*dist64, rdx, YMM_SCD2, 4
	yloop	4, c3b, 0, rdx, -4*YMM_SCD2 ;; Loop 4 times, restore sine/cosine pointer
	yloop	4, c3b, -3*4*4*dist64, rdx, 4*YMM_SCD2 ;; 3 iterations!!  Restore source pointer, next sine/cosine pointer
	loops_undo 4*4			;; Back up loop counter
	end_timer 13

	start_timer 14
	yloop_adjust 4			;; Four fewer iterations this loop
	yloop_set_prefetch YQUARTER	;; Prefetch 192 cache lines in this loop
	yr4_s4cl_four_complex_djbunfft_preload
c2b:	yr4_s4cl_four_complex_djbunfft rsi, dist64, 4*dist64, 8*dist64, rbp, 64*YMM_SCD2, 4, L1PREFETCH_ALL, 64
	yloop	4, c2b, -4*dist64+16*dist64, rbp, -4*64*YMM_SCD2 ;; Test loop counter, next source pointer, restore sine/cosine pointer
	yloop	4, c2b, -3*16*dist64, rbp, YMM_SCD2 ;; 3 iterations!!  Restore source pointer, next sine/cosine pointer
	loops_undo 4*4			;; Back up loop counter
	end_timer 14

	start_timer 15
	yloop_set_prefetch YQUARTER	;; Prefetch 256 cache lines in this loop
	yr3_3cl_three_complex_djbunfft_preload
c1b:	yr3_3cl_three_complex_djbunfft rsi, dist64, 16*dist64, rdi, 64*YMM_SCD1, 16
	yloop	16, c1b, -16*dist64+64, rdi, -16*64*YMM_SCD1+YMM_SCD1 ;; Test loop counter, next source pointer, next sine/cosine pointer
	end_timer 15

	yloop	64, c3a			;; Loop 64 times

ypass2_12288_complex_done:
	ENDM


; *************** radix4-pass2-15360-real macro ******************
; This macro takes 30720 real values and performs the final 14.907 levels of the FFT.
; The results are squared, and the first 14.907 levels of the inverse FFT are done.

yr4dwpn_pass2_15360_real MACRO
	LOCAL	b1b, b2a, b2b, b3a, b3b, b4a, b4b, b5a, b5b, b6a, b6b, b7a, b7b, b7c, b7d
	LOCAL	c1b, c2a, c2b, c3a, c3b, c4a, c4b, c5a, c5b, c6a, c6b
	LOCAL	ypass2_15360_real_1, ypass2_15360_real_2
	LOCAL	ypass2_15360_real_3, ypass2_15360_real_4
	LOCAL	ypass2_15360_real_unfft, ypass2_15360_real_done

;; Type 4 FFTs skip the forward FFT process

	start_timer 2
	mov	rsi, DATA_ADDR		;; Load source address
	mov	rbx, DIST_TO_FFTSRCARG
	mov	rcx, DATA_PREFETCH	;; Prefetch pointer (we need to prefetch 
					;; 3840 cache lines + 59 pad cache lines)
	cmp	ffttype, 4
	je	ypass2_15360_real_4

;; Do FFT levels 1,2,3 which correspond to complex levels 1,2
;; On input the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	15360	+1	+1	+1
;;	4	...
;;	...
;;	15356
;; On output the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	5120	+1	+1	+1
;;	4	...
;;	...
;;	5116	...
;;	10240	...
;;	...

;; Do 5120 six_reals_fft operations
;;	distance between fft data elements is 5120
;;	do 1280 macros each processing 24 data values

	mov	rdi, sincos3		;; Real sin/cos pointer
	mov	rbp, xsincos_complex	;; Complex sin/cos data
	yloop_init 1280			;; 1280 iterations
	yloop_set_L3_prefetch YHALF
	yr3_f3cl_2sc_six_reals_fft_preload
b1b:	yr3_f3cl_2sc_six_reals_fft rsi, 64, 20*dist64, rbp, YMM_SCD1, rdi, YMM_SCD1, 64
	yloop	64, b1b, -64*64+dist64	;; Test loop counter, next source pointer
	yloop	20, b1b, -20*dist64	;; Test loop counter, restore source pointer

;; Do FFT levels 3,4
;; On input the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	5120	+1	+1	+1
;;	4	...
;;	...
;;	5116	...
;;	10240	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	1024	+1	+1	+1
;;	4	...
;;	...
;;	1020	...
;;	2048	...
;;	...

;; Do 1024 ten_reals_fft operations
;; Do 2048 five_complex_fft operations
;;	distance between fft data elements is 1024
;;	do 768 macros each processing 40 data values

	yloop_init 768			;; 3 iters of 256
	yloop_set_L3_prefetch YHALF
	yr5_5cl_2sc_ten_reals_fft_preload
b2a:	yr5_5cl_2sc_ten_reals_fft rsi, 64, 4*dist64, rbp, YMM_SCD2, rdi, YMM_SCD2, 64
	yloop	64, b2a, -64*64+dist64	;; Test loop counter, next source pointer
	yloop	4, b2a, -4*dist64+20*dist64, rbp, -256*YMM_SCD2 ;; Test loop counter, next source pointer, restore sine/cosine pointer

	loops_reset			;; 2 more iters of 256
	yr5_5cl_five_complex_djbfft_preload
b2b:	yr5_5cl_five_complex_djbfft rsi, 64, 4*dist64, rbp, YMM_SCD2, 64
	yloop	64, b2b, -64*64+dist64	;; Test loop counter, next source pointer
	yloop	4, b2b, -4*dist64+20*dist64, rbp, -256*YMM_SCD2 ;; Test loop counter, next source pointer, restore sin/cos ptr
	yloop	3, b2b, -3*20*dist64, rbp, 256*YMM_SCD2 ;; Loop 3 times, restore source pointer, next sin/cos pointer

;; Do FFT levels 5,6
;; On input the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	1024	+1	+1	+1
;;	4	...
;;	...
;;	1020	...
;;	2048	...
;;	...
;; On output the 64-byte cache lines are swizzled to hold these data values:
;;	0	+512	+512	+512	256	+512	+512	+512
;;	4	...
;;	...
;;	508	...
;;	1	...
;;	...
;;	2048	...
;;	...

;; Do 256 eight_reals_fft operations
;; Do 3584 four_complex_fft operations
;;	distance between fft data elements is 256
;;	do 960 macros each processing 32 data values

	yloop_init 960			;; 15 iterations of 64
	yloop_set_L3_prefetch YHALF
	yr4_s4cl_2sc_eight_reals_fft_preload
b3a:	yr4_s4cl_2sc_eight_reals_fft rsi, 64, dist64, 2*dist64, rbp, YMM_SCD2, rdi, YMM_SCD2, 64
	yloop	64, b3a, -64*64+4*dist64, rbp, -64*YMM_SCD2 ;; Test loop counter, next source pointer, restore sine/cosine pointer

	loops_reset			;; 14 more iters of 64
	yr4_s4cl_four_complex_djbfft_preload
b3b:	yr4_s4cl_four_complex_djbfft rsi, 64, dist64, 2*dist64, rbp, YMM_SCD2, 64
	yloop	64, b3b, -64*64+4*dist64, rbp, -64*YMM_SCD2 ;; Test loop counter, next source pointer, restore sin/cos ptr
	yloop	15, b3b, -15*4*dist64	;; Loop 15 times, restore source pointer

;; Do FFT levels 7,8
;; On input the 64-byte cache lines hold these data values:
;;	0	+512	+512	+512	256	+512	+512	+512
;;	4	...
;;	...
;;	508	...
;;	1	...
;;	...
;;	2048	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+512	+512	+512	64	+512	+512	+512
;;	4	...
;;	...
;;	60	...
;;	128	...
;;	...
;;	1	...
;;	...
;;	2048	...
;;	...

;; Do 64 eight_reals_fft operations
;; Do 3776 four_complex_fft operations
;;	distance between fft data elements is 64
;;	do 960 macros each processing 32 data values

	yloop_init 960			;; 15 iterations of 64
	yloop_set_L3_prefetch YHALF
	yr4_4cl_eight_reals_four_complex_djbfft_preload
b4a:	yr4_4cl_eight_reals_four_complex_djbfft rsi, dist64, 16*64, 32*64, rdi, YMM_SCD3, 4
	yloop	4, b4a, -4*dist64+64	;; Test loop counter, next source pointer
	yloop	16, b4a, -16*64+4*dist64, rdi, -64*YMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer

	loops_reset			;; 14 more iters of 64
	yr4_rb4cl_four_complex_djbfft_preload
b4b:	yr4_rb4cl_four_complex_djbfft rsi, dist64, 16*64, 32*64, rdi, YMM_SCD3, 4
	yloop	4, b4b, -4*dist64+64	;; Test loop counter, next source pointer
	yloop	16, b4b, -16*64+4*dist64, rdi, -64*YMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer
	yloop	15, b4b, -15*4*dist64	;; Test loop counter, restore source pointer

;; Do FFT levels 9,10
;; On input the 64-byte cache lines hold these data values:
;;	0	+512	+512	+512	64	+512	+512	+512
;;	4	...
;;	...
;;	60	...
;;	128	...
;;	...
;;	1	...
;;	...
;;	2048	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+512	+512	+512	16	+512	+512	+512
;;	4	...
;;	...
;;	12	...
;;	32	...
;;	...
;;	1	...
;;	...
;;	2048	...
;;	...

;; Do 16 eight_reals_fft operations
;; Do 3824 four_complex_fft operations
;;	distance between fft data elements is 16
;;	do 960 macros each processing 32 data values

	yloop_init 960			;; 60 iterations of 16
	yloop_set_L3_prefetch YHALF
	yr4_4cl_eight_reals_four_complex_djbfft_preload
b5a:	yr4_4cl_eight_reals_four_complex_djbfft rsi, dist64, 4*64, 8*64, rdi, 4*YMM_SCD3, 4
	yloop	4, b5a, -4*dist64+64	;; Test loop counter, next source pointer
	yloop	4, b5a, -4*64+16*64, rdi, -16*4*YMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer

	loops_reset			;; 59 more iters of 16
	yr4_rb4cl_four_complex_djbfft_preload
b5b:	yr4_rb4cl_four_complex_djbfft rsi, dist64, 4*64, 8*64, rdi, 4*YMM_SCD3, 4
	yloop	4, b5b, -4*dist64+64	;; Test loop counter, next source pointer
	yloop	4, b5b, -4*64+16*64, rdi, -16*4*YMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer
	yloop	4, b5b, -4*16*64+4*dist64 ;; Test loop counter, next source pointer
	yloop	15, b5b, -15*4*dist64	;; Test loop counter, restore source pointer

;; Do FFT levels 11,12
;; On input the 64-byte cache lines hold these data values:
;;	0	+512	+512	+512	16	+512	+512	+512
;;	4	...
;;	...
;;	12	...
;;	32	...
;;	...
;;	1	...
;;	...
;;	2048	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+512	+512	+512	4	+512	+512	+512
;;	8	...
;;	...
;;	1	...
;;	...
;;	2048	...
;;	...

;; Do 4 eight_reals_fft operations
;; Do 3836 four_complex_fft operations
;;	distance between fft data elements is 4
;;	do 960 macros each processing 32 data values

	yloop_init 960			;; 240 iterations of 4
	yloop_set_L3_prefetch YQUARTER
	yr4_4cl_eight_reals_four_complex_djbfft_preload
b6a:	yr4_4cl_eight_reals_four_complex_djbfft rsi, dist64, 64, 2*64, rdi, 16*YMM_SCD3, 4
	yloop	4, b6a, -4*dist64+4*64, rdi, -4*16*YMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer

	loops_reset			;; 239 more iters of 4
	yr4_rb4cl_four_complex_djbfft_preload
b6b:	yr4_rb4cl_four_complex_djbfft rsi, dist64, 64, 2*64, rdi, 16*YMM_SCD3, 4
	yloop	4, b6b, -4*dist64+4*64, rdi, -4*16*YMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer
	yloop	16, b6b, -16*4*64+4*dist64 ;; Test loop counter, next source pointer
	yloop	15, b6b, -15*4*dist64	;; Test loop counter, restore source pointer

;; Do FFT levels 13,14 as well as inverse FFT levels 13,14
;; On input and output the 64-byte cache lines hold these data values:
;;	0	+512	+512	+512	4	+512	+512	+512
;;	8	...
;;	...
;;	1	...
;;	...
;;	2048	...
;;	...

;; Do 1 eight_reals_fft operation
;; Do 3839 four_complex_fft operations
;;	distance between fft data elements is 1
;;	do 960 macros each processing 32 data values

	cmp	ffttype, 2
	jl	ypass2_15360_real_1
	jg	ypass2_15360_real_3

ypass2_15360_real_2:
;;	yr4_4cl_eight_reals_four_complex_with_square rsi, 64, dist64, 2*dist64
	yr4_dispatch call, yreal_square, dist64, 2*dist64
	yloop_init 960			;; 959 more iterations
	yloop_adjust 1
	yloop_set_L3_prefetch YQUARTER
	yr4_4cl_four_complex_with_square_preload
b7b:	yr4_4cl_four_complex_with_square rsi, 64, dist64, 2*dist64, 1
	yloop	64, b7b, -64*64+4*dist64	;; Test loop counter, next source pointer
	yloop	15, b7b, -15*4*dist64	;; Test loop counter, restore source pointer
	jmp	ypass2_15360_real_unfft

ypass2_15360_real_1:
;;	yr4_4cl_eight_reals_four_complex_fft_final rsi, 64, dist64, 2*dist64
	yr4_dispatch call, yreal_fft_final, dist64, 2*dist64
	mov	rcx, xsincos_complex	;; Prefetch complex sin/cos data, 1920 cache lines
	yloop_init 960			;; 959 more iterations
	yloop_adjust 1
	yloop_set_L3_prefetch YDOUBLE
b7a:	yr4_4cl_four_complex_fft_final rsi, 64, dist64, 2*dist64
	yloop	64, b7a, -64*64+4*dist64	;; Test loop counter, next source pointer
	yloop	15, b7a			;; Test loop counter
	jmp	ypass2_15360_real_done

ypass2_15360_real_3:
x64	mov	r8, DEST2ARG
	mov	rbp, DIST_TO_MULSRCARG
;;	yr4_4cl_eight_reals_four_complex_with_mult rsi, 64, dist64, 2*dist64
	yr4_dispatch call, yreal_mult, dist64, 2*dist64
	yloop_init 960			;; 959 more iterations
	yloop_adjust 1
	yloop_set_L3_prefetch YQUARTER
	yr4_4cl_four_complex_with_mult_preload
b7c:	yr4_4cl_four_complex_with_mult rsi, 64, dist64, 2*dist64
	yloop	64, b7c, -64*64+4*dist64	;; Test loop counter, next source pointer
	yloop	15, b7c, -15*4*dist64	;; Test loop counter, restore source pointer
x64	add	rsi, r8
	jmp	ypass2_15360_real_unfft

ypass2_15360_real_4:
	mov	rbp, DIST_TO_MULSRCARG
;;	yr4_4cl_eight_reals_four_complex_with_mulf rsi, 64, dist64, 2*dist64
	yr4_dispatch call, yreal_mulf, dist64, 2*dist64
	yloop_init 960			;; 959 more iterations
	yloop_adjust 1
	yloop_set_L3_prefetch YDOUBLE
	yr4_4cl_four_complex_with_mulf_preload
b7d:	yr4_4cl_four_complex_with_mulf rsi, 64, dist64, 2*dist64
	yloop	64, b7d, -64*64+4*dist64	;; Test loop counter, next source pointer
	yloop	15, b7d, -15*4*dist64	;; Test loop counter, restore source pointer
;;	bump	rcx, (-1918+640+384+3*480+2*240)*64 ;; Move prefetch pointer back to where squaring would be

;; Do inverse FFT levels 11,12
;; On input the 64-byte cache lines hold these data values:
;;	0	+512	+512	+512	4	+512	+512	+512
;;	8	...
;;	...
;;	1	...
;;	...
;;	2048	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+512	+512	+512	16	+512	+512	+512
;;	4	...
;;	...
;;	12	...
;;	32	...
;;	...
;;	1	...
;;	...
;;	2048	...
;;	...

;; Do 4 eight_reals_unfft operations
;; Do 3836 four_complex_unfft operations
;;	distance between fft data elements is 4
;;	do 960 macros each processing 32 data values

ypass2_15360_real_unfft:
	mov	rdi, sincos3		;; Load sin/cos pointer
	bump	rdi, 64*YMM_SCD2+256*YMM_SCD2+1280*YMM_SCD1
	yloop_init 960			;; 240 iterations of 4
	yloop_set_L3_prefetch YQUARTER
	yr4_4cl_eight_reals_four_complex_djbunfft_preload
c6a:	yr4_4cl_eight_reals_four_complex_djbunfft rsi, dist64, 64, 2*64, rdi, 16*YMM_SCD3, 4
	yloop	4, c6a, -4*dist64+4*64, rdi, -4*16*YMM_SCD3 ;; Test loop counter, next source pointer, restore sine/cosine pointer

	loops_reset			;; 239 more iters of 4
	yr4_rb4cl_four_complex_djbunfft_preload
c6b:	yr4_rb4cl_four_complex_djbunfft rsi, dist64, 64, 2*64, rdi, 16*YMM_SCD3, 4
	yloop	4, c6b, -4*dist64+4*64, rdi, -4*16*YMM_SCD3 ;; Test loop counter, next source pointer, restore sine/cosine pointer
	yloop	16, c6b, -16*4*64+4*dist64	;; Test loop counter, next source pointer
	yloop	15, c6b, -15*4*dist64	;; Test loop counter, restore source pointer

;; Do inverse FFT levels 9,10
;; On input the 64-byte cache lines hold these data values:
;;	0	+512	+512	+512	16	+512	+512	+512
;;	4	...
;;	...
;;	12	...
;;	32	...
;;	...
;;	1	...
;;	...
;;	2048	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+512	+512	+512	64	+512	+512	+512
;;	4	...
;;	...
;;	60	...
;;	128	...
;;	...
;;	1	...
;;	...
;;	2048	...
;;	...

;; Do 16 eight_reals_unfft operations
;; Do 3824 four_complex_unfft operations
;;	distance between fft data elements is 16
;;	do 960 macros each processing 32 data values

	yloop_init 960			;; 60 iterations of 16
	yloop_set_L3_prefetch YQUARTER
	yr4_4cl_eight_reals_four_complex_djbunfft_preload
c5a:	yr4_4cl_eight_reals_four_complex_djbunfft rsi, dist64, 4*64, 8*64, rdi, 4*YMM_SCD3, 4
	yloop	4, c5a, -4*dist64+64	;; Test loop counter, next source pointer
	yloop	4, c5a, -4*64+16*64, rdi, -16*4*YMM_SCD3 ;; Test loop counter, next source pointer, restore sine/cosine pointer

	loops_reset			;; 59 more iters of 16
	yr4_rb4cl_four_complex_djbunfft_preload
c5b:	yr4_rb4cl_four_complex_djbunfft rsi, dist64, 4*64, 8*64, rdi, 4*YMM_SCD3, 4
	yloop	4, c5b, -4*dist64+64	;; Test loop counter, next source pointer
	yloop	4, c5b, -4*64+16*64, rdi, -16*4*YMM_SCD3 ;; Test loop counter, next source pointer, restore sine/cosine pointer
	yloop	4, c5b, -4*16*64+4*dist64	;; Test loop counter, next source pointer
	yloop	15, c5b, -15*4*dist64	;; Test loop counter, restore source pointer

;; Do inverse FFT levels 7,8
;; On input the 64-byte cache lines hold these data values:
;;	0	+512	+512	+512	64	+512	+512	+512
;;	4	...
;;	...
;;	60	...
;;	128	...
;;	...
;;	1	...
;;	...
;;	2048	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+512	+512	+512	256	+512	+512	+512
;;	4	...
;;	...
;;	508	...
;;	1	...
;;	...
;;	2048	...
;;	...

;; Do 64 eight_reals_unfft operations
;; Do 3776 four_complex_unfft operations
;;	distance between fft data elements is 64
;;	do 960 macros each processing 32 data values

	bump	rcx, (3840+59-640-384-3*480-4*240-480)*64 ;; Move prefetch pointer along
	yloop_init 960			;; 15 iterations of 64
	yloop_set_L3_prefetch YHALF
	yr4_4cl_eight_reals_four_complex_djbunfft_preload
c4a:	yr4_4cl_eight_reals_four_complex_djbunfft rsi, dist64, 16*64, 32*64, rdi, YMM_SCD3, 4
	yloop	4, c4a, -4*dist64+64	;; Test loop counter, next source pointer
	yloop	16, c4a, -16*64+4*dist64, rdi, -64*YMM_SCD3 ;; Test loop counter, next source pointer, restore sine/cosine pointer

	loops_reset			;; 14 more iters of 64
	yr4_rb4cl_four_complex_djbunfft_preload
c4b:	yr4_rb4cl_four_complex_djbunfft rsi, dist64, 16*64, 32*64, rdi, YMM_SCD3, 4
	yloop	4, c4b, -4*dist64+64	;; Test loop counter, next source pointer
	yloop	16, c4b, -16*64+4*dist64, rdi, -64*YMM_SCD3 ;; Test loop counter, next source pointer, restore sine/cosine pointer
	yloop	15, c4b, -15*4*dist64	;; Test loop counter, restore source pointer

;; Do inverse FFT levels 5,6
;; On input the 64-byte cache lines hold these data values:
;;	0	+512	+512	+512	256	+512	+512	+512
;;	4	...
;;	...
;;	508	...
;;	1	...
;;	...
;;	2048	...
;;	...
;; On output the 64-byte cache lines are swizzled to hold these data values:
;;	0	+1	+1	+1	1024	+1	+1	+1
;;	4	...
;;	...
;;	1020	...
;;	2048	...
;;	...

;; Do 256 eight_reals_unfft operations
;; Do 3584 four_complex_unfft operations
;;	distance between fft data elements is 256
;;	do 960 macros each processing 32 data values

	bump	rdi, -64*YMM_SCD2	;; Real sin/cos ptr
	mov	rbp, xsincos_complex	;; Complex sin/cos data
	bump	rbp, 256*YMM_SCD2+1280*YMM_SCD1
	mov	rcx, xsincos_complex	;; Prefetch complex sin/cos data, 1920 cache lines
	yloop_init 960			;; 15 iters of 64
	yloop_set_L3_prefetch YSINGLE
	yr4_s4cl_2sc_eight_reals_unfft_preload
c3a:	yr4_s4cl_2sc_eight_reals_unfft rsi, 64, dist64, 2*dist64, rbp, YMM_SCD2, rdi, YMM_SCD2, 64
	yloop	64, c3a, -64*64+4*dist64, rbp, -64*YMM_SCD2 ;; Test loop counter, next source pointer, restore sine/cosine pointer

	loops_reset			;; 14 more iters of 64
	yr4_s4cl_four_complex_djbunfft_preload
c3b:	yr4_s4cl_four_complex_djbunfft rsi, 64, dist64, 2*dist64, rbp, YMM_SCD2, 64
	yloop	64, c3b, -64*64+4*dist64, rbp, -64*YMM_SCD2 ;; Test loop counter, next source pointer, restore sine/cosine pointer
	yloop	15, c3b, -15*4*dist64	;; Test loop counter, restore source pointer

;; Do inverse FFT levels 3,4
;; On input the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	1024	+1	+1	+1
;;	4	...
;;	...
;;	1020	...
;;	2048	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	5120	+1	+1	+1
;;	4	...
;;	...
;;	5116	...
;;	10240	...
;;	...

;; Do 1024 ten_reals_unfft operations
;; Do 2048 five_complex_unfft operations
;;	distance between fft data elements is 1024
;;	do 768 macros each processing 40 data values

	bump	rdi, -64*YMM_SCD2-256*YMM_SCD2 ;; Real sin/cos ptr
	bump	rbp, -256*YMM_SCD2	;; Complex sin/cos data
	yloop_init 768			;; 3 iters of 256
	yloop_set_L3_prefetch YHALF
	yr5_5cl_2sc_ten_reals_unfft_preload
c2a:	yr5_5cl_2sc_ten_reals_unfft rsi, 64, 4*dist64, rbp, YMM_SCD2, rdi, YMM_SCD2, 64
	yloop	64, c2a, -64*64+dist64	;; Test loop counter, next source pointer
	yloop	4, c2a, -4*dist64+20*dist64, rbp, -256*YMM_SCD2 ;; Test loop counter, next source pointer, restore sine/cosine pointer

	loops_reset			;; 2 more iters of 256
	yr5_5cl_five_complex_djbunfft_preload
c2b:	yr5_5cl_five_complex_djbunfft rsi, 64, 4*dist64, rbp, YMM_SCD2, 64
	yloop	64, c2b, -64*64+dist64	;; Test loop counter, next source pointer
	yloop	4, c2b, -4*dist64+20*dist64, rbp, -256*YMM_SCD2 ;; Test loop counter, next source pointer, restore sine/cosine pointer
	yloop	3, c2b, -3*20*dist64	;; Test loop counter, restore source pointer

;; Do inverse FFT levels 1,2,3 which correspond to complex levels 1,2
;; On input the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	5120	+1	+1	+1
;;	4	...
;;	...
;;	5116	...
;;	10240	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	15360	+1	+1	+1
;;	4	...
;;	...
;;	15356

;; Do 3072 six_reals_unfft operations
;;	distance between fft data elements is 3072
;;	do 1280 macros each processing 24 data values

	bump	rdi, -256*YMM_SCD2-1280*YMM_SCD1 ;; Real sin/cos ptr
	bump	rbp, -1280*YMM_SCD1	;; Complex sin/cos data
	bump	rcx, (3840+59-960-384-640)*64 ;; Move prefetch pointer along
	yloop_init 1280			;; 1280 iterations
	yloop_set_L3_prefetch YHALF
	yr3_3cl_2sc_six_reals_unfft_preload
c1b:	yr3_3cl_2sc_six_reals_unfft rsi, 64, 20*dist64, rbp, YMM_SCD1, rdi, YMM_SCD1, 64
	yloop	64, c1b, -64*64+dist64	;; Test loop counter, next source pointer
	yloop	20, c1b			;; Test loop counter

ypass2_15360_real_done:
	end_timer 2
	ENDM

; *************** radix4-pass2-15360-complex macro ******************
; This macro takes 15360 complex values and performs the final 13.907 levels of a standard
; radix-4 FFT, squares the results, and does the first 13.907 levels of the inverse FFT.

yr4dwpn_pass2_15360_complex MACRO
	LOCAL	b1a, b1b, b2b, b3a0, b3a, b3b0, b3b, b3c0, b3c, b4a, b4b, b4c, b5a, b5b, b5c, b6a, b6b, b6c, b7a, b7b, b7c, b7d
	LOCAL	c1b, c2a, c2b, c3b, c3c, c3d, c4b, c4c, c4d, c5b, c5c, c5d, c6b, c6c, c6d
	LOCAL	ypass2_15360_complex_1, ypass2_15360_complex_2
	LOCAL	ypass2_15360_complex_3, ypass2_15360_complex_4
	LOCAL	ypass2_15360_complex_unfft, ypass2_15360_complex_done

;; Type 4 FFTs skip the forward FFT process

	mov	rsi, DATA_ADDR		;; Load address of FFT data
	mov	rbx, DIST_TO_FFTSRCARG
	mov	rcx, DATA_PREFETCH	;; Prefetch pointer (we need to prefetch 
					;; 3840 cache lines + 59 pad cache lines)
	cmp	ffttype, 4
	je	ypass2_15360_complex_4

;; Do FFT level 1,2 (actually 1.585 levels)
;; On input the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	15360	+1	+1	+1
;;	4	...
;;	...
;;	15356
;; On output the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	5120	+1	+1	+1
;;	4	...
;;	...
;;	5116	...
;;	10240	...
;;	...

;; Do 5120 three_complex_fft operations
;;	distance between fft data elements is 5120
;;	do 1280 macros each processing 24 data values

;; Do FFT levels 3,4 (actually 2.322 levels)
;; On input the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	5120	+1	+1	+1
;;	4	...
;;	...
;;	5116	...
;;	10240	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	1024	+1	+1	+1
;;	4	...
;;	...
;;	1020	...
;;	2048	...
;;	...

;; Do 3072 five_complex_fft operations
;;	distance between fft data elements is 1024
;;	do 768 macros each processing 40 data values

	mov	rdi, xsincos_complex	;; The sin/cos ptrs
	lea	rbp, [rdi+1280*YMM_SCD1]
	yloop_init 2048			;; 1280 iterations or 768 iterations

b1a:	start_timer 3
	yloop_adjust 3			;; Three fewer iterations this loop
	yloop_set_L3_prefetch YOTHER, 8	;; Prefetch 256 cache lines
	yr3_f3cl_three_complex_djbfft_preload
b1b:	yr3_f3cl_three_complex_djbfft rsi, 4*dist64, 20*dist64, rdi, 256*YMM_SCD1, 5
	yloop	8, b1b, -5*4*dist64, rdi, -5*256*YMM_SCD1+YMM_SCD1 ;; 5 iterations!!  Restore source pointer, next sin/cos data
	loops_undo 8			;; Back up loop counter
	end_timer 3

	start_timer 4
	yloop_adjust 5			;; Five fewer iterations this loop
	yloop_set_L3_prefetch YOTHER, 8	;; Prefetch 256 cache lines
	yr5_5cl_five_complex_djbfft_preload
b2b:	yr5_5cl_five_complex_djbfft rsi, 20*dist64, 4*dist64, rbp, 0, 3, L1PREFETCH_ALL, 64
	yloop	8, b2b, -3*20*dist64+64, rbp, YMM_SCD2 ;; 3 iterations!!  Next source pointer, next sin/cos data
	end_timer 4

	yloop	64, b1a, -64*64+dist64	;; Test loop counter, next source pointer
	yloop	4, b1a, -4*dist64, rdi, -256*YMM_SCD1+1280*YMM_SCD1+256*YMM_SCD2 ;; Test loop counter, restore source pointer, next sin/cos ptr

;; Do FFT levels 5,6
;; On input the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	1024	+1	+1	+1
;;	4	...
;;	...
;;	1020	...
;;	2048	...
;;	...
;; On output the 64-byte cache lines are swizzled to hold these data values:
;;	0	+512	+512	+512	256	+512	+512	+512
;;	4	...
;;	...
;;	508	...
;;	1	...
;;	...
;;	2048	...
;;	...

;; Do 3840 four_complex_fft operations
;;	distance between fft data elements is 256
;;	do 960 macros each processing 32 data values

;; Do FFT levels 7,8
;; On input the 64-byte cache lines hold these data values:
;;	0	+512	+512	+512	256	+512	+512	+512
;;	4	...
;;	...
;;	508	...
;;	1	...
;;	...
;;	2048	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+512	+512	+512	64	+512	+512	+512
;;	4	...
;;	...
;;	60	...
;;	128	...
;;	...
;;	1	...
;;	...
;;	2048	...
;;	...

;; Do 3840 four_complex_fft operations
;;	distance between fft data elements is 64
;;	do 960 macros each processing 32 data values

;; Do FFT levels 9,10
;; On input the 64-byte cache lines hold these data values:
;;	0	+512	+512	+512	64	+512	+512	+512
;;	4	...
;;	...
;;	60	...
;;	128	...
;;	...
;;	1	...
;;	...
;;	2048	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+512	+512	+512	16	+512	+512	+512
;;	4	...
;;	...
;;	12	...
;;	32	...
;;	...
;;	1	...
;;	...
;;	2048	...
;;	...

;; Do 3840 four_complex_fft operations
;;	distance between fft data elements is 16
;;	do 960 macros each processing 32 data values

;; Do FFT levels 11,12
;; On input the 64-byte cache lines hold these data values:
;;	0	+512	+512	+512	16	+512	+512	+512
;;	4	...
;;	...
;;	12	...
;;	32	...
;;	...
;;	1	...
;;	...
;;	2048	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+512	+512	+512	4	+512	+512	+512
;;	8	...
;;	...
;;	1	...
;;	...
;;	2048	...
;;	...

;; Do 3840 four_complex_fft operations
;;	distance between fft data elements is 4
;;	do 960 macros each processing 32 data values

;; Do FFT levels 13,14 as well as inverse FFT levels 13,14
;; On input and output the 64-byte cache lines hold these data values:
;;	0	+512	+512	+512	4	+512	+512	+512
;;	8	...
;;	...
;;	1	...
;;	...
;;	2048	...
;;	...

;; Do 3840 four_complex_fft operations
;;	distance between fft data elements is 1
;;	do 960 macros each processing 32 data values

;; Do inverse FFT levels 11,12
;; On input the 64-byte cache lines hold these data values:
;;	0	+512	+512	+512	4	+512	+512	+512
;;	8	...
;;	...
;;	1	...
;;	...
;;	2048	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+512	+512	+512	16	+512	+512	+512
;;	4	...
;;	...
;;	12	...
;;	32	...
;;	...
;;	1	...
;;	...
;;	2048	...
;;	...

;; Do 3840 four_complex_unfft operations
;;	distance between fft data elements is 4
;;	do 960 macros each processing 32 data values

;; Do inverse FFT levels 9,10
;; On input the 64-byte cache lines hold these data values:
;;	0	+512	+512	+512	16	+512	+512	+512
;;	4	...
;;	...
;;	12	...
;;	32	...
;;	...
;;	1	...
;;	...
;;	2048	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+512	+512	+512	64	+512	+512	+512
;;	4	...
;;	...
;;	60	...
;;	128	...
;;	...
;;	1	...
;;	...
;;	2048	...
;;	...

;; Do 3840 four_complex_unfft operations
;;	distance between fft data elements is 16
;;	do 960 macros each processing 32 data values

;; Do inverse FFT levels 7,8
;; On input the 64-byte cache lines hold these data values:
;;	0	+512	+512	+512	64	+512	+512	+512
;;	4	...
;;	...
;;	60	...
;;	128	...
;;	...
;;	1	...
;;	...
;;	2048	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+512	+512	+512	256	+512	+512	+512
;;	4	...
;;	...
;;	508	...
;;	1	...
;;	...
;;	2048	...
;;	...

;; Do 3840 four_complex_unfft operations
;;	distance between fft data elements is 64
;;	do 960 macros each processing 32 data values

;; Do inverse FFT levels 5,6
;; On input the 64-byte cache lines hold these data values:
;;	0	+512	+512	+512	256	+512	+512	+512
;;	4	...
;;	...
;;	508	...
;;	1	...
;;	...
;;	2048	...
;;	...
;; On output the 64-byte cache lines are swizzled to hold these data values:
;;	0	+1	+1	+1	1024	+1	+1	+1
;;	4	...
;;	...
;;	1020	...
;;	2048	...
;;	...

;; Do 3840 four_complex_unfft operations
;;	distance between fft data elements is 256
;;	do 960 macros each processing 32 data values

;; Do this 256 cache lines (16KB) at a time to maximize accessing data
;; while it is in the L1 cache.

;; Execute the right middle step

	cmp	ffttype, 2
	jl	ypass2_15360_complex_1
	jg	ypass2_15360_complex_3

ypass2_15360_complex_2:
	yloop_init 960			;; 15 iters of 256 cache lines

b3b0:	start_timer 5
	yloop_set_L3_prefetch YHALF	;; Prefetch 480 cache lines in this loop
	yr4_s4cl_four_complex_djbfft_preload
;; Loop below reworked to avoid 4KB distances
;;b3b:	yr4_s4cl_four_complex_djbfft rsi, 64, dist64, 2*dist64, rdi, YMM_SCD2, 64
;;	yloop	64, b3b, -64*64, rdi, -64*YMM_SCD2 ;; Loop 64 times, restore source pointer, restore sine/cosine pointer
b3b:	yr4_s4cl_four_complex_djbfft rsi, 4*64, dist64, 2*dist64, rdi, 4*YMM_SCD2, 16
	yloop	16, b3b, -16*4*64+64, rdi, -16*4*YMM_SCD2+YMM_SCD2 ;; Loop 16 times, next source pointer, next sine/cosine pointer
	yloop	4, b3b, -4*64, rdi, -4*YMM_SCD2 ;; Loop 4 times, restore source pointer, restore sine/cosine pointer
	loops_undo 64			;; Back up loop counter
	end_timer 5

	start_timer 6
	yloop_set_L3_prefetch YQUARTER	;; Prefetch 240 cache lines in this loop
	yr4_eb4cl_four_complex_djbfft_preload
b4b:	yr4_eb4cl_four_complex_djbfft rsi, 64, 16*64, 32*64, rdi, 4*YMM_SCD2, 16
	yloop	16, b4b, -16*64+dist64, rdi, -16*4*YMM_SCD2+YMM_SCD2 ;; Loop 16 times, next source pointer, next sine/cosine pointer
	yloop	4, b4b, -4*dist64, rdi, -4*YMM_SCD2 ;; Loop 4 times, restore source pointer, restore sine/cosine pointer
	loops_undo 4*16			;; Back up loop counter
	end_timer 6

	start_timer 7
	yloop_set_L3_prefetch YQUARTER	;; Prefetch 240 cache lines in this loop
;;	yr4_eb4cl_four_complex_djbfft_preload
b5b:	yr4_eb4cl_four_complex_djbfft rsi, 64, 4*64, 8*64, rdi, 16*YMM_SCD2, 4
	yloop	4, b5b, -4*64+16*64, rdi, -4*16*YMM_SCD2 ;; Loop 4 times, next source pointer, restore sine/cosine pointer
	yloop	4, b5b, -4*16*64+dist64, rdi, 4*YMM_SCD2 ;; Loop 4 times, next source pointer, next sine/cosine pointer
	yloop	4, b5b, -4*dist64, rdi, -4*4*YMM_SCD2 ;; Test loop counter, restore source pointer, restore sine/cosine pointer
	loops_undo 4*4*4		;; Back up loop counter
	end_timer 7

	start_timer 8
	yloop_set_L3_prefetch YQUARTER	;; Prefetch 240 cache lines in this loop
;;	yr4_eb4cl_four_complex_djbfft_preload
b6b:	yr4_eb4cl_four_complex_djbfft rsi, 4*64, 64, 2*64, rdi, 0, 16
	yloop	16, b6b, -16*4*64+dist64, rdi, 16*YMM_SCD2 ;; Loop 4 times, next source pointer, next sine/cosine pointer
	yloop	4, b6b, -4*dist64, rdi, -4*16*YMM_SCD2 ;; Test loop counter, restore source pointer, restore sine/cosine pointer
	loops_undo 4*16			;; Back up loop counter
	end_timer 8

	start_timer 9
	yloop_set_L3_prefetch YQUARTER	;; Prefetch 240 cache lines in this loop
	yr4_4cl_four_complex_with_square_preload
;; Loop below reworked to avoid 4KB distances
;;b7b:	yr4_4cl_four_complex_with_square rsi, 64, dist64, 2*dist64
;;	yloop	64, b7b, -64*64	;; Loop 64 times, restore source pointer
b7b:	yr4_4cl_four_complex_with_square rsi, 4*64, dist64, 2*dist64, 16
	yloop	16, b7b, -16*4*64+64	;; Loop 16 times, next source pointer
	yloop	4, b7b, -4*64	;; Loop 4 times, restore source pointer
	loops_undo 64			;; Back up loop counter
	end_timer 9

	start_timer 10
	yloop_set_L3_prefetch YQUARTER	;; Prefetch 240 cache lines in this loop
	yr4_eb4cl_four_complex_djbunfft_preload
c6b:	yr4_eb4cl_four_complex_djbunfft rsi, 4*64, 64, 2*64, rdi, 0, 16
	yloop	16, c6b, -16*4*64+dist64, rdi, 16*YMM_SCD2 ;; Loop 4 times, next source pointer, next sine/cosine pointer
	yloop	4, c6b, -4*dist64, rdi, -4*16*YMM_SCD2 ;; Test loop counter, restore source pointer, restore sine/cosine pointer
	loops_undo 4*16			;; Back up loop counter
	end_timer 10

	start_timer 11
	yloop_set_L3_prefetch YQUARTER	;; Prefetch 240 cache lines in this loop
;;	yr4_eb4cl_four_complex_djbunfft_preload
c5b:	yr4_eb4cl_four_complex_djbunfft rsi, 64, 4*64, 8*64, rdi, 16*YMM_SCD2, 4
	yloop	4, c5b, -4*64+16*64, rdi, -4*16*YMM_SCD2 ;; Loop 4 times, next source pointer, restore sine/cosine pointer
	yloop	4, c5b, -4*16*64+dist64, rdi, 4*YMM_SCD2 ;; Loop 4 times, next source pointer, next sine/cosine pointer
	yloop	4, c5b, -4*dist64, rdi, -4*4*YMM_SCD2 ;; Test loop counter, restore source pointer, restore sine/cosine pointer
	loops_undo 4*4*4		;; Back up loop counter
	end_timer 11

	start_timer 12
	yloop_set_L3_prefetch YHALF	;; Prefetch 480 cache lines in this loop
;;	yr4_eb4cl_four_complex_djbunfft_preload
c4b:	yr4_eb4cl_four_complex_djbunfft rsi, 64, 16*64, 32*64, rdi, 4*YMM_SCD2, 16
	yloop	16, c4b, -16*64+dist64, rdi, -16*4*YMM_SCD2+YMM_SCD2 ;; Loop 16 times, next source pointer, next sine/cosine pointer
	yloop	4, c4b, -4*dist64, rdi, -4*YMM_SCD2 ;; Loop 4 times, restore source pointer, restore sine/cosine pointer
	loops_undo 4*16			;; Back up loop counter
	end_timer 12

	start_timer 13
	yloop_set_L3_prefetch YHALF	;; Prefetch 480 cache lines in this loop
	yr4_s4cl_four_complex_djbunfft_preload
;; Loop below reworked to avoid 4KB distances
;;c3b:	yr4_s4cl_four_complex_djbunfft rsi, 64, dist64, 2*dist64, rdi, YMM_SCD2, 64
;;	yloop	64, c3b, -64*64, rdi, -64*YMM_SCD2 ;; Loop 64 times, restore source pointer, restore sine/cosine pointer
c3b:	yr4_s4cl_four_complex_djbunfft rsi, 4*64, dist64, 2*dist64, rdi, 4*YMM_SCD2, 16
	yloop	16, c3b, -16*4*64+64, rdi, -16*4*YMM_SCD2+YMM_SCD2 ;; Loop 16 times, next source pointer, next sine/cosine pointer
	yloop	4, c3b, -4*64+4*dist64, rdi, -4*YMM_SCD2 ;; Loop 4 times, next source pointer, restore sine/cosine pointer
	end_timer 13

	yloop	15, b3b0, -15*4*dist64	;; Loop 15 times, restore source pointer
	jmp	ypass2_15360_complex_unfft


ypass2_15360_complex_1:
	start_timer 6
	bump	rcx, (3840+59-2*256-3*960-480-30)*64 ;; Move prefetch pointer along
	yloop_init 960			;; 15 iters of 256 cache lines

b3a0:	yloop_set_L3_prefetch YSINGLE	;; Prefetch 960 cache lines in this loop
	yr4_s4cl_four_complex_djbfft_preload
;; Loop below reworked to avoid 4KB distances
;;b3a:	yr4_s4cl_four_complex_djbfft rsi, 64, dist64, 2*dist64, rdi, YMM_SCD2, 64
;;	yloop	64, b3a, -64*64, rdi, -64*YMM_SCD2 ;; Loop 64 times, restore source pointer, restore sine/cosine pointer
b3a:	yr4_s4cl_four_complex_djbfft rsi, 4*64, dist64, 2*dist64, rdi, 4*YMM_SCD2, 16
	yloop	16, b3a, -16*4*64+64, rdi, -16*4*YMM_SCD2+YMM_SCD2 ;; Loop 16 times, next source pointer, next sine/cosine pointer
	yloop	4, b3a, -4*64, rdi, -4*YMM_SCD2 ;; Loop 4 times, restore source pointer, restore sine/cosine pointer
	loops_undo 64			;; Back up loop counter

	yloop_set_L3_prefetch YSINGLE	;; Prefetch 960 cache lines in this loop
	yr4_eb4cl_four_complex_djbfft_preload
b4a:	yr4_eb4cl_four_complex_djbfft rsi, 64, 16*64, 32*64, rdi, 4*YMM_SCD2, 16
	yloop	16, b4a, -16*64+dist64, rdi, -16*4*YMM_SCD2+YMM_SCD2 ;; Loop 16 times, next source pointer, next sine/cosine pointer
	yloop	4, b4a, -4*dist64, rdi, -4*YMM_SCD2 ;; Loop 4 times, restore source pointer, restore sine/cosine pointer
	loops_undo 4*16			;; Back up loop counter

	yloop_set_L3_prefetch YSINGLE	;; Prefetch 960 cache lines in this loop
;;	yr4_eb4cl_four_complex_djbfft_preload
b5a:	yr4_eb4cl_four_complex_djbfft rsi, 64, 4*64, 8*64, rdi, 16*YMM_SCD2, 4
	yloop	4, b5a, -4*64+16*64, rdi, -4*16*YMM_SCD2 ;; Loop 4 times, next source pointer, restore sine/cosine pointer
	yloop	4, b5a, -4*16*64+dist64, rdi, 4*YMM_SCD2 ;; Loop 4 times, next source pointer, next sine/cosine pointer
	yloop	4, b5a, -4*dist64, rdi, -4*4*YMM_SCD2 ;; Test loop counter, restore source pointer, restore sine/cosine pointer
	loops_undo 4*4*4		;; Back up loop counter

	yloop_set_L3_prefetch YHALF	;; Prefetch 480 cache lines in this loop
;;	yr4_eb4cl_four_complex_djbfft_preload
b6a:	yr4_eb4cl_four_complex_djbfft rsi, 4*64, 64, 2*64, rdi, 0, 16
	yloop	16, b6a, -16*4*64+dist64, rdi, 16*YMM_SCD2 ;; Loop 4 times, next source pointer, next sine/cosine pointer
	yloop	4, b6a, -4*dist64, rdi, -4*16*YMM_SCD2 ;; Test loop counter, restore source pointer, restore sine/cosine pointer
	loops_undo 4*16			;; Back up loop counter

	yloop_set_L3_prefetch YOTHER, 32	;; Prefetch 30 cache lines in this loop
	yr4_4cl_four_complex_fft_final_preload
;; Loop below reworked to avoid 4KB distances
;;b7a:	yr4_4cl_four_complex_fft_final rsi, 64, dist64, 2*dist64
;;	yloop	64, b7a, -64*64	;; Loop 64 times, restore source pointer
b7a:	yr4_4cl_four_complex_fft_final rsi, 4*64, dist64, 2*dist64
	yloop	16, b7a, -16*4*64+64	;; Loop 16 times, next source pointer
	yloop	4, b7a, -4*64+4*dist64	;; Loop 4 times, next source pointer

	yloop	15, b3a0		;; Loop 15 times
	end_timer 6
	jmp	ypass2_15360_complex_done


ypass2_15360_complex_3:
	start_timer 6
x64	mov	r8, DEST2ARG
	mov	rbp, DIST_TO_MULSRCARG
	yloop_init 960			;; 15 iters of 256 cache lines

b3c0:	yloop_set_L3_prefetch YHALF	;; Prefetch 480 cache lines in this loop
	yr4_s4cl_four_complex_djbfft_preload
;; Loop below reworked to avoid 4KB distances
;;b3c:	yr4_s4cl_four_complex_djbfft rsi, 64, dist64, 2*dist64, rdi, YMM_SCD2, 64
;;	yloop	64, b3c, -64*64, rdi, -64*YMM_SCD2 ;; Loop 64 times, restore source pointer, restore sine/cosine pointer
b3c:	yr4_s4cl_four_complex_djbfft rsi, 4*64, dist64, 2*dist64, rdi, 4*YMM_SCD2, 16
	yloop	16, b3c, -16*4*64+64, rdi, -16*4*YMM_SCD2+YMM_SCD2 ;; Loop 16 times, next source pointer, next sine/cosine pointer
	yloop	4, b3c, -4*64, rdi, -4*YMM_SCD2 ;; Loop 4 times, restore source pointer, restore sine/cosine pointer
	loops_undo 64			;; Back up loop counter

	yloop_set_L3_prefetch YQUARTER	;; Prefetch 240 cache lines in this loop
	yr4_eb4cl_four_complex_djbfft_preload
b4c:	yr4_eb4cl_four_complex_djbfft rsi, 64, 16*64, 32*64, rdi, 4*YMM_SCD2, 16
	yloop	16, b4c, -16*64+dist64, rdi, -16*4*YMM_SCD2+YMM_SCD2 ;; Loop 16 times, next source pointer, next sine/cosine pointer
	yloop	4, b4c, -4*dist64, rdi, -4*YMM_SCD2 ;; Loop 4 times, restore source pointer, restore sine/cosine pointer
	loops_undo 4*16			;; Back up loop counter

	yloop_set_L3_prefetch YQUARTER	;; Prefetch 240 cache lines in this loop
;;	yr4_eb4cl_four_complex_djbfft_preload
b5c:	yr4_eb4cl_four_complex_djbfft rsi, 64, 4*64, 8*64, rdi, 16*YMM_SCD2, 4
	yloop	4, b5c, -4*64+16*64, rdi, -4*16*YMM_SCD2 ;; Loop 4 times, next source pointer, restore sine/cosine pointer
	yloop	4, b5c, -4*16*64+dist64, rdi, 4*YMM_SCD2 ;; Loop 4 times, next source pointer, next sine/cosine pointer
	yloop	4, b5c, -4*dist64, rdi, -4*4*YMM_SCD2 ;; Test loop counter, restore source pointer, restore sine/cosine pointer
	loops_undo 4*4*4		;; Back up loop counter

	yloop_set_L3_prefetch YQUARTER	;; Prefetch 240 cache lines in this loop
;;	yr4_eb4cl_four_complex_djbfft_preload
b6c:	yr4_eb4cl_four_complex_djbfft rsi, 4*64, 64, 2*64, rdi, 0, 16
	yloop	16, b6c, -16*4*64+dist64, rdi, 16*YMM_SCD2 ;; Loop 4 times, next source pointer, next sine/cosine pointer
	yloop	4, b6c, -4*dist64, rdi, -4*16*YMM_SCD2 ;; Test loop counter, restore source pointer, restore sine/cosine pointer
	loops_undo 4*16			;; Back up loop counter

	yloop_set_L3_prefetch YQUARTER	;; Prefetch 240 cache lines in this loop
	yr4_4cl_four_complex_with_mult_preload
;; Loop below reworked to avoid 4KB distances
;;b7c:	yr4_4cl_four_complex_with_mult rsi, 64, dist64, 2*dist64
;;	yloop	64, b7c, -64*64	;; Loop 64 times, restore source pointer
b7c:	yr4_4cl_four_complex_with_mult rsi, 4*64, dist64, 2*dist64
	yloop	16, b7c, -16*4*64+64	;; Loop 16 times, next source pointer
	yloop	4, b7c, -4*64		;; Loop 4 times, restore source pointer
x64	add	rsi, r8
	loops_undo 64			;; Back up loop counter

	yloop_set_L3_prefetch YQUARTER	;; Prefetch 240 cache lines in this loop
	yr4_eb4cl_four_complex_djbunfft_preload
c6c:	yr4_eb4cl_four_complex_djbunfft rsi, 4*64, 64, 2*64, rdi, 0, 16
	yloop	16, c6c, -16*4*64+dist64, rdi, 16*YMM_SCD2 ;; Loop 4 times, next source pointer, next sine/cosine pointer
	yloop	4, c6c, -4*dist64, rdi, -4*16*YMM_SCD2 ;; Test loop counter, restore source pointer, restore sine/cosine pointer
	loops_undo 4*16			;; Back up loop counter

	yloop_set_L3_prefetch YQUARTER	;; Prefetch 240 cache lines in this loop
;;	yr4_eb4cl_four_complex_djbunfft_preload
c5c:	yr4_eb4cl_four_complex_djbunfft rsi, 64, 4*64, 8*64, rdi, 16*YMM_SCD2, 4
	yloop	4, c5c, -4*64+16*64, rdi, -4*16*YMM_SCD2 ;; Loop 4 times, next source pointer, restore sine/cosine pointer
	yloop	4, c5c, -4*16*64+dist64, rdi, 4*YMM_SCD2 ;; Loop 4 times, next source pointer, next sine/cosine pointer
	yloop	4, c5c, -4*dist64, rdi, -4*4*YMM_SCD2 ;; Test loop counter, restore source pointer, restore sine/cosine pointer
	loops_undo 4*4*4		;; Back up loop counter

	yloop_set_L3_prefetch YHALF	;; Prefetch 480 cache lines in this loop
;;	yr4_eb4cl_four_complex_djbunfft_preload
c4c:	yr4_eb4cl_four_complex_djbunfft rsi, 64, 16*64, 32*64, rdi, 4*YMM_SCD2, 16
	yloop	16, c4c, -16*64+dist64, rdi, -16*4*YMM_SCD2+YMM_SCD2 ;; Loop 16 times, next source pointer, next sine/cosine pointer
	yloop	4, c4c, -4*dist64, rdi, -4*YMM_SCD2 ;; Loop 4 times, restore source pointer, restore sine/cosine pointer
	loops_undo 4*16			;; Back up loop counter

	yloop_set_L3_prefetch YHALF	;; Prefetch 480 cache lines in this loop
	yr4_s4cl_four_complex_djbunfft_preload
;; Loop below reworked to avoid 4KB distances
;;c3c:	yr4_s4cl_four_complex_djbunfft rsi, 64, dist64, 2*dist64, rdi, YMM_SCD2, 64
;;	yloop	64, c3c, -64*64, rdi, -64*YMM_SCD2 ;; Loop 64 times, restore source pointer, restore sine/cosine pointer
c3c:	yr4_s4cl_four_complex_djbunfft rsi, 4*64, dist64, 2*dist64, rdi, 4*YMM_SCD2, 16
	yloop	16, c3c, -16*4*64+64, rdi, -16*4*YMM_SCD2+YMM_SCD2 ;; Loop 16 times, next source pointer, next sine/cosine pointer
	yloop	4, c3c, -4*64+4*dist64, rdi, -4*YMM_SCD2 ;; Loop 4 times, next source pointer, restore sine/cosine pointer

x64	sub	rsi, r8
	yloop	15, b3c0, -15*4*dist64	;; Loop 15 times, restore source pointer
x64	add	rsi, r8
	end_timer 6
	jmp	ypass2_15360_complex_unfft


ypass2_15360_complex_4:
	start_timer 6
	mov	rdi, xsincos_complex	;; The sin/cos ptrs
	bump	rdi, 1280*YMM_SCD1+256*YMM_SCD2
	mov	rbp, DIST_TO_MULSRCARG
	yloop_init 960			;; 15 iters of 256 cache lines

	yloop_set_L3_prefetch YOTHER, 16	;; Prefetch 60 cache lines in this loop
	yr4_4cl_four_complex_with_mulf_preload
;; Loop below reworked to avoid 4KB distances
;;b7d:	yr4_4cl_four_complex_with_mulf rsi, 64, dist64, 2*dist64
;;	yloop	64, b7d, -64*64	;; Loop 64 times, restore source pointer
b7d:	yr4_4cl_four_complex_with_mulf rsi, 4*64, dist64, 2*dist64
	yloop	16, b7d, -16*4*64+64	;; Loop 16 times, next source pointer
	yloop	4, b7d, -4*64		;; Loop 4 times, restore source pointer
	loops_undo 64			;; Back up loop counter

	yloop_set_L3_prefetch YHALF	;; Prefetch 480 cache lines in this loop
	yr4_eb4cl_four_complex_djbunfft_preload
c6d:	yr4_eb4cl_four_complex_djbunfft rsi, 4*64, 64, 2*64, rdi, 0, 16
	yloop	16, c6d, -16*4*64+dist64, rdi, 16*YMM_SCD2 ;; Loop 4 times, next source pointer, next sine/cosine pointer
	yloop	4, c6d, -4*dist64, rdi, -4*16*YMM_SCD2 ;; Test loop counter, restore source pointer, restore sine/cosine pointer
	loops_undo 4*16			;; Back up loop counter

	yloop_set_L3_prefetch YSINGLE	;; Prefetch 960 cache lines in this loop
;;	yr4_eb4cl_four_complex_djbunfft_preload
c5d:	yr4_eb4cl_four_complex_djbunfft rsi, 64, 4*64, 8*64, rdi, 16*YMM_SCD2, 4
	yloop	4, c5d, -4*64+16*64, rdi, -4*16*YMM_SCD2 ;; Loop 4 times, next source pointer, restore sine/cosine pointer
	yloop	4, c5d, -4*16*64+dist64, rdi, 4*YMM_SCD2 ;; Loop 4 times, next source pointer, next sine/cosine pointer
	yloop	4, c5d, -4*dist64, rdi, -4*4*YMM_SCD2 ;; Test loop counter, restore source pointer, restore sine/cosine pointer
	loops_undo 4*4*4		;; Back up loop counter

	yloop_set_L3_prefetch YSINGLE	;; Prefetch 960 cache lines in this loop
;;	yr4_eb4cl_four_complex_djbunfft_preload
c4d:	yr4_eb4cl_four_complex_djbunfft rsi, 64, 16*64, 32*64, rdi, 4*YMM_SCD2, 16
	yloop	16, c4d, -16*64+dist64, rdi, -16*4*YMM_SCD2+YMM_SCD2 ;; Loop 16 times, next source pointer, next sine/cosine pointer
	yloop	4, c4d, -4*dist64, rdi, -4*YMM_SCD2 ;; Loop 4 times, restore source pointer, restore sine/cosine pointer
	loops_undo 4*16			;; Back up loop counter

	yloop_set_L3_prefetch YSINGLE	;; Prefetch 960 cache lines in this loop
	yr4_s4cl_four_complex_djbunfft_preload
;; Loop below reworked to avoid 4KB distances
;;c3d:	yr4_s4cl_four_complex_djbunfft rsi, 64, dist64, 2*dist64, rdi, YMM_SCD2, 64
;;	yloop	64, c3d, -64*64, rdi, -64*YMM_SCD2 ;; Loop 64 times, restore source pointer, restore sine/cosine pointer
c3d:	yr4_s4cl_four_complex_djbunfft rsi, 4*64, dist64, 2*dist64, rdi, 4*YMM_SCD2, 16
	yloop	16, c3d, -16*4*64+64, rdi, -16*4*YMM_SCD2+YMM_SCD2 ;; Loop 16 times, next source pointer, next sine/cosine pointer
	yloop	4, c3d, -4*64+4*dist64, rdi, -4*YMM_SCD2 ;; Loop 4 times, next source pointer, restore sine/cosine pointer

	yloop	15, b7d, -15*4*dist64	;; Loop 15 times, restore source pointer
	bump	rcx, (-60-480-3*960+2*256+480+6*240+2*480)*64 ;; Move prefetch pointer to where squaring would be
	end_timer 6

;; Do inverse FFT levels 3,4
;; On input the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	1024	+1	+1	+1
;;	4	...
;;	...
;;	1020	...
;;	2048	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	5120	+1	+1	+1
;;	4	...
;;	...
;;	5116	...
;;	10240	...
;;	...

;; Do 3072 five_complex_unfft operations
;;	distance between fft data elements is 1024
;;	do 768 macros each processing 40 data values

;; Do inverse FFT levels 1,2
;; On input the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	5120	+1	+1	+1
;;	4	...
;;	...
;;	5116	...
;;	10240	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	15360	+1	+1	+1
;;	4	...
;;	...
;;	15356

;; Do 5120 three_complex_unfft operations
;;	distance between fft data elements is 5120
;;	do 1280 macros each processing 24 data values

ypass2_15360_complex_unfft:
	mov	rdi, xsincos_complex	;; Load sin/cos pointers
	lea	rbp, [rdi+1280*YMM_SCD1]
	bump	rcx, (3840+59-2*256-480-6*240-2*480-2*256)*64 ;; Move prefetch pointer along
	yloop_init 2048			;; 768 iterations or 1280 iterations

c2a:	start_timer 14
	yloop_adjust 5			;; Five fewer iterations this loop
	yloop_set_L3_prefetch YOTHER, 8	;; Prefetch 256 cache lines
	yr5_5cl_five_complex_djbunfft_preload
c2b:	yr5_5cl_five_complex_djbunfft rsi, 20*dist64, 4*dist64, rbp, 0, 3
	yloop	8, c2b, -3*20*dist64, rbp, YMM_SCD2 ;; 3 iterations!!  Restore source pointer, next sine/cosine pointer
	loops_undo 8			;; Restore loop counter
	end_timer 14

	start_timer 15
	yloop_adjust 3			;; Three fewer iterations this loop
	yloop_set_L3_prefetch YOTHER, 8	;; Prefetch 256 cache lines
	yr3_3cl_three_complex_djbunfft_preload
c1b:	yr3_3cl_three_complex_djbunfft rsi, 4*dist64, 20*dist64, rdi, 256*YMM_SCD1, 5, L1PREFETCH_ALL, 64
	yloop	8, c1b, -5*4*dist64+64, rdi, -5*256*YMM_SCD1+YMM_SCD1 ;; 5 iterations!!  Next source pointer, next sine/cosine pointer
	end_timer 15

	yloop	64, c2a, -64*64+dist64	;; Loop 64 times, next source pointer
	yloop	4, c2a			;; Loop 4 times

ypass2_15360_complex_done:
	ENDM


; *************** radix4-pass2-14-levels-real macro ******************
; This macro takes 32768 real values and performs the final 15 levels of the FFT.
; The results are squared, and the first 15 levels of the inverse FFT are done.

yr4dwpn_pass2_14_levels_real MACRO
	LOCAL	b1b, b2b, b3a, b3b, b4a, b4b, b5a, b5b, b6a, b6b, b7a, b7b, b7c, b7d
	LOCAL	c1b, c2b, c3a, c3b, c4a, c4b, c5a, c5b, c6a, c6b
	LOCAL	ypass2_14_levels_real_1, ypass2_14_levels_real_2
	LOCAL	ypass2_14_levels_real_3, ypass2_14_levels_real_4
	LOCAL	ypass2_14_levels_real_unfft, ypass2_14_levels_real_done

;; Type 4 FFTs skip the forward FFT process

	start_timer 2
	mov	rsi, DATA_ADDR		;; Load source address
	mov	rbx, DIST_TO_FFTSRCARG
	mov	rcx, DATA_PREFETCH	;; Prefetch pointer (we need to prefetch 
					;; 4096 cache lines + 63 pad cache lines)
	cmp	ffttype, 4
	je	ypass2_14_levels_real_4

;; Do FFT levels 1,2,3 which correspond to complex levels 1,2
;; On input the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	16384	+1	+1	+1
;;	4	...
;;	...
;;	16380
;; On output the 64-byte cache lines are swizzled to hold these data values:
;;	0	+8K	+8K	+8K	4096	+8K	+8K	+8K
;;	4	...
;;	...
;;	4092	...
;;	1	...
;;	...

;; Do 4096 eight_reals_fft operations
;;	distance between fft data elements is 4096
;;	do 1024 macros each processing 32 data values

	mov	rdi, sincos3		;; Real sin/cos pointer
	mov	rbp, xsincos_complex	;; Complex sin/cos data
	yloop_init 1024			;; 1024 iterations
	yloop_set_L3_prefetch YHALF
	yr4_fs4cl_2sc_eight_reals_fft_preload
b1b:	yr4_fs4cl_2sc_eight_reals_fft rsi, 64, 16*dist64, 32*dist64, rbp, YMM_SCD2, rdi, YMM_SCD2, 64
	yloop	64, b1b, -64*64+dist64	;; Test loop counter, next source pointer
	yloop	16, b1b, -16*dist64	;; Test loop counter, restore source pointer

;; Do FFT levels 3,4
;; On input the 64-byte cache lines hold these data values:
;;	0	+8K	+8K	+8K	4096	+8K	+8K	+8K
;;	4	...
;;	...
;;	4092	...
;;	1	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+8K	+8K	+8K	1024	+8K	+8K	+8K
;;	4	...
;;	...
;;	1020	...
;;	2048	...
;;	...
;;	1	...
;;	...

;; Do 1024 eight_reals_fft operations
;; Do 3072 four_complex_fft operations
;;	distance between fft data elements is 1024
;;	do 1024 macros each processing 32 data values

	yloop_init 1024			;; 1024 iterations
	yloop_set_L3_prefetch YHALF
	yr4_4cl_eight_reals_four_complex_djbfft_preload
b2b:	yr4_4cl_eight_reals_four_complex_djbfft rsi, 16*dist64, 4*dist64, 8*dist64, rdi, YMM_SCD3, 4
	yloop	4, b2b, -4*16*dist64+64	;; Test loop counter, next source pointer
	yloop	64, b2b, -64*64+dist64	;; Test loop counter, next source pointer
	yloop	4, b2b, -4*dist64, rdi, -1024*YMM_SCD3 ;; Test loop counter, restore source pointer, restore sin/cos ptr

;; Do FFT levels 5,6
;; On input the 64-byte cache lines hold these data values:
;;	0	+8K	+8K	+8K	1024	+8K	+8K	+8K
;;	4	...
;;	...
;;	1020	...
;;	2048	...
;;	...
;;	1	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+8K	+8K	+8K	256	+8K	+8K	+8K
;;	4	...
;;	...
;;	252	...
;;	512	...
;;	...
;;	1	...
;;	...

;; Do 256 eight_reals_fft operations
;; Do 3840 four_complex_fft operations
;;	distance between fft data elements is 256
;;	do 1024 macros each processing 32 data values

	yloop_init 1024			;; 4 iterations of 256
	yloop_set_L3_prefetch YHALF
	yr4_4cl_eight_reals_four_complex_djbfft_preload
b3a:	yr4_4cl_eight_reals_four_complex_djbfft rsi, 16*dist64, dist64, 2*dist64, rdi, 4*YMM_SCD3, 4
	yloop	4, b3a, -4*16*dist64+64	;; Test loop counter, next source pointer
	yloop	64, b3a, -64*64+4*dist64, rdi, -256*4*YMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer

	loops_reset			;; 3 more iters of 256
	yr4_rb4cl_four_complex_djbfft_preload
b3b:	yr4_rb4cl_four_complex_djbfft rsi, 16*dist64, dist64, 2*dist64, rdi, 4*YMM_SCD3, 4
	yloop	4, b3b, -4*16*dist64+64	;; Test loop counter, next source pointer
	yloop	64, b3b, -64*64+4*dist64, rdi, -256*4*YMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer
	yloop	4, b3b, -4*4*dist64	;; Test loop counter, restore source pointer

;; Do FFT levels 7,8
;; On input the 64-byte cache lines hold these data values:
;;	0	+8K	+8K	+8K	256	+8K	+8K	+8K
;;	4	...
;;	...
;;	252	...
;;	512	...
;;	...
;;	1	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+8K	+8K	+8K	64	+8K	+8K	+8K
;;	4	...
;;	...
;;	60	...
;;	128	...
;;	...
;;	1	...
;;	...

;; Do 64 eight_reals_fft operations
;; Do 4032 four_complex_fft operations
;;	distance between fft data elements is 64
;;	do 1024 macros each processing 32 data values

	yloop_init 1024			;; 16 iterations of 64
	yloop_set_L3_prefetch YHALF
	yr4_4cl_eight_reals_four_complex_djbfft_preload
b4a:	yr4_4cl_eight_reals_four_complex_djbfft rsi, 16*dist64, 16*64, 32*64, rdi, 16*YMM_SCD3, 4
	yloop	4, b4a, -4*16*dist64+64	;; Test loop counter, next source pointer
	yloop	16, b4a, -16*64+dist64, rdi, -64*16*YMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer

	loops_reset			;; 15 more iters of 64
	yr4_rb4cl_four_complex_djbfft_preload
b4b:	yr4_rb4cl_four_complex_djbfft rsi, 16*dist64, 16*64, 32*64, rdi, 16*YMM_SCD3, 4
	yloop	4, b4b, -4*16*dist64+64	;; Test loop counter, next source pointer
	yloop	16, b4b, -16*64+dist64, rdi, -64*16*YMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer
	yloop	16, b4b, -16*dist64	;; Test loop counter, restore source pointer

;; Do FFT levels 9,10
;; On input the 64-byte cache lines hold these data values:
;;	0	+8K	+8K	+8K	64	+8K	+8K	+8K
;;	4	...
;;	...
;;	60	...
;;	128	...
;;	...
;;	1	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+8K	+8K	+8K	16	+8K	+8K	+8K
;;	4	...
;;	...
;;	12	...
;;	32	...
;;	...
;;	1	...
;;	...

;; Do 16 eight_reals_fft operations
;; Do 4080 four_complex_fft operations
;;	distance between fft data elements is 16
;;	do 1024 macros each processing 32 data values

	yloop_init 1024			;; 64 iterations of 16
	yloop_set_L3_prefetch YHALF
	yr4_4cl_eight_reals_four_complex_djbfft_preload
b5a:	yr4_4cl_eight_reals_four_complex_djbfft rsi, 16*dist64, 4*64, 8*64, rdi, 64*YMM_SCD3, 4
	yloop	4, b5a, -4*16*dist64+64	;; Test loop counter, next source pointer
	yloop	4, b5a, -4*64+16*64, rdi, -16*64*YMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer

	loops_reset			;; 63 more iters of 16
	yr4_rb4cl_four_complex_djbfft_preload
b5b:	yr4_rb4cl_four_complex_djbfft rsi, 16*dist64, 4*64, 8*64, rdi, 64*YMM_SCD3, 4
	yloop	4, b5b, -4*16*dist64+64	;; Test loop counter, next source pointer
	yloop	4, b5b, -4*64+16*64, rdi, -16*64*YMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer
	yloop	4, b5b, -4*16*64+dist64	;; Test loop counter, next source pointer
	yloop	16, b5b, -16*dist64	;; Test loop counter, restore source pointer

;; Do FFT levels 11,12
;; On input the 64-byte cache lines hold these data values:
;;	0	+8K	+8K	+8K	16	+8K	+8K	+8K
;;	4	...
;;	...
;;	12	...
;;	32	...
;;	...
;;	1	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+8K	+8K	+8K	4	+8K	+8K	+8K
;;	8	...
;;	...
;;	1	...
;;	...

;; Do 4 eight_reals_fft operations
;; Do 4092 four_complex_fft operations
;;	distance between fft data elements is 4
;;	do 1024 macros each processing 32 data values

	yloop_init 1024			;; 256 iterations of 4
	yloop_set_L3_prefetch YQUARTER
	yr4_4cl_eight_reals_four_complex_djbfft_preload
b6a:	yr4_4cl_eight_reals_four_complex_djbfft rsi, 16*dist64, 64, 2*64, rdi, 256*YMM_SCD3, 4
	yloop	4, b6a, -4*16*dist64+4*64, rdi, -4*256*YMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer

	loops_reset			;; 255 more iters of 4
	yr4_rb4cl_four_complex_djbfft_preload
b6b:	yr4_rb4cl_four_complex_djbfft rsi, 16*dist64, 64, 2*64, rdi, 256*YMM_SCD3, 4
	yloop	4, b6b, -4*16*dist64+4*64, rdi, -4*256*YMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer
	yloop	16, b6b, -16*4*64+dist64 ;; Test loop counter, next source pointer
	yloop	16, b6b, -16*dist64	;; Test loop counter, restore source pointer

;; Do FFT levels 13,14 as well as inverse FFT levels 13,14
;; On input and output the 64-byte cache lines hold these data values:
;;	0	+8K	+8K	+8K	4	+8K	+8K	+8K
;;	8	...
;;	...
;;	1	...
;;	...

;; Do 1 eight_reals_fft operation
;; Do 4095 four_complex_fft operations
;;	distance between fft data elements is 1
;;	do 1024 macros each processing 32 data values

	cmp	ffttype, 2
	jl	ypass2_14_levels_real_1
	jg	ypass2_14_levels_real_3

ypass2_14_levels_real_2:
;;	yr4_4cl_eight_reals_four_complex_with_square rsi, 64, 16*dist64, 32*dist64
	yr4_dispatch call, yreal_square, 16*dist64, 32*dist64
	yloop_init 1024			;; 1023 more iterations
	yloop_adjust 1
	yloop_set_L3_prefetch YOTHER, 16
	yr4_4cl_four_complex_with_square_preload
b7b:	yr4_4cl_four_complex_with_square rsi, 64, 16*dist64, 32*dist64, 1
	yloop	64, b7b, -64*64+dist64	;; Test loop counter, next source pointer
	yloop	16, b7b, -16*dist64	;; Test loop counter, restore source pointer
	jmp	ypass2_14_levels_real_unfft

ypass2_14_levels_real_1:
;;	yr4_4cl_eight_reals_four_complex_fft_final rsi, 64, 16*dist64, 32*dist64
	yr4_dispatch call, yreal_fft_final, 16*dist64, 32*dist64
	mov	rcx, xsincos_complex	;; Prefetch complex sin/cos data, 2080 cache lines
	yloop_init 1024			;; 1023 more iterations
	yloop_adjust 1
	yloop_set_L3_prefetch YDOUBLE
b7a:	yr4_4cl_four_complex_fft_final rsi, 64, 16*dist64, 32*dist64
	yloop	64, b7a, -64*64+dist64	;; Test loop counter, next source pointer
	yloop	16, b7a			;; Test loop counter
	jmp	ypass2_14_levels_real_done

ypass2_14_levels_real_3:
x64	mov	r8, DEST2ARG
	mov	rbp, DIST_TO_MULSRCARG
;;	yr4_4cl_eight_reals_four_complex_with_mult rsi, 64, 16*dist64, 32*dist64
	yr4_dispatch call, yreal_mult, 16*dist64, 32*dist64
	yloop_init 1024			;; 1023 more iterations
	yloop_adjust 1
	yloop_set_L3_prefetch YOTHER, 16
	yr4_4cl_four_complex_with_mult_preload
b7c:	yr4_4cl_four_complex_with_mult rsi, 64, 16*dist64, 32*dist64
	yloop	64, b7c, -64*64+dist64	;; Test loop counter, next source pointer
	yloop	16, b7c, -16*dist64	;; Test loop counter, restore source pointer
x64	add	rsi, r8
	jmp	ypass2_14_levels_real_unfft

ypass2_14_levels_real_4:
	mov	rbp, DIST_TO_MULSRCARG
;;	yr4_4cl_eight_reals_four_complex_with_mulf rsi, 64, 16*dist64, 32*dist64
	yr4_dispatch call, yreal_mulf, 16*dist64, 32*dist64
	yloop_init 1024			;; 1023 more iterations
	yloop_adjust 1
	yloop_set_L3_prefetch YDOUBLE
	yr4_4cl_four_complex_with_mulf_preload
b7d:	yr4_4cl_four_complex_with_mulf rsi, 64, 16*dist64, 32*dist64
	yloop	64, b7d, -64*64+dist64	;; Test loop counter, next source pointer
	yloop	16, b7d, -16*dist64	;; Test loop counter, restore source pointer
;;	bump	rcx, (-2046+5*512+256+64)*64 ;; Move prefetch pointer back to where squaring would be

;; Do inverse FFT levels 11,12
;; On input the 64-byte cache lines hold these data values:
;;	0	+8K	+8K	+8K	4	+8K	+8K	+8K
;;	8	...
;;	...
;;	1	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+8K	+8K	+8K	16	+8K	+8K	+8K
;;	4	...
;;	...
;;	12	...
;;	32	...
;;	...
;;	1	...
;;	...

;; Do 4 eight_reals_unfft operations
;; Do 4092 four_complex_unfft operations
;;	distance between fft data elements is 4
;;	do 1024 macros each processing 32 data values

ypass2_14_levels_real_unfft:
	mov	rdi, sincos3		;; Load sin/cos pointer
	bump	rdi, 1024*YMM_SCD2
	yloop_init 1024			;; 256 iterations of 4
	yloop_set_L3_prefetch YQUARTER
	yr4_4cl_eight_reals_four_complex_djbunfft_preload
c6a:	yr4_4cl_eight_reals_four_complex_djbunfft rsi, 16*dist64, 64, 2*64, rdi, 256*YMM_SCD3, 4
	yloop	4, c6a, -4*16*dist64+4*64, rdi, -4*256*YMM_SCD3 ;; Test loop counter, next source pointer, restore sine/cosine pointer

	loops_reset			;; 255 more iters of 4
	yr4_rb4cl_four_complex_djbunfft_preload
c6b:	yr4_rb4cl_four_complex_djbunfft rsi, 16*dist64, 64, 2*64, rdi, 256*YMM_SCD3, 4
	yloop	4, c6b, -4*16*dist64+4*64, rdi, -4*256*YMM_SCD3 ;; Test loop counter, next source pointer, restore sine/cosine pointer
	yloop	16, c6b, -16*4*64+dist64	;; Test loop counter, next source pointer
	yloop	16, c6b, -16*dist64	;; Test loop counter, restore source pointer

;; Do inverse FFT levels 9,10
;; On input the 64-byte cache lines hold these data values:
;;	0	+8K	+8K	+8K	16	+8K	+8K	+8K
;;	4	...
;;	...
;;	12	...
;;	32	...
;;	...
;;	1	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+8K	+8K	+8K	64	+8K	+8K	+8K
;;	4	...
;;	...
;;	60	...
;;	128	...
;;	...
;;	1	...
;;	...

;; Do 16 eight_reals_unfft operations
;; Do 4080 four_complex_unfft operations
;;	distance between fft data elements is 16
;;	do 1024 macros each processing 32 data values

	yloop_init 1024			;; 64 iterations of 16
	yloop_set_L3_prefetch YHALF
	yr4_4cl_eight_reals_four_complex_djbunfft_preload
c5a:	yr4_4cl_eight_reals_four_complex_djbunfft rsi, 16*dist64, 4*64, 8*64, rdi, 64*YMM_SCD3, 4
	yloop	4, c5a, -4*16*dist64+64	;; Test loop counter, next source pointer
	yloop	4, c5a, -4*64+16*64, rdi, -16*64*YMM_SCD3 ;; Test loop counter, next source pointer, restore sine/cosine pointer

	loops_reset			;; 63 more iters of 16
	yr4_rb4cl_four_complex_djbunfft_preload
c5b:	yr4_rb4cl_four_complex_djbunfft rsi, 16*dist64, 4*64, 8*64, rdi, 64*YMM_SCD3, 4
	yloop	4, c5b, -4*16*dist64+64	;; Test loop counter, next source pointer
	yloop	4, c5b, -4*64+16*64, rdi, -16*64*YMM_SCD3 ;; Test loop counter, next source pointer, restore sine/cosine pointer
	yloop	4, c5b, -4*16*64+dist64	;; Test loop counter, next source pointer
	yloop	16, c5b, -16*dist64	;; Test loop counter, restore source pointer

;; Do inverse FFT levels 7,8
;; On input the 64-byte cache lines hold these data values:
;;	0	+8K	+8K	+8K	64	+8K	+8K	+8K
;;	4	...
;;	...
;;	60	...
;;	128	...
;;	...
;;	1	...
;;	...
;;
;; On output the 64-byte cache lines hold these data values:
;;	0	+8K	+8K	+8K	256	+8K	+8K	+8K
;;	4	...
;;	...
;;	252	...
;;	512	...
;;	...
;;	1	...
;;	...

;; Do 64 eight_reals_unfft operations
;; Do 4032 four_complex_unfft operations
;;	distance between fft data elements is 64
;;	do 1024 macros each processing 32 data values

	bump	rcx, (4096+63-5*512-256-64-256-512-512)*64 ;; Move prefetch pointer along
	yloop_init 1024			;; 16 iterations of 64
	yloop_set_L3_prefetch YHALF
	yr4_4cl_eight_reals_four_complex_djbunfft_preload
c4a:	yr4_4cl_eight_reals_four_complex_djbunfft rsi, 16*dist64, 16*64, 32*64, rdi, 16*YMM_SCD3, 4
	yloop	4, c4a, -4*16*dist64+64	;; Test loop counter, next source pointer
	yloop	16, c4a, -16*64+dist64, rdi, -64*16*YMM_SCD3 ;; Test loop counter, next source pointer, restore sine/cosine pointer

	loops_reset			;; 15 more iters of 64
	yr4_rb4cl_four_complex_djbunfft_preload
c4b:	yr4_rb4cl_four_complex_djbunfft rsi, 16*dist64, 16*64, 32*64, rdi, 16*YMM_SCD3, 4
	yloop	4, c4b, -4*16*dist64+64	;; Test loop counter, next source pointer
	yloop	16, c4b, -16*64+dist64, rdi, -64*16*YMM_SCD3 ;; Test loop counter, next source pointer, restore sine/cosine pointer
	yloop	16, c4b, -16*dist64	;; Test loop counter, restore source pointer

;; Do inverse FFT levels 5,6
;; On input the 64-byte cache lines hold these data values:
;;	0	+8K	+8K	+8K	256	+8K	+8K	+8K
;;	4	...
;;	...
;;	252	...
;;	512	...
;;	...
;;	1	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+8K	+8K	+8K	1024	+8K	+8K	+8K
;;	4	...
;;	...
;;	1020	...
;;	2048	...
;;	...
;;	1	...
;;	...

;; Do 256 eight_reals_unfft operations
;; Do 3840 four_complex_unfft operations
;;	distance between fft data elements is 256
;;	do 1024 macros each processing 32 data values

	mov	rcx, xsincos_complex	;; Prefetch complex sin/cos data, 2080 cache lines
	yloop_init 1024			;; 4 iterations of 256
	yloop_set_L3_prefetch YOTHER, 32
	yr4_4cl_eight_reals_four_complex_djbunfft_preload
c3a:	yr4_4cl_eight_reals_four_complex_djbunfft rsi, 16*dist64, dist64, 2*dist64, rdi, 4*YMM_SCD3, 4
	yloop	4, c3a, -4*16*dist64+64	;; Test loop counter, next source pointer
	yloop	64, c3a, -64*64+4*dist64, rdi, -256*4*YMM_SCD3 ;; Test loop counter, next source pointer, restore sine/cosine pointer

	loops_reset			;; 3 more iters of 256
	yr4_rb4cl_four_complex_djbunfft_preload
c3b:	yr4_rb4cl_four_complex_djbunfft rsi, 16*dist64, dist64, 2*dist64, rdi, 4*YMM_SCD3, 4
	yloop	4, c3b, -4*16*dist64+64	;; Test loop counter, next source pointer
	yloop	64, c3b, -64*64+4*dist64, rdi, -256*4*YMM_SCD3 ;; Test loop counter, next source pointer, restore sine/cosine pointer
	yloop	4, c3b, -4*4*dist64	;; Test loop counter, restore source pointer

;; Do inverse FFT levels 3,4
;; On input the 64-byte cache lines hold these data values:
;;	0	+8K	+8K	+8K	1024	+8K	+8K	+8K
;;	4	...
;;	...
;;	1020	...
;;	2048	...
;;	...
;;	1	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+8K	+8K	+8K	4096	+8K	+8K	+8K
;;	4	...
;;	...
;;	4092	...
;;	1	...
;;	...

;; Do 1024 eight_reals_unfft operations
;; Do 3072 four_complex_unfft operations
;;	distance between fft data elements is 1024
;;	do 1024 macros each processing 32 data values

	yloop_init 1024			;; 1024 iterations
	yloop_set_L3_prefetch YSINGLE
	yr4_4cl_eight_reals_four_complex_djbunfft_preload
c2b:	yr4_4cl_eight_reals_four_complex_djbunfft rsi, 16*dist64, 4*dist64, 8*dist64, rdi, YMM_SCD3, 4
	yloop	4, c2b, -4*16*dist64+64	;; Test loop counter, next source pointer
	yloop	64, c2b, -64*64+dist64	;; Test loop counter, next source pointer
	yloop	4, c2b, -4*dist64	;; Test loop counter, restore source pointer

;; Do inverse FFT levels 1,2,3 which correspond to complex levels 1,2
;; On input the 64-byte cache lines hold these data values:
;;	0	+8K	+8K	+8K	4096	+8K	+8K	+8K
;;	4	...
;;	...
;;	4092	...
;;	1	...
;;	...
;; On output the 64-byte cache lines are swizzled to hold these data values:
;;	0	+1	+1	+1	16384	+1	+1	+1
;;	4	...
;;	...
;;	16380

;; Do 4096 eight_reals_unfft operations
;;	distance between fft data elements is 4096
;;	do 1024 macros each processing 32 data values

	bump	rdi, -1024*YMM_SCD3-1024*YMM_SCD2 ;; Real sin/cos pointer
	mov	rbp, xsincos_complex	;; Complex sin/cos data
	bump	rcx, (2080-32-1024-1024)*64 ;; Move prefetch pointer along
	yloop_init 1024			;; 1024 iterations
	yloop_set_L3_prefetch YSINGLE
	yr4_s4cl_2sc_eight_reals_unfft_preload
c1b:	yr4_s4cl_2sc_eight_reals_unfft rsi, 64, 16*dist64, 32*dist64, rbp, YMM_SCD2, rdi, YMM_SCD2, 64
	yloop	64, c1b, -64*64+dist64	;; Test loop counter, next source pointer
	yloop	16, c1b			;; Test loop counter

ypass2_14_levels_real_done:
	end_timer 2
	ENDM

; *************** radix4-pass2-14-levels-complex macro ******************
; This macro takes 16384 complex values and performs the final 14 levels of a standard
; radix-4 FFT, squares the results, and does the first 14 levels of the inverse FFT.

yr4dwpn_pass2_14_levels_complex MACRO
	LOCAL	b1a, b1b, b2b, b3b, b4a0, b4a, b4b0, b4b, b4c0, b4c, b5a, b5b, b5c, b6a, b6b, b6c, b7a, b7b, b7c, b7d
	LOCAL	c1b, c2b, c3a, c3b, c4b, c4c, c4d, c5b, c5c, c5d, c6b, c6c, c6d
	LOCAL	ypass2_14_levels_complex_1, ypass2_14_levels_complex_2
	LOCAL	ypass2_14_levels_complex_3, ypass2_14_levels_complex_4
	LOCAL	ypass2_14_levels_complex_unfft, ypass2_14_levels_complex_done

;; Type 4 FFTs skip the forward FFT process

	mov	rsi, DATA_ADDR		;; Load address of FFT data
	mov	rbx, DIST_TO_FFTSRCARG
	mov	rcx, DATA_PREFETCH	;; Prefetch pointer (we need to prefetch 
					;; 4096 cache lines + 63 pad cache lines)
	cmp	ffttype, 4
	je	ypass2_14_levels_complex_4

;; Do FFT level 1,2
;; On input the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	16384	+1	+1	+1
;;	4	...
;;	...
;;	16380
;; On output the 64-byte cache lines are swizzled to hold these data values:
;;	0	+8K	+8K	+8K	4096	+8K	+8K	+8K
;;	4	...
;;	...
;;	4092	...
;;	1	...
;;	...

;; Do 4096 four_complex_fft operations
;;	distance between fft data elements is 4096
;;	do 1024 macros each processing 32 data values

;; Do FFT levels 3,4
;; On input the 64-byte cache lines hold these data values:
;;	0	+8K	+8K	+8K	4096	+8K	+8K	+8K
;;	4	...
;;	...
;;	4092	...
;;	1	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+8K	+8K	+8K	1024	+8K	+8K	+8K
;;	4	...
;;	...
;;	1020	...
;;	2048	...
;;	...
;;	1	...
;;	...

;; Do 4096 four_complex_fft operations
;;	distance between fft data elements is 1024
;;	do 1024 macros each processing 32 data values

;; Do FFT levels 5,6
;; On input the 64-byte cache lines hold these data values:
;;	0	+8K	+8K	+8K	1024	+8K	+8K	+8K
;;	4	...
;;	...
;;	1020	...
;;	2048	...
;;	...
;;	1	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+8K	+8K	+8K	256	+8K	+8K	+8K
;;	4	...
;;	...
;;	252	...
;;	512	...
;;	...
;;	1	...
;;	...

;; Do 4096 four_complex_fft operations
;;	distance between fft data elements is 256
;;	do 1024 macros each processing 32 data values

	mov	rdi, xsincos_complex	;; The sin/cos ptrs
	mov	rbp, rdi
	mov	rdx, rdi
	yloop_init 1024			;; 1024 iterations

b1a:	start_timer 3
	yloop_set_L3_prefetch YHALF	;; Prefetch 512 cache lines in this loop
	yr4_fs4cl_four_complex_djbfft_preload
b1b:	yr4_fs4cl_four_complex_djbfft rsi, dist64, 16*dist64, 32*dist64, rdi, 64*YMM_SCD2, 16
	yloop	16, b1b, -16*dist64, rdi, -16*64*YMM_SCD2+YMM_SCD2 ;; Test loop counter, restore source pointer, next sin/cos data
	loops_undo 16			;; Back up loop counter
	end_timer 3

	start_timer 4
	yloop_set_L3_prefetch YHALF	;; Prefetch 512 cache lines in this loop
	yr4_eb4cl_four_complex_djbfft_preload
b2b:	yr4_eb4cl_four_complex_djbfft rsi, dist64, 4*dist64, 8*dist64, rbp, 256*YMM_SCD2, 4
	yloop	4, b2b, -4*dist64+16*dist64, rbp, -4*256*YMM_SCD2+YMM_SCD2 ;; Test loop counter, next source pointer, next sin/cos data
	yloop	4, b2b, -4*16*dist64	;; Test loop counter, restore source pointer
	loops_undo 4*4			;; Back up loop counter
	end_timer 4

	start_timer 5
	yloop_set_L3_prefetch YQUARTER	;; Prefetch 256 cache lines in this loop
;;	yr4_eb4cl_four_complex_djbfft_preload
b3b:	yr4_eb4cl_four_complex_djbfft rsi, 4*dist64, dist64, 2*dist64, rdx, 0, 4, L1PREFETCH_ALL, 64
	yloop	4, b3b, 0, rdx, 4*YMM_SCD2 ;; Loop 4 times, next sine/cosine pointer
	yloop	4, b3b, -4*4*4*dist64+64	;; Loop 4 times, next source pointer
	end_timer 5

	yloop	64, b1a, -64*64, rdi, -64*YMM_SCD2+1024*YMM_SCD2 ;; Test loop counter, restore source pointer, next sin/cos ptr

;; Do FFT levels 7,8
;; On input the 64-byte cache lines hold these data values:
;;	0	+8K	+8K	+8K	256	+8K	+8K	+8K
;;	4	...
;;	...
;;	252	...
;;	512	...
;;	...
;;	1	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+8K	+8K	+8K	64	+8K	+8K	+8K
;;	4	...
;;	...
;;	60	...
;;	128	...
;;	...
;;	1	...
;;	...

;; Do 4096 four_complex_fft operations
;;	distance between fft data elements is 64
;;	do 1024 macros each processing 32 data values

;; Do FFT levels 9,10
;; On input the 64-byte cache lines hold these data values:
;;	0	+8K	+8K	+8K	64	+8K	+8K	+8K
;;	4	...
;;	...
;;	60	...
;;	128	...
;;	...
;;	1	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+8K	+8K	+8K	16	+8K	+8K	+8K
;;	4	...
;;	...
;;	12	...
;;	32	...
;;	...
;;	1	...
;;	...

;; Do 4096 four_complex_fft operations
;;	distance between fft data elements is 16
;;	do 1024 macros each processing 32 data values

;; Do FFT levels 11,12
;; On input the 64-byte cache lines hold these data values:
;;	0	+8K	+8K	+8K	16	+8K	+8K	+8K
;;	4	...
;;	...
;;	12	...
;;	32	...
;;	...
;;	1	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+8K	+8K	+8K	4	+8K	+8K	+8K
;;	8	...
;;	...
;;	1	...
;;	...

;; Do 4096 four_complex_fft operations
;;	distance between fft data elements is 4
;;	do 1024 macros each processing 32 data values

;; Do FFT levels 13,14 as well as inverse FFT levels 13,14
;; On input and output the 64-byte cache lines hold these data values:
;;	0	+8K	+8K	+8K	4	+8K	+8K	+8K
;;	8	...
;;	...
;;	1	...
;;	...

;; Do 4096 four_complex_fft operations
;;	distance between fft data elements is 1
;;	do 1024 macros each processing 32 data values

;; Do inverse FFT levels 11,12
;; On input the 64-byte cache lines hold these data values:
;;	0	+8K	+8K	+8K	4	+8K	+8K	+8K
;;	8	...
;;	...
;;	1	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+8K	+8K	+8K	16	+8K	+8K	+8K
;;	4	...
;;	...
;;	12	...
;;	32	...
;;	...
;;	1	...
;;	...

;; Do 4096 four_complex_unfft operations
;;	distance between fft data elements is 4
;;	do 1024 macros each processing 32 data values

;; Do inverse FFT levels 9,10
;; On input the 64-byte cache lines hold these data values:
;;	0	+8K	+8K	+8K	16	+8K	+8K	+8K
;;	4	...
;;	...
;;	12	...
;;	32	...
;;	...
;;	1	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+8K	+8K	+8K	64	+8K	+8K	+8K
;;	4	...
;;	...
;;	60	...
;;	128	...
;;	...
;;	1	...
;;	...

;; Do 4096 four_complex_unfft operations
;;	distance between fft data elements is 16
;;	do 1024 macros each processing 32 data values

;; Do inverse FFT levels 7,8
;; On input the 64-byte cache lines hold these data values:
;;	0	+8K	+8K	+8K	64	+8K	+8K	+8K
;;	4	...
;;	...
;;	60	...
;;	128	...
;;	...
;;	1	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+8K	+8K	+8K	256	+8K	+8K	+8K
;;	4	...
;;	...
;;	252	...
;;	512	...
;;	...
;;	1	...
;;	...

;; Do 4096 four_complex_unfft operations
;;	distance between fft data elements is 64
;;	do 1024 macros each processing 32 data values

;; Do this 256 cache lines (16KB) at a time to maximize accessing data
;; while it is in the L1 cache.

;; Execute the right middle step

	cmp	ffttype, 2
	jl	ypass2_14_levels_complex_1
	jg	ypass2_14_levels_complex_3

ypass2_14_levels_complex_2:
	yloop_init 1024			;; 16 iters of 256 cache lines

b4b0:	start_timer 6
	yloop_set_L3_prefetch YQUARTER	;; Prefetch 256 cache lines in this loop
	yr4_b4cl_four_complex_djbfft_preload
b4b:	yr4_b4cl_four_complex_djbfft rsi, 64, 16*64, 32*64, rdi, YMM_SCD2/4, 16
	yloop	16, b4b, -16*64+16*dist64	;; Loop 16 times, next source pointer
	yloop	4, b4b, -4*16*dist64, rdi, -64*YMM_SCD2/4 ;; Loop 4 times, restore source pointer, restore sine/cosine pointer
	loops_undo 4*16			;; Back up loop counter
	end_timer 6

	start_timer 7
	yloop_set_L3_prefetch YQUARTER	;; Prefetch 256 cache lines in this loop
;;	yr4_b4cl_four_complex_djbfft_preload
b5b:	yr4_b4cl_four_complex_djbfft rsi, 64, 4*64, 8*64, rdi, 4*YMM_SCD2/4, 4
	yloop	4, b5b, -4*64+16*64, rdi, -4*4*YMM_SCD2/4 ;; Loop 4 times, next source pointer, restore sine/cosine pointer
	yloop	4, b5b, -4*16*64+16*dist64, rdi, YMM_SCD2/4 ;; Loop 4 times, next source pointer, next sine/cosine pointer
	yloop	4, b5b, -4*16*dist64, rdi, -4*YMM_SCD2/4 ;; Loop 4 times, restore source pointer, next sine/cosine pointer
	loops_undo 4*4*4		;; Back up loop counter
	end_timer 7

	start_timer 8
	yloop_set_L3_prefetch YQUARTER	;; Prefetch 256 cache lines in this loop
;;	yr4_b4cl_four_complex_djbfft_preload
b6b:	yr4_b4cl_four_complex_djbfft rsi, 4*64, 64, 2*64, rdi, 0, 16
	yloop	16, b6b, -16*4*64+16*dist64, rdi, 4*YMM_SCD2/4 ;; Loop 16 times, next source pointer, next sine/cosine pointer
	yloop	4, b6b, -4*16*dist64, rdi, -4*4*YMM_SCD2/4 ;; Loop 4 times, restore source pointer, restore sine/cosine pointer
	loops_undo 4*16			;; Back up loop counter
	end_timer 8

	start_timer 9
	yloop_set_L3_prefetch YOTHER, 16	;; Prefetch 64 cache lines in this loop
	yr4_4cl_four_complex_with_square_preload
b7b:	yr4_4cl_four_complex_with_square rsi, 64, 16*dist64, 32*dist64, 64
	yloop	64, b7b, -64*64	;; Loop 64 times, restore source pointer
	loops_undo 64			;; Back up loop counter
	end_timer 9

	start_timer 10
	yloop_set_L3_prefetch YQUARTER	;; Prefetch 256 cache lines in this loop
	yr4_b4cl_four_complex_djbunfft_preload
c6b:	yr4_b4cl_four_complex_djbunfft rsi, 4*64, 64, 2*64, rdi, 0, 16
	yloop	16, c6b, -16*4*64+16*dist64, rdi, 4*YMM_SCD2/4 ;; Loop 16 times, next source pointer, next sine/cosine pointer
	yloop	4, c6b, -4*16*dist64, rdi, -4*4*YMM_SCD2/4 ;; Loop 4 times, restore source pointer, restore sine/cosine pointer
	loops_undo 4*16			;; Restore loop counter
	end_timer 10

	start_timer 11
	yloop_set_L3_prefetch YQUARTER	;; Prefetch 256 cache lines in this loop
;;	yr4_b4cl_four_complex_djbunfft_preload
c5b:	yr4_b4cl_four_complex_djbunfft rsi, 64, 4*64, 8*64, rdi, 4*YMM_SCD2/4, 4
	yloop	4, c5b, -4*64+16*64, rdi, -4*4*YMM_SCD2/4 ;; Loop 4 times, next source pointer, restore sine/cosine pointer
	yloop	4, c5b, -4*16*64+16*dist64, rdi, YMM_SCD2/4 ;; Loop 4 times, next source pointer, next sine/cosine pointer
	yloop	4, c5b, -4*16*dist64, rdi, -4*YMM_SCD2/4 ;; Loop 4 times, restore source pointer, next sine/cosine pointer
	loops_undo 4*4*4		;; Back up loop counter
	end_timer 11

	start_timer 12
	yloop_set_L3_prefetch YQUARTER	;; Prefetch 256 cache lines in this loop
;;	yr4_b4cl_four_complex_djbunfft_preload
c4b:	yr4_b4cl_four_complex_djbunfft rsi, 64, 16*64, 32*64, rdi, YMM_SCD2/4, 16
	yloop	16, c4b, -16*64+16*dist64	;; Loop 16 times, next source pointer
	yloop	4, c4b, -4*16*dist64+dist64, rdi, -64*YMM_SCD2/4 ;; Loop 4 times, next source pointer, restore sine/cosine pointer
	end_timer 12

	yloop	16, b4b0, -16*dist64	;; Loop 16 times, restore source pointer
	jmp	ypass2_14_levels_complex_unfft


ypass2_14_levels_complex_1:
	start_timer 6
	bump	rcx, (4096+63-2*512-256-2*1024-2*512)*64 ;; Move prefetch pointer along
	yloop_init 1024			;; 16 iters of 256 cache lines

b4a0:	yloop_set_L3_prefetch YSINGLE	;; Prefetch 1024 cache lines in this loop
	yr4_b4cl_four_complex_djbfft_preload
b4a:	yr4_b4cl_four_complex_djbfft rsi, 64, 16*64, 32*64, rdi, YMM_SCD2/4, 16
	yloop	16, b4a, -16*64+16*dist64 ;; Loop 16 times, next source pointer
	yloop	4, b4a, -4*16*dist64, rdi, -64*YMM_SCD2/4 ;; Loop 4 times, restore source pointer, restore sine/cosine pointer
	loops_undo 4*16			;; Back up loop counter

	yloop_set_L3_prefetch YSINGLE	;; Prefetch 1024 cache lines in this loop
;;	yr4_b4cl_four_complex_djbfft_preload
b5a:	yr4_b4cl_four_complex_djbfft rsi, 64, 4*64, 8*64, rdi, 4*YMM_SCD2/4, 4
	yloop	4, b5a, -4*64+16*64, rdi, -4*4*YMM_SCD2/4 ;; Loop 4 times, next source pointer, restore sine/cosine pointer
	yloop	4, b5a, -4*16*64+16*dist64, rdi, YMM_SCD2/4 ;; Loop 4 times, next source pointer, next sine/cosine pointer
	yloop	4, b5a, -4*16*dist64, rdi, -4*YMM_SCD2/4 ;; Loop 4 times, restore source pointer, next sine/cosine pointer
	loops_undo 4*4*4		;; Back up loop counter

	yloop_set_L3_prefetch YHALF	;; Prefetch 512 cache lines in this loop
;;	yr4_b4cl_four_complex_djbfft_preload
b6a:	yr4_b4cl_four_complex_djbfft rsi, 4*64, 64, 2*64, rdi, 0, 16
	yloop	16, b6a, -16*4*64+16*dist64, rdi, 4*YMM_SCD2/4 ;; Loop 16 times, next source pointer, next sine/cosine pointer
	yloop	4, b6a, -4*16*dist64, rdi, -4*4*YMM_SCD2/4 ;; Loop 4 times, restore source pointer, restore sine/cosine pointer
	loops_undo 4*16			;; Back up loop counter

	yloop_set_L3_prefetch YHALF	;; Prefetch 512 cache lines in this loop
	yr4_4cl_four_complex_fft_final_preload
b7a:	yr4_4cl_four_complex_fft_final rsi, 64, 16*dist64, 32*dist64
	yloop	64, b7a, -64*64+dist64	;; Loop 64 times, next source pointer

	yloop	16, b4a0		;; Loop 16 times
	end_timer 6
	jmp	ypass2_14_levels_complex_done


ypass2_14_levels_complex_3:
	start_timer 6
x64	mov	r8, DEST2ARG
	mov	rbp, DIST_TO_MULSRCARG
	yloop_init 1024			;; 16 iters of 256 cache lines

b4c0:	yloop_set_L3_prefetch YQUARTER	;; Prefetch 256 cache lines in this loop
	yr4_b4cl_four_complex_djbfft_preload
b4c:	yr4_b4cl_four_complex_djbfft rsi, 64, 16*64, 32*64, rdi, YMM_SCD2/4, 16
	yloop	16, b4c, -16*64+16*dist64 ;; Loop 16 times, next source pointer
	yloop	4, b4c, -4*16*dist64, rdi, -64*YMM_SCD2/4 ;; Loop 4 times, restore source pointer, restore sine/cosine pointer
	loops_undo 4*16			;; Back up loop counter

	yloop_set_L3_prefetch YQUARTER	;; Prefetch 256 cache lines in this loop
;;	yr4_b4cl_four_complex_djbfft_preload
b5c:	yr4_b4cl_four_complex_djbfft rsi, 64, 4*64, 8*64, rdi, 4*YMM_SCD2/4, 4
	yloop	4, b5c, -4*64+16*64, rdi, -4*4*YMM_SCD2/4 ;; Loop 4 times, next source pointer, restore sine/cosine pointer
	yloop	4, b5c, -4*16*64+16*dist64, rdi, YMM_SCD2/4 ;; Loop 4 times, next source pointer, next sine/cosine pointer
	yloop	4, b5c, -4*16*dist64, rdi, -4*YMM_SCD2/4 ;; Loop 4 times, restore source pointer, next sine/cosine pointer
	loops_undo 4*4*4		;; Back up loop counter

	yloop_set_L3_prefetch YQUARTER	;; Prefetch 256 cache lines in this loop
;;	yr4_b4cl_four_complex_djbfft_preload
b6c:	yr4_b4cl_four_complex_djbfft rsi, 4*64, 64, 2*64, rdi, 0, 16
	yloop	16, b6c, -16*4*64+16*dist64, rdi, 4*YMM_SCD2/4 ;; Loop 16 times, next source pointer, next sine/cosine pointer
	yloop	4, b6c, -4*16*dist64, rdi, -4*4*YMM_SCD2/4 ;; Loop 4 times, restore source pointer, restore sine/cosine pointer
	loops_undo 4*16			;; Back up loop counter

	yloop_set_L3_prefetch YOTHER, 16	;; Prefetch 64 cache lines in this loop
	yr4_4cl_four_complex_with_mult_preload
b7c:	yr4_4cl_four_complex_with_mult rsi, 64, 16*dist64, 32*dist64
	yloop	64, b7c, -64*64		;; Loop 64 times, restore source pointer
x64	add	rsi, r8
	loops_undo 64			;; Back up loop counter

	yloop_set_L3_prefetch YQUARTER	;; Prefetch 256 cache lines in this loop
	yr4_b4cl_four_complex_djbunfft_preload
c6c:	yr4_b4cl_four_complex_djbunfft rsi, 4*64, 64, 2*64, rdi, 0, 16
	yloop	16, c6c, -16*4*64+16*dist64, rdi, 4*YMM_SCD2/4 ;; Loop 16 times, next source pointer, next sine/cosine pointer
	yloop	4, c6c, -4*16*dist64, rdi, -4*4*YMM_SCD2/4 ;; Loop 4 times, restore source pointer, restore sine/cosine pointer
	loops_undo 4*16			;; Restore loop counter

	yloop_set_L3_prefetch YQUARTER	;; Prefetch 256 cache lines in this loop
;;	yr4_b4cl_four_complex_djbunfft_preload
c5c:	yr4_b4cl_four_complex_djbunfft rsi, 64, 4*64, 8*64, rdi, 4*YMM_SCD2/4, 4
	yloop	4, c5c, -4*64+16*64, rdi, -4*4*YMM_SCD2/4 ;; Loop 4 times, next source pointer, restore sine/cosine pointer
	yloop	4, c5c, -4*16*64+16*dist64, rdi, YMM_SCD2/4 ;; Loop 4 times, next source pointer, next sine/cosine pointer
	yloop	4, c5c, -4*16*dist64, rdi, -4*YMM_SCD2/4 ;; Loop 4 times, restore source pointer, next sine/cosine pointer
	loops_undo 4*4*4		;; Back up loop counter

	yloop_set_L3_prefetch YQUARTER	;; Prefetch 256 cache lines in this loop
;;	yr4_b4cl_four_complex_djbunfft_preload
c4c:	yr4_b4cl_four_complex_djbunfft rsi, 64, 16*64, 32*64, rdi, YMM_SCD2/4, 16
	yloop	16, c4c, -16*64+16*dist64 ;; Loop 16 times, next source pointer
	yloop	4, c4c, -4*16*dist64+dist64, rdi, -64*YMM_SCD2/4 ;; Loop 4 times, next source pointer, restore sine/cosine pointer

x64	sub	rsi, r8
	yloop	16, b4c0, -16*dist64	;; Loop 16 times, restore source pointer
x64	add	rsi, r8
	end_timer 6
	jmp	ypass2_14_levels_complex_unfft


ypass2_14_levels_complex_4:
	start_timer 6
	mov	rdi, xsincos_complex	;; The sin/cos ptrs
	bump	rdi, 1024*YMM_SCD2
	mov	rbp, DIST_TO_MULSRCARG
	yloop_init 1024			;; 16 iters of 256 cache lines

	yloop_set_L3_prefetch YHALF	;; Prefetch 512 cache lines in this loop
	yr4_4cl_four_complex_with_mulf_preload
b7d:	yr4_4cl_four_complex_with_mulf rsi, 64, 16*dist64, 32*dist64
	yloop	64, b7d, -64*64		;; Loop 64 times, restore source pointer
	loops_undo 64			;; Back up loop counter

	yloop_set_L3_prefetch YHALF	;; Prefetch 512 cache lines in this loop
	yr4_b4cl_four_complex_djbunfft_preload
c6d:	yr4_b4cl_four_complex_djbunfft rsi, 4*64, 64, 2*64, rdi, 0, 16
	yloop	16, c6d, -16*4*64+16*dist64, rdi, 4*YMM_SCD2/4 ;; Loop 16 times, next source pointer, next sine/cosine pointer
	yloop	4, c6d, -4*16*dist64, rdi, -4*4*YMM_SCD2/4 ;; Loop 4 times, restore source pointer, restore sine/cosine pointer
	loops_undo 4*16			;; Restore loop counter

	yloop_set_L3_prefetch YSINGLE	;; Prefetch 1024 cache lines in this loop
;;	yr4_b4cl_four_complex_djbunfft_preload
c5d:	yr4_b4cl_four_complex_djbunfft rsi, 64, 4*64, 8*64, rdi, 4*YMM_SCD2/4, 4
	yloop	4, c5d, -4*64+16*64, rdi, -4*4*YMM_SCD2/4 ;; Loop 4 times, next source pointer, restore sine/cosine pointer
	yloop	4, c5d, -4*16*64+16*dist64, rdi, YMM_SCD2/4 ;; Loop 4 times, next source pointer, next sine/cosine pointer
	yloop	4, c5d, -4*16*dist64, rdi, -4*YMM_SCD2/4 ;; Loop 4 times, restore source pointer, next sine/cosine pointer
	loops_undo 4*4*4		;; Back up loop counter

	yloop_set_L3_prefetch YSINGLE	;; Prefetch 1024 cache lines in this loop
;;	yr4_b4cl_four_complex_djbunfft_preload
c4d:	yr4_b4cl_four_complex_djbunfft rsi, 64, 16*64, 32*64, rdi, YMM_SCD2/4, 16
	yloop	16, c4d, -16*64+16*dist64	;; Loop 16 times, next source pointer
	yloop	4, c4d, -4*16*dist64+dist64, rdi, -64*YMM_SCD2/4 ;; Loop 4 times, next source pointer, restore sine/cosine pointer

	yloop	16, b7d, -16*dist64	;; Loop 16 times, restore source pointer
	bump	rcx, (-2*512-2*1024+2*512+4*256+64+3*256)*64 ;; Move prefetch pointer to where squaring would be
	end_timer 6

;; Do inverse FFT levels 5,6
;; On input the 64-byte cache lines hold these data values:
;;	0	+8K	+8K	+8K	256	+8K	+8K	+8K
;;	4	...
;;	...
;;	252	...
;;	512	...
;;	...
;;	1	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+8K	+8K	+8K	1024	+8K	+8K	+8K
;;	4	...
;;	...
;;	1020	...
;;	2048	...
;;	...
;;	1	...
;;	...
;;
;; Do 4096 four_complex_unfft operations
;;	distance between fft data elements is 256
;;	do 1024 macros each processing 32 data values

;; Do inverse FFT levels 3,4
;; On input the 64-byte cache lines hold these data values:
;;	0	+8K	+8K	+8K	1024	+8K	+8K	+8K
;;	4	...
;;	...
;;	1020	...
;;	2048	...
;;	...
;;	1	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+8K	+8K	+8K	4096	+8K	+8K	+8K
;;	4	...
;;	...
;;	4092	...
;;	1	...
;;	...

;; Do 4096 four_complex_unfft operations
;;	distance between fft data elements is 1024
;;	do 1024 macros each processing 32 data values

;; Do inverse FFT levels 1,2
;; On input the 64-byte cache lines hold these data values:
;;	0	+8K	+8K	+8K	4096	+8K	+8K	+8K
;;	4	...
;;	...
;;	4092	...
;;	1	...
;;	...
;; On output the 64-byte cache lines are swizzled to hold these data values:
;;	0	+1	+1	+1	16384	+1	+1	+1
;;	4	...
;;	...
;;	16380

;; Do 4096 four_complex_unfft operations
;;	distance between fft data elements is 4096
;;	do 1024 macros each processing 32 data values

ypass2_14_levels_complex_unfft:
	mov	rdi, xsincos_complex	;; Load sin/cos pointers
	mov	rbp, rdi
	mov	rdx, rdi
	bump	rcx, (4096+63-2*512-4*256-64-3*256-256-2*512)*64 ;; Move prefetch pointer along
	yloop_init 1024			;; 1024 iterations

c3a:	start_timer 13
	yloop_set_L3_prefetch YQUARTER	;; Prefetch 256 cache lines in this loop
	yr4_eb4cl_four_complex_djbunfft_preload
c3b:	yr4_eb4cl_four_complex_djbunfft rsi, 4*dist64, dist64, 2*dist64, rdx, 0, 4
	yloop	4, c3b, 0, rdx, 4*YMM_SCD2 ;; Loop 4 times, next sine/cosine pointer
	yloop	4, c3b, -4*4*4*dist64	;; Loop 4 times, restore source pointer
	loops_undo 4*4			;; Back up loop counter
	end_timer 13

	start_timer 14
	yloop_set_L3_prefetch YHALF	;; Prefetch 512 cache lines in this loop
;;	yr4_eb4cl_four_complex_djbunfft_preload
c2b:	yr4_eb4cl_four_complex_djbunfft rsi, dist64, 4*dist64, 8*dist64, rbp, 256*YMM_SCD2, 4
	yloop	4, c2b, -4*dist64+16*dist64, rbp, -4*256*YMM_SCD2+YMM_SCD2 ;; Test loop counter, next source pointer, next sine/cosine pointer
	yloop	4, c2b, -4*16*dist64	;; Test loop counter, restore source pointer
	loops_undo 4*4			;; Back up loop counter
	end_timer 14

	start_timer 15
	yloop_set_L3_prefetch YHALF	;; Prefetch 512 cache lines in this loop
	yr4_s4cl_four_complex_djbunfft_preload
c1b:	yr4_s4cl_four_complex_djbunfft rsi, dist64, 16*dist64, 32*dist64, rdi, 64*YMM_SCD2, 16, L1PREFETCH_ALL, 64
	yloop	16, c1b, -16*dist64+64, rdi, -16*64*YMM_SCD2+YMM_SCD2 ;; Test loop counter, next source pointer, next sine/cosine pointer
	end_timer 15

	yloop	64, c3a			;; Loop 64 times

ypass2_14_levels_complex_done:
	ENDM


; *************** radix4-pass2-20480-real macro ******************
; This macro takes 40960 real values and performs the final 15.322 levels of the FFT.
; The results are squared, and the first 15.322 levels of the inverse FFT are done.

yr4dwpn_pass2_20480_real MACRO
	LOCAL	b1b, b2a, b2b, b3a, b3b, b4a, b4b, b5a, b5b, b6a, b6b, b7a, b7b, b7c, b7d
	LOCAL	c1b, c2a, c2b, c3a, c3b, c4a, c4b, c5a, c5b, c6a, c6b
	LOCAL	ypass2_20480_real_1, ypass2_20480_real_2
	LOCAL	ypass2_20480_real_3, ypass2_20480_real_4
	LOCAL	ypass2_20480_real_unfft, ypass2_20480_real_done

;; Type 4 FFTs skip the forward FFT process

	start_timer 2
	mov	rsi, DATA_ADDR		;; Load source address
	mov	rbx, DIST_TO_FFTSRCARG
	mov	rcx, DATA_PREFETCH	;; Prefetch pointer (we need to prefetch 
					;; 5120 cache lines + 79 pad cache lines)
	cmp	ffttype, 4
	je	ypass2_20480_real_4

;; Do FFT levels 1,2,3 which correspond to complex levels 1,2
;; On input the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	20480	+1	+1	+1
;;	4	...
;;	...
;;	20476
;; On output the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	4096	+1	+1	+1
;;	4	...
;;	...
;;	4092	...
;;	8192	...
;;	...

;; Do 4096 ten_reals_fft operations
;;	distance between fft data elements is 4096
;;	do 1024 macros each processing 40 data values

	mov	rdi, sincos3		;; Real sin/cos pointer
	mov	rbp, xsincos_complex	;; Complex sin/cos data
	yloop_init 1024			;; 1024 iterations
	yloop_set_L3_prefetch YHALF
	yr5_f5cl_2sc_ten_reals_fft_preload
b1b:	yr5_f5cl_2sc_ten_reals_fft rsi, 64, 16*dist64, rbp, YMM_SCD2, rdi, YMM_SCD2, 64
	yloop	64, b1b, -64*64+dist64	;; Test loop counter, next source pointer
	yloop	16, b1b, -16*dist64	;; Test loop counter, restore source pointer

;; Do FFT levels 3,4
;; On input the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	4096	+1	+1	+1
;;	4	...
;;	...
;;	4092	...
;;	8192	...
;;	...
;; On output the 64-byte cache lines are swizzled to hold these data values:
;;	0	+2K	+2K	+2K	1024	+2K	+2K	+2K
;;	4	...
;;	...
;;	1020	...
;;	1	...
;;	...
;;	8192	...
;;	...

;; Do 1024 eight_reals_fft operations
;; Do 4096 four_complex_fft operations
;;	distance between fft data elements is 1024
;;	do 1280 macros each processing 32 data values

	yloop_init 1280			;; 5 iterations of 256
	yloop_set_L3_prefetch YHALF
	yr4_s4cl_2sc_eight_reals_fft_preload
b2a:	yr4_s4cl_2sc_eight_reals_fft rsi, 64, 4*dist64, 8*dist64, rbp, YMM_SCD2, rdi, YMM_SCD2, 64
	yloop	64, b2a, -64*64+dist64	;; Test loop counter, next source pointer
	yloop	4, b2a, -4*dist64+16*dist64, rbp, -256*YMM_SCD2 ;; Test loop counter, next source pointer, restore sine/cosine pointer

	loops_reset			;; 4 more iters of 256
	yr4_s4cl_four_complex_djbfft_preload
b2b:	yr4_s4cl_four_complex_djbfft rsi, 64, 4*dist64, 8*dist64, rbp, YMM_SCD2, 64
	yloop	64, b2b, -64*64+dist64	;; Test loop counter, next source pointer
	yloop	4, b2b, -4*dist64+16*dist64, rbp, -256*YMM_SCD2 ;; Test loop counter, next source pointer, restore sin/cos ptr
	yloop	5, b2b, -5*16*dist64	;; Loop 5 times, restore source pointer

;; Do FFT levels 5,6
;; On input the 64-byte cache lines hold these data values:
;;	0	+2K	+2K	+2K	1024	+2K	+2K	+2K
;;	4	...
;;	...
;;	1020	...
;;	1	...
;;	...
;;	8192	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+2K	+2K	+2K	256	+2K	+2K	+2K
;;	4	...
;;	...
;;	252	...
;;	512	...
;;	...
;;	1	...
;;	...
;;	8192	...
;;	...

;; Do 256 eight_reals_fft operations
;; Do 4864 four_complex_fft operations
;;	distance between fft data elements is 256
;;	do 1280 macros each processing 32 data values

	yloop_init 1280			;; 5 iterations of 256
	yloop_set_L3_prefetch YHALF
	yr4_4cl_eight_reals_four_complex_djbfft_preload
b3a:	yr4_4cl_eight_reals_four_complex_djbfft rsi, 4*dist64, dist64, 2*dist64, rdi, YMM_SCD3, 4
	yloop	4, b3a, -4*4*dist64+64	;; Test loop counter, next source pointer
	yloop	64, b3a, -64*64+16*dist64, rdi, -256*YMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer

	loops_reset			;; 4 more iters of 256
	yr4_rb4cl_four_complex_djbfft_preload
b3b:	yr4_rb4cl_four_complex_djbfft rsi, 4*dist64, dist64, 2*dist64, rdi, YMM_SCD3, 4
	yloop	4, b3b, -4*4*dist64+64	;; Test loop counter, next source pointer
	yloop	64, b3b, -64*64+16*dist64, rdi, -256*YMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer
	yloop	5, b3b, -5*16*dist64	;; Test loop counter, restore source pointer

;; Do FFT levels 7,8
;; On input the 64-byte cache lines hold these data values:
;;	0	+2K	+2K	+2K	256	+2K	+2K	+2K
;;	4	...
;;	...
;;	252	...
;;	512	...
;;	...
;;	1	...
;;	...
;;	8192	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+2K	+2K	+2K	64	+2K	+2K	+2K
;;	4	...
;;	...
;;	60	...
;;	128	...
;;	...
;;	1	...
;;	...
;;	8192	...
;;	...

;; Do 64 eight_reals_fft operations
;; Do 5056 four_complex_fft operations
;;	distance between fft data elements is 64
;;	do 1280 macros each processing 32 data values

	yloop_init 1280			;; 20 iterations of 64
	yloop_set_L3_prefetch YHALF
	yr4_4cl_eight_reals_four_complex_djbfft_preload
b4a:	yr4_4cl_eight_reals_four_complex_djbfft rsi, 4*dist64, 16*64, 32*64, rdi, 4*YMM_SCD3, 4
	yloop	4, b4a, -4*4*dist64+64	;; Test loop counter, next source pointer
	yloop	16, b4a, -16*64+dist64, rdi, -64*4*YMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer

	loops_reset			;; 19 more iters of 64
	yr4_rb4cl_four_complex_djbfft_preload
b4b:	yr4_rb4cl_four_complex_djbfft rsi, 4*dist64, 16*64, 32*64, rdi, 4*YMM_SCD3, 4
	yloop	4, b4b, -4*4*dist64+64	;; Test loop counter, next source pointer
	yloop	16, b4b, -16*64+dist64, rdi, -64*4*YMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer
	yloop	4, b4b, -4*dist64+16*dist64 ;; Test loop counter, next source pointer
	yloop	5, b4b, -5*16*dist64	;; Test loop counter, restore source pointer

;; Do FFT levels 9,10
;; On input the 64-byte cache lines hold these data values:
;;	0	+2K	+2K	+2K	64	+2K	+2K	+2K
;;	4	...
;;	...
;;	60	...
;;	128	...
;;	...
;;	1	...
;;	...
;;	8192	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+2K	+2K	+2K	16	+2K	+2K	+2K
;;	4	...
;;	...
;;	12	...
;;	32	...
;;	...
;;	1	...
;;	...
;;	8192	...
;;	...

;; Do 16 eight_reals_fft operations
;; Do 5104 four_complex_fft operations
;;	distance between fft data elements is 16
;;	do 1280 macros each processing 32 data values

	yloop_init 1280			;; 80 iterations of 16
	yloop_set_L3_prefetch YHALF
	yr4_4cl_eight_reals_four_complex_djbfft_preload
b5a:	yr4_4cl_eight_reals_four_complex_djbfft rsi, 4*dist64, 4*64, 8*64, rdi, 16*YMM_SCD3, 4
	yloop	4, b5a, -4*4*dist64+64	;; Test loop counter, next source pointer
	yloop	4, b5a, -4*64+16*64, rdi, -16*16*YMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer

	loops_reset			;; 79 more iters of 16
	yr4_rb4cl_four_complex_djbfft_preload
b5b:	yr4_rb4cl_four_complex_djbfft rsi, 4*dist64, 4*64, 8*64, rdi, 16*YMM_SCD3, 4
	yloop	4, b5b, -4*4*dist64+64	;; Test loop counter, next source pointer
	yloop	4, b5b, -4*64+16*64, rdi, -16*16*YMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer
	yloop	4, b5b, -4*16*64+dist64	;; Test loop counter, next source pointer
	yloop	4, b5b, -4*dist64+16*dist64 ;; Test loop counter, next source pointer
	yloop	5, b5b, -5*16*dist64	;; Test loop counter, restore source pointer

;; Do FFT levels 11,12
;; On input the 64-byte cache lines hold these data values:
;;	0	+2K	+2K	+2K	16	+2K	+2K	+2K
;;	4	...
;;	...
;;	12	...
;;	32	...
;;	...
;;	1	...
;;	...
;;	8192	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+2K	+2K	+2K	4	+2K	+2K	+2K
;;	8	...
;;	...
;;	1	...
;;	...
;;	8192	...
;;	...

;; Do 4 eight_reals_fft operations
;; Do 5116 four_complex_fft operations
;;	distance between fft data elements is 4
;;	do 1280 macros each processing 32 data values

	yloop_init 1280			;; 320 iterations of 4
	yloop_set_L3_prefetch YQUARTER
	yr4_4cl_eight_reals_four_complex_djbfft_preload
b6a:	yr4_4cl_eight_reals_four_complex_djbfft rsi, 4*dist64, 64, 2*64, rdi, 64*YMM_SCD3, 4
	yloop	4, b6a, -4*4*dist64+4*64, rdi, -4*64*YMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer

	loops_reset			;; 319 more iters of 4
	yr4_rb4cl_four_complex_djbfft_preload
b6b:	yr4_rb4cl_four_complex_djbfft rsi, 4*dist64, 64, 2*64, rdi, 64*YMM_SCD3, 4
	yloop	4, b6b, -4*4*dist64+4*64, rdi, -4*64*YMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer
	yloop	16, b6b, -16*4*64+dist64 ;; Test loop counter, next source pointer
	yloop	4, b6b, -4*dist64+16*dist64 ;; Test loop counter, next source pointer
	yloop	5, b6b, -5*16*dist64	;; Test loop counter, restore source pointer

;; Do FFT levels 13,14 as well as inverse FFT levels 13,14
;; On input and output the 64-byte cache lines hold these data values:
;;	0	+2K	+2K	+2K	4	+2K	+2K	+2K
;;	8	...
;;	...
;;	1	...
;;	...
;;	8192	...
;;	...

;; Do 1 eight_reals_fft operation
;; Do 5119 four_complex_fft operations
;;	distance between fft data elements is 1
;;	do 1280 macros each processing 32 data values

	cmp	ffttype, 2
	jl	ypass2_20480_real_1
	jg	ypass2_20480_real_3

ypass2_20480_real_2:
;;	yr4_4cl_eight_reals_four_complex_with_square rsi, 64, 4*dist64, 8*dist64
	yr4_dispatch call, yreal_square, 4*dist64, 8*dist64
	yloop_init 1280			;; 1279 more iterations
	yloop_adjust 1
	yloop_set_L3_prefetch YQUARTER
	yr4_4cl_four_complex_with_square_preload
b7b:	yr4_4cl_four_complex_with_square rsi, 64, 4*dist64, 8*dist64, 1
	yloop	64, b7b, -64*64+dist64	;; Test loop counter, next source pointer
	yloop	4, b7b, -4*dist64+16*dist64	;; Test loop counter, next source pointer
	yloop	5, b7b, -5*16*dist64	;; Test loop counter, restore source pointer
	jmp	ypass2_20480_real_unfft

ypass2_20480_real_1:
;;	yr4_4cl_eight_reals_four_complex_fft_final rsi, 64, 4*dist64, 8*dist64
	yr4_dispatch call, yreal_fft_final, 4*dist64, 8*dist64
	yloop_init 1280			;; 1279 more iterations
	yloop_adjust 1
	yloop_set_L3_prefetch YDOUBLE
b7a:	yr4_4cl_four_complex_fft_final rsi, 64, 4*dist64, 8*dist64
	yloop	64, b7a, -64*64+dist64	;; Test loop counter, next source pointer
	yloop	4, b7a, -4*dist64+16*dist64	;; Test loop counter, next source pointer
	yloop	5, b7a			;; Test loop counter
	jmp	ypass2_20480_real_done

ypass2_20480_real_3:
x64	mov	r8, DEST2ARG
	mov	rbp, DIST_TO_MULSRCARG
;;	yr4_4cl_eight_reals_four_complex_with_mult rsi, 64, 4*dist64, 8*dist64
	yr4_dispatch call, yreal_mult, 4*dist64, 8*dist64
	yloop_init 1280			;; 1279 more iterations
	yloop_adjust 1
	yloop_set_L3_prefetch YQUARTER
	yr4_4cl_four_complex_with_mult_preload
b7c:	yr4_4cl_four_complex_with_mult rsi, 64, 4*dist64, 8*dist64
	yloop	64, b7c, -64*64+dist64	;; Test loop counter, next source pointer
	yloop	4, b7c, -4*dist64+16*dist64	;; Test loop counter, next source pointer
	yloop	5, b7c, -5*16*dist64	;; Test loop counter, restore source pointer
x64	add	rsi, r8
	jmp	ypass2_20480_real_unfft

ypass2_20480_real_4:
	mov	rbp, DIST_TO_MULSRCARG
;;	yr4_4cl_eight_reals_four_complex_with_mulf rsi, 64, 4*dist64, 8*dist64
	yr4_dispatch call, yreal_mulf, 4*dist64, 8*dist64
	yloop_init 1280			;; 1279 more iterations
	yloop_adjust 1
	yloop_set_L3_prefetch YDOUBLE
	yr4_4cl_four_complex_with_mulf_preload
b7d:	yr4_4cl_four_complex_with_mulf rsi, 64, 4*dist64, 8*dist64
	yloop	64, b7d, -64*64+dist64	;; Test loop counter, next source pointer
	yloop	4, b7d, -4*dist64+16*dist64	;; Test loop counter, next source pointer
	yloop	5, b7d, -5*16*dist64	;; Test loop counter, restore source pointer

;; Do inverse FFT levels 11,12
;; On input the 64-byte cache lines hold these data values:
;;	0	+2K	+2K	+2K	4	+2K	+2K	+2K
;;	8	...
;;	...
;;	1	...
;;	...
;;	8192	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+2K	+2K	+2K	16	+2K	+2K	+2K
;;	4	...
;;	...
;;	12	...
;;	32	...
;;	...
;;	1	...
;;	...
;;	8192	...
;;	...

;; Do 4 eight_reals_unfft operations
;; Do 5116 four_complex_unfft operations
;;	distance between fft data elements is 4
;;	do 1280 macros each processing 32 data values

ypass2_20480_real_unfft:
	mov	rdi, sincos3		;; Load sin/cos pointer
	bump	rdi, 256*YMM_SCD2+1024*YMM_SCD2
	yloop_init 1280			;; 320 iterations of 4
	yloop_set_L3_prefetch YQUARTER
	yr4_4cl_eight_reals_four_complex_djbunfft_preload
c6a:	yr4_4cl_eight_reals_four_complex_djbunfft rsi, 4*dist64, 64, 2*64, rdi, 64*YMM_SCD3, 4
	yloop	4, c6a, -4*4*dist64+4*64, rdi, -4*64*YMM_SCD3 ;; Test loop counter, next source pointer, restore sine/cosine pointer

	loops_reset			;; 319 more iters of 4
	yr4_rb4cl_four_complex_djbunfft_preload
c6b:	yr4_rb4cl_four_complex_djbunfft rsi, 4*dist64, 64, 2*64, rdi, 64*YMM_SCD3, 4
	yloop	4, c6b, -4*4*dist64+4*64, rdi, -4*64*YMM_SCD3 ;; Test loop counter, next source pointer, restore sine/cosine pointer
	yloop	16, c6b, -16*4*64+dist64	;; Test loop counter, next source pointer
	yloop	4, c6b, -4*dist64+16*dist64	;; Test loop counter, next source pointer
	yloop	5, c6b, -5*16*dist64	;; Test loop counter, restore source pointer

;; Do inverse FFT levels 9,10
;; On input the 64-byte cache lines hold these data values:
;;	0	+2K	+2K	+2K	16	+2K	+2K	+2K
;;	4	...
;;	...
;;	12	...
;;	32	...
;;	...
;;	1	...
;;	...
;;	8192	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+2K	+2K	+2K	64	+2K	+2K	+2K
;;	4	...
;;	...
;;	60	...
;;	128	...
;;	...
;;	1	...
;;	...
;;	8192	...
;;	...

;; Do 16 eight_reals_unfft operations
;; Do 5104 four_complex_unfft operations
;;	distance between fft data elements is 16
;;	do 1280 macros each processing 32 data values

	yloop_init 1280			;; 80 iterations of 16
	yloop_set_L3_prefetch YHALF
	yr4_4cl_eight_reals_four_complex_djbunfft_preload
c5a:	yr4_4cl_eight_reals_four_complex_djbunfft rsi, 4*dist64, 4*64, 8*64, rdi, 16*YMM_SCD3, 4
	yloop	4, c5a, -4*4*dist64+64	;; Test loop counter, next source pointer
	yloop	4, c5a, -4*64+16*64, rdi, -16*16*YMM_SCD3 ;; Test loop counter, next source pointer, restore sine/cosine pointer

	loops_reset			;; 79 more iters of 16
	yr4_rb4cl_four_complex_djbunfft_preload
c5b:	yr4_rb4cl_four_complex_djbunfft rsi, 4*dist64, 4*64, 8*64, rdi, 16*YMM_SCD3, 4
	yloop	4, c5b, -4*4*dist64+64	;; Test loop counter, next source pointer
	yloop	4, c5b, -4*64+16*64, rdi, -16*16*YMM_SCD3 ;; Test loop counter, next source pointer, restore sine/cosine pointer
	yloop	4, c5b, -4*16*64+dist64	;; Test loop counter, next source pointer
	yloop	4, c5b, -4*dist64+16*dist64	;; Test loop counter, next source pointer
	yloop	5, c5b, -5*16*dist64	;; Test loop counter, restore source pointer

;; Do inverse FFT levels 7,8
;; On input the 64-byte cache lines hold these data values:
;;	0	+2K	+2K	+2K	64	+2K	+2K	+2K
;;	4	...
;;	...
;;	60	...
;;	128	...
;;	...
;;	1	...
;;	...
;;	8192	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+2K	+2K	+2K	256	+2K	+2K	+2K
;;	4	...
;;	...
;;	252	...
;;	512	...
;;	...
;;	1	...
;;	...
;;	8192	...
;;	...

;; Do 64 eight_reals_unfft operations
;; Do 5056 four_complex_unfft operations
;;	distance between fft data elements is 64
;;	do 1280 macros each processing 32 data values

	bump	rcx, (5120+79-512-4*640-3*320-640-640)*64 ;; Move prefetch pointer along
	yloop_init 1280			;; 20 iterations of 64
	yloop_set_L3_prefetch YHALF
	yr4_4cl_eight_reals_four_complex_djbunfft_preload
c4a:	yr4_4cl_eight_reals_four_complex_djbunfft rsi, 4*dist64, 16*64, 32*64, rdi, 4*YMM_SCD3, 4
	yloop	4, c4a, -4*4*dist64+64	;; Test loop counter, next source pointer
	yloop	16, c4a, -16*64+dist64, rdi, -64*4*YMM_SCD3 ;; Test loop counter, next source pointer, restore sine/cosine pointer

	loops_reset			;; 19 more iters of 64
	yr4_rb4cl_four_complex_djbunfft_preload
c4b:	yr4_rb4cl_four_complex_djbunfft rsi, 4*dist64, 16*64, 32*64, rdi, 4*YMM_SCD3, 4
	yloop	4, c4b, -4*4*dist64+64	;; Test loop counter, next source pointer
	yloop	16, c4b, -16*64+dist64, rdi, -64*4*YMM_SCD3 ;; Test loop counter, next source pointer, restore sine/cosine pointer
	yloop	4, c4b, -4*dist64+16*dist64	;; Test loop counter, next source pointer
	yloop	5, c4b, -5*16*dist64	;; Test loop counter, restore source pointer

;; Do inverse FFT levels 5,6
;; On input the 64-byte cache lines hold these data values:
;;	0	+2K	+2K	+2K	256	+2K	+2K	+2K
;;	4	...
;;	...
;;	252	...
;;	512	...
;;	...
;;	1	...
;;	...
;;	8192	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+2K	+2K	+2K	1024	+2K	+2K	+2K
;;	4	...
;;	...
;;	1020	...
;;	1	...
;;	...
;;	8192	...
;;	...

;; Do 256 eight_reals_unfft operations
;; Do 4864 four_complex_unfft operations
;;	distance between fft data elements is 256
;;	do 1280 macros each processing 32 data values

	mov	rcx, xsincos_complex	;; Prefetch complex sin/cos data, 2592 cache lines
	yloop_init 1280			;; 5 iterations of 256
	yloop_set_L3_prefetch YSINGLE
	yr4_4cl_eight_reals_four_complex_djbunfft_preload
c3a:	yr4_4cl_eight_reals_four_complex_djbunfft rsi, 4*dist64, dist64, 2*dist64, rdi, YMM_SCD3, 4
	yloop	4, c3a, -4*4*dist64+64	;; Test loop counter, next source pointer
	yloop	64, c3a, -64*64+16*dist64, rdi, -256*YMM_SCD3 ;; Test loop counter, next source pointer, restore sine/cosine pointer

	loops_reset			;; 4 more iters of 256
	yr4_rb4cl_four_complex_djbunfft_preload
c3b:	yr4_rb4cl_four_complex_djbunfft rsi, 4*dist64, dist64, 2*dist64, rdi, YMM_SCD3, 4
	yloop	4, c3b, -4*4*dist64+64	;; Test loop counter, next source pointer
	yloop	64, c3b, -64*64+16*dist64, rdi, -256*YMM_SCD3 ;; Test loop counter, next source pointer, restore sine/cosine pointer
	yloop	5, c3b, -5*16*dist64	;; Test loop counter, restore source pointer

;; Do inverse FFT levels 3,4
;; On input the 64-byte cache lines hold these data values:
;;	0	+2K	+2K	+2K	1024	+2K	+2K	+2K
;;	4	...
;;	...
;;	1020	...
;;	1	...
;;	...
;;	8192	...
;;	...
;; On output the 64-byte cache lines are swizzled to hold these data values:
;;	0	+1	+1	+1	4096	+1	+1	+1
;;	4	...
;;	...
;;	4092	...
;;	8192	...
;;	...

;; Do 1024 eight_reals_unfft operations
;; Do 4096 four_complex_unfft operations
;;	distance between fft data elements is 1024
;;	do 1280 macros each processing 32 data values

	bump	rdi, -256*YMM_SCD2	;; Real sin/cos ptr
	mov	rbp, xsincos_complex	;; Complex sin/cos data
	bump	rbp, 1024*YMM_SCD2
	yloop_init 1280			;; 5 iterations of 256
	yloop_set_L3_prefetch YSINGLE
	yr4_s4cl_2sc_eight_reals_unfft_preload
c2a:	yr4_s4cl_2sc_eight_reals_unfft rsi, 64, 4*dist64, 8*dist64, rbp, YMM_SCD2, rdi, YMM_SCD2, 64
	yloop	64, c2a, -64*64+dist64	;; Test loop counter, next source pointer
	yloop	4, c2a, -4*dist64+16*dist64, rbp, -256*YMM_SCD2 ;; Test loop counter, next source pointer, restore sine/cosine pointer

	loops_reset			;; 4 more iters of 256
	yr4_s4cl_four_complex_djbunfft_preload
c2b:	yr4_s4cl_four_complex_djbunfft rsi, 64, 4*dist64, 8*dist64, rbp, YMM_SCD2, 64
	yloop	64, c2b, -64*64+dist64	;; Test loop counter, next source pointer
	yloop	4, c2b, -4*dist64+16*dist64, rbp, -256*YMM_SCD2 ;; Test loop counter, next source pointer, restore sine/cosine pointer
	yloop	5, c2b, -5*16*dist64	;; Test loop counter, restore source pointer

;; Do inverse FFT levels 1,2,3 which correspond to complex levels 1,2
;; On input the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	4096	+1	+1	+1
;;	4	...
;;	...
;;	4092	...
;;	8192	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	20480	+1	+1	+1
;;	4	...
;;	...
;;	20476

;; Do 4096 ten_reals_unfft operations
;;	distance between fft data elements is 4096
;;	do 1024 macros each processing 40 data values

	bump	rdi, -256*YMM_SCD2-1024*YMM_SCD2 ;; Real sin/cos ptr
	bump	rbp, -1024*YMM_SCD2	;; Complex sin/cos data
	bump	rcx, (2592-2*1280-32)*64 ;; Move prefetch pointer along
	yloop_init 1024			;; 1024 iterations
	yloop_set_L3_prefetch YOTHER, 32
	yr5_5cl_2sc_ten_reals_unfft_preload
c1b:	yr5_5cl_2sc_ten_reals_unfft rsi, 64, 16*dist64, rbp, YMM_SCD2, rdi, YMM_SCD2, 64
	yloop	64, c1b, -64*64+dist64	;; Test loop counter, next source pointer
	yloop	16, c1b			;; Test loop counter

ypass2_20480_real_done:
	end_timer 2
	ENDM

; *************** radix4-pass2-20480-complex macro ******************
; This macro takes 20480 complex values and performs the final 14.322 levels of a standard
; radix-4 FFT, squares the results, and does the first 14.322 levels of the inverse FFT.

yr4dwpn_pass2_20480_complex MACRO
	LOCAL	b1a, b1b, b2b, b3b, b4a0, b4a, b4b0, b4b, b4c0, b4c, b5a, b5b, b5c, b6a, b6b, b6c, b7a, b7b, b7c, b7d
	LOCAL	c1b, c2b, c3a, c3b, c4b, c4c, c4d, c5b, c5c, c5d, c6b, c6c, c6d
	LOCAL	ypass2_20480_complex_1, ypass2_20480_complex_2
	LOCAL	ypass2_20480_complex_3, ypass2_20480_complex_4
	LOCAL	ypass2_20480_complex_unfft, ypass2_20480_complex_done

;; Type 4 FFTs skip the forward FFT process

	mov	rsi, DATA_ADDR		;; Load address of FFT data
	mov	rbx, DIST_TO_FFTSRCARG
	mov	rcx, DATA_PREFETCH	;; Prefetch pointer (we need to prefetch 
					;; 5120 cache lines + 79 pad cache lines)
	cmp	ffttype, 4
	je	ypass2_20480_complex_4

;; Do FFT level 1,2
;; On input the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	20480	+1	+1	+1
;;	4	...
;;	...
;;	20476
;; On output the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	4096	+1	+1	+1
;;	4	...
;;	...
;;	4092	...
;;	8192	...
;;	...

;; Do 4096 five_complex_fft operations
;;	distance between fft data elements is 4096
;;	do 1024 macros each processing 40 data values

;; Do FFT levels 3,4
;; On input the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	4096	+1	+1	+1
;;	4	...
;;	...
;;	4092	...
;;	8192	...
;;	...
;; On output the 64-byte cache lines are swizzled to hold these data values:
;;	0	+2K	+2K	+2K	1024	+2K	+2K	+2K
;;	4	...
;;	...
;;	1020	...
;;	1	...
;;	...
;;	8192	...
;;	...

;; Do 5120 four_complex_fft operations
;;	distance between fft data elements is 1024
;;	do 1280 macros each processing 32 data values

;; Do FFT levels 5,6
;; On input the 64-byte cache lines hold these data values:
;;	0	+2K	+2K	+2K	1024	+2K	+2K	+2K
;;	4	...
;;	...
;;	1020	...
;;	1	...
;;	...
;;	8192	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+2K	+2K	+2K	256	+2K	+2K	+2K
;;	4	...
;;	...
;;	252	...
;;	512	...
;;	...
;;	1	...
;;	...
;;	8192	...
;;	...

;; Do 5120 four_complex_fft operations
;;	distance between fft data elements is 256
;;	do 1280 macros each processing 32 data values

	mov	rdi, xsincos_complex	;; The sin/cos ptrs
	lea	rbp, [rdi+1024*YMM_SCD2]
	mov	rdx, rbp
	yloop_init 2048			;; 1024 iterations or 1280 iterations

b1a:	start_timer 3
	yloop_adjust 16			;; 16 fewer iterations this loop
	yloop_set_L3_prefetch YQUARTER	;; Prefetch 256 cache lines in this loop
	yr5_f5cl_five_complex_djbfft_preload
b1b:	yr5_f5cl_five_complex_djbfft rsi, dist64, 16*dist64, rdi, 64*YMM_SCD2, 16
	yloop	32, b1b, -16*dist64, rdi, -16*64*YMM_SCD2+YMM_SCD2 ;; 16 iterations!!  Restore source pointer, next sin/cos data
	loops_undo 32			;; Back up loop counter
	end_timer 3

	start_timer 4
	yloop_adjust 3*4		;; Fewer iterations this loop
	yloop_set_L3_prefetch YHALF	;; Prefetch 640 cache lines in this loop
	yr4_s4cl_four_complex_djbfft_preload
b2b:	yr4_s4cl_four_complex_djbfft rsi, dist64, 4*dist64, 8*dist64, rbp, 64*YMM_SCD2, 4
	yloop	4, b2b, -4*dist64+16*dist64, rbp, -4*64*YMM_SCD2 ;; Test loop counter, next source pointer, restore sin/cos data
	yloop	8, b2b, -5*16*dist64, rbp, YMM_SCD2 ;; 5 iterations!!  Restore source pointer, next sin/cos data
	loops_undo 8*4			;; Back up loop counter
	end_timer 4

	start_timer 5
	yloop_adjust 3*4		;; Fewer iterations this loop
	yloop_set_L3_prefetch YQUARTER	;; Prefetch 320 cache lines in this loop
	yr4_eb4cl_four_complex_djbfft_preload
b3b:	yr4_eb4cl_four_complex_djbfft rsi, 4*dist64, dist64, 2*dist64, rdx, YMM_SCD2, 4, L1PREFETCH_ALL, 64
	yloop	4, b3b, 0, rdx, -4*YMM_SCD2 ;; Test loop counter, restore sine/cosine pointer
	yloop	8, b3b, -5*4*4*dist64+64, rdx, 4*YMM_SCD2 ;; 5 iterations!!  Next source pointer, next sine/cosine pointer
	end_timer 5

	yloop	64, b1a, -64*64, rdi, -64*YMM_SCD2+1024*YMM_SCD2+256*YMM_SCD2 ;; Test loop counter, restore source pointer, next sin/cos ptr

;; Do FFT levels 7,8
;; On input the 64-byte cache lines hold these data values:
;;	0	+2K	+2K	+2K	256	+2K	+2K	+2K
;;	4	...
;;	...
;;	252	...
;;	512	...
;;	...
;;	1	...
;;	...
;;	8192	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+2K	+2K	+2K	64	+2K	+2K	+2K
;;	4	...
;;	...
;;	60	...
;;	128	...
;;	...
;;	1	...
;;	...
;;	8192	...
;;	...

;; Do 5120 four_complex_fft operations
;;	distance between fft data elements is 64
;;	do 1280 macros each processing 32 data values

;; Do FFT levels 9,10
;; On input the 64-byte cache lines hold these data values:
;;	0	+2K	+2K	+2K	64	+2K	+2K	+2K
;;	4	...
;;	...
;;	60	...
;;	128	...
;;	...
;;	1	...
;;	...
;;	8192	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+2K	+2K	+2K	16	+2K	+2K	+2K
;;	4	...
;;	...
;;	12	...
;;	32	...
;;	...
;;	1	...
;;	...
;;	8192	...
;;	...

;; Do 5120 four_complex_fft operations
;;	distance between fft data elements is 16
;;	do 1280 macros each processing 32 data values

;; Do FFT levels 11,12
;; On input the 64-byte cache lines hold these data values:
;;	0	+2K	+2K	+2K	16	+2K	+2K	+2K
;;	4	...
;;	...
;;	12	...
;;	32	...
;;	...
;;	1	...
;;	...
;;	8192	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+2K	+2K	+2K	4	+2K	+2K	+2K
;;	8	...
;;	...
;;	1	...
;;	...
;;	8192	...
;;	...

;; Do 5120 four_complex_fft operations
;;	distance between fft data elements is 4
;;	do 1280 macros each processing 32 data values

;; Do FFT levels 13,14 as well as inverse FFT levels 13,14
;; On input and output the 64-byte cache lines hold these data values:
;;	0	+2K	+2K	+2K	4	+2K	+2K	+2K
;;	8	...
;;	...
;;	1	...
;;	...
;;	8192	...
;;	...

;; Do 5120 four_complex_fft operations
;;	distance between fft data elements is 1
;;	do 1280 macros each processing 32 data values

;; Do inverse FFT levels 11,12
;; On input the 64-byte cache lines hold these data values:
;;	0	+2K	+2K	+2K	4	+2K	+2K	+2K
;;	8	...
;;	...
;;	1	...
;;	...
;;	8192	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+2K	+2K	+2K	16	+2K	+2K	+2K
;;	4	...
;;	...
;;	12	...
;;	32	...
;;	...
;;	1	...
;;	...
;;	8192	...
;;	...

;; Do 5120 four_complex_unfft operations
;;	distance between fft data elements is 4
;;	do 1280 macros each processing 32 data values

;; Do inverse FFT levels 9,10
;; On input the 64-byte cache lines hold these data values:
;;	0	+2K	+2K	+2K	16	+2K	+2K	+2K
;;	4	...
;;	...
;;	12	...
;;	32	...
;;	...
;;	1	...
;;	...
;;	8192	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+2K	+2K	+2K	64	+2K	+2K	+2K
;;	4	...
;;	...
;;	60	...
;;	128	...
;;	...
;;	1	...
;;	...
;;	8192	...
;;	...

;; Do 5120 four_complex_unfft operations
;;	distance between fft data elements is 16
;;	do 1280 macros each processing 32 data values

;; Do inverse FFT levels 7,8
;; On input the 64-byte cache lines hold these data values:
;;	0	+2K	+2K	+2K	64	+2K	+2K	+2K
;;	4	...
;;	...
;;	60	...
;;	128	...
;;	...
;;	1	...
;;	...
;;	8192	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+2K	+2K	+2K	256	+2K	+2K	+2K
;;	4	...
;;	...
;;	252	...
;;	512	...
;;	...
;;	1	...
;;	...
;;	8192	...
;;	...

;; Do 5120 four_complex_unfft operations
;;	distance between fft data elements is 64
;;	do 1280 macros each processing 32 data values

;; Do this 256 cache lines (16KB) at a time to maximize accessing data
;; while it is in the L1 cache.

;; Execute the right middle step

	cmp	ffttype, 2
	jl	ypass2_20480_complex_1
	jg	ypass2_20480_complex_3

ypass2_20480_complex_2:
	yloop_init 1280			;; 20 iters of 256 cache lines
	yloop_set_L3_prefetch YQUARTER

b4b0:	start_timer 6
	yr4_b4cl_four_complex_djbfft_preload
b4b:	yr4_b4cl_four_complex_djbfft rsi, 64, 16*64, 32*64, rdi, YMM_SCD2/4, 16
	yloop	16, b4b, -16*64+4*dist64	;; Loop 16 times, next source pointer
	yloop	4, b4b, -4*4*dist64, rdi, -64*YMM_SCD2/4 ;; Loop 4 times, restore source pointer, restore sine/cosine pointer
	loops_undo 4*16			;; Back up loop counter
	end_timer 6

	start_timer 7
;;	yr4_b4cl_four_complex_djbfft_preload
b5b:	yr4_b4cl_four_complex_djbfft rsi, 64, 4*64, 8*64, rdi, 4*YMM_SCD2/4, 4
	yloop	4, b5b, -4*64+16*64, rdi, -4*4*YMM_SCD2/4 ;; Loop 4 times, next source pointer, restore sine/cosine pointer
	yloop	4, b5b, -4*16*64+4*dist64, rdi, YMM_SCD2/4 ;; Loop 4 times, next source pointer, next sine/cosine pointer
	yloop	4, b5b, -4*4*dist64, rdi, -4*YMM_SCD2/4 ;; Loop 4 times, restore source pointer, next sine/cosine pointer
	loops_undo 4*4*4		;; Back up loop counter
	end_timer 7

	start_timer 8
;;	yr4_b4cl_four_complex_djbfft_preload
b6b:	yr4_b4cl_four_complex_djbfft rsi, 4*64, 64, 2*64, rdi, 0, 16
	yloop	16, b6b, -16*4*64+4*dist64, rdi, 4*YMM_SCD2/4 ;; Loop 16 times, next source pointer, next sine/cosine pointer
	yloop	4, b6b, -4*4*dist64, rdi, -4*4*YMM_SCD2/4 ;; Loop 4 times, restore source pointer, restore sine/cosine pointer
	loops_undo 4*16			;; Back up loop counter
	end_timer 8

	start_timer 9
	yr4_4cl_four_complex_with_square_preload
b7b:	yr4_4cl_four_complex_with_square rsi, 64, 4*dist64, 8*dist64, 64
	yloop	64, b7b, -64*64		;; Loop 64 times, restore source pointer
	loops_undo 64			;; Back up loop counter
	end_timer 9

	start_timer 10
	yr4_b4cl_four_complex_djbunfft_preload
c6b:	yr4_b4cl_four_complex_djbunfft rsi, 4*64, 64, 2*64, rdi, 0, 16
	yloop	16, c6b, -16*4*64+4*dist64, rdi, 4*YMM_SCD2/4 ;; Loop 16 times, next source pointer, next sine/cosine pointer
	yloop	4, c6b, -4*4*dist64, rdi, -4*4*YMM_SCD2/4 ;; Loop 4 times, restore source pointer, restore sine/cosine pointer
	loops_undo 4*16			;; Restore loop counter
	end_timer 10

	start_timer 11
;;	yr4_b4cl_four_complex_djbunfft_preload
c5b:	yr4_b4cl_four_complex_djbunfft rsi, 64, 4*64, 8*64, rdi, 4*YMM_SCD2/4, 4
	yloop	4, c5b, -4*64+16*64, rdi, -4*4*YMM_SCD2/4 ;; Loop 4 times, next source pointer, restore sine/cosine pointer
	yloop	4, c5b, -4*16*64+4*dist64, rdi, YMM_SCD2/4 ;; Loop 4 times, next source pointer, next sine/cosine pointer
	yloop	4, c5b, -4*4*dist64, rdi, -4*YMM_SCD2/4 ;; Loop 4 times, restore source pointer, next sine/cosine pointer
	loops_undo 4*4*4		;; Back up loop counter
	end_timer 11

	start_timer 12
;;	yr4_b4cl_four_complex_djbunfft_preload
c4b:	yr4_b4cl_four_complex_djbunfft rsi, 64, 16*64, 32*64, rdi, YMM_SCD2/4, 16
	yloop	16, c4b, -16*64+4*dist64 ;; Loop 16 times, next source pointer
	yloop	4, c4b, -4*4*dist64+dist64, rdi, -64*YMM_SCD2/4 ;; Loop 4 times, next source pointer, restore sine/cosine pointer
	end_timer 12

	yloop	4, b4b0, -4*dist64+16*dist64 ;; Loop 4 times, next source pointer
	yloop	5, b4b0, -5*16*dist64	;; Loop 5 times, restore source pointer
	jmp	ypass2_20480_complex_unfft


ypass2_20480_complex_1:
	start_timer 6
	bump	rcx, (5120+79-256-640-320-3*1280-160)*64 ;; Move prefetch pointer along
	yloop_init 1280			;; 20 iters of 256 cache lines

b4a0:	yloop_set_L3_prefetch YSINGLE	;; Prefetch 1280 cache lines in this loop
	yr4_b4cl_four_complex_djbfft_preload
b4a:	yr4_b4cl_four_complex_djbfft rsi, 64, 16*64, 32*64, rdi, YMM_SCD2/4, 16
	yloop	16, b4a, -16*64+4*dist64 ;; Loop 16 times, next source pointer
	yloop	4, b4a, -4*4*dist64, rdi, -64*YMM_SCD2/4 ;; Loop 4 times, restore source pointer, restore sine/cosine pointer
	loops_undo 4*16			;; Back up loop counter

	yloop_set_L3_prefetch YSINGLE	;; Prefetch 1280 cache lines in this loop
;;	yr4_b4cl_four_complex_djbfft_preload
b5a:	yr4_b4cl_four_complex_djbfft rsi, 64, 4*64, 8*64, rdi, 4*YMM_SCD2/4, 4
	yloop	4, b5a, -4*64+16*64, rdi, -4*4*YMM_SCD2/4 ;; Loop 4 times, next source pointer, restore sine/cosine pointer
	yloop	4, b5a, -4*16*64+4*dist64, rdi, YMM_SCD2/4 ;; Loop 4 times, next source pointer, next sine/cosine pointer
	yloop	4, b5a, -4*4*dist64, rdi, -4*YMM_SCD2/4 ;; Loop 4 times, restore source pointer, next sine/cosine pointer
	loops_undo 4*4*4		;; Back up loop counter

	yloop_set_L3_prefetch YSINGLE	;; Prefetch 1280 cache lines in this loop
;;	yr4_b4cl_four_complex_djbfft_preload
b6a:	yr4_b4cl_four_complex_djbfft rsi, 4*64, 64, 2*64, rdi, 0, 16
	yloop	16, b6a, -16*4*64+4*dist64, rdi, 4*YMM_SCD2/4 ;; Loop 16 times, next source pointer, next sine/cosine pointer
	yloop	4, b6a, -4*4*dist64, rdi, -4*4*YMM_SCD2/4 ;; Loop 4 times, restore source pointer, restore sine/cosine pointer
	loops_undo 4*16			;; Back up loop counter

	yloop_set_L3_prefetch YOTHER, 8	;; Prefetch 160 cache lines in this loop
	yr4_4cl_four_complex_fft_final_preload
b7a:	yr4_4cl_four_complex_fft_final rsi, 64, 4*dist64, 8*dist64
	yloop	64, b7a, -64*64+dist64	;; Loop 64 times, next source pointer

	yloop	4, b4a0, -4*dist64+16*dist64 ;; Loop 4 times, next source pointer
	yloop	5, b4a0			;; Loop 5 times
	end_timer 6
	jmp	ypass2_20480_complex_done


ypass2_20480_complex_3:
	start_timer 6
x64	mov	r8, DEST2ARG
	mov	rbp, DIST_TO_MULSRCARG
	yloop_init 1280			;; 20 iters of 256 cache lines
	yloop_set_L3_prefetch YQUARTER

b4c0:	yr4_b4cl_four_complex_djbfft_preload
b4c:	yr4_b4cl_four_complex_djbfft rsi, 64, 16*64, 32*64, rdi, YMM_SCD2/4, 16
	yloop	16, b4c, -16*64+4*dist64 ;; Loop 16 times, next source pointer
	yloop	4, b4c, -4*4*dist64, rdi, -64*YMM_SCD2/4 ;; Loop 4 times, restore source pointer, restore sine/cosine pointer
	loops_undo 4*16			;; Back up loop counter

;;	yr4_b4cl_four_complex_djbfft_preload
b5c:	yr4_b4cl_four_complex_djbfft rsi, 64, 4*64, 8*64, rdi, 4*YMM_SCD2/4, 4
	yloop	4, b5c, -4*64+16*64, rdi, -4*4*YMM_SCD2/4 ;; Loop 4 times, next source pointer, restore sine/cosine pointer
	yloop	4, b5c, -4*16*64+4*dist64, rdi, YMM_SCD2/4 ;; Loop 4 times, next source pointer, next sine/cosine pointer
	yloop	4, b5c, -4*4*dist64, rdi, -4*YMM_SCD2/4 ;; Loop 4 times, restore source pointer, next sine/cosine pointer
	loops_undo 4*4*4		;; Back up loop counter

;;	yr4_b4cl_four_complex_djbfft_preload
b6c:	yr4_b4cl_four_complex_djbfft rsi, 4*64, 64, 2*64, rdi, 0, 16
	yloop	16, b6c, -16*4*64+4*dist64, rdi, 4*YMM_SCD2/4 ;; Loop 16 times, next source pointer, next sine/cosine pointer
	yloop	4, b6c, -4*4*dist64, rdi, -4*4*YMM_SCD2/4 ;; Loop 4 times, restore source pointer, restore sine/cosine pointer
	loops_undo 4*16			;; Back up loop counter

	yr4_4cl_four_complex_with_mult_preload
b7c:	yr4_4cl_four_complex_with_mult rsi, 64, 4*dist64, 8*dist64
	yloop	64, b7c, -64*64		;; Loop 64 times, restore source pointer
x64	add	rsi, r8
	loops_undo 64			;; Back up loop counter

	yr4_b4cl_four_complex_djbunfft_preload
c6c:	yr4_b4cl_four_complex_djbunfft rsi, 4*64, 64, 2*64, rdi, 0, 16
	yloop	16, c6c, -16*4*64+4*dist64, rdi, 4*YMM_SCD2/4 ;; Loop 16 times, next source pointer, next sine/cosine pointer
	yloop	4, c6c, -4*4*dist64, rdi, -4*4*YMM_SCD2/4 ;; Loop 4 times, restore source pointer, restore sine/cosine pointer
	loops_undo 4*16			;; Restore loop counter

;;	yr4_b4cl_four_complex_djbunfft_preload
c5c:	yr4_b4cl_four_complex_djbunfft rsi, 64, 4*64, 8*64, rdi, 4*YMM_SCD2/4, 4
	yloop	4, c5c, -4*64+16*64, rdi, -4*4*YMM_SCD2/4 ;; Loop 4 times, next source pointer, restore sine/cosine pointer
	yloop	4, c5c, -4*16*64+4*dist64, rdi, YMM_SCD2/4 ;; Loop 4 times, next source pointer, next sine/cosine pointer
	yloop	4, c5c, -4*4*dist64, rdi, -4*YMM_SCD2/4 ;; Loop 4 times, restore source pointer, next sine/cosine pointer
	loops_undo 4*4*4		;; Back up loop counter

;;	yr4_b4cl_four_complex_djbunfft_preload
c4c:	yr4_b4cl_four_complex_djbunfft rsi, 64, 16*64, 32*64, rdi, YMM_SCD2/4, 16
	yloop	16, c4c, -16*64+4*dist64 ;; Loop 16 times, next source pointer
	yloop	4, c4c, -4*4*dist64+dist64, rdi, -64*YMM_SCD2/4 ;; Loop 4 times, next source pointer, restore sine/cosine pointer

x64	sub	rsi, r8
	yloop	4, b4c0, -4*dist64+16*dist64 ;; Loop 4 times, next source pointer
	yloop	5, b4c0, -5*16*dist64	;; Loop 5 times, restore source pointer
x64	add	rsi, r8
	end_timer 6
	jmp	ypass2_20480_complex_unfft


ypass2_20480_complex_4:
	start_timer 6
	mov	rdi, xsincos_complex	;; The sin/cos ptrs
	bump	rdi, 1024*YMM_SCD2+256*YMM_SCD2
	mov	rbp, DIST_TO_MULSRCARG
	yloop_init 1280			;; 20 iters of 256 cache lines

	yloop_set_L3_prefetch YQUARTER	;; Prefetch 320 cache lines in this loop
	yr4_4cl_four_complex_with_mulf_preload
b7d:	yr4_4cl_four_complex_with_mulf rsi, 64, 4*dist64, 8*dist64
	yloop	64, b7d, -64*64		;; Loop 64 times, restore source pointer
	loops_undo 64			;; Back up loop counter

	yloop_set_L3_prefetch YHALF	;; Prefetch 640 cache lines in this loop
	yr4_b4cl_four_complex_djbunfft_preload
c6d:	yr4_b4cl_four_complex_djbunfft rsi, 4*64, 64, 2*64, rdi, 0, 16
	yloop	16, c6d, -16*4*64+4*dist64, rdi, 4*YMM_SCD2/4 ;; Loop 16 times, next source pointer, next sine/cosine pointer
	yloop	4, c6d, -4*4*dist64, rdi, -4*4*YMM_SCD2/4 ;; Loop 4 times, restore source pointer, restore sine/cosine pointer
	loops_undo 4*16			;; Restore loop counter

	yloop_set_L3_prefetch YSINGLE	;; Prefetch 1280 cache lines in this loop
;;	yr4_b4cl_four_complex_djbunfft_preload
c5d:	yr4_b4cl_four_complex_djbunfft rsi, 64, 4*64, 8*64, rdi, 4*YMM_SCD2/4, 4
	yloop	4, c5d, -4*64+16*64, rdi, -4*4*YMM_SCD2/4 ;; Loop 4 times, next source pointer, restore sine/cosine pointer
	yloop	4, c5d, -4*16*64+4*dist64, rdi, YMM_SCD2/4 ;; Loop 4 times, next source pointer, next sine/cosine pointer
	yloop	4, c5d, -4*4*dist64, rdi, -4*YMM_SCD2/4 ;; Loop 4 times, restore source pointer, next sine/cosine pointer
	loops_undo 4*4*4		;; Back up loop counter

	yloop_set_L3_prefetch YSINGLE	;; Prefetch 1280 cache lines in this loop
;;	yr4_b4cl_four_complex_djbunfft_preload
c4d:	yr4_b4cl_four_complex_djbunfft rsi, 64, 16*64, 32*64, rdi, YMM_SCD2/4, 16
	yloop	16, c4d, -16*64+4*dist64	;; Loop 16 times, next source pointer
	yloop	4, c4d, -4*4*dist64+dist64, rdi, -64*YMM_SCD2/4 ;; Loop 4 times, next source pointer, restore sine/cosine pointer

	yloop	4, b7d, -4*dist64+16*dist64	;; Loop 4 times, next source pointer
	yloop	5, b7d, -5*16*dist64	;; Loop 5 times, restore source pointer
	bump	rcx, (-320-640-2*1280+256+640+320+7*320)*64 ;; Move prefetch pointer to where squaring would be
	end_timer 6

;; Do inverse FFT levels 5,6
;; On input the 64-byte cache lines hold these data values:
;;	0	+2K	+2K	+2K	256	+2K	+2K	+2K
;;	4	...
;;	...
;;	252	...
;;	512	...
;;	...
;;	1	...
;;	...
;;	8192	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+2K	+2K	+2K	1024	+2K	+2K	+2K
;;	4	...
;;	...
;;	1020	...
;;	1	...
;;	...
;;	8192	...
;;	...

;; Do 5120 four_complex_unfft operations
;;	distance between fft data elements is 256
;;	do 1280 macros each processing 32 data values

;; Do inverse FFT levels 3,4
;; On input the 64-byte cache lines hold these data values:
;;	0	+2K	+2K	+2K	1024	+2K	+2K	+2K
;;	4	...
;;	...
;;	1020	...
;;	1	...
;;	...
;;	8192	...
;;	...
;; On output the 64-byte cache lines are swizzled to hold these data values:
;;	0	+1	+1	+1	4096	+1	+1	+1
;;	4	...
;;	...
;;	4092	...
;;	8192	...
;;	...

;; Do 5120 four_complex_unfft operations
;;	distance between fft data elements is 1024
;;	do 1280 macros each processing 32 data values

;; Do inverse FFT levels 1,2
;; On input the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	4096	+1	+1	+1
;;	4	...
;;	...
;;	4092	...
;;	8192	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	20480	+1	+1	+1
;;	4	...
;;	...
;;	12284

;; Do 4096 five_complex_unfft operations
;;	distance between fft data elements is 4096
;;	do 1024 macros each processing 40 data values

ypass2_20480_complex_unfft:
	mov	rdi, xsincos_complex	;; Load sin/cos pointers
	lea	rbp, [rdi+1024*YMM_SCD2]
	mov	rdx, rbp
	bump	rcx, (5120+79-256-640-320-7*320-2*640-512)*64 ;; Move prefetch pointer along
	yloop_init 2048			;; 1024 iterations or 1280 iterations

c3a:	start_timer 13
	yloop_adjust 3*4		;; Fewer iterations this loop
	yloop_set_L3_prefetch YHALF	;; Prefetch 640 cache lines in this loop
	yr4_eb4cl_four_complex_djbunfft_preload
c3b:	yr4_eb4cl_four_complex_djbunfft rsi, 4*dist64, dist64, 2*dist64, rdx, YMM_SCD2, 4
	yloop	4, c3b, 0, rdx, -4*YMM_SCD2 ;; Test loop counter, restore sine/cosine pointer
	yloop	8, c3b, -5*4*4*dist64, rdx, 4*YMM_SCD2 ;; 5 iterations!!  Restore source pointer, next sine/cosine pointer
	loops_undo 8*4			;; Back up loop counter
	end_timer 13

	start_timer 14
	yloop_adjust 3*4		;; Fewer iterations this loop
	yloop_set_L3_prefetch YHALF	;; Prefetch 640 cache lines in this loop
	yr4_s4cl_four_complex_djbunfft_preload
c2b:	yr4_s4cl_four_complex_djbunfft rsi, dist64, 4*dist64, 8*dist64, rbp, 64*YMM_SCD2, 4, L1PREFETCH_ALL, 64
	yloop	4, c2b, -4*dist64+16*dist64, rbp, -4*64*YMM_SCD2 ;; Test loop counter, next source pointer, restore sine/cosine pointer
	yloop	8, c2b, -5*16*dist64, rbp, YMM_SCD2 ;; 5 iterations!!  Restore source pointer, next sine/cosine pointer
	loops_undo 8*4			;; Back up loop counter
	end_timer 14

	start_timer 15
	yloop_adjust 16			;; Fewer iterations this loop
	yloop_set_L3_prefetch YHALF	;; Prefetch 512 cache lines in this loop
	yr5_5cl_five_complex_djbunfft_preload
c1b:	yr5_5cl_five_complex_djbunfft rsi, dist64, 16*dist64, rdi, 64*YMM_SCD2, 16
	yloop	32, c1b, -16*dist64+64, rdi, -16*64*YMM_SCD2+YMM_SCD2 ;; 16 iterations!!  Next source pointer, next sine/cosine pointer
	end_timer 15

	yloop	64, c3a			;; Loop 64 times

ypass2_20480_complex_done:
	ENDM


; *************** radix4-pass2-25600-real macro ******************
; This macro takes 51200 real values and performs the final 15.644 levels of the FFT.
; The results are squared, and the first 15.644 levels of the inverse FFT are done.

yr4dwpn_pass2_25600_real MACRO
	LOCAL	b1b, b2a, b2b, b3a, b3b, b4a, b4b, b5a, b5b, b6a, b6b, b7a, b7b, b7c, b7d
	LOCAL	c1b, c2a, c2b, c3a, c3b, c4a, c4b, c5a, c5b, c6a, c6b
	LOCAL	ypass2_25600_real_1, ypass2_25600_real_2
	LOCAL	ypass2_25600_real_3, ypass2_25600_real_4
	LOCAL	ypass2_25600_real_unfft, ypass2_25600_real_done

;; Type 4 FFTs skip the forward FFT process

	start_timer 2
	mov	rsi, DATA_ADDR		;; Load source address
	mov	rbx, DIST_TO_FFTSRCARG
	mov	rcx, DATA_PREFETCH	;; Prefetch pointer (we need to prefetch 
					;; 6400 cache lines + 99 pad cache lines)
	cmp	ffttype, 4
	je	ypass2_25600_real_4

;; Do FFT levels 1,2,3 which correspond to complex levels 1,2
;; On input the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	25600	+1	+1	+1
;;	4	...
;;	...
;;	25596
;; On output the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	5120	+1	+1	+1
;;	4	...
;;	...
;;	5116	...
;;	10240	...
;;	...

;; Do 5120 ten_reals_fft operations
;;	distance between fft data elements is 5120
;;	do 1280 macros each processing 40 data values

	mov	rdi, sincos3		;; Real sin/cos pointer
	mov	rbp, xsincos_complex	;; Complex sin/cos data
	yloop_init 1280			;; 1280 iterations
	yloop_set_L3_prefetch YQUARTER
	yr5_f5cl_2sc_ten_reals_fft_preload
b1b:	yr5_f5cl_2sc_ten_reals_fft rsi, 64, 20*dist64, rbp, YMM_SCD2, rdi, YMM_SCD2, 64
	yloop	64, b1b, -64*64+dist64	;; Test loop counter, next source pointer
	yloop	20, b1b, -20*dist64	;; Test loop counter, restore source pointer

;; Do FFT levels 3,4
;; On input the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	5120	+1	+1	+1
;;	4	...
;;	...
;;	5116	...
;;	10240	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	1024	+1	+1	+1
;;	4	...
;;	...
;;	1020	...
;;	2048	...
;;	...

;; Do 1024 ten_reals_fft operations
;; Do 2048 five_complex_fft operations
;;	distance between fft data elements is 1024
;;	do 1280 macros each processing 40 data values

	yloop_init 1280			;; 5 iters of 256
	yloop_set_L3_prefetch YHALF
	yr5_5cl_2sc_ten_reals_fft_preload
b2a:	yr5_5cl_2sc_ten_reals_fft rsi, 64, 4*dist64, rbp, YMM_SCD2, rdi, YMM_SCD2, 64
	yloop	64, b2a, -64*64+dist64	;; Test loop counter, next source pointer
	yloop	4, b2a, -4*dist64+20*dist64, rbp, -256*YMM_SCD2 ;; Test loop counter, next source pointer, restore sine/cosine pointer

	loops_reset			;; 4 more iters of 256
	yr5_5cl_five_complex_djbfft_preload
b2b:	yr5_5cl_five_complex_djbfft rsi, 64, 4*dist64, rbp, YMM_SCD2, 64
	yloop	64, b2b, -64*64+dist64	;; Test loop counter, next source pointer
	yloop	4, b2b, -4*dist64+20*dist64, rbp, -256*YMM_SCD2 ;; Test loop counter, next source pointer, restore sin/cos ptr
	yloop	5, b2b, -5*20*dist64, rbp, 256*YMM_SCD2 ;; Loop 5 times, restore source pointer, next sin/cos pointer

;; Do FFT levels 5,6
;; On input the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	1024	+1	+1	+1
;;	4	...
;;	...
;;	1020	...
;;	2048	...
;;	...
;; On output the 64-byte cache lines are swizzled to hold these data values:
;;	0	+512	+512	+512	256	+512	+512	+512
;;	4	...
;;	...
;;	508	...
;;	1	...
;;	...
;;	2048	...
;;	...

;; Do 256 eight_reals_fft operations
;; Do 6144 four_complex_fft operations
;;	distance between fft data elements is 256
;;	do 1600 macros each processing 32 data values

	yloop_init 1600			;; 25 iterations of 64
	yloop_set_L3_prefetch YHALF
	yr4_s4cl_2sc_eight_reals_fft_preload
b3a:	yr4_s4cl_2sc_eight_reals_fft rsi, 64, dist64, 2*dist64, rbp, YMM_SCD2, rdi, YMM_SCD2, 64
	yloop	64, b3a, -64*64+4*dist64, rbp, -64*YMM_SCD2 ;; Test loop counter, next source pointer, restore sine/cosine pointer

	loops_reset			;; 24 more iters of 64
	yr4_s4cl_four_complex_djbfft_preload
b3b:	yr4_s4cl_four_complex_djbfft rsi, 64, dist64, 2*dist64, rbp, YMM_SCD2, 64
	yloop	64, b3b, -64*64+4*dist64, rbp, -64*YMM_SCD2 ;; Test loop counter, next source pointer, restore sin/cos ptr
	yloop	25, b3b, -25*4*dist64	;; Loop 25 times, restore source pointer

;; Do FFT levels 7,8
;; On input the 64-byte cache lines hold these data values:
;;	0	+512	+512	+512	256	+512	+512	+512
;;	4	...
;;	...
;;	508	...
;;	1	...
;;	...
;;	2048	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+512	+512	+512	64	+512	+512	+512
;;	4	...
;;	...
;;	60	...
;;	128	...
;;	...
;;	1	...
;;	...
;;	2048	...
;;	...

;; Do 64 eight_reals_fft operations
;; Do 6336 four_complex_fft operations
;;	distance between fft data elements is 64
;;	do 1600 macros each processing 32 data values

	yloop_init 1600			;; 25 iterations of 64
	yloop_set_L3_prefetch YHALF
	yr4_4cl_eight_reals_four_complex_djbfft_preload
b4a:	yr4_4cl_eight_reals_four_complex_djbfft rsi, dist64, 16*64, 32*64, rdi, YMM_SCD3, 4
	yloop	4, b4a, -4*dist64+64	;; Test loop counter, next source pointer
	yloop	16, b4a, -16*64+4*dist64, rdi, -64*YMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer

	loops_reset			;; 24 more iters of 64
	yr4_rb4cl_four_complex_djbfft_preload
b4b:	yr4_rb4cl_four_complex_djbfft rsi, dist64, 16*64, 32*64, rdi, YMM_SCD3, 4
	yloop	4, b4b, -4*dist64+64	;; Test loop counter, next source pointer
	yloop	16, b4b, -16*64+4*dist64, rdi, -64*YMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer
	yloop	25, b4b, -25*4*dist64	;; Test loop counter, restore source pointer

;; Do FFT levels 9,10
;; On input the 64-byte cache lines hold these data values:
;;	0	+512	+512	+512	64	+512	+512	+512
;;	4	...
;;	...
;;	60	...
;;	128	...
;;	...
;;	1	...
;;	...
;;	2048	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+512	+512	+512	16	+512	+512	+512
;;	4	...
;;	...
;;	12	...
;;	32	...
;;	...
;;	1	...
;;	...
;;	2048	...
;;	...

;; Do 16 eight_reals_fft operations
;; Do 6384 four_complex_fft operations
;;	distance between fft data elements is 16
;;	do 1600 macros each processing 32 data values

	yloop_init 1600			;; 100 iterations of 16
	yloop_set_L3_prefetch YHALF
	yr4_4cl_eight_reals_four_complex_djbfft_preload
b5a:	yr4_4cl_eight_reals_four_complex_djbfft rsi, dist64, 4*64, 8*64, rdi, 4*YMM_SCD3, 4
	yloop	4, b5a, -4*dist64+64	;; Test loop counter, next source pointer
	yloop	4, b5a, -4*64+16*64, rdi, -16*4*YMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer

	loops_reset			;; 99 more iters of 16
	yr4_rb4cl_four_complex_djbfft_preload
b5b:	yr4_rb4cl_four_complex_djbfft rsi, dist64, 4*64, 8*64, rdi, 4*YMM_SCD3, 4
	yloop	4, b5b, -4*dist64+64	;; Test loop counter, next source pointer
	yloop	4, b5b, -4*64+16*64, rdi, -16*4*YMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer
	yloop	4, b5b, -4*16*64+4*dist64 ;; Test loop counter, next source pointer
	yloop	25, b5b, -25*4*dist64	;; Test loop counter, restore source pointer

;; Do FFT levels 11,12
;; On input the 64-byte cache lines hold these data values:
;;	0	+512	+512	+512	16	+512	+512	+512
;;	4	...
;;	...
;;	12	...
;;	32	...
;;	...
;;	1	...
;;	...
;;	2048	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+512	+512	+512	4	+512	+512	+512
;;	8	...
;;	...
;;	1	...
;;	...
;;	2048	...
;;	...

;; Do 4 eight_reals_fft operations
;; Do 6396 four_complex_fft operations
;;	distance between fft data elements is 4
;;	do 1600 macros each processing 32 data values

	yloop_init 1600			;; 400 iterations of 4
	yloop_set_L3_prefetch YHALF
	yr4_4cl_eight_reals_four_complex_djbfft_preload
b6a:	yr4_4cl_eight_reals_four_complex_djbfft rsi, dist64, 64, 2*64, rdi, 16*YMM_SCD3, 4
	yloop	4, b6a, -4*dist64+4*64, rdi, -4*16*YMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer

	loops_reset			;; 399 more iters of 4
	yr4_rb4cl_four_complex_djbfft_preload
b6b:	yr4_rb4cl_four_complex_djbfft rsi, dist64, 64, 2*64, rdi, 16*YMM_SCD3, 4
	yloop	4, b6b, -4*dist64+4*64, rdi, -4*16*YMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer
	yloop	16, b6b, -16*4*64+4*dist64 ;; Test loop counter, next source pointer
	yloop	25, b6b, -25*4*dist64	;; Test loop counter, restore source pointer

;; Do FFT levels 13,14 as well as inverse FFT levels 13,14
;; On input and output the 64-byte cache lines hold these data values:
;;	0	+512	+512	+512	4	+512	+512	+512
;;	8	...
;;	...
;;	1	...
;;	...
;;	2048	...
;;	...

;; Do 1 eight_reals_fft operation
;; Do 6399 four_complex_fft operations
;;	distance between fft data elements is 1
;;	do 1600 macros each processing 32 data values

	cmp	ffttype, 2
	jl	ypass2_25600_real_1
	jg	ypass2_25600_real_3

ypass2_25600_real_2:
;;	yr4_4cl_eight_reals_four_complex_with_square rsi, 64, dist64, 2*dist64
	yr4_dispatch call, yreal_square, dist64, 2*dist64
	yloop_init 1600			;; 1599 more iterations
	yloop_adjust 1
	yloop_set_L3_prefetch YQUARTER
	yr4_4cl_four_complex_with_square_preload
b7b:	yr4_4cl_four_complex_with_square rsi, 64, dist64, 2*dist64, 1
	yloop	64, b7b, -64*64+4*dist64	;; Test loop counter, next source pointer
	yloop	25, b7b, -25*4*dist64	;; Test loop counter, restore source pointer
	jmp	ypass2_25600_real_unfft

ypass2_25600_real_1:
;;	yr4_4cl_eight_reals_four_complex_fft_final rsi, 64, dist64, 2*dist64
	yr4_dispatch call, yreal_fft_final, dist64, 2*dist64
	mov	rcx, xsincos_complex	;; Prefetch complex sin/cos data, 3200 cache lines
	yloop_init 1600			;; 1599 more iterations
	yloop_adjust 1
	yloop_set_L3_prefetch YDOUBLE
b7a:	yr4_4cl_four_complex_fft_final rsi, 64, dist64, 2*dist64
	yloop	64, b7a, -64*64+4*dist64	;; Test loop counter, next source pointer
	yloop	25, b7a			;; Test loop counter
	jmp	ypass2_25600_real_done

ypass2_25600_real_3:
x64	mov	r8, DEST2ARG
	mov	rbp, DIST_TO_MULSRCARG
;;	yr4_4cl_eight_reals_four_complex_with_mult rsi, 64, dist64, 2*dist64
	yr4_dispatch call, yreal_mult, dist64, 2*dist64
	yloop_init 1600			;; 1599 more iterations
	yloop_adjust 1
	yloop_set_L3_prefetch YQUARTER
	yr4_4cl_four_complex_with_mult_preload
b7c:	yr4_4cl_four_complex_with_mult rsi, 64, dist64, 2*dist64
	yloop	64, b7c, -64*64+4*dist64	;; Test loop counter, next source pointer
	yloop	25, b7c, -25*4*dist64	;; Test loop counter, restore source pointer
x64	add	rsi, r8
	jmp	ypass2_25600_real_unfft

ypass2_25600_real_4:
	mov	rbp, DIST_TO_MULSRCARG
;;	yr4_4cl_eight_reals_four_complex_with_mulf rsi, 64, dist64, 2*dist64
	yr4_dispatch call, yreal_mulf, dist64, 2*dist64
	yloop_init 1600			;; 1599 more iterations
	yloop_adjust 1
	yloop_set_L3_prefetch YDOUBLE
	yr4_4cl_four_complex_with_mulf_preload
b7d:	yr4_4cl_four_complex_with_mulf rsi, 64, dist64, 2*dist64
	yloop	64, b7d, -64*64+4*dist64	;; Test loop counter, next source pointer
	yloop	25, b7d, -25*4*dist64	;; Test loop counter, restore source pointer
;;	bump	rcx, (-3198+320+640+4*800+400)*64 ;; Move prefetch pointer back to where squaring would be

;; Do inverse FFT levels 11,12
;; On input the 64-byte cache lines hold these data values:
;;	0	+512	+512	+512	4	+512	+512	+512
;;	8	...
;;	...
;;	1	...
;;	...
;;	2048	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+512	+512	+512	16	+512	+512	+512
;;	4	...
;;	...
;;	12	...
;;	32	...
;;	...
;;	1	...
;;	...
;;	2048	...
;;	...

;; Do 4 eight_reals_unfft operations
;; Do 6396 four_complex_unfft operations
;;	distance between fft data elements is 4
;;	do 1600 macros each processing 32 data values

ypass2_25600_real_unfft:
	mov	rdi, sincos3		;; Load sin/cos pointer
	bump	rdi, 64*YMM_SCD2+256*YMM_SCD2+1280*YMM_SCD2
	yloop_init 1600			;; 400 iterations of 4
	yloop_set_L3_prefetch YQUARTER
	yr4_4cl_eight_reals_four_complex_djbunfft_preload
c6a:	yr4_4cl_eight_reals_four_complex_djbunfft rsi, dist64, 64, 2*64, rdi, 16*YMM_SCD3, 4
	yloop	4, c6a, -4*dist64+4*64, rdi, -4*16*YMM_SCD3 ;; Test loop counter, next source pointer, restore sine/cosine pointer

	loops_reset			;; 399 more iters of 4
	yr4_rb4cl_four_complex_djbunfft_preload
c6b:	yr4_rb4cl_four_complex_djbunfft rsi, dist64, 64, 2*64, rdi, 16*YMM_SCD3, 4
	yloop	4, c6b, -4*dist64+4*64, rdi, -4*16*YMM_SCD3 ;; Test loop counter, next source pointer, restore sine/cosine pointer
	yloop	16, c6b, -16*4*64+4*dist64	;; Test loop counter, next source pointer
	yloop	25, c6b, -25*4*dist64	;; Test loop counter, restore source pointer

;; Do inverse FFT levels 9,10
;; On input the 64-byte cache lines hold these data values:
;;	0	+512	+512	+512	16	+512	+512	+512
;;	4	...
;;	...
;;	12	...
;;	32	...
;;	...
;;	1	...
;;	...
;;	2048	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+512	+512	+512	64	+512	+512	+512
;;	4	...
;;	...
;;	60	...
;;	128	...
;;	...
;;	1	...
;;	...
;;	2048	...
;;	...

;; Do 16 eight_reals_unfft operations
;; Do 6384 four_complex_unfft operations
;;	distance between fft data elements is 16
;;	do 1600 macros each processing 32 data values

	yloop_init 1600			;; 100 iterations of 16
	yloop_set_L3_prefetch YHALF
	yr4_4cl_eight_reals_four_complex_djbunfft_preload
c5a:	yr4_4cl_eight_reals_four_complex_djbunfft rsi, dist64, 4*64, 8*64, rdi, 4*YMM_SCD3, 4
	yloop	4, c5a, -4*dist64+64	;; Test loop counter, next source pointer
	yloop	4, c5a, -4*64+16*64, rdi, -16*4*YMM_SCD3 ;; Test loop counter, next source pointer, restore sine/cosine pointer

	loops_reset			;; 99 more iters of 16
	yr4_rb4cl_four_complex_djbunfft_preload
c5b:	yr4_rb4cl_four_complex_djbunfft rsi, dist64, 4*64, 8*64, rdi, 4*YMM_SCD3, 4
	yloop	4, c5b, -4*dist64+64	;; Test loop counter, next source pointer
	yloop	4, c5b, -4*64+16*64, rdi, -16*4*YMM_SCD3 ;; Test loop counter, next source pointer, restore sine/cosine pointer
	yloop	4, c5b, -4*16*64+4*dist64	;; Test loop counter, next source pointer
	yloop	25, c5b, -25*4*dist64	;; Test loop counter, restore source pointer

;; Do inverse FFT levels 7,8
;; On input the 64-byte cache lines hold these data values:
;;	0	+512	+512	+512	64	+512	+512	+512
;;	4	...
;;	...
;;	60	...
;;	128	...
;;	...
;;	1	...
;;	...
;;	2048	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+512	+512	+512	256	+512	+512	+512
;;	4	...
;;	...
;;	508	...
;;	1	...
;;	...
;;	2048	...
;;	...

;; Do 64 eight_reals_unfft operations
;; Do 6336 four_complex_unfft operations
;;	distance between fft data elements is 64
;;	do 1600 macros each processing 32 data values

	bump	rcx, (6400+99-320-640-4*800-2*400-800-800)*64 ;; Move prefetch pointer along
	yloop_init 1600			;; 25 iterations of 64
	yloop_set_L3_prefetch YHALF
	yr4_4cl_eight_reals_four_complex_djbunfft_preload
c4a:	yr4_4cl_eight_reals_four_complex_djbunfft rsi, dist64, 16*64, 32*64, rdi, YMM_SCD3, 4
	yloop	4, c4a, -4*dist64+64	;; Test loop counter, next source pointer
	yloop	16, c4a, -16*64+4*dist64, rdi, -64*YMM_SCD3 ;; Test loop counter, next source pointer, restore sine/cosine pointer

	loops_reset			;; 24 more iters of 64
	yr4_rb4cl_four_complex_djbunfft_preload
c4b:	yr4_rb4cl_four_complex_djbunfft rsi, dist64, 16*64, 32*64, rdi, YMM_SCD3, 4
	yloop	4, c4b, -4*dist64+64	;; Test loop counter, next source pointer
	yloop	16, c4b, -16*64+4*dist64, rdi, -64*YMM_SCD3 ;; Test loop counter, next source pointer, restore sine/cosine pointer
	yloop	25, c4b, -25*4*dist64	;; Test loop counter, restore source pointer

;; Do inverse FFT levels 5,6
;; On input the 64-byte cache lines hold these data values:
;;	0	+512	+512	+512	256	+512	+512	+512
;;	4	...
;;	...
;;	508	...
;;	1	...
;;	...
;;	2048	...
;;	...
;; On output the 64-byte cache lines are swizzled to hold these data values:
;;	0	+1	+1	+1	1024	+1	+1	+1
;;	4	...
;;	...
;;	1020	...
;;	2048	...
;;	...

;; Do 256 eight_reals_unfft operations
;; Do 6144 four_complex_unfft operations
;;	distance between fft data elements is 256
;;	do 1600 macros each processing 32 data values

	bump	rdi, -64*YMM_SCD2	;; Real sin/cos ptr
	mov	rbp, xsincos_complex	;; Complex sin/cos data
	bump	rbp, 256*YMM_SCD2+1280*YMM_SCD2
	mov	rcx, xsincos_complex	;; Prefetch complex sin/cos data, 3200 cache lines
	yloop_init 1600			;; 25 iters of 64
	yloop_set_L3_prefetch YHALF
	yr4_s4cl_2sc_eight_reals_unfft_preload
c3a:	yr4_s4cl_2sc_eight_reals_unfft rsi, 64, dist64, 2*dist64, rbp, YMM_SCD2, rdi, YMM_SCD2, 64
	yloop	64, c3a, -64*64+4*dist64, rbp, -64*YMM_SCD2 ;; Test loop counter, next source pointer, restore sine/cosine pointer

	loops_reset			;; 24 more iters of 64
	yr4_s4cl_four_complex_djbunfft_preload
c3b:	yr4_s4cl_four_complex_djbunfft rsi, 64, dist64, 2*dist64, rbp, YMM_SCD2, 64
	yloop	64, c3b, -64*64+4*dist64, rbp, -64*YMM_SCD2 ;; Test loop counter, next source pointer, restore sine/cosine pointer
	yloop	25, c3b, -25*4*dist64	;; Test loop counter, restore source pointer

;; Do inverse FFT levels 3,4
;; On input the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	1024	+1	+1	+1
;;	4	...
;;	...
;;	1020	...
;;	2048	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	5120	+1	+1	+1
;;	4	...
;;	...
;;	5116	...
;;	10240	...
;;	...

;; Do 1024 ten_reals_unfft operations
;; Do 2048 five_complex_unfft operations
;;	distance between fft data elements is 1024
;;	do 1280 macros each processing 40 data values

	bump	rdi, -64*YMM_SCD2-256*YMM_SCD2 ;; Real sin/cos ptr
	bump	rbp, -256*YMM_SCD2	;; Complex sin/cos data
	yloop_init 1280			;; 5 iters of 256
	yloop_set_L3_prefetch YSINGLE
	yr5_5cl_2sc_ten_reals_unfft_preload
c2a:	yr5_5cl_2sc_ten_reals_unfft rsi, 64, 4*dist64, rbp, YMM_SCD2, rdi, YMM_SCD2, 64
	yloop	64, c2a, -64*64+dist64	;; Test loop counter, next source pointer
	yloop	4, c2a, -4*dist64+20*dist64, rbp, -256*YMM_SCD2 ;; Test loop counter, next source pointer, restore sine/cosine pointer

	loops_reset			;; 4 more iters of 256
	yr5_5cl_five_complex_djbunfft_preload
c2b:	yr5_5cl_five_complex_djbunfft rsi, 64, 4*dist64, rbp, YMM_SCD2, 64
	yloop	64, c2b, -64*64+dist64	;; Test loop counter, next source pointer
	yloop	4, c2b, -4*dist64+20*dist64, rbp, -256*YMM_SCD2 ;; Test loop counter, next source pointer, restore sine/cosine pointer
	yloop	5, c2b, -5*20*dist64	;; Test loop counter, restore source pointer

;; Do inverse FFT levels 1,2,3 which correspond to complex levels 1,2
;; On input the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	5120	+1	+1	+1
;;	4	...
;;	...
;;	5116	...
;;	10240	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	25600	+1	+1	+1
;;	4	...
;;	...
;;	25596

;; Do 5120 ten_reals_unfft operations
;;	distance between fft data elements is 5120
;;	do 1280 macros each processing 40 data values

	bump	rdi, -256*YMM_SCD2-1280*YMM_SCD2 ;; Real sin/cos ptr
	bump	rbp, -1280*YMM_SCD2	;; Complex sin/cos data
	bump	rcx, (3200-800-1280-1280)*64 ;; Move prefetch pointer along
	yloop_init 1280			;; 1280 iterations
	yloop_set_L3_prefetch YSINGLE
	yr5_5cl_2sc_ten_reals_unfft_preload
c1b:	yr5_5cl_2sc_ten_reals_unfft rsi, 64, 20*dist64, rbp, YMM_SCD2, rdi, YMM_SCD2, 64
	yloop	64, c1b, -64*64+dist64	;; Test loop counter, next source pointer
	yloop	20, c1b			;; Test loop counter

ypass2_25600_real_done:
	end_timer 2
	ENDM

; *************** radix4-pass2-25600-complex macro ******************
; This macro takes 25600 complex values and performs the final 14.644 levels of a standard
; radix-4 FFT, squares the results, and does the first 14.644 levels of the inverse FFT.

yr4dwpn_pass2_25600_complex MACRO
	LOCAL	b1a, b1b, b2b, b3a0, b3a, b3b0, b3b, b3c0, b3c, b4a, b4b, b4c, b5a, b5b, b5c, b6a, b6b, b6c, b7a, b7b, b7c, b7d
	LOCAL	c1b, c2a, c2b, c3b, c3c, c3d, c4b, c4c, c4d, c5b, c5c, c5d, c6b, c6c, c6d
	LOCAL	ypass2_25600_complex_1, ypass2_25600_complex_2
	LOCAL	ypass2_25600_complex_3, ypass2_25600_complex_4
	LOCAL	ypass2_25600_complex_unfft, ypass2_25600_complex_done

;; Type 4 FFTs skip the forward FFT process

	mov	rsi, DATA_ADDR		;; Load address of FFT data
	mov	rbx, DIST_TO_FFTSRCARG
	mov	rcx, DATA_PREFETCH	;; Prefetch pointer (we need to prefetch 
					;; 6400 cache lines + 99 pad cache lines)
	cmp	ffttype, 4
	je	ypass2_25600_complex_4

;; Do FFT level 1,2 (actually 2.322 levels)
;; On input the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	25600	+1	+1	+1
;;	4	...
;;	...
;;	25596
;; On output the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	5120	+1	+1	+1
;;	4	...
;;	...
;;	5116	...
;;	10240	...
;;	...

;; Do 5120 five_complex_fft operations
;;	distance between fft data elements is 5120
;;	do 1280 macros each processing 40 data values

;; Do FFT levels 3,4 (actually 2.322 levels)
;; On input the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	5120	+1	+1	+1
;;	4	...
;;	...
;;	5116	...
;;	10240	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	1024	+1	+1	+1
;;	4	...
;;	...
;;	1020	...
;;	2048	...
;;	...

;; Do 5120 five_complex_fft operations
;;	distance between fft data elements is 1024
;;	do 1280 macros each processing 40 data values

	mov	rdi, xsincos_complex	;; The sin/cos ptrs
	lea	rbp, [rdi+1280*YMM_SCD2]
	yloop_init 2048			;; 1280 iterations

	yr5_f5cl_five_complex_djbfft_preload
b1a:	start_timer 3
	yloop_adjust 3			;; Three fewer iterations this loop
	yloop_set_L3_prefetch YOTHER, 8	;; Prefetch 256 cache lines
b1b:	yr5_f5cl_five_complex_djbfft rsi, 4*dist64, 20*dist64, rdi, 256*YMM_SCD2, 5
	yloop	8, b1b, -5*4*dist64, rdi, -5*256*YMM_SCD2+YMM_SCD2 ;; 5 iterations!!  Restore source pointer, next sin/cos data
	loops_undo 8			;; Back up loop counter
	end_timer 3

	start_timer 4
	yloop_adjust 3			;; Three fewer iterations this loop
	yloop_set_L3_prefetch YOTHER, 8	;; Prefetch 256 cache lines
;;	yr5_5cl_five_complex_djbfft_preload
b2b:	yr5_5cl_five_complex_djbfft rsi, 20*dist64, 4*dist64, rbp, 0, 5, L1PREFETCH_ALL, 64
	yloop	8, b2b, -5*20*dist64+64, rbp, YMM_SCD2 ;; 5 iterations!!  Next source pointer, next sin/cos data
	end_timer 4

	yloop	64, b1a, -64*64+dist64	;; Test loop counter, next source pointer
	yloop	4, b1a, -4*dist64, rdi, -256*YMM_SCD2+1280*YMM_SCD2+256*YMM_SCD2 ;; Test loop counter, restore source pointer, next sin/cos ptr

;; Do FFT levels 5,6
;; On input the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	1024	+1	+1	+1
;;	4	...
;;	...
;;	1020	...
;;	2048	...
;;	...
;; On output the 64-byte cache lines are swizzled to hold these data values:
;;	0	+512	+512	+512	256	+512	+512	+512
;;	4	...
;;	...
;;	508	...
;;	1	...
;;	...
;;	2048	...
;;	...

;; Do 6400 four_complex_fft operations
;;	distance between fft data elements is 256
;;	do 1600 macros each processing 32 data values

;; Do FFT levels 7,8
;; On input the 64-byte cache lines hold these data values:
;;	0	+512	+512	+512	256	+512	+512	+512
;;	4	...
;;	...
;;	508	...
;;	1	...
;;	...
;;	2048	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+512	+512	+512	64	+512	+512	+512
;;	4	...
;;	...
;;	60	...
;;	128	...
;;	...
;;	1	...
;;	...
;;	2048	...
;;	...

;; Do 6400 four_complex_fft operations
;;	distance between fft data elements is 64
;;	do 1600 macros each processing 32 data values

;; Do FFT levels 9,10
;; On input the 64-byte cache lines hold these data values:
;;	0	+512	+512	+512	64	+512	+512	+512
;;	4	...
;;	...
;;	60	...
;;	128	...
;;	...
;;	1	...
;;	...
;;	2048	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+512	+512	+512	16	+512	+512	+512
;;	4	...
;;	...
;;	12	...
;;	32	...
;;	...
;;	1	...
;;	...
;;	2048	...
;;	...

;; Do 6400 four_complex_fft operations
;;	distance between fft data elements is 16
;;	do 1600 macros each processing 32 data values

;; Do FFT levels 11,12
;; On input the 64-byte cache lines hold these data values:
;;	0	+512	+512	+512	16	+512	+512	+512
;;	4	...
;;	...
;;	12	...
;;	32	...
;;	...
;;	1	...
;;	...
;;	2048	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+512	+512	+512	4	+512	+512	+512
;;	8	...
;;	...
;;	1	...
;;	...
;;	2048	...
;;	...

;; Do 6400 four_complex_fft operations
;;	distance between fft data elements is 4
;;	do 1600 macros each processing 32 data values

;; Do FFT levels 13,14 as well as inverse FFT levels 13,14
;; On input and output the 64-byte cache lines hold these data values:
;;	0	+512	+512	+512	4	+512	+512	+512
;;	8	...
;;	...
;;	1	...
;;	...
;;	2048	...
;;	...

;; Do 6400 four_complex_fft operations
;;	distance between fft data elements is 1
;;	do 1600 macros each processing 32 data values

;; Do inverse FFT levels 11,12
;; On input the 64-byte cache lines hold these data values:
;;	0	+512	+512	+512	4	+512	+512	+512
;;	8	...
;;	...
;;	1	...
;;	...
;;	2048	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+512	+512	+512	16	+512	+512	+512
;;	4	...
;;	...
;;	12	...
;;	32	...
;;	...
;;	1	...
;;	...
;;	2048	...
;;	...

;; Do 6400 four_complex_unfft operations
;;	distance between fft data elements is 4
;;	do 1600 macros each processing 32 data values

;; Do inverse FFT levels 9,10
;; On input the 64-byte cache lines hold these data values:
;;	0	+512	+512	+512	16	+512	+512	+512
;;	4	...
;;	...
;;	12	...
;;	32	...
;;	...
;;	1	...
;;	...
;;	2048	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+512	+512	+512	64	+512	+512	+512
;;	4	...
;;	...
;;	60	...
;;	128	...
;;	...
;;	1	...
;;	...
;;	2048	...
;;	...

;; Do 6400 four_complex_unfft operations
;;	distance between fft data elements is 16
;;	do 1600 macros each processing 32 data values

;; Do inverse FFT levels 7,8
;; On input the 64-byte cache lines hold these data values:
;;	0	+512	+512	+512	64	+512	+512	+512
;;	4	...
;;	...
;;	60	...
;;	128	...
;;	...
;;	1	...
;;	...
;;	2048	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+512	+512	+512	256	+512	+512	+512
;;	4	...
;;	...
;;	508	...
;;	1	...
;;	...
;;	2048	...
;;	...

;; Do 6400 four_complex_unfft operations
;;	distance between fft data elements is 64
;;	do 1600 macros each processing 32 data values

;; Do inverse FFT levels 5,6
;; On input the 64-byte cache lines hold these data values:
;;	0	+512	+512	+512	256	+512	+512	+512
;;	4	...
;;	...
;;	508	...
;;	1	...
;;	...
;;	2048	...
;;	...
;; On output the 64-byte cache lines are swizzled to hold these data values:
;;	0	+1	+1	+1	1024	+1	+1	+1
;;	4	...
;;	...
;;	1020	...
;;	2048	...
;;	...

;; Do 6400 four_complex_unfft operations
;;	distance between fft data elements is 256
;;	do 1600 macros each processing 32 data values

;; Do this 256 cache lines (16KB) at a time to maximize accessing data
;; while it is in the L1 cache.

;; Execute the right middle step

	cmp	ffttype, 2
	jl	ypass2_25600_complex_1
	jg	ypass2_25600_complex_3

ypass2_25600_complex_2:
	yloop_init 1600			;; 25 iters of 256 cache lines

b3b0:	start_timer 5
	yloop_set_L3_prefetch YHALF	;; Prefetch 800 cache lines in this loop
	yr4_s4cl_four_complex_djbfft_preload
;; Loop below reworked to avoid 4KB distances
;;b3b:	yr4_s4cl_four_complex_djbfft rsi, 64, dist64, 2*dist64, rdi, YMM_SCD2, 64
;;	yloop	64, b3b, -64*64, rdi, -64*YMM_SCD2 ;; Loop 64 times, restore source pointer, restore sine/cosine pointer
b3b:	yr4_s4cl_four_complex_djbfft rsi, 4*64, dist64, 2*dist64, rdi, 4*YMM_SCD2, 16
	yloop	16, b3b, -16*4*64+64, rdi, -16*4*YMM_SCD2+YMM_SCD2 ;; Loop 16 times, next source pointer, next sine/cosine pointer
	yloop	4, b3b, -4*64, rdi, -4*YMM_SCD2 ;; Loop 4 times, restore source pointer, restore sine/cosine pointer
	loops_undo 64			;; Back up loop counter
	end_timer 5

	start_timer 6
	yloop_set_L3_prefetch YHALF	;; Prefetch 800 cache lines in this loop
	yr4_eb4cl_four_complex_djbfft_preload
b4b:	yr4_eb4cl_four_complex_djbfft rsi, 64, 16*64, 32*64, rdi, 4*YMM_SCD2, 16
	yloop	16, b4b, -16*64+dist64, rdi, -16*4*YMM_SCD2+YMM_SCD2 ;; Loop 16 times, next source pointer, next sine/cosine pointer
	yloop	4, b4b, -4*dist64, rdi, -4*YMM_SCD2 ;; Loop 4 times, restore source pointer, restore sine/cosine pointer
	loops_undo 4*16			;; Back up loop counter
	end_timer 6

	start_timer 7
	yloop_set_L3_prefetch YQUARTER	;; Prefetch 400 cache lines in this loop
;;	yr4_eb4cl_four_complex_djbfft_preload
b5b:	yr4_eb4cl_four_complex_djbfft rsi, 64, 4*64, 8*64, rdi, 16*YMM_SCD2, 4
	yloop	4, b5b, -4*64+16*64, rdi, -4*16*YMM_SCD2 ;; Loop 4 times, next source pointer, restore sine/cosine pointer
	yloop	4, b5b, -4*16*64+dist64, rdi, 4*YMM_SCD2 ;; Loop 4 times, next source pointer, next sine/cosine pointer
	yloop	4, b5b, -4*dist64, rdi, -4*4*YMM_SCD2 ;; Test loop counter, restore source pointer, restore sine/cosine pointer
	loops_undo 4*4*4		;; Back up loop counter
	end_timer 7

	start_timer 8
	yloop_set_L3_prefetch YQUARTER	;; Prefetch 400 cache lines in this loop
;;	yr4_eb4cl_four_complex_djbfft_preload
b6b:	yr4_eb4cl_four_complex_djbfft rsi, 4*64, 64, 2*64, rdi, 0, 16
	yloop	16, b6b, -16*4*64+dist64, rdi, 16*YMM_SCD2 ;; Loop 4 times, next source pointer, next sine/cosine pointer
	yloop	4, b6b, -4*dist64, rdi, -4*16*YMM_SCD2 ;; Test loop counter, restore source pointer, restore sine/cosine pointer
	loops_undo 4*16			;; Back up loop counter
	end_timer 8

	start_timer 9
	yloop_set_L3_prefetch YQUARTER	;; Prefetch 400 cache lines in this loop
	yr4_4cl_four_complex_with_square_preload
;; Loop below reworked to avoid 4KB distances
;;b7b:	yr4_4cl_four_complex_with_square rsi, 64, dist64, 2*dist64
;;	yloop	64, b7b, -64*64	;; Loop 64 times, restore source pointer
b7b:	yr4_4cl_four_complex_with_square rsi, 4*64, dist64, 2*dist64, 16
	yloop	16, b7b, -16*4*64+64	;; Loop 16 times, next source pointer
	yloop	4, b7b, -4*64	;; Loop 4 times, restore source pointer
	loops_undo 64			;; Back up loop counter
	end_timer 9

	start_timer 10
	yloop_set_L3_prefetch YQUARTER	;; Prefetch 400 cache lines in this loop
	yr4_eb4cl_four_complex_djbunfft_preload
c6b:	yr4_eb4cl_four_complex_djbunfft rsi, 4*64, 64, 2*64, rdi, 0, 16
	yloop	16, c6b, -16*4*64+dist64, rdi, 16*YMM_SCD2 ;; Loop 4 times, next source pointer, next sine/cosine pointer
	yloop	4, c6b, -4*dist64, rdi, -4*16*YMM_SCD2 ;; Test loop counter, restore source pointer, restore sine/cosine pointer
	loops_undo 4*16			;; Back up loop counter
	end_timer 10

	start_timer 11
	yloop_set_L3_prefetch YHALF	;; Prefetch 800 cache lines in this loop
;;	yr4_eb4cl_four_complex_djbunfft_preload
c5b:	yr4_eb4cl_four_complex_djbunfft rsi, 64, 4*64, 8*64, rdi, 16*YMM_SCD2, 4
	yloop	4, c5b, -4*64+16*64, rdi, -4*16*YMM_SCD2 ;; Loop 4 times, next source pointer, restore sine/cosine pointer
	yloop	4, c5b, -4*16*64+dist64, rdi, 4*YMM_SCD2 ;; Loop 4 times, next source pointer, next sine/cosine pointer
	yloop	4, c5b, -4*dist64, rdi, -4*4*YMM_SCD2 ;; Test loop counter, restore source pointer, restore sine/cosine pointer
	loops_undo 4*4*4		;; Back up loop counter
	end_timer 11

	start_timer 12
	yloop_set_L3_prefetch YHALF	;; Prefetch 800 cache lines in this loop
;;	yr4_eb4cl_four_complex_djbunfft_preload
c4b:	yr4_eb4cl_four_complex_djbunfft rsi, 64, 16*64, 32*64, rdi, 4*YMM_SCD2, 16
	yloop	16, c4b, -16*64+dist64, rdi, -16*4*YMM_SCD2+YMM_SCD2 ;; Loop 16 times, next source pointer, next sine/cosine pointer
	yloop	4, c4b, -4*dist64, rdi, -4*YMM_SCD2 ;; Loop 4 times, restore source pointer, restore sine/cosine pointer
	loops_undo 4*16			;; Back up loop counter
	end_timer 12

	start_timer 13
	yloop_set_L3_prefetch YHALF	;; Prefetch 800 cache lines in this loop
	yr4_s4cl_four_complex_djbunfft_preload
;; Loop below reworked to avoid 4KB distances
;;c3b:	yr4_s4cl_four_complex_djbunfft rsi, 64, dist64, 2*dist64, rdi, YMM_SCD2, 64
;;	yloop	64, c3b, -64*64, rdi, -64*YMM_SCD2 ;; Loop 64 times, restore source pointer, restore sine/cosine pointer
c3b:	yr4_s4cl_four_complex_djbunfft rsi, 4*64, dist64, 2*dist64, rdi, 4*YMM_SCD2, 16
	yloop	16, c3b, -16*4*64+64, rdi, -16*4*YMM_SCD2+YMM_SCD2 ;; Loop 16 times, next source pointer, next sine/cosine pointer
	yloop	4, c3b, -4*64+4*dist64, rdi, -4*YMM_SCD2 ;; Loop 4 times, next source pointer, restore sine/cosine pointer
	end_timer 13

	yloop	25, b3b0, -25*4*dist64	;; Loop 25 times, restore source pointer
	jmp	ypass2_25600_complex_unfft


ypass2_25600_complex_1:
	start_timer 6
	bump	rcx, (6400+99-2*256-3*1600-800-400)*64 ;; Move prefetch pointer along
	yloop_init 1600			;; 25 iters of 256 cache lines

b3a0:	yloop_set_L3_prefetch YSINGLE	;; Prefetch 1600 cache lines in this loop
	yr4_s4cl_four_complex_djbfft_preload
;; Loop below reworked to avoid 4KB distances
;;b3a:	yr4_s4cl_four_complex_djbfft rsi, 64, dist64, 2*dist64, rdi, YMM_SCD2, 64
;;	yloop	64, b3a, -64*64, rdi, -64*YMM_SCD2 ;; Loop 64 times, restore source pointer, restore sine/cosine pointer
b3a:	yr4_s4cl_four_complex_djbfft rsi, 4*64, dist64, 2*dist64, rdi, 4*YMM_SCD2, 16
	yloop	16, b3a, -16*4*64+64, rdi, -16*4*YMM_SCD2+YMM_SCD2 ;; Loop 16 times, next source pointer, next sine/cosine pointer
	yloop	4, b3a, -4*64, rdi, -4*YMM_SCD2 ;; Loop 4 times, restore source pointer, restore sine/cosine pointer
	loops_undo 64			;; Back up loop counter

	yloop_set_L3_prefetch YSINGLE	;; Prefetch 1600 cache lines in this loop
	yr4_eb4cl_four_complex_djbfft_preload
b4a:	yr4_eb4cl_four_complex_djbfft rsi, 64, 16*64, 32*64, rdi, 4*YMM_SCD2, 16
	yloop	16, b4a, -16*64+dist64, rdi, -16*4*YMM_SCD2+YMM_SCD2 ;; Loop 16 times, next source pointer, next sine/cosine pointer
	yloop	4, b4a, -4*dist64, rdi, -4*YMM_SCD2 ;; Loop 4 times, restore source pointer, restore sine/cosine pointer
	loops_undo 4*16			;; Back up loop counter

	yloop_set_L3_prefetch YSINGLE	;; Prefetch 1600 cache lines in this loop
;;	yr4_eb4cl_four_complex_djbfft_preload
b5a:	yr4_eb4cl_four_complex_djbfft rsi, 64, 4*64, 8*64, rdi, 16*YMM_SCD2, 4
	yloop	4, b5a, -4*64+16*64, rdi, -4*16*YMM_SCD2 ;; Loop 4 times, next source pointer, restore sine/cosine pointer
	yloop	4, b5a, -4*16*64+dist64, rdi, 4*YMM_SCD2 ;; Loop 4 times, next source pointer, next sine/cosine pointer
	yloop	4, b5a, -4*dist64, rdi, -4*4*YMM_SCD2 ;; Test loop counter, restore source pointer, restore sine/cosine pointer
	loops_undo 4*4*4		;; Back up loop counter

	yloop_set_L3_prefetch YHALF	;; Prefetch 800 cache lines in this loop
;;	yr4_eb4cl_four_complex_djbfft_preload
b6a:	yr4_eb4cl_four_complex_djbfft rsi, 4*64, 64, 2*64, rdi, 0, 16
	yloop	16, b6a, -16*4*64+dist64, rdi, 16*YMM_SCD2 ;; Loop 4 times, next source pointer, next sine/cosine pointer
	yloop	4, b6a, -4*dist64, rdi, -4*16*YMM_SCD2 ;; Test loop counter, restore source pointer, restore sine/cosine pointer
	loops_undo 4*16			;; Back up loop counter

	yloop_set_L3_prefetch YQUARTER	;; Prefetch 400 cache lines in this loop
	yr4_4cl_four_complex_fft_final_preload
;; Loop below reworked to avoid 4KB distances
;;b7a:	yr4_4cl_four_complex_fft_final rsi, 64, dist64, 2*dist64
;;	yloop	64, b7a, -64*64	;; Loop 64 times, restore source pointer
b7a:	yr4_4cl_four_complex_fft_final rsi, 4*64, dist64, 2*dist64
	yloop	16, b7a, -16*4*64+64	;; Loop 16 times, next source pointer
	yloop	4, b7a, -4*64+4*dist64	;; Loop 4 times, next source pointer

	yloop	25, b3a0		;; Loop 25 times
	end_timer 6
	jmp	ypass2_25600_complex_done


ypass2_25600_complex_3:
	start_timer 6
x64	mov	r8, DEST2ARG
	mov	rbp, DIST_TO_MULSRCARG
	yloop_init 1600			;; 25 iters of 256 cache lines

b3c0:	yloop_set_L3_prefetch YHALF	;; Prefetch 800 cache lines in this loop
	yr4_s4cl_four_complex_djbfft_preload
;; Loop below reworked to avoid 4KB distances
;;b3c:	yr4_s4cl_four_complex_djbfft rsi, 64, dist64, 2*dist64, rdi, YMM_SCD2, 64
;;	yloop	64, b3c, -64*64, rdi, -64*YMM_SCD2 ;; Loop 64 times, restore source pointer, restore sine/cosine pointer
b3c:	yr4_s4cl_four_complex_djbfft rsi, 4*64, dist64, 2*dist64, rdi, 4*YMM_SCD2, 16
	yloop	16, b3c, -16*4*64+64, rdi, -16*4*YMM_SCD2+YMM_SCD2 ;; Loop 16 times, next source pointer, next sine/cosine pointer
	yloop	4, b3c, -4*64, rdi, -4*YMM_SCD2 ;; Loop 4 times, restore source pointer, restore sine/cosine pointer
	loops_undo 64			;; Back up loop counter

	yloop_set_L3_prefetch YHALF	;; Prefetch 800 cache lines in this loop
	yr4_eb4cl_four_complex_djbfft_preload
b4c:	yr4_eb4cl_four_complex_djbfft rsi, 64, 16*64, 32*64, rdi, 4*YMM_SCD2, 16
	yloop	16, b4c, -16*64+dist64, rdi, -16*4*YMM_SCD2+YMM_SCD2 ;; Loop 16 times, next source pointer, next sine/cosine pointer
	yloop	4, b4c, -4*dist64, rdi, -4*YMM_SCD2 ;; Loop 4 times, restore source pointer, restore sine/cosine pointer
	loops_undo 4*16			;; Back up loop counter

	yloop_set_L3_prefetch YQUARTER	;; Prefetch 400 cache lines in this loop
;;	yr4_eb4cl_four_complex_djbfft_preload
b5c:	yr4_eb4cl_four_complex_djbfft rsi, 64, 4*64, 8*64, rdi, 16*YMM_SCD2, 4
	yloop	4, b5c, -4*64+16*64, rdi, -4*16*YMM_SCD2 ;; Loop 4 times, next source pointer, restore sine/cosine pointer
	yloop	4, b5c, -4*16*64+dist64, rdi, 4*YMM_SCD2 ;; Loop 4 times, next source pointer, next sine/cosine pointer
	yloop	4, b5c, -4*dist64, rdi, -4*4*YMM_SCD2 ;; Test loop counter, restore source pointer, restore sine/cosine pointer
	loops_undo 4*4*4		;; Back up loop counter

	yloop_set_L3_prefetch YQUARTER	;; Prefetch 400 cache lines in this loop
;;	yr4_eb4cl_four_complex_djbfft_preload
b6c:	yr4_eb4cl_four_complex_djbfft rsi, 4*64, 64, 2*64, rdi, 0, 16
	yloop	16, b6c, -16*4*64+dist64, rdi, 16*YMM_SCD2 ;; Loop 4 times, next source pointer, next sine/cosine pointer
	yloop	4, b6c, -4*dist64, rdi, -4*16*YMM_SCD2 ;; Test loop counter, restore source pointer, restore sine/cosine pointer
	loops_undo 4*16			;; Back up loop counter

	yloop_set_L3_prefetch YQUARTER	;; Prefetch 400 cache lines in this loop
	yr4_4cl_four_complex_with_mult_preload
;; Loop below reworked to avoid 4KB distances
;;b7c:	yr4_4cl_four_complex_with_mult rsi, 64, dist64, 2*dist64
;;	yloop	64, b7c, -64*64	;; Loop 64 times, restore source pointer
b7c:	yr4_4cl_four_complex_with_mult rsi, 4*64, dist64, 2*dist64
	yloop	16, b7c, -16*4*64+64	;; Loop 16 times, next source pointer
	yloop	4, b7c, -4*64		;; Loop 4 times, restore source pointer
x64	add	rsi, r8
	loops_undo 64			;; Back up loop counter

	yloop_set_L3_prefetch YQUARTER	;; Prefetch 400 cache lines in this loop
	yr4_eb4cl_four_complex_djbunfft_preload
c6c:	yr4_eb4cl_four_complex_djbunfft rsi, 4*64, 64, 2*64, rdi, 0, 16
	yloop	16, c6c, -16*4*64+dist64, rdi, 16*YMM_SCD2 ;; Loop 16 times, next source pointer, next sine/cosine pointer
	yloop	4, c6c, -4*dist64, rdi, -4*16*YMM_SCD2 ;; Test loop counter, restore source pointer, restore sine/cosine pointer
	loops_undo 4*16			;; Back up loop counter

	yloop_set_L3_prefetch YHALF	;; Prefetch 800 cache lines in this loop
;;	yr4_eb4cl_four_complex_djbunfft_preload
c5c:	yr4_eb4cl_four_complex_djbunfft rsi, 64, 4*64, 8*64, rdi, 16*YMM_SCD2, 4
	yloop	4, c5c, -4*64+16*64, rdi, -4*16*YMM_SCD2 ;; Loop 4 times, next source pointer, restore sine/cosine pointer
	yloop	4, c5c, -4*16*64+dist64, rdi, 4*YMM_SCD2 ;; Loop 4 times, next source pointer, next sine/cosine pointer
	yloop	4, c5c, -4*dist64, rdi, -4*4*YMM_SCD2 ;; Test loop counter, restore source pointer, restore sine/cosine pointer
	loops_undo 4*4*4		;; Back up loop counter

	yloop_set_L3_prefetch YHALF	;; Prefetch 800 cache lines in this loop
;;	yr4_eb4cl_four_complex_djbunfft_preload
c4c:	yr4_eb4cl_four_complex_djbunfft rsi, 64, 16*64, 32*64, rdi, 4*YMM_SCD2, 16
	yloop	16, c4c, -16*64+dist64, rdi, -16*4*YMM_SCD2+YMM_SCD2 ;; Loop 16 times, next source pointer, next sine/cosine pointer
	yloop	4, c4c, -4*dist64, rdi, -4*YMM_SCD2 ;; Loop 4 times, restore source pointer, restore sine/cosine pointer
	loops_undo 4*16			;; Back up loop counter

	yloop_set_L3_prefetch YHALF	;; Prefetch 800 cache lines in this loop
	yr4_s4cl_four_complex_djbunfft_preload
;; Loop below reworked to avoid 4KB distances
;;c3c:	yr4_s4cl_four_complex_djbunfft rsi, 64, dist64, 2*dist64, rdi, YMM_SCD2, 64
;;	yloop	64, c3c, -64*64, rdi, -64*YMM_SCD2 ;; Loop 64 times, restore source pointer, restore sine/cosine pointer
c3c:	yr4_s4cl_four_complex_djbunfft rsi, 4*64, dist64, 2*dist64, rdi, 4*YMM_SCD2, 16
	yloop	16, c3c, -16*4*64+64, rdi, -16*4*YMM_SCD2+YMM_SCD2 ;; Loop 16 times, next source pointer, next sine/cosine pointer
	yloop	4, c3c, -4*64+4*dist64, rdi, -4*YMM_SCD2 ;; Loop 4 times, next source pointer, restore sine/cosine pointer

x64	sub	rsi, r8
	yloop	25, b3c0, -25*4*dist64	;; Loop 25 times, restore source pointer
x64	add	rsi, r8
	end_timer 6
	jmp	ypass2_25600_complex_unfft


ypass2_25600_complex_4:
	start_timer 6
	mov	rdi, xsincos_complex	;; The sin/cos ptrs
	bump	rdi, 1280*YMM_SCD2+256*YMM_SCD2
	mov	rbp, DIST_TO_MULSRCARG
	yloop_init 1600			;; 25 iters of 256 cache lines

	yloop_set_L3_prefetch YHALF	;; Prefetch 800 cache lines in this loop
	yr4_4cl_four_complex_with_mulf_preload
;; Loop below reworked to avoid 4KB distances
;;b7d:	yr4_4cl_four_complex_with_mulf rsi, 64, dist64, 2*dist64
;;	yloop	64, b7d, -64*64	;; Loop 64 times, restore source pointer
b7d:	yr4_4cl_four_complex_with_mulf rsi, 4*64, dist64, 2*dist64
	yloop	16, b7d, -16*4*64+64	;; Loop 16 times, next source pointer
	yloop	4, b7d, -4*64		;; Loop 4 times, restore source pointer
	loops_undo 64			;; Back up loop counter

	yloop_set_L3_prefetch YHALF	;; Prefetch 800 cache lines in this loop
	yr4_eb4cl_four_complex_djbunfft_preload
c6d:	yr4_eb4cl_four_complex_djbunfft rsi, 4*64, 64, 2*64, rdi, 0, 16
	yloop	16, c6d, -16*4*64+dist64, rdi, 16*YMM_SCD2 ;; Loop 16 times, next source pointer, next sine/cosine pointer
	yloop	4, c6d, -4*dist64, rdi, -4*16*YMM_SCD2 ;; Test loop counter, restore source pointer, restore sine/cosine pointer
	loops_undo 4*16			;; Back up loop counter

	yloop_set_L3_prefetch YSINGLE	;; Prefetch 1600 cache lines in this loop
;;	yr4_eb4cl_four_complex_djbunfft_preload
c5d:	yr4_eb4cl_four_complex_djbunfft rsi, 64, 4*64, 8*64, rdi, 16*YMM_SCD2, 4
	yloop	4, c5d, -4*64+16*64, rdi, -4*16*YMM_SCD2 ;; Loop 4 times, next source pointer, restore sine/cosine pointer
	yloop	4, c5d, -4*16*64+dist64, rdi, 4*YMM_SCD2 ;; Loop 4 times, next source pointer, next sine/cosine pointer
	yloop	4, c5d, -4*dist64, rdi, -4*4*YMM_SCD2 ;; Test loop counter, restore source pointer, restore sine/cosine pointer
	loops_undo 4*4*4		;; Back up loop counter

	yloop_set_L3_prefetch YSINGLE	;; Prefetch 1600 cache lines in this loop
;;	yr4_eb4cl_four_complex_djbunfft_preload
c4d:	yr4_eb4cl_four_complex_djbunfft rsi, 64, 16*64, 32*64, rdi, 4*YMM_SCD2, 16
	yloop	16, c4d, -16*64+dist64, rdi, -16*4*YMM_SCD2+YMM_SCD2 ;; Loop 16 times, next source pointer, next sine/cosine pointer
	yloop	4, c4d, -4*dist64, rdi, -4*YMM_SCD2 ;; Loop 4 times, restore source pointer, restore sine/cosine pointer
	loops_undo 4*16			;; Back up loop counter

	yloop_set_L3_prefetch YSINGLE	;; Prefetch 1600 cache lines in this loop
	yr4_s4cl_four_complex_djbunfft_preload
;; Loop below reworked to avoid 4KB distances
;;c3d:	yr4_s4cl_four_complex_djbunfft rsi, 64, dist64, 2*dist64, rdi, YMM_SCD2, 64
;;	yloop	64, c3d, -64*64, rdi, -64*YMM_SCD2 ;; Loop 64 times, restore source pointer, restore sine/cosine pointer
c3d:	yr4_s4cl_four_complex_djbunfft rsi, 4*64, dist64, 2*dist64, rdi, 4*YMM_SCD2, 16
	yloop	16, c3d, -16*4*64+64, rdi, -16*4*YMM_SCD2+YMM_SCD2 ;; Loop 16 times, next source pointer, next sine/cosine pointer
	yloop	4, c3d, -4*64+4*dist64, rdi, -4*YMM_SCD2 ;; Loop 4 times, next source pointer, restore sine/cosine pointer

	yloop	25, b7d, -25*4*dist64	;; Loop 25 times, restore source pointer
	bump	rcx, (-2*800-3*1600+2*256+2*800+4*400+3*800)*64 ;; Move prefetch pointer to where squaring would be
	end_timer 6

;; Do inverse FFT levels 3,4
;; On input the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	1024	+1	+1	+1
;;	4	...
;;	...
;;	1020	...
;;	2048	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	5120	+1	+1	+1
;;	4	...
;;	...
;;	5116	...
;;	10240	...
;;	...

;; Do 5120 five_complex_unfft operations
;;	distance between fft data elements is 1024
;;	do 1280 macros each processing 40 data values

;; Do inverse FFT levels 1,2
;; On input the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	5120	+1	+1	+1
;;	4	...
;;	...
;;	5116	...
;;	10240	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+1	+1	+1	25600	+1	+1	+1
;;	4	...
;;	...
;;	25596

;; Do 5120 five_complex_unfft operations
;;	distance between fft data elements is 5120
;;	do 1280 macros each processing 40 data values

ypass2_25600_complex_unfft:
	mov	rdi, xsincos_complex	;; Load sin/cos pointers
	lea	rbp, [rdi+1280*YMM_SCD2]
	bump	rcx, (6400+99-2*256-2*800-4*400-3*800-2*256)*64 ;; Move prefetch pointer along
	yloop_init 2048			;; 1280 iterations

	yr5_5cl_five_complex_djbunfft_preload
c2a:	start_timer 14
	yloop_adjust 3			;; Three fewer iterations this loop
	yloop_set_L3_prefetch YOTHER, 8	;; Prefetch 256 cache lines
c2b:	yr5_5cl_five_complex_djbunfft rsi, 20*dist64, 4*dist64, rbp, 0, 5
	yloop	8, c2b, -5*20*dist64, rbp, YMM_SCD2 ;; 5 iterations!!  Restore source pointer, next sine/cosine pointer
	loops_undo 8			;; Restore loop counter
	end_timer 14

	start_timer 15
	yloop_adjust 3			;; Three fewer iterations this loop
	yloop_set_L3_prefetch YOTHER, 8	;; Prefetch 256 cache lines
;;	yr5_5cl_five_complex_djbunfft_preload
c1b:	yr5_5cl_five_complex_djbunfft rsi, 4*dist64, 20*dist64, rdi, 256*YMM_SCD2, 5, L1PREFETCH_ALL, 64
	yloop	8, c1b, -5*4*dist64+64, rdi, -5*256*YMM_SCD2+YMM_SCD2 ;; 5 iterations!!  Next source pointer, next sine/cosine pointer
	end_timer 15

	yloop	64, c2a, -64*64+dist64	;; Loop 64 times, next source pointer
	yloop	4, c2a			;; Loop 4 times

ypass2_25600_complex_done:
	ENDM

